added temperature pause

This commit is contained in:
Linrador 2026-06-26 22:06:05 +02:00
parent 5e6c841b26
commit 6b6528acca
9 changed files with 771 additions and 88 deletions

Binary file not shown.

View File

@ -13,11 +13,13 @@ import (
"runtime" "runtime"
"strconv" "strconv"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"time" "time"
gocpu "github.com/shirou/gopsutil/v3/cpu" gocpu "github.com/shirou/gopsutil/v3/cpu"
godisk "github.com/shirou/gopsutil/v3/disk" godisk "github.com/shirou/gopsutil/v3/disk"
gohost "github.com/shirou/gopsutil/v3/host"
) )
// Server-Startzeit für Perf/Uptime // Server-Startzeit für Perf/Uptime
@ -25,6 +27,13 @@ var serverStartedAt = time.Now()
var lastCPUUsageBits uint64 // atomic float64 bits var lastCPUUsageBits uint64 // atomic float64 bits
var cpuTemperatureCache = struct {
sync.Mutex
value float64
available bool
sampledAt time.Time
}{}
func setLastCPUUsage(v float64) { func setLastCPUUsage(v float64) {
atomic.StoreUint64(&lastCPUUsageBits, math.Float64bits(v)) atomic.StoreUint64(&lastCPUUsageBits, math.Float64bits(v))
} }
@ -33,6 +42,71 @@ func getLastCPUUsage() float64 {
return math.Float64frombits(atomic.LoadUint64(&lastCPUUsageBits)) return math.Float64frombits(atomic.LoadUint64(&lastCPUUsageBits))
} }
func getMaxCPUTemperatureC() (float64, bool) {
now := time.Now()
cpuTemperatureCache.Lock()
if now.Sub(cpuTemperatureCache.sampledAt) < 15*time.Second {
value := cpuTemperatureCache.value
available := cpuTemperatureCache.available
cpuTemperatureCache.Unlock()
return value, available
}
cpuTemperatureCache.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
temps, err := gohost.SensorsTemperaturesWithContext(ctx)
if err != nil || len(temps) == 0 {
cpuTemperatureCache.Lock()
cpuTemperatureCache.value = 0
cpuTemperatureCache.available = false
cpuTemperatureCache.sampledAt = now
cpuTemperatureCache.Unlock()
return 0, false
}
maxCPU := 0.0
maxAny := 0.0
for _, temp := range temps {
v := temp.Temperature
if math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 || v > 150 {
continue
}
if v > maxAny {
maxAny = v
}
key := strings.ToLower(strings.TrimSpace(temp.SensorKey))
if strings.Contains(key, "cpu") ||
strings.Contains(key, "core") ||
strings.Contains(key, "package") ||
strings.Contains(key, "tdie") ||
strings.Contains(key, "tctl") ||
strings.Contains(key, "k10temp") {
if v > maxCPU {
maxCPU = v
}
}
}
value := maxCPU
if value <= 0 {
value = maxAny
}
available := value > 0
cpuTemperatureCache.Lock()
cpuTemperatureCache.value = value
cpuTemperatureCache.available = available
cpuTemperatureCache.sampledAt = now
cpuTemperatureCache.Unlock()
return value, available
}
func startCPUUsageSampler(ctx context.Context) { func startCPUUsageSampler(ctx context.Context) {
_, _ = gocpu.Percent(200*time.Millisecond, false) _, _ = gocpu.Percent(200*time.Millisecond, false)
@ -194,6 +268,17 @@ func buildPerfSnapshot() map[string]any {
} }
return v return v
}(), }(),
"cpuTemperatureC": func() float64 {
v, ok := getMaxCPUTemperatureC()
if !ok {
return 0
}
return math.Round(v*10) / 10
}(),
"cpuTemperatureAvailable": func() bool {
_, ok := getMaxCPUTemperatureC()
return ok
}(),
"diskPath": diskPath, "diskPath": diskPath,
"diskFreeBytes": diskFreeBytes, "diskFreeBytes": diskFreeBytes,
"diskTotalBytes": diskTotalBytes, "diskTotalBytes": diskTotalBytes,

View File

@ -56,15 +56,18 @@ type RecorderSettings struct {
GenerateAssetsAnalyze bool `json:"generateAssetsAnalyze"` GenerateAssetsAnalyze bool `json:"generateAssetsAnalyze"`
EnrichPostworkEnabled bool `json:"enrichPostworkEnabled"` EnrichPostworkEnabled bool `json:"enrichPostworkEnabled"`
TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"` TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"`
TrainingDetectorEpochs int `json:"trainingDetectorEpochs"` TrainingDetectorEpochs int `json:"trainingDetectorEpochs"`
TrainingPerformanceMode string `json:"trainingPerformanceMode"` TrainingPerformanceMode string `json:"trainingPerformanceMode"`
TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"` TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"`
TrainingCPUThreads int `json:"trainingCpuThreads"` TrainingCPUThreads int `json:"trainingCpuThreads"`
TrainingWorkers int `json:"trainingWorkers"` TrainingWorkers int `json:"trainingWorkers"`
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"` TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
TrainingLowPriority bool `json:"trainingLowPriority"` TrainingLowPriority bool `json:"trainingLowPriority"`
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"` TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"`
TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"`
TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"`
// EncryptedCookies contains base64(nonce+ciphertext) of a JSON cookie map. // EncryptedCookies contains base64(nonce+ciphertext) of a JSON cookie map.
EncryptedCookies string `json:"encryptedCookies"` EncryptedCookies string `json:"encryptedCookies"`
@ -112,15 +115,18 @@ var (
GenerateAssetsAnalyze: false, GenerateAssetsAnalyze: false,
EnrichPostworkEnabled: true, EnrichPostworkEnabled: true,
TrainingRecognitionEnabled: true, TrainingRecognitionEnabled: true,
TrainingDetectorEpochs: 60, TrainingDetectorEpochs: 60,
TrainingPerformanceMode: trainingPerformanceAuto, TrainingPerformanceMode: trainingPerformanceAuto,
TrainingPowerSaveMode: false, TrainingPowerSaveMode: false,
TrainingCPUThreads: 0, TrainingCPUThreads: 0,
TrainingWorkers: 2, TrainingWorkers: 2,
TrainingYoloBatchSize: 0, TrainingYoloBatchSize: 0,
TrainingLowPriority: false, TrainingLowPriority: false,
TrainingVideoMAEEnabled: true, TrainingVideoMAEEnabled: true,
TrainingAutoPauseEnabled: false,
TrainingAutoPauseCPUPercent: 95,
TrainingAutoPauseTemperatureC: 85,
EncryptedCookies: "", EncryptedCookies: "",
} }
@ -189,6 +195,20 @@ func normalizeTrainingSettings(s *RecorderSettings) {
if s.TrainingYoloBatchSize > 64 { if s.TrainingYoloBatchSize > 64 {
s.TrainingYoloBatchSize = 64 s.TrainingYoloBatchSize = 64
} }
if s.TrainingAutoPauseCPUPercent < 1 {
s.TrainingAutoPauseCPUPercent = 95
}
if s.TrainingAutoPauseCPUPercent > 100 {
s.TrainingAutoPauseCPUPercent = 100
}
if s.TrainingAutoPauseTemperatureC < 1 {
s.TrainingAutoPauseTemperatureC = 85
}
if s.TrainingAutoPauseTemperatureC > 120 {
s.TrainingAutoPauseTemperatureC = 120
}
} }
func settingsFilePath() string { func settingsFilePath() string {
@ -375,6 +395,9 @@ type RecorderSettingsPublic struct {
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"` TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
TrainingLowPriority bool `json:"trainingLowPriority"` TrainingLowPriority bool `json:"trainingLowPriority"`
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"` TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"`
TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"`
TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"`
TrainingCPUCoreCount int `json:"trainingCpuCoreCount"` TrainingCPUCoreCount int `json:"trainingCpuCoreCount"`
TrainingEffectiveMode string `json:"trainingEffectiveMode"` TrainingEffectiveMode string `json:"trainingEffectiveMode"`
TrainingEffectiveCPUThreads int `json:"trainingEffectiveCpuThreads"` TrainingEffectiveCPUThreads int `json:"trainingEffectiveCpuThreads"`
@ -436,6 +459,9 @@ func toPublicSettings(s RecorderSettings) RecorderSettingsPublic {
TrainingYoloBatchSize: s.TrainingYoloBatchSize, TrainingYoloBatchSize: s.TrainingYoloBatchSize,
TrainingLowPriority: s.TrainingLowPriority, TrainingLowPriority: s.TrainingLowPriority,
TrainingVideoMAEEnabled: s.TrainingVideoMAEEnabled, TrainingVideoMAEEnabled: s.TrainingVideoMAEEnabled,
TrainingAutoPauseEnabled: s.TrainingAutoPauseEnabled,
TrainingAutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent,
TrainingAutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC,
TrainingCPUCoreCount: trainingOpts.CPUCoreCount, TrainingCPUCoreCount: trainingOpts.CPUCoreCount,
TrainingEffectiveMode: trainingOpts.PerformanceMode, TrainingEffectiveMode: trainingOpts.PerformanceMode,
TrainingEffectiveCPUThreads: trainingOpts.CPUThreads, TrainingEffectiveCPUThreads: trainingOpts.CPUThreads,
@ -631,6 +657,9 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
next.TrainingYoloBatchSize = in.TrainingYoloBatchSize next.TrainingYoloBatchSize = in.TrainingYoloBatchSize
next.TrainingLowPriority = in.TrainingLowPriority next.TrainingLowPriority = in.TrainingLowPriority
next.TrainingVideoMAEEnabled = in.TrainingVideoMAEEnabled next.TrainingVideoMAEEnabled = in.TrainingVideoMAEEnabled
next.TrainingAutoPauseEnabled = in.TrainingAutoPauseEnabled
next.TrainingAutoPauseCPUPercent = in.TrainingAutoPauseCPUPercent
next.TrainingAutoPauseTemperatureC = in.TrainingAutoPauseTemperatureC
dbChanged := dbChanged :=
normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) || normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) ||

View File

@ -24,6 +24,8 @@ import (
"strings" "strings"
"sync" "sync"
"time" "time"
goprocess "github.com/shirou/gopsutil/v3/process"
) )
const trainingUncertainCandidateCount = 10 const trainingUncertainCandidateCount = 10
@ -262,15 +264,19 @@ type TrainingPosePrediction struct {
} }
type TrainingJobStatus struct { type TrainingJobStatus struct {
Running bool `json:"running"` Running bool `json:"running"`
Progress int `json:"progress"` Progress int `json:"progress"`
Step string `json:"step"` Step string `json:"step"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
StartedAt string `json:"startedAt,omitempty"` StartedAt string `json:"startedAt,omitempty"`
FinishedAt string `json:"finishedAt,omitempty"` FinishedAt string `json:"finishedAt,omitempty"`
DurationMs int64 `json:"durationMs,omitempty"` DurationMs int64 `json:"durationMs,omitempty"`
PreviewURL string `json:"previewUrl,omitempty"` PreviewURL string `json:"previewUrl,omitempty"`
Paused bool `json:"paused,omitempty"`
PauseReason string `json:"pauseReason,omitempty"`
CPUPercent float64 `json:"cpuPercent,omitempty"`
TemperatureC float64 `json:"temperatureC,omitempty"`
Stage string `json:"stage,omitempty"` Stage string `json:"stage,omitempty"`
Epoch int `json:"epoch,omitempty"` Epoch int `json:"epoch,omitempty"`
@ -1181,6 +1187,8 @@ func trainingRunCommandStreaming(
return "", err return "", err
} }
applyTrainingLowPriorityAfterStart(cmd, opts.LowPriority) applyTrainingLowPriorityAfterStart(cmd, opts.LowPriority)
monitorCtx, stopResourcePauseMonitor := context.WithCancel(ctx)
resumeResourcePause := startTrainingResourcePauseMonitor(monitorCtx, cmd, opts)
var outMu sync.Mutex var outMu sync.Mutex
var lines []string var lines []string
@ -1231,9 +1239,12 @@ func trainingRunCommandStreaming(
select { select {
case err = <-waitCh: case err = <-waitCh:
case <-ctx.Done(): case <-ctx.Done():
resumeResourcePause()
terminateTrainingCommand(cmd) terminateTrainingCommand(cmd)
err = <-waitCh err = <-waitCh
} }
stopResourcePauseMonitor()
resumeResourcePause()
wg.Wait() wg.Wait()
outMu.Lock() outMu.Lock()
@ -1274,14 +1285,17 @@ const (
) )
type trainingRuntimeOptions struct { type trainingRuntimeOptions struct {
PerformanceMode string PerformanceMode string
PowerSaveMode bool PowerSaveMode bool
CPUCoreCount int CPUCoreCount int
CPUThreads int CPUThreads int
Workers int Workers int
YoloBatchSize int YoloBatchSize int
LowPriority bool LowPriority bool
VideoMAEEnabled bool VideoMAEEnabled bool
AutoPauseEnabled bool
AutoPauseCPUPercent int
AutoPauseTemperatureC int
} }
func normalizeTrainingPerformanceMode(raw string) string { func normalizeTrainingPerformanceMode(raw string) string {
@ -1381,14 +1395,17 @@ func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRunt
presetThreads, presetWorkers, presetBatch, presetLowPriority := trainingPresetValues(mode, cpuCores) presetThreads, presetWorkers, presetBatch, presetLowPriority := trainingPresetValues(mode, cpuCores)
opts := trainingRuntimeOptions{ opts := trainingRuntimeOptions{
PerformanceMode: mode, PerformanceMode: mode,
PowerSaveMode: mode == trainingPerformanceEco, PowerSaveMode: mode == trainingPerformanceEco,
CPUCoreCount: cpuCores, CPUCoreCount: cpuCores,
CPUThreads: presetThreads, CPUThreads: presetThreads,
Workers: presetWorkers, Workers: presetWorkers,
YoloBatchSize: presetBatch, YoloBatchSize: presetBatch,
LowPriority: presetLowPriority, LowPriority: presetLowPriority,
VideoMAEEnabled: s.TrainingVideoMAEEnabled, VideoMAEEnabled: s.TrainingVideoMAEEnabled,
AutoPauseEnabled: s.TrainingAutoPauseEnabled,
AutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent,
AutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC,
} }
if mode == trainingPerformanceCustom { if mode == trainingPerformanceCustom {
@ -1448,6 +1465,331 @@ func trainingCommandEnv(opts trainingRuntimeOptions) []string {
return env return env
} }
type trainingResourceSnapshot struct {
CPUPercent float64
TemperatureC float64
TemperatureAvailable bool
}
func trainingRoundMetric(v float64) float64 {
if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 {
return 0
}
return math.Round(v*10) / 10
}
func trainingReadResourceSnapshot() trainingResourceSnapshot {
temp, tempOK := getMaxCPUTemperatureC()
return trainingResourceSnapshot{
CPUPercent: trainingRoundMetric(getLastCPUUsage()),
TemperatureC: trainingRoundMetric(temp),
TemperatureAvailable: tempOK,
}
}
func trainingPauseReasonForResources(opts trainingRuntimeOptions, snap trainingResourceSnapshot) (string, bool) {
if !opts.AutoPauseEnabled {
return "", false
}
reasons := []string{}
if opts.AutoPauseCPUPercent > 0 && snap.CPUPercent >= float64(opts.AutoPauseCPUPercent) {
reasons = append(
reasons,
fmt.Sprintf("CPU %.0f%% >= %d%%", snap.CPUPercent, opts.AutoPauseCPUPercent),
)
}
if opts.AutoPauseTemperatureC > 0 &&
snap.TemperatureAvailable &&
snap.TemperatureC >= float64(opts.AutoPauseTemperatureC) {
reasons = append(
reasons,
fmt.Sprintf("Temperatur %.0f°C >= %d°C", snap.TemperatureC, opts.AutoPauseTemperatureC),
)
}
if len(reasons) == 0 {
return "", false
}
return strings.Join(reasons, ", "), true
}
func trainingResourcesRecovered(opts trainingRuntimeOptions, snap trainingResourceSnapshot) bool {
if !opts.AutoPauseEnabled {
return true
}
if opts.AutoPauseCPUPercent > 0 {
resumeCPU := float64(opts.AutoPauseCPUPercent - 15)
if resumeCPU < 50 {
resumeCPU = 50
}
if snap.CPUPercent > resumeCPU {
return false
}
}
if opts.AutoPauseTemperatureC > 0 && snap.TemperatureAvailable {
resumeTemp := float64(opts.AutoPauseTemperatureC - 5)
if resumeTemp < 40 {
resumeTemp = 40
}
if snap.TemperatureC > resumeTemp {
return false
}
}
return true
}
func trainingProcessTree(pid int32) []*goprocess.Process {
if pid <= 0 {
return nil
}
root, err := goprocess.NewProcess(pid)
if err != nil || root == nil {
return nil
}
seen := map[int32]bool{}
var out []*goprocess.Process
var walk func(*goprocess.Process)
walk = func(p *goprocess.Process) {
if p == nil || seen[p.Pid] {
return
}
seen[p.Pid] = true
out = append(out, p)
children, err := p.Children()
if err != nil {
return
}
for _, child := range children {
walk(child)
}
}
walk(root)
return out
}
func trainingSuspendProcessTree(cmd *exec.Cmd) error {
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
return nil
}
procs := trainingProcessTree(int32(cmd.Process.Pid))
if len(procs) == 0 {
return nil
}
var firstErr error
success := 0
for i := len(procs) - 1; i >= 0; i-- {
if err := procs[i].Suspend(); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
success++
}
if success > 0 {
return nil
}
return firstErr
}
func trainingResumeProcessTree(cmd *exec.Cmd) error {
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
return nil
}
procs := trainingProcessTree(int32(cmd.Process.Pid))
if len(procs) == 0 {
return nil
}
var firstErr error
success := 0
for _, proc := range procs {
if err := proc.Resume(); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
success++
}
if success > 0 {
return nil
}
return firstErr
}
func startTrainingResourcePauseMonitor(ctx context.Context, cmd *exec.Cmd, opts trainingRuntimeOptions) func() {
if !opts.AutoPauseEnabled || cmd == nil || cmd.Process == nil {
return func() {}
}
const (
checkInterval = 5 * time.Second
requiredHighSamples = 2
requiredCoolSamples = 2
minPauseDuration = 45 * time.Second
)
var pauseMu sync.Mutex
paused := false
previousStep := ""
highSamples := 0
coolSamples := 0
pauseStartedAt := time.Time{}
loggedSuspendError := false
resumeNow := func(reason string) {
pauseMu.Lock()
if !paused {
pauseMu.Unlock()
return
}
err := trainingResumeProcessTree(cmd)
paused = false
restoreStep := strings.TrimSpace(previousStep)
previousStep = ""
pauseMu.Unlock()
if err != nil {
appLogln("⚠️ Training konnte nach Ressourcenpause nicht sauber fortgesetzt werden:", err)
}
if strings.TrimSpace(reason) != "" {
appLogln(reason)
}
trainingSetJobStatus(func(s *TrainingJobStatus) {
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
s.Message = ""
if restoreStep != "" && s.Running {
s.Step = restoreStep
}
})
}
go func() {
ticker := time.NewTicker(checkInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
resumeNow("Training-Ressourcenpause wurde beendet.")
return
case <-ticker.C:
}
snap := trainingReadResourceSnapshot()
pauseMu.Lock()
isPaused := paused
pauseAge := time.Since(pauseStartedAt)
pauseMu.Unlock()
if !isPaused {
reason, shouldPause := trainingPauseReasonForResources(opts, snap)
if !shouldPause {
highSamples = 0
continue
}
highSamples++
if highSamples < requiredHighSamples {
continue
}
pauseMu.Lock()
if paused {
pauseMu.Unlock()
continue
}
currentStatus := trainingGetJobStatus()
previousStep = strings.TrimSpace(currentStatus.Step)
if previousStep == "" {
previousStep = "Training läuft..."
}
if err := trainingSuspendProcessTree(cmd); err != nil {
pauseMu.Unlock()
highSamples = 0
if !loggedSuspendError {
loggedSuspendError = true
appLogln("⚠️ Training-Ressourcenpause konnte Prozess nicht pausieren:", err)
}
continue
}
paused = true
pauseStartedAt = time.Now()
coolSamples = 0
pauseMu.Unlock()
appLogln("⏸ Training pausiert wegen Ressourcenlimit:", reason)
trainingSetJobStatus(func(s *TrainingJobStatus) {
s.Paused = true
s.PauseReason = reason
s.CPUPercent = snap.CPUPercent
if snap.TemperatureAvailable {
s.TemperatureC = snap.TemperatureC
} else {
s.TemperatureC = 0
}
s.Step = "Training pausiert: " + reason
s.Message = "Training wird automatisch fortgesetzt, sobald CPU/Temperatur wieder im grünen Bereich sind."
})
continue
}
trainingSetJobStatus(func(s *TrainingJobStatus) {
if !s.Running || !s.Paused {
return
}
s.CPUPercent = snap.CPUPercent
if snap.TemperatureAvailable {
s.TemperatureC = snap.TemperatureC
}
})
if pauseAge < minPauseDuration || !trainingResourcesRecovered(opts, snap) {
coolSamples = 0
continue
}
coolSamples++
if coolSamples < requiredCoolSamples {
continue
}
highSamples = 0
coolSamples = 0
resumeNow("▶ Training nach Ressourcenpause fortgesetzt.")
}
}()
return func() {
resumeNow("")
}
}
var trainingJob = struct { var trainingJob = struct {
mu sync.Mutex mu sync.Mutex
status TrainingJobStatus status TrainingJobStatus
@ -1526,6 +1868,10 @@ func trainingFinishCancelled(root string) {
s.FinishedAt = finishedAt.Format(time.RFC3339) s.FinishedAt = finishedAt.Format(time.RFC3339)
s.DurationMs = durationMs s.DurationMs = durationMs
s.PreviewURL = "" s.PreviewURL = ""
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
if cleanupErr != nil { if cleanupErr != nil {
s.Message = "Training wurde abgebrochen, aber temporäre Trainingsausgaben konnten nicht vollständig gelöscht werden." s.Message = "Training wurde abgebrochen, aber temporäre Trainingsausgaben konnten nicht vollständig gelöscht werden."
@ -3641,6 +3987,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
s.FinishedAt = finishedAt.Format(time.RFC3339) s.FinishedAt = finishedAt.Format(time.RFC3339)
s.DurationMs = durationMs s.DurationMs = durationMs
s.PreviewURL = "" s.PreviewURL = ""
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
}) })
trainingClearJobCancel() trainingClearJobCancel()
return return
@ -3650,7 +4000,7 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
appLogln("ML-Python für Training:", python) appLogln("ML-Python für Training:", python)
runtimeOpts := trainingRuntimeOptionsFromSettings() runtimeOpts := trainingRuntimeOptionsFromSettings()
appLogf( appLogf(
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v targets=%s", "Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v autoPause=%v cpuLimit=%d tempLimit=%d targets=%s",
runtimeOpts.PerformanceMode, runtimeOpts.PerformanceMode,
runtimeOpts.CPUCoreCount, runtimeOpts.CPUCoreCount,
runtimeOpts.PowerSaveMode, runtimeOpts.PowerSaveMode,
@ -3659,6 +4009,9 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
runtimeOpts.YoloBatchSize, runtimeOpts.YoloBatchSize,
runtimeOpts.LowPriority, runtimeOpts.LowPriority,
runtimeOpts.VideoMAEEnabled, runtimeOpts.VideoMAEEnabled,
runtimeOpts.AutoPauseEnabled,
runtimeOpts.AutoPauseCPUPercent,
runtimeOpts.AutoPauseTemperatureC,
strings.Join(targets.list(), ","), strings.Join(targets.list(), ","),
) )
@ -4209,6 +4562,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
s.FinishedAt = finishedAt.Format(time.RFC3339) s.FinishedAt = finishedAt.Format(time.RFC3339)
s.DurationMs = durationMs s.DurationMs = durationMs
s.PreviewURL = "" s.PreviewURL = ""
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
}) })
trainingClearJobCancel() trainingClearJobCancel()
@ -4765,24 +5122,27 @@ func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo {
} }
type TrainingHistoryEntry struct { type TrainingHistoryEntry struct {
TrainedAt string `json:"trainedAt,omitempty"` TrainedAt string `json:"trainedAt,omitempty"`
TrainedAtMs int64 `json:"trainedAtMs,omitempty"` TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
Target string `json:"target,omitempty"` Target string `json:"target,omitempty"`
Status string `json:"status,omitempty"` Status string `json:"status,omitempty"`
DurationMs int64 `json:"durationMs,omitempty"` DurationMs int64 `json:"durationMs,omitempty"`
Epochs int `json:"epochs,omitempty"` Epochs int `json:"epochs,omitempty"`
TrainSamples int `json:"trainSamples,omitempty"` TrainSamples int `json:"trainSamples,omitempty"`
ValSamples int `json:"valSamples,omitempty"` ValSamples int `json:"valSamples,omitempty"`
Imgsz int `json:"imgsz,omitempty"` Imgsz int `json:"imgsz,omitempty"`
Device string `json:"device,omitempty"` Device string `json:"device,omitempty"`
MAP50 float64 `json:"map50,omitempty"` MAP50 float64 `json:"map50,omitempty"`
MAP5095 float64 `json:"map5095,omitempty"` MAP5095 float64 `json:"map5095,omitempty"`
PerformanceMode string `json:"performanceMode,omitempty"` PerformanceMode string `json:"performanceMode,omitempty"`
CPUCoreCount int `json:"cpuCoreCount,omitempty"` CPUCoreCount int `json:"cpuCoreCount,omitempty"`
CPUThreads int `json:"cpuThreads,omitempty"` CPUThreads int `json:"cpuThreads,omitempty"`
Workers int `json:"workers,omitempty"` Workers int `json:"workers,omitempty"`
YoloBatchSize int `json:"yoloBatchSize,omitempty"` YoloBatchSize int `json:"yoloBatchSize,omitempty"`
LowPriority bool `json:"lowPriority,omitempty"` LowPriority bool `json:"lowPriority,omitempty"`
AutoPauseEnabled bool `json:"autoPauseEnabled,omitempty"`
AutoPauseCPUPercent int `json:"autoPauseCpuPercent,omitempty"`
AutoPauseTemperatureC int `json:"autoPauseTemperatureC,omitempty"`
} }
type TrainingHistoryResponse struct { type TrainingHistoryResponse struct {
@ -4813,24 +5173,27 @@ func trainingAppendTargetHistory(root string, target string, status string, dura
now := time.Now().UTC() now := time.Now().UTC()
entry := TrainingHistoryEntry{ entry := TrainingHistoryEntry{
TrainedAt: now.Format(time.RFC3339), TrainedAt: now.Format(time.RFC3339),
TrainedAtMs: now.UnixMilli(), TrainedAtMs: now.UnixMilli(),
Target: kind, Target: kind,
Status: strings.TrimSpace(status), Status: strings.TrimSpace(status),
DurationMs: durationMs, DurationMs: durationMs,
Epochs: fallback.Epochs, Epochs: fallback.Epochs,
TrainSamples: fallback.TrainSamples, TrainSamples: fallback.TrainSamples,
ValSamples: fallback.ValSamples, ValSamples: fallback.ValSamples,
Imgsz: fallback.Imgsz, Imgsz: fallback.Imgsz,
Device: strings.TrimSpace(fallback.Device), Device: strings.TrimSpace(fallback.Device),
MAP50: fallback.MAP50, MAP50: fallback.MAP50,
MAP5095: fallback.MAP5095, MAP5095: fallback.MAP5095,
PerformanceMode: runtimeOpts.PerformanceMode, PerformanceMode: runtimeOpts.PerformanceMode,
CPUCoreCount: runtimeOpts.CPUCoreCount, CPUCoreCount: runtimeOpts.CPUCoreCount,
CPUThreads: runtimeOpts.CPUThreads, CPUThreads: runtimeOpts.CPUThreads,
Workers: runtimeOpts.Workers, Workers: runtimeOpts.Workers,
YoloBatchSize: runtimeOpts.YoloBatchSize, YoloBatchSize: runtimeOpts.YoloBatchSize,
LowPriority: runtimeOpts.LowPriority, LowPriority: runtimeOpts.LowPriority,
AutoPauseEnabled: runtimeOpts.AutoPauseEnabled,
AutoPauseCPUPercent: runtimeOpts.AutoPauseCPUPercent,
AutoPauseTemperatureC: runtimeOpts.AutoPauseTemperatureC,
} }
if info != nil { if info != nil {

View File

@ -1230,6 +1230,20 @@ export default function App() {
} }
}, [authed]) }, [authed])
useEffect(() => {
try {
const appWindow = window as any
appWindow.__nsfwappTrainingRunning = trainingTabRunning
window.dispatchEvent(
new CustomEvent('app:training-running', {
detail: { running: trainingTabRunning },
})
)
} catch {
// ignore
}
}, [trainingTabRunning])
const loadPendingAutoStarts = useCallback(async (opts?: { force?: boolean }) => { const loadPendingAutoStarts = useCallback(async (opts?: { force?: boolean }) => {
const force = Boolean(opts?.force) const force = Boolean(opts?.force)
const now = Date.now() const now = Date.now()

View File

@ -76,6 +76,7 @@ type Props = {
onAddToDownloads?: ActionFn onAddToDownloads?: ActionFn
onSplit?: ActionFn onSplit?: ActionFn
onAddToTraining?: ActionFn onAddToTraining?: ActionFn
trainingImportDisabled?: boolean
order?: ActionKey[] order?: ActionKey[]
@ -127,6 +128,7 @@ export default function RecordJobActions({
onAddToDownloads, onAddToDownloads,
onSplit, onSplit,
onAddToTraining, onAddToTraining,
trainingImportDisabled = false,
order, order,
className, className,
}: Props) { }: Props) {
@ -205,6 +207,10 @@ export default function RecordJobActions({
const [addState, setAddState] = useState<'idle' | 'busy' | 'ok'>('idle') const [addState, setAddState] = useState<'idle' | 'busy' | 'ok'>('idle')
const addTimerRef = useRef<number | null>(null) const addTimerRef = useRef<number | null>(null)
const [appTrainingRunning, setAppTrainingRunning] = useState(() => {
if (typeof window === 'undefined') return false
return Boolean((window as any).__nsfwappTrainingRunning)
})
useEffect(() => { useEffect(() => {
return () => { return () => {
@ -212,6 +218,19 @@ export default function RecordJobActions({
} }
}, []) }, [])
useEffect(() => {
const onTrainingRunning = (event: Event) => {
const detail = (event as CustomEvent<{ running?: boolean }>).detail
setAppTrainingRunning(Boolean(detail?.running))
}
window.addEventListener('app:training-running', onTrainingRunning as EventListener)
return () => {
window.removeEventListener('app:training-running', onTrainingRunning as EventListener)
}
}, [])
const flashAddOk = () => { const flashAddOk = () => {
setAddState('ok') setAddState('ok')
if (addTimerRef.current) window.clearTimeout(addTimerRef.current) if (addTimerRef.current) window.clearTimeout(addTimerRef.current)
@ -249,8 +268,11 @@ export default function RecordJobActions({
} }
} }
const trainingImportBlocked = trainingImportDisabled || appTrainingRunning
const doTrainingImport = async (): Promise<boolean> => { const doTrainingImport = async (): Promise<boolean> => {
if (busy) return false if (busy) return false
if (trainingImportBlocked) return false
const output = String(job.output || '').trim() const output = String(job.output || '').trim()
if (!output) return false if (!output) return false
@ -364,7 +386,7 @@ export default function RecordJobActions({
className={btnBase} className={btnBase}
title="Video ins Training übernehmen" title="Video ins Training übernehmen"
aria-label="Video ins Training übernehmen" aria-label="Video ins Training übernehmen"
disabled={busy || (!onAddToTraining && !job.output)} disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)}
onClick={async (e) => { onClick={async (e) => {
e.preventDefault() e.preventDefault()
e.stopPropagation() e.stopPropagation()
@ -774,7 +796,7 @@ export default function RecordJobActions({
key="training" key="training"
type="button" type="button"
className="flex w-full items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-gray-100/70 dark:hover:bg-white/5 disabled:opacity-50" className="flex w-full items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-gray-100/70 dark:hover:bg-white/5 disabled:opacity-50"
disabled={busy || (!onAddToTraining && !job.output)} disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)}
onClick={async (e) => { onClick={async (e) => {
e.preventDefault() e.preventDefault()
e.stopPropagation() e.stopPropagation()

View File

@ -55,6 +55,9 @@ type Settings = {
trainingYoloBatchSize?: number trainingYoloBatchSize?: number
trainingLowPriority?: boolean trainingLowPriority?: boolean
trainingVideoMAEEnabled?: boolean trainingVideoMAEEnabled?: boolean
trainingAutoPauseEnabled?: boolean
trainingAutoPauseCpuPercent?: number
trainingAutoPauseTemperatureC?: number
trainingCpuCoreCount?: number trainingCpuCoreCount?: number
trainingEffectiveMode?: TrainingPerformanceMode | string trainingEffectiveMode?: TrainingPerformanceMode | string
trainingEffectiveCpuThreads?: number trainingEffectiveCpuThreads?: number
@ -99,6 +102,10 @@ type TrainingJobStatus = {
progress?: number progress?: number
step?: string step?: string
message?: string message?: string
paused?: boolean
pauseReason?: string
cpuPercent?: number
temperatureC?: number
} }
type AppLog = { type AppLog = {
@ -342,6 +349,9 @@ const DEFAULTS: Settings = {
trainingYoloBatchSize: 0, trainingYoloBatchSize: 0,
trainingLowPriority: false, trainingLowPriority: false,
trainingVideoMAEEnabled: true, trainingVideoMAEEnabled: true,
trainingAutoPauseEnabled: false,
trainingAutoPauseCpuPercent: 95,
trainingAutoPauseTemperatureC: 85,
} }
const TRAINING_PERFORMANCE_MODES: Array<{ const TRAINING_PERFORMANCE_MODES: Array<{
@ -1024,6 +1034,12 @@ export default function Settings({ onAssetsGenerated }: Props) {
(data as any).trainingLowPriority ?? DEFAULTS.trainingLowPriority, (data as any).trainingLowPriority ?? DEFAULTS.trainingLowPriority,
trainingVideoMAEEnabled: trainingVideoMAEEnabled:
(data as any).trainingVideoMAEEnabled ?? DEFAULTS.trainingVideoMAEEnabled, (data as any).trainingVideoMAEEnabled ?? DEFAULTS.trainingVideoMAEEnabled,
trainingAutoPauseEnabled:
(data as any).trainingAutoPauseEnabled ?? DEFAULTS.trainingAutoPauseEnabled,
trainingAutoPauseCpuPercent:
(data as any).trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent,
trainingAutoPauseTemperatureC:
(data as any).trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC,
trainingCpuCoreCount: trainingCpuCoreCount:
(data as any).trainingCpuCoreCount ?? DEFAULTS.trainingCpuCoreCount, (data as any).trainingCpuCoreCount ?? DEFAULTS.trainingCpuCoreCount,
trainingEffectiveMode: trainingEffectiveMode:
@ -1073,6 +1089,10 @@ export default function Settings({ onAssetsGenerated }: Props) {
progress: Number(job.progress || 0), progress: Number(job.progress || 0),
step: String(job.step || ''), step: String(job.step || ''),
message: String(job.message || ''), message: String(job.message || ''),
paused: Boolean(job.paused),
pauseReason: String(job.pauseReason || ''),
cpuPercent: Number(job.cpuPercent || 0),
temperatureC: Number(job.temperatureC || 0),
} }
: null) : null)
} catch { } catch {
@ -1611,6 +1631,15 @@ export default function Settings({ onAssetsGenerated }: Props) {
) )
const trainingLowPriority = !!value.trainingLowPriority const trainingLowPriority = !!value.trainingLowPriority
const trainingVideoMAEEnabled = value.trainingVideoMAEEnabled !== false const trainingVideoMAEEnabled = value.trainingVideoMAEEnabled !== false
const trainingAutoPauseEnabled = !!value.trainingAutoPauseEnabled
const trainingAutoPauseCpuPercent = Math.max(
1,
Math.min(100, Math.floor(Number(value.trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent ?? 95)))
)
const trainingAutoPauseTemperatureC = Math.max(
1,
Math.min(120, Math.floor(Number(value.trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC ?? 85)))
)
setSaving(true) setSaving(true)
try { try {
@ -1656,6 +1685,9 @@ export default function Settings({ onAssetsGenerated }: Props) {
trainingYoloBatchSize, trainingYoloBatchSize,
trainingLowPriority, trainingLowPriority,
trainingVideoMAEEnabled, trainingVideoMAEEnabled,
trainingAutoPauseEnabled,
trainingAutoPauseCpuPercent,
trainingAutoPauseTemperatureC,
}), }),
}) })
if (!res.ok) { if (!res.ok) {
@ -3089,6 +3121,100 @@ export default function Settings({ onAssetsGenerated }: Props) {
description="Deaktivieren spart viel CPU-Zeit. YOLO Detector und Pose können weiterhin trainiert werden." description="Deaktivieren spart viel CPU-Zeit. YOLO Detector und Pose können weiterhin trainiert werden."
/> />
</div> </div>
<div className="mt-3 rounded-xl border border-gray-200 bg-white p-3 dark:border-white/10 dark:bg-gray-950/40">
<LabeledSwitch
checked={!!value.trainingAutoPauseEnabled}
onChange={(checked) =>
setValue((v) => ({
...v,
trainingAutoPauseEnabled: checked,
}))
}
disabled={trainingSettingsLocked}
label="Training automatisch pausieren"
description="Pausiert laufende Python-Trainingsprozesse bei dauerhaft hoher CPU-Last oder hoher CPU-Temperatur und setzt sie nach einer kurzen Abkühlphase fort."
/>
<div
className={[
'mt-3 grid grid-cols-1 gap-3 sm:grid-cols-2',
value.trainingAutoPauseEnabled && !trainingSettingsLocked ? '' : 'opacity-60',
].join(' ')}
>
<div>
<label
htmlFor="trainingAutoPauseCpuPercent"
className="text-sm font-medium text-gray-900 dark:text-gray-200"
>
CPU-Grenze
</label>
<div className="mt-1 flex items-center gap-2">
<input
id="trainingAutoPauseCpuPercent"
type="number"
min={1}
max={100}
step={1}
disabled={!value.trainingAutoPauseEnabled || trainingSettingsLocked}
value={value.trainingAutoPauseCpuPercent ?? 95}
onChange={(e) =>
setValue((v) => ({
...v,
trainingAutoPauseCpuPercent: Number(e.target.value || 95),
}))
}
className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100"
/>
<span className="shrink-0 text-xs text-gray-600 dark:text-gray-300">
%
</span>
</div>
</div>
<div>
<label
htmlFor="trainingAutoPauseTemperatureC"
className="text-sm font-medium text-gray-900 dark:text-gray-200"
>
Temperatur-Grenze
</label>
<div className="mt-1 flex items-center gap-2">
<input
id="trainingAutoPauseTemperatureC"
type="number"
min={1}
max={120}
step={1}
disabled={!value.trainingAutoPauseEnabled || trainingSettingsLocked}
value={value.trainingAutoPauseTemperatureC ?? 85}
onChange={(e) =>
setValue((v) => ({
...v,
trainingAutoPauseTemperatureC: Number(e.target.value || 85),
}))
}
className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100"
/>
<span className="shrink-0 text-xs text-gray-600 dark:text-gray-300">
°C
</span>
</div>
</div>
</div>
{trainingJobStatus?.paused ? (
<div className="mt-3 rounded-lg border border-amber-200 bg-amber-50 px-3 py-2 text-xs font-medium text-amber-900 dark:border-amber-400/30 dark:bg-amber-500/10 dark:text-amber-100">
Training pausiert: {trainingJobStatus.pauseReason || 'Ressourcenlimit erreicht'}
{trainingJobStatus.cpuPercent ? ` · CPU ${Math.round(trainingJobStatus.cpuPercent)}%` : ''}
{trainingJobStatus.temperatureC ? ` · ${Math.round(trainingJobStatus.temperatureC)}°C` : ''}
</div>
) : (
<div className="mt-2 text-xs text-gray-500 dark:text-gray-400">
Temperatur greift nur, wenn der Server passende Sensorwerte liefert.
</div>
)}
</div>
</div> </div>
</div> </div>

View File

@ -14,6 +14,7 @@ import {
CheckIcon, CheckIcon,
ForwardIcon, ForwardIcon,
InboxArrowDownIcon, InboxArrowDownIcon,
PauseIcon,
RectangleGroupIcon, RectangleGroupIcon,
UserGroupIcon, UserGroupIcon,
TrashIcon, TrashIcon,
@ -106,6 +107,10 @@ type TrainingJobStatus = {
epoch?: number epoch?: number
epochs?: number epochs?: number
previewUrl?: string previewUrl?: string
paused?: boolean
pauseReason?: string
cpuPercent?: number
temperatureC?: number
map50?: number map50?: number
map5095?: number map5095?: number
} }
@ -260,6 +265,9 @@ type TrainingHistoryEntry = {
workers?: number workers?: number
yoloBatchSize?: number yoloBatchSize?: number
lowPriority?: boolean lowPriority?: boolean
autoPauseEnabled?: boolean
autoPauseCpuPercent?: number
autoPauseTemperatureC?: number
} }
type TrainingStats = { type TrainingStats = {
@ -327,6 +335,7 @@ type TrainingEstimateRuntime = {
workers: number workers: number
yoloBatchLabel: string yoloBatchLabel: string
lowPriority: boolean lowPriority: boolean
autoPause: boolean
factor: number factor: number
yoloFactor: number yoloFactor: number
} }
@ -718,6 +727,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
const lowPriority = Boolean( const lowPriority = Boolean(
settings?.trainingEffectiveLowPriority ?? settings?.trainingLowPriority settings?.trainingEffectiveLowPriority ?? settings?.trainingLowPriority
) )
const autoPause = Boolean(settings?.trainingAutoPauseEnabled)
const autoThreads = cpuCores > 0 const autoThreads = cpuCores > 0
? Math.max(1, Math.min(16, Math.round(cpuCores * 0.75))) ? Math.max(1, Math.min(16, Math.round(cpuCores * 0.75)))
: 4 : 4
@ -769,11 +779,12 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
? 0.98 ? 0.98
: 1 : 1
const lowPriorityFactor = lowPriority ? 1.12 : 1 const lowPriorityFactor = lowPriority ? 1.12 : 1
const autoPauseFactor = autoPause ? 1.12 : 1
const corePenalty = const corePenalty =
cpuCores > 0 && effectiveThreads > cpuCores cpuCores > 0 && effectiveThreads > cpuCores
? 1 + Math.min(0.3, (effectiveThreads - cpuCores) * 0.04) ? 1 + Math.min(0.3, (effectiveThreads - cpuCores) * 0.04)
: 1 : 1
const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * corePenalty const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * autoPauseFactor * corePenalty
const yoloFactor = factor * batchFactor const yoloFactor = factor * batchFactor
return { return {
@ -783,6 +794,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
workers, workers,
yoloBatchLabel: rawBatch > 0 ? String(rawBatch) : 'Auto', yoloBatchLabel: rawBatch > 0 ? String(rawBatch) : 'Auto',
lowPriority, lowPriority,
autoPause,
factor: Number.isFinite(factor) && factor > 0 ? factor : 1, factor: Number.isFinite(factor) && factor > 0 ? factor : 1,
yoloFactor: Number.isFinite(yoloFactor) && yoloFactor > 0 ? yoloFactor : 1, yoloFactor: Number.isFinite(yoloFactor) && yoloFactor > 0 ? yoloFactor : 1,
} }
@ -790,7 +802,8 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
function trainingEstimateRuntimeText(runtime: TrainingEstimateRuntime) { function trainingEstimateRuntimeText(runtime: TrainingEstimateRuntime) {
const priority = runtime.lowPriority ? ' · niedrige Priorität' : '' const priority = runtime.lowPriority ? ' · niedrige Priorität' : ''
return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}` const autoPause = runtime.autoPause ? ' · Auto-Pause' : ''
return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}${autoPause}`
} }
function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) { function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) {
@ -802,6 +815,7 @@ function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) {
trainingEffectiveWorkers: entry.workers, trainingEffectiveWorkers: entry.workers,
trainingEffectiveYoloBatchSize: entry.yoloBatchSize, trainingEffectiveYoloBatchSize: entry.yoloBatchSize,
trainingEffectiveLowPriority: entry.lowPriority, trainingEffectiveLowPriority: entry.lowPriority,
trainingAutoPauseEnabled: entry.autoPauseEnabled,
} as RecorderSettingsState) } as RecorderSettingsState)
} }
@ -4644,6 +4658,10 @@ export default function TrainingTab(props: {
epoch: Number(job.epoch ?? 0), epoch: Number(job.epoch ?? 0),
epochs: Number(job.epochs ?? 0), epochs: Number(job.epochs ?? 0),
previewUrl: String(job.previewUrl ?? ''), previewUrl: String(job.previewUrl ?? ''),
paused: Boolean(job.paused),
pauseReason: String(job.pauseReason ?? ''),
cpuPercent: Number(job.cpuPercent ?? 0),
temperatureC: Number(job.temperatureC ?? 0),
map50: Number(job.map50 ?? 0), map50: Number(job.map50 ?? 0),
map5095: Number(job.map5095 ?? 0), map5095: Number(job.map5095 ?? 0),
} }
@ -5125,6 +5143,10 @@ export default function TrainingTab(props: {
const importVideoIntoTraining = useCallback(async (raw: any) => { const importVideoIntoTraining = useCallback(async (raw: any) => {
const output = String(raw?.output || '').trim() const output = String(raw?.output || '').trim()
if (!output) return false if (!output) return false
if (trainingRunning) {
setMessage('Während das Training läuft, kann kein Video ins Training übernommen werden.')
return false
}
setLoadingPreviewCandidate('') setLoadingPreviewCandidate('')
@ -5277,6 +5299,7 @@ export default function TrainingTab(props: {
loadPriorityTrainingSamples, loadPriorityTrainingSamples,
loadTrainingStatus, loadTrainingStatus,
setLoadingPreviewCandidate, setLoadingPreviewCandidate,
trainingRunning,
waitForVideoImportResult, waitForVideoImportResult,
]) ])
@ -7243,13 +7266,25 @@ export default function TrainingTab(props: {
}, },
] ]
const trainingPaused = Boolean(trainingStatus?.training?.paused)
const pauseReason = String(trainingStatus?.training?.pauseReason || '').trim()
const pauseMetrics = [
trainingStatus?.training?.cpuPercent
? `CPU ${Math.round(Number(trainingStatus.training.cpuPercent))}%`
: '',
trainingStatus?.training?.temperatureC
? `${Math.round(Number(trainingStatus.training.temperatureC))}°C`
: '',
].filter(Boolean).join(' · ')
const progress = clampPercent( const progress = clampPercent(
trainingRunning trainingRunning
? shownTrainingProgress ? shownTrainingProgress
: readinessProgress * 100 : readinessProgress * 100
) )
const statusText = trainingRunning const statusText = trainingRunning
? shownTrainingStep || 'Training läuft…' ? trainingPaused
? `Pausiert${pauseReason ? `: ${pauseReason}` : ''}${pauseMetrics ? ` · ${pauseMetrics}` : ''}`
: shownTrainingStep || 'Training läuft…'
: !feedbackReady : !feedbackReady
? `${Math.max(0, requiredCount - feedbackCount)} Feedback fehlen noch` ? `${Math.max(0, requiredCount - feedbackCount)} Feedback fehlen noch`
: canStartTraining : canStartTraining
@ -7280,14 +7315,18 @@ export default function TrainingTab(props: {
<div <div
className={[ className={[
'flex h-8 w-8 shrink-0 items-center justify-center rounded-lg ring-1', 'flex h-8 w-8 shrink-0 items-center justify-center rounded-lg ring-1',
trainingRunning trainingPaused
? 'bg-amber-50 text-amber-700 ring-amber-200 dark:bg-amber-500/15 dark:text-amber-200 dark:ring-amber-400/30'
: trainingRunning
? 'bg-indigo-50 text-indigo-700 ring-indigo-200 dark:bg-indigo-500/15 dark:text-indigo-200 dark:ring-indigo-400/30' ? 'bg-indigo-50 text-indigo-700 ring-indigo-200 dark:bg-indigo-500/15 dark:text-indigo-200 dark:ring-indigo-400/30'
: canStartTraining : canStartTraining
? 'bg-emerald-50 text-emerald-700 ring-emerald-200 dark:bg-emerald-500/15 dark:text-emerald-200 dark:ring-emerald-400/30' ? 'bg-emerald-50 text-emerald-700 ring-emerald-200 dark:bg-emerald-500/15 dark:text-emerald-200 dark:ring-emerald-400/30'
: 'bg-gray-50 text-gray-600 ring-gray-200 dark:bg-white/5 dark:text-gray-300 dark:ring-white/10', : 'bg-gray-50 text-gray-600 ring-gray-200 dark:bg-white/5 dark:text-gray-300 dark:ring-white/10',
].join(' ')} ].join(' ')}
> >
{trainingRunning ? ( {trainingPaused ? (
<PauseIcon className="h-4 w-4" aria-hidden="true" />
) : trainingRunning ? (
<ArrowPathIcon className="h-4 w-4 animate-spin" aria-hidden="true" /> <ArrowPathIcon className="h-4 w-4 animate-spin" aria-hidden="true" />
) : ( ) : (
<BoltIcon className="h-4 w-4" aria-hidden="true" /> <BoltIcon className="h-4 w-4" aria-hidden="true" />
@ -7394,7 +7433,9 @@ export default function TrainingTab(props: {
<div <div
className={[ className={[
'h-full rounded-full transition-all duration-500', 'h-full rounded-full transition-all duration-500',
trainingRunning trainingPaused
? 'bg-amber-500'
: trainingRunning
? 'bg-indigo-500' ? 'bg-indigo-500'
: canStartTraining : canStartTraining
? 'bg-emerald-500' ? 'bg-emerald-500'

View File

@ -289,6 +289,9 @@ export type RecorderSettingsState = {
trainingYoloBatchSize?: number trainingYoloBatchSize?: number
trainingLowPriority?: boolean trainingLowPriority?: boolean
trainingVideoMAEEnabled?: boolean trainingVideoMAEEnabled?: boolean
trainingAutoPauseEnabled?: boolean
trainingAutoPauseCpuPercent?: number
trainingAutoPauseTemperatureC?: number
trainingCpuCoreCount?: number trainingCpuCoreCount?: number
trainingEffectiveMode?: string trainingEffectiveMode?: string
trainingEffectiveCpuThreads?: number trainingEffectiveCpuThreads?: number