diff --git a/backend/dist/nsfwapp.exe b/backend/dist/nsfwapp.exe deleted file mode 100644 index b81984a..0000000 Binary files a/backend/dist/nsfwapp.exe and /dev/null differ diff --git a/backend/performance.go b/backend/performance.go index 0532056..a02fb3f 100644 --- a/backend/performance.go +++ b/backend/performance.go @@ -13,11 +13,13 @@ import ( "runtime" "strconv" "strings" + "sync" "sync/atomic" "time" gocpu "github.com/shirou/gopsutil/v3/cpu" godisk "github.com/shirou/gopsutil/v3/disk" + gohost "github.com/shirou/gopsutil/v3/host" ) // Server-Startzeit für Perf/Uptime @@ -25,6 +27,13 @@ var serverStartedAt = time.Now() var lastCPUUsageBits uint64 // atomic float64 bits +var cpuTemperatureCache = struct { + sync.Mutex + value float64 + available bool + sampledAt time.Time +}{} + func setLastCPUUsage(v float64) { atomic.StoreUint64(&lastCPUUsageBits, math.Float64bits(v)) } @@ -33,6 +42,71 @@ func getLastCPUUsage() float64 { return math.Float64frombits(atomic.LoadUint64(&lastCPUUsageBits)) } +func getMaxCPUTemperatureC() (float64, bool) { + now := time.Now() + + cpuTemperatureCache.Lock() + if now.Sub(cpuTemperatureCache.sampledAt) < 15*time.Second { + value := cpuTemperatureCache.value + available := cpuTemperatureCache.available + cpuTemperatureCache.Unlock() + return value, available + } + cpuTemperatureCache.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + temps, err := gohost.SensorsTemperaturesWithContext(ctx) + if err != nil || len(temps) == 0 { + cpuTemperatureCache.Lock() + cpuTemperatureCache.value = 0 + cpuTemperatureCache.available = false + cpuTemperatureCache.sampledAt = now + cpuTemperatureCache.Unlock() + return 0, false + } + + maxCPU := 0.0 + maxAny := 0.0 + for _, temp := range temps { + v := temp.Temperature + if math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 || v > 150 { + continue + } + + if v > maxAny { + maxAny = v + } + + key := strings.ToLower(strings.TrimSpace(temp.SensorKey)) + if strings.Contains(key, "cpu") || + strings.Contains(key, "core") || + strings.Contains(key, "package") || + strings.Contains(key, "tdie") || + strings.Contains(key, "tctl") || + strings.Contains(key, "k10temp") { + if v > maxCPU { + maxCPU = v + } + } + } + + value := maxCPU + if value <= 0 { + value = maxAny + } + available := value > 0 + + cpuTemperatureCache.Lock() + cpuTemperatureCache.value = value + cpuTemperatureCache.available = available + cpuTemperatureCache.sampledAt = now + cpuTemperatureCache.Unlock() + + return value, available +} + func startCPUUsageSampler(ctx context.Context) { _, _ = gocpu.Percent(200*time.Millisecond, false) @@ -194,6 +268,17 @@ func buildPerfSnapshot() map[string]any { } return v }(), + "cpuTemperatureC": func() float64 { + v, ok := getMaxCPUTemperatureC() + if !ok { + return 0 + } + return math.Round(v*10) / 10 + }(), + "cpuTemperatureAvailable": func() bool { + _, ok := getMaxCPUTemperatureC() + return ok + }(), "diskPath": diskPath, "diskFreeBytes": diskFreeBytes, "diskTotalBytes": diskTotalBytes, diff --git a/backend/settings.go b/backend/settings.go index 5e79abf..8971e99 100644 --- a/backend/settings.go +++ b/backend/settings.go @@ -56,15 +56,18 @@ type RecorderSettings struct { GenerateAssetsAnalyze bool `json:"generateAssetsAnalyze"` EnrichPostworkEnabled bool `json:"enrichPostworkEnabled"` - TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"` - TrainingDetectorEpochs int `json:"trainingDetectorEpochs"` - TrainingPerformanceMode string `json:"trainingPerformanceMode"` - TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"` - TrainingCPUThreads int `json:"trainingCpuThreads"` - TrainingWorkers int `json:"trainingWorkers"` - TrainingYoloBatchSize int `json:"trainingYoloBatchSize"` - TrainingLowPriority bool `json:"trainingLowPriority"` - TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"` + TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"` + TrainingDetectorEpochs int `json:"trainingDetectorEpochs"` + TrainingPerformanceMode string `json:"trainingPerformanceMode"` + TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"` + TrainingCPUThreads int `json:"trainingCpuThreads"` + TrainingWorkers int `json:"trainingWorkers"` + TrainingYoloBatchSize int `json:"trainingYoloBatchSize"` + TrainingLowPriority bool `json:"trainingLowPriority"` + TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"` + TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"` + TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"` + TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"` // EncryptedCookies contains base64(nonce+ciphertext) of a JSON cookie map. EncryptedCookies string `json:"encryptedCookies"` @@ -112,15 +115,18 @@ var ( GenerateAssetsAnalyze: false, EnrichPostworkEnabled: true, - TrainingRecognitionEnabled: true, - TrainingDetectorEpochs: 60, - TrainingPerformanceMode: trainingPerformanceAuto, - TrainingPowerSaveMode: false, - TrainingCPUThreads: 0, - TrainingWorkers: 2, - TrainingYoloBatchSize: 0, - TrainingLowPriority: false, - TrainingVideoMAEEnabled: true, + TrainingRecognitionEnabled: true, + TrainingDetectorEpochs: 60, + TrainingPerformanceMode: trainingPerformanceAuto, + TrainingPowerSaveMode: false, + TrainingCPUThreads: 0, + TrainingWorkers: 2, + TrainingYoloBatchSize: 0, + TrainingLowPriority: false, + TrainingVideoMAEEnabled: true, + TrainingAutoPauseEnabled: false, + TrainingAutoPauseCPUPercent: 95, + TrainingAutoPauseTemperatureC: 85, EncryptedCookies: "", } @@ -189,6 +195,20 @@ func normalizeTrainingSettings(s *RecorderSettings) { if s.TrainingYoloBatchSize > 64 { s.TrainingYoloBatchSize = 64 } + + if s.TrainingAutoPauseCPUPercent < 1 { + s.TrainingAutoPauseCPUPercent = 95 + } + if s.TrainingAutoPauseCPUPercent > 100 { + s.TrainingAutoPauseCPUPercent = 100 + } + + if s.TrainingAutoPauseTemperatureC < 1 { + s.TrainingAutoPauseTemperatureC = 85 + } + if s.TrainingAutoPauseTemperatureC > 120 { + s.TrainingAutoPauseTemperatureC = 120 + } } func settingsFilePath() string { @@ -375,6 +395,9 @@ type RecorderSettingsPublic struct { TrainingYoloBatchSize int `json:"trainingYoloBatchSize"` TrainingLowPriority bool `json:"trainingLowPriority"` TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"` + TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"` + TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"` + TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"` TrainingCPUCoreCount int `json:"trainingCpuCoreCount"` TrainingEffectiveMode string `json:"trainingEffectiveMode"` TrainingEffectiveCPUThreads int `json:"trainingEffectiveCpuThreads"` @@ -436,6 +459,9 @@ func toPublicSettings(s RecorderSettings) RecorderSettingsPublic { TrainingYoloBatchSize: s.TrainingYoloBatchSize, TrainingLowPriority: s.TrainingLowPriority, TrainingVideoMAEEnabled: s.TrainingVideoMAEEnabled, + TrainingAutoPauseEnabled: s.TrainingAutoPauseEnabled, + TrainingAutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent, + TrainingAutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC, TrainingCPUCoreCount: trainingOpts.CPUCoreCount, TrainingEffectiveMode: trainingOpts.PerformanceMode, TrainingEffectiveCPUThreads: trainingOpts.CPUThreads, @@ -631,6 +657,9 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { next.TrainingYoloBatchSize = in.TrainingYoloBatchSize next.TrainingLowPriority = in.TrainingLowPriority next.TrainingVideoMAEEnabled = in.TrainingVideoMAEEnabled + next.TrainingAutoPauseEnabled = in.TrainingAutoPauseEnabled + next.TrainingAutoPauseCPUPercent = in.TrainingAutoPauseCPUPercent + next.TrainingAutoPauseTemperatureC = in.TrainingAutoPauseTemperatureC dbChanged := normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) || diff --git a/backend/training.go b/backend/training.go index 33e2872..50fde8c 100644 --- a/backend/training.go +++ b/backend/training.go @@ -24,6 +24,8 @@ import ( "strings" "sync" "time" + + goprocess "github.com/shirou/gopsutil/v3/process" ) const trainingUncertainCandidateCount = 10 @@ -262,15 +264,19 @@ type TrainingPosePrediction struct { } type TrainingJobStatus struct { - Running bool `json:"running"` - Progress int `json:"progress"` - Step string `json:"step"` - Message string `json:"message,omitempty"` - Error string `json:"error,omitempty"` - StartedAt string `json:"startedAt,omitempty"` - FinishedAt string `json:"finishedAt,omitempty"` - DurationMs int64 `json:"durationMs,omitempty"` - PreviewURL string `json:"previewUrl,omitempty"` + Running bool `json:"running"` + Progress int `json:"progress"` + Step string `json:"step"` + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` + DurationMs int64 `json:"durationMs,omitempty"` + PreviewURL string `json:"previewUrl,omitempty"` + Paused bool `json:"paused,omitempty"` + PauseReason string `json:"pauseReason,omitempty"` + CPUPercent float64 `json:"cpuPercent,omitempty"` + TemperatureC float64 `json:"temperatureC,omitempty"` Stage string `json:"stage,omitempty"` Epoch int `json:"epoch,omitempty"` @@ -1181,6 +1187,8 @@ func trainingRunCommandStreaming( return "", err } applyTrainingLowPriorityAfterStart(cmd, opts.LowPriority) + monitorCtx, stopResourcePauseMonitor := context.WithCancel(ctx) + resumeResourcePause := startTrainingResourcePauseMonitor(monitorCtx, cmd, opts) var outMu sync.Mutex var lines []string @@ -1231,9 +1239,12 @@ func trainingRunCommandStreaming( select { case err = <-waitCh: case <-ctx.Done(): + resumeResourcePause() terminateTrainingCommand(cmd) err = <-waitCh } + stopResourcePauseMonitor() + resumeResourcePause() wg.Wait() outMu.Lock() @@ -1274,14 +1285,17 @@ const ( ) type trainingRuntimeOptions struct { - PerformanceMode string - PowerSaveMode bool - CPUCoreCount int - CPUThreads int - Workers int - YoloBatchSize int - LowPriority bool - VideoMAEEnabled bool + PerformanceMode string + PowerSaveMode bool + CPUCoreCount int + CPUThreads int + Workers int + YoloBatchSize int + LowPriority bool + VideoMAEEnabled bool + AutoPauseEnabled bool + AutoPauseCPUPercent int + AutoPauseTemperatureC int } func normalizeTrainingPerformanceMode(raw string) string { @@ -1381,14 +1395,17 @@ func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRunt presetThreads, presetWorkers, presetBatch, presetLowPriority := trainingPresetValues(mode, cpuCores) opts := trainingRuntimeOptions{ - PerformanceMode: mode, - PowerSaveMode: mode == trainingPerformanceEco, - CPUCoreCount: cpuCores, - CPUThreads: presetThreads, - Workers: presetWorkers, - YoloBatchSize: presetBatch, - LowPriority: presetLowPriority, - VideoMAEEnabled: s.TrainingVideoMAEEnabled, + PerformanceMode: mode, + PowerSaveMode: mode == trainingPerformanceEco, + CPUCoreCount: cpuCores, + CPUThreads: presetThreads, + Workers: presetWorkers, + YoloBatchSize: presetBatch, + LowPriority: presetLowPriority, + VideoMAEEnabled: s.TrainingVideoMAEEnabled, + AutoPauseEnabled: s.TrainingAutoPauseEnabled, + AutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent, + AutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC, } if mode == trainingPerformanceCustom { @@ -1448,6 +1465,331 @@ func trainingCommandEnv(opts trainingRuntimeOptions) []string { return env } +type trainingResourceSnapshot struct { + CPUPercent float64 + TemperatureC float64 + TemperatureAvailable bool +} + +func trainingRoundMetric(v float64) float64 { + if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 { + return 0 + } + return math.Round(v*10) / 10 +} + +func trainingReadResourceSnapshot() trainingResourceSnapshot { + temp, tempOK := getMaxCPUTemperatureC() + return trainingResourceSnapshot{ + CPUPercent: trainingRoundMetric(getLastCPUUsage()), + TemperatureC: trainingRoundMetric(temp), + TemperatureAvailable: tempOK, + } +} + +func trainingPauseReasonForResources(opts trainingRuntimeOptions, snap trainingResourceSnapshot) (string, bool) { + if !opts.AutoPauseEnabled { + return "", false + } + + reasons := []string{} + if opts.AutoPauseCPUPercent > 0 && snap.CPUPercent >= float64(opts.AutoPauseCPUPercent) { + reasons = append( + reasons, + fmt.Sprintf("CPU %.0f%% >= %d%%", snap.CPUPercent, opts.AutoPauseCPUPercent), + ) + } + if opts.AutoPauseTemperatureC > 0 && + snap.TemperatureAvailable && + snap.TemperatureC >= float64(opts.AutoPauseTemperatureC) { + reasons = append( + reasons, + fmt.Sprintf("Temperatur %.0f°C >= %d°C", snap.TemperatureC, opts.AutoPauseTemperatureC), + ) + } + + if len(reasons) == 0 { + return "", false + } + return strings.Join(reasons, ", "), true +} + +func trainingResourcesRecovered(opts trainingRuntimeOptions, snap trainingResourceSnapshot) bool { + if !opts.AutoPauseEnabled { + return true + } + + if opts.AutoPauseCPUPercent > 0 { + resumeCPU := float64(opts.AutoPauseCPUPercent - 15) + if resumeCPU < 50 { + resumeCPU = 50 + } + if snap.CPUPercent > resumeCPU { + return false + } + } + + if opts.AutoPauseTemperatureC > 0 && snap.TemperatureAvailable { + resumeTemp := float64(opts.AutoPauseTemperatureC - 5) + if resumeTemp < 40 { + resumeTemp = 40 + } + if snap.TemperatureC > resumeTemp { + return false + } + } + + return true +} + +func trainingProcessTree(pid int32) []*goprocess.Process { + if pid <= 0 { + return nil + } + + root, err := goprocess.NewProcess(pid) + if err != nil || root == nil { + return nil + } + + seen := map[int32]bool{} + var out []*goprocess.Process + var walk func(*goprocess.Process) + walk = func(p *goprocess.Process) { + if p == nil || seen[p.Pid] { + return + } + + seen[p.Pid] = true + out = append(out, p) + + children, err := p.Children() + if err != nil { + return + } + for _, child := range children { + walk(child) + } + } + + walk(root) + return out +} + +func trainingSuspendProcessTree(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 { + return nil + } + + procs := trainingProcessTree(int32(cmd.Process.Pid)) + if len(procs) == 0 { + return nil + } + + var firstErr error + success := 0 + for i := len(procs) - 1; i >= 0; i-- { + if err := procs[i].Suspend(); err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + success++ + } + + if success > 0 { + return nil + } + return firstErr +} + +func trainingResumeProcessTree(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 { + return nil + } + + procs := trainingProcessTree(int32(cmd.Process.Pid)) + if len(procs) == 0 { + return nil + } + + var firstErr error + success := 0 + for _, proc := range procs { + if err := proc.Resume(); err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + success++ + } + + if success > 0 { + return nil + } + return firstErr +} + +func startTrainingResourcePauseMonitor(ctx context.Context, cmd *exec.Cmd, opts trainingRuntimeOptions) func() { + if !opts.AutoPauseEnabled || cmd == nil || cmd.Process == nil { + return func() {} + } + + const ( + checkInterval = 5 * time.Second + requiredHighSamples = 2 + requiredCoolSamples = 2 + minPauseDuration = 45 * time.Second + ) + + var pauseMu sync.Mutex + paused := false + previousStep := "" + highSamples := 0 + coolSamples := 0 + pauseStartedAt := time.Time{} + loggedSuspendError := false + + resumeNow := func(reason string) { + pauseMu.Lock() + if !paused { + pauseMu.Unlock() + return + } + + err := trainingResumeProcessTree(cmd) + paused = false + restoreStep := strings.TrimSpace(previousStep) + previousStep = "" + pauseMu.Unlock() + + if err != nil { + appLogln("⚠️ Training konnte nach Ressourcenpause nicht sauber fortgesetzt werden:", err) + } + + if strings.TrimSpace(reason) != "" { + appLogln(reason) + } + + trainingSetJobStatus(func(s *TrainingJobStatus) { + s.Paused = false + s.PauseReason = "" + s.CPUPercent = 0 + s.TemperatureC = 0 + s.Message = "" + if restoreStep != "" && s.Running { + s.Step = restoreStep + } + }) + } + + go func() { + ticker := time.NewTicker(checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + resumeNow("Training-Ressourcenpause wurde beendet.") + return + case <-ticker.C: + } + + snap := trainingReadResourceSnapshot() + + pauseMu.Lock() + isPaused := paused + pauseAge := time.Since(pauseStartedAt) + pauseMu.Unlock() + + if !isPaused { + reason, shouldPause := trainingPauseReasonForResources(opts, snap) + if !shouldPause { + highSamples = 0 + continue + } + + highSamples++ + if highSamples < requiredHighSamples { + continue + } + + pauseMu.Lock() + if paused { + pauseMu.Unlock() + continue + } + + currentStatus := trainingGetJobStatus() + previousStep = strings.TrimSpace(currentStatus.Step) + if previousStep == "" { + previousStep = "Training läuft..." + } + + if err := trainingSuspendProcessTree(cmd); err != nil { + pauseMu.Unlock() + highSamples = 0 + if !loggedSuspendError { + loggedSuspendError = true + appLogln("⚠️ Training-Ressourcenpause konnte Prozess nicht pausieren:", err) + } + continue + } + + paused = true + pauseStartedAt = time.Now() + coolSamples = 0 + pauseMu.Unlock() + + appLogln("⏸ Training pausiert wegen Ressourcenlimit:", reason) + trainingSetJobStatus(func(s *TrainingJobStatus) { + s.Paused = true + s.PauseReason = reason + s.CPUPercent = snap.CPUPercent + if snap.TemperatureAvailable { + s.TemperatureC = snap.TemperatureC + } else { + s.TemperatureC = 0 + } + s.Step = "Training pausiert: " + reason + s.Message = "Training wird automatisch fortgesetzt, sobald CPU/Temperatur wieder im grünen Bereich sind." + }) + continue + } + + trainingSetJobStatus(func(s *TrainingJobStatus) { + if !s.Running || !s.Paused { + return + } + s.CPUPercent = snap.CPUPercent + if snap.TemperatureAvailable { + s.TemperatureC = snap.TemperatureC + } + }) + + if pauseAge < minPauseDuration || !trainingResourcesRecovered(opts, snap) { + coolSamples = 0 + continue + } + + coolSamples++ + if coolSamples < requiredCoolSamples { + continue + } + + highSamples = 0 + coolSamples = 0 + resumeNow("▶ Training nach Ressourcenpause fortgesetzt.") + } + }() + + return func() { + resumeNow("") + } +} + var trainingJob = struct { mu sync.Mutex status TrainingJobStatus @@ -1526,6 +1868,10 @@ func trainingFinishCancelled(root string) { s.FinishedAt = finishedAt.Format(time.RFC3339) s.DurationMs = durationMs s.PreviewURL = "" + s.Paused = false + s.PauseReason = "" + s.CPUPercent = 0 + s.TemperatureC = 0 if cleanupErr != nil { s.Message = "Training wurde abgebrochen, aber temporäre Trainingsausgaben konnten nicht vollständig gelöscht werden." @@ -3641,6 +3987,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin s.FinishedAt = finishedAt.Format(time.RFC3339) s.DurationMs = durationMs s.PreviewURL = "" + s.Paused = false + s.PauseReason = "" + s.CPUPercent = 0 + s.TemperatureC = 0 }) trainingClearJobCancel() return @@ -3650,7 +4000,7 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin appLogln("ML-Python für Training:", python) runtimeOpts := trainingRuntimeOptionsFromSettings() appLogf( - "Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v targets=%s", + "Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v autoPause=%v cpuLimit=%d tempLimit=%d targets=%s", runtimeOpts.PerformanceMode, runtimeOpts.CPUCoreCount, runtimeOpts.PowerSaveMode, @@ -3659,6 +4009,9 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin runtimeOpts.YoloBatchSize, runtimeOpts.LowPriority, runtimeOpts.VideoMAEEnabled, + runtimeOpts.AutoPauseEnabled, + runtimeOpts.AutoPauseCPUPercent, + runtimeOpts.AutoPauseTemperatureC, strings.Join(targets.list(), ","), ) @@ -4209,6 +4562,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin s.FinishedAt = finishedAt.Format(time.RFC3339) s.DurationMs = durationMs s.PreviewURL = "" + s.Paused = false + s.PauseReason = "" + s.CPUPercent = 0 + s.TemperatureC = 0 }) trainingClearJobCancel() @@ -4765,24 +5122,27 @@ func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo { } type TrainingHistoryEntry struct { - TrainedAt string `json:"trainedAt,omitempty"` - TrainedAtMs int64 `json:"trainedAtMs,omitempty"` - Target string `json:"target,omitempty"` - Status string `json:"status,omitempty"` - DurationMs int64 `json:"durationMs,omitempty"` - Epochs int `json:"epochs,omitempty"` - TrainSamples int `json:"trainSamples,omitempty"` - ValSamples int `json:"valSamples,omitempty"` - Imgsz int `json:"imgsz,omitempty"` - Device string `json:"device,omitempty"` - MAP50 float64 `json:"map50,omitempty"` - MAP5095 float64 `json:"map5095,omitempty"` - PerformanceMode string `json:"performanceMode,omitempty"` - CPUCoreCount int `json:"cpuCoreCount,omitempty"` - CPUThreads int `json:"cpuThreads,omitempty"` - Workers int `json:"workers,omitempty"` - YoloBatchSize int `json:"yoloBatchSize,omitempty"` - LowPriority bool `json:"lowPriority,omitempty"` + TrainedAt string `json:"trainedAt,omitempty"` + TrainedAtMs int64 `json:"trainedAtMs,omitempty"` + Target string `json:"target,omitempty"` + Status string `json:"status,omitempty"` + DurationMs int64 `json:"durationMs,omitempty"` + Epochs int `json:"epochs,omitempty"` + TrainSamples int `json:"trainSamples,omitempty"` + ValSamples int `json:"valSamples,omitempty"` + Imgsz int `json:"imgsz,omitempty"` + Device string `json:"device,omitempty"` + MAP50 float64 `json:"map50,omitempty"` + MAP5095 float64 `json:"map5095,omitempty"` + PerformanceMode string `json:"performanceMode,omitempty"` + CPUCoreCount int `json:"cpuCoreCount,omitempty"` + CPUThreads int `json:"cpuThreads,omitempty"` + Workers int `json:"workers,omitempty"` + YoloBatchSize int `json:"yoloBatchSize,omitempty"` + LowPriority bool `json:"lowPriority,omitempty"` + AutoPauseEnabled bool `json:"autoPauseEnabled,omitempty"` + AutoPauseCPUPercent int `json:"autoPauseCpuPercent,omitempty"` + AutoPauseTemperatureC int `json:"autoPauseTemperatureC,omitempty"` } type TrainingHistoryResponse struct { @@ -4813,24 +5173,27 @@ func trainingAppendTargetHistory(root string, target string, status string, dura now := time.Now().UTC() entry := TrainingHistoryEntry{ - TrainedAt: now.Format(time.RFC3339), - TrainedAtMs: now.UnixMilli(), - Target: kind, - Status: strings.TrimSpace(status), - DurationMs: durationMs, - Epochs: fallback.Epochs, - TrainSamples: fallback.TrainSamples, - ValSamples: fallback.ValSamples, - Imgsz: fallback.Imgsz, - Device: strings.TrimSpace(fallback.Device), - MAP50: fallback.MAP50, - MAP5095: fallback.MAP5095, - PerformanceMode: runtimeOpts.PerformanceMode, - CPUCoreCount: runtimeOpts.CPUCoreCount, - CPUThreads: runtimeOpts.CPUThreads, - Workers: runtimeOpts.Workers, - YoloBatchSize: runtimeOpts.YoloBatchSize, - LowPriority: runtimeOpts.LowPriority, + TrainedAt: now.Format(time.RFC3339), + TrainedAtMs: now.UnixMilli(), + Target: kind, + Status: strings.TrimSpace(status), + DurationMs: durationMs, + Epochs: fallback.Epochs, + TrainSamples: fallback.TrainSamples, + ValSamples: fallback.ValSamples, + Imgsz: fallback.Imgsz, + Device: strings.TrimSpace(fallback.Device), + MAP50: fallback.MAP50, + MAP5095: fallback.MAP5095, + PerformanceMode: runtimeOpts.PerformanceMode, + CPUCoreCount: runtimeOpts.CPUCoreCount, + CPUThreads: runtimeOpts.CPUThreads, + Workers: runtimeOpts.Workers, + YoloBatchSize: runtimeOpts.YoloBatchSize, + LowPriority: runtimeOpts.LowPriority, + AutoPauseEnabled: runtimeOpts.AutoPauseEnabled, + AutoPauseCPUPercent: runtimeOpts.AutoPauseCPUPercent, + AutoPauseTemperatureC: runtimeOpts.AutoPauseTemperatureC, } if info != nil { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 4b4a478..5c4c899 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1230,6 +1230,20 @@ export default function App() { } }, [authed]) + useEffect(() => { + try { + const appWindow = window as any + appWindow.__nsfwappTrainingRunning = trainingTabRunning + window.dispatchEvent( + new CustomEvent('app:training-running', { + detail: { running: trainingTabRunning }, + }) + ) + } catch { + // ignore + } + }, [trainingTabRunning]) + const loadPendingAutoStarts = useCallback(async (opts?: { force?: boolean }) => { const force = Boolean(opts?.force) const now = Date.now() diff --git a/frontend/src/components/ui/RecordJobActions.tsx b/frontend/src/components/ui/RecordJobActions.tsx index f0b97f3..a0a71c7 100644 --- a/frontend/src/components/ui/RecordJobActions.tsx +++ b/frontend/src/components/ui/RecordJobActions.tsx @@ -76,6 +76,7 @@ type Props = { onAddToDownloads?: ActionFn onSplit?: ActionFn onAddToTraining?: ActionFn + trainingImportDisabled?: boolean order?: ActionKey[] @@ -127,6 +128,7 @@ export default function RecordJobActions({ onAddToDownloads, onSplit, onAddToTraining, + trainingImportDisabled = false, order, className, }: Props) { @@ -205,6 +207,10 @@ export default function RecordJobActions({ const [addState, setAddState] = useState<'idle' | 'busy' | 'ok'>('idle') const addTimerRef = useRef(null) + const [appTrainingRunning, setAppTrainingRunning] = useState(() => { + if (typeof window === 'undefined') return false + return Boolean((window as any).__nsfwappTrainingRunning) + }) useEffect(() => { return () => { @@ -212,6 +218,19 @@ export default function RecordJobActions({ } }, []) + useEffect(() => { + const onTrainingRunning = (event: Event) => { + const detail = (event as CustomEvent<{ running?: boolean }>).detail + setAppTrainingRunning(Boolean(detail?.running)) + } + + window.addEventListener('app:training-running', onTrainingRunning as EventListener) + + return () => { + window.removeEventListener('app:training-running', onTrainingRunning as EventListener) + } + }, []) + const flashAddOk = () => { setAddState('ok') if (addTimerRef.current) window.clearTimeout(addTimerRef.current) @@ -249,8 +268,11 @@ export default function RecordJobActions({ } } + const trainingImportBlocked = trainingImportDisabled || appTrainingRunning + const doTrainingImport = async (): Promise => { if (busy) return false + if (trainingImportBlocked) return false const output = String(job.output || '').trim() if (!output) return false @@ -364,7 +386,7 @@ export default function RecordJobActions({ className={btnBase} title="Video ins Training übernehmen" aria-label="Video ins Training übernehmen" - disabled={busy || (!onAddToTraining && !job.output)} + disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)} onClick={async (e) => { e.preventDefault() e.stopPropagation() @@ -774,7 +796,7 @@ export default function RecordJobActions({ key="training" type="button" className="flex w-full items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-gray-100/70 dark:hover:bg-white/5 disabled:opacity-50" - disabled={busy || (!onAddToTraining && !job.output)} + disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)} onClick={async (e) => { e.preventDefault() e.stopPropagation() diff --git a/frontend/src/components/ui/Settings.tsx b/frontend/src/components/ui/Settings.tsx index 3fc50ef..7f9a137 100644 --- a/frontend/src/components/ui/Settings.tsx +++ b/frontend/src/components/ui/Settings.tsx @@ -55,6 +55,9 @@ type Settings = { trainingYoloBatchSize?: number trainingLowPriority?: boolean trainingVideoMAEEnabled?: boolean + trainingAutoPauseEnabled?: boolean + trainingAutoPauseCpuPercent?: number + trainingAutoPauseTemperatureC?: number trainingCpuCoreCount?: number trainingEffectiveMode?: TrainingPerformanceMode | string trainingEffectiveCpuThreads?: number @@ -99,6 +102,10 @@ type TrainingJobStatus = { progress?: number step?: string message?: string + paused?: boolean + pauseReason?: string + cpuPercent?: number + temperatureC?: number } type AppLog = { @@ -342,6 +349,9 @@ const DEFAULTS: Settings = { trainingYoloBatchSize: 0, trainingLowPriority: false, trainingVideoMAEEnabled: true, + trainingAutoPauseEnabled: false, + trainingAutoPauseCpuPercent: 95, + trainingAutoPauseTemperatureC: 85, } const TRAINING_PERFORMANCE_MODES: Array<{ @@ -1024,6 +1034,12 @@ export default function Settings({ onAssetsGenerated }: Props) { (data as any).trainingLowPriority ?? DEFAULTS.trainingLowPriority, trainingVideoMAEEnabled: (data as any).trainingVideoMAEEnabled ?? DEFAULTS.trainingVideoMAEEnabled, + trainingAutoPauseEnabled: + (data as any).trainingAutoPauseEnabled ?? DEFAULTS.trainingAutoPauseEnabled, + trainingAutoPauseCpuPercent: + (data as any).trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent, + trainingAutoPauseTemperatureC: + (data as any).trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC, trainingCpuCoreCount: (data as any).trainingCpuCoreCount ?? DEFAULTS.trainingCpuCoreCount, trainingEffectiveMode: @@ -1073,6 +1089,10 @@ export default function Settings({ onAssetsGenerated }: Props) { progress: Number(job.progress || 0), step: String(job.step || ''), message: String(job.message || ''), + paused: Boolean(job.paused), + pauseReason: String(job.pauseReason || ''), + cpuPercent: Number(job.cpuPercent || 0), + temperatureC: Number(job.temperatureC || 0), } : null) } catch { @@ -1611,6 +1631,15 @@ export default function Settings({ onAssetsGenerated }: Props) { ) const trainingLowPriority = !!value.trainingLowPriority const trainingVideoMAEEnabled = value.trainingVideoMAEEnabled !== false + const trainingAutoPauseEnabled = !!value.trainingAutoPauseEnabled + const trainingAutoPauseCpuPercent = Math.max( + 1, + Math.min(100, Math.floor(Number(value.trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent ?? 95))) + ) + const trainingAutoPauseTemperatureC = Math.max( + 1, + Math.min(120, Math.floor(Number(value.trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC ?? 85))) + ) setSaving(true) try { @@ -1656,6 +1685,9 @@ export default function Settings({ onAssetsGenerated }: Props) { trainingYoloBatchSize, trainingLowPriority, trainingVideoMAEEnabled, + trainingAutoPauseEnabled, + trainingAutoPauseCpuPercent, + trainingAutoPauseTemperatureC, }), }) if (!res.ok) { @@ -3089,6 +3121,100 @@ export default function Settings({ onAssetsGenerated }: Props) { description="Deaktivieren spart viel CPU-Zeit. YOLO Detector und Pose können weiterhin trainiert werden." /> + +
+ + setValue((v) => ({ + ...v, + trainingAutoPauseEnabled: checked, + })) + } + disabled={trainingSettingsLocked} + label="Training automatisch pausieren" + description="Pausiert laufende Python-Trainingsprozesse bei dauerhaft hoher CPU-Last oder hoher CPU-Temperatur und setzt sie nach einer kurzen Abkühlphase fort." + /> + +
+
+ +
+ + setValue((v) => ({ + ...v, + trainingAutoPauseCpuPercent: Number(e.target.value || 95), + })) + } + className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100" + /> + + % + +
+
+ +
+ +
+ + setValue((v) => ({ + ...v, + trainingAutoPauseTemperatureC: Number(e.target.value || 85), + })) + } + className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100" + /> + + °C + +
+
+
+ + {trainingJobStatus?.paused ? ( +
+ Training pausiert: {trainingJobStatus.pauseReason || 'Ressourcenlimit erreicht'} + {trainingJobStatus.cpuPercent ? ` · CPU ${Math.round(trainingJobStatus.cpuPercent)}%` : ''} + {trainingJobStatus.temperatureC ? ` · ${Math.round(trainingJobStatus.temperatureC)}°C` : ''} +
+ ) : ( +
+ Temperatur greift nur, wenn der Server passende Sensorwerte liefert. +
+ )} +
diff --git a/frontend/src/components/ui/TrainingTab.tsx b/frontend/src/components/ui/TrainingTab.tsx index 1e347dd..eb2be13 100644 --- a/frontend/src/components/ui/TrainingTab.tsx +++ b/frontend/src/components/ui/TrainingTab.tsx @@ -14,6 +14,7 @@ import { CheckIcon, ForwardIcon, InboxArrowDownIcon, + PauseIcon, RectangleGroupIcon, UserGroupIcon, TrashIcon, @@ -106,6 +107,10 @@ type TrainingJobStatus = { epoch?: number epochs?: number previewUrl?: string + paused?: boolean + pauseReason?: string + cpuPercent?: number + temperatureC?: number map50?: number map5095?: number } @@ -260,6 +265,9 @@ type TrainingHistoryEntry = { workers?: number yoloBatchSize?: number lowPriority?: boolean + autoPauseEnabled?: boolean + autoPauseCpuPercent?: number + autoPauseTemperatureC?: number } type TrainingStats = { @@ -327,6 +335,7 @@ type TrainingEstimateRuntime = { workers: number yoloBatchLabel: string lowPriority: boolean + autoPause: boolean factor: number yoloFactor: number } @@ -718,6 +727,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState | const lowPriority = Boolean( settings?.trainingEffectiveLowPriority ?? settings?.trainingLowPriority ) + const autoPause = Boolean(settings?.trainingAutoPauseEnabled) const autoThreads = cpuCores > 0 ? Math.max(1, Math.min(16, Math.round(cpuCores * 0.75))) : 4 @@ -769,11 +779,12 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState | ? 0.98 : 1 const lowPriorityFactor = lowPriority ? 1.12 : 1 + const autoPauseFactor = autoPause ? 1.12 : 1 const corePenalty = cpuCores > 0 && effectiveThreads > cpuCores ? 1 + Math.min(0.3, (effectiveThreads - cpuCores) * 0.04) : 1 - const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * corePenalty + const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * autoPauseFactor * corePenalty const yoloFactor = factor * batchFactor return { @@ -783,6 +794,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState | workers, yoloBatchLabel: rawBatch > 0 ? String(rawBatch) : 'Auto', lowPriority, + autoPause, factor: Number.isFinite(factor) && factor > 0 ? factor : 1, yoloFactor: Number.isFinite(yoloFactor) && yoloFactor > 0 ? yoloFactor : 1, } @@ -790,7 +802,8 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState | function trainingEstimateRuntimeText(runtime: TrainingEstimateRuntime) { const priority = runtime.lowPriority ? ' · niedrige Priorität' : '' - return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}` + const autoPause = runtime.autoPause ? ' · Auto-Pause' : '' + return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}${autoPause}` } function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) { @@ -802,6 +815,7 @@ function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) { trainingEffectiveWorkers: entry.workers, trainingEffectiveYoloBatchSize: entry.yoloBatchSize, trainingEffectiveLowPriority: entry.lowPriority, + trainingAutoPauseEnabled: entry.autoPauseEnabled, } as RecorderSettingsState) } @@ -4644,6 +4658,10 @@ export default function TrainingTab(props: { epoch: Number(job.epoch ?? 0), epochs: Number(job.epochs ?? 0), previewUrl: String(job.previewUrl ?? ''), + paused: Boolean(job.paused), + pauseReason: String(job.pauseReason ?? ''), + cpuPercent: Number(job.cpuPercent ?? 0), + temperatureC: Number(job.temperatureC ?? 0), map50: Number(job.map50 ?? 0), map5095: Number(job.map5095 ?? 0), } @@ -5125,6 +5143,10 @@ export default function TrainingTab(props: { const importVideoIntoTraining = useCallback(async (raw: any) => { const output = String(raw?.output || '').trim() if (!output) return false + if (trainingRunning) { + setMessage('Während das Training läuft, kann kein Video ins Training übernommen werden.') + return false + } setLoadingPreviewCandidate('') @@ -5277,6 +5299,7 @@ export default function TrainingTab(props: { loadPriorityTrainingSamples, loadTrainingStatus, setLoadingPreviewCandidate, + trainingRunning, waitForVideoImportResult, ]) @@ -7243,13 +7266,25 @@ export default function TrainingTab(props: { }, ] + const trainingPaused = Boolean(trainingStatus?.training?.paused) + const pauseReason = String(trainingStatus?.training?.pauseReason || '').trim() + const pauseMetrics = [ + trainingStatus?.training?.cpuPercent + ? `CPU ${Math.round(Number(trainingStatus.training.cpuPercent))}%` + : '', + trainingStatus?.training?.temperatureC + ? `${Math.round(Number(trainingStatus.training.temperatureC))}°C` + : '', + ].filter(Boolean).join(' · ') const progress = clampPercent( trainingRunning ? shownTrainingProgress : readinessProgress * 100 ) const statusText = trainingRunning - ? shownTrainingStep || 'Training läuft…' + ? trainingPaused + ? `Pausiert${pauseReason ? `: ${pauseReason}` : ''}${pauseMetrics ? ` · ${pauseMetrics}` : ''}` + : shownTrainingStep || 'Training läuft…' : !feedbackReady ? `${Math.max(0, requiredCount - feedbackCount)} Feedback fehlen noch` : canStartTraining @@ -7280,14 +7315,18 @@ export default function TrainingTab(props: {
- {trainingRunning ? ( + {trainingPaused ? ( +