chris/nsfwapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added temperature pause

Browse Source
This commit is contained in:
Linrador 2026-06-26 22:06:05 +02:00
parent 5e6c841b26
commit 6b6528acca
9 changed files with 771 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
backend/dist/nsfwapp.exe vendored
View File

Binary file not shown.

85
backend/performance.go
View File

@ -13,11 +13,13 @@ import (
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
gocpu "github.com/shirou/gopsutil/v3/cpu"
godisk "github.com/shirou/gopsutil/v3/disk"
gohost "github.com/shirou/gopsutil/v3/host"
)
// Server-Startzeit für Perf/Uptime
@ -25,6 +27,13 @@ var serverStartedAt = time.Now()
var lastCPUUsageBits uint64 // atomic float64 bits
var cpuTemperatureCache = struct {
sync.Mutex
value float64
available bool
sampledAt time.Time
}{}
func setLastCPUUsage(v float64) {
atomic.StoreUint64(&lastCPUUsageBits, math.Float64bits(v))
}
@ -33,6 +42,71 @@ func getLastCPUUsage() float64 {
return math.Float64frombits(atomic.LoadUint64(&lastCPUUsageBits))
}
func getMaxCPUTemperatureC() (float64, bool) {
now := time.Now()
cpuTemperatureCache.Lock()
if now.Sub(cpuTemperatureCache.sampledAt) < 15*time.Second {
value := cpuTemperatureCache.value
available := cpuTemperatureCache.available
cpuTemperatureCache.Unlock()
return value, available
}
cpuTemperatureCache.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
temps, err := gohost.SensorsTemperaturesWithContext(ctx)
if err != nil || len(temps) == 0 {
cpuTemperatureCache.Lock()
cpuTemperatureCache.value = 0
cpuTemperatureCache.available = false
cpuTemperatureCache.sampledAt = now
cpuTemperatureCache.Unlock()
return 0, false
}
maxCPU := 0.0
maxAny := 0.0
for _, temp := range temps {
v := temp.Temperature
if math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 || v > 150 {
continue
}
if v > maxAny {
maxAny = v
}
key := strings.ToLower(strings.TrimSpace(temp.SensorKey))
if strings.Contains(key, "cpu") ||
strings.Contains(key, "core") ||
strings.Contains(key, "package") ||
strings.Contains(key, "tdie") ||
strings.Contains(key, "tctl") ||
strings.Contains(key, "k10temp") {
if v > maxCPU {
maxCPU = v
}
}
}
value := maxCPU
if value <= 0 {
value = maxAny
}
available := value > 0
cpuTemperatureCache.Lock()
cpuTemperatureCache.value = value
cpuTemperatureCache.available = available
cpuTemperatureCache.sampledAt = now
cpuTemperatureCache.Unlock()
return value, available
}
func startCPUUsageSampler(ctx context.Context) {
_, _ = gocpu.Percent(200*time.Millisecond, false)
@ -194,6 +268,17 @@ func buildPerfSnapshot() map[string]any {
}
return v
}(),
"cpuTemperatureC": func() float64 {
v, ok := getMaxCPUTemperatureC()
if !ok {
return 0
}
return math.Round(v*10) / 10
}(),
"cpuTemperatureAvailable": func() bool {
_, ok := getMaxCPUTemperatureC()
return ok
}(),
"diskPath": diskPath,
"diskFreeBytes": diskFreeBytes,
"diskTotalBytes": diskTotalBytes,

65
backend/settings.go
View File

@ -56,15 +56,18 @@ type RecorderSettings struct {
GenerateAssetsAnalyze bool `json:"generateAssetsAnalyze"`
EnrichPostworkEnabled bool `json:"enrichPostworkEnabled"`
TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"`
TrainingDetectorEpochs int `json:"trainingDetectorEpochs"`
TrainingPerformanceMode string `json:"trainingPerformanceMode"`
TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"`
TrainingCPUThreads int `json:"trainingCpuThreads"`
TrainingWorkers int `json:"trainingWorkers"`
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
TrainingLowPriority bool `json:"trainingLowPriority"`
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"`
TrainingDetectorEpochs int `json:"trainingDetectorEpochs"`
TrainingPerformanceMode string `json:"trainingPerformanceMode"`
TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"`
TrainingCPUThreads int `json:"trainingCpuThreads"`
TrainingWorkers int `json:"trainingWorkers"`
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
TrainingLowPriority bool `json:"trainingLowPriority"`
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"`
TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"`
TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"`
// EncryptedCookies contains base64(nonce+ciphertext) of a JSON cookie map.
EncryptedCookies string `json:"encryptedCookies"`
@ -112,15 +115,18 @@ var (
GenerateAssetsAnalyze: false,
EnrichPostworkEnabled: true,
TrainingRecognitionEnabled: true,
TrainingDetectorEpochs: 60,
TrainingPerformanceMode: trainingPerformanceAuto,
TrainingPowerSaveMode: false,
TrainingCPUThreads: 0,
TrainingWorkers: 2,
TrainingYoloBatchSize: 0,
TrainingLowPriority: false,
TrainingVideoMAEEnabled: true,
TrainingRecognitionEnabled: true,
TrainingDetectorEpochs: 60,
TrainingPerformanceMode: trainingPerformanceAuto,
TrainingPowerSaveMode: false,
TrainingCPUThreads: 0,
TrainingWorkers: 2,
TrainingYoloBatchSize: 0,
TrainingLowPriority: false,
TrainingVideoMAEEnabled: true,
TrainingAutoPauseEnabled: false,
TrainingAutoPauseCPUPercent: 95,
TrainingAutoPauseTemperatureC: 85,
EncryptedCookies: "",
}
@ -189,6 +195,20 @@ func normalizeTrainingSettings(s *RecorderSettings) {
if s.TrainingYoloBatchSize > 64 {
s.TrainingYoloBatchSize = 64
}
if s.TrainingAutoPauseCPUPercent < 1 {
s.TrainingAutoPauseCPUPercent = 95
}
if s.TrainingAutoPauseCPUPercent > 100 {
s.TrainingAutoPauseCPUPercent = 100
}
if s.TrainingAutoPauseTemperatureC < 1 {
s.TrainingAutoPauseTemperatureC = 85
}
if s.TrainingAutoPauseTemperatureC > 120 {
s.TrainingAutoPauseTemperatureC = 120
}
}
func settingsFilePath() string {
@ -375,6 +395,9 @@ type RecorderSettingsPublic struct {
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
TrainingLowPriority bool `json:"trainingLowPriority"`
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"`
TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"`
TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"`
TrainingCPUCoreCount int `json:"trainingCpuCoreCount"`
TrainingEffectiveMode string `json:"trainingEffectiveMode"`
TrainingEffectiveCPUThreads int `json:"trainingEffectiveCpuThreads"`
@ -436,6 +459,9 @@ func toPublicSettings(s RecorderSettings) RecorderSettingsPublic {
TrainingYoloBatchSize: s.TrainingYoloBatchSize,
TrainingLowPriority: s.TrainingLowPriority,
TrainingVideoMAEEnabled: s.TrainingVideoMAEEnabled,
TrainingAutoPauseEnabled: s.TrainingAutoPauseEnabled,
TrainingAutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent,
TrainingAutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC,
TrainingCPUCoreCount: trainingOpts.CPUCoreCount,
TrainingEffectiveMode: trainingOpts.PerformanceMode,
TrainingEffectiveCPUThreads: trainingOpts.CPUThreads,
@ -631,6 +657,9 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
next.TrainingYoloBatchSize = in.TrainingYoloBatchSize
next.TrainingLowPriority = in.TrainingLowPriority
next.TrainingVideoMAEEnabled = in.TrainingVideoMAEEnabled
next.TrainingAutoPauseEnabled = in.TrainingAutoPauseEnabled
next.TrainingAutoPauseCPUPercent = in.TrainingAutoPauseCPUPercent
next.TrainingAutoPauseTemperatureC = in.TrainingAutoPauseTemperatureC
dbChanged :=
normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) ||

487
backend/training.go
View File

@ -24,6 +24,8 @@ import (
"strings"
"sync"
"time"
goprocess "github.com/shirou/gopsutil/v3/process"
)
const trainingUncertainCandidateCount = 10
@ -262,15 +264,19 @@ type TrainingPosePrediction struct {
}
type TrainingJobStatus struct {
Running bool `json:"running"`
Progress int `json:"progress"`
Step string `json:"step"`
Message string `json:"message,omitempty"`
Error string `json:"error,omitempty"`
StartedAt string `json:"startedAt,omitempty"`
FinishedAt string `json:"finishedAt,omitempty"`
DurationMs int64 `json:"durationMs,omitempty"`
PreviewURL string `json:"previewUrl,omitempty"`
Running bool `json:"running"`
Progress int `json:"progress"`
Step string `json:"step"`
Message string `json:"message,omitempty"`
Error string `json:"error,omitempty"`
StartedAt string `json:"startedAt,omitempty"`
FinishedAt string `json:"finishedAt,omitempty"`
DurationMs int64 `json:"durationMs,omitempty"`
PreviewURL string `json:"previewUrl,omitempty"`
Paused bool `json:"paused,omitempty"`
PauseReason string `json:"pauseReason,omitempty"`
CPUPercent float64 `json:"cpuPercent,omitempty"`
TemperatureC float64 `json:"temperatureC,omitempty"`
Stage string `json:"stage,omitempty"`
Epoch int `json:"epoch,omitempty"`
@ -1181,6 +1187,8 @@ func trainingRunCommandStreaming(
return "", err
}
applyTrainingLowPriorityAfterStart(cmd, opts.LowPriority)
monitorCtx, stopResourcePauseMonitor := context.WithCancel(ctx)
resumeResourcePause := startTrainingResourcePauseMonitor(monitorCtx, cmd, opts)
var outMu sync.Mutex
var lines []string
@ -1231,9 +1239,12 @@ func trainingRunCommandStreaming(
select {
case err = <-waitCh:
case <-ctx.Done():
resumeResourcePause()
terminateTrainingCommand(cmd)
err = <-waitCh
}
stopResourcePauseMonitor()
resumeResourcePause()
wg.Wait()
outMu.Lock()
@ -1274,14 +1285,17 @@ const (
)
type trainingRuntimeOptions struct {
PerformanceMode string
PowerSaveMode bool
CPUCoreCount int
CPUThreads int
Workers int
YoloBatchSize int
LowPriority bool
VideoMAEEnabled bool
PerformanceMode string
PowerSaveMode bool
CPUCoreCount int
CPUThreads int
Workers int
YoloBatchSize int
LowPriority bool
VideoMAEEnabled bool
AutoPauseEnabled bool
AutoPauseCPUPercent int
AutoPauseTemperatureC int
}
func normalizeTrainingPerformanceMode(raw string) string {
@ -1381,14 +1395,17 @@ func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRunt
presetThreads, presetWorkers, presetBatch, presetLowPriority := trainingPresetValues(mode, cpuCores)
opts := trainingRuntimeOptions{
PerformanceMode: mode,
PowerSaveMode: mode == trainingPerformanceEco,
CPUCoreCount: cpuCores,
CPUThreads: presetThreads,
Workers: presetWorkers,
YoloBatchSize: presetBatch,
LowPriority: presetLowPriority,
VideoMAEEnabled: s.TrainingVideoMAEEnabled,
PerformanceMode: mode,
PowerSaveMode: mode == trainingPerformanceEco,
CPUCoreCount: cpuCores,
CPUThreads: presetThreads,
Workers: presetWorkers,
YoloBatchSize: presetBatch,
LowPriority: presetLowPriority,
VideoMAEEnabled: s.TrainingVideoMAEEnabled,
AutoPauseEnabled: s.TrainingAutoPauseEnabled,
AutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent,
AutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC,
}
if mode == trainingPerformanceCustom {
@ -1448,6 +1465,331 @@ func trainingCommandEnv(opts trainingRuntimeOptions) []string {
return env
}
type trainingResourceSnapshot struct {
CPUPercent float64
TemperatureC float64
TemperatureAvailable bool
}
func trainingRoundMetric(v float64) float64 {
if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 {
return 0
}
return math.Round(v*10) / 10
}
func trainingReadResourceSnapshot() trainingResourceSnapshot {
temp, tempOK := getMaxCPUTemperatureC()
return trainingResourceSnapshot{
CPUPercent: trainingRoundMetric(getLastCPUUsage()),
TemperatureC: trainingRoundMetric(temp),
TemperatureAvailable: tempOK,
}
}
func trainingPauseReasonForResources(opts trainingRuntimeOptions, snap trainingResourceSnapshot) (string, bool) {
if !opts.AutoPauseEnabled {
return "", false
}
reasons := []string{}
if opts.AutoPauseCPUPercent > 0 && snap.CPUPercent >= float64(opts.AutoPauseCPUPercent) {
reasons = append(
reasons,
fmt.Sprintf("CPU %.0f%% >= %d%%", snap.CPUPercent, opts.AutoPauseCPUPercent),
)
}
if opts.AutoPauseTemperatureC > 0 &&
snap.TemperatureAvailable &&
snap.TemperatureC >= float64(opts.AutoPauseTemperatureC) {
reasons = append(
reasons,
fmt.Sprintf("Temperatur %.0f°C >= %d°C", snap.TemperatureC, opts.AutoPauseTemperatureC),
)
}
if len(reasons) == 0 {
return "", false
}
return strings.Join(reasons, ", "), true
}
func trainingResourcesRecovered(opts trainingRuntimeOptions, snap trainingResourceSnapshot) bool {
if !opts.AutoPauseEnabled {
return true
}
if opts.AutoPauseCPUPercent > 0 {
resumeCPU := float64(opts.AutoPauseCPUPercent - 15)
if resumeCPU < 50 {
resumeCPU = 50
}
if snap.CPUPercent > resumeCPU {
return false
}
}
if opts.AutoPauseTemperatureC > 0 && snap.TemperatureAvailable {
resumeTemp := float64(opts.AutoPauseTemperatureC - 5)
if resumeTemp < 40 {
resumeTemp = 40
}
if snap.TemperatureC > resumeTemp {
return false
}
}
return true
}
func trainingProcessTree(pid int32) []*goprocess.Process {
if pid <= 0 {
return nil
}
root, err := goprocess.NewProcess(pid)
if err != nil || root == nil {
return nil
}
seen := map[int32]bool{}
var out []*goprocess.Process
var walk func(*goprocess.Process)
walk = func(p *goprocess.Process) {
if p == nil || seen[p.Pid] {
return
}
seen[p.Pid] = true
out = append(out, p)
children, err := p.Children()
if err != nil {
return
}
for _, child := range children {
walk(child)
}
}
walk(root)
return out
}
func trainingSuspendProcessTree(cmd *exec.Cmd) error {
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
return nil
}
procs := trainingProcessTree(int32(cmd.Process.Pid))
if len(procs) == 0 {
return nil
}
var firstErr error
success := 0
for i := len(procs) - 1; i >= 0; i-- {
if err := procs[i].Suspend(); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
success++
}
if success > 0 {
return nil
}
return firstErr
}
func trainingResumeProcessTree(cmd *exec.Cmd) error {
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
return nil
}
procs := trainingProcessTree(int32(cmd.Process.Pid))
if len(procs) == 0 {
return nil
}
var firstErr error
success := 0
for _, proc := range procs {
if err := proc.Resume(); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
success++
}
if success > 0 {
return nil
}
return firstErr
}
func startTrainingResourcePauseMonitor(ctx context.Context, cmd *exec.Cmd, opts trainingRuntimeOptions) func() {
if !opts.AutoPauseEnabled || cmd == nil || cmd.Process == nil {
return func() {}
}
const (
checkInterval = 5 * time.Second
requiredHighSamples = 2
requiredCoolSamples = 2
minPauseDuration = 45 * time.Second
)
var pauseMu sync.Mutex
paused := false
previousStep := ""
highSamples := 0
coolSamples := 0
pauseStartedAt := time.Time{}
loggedSuspendError := false
resumeNow := func(reason string) {
pauseMu.Lock()
if !paused {
pauseMu.Unlock()
return
}
err := trainingResumeProcessTree(cmd)
paused = false
restoreStep := strings.TrimSpace(previousStep)
previousStep = ""
pauseMu.Unlock()
if err != nil {
appLogln("⚠️ Training konnte nach Ressourcenpause nicht sauber fortgesetzt werden:", err)
}
if strings.TrimSpace(reason) != "" {
appLogln(reason)
}
trainingSetJobStatus(func(s *TrainingJobStatus) {
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
s.Message = ""
if restoreStep != "" && s.Running {
s.Step = restoreStep
}
})
}
go func() {
ticker := time.NewTicker(checkInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
resumeNow("Training-Ressourcenpause wurde beendet.")
return
case <-ticker.C:
}
snap := trainingReadResourceSnapshot()
pauseMu.Lock()
isPaused := paused
pauseAge := time.Since(pauseStartedAt)
pauseMu.Unlock()
if !isPaused {
reason, shouldPause := trainingPauseReasonForResources(opts, snap)
if !shouldPause {
highSamples = 0
continue
}
highSamples++
if highSamples < requiredHighSamples {
continue
}
pauseMu.Lock()
if paused {
pauseMu.Unlock()
continue
}
currentStatus := trainingGetJobStatus()
previousStep = strings.TrimSpace(currentStatus.Step)
if previousStep == "" {
previousStep = "Training läuft..."
}
if err := trainingSuspendProcessTree(cmd); err != nil {
pauseMu.Unlock()
highSamples = 0
if !loggedSuspendError {
loggedSuspendError = true
appLogln("⚠️ Training-Ressourcenpause konnte Prozess nicht pausieren:", err)
}
continue
}
paused = true
pauseStartedAt = time.Now()
coolSamples = 0
pauseMu.Unlock()
appLogln("⏸ Training pausiert wegen Ressourcenlimit:", reason)
trainingSetJobStatus(func(s *TrainingJobStatus) {
s.Paused = true
s.PauseReason = reason
s.CPUPercent = snap.CPUPercent
if snap.TemperatureAvailable {
s.TemperatureC = snap.TemperatureC
} else {
s.TemperatureC = 0
}
s.Step = "Training pausiert: " + reason
s.Message = "Training wird automatisch fortgesetzt, sobald CPU/Temperatur wieder im grünen Bereich sind."
})
continue
}
trainingSetJobStatus(func(s *TrainingJobStatus) {
if !s.Running || !s.Paused {
return
}
s.CPUPercent = snap.CPUPercent
if snap.TemperatureAvailable {
s.TemperatureC = snap.TemperatureC
}
})
if pauseAge < minPauseDuration || !trainingResourcesRecovered(opts, snap) {
coolSamples = 0
continue
}
coolSamples++
if coolSamples < requiredCoolSamples {
continue
}
highSamples = 0
coolSamples = 0
resumeNow("▶ Training nach Ressourcenpause fortgesetzt.")
}
}()
return func() {
resumeNow("")
}
}
var trainingJob = struct {
mu sync.Mutex
status TrainingJobStatus
@ -1526,6 +1868,10 @@ func trainingFinishCancelled(root string) {
s.FinishedAt = finishedAt.Format(time.RFC3339)
s.DurationMs = durationMs
s.PreviewURL = ""
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
if cleanupErr != nil {
s.Message = "Training wurde abgebrochen, aber temporäre Trainingsausgaben konnten nicht vollständig gelöscht werden."
@ -3641,6 +3987,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
s.FinishedAt = finishedAt.Format(time.RFC3339)
s.DurationMs = durationMs
s.PreviewURL = ""
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
})
trainingClearJobCancel()
return
@ -3650,7 +4000,7 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
appLogln("ML-Python für Training:", python)
runtimeOpts := trainingRuntimeOptionsFromSettings()
appLogf(
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v targets=%s",
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v autoPause=%v cpuLimit=%d tempLimit=%d targets=%s",
runtimeOpts.PerformanceMode,
runtimeOpts.CPUCoreCount,
runtimeOpts.PowerSaveMode,
@ -3659,6 +4009,9 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
runtimeOpts.YoloBatchSize,
runtimeOpts.LowPriority,
runtimeOpts.VideoMAEEnabled,
runtimeOpts.AutoPauseEnabled,
runtimeOpts.AutoPauseCPUPercent,
runtimeOpts.AutoPauseTemperatureC,
strings.Join(targets.list(), ","),
)
@ -4209,6 +4562,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
s.FinishedAt = finishedAt.Format(time.RFC3339)
s.DurationMs = durationMs
s.PreviewURL = ""
s.Paused = false
s.PauseReason = ""
s.CPUPercent = 0
s.TemperatureC = 0
})
trainingClearJobCancel()
@ -4765,24 +5122,27 @@ func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo {
}
type TrainingHistoryEntry struct {
TrainedAt string `json:"trainedAt,omitempty"`
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
Target string `json:"target,omitempty"`
Status string `json:"status,omitempty"`
DurationMs int64 `json:"durationMs,omitempty"`
Epochs int `json:"epochs,omitempty"`
TrainSamples int `json:"trainSamples,omitempty"`
ValSamples int `json:"valSamples,omitempty"`
Imgsz int `json:"imgsz,omitempty"`
Device string `json:"device,omitempty"`
MAP50 float64 `json:"map50,omitempty"`
MAP5095 float64 `json:"map5095,omitempty"`
PerformanceMode string `json:"performanceMode,omitempty"`
CPUCoreCount int `json:"cpuCoreCount,omitempty"`
CPUThreads int `json:"cpuThreads,omitempty"`
Workers int `json:"workers,omitempty"`
YoloBatchSize int `json:"yoloBatchSize,omitempty"`
LowPriority bool `json:"lowPriority,omitempty"`
TrainedAt string `json:"trainedAt,omitempty"`
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
Target string `json:"target,omitempty"`
Status string `json:"status,omitempty"`
DurationMs int64 `json:"durationMs,omitempty"`
Epochs int `json:"epochs,omitempty"`
TrainSamples int `json:"trainSamples,omitempty"`
ValSamples int `json:"valSamples,omitempty"`
Imgsz int `json:"imgsz,omitempty"`
Device string `json:"device,omitempty"`
MAP50 float64 `json:"map50,omitempty"`
MAP5095 float64 `json:"map5095,omitempty"`
PerformanceMode string `json:"performanceMode,omitempty"`
CPUCoreCount int `json:"cpuCoreCount,omitempty"`
CPUThreads int `json:"cpuThreads,omitempty"`
Workers int `json:"workers,omitempty"`
YoloBatchSize int `json:"yoloBatchSize,omitempty"`
LowPriority bool `json:"lowPriority,omitempty"`
AutoPauseEnabled bool `json:"autoPauseEnabled,omitempty"`
AutoPauseCPUPercent int `json:"autoPauseCpuPercent,omitempty"`
AutoPauseTemperatureC int `json:"autoPauseTemperatureC,omitempty"`
}
type TrainingHistoryResponse struct {
@ -4813,24 +5173,27 @@ func trainingAppendTargetHistory(root string, target string, status string, dura
now := time.Now().UTC()
entry := TrainingHistoryEntry{
TrainedAt: now.Format(time.RFC3339),
TrainedAtMs: now.UnixMilli(),
Target: kind,
Status: strings.TrimSpace(status),
DurationMs: durationMs,
Epochs: fallback.Epochs,
TrainSamples: fallback.TrainSamples,
ValSamples: fallback.ValSamples,
Imgsz: fallback.Imgsz,
Device: strings.TrimSpace(fallback.Device),
MAP50: fallback.MAP50,
MAP5095: fallback.MAP5095,
PerformanceMode: runtimeOpts.PerformanceMode,
CPUCoreCount: runtimeOpts.CPUCoreCount,
CPUThreads: runtimeOpts.CPUThreads,
Workers: runtimeOpts.Workers,
YoloBatchSize: runtimeOpts.YoloBatchSize,
LowPriority: runtimeOpts.LowPriority,
TrainedAt: now.Format(time.RFC3339),
TrainedAtMs: now.UnixMilli(),
Target: kind,
Status: strings.TrimSpace(status),
DurationMs: durationMs,
Epochs: fallback.Epochs,
TrainSamples: fallback.TrainSamples,
ValSamples: fallback.ValSamples,
Imgsz: fallback.Imgsz,
Device: strings.TrimSpace(fallback.Device),
MAP50: fallback.MAP50,
MAP5095: fallback.MAP5095,
PerformanceMode: runtimeOpts.PerformanceMode,
CPUCoreCount: runtimeOpts.CPUCoreCount,
CPUThreads: runtimeOpts.CPUThreads,
Workers: runtimeOpts.Workers,
YoloBatchSize: runtimeOpts.YoloBatchSize,
LowPriority: runtimeOpts.LowPriority,
AutoPauseEnabled: runtimeOpts.AutoPauseEnabled,
AutoPauseCPUPercent: runtimeOpts.AutoPauseCPUPercent,
AutoPauseTemperatureC: runtimeOpts.AutoPauseTemperatureC,
}
if info != nil {

14
frontend/src/App.tsx
View File

@ -1230,6 +1230,20 @@ export default function App() {
}
}, [authed])
useEffect(() => {
try {
const appWindow = window as any
appWindow.__nsfwappTrainingRunning = trainingTabRunning
window.dispatchEvent(
new CustomEvent('app:training-running', {
detail: { running: trainingTabRunning },
})
)
} catch {
// ignore
}
}, [trainingTabRunning])
const loadPendingAutoStarts = useCallback(async (opts?: { force?: boolean }) => {
const force = Boolean(opts?.force)
const now = Date.now()

26
frontend/src/components/ui/RecordJobActions.tsx
View File

@ -76,6 +76,7 @@ type Props = {
onAddToDownloads?: ActionFn
onSplit?: ActionFn
onAddToTraining?: ActionFn
trainingImportDisabled?: boolean
order?: ActionKey[]
@ -127,6 +128,7 @@ export default function RecordJobActions({
onAddToDownloads,
onSplit,
onAddToTraining,
trainingImportDisabled = false,
order,
className,
}: Props) {
@ -205,6 +207,10 @@ export default function RecordJobActions({
const [addState, setAddState] = useState<'idle' | 'busy' | 'ok'>('idle')
const addTimerRef = useRef<number | null>(null)
const [appTrainingRunning, setAppTrainingRunning] = useState(() => {
if (typeof window === 'undefined') return false
return Boolean((window as any).__nsfwappTrainingRunning)
})
useEffect(() => {
return () => {
@ -212,6 +218,19 @@ export default function RecordJobActions({
}
}, [])
useEffect(() => {
const onTrainingRunning = (event: Event) => {
const detail = (event as CustomEvent<{ running?: boolean }>).detail
setAppTrainingRunning(Boolean(detail?.running))
}
window.addEventListener('app:training-running', onTrainingRunning as EventListener)
return () => {
window.removeEventListener('app:training-running', onTrainingRunning as EventListener)
}
}, [])
const flashAddOk = () => {
setAddState('ok')
if (addTimerRef.current) window.clearTimeout(addTimerRef.current)
@ -249,8 +268,11 @@ export default function RecordJobActions({
}
}
const trainingImportBlocked = trainingImportDisabled || appTrainingRunning
const doTrainingImport = async (): Promise<boolean> => {
if (busy) return false
if (trainingImportBlocked) return false
const output = String(job.output || '').trim()
if (!output) return false
@ -364,7 +386,7 @@ export default function RecordJobActions({
className={btnBase}
title="Video ins Training übernehmen"
aria-label="Video ins Training übernehmen"
disabled={busy || (!onAddToTraining && !job.output)}
disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)}
onClick={async (e) => {
e.preventDefault()
e.stopPropagation()
@ -774,7 +796,7 @@ export default function RecordJobActions({
key="training"
type="button"
className="flex w-full items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-gray-100/70 dark:hover:bg-white/5 disabled:opacity-50"
disabled={busy || (!onAddToTraining && !job.output)}
disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)}
onClick={async (e) => {
e.preventDefault()
e.stopPropagation()

126
frontend/src/components/ui/Settings.tsx
View File

@ -55,6 +55,9 @@ type Settings = {
trainingYoloBatchSize?: number
trainingLowPriority?: boolean
trainingVideoMAEEnabled?: boolean
trainingAutoPauseEnabled?: boolean
trainingAutoPauseCpuPercent?: number
trainingAutoPauseTemperatureC?: number
trainingCpuCoreCount?: number
trainingEffectiveMode?: TrainingPerformanceMode | string
trainingEffectiveCpuThreads?: number
@ -99,6 +102,10 @@ type TrainingJobStatus = {
progress?: number
step?: string
message?: string
paused?: boolean
pauseReason?: string
cpuPercent?: number
temperatureC?: number
}
type AppLog = {
@ -342,6 +349,9 @@ const DEFAULTS: Settings = {
trainingYoloBatchSize: 0,
trainingLowPriority: false,
trainingVideoMAEEnabled: true,
trainingAutoPauseEnabled: false,
trainingAutoPauseCpuPercent: 95,
trainingAutoPauseTemperatureC: 85,
}
const TRAINING_PERFORMANCE_MODES: Array<{
@ -1024,6 +1034,12 @@ export default function Settings({ onAssetsGenerated }: Props) {
(data as any).trainingLowPriority ?? DEFAULTS.trainingLowPriority,
trainingVideoMAEEnabled:
(data as any).trainingVideoMAEEnabled ?? DEFAULTS.trainingVideoMAEEnabled,
trainingAutoPauseEnabled:
(data as any).trainingAutoPauseEnabled ?? DEFAULTS.trainingAutoPauseEnabled,
trainingAutoPauseCpuPercent:
(data as any).trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent,
trainingAutoPauseTemperatureC:
(data as any).trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC,
trainingCpuCoreCount:
(data as any).trainingCpuCoreCount ?? DEFAULTS.trainingCpuCoreCount,
trainingEffectiveMode:
@ -1073,6 +1089,10 @@ export default function Settings({ onAssetsGenerated }: Props) {
progress: Number(job.progress || 0),
step: String(job.step || ''),
message: String(job.message || ''),
paused: Boolean(job.paused),
pauseReason: String(job.pauseReason || ''),
cpuPercent: Number(job.cpuPercent || 0),
temperatureC: Number(job.temperatureC || 0),
}
: null)
} catch {
@ -1611,6 +1631,15 @@ export default function Settings({ onAssetsGenerated }: Props) {
)
const trainingLowPriority = !!value.trainingLowPriority
const trainingVideoMAEEnabled = value.trainingVideoMAEEnabled !== false
const trainingAutoPauseEnabled = !!value.trainingAutoPauseEnabled
const trainingAutoPauseCpuPercent = Math.max(
1,
Math.min(100, Math.floor(Number(value.trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent ?? 95)))
)
const trainingAutoPauseTemperatureC = Math.max(
1,
Math.min(120, Math.floor(Number(value.trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC ?? 85)))
)
setSaving(true)
try {
@ -1656,6 +1685,9 @@ export default function Settings({ onAssetsGenerated }: Props) {
trainingYoloBatchSize,
trainingLowPriority,
trainingVideoMAEEnabled,
trainingAutoPauseEnabled,
trainingAutoPauseCpuPercent,
trainingAutoPauseTemperatureC,
}),
})
if (!res.ok) {
@ -3089,6 +3121,100 @@ export default function Settings({ onAssetsGenerated }: Props) {
description="Deaktivieren spart viel CPU-Zeit. YOLO Detector und Pose können weiterhin trainiert werden."
/>
</div>
<div className="mt-3 rounded-xl border border-gray-200 bg-white p-3 dark:border-white/10 dark:bg-gray-950/40">
<LabeledSwitch
checked={!!value.trainingAutoPauseEnabled}
onChange={(checked) =>
setValue((v) => ({
...v,
trainingAutoPauseEnabled: checked,
}))
}
disabled={trainingSettingsLocked}
label="Training automatisch pausieren"
description="Pausiert laufende Python-Trainingsprozesse bei dauerhaft hoher CPU-Last oder hoher CPU-Temperatur und setzt sie nach einer kurzen Abkühlphase fort."
/>
<div
className={[
'mt-3 grid grid-cols-1 gap-3 sm:grid-cols-2',
value.trainingAutoPauseEnabled && !trainingSettingsLocked ? '' : 'opacity-60',
].join(' ')}
>
<div>
<label
htmlFor="trainingAutoPauseCpuPercent"
className="text-sm font-medium text-gray-900 dark:text-gray-200"
>
CPU-Grenze
</label>
<div className="mt-1 flex items-center gap-2">
<input
id="trainingAutoPauseCpuPercent"
type="number"
min={1}
max={100}
step={1}
disabled={!value.trainingAutoPauseEnabled || trainingSettingsLocked}
value={value.trainingAutoPauseCpuPercent ?? 95}
onChange={(e) =>
setValue((v) => ({
...v,
trainingAutoPauseCpuPercent: Number(e.target.value || 95),
}))
}
className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100"
/>
<span className="shrink-0 text-xs text-gray-600 dark:text-gray-300">
%
</span>
</div>
</div>
<div>
<label
htmlFor="trainingAutoPauseTemperatureC"
className="text-sm font-medium text-gray-900 dark:text-gray-200"
>
Temperatur-Grenze
</label>
<div className="mt-1 flex items-center gap-2">
<input
id="trainingAutoPauseTemperatureC"
type="number"
min={1}
max={120}
step={1}
disabled={!value.trainingAutoPauseEnabled || trainingSettingsLocked}
value={value.trainingAutoPauseTemperatureC ?? 85}
onChange={(e) =>
setValue((v) => ({
...v,
trainingAutoPauseTemperatureC: Number(e.target.value || 85),
}))
}
className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100"
/>
<span className="shrink-0 text-xs text-gray-600 dark:text-gray-300">
°C
</span>
</div>
</div>
</div>
{trainingJobStatus?.paused ? (
<div className="mt-3 rounded-lg border border-amber-200 bg-amber-50 px-3 py-2 text-xs font-medium text-amber-900 dark:border-amber-400/30 dark:bg-amber-500/10 dark:text-amber-100">
Training pausiert: {trainingJobStatus.pauseReason || 'Ressourcenlimit erreicht'}
{trainingJobStatus.cpuPercent ? ` · CPU ${Math.round(trainingJobStatus.cpuPercent)}%` : ''}
{trainingJobStatus.temperatureC ? ` · ${Math.round(trainingJobStatus.temperatureC)}°C` : ''}
</div>
) : (
<div className="mt-2 text-xs text-gray-500 dark:text-gray-400">
Temperatur greift nur, wenn der Server passende Sensorwerte liefert.
</div>
)}
</div>
</div>
</div>

53
frontend/src/components/ui/TrainingTab.tsx
View File

@ -14,6 +14,7 @@ import {
CheckIcon,
ForwardIcon,
InboxArrowDownIcon,
PauseIcon,
RectangleGroupIcon,
UserGroupIcon,
TrashIcon,
@ -106,6 +107,10 @@ type TrainingJobStatus = {
epoch?: number
epochs?: number
previewUrl?: string
paused?: boolean
pauseReason?: string
cpuPercent?: number
temperatureC?: number
map50?: number
map5095?: number
}
@ -260,6 +265,9 @@ type TrainingHistoryEntry = {
workers?: number
yoloBatchSize?: number
lowPriority?: boolean
autoPauseEnabled?: boolean
autoPauseCpuPercent?: number
autoPauseTemperatureC?: number
}
type TrainingStats = {
@ -327,6 +335,7 @@ type TrainingEstimateRuntime = {
workers: number
yoloBatchLabel: string
lowPriority: boolean
autoPause: boolean
factor: number
yoloFactor: number
}
@ -718,6 +727,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
const lowPriority = Boolean(
settings?.trainingEffectiveLowPriority ?? settings?.trainingLowPriority
)
const autoPause = Boolean(settings?.trainingAutoPauseEnabled)
const autoThreads = cpuCores > 0
? Math.max(1, Math.min(16, Math.round(cpuCores * 0.75)))
: 4
@ -769,11 +779,12 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
? 0.98
: 1
const lowPriorityFactor = lowPriority ? 1.12 : 1
const autoPauseFactor = autoPause ? 1.12 : 1
const corePenalty =
cpuCores > 0 && effectiveThreads > cpuCores
? 1 + Math.min(0.3, (effectiveThreads - cpuCores) * 0.04)
: 1
const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * corePenalty
const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * autoPauseFactor * corePenalty
const yoloFactor = factor * batchFactor
return {
@ -783,6 +794,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
workers,
yoloBatchLabel: rawBatch > 0 ? String(rawBatch) : 'Auto',
lowPriority,
autoPause,
factor: Number.isFinite(factor) && factor > 0 ? factor : 1,
yoloFactor: Number.isFinite(yoloFactor) && yoloFactor > 0 ? yoloFactor : 1,
}
@ -790,7 +802,8 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
function trainingEstimateRuntimeText(runtime: TrainingEstimateRuntime) {
const priority = runtime.lowPriority ? ' · niedrige Priorität' : ''
return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}`
const autoPause = runtime.autoPause ? ' · Auto-Pause' : ''
return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}${autoPause}`
}
function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) {
@ -802,6 +815,7 @@ function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) {
trainingEffectiveWorkers: entry.workers,
trainingEffectiveYoloBatchSize: entry.yoloBatchSize,
trainingEffectiveLowPriority: entry.lowPriority,
trainingAutoPauseEnabled: entry.autoPauseEnabled,
} as RecorderSettingsState)
}
@ -4644,6 +4658,10 @@ export default function TrainingTab(props: {
epoch: Number(job.epoch ?? 0),
epochs: Number(job.epochs ?? 0),
previewUrl: String(job.previewUrl ?? ''),
paused: Boolean(job.paused),
pauseReason: String(job.pauseReason ?? ''),
cpuPercent: Number(job.cpuPercent ?? 0),
temperatureC: Number(job.temperatureC ?? 0),
map50: Number(job.map50 ?? 0),
map5095: Number(job.map5095 ?? 0),
}
@ -5125,6 +5143,10 @@ export default function TrainingTab(props: {
const importVideoIntoTraining = useCallback(async (raw: any) => {
const output = String(raw?.output || '').trim()
if (!output) return false
if (trainingRunning) {
setMessage('Während das Training läuft, kann kein Video ins Training übernommen werden.')
return false
}
setLoadingPreviewCandidate('')
@ -5277,6 +5299,7 @@ export default function TrainingTab(props: {
loadPriorityTrainingSamples,
loadTrainingStatus,
setLoadingPreviewCandidate,
trainingRunning,
waitForVideoImportResult,
])
@ -7243,13 +7266,25 @@ export default function TrainingTab(props: {
},
]
const trainingPaused = Boolean(trainingStatus?.training?.paused)
const pauseReason = String(trainingStatus?.training?.pauseReason || '').trim()
const pauseMetrics = [
trainingStatus?.training?.cpuPercent
? `CPU ${Math.round(Number(trainingStatus.training.cpuPercent))}%`
: '',
trainingStatus?.training?.temperatureC
? `${Math.round(Number(trainingStatus.training.temperatureC))}°C`
: '',
].filter(Boolean).join(' · ')
const progress = clampPercent(
trainingRunning
? shownTrainingProgress
: readinessProgress * 100
)
const statusText = trainingRunning
? shownTrainingStep || 'Training läuft…'
? trainingPaused
? `Pausiert${pauseReason ? `: ${pauseReason}` : ''}${pauseMetrics ? ` · ${pauseMetrics}` : ''}`
: shownTrainingStep || 'Training läuft…'
: !feedbackReady
? `${Math.max(0, requiredCount - feedbackCount)} Feedback fehlen noch`
: canStartTraining
@ -7280,14 +7315,18 @@ export default function TrainingTab(props: {
<div
className={[
'flex h-8 w-8 shrink-0 items-center justify-center rounded-lg ring-1',
trainingRunning
trainingPaused
? 'bg-amber-50 text-amber-700 ring-amber-200 dark:bg-amber-500/15 dark:text-amber-200 dark:ring-amber-400/30'
: trainingRunning
? 'bg-indigo-50 text-indigo-700 ring-indigo-200 dark:bg-indigo-500/15 dark:text-indigo-200 dark:ring-indigo-400/30'
: canStartTraining
? 'bg-emerald-50 text-emerald-700 ring-emerald-200 dark:bg-emerald-500/15 dark:text-emerald-200 dark:ring-emerald-400/30'
: 'bg-gray-50 text-gray-600 ring-gray-200 dark:bg-white/5 dark:text-gray-300 dark:ring-white/10',
].join(' ')}
>
{trainingRunning ? (
{trainingPaused ? (
<PauseIcon className="h-4 w-4" aria-hidden="true" />
) : trainingRunning ? (
<ArrowPathIcon className="h-4 w-4 animate-spin" aria-hidden="true" />
) : (
<BoltIcon className="h-4 w-4" aria-hidden="true" />
@ -7394,7 +7433,9 @@ export default function TrainingTab(props: {
<div
className={[
'h-full rounded-full transition-all duration-500',
trainingRunning
trainingPaused
? 'bg-amber-500'
: trainingRunning
? 'bg-indigo-500'
: canStartTraining
? 'bg-emerald-500'

3
frontend/src/types.ts
View File

@ -289,6 +289,9 @@ export type RecorderSettingsState = {
trainingYoloBatchSize?: number
trainingLowPriority?: boolean
trainingVideoMAEEnabled?: boolean
trainingAutoPauseEnabled?: boolean
trainingAutoPauseCpuPercent?: number
trainingAutoPauseTemperatureC?: number
trainingCpuCoreCount?: number
trainingEffectiveMode?: string
trainingEffectiveCpuThreads?: number