added temperature pause
This commit is contained in:
parent
5e6c841b26
commit
6b6528acca
BIN
backend/dist/nsfwapp.exe
vendored
BIN
backend/dist/nsfwapp.exe
vendored
Binary file not shown.
@ -13,11 +13,13 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
gocpu "github.com/shirou/gopsutil/v3/cpu"
|
gocpu "github.com/shirou/gopsutil/v3/cpu"
|
||||||
godisk "github.com/shirou/gopsutil/v3/disk"
|
godisk "github.com/shirou/gopsutil/v3/disk"
|
||||||
|
gohost "github.com/shirou/gopsutil/v3/host"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Server-Startzeit für Perf/Uptime
|
// Server-Startzeit für Perf/Uptime
|
||||||
@ -25,6 +27,13 @@ var serverStartedAt = time.Now()
|
|||||||
|
|
||||||
var lastCPUUsageBits uint64 // atomic float64 bits
|
var lastCPUUsageBits uint64 // atomic float64 bits
|
||||||
|
|
||||||
|
var cpuTemperatureCache = struct {
|
||||||
|
sync.Mutex
|
||||||
|
value float64
|
||||||
|
available bool
|
||||||
|
sampledAt time.Time
|
||||||
|
}{}
|
||||||
|
|
||||||
func setLastCPUUsage(v float64) {
|
func setLastCPUUsage(v float64) {
|
||||||
atomic.StoreUint64(&lastCPUUsageBits, math.Float64bits(v))
|
atomic.StoreUint64(&lastCPUUsageBits, math.Float64bits(v))
|
||||||
}
|
}
|
||||||
@ -33,6 +42,71 @@ func getLastCPUUsage() float64 {
|
|||||||
return math.Float64frombits(atomic.LoadUint64(&lastCPUUsageBits))
|
return math.Float64frombits(atomic.LoadUint64(&lastCPUUsageBits))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getMaxCPUTemperatureC() (float64, bool) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
cpuTemperatureCache.Lock()
|
||||||
|
if now.Sub(cpuTemperatureCache.sampledAt) < 15*time.Second {
|
||||||
|
value := cpuTemperatureCache.value
|
||||||
|
available := cpuTemperatureCache.available
|
||||||
|
cpuTemperatureCache.Unlock()
|
||||||
|
return value, available
|
||||||
|
}
|
||||||
|
cpuTemperatureCache.Unlock()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
temps, err := gohost.SensorsTemperaturesWithContext(ctx)
|
||||||
|
if err != nil || len(temps) == 0 {
|
||||||
|
cpuTemperatureCache.Lock()
|
||||||
|
cpuTemperatureCache.value = 0
|
||||||
|
cpuTemperatureCache.available = false
|
||||||
|
cpuTemperatureCache.sampledAt = now
|
||||||
|
cpuTemperatureCache.Unlock()
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
maxCPU := 0.0
|
||||||
|
maxAny := 0.0
|
||||||
|
for _, temp := range temps {
|
||||||
|
v := temp.Temperature
|
||||||
|
if math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 || v > 150 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if v > maxAny {
|
||||||
|
maxAny = v
|
||||||
|
}
|
||||||
|
|
||||||
|
key := strings.ToLower(strings.TrimSpace(temp.SensorKey))
|
||||||
|
if strings.Contains(key, "cpu") ||
|
||||||
|
strings.Contains(key, "core") ||
|
||||||
|
strings.Contains(key, "package") ||
|
||||||
|
strings.Contains(key, "tdie") ||
|
||||||
|
strings.Contains(key, "tctl") ||
|
||||||
|
strings.Contains(key, "k10temp") {
|
||||||
|
if v > maxCPU {
|
||||||
|
maxCPU = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
value := maxCPU
|
||||||
|
if value <= 0 {
|
||||||
|
value = maxAny
|
||||||
|
}
|
||||||
|
available := value > 0
|
||||||
|
|
||||||
|
cpuTemperatureCache.Lock()
|
||||||
|
cpuTemperatureCache.value = value
|
||||||
|
cpuTemperatureCache.available = available
|
||||||
|
cpuTemperatureCache.sampledAt = now
|
||||||
|
cpuTemperatureCache.Unlock()
|
||||||
|
|
||||||
|
return value, available
|
||||||
|
}
|
||||||
|
|
||||||
func startCPUUsageSampler(ctx context.Context) {
|
func startCPUUsageSampler(ctx context.Context) {
|
||||||
_, _ = gocpu.Percent(200*time.Millisecond, false)
|
_, _ = gocpu.Percent(200*time.Millisecond, false)
|
||||||
|
|
||||||
@ -194,6 +268,17 @@ func buildPerfSnapshot() map[string]any {
|
|||||||
}
|
}
|
||||||
return v
|
return v
|
||||||
}(),
|
}(),
|
||||||
|
"cpuTemperatureC": func() float64 {
|
||||||
|
v, ok := getMaxCPUTemperatureC()
|
||||||
|
if !ok {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return math.Round(v*10) / 10
|
||||||
|
}(),
|
||||||
|
"cpuTemperatureAvailable": func() bool {
|
||||||
|
_, ok := getMaxCPUTemperatureC()
|
||||||
|
return ok
|
||||||
|
}(),
|
||||||
"diskPath": diskPath,
|
"diskPath": diskPath,
|
||||||
"diskFreeBytes": diskFreeBytes,
|
"diskFreeBytes": diskFreeBytes,
|
||||||
"diskTotalBytes": diskTotalBytes,
|
"diskTotalBytes": diskTotalBytes,
|
||||||
|
|||||||
@ -56,15 +56,18 @@ type RecorderSettings struct {
|
|||||||
GenerateAssetsAnalyze bool `json:"generateAssetsAnalyze"`
|
GenerateAssetsAnalyze bool `json:"generateAssetsAnalyze"`
|
||||||
EnrichPostworkEnabled bool `json:"enrichPostworkEnabled"`
|
EnrichPostworkEnabled bool `json:"enrichPostworkEnabled"`
|
||||||
|
|
||||||
TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"`
|
TrainingRecognitionEnabled bool `json:"trainingRecognitionEnabled"`
|
||||||
TrainingDetectorEpochs int `json:"trainingDetectorEpochs"`
|
TrainingDetectorEpochs int `json:"trainingDetectorEpochs"`
|
||||||
TrainingPerformanceMode string `json:"trainingPerformanceMode"`
|
TrainingPerformanceMode string `json:"trainingPerformanceMode"`
|
||||||
TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"`
|
TrainingPowerSaveMode bool `json:"trainingPowerSaveMode"`
|
||||||
TrainingCPUThreads int `json:"trainingCpuThreads"`
|
TrainingCPUThreads int `json:"trainingCpuThreads"`
|
||||||
TrainingWorkers int `json:"trainingWorkers"`
|
TrainingWorkers int `json:"trainingWorkers"`
|
||||||
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
|
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
|
||||||
TrainingLowPriority bool `json:"trainingLowPriority"`
|
TrainingLowPriority bool `json:"trainingLowPriority"`
|
||||||
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
|
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
|
||||||
|
TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"`
|
||||||
|
TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"`
|
||||||
|
TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"`
|
||||||
|
|
||||||
// EncryptedCookies contains base64(nonce+ciphertext) of a JSON cookie map.
|
// EncryptedCookies contains base64(nonce+ciphertext) of a JSON cookie map.
|
||||||
EncryptedCookies string `json:"encryptedCookies"`
|
EncryptedCookies string `json:"encryptedCookies"`
|
||||||
@ -112,15 +115,18 @@ var (
|
|||||||
GenerateAssetsAnalyze: false,
|
GenerateAssetsAnalyze: false,
|
||||||
EnrichPostworkEnabled: true,
|
EnrichPostworkEnabled: true,
|
||||||
|
|
||||||
TrainingRecognitionEnabled: true,
|
TrainingRecognitionEnabled: true,
|
||||||
TrainingDetectorEpochs: 60,
|
TrainingDetectorEpochs: 60,
|
||||||
TrainingPerformanceMode: trainingPerformanceAuto,
|
TrainingPerformanceMode: trainingPerformanceAuto,
|
||||||
TrainingPowerSaveMode: false,
|
TrainingPowerSaveMode: false,
|
||||||
TrainingCPUThreads: 0,
|
TrainingCPUThreads: 0,
|
||||||
TrainingWorkers: 2,
|
TrainingWorkers: 2,
|
||||||
TrainingYoloBatchSize: 0,
|
TrainingYoloBatchSize: 0,
|
||||||
TrainingLowPriority: false,
|
TrainingLowPriority: false,
|
||||||
TrainingVideoMAEEnabled: true,
|
TrainingVideoMAEEnabled: true,
|
||||||
|
TrainingAutoPauseEnabled: false,
|
||||||
|
TrainingAutoPauseCPUPercent: 95,
|
||||||
|
TrainingAutoPauseTemperatureC: 85,
|
||||||
|
|
||||||
EncryptedCookies: "",
|
EncryptedCookies: "",
|
||||||
}
|
}
|
||||||
@ -189,6 +195,20 @@ func normalizeTrainingSettings(s *RecorderSettings) {
|
|||||||
if s.TrainingYoloBatchSize > 64 {
|
if s.TrainingYoloBatchSize > 64 {
|
||||||
s.TrainingYoloBatchSize = 64
|
s.TrainingYoloBatchSize = 64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if s.TrainingAutoPauseCPUPercent < 1 {
|
||||||
|
s.TrainingAutoPauseCPUPercent = 95
|
||||||
|
}
|
||||||
|
if s.TrainingAutoPauseCPUPercent > 100 {
|
||||||
|
s.TrainingAutoPauseCPUPercent = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.TrainingAutoPauseTemperatureC < 1 {
|
||||||
|
s.TrainingAutoPauseTemperatureC = 85
|
||||||
|
}
|
||||||
|
if s.TrainingAutoPauseTemperatureC > 120 {
|
||||||
|
s.TrainingAutoPauseTemperatureC = 120
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func settingsFilePath() string {
|
func settingsFilePath() string {
|
||||||
@ -375,6 +395,9 @@ type RecorderSettingsPublic struct {
|
|||||||
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
|
TrainingYoloBatchSize int `json:"trainingYoloBatchSize"`
|
||||||
TrainingLowPriority bool `json:"trainingLowPriority"`
|
TrainingLowPriority bool `json:"trainingLowPriority"`
|
||||||
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
|
TrainingVideoMAEEnabled bool `json:"trainingVideoMAEEnabled"`
|
||||||
|
TrainingAutoPauseEnabled bool `json:"trainingAutoPauseEnabled"`
|
||||||
|
TrainingAutoPauseCPUPercent int `json:"trainingAutoPauseCpuPercent"`
|
||||||
|
TrainingAutoPauseTemperatureC int `json:"trainingAutoPauseTemperatureC"`
|
||||||
TrainingCPUCoreCount int `json:"trainingCpuCoreCount"`
|
TrainingCPUCoreCount int `json:"trainingCpuCoreCount"`
|
||||||
TrainingEffectiveMode string `json:"trainingEffectiveMode"`
|
TrainingEffectiveMode string `json:"trainingEffectiveMode"`
|
||||||
TrainingEffectiveCPUThreads int `json:"trainingEffectiveCpuThreads"`
|
TrainingEffectiveCPUThreads int `json:"trainingEffectiveCpuThreads"`
|
||||||
@ -436,6 +459,9 @@ func toPublicSettings(s RecorderSettings) RecorderSettingsPublic {
|
|||||||
TrainingYoloBatchSize: s.TrainingYoloBatchSize,
|
TrainingYoloBatchSize: s.TrainingYoloBatchSize,
|
||||||
TrainingLowPriority: s.TrainingLowPriority,
|
TrainingLowPriority: s.TrainingLowPriority,
|
||||||
TrainingVideoMAEEnabled: s.TrainingVideoMAEEnabled,
|
TrainingVideoMAEEnabled: s.TrainingVideoMAEEnabled,
|
||||||
|
TrainingAutoPauseEnabled: s.TrainingAutoPauseEnabled,
|
||||||
|
TrainingAutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent,
|
||||||
|
TrainingAutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC,
|
||||||
TrainingCPUCoreCount: trainingOpts.CPUCoreCount,
|
TrainingCPUCoreCount: trainingOpts.CPUCoreCount,
|
||||||
TrainingEffectiveMode: trainingOpts.PerformanceMode,
|
TrainingEffectiveMode: trainingOpts.PerformanceMode,
|
||||||
TrainingEffectiveCPUThreads: trainingOpts.CPUThreads,
|
TrainingEffectiveCPUThreads: trainingOpts.CPUThreads,
|
||||||
@ -631,6 +657,9 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
next.TrainingYoloBatchSize = in.TrainingYoloBatchSize
|
next.TrainingYoloBatchSize = in.TrainingYoloBatchSize
|
||||||
next.TrainingLowPriority = in.TrainingLowPriority
|
next.TrainingLowPriority = in.TrainingLowPriority
|
||||||
next.TrainingVideoMAEEnabled = in.TrainingVideoMAEEnabled
|
next.TrainingVideoMAEEnabled = in.TrainingVideoMAEEnabled
|
||||||
|
next.TrainingAutoPauseEnabled = in.TrainingAutoPauseEnabled
|
||||||
|
next.TrainingAutoPauseCPUPercent = in.TrainingAutoPauseCPUPercent
|
||||||
|
next.TrainingAutoPauseTemperatureC = in.TrainingAutoPauseTemperatureC
|
||||||
|
|
||||||
dbChanged :=
|
dbChanged :=
|
||||||
normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) ||
|
normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) ||
|
||||||
|
|||||||
@ -24,6 +24,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
goprocess "github.com/shirou/gopsutil/v3/process"
|
||||||
)
|
)
|
||||||
|
|
||||||
const trainingUncertainCandidateCount = 10
|
const trainingUncertainCandidateCount = 10
|
||||||
@ -262,15 +264,19 @@ type TrainingPosePrediction struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TrainingJobStatus struct {
|
type TrainingJobStatus struct {
|
||||||
Running bool `json:"running"`
|
Running bool `json:"running"`
|
||||||
Progress int `json:"progress"`
|
Progress int `json:"progress"`
|
||||||
Step string `json:"step"`
|
Step string `json:"step"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
StartedAt string `json:"startedAt,omitempty"`
|
StartedAt string `json:"startedAt,omitempty"`
|
||||||
FinishedAt string `json:"finishedAt,omitempty"`
|
FinishedAt string `json:"finishedAt,omitempty"`
|
||||||
DurationMs int64 `json:"durationMs,omitempty"`
|
DurationMs int64 `json:"durationMs,omitempty"`
|
||||||
PreviewURL string `json:"previewUrl,omitempty"`
|
PreviewURL string `json:"previewUrl,omitempty"`
|
||||||
|
Paused bool `json:"paused,omitempty"`
|
||||||
|
PauseReason string `json:"pauseReason,omitempty"`
|
||||||
|
CPUPercent float64 `json:"cpuPercent,omitempty"`
|
||||||
|
TemperatureC float64 `json:"temperatureC,omitempty"`
|
||||||
|
|
||||||
Stage string `json:"stage,omitempty"`
|
Stage string `json:"stage,omitempty"`
|
||||||
Epoch int `json:"epoch,omitempty"`
|
Epoch int `json:"epoch,omitempty"`
|
||||||
@ -1181,6 +1187,8 @@ func trainingRunCommandStreaming(
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
applyTrainingLowPriorityAfterStart(cmd, opts.LowPriority)
|
applyTrainingLowPriorityAfterStart(cmd, opts.LowPriority)
|
||||||
|
monitorCtx, stopResourcePauseMonitor := context.WithCancel(ctx)
|
||||||
|
resumeResourcePause := startTrainingResourcePauseMonitor(monitorCtx, cmd, opts)
|
||||||
|
|
||||||
var outMu sync.Mutex
|
var outMu sync.Mutex
|
||||||
var lines []string
|
var lines []string
|
||||||
@ -1231,9 +1239,12 @@ func trainingRunCommandStreaming(
|
|||||||
select {
|
select {
|
||||||
case err = <-waitCh:
|
case err = <-waitCh:
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
resumeResourcePause()
|
||||||
terminateTrainingCommand(cmd)
|
terminateTrainingCommand(cmd)
|
||||||
err = <-waitCh
|
err = <-waitCh
|
||||||
}
|
}
|
||||||
|
stopResourcePauseMonitor()
|
||||||
|
resumeResourcePause()
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
outMu.Lock()
|
outMu.Lock()
|
||||||
@ -1274,14 +1285,17 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type trainingRuntimeOptions struct {
|
type trainingRuntimeOptions struct {
|
||||||
PerformanceMode string
|
PerformanceMode string
|
||||||
PowerSaveMode bool
|
PowerSaveMode bool
|
||||||
CPUCoreCount int
|
CPUCoreCount int
|
||||||
CPUThreads int
|
CPUThreads int
|
||||||
Workers int
|
Workers int
|
||||||
YoloBatchSize int
|
YoloBatchSize int
|
||||||
LowPriority bool
|
LowPriority bool
|
||||||
VideoMAEEnabled bool
|
VideoMAEEnabled bool
|
||||||
|
AutoPauseEnabled bool
|
||||||
|
AutoPauseCPUPercent int
|
||||||
|
AutoPauseTemperatureC int
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeTrainingPerformanceMode(raw string) string {
|
func normalizeTrainingPerformanceMode(raw string) string {
|
||||||
@ -1381,14 +1395,17 @@ func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRunt
|
|||||||
presetThreads, presetWorkers, presetBatch, presetLowPriority := trainingPresetValues(mode, cpuCores)
|
presetThreads, presetWorkers, presetBatch, presetLowPriority := trainingPresetValues(mode, cpuCores)
|
||||||
|
|
||||||
opts := trainingRuntimeOptions{
|
opts := trainingRuntimeOptions{
|
||||||
PerformanceMode: mode,
|
PerformanceMode: mode,
|
||||||
PowerSaveMode: mode == trainingPerformanceEco,
|
PowerSaveMode: mode == trainingPerformanceEco,
|
||||||
CPUCoreCount: cpuCores,
|
CPUCoreCount: cpuCores,
|
||||||
CPUThreads: presetThreads,
|
CPUThreads: presetThreads,
|
||||||
Workers: presetWorkers,
|
Workers: presetWorkers,
|
||||||
YoloBatchSize: presetBatch,
|
YoloBatchSize: presetBatch,
|
||||||
LowPriority: presetLowPriority,
|
LowPriority: presetLowPriority,
|
||||||
VideoMAEEnabled: s.TrainingVideoMAEEnabled,
|
VideoMAEEnabled: s.TrainingVideoMAEEnabled,
|
||||||
|
AutoPauseEnabled: s.TrainingAutoPauseEnabled,
|
||||||
|
AutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent,
|
||||||
|
AutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC,
|
||||||
}
|
}
|
||||||
|
|
||||||
if mode == trainingPerformanceCustom {
|
if mode == trainingPerformanceCustom {
|
||||||
@ -1448,6 +1465,331 @@ func trainingCommandEnv(opts trainingRuntimeOptions) []string {
|
|||||||
return env
|
return env
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type trainingResourceSnapshot struct {
|
||||||
|
CPUPercent float64
|
||||||
|
TemperatureC float64
|
||||||
|
TemperatureAvailable bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingRoundMetric(v float64) float64 {
|
||||||
|
if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return math.Round(v*10) / 10
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingReadResourceSnapshot() trainingResourceSnapshot {
|
||||||
|
temp, tempOK := getMaxCPUTemperatureC()
|
||||||
|
return trainingResourceSnapshot{
|
||||||
|
CPUPercent: trainingRoundMetric(getLastCPUUsage()),
|
||||||
|
TemperatureC: trainingRoundMetric(temp),
|
||||||
|
TemperatureAvailable: tempOK,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingPauseReasonForResources(opts trainingRuntimeOptions, snap trainingResourceSnapshot) (string, bool) {
|
||||||
|
if !opts.AutoPauseEnabled {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
reasons := []string{}
|
||||||
|
if opts.AutoPauseCPUPercent > 0 && snap.CPUPercent >= float64(opts.AutoPauseCPUPercent) {
|
||||||
|
reasons = append(
|
||||||
|
reasons,
|
||||||
|
fmt.Sprintf("CPU %.0f%% >= %d%%", snap.CPUPercent, opts.AutoPauseCPUPercent),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if opts.AutoPauseTemperatureC > 0 &&
|
||||||
|
snap.TemperatureAvailable &&
|
||||||
|
snap.TemperatureC >= float64(opts.AutoPauseTemperatureC) {
|
||||||
|
reasons = append(
|
||||||
|
reasons,
|
||||||
|
fmt.Sprintf("Temperatur %.0f°C >= %d°C", snap.TemperatureC, opts.AutoPauseTemperatureC),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(reasons) == 0 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return strings.Join(reasons, ", "), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingResourcesRecovered(opts trainingRuntimeOptions, snap trainingResourceSnapshot) bool {
|
||||||
|
if !opts.AutoPauseEnabled {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if opts.AutoPauseCPUPercent > 0 {
|
||||||
|
resumeCPU := float64(opts.AutoPauseCPUPercent - 15)
|
||||||
|
if resumeCPU < 50 {
|
||||||
|
resumeCPU = 50
|
||||||
|
}
|
||||||
|
if snap.CPUPercent > resumeCPU {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if opts.AutoPauseTemperatureC > 0 && snap.TemperatureAvailable {
|
||||||
|
resumeTemp := float64(opts.AutoPauseTemperatureC - 5)
|
||||||
|
if resumeTemp < 40 {
|
||||||
|
resumeTemp = 40
|
||||||
|
}
|
||||||
|
if snap.TemperatureC > resumeTemp {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingProcessTree(pid int32) []*goprocess.Process {
|
||||||
|
if pid <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := goprocess.NewProcess(pid)
|
||||||
|
if err != nil || root == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[int32]bool{}
|
||||||
|
var out []*goprocess.Process
|
||||||
|
var walk func(*goprocess.Process)
|
||||||
|
walk = func(p *goprocess.Process) {
|
||||||
|
if p == nil || seen[p.Pid] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
seen[p.Pid] = true
|
||||||
|
out = append(out, p)
|
||||||
|
|
||||||
|
children, err := p.Children()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, child := range children {
|
||||||
|
walk(child)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
walk(root)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingSuspendProcessTree(cmd *exec.Cmd) error {
|
||||||
|
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
procs := trainingProcessTree(int32(cmd.Process.Pid))
|
||||||
|
if len(procs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var firstErr error
|
||||||
|
success := 0
|
||||||
|
for i := len(procs) - 1; i >= 0; i-- {
|
||||||
|
if err := procs[i].Suspend(); err != nil {
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
success++
|
||||||
|
}
|
||||||
|
|
||||||
|
if success > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return firstErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingResumeProcessTree(cmd *exec.Cmd) error {
|
||||||
|
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
procs := trainingProcessTree(int32(cmd.Process.Pid))
|
||||||
|
if len(procs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var firstErr error
|
||||||
|
success := 0
|
||||||
|
for _, proc := range procs {
|
||||||
|
if err := proc.Resume(); err != nil {
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
success++
|
||||||
|
}
|
||||||
|
|
||||||
|
if success > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return firstErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func startTrainingResourcePauseMonitor(ctx context.Context, cmd *exec.Cmd, opts trainingRuntimeOptions) func() {
|
||||||
|
if !opts.AutoPauseEnabled || cmd == nil || cmd.Process == nil {
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
checkInterval = 5 * time.Second
|
||||||
|
requiredHighSamples = 2
|
||||||
|
requiredCoolSamples = 2
|
||||||
|
minPauseDuration = 45 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
var pauseMu sync.Mutex
|
||||||
|
paused := false
|
||||||
|
previousStep := ""
|
||||||
|
highSamples := 0
|
||||||
|
coolSamples := 0
|
||||||
|
pauseStartedAt := time.Time{}
|
||||||
|
loggedSuspendError := false
|
||||||
|
|
||||||
|
resumeNow := func(reason string) {
|
||||||
|
pauseMu.Lock()
|
||||||
|
if !paused {
|
||||||
|
pauseMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
err := trainingResumeProcessTree(cmd)
|
||||||
|
paused = false
|
||||||
|
restoreStep := strings.TrimSpace(previousStep)
|
||||||
|
previousStep = ""
|
||||||
|
pauseMu.Unlock()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
appLogln("⚠️ Training konnte nach Ressourcenpause nicht sauber fortgesetzt werden:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.TrimSpace(reason) != "" {
|
||||||
|
appLogln(reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
|
s.Paused = false
|
||||||
|
s.PauseReason = ""
|
||||||
|
s.CPUPercent = 0
|
||||||
|
s.TemperatureC = 0
|
||||||
|
s.Message = ""
|
||||||
|
if restoreStep != "" && s.Running {
|
||||||
|
s.Step = restoreStep
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
ticker := time.NewTicker(checkInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
resumeNow("Training-Ressourcenpause wurde beendet.")
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
|
||||||
|
snap := trainingReadResourceSnapshot()
|
||||||
|
|
||||||
|
pauseMu.Lock()
|
||||||
|
isPaused := paused
|
||||||
|
pauseAge := time.Since(pauseStartedAt)
|
||||||
|
pauseMu.Unlock()
|
||||||
|
|
||||||
|
if !isPaused {
|
||||||
|
reason, shouldPause := trainingPauseReasonForResources(opts, snap)
|
||||||
|
if !shouldPause {
|
||||||
|
highSamples = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
highSamples++
|
||||||
|
if highSamples < requiredHighSamples {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
pauseMu.Lock()
|
||||||
|
if paused {
|
||||||
|
pauseMu.Unlock()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
currentStatus := trainingGetJobStatus()
|
||||||
|
previousStep = strings.TrimSpace(currentStatus.Step)
|
||||||
|
if previousStep == "" {
|
||||||
|
previousStep = "Training läuft..."
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := trainingSuspendProcessTree(cmd); err != nil {
|
||||||
|
pauseMu.Unlock()
|
||||||
|
highSamples = 0
|
||||||
|
if !loggedSuspendError {
|
||||||
|
loggedSuspendError = true
|
||||||
|
appLogln("⚠️ Training-Ressourcenpause konnte Prozess nicht pausieren:", err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
paused = true
|
||||||
|
pauseStartedAt = time.Now()
|
||||||
|
coolSamples = 0
|
||||||
|
pauseMu.Unlock()
|
||||||
|
|
||||||
|
appLogln("⏸ Training pausiert wegen Ressourcenlimit:", reason)
|
||||||
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
|
s.Paused = true
|
||||||
|
s.PauseReason = reason
|
||||||
|
s.CPUPercent = snap.CPUPercent
|
||||||
|
if snap.TemperatureAvailable {
|
||||||
|
s.TemperatureC = snap.TemperatureC
|
||||||
|
} else {
|
||||||
|
s.TemperatureC = 0
|
||||||
|
}
|
||||||
|
s.Step = "Training pausiert: " + reason
|
||||||
|
s.Message = "Training wird automatisch fortgesetzt, sobald CPU/Temperatur wieder im grünen Bereich sind."
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
|
if !s.Running || !s.Paused {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.CPUPercent = snap.CPUPercent
|
||||||
|
if snap.TemperatureAvailable {
|
||||||
|
s.TemperatureC = snap.TemperatureC
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if pauseAge < minPauseDuration || !trainingResourcesRecovered(opts, snap) {
|
||||||
|
coolSamples = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
coolSamples++
|
||||||
|
if coolSamples < requiredCoolSamples {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
highSamples = 0
|
||||||
|
coolSamples = 0
|
||||||
|
resumeNow("▶ Training nach Ressourcenpause fortgesetzt.")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return func() {
|
||||||
|
resumeNow("")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var trainingJob = struct {
|
var trainingJob = struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
status TrainingJobStatus
|
status TrainingJobStatus
|
||||||
@ -1526,6 +1868,10 @@ func trainingFinishCancelled(root string) {
|
|||||||
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
||||||
s.DurationMs = durationMs
|
s.DurationMs = durationMs
|
||||||
s.PreviewURL = ""
|
s.PreviewURL = ""
|
||||||
|
s.Paused = false
|
||||||
|
s.PauseReason = ""
|
||||||
|
s.CPUPercent = 0
|
||||||
|
s.TemperatureC = 0
|
||||||
|
|
||||||
if cleanupErr != nil {
|
if cleanupErr != nil {
|
||||||
s.Message = "Training wurde abgebrochen, aber temporäre Trainingsausgaben konnten nicht vollständig gelöscht werden."
|
s.Message = "Training wurde abgebrochen, aber temporäre Trainingsausgaben konnten nicht vollständig gelöscht werden."
|
||||||
@ -3641,6 +3987,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
|
|||||||
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
||||||
s.DurationMs = durationMs
|
s.DurationMs = durationMs
|
||||||
s.PreviewURL = ""
|
s.PreviewURL = ""
|
||||||
|
s.Paused = false
|
||||||
|
s.PauseReason = ""
|
||||||
|
s.CPUPercent = 0
|
||||||
|
s.TemperatureC = 0
|
||||||
})
|
})
|
||||||
trainingClearJobCancel()
|
trainingClearJobCancel()
|
||||||
return
|
return
|
||||||
@ -3650,7 +4000,7 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
|
|||||||
appLogln("ML-Python für Training:", python)
|
appLogln("ML-Python für Training:", python)
|
||||||
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
||||||
appLogf(
|
appLogf(
|
||||||
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v targets=%s",
|
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v autoPause=%v cpuLimit=%d tempLimit=%d targets=%s",
|
||||||
runtimeOpts.PerformanceMode,
|
runtimeOpts.PerformanceMode,
|
||||||
runtimeOpts.CPUCoreCount,
|
runtimeOpts.CPUCoreCount,
|
||||||
runtimeOpts.PowerSaveMode,
|
runtimeOpts.PowerSaveMode,
|
||||||
@ -3659,6 +4009,9 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
|
|||||||
runtimeOpts.YoloBatchSize,
|
runtimeOpts.YoloBatchSize,
|
||||||
runtimeOpts.LowPriority,
|
runtimeOpts.LowPriority,
|
||||||
runtimeOpts.VideoMAEEnabled,
|
runtimeOpts.VideoMAEEnabled,
|
||||||
|
runtimeOpts.AutoPauseEnabled,
|
||||||
|
runtimeOpts.AutoPauseCPUPercent,
|
||||||
|
runtimeOpts.AutoPauseTemperatureC,
|
||||||
strings.Join(targets.list(), ","),
|
strings.Join(targets.list(), ","),
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -4209,6 +4562,10 @@ func trainingRunJob(ctx context.Context, root string, count int, targets trainin
|
|||||||
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
||||||
s.DurationMs = durationMs
|
s.DurationMs = durationMs
|
||||||
s.PreviewURL = ""
|
s.PreviewURL = ""
|
||||||
|
s.Paused = false
|
||||||
|
s.PauseReason = ""
|
||||||
|
s.CPUPercent = 0
|
||||||
|
s.TemperatureC = 0
|
||||||
})
|
})
|
||||||
|
|
||||||
trainingClearJobCancel()
|
trainingClearJobCancel()
|
||||||
@ -4765,24 +5122,27 @@ func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TrainingHistoryEntry struct {
|
type TrainingHistoryEntry struct {
|
||||||
TrainedAt string `json:"trainedAt,omitempty"`
|
TrainedAt string `json:"trainedAt,omitempty"`
|
||||||
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
||||||
Target string `json:"target,omitempty"`
|
Target string `json:"target,omitempty"`
|
||||||
Status string `json:"status,omitempty"`
|
Status string `json:"status,omitempty"`
|
||||||
DurationMs int64 `json:"durationMs,omitempty"`
|
DurationMs int64 `json:"durationMs,omitempty"`
|
||||||
Epochs int `json:"epochs,omitempty"`
|
Epochs int `json:"epochs,omitempty"`
|
||||||
TrainSamples int `json:"trainSamples,omitempty"`
|
TrainSamples int `json:"trainSamples,omitempty"`
|
||||||
ValSamples int `json:"valSamples,omitempty"`
|
ValSamples int `json:"valSamples,omitempty"`
|
||||||
Imgsz int `json:"imgsz,omitempty"`
|
Imgsz int `json:"imgsz,omitempty"`
|
||||||
Device string `json:"device,omitempty"`
|
Device string `json:"device,omitempty"`
|
||||||
MAP50 float64 `json:"map50,omitempty"`
|
MAP50 float64 `json:"map50,omitempty"`
|
||||||
MAP5095 float64 `json:"map5095,omitempty"`
|
MAP5095 float64 `json:"map5095,omitempty"`
|
||||||
PerformanceMode string `json:"performanceMode,omitempty"`
|
PerformanceMode string `json:"performanceMode,omitempty"`
|
||||||
CPUCoreCount int `json:"cpuCoreCount,omitempty"`
|
CPUCoreCount int `json:"cpuCoreCount,omitempty"`
|
||||||
CPUThreads int `json:"cpuThreads,omitempty"`
|
CPUThreads int `json:"cpuThreads,omitempty"`
|
||||||
Workers int `json:"workers,omitempty"`
|
Workers int `json:"workers,omitempty"`
|
||||||
YoloBatchSize int `json:"yoloBatchSize,omitempty"`
|
YoloBatchSize int `json:"yoloBatchSize,omitempty"`
|
||||||
LowPriority bool `json:"lowPriority,omitempty"`
|
LowPriority bool `json:"lowPriority,omitempty"`
|
||||||
|
AutoPauseEnabled bool `json:"autoPauseEnabled,omitempty"`
|
||||||
|
AutoPauseCPUPercent int `json:"autoPauseCpuPercent,omitempty"`
|
||||||
|
AutoPauseTemperatureC int `json:"autoPauseTemperatureC,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TrainingHistoryResponse struct {
|
type TrainingHistoryResponse struct {
|
||||||
@ -4813,24 +5173,27 @@ func trainingAppendTargetHistory(root string, target string, status string, dura
|
|||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
|
|
||||||
entry := TrainingHistoryEntry{
|
entry := TrainingHistoryEntry{
|
||||||
TrainedAt: now.Format(time.RFC3339),
|
TrainedAt: now.Format(time.RFC3339),
|
||||||
TrainedAtMs: now.UnixMilli(),
|
TrainedAtMs: now.UnixMilli(),
|
||||||
Target: kind,
|
Target: kind,
|
||||||
Status: strings.TrimSpace(status),
|
Status: strings.TrimSpace(status),
|
||||||
DurationMs: durationMs,
|
DurationMs: durationMs,
|
||||||
Epochs: fallback.Epochs,
|
Epochs: fallback.Epochs,
|
||||||
TrainSamples: fallback.TrainSamples,
|
TrainSamples: fallback.TrainSamples,
|
||||||
ValSamples: fallback.ValSamples,
|
ValSamples: fallback.ValSamples,
|
||||||
Imgsz: fallback.Imgsz,
|
Imgsz: fallback.Imgsz,
|
||||||
Device: strings.TrimSpace(fallback.Device),
|
Device: strings.TrimSpace(fallback.Device),
|
||||||
MAP50: fallback.MAP50,
|
MAP50: fallback.MAP50,
|
||||||
MAP5095: fallback.MAP5095,
|
MAP5095: fallback.MAP5095,
|
||||||
PerformanceMode: runtimeOpts.PerformanceMode,
|
PerformanceMode: runtimeOpts.PerformanceMode,
|
||||||
CPUCoreCount: runtimeOpts.CPUCoreCount,
|
CPUCoreCount: runtimeOpts.CPUCoreCount,
|
||||||
CPUThreads: runtimeOpts.CPUThreads,
|
CPUThreads: runtimeOpts.CPUThreads,
|
||||||
Workers: runtimeOpts.Workers,
|
Workers: runtimeOpts.Workers,
|
||||||
YoloBatchSize: runtimeOpts.YoloBatchSize,
|
YoloBatchSize: runtimeOpts.YoloBatchSize,
|
||||||
LowPriority: runtimeOpts.LowPriority,
|
LowPriority: runtimeOpts.LowPriority,
|
||||||
|
AutoPauseEnabled: runtimeOpts.AutoPauseEnabled,
|
||||||
|
AutoPauseCPUPercent: runtimeOpts.AutoPauseCPUPercent,
|
||||||
|
AutoPauseTemperatureC: runtimeOpts.AutoPauseTemperatureC,
|
||||||
}
|
}
|
||||||
|
|
||||||
if info != nil {
|
if info != nil {
|
||||||
|
|||||||
@ -1230,6 +1230,20 @@ export default function App() {
|
|||||||
}
|
}
|
||||||
}, [authed])
|
}, [authed])
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
try {
|
||||||
|
const appWindow = window as any
|
||||||
|
appWindow.__nsfwappTrainingRunning = trainingTabRunning
|
||||||
|
window.dispatchEvent(
|
||||||
|
new CustomEvent('app:training-running', {
|
||||||
|
detail: { running: trainingTabRunning },
|
||||||
|
})
|
||||||
|
)
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}, [trainingTabRunning])
|
||||||
|
|
||||||
const loadPendingAutoStarts = useCallback(async (opts?: { force?: boolean }) => {
|
const loadPendingAutoStarts = useCallback(async (opts?: { force?: boolean }) => {
|
||||||
const force = Boolean(opts?.force)
|
const force = Boolean(opts?.force)
|
||||||
const now = Date.now()
|
const now = Date.now()
|
||||||
|
|||||||
@ -76,6 +76,7 @@ type Props = {
|
|||||||
onAddToDownloads?: ActionFn
|
onAddToDownloads?: ActionFn
|
||||||
onSplit?: ActionFn
|
onSplit?: ActionFn
|
||||||
onAddToTraining?: ActionFn
|
onAddToTraining?: ActionFn
|
||||||
|
trainingImportDisabled?: boolean
|
||||||
|
|
||||||
order?: ActionKey[]
|
order?: ActionKey[]
|
||||||
|
|
||||||
@ -127,6 +128,7 @@ export default function RecordJobActions({
|
|||||||
onAddToDownloads,
|
onAddToDownloads,
|
||||||
onSplit,
|
onSplit,
|
||||||
onAddToTraining,
|
onAddToTraining,
|
||||||
|
trainingImportDisabled = false,
|
||||||
order,
|
order,
|
||||||
className,
|
className,
|
||||||
}: Props) {
|
}: Props) {
|
||||||
@ -205,6 +207,10 @@ export default function RecordJobActions({
|
|||||||
|
|
||||||
const [addState, setAddState] = useState<'idle' | 'busy' | 'ok'>('idle')
|
const [addState, setAddState] = useState<'idle' | 'busy' | 'ok'>('idle')
|
||||||
const addTimerRef = useRef<number | null>(null)
|
const addTimerRef = useRef<number | null>(null)
|
||||||
|
const [appTrainingRunning, setAppTrainingRunning] = useState(() => {
|
||||||
|
if (typeof window === 'undefined') return false
|
||||||
|
return Boolean((window as any).__nsfwappTrainingRunning)
|
||||||
|
})
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
return () => {
|
return () => {
|
||||||
@ -212,6 +218,19 @@ export default function RecordJobActions({
|
|||||||
}
|
}
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const onTrainingRunning = (event: Event) => {
|
||||||
|
const detail = (event as CustomEvent<{ running?: boolean }>).detail
|
||||||
|
setAppTrainingRunning(Boolean(detail?.running))
|
||||||
|
}
|
||||||
|
|
||||||
|
window.addEventListener('app:training-running', onTrainingRunning as EventListener)
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
window.removeEventListener('app:training-running', onTrainingRunning as EventListener)
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
const flashAddOk = () => {
|
const flashAddOk = () => {
|
||||||
setAddState('ok')
|
setAddState('ok')
|
||||||
if (addTimerRef.current) window.clearTimeout(addTimerRef.current)
|
if (addTimerRef.current) window.clearTimeout(addTimerRef.current)
|
||||||
@ -249,8 +268,11 @@ export default function RecordJobActions({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const trainingImportBlocked = trainingImportDisabled || appTrainingRunning
|
||||||
|
|
||||||
const doTrainingImport = async (): Promise<boolean> => {
|
const doTrainingImport = async (): Promise<boolean> => {
|
||||||
if (busy) return false
|
if (busy) return false
|
||||||
|
if (trainingImportBlocked) return false
|
||||||
|
|
||||||
const output = String(job.output || '').trim()
|
const output = String(job.output || '').trim()
|
||||||
if (!output) return false
|
if (!output) return false
|
||||||
@ -364,7 +386,7 @@ export default function RecordJobActions({
|
|||||||
className={btnBase}
|
className={btnBase}
|
||||||
title="Video ins Training übernehmen"
|
title="Video ins Training übernehmen"
|
||||||
aria-label="Video ins Training übernehmen"
|
aria-label="Video ins Training übernehmen"
|
||||||
disabled={busy || (!onAddToTraining && !job.output)}
|
disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)}
|
||||||
onClick={async (e) => {
|
onClick={async (e) => {
|
||||||
e.preventDefault()
|
e.preventDefault()
|
||||||
e.stopPropagation()
|
e.stopPropagation()
|
||||||
@ -774,7 +796,7 @@ export default function RecordJobActions({
|
|||||||
key="training"
|
key="training"
|
||||||
type="button"
|
type="button"
|
||||||
className="flex w-full items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-gray-100/70 dark:hover:bg-white/5 disabled:opacity-50"
|
className="flex w-full items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-gray-100/70 dark:hover:bg-white/5 disabled:opacity-50"
|
||||||
disabled={busy || (!onAddToTraining && !job.output)}
|
disabled={busy || trainingImportBlocked || (!onAddToTraining && !job.output)}
|
||||||
onClick={async (e) => {
|
onClick={async (e) => {
|
||||||
e.preventDefault()
|
e.preventDefault()
|
||||||
e.stopPropagation()
|
e.stopPropagation()
|
||||||
|
|||||||
@ -55,6 +55,9 @@ type Settings = {
|
|||||||
trainingYoloBatchSize?: number
|
trainingYoloBatchSize?: number
|
||||||
trainingLowPriority?: boolean
|
trainingLowPriority?: boolean
|
||||||
trainingVideoMAEEnabled?: boolean
|
trainingVideoMAEEnabled?: boolean
|
||||||
|
trainingAutoPauseEnabled?: boolean
|
||||||
|
trainingAutoPauseCpuPercent?: number
|
||||||
|
trainingAutoPauseTemperatureC?: number
|
||||||
trainingCpuCoreCount?: number
|
trainingCpuCoreCount?: number
|
||||||
trainingEffectiveMode?: TrainingPerformanceMode | string
|
trainingEffectiveMode?: TrainingPerformanceMode | string
|
||||||
trainingEffectiveCpuThreads?: number
|
trainingEffectiveCpuThreads?: number
|
||||||
@ -99,6 +102,10 @@ type TrainingJobStatus = {
|
|||||||
progress?: number
|
progress?: number
|
||||||
step?: string
|
step?: string
|
||||||
message?: string
|
message?: string
|
||||||
|
paused?: boolean
|
||||||
|
pauseReason?: string
|
||||||
|
cpuPercent?: number
|
||||||
|
temperatureC?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
type AppLog = {
|
type AppLog = {
|
||||||
@ -342,6 +349,9 @@ const DEFAULTS: Settings = {
|
|||||||
trainingYoloBatchSize: 0,
|
trainingYoloBatchSize: 0,
|
||||||
trainingLowPriority: false,
|
trainingLowPriority: false,
|
||||||
trainingVideoMAEEnabled: true,
|
trainingVideoMAEEnabled: true,
|
||||||
|
trainingAutoPauseEnabled: false,
|
||||||
|
trainingAutoPauseCpuPercent: 95,
|
||||||
|
trainingAutoPauseTemperatureC: 85,
|
||||||
}
|
}
|
||||||
|
|
||||||
const TRAINING_PERFORMANCE_MODES: Array<{
|
const TRAINING_PERFORMANCE_MODES: Array<{
|
||||||
@ -1024,6 +1034,12 @@ export default function Settings({ onAssetsGenerated }: Props) {
|
|||||||
(data as any).trainingLowPriority ?? DEFAULTS.trainingLowPriority,
|
(data as any).trainingLowPriority ?? DEFAULTS.trainingLowPriority,
|
||||||
trainingVideoMAEEnabled:
|
trainingVideoMAEEnabled:
|
||||||
(data as any).trainingVideoMAEEnabled ?? DEFAULTS.trainingVideoMAEEnabled,
|
(data as any).trainingVideoMAEEnabled ?? DEFAULTS.trainingVideoMAEEnabled,
|
||||||
|
trainingAutoPauseEnabled:
|
||||||
|
(data as any).trainingAutoPauseEnabled ?? DEFAULTS.trainingAutoPauseEnabled,
|
||||||
|
trainingAutoPauseCpuPercent:
|
||||||
|
(data as any).trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent,
|
||||||
|
trainingAutoPauseTemperatureC:
|
||||||
|
(data as any).trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC,
|
||||||
trainingCpuCoreCount:
|
trainingCpuCoreCount:
|
||||||
(data as any).trainingCpuCoreCount ?? DEFAULTS.trainingCpuCoreCount,
|
(data as any).trainingCpuCoreCount ?? DEFAULTS.trainingCpuCoreCount,
|
||||||
trainingEffectiveMode:
|
trainingEffectiveMode:
|
||||||
@ -1073,6 +1089,10 @@ export default function Settings({ onAssetsGenerated }: Props) {
|
|||||||
progress: Number(job.progress || 0),
|
progress: Number(job.progress || 0),
|
||||||
step: String(job.step || ''),
|
step: String(job.step || ''),
|
||||||
message: String(job.message || ''),
|
message: String(job.message || ''),
|
||||||
|
paused: Boolean(job.paused),
|
||||||
|
pauseReason: String(job.pauseReason || ''),
|
||||||
|
cpuPercent: Number(job.cpuPercent || 0),
|
||||||
|
temperatureC: Number(job.temperatureC || 0),
|
||||||
}
|
}
|
||||||
: null)
|
: null)
|
||||||
} catch {
|
} catch {
|
||||||
@ -1611,6 +1631,15 @@ export default function Settings({ onAssetsGenerated }: Props) {
|
|||||||
)
|
)
|
||||||
const trainingLowPriority = !!value.trainingLowPriority
|
const trainingLowPriority = !!value.trainingLowPriority
|
||||||
const trainingVideoMAEEnabled = value.trainingVideoMAEEnabled !== false
|
const trainingVideoMAEEnabled = value.trainingVideoMAEEnabled !== false
|
||||||
|
const trainingAutoPauseEnabled = !!value.trainingAutoPauseEnabled
|
||||||
|
const trainingAutoPauseCpuPercent = Math.max(
|
||||||
|
1,
|
||||||
|
Math.min(100, Math.floor(Number(value.trainingAutoPauseCpuPercent ?? DEFAULTS.trainingAutoPauseCpuPercent ?? 95)))
|
||||||
|
)
|
||||||
|
const trainingAutoPauseTemperatureC = Math.max(
|
||||||
|
1,
|
||||||
|
Math.min(120, Math.floor(Number(value.trainingAutoPauseTemperatureC ?? DEFAULTS.trainingAutoPauseTemperatureC ?? 85)))
|
||||||
|
)
|
||||||
|
|
||||||
setSaving(true)
|
setSaving(true)
|
||||||
try {
|
try {
|
||||||
@ -1656,6 +1685,9 @@ export default function Settings({ onAssetsGenerated }: Props) {
|
|||||||
trainingYoloBatchSize,
|
trainingYoloBatchSize,
|
||||||
trainingLowPriority,
|
trainingLowPriority,
|
||||||
trainingVideoMAEEnabled,
|
trainingVideoMAEEnabled,
|
||||||
|
trainingAutoPauseEnabled,
|
||||||
|
trainingAutoPauseCpuPercent,
|
||||||
|
trainingAutoPauseTemperatureC,
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
@ -3089,6 +3121,100 @@ export default function Settings({ onAssetsGenerated }: Props) {
|
|||||||
description="Deaktivieren spart viel CPU-Zeit. YOLO Detector und Pose können weiterhin trainiert werden."
|
description="Deaktivieren spart viel CPU-Zeit. YOLO Detector und Pose können weiterhin trainiert werden."
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="mt-3 rounded-xl border border-gray-200 bg-white p-3 dark:border-white/10 dark:bg-gray-950/40">
|
||||||
|
<LabeledSwitch
|
||||||
|
checked={!!value.trainingAutoPauseEnabled}
|
||||||
|
onChange={(checked) =>
|
||||||
|
setValue((v) => ({
|
||||||
|
...v,
|
||||||
|
trainingAutoPauseEnabled: checked,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
disabled={trainingSettingsLocked}
|
||||||
|
label="Training automatisch pausieren"
|
||||||
|
description="Pausiert laufende Python-Trainingsprozesse bei dauerhaft hoher CPU-Last oder hoher CPU-Temperatur und setzt sie nach einer kurzen Abkühlphase fort."
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div
|
||||||
|
className={[
|
||||||
|
'mt-3 grid grid-cols-1 gap-3 sm:grid-cols-2',
|
||||||
|
value.trainingAutoPauseEnabled && !trainingSettingsLocked ? '' : 'opacity-60',
|
||||||
|
].join(' ')}
|
||||||
|
>
|
||||||
|
<div>
|
||||||
|
<label
|
||||||
|
htmlFor="trainingAutoPauseCpuPercent"
|
||||||
|
className="text-sm font-medium text-gray-900 dark:text-gray-200"
|
||||||
|
>
|
||||||
|
CPU-Grenze
|
||||||
|
</label>
|
||||||
|
<div className="mt-1 flex items-center gap-2">
|
||||||
|
<input
|
||||||
|
id="trainingAutoPauseCpuPercent"
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
max={100}
|
||||||
|
step={1}
|
||||||
|
disabled={!value.trainingAutoPauseEnabled || trainingSettingsLocked}
|
||||||
|
value={value.trainingAutoPauseCpuPercent ?? 95}
|
||||||
|
onChange={(e) =>
|
||||||
|
setValue((v) => ({
|
||||||
|
...v,
|
||||||
|
trainingAutoPauseCpuPercent: Number(e.target.value || 95),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100"
|
||||||
|
/>
|
||||||
|
<span className="shrink-0 text-xs text-gray-600 dark:text-gray-300">
|
||||||
|
%
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label
|
||||||
|
htmlFor="trainingAutoPauseTemperatureC"
|
||||||
|
className="text-sm font-medium text-gray-900 dark:text-gray-200"
|
||||||
|
>
|
||||||
|
Temperatur-Grenze
|
||||||
|
</label>
|
||||||
|
<div className="mt-1 flex items-center gap-2">
|
||||||
|
<input
|
||||||
|
id="trainingAutoPauseTemperatureC"
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
max={120}
|
||||||
|
step={1}
|
||||||
|
disabled={!value.trainingAutoPauseEnabled || trainingSettingsLocked}
|
||||||
|
value={value.trainingAutoPauseTemperatureC ?? 85}
|
||||||
|
onChange={(e) =>
|
||||||
|
setValue((v) => ({
|
||||||
|
...v,
|
||||||
|
trainingAutoPauseTemperatureC: Number(e.target.value || 85),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
className="h-9 w-full rounded-md border border-gray-200 bg-white px-3 text-sm text-gray-900 shadow-sm disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/10 dark:bg-gray-900 dark:text-gray-100"
|
||||||
|
/>
|
||||||
|
<span className="shrink-0 text-xs text-gray-600 dark:text-gray-300">
|
||||||
|
°C
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{trainingJobStatus?.paused ? (
|
||||||
|
<div className="mt-3 rounded-lg border border-amber-200 bg-amber-50 px-3 py-2 text-xs font-medium text-amber-900 dark:border-amber-400/30 dark:bg-amber-500/10 dark:text-amber-100">
|
||||||
|
Training pausiert: {trainingJobStatus.pauseReason || 'Ressourcenlimit erreicht'}
|
||||||
|
{trainingJobStatus.cpuPercent ? ` · CPU ${Math.round(trainingJobStatus.cpuPercent)}%` : ''}
|
||||||
|
{trainingJobStatus.temperatureC ? ` · ${Math.round(trainingJobStatus.temperatureC)}°C` : ''}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="mt-2 text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
Temperatur greift nur, wenn der Server passende Sensorwerte liefert.
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@ -14,6 +14,7 @@ import {
|
|||||||
CheckIcon,
|
CheckIcon,
|
||||||
ForwardIcon,
|
ForwardIcon,
|
||||||
InboxArrowDownIcon,
|
InboxArrowDownIcon,
|
||||||
|
PauseIcon,
|
||||||
RectangleGroupIcon,
|
RectangleGroupIcon,
|
||||||
UserGroupIcon,
|
UserGroupIcon,
|
||||||
TrashIcon,
|
TrashIcon,
|
||||||
@ -106,6 +107,10 @@ type TrainingJobStatus = {
|
|||||||
epoch?: number
|
epoch?: number
|
||||||
epochs?: number
|
epochs?: number
|
||||||
previewUrl?: string
|
previewUrl?: string
|
||||||
|
paused?: boolean
|
||||||
|
pauseReason?: string
|
||||||
|
cpuPercent?: number
|
||||||
|
temperatureC?: number
|
||||||
map50?: number
|
map50?: number
|
||||||
map5095?: number
|
map5095?: number
|
||||||
}
|
}
|
||||||
@ -260,6 +265,9 @@ type TrainingHistoryEntry = {
|
|||||||
workers?: number
|
workers?: number
|
||||||
yoloBatchSize?: number
|
yoloBatchSize?: number
|
||||||
lowPriority?: boolean
|
lowPriority?: boolean
|
||||||
|
autoPauseEnabled?: boolean
|
||||||
|
autoPauseCpuPercent?: number
|
||||||
|
autoPauseTemperatureC?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
type TrainingStats = {
|
type TrainingStats = {
|
||||||
@ -327,6 +335,7 @@ type TrainingEstimateRuntime = {
|
|||||||
workers: number
|
workers: number
|
||||||
yoloBatchLabel: string
|
yoloBatchLabel: string
|
||||||
lowPriority: boolean
|
lowPriority: boolean
|
||||||
|
autoPause: boolean
|
||||||
factor: number
|
factor: number
|
||||||
yoloFactor: number
|
yoloFactor: number
|
||||||
}
|
}
|
||||||
@ -718,6 +727,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
|
|||||||
const lowPriority = Boolean(
|
const lowPriority = Boolean(
|
||||||
settings?.trainingEffectiveLowPriority ?? settings?.trainingLowPriority
|
settings?.trainingEffectiveLowPriority ?? settings?.trainingLowPriority
|
||||||
)
|
)
|
||||||
|
const autoPause = Boolean(settings?.trainingAutoPauseEnabled)
|
||||||
const autoThreads = cpuCores > 0
|
const autoThreads = cpuCores > 0
|
||||||
? Math.max(1, Math.min(16, Math.round(cpuCores * 0.75)))
|
? Math.max(1, Math.min(16, Math.round(cpuCores * 0.75)))
|
||||||
: 4
|
: 4
|
||||||
@ -769,11 +779,12 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
|
|||||||
? 0.98
|
? 0.98
|
||||||
: 1
|
: 1
|
||||||
const lowPriorityFactor = lowPriority ? 1.12 : 1
|
const lowPriorityFactor = lowPriority ? 1.12 : 1
|
||||||
|
const autoPauseFactor = autoPause ? 1.12 : 1
|
||||||
const corePenalty =
|
const corePenalty =
|
||||||
cpuCores > 0 && effectiveThreads > cpuCores
|
cpuCores > 0 && effectiveThreads > cpuCores
|
||||||
? 1 + Math.min(0.3, (effectiveThreads - cpuCores) * 0.04)
|
? 1 + Math.min(0.3, (effectiveThreads - cpuCores) * 0.04)
|
||||||
: 1
|
: 1
|
||||||
const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * corePenalty
|
const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * autoPauseFactor * corePenalty
|
||||||
const yoloFactor = factor * batchFactor
|
const yoloFactor = factor * batchFactor
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@ -783,6 +794,7 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
|
|||||||
workers,
|
workers,
|
||||||
yoloBatchLabel: rawBatch > 0 ? String(rawBatch) : 'Auto',
|
yoloBatchLabel: rawBatch > 0 ? String(rawBatch) : 'Auto',
|
||||||
lowPriority,
|
lowPriority,
|
||||||
|
autoPause,
|
||||||
factor: Number.isFinite(factor) && factor > 0 ? factor : 1,
|
factor: Number.isFinite(factor) && factor > 0 ? factor : 1,
|
||||||
yoloFactor: Number.isFinite(yoloFactor) && yoloFactor > 0 ? yoloFactor : 1,
|
yoloFactor: Number.isFinite(yoloFactor) && yoloFactor > 0 ? yoloFactor : 1,
|
||||||
}
|
}
|
||||||
@ -790,7 +802,8 @@ function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState |
|
|||||||
|
|
||||||
function trainingEstimateRuntimeText(runtime: TrainingEstimateRuntime) {
|
function trainingEstimateRuntimeText(runtime: TrainingEstimateRuntime) {
|
||||||
const priority = runtime.lowPriority ? ' · niedrige Priorität' : ''
|
const priority = runtime.lowPriority ? ' · niedrige Priorität' : ''
|
||||||
return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}`
|
const autoPause = runtime.autoPause ? ' · Auto-Pause' : ''
|
||||||
|
return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}${autoPause}`
|
||||||
}
|
}
|
||||||
|
|
||||||
function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) {
|
function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) {
|
||||||
@ -802,6 +815,7 @@ function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) {
|
|||||||
trainingEffectiveWorkers: entry.workers,
|
trainingEffectiveWorkers: entry.workers,
|
||||||
trainingEffectiveYoloBatchSize: entry.yoloBatchSize,
|
trainingEffectiveYoloBatchSize: entry.yoloBatchSize,
|
||||||
trainingEffectiveLowPriority: entry.lowPriority,
|
trainingEffectiveLowPriority: entry.lowPriority,
|
||||||
|
trainingAutoPauseEnabled: entry.autoPauseEnabled,
|
||||||
} as RecorderSettingsState)
|
} as RecorderSettingsState)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -4644,6 +4658,10 @@ export default function TrainingTab(props: {
|
|||||||
epoch: Number(job.epoch ?? 0),
|
epoch: Number(job.epoch ?? 0),
|
||||||
epochs: Number(job.epochs ?? 0),
|
epochs: Number(job.epochs ?? 0),
|
||||||
previewUrl: String(job.previewUrl ?? ''),
|
previewUrl: String(job.previewUrl ?? ''),
|
||||||
|
paused: Boolean(job.paused),
|
||||||
|
pauseReason: String(job.pauseReason ?? ''),
|
||||||
|
cpuPercent: Number(job.cpuPercent ?? 0),
|
||||||
|
temperatureC: Number(job.temperatureC ?? 0),
|
||||||
map50: Number(job.map50 ?? 0),
|
map50: Number(job.map50 ?? 0),
|
||||||
map5095: Number(job.map5095 ?? 0),
|
map5095: Number(job.map5095 ?? 0),
|
||||||
}
|
}
|
||||||
@ -5125,6 +5143,10 @@ export default function TrainingTab(props: {
|
|||||||
const importVideoIntoTraining = useCallback(async (raw: any) => {
|
const importVideoIntoTraining = useCallback(async (raw: any) => {
|
||||||
const output = String(raw?.output || '').trim()
|
const output = String(raw?.output || '').trim()
|
||||||
if (!output) return false
|
if (!output) return false
|
||||||
|
if (trainingRunning) {
|
||||||
|
setMessage('Während das Training läuft, kann kein Video ins Training übernommen werden.')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
setLoadingPreviewCandidate('')
|
setLoadingPreviewCandidate('')
|
||||||
|
|
||||||
@ -5277,6 +5299,7 @@ export default function TrainingTab(props: {
|
|||||||
loadPriorityTrainingSamples,
|
loadPriorityTrainingSamples,
|
||||||
loadTrainingStatus,
|
loadTrainingStatus,
|
||||||
setLoadingPreviewCandidate,
|
setLoadingPreviewCandidate,
|
||||||
|
trainingRunning,
|
||||||
waitForVideoImportResult,
|
waitForVideoImportResult,
|
||||||
])
|
])
|
||||||
|
|
||||||
@ -7243,13 +7266,25 @@ export default function TrainingTab(props: {
|
|||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
const trainingPaused = Boolean(trainingStatus?.training?.paused)
|
||||||
|
const pauseReason = String(trainingStatus?.training?.pauseReason || '').trim()
|
||||||
|
const pauseMetrics = [
|
||||||
|
trainingStatus?.training?.cpuPercent
|
||||||
|
? `CPU ${Math.round(Number(trainingStatus.training.cpuPercent))}%`
|
||||||
|
: '',
|
||||||
|
trainingStatus?.training?.temperatureC
|
||||||
|
? `${Math.round(Number(trainingStatus.training.temperatureC))}°C`
|
||||||
|
: '',
|
||||||
|
].filter(Boolean).join(' · ')
|
||||||
const progress = clampPercent(
|
const progress = clampPercent(
|
||||||
trainingRunning
|
trainingRunning
|
||||||
? shownTrainingProgress
|
? shownTrainingProgress
|
||||||
: readinessProgress * 100
|
: readinessProgress * 100
|
||||||
)
|
)
|
||||||
const statusText = trainingRunning
|
const statusText = trainingRunning
|
||||||
? shownTrainingStep || 'Training läuft…'
|
? trainingPaused
|
||||||
|
? `Pausiert${pauseReason ? `: ${pauseReason}` : ''}${pauseMetrics ? ` · ${pauseMetrics}` : ''}`
|
||||||
|
: shownTrainingStep || 'Training läuft…'
|
||||||
: !feedbackReady
|
: !feedbackReady
|
||||||
? `${Math.max(0, requiredCount - feedbackCount)} Feedback fehlen noch`
|
? `${Math.max(0, requiredCount - feedbackCount)} Feedback fehlen noch`
|
||||||
: canStartTraining
|
: canStartTraining
|
||||||
@ -7280,14 +7315,18 @@ export default function TrainingTab(props: {
|
|||||||
<div
|
<div
|
||||||
className={[
|
className={[
|
||||||
'flex h-8 w-8 shrink-0 items-center justify-center rounded-lg ring-1',
|
'flex h-8 w-8 shrink-0 items-center justify-center rounded-lg ring-1',
|
||||||
trainingRunning
|
trainingPaused
|
||||||
|
? 'bg-amber-50 text-amber-700 ring-amber-200 dark:bg-amber-500/15 dark:text-amber-200 dark:ring-amber-400/30'
|
||||||
|
: trainingRunning
|
||||||
? 'bg-indigo-50 text-indigo-700 ring-indigo-200 dark:bg-indigo-500/15 dark:text-indigo-200 dark:ring-indigo-400/30'
|
? 'bg-indigo-50 text-indigo-700 ring-indigo-200 dark:bg-indigo-500/15 dark:text-indigo-200 dark:ring-indigo-400/30'
|
||||||
: canStartTraining
|
: canStartTraining
|
||||||
? 'bg-emerald-50 text-emerald-700 ring-emerald-200 dark:bg-emerald-500/15 dark:text-emerald-200 dark:ring-emerald-400/30'
|
? 'bg-emerald-50 text-emerald-700 ring-emerald-200 dark:bg-emerald-500/15 dark:text-emerald-200 dark:ring-emerald-400/30'
|
||||||
: 'bg-gray-50 text-gray-600 ring-gray-200 dark:bg-white/5 dark:text-gray-300 dark:ring-white/10',
|
: 'bg-gray-50 text-gray-600 ring-gray-200 dark:bg-white/5 dark:text-gray-300 dark:ring-white/10',
|
||||||
].join(' ')}
|
].join(' ')}
|
||||||
>
|
>
|
||||||
{trainingRunning ? (
|
{trainingPaused ? (
|
||||||
|
<PauseIcon className="h-4 w-4" aria-hidden="true" />
|
||||||
|
) : trainingRunning ? (
|
||||||
<ArrowPathIcon className="h-4 w-4 animate-spin" aria-hidden="true" />
|
<ArrowPathIcon className="h-4 w-4 animate-spin" aria-hidden="true" />
|
||||||
) : (
|
) : (
|
||||||
<BoltIcon className="h-4 w-4" aria-hidden="true" />
|
<BoltIcon className="h-4 w-4" aria-hidden="true" />
|
||||||
@ -7394,7 +7433,9 @@ export default function TrainingTab(props: {
|
|||||||
<div
|
<div
|
||||||
className={[
|
className={[
|
||||||
'h-full rounded-full transition-all duration-500',
|
'h-full rounded-full transition-all duration-500',
|
||||||
trainingRunning
|
trainingPaused
|
||||||
|
? 'bg-amber-500'
|
||||||
|
: trainingRunning
|
||||||
? 'bg-indigo-500'
|
? 'bg-indigo-500'
|
||||||
: canStartTraining
|
: canStartTraining
|
||||||
? 'bg-emerald-500'
|
? 'bg-emerald-500'
|
||||||
|
|||||||
@ -289,6 +289,9 @@ export type RecorderSettingsState = {
|
|||||||
trainingYoloBatchSize?: number
|
trainingYoloBatchSize?: number
|
||||||
trainingLowPriority?: boolean
|
trainingLowPriority?: boolean
|
||||||
trainingVideoMAEEnabled?: boolean
|
trainingVideoMAEEnabled?: boolean
|
||||||
|
trainingAutoPauseEnabled?: boolean
|
||||||
|
trainingAutoPauseCpuPercent?: number
|
||||||
|
trainingAutoPauseTemperatureC?: number
|
||||||
trainingCpuCoreCount?: number
|
trainingCpuCoreCount?: number
|
||||||
trainingEffectiveMode?: string
|
trainingEffectiveMode?: string
|
||||||
trainingEffectiveCpuThreads?: number
|
trainingEffectiveCpuThreads?: number
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user