diff --git a/backend/dist/nsfwapp-linux-amd64 b/backend/dist/nsfwapp-linux-amd64 index e3e551c..bce836d 100644 Binary files a/backend/dist/nsfwapp-linux-amd64 and b/backend/dist/nsfwapp-linux-amd64 differ diff --git a/backend/dist/nsfwapp.exe b/backend/dist/nsfwapp.exe index 9eb33f6..fd73935 100644 Binary files a/backend/dist/nsfwapp.exe and b/backend/dist/nsfwapp.exe differ diff --git a/backend/dist/nsfwapp_amd64.deb b/backend/dist/nsfwapp_amd64.deb index e94277d..ec46339 100644 Binary files a/backend/dist/nsfwapp_amd64.deb and b/backend/dist/nsfwapp_amd64.deb differ diff --git a/backend/ml/train_detector_model.py b/backend/ml/train_detector_model.py index 03a2115..db3f1ee 100644 --- a/backend/ml/train_detector_model.py +++ b/backend/ml/train_detector_model.py @@ -385,6 +385,7 @@ def main(): "ok": True, "model": str(final_model), "sourceModel": str(best), + "baseModel": str(base_used), "runs": str(result_path), "trainedAt": datetime.now(timezone.utc).isoformat(), "trainSamples": train_count, @@ -392,6 +393,7 @@ def main(): "epochs": epochs, "imgsz": imgsz, "batchSize": batch_size, + "patience": patience, "threads": threads, "workers": workers, "device": str(train_device), diff --git a/backend/ml/train_pose_model.py b/backend/ml/train_pose_model.py index 46e6853..2c8c6b2 100644 --- a/backend/ml/train_pose_model.py +++ b/backend/ml/train_pose_model.py @@ -371,6 +371,7 @@ def main(): "ok": True, "model": str(final_model), "sourceModel": str(best), + "baseModel": str(base_model), "runs": str(result_path), "trainedAt": datetime.now(timezone.utc).isoformat(), "trainSamples": train_count, @@ -378,6 +379,7 @@ def main(): "epochs": epochs, "imgsz": imgsz, "batchSize": batch_size, + "patience": patience, "threads": threads, "workers": workers, "device": str(train_device), diff --git a/backend/ml/train_videomae_model.py b/backend/ml/train_videomae_model.py index c5ad8a3..fc14fe8 100644 --- a/backend/ml/train_videomae_model.py +++ b/backend/ml/train_videomae_model.py @@ -200,6 +200,7 @@ def main(): parser.add_argument("--workers", default="0") parser.add_argument("--threads", default="0") parser.add_argument("--num-frames", default="16") + parser.add_argument("--patience", default="3") parser.add_argument("--freeze-backbone", action="store_true") args = parser.parse_args() @@ -213,6 +214,7 @@ def main(): workers = max(0, safe_int(args.workers, 0)) threads = max(0, safe_int(args.threads, 0)) num_frames = max(2, safe_int(args.num_frames, 16)) + patience = max(0, safe_int(args.patience, 3)) lr = max(1e-7, safe_float(args.lr, 5e-5)) if threads > 0: @@ -249,18 +251,23 @@ def main(): if not val_entries: raise SystemExit("no VideoMAE val clips found") + base_model = str(args.base or DEFAULT_BASE_MODEL).strip() or DEFAULT_BASE_MODEL + base_path = Path(base_model).expanduser() + if base_path.exists(): + base_model = str(base_path.resolve()) + emit_progress( "videomae", 0.03, "VideoMAE-Basismodell wird geladen...", - base=args.base, + base=base_model, labels=len(labels), device=str(device), ) - image_processor = AutoImageProcessor.from_pretrained(args.base) + image_processor = AutoImageProcessor.from_pretrained(base_model) model = VideoMAEForVideoClassification.from_pretrained( - args.base, + base_model, num_labels=len(labels), label2id=label_to_id, id2label=id_to_label, @@ -302,8 +309,11 @@ def main(): best_accuracy = -1.0 best_loss = math.inf best_epoch = 0 + completed_epochs = 0 + epochs_without_improvement = 0 for epoch in range(1, epochs + 1): + completed_epochs = epoch model.train() running_loss = 0.0 seen = 0 @@ -346,8 +356,11 @@ def main(): best_accuracy = val_accuracy best_loss = val_loss best_epoch = epoch + epochs_without_improvement = 0 model.save_pretrained(tmp_dir) image_processor.save_pretrained(tmp_dir) + else: + epochs_without_improvement += 1 emit_progress( "videomae", @@ -360,16 +373,36 @@ def main(): device=str(device), accuracy=val_accuracy, loss=val_loss, + patience=patience, + epochsWithoutImprovement=epochs_without_improvement, ) + if patience > 0 and epochs_without_improvement >= patience: + emit_progress( + "videomae", + 0.96, + "VideoMAE Early-Stopping: keine weitere Verbesserung.", + epoch=epoch, + epochs=epochs, + bestEpoch=best_epoch, + patience=patience, + trainSamples=len(train_entries), + valSamples=len(val_entries), + device=str(device), + accuracy=best_accuracy if best_accuracy >= 0 else 0.0, + loss=best_loss if math.isfinite(best_loss) else 0.0, + ) + break + if best_epoch <= 0: model.save_pretrained(tmp_dir) image_processor.save_pretrained(tmp_dir) - best_epoch = epochs + best_epoch = completed_epochs or epochs status = { "trainedAt": datetime.now(timezone.utc).isoformat(), "epochs": epochs, + "completedEpochs": completed_epochs, "bestEpoch": best_epoch, "trainSamples": len(train_entries), "valSamples": len(val_entries), @@ -377,7 +410,8 @@ def main(): "workers": workers, "threads": threads, "numFrames": num_frames, - "baseModel": args.base, + "patience": patience, + "baseModel": base_model, "device": str(device), "accuracy": best_accuracy if best_accuracy >= 0 else 0.0, "loss": best_loss if math.isfinite(best_loss) else 0.0, diff --git a/backend/training.go b/backend/training.go index 91edbc5..0275429 100644 --- a/backend/training.go +++ b/backend/training.go @@ -118,6 +118,103 @@ type TrainingSkipRequest struct { SampleID string `json:"sampleId"` } +type TrainingTrainRequest struct { + Scope string `json:"scope,omitempty"` + Targets []string `json:"targets,omitempty"` +} + +type trainingTrainTargets struct { + Detector bool + Pose bool + VideoMAE bool +} + +func trainingAllTrainTargets() trainingTrainTargets { + return trainingTrainTargets{ + Detector: true, + Pose: true, + VideoMAE: true, + } +} + +func (t trainingTrainTargets) empty() bool { + return !t.Detector && !t.Pose && !t.VideoMAE +} + +func (t trainingTrainTargets) list() []string { + out := make([]string, 0, 3) + if t.Detector { + out = append(out, "detector") + } + if t.Pose { + out = append(out, "pose") + } + if t.VideoMAE { + out = append(out, "videomae") + } + return out +} + +func trainingReadTrainRequest(r *http.Request) (TrainingTrainRequest, error) { + var req TrainingTrainRequest + if r.Body == nil { + return req, nil + } + + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + return req, err + } + + if strings.TrimSpace(string(body)) == "" { + return req, nil + } + + if err := json.Unmarshal(body, &req); err != nil { + return req, err + } + + return req, nil +} + +func trainingNormalizeTrainTargets(req TrainingTrainRequest) (trainingTrainTargets, bool, error) { + scope := strings.ToLower(strings.TrimSpace(req.Scope)) + if scope == "" || scope == "full" || scope == "all" || scope == "complete" { + return trainingAllTrainTargets(), false, nil + } + + if scope != "custom" && scope != "selected" && scope != "partial" { + return trainingTrainTargets{}, false, fmt.Errorf("unbekannter Trainingsumfang: %s", req.Scope) + } + + var targets trainingTrainTargets + for _, raw := range req.Targets { + key := strings.ToLower(strings.TrimSpace(raw)) + key = strings.ReplaceAll(key, "_", "") + key = strings.ReplaceAll(key, "-", "") + key = strings.ReplaceAll(key, " ", "") + + switch key { + case "detector", "yolo", "yolo26", "yolo26detector", "box", "boxes", "boxdetection": + targets.Detector = true + case "pose", "yolo26pose", "posedetection": + targets.Pose = true + case "videomae", "video", "clip", "scene", "clipanalysis", "clipanalyse": + targets.VideoMAE = true + case "": + continue + default: + return trainingTrainTargets{}, true, fmt.Errorf("unbekanntes Training: %s", raw) + } + } + + if targets.empty() { + return trainingTrainTargets{}, true, errors.New("kein Training ausgewählt") + } + + return targets, true, nil +} + type TrainingAnnotation struct { SampleID string `json:"sampleId"` FrameURL string `json:"frameUrl"` @@ -1313,6 +1410,21 @@ func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRunt return opts } +func trainingYoloEarlyStoppingPatience(epochs int) int { + epochs = clampTrainingInt(epochs, 1, 300) + patience := epochs / 4 + if patience < 5 { + patience = 5 + } + if patience > 20 { + patience = 20 + } + if patience > epochs { + patience = epochs + } + return patience +} + func trainingCommandEnv(opts trainingRuntimeOptions) []string { env := os.Environ() if opts.CPUThreads <= 0 { @@ -3212,6 +3324,18 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) { return } + req, err := trainingReadTrainRequest(r) + if err != nil { + trainingWriteError(w, http.StatusBadRequest, "invalid json") + return + } + + targets, customTargets, err := trainingNormalizeTrainTargets(req) + if err != nil { + trainingWriteError(w, http.StatusBadRequest, err.Error()) + return + } + current := trainingGetJobStatus() if current.Running { trainingWriteJSON(w, http.StatusOK, map[string]any{ @@ -3301,7 +3425,70 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) { videoMAEDataReady := videoMAEEligibleCount >= minVideoMAETrainCount || (videoMAETrainCount >= minVideoMAETrainCount && videoMAEValCount >= minVideoMAEValCount) - if detectorDataReady || poseDataReady || videoMAEDataReady { + runtimeOpts := trainingRuntimeOptionsFromSettings() + selectedDataReady := + (targets.Detector && detectorDataReady) || + (targets.Pose && poseDataReady) || + (targets.VideoMAE && videoMAEDataReady) + + if customTargets { + if targets.Detector && !detectorDataReady { + trainingWriteError( + w, + http.StatusBadRequest, + fmt.Sprintf( + "YOLO26 Detector ist noch nicht trainingsbereit. Train=%d (%d positiv), Val=%d (%d positiv). Benoetigt: mindestens %d Train, %d Val und je ein positives Beispiel.", + trainCount, + positiveTrainCount, + valCount, + positiveValCount, + minDetectorTrainCount, + minDetectorValCount, + ), + ) + return + } + + if targets.Pose && !poseDataReady { + trainingWriteError( + w, + http.StatusBadRequest, + fmt.Sprintf( + "YOLO26 Pose ist noch nicht trainingsbereit. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.", + poseTrainCount, + poseValCount, + minPoseTrainCount, + minPoseValCount, + ), + ) + return + } + + if targets.VideoMAE && !runtimeOpts.VideoMAEEnabled { + trainingWriteError(w, http.StatusBadRequest, "VideoMAE ist in den Training-Settings deaktiviert.") + return + } + + if targets.VideoMAE && !videoMAEDataReady { + trainingWriteError( + w, + http.StatusBadRequest, + fmt.Sprintf( + "VideoMAE ist noch nicht trainingsbereit. Eligible=%d, Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.", + videoMAEEligibleCount, + videoMAETrainCount, + videoMAEValCount, + minVideoMAETrainCount, + minVideoMAEValCount, + ), + ) + return + } + + goto startTraining + } + + if selectedDataReady { goto startTraining } @@ -3348,12 +3535,13 @@ startTraining: trainingStartJob(cancel) - go trainingRunJob(ctx, root, feedbackCount) + go trainingRunJob(ctx, root, feedbackCount, targets) trainingWriteJSON(w, http.StatusAccepted, map[string]any{ "ok": true, "message": "Training gestartet.", "training": trainingGetJobStatus(), + "targets": targets.list(), "detector": map[string]any{ "trainCount": trainCount, "valCount": valCount, @@ -3382,6 +3570,7 @@ startTraining: "requiredTrain": minVideoMAETrainCount, "requiredVal": minVideoMAEValCount, "manifest": videoMAEManifest, + "dataReady": videoMAEDataReady, "source": "videomae_clip", }, }) @@ -3424,7 +3613,7 @@ func trainingCancelHandler(w http.ResponseWriter, r *http.Request) { }) } -func trainingRunJob(ctx context.Context, root string, count int) { +func trainingRunJob(ctx context.Context, root string, count int, targets trainingTrainTargets) { if err := ensureMLPythonSetup(ctx); err != nil { if errors.Is(err, context.Canceled) { trainingFinishCancelled(root) @@ -3459,7 +3648,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { appLogln("ML-Python für Training:", python) runtimeOpts := trainingRuntimeOptionsFromSettings() appLogf( - "Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v", + "Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v targets=%s", runtimeOpts.PerformanceMode, runtimeOpts.CPUCoreCount, runtimeOpts.PowerSaveMode, @@ -3468,6 +3657,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { runtimeOpts.YoloBatchSize, runtimeOpts.LowPriority, runtimeOpts.VideoMAEEnabled, + strings.Join(targets.list(), ","), ) cleanOutput := func(text string) string { @@ -3485,10 +3675,13 @@ func trainingRunJob(ctx context.Context, root string, count int) { detectorOutput := "" detectorStatus := "skipped" + detectorDurationMs := int64(0) poseOutput := "" poseStatus := "skipped" + poseDurationMs := int64(0) videoMAEOutput := "" videoMAEStatus := "skipped" + videoMAEDurationMs := int64(0) detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml") detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train") @@ -3514,29 +3707,50 @@ func trainingRunJob(ctx context.Context, root string, count int) { fileExistsNonEmpty(detectorDatasetYAML), ) - if fileExistsNonEmpty(detectorDatasetYAML) && + if !targets.Detector { + trainingSetJobStatus(func(s *TrainingJobStatus) { + if s.Progress < 58 { + s.Progress = 58 + } + s.Step = "YOLO26 Detector wurde nicht ausgewählt." + }) + detectorStatus = "skipped_unselected" + detectorOutput = "YOLO26 Detector übersprungen: nicht ausgewählt." + appLogln(detectorOutput) + } else if fileExistsNonEmpty(detectorDatasetYAML) && trainCount >= minDetectorTrainCount && valCount >= minDetectorValCount && positiveTrainCount > 0 && positiveValCount > 0 { + detectorStartedAt := time.Now() trainingSetJobStatus(func(s *TrainingJobStatus) { s.Progress = 15 s.Step = "YOLO26 Detector wird trainiert…" }) - detectorBasePath := "yolo26n.pt" - if p, err := trainingMLCacheFilePath("yolo26n.pt"); err == nil { - detectorBasePath = p + detectorModel := trainingResolveDetectorModel(root) + detectorBasePath := detectorModel.BestPath + if !detectorModel.TrainedExists { + detectorBasePath = "yolo26n.pt" + if p, err := trainingMLCacheFilePath("yolo26n.pt"); err == nil { + detectorBasePath = p + } + } + detectorEpochs := trainingDetectorEpochs() + detectorPatience := trainingYoloEarlyStoppingPatience(detectorEpochs) + if detectorModel.TrainedExists { + appLogln("YOLO26 Detector Fine-Tuning startet von:", detectorBasePath) } detectorScript := trainingScriptPath("train_detector_model.py") detectorArgs := []string{ "--root", root, "--base", detectorBasePath, - "--epochs", strconv.Itoa(trainingDetectorEpochs()), + "--epochs", strconv.Itoa(detectorEpochs), "--imgsz", "640", "--workers", strconv.Itoa(runtimeOpts.Workers), "--threads", strconv.Itoa(runtimeOpts.CPUThreads), + "--patience", strconv.Itoa(detectorPatience), } if runtimeOpts.YoloBatchSize > 0 { detectorArgs = append(detectorArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize)) @@ -3556,6 +3770,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { }, detectorArgs..., ) + detectorDurationMs = time.Since(detectorStartedAt).Milliseconds() if errors.Is(detectorErr, errTrainingCancelled) { appLogln("⛔ YOLO26 detector training cancelled") @@ -3601,24 +3816,38 @@ func trainingRunJob(ctx context.Context, root string, count int) { poseTrainLabels := filepath.Join(root, "pose", "dataset", "labels", "train") poseValImages := filepath.Join(root, "pose", "dataset", "images", "val") poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val") + poseStartedAt := time.Time{} - trainingSetJobStatus(func(s *TrainingJobStatus) { - if s.Progress < 60 { - s.Progress = 60 - } - s.Step = "YOLO26 Pose-Daten werden aufgebaut..." - }) - - if written, err := trainingSyncPoseDataset(root); err != nil { - poseStatus = "failed" - poseOutput = "YOLO26 Pose-Dataset konnte nicht aufgebaut werden: " + err.Error() + if !targets.Pose { + trainingSetJobStatus(func(s *TrainingJobStatus) { + if s.Progress < 82 { + s.Progress = 82 + } + s.Step = "YOLO26 Pose wurde nicht ausgewählt." + }) + poseStatus = "skipped_unselected" + poseOutput = "YOLO26 Pose übersprungen: nicht ausgewählt." appLogln(poseOutput) } else { - appLogln("pose samples synced:", written) - } + poseStartedAt = time.Now() + trainingSetJobStatus(func(s *TrainingJobStatus) { + if s.Progress < 60 { + s.Progress = 60 + } + s.Step = "YOLO26 Pose-Daten werden aufgebaut..." + }) - if err := trainingEnsurePoseValidationSample(root); err != nil { - appLogln("pose val sample ensure failed:", err) + if written, err := trainingSyncPoseDataset(root); err != nil { + poseStatus = "failed" + poseOutput = "YOLO26 Pose-Dataset konnte nicht aufgebaut werden: " + err.Error() + appLogln(poseOutput) + } else { + appLogln("pose samples synced:", written) + } + + if err := trainingEnsurePoseValidationSample(root); err != nil { + appLogln("pose val sample ensure failed:", err) + } } poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels) @@ -3631,7 +3860,8 @@ func trainingRunJob(ctx context.Context, root string, count int) { fileExistsNonEmpty(poseDatasetYAML), ) - if poseStatus != "failed" && + if targets.Pose && + poseStatus != "failed" && fileExistsNonEmpty(poseDatasetYAML) && poseTrainCount >= minPoseTrainCount && poseValCount >= minPoseValCount { @@ -3642,19 +3872,26 @@ func trainingRunJob(ctx context.Context, root string, count int) { s.Step = "YOLO26 Pose wird trainiert..." }) + poseModel := trainingResolvePoseModel(root) poseBasePath := "yolo26n-pose.pt" - if p, err := embeddedPoseModelPath(); err == nil && fileExistsNonEmpty(p) { - poseBasePath = p + if poseModel.EffectiveExists { + poseBasePath = poseModel.EffectivePath + } + poseEpochs := trainingDetectorEpochs() + posePatience := trainingYoloEarlyStoppingPatience(poseEpochs) + if poseModel.TrainedExists { + appLogln("YOLO26 Pose Fine-Tuning startet von:", poseBasePath) } poseScript := trainingScriptPath("train_pose_model.py") poseArgs := []string{ "--root", root, "--base", poseBasePath, - "--epochs", strconv.Itoa(trainingDetectorEpochs()), + "--epochs", strconv.Itoa(poseEpochs), "--imgsz", "640", "--workers", strconv.Itoa(runtimeOpts.Workers), "--threads", strconv.Itoa(runtimeOpts.CPUThreads), + "--patience", strconv.Itoa(posePatience), } if runtimeOpts.YoloBatchSize > 0 { poseArgs = append(poseArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize)) @@ -3674,6 +3911,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { }, poseArgs..., ) + poseDurationMs = time.Since(poseStartedAt).Milliseconds() if errors.Is(poseErr, errTrainingCancelled) { appLogln("YOLO26 pose training cancelled") @@ -3698,7 +3936,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { appLogln("YOLO26 pose training:", poseOutputClean) } } - } else if poseStatus != "failed" { + } else if targets.Pose && poseStatus != "failed" { poseStatus = "skipped_no_pose_data" poseOutput = fmt.Sprintf( "YOLO26 Pose übersprungen: zu wenige Skeleton-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.", @@ -3710,9 +3948,26 @@ func trainingRunJob(ctx context.Context, root string, count int) { appLogln(poseOutput) } - poseOutputClean := cleanOutput(poseOutput) + if poseStatus == "trained" && poseDurationMs <= 0 && !poseStartedAt.IsZero() { + poseDurationMs = time.Since(poseStartedAt).Milliseconds() + } - if !runtimeOpts.VideoMAEEnabled { + poseOutputClean := cleanOutput(poseOutput) + videoMAEStartedAt := time.Time{} + videoMAETrainCount := 0 + videoMAEValCount := 0 + + if !targets.VideoMAE { + trainingSetJobStatus(func(s *TrainingJobStatus) { + if s.Progress < 98 { + s.Progress = 98 + } + s.Step = "VideoMAE wurde nicht ausgewählt." + }) + videoMAEStatus = "skipped_unselected" + videoMAEOutput = "VideoMAE übersprungen: nicht ausgewählt." + appLogln(videoMAEOutput) + } else if !runtimeOpts.VideoMAEEnabled { trainingSetJobStatus(func(s *TrainingJobStatus) { if s.Progress < 84 { s.Progress = 84 @@ -3723,6 +3978,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { videoMAEOutput = "VideoMAE übersprungen: in den Training-Settings deaktiviert." appLogln(videoMAEOutput) } else { + videoMAEStartedAt = time.Now() trainingSetJobStatus(func(s *TrainingJobStatus) { if s.Progress < 84 { s.Progress = 84 @@ -3730,7 +3986,9 @@ func trainingRunJob(ctx context.Context, root string, count int) { s.Step = "VideoMAE Clip-Daten werden aufgebaut..." }) - videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr := + var videoMAEWritten int + var videoMAESyncErr error + videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr = trainingSyncVideoMAEDataset(ctx, root) if errors.Is(videoMAESyncErr, context.Canceled) || errors.Is(videoMAESyncErr, errTrainingCancelled) { appLogln("VideoMAE dataset sync cancelled") @@ -3761,6 +4019,13 @@ func trainingRunJob(ctx context.Context, root string, count int) { }) videoMAEScript := trainingScriptPath("train_videomae_model.py") + videoMAEBase := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL")) + if videoMAEBase == "" { + if model := trainingResolveVideoMAEModel(root); model.TrainedExists { + videoMAEBase = model.EffectivePath + appLogln("VideoMAE Fine-Tuning startet von:", videoMAEBase) + } + } videoMAEArgs := []string{ "--root", root, "--epochs", strconv.Itoa(trainingVideoMAEEpochs()), @@ -3768,9 +4033,10 @@ func trainingRunJob(ctx context.Context, root string, count int) { "--num-frames", strconv.Itoa(trainingVideoMAENumFrames), "--workers", strconv.Itoa(runtimeOpts.Workers), "--threads", strconv.Itoa(runtimeOpts.CPUThreads), + "--patience", strconv.Itoa(trainingVideoMAEEarlyStoppingPatience()), } - if base := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL")); base != "" { - videoMAEArgs = append(videoMAEArgs, "--base", base) + if videoMAEBase != "" { + videoMAEArgs = append(videoMAEArgs, "--base", videoMAEBase) } videoMAEOut, videoMAEErr := trainingRunCommandStreaming( @@ -3787,6 +4053,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { }, videoMAEArgs..., ) + videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds() if errors.Is(videoMAEErr, errTrainingCancelled) { appLogln("VideoMAE training cancelled") @@ -3822,6 +4089,10 @@ func trainingRunJob(ctx context.Context, root string, count int) { } } + if videoMAEStatus == "trained" && videoMAEDurationMs <= 0 && !videoMAEStartedAt.IsZero() { + videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds() + } + videoMAEOutputClean := cleanOutput(videoMAEOutput) message := "Training abgeschlossen." @@ -3831,6 +4102,9 @@ func trainingRunJob(ctx context.Context, root string, count int) { case "trained": message = "Training abgeschlossen. YOLO26 Detector wurde trainiert." + case "skipped_unselected": + message = "Training abgeschlossen." + case "skipped_no_detector_data": message = detectorOutput @@ -3887,8 +4161,31 @@ func trainingRunJob(ctx context.Context, root string, count int) { } if detectorStatus == "trained" || poseStatus == "trained" || videoMAEStatus == "trained" { - // Verlaufseintrag schreiben, solange die Job-Startzeit für die Dauer noch verfügbar ist. - trainingAppendRunHistory(root) + // Verlauf pro erfolgreich trainiertem Ziel schreiben. + if detectorStatus == "trained" { + trainingAppendTargetHistory(root, "detector", detectorStatus, detectorDurationMs, runtimeOpts, TrainingHistoryEntry{ + Epochs: trainingDetectorEpochs(), + TrainSamples: trainCount, + ValSamples: valCount, + Imgsz: 640, + }) + } + if poseStatus == "trained" { + trainingAppendTargetHistory(root, "pose", poseStatus, poseDurationMs, runtimeOpts, TrainingHistoryEntry{ + Epochs: trainingDetectorEpochs(), + TrainSamples: poseTrainCount, + ValSamples: poseValCount, + Imgsz: 640, + }) + } + if videoMAEStatus == "trained" { + trainingAppendTargetHistory(root, "videomae", videoMAEStatus, videoMAEDurationMs, runtimeOpts, TrainingHistoryEntry{ + Epochs: trainingVideoMAEEpochs(), + TrainSamples: videoMAETrainCount, + ValSamples: videoMAEValCount, + Imgsz: trainingVideoMAEFrameSize, + }) + } } trainingSetJobStatus(func(s *TrainingJobStatus) { @@ -4338,20 +4635,22 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { "predictsClothing": false, "predictsBoxes": false, - "feedbackCount": feedbackCount, - "eligibleCount": videoMAEEligibleCount, - "trainCount": videoMAETrainCount, - "valCount": videoMAEValCount, - "requiredTrain": minVideoMAETrainCount, - "requiredVal": minVideoMAEValCount, - "requiredCount": minVideoMAETrainCount, - "datasetReady": videoMAEDatasetReady, - "manifest": videoMAEManifest, - "dataReady": videoMAEDataReady, - "modelReady": videoMAEModel.EffectiveExists, - "modelExists": videoMAEModel.EffectiveExists, - "modelPath": videoMAEModel.EffectivePath, - "modelSource": videoMAEModel.Source, + "feedbackCount": feedbackCount, + "eligibleCount": videoMAEEligibleCount, + "trainCount": videoMAETrainCount, + "valCount": videoMAEValCount, + "requiredTrain": minVideoMAETrainCount, + "requiredVal": minVideoMAEValCount, + "requiredCount": minVideoMAETrainCount, + "datasetReady": videoMAEDatasetReady, + "manifest": videoMAEManifest, + "dataReady": videoMAEDataReady, + "modelReady": videoMAEModel.EffectiveExists, + "modelExists": videoMAEModel.EffectiveExists, + "modelPath": videoMAEModel.EffectivePath, + "modelSource": videoMAEModel.Source, + "trainedModelExists": videoMAEModel.TrainedExists, + "trainedModelPath": videoMAEModel.BestPath, }, "pipeline": map[string]any{ @@ -4464,16 +4763,24 @@ func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo { } type TrainingHistoryEntry struct { - TrainedAt string `json:"trainedAt,omitempty"` - TrainedAtMs int64 `json:"trainedAtMs,omitempty"` - DurationMs int64 `json:"durationMs,omitempty"` - Epochs int `json:"epochs,omitempty"` - TrainSamples int `json:"trainSamples,omitempty"` - ValSamples int `json:"valSamples,omitempty"` - Imgsz int `json:"imgsz,omitempty"` - Device string `json:"device,omitempty"` - MAP50 float64 `json:"map50,omitempty"` - MAP5095 float64 `json:"map5095,omitempty"` + TrainedAt string `json:"trainedAt,omitempty"` + TrainedAtMs int64 `json:"trainedAtMs,omitempty"` + Target string `json:"target,omitempty"` + Status string `json:"status,omitempty"` + DurationMs int64 `json:"durationMs,omitempty"` + Epochs int `json:"epochs,omitempty"` + TrainSamples int `json:"trainSamples,omitempty"` + ValSamples int `json:"valSamples,omitempty"` + Imgsz int `json:"imgsz,omitempty"` + Device string `json:"device,omitempty"` + MAP50 float64 `json:"map50,omitempty"` + MAP5095 float64 `json:"map5095,omitempty"` + PerformanceMode string `json:"performanceMode,omitempty"` + CPUCoreCount int `json:"cpuCoreCount,omitempty"` + CPUThreads int `json:"cpuThreads,omitempty"` + Workers int `json:"workers,omitempty"` + YoloBatchSize int `json:"yoloBatchSize,omitempty"` + LowPriority bool `json:"lowPriority,omitempty"` } type TrainingHistoryResponse struct { @@ -4485,31 +4792,81 @@ func trainingHistoryPath(root string) string { return filepath.Join(root, "detector", "training_history.jsonl") } -// trainingAppendRunHistory hängt nach einem erfolgreichen Trainingslauf einen -// Verlaufseintrag an (Datum, mAP, Samples, Epochen, Dauer). -func trainingAppendRunHistory(root string) { - info := trainingReadModelInfo(root) - if info == nil { - return +// trainingAppendTargetHistory haengt nach einem erfolgreichen Trainingsziel einen +// Verlaufseintrag an. Alte History-Zeilen ohne Target bleiben weiterhin lesbar. +func trainingHistoryKindForTarget(target string) string { + switch strings.ToLower(strings.TrimSpace(target)) { + case "pose": + return "pose" + case "videomae", "video_mae", "scene": + return "videomae" + default: + return "detector" } +} + +func trainingAppendTargetHistory(root string, target string, status string, durationMs int64, runtimeOpts trainingRuntimeOptions, fallback TrainingHistoryEntry) { + kind := trainingHistoryKindForTarget(target) + info := trainingReadModelInfoFor(root, kind) + now := time.Now().UTC() entry := TrainingHistoryEntry{ - TrainedAt: info.TrainedAt, - TrainedAtMs: info.TrainedAtMs, - Epochs: info.Epochs, - TrainSamples: info.TrainSamples, - ValSamples: info.ValSamples, - Imgsz: info.Imgsz, - Device: info.Device, - MAP50: info.MAP50, - MAP5095: info.MAP5095, + TrainedAt: now.Format(time.RFC3339), + TrainedAtMs: now.UnixMilli(), + Target: kind, + Status: strings.TrimSpace(status), + DurationMs: durationMs, + Epochs: fallback.Epochs, + TrainSamples: fallback.TrainSamples, + ValSamples: fallback.ValSamples, + Imgsz: fallback.Imgsz, + Device: strings.TrimSpace(fallback.Device), + MAP50: fallback.MAP50, + MAP5095: fallback.MAP5095, + PerformanceMode: runtimeOpts.PerformanceMode, + CPUCoreCount: runtimeOpts.CPUCoreCount, + CPUThreads: runtimeOpts.CPUThreads, + Workers: runtimeOpts.Workers, + YoloBatchSize: runtimeOpts.YoloBatchSize, + LowPriority: runtimeOpts.LowPriority, } - // Dauer aus der Startzeit des aktuellen Jobs ableiten. - job := trainingGetJobStatus() - if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(job.StartedAt)); err == nil { - if ms := time.Now().UTC().Sub(startedAt).Milliseconds(); ms > 0 { - entry.DurationMs = ms + if info != nil { + if strings.TrimSpace(info.TrainedAt) != "" { + entry.TrainedAt = info.TrainedAt + } + if info.TrainedAtMs > 0 { + entry.TrainedAtMs = info.TrainedAtMs + } + if info.Epochs > 0 { + entry.Epochs = info.Epochs + } + if info.TrainSamples > 0 { + entry.TrainSamples = info.TrainSamples + } + if info.ValSamples > 0 { + entry.ValSamples = info.ValSamples + } + if info.Imgsz > 0 { + entry.Imgsz = info.Imgsz + } + if strings.TrimSpace(info.Device) != "" { + entry.Device = strings.TrimSpace(info.Device) + } + if info.MAP50 > 0 { + entry.MAP50 = info.MAP50 + } + if info.MAP5095 > 0 { + entry.MAP5095 = info.MAP5095 + } + } + + if entry.DurationMs <= 0 { + job := trainingGetJobStatus() + if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(job.StartedAt)); err == nil { + if ms := now.Sub(startedAt).Milliseconds(); ms > 0 { + entry.DurationMs = ms + } } } diff --git a/backend/training_videomae.go b/backend/training_videomae.go index 19fc08e..419b70f 100644 --- a/backend/training_videomae.go +++ b/backend/training_videomae.go @@ -55,6 +55,21 @@ func trainingVideoMAEBatchSize() int { return n } +func trainingVideoMAEEarlyStoppingPatience() int { + epochs := trainingVideoMAEEpochs() + patience := epochs / 3 + if patience < 2 { + patience = 2 + } + if patience > 8 { + patience = 8 + } + if patience > epochs { + patience = epochs + } + return patience +} + type trainingVideoMAEManifestEntry struct { SampleID string `json:"sampleId"` Split string `json:"split"` diff --git a/frontend/src/components/ui/TrainingTab.tsx b/frontend/src/components/ui/TrainingTab.tsx index 16e763d..80fb11d 100644 --- a/frontend/src/components/ui/TrainingTab.tsx +++ b/frontend/src/components/ui/TrainingTab.tsx @@ -14,7 +14,10 @@ import { CheckIcon, ForwardIcon, InboxArrowDownIcon, + RectangleGroupIcon, + UserGroupIcon, TrashIcon, + VideoCameraIcon, XCircleIcon, } from '@heroicons/react/20/solid' import { getSegmentLabelItem } from './Icons' @@ -22,6 +25,7 @@ import Modal from './Modal' import { useNotify } from './notify' import { createPortal } from 'react-dom' import TrainingFeedbackHistoryModal from './TrainingFeedbackHistoryModal' +import type { RecorderSettingsState } from '../../types' type DrawingTrainingBox = TrainingBox & { @@ -45,6 +49,8 @@ type TrainingDetectorStatus = { dataReady: boolean modelExists: boolean modelPath?: string + trainedModelExists?: boolean + trainedModelPath?: string source?: string } @@ -57,6 +63,23 @@ type TrainingPoseStatus = { dataReady: boolean modelExists: boolean modelPath?: string + trainedModelExists?: boolean + trainedModelPath?: string + source?: string +} + +type TrainingVideoMAEStatus = { + eligibleCount: number + trainCount: number + valCount: number + requiredTrain: number + requiredVal: number + datasetReady: boolean + dataReady: boolean + modelExists: boolean + modelPath?: string + trainedModelExists?: boolean + trainedModelPath?: string source?: string } @@ -67,6 +90,7 @@ type TrainingStatus = { training?: TrainingJobStatus detector?: TrainingDetectorStatus pose?: TrainingPoseStatus + videoMAE?: TrainingVideoMAEStatus } type TrainingJobStatus = { @@ -220,6 +244,8 @@ type TrainingModelInfo = { type TrainingHistoryEntry = { trainedAt?: string trainedAtMs?: number + target?: string + status?: string durationMs?: number epochs?: number trainSamples?: number @@ -228,6 +254,12 @@ type TrainingHistoryEntry = { device?: string map50?: number map5095?: number + performanceMode?: string + cpuCoreCount?: number + cpuThreads?: number + workers?: number + yoloBatchSize?: number + lowPriority?: boolean } type TrainingStats = { @@ -279,6 +311,23 @@ type TrainingFeedbackListResponse = { } type TrainingSampleMode = 'random' | 'uncertain' +type TrainingTargetKey = 'detector' | 'pose' | 'videomae' +type TrainingStartMode = 'full' | 'custom' +type TrainingStartOptions = { + mode: TrainingStartMode + targets?: TrainingTargetKey[] +} +type TrainingEstimateMode = 'auto' | 'eco' | 'balanced' | 'performance' | 'custom' +type TrainingEstimateRuntime = { + mode: TrainingEstimateMode + modeLabel: string + threadsLabel: string + workers: number + yoloBatchLabel: string + lowPriority: boolean + factor: number + yoloFactor: number +} type FeedbackFilter = 'all' | 'accepted' | 'corrected' | 'negative' const POSE_KEYPOINT_MIN_CONFIDENCE = 0.15 @@ -471,6 +520,441 @@ function trainingDurationMs(job?: TrainingJobStatus | null) { return Math.max(0, finished - started) } +function roundedTrainingEstimateMs(ms: number) { + const safe = Number(ms) + if (!Number.isFinite(safe) || safe <= 0) return 0 + + const step = + safe >= 60 * 60 * 1000 + ? 5 * 60 * 1000 + : safe >= 10 * 60 * 1000 + ? 60 * 1000 + : safe >= 2 * 60 * 1000 + ? 30 * 1000 + : 10 * 1000 + + return Math.max(step, Math.round(safe / step) * step) +} + +function formatApproxTrainingDuration(ms: number) { + const rounded = roundedTrainingEstimateMs(ms) + if (rounded <= 0) return 'ca. —' + return `ca. ${formatDuration(rounded)}` +} + +function trainingHistoryDurationFloorMs(entries: TrainingHistoryEntry[]) { + const durations = (entries ?? []) + .filter((entry) => !String(entry.target ?? '').trim()) + .map((entry) => Number(entry.durationMs ?? 0)) + .filter((duration) => Number.isFinite(duration) && duration > 60 * 1000) + .slice(0, 5) + .sort((a, b) => a - b) + + if (durations.length === 0) return 0 + + return durations[Math.floor(durations.length / 2)] +} + +function trainingHistoryTarget(value?: string | null): TrainingTargetKey | '' { + switch (String(value ?? '').trim().toLowerCase()) { + case 'detector': + case 'yolo': + case 'yolo26': + case 'box': + case 'boxes': + return 'detector' + case 'pose': + case 'yolo26_pose': + return 'pose' + case 'videomae': + case 'video_mae': + case 'scene': + case 'clip': + return 'videomae' + default: + return '' + } +} + +function isTrainingTargetKey(value: unknown): value is TrainingTargetKey { + switch (String(value ?? '').trim().toLowerCase()) { + case 'detector': + case 'pose': + case 'videomae': + return true + default: + return false + } +} + +function trainingTargetFromStageText( + stage?: string | null, + step?: string | null +): TrainingTargetKey | '' { + const direct = trainingHistoryTarget(stage) + if (direct) return direct + + const text = `${stage ?? ''} ${step ?? ''}`.toLowerCase() + + if (text.includes('videomae') || text.includes('clip')) return 'videomae' + if (text.includes('pose')) return 'pose' + if ( + text.includes('detector') || + text.includes('object') || + text.includes('yolo26') || + text.includes('yolo') + ) { + return 'detector' + } + + return '' +} + +function trainingTargetProgressWindow(target: TrainingTargetKey | '') { + switch (target) { + case 'detector': + return { start: 15, end: 58 } + case 'pose': + return { start: 62, end: 82 } + case 'videomae': + return { start: 84, end: 98 } + default: + return { start: 0, end: 100 } + } +} + +function combineTrainingEtaMs(stageRemainingMs: number, epochRemainingMs: number) { + const stage = Number(stageRemainingMs) + const epoch = Number(epochRemainingMs) + const hasStage = Number.isFinite(stage) && stage > 0 + const hasEpoch = Number.isFinite(epoch) && epoch > 0 + + if (!hasStage && !hasEpoch) return 0 + if (!hasStage) return epoch + if (!hasEpoch) return stage + + const boundedEpoch = Math.max(stage * 0.35, Math.min(stage * 1.65, epoch)) + return boundedEpoch * 0.7 + stage * 0.3 +} + +function normalizeTrainingEstimateMode(value?: string | null): TrainingEstimateMode { + switch (String(value ?? '').trim().toLowerCase()) { + case 'eco': + case 'schonend': + case 'schonmodus': + case 'powersave': + case 'power-save': + case 'power_save': + return 'eco' + case 'balanced': + case 'ausgewogen': + case 'normal': + return 'balanced' + case 'performance': + case 'leistung': + case 'fast': + case 'schnell': + return 'performance' + case 'custom': + case 'manual': + case 'manuell': + return 'custom' + default: + return 'auto' + } +} + +function trainingEstimateModeLabel(mode: TrainingEstimateMode) { + switch (mode) { + case 'eco': + return 'Schonmodus' + case 'balanced': + return 'Ausgewogen' + case 'performance': + return 'Leistung' + case 'custom': + return 'Manuell' + default: + return 'Auto' + } +} + +function clampTrainingEstimate(value: number, minValue: number, maxValue: number) { + if (!Number.isFinite(value)) return minValue + return Math.max(minValue, Math.min(maxValue, value)) +} + +function trainingEstimateInt(value: unknown, fallback: number, minValue: number, maxValue: number) { + const raw = Math.floor(Number(value)) + const safe = Number.isFinite(raw) ? raw : fallback + return Math.max(minValue, Math.min(maxValue, safe)) +} + +function trainingEstimateRuntimeFromSettings(settings?: RecorderSettingsState | null): TrainingEstimateRuntime { + const mode = normalizeTrainingEstimateMode( + settings?.trainingEffectiveMode ?? settings?.trainingPerformanceMode + ) + const cpuCores = trainingEstimateInt(settings?.trainingCpuCoreCount, 0, 0, 256) + const rawThreads = trainingEstimateInt( + settings?.trainingEffectiveCpuThreads ?? settings?.trainingCpuThreads, + 0, + 0, + 256 + ) + const workers = trainingEstimateInt( + settings?.trainingEffectiveWorkers ?? settings?.trainingWorkers, + 1, + 0, + 32 + ) + const rawBatch = trainingEstimateInt( + settings?.trainingEffectiveYoloBatchSize ?? settings?.trainingYoloBatchSize, + 0, + 0, + 64 + ) + const lowPriority = Boolean( + settings?.trainingEffectiveLowPriority ?? settings?.trainingLowPriority + ) + const autoThreads = cpuCores > 0 + ? Math.max(1, Math.min(16, Math.round(cpuCores * 0.75))) + : 4 + const effectiveThreads = rawThreads > 0 ? rawThreads : autoThreads + const effectiveBatch = rawBatch > 0 ? rawBatch : 2 + + const modeFactor = + mode === 'eco' + ? 1.45 + : mode === 'balanced' + ? 1.15 + : mode === 'performance' + ? 0.92 + : mode === 'custom' + ? 1 + : 1.08 + const threadFactor = + effectiveThreads <= 1 + ? 1.7 + : effectiveThreads === 2 + ? 1.35 + : effectiveThreads === 3 + ? 1.18 + : effectiveThreads >= 12 + ? 0.86 + : effectiveThreads >= 8 + ? 0.9 + : effectiveThreads >= 6 + ? 0.96 + : 1.04 + const workerFactor = + workers <= 0 + ? 1.2 + : workers === 1 + ? 1.08 + : workers >= 6 + ? 0.98 + : 1 + const batchFactor = + effectiveBatch <= 1 + ? 1.25 + : effectiveBatch === 2 + ? 1.08 + : effectiveBatch >= 12 + ? 0.88 + : effectiveBatch >= 8 + ? 0.92 + : effectiveBatch >= 4 + ? 0.98 + : 1 + const lowPriorityFactor = lowPriority ? 1.12 : 1 + const corePenalty = + cpuCores > 0 && effectiveThreads > cpuCores + ? 1 + Math.min(0.3, (effectiveThreads - cpuCores) * 0.04) + : 1 + const factor = modeFactor * threadFactor * workerFactor * lowPriorityFactor * corePenalty + const yoloFactor = factor * batchFactor + + return { + mode, + modeLabel: trainingEstimateModeLabel(mode), + threadsLabel: rawThreads > 0 ? String(rawThreads) : 'Auto', + workers, + yoloBatchLabel: rawBatch > 0 ? String(rawBatch) : 'Auto', + lowPriority, + factor: Number.isFinite(factor) && factor > 0 ? factor : 1, + yoloFactor: Number.isFinite(yoloFactor) && yoloFactor > 0 ? yoloFactor : 1, + } +} + +function trainingEstimateRuntimeText(runtime: TrainingEstimateRuntime) { + const priority = runtime.lowPriority ? ' · niedrige Priorität' : '' + return `${runtime.modeLabel} · ${runtime.threadsLabel} Threads · ${runtime.workers} Worker · Batch ${runtime.yoloBatchLabel}${priority}` +} + +function trainingEstimateRuntimeFromHistory(entry: TrainingHistoryEntry) { + return trainingEstimateRuntimeFromSettings({ + trainingPerformanceMode: entry.performanceMode, + trainingEffectiveMode: entry.performanceMode, + trainingCpuCoreCount: entry.cpuCoreCount, + trainingEffectiveCpuThreads: entry.cpuThreads, + trainingEffectiveWorkers: entry.workers, + trainingEffectiveYoloBatchSize: entry.yoloBatchSize, + trainingEffectiveLowPriority: entry.lowPriority, + } as RecorderSettingsState) +} + +function trainingRuntimeFactorForTarget(runtime: TrainingEstimateRuntime, target: TrainingTargetKey) { + return target === 'videomae' ? runtime.factor : runtime.yoloFactor +} + +function trainingHasTargetHistory(entries: TrainingHistoryEntry[], target: TrainingTargetKey) { + return (entries ?? []).some((entry) => { + const duration = Number(entry.durationMs ?? 0) + if (!Number.isFinite(duration) || duration <= 60 * 1000) return false + const status = String(entry.status ?? 'trained').trim().toLowerCase() + if (status && status !== 'trained') return false + return trainingHistoryTarget(entry.target) === target + }) +} + +function trainingFineTuneEstimateFactor( + target: TrainingTargetKey, + trainedModelExists: boolean, + hasTargetHistory: boolean +) { + if (!trainedModelExists) return 1 + + if (hasTargetHistory) { + return 0.92 + } + + return target === 'videomae' ? 0.85 : 0.75 +} + +function estimateTrainingHistoryDurationMs( + entries: TrainingHistoryEntry[], + target: TrainingTargetKey, + trainCount: number, + valCount: number, + eligibleCount: number, + runtime: TrainingEstimateRuntime, + epochs: number +) { + const currentSamples = Math.max( + 1, + Math.max(0, Number(trainCount) || 0) + + Math.max(0, Number(valCount) || 0), + target === 'videomae' ? Math.max(0, Number(eligibleCount) || 0) : 0 + ) + const currentEpochs = trainingEstimateInt( + target === 'videomae' ? epochs || 8 : epochs, + target === 'videomae' ? 8 : 60, + 1, + target === 'videomae' ? 200 : 300 + ) + const currentRuntimeFactor = trainingRuntimeFactorForTarget(runtime, target) + const estimates = (entries ?? []) + .filter((entry) => { + const duration = Number(entry.durationMs ?? 0) + if (!Number.isFinite(duration) || duration <= 60 * 1000) return false + const status = String(entry.status ?? 'trained').trim().toLowerCase() + if (status && status !== 'trained') return false + + return trainingHistoryTarget(entry.target) === target + }) + .slice(0, 8) + .map((entry) => { + const duration = Number(entry.durationMs ?? 0) + const historySamples = Math.max( + 1, + Math.max(0, Number(entry.trainSamples ?? 0)) + + Math.max(0, Number(entry.valSamples ?? 0)) + ) + const historyEpochs = trainingEstimateInt( + entry.epochs, + target === 'videomae' ? 8 : 60, + 1, + target === 'videomae' ? 200 : 300 + ) + const historyRuntime = trainingEstimateRuntimeFromHistory(entry) + const historyRuntimeFactor = trainingRuntimeFactorForTarget(historyRuntime, target) + const sampleFactor = clampTrainingEstimate( + Math.pow(currentSamples / historySamples, 0.85), + 0.55, + 2.8 + ) + const epochFactor = clampTrainingEstimate(currentEpochs / historyEpochs, 0.35, 4) + const runtimeFactor = historyRuntimeFactor > 0 + ? clampTrainingEstimate(currentRuntimeFactor / historyRuntimeFactor, 0.45, 2.4) + : 1 + + return duration * sampleFactor * epochFactor * runtimeFactor + }) + .filter((value) => Number.isFinite(value) && value > 0) + .sort((a, b) => a - b) + + if (estimates.length === 0) return 0 + + return estimates[Math.floor(estimates.length / 2)] +} + +function estimateTrainingDurationMs( + target: TrainingTargetKey, + trainCount: number, + valCount: number, + eligibleCount = 0, + historyEstimateMs = 0, + runtimeFactor = 1, + detectorEpochs = 60, + fineTuneFactor = 1 +) { + const samples = Math.max( + 0, + Number.isFinite(trainCount) ? trainCount : 0, + ) + Math.max( + 0, + Number.isFinite(valCount) ? valCount : 0, + ) + const eligible = Math.max(0, Number.isFinite(eligibleCount) ? eligibleCount : 0) + const historyEstimate = Number.isFinite(historyEstimateMs) + ? Math.max(0, historyEstimateMs) + : 0 + const factor = Number.isFinite(runtimeFactor) && runtimeFactor > 0 + ? runtimeFactor + : 1 + const fineTune = Number.isFinite(fineTuneFactor) && fineTuneFactor > 0 + ? clampTrainingEstimate(fineTuneFactor, 0.5, 1) + : 1 + const epochs = trainingEstimateInt(detectorEpochs, 60, 1, 300) + const yoloEpochFactor = Math.max(0.5, epochs / 60) + + switch (target) { + case 'detector': { + const base = Math.max( + 12 * 60 * 1000, + (10 * 60 * 1000 + samples * 8500) * yoloEpochFactor + ) + return Math.max(base * factor * fineTune, historyEstimate) + } + case 'pose': { + const base = Math.max( + 14 * 60 * 1000, + (12 * 60 * 1000 + samples * 9500) * yoloEpochFactor + ) + return Math.max(base * factor * fineTune, historyEstimate) + } + case 'videomae': { + const base = Math.max( + 25 * 60 * 1000, + 18 * 60 * 1000 + Math.max(samples, eligible) * 16000 + ) + return Math.max(base * factor * fineTune, historyEstimate) + } + default: + return 0 + } +} + function countPercent(count: number, total: number) { if (!Number.isFinite(count) || !Number.isFinite(total) || total <= 0) return '0%' return `${Math.round((count / total) * 100)}%` @@ -2678,6 +3162,10 @@ function TrainingStatsModal(props: { ) const meta: string[] = [] + const target = trainingHistoryTarget(entry.target) + if (target === 'detector') meta.push('Detector') + if (target === 'pose') meta.push('Pose') + if (target === 'videomae') meta.push('VideoMAE') if (Number(entry.epochs) > 0) meta.push(`${entry.epochs} Ep.`) if (Number(entry.trainSamples) > 0) meta.push(`${entry.trainSamples} Train`) if (duration) meta.push(duration) @@ -2864,6 +3352,7 @@ const TRAINING_IMAGE_EXPANDED_STORAGE_KEY = 'training:image-expanded' const TRAINING_PENDING_IMPORT_VIDEO_STORAGE_KEY = 'training:pending-import-video' const TRAINING_ACTIVE_IMPORT_VIDEO_STORAGE_KEY = 'training:active-import-video' const TRAINING_ACTIVE_NEXT_STORAGE_KEY = 'training:active-next' +const ALL_TRAINING_TARGETS: TrainingTargetKey[] = ['detector', 'pose', 'videomae'] export default function TrainingTab(props: { active?: boolean @@ -2890,11 +3379,16 @@ export default function TrainingTab(props: { const [error, setError] = useState(null) const [message, setMessage] = useState(null) const [statsModalOpen, setStatsModalOpen] = useState(false) + const [trainingStartModalOpen, setTrainingStartModalOpen] = useState(false) + const [trainingStartMode, setTrainingStartMode] = useState('full') + const [trainingStartTargets, setTrainingStartTargets] = useState(ALL_TRAINING_TARGETS) + const [activeTrainingTargets, setActiveTrainingTargets] = useState([]) const [cancellingTraining, setCancellingTraining] = useState(false) const [trainingStats, setTrainingStats] = useState(null) const [trainingStatsLoading, setTrainingStatsLoading] = useState(false) const [trainingStatsError, setTrainingStatsError] = useState(null) const [trainingHistory, setTrainingHistory] = useState([]) + const [trainingEstimateSettings, setTrainingEstimateSettings] = useState(null) const wasTrainingRunningRef = useRef(false) const shownTrainingCompletionRef = useRef(null) const [dismissedTrainingInfoKey, setDismissedTrainingInfoKey] = useState(() => { @@ -3216,10 +3710,12 @@ export default function TrainingTab(props: { const epochTimingRef = useRef<{ + target: string firstEpochAt: number lastEpoch: number lastAt: number }>({ + target: '', firstEpochAt: 0, lastEpoch: 0, lastAt: 0, @@ -3607,13 +4103,346 @@ export default function TrainingTab(props: { const trainingRunning = training || Boolean(trainingStatus?.training?.running) const uiLocked = loading || saving || trainingRunning || deletingTrainingData - const shownTrainingProgress = trainingRunning + const serverTrainingProgress = trainingRunning ? trainingStatus?.training?.progress ?? trainingProgress : trainingProgress const shownTrainingStep = trainingRunning ? trainingStatus?.training?.step || trainingStep || 'Training läuft…' : trainingStep + const trainingHistoryFloorMs = useMemo( + () => trainingHistoryDurationFloorMs(trainingHistory), + [trainingHistory] + ) + const trainingEstimateRuntime = useMemo( + () => trainingEstimateRuntimeFromSettings(trainingEstimateSettings), + [trainingEstimateSettings] + ) + const trainingEstimateRuntimeLabel = useMemo( + () => trainingEstimateRuntimeText(trainingEstimateRuntime), + [trainingEstimateRuntime] + ) + const trainingEstimateDetectorEpochs = useMemo( + () => trainingEstimateInt(trainingEstimateSettings?.trainingDetectorEpochs, 60, 1, 300), + [trainingEstimateSettings?.trainingDetectorEpochs] + ) + + const trainingStartOptions = useMemo(() => { + const feedbackReady = feedbackCount >= requiredCount + const detector = trainingStatus?.detector + const pose = trainingStatus?.pose + const videoMAE = trainingStatus?.videoMAE + + const detectorReady = feedbackReady && Boolean(detector?.dataReady) + const poseReady = feedbackReady && Boolean(pose?.dataReady) + const videoMAEReady = feedbackReady && Boolean(videoMAE?.dataReady) + const detectorHasHistory = trainingHasTargetHistory(trainingHistory, 'detector') + const poseHasHistory = trainingHasTargetHistory(trainingHistory, 'pose') + const videoMAEHasHistory = trainingHasTargetHistory(trainingHistory, 'videomae') + const detectorHistoryEstimateMs = estimateTrainingHistoryDurationMs( + trainingHistory, + 'detector', + Number(detector?.trainCount ?? 0), + Number(detector?.valCount ?? 0), + 0, + trainingEstimateRuntime, + trainingEstimateDetectorEpochs + ) + const poseHistoryEstimateMs = estimateTrainingHistoryDurationMs( + trainingHistory, + 'pose', + Number(pose?.trainCount ?? 0), + Number(pose?.valCount ?? 0), + 0, + trainingEstimateRuntime, + trainingEstimateDetectorEpochs + ) + const videoMAEHistoryEstimateMs = estimateTrainingHistoryDurationMs( + trainingHistory, + 'videomae', + Number(videoMAE?.trainCount ?? 0), + Number(videoMAE?.valCount ?? 0), + Number(videoMAE?.eligibleCount ?? 0), + trainingEstimateRuntime, + 8 + ) + const detectorFineTuneFactor = trainingFineTuneEstimateFactor( + 'detector', + Boolean(detector?.trainedModelExists), + detectorHasHistory + ) + const poseFineTuneFactor = trainingFineTuneEstimateFactor( + 'pose', + Boolean(pose?.trainedModelExists), + poseHasHistory + ) + const videoMAEFineTuneFactor = trainingFineTuneEstimateFactor( + 'videomae', + Boolean(videoMAE?.trainedModelExists), + videoMAEHasHistory + ) + const detectorFallbackHistoryMs = detectorHistoryEstimateMs || + (trainingHistoryFloorMs > 0 ? trainingHistoryFloorMs * detectorFineTuneFactor : 0) + const poseFallbackHistoryMs = poseHistoryEstimateMs || + (trainingHistoryFloorMs > 0 ? trainingHistoryFloorMs * poseFineTuneFactor : 0) + const videoMAEFallbackHistoryMs = videoMAEHistoryEstimateMs || + (trainingHistoryFloorMs > 0 ? trainingHistoryFloorMs * videoMAEFineTuneFactor : 0) + const detectorEstimateMs = estimateTrainingDurationMs( + 'detector', + Number(detector?.trainCount ?? 0), + Number(detector?.valCount ?? 0), + 0, + detectorFallbackHistoryMs, + trainingEstimateRuntime.yoloFactor, + trainingEstimateDetectorEpochs, + detectorFineTuneFactor + ) + const poseEstimateMs = estimateTrainingDurationMs( + 'pose', + Number(pose?.trainCount ?? 0), + Number(pose?.valCount ?? 0), + 0, + poseFallbackHistoryMs, + trainingEstimateRuntime.yoloFactor, + trainingEstimateDetectorEpochs, + poseFineTuneFactor + ) + const videoMAEEstimateMs = estimateTrainingDurationMs( + 'videomae', + Number(videoMAE?.trainCount ?? 0), + Number(videoMAE?.valCount ?? 0), + Number(videoMAE?.eligibleCount ?? 0), + videoMAEFallbackHistoryMs, + trainingEstimateRuntime.factor, + 60, + videoMAEFineTuneFactor + ) + + return [ + { + key: 'detector' as const, + label: 'YOLO26 Detector', + icon: RectangleGroupIcon, + description: 'Boxen, Personen, Körperteile, Objekte und Kleidung.', + ready: detectorReady, + estimateMs: detectorEstimateMs, + estimateText: formatApproxTrainingDuration(detectorEstimateMs), + detail: `Train ${Number(detector?.trainCount ?? 0)}/${Number(detector?.requiredTrain ?? 20)}, Val ${Number(detector?.valCount ?? 0)}/${Number(detector?.requiredVal ?? 3)}, Positiv ${Number(detector?.positiveTrainCount ?? 0)}/${Number(detector?.positiveValCount ?? 0)}${detector?.trainedModelExists ? ', Fine-Tuning' : ''}`, + blockedText: !feedbackReady + ? `Noch zu wenig Feedback: ${feedbackCount}/${requiredCount}.` + : 'Detector-Datensatz ist noch nicht bereit.', + }, + { + key: 'pose' as const, + label: 'YOLO26 Pose', + icon: UserGroupIcon, + description: 'Keypoints und Positions-Kontext aus Skeleton-Beispielen.', + ready: poseReady, + estimateMs: poseEstimateMs, + estimateText: formatApproxTrainingDuration(poseEstimateMs), + detail: `Train ${Number(pose?.trainCount ?? 0)}/${Number(pose?.requiredTrain ?? 20)}, Val ${Number(pose?.valCount ?? 0)}/${Number(pose?.requiredVal ?? 3)}${pose?.trainedModelExists ? ', Fine-Tuning' : ''}`, + blockedText: !feedbackReady + ? `Noch zu wenig Feedback: ${feedbackCount}/${requiredCount}.` + : 'Pose-Datensatz ist noch nicht bereit.', + }, + { + key: 'videomae' as const, + label: 'VideoMAE Clip-Analyse', + icon: VideoCameraIcon, + description: 'Clip-basierte Positionsanalyse aus mehreren Video-Frames.', + ready: videoMAEReady, + estimateMs: videoMAEEstimateMs, + estimateText: formatApproxTrainingDuration(videoMAEEstimateMs), + detail: `Eligible ${Number(videoMAE?.eligibleCount ?? 0)}, Train ${Number(videoMAE?.trainCount ?? 0)}/${Number(videoMAE?.requiredTrain ?? 40)}, Val ${Number(videoMAE?.valCount ?? 0)}/${Number(videoMAE?.requiredVal ?? 5)}${videoMAE?.trainedModelExists ? ', Fine-Tuning' : ''}`, + blockedText: !feedbackReady + ? `Noch zu wenig Feedback: ${feedbackCount}/${requiredCount}.` + : 'VideoMAE-Datensatz ist noch nicht bereit.', + }, + ] + }, [ + feedbackCount, + requiredCount, + trainingStatus?.detector, + trainingStatus?.pose, + trainingStatus?.videoMAE, + trainingHistory, + trainingHistoryFloorMs, + trainingEstimateRuntime.factor, + trainingEstimateRuntime.yoloFactor, + trainingEstimateDetectorEpochs, + ]) + + const selectableTrainingTargets = useMemo( + () => trainingStartOptions + .filter((option) => option.ready) + .map((option) => option.key), + [trainingStartOptions] + ) + + const selectedTrainingTargets = useMemo( + () => trainingStartTargets.filter((key) => selectableTrainingTargets.includes(key)), + [selectableTrainingTargets, trainingStartTargets] + ) + + const canConfirmTrainingStart = + trainingStartMode === 'full' + ? canStartTraining + : selectedTrainingTargets.length > 0 + + const plannedTrainingTargets = useMemo( + () => trainingStartMode === 'full' + ? selectableTrainingTargets + : selectedTrainingTargets, + [selectableTrainingTargets, selectedTrainingTargets, trainingStartMode] + ) + + const trainingStartTotalEstimateMs = useMemo(() => { + const planned = new Set(plannedTrainingTargets) + const estimate = trainingStartOptions.reduce((sum, option) => ( + planned.has(option.key) ? sum + Number(option.estimateMs ?? 0) : sum + ), 0) + + return trainingHistoryFloorMs > 0 && plannedTrainingTargets.length > 1 + ? Math.max(estimate, trainingHistoryFloorMs) + : estimate + }, [plannedTrainingTargets, trainingHistoryFloorMs, trainingStartOptions]) + + const trainingStartTotalEstimateText = + plannedTrainingTargets.length > 0 + ? formatApproxTrainingDuration(trainingStartTotalEstimateMs) + : 'ca. —' + + const trainingEstimateByTarget = useMemo(() => { + const estimates: Record = { + detector: 0, + pose: 0, + videomae: 0, + } + + for (const option of trainingStartOptions) { + estimates[option.key] = Math.max(0, Number(option.estimateMs ?? 0) || 0) + } + + return estimates + }, [trainingStartOptions]) + + const activeTrainingTarget = useMemo( + () => trainingTargetFromStageText( + trainingStatus?.training?.stage, + trainingStatus?.training?.step + ), + [ + trainingStatus?.training?.stage, + trainingStatus?.training?.step, + ] + ) + + const effectiveTrainingTargets = useMemo(() => { + const selected = activeTrainingTargets.filter(isTrainingTargetKey) + if (selected.length > 0) return selected + + if (selectableTrainingTargets.length > 0) { + return selectableTrainingTargets + } + + if (activeTrainingTarget) { + const activeIndex = ALL_TRAINING_TARGETS.indexOf(activeTrainingTarget) + return activeIndex >= 0 + ? ALL_TRAINING_TARGETS.slice(activeIndex) + : [activeTrainingTarget] + } + + return ALL_TRAINING_TARGETS + }, [ + activeTrainingTarget, + activeTrainingTargets, + selectableTrainingTargets, + ]) + + const currentTrainingTarget = useMemo(() => { + if (!trainingRunning || effectiveTrainingTargets.length === 0) return '' + + const progress = clampPercent(Number(serverTrainingProgress) || 0) + const activeIndex = activeTrainingTarget + ? effectiveTrainingTargets.indexOf(activeTrainingTarget) + : -1 + + if (activeIndex >= 0) { + const window = trainingTargetProgressWindow(activeTrainingTarget) + if (progress < window.end || activeIndex === effectiveTrainingTargets.length - 1) { + return activeTrainingTarget + } + } + + return effectiveTrainingTargets.find((target) => { + const window = trainingTargetProgressWindow(target) + return progress < window.end + }) ?? effectiveTrainingTargets[effectiveTrainingTargets.length - 1] ?? '' + }, [ + activeTrainingTarget, + effectiveTrainingTargets, + serverTrainingProgress, + trainingRunning, + ]) + + const currentTrainingTargetProgress = useMemo(() => { + if (!currentTrainingTarget) return 0 + + const window = trainingTargetProgressWindow(currentTrainingTarget) + const span = Math.max(1, window.end - window.start) + const progress = clampPercent(Number(serverTrainingProgress) || 0) + + return clamp01((progress - window.start) / span) + }, [ + currentTrainingTarget, + serverTrainingProgress, + ]) + + const estimatedTrainingProgress = useMemo(() => { + if (!trainingRunning || effectiveTrainingTargets.length === 0) return 0 + + const totalEstimateMs = effectiveTrainingTargets.reduce( + (sum, target) => sum + Math.max(0, trainingEstimateByTarget[target] || 0), + 0 + ) + + if (totalEstimateMs <= 0) return 0 + + const currentTarget = currentTrainingTarget + if (!currentTarget) return 0 + + const currentIndex = effectiveTrainingTargets.indexOf(currentTarget) + + if (currentIndex < 0) return 0 + + const completedMs = effectiveTrainingTargets + .slice(0, currentIndex) + .reduce((sum, target) => sum + Math.max(0, trainingEstimateByTarget[target] || 0), 0) + const currentMs = Math.max(0, trainingEstimateByTarget[currentTarget] || 0) + const progressMs = completedMs + currentMs * currentTrainingTargetProgress + + return clampPercent((progressMs / totalEstimateMs) * 100) + }, [ + currentTrainingTarget, + currentTrainingTargetProgress, + effectiveTrainingTargets, + trainingEstimateByTarget, + trainingRunning, + ]) + + const serverTrainingProgressPercent = clampPercent(Number(serverTrainingProgress) || 0) + const usesSelectedTargetProgress = + activeTrainingTargets.length > 0 && + activeTrainingTargets.length < ALL_TRAINING_TARGETS.length + const shownTrainingProgress = trainingRunning + ? usesSelectedTargetProgress + ? Math.max( + Math.min(serverTrainingProgressPercent, 6), + estimatedTrainingProgress + ) + : Math.max(serverTrainingProgressPercent, estimatedTrainingProgress) + : serverTrainingProgress + const drawingCursorClass = boxInteraction?.type === 'move' ? '[@media_(hover:hover)_and_(pointer:fine)]:cursor-grabbing' @@ -3691,6 +4520,7 @@ export default function TrainingTab(props: { if (!data) return const job = data.training || null + const videoMAE = data.videoMAE || data.videomae || data.scene || null setTrainingStatus((prev) => ({ feedbackCount: Number(data.feedbackCount ?? prev?.feedbackCount ?? 0), @@ -3709,7 +4539,9 @@ export default function TrainingTab(props: { dataReady: Boolean(data.detector.dataReady), modelExists: Boolean(data.detector.modelExists), modelPath: data.detector.modelPath, - source: data.detector.source, + trainedModelExists: Boolean(data.detector.trainedModelExists ?? data.detector.modelExists), + trainedModelPath: data.detector.trainedModelPath, + source: data.detector.source || data.detector.modelSource, } : prev?.detector, @@ -3723,10 +4555,34 @@ export default function TrainingTab(props: { dataReady: Boolean(data.pose.dataReady), modelExists: Boolean(data.pose.modelExists), modelPath: data.pose.modelPath, - source: data.pose.source, + trainedModelExists: Boolean(data.pose.trainedModelExists ?? data.pose.modelExists), + trainedModelPath: data.pose.trainedModelPath, + source: data.pose.source || data.pose.modelSource, } : prev?.pose, + videoMAE: videoMAE + ? { + eligibleCount: Number(videoMAE.eligibleCount ?? 0), + trainCount: Number(videoMAE.trainCount ?? 0), + valCount: Number(videoMAE.valCount ?? 0), + requiredTrain: Number(videoMAE.requiredTrain ?? videoMAE.requiredCount ?? 40), + requiredVal: Number(videoMAE.requiredVal ?? 5), + datasetReady: Boolean(videoMAE.datasetReady), + dataReady: Boolean(videoMAE.dataReady), + modelExists: Boolean(videoMAE.modelExists ?? videoMAE.modelReady), + modelPath: videoMAE.modelPath, + trainedModelExists: Boolean( + videoMAE.trainedModelExists ?? + videoMAE.trainedModelReady ?? + videoMAE.modelExists ?? + videoMAE.modelReady + ), + trainedModelPath: videoMAE.trainedModelPath, + source: videoMAE.source || videoMAE.modelSource, + } + : prev?.videoMAE, + training: job ? { running: Boolean(job.running), @@ -4434,6 +5290,8 @@ export default function TrainingTab(props: { ? data.entries.map((e: any) => ({ trainedAt: e?.trainedAt, trainedAtMs: Number(e?.trainedAtMs ?? 0), + target: e?.target, + status: e?.status, durationMs: Number(e?.durationMs ?? 0), epochs: Number(e?.epochs ?? 0), trainSamples: Number(e?.trainSamples ?? 0), @@ -4442,6 +5300,12 @@ export default function TrainingTab(props: { device: e?.device, map50: Number(e?.map50 ?? 0), map5095: Number(e?.map5095 ?? 0), + performanceMode: e?.performanceMode, + cpuCoreCount: Number(e?.cpuCoreCount ?? 0), + cpuThreads: Number(e?.cpuThreads ?? 0), + workers: Number(e?.workers ?? 0), + yoloBatchSize: Number(e?.yoloBatchSize ?? 0), + lowPriority: Boolean(e?.lowPriority), })) : [] ) @@ -4450,6 +5314,19 @@ export default function TrainingTab(props: { } }, []) + const loadTrainingEstimateSettings = useCallback(async () => { + try { + const res = await fetch('/api/settings', { cache: 'no-store' }) + const data = await res.json().catch(() => null) + + if (!res.ok || !data) return + + setTrainingEstimateSettings(data as RecorderSettingsState) + } catch { + // ignore + } + }, []) + useEffect(() => { if (!tabActive) return @@ -4483,6 +5360,8 @@ export default function TrainingTab(props: { if (trainingRunning) { setMobilePanel('training') + } else { + setActiveTrainingTargets([]) } }, [trainingRunning]) @@ -4994,6 +5873,7 @@ export default function TrainingTab(props: { if (!trainingRunning || epoch <= 0 || epochs <= 0) { epochTimingRef.current = { + target: '', firstEpochAt: 0, lastEpoch: 0, lastAt: 0, @@ -5005,9 +5885,17 @@ export default function TrainingTab(props: { const now = Date.now() const previous = epochTimingRef.current + const target = currentTrainingTarget || activeTrainingTarget || '' + const targetChanged = previous.target !== target + + if (targetChanged) { + setEstimatedEpochMs(0) + } const firstEpochAt = - previous.firstEpochAt > 0 + targetChanged + ? now + : previous.firstEpochAt > 0 ? previous.firstEpochAt : job?.startedAt && Number.isFinite(Date.parse(job.startedAt)) ? Date.parse(job.startedAt) @@ -5039,6 +5927,7 @@ export default function TrainingTab(props: { } epochTimingRef.current = { + target, firstEpochAt, lastEpoch: safeEpoch, lastAt: now, @@ -5048,6 +5937,8 @@ export default function TrainingTab(props: { trainingStatus?.training?.epoch, trainingStatus?.training?.epochs, trainingStatus?.training?.startedAt, + activeTrainingTarget, + currentTrainingTarget, trainingNowMs, ]) @@ -5291,7 +6182,27 @@ export default function TrainingTab(props: { loadNextImportedQueuedSample, ]) - const startTraining = useCallback(async () => { + const openTrainingStartModal = useCallback(() => { + const readyTargets = selectableTrainingTargets.length > 0 + ? selectableTrainingTargets + : ALL_TRAINING_TARGETS + + setTrainingStartMode('full') + setTrainingStartTargets(readyTargets) + setTrainingStartModalOpen(true) + void loadTrainingHistory() + void loadTrainingEstimateSettings() + }, [loadTrainingEstimateSettings, loadTrainingHistory, selectableTrainingTargets]) + + const toggleTrainingStartTarget = useCallback((target: TrainingTargetKey) => { + setTrainingStartTargets((prev) => + prev.includes(target) + ? prev.filter((item) => item !== target) + : [...prev, target] + ) + }, []) + + const startTraining = useCallback(async (options?: TrainingStartOptions) => { shownTrainingCompletionRef.current = null setDismissedTrainingInfoKey('') @@ -5301,6 +6212,14 @@ export default function TrainingTab(props: { // ignore } + const nextActiveTargets = + options?.mode === 'custom' + ? (options.targets ?? []).filter(isTrainingTargetKey) + : selectableTrainingTargets.length > 0 + ? selectableTrainingTargets + : ALL_TRAINING_TARGETS + + setActiveTrainingTargets(nextActiveTargets) setTraining(true) setTrainingProgress(5) setTrainingStep('Training wird gestartet…') @@ -5308,8 +6227,22 @@ export default function TrainingTab(props: { setMessage(null) try { + const payload = + options?.mode === 'custom' + ? { + scope: 'custom', + targets: options.targets ?? [], + } + : { + scope: 'full', + } + const res = await fetch('/api/training/train', { method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), }) const data = await res.json().catch(() => null) @@ -5324,12 +6257,35 @@ export default function TrainingTab(props: { // Hier NICHT direkt loadNext() aufrufen. // Das Training läuft im Backend asynchron weiter. } catch (e) { + setActiveTrainingTargets([]) setTraining(false) setTrainingProgress(0) setTrainingStep('') setError(e instanceof Error ? e.message : String(e)) } - }, [loadTrainingStatus]) + }, [loadTrainingStatus, selectableTrainingTargets]) + + const startTrainingFromModal = useCallback(() => { + if (!canConfirmTrainingStart) return + + const options: TrainingStartOptions = + trainingStartMode === 'custom' + ? { + mode: 'custom', + targets: selectedTrainingTargets, + } + : { + mode: 'full', + } + + setTrainingStartModalOpen(false) + void startTraining(options) + }, [ + canConfirmTrainingStart, + selectedTrainingTargets, + startTraining, + trainingStartMode, + ]) const cancelTraining = useCallback(async () => { const confirmed = window.confirm( @@ -5977,26 +6933,54 @@ export default function TrainingTab(props: { const epoch = Number(job?.epoch ?? 0) const epochs = Number(job?.epochs ?? 0) - if ( - !Number.isFinite(epoch) || - !Number.isFinite(epochs) || - !Number.isFinite(estimatedEpochMs) || - epoch <= 0 || - epochs <= 0 || - estimatedEpochMs <= 0 - ) { - return 0 + const hasEpochInfo = + Number.isFinite(epoch) && + Number.isFinite(epochs) && + epoch > 0 && + epochs > 0 + const completedEpochs = hasEpochInfo + ? Math.max(0, Math.min(epochs, Math.floor(epoch))) + : 0 + const remainingEpochs = + Number.isFinite(epochs) && epochs > 0 + ? Math.max(0, epochs - completedEpochs) + : 0 + const epochEtaMs = + Number.isFinite(estimatedEpochMs) && estimatedEpochMs > 0 + ? remainingEpochs * estimatedEpochMs + : 0 + + if (!currentTrainingTarget || effectiveTrainingTargets.length === 0) { + return Math.max(0, epochEtaMs) } - const completedEpochs = Math.max(1, Math.min(epochs, Math.floor(epoch))) - const remainingEpochs = Math.max(0, epochs - completedEpochs) + const currentIndex = effectiveTrainingTargets.indexOf(currentTrainingTarget) + if (currentIndex < 0) { + return Math.max(0, epochEtaMs) + } - return Math.max(0, remainingEpochs * estimatedEpochMs) + const currentEstimateMs = Math.max( + 0, + trainingEstimateByTarget[currentTrainingTarget] || 0 + ) + const stageRemainingMs = currentEstimateMs > 0 + ? currentEstimateMs * (1 - currentTrainingTargetProgress) + : 0 + const currentRemainingMs = combineTrainingEtaMs(stageRemainingMs, epochEtaMs) + const laterTargetsMs = effectiveTrainingTargets + .slice(currentIndex + 1) + .reduce((sum, target) => sum + Math.max(0, trainingEstimateByTarget[target] || 0), 0) + + return Math.max(0, currentRemainingMs + laterTargetsMs) }, [ + currentTrainingTarget, + currentTrainingTargetProgress, + effectiveTrainingTargets, trainingRunning, trainingStatus?.training?.epoch, trainingStatus?.training?.epochs, estimatedEpochMs, + trainingEstimateByTarget, ]) useEffect(() => { @@ -6107,6 +7091,8 @@ export default function TrainingTab(props: { const detectorReady = Boolean(detector?.dataReady) const pose = trainingStatus?.pose const poseReady = Boolean(pose?.dataReady) + const videoMAE = trainingStatus?.videoMAE + const videoMAEReady = Boolean(videoMAE?.dataReady) const missingTrain = Math.max( 0, Number(detector?.requiredTrain ?? 20) - Number(detector?.trainCount ?? 0) @@ -6160,11 +7146,22 @@ export default function TrainingTab(props: { const poseRequired = Math.max(0, poseRequiredTrain) + Math.max(0, poseRequiredVal) + const videoMAETrainCount = Number(videoMAE?.trainCount ?? 0) + const videoMAEValCount = Number(videoMAE?.valCount ?? 0) + const videoMAEEligibleCount = Number(videoMAE?.eligibleCount ?? 0) + const videoMAERequiredTrain = Number(videoMAE?.requiredTrain ?? 40) + const videoMAERequiredVal = Number(videoMAE?.requiredVal ?? 5) + const videoMAEDone = + boundedCount(videoMAETrainCount, videoMAERequiredTrain) + + boundedCount(videoMAEValCount, videoMAERequiredVal) + const videoMAERequired = + Math.max(0, videoMAERequiredTrain) + + Math.max(0, videoMAERequiredVal) const feedbackDone = boundedCount(feedbackCount, requiredCount) const feedbackRequired = Math.max(0, requiredCount) - const readinessDone = feedbackDone + detectorDone + poseDone - const readinessRequired = feedbackRequired + detectorRequired + poseRequired + const readinessDone = feedbackDone + detectorDone + poseDone + videoMAEDone + const readinessRequired = feedbackRequired + detectorRequired + poseRequired + videoMAERequired const readinessProgress = readinessRequired > 0 ? readinessDone / readinessRequired : 0 const readinessItems = [ { @@ -6181,6 +7178,13 @@ export default function TrainingTab(props: { text: `${poseDone}/${poseRequired}`, detail: `Train ${poseTrainCount}/${poseRequiredTrain}, Val ${poseValCount}/${poseRequiredVal}`, }, + { + key: 'videomae', + label: 'VideoMAE', + value: videoMAERequired > 0 ? videoMAEDone / videoMAERequired : 1, + text: `${videoMAEDone}/${videoMAERequired}`, + detail: `Eligible ${videoMAEEligibleCount}, Train ${videoMAETrainCount}/${videoMAERequiredTrain}, Val ${videoMAEValCount}/${videoMAERequiredVal}`, + }, ] const progress = clampPercent( @@ -6192,6 +7196,8 @@ export default function TrainingTab(props: { ? shownTrainingStep || 'Training läuft…' : !feedbackReady ? `${Math.max(0, requiredCount - feedbackCount)} Feedback fehlen noch` + : canStartTraining + ? 'Bereit zum Trainieren' : !detectorReady ? missingTrain > 0 || missingVal > 0 ? `YOLO-Beispiele fehlen: ${missingTrain} Train, ${missingVal} Val` @@ -6202,9 +7208,9 @@ export default function TrainingTab(props: { ? missingPoseTrain > 0 || missingPoseVal > 0 ? `Pose-Beispiele fehlen: ${missingPoseTrain} Train, ${missingPoseVal} Val` : 'Pose-Datensatz noch nicht bereit' - : canStartTraining - ? 'Bereit zum Trainieren' - : 'Noch nicht trainingsbereit' + : !videoMAEReady + ? 'VideoMAE-Datensatz noch nicht bereit' + : 'Noch nicht trainingsbereit' return (
+ setTrainingStartModalOpen(false)} + title="Training starten" + width="max-w-xl" + footer={ + <> + + + + + } + > +
+
+ + + +
+ +
+
+ + Geschätzte Gesamtdauer + + + + {trainingStartTotalEstimateText} + +
+ +
+ {plannedTrainingTargets.length > 0 + ? `${plannedTrainingTargets.length} Training${plannedTrainingTargets.length === 1 ? '' : 's'} eingeplant` + : 'Noch kein bereites Training ausgewählt'} +
+ +
+ {trainingEstimateSettings + ? trainingEstimateRuntimeLabel + : 'Trainingsmodus wird geladen...'} +
+
+ +
+ {trainingStartOptions.map((option) => { + const fullMode = trainingStartMode === 'full' + const selected = + !fullMode && trainingStartTargets.includes(option.key) + const disabled = fullMode || !option.ready + const Icon = option.icon + + return ( + + ) + })} +
+ + {trainingStartMode === 'custom' && selectedTrainingTargets.length === 0 ? ( +
+ Wähle mindestens ein bereites Training aus. +
+ ) : null} +
+ + setStatsModalOpen(false)}