bugfixes
This commit is contained in:
parent
e4a474ad54
commit
af60ee5319
BIN
backend/dist/nsfwapp-linux-amd64
vendored
BIN
backend/dist/nsfwapp-linux-amd64
vendored
Binary file not shown.
BIN
backend/dist/nsfwapp.exe
vendored
BIN
backend/dist/nsfwapp.exe
vendored
Binary file not shown.
BIN
backend/dist/nsfwapp_amd64.deb
vendored
BIN
backend/dist/nsfwapp_amd64.deb
vendored
Binary file not shown.
@ -385,6 +385,7 @@ def main():
|
||||
"ok": True,
|
||||
"model": str(final_model),
|
||||
"sourceModel": str(best),
|
||||
"baseModel": str(base_used),
|
||||
"runs": str(result_path),
|
||||
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
||||
"trainSamples": train_count,
|
||||
@ -392,6 +393,7 @@ def main():
|
||||
"epochs": epochs,
|
||||
"imgsz": imgsz,
|
||||
"batchSize": batch_size,
|
||||
"patience": patience,
|
||||
"threads": threads,
|
||||
"workers": workers,
|
||||
"device": str(train_device),
|
||||
|
||||
@ -371,6 +371,7 @@ def main():
|
||||
"ok": True,
|
||||
"model": str(final_model),
|
||||
"sourceModel": str(best),
|
||||
"baseModel": str(base_model),
|
||||
"runs": str(result_path),
|
||||
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
||||
"trainSamples": train_count,
|
||||
@ -378,6 +379,7 @@ def main():
|
||||
"epochs": epochs,
|
||||
"imgsz": imgsz,
|
||||
"batchSize": batch_size,
|
||||
"patience": patience,
|
||||
"threads": threads,
|
||||
"workers": workers,
|
||||
"device": str(train_device),
|
||||
|
||||
@ -200,6 +200,7 @@ def main():
|
||||
parser.add_argument("--workers", default="0")
|
||||
parser.add_argument("--threads", default="0")
|
||||
parser.add_argument("--num-frames", default="16")
|
||||
parser.add_argument("--patience", default="3")
|
||||
parser.add_argument("--freeze-backbone", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
@ -213,6 +214,7 @@ def main():
|
||||
workers = max(0, safe_int(args.workers, 0))
|
||||
threads = max(0, safe_int(args.threads, 0))
|
||||
num_frames = max(2, safe_int(args.num_frames, 16))
|
||||
patience = max(0, safe_int(args.patience, 3))
|
||||
lr = max(1e-7, safe_float(args.lr, 5e-5))
|
||||
|
||||
if threads > 0:
|
||||
@ -249,18 +251,23 @@ def main():
|
||||
if not val_entries:
|
||||
raise SystemExit("no VideoMAE val clips found")
|
||||
|
||||
base_model = str(args.base or DEFAULT_BASE_MODEL).strip() or DEFAULT_BASE_MODEL
|
||||
base_path = Path(base_model).expanduser()
|
||||
if base_path.exists():
|
||||
base_model = str(base_path.resolve())
|
||||
|
||||
emit_progress(
|
||||
"videomae",
|
||||
0.03,
|
||||
"VideoMAE-Basismodell wird geladen...",
|
||||
base=args.base,
|
||||
base=base_model,
|
||||
labels=len(labels),
|
||||
device=str(device),
|
||||
)
|
||||
|
||||
image_processor = AutoImageProcessor.from_pretrained(args.base)
|
||||
image_processor = AutoImageProcessor.from_pretrained(base_model)
|
||||
model = VideoMAEForVideoClassification.from_pretrained(
|
||||
args.base,
|
||||
base_model,
|
||||
num_labels=len(labels),
|
||||
label2id=label_to_id,
|
||||
id2label=id_to_label,
|
||||
@ -302,8 +309,11 @@ def main():
|
||||
best_accuracy = -1.0
|
||||
best_loss = math.inf
|
||||
best_epoch = 0
|
||||
completed_epochs = 0
|
||||
epochs_without_improvement = 0
|
||||
|
||||
for epoch in range(1, epochs + 1):
|
||||
completed_epochs = epoch
|
||||
model.train()
|
||||
running_loss = 0.0
|
||||
seen = 0
|
||||
@ -346,8 +356,11 @@ def main():
|
||||
best_accuracy = val_accuracy
|
||||
best_loss = val_loss
|
||||
best_epoch = epoch
|
||||
epochs_without_improvement = 0
|
||||
model.save_pretrained(tmp_dir)
|
||||
image_processor.save_pretrained(tmp_dir)
|
||||
else:
|
||||
epochs_without_improvement += 1
|
||||
|
||||
emit_progress(
|
||||
"videomae",
|
||||
@ -360,16 +373,36 @@ def main():
|
||||
device=str(device),
|
||||
accuracy=val_accuracy,
|
||||
loss=val_loss,
|
||||
patience=patience,
|
||||
epochsWithoutImprovement=epochs_without_improvement,
|
||||
)
|
||||
|
||||
if patience > 0 and epochs_without_improvement >= patience:
|
||||
emit_progress(
|
||||
"videomae",
|
||||
0.96,
|
||||
"VideoMAE Early-Stopping: keine weitere Verbesserung.",
|
||||
epoch=epoch,
|
||||
epochs=epochs,
|
||||
bestEpoch=best_epoch,
|
||||
patience=patience,
|
||||
trainSamples=len(train_entries),
|
||||
valSamples=len(val_entries),
|
||||
device=str(device),
|
||||
accuracy=best_accuracy if best_accuracy >= 0 else 0.0,
|
||||
loss=best_loss if math.isfinite(best_loss) else 0.0,
|
||||
)
|
||||
break
|
||||
|
||||
if best_epoch <= 0:
|
||||
model.save_pretrained(tmp_dir)
|
||||
image_processor.save_pretrained(tmp_dir)
|
||||
best_epoch = epochs
|
||||
best_epoch = completed_epochs or epochs
|
||||
|
||||
status = {
|
||||
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
||||
"epochs": epochs,
|
||||
"completedEpochs": completed_epochs,
|
||||
"bestEpoch": best_epoch,
|
||||
"trainSamples": len(train_entries),
|
||||
"valSamples": len(val_entries),
|
||||
@ -377,7 +410,8 @@ def main():
|
||||
"workers": workers,
|
||||
"threads": threads,
|
||||
"numFrames": num_frames,
|
||||
"baseModel": args.base,
|
||||
"patience": patience,
|
||||
"baseModel": base_model,
|
||||
"device": str(device),
|
||||
"accuracy": best_accuracy if best_accuracy >= 0 else 0.0,
|
||||
"loss": best_loss if math.isfinite(best_loss) else 0.0,
|
||||
|
||||
@ -118,6 +118,103 @@ type TrainingSkipRequest struct {
|
||||
SampleID string `json:"sampleId"`
|
||||
}
|
||||
|
||||
type TrainingTrainRequest struct {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Targets []string `json:"targets,omitempty"`
|
||||
}
|
||||
|
||||
type trainingTrainTargets struct {
|
||||
Detector bool
|
||||
Pose bool
|
||||
VideoMAE bool
|
||||
}
|
||||
|
||||
func trainingAllTrainTargets() trainingTrainTargets {
|
||||
return trainingTrainTargets{
|
||||
Detector: true,
|
||||
Pose: true,
|
||||
VideoMAE: true,
|
||||
}
|
||||
}
|
||||
|
||||
func (t trainingTrainTargets) empty() bool {
|
||||
return !t.Detector && !t.Pose && !t.VideoMAE
|
||||
}
|
||||
|
||||
func (t trainingTrainTargets) list() []string {
|
||||
out := make([]string, 0, 3)
|
||||
if t.Detector {
|
||||
out = append(out, "detector")
|
||||
}
|
||||
if t.Pose {
|
||||
out = append(out, "pose")
|
||||
}
|
||||
if t.VideoMAE {
|
||||
out = append(out, "videomae")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func trainingReadTrainRequest(r *http.Request) (TrainingTrainRequest, error) {
|
||||
var req TrainingTrainRequest
|
||||
if r.Body == nil {
|
||||
return req, nil
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||||
if err != nil {
|
||||
return req, err
|
||||
}
|
||||
|
||||
if strings.TrimSpace(string(body)) == "" {
|
||||
return req, nil
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
return req, err
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func trainingNormalizeTrainTargets(req TrainingTrainRequest) (trainingTrainTargets, bool, error) {
|
||||
scope := strings.ToLower(strings.TrimSpace(req.Scope))
|
||||
if scope == "" || scope == "full" || scope == "all" || scope == "complete" {
|
||||
return trainingAllTrainTargets(), false, nil
|
||||
}
|
||||
|
||||
if scope != "custom" && scope != "selected" && scope != "partial" {
|
||||
return trainingTrainTargets{}, false, fmt.Errorf("unbekannter Trainingsumfang: %s", req.Scope)
|
||||
}
|
||||
|
||||
var targets trainingTrainTargets
|
||||
for _, raw := range req.Targets {
|
||||
key := strings.ToLower(strings.TrimSpace(raw))
|
||||
key = strings.ReplaceAll(key, "_", "")
|
||||
key = strings.ReplaceAll(key, "-", "")
|
||||
key = strings.ReplaceAll(key, " ", "")
|
||||
|
||||
switch key {
|
||||
case "detector", "yolo", "yolo26", "yolo26detector", "box", "boxes", "boxdetection":
|
||||
targets.Detector = true
|
||||
case "pose", "yolo26pose", "posedetection":
|
||||
targets.Pose = true
|
||||
case "videomae", "video", "clip", "scene", "clipanalysis", "clipanalyse":
|
||||
targets.VideoMAE = true
|
||||
case "":
|
||||
continue
|
||||
default:
|
||||
return trainingTrainTargets{}, true, fmt.Errorf("unbekanntes Training: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
if targets.empty() {
|
||||
return trainingTrainTargets{}, true, errors.New("kein Training ausgewählt")
|
||||
}
|
||||
|
||||
return targets, true, nil
|
||||
}
|
||||
|
||||
type TrainingAnnotation struct {
|
||||
SampleID string `json:"sampleId"`
|
||||
FrameURL string `json:"frameUrl"`
|
||||
@ -1313,6 +1410,21 @@ func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRunt
|
||||
return opts
|
||||
}
|
||||
|
||||
func trainingYoloEarlyStoppingPatience(epochs int) int {
|
||||
epochs = clampTrainingInt(epochs, 1, 300)
|
||||
patience := epochs / 4
|
||||
if patience < 5 {
|
||||
patience = 5
|
||||
}
|
||||
if patience > 20 {
|
||||
patience = 20
|
||||
}
|
||||
if patience > epochs {
|
||||
patience = epochs
|
||||
}
|
||||
return patience
|
||||
}
|
||||
|
||||
func trainingCommandEnv(opts trainingRuntimeOptions) []string {
|
||||
env := os.Environ()
|
||||
if opts.CPUThreads <= 0 {
|
||||
@ -3212,6 +3324,18 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := trainingReadTrainRequest(r)
|
||||
if err != nil {
|
||||
trainingWriteError(w, http.StatusBadRequest, "invalid json")
|
||||
return
|
||||
}
|
||||
|
||||
targets, customTargets, err := trainingNormalizeTrainTargets(req)
|
||||
if err != nil {
|
||||
trainingWriteError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
current := trainingGetJobStatus()
|
||||
if current.Running {
|
||||
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
||||
@ -3301,7 +3425,70 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
|
||||
videoMAEDataReady := videoMAEEligibleCount >= minVideoMAETrainCount ||
|
||||
(videoMAETrainCount >= minVideoMAETrainCount && videoMAEValCount >= minVideoMAEValCount)
|
||||
|
||||
if detectorDataReady || poseDataReady || videoMAEDataReady {
|
||||
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
||||
selectedDataReady :=
|
||||
(targets.Detector && detectorDataReady) ||
|
||||
(targets.Pose && poseDataReady) ||
|
||||
(targets.VideoMAE && videoMAEDataReady)
|
||||
|
||||
if customTargets {
|
||||
if targets.Detector && !detectorDataReady {
|
||||
trainingWriteError(
|
||||
w,
|
||||
http.StatusBadRequest,
|
||||
fmt.Sprintf(
|
||||
"YOLO26 Detector ist noch nicht trainingsbereit. Train=%d (%d positiv), Val=%d (%d positiv). Benoetigt: mindestens %d Train, %d Val und je ein positives Beispiel.",
|
||||
trainCount,
|
||||
positiveTrainCount,
|
||||
valCount,
|
||||
positiveValCount,
|
||||
minDetectorTrainCount,
|
||||
minDetectorValCount,
|
||||
),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if targets.Pose && !poseDataReady {
|
||||
trainingWriteError(
|
||||
w,
|
||||
http.StatusBadRequest,
|
||||
fmt.Sprintf(
|
||||
"YOLO26 Pose ist noch nicht trainingsbereit. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
||||
poseTrainCount,
|
||||
poseValCount,
|
||||
minPoseTrainCount,
|
||||
minPoseValCount,
|
||||
),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if targets.VideoMAE && !runtimeOpts.VideoMAEEnabled {
|
||||
trainingWriteError(w, http.StatusBadRequest, "VideoMAE ist in den Training-Settings deaktiviert.")
|
||||
return
|
||||
}
|
||||
|
||||
if targets.VideoMAE && !videoMAEDataReady {
|
||||
trainingWriteError(
|
||||
w,
|
||||
http.StatusBadRequest,
|
||||
fmt.Sprintf(
|
||||
"VideoMAE ist noch nicht trainingsbereit. Eligible=%d, Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
||||
videoMAEEligibleCount,
|
||||
videoMAETrainCount,
|
||||
videoMAEValCount,
|
||||
minVideoMAETrainCount,
|
||||
minVideoMAEValCount,
|
||||
),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
goto startTraining
|
||||
}
|
||||
|
||||
if selectedDataReady {
|
||||
goto startTraining
|
||||
}
|
||||
|
||||
@ -3348,12 +3535,13 @@ startTraining:
|
||||
|
||||
trainingStartJob(cancel)
|
||||
|
||||
go trainingRunJob(ctx, root, feedbackCount)
|
||||
go trainingRunJob(ctx, root, feedbackCount, targets)
|
||||
|
||||
trainingWriteJSON(w, http.StatusAccepted, map[string]any{
|
||||
"ok": true,
|
||||
"message": "Training gestartet.",
|
||||
"training": trainingGetJobStatus(),
|
||||
"targets": targets.list(),
|
||||
"detector": map[string]any{
|
||||
"trainCount": trainCount,
|
||||
"valCount": valCount,
|
||||
@ -3382,6 +3570,7 @@ startTraining:
|
||||
"requiredTrain": minVideoMAETrainCount,
|
||||
"requiredVal": minVideoMAEValCount,
|
||||
"manifest": videoMAEManifest,
|
||||
"dataReady": videoMAEDataReady,
|
||||
"source": "videomae_clip",
|
||||
},
|
||||
})
|
||||
@ -3424,7 +3613,7 @@ func trainingCancelHandler(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
func trainingRunJob(ctx context.Context, root string, count int, targets trainingTrainTargets) {
|
||||
if err := ensureMLPythonSetup(ctx); err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
trainingFinishCancelled(root)
|
||||
@ -3459,7 +3648,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
appLogln("ML-Python für Training:", python)
|
||||
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
||||
appLogf(
|
||||
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v",
|
||||
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v targets=%s",
|
||||
runtimeOpts.PerformanceMode,
|
||||
runtimeOpts.CPUCoreCount,
|
||||
runtimeOpts.PowerSaveMode,
|
||||
@ -3468,6 +3657,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
runtimeOpts.YoloBatchSize,
|
||||
runtimeOpts.LowPriority,
|
||||
runtimeOpts.VideoMAEEnabled,
|
||||
strings.Join(targets.list(), ","),
|
||||
)
|
||||
|
||||
cleanOutput := func(text string) string {
|
||||
@ -3485,10 +3675,13 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
|
||||
detectorOutput := ""
|
||||
detectorStatus := "skipped"
|
||||
detectorDurationMs := int64(0)
|
||||
poseOutput := ""
|
||||
poseStatus := "skipped"
|
||||
poseDurationMs := int64(0)
|
||||
videoMAEOutput := ""
|
||||
videoMAEStatus := "skipped"
|
||||
videoMAEDurationMs := int64(0)
|
||||
|
||||
detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml")
|
||||
detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train")
|
||||
@ -3514,29 +3707,50 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
fileExistsNonEmpty(detectorDatasetYAML),
|
||||
)
|
||||
|
||||
if fileExistsNonEmpty(detectorDatasetYAML) &&
|
||||
if !targets.Detector {
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
if s.Progress < 58 {
|
||||
s.Progress = 58
|
||||
}
|
||||
s.Step = "YOLO26 Detector wurde nicht ausgewählt."
|
||||
})
|
||||
detectorStatus = "skipped_unselected"
|
||||
detectorOutput = "YOLO26 Detector übersprungen: nicht ausgewählt."
|
||||
appLogln(detectorOutput)
|
||||
} else if fileExistsNonEmpty(detectorDatasetYAML) &&
|
||||
trainCount >= minDetectorTrainCount &&
|
||||
valCount >= minDetectorValCount &&
|
||||
positiveTrainCount > 0 &&
|
||||
positiveValCount > 0 {
|
||||
detectorStartedAt := time.Now()
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
s.Progress = 15
|
||||
s.Step = "YOLO26 Detector wird trainiert…"
|
||||
})
|
||||
|
||||
detectorBasePath := "yolo26n.pt"
|
||||
if p, err := trainingMLCacheFilePath("yolo26n.pt"); err == nil {
|
||||
detectorBasePath = p
|
||||
detectorModel := trainingResolveDetectorModel(root)
|
||||
detectorBasePath := detectorModel.BestPath
|
||||
if !detectorModel.TrainedExists {
|
||||
detectorBasePath = "yolo26n.pt"
|
||||
if p, err := trainingMLCacheFilePath("yolo26n.pt"); err == nil {
|
||||
detectorBasePath = p
|
||||
}
|
||||
}
|
||||
detectorEpochs := trainingDetectorEpochs()
|
||||
detectorPatience := trainingYoloEarlyStoppingPatience(detectorEpochs)
|
||||
if detectorModel.TrainedExists {
|
||||
appLogln("YOLO26 Detector Fine-Tuning startet von:", detectorBasePath)
|
||||
}
|
||||
|
||||
detectorScript := trainingScriptPath("train_detector_model.py")
|
||||
detectorArgs := []string{
|
||||
"--root", root,
|
||||
"--base", detectorBasePath,
|
||||
"--epochs", strconv.Itoa(trainingDetectorEpochs()),
|
||||
"--epochs", strconv.Itoa(detectorEpochs),
|
||||
"--imgsz", "640",
|
||||
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
||||
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
||||
"--patience", strconv.Itoa(detectorPatience),
|
||||
}
|
||||
if runtimeOpts.YoloBatchSize > 0 {
|
||||
detectorArgs = append(detectorArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
||||
@ -3556,6 +3770,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
},
|
||||
detectorArgs...,
|
||||
)
|
||||
detectorDurationMs = time.Since(detectorStartedAt).Milliseconds()
|
||||
|
||||
if errors.Is(detectorErr, errTrainingCancelled) {
|
||||
appLogln("⛔ YOLO26 detector training cancelled")
|
||||
@ -3601,24 +3816,38 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
poseTrainLabels := filepath.Join(root, "pose", "dataset", "labels", "train")
|
||||
poseValImages := filepath.Join(root, "pose", "dataset", "images", "val")
|
||||
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
|
||||
poseStartedAt := time.Time{}
|
||||
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
if s.Progress < 60 {
|
||||
s.Progress = 60
|
||||
}
|
||||
s.Step = "YOLO26 Pose-Daten werden aufgebaut..."
|
||||
})
|
||||
|
||||
if written, err := trainingSyncPoseDataset(root); err != nil {
|
||||
poseStatus = "failed"
|
||||
poseOutput = "YOLO26 Pose-Dataset konnte nicht aufgebaut werden: " + err.Error()
|
||||
if !targets.Pose {
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
if s.Progress < 82 {
|
||||
s.Progress = 82
|
||||
}
|
||||
s.Step = "YOLO26 Pose wurde nicht ausgewählt."
|
||||
})
|
||||
poseStatus = "skipped_unselected"
|
||||
poseOutput = "YOLO26 Pose übersprungen: nicht ausgewählt."
|
||||
appLogln(poseOutput)
|
||||
} else {
|
||||
appLogln("pose samples synced:", written)
|
||||
}
|
||||
poseStartedAt = time.Now()
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
if s.Progress < 60 {
|
||||
s.Progress = 60
|
||||
}
|
||||
s.Step = "YOLO26 Pose-Daten werden aufgebaut..."
|
||||
})
|
||||
|
||||
if err := trainingEnsurePoseValidationSample(root); err != nil {
|
||||
appLogln("pose val sample ensure failed:", err)
|
||||
if written, err := trainingSyncPoseDataset(root); err != nil {
|
||||
poseStatus = "failed"
|
||||
poseOutput = "YOLO26 Pose-Dataset konnte nicht aufgebaut werden: " + err.Error()
|
||||
appLogln(poseOutput)
|
||||
} else {
|
||||
appLogln("pose samples synced:", written)
|
||||
}
|
||||
|
||||
if err := trainingEnsurePoseValidationSample(root); err != nil {
|
||||
appLogln("pose val sample ensure failed:", err)
|
||||
}
|
||||
}
|
||||
|
||||
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
|
||||
@ -3631,7 +3860,8 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
fileExistsNonEmpty(poseDatasetYAML),
|
||||
)
|
||||
|
||||
if poseStatus != "failed" &&
|
||||
if targets.Pose &&
|
||||
poseStatus != "failed" &&
|
||||
fileExistsNonEmpty(poseDatasetYAML) &&
|
||||
poseTrainCount >= minPoseTrainCount &&
|
||||
poseValCount >= minPoseValCount {
|
||||
@ -3642,19 +3872,26 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
s.Step = "YOLO26 Pose wird trainiert..."
|
||||
})
|
||||
|
||||
poseModel := trainingResolvePoseModel(root)
|
||||
poseBasePath := "yolo26n-pose.pt"
|
||||
if p, err := embeddedPoseModelPath(); err == nil && fileExistsNonEmpty(p) {
|
||||
poseBasePath = p
|
||||
if poseModel.EffectiveExists {
|
||||
poseBasePath = poseModel.EffectivePath
|
||||
}
|
||||
poseEpochs := trainingDetectorEpochs()
|
||||
posePatience := trainingYoloEarlyStoppingPatience(poseEpochs)
|
||||
if poseModel.TrainedExists {
|
||||
appLogln("YOLO26 Pose Fine-Tuning startet von:", poseBasePath)
|
||||
}
|
||||
|
||||
poseScript := trainingScriptPath("train_pose_model.py")
|
||||
poseArgs := []string{
|
||||
"--root", root,
|
||||
"--base", poseBasePath,
|
||||
"--epochs", strconv.Itoa(trainingDetectorEpochs()),
|
||||
"--epochs", strconv.Itoa(poseEpochs),
|
||||
"--imgsz", "640",
|
||||
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
||||
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
||||
"--patience", strconv.Itoa(posePatience),
|
||||
}
|
||||
if runtimeOpts.YoloBatchSize > 0 {
|
||||
poseArgs = append(poseArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
||||
@ -3674,6 +3911,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
},
|
||||
poseArgs...,
|
||||
)
|
||||
poseDurationMs = time.Since(poseStartedAt).Milliseconds()
|
||||
|
||||
if errors.Is(poseErr, errTrainingCancelled) {
|
||||
appLogln("YOLO26 pose training cancelled")
|
||||
@ -3698,7 +3936,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
appLogln("YOLO26 pose training:", poseOutputClean)
|
||||
}
|
||||
}
|
||||
} else if poseStatus != "failed" {
|
||||
} else if targets.Pose && poseStatus != "failed" {
|
||||
poseStatus = "skipped_no_pose_data"
|
||||
poseOutput = fmt.Sprintf(
|
||||
"YOLO26 Pose übersprungen: zu wenige Skeleton-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
||||
@ -3710,9 +3948,26 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
appLogln(poseOutput)
|
||||
}
|
||||
|
||||
poseOutputClean := cleanOutput(poseOutput)
|
||||
if poseStatus == "trained" && poseDurationMs <= 0 && !poseStartedAt.IsZero() {
|
||||
poseDurationMs = time.Since(poseStartedAt).Milliseconds()
|
||||
}
|
||||
|
||||
if !runtimeOpts.VideoMAEEnabled {
|
||||
poseOutputClean := cleanOutput(poseOutput)
|
||||
videoMAEStartedAt := time.Time{}
|
||||
videoMAETrainCount := 0
|
||||
videoMAEValCount := 0
|
||||
|
||||
if !targets.VideoMAE {
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
if s.Progress < 98 {
|
||||
s.Progress = 98
|
||||
}
|
||||
s.Step = "VideoMAE wurde nicht ausgewählt."
|
||||
})
|
||||
videoMAEStatus = "skipped_unselected"
|
||||
videoMAEOutput = "VideoMAE übersprungen: nicht ausgewählt."
|
||||
appLogln(videoMAEOutput)
|
||||
} else if !runtimeOpts.VideoMAEEnabled {
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
if s.Progress < 84 {
|
||||
s.Progress = 84
|
||||
@ -3723,6 +3978,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
videoMAEOutput = "VideoMAE übersprungen: in den Training-Settings deaktiviert."
|
||||
appLogln(videoMAEOutput)
|
||||
} else {
|
||||
videoMAEStartedAt = time.Now()
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
if s.Progress < 84 {
|
||||
s.Progress = 84
|
||||
@ -3730,7 +3986,9 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
s.Step = "VideoMAE Clip-Daten werden aufgebaut..."
|
||||
})
|
||||
|
||||
videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr :=
|
||||
var videoMAEWritten int
|
||||
var videoMAESyncErr error
|
||||
videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr =
|
||||
trainingSyncVideoMAEDataset(ctx, root)
|
||||
if errors.Is(videoMAESyncErr, context.Canceled) || errors.Is(videoMAESyncErr, errTrainingCancelled) {
|
||||
appLogln("VideoMAE dataset sync cancelled")
|
||||
@ -3761,6 +4019,13 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
})
|
||||
|
||||
videoMAEScript := trainingScriptPath("train_videomae_model.py")
|
||||
videoMAEBase := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL"))
|
||||
if videoMAEBase == "" {
|
||||
if model := trainingResolveVideoMAEModel(root); model.TrainedExists {
|
||||
videoMAEBase = model.EffectivePath
|
||||
appLogln("VideoMAE Fine-Tuning startet von:", videoMAEBase)
|
||||
}
|
||||
}
|
||||
videoMAEArgs := []string{
|
||||
"--root", root,
|
||||
"--epochs", strconv.Itoa(trainingVideoMAEEpochs()),
|
||||
@ -3768,9 +4033,10 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
"--num-frames", strconv.Itoa(trainingVideoMAENumFrames),
|
||||
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
||||
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
||||
"--patience", strconv.Itoa(trainingVideoMAEEarlyStoppingPatience()),
|
||||
}
|
||||
if base := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL")); base != "" {
|
||||
videoMAEArgs = append(videoMAEArgs, "--base", base)
|
||||
if videoMAEBase != "" {
|
||||
videoMAEArgs = append(videoMAEArgs, "--base", videoMAEBase)
|
||||
}
|
||||
|
||||
videoMAEOut, videoMAEErr := trainingRunCommandStreaming(
|
||||
@ -3787,6 +4053,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
},
|
||||
videoMAEArgs...,
|
||||
)
|
||||
videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds()
|
||||
|
||||
if errors.Is(videoMAEErr, errTrainingCancelled) {
|
||||
appLogln("VideoMAE training cancelled")
|
||||
@ -3822,6 +4089,10 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
}
|
||||
}
|
||||
|
||||
if videoMAEStatus == "trained" && videoMAEDurationMs <= 0 && !videoMAEStartedAt.IsZero() {
|
||||
videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds()
|
||||
}
|
||||
|
||||
videoMAEOutputClean := cleanOutput(videoMAEOutput)
|
||||
|
||||
message := "Training abgeschlossen."
|
||||
@ -3831,6 +4102,9 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
case "trained":
|
||||
message = "Training abgeschlossen. YOLO26 Detector wurde trainiert."
|
||||
|
||||
case "skipped_unselected":
|
||||
message = "Training abgeschlossen."
|
||||
|
||||
case "skipped_no_detector_data":
|
||||
message = detectorOutput
|
||||
|
||||
@ -3887,8 +4161,31 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
||||
}
|
||||
|
||||
if detectorStatus == "trained" || poseStatus == "trained" || videoMAEStatus == "trained" {
|
||||
// Verlaufseintrag schreiben, solange die Job-Startzeit für die Dauer noch verfügbar ist.
|
||||
trainingAppendRunHistory(root)
|
||||
// Verlauf pro erfolgreich trainiertem Ziel schreiben.
|
||||
if detectorStatus == "trained" {
|
||||
trainingAppendTargetHistory(root, "detector", detectorStatus, detectorDurationMs, runtimeOpts, TrainingHistoryEntry{
|
||||
Epochs: trainingDetectorEpochs(),
|
||||
TrainSamples: trainCount,
|
||||
ValSamples: valCount,
|
||||
Imgsz: 640,
|
||||
})
|
||||
}
|
||||
if poseStatus == "trained" {
|
||||
trainingAppendTargetHistory(root, "pose", poseStatus, poseDurationMs, runtimeOpts, TrainingHistoryEntry{
|
||||
Epochs: trainingDetectorEpochs(),
|
||||
TrainSamples: poseTrainCount,
|
||||
ValSamples: poseValCount,
|
||||
Imgsz: 640,
|
||||
})
|
||||
}
|
||||
if videoMAEStatus == "trained" {
|
||||
trainingAppendTargetHistory(root, "videomae", videoMAEStatus, videoMAEDurationMs, runtimeOpts, TrainingHistoryEntry{
|
||||
Epochs: trainingVideoMAEEpochs(),
|
||||
TrainSamples: videoMAETrainCount,
|
||||
ValSamples: videoMAEValCount,
|
||||
Imgsz: trainingVideoMAEFrameSize,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||
@ -4338,20 +4635,22 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
"predictsClothing": false,
|
||||
"predictsBoxes": false,
|
||||
|
||||
"feedbackCount": feedbackCount,
|
||||
"eligibleCount": videoMAEEligibleCount,
|
||||
"trainCount": videoMAETrainCount,
|
||||
"valCount": videoMAEValCount,
|
||||
"requiredTrain": minVideoMAETrainCount,
|
||||
"requiredVal": minVideoMAEValCount,
|
||||
"requiredCount": minVideoMAETrainCount,
|
||||
"datasetReady": videoMAEDatasetReady,
|
||||
"manifest": videoMAEManifest,
|
||||
"dataReady": videoMAEDataReady,
|
||||
"modelReady": videoMAEModel.EffectiveExists,
|
||||
"modelExists": videoMAEModel.EffectiveExists,
|
||||
"modelPath": videoMAEModel.EffectivePath,
|
||||
"modelSource": videoMAEModel.Source,
|
||||
"feedbackCount": feedbackCount,
|
||||
"eligibleCount": videoMAEEligibleCount,
|
||||
"trainCount": videoMAETrainCount,
|
||||
"valCount": videoMAEValCount,
|
||||
"requiredTrain": minVideoMAETrainCount,
|
||||
"requiredVal": minVideoMAEValCount,
|
||||
"requiredCount": minVideoMAETrainCount,
|
||||
"datasetReady": videoMAEDatasetReady,
|
||||
"manifest": videoMAEManifest,
|
||||
"dataReady": videoMAEDataReady,
|
||||
"modelReady": videoMAEModel.EffectiveExists,
|
||||
"modelExists": videoMAEModel.EffectiveExists,
|
||||
"modelPath": videoMAEModel.EffectivePath,
|
||||
"modelSource": videoMAEModel.Source,
|
||||
"trainedModelExists": videoMAEModel.TrainedExists,
|
||||
"trainedModelPath": videoMAEModel.BestPath,
|
||||
},
|
||||
|
||||
"pipeline": map[string]any{
|
||||
@ -4464,16 +4763,24 @@ func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo {
|
||||
}
|
||||
|
||||
type TrainingHistoryEntry struct {
|
||||
TrainedAt string `json:"trainedAt,omitempty"`
|
||||
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
||||
DurationMs int64 `json:"durationMs,omitempty"`
|
||||
Epochs int `json:"epochs,omitempty"`
|
||||
TrainSamples int `json:"trainSamples,omitempty"`
|
||||
ValSamples int `json:"valSamples,omitempty"`
|
||||
Imgsz int `json:"imgsz,omitempty"`
|
||||
Device string `json:"device,omitempty"`
|
||||
MAP50 float64 `json:"map50,omitempty"`
|
||||
MAP5095 float64 `json:"map5095,omitempty"`
|
||||
TrainedAt string `json:"trainedAt,omitempty"`
|
||||
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
||||
Target string `json:"target,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
DurationMs int64 `json:"durationMs,omitempty"`
|
||||
Epochs int `json:"epochs,omitempty"`
|
||||
TrainSamples int `json:"trainSamples,omitempty"`
|
||||
ValSamples int `json:"valSamples,omitempty"`
|
||||
Imgsz int `json:"imgsz,omitempty"`
|
||||
Device string `json:"device,omitempty"`
|
||||
MAP50 float64 `json:"map50,omitempty"`
|
||||
MAP5095 float64 `json:"map5095,omitempty"`
|
||||
PerformanceMode string `json:"performanceMode,omitempty"`
|
||||
CPUCoreCount int `json:"cpuCoreCount,omitempty"`
|
||||
CPUThreads int `json:"cpuThreads,omitempty"`
|
||||
Workers int `json:"workers,omitempty"`
|
||||
YoloBatchSize int `json:"yoloBatchSize,omitempty"`
|
||||
LowPriority bool `json:"lowPriority,omitempty"`
|
||||
}
|
||||
|
||||
type TrainingHistoryResponse struct {
|
||||
@ -4485,31 +4792,81 @@ func trainingHistoryPath(root string) string {
|
||||
return filepath.Join(root, "detector", "training_history.jsonl")
|
||||
}
|
||||
|
||||
// trainingAppendRunHistory hängt nach einem erfolgreichen Trainingslauf einen
|
||||
// Verlaufseintrag an (Datum, mAP, Samples, Epochen, Dauer).
|
||||
func trainingAppendRunHistory(root string) {
|
||||
info := trainingReadModelInfo(root)
|
||||
if info == nil {
|
||||
return
|
||||
// trainingAppendTargetHistory haengt nach einem erfolgreichen Trainingsziel einen
|
||||
// Verlaufseintrag an. Alte History-Zeilen ohne Target bleiben weiterhin lesbar.
|
||||
func trainingHistoryKindForTarget(target string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(target)) {
|
||||
case "pose":
|
||||
return "pose"
|
||||
case "videomae", "video_mae", "scene":
|
||||
return "videomae"
|
||||
default:
|
||||
return "detector"
|
||||
}
|
||||
}
|
||||
|
||||
func trainingAppendTargetHistory(root string, target string, status string, durationMs int64, runtimeOpts trainingRuntimeOptions, fallback TrainingHistoryEntry) {
|
||||
kind := trainingHistoryKindForTarget(target)
|
||||
info := trainingReadModelInfoFor(root, kind)
|
||||
now := time.Now().UTC()
|
||||
|
||||
entry := TrainingHistoryEntry{
|
||||
TrainedAt: info.TrainedAt,
|
||||
TrainedAtMs: info.TrainedAtMs,
|
||||
Epochs: info.Epochs,
|
||||
TrainSamples: info.TrainSamples,
|
||||
ValSamples: info.ValSamples,
|
||||
Imgsz: info.Imgsz,
|
||||
Device: info.Device,
|
||||
MAP50: info.MAP50,
|
||||
MAP5095: info.MAP5095,
|
||||
TrainedAt: now.Format(time.RFC3339),
|
||||
TrainedAtMs: now.UnixMilli(),
|
||||
Target: kind,
|
||||
Status: strings.TrimSpace(status),
|
||||
DurationMs: durationMs,
|
||||
Epochs: fallback.Epochs,
|
||||
TrainSamples: fallback.TrainSamples,
|
||||
ValSamples: fallback.ValSamples,
|
||||
Imgsz: fallback.Imgsz,
|
||||
Device: strings.TrimSpace(fallback.Device),
|
||||
MAP50: fallback.MAP50,
|
||||
MAP5095: fallback.MAP5095,
|
||||
PerformanceMode: runtimeOpts.PerformanceMode,
|
||||
CPUCoreCount: runtimeOpts.CPUCoreCount,
|
||||
CPUThreads: runtimeOpts.CPUThreads,
|
||||
Workers: runtimeOpts.Workers,
|
||||
YoloBatchSize: runtimeOpts.YoloBatchSize,
|
||||
LowPriority: runtimeOpts.LowPriority,
|
||||
}
|
||||
|
||||
// Dauer aus der Startzeit des aktuellen Jobs ableiten.
|
||||
job := trainingGetJobStatus()
|
||||
if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(job.StartedAt)); err == nil {
|
||||
if ms := time.Now().UTC().Sub(startedAt).Milliseconds(); ms > 0 {
|
||||
entry.DurationMs = ms
|
||||
if info != nil {
|
||||
if strings.TrimSpace(info.TrainedAt) != "" {
|
||||
entry.TrainedAt = info.TrainedAt
|
||||
}
|
||||
if info.TrainedAtMs > 0 {
|
||||
entry.TrainedAtMs = info.TrainedAtMs
|
||||
}
|
||||
if info.Epochs > 0 {
|
||||
entry.Epochs = info.Epochs
|
||||
}
|
||||
if info.TrainSamples > 0 {
|
||||
entry.TrainSamples = info.TrainSamples
|
||||
}
|
||||
if info.ValSamples > 0 {
|
||||
entry.ValSamples = info.ValSamples
|
||||
}
|
||||
if info.Imgsz > 0 {
|
||||
entry.Imgsz = info.Imgsz
|
||||
}
|
||||
if strings.TrimSpace(info.Device) != "" {
|
||||
entry.Device = strings.TrimSpace(info.Device)
|
||||
}
|
||||
if info.MAP50 > 0 {
|
||||
entry.MAP50 = info.MAP50
|
||||
}
|
||||
if info.MAP5095 > 0 {
|
||||
entry.MAP5095 = info.MAP5095
|
||||
}
|
||||
}
|
||||
|
||||
if entry.DurationMs <= 0 {
|
||||
job := trainingGetJobStatus()
|
||||
if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(job.StartedAt)); err == nil {
|
||||
if ms := now.Sub(startedAt).Milliseconds(); ms > 0 {
|
||||
entry.DurationMs = ms
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -55,6 +55,21 @@ func trainingVideoMAEBatchSize() int {
|
||||
return n
|
||||
}
|
||||
|
||||
func trainingVideoMAEEarlyStoppingPatience() int {
|
||||
epochs := trainingVideoMAEEpochs()
|
||||
patience := epochs / 3
|
||||
if patience < 2 {
|
||||
patience = 2
|
||||
}
|
||||
if patience > 8 {
|
||||
patience = 8
|
||||
}
|
||||
if patience > epochs {
|
||||
patience = epochs
|
||||
}
|
||||
return patience
|
||||
}
|
||||
|
||||
type trainingVideoMAEManifestEntry struct {
|
||||
SampleID string `json:"sampleId"`
|
||||
Split string `json:"split"`
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user