bugfixes
This commit is contained in:
parent
e4a474ad54
commit
af60ee5319
BIN
backend/dist/nsfwapp-linux-amd64
vendored
BIN
backend/dist/nsfwapp-linux-amd64
vendored
Binary file not shown.
BIN
backend/dist/nsfwapp.exe
vendored
BIN
backend/dist/nsfwapp.exe
vendored
Binary file not shown.
BIN
backend/dist/nsfwapp_amd64.deb
vendored
BIN
backend/dist/nsfwapp_amd64.deb
vendored
Binary file not shown.
@ -385,6 +385,7 @@ def main():
|
|||||||
"ok": True,
|
"ok": True,
|
||||||
"model": str(final_model),
|
"model": str(final_model),
|
||||||
"sourceModel": str(best),
|
"sourceModel": str(best),
|
||||||
|
"baseModel": str(base_used),
|
||||||
"runs": str(result_path),
|
"runs": str(result_path),
|
||||||
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
||||||
"trainSamples": train_count,
|
"trainSamples": train_count,
|
||||||
@ -392,6 +393,7 @@ def main():
|
|||||||
"epochs": epochs,
|
"epochs": epochs,
|
||||||
"imgsz": imgsz,
|
"imgsz": imgsz,
|
||||||
"batchSize": batch_size,
|
"batchSize": batch_size,
|
||||||
|
"patience": patience,
|
||||||
"threads": threads,
|
"threads": threads,
|
||||||
"workers": workers,
|
"workers": workers,
|
||||||
"device": str(train_device),
|
"device": str(train_device),
|
||||||
|
|||||||
@ -371,6 +371,7 @@ def main():
|
|||||||
"ok": True,
|
"ok": True,
|
||||||
"model": str(final_model),
|
"model": str(final_model),
|
||||||
"sourceModel": str(best),
|
"sourceModel": str(best),
|
||||||
|
"baseModel": str(base_model),
|
||||||
"runs": str(result_path),
|
"runs": str(result_path),
|
||||||
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
||||||
"trainSamples": train_count,
|
"trainSamples": train_count,
|
||||||
@ -378,6 +379,7 @@ def main():
|
|||||||
"epochs": epochs,
|
"epochs": epochs,
|
||||||
"imgsz": imgsz,
|
"imgsz": imgsz,
|
||||||
"batchSize": batch_size,
|
"batchSize": batch_size,
|
||||||
|
"patience": patience,
|
||||||
"threads": threads,
|
"threads": threads,
|
||||||
"workers": workers,
|
"workers": workers,
|
||||||
"device": str(train_device),
|
"device": str(train_device),
|
||||||
|
|||||||
@ -200,6 +200,7 @@ def main():
|
|||||||
parser.add_argument("--workers", default="0")
|
parser.add_argument("--workers", default="0")
|
||||||
parser.add_argument("--threads", default="0")
|
parser.add_argument("--threads", default="0")
|
||||||
parser.add_argument("--num-frames", default="16")
|
parser.add_argument("--num-frames", default="16")
|
||||||
|
parser.add_argument("--patience", default="3")
|
||||||
parser.add_argument("--freeze-backbone", action="store_true")
|
parser.add_argument("--freeze-backbone", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
@ -213,6 +214,7 @@ def main():
|
|||||||
workers = max(0, safe_int(args.workers, 0))
|
workers = max(0, safe_int(args.workers, 0))
|
||||||
threads = max(0, safe_int(args.threads, 0))
|
threads = max(0, safe_int(args.threads, 0))
|
||||||
num_frames = max(2, safe_int(args.num_frames, 16))
|
num_frames = max(2, safe_int(args.num_frames, 16))
|
||||||
|
patience = max(0, safe_int(args.patience, 3))
|
||||||
lr = max(1e-7, safe_float(args.lr, 5e-5))
|
lr = max(1e-7, safe_float(args.lr, 5e-5))
|
||||||
|
|
||||||
if threads > 0:
|
if threads > 0:
|
||||||
@ -249,18 +251,23 @@ def main():
|
|||||||
if not val_entries:
|
if not val_entries:
|
||||||
raise SystemExit("no VideoMAE val clips found")
|
raise SystemExit("no VideoMAE val clips found")
|
||||||
|
|
||||||
|
base_model = str(args.base or DEFAULT_BASE_MODEL).strip() or DEFAULT_BASE_MODEL
|
||||||
|
base_path = Path(base_model).expanduser()
|
||||||
|
if base_path.exists():
|
||||||
|
base_model = str(base_path.resolve())
|
||||||
|
|
||||||
emit_progress(
|
emit_progress(
|
||||||
"videomae",
|
"videomae",
|
||||||
0.03,
|
0.03,
|
||||||
"VideoMAE-Basismodell wird geladen...",
|
"VideoMAE-Basismodell wird geladen...",
|
||||||
base=args.base,
|
base=base_model,
|
||||||
labels=len(labels),
|
labels=len(labels),
|
||||||
device=str(device),
|
device=str(device),
|
||||||
)
|
)
|
||||||
|
|
||||||
image_processor = AutoImageProcessor.from_pretrained(args.base)
|
image_processor = AutoImageProcessor.from_pretrained(base_model)
|
||||||
model = VideoMAEForVideoClassification.from_pretrained(
|
model = VideoMAEForVideoClassification.from_pretrained(
|
||||||
args.base,
|
base_model,
|
||||||
num_labels=len(labels),
|
num_labels=len(labels),
|
||||||
label2id=label_to_id,
|
label2id=label_to_id,
|
||||||
id2label=id_to_label,
|
id2label=id_to_label,
|
||||||
@ -302,8 +309,11 @@ def main():
|
|||||||
best_accuracy = -1.0
|
best_accuracy = -1.0
|
||||||
best_loss = math.inf
|
best_loss = math.inf
|
||||||
best_epoch = 0
|
best_epoch = 0
|
||||||
|
completed_epochs = 0
|
||||||
|
epochs_without_improvement = 0
|
||||||
|
|
||||||
for epoch in range(1, epochs + 1):
|
for epoch in range(1, epochs + 1):
|
||||||
|
completed_epochs = epoch
|
||||||
model.train()
|
model.train()
|
||||||
running_loss = 0.0
|
running_loss = 0.0
|
||||||
seen = 0
|
seen = 0
|
||||||
@ -346,8 +356,11 @@ def main():
|
|||||||
best_accuracy = val_accuracy
|
best_accuracy = val_accuracy
|
||||||
best_loss = val_loss
|
best_loss = val_loss
|
||||||
best_epoch = epoch
|
best_epoch = epoch
|
||||||
|
epochs_without_improvement = 0
|
||||||
model.save_pretrained(tmp_dir)
|
model.save_pretrained(tmp_dir)
|
||||||
image_processor.save_pretrained(tmp_dir)
|
image_processor.save_pretrained(tmp_dir)
|
||||||
|
else:
|
||||||
|
epochs_without_improvement += 1
|
||||||
|
|
||||||
emit_progress(
|
emit_progress(
|
||||||
"videomae",
|
"videomae",
|
||||||
@ -360,16 +373,36 @@ def main():
|
|||||||
device=str(device),
|
device=str(device),
|
||||||
accuracy=val_accuracy,
|
accuracy=val_accuracy,
|
||||||
loss=val_loss,
|
loss=val_loss,
|
||||||
|
patience=patience,
|
||||||
|
epochsWithoutImprovement=epochs_without_improvement,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if patience > 0 and epochs_without_improvement >= patience:
|
||||||
|
emit_progress(
|
||||||
|
"videomae",
|
||||||
|
0.96,
|
||||||
|
"VideoMAE Early-Stopping: keine weitere Verbesserung.",
|
||||||
|
epoch=epoch,
|
||||||
|
epochs=epochs,
|
||||||
|
bestEpoch=best_epoch,
|
||||||
|
patience=patience,
|
||||||
|
trainSamples=len(train_entries),
|
||||||
|
valSamples=len(val_entries),
|
||||||
|
device=str(device),
|
||||||
|
accuracy=best_accuracy if best_accuracy >= 0 else 0.0,
|
||||||
|
loss=best_loss if math.isfinite(best_loss) else 0.0,
|
||||||
|
)
|
||||||
|
break
|
||||||
|
|
||||||
if best_epoch <= 0:
|
if best_epoch <= 0:
|
||||||
model.save_pretrained(tmp_dir)
|
model.save_pretrained(tmp_dir)
|
||||||
image_processor.save_pretrained(tmp_dir)
|
image_processor.save_pretrained(tmp_dir)
|
||||||
best_epoch = epochs
|
best_epoch = completed_epochs or epochs
|
||||||
|
|
||||||
status = {
|
status = {
|
||||||
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
"trainedAt": datetime.now(timezone.utc).isoformat(),
|
||||||
"epochs": epochs,
|
"epochs": epochs,
|
||||||
|
"completedEpochs": completed_epochs,
|
||||||
"bestEpoch": best_epoch,
|
"bestEpoch": best_epoch,
|
||||||
"trainSamples": len(train_entries),
|
"trainSamples": len(train_entries),
|
||||||
"valSamples": len(val_entries),
|
"valSamples": len(val_entries),
|
||||||
@ -377,7 +410,8 @@ def main():
|
|||||||
"workers": workers,
|
"workers": workers,
|
||||||
"threads": threads,
|
"threads": threads,
|
||||||
"numFrames": num_frames,
|
"numFrames": num_frames,
|
||||||
"baseModel": args.base,
|
"patience": patience,
|
||||||
|
"baseModel": base_model,
|
||||||
"device": str(device),
|
"device": str(device),
|
||||||
"accuracy": best_accuracy if best_accuracy >= 0 else 0.0,
|
"accuracy": best_accuracy if best_accuracy >= 0 else 0.0,
|
||||||
"loss": best_loss if math.isfinite(best_loss) else 0.0,
|
"loss": best_loss if math.isfinite(best_loss) else 0.0,
|
||||||
|
|||||||
@ -118,6 +118,103 @@ type TrainingSkipRequest struct {
|
|||||||
SampleID string `json:"sampleId"`
|
SampleID string `json:"sampleId"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TrainingTrainRequest struct {
|
||||||
|
Scope string `json:"scope,omitempty"`
|
||||||
|
Targets []string `json:"targets,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type trainingTrainTargets struct {
|
||||||
|
Detector bool
|
||||||
|
Pose bool
|
||||||
|
VideoMAE bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingAllTrainTargets() trainingTrainTargets {
|
||||||
|
return trainingTrainTargets{
|
||||||
|
Detector: true,
|
||||||
|
Pose: true,
|
||||||
|
VideoMAE: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t trainingTrainTargets) empty() bool {
|
||||||
|
return !t.Detector && !t.Pose && !t.VideoMAE
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t trainingTrainTargets) list() []string {
|
||||||
|
out := make([]string, 0, 3)
|
||||||
|
if t.Detector {
|
||||||
|
out = append(out, "detector")
|
||||||
|
}
|
||||||
|
if t.Pose {
|
||||||
|
out = append(out, "pose")
|
||||||
|
}
|
||||||
|
if t.VideoMAE {
|
||||||
|
out = append(out, "videomae")
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingReadTrainRequest(r *http.Request) (TrainingTrainRequest, error) {
|
||||||
|
var req TrainingTrainRequest
|
||||||
|
if r.Body == nil {
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||||||
|
if err != nil {
|
||||||
|
return req, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.TrimSpace(string(body)) == "" {
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.Unmarshal(body, &req); err != nil {
|
||||||
|
return req, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingNormalizeTrainTargets(req TrainingTrainRequest) (trainingTrainTargets, bool, error) {
|
||||||
|
scope := strings.ToLower(strings.TrimSpace(req.Scope))
|
||||||
|
if scope == "" || scope == "full" || scope == "all" || scope == "complete" {
|
||||||
|
return trainingAllTrainTargets(), false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if scope != "custom" && scope != "selected" && scope != "partial" {
|
||||||
|
return trainingTrainTargets{}, false, fmt.Errorf("unbekannter Trainingsumfang: %s", req.Scope)
|
||||||
|
}
|
||||||
|
|
||||||
|
var targets trainingTrainTargets
|
||||||
|
for _, raw := range req.Targets {
|
||||||
|
key := strings.ToLower(strings.TrimSpace(raw))
|
||||||
|
key = strings.ReplaceAll(key, "_", "")
|
||||||
|
key = strings.ReplaceAll(key, "-", "")
|
||||||
|
key = strings.ReplaceAll(key, " ", "")
|
||||||
|
|
||||||
|
switch key {
|
||||||
|
case "detector", "yolo", "yolo26", "yolo26detector", "box", "boxes", "boxdetection":
|
||||||
|
targets.Detector = true
|
||||||
|
case "pose", "yolo26pose", "posedetection":
|
||||||
|
targets.Pose = true
|
||||||
|
case "videomae", "video", "clip", "scene", "clipanalysis", "clipanalyse":
|
||||||
|
targets.VideoMAE = true
|
||||||
|
case "":
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
return trainingTrainTargets{}, true, fmt.Errorf("unbekanntes Training: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if targets.empty() {
|
||||||
|
return trainingTrainTargets{}, true, errors.New("kein Training ausgewählt")
|
||||||
|
}
|
||||||
|
|
||||||
|
return targets, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
type TrainingAnnotation struct {
|
type TrainingAnnotation struct {
|
||||||
SampleID string `json:"sampleId"`
|
SampleID string `json:"sampleId"`
|
||||||
FrameURL string `json:"frameUrl"`
|
FrameURL string `json:"frameUrl"`
|
||||||
@ -1313,6 +1410,21 @@ func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRunt
|
|||||||
return opts
|
return opts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func trainingYoloEarlyStoppingPatience(epochs int) int {
|
||||||
|
epochs = clampTrainingInt(epochs, 1, 300)
|
||||||
|
patience := epochs / 4
|
||||||
|
if patience < 5 {
|
||||||
|
patience = 5
|
||||||
|
}
|
||||||
|
if patience > 20 {
|
||||||
|
patience = 20
|
||||||
|
}
|
||||||
|
if patience > epochs {
|
||||||
|
patience = epochs
|
||||||
|
}
|
||||||
|
return patience
|
||||||
|
}
|
||||||
|
|
||||||
func trainingCommandEnv(opts trainingRuntimeOptions) []string {
|
func trainingCommandEnv(opts trainingRuntimeOptions) []string {
|
||||||
env := os.Environ()
|
env := os.Environ()
|
||||||
if opts.CPUThreads <= 0 {
|
if opts.CPUThreads <= 0 {
|
||||||
@ -3212,6 +3324,18 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
req, err := trainingReadTrainRequest(r)
|
||||||
|
if err != nil {
|
||||||
|
trainingWriteError(w, http.StatusBadRequest, "invalid json")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
targets, customTargets, err := trainingNormalizeTrainTargets(req)
|
||||||
|
if err != nil {
|
||||||
|
trainingWriteError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
current := trainingGetJobStatus()
|
current := trainingGetJobStatus()
|
||||||
if current.Running {
|
if current.Running {
|
||||||
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
||||||
@ -3301,7 +3425,70 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
videoMAEDataReady := videoMAEEligibleCount >= minVideoMAETrainCount ||
|
videoMAEDataReady := videoMAEEligibleCount >= minVideoMAETrainCount ||
|
||||||
(videoMAETrainCount >= minVideoMAETrainCount && videoMAEValCount >= minVideoMAEValCount)
|
(videoMAETrainCount >= minVideoMAETrainCount && videoMAEValCount >= minVideoMAEValCount)
|
||||||
|
|
||||||
if detectorDataReady || poseDataReady || videoMAEDataReady {
|
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
||||||
|
selectedDataReady :=
|
||||||
|
(targets.Detector && detectorDataReady) ||
|
||||||
|
(targets.Pose && poseDataReady) ||
|
||||||
|
(targets.VideoMAE && videoMAEDataReady)
|
||||||
|
|
||||||
|
if customTargets {
|
||||||
|
if targets.Detector && !detectorDataReady {
|
||||||
|
trainingWriteError(
|
||||||
|
w,
|
||||||
|
http.StatusBadRequest,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"YOLO26 Detector ist noch nicht trainingsbereit. Train=%d (%d positiv), Val=%d (%d positiv). Benoetigt: mindestens %d Train, %d Val und je ein positives Beispiel.",
|
||||||
|
trainCount,
|
||||||
|
positiveTrainCount,
|
||||||
|
valCount,
|
||||||
|
positiveValCount,
|
||||||
|
minDetectorTrainCount,
|
||||||
|
minDetectorValCount,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if targets.Pose && !poseDataReady {
|
||||||
|
trainingWriteError(
|
||||||
|
w,
|
||||||
|
http.StatusBadRequest,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"YOLO26 Pose ist noch nicht trainingsbereit. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
||||||
|
poseTrainCount,
|
||||||
|
poseValCount,
|
||||||
|
minPoseTrainCount,
|
||||||
|
minPoseValCount,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if targets.VideoMAE && !runtimeOpts.VideoMAEEnabled {
|
||||||
|
trainingWriteError(w, http.StatusBadRequest, "VideoMAE ist in den Training-Settings deaktiviert.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if targets.VideoMAE && !videoMAEDataReady {
|
||||||
|
trainingWriteError(
|
||||||
|
w,
|
||||||
|
http.StatusBadRequest,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"VideoMAE ist noch nicht trainingsbereit. Eligible=%d, Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
||||||
|
videoMAEEligibleCount,
|
||||||
|
videoMAETrainCount,
|
||||||
|
videoMAEValCount,
|
||||||
|
minVideoMAETrainCount,
|
||||||
|
minVideoMAEValCount,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
goto startTraining
|
||||||
|
}
|
||||||
|
|
||||||
|
if selectedDataReady {
|
||||||
goto startTraining
|
goto startTraining
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3348,12 +3535,13 @@ startTraining:
|
|||||||
|
|
||||||
trainingStartJob(cancel)
|
trainingStartJob(cancel)
|
||||||
|
|
||||||
go trainingRunJob(ctx, root, feedbackCount)
|
go trainingRunJob(ctx, root, feedbackCount, targets)
|
||||||
|
|
||||||
trainingWriteJSON(w, http.StatusAccepted, map[string]any{
|
trainingWriteJSON(w, http.StatusAccepted, map[string]any{
|
||||||
"ok": true,
|
"ok": true,
|
||||||
"message": "Training gestartet.",
|
"message": "Training gestartet.",
|
||||||
"training": trainingGetJobStatus(),
|
"training": trainingGetJobStatus(),
|
||||||
|
"targets": targets.list(),
|
||||||
"detector": map[string]any{
|
"detector": map[string]any{
|
||||||
"trainCount": trainCount,
|
"trainCount": trainCount,
|
||||||
"valCount": valCount,
|
"valCount": valCount,
|
||||||
@ -3382,6 +3570,7 @@ startTraining:
|
|||||||
"requiredTrain": minVideoMAETrainCount,
|
"requiredTrain": minVideoMAETrainCount,
|
||||||
"requiredVal": minVideoMAEValCount,
|
"requiredVal": minVideoMAEValCount,
|
||||||
"manifest": videoMAEManifest,
|
"manifest": videoMAEManifest,
|
||||||
|
"dataReady": videoMAEDataReady,
|
||||||
"source": "videomae_clip",
|
"source": "videomae_clip",
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@ -3424,7 +3613,7 @@ func trainingCancelHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func trainingRunJob(ctx context.Context, root string, count int) {
|
func trainingRunJob(ctx context.Context, root string, count int, targets trainingTrainTargets) {
|
||||||
if err := ensureMLPythonSetup(ctx); err != nil {
|
if err := ensureMLPythonSetup(ctx); err != nil {
|
||||||
if errors.Is(err, context.Canceled) {
|
if errors.Is(err, context.Canceled) {
|
||||||
trainingFinishCancelled(root)
|
trainingFinishCancelled(root)
|
||||||
@ -3459,7 +3648,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
appLogln("ML-Python für Training:", python)
|
appLogln("ML-Python für Training:", python)
|
||||||
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
||||||
appLogf(
|
appLogf(
|
||||||
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v",
|
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v targets=%s",
|
||||||
runtimeOpts.PerformanceMode,
|
runtimeOpts.PerformanceMode,
|
||||||
runtimeOpts.CPUCoreCount,
|
runtimeOpts.CPUCoreCount,
|
||||||
runtimeOpts.PowerSaveMode,
|
runtimeOpts.PowerSaveMode,
|
||||||
@ -3468,6 +3657,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
runtimeOpts.YoloBatchSize,
|
runtimeOpts.YoloBatchSize,
|
||||||
runtimeOpts.LowPriority,
|
runtimeOpts.LowPriority,
|
||||||
runtimeOpts.VideoMAEEnabled,
|
runtimeOpts.VideoMAEEnabled,
|
||||||
|
strings.Join(targets.list(), ","),
|
||||||
)
|
)
|
||||||
|
|
||||||
cleanOutput := func(text string) string {
|
cleanOutput := func(text string) string {
|
||||||
@ -3485,10 +3675,13 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
|
|
||||||
detectorOutput := ""
|
detectorOutput := ""
|
||||||
detectorStatus := "skipped"
|
detectorStatus := "skipped"
|
||||||
|
detectorDurationMs := int64(0)
|
||||||
poseOutput := ""
|
poseOutput := ""
|
||||||
poseStatus := "skipped"
|
poseStatus := "skipped"
|
||||||
|
poseDurationMs := int64(0)
|
||||||
videoMAEOutput := ""
|
videoMAEOutput := ""
|
||||||
videoMAEStatus := "skipped"
|
videoMAEStatus := "skipped"
|
||||||
|
videoMAEDurationMs := int64(0)
|
||||||
|
|
||||||
detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml")
|
detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml")
|
||||||
detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train")
|
detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train")
|
||||||
@ -3514,29 +3707,50 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
fileExistsNonEmpty(detectorDatasetYAML),
|
fileExistsNonEmpty(detectorDatasetYAML),
|
||||||
)
|
)
|
||||||
|
|
||||||
if fileExistsNonEmpty(detectorDatasetYAML) &&
|
if !targets.Detector {
|
||||||
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
|
if s.Progress < 58 {
|
||||||
|
s.Progress = 58
|
||||||
|
}
|
||||||
|
s.Step = "YOLO26 Detector wurde nicht ausgewählt."
|
||||||
|
})
|
||||||
|
detectorStatus = "skipped_unselected"
|
||||||
|
detectorOutput = "YOLO26 Detector übersprungen: nicht ausgewählt."
|
||||||
|
appLogln(detectorOutput)
|
||||||
|
} else if fileExistsNonEmpty(detectorDatasetYAML) &&
|
||||||
trainCount >= minDetectorTrainCount &&
|
trainCount >= minDetectorTrainCount &&
|
||||||
valCount >= minDetectorValCount &&
|
valCount >= minDetectorValCount &&
|
||||||
positiveTrainCount > 0 &&
|
positiveTrainCount > 0 &&
|
||||||
positiveValCount > 0 {
|
positiveValCount > 0 {
|
||||||
|
detectorStartedAt := time.Now()
|
||||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
s.Progress = 15
|
s.Progress = 15
|
||||||
s.Step = "YOLO26 Detector wird trainiert…"
|
s.Step = "YOLO26 Detector wird trainiert…"
|
||||||
})
|
})
|
||||||
|
|
||||||
detectorBasePath := "yolo26n.pt"
|
detectorModel := trainingResolveDetectorModel(root)
|
||||||
if p, err := trainingMLCacheFilePath("yolo26n.pt"); err == nil {
|
detectorBasePath := detectorModel.BestPath
|
||||||
detectorBasePath = p
|
if !detectorModel.TrainedExists {
|
||||||
|
detectorBasePath = "yolo26n.pt"
|
||||||
|
if p, err := trainingMLCacheFilePath("yolo26n.pt"); err == nil {
|
||||||
|
detectorBasePath = p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
detectorEpochs := trainingDetectorEpochs()
|
||||||
|
detectorPatience := trainingYoloEarlyStoppingPatience(detectorEpochs)
|
||||||
|
if detectorModel.TrainedExists {
|
||||||
|
appLogln("YOLO26 Detector Fine-Tuning startet von:", detectorBasePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
detectorScript := trainingScriptPath("train_detector_model.py")
|
detectorScript := trainingScriptPath("train_detector_model.py")
|
||||||
detectorArgs := []string{
|
detectorArgs := []string{
|
||||||
"--root", root,
|
"--root", root,
|
||||||
"--base", detectorBasePath,
|
"--base", detectorBasePath,
|
||||||
"--epochs", strconv.Itoa(trainingDetectorEpochs()),
|
"--epochs", strconv.Itoa(detectorEpochs),
|
||||||
"--imgsz", "640",
|
"--imgsz", "640",
|
||||||
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
||||||
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
||||||
|
"--patience", strconv.Itoa(detectorPatience),
|
||||||
}
|
}
|
||||||
if runtimeOpts.YoloBatchSize > 0 {
|
if runtimeOpts.YoloBatchSize > 0 {
|
||||||
detectorArgs = append(detectorArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
detectorArgs = append(detectorArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
||||||
@ -3556,6 +3770,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
},
|
},
|
||||||
detectorArgs...,
|
detectorArgs...,
|
||||||
)
|
)
|
||||||
|
detectorDurationMs = time.Since(detectorStartedAt).Milliseconds()
|
||||||
|
|
||||||
if errors.Is(detectorErr, errTrainingCancelled) {
|
if errors.Is(detectorErr, errTrainingCancelled) {
|
||||||
appLogln("⛔ YOLO26 detector training cancelled")
|
appLogln("⛔ YOLO26 detector training cancelled")
|
||||||
@ -3601,24 +3816,38 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
poseTrainLabels := filepath.Join(root, "pose", "dataset", "labels", "train")
|
poseTrainLabels := filepath.Join(root, "pose", "dataset", "labels", "train")
|
||||||
poseValImages := filepath.Join(root, "pose", "dataset", "images", "val")
|
poseValImages := filepath.Join(root, "pose", "dataset", "images", "val")
|
||||||
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
|
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
|
||||||
|
poseStartedAt := time.Time{}
|
||||||
|
|
||||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
if !targets.Pose {
|
||||||
if s.Progress < 60 {
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
s.Progress = 60
|
if s.Progress < 82 {
|
||||||
}
|
s.Progress = 82
|
||||||
s.Step = "YOLO26 Pose-Daten werden aufgebaut..."
|
}
|
||||||
})
|
s.Step = "YOLO26 Pose wurde nicht ausgewählt."
|
||||||
|
})
|
||||||
if written, err := trainingSyncPoseDataset(root); err != nil {
|
poseStatus = "skipped_unselected"
|
||||||
poseStatus = "failed"
|
poseOutput = "YOLO26 Pose übersprungen: nicht ausgewählt."
|
||||||
poseOutput = "YOLO26 Pose-Dataset konnte nicht aufgebaut werden: " + err.Error()
|
|
||||||
appLogln(poseOutput)
|
appLogln(poseOutput)
|
||||||
} else {
|
} else {
|
||||||
appLogln("pose samples synced:", written)
|
poseStartedAt = time.Now()
|
||||||
}
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
|
if s.Progress < 60 {
|
||||||
|
s.Progress = 60
|
||||||
|
}
|
||||||
|
s.Step = "YOLO26 Pose-Daten werden aufgebaut..."
|
||||||
|
})
|
||||||
|
|
||||||
if err := trainingEnsurePoseValidationSample(root); err != nil {
|
if written, err := trainingSyncPoseDataset(root); err != nil {
|
||||||
appLogln("pose val sample ensure failed:", err)
|
poseStatus = "failed"
|
||||||
|
poseOutput = "YOLO26 Pose-Dataset konnte nicht aufgebaut werden: " + err.Error()
|
||||||
|
appLogln(poseOutput)
|
||||||
|
} else {
|
||||||
|
appLogln("pose samples synced:", written)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := trainingEnsurePoseValidationSample(root); err != nil {
|
||||||
|
appLogln("pose val sample ensure failed:", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
|
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
|
||||||
@ -3631,7 +3860,8 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
fileExistsNonEmpty(poseDatasetYAML),
|
fileExistsNonEmpty(poseDatasetYAML),
|
||||||
)
|
)
|
||||||
|
|
||||||
if poseStatus != "failed" &&
|
if targets.Pose &&
|
||||||
|
poseStatus != "failed" &&
|
||||||
fileExistsNonEmpty(poseDatasetYAML) &&
|
fileExistsNonEmpty(poseDatasetYAML) &&
|
||||||
poseTrainCount >= minPoseTrainCount &&
|
poseTrainCount >= minPoseTrainCount &&
|
||||||
poseValCount >= minPoseValCount {
|
poseValCount >= minPoseValCount {
|
||||||
@ -3642,19 +3872,26 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
s.Step = "YOLO26 Pose wird trainiert..."
|
s.Step = "YOLO26 Pose wird trainiert..."
|
||||||
})
|
})
|
||||||
|
|
||||||
|
poseModel := trainingResolvePoseModel(root)
|
||||||
poseBasePath := "yolo26n-pose.pt"
|
poseBasePath := "yolo26n-pose.pt"
|
||||||
if p, err := embeddedPoseModelPath(); err == nil && fileExistsNonEmpty(p) {
|
if poseModel.EffectiveExists {
|
||||||
poseBasePath = p
|
poseBasePath = poseModel.EffectivePath
|
||||||
|
}
|
||||||
|
poseEpochs := trainingDetectorEpochs()
|
||||||
|
posePatience := trainingYoloEarlyStoppingPatience(poseEpochs)
|
||||||
|
if poseModel.TrainedExists {
|
||||||
|
appLogln("YOLO26 Pose Fine-Tuning startet von:", poseBasePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
poseScript := trainingScriptPath("train_pose_model.py")
|
poseScript := trainingScriptPath("train_pose_model.py")
|
||||||
poseArgs := []string{
|
poseArgs := []string{
|
||||||
"--root", root,
|
"--root", root,
|
||||||
"--base", poseBasePath,
|
"--base", poseBasePath,
|
||||||
"--epochs", strconv.Itoa(trainingDetectorEpochs()),
|
"--epochs", strconv.Itoa(poseEpochs),
|
||||||
"--imgsz", "640",
|
"--imgsz", "640",
|
||||||
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
||||||
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
||||||
|
"--patience", strconv.Itoa(posePatience),
|
||||||
}
|
}
|
||||||
if runtimeOpts.YoloBatchSize > 0 {
|
if runtimeOpts.YoloBatchSize > 0 {
|
||||||
poseArgs = append(poseArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
poseArgs = append(poseArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
||||||
@ -3674,6 +3911,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
},
|
},
|
||||||
poseArgs...,
|
poseArgs...,
|
||||||
)
|
)
|
||||||
|
poseDurationMs = time.Since(poseStartedAt).Milliseconds()
|
||||||
|
|
||||||
if errors.Is(poseErr, errTrainingCancelled) {
|
if errors.Is(poseErr, errTrainingCancelled) {
|
||||||
appLogln("YOLO26 pose training cancelled")
|
appLogln("YOLO26 pose training cancelled")
|
||||||
@ -3698,7 +3936,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
appLogln("YOLO26 pose training:", poseOutputClean)
|
appLogln("YOLO26 pose training:", poseOutputClean)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if poseStatus != "failed" {
|
} else if targets.Pose && poseStatus != "failed" {
|
||||||
poseStatus = "skipped_no_pose_data"
|
poseStatus = "skipped_no_pose_data"
|
||||||
poseOutput = fmt.Sprintf(
|
poseOutput = fmt.Sprintf(
|
||||||
"YOLO26 Pose übersprungen: zu wenige Skeleton-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
"YOLO26 Pose übersprungen: zu wenige Skeleton-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
||||||
@ -3710,9 +3948,26 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
appLogln(poseOutput)
|
appLogln(poseOutput)
|
||||||
}
|
}
|
||||||
|
|
||||||
poseOutputClean := cleanOutput(poseOutput)
|
if poseStatus == "trained" && poseDurationMs <= 0 && !poseStartedAt.IsZero() {
|
||||||
|
poseDurationMs = time.Since(poseStartedAt).Milliseconds()
|
||||||
|
}
|
||||||
|
|
||||||
if !runtimeOpts.VideoMAEEnabled {
|
poseOutputClean := cleanOutput(poseOutput)
|
||||||
|
videoMAEStartedAt := time.Time{}
|
||||||
|
videoMAETrainCount := 0
|
||||||
|
videoMAEValCount := 0
|
||||||
|
|
||||||
|
if !targets.VideoMAE {
|
||||||
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
|
if s.Progress < 98 {
|
||||||
|
s.Progress = 98
|
||||||
|
}
|
||||||
|
s.Step = "VideoMAE wurde nicht ausgewählt."
|
||||||
|
})
|
||||||
|
videoMAEStatus = "skipped_unselected"
|
||||||
|
videoMAEOutput = "VideoMAE übersprungen: nicht ausgewählt."
|
||||||
|
appLogln(videoMAEOutput)
|
||||||
|
} else if !runtimeOpts.VideoMAEEnabled {
|
||||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
if s.Progress < 84 {
|
if s.Progress < 84 {
|
||||||
s.Progress = 84
|
s.Progress = 84
|
||||||
@ -3723,6 +3978,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
videoMAEOutput = "VideoMAE übersprungen: in den Training-Settings deaktiviert."
|
videoMAEOutput = "VideoMAE übersprungen: in den Training-Settings deaktiviert."
|
||||||
appLogln(videoMAEOutput)
|
appLogln(videoMAEOutput)
|
||||||
} else {
|
} else {
|
||||||
|
videoMAEStartedAt = time.Now()
|
||||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
if s.Progress < 84 {
|
if s.Progress < 84 {
|
||||||
s.Progress = 84
|
s.Progress = 84
|
||||||
@ -3730,7 +3986,9 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
s.Step = "VideoMAE Clip-Daten werden aufgebaut..."
|
s.Step = "VideoMAE Clip-Daten werden aufgebaut..."
|
||||||
})
|
})
|
||||||
|
|
||||||
videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr :=
|
var videoMAEWritten int
|
||||||
|
var videoMAESyncErr error
|
||||||
|
videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr =
|
||||||
trainingSyncVideoMAEDataset(ctx, root)
|
trainingSyncVideoMAEDataset(ctx, root)
|
||||||
if errors.Is(videoMAESyncErr, context.Canceled) || errors.Is(videoMAESyncErr, errTrainingCancelled) {
|
if errors.Is(videoMAESyncErr, context.Canceled) || errors.Is(videoMAESyncErr, errTrainingCancelled) {
|
||||||
appLogln("VideoMAE dataset sync cancelled")
|
appLogln("VideoMAE dataset sync cancelled")
|
||||||
@ -3761,6 +4019,13 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
videoMAEScript := trainingScriptPath("train_videomae_model.py")
|
videoMAEScript := trainingScriptPath("train_videomae_model.py")
|
||||||
|
videoMAEBase := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL"))
|
||||||
|
if videoMAEBase == "" {
|
||||||
|
if model := trainingResolveVideoMAEModel(root); model.TrainedExists {
|
||||||
|
videoMAEBase = model.EffectivePath
|
||||||
|
appLogln("VideoMAE Fine-Tuning startet von:", videoMAEBase)
|
||||||
|
}
|
||||||
|
}
|
||||||
videoMAEArgs := []string{
|
videoMAEArgs := []string{
|
||||||
"--root", root,
|
"--root", root,
|
||||||
"--epochs", strconv.Itoa(trainingVideoMAEEpochs()),
|
"--epochs", strconv.Itoa(trainingVideoMAEEpochs()),
|
||||||
@ -3768,9 +4033,10 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
"--num-frames", strconv.Itoa(trainingVideoMAENumFrames),
|
"--num-frames", strconv.Itoa(trainingVideoMAENumFrames),
|
||||||
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
||||||
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
||||||
|
"--patience", strconv.Itoa(trainingVideoMAEEarlyStoppingPatience()),
|
||||||
}
|
}
|
||||||
if base := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL")); base != "" {
|
if videoMAEBase != "" {
|
||||||
videoMAEArgs = append(videoMAEArgs, "--base", base)
|
videoMAEArgs = append(videoMAEArgs, "--base", videoMAEBase)
|
||||||
}
|
}
|
||||||
|
|
||||||
videoMAEOut, videoMAEErr := trainingRunCommandStreaming(
|
videoMAEOut, videoMAEErr := trainingRunCommandStreaming(
|
||||||
@ -3787,6 +4053,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
},
|
},
|
||||||
videoMAEArgs...,
|
videoMAEArgs...,
|
||||||
)
|
)
|
||||||
|
videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds()
|
||||||
|
|
||||||
if errors.Is(videoMAEErr, errTrainingCancelled) {
|
if errors.Is(videoMAEErr, errTrainingCancelled) {
|
||||||
appLogln("VideoMAE training cancelled")
|
appLogln("VideoMAE training cancelled")
|
||||||
@ -3822,6 +4089,10 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if videoMAEStatus == "trained" && videoMAEDurationMs <= 0 && !videoMAEStartedAt.IsZero() {
|
||||||
|
videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds()
|
||||||
|
}
|
||||||
|
|
||||||
videoMAEOutputClean := cleanOutput(videoMAEOutput)
|
videoMAEOutputClean := cleanOutput(videoMAEOutput)
|
||||||
|
|
||||||
message := "Training abgeschlossen."
|
message := "Training abgeschlossen."
|
||||||
@ -3831,6 +4102,9 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
case "trained":
|
case "trained":
|
||||||
message = "Training abgeschlossen. YOLO26 Detector wurde trainiert."
|
message = "Training abgeschlossen. YOLO26 Detector wurde trainiert."
|
||||||
|
|
||||||
|
case "skipped_unselected":
|
||||||
|
message = "Training abgeschlossen."
|
||||||
|
|
||||||
case "skipped_no_detector_data":
|
case "skipped_no_detector_data":
|
||||||
message = detectorOutput
|
message = detectorOutput
|
||||||
|
|
||||||
@ -3887,8 +4161,31 @@ func trainingRunJob(ctx context.Context, root string, count int) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if detectorStatus == "trained" || poseStatus == "trained" || videoMAEStatus == "trained" {
|
if detectorStatus == "trained" || poseStatus == "trained" || videoMAEStatus == "trained" {
|
||||||
// Verlaufseintrag schreiben, solange die Job-Startzeit für die Dauer noch verfügbar ist.
|
// Verlauf pro erfolgreich trainiertem Ziel schreiben.
|
||||||
trainingAppendRunHistory(root)
|
if detectorStatus == "trained" {
|
||||||
|
trainingAppendTargetHistory(root, "detector", detectorStatus, detectorDurationMs, runtimeOpts, TrainingHistoryEntry{
|
||||||
|
Epochs: trainingDetectorEpochs(),
|
||||||
|
TrainSamples: trainCount,
|
||||||
|
ValSamples: valCount,
|
||||||
|
Imgsz: 640,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if poseStatus == "trained" {
|
||||||
|
trainingAppendTargetHistory(root, "pose", poseStatus, poseDurationMs, runtimeOpts, TrainingHistoryEntry{
|
||||||
|
Epochs: trainingDetectorEpochs(),
|
||||||
|
TrainSamples: poseTrainCount,
|
||||||
|
ValSamples: poseValCount,
|
||||||
|
Imgsz: 640,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if videoMAEStatus == "trained" {
|
||||||
|
trainingAppendTargetHistory(root, "videomae", videoMAEStatus, videoMAEDurationMs, runtimeOpts, TrainingHistoryEntry{
|
||||||
|
Epochs: trainingVideoMAEEpochs(),
|
||||||
|
TrainSamples: videoMAETrainCount,
|
||||||
|
ValSamples: videoMAEValCount,
|
||||||
|
Imgsz: trainingVideoMAEFrameSize,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
||||||
@ -4338,20 +4635,22 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
"predictsClothing": false,
|
"predictsClothing": false,
|
||||||
"predictsBoxes": false,
|
"predictsBoxes": false,
|
||||||
|
|
||||||
"feedbackCount": feedbackCount,
|
"feedbackCount": feedbackCount,
|
||||||
"eligibleCount": videoMAEEligibleCount,
|
"eligibleCount": videoMAEEligibleCount,
|
||||||
"trainCount": videoMAETrainCount,
|
"trainCount": videoMAETrainCount,
|
||||||
"valCount": videoMAEValCount,
|
"valCount": videoMAEValCount,
|
||||||
"requiredTrain": minVideoMAETrainCount,
|
"requiredTrain": minVideoMAETrainCount,
|
||||||
"requiredVal": minVideoMAEValCount,
|
"requiredVal": minVideoMAEValCount,
|
||||||
"requiredCount": minVideoMAETrainCount,
|
"requiredCount": minVideoMAETrainCount,
|
||||||
"datasetReady": videoMAEDatasetReady,
|
"datasetReady": videoMAEDatasetReady,
|
||||||
"manifest": videoMAEManifest,
|
"manifest": videoMAEManifest,
|
||||||
"dataReady": videoMAEDataReady,
|
"dataReady": videoMAEDataReady,
|
||||||
"modelReady": videoMAEModel.EffectiveExists,
|
"modelReady": videoMAEModel.EffectiveExists,
|
||||||
"modelExists": videoMAEModel.EffectiveExists,
|
"modelExists": videoMAEModel.EffectiveExists,
|
||||||
"modelPath": videoMAEModel.EffectivePath,
|
"modelPath": videoMAEModel.EffectivePath,
|
||||||
"modelSource": videoMAEModel.Source,
|
"modelSource": videoMAEModel.Source,
|
||||||
|
"trainedModelExists": videoMAEModel.TrainedExists,
|
||||||
|
"trainedModelPath": videoMAEModel.BestPath,
|
||||||
},
|
},
|
||||||
|
|
||||||
"pipeline": map[string]any{
|
"pipeline": map[string]any{
|
||||||
@ -4464,16 +4763,24 @@ func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TrainingHistoryEntry struct {
|
type TrainingHistoryEntry struct {
|
||||||
TrainedAt string `json:"trainedAt,omitempty"`
|
TrainedAt string `json:"trainedAt,omitempty"`
|
||||||
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
||||||
DurationMs int64 `json:"durationMs,omitempty"`
|
Target string `json:"target,omitempty"`
|
||||||
Epochs int `json:"epochs,omitempty"`
|
Status string `json:"status,omitempty"`
|
||||||
TrainSamples int `json:"trainSamples,omitempty"`
|
DurationMs int64 `json:"durationMs,omitempty"`
|
||||||
ValSamples int `json:"valSamples,omitempty"`
|
Epochs int `json:"epochs,omitempty"`
|
||||||
Imgsz int `json:"imgsz,omitempty"`
|
TrainSamples int `json:"trainSamples,omitempty"`
|
||||||
Device string `json:"device,omitempty"`
|
ValSamples int `json:"valSamples,omitempty"`
|
||||||
MAP50 float64 `json:"map50,omitempty"`
|
Imgsz int `json:"imgsz,omitempty"`
|
||||||
MAP5095 float64 `json:"map5095,omitempty"`
|
Device string `json:"device,omitempty"`
|
||||||
|
MAP50 float64 `json:"map50,omitempty"`
|
||||||
|
MAP5095 float64 `json:"map5095,omitempty"`
|
||||||
|
PerformanceMode string `json:"performanceMode,omitempty"`
|
||||||
|
CPUCoreCount int `json:"cpuCoreCount,omitempty"`
|
||||||
|
CPUThreads int `json:"cpuThreads,omitempty"`
|
||||||
|
Workers int `json:"workers,omitempty"`
|
||||||
|
YoloBatchSize int `json:"yoloBatchSize,omitempty"`
|
||||||
|
LowPriority bool `json:"lowPriority,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TrainingHistoryResponse struct {
|
type TrainingHistoryResponse struct {
|
||||||
@ -4485,31 +4792,81 @@ func trainingHistoryPath(root string) string {
|
|||||||
return filepath.Join(root, "detector", "training_history.jsonl")
|
return filepath.Join(root, "detector", "training_history.jsonl")
|
||||||
}
|
}
|
||||||
|
|
||||||
// trainingAppendRunHistory hängt nach einem erfolgreichen Trainingslauf einen
|
// trainingAppendTargetHistory haengt nach einem erfolgreichen Trainingsziel einen
|
||||||
// Verlaufseintrag an (Datum, mAP, Samples, Epochen, Dauer).
|
// Verlaufseintrag an. Alte History-Zeilen ohne Target bleiben weiterhin lesbar.
|
||||||
func trainingAppendRunHistory(root string) {
|
func trainingHistoryKindForTarget(target string) string {
|
||||||
info := trainingReadModelInfo(root)
|
switch strings.ToLower(strings.TrimSpace(target)) {
|
||||||
if info == nil {
|
case "pose":
|
||||||
return
|
return "pose"
|
||||||
|
case "videomae", "video_mae", "scene":
|
||||||
|
return "videomae"
|
||||||
|
default:
|
||||||
|
return "detector"
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func trainingAppendTargetHistory(root string, target string, status string, durationMs int64, runtimeOpts trainingRuntimeOptions, fallback TrainingHistoryEntry) {
|
||||||
|
kind := trainingHistoryKindForTarget(target)
|
||||||
|
info := trainingReadModelInfoFor(root, kind)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
|
||||||
entry := TrainingHistoryEntry{
|
entry := TrainingHistoryEntry{
|
||||||
TrainedAt: info.TrainedAt,
|
TrainedAt: now.Format(time.RFC3339),
|
||||||
TrainedAtMs: info.TrainedAtMs,
|
TrainedAtMs: now.UnixMilli(),
|
||||||
Epochs: info.Epochs,
|
Target: kind,
|
||||||
TrainSamples: info.TrainSamples,
|
Status: strings.TrimSpace(status),
|
||||||
ValSamples: info.ValSamples,
|
DurationMs: durationMs,
|
||||||
Imgsz: info.Imgsz,
|
Epochs: fallback.Epochs,
|
||||||
Device: info.Device,
|
TrainSamples: fallback.TrainSamples,
|
||||||
MAP50: info.MAP50,
|
ValSamples: fallback.ValSamples,
|
||||||
MAP5095: info.MAP5095,
|
Imgsz: fallback.Imgsz,
|
||||||
|
Device: strings.TrimSpace(fallback.Device),
|
||||||
|
MAP50: fallback.MAP50,
|
||||||
|
MAP5095: fallback.MAP5095,
|
||||||
|
PerformanceMode: runtimeOpts.PerformanceMode,
|
||||||
|
CPUCoreCount: runtimeOpts.CPUCoreCount,
|
||||||
|
CPUThreads: runtimeOpts.CPUThreads,
|
||||||
|
Workers: runtimeOpts.Workers,
|
||||||
|
YoloBatchSize: runtimeOpts.YoloBatchSize,
|
||||||
|
LowPriority: runtimeOpts.LowPriority,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dauer aus der Startzeit des aktuellen Jobs ableiten.
|
if info != nil {
|
||||||
job := trainingGetJobStatus()
|
if strings.TrimSpace(info.TrainedAt) != "" {
|
||||||
if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(job.StartedAt)); err == nil {
|
entry.TrainedAt = info.TrainedAt
|
||||||
if ms := time.Now().UTC().Sub(startedAt).Milliseconds(); ms > 0 {
|
}
|
||||||
entry.DurationMs = ms
|
if info.TrainedAtMs > 0 {
|
||||||
|
entry.TrainedAtMs = info.TrainedAtMs
|
||||||
|
}
|
||||||
|
if info.Epochs > 0 {
|
||||||
|
entry.Epochs = info.Epochs
|
||||||
|
}
|
||||||
|
if info.TrainSamples > 0 {
|
||||||
|
entry.TrainSamples = info.TrainSamples
|
||||||
|
}
|
||||||
|
if info.ValSamples > 0 {
|
||||||
|
entry.ValSamples = info.ValSamples
|
||||||
|
}
|
||||||
|
if info.Imgsz > 0 {
|
||||||
|
entry.Imgsz = info.Imgsz
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(info.Device) != "" {
|
||||||
|
entry.Device = strings.TrimSpace(info.Device)
|
||||||
|
}
|
||||||
|
if info.MAP50 > 0 {
|
||||||
|
entry.MAP50 = info.MAP50
|
||||||
|
}
|
||||||
|
if info.MAP5095 > 0 {
|
||||||
|
entry.MAP5095 = info.MAP5095
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if entry.DurationMs <= 0 {
|
||||||
|
job := trainingGetJobStatus()
|
||||||
|
if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(job.StartedAt)); err == nil {
|
||||||
|
if ms := now.Sub(startedAt).Milliseconds(); ms > 0 {
|
||||||
|
entry.DurationMs = ms
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -55,6 +55,21 @@ func trainingVideoMAEBatchSize() int {
|
|||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func trainingVideoMAEEarlyStoppingPatience() int {
|
||||||
|
epochs := trainingVideoMAEEpochs()
|
||||||
|
patience := epochs / 3
|
||||||
|
if patience < 2 {
|
||||||
|
patience = 2
|
||||||
|
}
|
||||||
|
if patience > 8 {
|
||||||
|
patience = 8
|
||||||
|
}
|
||||||
|
if patience > epochs {
|
||||||
|
patience = epochs
|
||||||
|
}
|
||||||
|
return patience
|
||||||
|
}
|
||||||
|
|
||||||
type trainingVideoMAEManifestEntry struct {
|
type trainingVideoMAEManifestEntry struct {
|
||||||
SampleID string `json:"sampleId"`
|
SampleID string `json:"sampleId"`
|
||||||
Split string `json:"split"`
|
Split string `json:"split"`
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user