From 0a6fefa6630ed0575cc17e7e9d2d629550437516 Mon Sep 17 00:00:00 2001 From: Linrador <68631622+Linrador@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:22:29 +0200 Subject: [PATCH] added videomae --- backend/ai_server.py | 269 ++++++++++++ backend/analyze.go | 33 +- backend/analyze_videomae.go | 271 ++++++++++++ backend/embed.go | 2 + backend/ml/predict_videomae_model.py | 174 ++++++++ backend/ml/train_videomae_model.py | 397 +++++++++++++++++ backend/training.go | 205 ++++++++- backend/training_videomae.go | 610 +++++++++++++++++++++++++++ backend/videomae_test.go | 70 +++ 9 files changed, 2019 insertions(+), 12 deletions(-) create mode 100644 backend/analyze_videomae.go create mode 100644 backend/ml/predict_videomae_model.py create mode 100644 backend/ml/train_videomae_model.py create mode 100644 backend/training_videomae.go create mode 100644 backend/videomae_test.go diff --git a/backend/ai_server.py b/backend/ai_server.py index 66cd1b5..fdd71d7 100644 --- a/backend/ai_server.py +++ b/backend/ai_server.py @@ -70,6 +70,15 @@ def existing_file(path: Path) -> Optional[Path]: return None +def existing_model_dir(path: Path) -> Optional[Path]: + try: + if path.exists() and path.is_dir() and existing_file(path / "config.json"): + return path + except OSError: + pass + return None + + def resolve_training_root() -> Path: env_root = os.environ.get("TRAINING_ROOT", "").strip() if env_root: @@ -95,6 +104,7 @@ def resolve_training_root() -> Path: if ( existing_file(root / "detection_labels.json") or existing_file(root / "detector" / "model" / "best.pt") + or existing_model_dir(root / "videomae" / "model") ): root.mkdir(parents=True, exist_ok=True) return root.resolve() @@ -108,6 +118,7 @@ def resolve_training_root() -> Path: TRAINING_ROOT = resolve_training_root() DEFAULT_MODEL_PATH = TRAINING_ROOT / "detector" / "model" / "best.pt" DEFAULT_POSE_MODEL_PATH = TRAINING_ROOT / "pose" / "model" / "best.pt" +DEFAULT_VIDEOMAE_MODEL_PATH = TRAINING_ROOT / "videomae" / "model" def resolve_detection_labels_path() -> Path: @@ -153,6 +164,20 @@ def resolve_pose_model_path() -> Optional[Path]: return None +def resolve_videomae_model_path() -> Optional[Path]: + env_path = os.environ.get("VIDEOMAE_MODEL", "").strip() + if env_path: + p = Path(env_path).expanduser().resolve() + if existing_model_dir(p): + return p + raise RuntimeError(f"VIDEOMAE_MODEL not found: {p}") + + if existing_model_dir(DEFAULT_VIDEOMAE_MODEL_PATH): + return DEFAULT_VIDEOMAE_MODEL_PATH.resolve() + + return None + + # Server darf auch ohne Labels/Model starten. DETECTION_LABELS_PATH: Optional[Path] = None @@ -178,6 +203,9 @@ _MODEL_PATH = "" _MODEL_ERROR = "" _POSE_MODEL_PATH = "" _POSE_MODEL_ERROR = "" +_VIDEOMAE_MODEL_PATH = "" +_VIDEOMAE_MODEL_ERROR = "" +_VIDEOMAE_DEVICE_ACTIVE = "" _LABEL_ERROR = "" _DEVICE = os.environ.get("YOLO_DEVICE", "") @@ -193,9 +221,13 @@ _POSITION_CONTEXT_OVERRIDE_MARGIN = float(os.environ.get("YOLO_POSITION_CONTEXT_ _BATCH = int(os.environ.get("YOLO_BATCH", "16")) _IMGSZ = int(os.environ.get("YOLO_IMGSZ", "640")) _HALF = os.environ.get("YOLO_HALF", "0").lower() in {"1", "true", "yes", "on"} +_VIDEOMAE_DEVICE = os.environ.get("VIDEOMAE_DEVICE", "auto") +_VIDEOMAE_NUM_FRAMES = int(os.environ.get("VIDEOMAE_NUM_FRAMES", "16")) model = None pose_model = None +videomae_model = None +videomae_processor = None app = FastAPI() @@ -243,6 +275,18 @@ class PredictBatchRequest(BaseModel): model: Optional[str] = None +class PositionClipItem(BaseModel): + time: float = 0.0 + start: float = 0.0 + end: float = 0.0 + paths: List[str] + + +class PredictPositionClipsRequest(BaseModel): + clips: List[PositionClipItem] + numFrames: int = 16 + + def empty_prediction(source: str = "model_missing") -> dict: return { "modelAvailable": False, @@ -390,6 +434,155 @@ def get_pose_model(): return None +def get_videomae_components(): + global videomae_model + global videomae_processor + global _VIDEOMAE_MODEL_PATH + global _VIDEOMAE_MODEL_ERROR + global _VIDEOMAE_DEVICE_ACTIVE + + if videomae_model is not None and videomae_processor is not None: + return videomae_model, videomae_processor + + try: + path = resolve_videomae_model_path() + if path is None: + videomae_model = None + videomae_processor = None + _VIDEOMAE_MODEL_PATH = "" + _VIDEOMAE_MODEL_ERROR = "" + _VIDEOMAE_DEVICE_ACTIVE = "" + return None, None + + import torch + from transformers import AutoImageProcessor, VideoMAEForVideoClassification + + if str(_VIDEOMAE_DEVICE).lower() == "auto": + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + else: + device = torch.device(_VIDEOMAE_DEVICE) + + processor = AutoImageProcessor.from_pretrained(path) + loaded = VideoMAEForVideoClassification.from_pretrained(path) + loaded.to(device) + loaded.eval() + + videomae_model = loaded + videomae_processor = processor + _VIDEOMAE_MODEL_PATH = str(path) + _VIDEOMAE_MODEL_ERROR = "" + _VIDEOMAE_DEVICE_ACTIVE = str(device) + + return videomae_model, videomae_processor + + except Exception as exc: + videomae_model = None + videomae_processor = None + _VIDEOMAE_MODEL_PATH = "" + _VIDEOMAE_DEVICE_ACTIVE = "" + _VIDEOMAE_MODEL_ERROR = str(exc) + return None, None + + +def resample_values(values: list, count: int) -> list: + if not values: + return [] + if count <= 1: + return [values[0]] + if len(values) == 1: + return [values[0] for _ in range(count)] + + last = len(values) - 1 + return [ + values[int(round((i * last) / max(1, count - 1)))] + for i in range(count) + ] + + +def load_videomae_clip_frames(paths: list[str], num_frames: int): + from PIL import Image + + clean_paths = [ + str(path).strip() + for path in paths or [] + if str(path).strip() + ] + selected = resample_values(clean_paths, max(2, int(num_frames or _VIDEOMAE_NUM_FRAMES or 16))) + frames = [] + + for path in selected: + with Image.open(path) as img: + frames.append(img.convert("RGB").copy()) + + return frames + + +def predict_videomae_clip(clip: PositionClipItem, num_frames: int) -> dict: + current_model, current_processor = get_videomae_components() + if current_model is None or current_processor is None: + return { + "time": float(clip.time or 0.0), + "start": float(clip.start or 0.0), + "end": float(clip.end or 0.0), + "sexPosition": NO_SEX_POSITION_LABEL, + "sexPositionScore": 0.0, + "source": "videomae_missing", + "scores": [], + } + + import torch + + frames = load_videomae_clip_frames(clip.paths, num_frames) + if not frames: + return { + "time": float(clip.time or 0.0), + "start": float(clip.start or 0.0), + "end": float(clip.end or 0.0), + "sexPosition": NO_SEX_POSITION_LABEL, + "sexPositionScore": 0.0, + "source": "videomae_no_frames", + "scores": [], + } + + inputs = current_processor(frames, return_tensors="pt") + device = next(current_model.parameters()).device + pixel_values = inputs["pixel_values"].to(device) + + with torch.no_grad(): + logits = current_model(pixel_values=pixel_values).logits + probs = torch.softmax(logits, dim=-1)[0].detach().cpu().tolist() + + id_to_label = getattr(current_model.config, "id2label", {}) or {} + scores = [] + best_label = NO_SEX_POSITION_LABEL + best_score = 0.0 + + for idx, score in enumerate(probs): + raw_label = id_to_label.get(idx, id_to_label.get(str(idx), idx)) + label = normalize_sex_position_label(raw_label) + score = clamp01(score) + scores.append({ + "label": label, + "score": score, + }) + + if score > best_score: + best_label = label + best_score = score + + scores.sort(key=lambda item: item["score"], reverse=True) + + return { + "time": float(clip.time or 0.0), + "start": float(clip.start or 0.0), + "end": float(clip.end or 0.0), + "sexPosition": best_label, + "sexPositionScore": best_score, + "source": "videomae", + "scores": scores[:10], + } + + def scored(label: str, score: float) -> dict: return { "label": label, @@ -1383,6 +1576,27 @@ def pose_model_status() -> dict: } +def videomae_model_status() -> dict: + try: + expected = resolve_videomae_model_path() + expected_text = str(expected) if expected else str(DEFAULT_VIDEOMAE_MODEL_PATH) + exists = expected is not None + error = _VIDEOMAE_MODEL_ERROR + except Exception as exc: + expected_text = str(DEFAULT_VIDEOMAE_MODEL_PATH) + exists = False + error = str(exc) + + return { + "videoMAEModelAvailable": exists, + "videoMAEModelLoaded": videomae_model is not None, + "videoMAEModel": _VIDEOMAE_MODEL_PATH or (expected_text if exists else ""), + "videoMAEModelError": error, + "expectedVideoMAEModel": expected_text, + "videoMAEDevice": _VIDEOMAE_DEVICE_ACTIVE, + } + + @app.post("/predict-batch", dependencies=[Depends(require_ai_server_auth)]) def predict_batch(req: PredictBatchRequest): paths = [str(path).strip() for path in req.paths if str(path).strip()] @@ -1439,6 +1653,49 @@ def predict_batch(req: PredictBatchRequest): } +@app.post("/predict-position-clips", dependencies=[Depends(require_ai_server_auth)]) +def predict_position_clips(req: PredictPositionClipsRequest): + clips = [clip for clip in req.clips if clip.paths] + if not clips: + return { + "ok": True, + "available": False, + "predictions": [], + "error": "no clips supplied", + } + + current_model, current_processor = get_videomae_components() + if current_model is None or current_processor is None: + return { + "ok": True, + "available": False, + "predictions": [], + "error": _VIDEOMAE_MODEL_ERROR or f"VideoMAE model not found: {DEFAULT_VIDEOMAE_MODEL_PATH}", + } + + predictions = [] + for clip in clips: + try: + predictions.append(predict_videomae_clip(clip, int(req.numFrames or _VIDEOMAE_NUM_FRAMES or 16))) + except Exception as exc: + predictions.append({ + "time": float(clip.time or 0.0), + "start": float(clip.start or 0.0), + "end": float(clip.end or 0.0), + "sexPosition": NO_SEX_POSITION_LABEL, + "sexPositionScore": 0.0, + "source": "videomae_predict_failed", + "error": repr(exc), + "scores": [], + }) + + return { + "ok": True, + "available": True, + "predictions": predictions, + } + + @app.get("/health", dependencies=[Depends(require_ai_server_auth)]) def health(): current_model = get_model() @@ -1459,24 +1716,35 @@ def health(): } status_payload.update(pose_model_status()) + status_payload.update(videomae_model_status()) return status_payload @app.post("/reload", dependencies=[Depends(require_ai_server_auth)]) def reload_model(): global model global pose_model + global videomae_model + global videomae_processor global _MODEL_PATH global _MODEL_ERROR global _POSE_MODEL_PATH global _POSE_MODEL_ERROR + global _VIDEOMAE_MODEL_PATH + global _VIDEOMAE_MODEL_ERROR + global _VIDEOMAE_DEVICE_ACTIVE global DETECTION_LABELS_PATH model = None pose_model = None + videomae_model = None + videomae_processor = None _MODEL_PATH = "" _MODEL_ERROR = "" _POSE_MODEL_PATH = "" _POSE_MODEL_ERROR = "" + _VIDEOMAE_MODEL_PATH = "" + _VIDEOMAE_MODEL_ERROR = "" + _VIDEOMAE_DEVICE_ACTIVE = "" DETECTION_LABELS_PATH = None current_model = get_model() @@ -1497,4 +1765,5 @@ def reload_model(): } status_payload.update(pose_model_status()) + status_payload.update(videomae_model_status()) return status_payload diff --git a/backend/analyze.go b/backend/analyze.go index d391ccc..9e5f9fc 100644 --- a/backend/analyze.go +++ b/backend/analyze.go @@ -106,6 +106,7 @@ type analyzePositionEvidence struct { PersonCount int HasPose bool HasContext bool + HasClip bool } func analyzeVideoFrameFilter(intervalSeconds int) string { @@ -1683,7 +1684,11 @@ func analyzePositionEvidenceFromPrediction( func analyzePositionEvidenceWeight(item analyzePositionEvidence) float64 { weight := 1.0 - if item.HasPose && item.HasContext { + if item.HasClip && item.HasPose { + weight = 1.28 + } else if item.HasClip { + weight = 1.18 + } else if item.HasPose && item.HasContext { weight = 1.15 } else if item.HasPose { weight = 1.0 @@ -1733,6 +1738,7 @@ func buildClipPositionHitsFromEvidence( Count int PoseCount int ContextCount int + ClipCount int Start float64 End float64 Marker float64 @@ -1795,6 +1801,9 @@ func buildClipPositionHitsFromEvidence( if item.HasContext { agg.ContextCount++ } + if item.HasClip { + agg.ClipCount++ + } if item.Time < agg.Start { agg.Start = item.Time } @@ -1820,12 +1829,16 @@ func buildClipPositionHitsFromEvidence( sourceBonus := 0.0 if agg.PoseCount > 0 && agg.ContextCount > 0 { sourceBonus = 0.04 + } else if agg.ClipCount > 0 && (agg.PoseCount > 0 || agg.ContextCount > 0) { + sourceBonus = 0.04 + } else if agg.ClipCount > 0 { + sourceBonus = 0.03 } else if agg.PoseCount > 0 { sourceBonus = 0.02 } score := clamp01(avg*(0.86+0.14*stability) + sourceBonus) - if agg.PoseCount == 0 { + if agg.PoseCount == 0 && agg.ClipCount == 0 { score = math.Min(score, trainingPositionContextMaxScore) } @@ -2292,6 +2305,14 @@ func analyzeVideoFromFramesForGoal( ) } + highlightHits, positionEvidence = applyVideoMAEPositionClipsForAnalyze( + ctx, + samples, + durationSec, + highlightHits, + positionEvidence, + ) + highlightHits = append( highlightHits, buildClipPositionHitsFromEvidence(positionEvidence, durationSec)..., @@ -2349,6 +2370,14 @@ func analyzeVideoFromFramesForGoal( ) } + highlightHits, positionEvidence = applyVideoMAEPositionClipsForAnalyze( + ctx, + samples, + durationSec, + highlightHits, + positionEvidence, + ) + highlightHits = append( highlightHits, buildClipPositionHitsFromEvidence(positionEvidence, durationSec)..., diff --git a/backend/analyze_videomae.go b/backend/analyze_videomae.go new file mode 100644 index 0000000..d6a46d8 --- /dev/null +++ b/backend/analyze_videomae.go @@ -0,0 +1,271 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "os" + "strings" + "time" +) + +const ( + analyzeVideoMAEClipWindowSeconds = 4.0 + analyzeVideoMAEClipStrideSeconds = 2.0 + analyzeVideoMAEMinScore = 0.34 + analyzeVideoMAERequestBatchSize = 48 +) + +type analyzeVideoMAEClipReqItem struct { + Time float64 `json:"time"` + Start float64 `json:"start"` + End float64 `json:"end"` + Paths []string `json:"paths"` +} + +type analyzeVideoMAEClipPredictReq struct { + Clips []analyzeVideoMAEClipReqItem `json:"clips"` + NumFrames int `json:"numFrames,omitempty"` +} + +type analyzeVideoMAEClipPrediction struct { + Time float64 `json:"time"` + Start float64 `json:"start"` + End float64 `json:"end"` + SexPosition string `json:"sexPosition"` + SexPositionScore float64 `json:"sexPositionScore"` + Source string `json:"source,omitempty"` + Scores []TrainingScoredLabel `json:"scores,omitempty"` +} + +type analyzeVideoMAEClipPredictResp struct { + OK bool `json:"ok"` + Available bool `json:"available"` + Predictions []analyzeVideoMAEClipPrediction `json:"predictions"` + Error string `json:"error,omitempty"` +} + +func analyzeVideoMAEEnabled() bool { + raw := strings.ToLower(strings.TrimSpace(os.Getenv("VIDEOMAE_ANALYZE_ENABLED"))) + return raw == "" || raw == "1" || raw == "true" || raw == "yes" || raw == "on" +} + +func buildAnalyzeVideoMAEClips( + samples []videoFrameSample, + duration float64, +) []analyzeVideoMAEClipReqItem { + if len(samples) == 0 || duration <= 0 { + return []analyzeVideoMAEClipReqItem{} + } + + clips := []analyzeVideoMAEClipReqItem{} + halfWindow := analyzeVideoMAEClipWindowSeconds / 2 + if halfWindow <= 0 { + halfWindow = 2 + } + + lastCenter := -math.MaxFloat64 + for _, sample := range samples { + center := math.Max(0, sample.Time) + if len(clips) > 0 && center-lastCenter < analyzeVideoMAEClipStrideSeconds-0.001 { + continue + } + + start := math.Max(0, center-halfWindow) + end := center + halfWindow + if duration > 0 { + end = math.Min(duration, end) + } + if end <= start { + end = math.Min(duration, start+math.Max(1, float64(analyzeVideoFrameIntervalSeconds))) + } + + paths := []string{} + for _, candidate := range samples { + if candidate.Time < start-0.001 || candidate.Time > end+0.001 { + continue + } + + path := strings.TrimSpace(candidate.Path) + if path != "" { + paths = append(paths, path) + } + } + + if len(paths) == 0 { + continue + } + + clips = append(clips, analyzeVideoMAEClipReqItem{ + Time: center, + Start: start, + End: end, + Paths: paths, + }) + lastCenter = center + } + + return clips +} + +func predictVideoMAEPositionClipsForAnalyze( + ctx context.Context, + clips []analyzeVideoMAEClipReqItem, +) ([]analyzeVideoMAEClipPrediction, error) { + if len(clips) == 0 { + return []analyzeVideoMAEClipPrediction{}, nil + } + + if !trainingRecognitionEnabled() { + return []analyzeVideoMAEClipPrediction{}, nil + } + + out := []analyzeVideoMAEClipPrediction{} + for start := 0; start < len(clips); start += analyzeVideoMAERequestBatchSize { + end := start + analyzeVideoMAERequestBatchSize + if end > len(clips) { + end = len(clips) + } + + payload := analyzeVideoMAEClipPredictReq{ + Clips: clips[start:end], + NumFrames: trainingVideoMAENumFrames, + } + + body, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + analyzeAIServerURL()+"/predict-position-clips", + bytes.NewReader(body), + ) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + addAIServerAuth(req) + + client := &http.Client{ + Timeout: 180 * time.Second, + } + + res, err := client.Do(req) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, err + } + + rawBody, readErr := io.ReadAll(res.Body) + _ = res.Body.Close() + if readErr != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, readErr + } + + var parsed analyzeVideoMAEClipPredictResp + if err := json.Unmarshal(rawBody, &parsed); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, fmt.Errorf("AI server VideoMAE JSON ungueltig: HTTP %d: %s", res.StatusCode, strings.TrimSpace(string(rawBody))) + } + + if res.StatusCode < 200 || res.StatusCode >= 300 || !parsed.OK { + msg := strings.TrimSpace(parsed.Error) + if msg == "" { + msg = fmt.Sprintf("AI server VideoMAE HTTP %d", res.StatusCode) + } + return nil, fmt.Errorf("%s", msg) + } + + if !parsed.Available { + return out, nil + } + + out = append(out, parsed.Predictions...) + } + + return out, nil +} + +func applyVideoMAEPositionClipsForAnalyze( + ctx context.Context, + samples []videoFrameSample, + duration float64, + highlightHits []analyzeHit, + positionEvidence []analyzePositionEvidence, +) ([]analyzeHit, []analyzePositionEvidence) { + if !analyzeVideoMAEEnabled() { + return highlightHits, positionEvidence + } + + clips := buildAnalyzeVideoMAEClips(samples, duration) + if len(clips) == 0 { + return highlightHits, positionEvidence + } + + predictions, err := predictVideoMAEPositionClipsForAnalyze(ctx, clips) + if err != nil { + if ctx.Err() == nil { + appLogln("VideoMAE Clip-Analyse uebersprungen:", err) + } + return highlightHits, positionEvidence + } + + for _, pred := range predictions { + label := strings.ToLower(strings.TrimSpace(pred.SexPosition)) + if isNoSexPositionLabel(label) || !isKnownPositionLabel(label) { + continue + } + + score := clamp01(pred.SexPositionScore) + if score < analyzeVideoMAEMinScore { + continue + } + + start := math.Max(0, pred.Start) + end := pred.End + if duration > 0 { + end = math.Min(duration, end) + } + if end <= start { + end = math.Min(duration, start+math.Max(1, float64(analyzeVideoFrameIntervalSeconds))) + } + + source := strings.ToLower(strings.TrimSpace(pred.Source)) + if source == "" { + source = "videomae" + } + + highlightHits = append(highlightHits, analyzeHit{ + Time: pred.Time, + Label: "position:" + label, + Score: score, + Start: start, + End: end, + }) + + positionEvidence = append(positionEvidence, analyzePositionEvidence{ + Time: pred.Time, + Label: label, + Score: score, + Source: source, + HasClip: true, + }) + } + + return highlightHits, positionEvidence +} diff --git a/backend/embed.go b/backend/embed.go index 2b9bee4..859d198 100644 --- a/backend/embed.go +++ b/backend/embed.go @@ -32,6 +32,8 @@ func trainingEmbeddedMLDir() (string, error) { "detection_labels.json", "predict_pose_model.py", "train_pose_model.py", + "predict_videomae_model.py", + "train_videomae_model.py", } // Falls du die alten Scene-Skripte noch embedded hast, kannst du sie optional mitkopieren. diff --git a/backend/ml/predict_videomae_model.py b/backend/ml/predict_videomae_model.py new file mode 100644 index 0000000..8d30f87 --- /dev/null +++ b/backend/ml/predict_videomae_model.py @@ -0,0 +1,174 @@ +import argparse +import json +from pathlib import Path + +import torch +from PIL import Image +from transformers import AutoImageProcessor, VideoMAEForVideoClassification + + +def clamp01(value): + try: + n = float(value) + except Exception: + return 0.0 + return max(0.0, min(1.0, n)) + + +def existing_model_dir(path: Path): + try: + if path.exists() and path.is_dir() and (path / "config.json").exists(): + return path + except Exception: + pass + return None + + +def resolve_model_path(root: Path, requested: str): + requested = str(requested or "").strip() + if requested: + p = existing_model_dir(Path(requested).expanduser().resolve()) + if p: + return p, "videomae_model" + return Path(requested).expanduser(), "videomae_missing" + + trained = root / "videomae" / "model" + p = existing_model_dir(trained) + if p: + return p, "videomae_clip" + + return trained, "videomae_missing" + + +def resample_values(values: list, count: int) -> list: + if not values: + return [] + if len(values) == 1: + return [values[0] for _ in range(count)] + if count <= 1: + return [values[0]] + + last = len(values) - 1 + return [values[int(round((i * last) / max(1, count - 1)))] for i in range(count)] + + +def load_frames(paths: list[str], num_frames: int): + selected = resample_values(paths, num_frames) + frames = [] + for path in selected: + with Image.open(path) as img: + frames.append(img.convert("RGB").copy()) + return frames + + +def frame_paths_from_args(args): + paths = [] + if args.frames_json: + with Path(args.frames_json).open("r", encoding="utf-8") as f: + data = json.load(f) + paths.extend(str(p) for p in data) + + if args.clip_dir: + clip_dir = Path(args.clip_dir) + paths.extend( + str(p) for p in sorted(clip_dir.iterdir()) + if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp"} + ) + + paths.extend(str(p) for p in args.frames) + return [p for p in paths if p.strip()] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True) + parser.add_argument("--model", default="") + parser.add_argument("--clip-dir", default="") + parser.add_argument("--frames-json", default="") + parser.add_argument("--frames", nargs="*", default=[]) + parser.add_argument("--num-frames", type=int, default=16) + parser.add_argument("--device", default="auto") + args = parser.parse_args() + + root = Path(args.root).resolve() + model_path, model_source = resolve_model_path(root, args.model) + if not existing_model_dir(model_path): + print(json.dumps({ + "available": False, + "source": model_source, + "modelPath": str(model_path), + "sexPosition": "keine", + "sexPositionScore": 0.0, + "scores": [], + }, ensure_ascii=False)) + return + + paths = frame_paths_from_args(args) + if not paths: + print(json.dumps({ + "available": False, + "source": "videomae_no_frames", + "modelPath": str(model_path), + "sexPosition": "keine", + "sexPositionScore": 0.0, + "scores": [], + }, ensure_ascii=False)) + return + + if str(args.device).lower() == "auto": + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + else: + device = torch.device(args.device) + + try: + processor = AutoImageProcessor.from_pretrained(model_path) + model = VideoMAEForVideoClassification.from_pretrained(model_path).to(device) + model.eval() + + frames = load_frames(paths, max(2, int(args.num_frames or 16))) + inputs = processor(frames, return_tensors="pt") + pixel_values = inputs["pixel_values"].to(device) + + with torch.no_grad(): + logits = model(pixel_values=pixel_values).logits + probs = torch.softmax(logits, dim=-1)[0].detach().cpu().tolist() + + id_to_label = getattr(model.config, "id2label", {}) or {} + scores = [] + best_label = "keine" + best_score = 0.0 + + for idx, score in enumerate(probs): + label = str(id_to_label.get(idx, id_to_label.get(str(idx), idx))).strip() + score = clamp01(score) + scores.append({"label": label, "score": score}) + if score > best_score: + best_label = label + best_score = score + + scores.sort(key=lambda item: item["score"], reverse=True) + + print(json.dumps({ + "available": True, + "source": model_source, + "modelPath": str(model_path), + "device": str(device), + "sexPosition": best_label, + "sexPositionScore": best_score, + "scores": scores[:10], + }, ensure_ascii=False)) + + except Exception as exc: + print(json.dumps({ + "available": False, + "source": "videomae_predict_failed", + "modelPath": str(model_path), + "error": repr(exc), + "sexPosition": "keine", + "sexPositionScore": 0.0, + "scores": [], + }, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/backend/ml/train_videomae_model.py b/backend/ml/train_videomae_model.py new file mode 100644 index 0000000..34a2f3c --- /dev/null +++ b/backend/ml/train_videomae_model.py @@ -0,0 +1,397 @@ +import argparse +import json +import math +import shutil +from datetime import datetime, timezone +from pathlib import Path + +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from transformers import AutoImageProcessor, VideoMAEForVideoClassification + + +DEFAULT_BASE_MODEL = "MCG-NJU/videomae-base-finetuned-kinetics" + + +def emit_progress(stage, progress, message="", **extra): + out = { + "type": "progress", + "stage": stage, + "progress": max(0.0, min(1.0, float(progress))), + "message": message, + } + out.update(extra) + print(json.dumps(out, ensure_ascii=False), flush=True) + + +def safe_int(value, fallback): + try: + return int(value) + except Exception: + return fallback + + +def safe_float(value, fallback): + try: + return float(value) + except Exception: + return fallback + + +def load_labels(dataset_root: Path) -> list[str]: + path = dataset_root / "labels.json" + if not path.exists(): + raise SystemExit(f"labels.json not found: {path}") + + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + + labels = data.get("labels", []) + out = [] + seen = set() + + for value in labels: + label = str(value or "").strip() + if not label or label in seen: + continue + seen.add(label) + out.append(label) + + if len(out) < 2: + raise SystemExit("VideoMAE needs at least two labels") + + return out + + +def load_manifest(dataset_root: Path, label_to_id: dict[str, int]) -> list[dict]: + path = dataset_root / "manifest.jsonl" + if not path.exists(): + raise SystemExit(f"manifest.jsonl not found: {path}") + + entries = [] + with path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + + try: + item = json.loads(line) + except Exception: + continue + + label = str(item.get("label") or "").strip() + clip_dir = Path(str(item.get("clipDir") or "")).expanduser() + split = str(item.get("split") or "").strip().lower() + + if label not in label_to_id or split not in {"train", "val"}: + continue + if not clip_dir.exists() or not clip_dir.is_dir(): + continue + + frames = sorted( + p for p in clip_dir.iterdir() + if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp"} + ) + if not frames: + continue + + item["label"] = label + item["split"] = split + item["clipDir"] = str(clip_dir) + item["frames"] = [str(p) for p in frames] + entries.append(item) + + return entries + + +def resample_values(values: list, count: int) -> list: + if not values: + return [] + if count <= 1: + return [values[0]] + if len(values) == 1: + return [values[0] for _ in range(count)] + + out = [] + last = len(values) - 1 + for i in range(count): + idx = int(round((i * last) / max(1, count - 1))) + out.append(values[idx]) + return out + + +def load_clip_frames(paths: list[str], num_frames: int) -> list[Image.Image]: + selected = resample_values(paths, num_frames) + frames = [] + + for path in selected: + with Image.open(path) as img: + frames.append(img.convert("RGB").copy()) + + return frames + + +class VideoMAEClipDataset(Dataset): + def __init__(self, entries, label_to_id, image_processor, num_frames): + self.entries = entries + self.label_to_id = label_to_id + self.image_processor = image_processor + self.num_frames = num_frames + + def __len__(self): + return len(self.entries) + + def __getitem__(self, idx): + item = self.entries[idx] + frames = load_clip_frames(item["frames"], self.num_frames) + inputs = self.image_processor(frames, return_tensors="pt") + pixel_values = inputs["pixel_values"].squeeze(0) + label_id = self.label_to_id[item["label"]] + return { + "pixel_values": pixel_values, + "labels": torch.tensor(label_id, dtype=torch.long), + "sample_id": str(item.get("sampleId") or ""), + } + + +def collate_batch(batch): + return { + "pixel_values": torch.stack([item["pixel_values"] for item in batch]), + "labels": torch.stack([item["labels"] for item in batch]), + "sample_ids": [item["sample_id"] for item in batch], + } + + +def evaluate(model, loader, device): + model.eval() + total = 0 + correct = 0 + loss_sum = 0.0 + + with torch.no_grad(): + for batch in loader: + pixel_values = batch["pixel_values"].to(device) + labels = batch["labels"].to(device) + outputs = model(pixel_values=pixel_values, labels=labels) + logits = outputs.logits + loss = outputs.loss + + preds = logits.argmax(dim=-1) + total += int(labels.numel()) + correct += int((preds == labels).sum().item()) + loss_sum += float(loss.item()) * int(labels.numel()) + + if total <= 0: + return 0.0, 0.0 + + return correct / total, loss_sum / total + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True) + parser.add_argument("--base", default=DEFAULT_BASE_MODEL) + parser.add_argument("--epochs", default="8") + parser.add_argument("--batch-size", default="2") + parser.add_argument("--lr", default="5e-5") + parser.add_argument("--device", default="auto") + parser.add_argument("--workers", default="0") + parser.add_argument("--num-frames", default="16") + parser.add_argument("--freeze-backbone", action="store_true") + args = parser.parse_args() + + root = Path(args.root).resolve() + dataset_root = root / "videomae" / "dataset" + out_dir = root / "videomae" / "model" + tmp_dir = root / "videomae" / "runs" / "model_tmp" + + epochs = max(1, safe_int(args.epochs, 8)) + batch_size = max(1, safe_int(args.batch_size, 2)) + workers = max(0, safe_int(args.workers, 0)) + num_frames = max(2, safe_int(args.num_frames, 16)) + lr = max(1e-7, safe_float(args.lr, 5e-5)) + + if str(args.device).lower() == "auto": + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + else: + device = torch.device(args.device) + + labels = load_labels(dataset_root) + label_to_id = {label: i for i, label in enumerate(labels)} + id_to_label = {i: label for label, i in label_to_id.items()} + entries = load_manifest(dataset_root, label_to_id) + train_entries = [item for item in entries if item["split"] == "train"] + val_entries = [item for item in entries if item["split"] == "val"] + + emit_progress( + "videomae", + 0.01, + "VideoMAE-Dataset wird geprueft...", + trainSamples=len(train_entries), + valSamples=len(val_entries), + epochs=epochs, + device=str(device), + ) + + if not train_entries: + raise SystemExit("no VideoMAE train clips found") + if not val_entries: + raise SystemExit("no VideoMAE val clips found") + + emit_progress( + "videomae", + 0.03, + "VideoMAE-Basismodell wird geladen...", + base=args.base, + labels=len(labels), + device=str(device), + ) + + image_processor = AutoImageProcessor.from_pretrained(args.base) + model = VideoMAEForVideoClassification.from_pretrained( + args.base, + num_labels=len(labels), + label2id=label_to_id, + id2label=id_to_label, + ignore_mismatched_sizes=True, + ) + + if args.freeze_backbone and hasattr(model, "videomae"): + for param in model.videomae.parameters(): + param.requires_grad = False + + model.to(device) + + train_ds = VideoMAEClipDataset(train_entries, label_to_id, image_processor, num_frames) + val_ds = VideoMAEClipDataset(val_entries, label_to_id, image_processor, num_frames) + train_loader = DataLoader( + train_ds, + batch_size=batch_size, + shuffle=True, + num_workers=workers, + collate_fn=collate_batch, + ) + val_loader = DataLoader( + val_ds, + batch_size=batch_size, + shuffle=False, + num_workers=workers, + collate_fn=collate_batch, + ) + + optimizer = torch.optim.AdamW( + [p for p in model.parameters() if p.requires_grad], + lr=lr, + ) + + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True, exist_ok=True) + + best_accuracy = -1.0 + best_loss = math.inf + best_epoch = 0 + + for epoch in range(1, epochs + 1): + model.train() + running_loss = 0.0 + seen = 0 + total_batches = max(1, len(train_loader)) + + for batch_idx, batch in enumerate(train_loader, start=1): + pixel_values = batch["pixel_values"].to(device) + labels_tensor = batch["labels"].to(device) + + optimizer.zero_grad(set_to_none=True) + outputs = model(pixel_values=pixel_values, labels=labels_tensor) + loss = outputs.loss + loss.backward() + optimizer.step() + + batch_size_seen = int(labels_tensor.numel()) + seen += batch_size_seen + running_loss += float(loss.item()) * batch_size_seen + + completed = (epoch - 1) + min(1.0, batch_idx / total_batches) + emit_progress( + "videomae", + 0.04 + 0.84 * (completed / max(1, epochs)), + "VideoMAE trainiert...", + epoch=epoch, + epochs=epochs, + sampleId=(batch["sample_ids"][0] if batch["sample_ids"] else ""), + trainSamples=len(train_entries), + valSamples=len(val_entries), + device=str(device), + loss=(running_loss / max(1, seen)), + ) + + val_accuracy, val_loss = evaluate(model, val_loader, device) + is_best = val_accuracy > best_accuracy or ( + math.isclose(val_accuracy, best_accuracy) and val_loss < best_loss + ) + + if is_best: + best_accuracy = val_accuracy + best_loss = val_loss + best_epoch = epoch + model.save_pretrained(tmp_dir) + image_processor.save_pretrained(tmp_dir) + + emit_progress( + "videomae", + 0.88 + 0.08 * (epoch / max(1, epochs)), + "VideoMAE wird validiert...", + epoch=epoch, + epochs=epochs, + trainSamples=len(train_entries), + valSamples=len(val_entries), + device=str(device), + accuracy=val_accuracy, + loss=val_loss, + ) + + if best_epoch <= 0: + model.save_pretrained(tmp_dir) + image_processor.save_pretrained(tmp_dir) + best_epoch = epochs + + status = { + "trainedAt": datetime.now(timezone.utc).isoformat(), + "epochs": epochs, + "bestEpoch": best_epoch, + "trainSamples": len(train_entries), + "valSamples": len(val_entries), + "numFrames": num_frames, + "baseModel": args.base, + "device": str(device), + "accuracy": best_accuracy if best_accuracy >= 0 else 0.0, + "loss": best_loss if math.isfinite(best_loss) else 0.0, + "labels": labels, + } + + with (tmp_dir / "status.json").open("w", encoding="utf-8") as f: + json.dump(status, f, ensure_ascii=False, indent=2) + + if out_dir.exists(): + shutil.rmtree(out_dir) + shutil.copytree(tmp_dir, out_dir) + + emit_progress( + "videomae", + 1.0, + "VideoMAE-Training abgeschlossen.", + epoch=best_epoch, + epochs=epochs, + trainSamples=len(train_entries), + valSamples=len(val_entries), + device=str(device), + accuracy=status["accuracy"], + loss=status["loss"], + ) + + +if __name__ == "__main__": + main() diff --git a/backend/training.go b/backend/training.go index a89df0a..fb4f5e3 100644 --- a/backend/training.go +++ b/backend/training.go @@ -229,6 +229,8 @@ type TrainingStatsResponse struct { DetectorModelInfo *TrainingModelInfo `json:"detectorModelInfo,omitempty"` PoseModelAvailable bool `json:"poseModelAvailable"` PoseModelInfo *TrainingModelInfo `json:"poseModelInfo,omitempty"` + VideoMAEModelAvailable bool `json:"videoMAEModelAvailable"` + VideoMAEModelInfo *TrainingModelInfo `json:"videoMAEModelInfo,omitempty"` Confidence TrainingConfidence `json:"confidence"` Labels TrainingStatsLabels `json:"labels"` } @@ -2377,6 +2379,10 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) { trainingWriteError(w, http.StatusInternalServerError, err.Error()) return } + if err := trainingEnsureVideoMAEDirs(root); err != nil { + trainingWriteError(w, http.StatusInternalServerError, err.Error()) + return + } if err := trainingEnsureDetectorValidationSample(root); err != nil { appLogln("⚠️ detector val sample ensure failed:", err) @@ -2412,6 +2418,7 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) { poseValImages := filepath.Join(root, "pose", "dataset", "images", "val") poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val") poseDatasetYAML := filepath.Join(root, "pose", "dataset", "dataset.yaml") + videoMAEManifest := trainingVideoMAEManifestPath(root) trainCount := trainingCountDetectorSamples(detectorTrainImages, detectorTrainLabels) valCount := trainingCountDetectorSamples(detectorValImages, detectorValLabels) @@ -2419,6 +2426,23 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) { positiveValCount := trainingCountPositiveDetectorSamples(detectorValImages, detectorValLabels) poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels) poseValCount := trainingCountDetectorSamples(poseValImages, poseValLabels) + videoMAETrainCount, videoMAEValCount := trainingCountVideoMAEManifestSamples(root) + videoMAEEligibleCount, _ := trainingCountVideoMAEEligibleAnnotations(root) + + detectorDataReady := fileExistsNonEmpty(detectorDatasetYAML) && + trainCount >= minDetectorTrainCount && + valCount >= minDetectorValCount && + positiveTrainCount > 0 && + positiveValCount > 0 + poseDataReady := fileExistsNonEmpty(poseDatasetYAML) && + poseTrainCount >= minPoseTrainCount && + poseValCount >= minPoseValCount + videoMAEDataReady := videoMAEEligibleCount >= minVideoMAETrainCount || + (videoMAETrainCount >= minVideoMAETrainCount && videoMAEValCount >= minVideoMAEValCount) + + if detectorDataReady || poseDataReady || videoMAEDataReady { + goto startTraining + } if !fileExistsNonEmpty(detectorDatasetYAML) || trainCount < minDetectorTrainCount || @@ -2458,6 +2482,7 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) { return } +startTraining: ctx, cancel := context.WithCancel(context.Background()) trainingStartJob(cancel) @@ -2489,6 +2514,15 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) { "datasetYAML": poseDatasetYAML, "source": "yolo26_pose", }, + "videomae": map[string]any{ + "eligibleCount": videoMAEEligibleCount, + "trainCount": videoMAETrainCount, + "valCount": videoMAEValCount, + "requiredTrain": minVideoMAETrainCount, + "requiredVal": minVideoMAEValCount, + "manifest": videoMAEManifest, + "source": "videomae_clip", + }, }) } @@ -2549,6 +2583,8 @@ func trainingRunJob(ctx context.Context, root string, count int) { detectorStatus := "skipped" poseOutput := "" poseStatus := "skipped" + videoMAEOutput := "" + videoMAEStatus := "skipped" detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml") detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train") @@ -2702,7 +2738,7 @@ func trainingRunJob(ctx context.Context, root string, count int) { return trainingHandleProgressLine( line, 62, - 98, + 82, "YOLO26 Pose wird trainiert...", ) }, @@ -2749,6 +2785,104 @@ func trainingRunJob(ctx context.Context, root string, count int) { poseOutputClean := cleanOutput(poseOutput) + trainingSetJobStatus(func(s *TrainingJobStatus) { + if s.Progress < 84 { + s.Progress = 84 + } + s.Step = "VideoMAE Clip-Daten werden aufgebaut..." + }) + + videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr := + trainingSyncVideoMAEDataset(ctx, root) + if errors.Is(videoMAESyncErr, context.Canceled) || errors.Is(videoMAESyncErr, errTrainingCancelled) { + appLogln("VideoMAE dataset sync cancelled") + trainingFinishCancelled(root) + return + } + if videoMAESyncErr != nil { + videoMAEStatus = "failed" + videoMAEOutput = "VideoMAE-Dataset konnte nicht aufgebaut werden: " + videoMAESyncErr.Error() + appLogln(videoMAEOutput) + } else { + appLogf( + "VideoMAE samples synced: written=%d train=%d val=%d", + videoMAEWritten, + videoMAETrainCount, + videoMAEValCount, + ) + } + + if videoMAEStatus != "failed" && + videoMAETrainCount >= minVideoMAETrainCount && + videoMAEValCount >= minVideoMAEValCount { + trainingSetJobStatus(func(s *TrainingJobStatus) { + if s.Progress < 86 { + s.Progress = 86 + } + s.Step = "VideoMAE Clip-Classifier wird trainiert..." + }) + + videoMAEScript := trainingScriptPath("train_videomae_model.py") + videoMAEArgs := []string{ + "--root", root, + "--epochs", strconv.Itoa(trainingVideoMAEEpochs()), + "--batch-size", strconv.Itoa(trainingVideoMAEBatchSize()), + "--num-frames", strconv.Itoa(trainingVideoMAENumFrames), + } + if base := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL")); base != "" { + videoMAEArgs = append(videoMAEArgs, "--base", base) + } + + videoMAEOut, videoMAEErr := trainingRunCommandStreaming( + ctx, + python, + videoMAEScript, + func(line string) bool { + return trainingHandleProgressLine( + line, + 86, + 98, + "VideoMAE Clip-Classifier wird trainiert...", + ) + }, + videoMAEArgs..., + ) + + if errors.Is(videoMAEErr, errTrainingCancelled) { + appLogln("VideoMAE training cancelled") + trainingFinishCancelled(root) + return + } + + videoMAEOutput = videoMAEOut + videoMAEOutputClean := cleanOutput(videoMAEOutput) + + if videoMAEErr != nil { + videoMAEStatus = "failed" + appLogln("VideoMAE training failed:", videoMAEErr) + if videoMAEOutputClean != "" { + appLogln("VideoMAE output:", videoMAEOutputClean) + } + } else { + videoMAEStatus = "trained" + if videoMAEOutputClean != "" { + appLogln("VideoMAE training:", videoMAEOutputClean) + } + } + } else if videoMAEStatus != "failed" { + videoMAEStatus = "skipped_no_videomae_data" + videoMAEOutput = fmt.Sprintf( + "VideoMAE uebersprungen: zu wenige Clip-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.", + videoMAETrainCount, + videoMAEValCount, + minVideoMAETrainCount, + minVideoMAEValCount, + ) + appLogln(videoMAEOutput) + } + + videoMAEOutputClean := cleanOutput(videoMAEOutput) + message := "Training abgeschlossen." errorText := "" @@ -2791,7 +2925,23 @@ func trainingRunJob(ctx context.Context, root string, count int) { errorText = message } - if detectorStatus == "trained" { + if videoMAEStatus == "trained" { + if !strings.Contains(message, "Training abgeschlossen.") { + message = "Training abgeschlossen. " + message + } + message += " VideoMAE wurde trainiert." + } else if videoMAEStatus == "skipped_no_videomae_data" { + if detectorStatus == "trained" || poseStatus == "trained" { + message += " VideoMAE wurde uebersprungen: zu wenige Clip-Beispiele." + } + } else if videoMAEStatus == "failed" { + message += " VideoMAE ist fehlgeschlagen." + if videoMAEOutputClean != "" { + message += " Grund: " + videoMAEOutputClean + } + } + + if detectorStatus == "trained" || poseStatus == "trained" || videoMAEStatus == "trained" { // Verlaufseintrag schreiben, solange die Job-Startzeit für die Dauer noch verfügbar ist. trainingAppendRunHistory(root) } @@ -3109,6 +3259,10 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { trainingWriteError(w, http.StatusInternalServerError, err.Error()) return } + if err := trainingEnsureVideoMAEDirs(root); err != nil { + trainingWriteError(w, http.StatusInternalServerError, err.Error()) + return + } if err := trainingEnsureDetectorValidationSample(root); err != nil { appLogln("⚠️ detector val sample ensure failed:", err) @@ -3133,6 +3287,8 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val") detectorModel := trainingResolveDetectorModel(root) poseModel := trainingResolvePoseModel(root) + videoMAEModel := trainingResolveVideoMAEModel(root) + videoMAEManifest := trainingVideoMAEManifestPath(root) trainCount := trainingCountDetectorSamples(detectorTrainImages, detectorTrainLabels) valCount := trainingCountDetectorSamples(detectorValImages, detectorValLabels) @@ -3140,6 +3296,8 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { positiveValCount := trainingCountPositiveDetectorSamples(detectorValImages, detectorValLabels) poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels) poseValCount := trainingCountDetectorSamples(poseValImages, poseValLabels) + videoMAETrainCount, videoMAEValCount := trainingCountVideoMAEManifestSamples(root) + videoMAEEligibleCount, _ := trainingCountVideoMAEEligibleAnnotations(root) datasetReady := fileExistsNonEmpty(detectorDatasetYAML) detectorDataReady := datasetReady && @@ -3151,8 +3309,13 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { poseDataReady := poseDatasetReady && poseTrainCount >= minPoseTrainCount && poseValCount >= minPoseValCount + videoMAEDatasetReady := fileExistsNonEmpty(videoMAEManifest) + videoMAEDataReady := (videoMAETrainCount >= minVideoMAETrainCount && + videoMAEValCount >= minVideoMAEValCount) || + videoMAEEligibleCount >= minVideoMAETrainCount - canTrain := feedbackCount >= minTrainingFeedbackCount && detectorDataReady && poseDataReady + canTrain := feedbackCount >= minTrainingFeedbackCount && + (detectorDataReady || poseDataReady || videoMAEDataReady) trainingWriteJSON(w, http.StatusOK, map[string]any{ "ok": true, @@ -3215,13 +3378,14 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { }, "scene": map[string]any{ - "source": "disabled", + "source": "videomae_clip", + "usesVideoMAE": true, "usesSceneCLIP": false, "usesSceneKNN": false, "usesResNet18KNN": false, "usesLogisticRegression": false, - "predictsSexPosition": false, + "predictsSexPosition": videoMAEModel.TrainedExists, "predictsPeople": false, "predictsGender": false, "predictsBodyParts": false, @@ -3230,17 +3394,27 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { "predictsBoxes": false, "feedbackCount": feedbackCount, - "requiredCount": minTrainingFeedbackCount, - "dataReady": false, - "modelReady": false, + "eligibleCount": videoMAEEligibleCount, + "trainCount": videoMAETrainCount, + "valCount": videoMAEValCount, + "requiredTrain": minVideoMAETrainCount, + "requiredVal": minVideoMAEValCount, + "requiredCount": minVideoMAETrainCount, + "datasetReady": videoMAEDatasetReady, + "manifest": videoMAEManifest, + "dataReady": videoMAEDataReady, + "modelReady": videoMAEModel.EffectiveExists, + "modelExists": videoMAEModel.EffectiveExists, + "modelPath": videoMAEModel.EffectivePath, + "modelSource": videoMAEModel.Source, }, "pipeline": map[string]any{ - "variant": "YOLO26_ONLY", + "variant": "YOLO26_VIDEO_CLIP_HYBRID", "peopleSource": "yolo26_detector", "genderSource": "yolo26_detector", - "sexPositionSource": "yolo26_pose", + "sexPositionSource": "yolo26_pose+box_context+videomae_clip", "bodyPartsSource": "yolo26_detector", "objectsSource": "yolo26_detector", "clothingSource": "yolo26_detector", @@ -3249,6 +3423,7 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) { "usesSceneKNNForDetection": false, "usesSceneCLIP": false, "usesSceneKNN": false, + "usesVideoMAE": true, "usesYOLOForDetection": true, "usesYOLOForSexPosition": true, }, @@ -3260,6 +3435,8 @@ func trainingApplyStatsModelInfo(root string, stats *TrainingStatsResponse) { detectorInfo := trainingReadModelInfoFor(root, "detector") poseAvailable := trainingStatsModelAvailableFor(root, "pose") poseInfo := trainingReadModelInfoFor(root, "pose") + videoMAEModel := trainingResolveVideoMAEModel(root) + videoMAEInfo := trainingReadModelInfoFor(root, "videomae") // modelAvailable/modelInfo bleiben aus Kompatibilitaetsgruenden der Detector. stats.ModelAvailable = detectorAvailable @@ -3268,6 +3445,8 @@ func trainingApplyStatsModelInfo(root string, stats *TrainingStatsResponse) { stats.DetectorModelInfo = detectorInfo stats.PoseModelAvailable = poseAvailable stats.PoseModelInfo = poseInfo + stats.VideoMAEModelAvailable = videoMAEModel.EffectiveExists + stats.VideoMAEModelInfo = videoMAEInfo } func trainingStatsModelAvailable(root string) bool { @@ -3276,6 +3455,9 @@ func trainingStatsModelAvailable(root string) bool { func trainingStatsModelAvailableFor(root string, kind string) bool { modelPath := filepath.Join(root, kind, "model", "best.pt") + if kind == "videomae" { + modelPath = filepath.Join(root, kind, "model", "config.json") + } return fileExistsNonEmpty(modelPath) } @@ -3288,6 +3470,9 @@ func trainingReadModelInfo(root string) *TrainingModelInfo { func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo { modelPath := filepath.Join(root, kind, "model", "best.pt") + if kind == "videomae" { + modelPath = filepath.Join(root, kind, "model", "config.json") + } fi, err := os.Stat(modelPath) if err != nil || fi.IsDir() || fi.Size() <= 0 { diff --git a/backend/training_videomae.go b/backend/training_videomae.go new file mode 100644 index 0000000..7c95d45 --- /dev/null +++ b/backend/training_videomae.go @@ -0,0 +1,610 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" +) + +const ( + trainingVideoMAEClipSeconds = 4.0 + trainingVideoMAENumFrames = 16 + trainingVideoMAEFrameSize = 224 + minVideoMAETrainCount = 5 + minVideoMAEValCount = 1 +) + +func trainingVideoMAEEpochs() int { + raw := strings.TrimSpace(os.Getenv("TRAINING_VIDEOMAE_EPOCHS")) + if raw == "" { + return 8 + } + + n, err := strconv.Atoi(raw) + if err != nil || n < 1 { + return 8 + } + if n > 200 { + return 200 + } + return n +} + +func trainingVideoMAEBatchSize() int { + raw := strings.TrimSpace(os.Getenv("TRAINING_VIDEOMAE_BATCH")) + if raw == "" { + return 2 + } + + n, err := strconv.Atoi(raw) + if err != nil || n < 1 { + return 2 + } + if n > 32 { + return 32 + } + return n +} + +type trainingVideoMAEManifestEntry struct { + SampleID string `json:"sampleId"` + Split string `json:"split"` + Label string `json:"label"` + ClipDir string `json:"clipDir"` + SourcePath string `json:"sourcePath,omitempty"` + Second float64 `json:"second,omitempty"` + FrameCount int `json:"frameCount"` +} + +func trainingResolveVideoMAEModel(root string) trainingModelResolution { + modelDir := filepath.Join(root, "videomae", "model") + configPath := filepath.Join(modelDir, "config.json") + if fileExistsNonEmpty(configPath) { + return trainingModelResolution{ + BestPath: modelDir, + EffectivePath: modelDir, + Source: "videomae_clip", + TrainedExists: true, + EffectiveExists: true, + } + } + + return trainingModelResolution{ + BestPath: modelDir, + EffectivePath: modelDir, + Source: "videomae_missing", + TrainedExists: false, + EffectiveExists: false, + } +} + +func trainingEnsureVideoMAEDirs(root string) error { + dirs := []string{ + filepath.Join(root, "videomae"), + filepath.Join(root, "videomae", "dataset"), + filepath.Join(root, "videomae", "dataset", "clips"), + filepath.Join(root, "videomae", "dataset", "clips", "train"), + filepath.Join(root, "videomae", "dataset", "clips", "val"), + filepath.Join(root, "videomae", "model"), + filepath.Join(root, "videomae", "runs"), + } + + for _, dir := range dirs { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } + + return trainingWriteVideoMAELabelsFile(root) +} + +func trainingVideoMAELabels() ([]string, error) { + grouped, err := trainingGroupedLabels() + if err != nil { + return nil, err + } + + out := []string{trainingNoSexPositionLabel} + seen := map[string]bool{ + trainingNoSexPositionLabel: true, + } + + for _, value := range grouped.SexPositions { + label := normalizeSexPositionLabel(value) + if seen[label] { + continue + } + + seen[label] = true + out = append(out, label) + } + + return out, nil +} + +func trainingVideoMAELabelSet() (map[string]bool, error) { + labels, err := trainingVideoMAELabels() + if err != nil { + return nil, err + } + + out := map[string]bool{} + for _, label := range labels { + out[label] = true + } + return out, nil +} + +func trainingWriteVideoMAELabelsFile(root string) error { + labels, err := trainingVideoMAELabels() + if err != nil { + return err + } + + body, err := json.MarshalIndent(map[string]any{ + "labels": labels, + }, "", " ") + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(root, "videomae", "dataset", "labels.json"), body, 0644) +} + +func trainingVideoMAEManifestPath(root string) string { + return filepath.Join(root, "videomae", "dataset", "manifest.jsonl") +} + +func trainingVideoMAELabelForAnnotation(item TrainingAnnotation) string { + if item.Negative { + return trainingNoSexPositionLabel + } + + effective := trainingEffectiveCorrection(item) + label := normalizeSexPositionLabel(effective.SexPosition) + if isNoSexPositionLabel(label) { + return trainingNoSexPositionLabel + } + + return label +} + +func trainingVideoMAEFrameFallbackPath(root string, sampleID string) string { + return filepath.Join(root, "frames", sampleID+".jpg") +} + +func trainingVideoMAEAnnotationHasSource(root string, item TrainingAnnotation) bool { + sourcePath := strings.TrimSpace(item.SourcePath) + if sourcePath != "" && trainingSupportedImportVideo(sourcePath) && fileExistsNonEmpty(sourcePath) { + return true + } + + return fileExistsNonEmpty(trainingVideoMAEFrameFallbackPath(root, item.SampleID)) +} + +func trainingCountVideoMAEEligibleAnnotations(root string) (int, error) { + items, err := trainingReadAnnotations(root) + if err != nil { + return 0, err + } + + labelSet, err := trainingVideoMAELabelSet() + if err != nil { + return 0, err + } + + count := 0 + for _, item := range items { + sampleID := strings.TrimSpace(item.SampleID) + if sampleID == "" || strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") { + continue + } + + if !labelSet[trainingVideoMAELabelForAnnotation(item)] { + continue + } + + if trainingVideoMAEAnnotationHasSource(root, item) { + count++ + } + } + + return count, nil +} + +func trainingReadVideoMAEManifest(root string) ([]trainingVideoMAEManifestEntry, error) { + path := trainingVideoMAEManifestPath(root) + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return []trainingVideoMAEManifestEntry{}, nil + } + return nil, err + } + defer f.Close() + + out := []trainingVideoMAEManifestEntry{} + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + var entry trainingVideoMAEManifestEntry + if err := json.Unmarshal([]byte(line), &entry); err != nil { + continue + } + + out = append(out, entry) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return out, nil +} + +func trainingWriteVideoMAEManifest(root string, entries []trainingVideoMAEManifestEntry) error { + path := trainingVideoMAEManifestPath(root) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + + var b strings.Builder + for _, entry := range entries { + line, err := json.Marshal(entry) + if err != nil { + return err + } + b.Write(line) + b.WriteByte('\n') + } + + return os.WriteFile(path, []byte(b.String()), 0644) +} + +func trainingCountVideoMAEManifestSamples(root string) (train int, val int) { + entries, err := trainingReadVideoMAEManifest(root) + if err != nil { + return 0, 0 + } + + for _, entry := range entries { + if entry.FrameCount <= 0 || !fileExistsNonEmpty(filepath.Join(entry.ClipDir, "frame_001.jpg")) { + continue + } + + switch strings.ToLower(strings.TrimSpace(entry.Split)) { + case "train": + train++ + case "val": + val++ + } + } + + return train, val +} + +func trainingRemoveVideoMAEGeneratedClips(root string) error { + clipsDir := filepath.Join(root, "videomae", "dataset", "clips") + if err := os.RemoveAll(clipsDir); err != nil { + return err + } + + for _, split := range []string{"train", "val"} { + if err := os.MkdirAll(filepath.Join(clipsDir, split), 0755); err != nil { + return err + } + } + + return nil +} + +func trainingVideoMAEClipFrameCount(clipDir string) int { + entries, err := os.ReadDir(clipDir) + if err != nil { + return 0 + } + + count := 0 + for _, entry := range entries { + if entry.IsDir() { + continue + } + + ext := strings.ToLower(filepath.Ext(entry.Name())) + if ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".webp" { + count++ + } + } + + return count +} + +func trainingExtractVideoMAEClipFrames( + ctx context.Context, + videoPath string, + centerSecond float64, + clipDir string, +) error { + if strings.TrimSpace(videoPath) == "" { + return errors.New("video path missing") + } + + if err := os.MkdirAll(clipDir, 0755); err != nil { + return err + } + + ffmpegPath := strings.TrimSpace(getSettings().FFmpegPath) + if ffmpegPath == "" { + ffmpegPath = "ffmpeg" + } + + start := math.Max(0, centerSecond-trainingVideoMAEClipSeconds/2) + fps := float64(trainingVideoMAENumFrames) / trainingVideoMAEClipSeconds + vf := fmt.Sprintf( + "fps=%.6f,scale=%d:%d:force_original_aspect_ratio=decrease,pad=%d:%d:(ow-iw)/2:(oh-ih)/2", + fps, + trainingVideoMAEFrameSize, + trainingVideoMAEFrameSize, + trainingVideoMAEFrameSize, + trainingVideoMAEFrameSize, + ) + + cmd := exec.CommandContext( + ctx, + ffmpegPath, + "-hide_banner", + "-loglevel", "error", + "-ss", fmt.Sprintf("%.3f", start), + "-i", videoPath, + "-t", fmt.Sprintf("%.3f", trainingVideoMAEClipSeconds), + "-vf", vf, + "-frames:v", strconv.Itoa(trainingVideoMAENumFrames), + "-q:v", "3", + filepath.Join(clipDir, "frame_%03d.jpg"), + ) + + cmd.SysProcAttr = &syscall.SysProcAttr{ + HideWindow: true, + CreationFlags: 0x08000000, + } + + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("ffmpeg clip extract failed: %w: %s", err, strings.TrimSpace(string(out))) + } + + if trainingVideoMAEClipFrameCount(clipDir) <= 0 { + return errors.New("ffmpeg erzeugte keine Clip-Frames") + } + + return nil +} + +func trainingWriteStillVideoMAEClip(framePath string, clipDir string) error { + if !fileExistsNonEmpty(framePath) { + return errors.New("fallback frame missing") + } + + if err := os.MkdirAll(clipDir, 0755); err != nil { + return err + } + + for i := 1; i <= trainingVideoMAENumFrames; i++ { + dst := filepath.Join(clipDir, fmt.Sprintf("frame_%03d.jpg", i)) + if err := copyFile(framePath, dst); err != nil { + return err + } + } + + return nil +} + +func trainingWriteVideoMAEClipForAnnotation( + ctx context.Context, + root string, + item TrainingAnnotation, + clipDir string, +) (int, error) { + if err := os.RemoveAll(clipDir); err != nil { + return 0, err + } + if err := os.MkdirAll(clipDir, 0755); err != nil { + return 0, err + } + + sourcePath := strings.TrimSpace(item.SourcePath) + if sourcePath != "" && trainingSupportedImportVideo(sourcePath) && fileExistsNonEmpty(sourcePath) { + if err := trainingExtractVideoMAEClipFrames(ctx, sourcePath, item.Second, clipDir); err == nil { + return trainingVideoMAEClipFrameCount(clipDir), nil + } else { + appLogln("videomae clip extract fallback:", item.SampleID, err) + } + } + + fallbackFrame := trainingVideoMAEFrameFallbackPath(root, item.SampleID) + if err := trainingWriteStillVideoMAEClip(fallbackFrame, clipDir); err != nil { + return 0, err + } + + return trainingVideoMAEClipFrameCount(clipDir), nil +} + +func trainingCopyVideoMAEClip(srcDir string, dstDir string) (int, error) { + if err := os.RemoveAll(dstDir); err != nil { + return 0, err + } + if err := os.MkdirAll(dstDir, 0755); err != nil { + return 0, err + } + + entries, err := os.ReadDir(srcDir) + if err != nil { + return 0, err + } + + copied := 0 + for _, entry := range entries { + if entry.IsDir() { + continue + } + + ext := strings.ToLower(filepath.Ext(entry.Name())) + if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" { + continue + } + + if err := copyFile(filepath.Join(srcDir, entry.Name()), filepath.Join(dstDir, entry.Name())); err != nil { + return copied, err + } + copied++ + } + + return copied, nil +} + +func trainingEnsureVideoMAEValidationEntries( + root string, + entries []trainingVideoMAEManifestEntry, +) []trainingVideoMAEManifestEntry { + trainCount := 0 + valCount := 0 + trainEntries := []trainingVideoMAEManifestEntry{} + + for _, entry := range entries { + switch strings.ToLower(strings.TrimSpace(entry.Split)) { + case "train": + trainCount++ + trainEntries = append(trainEntries, entry) + case "val": + valCount++ + } + } + + if valCount >= minVideoMAEValCount || trainCount < minVideoMAETrainCount { + return entries + } + + sort.SliceStable(trainEntries, func(i, j int) bool { + if trainEntries[i].Label == trainEntries[j].Label { + return trainEntries[i].SampleID < trainEntries[j].SampleID + } + return trainEntries[i].Label < trainEntries[j].Label + }) + + for _, entry := range trainEntries { + if valCount >= minVideoMAEValCount { + break + } + + copyID := entry.SampleID + "_valcopy" + dstDir := filepath.Join(root, "videomae", "dataset", "clips", "val", copyID) + frameCount, err := trainingCopyVideoMAEClip(entry.ClipDir, dstDir) + if err != nil || frameCount <= 0 { + if err != nil { + appLogln("videomae val copy failed:", entry.SampleID, err) + } + continue + } + + copyEntry := entry + copyEntry.SampleID = copyID + copyEntry.Split = "val" + copyEntry.ClipDir = dstDir + copyEntry.FrameCount = frameCount + entries = append(entries, copyEntry) + valCount++ + } + + return entries +} + +func trainingSyncVideoMAEDataset( + ctx context.Context, + root string, +) (trainCount int, valCount int, written int, err error) { + if err := trainingEnsureVideoMAEDirs(root); err != nil { + return 0, 0, 0, err + } + if err := trainingRemoveVideoMAEGeneratedClips(root); err != nil { + return 0, 0, 0, err + } + + items, err := trainingReadAnnotations(root) + if err != nil { + return 0, 0, 0, err + } + + labelSet, err := trainingVideoMAELabelSet() + if err != nil { + return 0, 0, 0, err + } + + entries := []trainingVideoMAEManifestEntry{} + + for _, item := range items { + if ctx.Err() != nil { + return 0, 0, written, ctx.Err() + } + + sampleID := strings.TrimSpace(item.SampleID) + if sampleID == "" || strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") { + continue + } + + label := trainingVideoMAELabelForAnnotation(item) + if !labelSet[label] { + continue + } + + split := trainingStableSplit(sampleID) + clipDir := filepath.Join(root, "videomae", "dataset", "clips", split, sampleID) + frameCount, clipErr := trainingWriteVideoMAEClipForAnnotation(ctx, root, item, clipDir) + if clipErr != nil { + appLogln("videomae sample sync skipped:", sampleID, clipErr) + continue + } + + entry := trainingVideoMAEManifestEntry{ + SampleID: sampleID, + Split: split, + Label: label, + ClipDir: clipDir, + SourcePath: strings.TrimSpace(item.SourcePath), + Second: item.Second, + FrameCount: frameCount, + } + entries = append(entries, entry) + written++ + } + + entries = trainingEnsureVideoMAEValidationEntries(root, entries) + + for _, entry := range entries { + switch strings.ToLower(strings.TrimSpace(entry.Split)) { + case "train": + trainCount++ + case "val": + valCount++ + } + } + + if err := trainingWriteVideoMAEManifest(root, entries); err != nil { + return trainCount, valCount, written, err + } + + return trainCount, valCount, written, nil +} diff --git a/backend/videomae_test.go b/backend/videomae_test.go new file mode 100644 index 0000000..4ce9312 --- /dev/null +++ b/backend/videomae_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestTrainingVideoMAELabelForAnnotationUsesNegativeAsKeine(t *testing.T) { + got := trainingVideoMAELabelForAnnotation(TrainingAnnotation{ + Negative: true, + Correction: &TrainingCorrection{ + SexPosition: "doggy", + }, + }) + + if got != trainingNoSexPositionLabel { + t.Fatalf("label = %q, want %q", got, trainingNoSexPositionLabel) + } +} + +func TestTrainingVideoMAELabelForAnnotationUsesCorrection(t *testing.T) { + got := trainingVideoMAELabelForAnnotation(TrainingAnnotation{ + Correction: &TrainingCorrection{ + SexPosition: "cowgirl", + }, + }) + + if got != "cowgirl" { + t.Fatalf("label = %q, want cowgirl", got) + } +} + +func TestBuildAnalyzeVideoMAEClipsUsesWindowAndStride(t *testing.T) { + samples := []videoFrameSample{ + {Time: 0, Path: "f0.jpg"}, + {Time: 1, Path: "f1.jpg"}, + {Time: 2, Path: "f2.jpg"}, + {Time: 3, Path: "f3.jpg"}, + {Time: 4, Path: "f4.jpg"}, + {Time: 5, Path: "f5.jpg"}, + } + + clips := buildAnalyzeVideoMAEClips(samples, 6) + if len(clips) != 3 { + t.Fatalf("clip count = %d, want 3", len(clips)) + } + if clips[0].Time != 0 || clips[1].Time != 2 || clips[2].Time != 4 { + t.Fatalf("clip times = %.1f, %.1f, %.1f; want 0, 2, 4", clips[0].Time, clips[1].Time, clips[2].Time) + } + if len(clips[1].Paths) < 4 { + t.Fatalf("middle clip paths = %d, want at least 4", len(clips[1].Paths)) + } +} + +func TestTrainingResolveVideoMAEModelUsesConfigDir(t *testing.T) { + root := t.TempDir() + modelDir := filepath.Join(root, "videomae", "model") + if err := os.MkdirAll(modelDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(modelDir, "config.json"), []byte("{}"), 0644); err != nil { + t.Fatal(err) + } + + got := trainingResolveVideoMAEModel(root) + if !got.EffectiveExists || got.EffectivePath != modelDir || got.Source != "videomae_clip" { + t.Fatalf("unexpected model resolution: %+v", got) + } +}