chris/nsfwapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added videomae

Browse Source
This commit is contained in:
Linrador 2026-06-22 15:22:29 +02:00
parent 9d3caeca86
commit 0a6fefa663
9 changed files with 2019 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

269
backend/ai_server.py
View File

@ -70,6 +70,15 @@ def existing_file(path: Path) -> Optional[Path]:
return None
def existing_model_dir(path: Path) -> Optional[Path]:
try:
if path.exists() and path.is_dir() and existing_file(path / "config.json"):
return path
except OSError:
pass
return None
def resolve_training_root() -> Path:
env_root = os.environ.get("TRAINING_ROOT", "").strip()
if env_root:
@ -95,6 +104,7 @@ def resolve_training_root() -> Path:
if (
existing_file(root / "detection_labels.json")
or existing_file(root / "detector" / "model" / "best.pt")
or existing_model_dir(root / "videomae" / "model")
):
root.mkdir(parents=True, exist_ok=True)
return root.resolve()
@ -108,6 +118,7 @@ def resolve_training_root() -> Path:
TRAINING_ROOT = resolve_training_root()
DEFAULT_MODEL_PATH = TRAINING_ROOT / "detector" / "model" / "best.pt"
DEFAULT_POSE_MODEL_PATH = TRAINING_ROOT / "pose" / "model" / "best.pt"
DEFAULT_VIDEOMAE_MODEL_PATH = TRAINING_ROOT / "videomae" / "model"
def resolve_detection_labels_path() -> Path:
@ -153,6 +164,20 @@ def resolve_pose_model_path() -> Optional[Path]:
return None
def resolve_videomae_model_path() -> Optional[Path]:
env_path = os.environ.get("VIDEOMAE_MODEL", "").strip()
if env_path:
p = Path(env_path).expanduser().resolve()
if existing_model_dir(p):
return p
raise RuntimeError(f"VIDEOMAE_MODEL not found: {p}")
if existing_model_dir(DEFAULT_VIDEOMAE_MODEL_PATH):
return DEFAULT_VIDEOMAE_MODEL_PATH.resolve()
return None
# Server darf auch ohne Labels/Model starten.
DETECTION_LABELS_PATH: Optional[Path] = None
@ -178,6 +203,9 @@ _MODEL_PATH = ""
_MODEL_ERROR = ""
_POSE_MODEL_PATH = ""
_POSE_MODEL_ERROR = ""
_VIDEOMAE_MODEL_PATH = ""
_VIDEOMAE_MODEL_ERROR = ""
_VIDEOMAE_DEVICE_ACTIVE = ""
_LABEL_ERROR = ""
_DEVICE = os.environ.get("YOLO_DEVICE", "")
@ -193,9 +221,13 @@ _POSITION_CONTEXT_OVERRIDE_MARGIN = float(os.environ.get("YOLO_POSITION_CONTEXT_
_BATCH = int(os.environ.get("YOLO_BATCH", "16"))
_IMGSZ = int(os.environ.get("YOLO_IMGSZ", "640"))
_HALF = os.environ.get("YOLO_HALF", "0").lower() in {"1", "true", "yes", "on"}
_VIDEOMAE_DEVICE = os.environ.get("VIDEOMAE_DEVICE", "auto")
_VIDEOMAE_NUM_FRAMES = int(os.environ.get("VIDEOMAE_NUM_FRAMES", "16"))
model = None
pose_model = None
videomae_model = None
videomae_processor = None
app = FastAPI()
@ -243,6 +275,18 @@ class PredictBatchRequest(BaseModel):
model: Optional[str] = None
class PositionClipItem(BaseModel):
time: float = 0.0
start: float = 0.0
end: float = 0.0
paths: List[str]
class PredictPositionClipsRequest(BaseModel):
clips: List[PositionClipItem]
numFrames: int = 16
def empty_prediction(source: str = "model_missing") -> dict:
return {
"modelAvailable": False,
@ -390,6 +434,155 @@ def get_pose_model():
return None
def get_videomae_components():
global videomae_model
global videomae_processor
global _VIDEOMAE_MODEL_PATH
global _VIDEOMAE_MODEL_ERROR
global _VIDEOMAE_DEVICE_ACTIVE
if videomae_model is not None and videomae_processor is not None:
return videomae_model, videomae_processor
try:
path = resolve_videomae_model_path()
if path is None:
videomae_model = None
videomae_processor = None
_VIDEOMAE_MODEL_PATH = ""
_VIDEOMAE_MODEL_ERROR = ""
_VIDEOMAE_DEVICE_ACTIVE = ""
return None, None
import torch
from transformers import AutoImageProcessor, VideoMAEForVideoClassification
if str(_VIDEOMAE_DEVICE).lower() == "auto":
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
else:
device = torch.device(_VIDEOMAE_DEVICE)
processor = AutoImageProcessor.from_pretrained(path)
loaded = VideoMAEForVideoClassification.from_pretrained(path)
loaded.to(device)
loaded.eval()
videomae_model = loaded
videomae_processor = processor
_VIDEOMAE_MODEL_PATH = str(path)
_VIDEOMAE_MODEL_ERROR = ""
_VIDEOMAE_DEVICE_ACTIVE = str(device)
return videomae_model, videomae_processor
except Exception as exc:
videomae_model = None
videomae_processor = None
_VIDEOMAE_MODEL_PATH = ""
_VIDEOMAE_DEVICE_ACTIVE = ""
_VIDEOMAE_MODEL_ERROR = str(exc)
return None, None
def resample_values(values: list, count: int) -> list:
if not values:
return []
if count <= 1:
return [values[0]]
if len(values) == 1:
return [values[0] for _ in range(count)]
last = len(values) - 1
return [
values[int(round((i * last) / max(1, count - 1)))]
for i in range(count)
]
def load_videomae_clip_frames(paths: list[str], num_frames: int):
from PIL import Image
clean_paths = [
str(path).strip()
for path in paths or []
if str(path).strip()
]
selected = resample_values(clean_paths, max(2, int(num_frames or _VIDEOMAE_NUM_FRAMES or 16)))
frames = []
for path in selected:
with Image.open(path) as img:
frames.append(img.convert("RGB").copy())
return frames
def predict_videomae_clip(clip: PositionClipItem, num_frames: int) -> dict:
current_model, current_processor = get_videomae_components()
if current_model is None or current_processor is None:
return {
"time": float(clip.time or 0.0),
"start": float(clip.start or 0.0),
"end": float(clip.end or 0.0),
"sexPosition": NO_SEX_POSITION_LABEL,
"sexPositionScore": 0.0,
"source": "videomae_missing",
"scores": [],
}
import torch
frames = load_videomae_clip_frames(clip.paths, num_frames)
if not frames:
return {
"time": float(clip.time or 0.0),
"start": float(clip.start or 0.0),
"end": float(clip.end or 0.0),
"sexPosition": NO_SEX_POSITION_LABEL,
"sexPositionScore": 0.0,
"source": "videomae_no_frames",
"scores": [],
}
inputs = current_processor(frames, return_tensors="pt")
device = next(current_model.parameters()).device
pixel_values = inputs["pixel_values"].to(device)
with torch.no_grad():
logits = current_model(pixel_values=pixel_values).logits
probs = torch.softmax(logits, dim=-1)[0].detach().cpu().tolist()
id_to_label = getattr(current_model.config, "id2label", {}) or {}
scores = []
best_label = NO_SEX_POSITION_LABEL
best_score = 0.0
for idx, score in enumerate(probs):
raw_label = id_to_label.get(idx, id_to_label.get(str(idx), idx))
label = normalize_sex_position_label(raw_label)
score = clamp01(score)
scores.append({
"label": label,
"score": score,
})
if score > best_score:
best_label = label
best_score = score
scores.sort(key=lambda item: item["score"], reverse=True)
return {
"time": float(clip.time or 0.0),
"start": float(clip.start or 0.0),
"end": float(clip.end or 0.0),
"sexPosition": best_label,
"sexPositionScore": best_score,
"source": "videomae",
"scores": scores[:10],
}
def scored(label: str, score: float) -> dict:
return {
"label": label,
@ -1383,6 +1576,27 @@ def pose_model_status() -> dict:
}
def videomae_model_status() -> dict:
try:
expected = resolve_videomae_model_path()
expected_text = str(expected) if expected else str(DEFAULT_VIDEOMAE_MODEL_PATH)
exists = expected is not None
error = _VIDEOMAE_MODEL_ERROR
except Exception as exc:
expected_text = str(DEFAULT_VIDEOMAE_MODEL_PATH)
exists = False
error = str(exc)
return {
"videoMAEModelAvailable": exists,
"videoMAEModelLoaded": videomae_model is not None,
"videoMAEModel": _VIDEOMAE_MODEL_PATH or (expected_text if exists else ""),
"videoMAEModelError": error,
"expectedVideoMAEModel": expected_text,
"videoMAEDevice": _VIDEOMAE_DEVICE_ACTIVE,
}
@app.post("/predict-batch", dependencies=[Depends(require_ai_server_auth)])
def predict_batch(req: PredictBatchRequest):
paths = [str(path).strip() for path in req.paths if str(path).strip()]
@ -1439,6 +1653,49 @@ def predict_batch(req: PredictBatchRequest):
}
@app.post("/predict-position-clips", dependencies=[Depends(require_ai_server_auth)])
def predict_position_clips(req: PredictPositionClipsRequest):
clips = [clip for clip in req.clips if clip.paths]
if not clips:
return {
"ok": True,
"available": False,
"predictions": [],
"error": "no clips supplied",
}
current_model, current_processor = get_videomae_components()
if current_model is None or current_processor is None:
return {
"ok": True,
"available": False,
"predictions": [],
"error": _VIDEOMAE_MODEL_ERROR or f"VideoMAE model not found: {DEFAULT_VIDEOMAE_MODEL_PATH}",
}
predictions = []
for clip in clips:
try:
predictions.append(predict_videomae_clip(clip, int(req.numFrames or _VIDEOMAE_NUM_FRAMES or 16)))
except Exception as exc:
predictions.append({
"time": float(clip.time or 0.0),
"start": float(clip.start or 0.0),
"end": float(clip.end or 0.0),
"sexPosition": NO_SEX_POSITION_LABEL,
"sexPositionScore": 0.0,
"source": "videomae_predict_failed",
"error": repr(exc),
"scores": [],
})
return {
"ok": True,
"available": True,
"predictions": predictions,
}
@app.get("/health", dependencies=[Depends(require_ai_server_auth)])
def health():
current_model = get_model()
@ -1459,24 +1716,35 @@ def health():
}
status_payload.update(pose_model_status())
status_payload.update(videomae_model_status())
return status_payload
@app.post("/reload", dependencies=[Depends(require_ai_server_auth)])
def reload_model():
global model
global pose_model
global videomae_model
global videomae_processor
global _MODEL_PATH
global _MODEL_ERROR
global _POSE_MODEL_PATH
global _POSE_MODEL_ERROR
global _VIDEOMAE_MODEL_PATH
global _VIDEOMAE_MODEL_ERROR
global _VIDEOMAE_DEVICE_ACTIVE
global DETECTION_LABELS_PATH
model = None
pose_model = None
videomae_model = None
videomae_processor = None
_MODEL_PATH = ""
_MODEL_ERROR = ""
_POSE_MODEL_PATH = ""
_POSE_MODEL_ERROR = ""
_VIDEOMAE_MODEL_PATH = ""
_VIDEOMAE_MODEL_ERROR = ""
_VIDEOMAE_DEVICE_ACTIVE = ""
DETECTION_LABELS_PATH = None
current_model = get_model()
@ -1497,4 +1765,5 @@ def reload_model():
}
status_payload.update(pose_model_status())
status_payload.update(videomae_model_status())
return status_payload

33
backend/analyze.go
View File

@ -106,6 +106,7 @@ type analyzePositionEvidence struct {
PersonCount int
HasPose bool
HasContext bool
HasClip bool
}
func analyzeVideoFrameFilter(intervalSeconds int) string {
@ -1683,7 +1684,11 @@ func analyzePositionEvidenceFromPrediction(
func analyzePositionEvidenceWeight(item analyzePositionEvidence) float64 {
weight := 1.0
if item.HasPose && item.HasContext {
if item.HasClip && item.HasPose {
weight = 1.28
} else if item.HasClip {
weight = 1.18
} else if item.HasPose && item.HasContext {
weight = 1.15
} else if item.HasPose {
weight = 1.0
@ -1733,6 +1738,7 @@ func buildClipPositionHitsFromEvidence(
Count int
PoseCount int
ContextCount int
ClipCount int
Start float64
End float64
Marker float64
@ -1795,6 +1801,9 @@ func buildClipPositionHitsFromEvidence(
if item.HasContext {
agg.ContextCount++
}
if item.HasClip {
agg.ClipCount++
}
if item.Time < agg.Start {
agg.Start = item.Time
}
@ -1820,12 +1829,16 @@ func buildClipPositionHitsFromEvidence(
sourceBonus := 0.0
if agg.PoseCount > 0 && agg.ContextCount > 0 {
sourceBonus = 0.04
} else if agg.ClipCount > 0 && (agg.PoseCount > 0 || agg.ContextCount > 0) {
sourceBonus = 0.04
} else if agg.ClipCount > 0 {
sourceBonus = 0.03
} else if agg.PoseCount > 0 {
sourceBonus = 0.02
}
score := clamp01(avg*(0.86+0.14*stability) + sourceBonus)
if agg.PoseCount == 0 {
if agg.PoseCount == 0 && agg.ClipCount == 0 {
score = math.Min(score, trainingPositionContextMaxScore)
}
@ -2292,6 +2305,14 @@ func analyzeVideoFromFramesForGoal(
)
}
highlightHits, positionEvidence = applyVideoMAEPositionClipsForAnalyze(
ctx,
samples,
durationSec,
highlightHits,
positionEvidence,
)
highlightHits = append(
highlightHits,
buildClipPositionHitsFromEvidence(positionEvidence, durationSec)...,
@ -2349,6 +2370,14 @@ func analyzeVideoFromFramesForGoal(
)
}
highlightHits, positionEvidence = applyVideoMAEPositionClipsForAnalyze(
ctx,
samples,
durationSec,
highlightHits,
positionEvidence,
)
highlightHits = append(
highlightHits,
buildClipPositionHitsFromEvidence(positionEvidence, durationSec)...,

271
backend/analyze_videomae.go Normal file
View File

@ -0,0 +1,271 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"os"
"strings"
"time"
)
const (
analyzeVideoMAEClipWindowSeconds = 4.0
analyzeVideoMAEClipStrideSeconds = 2.0
analyzeVideoMAEMinScore = 0.34
analyzeVideoMAERequestBatchSize = 48
)
type analyzeVideoMAEClipReqItem struct {
Time float64 `json:"time"`
Start float64 `json:"start"`
End float64 `json:"end"`
Paths []string `json:"paths"`
}
type analyzeVideoMAEClipPredictReq struct {
Clips []analyzeVideoMAEClipReqItem `json:"clips"`
NumFrames int `json:"numFrames,omitempty"`
}
type analyzeVideoMAEClipPrediction struct {
Time float64 `json:"time"`
Start float64 `json:"start"`
End float64 `json:"end"`
SexPosition string `json:"sexPosition"`
SexPositionScore float64 `json:"sexPositionScore"`
Source string `json:"source,omitempty"`
Scores []TrainingScoredLabel `json:"scores,omitempty"`
}
type analyzeVideoMAEClipPredictResp struct {
OK bool `json:"ok"`
Available bool `json:"available"`
Predictions []analyzeVideoMAEClipPrediction `json:"predictions"`
Error string `json:"error,omitempty"`
}
func analyzeVideoMAEEnabled() bool {
raw := strings.ToLower(strings.TrimSpace(os.Getenv("VIDEOMAE_ANALYZE_ENABLED")))
return raw == "" || raw == "1" || raw == "true" || raw == "yes" || raw == "on"
}
func buildAnalyzeVideoMAEClips(
samples []videoFrameSample,
duration float64,
) []analyzeVideoMAEClipReqItem {
if len(samples) == 0 || duration <= 0 {
return []analyzeVideoMAEClipReqItem{}
}
clips := []analyzeVideoMAEClipReqItem{}
halfWindow := analyzeVideoMAEClipWindowSeconds / 2
if halfWindow <= 0 {
halfWindow = 2
}
lastCenter := -math.MaxFloat64
for _, sample := range samples {
center := math.Max(0, sample.Time)
if len(clips) > 0 && center-lastCenter < analyzeVideoMAEClipStrideSeconds-0.001 {
continue
}
start := math.Max(0, center-halfWindow)
end := center + halfWindow
if duration > 0 {
end = math.Min(duration, end)
}
if end <= start {
end = math.Min(duration, start+math.Max(1, float64(analyzeVideoFrameIntervalSeconds)))
}
paths := []string{}
for _, candidate := range samples {
if candidate.Time < start-0.001 || candidate.Time > end+0.001 {
continue
}
path := strings.TrimSpace(candidate.Path)
if path != "" {
paths = append(paths, path)
}
}
if len(paths) == 0 {
continue
}
clips = append(clips, analyzeVideoMAEClipReqItem{
Time: center,
Start: start,
End: end,
Paths: paths,
})
lastCenter = center
}
return clips
}
func predictVideoMAEPositionClipsForAnalyze(
ctx context.Context,
clips []analyzeVideoMAEClipReqItem,
) ([]analyzeVideoMAEClipPrediction, error) {
if len(clips) == 0 {
return []analyzeVideoMAEClipPrediction{}, nil
}
if !trainingRecognitionEnabled() {
return []analyzeVideoMAEClipPrediction{}, nil
}
out := []analyzeVideoMAEClipPrediction{}
for start := 0; start < len(clips); start += analyzeVideoMAERequestBatchSize {
end := start + analyzeVideoMAERequestBatchSize
if end > len(clips) {
end = len(clips)
}
payload := analyzeVideoMAEClipPredictReq{
Clips: clips[start:end],
NumFrames: trainingVideoMAENumFrames,
}
body, err := json.Marshal(payload)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
analyzeAIServerURL()+"/predict-position-clips",
bytes.NewReader(body),
)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
addAIServerAuth(req)
client := &http.Client{
Timeout: 180 * time.Second,
}
res, err := client.Do(req)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return nil, err
}
rawBody, readErr := io.ReadAll(res.Body)
_ = res.Body.Close()
if readErr != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return nil, readErr
}
var parsed analyzeVideoMAEClipPredictResp
if err := json.Unmarshal(rawBody, &parsed); err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return nil, fmt.Errorf("AI server VideoMAE JSON ungueltig: HTTP %d: %s", res.StatusCode, strings.TrimSpace(string(rawBody)))
}
if res.StatusCode < 200 || res.StatusCode >= 300 || !parsed.OK {
msg := strings.TrimSpace(parsed.Error)
if msg == "" {
msg = fmt.Sprintf("AI server VideoMAE HTTP %d", res.StatusCode)
}
return nil, fmt.Errorf("%s", msg)
}
if !parsed.Available {
return out, nil
}
out = append(out, parsed.Predictions...)
}
return out, nil
}
func applyVideoMAEPositionClipsForAnalyze(
ctx context.Context,
samples []videoFrameSample,
duration float64,
highlightHits []analyzeHit,
positionEvidence []analyzePositionEvidence,
) ([]analyzeHit, []analyzePositionEvidence) {
if !analyzeVideoMAEEnabled() {
return highlightHits, positionEvidence
}
clips := buildAnalyzeVideoMAEClips(samples, duration)
if len(clips) == 0 {
return highlightHits, positionEvidence
}
predictions, err := predictVideoMAEPositionClipsForAnalyze(ctx, clips)
if err != nil {
if ctx.Err() == nil {
appLogln("VideoMAE Clip-Analyse uebersprungen:", err)
}
return highlightHits, positionEvidence
}
for _, pred := range predictions {
label := strings.ToLower(strings.TrimSpace(pred.SexPosition))
if isNoSexPositionLabel(label) || !isKnownPositionLabel(label) {
continue
}
score := clamp01(pred.SexPositionScore)
if score < analyzeVideoMAEMinScore {
continue
}
start := math.Max(0, pred.Start)
end := pred.End
if duration > 0 {
end = math.Min(duration, end)
}
if end <= start {
end = math.Min(duration, start+math.Max(1, float64(analyzeVideoFrameIntervalSeconds)))
}
source := strings.ToLower(strings.TrimSpace(pred.Source))
if source == "" {
source = "videomae"
}
highlightHits = append(highlightHits, analyzeHit{
Time: pred.Time,
Label: "position:" + label,
Score: score,
Start: start,
End: end,
})
positionEvidence = append(positionEvidence, analyzePositionEvidence{
Time: pred.Time,
Label: label,
Score: score,
Source: source,
HasClip: true,
})
}
return highlightHits, positionEvidence
}

2
backend/embed.go
View File

@ -32,6 +32,8 @@ func trainingEmbeddedMLDir() (string, error) {
"detection_labels.json",
"predict_pose_model.py",
"train_pose_model.py",
"predict_videomae_model.py",
"train_videomae_model.py",
}
// Falls du die alten Scene-Skripte noch embedded hast, kannst du sie optional mitkopieren.

174
backend/ml/predict_videomae_model.py Normal file
View File

@ -0,0 +1,174 @@
import argparse
import json
from pathlib import Path
import torch
from PIL import Image
from transformers import AutoImageProcessor, VideoMAEForVideoClassification
def clamp01(value):
try:
n = float(value)
except Exception:
return 0.0
return max(0.0, min(1.0, n))
def existing_model_dir(path: Path):
try:
if path.exists() and path.is_dir() and (path / "config.json").exists():
return path
except Exception:
pass
return None
def resolve_model_path(root: Path, requested: str):
requested = str(requested or "").strip()
if requested:
p = existing_model_dir(Path(requested).expanduser().resolve())
if p:
return p, "videomae_model"
return Path(requested).expanduser(), "videomae_missing"
trained = root / "videomae" / "model"
p = existing_model_dir(trained)
if p:
return p, "videomae_clip"
return trained, "videomae_missing"
def resample_values(values: list, count: int) -> list:
if not values:
return []
if len(values) == 1:
return [values[0] for _ in range(count)]
if count <= 1:
return [values[0]]
last = len(values) - 1
return [values[int(round((i * last) / max(1, count - 1)))] for i in range(count)]
def load_frames(paths: list[str], num_frames: int):
selected = resample_values(paths, num_frames)
frames = []
for path in selected:
with Image.open(path) as img:
frames.append(img.convert("RGB").copy())
return frames
def frame_paths_from_args(args):
paths = []
if args.frames_json:
with Path(args.frames_json).open("r", encoding="utf-8") as f:
data = json.load(f)
paths.extend(str(p) for p in data)
if args.clip_dir:
clip_dir = Path(args.clip_dir)
paths.extend(
str(p) for p in sorted(clip_dir.iterdir())
if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp"}
)
paths.extend(str(p) for p in args.frames)
return [p for p in paths if p.strip()]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--root", required=True)
parser.add_argument("--model", default="")
parser.add_argument("--clip-dir", default="")
parser.add_argument("--frames-json", default="")
parser.add_argument("--frames", nargs="*", default=[])
parser.add_argument("--num-frames", type=int, default=16)
parser.add_argument("--device", default="auto")
args = parser.parse_args()
root = Path(args.root).resolve()
model_path, model_source = resolve_model_path(root, args.model)
if not existing_model_dir(model_path):
print(json.dumps({
"available": False,
"source": model_source,
"modelPath": str(model_path),
"sexPosition": "keine",
"sexPositionScore": 0.0,
"scores": [],
}, ensure_ascii=False))
return
paths = frame_paths_from_args(args)
if not paths:
print(json.dumps({
"available": False,
"source": "videomae_no_frames",
"modelPath": str(model_path),
"sexPosition": "keine",
"sexPositionScore": 0.0,
"scores": [],
}, ensure_ascii=False))
return
if str(args.device).lower() == "auto":
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
else:
device = torch.device(args.device)
try:
processor = AutoImageProcessor.from_pretrained(model_path)
model = VideoMAEForVideoClassification.from_pretrained(model_path).to(device)
model.eval()
frames = load_frames(paths, max(2, int(args.num_frames or 16)))
inputs = processor(frames, return_tensors="pt")
pixel_values = inputs["pixel_values"].to(device)
with torch.no_grad():
logits = model(pixel_values=pixel_values).logits
probs = torch.softmax(logits, dim=-1)[0].detach().cpu().tolist()
id_to_label = getattr(model.config, "id2label", {}) or {}
scores = []
best_label = "keine"
best_score = 0.0
for idx, score in enumerate(probs):
label = str(id_to_label.get(idx, id_to_label.get(str(idx), idx))).strip()
score = clamp01(score)
scores.append({"label": label, "score": score})
if score > best_score:
best_label = label
best_score = score
scores.sort(key=lambda item: item["score"], reverse=True)
print(json.dumps({
"available": True,
"source": model_source,
"modelPath": str(model_path),
"device": str(device),
"sexPosition": best_label,
"sexPositionScore": best_score,
"scores": scores[:10],
}, ensure_ascii=False))
except Exception as exc:
print(json.dumps({
"available": False,
"source": "videomae_predict_failed",
"modelPath": str(model_path),
"error": repr(exc),
"sexPosition": "keine",
"sexPositionScore": 0.0,
"scores": [],
}, ensure_ascii=False))
if __name__ == "__main__":
main()

397
backend/ml/train_videomae_model.py Normal file
View File

@ -0,0 +1,397 @@
import argparse
import json
import math
import shutil
from datetime import datetime, timezone
from pathlib import Path
import torch
from PIL import Image
from torch.utils.data import DataLoader, Dataset
from transformers import AutoImageProcessor, VideoMAEForVideoClassification
DEFAULT_BASE_MODEL = "MCG-NJU/videomae-base-finetuned-kinetics"
def emit_progress(stage, progress, message="", **extra):
out = {
"type": "progress",
"stage": stage,
"progress": max(0.0, min(1.0, float(progress))),
"message": message,
}
out.update(extra)
print(json.dumps(out, ensure_ascii=False), flush=True)
def safe_int(value, fallback):
try:
return int(value)
except Exception:
return fallback
def safe_float(value, fallback):
try:
return float(value)
except Exception:
return fallback
def load_labels(dataset_root: Path) -> list[str]:
path = dataset_root / "labels.json"
if not path.exists():
raise SystemExit(f"labels.json not found: {path}")
with path.open("r", encoding="utf-8") as f:
data = json.load(f)
labels = data.get("labels", [])
out = []
seen = set()
for value in labels:
label = str(value or "").strip()
if not label or label in seen:
continue
seen.add(label)
out.append(label)
if len(out) < 2:
raise SystemExit("VideoMAE needs at least two labels")
return out
def load_manifest(dataset_root: Path, label_to_id: dict[str, int]) -> list[dict]:
path = dataset_root / "manifest.jsonl"
if not path.exists():
raise SystemExit(f"manifest.jsonl not found: {path}")
entries = []
with path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
item = json.loads(line)
except Exception:
continue
label = str(item.get("label") or "").strip()
clip_dir = Path(str(item.get("clipDir") or "")).expanduser()
split = str(item.get("split") or "").strip().lower()
if label not in label_to_id or split not in {"train", "val"}:
continue
if not clip_dir.exists() or not clip_dir.is_dir():
continue
frames = sorted(
p for p in clip_dir.iterdir()
if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp"}
)
if not frames:
continue
item["label"] = label
item["split"] = split
item["clipDir"] = str(clip_dir)
item["frames"] = [str(p) for p in frames]
entries.append(item)
return entries
def resample_values(values: list, count: int) -> list:
if not values:
return []
if count <= 1:
return [values[0]]
if len(values) == 1:
return [values[0] for _ in range(count)]
out = []
last = len(values) - 1
for i in range(count):
idx = int(round((i * last) / max(1, count - 1)))
out.append(values[idx])
return out
def load_clip_frames(paths: list[str], num_frames: int) -> list[Image.Image]:
selected = resample_values(paths, num_frames)
frames = []
for path in selected:
with Image.open(path) as img:
frames.append(img.convert("RGB").copy())
return frames
class VideoMAEClipDataset(Dataset):
def __init__(self, entries, label_to_id, image_processor, num_frames):
self.entries = entries
self.label_to_id = label_to_id
self.image_processor = image_processor
self.num_frames = num_frames
def __len__(self):
return len(self.entries)
def __getitem__(self, idx):
item = self.entries[idx]
frames = load_clip_frames(item["frames"], self.num_frames)
inputs = self.image_processor(frames, return_tensors="pt")
pixel_values = inputs["pixel_values"].squeeze(0)
label_id = self.label_to_id[item["label"]]
return {
"pixel_values": pixel_values,
"labels": torch.tensor(label_id, dtype=torch.long),
"sample_id": str(item.get("sampleId") or ""),
}
def collate_batch(batch):
return {
"pixel_values": torch.stack([item["pixel_values"] for item in batch]),
"labels": torch.stack([item["labels"] for item in batch]),
"sample_ids": [item["sample_id"] for item in batch],
}
def evaluate(model, loader, device):
model.eval()
total = 0
correct = 0
loss_sum = 0.0
with torch.no_grad():
for batch in loader:
pixel_values = batch["pixel_values"].to(device)
labels = batch["labels"].to(device)
outputs = model(pixel_values=pixel_values, labels=labels)
logits = outputs.logits
loss = outputs.loss
preds = logits.argmax(dim=-1)
total += int(labels.numel())
correct += int((preds == labels).sum().item())
loss_sum += float(loss.item()) * int(labels.numel())
if total <= 0:
return 0.0, 0.0
return correct / total, loss_sum / total
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--root", required=True)
parser.add_argument("--base", default=DEFAULT_BASE_MODEL)
parser.add_argument("--epochs", default="8")
parser.add_argument("--batch-size", default="2")
parser.add_argument("--lr", default="5e-5")
parser.add_argument("--device", default="auto")
parser.add_argument("--workers", default="0")
parser.add_argument("--num-frames", default="16")
parser.add_argument("--freeze-backbone", action="store_true")
args = parser.parse_args()
root = Path(args.root).resolve()
dataset_root = root / "videomae" / "dataset"
out_dir = root / "videomae" / "model"
tmp_dir = root / "videomae" / "runs" / "model_tmp"
epochs = max(1, safe_int(args.epochs, 8))
batch_size = max(1, safe_int(args.batch_size, 2))
workers = max(0, safe_int(args.workers, 0))
num_frames = max(2, safe_int(args.num_frames, 16))
lr = max(1e-7, safe_float(args.lr, 5e-5))
if str(args.device).lower() == "auto":
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
else:
device = torch.device(args.device)
labels = load_labels(dataset_root)
label_to_id = {label: i for i, label in enumerate(labels)}
id_to_label = {i: label for label, i in label_to_id.items()}
entries = load_manifest(dataset_root, label_to_id)
train_entries = [item for item in entries if item["split"] == "train"]
val_entries = [item for item in entries if item["split"] == "val"]
emit_progress(
"videomae",
0.01,
"VideoMAE-Dataset wird geprueft...",
trainSamples=len(train_entries),
valSamples=len(val_entries),
epochs=epochs,
device=str(device),
)
if not train_entries:
raise SystemExit("no VideoMAE train clips found")
if not val_entries:
raise SystemExit("no VideoMAE val clips found")
emit_progress(
"videomae",
0.03,
"VideoMAE-Basismodell wird geladen...",
base=args.base,
labels=len(labels),
device=str(device),
)
image_processor = AutoImageProcessor.from_pretrained(args.base)
model = VideoMAEForVideoClassification.from_pretrained(
args.base,
num_labels=len(labels),
label2id=label_to_id,
id2label=id_to_label,
ignore_mismatched_sizes=True,
)
if args.freeze_backbone and hasattr(model, "videomae"):
for param in model.videomae.parameters():
param.requires_grad = False
model.to(device)
train_ds = VideoMAEClipDataset(train_entries, label_to_id, image_processor, num_frames)
val_ds = VideoMAEClipDataset(val_entries, label_to_id, image_processor, num_frames)
train_loader = DataLoader(
train_ds,
batch_size=batch_size,
shuffle=True,
num_workers=workers,
collate_fn=collate_batch,
)
val_loader = DataLoader(
val_ds,
batch_size=batch_size,
shuffle=False,
num_workers=workers,
collate_fn=collate_batch,
)
optimizer = torch.optim.AdamW(
[p for p in model.parameters() if p.requires_grad],
lr=lr,
)
if tmp_dir.exists():
shutil.rmtree(tmp_dir)
tmp_dir.mkdir(parents=True, exist_ok=True)
best_accuracy = -1.0
best_loss = math.inf
best_epoch = 0
for epoch in range(1, epochs + 1):
model.train()
running_loss = 0.0
seen = 0
total_batches = max(1, len(train_loader))
for batch_idx, batch in enumerate(train_loader, start=1):
pixel_values = batch["pixel_values"].to(device)
labels_tensor = batch["labels"].to(device)
optimizer.zero_grad(set_to_none=True)
outputs = model(pixel_values=pixel_values, labels=labels_tensor)
loss = outputs.loss
loss.backward()
optimizer.step()
batch_size_seen = int(labels_tensor.numel())
seen += batch_size_seen
running_loss += float(loss.item()) * batch_size_seen
completed = (epoch - 1) + min(1.0, batch_idx / total_batches)
emit_progress(
"videomae",
0.04 + 0.84 * (completed / max(1, epochs)),
"VideoMAE trainiert...",
epoch=epoch,
epochs=epochs,
sampleId=(batch["sample_ids"][0] if batch["sample_ids"] else ""),
trainSamples=len(train_entries),
valSamples=len(val_entries),
device=str(device),
loss=(running_loss / max(1, seen)),
)
val_accuracy, val_loss = evaluate(model, val_loader, device)
is_best = val_accuracy > best_accuracy or (
math.isclose(val_accuracy, best_accuracy) and val_loss < best_loss
)
if is_best:
best_accuracy = val_accuracy
best_loss = val_loss
best_epoch = epoch
model.save_pretrained(tmp_dir)
image_processor.save_pretrained(tmp_dir)
emit_progress(
"videomae",
0.88 + 0.08 * (epoch / max(1, epochs)),
"VideoMAE wird validiert...",
epoch=epoch,
epochs=epochs,
trainSamples=len(train_entries),
valSamples=len(val_entries),
device=str(device),
accuracy=val_accuracy,
loss=val_loss,
)
if best_epoch <= 0:
model.save_pretrained(tmp_dir)
image_processor.save_pretrained(tmp_dir)
best_epoch = epochs
status = {
"trainedAt": datetime.now(timezone.utc).isoformat(),
"epochs": epochs,
"bestEpoch": best_epoch,
"trainSamples": len(train_entries),
"valSamples": len(val_entries),
"numFrames": num_frames,
"baseModel": args.base,
"device": str(device),
"accuracy": best_accuracy if best_accuracy >= 0 else 0.0,
"loss": best_loss if math.isfinite(best_loss) else 0.0,
"labels": labels,
}
with (tmp_dir / "status.json").open("w", encoding="utf-8") as f:
json.dump(status, f, ensure_ascii=False, indent=2)
if out_dir.exists():
shutil.rmtree(out_dir)
shutil.copytree(tmp_dir, out_dir)
emit_progress(
"videomae",
1.0,
"VideoMAE-Training abgeschlossen.",
epoch=best_epoch,
epochs=epochs,
trainSamples=len(train_entries),
valSamples=len(val_entries),
device=str(device),
accuracy=status["accuracy"],
loss=status["loss"],
)
if __name__ == "__main__":
main()

205
backend/training.go
View File

@ -229,6 +229,8 @@ type TrainingStatsResponse struct {
DetectorModelInfo *TrainingModelInfo `json:"detectorModelInfo,omitempty"`
PoseModelAvailable bool `json:"poseModelAvailable"`
PoseModelInfo *TrainingModelInfo `json:"poseModelInfo,omitempty"`
VideoMAEModelAvailable bool `json:"videoMAEModelAvailable"`
VideoMAEModelInfo *TrainingModelInfo `json:"videoMAEModelInfo,omitempty"`
Confidence TrainingConfidence `json:"confidence"`
Labels TrainingStatsLabels `json:"labels"`
}
@ -2377,6 +2379,10 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
trainingWriteError(w, http.StatusInternalServerError, err.Error())
return
}
if err := trainingEnsureVideoMAEDirs(root); err != nil {
trainingWriteError(w, http.StatusInternalServerError, err.Error())
return
}
if err := trainingEnsureDetectorValidationSample(root); err != nil {
appLogln("⚠️ detector val sample ensure failed:", err)
@ -2412,6 +2418,7 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
poseValImages := filepath.Join(root, "pose", "dataset", "images", "val")
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
poseDatasetYAML := filepath.Join(root, "pose", "dataset", "dataset.yaml")
videoMAEManifest := trainingVideoMAEManifestPath(root)
trainCount := trainingCountDetectorSamples(detectorTrainImages, detectorTrainLabels)
valCount := trainingCountDetectorSamples(detectorValImages, detectorValLabels)
@ -2419,6 +2426,23 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
positiveValCount := trainingCountPositiveDetectorSamples(detectorValImages, detectorValLabels)
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
poseValCount := trainingCountDetectorSamples(poseValImages, poseValLabels)
videoMAETrainCount, videoMAEValCount := trainingCountVideoMAEManifestSamples(root)
videoMAEEligibleCount, _ := trainingCountVideoMAEEligibleAnnotations(root)
detectorDataReady := fileExistsNonEmpty(detectorDatasetYAML) &&
trainCount >= minDetectorTrainCount &&
valCount >= minDetectorValCount &&
positiveTrainCount > 0 &&
positiveValCount > 0
poseDataReady := fileExistsNonEmpty(poseDatasetYAML) &&
poseTrainCount >= minPoseTrainCount &&
poseValCount >= minPoseValCount
videoMAEDataReady := videoMAEEligibleCount >= minVideoMAETrainCount ||
(videoMAETrainCount >= minVideoMAETrainCount && videoMAEValCount >= minVideoMAEValCount)
if detectorDataReady || poseDataReady || videoMAEDataReady {
goto startTraining
}
if !fileExistsNonEmpty(detectorDatasetYAML) ||
trainCount < minDetectorTrainCount ||
@ -2458,6 +2482,7 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
return
}
startTraining:
ctx, cancel := context.WithCancel(context.Background())
trainingStartJob(cancel)
@ -2489,6 +2514,15 @@ func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
"datasetYAML": poseDatasetYAML,
"source": "yolo26_pose",
},
"videomae": map[string]any{
"eligibleCount": videoMAEEligibleCount,
"trainCount": videoMAETrainCount,
"valCount": videoMAEValCount,
"requiredTrain": minVideoMAETrainCount,
"requiredVal": minVideoMAEValCount,
"manifest": videoMAEManifest,
"source": "videomae_clip",
},
})
}
@ -2549,6 +2583,8 @@ func trainingRunJob(ctx context.Context, root string, count int) {
detectorStatus := "skipped"
poseOutput := ""
poseStatus := "skipped"
videoMAEOutput := ""
videoMAEStatus := "skipped"
detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml")
detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train")
@ -2702,7 +2738,7 @@ func trainingRunJob(ctx context.Context, root string, count int) {
return trainingHandleProgressLine(
line,
62,
98,
82,
"YOLO26 Pose wird trainiert...",
)
},
@ -2749,6 +2785,104 @@ func trainingRunJob(ctx context.Context, root string, count int) {
poseOutputClean := cleanOutput(poseOutput)
trainingSetJobStatus(func(s *TrainingJobStatus) {
if s.Progress < 84 {
s.Progress = 84
}
s.Step = "VideoMAE Clip-Daten werden aufgebaut..."
})
videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr :=
trainingSyncVideoMAEDataset(ctx, root)
if errors.Is(videoMAESyncErr, context.Canceled) || errors.Is(videoMAESyncErr, errTrainingCancelled) {
appLogln("VideoMAE dataset sync cancelled")
trainingFinishCancelled(root)
return
}
if videoMAESyncErr != nil {
videoMAEStatus = "failed"
videoMAEOutput = "VideoMAE-Dataset konnte nicht aufgebaut werden: " + videoMAESyncErr.Error()
appLogln(videoMAEOutput)
} else {
appLogf(
"VideoMAE samples synced: written=%d train=%d val=%d",
videoMAEWritten,
videoMAETrainCount,
videoMAEValCount,
)
}
if videoMAEStatus != "failed" &&
videoMAETrainCount >= minVideoMAETrainCount &&
videoMAEValCount >= minVideoMAEValCount {
trainingSetJobStatus(func(s *TrainingJobStatus) {
if s.Progress < 86 {
s.Progress = 86
}
s.Step = "VideoMAE Clip-Classifier wird trainiert..."
})
videoMAEScript := trainingScriptPath("train_videomae_model.py")
videoMAEArgs := []string{
"--root", root,
"--epochs", strconv.Itoa(trainingVideoMAEEpochs()),
"--batch-size", strconv.Itoa(trainingVideoMAEBatchSize()),
"--num-frames", strconv.Itoa(trainingVideoMAENumFrames),
}
if base := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL")); base != "" {
videoMAEArgs = append(videoMAEArgs, "--base", base)
}
videoMAEOut, videoMAEErr := trainingRunCommandStreaming(
ctx,
python,
videoMAEScript,
func(line string) bool {
return trainingHandleProgressLine(
line,
86,
98,
"VideoMAE Clip-Classifier wird trainiert...",
)
},
videoMAEArgs...,
)
if errors.Is(videoMAEErr, errTrainingCancelled) {
appLogln("VideoMAE training cancelled")
trainingFinishCancelled(root)
return
}
videoMAEOutput = videoMAEOut
videoMAEOutputClean := cleanOutput(videoMAEOutput)
if videoMAEErr != nil {
videoMAEStatus = "failed"
appLogln("VideoMAE training failed:", videoMAEErr)
if videoMAEOutputClean != "" {
appLogln("VideoMAE output:", videoMAEOutputClean)
}
} else {
videoMAEStatus = "trained"
if videoMAEOutputClean != "" {
appLogln("VideoMAE training:", videoMAEOutputClean)
}
}
} else if videoMAEStatus != "failed" {
videoMAEStatus = "skipped_no_videomae_data"
videoMAEOutput = fmt.Sprintf(
"VideoMAE uebersprungen: zu wenige Clip-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
videoMAETrainCount,
videoMAEValCount,
minVideoMAETrainCount,
minVideoMAEValCount,
)
appLogln(videoMAEOutput)
}
videoMAEOutputClean := cleanOutput(videoMAEOutput)
message := "Training abgeschlossen."
errorText := ""
@ -2791,7 +2925,23 @@ func trainingRunJob(ctx context.Context, root string, count int) {
errorText = message
}
if detectorStatus == "trained" {
if videoMAEStatus == "trained" {
if !strings.Contains(message, "Training abgeschlossen.") {
message = "Training abgeschlossen. " + message
}
message += " VideoMAE wurde trainiert."
} else if videoMAEStatus == "skipped_no_videomae_data" {
if detectorStatus == "trained" || poseStatus == "trained" {
message += " VideoMAE wurde uebersprungen: zu wenige Clip-Beispiele."
}
} else if videoMAEStatus == "failed" {
message += " VideoMAE ist fehlgeschlagen."
if videoMAEOutputClean != "" {
message += " Grund: " + videoMAEOutputClean
}
}
if detectorStatus == "trained" || poseStatus == "trained" || videoMAEStatus == "trained" {
// Verlaufseintrag schreiben, solange die Job-Startzeit für die Dauer noch verfügbar ist.
trainingAppendRunHistory(root)
}
@ -3109,6 +3259,10 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
trainingWriteError(w, http.StatusInternalServerError, err.Error())
return
}
if err := trainingEnsureVideoMAEDirs(root); err != nil {
trainingWriteError(w, http.StatusInternalServerError, err.Error())
return
}
if err := trainingEnsureDetectorValidationSample(root); err != nil {
appLogln("⚠️ detector val sample ensure failed:", err)
@ -3133,6 +3287,8 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
detectorModel := trainingResolveDetectorModel(root)
poseModel := trainingResolvePoseModel(root)
videoMAEModel := trainingResolveVideoMAEModel(root)
videoMAEManifest := trainingVideoMAEManifestPath(root)
trainCount := trainingCountDetectorSamples(detectorTrainImages, detectorTrainLabels)
valCount := trainingCountDetectorSamples(detectorValImages, detectorValLabels)
@ -3140,6 +3296,8 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
positiveValCount := trainingCountPositiveDetectorSamples(detectorValImages, detectorValLabels)
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
poseValCount := trainingCountDetectorSamples(poseValImages, poseValLabels)
videoMAETrainCount, videoMAEValCount := trainingCountVideoMAEManifestSamples(root)
videoMAEEligibleCount, _ := trainingCountVideoMAEEligibleAnnotations(root)
datasetReady := fileExistsNonEmpty(detectorDatasetYAML)
detectorDataReady := datasetReady &&
@ -3151,8 +3309,13 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
poseDataReady := poseDatasetReady &&
poseTrainCount >= minPoseTrainCount &&
poseValCount >= minPoseValCount
videoMAEDatasetReady := fileExistsNonEmpty(videoMAEManifest)
videoMAEDataReady := (videoMAETrainCount >= minVideoMAETrainCount &&
videoMAEValCount >= minVideoMAEValCount) ||
videoMAEEligibleCount >= minVideoMAETrainCount
canTrain := feedbackCount >= minTrainingFeedbackCount && detectorDataReady && poseDataReady
canTrain := feedbackCount >= minTrainingFeedbackCount &&
(detectorDataReady || poseDataReady || videoMAEDataReady)
trainingWriteJSON(w, http.StatusOK, map[string]any{
"ok": true,
@ -3215,13 +3378,14 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
},
"scene": map[string]any{
"source": "disabled",
"source": "videomae_clip",
"usesVideoMAE": true,
"usesSceneCLIP": false,
"usesSceneKNN": false,
"usesResNet18KNN": false,
"usesLogisticRegression": false,
"predictsSexPosition": false,
"predictsSexPosition": videoMAEModel.TrainedExists,
"predictsPeople": false,
"predictsGender": false,
"predictsBodyParts": false,
@ -3230,17 +3394,27 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
"predictsBoxes": false,
"feedbackCount": feedbackCount,
"requiredCount": minTrainingFeedbackCount,
"dataReady": false,
"modelReady": false,
"eligibleCount": videoMAEEligibleCount,
"trainCount": videoMAETrainCount,
"valCount": videoMAEValCount,
"requiredTrain": minVideoMAETrainCount,
"requiredVal": minVideoMAEValCount,
"requiredCount": minVideoMAETrainCount,
"datasetReady": videoMAEDatasetReady,
"manifest": videoMAEManifest,
"dataReady": videoMAEDataReady,
"modelReady": videoMAEModel.EffectiveExists,
"modelExists": videoMAEModel.EffectiveExists,
"modelPath": videoMAEModel.EffectivePath,
"modelSource": videoMAEModel.Source,
},
"pipeline": map[string]any{
"variant": "YOLO26_ONLY",
"variant": "YOLO26_VIDEO_CLIP_HYBRID",
"peopleSource": "yolo26_detector",
"genderSource": "yolo26_detector",
"sexPositionSource": "yolo26_pose",
"sexPositionSource": "yolo26_pose+box_context+videomae_clip",
"bodyPartsSource": "yolo26_detector",
"objectsSource": "yolo26_detector",
"clothingSource": "yolo26_detector",
@ -3249,6 +3423,7 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
"usesSceneKNNForDetection": false,
"usesSceneCLIP": false,
"usesSceneKNN": false,
"usesVideoMAE": true,
"usesYOLOForDetection": true,
"usesYOLOForSexPosition": true,
},
@ -3260,6 +3435,8 @@ func trainingApplyStatsModelInfo(root string, stats *TrainingStatsResponse) {
detectorInfo := trainingReadModelInfoFor(root, "detector")
poseAvailable := trainingStatsModelAvailableFor(root, "pose")
poseInfo := trainingReadModelInfoFor(root, "pose")
videoMAEModel := trainingResolveVideoMAEModel(root)
videoMAEInfo := trainingReadModelInfoFor(root, "videomae")
// modelAvailable/modelInfo bleiben aus Kompatibilitaetsgruenden der Detector.
stats.ModelAvailable = detectorAvailable
@ -3268,6 +3445,8 @@ func trainingApplyStatsModelInfo(root string, stats *TrainingStatsResponse) {
stats.DetectorModelInfo = detectorInfo
stats.PoseModelAvailable = poseAvailable
stats.PoseModelInfo = poseInfo
stats.VideoMAEModelAvailable = videoMAEModel.EffectiveExists
stats.VideoMAEModelInfo = videoMAEInfo
}
func trainingStatsModelAvailable(root string) bool {
@ -3276,6 +3455,9 @@ func trainingStatsModelAvailable(root string) bool {
func trainingStatsModelAvailableFor(root string, kind string) bool {
modelPath := filepath.Join(root, kind, "model", "best.pt")
if kind == "videomae" {
modelPath = filepath.Join(root, kind, "model", "config.json")
}
return fileExistsNonEmpty(modelPath)
}
@ -3288,6 +3470,9 @@ func trainingReadModelInfo(root string) *TrainingModelInfo {
func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo {
modelPath := filepath.Join(root, kind, "model", "best.pt")
if kind == "videomae" {
modelPath = filepath.Join(root, kind, "model", "config.json")
}
fi, err := os.Stat(modelPath)
if err != nil || fi.IsDir() || fi.Size() <= 0 {

610
backend/training_videomae.go Normal file
View File

@ -0,0 +1,610 @@
package main
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
)
const (
trainingVideoMAEClipSeconds = 4.0
trainingVideoMAENumFrames = 16
trainingVideoMAEFrameSize = 224
minVideoMAETrainCount = 5
minVideoMAEValCount = 1
)
func trainingVideoMAEEpochs() int {
raw := strings.TrimSpace(os.Getenv("TRAINING_VIDEOMAE_EPOCHS"))
if raw == "" {
return 8
}
n, err := strconv.Atoi(raw)
if err != nil || n < 1 {
return 8
}
if n > 200 {
return 200
}
return n
}
func trainingVideoMAEBatchSize() int {
raw := strings.TrimSpace(os.Getenv("TRAINING_VIDEOMAE_BATCH"))
if raw == "" {
return 2
}
n, err := strconv.Atoi(raw)
if err != nil || n < 1 {
return 2
}
if n > 32 {
return 32
}
return n
}
type trainingVideoMAEManifestEntry struct {
SampleID string `json:"sampleId"`
Split string `json:"split"`
Label string `json:"label"`
ClipDir string `json:"clipDir"`
SourcePath string `json:"sourcePath,omitempty"`
Second float64 `json:"second,omitempty"`
FrameCount int `json:"frameCount"`
}
func trainingResolveVideoMAEModel(root string) trainingModelResolution {
modelDir := filepath.Join(root, "videomae", "model")
configPath := filepath.Join(modelDir, "config.json")
if fileExistsNonEmpty(configPath) {
return trainingModelResolution{
BestPath: modelDir,
EffectivePath: modelDir,
Source: "videomae_clip",
TrainedExists: true,
EffectiveExists: true,
}
}
return trainingModelResolution{
BestPath: modelDir,
EffectivePath: modelDir,
Source: "videomae_missing",
TrainedExists: false,
EffectiveExists: false,
}
}
func trainingEnsureVideoMAEDirs(root string) error {
dirs := []string{
filepath.Join(root, "videomae"),
filepath.Join(root, "videomae", "dataset"),
filepath.Join(root, "videomae", "dataset", "clips"),
filepath.Join(root, "videomae", "dataset", "clips", "train"),
filepath.Join(root, "videomae", "dataset", "clips", "val"),
filepath.Join(root, "videomae", "model"),
filepath.Join(root, "videomae", "runs"),
}
for _, dir := range dirs {
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
}
return trainingWriteVideoMAELabelsFile(root)
}
func trainingVideoMAELabels() ([]string, error) {
grouped, err := trainingGroupedLabels()
if err != nil {
return nil, err
}
out := []string{trainingNoSexPositionLabel}
seen := map[string]bool{
trainingNoSexPositionLabel: true,
}
for _, value := range grouped.SexPositions {
label := normalizeSexPositionLabel(value)
if seen[label] {
continue
}
seen[label] = true
out = append(out, label)
}
return out, nil
}
func trainingVideoMAELabelSet() (map[string]bool, error) {
labels, err := trainingVideoMAELabels()
if err != nil {
return nil, err
}
out := map[string]bool{}
for _, label := range labels {
out[label] = true
}
return out, nil
}
func trainingWriteVideoMAELabelsFile(root string) error {
labels, err := trainingVideoMAELabels()
if err != nil {
return err
}
body, err := json.MarshalIndent(map[string]any{
"labels": labels,
}, "", " ")
if err != nil {
return err
}
return os.WriteFile(filepath.Join(root, "videomae", "dataset", "labels.json"), body, 0644)
}
func trainingVideoMAEManifestPath(root string) string {
return filepath.Join(root, "videomae", "dataset", "manifest.jsonl")
}
func trainingVideoMAELabelForAnnotation(item TrainingAnnotation) string {
if item.Negative {
return trainingNoSexPositionLabel
}
effective := trainingEffectiveCorrection(item)
label := normalizeSexPositionLabel(effective.SexPosition)
if isNoSexPositionLabel(label) {
return trainingNoSexPositionLabel
}
return label
}
func trainingVideoMAEFrameFallbackPath(root string, sampleID string) string {
return filepath.Join(root, "frames", sampleID+".jpg")
}
func trainingVideoMAEAnnotationHasSource(root string, item TrainingAnnotation) bool {
sourcePath := strings.TrimSpace(item.SourcePath)
if sourcePath != "" && trainingSupportedImportVideo(sourcePath) && fileExistsNonEmpty(sourcePath) {
return true
}
return fileExistsNonEmpty(trainingVideoMAEFrameFallbackPath(root, item.SampleID))
}
func trainingCountVideoMAEEligibleAnnotations(root string) (int, error) {
items, err := trainingReadAnnotations(root)
if err != nil {
return 0, err
}
labelSet, err := trainingVideoMAELabelSet()
if err != nil {
return 0, err
}
count := 0
for _, item := range items {
sampleID := strings.TrimSpace(item.SampleID)
if sampleID == "" || strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") {
continue
}
if !labelSet[trainingVideoMAELabelForAnnotation(item)] {
continue
}
if trainingVideoMAEAnnotationHasSource(root, item) {
count++
}
}
return count, nil
}
func trainingReadVideoMAEManifest(root string) ([]trainingVideoMAEManifestEntry, error) {
path := trainingVideoMAEManifestPath(root)
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return []trainingVideoMAEManifestEntry{}, nil
}
return nil, err
}
defer f.Close()
out := []trainingVideoMAEManifestEntry{}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var entry trainingVideoMAEManifestEntry
if err := json.Unmarshal([]byte(line), &entry); err != nil {
continue
}
out = append(out, entry)
}
if err := scanner.Err(); err != nil {
return nil, err
}
return out, nil
}
func trainingWriteVideoMAEManifest(root string, entries []trainingVideoMAEManifestEntry) error {
path := trainingVideoMAEManifestPath(root)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
var b strings.Builder
for _, entry := range entries {
line, err := json.Marshal(entry)
if err != nil {
return err
}
b.Write(line)
b.WriteByte('\n')
}
return os.WriteFile(path, []byte(b.String()), 0644)
}
func trainingCountVideoMAEManifestSamples(root string) (train int, val int) {
entries, err := trainingReadVideoMAEManifest(root)
if err != nil {
return 0, 0
}
for _, entry := range entries {
if entry.FrameCount <= 0 || !fileExistsNonEmpty(filepath.Join(entry.ClipDir, "frame_001.jpg")) {
continue
}
switch strings.ToLower(strings.TrimSpace(entry.Split)) {
case "train":
train++
case "val":
val++
}
}
return train, val
}
func trainingRemoveVideoMAEGeneratedClips(root string) error {
clipsDir := filepath.Join(root, "videomae", "dataset", "clips")
if err := os.RemoveAll(clipsDir); err != nil {
return err
}
for _, split := range []string{"train", "val"} {
if err := os.MkdirAll(filepath.Join(clipsDir, split), 0755); err != nil {
return err
}
}
return nil
}
func trainingVideoMAEClipFrameCount(clipDir string) int {
entries, err := os.ReadDir(clipDir)
if err != nil {
return 0
}
count := 0
for _, entry := range entries {
if entry.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(entry.Name()))
if ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".webp" {
count++
}
}
return count
}
func trainingExtractVideoMAEClipFrames(
ctx context.Context,
videoPath string,
centerSecond float64,
clipDir string,
) error {
if strings.TrimSpace(videoPath) == "" {
return errors.New("video path missing")
}
if err := os.MkdirAll(clipDir, 0755); err != nil {
return err
}
ffmpegPath := strings.TrimSpace(getSettings().FFmpegPath)
if ffmpegPath == "" {
ffmpegPath = "ffmpeg"
}
start := math.Max(0, centerSecond-trainingVideoMAEClipSeconds/2)
fps := float64(trainingVideoMAENumFrames) / trainingVideoMAEClipSeconds
vf := fmt.Sprintf(
"fps=%.6f,scale=%d:%d:force_original_aspect_ratio=decrease,pad=%d:%d:(ow-iw)/2:(oh-ih)/2",
fps,
trainingVideoMAEFrameSize,
trainingVideoMAEFrameSize,
trainingVideoMAEFrameSize,
trainingVideoMAEFrameSize,
)
cmd := exec.CommandContext(
ctx,
ffmpegPath,
"-hide_banner",
"-loglevel", "error",
"-ss", fmt.Sprintf("%.3f", start),
"-i", videoPath,
"-t", fmt.Sprintf("%.3f", trainingVideoMAEClipSeconds),
"-vf", vf,
"-frames:v", strconv.Itoa(trainingVideoMAENumFrames),
"-q:v", "3",
filepath.Join(clipDir, "frame_%03d.jpg"),
)
cmd.SysProcAttr = &syscall.SysProcAttr{
HideWindow: true,
CreationFlags: 0x08000000,
}
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("ffmpeg clip extract failed: %w: %s", err, strings.TrimSpace(string(out)))
}
if trainingVideoMAEClipFrameCount(clipDir) <= 0 {
return errors.New("ffmpeg erzeugte keine Clip-Frames")
}
return nil
}
func trainingWriteStillVideoMAEClip(framePath string, clipDir string) error {
if !fileExistsNonEmpty(framePath) {
return errors.New("fallback frame missing")
}
if err := os.MkdirAll(clipDir, 0755); err != nil {
return err
}
for i := 1; i <= trainingVideoMAENumFrames; i++ {
dst := filepath.Join(clipDir, fmt.Sprintf("frame_%03d.jpg", i))
if err := copyFile(framePath, dst); err != nil {
return err
}
}
return nil
}
func trainingWriteVideoMAEClipForAnnotation(
ctx context.Context,
root string,
item TrainingAnnotation,
clipDir string,
) (int, error) {
if err := os.RemoveAll(clipDir); err != nil {
return 0, err
}
if err := os.MkdirAll(clipDir, 0755); err != nil {
return 0, err
}
sourcePath := strings.TrimSpace(item.SourcePath)
if sourcePath != "" && trainingSupportedImportVideo(sourcePath) && fileExistsNonEmpty(sourcePath) {
if err := trainingExtractVideoMAEClipFrames(ctx, sourcePath, item.Second, clipDir); err == nil {
return trainingVideoMAEClipFrameCount(clipDir), nil
} else {
appLogln("videomae clip extract fallback:", item.SampleID, err)
}
}
fallbackFrame := trainingVideoMAEFrameFallbackPath(root, item.SampleID)
if err := trainingWriteStillVideoMAEClip(fallbackFrame, clipDir); err != nil {
return 0, err
}
return trainingVideoMAEClipFrameCount(clipDir), nil
}
func trainingCopyVideoMAEClip(srcDir string, dstDir string) (int, error) {
if err := os.RemoveAll(dstDir); err != nil {
return 0, err
}
if err := os.MkdirAll(dstDir, 0755); err != nil {
return 0, err
}
entries, err := os.ReadDir(srcDir)
if err != nil {
return 0, err
}
copied := 0
for _, entry := range entries {
if entry.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(entry.Name()))
if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" {
continue
}
if err := copyFile(filepath.Join(srcDir, entry.Name()), filepath.Join(dstDir, entry.Name())); err != nil {
return copied, err
}
copied++
}
return copied, nil
}
func trainingEnsureVideoMAEValidationEntries(
root string,
entries []trainingVideoMAEManifestEntry,
) []trainingVideoMAEManifestEntry {
trainCount := 0
valCount := 0
trainEntries := []trainingVideoMAEManifestEntry{}
for _, entry := range entries {
switch strings.ToLower(strings.TrimSpace(entry.Split)) {
case "train":
trainCount++
trainEntries = append(trainEntries, entry)
case "val":
valCount++
}
}
if valCount >= minVideoMAEValCount || trainCount < minVideoMAETrainCount {
return entries
}
sort.SliceStable(trainEntries, func(i, j int) bool {
if trainEntries[i].Label == trainEntries[j].Label {
return trainEntries[i].SampleID < trainEntries[j].SampleID
}
return trainEntries[i].Label < trainEntries[j].Label
})
for _, entry := range trainEntries {
if valCount >= minVideoMAEValCount {
break
}
copyID := entry.SampleID + "_valcopy"
dstDir := filepath.Join(root, "videomae", "dataset", "clips", "val", copyID)
frameCount, err := trainingCopyVideoMAEClip(entry.ClipDir, dstDir)
if err != nil || frameCount <= 0 {
if err != nil {
appLogln("videomae val copy failed:", entry.SampleID, err)
}
continue
}
copyEntry := entry
copyEntry.SampleID = copyID
copyEntry.Split = "val"
copyEntry.ClipDir = dstDir
copyEntry.FrameCount = frameCount
entries = append(entries, copyEntry)
valCount++
}
return entries
}
func trainingSyncVideoMAEDataset(
ctx context.Context,
root string,
) (trainCount int, valCount int, written int, err error) {
if err := trainingEnsureVideoMAEDirs(root); err != nil {
return 0, 0, 0, err
}
if err := trainingRemoveVideoMAEGeneratedClips(root); err != nil {
return 0, 0, 0, err
}
items, err := trainingReadAnnotations(root)
if err != nil {
return 0, 0, 0, err
}
labelSet, err := trainingVideoMAELabelSet()
if err != nil {
return 0, 0, 0, err
}
entries := []trainingVideoMAEManifestEntry{}
for _, item := range items {
if ctx.Err() != nil {
return 0, 0, written, ctx.Err()
}
sampleID := strings.TrimSpace(item.SampleID)
if sampleID == "" || strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") {
continue
}
label := trainingVideoMAELabelForAnnotation(item)
if !labelSet[label] {
continue
}
split := trainingStableSplit(sampleID)
clipDir := filepath.Join(root, "videomae", "dataset", "clips", split, sampleID)
frameCount, clipErr := trainingWriteVideoMAEClipForAnnotation(ctx, root, item, clipDir)
if clipErr != nil {
appLogln("videomae sample sync skipped:", sampleID, clipErr)
continue
}
entry := trainingVideoMAEManifestEntry{
SampleID: sampleID,
Split: split,
Label: label,
ClipDir: clipDir,
SourcePath: strings.TrimSpace(item.SourcePath),
Second: item.Second,
FrameCount: frameCount,
}
entries = append(entries, entry)
written++
}
entries = trainingEnsureVideoMAEValidationEntries(root, entries)
for _, entry := range entries {
switch strings.ToLower(strings.TrimSpace(entry.Split)) {
case "train":
trainCount++
case "val":
valCount++
}
}
if err := trainingWriteVideoMAEManifest(root, entries); err != nil {
return trainCount, valCount, written, err
}
return trainCount, valCount, written, nil
}

70
backend/videomae_test.go Normal file
View File

@ -0,0 +1,70 @@
package main
import (
"os"
"path/filepath"
"testing"
)
func TestTrainingVideoMAELabelForAnnotationUsesNegativeAsKeine(t *testing.T) {
got := trainingVideoMAELabelForAnnotation(TrainingAnnotation{
Negative: true,
Correction: &TrainingCorrection{
SexPosition: "doggy",
},
})
if got != trainingNoSexPositionLabel {
t.Fatalf("label = %q, want %q", got, trainingNoSexPositionLabel)
}
}
func TestTrainingVideoMAELabelForAnnotationUsesCorrection(t *testing.T) {
got := trainingVideoMAELabelForAnnotation(TrainingAnnotation{
Correction: &TrainingCorrection{
SexPosition: "cowgirl",
},
})
if got != "cowgirl" {
t.Fatalf("label = %q, want cowgirl", got)
}
}
func TestBuildAnalyzeVideoMAEClipsUsesWindowAndStride(t *testing.T) {
samples := []videoFrameSample{
{Time: 0, Path: "f0.jpg"},
{Time: 1, Path: "f1.jpg"},
{Time: 2, Path: "f2.jpg"},
{Time: 3, Path: "f3.jpg"},
{Time: 4, Path: "f4.jpg"},
{Time: 5, Path: "f5.jpg"},
}
clips := buildAnalyzeVideoMAEClips(samples, 6)
if len(clips) != 3 {
t.Fatalf("clip count = %d, want 3", len(clips))
}
if clips[0].Time != 0 || clips[1].Time != 2 || clips[2].Time != 4 {
t.Fatalf("clip times = %.1f, %.1f, %.1f; want 0, 2, 4", clips[0].Time, clips[1].Time, clips[2].Time)
}
if len(clips[1].Paths) < 4 {
t.Fatalf("middle clip paths = %d, want at least 4", len(clips[1].Paths))
}
}
func TestTrainingResolveVideoMAEModelUsesConfigDir(t *testing.T) {
root := t.TempDir()
modelDir := filepath.Join(root, "videomae", "model")
if err := os.MkdirAll(modelDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(modelDir, "config.json"), []byte("{}"), 0644); err != nil {
t.Fatal(err)
}
got := trainingResolveVideoMAEModel(root)
if !got.EffectiveExists || got.EffectivePath != modelDir || got.Source != "videomae_clip" {
t.Fatalf("unexpected model resolution: %+v", got)
}
}