8612 lines
214 KiB
Go
8612 lines
214 KiB
Go
// backend\training.go
|
|
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"crypto/sha1"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
goprocess "github.com/shirou/gopsutil/v3/process"
|
|
)
|
|
|
|
const trainingUncertainCandidateCount = 10
|
|
|
|
type TrainingLabels struct {
|
|
People []string `json:"people"`
|
|
SexPositions []string `json:"sexPositions"`
|
|
BodyParts []string `json:"bodyParts"`
|
|
Objects []string `json:"objects"`
|
|
Clothing []string `json:"clothing"`
|
|
}
|
|
|
|
type TrainingBox struct {
|
|
Label string `json:"label"`
|
|
Score float64 `json:"score,omitempty"`
|
|
X float64 `json:"x"`
|
|
Y float64 `json:"y"`
|
|
W float64 `json:"w"`
|
|
H float64 `json:"h"`
|
|
}
|
|
|
|
type TrainingScoredLabel struct {
|
|
Label string `json:"label"`
|
|
Score float64 `json:"score"`
|
|
}
|
|
|
|
type TrainingPrediction struct {
|
|
ModelAvailable bool `json:"modelAvailable"`
|
|
Source string `json:"source,omitempty"`
|
|
|
|
SexPosition string `json:"sexPosition"`
|
|
SexPositionScore float64 `json:"sexPositionScore"`
|
|
PeoplePresent []TrainingScoredLabel `json:"peoplePresent"`
|
|
BodyPartsPresent []TrainingScoredLabel `json:"bodyPartsPresent"`
|
|
ObjectsPresent []TrainingScoredLabel `json:"objectsPresent"`
|
|
ClothingPresent []TrainingScoredLabel `json:"clothingPresent"`
|
|
Boxes []TrainingBox `json:"boxes"`
|
|
Persons []TrainingPosePerson `json:"persons,omitempty"`
|
|
}
|
|
|
|
type TrainingCorrection struct {
|
|
SexPosition string `json:"sexPosition"`
|
|
PeoplePresent []string `json:"peoplePresent"`
|
|
BodyPartsPresent []string `json:"bodyPartsPresent"`
|
|
ObjectsPresent []string `json:"objectsPresent"`
|
|
ClothingPresent []string `json:"clothingPresent"`
|
|
Boxes []TrainingBox `json:"boxes"`
|
|
PosePersons []TrainingPosePerson `json:"posePersons,omitempty"`
|
|
}
|
|
|
|
type TrainingSample struct {
|
|
SampleID string `json:"sampleId"`
|
|
FrameURL string `json:"frameUrl"`
|
|
SourceFile string `json:"sourceFile"`
|
|
SourcePath string `json:"sourcePath,omitempty"`
|
|
SourceSizeBytes int64 `json:"sourceSizeBytes,omitempty"`
|
|
Second float64 `json:"second"`
|
|
CreatedAt string `json:"createdAt"`
|
|
UncertaintyScore float64 `json:"uncertaintyScore,omitempty"`
|
|
Prediction TrainingPrediction `json:"prediction"`
|
|
}
|
|
|
|
type trainingUncertainQueueItem struct {
|
|
SampleID string `json:"sampleId"`
|
|
UncertaintyScore float64 `json:"uncertaintyScore"`
|
|
SourceFile string `json:"sourceFile,omitempty"`
|
|
CreatedAt string `json:"createdAt,omitempty"`
|
|
}
|
|
|
|
type trainingUncertainCandidate struct {
|
|
sample *TrainingSample
|
|
score float64
|
|
}
|
|
|
|
type TrainingFeedbackRequest struct {
|
|
SampleID string `json:"sampleId"`
|
|
Accepted bool `json:"accepted"`
|
|
Negative bool `json:"negative,omitempty"`
|
|
Correction *TrainingCorrection `json:"correction,omitempty"`
|
|
Notes string `json:"notes,omitempty"`
|
|
}
|
|
|
|
type TrainingFeedbackUpdateRequest struct {
|
|
SampleID string `json:"sampleId"`
|
|
AnsweredAt string `json:"answeredAt"`
|
|
Accepted bool `json:"accepted"`
|
|
Negative bool `json:"negative,omitempty"`
|
|
Correction *TrainingCorrection `json:"correction,omitempty"`
|
|
Notes string `json:"notes,omitempty"`
|
|
}
|
|
|
|
type TrainingSkipRequest struct {
|
|
SampleID string `json:"sampleId"`
|
|
}
|
|
|
|
type TrainingTrainRequest struct {
|
|
Scope string `json:"scope,omitempty"`
|
|
Targets []string `json:"targets,omitempty"`
|
|
}
|
|
|
|
type trainingTrainTargets struct {
|
|
Detector bool
|
|
Pose bool
|
|
VideoMAE bool
|
|
}
|
|
|
|
func trainingAllTrainTargets() trainingTrainTargets {
|
|
return trainingTrainTargets{
|
|
Detector: true,
|
|
Pose: true,
|
|
VideoMAE: true,
|
|
}
|
|
}
|
|
|
|
func (t trainingTrainTargets) empty() bool {
|
|
return !t.Detector && !t.Pose && !t.VideoMAE
|
|
}
|
|
|
|
func (t trainingTrainTargets) list() []string {
|
|
out := make([]string, 0, 3)
|
|
if t.Detector {
|
|
out = append(out, "detector")
|
|
}
|
|
if t.Pose {
|
|
out = append(out, "pose")
|
|
}
|
|
if t.VideoMAE {
|
|
out = append(out, "videomae")
|
|
}
|
|
return out
|
|
}
|
|
|
|
func trainingReadTrainRequest(r *http.Request) (TrainingTrainRequest, error) {
|
|
var req TrainingTrainRequest
|
|
if r.Body == nil {
|
|
return req, nil
|
|
}
|
|
|
|
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
|
if err != nil {
|
|
return req, err
|
|
}
|
|
|
|
if strings.TrimSpace(string(body)) == "" {
|
|
return req, nil
|
|
}
|
|
|
|
if err := json.Unmarshal(body, &req); err != nil {
|
|
return req, err
|
|
}
|
|
|
|
return req, nil
|
|
}
|
|
|
|
func trainingNormalizeTrainTargets(req TrainingTrainRequest) (trainingTrainTargets, bool, error) {
|
|
scope := strings.ToLower(strings.TrimSpace(req.Scope))
|
|
if scope == "" || scope == "full" || scope == "all" || scope == "complete" {
|
|
return trainingAllTrainTargets(), false, nil
|
|
}
|
|
|
|
if scope != "custom" && scope != "selected" && scope != "partial" {
|
|
return trainingTrainTargets{}, false, fmt.Errorf("unbekannter Trainingsumfang: %s", req.Scope)
|
|
}
|
|
|
|
var targets trainingTrainTargets
|
|
for _, raw := range req.Targets {
|
|
key := strings.ToLower(strings.TrimSpace(raw))
|
|
key = strings.ReplaceAll(key, "_", "")
|
|
key = strings.ReplaceAll(key, "-", "")
|
|
key = strings.ReplaceAll(key, " ", "")
|
|
|
|
switch key {
|
|
case "detector", "yolo", "yolo26", "yolo26detector", "box", "boxes", "boxdetection":
|
|
targets.Detector = true
|
|
case "pose", "yolo26pose", "posedetection":
|
|
targets.Pose = true
|
|
case "videomae", "video", "clip", "scene", "clipanalysis", "clipanalyse":
|
|
targets.VideoMAE = true
|
|
case "":
|
|
continue
|
|
default:
|
|
return trainingTrainTargets{}, true, fmt.Errorf("unbekanntes Training: %s", raw)
|
|
}
|
|
}
|
|
|
|
if targets.empty() {
|
|
return trainingTrainTargets{}, true, errors.New("kein Training ausgewählt")
|
|
}
|
|
|
|
return targets, true, nil
|
|
}
|
|
|
|
type TrainingAnnotation struct {
|
|
SampleID string `json:"sampleId"`
|
|
FrameURL string `json:"frameUrl"`
|
|
SourceFile string `json:"sourceFile"`
|
|
SourcePath string `json:"sourcePath,omitempty"`
|
|
SourceSizeBytes int64 `json:"sourceSizeBytes,omitempty"`
|
|
Second float64 `json:"second"`
|
|
CreatedAt string `json:"createdAt"`
|
|
AnsweredAt string `json:"answeredAt"`
|
|
Prediction TrainingPrediction `json:"prediction"`
|
|
Accepted bool `json:"accepted"`
|
|
Negative bool `json:"negative,omitempty"`
|
|
Correction *TrainingCorrection `json:"correction,omitempty"`
|
|
Notes string `json:"notes,omitempty"`
|
|
}
|
|
|
|
type TrainingDetectorPrediction struct {
|
|
Available bool `json:"available"`
|
|
Source string `json:"source,omitempty"`
|
|
Boxes []TrainingBox `json:"boxes"`
|
|
}
|
|
|
|
type TrainingKeypoint struct {
|
|
Name string `json:"name"`
|
|
X float64 `json:"x"`
|
|
Y float64 `json:"y"`
|
|
Conf float64 `json:"conf"`
|
|
}
|
|
|
|
type TrainingPosePerson struct {
|
|
Label string `json:"label,omitempty"`
|
|
Score float64 `json:"score"`
|
|
Box TrainingBox `json:"box"`
|
|
Keypoints []TrainingKeypoint `json:"keypoints"`
|
|
Quality float64 `json:"quality,omitempty"`
|
|
VisibleKeypoints int `json:"visibleKeypoints,omitempty"`
|
|
Reliable bool `json:"reliable,omitempty"`
|
|
}
|
|
|
|
type TrainingPosePrediction struct {
|
|
Available bool `json:"available"`
|
|
Source string `json:"source,omitempty"`
|
|
PersonCount int `json:"personCount"`
|
|
Persons []TrainingPosePerson `json:"persons"`
|
|
}
|
|
|
|
type TrainingJobStatus struct {
|
|
Running bool `json:"running"`
|
|
Progress int `json:"progress"`
|
|
Step string `json:"step"`
|
|
Message string `json:"message,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
StartedAt string `json:"startedAt,omitempty"`
|
|
FinishedAt string `json:"finishedAt,omitempty"`
|
|
DurationMs int64 `json:"durationMs,omitempty"`
|
|
PreviewURL string `json:"previewUrl,omitempty"`
|
|
Paused bool `json:"paused,omitempty"`
|
|
PauseReason string `json:"pauseReason,omitempty"`
|
|
CPUPercent float64 `json:"cpuPercent,omitempty"`
|
|
TemperatureC float64 `json:"temperatureC,omitempty"`
|
|
|
|
Stage string `json:"stage,omitempty"`
|
|
StageStartedAt string `json:"stageStartedAt,omitempty"`
|
|
StageProgress float64 `json:"stageProgress,omitempty"`
|
|
Epoch int `json:"epoch,omitempty"`
|
|
Epochs int `json:"epochs,omitempty"`
|
|
|
|
MAP50 float64 `json:"map50,omitempty"`
|
|
MAP5095 float64 `json:"map5095,omitempty"`
|
|
Accuracy float64 `json:"accuracy,omitempty"`
|
|
Loss float64 `json:"loss,omitempty"`
|
|
}
|
|
|
|
type TrainingConfidence struct {
|
|
Score float64 `json:"score"`
|
|
Level string `json:"level"`
|
|
Label string `json:"label"`
|
|
}
|
|
|
|
type TrainingLabelStat struct {
|
|
Label string `json:"label"`
|
|
Count int `json:"count"`
|
|
Confidence TrainingConfidence `json:"confidence"`
|
|
}
|
|
|
|
type TrainingStatsLabels struct {
|
|
People []TrainingLabelStat `json:"people"`
|
|
SexPositions []TrainingLabelStat `json:"sexPositions"`
|
|
BodyParts []TrainingLabelStat `json:"bodyParts"`
|
|
Objects []TrainingLabelStat `json:"objects"`
|
|
Clothing []TrainingLabelStat `json:"clothing"`
|
|
}
|
|
|
|
type TrainingModelInfo struct {
|
|
TrainedAt string `json:"trainedAt,omitempty"`
|
|
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
|
Epochs int `json:"epochs,omitempty"`
|
|
TrainSamples int `json:"trainSamples,omitempty"`
|
|
ValSamples int `json:"valSamples,omitempty"`
|
|
Imgsz int `json:"imgsz,omitempty"`
|
|
Device string `json:"device,omitempty"`
|
|
MAP50 float64 `json:"map50,omitempty"`
|
|
MAP5095 float64 `json:"map5095,omitempty"`
|
|
}
|
|
|
|
type TrainingStatsResponse struct {
|
|
OK bool `json:"ok"`
|
|
FeedbackCount int `json:"feedbackCount"`
|
|
AcceptedCount int `json:"acceptedCount"`
|
|
CorrectedCount int `json:"correctedCount"`
|
|
NegativeCount int `json:"negativeCount"`
|
|
SampleCount int `json:"sampleCount"`
|
|
BoxCount int `json:"boxCount"`
|
|
ModelAvailable bool `json:"modelAvailable"`
|
|
ModelInfo *TrainingModelInfo `json:"modelInfo,omitempty"`
|
|
DetectorModelAvailable bool `json:"detectorModelAvailable"`
|
|
DetectorModelInfo *TrainingModelInfo `json:"detectorModelInfo,omitempty"`
|
|
PoseModelAvailable bool `json:"poseModelAvailable"`
|
|
PoseModelInfo *TrainingModelInfo `json:"poseModelInfo,omitempty"`
|
|
VideoMAEModelAvailable bool `json:"videoMAEModelAvailable"`
|
|
VideoMAEModelInfo *TrainingModelInfo `json:"videoMAEModelInfo,omitempty"`
|
|
Confidence TrainingConfidence `json:"confidence"`
|
|
Labels TrainingStatsLabels `json:"labels"`
|
|
}
|
|
|
|
type trainingProgressEvent struct {
|
|
Type string `json:"type"`
|
|
Stage string `json:"stage"`
|
|
Progress float64 `json:"progress"` // 0..1
|
|
Message string `json:"message,omitempty"`
|
|
Epoch int `json:"epoch,omitempty"`
|
|
Epochs int `json:"epochs,omitempty"`
|
|
SampleID string `json:"sampleId,omitempty"`
|
|
MAP50 *float64 `json:"mAP50,omitempty"`
|
|
MAP5095 *float64 `json:"mAP5095,omitempty"`
|
|
Accuracy *float64 `json:"accuracy,omitempty"`
|
|
Loss *float64 `json:"loss,omitempty"`
|
|
}
|
|
|
|
type TrainingFeedbackListResponse struct {
|
|
OK bool `json:"ok"`
|
|
Items []TrainingAnnotation `json:"items"`
|
|
Total int `json:"total"`
|
|
Limit int `json:"limit"`
|
|
Offset int `json:"offset"`
|
|
HasMore bool `json:"hasMore"`
|
|
}
|
|
|
|
func trainingFeedbackListHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
limit := 30
|
|
offset := 0
|
|
|
|
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
|
|
if n, err := strconv.Atoi(raw); err == nil {
|
|
limit = n
|
|
}
|
|
}
|
|
|
|
if raw := strings.TrimSpace(r.URL.Query().Get("offset")); raw != "" {
|
|
if n, err := strconv.Atoi(raw); err == nil {
|
|
offset = n
|
|
}
|
|
}
|
|
|
|
if limit < 1 {
|
|
limit = 30
|
|
}
|
|
if limit > 200 {
|
|
limit = 200
|
|
}
|
|
if offset < 0 {
|
|
offset = 0
|
|
}
|
|
|
|
items, err := trainingReadAnnotations(root)
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
// Neueste zuerst.
|
|
sort.SliceStable(items, func(i, j int) bool {
|
|
ai := strings.TrimSpace(items[i].AnsweredAt)
|
|
aj := strings.TrimSpace(items[j].AnsweredAt)
|
|
|
|
if ai == aj {
|
|
return items[i].CreatedAt > items[j].CreatedAt
|
|
}
|
|
|
|
return ai > aj
|
|
})
|
|
|
|
query := strings.TrimSpace(r.URL.Query().Get("q"))
|
|
filter := strings.TrimSpace(r.URL.Query().Get("filter"))
|
|
|
|
items = trainingFilterAnnotations(items, query, filter)
|
|
|
|
total := len(items)
|
|
|
|
if offset > total {
|
|
offset = total
|
|
}
|
|
|
|
end := offset + limit
|
|
if end > total {
|
|
end = total
|
|
}
|
|
|
|
page := items[offset:end]
|
|
|
|
trainingWriteJSON(w, http.StatusOK, TrainingFeedbackListResponse{
|
|
OK: true,
|
|
Items: page,
|
|
Total: total,
|
|
Limit: limit,
|
|
Offset: offset,
|
|
HasMore: end < total,
|
|
})
|
|
}
|
|
|
|
func trainingReadAnnotations(root string) ([]TrainingAnnotation, error) {
|
|
path := filepath.Join(root, "feedback.jsonl")
|
|
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return []TrainingAnnotation{}, nil
|
|
}
|
|
|
|
return nil, err
|
|
}
|
|
|
|
items := []TrainingAnnotation{}
|
|
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
var item TrainingAnnotation
|
|
if err := json.Unmarshal([]byte(line), &item); err != nil {
|
|
continue
|
|
}
|
|
|
|
// Alte Einträge robust machen.
|
|
if strings.TrimSpace(item.FrameURL) == "" && strings.TrimSpace(item.SampleID) != "" {
|
|
item.FrameURL = "/api/training/frame?id=" + item.SampleID
|
|
}
|
|
|
|
items = append(items, item)
|
|
}
|
|
|
|
return items, nil
|
|
}
|
|
|
|
func trainingFilterAnnotations(
|
|
items []TrainingAnnotation,
|
|
query string,
|
|
filter string,
|
|
) []TrainingAnnotation {
|
|
cleanQuery := strings.ToLower(strings.TrimSpace(query))
|
|
cleanFilter := strings.ToLower(strings.TrimSpace(filter))
|
|
|
|
out := make([]TrainingAnnotation, 0, len(items))
|
|
|
|
for _, item := range items {
|
|
switch cleanFilter {
|
|
case "accepted":
|
|
if !item.Accepted || item.Negative {
|
|
continue
|
|
}
|
|
|
|
case "corrected":
|
|
if item.Accepted || item.Negative {
|
|
continue
|
|
}
|
|
|
|
case "negative":
|
|
if !item.Negative {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if cleanQuery != "" && !trainingAnnotationMatchesQuery(item, cleanQuery) {
|
|
continue
|
|
}
|
|
|
|
out = append(out, item)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingAnnotationMatchesQuery(item TrainingAnnotation, cleanQuery string) bool {
|
|
effective := trainingEffectiveCorrection(item)
|
|
|
|
parts := []string{
|
|
item.SampleID,
|
|
item.SourceFile,
|
|
item.SourcePath,
|
|
item.CreatedAt,
|
|
item.AnsweredAt,
|
|
item.Notes,
|
|
effective.SexPosition,
|
|
}
|
|
if item.Negative {
|
|
parts = append(parts, "negative negativ leer keine labels")
|
|
}
|
|
|
|
parts = append(parts, effective.PeoplePresent...)
|
|
parts = append(parts, effective.BodyPartsPresent...)
|
|
parts = append(parts, effective.ObjectsPresent...)
|
|
parts = append(parts, effective.ClothingPresent...)
|
|
|
|
for _, box := range effective.Boxes {
|
|
parts = append(parts, box.Label)
|
|
}
|
|
|
|
haystack := strings.ToLower(strings.Join(parts, " "))
|
|
|
|
return strings.Contains(haystack, cleanQuery)
|
|
}
|
|
|
|
func trainingRemoveSampleFromUncertainQueue(root string, sampleID string) error {
|
|
sampleID = strings.TrimSpace(sampleID)
|
|
if sampleID == "" {
|
|
return nil
|
|
}
|
|
|
|
items, err := trainingReadUncertainQueue(root)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
|
|
next := make([]trainingUncertainQueueItem, 0, len(items))
|
|
changed := false
|
|
|
|
for _, item := range items {
|
|
if strings.TrimSpace(item.SampleID) == sampleID {
|
|
changed = true
|
|
continue
|
|
}
|
|
|
|
next = append(next, item)
|
|
}
|
|
|
|
if !changed {
|
|
return nil
|
|
}
|
|
|
|
return trainingWriteUncertainQueue(root, next)
|
|
}
|
|
|
|
func trainingReadValidUncertainCandidates(root string) ([]trainingUncertainCandidate, error) {
|
|
items, err := trainingReadUncertainQueue(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(items) == 0 {
|
|
return []trainingUncertainCandidate{}, nil
|
|
}
|
|
|
|
answered, err := trainingAnsweredSampleIDs(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
candidates := make([]trainingUncertainCandidate, 0, len(items))
|
|
|
|
for _, item := range items {
|
|
id := strings.TrimSpace(item.SampleID)
|
|
if id == "" || answered[id] {
|
|
continue
|
|
}
|
|
|
|
framePath := filepath.Join(root, "frames", id+".jpg")
|
|
if !fileExistsNonEmpty(framePath) {
|
|
continue
|
|
}
|
|
|
|
sample, err := trainingReadSample(root, id)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
score := item.UncertaintyScore
|
|
if score <= 0 {
|
|
score = sample.UncertaintyScore
|
|
}
|
|
if score <= 0 {
|
|
score = trainingPredictionUncertaintyScore(sample.Prediction)
|
|
}
|
|
|
|
score = clamp01(score)
|
|
sample.UncertaintyScore = score
|
|
|
|
candidates = append(candidates, trainingUncertainCandidate{
|
|
sample: sample,
|
|
score: score,
|
|
})
|
|
}
|
|
|
|
sort.Slice(candidates, func(i, j int) bool {
|
|
if candidates[i].score == candidates[j].score {
|
|
return candidates[i].sample.CreatedAt < candidates[j].sample.CreatedAt
|
|
}
|
|
|
|
return candidates[i].score > candidates[j].score
|
|
})
|
|
|
|
return candidates, nil
|
|
}
|
|
|
|
func trainingWriteUncertainCandidateQueue(root string, candidates []trainingUncertainCandidate) error {
|
|
items := make([]trainingUncertainQueueItem, 0, len(candidates))
|
|
|
|
for _, candidate := range candidates {
|
|
if candidate.sample == nil {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSpace(candidate.sample.SampleID)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
|
|
items = append(items, trainingUncertainQueueItem{
|
|
SampleID: id,
|
|
UncertaintyScore: clamp01(candidate.score),
|
|
SourceFile: candidate.sample.SourceFile,
|
|
CreatedAt: candidate.sample.CreatedAt,
|
|
})
|
|
}
|
|
|
|
return trainingWriteUncertainQueue(root, items)
|
|
}
|
|
|
|
func trainingCreateUncertainCandidateWithProgress(
|
|
root string,
|
|
startedAtMs int64,
|
|
requestID string,
|
|
stepStart int,
|
|
stepTotal int,
|
|
prefix string,
|
|
) (*trainingUncertainCandidate, error) {
|
|
sample, err := trainingCreateNextSampleWithProgressRange(
|
|
startedAtMs,
|
|
requestID,
|
|
stepStart,
|
|
stepTotal,
|
|
prefix,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
score := trainingPredictionUncertaintyScore(sample.Prediction)
|
|
score = clamp01(score)
|
|
|
|
sample.UncertaintyScore = score
|
|
|
|
if err := trainingWriteSample(root, sample); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &trainingUncertainCandidate{
|
|
sample: sample,
|
|
score: score,
|
|
}, nil
|
|
}
|
|
|
|
func trainingUncertainQueuePath(root string) string {
|
|
return filepath.Join(root, "uncertain_queue.json")
|
|
}
|
|
|
|
func trainingReadUncertainQueue(root string) ([]trainingUncertainQueueItem, error) {
|
|
path := trainingUncertainQueuePath(root)
|
|
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return []trainingUncertainQueueItem{}, nil
|
|
}
|
|
|
|
return nil, err
|
|
}
|
|
|
|
var items []trainingUncertainQueueItem
|
|
if err := json.Unmarshal(b, &items); err != nil {
|
|
return []trainingUncertainQueueItem{}, nil
|
|
}
|
|
|
|
return items, nil
|
|
}
|
|
|
|
func trainingWriteUncertainQueue(root string, items []trainingUncertainQueueItem) error {
|
|
path := trainingUncertainQueuePath(root)
|
|
|
|
if len(items) == 0 {
|
|
_ = os.Remove(path)
|
|
return nil
|
|
}
|
|
|
|
b, err := json.MarshalIndent(items, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return os.WriteFile(path, b, 0644)
|
|
}
|
|
|
|
func trainingPopQueuedUncertainSample(root string) (*TrainingSample, bool, error) {
|
|
items, err := trainingReadUncertainQueue(root)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
if len(items) == 0 {
|
|
return nil, false, nil
|
|
}
|
|
|
|
answered, err := trainingAnsweredSampleIDs(root)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
remaining := make([]trainingUncertainQueueItem, 0, len(items))
|
|
|
|
for index, item := range items {
|
|
id := strings.TrimSpace(item.SampleID)
|
|
if id == "" || answered[id] {
|
|
continue
|
|
}
|
|
|
|
framePath := filepath.Join(root, "frames", id+".jpg")
|
|
if !fileExistsNonEmpty(framePath) {
|
|
continue
|
|
}
|
|
|
|
sample, err := trainingReadSample(root, id)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
sample.UncertaintyScore = item.UncertaintyScore
|
|
|
|
remaining = append(remaining, items[index+1:]...)
|
|
|
|
if err := trainingWriteUncertainQueue(root, remaining); err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
return sample, true, nil
|
|
}
|
|
|
|
_ = trainingWriteUncertainQueue(root, []trainingUncertainQueueItem{})
|
|
return nil, false, nil
|
|
}
|
|
|
|
func trainingScaleProgress(local float64, start int, end int) int {
|
|
if math.IsNaN(local) || math.IsInf(local, 0) {
|
|
local = 0
|
|
}
|
|
|
|
local = clamp01(local)
|
|
|
|
if end < start {
|
|
end = start
|
|
}
|
|
|
|
return start + int(math.Round(local*float64(end-start)))
|
|
}
|
|
|
|
func trainingApplyStageProgress(s *TrainingJobStatus, stage string, localProgress float64) {
|
|
stage = strings.TrimSpace(stage)
|
|
if stage == "" {
|
|
return
|
|
}
|
|
|
|
if math.IsNaN(localProgress) || math.IsInf(localProgress, 0) {
|
|
localProgress = 0
|
|
}
|
|
localProgress = clamp01(localProgress)
|
|
|
|
if strings.TrimSpace(s.Stage) != stage {
|
|
s.StageStartedAt = time.Now().UTC().Format(time.RFC3339)
|
|
s.Epoch = 0
|
|
s.Epochs = 0
|
|
s.MAP50 = 0
|
|
s.MAP5095 = 0
|
|
s.Accuracy = 0
|
|
s.Loss = 0
|
|
}
|
|
|
|
s.Stage = stage
|
|
s.StageProgress = localProgress
|
|
}
|
|
|
|
func trainingHandleProgressLine(line string, start int, end int, defaultStep string) bool {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
return false
|
|
}
|
|
|
|
var ev trainingProgressEvent
|
|
if err := json.Unmarshal([]byte(line), &ev); err != nil {
|
|
return false
|
|
}
|
|
|
|
if ev.Type != "progress" {
|
|
return false
|
|
}
|
|
|
|
progress := trainingScaleProgress(ev.Progress, start, end)
|
|
step := strings.TrimSpace(ev.Message)
|
|
if step == "" {
|
|
step = defaultStep
|
|
}
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
if progress > s.Progress {
|
|
s.Progress = progress
|
|
}
|
|
|
|
s.Step = step
|
|
|
|
trainingApplyStageProgress(s, ev.Stage, ev.Progress)
|
|
|
|
if ev.Epoch > 0 {
|
|
if s.Epoch <= 0 {
|
|
s.StageStartedAt = time.Now().UTC().Format(time.RFC3339)
|
|
}
|
|
s.Epoch = ev.Epoch
|
|
}
|
|
|
|
if ev.Epochs > 0 {
|
|
s.Epochs = ev.Epochs
|
|
}
|
|
|
|
if ev.MAP50 != nil && *ev.MAP50 > 0 {
|
|
s.MAP50 = *ev.MAP50
|
|
}
|
|
if ev.MAP5095 != nil && *ev.MAP5095 > 0 {
|
|
s.MAP5095 = *ev.MAP5095
|
|
}
|
|
if ev.Accuracy != nil {
|
|
s.Accuracy = *ev.Accuracy
|
|
}
|
|
if ev.Loss != nil {
|
|
s.Loss = *ev.Loss
|
|
}
|
|
|
|
sampleID := strings.TrimSpace(ev.SampleID)
|
|
if sampleID != "" &&
|
|
!strings.Contains(sampleID, "/") &&
|
|
!strings.Contains(sampleID, "\\") {
|
|
s.PreviewURL = "/api/training/frame?id=" + url.QueryEscape(sampleID)
|
|
}
|
|
})
|
|
|
|
return true
|
|
}
|
|
|
|
func trainingPublishJobStatus(status TrainingJobStatus) {
|
|
b, err := json.Marshal(map[string]any{
|
|
"type": "training_status",
|
|
"training": status,
|
|
"ts": time.Now().UnixMilli(),
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
publishSSE("training", b)
|
|
}
|
|
|
|
type trainingAnalysisStatusEntry struct {
|
|
updatedAt time.Time
|
|
payload map[string]any
|
|
}
|
|
|
|
var trainingAnalysisStatuses = struct {
|
|
sync.Mutex
|
|
items map[string]trainingAnalysisStatusEntry
|
|
}{
|
|
items: map[string]trainingAnalysisStatusEntry{},
|
|
}
|
|
|
|
const trainingAnalysisStatusTTL = 2 * time.Hour
|
|
|
|
func trainingRememberAnalysisPayload(payload map[string]any) {
|
|
requestID := strings.TrimSpace(fmt.Sprint(payload["requestId"]))
|
|
if requestID == "" {
|
|
return
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
copied := make(map[string]any, len(payload))
|
|
for k, v := range payload {
|
|
copied[k] = v
|
|
}
|
|
|
|
trainingAnalysisStatuses.Lock()
|
|
for id, entry := range trainingAnalysisStatuses.items {
|
|
if !entry.updatedAt.IsZero() && now.Sub(entry.updatedAt) > trainingAnalysisStatusTTL {
|
|
delete(trainingAnalysisStatuses.items, id)
|
|
}
|
|
}
|
|
trainingAnalysisStatuses.items[requestID] = trainingAnalysisStatusEntry{
|
|
updatedAt: now,
|
|
payload: copied,
|
|
}
|
|
trainingAnalysisStatuses.Unlock()
|
|
}
|
|
|
|
func trainingAnalysisStatusPayload(requestID string) map[string]any {
|
|
requestID = strings.TrimSpace(requestID)
|
|
|
|
trainingAnalysisStatuses.Lock()
|
|
defer trainingAnalysisStatuses.Unlock()
|
|
|
|
if requestID != "" {
|
|
entry, ok := trainingAnalysisStatuses.items[requestID]
|
|
if !ok || entry.payload == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make(map[string]any, len(entry.payload))
|
|
for k, v := range entry.payload {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
var latest *trainingAnalysisStatusEntry
|
|
for _, entry := range trainingAnalysisStatuses.items {
|
|
if entry.payload == nil || !trainingAnalysisValueTruthy(entry.payload["running"]) {
|
|
continue
|
|
}
|
|
if latest == nil || entry.updatedAt.After(latest.updatedAt) {
|
|
copyEntry := entry
|
|
latest = ©Entry
|
|
}
|
|
}
|
|
|
|
if latest == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make(map[string]any, len(latest.payload))
|
|
for k, v := range latest.payload {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
func trainingAnalysisValueTruthy(value any) bool {
|
|
switch v := value.(type) {
|
|
case bool:
|
|
return v
|
|
case string:
|
|
return strings.EqualFold(strings.TrimSpace(v), "true") || strings.TrimSpace(v) == "1"
|
|
case int:
|
|
return v != 0
|
|
case int64:
|
|
return v != 0
|
|
case float64:
|
|
return v != 0
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func trainingPublishAnalysisPayload(payload map[string]any) {
|
|
trainingRememberAnalysisPayload(payload)
|
|
|
|
b, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
publishSSE("analysisProgress", b)
|
|
}
|
|
|
|
func trainingPublishAnalysisStep(
|
|
requestID string,
|
|
startedAtMs int64,
|
|
current int,
|
|
total int,
|
|
sourceFile string,
|
|
message string,
|
|
) {
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
current,
|
|
total,
|
|
sourceFile,
|
|
"",
|
|
message,
|
|
)
|
|
}
|
|
|
|
func trainingPublishAnalysisStepWithPreview(
|
|
requestID string,
|
|
startedAtMs int64,
|
|
current int,
|
|
total int,
|
|
sourceFile string,
|
|
previewURL string,
|
|
message string,
|
|
) {
|
|
progress := 0.0
|
|
if total > 0 {
|
|
progress = float64(current) / float64(total)
|
|
}
|
|
|
|
payload := map[string]any{
|
|
"type": "analysis_progress",
|
|
"scope": "training",
|
|
"requestId": requestID,
|
|
"running": true,
|
|
"phase": "running",
|
|
"progress": progress,
|
|
"startedAtMs": startedAtMs,
|
|
"current": current,
|
|
"total": total,
|
|
"sourceFile": strings.TrimSpace(sourceFile),
|
|
"message": strings.TrimSpace(message),
|
|
"ts": time.Now().UnixMilli(),
|
|
}
|
|
|
|
if strings.TrimSpace(previewURL) != "" {
|
|
payload["previewUrl"] = strings.TrimSpace(previewURL)
|
|
}
|
|
|
|
trainingPublishAnalysisPayload(payload)
|
|
}
|
|
|
|
func trainingPublishAnalysisStarted(
|
|
requestID string,
|
|
total int,
|
|
sourceFile string,
|
|
message string,
|
|
) int64 {
|
|
return trainingPublishAnalysisStartedWithPreview(
|
|
requestID,
|
|
total,
|
|
sourceFile,
|
|
"",
|
|
message,
|
|
)
|
|
}
|
|
|
|
func trainingPublishAnalysisStartedWithPreview(
|
|
requestID string,
|
|
total int,
|
|
sourceFile string,
|
|
previewURL string,
|
|
message string,
|
|
) int64 {
|
|
startedAtMs := time.Now().UnixMilli()
|
|
|
|
payload := map[string]any{
|
|
"type": "analysis_progress",
|
|
"scope": "training",
|
|
"requestId": requestID,
|
|
"running": true,
|
|
"phase": "starting",
|
|
"progress": 0,
|
|
"startedAtMs": startedAtMs,
|
|
"current": 0,
|
|
"total": total,
|
|
"sourceFile": strings.TrimSpace(sourceFile),
|
|
"message": strings.TrimSpace(message),
|
|
"ts": time.Now().UnixMilli(),
|
|
}
|
|
|
|
if strings.TrimSpace(previewURL) != "" {
|
|
payload["previewUrl"] = strings.TrimSpace(previewURL)
|
|
}
|
|
|
|
trainingPublishAnalysisPayload(payload)
|
|
|
|
return startedAtMs
|
|
}
|
|
|
|
func trainingPublishAnalysisFinished(
|
|
requestID string,
|
|
startedAtMs int64,
|
|
total int,
|
|
sourceFile string,
|
|
message string,
|
|
) {
|
|
finishedAtMs := time.Now().UnixMilli()
|
|
durationMs := finishedAtMs - startedAtMs
|
|
if durationMs < 0 {
|
|
durationMs = 0
|
|
}
|
|
|
|
payload := map[string]any{
|
|
"type": "analysis_progress",
|
|
"scope": "training",
|
|
"requestId": requestID,
|
|
"running": false,
|
|
"phase": "done",
|
|
"progress": 1,
|
|
"startedAtMs": startedAtMs,
|
|
"finishedAtMs": finishedAtMs,
|
|
"durationMs": durationMs,
|
|
"current": total,
|
|
"total": total,
|
|
"sourceFile": strings.TrimSpace(sourceFile),
|
|
"message": strings.TrimSpace(message),
|
|
"ts": time.Now().UnixMilli(),
|
|
}
|
|
|
|
trainingPublishAnalysisPayload(payload)
|
|
}
|
|
|
|
func trainingPublishAnalysisError(
|
|
requestID string,
|
|
startedAtMs int64,
|
|
sourceFile string,
|
|
message string,
|
|
err error,
|
|
) {
|
|
finishedAtMs := time.Now().UnixMilli()
|
|
durationMs := finishedAtMs - startedAtMs
|
|
if durationMs < 0 {
|
|
durationMs = 0
|
|
}
|
|
|
|
errText := ""
|
|
if err != nil {
|
|
errText = err.Error()
|
|
}
|
|
|
|
payload := map[string]any{
|
|
"type": "analysis_progress",
|
|
"scope": "training",
|
|
"requestId": requestID,
|
|
"running": false,
|
|
"phase": "error",
|
|
"progress": 0,
|
|
"startedAtMs": startedAtMs,
|
|
"finishedAtMs": finishedAtMs,
|
|
"durationMs": durationMs,
|
|
"sourceFile": strings.TrimSpace(sourceFile),
|
|
"message": strings.TrimSpace(message),
|
|
"error": errText,
|
|
"ts": time.Now().UnixMilli(),
|
|
}
|
|
|
|
trainingPublishAnalysisPayload(payload)
|
|
}
|
|
|
|
func trainingRunCommandStreaming(
|
|
ctx context.Context,
|
|
python string,
|
|
script string,
|
|
onLine func(line string) bool,
|
|
args ...string,
|
|
) (string, error) {
|
|
cmdArgs := append([]string{script}, args...)
|
|
cmd := exec.Command(python, cmdArgs...)
|
|
|
|
hideCommandWindow(cmd)
|
|
opts := trainingRuntimeOptionsFromSettings()
|
|
cmd.Env = trainingCommandEnv(opts)
|
|
prepareTrainingCommandForCancel(cmd)
|
|
applyTrainingLowPriorityBeforeStart(cmd, opts.LowPriority)
|
|
|
|
stdout, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
stderr, err := cmd.StderrPipe()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if ctx.Err() != nil {
|
|
return "", errTrainingCancelled
|
|
}
|
|
|
|
if err := cmd.Start(); err != nil {
|
|
return "", err
|
|
}
|
|
applyTrainingLowPriorityAfterStart(cmd, opts.LowPriority)
|
|
monitorCtx, stopResourcePauseMonitor := context.WithCancel(ctx)
|
|
resumeResourcePause := startTrainingResourcePauseMonitor(monitorCtx, cmd, opts)
|
|
|
|
var outMu sync.Mutex
|
|
var lines []string
|
|
|
|
readPipe := func(scanner *bufio.Scanner) {
|
|
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
|
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
handled := false
|
|
if onLine != nil {
|
|
handled = onLine(line)
|
|
}
|
|
|
|
// Progress-Events nicht in den finalen Output übernehmen.
|
|
if handled {
|
|
continue
|
|
}
|
|
|
|
outMu.Lock()
|
|
lines = append(lines, line)
|
|
outMu.Unlock()
|
|
}
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
readPipe(bufio.NewScanner(stdout))
|
|
}()
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
readPipe(bufio.NewScanner(stderr))
|
|
}()
|
|
|
|
waitCh := make(chan error, 1)
|
|
go func() {
|
|
waitCh <- cmd.Wait()
|
|
}()
|
|
|
|
select {
|
|
case err = <-waitCh:
|
|
case <-ctx.Done():
|
|
resumeResourcePause()
|
|
terminateTrainingCommand(cmd)
|
|
err = <-waitCh
|
|
}
|
|
stopResourcePauseMonitor()
|
|
resumeResourcePause()
|
|
wg.Wait()
|
|
|
|
outMu.Lock()
|
|
out := strings.Join(lines, "\n")
|
|
outMu.Unlock()
|
|
|
|
if ctx.Err() != nil {
|
|
return strings.TrimSpace(out), errTrainingCancelled
|
|
}
|
|
|
|
return strings.TrimSpace(out), err
|
|
}
|
|
|
|
const minTrainingFeedbackCount = 5
|
|
|
|
const minDetectorTrainCount = 20
|
|
const minDetectorValCount = 3
|
|
const minPoseTrainCount = 20
|
|
const minPoseValCount = 3
|
|
const trainingPoseKeypointCount = 17
|
|
const trainingPoseKeypointMinConfidence = 0.20
|
|
const trainingPoseReliableMinScore = 0.30
|
|
const trainingPoseReliableMinKeypoints = 6
|
|
const trainingPoseReliableMinQuality = 0.45
|
|
const trainingPositionContextMinScore = 0.22
|
|
const trainingPositionContextMaxScore = 0.44
|
|
const trainingPositionContextBoostWeight = 0.60
|
|
const trainingPositionContextOverrideMargin = 0.16
|
|
|
|
var errTrainingCancelled = errors.New("training cancelled")
|
|
|
|
const (
|
|
trainingPerformanceAuto = "auto"
|
|
trainingPerformanceEco = "eco"
|
|
trainingPerformanceBalanced = "balanced"
|
|
trainingPerformancePerformance = "performance"
|
|
trainingPerformanceCustom = "custom"
|
|
)
|
|
|
|
type trainingRuntimeOptions struct {
|
|
PerformanceMode string
|
|
PowerSaveMode bool
|
|
CPUCoreCount int
|
|
CPUThreads int
|
|
Workers int
|
|
YoloBatchSize int
|
|
LowPriority bool
|
|
VideoMAEEnabled bool
|
|
AutoPauseEnabled bool
|
|
AutoPauseCPUPercent int
|
|
AutoPauseTemperatureC int
|
|
}
|
|
|
|
func normalizeTrainingPerformanceMode(raw string) string {
|
|
switch strings.ToLower(strings.TrimSpace(raw)) {
|
|
case "", "auto", "automatic", "automatisch":
|
|
return trainingPerformanceAuto
|
|
case "eco", "schonend", "schonmodus", "powersave", "power-save", "power_save":
|
|
return trainingPerformanceEco
|
|
case "balanced", "ausgewogen", "normal":
|
|
return trainingPerformanceBalanced
|
|
case "performance", "leistung", "fast", "schnell":
|
|
return trainingPerformancePerformance
|
|
case "custom", "manual", "manuell":
|
|
return trainingPerformanceCustom
|
|
default:
|
|
return trainingPerformanceAuto
|
|
}
|
|
}
|
|
|
|
func clampTrainingInt(value int, minValue int, maxValue int) int {
|
|
if value < minValue {
|
|
return minValue
|
|
}
|
|
if value > maxValue {
|
|
return maxValue
|
|
}
|
|
return value
|
|
}
|
|
|
|
func trainingPresetValues(mode string, cpuCores int) (threads int, workers int, yoloBatch int, lowPriority bool) {
|
|
cpuCores = clampTrainingInt(cpuCores, 1, 256)
|
|
|
|
switch normalizeTrainingPerformanceMode(mode) {
|
|
case trainingPerformanceEco:
|
|
threads = 1
|
|
if cpuCores >= 4 {
|
|
threads = 2
|
|
}
|
|
workers = 0
|
|
yoloBatch = 1
|
|
if cpuCores >= 6 {
|
|
yoloBatch = 2
|
|
}
|
|
lowPriority = true
|
|
|
|
case trainingPerformancePerformance:
|
|
threads = clampTrainingInt(cpuCores-1, 2, 16)
|
|
workers = clampTrainingInt(threads/2, 2, 4)
|
|
yoloBatch = 4
|
|
if cpuCores >= 8 {
|
|
yoloBatch = 8
|
|
}
|
|
if cpuCores >= 16 {
|
|
yoloBatch = 12
|
|
}
|
|
lowPriority = false
|
|
|
|
default:
|
|
threads = clampTrainingInt(cpuCores/2, 2, 8)
|
|
workers = clampTrainingInt(threads/2, 1, 2)
|
|
yoloBatch = 2
|
|
if cpuCores >= 8 {
|
|
yoloBatch = 4
|
|
}
|
|
if cpuCores >= 16 {
|
|
yoloBatch = 6
|
|
}
|
|
lowPriority = false
|
|
}
|
|
|
|
return threads, workers, yoloBatch, lowPriority
|
|
}
|
|
|
|
func trainingRuntimeOptionsFromSettings() trainingRuntimeOptions {
|
|
s := getSettings()
|
|
return trainingRuntimeOptionsFromRecorderSettings(s)
|
|
}
|
|
|
|
func trainingRuntimeOptionsFromRecorderSettings(s RecorderSettings) trainingRuntimeOptions {
|
|
normalizeTrainingSettings(&s)
|
|
cpuCores := runtime.NumCPU()
|
|
if cpuCores < 1 {
|
|
cpuCores = 1
|
|
}
|
|
|
|
mode := normalizeTrainingPerformanceMode(s.TrainingPerformanceMode)
|
|
if mode == trainingPerformanceAuto {
|
|
if cpuCores <= 4 {
|
|
mode = trainingPerformanceEco
|
|
} else if cpuCores >= 12 {
|
|
mode = trainingPerformancePerformance
|
|
} else {
|
|
mode = trainingPerformanceBalanced
|
|
}
|
|
}
|
|
|
|
presetThreads, presetWorkers, presetBatch, presetLowPriority := trainingPresetValues(mode, cpuCores)
|
|
|
|
opts := trainingRuntimeOptions{
|
|
PerformanceMode: mode,
|
|
PowerSaveMode: mode == trainingPerformanceEco,
|
|
CPUCoreCount: cpuCores,
|
|
CPUThreads: presetThreads,
|
|
Workers: presetWorkers,
|
|
YoloBatchSize: presetBatch,
|
|
LowPriority: presetLowPriority,
|
|
VideoMAEEnabled: s.TrainingVideoMAEEnabled,
|
|
AutoPauseEnabled: s.TrainingAutoPauseEnabled,
|
|
AutoPauseCPUPercent: s.TrainingAutoPauseCPUPercent,
|
|
AutoPauseTemperatureC: s.TrainingAutoPauseTemperatureC,
|
|
}
|
|
|
|
if mode == trainingPerformanceCustom {
|
|
opts.PowerSaveMode = s.TrainingPowerSaveMode
|
|
if s.TrainingCPUThreads > 0 {
|
|
opts.CPUThreads = clampTrainingInt(s.TrainingCPUThreads, 1, cpuCores)
|
|
} else {
|
|
opts.CPUThreads = 0
|
|
}
|
|
opts.Workers = clampTrainingInt(s.TrainingWorkers, 0, 16)
|
|
opts.YoloBatchSize = clampTrainingInt(s.TrainingYoloBatchSize, 0, 64)
|
|
opts.LowPriority = s.TrainingLowPriority
|
|
} else {
|
|
if opts.CPUThreads > cpuCores {
|
|
opts.CPUThreads = cpuCores
|
|
}
|
|
}
|
|
|
|
return opts
|
|
}
|
|
|
|
func trainingYoloEarlyStoppingPatience(epochs int) int {
|
|
epochs = clampTrainingInt(epochs, 1, 300)
|
|
patience := epochs / 4
|
|
if patience < 5 {
|
|
patience = 5
|
|
}
|
|
if patience > 20 {
|
|
patience = 20
|
|
}
|
|
if patience > epochs {
|
|
patience = epochs
|
|
}
|
|
return patience
|
|
}
|
|
|
|
func trainingCommandEnv(opts trainingRuntimeOptions) []string {
|
|
env := os.Environ()
|
|
if opts.CPUThreads <= 0 {
|
|
return env
|
|
}
|
|
|
|
threads := strconv.Itoa(opts.CPUThreads)
|
|
limitVars := []string{
|
|
"OMP_NUM_THREADS",
|
|
"MKL_NUM_THREADS",
|
|
"OPENBLAS_NUM_THREADS",
|
|
"NUMEXPR_NUM_THREADS",
|
|
"VECLIB_MAXIMUM_THREADS",
|
|
"TORCH_NUM_THREADS",
|
|
}
|
|
|
|
for _, name := range limitVars {
|
|
env = append(env, name+"="+threads)
|
|
}
|
|
|
|
return env
|
|
}
|
|
|
|
type trainingResourceSnapshot struct {
|
|
CPUPercent float64
|
|
TemperatureC float64
|
|
TemperatureAvailable bool
|
|
}
|
|
|
|
func trainingRoundMetric(v float64) float64 {
|
|
if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 {
|
|
return 0
|
|
}
|
|
return math.Round(v*10) / 10
|
|
}
|
|
|
|
func trainingReadResourceSnapshot() trainingResourceSnapshot {
|
|
temp, tempOK := getMaxCPUTemperatureC()
|
|
return trainingResourceSnapshot{
|
|
CPUPercent: trainingRoundMetric(getLastCPUUsage()),
|
|
TemperatureC: trainingRoundMetric(temp),
|
|
TemperatureAvailable: tempOK,
|
|
}
|
|
}
|
|
|
|
func trainingPauseReasonForResources(opts trainingRuntimeOptions, snap trainingResourceSnapshot) (string, bool) {
|
|
if !opts.AutoPauseEnabled {
|
|
return "", false
|
|
}
|
|
|
|
reasons := []string{}
|
|
if opts.AutoPauseCPUPercent > 0 && snap.CPUPercent >= float64(opts.AutoPauseCPUPercent) {
|
|
reasons = append(
|
|
reasons,
|
|
fmt.Sprintf("CPU %.0f%% >= %d%%", snap.CPUPercent, opts.AutoPauseCPUPercent),
|
|
)
|
|
}
|
|
if opts.AutoPauseTemperatureC > 0 &&
|
|
snap.TemperatureAvailable &&
|
|
snap.TemperatureC >= float64(opts.AutoPauseTemperatureC) {
|
|
reasons = append(
|
|
reasons,
|
|
fmt.Sprintf("Temperatur %.0f°C >= %d°C", snap.TemperatureC, opts.AutoPauseTemperatureC),
|
|
)
|
|
}
|
|
|
|
if len(reasons) == 0 {
|
|
return "", false
|
|
}
|
|
return strings.Join(reasons, ", "), true
|
|
}
|
|
|
|
func trainingResourcesRecovered(opts trainingRuntimeOptions, snap trainingResourceSnapshot) bool {
|
|
if !opts.AutoPauseEnabled {
|
|
return true
|
|
}
|
|
|
|
if opts.AutoPauseCPUPercent > 0 {
|
|
resumeCPU := float64(opts.AutoPauseCPUPercent - 15)
|
|
if resumeCPU < 50 {
|
|
resumeCPU = 50
|
|
}
|
|
if snap.CPUPercent > resumeCPU {
|
|
return false
|
|
}
|
|
}
|
|
|
|
if opts.AutoPauseTemperatureC > 0 && snap.TemperatureAvailable {
|
|
resumeTemp := float64(opts.AutoPauseTemperatureC - 5)
|
|
if resumeTemp < 40 {
|
|
resumeTemp = 40
|
|
}
|
|
if snap.TemperatureC > resumeTemp {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func trainingProcessTree(pid int32) []*goprocess.Process {
|
|
if pid <= 0 {
|
|
return nil
|
|
}
|
|
|
|
root, err := goprocess.NewProcess(pid)
|
|
if err != nil || root == nil {
|
|
return nil
|
|
}
|
|
|
|
seen := map[int32]bool{}
|
|
var out []*goprocess.Process
|
|
var walk func(*goprocess.Process)
|
|
walk = func(p *goprocess.Process) {
|
|
if p == nil || seen[p.Pid] {
|
|
return
|
|
}
|
|
|
|
seen[p.Pid] = true
|
|
out = append(out, p)
|
|
|
|
children, err := p.Children()
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, child := range children {
|
|
walk(child)
|
|
}
|
|
}
|
|
|
|
walk(root)
|
|
return out
|
|
}
|
|
|
|
func trainingSuspendProcessTree(cmd *exec.Cmd) error {
|
|
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
|
|
return nil
|
|
}
|
|
|
|
procs := trainingProcessTree(int32(cmd.Process.Pid))
|
|
if len(procs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var firstErr error
|
|
success := 0
|
|
for i := len(procs) - 1; i >= 0; i-- {
|
|
if err := procs[i].Suspend(); err != nil {
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
continue
|
|
}
|
|
success++
|
|
}
|
|
|
|
if success > 0 {
|
|
return nil
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
func trainingResumeProcessTree(cmd *exec.Cmd) error {
|
|
if cmd == nil || cmd.Process == nil || cmd.Process.Pid <= 0 {
|
|
return nil
|
|
}
|
|
|
|
procs := trainingProcessTree(int32(cmd.Process.Pid))
|
|
if len(procs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var firstErr error
|
|
success := 0
|
|
for _, proc := range procs {
|
|
if err := proc.Resume(); err != nil {
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
continue
|
|
}
|
|
success++
|
|
}
|
|
|
|
if success > 0 {
|
|
return nil
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
func startTrainingResourcePauseMonitor(ctx context.Context, cmd *exec.Cmd, opts trainingRuntimeOptions) func() {
|
|
if !opts.AutoPauseEnabled || cmd == nil || cmd.Process == nil {
|
|
return func() {}
|
|
}
|
|
|
|
const (
|
|
checkInterval = 5 * time.Second
|
|
requiredHighSamples = 2
|
|
requiredCoolSamples = 2
|
|
minPauseDuration = 45 * time.Second
|
|
)
|
|
|
|
var pauseMu sync.Mutex
|
|
paused := false
|
|
previousStep := ""
|
|
highSamples := 0
|
|
coolSamples := 0
|
|
pauseStartedAt := time.Time{}
|
|
loggedSuspendError := false
|
|
|
|
resumeNow := func(reason string) {
|
|
pauseMu.Lock()
|
|
if !paused {
|
|
pauseMu.Unlock()
|
|
return
|
|
}
|
|
|
|
err := trainingResumeProcessTree(cmd)
|
|
paused = false
|
|
restoreStep := strings.TrimSpace(previousStep)
|
|
previousStep = ""
|
|
pauseMu.Unlock()
|
|
|
|
if err != nil {
|
|
appLogln("⚠️ Training konnte nach Ressourcenpause nicht sauber fortgesetzt werden:", err)
|
|
}
|
|
|
|
if strings.TrimSpace(reason) != "" {
|
|
appLogln(reason)
|
|
}
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
s.Paused = false
|
|
s.PauseReason = ""
|
|
s.CPUPercent = 0
|
|
s.TemperatureC = 0
|
|
s.Message = ""
|
|
if restoreStep != "" && s.Running {
|
|
s.Step = restoreStep
|
|
}
|
|
})
|
|
}
|
|
|
|
go func() {
|
|
ticker := time.NewTicker(checkInterval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
resumeNow("Training-Ressourcenpause wurde beendet.")
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
|
|
snap := trainingReadResourceSnapshot()
|
|
|
|
pauseMu.Lock()
|
|
isPaused := paused
|
|
pauseAge := time.Since(pauseStartedAt)
|
|
pauseMu.Unlock()
|
|
|
|
if !isPaused {
|
|
reason, shouldPause := trainingPauseReasonForResources(opts, snap)
|
|
if !shouldPause {
|
|
highSamples = 0
|
|
continue
|
|
}
|
|
|
|
highSamples++
|
|
if highSamples < requiredHighSamples {
|
|
continue
|
|
}
|
|
|
|
pauseMu.Lock()
|
|
if paused {
|
|
pauseMu.Unlock()
|
|
continue
|
|
}
|
|
|
|
currentStatus := trainingGetJobStatus()
|
|
previousStep = strings.TrimSpace(currentStatus.Step)
|
|
if previousStep == "" {
|
|
previousStep = "Training läuft..."
|
|
}
|
|
|
|
if err := trainingSuspendProcessTree(cmd); err != nil {
|
|
pauseMu.Unlock()
|
|
highSamples = 0
|
|
if !loggedSuspendError {
|
|
loggedSuspendError = true
|
|
appLogln("⚠️ Training-Ressourcenpause konnte Prozess nicht pausieren:", err)
|
|
}
|
|
continue
|
|
}
|
|
|
|
paused = true
|
|
pauseStartedAt = time.Now()
|
|
coolSamples = 0
|
|
pauseMu.Unlock()
|
|
|
|
appLogln("⏸ Training pausiert wegen Ressourcenlimit:", reason)
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
s.Paused = true
|
|
s.PauseReason = reason
|
|
s.CPUPercent = snap.CPUPercent
|
|
if snap.TemperatureAvailable {
|
|
s.TemperatureC = snap.TemperatureC
|
|
} else {
|
|
s.TemperatureC = 0
|
|
}
|
|
s.Step = "Training pausiert: " + reason
|
|
s.Message = "Training wird automatisch fortgesetzt, sobald CPU/Temperatur wieder im grünen Bereich sind."
|
|
})
|
|
continue
|
|
}
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
if !s.Running || !s.Paused {
|
|
return
|
|
}
|
|
s.CPUPercent = snap.CPUPercent
|
|
if snap.TemperatureAvailable {
|
|
s.TemperatureC = snap.TemperatureC
|
|
}
|
|
})
|
|
|
|
if pauseAge < minPauseDuration || !trainingResourcesRecovered(opts, snap) {
|
|
coolSamples = 0
|
|
continue
|
|
}
|
|
|
|
coolSamples++
|
|
if coolSamples < requiredCoolSamples {
|
|
continue
|
|
}
|
|
|
|
highSamples = 0
|
|
coolSamples = 0
|
|
resumeNow("▶ Training nach Ressourcenpause fortgesetzt.")
|
|
}
|
|
}()
|
|
|
|
return func() {
|
|
resumeNow("")
|
|
}
|
|
}
|
|
|
|
var trainingJob = struct {
|
|
mu sync.Mutex
|
|
status TrainingJobStatus
|
|
cancel context.CancelFunc
|
|
}{}
|
|
|
|
func trainingSetJobStatus(update func(*TrainingJobStatus)) {
|
|
trainingJob.mu.Lock()
|
|
update(&trainingJob.status)
|
|
snapshot := trainingJob.status
|
|
trainingJob.mu.Unlock()
|
|
|
|
trainingPublishJobStatus(snapshot)
|
|
}
|
|
|
|
func trainingGetJobStatus() TrainingJobStatus {
|
|
trainingJob.mu.Lock()
|
|
defer trainingJob.mu.Unlock()
|
|
return trainingJob.status
|
|
}
|
|
|
|
func trainingStartJob(cancel context.CancelFunc) {
|
|
trainingJob.mu.Lock()
|
|
|
|
startedAt := time.Now().UTC().Format(time.RFC3339)
|
|
trainingJob.status = TrainingJobStatus{
|
|
Running: true,
|
|
Progress: 5,
|
|
Step: "Training wird vorbereitet…",
|
|
StartedAt: startedAt,
|
|
StageStartedAt: startedAt,
|
|
}
|
|
|
|
trainingJob.cancel = cancel
|
|
snapshot := trainingJob.status
|
|
|
|
trainingJob.mu.Unlock()
|
|
|
|
trainingPublishJobStatus(snapshot)
|
|
}
|
|
|
|
func trainingClearJobCancel() {
|
|
trainingJob.mu.Lock()
|
|
trainingJob.cancel = nil
|
|
trainingJob.mu.Unlock()
|
|
}
|
|
|
|
func trainingCleanupPartialDetectorTraining(root string) error {
|
|
// Löscht nur temporäre Trainingsläufe.
|
|
// Feedback, Samples, Frames und Dataset bleiben erhalten.
|
|
runsDir := filepath.Join(root, "detector", "runs")
|
|
|
|
if err := os.RemoveAll(runsDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
return os.MkdirAll(runsDir, 0755)
|
|
}
|
|
|
|
func trainingFinishCancelled(root string) {
|
|
cleanupErr := trainingCleanupPartialDetectorTraining(root)
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
finishedAt := time.Now().UTC()
|
|
|
|
var durationMs int64
|
|
if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(s.StartedAt)); err == nil {
|
|
durationMs = finishedAt.Sub(startedAt).Milliseconds()
|
|
if durationMs < 0 {
|
|
durationMs = 0
|
|
}
|
|
}
|
|
|
|
s.Running = false
|
|
s.Step = "Training abgebrochen."
|
|
s.Message = "Training wurde abgebrochen. Temporäre Trainingsausgaben wurden gelöscht."
|
|
s.Error = ""
|
|
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
|
s.DurationMs = durationMs
|
|
s.PreviewURL = ""
|
|
s.Paused = false
|
|
s.PauseReason = ""
|
|
s.CPUPercent = 0
|
|
s.TemperatureC = 0
|
|
|
|
if cleanupErr != nil {
|
|
s.Message = "Training wurde abgebrochen, aber temporäre Trainingsausgaben konnten nicht vollständig gelöscht werden."
|
|
s.Error = cleanupErr.Error()
|
|
}
|
|
})
|
|
|
|
trainingClearJobCancel()
|
|
}
|
|
|
|
func trainingRunCommand(python string, script string, args ...string) (string, error) {
|
|
cmdArgs := append([]string{script}, args...)
|
|
cmd := exec.Command(python, cmdArgs...)
|
|
|
|
hideCommandWindow(cmd)
|
|
|
|
out, err := cmd.CombinedOutput()
|
|
return strings.TrimSpace(string(out)), err
|
|
}
|
|
|
|
type trainingModelResolution struct {
|
|
BestPath string
|
|
EffectivePath string
|
|
Source string
|
|
TrainedExists bool
|
|
EffectiveExists bool
|
|
}
|
|
|
|
func trainingResolveModel(
|
|
root string,
|
|
kind string,
|
|
trainedSource string,
|
|
) trainingModelResolution {
|
|
bestPath := filepath.Join(root, kind, "model", "best.pt")
|
|
if fileExistsNonEmpty(bestPath) {
|
|
return trainingModelResolution{
|
|
BestPath: bestPath,
|
|
EffectivePath: bestPath,
|
|
Source: trainedSource,
|
|
TrainedExists: true,
|
|
EffectiveExists: true,
|
|
}
|
|
}
|
|
|
|
return trainingModelResolution{
|
|
BestPath: bestPath,
|
|
EffectivePath: bestPath,
|
|
Source: kind + "_missing",
|
|
TrainedExists: false,
|
|
EffectiveExists: false,
|
|
}
|
|
}
|
|
|
|
func trainingResolveDetectorModel(root string) trainingModelResolution {
|
|
return trainingResolveModel(
|
|
root,
|
|
"detector",
|
|
"yolo26_detector",
|
|
)
|
|
}
|
|
|
|
func trainingResolvePoseModel(root string) trainingModelResolution {
|
|
res := trainingResolveModel(
|
|
root,
|
|
"pose",
|
|
"yolo_pose",
|
|
)
|
|
if res.EffectiveExists {
|
|
return res
|
|
}
|
|
|
|
if poseBase, err := embeddedPoseModelPath(); err == nil && fileExistsNonEmpty(poseBase) {
|
|
res.EffectivePath = poseBase
|
|
res.Source = "yolo26_pose_base"
|
|
res.EffectiveExists = true
|
|
}
|
|
|
|
return res
|
|
}
|
|
|
|
func trainingLabelsHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusOK, defaultTrainingLabelsFromJSON())
|
|
}
|
|
|
|
const trainingImportVideoDefaultFrameCount = 10
|
|
const trainingImportVideoMaxFrameCount = 30
|
|
|
|
type TrainingImportVideoRequest struct {
|
|
JobID string `json:"jobId"`
|
|
Output string `json:"output"`
|
|
Count int `json:"count"`
|
|
AnalysisRequestID string `json:"analysisRequestId"`
|
|
}
|
|
|
|
type TrainingImportVideoResponse struct {
|
|
OK bool `json:"ok"`
|
|
Accepted bool `json:"accepted,omitempty"`
|
|
Running bool `json:"running,omitempty"`
|
|
RequestID string `json:"requestId,omitempty"`
|
|
Count int `json:"count"`
|
|
Sample *TrainingSample `json:"sample,omitempty"`
|
|
Samples []TrainingSample `json:"samples,omitempty"`
|
|
Errors []string `json:"errors,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
Analysis map[string]any `json:"analysis,omitempty"`
|
|
}
|
|
|
|
type TrainingNextRequest struct {
|
|
ForceNew bool
|
|
RefreshPrediction bool
|
|
PreferUncertain bool
|
|
AnalysisRequestID string
|
|
ExcludeIDs map[string]bool
|
|
}
|
|
|
|
type TrainingNextResponse struct {
|
|
OK bool `json:"ok"`
|
|
Accepted bool `json:"accepted,omitempty"`
|
|
Running bool `json:"running,omitempty"`
|
|
RequestID string `json:"requestId,omitempty"`
|
|
Sample *TrainingSample `json:"sample,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
Analysis map[string]any `json:"analysis,omitempty"`
|
|
}
|
|
|
|
type trainingNextJobState struct {
|
|
requestID string
|
|
startedAt time.Time
|
|
finishedAt time.Time
|
|
running bool
|
|
statusCode int
|
|
response *TrainingNextResponse
|
|
errorText string
|
|
}
|
|
|
|
var trainingNextJobs = struct {
|
|
sync.Mutex
|
|
items map[string]*trainingNextJobState
|
|
}{
|
|
items: map[string]*trainingNextJobState{},
|
|
}
|
|
|
|
const trainingNextJobTTL = 2 * time.Hour
|
|
|
|
type trainingImportVideoJobState struct {
|
|
requestID string
|
|
startedAt time.Time
|
|
finishedAt time.Time
|
|
running bool
|
|
statusCode int
|
|
response *TrainingImportVideoResponse
|
|
errorText string
|
|
}
|
|
|
|
var trainingImportVideoJobs = struct {
|
|
sync.Mutex
|
|
items map[string]*trainingImportVideoJobState
|
|
}{
|
|
items: map[string]*trainingImportVideoJobState{},
|
|
}
|
|
|
|
const trainingImportVideoJobTTL = 2 * time.Hour
|
|
|
|
func trainingCleanImportVideoCount(count int) int {
|
|
if count <= 0 {
|
|
return trainingImportVideoDefaultFrameCount
|
|
}
|
|
|
|
if count > trainingImportVideoMaxFrameCount {
|
|
return trainingImportVideoMaxFrameCount
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
func trainingSupportedImportVideo(path string) bool {
|
|
switch strings.ToLower(filepath.Ext(path)) {
|
|
case ".mp4", ".m4v", ".mov", ".mkv", ".webm":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func trainingGeneratedAssetIDCandidatesForVideo(videoPath string) []string {
|
|
videoPath = strings.TrimSpace(videoPath)
|
|
if videoPath == "" {
|
|
return nil
|
|
}
|
|
|
|
out := []string{}
|
|
seen := map[string]bool{}
|
|
|
|
add := func(id string) {
|
|
id = stripHotPrefix(strings.TrimSpace(id))
|
|
|
|
if id == "" ||
|
|
id == "." ||
|
|
id == ".." ||
|
|
strings.Contains(id, "/") ||
|
|
strings.Contains(id, "\\") {
|
|
return
|
|
}
|
|
|
|
if seen[id] {
|
|
return
|
|
}
|
|
|
|
seen[id] = true
|
|
out = append(out, id)
|
|
}
|
|
|
|
// Fall 1:
|
|
// Video liegt selbst unter /generated/<id>/...
|
|
//
|
|
// Beispiel:
|
|
// C:\app\generated\abc123\video.mp4
|
|
// => abc123
|
|
slashPath := filepath.ToSlash(filepath.Clean(videoPath))
|
|
parts := strings.Split(slashPath, "/")
|
|
|
|
for i := 0; i+1 < len(parts); i++ {
|
|
if strings.EqualFold(parts[i], "generated") {
|
|
add(parts[i+1])
|
|
}
|
|
}
|
|
|
|
// Fall 2:
|
|
// Video liegt z.B. in done/keep, aber generated/<id>/preview.jpg
|
|
// basiert auf dem Dateinamen ohne Extension.
|
|
//
|
|
// Beispiel:
|
|
// done/keep/model/abc123.mp4
|
|
// => generated/abc123/preview.jpg
|
|
base := filepath.Base(videoPath)
|
|
stem := strings.TrimSuffix(base, filepath.Ext(base))
|
|
add(stem)
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingGeneratedPreviewPathForAssetID(assetID string) (string, bool) {
|
|
assetID = stripHotPrefix(strings.TrimSpace(assetID))
|
|
|
|
if assetID == "" ||
|
|
assetID == "." ||
|
|
assetID == ".." ||
|
|
strings.Contains(assetID, "/") ||
|
|
strings.Contains(assetID, "\\") {
|
|
return "", false
|
|
}
|
|
|
|
previewPath, err := resolvePathRelativeToApp(
|
|
filepath.Join("generated", assetID, "preview.jpg"),
|
|
)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
|
|
if !fileExistsNonEmpty(previewPath) {
|
|
return "", false
|
|
}
|
|
|
|
return previewPath, true
|
|
}
|
|
|
|
func trainingPreviewPathForVideo(videoPath string) (string, bool) {
|
|
for _, assetID := range trainingGeneratedAssetIDCandidatesForVideo(videoPath) {
|
|
if previewPath, ok := trainingGeneratedPreviewPathForAssetID(assetID); ok {
|
|
return previewPath, true
|
|
}
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
func trainingPreviewURLForVideoPath(videoPath string) string {
|
|
videoPath = strings.TrimSpace(videoPath)
|
|
if videoPath == "" {
|
|
return ""
|
|
}
|
|
|
|
if !trainingSupportedImportVideo(videoPath) {
|
|
return ""
|
|
}
|
|
|
|
return "/api/training/video-preview?output=" + url.QueryEscape(videoPath)
|
|
}
|
|
|
|
func trainingPreviewAssetIDForVideo(videoPath string) string {
|
|
candidates := trainingGeneratedAssetIDCandidatesForVideo(videoPath)
|
|
|
|
for _, assetID := range candidates {
|
|
if _, ok := trainingGeneratedPreviewPathForAssetID(assetID); ok {
|
|
return assetID
|
|
}
|
|
}
|
|
|
|
for _, assetID := range candidates {
|
|
if _, err := findFinishedFileByID(assetID); err == nil {
|
|
return assetID
|
|
}
|
|
}
|
|
|
|
if len(candidates) > 0 {
|
|
return candidates[0]
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func trainingVideoPreviewHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
outPath := strings.TrimSpace(r.URL.Query().Get("output"))
|
|
if outPath == "" {
|
|
trainingWriteError(w, http.StatusBadRequest, "output missing")
|
|
return
|
|
}
|
|
|
|
if !trainingSupportedImportVideo(outPath) {
|
|
trainingWriteError(w, http.StatusBadRequest, "unsupported video type")
|
|
return
|
|
}
|
|
|
|
st, err := os.Stat(outPath)
|
|
if err != nil || st == nil || st.IsDir() || st.Size() <= 0 {
|
|
trainingWriteError(w, http.StatusNotFound, "video not found")
|
|
return
|
|
}
|
|
|
|
// Fast path: Wenn /generated/<id>/preview.jpg schon existiert, direkt ausliefern.
|
|
if previewPath, ok := trainingPreviewPathForVideo(outPath); ok {
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
servePreviewJPGFile(w, r, previewPath)
|
|
return
|
|
}
|
|
|
|
assetID := trainingPreviewAssetIDForVideo(outPath)
|
|
if assetID == "" {
|
|
trainingWriteError(w, http.StatusNotFound, "preview asset id not found")
|
|
return
|
|
}
|
|
|
|
// Wichtig:
|
|
// Nicht file=preview.jpg setzen.
|
|
// Ohne file=... darf recordPreviewWithBase die Preview bei Bedarf erzeugen.
|
|
r2 := r.Clone(r.Context())
|
|
u := *r.URL
|
|
q := u.Query()
|
|
|
|
q.Set("id", assetID)
|
|
q.Del("output")
|
|
q.Del("file")
|
|
q.Del("fallbackOnly")
|
|
|
|
u.RawQuery = q.Encode()
|
|
r2.URL = &u
|
|
|
|
recordPreviewWithBase(w, r2, "/api/training/video-preview")
|
|
}
|
|
|
|
func trainingFrameSecondsForVideo(duration float64, count int) []float64 {
|
|
count = trainingCleanImportVideoCount(count)
|
|
|
|
if duration <= 0 {
|
|
return []float64{0}
|
|
}
|
|
|
|
if duration <= 2 {
|
|
return []float64{0}
|
|
}
|
|
|
|
minSec := 0.5
|
|
maxSec := duration - 0.5
|
|
|
|
if maxSec <= minSec {
|
|
return []float64{0}
|
|
}
|
|
|
|
out := make([]float64, 0, count)
|
|
|
|
// Lokaler RNG, damit jeder Import neue Frames bekommt.
|
|
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
|
|
// Mindestabstand zwischen Frames, damit nicht mehrfach fast dieselbe Stelle kommt.
|
|
minDistance := 0.4
|
|
|
|
// Bei sehr kurzen Videos Abstand automatisch verkleinern.
|
|
availableRange := maxSec - minSec
|
|
if availableRange/float64(count) < minDistance {
|
|
minDistance = math.Max(0.1, availableRange/float64(count+1))
|
|
}
|
|
|
|
const maxAttempts = 4000
|
|
|
|
for attempts := 0; len(out) < count && attempts < maxAttempts; attempts++ {
|
|
sec := minSec + rng.Float64()*(maxSec-minSec)
|
|
|
|
// Auf 0.1s runden, damit IDs/Logs lesbar bleiben.
|
|
sec = math.Round(sec*10) / 10
|
|
|
|
tooClose := false
|
|
for _, existing := range out {
|
|
if math.Abs(sec-existing) < minDistance {
|
|
tooClose = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if tooClose {
|
|
continue
|
|
}
|
|
|
|
out = append(out, sec)
|
|
}
|
|
|
|
// Fallback, falls ein extrem kurzes Video nicht genug unterschiedliche Random-Punkte hergibt.
|
|
for len(out) < count {
|
|
sec := minSec + rng.Float64()*(maxSec-minSec)
|
|
sec = math.Round(sec*10) / 10
|
|
out = append(out, sec)
|
|
}
|
|
|
|
sort.Float64s(out)
|
|
|
|
if len(out) == 0 {
|
|
out = append(out, 0)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingRunImportVideoRequest(req TrainingImportVideoRequest) (TrainingImportVideoResponse, int, string) {
|
|
outPath := strings.TrimSpace(req.Output)
|
|
if outPath == "" {
|
|
msg := "output missing"
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusBadRequest, msg
|
|
}
|
|
|
|
if !trainingSupportedImportVideo(outPath) {
|
|
msg := "unsupported video type"
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusBadRequest, msg
|
|
}
|
|
|
|
fi, err := os.Stat(outPath)
|
|
if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
|
|
if err == nil {
|
|
err = errors.New("video file missing or empty")
|
|
}
|
|
|
|
msg := "video not found: " + err.Error()
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusBadRequest, msg
|
|
}
|
|
|
|
duration := trainingProbeDurationSeconds(outPath)
|
|
if duration <= 0 {
|
|
msg := "Videolaenge konnte nicht bestimmt werden"
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusBadRequest, msg
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
msg := err.Error()
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusInternalServerError, msg
|
|
}
|
|
|
|
if err := trainingEnsureDetectorDirs(root); err != nil {
|
|
msg := err.Error()
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusInternalServerError, msg
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Join(root, "frames"), 0755); err != nil {
|
|
msg := err.Error()
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusInternalServerError, msg
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Join(root, "samples"), 0755); err != nil {
|
|
msg := err.Error()
|
|
return TrainingImportVideoResponse{OK: false, Error: msg}, http.StatusInternalServerError, msg
|
|
}
|
|
|
|
seconds := trainingFrameSecondsForVideo(duration, req.Count)
|
|
sourceFile := filepath.Base(outPath)
|
|
previewURL := ""
|
|
sourceFileWithFrame := func(index int) string {
|
|
return fmt.Sprintf("%s (%d / %d)", sourceFile, index+1, len(seconds))
|
|
}
|
|
|
|
requestID := strings.TrimSpace(req.AnalysisRequestID)
|
|
if requestID == "" {
|
|
requestID = trainingMakeSampleID(outPath, float64(time.Now().UnixNano()))
|
|
}
|
|
|
|
totalSteps := len(seconds) * 3
|
|
if totalSteps < 1 {
|
|
totalSteps = 1
|
|
}
|
|
|
|
startedAtMs := trainingPublishAnalysisStartedWithPreview(
|
|
requestID,
|
|
totalSteps,
|
|
sourceFile,
|
|
previewURL,
|
|
"Video wird ins Training uebernommen...",
|
|
)
|
|
|
|
var sourceSizeBytes int64
|
|
if st, err := os.Stat(outPath); err == nil && st != nil && !st.IsDir() {
|
|
sourceSizeBytes = st.Size()
|
|
}
|
|
|
|
samples := make([]TrainingSample, 0, len(seconds))
|
|
errs := []string{}
|
|
|
|
for i, second := range seconds {
|
|
stepBase := i * 3
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
stepBase+1,
|
|
totalSteps,
|
|
sourceFileWithFrame(i),
|
|
previewURL,
|
|
fmt.Sprintf("Frame %d/%d wird extrahiert...", i+1, len(seconds)),
|
|
)
|
|
|
|
id := trainingMakeSampleID(outPath, second)
|
|
framePath := filepath.Join(root, "frames", id+".jpg")
|
|
|
|
if err := trainingExtractFrame(outPath, framePath, second); err != nil {
|
|
errs = append(errs, fmt.Sprintf("Frame bei %.1fs: %v", second, err))
|
|
continue
|
|
}
|
|
|
|
previewURL = "/api/training/frame?id=" + url.QueryEscape(id)
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
stepBase+2,
|
|
totalSteps,
|
|
sourceFileWithFrame(i),
|
|
previewURL,
|
|
fmt.Sprintf("Frame %d/%d wird analysiert...", i+1, len(seconds)),
|
|
)
|
|
|
|
prediction := trainingPredictFrame(framePath)
|
|
|
|
sample := &TrainingSample{
|
|
SampleID: id,
|
|
FrameURL: "/api/training/frame?id=" + id,
|
|
SourceFile: sourceFileWithFrame(i),
|
|
SourcePath: outPath,
|
|
SourceSizeBytes: sourceSizeBytes,
|
|
Second: second,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
Prediction: prediction,
|
|
}
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
stepBase+3,
|
|
totalSteps,
|
|
sourceFileWithFrame(i),
|
|
previewURL,
|
|
fmt.Sprintf("Frame %d/%d wird gespeichert...", i+1, len(seconds)),
|
|
)
|
|
|
|
if err := trainingWriteSample(root, sample); err != nil {
|
|
_ = os.Remove(framePath)
|
|
errs = append(errs, fmt.Sprintf("Frame bei %.1fs speichern: %v", second, err))
|
|
continue
|
|
}
|
|
|
|
samples = append(samples, *sample)
|
|
}
|
|
|
|
if len(samples) == 0 {
|
|
msg := "keine Trainingsframes erzeugt"
|
|
if len(errs) > 0 {
|
|
msg += ": " + strings.Join(errs, "; ")
|
|
}
|
|
|
|
err := errors.New(msg)
|
|
trainingPublishAnalysisError(
|
|
requestID,
|
|
startedAtMs,
|
|
sourceFile,
|
|
"Video konnte nicht ins Training uebernommen werden.",
|
|
err,
|
|
)
|
|
|
|
return TrainingImportVideoResponse{OK: false, RequestID: requestID, Error: msg}, http.StatusInternalServerError, msg
|
|
}
|
|
|
|
trainingPublishAnalysisFinished(
|
|
requestID,
|
|
startedAtMs,
|
|
totalSteps,
|
|
sourceFile,
|
|
fmt.Sprintf("%d Frames ins Training uebernommen.", len(samples)),
|
|
)
|
|
|
|
return TrainingImportVideoResponse{
|
|
OK: true,
|
|
RequestID: requestID,
|
|
Count: len(samples),
|
|
Sample: &samples[0],
|
|
Samples: samples,
|
|
Errors: errs,
|
|
}, http.StatusOK, ""
|
|
}
|
|
|
|
func trainingPruneImportVideoJobsLocked(now time.Time) {
|
|
for requestID, job := range trainingImportVideoJobs.items {
|
|
if job == nil || job.running {
|
|
continue
|
|
}
|
|
if !job.finishedAt.IsZero() && now.Sub(job.finishedAt) > trainingImportVideoJobTTL {
|
|
delete(trainingImportVideoJobs.items, requestID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func trainingLatestRunningImportVideoJobRequestID() string {
|
|
now := time.Now().UTC()
|
|
|
|
trainingImportVideoJobs.Lock()
|
|
defer trainingImportVideoJobs.Unlock()
|
|
|
|
trainingPruneImportVideoJobsLocked(now)
|
|
|
|
requestID := ""
|
|
var latestStarted time.Time
|
|
|
|
for id, job := range trainingImportVideoJobs.items {
|
|
if job == nil || !job.running {
|
|
continue
|
|
}
|
|
|
|
if requestID == "" || job.startedAt.After(latestStarted) {
|
|
requestID = id
|
|
latestStarted = job.startedAt
|
|
}
|
|
}
|
|
|
|
return requestID
|
|
}
|
|
|
|
func trainingStartImportVideoJob(req TrainingImportVideoRequest) {
|
|
requestID := strings.TrimSpace(req.AnalysisRequestID)
|
|
if requestID == "" {
|
|
requestID = trainingMakeSampleID(req.Output, float64(time.Now().UnixNano()))
|
|
req.AnalysisRequestID = requestID
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
|
|
trainingImportVideoJobs.Lock()
|
|
trainingPruneImportVideoJobsLocked(now)
|
|
if _, exists := trainingImportVideoJobs.items[requestID]; exists {
|
|
trainingImportVideoJobs.Unlock()
|
|
return
|
|
}
|
|
|
|
trainingImportVideoJobs.items[requestID] = &trainingImportVideoJobState{
|
|
requestID: requestID,
|
|
startedAt: now,
|
|
running: true,
|
|
statusCode: http.StatusAccepted,
|
|
}
|
|
trainingImportVideoJobs.Unlock()
|
|
|
|
go func() {
|
|
resp, statusCode, errorText := trainingRunImportVideoRequest(req)
|
|
resp.RequestID = requestID
|
|
|
|
trainingImportVideoJobs.Lock()
|
|
if job := trainingImportVideoJobs.items[requestID]; job != nil {
|
|
job.running = false
|
|
job.finishedAt = time.Now().UTC()
|
|
job.statusCode = statusCode
|
|
job.response = &resp
|
|
job.errorText = strings.TrimSpace(errorText)
|
|
}
|
|
trainingImportVideoJobs.Unlock()
|
|
}()
|
|
}
|
|
|
|
func trainingImportVideoJobResponse(requestID string) (TrainingImportVideoResponse, int) {
|
|
requestID = strings.TrimSpace(requestID)
|
|
if requestID == "" {
|
|
requestID = trainingLatestRunningImportVideoJobRequestID()
|
|
}
|
|
|
|
if requestID == "" {
|
|
return TrainingImportVideoResponse{OK: false, Error: "Kein laufender Import-Job gefunden."}, http.StatusNotFound
|
|
}
|
|
|
|
trainingImportVideoJobs.Lock()
|
|
job := trainingImportVideoJobs.items[requestID]
|
|
trainingImportVideoJobs.Unlock()
|
|
|
|
if job == nil {
|
|
return TrainingImportVideoResponse{OK: false, RequestID: requestID, Error: "Import-Job nicht gefunden."}, http.StatusNotFound
|
|
}
|
|
|
|
if job.running {
|
|
return TrainingImportVideoResponse{
|
|
OK: true,
|
|
Accepted: true,
|
|
Running: true,
|
|
RequestID: requestID,
|
|
Analysis: trainingAnalysisStatusPayload(requestID),
|
|
}, http.StatusAccepted
|
|
}
|
|
|
|
if job.response != nil {
|
|
resp := *job.response
|
|
resp.Running = false
|
|
resp.RequestID = requestID
|
|
resp.Analysis = trainingAnalysisStatusPayload(requestID)
|
|
if resp.Error == "" {
|
|
resp.Error = job.errorText
|
|
}
|
|
statusCode := job.statusCode
|
|
if statusCode < 100 {
|
|
statusCode = http.StatusOK
|
|
}
|
|
return resp, statusCode
|
|
}
|
|
|
|
return TrainingImportVideoResponse{OK: false, RequestID: requestID, Error: job.errorText}, http.StatusInternalServerError
|
|
}
|
|
|
|
func trainingImportVideoStatusHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
requestID := strings.TrimSpace(r.URL.Query().Get("requestId"))
|
|
if requestID == "" {
|
|
requestID = strings.TrimSpace(r.URL.Query().Get("id"))
|
|
}
|
|
|
|
resp, statusCode := trainingImportVideoJobResponse(requestID)
|
|
trainingWriteJSON(w, statusCode, resp)
|
|
}
|
|
|
|
func trainingAnalysisStatusHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
requestID := strings.TrimSpace(r.URL.Query().Get("requestId"))
|
|
if requestID == "" {
|
|
requestID = strings.TrimSpace(r.URL.Query().Get("id"))
|
|
}
|
|
|
|
payload := trainingAnalysisStatusPayload(requestID)
|
|
if payload == nil {
|
|
trainingWriteJSON(w, http.StatusNotFound, map[string]any{
|
|
"ok": false,
|
|
"error": "Analyse-Status nicht gefunden.",
|
|
})
|
|
return
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
"analysis": payload,
|
|
})
|
|
}
|
|
|
|
func trainingImportVideoHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
var req TrainingImportVideoRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid json")
|
|
return
|
|
}
|
|
|
|
analysisRequestID := strings.TrimSpace(req.AnalysisRequestID)
|
|
if analysisRequestID == "" {
|
|
analysisRequestID = trainingMakeSampleID(req.Output, float64(time.Now().UnixNano()))
|
|
req.AnalysisRequestID = analysisRequestID
|
|
}
|
|
|
|
trainingStartImportVideoJob(req)
|
|
resp, statusCode := trainingImportVideoJobResponse(analysisRequestID)
|
|
trainingWriteJSON(w, statusCode, resp)
|
|
return
|
|
|
|
outPath := strings.TrimSpace(req.Output)
|
|
if outPath == "" {
|
|
trainingWriteError(w, http.StatusBadRequest, "output missing")
|
|
return
|
|
}
|
|
|
|
if !trainingSupportedImportVideo(outPath) {
|
|
trainingWriteError(w, http.StatusBadRequest, "unsupported video type")
|
|
return
|
|
}
|
|
|
|
fi, err := os.Stat(outPath)
|
|
if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
|
|
if err == nil {
|
|
err = errors.New("video file missing or empty")
|
|
}
|
|
|
|
trainingWriteError(w, http.StatusBadRequest, "video not found: "+err.Error())
|
|
return
|
|
}
|
|
|
|
duration := trainingProbeDurationSeconds(outPath)
|
|
if duration <= 0 {
|
|
trainingWriteError(w, http.StatusBadRequest, "Videolänge konnte nicht bestimmt werden")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
if err := trainingEnsureDetectorDirs(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Join(root, "frames"), 0755); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Join(root, "samples"), 0755); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
seconds := trainingFrameSecondsForVideo(duration, req.Count)
|
|
sourceFile := filepath.Base(outPath)
|
|
previewURL := ""
|
|
sourceFileWithFrame := func(index int) string {
|
|
return fmt.Sprintf("%s (%d / %d)", sourceFile, index+1, len(seconds))
|
|
}
|
|
|
|
requestID := strings.TrimSpace(req.AnalysisRequestID)
|
|
if requestID == "" {
|
|
requestID = trainingMakeSampleID(outPath, float64(time.Now().UnixNano()))
|
|
}
|
|
|
|
totalSteps := len(seconds) * 3
|
|
if totalSteps < 1 {
|
|
totalSteps = 1
|
|
}
|
|
|
|
startedAtMs := trainingPublishAnalysisStartedWithPreview(
|
|
requestID,
|
|
totalSteps,
|
|
sourceFile,
|
|
previewURL,
|
|
"Video wird ins Training übernommen…",
|
|
)
|
|
|
|
var sourceSizeBytes int64
|
|
if st, err := os.Stat(outPath); err == nil && st != nil && !st.IsDir() {
|
|
sourceSizeBytes = st.Size()
|
|
}
|
|
|
|
samples := make([]TrainingSample, 0, len(seconds))
|
|
errs := []string{}
|
|
|
|
for i, second := range seconds {
|
|
stepBase := i * 3
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
stepBase+1,
|
|
totalSteps,
|
|
sourceFileWithFrame(i),
|
|
previewURL,
|
|
fmt.Sprintf("Frame %d/%d wird extrahiert…", i+1, len(seconds)),
|
|
)
|
|
|
|
id := trainingMakeSampleID(outPath, second)
|
|
framePath := filepath.Join(root, "frames", id+".jpg")
|
|
|
|
if err := trainingExtractFrame(outPath, framePath, second); err != nil {
|
|
errs = append(errs, fmt.Sprintf("Frame bei %.1fs: %v", second, err))
|
|
continue
|
|
}
|
|
|
|
// Nach jeder erfolgreichen Extraktion das aktuell verarbeitete Frame zeigen.
|
|
previewURL = "/api/training/frame?id=" + url.QueryEscape(id)
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
stepBase+2,
|
|
totalSteps,
|
|
sourceFileWithFrame(i),
|
|
previewURL,
|
|
fmt.Sprintf("Frame %d/%d wird analysiert…", i+1, len(seconds)),
|
|
)
|
|
|
|
prediction := trainingPredictFrame(framePath)
|
|
|
|
sample := &TrainingSample{
|
|
SampleID: id,
|
|
FrameURL: "/api/training/frame?id=" + id,
|
|
SourceFile: sourceFileWithFrame(i),
|
|
SourcePath: outPath,
|
|
SourceSizeBytes: sourceSizeBytes,
|
|
Second: second,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
Prediction: prediction,
|
|
}
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
stepBase+3,
|
|
totalSteps,
|
|
sourceFileWithFrame(i),
|
|
previewURL,
|
|
fmt.Sprintf("Frame %d/%d wird gespeichert…", i+1, len(seconds)),
|
|
)
|
|
|
|
if err := trainingWriteSample(root, sample); err != nil {
|
|
_ = os.Remove(framePath)
|
|
errs = append(errs, fmt.Sprintf("Frame bei %.1fs speichern: %v", second, err))
|
|
continue
|
|
}
|
|
|
|
samples = append(samples, *sample)
|
|
}
|
|
|
|
if len(samples) == 0 {
|
|
msg := "keine Trainingsframes erzeugt"
|
|
if len(errs) > 0 {
|
|
msg += ": " + strings.Join(errs, "; ")
|
|
}
|
|
|
|
err := errors.New(msg)
|
|
|
|
trainingPublishAnalysisError(
|
|
requestID,
|
|
startedAtMs,
|
|
sourceFile,
|
|
"Video konnte nicht ins Training übernommen werden.",
|
|
err,
|
|
)
|
|
|
|
trainingWriteError(w, http.StatusInternalServerError, msg)
|
|
return
|
|
}
|
|
|
|
trainingPublishAnalysisFinished(
|
|
requestID,
|
|
startedAtMs,
|
|
totalSteps,
|
|
sourceFile,
|
|
fmt.Sprintf("%d Frames ins Training übernommen.", len(samples)),
|
|
)
|
|
|
|
trainingWriteJSON(w, http.StatusOK, TrainingImportVideoResponse{
|
|
OK: true,
|
|
Count: len(samples),
|
|
Sample: &samples[0],
|
|
Samples: samples,
|
|
Errors: errs,
|
|
})
|
|
}
|
|
|
|
func trainingNextHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
req := trainingNextRequestFromHTTP(r)
|
|
|
|
if strings.TrimSpace(req.AnalysisRequestID) == "" {
|
|
req.AnalysisRequestID = trainingMakeSampleID("training-next", float64(time.Now().UnixNano()))
|
|
}
|
|
|
|
async := r.URL.Query().Get("async") == "1" ||
|
|
strings.EqualFold(r.URL.Query().Get("async"), "true")
|
|
|
|
if async {
|
|
trainingStartNextJob(req)
|
|
resp, statusCode := trainingNextJobResponse(req.AnalysisRequestID)
|
|
trainingWriteJSON(w, statusCode, resp)
|
|
return
|
|
}
|
|
|
|
resp, statusCode, errorText := trainingRunNextRequest(req)
|
|
if statusCode >= 400 || !resp.OK || resp.Sample == nil {
|
|
if errorText == "" {
|
|
errorText = resp.Error
|
|
}
|
|
if errorText == "" {
|
|
errorText = "Trainingsbild konnte nicht geladen werden."
|
|
}
|
|
trainingWriteError(w, statusCode, errorText)
|
|
return
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusOK, resp.Sample)
|
|
}
|
|
|
|
func trainingNextRequestFromHTTP(r *http.Request) TrainingNextRequest {
|
|
forceNew := r.URL.Query().Get("force") == "1" ||
|
|
strings.EqualFold(r.URL.Query().Get("force"), "true")
|
|
|
|
analysisRequestID := strings.TrimSpace(r.URL.Query().Get("analysisRequestId"))
|
|
|
|
excludeIDs := trainingExcludedSampleIDs(r)
|
|
|
|
preferUncertain := strings.EqualFold(r.URL.Query().Get("mode"), "uncertain") ||
|
|
strings.EqualFold(r.URL.Query().Get("sampleMode"), "uncertain")
|
|
|
|
refreshPrediction := r.URL.Query().Get("refresh") == "1" ||
|
|
strings.EqualFold(r.URL.Query().Get("refresh"), "true")
|
|
|
|
return TrainingNextRequest{
|
|
ForceNew: forceNew,
|
|
RefreshPrediction: refreshPrediction,
|
|
PreferUncertain: preferUncertain,
|
|
AnalysisRequestID: analysisRequestID,
|
|
ExcludeIDs: excludeIDs,
|
|
}
|
|
}
|
|
|
|
func trainingRunNextRequest(req TrainingNextRequest) (TrainingNextResponse, int, string) {
|
|
forceNew := req.ForceNew
|
|
refreshPrediction := req.RefreshPrediction
|
|
preferUncertain := req.PreferUncertain
|
|
analysisRequestID := strings.TrimSpace(req.AnalysisRequestID)
|
|
if analysisRequestID == "" {
|
|
analysisRequestID = trainingMakeSampleID("training-next", float64(time.Now().UnixNano()))
|
|
}
|
|
|
|
excludeIDs := req.ExcludeIDs
|
|
if excludeIDs == nil {
|
|
excludeIDs = map[string]bool{}
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
return TrainingNextResponse{
|
|
OK: false,
|
|
RequestID: analysisRequestID,
|
|
Error: err.Error(),
|
|
Analysis: trainingAnalysisStatusPayload(analysisRequestID),
|
|
}, http.StatusInternalServerError, err.Error()
|
|
}
|
|
|
|
if !forceNew && !preferUncertain {
|
|
var startedAtMs int64
|
|
|
|
if refreshPrediction {
|
|
startedAtMs = time.Now().UnixMilli()
|
|
trainingPublishAnalysisStep(
|
|
analysisRequestID,
|
|
startedAtMs,
|
|
1,
|
|
2,
|
|
"",
|
|
"Aktuelles Bild wird neu analysiert…",
|
|
)
|
|
}
|
|
|
|
if sample, ok, err := trainingLatestOpenSample(root, refreshPrediction, startedAtMs, analysisRequestID, excludeIDs); err != nil {
|
|
if refreshPrediction {
|
|
trainingPublishAnalysisError(
|
|
analysisRequestID,
|
|
startedAtMs,
|
|
"",
|
|
"Aktuelles Bild konnte nicht neu analysiert werden.",
|
|
err,
|
|
)
|
|
}
|
|
|
|
return TrainingNextResponse{
|
|
OK: false,
|
|
RequestID: analysisRequestID,
|
|
Error: err.Error(),
|
|
Analysis: trainingAnalysisStatusPayload(analysisRequestID),
|
|
}, http.StatusInternalServerError, err.Error()
|
|
} else if ok {
|
|
if refreshPrediction {
|
|
trainingPublishAnalysisFinished(
|
|
analysisRequestID,
|
|
startedAtMs,
|
|
2,
|
|
sample.SourceFile,
|
|
"Analyse abgeschlossen.",
|
|
)
|
|
}
|
|
|
|
return TrainingNextResponse{
|
|
OK: true,
|
|
RequestID: analysisRequestID,
|
|
Sample: sample,
|
|
Analysis: trainingAnalysisStatusPayload(analysisRequestID),
|
|
}, http.StatusOK, ""
|
|
}
|
|
}
|
|
|
|
totalSteps := 4
|
|
if preferUncertain {
|
|
totalSteps = trainingUncertainCandidateCount*4 + 1
|
|
}
|
|
|
|
startedAtMs := trainingPublishAnalysisStarted(
|
|
analysisRequestID,
|
|
totalSteps,
|
|
"",
|
|
func() string {
|
|
if preferUncertain {
|
|
return "Unsichere Prediction wird gesucht…"
|
|
}
|
|
|
|
return "Neues Trainingsbild wird vorbereitet…"
|
|
}(),
|
|
)
|
|
|
|
var sample *TrainingSample
|
|
|
|
if preferUncertain {
|
|
sample, err = trainingCreateUncertainNextSampleWithProgress(startedAtMs, analysisRequestID)
|
|
} else {
|
|
sample, err = trainingCreateNextSampleWithProgress(startedAtMs, analysisRequestID)
|
|
}
|
|
|
|
if err != nil {
|
|
trainingPublishAnalysisError(
|
|
analysisRequestID,
|
|
startedAtMs,
|
|
"",
|
|
"Trainingsbild konnte nicht erstellt werden.",
|
|
err,
|
|
)
|
|
return TrainingNextResponse{
|
|
OK: false,
|
|
RequestID: analysisRequestID,
|
|
Error: err.Error(),
|
|
Analysis: trainingAnalysisStatusPayload(analysisRequestID),
|
|
}, http.StatusInternalServerError, err.Error()
|
|
}
|
|
|
|
trainingPublishAnalysisFinished(
|
|
analysisRequestID,
|
|
startedAtMs,
|
|
totalSteps,
|
|
sample.SourceFile,
|
|
"Analyse abgeschlossen.",
|
|
)
|
|
|
|
return TrainingNextResponse{
|
|
OK: true,
|
|
RequestID: analysisRequestID,
|
|
Sample: sample,
|
|
Analysis: trainingAnalysisStatusPayload(analysisRequestID),
|
|
}, http.StatusOK, ""
|
|
}
|
|
|
|
func trainingPruneNextJobsLocked(now time.Time) {
|
|
for requestID, job := range trainingNextJobs.items {
|
|
if job == nil || job.running {
|
|
continue
|
|
}
|
|
if !job.finishedAt.IsZero() && now.Sub(job.finishedAt) > trainingNextJobTTL {
|
|
delete(trainingNextJobs.items, requestID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func trainingLatestRunningNextJobRequestID() string {
|
|
now := time.Now().UTC()
|
|
|
|
trainingNextJobs.Lock()
|
|
defer trainingNextJobs.Unlock()
|
|
|
|
trainingPruneNextJobsLocked(now)
|
|
|
|
requestID := ""
|
|
var latestStarted time.Time
|
|
|
|
for id, job := range trainingNextJobs.items {
|
|
if job == nil || !job.running {
|
|
continue
|
|
}
|
|
|
|
if requestID == "" || job.startedAt.After(latestStarted) {
|
|
requestID = id
|
|
latestStarted = job.startedAt
|
|
}
|
|
}
|
|
|
|
return requestID
|
|
}
|
|
|
|
func trainingStartNextJob(req TrainingNextRequest) {
|
|
requestID := strings.TrimSpace(req.AnalysisRequestID)
|
|
if requestID == "" {
|
|
requestID = trainingMakeSampleID("training-next", float64(time.Now().UnixNano()))
|
|
req.AnalysisRequestID = requestID
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
|
|
trainingNextJobs.Lock()
|
|
trainingPruneNextJobsLocked(now)
|
|
if _, exists := trainingNextJobs.items[requestID]; exists {
|
|
trainingNextJobs.Unlock()
|
|
return
|
|
}
|
|
|
|
trainingNextJobs.items[requestID] = &trainingNextJobState{
|
|
requestID: requestID,
|
|
startedAt: now,
|
|
running: true,
|
|
statusCode: http.StatusAccepted,
|
|
}
|
|
trainingNextJobs.Unlock()
|
|
|
|
go func() {
|
|
resp, statusCode, errorText := trainingRunNextRequest(req)
|
|
resp.RequestID = requestID
|
|
resp.Analysis = trainingAnalysisStatusPayload(requestID)
|
|
|
|
trainingNextJobs.Lock()
|
|
if job := trainingNextJobs.items[requestID]; job != nil {
|
|
job.running = false
|
|
job.finishedAt = time.Now().UTC()
|
|
job.statusCode = statusCode
|
|
job.response = &resp
|
|
job.errorText = strings.TrimSpace(errorText)
|
|
}
|
|
trainingNextJobs.Unlock()
|
|
}()
|
|
}
|
|
|
|
func trainingNextJobResponse(requestID string) (TrainingNextResponse, int) {
|
|
requestID = strings.TrimSpace(requestID)
|
|
if requestID == "" {
|
|
requestID = trainingLatestRunningNextJobRequestID()
|
|
}
|
|
|
|
if requestID == "" {
|
|
return TrainingNextResponse{OK: false, Error: "Kein laufender Analyse-Job gefunden."}, http.StatusNotFound
|
|
}
|
|
|
|
trainingNextJobs.Lock()
|
|
job := trainingNextJobs.items[requestID]
|
|
trainingNextJobs.Unlock()
|
|
|
|
if job == nil {
|
|
return TrainingNextResponse{
|
|
OK: false,
|
|
RequestID: requestID,
|
|
Error: "Analyse-Job nicht gefunden.",
|
|
Analysis: trainingAnalysisStatusPayload(requestID),
|
|
}, http.StatusNotFound
|
|
}
|
|
|
|
if job.running {
|
|
return TrainingNextResponse{
|
|
OK: true,
|
|
Accepted: true,
|
|
Running: true,
|
|
RequestID: requestID,
|
|
Analysis: trainingAnalysisStatusPayload(requestID),
|
|
}, http.StatusAccepted
|
|
}
|
|
|
|
if job.response != nil {
|
|
resp := *job.response
|
|
resp.Running = false
|
|
resp.RequestID = requestID
|
|
resp.Analysis = trainingAnalysisStatusPayload(requestID)
|
|
if resp.Error == "" {
|
|
resp.Error = job.errorText
|
|
}
|
|
statusCode := job.statusCode
|
|
if statusCode < 100 {
|
|
statusCode = http.StatusOK
|
|
}
|
|
return resp, statusCode
|
|
}
|
|
|
|
return TrainingNextResponse{
|
|
OK: false,
|
|
RequestID: requestID,
|
|
Error: job.errorText,
|
|
Analysis: trainingAnalysisStatusPayload(requestID),
|
|
}, http.StatusInternalServerError
|
|
}
|
|
|
|
func trainingNextStatusHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
requestID := strings.TrimSpace(r.URL.Query().Get("requestId"))
|
|
if requestID == "" {
|
|
requestID = strings.TrimSpace(r.URL.Query().Get("id"))
|
|
}
|
|
|
|
resp, statusCode := trainingNextJobResponse(requestID)
|
|
trainingWriteJSON(w, statusCode, resp)
|
|
}
|
|
|
|
func trainingLatestOpenSample(
|
|
root string,
|
|
refreshPrediction bool,
|
|
startedAtMs int64,
|
|
requestID string,
|
|
excludeIDs map[string]bool,
|
|
) (*TrainingSample, bool, error) {
|
|
answered, err := trainingAnsweredSampleIDs(root)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
samplesDir := filepath.Join(root, "samples")
|
|
|
|
entries, err := os.ReadDir(samplesDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, false, nil
|
|
}
|
|
return nil, false, err
|
|
}
|
|
|
|
type sampleFile struct {
|
|
id string
|
|
path string
|
|
modTime time.Time
|
|
}
|
|
|
|
files := []sampleFile{}
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
|
|
name := entry.Name()
|
|
if strings.ToLower(filepath.Ext(name)) != ".json" {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSuffix(name, filepath.Ext(name))
|
|
if id == "" || answered[id] || excludeIDs[id] {
|
|
continue
|
|
}
|
|
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
files = append(files, sampleFile{
|
|
id: id,
|
|
path: filepath.Join(samplesDir, name),
|
|
modTime: info.ModTime(),
|
|
})
|
|
}
|
|
|
|
sort.Slice(files, func(i, j int) bool {
|
|
return files[i].modTime.After(files[j].modTime)
|
|
})
|
|
|
|
for _, file := range files {
|
|
sample, err := trainingReadSample(root, file.id)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
framePath := filepath.Join(root, "frames", sample.SampleID+".jpg")
|
|
if !fileExistsNonEmpty(framePath) {
|
|
continue
|
|
}
|
|
|
|
if refreshPrediction {
|
|
sourceFile := strings.TrimSpace(sample.SourceFile)
|
|
if sourceFile == "" {
|
|
sourceFile = filepath.Base(sample.SourcePath)
|
|
}
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
1,
|
|
2,
|
|
sourceFile,
|
|
sample.FrameURL,
|
|
"Aktuelles Bild wird analysiert…",
|
|
)
|
|
|
|
sample.Prediction = trainingPredictFrame(framePath)
|
|
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
2,
|
|
2,
|
|
sourceFile,
|
|
sample.FrameURL,
|
|
"Analyse-Ergebnis wird gespeichert…",
|
|
)
|
|
|
|
if err := trainingWriteSample(root, sample); err != nil {
|
|
return nil, false, err
|
|
}
|
|
}
|
|
|
|
return sample, true, nil
|
|
}
|
|
|
|
return nil, false, nil
|
|
}
|
|
|
|
func trainingExcludedSampleIDs(r *http.Request) map[string]bool {
|
|
out := map[string]bool{}
|
|
|
|
for _, raw := range r.URL.Query()["exclude"] {
|
|
for _, part := range strings.Split(raw, ",") {
|
|
id := strings.TrimSpace(part)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
|
|
if strings.Contains(id, "/") || strings.Contains(id, "\\") {
|
|
continue
|
|
}
|
|
|
|
out[id] = true
|
|
}
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingAnsweredSampleIDs(root string) (map[string]bool, error) {
|
|
out := map[string]bool{}
|
|
|
|
path := filepath.Join(root, "feedback.jsonl")
|
|
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return out, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
var row TrainingAnnotation
|
|
if err := json.Unmarshal([]byte(line), &row); err != nil {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSpace(row.SampleID)
|
|
if id != "" {
|
|
out[id] = true
|
|
}
|
|
}
|
|
|
|
return out, nil
|
|
}
|
|
|
|
func trainingFrameHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
id := strings.TrimSpace(r.URL.Query().Get("id"))
|
|
if id == "" || strings.Contains(id, "/") || strings.Contains(id, "\\") {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid frame id")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
path := filepath.Join(root, "frames", id+".jpg")
|
|
|
|
if _, err := os.Stat(path); err != nil {
|
|
trainingWriteError(w, http.StatusNotFound, "frame not found")
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
http.ServeFile(w, r, path)
|
|
}
|
|
|
|
func trainingDetectorBoxesForAnnotation(sample *TrainingSample, req TrainingFeedbackRequest) []TrainingBox {
|
|
if req.Negative {
|
|
return []TrainingBox{}
|
|
}
|
|
|
|
boxes := []TrainingBox{}
|
|
|
|
if req.Correction != nil {
|
|
boxes = append(boxes, req.Correction.Boxes...)
|
|
} else if req.Accepted {
|
|
boxes = append(boxes, sample.Prediction.Boxes...)
|
|
}
|
|
|
|
return boxes
|
|
}
|
|
|
|
func trainingNegativeCorrection() *TrainingCorrection {
|
|
return &TrainingCorrection{
|
|
SexPosition: trainingNoSexPositionLabel,
|
|
PeoplePresent: []string{},
|
|
BodyPartsPresent: []string{},
|
|
ObjectsPresent: []string{},
|
|
ClothingPresent: []string{},
|
|
Boxes: []TrainingBox{},
|
|
PosePersons: []TrainingPosePerson{},
|
|
}
|
|
}
|
|
|
|
func trainingFeedbackHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
var req TrainingFeedbackRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid json")
|
|
return
|
|
}
|
|
|
|
req.SampleID = strings.TrimSpace(req.SampleID)
|
|
if req.SampleID == "" {
|
|
trainingWriteError(w, http.StatusBadRequest, "sampleId missing")
|
|
return
|
|
}
|
|
if req.Negative {
|
|
req.Accepted = false
|
|
req.Correction = trainingNegativeCorrection()
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
sample, err := trainingReadSample(root, req.SampleID)
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusNotFound, "sample not found")
|
|
return
|
|
}
|
|
|
|
annotation := TrainingAnnotation{
|
|
SampleID: sample.SampleID,
|
|
FrameURL: sample.FrameURL,
|
|
SourceFile: sample.SourceFile,
|
|
SourcePath: sample.SourcePath,
|
|
SourceSizeBytes: sample.SourceSizeBytes,
|
|
Second: sample.Second,
|
|
CreatedAt: sample.CreatedAt,
|
|
AnsweredAt: time.Now().UTC().Format(time.RFC3339),
|
|
Prediction: sample.Prediction,
|
|
Accepted: req.Accepted,
|
|
Negative: req.Negative,
|
|
Correction: req.Correction,
|
|
Notes: strings.TrimSpace(req.Notes),
|
|
}
|
|
|
|
if !annotation.Accepted && annotation.Correction == nil {
|
|
trainingWriteError(w, http.StatusBadRequest, "correction missing")
|
|
return
|
|
}
|
|
|
|
if err := trainingAppendAnnotation(root, annotation); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
detectorBoxes := trainingDetectorBoxesForAnnotation(sample, req)
|
|
|
|
if req.Negative || len(detectorBoxes) > 0 {
|
|
if err := trainingWriteDetectorSample(root, sample, detectorBoxes, req.Negative); err != nil {
|
|
appLogln("⚠️ detector sample write failed:", err)
|
|
}
|
|
}
|
|
|
|
trainingDeletePoseSample(root, req.SampleID)
|
|
if sexPosition := trainingSexPositionForFeedback(sample, req); !isNoSexPositionLabel(sexPosition) {
|
|
if err := trainingWritePoseSample(root, sample, sexPosition, detectorBoxes, req.Correction); err != nil {
|
|
appLogln("pose sample write failed:", err)
|
|
}
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
})
|
|
}
|
|
|
|
func trainingFeedbackUpdateHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPut && r.Method != http.MethodPost {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
var req TrainingFeedbackUpdateRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid json")
|
|
return
|
|
}
|
|
|
|
req.SampleID = strings.TrimSpace(req.SampleID)
|
|
req.AnsweredAt = strings.TrimSpace(req.AnsweredAt)
|
|
if req.Negative {
|
|
req.Accepted = false
|
|
req.Correction = trainingNegativeCorrection()
|
|
}
|
|
|
|
if req.SampleID == "" {
|
|
trainingWriteError(w, http.StatusBadRequest, "sampleId missing")
|
|
return
|
|
}
|
|
|
|
if req.AnsweredAt == "" {
|
|
trainingWriteError(w, http.StatusBadRequest, "answeredAt missing")
|
|
return
|
|
}
|
|
|
|
if strings.Contains(req.SampleID, "/") || strings.Contains(req.SampleID, "\\") {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid sampleId")
|
|
return
|
|
}
|
|
|
|
if !req.Accepted && req.Correction == nil {
|
|
trainingWriteError(w, http.StatusBadRequest, "correction missing")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
items, err := trainingReadAnnotations(root)
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
matchIndex := -1
|
|
|
|
for i, item := range items {
|
|
if strings.TrimSpace(item.SampleID) == req.SampleID &&
|
|
strings.TrimSpace(item.AnsweredAt) == req.AnsweredAt {
|
|
matchIndex = i
|
|
break
|
|
}
|
|
}
|
|
|
|
if matchIndex < 0 {
|
|
trainingWriteError(w, http.StatusNotFound, "feedback not found")
|
|
return
|
|
}
|
|
|
|
old := items[matchIndex]
|
|
|
|
updated := old
|
|
updated.Accepted = req.Accepted
|
|
updated.Negative = req.Negative
|
|
updated.Notes = strings.TrimSpace(req.Notes)
|
|
|
|
if req.Accepted {
|
|
updated.Correction = nil
|
|
} else {
|
|
updated.Correction = req.Correction
|
|
}
|
|
|
|
items[matchIndex] = updated
|
|
|
|
if err := trainingWriteAnnotations(root, items); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
sample, sampleErr := trainingReadSample(root, req.SampleID)
|
|
if sampleErr != nil {
|
|
sample = &TrainingSample{
|
|
SampleID: old.SampleID,
|
|
FrameURL: old.FrameURL,
|
|
SourceFile: old.SourceFile,
|
|
SourcePath: old.SourcePath,
|
|
SourceSizeBytes: old.SourceSizeBytes,
|
|
Second: old.Second,
|
|
CreatedAt: old.CreatedAt,
|
|
Prediction: old.Prediction,
|
|
}
|
|
}
|
|
|
|
trainingDeleteDetectorSample(root, req.SampleID)
|
|
|
|
detectorBoxes := trainingDetectorBoxesForAnnotation(sample, TrainingFeedbackRequest{
|
|
SampleID: req.SampleID,
|
|
Accepted: req.Accepted,
|
|
Negative: req.Negative,
|
|
Correction: req.Correction,
|
|
Notes: req.Notes,
|
|
})
|
|
|
|
if req.Negative || len(detectorBoxes) > 0 {
|
|
if err := trainingWriteDetectorSample(root, sample, detectorBoxes, req.Negative); err != nil {
|
|
appLogln("⚠️ detector sample update failed:", err)
|
|
}
|
|
}
|
|
|
|
trainingDeletePoseSample(root, req.SampleID)
|
|
if sexPosition := trainingSexPositionForFeedback(sample, TrainingFeedbackRequest{
|
|
SampleID: req.SampleID,
|
|
Accepted: req.Accepted,
|
|
Negative: req.Negative,
|
|
Correction: req.Correction,
|
|
Notes: req.Notes,
|
|
}); !isNoSexPositionLabel(sexPosition) {
|
|
if err := trainingWritePoseSample(root, sample, sexPosition, detectorBoxes, req.Correction); err != nil {
|
|
appLogln("pose sample update failed:", err)
|
|
}
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
"item": updated,
|
|
})
|
|
}
|
|
|
|
func trainingSkipHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost && r.Method != http.MethodDelete {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
var req TrainingSkipRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid json")
|
|
return
|
|
}
|
|
|
|
sampleID := strings.TrimSpace(req.SampleID)
|
|
if sampleID == "" {
|
|
trainingWriteError(w, http.StatusBadRequest, "sampleId missing")
|
|
return
|
|
}
|
|
|
|
if strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid sampleId")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
// Aus Uncertain-Queue entfernen, falls es dort noch liegt.
|
|
if err := trainingRemoveSampleFromUncertainQueue(root, sampleID); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
// Sample + Frame löschen.
|
|
trainingDeleteSampleFiles(root, sampleID)
|
|
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
"sampleId": sampleID,
|
|
})
|
|
}
|
|
|
|
func trainingHasDetectorTrainingData(imagesDir string, labelsDir string) bool {
|
|
imageExts := map[string]bool{
|
|
".jpg": true,
|
|
".jpeg": true,
|
|
".png": true,
|
|
".webp": true,
|
|
}
|
|
|
|
entries, err := os.ReadDir(imagesDir)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
count := 0
|
|
|
|
for _, e := range entries {
|
|
if e.IsDir() {
|
|
continue
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(e.Name()))
|
|
if !imageExts[ext] {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name()))
|
|
labelPath := filepath.Join(labelsDir, id+".txt")
|
|
|
|
if fileExists(labelPath) {
|
|
count++
|
|
}
|
|
}
|
|
|
|
return count >= 5
|
|
}
|
|
|
|
func trainingTrainHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
req, err := trainingReadTrainRequest(r)
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusBadRequest, "invalid json")
|
|
return
|
|
}
|
|
|
|
targets, customTargets, err := trainingNormalizeTrainTargets(req)
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
|
|
current := trainingGetJobStatus()
|
|
if current.Running {
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
"message": "Training läuft bereits.",
|
|
"training": current,
|
|
})
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
if err := trainingEnsureDetectorDirs(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
// Falls bisher alles zufällig in train gelandet ist, erzeugen wir mindestens
|
|
// ein Validation-Sample durch Kopieren. Bei mehr Daten solltest du später
|
|
// einen echten 80/20 Split verwenden.
|
|
if err := trainingEnsurePoseDirs(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
if err := trainingEnsureVideoMAEDirs(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
if err := trainingEnsureDetectorValidationSample(root); err != nil {
|
|
appLogln("⚠️ detector val sample ensure failed:", err)
|
|
}
|
|
|
|
if err := trainingEnsurePoseValidationSample(root); err != nil {
|
|
appLogln("pose val sample ensure failed:", err)
|
|
}
|
|
|
|
feedbackPath := filepath.Join(root, "feedback.jsonl")
|
|
feedbackCount, _ := trainingCountAnnotations(feedbackPath)
|
|
|
|
if feedbackCount < minTrainingFeedbackCount {
|
|
trainingWriteError(
|
|
w,
|
|
http.StatusBadRequest,
|
|
fmt.Sprintf(
|
|
"Zu wenige Bewertungen für das YOLO26-Training. Mindestens %d, aktuell %d.",
|
|
minTrainingFeedbackCount,
|
|
feedbackCount,
|
|
),
|
|
)
|
|
return
|
|
}
|
|
|
|
detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train")
|
|
detectorTrainLabels := filepath.Join(root, "detector", "dataset", "labels", "train")
|
|
detectorValImages := filepath.Join(root, "detector", "dataset", "images", "val")
|
|
detectorValLabels := filepath.Join(root, "detector", "dataset", "labels", "val")
|
|
detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml")
|
|
poseTrainImages := filepath.Join(root, "pose", "dataset", "images", "train")
|
|
poseTrainLabels := filepath.Join(root, "pose", "dataset", "labels", "train")
|
|
poseValImages := filepath.Join(root, "pose", "dataset", "images", "val")
|
|
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
|
|
poseDatasetYAML := filepath.Join(root, "pose", "dataset", "dataset.yaml")
|
|
videoMAEManifest := trainingVideoMAEManifestPath(root)
|
|
|
|
trainCount := trainingCountDetectorSamples(detectorTrainImages, detectorTrainLabels)
|
|
valCount := trainingCountDetectorSamples(detectorValImages, detectorValLabels)
|
|
positiveTrainCount := trainingCountPositiveDetectorSamples(detectorTrainImages, detectorTrainLabels)
|
|
positiveValCount := trainingCountPositiveDetectorSamples(detectorValImages, detectorValLabels)
|
|
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
|
|
poseValCount := trainingCountDetectorSamples(poseValImages, poseValLabels)
|
|
videoMAETrainCount, videoMAEValCount := trainingCountVideoMAEManifestSamples(root)
|
|
videoMAEEligibleCount, _ := trainingCountVideoMAEEligibleAnnotations(root)
|
|
|
|
detectorDataReady := fileExistsNonEmpty(detectorDatasetYAML) &&
|
|
trainCount >= minDetectorTrainCount &&
|
|
valCount >= minDetectorValCount &&
|
|
positiveTrainCount > 0 &&
|
|
positiveValCount > 0
|
|
poseDataReady := fileExistsNonEmpty(poseDatasetYAML) &&
|
|
poseTrainCount >= minPoseTrainCount &&
|
|
poseValCount >= minPoseValCount
|
|
videoMAEDataReady := videoMAEEligibleCount >= minVideoMAETrainCount ||
|
|
(videoMAETrainCount >= minVideoMAETrainCount && videoMAEValCount >= minVideoMAEValCount)
|
|
|
|
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
|
selectedDataReady :=
|
|
(targets.Detector && detectorDataReady) ||
|
|
(targets.Pose && poseDataReady) ||
|
|
(targets.VideoMAE && videoMAEDataReady)
|
|
|
|
if customTargets {
|
|
if targets.Detector && !detectorDataReady {
|
|
trainingWriteError(
|
|
w,
|
|
http.StatusBadRequest,
|
|
fmt.Sprintf(
|
|
"YOLO26 Detector ist noch nicht trainingsbereit. Train=%d (%d positiv), Val=%d (%d positiv). Benoetigt: mindestens %d Train, %d Val und je ein positives Beispiel.",
|
|
trainCount,
|
|
positiveTrainCount,
|
|
valCount,
|
|
positiveValCount,
|
|
minDetectorTrainCount,
|
|
minDetectorValCount,
|
|
),
|
|
)
|
|
return
|
|
}
|
|
|
|
if targets.Pose && !poseDataReady {
|
|
trainingWriteError(
|
|
w,
|
|
http.StatusBadRequest,
|
|
fmt.Sprintf(
|
|
"YOLO26 Pose ist noch nicht trainingsbereit. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
|
poseTrainCount,
|
|
poseValCount,
|
|
minPoseTrainCount,
|
|
minPoseValCount,
|
|
),
|
|
)
|
|
return
|
|
}
|
|
|
|
if targets.VideoMAE && !runtimeOpts.VideoMAEEnabled {
|
|
trainingWriteError(w, http.StatusBadRequest, "VideoMAE ist in den Training-Settings deaktiviert.")
|
|
return
|
|
}
|
|
|
|
if targets.VideoMAE && !videoMAEDataReady {
|
|
trainingWriteError(
|
|
w,
|
|
http.StatusBadRequest,
|
|
fmt.Sprintf(
|
|
"VideoMAE ist noch nicht trainingsbereit. Eligible=%d, Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
|
videoMAEEligibleCount,
|
|
videoMAETrainCount,
|
|
videoMAEValCount,
|
|
minVideoMAETrainCount,
|
|
minVideoMAEValCount,
|
|
),
|
|
)
|
|
return
|
|
}
|
|
|
|
goto startTraining
|
|
}
|
|
|
|
if selectedDataReady {
|
|
goto startTraining
|
|
}
|
|
|
|
if !fileExistsNonEmpty(detectorDatasetYAML) ||
|
|
trainCount < minDetectorTrainCount ||
|
|
valCount < minDetectorValCount ||
|
|
positiveTrainCount == 0 ||
|
|
positiveValCount == 0 {
|
|
trainingWriteError(
|
|
w,
|
|
http.StatusBadRequest,
|
|
fmt.Sprintf(
|
|
"Zu wenige YOLO26-Beispiele. Train=%d (%d positiv), Val=%d (%d positiv). Benötigt: mindestens %d Train, %d Val und je ein positives Beispiel.",
|
|
trainCount,
|
|
positiveTrainCount,
|
|
valCount,
|
|
positiveValCount,
|
|
minDetectorTrainCount,
|
|
minDetectorValCount,
|
|
),
|
|
)
|
|
return
|
|
}
|
|
|
|
if !fileExistsNonEmpty(poseDatasetYAML) ||
|
|
poseTrainCount < minPoseTrainCount ||
|
|
poseValCount < minPoseValCount {
|
|
trainingWriteError(
|
|
w,
|
|
http.StatusBadRequest,
|
|
fmt.Sprintf(
|
|
"Zu wenige YOLO26-Pose-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
|
poseTrainCount,
|
|
poseValCount,
|
|
minPoseTrainCount,
|
|
minPoseValCount,
|
|
),
|
|
)
|
|
return
|
|
}
|
|
|
|
startTraining:
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
trainingStartJob(cancel)
|
|
|
|
go trainingRunJob(ctx, root, feedbackCount, targets)
|
|
|
|
trainingWriteJSON(w, http.StatusAccepted, map[string]any{
|
|
"ok": true,
|
|
"message": "Training gestartet.",
|
|
"training": trainingGetJobStatus(),
|
|
"targets": targets.list(),
|
|
"detector": map[string]any{
|
|
"trainCount": trainCount,
|
|
"valCount": valCount,
|
|
"positiveTrainCount": positiveTrainCount,
|
|
"positiveValCount": positiveValCount,
|
|
"requiredTrain": minDetectorTrainCount,
|
|
"requiredVal": minDetectorValCount,
|
|
"datasetYAML": detectorDatasetYAML,
|
|
"usesSceneCLIP": false,
|
|
"usesSceneKNN": false,
|
|
"source": "yolo26_detector",
|
|
"detectsPosition": false,
|
|
},
|
|
"pose": map[string]any{
|
|
"trainCount": poseTrainCount,
|
|
"valCount": poseValCount,
|
|
"requiredTrain": minPoseTrainCount,
|
|
"requiredVal": minPoseValCount,
|
|
"datasetYAML": poseDatasetYAML,
|
|
"source": "yolo26_pose",
|
|
},
|
|
"videomae": map[string]any{
|
|
"eligibleCount": videoMAEEligibleCount,
|
|
"trainCount": videoMAETrainCount,
|
|
"valCount": videoMAEValCount,
|
|
"requiredTrain": minVideoMAETrainCount,
|
|
"requiredVal": minVideoMAEValCount,
|
|
"manifest": videoMAEManifest,
|
|
"dataReady": videoMAEDataReady,
|
|
"source": "videomae_clip",
|
|
},
|
|
})
|
|
}
|
|
|
|
func trainingCancelHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost && r.Method != http.MethodDelete {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
trainingJob.mu.Lock()
|
|
status := trainingJob.status
|
|
cancel := trainingJob.cancel
|
|
trainingJob.mu.Unlock()
|
|
|
|
if !status.Running {
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
"message": "Es läuft kein Training.",
|
|
"training": status,
|
|
})
|
|
return
|
|
}
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
s.Step = "Training wird abgebrochen…"
|
|
s.Message = ""
|
|
s.Error = ""
|
|
})
|
|
|
|
if cancel != nil {
|
|
cancel()
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusAccepted, map[string]any{
|
|
"ok": true,
|
|
"message": "Training wird abgebrochen.",
|
|
"training": trainingGetJobStatus(),
|
|
})
|
|
}
|
|
|
|
func trainingRunJob(ctx context.Context, root string, count int, targets trainingTrainTargets) {
|
|
if err := ensureMLPythonSetup(ctx); err != nil {
|
|
if errors.Is(err, context.Canceled) {
|
|
trainingFinishCancelled(root)
|
|
return
|
|
}
|
|
|
|
appLogln("ML-Python setup for training failed:", err)
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
finishedAt := time.Now().UTC()
|
|
var durationMs int64
|
|
if startedAt, parseErr := time.Parse(time.RFC3339, strings.TrimSpace(s.StartedAt)); parseErr == nil {
|
|
durationMs = finishedAt.Sub(startedAt).Milliseconds()
|
|
if durationMs < 0 {
|
|
durationMs = 0
|
|
}
|
|
}
|
|
|
|
s.Running = false
|
|
s.Progress = 100
|
|
s.Step = "Training fehlgeschlagen."
|
|
s.Message = "ML-Python-Umgebung konnte nicht vorbereitet werden."
|
|
s.Error = err.Error()
|
|
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
|
s.DurationMs = durationMs
|
|
s.PreviewURL = ""
|
|
s.Paused = false
|
|
s.PauseReason = ""
|
|
s.CPUPercent = 0
|
|
s.TemperatureC = 0
|
|
})
|
|
trainingClearJobCancel()
|
|
return
|
|
}
|
|
|
|
python := trainingPythonExe()
|
|
appLogln("ML-Python für Training:", python)
|
|
runtimeOpts := trainingRuntimeOptionsFromSettings()
|
|
appLogf(
|
|
"Training-Laufzeit: mode=%s cpuCores=%d schonmodus=%v threads=%d workers=%d yoloBatch=%d lowPriority=%v videoMAE=%v autoPause=%v cpuLimit=%d tempLimit=%d targets=%s",
|
|
runtimeOpts.PerformanceMode,
|
|
runtimeOpts.CPUCoreCount,
|
|
runtimeOpts.PowerSaveMode,
|
|
runtimeOpts.CPUThreads,
|
|
runtimeOpts.Workers,
|
|
runtimeOpts.YoloBatchSize,
|
|
runtimeOpts.LowPriority,
|
|
runtimeOpts.VideoMAEEnabled,
|
|
runtimeOpts.AutoPauseEnabled,
|
|
runtimeOpts.AutoPauseCPUPercent,
|
|
runtimeOpts.AutoPauseTemperatureC,
|
|
strings.Join(targets.list(), ","),
|
|
)
|
|
|
|
cleanOutput := func(text string) string {
|
|
out := strings.TrimSpace(text)
|
|
if len(out) > 1500 {
|
|
out = out[:1500] + "…"
|
|
}
|
|
return out
|
|
}
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
s.Progress = 10
|
|
s.Step = "YOLO26-Daten werden geprüft…"
|
|
})
|
|
|
|
detectorOutput := ""
|
|
detectorStatus := "skipped"
|
|
detectorDurationMs := int64(0)
|
|
poseOutput := ""
|
|
poseStatus := "skipped"
|
|
poseDurationMs := int64(0)
|
|
videoMAEOutput := ""
|
|
videoMAEStatus := "skipped"
|
|
videoMAEDurationMs := int64(0)
|
|
|
|
detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml")
|
|
detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train")
|
|
detectorTrainLabels := filepath.Join(root, "detector", "dataset", "labels", "train")
|
|
detectorValImages := filepath.Join(root, "detector", "dataset", "images", "val")
|
|
detectorValLabels := filepath.Join(root, "detector", "dataset", "labels", "val")
|
|
|
|
if err := trainingEnsureDetectorValidationSample(root); err != nil {
|
|
appLogln("⚠️ detector val sample ensure failed:", err)
|
|
}
|
|
|
|
trainCount := trainingCountDetectorSamples(detectorTrainImages, detectorTrainLabels)
|
|
valCount := trainingCountDetectorSamples(detectorValImages, detectorValLabels)
|
|
positiveTrainCount := trainingCountPositiveDetectorSamples(detectorTrainImages, detectorTrainLabels)
|
|
positiveValCount := trainingCountPositiveDetectorSamples(detectorValImages, detectorValLabels)
|
|
|
|
fmt.Printf(
|
|
"🔎 detector data: train=%d (%d positive) val=%d (%d positive) yaml=%v\n",
|
|
trainCount,
|
|
positiveTrainCount,
|
|
valCount,
|
|
positiveValCount,
|
|
fileExistsNonEmpty(detectorDatasetYAML),
|
|
)
|
|
|
|
if !targets.Detector {
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "detector", 1)
|
|
if s.Progress < 58 {
|
|
s.Progress = 58
|
|
}
|
|
s.Step = "YOLO26 Detector wurde nicht ausgewählt."
|
|
})
|
|
detectorStatus = "skipped_unselected"
|
|
detectorOutput = "YOLO26 Detector übersprungen: nicht ausgewählt."
|
|
appLogln(detectorOutput)
|
|
} else if fileExistsNonEmpty(detectorDatasetYAML) &&
|
|
trainCount >= minDetectorTrainCount &&
|
|
valCount >= minDetectorValCount &&
|
|
positiveTrainCount > 0 &&
|
|
positiveValCount > 0 {
|
|
detectorStartedAt := time.Now()
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "detector", 0)
|
|
s.Epochs = trainingDetectorEpochs()
|
|
s.Progress = 15
|
|
s.Step = "YOLO26 Detector wird trainiert…"
|
|
})
|
|
|
|
detectorModel := trainingResolveDetectorModel(root)
|
|
detectorBasePath := detectorModel.BestPath
|
|
if !detectorModel.TrainedExists {
|
|
detectorBasePath = "yolo26n.pt"
|
|
if p, err := trainingMLCacheFilePath("yolo26n.pt"); err == nil {
|
|
detectorBasePath = p
|
|
}
|
|
}
|
|
detectorEpochs := trainingDetectorEpochs()
|
|
detectorPatience := trainingYoloEarlyStoppingPatience(detectorEpochs)
|
|
if detectorModel.TrainedExists {
|
|
appLogln("YOLO26 Detector Fine-Tuning startet von:", detectorBasePath)
|
|
}
|
|
|
|
detectorScript := trainingScriptPath("train_detector_model.py")
|
|
detectorArgs := []string{
|
|
"--root", root,
|
|
"--base", detectorBasePath,
|
|
"--epochs", strconv.Itoa(detectorEpochs),
|
|
"--imgsz", "640",
|
|
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
|
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
|
"--patience", strconv.Itoa(detectorPatience),
|
|
}
|
|
if runtimeOpts.YoloBatchSize > 0 {
|
|
detectorArgs = append(detectorArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
|
}
|
|
|
|
detectorOut, detectorErr := trainingRunCommandStreaming(
|
|
ctx,
|
|
python,
|
|
detectorScript,
|
|
func(line string) bool {
|
|
return trainingHandleProgressLine(
|
|
line,
|
|
15,
|
|
58,
|
|
"YOLO26 Detector wird trainiert…",
|
|
)
|
|
},
|
|
detectorArgs...,
|
|
)
|
|
detectorDurationMs = time.Since(detectorStartedAt).Milliseconds()
|
|
|
|
if errors.Is(detectorErr, errTrainingCancelled) {
|
|
appLogln("⛔ YOLO26 detector training cancelled")
|
|
trainingFinishCancelled(root)
|
|
return
|
|
}
|
|
|
|
detectorOutput = detectorOut
|
|
detectorOutputClean := cleanOutput(detectorOutput)
|
|
|
|
if detectorErr != nil {
|
|
detectorStatus = "failed"
|
|
|
|
appLogln("⚠️ YOLO26 detector training failed:", detectorErr)
|
|
if detectorOutputClean != "" {
|
|
appLogln("⚠️ YOLO26 detector output:", detectorOutputClean)
|
|
}
|
|
} else {
|
|
detectorStatus = "trained"
|
|
|
|
if detectorOutputClean != "" {
|
|
appLogln("✅ YOLO26 detector training:", detectorOutputClean)
|
|
}
|
|
}
|
|
} else {
|
|
detectorStatus = "skipped_no_detector_data"
|
|
detectorOutput = fmt.Sprintf(
|
|
"YOLO26 Detector übersprungen: zu wenige Beispiele. Train=%d (%d positiv), Val=%d (%d positiv). Benötigt: mindestens %d Train, %d Val und je ein positives Beispiel.",
|
|
trainCount,
|
|
positiveTrainCount,
|
|
valCount,
|
|
positiveValCount,
|
|
minDetectorTrainCount,
|
|
minDetectorValCount,
|
|
)
|
|
|
|
appLogln("⚠️", detectorOutput)
|
|
}
|
|
|
|
detectorOutputClean := cleanOutput(detectorOutput)
|
|
poseDatasetYAML := filepath.Join(root, "pose", "dataset", "dataset.yaml")
|
|
poseTrainImages := filepath.Join(root, "pose", "dataset", "images", "train")
|
|
poseTrainLabels := filepath.Join(root, "pose", "dataset", "labels", "train")
|
|
poseValImages := filepath.Join(root, "pose", "dataset", "images", "val")
|
|
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
|
|
poseStartedAt := time.Time{}
|
|
|
|
if !targets.Pose {
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "pose", 1)
|
|
if s.Progress < 82 {
|
|
s.Progress = 82
|
|
}
|
|
s.Step = "YOLO26 Pose wurde nicht ausgewählt."
|
|
})
|
|
poseStatus = "skipped_unselected"
|
|
poseOutput = "YOLO26 Pose übersprungen: nicht ausgewählt."
|
|
appLogln(poseOutput)
|
|
} else {
|
|
poseStartedAt = time.Now()
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "pose", 0)
|
|
s.Epochs = trainingDetectorEpochs()
|
|
if s.Progress < 60 {
|
|
s.Progress = 60
|
|
}
|
|
s.Step = "YOLO26 Pose-Daten werden aufgebaut..."
|
|
})
|
|
|
|
if written, err := trainingSyncPoseDataset(root); err != nil {
|
|
poseStatus = "failed"
|
|
poseOutput = "YOLO26 Pose-Dataset konnte nicht aufgebaut werden: " + err.Error()
|
|
appLogln(poseOutput)
|
|
} else {
|
|
appLogln("pose samples synced:", written)
|
|
}
|
|
|
|
if err := trainingEnsurePoseValidationSample(root); err != nil {
|
|
appLogln("pose val sample ensure failed:", err)
|
|
}
|
|
}
|
|
|
|
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
|
|
poseValCount := trainingCountDetectorSamples(poseValImages, poseValLabels)
|
|
|
|
fmt.Printf(
|
|
"pose data: train=%d val=%d yaml=%v\n",
|
|
poseTrainCount,
|
|
poseValCount,
|
|
fileExistsNonEmpty(poseDatasetYAML),
|
|
)
|
|
|
|
if targets.Pose &&
|
|
poseStatus != "failed" &&
|
|
fileExistsNonEmpty(poseDatasetYAML) &&
|
|
poseTrainCount >= minPoseTrainCount &&
|
|
poseValCount >= minPoseValCount {
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "pose", 0.04)
|
|
s.Epochs = trainingDetectorEpochs()
|
|
if s.Progress < 62 {
|
|
s.Progress = 62
|
|
}
|
|
s.Step = "YOLO26 Pose wird trainiert..."
|
|
})
|
|
|
|
poseModel := trainingResolvePoseModel(root)
|
|
poseBasePath := "yolo26n-pose.pt"
|
|
if poseModel.EffectiveExists {
|
|
poseBasePath = poseModel.EffectivePath
|
|
}
|
|
poseEpochs := trainingDetectorEpochs()
|
|
posePatience := trainingYoloEarlyStoppingPatience(poseEpochs)
|
|
if poseModel.TrainedExists {
|
|
appLogln("YOLO26 Pose Fine-Tuning startet von:", poseBasePath)
|
|
}
|
|
|
|
poseScript := trainingScriptPath("train_pose_model.py")
|
|
poseArgs := []string{
|
|
"--root", root,
|
|
"--base", poseBasePath,
|
|
"--epochs", strconv.Itoa(poseEpochs),
|
|
"--imgsz", "640",
|
|
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
|
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
|
"--patience", strconv.Itoa(posePatience),
|
|
}
|
|
if runtimeOpts.YoloBatchSize > 0 {
|
|
poseArgs = append(poseArgs, "--batch", strconv.Itoa(runtimeOpts.YoloBatchSize))
|
|
}
|
|
|
|
poseOut, poseErr := trainingRunCommandStreaming(
|
|
ctx,
|
|
python,
|
|
poseScript,
|
|
func(line string) bool {
|
|
return trainingHandleProgressLine(
|
|
line,
|
|
62,
|
|
82,
|
|
"YOLO26 Pose wird trainiert...",
|
|
)
|
|
},
|
|
poseArgs...,
|
|
)
|
|
poseDurationMs = time.Since(poseStartedAt).Milliseconds()
|
|
|
|
if errors.Is(poseErr, errTrainingCancelled) {
|
|
appLogln("YOLO26 pose training cancelled")
|
|
trainingFinishCancelled(root)
|
|
return
|
|
}
|
|
|
|
poseOutput = poseOut
|
|
poseOutputClean := cleanOutput(poseOutput)
|
|
|
|
if poseErr != nil {
|
|
poseStatus = "failed"
|
|
|
|
appLogln("YOLO26 pose training failed:", poseErr)
|
|
if poseOutputClean != "" {
|
|
appLogln("YOLO26 pose output:", poseOutputClean)
|
|
}
|
|
} else {
|
|
poseStatus = "trained"
|
|
|
|
if poseOutputClean != "" {
|
|
appLogln("YOLO26 pose training:", poseOutputClean)
|
|
}
|
|
}
|
|
} else if targets.Pose && poseStatus != "failed" {
|
|
poseStatus = "skipped_no_pose_data"
|
|
poseOutput = fmt.Sprintf(
|
|
"YOLO26 Pose übersprungen: zu wenige Skeleton-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
|
poseTrainCount,
|
|
poseValCount,
|
|
minPoseTrainCount,
|
|
minPoseValCount,
|
|
)
|
|
appLogln(poseOutput)
|
|
}
|
|
|
|
if poseStatus == "trained" && poseDurationMs <= 0 && !poseStartedAt.IsZero() {
|
|
poseDurationMs = time.Since(poseStartedAt).Milliseconds()
|
|
}
|
|
|
|
poseOutputClean := cleanOutput(poseOutput)
|
|
videoMAEStartedAt := time.Time{}
|
|
videoMAETrainCount := 0
|
|
videoMAEValCount := 0
|
|
|
|
if !targets.VideoMAE {
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "videomae", 1)
|
|
if s.Progress < 98 {
|
|
s.Progress = 98
|
|
}
|
|
s.Step = "VideoMAE wurde nicht ausgewählt."
|
|
})
|
|
videoMAEStatus = "skipped_unselected"
|
|
videoMAEOutput = "VideoMAE übersprungen: nicht ausgewählt."
|
|
appLogln(videoMAEOutput)
|
|
} else if !runtimeOpts.VideoMAEEnabled {
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "videomae", 1)
|
|
if s.Progress < 84 {
|
|
s.Progress = 84
|
|
}
|
|
s.Step = "VideoMAE wurde in den Settings übersprungen."
|
|
})
|
|
videoMAEStatus = "skipped_disabled"
|
|
videoMAEOutput = "VideoMAE übersprungen: in den Training-Settings deaktiviert."
|
|
appLogln(videoMAEOutput)
|
|
} else {
|
|
videoMAEStartedAt = time.Now()
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "videomae", 0)
|
|
s.Epochs = trainingVideoMAEEpochs()
|
|
if s.Progress < 84 {
|
|
s.Progress = 84
|
|
}
|
|
s.Step = "VideoMAE Clip-Daten werden aufgebaut..."
|
|
})
|
|
|
|
var videoMAEWritten int
|
|
var videoMAESyncErr error
|
|
videoMAETrainCount, videoMAEValCount, videoMAEWritten, videoMAESyncErr =
|
|
trainingSyncVideoMAEDataset(ctx, root)
|
|
if errors.Is(videoMAESyncErr, context.Canceled) || errors.Is(videoMAESyncErr, errTrainingCancelled) {
|
|
appLogln("VideoMAE dataset sync cancelled")
|
|
trainingFinishCancelled(root)
|
|
return
|
|
}
|
|
if videoMAESyncErr != nil {
|
|
videoMAEStatus = "failed"
|
|
videoMAEOutput = "VideoMAE-Dataset konnte nicht aufgebaut werden: " + videoMAESyncErr.Error()
|
|
appLogln(videoMAEOutput)
|
|
} else {
|
|
appLogf(
|
|
"VideoMAE samples synced: written=%d train=%d val=%d",
|
|
videoMAEWritten,
|
|
videoMAETrainCount,
|
|
videoMAEValCount,
|
|
)
|
|
}
|
|
|
|
if videoMAEStatus != "failed" &&
|
|
videoMAETrainCount >= minVideoMAETrainCount &&
|
|
videoMAEValCount >= minVideoMAEValCount {
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
trainingApplyStageProgress(s, "videomae", 0.04)
|
|
s.Epochs = trainingVideoMAEEpochs()
|
|
if s.Progress < 86 {
|
|
s.Progress = 86
|
|
}
|
|
s.Step = "VideoMAE Clip-Classifier wird trainiert..."
|
|
})
|
|
|
|
videoMAEScript := trainingScriptPath("train_videomae_model.py")
|
|
videoMAEBase := strings.TrimSpace(os.Getenv("VIDEOMAE_BASE_MODEL"))
|
|
if videoMAEBase == "" {
|
|
if model := trainingResolveVideoMAEModel(root); model.TrainedExists {
|
|
videoMAEBase = model.EffectivePath
|
|
appLogln("VideoMAE Fine-Tuning startet von:", videoMAEBase)
|
|
}
|
|
}
|
|
videoMAEArgs := []string{
|
|
"--root", root,
|
|
"--epochs", strconv.Itoa(trainingVideoMAEEpochs()),
|
|
"--batch-size", strconv.Itoa(trainingVideoMAEBatchSize()),
|
|
"--num-frames", strconv.Itoa(trainingVideoMAENumFrames),
|
|
"--workers", strconv.Itoa(runtimeOpts.Workers),
|
|
"--threads", strconv.Itoa(runtimeOpts.CPUThreads),
|
|
"--patience", strconv.Itoa(trainingVideoMAEEarlyStoppingPatience()),
|
|
}
|
|
if videoMAEBase != "" {
|
|
videoMAEArgs = append(videoMAEArgs, "--base", videoMAEBase)
|
|
}
|
|
|
|
videoMAEOut, videoMAEErr := trainingRunCommandStreaming(
|
|
ctx,
|
|
python,
|
|
videoMAEScript,
|
|
func(line string) bool {
|
|
return trainingHandleProgressLine(
|
|
line,
|
|
86,
|
|
98,
|
|
"VideoMAE Clip-Classifier wird trainiert...",
|
|
)
|
|
},
|
|
videoMAEArgs...,
|
|
)
|
|
videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds()
|
|
|
|
if errors.Is(videoMAEErr, errTrainingCancelled) {
|
|
appLogln("VideoMAE training cancelled")
|
|
trainingFinishCancelled(root)
|
|
return
|
|
}
|
|
|
|
videoMAEOutput = videoMAEOut
|
|
videoMAEOutputClean := cleanOutput(videoMAEOutput)
|
|
|
|
if videoMAEErr != nil {
|
|
videoMAEStatus = "failed"
|
|
appLogln("VideoMAE training failed:", videoMAEErr)
|
|
if videoMAEOutputClean != "" {
|
|
appLogln("VideoMAE output:", videoMAEOutputClean)
|
|
}
|
|
} else {
|
|
videoMAEStatus = "trained"
|
|
if videoMAEOutputClean != "" {
|
|
appLogln("VideoMAE training:", videoMAEOutputClean)
|
|
}
|
|
}
|
|
} else if videoMAEStatus != "failed" {
|
|
videoMAEStatus = "skipped_no_videomae_data"
|
|
videoMAEOutput = fmt.Sprintf(
|
|
"VideoMAE übersprungen: zu wenige Clip-Beispiele. Train=%d, Val=%d. Benoetigt: mindestens %d Train und %d Val.",
|
|
videoMAETrainCount,
|
|
videoMAEValCount,
|
|
minVideoMAETrainCount,
|
|
minVideoMAEValCount,
|
|
)
|
|
appLogln(videoMAEOutput)
|
|
}
|
|
}
|
|
|
|
if videoMAEStatus == "trained" && videoMAEDurationMs <= 0 && !videoMAEStartedAt.IsZero() {
|
|
videoMAEDurationMs = time.Since(videoMAEStartedAt).Milliseconds()
|
|
}
|
|
|
|
videoMAEOutputClean := cleanOutput(videoMAEOutput)
|
|
|
|
message := "Training abgeschlossen."
|
|
errorText := ""
|
|
|
|
switch detectorStatus {
|
|
case "trained":
|
|
message = "Training abgeschlossen. YOLO26 Detector wurde trainiert."
|
|
|
|
case "skipped_unselected":
|
|
message = "Training abgeschlossen."
|
|
|
|
case "skipped_no_detector_data":
|
|
message = detectorOutput
|
|
|
|
case "failed":
|
|
message = "YOLO26 Detector ist fehlgeschlagen."
|
|
if detectorOutputClean != "" {
|
|
message += " Grund: " + detectorOutputClean
|
|
}
|
|
errorText = message
|
|
|
|
default:
|
|
message = "Training abgeschlossen, aber YOLO26 wurde nicht trainiert."
|
|
if detectorOutputClean != "" {
|
|
message += " Ausgabe: " + detectorOutputClean
|
|
}
|
|
}
|
|
|
|
if poseStatus == "trained" && detectorStatus == "trained" {
|
|
message = "Training abgeschlossen. YOLO26 Detector und YOLO26 Pose wurden trainiert."
|
|
} else if poseStatus == "trained" && detectorStatus != "failed" {
|
|
message = "Training abgeschlossen. YOLO26 Pose wurde trainiert."
|
|
} else if poseStatus == "skipped_no_pose_data" && detectorStatus == "trained" {
|
|
message += " YOLO26 Pose wurde übersprungen: zu wenige Skeleton-Beispiele."
|
|
} else if poseStatus == "failed" {
|
|
if detectorStatus == "failed" {
|
|
message += " YOLO26 Pose ist ebenfalls fehlgeschlagen."
|
|
} else {
|
|
message = "YOLO26 Pose ist fehlgeschlagen."
|
|
}
|
|
if poseOutputClean != "" {
|
|
message += " Grund: " + poseOutputClean
|
|
}
|
|
errorText = message
|
|
}
|
|
|
|
if videoMAEStatus == "trained" {
|
|
if !strings.Contains(message, "Training abgeschlossen.") {
|
|
message = "Training abgeschlossen. " + message
|
|
}
|
|
message += " VideoMAE wurde trainiert."
|
|
} else if videoMAEStatus == "skipped_no_videomae_data" {
|
|
if detectorStatus == "trained" || poseStatus == "trained" {
|
|
message += " VideoMAE wurde übersprungen: zu wenige Clip-Beispiele."
|
|
}
|
|
} else if videoMAEStatus == "skipped_disabled" {
|
|
if detectorStatus == "trained" || poseStatus == "trained" {
|
|
message += " VideoMAE wurde übersprungen: in den Settings deaktiviert."
|
|
}
|
|
} else if videoMAEStatus == "failed" {
|
|
message += " VideoMAE ist fehlgeschlagen."
|
|
if videoMAEOutputClean != "" {
|
|
message += " Grund: " + videoMAEOutputClean
|
|
}
|
|
}
|
|
|
|
if detectorStatus == "trained" || poseStatus == "trained" || videoMAEStatus == "trained" {
|
|
// Verlauf pro erfolgreich trainiertem Ziel schreiben.
|
|
if detectorStatus == "trained" {
|
|
trainingAppendTargetHistory(root, "detector", detectorStatus, detectorDurationMs, runtimeOpts, TrainingHistoryEntry{
|
|
Epochs: trainingDetectorEpochs(),
|
|
TrainSamples: trainCount,
|
|
ValSamples: valCount,
|
|
Imgsz: 640,
|
|
})
|
|
}
|
|
if poseStatus == "trained" {
|
|
trainingAppendTargetHistory(root, "pose", poseStatus, poseDurationMs, runtimeOpts, TrainingHistoryEntry{
|
|
Epochs: trainingDetectorEpochs(),
|
|
TrainSamples: poseTrainCount,
|
|
ValSamples: poseValCount,
|
|
Imgsz: 640,
|
|
})
|
|
}
|
|
if videoMAEStatus == "trained" {
|
|
trainingAppendTargetHistory(root, "videomae", videoMAEStatus, videoMAEDurationMs, runtimeOpts, TrainingHistoryEntry{
|
|
Epochs: trainingVideoMAEEpochs(),
|
|
TrainSamples: videoMAETrainCount,
|
|
ValSamples: videoMAEValCount,
|
|
Imgsz: trainingVideoMAEFrameSize,
|
|
})
|
|
}
|
|
}
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
finishedAt := time.Now().UTC()
|
|
|
|
var durationMs int64
|
|
if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(s.StartedAt)); err == nil {
|
|
durationMs = finishedAt.Sub(startedAt).Milliseconds()
|
|
if durationMs < 0 {
|
|
durationMs = 0
|
|
}
|
|
}
|
|
|
|
s.Running = false
|
|
s.Progress = 100
|
|
if strings.TrimSpace(s.Stage) != "" {
|
|
s.StageProgress = 1
|
|
}
|
|
s.Step = "Training abgeschlossen."
|
|
s.Message = message
|
|
s.Error = errorText
|
|
s.FinishedAt = finishedAt.Format(time.RFC3339)
|
|
s.DurationMs = durationMs
|
|
s.PreviewURL = ""
|
|
s.Paused = false
|
|
s.PauseReason = ""
|
|
s.CPUPercent = 0
|
|
s.TemperatureC = 0
|
|
})
|
|
|
|
trainingClearJobCancel()
|
|
}
|
|
|
|
func trainingStatsHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
stats, err := trainingBuildStats(root)
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusOK, stats)
|
|
}
|
|
|
|
func trainingBuildStats(root string) (*TrainingStatsResponse, error) {
|
|
grouped, err := trainingGroupedLabels()
|
|
if err != nil {
|
|
// Fallback: Stats sollen trotzdem funktionieren, auch wenn Label-Gruppierung scheitert.
|
|
fallbackLabels := defaultTrainingLabelsFromJSON()
|
|
|
|
grouped = TrainingGroupedLabels{
|
|
People: fallbackLabels.People,
|
|
SexPositions: fallbackLabels.SexPositions,
|
|
BodyParts: fallbackLabels.BodyParts,
|
|
Objects: fallbackLabels.Objects,
|
|
Clothing: fallbackLabels.Clothing,
|
|
}
|
|
}
|
|
|
|
peopleSet := stringSet(grouped.People)
|
|
sexPositionSet := stringSet(grouped.SexPositions)
|
|
bodyPartSet := stringSet(grouped.BodyParts)
|
|
objectSet := stringSet(grouped.Objects)
|
|
clothingSet := stringSet(grouped.Clothing)
|
|
|
|
peopleCounts := map[string]int{}
|
|
sexPositionCounts := map[string]int{}
|
|
bodyPartCounts := map[string]int{}
|
|
objectCounts := map[string]int{}
|
|
clothingCounts := map[string]int{}
|
|
|
|
stats := &TrainingStatsResponse{
|
|
OK: true,
|
|
Labels: TrainingStatsLabels{
|
|
People: []TrainingLabelStat{},
|
|
SexPositions: []TrainingLabelStat{},
|
|
BodyParts: []TrainingLabelStat{},
|
|
Objects: []TrainingLabelStat{},
|
|
Clothing: []TrainingLabelStat{},
|
|
},
|
|
}
|
|
|
|
feedbackPath := filepath.Join(root, "feedback.jsonl")
|
|
b, err := os.ReadFile(feedbackPath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
stats.SampleCount = trainingCountSampleFiles(filepath.Join(root, "samples"))
|
|
trainingApplyStatsModelInfo(root, stats)
|
|
return stats, nil
|
|
}
|
|
|
|
return nil, err
|
|
}
|
|
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
var annotation TrainingAnnotation
|
|
if err := json.Unmarshal([]byte(line), &annotation); err != nil {
|
|
continue
|
|
}
|
|
|
|
stats.FeedbackCount++
|
|
|
|
if annotation.Negative {
|
|
stats.NegativeCount++
|
|
} else if annotation.Accepted {
|
|
stats.AcceptedCount++
|
|
} else {
|
|
stats.CorrectedCount++
|
|
}
|
|
|
|
effective := trainingEffectiveCorrection(annotation)
|
|
|
|
sexPosition := normalizeSexPositionLabel(effective.SexPosition)
|
|
if len(sexPositionSet) == 0 || sexPositionSet[sexPosition] {
|
|
sexPositionCounts[sexPosition]++
|
|
}
|
|
|
|
for _, label := range effective.BodyPartsPresent {
|
|
clean := strings.TrimSpace(label)
|
|
if clean == "" {
|
|
continue
|
|
}
|
|
if len(bodyPartSet) == 0 || bodyPartSet[clean] {
|
|
bodyPartCounts[clean]++
|
|
}
|
|
}
|
|
|
|
for _, label := range effective.ObjectsPresent {
|
|
clean := strings.TrimSpace(label)
|
|
if clean == "" {
|
|
continue
|
|
}
|
|
if len(objectSet) == 0 || objectSet[clean] {
|
|
objectCounts[clean]++
|
|
}
|
|
}
|
|
|
|
for _, label := range effective.ClothingPresent {
|
|
clean := strings.TrimSpace(label)
|
|
if clean == "" {
|
|
continue
|
|
}
|
|
if len(clothingSet) == 0 || clothingSet[clean] {
|
|
clothingCounts[clean]++
|
|
}
|
|
}
|
|
|
|
for _, box := range effective.Boxes {
|
|
label := strings.TrimSpace(box.Label)
|
|
if label == "" {
|
|
continue
|
|
}
|
|
|
|
stats.BoxCount++
|
|
|
|
switch {
|
|
case peopleSet[label]:
|
|
peopleCounts[label]++
|
|
|
|
case bodyPartSet[label]:
|
|
bodyPartCounts[label]++
|
|
|
|
case objectSet[label]:
|
|
objectCounts[label]++
|
|
|
|
case clothingSet[label]:
|
|
clothingCounts[label]++
|
|
}
|
|
}
|
|
}
|
|
|
|
stats.SampleCount = trainingCountSampleFiles(filepath.Join(root, "samples"))
|
|
trainingApplyStatsModelInfo(root, stats)
|
|
|
|
stats.Labels = TrainingStatsLabels{
|
|
// Personen/Box-Labels brauchen mehr Beispiele, weil der Detector Boxen lernen muss.
|
|
People: trainingStatsMapToList(peopleCounts, 20),
|
|
|
|
// Scene-Positionen sind Sample-Labels, hier reichen grob weniger pro Klasse.
|
|
SexPositions: trainingStatsMapToList(sexPositionCounts, 8),
|
|
|
|
// Detector-Klassen: grob 15 Beispiele pro Label als solide Untergrenze.
|
|
BodyParts: trainingStatsMapToList(bodyPartCounts, 15),
|
|
Objects: trainingStatsMapToList(objectCounts, 15),
|
|
Clothing: trainingStatsMapToList(clothingCounts, 15),
|
|
}
|
|
|
|
stats.Confidence = trainingOverallConfidence(
|
|
stats.FeedbackCount,
|
|
stats.BoxCount,
|
|
stats.AcceptedCount,
|
|
stats.CorrectedCount,
|
|
stats.Labels,
|
|
)
|
|
|
|
return stats, nil
|
|
}
|
|
|
|
func trainingEffectiveCorrection(annotation TrainingAnnotation) TrainingCorrection {
|
|
if annotation.Negative {
|
|
return *trainingNegativeCorrection()
|
|
}
|
|
|
|
if annotation.Correction != nil {
|
|
return *annotation.Correction
|
|
}
|
|
|
|
p := annotation.Prediction
|
|
|
|
return TrainingCorrection{
|
|
SexPosition: p.SexPosition,
|
|
PeoplePresent: trainingScoredLabelsToStrings(p.PeoplePresent),
|
|
BodyPartsPresent: trainingScoredLabelsToStrings(p.BodyPartsPresent),
|
|
ObjectsPresent: trainingScoredLabelsToStrings(p.ObjectsPresent),
|
|
ClothingPresent: trainingScoredLabelsToStrings(p.ClothingPresent),
|
|
Boxes: p.Boxes,
|
|
PosePersons: p.Persons,
|
|
}
|
|
}
|
|
|
|
func trainingScoredLabelsToStrings(values []TrainingScoredLabel) []string {
|
|
out := make([]string, 0, len(values))
|
|
seen := map[string]bool{}
|
|
|
|
for _, value := range values {
|
|
label := strings.TrimSpace(value.Label)
|
|
if label == "" || seen[label] {
|
|
continue
|
|
}
|
|
|
|
seen[label] = true
|
|
out = append(out, label)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingStatsMapToList(values map[string]int, target int) []TrainingLabelStat {
|
|
out := make([]TrainingLabelStat, 0, len(values))
|
|
|
|
for label, count := range values {
|
|
label = strings.TrimSpace(label)
|
|
if label == "" || count <= 0 {
|
|
continue
|
|
}
|
|
|
|
out = append(out, TrainingLabelStat{
|
|
Label: label,
|
|
Count: count,
|
|
Confidence: trainingLabelConfidence(count, target),
|
|
})
|
|
}
|
|
|
|
sort.Slice(out, func(i, j int) bool {
|
|
if out[i].Count == out[j].Count {
|
|
return out[i].Label < out[j].Label
|
|
}
|
|
|
|
return out[i].Count > out[j].Count
|
|
})
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingCountSampleFiles(samplesDir string) int {
|
|
entries, err := os.ReadDir(samplesDir)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
count := 0
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
|
|
if strings.ToLower(filepath.Ext(entry.Name())) == ".json" {
|
|
count++
|
|
}
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
job := trainingGetJobStatus()
|
|
|
|
if !job.Running {
|
|
if err := trainingEnsureDetectorDirs(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
if err := trainingEnsurePoseDirs(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
if err := trainingEnsureVideoMAEDirs(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
if err := trainingEnsureDetectorValidationSample(root); err != nil {
|
|
appLogln("⚠️ detector val sample ensure failed:", err)
|
|
}
|
|
if err := trainingEnsurePoseValidationSample(root); err != nil {
|
|
appLogln("pose val sample ensure failed:", err)
|
|
}
|
|
}
|
|
|
|
feedbackPath := filepath.Join(root, "feedback.jsonl")
|
|
feedbackCount, _ := trainingCountAnnotations(feedbackPath)
|
|
|
|
detectorDatasetYAML := filepath.Join(root, "detector", "dataset", "dataset.yaml")
|
|
detectorTrainImages := filepath.Join(root, "detector", "dataset", "images", "train")
|
|
detectorTrainLabels := filepath.Join(root, "detector", "dataset", "labels", "train")
|
|
detectorValImages := filepath.Join(root, "detector", "dataset", "images", "val")
|
|
detectorValLabels := filepath.Join(root, "detector", "dataset", "labels", "val")
|
|
poseDatasetYAML := filepath.Join(root, "pose", "dataset", "dataset.yaml")
|
|
poseTrainImages := filepath.Join(root, "pose", "dataset", "images", "train")
|
|
poseTrainLabels := filepath.Join(root, "pose", "dataset", "labels", "train")
|
|
poseValImages := filepath.Join(root, "pose", "dataset", "images", "val")
|
|
poseValLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
|
|
detectorModel := trainingResolveDetectorModel(root)
|
|
poseModel := trainingResolvePoseModel(root)
|
|
videoMAEModel := trainingResolveVideoMAEModel(root)
|
|
videoMAEManifest := trainingVideoMAEManifestPath(root)
|
|
|
|
trainCount := trainingCountDetectorSamples(detectorTrainImages, detectorTrainLabels)
|
|
valCount := trainingCountDetectorSamples(detectorValImages, detectorValLabels)
|
|
positiveTrainCount := trainingCountPositiveDetectorSamples(detectorTrainImages, detectorTrainLabels)
|
|
positiveValCount := trainingCountPositiveDetectorSamples(detectorValImages, detectorValLabels)
|
|
poseTrainCount := trainingCountDetectorSamples(poseTrainImages, poseTrainLabels)
|
|
poseValCount := trainingCountDetectorSamples(poseValImages, poseValLabels)
|
|
videoMAETrainCount, videoMAEValCount := trainingCountVideoMAEManifestSamples(root)
|
|
videoMAEEligibleCount, _ := trainingCountVideoMAEEligibleAnnotations(root)
|
|
|
|
datasetReady := fileExistsNonEmpty(detectorDatasetYAML)
|
|
detectorDataReady := datasetReady &&
|
|
trainCount >= minDetectorTrainCount &&
|
|
valCount >= minDetectorValCount &&
|
|
positiveTrainCount > 0 &&
|
|
positiveValCount > 0
|
|
poseDatasetReady := fileExistsNonEmpty(poseDatasetYAML)
|
|
poseDataReady := poseDatasetReady &&
|
|
poseTrainCount >= minPoseTrainCount &&
|
|
poseValCount >= minPoseValCount
|
|
videoMAEDatasetReady := fileExistsNonEmpty(videoMAEManifest)
|
|
videoMAEDataReady := (videoMAETrainCount >= minVideoMAETrainCount &&
|
|
videoMAEValCount >= minVideoMAEValCount) ||
|
|
videoMAEEligibleCount >= minVideoMAETrainCount
|
|
|
|
canTrain := feedbackCount >= minTrainingFeedbackCount &&
|
|
(detectorDataReady || poseDataReady || videoMAEDataReady)
|
|
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
"feedbackCount": feedbackCount,
|
|
"requiredCount": minTrainingFeedbackCount,
|
|
"canTrain": canTrain,
|
|
|
|
"training": job,
|
|
|
|
"detector": map[string]any{
|
|
"source": "yolo26_detector",
|
|
"usesSceneCLIP": false,
|
|
"usesSceneKNN": false,
|
|
"usesResNet18KNN": false,
|
|
|
|
"detectsPeople": true,
|
|
"detectsGender": true,
|
|
"detectsSexPosition": false,
|
|
"detectsBodyParts": true,
|
|
"detectsObjects": true,
|
|
"detectsClothing": true,
|
|
"detectsBoxes": true,
|
|
|
|
"trainCount": trainCount,
|
|
"valCount": valCount,
|
|
"positiveTrainCount": positiveTrainCount,
|
|
"positiveValCount": positiveValCount,
|
|
"requiredTrain": minDetectorTrainCount,
|
|
"requiredVal": minDetectorValCount,
|
|
|
|
"datasetReady": datasetReady,
|
|
"datasetYAML": detectorDatasetYAML,
|
|
"dataReady": detectorDataReady,
|
|
|
|
"modelExists": detectorModel.EffectiveExists,
|
|
"modelPath": detectorModel.EffectivePath,
|
|
"trainedModelExists": detectorModel.TrainedExists,
|
|
"trainedModelPath": detectorModel.BestPath,
|
|
"modelSource": detectorModel.Source,
|
|
},
|
|
|
|
"pose": map[string]any{
|
|
"source": "yolo26_pose",
|
|
"usesKeypoints": true,
|
|
"predictsPersons": true,
|
|
"predictsSexPosition": poseModel.TrainedExists,
|
|
"trainedFromFeedback": true,
|
|
"trainCount": poseTrainCount,
|
|
"valCount": poseValCount,
|
|
"requiredTrain": minPoseTrainCount,
|
|
"requiredVal": minPoseValCount,
|
|
"datasetReady": poseDatasetReady,
|
|
"datasetYAML": poseDatasetYAML,
|
|
"dataReady": poseDataReady,
|
|
"modelExists": poseModel.EffectiveExists,
|
|
"modelPath": poseModel.EffectivePath,
|
|
"trainedModelExists": poseModel.TrainedExists,
|
|
"trainedModelPath": poseModel.BestPath,
|
|
"modelSource": poseModel.Source,
|
|
},
|
|
|
|
"scene": map[string]any{
|
|
"source": "videomae_clip",
|
|
"usesVideoMAE": true,
|
|
"usesSceneCLIP": false,
|
|
"usesSceneKNN": false,
|
|
"usesResNet18KNN": false,
|
|
"usesLogisticRegression": false,
|
|
|
|
"predictsSexPosition": videoMAEModel.TrainedExists,
|
|
"predictsPeople": false,
|
|
"predictsGender": false,
|
|
"predictsBodyParts": false,
|
|
"predictsObjects": false,
|
|
"predictsClothing": false,
|
|
"predictsBoxes": false,
|
|
|
|
"feedbackCount": feedbackCount,
|
|
"eligibleCount": videoMAEEligibleCount,
|
|
"trainCount": videoMAETrainCount,
|
|
"valCount": videoMAEValCount,
|
|
"requiredTrain": minVideoMAETrainCount,
|
|
"requiredVal": minVideoMAEValCount,
|
|
"requiredCount": minVideoMAETrainCount,
|
|
"datasetReady": videoMAEDatasetReady,
|
|
"manifest": videoMAEManifest,
|
|
"dataReady": videoMAEDataReady,
|
|
"modelReady": videoMAEModel.EffectiveExists,
|
|
"modelExists": videoMAEModel.EffectiveExists,
|
|
"modelPath": videoMAEModel.EffectivePath,
|
|
"modelSource": videoMAEModel.Source,
|
|
"trainedModelExists": videoMAEModel.TrainedExists,
|
|
"trainedModelPath": videoMAEModel.BestPath,
|
|
},
|
|
|
|
"pipeline": map[string]any{
|
|
"variant": "YOLO26_VIDEO_CLIP_HYBRID",
|
|
|
|
"peopleSource": "yolo26_detector",
|
|
"genderSource": "yolo26_detector",
|
|
"sexPositionSource": "yolo26_pose+box_context+videomae_clip",
|
|
"bodyPartsSource": "yolo26_detector",
|
|
"objectsSource": "yolo26_detector",
|
|
"clothingSource": "yolo26_detector",
|
|
"boxesSource": "yolo26_detector",
|
|
|
|
"usesSceneKNNForDetection": false,
|
|
"usesSceneCLIP": false,
|
|
"usesSceneKNN": false,
|
|
"usesVideoMAE": true,
|
|
"usesYOLOForDetection": true,
|
|
"usesYOLOForSexPosition": true,
|
|
},
|
|
})
|
|
}
|
|
|
|
func trainingApplyStatsModelInfo(root string, stats *TrainingStatsResponse) {
|
|
detectorAvailable := trainingStatsModelAvailableFor(root, "detector")
|
|
detectorInfo := trainingReadModelInfoFor(root, "detector")
|
|
poseAvailable := trainingStatsModelAvailableFor(root, "pose")
|
|
poseInfo := trainingReadModelInfoFor(root, "pose")
|
|
videoMAEModel := trainingResolveVideoMAEModel(root)
|
|
videoMAEInfo := trainingReadModelInfoFor(root, "videomae")
|
|
|
|
// modelAvailable/modelInfo bleiben aus Kompatibilitaetsgruenden der Detector.
|
|
stats.ModelAvailable = detectorAvailable
|
|
stats.ModelInfo = detectorInfo
|
|
stats.DetectorModelAvailable = detectorAvailable
|
|
stats.DetectorModelInfo = detectorInfo
|
|
stats.PoseModelAvailable = poseAvailable
|
|
stats.PoseModelInfo = poseInfo
|
|
stats.VideoMAEModelAvailable = videoMAEModel.EffectiveExists
|
|
stats.VideoMAEModelInfo = videoMAEInfo
|
|
}
|
|
|
|
func trainingStatsModelAvailable(root string) bool {
|
|
return trainingStatsModelAvailableFor(root, "detector")
|
|
}
|
|
|
|
func trainingStatsModelAvailableFor(root string, kind string) bool {
|
|
modelPath := filepath.Join(root, kind, "model", "best.pt")
|
|
if kind == "videomae" {
|
|
modelPath = filepath.Join(root, kind, "model", "config.json")
|
|
}
|
|
return fileExistsNonEmpty(modelPath)
|
|
}
|
|
|
|
// trainingReadModelInfo liest Versions-/Datums-Infos zum aktuell trainierten
|
|
// Detector-Modell. Datum/Version stammen primär aus status.json (vom Trainingsskript),
|
|
// Fallback ist die Änderungszeit der best.pt-Datei.
|
|
func trainingReadModelInfo(root string) *TrainingModelInfo {
|
|
return trainingReadModelInfoFor(root, "detector")
|
|
}
|
|
|
|
func trainingReadModelInfoFor(root string, kind string) *TrainingModelInfo {
|
|
modelPath := filepath.Join(root, kind, "model", "best.pt")
|
|
if kind == "videomae" {
|
|
modelPath = filepath.Join(root, kind, "model", "config.json")
|
|
}
|
|
|
|
fi, err := os.Stat(modelPath)
|
|
if err != nil || fi.IsDir() || fi.Size() <= 0 {
|
|
return nil
|
|
}
|
|
|
|
info := &TrainingModelInfo{
|
|
TrainedAt: fi.ModTime().UTC().Format(time.RFC3339),
|
|
TrainedAtMs: fi.ModTime().UnixMilli(),
|
|
}
|
|
|
|
statusPath := filepath.Join(root, kind, "model", "status.json")
|
|
if b, err := os.ReadFile(statusPath); err == nil {
|
|
var raw struct {
|
|
TrainedAt string `json:"trainedAt"`
|
|
Epochs int `json:"epochs"`
|
|
TrainSamples int `json:"trainSamples"`
|
|
ValSamples int `json:"valSamples"`
|
|
Imgsz int `json:"imgsz"`
|
|
Device string `json:"device"`
|
|
MAP50 float64 `json:"mAP50"`
|
|
MAP5095 float64 `json:"mAP5095"`
|
|
}
|
|
|
|
if json.Unmarshal(b, &raw) == nil {
|
|
if trimmed := strings.TrimSpace(raw.TrainedAt); trimmed != "" {
|
|
if t, err := time.Parse(time.RFC3339, trimmed); err == nil {
|
|
info.TrainedAt = t.UTC().Format(time.RFC3339)
|
|
info.TrainedAtMs = t.UnixMilli()
|
|
}
|
|
}
|
|
|
|
info.Epochs = raw.Epochs
|
|
info.TrainSamples = raw.TrainSamples
|
|
info.ValSamples = raw.ValSamples
|
|
info.Imgsz = raw.Imgsz
|
|
info.Device = strings.TrimSpace(raw.Device)
|
|
info.MAP50 = raw.MAP50
|
|
info.MAP5095 = raw.MAP5095
|
|
}
|
|
}
|
|
|
|
return info
|
|
}
|
|
|
|
type TrainingHistoryEntry struct {
|
|
TrainedAt string `json:"trainedAt,omitempty"`
|
|
TrainedAtMs int64 `json:"trainedAtMs,omitempty"`
|
|
Target string `json:"target,omitempty"`
|
|
Status string `json:"status,omitempty"`
|
|
DurationMs int64 `json:"durationMs,omitempty"`
|
|
Epochs int `json:"epochs,omitempty"`
|
|
TrainSamples int `json:"trainSamples,omitempty"`
|
|
ValSamples int `json:"valSamples,omitempty"`
|
|
Imgsz int `json:"imgsz,omitempty"`
|
|
Device string `json:"device,omitempty"`
|
|
MAP50 float64 `json:"map50,omitempty"`
|
|
MAP5095 float64 `json:"map5095,omitempty"`
|
|
PerformanceMode string `json:"performanceMode,omitempty"`
|
|
CPUCoreCount int `json:"cpuCoreCount,omitempty"`
|
|
CPUThreads int `json:"cpuThreads,omitempty"`
|
|
Workers int `json:"workers,omitempty"`
|
|
YoloBatchSize int `json:"yoloBatchSize,omitempty"`
|
|
LowPriority bool `json:"lowPriority,omitempty"`
|
|
AutoPauseEnabled bool `json:"autoPauseEnabled,omitempty"`
|
|
AutoPauseCPUPercent int `json:"autoPauseCpuPercent,omitempty"`
|
|
AutoPauseTemperatureC int `json:"autoPauseTemperatureC,omitempty"`
|
|
}
|
|
|
|
type TrainingHistoryResponse struct {
|
|
OK bool `json:"ok"`
|
|
Entries []TrainingHistoryEntry `json:"entries"`
|
|
}
|
|
|
|
func trainingHistoryPath(root string) string {
|
|
return filepath.Join(root, "detector", "training_history.jsonl")
|
|
}
|
|
|
|
// trainingAppendTargetHistory haengt nach einem erfolgreichen Trainingsziel einen
|
|
// Verlaufseintrag an. Alte History-Zeilen ohne Target bleiben weiterhin lesbar.
|
|
func trainingHistoryKindForTarget(target string) string {
|
|
switch strings.ToLower(strings.TrimSpace(target)) {
|
|
case "pose":
|
|
return "pose"
|
|
case "videomae", "video_mae", "scene":
|
|
return "videomae"
|
|
default:
|
|
return "detector"
|
|
}
|
|
}
|
|
|
|
func trainingAppendTargetHistory(root string, target string, status string, durationMs int64, runtimeOpts trainingRuntimeOptions, fallback TrainingHistoryEntry) {
|
|
kind := trainingHistoryKindForTarget(target)
|
|
info := trainingReadModelInfoFor(root, kind)
|
|
now := time.Now().UTC()
|
|
|
|
entry := TrainingHistoryEntry{
|
|
TrainedAt: now.Format(time.RFC3339),
|
|
TrainedAtMs: now.UnixMilli(),
|
|
Target: kind,
|
|
Status: strings.TrimSpace(status),
|
|
DurationMs: durationMs,
|
|
Epochs: fallback.Epochs,
|
|
TrainSamples: fallback.TrainSamples,
|
|
ValSamples: fallback.ValSamples,
|
|
Imgsz: fallback.Imgsz,
|
|
Device: strings.TrimSpace(fallback.Device),
|
|
MAP50: fallback.MAP50,
|
|
MAP5095: fallback.MAP5095,
|
|
PerformanceMode: runtimeOpts.PerformanceMode,
|
|
CPUCoreCount: runtimeOpts.CPUCoreCount,
|
|
CPUThreads: runtimeOpts.CPUThreads,
|
|
Workers: runtimeOpts.Workers,
|
|
YoloBatchSize: runtimeOpts.YoloBatchSize,
|
|
LowPriority: runtimeOpts.LowPriority,
|
|
AutoPauseEnabled: runtimeOpts.AutoPauseEnabled,
|
|
AutoPauseCPUPercent: runtimeOpts.AutoPauseCPUPercent,
|
|
AutoPauseTemperatureC: runtimeOpts.AutoPauseTemperatureC,
|
|
}
|
|
|
|
if info != nil {
|
|
if strings.TrimSpace(info.TrainedAt) != "" {
|
|
entry.TrainedAt = info.TrainedAt
|
|
}
|
|
if info.TrainedAtMs > 0 {
|
|
entry.TrainedAtMs = info.TrainedAtMs
|
|
}
|
|
if info.Epochs > 0 {
|
|
entry.Epochs = info.Epochs
|
|
}
|
|
if info.TrainSamples > 0 {
|
|
entry.TrainSamples = info.TrainSamples
|
|
}
|
|
if info.ValSamples > 0 {
|
|
entry.ValSamples = info.ValSamples
|
|
}
|
|
if info.Imgsz > 0 {
|
|
entry.Imgsz = info.Imgsz
|
|
}
|
|
if strings.TrimSpace(info.Device) != "" {
|
|
entry.Device = strings.TrimSpace(info.Device)
|
|
}
|
|
if info.MAP50 > 0 {
|
|
entry.MAP50 = info.MAP50
|
|
}
|
|
if info.MAP5095 > 0 {
|
|
entry.MAP5095 = info.MAP5095
|
|
}
|
|
}
|
|
|
|
if entry.DurationMs <= 0 {
|
|
job := trainingGetJobStatus()
|
|
if startedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(job.StartedAt)); err == nil {
|
|
if ms := now.Sub(startedAt).Milliseconds(); ms > 0 {
|
|
entry.DurationMs = ms
|
|
}
|
|
}
|
|
}
|
|
|
|
b, err := json.Marshal(entry)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
path := trainingHistoryPath(root)
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
appLogln("⚠️ training history dir konnte nicht erstellt werden:", err)
|
|
return
|
|
}
|
|
|
|
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
|
if err != nil {
|
|
appLogln("⚠️ training history konnte nicht geöffnet werden:", err)
|
|
return
|
|
}
|
|
defer f.Close()
|
|
|
|
if _, err := f.Write(append(b, '\n')); err != nil {
|
|
appLogln("⚠️ training history konnte nicht geschrieben werden:", err)
|
|
}
|
|
}
|
|
|
|
// trainingReadHistory liest die Verlaufseinträge, neueste zuerst.
|
|
func trainingReadHistory(root string, limit int) []TrainingHistoryEntry {
|
|
b, err := os.ReadFile(trainingHistoryPath(root))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
lines := strings.Split(string(b), "\n")
|
|
entries := make([]TrainingHistoryEntry, 0, len(lines))
|
|
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
var e TrainingHistoryEntry
|
|
if json.Unmarshal([]byte(line), &e) != nil {
|
|
continue
|
|
}
|
|
|
|
entries = append(entries, e)
|
|
}
|
|
|
|
// Datei ist chronologisch → umdrehen, damit der neueste Lauf oben steht.
|
|
for i, j := 0, len(entries)-1; i < j; i, j = i+1, j-1 {
|
|
entries[i], entries[j] = entries[j], entries[i]
|
|
}
|
|
|
|
if limit > 0 && len(entries) > limit {
|
|
entries = entries[:limit]
|
|
}
|
|
|
|
return entries
|
|
}
|
|
|
|
func trainingHistoryHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
entries := trainingReadHistory(root, 50)
|
|
if entries == nil {
|
|
entries = []TrainingHistoryEntry{}
|
|
}
|
|
|
|
trainingWriteJSON(w, http.StatusOK, TrainingHistoryResponse{
|
|
OK: true,
|
|
Entries: entries,
|
|
})
|
|
}
|
|
|
|
func trainingConfidenceFromScore(score float64) TrainingConfidence {
|
|
if math.IsNaN(score) || math.IsInf(score, 0) {
|
|
score = 0
|
|
}
|
|
|
|
score = clamp01(score)
|
|
|
|
level := "none"
|
|
label := "Keine"
|
|
|
|
switch {
|
|
case score >= 0.75:
|
|
level = "high"
|
|
label = "Hoch"
|
|
case score >= 0.45:
|
|
level = "mid"
|
|
label = "Mittel"
|
|
case score > 0:
|
|
level = "low"
|
|
label = "Niedrig"
|
|
}
|
|
|
|
return TrainingConfidence{
|
|
Score: score,
|
|
Level: level,
|
|
Label: label,
|
|
}
|
|
}
|
|
|
|
func trainingLabelConfidence(count int, target int) TrainingConfidence {
|
|
if target <= 0 {
|
|
target = 10
|
|
}
|
|
|
|
if count <= 0 {
|
|
return trainingConfidenceFromScore(0)
|
|
}
|
|
|
|
// Grobe Datenabdeckung: target erreicht = 100%.
|
|
// sqrt macht kleine Mengen etwas weniger hart, aber 1 Treffer bleibt niedrig.
|
|
score := math.Sqrt(float64(count) / float64(target*2))
|
|
|
|
return trainingConfidenceFromScore(score)
|
|
}
|
|
|
|
func trainingSaturationScore(value int, target int) float64 {
|
|
if value <= 0 || target <= 0 {
|
|
return 0
|
|
}
|
|
|
|
// Sanfter Anstieg, aber nie über 1.
|
|
return clamp01(math.Sqrt(float64(value) / float64(target)))
|
|
}
|
|
|
|
func trainingAverageLabelConfidence(labels TrainingStatsLabels) float64 {
|
|
values := []float64{}
|
|
|
|
appendScores := func(items []TrainingLabelStat) {
|
|
for _, item := range items {
|
|
values = append(values, clamp01(item.Confidence.Score))
|
|
}
|
|
}
|
|
|
|
appendScores(labels.People)
|
|
appendScores(labels.SexPositions)
|
|
appendScores(labels.BodyParts)
|
|
appendScores(labels.Objects)
|
|
appendScores(labels.Clothing)
|
|
|
|
if len(values) == 0 {
|
|
return 0
|
|
}
|
|
|
|
sum := 0.0
|
|
for _, value := range values {
|
|
sum += value
|
|
}
|
|
|
|
return clamp01(sum / float64(len(values)))
|
|
}
|
|
|
|
func trainingOverallConfidence(
|
|
feedbackCount int,
|
|
boxCount int,
|
|
acceptedCount int,
|
|
correctedCount int,
|
|
labels TrainingStatsLabels,
|
|
) TrainingConfidence {
|
|
if feedbackCount <= 0 {
|
|
return trainingConfidenceFromScore(0)
|
|
}
|
|
|
|
// Datenmenge: 300 Feedbacks sind grob "voll", darunter anteilig.
|
|
feedbackScore := trainingSaturationScore(feedbackCount, 300)
|
|
|
|
// Detector-Daten: 1000 Boxen sind grob "voll", darunter anteilig.
|
|
boxScore := trainingSaturationScore(boxCount, 1000)
|
|
|
|
// Label-Abdeckung aus den einzelnen Label-Confidence-Werten.
|
|
labelScore := trainingAverageLabelConfidence(labels)
|
|
|
|
// Modell-/Prediction-Zustimmung:
|
|
// Viele "Passt so"-Antworten bedeuten, dass die Vorhersagen brauchbar sind.
|
|
// Bei 4/229 ist dieser Teil bewusst sehr niedrig.
|
|
agreementScore := 0.0
|
|
decisionCount := acceptedCount + correctedCount
|
|
if decisionCount > 0 {
|
|
agreementScore = clamp01(float64(acceptedCount) / float64(decisionCount))
|
|
}
|
|
|
|
// Korrekturquote als zusätzlicher Dämpfer.
|
|
// 98% korrigiert soll die Gesamt-Confidence sichtbar drücken,
|
|
// aber nicht alle gesammelten Daten entwerten.
|
|
correctionRate := 0.0
|
|
if decisionCount > 0 {
|
|
correctionRate = clamp01(float64(correctedCount) / float64(decisionCount))
|
|
}
|
|
|
|
correctionPenalty := 1.0 - math.Min(0.45, correctionRate*0.45)
|
|
|
|
// Gesamt:
|
|
// - Datenmenge zählt viel
|
|
// - Boxen und Label-Abdeckung zählen mittel
|
|
// - echte Modell-Zustimmung zählt bewusst mit rein
|
|
score :=
|
|
feedbackScore*0.30 +
|
|
boxScore*0.25 +
|
|
labelScore*0.25 +
|
|
agreementScore*0.20
|
|
|
|
score *= correctionPenalty
|
|
|
|
return trainingConfidenceFromScore(score)
|
|
}
|
|
|
|
func stringSet(values []string) map[string]bool {
|
|
out := map[string]bool{}
|
|
|
|
for _, value := range values {
|
|
clean := strings.TrimSpace(value)
|
|
if clean == "" {
|
|
continue
|
|
}
|
|
|
|
out[clean] = true
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingRecognitionEnabled() bool {
|
|
return getSettings().TrainingRecognitionEnabled
|
|
}
|
|
|
|
func trainingDetectorEpochs() int {
|
|
epochs := getSettings().TrainingDetectorEpochs
|
|
|
|
if epochs < 1 {
|
|
return 60
|
|
}
|
|
|
|
if epochs > 300 {
|
|
return 300
|
|
}
|
|
|
|
return epochs
|
|
}
|
|
|
|
func trainingDeleteAllHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodDelete && r.Method != http.MethodPost {
|
|
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
|
return
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
if err := os.RemoveAll(root); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, fmt.Sprintf("Trainingsdaten konnten nicht gelöscht werden: %v", err))
|
|
return
|
|
}
|
|
|
|
// Basisordner direkt wieder anlegen, damit die nächsten API-Aufrufe sauber funktionieren.
|
|
if _, err := trainingRootDir(); err != nil {
|
|
trainingWriteError(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
trainingSetJobStatus(func(s *TrainingJobStatus) {
|
|
*s = TrainingJobStatus{}
|
|
})
|
|
|
|
trainingWriteJSON(w, http.StatusOK, map[string]any{
|
|
"ok": true,
|
|
"message": "Alle Trainingsdaten wurden gelöscht.",
|
|
})
|
|
}
|
|
|
|
func trainingCountDetectorSamples(imagesDir string, labelsDir string) int {
|
|
imageExts := map[string]bool{
|
|
".jpg": true,
|
|
".jpeg": true,
|
|
".png": true,
|
|
".webp": true,
|
|
}
|
|
|
|
entries, err := os.ReadDir(imagesDir)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
count := 0
|
|
|
|
for _, e := range entries {
|
|
if e.IsDir() {
|
|
continue
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(e.Name()))
|
|
if !imageExts[ext] {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name()))
|
|
labelPath := filepath.Join(labelsDir, id+".txt")
|
|
|
|
if fileExists(labelPath) {
|
|
count++
|
|
}
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
func trainingCountPositiveDetectorSamples(imagesDir string, labelsDir string) int {
|
|
imageExts := map[string]bool{
|
|
".jpg": true,
|
|
".jpeg": true,
|
|
".png": true,
|
|
".webp": true,
|
|
}
|
|
|
|
entries, err := os.ReadDir(imagesDir)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
count := 0
|
|
|
|
for _, e := range entries {
|
|
if e.IsDir() {
|
|
continue
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(e.Name()))
|
|
if !imageExts[ext] {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name()))
|
|
labelPath := filepath.Join(labelsDir, id+".txt")
|
|
|
|
if fileExistsNonEmpty(labelPath) {
|
|
count++
|
|
}
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
func trainingWriteDetectorDatasetYAML(root string) error {
|
|
datasetDir := filepath.Join(root, "detector", "dataset")
|
|
|
|
absDatasetDir, err := filepath.Abs(datasetDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
namesYAML, err := trainingDetectorDatasetNamesYAML()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
path := filepath.Join(datasetDir, "dataset.yaml")
|
|
yoloPath := filepath.ToSlash(absDatasetDir)
|
|
|
|
body := fmt.Sprintf(`path: %s
|
|
train: images/train
|
|
val: images/val
|
|
|
|
names:
|
|
%s`, yoloPath, namesYAML)
|
|
|
|
return os.WriteFile(path, []byte(body), 0644)
|
|
}
|
|
|
|
func trainingWritePoseDatasetYAML(root string) error {
|
|
datasetDir := filepath.Join(root, "pose", "dataset")
|
|
|
|
absDatasetDir, err := filepath.Abs(datasetDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
namesYAML, err := trainingPoseDatasetNamesYAML()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
path := filepath.Join(datasetDir, "dataset.yaml")
|
|
yoloPath := filepath.ToSlash(absDatasetDir)
|
|
|
|
body := fmt.Sprintf(`path: %s
|
|
train: images/train
|
|
val: images/val
|
|
|
|
kpt_shape: [17, 3]
|
|
flip_idx: [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15]
|
|
|
|
names:
|
|
%s`, yoloPath, namesYAML)
|
|
|
|
return os.WriteFile(path, []byte(body), 0644)
|
|
}
|
|
|
|
func trainingEnsureDetectorDirs(root string) error {
|
|
dirs := []string{
|
|
filepath.Join(root, "detector"),
|
|
filepath.Join(root, "detector", "dataset"),
|
|
filepath.Join(root, "detector", "dataset", "images"),
|
|
filepath.Join(root, "detector", "dataset", "images", "train"),
|
|
filepath.Join(root, "detector", "dataset", "images", "val"),
|
|
filepath.Join(root, "detector", "dataset", "labels"),
|
|
filepath.Join(root, "detector", "dataset", "labels", "train"),
|
|
filepath.Join(root, "detector", "dataset", "labels", "val"),
|
|
filepath.Join(root, "detector", "runs"),
|
|
}
|
|
|
|
for _, dir := range dirs {
|
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := trainingWriteDetectorDatasetYAML(root); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func trainingEnsurePoseDirs(root string) error {
|
|
dirs := []string{
|
|
filepath.Join(root, "pose"),
|
|
filepath.Join(root, "pose", "dataset"),
|
|
filepath.Join(root, "pose", "dataset", "images"),
|
|
filepath.Join(root, "pose", "dataset", "images", "train"),
|
|
filepath.Join(root, "pose", "dataset", "images", "val"),
|
|
filepath.Join(root, "pose", "dataset", "labels"),
|
|
filepath.Join(root, "pose", "dataset", "labels", "train"),
|
|
filepath.Join(root, "pose", "dataset", "labels", "val"),
|
|
filepath.Join(root, "pose", "runs"),
|
|
}
|
|
|
|
for _, dir := range dirs {
|
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := trainingWritePoseDatasetYAML(root); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func trainingCreateNextSample() (*TrainingSample, error) {
|
|
settings := getSettings()
|
|
|
|
doneDir := strings.TrimSpace(settings.DoneDir)
|
|
if doneDir == "" {
|
|
return nil, errors.New("doneDir ist leer")
|
|
}
|
|
|
|
videoPath, err := trainingPickRandomVideo(doneDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
duration := trainingProbeDurationSeconds(videoPath)
|
|
second := trainingRandomSecond(duration)
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := trainingEnsureDetectorDirs(root); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Join(root, "frames"), 0755); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(root, "samples"), 0755); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
id := trainingMakeSampleID(videoPath, second)
|
|
framePath := filepath.Join(root, "frames", id+".jpg")
|
|
|
|
if err := trainingExtractFrame(videoPath, framePath, second); err != nil {
|
|
// Fallback auf Sekunde 0, falls random second nicht funktioniert.
|
|
second = 0
|
|
id = trainingMakeSampleID(videoPath, second)
|
|
framePath = filepath.Join(root, "frames", id+".jpg")
|
|
|
|
if err2 := trainingExtractFrame(videoPath, framePath, second); err2 != nil {
|
|
return nil, appErrorf("frame extraction failed: %v / fallback: %w", err, err2)
|
|
}
|
|
}
|
|
|
|
prediction := trainingPredictFrame(framePath)
|
|
|
|
var sourceSizeBytes int64
|
|
if st, err := os.Stat(videoPath); err == nil && st != nil && !st.IsDir() {
|
|
sourceSizeBytes = st.Size()
|
|
}
|
|
|
|
sample := &TrainingSample{
|
|
SampleID: id,
|
|
FrameURL: "/api/training/frame?id=" + id,
|
|
SourceFile: filepath.Base(videoPath),
|
|
SourcePath: videoPath,
|
|
SourceSizeBytes: sourceSizeBytes,
|
|
Second: second,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
Prediction: prediction,
|
|
}
|
|
|
|
if err := trainingWriteSample(root, sample); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return sample, nil
|
|
}
|
|
|
|
func trainingCreateUncertainNextSampleWithProgress(startedAtMs int64, requestID string) (*TrainingSample, error) {
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
candidates, err := trainingReadValidUncertainCandidates(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Ziel:
|
|
// Vor jeder Auswahl sollen wieder 5 Kandidaten verglichen werden.
|
|
// Wenn noch 4 aus der Queue übrig sind, wird genau 1 neues Bild extrahiert.
|
|
// Wenn keine Queue existiert, wird ein kompletter 5er-Batch aufgebaut.
|
|
missing := trainingUncertainCandidateCount - len(candidates)
|
|
if missing < 1 {
|
|
// Trotzdem mindestens 1 neues Bild nachziehen,
|
|
// damit die Auswahl nach jedem Speichern frisch bleibt.
|
|
missing = 1
|
|
}
|
|
|
|
candidateWindowTotal := trainingUncertainCandidateCount
|
|
|
|
const stepsPerCandidate = 4
|
|
totalSteps := missing*stepsPerCandidate + 1
|
|
|
|
errs := []string{}
|
|
|
|
for i := 0; i < missing; i++ {
|
|
attempt := i + 1
|
|
stepStart := i*stepsPerCandidate + 1
|
|
|
|
prefix := fmt.Sprintf("Kandidat %d/%d: ", attempt, candidateWindowTotal)
|
|
|
|
candidate, err := trainingCreateUncertainCandidateWithProgress(
|
|
root,
|
|
startedAtMs,
|
|
requestID,
|
|
stepStart,
|
|
totalSteps,
|
|
prefix,
|
|
)
|
|
|
|
if err != nil {
|
|
errs = append(errs, err.Error())
|
|
|
|
trainingPublishAnalysisStep(
|
|
requestID,
|
|
startedAtMs,
|
|
stepStart+stepsPerCandidate-1,
|
|
totalSteps,
|
|
"",
|
|
fmt.Sprintf("Kandidat %d/%d fehlgeschlagen…", attempt, candidateWindowTotal),
|
|
)
|
|
|
|
continue
|
|
}
|
|
|
|
candidates = append(candidates, *candidate)
|
|
|
|
trainingPublishAnalysisStep(
|
|
requestID,
|
|
startedAtMs,
|
|
stepStart+stepsPerCandidate-1,
|
|
totalSteps,
|
|
candidate.sample.SourceFile,
|
|
fmt.Sprintf(
|
|
"Kandidat %d/%d bewertet · Unsicherheit %.0f%%",
|
|
attempt,
|
|
candidateWindowTotal,
|
|
candidate.score*100,
|
|
),
|
|
)
|
|
}
|
|
|
|
if len(candidates) == 0 {
|
|
if len(errs) > 0 {
|
|
return nil, errors.New(strings.Join(errs, "; "))
|
|
}
|
|
|
|
return nil, errors.New("keine unsicheren Trainingskandidaten gefunden")
|
|
}
|
|
|
|
sort.Slice(candidates, func(i, j int) bool {
|
|
if candidates[i].score == candidates[j].score {
|
|
return candidates[i].sample.CreatedAt < candidates[j].sample.CreatedAt
|
|
}
|
|
|
|
return candidates[i].score > candidates[j].score
|
|
})
|
|
|
|
best := candidates[0]
|
|
remaining := candidates[1:]
|
|
|
|
if len(remaining) > trainingUncertainCandidateCount-1 {
|
|
remaining = remaining[:trainingUncertainCandidateCount-1]
|
|
}
|
|
|
|
if err := trainingWriteUncertainCandidateQueue(root, remaining); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
trainingPublishAnalysisStep(
|
|
requestID,
|
|
startedAtMs,
|
|
totalSteps,
|
|
totalSteps,
|
|
best.sample.SourceFile,
|
|
fmt.Sprintf(
|
|
"Unsicherster Kandidat gewählt · Score %.0f%% · Fenster %d/%d",
|
|
best.score*100,
|
|
len(remaining)+1,
|
|
trainingUncertainCandidateCount,
|
|
),
|
|
)
|
|
|
|
best.sample.UncertaintyScore = best.score
|
|
|
|
return best.sample, nil
|
|
}
|
|
|
|
func reloadAIServerModelAfterTraining() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, aiServerURL()+"/reload", nil)
|
|
if err != nil {
|
|
appLogln("⚠️ AI Server Reload Request konnte nicht gebaut werden:", err)
|
|
return
|
|
}
|
|
|
|
addAIServerAuth(req)
|
|
|
|
client := &http.Client{
|
|
Timeout: 30 * time.Second,
|
|
}
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
appLogln("⚠️ AI Server Reload fehlgeschlagen:", err)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
appLogln("⚠️ AI Server Reload HTTP", resp.StatusCode, strings.TrimSpace(string(body)))
|
|
return
|
|
}
|
|
|
|
appLogln("✅ AI Server Modell nach Training neu geladen:", strings.TrimSpace(string(body)))
|
|
}
|
|
|
|
func trainingDeleteSampleFiles(root string, sampleID string) {
|
|
sampleID = strings.TrimSpace(sampleID)
|
|
if sampleID == "" || strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") {
|
|
return
|
|
}
|
|
|
|
_ = os.Remove(filepath.Join(root, "samples", sampleID+".json"))
|
|
_ = os.Remove(filepath.Join(root, "frames", sampleID+".jpg"))
|
|
}
|
|
|
|
func trainingPredictionUncertaintyScore(pred TrainingPrediction) float64 {
|
|
if !pred.ModelAvailable {
|
|
return 0.10
|
|
}
|
|
|
|
scores := []float64{}
|
|
|
|
addScore := func(score float64) {
|
|
if math.IsNaN(score) || math.IsInf(score, 0) {
|
|
return
|
|
}
|
|
|
|
if score <= 0 {
|
|
return
|
|
}
|
|
|
|
scores = append(scores, clamp01(score))
|
|
}
|
|
|
|
if !isNoSexPositionLabel(pred.SexPosition) {
|
|
addScore(pred.SexPositionScore)
|
|
}
|
|
|
|
for _, box := range pred.Boxes {
|
|
addScore(box.Score)
|
|
}
|
|
|
|
if len(pred.Boxes) == 0 {
|
|
for _, item := range pred.BodyPartsPresent {
|
|
addScore(item.Score)
|
|
}
|
|
|
|
for _, item := range pred.ObjectsPresent {
|
|
addScore(item.Score)
|
|
}
|
|
|
|
for _, item := range pred.ClothingPresent {
|
|
addScore(item.Score)
|
|
}
|
|
}
|
|
|
|
if len(scores) == 0 {
|
|
// Modell ist verfügbar, erkennt aber nichts.
|
|
// Das kann ein nützliches Hard-Negative oder ein False-Negative sein.
|
|
return 0.35
|
|
}
|
|
|
|
sum := 0.0
|
|
|
|
for _, score := range scores {
|
|
// Höchste Unsicherheit ungefähr im mittleren Bereich.
|
|
// 0.55 ist absichtlich etwas niedriger als 0.75,
|
|
// damit Low/Mid-Confidence-Fälle bevorzugt werden.
|
|
distance := math.Abs(score - 0.55)
|
|
uncertainty := 1.0 - distance/0.55
|
|
sum += clamp01(uncertainty)
|
|
}
|
|
|
|
avg := sum / float64(len(scores))
|
|
|
|
// Viele Boxen mit mittlerer Confidence sind besonders wertvoll.
|
|
boxBonus := math.Min(0.12, float64(len(pred.Boxes))*0.025)
|
|
|
|
// Sehr niedrige Confidence soll ebenfalls auftauchen,
|
|
// aber nicht alle Ergebnisse dominieren.
|
|
lowConfidenceBonus := 0.0
|
|
for _, score := range scores {
|
|
if score >= 0.25 && score <= 0.75 {
|
|
lowConfidenceBonus = 0.08
|
|
break
|
|
}
|
|
}
|
|
|
|
return clamp01(avg + boxBonus + lowConfidenceBonus)
|
|
}
|
|
|
|
func trainingCreateNextSampleWithProgress(startedAtMs int64, requestID string) (*TrainingSample, error) {
|
|
return trainingCreateNextSampleWithProgressRange(
|
|
startedAtMs,
|
|
requestID,
|
|
1,
|
|
4,
|
|
"",
|
|
)
|
|
}
|
|
|
|
func trainingCreateNextSampleWithProgressRange(
|
|
startedAtMs int64,
|
|
requestID string,
|
|
stepStart int,
|
|
stepTotal int,
|
|
prefix string,
|
|
) (*TrainingSample, error) {
|
|
publishStep := func(localStep int, sourceFile string, message string) {
|
|
trainingPublishAnalysisStep(
|
|
requestID,
|
|
startedAtMs,
|
|
stepStart+localStep-1,
|
|
stepTotal,
|
|
sourceFile,
|
|
prefix+message,
|
|
)
|
|
}
|
|
|
|
publishStep(1, "", "Video wird ausgewählt…")
|
|
|
|
settings := getSettings()
|
|
|
|
doneDir := strings.TrimSpace(settings.DoneDir)
|
|
if doneDir == "" {
|
|
return nil, errors.New("doneDir ist leer")
|
|
}
|
|
|
|
videoPath, err := trainingPickRandomVideo(doneDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sourceFile := filepath.Base(videoPath)
|
|
previewURL := ""
|
|
|
|
publishStep = func(localStep int, sourceFile string, message string) {
|
|
trainingPublishAnalysisStepWithPreview(
|
|
requestID,
|
|
startedAtMs,
|
|
stepStart+localStep-1,
|
|
stepTotal,
|
|
sourceFile,
|
|
previewURL,
|
|
prefix+message,
|
|
)
|
|
}
|
|
|
|
publishStep(2, sourceFile, "Bild wird extrahiert…")
|
|
|
|
duration := trainingProbeDurationSeconds(videoPath)
|
|
second := trainingRandomSecond(duration)
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := trainingEnsureDetectorDirs(root); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Join(root, "frames"), 0755); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Join(root, "samples"), 0755); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
id := trainingMakeSampleID(videoPath, second)
|
|
framePath := filepath.Join(root, "frames", id+".jpg")
|
|
|
|
if err := trainingExtractFrame(videoPath, framePath, second); err != nil {
|
|
publishStep(2, sourceFile, "Bild wird erneut bei Sekunde 0 extrahiert…")
|
|
|
|
second = 0
|
|
id = trainingMakeSampleID(videoPath, second)
|
|
framePath = filepath.Join(root, "frames", id+".jpg")
|
|
|
|
if err2 := trainingExtractFrame(videoPath, framePath, second); err2 != nil {
|
|
return nil, appErrorf("frame extraction failed: %v / fallback: %w", err, err2)
|
|
}
|
|
}
|
|
|
|
previewURL = "/api/training/frame?id=" + url.QueryEscape(id)
|
|
|
|
publishStep(3, sourceFile, "Bild wird analysiert…")
|
|
|
|
prediction := trainingPredictFrame(framePath)
|
|
|
|
var sourceSizeBytes int64
|
|
if st, err := os.Stat(videoPath); err == nil && st != nil && !st.IsDir() {
|
|
sourceSizeBytes = st.Size()
|
|
}
|
|
|
|
sample := &TrainingSample{
|
|
SampleID: id,
|
|
FrameURL: "/api/training/frame?id=" + id,
|
|
SourceFile: sourceFile,
|
|
SourcePath: videoPath,
|
|
SourceSizeBytes: sourceSizeBytes,
|
|
Second: second,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
Prediction: prediction,
|
|
}
|
|
|
|
publishStep(4, sourceFile, "Analyse-Ergebnis wird gespeichert…")
|
|
|
|
if err := trainingWriteSample(root, sample); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return sample, nil
|
|
}
|
|
|
|
func trainingPickRandomVideo(doneDir string) (string, error) {
|
|
extOK := map[string]bool{
|
|
".mp4": true,
|
|
}
|
|
|
|
doneDir = strings.TrimSpace(doneDir)
|
|
if doneDir == "" {
|
|
return "", errors.New("doneDir ist leer")
|
|
}
|
|
|
|
doneAbs, err := filepath.Abs(doneDir)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
var files []string
|
|
|
|
err = filepath.WalkDir(doneAbs, func(path string, d os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return nil
|
|
}
|
|
|
|
rel, err := filepath.Rel(doneAbs, path)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
rel = filepath.Clean(rel)
|
|
|
|
// Root "done" selbst erlauben.
|
|
if rel == "." {
|
|
return nil
|
|
}
|
|
|
|
parts := strings.Split(rel, string(os.PathSeparator))
|
|
top := strings.ToLower(strings.TrimSpace(parts[0]))
|
|
name := strings.ToLower(strings.TrimSpace(d.Name()))
|
|
|
|
if d.IsDir() {
|
|
// Nur diese Bereiche fürs Trainingsbild:
|
|
// - done/
|
|
// - done/keep/...
|
|
//
|
|
// Alles andere unter done wird ignoriert, z.B.:
|
|
// .postwork_tmp, .trash, generated, training, sonstige Temp-Ordner.
|
|
if top != "keep" {
|
|
return filepath.SkipDir
|
|
}
|
|
|
|
// Innerhalb von keep trotzdem versteckte/temporäre Ordner überspringen.
|
|
if name == ".trash" ||
|
|
name == ".postwork_tmp" ||
|
|
name == "training" ||
|
|
name == "generated" ||
|
|
strings.HasPrefix(name, ".") {
|
|
return filepath.SkipDir
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Dateien nur direkt in done/ oder unter done/keep/... erlauben.
|
|
if len(parts) > 1 && top != "keep" {
|
|
return nil
|
|
}
|
|
|
|
// Keine versteckten/temp-Dateien verwenden.
|
|
if strings.HasPrefix(name, ".") ||
|
|
strings.Contains(name, ".tmp.") ||
|
|
strings.Contains(name, ".part") {
|
|
return nil
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
if extOK[ext] {
|
|
files = append(files, path)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if len(files) == 0 {
|
|
return "", errors.New("keine Videos in done oder done/keep gefunden")
|
|
}
|
|
|
|
return files[rand.Intn(len(files))], nil
|
|
}
|
|
|
|
func trainingExtractFrame(videoPath string, framePath string, second float64) error {
|
|
settings := getSettings()
|
|
|
|
ffmpeg := strings.TrimSpace(settings.FFmpegPath)
|
|
if ffmpeg == "" {
|
|
ffmpeg = "ffmpeg"
|
|
}
|
|
|
|
_ = os.Remove(framePath)
|
|
|
|
ss := strconv.FormatFloat(math.Max(0, second), 'f', 3, 64)
|
|
|
|
cmd := exec.Command(
|
|
ffmpeg,
|
|
"-hide_banner",
|
|
"-loglevel", "error",
|
|
"-ss", ss,
|
|
"-i", videoPath,
|
|
"-frames:v", "1",
|
|
"-q:v", "2",
|
|
"-y",
|
|
framePath,
|
|
)
|
|
|
|
hideCommandWindow(cmd)
|
|
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return appErrorf("%w: %s", err, strings.TrimSpace(string(out)))
|
|
}
|
|
|
|
if st, err := os.Stat(framePath); err != nil || st.Size() == 0 {
|
|
return errors.New("ffmpeg erzeugte kein Frame")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func trainingPredictFrame(framePath string) TrainingPrediction {
|
|
settings := getSettings()
|
|
if !settings.TrainingRecognitionEnabled {
|
|
return trainingEmptyPrediction("recognition_disabled")
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
appLogln("⚠️ training predict root error:", err)
|
|
return trainingEmptyPrediction("root_error")
|
|
}
|
|
|
|
det := trainingPredictDetector(root, framePath)
|
|
pred := trainingPredictionFromDetector(det)
|
|
|
|
pose := trainingPredictPose(root, framePath)
|
|
pred = trainingApplyPoseToPrediction(pred, pose)
|
|
|
|
return pred
|
|
}
|
|
|
|
func trainingPredictFrameDetectorOnly(framePath string) TrainingPrediction {
|
|
settings := getSettings()
|
|
if !settings.TrainingRecognitionEnabled {
|
|
return trainingEmptyPrediction("recognition_disabled")
|
|
}
|
|
|
|
root, err := trainingRootDir()
|
|
if err != nil {
|
|
appLogln("⚠️ training predict root error:", err)
|
|
return trainingEmptyPrediction("root_error")
|
|
}
|
|
|
|
det := trainingPredictDetector(root, framePath)
|
|
return trainingPredictionFromDetector(det)
|
|
}
|
|
|
|
func trainingPredictionFromDetector(det TrainingDetectorPrediction) TrainingPrediction {
|
|
rawBoxes := det.Boxes
|
|
if rawBoxes == nil {
|
|
rawBoxes = []TrainingBox{}
|
|
}
|
|
|
|
pred := TrainingPrediction{
|
|
ModelAvailable: det.Available,
|
|
Source: det.Source,
|
|
SexPosition: trainingNoSexPositionLabel,
|
|
SexPositionScore: 0,
|
|
PeoplePresent: []TrainingScoredLabel{},
|
|
BodyPartsPresent: []TrainingScoredLabel{},
|
|
ObjectsPresent: []TrainingScoredLabel{},
|
|
ClothingPresent: []TrainingScoredLabel{},
|
|
Boxes: []TrainingBox{},
|
|
}
|
|
|
|
if pred.Source == "" {
|
|
if det.Available {
|
|
pred.Source = "yolo26_detector"
|
|
} else {
|
|
pred.Source = "detector_missing"
|
|
}
|
|
}
|
|
|
|
grouped, err := trainingGroupedLabels()
|
|
if err != nil {
|
|
appLogln("⚠️ detector label grouping failed:", err)
|
|
return pred
|
|
}
|
|
|
|
peopleSet := map[string]bool{}
|
|
for _, label := range grouped.People {
|
|
clean := strings.TrimSpace(label)
|
|
if clean != "" {
|
|
peopleSet[clean] = true
|
|
}
|
|
}
|
|
|
|
detectionSet := map[string]bool{}
|
|
|
|
for _, label := range grouped.BodyParts {
|
|
clean := strings.TrimSpace(label)
|
|
if clean != "" {
|
|
detectionSet[clean] = true
|
|
}
|
|
}
|
|
|
|
for _, label := range grouped.Objects {
|
|
clean := strings.TrimSpace(label)
|
|
if clean != "" {
|
|
detectionSet[clean] = true
|
|
}
|
|
}
|
|
|
|
for _, label := range grouped.Clothing {
|
|
clean := strings.TrimSpace(label)
|
|
if clean != "" {
|
|
detectionSet[clean] = true
|
|
}
|
|
}
|
|
|
|
visibleBoxes := []TrainingBox{}
|
|
|
|
for _, box := range rawBoxes {
|
|
if box.Score > 0 && box.Score < 0.25 {
|
|
continue
|
|
}
|
|
|
|
label := strings.TrimSpace(box.Label)
|
|
if label == "" {
|
|
continue
|
|
}
|
|
|
|
box.Label = label
|
|
|
|
if peopleSet[label] {
|
|
visibleBoxes = append(visibleBoxes, box)
|
|
continue
|
|
}
|
|
|
|
if detectionSet[label] {
|
|
visibleBoxes = append(visibleBoxes, box)
|
|
}
|
|
}
|
|
|
|
pred.Boxes = visibleBoxes
|
|
|
|
pred.PeoplePresent = trainingScoredLabelsFromDetectorBoxes(visibleBoxes, grouped.People)
|
|
pred.BodyPartsPresent = trainingScoredLabelsFromDetectorBoxes(visibleBoxes, grouped.BodyParts)
|
|
pred.ObjectsPresent = trainingScoredLabelsFromDetectorBoxes(visibleBoxes, grouped.Objects)
|
|
pred.ClothingPresent = trainingScoredLabelsFromDetectorBoxes(visibleBoxes, grouped.Clothing)
|
|
|
|
return pred
|
|
}
|
|
|
|
func trainingPredictDetector(root string, framePath string) TrainingDetectorPrediction {
|
|
python := trainingPythonExe()
|
|
script := trainingScriptPath("predict_detector_model.py")
|
|
|
|
model := trainingResolveDetectorModel(root)
|
|
if !model.EffectiveExists {
|
|
return TrainingDetectorPrediction{
|
|
Available: false,
|
|
Source: model.Source,
|
|
Boxes: []TrainingBox{},
|
|
}
|
|
}
|
|
|
|
confValues := []string{"0.30"}
|
|
|
|
best := TrainingDetectorPrediction{
|
|
Available: true,
|
|
Source: "yolo26_detector",
|
|
Boxes: []TrainingBox{},
|
|
}
|
|
|
|
for _, conf := range confValues {
|
|
cmd := exec.Command(
|
|
python,
|
|
script,
|
|
"--root", root,
|
|
"--image", framePath,
|
|
"--model", model.EffectivePath,
|
|
"--conf", conf,
|
|
"--imgsz", "640",
|
|
)
|
|
|
|
hideCommandWindow(cmd)
|
|
|
|
var stdout strings.Builder
|
|
var stderr strings.Builder
|
|
|
|
cmd.Stdout = &stdout
|
|
cmd.Stderr = &stderr
|
|
|
|
err := cmd.Run()
|
|
|
|
outText := strings.TrimSpace(stdout.String())
|
|
errText := strings.TrimSpace(stderr.String())
|
|
|
|
if errText != "" {
|
|
appLogln("🔎 detector stderr:", errText)
|
|
}
|
|
|
|
if err != nil {
|
|
appLogln("⚠️ detector predict failed")
|
|
appLogln(" conf:", conf)
|
|
appLogln(" error:", err)
|
|
appLogln(" stdout:", outText)
|
|
appLogln(" stderr:", errText)
|
|
continue
|
|
}
|
|
|
|
if outText == "" {
|
|
appLogln("⚠️ detector predict empty stdout")
|
|
appLogln(" conf:", conf)
|
|
appLogln(" stderr:", errText)
|
|
continue
|
|
}
|
|
|
|
var det TrainingDetectorPrediction
|
|
if err := json.Unmarshal([]byte(outText), &det); err != nil {
|
|
appLogln("⚠️ detector predict json failed:", err)
|
|
appLogln(" conf:", conf)
|
|
appLogln(" stdout:", outText)
|
|
appLogln(" stderr:", errText)
|
|
continue
|
|
}
|
|
|
|
if det.Boxes == nil {
|
|
det.Boxes = []TrainingBox{}
|
|
}
|
|
|
|
det.Source = model.Source
|
|
|
|
best = det
|
|
|
|
if len(det.Boxes) > 0 {
|
|
return det
|
|
}
|
|
}
|
|
|
|
if best.Boxes == nil {
|
|
best.Boxes = []TrainingBox{}
|
|
}
|
|
|
|
return best
|
|
}
|
|
|
|
func trainingScoredLabelsFromDetectorBoxes(
|
|
boxes []TrainingBox,
|
|
group []string,
|
|
) []TrainingScoredLabel {
|
|
groupSet := map[string]bool{}
|
|
|
|
for _, label := range group {
|
|
clean := strings.TrimSpace(label)
|
|
if clean != "" {
|
|
groupSet[clean] = true
|
|
}
|
|
}
|
|
|
|
best := map[string]float64{}
|
|
|
|
for _, box := range boxes {
|
|
label := strings.TrimSpace(box.Label)
|
|
if label == "" || !groupSet[label] {
|
|
continue
|
|
}
|
|
|
|
score := box.Score
|
|
if score <= 0 {
|
|
score = 1
|
|
}
|
|
|
|
if old, ok := best[label]; !ok || score > old {
|
|
best[label] = score
|
|
}
|
|
}
|
|
|
|
out := make([]TrainingScoredLabel, 0, len(best))
|
|
|
|
for label, score := range best {
|
|
out = append(out, TrainingScoredLabel{
|
|
Label: label,
|
|
Score: score,
|
|
})
|
|
}
|
|
|
|
sort.Slice(out, func(i, j int) bool {
|
|
if out[i].Score == out[j].Score {
|
|
return out[i].Label < out[j].Label
|
|
}
|
|
|
|
return out[i].Score > out[j].Score
|
|
})
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingApplyDetectorToPrediction(pred TrainingPrediction, det TrainingDetectorPrediction) TrainingPrediction {
|
|
boxes := det.Boxes
|
|
if boxes == nil {
|
|
boxes = []TrainingBox{}
|
|
}
|
|
|
|
grouped, err := trainingGroupedLabels()
|
|
if err != nil {
|
|
appLogln("⚠️ detector label grouping failed:", err)
|
|
|
|
pred.Boxes = boxes
|
|
pred.BodyPartsPresent = []TrainingScoredLabel{}
|
|
pred.ObjectsPresent = []TrainingScoredLabel{}
|
|
pred.ClothingPresent = []TrainingScoredLabel{}
|
|
return pred
|
|
}
|
|
|
|
// Wichtig:
|
|
// Ab jetzt kommen diese drei Bereiche ausschließlich vom Object Detector.
|
|
// Kein Scene-KNN-Fallback, damit keine Labels ohne Box angezeigt werden.
|
|
pred.BodyPartsPresent = trainingScoredLabelsFromDetectorBoxes(boxes, grouped.BodyParts)
|
|
pred.ObjectsPresent = trainingScoredLabelsFromDetectorBoxes(boxes, grouped.Objects)
|
|
pred.ClothingPresent = trainingScoredLabelsFromDetectorBoxes(boxes, grouped.Clothing)
|
|
pred.Boxes = boxes
|
|
|
|
if det.Available {
|
|
if pred.Source == "" {
|
|
pred.Source = "yolo26_detector"
|
|
} else {
|
|
pred.Source = pred.Source + "+yolo_detector"
|
|
}
|
|
pred.ModelAvailable = true
|
|
}
|
|
|
|
return pred
|
|
}
|
|
|
|
func trainingPredictPose(root string, framePath string) TrainingPosePrediction {
|
|
python := trainingPythonExe()
|
|
script := trainingScriptPath("predict_pose_model.py")
|
|
|
|
model := trainingResolvePoseModel(root)
|
|
if !model.EffectiveExists {
|
|
return TrainingPosePrediction{
|
|
Available: false,
|
|
Source: model.Source,
|
|
Persons: []TrainingPosePerson{},
|
|
}
|
|
}
|
|
|
|
cmd := exec.Command(
|
|
python,
|
|
script,
|
|
"--root", root,
|
|
"--image", framePath,
|
|
"--model", model.EffectivePath,
|
|
"--conf", "0.30",
|
|
"--imgsz", "640",
|
|
)
|
|
|
|
hideCommandWindow(cmd)
|
|
|
|
var stdout strings.Builder
|
|
var stderr strings.Builder
|
|
|
|
cmd.Stdout = &stdout
|
|
cmd.Stderr = &stderr
|
|
|
|
err := cmd.Run()
|
|
|
|
outText := strings.TrimSpace(stdout.String())
|
|
errText := strings.TrimSpace(stderr.String())
|
|
|
|
if errText != "" {
|
|
appLogln("🔎 pose stderr:", errText)
|
|
}
|
|
|
|
if err != nil {
|
|
appLogln("⚠️ pose predict failed:", err)
|
|
appLogln(" stdout:", outText)
|
|
return TrainingPosePrediction{
|
|
Available: false,
|
|
Source: "pose_error",
|
|
Persons: []TrainingPosePerson{},
|
|
}
|
|
}
|
|
|
|
if outText == "" {
|
|
return TrainingPosePrediction{
|
|
Available: false,
|
|
Source: "pose_empty",
|
|
Persons: []TrainingPosePerson{},
|
|
}
|
|
}
|
|
|
|
var pose TrainingPosePrediction
|
|
if err := json.Unmarshal([]byte(outText), &pose); err != nil {
|
|
appLogln("⚠️ pose predict json failed:", err)
|
|
return TrainingPosePrediction{
|
|
Available: false,
|
|
Source: "pose_json_error",
|
|
Persons: []TrainingPosePerson{},
|
|
}
|
|
}
|
|
|
|
if pose.Persons == nil {
|
|
pose.Persons = []TrainingPosePerson{}
|
|
}
|
|
|
|
pose = trainingAnnotatePosePrediction(pose)
|
|
pose.Source = model.Source
|
|
|
|
return pose
|
|
}
|
|
|
|
func trainingCombinePositionScore(scores map[string]float64, positionSet map[string]bool, label string, score float64) {
|
|
label = normalizeSexPositionLabel(label)
|
|
if isNoSexPositionLabel(label) || !positionSet[label] || score <= 0 {
|
|
return
|
|
}
|
|
|
|
score = clamp01(score)
|
|
current := clamp01(scores[label])
|
|
scores[label] = clamp01(1 - (1-current)*(1-score))
|
|
}
|
|
|
|
func trainingAppendPredictionSource(pred *TrainingPrediction, source string) {
|
|
source = strings.TrimSpace(source)
|
|
if source == "" {
|
|
return
|
|
}
|
|
|
|
current := strings.TrimSpace(pred.Source)
|
|
if current == "" {
|
|
pred.Source = source
|
|
return
|
|
}
|
|
|
|
for _, part := range strings.Split(current, "+") {
|
|
if strings.TrimSpace(part) == source {
|
|
return
|
|
}
|
|
}
|
|
|
|
pred.Source = current + "+" + source
|
|
}
|
|
|
|
func trainingFuseHybridPositionScores(
|
|
poseScores map[string]float64,
|
|
contextScores map[string]float64,
|
|
) (string, float64, bool, bool) {
|
|
labels := map[string]bool{}
|
|
|
|
for label := range poseScores {
|
|
if !isNoSexPositionLabel(label) {
|
|
labels[label] = true
|
|
}
|
|
}
|
|
|
|
for label := range contextScores {
|
|
if !isNoSexPositionLabel(label) {
|
|
labels[label] = true
|
|
}
|
|
}
|
|
|
|
bestPosition := ""
|
|
bestPositionScore := 0.0
|
|
bestHasPose := false
|
|
bestHasContext := false
|
|
bestPosePosition := ""
|
|
bestPoseScore := 0.0
|
|
bestPoseHasContext := false
|
|
|
|
for label := range labels {
|
|
poseScore := clamp01(poseScores[label])
|
|
contextScore := clamp01(contextScores[label])
|
|
|
|
score := 0.0
|
|
hasPose := poseScore > 0
|
|
hasContext := contextScore > 0
|
|
|
|
if hasPose {
|
|
score = poseScore
|
|
|
|
// Kontext boostet die Pose, bleibt aber bewusst Nebeninformation.
|
|
if hasContext {
|
|
boost := clamp01(contextScore * trainingPositionContextBoostWeight)
|
|
score = clamp01(1 - (1-score)*(1-boost))
|
|
}
|
|
} else if contextScore >= trainingPositionContextMinScore {
|
|
// Reiner Box-/Szenen-Kontext darf eine unsichere Prediction liefern,
|
|
// aber keine hohe Sicherheit vortaeuschen.
|
|
score = math.Min(trainingPositionContextMaxScore, contextScore)
|
|
}
|
|
|
|
if score > bestPositionScore {
|
|
bestPosition = label
|
|
bestPositionScore = score
|
|
bestHasPose = hasPose
|
|
bestHasContext = hasContext
|
|
}
|
|
|
|
if hasPose && score > bestPoseScore {
|
|
bestPosePosition = label
|
|
bestPoseScore = score
|
|
bestPoseHasContext = hasContext
|
|
}
|
|
}
|
|
|
|
if bestPosePosition != "" &&
|
|
!bestHasPose &&
|
|
bestPositionScore <= bestPoseScore+trainingPositionContextOverrideMargin {
|
|
bestPosition = bestPosePosition
|
|
bestPositionScore = bestPoseScore
|
|
bestHasPose = true
|
|
bestHasContext = bestPoseHasContext
|
|
}
|
|
|
|
return bestPosition, clamp01(bestPositionScore), bestHasPose, bestHasContext
|
|
}
|
|
|
|
func trainingIsFinite01(v float64) bool {
|
|
return !math.IsNaN(v) && !math.IsInf(v, 0) && v >= 0 && v <= 1
|
|
}
|
|
|
|
func trainingNormalizedBox(box TrainingBox) (TrainingBox, bool) {
|
|
x := clamp01(box.X)
|
|
y := clamp01(box.Y)
|
|
w := clamp01(box.W)
|
|
h := clamp01(box.H)
|
|
|
|
if w <= 0 || h <= 0 {
|
|
return TrainingBox{}, false
|
|
}
|
|
|
|
if x+w > 1 {
|
|
w = 1 - x
|
|
}
|
|
if y+h > 1 {
|
|
h = 1 - y
|
|
}
|
|
|
|
if w <= 0 || h <= 0 {
|
|
return TrainingBox{}, false
|
|
}
|
|
|
|
box.X = x
|
|
box.Y = y
|
|
box.W = w
|
|
box.H = h
|
|
return box, true
|
|
}
|
|
|
|
func trainingBoxArea(box TrainingBox) float64 {
|
|
box, ok := trainingNormalizedBox(box)
|
|
if !ok {
|
|
return 0
|
|
}
|
|
|
|
return box.W * box.H
|
|
}
|
|
|
|
func trainingBoxCenter(box TrainingBox) (float64, float64, bool) {
|
|
box, ok := trainingNormalizedBox(box)
|
|
if !ok {
|
|
return 0, 0, false
|
|
}
|
|
|
|
return box.X + box.W/2, box.Y + box.H/2, true
|
|
}
|
|
|
|
func trainingBoxGap(a TrainingBox, b TrainingBox) float64 {
|
|
a, okA := trainingNormalizedBox(a)
|
|
b, okB := trainingNormalizedBox(b)
|
|
if !okA || !okB {
|
|
return 1
|
|
}
|
|
|
|
dx := math.Max(0, math.Max(a.X-(b.X+b.W), b.X-(a.X+a.W)))
|
|
dy := math.Max(0, math.Max(a.Y-(b.Y+b.H), b.Y-(a.Y+a.H)))
|
|
|
|
return math.Sqrt(dx*dx + dy*dy)
|
|
}
|
|
|
|
func trainingBoxOverlapRatio(a TrainingBox, b TrainingBox) float64 {
|
|
a, okA := trainingNormalizedBox(a)
|
|
b, okB := trainingNormalizedBox(b)
|
|
if !okA || !okB {
|
|
return 0
|
|
}
|
|
|
|
left := math.Max(a.X, b.X)
|
|
top := math.Max(a.Y, b.Y)
|
|
right := math.Min(a.X+a.W, b.X+b.W)
|
|
bottom := math.Min(a.Y+a.H, b.Y+b.H)
|
|
|
|
if right <= left || bottom <= top {
|
|
return 0
|
|
}
|
|
|
|
intersection := (right - left) * (bottom - top)
|
|
minArea := math.Min(trainingBoxArea(a), trainingBoxArea(b))
|
|
if minArea <= 0 {
|
|
return 0
|
|
}
|
|
|
|
return clamp01(intersection / minArea)
|
|
}
|
|
|
|
func trainingBoxHorizontalOverlapRatio(a TrainingBox, b TrainingBox) float64 {
|
|
a, okA := trainingNormalizedBox(a)
|
|
b, okB := trainingNormalizedBox(b)
|
|
if !okA || !okB {
|
|
return 0
|
|
}
|
|
|
|
left := math.Max(a.X, b.X)
|
|
right := math.Min(a.X+a.W, b.X+b.W)
|
|
if right <= left {
|
|
return 0
|
|
}
|
|
|
|
minWidth := math.Min(a.W, b.W)
|
|
if minWidth <= 0 {
|
|
return 0
|
|
}
|
|
|
|
return clamp01((right - left) / minWidth)
|
|
}
|
|
|
|
func trainingBoxesByLabel(boxes []TrainingBox, labels ...string) []TrainingBox {
|
|
wanted := map[string]bool{}
|
|
for _, label := range labels {
|
|
clean := strings.TrimSpace(label)
|
|
if clean != "" {
|
|
wanted[clean] = true
|
|
}
|
|
}
|
|
|
|
out := []TrainingBox{}
|
|
for _, box := range boxes {
|
|
label := strings.TrimSpace(box.Label)
|
|
if !wanted[label] {
|
|
continue
|
|
}
|
|
if normalized, ok := trainingNormalizedBox(box); ok {
|
|
out = append(out, normalized)
|
|
}
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingAnyBoxesNear(a []TrainingBox, b []TrainingBox, margin float64) bool {
|
|
for _, left := range a {
|
|
for _, right := range b {
|
|
if trainingBoxGap(left, right) <= margin || trainingBoxOverlapRatio(left, right) > 0 {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func trainingPointNearBox(x float64, y float64, box TrainingBox, margin float64) bool {
|
|
if !trainingIsFinite01(x) || !trainingIsFinite01(y) {
|
|
return false
|
|
}
|
|
|
|
box, ok := trainingNormalizedBox(box)
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
return x >= box.X-margin &&
|
|
x <= box.X+box.W+margin &&
|
|
y >= box.Y-margin &&
|
|
y <= box.Y+box.H+margin
|
|
}
|
|
|
|
func trainingAnyPoseKeypointNearBoxes(
|
|
pose TrainingPosePrediction,
|
|
keypointNames []string,
|
|
boxes []TrainingBox,
|
|
margin float64,
|
|
) bool {
|
|
if len(boxes) == 0 {
|
|
return false
|
|
}
|
|
|
|
nameSet := stringSet(keypointNames)
|
|
|
|
for _, person := range pose.Persons {
|
|
if !trainingPosePersonReliable(person) {
|
|
continue
|
|
}
|
|
|
|
for _, point := range person.Keypoints {
|
|
if !nameSet[strings.TrimSpace(point.Name)] {
|
|
continue
|
|
}
|
|
if point.Conf < trainingPoseKeypointMinConfidence {
|
|
continue
|
|
}
|
|
|
|
for _, box := range boxes {
|
|
if trainingPointNearBox(point.X, point.Y, box, margin) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func trainingPoseKeypointStats(person TrainingPosePerson) (int, float64) {
|
|
if len(person.Keypoints) == 0 {
|
|
return 0, 0
|
|
}
|
|
|
|
visible := 0
|
|
totalConf := 0.0
|
|
|
|
for _, point := range person.Keypoints {
|
|
if point.Conf < trainingPoseKeypointMinConfidence ||
|
|
!trainingIsFinite01(point.X) ||
|
|
!trainingIsFinite01(point.Y) {
|
|
continue
|
|
}
|
|
|
|
visible++
|
|
totalConf += clamp01(point.Conf)
|
|
}
|
|
|
|
if visible == 0 {
|
|
return 0, 0
|
|
}
|
|
|
|
coverage := clamp01(float64(visible) / float64(trainingPoseKeypointCount))
|
|
avgConf := clamp01(totalConf / float64(visible))
|
|
|
|
return visible, clamp01(coverage*0.45 + avgConf*0.55)
|
|
}
|
|
|
|
func trainingPoseKeypointQuality(person TrainingPosePerson) float64 {
|
|
_, quality := trainingPoseKeypointStats(person)
|
|
return quality
|
|
}
|
|
|
|
func trainingAnnotatePosePersonQuality(person TrainingPosePerson) TrainingPosePerson {
|
|
visible, quality := trainingPoseKeypointStats(person)
|
|
person.VisibleKeypoints = visible
|
|
person.Quality = quality
|
|
person.Reliable = person.Score >= trainingPoseReliableMinScore &&
|
|
visible >= trainingPoseReliableMinKeypoints &&
|
|
quality >= trainingPoseReliableMinQuality
|
|
|
|
return person
|
|
}
|
|
|
|
func trainingPosePersonReliable(person TrainingPosePerson) bool {
|
|
visible := person.VisibleKeypoints
|
|
quality := person.Quality
|
|
|
|
if visible == 0 && quality == 0 && len(person.Keypoints) > 0 {
|
|
visible, quality = trainingPoseKeypointStats(person)
|
|
}
|
|
|
|
return person.Score >= trainingPoseReliableMinScore &&
|
|
visible >= trainingPoseReliableMinKeypoints &&
|
|
quality >= trainingPoseReliableMinQuality
|
|
}
|
|
|
|
func trainingAnnotatePosePrediction(pose TrainingPosePrediction) TrainingPosePrediction {
|
|
for i := range pose.Persons {
|
|
pose.Persons[i] = trainingAnnotatePosePersonQuality(pose.Persons[i])
|
|
}
|
|
|
|
if pose.PersonCount == 0 {
|
|
pose.PersonCount = len(pose.Persons)
|
|
}
|
|
|
|
return pose
|
|
}
|
|
|
|
func trainingReliablePosePrediction(pose TrainingPosePrediction) TrainingPosePrediction {
|
|
pose = trainingAnnotatePosePrediction(pose)
|
|
|
|
reliable := make([]TrainingPosePerson, 0, len(pose.Persons))
|
|
for _, person := range pose.Persons {
|
|
if trainingPosePersonReliable(person) {
|
|
reliable = append(reliable, person)
|
|
}
|
|
}
|
|
|
|
pose.Persons = reliable
|
|
pose.PersonCount = len(reliable)
|
|
|
|
return pose
|
|
}
|
|
|
|
func trainingScenePersonBoxes(pred TrainingPrediction, pose TrainingPosePrediction) []TrainingBox {
|
|
detectorBoxes := []TrainingBox{}
|
|
poseBoxes := []TrainingBox{}
|
|
|
|
for _, box := range pred.Boxes {
|
|
if trainingIsPersonLikeLabel(box.Label) {
|
|
if normalized, ok := trainingNormalizedBox(box); ok {
|
|
detectorBoxes = append(detectorBoxes, normalized)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, person := range pose.Persons {
|
|
if !trainingPosePersonReliable(person) {
|
|
continue
|
|
}
|
|
|
|
if normalized, ok := trainingNormalizedBox(person.Box); ok {
|
|
poseBoxes = append(poseBoxes, normalized)
|
|
}
|
|
}
|
|
|
|
if len(detectorBoxes) >= len(poseBoxes) && len(detectorBoxes) > 0 {
|
|
return detectorBoxes
|
|
}
|
|
|
|
return poseBoxes
|
|
}
|
|
|
|
func trainingIsPersonLikeLabel(label string) bool {
|
|
label = strings.TrimSpace(label)
|
|
return label == "person" || strings.HasPrefix(label, "person_")
|
|
}
|
|
|
|
type trainingPersonPairSignals struct {
|
|
close bool
|
|
overlap bool
|
|
horizontal bool
|
|
vertical bool
|
|
stacked bool
|
|
}
|
|
|
|
func trainingScenePersonPairSignals(boxes []TrainingBox) trainingPersonPairSignals {
|
|
signals := trainingPersonPairSignals{}
|
|
|
|
for i := 0; i < len(boxes); i++ {
|
|
a, okA := trainingNormalizedBox(boxes[i])
|
|
if !okA {
|
|
continue
|
|
}
|
|
|
|
for j := i + 1; j < len(boxes); j++ {
|
|
b, okB := trainingNormalizedBox(boxes[j])
|
|
if !okB {
|
|
continue
|
|
}
|
|
|
|
gap := trainingBoxGap(a, b)
|
|
overlap := trainingBoxOverlapRatio(a, b)
|
|
close := gap <= 0.08 || overlap >= 0.08
|
|
if close {
|
|
signals.close = true
|
|
}
|
|
if overlap >= 0.18 {
|
|
signals.overlap = true
|
|
}
|
|
|
|
if close && a.W > a.H*1.15 && b.W > b.H*1.15 {
|
|
signals.horizontal = true
|
|
}
|
|
if close && a.H > a.W*1.25 && b.H > b.W*1.25 {
|
|
signals.vertical = true
|
|
}
|
|
|
|
_, ay, okACenter := trainingBoxCenter(a)
|
|
_, by, okBCenter := trainingBoxCenter(b)
|
|
if okACenter && okBCenter &&
|
|
gap <= 0.12 &&
|
|
trainingBoxHorizontalOverlapRatio(a, b) >= 0.35 &&
|
|
math.Abs(ay-by) >= 0.12 {
|
|
signals.stacked = true
|
|
}
|
|
}
|
|
}
|
|
|
|
return signals
|
|
}
|
|
|
|
type trainingPosePoint struct {
|
|
x float64
|
|
y float64
|
|
}
|
|
|
|
type trainingPosePersonGeometry struct {
|
|
box TrainingBox
|
|
hasBox bool
|
|
center trainingPosePoint
|
|
lying bool
|
|
upright bool
|
|
straddling bool
|
|
kneesBelowHips bool
|
|
kneesWide bool
|
|
allFours bool
|
|
bentOrKneeling bool
|
|
}
|
|
|
|
func trainingPosePointByName(person TrainingPosePerson, name string) (trainingPosePoint, bool) {
|
|
name = strings.TrimSpace(name)
|
|
if name == "" {
|
|
return trainingPosePoint{}, false
|
|
}
|
|
|
|
for _, point := range person.Keypoints {
|
|
if strings.TrimSpace(point.Name) != name {
|
|
continue
|
|
}
|
|
if point.Conf < trainingPoseKeypointMinConfidence ||
|
|
!trainingIsFinite01(point.X) ||
|
|
!trainingIsFinite01(point.Y) {
|
|
continue
|
|
}
|
|
|
|
return trainingPosePoint{x: point.X, y: point.Y}, true
|
|
}
|
|
|
|
return trainingPosePoint{}, false
|
|
}
|
|
|
|
func trainingPoseMidpoint(person TrainingPosePerson, names ...string) (trainingPosePoint, bool) {
|
|
count := 0
|
|
x := 0.0
|
|
y := 0.0
|
|
|
|
for _, name := range names {
|
|
point, ok := trainingPosePointByName(person, name)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
count++
|
|
x += point.x
|
|
y += point.y
|
|
}
|
|
|
|
if count == 0 {
|
|
return trainingPosePoint{}, false
|
|
}
|
|
|
|
return trainingPosePoint{
|
|
x: x / float64(count),
|
|
y: y / float64(count),
|
|
}, true
|
|
}
|
|
|
|
func trainingPosePointDistance(a trainingPosePoint, okA bool, b trainingPosePoint, okB bool) float64 {
|
|
if !okA || !okB {
|
|
return 0
|
|
}
|
|
|
|
dx := a.x - b.x
|
|
dy := a.y - b.y
|
|
return math.Sqrt(dx*dx + dy*dy)
|
|
}
|
|
|
|
func trainingPosePersonGeometryFor(person TrainingPosePerson) trainingPosePersonGeometry {
|
|
box, hasBox := trainingNormalizedBox(person.Box)
|
|
center := trainingPosePoint{x: 0.5, y: 0.5}
|
|
if hasBox {
|
|
if x, y, ok := trainingBoxCenter(box); ok {
|
|
center = trainingPosePoint{x: x, y: y}
|
|
}
|
|
}
|
|
|
|
leftHip, okLeftHip := trainingPosePointByName(person, "left_hip")
|
|
rightHip, okRightHip := trainingPosePointByName(person, "right_hip")
|
|
leftKnee, okLeftKnee := trainingPosePointByName(person, "left_knee")
|
|
rightKnee, okRightKnee := trainingPosePointByName(person, "right_knee")
|
|
|
|
hip, okHip := trainingPoseMidpoint(person, "left_hip", "right_hip")
|
|
shoulder, okShoulder := trainingPoseMidpoint(person, "left_shoulder", "right_shoulder")
|
|
knee, okKnee := trainingPoseMidpoint(person, "left_knee", "right_knee")
|
|
|
|
if okHip {
|
|
center = hip
|
|
}
|
|
|
|
torsoDX := 0.0
|
|
torsoDY := 0.0
|
|
torsoLen := 0.0
|
|
if okHip && okShoulder {
|
|
torsoDX = math.Abs(hip.x - shoulder.x)
|
|
torsoDY = math.Abs(hip.y - shoulder.y)
|
|
torsoLen = math.Sqrt(torsoDX*torsoDX + torsoDY*torsoDY)
|
|
}
|
|
|
|
hipWidth := trainingPosePointDistance(leftHip, okLeftHip, rightHip, okRightHip)
|
|
kneeWidth := trainingPosePointDistance(leftKnee, okLeftKnee, rightKnee, okRightKnee)
|
|
|
|
kneesBelowHips := okKnee && okHip && knee.y > hip.y+0.045
|
|
kneesWide := kneeWidth > 0 && kneeWidth >= math.Max(0.11, hipWidth*1.12)
|
|
straddling := kneesBelowHips && kneesWide
|
|
|
|
torsoHorizontal := torsoLen >= 0.07 && torsoDX >= torsoDY*1.15
|
|
torsoVertical := torsoLen >= 0.07 && torsoDY >= torsoDX*1.15
|
|
|
|
boxHorizontal := hasBox && box.W >= box.H*1.05
|
|
boxVertical := hasBox && box.H >= box.W*1.25
|
|
|
|
lying := torsoHorizontal || boxHorizontal
|
|
upright := torsoVertical || boxVertical
|
|
allFours := torsoHorizontal && kneesBelowHips
|
|
bentOrKneeling := allFours || (kneesBelowHips && !straddling)
|
|
|
|
return trainingPosePersonGeometry{
|
|
box: box,
|
|
hasBox: hasBox,
|
|
center: center,
|
|
lying: lying,
|
|
upright: upright,
|
|
straddling: straddling,
|
|
kneesBelowHips: kneesBelowHips,
|
|
kneesWide: kneesWide,
|
|
allFours: allFours,
|
|
bentOrKneeling: bentOrKneeling,
|
|
}
|
|
}
|
|
|
|
func trainingAddPosePairGeometryScores(
|
|
scores map[string]float64,
|
|
positionSet map[string]bool,
|
|
pose TrainingPosePrediction,
|
|
) {
|
|
add := func(label string, score float64) {
|
|
trainingCombinePositionScore(scores, positionSet, label, score)
|
|
}
|
|
|
|
pose = trainingReliablePosePrediction(pose)
|
|
if len(pose.Persons) < 2 {
|
|
return
|
|
}
|
|
|
|
geometries := make([]trainingPosePersonGeometry, 0, len(pose.Persons))
|
|
for _, person := range pose.Persons {
|
|
geometries = append(geometries, trainingPosePersonGeometryFor(person))
|
|
}
|
|
|
|
for i := 0; i < len(geometries); i++ {
|
|
left := geometries[i]
|
|
if !left.hasBox {
|
|
continue
|
|
}
|
|
|
|
for j := i + 1; j < len(geometries); j++ {
|
|
right := geometries[j]
|
|
if !right.hasBox {
|
|
continue
|
|
}
|
|
|
|
gap := trainingBoxGap(left.box, right.box)
|
|
overlap := trainingBoxOverlapRatio(left.box, right.box)
|
|
horizontalOverlap := trainingBoxHorizontalOverlapRatio(left.box, right.box)
|
|
close := gap <= 0.12 || overlap >= 0.08
|
|
if !close {
|
|
continue
|
|
}
|
|
|
|
top := left
|
|
bottom := right
|
|
if right.center.y < left.center.y {
|
|
top = right
|
|
bottom = left
|
|
}
|
|
|
|
topAbove := top.center.y <= bottom.center.y-0.055
|
|
strongStack := horizontalOverlap >= 0.35 && (topAbove || overlap >= 0.22)
|
|
topHasRiderShape := top.straddling ||
|
|
top.kneesWide ||
|
|
(top.upright && top.kneesBelowHips)
|
|
|
|
if strongStack && topHasRiderShape {
|
|
add("cowgirl", 0.20)
|
|
add("reverse_cowgirl", 0.17)
|
|
|
|
if bottom.lying {
|
|
add("cowgirl", 0.12)
|
|
add("reverse_cowgirl", 0.10)
|
|
}
|
|
if top.straddling {
|
|
add("cowgirl", 0.08)
|
|
add("reverse_cowgirl", 0.06)
|
|
}
|
|
}
|
|
|
|
if strongStack && bottom.lying {
|
|
add("missionary", 0.14)
|
|
if !top.straddling {
|
|
add("missionary", 0.08)
|
|
}
|
|
}
|
|
|
|
bothLying := left.lying && right.lying
|
|
sameLevel := math.Abs(left.center.y-right.center.y) <= 0.15
|
|
sideBySide := math.Abs(left.center.x-right.center.x) >= 0.10
|
|
if bothLying && (sameLevel || sideBySide) {
|
|
add("spooning", 0.18)
|
|
if overlap >= 0.10 {
|
|
add("prone_bone", 0.07)
|
|
}
|
|
}
|
|
|
|
leftBent := left.allFours || left.bentOrKneeling
|
|
rightBent := right.allFours || right.bentOrKneeling
|
|
if leftBent != rightBent {
|
|
other := right
|
|
if rightBent {
|
|
other = left
|
|
}
|
|
|
|
add("doggy", 0.16)
|
|
if other.upright {
|
|
add("doggy", 0.06)
|
|
add("standing_doggy", 0.07)
|
|
}
|
|
if left.lying || right.lying {
|
|
add("prone_bone", 0.06)
|
|
}
|
|
} else if left.allFours || right.allFours {
|
|
add("doggy", 0.13)
|
|
if left.lying || right.lying {
|
|
add("prone_bone", 0.07)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func trainingAddPoseSceneContextScores(
|
|
scores map[string]float64,
|
|
positionSet map[string]bool,
|
|
pred TrainingPrediction,
|
|
pose TrainingPosePrediction,
|
|
) {
|
|
add := func(label string, score float64) {
|
|
trainingCombinePositionScore(scores, positionSet, label, score)
|
|
}
|
|
|
|
pose = trainingReliablePosePrediction(pose)
|
|
personBoxes := trainingScenePersonBoxes(pred, pose)
|
|
personCount := len(personBoxes)
|
|
if len(pose.Persons) > personCount {
|
|
personCount = len(pose.Persons)
|
|
}
|
|
|
|
pair := trainingScenePersonPairSignals(personBoxes)
|
|
trainingAddPosePairGeometryScores(scores, positionSet, pose)
|
|
|
|
penisBoxes := trainingBoxesByLabel(pred.Boxes, "penis")
|
|
pussyBoxes := trainingBoxesByLabel(pred.Boxes, "pussy", "vagina", "vulva", "labia")
|
|
assBoxes := trainingBoxesByLabel(pred.Boxes, "ass", "anus")
|
|
breastBoxes := trainingBoxesByLabel(pred.Boxes, "breasts")
|
|
tongueBoxes := trainingBoxesByLabel(pred.Boxes, "tongue")
|
|
toyBoxes := trainingBoxesByLabel(pred.Boxes, "dildo", "vibrator", "strapon", "buttplug")
|
|
|
|
hasPenis := len(penisBoxes) > 0
|
|
hasPussy := len(pussyBoxes) > 0
|
|
hasAss := len(assBoxes) > 0
|
|
hasBreasts := len(breastBoxes) > 0
|
|
hasTongue := len(tongueBoxes) > 0
|
|
hasToy := len(toyBoxes) > 0
|
|
|
|
headNames := []string{"nose", "left_eye", "right_eye", "left_ear", "right_ear"}
|
|
handNames := []string{"left_wrist", "right_wrist"}
|
|
hipNames := []string{"left_hip", "right_hip"}
|
|
|
|
headNearPenis := trainingAnyPoseKeypointNearBoxes(pose, headNames, penisBoxes, 0.09)
|
|
headNearPussy := trainingAnyPoseKeypointNearBoxes(pose, headNames, pussyBoxes, 0.09)
|
|
headNearAss := trainingAnyPoseKeypointNearBoxes(pose, headNames, assBoxes, 0.09)
|
|
handNearPenis := trainingAnyPoseKeypointNearBoxes(pose, handNames, penisBoxes, 0.08)
|
|
handNearPussy := trainingAnyPoseKeypointNearBoxes(pose, handNames, pussyBoxes, 0.08)
|
|
handNearToy := trainingAnyPoseKeypointNearBoxes(pose, handNames, toyBoxes, 0.08)
|
|
hipsNearGenitals := trainingAnyPoseKeypointNearBoxes(pose, hipNames, append(penisBoxes, pussyBoxes...), 0.08)
|
|
|
|
if personCount >= 2 {
|
|
add("missionary", 0.04)
|
|
add("doggy", 0.04)
|
|
add("cowgirl", 0.04)
|
|
add("reverse_cowgirl", 0.04)
|
|
add("standing_doggy", 0.04)
|
|
add("spooning", 0.04)
|
|
add("69", 0.03)
|
|
}
|
|
|
|
if pair.close {
|
|
add("missionary", 0.04)
|
|
add("doggy", 0.04)
|
|
add("cowgirl", 0.04)
|
|
add("spooning", 0.04)
|
|
}
|
|
if pair.overlap {
|
|
add("missionary", 0.05)
|
|
add("cowgirl", 0.05)
|
|
add("prone_bone", 0.04)
|
|
}
|
|
if pair.horizontal {
|
|
add("spooning", 0.12)
|
|
add("prone_bone", 0.07)
|
|
}
|
|
if pair.vertical {
|
|
add("standing", 0.08)
|
|
add("standing_doggy", 0.10)
|
|
}
|
|
if pair.stacked {
|
|
add("missionary", 0.07)
|
|
add("cowgirl", 0.07)
|
|
add("reverse_cowgirl", 0.06)
|
|
add("facesitting", 0.05)
|
|
}
|
|
|
|
if hasPenis && hasPussy {
|
|
add("missionary", 0.08)
|
|
add("doggy", 0.08)
|
|
add("cowgirl", 0.08)
|
|
add("reverse_cowgirl", 0.07)
|
|
add("prone_bone", 0.06)
|
|
add("standing_doggy", 0.06)
|
|
add("spooning", 0.05)
|
|
|
|
if trainingAnyBoxesNear(penisBoxes, pussyBoxes, 0.09) || hipsNearGenitals {
|
|
add("missionary", 0.08)
|
|
add("doggy", 0.08)
|
|
add("cowgirl", 0.08)
|
|
add("reverse_cowgirl", 0.07)
|
|
}
|
|
}
|
|
|
|
if hasPenis && (headNearPenis || hasTongue) {
|
|
add("blowjob", 0.16)
|
|
if headNearPenis {
|
|
add("blowjob", 0.10)
|
|
}
|
|
}
|
|
if hasPussy && (headNearPussy || hasTongue || trainingAnyBoxesNear(tongueBoxes, pussyBoxes, 0.08)) {
|
|
add("cunnilingus", 0.16)
|
|
if headNearPussy || trainingAnyBoxesNear(tongueBoxes, pussyBoxes, 0.08) {
|
|
add("cunnilingus", 0.10)
|
|
}
|
|
}
|
|
if hasPenis && handNearPenis {
|
|
add("handjob", 0.18)
|
|
}
|
|
if hasPussy && handNearPussy {
|
|
add("fingering", 0.18)
|
|
}
|
|
if hasPenis && hasBreasts && trainingAnyBoxesNear(penisBoxes, breastBoxes, 0.10) {
|
|
add("boobjob", 0.20)
|
|
}
|
|
if hasToy {
|
|
add("toy_play", 0.12)
|
|
if handNearToy ||
|
|
trainingAnyBoxesNear(toyBoxes, pussyBoxes, 0.10) ||
|
|
trainingAnyBoxesNear(toyBoxes, penisBoxes, 0.10) ||
|
|
trainingAnyBoxesNear(toyBoxes, assBoxes, 0.10) {
|
|
add("toy_play", 0.12)
|
|
}
|
|
}
|
|
if personCount >= 2 && (headNearAss || headNearPussy) {
|
|
if hasAss {
|
|
add("facesitting", 0.14)
|
|
}
|
|
if hasPussy && hasPenis {
|
|
add("69", 0.10)
|
|
}
|
|
}
|
|
if hasAss && hasPussy && pair.horizontal {
|
|
add("doggy", 0.07)
|
|
add("prone_bone", 0.07)
|
|
}
|
|
}
|
|
|
|
func trainingApplyPoseToPrediction(pred TrainingPrediction, pose TrainingPosePrediction) TrainingPrediction {
|
|
if pose.Available {
|
|
pred.ModelAvailable = true
|
|
}
|
|
|
|
positionSet := stringSet(defaultTrainingLabelsFromJSON().SexPositions)
|
|
poseScores := map[string]float64{}
|
|
contextScores := map[string]float64{}
|
|
reliablePose := TrainingPosePrediction{
|
|
Available: pose.Available,
|
|
Source: pose.Source,
|
|
Persons: []TrainingPosePerson{},
|
|
}
|
|
|
|
if pose.Available && len(pose.Persons) > 0 {
|
|
pose = trainingAnnotatePosePrediction(pose)
|
|
pred.Persons = pose.Persons
|
|
reliablePose = trainingReliablePosePrediction(pose)
|
|
}
|
|
|
|
for _, person := range reliablePose.Persons {
|
|
label := normalizeSexPositionLabel(person.Label)
|
|
if isNoSexPositionLabel(label) || !positionSet[label] {
|
|
continue
|
|
}
|
|
|
|
trainingCombinePositionScore(poseScores, positionSet, label, person.Score)
|
|
|
|
if quality := trainingPoseKeypointQuality(person); quality > 0 {
|
|
trainingCombinePositionScore(poseScores, positionSet, label, 0.04*quality)
|
|
}
|
|
}
|
|
|
|
trainingAddPoseSceneContextScores(contextScores, positionSet, pred, reliablePose)
|
|
|
|
bestPosition, bestPositionScore, hasPoseSignal, hasContextSignal :=
|
|
trainingFuseHybridPositionScores(poseScores, contextScores)
|
|
|
|
if bestPosition != "" && (hasPoseSignal || bestPositionScore >= trainingPositionContextMinScore) {
|
|
pred.SexPosition = bestPosition
|
|
pred.SexPositionScore = clamp01(bestPositionScore)
|
|
}
|
|
|
|
if pose.Available {
|
|
trainingAppendPredictionSource(&pred, "yolo_pose")
|
|
}
|
|
|
|
if hasContextSignal {
|
|
trainingAppendPredictionSource(&pred, "box_context")
|
|
}
|
|
|
|
return pred
|
|
}
|
|
|
|
func trainingDetectorLabelContent(
|
|
boxes []TrainingBox,
|
|
classMap map[string]int,
|
|
allowEmpty bool,
|
|
) ([]byte, error) {
|
|
var lines []string
|
|
|
|
for _, box := range boxes {
|
|
label := strings.TrimSpace(box.Label)
|
|
if label == "" {
|
|
continue
|
|
}
|
|
|
|
classID, ok := classMap[label]
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
x := clamp01(box.X)
|
|
y := clamp01(box.Y)
|
|
w := clamp01(box.W)
|
|
h := clamp01(box.H)
|
|
|
|
if w <= 0 || h <= 0 {
|
|
continue
|
|
}
|
|
|
|
// Frontend/Predictor nutzen x/y als linke obere Ecke.
|
|
// YOLO erwartet x_center/y_center.
|
|
xCenter := clamp01(x + w/2)
|
|
yCenter := clamp01(y + h/2)
|
|
|
|
lines = append(lines, fmt.Sprintf(
|
|
"%d %.6f %.6f %.6f %.6f",
|
|
classID,
|
|
xCenter,
|
|
yCenter,
|
|
w,
|
|
h,
|
|
))
|
|
}
|
|
|
|
if len(lines) == 0 {
|
|
if allowEmpty {
|
|
return []byte{}, nil
|
|
}
|
|
return nil, errors.New("no valid detector boxes")
|
|
}
|
|
|
|
return []byte(strings.Join(lines, "\n") + "\n"), nil
|
|
}
|
|
|
|
func trainingWriteDetectorSample(
|
|
root string,
|
|
sample *TrainingSample,
|
|
boxes []TrainingBox,
|
|
allowEmpty bool,
|
|
) error {
|
|
if sample == nil {
|
|
return errors.New("sample missing")
|
|
}
|
|
|
|
classMap, err := trainingDetectorClassMap()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
labelContent, err := trainingDetectorLabelContent(boxes, classMap, allowEmpty)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
srcFrame := filepath.Join(root, "frames", sample.SampleID+".jpg")
|
|
if _, err := os.Stat(srcFrame); err != nil {
|
|
return appErrorf("frame missing: %w", err)
|
|
}
|
|
|
|
// Stabiler 80/20 Split: gleicher sampleID landet immer im gleichen Split.
|
|
split := trainingStableSplit(sample.SampleID)
|
|
|
|
imgDir := filepath.Join(root, "detector", "dataset", "images", split)
|
|
lblDir := filepath.Join(root, "detector", "dataset", "labels", split)
|
|
|
|
if err := os.MkdirAll(imgDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(lblDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
dstFrame := filepath.Join(imgDir, sample.SampleID+".jpg")
|
|
if err := copyFile(srcFrame, dstFrame); err != nil {
|
|
return err
|
|
}
|
|
|
|
labelPath := filepath.Join(lblDir, sample.SampleID+".txt")
|
|
return os.WriteFile(labelPath, labelContent, 0644)
|
|
}
|
|
|
|
func trainingSexPositionForFeedback(sample *TrainingSample, req TrainingFeedbackRequest) string {
|
|
if req.Negative {
|
|
return ""
|
|
}
|
|
|
|
if req.Correction != nil {
|
|
return normalizeSexPositionLabel(req.Correction.SexPosition)
|
|
}
|
|
|
|
if req.Accepted && sample != nil {
|
|
return normalizeSexPositionLabel(sample.Prediction.SexPosition)
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func trainingPosePersonsForSample(root string, sample *TrainingSample) []TrainingPosePerson {
|
|
if sample == nil {
|
|
return nil
|
|
}
|
|
|
|
if len(sample.Prediction.Persons) > 0 {
|
|
return sample.Prediction.Persons
|
|
}
|
|
|
|
framePath := filepath.Join(root, "frames", sample.SampleID+".jpg")
|
|
if !fileExistsNonEmpty(framePath) {
|
|
return nil
|
|
}
|
|
|
|
pose := trainingPredictPose(root, framePath)
|
|
if !pose.Available || len(pose.Persons) == 0 {
|
|
return nil
|
|
}
|
|
|
|
return pose.Persons
|
|
}
|
|
|
|
func trainingPosePersonsForCorrection(root string, sample *TrainingSample, correction *TrainingCorrection) []TrainingPosePerson {
|
|
if correction != nil && correction.PosePersons != nil {
|
|
persons := make([]TrainingPosePerson, 0, len(correction.PosePersons))
|
|
for _, person := range correction.PosePersons {
|
|
persons = append(persons, trainingAnnotatePosePersonQuality(person))
|
|
}
|
|
return persons
|
|
}
|
|
|
|
return trainingPosePersonsForSample(root, sample)
|
|
}
|
|
|
|
func trainingPoseContextPersonBoxes(boxes []TrainingBox) []TrainingBox {
|
|
out := []TrainingBox{}
|
|
|
|
for _, box := range boxes {
|
|
if !trainingIsPersonLikeLabel(box.Label) {
|
|
continue
|
|
}
|
|
|
|
if normalized, ok := trainingNormalizedBox(box); ok {
|
|
out = append(out, normalized)
|
|
}
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func trainingFilterPosePersonsByContext(
|
|
persons []TrainingPosePerson,
|
|
contextBoxes []TrainingBox,
|
|
) []TrainingPosePerson {
|
|
if len(persons) == 0 {
|
|
return persons
|
|
}
|
|
|
|
personBoxes := trainingPoseContextPersonBoxes(contextBoxes)
|
|
if len(personBoxes) == 0 {
|
|
return persons
|
|
}
|
|
|
|
filtered := []TrainingPosePerson{}
|
|
|
|
for _, person := range persons {
|
|
personBox, ok := trainingNormalizedBox(person.Box)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
for _, contextBox := range personBoxes {
|
|
if trainingBoxOverlapRatio(personBox, contextBox) >= 0.12 ||
|
|
trainingBoxGap(personBox, contextBox) <= 0.08 {
|
|
filtered = append(filtered, person)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(filtered) == 0 {
|
|
return persons
|
|
}
|
|
|
|
return filtered
|
|
}
|
|
|
|
func trainingPoseLabelContent(persons []TrainingPosePerson, classID int) ([]byte, error) {
|
|
lines := []string{}
|
|
|
|
for _, person := range persons {
|
|
person = trainingAnnotatePosePersonQuality(person)
|
|
if !trainingPosePersonReliable(person) {
|
|
continue
|
|
}
|
|
|
|
if len(person.Keypoints) < trainingPoseKeypointCount {
|
|
continue
|
|
}
|
|
|
|
x := clamp01(person.Box.X)
|
|
y := clamp01(person.Box.Y)
|
|
w := clamp01(person.Box.W)
|
|
h := clamp01(person.Box.H)
|
|
|
|
if w <= 0 || h <= 0 {
|
|
continue
|
|
}
|
|
|
|
xCenter := clamp01(x + w/2)
|
|
yCenter := clamp01(y + h/2)
|
|
|
|
parts := []string{
|
|
strconv.Itoa(classID),
|
|
fmt.Sprintf("%.6f", xCenter),
|
|
fmt.Sprintf("%.6f", yCenter),
|
|
fmt.Sprintf("%.6f", w),
|
|
fmt.Sprintf("%.6f", h),
|
|
}
|
|
|
|
visible := 0
|
|
for i := 0; i < trainingPoseKeypointCount; i++ {
|
|
kp := person.Keypoints[i]
|
|
kx := clamp01(kp.X)
|
|
ky := clamp01(kp.Y)
|
|
visibility := 0
|
|
|
|
if kp.Conf >= trainingPoseKeypointMinConfidence && kx > 0 && ky > 0 {
|
|
visibility = 2
|
|
visible++
|
|
} else {
|
|
kx = 0
|
|
ky = 0
|
|
}
|
|
|
|
parts = append(
|
|
parts,
|
|
fmt.Sprintf("%.6f", kx),
|
|
fmt.Sprintf("%.6f", ky),
|
|
strconv.Itoa(visibility),
|
|
)
|
|
}
|
|
|
|
if visible < 5 {
|
|
continue
|
|
}
|
|
|
|
lines = append(lines, strings.Join(parts, " "))
|
|
}
|
|
|
|
if len(lines) == 0 {
|
|
return nil, errors.New("no valid pose persons")
|
|
}
|
|
|
|
return []byte(strings.Join(lines, "\n") + "\n"), nil
|
|
}
|
|
|
|
func trainingWritePoseSample(
|
|
root string,
|
|
sample *TrainingSample,
|
|
sexPosition string,
|
|
contextBoxes []TrainingBox,
|
|
correction *TrainingCorrection,
|
|
) error {
|
|
if sample == nil {
|
|
return errors.New("sample missing")
|
|
}
|
|
|
|
sexPosition = strings.TrimSpace(sexPosition)
|
|
if isNoSexPositionLabel(sexPosition) {
|
|
return errors.New("pose sex position missing")
|
|
}
|
|
|
|
classMap, err := trainingPoseClassMap()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
classID, ok := classMap[sexPosition]
|
|
if !ok {
|
|
return fmt.Errorf("pose class not configured: %s", sexPosition)
|
|
}
|
|
|
|
persons := trainingPosePersonsForCorrection(root, sample, correction)
|
|
persons = trainingFilterPosePersonsByContext(persons, contextBoxes)
|
|
labelContent, err := trainingPoseLabelContent(persons, classID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
srcFrame := filepath.Join(root, "frames", sample.SampleID+".jpg")
|
|
if _, err := os.Stat(srcFrame); err != nil {
|
|
return appErrorf("frame missing: %w", err)
|
|
}
|
|
|
|
split := trainingStableSplit(sample.SampleID)
|
|
|
|
imgDir := filepath.Join(root, "pose", "dataset", "images", split)
|
|
lblDir := filepath.Join(root, "pose", "dataset", "labels", split)
|
|
|
|
if err := os.MkdirAll(imgDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(lblDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
dstFrame := filepath.Join(imgDir, sample.SampleID+".jpg")
|
|
if err := copyFile(srcFrame, dstFrame); err != nil {
|
|
return err
|
|
}
|
|
|
|
labelPath := filepath.Join(lblDir, sample.SampleID+".txt")
|
|
return os.WriteFile(labelPath, labelContent, 0644)
|
|
}
|
|
|
|
func trainingDeleteDetectorSample(root string, sampleID string) {
|
|
sampleID = strings.TrimSpace(sampleID)
|
|
if sampleID == "" || strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") {
|
|
return
|
|
}
|
|
|
|
for _, split := range []string{"train", "val"} {
|
|
labelsDir := filepath.Join(root, "detector", "dataset", "labels", split)
|
|
imagesDir := filepath.Join(root, "detector", "dataset", "images", split)
|
|
|
|
_ = os.Remove(filepath.Join(labelsDir, sampleID+".txt"))
|
|
|
|
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
|
_ = os.Remove(filepath.Join(imagesDir, sampleID+ext))
|
|
}
|
|
}
|
|
}
|
|
|
|
func trainingDeletePoseSample(root string, sampleID string) {
|
|
sampleID = strings.TrimSpace(sampleID)
|
|
if sampleID == "" || strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") {
|
|
return
|
|
}
|
|
|
|
for _, split := range []string{"train", "val"} {
|
|
labelsDir := filepath.Join(root, "pose", "dataset", "labels", split)
|
|
imagesDir := filepath.Join(root, "pose", "dataset", "images", split)
|
|
|
|
_ = os.Remove(filepath.Join(labelsDir, sampleID+".txt"))
|
|
|
|
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
|
_ = os.Remove(filepath.Join(imagesDir, sampleID+ext))
|
|
}
|
|
}
|
|
}
|
|
|
|
func trainingSyncPoseDataset(root string) (int, error) {
|
|
if err := trainingEnsurePoseDirs(root); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
items, err := trainingReadAnnotations(root)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
written := 0
|
|
|
|
for _, item := range items {
|
|
sampleID := strings.TrimSpace(item.SampleID)
|
|
if sampleID == "" {
|
|
continue
|
|
}
|
|
|
|
sample := &TrainingSample{
|
|
SampleID: item.SampleID,
|
|
FrameURL: item.FrameURL,
|
|
SourceFile: item.SourceFile,
|
|
SourcePath: item.SourcePath,
|
|
SourceSizeBytes: item.SourceSizeBytes,
|
|
Second: item.Second,
|
|
CreatedAt: item.CreatedAt,
|
|
UncertaintyScore: 0,
|
|
Prediction: item.Prediction,
|
|
}
|
|
|
|
effective := trainingEffectiveCorrection(item)
|
|
sexPosition := strings.TrimSpace(effective.SexPosition)
|
|
|
|
trainingDeletePoseSample(root, sampleID)
|
|
|
|
if item.Negative || isNoSexPositionLabel(sexPosition) {
|
|
continue
|
|
}
|
|
|
|
if err := trainingWritePoseSample(root, sample, sexPosition, effective.Boxes, &effective); err != nil {
|
|
appLogln("pose sample sync skipped:", sampleID, err)
|
|
continue
|
|
}
|
|
|
|
written++
|
|
}
|
|
|
|
return written, nil
|
|
}
|
|
|
|
func trainingEnsureDetectorValidationSample(root string) error {
|
|
trainImages := filepath.Join(root, "detector", "dataset", "images", "train")
|
|
trainLabels := filepath.Join(root, "detector", "dataset", "labels", "train")
|
|
valImages := filepath.Join(root, "detector", "dataset", "images", "val")
|
|
valLabels := filepath.Join(root, "detector", "dataset", "labels", "val")
|
|
|
|
currentVal := trainingCountDetectorSamples(valImages, valLabels)
|
|
currentPositiveVal := trainingCountPositiveDetectorSamples(valImages, valLabels)
|
|
if currentVal >= minDetectorValCount && currentPositiveVal > 0 {
|
|
return nil
|
|
}
|
|
|
|
if trainingCountDetectorSamples(trainImages, trainLabels) < minDetectorTrainCount ||
|
|
trainingCountPositiveDetectorSamples(trainImages, trainLabels) == 0 {
|
|
return nil
|
|
}
|
|
|
|
entries, err := os.ReadDir(trainImages)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
if err := os.MkdirAll(valImages, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(valLabels, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
copied := 0
|
|
needed := max(0, minDetectorValCount-currentVal)
|
|
needsPositive := currentPositiveVal == 0
|
|
|
|
sort.SliceStable(entries, func(i, j int) bool {
|
|
iID := strings.TrimSuffix(entries[i].Name(), filepath.Ext(entries[i].Name()))
|
|
jID := strings.TrimSuffix(entries[j].Name(), filepath.Ext(entries[j].Name()))
|
|
|
|
return fileExistsNonEmpty(filepath.Join(trainLabels, iID+".txt")) &&
|
|
!fileExistsNonEmpty(filepath.Join(trainLabels, jID+".txt"))
|
|
})
|
|
|
|
for _, e := range entries {
|
|
if copied >= needed && !needsPositive {
|
|
break
|
|
}
|
|
|
|
if e.IsDir() {
|
|
continue
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(e.Name()))
|
|
if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name()))
|
|
|
|
srcImage := filepath.Join(trainImages, e.Name())
|
|
srcLabel := filepath.Join(trainLabels, id+".txt")
|
|
|
|
if !fileExistsNonEmpty(srcImage) || !fileExists(srcLabel) {
|
|
continue
|
|
}
|
|
|
|
dstImage := filepath.Join(valImages, e.Name())
|
|
dstLabel := filepath.Join(valLabels, id+".txt")
|
|
|
|
if fileExistsNonEmpty(dstImage) && fileExists(dstLabel) {
|
|
continue
|
|
}
|
|
|
|
if err := copyFile(srcImage, dstImage); err != nil {
|
|
return err
|
|
}
|
|
if err := copyFile(srcLabel, dstLabel); err != nil {
|
|
return err
|
|
}
|
|
|
|
copied++
|
|
if fileExistsNonEmpty(srcLabel) {
|
|
needsPositive = false
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func trainingEnsurePoseValidationSample(root string) error {
|
|
trainImages := filepath.Join(root, "pose", "dataset", "images", "train")
|
|
trainLabels := filepath.Join(root, "pose", "dataset", "labels", "train")
|
|
valImages := filepath.Join(root, "pose", "dataset", "images", "val")
|
|
valLabels := filepath.Join(root, "pose", "dataset", "labels", "val")
|
|
|
|
currentVal := trainingCountDetectorSamples(valImages, valLabels)
|
|
if currentVal >= minPoseValCount {
|
|
return nil
|
|
}
|
|
|
|
if trainingCountDetectorSamples(trainImages, trainLabels) < minPoseTrainCount {
|
|
return nil
|
|
}
|
|
|
|
entries, err := os.ReadDir(trainImages)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
if err := os.MkdirAll(valImages, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(valLabels, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
copied := 0
|
|
needed := max(0, minPoseValCount-currentVal)
|
|
|
|
for _, e := range entries {
|
|
if copied >= needed {
|
|
break
|
|
}
|
|
|
|
if e.IsDir() {
|
|
continue
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(e.Name()))
|
|
if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" {
|
|
continue
|
|
}
|
|
|
|
id := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name()))
|
|
|
|
srcImage := filepath.Join(trainImages, e.Name())
|
|
srcLabel := filepath.Join(trainLabels, id+".txt")
|
|
|
|
if !fileExistsNonEmpty(srcImage) || !fileExistsNonEmpty(srcLabel) {
|
|
continue
|
|
}
|
|
|
|
dstImage := filepath.Join(valImages, e.Name())
|
|
dstLabel := filepath.Join(valLabels, id+".txt")
|
|
|
|
if fileExistsNonEmpty(dstImage) && fileExistsNonEmpty(dstLabel) {
|
|
continue
|
|
}
|
|
|
|
if err := copyFile(srcImage, dstImage); err != nil {
|
|
return err
|
|
}
|
|
if err := copyFile(srcLabel, dstLabel); err != nil {
|
|
return err
|
|
}
|
|
|
|
copied++
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func trainingStableSplit(sampleID string) string {
|
|
sum := sha1.Sum([]byte(sampleID))
|
|
if int(sum[0])%5 == 0 {
|
|
return "val"
|
|
}
|
|
return "train"
|
|
}
|
|
|
|
func copyFile(src string, dst string) error {
|
|
b, err := os.ReadFile(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(dst, b, 0644)
|
|
}
|
|
|
|
func trainingEmptyPrediction(source string) TrainingPrediction {
|
|
return TrainingPrediction{
|
|
ModelAvailable: false,
|
|
Source: source,
|
|
SexPosition: trainingNoSexPositionLabel,
|
|
SexPositionScore: 0,
|
|
PeoplePresent: []TrainingScoredLabel{},
|
|
BodyPartsPresent: []TrainingScoredLabel{},
|
|
ObjectsPresent: []TrainingScoredLabel{},
|
|
ClothingPresent: []TrainingScoredLabel{},
|
|
Boxes: []TrainingBox{},
|
|
}
|
|
}
|
|
|
|
func trainingPythonExe() string {
|
|
v := strings.TrimSpace(os.Getenv("TRAINING_PYTHON"))
|
|
if v != "" {
|
|
return v
|
|
}
|
|
return aiServerPythonPath()
|
|
}
|
|
|
|
func trainingProjectRoot() string {
|
|
wd, err := os.Getwd()
|
|
if err != nil {
|
|
return "."
|
|
}
|
|
|
|
if _, err := os.Stat(filepath.Join(wd, "backend", "ml", "predict_detector_model.py")); err == nil {
|
|
return wd
|
|
}
|
|
|
|
if _, err := os.Stat(filepath.Join(wd, "ml", "predict_detector_model.py")); err == nil {
|
|
return filepath.Dir(wd)
|
|
}
|
|
|
|
parent := filepath.Dir(wd)
|
|
if _, err := os.Stat(filepath.Join(parent, "backend", "ml", "predict_detector_model.py")); err == nil {
|
|
return parent
|
|
}
|
|
|
|
return wd
|
|
}
|
|
|
|
func trainingScriptPath(name string) string {
|
|
// 1) Eingebettete Scripts bevorzugen.
|
|
if dir, err := trainingEmbeddedMLDir(); err == nil {
|
|
p := filepath.Join(dir, name)
|
|
if _, err := os.Stat(p); err == nil {
|
|
return p
|
|
}
|
|
}
|
|
|
|
// 2) App-/Backend-relativ wie record_paths.go.
|
|
if backendRoot, err := trainingBackendRootDir(); err == nil {
|
|
p := filepath.Join(backendRoot, "ml", name)
|
|
if _, err := os.Stat(p); err == nil {
|
|
return p
|
|
}
|
|
}
|
|
|
|
// 3) Dev-Fallback.
|
|
root := trainingProjectRoot()
|
|
|
|
p := filepath.Join(root, "backend", "ml", name)
|
|
if _, err := os.Stat(p); err == nil {
|
|
return p
|
|
}
|
|
|
|
p = filepath.Join("ml", name)
|
|
if _, err := os.Stat(p); err == nil {
|
|
return p
|
|
}
|
|
|
|
return filepath.Join(root, "backend", "ml", name)
|
|
}
|
|
|
|
func isTempBuildDir(dir string) bool {
|
|
low := strings.ToLower(filepath.Clean(dir))
|
|
|
|
return strings.Contains(low, `\appdata\local\temp`) ||
|
|
strings.Contains(low, `\temp\`) ||
|
|
strings.Contains(low, `\tmp\`) ||
|
|
strings.Contains(low, `\go-build`) ||
|
|
strings.Contains(low, `/tmp/`) ||
|
|
strings.Contains(low, `/go-build`)
|
|
}
|
|
|
|
func trainingBackendRootDir() (string, error) {
|
|
if script, err := resolvePathRelativeToApp(filepath.Join("ml", "predict_detector_model.py")); err == nil {
|
|
if st, statErr := os.Stat(script); statErr == nil && !st.IsDir() {
|
|
return filepath.Dir(filepath.Dir(script)), nil
|
|
}
|
|
}
|
|
|
|
if script, err := resolvePathRelativeToApp(filepath.Join("backend", "ml", "predict_detector_model.py")); err == nil {
|
|
if st, statErr := os.Stat(script); statErr == nil && !st.IsDir() {
|
|
return filepath.Dir(filepath.Dir(script)), nil
|
|
}
|
|
}
|
|
|
|
if dir, err := exeDir(); err == nil && strings.TrimSpace(dir) != "" && !isTempBuildDir(dir) {
|
|
return dir, nil
|
|
}
|
|
|
|
wd, err := os.Getwd()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if _, err := os.Stat(filepath.Join(wd, "ml", "predict_detector_model.py")); err == nil {
|
|
return wd, nil
|
|
}
|
|
|
|
if _, err := os.Stat(filepath.Join(wd, "backend", "ml", "predict_detector_model.py")); err == nil {
|
|
return filepath.Join(wd, "backend"), nil
|
|
}
|
|
|
|
projectRoot := trainingProjectRoot()
|
|
return filepath.Join(projectRoot, "backend"), nil
|
|
}
|
|
|
|
func trainingRootDir() (string, error) {
|
|
// Optionaler Override, falls du später explizit einen anderen Speicherort willst.
|
|
// Relative Pfade werden wie in record_paths.go app-relativ aufgelöst.
|
|
if override := strings.TrimSpace(os.Getenv("TRAINING_ROOT")); override != "" {
|
|
root, err := resolvePathRelativeToApp(override)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
root, err = filepath.Abs(root)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if err := os.MkdirAll(root, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return root, nil
|
|
}
|
|
|
|
backendRoot, err := trainingBackendRootDir()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
root, err := filepath.Abs(filepath.Join(backendRoot, "generated", "training"))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if err := os.MkdirAll(root, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return root, nil
|
|
}
|
|
|
|
func trainingWriteSample(root string, sample *TrainingSample) error {
|
|
path := filepath.Join(root, "samples", sample.SampleID+".json")
|
|
b, err := json.MarshalIndent(sample, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, b, 0644)
|
|
}
|
|
|
|
func trainingReadSample(root string, sampleID string) (*TrainingSample, error) {
|
|
if strings.Contains(sampleID, "/") || strings.Contains(sampleID, "\\") {
|
|
return nil, errors.New("invalid sample id")
|
|
}
|
|
|
|
path := filepath.Join(root, "samples", sampleID+".json")
|
|
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var sample TrainingSample
|
|
if err := json.Unmarshal(b, &sample); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &sample, nil
|
|
}
|
|
|
|
func trainingAppendAnnotation(root string, annotation TrainingAnnotation) error {
|
|
path := filepath.Join(root, "feedback.jsonl")
|
|
|
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
b, err := json.Marshal(annotation)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := f.Write(append(b, '\n')); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func trainingWriteAnnotations(root string, items []TrainingAnnotation) error {
|
|
path := filepath.Join(root, "feedback.jsonl")
|
|
tmpPath := path + ".tmp"
|
|
|
|
var b strings.Builder
|
|
|
|
for _, item := range items {
|
|
line, err := json.Marshal(item)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
b.Write(line)
|
|
b.WriteByte('\n')
|
|
}
|
|
|
|
if err := os.WriteFile(tmpPath, []byte(b.String()), 0644); err != nil {
|
|
return err
|
|
}
|
|
|
|
return os.Rename(tmpPath, path)
|
|
}
|
|
|
|
func trainingCountAnnotations(path string) (int, error) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
text := strings.TrimSpace(string(b))
|
|
if text == "" {
|
|
return 0, nil
|
|
}
|
|
|
|
return len(strings.Split(text, "\n")), nil
|
|
}
|
|
|
|
func trainingProbeDurationSeconds(videoPath string) float64 {
|
|
settings := getSettings()
|
|
|
|
ffmpeg := strings.TrimSpace(settings.FFmpegPath)
|
|
ffprobe := "ffprobe"
|
|
|
|
if ffmpeg != "" {
|
|
dir := filepath.Dir(ffmpeg)
|
|
base := filepath.Base(ffmpeg)
|
|
if strings.Contains(strings.ToLower(base), "ffmpeg") {
|
|
ffprobeBase := strings.Replace(base, "ffmpeg", "ffprobe", 1)
|
|
ffprobe = filepath.Join(dir, ffprobeBase)
|
|
}
|
|
}
|
|
|
|
cmd := exec.Command(
|
|
ffprobe,
|
|
"-v", "error",
|
|
"-show_entries", "format=duration",
|
|
"-of", "default=noprint_wrappers=1:nokey=1",
|
|
videoPath,
|
|
)
|
|
|
|
hideCommandWindow(cmd)
|
|
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
v, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
|
|
if err != nil || math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 {
|
|
return 0
|
|
}
|
|
|
|
return v
|
|
}
|
|
|
|
func trainingRandomSecond(duration float64) float64 {
|
|
if duration <= 2 {
|
|
return 0
|
|
}
|
|
|
|
minSec := 1.0
|
|
maxSec := math.Max(minSec, duration-1)
|
|
|
|
return minSec + rand.Float64()*(maxSec-minSec)
|
|
}
|
|
|
|
func trainingMakeSampleID(videoPath string, second float64) string {
|
|
h := sha1.New()
|
|
_, _ = h.Write([]byte(videoPath))
|
|
_, _ = h.Write([]byte("|"))
|
|
_, _ = h.Write([]byte(strconv.FormatFloat(second, 'f', 3, 64)))
|
|
_, _ = h.Write([]byte("|"))
|
|
_, _ = h.Write([]byte(strconv.FormatInt(time.Now().UnixNano(), 10)))
|
|
return hex.EncodeToString(h.Sum(nil))[:20]
|
|
}
|
|
|
|
func trainingWriteJSON(w http.ResponseWriter, status int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
|
|
func trainingWriteError(w http.ResponseWriter, status int, msg string) {
|
|
trainingWriteJSON(w, status, map[string]any{
|
|
"ok": false,
|
|
"error": msg,
|
|
})
|
|
}
|