// backend\postwork.go package main import ( "context" "os" "strings" "sync" "time" ) // Eine Nacharbeit (kann ffmpeg, ffprobe, thumbnails, rename, etc. enthalten) type PostWorkTask struct { Key string // z.B. Dateiname oder Job-ID, zum Deduplizieren Path string // Datei, die während queued/running gegen Explorer-Löschen gelockt wird Run func(ctx context.Context) error Added time.Time SortBucket int SortName string } type PostWorkQueue struct { ffmpegSem chan struct{} // "heavy" gate: maxParallelFFmpeg gleichzeitig maxQueued int mu sync.Mutex cond *sync.Cond queue []PostWorkTask inflight map[string]struct{} // dedupe: queued ODER running queued int // Anzahl inflight (queued + running) // ✅ für UI: Warteschlange + Running-Keys tracken waitingKeys []string // sortierte wartende Keys runningKeys map[string]struct{} // Keys, die gerade wirklich laufen (Semaphor gehalten) cancelByKey map[string]context.CancelFunc fileLocks map[string]*os.File } func NewPostWorkQueue(queueSize int, maxParallelFFmpeg int) *PostWorkQueue { if queueSize <= 0 { queueSize = 256 } if maxParallelFFmpeg <= 0 { maxParallelFFmpeg = 1 // Default: "nacheinander" } pq := &PostWorkQueue{ ffmpegSem: make(chan struct{}, maxParallelFFmpeg), maxQueued: queueSize, queue: make([]PostWorkTask, 0, queueSize), inflight: make(map[string]struct{}), waitingKeys: make([]string, 0, queueSize), runningKeys: make(map[string]struct{}), cancelByKey: make(map[string]context.CancelFunc), fileLocks: make(map[string]*os.File), } pq.cond = sync.NewCond(&pq.mu) return pq } func lessPostWorkTask(a, b PostWorkTask) bool { aHasSort := strings.TrimSpace(a.SortName) != "" bHasSort := strings.TrimSpace(b.SortName) != "" if aHasSort || bHasSort { ab := a.SortBucket bb := b.SortBucket if !aHasSort { ab = 999 } if !bHasSort { bb = 999 } if ab != bb { return ab < bb } an := strings.ToLower(strings.TrimSpace(a.SortName)) bn := strings.ToLower(strings.TrimSpace(b.SortName)) if an != bn { return an < bn } } if !a.Added.Equal(b.Added) { return a.Added.Before(b.Added) } return a.Key < b.Key } func containsWaitingKey(keys []string, key string) bool { for _, k := range keys { if k == key { return true } } return false } func removeQueuedTaskLocked(tasks []PostWorkTask, key string) []PostWorkTask { for i, t := range tasks { if t.Key == key { return append(tasks[:i], tasks[i+1:]...) } } return tasks } func postWorkTaskLockPath(task PostWorkTask) string { return strings.TrimSpace(task.Path) } func (pq *PostWorkQueue) unlockFileLocked(key string) { f := pq.fileLocks[key] if f == nil { return } _ = f.Close() delete(pq.fileLocks, key) } // Enqueue dedupliziert nach Key (damit du nicht durch Events doppelt queue-st) func (pq *PostWorkQueue) Enqueue(task PostWorkTask) bool { if task.Key == "" || task.Run == nil { return false } if task.Added.IsZero() { task.Added = time.Now() } pq.mu.Lock() defer pq.mu.Unlock() if _, ok := pq.inflight[task.Key]; ok { return false } if pq.maxQueued > 0 && len(pq.queue) >= pq.maxQueued { return false } lockPath := postWorkTaskLockPath(task) var lockFile *os.File if lockPath != "" { f, err := lockPostWorkFile(lockPath) if err != nil { return false } lockFile = f } pq.inflight[task.Key] = struct{}{} pq.queued++ if lockFile != nil { pq.fileLocks[task.Key] = lockFile } insertAt := len(pq.queue) for i, existing := range pq.queue { if lessPostWorkTask(task, existing) { insertAt = i break } } pq.queue = append(pq.queue, PostWorkTask{}) copy(pq.queue[insertAt+1:], pq.queue[insertAt:]) pq.queue[insertAt] = task pq.waitingKeys = append(pq.waitingKeys, "") copy(pq.waitingKeys[insertAt+1:], pq.waitingKeys[insertAt:]) pq.waitingKeys[insertAt] = task.Key pq.cond.Signal() return true } func (pq *PostWorkQueue) Cancel(key string) bool { if strings.TrimSpace(key) == "" { return false } pq.mu.Lock() cancel := pq.cancelByKey[key] pq.mu.Unlock() if cancel != nil { cancel() return true } return false } func (pq *PostWorkQueue) removeWaitingKeyLocked(key string) { for i, k := range pq.waitingKeys { if k == key { pq.waitingKeys = append(pq.waitingKeys[:i], pq.waitingKeys[i+1:]...) return } } } func (pq *PostWorkQueue) RemoveQueued(key string) bool { if strings.TrimSpace(key) == "" { return false } pq.mu.Lock() defer pq.mu.Unlock() // running darf hier NICHT entfernt werden if _, running := pq.runningKeys[key]; running { return false } found := false for i, k := range pq.waitingKeys { if k == key { pq.waitingKeys = append(pq.waitingKeys[:i], pq.waitingKeys[i+1:]...) found = true break } } if !found { return false } delete(pq.inflight, key) pq.unlockFileLocked(key) if pq.queued > 0 { pq.queued-- } pq.queue = removeQueuedTaskLocked(pq.queue, key) // Das Task-Element bleibt evtl. noch im Channel liegen. // Beim Worker-Start muss es dann erkannt und übersprungen werden. return true } func (pq *PostWorkQueue) CancelAll() (queuedRemoved int, runningCancelled int) { pq.mu.Lock() waiting := append([]string(nil), pq.waitingKeys...) runningCancels := make([]context.CancelFunc, 0, len(pq.cancelByKey)) for key, cancel := range pq.cancelByKey { if _, running := pq.runningKeys[key]; running && cancel != nil { runningCancels = append(runningCancels, cancel) } } for _, key := range waiting { delete(pq.inflight, key) pq.unlockFileLocked(key) if pq.queued > 0 { pq.queued-- } queuedRemoved++ } pq.waitingKeys = nil pq.queue = nil runningCancelled = len(runningCancels) pq.mu.Unlock() for _, cancel := range runningCancels { cancel() } publishTaskState() return queuedRemoved, runningCancelled } func (pq *PostWorkQueue) workerLoop(id int) { for { pq.mu.Lock() for len(pq.queue) == 0 { pq.cond.Wait() } task := pq.queue[0] pq.queue = pq.queue[1:] pq.mu.Unlock() pq.mu.Lock() _, stillInflight := pq.inflight[task.Key] _, alreadyRunning := pq.runningKeys[task.Key] waitingFound := containsWaitingKey(pq.waitingKeys, task.Key) pq.mu.Unlock() // queued Job wurde zwischenzeitlich entfernt -> verwerfen if !stillInflight || (!waitingFound && !alreadyRunning) { continue } func() { defer func() { if r := recover(); r != nil { _ = r } pq.mu.Lock() pq.removeWaitingKeyLocked(task.Key) delete(pq.runningKeys, task.Key) delete(pq.inflight, task.Key) delete(pq.cancelByKey, task.Key) pq.unlockFileLocked(task.Key) if pq.queued > 0 { pq.queued-- } pq.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) pq.mu.Lock() pq.cancelByKey[task.Key] = cancel pq.mu.Unlock() defer cancel() select { case pq.ffmpegSem <- struct{}{}: case <-ctx.Done(): return } defer func() { <-pq.ffmpegSem }() pq.mu.Lock() pq.removeWaitingKeyLocked(task.Key) pq.runningKeys[task.Key] = struct{}{} // Wichtig: Lock vor der eigentlichen Nacharbeit freigeben, // damit moveToDoneDir/removeWithRetry unter Windows nicht blockiert werden. pq.unlockFileLocked(task.Key) pq.mu.Unlock() if task.Run != nil { _ = task.Run(ctx) } }() } } func (pq *PostWorkQueue) StartWorkers(n int) { if n <= 0 { n = 1 } for i := 0; i < n; i++ { go pq.workerLoop(i + 1) } } // Stats bleibt kompatibel zu deinem bisherigen Code func (pq *PostWorkQueue) Stats() (queued int, inflight int, maxParallel int) { pq.mu.Lock() defer pq.mu.Unlock() return len(pq.waitingKeys), len(pq.runningKeys), cap(pq.ffmpegSem) } // Optional für UI/Debug (praktisch für "Warte… Position X/Y") type PostWorkKeyStatus struct { State string `json:"state"` // "queued" | "running" | "missing" Position int `json:"position"` // 1..n (nur queued), 0 sonst Waiting int `json:"waiting"` // Anzahl wartend Running int `json:"running"` // Anzahl running MaxParallel int `json:"maxParallel"` // cap(ffmpegSem) } func (pq *PostWorkQueue) StatusForKey(key string) PostWorkKeyStatus { pq.mu.Lock() defer pq.mu.Unlock() waiting := len(pq.waitingKeys) running := len(pq.runningKeys) maxPar := cap(pq.ffmpegSem) if _, ok := pq.runningKeys[key]; ok { return PostWorkKeyStatus{ State: "running", Position: 0, Waiting: waiting, Running: running, MaxParallel: maxPar, } } for i, k := range pq.waitingKeys { if k == key { return PostWorkKeyStatus{ State: "queued", Position: i + 1, // 1-basiert Waiting: waiting, Running: running, MaxParallel: maxPar, } } } // Key ist weder queued noch running (wahrscheinlich schon fertig oder nie queued) return PostWorkKeyStatus{ State: "missing", Position: 0, Waiting: waiting, Running: running, MaxParallel: maxPar, } } // global (oder in deinem app struct halten) var postWorkQ = NewPostWorkQueue(1024, 1) // schneller kritischer Pfad var enrichQ = NewPostWorkQueue(128, 1) // langsame Hintergrundaufgaben func enrichPostworkEnabled() bool { return getSettings().EnrichPostworkEnabled } func enqueueEnrichPostwork(task PostWorkTask) bool { if !enrichPostworkEnabled() { appLogln("⏭️ enrich postwork deaktiviert, überspringe:", task.Key) // Wichtig: // Deaktiviert heißt "bewusst übersprungen", nicht "enqueue fehlgeschlagen". // Caller, die false als Fehler behandeln, sollen hier keinen Fehler erzeugen. return true } return enrichQ.Enqueue(task) } // --- Status Refresher (ehemals postwork_refresh.go) --- type enrichStatusSnapshot struct { State string Position int Waiting int Running int MaxParallel int } var enrichStatusLastMu sync.Mutex var enrichStatusLast = map[string]enrichStatusSnapshot{} func shouldPublishEnrichStatus(key string, st PostWorkKeyStatus) bool { enrichStatusLastMu.Lock() defer enrichStatusLastMu.Unlock() cur := enrichStatusSnapshot{ State: st.State, Position: st.Position, Waiting: st.Waiting, Running: st.Running, MaxParallel: st.MaxParallel, } prev, ok := enrichStatusLast[key] if ok && prev == cur { return false } enrichStatusLast[key] = cur return true } func clearEnrichStatusSnapshot(key string) { enrichStatusLastMu.Lock() delete(enrichStatusLast, key) enrichStatusLastMu.Unlock() } func waitUntilEnrichMayRun(ctx context.Context) error { t := time.NewTicker(2 * time.Second) defer t.Stop() for { waiting, running, _ := postWorkQ.Stats() // enrich erst starten, wenn keine primäre Postwork mehr läuft if waiting == 0 && running == 0 { return nil } select { case <-ctx.Done(): return ctx.Err() case <-t.C: } } } func startEnrichStatusRefresher() { t := time.NewTicker(2 * time.Second) go func() { defer t.Stop() for range t.C { jobsMu.RLock() snapshot := make([]*RecordJob, 0, len(jobs)) for _, job := range jobs { if job == nil { continue } snapshot = append(snapshot, job) } jobsMu.RUnlock() for _, job := range snapshot { if job == nil { continue } videoPath := strings.TrimSpace(job.Output) if videoPath == "" { continue } id := assetIDFromVideoPath(videoPath) if id == "" { continue } key := "enrich:" + id st := enrichQ.StatusForKey(key) if st.State != "queued" && st.State != "running" { clearEnrichStatusSnapshot(key) continue } if !shouldPublishEnrichStatus(key, st) { continue } // Wichtig: // Hier NICHT mehr pauschal "analyze" publishen. // Die echten Phasen kommen direkt aus dem Enrich-Worker selbst: // meta/thumb/teaser (nur falls fehlend), danach sprites, danach analyze. } } }() } func startPostWorkStatusRefresher() { t := time.NewTicker(1 * time.Second) go func() { defer t.Stop() for range t.C { snapshots := make([]jobEventSnapshot, 0, 16) jobsMu.Lock() for _, job := range jobs { if job == nil { continue } key := strings.TrimSpace(job.PostWorkKey) if key == "" { continue } st := postWorkQ.StatusForKey(key) changed := false if job.PostWork == nil || job.PostWork.State != st.State || job.PostWork.Position != st.Position || job.PostWork.Waiting != st.Waiting || job.PostWork.Running != st.Running || job.PostWork.MaxParallel != st.MaxParallel { tmp := st job.PostWork = &tmp changed = true } switch st.State { case "queued": if job.Status != JobPostwork { job.Status = JobPostwork changed = true } phaseLower := strings.TrimSpace(strings.ToLower(job.Phase)) if phaseLower == "" || phaseLower == "recording" { job.Phase = "postwork" changed = true } if job.Progress < 0 || job.Progress > 100 { job.Progress = 0 changed = true } case "running": if job.Status != JobPostwork { job.Status = JobPostwork changed = true } phaseLower := strings.TrimSpace(strings.ToLower(job.Phase)) switch phaseLower { case "probe", "remuxing", "moving", "assets", "analyze": // konkrete Unterphasen beibehalten default: if phaseLower == "" || phaseLower == "recording" || phaseLower == "postwork" { if phaseLower != "postwork" { job.Phase = "postwork" changed = true } } } case "missing": continue } if !changed { continue } var pw *PostWorkKeyStatus if job.PostWork != nil { tmp := *job.PostWork pw = &tmp } var endedAtCopy *time.Time if job.EndedAt != nil { tm := *job.EndedAt endedAtCopy = &tm } snapshots = append(snapshots, jobEventSnapshot{ ID: job.ID, SourceURL: job.SourceURL, Output: job.Output, Status: job.Status, StartedAt: job.StartedAt, StartedAtMs: job.StartedAtMs, EndedAt: endedAtCopy, EndedAtMs: job.EndedAtMs, Error: job.Error, Phase: job.Phase, Progress: job.Progress, SizeBytes: job.SizeBytes, DurationSeconds: job.DurationSeconds, PreviewState: job.PreviewState, PostWorkKey: job.PostWorkKey, PostWork: pw, }) } jobsMu.Unlock() for _, snap := range snapshots { publishJobUpsertSnapshot(snap) } } }() } func (pq *PostWorkQueue) CancelRunning() int { pq.mu.Lock() cancels := make([]context.CancelFunc, 0, len(pq.cancelByKey)) for key, cancel := range pq.cancelByKey { if cancel == nil { continue } if _, running := pq.runningKeys[key]; running { cancels = append(cancels, cancel) } } pq.mu.Unlock() for _, cancel := range cancels { cancel() } return len(cancels) }