707 lines
15 KiB
Go
707 lines
15 KiB
Go
// backend\postwork.go
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// Eine Nacharbeit (kann ffmpeg, ffprobe, thumbnails, rename, etc. enthalten)
|
|
type PostWorkTask struct {
|
|
Key string // z.B. Dateiname oder Job-ID, zum Deduplizieren
|
|
Path string // Datei, die während queued/running gegen Explorer-Löschen gelockt wird
|
|
Run func(ctx context.Context) error
|
|
Added time.Time
|
|
SortBucket int
|
|
SortName string
|
|
}
|
|
|
|
type PostWorkQueue struct {
|
|
ffmpegSem chan struct{} // "heavy" gate: maxParallelFFmpeg gleichzeitig
|
|
maxQueued int
|
|
|
|
mu sync.Mutex
|
|
cond *sync.Cond
|
|
|
|
queue []PostWorkTask
|
|
inflight map[string]struct{} // dedupe: queued ODER running
|
|
|
|
queued int // Anzahl inflight (queued + running)
|
|
|
|
// ✅ für UI: Warteschlange + Running-Keys tracken
|
|
waitingKeys []string // sortierte wartende Keys
|
|
runningKeys map[string]struct{} // Keys, die gerade wirklich laufen (Semaphor gehalten)
|
|
cancelByKey map[string]context.CancelFunc
|
|
|
|
fileLocks map[string]*os.File
|
|
}
|
|
|
|
func NewPostWorkQueue(queueSize int, maxParallelFFmpeg int) *PostWorkQueue {
|
|
if queueSize <= 0 {
|
|
queueSize = 256
|
|
}
|
|
if maxParallelFFmpeg <= 0 {
|
|
maxParallelFFmpeg = 1 // Default: "nacheinander"
|
|
}
|
|
|
|
pq := &PostWorkQueue{
|
|
ffmpegSem: make(chan struct{}, maxParallelFFmpeg),
|
|
maxQueued: queueSize,
|
|
|
|
queue: make([]PostWorkTask, 0, queueSize),
|
|
inflight: make(map[string]struct{}),
|
|
|
|
waitingKeys: make([]string, 0, queueSize),
|
|
runningKeys: make(map[string]struct{}),
|
|
cancelByKey: make(map[string]context.CancelFunc),
|
|
|
|
fileLocks: make(map[string]*os.File),
|
|
}
|
|
|
|
pq.cond = sync.NewCond(&pq.mu)
|
|
return pq
|
|
}
|
|
|
|
func lessPostWorkTask(a, b PostWorkTask) bool {
|
|
aHasSort := strings.TrimSpace(a.SortName) != ""
|
|
bHasSort := strings.TrimSpace(b.SortName) != ""
|
|
|
|
if aHasSort || bHasSort {
|
|
ab := a.SortBucket
|
|
bb := b.SortBucket
|
|
|
|
if !aHasSort {
|
|
ab = 999
|
|
}
|
|
if !bHasSort {
|
|
bb = 999
|
|
}
|
|
|
|
if ab != bb {
|
|
return ab < bb
|
|
}
|
|
|
|
an := strings.ToLower(strings.TrimSpace(a.SortName))
|
|
bn := strings.ToLower(strings.TrimSpace(b.SortName))
|
|
if an != bn {
|
|
return an < bn
|
|
}
|
|
}
|
|
|
|
if !a.Added.Equal(b.Added) {
|
|
return a.Added.Before(b.Added)
|
|
}
|
|
|
|
return a.Key < b.Key
|
|
}
|
|
|
|
func containsWaitingKey(keys []string, key string) bool {
|
|
for _, k := range keys {
|
|
if k == key {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func removeQueuedTaskLocked(tasks []PostWorkTask, key string) []PostWorkTask {
|
|
for i, t := range tasks {
|
|
if t.Key == key {
|
|
return append(tasks[:i], tasks[i+1:]...)
|
|
}
|
|
}
|
|
return tasks
|
|
}
|
|
|
|
func postWorkTaskLockPath(task PostWorkTask) string {
|
|
return strings.TrimSpace(task.Path)
|
|
}
|
|
|
|
func (pq *PostWorkQueue) unlockFileLocked(key string) {
|
|
f := pq.fileLocks[key]
|
|
if f == nil {
|
|
return
|
|
}
|
|
|
|
_ = f.Close()
|
|
delete(pq.fileLocks, key)
|
|
}
|
|
|
|
// Enqueue dedupliziert nach Key (damit du nicht durch Events doppelt queue-st)
|
|
func (pq *PostWorkQueue) Enqueue(task PostWorkTask) bool {
|
|
if task.Key == "" || task.Run == nil {
|
|
return false
|
|
}
|
|
if task.Added.IsZero() {
|
|
task.Added = time.Now()
|
|
}
|
|
|
|
pq.mu.Lock()
|
|
defer pq.mu.Unlock()
|
|
|
|
if _, ok := pq.inflight[task.Key]; ok {
|
|
return false
|
|
}
|
|
if pq.maxQueued > 0 && len(pq.queue) >= pq.maxQueued {
|
|
return false
|
|
}
|
|
|
|
lockPath := postWorkTaskLockPath(task)
|
|
var lockFile *os.File
|
|
|
|
if lockPath != "" {
|
|
f, err := lockPostWorkFile(lockPath)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
lockFile = f
|
|
}
|
|
|
|
pq.inflight[task.Key] = struct{}{}
|
|
pq.queued++
|
|
|
|
if lockFile != nil {
|
|
pq.fileLocks[task.Key] = lockFile
|
|
}
|
|
|
|
insertAt := len(pq.queue)
|
|
for i, existing := range pq.queue {
|
|
if lessPostWorkTask(task, existing) {
|
|
insertAt = i
|
|
break
|
|
}
|
|
}
|
|
|
|
pq.queue = append(pq.queue, PostWorkTask{})
|
|
copy(pq.queue[insertAt+1:], pq.queue[insertAt:])
|
|
pq.queue[insertAt] = task
|
|
|
|
pq.waitingKeys = append(pq.waitingKeys, "")
|
|
copy(pq.waitingKeys[insertAt+1:], pq.waitingKeys[insertAt:])
|
|
pq.waitingKeys[insertAt] = task.Key
|
|
|
|
pq.cond.Signal()
|
|
return true
|
|
}
|
|
|
|
func (pq *PostWorkQueue) Cancel(key string) bool {
|
|
if strings.TrimSpace(key) == "" {
|
|
return false
|
|
}
|
|
|
|
pq.mu.Lock()
|
|
cancel := pq.cancelByKey[key]
|
|
pq.mu.Unlock()
|
|
|
|
if cancel != nil {
|
|
cancel()
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func (pq *PostWorkQueue) removeWaitingKeyLocked(key string) {
|
|
for i, k := range pq.waitingKeys {
|
|
if k == key {
|
|
pq.waitingKeys = append(pq.waitingKeys[:i], pq.waitingKeys[i+1:]...)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (pq *PostWorkQueue) RemoveQueued(key string) bool {
|
|
if strings.TrimSpace(key) == "" {
|
|
return false
|
|
}
|
|
|
|
pq.mu.Lock()
|
|
defer pq.mu.Unlock()
|
|
|
|
// running darf hier NICHT entfernt werden
|
|
if _, running := pq.runningKeys[key]; running {
|
|
return false
|
|
}
|
|
|
|
found := false
|
|
for i, k := range pq.waitingKeys {
|
|
if k == key {
|
|
pq.waitingKeys = append(pq.waitingKeys[:i], pq.waitingKeys[i+1:]...)
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return false
|
|
}
|
|
|
|
delete(pq.inflight, key)
|
|
pq.unlockFileLocked(key)
|
|
|
|
if pq.queued > 0 {
|
|
pq.queued--
|
|
}
|
|
|
|
pq.queue = removeQueuedTaskLocked(pq.queue, key)
|
|
|
|
// Das Task-Element bleibt evtl. noch im Channel liegen.
|
|
// Beim Worker-Start muss es dann erkannt und übersprungen werden.
|
|
return true
|
|
}
|
|
|
|
func (pq *PostWorkQueue) CancelAll() (queuedRemoved int, runningCancelled int) {
|
|
pq.mu.Lock()
|
|
|
|
waiting := append([]string(nil), pq.waitingKeys...)
|
|
runningCancels := make([]context.CancelFunc, 0, len(pq.cancelByKey))
|
|
|
|
for key, cancel := range pq.cancelByKey {
|
|
if _, running := pq.runningKeys[key]; running && cancel != nil {
|
|
runningCancels = append(runningCancels, cancel)
|
|
}
|
|
}
|
|
|
|
for _, key := range waiting {
|
|
delete(pq.inflight, key)
|
|
pq.unlockFileLocked(key)
|
|
|
|
if pq.queued > 0 {
|
|
pq.queued--
|
|
}
|
|
|
|
queuedRemoved++
|
|
}
|
|
|
|
pq.waitingKeys = nil
|
|
pq.queue = nil
|
|
|
|
runningCancelled = len(runningCancels)
|
|
|
|
pq.mu.Unlock()
|
|
|
|
for _, cancel := range runningCancels {
|
|
cancel()
|
|
}
|
|
|
|
publishTaskState()
|
|
|
|
return queuedRemoved, runningCancelled
|
|
}
|
|
|
|
func (pq *PostWorkQueue) workerLoop(id int) {
|
|
for {
|
|
pq.mu.Lock()
|
|
for len(pq.queue) == 0 {
|
|
pq.cond.Wait()
|
|
}
|
|
task := pq.queue[0]
|
|
pq.queue = pq.queue[1:]
|
|
pq.mu.Unlock()
|
|
|
|
pq.mu.Lock()
|
|
_, stillInflight := pq.inflight[task.Key]
|
|
_, alreadyRunning := pq.runningKeys[task.Key]
|
|
waitingFound := containsWaitingKey(pq.waitingKeys, task.Key)
|
|
pq.mu.Unlock()
|
|
|
|
// queued Job wurde zwischenzeitlich entfernt -> verwerfen
|
|
if !stillInflight || (!waitingFound && !alreadyRunning) {
|
|
continue
|
|
}
|
|
|
|
func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
_ = r
|
|
}
|
|
|
|
pq.mu.Lock()
|
|
pq.removeWaitingKeyLocked(task.Key)
|
|
delete(pq.runningKeys, task.Key)
|
|
delete(pq.inflight, task.Key)
|
|
delete(pq.cancelByKey, task.Key)
|
|
pq.unlockFileLocked(task.Key)
|
|
if pq.queued > 0 {
|
|
pq.queued--
|
|
}
|
|
pq.mu.Unlock()
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
|
|
|
|
pq.mu.Lock()
|
|
pq.cancelByKey[task.Key] = cancel
|
|
pq.mu.Unlock()
|
|
|
|
defer cancel()
|
|
|
|
select {
|
|
case pq.ffmpegSem <- struct{}{}:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
defer func() {
|
|
<-pq.ffmpegSem
|
|
}()
|
|
|
|
pq.mu.Lock()
|
|
pq.removeWaitingKeyLocked(task.Key)
|
|
pq.runningKeys[task.Key] = struct{}{}
|
|
|
|
// Wichtig: Lock vor der eigentlichen Nacharbeit freigeben,
|
|
// damit moveToDoneDir/removeWithRetry unter Windows nicht blockiert werden.
|
|
pq.unlockFileLocked(task.Key)
|
|
pq.mu.Unlock()
|
|
|
|
if task.Run != nil {
|
|
_ = task.Run(ctx)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
func (pq *PostWorkQueue) StartWorkers(n int) {
|
|
if n <= 0 {
|
|
n = 1
|
|
}
|
|
for i := 0; i < n; i++ {
|
|
go pq.workerLoop(i + 1)
|
|
}
|
|
}
|
|
|
|
// Stats bleibt kompatibel zu deinem bisherigen Code
|
|
func (pq *PostWorkQueue) Stats() (queued int, inflight int, maxParallel int) {
|
|
pq.mu.Lock()
|
|
defer pq.mu.Unlock()
|
|
return len(pq.waitingKeys), len(pq.runningKeys), cap(pq.ffmpegSem)
|
|
}
|
|
|
|
// Optional für UI/Debug (praktisch für "Warte… Position X/Y")
|
|
type PostWorkKeyStatus struct {
|
|
State string `json:"state"` // "queued" | "running" | "missing"
|
|
Position int `json:"position"` // 1..n (nur queued), 0 sonst
|
|
Waiting int `json:"waiting"` // Anzahl wartend
|
|
Running int `json:"running"` // Anzahl running
|
|
MaxParallel int `json:"maxParallel"` // cap(ffmpegSem)
|
|
}
|
|
|
|
func (pq *PostWorkQueue) StatusForKey(key string) PostWorkKeyStatus {
|
|
pq.mu.Lock()
|
|
defer pq.mu.Unlock()
|
|
|
|
waiting := len(pq.waitingKeys)
|
|
running := len(pq.runningKeys)
|
|
maxPar := cap(pq.ffmpegSem)
|
|
|
|
if _, ok := pq.runningKeys[key]; ok {
|
|
return PostWorkKeyStatus{
|
|
State: "running",
|
|
Position: 0,
|
|
Waiting: waiting,
|
|
Running: running,
|
|
MaxParallel: maxPar,
|
|
}
|
|
}
|
|
|
|
for i, k := range pq.waitingKeys {
|
|
if k == key {
|
|
return PostWorkKeyStatus{
|
|
State: "queued",
|
|
Position: i + 1, // 1-basiert
|
|
Waiting: waiting,
|
|
Running: running,
|
|
MaxParallel: maxPar,
|
|
}
|
|
}
|
|
}
|
|
|
|
// Key ist weder queued noch running (wahrscheinlich schon fertig oder nie queued)
|
|
return PostWorkKeyStatus{
|
|
State: "missing",
|
|
Position: 0,
|
|
Waiting: waiting,
|
|
Running: running,
|
|
MaxParallel: maxPar,
|
|
}
|
|
}
|
|
|
|
// global (oder in deinem app struct halten)
|
|
var postWorkQ = NewPostWorkQueue(1024, 1) // schneller kritischer Pfad
|
|
var enrichQ = NewPostWorkQueue(128, 1) // langsame Hintergrundaufgaben
|
|
|
|
func enrichPostworkEnabled() bool {
|
|
return getSettings().EnrichPostworkEnabled
|
|
}
|
|
|
|
func enqueueEnrichPostwork(task PostWorkTask) bool {
|
|
if !enrichPostworkEnabled() {
|
|
appLogln("⏭️ enrich postwork deaktiviert, überspringe:", task.Key)
|
|
|
|
// Wichtig:
|
|
// Deaktiviert heißt "bewusst übersprungen", nicht "enqueue fehlgeschlagen".
|
|
// Caller, die false als Fehler behandeln, sollen hier keinen Fehler erzeugen.
|
|
return true
|
|
}
|
|
|
|
return enrichQ.Enqueue(task)
|
|
}
|
|
|
|
// --- Status Refresher (ehemals postwork_refresh.go) ---
|
|
|
|
type enrichStatusSnapshot struct {
|
|
State string
|
|
Position int
|
|
Waiting int
|
|
Running int
|
|
MaxParallel int
|
|
}
|
|
|
|
var enrichStatusLastMu sync.Mutex
|
|
var enrichStatusLast = map[string]enrichStatusSnapshot{}
|
|
|
|
func shouldPublishEnrichStatus(key string, st PostWorkKeyStatus) bool {
|
|
enrichStatusLastMu.Lock()
|
|
defer enrichStatusLastMu.Unlock()
|
|
|
|
cur := enrichStatusSnapshot{
|
|
State: st.State,
|
|
Position: st.Position,
|
|
Waiting: st.Waiting,
|
|
Running: st.Running,
|
|
MaxParallel: st.MaxParallel,
|
|
}
|
|
|
|
prev, ok := enrichStatusLast[key]
|
|
if ok && prev == cur {
|
|
return false
|
|
}
|
|
|
|
enrichStatusLast[key] = cur
|
|
return true
|
|
}
|
|
|
|
func clearEnrichStatusSnapshot(key string) {
|
|
enrichStatusLastMu.Lock()
|
|
delete(enrichStatusLast, key)
|
|
enrichStatusLastMu.Unlock()
|
|
}
|
|
|
|
func waitUntilEnrichMayRun(ctx context.Context) error {
|
|
t := time.NewTicker(2 * time.Second)
|
|
defer t.Stop()
|
|
|
|
for {
|
|
waiting, running, _ := postWorkQ.Stats()
|
|
trainingRunning := trainingGetJobStatus().Running
|
|
// Auch waehrend eines Trainings pausieren, damit EnrichQ keine CPU stiehlt.
|
|
|
|
// enrich erst starten, wenn keine primäre Postwork mehr läuft
|
|
if waiting == 0 && running == 0 && !trainingRunning {
|
|
return nil
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-t.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func startEnrichStatusRefresher() {
|
|
t := time.NewTicker(2 * time.Second)
|
|
go func() {
|
|
defer t.Stop()
|
|
|
|
for range t.C {
|
|
jobsMu.RLock()
|
|
snapshot := make([]*RecordJob, 0, len(jobs))
|
|
for _, job := range jobs {
|
|
if job == nil {
|
|
continue
|
|
}
|
|
snapshot = append(snapshot, job)
|
|
}
|
|
jobsMu.RUnlock()
|
|
|
|
for _, job := range snapshot {
|
|
if job == nil {
|
|
continue
|
|
}
|
|
|
|
videoPath := strings.TrimSpace(job.Output)
|
|
if videoPath == "" {
|
|
continue
|
|
}
|
|
|
|
id := assetIDFromVideoPath(videoPath)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
|
|
key := "enrich:" + id
|
|
st := enrichQ.StatusForKey(key)
|
|
|
|
if st.State != "queued" && st.State != "running" {
|
|
clearEnrichStatusSnapshot(key)
|
|
continue
|
|
}
|
|
|
|
if !shouldPublishEnrichStatus(key, st) {
|
|
continue
|
|
}
|
|
|
|
// Wichtig:
|
|
// Hier NICHT mehr pauschal "analyze" publishen.
|
|
// Die echten Phasen kommen direkt aus dem Enrich-Worker selbst:
|
|
// meta/thumb/teaser (nur falls fehlend), danach sprites, danach analyze.
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func startPostWorkStatusRefresher() {
|
|
t := time.NewTicker(1 * time.Second)
|
|
go func() {
|
|
defer t.Stop()
|
|
|
|
for range t.C {
|
|
snapshots := make([]jobEventSnapshot, 0, 16)
|
|
|
|
jobsMu.Lock()
|
|
for _, job := range jobs {
|
|
if job == nil {
|
|
continue
|
|
}
|
|
|
|
key := strings.TrimSpace(job.PostWorkKey)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
|
|
st := postWorkQ.StatusForKey(key)
|
|
changed := false
|
|
|
|
if job.PostWork == nil ||
|
|
job.PostWork.State != st.State ||
|
|
job.PostWork.Position != st.Position ||
|
|
job.PostWork.Waiting != st.Waiting ||
|
|
job.PostWork.Running != st.Running ||
|
|
job.PostWork.MaxParallel != st.MaxParallel {
|
|
tmp := st
|
|
job.PostWork = &tmp
|
|
changed = true
|
|
}
|
|
|
|
switch st.State {
|
|
case "queued":
|
|
if job.Status != JobPostwork {
|
|
job.Status = JobPostwork
|
|
changed = true
|
|
}
|
|
|
|
phaseLower := strings.TrimSpace(strings.ToLower(job.Phase))
|
|
if phaseLower == "" || phaseLower == "recording" {
|
|
job.Phase = "postwork"
|
|
changed = true
|
|
}
|
|
|
|
if job.Progress < 0 || job.Progress > 100 {
|
|
job.Progress = 0
|
|
changed = true
|
|
}
|
|
|
|
case "running":
|
|
if job.Status != JobPostwork {
|
|
job.Status = JobPostwork
|
|
changed = true
|
|
}
|
|
|
|
phaseLower := strings.TrimSpace(strings.ToLower(job.Phase))
|
|
switch phaseLower {
|
|
case "probe", "remuxing", "moving", "assets", "analyze":
|
|
// konkrete Unterphasen beibehalten
|
|
default:
|
|
if phaseLower == "" || phaseLower == "recording" || phaseLower == "postwork" {
|
|
if phaseLower != "postwork" {
|
|
job.Phase = "postwork"
|
|
changed = true
|
|
}
|
|
}
|
|
}
|
|
|
|
case "missing":
|
|
continue
|
|
}
|
|
|
|
if !changed {
|
|
continue
|
|
}
|
|
|
|
var pw *PostWorkKeyStatus
|
|
if job.PostWork != nil {
|
|
tmp := *job.PostWork
|
|
pw = &tmp
|
|
}
|
|
|
|
var endedAtCopy *time.Time
|
|
if job.EndedAt != nil {
|
|
tm := *job.EndedAt
|
|
endedAtCopy = &tm
|
|
}
|
|
|
|
snapshots = append(snapshots, jobEventSnapshot{
|
|
ID: job.ID,
|
|
SourceURL: job.SourceURL,
|
|
Output: job.Output,
|
|
Status: job.Status,
|
|
StartedAt: job.StartedAt,
|
|
StartedAtMs: job.StartedAtMs,
|
|
EndedAt: endedAtCopy,
|
|
EndedAtMs: job.EndedAtMs,
|
|
Error: job.Error,
|
|
Phase: job.Phase,
|
|
Progress: job.Progress,
|
|
SizeBytes: job.SizeBytes,
|
|
DurationSeconds: job.DurationSeconds,
|
|
PreviewState: job.PreviewState,
|
|
PostWorkKey: job.PostWorkKey,
|
|
PostWork: pw,
|
|
})
|
|
}
|
|
jobsMu.Unlock()
|
|
|
|
for _, snap := range snapshots {
|
|
publishJobUpsertSnapshot(snap)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (pq *PostWorkQueue) CancelRunning() int {
|
|
pq.mu.Lock()
|
|
|
|
cancels := make([]context.CancelFunc, 0, len(pq.cancelByKey))
|
|
|
|
for key, cancel := range pq.cancelByKey {
|
|
if cancel == nil {
|
|
continue
|
|
}
|
|
|
|
if _, running := pq.runningKeys[key]; running {
|
|
cancels = append(cancels, cancel)
|
|
}
|
|
}
|
|
|
|
pq.mu.Unlock()
|
|
|
|
for _, cancel := range cancels {
|
|
cancel()
|
|
}
|
|
|
|
return len(cancels)
|
|
}
|