From 7a5b29c3e4938f4c791755ef53cdb6aa4505ae03 Mon Sep 17 00:00:00 2001 From: Linrador <68631622+Linrador@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:46:21 +0200 Subject: [PATCH] bugfixes --- backend/tasks_cleanup.go | 428 +++++++++++++++++++++++++-------------- 1 file changed, 280 insertions(+), 148 deletions(-) diff --git a/backend/tasks_cleanup.go b/backend/tasks_cleanup.go index 20f85cd..38da912 100644 --- a/backend/tasks_cleanup.go +++ b/backend/tasks_cleanup.go @@ -23,6 +23,19 @@ type cleanupCandidate struct { size int64 } +type recordCleanupFile struct { + path string + name string + lowName string + size int64 + modTime time.Time +} + +type recordCleanupDeleteCandidate struct { + file recordCleanupFile + reason string +} + type cleanupResp struct { // Small / broken downloads cleanup ScannedFiles int `json:"scannedFiles"` @@ -435,14 +448,39 @@ func cleanupVideoLooksBroken(ctx context.Context, path string) (bool, string) { return false, "" } -const cleanupRecordOrphanMinAge = 60 * time.Second +const ( + cleanupRecordOrphanMinAge = 60 * time.Second + cleanupRecordStableDelay = 1200 * time.Millisecond +) -func cleanupRecordFileIsActive(path string) bool { +func cleanupRecordPathKey(path string) string { path = filepath.Clean(strings.TrimSpace(path)) if path == "" { + return "" + } + return strings.ToLower(path) +} + +func cleanupRecordPathIsActive(path string, activePaths map[string]struct{}) bool { + key := cleanupRecordPathKey(path) + if key == "" || activePaths == nil { return false } + _, ok := activePaths[key] + return ok +} + +func snapshotCleanupRecordActivePaths() map[string]struct{} { + active := make(map[string]struct{}, 32) + + add := func(path string) { + key := cleanupRecordPathKey(path) + if key != "" { + active[key] = struct{}{} + } + } + jobsMu.RLock() defer jobsMu.RUnlock() @@ -452,132 +490,158 @@ func cleanupRecordFileIsActive(path string) bool { } out := filepath.Clean(strings.TrimSpace(j.Output)) - if out != "" && strings.EqualFold(out, path) { - return true - } + add(out) for _, sidecar := range postworkAudioSidecarCandidatesForOutput(out) { - if sidecar != "" && strings.EqualFold(filepath.Clean(sidecar), path) { - return true - } + add(sidecar) } } + return active +} + +func cleanupAudioSidecarNamesForTS(tsName string) []string { + tsName = strings.TrimSpace(filepath.Base(tsName)) + if tsName == "" || strings.HasPrefix(tsName, ".") || !strings.EqualFold(filepath.Ext(tsName), ".ts") { + return nil + } + + base := strings.TrimSuffix(tsName, filepath.Ext(tsName)) + if strings.TrimSpace(base) == "" { + return nil + } + + names := []string{ + base + ".audio.m4s", + "." + base + ".audio.m4s", + } + + if canonical := strings.TrimSpace(canonicalAssetIDFromName(base)); canonical != "" && !strings.EqualFold(canonical, base) { + names = append(names, + canonical+".audio.m4s", + "."+canonical+".audio.m4s", + ) + } + + return names +} + +func cleanupHasAudioSidecarForTS(tsName string, filesByLowName map[string]recordCleanupFile) bool { + for _, sidecarName := range cleanupAudioSidecarNamesForTS(tsName) { + f, ok := filesByLowName[strings.ToLower(sidecarName)] + if ok && f.size > 0 { + return true + } + } return false } -func cleanupFileStable(path string) bool { - path = strings.TrimSpace(path) - if path == "" { - return false +func cleanupWaitForStableRecordCandidates( + ctx context.Context, + candidates []recordCleanupDeleteCandidate, + activePaths map[string]struct{}, + resp *cleanupResp, +) ([]recordCleanupDeleteCandidate, error) { + if len(candidates) == 0 { + return nil, nil } - fi1, err := os.Stat(path) - if err != nil || fi1 == nil || fi1.IsDir() || fi1.Size() <= 0 { - return false + now := time.Now() + pendingStable := make([]recordCleanupDeleteCandidate, 0, len(candidates)) + ready := make([]recordCleanupDeleteCandidate, 0, len(candidates)) + + for _, candidate := range candidates { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + if now.Sub(candidate.file.modTime) < cleanupRecordOrphanMinAge { + resp.SkippedFiles++ + continue + } + + if cleanupRecordPathIsActive(candidate.file.path, activePaths) { + resp.SkippedFiles++ + continue + } + + if candidate.file.size <= 0 { + ready = append(ready, candidate) + continue + } + + pendingStable = append(pendingStable, candidate) } - time.Sleep(1200 * time.Millisecond) - - fi2, err := os.Stat(path) - if err != nil || fi2 == nil || fi2.IsDir() || fi2.Size() <= 0 { - return false + if len(pendingStable) == 0 { + return ready, nil } - return fi1.Size() == fi2.Size() + timer := time.NewTimer(cleanupRecordStableDelay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + + for _, candidate := range pendingStable { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + fi, err := os.Stat(candidate.file.path) + if err != nil || fi == nil || fi.IsDir() { + resp.SkippedFiles++ + continue + } + + if fi.Size() != candidate.file.size { + resp.SkippedFiles++ + continue + } + + candidate.file.size = fi.Size() + candidate.file.modTime = fi.ModTime() + ready = append(ready, candidate) + } + + return ready, nil } -func cleanupTSPathForAudioSidecar(audioPath string) string { - audioPath = strings.TrimSpace(audioPath) - if audioPath == "" { - return "" - } - - dir := filepath.Dir(audioPath) - name := filepath.Base(audioPath) - lowName := strings.ToLower(name) - - if !strings.HasSuffix(lowName, ".audio.m4s") { - return "" - } - - base := name[:len(name)-len(".audio.m4s")] - base = strings.TrimPrefix(base, ".") - - if strings.TrimSpace(base) == "" { - return "" - } - - return filepath.Join(dir, base+".ts") -} - -func cleanupDeleteRecordOrphanFile(ctx context.Context, jobID string, path string, resp *cleanupResp, reason string) { +func cleanupDeleteRecordOrphanCandidate(candidate recordCleanupDeleteCandidate, resp *cleanupResp) { if resp == nil { return } - select { - case <-ctx.Done(): - return - default: - } + file := candidate.file - path = strings.TrimSpace(path) - if path == "" { - return - } - - fi, err := os.Stat(path) - if err != nil || fi == nil || fi.IsDir() { - return - } - - size := fi.Size() - name := filepath.Base(path) - lowName := strings.ToLower(name) - - if time.Since(fi.ModTime()) < cleanupRecordOrphanMinAge { - resp.SkippedFiles++ - return - } - - if cleanupRecordFileIsActive(path) { - resp.SkippedFiles++ - return - } - - // 0-KB-Dateien sind stabil genug zum Löschen. - // Für >0 Bytes weiter die Stabilitätsprüfung behalten. - if size > 0 && !cleanupFileStable(path) { - resp.SkippedFiles++ - return - } - - // Fortschritt (Done/Total/CurrentFile) steuert der Aufrufer - // cleanupRecordDirOrphanAVFiles, damit die Progress-Bar sichtbar bleibt. - - if err := removeWithRetry(path); err != nil && !os.IsNotExist(err) { + if err := removeWithRetry(file.path); err != nil && !os.IsNotExist(err) { resp.ErrorCount++ - appLogln("⚠️ cleanup record orphan konnte nicht gelöscht werden:", name, reason, err) + appLogln("⚠️ cleanup record orphan konnte nicht gelöscht werden:", file.name, candidate.reason, err) return } resp.DeletedFiles++ resp.DeletedRecordOrphans++ - resp.DeletedBytes += size - resp.DeletedRecordOrphanBytes += size + resp.DeletedBytes += file.size + resp.DeletedRecordOrphanBytes += file.size - if strings.HasSuffix(lowName, ".muxing.ts") { + if strings.HasSuffix(file.lowName, ".muxing.ts") { resp.DeletedMuxingTemps++ - } else if strings.HasSuffix(lowName, ".audio.m4s") { + } else if strings.HasSuffix(file.lowName, ".audio.m4s") { resp.DeletedAudioOrphans++ - } else if !strings.HasPrefix(name, ".") && strings.EqualFold(filepath.Ext(name), ".ts") { + } else if !strings.HasPrefix(file.name, ".") && strings.EqualFold(filepath.Ext(file.name), ".ts") { resp.DeletedTSOrphans++ } - purgeDurationCacheForPath(path) + purgeDurationCacheForPath(file.path) - base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + base := strings.TrimSuffix(filepath.Base(file.path), filepath.Ext(file.path)) base = strings.TrimPrefix(base, ".") base = strings.TrimSuffix(base, ".audio") base = strings.TrimSuffix(base, ".muxing") @@ -586,7 +650,7 @@ func cleanupDeleteRecordOrphanFile(ctx context.Context, jobID string, path strin removeGeneratedForID(stripHotPrefix(base)) } - appLogln("🧹 cleanup deleted record orphan:", name, "reason:", reason, "size=", size) + appLogln("🧹 cleanup deleted record orphan:", file.name, "reason:", candidate.reason, "size=", file.size) } func cleanupRecordDirOrphanAVFiles(ctx context.Context, jobID string, recordAbs string, resp *cleanupResp) error { @@ -600,41 +664,10 @@ func cleanupRecordDirOrphanAVFiles(ctx context.Context, jobID string, recordAbs return nil } - // Gesamtzahl der zu prüfenden Dateien für einen sichtbaren Fortschritt. - total := 0 - for _, e := range entries { - if e == nil || e.IsDir() { - continue - } - if strings.TrimSpace(e.Name()) == "" { - continue - } - total++ - } - - updateCleanupJobState(jobID, func(st *CleanupTaskState) { - st.Queued = false - st.Running = true - if st.StartedAt.IsZero() { - st.StartedAt = time.Now() - } - st.Done = 0 - st.Total = total - st.CurrentFile = "" - st.Text = "Räume Record-Reste auf…" - st.Error = "" - st.FinishedAt = nil - }) - - processed := 0 + files := make([]recordCleanupFile, 0, len(entries)) + filesByLowName := make(map[string]recordCleanupFile, len(entries)) for _, e := range entries { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if e == nil || e.IsDir() { continue } @@ -644,54 +677,153 @@ func cleanupRecordDirOrphanAVFiles(ctx context.Context, jobID string, recordAbs continue } - processed++ - updateCleanupJobState(jobID, func(st *CleanupTaskState) { - st.Done = processed - st.Total = total - st.CurrentFile = name - }) + info, err := e.Info() + if err != nil || info == nil || info.IsDir() { + continue + } - full := filepath.Join(recordAbs, name) - low := strings.ToLower(name) + file := recordCleanupFile{ + path: filepath.Join(recordAbs, name), + name: name, + lowName: strings.ToLower(name), + size: info.Size(), + modTime: info.ModTime(), + } + + files = append(files, file) + filesByLowName[file.lowName] = file + } + + audioSidecarMatchedByLowName := make(map[string]bool, len(files)) + for _, file := range files { + if strings.HasPrefix(file.name, ".") || !strings.EqualFold(filepath.Ext(file.name), ".ts") || file.size <= 0 { + continue + } + + for _, sidecarName := range cleanupAudioSidecarNamesForTS(file.name) { + lowSidecarName := strings.ToLower(sidecarName) + if _, ok := filesByLowName[lowSidecarName]; ok { + audioSidecarMatchedByLowName[lowSidecarName] = true + } + } + } + + updateCleanupJobState(jobID, func(st *CleanupTaskState) { + st.Queued = false + st.Running = true + if st.StartedAt.IsZero() { + st.StartedAt = time.Now() + } + st.Done = 0 + st.Total = len(files) + st.CurrentFile = "" + st.Text = "Suche Record-Reste…" + st.Error = "" + st.FinishedAt = nil + }) + + activePaths := snapshotCleanupRecordActivePaths() + candidates := make([]recordCleanupDeleteCandidate, 0, 64) + lastProgressAt := time.Time{} + + updateProgress := func(done, total int, currentFile, text string, force bool) { + now := time.Now() + if !force && now.Sub(lastProgressAt) < 250*time.Millisecond && done%100 != 0 { + return + } + lastProgressAt = now + + updateCleanupJobState(jobID, func(st *CleanupTaskState) { + st.Done = done + st.Total = total + st.CurrentFile = strings.TrimSpace(currentFile) + if strings.TrimSpace(text) != "" { + st.Text = text + } + }) + } + + for i, file := range files { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + updateProgress(i+1, len(files), file.name, "Suche Record-Reste…", false) // Fall 0: // alte .muxing.ts sind temporäre Remux-/Muxing-Reste. - if strings.HasSuffix(low, ".muxing.ts") { + if strings.HasSuffix(file.lowName, ".muxing.ts") { resp.ScannedFiles++ - cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "stale muxing temp") + candidates = append(candidates, recordCleanupDeleteCandidate{ + file: file, + reason: "stale muxing temp", + }) continue } // Fall 1: // foo.ts existiert, aber kein passendes foo.audio.m4s/.foo.audio.m4s. - if !strings.HasPrefix(name, ".") && strings.EqualFold(filepath.Ext(name), ".ts") { + if !strings.HasPrefix(file.name, ".") && strings.EqualFold(filepath.Ext(file.name), ".ts") { resp.ScannedFiles++ - if _, ok := findPostworkAudioSidecarForTS(full); ok { + if cleanupHasAudioSidecarForTS(file.name, filesByLowName) { continue } - cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "missing audio sidecar") + candidates = append(candidates, recordCleanupDeleteCandidate{ + file: file, + reason: "missing audio sidecar", + }) continue } // Fall 2: // foo.audio.m4s/.foo.audio.m4s existiert, aber foo.ts fehlt. - if strings.HasSuffix(low, ".audio.m4s") { + if strings.HasSuffix(file.lowName, ".audio.m4s") { resp.ScannedFiles++ - tsPath := cleanupTSPathForAudioSidecar(full) - if strings.TrimSpace(tsPath) != "" { - if fi, err := os.Stat(tsPath); err == nil && fi != nil && !fi.IsDir() && fi.Size() > 0 { - continue - } + if audioSidecarMatchedByLowName[file.lowName] { + continue } - cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "missing ts file") + candidates = append(candidates, recordCleanupDeleteCandidate{ + file: file, + reason: "missing ts file", + }) continue } } + updateProgress(len(files), len(files), "", "Prüfe stabile Record-Reste…", true) + + stableCandidates, err := cleanupWaitForStableRecordCandidates(ctx, candidates, activePaths, resp) + if err != nil { + return err + } + + updateProgress(0, len(stableCandidates), "", "Lösche Record-Reste…", true) + activePaths = snapshotCleanupRecordActivePaths() + + for i, candidate := range stableCandidates { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + updateProgress(i+1, len(stableCandidates), candidate.file.name, "Lösche Record-Reste…", false) + if cleanupRecordPathIsActive(candidate.file.path, activePaths) { + resp.SkippedFiles++ + continue + } + + cleanupDeleteRecordOrphanCandidate(candidate, resp) + } + + updateProgress(len(stableCandidates), len(stableCandidates), "", cleanupProgressText(*resp), true) + return nil }