// backend\tasks_cleanup.go package main import ( "context" "encoding/json" "errors" "fmt" "io/fs" "net/http" "os" "path/filepath" "strings" "sync" "sync/atomic" "time" ) type cleanupCandidate struct { path string name string size int64 } type cleanupResp struct { // Small downloads cleanup ScannedFiles int `json:"scannedFiles"` DeletedFiles int `json:"deletedFiles"` SkippedFiles int `json:"skippedFiles"` DeletedBytes int64 `json:"deletedBytes"` DeletedBytesHuman string `json:"deletedBytesHuman"` ErrorCount int `json:"errorCount"` // Generated-GC GeneratedOrphansChecked int `json:"generatedOrphansChecked"` GeneratedOrphansRemoved int `json:"generatedOrphansRemoved"` GeneratedOrphansRemovedBytes int64 `json:"generatedOrphansRemovedBytes"` GeneratedOrphansRemovedBytesHuman string `json:"generatedOrphansRemovedBytesHuman"` } // Optional: falls du später Threshold per Body überschreiben willst. // Frontend sendet aktuell nichts -> wir nutzen Settings. type cleanupReq struct { BelowMB *int `json:"belowMB,omitempty"` } type cleanupTaskJob struct { State CleanupTaskState Cancel context.CancelFunc } var cleanupJobsMu sync.Mutex var cleanupJobs = map[string]*cleanupTaskJob{} func snapshotCleanupJobs() []CleanupTaskState { cleanupJobsMu.Lock() defer cleanupJobsMu.Unlock() now := time.Now() out := make([]CleanupTaskState, 0, len(cleanupJobs)) for id, job := range cleanupJobs { if job == nil { delete(cleanupJobs, id) continue } st := job.State if !st.Queued && !st.Running && st.FinishedAt != nil && now.Sub(*st.FinishedAt) > 4*time.Second { delete(cleanupJobs, id) continue } out = append(out, st) } return out } func updateCleanupJobState(jobID string, fn func(st *CleanupTaskState)) (CleanupTaskState, bool) { cleanupJobsMu.Lock() job := cleanupJobs[jobID] if job == nil { cleanupJobsMu.Unlock() return CleanupTaskState{}, false } fn(&job.State) st := job.State cleanupJobsMu.Unlock() publishTaskState() return st, true } func removeCleanupJob(jobID string) { cleanupJobsMu.Lock() delete(cleanupJobs, jobID) cleanupJobsMu.Unlock() publishTaskState() } func finishCleanupJob(jobID string, text string, err error) { now := time.Now() updateCleanupJobState(jobID, func(st *CleanupTaskState) { st.Queued = false st.Running = false st.FinishedAt = &now st.CurrentFile = "" if err == nil { st.Error = "" st.Text = strings.TrimSpace(text) if st.Total > 0 { st.Done = st.Total } return } if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { st.Error = "Abgebrochen" st.Text = "Abgebrochen." } else { st.Error = strings.TrimSpace(err.Error()) st.Text = "" } }) time.AfterFunc(4*time.Second, func() { removeCleanupJob(jobID) }) } // /api/settings/cleanup (POST) // - löscht kleine Dateien < threshold MB (mp4/ts; skip .part/.tmp; skip keep-Ordner) // - räumt Orphans (preview/thumbs + generated) auf func settingsCleanupHandler(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodPost: s := getSettings() doneAbs, err := resolvePathRelativeToApp(s.DoneDir) if err != nil || strings.TrimSpace(doneAbs) == "" { http.Error(w, "doneDir auflösung fehlgeschlagen", http.StatusBadRequest) return } mb := int(s.AutoDeleteSmallDownloadsBelowMB) var req cleanupReq if r.Body != nil { _ = json.NewDecoder(r.Body).Decode(&req) } if req.BelowMB != nil { mb = *req.BelowMB } if mb < 0 { mb = 0 } jobID := newTaskID("cleanup") ctx, cancel := context.WithCancel(context.Background()) st := CleanupTaskState{ ID: jobID, Queued: true, Running: false, QueuedAt: time.Now(), StartedAt: time.Time{}, FinishedAt: nil, Text: "Wartet…", Error: "", CurrentFile: "", Done: 0, Total: 0, } cleanupJobsMu.Lock() cleanupJobs[jobID] = &cleanupTaskJob{ State: st, Cancel: cancel, } cleanupJobsMu.Unlock() publishTaskState() go func(jobID string, ctx context.Context, doneAbs string, mb int) { if err := acquireExclusiveTask(ctx); err != nil { finishCleanupJob(jobID, "", err) return } defer releaseExclusiveTask() updateCleanupJobState(jobID, func(st *CleanupTaskState) { st.Queued = false st.Running = true st.StartedAt = time.Now() st.Text = "Räume auf…" }) resp := cleanupResp{} if mb > 0 { threshold := int64(mb) * 1024 * 1024 if err := cleanupSmallFiles(ctx, jobID, doneAbs, threshold, &resp); err != nil { finishCleanupJob(jobID, "", err) return } } select { case <-ctx.Done(): finishCleanupJob(jobID, "", ctx.Err()) return default: } gcStats := triggerGeneratedGarbageCollectorSync() resp.GeneratedOrphansChecked = gcStats.Checked resp.GeneratedOrphansRemoved = gcStats.Removed resp.GeneratedOrphansRemovedBytes = gcStats.RemovedBytes resp.GeneratedOrphansRemovedBytesHuman = formatBytesSI(gcStats.RemovedBytes) resp.DeletedBytes += gcStats.RemovedBytes resp.DeletedBytesHuman = formatBytesSI(resp.DeletedBytes) if resp.DeletedFiles > 0 || resp.GeneratedOrphansRemoved > 0 { notifyDoneChanged() } finishCleanupJob( jobID, cleanupProgressText(resp.DeletedFiles, resp.ScannedFiles, resp.DeletedBytes), nil, ) }(jobID, ctx, doneAbs, mb) writeJSON(w, http.StatusOK, st) return case http.MethodDelete: id := strings.TrimSpace(r.URL.Query().Get("id")) if id == "" { http.Error(w, "id fehlt", http.StatusBadRequest) return } cleanupJobsMu.Lock() job := cleanupJobs[id] cleanupJobsMu.Unlock() if job != nil && job.Cancel != nil { job.Cancel() } w.WriteHeader(http.StatusNoContent) return default: http.Error(w, "Nur POST/DELETE erlaubt", http.StatusMethodNotAllowed) return } } func cleanupProgressText(deleted int, total int, freedBytes int64) string { return fmt.Sprintf( "gelöscht: %d/%d · freigegeben: %s", deleted, total, formatBytesSI(freedBytes), ) } func cleanupSmallFiles(ctx context.Context, jobID string, doneAbs string, threshold int64, resp *cleanupResp) error { isCandidate := func(name string) bool { low := strings.ToLower(name) if strings.Contains(low, ".part") || strings.Contains(low, ".tmp") { return false } ext := strings.ToLower(filepath.Ext(name)) return ext == ".mp4" || ext == ".ts" } candidates := make([]cleanupCandidate, 0, 1024) collectCandidates := func(dir string, allowSubdirs bool) error { ents, err := os.ReadDir(dir) if err != nil { return nil } for _, e := range ents { select { case <-ctx.Done(): return ctx.Err() default: } full := filepath.Join(dir, e.Name()) if e.IsDir() { if !allowSubdirs { continue } if e.Name() == "keep" { continue } sub, err := os.ReadDir(full) if err != nil { continue } for _, se := range sub { if se.IsDir() { continue } name := se.Name() if !isCandidate(name) { resp.SkippedFiles++ continue } p := filepath.Join(full, name) fi, err := os.Stat(p) if err != nil || fi.IsDir() || fi.Size() <= 0 { resp.SkippedFiles++ continue } candidates = append(candidates, cleanupCandidate{ path: p, name: name, size: fi.Size(), }) } continue } name := e.Name() if !isCandidate(name) { resp.SkippedFiles++ continue } fi, err := os.Stat(full) if err != nil || fi.IsDir() || fi.Size() <= 0 { resp.SkippedFiles++ continue } candidates = append(candidates, cleanupCandidate{ path: full, name: name, size: fi.Size(), }) } return nil } if err := collectCandidates(doneAbs, true); err != nil { return err } resp.ScannedFiles = len(candidates) updateCleanupJobState(jobID, func(st *CleanupTaskState) { st.Queued = false st.Running = true if st.StartedAt.IsZero() { st.StartedAt = time.Now() } st.Done = 0 st.Total = resp.ScannedFiles st.CurrentFile = "" st.Text = cleanupProgressText(resp.DeletedFiles, resp.ScannedFiles, resp.DeletedBytes) st.Error = "" st.FinishedAt = nil }) for i, c := range candidates { select { case <-ctx.Done(): return ctx.Err() default: } if c.size < threshold { base := strings.TrimSuffix(filepath.Base(c.path), filepath.Ext(c.path)) id := stripHotPrefix(base) if derr := removeWithRetry(c.path); derr == nil || os.IsNotExist(derr) { resp.DeletedFiles++ resp.DeletedBytes += c.size if strings.TrimSpace(id) != "" { removeGeneratedForID(id) } purgeDurationCacheForPath(c.path) } else { resp.ErrorCount++ } } updateCleanupJobState(jobID, func(st *CleanupTaskState) { st.Queued = false st.Running = true if st.StartedAt.IsZero() { st.StartedAt = time.Now() } st.Done = i + 1 st.Total = resp.ScannedFiles st.CurrentFile = "" st.Text = cleanupProgressText(resp.DeletedFiles, resp.ScannedFiles, resp.DeletedBytes) st.Error = "" st.FinishedAt = nil }) } return nil } var generatedGCRunning int32 type generatedGCStats struct { Checked int Removed int RemovedBytes int64 } func dirSizeBytes(root string) int64 { root = strings.TrimSpace(root) if root == "" { return 0 } var total int64 _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { if err != nil { return nil } if d.IsDir() { return nil } info, err := d.Info() if err != nil { return nil } if info.Size() > 0 { total += info.Size() } return nil }) return total } func removeDirIfEmpty(dir string) (bool, error) { dir = strings.TrimSpace(dir) if dir == "" { return false, nil } entries, err := os.ReadDir(dir) if err != nil { if os.IsNotExist(err) { return false, nil } return false, err } if len(entries) > 0 { return false, nil } if err := os.Remove(dir); err != nil { if os.IsNotExist(err) { return false, nil } return false, err } return true, nil } // Läuft synchron und liefert Zahlen zurück (für /api/settings/cleanup Response). func triggerGeneratedGarbageCollectorSync() generatedGCStats { // nur 1 GC gleichzeitig if !atomic.CompareAndSwapInt32(&generatedGCRunning, 0, 1) { appLogln("🧹 [gc] skip: already running") return generatedGCStats{} } defer atomic.StoreInt32(&generatedGCRunning, 0) stats := runGeneratedGarbageCollector() return stats } // Läuft 1× nach Serverstart (mit Delay), löscht /generated/* Orphans. func startGeneratedGarbageCollector() { go func() { time.Sleep(3 * time.Second) triggerGeneratedGarbageCollectorSync() }() } func collectLiveIDsFromTrash(trashDir string, live map[string]struct{}) { trashDir = strings.TrimSpace(trashDir) if trashDir == "" || live == nil { return } entries, err := os.ReadDir(trashDir) if err != nil { return } for _, e := range entries { if e == nil || e.IsDir() { continue } name := strings.TrimSpace(e.Name()) if name == "" { continue } // last.json ignorieren if strings.EqualFold(name, "last.json") { continue } // Format in .trash: // __ parts := strings.SplitN(name, "__", 2) if len(parts) != 2 { continue } originalFile := strings.TrimSpace(parts[1]) if originalFile == "" { continue } ext := strings.ToLower(filepath.Ext(originalFile)) if ext != ".mp4" && ext != ".ts" { continue } full := filepath.Join(trashDir, name) fi, err := os.Stat(full) if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 { continue } id := stripHotPrefix(strings.TrimSuffix(filepath.Base(originalFile), ext)) id = strings.TrimSpace(id) if id == "" { continue } if sid, err := sanitizeID(id); err == nil && sid != "" { live[sid] = struct{}{} } } } // Core-Logik ohne Delay (für manuelle Trigger, z.B. nach Cleanup) // Liefert Stats zurück, damit /api/settings/cleanup die Zahlen anzeigen kann. func runGeneratedGarbageCollector() generatedGCStats { stats := generatedGCStats{} s := getSettings() doneAbs, err := resolvePathRelativeToApp(s.DoneDir) if err != nil { appLogln("🧹 [gc] resolve doneDir failed:", err) return stats } doneAbs = strings.TrimSpace(doneAbs) if doneAbs == "" { return stats } // 1) Live-IDs sammeln: alle mp4/ts unter /done (rekursiv), .trash zunächst ignorieren live := make(map[string]struct{}, 4096) _ = filepath.WalkDir(doneAbs, func(p string, d fs.DirEntry, err error) error { if err != nil { return nil } name := d.Name() if d.IsDir() { if strings.EqualFold(name, ".trash") { return fs.SkipDir } return nil } ext := strings.ToLower(filepath.Ext(name)) if ext != ".mp4" && ext != ".ts" { return nil } info, err := d.Info() if err != nil || info.IsDir() || info.Size() <= 0 { return nil } base := strings.TrimSuffix(name, ext) id, err := sanitizeID(stripHotPrefix(base)) if err != nil || id == "" { return nil } live[id] = struct{}{} return nil }) // 1a) Dateien in .trash ebenfalls als "live" behandeln, // damit ihre Generated-Dateien erst gelöscht werden, // wenn sie wirklich endgültig aus .trash verschwinden. collectLiveIDsFromTrash(filepath.Join(doneAbs, ".trash"), live) // 1b) Auch aktuell laufende / nachbearbeitete Jobs als "live" behandeln. // Zusätzlich separat merken, damit leere Ordner aktiver Jobs nicht gelöscht werden. activeJobIDs := make(map[string]struct{}, 128) jobsMu.RLock() for _, j := range jobs { if j == nil { continue } id := strings.TrimSpace(assetIDForJob(j)) if id == "" { continue } live[id] = struct{}{} activeJobIDs[id] = struct{}{} } jobsMu.RUnlock() // 2) /generated/meta/ prüfen metaRoot, err := generatedMetaRoot() if err == nil { metaRoot = strings.TrimSpace(metaRoot) } if err != nil || metaRoot == "" { return stats } removedMeta := 0 checkedMeta := 0 removedBytes := int64(0) if entries, err := os.ReadDir(metaRoot); err == nil { for _, e := range entries { if !e.IsDir() { continue } id := strings.TrimSpace(e.Name()) if id == "" || strings.HasPrefix(id, ".") { continue } checkedMeta++ orphanDir := filepath.Join(metaRoot, id) // Leere generated/meta/-Ordner immer entfernen, // außer ein aktiver Job könnte gerade hinein schreiben. if _, active := activeJobIDs[id]; !active { if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty { removedMeta++ continue } } if _, ok := live[id]; ok { continue } orphanBytes := dirSizeBytes(orphanDir) removeGeneratedForID(id) // Falls removeGeneratedForID nur Dateien gelöscht hat, den nun leeren Ordner entfernen. if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty { // ok } if _, err := os.Stat(orphanDir); os.IsNotExist(err) { removedMeta++ removedBytes += orphanBytes } } } //fmt.Printf("🧹 [gc] generated/meta checked=%d removed_orphans=%d bytes=%d\n", checkedMeta, removedMeta, removedBytes) stats.Checked += checkedMeta stats.Removed += removedMeta stats.RemovedBytes += removedBytes return stats }