nsfwapp/backend/tasks_cleanup.go
2026-06-17 09:24:18 +02:00

1308 lines
29 KiB
Go
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// backend\tasks_cleanup.go
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
)
type cleanupCandidate struct {
path string
name string
size int64
}
type cleanupResp struct {
// Small / broken downloads cleanup
ScannedFiles int `json:"scannedFiles"`
DeletedFiles int `json:"deletedFiles"`
DeletedBrokenFiles int `json:"deletedBrokenFiles"`
DeletedLowRatedFiles int `json:"deletedLowRatedFiles"`
SkippedFiles int `json:"skippedFiles"`
DeletedBytes int64 `json:"deletedBytes"`
DeletedBytesHuman string `json:"deletedBytesHuman"`
DeletedRecordOrphans int `json:"deletedRecordOrphans"`
DeletedRecordOrphanBytes int64 `json:"deletedRecordOrphanBytes"`
DeletedMuxingTemps int `json:"deletedMuxingTemps"`
DeletedAudioOrphans int `json:"deletedAudioOrphans"`
DeletedTSOrphans int `json:"deletedTSOrphans"`
ErrorCount int `json:"errorCount"`
// Generated-GC
GeneratedOrphansChecked int `json:"generatedOrphansChecked"`
GeneratedOrphansRemoved int `json:"generatedOrphansRemoved"`
GeneratedOrphansRemovedBytes int64 `json:"generatedOrphansRemovedBytes"`
GeneratedOrphansRemovedBytesHuman string `json:"generatedOrphansRemovedBytesHuman"`
}
// Optional: falls du später Threshold per Body überschreiben willst.
// Frontend sendet aktuell nichts -> wir nutzen Settings.
type cleanupReq struct {
BelowMB *int `json:"belowMB,omitempty"`
}
type cleanupTaskJob struct {
State CleanupTaskState
Cancel context.CancelFunc
}
var cleanupJobsMu sync.Mutex
var cleanupJobs = map[string]*cleanupTaskJob{}
func snapshotCleanupJobs() []CleanupTaskState {
cleanupJobsMu.Lock()
defer cleanupJobsMu.Unlock()
now := time.Now()
out := make([]CleanupTaskState, 0, len(cleanupJobs))
for id, job := range cleanupJobs {
if job == nil {
delete(cleanupJobs, id)
continue
}
st := job.State
if !st.Queued && !st.Running && st.FinishedAt != nil && now.Sub(*st.FinishedAt) > 4*time.Second {
delete(cleanupJobs, id)
continue
}
out = append(out, st)
}
return out
}
func updateCleanupJobState(jobID string, fn func(st *CleanupTaskState)) (CleanupTaskState, bool) {
cleanupJobsMu.Lock()
job := cleanupJobs[jobID]
if job == nil {
cleanupJobsMu.Unlock()
return CleanupTaskState{}, false
}
fn(&job.State)
st := job.State
cleanupJobsMu.Unlock()
publishTaskState()
return st, true
}
func removeCleanupJob(jobID string) {
cleanupJobsMu.Lock()
delete(cleanupJobs, jobID)
cleanupJobsMu.Unlock()
publishTaskState()
}
func finishCleanupJob(jobID string, text string, err error) {
now := time.Now()
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Queued = false
st.Running = false
st.FinishedAt = &now
st.CurrentFile = ""
if err == nil {
st.Error = ""
st.Text = strings.TrimSpace(text)
if st.Total > 0 {
st.Done = st.Total
}
return
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
st.Error = "Abgebrochen"
st.Text = "Abgebrochen."
} else {
st.Error = strings.TrimSpace(err.Error())
st.Text = ""
}
})
time.AfterFunc(4*time.Second, func() {
removeCleanupJob(jobID)
})
}
// /api/settings/cleanup (POST)
// - löscht kleine Dateien < threshold MB (mp4/ts; skip .part/.tmp; skip keep-Ordner)
// - räumt Orphans (preview/thumbs + generated) auf
func settingsCleanupHandler(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
s := getSettings()
doneAbs, err := resolvePathRelativeToApp(s.DoneDir)
if err != nil || strings.TrimSpace(doneAbs) == "" {
http.Error(w, "doneDir auflösung fehlgeschlagen", http.StatusBadRequest)
return
}
recordAbs, err := resolvePathRelativeToApp(s.RecordDir)
if err != nil || strings.TrimSpace(recordAbs) == "" {
recordAbs = strings.TrimSpace(s.RecordDir)
}
mb := int(s.AutoDeleteSmallDownloadsBelowMB)
deleteLowRated := s.AutoDeleteLowRatedDownloads
maxStars := clampAutoDeleteRatingStars(s.AutoDeleteLowRatedDownloadsMaxStars)
var req cleanupReq
if r.Body != nil {
_ = json.NewDecoder(r.Body).Decode(&req)
}
if req.BelowMB != nil {
mb = *req.BelowMB
}
if mb < 0 {
mb = 0
}
jobID := newTaskID("cleanup")
ctx, cancel := context.WithCancel(context.Background())
st := CleanupTaskState{
ID: jobID,
Queued: true,
Running: false,
QueuedAt: time.Now(),
StartedAt: time.Time{},
FinishedAt: nil,
Text: "Wartet…",
Error: "",
CurrentFile: "",
Done: 0,
Total: 0,
}
cleanupJobsMu.Lock()
cleanupJobs[jobID] = &cleanupTaskJob{
State: st,
Cancel: cancel,
}
cleanupJobsMu.Unlock()
publishTaskState()
go func(jobID string, ctx context.Context, doneAbs string, recordAbs string, mb int, deleteLowRated bool, maxStars int) {
if err := acquireExclusiveTask(ctx); err != nil {
finishCleanupJob(jobID, "", err)
return
}
defer releaseExclusiveTask()
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Queued = false
st.Running = true
st.StartedAt = time.Now()
st.Text = "Räume auf…"
})
resp := cleanupResp{}
if mb > 0 || deleteLowRated {
threshold := int64(mb) * 1024 * 1024
if err := cleanupSmallFiles(ctx, jobID, doneAbs, threshold, deleteLowRated, maxStars, &resp); err != nil {
finishCleanupJob(jobID, "", err)
return
}
}
// Neuer Cleanup-Abschnitt: alten 100%-Fortschritt vom Small-Files-Scan zurücksetzen.
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Queued = false
st.Running = true
if st.StartedAt.IsZero() {
st.StartedAt = time.Now()
}
st.Done = 0
st.Total = 0
st.CurrentFile = ""
st.Text = "Räume Record-Reste auf…"
st.Error = ""
st.FinishedAt = nil
})
if err := cleanupRecordDirOrphanAVFiles(ctx, jobID, recordAbs, &resp); err != nil {
finishCleanupJob(jobID, "", err)
return
}
select {
case <-ctx.Done():
finishCleanupJob(jobID, "", ctx.Err())
return
default:
}
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Queued = false
st.Running = true
st.Done = 0
st.Total = 0
st.CurrentFile = ""
st.Text = "Räume Generated-Assets auf…"
st.Error = ""
st.FinishedAt = nil
})
gcStats := triggerGeneratedGarbageCollectorSync()
resp.GeneratedOrphansChecked = gcStats.Checked
resp.GeneratedOrphansRemoved = gcStats.Removed
resp.GeneratedOrphansRemovedBytes = gcStats.RemovedBytes
resp.GeneratedOrphansRemovedBytesHuman = formatBytesSI(gcStats.RemovedBytes)
resp.DeletedBytes += gcStats.RemovedBytes
resp.DeletedBytesHuman = formatBytesSI(resp.DeletedBytes)
if resp.DeletedFiles > 0 || resp.GeneratedOrphansRemoved > 0 {
notifyDoneChanged()
}
finishCleanupJob(
jobID,
cleanupProgressText(resp),
nil,
)
}(jobID, ctx, doneAbs, recordAbs, mb, deleteLowRated, maxStars)
writeJSON(w, http.StatusOK, st)
return
case http.MethodDelete:
id := strings.TrimSpace(r.URL.Query().Get("id"))
if id == "" {
http.Error(w, "id fehlt", http.StatusBadRequest)
return
}
cleanupJobsMu.Lock()
job := cleanupJobs[id]
cleanupJobsMu.Unlock()
if job != nil && job.Cancel != nil {
job.Cancel()
}
w.WriteHeader(http.StatusNoContent)
return
default:
http.Error(w, "Nur POST/DELETE erlaubt", http.StatusMethodNotAllowed)
return
}
}
func cleanupProgressText(resp cleanupResp) string {
parts := []string{
fmt.Sprintf("gelöscht: %d/%d", resp.DeletedFiles, resp.ScannedFiles),
}
if resp.DeletedBrokenFiles > 0 {
parts = append(parts, fmt.Sprintf("defekt: %d", resp.DeletedBrokenFiles))
}
if resp.DeletedLowRatedFiles > 0 {
parts = append(parts, fmt.Sprintf("Rating: %d", resp.DeletedLowRatedFiles))
}
if resp.DeletedRecordOrphans > 0 {
parts = append(parts, fmt.Sprintf("Records: %d", resp.DeletedRecordOrphans))
}
if resp.DeletedTSOrphans > 0 {
parts = append(parts, fmt.Sprintf("TS ohne Audio: %d", resp.DeletedTSOrphans))
}
if resp.DeletedAudioOrphans > 0 {
parts = append(parts, fmt.Sprintf("Audio ohne TS: %d", resp.DeletedAudioOrphans))
}
if resp.DeletedMuxingTemps > 0 {
parts = append(parts, fmt.Sprintf("Muxing-Temp: %d", resp.DeletedMuxingTemps))
}
if resp.GeneratedOrphansRemoved > 0 {
parts = append(parts, fmt.Sprintf("Generated: %d", resp.GeneratedOrphansRemoved))
}
if resp.ErrorCount > 0 {
parts = append(parts, fmt.Sprintf("Fehler: %d", resp.ErrorCount))
}
parts = append(parts, fmt.Sprintf("freigegeben: %s", formatBytesSI(resp.DeletedBytes)))
return strings.Join(parts, " · ")
}
func cleanupVideoLooksBroken(ctx context.Context, path string) (bool, string) {
path = strings.TrimSpace(path)
if path == "" {
return false, ""
}
ext := strings.ToLower(filepath.Ext(path))
if ext != ".mp4" && ext != ".ts" {
return false, ""
}
fi, err := os.Stat(path)
if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
return true, "file missing/empty"
}
// Erst ffprobe/duration: wenn keine Dauer lesbar ist, ist die Datei meistens kaputt.
dctx, cancel := context.WithTimeout(ctx, 8*time.Second)
dur, derr := durationSecondsCached(dctx, path)
dctxErr := dctx.Err()
cancel()
if errors.Is(dctxErr, context.DeadlineExceeded) || errors.Is(dctxErr, context.Canceled) {
// Timeout beim Prüfen nicht als "kaputt" werten, um keine guten Dateien wegen Last zu löschen.
return false, "duration check timeout"
}
if derr != nil || dur <= 0 {
if derr != nil {
return true, "duration failed: " + derr.Error()
}
return true, "duration <= 0"
}
// Danach konservative Decode-Prüfung am Ende.
// Diese Funktion nutzt du bereits nach dem Remux in recorder.go.
vctx, vcancel := context.WithTimeout(ctx, 20*time.Second)
verr := validateVideoDecodesNearEnd(vctx, path)
vctxErr := vctx.Err()
vcancel()
if errors.Is(vctxErr, context.DeadlineExceeded) || errors.Is(vctxErr, context.Canceled) {
return false, "decode check timeout"
}
if verr != nil {
return true, "decode failed: " + verr.Error()
}
return false, ""
}
const cleanupRecordOrphanMinAge = 60 * time.Second
func cleanupRecordFileIsActive(path string) bool {
path = filepath.Clean(strings.TrimSpace(path))
if path == "" {
return false
}
jobsMu.RLock()
defer jobsMu.RUnlock()
for _, j := range jobs {
if j == nil {
continue
}
out := filepath.Clean(strings.TrimSpace(j.Output))
if out != "" && strings.EqualFold(out, path) {
return true
}
for _, sidecar := range postworkAudioSidecarCandidatesForOutput(out) {
if sidecar != "" && strings.EqualFold(filepath.Clean(sidecar), path) {
return true
}
}
}
return false
}
func cleanupFileStable(path string) bool {
path = strings.TrimSpace(path)
if path == "" {
return false
}
fi1, err := os.Stat(path)
if err != nil || fi1 == nil || fi1.IsDir() || fi1.Size() <= 0 {
return false
}
time.Sleep(1200 * time.Millisecond)
fi2, err := os.Stat(path)
if err != nil || fi2 == nil || fi2.IsDir() || fi2.Size() <= 0 {
return false
}
return fi1.Size() == fi2.Size()
}
func cleanupTSPathForAudioSidecar(audioPath string) string {
audioPath = strings.TrimSpace(audioPath)
if audioPath == "" {
return ""
}
dir := filepath.Dir(audioPath)
name := filepath.Base(audioPath)
if !strings.HasPrefix(name, ".") {
return ""
}
if !strings.HasSuffix(strings.ToLower(name), ".audio.m4s") {
return ""
}
base := strings.TrimPrefix(name, ".")
base = strings.TrimSuffix(base, ".audio.m4s")
if strings.TrimSpace(base) == "" {
return ""
}
return filepath.Join(dir, base+".ts")
}
func cleanupDeleteRecordOrphanFile(ctx context.Context, jobID string, path string, resp *cleanupResp, reason string) {
if resp == nil {
return
}
select {
case <-ctx.Done():
return
default:
}
path = strings.TrimSpace(path)
if path == "" {
return
}
fi, err := os.Stat(path)
if err != nil || fi == nil || fi.IsDir() {
return
}
size := fi.Size()
name := filepath.Base(path)
lowName := strings.ToLower(name)
if time.Since(fi.ModTime()) < cleanupRecordOrphanMinAge {
resp.SkippedFiles++
return
}
if cleanupRecordFileIsActive(path) {
resp.SkippedFiles++
return
}
// 0-KB-Dateien sind stabil genug zum Löschen.
// Für >0 Bytes weiter die Stabilitätsprüfung behalten.
if size > 0 && !cleanupFileStable(path) {
resp.SkippedFiles++
return
}
// Fortschritt (Done/Total/CurrentFile) steuert der Aufrufer
// cleanupRecordDirOrphanAVFiles, damit die Progress-Bar sichtbar bleibt.
if err := removeWithRetry(path); err != nil && !os.IsNotExist(err) {
resp.ErrorCount++
appLogln("⚠️ cleanup record orphan konnte nicht gelöscht werden:", name, reason, err)
return
}
resp.DeletedFiles++
resp.DeletedRecordOrphans++
resp.DeletedBytes += size
resp.DeletedRecordOrphanBytes += size
if strings.HasSuffix(lowName, ".muxing.ts") {
resp.DeletedMuxingTemps++
} else if strings.HasPrefix(name, ".") && strings.HasSuffix(lowName, ".audio.m4s") {
resp.DeletedAudioOrphans++
} else if !strings.HasPrefix(name, ".") && strings.EqualFold(filepath.Ext(name), ".ts") {
resp.DeletedTSOrphans++
}
purgeDurationCacheForPath(path)
base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
base = strings.TrimPrefix(base, ".")
base = strings.TrimSuffix(base, ".audio")
base = strings.TrimSuffix(base, ".muxing")
if strings.TrimSpace(base) != "" {
removeGeneratedForID(stripHotPrefix(base))
}
appLogln("🧹 cleanup deleted record orphan:", name, "reason:", reason, "size=", size)
}
func cleanupRecordDirOrphanAVFiles(ctx context.Context, jobID string, recordAbs string, resp *cleanupResp) error {
recordAbs = strings.TrimSpace(recordAbs)
if recordAbs == "" || resp == nil {
return nil
}
entries, err := os.ReadDir(recordAbs)
if err != nil {
return nil
}
// Gesamtzahl der zu prüfenden Dateien für einen sichtbaren Fortschritt.
total := 0
for _, e := range entries {
if e == nil || e.IsDir() {
continue
}
if strings.TrimSpace(e.Name()) == "" {
continue
}
total++
}
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Queued = false
st.Running = true
if st.StartedAt.IsZero() {
st.StartedAt = time.Now()
}
st.Done = 0
st.Total = total
st.CurrentFile = ""
st.Text = "Räume Record-Reste auf…"
st.Error = ""
st.FinishedAt = nil
})
processed := 0
for _, e := range entries {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if e == nil || e.IsDir() {
continue
}
name := strings.TrimSpace(e.Name())
if name == "" {
continue
}
processed++
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Done = processed
st.Total = total
st.CurrentFile = name
})
full := filepath.Join(recordAbs, name)
low := strings.ToLower(name)
// Fall 0:
// alte .muxing.ts sind temporäre Remux-/Muxing-Reste.
if strings.HasSuffix(low, ".muxing.ts") {
resp.ScannedFiles++
cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "stale muxing temp")
continue
}
// Fall 1:
// foo.ts existiert, aber .foo.audio.m4s fehlt.
if !strings.HasPrefix(name, ".") && strings.EqualFold(filepath.Ext(name), ".ts") {
resp.ScannedFiles++
if _, ok := findPostworkAudioSidecarForTS(full); ok {
continue
}
cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "missing audio sidecar")
continue
}
// Fall 2:
// .foo.audio.m4s existiert, aber foo.ts fehlt.
if strings.HasPrefix(name, ".") && strings.HasSuffix(low, ".audio.m4s") {
resp.ScannedFiles++
tsPath := cleanupTSPathForAudioSidecar(full)
if strings.TrimSpace(tsPath) != "" {
if fi, err := os.Stat(tsPath); err == nil && fi != nil && !fi.IsDir() && fi.Size() > 0 {
continue
}
}
cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "missing ts file")
continue
}
}
return nil
}
func clampAutoDeleteRatingStars(v int) int {
if v < 1 {
return 3
}
if v > 5 {
return 5
}
return v
}
func ratingStarsForVideoPath(videoPath string) (int, bool, error) {
videoPath = strings.TrimSpace(videoPath)
if videoPath == "" {
return 0, false, nil
}
base := strings.TrimSuffix(filepath.Base(videoPath), filepath.Ext(videoPath))
assetID := stripHotPrefix(base)
if strings.TrimSpace(assetID) == "" {
return 0, false, nil
}
metaPath, err := generatedMetaFile(assetID)
if err != nil {
return 0, false, err
}
m, ok := readVideoMetaLoose(metaPath)
if !ok || m == nil {
return 0, false, nil
}
ai := pickAnalysisAIForGoal(m.Analysis, "highlights")
if ai == nil || ai.Rating == nil {
return 0, false, nil
}
stars := ai.Rating.Stars
if stars <= 0 && ai.Rating.Score > 0 {
stars = starsFromHighlightScore(ai.Rating.Score)
}
if stars < 1 || stars > 5 {
return 0, false, nil
}
return stars, true, nil
}
func deleteDoneVideoAndGenerated(videoPath string) (int64, error) {
videoPath = filepath.Clean(strings.TrimSpace(videoPath))
if videoPath == "" {
return 0, nil
}
ext := strings.ToLower(filepath.Ext(videoPath))
if ext != ".mp4" && ext != ".ts" {
return 0, nil
}
fi, err := os.Stat(videoPath)
if err != nil {
if os.IsNotExist(err) {
return 0, nil
}
return 0, err
}
if fi == nil || fi.IsDir() {
return 0, nil
}
name := filepath.Base(videoPath)
base := strings.TrimSuffix(name, filepath.Ext(name))
assetID := stripHotPrefix(base)
// Stoppt ggf. laufende Asset-/Enrich-Tasks für genau diese Datei,
// damit Windows-Dateihandles nicht blockieren.
releaseFileForMutation(name)
if err := removeWithRetry(videoPath); err != nil && !os.IsNotExist(err) {
return 0, err
}
if strings.TrimSpace(assetID) != "" {
removeGeneratedForID(assetID)
}
purgeDurationCacheForPath(videoPath)
return fi.Size(), nil
}
func autoDeleteLowRatedDownloadAfterAnalysis(ctx context.Context, videoPath string, rating *aiRatingMeta) {
select {
case <-ctx.Done():
return
default:
}
videoPath = strings.TrimSpace(videoPath)
if videoPath == "" {
return
}
s := getSettings()
if !s.AutoDeleteLowRatedDownloads {
return
}
maxStars := clampAutoDeleteRatingStars(s.AutoDeleteLowRatedDownloadsMaxStars)
stars := 0
if rating != nil {
stars = rating.Stars
if stars <= 0 && rating.Score > 0 {
stars = starsFromHighlightScore(rating.Score)
}
}
// Wichtig für Background-/Regenerate-Analyse:
// Dort wird rating teilweise als nil übergeben, obwohl es kurz vorher
// in meta.json gespeichert wurde. Dann lesen wir die Sterne aus der Meta.
if stars <= 0 {
if metaStars, ok, err := ratingStarsForVideoPath(videoPath); err != nil {
appLogln("⚠️ [rating-delete] rating lookup failed:", filepath.Base(videoPath), err)
} else if ok {
stars = metaStars
}
}
if stars < 1 || stars > 5 {
appLogf(
" [rating-delete] übersprungen, kein gültiges Rating: %s stars=%d threshold<=%d",
filepath.Base(videoPath),
stars,
maxStars,
)
return
}
if stars > maxStars {
appLogf(
" [rating-delete] behalten: %s stars=%d threshold<=%d",
filepath.Base(videoPath),
stars,
maxStars,
)
return
}
deletedBytes, err := deleteDoneVideoAndGenerated(videoPath)
if err != nil {
appLogln("⚠️ [rating-delete] konnte Download nicht löschen:", filepath.Base(videoPath), err)
return
}
appLogf(
"🗑️ [rating-delete] gelöscht nach Analyse: %s stars=%d threshold<=%d bytes=%s",
filepath.Base(videoPath),
stars,
maxStars,
formatBytesSI(deletedBytes),
)
notifyDoneChanged()
}
func cleanupSmallFiles(ctx context.Context, jobID string, doneAbs string, threshold int64, deleteLowRated bool, maxStars int, resp *cleanupResp) error {
isCandidate := func(name string) bool {
low := strings.ToLower(name)
if strings.Contains(low, ".part") || strings.Contains(low, ".tmp") {
return false
}
ext := strings.ToLower(filepath.Ext(name))
return ext == ".mp4" || ext == ".ts"
}
candidates := make([]cleanupCandidate, 0, 1024)
collectCandidates := func(dir string, allowSubdirs bool) error {
ents, err := os.ReadDir(dir)
if err != nil {
return nil
}
for _, e := range ents {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
full := filepath.Join(dir, e.Name())
if e.IsDir() {
if !allowSubdirs {
continue
}
if e.Name() == "keep" {
continue
}
sub, err := os.ReadDir(full)
if err != nil {
continue
}
for _, se := range sub {
if se.IsDir() {
continue
}
name := se.Name()
if !isCandidate(name) {
resp.SkippedFiles++
continue
}
p := filepath.Join(full, name)
fi, err := os.Stat(p)
if err != nil || fi.IsDir() || fi.Size() <= 0 {
resp.SkippedFiles++
continue
}
candidates = append(candidates, cleanupCandidate{
path: p,
name: name,
size: fi.Size(),
})
}
continue
}
name := e.Name()
if !isCandidate(name) {
resp.SkippedFiles++
continue
}
fi, err := os.Stat(full)
if err != nil || fi.IsDir() || fi.Size() <= 0 {
resp.SkippedFiles++
continue
}
candidates = append(candidates, cleanupCandidate{
path: full,
name: name,
size: fi.Size(),
})
}
return nil
}
if err := collectCandidates(doneAbs, true); err != nil {
return err
}
resp.ScannedFiles = len(candidates)
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Queued = false
st.Running = true
if st.StartedAt.IsZero() {
st.StartedAt = time.Now()
}
st.Done = 0
st.Total = resp.ScannedFiles
st.CurrentFile = ""
st.Text = cleanupProgressText(*resp)
st.Error = ""
st.FinishedAt = nil
})
for i, c := range candidates {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
deleteReason := ""
if threshold > 0 && c.size < threshold {
deleteReason = "small"
} else if deleteLowRated {
stars, ok, err := ratingStarsForVideoPath(c.path)
if err != nil {
appLogln("⚠️ cleanup rating check failed:", filepath.Base(c.path), err)
} else if ok && stars <= maxStars {
deleteReason = "rating"
appLogf(
"🧹 cleanup deleting low-rated video: %s stars=%d threshold<=%d",
filepath.Base(c.path),
stars,
maxStars,
)
}
}
if deleteReason == "" {
broken, why := cleanupVideoLooksBroken(ctx, c.path)
if broken {
deleteReason = "broken"
appLogln("🧹 cleanup deleting broken video:", filepath.Base(c.path), why)
}
}
if deleteReason != "" {
deletedBytes, derr := deleteDoneVideoAndGenerated(c.path)
if derr == nil {
resp.DeletedFiles++
if deletedBytes > 0 {
resp.DeletedBytes += deletedBytes
} else {
resp.DeletedBytes += c.size
}
switch deleteReason {
case "broken":
resp.DeletedBrokenFiles++
case "rating":
resp.DeletedLowRatedFiles++
}
} else {
resp.ErrorCount++
}
}
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
st.Queued = false
st.Running = true
if st.StartedAt.IsZero() {
st.StartedAt = time.Now()
}
st.Done = i + 1
st.Total = resp.ScannedFiles
st.CurrentFile = c.name
st.Text = cleanupProgressText(*resp)
st.Error = ""
st.FinishedAt = nil
})
}
return nil
}
var generatedGCRunning int32
type generatedGCStats struct {
Checked int
Removed int
RemovedBytes int64
}
func dirSizeBytes(root string) int64 {
root = strings.TrimSpace(root)
if root == "" {
return 0
}
var total int64
_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
if d.IsDir() {
return nil
}
info, err := d.Info()
if err != nil {
return nil
}
if info.Size() > 0 {
total += info.Size()
}
return nil
})
return total
}
func removeDirIfEmpty(dir string) (bool, error) {
dir = strings.TrimSpace(dir)
if dir == "" {
return false, nil
}
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
if len(entries) > 0 {
return false, nil
}
if err := os.Remove(dir); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
// Läuft synchron und liefert Zahlen zurück (für /api/settings/cleanup Response).
func triggerGeneratedGarbageCollectorSync() generatedGCStats {
// nur 1 GC gleichzeitig
if !atomic.CompareAndSwapInt32(&generatedGCRunning, 0, 1) {
appLogln("🧹 [gc] skip: already running")
return generatedGCStats{}
}
defer atomic.StoreInt32(&generatedGCRunning, 0)
stats := runGeneratedGarbageCollector()
return stats
}
// Läuft 1× nach Serverstart (mit Delay), löscht /generated/* Orphans.
func startGeneratedGarbageCollector() {
go func() {
time.Sleep(3 * time.Second)
triggerGeneratedGarbageCollectorSync()
}()
}
func collectLiveIDsFromTrash(trashDir string, live map[string]struct{}) {
trashDir = strings.TrimSpace(trashDir)
if trashDir == "" || live == nil {
return
}
entries, err := os.ReadDir(trashDir)
if err != nil {
return
}
for _, e := range entries {
if e == nil || e.IsDir() {
continue
}
name := strings.TrimSpace(e.Name())
if name == "" {
continue
}
// last.json ignorieren
if strings.EqualFold(name, "last.json") {
continue
}
// Format in .trash:
// <token>__<original-file>
parts := strings.SplitN(name, "__", 2)
if len(parts) != 2 {
continue
}
originalFile := strings.TrimSpace(parts[1])
if originalFile == "" {
continue
}
ext := strings.ToLower(filepath.Ext(originalFile))
if ext != ".mp4" && ext != ".ts" {
continue
}
full := filepath.Join(trashDir, name)
fi, err := os.Stat(full)
if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
continue
}
id := stripHotPrefix(strings.TrimSuffix(filepath.Base(originalFile), ext))
id = strings.TrimSpace(id)
if id == "" {
continue
}
if sid, err := sanitizeID(id); err == nil && sid != "" {
live[sid] = struct{}{}
}
}
}
// Core-Logik ohne Delay (für manuelle Trigger, z.B. nach Cleanup)
// Liefert Stats zurück, damit /api/settings/cleanup die Zahlen anzeigen kann.
func runGeneratedGarbageCollector() generatedGCStats {
stats := generatedGCStats{}
s := getSettings()
doneAbs, err := resolvePathRelativeToApp(s.DoneDir)
if err != nil {
appLogln("🧹 [gc] resolve doneDir failed:", err)
return stats
}
doneAbs = strings.TrimSpace(doneAbs)
if doneAbs == "" {
return stats
}
// 1) Live-IDs sammeln: alle mp4/ts unter /done (rekursiv), .trash zunächst ignorieren
live := make(map[string]struct{}, 4096)
_ = filepath.WalkDir(doneAbs, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
name := d.Name()
if d.IsDir() {
if strings.EqualFold(name, ".trash") {
return fs.SkipDir
}
return nil
}
ext := strings.ToLower(filepath.Ext(name))
if ext != ".mp4" && ext != ".ts" {
return nil
}
info, err := d.Info()
if err != nil || info.IsDir() || info.Size() <= 0 {
return nil
}
base := strings.TrimSuffix(name, ext)
id, err := sanitizeID(stripHotPrefix(base))
if err != nil || id == "" {
return nil
}
live[id] = struct{}{}
return nil
})
// 1a) Dateien in .trash ebenfalls als "live" behandeln,
// damit ihre Generated-Dateien erst gelöscht werden,
// wenn sie wirklich endgültig aus .trash verschwinden.
collectLiveIDsFromTrash(filepath.Join(doneAbs, ".trash"), live)
// 1b) Auch aktuell laufende / nachbearbeitete Jobs als "live" behandeln.
// Zusätzlich separat merken, damit leere Ordner aktiver Jobs nicht gelöscht werden.
activeJobIDs := make(map[string]struct{}, 128)
jobsMu.RLock()
for _, j := range jobs {
if j == nil {
continue
}
id := strings.TrimSpace(assetIDForJob(j))
if id == "" {
continue
}
live[id] = struct{}{}
activeJobIDs[id] = struct{}{}
}
jobsMu.RUnlock()
// 2) /generated/meta/<id> prüfen
metaRoot, err := generatedMetaRoot()
if err == nil {
metaRoot = strings.TrimSpace(metaRoot)
}
if err != nil || metaRoot == "" {
return stats
}
removedMeta := 0
checkedMeta := 0
removedBytes := int64(0)
if entries, err := os.ReadDir(metaRoot); err == nil {
for _, e := range entries {
if !e.IsDir() {
continue
}
id := strings.TrimSpace(e.Name())
if id == "" || strings.HasPrefix(id, ".") {
continue
}
checkedMeta++
orphanDir := filepath.Join(metaRoot, id)
// Leere generated/meta/<id>-Ordner immer entfernen,
// außer ein aktiver Job könnte gerade hinein schreiben.
if _, active := activeJobIDs[id]; !active {
if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty {
removedMeta++
continue
}
}
if _, ok := live[id]; ok {
continue
}
orphanBytes := dirSizeBytes(orphanDir)
removeGeneratedForID(id)
// Falls removeGeneratedForID nur Dateien gelöscht hat, den nun leeren Ordner entfernen.
if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty {
// ok
}
if _, err := os.Stat(orphanDir); os.IsNotExist(err) {
removedMeta++
removedBytes += orphanBytes
}
}
}
//fmt.Printf("🧹 [gc] generated/meta checked=%d removed_orphans=%d bytes=%d\n", checkedMeta, removedMeta, removedBytes)
stats.Checked += checkedMeta
stats.Removed += removedMeta
stats.RemovedBytes += removedBytes
return stats
}