1308 lines
29 KiB
Go
1308 lines
29 KiB
Go
// backend\tasks_cleanup.go
|
||
|
||
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io/fs"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
)
|
||
|
||
type cleanupCandidate struct {
|
||
path string
|
||
name string
|
||
size int64
|
||
}
|
||
|
||
type cleanupResp struct {
|
||
// Small / broken downloads cleanup
|
||
ScannedFiles int `json:"scannedFiles"`
|
||
DeletedFiles int `json:"deletedFiles"`
|
||
DeletedBrokenFiles int `json:"deletedBrokenFiles"`
|
||
DeletedLowRatedFiles int `json:"deletedLowRatedFiles"`
|
||
SkippedFiles int `json:"skippedFiles"`
|
||
DeletedBytes int64 `json:"deletedBytes"`
|
||
DeletedBytesHuman string `json:"deletedBytesHuman"`
|
||
DeletedRecordOrphans int `json:"deletedRecordOrphans"`
|
||
DeletedRecordOrphanBytes int64 `json:"deletedRecordOrphanBytes"`
|
||
|
||
DeletedMuxingTemps int `json:"deletedMuxingTemps"`
|
||
DeletedAudioOrphans int `json:"deletedAudioOrphans"`
|
||
DeletedTSOrphans int `json:"deletedTSOrphans"`
|
||
|
||
ErrorCount int `json:"errorCount"`
|
||
|
||
// Generated-GC
|
||
GeneratedOrphansChecked int `json:"generatedOrphansChecked"`
|
||
GeneratedOrphansRemoved int `json:"generatedOrphansRemoved"`
|
||
GeneratedOrphansRemovedBytes int64 `json:"generatedOrphansRemovedBytes"`
|
||
GeneratedOrphansRemovedBytesHuman string `json:"generatedOrphansRemovedBytesHuman"`
|
||
}
|
||
|
||
// Optional: falls du später Threshold per Body überschreiben willst.
|
||
// Frontend sendet aktuell nichts -> wir nutzen Settings.
|
||
type cleanupReq struct {
|
||
BelowMB *int `json:"belowMB,omitempty"`
|
||
}
|
||
|
||
type cleanupTaskJob struct {
|
||
State CleanupTaskState
|
||
Cancel context.CancelFunc
|
||
}
|
||
|
||
var cleanupJobsMu sync.Mutex
|
||
var cleanupJobs = map[string]*cleanupTaskJob{}
|
||
|
||
func snapshotCleanupJobs() []CleanupTaskState {
|
||
cleanupJobsMu.Lock()
|
||
defer cleanupJobsMu.Unlock()
|
||
|
||
now := time.Now()
|
||
out := make([]CleanupTaskState, 0, len(cleanupJobs))
|
||
|
||
for id, job := range cleanupJobs {
|
||
if job == nil {
|
||
delete(cleanupJobs, id)
|
||
continue
|
||
}
|
||
|
||
st := job.State
|
||
if !st.Queued && !st.Running && st.FinishedAt != nil && now.Sub(*st.FinishedAt) > 4*time.Second {
|
||
delete(cleanupJobs, id)
|
||
continue
|
||
}
|
||
|
||
out = append(out, st)
|
||
}
|
||
|
||
return out
|
||
}
|
||
|
||
func updateCleanupJobState(jobID string, fn func(st *CleanupTaskState)) (CleanupTaskState, bool) {
|
||
cleanupJobsMu.Lock()
|
||
job := cleanupJobs[jobID]
|
||
if job == nil {
|
||
cleanupJobsMu.Unlock()
|
||
return CleanupTaskState{}, false
|
||
}
|
||
|
||
fn(&job.State)
|
||
st := job.State
|
||
cleanupJobsMu.Unlock()
|
||
|
||
publishTaskState()
|
||
return st, true
|
||
}
|
||
|
||
func removeCleanupJob(jobID string) {
|
||
cleanupJobsMu.Lock()
|
||
delete(cleanupJobs, jobID)
|
||
cleanupJobsMu.Unlock()
|
||
|
||
publishTaskState()
|
||
}
|
||
|
||
func finishCleanupJob(jobID string, text string, err error) {
|
||
now := time.Now()
|
||
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Queued = false
|
||
st.Running = false
|
||
st.FinishedAt = &now
|
||
st.CurrentFile = ""
|
||
|
||
if err == nil {
|
||
st.Error = ""
|
||
st.Text = strings.TrimSpace(text)
|
||
if st.Total > 0 {
|
||
st.Done = st.Total
|
||
}
|
||
return
|
||
}
|
||
|
||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||
st.Error = "Abgebrochen"
|
||
st.Text = "Abgebrochen."
|
||
} else {
|
||
st.Error = strings.TrimSpace(err.Error())
|
||
st.Text = ""
|
||
}
|
||
})
|
||
|
||
time.AfterFunc(4*time.Second, func() {
|
||
removeCleanupJob(jobID)
|
||
})
|
||
}
|
||
|
||
// /api/settings/cleanup (POST)
|
||
// - löscht kleine Dateien < threshold MB (mp4/ts; skip .part/.tmp; skip keep-Ordner)
|
||
// - räumt Orphans (preview/thumbs + generated) auf
|
||
func settingsCleanupHandler(w http.ResponseWriter, r *http.Request) {
|
||
switch r.Method {
|
||
case http.MethodPost:
|
||
s := getSettings()
|
||
|
||
doneAbs, err := resolvePathRelativeToApp(s.DoneDir)
|
||
if err != nil || strings.TrimSpace(doneAbs) == "" {
|
||
http.Error(w, "doneDir auflösung fehlgeschlagen", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
recordAbs, err := resolvePathRelativeToApp(s.RecordDir)
|
||
if err != nil || strings.TrimSpace(recordAbs) == "" {
|
||
recordAbs = strings.TrimSpace(s.RecordDir)
|
||
}
|
||
|
||
mb := int(s.AutoDeleteSmallDownloadsBelowMB)
|
||
deleteLowRated := s.AutoDeleteLowRatedDownloads
|
||
maxStars := clampAutoDeleteRatingStars(s.AutoDeleteLowRatedDownloadsMaxStars)
|
||
|
||
var req cleanupReq
|
||
if r.Body != nil {
|
||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||
}
|
||
if req.BelowMB != nil {
|
||
mb = *req.BelowMB
|
||
}
|
||
if mb < 0 {
|
||
mb = 0
|
||
}
|
||
|
||
jobID := newTaskID("cleanup")
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
|
||
st := CleanupTaskState{
|
||
ID: jobID,
|
||
Queued: true,
|
||
Running: false,
|
||
QueuedAt: time.Now(),
|
||
StartedAt: time.Time{},
|
||
FinishedAt: nil,
|
||
Text: "Wartet…",
|
||
Error: "",
|
||
CurrentFile: "",
|
||
Done: 0,
|
||
Total: 0,
|
||
}
|
||
|
||
cleanupJobsMu.Lock()
|
||
cleanupJobs[jobID] = &cleanupTaskJob{
|
||
State: st,
|
||
Cancel: cancel,
|
||
}
|
||
cleanupJobsMu.Unlock()
|
||
|
||
publishTaskState()
|
||
|
||
go func(jobID string, ctx context.Context, doneAbs string, recordAbs string, mb int, deleteLowRated bool, maxStars int) {
|
||
if err := acquireExclusiveTask(ctx); err != nil {
|
||
finishCleanupJob(jobID, "", err)
|
||
return
|
||
}
|
||
defer releaseExclusiveTask()
|
||
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Queued = false
|
||
st.Running = true
|
||
st.StartedAt = time.Now()
|
||
st.Text = "Räume auf…"
|
||
})
|
||
|
||
resp := cleanupResp{}
|
||
|
||
if mb > 0 || deleteLowRated {
|
||
threshold := int64(mb) * 1024 * 1024
|
||
if err := cleanupSmallFiles(ctx, jobID, doneAbs, threshold, deleteLowRated, maxStars, &resp); err != nil {
|
||
finishCleanupJob(jobID, "", err)
|
||
return
|
||
}
|
||
}
|
||
|
||
// Neuer Cleanup-Abschnitt: alten 100%-Fortschritt vom Small-Files-Scan zurücksetzen.
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Queued = false
|
||
st.Running = true
|
||
if st.StartedAt.IsZero() {
|
||
st.StartedAt = time.Now()
|
||
}
|
||
st.Done = 0
|
||
st.Total = 0
|
||
st.CurrentFile = ""
|
||
st.Text = "Räume Record-Reste auf…"
|
||
st.Error = ""
|
||
st.FinishedAt = nil
|
||
})
|
||
|
||
if err := cleanupRecordDirOrphanAVFiles(ctx, jobID, recordAbs, &resp); err != nil {
|
||
finishCleanupJob(jobID, "", err)
|
||
return
|
||
}
|
||
|
||
select {
|
||
case <-ctx.Done():
|
||
finishCleanupJob(jobID, "", ctx.Err())
|
||
return
|
||
default:
|
||
}
|
||
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Queued = false
|
||
st.Running = true
|
||
st.Done = 0
|
||
st.Total = 0
|
||
st.CurrentFile = ""
|
||
st.Text = "Räume Generated-Assets auf…"
|
||
st.Error = ""
|
||
st.FinishedAt = nil
|
||
})
|
||
|
||
gcStats := triggerGeneratedGarbageCollectorSync()
|
||
|
||
resp.GeneratedOrphansChecked = gcStats.Checked
|
||
resp.GeneratedOrphansRemoved = gcStats.Removed
|
||
resp.GeneratedOrphansRemovedBytes = gcStats.RemovedBytes
|
||
resp.GeneratedOrphansRemovedBytesHuman = formatBytesSI(gcStats.RemovedBytes)
|
||
|
||
resp.DeletedBytes += gcStats.RemovedBytes
|
||
resp.DeletedBytesHuman = formatBytesSI(resp.DeletedBytes)
|
||
|
||
if resp.DeletedFiles > 0 || resp.GeneratedOrphansRemoved > 0 {
|
||
notifyDoneChanged()
|
||
}
|
||
|
||
finishCleanupJob(
|
||
jobID,
|
||
cleanupProgressText(resp),
|
||
nil,
|
||
)
|
||
}(jobID, ctx, doneAbs, recordAbs, mb, deleteLowRated, maxStars)
|
||
|
||
writeJSON(w, http.StatusOK, st)
|
||
return
|
||
|
||
case http.MethodDelete:
|
||
id := strings.TrimSpace(r.URL.Query().Get("id"))
|
||
if id == "" {
|
||
http.Error(w, "id fehlt", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
cleanupJobsMu.Lock()
|
||
job := cleanupJobs[id]
|
||
cleanupJobsMu.Unlock()
|
||
|
||
if job != nil && job.Cancel != nil {
|
||
job.Cancel()
|
||
}
|
||
|
||
w.WriteHeader(http.StatusNoContent)
|
||
return
|
||
|
||
default:
|
||
http.Error(w, "Nur POST/DELETE erlaubt", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
}
|
||
|
||
func cleanupProgressText(resp cleanupResp) string {
|
||
parts := []string{
|
||
fmt.Sprintf("gelöscht: %d/%d", resp.DeletedFiles, resp.ScannedFiles),
|
||
}
|
||
|
||
if resp.DeletedBrokenFiles > 0 {
|
||
parts = append(parts, fmt.Sprintf("defekt: %d", resp.DeletedBrokenFiles))
|
||
}
|
||
|
||
if resp.DeletedLowRatedFiles > 0 {
|
||
parts = append(parts, fmt.Sprintf("Rating: %d", resp.DeletedLowRatedFiles))
|
||
}
|
||
|
||
if resp.DeletedRecordOrphans > 0 {
|
||
parts = append(parts, fmt.Sprintf("Records: %d", resp.DeletedRecordOrphans))
|
||
}
|
||
|
||
if resp.DeletedTSOrphans > 0 {
|
||
parts = append(parts, fmt.Sprintf("TS ohne Audio: %d", resp.DeletedTSOrphans))
|
||
}
|
||
|
||
if resp.DeletedAudioOrphans > 0 {
|
||
parts = append(parts, fmt.Sprintf("Audio ohne TS: %d", resp.DeletedAudioOrphans))
|
||
}
|
||
|
||
if resp.DeletedMuxingTemps > 0 {
|
||
parts = append(parts, fmt.Sprintf("Muxing-Temp: %d", resp.DeletedMuxingTemps))
|
||
}
|
||
|
||
if resp.GeneratedOrphansRemoved > 0 {
|
||
parts = append(parts, fmt.Sprintf("Generated: %d", resp.GeneratedOrphansRemoved))
|
||
}
|
||
|
||
if resp.ErrorCount > 0 {
|
||
parts = append(parts, fmt.Sprintf("Fehler: %d", resp.ErrorCount))
|
||
}
|
||
|
||
parts = append(parts, fmt.Sprintf("freigegeben: %s", formatBytesSI(resp.DeletedBytes)))
|
||
|
||
return strings.Join(parts, " · ")
|
||
}
|
||
|
||
func cleanupVideoLooksBroken(ctx context.Context, path string) (bool, string) {
|
||
path = strings.TrimSpace(path)
|
||
if path == "" {
|
||
return false, ""
|
||
}
|
||
|
||
ext := strings.ToLower(filepath.Ext(path))
|
||
if ext != ".mp4" && ext != ".ts" {
|
||
return false, ""
|
||
}
|
||
|
||
fi, err := os.Stat(path)
|
||
if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
|
||
return true, "file missing/empty"
|
||
}
|
||
|
||
// Erst ffprobe/duration: wenn keine Dauer lesbar ist, ist die Datei meistens kaputt.
|
||
dctx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
||
dur, derr := durationSecondsCached(dctx, path)
|
||
dctxErr := dctx.Err()
|
||
cancel()
|
||
|
||
if errors.Is(dctxErr, context.DeadlineExceeded) || errors.Is(dctxErr, context.Canceled) {
|
||
// Timeout beim Prüfen nicht als "kaputt" werten, um keine guten Dateien wegen Last zu löschen.
|
||
return false, "duration check timeout"
|
||
}
|
||
|
||
if derr != nil || dur <= 0 {
|
||
if derr != nil {
|
||
return true, "duration failed: " + derr.Error()
|
||
}
|
||
return true, "duration <= 0"
|
||
}
|
||
|
||
// Danach konservative Decode-Prüfung am Ende.
|
||
// Diese Funktion nutzt du bereits nach dem Remux in recorder.go.
|
||
vctx, vcancel := context.WithTimeout(ctx, 20*time.Second)
|
||
verr := validateVideoDecodesNearEnd(vctx, path)
|
||
vctxErr := vctx.Err()
|
||
vcancel()
|
||
|
||
if errors.Is(vctxErr, context.DeadlineExceeded) || errors.Is(vctxErr, context.Canceled) {
|
||
return false, "decode check timeout"
|
||
}
|
||
|
||
if verr != nil {
|
||
return true, "decode failed: " + verr.Error()
|
||
}
|
||
|
||
return false, ""
|
||
}
|
||
|
||
const cleanupRecordOrphanMinAge = 60 * time.Second
|
||
|
||
func cleanupRecordFileIsActive(path string) bool {
|
||
path = filepath.Clean(strings.TrimSpace(path))
|
||
if path == "" {
|
||
return false
|
||
}
|
||
|
||
jobsMu.RLock()
|
||
defer jobsMu.RUnlock()
|
||
|
||
for _, j := range jobs {
|
||
if j == nil {
|
||
continue
|
||
}
|
||
|
||
out := filepath.Clean(strings.TrimSpace(j.Output))
|
||
if out != "" && strings.EqualFold(out, path) {
|
||
return true
|
||
}
|
||
|
||
for _, sidecar := range postworkAudioSidecarCandidatesForOutput(out) {
|
||
if sidecar != "" && strings.EqualFold(filepath.Clean(sidecar), path) {
|
||
return true
|
||
}
|
||
}
|
||
}
|
||
|
||
return false
|
||
}
|
||
|
||
func cleanupFileStable(path string) bool {
|
||
path = strings.TrimSpace(path)
|
||
if path == "" {
|
||
return false
|
||
}
|
||
|
||
fi1, err := os.Stat(path)
|
||
if err != nil || fi1 == nil || fi1.IsDir() || fi1.Size() <= 0 {
|
||
return false
|
||
}
|
||
|
||
time.Sleep(1200 * time.Millisecond)
|
||
|
||
fi2, err := os.Stat(path)
|
||
if err != nil || fi2 == nil || fi2.IsDir() || fi2.Size() <= 0 {
|
||
return false
|
||
}
|
||
|
||
return fi1.Size() == fi2.Size()
|
||
}
|
||
|
||
func cleanupTSPathForAudioSidecar(audioPath string) string {
|
||
audioPath = strings.TrimSpace(audioPath)
|
||
if audioPath == "" {
|
||
return ""
|
||
}
|
||
|
||
dir := filepath.Dir(audioPath)
|
||
name := filepath.Base(audioPath)
|
||
|
||
if !strings.HasPrefix(name, ".") {
|
||
return ""
|
||
}
|
||
if !strings.HasSuffix(strings.ToLower(name), ".audio.m4s") {
|
||
return ""
|
||
}
|
||
|
||
base := strings.TrimPrefix(name, ".")
|
||
base = strings.TrimSuffix(base, ".audio.m4s")
|
||
|
||
if strings.TrimSpace(base) == "" {
|
||
return ""
|
||
}
|
||
|
||
return filepath.Join(dir, base+".ts")
|
||
}
|
||
|
||
func cleanupDeleteRecordOrphanFile(ctx context.Context, jobID string, path string, resp *cleanupResp, reason string) {
|
||
if resp == nil {
|
||
return
|
||
}
|
||
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
default:
|
||
}
|
||
|
||
path = strings.TrimSpace(path)
|
||
if path == "" {
|
||
return
|
||
}
|
||
|
||
fi, err := os.Stat(path)
|
||
if err != nil || fi == nil || fi.IsDir() {
|
||
return
|
||
}
|
||
|
||
size := fi.Size()
|
||
name := filepath.Base(path)
|
||
lowName := strings.ToLower(name)
|
||
|
||
if time.Since(fi.ModTime()) < cleanupRecordOrphanMinAge {
|
||
resp.SkippedFiles++
|
||
return
|
||
}
|
||
|
||
if cleanupRecordFileIsActive(path) {
|
||
resp.SkippedFiles++
|
||
return
|
||
}
|
||
|
||
// 0-KB-Dateien sind stabil genug zum Löschen.
|
||
// Für >0 Bytes weiter die Stabilitätsprüfung behalten.
|
||
if size > 0 && !cleanupFileStable(path) {
|
||
resp.SkippedFiles++
|
||
return
|
||
}
|
||
|
||
// Fortschritt (Done/Total/CurrentFile) steuert der Aufrufer
|
||
// cleanupRecordDirOrphanAVFiles, damit die Progress-Bar sichtbar bleibt.
|
||
|
||
if err := removeWithRetry(path); err != nil && !os.IsNotExist(err) {
|
||
resp.ErrorCount++
|
||
appLogln("⚠️ cleanup record orphan konnte nicht gelöscht werden:", name, reason, err)
|
||
return
|
||
}
|
||
|
||
resp.DeletedFiles++
|
||
resp.DeletedRecordOrphans++
|
||
resp.DeletedBytes += size
|
||
resp.DeletedRecordOrphanBytes += size
|
||
|
||
if strings.HasSuffix(lowName, ".muxing.ts") {
|
||
resp.DeletedMuxingTemps++
|
||
} else if strings.HasPrefix(name, ".") && strings.HasSuffix(lowName, ".audio.m4s") {
|
||
resp.DeletedAudioOrphans++
|
||
} else if !strings.HasPrefix(name, ".") && strings.EqualFold(filepath.Ext(name), ".ts") {
|
||
resp.DeletedTSOrphans++
|
||
}
|
||
|
||
purgeDurationCacheForPath(path)
|
||
|
||
base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
||
base = strings.TrimPrefix(base, ".")
|
||
base = strings.TrimSuffix(base, ".audio")
|
||
base = strings.TrimSuffix(base, ".muxing")
|
||
|
||
if strings.TrimSpace(base) != "" {
|
||
removeGeneratedForID(stripHotPrefix(base))
|
||
}
|
||
|
||
appLogln("🧹 cleanup deleted record orphan:", name, "reason:", reason, "size=", size)
|
||
}
|
||
|
||
func cleanupRecordDirOrphanAVFiles(ctx context.Context, jobID string, recordAbs string, resp *cleanupResp) error {
|
||
recordAbs = strings.TrimSpace(recordAbs)
|
||
if recordAbs == "" || resp == nil {
|
||
return nil
|
||
}
|
||
|
||
entries, err := os.ReadDir(recordAbs)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
|
||
// Gesamtzahl der zu prüfenden Dateien für einen sichtbaren Fortschritt.
|
||
total := 0
|
||
for _, e := range entries {
|
||
if e == nil || e.IsDir() {
|
||
continue
|
||
}
|
||
if strings.TrimSpace(e.Name()) == "" {
|
||
continue
|
||
}
|
||
total++
|
||
}
|
||
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Queued = false
|
||
st.Running = true
|
||
if st.StartedAt.IsZero() {
|
||
st.StartedAt = time.Now()
|
||
}
|
||
st.Done = 0
|
||
st.Total = total
|
||
st.CurrentFile = ""
|
||
st.Text = "Räume Record-Reste auf…"
|
||
st.Error = ""
|
||
st.FinishedAt = nil
|
||
})
|
||
|
||
processed := 0
|
||
|
||
for _, e := range entries {
|
||
select {
|
||
case <-ctx.Done():
|
||
return ctx.Err()
|
||
default:
|
||
}
|
||
|
||
if e == nil || e.IsDir() {
|
||
continue
|
||
}
|
||
|
||
name := strings.TrimSpace(e.Name())
|
||
if name == "" {
|
||
continue
|
||
}
|
||
|
||
processed++
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Done = processed
|
||
st.Total = total
|
||
st.CurrentFile = name
|
||
})
|
||
|
||
full := filepath.Join(recordAbs, name)
|
||
low := strings.ToLower(name)
|
||
|
||
// Fall 0:
|
||
// alte .muxing.ts sind temporäre Remux-/Muxing-Reste.
|
||
if strings.HasSuffix(low, ".muxing.ts") {
|
||
resp.ScannedFiles++
|
||
cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "stale muxing temp")
|
||
continue
|
||
}
|
||
|
||
// Fall 1:
|
||
// foo.ts existiert, aber .foo.audio.m4s fehlt.
|
||
if !strings.HasPrefix(name, ".") && strings.EqualFold(filepath.Ext(name), ".ts") {
|
||
resp.ScannedFiles++
|
||
|
||
if _, ok := findPostworkAudioSidecarForTS(full); ok {
|
||
continue
|
||
}
|
||
|
||
cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "missing audio sidecar")
|
||
continue
|
||
}
|
||
|
||
// Fall 2:
|
||
// .foo.audio.m4s existiert, aber foo.ts fehlt.
|
||
if strings.HasPrefix(name, ".") && strings.HasSuffix(low, ".audio.m4s") {
|
||
resp.ScannedFiles++
|
||
|
||
tsPath := cleanupTSPathForAudioSidecar(full)
|
||
if strings.TrimSpace(tsPath) != "" {
|
||
if fi, err := os.Stat(tsPath); err == nil && fi != nil && !fi.IsDir() && fi.Size() > 0 {
|
||
continue
|
||
}
|
||
}
|
||
|
||
cleanupDeleteRecordOrphanFile(ctx, jobID, full, resp, "missing ts file")
|
||
continue
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
func clampAutoDeleteRatingStars(v int) int {
|
||
if v < 1 {
|
||
return 3
|
||
}
|
||
if v > 5 {
|
||
return 5
|
||
}
|
||
return v
|
||
}
|
||
|
||
func ratingStarsForVideoPath(videoPath string) (int, bool, error) {
|
||
videoPath = strings.TrimSpace(videoPath)
|
||
if videoPath == "" {
|
||
return 0, false, nil
|
||
}
|
||
|
||
base := strings.TrimSuffix(filepath.Base(videoPath), filepath.Ext(videoPath))
|
||
assetID := stripHotPrefix(base)
|
||
if strings.TrimSpace(assetID) == "" {
|
||
return 0, false, nil
|
||
}
|
||
|
||
metaPath, err := generatedMetaFile(assetID)
|
||
if err != nil {
|
||
return 0, false, err
|
||
}
|
||
|
||
m, ok := readVideoMetaLoose(metaPath)
|
||
if !ok || m == nil {
|
||
return 0, false, nil
|
||
}
|
||
|
||
ai := pickAnalysisAIForGoal(m.Analysis, "highlights")
|
||
if ai == nil || ai.Rating == nil {
|
||
return 0, false, nil
|
||
}
|
||
|
||
stars := ai.Rating.Stars
|
||
if stars <= 0 && ai.Rating.Score > 0 {
|
||
stars = starsFromHighlightScore(ai.Rating.Score)
|
||
}
|
||
if stars < 1 || stars > 5 {
|
||
return 0, false, nil
|
||
}
|
||
|
||
return stars, true, nil
|
||
}
|
||
|
||
func deleteDoneVideoAndGenerated(videoPath string) (int64, error) {
|
||
videoPath = filepath.Clean(strings.TrimSpace(videoPath))
|
||
if videoPath == "" {
|
||
return 0, nil
|
||
}
|
||
|
||
ext := strings.ToLower(filepath.Ext(videoPath))
|
||
if ext != ".mp4" && ext != ".ts" {
|
||
return 0, nil
|
||
}
|
||
|
||
fi, err := os.Stat(videoPath)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
return 0, nil
|
||
}
|
||
return 0, err
|
||
}
|
||
if fi == nil || fi.IsDir() {
|
||
return 0, nil
|
||
}
|
||
|
||
name := filepath.Base(videoPath)
|
||
base := strings.TrimSuffix(name, filepath.Ext(name))
|
||
assetID := stripHotPrefix(base)
|
||
|
||
// Stoppt ggf. laufende Asset-/Enrich-Tasks für genau diese Datei,
|
||
// damit Windows-Dateihandles nicht blockieren.
|
||
releaseFileForMutation(name)
|
||
|
||
if err := removeWithRetry(videoPath); err != nil && !os.IsNotExist(err) {
|
||
return 0, err
|
||
}
|
||
|
||
if strings.TrimSpace(assetID) != "" {
|
||
removeGeneratedForID(assetID)
|
||
}
|
||
|
||
purgeDurationCacheForPath(videoPath)
|
||
return fi.Size(), nil
|
||
}
|
||
|
||
func autoDeleteLowRatedDownloadAfterAnalysis(ctx context.Context, videoPath string, rating *aiRatingMeta) {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
default:
|
||
}
|
||
|
||
videoPath = strings.TrimSpace(videoPath)
|
||
if videoPath == "" {
|
||
return
|
||
}
|
||
|
||
s := getSettings()
|
||
if !s.AutoDeleteLowRatedDownloads {
|
||
return
|
||
}
|
||
|
||
maxStars := clampAutoDeleteRatingStars(s.AutoDeleteLowRatedDownloadsMaxStars)
|
||
|
||
stars := 0
|
||
|
||
if rating != nil {
|
||
stars = rating.Stars
|
||
if stars <= 0 && rating.Score > 0 {
|
||
stars = starsFromHighlightScore(rating.Score)
|
||
}
|
||
}
|
||
|
||
// Wichtig für Background-/Regenerate-Analyse:
|
||
// Dort wird rating teilweise als nil übergeben, obwohl es kurz vorher
|
||
// in meta.json gespeichert wurde. Dann lesen wir die Sterne aus der Meta.
|
||
if stars <= 0 {
|
||
if metaStars, ok, err := ratingStarsForVideoPath(videoPath); err != nil {
|
||
appLogln("⚠️ [rating-delete] rating lookup failed:", filepath.Base(videoPath), err)
|
||
} else if ok {
|
||
stars = metaStars
|
||
}
|
||
}
|
||
|
||
if stars < 1 || stars > 5 {
|
||
appLogf(
|
||
"ℹ️ [rating-delete] übersprungen, kein gültiges Rating: %s stars=%d threshold<=%d",
|
||
filepath.Base(videoPath),
|
||
stars,
|
||
maxStars,
|
||
)
|
||
return
|
||
}
|
||
|
||
if stars > maxStars {
|
||
appLogf(
|
||
"ℹ️ [rating-delete] behalten: %s stars=%d threshold<=%d",
|
||
filepath.Base(videoPath),
|
||
stars,
|
||
maxStars,
|
||
)
|
||
return
|
||
}
|
||
|
||
deletedBytes, err := deleteDoneVideoAndGenerated(videoPath)
|
||
if err != nil {
|
||
appLogln("⚠️ [rating-delete] konnte Download nicht löschen:", filepath.Base(videoPath), err)
|
||
return
|
||
}
|
||
|
||
appLogf(
|
||
"🗑️ [rating-delete] gelöscht nach Analyse: %s stars=%d threshold<=%d bytes=%s",
|
||
filepath.Base(videoPath),
|
||
stars,
|
||
maxStars,
|
||
formatBytesSI(deletedBytes),
|
||
)
|
||
|
||
notifyDoneChanged()
|
||
}
|
||
|
||
func cleanupSmallFiles(ctx context.Context, jobID string, doneAbs string, threshold int64, deleteLowRated bool, maxStars int, resp *cleanupResp) error {
|
||
isCandidate := func(name string) bool {
|
||
low := strings.ToLower(name)
|
||
if strings.Contains(low, ".part") || strings.Contains(low, ".tmp") {
|
||
return false
|
||
}
|
||
ext := strings.ToLower(filepath.Ext(name))
|
||
return ext == ".mp4" || ext == ".ts"
|
||
}
|
||
|
||
candidates := make([]cleanupCandidate, 0, 1024)
|
||
|
||
collectCandidates := func(dir string, allowSubdirs bool) error {
|
||
ents, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
|
||
for _, e := range ents {
|
||
select {
|
||
case <-ctx.Done():
|
||
return ctx.Err()
|
||
default:
|
||
}
|
||
|
||
full := filepath.Join(dir, e.Name())
|
||
|
||
if e.IsDir() {
|
||
if !allowSubdirs {
|
||
continue
|
||
}
|
||
if e.Name() == "keep" {
|
||
continue
|
||
}
|
||
|
||
sub, err := os.ReadDir(full)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
|
||
for _, se := range sub {
|
||
if se.IsDir() {
|
||
continue
|
||
}
|
||
|
||
name := se.Name()
|
||
if !isCandidate(name) {
|
||
resp.SkippedFiles++
|
||
continue
|
||
}
|
||
|
||
p := filepath.Join(full, name)
|
||
fi, err := os.Stat(p)
|
||
if err != nil || fi.IsDir() || fi.Size() <= 0 {
|
||
resp.SkippedFiles++
|
||
continue
|
||
}
|
||
|
||
candidates = append(candidates, cleanupCandidate{
|
||
path: p,
|
||
name: name,
|
||
size: fi.Size(),
|
||
})
|
||
}
|
||
continue
|
||
}
|
||
|
||
name := e.Name()
|
||
if !isCandidate(name) {
|
||
resp.SkippedFiles++
|
||
continue
|
||
}
|
||
|
||
fi, err := os.Stat(full)
|
||
if err != nil || fi.IsDir() || fi.Size() <= 0 {
|
||
resp.SkippedFiles++
|
||
continue
|
||
}
|
||
|
||
candidates = append(candidates, cleanupCandidate{
|
||
path: full,
|
||
name: name,
|
||
size: fi.Size(),
|
||
})
|
||
}
|
||
return nil
|
||
}
|
||
|
||
if err := collectCandidates(doneAbs, true); err != nil {
|
||
return err
|
||
}
|
||
|
||
resp.ScannedFiles = len(candidates)
|
||
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Queued = false
|
||
st.Running = true
|
||
if st.StartedAt.IsZero() {
|
||
st.StartedAt = time.Now()
|
||
}
|
||
st.Done = 0
|
||
st.Total = resp.ScannedFiles
|
||
st.CurrentFile = ""
|
||
st.Text = cleanupProgressText(*resp)
|
||
st.Error = ""
|
||
st.FinishedAt = nil
|
||
})
|
||
|
||
for i, c := range candidates {
|
||
select {
|
||
case <-ctx.Done():
|
||
return ctx.Err()
|
||
default:
|
||
}
|
||
deleteReason := ""
|
||
|
||
if threshold > 0 && c.size < threshold {
|
||
deleteReason = "small"
|
||
} else if deleteLowRated {
|
||
stars, ok, err := ratingStarsForVideoPath(c.path)
|
||
if err != nil {
|
||
appLogln("⚠️ cleanup rating check failed:", filepath.Base(c.path), err)
|
||
} else if ok && stars <= maxStars {
|
||
deleteReason = "rating"
|
||
appLogf(
|
||
"🧹 cleanup deleting low-rated video: %s stars=%d threshold<=%d",
|
||
filepath.Base(c.path),
|
||
stars,
|
||
maxStars,
|
||
)
|
||
}
|
||
}
|
||
|
||
if deleteReason == "" {
|
||
broken, why := cleanupVideoLooksBroken(ctx, c.path)
|
||
if broken {
|
||
deleteReason = "broken"
|
||
appLogln("🧹 cleanup deleting broken video:", filepath.Base(c.path), why)
|
||
}
|
||
}
|
||
|
||
if deleteReason != "" {
|
||
deletedBytes, derr := deleteDoneVideoAndGenerated(c.path)
|
||
if derr == nil {
|
||
resp.DeletedFiles++
|
||
if deletedBytes > 0 {
|
||
resp.DeletedBytes += deletedBytes
|
||
} else {
|
||
resp.DeletedBytes += c.size
|
||
}
|
||
|
||
switch deleteReason {
|
||
case "broken":
|
||
resp.DeletedBrokenFiles++
|
||
case "rating":
|
||
resp.DeletedLowRatedFiles++
|
||
}
|
||
} else {
|
||
resp.ErrorCount++
|
||
}
|
||
}
|
||
|
||
updateCleanupJobState(jobID, func(st *CleanupTaskState) {
|
||
st.Queued = false
|
||
st.Running = true
|
||
if st.StartedAt.IsZero() {
|
||
st.StartedAt = time.Now()
|
||
}
|
||
st.Done = i + 1
|
||
st.Total = resp.ScannedFiles
|
||
st.CurrentFile = c.name
|
||
st.Text = cleanupProgressText(*resp)
|
||
st.Error = ""
|
||
st.FinishedAt = nil
|
||
})
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
var generatedGCRunning int32
|
||
|
||
type generatedGCStats struct {
|
||
Checked int
|
||
Removed int
|
||
RemovedBytes int64
|
||
}
|
||
|
||
func dirSizeBytes(root string) int64 {
|
||
root = strings.TrimSpace(root)
|
||
if root == "" {
|
||
return 0
|
||
}
|
||
|
||
var total int64
|
||
|
||
_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
if d.IsDir() {
|
||
return nil
|
||
}
|
||
|
||
info, err := d.Info()
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
if info.Size() > 0 {
|
||
total += info.Size()
|
||
}
|
||
return nil
|
||
})
|
||
|
||
return total
|
||
}
|
||
|
||
func removeDirIfEmpty(dir string) (bool, error) {
|
||
dir = strings.TrimSpace(dir)
|
||
if dir == "" {
|
||
return false, nil
|
||
}
|
||
|
||
entries, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
return false, nil
|
||
}
|
||
return false, err
|
||
}
|
||
|
||
if len(entries) > 0 {
|
||
return false, nil
|
||
}
|
||
|
||
if err := os.Remove(dir); err != nil {
|
||
if os.IsNotExist(err) {
|
||
return false, nil
|
||
}
|
||
return false, err
|
||
}
|
||
|
||
return true, nil
|
||
}
|
||
|
||
// Läuft synchron und liefert Zahlen zurück (für /api/settings/cleanup Response).
|
||
func triggerGeneratedGarbageCollectorSync() generatedGCStats {
|
||
// nur 1 GC gleichzeitig
|
||
if !atomic.CompareAndSwapInt32(&generatedGCRunning, 0, 1) {
|
||
appLogln("🧹 [gc] skip: already running")
|
||
return generatedGCStats{}
|
||
}
|
||
defer atomic.StoreInt32(&generatedGCRunning, 0)
|
||
|
||
stats := runGeneratedGarbageCollector()
|
||
return stats
|
||
}
|
||
|
||
// Läuft 1× nach Serverstart (mit Delay), löscht /generated/* Orphans.
|
||
func startGeneratedGarbageCollector() {
|
||
go func() {
|
||
time.Sleep(3 * time.Second)
|
||
triggerGeneratedGarbageCollectorSync()
|
||
}()
|
||
}
|
||
|
||
func collectLiveIDsFromTrash(trashDir string, live map[string]struct{}) {
|
||
trashDir = strings.TrimSpace(trashDir)
|
||
if trashDir == "" || live == nil {
|
||
return
|
||
}
|
||
|
||
entries, err := os.ReadDir(trashDir)
|
||
if err != nil {
|
||
return
|
||
}
|
||
|
||
for _, e := range entries {
|
||
if e == nil || e.IsDir() {
|
||
continue
|
||
}
|
||
|
||
name := strings.TrimSpace(e.Name())
|
||
if name == "" {
|
||
continue
|
||
}
|
||
|
||
// last.json ignorieren
|
||
if strings.EqualFold(name, "last.json") {
|
||
continue
|
||
}
|
||
|
||
// Format in .trash:
|
||
// <token>__<original-file>
|
||
parts := strings.SplitN(name, "__", 2)
|
||
if len(parts) != 2 {
|
||
continue
|
||
}
|
||
|
||
originalFile := strings.TrimSpace(parts[1])
|
||
if originalFile == "" {
|
||
continue
|
||
}
|
||
|
||
ext := strings.ToLower(filepath.Ext(originalFile))
|
||
if ext != ".mp4" && ext != ".ts" {
|
||
continue
|
||
}
|
||
|
||
full := filepath.Join(trashDir, name)
|
||
fi, err := os.Stat(full)
|
||
if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
|
||
continue
|
||
}
|
||
|
||
id := stripHotPrefix(strings.TrimSuffix(filepath.Base(originalFile), ext))
|
||
id = strings.TrimSpace(id)
|
||
if id == "" {
|
||
continue
|
||
}
|
||
|
||
if sid, err := sanitizeID(id); err == nil && sid != "" {
|
||
live[sid] = struct{}{}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Core-Logik ohne Delay (für manuelle Trigger, z.B. nach Cleanup)
|
||
// Liefert Stats zurück, damit /api/settings/cleanup die Zahlen anzeigen kann.
|
||
func runGeneratedGarbageCollector() generatedGCStats {
|
||
stats := generatedGCStats{}
|
||
|
||
s := getSettings()
|
||
|
||
doneAbs, err := resolvePathRelativeToApp(s.DoneDir)
|
||
if err != nil {
|
||
appLogln("🧹 [gc] resolve doneDir failed:", err)
|
||
return stats
|
||
}
|
||
doneAbs = strings.TrimSpace(doneAbs)
|
||
if doneAbs == "" {
|
||
return stats
|
||
}
|
||
|
||
// 1) Live-IDs sammeln: alle mp4/ts unter /done (rekursiv), .trash zunächst ignorieren
|
||
live := make(map[string]struct{}, 4096)
|
||
|
||
_ = filepath.WalkDir(doneAbs, func(p string, d fs.DirEntry, err error) error {
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
|
||
name := d.Name()
|
||
|
||
if d.IsDir() {
|
||
if strings.EqualFold(name, ".trash") {
|
||
return fs.SkipDir
|
||
}
|
||
return nil
|
||
}
|
||
|
||
ext := strings.ToLower(filepath.Ext(name))
|
||
if ext != ".mp4" && ext != ".ts" {
|
||
return nil
|
||
}
|
||
|
||
info, err := d.Info()
|
||
if err != nil || info.IsDir() || info.Size() <= 0 {
|
||
return nil
|
||
}
|
||
|
||
base := strings.TrimSuffix(name, ext)
|
||
id, err := sanitizeID(stripHotPrefix(base))
|
||
if err != nil || id == "" {
|
||
return nil
|
||
}
|
||
|
||
live[id] = struct{}{}
|
||
return nil
|
||
})
|
||
|
||
// 1a) Dateien in .trash ebenfalls als "live" behandeln,
|
||
// damit ihre Generated-Dateien erst gelöscht werden,
|
||
// wenn sie wirklich endgültig aus .trash verschwinden.
|
||
collectLiveIDsFromTrash(filepath.Join(doneAbs, ".trash"), live)
|
||
|
||
// 1b) Auch aktuell laufende / nachbearbeitete Jobs als "live" behandeln.
|
||
// Zusätzlich separat merken, damit leere Ordner aktiver Jobs nicht gelöscht werden.
|
||
activeJobIDs := make(map[string]struct{}, 128)
|
||
|
||
jobsMu.RLock()
|
||
for _, j := range jobs {
|
||
if j == nil {
|
||
continue
|
||
}
|
||
|
||
id := strings.TrimSpace(assetIDForJob(j))
|
||
if id == "" {
|
||
continue
|
||
}
|
||
|
||
live[id] = struct{}{}
|
||
activeJobIDs[id] = struct{}{}
|
||
}
|
||
jobsMu.RUnlock()
|
||
|
||
// 2) /generated/meta/<id> prüfen
|
||
metaRoot, err := generatedMetaRoot()
|
||
if err == nil {
|
||
metaRoot = strings.TrimSpace(metaRoot)
|
||
}
|
||
if err != nil || metaRoot == "" {
|
||
return stats
|
||
}
|
||
|
||
removedMeta := 0
|
||
checkedMeta := 0
|
||
removedBytes := int64(0)
|
||
|
||
if entries, err := os.ReadDir(metaRoot); err == nil {
|
||
for _, e := range entries {
|
||
if !e.IsDir() {
|
||
continue
|
||
}
|
||
|
||
id := strings.TrimSpace(e.Name())
|
||
if id == "" || strings.HasPrefix(id, ".") {
|
||
continue
|
||
}
|
||
|
||
checkedMeta++
|
||
|
||
orphanDir := filepath.Join(metaRoot, id)
|
||
|
||
// Leere generated/meta/<id>-Ordner immer entfernen,
|
||
// außer ein aktiver Job könnte gerade hinein schreiben.
|
||
if _, active := activeJobIDs[id]; !active {
|
||
if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty {
|
||
removedMeta++
|
||
continue
|
||
}
|
||
}
|
||
|
||
if _, ok := live[id]; ok {
|
||
continue
|
||
}
|
||
|
||
orphanBytes := dirSizeBytes(orphanDir)
|
||
|
||
removeGeneratedForID(id)
|
||
|
||
// Falls removeGeneratedForID nur Dateien gelöscht hat, den nun leeren Ordner entfernen.
|
||
if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty {
|
||
// ok
|
||
}
|
||
|
||
if _, err := os.Stat(orphanDir); os.IsNotExist(err) {
|
||
removedMeta++
|
||
removedBytes += orphanBytes
|
||
}
|
||
}
|
||
}
|
||
|
||
//fmt.Printf("🧹 [gc] generated/meta checked=%d removed_orphans=%d bytes=%d\n", checkedMeta, removedMeta, removedBytes)
|
||
stats.Checked += checkedMeta
|
||
stats.Removed += removedMeta
|
||
stats.RemovedBytes += removedBytes
|
||
|
||
return stats
|
||
}
|