nsfwapp/backend/tasks_cleanup.go

// backend\tasks_cleanup.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type cleanupCandidate struct {
	path string
	name string
	size int64
}

type cleanupResp struct {
	// Small / broken downloads cleanup
	ScannedFiles       int    `json:"scannedFiles"`
	DeletedFiles       int    `json:"deletedFiles"`
	DeletedBrokenFiles int    `json:"deletedBrokenFiles"`
	SkippedFiles       int    `json:"skippedFiles"`
	DeletedBytes       int64  `json:"deletedBytes"`
	DeletedBytesHuman  string `json:"deletedBytesHuman"`
	ErrorCount         int    `json:"errorCount"`

	// Generated-GC
	GeneratedOrphansChecked           int    `json:"generatedOrphansChecked"`
	GeneratedOrphansRemoved           int    `json:"generatedOrphansRemoved"`
	GeneratedOrphansRemovedBytes      int64  `json:"generatedOrphansRemovedBytes"`
	GeneratedOrphansRemovedBytesHuman string `json:"generatedOrphansRemovedBytesHuman"`
}

// Optional: falls du später Threshold per Body überschreiben willst.
// Frontend sendet aktuell nichts -> wir nutzen Settings.
type cleanupReq struct {
	BelowMB *int `json:"belowMB,omitempty"`
}

type cleanupTaskJob struct {
	State  CleanupTaskState
	Cancel context.CancelFunc
}

var cleanupJobsMu sync.Mutex
var cleanupJobs = map[string]*cleanupTaskJob{}

func snapshotCleanupJobs() []CleanupTaskState {
	cleanupJobsMu.Lock()
	defer cleanupJobsMu.Unlock()

	now := time.Now()
	out := make([]CleanupTaskState, 0, len(cleanupJobs))

	for id, job := range cleanupJobs {
		if job == nil {
			delete(cleanupJobs, id)
			continue
		}

		st := job.State
		if !st.Queued && !st.Running && st.FinishedAt != nil && now.Sub(*st.FinishedAt) > 4*time.Second {
			delete(cleanupJobs, id)
			continue
		}

		out = append(out, st)
	}

	return out
}

func updateCleanupJobState(jobID string, fn func(st *CleanupTaskState)) (CleanupTaskState, bool) {
	cleanupJobsMu.Lock()
	job := cleanupJobs[jobID]
	if job == nil {
		cleanupJobsMu.Unlock()
		return CleanupTaskState{}, false
	}

	fn(&job.State)
	st := job.State
	cleanupJobsMu.Unlock()

	publishTaskState()
	return st, true
}

func removeCleanupJob(jobID string) {
	cleanupJobsMu.Lock()
	delete(cleanupJobs, jobID)
	cleanupJobsMu.Unlock()

	publishTaskState()
}

func finishCleanupJob(jobID string, text string, err error) {
	now := time.Now()

	updateCleanupJobState(jobID, func(st *CleanupTaskState) {
		st.Queued = false
		st.Running = false
		st.FinishedAt = &now
		st.CurrentFile = ""

		if err == nil {
			st.Error = ""
			st.Text = strings.TrimSpace(text)
			if st.Total > 0 {
				st.Done = st.Total
			}
			return
		}

		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
			st.Error = "Abgebrochen"
			st.Text = "Abgebrochen."
		} else {
			st.Error = strings.TrimSpace(err.Error())
			st.Text = ""
		}
	})

	time.AfterFunc(4*time.Second, func() {
		removeCleanupJob(jobID)
	})
}

// /api/settings/cleanup (POST)
// - löscht kleine Dateien < threshold MB (mp4/ts; skip .part/.tmp; skip keep-Ordner)
// - räumt Orphans (preview/thumbs + generated) auf
func settingsCleanupHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case http.MethodPost:
		s := getSettings()

		doneAbs, err := resolvePathRelativeToApp(s.DoneDir)
		if err != nil || strings.TrimSpace(doneAbs) == "" {
			http.Error(w, "doneDir auflösung fehlgeschlagen", http.StatusBadRequest)
			return
		}

		mb := int(s.AutoDeleteSmallDownloadsBelowMB)

		var req cleanupReq
		if r.Body != nil {
			_ = json.NewDecoder(r.Body).Decode(&req)
		}
		if req.BelowMB != nil {
			mb = *req.BelowMB
		}
		if mb < 0 {
			mb = 0
		}

		jobID := newTaskID("cleanup")
		ctx, cancel := context.WithCancel(context.Background())

		st := CleanupTaskState{
			ID:          jobID,
			Queued:      true,
			Running:     false,
			QueuedAt:    time.Now(),
			StartedAt:   time.Time{},
			FinishedAt:  nil,
			Text:        "Wartet…",
			Error:       "",
			CurrentFile: "",
			Done:        0,
			Total:       0,
		}

		cleanupJobsMu.Lock()
		cleanupJobs[jobID] = &cleanupTaskJob{
			State:  st,
			Cancel: cancel,
		}
		cleanupJobsMu.Unlock()

		publishTaskState()

		go func(jobID string, ctx context.Context, doneAbs string, mb int) {
			if err := acquireExclusiveTask(ctx); err != nil {
				finishCleanupJob(jobID, "", err)
				return
			}
			defer releaseExclusiveTask()

			updateCleanupJobState(jobID, func(st *CleanupTaskState) {
				st.Queued = false
				st.Running = true
				st.StartedAt = time.Now()
				st.Text = "Räume auf…"
			})

			resp := cleanupResp{}

			if mb > 0 {
				threshold := int64(mb) * 1024 * 1024
				if err := cleanupSmallFiles(ctx, jobID, doneAbs, threshold, &resp); err != nil {
					finishCleanupJob(jobID, "", err)
					return
				}
			}

			select {
			case <-ctx.Done():
				finishCleanupJob(jobID, "", ctx.Err())
				return
			default:
			}

			gcStats := triggerGeneratedGarbageCollectorSync()
			resp.GeneratedOrphansChecked = gcStats.Checked
			resp.GeneratedOrphansRemoved = gcStats.Removed
			resp.GeneratedOrphansRemovedBytes = gcStats.RemovedBytes
			resp.GeneratedOrphansRemovedBytesHuman = formatBytesSI(gcStats.RemovedBytes)

			resp.DeletedBytes += gcStats.RemovedBytes
			resp.DeletedBytesHuman = formatBytesSI(resp.DeletedBytes)

			if resp.DeletedFiles > 0 || resp.GeneratedOrphansRemoved > 0 {
				notifyDoneChanged()
			}

			finishCleanupJob(
				jobID,
				cleanupProgressText(resp.DeletedFiles, resp.DeletedBrokenFiles, resp.ScannedFiles, resp.DeletedBytes),
				nil,
			)
		}(jobID, ctx, doneAbs, mb)

		writeJSON(w, http.StatusOK, st)
		return

	case http.MethodDelete:
		id := strings.TrimSpace(r.URL.Query().Get("id"))
		if id == "" {
			http.Error(w, "id fehlt", http.StatusBadRequest)
			return
		}

		cleanupJobsMu.Lock()
		job := cleanupJobs[id]
		cleanupJobsMu.Unlock()

		if job != nil && job.Cancel != nil {
			job.Cancel()
		}

		w.WriteHeader(http.StatusNoContent)
		return

	default:
		http.Error(w, "Nur POST/DELETE erlaubt", http.StatusMethodNotAllowed)
		return
	}
}

func cleanupProgressText(deleted int, broken int, total int, freedBytes int64) string {
	if broken > 0 {
		return fmt.Sprintf(
			"gelöscht: %d/%d · fehlerhaft: %d · freigegeben: %s",
			deleted,
			total,
			broken,
			formatBytesSI(freedBytes),
		)
	}

	return fmt.Sprintf(
		"gelöscht: %d/%d · freigegeben: %s",
		deleted,
		total,
		formatBytesSI(freedBytes),
	)
}

func cleanupVideoLooksBroken(ctx context.Context, path string) (bool, string) {
	path = strings.TrimSpace(path)
	if path == "" {
		return false, ""
	}

	ext := strings.ToLower(filepath.Ext(path))
	if ext != ".mp4" && ext != ".ts" {
		return false, ""
	}

	fi, err := os.Stat(path)
	if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
		return true, "file missing/empty"
	}

	// Erst ffprobe/duration: wenn keine Dauer lesbar ist, ist die Datei meistens kaputt.
	dctx, cancel := context.WithTimeout(ctx, 8*time.Second)
	dur, derr := durationSecondsCached(dctx, path)
	dctxErr := dctx.Err()
	cancel()

	if errors.Is(dctxErr, context.DeadlineExceeded) || errors.Is(dctxErr, context.Canceled) {
		// Timeout beim Prüfen nicht als "kaputt" werten, um keine guten Dateien wegen Last zu löschen.
		return false, "duration check timeout"
	}

	if derr != nil || dur <= 0 {
		if derr != nil {
			return true, "duration failed: " + derr.Error()
		}
		return true, "duration <= 0"
	}

	// Danach konservative Decode-Prüfung am Ende.
	// Diese Funktion nutzt du bereits nach dem Remux in recorder.go.
	vctx, vcancel := context.WithTimeout(ctx, 20*time.Second)
	verr := validateVideoDecodesNearEnd(vctx, path)
	vctxErr := vctx.Err()
	vcancel()

	if errors.Is(vctxErr, context.DeadlineExceeded) || errors.Is(vctxErr, context.Canceled) {
		return false, "decode check timeout"
	}

	if verr != nil {
		return true, "decode failed: " + verr.Error()
	}

	return false, ""
}

func cleanupSmallFiles(ctx context.Context, jobID string, doneAbs string, threshold int64, resp *cleanupResp) error {
	isCandidate := func(name string) bool {
		low := strings.ToLower(name)
		if strings.Contains(low, ".part") || strings.Contains(low, ".tmp") {
			return false
		}
		ext := strings.ToLower(filepath.Ext(name))
		return ext == ".mp4" || ext == ".ts"
	}

	candidates := make([]cleanupCandidate, 0, 1024)

	collectCandidates := func(dir string, allowSubdirs bool) error {
		ents, err := os.ReadDir(dir)
		if err != nil {
			return nil
		}

		for _, e := range ents {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
			}

			full := filepath.Join(dir, e.Name())

			if e.IsDir() {
				if !allowSubdirs {
					continue
				}
				if e.Name() == "keep" {
					continue
				}

				sub, err := os.ReadDir(full)
				if err != nil {
					continue
				}

				for _, se := range sub {
					if se.IsDir() {
						continue
					}

					name := se.Name()
					if !isCandidate(name) {
						resp.SkippedFiles++
						continue
					}

					p := filepath.Join(full, name)
					fi, err := os.Stat(p)
					if err != nil || fi.IsDir() || fi.Size() <= 0 {
						resp.SkippedFiles++
						continue
					}

					candidates = append(candidates, cleanupCandidate{
						path: p,
						name: name,
						size: fi.Size(),
					})
				}
				continue
			}

			name := e.Name()
			if !isCandidate(name) {
				resp.SkippedFiles++
				continue
			}

			fi, err := os.Stat(full)
			if err != nil || fi.IsDir() || fi.Size() <= 0 {
				resp.SkippedFiles++
				continue
			}

			candidates = append(candidates, cleanupCandidate{
				path: full,
				name: name,
				size: fi.Size(),
			})
		}
		return nil
	}

	if err := collectCandidates(doneAbs, true); err != nil {
		return err
	}

	resp.ScannedFiles = len(candidates)

	updateCleanupJobState(jobID, func(st *CleanupTaskState) {
		st.Queued = false
		st.Running = true
		if st.StartedAt.IsZero() {
			st.StartedAt = time.Now()
		}
		st.Done = 0
		st.Total = resp.ScannedFiles
		st.CurrentFile = ""
		st.Text = cleanupProgressText(resp.DeletedFiles, resp.DeletedBrokenFiles, resp.ScannedFiles, resp.DeletedBytes)
		st.Error = ""
		st.FinishedAt = nil
	})

	for i, c := range candidates {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}
		deleteReason := ""

		if c.size < threshold {
			deleteReason = "small"
		} else {
			broken, why := cleanupVideoLooksBroken(ctx, c.path)
			if broken {
				deleteReason = "broken"
				appLogln("🧹 cleanup deleting broken video:", filepath.Base(c.path), why)
			}
		}

		if deleteReason != "" {
			base := strings.TrimSuffix(filepath.Base(c.path), filepath.Ext(c.path))
			id := stripHotPrefix(base)

			// Stoppt ggf. laufende Asset-/Enrich-Tasks für genau diese Datei,
			// damit Windows-Dateihandles nicht blockieren.
			releaseFileForMutation(c.name)

			if derr := removeWithRetry(c.path); derr == nil || os.IsNotExist(derr) {
				resp.DeletedFiles++
				resp.DeletedBytes += c.size

				if deleteReason == "broken" {
					resp.DeletedBrokenFiles++
				}

				if strings.TrimSpace(id) != "" {
					removeGeneratedForID(id)
				}

				purgeDurationCacheForPath(c.path)
			} else {
				resp.ErrorCount++
			}
		}

		updateCleanupJobState(jobID, func(st *CleanupTaskState) {
			st.Queued = false
			st.Running = true
			if st.StartedAt.IsZero() {
				st.StartedAt = time.Now()
			}
			st.Done = i + 1
			st.Total = resp.ScannedFiles
			st.CurrentFile = c.name
			st.Text = cleanupProgressText(resp.DeletedFiles, resp.DeletedBrokenFiles, resp.ScannedFiles, resp.DeletedBytes)
			st.Error = ""
			st.FinishedAt = nil
		})
	}

	return nil
}

var generatedGCRunning int32

type generatedGCStats struct {
	Checked      int
	Removed      int
	RemovedBytes int64
}

func dirSizeBytes(root string) int64 {
	root = strings.TrimSpace(root)
	if root == "" {
		return 0
	}

	var total int64

	_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		if d.IsDir() {
			return nil
		}

		info, err := d.Info()
		if err != nil {
			return nil
		}
		if info.Size() > 0 {
			total += info.Size()
		}
		return nil
	})

	return total
}

func removeDirIfEmpty(dir string) (bool, error) {
	dir = strings.TrimSpace(dir)
	if dir == "" {
		return false, nil
	}

	entries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	if len(entries) > 0 {
		return false, nil
	}

	if err := os.Remove(dir); err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// Läuft synchron und liefert Zahlen zurück (für /api/settings/cleanup Response).
func triggerGeneratedGarbageCollectorSync() generatedGCStats {
	// nur 1 GC gleichzeitig
	if !atomic.CompareAndSwapInt32(&generatedGCRunning, 0, 1) {
		appLogln("🧹 [gc] skip: already running")
		return generatedGCStats{}
	}
	defer atomic.StoreInt32(&generatedGCRunning, 0)

	stats := runGeneratedGarbageCollector()
	return stats
}

// Läuft 1× nach Serverstart (mit Delay), löscht /generated/* Orphans.
func startGeneratedGarbageCollector() {
	go func() {
		time.Sleep(3 * time.Second)
		triggerGeneratedGarbageCollectorSync()
	}()
}

func collectLiveIDsFromTrash(trashDir string, live map[string]struct{}) {
	trashDir = strings.TrimSpace(trashDir)
	if trashDir == "" || live == nil {
		return
	}

	entries, err := os.ReadDir(trashDir)
	if err != nil {
		return
	}

	for _, e := range entries {
		if e == nil || e.IsDir() {
			continue
		}

		name := strings.TrimSpace(e.Name())
		if name == "" {
			continue
		}

		// last.json ignorieren
		if strings.EqualFold(name, "last.json") {
			continue
		}

		// Format in .trash:
		// <token>__<original-file>
		parts := strings.SplitN(name, "__", 2)
		if len(parts) != 2 {
			continue
		}

		originalFile := strings.TrimSpace(parts[1])
		if originalFile == "" {
			continue
		}

		ext := strings.ToLower(filepath.Ext(originalFile))
		if ext != ".mp4" && ext != ".ts" {
			continue
		}

		full := filepath.Join(trashDir, name)
		fi, err := os.Stat(full)
		if err != nil || fi == nil || fi.IsDir() || fi.Size() <= 0 {
			continue
		}

		id := stripHotPrefix(strings.TrimSuffix(filepath.Base(originalFile), ext))
		id = strings.TrimSpace(id)
		if id == "" {
			continue
		}

		if sid, err := sanitizeID(id); err == nil && sid != "" {
			live[sid] = struct{}{}
		}
	}
}

// Core-Logik ohne Delay (für manuelle Trigger, z.B. nach Cleanup)
// Liefert Stats zurück, damit /api/settings/cleanup die Zahlen anzeigen kann.
func runGeneratedGarbageCollector() generatedGCStats {
	stats := generatedGCStats{}

	s := getSettings()

	doneAbs, err := resolvePathRelativeToApp(s.DoneDir)
	if err != nil {
		appLogln("🧹 [gc] resolve doneDir failed:", err)
		return stats
	}
	doneAbs = strings.TrimSpace(doneAbs)
	if doneAbs == "" {
		return stats
	}

	// 1) Live-IDs sammeln: alle mp4/ts unter /done (rekursiv), .trash zunächst ignorieren
	live := make(map[string]struct{}, 4096)

	_ = filepath.WalkDir(doneAbs, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}

		name := d.Name()

		if d.IsDir() {
			if strings.EqualFold(name, ".trash") {
				return fs.SkipDir
			}
			return nil
		}

		ext := strings.ToLower(filepath.Ext(name))
		if ext != ".mp4" && ext != ".ts" {
			return nil
		}

		info, err := d.Info()
		if err != nil || info.IsDir() || info.Size() <= 0 {
			return nil
		}

		base := strings.TrimSuffix(name, ext)
		id, err := sanitizeID(stripHotPrefix(base))
		if err != nil || id == "" {
			return nil
		}

		live[id] = struct{}{}
		return nil
	})

	// 1a) Dateien in .trash ebenfalls als "live" behandeln,
	// damit ihre Generated-Dateien erst gelöscht werden,
	// wenn sie wirklich endgültig aus .trash verschwinden.
	collectLiveIDsFromTrash(filepath.Join(doneAbs, ".trash"), live)

	// 1b) Auch aktuell laufende / nachbearbeitete Jobs als "live" behandeln.
	// Zusätzlich separat merken, damit leere Ordner aktiver Jobs nicht gelöscht werden.
	activeJobIDs := make(map[string]struct{}, 128)

	jobsMu.RLock()
	for _, j := range jobs {
		if j == nil {
			continue
		}

		id := strings.TrimSpace(assetIDForJob(j))
		if id == "" {
			continue
		}

		live[id] = struct{}{}
		activeJobIDs[id] = struct{}{}
	}
	jobsMu.RUnlock()

	// 2) /generated/meta/<id> prüfen
	metaRoot, err := generatedMetaRoot()
	if err == nil {
		metaRoot = strings.TrimSpace(metaRoot)
	}
	if err != nil || metaRoot == "" {
		return stats
	}

	removedMeta := 0
	checkedMeta := 0
	removedBytes := int64(0)

	if entries, err := os.ReadDir(metaRoot); err == nil {
		for _, e := range entries {
			if !e.IsDir() {
				continue
			}

			id := strings.TrimSpace(e.Name())
			if id == "" || strings.HasPrefix(id, ".") {
				continue
			}

			checkedMeta++

			orphanDir := filepath.Join(metaRoot, id)

			// Leere generated/meta/<id>-Ordner immer entfernen,
			// außer ein aktiver Job könnte gerade hinein schreiben.
			if _, active := activeJobIDs[id]; !active {
				if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty {
					removedMeta++
					continue
				}
			}

			if _, ok := live[id]; ok {
				continue
			}

			orphanBytes := dirSizeBytes(orphanDir)

			removeGeneratedForID(id)

			// Falls removeGeneratedForID nur Dateien gelöscht hat, den nun leeren Ordner entfernen.
			if removedEmpty, err := removeDirIfEmpty(orphanDir); err == nil && removedEmpty {
				// ok
			}

			if _, err := os.Stat(orphanDir); os.IsNotExist(err) {
				removedMeta++
				removedBytes += orphanBytes
			}
		}
	}

	//fmt.Printf("🧹 [gc] generated/meta checked=%d removed_orphans=%d bytes=%d\n", checkedMeta, removedMeta, removedBytes)
	stats.Checked += checkedMeta
	stats.Removed += removedMeta
	stats.RemovedBytes += removedBytes

	return stats
}