// backend\sse.go package main import ( "bytes" "encoding/json" "net/http" "path/filepath" "sort" "strings" "sync/atomic" "time" "github.com/r3labs/sse/v2" ) // -------------------- SSE server -------------------- type appSSE struct { server *sse.Server } var sseApp *appSSE type jobEvent struct { Type string `json:"type"` Model string `json:"model"` JobID string `json:"jobId"` Status JobStatus `json:"status"` Phase string `json:"phase,omitempty"` Progress int `json:"progress,omitempty"` StopRequestedAtMs int64 `json:"stopRequestedAtMs,omitempty"` StopAttempts int `json:"stopAttempts,omitempty"` SourceURL string `json:"sourceUrl,omitempty"` Output string `json:"output,omitempty"` StartedAt string `json:"startedAt,omitempty"` StartedAtMs int64 `json:"startedAtMs,omitempty"` EndedAt string `json:"endedAt,omitempty"` EndedAtMs int64 `json:"endedAtMs,omitempty"` SizeBytes int64 `json:"sizeBytes,omitempty"` DurationSeconds float64 `json:"durationSeconds,omitempty"` PreviewState string `json:"previewState,omitempty"` RoomStatus string `json:"roomStatus,omitempty"` IsOnline bool `json:"isOnline,omitempty"` ModelImageURL string `json:"modelImageUrl,omitempty"` ModelChatRoomURL string `json:"modelChatRoomUrl,omitempty"` PostWorkKey string `json:"postWorkKey,omitempty"` PostWork any `json:"postWork,omitempty"` TS int64 `json:"ts"` } type ssePublishItem struct { EventName string CacheKey string Data []byte } type finishedPostworkEvent struct { Type string `json:"type"` // "finished_postwork" Model string `json:"model"` // model event name, z. B. "maypeach" File string `json:"file"` AssetID string `json:"assetId,omitempty"` Queue string `json:"queue"` // "postwork" | "enrich" State string `json:"state"` // "queued" | "running" | "done" | "error" | "missing" Phase string `json:"phase,omitempty"` Label string `json:"label,omitempty"` Error string `json:"error,omitempty"` Position int `json:"position,omitempty"` Waiting int `json:"waiting,omitempty"` Running int `json:"running,omitempty"` MaxParallel int `json:"maxParallel,omitempty"` TS int64 `json:"ts"` } type taskStateEvent struct { Type string `json:"type"` // "task_state" GenerateAssetsRunning bool `json:"generateAssetsRunning"` RegenerateAssetsRunning bool `json:"regenerateAssetsRunning"` CheckVideosRunning bool `json:"checkVideosRunning"` CleanupRunning bool `json:"cleanupRunning"` AnyRunning bool `json:"anyRunning"` TS int64 `json:"ts"` } type appNotificationEvent struct { Type string `json:"type"` // "notification" Level string `json:"level"` Title string `json:"title"` Message string `json:"message,omitempty"` DurationMs int `json:"durationMs,omitempty"` TS int64 `json:"ts"` } func publishAppNotification(level string, title string, message string, durationMs int) { level = strings.TrimSpace(strings.ToLower(level)) if level == "" { level = "info" } ev := appNotificationEvent{ Type: "notification", Level: level, Title: strings.TrimSpace(title), Message: strings.TrimSpace(message), DurationMs: durationMs, TS: time.Now().UnixMilli(), } if ev.Title == "" { return } b, err := json.Marshal(ev) if err != nil { return } publishSSE("notification", b) } type analysisProgressEvent struct { Type string `json:"type"` Scope string `json:"scope,omitempty"` RequestID string `json:"requestId,omitempty"` Running bool `json:"running"` Phase string `json:"phase,omitempty"` Progress float64 `json:"progress"` Current int `json:"current,omitempty"` Total int `json:"total,omitempty"` File string `json:"file,omitempty"` SourceFile string `json:"sourceFile,omitempty"` Message string `json:"message,omitempty"` Error string `json:"error,omitempty"` StartedAtMs int64 `json:"startedAtMs,omitempty"` FinishedAtMs int64 `json:"finishedAtMs,omitempty"` DurationMs int64 `json:"durationMs,omitempty"` TS int64 `json:"ts"` } func publishAnalysisProgress(ev analysisProgressEvent) { ev.Type = "analysis_progress" ev.TS = time.Now().UnixMilli() if ev.Progress < 0 { ev.Progress = 0 } if ev.Progress > 1 { ev.Progress = 1 } b, err := json.Marshal(ev) if err != nil { return } publishSSE("analysisProgress", b) } func publishAnalysisStarted(file string, total int, message string) int64 { startedAtMs := time.Now().UnixMilli() publishAnalysisProgress(analysisProgressEvent{ Running: true, Phase: "starting", Progress: 0, Current: 0, Total: total, File: strings.TrimSpace(file), Message: message, StartedAtMs: startedAtMs, }) return startedAtMs } func publishAnalysisStep(startedAtMs int64, current int, total int, file string, message string) { progress := 0.0 if total > 0 { progress = float64(current) / float64(total) } publishAnalysisProgress(analysisProgressEvent{ Running: true, Phase: "running", Progress: progress, Current: current, Total: total, File: strings.TrimSpace(file), Message: message, StartedAtMs: startedAtMs, }) } func publishAnalysisFinished(startedAtMs int64, total int, file string, message string) { finishedAtMs := time.Now().UnixMilli() durationMs := finishedAtMs - startedAtMs if durationMs < 0 { durationMs = 0 } publishAnalysisProgress(analysisProgressEvent{ Running: false, Phase: "done", Progress: 1, Current: total, Total: total, File: strings.TrimSpace(file), Message: message, StartedAtMs: startedAtMs, FinishedAtMs: finishedAtMs, DurationMs: durationMs, }) } func publishAnalysisError(startedAtMs int64, file string, message string, err error) { finishedAtMs := time.Now().UnixMilli() durationMs := finishedAtMs - startedAtMs if durationMs < 0 { durationMs = 0 } errText := "" if err != nil { errText = err.Error() } publishAnalysisProgress(analysisProgressEvent{ Running: false, Phase: "error", Progress: 0, File: strings.TrimSpace(file), Message: message, Error: errText, StartedAtMs: startedAtMs, FinishedAtMs: finishedAtMs, DurationMs: durationMs, }) } func publishFinishedPostworkStateForJob( j *RecordJob, queue string, state string, phase string, label string, position int, waiting int, running int, maxParallel int, ) { if j == nil { return } file := strings.TrimSpace(filepath.Base(j.Output)) if file == "" { return } assetID := assetIDFromVideoPath(j.Output) publishFinishedPostworkStatusForJob(j, finishedPostworkEvent{ File: file, AssetID: assetID, Queue: queue, State: state, Phase: phase, Label: label, Position: position, Waiting: waiting, Running: running, MaxParallel: maxParallel, }) } func anyGenerateAssetsTaskRunning() bool { for _, st := range snapshotAssetsJobs() { if st.Queued || st.Running { return true } } return false } func anyRegenerateAssetsTaskRunning() bool { for _, job := range getRegenerateAssetsJobsSnapshot() { state := strings.TrimSpace(strings.ToLower(anyToString(job["state"]))) if state == "queued" || state == "running" { return true } } return false } func anyCheckVideosTaskRunning() bool { for _, st := range snapshotCheckVideosJobs() { if st.Queued || st.Running { return true } } return false } func anyCleanupTaskRunning() bool { for _, st := range snapshotCleanupJobs() { if st.Queued || st.Running { return true } } return false } func publishTaskState() { assetsRunning := anyGenerateAssetsTaskRunning() regenerateRunning := anyRegenerateAssetsTaskRunning() checkVideosRunning := anyCheckVideosTaskRunning() cleanupRunning := anyCleanupTaskRunning() ev := taskStateEvent{ Type: "task_state", GenerateAssetsRunning: assetsRunning, RegenerateAssetsRunning: regenerateRunning, CheckVideosRunning: checkVideosRunning, CleanupRunning: cleanupRunning, AnyRunning: assetsRunning || regenerateRunning || checkVideosRunning || cleanupRunning, TS: time.Now().UnixMilli(), } b, err := json.Marshal(ev) if err != nil { return } publishSSE("taskState", b) } func publishFinishedPostworkStatusForJob(j *RecordJob, ev finishedPostworkEvent) { if j == nil { return } if strings.TrimSpace(ev.File) == "" { return } eventName := sseModelEventNameForJob(j) if eventName == "" { return } ev.Type = "finished_postwork" ev.Model = eventName ev.TS = time.Now().UnixMilli() b, _ := json.Marshal(ev) // nur global für FinishedDownloads publishSSE("finishedPostwork", b) } func isTerminalJobStatusForSSE(status JobStatus) bool { s := strings.ToLower(strings.TrimSpace(string(status))) return s == "stopped" || s == "finished" || s == "failed" || s == "done" || s == "completed" || s == "canceled" || s == "cancelled" } func isPostworkJobForSSE(j *RecordJob) bool { if j == nil { return false } phase := strings.ToLower(strings.TrimSpace(j.Phase)) pwKey := strings.TrimSpace(j.PostWorkKey) if pwKey != "" { return true } if j.PostWork != nil { // falls PostWork als struct/map vorliegt, reicht für SSE der generelle Hinweis: return true } if j.EndedAt != nil && phase != "" { return true } if phase == "postwork" { return true } return false } func isVisibleDownloadJobForSSE(j *RecordJob) bool { if j == nil { return false } if isPostworkJobForSSE(j) { return false } if isTerminalJobStatusForSSE(j.Status) { return false } if j.EndedAt != nil { return false } return true } func isVisiblePostworkJobForSSE(j *RecordJob) bool { if j == nil { return false } if !isPostworkJobForSSE(j) { return false } if isActivePostworkJobForSSE(j) { return true } if isTerminalJobStatusForSSE(j.Status) { return false } return true } func isActivePostworkJobForSSE(j *RecordJob) bool { if j == nil { return false } if strings.TrimSpace(j.PostWorkKey) == "" { return false } if j.PostWork == nil { return true } state := strings.ToLower(strings.TrimSpace(j.PostWork.State)) return state == "" || state == "queued" || state == "running" } func shouldPublishModelEventForJob(j *RecordJob) bool { return isVisibleDownloadJobForSSE(j) || isVisiblePostworkJobForSSE(j) } func visibleJobEventsJSON() []ssePublishItem { out := make([]ssePublishItem, 0, 64) jobsMu.RLock() defer jobsMu.RUnlock() for _, j := range jobs { if j == nil || j.Hidden { continue } if !shouldPublishModelEventForJob(j) { continue } eventName := sseModelEventNameForJob(j) if eventName == "" { continue } payload := jobEvent{ Type: "job_upsert", Model: eventName, JobID: j.ID, Status: j.Status, Phase: j.Phase, Progress: j.Progress, StopRequestedAtMs: j.StopRequestedAtMs, StopAttempts: j.StopAttempts, SourceURL: j.SourceURL, Output: j.Output, StartedAt: j.StartedAt.Format(time.RFC3339Nano), StartedAtMs: j.StartedAtMs, SizeBytes: j.SizeBytes, DurationSeconds: j.DurationSeconds, PreviewState: j.PreviewState, PostWorkKey: strings.TrimSpace(j.PostWorkKey), PostWork: j.PostWork, TS: 0, // wichtig: stabil halten, damit Dedupe funktioniert } if sm := sseStoredModelForJob(j); sm != nil { payload.RoomStatus = strings.ToLower(strings.TrimSpace(sm.RoomStatus)) payload.IsOnline = sm.IsOnline payload.ModelImageURL = strings.TrimSpace(sm.ImageURL) payload.ModelChatRoomURL = strings.TrimSpace(sm.ChatRoomURL) } if j.EndedAt != nil { payload.EndedAt = j.EndedAt.Format(time.RFC3339Nano) payload.EndedAtMs = j.EndedAtMs } b, err := json.Marshal(payload) if err != nil { continue } out = append(out, ssePublishItem{ EventName: eventName, CacheKey: eventName + "|" + j.ID, Data: b, }) } return out } func publishJobRemove(j *RecordJob) { if j == nil || j.Hidden { return } eventName := sseModelEventNameForJob(j) if eventName == "" { return } b, _ := json.Marshal(jobEvent{ Type: "job_remove", Model: eventName, JobID: j.ID, TS: time.Now().UnixMilli(), }) publishSSE(eventName, b) } func publishModelJobUpsert(jobID string) bool { jobsMu.RLock() j := jobs[jobID] if j == nil || j.Hidden { jobsMu.RUnlock() return false } eventName := sseModelEventNameForJob(j) if eventName == "" { jobsMu.RUnlock() return false } var postWork any if j.PostWork != nil { tmp := *j.PostWork postWork = &tmp } payload := jobEvent{ Type: "job_upsert", Model: eventName, JobID: j.ID, Status: j.Status, Phase: j.Phase, Progress: j.Progress, StopRequestedAtMs: j.StopRequestedAtMs, StopAttempts: j.StopAttempts, SourceURL: j.SourceURL, Output: j.Output, StartedAt: j.StartedAt.Format(time.RFC3339Nano), StartedAtMs: j.StartedAtMs, SizeBytes: j.SizeBytes, DurationSeconds: j.DurationSeconds, PreviewState: j.PreviewState, PostWorkKey: strings.TrimSpace(j.PostWorkKey), PostWork: postWork, TS: time.Now().UnixMilli(), } if sm := sseStoredModelForJob(j); sm != nil { payload.RoomStatus = strings.ToLower(strings.TrimSpace(sm.RoomStatus)) payload.IsOnline = sm.IsOnline payload.ModelImageURL = strings.TrimSpace(sm.ImageURL) payload.ModelChatRoomURL = strings.TrimSpace(sm.ChatRoomURL) } if j.EndedAt != nil { payload.EndedAt = j.EndedAt.Format(time.RFC3339Nano) payload.EndedAtMs = j.EndedAtMs } jobsMu.RUnlock() b, err := json.Marshal(payload) if err != nil { return false } publishSSE(eventName, b) return true } func sseModelEventNameForJob(j *RecordJob) string { if j == nil { return "" } src := strings.TrimSpace(j.SourceURL) switch detectProvider(src) { case "chaturbate": if u := strings.TrimSpace(extractUsername(src)); u != "" { return strings.ToLower(u) } case "mfc": if u := strings.TrimSpace(extractMFCUsername(src)); u != "" { return strings.ToLower(u) } } return "" } func sseStoredModelForJob(j *RecordJob) *StoredModel { if j == nil || cbModelStore == nil { return nil } src := strings.TrimSpace(j.SourceURL) if src == "" { return nil } host := "" modelKey := "" switch detectProvider(src) { case "chaturbate": host = "chaturbate.com" modelKey = strings.ToLower(strings.TrimSpace(extractUsername(src))) default: return nil } if host == "" || modelKey == "" { return nil } m, ok := cbModelStore.GetByHostAndModelKey(host, modelKey) if !ok { return nil } return m } func initSSE() { srv := sse.New() srv.SplitData = true stream := srv.CreateStream("events") stream.AutoReplay = false sseApp = &appSSE{ server: srv, } // Debounced broadcaster (done changed) go func() { for range doneNotify { time.Sleep(40 * time.Millisecond) for { select { case <-doneNotify: default: goto SEND } } SEND: seq := atomic.AddUint64(&doneSeq, 1) b, _ := json.Marshal(map[string]any{ "type": "doneChanged", "seq": seq, "ts": time.Now().UnixMilli(), }) publishSSE("doneChanged", b) } }() // Debounced broadcaster (assets) go func() { for range assetsNotify { time.Sleep(80 * time.Millisecond) for { select { case <-assetsNotify: default: goto SEND } } SEND: b := assetsSnapshotJSON() if len(b) > 0 { publishSSE("state", b) } } }() // Per running job: 1 SSE event per second go func() { t := time.NewTicker(2 * time.Second) defer t.Stop() lastByKey := map[string][]byte{} lastHeartbeat := time.Now() for range t.C { events := visibleJobEventsJSON() liveKeys := make(map[string]struct{}, len(events)) forceHeartbeat := time.Since(lastHeartbeat) >= 15*time.Second for _, ev := range events { if len(ev.Data) == 0 || ev.EventName == "" || ev.CacheKey == "" { continue } liveKeys[ev.CacheKey] = struct{}{} prev, ok := lastByKey[ev.CacheKey] if !forceHeartbeat && ok && bytes.Equal(prev, ev.Data) { continue } lastByKey[ev.CacheKey] = append([]byte(nil), ev.Data...) publishSSE(ev.EventName, ev.Data) } for k := range lastByKey { if _, ok := liveKeys[k]; !ok { delete(lastByKey, k) } } if forceHeartbeat { lastHeartbeat = time.Now() } } }() } func publishSSE(eventName string, data []byte) { if sseApp == nil || sseApp.server == nil { return } if len(data) == 0 { return } sseApp.server.Publish("events", &sse.Event{ Event: []byte(eventName), Data: data, }) } // -------------------- notify channels -------------------- var ( doneNotify = make(chan struct{}, 1) doneSeq uint64 assetsNotify = make(chan struct{}, 1) ) func notifyDoneChanged() { select { case doneNotify <- struct{}{}: default: } } func notifyAssetsChanged() { select { case assetsNotify <- struct{}{}: default: } } // -------------------- snapshots -------------------- func jobsSnapshotJSON() []byte { jobsMu.RLock() list := make([]*RecordJob, 0, len(jobs)) for _, j := range jobs { if j == nil || j.Hidden { continue } c := *j c.cancel = nil list = append(list, &c) } jobsMu.RUnlock() sort.Slice(list, func(i, j int) bool { return list[i].StartedAt.After(list[j].StartedAt) }) b, _ := json.Marshal(list) return b } func assetsSnapshotJSON() []byte { b, _ := json.Marshal(snapshotAssetsJobs()) return b } // -------------------- unified stream handler -------------------- func eventsStream(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Nur GET erlaubt", http.StatusMethodNotAllowed) return } if sseApp == nil || sseApp.server == nil { http.Error(w, "SSE nicht initialisiert", http.StatusInternalServerError) return } q := r.URL.Query() q.Set("stream", "events") r2 := r.Clone(r.Context()) r2.URL.RawQuery = q.Encode() sseApp.server.ServeHTTP(w, r2) } // -------------------- optional helper -------------------- func publishRawSSE(eventName string, buf *bytes.Buffer) { if buf == nil { return } publishSSE(eventName, buf.Bytes()) }