bugfixes

backend/models_api.go

@@ -3,12 +3,14 @@
 package main
 
 import (
+	"database/sql"
 	"encoding/csv"
 	"encoding/json"
 	"errors"
 	"io"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -252,6 +254,146 @@ func buildModelsOverview(rawList any, req modelsOverviewRequest) (modelsOverview
 	return resp, nil
 }
 
+func loadModelsOverview(store *ModelStore, req modelsOverviewRequest) (modelsOverviewResponse, error) {
+	if err := store.ensureInit(); err != nil {
+		return modelsOverviewResponse{}, err
+	}
+
+	onlineExpr := `(lower(trim(COALESCE(room_status,''))) <> 'offline' AND (COALESCE(is_online,false) = true OR trim(COALESCE(room_status,'')) <> ''))`
+
+	var totalCount, onlineCount, onlineWatchedCount, onlineFavCount, onlineLikedCount int64
+	err := store.db.QueryRow(`
+SELECT
+  COUNT(*),
+  COALESCE(SUM(CASE WHEN lower(COALESCE(host,'')) LIKE '%chaturbate%' AND `+onlineExpr+` THEN 1 ELSE 0 END), 0),
+  COALESCE(SUM(CASE WHEN lower(COALESCE(host,'')) LIKE '%chaturbate%' AND COALESCE(watching,false) = true AND `+onlineExpr+` THEN 1 ELSE 0 END), 0),
+  COALESCE(SUM(CASE WHEN COALESCE(favorite,false) = true AND `+onlineExpr+` THEN 1 ELSE 0 END), 0),
+  COALESCE(SUM(CASE WHEN COALESCE(liked,false) = true AND `+onlineExpr+` THEN 1 ELSE 0 END), 0)
+FROM models;
+`).Scan(&totalCount, &onlineCount, &onlineWatchedCount, &onlineFavCount, &onlineLikedCount)
+	if err != nil {
+		return modelsOverviewResponse{}, err
+	}
+
+	resp := modelsOverviewResponse{
+		TotalCount:               int(totalCount),
+		OnlineModelsCount:        int(onlineCount),
+		OnlineWatchedModelsCount: int(onlineWatchedCount),
+		OnlineFavCount:           int(onlineFavCount),
+		OnlineLikedCount:         int(onlineLikedCount),
+		Models:                   make([]modelsOverviewModel, 0, len(req.Keys)+32),
+	}
+
+	keySet := normalizeModelKeySet(req.Keys)
+	keys := make([]string, 0, len(keySet))
+	for key := range keySet {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	whereParts := make([]string, 0, 2)
+	args := make([]any, 0, len(keys))
+
+	if req.IncludeWatched {
+		whereParts = append(whereParts, "COALESCE(watching,false) = true")
+	}
+	if len(keys) > 0 {
+		placeholders := make([]string, 0, len(keys))
+		for _, key := range keys {
+			args = append(args, key)
+			placeholders = append(placeholders, "$"+strconv.Itoa(len(args)))
+		}
+		whereParts = append(whereParts, "lower(trim(model_key)) IN ("+strings.Join(placeholders, ",")+")")
+	}
+
+	if len(whereParts) == 0 {
+		return resp, nil
+	}
+
+	rows, err := store.db.Query(`
+SELECT
+  id,
+  COALESCE(input,''),
+  COALESCE(host,''),
+  COALESCE(model_key,''),
+  COALESCE(tags,''),
+  COALESCE(cb_online_json,''),
+  COALESCE(watching,false),
+  COALESCE(favorite,false),
+  liked,
+  COALESCE(is_url,false),
+  COALESCE(path,''),
+  COALESCE(room_status,''),
+  COALESCE(is_online,false),
+  COALESCE(chat_room_url,''),
+  COALESCE(image_url,''),
+  last_online_at,
+  last_offline_at,
+  last_room_sync_at
+FROM models
+WHERE `+strings.Join(whereParts, " OR ")+`
+ORDER BY updated_at DESC;
+`, args...)
+	if err != nil {
+		return modelsOverviewResponse{}, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			m              modelsOverviewModel
+			cbOnlineJSON   string
+			liked          sql.NullBool
+			lastOnlineAt   dbNullTime
+			lastOfflineAt  dbNullTime
+			lastRoomSyncAt dbNullTime
+		)
+
+		if err := rows.Scan(
+			&m.ID,
+			&m.Input,
+			&m.Host,
+			&m.ModelKey,
+			&m.Tags,
+			&cbOnlineJSON,
+			&m.Watching,
+			&m.Favorite,
+			&liked,
+			&m.IsURL,
+			&m.Path,
+			&m.RoomStatus,
+			&m.IsOnline,
+			&m.ChatRoomURL,
+			&m.ImageURL,
+			&lastOnlineAt,
+			&lastOfflineAt,
+			&lastRoomSyncAt,
+		); err != nil {
+			return modelsOverviewResponse{}, err
+		}
+
+		if strings.TrimSpace(cbOnlineJSON) != "" {
+			var snap ChaturbateOnlineSnapshot
+			if err := json.Unmarshal([]byte(cbOnlineJSON), &snap); err == nil {
+				m.Gender = strings.TrimSpace(snap.Gender)
+				m.Country = strings.TrimSpace(snap.Country)
+			}
+		}
+
+		m.Liked = ptrLikedFromNullBool(liked)
+		m.LastOnlineAt = fmtNullTime(lastOnlineAt.NullTime)
+		m.LastOfflineAt = fmtNullTime(lastOfflineAt.NullTime)
+		m.LastRoomSyncAt = fmtNullTime(lastRoomSyncAt.NullTime)
+
+		resp.Models = append(resp.Models, m)
+	}
+	if err := rows.Err(); err != nil {
+		return modelsOverviewResponse{}, err
+	}
+
+	return resp, nil
+}
+
 func importModelsCSV(store *ModelStore, r io.Reader, kind string) (importResult, error) {
 	cr := csv.NewReader(r)
 	cr.Comma = ';'
@@ -427,9 +569,7 @@ func RegisterModelAPI(mux *http.ServeMux, store *ModelStore) {
 			return
 		}
 
-		list := store.List()
+		resp, err := loadModelsOverview(store, req)
-
-		resp, err := buildModelsOverview(list, req)
 		if err != nil {
 			modelsWriteJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
 			return
@@ -697,7 +837,7 @@ func RegisterModelAPI(mux *http.ServeMux, store *ModelStore) {
 			req.ID = ensured.ID
 		}
 
-		m, err := store.PatchFlags(req)
+		m, err := store.PatchFlagsFast(req)
 		if err != nil {
 			modelsWriteJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
 			return

backend/models_store.go

@@ -12,6 +12,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -124,7 +125,7 @@ type ModelStore struct {
 	initOnce sync.Once
 	initErr  error
 
-	// serialize writes (einfach & robust)
+	// coarse lock for legacy write paths that still need local serialization
 	mu sync.Mutex
 
 	statusMu         sync.Mutex
@@ -240,6 +241,13 @@ func ptrLikedFromNullBool(n sql.NullBool) *bool {
 	return &v
 }
 
+func nullableBoolPtrArg(v *bool) any {
+	if v == nil {
+		return nil
+	}
+	return *v
+}
+
 // parseRFC3339Nano: akzeptiert RFC3339/RFC3339Nano, sonst "invalid" -> (Valid=false)
 func parseRFC3339Nano(s string) sql.NullTime {
 	s = strings.TrimSpace(s)
@@ -378,6 +386,8 @@ func (s *ModelStore) ensurePostgresSchema() error {
 		);`,
 		`CREATE INDEX IF NOT EXISTS idx_models_host_model_key ON models(host, model_key);`,
 		`CREATE INDEX IF NOT EXISTS idx_models_model_key ON models(model_key);`,
+		`CREATE INDEX IF NOT EXISTS idx_models_host_model_key_norm ON models ((lower(trim(host))), (lower(trim(model_key))));`,
+		`CREATE INDEX IF NOT EXISTS idx_models_model_key_norm_updated ON models ((lower(trim(model_key))), updated_at DESC);`,
 		`CREATE INDEX IF NOT EXISTS idx_models_watching_host ON models(watching, host);`,
 		`CREATE INDEX IF NOT EXISTS idx_models_watching_updated ON models(watching, updated_at DESC);`,
 		`CREATE INDEX IF NOT EXISTS idx_models_favorite_updated ON models(favorite, updated_at DESC);`,
@@ -475,9 +485,6 @@ func (s *ModelStore) flushPendingStatusUpdates() error {
 }
 
 func (s *ModelStore) flushStatusUpdateBatch(lastSeen map[string]modelLastSeenUpdate, roomState map[string]modelRoomStateUpdate) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
 	tx, err := s.db.Begin()
 	if err != nil {
 		return err
@@ -495,7 +502,15 @@ WHERE lower(trim(host)) = lower(trim($4))
 		if err != nil {
 			return err
 		}
-		for _, update := range lastSeen {
+
+		keys := make([]string, 0, len(lastSeen))
+		for key := range lastSeen {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+
+		for _, key := range keys {
+			update := lastSeen[key]
 			if _, err := stmt.Exec(update.online, update.seenAt, update.updatedAt, update.host, update.modelKey); err != nil {
 				_ = stmt.Close()
 				return err
@@ -539,7 +554,15 @@ WHERE lower(trim(host)) = lower(trim($7))
 		if err != nil {
 			return err
 		}
-		for _, update := range roomState {
+
+		keys := make([]string, 0, len(roomState))
+		for key := range roomState {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+
+		for _, key := range keys {
+			update := roomState[key]
 			if _, err := stmt.Exec(
 				update.roomStatus,
 				update.isOnline,
@@ -741,9 +764,6 @@ LIMIT 1;
 	now := time.Now().UTC()
 	id := canonicalID(h, key)
 
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
 	_, err = s.db.Exec(`
 INSERT INTO models (
   id,input,is_url,host,path,model_key,
@@ -804,9 +824,6 @@ LIMIT 1;
 	now := time.Now().UTC()
 	id := canonicalID("", key)
 
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
 	_, err = s.db.Exec(`
 INSERT INTO models (
   id,input,is_url,host,path,model_key,
@@ -2040,9 +2057,6 @@ func (s *ModelStore) UpsertFromParsed(p ParsedModelDTO) (StoredModel, error) {
 
 	now := time.Now().UTC()
 
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
 	_, err = s.db.Exec(`
 INSERT INTO models (
   id,input,is_url,host,path,model_key,
@@ -2139,6 +2153,63 @@ WHERE id=$7;
 	return s.getByID(patch.ID)
 }
 
+func (s *ModelStore) PatchFlagsFast(patch ModelFlagsPatch) (StoredModel, error) {
+	if err := s.ensureInit(); err != nil {
+		return StoredModel{}, err
+	}
+	if patch.ID == "" {
+		return StoredModel{}, errors.New("id fehlt")
+	}
+
+	now := time.Now().UTC()
+
+	var updatedID string
+	err := s.db.QueryRow(`
+WITH cur AS (
+  SELECT id, watching, favorite, liked
+  FROM models
+  WHERE id=$1
+),
+updated AS (
+  UPDATE models AS m
+  SET
+    watching = CASE
+      WHEN $2::boolean IS NULL THEN cur.watching
+      ELSE $2::boolean
+    END,
+    favorite = CASE
+      WHEN $4::boolean IS TRUE THEN false
+      WHEN $3::boolean IS NULL THEN cur.favorite
+      ELSE $3::boolean
+    END,
+    liked = CASE
+      WHEN $4::boolean IS NOT NULL THEN $4::boolean
+      WHEN $3::boolean IS TRUE THEN false
+      ELSE cur.liked
+    END,
+    updated_at=$5
+  FROM cur
+  WHERE m.id=cur.id
+  RETURNING m.id
+)
+SELECT id FROM updated;
+`,
+		patch.ID,
+		nullableBoolPtrArg(patch.Watched),
+		nullableBoolPtrArg(patch.Favorite),
+		nullableBoolPtrArg(patch.Liked),
+		now,
+	).Scan(&updatedID)
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return StoredModel{}, errors.New("model nicht gefunden")
+		}
+		return StoredModel{}, err
+	}
+
+	return s.getByID(updatedID)
+}
+
 func (s *ModelStore) Delete(id string) error {
 	if err := s.ensureInit(); err != nil {
 		return err