diff --git a/backend/build_deb.sh b/backend/build_deb.sh index 8285d57..51fd3fb 100644 --- a/backend/build_deb.sh +++ b/backend/build_deb.sh @@ -257,11 +257,11 @@ User-Daten: Die installierte Binary liegt unter /opt/nsfwapp/nsfwapp. Beim Start wird sie nach ${NSFWAPP_HOME:-$HOME/nsfwapp}/nsfwapp kopiert und von dort ausgefuehrt. Dadurch bleiben recorder.log, recorder_settings.json, - generated/, record/ und record/done im User-Bereich. + nsfwapp.sqlite, generated/, record/ und record/done im User-Bereich. Wichtige Laufzeitprogramme: ffmpeg/ffprobe muessen fuer Recording, Teaser, Analyse und Videofunktionen verfuegbar sein. - pg_dump/pg_restore werden fuer Datenbank-Backup und Restore benoetigt. + SQLite ist die Standard-Datenbank. pg_dump/pg_restore werden nur fuer Postgres-Backup und Restore benoetigt. Der AI-Server braucht Python und die benoetigten Python-Pakete deiner ML-Umgebung. EOF @@ -272,7 +272,7 @@ Section: $SECTION Priority: $PRIORITY Architecture: $ARCH Maintainer: $MAINTAINER -Depends: ca-certificates, ffmpeg, python3, postgresql-client, xdg-utils, libnotify-bin, procps +Depends: ca-certificates, ffmpeg, python3, xdg-utils, libnotify-bin, procps Description: $DESCRIPTION_SHORT $DESCRIPTION_LONG EOF diff --git a/backend/dist/nsfwapp-linux-amd64 b/backend/dist/nsfwapp-linux-amd64 index cfd8da9..4f4091e 100644 Binary files a/backend/dist/nsfwapp-linux-amd64 and b/backend/dist/nsfwapp-linux-amd64 differ diff --git a/backend/dist/nsfwapp.exe b/backend/dist/nsfwapp.exe index 17be6bd..8ddbc5e 100644 Binary files a/backend/dist/nsfwapp.exe and b/backend/dist/nsfwapp.exe differ diff --git a/backend/dist/nsfwapp_amd64.deb b/backend/dist/nsfwapp_amd64.deb index aaa425a..5fba590 100644 Binary files a/backend/dist/nsfwapp_amd64.deb and b/backend/dist/nsfwapp_amd64.deb differ diff --git a/backend/go.mod b/backend/go.mod index 95aa51c..2bbb050 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -14,10 +14,12 @@ require ( github.com/r3labs/sse/v2 v2.10.0 github.com/yalue/onnxruntime_go v1.27.0 golang.org/x/crypto v0.50.0 + modernc.org/sqlite v1.53.0 ) require ( github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 // indirect github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 // indirect @@ -35,9 +37,12 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tinylib/msgp v1.6.4 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -47,6 +52,9 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.36.0 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect + modernc.org/libc v1.73.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) require ( @@ -56,5 +64,5 @@ require ( github.com/sqweek/dialog v0.0.0-20240226140203-065105509627 golang.org/x/image v0.37.0 golang.org/x/net v0.52.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/sys v0.44.0 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index be1de44..98359ea 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -9,6 +9,8 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBW github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 h1:NRUJuo3v3WGC/g5YiyF790gut6oQr5f3FBI88Wv0dx4= @@ -62,6 +64,10 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= @@ -74,6 +80,8 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -149,6 +157,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -157,6 +166,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -192,3 +203,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= diff --git a/backend/model_store_config.go b/backend/model_store_config.go new file mode 100644 index 0000000..6ed8933 --- /dev/null +++ b/backend/model_store_config.go @@ -0,0 +1,60 @@ +package main + +import ( + "path/filepath" + "strings" +) + +const ( + databaseTypeSQLite = "sqlite" + databaseTypePostgres = "postgres" + defaultSQLitePath = "nsfwapp.sqlite" +) + +type ModelStoreConfig struct { + Type string + DSN string + SQLitePath string +} + +func normalizeDatabaseType(raw string) string { + switch strings.ToLower(strings.TrimSpace(raw)) { + case databaseTypePostgres: + return databaseTypePostgres + default: + return databaseTypeSQLite + } +} + +func normalizeSQLitePath(raw string) string { + value := strings.TrimSpace(raw) + if value == "" { + value = defaultSQLitePath + } + return filepath.Clean(filepath.FromSlash(value)) +} + +func buildModelStoreConfigFromSettings() (ModelStoreConfig, error) { + return buildModelStoreConfigFromRecorderSettings(getSettings()) +} + +func buildModelStoreConfigFromRecorderSettings(s RecorderSettings) (ModelStoreConfig, error) { + dbType := normalizeDatabaseType(s.DatabaseType) + if dbType == databaseTypePostgres { + dsn, err := buildPostgresDSNFromRecorderSettings(s) + if err != nil { + return ModelStoreConfig{}, err + } + return ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn}, nil + } + + sqlitePath, err := resolvePathRelativeToApp(normalizeSQLitePath(s.SQLitePath)) + if err != nil { + return ModelStoreConfig{}, err + } + return ModelStoreConfig{ + Type: databaseTypeSQLite, + DSN: sqlitePath, + SQLitePath: sqlitePath, + }, nil +} diff --git a/backend/models_store.go b/backend/models_store.go index b8f7bd9..0ed14cf 100644 --- a/backend/models_store.go +++ b/backend/models_store.go @@ -6,13 +6,17 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "net/http" "net/url" + "os" + "path/filepath" "strings" "sync" "time" _ "github.com/jackc/pgx/v5/stdlib" + _ "modernc.org/sqlite" ) type StoredModel struct { @@ -91,7 +95,7 @@ type ModelFlagsPatch struct { } type ModelStore struct { - dsn string + config ModelStoreConfig db *sql.DB initOnce sync.Once @@ -115,6 +119,73 @@ func fmtNullTime(nt sql.NullTime) string { return nt.Time.UTC().Format(time.RFC3339Nano) } +type dbNullTime struct { + sql.NullTime +} + +func (nt *dbNullTime) Scan(value any) error { + if value == nil { + nt.Valid = false + nt.Time = time.Time{} + return nil + } + + switch v := value.(type) { + case time.Time: + nt.Valid = true + nt.Time = v.UTC() + return nil + case string: + return nt.scanString(v) + case []byte: + return nt.scanString(string(v)) + default: + var raw sql.NullTime + if err := raw.Scan(value); err != nil { + return err + } + nt.NullTime = raw + if nt.Valid { + nt.Time = nt.Time.UTC() + } + return nil + } +} + +func (nt *dbNullTime) scanString(value string) error { + value = strings.TrimSpace(value) + if value == "" { + nt.Valid = false + nt.Time = time.Time{} + return nil + } + + layouts := []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999-07:00", + "2006-01-02 15:04:05-07:00", + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05.999999Z07:00", + "2006-01-02 15:04:05Z07:00", + "2006-01-02 15:04:05.999999999 -0700 MST", + "2006-01-02 15:04:05.999999 -0700 MST", + "2006-01-02 15:04:05 -0700 MST", + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05.999999", + "2006-01-02 15:04:05", + } + for _, layout := range layouts { + if t, err := time.Parse(layout, value); err == nil { + nt.Valid = true + nt.Time = t.UTC() + return nil + } + } + return fmt.Errorf("invalid time value %q", value) +} + func nullableTimeArg(nt sql.NullTime) any { if !nt.Valid { return nil @@ -154,12 +225,22 @@ func parseRFC3339Nano(s string) sql.NullTime { return sql.NullTime{Valid: false} } -func NewModelStore(dsn string) *ModelStore { - return &ModelStore{dsn: strings.TrimSpace(dsn)} +func NewModelStore(config ModelStoreConfig) *ModelStore { + config.Type = normalizeDatabaseType(config.Type) + config.DSN = strings.TrimSpace(config.DSN) + config.SQLitePath = strings.TrimSpace(config.SQLitePath) + return &ModelStore{config: config} } func (s *ModelStore) Load() error { return s.ensureInit() } +func (s *ModelStore) Close() error { + if s == nil || s.db == nil { + return nil + } + return s.db.Close() +} + func (s *ModelStore) ensureInit() error { s.initOnce.Do(func() { s.initErr = s.init() @@ -168,26 +249,61 @@ func (s *ModelStore) ensureInit() error { } func (s *ModelStore) init() error { - if strings.TrimSpace(s.dsn) == "" { + config := s.config + config.Type = normalizeDatabaseType(config.Type) + + driverName := "pgx" + dsn := strings.TrimSpace(config.DSN) + if config.Type == databaseTypeSQLite { + sqlitePath := strings.TrimSpace(config.SQLitePath) + if sqlitePath == "" { + sqlitePath = dsn + } + if strings.TrimSpace(sqlitePath) == "" { + return errors.New("sqlite path fehlt") + } + info, err := os.Stat(sqlitePath) + if err == nil && info.IsDir() { + return fmt.Errorf("sqlite path ist ein ordner: %s", sqlitePath) + } + if err := os.MkdirAll(filepath.Dir(sqlitePath), 0755); err != nil { + return err + } + driverName = "sqlite" + dsn = sqlitePath + config.SQLitePath = sqlitePath + } + if strings.TrimSpace(dsn) == "" { return errors.New("db dsn fehlt") } - db, err := sql.Open("pgx", s.dsn) + db, err := sql.Open(driverName, dsn) if err != nil { return err } - db.SetMaxOpenConns(10) - db.SetMaxIdleConns(10) + if config.Type == databaseTypeSQLite { + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + } else { + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(10) + } if err := db.Ping(); err != nil { _ = db.Close() return err } - // ✅ Du hast die Tabelle schon in Postgres angelegt (mit richtigen Typen). - // Deshalb hier KEIN create/alter mehr, sonst riskierst du falsche Typen. s.db = db + s.config = config + + if config.Type == databaseTypeSQLite { + if err := s.ensureSQLiteSchema(); err != nil { + _ = db.Close() + return err + } + } if err := s.normalizeNameOnlyChaturbate(); err != nil { return err @@ -196,6 +312,77 @@ func (s *ModelStore) init() error { return nil } +func (s *ModelStore) isSQLite() bool { + return s != nil && normalizeDatabaseType(s.config.Type) == databaseTypeSQLite +} + +func (s *ModelStore) blobLengthExpr() string { + if s.isSQLite() { + return "length(profile_image_blob)" + } + return "octet_length(profile_image_blob)" +} + +func (s *ModelStore) emptyTextExpr() string { + if s.isSQLite() { + return "''" + } + return "''::text" +} + +func (s *ModelStore) ensureSQLiteSchema() error { + stmts := []string{ + `PRAGMA busy_timeout = 5000;`, + `PRAGMA foreign_keys = ON;`, + `PRAGMA journal_mode = WAL;`, + `CREATE TABLE IF NOT EXISTS models ( + id TEXT PRIMARY KEY, + input TEXT NOT NULL DEFAULT '', + is_url BOOLEAN NOT NULL DEFAULT 0, + host TEXT DEFAULT '', + path TEXT DEFAULT '', + model_key TEXT NOT NULL DEFAULT '', + tags TEXT DEFAULT '', + last_stream DATETIME NULL, + last_seen_online BOOLEAN NULL, + last_seen_online_at DATETIME NULL, + cb_online_json TEXT DEFAULT '', + cb_online_fetched_at DATETIME NULL, + cb_online_last_error TEXT DEFAULT '', + profile_image_url TEXT DEFAULT '', + profile_image_mime TEXT DEFAULT '', + profile_image_blob BLOB NULL, + profile_image_updated_at DATETIME NULL, + room_status TEXT DEFAULT '', + is_online BOOLEAN NOT NULL DEFAULT 0, + chat_room_url TEXT DEFAULT '', + image_url TEXT DEFAULT '', + last_online_at DATETIME NULL, + last_offline_at DATETIME NULL, + last_room_sync_at DATETIME NULL, + biocontext_json TEXT DEFAULT '', + biocontext_fetched_at DATETIME NULL, + watching BOOLEAN NOT NULL DEFAULT 0, + favorite BOOLEAN NOT NULL DEFAULT 0, + hot BOOLEAN NOT NULL DEFAULT 0, + keep BOOLEAN NOT NULL DEFAULT 0, + liked BOOLEAN NULL, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL + );`, + `CREATE INDEX IF NOT EXISTS idx_models_host_model_key ON models(host, model_key);`, + `CREATE INDEX IF NOT EXISTS idx_models_model_key ON models(model_key);`, + `CREATE INDEX IF NOT EXISTS idx_models_watching_host ON models(watching, host);`, + `CREATE INDEX IF NOT EXISTS idx_models_updated_at ON models(updated_at);`, + } + for _, stmt := range stmts { + if _, err := s.db.Exec(stmt); err != nil { + return err + } + } + return nil +} + func (s *ModelStore) SetChaturbateRoomState( host string, modelKey string, @@ -672,7 +859,7 @@ func (s *ModelStore) GetBioContext(host, modelKey string) (jsonStr string, fetch } var js sql.NullString - var ts sql.NullTime + var ts dbNullTime err = s.db.QueryRow(` SELECT biocontext_json, biocontext_fetched_at FROM models @@ -690,9 +877,9 @@ LIMIT 1; val := strings.TrimSpace(js.String) if val == "" { - return "", fmtNullTime(ts), false, nil + return "", fmtNullTime(ts.NullTime), false, nil } - return val, fmtNullTime(ts), true, nil + return val, fmtNullTime(ts.NullTime), true, nil } func (s *ModelStore) SetBioContext(host, modelKey, jsonStr, fetchedAt string) error { @@ -799,12 +986,12 @@ WHERE is_url = false type rowT struct { oldID, key, tags string - lastStream sql.NullTime + lastStream dbNullTime watching, favorite, hot, keep bool liked sql.NullBool - createdAt, updatedAt sql.NullTime + createdAt, updatedAt dbNullTime } var items []rowT @@ -890,7 +1077,7 @@ LIMIT 1; likedArg = nil } - lastStreamArg := nullableTimeArg(it.lastStream) + lastStreamArg := nullableTimeArg(it.lastStream.NullTime) if targetID == "" { targetID = canonicalID(host, it.key) @@ -912,7 +1099,7 @@ INSERT INTO models ( return err } } else { - _, err = tx.Exec(` + updateSQL := ` UPDATE models SET input = CASE WHEN is_url=false OR input IS NULL OR trim(input)='' OR lower(trim(input))=lower(trim(model_key)) @@ -940,7 +1127,12 @@ UPDATE models SET updated_at = CASE WHEN updated_at < $12 THEN $12 ELSE updated_at END WHERE id = $13; -`, +` + if s.isSQLite() { + updateSQL = strings.ReplaceAll(updateSQL, "$6::timestamptz", "$6") + updateSQL = strings.ReplaceAll(updateSQL, "$11::boolean", "$11") + } + _, err = tx.Exec(updateSQL, newInput, host, newPath, it.tags, it.tags, lastStreamArg, @@ -969,7 +1161,10 @@ func (s *ModelStore) List() []StoredModel { return []StoredModel{} } - q1 := ` + blobLengthExpr := s.blobLengthExpr() + emptyTextExpr := s.emptyTextExpr() + + q1 := fmt.Sprintf(` SELECT id, COALESCE(input,'') as input, @@ -986,7 +1181,7 @@ SELECT COALESCE(cb_online_last_error,''), COALESCE(profile_image_url,''), profile_image_updated_at, - CASE WHEN profile_image_blob IS NOT NULL AND octet_length(profile_image_blob) > 0 THEN 1 ELSE 0 END as has_profile_image, + CASE WHEN profile_image_blob IS NOT NULL AND %s > 0 THEN 1 ELSE 0 END as has_profile_image, COALESCE(room_status,'') as room_status, COALESCE(is_online,false) as is_online, COALESCE(chat_room_url,'') as chat_room_url, @@ -998,9 +1193,9 @@ SELECT created_at, updated_at FROM models ORDER BY updated_at DESC; -` +`, blobLengthExpr) - q2 := ` + q2 := fmt.Sprintf(` SELECT id, COALESCE(input,'') as input, @@ -1014,10 +1209,10 @@ SELECT last_seen_online_at, COALESCE(cb_online_json,''), cb_online_fetched_at, - ''::text as cb_online_last_error, + %s as cb_online_last_error, COALESCE(profile_image_url,''), profile_image_updated_at, - CASE WHEN profile_image_blob IS NOT NULL AND octet_length(profile_image_blob) > 0 THEN 1 ELSE 0 END as has_profile_image, + CASE WHEN profile_image_blob IS NOT NULL AND %s > 0 THEN 1 ELSE 0 END as has_profile_image, COALESCE(room_status,'') as room_status, COALESCE(is_online,false) as is_online, COALESCE(chat_room_url,'') as chat_room_url, @@ -1029,7 +1224,7 @@ SELECT created_at, updated_at FROM models ORDER BY updated_at DESC; -` +`, emptyTextExpr, blobLengthExpr) rows, err := s.db.Query(q1) if err != nil { @@ -1050,31 +1245,31 @@ ORDER BY updated_at DESC; id, input, host, path, modelKey, tags string isURL bool - lastStream sql.NullTime + lastStream dbNullTime lastSeenOnline sql.NullBool - lastSeenOnlineAt sql.NullTime + lastSeenOnlineAt dbNullTime cbOnlineJSON string - cbOnlineFetchedAt sql.NullTime + cbOnlineFetchedAt dbNullTime cbOnlineLastError string profileImageURL string - profileImageUpdatedAt sql.NullTime + profileImageUpdatedAt dbNullTime hasProfileImage int64 roomStatus string isOnline bool chatRoomURL string imageURL string - lastOnlineAt sql.NullTime - lastOfflineAt sql.NullTime - lastRoomSyncAt sql.NullTime + lastOnlineAt dbNullTime + lastOfflineAt dbNullTime + lastRoomSyncAt dbNullTime watching, favorite, hot, keep bool liked sql.NullBool - createdAt, updatedAt time.Time + createdAt, updatedAt dbNullTime ) if err := rows.Scan( @@ -1111,24 +1306,24 @@ ORDER BY updated_at DESC; Gender: gender, Country: country, Tags: tags, - LastStream: fmtNullTime(lastStream), + LastStream: fmtNullTime(lastStream.NullTime), LastSeenOnline: ptrBoolFromNullBool(lastSeenOnline), - LastSeenOnlineAt: fmtNullTime(lastSeenOnlineAt), + LastSeenOnlineAt: fmtNullTime(lastSeenOnlineAt.NullTime), CbOnlineJSON: cbOnlineJSON, - CbOnlineFetchedAt: fmtNullTime(cbOnlineFetchedAt), + CbOnlineFetchedAt: fmtNullTime(cbOnlineFetchedAt.NullTime), CbOnlineLastError: cbOnlineLastError, ProfileImageURL: profileImageURL, - ProfileImageUpdatedAt: fmtNullTime(profileImageUpdatedAt), + ProfileImageUpdatedAt: fmtNullTime(profileImageUpdatedAt.NullTime), RoomStatus: roomStatus, IsOnline: isOnline, ChatRoomURL: chatRoomURL, ImageURL: imageURL, - LastOnlineAt: fmtNullTime(lastOnlineAt), - LastOfflineAt: fmtNullTime(lastOfflineAt), - LastRoomSyncAt: fmtNullTime(lastRoomSyncAt), + LastOnlineAt: fmtNullTime(lastOnlineAt.NullTime), + LastOfflineAt: fmtNullTime(lastOfflineAt.NullTime), + LastRoomSyncAt: fmtNullTime(lastRoomSyncAt.NullTime), Watching: watching, Favorite: favorite, @@ -1136,8 +1331,8 @@ ORDER BY updated_at DESC; Keep: keep, Liked: ptrLikedFromNullBool(liked), - CreatedAt: fmtTime(createdAt), - UpdatedAt: fmtTime(updatedAt), + CreatedAt: fmtNullTime(createdAt.NullTime), + UpdatedAt: fmtNullTime(updatedAt.NullTime), } if hasProfileImage != 0 { @@ -1156,12 +1351,12 @@ func (s *ModelStore) Meta() ModelsMeta { } var count int - var updatedAt sql.NullTime + var updatedAt dbNullTime err := s.db.QueryRow(`SELECT COUNT(*), MAX(updated_at) FROM models;`).Scan(&count, &updatedAt) if err != nil { return ModelsMeta{Count: 0, UpdatedAt: ""} } - return ModelsMeta{Count: count, UpdatedAt: fmtNullTime(updatedAt)} + return ModelsMeta{Count: count, UpdatedAt: fmtNullTime(updatedAt.NullTime)} } type ChaturbateOnlineSnapshot struct { @@ -1493,7 +1688,7 @@ func (s *ModelStore) GetChaturbateOnlineSnapshot(host, modelKey string) (*Chatur } var js sql.NullString - var fetchedAt sql.NullTime + var fetchedAt dbNullTime err := s.db.QueryRow(` SELECT cb_online_json, cb_online_fetched_at @@ -1512,15 +1707,15 @@ LIMIT 1; raw := strings.TrimSpace(js.String) if raw == "" { - return nil, fmtNullTime(fetchedAt), false, nil + return nil, fmtNullTime(fetchedAt.NullTime), false, nil } var snap ChaturbateOnlineSnapshot if err := json.Unmarshal([]byte(raw), &snap); err != nil { - return nil, fmtNullTime(fetchedAt), false, err + return nil, fmtNullTime(fetchedAt.NullTime), false, err } - return &snap, fmtNullTime(fetchedAt), true, nil + return &snap, fmtNullTime(fetchedAt.NullTime), true, nil } func nullableStringArg(s string) any { @@ -1802,35 +1997,37 @@ func (s *ModelStore) getByID(id string) (StoredModel, error) { isURL bool - lastStream sql.NullTime + lastStream dbNullTime lastSeenOnline sql.NullBool - lastSeenOnlineAt sql.NullTime + lastSeenOnlineAt dbNullTime cbOnlineJSON string - cbOnlineFetchedAt sql.NullTime + cbOnlineFetchedAt dbNullTime cbOnlineLastError string profileImageURL string - profileImageUpdatedAt sql.NullTime + profileImageUpdatedAt dbNullTime hasProfileImage int64 roomStatus string isOnline bool chatRoomURL string imageURL string - lastOnlineAt sql.NullTime - lastOfflineAt sql.NullTime - lastRoomSyncAt sql.NullTime + lastOnlineAt dbNullTime + lastOfflineAt dbNullTime + lastRoomSyncAt dbNullTime watching, favorite, hot, keep bool liked sql.NullBool - createdAt, updatedAt time.Time + createdAt, updatedAt dbNullTime ) + blobLengthExpr := s.blobLengthExpr() + // q1: mit optionaler Spalte cb_online_last_error - q1 := ` + q1 := fmt.Sprintf(` SELECT COALESCE(input,'') as input, is_url, @@ -1846,7 +2043,7 @@ SELECT COALESCE(cb_online_last_error,''), COALESCE(profile_image_url,''), profile_image_updated_at, - CASE WHEN profile_image_blob IS NOT NULL AND octet_length(profile_image_blob) > 0 THEN 1 ELSE 0 END as has_profile_image, + CASE WHEN profile_image_blob IS NOT NULL AND %s > 0 THEN 1 ELSE 0 END as has_profile_image, COALESCE(room_status,'') as room_status, COALESCE(is_online,false) as is_online, COALESCE(chat_room_url,'') as chat_room_url, @@ -1858,11 +2055,11 @@ SELECT created_at, updated_at FROM models WHERE id=$1; -` +`, blobLengthExpr) // q2: Fallback, falls cb_online_last_error nicht existiert (oder q1 aus anderen Gründen scheitert) // Wichtig: gleiche Spaltenreihenfolge + Typen kompatibel halten. - q2 := ` + q2 := fmt.Sprintf(` SELECT COALESCE(input,'') as input, is_url, @@ -1878,7 +2075,7 @@ SELECT '' as cb_online_last_error, COALESCE(profile_image_url,''), profile_image_updated_at, - CASE WHEN profile_image_blob IS NOT NULL AND octet_length(profile_image_blob) > 0 THEN 1 ELSE 0 END as has_profile_image, + CASE WHEN profile_image_blob IS NOT NULL AND %s > 0 THEN 1 ELSE 0 END as has_profile_image, COALESCE(room_status,'') as room_status, COALESCE(is_online,false) as is_online, COALESCE(chat_room_url,'') as chat_room_url, @@ -1890,7 +2087,7 @@ SELECT created_at, updated_at FROM models WHERE id=$1; -` +`, blobLengthExpr) scan := func(q string) error { return s.db.QueryRow(q, id).Scan( @@ -1946,21 +2143,21 @@ WHERE id=$1; Gender: gender, Country: country, Tags: tags, - LastStream: fmtNullTime(lastStream), + LastStream: fmtNullTime(lastStream.NullTime), LastSeenOnline: ptrBoolFromNullBool(lastSeenOnline), - LastSeenOnlineAt: fmtNullTime(lastSeenOnlineAt), + LastSeenOnlineAt: fmtNullTime(lastSeenOnlineAt.NullTime), CbOnlineJSON: cbOnlineJSON, - CbOnlineFetchedAt: fmtNullTime(cbOnlineFetchedAt), + CbOnlineFetchedAt: fmtNullTime(cbOnlineFetchedAt.NullTime), CbOnlineLastError: cbOnlineLastError, RoomStatus: roomStatus, IsOnline: isOnline, ChatRoomURL: chatRoomURL, ImageURL: imageURL, - LastOnlineAt: fmtNullTime(lastOnlineAt), - LastOfflineAt: fmtNullTime(lastOfflineAt), - LastRoomSyncAt: fmtNullTime(lastRoomSyncAt), + LastOnlineAt: fmtNullTime(lastOnlineAt.NullTime), + LastOfflineAt: fmtNullTime(lastOfflineAt.NullTime), + LastRoomSyncAt: fmtNullTime(lastRoomSyncAt.NullTime), Watching: watching, Favorite: favorite, @@ -1968,11 +2165,11 @@ WHERE id=$1; Keep: keep, Liked: ptrLikedFromNullBool(liked), - CreatedAt: fmtTime(createdAt), - UpdatedAt: fmtTime(updatedAt), + CreatedAt: fmtNullTime(createdAt.NullTime), + UpdatedAt: fmtNullTime(updatedAt.NullTime), ProfileImageURL: profileImageURL, - ProfileImageUpdatedAt: fmtNullTime(profileImageUpdatedAt), + ProfileImageUpdatedAt: fmtNullTime(profileImageUpdatedAt.NullTime), } if hasProfileImage != 0 { diff --git a/backend/models_store_sqlite_test.go b/backend/models_store_sqlite_test.go new file mode 100644 index 0000000..57d7119 --- /dev/null +++ b/backend/models_store_sqlite_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestModelStoreSQLiteCreatesSchemaAndPersists(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "models.sqlite") + store := NewModelStore(ModelStoreConfig{ + Type: databaseTypeSQLite, + DSN: dbPath, + SQLitePath: dbPath, + }) + defer store.Close() + + if err := store.Load(); err != nil { + t.Fatalf("load sqlite store: %v", err) + } + if _, err := os.Stat(dbPath); err != nil { + t.Fatalf("sqlite file not created: %v", err) + } + + model, err := store.EnsureByHostModelKey("chaturbate.com", "alice") + if err != nil { + t.Fatalf("ensure model: %v", err) + } + if model.ID != "chaturbate.com:alice" { + t.Fatalf("unexpected model id: %q", model.ID) + } + + if err := store.SetLastSeenOnline("chaturbate.com", "alice", true, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + t.Fatalf("set last seen online: %v", err) + } + + models := store.List() + if len(models) != 1 { + t.Fatalf("expected 1 model, got %d", len(models)) + } + if models[0].LastSeenOnline == nil || !*models[0].LastSeenOnline { + t.Fatalf("lastSeenOnline not persisted: %#v", models[0].LastSeenOnline) + } + + meta := store.Meta() + if meta.Count != 1 { + t.Fatalf("expected meta count 1, got %d", meta.Count) + } + if meta.UpdatedAt == "" { + t.Fatal("expected meta updatedAt") + } +} diff --git a/backend/routes.go b/backend/routes.go index 14340ff..00b819e 100644 --- a/backend/routes.go +++ b/backend/routes.go @@ -119,17 +119,20 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore { api.HandleFunc("/api/tasks/check-videos", checkVideosTaskHandler) // -------------------------- - // 3) ModelStore (Postgres) - // DSN kommt aus Settings: databaseUrl + gespeichertes Passwort + // 3) ModelStore // -------------------------- - dsn, err := buildPostgresDSNFromSettings() + storeConfig, err := buildModelStoreConfigFromSettings() if err != nil { - appLogln("⚠️ models DSN:", err) + appLogln("models DB config:", err) + } + if storeConfig.Type == databaseTypePostgres { + appLogln("Models DB (Postgres):", sanitizeDSNForLog(storeConfig.DSN)) + } else { + appLogln("Models DB (SQLite):", storeConfig.SQLitePath) } - appLogln("📦 Models DB (Postgres):", sanitizeDSNForLog(dsn)) - store := NewModelStore(dsn) + store := NewModelStore(storeConfig) if err := store.Load(); err != nil { appLogln("⚠️ models load:", err) } diff --git a/backend/settings.go b/backend/settings.go index a09a647..d197c07 100644 --- a/backend/settings.go +++ b/backend/settings.go @@ -14,6 +14,8 @@ import ( ) type RecorderSettings struct { + DatabaseType string `json:"databaseType"` // sqlite | postgres + SQLitePath string `json:"sqlitePath"` DatabaseURL string `json:"databaseUrl"` EncryptedDBPassword string `json:"encryptedDbPassword,omitempty"` // base64(nonce+ciphertext) @@ -65,6 +67,8 @@ type RecorderSettings struct { var ( settingsMu sync.Mutex settings = RecorderSettings{ + DatabaseType: databaseTypeSQLite, + SQLitePath: defaultSQLitePath, DatabaseURL: "", EncryptedDBPassword: "", @@ -160,6 +164,8 @@ func loadSettings() { if json.Unmarshal(b, &s) == nil { s.RecordDir = normalizeSettingsDir(s.RecordDir, defaultRecordDir, legacyDefaultRecordDir) s.DoneDir = normalizeSettingsDir(s.DoneDir, defaultDoneDir, legacyDefaultDoneDir) + s.DatabaseType = normalizeDatabaseType(s.DatabaseType) + s.SQLitePath = normalizeSQLitePath(s.SQLitePath) if strings.TrimSpace(s.FFmpegPath) != "" { s.FFmpegPath = strings.TrimSpace(s.FFmpegPath) } @@ -206,6 +212,8 @@ func loadSettings() { settingsMu.Unlock() } + s.DatabaseType = normalizeDatabaseType(s.DatabaseType) + s.SQLitePath = normalizeSQLitePath(s.SQLitePath) s.DatabaseURL = strings.TrimSpace(s.DatabaseURL) // Optional: falls in der JSON mal ein URL MIT Passwort steht (Altbestand) @@ -275,6 +283,8 @@ type RecorderSettingsPublic struct { DoneDir string `json:"doneDir"` FFmpegPath string `json:"ffmpegPath"` + DatabaseType string `json:"databaseType"` + SQLitePath string `json:"sqlitePath"` DatabaseURL string `json:"databaseUrl"` HasDBPassword bool `json:"hasDbPassword"` @@ -319,6 +329,8 @@ func toPublicSettings(s RecorderSettings) RecorderSettingsPublic { DoneDir: s.DoneDir, FFmpegPath: s.FFmpegPath, + DatabaseType: normalizeDatabaseType(s.DatabaseType), + SQLitePath: normalizeSQLitePath(s.SQLitePath), DatabaseURL: strings.TrimSpace(s.DatabaseURL), HasDBPassword: strings.TrimSpace(s.EncryptedDBPassword) != "", @@ -459,6 +471,8 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { // --- DB URL + Passwort behandeln --- // 1) Trim + in.DatabaseType = normalizeDatabaseType(in.DatabaseType) + in.SQLitePath = normalizeSQLitePath(in.SQLitePath) in.DatabaseURL = strings.TrimSpace(in.DatabaseURL) // 2) Migration: wenn in.DatabaseURL ein Passwort enthält, extrahieren @@ -499,6 +513,8 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { next.DoneDir = in.DoneDir next.FFmpegPath = in.FFmpegPath + next.DatabaseType = in.DatabaseType + next.SQLitePath = in.SQLitePath next.DatabaseURL = in.DatabaseURL next.EncryptedDBPassword = in.EncryptedDBPassword @@ -537,20 +553,22 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { next.TrainingDetectorEpochs = in.TrainingDetectorEpochs dbChanged := - strings.TrimSpace(next.DatabaseURL) != strings.TrimSpace(current.DatabaseURL) || + normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) || + normalizeSQLitePath(next.SQLitePath) != normalizeSQLitePath(current.SQLitePath) || + strings.TrimSpace(next.DatabaseURL) != strings.TrimSpace(current.DatabaseURL) || strings.TrimSpace(next.EncryptedDBPassword) != strings.TrimSpace(current.EncryptedDBPassword) var newStore *ModelStore // DB erst prüfen, BEVOR gespeichert wird if dbChanged { - dsn, err := buildPostgresDSNFromRecorderSettings(next) + storeConfig, err := buildModelStoreConfigFromRecorderSettings(next) if err != nil { http.Error(w, "ungültige Datenbank-Konfiguration: "+err.Error(), http.StatusBadRequest) return } - newStore = NewModelStore(dsn) + newStore = NewModelStore(storeConfig) if err := newStore.Load(); err != nil { http.Error(w, "Datenbank-Verbindung fehlgeschlagen: "+err.Error(), http.StatusBadRequest) return diff --git a/backend/settings_database_backup.go b/backend/settings_database_backup.go index 23635a7..f444b6e 100644 --- a/backend/settings_database_backup.go +++ b/backend/settings_database_backup.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "database/sql" "errors" "fmt" "io" @@ -126,6 +127,16 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) { return } + storeConfig, err := buildModelStoreConfigFromSettings() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if storeConfig.Type == databaseTypeSQLite { + settingsSQLiteDatabaseBackupHandler(w, r, storeConfig) + return + } + cfg, _, err := postgresToolConfigFromSettings() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -203,13 +214,65 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, filename, info.ModTime(), f) } +func settingsSQLiteDatabaseBackupHandler(w http.ResponseWriter, r *http.Request, storeConfig ModelStoreConfig) { + tmp, err := os.CreateTemp("", "nsfwapp-db-backup-*.sqlite") + if err != nil { + http.Error(w, "Backup-Datei konnte nicht erstellt werden: "+err.Error(), http.StatusInternalServerError) + return + } + tmpPath := tmp.Name() + _ = tmp.Close() + _ = os.Remove(tmpPath) + defer os.Remove(tmpPath) + + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute) + defer cancel() + + db, err := sql.Open("sqlite", storeConfig.SQLitePath) + if err != nil { + http.Error(w, "SQLite konnte nicht geoeffnet werden: "+err.Error(), http.StatusInternalServerError) + return + } + defer db.Close() + + if _, err := db.ExecContext(ctx, `PRAGMA busy_timeout = 5000;`); err != nil { + http.Error(w, "SQLite Backup konnte nicht vorbereitet werden: "+err.Error(), http.StatusInternalServerError) + return + } + if _, err := db.ExecContext(ctx, "VACUUM INTO "+sqliteLiteral(tmpPath)); err != nil { + http.Error(w, "SQLite Backup fehlgeschlagen: "+err.Error(), http.StatusInternalServerError) + return + } + + info, err := os.Stat(tmpPath) + if err != nil { + http.Error(w, "Backup-Datei konnte nicht gelesen werden: "+err.Error(), http.StatusInternalServerError) + return + } + + f, err := os.Open(tmpPath) + if err != nil { + http.Error(w, "Backup-Datei konnte nicht geoeffnet werden: "+err.Error(), http.StatusInternalServerError) + return + } + defer f.Close() + + filename := "nsfwapp-db-backup-" + time.Now().Format("20060102-150405") + ".sqlite" + w.Header().Set("Content-Type", "application/vnd.sqlite3") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename)) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size())) + + http.ServeContent(w, r, filename, info.ModTime(), f) +} + func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Nur POST erlaubt", http.StatusMethodNotAllowed) return } - cfg, dsn, err := postgresToolConfigFromSettings() + storeConfig, err := buildModelStoreConfigFromSettings() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -254,6 +317,33 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { return } + if storeConfig.Type == databaseTypeSQLite { + if err := restoreSQLiteDatabase(storeConfig, tmpPath); err != nil { + http.Error(w, "SQLite Restore fehlgeschlagen: "+err.Error(), http.StatusInternalServerError) + return + } + + refreshedStore := NewModelStore(storeConfig) + if err := refreshedStore.Load(); err != nil { + appLogln("Models DB nach Restore konnte nicht neu verbunden werden:", err) + } else { + setModelStore(refreshedStore) + } + + writeJSON(w, http.StatusOK, map[string]any{ + "ok": true, + "fileName": strings.TrimSpace(header.Filename), + "bytes": written, + }) + return + } + + cfg, dsn, err := postgresToolConfigFromSettings() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute) defer cancel() @@ -274,7 +364,7 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { return } - refreshedStore := NewModelStore(dsn) + refreshedStore := NewModelStore(ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn}) if err := refreshedStore.Load(); err != nil { appLogln("⚠️ Models DB nach Restore konnte nicht neu verbunden werden:", err) } else { @@ -287,3 +377,88 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { "bytes": written, }) } + +func restoreSQLiteDatabase(storeConfig ModelStoreConfig, restorePath string) error { + restorePath = filepath.Clean(strings.TrimSpace(restorePath)) + if restorePath == "" { + return errors.New("restore path fehlt") + } + if err := validateSQLiteDatabaseFile(restorePath); err != nil { + return err + } + + targetPath := filepath.Clean(strings.TrimSpace(storeConfig.SQLitePath)) + if targetPath == "" { + return errors.New("sqlite path fehlt") + } + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + return err + } + + oldStore := getModelStore() + if oldStore != nil && oldStore.isSQLite() && oldStore.db != nil { + _, _ = oldStore.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`) + _ = oldStore.Close() + } + + if _, err := os.Stat(targetPath); err == nil { + backupPath := targetPath + ".before-restore-" + time.Now().Format("20060102-150405") + ".bak" + if _, err := copyDatabaseFile(targetPath, backupPath); err != nil { + return appErrorf("Sicherung der bisherigen SQLite-Datei fehlgeschlagen: %w", err) + } + } + + _ = os.Remove(targetPath + "-wal") + _ = os.Remove(targetPath + "-shm") + _ = os.Remove(targetPath) + + if _, err := copyDatabaseFile(restorePath, targetPath); err != nil { + return err + } + return nil +} + +func validateSQLiteDatabaseFile(path string) error { + db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer db.Close() + + if err := db.Ping(); err != nil { + return err + } + + var name string + err = db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='models' LIMIT 1;`).Scan(&name) + if errors.Is(err, sql.ErrNoRows) { + return errors.New("SQLite-Datei enthaelt keine models-Tabelle") + } + return err +} + +func copyDatabaseFile(src, dst string) (int64, error) { + in, err := os.Open(src) + if err != nil { + return 0, err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return 0, err + } + written, copyErr := io.Copy(out, in) + closeErr := out.Close() + if copyErr != nil { + return written, copyErr + } + if closeErr != nil { + return written, closeErr + } + return written, nil +} + +func sqliteLiteral(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} diff --git a/frontend/src/components/ui/RecorderSettings.tsx b/frontend/src/components/ui/RecorderSettings.tsx index c52ceb1..4150980 100644 --- a/frontend/src/components/ui/RecorderSettings.tsx +++ b/frontend/src/components/ui/RecorderSettings.tsx @@ -13,6 +13,8 @@ import { CheckIcon, ChevronDownIcon, XMarkIcon } from '@heroicons/react/24/solid import { startRegistration } from '@simplewebauthn/browser' type RecorderSettings = { + databaseType?: 'sqlite' | 'postgres' + sqlitePath?: string databaseUrl?: string hasDbPassword?: boolean recordDir: string @@ -279,6 +281,8 @@ function SettingsSection(props: { } const DEFAULTS: RecorderSettings = { + databaseType: 'sqlite', + sqlitePath: 'nsfwapp.sqlite', databaseUrl: '', hasDbPassword: false, recordDir: 'record', @@ -534,6 +538,15 @@ function databaseBackupFilename(contentDisposition: string | null) { return plain?.[1]?.trim() || 'nsfwapp-db-backup.dump' } +function normalizeDatabaseType(value: unknown): 'sqlite' | 'postgres' { + return String(value ?? '').trim().toLowerCase() === 'postgres' ? 'postgres' : 'sqlite' +} + +function normalizeSqlitePath(value: unknown) { + const path = String(value ?? '').trim() + return path || 'nsfwapp.sqlite' +} + function formatDatabaseBackupBytes(bytes: unknown) { const n = Number(bytes) if (!Number.isFinite(n) || n <= 0) return '' @@ -684,6 +697,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const [appLogLoading, setAppLogLoading] = useState(false) const [appLogOpen, setAppLogOpen] = useState(false) const [dbModalOpen, setDbModalOpen] = useState(false) + const [loadedDatabaseType, setLoadedDatabaseType] = useState<'sqlite' | 'postgres'>('sqlite') + const [loadedSqlitePath, setLoadedSqlitePath] = useState('nsfwapp.sqlite') const [loadedDatabaseUrl, setLoadedDatabaseUrl] = useState('') const [pendingDbPassword, setPendingDbPassword] = useState('') // wird nur beim Speichern gesendet const [dbBackupBusy, setDbBackupBusy] = useState(false) @@ -693,9 +708,15 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const now = Date.now() const saveSucceeded = saveSuccessUntilMs > now const saveFailed = Boolean(err) && !saving && !saveSucceeded + const currentDatabaseType = normalizeDatabaseType((value as any).databaseType) + const currentSqlitePath = normalizeSqlitePath((value as any).sqlitePath) const currentDatabaseUrl = String((value as any).databaseUrl ?? '').trim() const databaseSettingsDirty = - currentDatabaseUrl !== loadedDatabaseUrl || Boolean((pendingDbPassword || '').trim()) + currentDatabaseType !== loadedDatabaseType || + currentSqlitePath !== loadedSqlitePath || + currentDatabaseUrl !== loadedDatabaseUrl || + Boolean((pendingDbPassword || '').trim()) + const databaseConfigured = currentDatabaseType === 'sqlite' ? Boolean(loadedSqlitePath) : Boolean(loadedDatabaseUrl) const dbMaintenanceBusy = dbBackupBusy || dbRestoreBusy const [authStatus, setAuthStatus] = useState(null) @@ -834,7 +855,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { }) .then((data: RecorderSettings) => { if (!alive) return + const databaseType = normalizeDatabaseType((data as any).databaseType) + const sqlitePath = normalizeSqlitePath((data as any).sqlitePath) setValue({ + databaseType, + sqlitePath, databaseUrl: String((data as any).databaseUrl ?? ''), hasDbPassword: Boolean((data as any).hasDbPassword ?? false), recordDir: (data.recordDir || DEFAULTS.recordDir).toString(), @@ -885,6 +910,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { trainingDetectorEpochs: (data as any).trainingDetectorEpochs ?? DEFAULTS.trainingDetectorEpochs, }) + setLoadedDatabaseType(databaseType) + setLoadedSqlitePath(sqlitePath) setLoadedDatabaseUrl(String((data as any).databaseUrl ?? '').trim()) }) .catch(() => { @@ -1235,8 +1262,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setDbMaintenanceMsg(null) setDbMaintenanceErr(null) - if (!loadedDatabaseUrl) { - setDbMaintenanceErr('Bitte zuerst eine Datenbank-Verbindung speichern.') + if (!databaseConfigured) { + setDbMaintenanceErr('Bitte zuerst eine Datenbank-Konfiguration speichern.') return } if (databaseSettingsDirty) { @@ -1284,8 +1311,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setDbMaintenanceMsg(null) setDbMaintenanceErr(null) - if (!loadedDatabaseUrl) { - setDbMaintenanceErr('Bitte zuerst eine Datenbank-Verbindung speichern.') + if (!databaseConfigured) { + setDbMaintenanceErr('Bitte zuerst eine Datenbank-Konfiguration speichern.') return } if (databaseSettingsDirty) { @@ -1320,7 +1347,12 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { window.dispatchEvent(new Event('models-changed')) window.dispatchEvent( new CustomEvent('models-db-changed', { - detail: { databaseUrl: loadedDatabaseUrl, restored: true }, + detail: { + databaseType: loadedDatabaseType, + sqlitePath: loadedSqlitePath, + databaseUrl: loadedDatabaseUrl, + restored: true, + }, }) ) } catch (e: any) { @@ -1337,6 +1369,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const recordDir = value.recordDir.trim() const doneDir = value.doneDir.trim() const ffmpegPath = (value.ffmpegPath ?? '').trim() + const databaseType = normalizeDatabaseType((value as any).databaseType) + const sqlitePath = normalizeSqlitePath((value as any).sqlitePath) const databaseUrl = String((value as any).databaseUrl ?? '').trim() const hadPendingDbPassword = Boolean((pendingDbPassword || '').trim()) @@ -1344,6 +1378,14 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setErr('Bitte Aufnahme-Ordner und Ziel-Ordner angeben.') return } + if (databaseType === 'sqlite' && !sqlitePath) { + setErr('Bitte SQLite-Datei angeben.') + return + } + if (databaseType === 'postgres' && !databaseUrl) { + setErr('Bitte Database URL angeben oder SQLite auswaehlen.') + return + } // ✅ Switch-Logik: Autostart nur sinnvoll, wenn Auto-Add aktiv ist const autoAddToDownloadList = !!value.autoAddToDownloadList @@ -1395,6 +1437,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ + databaseType, + sqlitePath, databaseUrl, dbPassword: pendingDbPassword || undefined, recordDir, @@ -1444,15 +1488,20 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { }, 2500) window.dispatchEvent(new CustomEvent('recorder-settings-updated')) - const databaseUrlChanged = - databaseUrl !== loadedDatabaseUrl || hadPendingDbPassword + const databaseChanged = + databaseType !== loadedDatabaseType || + sqlitePath !== loadedSqlitePath || + databaseUrl !== loadedDatabaseUrl || + hadPendingDbPassword - if (databaseUrlChanged) { + if (databaseChanged) { window.dispatchEvent( new CustomEvent('models-db-changed', { - detail: { databaseUrl }, + detail: { databaseType, sqlitePath, databaseUrl }, }) ) + setLoadedDatabaseType(databaseType) + setLoadedSqlitePath(sqlitePath) setLoadedDatabaseUrl(databaseUrl) setPendingDbPassword('') } @@ -2125,12 +2174,68 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
Datenbank
- Postgres Verbindung. Passwort wird verschlüsselt gespeichert. + SQLite-Datei als portable Standard-Datenbank. Postgres bleibt optional verfuegbar.
+ + +
+
+ {(['sqlite', 'postgres'] as const).map((type) => { + const active = currentDatabaseType === type + return ( + + ) + })} +
+
+
+ + {currentDatabaseType === 'sqlite' ? ( +
+ + +
+ setValue((v) => ({ ...v, sqlitePath: e.target.value }))} + placeholder="nsfwapp.sqlite" + className="w-full rounded-lg px-3 py-2 text-sm bg-white text-gray-900 ring-1 ring-gray-200 + focus:outline-none focus:ring-2 focus:ring-indigo-500 + dark:bg-white/10 dark:text-white dark:ring-white/10" + /> +
+ Relativ zur App oder absolut, z.B. /home/chris/nsfwapp/nsfwapp.sqlite. +
+
+
+ ) : null} + +
@@ -2158,7 +2263,9 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { Backup / Restore
- Exportiert die gespeicherte PostgreSQL-Datenbank als pg_dump-Datei oder stellt sie aus einem Backup wieder her. + {currentDatabaseType === 'sqlite' + ? 'Exportiert die aktive SQLite-Datei oder stellt sie aus einem SQLite-Backup wieder her.' + : 'Exportiert die gespeicherte PostgreSQL-Datenbank als pg_dump-Datei oder stellt sie aus einem Backup wieder her.'}
{databaseSettingsDirty ? (
@@ -2172,7 +2279,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { variant="secondary" color="emerald" isLoading={dbBackupBusy} - disabled={saving || dbMaintenanceBusy || !loadedDatabaseUrl || databaseSettingsDirty} + disabled={saving || dbMaintenanceBusy || !databaseConfigured || databaseSettingsDirty} onClick={() => void downloadDatabaseBackup()} > Backup herunterladen @@ -2182,7 +2289,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { variant="secondary" color="amber" isLoading={dbRestoreBusy} - disabled={saving || dbMaintenanceBusy || !loadedDatabaseUrl || databaseSettingsDirty} + disabled={saving || dbMaintenanceBusy || !databaseConfigured || databaseSettingsDirty} onClick={chooseDatabaseRestoreFile} > Restore @@ -2193,7 +2300,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { { const file = e.currentTarget.files?.[0] ?? null