diff --git a/backend/build_deb.sh b/backend/build_deb.sh index 0e7fc63..210fe27 100644 --- a/backend/build_deb.sh +++ b/backend/build_deb.sh @@ -112,7 +112,7 @@ if [ ! -f "$runtime_bin" ] || ! cmp -s "$install_dir/nsfwapp" "$runtime_bin"; th mv "$tmp" "$runtime_bin" fi -setup_ml_python() { +configure_ml_python_setup() { if [ "${NSFWAPP_SKIP_ML_SETUP:-0}" = "1" ]; then return 0 fi @@ -125,45 +125,15 @@ setup_ml_python() { return 0 fi - venv_dir="${NSFWAPP_ML_VENV:-$app_dir/ml-venv}" - venv_python="$venv_dir/bin/python" - requirements_marker="$venv_dir/.requirements-ml.txt" - setup_log="$app_dir/ml-setup.log" - - if [ ! -x "$venv_python" ]; then - echo "Erstelle Python-ML-Umgebung: $venv_dir" >&2 - if ! python3 -m venv "$venv_dir" >>"$setup_log" 2>&1; then - return 1 - fi - fi - - if [ ! -f "$requirements_marker" ] || ! cmp -s "$requirements_file" "$requirements_marker"; then - echo "Installiere Python-ML-Pakete. Details: $setup_log" >&2 - if command -v notify-send >/dev/null 2>&1; then - notify-send "NSFW App" "ML-Python-Pakete werden installiert. Das kann beim ersten Start dauern." || true - fi - if ! "$venv_python" -m pip install --upgrade pip >>"$setup_log" 2>&1; then - return 1 - fi - if ! "$venv_python" -m pip install -r "$requirements_file" >>"$setup_log" 2>&1; then - return 1 - fi - if ! cp "$requirements_file" "$requirements_marker"; then - return 1 - fi - fi - - AI_SERVER_PYTHON="$venv_python" - export AI_SERVER_PYTHON + NSFWAPP_ML_SETUP=1 + NSFWAPP_ML_REQUIREMENTS="$requirements_file" + NSFWAPP_ML_VENV="${NSFWAPP_ML_VENV:-$app_dir/ml-venv}" + NSFWAPP_ML_SETUP_LOG="${NSFWAPP_ML_SETUP_LOG:-$app_dir/ml-setup.log}" + export NSFWAPP_ML_SETUP NSFWAPP_ML_REQUIREMENTS NSFWAPP_ML_VENV NSFWAPP_ML_SETUP_LOG return 0 } -if ! setup_ml_python; then - echo "Warnung: ML-Python-Umgebung konnte nicht vorbereitet werden. Details: $app_dir/ml-setup.log" >&2 - if command -v notify-send >/dev/null 2>&1; then - notify-send "NSFW App" "ML-Python-Umgebung konnte nicht vorbereitet werden. Siehe ml-setup.log." || true - fi -fi +configure_ml_python_setup cd "$app_dir" exec "$runtime_bin" "$@" @@ -313,11 +283,11 @@ User-Daten: Die installierte Binary liegt unter /opt/nsfwapp/nsfwapp. Beim Start wird sie nach ${NSFWAPP_HOME:-$HOME/nsfwapp}/nsfwapp kopiert und von dort ausgefuehrt. Dadurch bleiben recorder.log, recorder_settings.json, - nsfwapp.sqlite, generated/, record/ und record/done im User-Bereich. + recorder_settings.json, PostgreSQL-Datenordner, generated/, record/ und record/done im User-Bereich. Wichtige Laufzeitprogramme: ffmpeg/ffprobe muessen fuer Recording, Teaser, Analyse und Videofunktionen verfuegbar sein. - SQLite ist die Standard-Datenbank. pg_dump/pg_restore werden nur fuer Postgres-Backup und Restore benoetigt. + PostgreSQL ist die Datenbank. Fuer lokale Datenordner werden initdb/pg_ctl und fuer Migration/Backup pg_dump/pg_restore benoetigt. Der Launcher erstellt beim ersten Start automatisch ${NSFWAPP_HOME:-$HOME/nsfwapp}/ml-venv und installiert die mitgelieferte requirements-ml.txt. Details stehen in ml-setup.log. Setze AI_SERVER_PYTHON, wenn du eine eigene Python-Umgebung verwenden willst. @@ -330,7 +300,7 @@ Section: $SECTION Priority: $PRIORITY Architecture: $ARCH Maintainer: $MAINTAINER -Depends: ca-certificates, ffmpeg, python3, python3-venv, xdg-utils, libnotify-bin, procps +Depends: ca-certificates, ffmpeg, python3, python3-venv, xdg-utils, libnotify-bin, procps, postgresql Description: $DESCRIPTION_SHORT $DESCRIPTION_LONG EOF diff --git a/backend/dist/nsfwapp-linux-amd64 b/backend/dist/nsfwapp-linux-amd64 index 914d495..829fe6d 100644 Binary files a/backend/dist/nsfwapp-linux-amd64 and b/backend/dist/nsfwapp-linux-amd64 differ diff --git a/backend/dist/nsfwapp.exe b/backend/dist/nsfwapp.exe index 49d2e21..02163f4 100644 Binary files a/backend/dist/nsfwapp.exe and b/backend/dist/nsfwapp.exe differ diff --git a/backend/dist/nsfwapp_amd64.deb b/backend/dist/nsfwapp_amd64.deb index dfa6ef3..c4759e0 100644 Binary files a/backend/dist/nsfwapp_amd64.deb and b/backend/dist/nsfwapp_amd64.deb differ diff --git a/backend/go.mod b/backend/go.mod index 2bbb050..c39cdce 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,7 +3,6 @@ module nsfwapp go 1.25.3 require ( - github.com/PuerkitoBio/goquery v1.11.0 github.com/getlantern/systray v1.2.2 github.com/go-webauthn/webauthn v0.17.2 github.com/google/uuid v1.6.0 @@ -14,12 +13,10 @@ require ( github.com/r3labs/sse/v2 v2.10.0 github.com/yalue/onnxruntime_go v1.27.0 golang.org/x/crypto v0.50.0 - modernc.org/sqlite v1.53.0 ) require ( github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 // indirect github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 // indirect @@ -37,12 +34,9 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tinylib/msgp v1.6.4 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -52,14 +46,10 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/text v0.36.0 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect - modernc.org/libc v1.73.4 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect ) require ( github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf // indirect - github.com/andybalholm/cascadia v1.3.3 // indirect github.com/shirou/gopsutil/v3 v3.24.5 github.com/sqweek/dialog v0.0.0-20240226140203-065105509627 golang.org/x/image v0.37.0 diff --git a/backend/go.sum b/backend/go.sum index 98359ea..e3a5309 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,16 +1,10 @@ -github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= -github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ= github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf h1:FPsprx82rdrX2jiKyS17BH6IrTmUBYqZa/CXT4uvb+I= github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf/go.mod h1:peYoMncQljjNS6tZwI9WVyQB3qZS6u79/N3mBOcnd3I= -github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= -github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 h1:NRUJuo3v3WGC/g5YiyF790gut6oQr5f3FBI88Wv0dx4= @@ -64,10 +58,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= -github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= @@ -80,8 +70,6 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -106,95 +94,32 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yalue/onnxruntime_go v1.27.0 h1:c1YSgDNtpf0WGtxj3YeRIb8VC5LmM1J+Ve3uHdteC1U= github.com/yalue/onnxruntime_go v1.27.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/image v0.37.0 h1:ZiRjArKI8GwxZOoEtUfhrBtaCN+4b/7709dlT6SSnQA= golang.org/x/image v0.37.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/Knetic/govaluate.v3 v3.0.0/go.mod h1:csKLBORsPbafmSCGTEh3U7Ozmsuq8ZSIlKk1bcqph0E= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= @@ -203,11 +128,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= -modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= diff --git a/backend/ml_setup.go b/backend/ml_setup.go new file mode 100644 index 0000000..068e978 --- /dev/null +++ b/backend/ml_setup.go @@ -0,0 +1,360 @@ +package main + +import ( + "bufio" + "context" + "errors" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type mlPythonSetupStatus struct { + Required bool `json:"required"` + Running bool `json:"running"` + Done bool `json:"done"` + OK bool `json:"ok"` + Progress int `json:"progress"` + Phase string `json:"phase"` + Message string `json:"message"` + Error string `json:"error,omitempty"` + LogFile string `json:"logFile,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + UpdatedAt string `json:"updatedAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` +} + +type mlPythonSetupConfig struct { + enabled bool + requirementsPath string + venvDir string + venvPython string + requirementsMark string + logPath string +} + +var ( + mlSetupMu sync.RWMutex + mlSetupRunMu sync.Mutex + mlSetupStatus mlPythonSetupStatus +) + +func initMLPythonSetupStatus() { + cfg := mlPythonSetupConfigFromEnv() + now := time.Now().UTC().Format(time.RFC3339Nano) + + mlSetupMu.Lock() + defer mlSetupMu.Unlock() + + if cfg.enabled { + mlSetupStatus = mlPythonSetupStatus{ + Required: true, + Running: false, + Done: false, + OK: false, + Progress: 0, + Phase: "pending", + Message: "ML-Python-Umgebung wartet auf Start.", + LogFile: filepath.Base(cfg.logPath), + UpdatedAt: now, + } + return + } + + mlSetupStatus = mlPythonSetupStatus{ + Required: false, + Running: false, + Done: true, + OK: true, + Progress: 100, + Phase: "skipped", + Message: "ML-Python-Setup nicht erforderlich.", + UpdatedAt: now, + } +} + +func mlPythonSetupConfigFromEnv() mlPythonSetupConfig { + if strings.TrimSpace(os.Getenv("NSFWAPP_SKIP_ML_SETUP")) == "1" { + return mlPythonSetupConfig{} + } + if strings.TrimSpace(os.Getenv("AI_SERVER_PYTHON")) != "" { + return mlPythonSetupConfig{} + } + if strings.TrimSpace(os.Getenv("NSFWAPP_ML_SETUP")) != "1" { + return mlPythonSetupConfig{} + } + + appDir := strings.TrimSpace(os.Getenv("NSFWAPP_HOME")) + if appDir == "" { + if wd, err := os.Getwd(); err == nil { + appDir = wd + } + } + if appDir == "" { + appDir = "." + } + + requirementsPath := strings.TrimSpace(os.Getenv("NSFWAPP_ML_REQUIREMENTS")) + if requirementsPath == "" { + requirementsPath = filepath.Join(appDir, "requirements-ml.txt") + } + + venvDir := strings.TrimSpace(os.Getenv("NSFWAPP_ML_VENV")) + if venvDir == "" { + venvDir = filepath.Join(appDir, "ml-venv") + } + + logPath := strings.TrimSpace(os.Getenv("NSFWAPP_ML_SETUP_LOG")) + if logPath == "" { + logPath = filepath.Join(appDir, "ml-setup.log") + } + + return mlPythonSetupConfig{ + enabled: true, + requirementsPath: filepath.Clean(requirementsPath), + venvDir: filepath.Clean(venvDir), + venvPython: filepath.Join(filepath.Clean(venvDir), "bin", "python"), + requirementsMark: filepath.Join(filepath.Clean(venvDir), ".requirements-ml.txt"), + logPath: filepath.Clean(logPath), + } +} + +func mlPythonSetupStatusSnapshot() mlPythonSetupStatus { + mlSetupMu.RLock() + defer mlSetupMu.RUnlock() + return mlSetupStatus +} + +func updateMLPythonSetupStatus(update func(*mlPythonSetupStatus)) { + mlSetupMu.Lock() + defer mlSetupMu.Unlock() + + update(&mlSetupStatus) + mlSetupStatus.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) +} + +func mlPythonSetupStatusHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Nur GET erlaubt", http.StatusMethodNotAllowed) + return + } + writeJSON(w, http.StatusOK, mlPythonSetupStatusSnapshot()) +} + +func ensureMLPythonSetup(ctx context.Context) error { + cfg := mlPythonSetupConfigFromEnv() + if !cfg.enabled { + initMLPythonSetupStatus() + return nil + } + + mlSetupRunMu.Lock() + defer mlSetupRunMu.Unlock() + + if strings.TrimSpace(os.Getenv("AI_SERVER_PYTHON")) != "" { + initMLPythonSetupStatus() + return nil + } + + startedAt := time.Now().UTC().Format(time.RFC3339Nano) + updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) { + s.Required = true + s.Running = true + s.Done = false + s.OK = false + s.Progress = 1 + s.Phase = "prepare" + s.Message = "ML-Python-Umgebung wird vorbereitet." + s.Error = "" + s.LogFile = filepath.Base(cfg.logPath) + s.StartedAt = startedAt + s.FinishedAt = "" + }) + + reqBytes, err := os.ReadFile(cfg.requirementsPath) + if err != nil { + return finishMLPythonSetupError("Requirements konnten nicht gelesen werden.", err) + } + + if err := os.MkdirAll(cfg.venvDir, 0755); err != nil { + return finishMLPythonSetupError("ML-Venv-Ordner konnte nicht erstellt werden.", err) + } + if err := os.MkdirAll(filepath.Dir(cfg.logPath), 0755); err != nil { + return finishMLPythonSetupError("ML-Setup-Logordner konnte nicht erstellt werden.", err) + } + + if current, err := os.ReadFile(cfg.requirementsMark); err == nil && string(current) == string(reqBytes) { + if fi, statErr := os.Stat(cfg.venvPython); statErr == nil && fi != nil && !fi.IsDir() { + _ = os.Setenv("AI_SERVER_PYTHON", cfg.venvPython) + finishMLPythonSetupSuccess("ML-Python-Pakete sind bereits installiert.") + return nil + } + } + + if fi, err := os.Stat(cfg.venvPython); err != nil || fi == nil || fi.IsDir() { + python3, lookErr := exec.LookPath("python3") + if lookErr != nil { + return finishMLPythonSetupError("python3 wurde nicht gefunden.", lookErr) + } + if err := runMLPythonSetupCommand(ctx, cfg, "venv", "Python-Umgebung wird erstellt.", 5, 20, python3, "-m", "venv", cfg.venvDir); err != nil { + return finishMLPythonSetupError("Python-Umgebung konnte nicht erstellt werden.", err) + } + } + + if err := runMLPythonSetupCommand(ctx, cfg, "pip", "pip wird aktualisiert.", 22, 38, cfg.venvPython, "-m", "pip", "install", "--upgrade", "pip"); err != nil { + return finishMLPythonSetupError("pip konnte nicht aktualisiert werden.", err) + } + + if err := runMLPythonSetupCommand(ctx, cfg, "requirements", "ML-Python-Pakete werden installiert.", 40, 94, cfg.venvPython, "-m", "pip", "install", "-r", cfg.requirementsPath); err != nil { + return finishMLPythonSetupError("ML-Python-Pakete konnten nicht installiert werden.", err) + } + + if err := atomicWriteFile(cfg.requirementsMark, reqBytes); err != nil { + return finishMLPythonSetupError("ML-Requirements-Marker konnte nicht geschrieben werden.", err) + } + + _ = os.Setenv("AI_SERVER_PYTHON", cfg.venvPython) + finishMLPythonSetupSuccess("ML-Python-Pakete wurden installiert.") + return nil +} + +func runMLPythonSetupCommand(ctx context.Context, cfg mlPythonSetupConfig, phase, message string, startProgress, endProgress int, name string, args ...string) error { + updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) { + s.Running = true + s.Progress = startProgress + s.Phase = phase + s.Message = message + }) + + logFile, err := os.OpenFile(cfg.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return err + } + defer logFile.Close() + + _, _ = logFile.WriteString("\n[" + time.Now().Format(time.RFC3339) + "] " + name + " " + strings.Join(args, " ") + "\n") + + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = filepath.Dir(cfg.logPath) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + + if err := cmd.Start(); err != nil { + return err + } + + lines := make(chan string, 64) + var scanWG sync.WaitGroup + scan := func(r io.Reader) { + defer scanWG.Done() + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := scanner.Text() + _, _ = logFile.WriteString(line + "\n") + select { + case lines <- line: + case <-ctx.Done(): + return + } + } + } + + scanWG.Add(2) + go scan(stdout) + go scan(stderr) + + done := make(chan error, 1) + go func() { + scanWG.Wait() + done <- cmd.Wait() + }() + + lineCount := 0 + progressRange := endProgress - startProgress + if progressRange < 1 { + progressRange = 1 + } + + for { + select { + case line := <-lines: + line = strings.TrimSpace(line) + if line == "" { + continue + } + lineCount++ + progress := startProgress + minInt(progressRange-1, lineCount) + updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) { + s.Progress = progress + s.Phase = phase + s.Message = line + }) + + case err := <-done: + if err != nil { + return err + } + updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) { + s.Progress = endProgress + s.Phase = phase + s.Message = message + }) + return nil + + case <-ctx.Done(): + _ = cmd.Process.Kill() + return ctx.Err() + } + } +} + +func finishMLPythonSetupSuccess(message string) { + updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) { + s.Required = true + s.Running = false + s.Done = true + s.OK = true + s.Progress = 100 + s.Phase = "done" + s.Message = message + s.Error = "" + s.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano) + }) +} + +func finishMLPythonSetupError(message string, err error) error { + if err == nil { + err = errors.New(message) + } + updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) { + s.Required = true + s.Running = false + s.Done = true + s.OK = false + s.Phase = "error" + s.Message = message + s.Error = err.Error() + s.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano) + }) + return appErrorf("%s: %w", message, err) +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/backend/model_store_config.go b/backend/model_store_config.go index 6ed8933..6f79634 100644 --- a/backend/model_store_config.go +++ b/backend/model_store_config.go @@ -1,37 +1,17 @@ package main -import ( - "path/filepath" - "strings" -) - const ( - databaseTypeSQLite = "sqlite" databaseTypePostgres = "postgres" - defaultSQLitePath = "nsfwapp.sqlite" ) type ModelStoreConfig struct { - Type string - DSN string - SQLitePath string + Type string + DSN string } func normalizeDatabaseType(raw string) string { - switch strings.ToLower(strings.TrimSpace(raw)) { - case databaseTypePostgres: - return databaseTypePostgres - default: - return databaseTypeSQLite - } -} - -func normalizeSQLitePath(raw string) string { - value := strings.TrimSpace(raw) - if value == "" { - value = defaultSQLitePath - } - return filepath.Clean(filepath.FromSlash(value)) + _ = raw + return databaseTypePostgres } func buildModelStoreConfigFromSettings() (ModelStoreConfig, error) { @@ -39,22 +19,9 @@ func buildModelStoreConfigFromSettings() (ModelStoreConfig, error) { } func buildModelStoreConfigFromRecorderSettings(s RecorderSettings) (ModelStoreConfig, error) { - dbType := normalizeDatabaseType(s.DatabaseType) - if dbType == databaseTypePostgres { - dsn, err := buildPostgresDSNFromRecorderSettings(s) - if err != nil { - return ModelStoreConfig{}, err - } - return ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn}, nil - } - - sqlitePath, err := resolvePathRelativeToApp(normalizeSQLitePath(s.SQLitePath)) + dsn, err := buildPostgresDSNFromRecorderSettings(s) if err != nil { return ModelStoreConfig{}, err } - return ModelStoreConfig{ - Type: databaseTypeSQLite, - DSN: sqlitePath, - SQLitePath: sqlitePath, - }, nil + return ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn}, nil } diff --git a/backend/models_store.go b/backend/models_store.go index 8acda29..0d0fe0d 100644 --- a/backend/models_store.go +++ b/backend/models_store.go @@ -17,7 +17,6 @@ import ( "time" _ "github.com/jackc/pgx/v5/stdlib" - _ "modernc.org/sqlite" ) type StoredModel struct { @@ -69,16 +68,7 @@ type ModelsMeta struct { } const ( - sqliteMaxOpenConns = 4 - sqliteMaxIdleConns = 4 - sqliteBusyTimeoutMS = 15000 - sqliteCacheSizeKiB = 65536 - sqliteMMapSizeBytes = 268435456 - sqliteWALAutoCheckpt = 1000 - sqliteAnalyzeLimit = 1000 - - modelProfileImageDirName = "model-profile-images" - modelStatusFlushDelay = 1500 * time.Millisecond + modelStatusFlushDelay = 1500 * time.Millisecond ) // Kleine Payload für "watched" Listen (für Autostart/Abgleich) @@ -269,42 +259,9 @@ func parseRFC3339Nano(s string) sql.NullTime { func NewModelStore(config ModelStoreConfig) *ModelStore { config.Type = normalizeDatabaseType(config.Type) config.DSN = strings.TrimSpace(config.DSN) - config.SQLitePath = strings.TrimSpace(config.SQLitePath) return &ModelStore{config: config} } -func sqliteTuningPragmas() []string { - return []string{ - fmt.Sprintf("busy_timeout=%d", sqliteBusyTimeoutMS), - "foreign_keys=ON", - "journal_mode=WAL", - "synchronous=NORMAL", - "temp_store=MEMORY", - fmt.Sprintf("cache_size=-%d", sqliteCacheSizeKiB), - fmt.Sprintf("mmap_size=%d", sqliteMMapSizeBytes), - fmt.Sprintf("wal_autocheckpoint=%d", sqliteWALAutoCheckpt), - fmt.Sprintf("analysis_limit=%d", sqliteAnalyzeLimit), - } -} - -func sqliteTunedDSN(path string) string { - path = strings.TrimSpace(path) - if path == "" { - return "" - } - - q := url.Values{} - for _, pragma := range sqliteTuningPragmas() { - q.Add("_pragma", pragma) - } - - sep := "?" - if strings.Contains(path, "?") { - sep = "&" - } - return path + sep + q.Encode() -} - func (s *ModelStore) Load() error { return s.ensureInit() } func (s *ModelStore) Close() error { @@ -336,25 +293,6 @@ func (s *ModelStore) init() error { driverName := "pgx" dsn := strings.TrimSpace(config.DSN) - if config.Type == databaseTypeSQLite { - sqlitePath := strings.TrimSpace(config.SQLitePath) - if sqlitePath == "" { - sqlitePath = dsn - } - if strings.TrimSpace(sqlitePath) == "" { - return errors.New("sqlite path fehlt") - } - info, err := os.Stat(sqlitePath) - if err == nil && info.IsDir() { - return fmt.Errorf("sqlite path ist ein ordner: %s", sqlitePath) - } - if err := os.MkdirAll(filepath.Dir(sqlitePath), 0755); err != nil { - return err - } - driverName = "sqlite" - dsn = sqliteTunedDSN(sqlitePath) - config.SQLitePath = sqlitePath - } if strings.TrimSpace(dsn) == "" { return errors.New("db dsn fehlt") } @@ -364,13 +302,8 @@ func (s *ModelStore) init() error { return err } - if config.Type == databaseTypeSQLite { - db.SetMaxOpenConns(sqliteMaxOpenConns) - db.SetMaxIdleConns(sqliteMaxIdleConns) - } else { - db.SetMaxOpenConns(10) - db.SetMaxIdleConns(10) - } + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(10) if err := db.Ping(); err != nil { _ = db.Close() @@ -380,16 +313,9 @@ func (s *ModelStore) init() error { s.db = db s.config = config - if config.Type == databaseTypeSQLite { - if err := s.ensureSQLiteSchema(); err != nil { - _ = db.Close() - return err - } - } else { - if err := s.ensurePostgresSchema(); err != nil { - _ = db.Close() - return err - } + if err := s.ensurePostgresSchema(); err != nil { + _ = db.Close() + return err } if err := s.normalizeNameOnlyChaturbate(); err != nil { @@ -399,147 +325,18 @@ func (s *ModelStore) init() error { return nil } -func (s *ModelStore) isSQLite() bool { - return s != nil && normalizeDatabaseType(s.config.Type) == databaseTypeSQLite -} - func (s *ModelStore) blobLengthExpr() string { - if s.isSQLite() { - return "length(profile_image_blob)" - } return "octet_length(profile_image_blob)" } func (s *ModelStore) profileImagePresenceExpr() string { - if s.isSQLite() { - return "CASE WHEN trim(COALESCE(profile_image_path,'')) <> '' OR profile_image_blob IS NOT NULL THEN 1 ELSE 0 END" - } return fmt.Sprintf("CASE WHEN trim(COALESCE(profile_image_path,'')) <> '' OR (profile_image_blob IS NOT NULL AND %s > 0) THEN 1 ELSE 0 END", s.blobLengthExpr()) } func (s *ModelStore) emptyTextExpr() string { - if s.isSQLite() { - return "''" - } return "''::text" } -func (s *ModelStore) ensureSQLiteModelColumns() error { - rows, err := s.db.Query(`PRAGMA table_info(models);`) - if err != nil { - return err - } - defer rows.Close() - - existing := make(map[string]struct{}, 64) - for rows.Next() { - var ( - cid int - name string - colType string - notNull int - defaultV sql.NullString - primaryID int - ) - if err := rows.Scan(&cid, &name, &colType, ¬Null, &defaultV, &primaryID); err != nil { - return err - } - existing[strings.ToLower(strings.TrimSpace(name))] = struct{}{} - } - if err := rows.Err(); err != nil { - return err - } - - columns := []struct { - name string - ddl string - }{ - {name: "cb_online_last_error", ddl: "cb_online_last_error TEXT DEFAULT ''"}, - {name: "profile_image_path", ddl: "profile_image_path TEXT DEFAULT ''"}, - {name: "profile_image_hash", ddl: "profile_image_hash TEXT DEFAULT ''"}, - } - - for _, column := range columns { - if _, ok := existing[strings.ToLower(column.name)]; ok { - continue - } - if _, err := s.db.Exec(`ALTER TABLE models ADD COLUMN ` + column.ddl + `;`); err != nil { - return err - } - } - - return nil -} - -func (s *ModelStore) ensureSQLiteSchema() error { - stmts := make([]string, 0, len(sqliteTuningPragmas())+6) - for _, pragma := range sqliteTuningPragmas() { - stmts = append(stmts, "PRAGMA "+pragma+";") - } - - stmts = append(stmts, - `CREATE TABLE IF NOT EXISTS models ( - id TEXT PRIMARY KEY, - input TEXT NOT NULL DEFAULT '', - is_url BOOLEAN NOT NULL DEFAULT 0, - host TEXT DEFAULT '', - path TEXT DEFAULT '', - model_key TEXT NOT NULL DEFAULT '', - tags TEXT DEFAULT '', - last_stream DATETIME NULL, - last_seen_online BOOLEAN NULL, - last_seen_online_at DATETIME NULL, - cb_online_json TEXT DEFAULT '', - cb_online_fetched_at DATETIME NULL, - cb_online_last_error TEXT DEFAULT '', - profile_image_url TEXT DEFAULT '', - profile_image_mime TEXT DEFAULT '', - profile_image_path TEXT DEFAULT '', - profile_image_hash TEXT DEFAULT '', - profile_image_blob BLOB NULL, - profile_image_updated_at DATETIME NULL, - room_status TEXT DEFAULT '', - is_online BOOLEAN NOT NULL DEFAULT 0, - chat_room_url TEXT DEFAULT '', - image_url TEXT DEFAULT '', - last_online_at DATETIME NULL, - last_offline_at DATETIME NULL, - last_room_sync_at DATETIME NULL, - biocontext_json TEXT DEFAULT '', - biocontext_fetched_at DATETIME NULL, - watching BOOLEAN NOT NULL DEFAULT 0, - favorite BOOLEAN NOT NULL DEFAULT 0, - hot BOOLEAN NOT NULL DEFAULT 0, - keep BOOLEAN NOT NULL DEFAULT 0, - liked BOOLEAN NULL, - created_at DATETIME NOT NULL, - updated_at DATETIME NOT NULL - );`, - `CREATE INDEX IF NOT EXISTS idx_models_host_model_key ON models(host, model_key);`, - `CREATE INDEX IF NOT EXISTS idx_models_model_key ON models(model_key);`, - `CREATE INDEX IF NOT EXISTS idx_models_watching_host ON models(watching, host);`, - `CREATE INDEX IF NOT EXISTS idx_models_watching_updated ON models(watching, updated_at DESC);`, - `CREATE INDEX IF NOT EXISTS idx_models_favorite_updated ON models(favorite, updated_at DESC);`, - `CREATE INDEX IF NOT EXISTS idx_models_liked_updated ON models(liked, updated_at DESC);`, - `CREATE INDEX IF NOT EXISTS idx_models_updated_at ON models(updated_at);`, - ) - for _, stmt := range stmts { - if _, err := s.db.Exec(stmt); err != nil { - return err - } - } - if err := s.ensureSQLiteModelColumns(); err != nil { - return err - } - if err := s.migrateSQLiteProfileImageBlobsToFiles(); err != nil { - appLogln("models sqlite profile image blob migration skipped:", err) - } - if _, err := s.db.Exec(`PRAGMA optimize;`); err != nil { - return err - } - return nil -} - func (s *ModelStore) ensurePostgresSchema() error { stmts := []string{ `CREATE TABLE IF NOT EXISTS models ( @@ -1044,38 +841,11 @@ WHERE lower(trim(host)) = 'chaturbate.com' // --- Profile image cache --- -func profileImageExtForMime(mime string) string { - m := strings.ToLower(strings.TrimSpace(mime)) - if i := strings.Index(m, ";"); i >= 0 { - m = strings.TrimSpace(m[:i]) - } - switch m { - case "image/png": - return ".png" - case "image/webp": - return ".webp" - case "image/gif": - return ".gif" - case "image/jpeg", "image/jpg": - return ".jpg" - default: - return ".img" - } -} - func profileImageHash(data []byte) string { sum := sha256.Sum256(data) return fmt.Sprintf("%x", sum[:]) } -func (s *ModelStore) profileImageRelPath(hash string, mime string) string { - hash = strings.ToLower(strings.TrimSpace(hash)) - if len(hash) < 2 { - hash = "unknown" - } - return filepath.ToSlash(filepath.Join(modelProfileImageDirName, hash[:2], hash+profileImageExtForMime(mime))) -} - func (s *ModelStore) resolveProfileImagePath(storedPath string) (string, error) { storedPath = strings.TrimSpace(storedPath) if storedPath == "" { @@ -1090,109 +860,15 @@ func (s *ModelStore) resolveProfileImagePath(storedPath string) (string, error) return "", fmt.Errorf("ungueltiger profile image path: %s", storedPath) } - sqlitePath := strings.TrimSpace(s.config.SQLitePath) - if sqlitePath == "" { - sqlitePath = strings.TrimSpace(s.config.DSN) + slashPath := filepath.ToSlash(local) + if slashPath == "generated" || strings.HasPrefix(slashPath, "generated/") { + return resolvePathRelativeToApp(slashPath) } - if sqlitePath == "" { - return "", errors.New("sqlite path fehlt") - } - return filepath.Join(filepath.Dir(sqlitePath), local), nil + + return "", fmt.Errorf("ungueltiger profile image path: %s", storedPath) } -func (s *ModelStore) storeSQLiteProfileImageFile(data []byte, mime string) (relPath string, hash string, err error) { - if len(data) == 0 { - return "", "", errors.New("image data fehlt") - } - hash = profileImageHash(data) - relPath = s.profileImageRelPath(hash, mime) - absPath, err := s.resolveProfileImagePath(relPath) - if err != nil { - return "", "", err - } - if err := atomicWriteFile(absPath, data); err != nil { - return "", "", err - } - return relPath, hash, nil -} - -func (s *ModelStore) migrateSQLiteProfileImageBlobsToFiles() error { - if !s.isSQLite() { - return nil - } - - rows, err := s.db.Query(` -SELECT id, COALESCE(profile_image_mime,''), profile_image_blob -FROM models -WHERE profile_image_blob IS NOT NULL - AND (profile_image_path IS NULL OR trim(profile_image_path) = ''); -`) - if err != nil { - return err - } - defer rows.Close() - - type migratedProfileImage struct { - id string - path string - hash string - } - - migrated := make([]migratedProfileImage, 0, 32) - for rows.Next() { - var ( - id string - mime string - blob []byte - ) - if err := rows.Scan(&id, &mime, &blob); err != nil { - return err - } - if strings.TrimSpace(id) == "" || len(blob) == 0 { - continue - } - if strings.TrimSpace(mime) == "" { - mime = http.DetectContentType(blob) - } - relPath, hash, err := s.storeSQLiteProfileImageFile(blob, mime) - if err != nil { - return err - } - migrated = append(migrated, migratedProfileImage{id: id, path: relPath, hash: hash}) - } - if err := rows.Err(); err != nil { - return err - } - if len(migrated) == 0 { - return nil - } - - tx, err := s.db.Begin() - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - stmt, err := tx.Prepare(` -UPDATE models -SET profile_image_path=$1, profile_image_hash=$2, profile_image_blob=NULL -WHERE id=$3; -`) - if err != nil { - return err - } - defer stmt.Close() - - for _, item := range migrated { - if _, err := stmt.Exec(item.path, item.hash, item.id); err != nil { - return err - } - } - - return tx.Commit() -} - -// SetProfileImage speichert Bild-URL + MIME. SQLite legt die Bilddaten als Datei neben der DB ab. +// SetProfileImage speichert Bild-URL, MIME und Bilddaten in Postgres. // Speichert nur fuer bereits bestehende Models. func (s *ModelStore) SetProfileImage(host, modelKey, sourceURL, mime string, data []byte, updatedAt string) error { if err := s.ensureInit(); err != nil { @@ -1236,14 +912,6 @@ LIMIT 1; } hash := profileImageHash(data) - imagePath := "" - if s.isSQLite() { - var err error - imagePath, hash, err = s.storeSQLiteProfileImageFile(data, mime) - if err != nil { - return err - } - } nt := parseRFC3339Nano(updatedAt) if !nt.Valid { @@ -1254,22 +922,12 @@ LIMIT 1; s.mu.Lock() defer s.mu.Unlock() - var res sql.Result - if s.isSQLite() { - res, err = s.db.Exec(` -UPDATE models -SET profile_image_url=$1, profile_image_mime=$2, profile_image_path=$3, profile_image_hash=$4, profile_image_blob=NULL, profile_image_updated_at=$5, updated_at=$6 -WHERE lower(trim(host)) = lower(trim($7)) - AND lower(trim(model_key)) = lower(trim($8)); -`, src, mime, imagePath, hash, nullableTimeArg(nt), now, host, key) - } else { - res, err = s.db.Exec(` + res, err := s.db.Exec(` UPDATE models SET profile_image_url=$1, profile_image_mime=$2, profile_image_path='', profile_image_hash=$3, profile_image_blob=$4, profile_image_updated_at=$5, updated_at=$6 WHERE lower(trim(host)) = lower(trim($7)) AND lower(trim(model_key)) = lower(trim($8)); `, src, mime, hash, data, nullableTimeArg(nt), now, host, key) - } if err != nil { return err } @@ -1381,12 +1039,6 @@ LIMIT 1; } } - if s.isSQLite() { - if relPath, hash, werr := s.storeSQLiteProfileImageFile(blob, m); werr == nil { - _, _ = s.db.Exec(`UPDATE models SET profile_image_path=$1, profile_image_hash=$2, profile_image_blob=NULL WHERE id=$3;`, relPath, hash, id) - } - } - return m, blob, true, nil } @@ -1656,10 +1308,6 @@ UPDATE models SET updated_at = CASE WHEN updated_at < $12 THEN $12 ELSE updated_at END WHERE id = $13; ` - if s.isSQLite() { - updateSQL = strings.ReplaceAll(updateSQL, "$6::timestamptz", "$6") - updateSQL = strings.ReplaceAll(updateSQL, "$11::boolean", "$11") - } _, err = tx.Exec(updateSQL, newInput, host, newPath, it.tags, it.tags, diff --git a/backend/models_store_sqlite_test.go b/backend/models_store_sqlite_test.go deleted file mode 100644 index 3c98e64..0000000 --- a/backend/models_store_sqlite_test.go +++ /dev/null @@ -1,235 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestModelStoreSQLiteCreatesSchemaAndPersists(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "models.sqlite") - store := NewModelStore(ModelStoreConfig{ - Type: databaseTypeSQLite, - DSN: dbPath, - SQLitePath: dbPath, - }) - defer store.Close() - - if err := store.Load(); err != nil { - t.Fatalf("load sqlite store: %v", err) - } - if _, err := os.Stat(dbPath); err != nil { - t.Fatalf("sqlite file not created: %v", err) - } - - model, err := store.EnsureByHostModelKey("chaturbate.com", "alice") - if err != nil { - t.Fatalf("ensure model: %v", err) - } - if model.ID != "chaturbate.com:alice" { - t.Fatalf("unexpected model id: %q", model.ID) - } - - if err := store.SetLastSeenOnline("chaturbate.com", "alice", true, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { - t.Fatalf("set last seen online: %v", err) - } - if err := store.flushPendingStatusUpdates(); err != nil { - t.Fatalf("flush status updates: %v", err) - } - - models := store.List() - if len(models) != 1 { - t.Fatalf("expected 1 model, got %d", len(models)) - } - if models[0].LastSeenOnline == nil || !*models[0].LastSeenOnline { - t.Fatalf("lastSeenOnline not persisted: %#v", models[0].LastSeenOnline) - } - - meta := store.Meta() - if meta.Count != 1 { - t.Fatalf("expected meta count 1, got %d", meta.Count) - } - if meta.UpdatedAt == "" { - t.Fatal("expected meta updatedAt") - } -} - -func TestModelStoreSQLiteStoresProfileImagesAsFiles(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "models.sqlite") - store := NewModelStore(ModelStoreConfig{ - Type: databaseTypeSQLite, - DSN: dbPath, - SQLitePath: dbPath, - }) - defer store.Close() - - if err := store.Load(); err != nil { - t.Fatalf("load sqlite store: %v", err) - } - if _, err := store.EnsureByHostModelKey("chaturbate.com", "alice"); err != nil { - t.Fatalf("ensure model: %v", err) - } - data := []byte{0xff, 0xd8, 0xff, 0xdb, 0x00, 0x43, 0x00} - if err := store.SetProfileImage("chaturbate.com", "alice", "https://example.test/a.jpg", "image/jpeg", data, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { - t.Fatalf("set profile image: %v", err) - } - - var relPath, hash string - var blob []byte - if err := store.db.QueryRow(`SELECT profile_image_path, profile_image_hash, profile_image_blob FROM models WHERE id=$1;`, "chaturbate.com:alice").Scan(&relPath, &hash, &blob); err != nil { - t.Fatalf("query profile image fields: %v", err) - } - if strings.TrimSpace(relPath) == "" { - t.Fatal("expected profile image path") - } - if strings.TrimSpace(hash) == "" { - t.Fatal("expected profile image hash") - } - if len(blob) != 0 { - t.Fatalf("expected sqlite blob cleared, got %d bytes", len(blob)) - } - - absPath, err := store.resolveProfileImagePath(relPath) - if err != nil { - t.Fatalf("resolve profile image path: %v", err) - } - if b, err := os.ReadFile(absPath); err != nil { - t.Fatalf("read profile image file: %v", err) - } else if string(b) != string(data) { - t.Fatalf("unexpected profile image data: %v", b) - } - - mime, got, ok, err := store.GetProfileImageByID("chaturbate.com:alice") - if err != nil { - t.Fatalf("get profile image: %v", err) - } - if !ok || mime != "image/jpeg" || string(got) != string(data) { - t.Fatalf("unexpected profile image response ok=%v mime=%q data=%v", ok, mime, got) - } -} - -func TestMigrateModelStoreDataSQLiteToSQLiteReplacesTarget(t *testing.T) { - tmp := t.TempDir() - sourcePath := filepath.Join(tmp, "source.sqlite") - targetPath := filepath.Join(tmp, "target.sqlite") - - sourceConfig := ModelStoreConfig{Type: databaseTypeSQLite, DSN: sourcePath, SQLitePath: sourcePath} - targetConfig := ModelStoreConfig{Type: databaseTypeSQLite, DSN: targetPath, SQLitePath: targetPath} - - source := NewModelStore(sourceConfig) - defer source.Close() - if err := source.Load(); err != nil { - t.Fatalf("load source: %v", err) - } - if _, err := source.EnsureByHostModelKey("chaturbate.com", "alice"); err != nil { - t.Fatalf("ensure source model: %v", err) - } - if err := source.SetProfileImage("chaturbate.com", "alice", "https://example.test/a.jpg", "image/jpeg", []byte{1, 2, 3, 4}, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { - t.Fatalf("set profile image: %v", err) - } - if err := source.SetBioContext("chaturbate.com", "alice", `{"ok":true}`, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { - t.Fatalf("set bio context: %v", err) - } - - target := NewModelStore(targetConfig) - if err := target.Load(); err != nil { - t.Fatalf("load target: %v", err) - } - if _, err := target.EnsureByHostModelKey("chaturbate.com", "stale"); err != nil { - t.Fatalf("ensure stale target model: %v", err) - } - _ = target.Close() - _ = source.Close() - - count, err := migrateModelStoreData(sourceConfig, targetConfig) - if err != nil { - t.Fatalf("migrate sqlite to sqlite: %v", err) - } - if count != 1 { - t.Fatalf("expected 1 migrated row, got %d", count) - } - - migrated := NewModelStore(targetConfig) - defer migrated.Close() - if err := migrated.Load(); err != nil { - t.Fatalf("load migrated target: %v", err) - } - models := migrated.List() - if len(models) != 1 { - t.Fatalf("expected target replacement with 1 row, got %d", len(models)) - } - if models[0].ModelKey != "alice" { - t.Fatalf("unexpected migrated model: %q", models[0].ModelKey) - } - if models[0].ProfileImageCached == "" { - t.Fatal("expected migrated profile image blob marker") - } - bio, _, ok, err := migrated.GetBioContext("chaturbate.com", "alice") - if err != nil { - t.Fatalf("get migrated bio context: %v", err) - } - if !ok || bio != `{"ok":true}` { - t.Fatalf("unexpected migrated bio context ok=%v bio=%q", ok, bio) - } -} - -func TestMoveSQLiteDatabaseFilesMovesSidecars(t *testing.T) { - tmp := t.TempDir() - sourcePath := filepath.Join(tmp, "old", "models.sqlite") - targetPath := filepath.Join(tmp, "new", "models.sqlite") - - if err := os.MkdirAll(filepath.Dir(sourcePath), 0755); err != nil { - t.Fatalf("mkdir source: %v", err) - } - files := map[string]string{ - sourcePath: "main", - sourcePath + "-wal": "wal", - sourcePath + "-shm": "shm", - sourcePath + "-journal": "journal", - } - for path, content := range files { - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatalf("write %s: %v", path, err) - } - } - sourceImageDir := filepath.Join(filepath.Dir(sourcePath), modelProfileImageDirName) - sourceImagePath := filepath.Join(sourceImageDir, "aa", "avatar.jpg") - if err := os.MkdirAll(filepath.Dir(sourceImagePath), 0755); err != nil { - t.Fatalf("mkdir profile image dir: %v", err) - } - if err := os.WriteFile(sourceImagePath, []byte("image"), 0644); err != nil { - t.Fatalf("write profile image: %v", err) - } - - if err := moveSQLiteDatabaseFiles(sourcePath, targetPath); err != nil { - t.Fatalf("move sqlite files: %v", err) - } - - for source, content := range files { - if _, err := os.Stat(source); !os.IsNotExist(err) { - t.Fatalf("expected source removed %s, err=%v", source, err) - } - target := strings.Replace(source, sourcePath, targetPath, 1) - b, err := os.ReadFile(target) - if err != nil { - t.Fatalf("read target %s: %v", target, err) - } - if string(b) != content { - t.Fatalf("unexpected target content for %s: %q", target, string(b)) - } - } - - if _, err := os.Stat(sourceImagePath); !os.IsNotExist(err) { - t.Fatalf("expected source profile image moved, err=%v", err) - } - targetImagePath := strings.Replace(sourceImagePath, filepath.Dir(sourcePath), filepath.Dir(targetPath), 1) - b, err := os.ReadFile(targetImagePath) - if err != nil { - t.Fatalf("read target profile image: %v", err) - } - if string(b) != "image" { - t.Fatalf("unexpected target profile image content: %q", string(b)) - } -} diff --git a/backend/postgres_local.go b/backend/postgres_local.go new file mode 100644 index 0000000..11daacd --- /dev/null +++ b/backend/postgres_local.go @@ -0,0 +1,250 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +const ( + postgresModeURL = "url" + postgresModeLocal = "local" + + defaultPostgresDataDir = "postgres-data" + defaultPostgresPort = 55432 + defaultPostgresDatabase = "nsfwapp" + defaultPostgresUser = "nsfwapp" +) + +func normalizePostgresMode(raw string) string { + switch strings.ToLower(strings.TrimSpace(raw)) { + case postgresModeLocal: + return postgresModeLocal + default: + return postgresModeURL + } +} + +func normalizePostgresDataDir(raw string) string { + value := strings.TrimSpace(raw) + if value == "" { + value = defaultPostgresDataDir + } + return filepath.Clean(filepath.FromSlash(value)) +} + +func normalizePostgresPort(port int) int { + if port < 1 || port > 65535 { + return defaultPostgresPort + } + return port +} + +func normalizePostgresName(raw, fallback string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + raw = fallback + } + + var b strings.Builder + for _, r := range raw { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' { + b.WriteRune(r) + } else if b.Len() > 0 { + b.WriteByte('_') + } + } + out := strings.Trim(b.String(), "_") + if out == "" { + return fallback + } + if len(out) > 63 { + out = out[:63] + } + return out +} + +func normalizePostgresSettings(s *RecorderSettings) { + if s == nil { + return + } + s.DatabaseType = normalizeDatabaseType(s.DatabaseType) + s.PostgresMode = normalizePostgresMode(s.PostgresMode) + s.PostgresDataDir = normalizePostgresDataDir(s.PostgresDataDir) + s.PostgresPort = normalizePostgresPort(s.PostgresPort) + s.PostgresDatabase = normalizePostgresName(s.PostgresDatabase, defaultPostgresDatabase) + s.PostgresUser = normalizePostgresName(s.PostgresUser, defaultPostgresUser) + s.DatabaseURL = strings.TrimSpace(s.DatabaseURL) +} + +func postgresSettingsKey(s RecorderSettings) string { + normalizePostgresSettings(&s) + if s.PostgresMode == postgresModeLocal { + return strings.Join([]string{ + s.PostgresMode, + filepath.ToSlash(s.PostgresDataDir), + strconv.Itoa(s.PostgresPort), + s.PostgresDatabase, + s.PostgresUser, + }, "\x00") + } + return strings.Join([]string{s.PostgresMode, strings.TrimSpace(s.DatabaseURL)}, "\x00") +} + +func buildLocalPostgresDSN(s RecorderSettings) (string, error) { + normalizePostgresSettings(&s) + dataDir, err := resolvePathRelativeToApp(s.PostgresDataDir) + if err != nil { + return "", err + } + if strings.TrimSpace(dataDir) == "" { + return "", appErrorf("postgresDataDir ist leer") + } + + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + + if err := ensureLocalPostgresCluster(ctx, dataDir, s.PostgresPort, s.PostgresUser, s.PostgresDatabase); err != nil { + return "", err + } + + u := url.URL{ + Scheme: "postgres", + User: url.User(s.PostgresUser), + Host: "127.0.0.1:" + strconv.Itoa(s.PostgresPort), + Path: "/" + s.PostgresDatabase, + } + q := u.Query() + q.Set("sslmode", "disable") + u.RawQuery = q.Encode() + return u.String(), nil +} + +func ensureLocalPostgresCluster(ctx context.Context, dataDir string, port int, user, database string) error { + dataDir = filepath.Clean(strings.TrimSpace(dataDir)) + if dataDir == "" { + return appErrorf("postgresDataDir ist leer") + } + if err := os.MkdirAll(filepath.Dir(dataDir), 0755); err != nil { + return err + } + + versionFile := filepath.Join(dataDir, "PG_VERSION") + if _, err := os.Stat(versionFile); err != nil { + if !os.IsNotExist(err) { + return err + } + if err := initLocalPostgresCluster(ctx, dataDir, user); err != nil { + return err + } + } + + if err := startLocalPostgresCluster(ctx, dataDir, port); err != nil { + return err + } + return ensureLocalPostgresDatabase(ctx, port, user, database) +} + +func initLocalPostgresCluster(ctx context.Context, dataDir, user string) error { + initdb, err := postgresToolPath("initdb") + if err != nil { + return appErrorf("initdb wurde nicht gefunden. Installiere PostgreSQL-Server-Tools oder nimm initdb in PATH auf.") + } + if err := os.MkdirAll(dataDir, 0755); err != nil { + return err + } + + out, err := runLocalPostgresTool(ctx, initdb, + "-D", dataDir, + "-U", user, + "--encoding=UTF8", + "--auth-local=trust", + "--auth-host=trust", + ) + if err != nil { + return appErrorf("PostgreSQL-Datenordner konnte nicht initialisiert werden: %s", out) + } + return nil +} + +func startLocalPostgresCluster(ctx context.Context, dataDir string, port int) error { + pgCtl, err := postgresToolPath("pg_ctl") + if err != nil { + return appErrorf("pg_ctl wurde nicht gefunden. Installiere PostgreSQL-Server-Tools oder nimm pg_ctl in PATH auf.") + } + + _, _ = runLocalPostgresTool(ctx, pgCtl, "-D", dataDir, "status") + + logPath := filepath.Join(dataDir, "postgresql.log") + opts := fmt.Sprintf("-p %d -h 127.0.0.1", port) + out, err := runLocalPostgresTool(ctx, pgCtl, "-D", dataDir, "-l", logPath, "-o", opts, "-w", "start") + if err == nil { + return nil + } + if strings.Contains(strings.ToLower(out), "server is running") || strings.Contains(strings.ToLower(out), "server running") { + return nil + } + return appErrorf("lokaler PostgreSQL-Server konnte nicht gestartet werden: %s", out) +} + +func ensureLocalPostgresDatabase(ctx context.Context, port int, user, database string) error { + psql, err := postgresToolPath("psql") + if err != nil { + return appErrorf("psql wurde nicht gefunden. Installiere postgresql-client oder nimm psql in PATH auf.") + } + + sql := "SELECT 1 FROM pg_database WHERE datname = '" + strings.ReplaceAll(database, "'", "''") + "';" + out, err := runLocalPostgresTool(ctx, psql, + "-h", "127.0.0.1", + "-p", strconv.Itoa(port), + "-U", user, + "-d", "postgres", + "-tAc", sql, + ) + if err == nil && strings.TrimSpace(out) == "1" { + return nil + } + + createdb, err := postgresToolPath("createdb") + if err != nil { + return appErrorf("createdb wurde nicht gefunden. Installiere postgresql-client oder nimm createdb in PATH auf.") + } + out, err = runLocalPostgresTool(ctx, createdb, + "-h", "127.0.0.1", + "-p", strconv.Itoa(port), + "-U", user, + database, + ) + if err != nil { + return appErrorf("lokale PostgreSQL-Datenbank konnte nicht angelegt werden: %s", out) + } + return nil +} + +func runLocalPostgresTool(ctx context.Context, tool string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, tool, args...) + cmd.Env = upsertEnv(os.Environ(), "PGCONNECT_TIMEOUT", "10") + hideCommandWindow(cmd) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + + out := strings.TrimSpace(stdout.String()) + errOut := strings.TrimSpace(stderr.String()) + if errOut != "" { + if out != "" { + out += "\n" + } + out += errOut + } + return trimCommandOutput(out), err +} diff --git a/backend/routes.go b/backend/routes.go index 65a6224..22d4f12 100644 --- a/backend/routes.go +++ b/backend/routes.go @@ -32,6 +32,7 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore { mux.HandleFunc("/api/auth/passkey/login/options", authPasskeyLoginOptionsHandler(auth)) mux.HandleFunc("/api/auth/passkey/login/verify", authPasskeyLoginVerifyHandler(auth)) mux.HandleFunc("/api/auth/passkey/delete", authPasskeyDeleteHandler(auth)) + mux.HandleFunc("/api/setup/ml/status", mlPythonSetupStatusHandler) // -------------------------- // 2) Protected API Mux @@ -128,11 +129,7 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore { if err != nil { appLogln("models DB config:", err) } - if storeConfig.Type == databaseTypePostgres { - appLogln("Models DB (Postgres):", sanitizeDSNForLog(storeConfig.DSN)) - } else { - appLogln("Models DB (SQLite):", storeConfig.SQLitePath) - } + appLogln("Models DB (Postgres):", sanitizeDSNForLog(storeConfig.DSN)) store := NewModelStore(storeConfig) if err := store.Load(); err != nil { @@ -174,6 +171,10 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore { func buildPostgresDSNFromSettings() (string, error) { // Settings sind bereits durch loadSettings() in main() geladen. s := getSettings() + normalizePostgresSettings(&s) + if s.PostgresMode == postgresModeLocal { + return buildLocalPostgresDSN(s) + } dbURL := strings.TrimSpace(s.DatabaseURL) if dbURL == "" { @@ -228,6 +229,11 @@ func buildPostgresDSNFromSettings() (string, error) { } func buildPostgresDSNFromRecorderSettings(s RecorderSettings) (string, error) { + normalizePostgresSettings(&s) + if s.PostgresMode == postgresModeLocal { + return buildLocalPostgresDSN(s) + } + dbURL := strings.TrimSpace(s.DatabaseURL) if dbURL == "" { return "", appErrorf("databaseUrl ist leer") diff --git a/backend/server.go b/backend/server.go index e367bcc..a1c02ae 100644 --- a/backend/server.go +++ b/backend/server.go @@ -467,6 +467,10 @@ func startAIServer(ctx context.Context) (*aiServerProcess, error) { return nil, nil } + if err := ensureMLPythonSetup(ctx); err != nil { + return nil, err + } + scriptDir, err := findAIServerScriptDir() if err != nil { return nil, err @@ -973,6 +977,12 @@ func main() { appCtx, appCancel := context.WithCancel(context.Background()) defer appCancel() + initMLPythonSetupStatus() + + aiServerMu.Lock() + aiServerCtx = appCtx + aiServerCurrent = nil + aiServerMu.Unlock() listenAddr := ":" + appPort() @@ -988,21 +998,6 @@ func main() { } defer appLn.Close() - aiProc, err := startAIServer(appCtx) - if err != nil { - appLogln("⚠️ AI Server konnte nicht gestartet werden:", err) - } - - // Handle/Context global hinterlegen, damit der Server später neu gestartet - // werden kann (z.B. nach einem neuen Modell). - aiServerMu.Lock() - aiServerCtx = appCtx - aiServerCurrent = aiProc - watchedTrainingRoot := aiServerRoot - aiServerMu.Unlock() - - go startTrainingBestModelWatcher(appCtx, watchedTrainingRoot) - // ✅ Hier: alte manuelle Autostart-Pause beim echten App-Start zurücksetzen. resetAutostartPauseOnStartup() @@ -1132,6 +1127,26 @@ func main() { serverErrCh <- nil }() + go func() { + aiProc, err := startAIServer(appCtx) + if err != nil { + appLogln("⚠️ AI Server konnte nicht gestartet werden:", err) + } + if appCtx.Err() != nil { + if aiProc != nil { + aiProc.Stop() + } + return + } + + aiServerMu.Lock() + aiServerCurrent = aiProc + watchedTrainingRoot := aiServerRoot + aiServerMu.Unlock() + + go startTrainingBestModelWatcher(appCtx, watchedTrainingRoot) + }() + go func() { _ = showAppNotification( "Recorder gestartet", diff --git a/backend/settings.go b/backend/settings.go index b73800c..4c5d344 100644 --- a/backend/settings.go +++ b/backend/settings.go @@ -14,8 +14,12 @@ import ( ) type RecorderSettings struct { - DatabaseType string `json:"databaseType"` // sqlite | postgres - SQLitePath string `json:"sqlitePath"` + DatabaseType string `json:"databaseType"` + PostgresMode string `json:"postgresMode"` // url | local + PostgresDataDir string `json:"postgresDataDir"` + PostgresPort int `json:"postgresPort"` + PostgresDatabase string `json:"postgresDatabase"` + PostgresUser string `json:"postgresUser"` DatabaseURL string `json:"databaseUrl"` EncryptedDBPassword string `json:"encryptedDbPassword,omitempty"` // base64(nonce+ciphertext) @@ -67,8 +71,12 @@ type RecorderSettings struct { var ( settingsMu sync.Mutex settings = RecorderSettings{ - DatabaseType: databaseTypeSQLite, - SQLitePath: defaultSQLitePath, + DatabaseType: databaseTypePostgres, + PostgresMode: postgresModeURL, + PostgresDataDir: defaultPostgresDataDir, + PostgresPort: defaultPostgresPort, + PostgresDatabase: defaultPostgresDatabase, + PostgresUser: defaultPostgresUser, DatabaseURL: "", EncryptedDBPassword: "", @@ -165,7 +173,7 @@ func loadSettings() { s.RecordDir = normalizeSettingsDir(s.RecordDir, defaultRecordDir, legacyDefaultRecordDir) s.DoneDir = normalizeSettingsDir(s.DoneDir, defaultDoneDir, legacyDefaultDoneDir) s.DatabaseType = normalizeDatabaseType(s.DatabaseType) - s.SQLitePath = normalizeSQLitePath(s.SQLitePath) + normalizePostgresSettings(&s) if strings.TrimSpace(s.FFmpegPath) != "" { s.FFmpegPath = strings.TrimSpace(s.FFmpegPath) } @@ -213,7 +221,7 @@ func loadSettings() { } s.DatabaseType = normalizeDatabaseType(s.DatabaseType) - s.SQLitePath = normalizeSQLitePath(s.SQLitePath) + normalizePostgresSettings(&s) s.DatabaseURL = strings.TrimSpace(s.DatabaseURL) // Optional: falls in der JSON mal ein URL MIT Passwort steht (Altbestand) @@ -283,10 +291,14 @@ type RecorderSettingsPublic struct { DoneDir string `json:"doneDir"` FFmpegPath string `json:"ffmpegPath"` - DatabaseType string `json:"databaseType"` - SQLitePath string `json:"sqlitePath"` - DatabaseURL string `json:"databaseUrl"` - HasDBPassword bool `json:"hasDbPassword"` + DatabaseType string `json:"databaseType"` + PostgresMode string `json:"postgresMode"` + PostgresDataDir string `json:"postgresDataDir"` + PostgresPort int `json:"postgresPort"` + PostgresDatabase string `json:"postgresDatabase"` + PostgresUser string `json:"postgresUser"` + DatabaseURL string `json:"databaseUrl"` + HasDBPassword bool `json:"hasDbPassword"` AutoAddToDownloadList bool `json:"autoAddToDownloadList"` AutoStartAddedDownloads bool `json:"autoStartAddedDownloads"` @@ -324,15 +336,20 @@ type RecorderSettingsPublic struct { } func toPublicSettings(s RecorderSettings) RecorderSettingsPublic { + normalizePostgresSettings(&s) return RecorderSettingsPublic{ RecordDir: s.RecordDir, DoneDir: s.DoneDir, FFmpegPath: s.FFmpegPath, - DatabaseType: normalizeDatabaseType(s.DatabaseType), - SQLitePath: normalizeSQLitePath(s.SQLitePath), - DatabaseURL: strings.TrimSpace(s.DatabaseURL), - HasDBPassword: strings.TrimSpace(s.EncryptedDBPassword) != "", + DatabaseType: normalizeDatabaseType(s.DatabaseType), + PostgresMode: s.PostgresMode, + PostgresDataDir: s.PostgresDataDir, + PostgresPort: s.PostgresPort, + PostgresDatabase: s.PostgresDatabase, + PostgresUser: s.PostgresUser, + DatabaseURL: strings.TrimSpace(s.DatabaseURL), + HasDBPassword: strings.TrimSpace(s.EncryptedDBPassword) != "", AutoAddToDownloadList: s.AutoAddToDownloadList, AutoStartAddedDownloads: s.AutoStartAddedDownloads, @@ -472,8 +489,12 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { // --- DB URL + Passwort behandeln --- // 1) Trim in.DatabaseType = normalizeDatabaseType(in.DatabaseType) - in.SQLitePath = normalizeSQLitePath(in.SQLitePath) + normalizePostgresSettings(&in.RecorderSettings) in.DatabaseURL = strings.TrimSpace(in.DatabaseURL) + if in.PostgresMode == postgresModeURL && in.DatabaseURL == "" { + http.Error(w, "Database URL fehlt", http.StatusBadRequest) + return + } // 2) Migration: wenn in.DatabaseURL ein Passwort enthält, extrahieren // und URL ohne Passwort zurückschreiben. @@ -514,7 +535,11 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { next.FFmpegPath = in.FFmpegPath next.DatabaseType = in.DatabaseType - next.SQLitePath = in.SQLitePath + next.PostgresMode = in.PostgresMode + next.PostgresDataDir = in.PostgresDataDir + next.PostgresPort = in.PostgresPort + next.PostgresDatabase = in.PostgresDatabase + next.PostgresUser = in.PostgresUser next.DatabaseURL = in.DatabaseURL next.EncryptedDBPassword = in.EncryptedDBPassword @@ -554,7 +579,7 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { dbChanged := normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) || - normalizeSQLitePath(next.SQLitePath) != normalizeSQLitePath(current.SQLitePath) || + postgresSettingsKey(next) != postgresSettingsKey(current) || strings.TrimSpace(next.DatabaseURL) != strings.TrimSpace(current.DatabaseURL) || strings.TrimSpace(next.EncryptedDBPassword) != strings.TrimSpace(current.EncryptedDBPassword) @@ -562,21 +587,8 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { // DB erst prüfen, BEVOR gespeichert wird if dbChanged { - sqliteMoved := false - if sqlitePathChanged(current, next) { - if err := moveSQLiteDatabaseForSettings(current, next); err != nil { - http.Error(w, "SQLite-Datei konnte nicht verschoben werden: "+err.Error(), http.StatusBadRequest) - return - } - sqliteMoved = true - } - storeConfig, err := buildModelStoreConfigFromRecorderSettings(next) if err != nil { - if sqliteMoved { - _ = moveSQLiteDatabaseForSettings(next, current) - restoreModelStoreForSettings(current) - } http.Error(w, "ungültige Datenbank-Konfiguration: "+err.Error(), http.StatusBadRequest) return } @@ -584,10 +596,6 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { newStore = NewModelStore(storeConfig) if err := newStore.Load(); err != nil { _ = newStore.Close() - if sqliteMoved { - _ = moveSQLiteDatabaseForSettings(next, current) - restoreModelStoreForSettings(current) - } http.Error(w, "Datenbank-Verbindung fehlgeschlagen: "+err.Error(), http.StatusBadRequest) return } @@ -642,8 +650,8 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) { func settingsBrowse(w http.ResponseWriter, r *http.Request) { target := r.URL.Query().Get("target") - if target != "record" && target != "done" && target != "ffmpeg" && target != "sqlite" { - http.Error(w, "target muss record, done, ffmpeg oder sqlite sein", http.StatusBadRequest) + if target != "record" && target != "done" && target != "ffmpeg" && target != "postgresData" { + http.Error(w, "target muss record, done, ffmpeg oder postgresData sein", http.StatusBadRequest) return } @@ -657,9 +665,9 @@ func settingsBrowse(w http.ResponseWriter, r *http.Request) { p, err = dialog.File(). Title("ffmpeg.exe auswählen"). Load() - } else if target == "sqlite" { + } else if target == "postgresData" { p, err = dialog.Directory(). - Title("SQLite-Speicherort auswaehlen"). + Title("PostgreSQL-Datenordner auswaehlen"). Browse() } else { // Ordnerauswahl für record/done diff --git a/backend/settings_database_backup.go b/backend/settings_database_backup.go index 81e707e..8d537d9 100644 --- a/backend/settings_database_backup.go +++ b/backend/settings_database_backup.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "database/sql" "errors" "fmt" "io" @@ -127,15 +126,10 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) { return } - storeConfig, err := buildModelStoreConfigFromSettings() - if err != nil { + if _, err := buildModelStoreConfigFromSettings(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - if storeConfig.Type == databaseTypeSQLite { - settingsSQLiteDatabaseBackupHandler(w, r, storeConfig) - return - } cfg, _, err := postgresToolConfigFromSettings() if err != nil { @@ -200,7 +194,7 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) { f, err := os.Open(tmpPath) if err != nil { - http.Error(w, "Backup-Datei konnte nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError) + http.Error(w, "Backup-Datei konnte nicht geoeffnet werden: "+err.Error(), http.StatusInternalServerError) return } defer f.Close() @@ -214,66 +208,13 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, filename, info.ModTime(), f) } -func settingsSQLiteDatabaseBackupHandler(w http.ResponseWriter, r *http.Request, storeConfig ModelStoreConfig) { - tmp, err := os.CreateTemp("", "nsfwapp-db-backup-*.sqlite") - if err != nil { - http.Error(w, "Backup-Datei konnte nicht erstellt werden: "+err.Error(), http.StatusInternalServerError) - return - } - tmpPath := tmp.Name() - _ = tmp.Close() - _ = os.Remove(tmpPath) - defer os.Remove(tmpPath) - - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute) - defer cancel() - - db, err := sql.Open("sqlite", storeConfig.SQLitePath) - if err != nil { - http.Error(w, "SQLite konnte nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError) - return - } - defer db.Close() - - if _, err := db.ExecContext(ctx, `PRAGMA busy_timeout = 5000;`); err != nil { - http.Error(w, "SQLite Backup konnte nicht vorbereitet werden: "+err.Error(), http.StatusInternalServerError) - return - } - if _, err := db.ExecContext(ctx, "VACUUM INTO "+sqliteLiteral(tmpPath)); err != nil { - http.Error(w, "SQLite Backup fehlgeschlagen: "+err.Error(), http.StatusInternalServerError) - return - } - - info, err := os.Stat(tmpPath) - if err != nil { - http.Error(w, "Backup-Datei konnte nicht gelesen werden: "+err.Error(), http.StatusInternalServerError) - return - } - - f, err := os.Open(tmpPath) - if err != nil { - http.Error(w, "Backup-Datei konnte nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError) - return - } - defer f.Close() - - filename := "nsfwapp-db-backup-" + time.Now().Format("20060102-150405") + ".sqlite" - w.Header().Set("Content-Type", "application/vnd.sqlite3") - w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename)) - w.Header().Set("Cache-Control", "no-store") - w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size())) - - http.ServeContent(w, r, filename, info.ModTime(), f) -} - func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Nur POST erlaubt", http.StatusMethodNotAllowed) return } - storeConfig, err := buildModelStoreConfigFromSettings() - if err != nil { + if _, err := buildModelStoreConfigFromSettings(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -317,27 +258,6 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { return } - if storeConfig.Type == databaseTypeSQLite { - if err := restoreSQLiteDatabase(storeConfig, tmpPath); err != nil { - http.Error(w, "SQLite Restore fehlgeschlagen: "+err.Error(), http.StatusInternalServerError) - return - } - - refreshedStore := NewModelStore(storeConfig) - if err := refreshedStore.Load(); err != nil { - appLogln("Models DB nach Restore konnte nicht neu verbunden werden:", err) - } else { - setModelStore(refreshedStore) - } - - writeJSON(w, http.StatusOK, map[string]any{ - "ok": true, - "fileName": strings.TrimSpace(header.Filename), - "bytes": written, - }) - return - } - cfg, dsn, err := postgresToolConfigFromSettings() if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -366,7 +286,7 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { refreshedStore := NewModelStore(ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn}) if err := refreshedStore.Load(); err != nil { - appLogln("⚠️ Models DB nach Restore konnte nicht neu verbunden werden:", err) + appLogln("Models DB nach Restore konnte nicht neu verbunden werden:", err) } else { setModelStore(refreshedStore) } @@ -377,88 +297,3 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) { "bytes": written, }) } - -func restoreSQLiteDatabase(storeConfig ModelStoreConfig, restorePath string) error { - restorePath = filepath.Clean(strings.TrimSpace(restorePath)) - if restorePath == "" { - return errors.New("restore path fehlt") - } - if err := validateSQLiteDatabaseFile(restorePath); err != nil { - return err - } - - targetPath := filepath.Clean(strings.TrimSpace(storeConfig.SQLitePath)) - if targetPath == "" { - return errors.New("sqlite path fehlt") - } - if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { - return err - } - - oldStore := getModelStore() - if oldStore != nil && oldStore.isSQLite() && oldStore.db != nil { - _, _ = oldStore.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`) - _ = oldStore.Close() - } - - if _, err := os.Stat(targetPath); err == nil { - backupPath := targetPath + ".before-restore-" + time.Now().Format("20060102-150405") + ".bak" - if _, err := copyDatabaseFile(targetPath, backupPath); err != nil { - return appErrorf("Sicherung der bisherigen SQLite-Datei fehlgeschlagen: %w", err) - } - } - - _ = os.Remove(targetPath + "-wal") - _ = os.Remove(targetPath + "-shm") - _ = os.Remove(targetPath) - - if _, err := copyDatabaseFile(restorePath, targetPath); err != nil { - return err - } - return nil -} - -func validateSQLiteDatabaseFile(path string) error { - db, err := sql.Open("sqlite", path) - if err != nil { - return err - } - defer db.Close() - - if err := db.Ping(); err != nil { - return err - } - - var name string - err = db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='models' LIMIT 1;`).Scan(&name) - if errors.Is(err, sql.ErrNoRows) { - return errors.New("SQLite-Datei enthaelt keine models-Tabelle") - } - return err -} - -func copyDatabaseFile(src, dst string) (int64, error) { - in, err := os.Open(src) - if err != nil { - return 0, err - } - defer in.Close() - - out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return 0, err - } - written, copyErr := io.Copy(out, in) - closeErr := out.Close() - if copyErr != nil { - return written, copyErr - } - if closeErr != nil { - return written, closeErr - } - return written, nil -} - -func sqliteLiteral(value string) string { - return "'" + strings.ReplaceAll(value, "'", "''") + "'" -} diff --git a/backend/settings_database_migrate.go b/backend/settings_database_migrate.go index 74118ba..67b9768 100644 --- a/backend/settings_database_migrate.go +++ b/backend/settings_database_migrate.go @@ -1,242 +1,119 @@ package main import ( - "database/sql" + "bytes" + "context" "encoding/json" - "errors" - "fmt" "net/http" "os" + "os/exec" "path/filepath" - "strconv" "strings" "time" ) -type databaseMigrationRequest struct { - DatabaseType string `json:"databaseType"` - SQLitePath string `json:"sqlitePath"` - DatabaseURL string `json:"databaseUrl"` - DBPassword string `json:"dbPassword,omitempty"` -} - -type modelMigrationColumn struct { - name string - kind string - notNull bool -} - -const ( - modelMigrationText = "text" - modelMigrationBool = "bool" - modelMigrationTime = "time" - modelMigrationBlob = "blob" -) - -var modelMigrationColumns = []modelMigrationColumn{ - {name: "id", kind: modelMigrationText, notNull: true}, - {name: "input", kind: modelMigrationText, notNull: true}, - {name: "is_url", kind: modelMigrationBool, notNull: true}, - {name: "host", kind: modelMigrationText}, - {name: "path", kind: modelMigrationText}, - {name: "model_key", kind: modelMigrationText, notNull: true}, - {name: "tags", kind: modelMigrationText}, - {name: "last_stream", kind: modelMigrationTime}, - {name: "last_seen_online", kind: modelMigrationBool}, - {name: "last_seen_online_at", kind: modelMigrationTime}, - {name: "cb_online_json", kind: modelMigrationText}, - {name: "cb_online_fetched_at", kind: modelMigrationTime}, - {name: "cb_online_last_error", kind: modelMigrationText}, - {name: "profile_image_url", kind: modelMigrationText}, - {name: "profile_image_mime", kind: modelMigrationText}, - {name: "profile_image_path", kind: modelMigrationText}, - {name: "profile_image_hash", kind: modelMigrationText}, - {name: "profile_image_blob", kind: modelMigrationBlob}, - {name: "profile_image_updated_at", kind: modelMigrationTime}, - {name: "room_status", kind: modelMigrationText}, - {name: "is_online", kind: modelMigrationBool, notNull: true}, - {name: "chat_room_url", kind: modelMigrationText}, - {name: "image_url", kind: modelMigrationText}, - {name: "last_online_at", kind: modelMigrationTime}, - {name: "last_offline_at", kind: modelMigrationTime}, - {name: "last_room_sync_at", kind: modelMigrationTime}, - {name: "biocontext_json", kind: modelMigrationText}, - {name: "biocontext_fetched_at", kind: modelMigrationTime}, - {name: "watching", kind: modelMigrationBool, notNull: true}, - {name: "favorite", kind: modelMigrationBool, notNull: true}, - {name: "hot", kind: modelMigrationBool, notNull: true}, - {name: "keep", kind: modelMigrationBool, notNull: true}, - {name: "liked", kind: modelMigrationBool}, - {name: "created_at", kind: modelMigrationTime, notNull: true}, - {name: "updated_at", kind: modelMigrationTime, notNull: true}, -} - -type dbNullBool struct { - sql.NullBool -} - -func (b *dbNullBool) Scan(value any) error { - if value == nil { - b.Valid = false - b.Bool = false - return nil - } - - switch v := value.(type) { - case bool: - b.Valid = true - b.Bool = v - return nil - case int64: - b.Valid = true - b.Bool = v != 0 - return nil - case int: - b.Valid = true - b.Bool = v != 0 - return nil - case []byte: - return b.scanString(string(v)) - case string: - return b.scanString(v) - default: - var raw sql.NullBool - if err := raw.Scan(value); err != nil { - return err - } - b.NullBool = raw - return nil - } -} - -func (b *dbNullBool) scanString(value string) error { - value = strings.TrimSpace(strings.ToLower(value)) - if value == "" { - b.Valid = false - b.Bool = false - return nil - } - switch value { - case "1", "t", "true", "yes", "y", "on": - b.Valid = true - b.Bool = true - return nil - case "0", "f", "false", "no", "n", "off": - b.Valid = true - b.Bool = false - return nil - default: - parsed, err := strconv.ParseBool(value) - if err != nil { - return fmt.Errorf("invalid bool value %q", value) - } - b.Valid = true - b.Bool = parsed - return nil - } -} - -type modelMigrationValue struct { - column modelMigrationColumn - text sql.NullString - bool dbNullBool - tm dbNullTime - blob []byte -} - -func (v *modelMigrationValue) dest() any { - switch v.column.kind { - case modelMigrationBool: - return &v.bool - case modelMigrationTime: - return &v.tm - case modelMigrationBlob: - return &v.blob - default: - return &v.text - } -} - -func (v modelMigrationValue) arg(now time.Time) any { - switch v.column.kind { - case modelMigrationBool: - if !v.bool.Valid { - if v.column.notNull { - return false - } - return nil - } - return v.bool.Bool - case modelMigrationTime: - if !v.tm.Valid || v.tm.Time.IsZero() { - if v.column.notNull { - return now - } - return nil - } - return v.tm.Time.UTC() - case modelMigrationBlob: - if len(v.blob) == 0 { - return nil - } - return v.blob - default: - if !v.text.Valid { - if v.column.notNull { - return "" - } - return nil - } - return v.text.String - } -} - -func (v modelMigrationValue) textValue() string { - if !v.text.Valid { - return "" - } - return strings.TrimSpace(v.text.String) -} - func settingsDatabaseMigrateHandler(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Nur POST erlaubt", http.StatusMethodNotAllowed) return } - var req databaseMigrationRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + var in RecorderSettingsIn + if err := json.NewDecoder(r.Body).Decode(&in); err != nil { http.Error(w, "invalid json: "+err.Error(), http.StatusBadRequest) return } current := getSettings() - target, err := databaseMigrationTargetSettings(current, req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + target := current + target.DatabaseType = databaseTypePostgres + target.PostgresMode = in.PostgresMode + target.PostgresDataDir = in.PostgresDataDir + target.PostgresPort = in.PostgresPort + target.PostgresDatabase = in.PostgresDatabase + target.PostgresUser = in.PostgresUser + target.DatabaseURL = strings.TrimSpace(in.DatabaseURL) + normalizePostgresSettings(&target) + + if target.PostgresMode == postgresModeURL && target.DatabaseURL == "" { + http.Error(w, "Database URL fehlt", http.StatusBadRequest) return } - sourceConfig, err := buildModelStoreConfigFromRecorderSettings(current) + sanitizedURL, pwFromURL := stripPasswordFromPostgresURL(target.DatabaseURL) + pwFromURL = strings.TrimSpace(pwFromURL) + if pwFromURL == "****" { + pwFromURL = "" + } + if sanitizedURL != "" { + target.DatabaseURL = sanitizedURL + } + + plainPW := strings.TrimSpace(in.DBPassword) + if plainPW == "" { + plainPW = pwFromURL + } + if plainPW != "" { + enc, err := encryptSettingString(plainPW) + if err != nil { + http.Error(w, "konnte DB-Passwort nicht verschluesseln: "+err.Error(), http.StatusInternalServerError) + return + } + target.EncryptedDBPassword = enc + } + + sourceDSN, err := buildPostgresDSNFromRecorderSettings(current) if err != nil { http.Error(w, "Quell-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest) return } - targetConfig, err := buildModelStoreConfigFromRecorderSettings(target) + targetDSN, err := buildPostgresDSNFromRecorderSettings(target) + if err != nil { + http.Error(w, "Ziel-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest) + return + } + if strings.TrimSpace(sourceDSN) == strings.TrimSpace(targetDSN) || + sanitizeDSNForLog(sourceDSN) == sanitizeDSNForLog(targetDSN) { + http.Error(w, "Quelle und Ziel sind identisch", http.StatusBadRequest) + return + } + + sourceDBURL, sourceEnv, err := postgresToolConnection(sourceDSN) + if err != nil { + http.Error(w, "Quell-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest) + return + } + targetDBURL, targetEnv, err := postgresToolConnection(targetDSN) if err != nil { http.Error(w, "Ziel-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest) return } - count, err := migrateModelStoreData(sourceConfig, targetConfig) + tmp, err := os.CreateTemp("", "nsfwapp-db-migrate-*.dump") if err != nil { - http.Error(w, "Migration fehlgeschlagen: "+err.Error(), http.StatusInternalServerError) + http.Error(w, "Migration konnte nicht vorbereitet werden: "+err.Error(), http.StatusInternalServerError) + return + } + tmpPath := tmp.Name() + _ = tmp.Close() + defer os.Remove(tmpPath) + + ctx, cancel := context.WithTimeout(r.Context(), 20*time.Minute) + defer cancel() + + if err := dumpPostgresDatabase(ctx, sourceDBURL, sourceEnv, tmpPath); err != nil { + http.Error(w, "pg_dump fehlgeschlagen: "+err.Error(), http.StatusInternalServerError) + return + } + if err := restorePostgresDatabase(ctx, targetDBURL, targetEnv, tmpPath); err != nil { + http.Error(w, "pg_restore fehlgeschlagen: "+err.Error(), http.StatusInternalServerError) return } - refreshedStore := NewModelStore(targetConfig) - if err := refreshedStore.Load(); err != nil { - http.Error(w, "Ziel-Datenbank konnte nach Migration nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError) + newStore := NewModelStore(ModelStoreConfig{Type: databaseTypePostgres, DSN: targetDSN}) + if err := newStore.Load(); err != nil { + _ = newStore.Close() + http.Error(w, "Ziel-Datenbank konnte nach Migration nicht geladen werden: "+err.Error(), http.StatusInternalServerError) return } @@ -244,186 +121,77 @@ func settingsDatabaseMigrateHandler(w http.ResponseWriter, r *http.Request) { settings = target settingsMu.Unlock() saveSettingsToDisk() - setModelStore(refreshedStore) + + setModelStore(newStore) + setCoverModelStore(newStore) + setChaturbateOnlineModelStore(newStore) + + info, _ := os.Stat(tmpPath) + var bytesWritten int64 + if info != nil { + bytesWritten = info.Size() + } writeJSON(w, http.StatusOK, map[string]any{ - "ok": true, - "rows": count, - "sourceType": sourceConfig.Type, - "targetType": targetConfig.Type, - "targetSQLite": targetConfig.SQLitePath, - "targetDatabase": sanitizeDSNForLog(targetConfig.DSN), + "ok": true, + "bytes": bytesWritten, + "mode": target.PostgresMode, }) } -func databaseMigrationTargetSettings(current RecorderSettings, req databaseMigrationRequest) (RecorderSettings, error) { - next := current - next.DatabaseType = normalizeDatabaseType(req.DatabaseType) - next.SQLitePath = normalizeSQLitePath(req.SQLitePath) - next.DatabaseURL = strings.TrimSpace(req.DatabaseURL) - - sanitizedURL, pwFromURL := stripPasswordFromPostgresURL(next.DatabaseURL) - pwFromURL = strings.TrimSpace(pwFromURL) - if pwFromURL == "****" { - pwFromURL = "" - } - if sanitizedURL != "" { - next.DatabaseURL = sanitizedURL - } - - plainPW := strings.TrimSpace(req.DBPassword) - if plainPW == "" { - plainPW = pwFromURL - } - if plainPW != "" { - enc, err := encryptSettingString(plainPW) - if err != nil { - return RecorderSettings{}, appErrorf("konnte DB-Passwort nicht verschluesseln: %w", err) - } - next.EncryptedDBPassword = enc - } - - if next.DatabaseType == databaseTypePostgres && strings.TrimSpace(next.DatabaseURL) == "" { - return RecorderSettings{}, errors.New("Database URL fehlt") - } - if next.DatabaseType == databaseTypeSQLite && strings.TrimSpace(next.SQLitePath) == "" { - return RecorderSettings{}, errors.New("SQLite-Datei fehlt") - } - - return next, nil -} - -func migrateModelStoreData(sourceConfig, targetConfig ModelStoreConfig) (int64, error) { - if sameModelStoreConfig(sourceConfig, targetConfig) { - return 0, errors.New("Quelle und Ziel sind identisch") - } - - source := NewModelStore(sourceConfig) - if err := source.Load(); err != nil { - return 0, appErrorf("Quelle konnte nicht geöffnet werden: %w", err) - } - defer source.Close() - - target := NewModelStore(targetConfig) - if err := target.Load(); err != nil { - return 0, appErrorf("Ziel konnte nicht geöffnet werden: %w", err) - } - defer target.Close() - - count, err := migrateModelsTable(source.db, target.db) +func dumpPostgresDatabase(ctx context.Context, dbURL string, env []string, outPath string) error { + toolPath, err := postgresToolPath("pg_dump") if err != nil { - return count, err - } - if normalizeDatabaseType(sourceConfig.Type) == databaseTypeSQLite && - normalizeDatabaseType(targetConfig.Type) == databaseTypeSQLite && - !sameCleanPath(filepath.Dir(sourceConfig.SQLitePath), filepath.Dir(targetConfig.SQLitePath)) { - sourceImageDir := sqliteProfileImageDir(sourceConfig.SQLitePath) - targetImageDir := sqliteProfileImageDir(targetConfig.SQLitePath) - if _, statErr := os.Stat(sourceImageDir); statErr == nil { - if _, targetErr := os.Stat(targetImageDir); errors.Is(targetErr, os.ErrNotExist) { - if copyErr := copyDirectory(sourceImageDir, targetImageDir); copyErr != nil { - return count, copyErr - } - } else if targetErr != nil { - return count, targetErr - } - } else if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { - return count, statErr - } - } - return count, nil -} - -func sameModelStoreConfig(a, b ModelStoreConfig) bool { - a.Type = normalizeDatabaseType(a.Type) - b.Type = normalizeDatabaseType(b.Type) - if a.Type != b.Type { - return false - } - if a.Type == databaseTypeSQLite { - return filepath.Clean(strings.TrimSpace(a.SQLitePath)) == filepath.Clean(strings.TrimSpace(b.SQLitePath)) - } - return strings.TrimSpace(a.DSN) == strings.TrimSpace(b.DSN) -} - -func migrateModelsTable(sourceDB, targetDB *sql.DB) (int64, error) { - columnNames := make([]string, 0, len(modelMigrationColumns)) - for _, column := range modelMigrationColumns { - columnNames = append(columnNames, column.name) + return err } - rows, err := sourceDB.Query("SELECT " + strings.Join(columnNames, ",") + " FROM models ORDER BY updated_at ASC, id ASC;") + out, err := os.Create(outPath) if err != nil { - return 0, err - } - defer rows.Close() - - tx, err := targetDB.Begin() - if err != nil { - return 0, err - } - defer func() { _ = tx.Rollback() }() - - if _, err := tx.Exec(`DELETE FROM models;`); err != nil { - return 0, err + return err } - insertSQL := modelMigrationInsertSQL(columnNames) - stmt, err := tx.Prepare(insertSQL) - if err != nil { - return 0, err - } - defer stmt.Close() - - var count int64 - now := time.Now().UTC() - for rows.Next() { - values := make([]modelMigrationValue, len(modelMigrationColumns)) - dests := make([]any, len(modelMigrationColumns)) - for i, column := range modelMigrationColumns { - values[i] = modelMigrationValue{column: column} - dests[i] = values[i].dest() - } - if err := rows.Scan(dests...); err != nil { - return count, err - } - if values[0].textValue() == "" { - continue - } - - args := make([]any, len(values)) - for i, value := range values { - args[i] = value.arg(now) - } - if _, err := stmt.Exec(args...); err != nil { - return count, err - } - count++ - } - if err := rows.Err(); err != nil { - return count, err - } - if err := tx.Commit(); err != nil { - return count, err - } - - return count, nil -} - -func modelMigrationInsertSQL(columnNames []string) string { - placeholders := make([]string, 0, len(columnNames)) - updates := make([]string, 0, len(columnNames)-1) - for i, column := range columnNames { - placeholders = append(placeholders, fmt.Sprintf("$%d", i+1)) - if column != "id" { - updates = append(updates, column+"=EXCLUDED."+column) - } - } - - return fmt.Sprintf( - "INSERT INTO models (%s) VALUES (%s) ON CONFLICT(id) DO UPDATE SET %s;", - strings.Join(columnNames, ","), - strings.Join(placeholders, ","), - strings.Join(updates, ","), + cmd := exec.CommandContext( + ctx, + toolPath, + "--format=custom", + "--blobs", + "--no-owner", + "--no-privileges", + "--dbname="+dbURL, ) + cmd.Env = env + cmd.Stdout = out + hideCommandWindow(cmd) + + var stderr bytes.Buffer + cmd.Stderr = &stderr + runErr := cmd.Run() + closeErr := out.Close() + if runErr != nil { + msg := trimCommandOutput(stderr.String()) + if msg == "" { + msg = runErr.Error() + } + return appErrorf("%s", msg) + } + return closeErr +} + +func restorePostgresDatabase(ctx context.Context, dbURL string, env []string, dumpPath string) error { + stderr, err := runPostgresTool( + ctx, + "pg_restore", + postgresToolConfig{dbURL: dbURL, env: env}, + "--clean", + "--if-exists", + "--no-owner", + "--no-privileges", + "--single-transaction", + "--dbname="+dbURL, + filepath.Clean(dumpPath), + ) + if err != nil { + return appErrorf("%s", stderr) + } + return nil } diff --git a/backend/settings_sqlite_move.go b/backend/settings_sqlite_move.go deleted file mode 100644 index bda8172..0000000 --- a/backend/settings_sqlite_move.go +++ /dev/null @@ -1,196 +0,0 @@ -package main - -import ( - "errors" - "os" - "path/filepath" - "strings" -) - -func sqlitePathChanged(current, next RecorderSettings) bool { - return normalizeDatabaseType(current.DatabaseType) == databaseTypeSQLite && - normalizeDatabaseType(next.DatabaseType) == databaseTypeSQLite && - normalizeSQLitePath(current.SQLitePath) != normalizeSQLitePath(next.SQLitePath) -} - -func moveSQLiteDatabaseForSettings(current, next RecorderSettings) error { - currentConfig, err := buildModelStoreConfigFromRecorderSettings(current) - if err != nil { - return err - } - nextConfig, err := buildModelStoreConfigFromRecorderSettings(next) - if err != nil { - return err - } - return moveSQLiteDatabaseFiles(currentConfig.SQLitePath, nextConfig.SQLitePath) -} - -func restoreModelStoreForSettings(rec RecorderSettings) { - config, err := buildModelStoreConfigFromRecorderSettings(rec) - if err != nil { - return - } - store := NewModelStore(config) - if err := store.Load(); err != nil { - _ = store.Close() - return - } - setModelStore(store) -} - -func moveSQLiteDatabaseFiles(sourcePath, targetPath string) error { - sourcePath = filepath.Clean(strings.TrimSpace(sourcePath)) - targetPath = filepath.Clean(strings.TrimSpace(targetPath)) - if sourcePath == "" || targetPath == "" { - return errors.New("sqlite source/target path fehlt") - } - if sourcePath == targetPath { - return nil - } - - sourceInfo, err := os.Stat(sourcePath) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return err - } - if sourceInfo.IsDir() { - return appErrorf("sqlite source path ist ein ordner: %s", sourcePath) - } - - if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { - return err - } - - for _, target := range append([]string{targetPath}, sqliteSidecarPaths(targetPath)...) { - if _, err := os.Stat(target); err == nil { - return appErrorf("Ziel-Datei existiert bereits: %s", target) - } else if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - } - sourceImageDir := sqliteProfileImageDir(sourcePath) - targetImageDir := sqliteProfileImageDir(targetPath) - if !sameCleanPath(sourceImageDir, targetImageDir) { - if _, err := os.Stat(sourceImageDir); err == nil { - if _, err := os.Stat(targetImageDir); err == nil { - return appErrorf("Ziel-Profilbildordner existiert bereits: %s", targetImageDir) - } else if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - } else if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - } - - if store := getModelStore(); store != nil && store.isSQLite() && store.db != nil { - _, _ = store.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`) - _ = store.Close() - } - - if err := moveDatabaseFile(sourcePath, targetPath); err != nil { - return err - } - - sourceSidecars := sqliteSidecarPaths(sourcePath) - targetSidecars := sqliteSidecarPaths(targetPath) - for i, source := range sourceSidecars { - if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) { - continue - } else if err != nil { - return err - } - if err := moveDatabaseFile(source, targetSidecars[i]); err != nil { - return err - } - } - - if !sameCleanPath(sourceImageDir, targetImageDir) { - if err := moveSQLiteProfileImageDir(sourceImageDir, targetImageDir); err != nil { - return err - } - } - - return nil -} - -func sqliteProfileImageDir(sqlitePath string) string { - return filepath.Join(filepath.Dir(filepath.Clean(strings.TrimSpace(sqlitePath))), modelProfileImageDirName) -} - -func sameCleanPath(a, b string) bool { - return strings.EqualFold(filepath.Clean(strings.TrimSpace(a)), filepath.Clean(strings.TrimSpace(b))) -} - -func moveSQLiteProfileImageDir(sourceDir, targetDir string) error { - sourceDir = filepath.Clean(strings.TrimSpace(sourceDir)) - targetDir = filepath.Clean(strings.TrimSpace(targetDir)) - if sourceDir == "" || targetDir == "" || sameCleanPath(sourceDir, targetDir) { - return nil - } - - info, err := os.Stat(sourceDir) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return err - } - if !info.IsDir() { - return appErrorf("Profilbildpfad ist kein Ordner: %s", sourceDir) - } - - if err := os.MkdirAll(filepath.Dir(targetDir), 0755); err != nil { - return err - } - if err := renameWithRetry(sourceDir, targetDir); err == nil { - return nil - } - if err := copyDirectory(sourceDir, targetDir); err != nil { - return err - } - return os.RemoveAll(sourceDir) -} - -func copyDirectory(sourceDir, targetDir string) error { - sourceDir = filepath.Clean(strings.TrimSpace(sourceDir)) - targetDir = filepath.Clean(strings.TrimSpace(targetDir)) - return filepath.WalkDir(sourceDir, func(path string, entry os.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - rel, err := filepath.Rel(sourceDir, path) - if err != nil { - return err - } - target := filepath.Join(targetDir, rel) - if entry.IsDir() { - return os.MkdirAll(target, 0755) - } - if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { - return err - } - _, err = copyDatabaseFile(path, target) - return err - }) -} - -func sqliteSidecarPaths(path string) []string { - return []string{ - path + "-wal", - path + "-shm", - path + "-journal", - } -} - -func moveDatabaseFile(sourcePath, targetPath string) error { - if err := renameWithRetry(sourcePath, targetPath); err == nil { - return nil - } - - if _, err := copyDatabaseFile(sourcePath, targetPath); err != nil { - return err - } - return removeWithRetry(sourcePath) -} diff --git a/frontend/src/components/ui/LoginPage.tsx b/frontend/src/components/ui/LoginPage.tsx index 8f1c059..0e02788 100644 --- a/frontend/src/components/ui/LoginPage.tsx +++ b/frontend/src/components/ui/LoginPage.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useRef, useState, type KeyboardEvent, type ClipboardEvent } from 'react' import { startRegistration, startAuthentication } from '@simplewebauthn/browser' import Button from './Button' +import ProgressBar from './ProgressBar' import LoginShell from './login/LoginShell' import ErrorAlert from './login/ErrorAlert' import LoginForm from './login/LoginForm' @@ -40,6 +41,18 @@ type MeResp = { passkeyCount?: number } +type MLSetupStatus = { + required?: boolean + running?: boolean + done?: boolean + ok?: boolean + progress?: number + phase?: string + message?: string + error?: string + logFile?: string +} + type SetupResp = { secret?: string otpauth?: string @@ -55,6 +68,65 @@ function unwrapWebAuthnOptions(options: any): T { return unwrapped as T } +function MLSetupModal({ status }: { status: MLSetupStatus | null }) { + if (!status?.required) return null + + const progress = Math.max(0, Math.min(100, Number(status.progress) || 0)) + const failed = Boolean(status.done && !status.ok) + const done = Boolean(status.done && status.ok) + if (done) return null + + const title = failed + ? 'ML-Python-Setup fehlgeschlagen' + : 'ML-Python-Pakete werden installiert' + const message = String(status.message || '').trim() || 'Die ML-Python-Umgebung wird vorbereitet.' + const detail = failed + ? String(status.error || '').trim() + : status.logFile + ? `Log: ${status.logFile}` + : '' + + return ( +
+
+
+
+
{title}
+
{message}
+
+
+ {Math.round(progress)}% +
+
+ + + + {detail ? ( +
+ {detail} +
+ ) : null} +
+
+ ) +} + export default function LoginPage({ onLoggedIn }: Props) { const nextPath = useMemo(() => getNextFromLocation(), []) @@ -77,6 +149,7 @@ export default function LoginPage({ onLoggedIn }: Props) { const [passkeySupported, setPasskeySupported] = useState(false) const [passkeyFeedback, setPasskeyFeedback] = useState(null) + const [mlSetupStatus, setMLSetupStatus] = useState(null) const submittedOnceRef = useRef(false) @@ -107,6 +180,39 @@ export default function LoginPage({ onLoggedIn }: Props) { ) }, []) + useEffect(() => { + let cancelled = false + let timer: number | undefined + + async function poll() { + try { + const res = await fetch('/api/setup/ml/status', { + cache: 'no-store', + credentials: 'include', + }) + if (!res.ok) return + const status = (await res.json()) as MLSetupStatus + if (cancelled) return + setMLSetupStatus(status) + + if (status?.required && (!status.done || status.running)) { + timer = window.setTimeout(poll, 1000) + } + } catch { + if (!cancelled) { + timer = window.setTimeout(poll, 2500) + } + } + } + + void poll() + + return () => { + cancelled = true + if (timer) window.clearTimeout(timer) + } + }, []) + function passkeyErrorMessage(e: any) { const raw = String(e?.message ?? e ?? '').trim() @@ -575,7 +681,7 @@ export default function LoginPage({ onLoggedIn }: Props) { ) return ( - + }> {stage === 'login' ? ( = 1 && n <= 65535 ? n : 55432 } -function databaseTypeLabel(value: 'sqlite' | 'postgres') { - return value === 'sqlite' ? 'SQLite' : 'Postgres' +function normalizePostgresName(value: unknown, fallback: string) { + const s = String(value ?? '').trim() + return s || fallback +} + +function postgresSourceKey(v: any) { + const mode = normalizePostgresMode(v?.postgresMode) + if (mode === 'local') { + return [ + mode, + normalizePostgresDataDir(v?.postgresDataDir), + normalizePostgresPort(v?.postgresPort), + normalizePostgresName(v?.postgresDatabase, 'nsfwapp'), + normalizePostgresName(v?.postgresUser, 'nsfwapp'), + ].join('\u0000') + } + return [mode, String(v?.databaseUrl ?? '').trim()].join('\u0000') } function formatDatabaseBackupBytes(bytes: unknown) { @@ -675,7 +693,7 @@ function RatingThresholdStars({ role="radio" aria-checked={checked} disabled={disabled} - title={`Bis einschließlich ${stars} Stern${stars === 1 ? '' : 'e'} löschen`} + title={`Bis einschließlich ${stars} Stern${stars === 1 ? '' : 'e'} löschen`} onClick={() => onChange(stars)} className={[ 'rounded-md p-0.5 transition', @@ -692,7 +710,7 @@ function RatingThresholdStars({
- Löscht bis einschließlich {selected} Stern{selected === 1 ? '' : 'e'}. + Löscht bis einschließlich {selected} Stern{selected === 1 ? '' : 'e'}.
) @@ -705,7 +723,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const saveSuccessTimerRef = useRef(null) const appLogPreRef = useRef(null) const dbRestoreInputRef = useRef(null) - const [browsing, setBrowsing] = useState<'record' | 'done' | 'ffmpeg' | 'sqlite' | null>(null) + const [browsing, setBrowsing] = useState<'record' | 'done' | 'ffmpeg' | 'postgresData' | null>(null) const [msg, setMsg] = useState(null) const [err, setErr] = useState(null) const [diskStatus, setDiskStatus] = useState(null) @@ -714,8 +732,12 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const [appLogLoading, setAppLogLoading] = useState(false) const [appLogOpen, setAppLogOpen] = useState(false) const [dbModalOpen, setDbModalOpen] = useState(false) - const [loadedDatabaseType, setLoadedDatabaseType] = useState<'sqlite' | 'postgres'>('sqlite') - const [loadedSqlitePath, setLoadedSqlitePath] = useState('nsfwapp.sqlite') + const [loadedDatabaseType, setLoadedDatabaseType] = useState<'postgres'>('postgres') + const [loadedPostgresMode, setLoadedPostgresMode] = useState<'url' | 'local'>('url') + const [loadedPostgresDataDir, setLoadedPostgresDataDir] = useState('postgres-data') + const [loadedPostgresPort, setLoadedPostgresPort] = useState(55432) + const [loadedPostgresDatabase, setLoadedPostgresDatabase] = useState('nsfwapp') + const [loadedPostgresUser, setLoadedPostgresUser] = useState('nsfwapp') const [loadedDatabaseUrl, setLoadedDatabaseUrl] = useState('') const [pendingDbPassword, setPendingDbPassword] = useState('') // wird nur beim Speichern gesendet const [dbBackupBusy, setDbBackupBusy] = useState(false) @@ -726,23 +748,29 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const now = Date.now() const saveSucceeded = saveSuccessUntilMs > now const saveFailed = Boolean(err) && !saving && !saveSucceeded - const currentDatabaseType = normalizeDatabaseType((value as any).databaseType) - const currentSqlitePath = normalizeSqlitePath((value as any).sqlitePath) + const currentPostgresMode = normalizePostgresMode((value as any).postgresMode) const currentDatabaseUrl = String((value as any).databaseUrl ?? '').trim() - const databaseLocationDirty = - currentDatabaseType !== loadedDatabaseType || - currentSqlitePath !== loadedSqlitePath || - currentDatabaseUrl !== loadedDatabaseUrl + const loadedPostgresSourceKey = postgresSourceKey({ + databaseType: loadedDatabaseType, + databaseUrl: loadedDatabaseUrl, + postgresMode: loadedPostgresMode, + postgresDataDir: loadedPostgresDataDir, + postgresPort: loadedPostgresPort, + postgresDatabase: loadedPostgresDatabase, + postgresUser: loadedPostgresUser, + }) + const currentPostgresSourceKey = postgresSourceKey(value) + const databaseLocationDirty = currentPostgresSourceKey !== loadedPostgresSourceKey const databaseSettingsDirty = databaseLocationDirty || Boolean((pendingDbPassword || '').trim()) - const savedDatabaseConfigured = loadedDatabaseType === 'sqlite' ? Boolean(loadedSqlitePath) : Boolean(loadedDatabaseUrl) - const targetDatabaseConfigured = currentDatabaseType === 'sqlite' ? Boolean(currentSqlitePath) : Boolean(currentDatabaseUrl) + const savedDatabaseConfigured = + loadedPostgresMode === 'local' ? Boolean(loadedPostgresDataDir) : Boolean(loadedDatabaseUrl) + const targetDatabaseConfigured = + currentPostgresMode === 'local' + ? Boolean(normalizePostgresDataDir((value as any).postgresDataDir)) + : Boolean(currentDatabaseUrl) const databaseConfigured = savedDatabaseConfigured - const canMigrateDatabase = - savedDatabaseConfigured && - targetDatabaseConfigured && - databaseLocationDirty && - currentDatabaseType !== loadedDatabaseType + const canMigrateDatabase = savedDatabaseConfigured && targetDatabaseConfigured && databaseLocationDirty const dbMaintenanceBusy = dbBackupBusy || dbRestoreBusy || dbMigrationBusy const [authStatus, setAuthStatus] = useState(null) @@ -781,9 +809,9 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const saveButton = (() => { if (saveUiState === 'saving') { return { - text: 'Speichern…', + text: 'Speichern…', color: 'blue' as const, - icon: null, // Spinner kommt über isLoading + icon: null, // Spinner kommt über isLoading isLoading: true, } } @@ -882,10 +910,18 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { .then((data: RecorderSettings) => { if (!alive) return const databaseType = normalizeDatabaseType((data as any).databaseType) - const sqlitePath = normalizeSqlitePath((data as any).sqlitePath) + const postgresMode = normalizePostgresMode((data as any).postgresMode) + const postgresDataDir = normalizePostgresDataDir((data as any).postgresDataDir) + const postgresPort = normalizePostgresPort((data as any).postgresPort) + const postgresDatabase = normalizePostgresName((data as any).postgresDatabase, 'nsfwapp') + const postgresUser = normalizePostgresName((data as any).postgresUser, 'nsfwapp') setValue({ databaseType, - sqlitePath, + postgresMode, + postgresDataDir, + postgresPort, + postgresDatabase, + postgresUser, databaseUrl: String((data as any).databaseUrl ?? ''), hasDbPassword: Boolean((data as any).hasDbPassword ?? false), recordDir: (data.recordDir || DEFAULTS.recordDir).toString(), @@ -893,7 +929,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { ffmpegPath: String(data.ffmpegPath ?? DEFAULTS.ffmpegPath ?? ''), - // ✅ falls backend die Felder noch nicht hat -> defaults nutzen + // ✅ falls backend die Felder noch nicht hat -> defaults nutzen autoAddToDownloadList: data.autoAddToDownloadList ?? DEFAULTS.autoAddToDownloadList, autoStartAddedDownloads: (data as any).autoStartAddedDownloads ?? DEFAULTS.autoStartAddedDownloads, @@ -937,7 +973,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { (data as any).trainingDetectorEpochs ?? DEFAULTS.trainingDetectorEpochs, }) setLoadedDatabaseType(databaseType) - setLoadedSqlitePath(sqlitePath) + setLoadedPostgresMode(postgresMode) + setLoadedPostgresDataDir(postgresDataDir) + setLoadedPostgresPort(postgresPort) + setLoadedPostgresDatabase(postgresDatabase) + setLoadedPostgresUser(postgresUser) setLoadedDatabaseUrl(String((data as any).databaseUrl ?? '').trim()) }) .catch(() => { @@ -1048,7 +1088,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const text = status === 'running' || status === 'queued' ? phaseLabel - ? `${shortTaskFilename(file, 38)} · ${phaseLabel}` + ? `${shortTaskFilename(file, 38)} · ${phaseLabel}` : shortTaskFilename(file, 52) : shortTaskFilename(file, 52) @@ -1254,7 +1294,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { scrollAppLogToBottom() }, [appLogOpen, appLog?.log, appLogLoading]) - async function browse(target: 'record' | 'done' | 'ffmpeg' | 'sqlite') { + async function browse(target: 'record' | 'done' | 'ffmpeg' | 'postgresData') { setErr(null) setMsg(null) setBrowsing(target) @@ -1275,9 +1315,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setValue((v) => { if (target === 'record') return { ...v, recordDir: p } if (target === 'done') return { ...v, doneDir: p } - if (target === 'sqlite') { - return { ...v, sqlitePath: joinSettingsPath(p, sqlitePathFilename((v as any).sqlitePath)) } - } + if (target === 'postgresData') return { ...v, postgresMode: 'local', postgresDataDir: p } return { ...v, ffmpegPath: p } }) } catch (e: any) { @@ -1351,7 +1389,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const ok = window.confirm( `Datenbank-Backup "${file.name}" wiederherstellen?\n\n` + - 'Das überschreibt die aktuelle Datenbank. Dieser Schritt kann nicht automatisch rueckgaengig gemacht werden.' + 'Das überschreibt die aktuelle Datenbank. Dieser Schritt kann nicht automatisch rueckgaengig gemacht werden.' ) if (!ok) return @@ -1378,7 +1416,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { new CustomEvent('models-db-changed', { detail: { databaseType: loadedDatabaseType, - sqlitePath: loadedSqlitePath, + postgresMode: loadedPostgresMode, + postgresDataDir: loadedPostgresDataDir, + postgresPort: loadedPostgresPort, + postgresDatabase: loadedPostgresDatabase, + postgresUser: loadedPostgresUser, databaseUrl: loadedDatabaseUrl, restored: true, }, @@ -1395,8 +1437,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setDbMaintenanceMsg(null) setDbMaintenanceErr(null) - const databaseType = currentDatabaseType - const sqlitePath = currentSqlitePath + const postgresMode = currentPostgresMode + const postgresDataDir = normalizePostgresDataDir((value as any).postgresDataDir) + const postgresPort = normalizePostgresPort((value as any).postgresPort) + const postgresDatabase = normalizePostgresName((value as any).postgresDatabase, 'nsfwapp') + const postgresUser = normalizePostgresName((value as any).postgresUser, 'nsfwapp') const databaseUrl = currentDatabaseUrl if (!savedDatabaseConfigured) { @@ -1412,9 +1457,13 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { return } + const targetLabel = + postgresMode === 'local' + ? `lokalen Datenordner "${postgresDataDir}"` + : `Datenbank "${databaseUrl}"` const ok = window.confirm( - `Aktive Datenbank von ${databaseTypeLabel(loadedDatabaseType)} nach ${databaseTypeLabel(databaseType)} migrieren?\n\n` + - 'Die Ziel-Datenbank wird mit den aktuellen Model-Daten überschrieben. Danach wird die App auf die Ziel-Datenbank umgestellt.' + `Aktive PostgreSQL-Datenbank nach ${targetLabel} migrieren?\n\n` + + 'Die Ziel-Datenbank wird mit den aktuellen Model-Daten ueberschrieben. Danach nutzt die App das Ziel.' ) if (!ok) return @@ -1425,8 +1474,12 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { headers: { 'Content-Type': 'application/json' }, cache: 'no-store', body: JSON.stringify({ - databaseType, - sqlitePath, + databaseType: 'postgres', + postgresMode, + postgresDataDir, + postgresPort, + postgresDatabase, + postgresUser, databaseUrl, dbPassword: pendingDbPassword || undefined, }), @@ -1437,11 +1490,15 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { } const data = await res.json().catch(() => null) - const rows = Number(data?.rows ?? 0) - setDbMaintenanceMsg(`Migration abgeschlossen (${rows} Model${rows === 1 ? '' : 's'}).`) + const size = formatDatabaseBackupBytes(data?.bytes ?? 0) + setDbMaintenanceMsg(size ? `Migration abgeschlossen (${size}).` : 'Migration abgeschlossen.') - setLoadedDatabaseType(databaseType) - setLoadedSqlitePath(sqlitePath) + setLoadedDatabaseType('postgres') + setLoadedPostgresMode(postgresMode) + setLoadedPostgresDataDir(postgresDataDir) + setLoadedPostgresPort(postgresPort) + setLoadedPostgresDatabase(postgresDatabase) + setLoadedPostgresUser(postgresUser) setLoadedDatabaseUrl(databaseUrl) setPendingDbPassword('') @@ -1449,7 +1506,16 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { window.dispatchEvent(new Event('models-changed')) window.dispatchEvent( new CustomEvent('models-db-changed', { - detail: { databaseType, sqlitePath, databaseUrl, migrated: true }, + detail: { + databaseType: 'postgres', + postgresMode, + postgresDataDir, + postgresPort, + postgresDatabase, + postgresUser, + databaseUrl, + migrated: true, + }, }) ) } catch (e: any) { @@ -1467,7 +1533,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const doneDir = value.doneDir.trim() const ffmpegPath = (value.ffmpegPath ?? '').trim() const databaseType = normalizeDatabaseType((value as any).databaseType) - const sqlitePath = normalizeSqlitePath((value as any).sqlitePath) + const postgresMode = normalizePostgresMode((value as any).postgresMode) + const postgresDataDir = normalizePostgresDataDir((value as any).postgresDataDir) + const postgresPort = normalizePostgresPort((value as any).postgresPort) + const postgresDatabase = normalizePostgresName((value as any).postgresDatabase, 'nsfwapp') + const postgresUser = normalizePostgresName((value as any).postgresUser, 'nsfwapp') const databaseUrl = String((value as any).databaseUrl ?? '').trim() const hadPendingDbPassword = Boolean((pendingDbPassword || '').trim()) @@ -1475,16 +1545,16 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setErr('Bitte Aufnahme-Ordner und Ziel-Ordner angeben.') return } - if (databaseType === 'sqlite' && !sqlitePath) { - setErr('Bitte SQLite-Datei angeben.') + if (postgresMode === 'url' && !databaseUrl) { + setErr('Bitte Database URL angeben.') return } - if (databaseType === 'postgres' && !databaseUrl) { - setErr('Bitte Database URL angeben oder SQLite auswaehlen.') + if (postgresMode === 'local' && !postgresDataDir) { + setErr('Bitte PostgreSQL-Datenordner angeben.') return } - // ✅ Switch-Logik: Autostart nur sinnvoll, wenn Auto-Add aktiv ist + // ✅ Switch-Logik: Autostart nur sinnvoll, wenn Auto-Add aktiv ist const autoAddToDownloadList = !!value.autoAddToDownloadList const autoStartAddedDownloads = !!value.autoStartAddedDownloads const enableConcurrentDownloadsLimit = !!(value as any).enableConcurrentDownloadsLimit @@ -1535,7 +1605,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ databaseType, - sqlitePath, + postgresMode, + postgresDataDir, + postgresPort, + postgresDatabase, + postgresUser, databaseUrl, dbPassword: pendingDbPassword || undefined, recordDir, @@ -1572,7 +1646,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const t = await res.text().catch(() => '') throw new Error(t || `HTTP ${res.status}`) } - // Button-Label: "Gespeichert" für 2.5s + // Button-Label: "Gespeichert" für 2.5s const until = Date.now() + 2500 setSaveSuccessUntilMs(until) @@ -1586,19 +1660,37 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { window.dispatchEvent(new CustomEvent('recorder-settings-updated')) const databaseChanged = - databaseType !== loadedDatabaseType || - sqlitePath !== loadedSqlitePath || - databaseUrl !== loadedDatabaseUrl || + postgresSourceKey({ + databaseType, + postgresMode, + postgresDataDir, + postgresPort, + postgresDatabase, + postgresUser, + databaseUrl, + }) !== loadedPostgresSourceKey || hadPendingDbPassword if (databaseChanged) { window.dispatchEvent( new CustomEvent('models-db-changed', { - detail: { databaseType, sqlitePath, databaseUrl }, + detail: { + databaseType, + postgresMode, + postgresDataDir, + postgresPort, + postgresDatabase, + postgresUser, + databaseUrl, + }, }) ) setLoadedDatabaseType(databaseType) - setLoadedSqlitePath(sqlitePath) + setLoadedPostgresMode(postgresMode) + setLoadedPostgresDataDir(postgresDataDir) + setLoadedPostgresPort(postgresPort) + setLoadedPostgresDatabase(postgresDatabase) + setLoadedPostgresUser(postgresUser) setLoadedDatabaseUrl(databaseUrl) setPendingDbPassword('') } @@ -1644,24 +1736,24 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { return null } if ((!mb || mb <= 0) && !deleteLowRated) { - setErr('Mindestgröße ist 0 und Rating-Löschen ist deaktiviert – es würde nichts gelöscht.') + setErr('Mindestgröße ist 0 und Rating-Löschen ist deaktiviert – es würde nichts gelöscht.') return null } const ratingLine = deleteLowRated - ? `• Rating ≤ ${maxStars} Stern${maxStars === 1 ? '' : 'e'}\n` + ? `• Rating ≤ ${maxStars} Stern${maxStars === 1 ? '' : 'e'}\n` : '' const sizeLine = mb > 0 - ? `• Dateien < ${mb} MB in "${doneDir}"\n` + ? `• Dateien < ${mb} MB in "${doneDir}"\n` : '' const ok = window.confirm( - `Aufräumen:\n` + + `Aufräumen:\n` + sizeLine + ratingLine + - `• Entfernt defekte Video-Dateien\n` + - `• Löscht Record-Reste und verwaiste Assets\n\n` + + `• Entfernt defekte Video-Dateien\n` + + `• Löscht Record-Reste und verwaiste Assets\n\n` + `Fortfahren?` ) if (!ok) return null @@ -1791,7 +1883,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { } if (newPassword !== newPasswordRepeat) { - setErr('Die neuen Passwörter stimmen nicht überein.') + setErr('Die neuen Passwörter stimmen nicht überein.') return } @@ -1816,7 +1908,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setCurrentPassword('') setNewPassword('') setNewPasswordRepeat('') - setMsg('Passwort wurde geändert.') + setMsg('Passwort wurde geändert.') } catch (e: any) { setErr(e?.message ?? String(e)) } finally { @@ -1850,7 +1942,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { return } - const ok = window.confirm('2FA wirklich deaktivieren? Dein Login ist danach weniger geschützt.') + const ok = window.confirm('2FA wirklich deaktivieren? Dein Login ist danach weniger geschützt.') if (!ok) return setSecurityBusy(true) @@ -1886,7 +1978,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { function passkeyErrorMessage(e: any) { const raw = String(e?.message ?? e ?? '').trim() - if (!raw) return 'Passkey konnte nicht hinzugefügt werden.' + if (!raw) return 'Passkey konnte nicht hinzugefügt werden.' if (raw.includes('passkey registration verify failed')) { return ( @@ -1929,14 +2021,14 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setMsg(null) setPasskeyFeedback({ kind: 'loading', - text: 'Starte Passkey-Einrichtung…', + text: 'Starte Passkey-Einrichtung…', }) setSecurityBusy(true) try { setPasskeyFeedback({ kind: 'loading', - text: 'Hole WebAuthn-Challenge vom Backend…', + text: 'Hole WebAuthn-Challenge vom Backend…', }) const optionsRes = await fetch('/api/auth/passkey/register/options', { @@ -1953,7 +2045,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setPasskeyFeedback({ kind: 'loading', - text: 'Browser-Passkey-Dialog geöffnet. Bitte Face ID, Fingerabdruck, Windows Hello oder Sicherheitsschlüssel bestätigen.', + text: 'Browser-Passkey-Dialog geöffnet. Bitte Face ID, Fingerabdruck, Windows Hello oder Sicherheitsschlüssel bestätigen.', }) const credential = await startRegistration({ @@ -1962,7 +2054,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { setPasskeyFeedback({ kind: 'loading', - text: 'Passkey wurde vom Browser erstellt. Verifiziere beim Backend…', + text: 'Passkey wurde vom Browser erstellt. Verifiziere beim Backend…', }) const verifyRes = await fetch('/api/auth/passkey/register/verify', { @@ -1977,10 +2069,10 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { throw new Error(text || `HTTP ${verifyRes.status}`) } - setMsg('Passkey wurde hinzugefügt.') + setMsg('Passkey wurde hinzugefügt.') setPasskeyFeedback({ kind: 'success', - text: 'Passkey wurde erfolgreich hinzugefügt.', + text: 'Passkey wurde erfolgreich hinzugefügt.', }) await loadAuthStatus() } catch (e: any) { @@ -2002,14 +2094,14 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { const credentialId = String(passkey?.id ?? '').trim() if (!credentialId) { - setErr('Passkey kann nicht gelöscht werden: Credential-ID fehlt.') + setErr('Passkey kann nicht gelöscht werden: Credential-ID fehlt.') return } const label = String(passkey?.label || 'diesen Passkey').trim() const ok = window.confirm( - `"${label}" wirklich löschen? Anmeldung mit diesem Passkey ist danach nicht mehr möglich.` + `"${label}" wirklich löschen? Anmeldung mit diesem Passkey ist danach nicht mehr möglich.` ) if (!ok) return @@ -2030,7 +2122,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { throw new Error(text || `HTTP ${res.status}`) } - setMsg('Passkey wurde gelöscht.') + setMsg('Passkey wurde gelöscht.') await loadAuthStatus() } catch (e: any) { setErr(e?.message ?? String(e)) @@ -2060,9 +2152,9 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { ].join(' ')} title={ aiServerStatus === null - ? 'AI Server Status wird geladen…' + ? 'AI Server Status wird geladen…' : aiServerStatus.running - ? `AI Server läuft${aiServerStatus.url ? `: ${aiServerStatus.url}` : ''}` + ? `AI Server läuft${aiServerStatus.url ? `: ${aiServerStatus.url}` : ''}` : `AI Server offline${aiServerStatus.error ? `: ${aiServerStatus.error}` : ''}` } > @@ -2079,7 +2171,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { /> {aiServerStatus === null - ? 'AI prüft…' + ? 'AI prüft…' : aiServerStatus.running ? 'AI online' : 'AI offline'} @@ -2160,15 +2252,15 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { id="tasks" title="Aufgaben" description="Hintergrundaufgaben wie z.B. Asset/Preview-Generierung." - icon="⚙️" + icon="⚙️" defaultOpen={true} >
{ if (!job?.id) return @@ -2185,44 +2277,44 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { checked={!!value.generateAssetsMeta} onChange={(checked) => setValue((v) => ({ ...v, generateAssetsMeta: checked }))} label="Meta" - description="Erzeugt oder ergänzt die meta.json mit Dauer, Auflösung, FPS und weiteren Videoinformationen." + description="Erzeugt oder ergänzt die meta.json mit Dauer, Auflösung, FPS und weiteren Videoinformationen." /> setValue((v) => ({ ...v, generateAssetsThumb: checked }))} label="Vorschaubild" - description="Erzeugt das statische Vorschaubild preview.jpg für Listen und Kartenansicht." + description="Erzeugt das statische Vorschaubild preview.jpg für Listen und Kartenansicht." /> setValue((v) => ({ ...v, generateAssetsTeaser: checked }))} label="Teaser" - description="Erzeugt die kurze Video-Vorschau preview.mp4 für Hover- und Preview-Wiedergabe." + description="Erzeugt die kurze Video-Vorschau preview.mp4 für Hover- und Preview-Wiedergabe." /> setValue((v) => ({ ...v, generateAssetsSprites: checked }))} label="Sprites" - description="Erzeugt das Sprite-Bild preview-sprite.jpg für die Scrubber- und Timeline-Vorschau." + description="Erzeugt das Sprite-Bild preview-sprite.jpg für die Scrubber- und Timeline-Vorschau." /> setValue((v) => ({ ...v, generateAssetsAnalyze: checked }))} label="Analyse" - description="Führt die Analyse aus und ergänzt Treffer, Segmente und weitere Auswertungsdaten." + description="Führt die Analyse aus und ergänzt Treffer, Segmente und weitere Auswertungsdaten." />
{ if (!job?.id) return @@ -2234,10 +2326,10 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { /> { @@ -2256,7 +2348,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { id="paths" title="Pfad-Einstellungen" description="Aufnahme- und Zielverzeichnisse sowie optionaler ffmpeg-Pfad." - icon="📁" + icon="📁" defaultOpen={false} >
@@ -2271,25 +2363,28 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
Datenbank
- SQLite-Datei als portable Standard-Datenbank. Postgres bleibt optional verfuegbar. + PostgreSQL als externe Datenbank oder lokal verwalteter Datenordner.
- {(['sqlite', 'postgres'] as const).map((type) => { - const active = currentDatabaseType === type + {([ + ['url', 'Datenbank'], + ['local', 'Datenordner'], + ] as const).map(([mode, label]) => { + const active = currentPostgresMode === mode return ( ) })} @@ -2305,39 +2400,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
- {currentDatabaseType === 'sqlite' ? ( -
- - -
-
- setValue((v) => ({ ...v, sqlitePath: e.target.value }))} - placeholder="nsfwapp.sqlite" - className="min-w-0 flex-1 rounded-lg px-3 py-2 text-sm bg-white text-gray-900 ring-1 ring-gray-200 - focus:outline-none focus:ring-2 focus:ring-indigo-500 - dark:bg-white/10 dark:text-white dark:ring-white/10" - /> - -
-
- Beim Speichern wird eine bestehende SQLite-Datei samt -wal/-shm an den neuen Ort verschoben. -
-
-
- ) : null} - -
+ {currentPostgresMode === 'url' ? ( +
@@ -2345,7 +2409,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
setValue((v) => ({ ...v, databaseUrl: e.target.value }))} + onChange={(e) => setValue((v) => ({ ...v, databaseType: 'postgres', databaseUrl: e.target.value }))} placeholder="postgres://user@host:5432/db?sslmode=disable" className="min-w-0 flex-1 rounded-lg px-3 py-2 text-sm bg-white text-gray-900 ring-1 ring-gray-200 focus:outline-none focus:ring-2 focus:ring-indigo-500 @@ -2353,10 +2417,69 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { />
+ ) : ( +
+
+ + +
+ setValue((v) => ({ ...v, postgresMode: 'local', postgresDataDir: e.target.value }))} + placeholder="postgres-data" + className="min-w-0 flex-1 rounded-lg px-3 py-2 text-sm bg-white text-gray-900 ring-1 ring-gray-200 + focus:outline-none focus:ring-2 focus:ring-indigo-500 + dark:bg-white/10 dark:text-white dark:ring-white/10" + /> + +
+
+ +
+ + +
+ setValue((v) => ({ ...v, postgresMode: 'local', postgresPort: Number(e.target.value || 55432) }))} + placeholder="Port" + className="rounded-lg px-3 py-2 text-sm bg-white text-gray-900 ring-1 ring-gray-200 + focus:outline-none focus:ring-2 focus:ring-indigo-500 + dark:bg-white/10 dark:text-white dark:ring-white/10" + /> + setValue((v) => ({ ...v, postgresMode: 'local', postgresDatabase: e.target.value }))} + placeholder="Datenbank" + className="rounded-lg px-3 py-2 text-sm bg-white text-gray-900 ring-1 ring-gray-200 + focus:outline-none focus:ring-2 focus:ring-indigo-500 + dark:bg-white/10 dark:text-white dark:ring-white/10" + /> + setValue((v) => ({ ...v, postgresMode: 'local', postgresUser: e.target.value }))} + placeholder="User" + className="rounded-lg px-3 py-2 text-sm bg-white text-gray-900 ring-1 ring-gray-200 + focus:outline-none focus:ring-2 focus:ring-indigo-500 + dark:bg-white/10 dark:text-white dark:ring-white/10" + /> +
+
+
+ )}
@@ -2365,17 +2488,11 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { Backup / Restore
- {currentDatabaseType === 'sqlite' - ? 'Exportiert die aktive SQLite-Datei oder stellt sie aus einem SQLite-Backup wieder her.' - : 'Exportiert die gespeicherte PostgreSQL-Datenbank als pg_dump-Datei oder stellt sie aus einem Backup wieder her.'} + Exportiert die gespeicherte PostgreSQL-Datenbank als pg_dump-Datei oder stellt sie aus einem Backup wieder her.
{databaseSettingsDirty ? (
- {currentDatabaseType !== loadedDatabaseType - ? 'Backup/Restore nutzen die gespeicherte Datenbank. Migration nutzt diese geänderten Werte als Ziel.' - : currentDatabaseType === 'sqlite' && currentSqlitePath !== loadedSqlitePath - ? 'Beim Speichern wird die bestehende SQLite-Datei an diesen Ort verschoben.' - : 'Die Datenbank-Einstellungen wurden geändert. Bitte erst speichern.'} + Die Datenbank-Einstellungen wurden geaendert. Bitte erst speichern.
) : null}
@@ -2388,7 +2505,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { disabled={saving || dbMaintenanceBusy || !canMigrateDatabase} onClick={() => void migrateDatabase()} > - Migrieren + Migrieren und wechseln
- setDbModalOpen(false)} initialUrl={String((value as any).databaseUrl ?? '')} initialHasPassword={Boolean((value as any).hasDbPassword)} onApply={({ databaseUrl, dbPassword }) => { - setValue((v) => ({ ...v, databaseUrl })) + setValue((v) => ({ ...v, postgresMode: 'url', databaseUrl })) setPendingDbPassword(dbPassword || '') }} /> @@ -2520,14 +2632,14 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
Automatisierung & Anzeige
- Verhalten beim Hinzufügen/Starten sowie Anzeigeoptionen. + Verhalten beim Hinzufügen/Starten sowie Anzeigeoptionen.
@@ -2589,7 +2701,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { })) } label="EnrichQ-Postwork im Hintergrund" - description="Wenn aktiv, werden fehlende Analyse-/Enrich-Assets automatisch im Hintergrund nachgeholt. Wenn deaktiviert, werden keine neuen EnrichQ-Hintergrundjobs gestartet; manuelle Asset-Generierung bleibt möglich." + description="Wenn aktiv, werden fehlende Analyse-/Enrich-Assets automatisch im Hintergrund nachgeholt. Wenn deaktiviert, werden keine neuen EnrichQ-Hintergrundjobs gestartet; manuelle Asset-Generierung bleibt möglich." />
@@ -2630,7 +2742,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { })) } label="Provider-APIs verwenden" - description="Master-Schalter für externe Provider-APIs. Wenn deaktiviert, werden keine automatischen API-Abfragen für Online-Status, Profilbilder oder Model-Daten ausgeführt." + description="Master-Schalter für externe Provider-APIs. Wenn deaktiviert, werden keine automatischen API-Abfragen für Online-Status, Profilbilder oder Model-Daten ausgeführt." />
- {/* ✅ NEU: Auto-Delete kleine Downloads */} + {/* ✅ NEU: Auto-Delete kleine Downloads */}
-
Mindestgröße
-
Alles darunter wird gelöscht.
+
Mindestgröße
+
Alles darunter wird gelöscht.
@@ -2730,8 +2842,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { autoDeleteSmallDownloadsKeepFavorites: checked, })) } - label="Favoriten nicht automatisch löschen" - description="Schützt Downloads von favorisierten Models vor dem automatischen Löschen, auch wenn die Datei kleiner als die Mindestgröße ist." + label="Favoriten nicht automatisch löschen" + description="Schützt Downloads von favorisierten Models vor dem automatischen Löschen, auch wenn die Datei kleiner als die Mindestgröße ist." />
@@ -2747,8 +2859,8 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { v.autoDeleteLowRatedDownloadsMaxStars ?? 3, })) } - label="Downloads nach Analyse nach Rating löschen" - description="Löscht fertige Downloads automatisch nach der Analyse, wenn das Rating höchstens der gewählten Sternezahl entspricht." + label="Downloads nach Analyse nach Rating löschen" + description="Löscht fertige Downloads automatisch nach der Analyse, wenn das Rating höchstens der gewählten Sternezahl entspricht." />
- Bis einschließlich dieser Bewertung wird gelöscht. + Bis einschließlich dieser Bewertung wird gelöscht.
@@ -2800,7 +2912,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { Object Detection Epochs
- Anzahl der Trainingsdurchläufe für die Box-Erkennung. Mehr Epochs können die Erkennung verbessern, dauern aber länger. 60 ist ein guter Standard. + Anzahl der Trainingsdurchläufe für die Box-Erkennung. Mehr Epochs können die Erkennung verbessern, dauern aber länger. 60 ist ein guter Standard.
@@ -2834,7 +2946,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { checked={!!value.blurPreviews} onChange={(checked) => setValue((v) => ({ ...v, blurPreviews: checked }))} label="Vorschaubilder blurren" - description="Weichzeichnet Vorschaubilder/Teaser (praktisch auf mobilen Geräten oder im öffentlichen Umfeld)." + description="Weichzeichnet Vorschaubilder/Teaser (praktisch auf mobilen Geräten oder im öffentlichen Umfeld)." />
@@ -2843,7 +2955,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { Teaser abspielen
- Standbild spart Leistung. „Bei Hover (Standard)“: Desktop spielt bei Hover ab, Mobile im Viewport. „Alle“ kann viel CPU ziehen. + Standbild spart Leistung. „Bei Hover (Standard)“: Desktop spielt bei Hover ab, Mobile im Viewport. „Alle“ kann viel CPU ziehen.
@@ -2909,7 +3021,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
Schwelle:{' '} Pause unter {uiPauseGB} GB - {' · '}Resume ab{' '} + {' · '}Resume ab{' '} {uiResumeGB} {' '} @@ -2919,7 +3031,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
{diskStatus ? `Frei: ${diskStatus.freeBytesHuman}${diskStatus.recordPath ? ` (Pfad: ${diskStatus.recordPath})` : ''}` - : 'Status wird geladen…'} + : 'Status wird geladen…'}
{diskStatus?.emergency && ( @@ -2961,7 +3073,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { ].join(' ')} aria-hidden="true" > - ▾ + â–¾ @@ -2974,7 +3086,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { Logdatei: {appLog.path} ) : appLogLoading ? ( - 'Logdatei wird geladen…' + 'Logdatei wird geladen…' ) : ( 'Logdatei noch nicht geladen.' )} @@ -2986,7 +3098,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { disabled={appLogLoading} onClick={() => void loadAppLog()} > - {appLogLoading ? 'Lade…' : 'Aktualisieren'} + {appLogLoading ? 'Lade…' : 'Aktualisieren'}
@@ -3001,7 +3113,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { className="max-h-96 overflow-auto whitespace-pre-wrap rounded-xl bg-gray-950 p-3 font-mono text-[11px] leading-relaxed text-gray-100 ring-1 ring-black/10 dark:ring-white/10" > {appLogLoading - ? 'Log wird geladen…' + ? 'Log wird geladen…' : appLog?.log?.trim() ? appLog.log : 'Noch kein Log vorhanden.'} @@ -3017,7 +3129,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { id="security" title="Konto-Sicherheit" description="Passwort, Zwei-Faktor-Authentifizierung und Passkeys getrennt von den Recorder-Einstellungen verwalten." - icon="🛡️" + icon="🛡️" defaultOpen={false} actions={ <> @@ -3031,10 +3143,10 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
- Passwort ändern + Passwort ändern
- Ändert das Login-Passwort. Andere Sessions werden abgemeldet. + Ändert das Login-Passwort. Andere Sessions werden abgemeldet.
@@ -3089,7 +3201,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { Zwei-Faktor-Authentifizierung
- Authenticator-App als zweiter Faktor. Einzelne Geräte können bei TOTP nicht getrennt erkannt werden. + Authenticator-App als zweiter Faktor. Einzelne Geräte können bei TOTP nicht getrennt erkannt werden.
@@ -3099,7 +3211,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) {
{authStatus?.totpDevice ? ( ) : (
- 2FA ist noch nicht eingerichtet. Die Einrichtung erfolgt aktuell beim nächsten Login-Setup. + 2FA ist noch nicht eingerichtet. Die Einrichtung erfolgt aktuell beim nächsten Login-Setup.
)}
@@ -3163,7 +3275,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { ) : (
Zum Deaktivieren brauchst du dein aktuelles Passwort - {authStatus?.totpEnabled ? ' und einen gültigen 2FA-Code.' : '.'} + {authStatus?.totpEnabled ? ' und einen gültigen 2FA-Code.' : '.'}
)} @@ -3188,7 +3300,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { Passkeys
- Anmeldung ohne Passwort über Face ID, Fingerabdruck, Windows Hello oder Sicherheitsschlüssel. + Anmeldung ohne Passwort über Face ID, Fingerabdruck, Windows Hello oder Sicherheitsschlüssel.
@@ -3200,20 +3312,20 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { (authStatus?.passkeys ?? []).map((pk, index) => ( void deletePasskey(pk)} > - Löschen + Löschen } /> @@ -3237,7 +3349,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { {!passkeySupported ? (
- Dein Browser oder diese Verbindung unterstützt Passkeys aktuell nicht. + Dein Browser oder diese Verbindung unterstützt Passkeys aktuell nicht.
) : (
@@ -3252,9 +3364,9 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { : 'border-blue-200 bg-blue-50 text-blue-800 dark:border-blue-500/30 dark:bg-blue-500/10 dark:text-blue-200', ].join(' ')} > - {passkeyFeedback.kind === 'loading' ? '⏳ ' : null} - {passkeyFeedback.kind === 'success' ? '✅ ' : null} - {passkeyFeedback.kind === 'error' ? '⚠️ ' : null} + {passkeyFeedback.kind === 'loading' ? '⏳ ' : null} + {passkeyFeedback.kind === 'success' ? '✅ ' : null} + {passkeyFeedback.kind === 'error' ? '⚠️ ' : null} {passkeyFeedback.text}
) : null} @@ -3265,7 +3377,7 @@ export default function RecorderSettings({ onAssetsGenerated }: Props) { disabled={securityBusy} onClick={() => void registerSettingsPasskey()} > - {securityBusy ? 'Passkey wird eingerichtet…' : 'Passkey hinzufügen'} + {securityBusy ? 'Passkey wird eingerichtet…' : 'Passkey hinzufügen'} diff --git a/frontend/src/components/ui/login/LoginShell.tsx b/frontend/src/components/ui/login/LoginShell.tsx index 0db3166..b5e03da 100644 --- a/frontend/src/components/ui/login/LoginShell.tsx +++ b/frontend/src/components/ui/login/LoginShell.tsx @@ -2,7 +2,7 @@ import type { ReactNode } from 'react' -export default function LoginShell(props: { children: ReactNode }) { +export default function LoginShell(props: { modal?: ReactNode; children: ReactNode }) { return (
-
-
-

Recorder Login

-

- Bitte melde dich an, um fortzufahren. -

-
+
+
+
+

Recorder Login

+

+ Bitte melde dich an, um fortzufahren. +

+
-
- {props.children} +
+ {props.children} +
+ + {props.modal}
) -} \ No newline at end of file +}