removed sqlite
This commit is contained in:
Linrador
parent
bee72d85e3
commit
3e32f9480a
@ -112,7 +112,7 @@ if [ ! -f "$runtime_bin" ] || ! cmp -s "$install_dir/nsfwapp" "$runtime_bin"; th
|
||||
mv "$tmp" "$runtime_bin"
|
||||
fi
|
||||
|
||||
setup_ml_python() {
|
||||
configure_ml_python_setup() {
|
||||
if [ "${NSFWAPP_SKIP_ML_SETUP:-0}" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
@ -125,45 +125,15 @@ setup_ml_python() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
venv_dir="${NSFWAPP_ML_VENV:-$app_dir/ml-venv}"
|
||||
venv_python="$venv_dir/bin/python"
|
||||
requirements_marker="$venv_dir/.requirements-ml.txt"
|
||||
setup_log="$app_dir/ml-setup.log"
|
||||
|
||||
if [ ! -x "$venv_python" ]; then
|
||||
echo "Erstelle Python-ML-Umgebung: $venv_dir" >&2
|
||||
if ! python3 -m venv "$venv_dir" >>"$setup_log" 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "$requirements_marker" ] || ! cmp -s "$requirements_file" "$requirements_marker"; then
|
||||
echo "Installiere Python-ML-Pakete. Details: $setup_log" >&2
|
||||
if command -v notify-send >/dev/null 2>&1; then
|
||||
notify-send "NSFW App" "ML-Python-Pakete werden installiert. Das kann beim ersten Start dauern." || true
|
||||
fi
|
||||
if ! "$venv_python" -m pip install --upgrade pip >>"$setup_log" 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
if ! "$venv_python" -m pip install -r "$requirements_file" >>"$setup_log" 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
if ! cp "$requirements_file" "$requirements_marker"; then
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
AI_SERVER_PYTHON="$venv_python"
|
||||
export AI_SERVER_PYTHON
|
||||
NSFWAPP_ML_SETUP=1
|
||||
NSFWAPP_ML_REQUIREMENTS="$requirements_file"
|
||||
NSFWAPP_ML_VENV="${NSFWAPP_ML_VENV:-$app_dir/ml-venv}"
|
||||
NSFWAPP_ML_SETUP_LOG="${NSFWAPP_ML_SETUP_LOG:-$app_dir/ml-setup.log}"
|
||||
export NSFWAPP_ML_SETUP NSFWAPP_ML_REQUIREMENTS NSFWAPP_ML_VENV NSFWAPP_ML_SETUP_LOG
|
||||
return 0
|
||||
}
|
||||
|
||||
if ! setup_ml_python; then
|
||||
echo "Warnung: ML-Python-Umgebung konnte nicht vorbereitet werden. Details: $app_dir/ml-setup.log" >&2
|
||||
if command -v notify-send >/dev/null 2>&1; then
|
||||
notify-send "NSFW App" "ML-Python-Umgebung konnte nicht vorbereitet werden. Siehe ml-setup.log." || true
|
||||
fi
|
||||
fi
|
||||
configure_ml_python_setup
|
||||
|
||||
cd "$app_dir"
|
||||
exec "$runtime_bin" "$@"
|
||||
@ -313,11 +283,11 @@ User-Daten:
|
||||
Die installierte Binary liegt unter /opt/nsfwapp/nsfwapp.
|
||||
Beim Start wird sie nach ${NSFWAPP_HOME:-$HOME/nsfwapp}/nsfwapp kopiert
|
||||
und von dort ausgefuehrt. Dadurch bleiben recorder.log, recorder_settings.json,
|
||||
nsfwapp.sqlite, generated/, record/ und record/done im User-Bereich.
|
||||
recorder_settings.json, PostgreSQL-Datenordner, generated/, record/ und record/done im User-Bereich.
|
||||
|
||||
Wichtige Laufzeitprogramme:
|
||||
ffmpeg/ffprobe muessen fuer Recording, Teaser, Analyse und Videofunktionen verfuegbar sein.
|
||||
SQLite ist die Standard-Datenbank. pg_dump/pg_restore werden nur fuer Postgres-Backup und Restore benoetigt.
|
||||
PostgreSQL ist die Datenbank. Fuer lokale Datenordner werden initdb/pg_ctl und fuer Migration/Backup pg_dump/pg_restore benoetigt.
|
||||
Der Launcher erstellt beim ersten Start automatisch ${NSFWAPP_HOME:-$HOME/nsfwapp}/ml-venv
|
||||
und installiert die mitgelieferte requirements-ml.txt. Details stehen in ml-setup.log.
|
||||
Setze AI_SERVER_PYTHON, wenn du eine eigene Python-Umgebung verwenden willst.
|
||||
@ -330,7 +300,7 @@ Section: $SECTION
|
||||
Priority: $PRIORITY
|
||||
Architecture: $ARCH
|
||||
Maintainer: $MAINTAINER
|
||||
Depends: ca-certificates, ffmpeg, python3, python3-venv, xdg-utils, libnotify-bin, procps
|
||||
Depends: ca-certificates, ffmpeg, python3, python3-venv, xdg-utils, libnotify-bin, procps, postgresql
|
||||
Description: $DESCRIPTION_SHORT
|
||||
$DESCRIPTION_LONG
|
||||
EOF
|
||||
|
||||
BIN
backend/dist/nsfwapp-linux-amd64
vendored
BIN
backend/dist/nsfwapp-linux-amd64
vendored
Binary file not shown.
BIN
backend/dist/nsfwapp.exe
vendored
BIN
backend/dist/nsfwapp.exe
vendored
Binary file not shown.
BIN
backend/dist/nsfwapp_amd64.deb
vendored
BIN
backend/dist/nsfwapp_amd64.deb
vendored
Binary file not shown.
@ -3,7 +3,6 @@ module nsfwapp
|
||||
go 1.25.3
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.11.0
|
||||
github.com/getlantern/systray v1.2.2
|
||||
github.com/go-webauthn/webauthn v0.17.2
|
||||
github.com/google/uuid v1.6.0
|
||||
@ -14,12 +13,10 @@ require (
|
||||
github.com/r3labs/sse/v2 v2.10.0
|
||||
github.com/yalue/onnxruntime_go v1.27.0
|
||||
golang.org/x/crypto v0.50.0
|
||||
modernc.org/sqlite v1.53.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.1 // indirect
|
||||
github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 // indirect
|
||||
github.com/getlantern/errors v0.0.0-20190325191628-abdb3e3e36f7 // indirect
|
||||
@ -37,12 +34,9 @@ require (
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
@ -52,14 +46,10 @@ require (
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.24.5
|
||||
github.com/sqweek/dialog v0.0.0-20240226140203-065105509627
|
||||
golang.org/x/image v0.37.0
|
||||
|
||||
@ -1,16 +1,10 @@
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf h1:FPsprx82rdrX2jiKyS17BH6IrTmUBYqZa/CXT4uvb+I=
|
||||
github.com/TheTitanrain/w32 v0.0.0-20180517000239-4f5cfb03fabf/go.mod h1:peYoMncQljjNS6tZwI9WVyQB3qZS6u79/N3mBOcnd3I=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
|
||||
github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
||||
github.com/getlantern/context v0.0.0-20190109183933-c447772a6520 h1:NRUJuo3v3WGC/g5YiyF790gut6oQr5f3FBI88Wv0dx4=
|
||||
@ -64,10 +58,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
|
||||
github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw=
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
@ -80,8 +70,6 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs=
|
||||
github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0=
|
||||
github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
@ -106,95 +94,32 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
github.com/yalue/onnxruntime_go v1.27.0 h1:c1YSgDNtpf0WGtxj3YeRIb8VC5LmM1J+Ve3uHdteC1U=
|
||||
github.com/yalue/onnxruntime_go v1.27.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
|
||||
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/image v0.37.0 h1:ZiRjArKI8GwxZOoEtUfhrBtaCN+4b/7709dlT6SSnQA=
|
||||
golang.org/x/image v0.37.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/Knetic/govaluate.v3 v3.0.0/go.mod h1:csKLBORsPbafmSCGTEh3U7Ozmsuq8ZSIlKk1bcqph0E=
|
||||
gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
|
||||
@ -203,11 +128,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
|
||||
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
|
||||
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
|
||||
|
||||
360
backend/ml_setup.go
Normal file
360
backend/ml_setup.go
Normal file
@ -0,0 +1,360 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type mlPythonSetupStatus struct {
|
||||
Required bool `json:"required"`
|
||||
Running bool `json:"running"`
|
||||
Done bool `json:"done"`
|
||||
OK bool `json:"ok"`
|
||||
Progress int `json:"progress"`
|
||||
Phase string `json:"phase"`
|
||||
Message string `json:"message"`
|
||||
Error string `json:"error,omitempty"`
|
||||
LogFile string `json:"logFile,omitempty"`
|
||||
StartedAt string `json:"startedAt,omitempty"`
|
||||
UpdatedAt string `json:"updatedAt,omitempty"`
|
||||
FinishedAt string `json:"finishedAt,omitempty"`
|
||||
}
|
||||
|
||||
type mlPythonSetupConfig struct {
|
||||
enabled bool
|
||||
requirementsPath string
|
||||
venvDir string
|
||||
venvPython string
|
||||
requirementsMark string
|
||||
logPath string
|
||||
}
|
||||
|
||||
var (
|
||||
mlSetupMu sync.RWMutex
|
||||
mlSetupRunMu sync.Mutex
|
||||
mlSetupStatus mlPythonSetupStatus
|
||||
)
|
||||
|
||||
func initMLPythonSetupStatus() {
|
||||
cfg := mlPythonSetupConfigFromEnv()
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
|
||||
mlSetupMu.Lock()
|
||||
defer mlSetupMu.Unlock()
|
||||
|
||||
if cfg.enabled {
|
||||
mlSetupStatus = mlPythonSetupStatus{
|
||||
Required: true,
|
||||
Running: false,
|
||||
Done: false,
|
||||
OK: false,
|
||||
Progress: 0,
|
||||
Phase: "pending",
|
||||
Message: "ML-Python-Umgebung wartet auf Start.",
|
||||
LogFile: filepath.Base(cfg.logPath),
|
||||
UpdatedAt: now,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
mlSetupStatus = mlPythonSetupStatus{
|
||||
Required: false,
|
||||
Running: false,
|
||||
Done: true,
|
||||
OK: true,
|
||||
Progress: 100,
|
||||
Phase: "skipped",
|
||||
Message: "ML-Python-Setup nicht erforderlich.",
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
func mlPythonSetupConfigFromEnv() mlPythonSetupConfig {
|
||||
if strings.TrimSpace(os.Getenv("NSFWAPP_SKIP_ML_SETUP")) == "1" {
|
||||
return mlPythonSetupConfig{}
|
||||
}
|
||||
if strings.TrimSpace(os.Getenv("AI_SERVER_PYTHON")) != "" {
|
||||
return mlPythonSetupConfig{}
|
||||
}
|
||||
if strings.TrimSpace(os.Getenv("NSFWAPP_ML_SETUP")) != "1" {
|
||||
return mlPythonSetupConfig{}
|
||||
}
|
||||
|
||||
appDir := strings.TrimSpace(os.Getenv("NSFWAPP_HOME"))
|
||||
if appDir == "" {
|
||||
if wd, err := os.Getwd(); err == nil {
|
||||
appDir = wd
|
||||
}
|
||||
}
|
||||
if appDir == "" {
|
||||
appDir = "."
|
||||
}
|
||||
|
||||
requirementsPath := strings.TrimSpace(os.Getenv("NSFWAPP_ML_REQUIREMENTS"))
|
||||
if requirementsPath == "" {
|
||||
requirementsPath = filepath.Join(appDir, "requirements-ml.txt")
|
||||
}
|
||||
|
||||
venvDir := strings.TrimSpace(os.Getenv("NSFWAPP_ML_VENV"))
|
||||
if venvDir == "" {
|
||||
venvDir = filepath.Join(appDir, "ml-venv")
|
||||
}
|
||||
|
||||
logPath := strings.TrimSpace(os.Getenv("NSFWAPP_ML_SETUP_LOG"))
|
||||
if logPath == "" {
|
||||
logPath = filepath.Join(appDir, "ml-setup.log")
|
||||
}
|
||||
|
||||
return mlPythonSetupConfig{
|
||||
enabled: true,
|
||||
requirementsPath: filepath.Clean(requirementsPath),
|
||||
venvDir: filepath.Clean(venvDir),
|
||||
venvPython: filepath.Join(filepath.Clean(venvDir), "bin", "python"),
|
||||
requirementsMark: filepath.Join(filepath.Clean(venvDir), ".requirements-ml.txt"),
|
||||
logPath: filepath.Clean(logPath),
|
||||
}
|
||||
}
|
||||
|
||||
func mlPythonSetupStatusSnapshot() mlPythonSetupStatus {
|
||||
mlSetupMu.RLock()
|
||||
defer mlSetupMu.RUnlock()
|
||||
return mlSetupStatus
|
||||
}
|
||||
|
||||
func updateMLPythonSetupStatus(update func(*mlPythonSetupStatus)) {
|
||||
mlSetupMu.Lock()
|
||||
defer mlSetupMu.Unlock()
|
||||
|
||||
update(&mlSetupStatus)
|
||||
mlSetupStatus.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func mlPythonSetupStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Nur GET erlaubt", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, mlPythonSetupStatusSnapshot())
|
||||
}
|
||||
|
||||
func ensureMLPythonSetup(ctx context.Context) error {
|
||||
cfg := mlPythonSetupConfigFromEnv()
|
||||
if !cfg.enabled {
|
||||
initMLPythonSetupStatus()
|
||||
return nil
|
||||
}
|
||||
|
||||
mlSetupRunMu.Lock()
|
||||
defer mlSetupRunMu.Unlock()
|
||||
|
||||
if strings.TrimSpace(os.Getenv("AI_SERVER_PYTHON")) != "" {
|
||||
initMLPythonSetupStatus()
|
||||
return nil
|
||||
}
|
||||
|
||||
startedAt := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) {
|
||||
s.Required = true
|
||||
s.Running = true
|
||||
s.Done = false
|
||||
s.OK = false
|
||||
s.Progress = 1
|
||||
s.Phase = "prepare"
|
||||
s.Message = "ML-Python-Umgebung wird vorbereitet."
|
||||
s.Error = ""
|
||||
s.LogFile = filepath.Base(cfg.logPath)
|
||||
s.StartedAt = startedAt
|
||||
s.FinishedAt = ""
|
||||
})
|
||||
|
||||
reqBytes, err := os.ReadFile(cfg.requirementsPath)
|
||||
if err != nil {
|
||||
return finishMLPythonSetupError("Requirements konnten nicht gelesen werden.", err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(cfg.venvDir, 0755); err != nil {
|
||||
return finishMLPythonSetupError("ML-Venv-Ordner konnte nicht erstellt werden.", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(cfg.logPath), 0755); err != nil {
|
||||
return finishMLPythonSetupError("ML-Setup-Logordner konnte nicht erstellt werden.", err)
|
||||
}
|
||||
|
||||
if current, err := os.ReadFile(cfg.requirementsMark); err == nil && string(current) == string(reqBytes) {
|
||||
if fi, statErr := os.Stat(cfg.venvPython); statErr == nil && fi != nil && !fi.IsDir() {
|
||||
_ = os.Setenv("AI_SERVER_PYTHON", cfg.venvPython)
|
||||
finishMLPythonSetupSuccess("ML-Python-Pakete sind bereits installiert.")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if fi, err := os.Stat(cfg.venvPython); err != nil || fi == nil || fi.IsDir() {
|
||||
python3, lookErr := exec.LookPath("python3")
|
||||
if lookErr != nil {
|
||||
return finishMLPythonSetupError("python3 wurde nicht gefunden.", lookErr)
|
||||
}
|
||||
if err := runMLPythonSetupCommand(ctx, cfg, "venv", "Python-Umgebung wird erstellt.", 5, 20, python3, "-m", "venv", cfg.venvDir); err != nil {
|
||||
return finishMLPythonSetupError("Python-Umgebung konnte nicht erstellt werden.", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := runMLPythonSetupCommand(ctx, cfg, "pip", "pip wird aktualisiert.", 22, 38, cfg.venvPython, "-m", "pip", "install", "--upgrade", "pip"); err != nil {
|
||||
return finishMLPythonSetupError("pip konnte nicht aktualisiert werden.", err)
|
||||
}
|
||||
|
||||
if err := runMLPythonSetupCommand(ctx, cfg, "requirements", "ML-Python-Pakete werden installiert.", 40, 94, cfg.venvPython, "-m", "pip", "install", "-r", cfg.requirementsPath); err != nil {
|
||||
return finishMLPythonSetupError("ML-Python-Pakete konnten nicht installiert werden.", err)
|
||||
}
|
||||
|
||||
if err := atomicWriteFile(cfg.requirementsMark, reqBytes); err != nil {
|
||||
return finishMLPythonSetupError("ML-Requirements-Marker konnte nicht geschrieben werden.", err)
|
||||
}
|
||||
|
||||
_ = os.Setenv("AI_SERVER_PYTHON", cfg.venvPython)
|
||||
finishMLPythonSetupSuccess("ML-Python-Pakete wurden installiert.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func runMLPythonSetupCommand(ctx context.Context, cfg mlPythonSetupConfig, phase, message string, startProgress, endProgress int, name string, args ...string) error {
|
||||
updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) {
|
||||
s.Running = true
|
||||
s.Progress = startProgress
|
||||
s.Phase = phase
|
||||
s.Message = message
|
||||
})
|
||||
|
||||
logFile, err := os.OpenFile(cfg.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer logFile.Close()
|
||||
|
||||
_, _ = logFile.WriteString("\n[" + time.Now().Format(time.RFC3339) + "] " + name + " " + strings.Join(args, " ") + "\n")
|
||||
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = filepath.Dir(cfg.logPath)
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lines := make(chan string, 64)
|
||||
var scanWG sync.WaitGroup
|
||||
scan := func(r io.Reader) {
|
||||
defer scanWG.Done()
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
_, _ = logFile.WriteString(line + "\n")
|
||||
select {
|
||||
case lines <- line:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
scanWG.Add(2)
|
||||
go scan(stdout)
|
||||
go scan(stderr)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
scanWG.Wait()
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
lineCount := 0
|
||||
progressRange := endProgress - startProgress
|
||||
if progressRange < 1 {
|
||||
progressRange = 1
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case line := <-lines:
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
lineCount++
|
||||
progress := startProgress + minInt(progressRange-1, lineCount)
|
||||
updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) {
|
||||
s.Progress = progress
|
||||
s.Phase = phase
|
||||
s.Message = line
|
||||
})
|
||||
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) {
|
||||
s.Progress = endProgress
|
||||
s.Phase = phase
|
||||
s.Message = message
|
||||
})
|
||||
return nil
|
||||
|
||||
case <-ctx.Done():
|
||||
_ = cmd.Process.Kill()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func finishMLPythonSetupSuccess(message string) {
|
||||
updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) {
|
||||
s.Required = true
|
||||
s.Running = false
|
||||
s.Done = true
|
||||
s.OK = true
|
||||
s.Progress = 100
|
||||
s.Phase = "done"
|
||||
s.Message = message
|
||||
s.Error = ""
|
||||
s.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
})
|
||||
}
|
||||
|
||||
func finishMLPythonSetupError(message string, err error) error {
|
||||
if err == nil {
|
||||
err = errors.New(message)
|
||||
}
|
||||
updateMLPythonSetupStatus(func(s *mlPythonSetupStatus) {
|
||||
s.Required = true
|
||||
s.Running = false
|
||||
s.Done = true
|
||||
s.OK = false
|
||||
s.Phase = "error"
|
||||
s.Message = message
|
||||
s.Error = err.Error()
|
||||
s.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
})
|
||||
return appErrorf("%s: %w", message, err)
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@ -1,37 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
databaseTypeSQLite = "sqlite"
|
||||
databaseTypePostgres = "postgres"
|
||||
defaultSQLitePath = "nsfwapp.sqlite"
|
||||
)
|
||||
|
||||
type ModelStoreConfig struct {
|
||||
Type string
|
||||
DSN string
|
||||
SQLitePath string
|
||||
Type string
|
||||
DSN string
|
||||
}
|
||||
|
||||
func normalizeDatabaseType(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case databaseTypePostgres:
|
||||
return databaseTypePostgres
|
||||
default:
|
||||
return databaseTypeSQLite
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeSQLitePath(raw string) string {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
value = defaultSQLitePath
|
||||
}
|
||||
return filepath.Clean(filepath.FromSlash(value))
|
||||
_ = raw
|
||||
return databaseTypePostgres
|
||||
}
|
||||
|
||||
func buildModelStoreConfigFromSettings() (ModelStoreConfig, error) {
|
||||
@ -39,22 +19,9 @@ func buildModelStoreConfigFromSettings() (ModelStoreConfig, error) {
|
||||
}
|
||||
|
||||
func buildModelStoreConfigFromRecorderSettings(s RecorderSettings) (ModelStoreConfig, error) {
|
||||
dbType := normalizeDatabaseType(s.DatabaseType)
|
||||
if dbType == databaseTypePostgres {
|
||||
dsn, err := buildPostgresDSNFromRecorderSettings(s)
|
||||
if err != nil {
|
||||
return ModelStoreConfig{}, err
|
||||
}
|
||||
return ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn}, nil
|
||||
}
|
||||
|
||||
sqlitePath, err := resolvePathRelativeToApp(normalizeSQLitePath(s.SQLitePath))
|
||||
dsn, err := buildPostgresDSNFromRecorderSettings(s)
|
||||
if err != nil {
|
||||
return ModelStoreConfig{}, err
|
||||
}
|
||||
return ModelStoreConfig{
|
||||
Type: databaseTypeSQLite,
|
||||
DSN: sqlitePath,
|
||||
SQLitePath: sqlitePath,
|
||||
}, nil
|
||||
return ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn}, nil
|
||||
}
|
||||
|
||||
@ -17,7 +17,6 @@ import (
|
||||
"time"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type StoredModel struct {
|
||||
@ -69,16 +68,7 @@ type ModelsMeta struct {
|
||||
}
|
||||
|
||||
const (
|
||||
sqliteMaxOpenConns = 4
|
||||
sqliteMaxIdleConns = 4
|
||||
sqliteBusyTimeoutMS = 15000
|
||||
sqliteCacheSizeKiB = 65536
|
||||
sqliteMMapSizeBytes = 268435456
|
||||
sqliteWALAutoCheckpt = 1000
|
||||
sqliteAnalyzeLimit = 1000
|
||||
|
||||
modelProfileImageDirName = "model-profile-images"
|
||||
modelStatusFlushDelay = 1500 * time.Millisecond
|
||||
modelStatusFlushDelay = 1500 * time.Millisecond
|
||||
)
|
||||
|
||||
// Kleine Payload für "watched" Listen (für Autostart/Abgleich)
|
||||
@ -269,42 +259,9 @@ func parseRFC3339Nano(s string) sql.NullTime {
|
||||
func NewModelStore(config ModelStoreConfig) *ModelStore {
|
||||
config.Type = normalizeDatabaseType(config.Type)
|
||||
config.DSN = strings.TrimSpace(config.DSN)
|
||||
config.SQLitePath = strings.TrimSpace(config.SQLitePath)
|
||||
return &ModelStore{config: config}
|
||||
}
|
||||
|
||||
func sqliteTuningPragmas() []string {
|
||||
return []string{
|
||||
fmt.Sprintf("busy_timeout=%d", sqliteBusyTimeoutMS),
|
||||
"foreign_keys=ON",
|
||||
"journal_mode=WAL",
|
||||
"synchronous=NORMAL",
|
||||
"temp_store=MEMORY",
|
||||
fmt.Sprintf("cache_size=-%d", sqliteCacheSizeKiB),
|
||||
fmt.Sprintf("mmap_size=%d", sqliteMMapSizeBytes),
|
||||
fmt.Sprintf("wal_autocheckpoint=%d", sqliteWALAutoCheckpt),
|
||||
fmt.Sprintf("analysis_limit=%d", sqliteAnalyzeLimit),
|
||||
}
|
||||
}
|
||||
|
||||
func sqliteTunedDSN(path string) string {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
q := url.Values{}
|
||||
for _, pragma := range sqliteTuningPragmas() {
|
||||
q.Add("_pragma", pragma)
|
||||
}
|
||||
|
||||
sep := "?"
|
||||
if strings.Contains(path, "?") {
|
||||
sep = "&"
|
||||
}
|
||||
return path + sep + q.Encode()
|
||||
}
|
||||
|
||||
func (s *ModelStore) Load() error { return s.ensureInit() }
|
||||
|
||||
func (s *ModelStore) Close() error {
|
||||
@ -336,25 +293,6 @@ func (s *ModelStore) init() error {
|
||||
|
||||
driverName := "pgx"
|
||||
dsn := strings.TrimSpace(config.DSN)
|
||||
if config.Type == databaseTypeSQLite {
|
||||
sqlitePath := strings.TrimSpace(config.SQLitePath)
|
||||
if sqlitePath == "" {
|
||||
sqlitePath = dsn
|
||||
}
|
||||
if strings.TrimSpace(sqlitePath) == "" {
|
||||
return errors.New("sqlite path fehlt")
|
||||
}
|
||||
info, err := os.Stat(sqlitePath)
|
||||
if err == nil && info.IsDir() {
|
||||
return fmt.Errorf("sqlite path ist ein ordner: %s", sqlitePath)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(sqlitePath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
driverName = "sqlite"
|
||||
dsn = sqliteTunedDSN(sqlitePath)
|
||||
config.SQLitePath = sqlitePath
|
||||
}
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
return errors.New("db dsn fehlt")
|
||||
}
|
||||
@ -364,13 +302,8 @@ func (s *ModelStore) init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if config.Type == databaseTypeSQLite {
|
||||
db.SetMaxOpenConns(sqliteMaxOpenConns)
|
||||
db.SetMaxIdleConns(sqliteMaxIdleConns)
|
||||
} else {
|
||||
db.SetMaxOpenConns(10)
|
||||
db.SetMaxIdleConns(10)
|
||||
}
|
||||
db.SetMaxOpenConns(10)
|
||||
db.SetMaxIdleConns(10)
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
_ = db.Close()
|
||||
@ -380,16 +313,9 @@ func (s *ModelStore) init() error {
|
||||
s.db = db
|
||||
s.config = config
|
||||
|
||||
if config.Type == databaseTypeSQLite {
|
||||
if err := s.ensureSQLiteSchema(); err != nil {
|
||||
_ = db.Close()
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := s.ensurePostgresSchema(); err != nil {
|
||||
_ = db.Close()
|
||||
return err
|
||||
}
|
||||
if err := s.ensurePostgresSchema(); err != nil {
|
||||
_ = db.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.normalizeNameOnlyChaturbate(); err != nil {
|
||||
@ -399,147 +325,18 @@ func (s *ModelStore) init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ModelStore) isSQLite() bool {
|
||||
return s != nil && normalizeDatabaseType(s.config.Type) == databaseTypeSQLite
|
||||
}
|
||||
|
||||
func (s *ModelStore) blobLengthExpr() string {
|
||||
if s.isSQLite() {
|
||||
return "length(profile_image_blob)"
|
||||
}
|
||||
return "octet_length(profile_image_blob)"
|
||||
}
|
||||
|
||||
func (s *ModelStore) profileImagePresenceExpr() string {
|
||||
if s.isSQLite() {
|
||||
return "CASE WHEN trim(COALESCE(profile_image_path,'')) <> '' OR profile_image_blob IS NOT NULL THEN 1 ELSE 0 END"
|
||||
}
|
||||
return fmt.Sprintf("CASE WHEN trim(COALESCE(profile_image_path,'')) <> '' OR (profile_image_blob IS NOT NULL AND %s > 0) THEN 1 ELSE 0 END", s.blobLengthExpr())
|
||||
}
|
||||
|
||||
func (s *ModelStore) emptyTextExpr() string {
|
||||
if s.isSQLite() {
|
||||
return "''"
|
||||
}
|
||||
return "''::text"
|
||||
}
|
||||
|
||||
func (s *ModelStore) ensureSQLiteModelColumns() error {
|
||||
rows, err := s.db.Query(`PRAGMA table_info(models);`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
existing := make(map[string]struct{}, 64)
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid int
|
||||
name string
|
||||
colType string
|
||||
notNull int
|
||||
defaultV sql.NullString
|
||||
primaryID int
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &colType, ¬Null, &defaultV, &primaryID); err != nil {
|
||||
return err
|
||||
}
|
||||
existing[strings.ToLower(strings.TrimSpace(name))] = struct{}{}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
columns := []struct {
|
||||
name string
|
||||
ddl string
|
||||
}{
|
||||
{name: "cb_online_last_error", ddl: "cb_online_last_error TEXT DEFAULT ''"},
|
||||
{name: "profile_image_path", ddl: "profile_image_path TEXT DEFAULT ''"},
|
||||
{name: "profile_image_hash", ddl: "profile_image_hash TEXT DEFAULT ''"},
|
||||
}
|
||||
|
||||
for _, column := range columns {
|
||||
if _, ok := existing[strings.ToLower(column.name)]; ok {
|
||||
continue
|
||||
}
|
||||
if _, err := s.db.Exec(`ALTER TABLE models ADD COLUMN ` + column.ddl + `;`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ModelStore) ensureSQLiteSchema() error {
|
||||
stmts := make([]string, 0, len(sqliteTuningPragmas())+6)
|
||||
for _, pragma := range sqliteTuningPragmas() {
|
||||
stmts = append(stmts, "PRAGMA "+pragma+";")
|
||||
}
|
||||
|
||||
stmts = append(stmts,
|
||||
`CREATE TABLE IF NOT EXISTS models (
|
||||
id TEXT PRIMARY KEY,
|
||||
input TEXT NOT NULL DEFAULT '',
|
||||
is_url BOOLEAN NOT NULL DEFAULT 0,
|
||||
host TEXT DEFAULT '',
|
||||
path TEXT DEFAULT '',
|
||||
model_key TEXT NOT NULL DEFAULT '',
|
||||
tags TEXT DEFAULT '',
|
||||
last_stream DATETIME NULL,
|
||||
last_seen_online BOOLEAN NULL,
|
||||
last_seen_online_at DATETIME NULL,
|
||||
cb_online_json TEXT DEFAULT '',
|
||||
cb_online_fetched_at DATETIME NULL,
|
||||
cb_online_last_error TEXT DEFAULT '',
|
||||
profile_image_url TEXT DEFAULT '',
|
||||
profile_image_mime TEXT DEFAULT '',
|
||||
profile_image_path TEXT DEFAULT '',
|
||||
profile_image_hash TEXT DEFAULT '',
|
||||
profile_image_blob BLOB NULL,
|
||||
profile_image_updated_at DATETIME NULL,
|
||||
room_status TEXT DEFAULT '',
|
||||
is_online BOOLEAN NOT NULL DEFAULT 0,
|
||||
chat_room_url TEXT DEFAULT '',
|
||||
image_url TEXT DEFAULT '',
|
||||
last_online_at DATETIME NULL,
|
||||
last_offline_at DATETIME NULL,
|
||||
last_room_sync_at DATETIME NULL,
|
||||
biocontext_json TEXT DEFAULT '',
|
||||
biocontext_fetched_at DATETIME NULL,
|
||||
watching BOOLEAN NOT NULL DEFAULT 0,
|
||||
favorite BOOLEAN NOT NULL DEFAULT 0,
|
||||
hot BOOLEAN NOT NULL DEFAULT 0,
|
||||
keep BOOLEAN NOT NULL DEFAULT 0,
|
||||
liked BOOLEAN NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_models_host_model_key ON models(host, model_key);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_models_model_key ON models(model_key);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_models_watching_host ON models(watching, host);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_models_watching_updated ON models(watching, updated_at DESC);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_models_favorite_updated ON models(favorite, updated_at DESC);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_models_liked_updated ON models(liked, updated_at DESC);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_models_updated_at ON models(updated_at);`,
|
||||
)
|
||||
for _, stmt := range stmts {
|
||||
if _, err := s.db.Exec(stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := s.ensureSQLiteModelColumns(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.migrateSQLiteProfileImageBlobsToFiles(); err != nil {
|
||||
appLogln("models sqlite profile image blob migration skipped:", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`PRAGMA optimize;`); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ModelStore) ensurePostgresSchema() error {
|
||||
stmts := []string{
|
||||
`CREATE TABLE IF NOT EXISTS models (
|
||||
@ -1044,38 +841,11 @@ WHERE lower(trim(host)) = 'chaturbate.com'
|
||||
|
||||
// --- Profile image cache ---
|
||||
|
||||
func profileImageExtForMime(mime string) string {
|
||||
m := strings.ToLower(strings.TrimSpace(mime))
|
||||
if i := strings.Index(m, ";"); i >= 0 {
|
||||
m = strings.TrimSpace(m[:i])
|
||||
}
|
||||
switch m {
|
||||
case "image/png":
|
||||
return ".png"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
case "image/gif":
|
||||
return ".gif"
|
||||
case "image/jpeg", "image/jpg":
|
||||
return ".jpg"
|
||||
default:
|
||||
return ".img"
|
||||
}
|
||||
}
|
||||
|
||||
func profileImageHash(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return fmt.Sprintf("%x", sum[:])
|
||||
}
|
||||
|
||||
func (s *ModelStore) profileImageRelPath(hash string, mime string) string {
|
||||
hash = strings.ToLower(strings.TrimSpace(hash))
|
||||
if len(hash) < 2 {
|
||||
hash = "unknown"
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(modelProfileImageDirName, hash[:2], hash+profileImageExtForMime(mime)))
|
||||
}
|
||||
|
||||
func (s *ModelStore) resolveProfileImagePath(storedPath string) (string, error) {
|
||||
storedPath = strings.TrimSpace(storedPath)
|
||||
if storedPath == "" {
|
||||
@ -1090,109 +860,15 @@ func (s *ModelStore) resolveProfileImagePath(storedPath string) (string, error)
|
||||
return "", fmt.Errorf("ungueltiger profile image path: %s", storedPath)
|
||||
}
|
||||
|
||||
sqlitePath := strings.TrimSpace(s.config.SQLitePath)
|
||||
if sqlitePath == "" {
|
||||
sqlitePath = strings.TrimSpace(s.config.DSN)
|
||||
slashPath := filepath.ToSlash(local)
|
||||
if slashPath == "generated" || strings.HasPrefix(slashPath, "generated/") {
|
||||
return resolvePathRelativeToApp(slashPath)
|
||||
}
|
||||
if sqlitePath == "" {
|
||||
return "", errors.New("sqlite path fehlt")
|
||||
}
|
||||
return filepath.Join(filepath.Dir(sqlitePath), local), nil
|
||||
|
||||
return "", fmt.Errorf("ungueltiger profile image path: %s", storedPath)
|
||||
}
|
||||
|
||||
func (s *ModelStore) storeSQLiteProfileImageFile(data []byte, mime string) (relPath string, hash string, err error) {
|
||||
if len(data) == 0 {
|
||||
return "", "", errors.New("image data fehlt")
|
||||
}
|
||||
hash = profileImageHash(data)
|
||||
relPath = s.profileImageRelPath(hash, mime)
|
||||
absPath, err := s.resolveProfileImagePath(relPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := atomicWriteFile(absPath, data); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return relPath, hash, nil
|
||||
}
|
||||
|
||||
func (s *ModelStore) migrateSQLiteProfileImageBlobsToFiles() error {
|
||||
if !s.isSQLite() {
|
||||
return nil
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(`
|
||||
SELECT id, COALESCE(profile_image_mime,''), profile_image_blob
|
||||
FROM models
|
||||
WHERE profile_image_blob IS NOT NULL
|
||||
AND (profile_image_path IS NULL OR trim(profile_image_path) = '');
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type migratedProfileImage struct {
|
||||
id string
|
||||
path string
|
||||
hash string
|
||||
}
|
||||
|
||||
migrated := make([]migratedProfileImage, 0, 32)
|
||||
for rows.Next() {
|
||||
var (
|
||||
id string
|
||||
mime string
|
||||
blob []byte
|
||||
)
|
||||
if err := rows.Scan(&id, &mime, &blob); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(id) == "" || len(blob) == 0 {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(mime) == "" {
|
||||
mime = http.DetectContentType(blob)
|
||||
}
|
||||
relPath, hash, err := s.storeSQLiteProfileImageFile(blob, mime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
migrated = append(migrated, migratedProfileImage{id: id, path: relPath, hash: hash})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(migrated) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
UPDATE models
|
||||
SET profile_image_path=$1, profile_image_hash=$2, profile_image_blob=NULL
|
||||
WHERE id=$3;
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, item := range migrated {
|
||||
if _, err := stmt.Exec(item.path, item.hash, item.id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// SetProfileImage speichert Bild-URL + MIME. SQLite legt die Bilddaten als Datei neben der DB ab.
|
||||
// SetProfileImage speichert Bild-URL, MIME und Bilddaten in Postgres.
|
||||
// Speichert nur fuer bereits bestehende Models.
|
||||
func (s *ModelStore) SetProfileImage(host, modelKey, sourceURL, mime string, data []byte, updatedAt string) error {
|
||||
if err := s.ensureInit(); err != nil {
|
||||
@ -1236,14 +912,6 @@ LIMIT 1;
|
||||
}
|
||||
|
||||
hash := profileImageHash(data)
|
||||
imagePath := ""
|
||||
if s.isSQLite() {
|
||||
var err error
|
||||
imagePath, hash, err = s.storeSQLiteProfileImageFile(data, mime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
nt := parseRFC3339Nano(updatedAt)
|
||||
if !nt.Valid {
|
||||
@ -1254,22 +922,12 @@ LIMIT 1;
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
var res sql.Result
|
||||
if s.isSQLite() {
|
||||
res, err = s.db.Exec(`
|
||||
UPDATE models
|
||||
SET profile_image_url=$1, profile_image_mime=$2, profile_image_path=$3, profile_image_hash=$4, profile_image_blob=NULL, profile_image_updated_at=$5, updated_at=$6
|
||||
WHERE lower(trim(host)) = lower(trim($7))
|
||||
AND lower(trim(model_key)) = lower(trim($8));
|
||||
`, src, mime, imagePath, hash, nullableTimeArg(nt), now, host, key)
|
||||
} else {
|
||||
res, err = s.db.Exec(`
|
||||
res, err := s.db.Exec(`
|
||||
UPDATE models
|
||||
SET profile_image_url=$1, profile_image_mime=$2, profile_image_path='', profile_image_hash=$3, profile_image_blob=$4, profile_image_updated_at=$5, updated_at=$6
|
||||
WHERE lower(trim(host)) = lower(trim($7))
|
||||
AND lower(trim(model_key)) = lower(trim($8));
|
||||
`, src, mime, hash, data, nullableTimeArg(nt), now, host, key)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -1381,12 +1039,6 @@ LIMIT 1;
|
||||
}
|
||||
}
|
||||
|
||||
if s.isSQLite() {
|
||||
if relPath, hash, werr := s.storeSQLiteProfileImageFile(blob, m); werr == nil {
|
||||
_, _ = s.db.Exec(`UPDATE models SET profile_image_path=$1, profile_image_hash=$2, profile_image_blob=NULL WHERE id=$3;`, relPath, hash, id)
|
||||
}
|
||||
}
|
||||
|
||||
return m, blob, true, nil
|
||||
}
|
||||
|
||||
@ -1656,10 +1308,6 @@ UPDATE models SET
|
||||
updated_at = CASE WHEN updated_at < $12 THEN $12 ELSE updated_at END
|
||||
WHERE id = $13;
|
||||
`
|
||||
if s.isSQLite() {
|
||||
updateSQL = strings.ReplaceAll(updateSQL, "$6::timestamptz", "$6")
|
||||
updateSQL = strings.ReplaceAll(updateSQL, "$11::boolean", "$11")
|
||||
}
|
||||
_, err = tx.Exec(updateSQL,
|
||||
newInput, host, newPath,
|
||||
it.tags, it.tags,
|
||||
|
||||
@ -1,235 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestModelStoreSQLiteCreatesSchemaAndPersists(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "models.sqlite")
|
||||
store := NewModelStore(ModelStoreConfig{
|
||||
Type: databaseTypeSQLite,
|
||||
DSN: dbPath,
|
||||
SQLitePath: dbPath,
|
||||
})
|
||||
defer store.Close()
|
||||
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("load sqlite store: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(dbPath); err != nil {
|
||||
t.Fatalf("sqlite file not created: %v", err)
|
||||
}
|
||||
|
||||
model, err := store.EnsureByHostModelKey("chaturbate.com", "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("ensure model: %v", err)
|
||||
}
|
||||
if model.ID != "chaturbate.com:alice" {
|
||||
t.Fatalf("unexpected model id: %q", model.ID)
|
||||
}
|
||||
|
||||
if err := store.SetLastSeenOnline("chaturbate.com", "alice", true, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
t.Fatalf("set last seen online: %v", err)
|
||||
}
|
||||
if err := store.flushPendingStatusUpdates(); err != nil {
|
||||
t.Fatalf("flush status updates: %v", err)
|
||||
}
|
||||
|
||||
models := store.List()
|
||||
if len(models) != 1 {
|
||||
t.Fatalf("expected 1 model, got %d", len(models))
|
||||
}
|
||||
if models[0].LastSeenOnline == nil || !*models[0].LastSeenOnline {
|
||||
t.Fatalf("lastSeenOnline not persisted: %#v", models[0].LastSeenOnline)
|
||||
}
|
||||
|
||||
meta := store.Meta()
|
||||
if meta.Count != 1 {
|
||||
t.Fatalf("expected meta count 1, got %d", meta.Count)
|
||||
}
|
||||
if meta.UpdatedAt == "" {
|
||||
t.Fatal("expected meta updatedAt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelStoreSQLiteStoresProfileImagesAsFiles(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "models.sqlite")
|
||||
store := NewModelStore(ModelStoreConfig{
|
||||
Type: databaseTypeSQLite,
|
||||
DSN: dbPath,
|
||||
SQLitePath: dbPath,
|
||||
})
|
||||
defer store.Close()
|
||||
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("load sqlite store: %v", err)
|
||||
}
|
||||
if _, err := store.EnsureByHostModelKey("chaturbate.com", "alice"); err != nil {
|
||||
t.Fatalf("ensure model: %v", err)
|
||||
}
|
||||
data := []byte{0xff, 0xd8, 0xff, 0xdb, 0x00, 0x43, 0x00}
|
||||
if err := store.SetProfileImage("chaturbate.com", "alice", "https://example.test/a.jpg", "image/jpeg", data, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
t.Fatalf("set profile image: %v", err)
|
||||
}
|
||||
|
||||
var relPath, hash string
|
||||
var blob []byte
|
||||
if err := store.db.QueryRow(`SELECT profile_image_path, profile_image_hash, profile_image_blob FROM models WHERE id=$1;`, "chaturbate.com:alice").Scan(&relPath, &hash, &blob); err != nil {
|
||||
t.Fatalf("query profile image fields: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(relPath) == "" {
|
||||
t.Fatal("expected profile image path")
|
||||
}
|
||||
if strings.TrimSpace(hash) == "" {
|
||||
t.Fatal("expected profile image hash")
|
||||
}
|
||||
if len(blob) != 0 {
|
||||
t.Fatalf("expected sqlite blob cleared, got %d bytes", len(blob))
|
||||
}
|
||||
|
||||
absPath, err := store.resolveProfileImagePath(relPath)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve profile image path: %v", err)
|
||||
}
|
||||
if b, err := os.ReadFile(absPath); err != nil {
|
||||
t.Fatalf("read profile image file: %v", err)
|
||||
} else if string(b) != string(data) {
|
||||
t.Fatalf("unexpected profile image data: %v", b)
|
||||
}
|
||||
|
||||
mime, got, ok, err := store.GetProfileImageByID("chaturbate.com:alice")
|
||||
if err != nil {
|
||||
t.Fatalf("get profile image: %v", err)
|
||||
}
|
||||
if !ok || mime != "image/jpeg" || string(got) != string(data) {
|
||||
t.Fatalf("unexpected profile image response ok=%v mime=%q data=%v", ok, mime, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateModelStoreDataSQLiteToSQLiteReplacesTarget(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
sourcePath := filepath.Join(tmp, "source.sqlite")
|
||||
targetPath := filepath.Join(tmp, "target.sqlite")
|
||||
|
||||
sourceConfig := ModelStoreConfig{Type: databaseTypeSQLite, DSN: sourcePath, SQLitePath: sourcePath}
|
||||
targetConfig := ModelStoreConfig{Type: databaseTypeSQLite, DSN: targetPath, SQLitePath: targetPath}
|
||||
|
||||
source := NewModelStore(sourceConfig)
|
||||
defer source.Close()
|
||||
if err := source.Load(); err != nil {
|
||||
t.Fatalf("load source: %v", err)
|
||||
}
|
||||
if _, err := source.EnsureByHostModelKey("chaturbate.com", "alice"); err != nil {
|
||||
t.Fatalf("ensure source model: %v", err)
|
||||
}
|
||||
if err := source.SetProfileImage("chaturbate.com", "alice", "https://example.test/a.jpg", "image/jpeg", []byte{1, 2, 3, 4}, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
t.Fatalf("set profile image: %v", err)
|
||||
}
|
||||
if err := source.SetBioContext("chaturbate.com", "alice", `{"ok":true}`, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
t.Fatalf("set bio context: %v", err)
|
||||
}
|
||||
|
||||
target := NewModelStore(targetConfig)
|
||||
if err := target.Load(); err != nil {
|
||||
t.Fatalf("load target: %v", err)
|
||||
}
|
||||
if _, err := target.EnsureByHostModelKey("chaturbate.com", "stale"); err != nil {
|
||||
t.Fatalf("ensure stale target model: %v", err)
|
||||
}
|
||||
_ = target.Close()
|
||||
_ = source.Close()
|
||||
|
||||
count, err := migrateModelStoreData(sourceConfig, targetConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("migrate sqlite to sqlite: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("expected 1 migrated row, got %d", count)
|
||||
}
|
||||
|
||||
migrated := NewModelStore(targetConfig)
|
||||
defer migrated.Close()
|
||||
if err := migrated.Load(); err != nil {
|
||||
t.Fatalf("load migrated target: %v", err)
|
||||
}
|
||||
models := migrated.List()
|
||||
if len(models) != 1 {
|
||||
t.Fatalf("expected target replacement with 1 row, got %d", len(models))
|
||||
}
|
||||
if models[0].ModelKey != "alice" {
|
||||
t.Fatalf("unexpected migrated model: %q", models[0].ModelKey)
|
||||
}
|
||||
if models[0].ProfileImageCached == "" {
|
||||
t.Fatal("expected migrated profile image blob marker")
|
||||
}
|
||||
bio, _, ok, err := migrated.GetBioContext("chaturbate.com", "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("get migrated bio context: %v", err)
|
||||
}
|
||||
if !ok || bio != `{"ok":true}` {
|
||||
t.Fatalf("unexpected migrated bio context ok=%v bio=%q", ok, bio)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoveSQLiteDatabaseFilesMovesSidecars(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
sourcePath := filepath.Join(tmp, "old", "models.sqlite")
|
||||
targetPath := filepath.Join(tmp, "new", "models.sqlite")
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(sourcePath), 0755); err != nil {
|
||||
t.Fatalf("mkdir source: %v", err)
|
||||
}
|
||||
files := map[string]string{
|
||||
sourcePath: "main",
|
||||
sourcePath + "-wal": "wal",
|
||||
sourcePath + "-shm": "shm",
|
||||
sourcePath + "-journal": "journal",
|
||||
}
|
||||
for path, content := range files {
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
sourceImageDir := filepath.Join(filepath.Dir(sourcePath), modelProfileImageDirName)
|
||||
sourceImagePath := filepath.Join(sourceImageDir, "aa", "avatar.jpg")
|
||||
if err := os.MkdirAll(filepath.Dir(sourceImagePath), 0755); err != nil {
|
||||
t.Fatalf("mkdir profile image dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sourceImagePath, []byte("image"), 0644); err != nil {
|
||||
t.Fatalf("write profile image: %v", err)
|
||||
}
|
||||
|
||||
if err := moveSQLiteDatabaseFiles(sourcePath, targetPath); err != nil {
|
||||
t.Fatalf("move sqlite files: %v", err)
|
||||
}
|
||||
|
||||
for source, content := range files {
|
||||
if _, err := os.Stat(source); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected source removed %s, err=%v", source, err)
|
||||
}
|
||||
target := strings.Replace(source, sourcePath, targetPath, 1)
|
||||
b, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatalf("read target %s: %v", target, err)
|
||||
}
|
||||
if string(b) != content {
|
||||
t.Fatalf("unexpected target content for %s: %q", target, string(b))
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := os.Stat(sourceImagePath); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected source profile image moved, err=%v", err)
|
||||
}
|
||||
targetImagePath := strings.Replace(sourceImagePath, filepath.Dir(sourcePath), filepath.Dir(targetPath), 1)
|
||||
b, err := os.ReadFile(targetImagePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read target profile image: %v", err)
|
||||
}
|
||||
if string(b) != "image" {
|
||||
t.Fatalf("unexpected target profile image content: %q", string(b))
|
||||
}
|
||||
}
|
||||
250
backend/postgres_local.go
Normal file
250
backend/postgres_local.go
Normal file
@ -0,0 +1,250 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
postgresModeURL = "url"
|
||||
postgresModeLocal = "local"
|
||||
|
||||
defaultPostgresDataDir = "postgres-data"
|
||||
defaultPostgresPort = 55432
|
||||
defaultPostgresDatabase = "nsfwapp"
|
||||
defaultPostgresUser = "nsfwapp"
|
||||
)
|
||||
|
||||
func normalizePostgresMode(raw string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case postgresModeLocal:
|
||||
return postgresModeLocal
|
||||
default:
|
||||
return postgresModeURL
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePostgresDataDir(raw string) string {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
value = defaultPostgresDataDir
|
||||
}
|
||||
return filepath.Clean(filepath.FromSlash(value))
|
||||
}
|
||||
|
||||
func normalizePostgresPort(port int) int {
|
||||
if port < 1 || port > 65535 {
|
||||
return defaultPostgresPort
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
func normalizePostgresName(raw, fallback string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
raw = fallback
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
for _, r := range raw {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
b.WriteRune(r)
|
||||
} else if b.Len() > 0 {
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "_")
|
||||
if out == "" {
|
||||
return fallback
|
||||
}
|
||||
if len(out) > 63 {
|
||||
out = out[:63]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizePostgresSettings(s *RecorderSettings) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.DatabaseType = normalizeDatabaseType(s.DatabaseType)
|
||||
s.PostgresMode = normalizePostgresMode(s.PostgresMode)
|
||||
s.PostgresDataDir = normalizePostgresDataDir(s.PostgresDataDir)
|
||||
s.PostgresPort = normalizePostgresPort(s.PostgresPort)
|
||||
s.PostgresDatabase = normalizePostgresName(s.PostgresDatabase, defaultPostgresDatabase)
|
||||
s.PostgresUser = normalizePostgresName(s.PostgresUser, defaultPostgresUser)
|
||||
s.DatabaseURL = strings.TrimSpace(s.DatabaseURL)
|
||||
}
|
||||
|
||||
func postgresSettingsKey(s RecorderSettings) string {
|
||||
normalizePostgresSettings(&s)
|
||||
if s.PostgresMode == postgresModeLocal {
|
||||
return strings.Join([]string{
|
||||
s.PostgresMode,
|
||||
filepath.ToSlash(s.PostgresDataDir),
|
||||
strconv.Itoa(s.PostgresPort),
|
||||
s.PostgresDatabase,
|
||||
s.PostgresUser,
|
||||
}, "\x00")
|
||||
}
|
||||
return strings.Join([]string{s.PostgresMode, strings.TrimSpace(s.DatabaseURL)}, "\x00")
|
||||
}
|
||||
|
||||
func buildLocalPostgresDSN(s RecorderSettings) (string, error) {
|
||||
normalizePostgresSettings(&s)
|
||||
dataDir, err := resolvePathRelativeToApp(s.PostgresDataDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.TrimSpace(dataDir) == "" {
|
||||
return "", appErrorf("postgresDataDir ist leer")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := ensureLocalPostgresCluster(ctx, dataDir, s.PostgresPort, s.PostgresUser, s.PostgresDatabase); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
u := url.URL{
|
||||
Scheme: "postgres",
|
||||
User: url.User(s.PostgresUser),
|
||||
Host: "127.0.0.1:" + strconv.Itoa(s.PostgresPort),
|
||||
Path: "/" + s.PostgresDatabase,
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("sslmode", "disable")
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func ensureLocalPostgresCluster(ctx context.Context, dataDir string, port int, user, database string) error {
|
||||
dataDir = filepath.Clean(strings.TrimSpace(dataDir))
|
||||
if dataDir == "" {
|
||||
return appErrorf("postgresDataDir ist leer")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dataDir), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionFile := filepath.Join(dataDir, "PG_VERSION")
|
||||
if _, err := os.Stat(versionFile); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if err := initLocalPostgresCluster(ctx, dataDir, user); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := startLocalPostgresCluster(ctx, dataDir, port); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureLocalPostgresDatabase(ctx, port, user, database)
|
||||
}
|
||||
|
||||
func initLocalPostgresCluster(ctx context.Context, dataDir, user string) error {
|
||||
initdb, err := postgresToolPath("initdb")
|
||||
if err != nil {
|
||||
return appErrorf("initdb wurde nicht gefunden. Installiere PostgreSQL-Server-Tools oder nimm initdb in PATH auf.")
|
||||
}
|
||||
if err := os.MkdirAll(dataDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := runLocalPostgresTool(ctx, initdb,
|
||||
"-D", dataDir,
|
||||
"-U", user,
|
||||
"--encoding=UTF8",
|
||||
"--auth-local=trust",
|
||||
"--auth-host=trust",
|
||||
)
|
||||
if err != nil {
|
||||
return appErrorf("PostgreSQL-Datenordner konnte nicht initialisiert werden: %s", out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func startLocalPostgresCluster(ctx context.Context, dataDir string, port int) error {
|
||||
pgCtl, err := postgresToolPath("pg_ctl")
|
||||
if err != nil {
|
||||
return appErrorf("pg_ctl wurde nicht gefunden. Installiere PostgreSQL-Server-Tools oder nimm pg_ctl in PATH auf.")
|
||||
}
|
||||
|
||||
_, _ = runLocalPostgresTool(ctx, pgCtl, "-D", dataDir, "status")
|
||||
|
||||
logPath := filepath.Join(dataDir, "postgresql.log")
|
||||
opts := fmt.Sprintf("-p %d -h 127.0.0.1", port)
|
||||
out, err := runLocalPostgresTool(ctx, pgCtl, "-D", dataDir, "-l", logPath, "-o", opts, "-w", "start")
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if strings.Contains(strings.ToLower(out), "server is running") || strings.Contains(strings.ToLower(out), "server running") {
|
||||
return nil
|
||||
}
|
||||
return appErrorf("lokaler PostgreSQL-Server konnte nicht gestartet werden: %s", out)
|
||||
}
|
||||
|
||||
func ensureLocalPostgresDatabase(ctx context.Context, port int, user, database string) error {
|
||||
psql, err := postgresToolPath("psql")
|
||||
if err != nil {
|
||||
return appErrorf("psql wurde nicht gefunden. Installiere postgresql-client oder nimm psql in PATH auf.")
|
||||
}
|
||||
|
||||
sql := "SELECT 1 FROM pg_database WHERE datname = '" + strings.ReplaceAll(database, "'", "''") + "';"
|
||||
out, err := runLocalPostgresTool(ctx, psql,
|
||||
"-h", "127.0.0.1",
|
||||
"-p", strconv.Itoa(port),
|
||||
"-U", user,
|
||||
"-d", "postgres",
|
||||
"-tAc", sql,
|
||||
)
|
||||
if err == nil && strings.TrimSpace(out) == "1" {
|
||||
return nil
|
||||
}
|
||||
|
||||
createdb, err := postgresToolPath("createdb")
|
||||
if err != nil {
|
||||
return appErrorf("createdb wurde nicht gefunden. Installiere postgresql-client oder nimm createdb in PATH auf.")
|
||||
}
|
||||
out, err = runLocalPostgresTool(ctx, createdb,
|
||||
"-h", "127.0.0.1",
|
||||
"-p", strconv.Itoa(port),
|
||||
"-U", user,
|
||||
database,
|
||||
)
|
||||
if err != nil {
|
||||
return appErrorf("lokale PostgreSQL-Datenbank konnte nicht angelegt werden: %s", out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runLocalPostgresTool(ctx context.Context, tool string, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, tool, args...)
|
||||
cmd.Env = upsertEnv(os.Environ(), "PGCONNECT_TIMEOUT", "10")
|
||||
hideCommandWindow(cmd)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
err := cmd.Run()
|
||||
|
||||
out := strings.TrimSpace(stdout.String())
|
||||
errOut := strings.TrimSpace(stderr.String())
|
||||
if errOut != "" {
|
||||
if out != "" {
|
||||
out += "\n"
|
||||
}
|
||||
out += errOut
|
||||
}
|
||||
return trimCommandOutput(out), err
|
||||
}
|
||||
@ -32,6 +32,7 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore {
|
||||
mux.HandleFunc("/api/auth/passkey/login/options", authPasskeyLoginOptionsHandler(auth))
|
||||
mux.HandleFunc("/api/auth/passkey/login/verify", authPasskeyLoginVerifyHandler(auth))
|
||||
mux.HandleFunc("/api/auth/passkey/delete", authPasskeyDeleteHandler(auth))
|
||||
mux.HandleFunc("/api/setup/ml/status", mlPythonSetupStatusHandler)
|
||||
|
||||
// --------------------------
|
||||
// 2) Protected API Mux
|
||||
@ -128,11 +129,7 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore {
|
||||
if err != nil {
|
||||
appLogln("models DB config:", err)
|
||||
}
|
||||
if storeConfig.Type == databaseTypePostgres {
|
||||
appLogln("Models DB (Postgres):", sanitizeDSNForLog(storeConfig.DSN))
|
||||
} else {
|
||||
appLogln("Models DB (SQLite):", storeConfig.SQLitePath)
|
||||
}
|
||||
appLogln("Models DB (Postgres):", sanitizeDSNForLog(storeConfig.DSN))
|
||||
|
||||
store := NewModelStore(storeConfig)
|
||||
if err := store.Load(); err != nil {
|
||||
@ -174,6 +171,10 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore {
|
||||
func buildPostgresDSNFromSettings() (string, error) {
|
||||
// Settings sind bereits durch loadSettings() in main() geladen.
|
||||
s := getSettings()
|
||||
normalizePostgresSettings(&s)
|
||||
if s.PostgresMode == postgresModeLocal {
|
||||
return buildLocalPostgresDSN(s)
|
||||
}
|
||||
|
||||
dbURL := strings.TrimSpace(s.DatabaseURL)
|
||||
if dbURL == "" {
|
||||
@ -228,6 +229,11 @@ func buildPostgresDSNFromSettings() (string, error) {
|
||||
}
|
||||
|
||||
func buildPostgresDSNFromRecorderSettings(s RecorderSettings) (string, error) {
|
||||
normalizePostgresSettings(&s)
|
||||
if s.PostgresMode == postgresModeLocal {
|
||||
return buildLocalPostgresDSN(s)
|
||||
}
|
||||
|
||||
dbURL := strings.TrimSpace(s.DatabaseURL)
|
||||
if dbURL == "" {
|
||||
return "", appErrorf("databaseUrl ist leer")
|
||||
|
||||
@ -467,6 +467,10 @@ func startAIServer(ctx context.Context) (*aiServerProcess, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err := ensureMLPythonSetup(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scriptDir, err := findAIServerScriptDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -973,6 +977,12 @@ func main() {
|
||||
|
||||
appCtx, appCancel := context.WithCancel(context.Background())
|
||||
defer appCancel()
|
||||
initMLPythonSetupStatus()
|
||||
|
||||
aiServerMu.Lock()
|
||||
aiServerCtx = appCtx
|
||||
aiServerCurrent = nil
|
||||
aiServerMu.Unlock()
|
||||
|
||||
listenAddr := ":" + appPort()
|
||||
|
||||
@ -988,21 +998,6 @@ func main() {
|
||||
}
|
||||
defer appLn.Close()
|
||||
|
||||
aiProc, err := startAIServer(appCtx)
|
||||
if err != nil {
|
||||
appLogln("⚠️ AI Server konnte nicht gestartet werden:", err)
|
||||
}
|
||||
|
||||
// Handle/Context global hinterlegen, damit der Server später neu gestartet
|
||||
// werden kann (z.B. nach einem neuen Modell).
|
||||
aiServerMu.Lock()
|
||||
aiServerCtx = appCtx
|
||||
aiServerCurrent = aiProc
|
||||
watchedTrainingRoot := aiServerRoot
|
||||
aiServerMu.Unlock()
|
||||
|
||||
go startTrainingBestModelWatcher(appCtx, watchedTrainingRoot)
|
||||
|
||||
// ✅ Hier: alte manuelle Autostart-Pause beim echten App-Start zurücksetzen.
|
||||
resetAutostartPauseOnStartup()
|
||||
|
||||
@ -1132,6 +1127,26 @@ func main() {
|
||||
serverErrCh <- nil
|
||||
}()
|
||||
|
||||
go func() {
|
||||
aiProc, err := startAIServer(appCtx)
|
||||
if err != nil {
|
||||
appLogln("⚠️ AI Server konnte nicht gestartet werden:", err)
|
||||
}
|
||||
if appCtx.Err() != nil {
|
||||
if aiProc != nil {
|
||||
aiProc.Stop()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
aiServerMu.Lock()
|
||||
aiServerCurrent = aiProc
|
||||
watchedTrainingRoot := aiServerRoot
|
||||
aiServerMu.Unlock()
|
||||
|
||||
go startTrainingBestModelWatcher(appCtx, watchedTrainingRoot)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
_ = showAppNotification(
|
||||
"Recorder gestartet",
|
||||
|
||||
@ -14,8 +14,12 @@ import (
|
||||
)
|
||||
|
||||
type RecorderSettings struct {
|
||||
DatabaseType string `json:"databaseType"` // sqlite | postgres
|
||||
SQLitePath string `json:"sqlitePath"`
|
||||
DatabaseType string `json:"databaseType"`
|
||||
PostgresMode string `json:"postgresMode"` // url | local
|
||||
PostgresDataDir string `json:"postgresDataDir"`
|
||||
PostgresPort int `json:"postgresPort"`
|
||||
PostgresDatabase string `json:"postgresDatabase"`
|
||||
PostgresUser string `json:"postgresUser"`
|
||||
DatabaseURL string `json:"databaseUrl"`
|
||||
EncryptedDBPassword string `json:"encryptedDbPassword,omitempty"` // base64(nonce+ciphertext)
|
||||
|
||||
@ -67,8 +71,12 @@ type RecorderSettings struct {
|
||||
var (
|
||||
settingsMu sync.Mutex
|
||||
settings = RecorderSettings{
|
||||
DatabaseType: databaseTypeSQLite,
|
||||
SQLitePath: defaultSQLitePath,
|
||||
DatabaseType: databaseTypePostgres,
|
||||
PostgresMode: postgresModeURL,
|
||||
PostgresDataDir: defaultPostgresDataDir,
|
||||
PostgresPort: defaultPostgresPort,
|
||||
PostgresDatabase: defaultPostgresDatabase,
|
||||
PostgresUser: defaultPostgresUser,
|
||||
DatabaseURL: "",
|
||||
EncryptedDBPassword: "",
|
||||
|
||||
@ -165,7 +173,7 @@ func loadSettings() {
|
||||
s.RecordDir = normalizeSettingsDir(s.RecordDir, defaultRecordDir, legacyDefaultRecordDir)
|
||||
s.DoneDir = normalizeSettingsDir(s.DoneDir, defaultDoneDir, legacyDefaultDoneDir)
|
||||
s.DatabaseType = normalizeDatabaseType(s.DatabaseType)
|
||||
s.SQLitePath = normalizeSQLitePath(s.SQLitePath)
|
||||
normalizePostgresSettings(&s)
|
||||
if strings.TrimSpace(s.FFmpegPath) != "" {
|
||||
s.FFmpegPath = strings.TrimSpace(s.FFmpegPath)
|
||||
}
|
||||
@ -213,7 +221,7 @@ func loadSettings() {
|
||||
}
|
||||
|
||||
s.DatabaseType = normalizeDatabaseType(s.DatabaseType)
|
||||
s.SQLitePath = normalizeSQLitePath(s.SQLitePath)
|
||||
normalizePostgresSettings(&s)
|
||||
s.DatabaseURL = strings.TrimSpace(s.DatabaseURL)
|
||||
|
||||
// Optional: falls in der JSON mal ein URL MIT Passwort steht (Altbestand)
|
||||
@ -283,10 +291,14 @@ type RecorderSettingsPublic struct {
|
||||
DoneDir string `json:"doneDir"`
|
||||
FFmpegPath string `json:"ffmpegPath"`
|
||||
|
||||
DatabaseType string `json:"databaseType"`
|
||||
SQLitePath string `json:"sqlitePath"`
|
||||
DatabaseURL string `json:"databaseUrl"`
|
||||
HasDBPassword bool `json:"hasDbPassword"`
|
||||
DatabaseType string `json:"databaseType"`
|
||||
PostgresMode string `json:"postgresMode"`
|
||||
PostgresDataDir string `json:"postgresDataDir"`
|
||||
PostgresPort int `json:"postgresPort"`
|
||||
PostgresDatabase string `json:"postgresDatabase"`
|
||||
PostgresUser string `json:"postgresUser"`
|
||||
DatabaseURL string `json:"databaseUrl"`
|
||||
HasDBPassword bool `json:"hasDbPassword"`
|
||||
|
||||
AutoAddToDownloadList bool `json:"autoAddToDownloadList"`
|
||||
AutoStartAddedDownloads bool `json:"autoStartAddedDownloads"`
|
||||
@ -324,15 +336,20 @@ type RecorderSettingsPublic struct {
|
||||
}
|
||||
|
||||
func toPublicSettings(s RecorderSettings) RecorderSettingsPublic {
|
||||
normalizePostgresSettings(&s)
|
||||
return RecorderSettingsPublic{
|
||||
RecordDir: s.RecordDir,
|
||||
DoneDir: s.DoneDir,
|
||||
FFmpegPath: s.FFmpegPath,
|
||||
|
||||
DatabaseType: normalizeDatabaseType(s.DatabaseType),
|
||||
SQLitePath: normalizeSQLitePath(s.SQLitePath),
|
||||
DatabaseURL: strings.TrimSpace(s.DatabaseURL),
|
||||
HasDBPassword: strings.TrimSpace(s.EncryptedDBPassword) != "",
|
||||
DatabaseType: normalizeDatabaseType(s.DatabaseType),
|
||||
PostgresMode: s.PostgresMode,
|
||||
PostgresDataDir: s.PostgresDataDir,
|
||||
PostgresPort: s.PostgresPort,
|
||||
PostgresDatabase: s.PostgresDatabase,
|
||||
PostgresUser: s.PostgresUser,
|
||||
DatabaseURL: strings.TrimSpace(s.DatabaseURL),
|
||||
HasDBPassword: strings.TrimSpace(s.EncryptedDBPassword) != "",
|
||||
|
||||
AutoAddToDownloadList: s.AutoAddToDownloadList,
|
||||
AutoStartAddedDownloads: s.AutoStartAddedDownloads,
|
||||
@ -472,8 +489,12 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// --- DB URL + Passwort behandeln ---
|
||||
// 1) Trim
|
||||
in.DatabaseType = normalizeDatabaseType(in.DatabaseType)
|
||||
in.SQLitePath = normalizeSQLitePath(in.SQLitePath)
|
||||
normalizePostgresSettings(&in.RecorderSettings)
|
||||
in.DatabaseURL = strings.TrimSpace(in.DatabaseURL)
|
||||
if in.PostgresMode == postgresModeURL && in.DatabaseURL == "" {
|
||||
http.Error(w, "Database URL fehlt", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// 2) Migration: wenn in.DatabaseURL ein Passwort enthält, extrahieren
|
||||
// und URL ohne Passwort zurückschreiben.
|
||||
@ -514,7 +535,11 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
next.FFmpegPath = in.FFmpegPath
|
||||
|
||||
next.DatabaseType = in.DatabaseType
|
||||
next.SQLitePath = in.SQLitePath
|
||||
next.PostgresMode = in.PostgresMode
|
||||
next.PostgresDataDir = in.PostgresDataDir
|
||||
next.PostgresPort = in.PostgresPort
|
||||
next.PostgresDatabase = in.PostgresDatabase
|
||||
next.PostgresUser = in.PostgresUser
|
||||
next.DatabaseURL = in.DatabaseURL
|
||||
next.EncryptedDBPassword = in.EncryptedDBPassword
|
||||
|
||||
@ -554,7 +579,7 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
dbChanged :=
|
||||
normalizeDatabaseType(next.DatabaseType) != normalizeDatabaseType(current.DatabaseType) ||
|
||||
normalizeSQLitePath(next.SQLitePath) != normalizeSQLitePath(current.SQLitePath) ||
|
||||
postgresSettingsKey(next) != postgresSettingsKey(current) ||
|
||||
strings.TrimSpace(next.DatabaseURL) != strings.TrimSpace(current.DatabaseURL) ||
|
||||
strings.TrimSpace(next.EncryptedDBPassword) != strings.TrimSpace(current.EncryptedDBPassword)
|
||||
|
||||
@ -562,21 +587,8 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// DB erst prüfen, BEVOR gespeichert wird
|
||||
if dbChanged {
|
||||
sqliteMoved := false
|
||||
if sqlitePathChanged(current, next) {
|
||||
if err := moveSQLiteDatabaseForSettings(current, next); err != nil {
|
||||
http.Error(w, "SQLite-Datei konnte nicht verschoben werden: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
sqliteMoved = true
|
||||
}
|
||||
|
||||
storeConfig, err := buildModelStoreConfigFromRecorderSettings(next)
|
||||
if err != nil {
|
||||
if sqliteMoved {
|
||||
_ = moveSQLiteDatabaseForSettings(next, current)
|
||||
restoreModelStoreForSettings(current)
|
||||
}
|
||||
http.Error(w, "ungültige Datenbank-Konfiguration: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
@ -584,10 +596,6 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
newStore = NewModelStore(storeConfig)
|
||||
if err := newStore.Load(); err != nil {
|
||||
_ = newStore.Close()
|
||||
if sqliteMoved {
|
||||
_ = moveSQLiteDatabaseForSettings(next, current)
|
||||
restoreModelStoreForSettings(current)
|
||||
}
|
||||
http.Error(w, "Datenbank-Verbindung fehlgeschlagen: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
@ -642,8 +650,8 @@ func recordSettingsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func settingsBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
target := r.URL.Query().Get("target")
|
||||
if target != "record" && target != "done" && target != "ffmpeg" && target != "sqlite" {
|
||||
http.Error(w, "target muss record, done, ffmpeg oder sqlite sein", http.StatusBadRequest)
|
||||
if target != "record" && target != "done" && target != "ffmpeg" && target != "postgresData" {
|
||||
http.Error(w, "target muss record, done, ffmpeg oder postgresData sein", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
@ -657,9 +665,9 @@ func settingsBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
p, err = dialog.File().
|
||||
Title("ffmpeg.exe auswählen").
|
||||
Load()
|
||||
} else if target == "sqlite" {
|
||||
} else if target == "postgresData" {
|
||||
p, err = dialog.Directory().
|
||||
Title("SQLite-Speicherort auswaehlen").
|
||||
Title("PostgreSQL-Datenordner auswaehlen").
|
||||
Browse()
|
||||
} else {
|
||||
// Ordnerauswahl für record/done
|
||||
|
||||
@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@ -127,15 +126,10 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
storeConfig, err := buildModelStoreConfigFromSettings()
|
||||
if err != nil {
|
||||
if _, err := buildModelStoreConfigFromSettings(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if storeConfig.Type == databaseTypeSQLite {
|
||||
settingsSQLiteDatabaseBackupHandler(w, r, storeConfig)
|
||||
return
|
||||
}
|
||||
|
||||
cfg, _, err := postgresToolConfigFromSettings()
|
||||
if err != nil {
|
||||
@ -200,7 +194,7 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
f, err := os.Open(tmpPath)
|
||||
if err != nil {
|
||||
http.Error(w, "Backup-Datei konnte nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError)
|
||||
http.Error(w, "Backup-Datei konnte nicht geoeffnet werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
@ -214,66 +208,13 @@ func settingsDatabaseBackupHandler(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(w, r, filename, info.ModTime(), f)
|
||||
}
|
||||
|
||||
func settingsSQLiteDatabaseBackupHandler(w http.ResponseWriter, r *http.Request, storeConfig ModelStoreConfig) {
|
||||
tmp, err := os.CreateTemp("", "nsfwapp-db-backup-*.sqlite")
|
||||
if err != nil {
|
||||
http.Error(w, "Backup-Datei konnte nicht erstellt werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
db, err := sql.Open("sqlite", storeConfig.SQLitePath)
|
||||
if err != nil {
|
||||
http.Error(w, "SQLite konnte nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if _, err := db.ExecContext(ctx, `PRAGMA busy_timeout = 5000;`); err != nil {
|
||||
http.Error(w, "SQLite Backup konnte nicht vorbereitet werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "VACUUM INTO "+sqliteLiteral(tmpPath)); err != nil {
|
||||
http.Error(w, "SQLite Backup fehlgeschlagen: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
info, err := os.Stat(tmpPath)
|
||||
if err != nil {
|
||||
http.Error(w, "Backup-Datei konnte nicht gelesen werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
f, err := os.Open(tmpPath)
|
||||
if err != nil {
|
||||
http.Error(w, "Backup-Datei konnte nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
filename := "nsfwapp-db-backup-" + time.Now().Format("20060102-150405") + ".sqlite"
|
||||
w.Header().Set("Content-Type", "application/vnd.sqlite3")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size()))
|
||||
|
||||
http.ServeContent(w, r, filename, info.ModTime(), f)
|
||||
}
|
||||
|
||||
func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Nur POST erlaubt", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
storeConfig, err := buildModelStoreConfigFromSettings()
|
||||
if err != nil {
|
||||
if _, err := buildModelStoreConfigFromSettings(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
@ -317,27 +258,6 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if storeConfig.Type == databaseTypeSQLite {
|
||||
if err := restoreSQLiteDatabase(storeConfig, tmpPath); err != nil {
|
||||
http.Error(w, "SQLite Restore fehlgeschlagen: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
refreshedStore := NewModelStore(storeConfig)
|
||||
if err := refreshedStore.Load(); err != nil {
|
||||
appLogln("Models DB nach Restore konnte nicht neu verbunden werden:", err)
|
||||
} else {
|
||||
setModelStore(refreshedStore)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"ok": true,
|
||||
"fileName": strings.TrimSpace(header.Filename),
|
||||
"bytes": written,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
cfg, dsn, err := postgresToolConfigFromSettings()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
@ -366,7 +286,7 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
refreshedStore := NewModelStore(ModelStoreConfig{Type: databaseTypePostgres, DSN: dsn})
|
||||
if err := refreshedStore.Load(); err != nil {
|
||||
appLogln("⚠️ Models DB nach Restore konnte nicht neu verbunden werden:", err)
|
||||
appLogln("Models DB nach Restore konnte nicht neu verbunden werden:", err)
|
||||
} else {
|
||||
setModelStore(refreshedStore)
|
||||
}
|
||||
@ -377,88 +297,3 @@ func settingsDatabaseRestoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
"bytes": written,
|
||||
})
|
||||
}
|
||||
|
||||
func restoreSQLiteDatabase(storeConfig ModelStoreConfig, restorePath string) error {
|
||||
restorePath = filepath.Clean(strings.TrimSpace(restorePath))
|
||||
if restorePath == "" {
|
||||
return errors.New("restore path fehlt")
|
||||
}
|
||||
if err := validateSQLiteDatabaseFile(restorePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targetPath := filepath.Clean(strings.TrimSpace(storeConfig.SQLitePath))
|
||||
if targetPath == "" {
|
||||
return errors.New("sqlite path fehlt")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
oldStore := getModelStore()
|
||||
if oldStore != nil && oldStore.isSQLite() && oldStore.db != nil {
|
||||
_, _ = oldStore.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`)
|
||||
_ = oldStore.Close()
|
||||
}
|
||||
|
||||
if _, err := os.Stat(targetPath); err == nil {
|
||||
backupPath := targetPath + ".before-restore-" + time.Now().Format("20060102-150405") + ".bak"
|
||||
if _, err := copyDatabaseFile(targetPath, backupPath); err != nil {
|
||||
return appErrorf("Sicherung der bisherigen SQLite-Datei fehlgeschlagen: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
_ = os.Remove(targetPath + "-wal")
|
||||
_ = os.Remove(targetPath + "-shm")
|
||||
_ = os.Remove(targetPath)
|
||||
|
||||
if _, err := copyDatabaseFile(restorePath, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSQLiteDatabaseFile(path string) error {
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var name string
|
||||
err = db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='models' LIMIT 1;`).Scan(&name)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return errors.New("SQLite-Datei enthaelt keine models-Tabelle")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func copyDatabaseFile(src, dst string) (int64, error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written, copyErr := io.Copy(out, in)
|
||||
closeErr := out.Close()
|
||||
if copyErr != nil {
|
||||
return written, copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return written, closeErr
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func sqliteLiteral(value string) string {
|
||||
return "'" + strings.ReplaceAll(value, "'", "''") + "'"
|
||||
}
|
||||
|
||||
@ -1,242 +1,119 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type databaseMigrationRequest struct {
|
||||
DatabaseType string `json:"databaseType"`
|
||||
SQLitePath string `json:"sqlitePath"`
|
||||
DatabaseURL string `json:"databaseUrl"`
|
||||
DBPassword string `json:"dbPassword,omitempty"`
|
||||
}
|
||||
|
||||
type modelMigrationColumn struct {
|
||||
name string
|
||||
kind string
|
||||
notNull bool
|
||||
}
|
||||
|
||||
const (
|
||||
modelMigrationText = "text"
|
||||
modelMigrationBool = "bool"
|
||||
modelMigrationTime = "time"
|
||||
modelMigrationBlob = "blob"
|
||||
)
|
||||
|
||||
var modelMigrationColumns = []modelMigrationColumn{
|
||||
{name: "id", kind: modelMigrationText, notNull: true},
|
||||
{name: "input", kind: modelMigrationText, notNull: true},
|
||||
{name: "is_url", kind: modelMigrationBool, notNull: true},
|
||||
{name: "host", kind: modelMigrationText},
|
||||
{name: "path", kind: modelMigrationText},
|
||||
{name: "model_key", kind: modelMigrationText, notNull: true},
|
||||
{name: "tags", kind: modelMigrationText},
|
||||
{name: "last_stream", kind: modelMigrationTime},
|
||||
{name: "last_seen_online", kind: modelMigrationBool},
|
||||
{name: "last_seen_online_at", kind: modelMigrationTime},
|
||||
{name: "cb_online_json", kind: modelMigrationText},
|
||||
{name: "cb_online_fetched_at", kind: modelMigrationTime},
|
||||
{name: "cb_online_last_error", kind: modelMigrationText},
|
||||
{name: "profile_image_url", kind: modelMigrationText},
|
||||
{name: "profile_image_mime", kind: modelMigrationText},
|
||||
{name: "profile_image_path", kind: modelMigrationText},
|
||||
{name: "profile_image_hash", kind: modelMigrationText},
|
||||
{name: "profile_image_blob", kind: modelMigrationBlob},
|
||||
{name: "profile_image_updated_at", kind: modelMigrationTime},
|
||||
{name: "room_status", kind: modelMigrationText},
|
||||
{name: "is_online", kind: modelMigrationBool, notNull: true},
|
||||
{name: "chat_room_url", kind: modelMigrationText},
|
||||
{name: "image_url", kind: modelMigrationText},
|
||||
{name: "last_online_at", kind: modelMigrationTime},
|
||||
{name: "last_offline_at", kind: modelMigrationTime},
|
||||
{name: "last_room_sync_at", kind: modelMigrationTime},
|
||||
{name: "biocontext_json", kind: modelMigrationText},
|
||||
{name: "biocontext_fetched_at", kind: modelMigrationTime},
|
||||
{name: "watching", kind: modelMigrationBool, notNull: true},
|
||||
{name: "favorite", kind: modelMigrationBool, notNull: true},
|
||||
{name: "hot", kind: modelMigrationBool, notNull: true},
|
||||
{name: "keep", kind: modelMigrationBool, notNull: true},
|
||||
{name: "liked", kind: modelMigrationBool},
|
||||
{name: "created_at", kind: modelMigrationTime, notNull: true},
|
||||
{name: "updated_at", kind: modelMigrationTime, notNull: true},
|
||||
}
|
||||
|
||||
type dbNullBool struct {
|
||||
sql.NullBool
|
||||
}
|
||||
|
||||
func (b *dbNullBool) Scan(value any) error {
|
||||
if value == nil {
|
||||
b.Valid = false
|
||||
b.Bool = false
|
||||
return nil
|
||||
}
|
||||
|
||||
switch v := value.(type) {
|
||||
case bool:
|
||||
b.Valid = true
|
||||
b.Bool = v
|
||||
return nil
|
||||
case int64:
|
||||
b.Valid = true
|
||||
b.Bool = v != 0
|
||||
return nil
|
||||
case int:
|
||||
b.Valid = true
|
||||
b.Bool = v != 0
|
||||
return nil
|
||||
case []byte:
|
||||
return b.scanString(string(v))
|
||||
case string:
|
||||
return b.scanString(v)
|
||||
default:
|
||||
var raw sql.NullBool
|
||||
if err := raw.Scan(value); err != nil {
|
||||
return err
|
||||
}
|
||||
b.NullBool = raw
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *dbNullBool) scanString(value string) error {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value == "" {
|
||||
b.Valid = false
|
||||
b.Bool = false
|
||||
return nil
|
||||
}
|
||||
switch value {
|
||||
case "1", "t", "true", "yes", "y", "on":
|
||||
b.Valid = true
|
||||
b.Bool = true
|
||||
return nil
|
||||
case "0", "f", "false", "no", "n", "off":
|
||||
b.Valid = true
|
||||
b.Bool = false
|
||||
return nil
|
||||
default:
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid bool value %q", value)
|
||||
}
|
||||
b.Valid = true
|
||||
b.Bool = parsed
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type modelMigrationValue struct {
|
||||
column modelMigrationColumn
|
||||
text sql.NullString
|
||||
bool dbNullBool
|
||||
tm dbNullTime
|
||||
blob []byte
|
||||
}
|
||||
|
||||
func (v *modelMigrationValue) dest() any {
|
||||
switch v.column.kind {
|
||||
case modelMigrationBool:
|
||||
return &v.bool
|
||||
case modelMigrationTime:
|
||||
return &v.tm
|
||||
case modelMigrationBlob:
|
||||
return &v.blob
|
||||
default:
|
||||
return &v.text
|
||||
}
|
||||
}
|
||||
|
||||
func (v modelMigrationValue) arg(now time.Time) any {
|
||||
switch v.column.kind {
|
||||
case modelMigrationBool:
|
||||
if !v.bool.Valid {
|
||||
if v.column.notNull {
|
||||
return false
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return v.bool.Bool
|
||||
case modelMigrationTime:
|
||||
if !v.tm.Valid || v.tm.Time.IsZero() {
|
||||
if v.column.notNull {
|
||||
return now
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return v.tm.Time.UTC()
|
||||
case modelMigrationBlob:
|
||||
if len(v.blob) == 0 {
|
||||
return nil
|
||||
}
|
||||
return v.blob
|
||||
default:
|
||||
if !v.text.Valid {
|
||||
if v.column.notNull {
|
||||
return ""
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return v.text.String
|
||||
}
|
||||
}
|
||||
|
||||
func (v modelMigrationValue) textValue() string {
|
||||
if !v.text.Valid {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(v.text.String)
|
||||
}
|
||||
|
||||
func settingsDatabaseMigrateHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Nur POST erlaubt", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
var req databaseMigrationRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
var in RecorderSettingsIn
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
http.Error(w, "invalid json: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
current := getSettings()
|
||||
target, err := databaseMigrationTargetSettings(current, req)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
target := current
|
||||
target.DatabaseType = databaseTypePostgres
|
||||
target.PostgresMode = in.PostgresMode
|
||||
target.PostgresDataDir = in.PostgresDataDir
|
||||
target.PostgresPort = in.PostgresPort
|
||||
target.PostgresDatabase = in.PostgresDatabase
|
||||
target.PostgresUser = in.PostgresUser
|
||||
target.DatabaseURL = strings.TrimSpace(in.DatabaseURL)
|
||||
normalizePostgresSettings(&target)
|
||||
|
||||
if target.PostgresMode == postgresModeURL && target.DatabaseURL == "" {
|
||||
http.Error(w, "Database URL fehlt", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
sourceConfig, err := buildModelStoreConfigFromRecorderSettings(current)
|
||||
sanitizedURL, pwFromURL := stripPasswordFromPostgresURL(target.DatabaseURL)
|
||||
pwFromURL = strings.TrimSpace(pwFromURL)
|
||||
if pwFromURL == "****" {
|
||||
pwFromURL = ""
|
||||
}
|
||||
if sanitizedURL != "" {
|
||||
target.DatabaseURL = sanitizedURL
|
||||
}
|
||||
|
||||
plainPW := strings.TrimSpace(in.DBPassword)
|
||||
if plainPW == "" {
|
||||
plainPW = pwFromURL
|
||||
}
|
||||
if plainPW != "" {
|
||||
enc, err := encryptSettingString(plainPW)
|
||||
if err != nil {
|
||||
http.Error(w, "konnte DB-Passwort nicht verschluesseln: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
target.EncryptedDBPassword = enc
|
||||
}
|
||||
|
||||
sourceDSN, err := buildPostgresDSNFromRecorderSettings(current)
|
||||
if err != nil {
|
||||
http.Error(w, "Quell-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
targetConfig, err := buildModelStoreConfigFromRecorderSettings(target)
|
||||
targetDSN, err := buildPostgresDSNFromRecorderSettings(target)
|
||||
if err != nil {
|
||||
http.Error(w, "Ziel-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(sourceDSN) == strings.TrimSpace(targetDSN) ||
|
||||
sanitizeDSNForLog(sourceDSN) == sanitizeDSNForLog(targetDSN) {
|
||||
http.Error(w, "Quelle und Ziel sind identisch", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
sourceDBURL, sourceEnv, err := postgresToolConnection(sourceDSN)
|
||||
if err != nil {
|
||||
http.Error(w, "Quell-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
targetDBURL, targetEnv, err := postgresToolConnection(targetDSN)
|
||||
if err != nil {
|
||||
http.Error(w, "Ziel-Datenbank ungueltig: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
count, err := migrateModelStoreData(sourceConfig, targetConfig)
|
||||
tmp, err := os.CreateTemp("", "nsfwapp-db-migrate-*.dump")
|
||||
if err != nil {
|
||||
http.Error(w, "Migration fehlgeschlagen: "+err.Error(), http.StatusInternalServerError)
|
||||
http.Error(w, "Migration konnte nicht vorbereitet werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
_ = tmp.Close()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
if err := dumpPostgresDatabase(ctx, sourceDBURL, sourceEnv, tmpPath); err != nil {
|
||||
http.Error(w, "pg_dump fehlgeschlagen: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := restorePostgresDatabase(ctx, targetDBURL, targetEnv, tmpPath); err != nil {
|
||||
http.Error(w, "pg_restore fehlgeschlagen: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
refreshedStore := NewModelStore(targetConfig)
|
||||
if err := refreshedStore.Load(); err != nil {
|
||||
http.Error(w, "Ziel-Datenbank konnte nach Migration nicht geöffnet werden: "+err.Error(), http.StatusInternalServerError)
|
||||
newStore := NewModelStore(ModelStoreConfig{Type: databaseTypePostgres, DSN: targetDSN})
|
||||
if err := newStore.Load(); err != nil {
|
||||
_ = newStore.Close()
|
||||
http.Error(w, "Ziel-Datenbank konnte nach Migration nicht geladen werden: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@ -244,186 +121,77 @@ func settingsDatabaseMigrateHandler(w http.ResponseWriter, r *http.Request) {
|
||||
settings = target
|
||||
settingsMu.Unlock()
|
||||
saveSettingsToDisk()
|
||||
setModelStore(refreshedStore)
|
||||
|
||||
setModelStore(newStore)
|
||||
setCoverModelStore(newStore)
|
||||
setChaturbateOnlineModelStore(newStore)
|
||||
|
||||
info, _ := os.Stat(tmpPath)
|
||||
var bytesWritten int64
|
||||
if info != nil {
|
||||
bytesWritten = info.Size()
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"ok": true,
|
||||
"rows": count,
|
||||
"sourceType": sourceConfig.Type,
|
||||
"targetType": targetConfig.Type,
|
||||
"targetSQLite": targetConfig.SQLitePath,
|
||||
"targetDatabase": sanitizeDSNForLog(targetConfig.DSN),
|
||||
"ok": true,
|
||||
"bytes": bytesWritten,
|
||||
"mode": target.PostgresMode,
|
||||
})
|
||||
}
|
||||
|
||||
func databaseMigrationTargetSettings(current RecorderSettings, req databaseMigrationRequest) (RecorderSettings, error) {
|
||||
next := current
|
||||
next.DatabaseType = normalizeDatabaseType(req.DatabaseType)
|
||||
next.SQLitePath = normalizeSQLitePath(req.SQLitePath)
|
||||
next.DatabaseURL = strings.TrimSpace(req.DatabaseURL)
|
||||
|
||||
sanitizedURL, pwFromURL := stripPasswordFromPostgresURL(next.DatabaseURL)
|
||||
pwFromURL = strings.TrimSpace(pwFromURL)
|
||||
if pwFromURL == "****" {
|
||||
pwFromURL = ""
|
||||
}
|
||||
if sanitizedURL != "" {
|
||||
next.DatabaseURL = sanitizedURL
|
||||
}
|
||||
|
||||
plainPW := strings.TrimSpace(req.DBPassword)
|
||||
if plainPW == "" {
|
||||
plainPW = pwFromURL
|
||||
}
|
||||
if plainPW != "" {
|
||||
enc, err := encryptSettingString(plainPW)
|
||||
if err != nil {
|
||||
return RecorderSettings{}, appErrorf("konnte DB-Passwort nicht verschluesseln: %w", err)
|
||||
}
|
||||
next.EncryptedDBPassword = enc
|
||||
}
|
||||
|
||||
if next.DatabaseType == databaseTypePostgres && strings.TrimSpace(next.DatabaseURL) == "" {
|
||||
return RecorderSettings{}, errors.New("Database URL fehlt")
|
||||
}
|
||||
if next.DatabaseType == databaseTypeSQLite && strings.TrimSpace(next.SQLitePath) == "" {
|
||||
return RecorderSettings{}, errors.New("SQLite-Datei fehlt")
|
||||
}
|
||||
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func migrateModelStoreData(sourceConfig, targetConfig ModelStoreConfig) (int64, error) {
|
||||
if sameModelStoreConfig(sourceConfig, targetConfig) {
|
||||
return 0, errors.New("Quelle und Ziel sind identisch")
|
||||
}
|
||||
|
||||
source := NewModelStore(sourceConfig)
|
||||
if err := source.Load(); err != nil {
|
||||
return 0, appErrorf("Quelle konnte nicht geöffnet werden: %w", err)
|
||||
}
|
||||
defer source.Close()
|
||||
|
||||
target := NewModelStore(targetConfig)
|
||||
if err := target.Load(); err != nil {
|
||||
return 0, appErrorf("Ziel konnte nicht geöffnet werden: %w", err)
|
||||
}
|
||||
defer target.Close()
|
||||
|
||||
count, err := migrateModelsTable(source.db, target.db)
|
||||
func dumpPostgresDatabase(ctx context.Context, dbURL string, env []string, outPath string) error {
|
||||
toolPath, err := postgresToolPath("pg_dump")
|
||||
if err != nil {
|
||||
return count, err
|
||||
}
|
||||
if normalizeDatabaseType(sourceConfig.Type) == databaseTypeSQLite &&
|
||||
normalizeDatabaseType(targetConfig.Type) == databaseTypeSQLite &&
|
||||
!sameCleanPath(filepath.Dir(sourceConfig.SQLitePath), filepath.Dir(targetConfig.SQLitePath)) {
|
||||
sourceImageDir := sqliteProfileImageDir(sourceConfig.SQLitePath)
|
||||
targetImageDir := sqliteProfileImageDir(targetConfig.SQLitePath)
|
||||
if _, statErr := os.Stat(sourceImageDir); statErr == nil {
|
||||
if _, targetErr := os.Stat(targetImageDir); errors.Is(targetErr, os.ErrNotExist) {
|
||||
if copyErr := copyDirectory(sourceImageDir, targetImageDir); copyErr != nil {
|
||||
return count, copyErr
|
||||
}
|
||||
} else if targetErr != nil {
|
||||
return count, targetErr
|
||||
}
|
||||
} else if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {
|
||||
return count, statErr
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func sameModelStoreConfig(a, b ModelStoreConfig) bool {
|
||||
a.Type = normalizeDatabaseType(a.Type)
|
||||
b.Type = normalizeDatabaseType(b.Type)
|
||||
if a.Type != b.Type {
|
||||
return false
|
||||
}
|
||||
if a.Type == databaseTypeSQLite {
|
||||
return filepath.Clean(strings.TrimSpace(a.SQLitePath)) == filepath.Clean(strings.TrimSpace(b.SQLitePath))
|
||||
}
|
||||
return strings.TrimSpace(a.DSN) == strings.TrimSpace(b.DSN)
|
||||
}
|
||||
|
||||
func migrateModelsTable(sourceDB, targetDB *sql.DB) (int64, error) {
|
||||
columnNames := make([]string, 0, len(modelMigrationColumns))
|
||||
for _, column := range modelMigrationColumns {
|
||||
columnNames = append(columnNames, column.name)
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := sourceDB.Query("SELECT " + strings.Join(columnNames, ",") + " FROM models ORDER BY updated_at ASC, id ASC;")
|
||||
out, err := os.Create(outPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
tx, err := targetDB.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
if _, err := tx.Exec(`DELETE FROM models;`); err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
|
||||
insertSQL := modelMigrationInsertSQL(columnNames)
|
||||
stmt, err := tx.Prepare(insertSQL)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
var count int64
|
||||
now := time.Now().UTC()
|
||||
for rows.Next() {
|
||||
values := make([]modelMigrationValue, len(modelMigrationColumns))
|
||||
dests := make([]any, len(modelMigrationColumns))
|
||||
for i, column := range modelMigrationColumns {
|
||||
values[i] = modelMigrationValue{column: column}
|
||||
dests[i] = values[i].dest()
|
||||
}
|
||||
if err := rows.Scan(dests...); err != nil {
|
||||
return count, err
|
||||
}
|
||||
if values[0].textValue() == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
args := make([]any, len(values))
|
||||
for i, value := range values {
|
||||
args[i] = value.arg(now)
|
||||
}
|
||||
if _, err := stmt.Exec(args...); err != nil {
|
||||
return count, err
|
||||
}
|
||||
count++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return count, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return count, err
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func modelMigrationInsertSQL(columnNames []string) string {
|
||||
placeholders := make([]string, 0, len(columnNames))
|
||||
updates := make([]string, 0, len(columnNames)-1)
|
||||
for i, column := range columnNames {
|
||||
placeholders = append(placeholders, fmt.Sprintf("$%d", i+1))
|
||||
if column != "id" {
|
||||
updates = append(updates, column+"=EXCLUDED."+column)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"INSERT INTO models (%s) VALUES (%s) ON CONFLICT(id) DO UPDATE SET %s;",
|
||||
strings.Join(columnNames, ","),
|
||||
strings.Join(placeholders, ","),
|
||||
strings.Join(updates, ","),
|
||||
cmd := exec.CommandContext(
|
||||
ctx,
|
||||
toolPath,
|
||||
"--format=custom",
|
||||
"--blobs",
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
"--dbname="+dbURL,
|
||||
)
|
||||
cmd.Env = env
|
||||
cmd.Stdout = out
|
||||
hideCommandWindow(cmd)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
runErr := cmd.Run()
|
||||
closeErr := out.Close()
|
||||
if runErr != nil {
|
||||
msg := trimCommandOutput(stderr.String())
|
||||
if msg == "" {
|
||||
msg = runErr.Error()
|
||||
}
|
||||
return appErrorf("%s", msg)
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
|
||||
func restorePostgresDatabase(ctx context.Context, dbURL string, env []string, dumpPath string) error {
|
||||
stderr, err := runPostgresTool(
|
||||
ctx,
|
||||
"pg_restore",
|
||||
postgresToolConfig{dbURL: dbURL, env: env},
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
"--single-transaction",
|
||||
"--dbname="+dbURL,
|
||||
filepath.Clean(dumpPath),
|
||||
)
|
||||
if err != nil {
|
||||
return appErrorf("%s", stderr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1,196 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func sqlitePathChanged(current, next RecorderSettings) bool {
|
||||
return normalizeDatabaseType(current.DatabaseType) == databaseTypeSQLite &&
|
||||
normalizeDatabaseType(next.DatabaseType) == databaseTypeSQLite &&
|
||||
normalizeSQLitePath(current.SQLitePath) != normalizeSQLitePath(next.SQLitePath)
|
||||
}
|
||||
|
||||
func moveSQLiteDatabaseForSettings(current, next RecorderSettings) error {
|
||||
currentConfig, err := buildModelStoreConfigFromRecorderSettings(current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nextConfig, err := buildModelStoreConfigFromRecorderSettings(next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return moveSQLiteDatabaseFiles(currentConfig.SQLitePath, nextConfig.SQLitePath)
|
||||
}
|
||||
|
||||
func restoreModelStoreForSettings(rec RecorderSettings) {
|
||||
config, err := buildModelStoreConfigFromRecorderSettings(rec)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
store := NewModelStore(config)
|
||||
if err := store.Load(); err != nil {
|
||||
_ = store.Close()
|
||||
return
|
||||
}
|
||||
setModelStore(store)
|
||||
}
|
||||
|
||||
func moveSQLiteDatabaseFiles(sourcePath, targetPath string) error {
|
||||
sourcePath = filepath.Clean(strings.TrimSpace(sourcePath))
|
||||
targetPath = filepath.Clean(strings.TrimSpace(targetPath))
|
||||
if sourcePath == "" || targetPath == "" {
|
||||
return errors.New("sqlite source/target path fehlt")
|
||||
}
|
||||
if sourcePath == targetPath {
|
||||
return nil
|
||||
}
|
||||
|
||||
sourceInfo, err := os.Stat(sourcePath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sourceInfo.IsDir() {
|
||||
return appErrorf("sqlite source path ist ein ordner: %s", sourcePath)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, target := range append([]string{targetPath}, sqliteSidecarPaths(targetPath)...) {
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
return appErrorf("Ziel-Datei existiert bereits: %s", target)
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
sourceImageDir := sqliteProfileImageDir(sourcePath)
|
||||
targetImageDir := sqliteProfileImageDir(targetPath)
|
||||
if !sameCleanPath(sourceImageDir, targetImageDir) {
|
||||
if _, err := os.Stat(sourceImageDir); err == nil {
|
||||
if _, err := os.Stat(targetImageDir); err == nil {
|
||||
return appErrorf("Ziel-Profilbildordner existiert bereits: %s", targetImageDir)
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if store := getModelStore(); store != nil && store.isSQLite() && store.db != nil {
|
||||
_, _ = store.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`)
|
||||
_ = store.Close()
|
||||
}
|
||||
|
||||
if err := moveDatabaseFile(sourcePath, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sourceSidecars := sqliteSidecarPaths(sourcePath)
|
||||
targetSidecars := sqliteSidecarPaths(targetPath)
|
||||
for i, source := range sourceSidecars {
|
||||
if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := moveDatabaseFile(source, targetSidecars[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if !sameCleanPath(sourceImageDir, targetImageDir) {
|
||||
if err := moveSQLiteProfileImageDir(sourceImageDir, targetImageDir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sqliteProfileImageDir(sqlitePath string) string {
|
||||
return filepath.Join(filepath.Dir(filepath.Clean(strings.TrimSpace(sqlitePath))), modelProfileImageDirName)
|
||||
}
|
||||
|
||||
func sameCleanPath(a, b string) bool {
|
||||
return strings.EqualFold(filepath.Clean(strings.TrimSpace(a)), filepath.Clean(strings.TrimSpace(b)))
|
||||
}
|
||||
|
||||
func moveSQLiteProfileImageDir(sourceDir, targetDir string) error {
|
||||
sourceDir = filepath.Clean(strings.TrimSpace(sourceDir))
|
||||
targetDir = filepath.Clean(strings.TrimSpace(targetDir))
|
||||
if sourceDir == "" || targetDir == "" || sameCleanPath(sourceDir, targetDir) {
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := os.Stat(sourceDir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return appErrorf("Profilbildpfad ist kein Ordner: %s", sourceDir)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(targetDir), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := renameWithRetry(sourceDir, targetDir); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err := copyDirectory(sourceDir, targetDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(sourceDir)
|
||||
}
|
||||
|
||||
func copyDirectory(sourceDir, targetDir string) error {
|
||||
sourceDir = filepath.Clean(strings.TrimSpace(sourceDir))
|
||||
targetDir = filepath.Clean(strings.TrimSpace(targetDir))
|
||||
return filepath.WalkDir(sourceDir, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
rel, err := filepath.Rel(sourceDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := filepath.Join(targetDir, rel)
|
||||
if entry.IsDir() {
|
||||
return os.MkdirAll(target, 0755)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = copyDatabaseFile(path, target)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func sqliteSidecarPaths(path string) []string {
|
||||
return []string{
|
||||
path + "-wal",
|
||||
path + "-shm",
|
||||
path + "-journal",
|
||||
}
|
||||
}
|
||||
|
||||
func moveDatabaseFile(sourcePath, targetPath string) error {
|
||||
if err := renameWithRetry(sourcePath, targetPath); err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := copyDatabaseFile(sourcePath, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return removeWithRetry(sourcePath)
|
||||
}
|
||||
@ -2,6 +2,7 @@
|
||||
import { useEffect, useMemo, useRef, useState, type KeyboardEvent, type ClipboardEvent } from 'react'
|
||||
import { startRegistration, startAuthentication } from '@simplewebauthn/browser'
|
||||
import Button from './Button'
|
||||
import ProgressBar from './ProgressBar'
|
||||
import LoginShell from './login/LoginShell'
|
||||
import ErrorAlert from './login/ErrorAlert'
|
||||
import LoginForm from './login/LoginForm'
|
||||
@ -40,6 +41,18 @@ type MeResp = {
|
||||
passkeyCount?: number
|
||||
}
|
||||
|
||||
type MLSetupStatus = {
|
||||
required?: boolean
|
||||
running?: boolean
|
||||
done?: boolean
|
||||
ok?: boolean
|
||||
progress?: number
|
||||
phase?: string
|
||||
message?: string
|
||||
error?: string
|
||||
logFile?: string
|
||||
}
|
||||
|
||||
type SetupResp = {
|
||||
secret?: string
|
||||
otpauth?: string
|
||||
@ -55,6 +68,65 @@ function unwrapWebAuthnOptions<T = any>(options: any): T {
|
||||
return unwrapped as T
|
||||
}
|
||||
|
||||
function MLSetupModal({ status }: { status: MLSetupStatus | null }) {
|
||||
if (!status?.required) return null
|
||||
|
||||
const progress = Math.max(0, Math.min(100, Number(status.progress) || 0))
|
||||
const failed = Boolean(status.done && !status.ok)
|
||||
const done = Boolean(status.done && status.ok)
|
||||
if (done) return null
|
||||
|
||||
const title = failed
|
||||
? 'ML-Python-Setup fehlgeschlagen'
|
||||
: 'ML-Python-Pakete werden installiert'
|
||||
const message = String(status.message || '').trim() || 'Die ML-Python-Umgebung wird vorbereitet.'
|
||||
const detail = failed
|
||||
? String(status.error || '').trim()
|
||||
: status.logFile
|
||||
? `Log: ${status.logFile}`
|
||||
: ''
|
||||
|
||||
return (
|
||||
<div
|
||||
className="absolute inset-0 z-20 grid place-items-center bg-white/80 p-4 backdrop-blur-sm dark:bg-gray-950/80"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
>
|
||||
<div
|
||||
className={[
|
||||
'w-full rounded-xl border p-4 shadow-lg',
|
||||
failed
|
||||
? 'border-red-200 bg-red-50 text-red-950 dark:border-red-500/30 dark:bg-red-500/10 dark:text-red-100'
|
||||
: 'border-indigo-200 bg-white text-gray-900 dark:border-indigo-400/30 dark:bg-gray-900 dark:text-gray-100',
|
||||
].join(' ')}
|
||||
>
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-sm font-semibold">{title}</div>
|
||||
<div className="mt-1 break-words text-xs opacity-80">{message}</div>
|
||||
</div>
|
||||
<div className="shrink-0 text-xs font-semibold tabular-nums">
|
||||
{Math.round(progress)}%
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ProgressBar
|
||||
className="mt-3"
|
||||
value={progress}
|
||||
showPercent={false}
|
||||
size="sm"
|
||||
/>
|
||||
|
||||
{detail ? (
|
||||
<div className="mt-2 break-words text-xs opacity-75">
|
||||
{detail}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function LoginPage({ onLoggedIn }: Props) {
|
||||
const nextPath = useMemo(() => getNextFromLocation(), [])
|
||||
|
||||
@ -77,6 +149,7 @@ export default function LoginPage({ onLoggedIn }: Props) {
|
||||
|
||||
const [passkeySupported, setPasskeySupported] = useState(false)
|
||||
const [passkeyFeedback, setPasskeyFeedback] = useState<PasskeyFeedback | null>(null)
|
||||
const [mlSetupStatus, setMLSetupStatus] = useState<MLSetupStatus | null>(null)
|
||||
|
||||
const submittedOnceRef = useRef(false)
|
||||
|
||||
@ -107,6 +180,39 @@ export default function LoginPage({ onLoggedIn }: Props) {
|
||||
)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
let timer: number | undefined
|
||||
|
||||
async function poll() {
|
||||
try {
|
||||
const res = await fetch('/api/setup/ml/status', {
|
||||
cache: 'no-store',
|
||||
credentials: 'include',
|
||||
})
|
||||
if (!res.ok) return
|
||||
const status = (await res.json()) as MLSetupStatus
|
||||
if (cancelled) return
|
||||
setMLSetupStatus(status)
|
||||
|
||||
if (status?.required && (!status.done || status.running)) {
|
||||
timer = window.setTimeout(poll, 1000)
|
||||
}
|
||||
} catch {
|
||||
if (!cancelled) {
|
||||
timer = window.setTimeout(poll, 2500)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void poll()
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
if (timer) window.clearTimeout(timer)
|
||||
}
|
||||
}, [])
|
||||
|
||||
function passkeyErrorMessage(e: any) {
|
||||
const raw = String(e?.message ?? e ?? '').trim()
|
||||
|
||||
@ -575,7 +681,7 @@ export default function LoginPage({ onLoggedIn }: Props) {
|
||||
)
|
||||
|
||||
return (
|
||||
<LoginShell>
|
||||
<LoginShell modal={<MLSetupModal status={mlSetupStatus} />}>
|
||||
{stage === 'login' ? (
|
||||
<LoginForm
|
||||
username={username}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -2,7 +2,7 @@
|
||||
|
||||
import type { ReactNode } from 'react'
|
||||
|
||||
export default function LoginShell(props: { children: ReactNode }) {
|
||||
export default function LoginShell(props: { modal?: ReactNode; children: ReactNode }) {
|
||||
return (
|
||||
<div className="min-h-[100dvh] bg-gray-50 text-gray-900 dark:bg-gray-950 dark:text-gray-100">
|
||||
<div aria-hidden="true" className="pointer-events-none fixed inset-0 overflow-hidden">
|
||||
@ -11,16 +11,20 @@ export default function LoginShell(props: { children: ReactNode }) {
|
||||
</div>
|
||||
|
||||
<div className="relative grid min-h-[100dvh] place-items-center px-4">
|
||||
<div className="w-full max-w-md rounded-2xl border border-gray-200/70 bg-white/80 p-6 shadow-sm backdrop-blur dark:border-white/10 dark:bg-white/5">
|
||||
<div className="space-y-1">
|
||||
<h1 className="text-lg font-semibold tracking-tight">Recorder Login</h1>
|
||||
<p className="text-sm text-gray-600 dark:text-gray-300">
|
||||
Bitte melde dich an, um fortzufahren.
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-full max-w-md">
|
||||
<div className="relative overflow-hidden rounded-2xl border border-gray-200/70 bg-white/80 p-6 shadow-sm backdrop-blur dark:border-white/10 dark:bg-white/5">
|
||||
<div className="space-y-1">
|
||||
<h1 className="text-lg font-semibold tracking-tight">Recorder Login</h1>
|
||||
<p className="text-sm text-gray-600 dark:text-gray-300">
|
||||
Bitte melde dich an, um fortzufahren.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="mt-5 space-y-3">
|
||||
{props.children}
|
||||
<div className="mt-5 space-y-3">
|
||||
{props.children}
|
||||
</div>
|
||||
|
||||
{props.modal}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user