chris/nsfwapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfixes

Browse Source
This commit is contained in:
Linrador 2026-05-02 13:02:57 +02:00
parent c905753f7a
commit 6e91c352a7
2 changed files with 465 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/routes.go
View File

@ -78,6 +78,7 @@ func registerRoutes(mux *http.ServeMux, auth *AuthManager) *ModelStore {
api.HandleFunc("/api/training/feedback", trainingFeedbackHandler)
api.HandleFunc("/api/training/train", trainingTrainHandler)
api.HandleFunc("/api/training/status", trainingStatusHandler)
api.HandleFunc("/api/training/stats", trainingStatsHandler)
api.HandleFunc("/api/training/delete-all", trainingDeleteAllHandler)
api.HandleFunc("/api/chaturbate/online", chaturbateOnlineHandler)

465
backend/training.go
View File

@ -128,6 +128,38 @@ type TrainingJobStatus struct {
FinishedAt string `json:"finishedAt,omitempty"`
}
type TrainingConfidence struct {
Score float64 `json:"score"`
Level string `json:"level"`
Label string `json:"label"`
}
type TrainingLabelStat struct {
Label string `json:"label"`
Count int `json:"count"`
Confidence TrainingConfidence `json:"confidence"`
}
type TrainingStatsLabels struct {
People []TrainingLabelStat `json:"people"`
SexPositions []TrainingLabelStat `json:"sexPositions"`
BodyParts []TrainingLabelStat `json:"bodyParts"`
Objects []TrainingLabelStat `json:"objects"`
Clothing []TrainingLabelStat `json:"clothing"`
}
type TrainingStatsResponse struct {
OK bool `json:"ok"`
FeedbackCount int `json:"feedbackCount"`
AcceptedCount int `json:"acceptedCount"`
CorrectedCount int `json:"correctedCount"`
SampleCount int `json:"sampleCount"`
BoxCount int `json:"boxCount"`
ModelAvailable bool `json:"modelAvailable"`
Confidence TrainingConfidence `json:"confidence"`
Labels TrainingStatsLabels `json:"labels"`
}
const minTrainingFeedbackCount = 5
const minDetectorTrainCount = 20
@ -874,6 +906,438 @@ func trainingStatusHandler(w http.ResponseWriter, r *http.Request) {
})
}
func trainingStatsHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
trainingWriteError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
root, err := trainingRootDir()
if err != nil {
trainingWriteError(w, http.StatusInternalServerError, err.Error())
return
}
stats, err := trainingBuildStats(root)
if err != nil {
trainingWriteError(w, http.StatusInternalServerError, err.Error())
return
}
trainingWriteJSON(w, http.StatusOK, stats)
}
func trainingBuildStats(root string) (*TrainingStatsResponse, error) {
grouped, err := trainingGroupedLabels()
if err != nil {
// Fallback: Stats sollen trotzdem funktionieren, auch wenn Label-Gruppierung scheitert.
fallbackLabels := defaultTrainingLabelsFromJSON()
grouped = TrainingGroupedLabels{
People: fallbackLabels.People,
SexPositions: fallbackLabels.SexPositions,
BodyParts: fallbackLabels.BodyParts,
Objects: fallbackLabels.Objects,
Clothing: fallbackLabels.Clothing,
}
}
peopleSet := stringSet(grouped.People)
sexPositionSet := stringSet(grouped.SexPositions)
bodyPartSet := stringSet(grouped.BodyParts)
objectSet := stringSet(grouped.Objects)
clothingSet := stringSet(grouped.Clothing)
peopleCounts := map[string]int{}
sexPositionCounts := map[string]int{}
bodyPartCounts := map[string]int{}
objectCounts := map[string]int{}
clothingCounts := map[string]int{}
stats := &TrainingStatsResponse{
OK: true,
Labels: TrainingStatsLabels{
People: []TrainingLabelStat{},
SexPositions: []TrainingLabelStat{},
BodyParts: []TrainingLabelStat{},
Objects: []TrainingLabelStat{},
Clothing: []TrainingLabelStat{},
},
}
feedbackPath := filepath.Join(root, "feedback.jsonl")
b, err := os.ReadFile(feedbackPath)
if err != nil {
if os.IsNotExist(err) {
stats.SampleCount = trainingCountSampleFiles(filepath.Join(root, "samples"))
stats.ModelAvailable = trainingStatsModelAvailable(root)
return stats, nil
}
return nil, err
}
for _, line := range strings.Split(string(b), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
var annotation TrainingAnnotation
if err := json.Unmarshal([]byte(line), &annotation); err != nil {
continue
}
stats.FeedbackCount++
if annotation.Accepted {
stats.AcceptedCount++
} else {
stats.CorrectedCount++
}
effective := trainingEffectiveCorrection(annotation)
sexPosition := strings.TrimSpace(effective.SexPosition)
if sexPosition == "" {
sexPosition = "unknown"
}
if len(sexPositionSet) == 0 || sexPositionSet[sexPosition] {
sexPositionCounts[sexPosition]++
}
for _, label := range effective.BodyPartsPresent {
clean := strings.TrimSpace(label)
if clean == "" {
continue
}
if len(bodyPartSet) == 0 || bodyPartSet[clean] {
bodyPartCounts[clean]++
}
}
for _, label := range effective.ObjectsPresent {
clean := strings.TrimSpace(label)
if clean == "" {
continue
}
if len(objectSet) == 0 || objectSet[clean] {
objectCounts[clean]++
}
}
for _, label := range effective.ClothingPresent {
clean := strings.TrimSpace(label)
if clean == "" {
continue
}
if len(clothingSet) == 0 || clothingSet[clean] {
clothingCounts[clean]++
}
}
for _, box := range effective.Boxes {
label := strings.TrimSpace(box.Label)
if label == "" {
continue
}
stats.BoxCount++
switch {
case peopleSet[label]:
peopleCounts[label]++
case bodyPartSet[label]:
bodyPartCounts[label]++
case objectSet[label]:
objectCounts[label]++
case clothingSet[label]:
clothingCounts[label]++
}
}
}
stats.SampleCount = trainingCountSampleFiles(filepath.Join(root, "samples"))
stats.ModelAvailable = trainingStatsModelAvailable(root)
stats.Labels = TrainingStatsLabels{
// Personen/Box-Labels brauchen mehr Beispiele, weil der Detector Boxen lernen muss.
People: trainingStatsMapToList(peopleCounts, 20),
// Scene-Positionen sind Sample-Labels, hier reichen grob weniger pro Klasse.
SexPositions: trainingStatsMapToList(sexPositionCounts, 8),
// Detector-Klassen: grob 15 Beispiele pro Label als solide Untergrenze.
BodyParts: trainingStatsMapToList(bodyPartCounts, 15),
Objects: trainingStatsMapToList(objectCounts, 15),
Clothing: trainingStatsMapToList(clothingCounts, 15),
}
stats.Confidence = trainingOverallConfidence(
stats.FeedbackCount,
stats.BoxCount,
stats.AcceptedCount,
stats.CorrectedCount,
stats.Labels,
)
return stats, nil
}
func trainingEffectiveCorrection(annotation TrainingAnnotation) TrainingCorrection {
if annotation.Correction != nil {
return *annotation.Correction
}
p := annotation.Prediction
return TrainingCorrection{
PeopleCount: p.PeopleCount,
MaleCount: p.MaleCount,
FemaleCount: p.FemaleCount,
UnknownCount: p.UnknownCount,
SexPosition: p.SexPosition,
BodyPartsPresent: trainingScoredLabelsToStrings(p.BodyPartsPresent),
ObjectsPresent: trainingScoredLabelsToStrings(p.ObjectsPresent),
ClothingPresent: trainingScoredLabelsToStrings(p.ClothingPresent),
Boxes: p.Boxes,
}
}
func trainingScoredLabelsToStrings(values []TrainingScoredLabel) []string {
out := make([]string, 0, len(values))
seen := map[string]bool{}
for _, value := range values {
label := strings.TrimSpace(value.Label)
if label == "" || seen[label] {
continue
}
seen[label] = true
out = append(out, label)
}
return out
}
func trainingStatsMapToList(values map[string]int, target int) []TrainingLabelStat {
out := make([]TrainingLabelStat, 0, len(values))
for label, count := range values {
label = strings.TrimSpace(label)
if label == "" || count <= 0 {
continue
}
out = append(out, TrainingLabelStat{
Label: label,
Count: count,
Confidence: trainingLabelConfidence(count, target),
})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Count == out[j].Count {
return out[i].Label < out[j].Label
}
return out[i].Count > out[j].Count
})
return out
}
func trainingCountSampleFiles(samplesDir string) int {
entries, err := os.ReadDir(samplesDir)
if err != nil {
return 0
}
count := 0
for _, entry := range entries {
if entry.IsDir() {
continue
}
if strings.ToLower(filepath.Ext(entry.Name())) == ".json" {
count++
}
}
return count
}
func trainingStatsModelAvailable(root string) bool {
detectorModelPath := filepath.Join(root, "detector", "model", "best.pt")
sceneKNNPath := filepath.Join(root, "model", "scene_clip_knn.joblib")
sceneLRPath := filepath.Join(root, "model", "scene_clip_lr.joblib")
sceneEmbeddingsPath := filepath.Join(root, "model", "scene_clip_embeddings.npz")
sceneTargetsPath := filepath.Join(root, "model", "scene_clip_targets.json")
detectorReady := fileExistsNonEmpty(detectorModelPath)
sceneReady :=
fileExistsNonEmpty(sceneEmbeddingsPath) &&
fileExistsNonEmpty(sceneTargetsPath) &&
(fileExistsNonEmpty(sceneKNNPath) || fileExistsNonEmpty(sceneLRPath))
return detectorReady || sceneReady
}
func trainingConfidenceFromScore(score float64) TrainingConfidence {
if math.IsNaN(score) || math.IsInf(score, 0) {
score = 0
}
score = clamp01(score)
level := "none"
label := "Keine"
switch {
case score >= 0.75:
level = "high"
label = "Hoch"
case score >= 0.45:
level = "mid"
label = "Mittel"
case score > 0:
level = "low"
label = "Niedrig"
}
return TrainingConfidence{
Score: score,
Level: level,
Label: label,
}
}
func trainingLabelConfidence(count int, target int) TrainingConfidence {
if target <= 0 {
target = 10
}
if count <= 0 {
return trainingConfidenceFromScore(0)
}
// Grobe Datenabdeckung: target erreicht = 100%.
// sqrt macht kleine Mengen etwas weniger hart, aber 1 Treffer bleibt niedrig.
score := math.Sqrt(float64(count) / float64(target*2))
return trainingConfidenceFromScore(score)
}
func trainingSaturationScore(value int, target int) float64 {
if value <= 0 || target <= 0 {
return 0
}
// Sanfter Anstieg, aber nie über 1.
return clamp01(math.Sqrt(float64(value) / float64(target)))
}
func trainingAverageLabelConfidence(labels TrainingStatsLabels) float64 {
values := []float64{}
appendScores := func(items []TrainingLabelStat) {
for _, item := range items {
values = append(values, clamp01(item.Confidence.Score))
}
}
appendScores(labels.People)
appendScores(labels.SexPositions)
appendScores(labels.BodyParts)
appendScores(labels.Objects)
appendScores(labels.Clothing)
if len(values) == 0 {
return 0
}
sum := 0.0
for _, value := range values {
sum += value
}
return clamp01(sum / float64(len(values)))
}
func trainingOverallConfidence(
feedbackCount int,
boxCount int,
acceptedCount int,
correctedCount int,
labels TrainingStatsLabels,
) TrainingConfidence {
if feedbackCount <= 0 {
return trainingConfidenceFromScore(0)
}
// Datenmenge: 300 Feedbacks sind grob "voll", darunter anteilig.
feedbackScore := trainingSaturationScore(feedbackCount, 300)
// Detector-Daten: 1000 Boxen sind grob "voll", darunter anteilig.
boxScore := trainingSaturationScore(boxCount, 1000)
// Label-Abdeckung aus den einzelnen Label-Confidence-Werten.
labelScore := trainingAverageLabelConfidence(labels)
// Modell-/Prediction-Zustimmung:
// Viele "Passt so"-Antworten bedeuten, dass die Vorhersagen brauchbar sind.
// Bei 4/229 ist dieser Teil bewusst sehr niedrig.
agreementScore := 0.0
if feedbackCount > 0 {
agreementScore = clamp01(float64(acceptedCount) / float64(feedbackCount))
}
// Korrekturquote als zusätzlicher Dämpfer.
// 98% korrigiert soll die Gesamt-Confidence sichtbar drücken,
// aber nicht alle gesammelten Daten entwerten.
correctionRate := 0.0
if feedbackCount > 0 {
correctionRate = clamp01(float64(correctedCount) / float64(feedbackCount))
}
correctionPenalty := 1.0 - math.Min(0.45, correctionRate*0.45)
// Gesamt:
// - Datenmenge zählt viel
// - Boxen und Label-Abdeckung zählen mittel
// - echte Modell-Zustimmung zählt bewusst mit rein
score :=
feedbackScore*0.30 +
boxScore*0.25 +
labelScore*0.25 +
agreementScore*0.20
score *= correctionPenalty
return trainingConfidenceFromScore(score)
}
func stringSet(values []string) map[string]bool {
out := map[string]bool{}
for _, value := range values {
clean := strings.TrimSpace(value)
if clean == "" {
continue
}
out[clean] = true
}
return out
}
func trainingRecognitionEnabled() bool {
return getSettings().TrainingRecognitionEnabled
}
@ -1754,7 +2218,6 @@ func trainingEnsureDetectorValidationSample(root string) error {
}
copied++
fmt.Println("✅ detector val sample duplicated:", id)
}
return nil