Add semantic tagging, search, favourite/NSFW filtering, and LLM job queue
Replaces old list-format tags (which duplicated prompt content) with structured dict tags per category (origin_series, outfit_type, participants, style_type, scene_type, etc.). Tags are now purely organizational metadata — removed from the prompt pipeline entirely. Adds is_favourite and is_nsfw columns to all 8 resource models. Favourite is DB-only (user preference); NSFW is mirrored in JSON tags for rescan persistence. All library pages get filter controls and favourites-first sorting. Introduces a parallel LLM job queue (_enqueue_task + _llm_queue_worker) for background tag regeneration, with the same status polling UI as ComfyUI jobs. Fixes call_llm() to use has_request_context() fallback for background threads. Adds global search (/search) across resources and gallery images, with navbar search bar. Adds gallery image sidecar JSON for per-image favourite/NSFW metadata. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Aodhan Collins
@@ -1,7 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
|
||||
from flask import render_template, request, redirect, url_for, flash, session, current_app
|
||||
@@ -10,7 +9,7 @@ from sqlalchemy.orm.attributes import flag_modified
|
||||
|
||||
from models import db, Checkpoint, Character, Settings
|
||||
from services.workflow import _prepare_workflow, _get_default_checkpoint, _apply_checkpoint_settings, _log_workflow_prompts
|
||||
from services.job_queue import _enqueue_job, _make_finalize
|
||||
from services.job_queue import _enqueue_job, _make_finalize, _enqueue_task
|
||||
from services.prompts import build_prompt, _resolve_character, _ensure_character_fields, _append_background
|
||||
from services.sync import sync_checkpoints, _default_checkpoint_data
|
||||
from services.file_io import get_available_checkpoints
|
||||
@@ -57,8 +56,17 @@ def register_routes(app):
|
||||
|
||||
@app.route('/checkpoints')
|
||||
def checkpoints_index():
|
||||
checkpoints = Checkpoint.query.order_by(Checkpoint.name).all()
|
||||
return render_template('checkpoints/index.html', checkpoints=checkpoints)
|
||||
query = Checkpoint.query
|
||||
fav = request.args.get('favourite')
|
||||
nsfw = request.args.get('nsfw', 'all')
|
||||
if fav == 'on':
|
||||
query = query.filter_by(is_favourite=True)
|
||||
if nsfw == 'sfw':
|
||||
query = query.filter_by(is_nsfw=False)
|
||||
elif nsfw == 'nsfw':
|
||||
query = query.filter_by(is_nsfw=True)
|
||||
checkpoints = query.order_by(Checkpoint.is_favourite.desc(), Checkpoint.name).all()
|
||||
return render_template('checkpoints/index.html', checkpoints=checkpoints, favourite_filter=fav or '', nsfw_filter=nsfw)
|
||||
|
||||
@app.route('/checkpoints/rescan', methods=['POST'])
|
||||
def rescan_checkpoints():
|
||||
@@ -189,9 +197,9 @@ def register_routes(app):
|
||||
os.makedirs(checkpoints_dir, exist_ok=True)
|
||||
|
||||
overwrite = request.form.get('overwrite') == 'true'
|
||||
created_count = 0
|
||||
skipped_count = 0
|
||||
overwritten_count = 0
|
||||
skipped = 0
|
||||
written_directly = 0
|
||||
job_ids = []
|
||||
|
||||
system_prompt = load_prompt('checkpoint_system.txt')
|
||||
if not system_prompt:
|
||||
@@ -219,7 +227,7 @@ def register_routes(app):
|
||||
|
||||
is_existing = os.path.exists(json_path)
|
||||
if is_existing and not overwrite:
|
||||
skipped_count += 1
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Look for a matching HTML file alongside the model file
|
||||
@@ -235,52 +243,72 @@ def register_routes(app):
|
||||
clean_html = re.sub(r'<[^>]+>', ' ', clean_html)
|
||||
html_content = ' '.join(clean_html.split())
|
||||
except Exception as e:
|
||||
print(f"Error reading HTML for {filename}: {e}")
|
||||
logger.error("Error reading HTML for %s: %s", filename, e)
|
||||
|
||||
defaults = _default_checkpoint_data(checkpoint_path, filename)
|
||||
|
||||
if html_content:
|
||||
try:
|
||||
print(f"Asking LLM to describe checkpoint: {filename}")
|
||||
prompt = (
|
||||
f"Generate checkpoint metadata JSON for the model file: '{filename}' "
|
||||
f"(checkpoint_path: '{checkpoint_path}').\n\n"
|
||||
f"Here is descriptive text extracted from an associated HTML file:\n###\n{html_content[:3000]}\n###"
|
||||
)
|
||||
llm_response = call_llm(prompt, system_prompt)
|
||||
clean_json = llm_response.replace('```json', '').replace('```', '').strip()
|
||||
ckpt_data = json.loads(clean_json)
|
||||
# Enforce fixed fields
|
||||
ckpt_data['checkpoint_path'] = checkpoint_path
|
||||
ckpt_data['checkpoint_name'] = filename
|
||||
# Fill missing fields with defaults
|
||||
for key, val in defaults.items():
|
||||
if key not in ckpt_data or ckpt_data[key] is None:
|
||||
ckpt_data[key] = val
|
||||
time.sleep(0.5)
|
||||
except Exception as e:
|
||||
print(f"LLM error for {filename}: {e}. Using defaults.")
|
||||
ckpt_data = defaults
|
||||
# Has HTML companion — enqueue LLM task
|
||||
def make_task(filename, checkpoint_path, json_path, html_content, system_prompt, defaults, is_existing):
|
||||
def task_fn(job):
|
||||
prompt = (
|
||||
f"Generate checkpoint metadata JSON for the model file: '{filename}' "
|
||||
f"(checkpoint_path: '{checkpoint_path}').\n\n"
|
||||
f"Here is descriptive text extracted from an associated HTML file:\n###\n{html_content[:3000]}\n###"
|
||||
)
|
||||
try:
|
||||
llm_response = call_llm(prompt, system_prompt)
|
||||
clean_json = llm_response.replace('```json', '').replace('```', '').strip()
|
||||
ckpt_data = json.loads(clean_json)
|
||||
ckpt_data['checkpoint_path'] = checkpoint_path
|
||||
ckpt_data['checkpoint_name'] = filename
|
||||
for key, val in defaults.items():
|
||||
if key not in ckpt_data or ckpt_data[key] is None:
|
||||
ckpt_data[key] = val
|
||||
except Exception as e:
|
||||
logger.error("LLM error for %s: %s. Using defaults.", filename, e)
|
||||
ckpt_data = defaults
|
||||
|
||||
with open(json_path, 'w') as f:
|
||||
json.dump(ckpt_data, f, indent=2)
|
||||
|
||||
job['result'] = {'name': filename, 'action': 'overwritten' if is_existing else 'created'}
|
||||
return task_fn
|
||||
|
||||
job = _enqueue_task(f"Create checkpoint: {filename}", make_task(filename, checkpoint_path, json_path, html_content, system_prompt, defaults, is_existing))
|
||||
job_ids.append(job['id'])
|
||||
else:
|
||||
ckpt_data = defaults
|
||||
# No HTML — write defaults directly (no LLM needed)
|
||||
try:
|
||||
with open(json_path, 'w') as f:
|
||||
json.dump(defaults, f, indent=2)
|
||||
written_directly += 1
|
||||
except Exception as e:
|
||||
logger.error("Error saving JSON for %s: %s", filename, e)
|
||||
|
||||
try:
|
||||
with open(json_path, 'w') as f:
|
||||
json.dump(ckpt_data, f, indent=2)
|
||||
if is_existing:
|
||||
overwritten_count += 1
|
||||
else:
|
||||
created_count += 1
|
||||
except Exception as e:
|
||||
print(f"Error saving JSON for {filename}: {e}")
|
||||
needs_sync = len(job_ids) > 0 or written_directly > 0
|
||||
|
||||
if created_count > 0 or overwritten_count > 0:
|
||||
sync_checkpoints()
|
||||
msg = f'Successfully processed checkpoints: {created_count} created, {overwritten_count} overwritten.'
|
||||
if skipped_count > 0:
|
||||
msg += f' (Skipped {skipped_count} existing)'
|
||||
flash(msg)
|
||||
else:
|
||||
flash(f'No checkpoints created or overwritten. {skipped_count} existing entries found.')
|
||||
if needs_sync:
|
||||
if job_ids:
|
||||
# Sync after all LLM tasks complete
|
||||
def sync_task(job):
|
||||
sync_checkpoints()
|
||||
job['result'] = {'synced': True}
|
||||
_enqueue_task("Sync checkpoints DB", sync_task)
|
||||
else:
|
||||
# No LLM tasks — sync immediately
|
||||
sync_checkpoints()
|
||||
|
||||
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
|
||||
return {'success': True, 'queued': len(job_ids), 'written_directly': written_directly, 'skipped': skipped}
|
||||
flash(f'Queued {len(job_ids)} checkpoint tasks, {written_directly} written directly ({skipped} skipped).')
|
||||
return redirect(url_for('checkpoints_index'))
|
||||
|
||||
@app.route('/checkpoint/<path:slug>/favourite', methods=['POST'])
|
||||
def toggle_checkpoint_favourite(slug):
|
||||
ckpt = Checkpoint.query.filter_by(slug=slug).first_or_404()
|
||||
ckpt.is_favourite = not ckpt.is_favourite
|
||||
db.session.commit()
|
||||
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
|
||||
return {'success': True, 'is_favourite': ckpt.is_favourite}
|
||||
return redirect(url_for('checkpoint_detail', slug=slug))
|
||||
|
||||
Reference in New Issue
Block a user