aodhan/character-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add semantic tagging, search, favourite/NSFW filtering, and LLM job queue

Browse Source 
Replaces old list-format tags (which duplicated prompt content) with structured
dict tags per category (origin_series, outfit_type, participants, style_type,
scene_type, etc.). Tags are now purely organizational metadata — removed from
the prompt pipeline entirely.

Adds is_favourite and is_nsfw columns to all 8 resource models. Favourite is
DB-only (user preference); NSFW is mirrored in JSON tags for rescan persistence.
All library pages get filter controls and favourites-first sorting.

Introduces a parallel LLM job queue (_enqueue_task + _llm_queue_worker) for
background tag regeneration, with the same status polling UI as ComfyUI jobs.
Fixes call_llm() to use has_request_context() fallback for background threads.

Adds global search (/search) across resources and gallery images, with navbar
search bar. Adds gallery image sidecar JSON for per-image favourite/NSFW metadata.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Aodhan Collins
2026-03-21 03:22:09 +00:00
parent 7d79e626a5
commit 32a73b02f5
72 changed files with 3163 additions and 2212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
services/generation.py
View File

@@ -11,13 +11,14 @@ from services.sync import _resolve_preset_entity, _resolve_preset_fields
logger = logging.getLogger('gaze')
def generate_from_preset(preset, overrides=None):
def generate_from_preset(preset, overrides=None, save_category='presets'):
"""Execute preset-based generation.
Args:
preset: Preset ORM object
overrides: optional dict with keys:
checkpoint, extra_positive, extra_negative, seed, width, height, action
save_category: upload sub-directory ('presets' or 'generator')
Returns:
job dict from _enqueue_job()
@@ -52,6 +53,19 @@ def generate_from_preset(preset, overrides=None):
detailer_obj = _resolve_preset_entity('detailer', detailer_cfg.get('detailer_id'))
look_obj = _resolve_preset_entity('look', look_cfg.get('look_id'))
# Build sidecar metadata with resolved entity slugs
resolved_meta = {
'preset_slug': preset.slug,
'preset_name': preset.name,
'character_slug': character.slug if character else None,
'outfit_slug': outfit.slug if outfit else None,
'action_slug': action_obj.slug if action_obj else None,
'style_slug': style_obj.slug if style_obj else None,
'scene_slug': scene_obj.slug if scene_obj else None,
'detailer_slug': detailer_obj.slug if detailer_obj else None,
'look_slug': look_obj.slug if look_obj else None,
}
# Checkpoint: override > preset config > default
checkpoint_override = overrides.get('checkpoint', '').strip() if overrides.get('checkpoint') else ''
if checkpoint_override:
@@ -71,14 +85,31 @@ def generate_from_preset(preset, overrides=None):
else:
ckpt_path, ckpt_data = _get_default_checkpoint()
resolved_meta['checkpoint_path'] = ckpt_path
# Resolve selected fields from preset toggles
selected_fields = _resolve_preset_fields(data)
# Check suppress_wardrobe: preset override > action default
suppress_wardrobe = False
preset_suppress = action_cfg.get('suppress_wardrobe')
if preset_suppress == 'random':
suppress_wardrobe = random.choice([True, False])
elif preset_suppress is not None:
suppress_wardrobe = bool(preset_suppress)
elif action_obj:
suppress_wardrobe = action_obj.data.get('suppress_wardrobe', False)
if suppress_wardrobe:
selected_fields = [f for f in selected_fields if not f.startswith('wardrobe::')]
# Build combined data for prompt building
active_wardrobe = char_cfg.get('fields', {}).get('wardrobe', {}).get('outfit', 'default')
wardrobe_source = outfit.data.get('wardrobe', {}) if outfit else None
if wardrobe_source is None:
wardrobe_source = character.get_active_wardrobe() if character else {}
if suppress_wardrobe:
wardrobe_source = {}
combined_data = {
'character_id': character.character_id if character else 'unknown',
@@ -88,7 +119,6 @@ def generate_from_preset(preset, overrides=None):
'styles': character.data.get('styles', {}) if character else {},
'lora': (look_obj.data.get('lora', {}) if look_obj
else (character.data.get('lora', {}) if character else {})),
'tags': (character.data.get('tags', []) if character else []) + data.get('tags', []),
}
# Build extras prompt from secondary resources
@@ -108,7 +138,6 @@ def generate_from_preset(preset, overrides=None):
trg = action_obj.data.get('lora', {}).get('lora_triggers', '')
if trg:
extras_parts.append(trg)
extras_parts.extend(action_obj.data.get('tags', []))
if style_obj:
s = style_obj.data.get('style', {})
if s.get('artist_name'):
@@ -133,7 +162,6 @@ def generate_from_preset(preset, overrides=None):
trg = scene_obj.data.get('lora', {}).get('lora_triggers', '')
if trg:
extras_parts.append(trg)
extras_parts.extend(scene_obj.data.get('tags', []))
if detailer_obj:
prompt_val = detailer_obj.data.get('prompt', '')
if isinstance(prompt_val, list):
@@ -195,6 +223,7 @@ def generate_from_preset(preset, overrides=None):
)
label = f"Preset: {preset.name} – {action}"
job = _enqueue_job(label, workflow, _make_finalize('presets', preset.slug, Preset, action))
db_model = Preset if save_category == 'presets' else None
job = _enqueue_job(label, workflow, _make_finalize(save_category, preset.slug, db_model, action, metadata=resolved_meta))
return job

84
services/job_queue.py
View File

@@ -1,3 +1,4 @@
import json
import os
import time
import uuid
@@ -27,8 +28,10 @@ logger = logging.getLogger('gaze')
_job_queue_lock = threading.Lock()
_job_queue = deque() # ordered list of job dicts (pending + paused + processing)
_llm_queue = deque() # ordered list of LLM task dicts (pending + paused + processing)
_job_history = {} # job_id -> job dict (all jobs ever added, for status lookup)
_queue_worker_event = threading.Event() # signals worker that a new job is available
_llm_worker_event = threading.Event() # signals LLM worker that a new task is available
# Stored reference to the Flask app, set by init_queue_worker()
_app = None
@@ -39,6 +42,7 @@ def _enqueue_job(label, workflow, finalize_fn):
job = {
'id': str(uuid.uuid4()),
'label': label,
'job_type': 'comfyui',
'status': 'pending',
'workflow': workflow,
'finalize_fn': finalize_fn,
@@ -55,6 +59,26 @@ def _enqueue_job(label, workflow, finalize_fn):
return job
def _enqueue_task(label, task_fn):
"""Add a generic task job (e.g. LLM call) to the LLM queue. Returns the job dict."""
job = {
'id': str(uuid.uuid4()),
'label': label,
'job_type': 'llm',
'status': 'pending',
'task_fn': task_fn,
'error': None,
'result': None,
'created_at': time.time(),
}
with _job_queue_lock:
_llm_queue.append(job)
_job_history[job['id']] = job
logger.info("LLM task queued: [%s] %s", job['id'][:8], label)
_llm_worker_event.set()
return job
def _queue_worker():
"""Background thread: processes jobs from _job_queue sequentially."""
while True:
@@ -174,13 +198,14 @@ def _queue_worker():
_prune_job_history()
def _make_finalize(category, slug, db_model_class=None, action=None):
def _make_finalize(category, slug, db_model_class=None, action=None, metadata=None):
"""Return a finalize callback for a standard queue job.
category — upload sub-directory name (e.g. 'characters', 'outfits')
slug — entity slug used for the upload folder name
db_model_class — SQLAlchemy model class for cover-image DB update; None = skip
action — 'replace' → update DB; None → always update; anything else → skip
metadata — optional dict to write as JSON sidecar alongside the image
"""
def _finalize(comfy_prompt_id, job):
logger.debug("=" * 80)
@@ -212,6 +237,14 @@ def _make_finalize(category, slug, db_model_class=None, action=None):
f.write(image_data)
logger.info("Image saved: %s (%d bytes)", full_path, len(image_data))
# Write JSON sidecar with generation metadata (if provided)
if metadata is not None:
sidecar_name = filename.rsplit('.', 1)[0] + '.json'
sidecar_path = os.path.join(folder, sidecar_name)
with open(sidecar_path, 'w') as sf:
json.dump(metadata, sf)
logger.debug(" Sidecar written: %s", sidecar_path)
relative_path = f"{category}/{slug}/{filename}"
# Include the seed used for this generation
used_seed = job['workflow'].get('3', {}).get('inputs', {}).get('seed')
@@ -244,6 +277,51 @@ def _make_finalize(category, slug, db_model_class=None, action=None):
return _finalize
def _llm_queue_worker():
"""Background thread: processes LLM task jobs sequentially."""
while True:
_llm_worker_event.wait()
_llm_worker_event.clear()
while True:
job = None
with _job_queue_lock:
for j in _llm_queue:
if j['status'] == 'pending':
job = j
break
if job is None:
break
with _job_queue_lock:
job['status'] = 'processing'
logger.info("LLM task started: [%s] %s", job['id'][:8], job['label'])
try:
with _app.app_context():
job['task_fn'](job)
with _job_queue_lock:
job['status'] = 'done'
logger.info("LLM task completed: [%s] %s", job['id'][:8], job['label'])
except Exception as e:
logger.exception("LLM task failed: [%s] %s — %s", job['id'][:8], job['label'], e)
with _job_queue_lock:
job['status'] = 'failed'
job['error'] = str(e)
with _job_queue_lock:
try:
_llm_queue.remove(job)
except ValueError:
pass
_prune_job_history()
def _prune_job_history(max_age_seconds=3600):
"""Remove completed/failed jobs older than max_age_seconds from _job_history."""
cutoff = time.time() - max_age_seconds
@@ -261,5 +339,5 @@ def init_queue_worker(flask_app):
"""
global _app
_app = flask_app
worker = threading.Thread(target=_queue_worker, daemon=True, name='queue-worker')
worker.start()
threading.Thread(target=_queue_worker, daemon=True, name='comfyui-worker').start()
threading.Thread(target=_llm_queue_worker, daemon=True, name='llm-worker').start()

38
services/llm.py
View File

@@ -2,7 +2,7 @@ import os
import json
import asyncio
import requests
from flask import request as flask_request
from flask import has_request_context, request as flask_request
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from models import Settings
@@ -77,6 +77,28 @@ def call_mcp_tool(name, arguments):
return json.dumps({"error": str(e)})
async def _run_character_mcp_tool(name, arguments):
server_params = StdioServerParameters(
command="docker",
args=["run", "--rm", "-i",
"-v", "character-cache:/root/.local/share/character_details",
"character-mcp:latest"],
)
async with stdio_client(server_params) as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
result = await session.call_tool(name, arguments)
return result.content[0].text
def call_character_mcp_tool(name, arguments):
try:
return asyncio.run(_run_character_mcp_tool(name, arguments))
except Exception as e:
print(f"Character MCP Tool Error: {e}")
return None
def load_prompt(filename):
path = os.path.join('data/prompts', filename)
if os.path.exists(path):
@@ -100,7 +122,7 @@ def call_llm(prompt, system_prompt="You are a creative assistant."):
headers = {
"Authorization": f"Bearer {settings.openrouter_api_key}",
"Content-Type": "application/json",
"HTTP-Referer": flask_request.url_root,
"HTTP-Referer": flask_request.url_root if has_request_context() else "http://localhost:5000/",
"X-Title": "Character Browser"
}
model = settings.openrouter_model or 'google/gemini-2.0-flash-001'
@@ -120,7 +142,8 @@ def call_llm(prompt, system_prompt="You are a creative assistant."):
{"role": "user", "content": prompt}
]
max_turns = 10
max_turns = 15
tool_turns_remaining = 8 # stop offering tools after this many tool-calling turns
use_tools = True
format_retries = 3 # retries allowed for unexpected response format
@@ -131,13 +154,13 @@ def call_llm(prompt, system_prompt="You are a creative assistant."):
"messages": messages,
}
# Only add tools if supported/requested
if use_tools:
# Only add tools if supported/requested and we haven't exhausted tool turns
if use_tools and tool_turns_remaining > 0:
data["tools"] = DANBOORU_TOOLS
data["tool_choice"] = "auto"
try:
response = requests.post(url, headers=headers, json=data)
response = requests.post(url, headers=headers, json=data, timeout=120)
# If 400 Bad Request and we were using tools, try once without tools
if response.status_code == 400 and use_tools:
@@ -158,6 +181,7 @@ def call_llm(prompt, system_prompt="You are a creative assistant."):
raise KeyError('message')
if message.get('tool_calls'):
tool_turns_remaining -= 1
messages.append(message)
for tool_call in message['tool_calls']:
name = tool_call['function']['name']
@@ -170,6 +194,8 @@ def call_llm(prompt, system_prompt="You are a creative assistant."):
"name": name,
"content": tool_result
})
if tool_turns_remaining <= 0:
print("Tool turn limit reached — next request will not offer tools")
continue
return message['content']

7
services/prompts.py
View File

@@ -171,10 +171,6 @@ def build_prompt(data, selected_fields=None, default_fields=None, active_outfit=
if style_data.get('artistic_style') and is_selected('style', 'artistic_style'):
parts.append(style_data['artistic_style'])
tags = data.get('tags', [])
if tags and is_selected('special', 'tags'):
parts.extend(tags)
lora = data.get('lora', {})
if lora.get('lora_triggers') and is_selected('lora', 'lora_triggers'):
parts.append(lora.get('lora_triggers'))
@@ -283,7 +279,6 @@ def build_extras_prompt(actions, outfits, scenes, styles, detailers):
lora = data.get('lora', {})
if lora.get('lora_triggers'):
parts.append(lora['lora_triggers'])
parts.extend(data.get('tags', []))
for key in _BODY_GROUP_KEYS:
val = data.get('action', {}).get(key)
if val:
@@ -299,7 +294,6 @@ def build_extras_prompt(actions, outfits, scenes, styles, detailers):
lora = data.get('lora', {})
if lora.get('lora_triggers'):
parts.append(lora['lora_triggers'])
parts.extend(data.get('tags', []))
for scene in scenes:
data = scene.data
@@ -311,7 +305,6 @@ def build_extras_prompt(actions, outfits, scenes, styles, detailers):
lora = data.get('lora', {})
if lora.get('lora_triggers'):
parts.append(lora['lora_triggers'])
parts.extend(data.get('tags', []))
for style in styles:
data = style.data

28
services/sync.py
View File

@@ -14,6 +14,13 @@ from models import (
logger = logging.getLogger('gaze')
def _sync_nsfw_from_tags(entity, data):
"""Sync is_nsfw from data['tags']['nsfw'] if tags is a dict. Never touches is_favourite."""
tags = data.get('tags')
if isinstance(tags, dict):
entity.is_nsfw = bool(tags.get('nsfw', False))
def sync_characters():
if not os.path.exists(current_app.config['CHARACTERS_DIR']):
return
@@ -44,6 +51,7 @@ def sync_characters():
character.name = name
character.slug = slug
character.filename = filename
_sync_nsfw_from_tags(character, data)
# Check if cover image still exists
if character.image_path:
@@ -62,6 +70,7 @@ def sync_characters():
name=name,
data=data
)
_sync_nsfw_from_tags(new_char, data)
db.session.add(new_char)
except Exception as e:
print(f"Error importing {filename}: {e}")
@@ -102,6 +111,7 @@ def sync_outfits():
outfit.name = name
outfit.slug = slug
outfit.filename = filename
_sync_nsfw_from_tags(outfit, data)
# Check if cover image still exists
if outfit.image_path:
@@ -120,6 +130,7 @@ def sync_outfits():
name=name,
data=data
)
_sync_nsfw_from_tags(new_outfit, data)
db.session.add(new_outfit)
except Exception as e:
print(f"Error importing outfit {filename}: {e}")
@@ -243,6 +254,7 @@ def sync_looks():
look.slug = slug
look.filename = filename
look.character_id = character_id
_sync_nsfw_from_tags(look, data)
if look.image_path:
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], look.image_path)
@@ -259,6 +271,7 @@ def sync_looks():
character_id=character_id,
data=data
)
_sync_nsfw_from_tags(new_look, data)
db.session.add(new_look)
except Exception as e:
print(f"Error importing look {filename}: {e}")
@@ -418,6 +431,7 @@ def sync_actions():
action.name = name
action.slug = slug
action.filename = filename
_sync_nsfw_from_tags(action, data)
# Check if cover image still exists
if action.image_path:
@@ -435,6 +449,7 @@ def sync_actions():
name=name,
data=data
)
_sync_nsfw_from_tags(new_action, data)
db.session.add(new_action)
except Exception as e:
print(f"Error importing action {filename}: {e}")
@@ -475,6 +490,7 @@ def sync_styles():
style.name = name
style.slug = slug
style.filename = filename
_sync_nsfw_from_tags(style, data)
# Check if cover image still exists
if style.image_path:
@@ -492,6 +508,7 @@ def sync_styles():
name=name,
data=data
)
_sync_nsfw_from_tags(new_style, data)
db.session.add(new_style)
except Exception as e:
print(f"Error importing style {filename}: {e}")
@@ -532,6 +549,7 @@ def sync_detailers():
detailer.name = name
detailer.slug = slug
detailer.filename = filename
_sync_nsfw_from_tags(detailer, data)
# Check if cover image still exists
if detailer.image_path:
@@ -549,6 +567,7 @@ def sync_detailers():
name=name,
data=data
)
_sync_nsfw_from_tags(new_detailer, data)
db.session.add(new_detailer)
except Exception as e:
print(f"Error importing detailer {filename}: {e}")
@@ -589,6 +608,7 @@ def sync_scenes():
scene.name = name
scene.slug = slug
scene.filename = filename
_sync_nsfw_from_tags(scene, data)
# Check if cover image still exists
if scene.image_path:
@@ -606,6 +626,7 @@ def sync_scenes():
name=name,
data=data
)
_sync_nsfw_from_tags(new_scene, data)
db.session.add(new_scene)
except Exception as e:
print(f"Error importing scene {filename}: {e}")
@@ -679,19 +700,22 @@ def sync_checkpoints():
ckpt.slug = slug
ckpt.checkpoint_path = checkpoint_path
ckpt.data = data
_sync_nsfw_from_tags(ckpt, data)
flag_modified(ckpt, "data")
if ckpt.image_path:
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], ckpt.image_path)
if not os.path.exists(full_img_path):
ckpt.image_path = None
else:
db.session.add(Checkpoint(
new_ckpt = Checkpoint(
checkpoint_id=checkpoint_id,
slug=slug,
name=display_name,
checkpoint_path=checkpoint_path,
data=data,
))
)
_sync_nsfw_from_tags(new_ckpt, data)
db.session.add(new_ckpt)
all_ckpts = Checkpoint.query.all()
for ckpt in all_ckpts: