Major refactor: deduplicate routes, sync, JS, and fix bugs
- Extract 8 common route patterns into factory functions in routes/shared.py (favourite, upload, replace cover, save defaults, clone, save JSON, get missing, clear covers) — removes ~1,100 lines across 9 route files - Extract generic _sync_category() in sync.py — 7 sync functions become one-liner wrappers, removing ~350 lines - Extract shared detail page JS into static/js/detail-common.js — all 9 detail templates now call initDetailPage() with minimal config - Extract layout inline JS into static/js/layout-utils.js (~185 lines) - Extract library toolbar JS into static/js/library-toolbar.js - Fix finalize missing-image bug: raise RuntimeError instead of logging warning so job is marked failed - Fix missing scheduler default in _default_checkpoint_data() - Fix N+1 query in Character.get_available_outfits() with batch IN query - Convert all print() to logger across services and routes - Add missing tags display to styles, scenes, detailers, checkpoints detail - Update delete buttons to use trash.png icon with solid red background - Update CLAUDE.md to reflect new architecture Net reduction: ~1,600 lines Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Aodhan Collins
449
services/sync.py
449
services/sync.py
@@ -57,7 +57,7 @@ def sync_characters():
|
||||
if character.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], character.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
print(f"Image missing for {character.name}, clearing path.")
|
||||
logger.warning("Image missing for %s, clearing path.", character.name)
|
||||
character.image_path = None
|
||||
|
||||
# Explicitly tell SQLAlchemy the JSON field was modified
|
||||
@@ -73,7 +73,7 @@ def sync_characters():
|
||||
_sync_nsfw_from_tags(new_char, data)
|
||||
db.session.add(new_char)
|
||||
except Exception as e:
|
||||
print(f"Error importing {filename}: {e}")
|
||||
logger.error("Error importing %s: %s", filename, e)
|
||||
|
||||
# Remove characters that are no longer in the folder
|
||||
all_characters = Character.query.all()
|
||||
@@ -83,66 +83,82 @@ def sync_characters():
|
||||
|
||||
db.session.commit()
|
||||
|
||||
def sync_outfits():
|
||||
if not os.path.exists(current_app.config['CLOTHING_DIR']):
|
||||
def _sync_category(config_key, model_class, id_field, name_field,
|
||||
extra_fn=None, sync_nsfw=True):
|
||||
"""Generic sync: load JSON files from a data directory into the database.
|
||||
|
||||
Args:
|
||||
config_key: app.config key for the data directory (e.g. 'CLOTHING_DIR')
|
||||
model_class: SQLAlchemy model class (e.g. Outfit)
|
||||
id_field: JSON key for the entity ID (e.g. 'outfit_id')
|
||||
name_field: JSON key for the display name (e.g. 'outfit_name')
|
||||
extra_fn: optional callable(entity, data) for category-specific field updates
|
||||
sync_nsfw: if True, call _sync_nsfw_from_tags on create/update
|
||||
"""
|
||||
data_dir = current_app.config.get(config_key)
|
||||
if not data_dir or not os.path.exists(data_dir):
|
||||
return
|
||||
|
||||
current_ids = []
|
||||
|
||||
for filename in os.listdir(current_app.config['CLOTHING_DIR']):
|
||||
for filename in os.listdir(data_dir):
|
||||
if filename.endswith('.json'):
|
||||
file_path = os.path.join(current_app.config['CLOTHING_DIR'], filename)
|
||||
file_path = os.path.join(data_dir, filename)
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
outfit_id = data.get('outfit_id') or filename.replace('.json', '')
|
||||
entity_id = data.get(id_field) or filename.replace('.json', '')
|
||||
current_ids.append(entity_id)
|
||||
|
||||
current_ids.append(outfit_id)
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', entity_id)
|
||||
entity = model_class.query.filter_by(**{id_field: entity_id}).first()
|
||||
name = data.get(name_field, entity_id.replace('_', ' ').title())
|
||||
|
||||
# Generate URL-safe slug: remove special characters from outfit_id
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', outfit_id)
|
||||
if entity:
|
||||
entity.data = data
|
||||
entity.name = name
|
||||
entity.slug = slug
|
||||
entity.filename = filename
|
||||
if sync_nsfw:
|
||||
_sync_nsfw_from_tags(entity, data)
|
||||
if extra_fn:
|
||||
extra_fn(entity, data)
|
||||
|
||||
# Check if outfit already exists
|
||||
outfit = Outfit.query.filter_by(outfit_id=outfit_id).first()
|
||||
name = data.get('outfit_name', outfit_id.replace('_', ' ').title())
|
||||
|
||||
if outfit:
|
||||
outfit.data = data
|
||||
outfit.name = name
|
||||
outfit.slug = slug
|
||||
outfit.filename = filename
|
||||
_sync_nsfw_from_tags(outfit, data)
|
||||
|
||||
# Check if cover image still exists
|
||||
if outfit.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], outfit.image_path)
|
||||
if entity.image_path:
|
||||
full_img_path = os.path.join(
|
||||
current_app.config['UPLOAD_FOLDER'], entity.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
print(f"Image missing for {outfit.name}, clearing path.")
|
||||
outfit.image_path = None
|
||||
logger.warning("Image missing for %s, clearing path.", entity.name)
|
||||
entity.image_path = None
|
||||
|
||||
# Explicitly tell SQLAlchemy the JSON field was modified
|
||||
flag_modified(outfit, "data")
|
||||
flag_modified(entity, "data")
|
||||
else:
|
||||
new_outfit = Outfit(
|
||||
outfit_id=outfit_id,
|
||||
slug=slug,
|
||||
filename=filename,
|
||||
name=name,
|
||||
data=data
|
||||
)
|
||||
_sync_nsfw_from_tags(new_outfit, data)
|
||||
db.session.add(new_outfit)
|
||||
kwargs = {
|
||||
id_field: entity_id,
|
||||
'slug': slug,
|
||||
'filename': filename,
|
||||
'name': name,
|
||||
'data': data,
|
||||
}
|
||||
new_entity = model_class(**kwargs)
|
||||
if sync_nsfw:
|
||||
_sync_nsfw_from_tags(new_entity, data)
|
||||
if extra_fn:
|
||||
extra_fn(new_entity, data)
|
||||
db.session.add(new_entity)
|
||||
except Exception as e:
|
||||
print(f"Error importing outfit {filename}: {e}")
|
||||
logger.error("Error importing %s: %s", filename, e)
|
||||
|
||||
# Remove outfits that are no longer in the folder
|
||||
all_outfits = Outfit.query.all()
|
||||
for outfit in all_outfits:
|
||||
if outfit.outfit_id not in current_ids:
|
||||
db.session.delete(outfit)
|
||||
for entity in model_class.query.all():
|
||||
if getattr(entity, id_field) not in current_ids:
|
||||
db.session.delete(entity)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def sync_outfits():
|
||||
_sync_category('CLOTHING_DIR', Outfit, 'outfit_id', 'outfit_name')
|
||||
|
||||
def ensure_default_outfit():
|
||||
"""Ensure a default outfit file exists and is registered in the database.
|
||||
|
||||
@@ -226,114 +242,16 @@ def ensure_default_outfit():
|
||||
|
||||
|
||||
|
||||
def _sync_look_extra(entity, data):
|
||||
entity.character_id = data.get('character_id', None)
|
||||
|
||||
def sync_looks():
|
||||
if not os.path.exists(current_app.config['LOOKS_DIR']):
|
||||
return
|
||||
|
||||
current_ids = []
|
||||
|
||||
for filename in os.listdir(current_app.config['LOOKS_DIR']):
|
||||
if filename.endswith('.json'):
|
||||
file_path = os.path.join(current_app.config['LOOKS_DIR'], filename)
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
look_id = data.get('look_id') or filename.replace('.json', '')
|
||||
|
||||
current_ids.append(look_id)
|
||||
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', look_id)
|
||||
|
||||
look = Look.query.filter_by(look_id=look_id).first()
|
||||
name = data.get('look_name', look_id.replace('_', ' ').title())
|
||||
character_id = data.get('character_id', None)
|
||||
|
||||
if look:
|
||||
look.data = data
|
||||
look.name = name
|
||||
look.slug = slug
|
||||
look.filename = filename
|
||||
look.character_id = character_id
|
||||
_sync_nsfw_from_tags(look, data)
|
||||
|
||||
if look.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], look.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
look.image_path = None
|
||||
|
||||
flag_modified(look, "data")
|
||||
else:
|
||||
new_look = Look(
|
||||
look_id=look_id,
|
||||
slug=slug,
|
||||
filename=filename,
|
||||
name=name,
|
||||
character_id=character_id,
|
||||
data=data
|
||||
)
|
||||
_sync_nsfw_from_tags(new_look, data)
|
||||
db.session.add(new_look)
|
||||
except Exception as e:
|
||||
print(f"Error importing look {filename}: {e}")
|
||||
|
||||
all_looks = Look.query.all()
|
||||
for look in all_looks:
|
||||
if look.look_id not in current_ids:
|
||||
db.session.delete(look)
|
||||
|
||||
db.session.commit()
|
||||
_sync_category('LOOKS_DIR', Look, 'look_id', 'look_name',
|
||||
extra_fn=_sync_look_extra)
|
||||
|
||||
def sync_presets():
|
||||
if not os.path.exists(current_app.config['PRESETS_DIR']):
|
||||
return
|
||||
|
||||
current_ids = []
|
||||
|
||||
for filename in os.listdir(current_app.config['PRESETS_DIR']):
|
||||
if filename.endswith('.json'):
|
||||
file_path = os.path.join(current_app.config['PRESETS_DIR'], filename)
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
preset_id = data.get('preset_id') or filename.replace('.json', '')
|
||||
|
||||
current_ids.append(preset_id)
|
||||
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', preset_id)
|
||||
|
||||
preset = Preset.query.filter_by(preset_id=preset_id).first()
|
||||
name = data.get('preset_name', preset_id.replace('_', ' ').title())
|
||||
|
||||
if preset:
|
||||
preset.data = data
|
||||
preset.name = name
|
||||
preset.slug = slug
|
||||
preset.filename = filename
|
||||
|
||||
if preset.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], preset.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
preset.image_path = None
|
||||
|
||||
flag_modified(preset, "data")
|
||||
else:
|
||||
new_preset = Preset(
|
||||
preset_id=preset_id,
|
||||
slug=slug,
|
||||
filename=filename,
|
||||
name=name,
|
||||
data=data
|
||||
)
|
||||
db.session.add(new_preset)
|
||||
except Exception as e:
|
||||
print(f"Error importing preset {filename}: {e}")
|
||||
|
||||
all_presets = Preset.query.all()
|
||||
for preset in all_presets:
|
||||
if preset.preset_id not in current_ids:
|
||||
db.session.delete(preset)
|
||||
|
||||
db.session.commit()
|
||||
_sync_category('PRESETS_DIR', Preset, 'preset_id', 'preset_name',
|
||||
sync_nsfw=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -404,240 +322,16 @@ def _resolve_preset_fields(preset_data):
|
||||
|
||||
|
||||
def sync_actions():
|
||||
if not os.path.exists(current_app.config['ACTIONS_DIR']):
|
||||
return
|
||||
|
||||
current_ids = []
|
||||
|
||||
for filename in os.listdir(current_app.config['ACTIONS_DIR']):
|
||||
if filename.endswith('.json'):
|
||||
file_path = os.path.join(current_app.config['ACTIONS_DIR'], filename)
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
action_id = data.get('action_id') or filename.replace('.json', '')
|
||||
|
||||
current_ids.append(action_id)
|
||||
|
||||
# Generate URL-safe slug
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', action_id)
|
||||
|
||||
# Check if action already exists
|
||||
action = Action.query.filter_by(action_id=action_id).first()
|
||||
name = data.get('action_name', action_id.replace('_', ' ').title())
|
||||
|
||||
if action:
|
||||
action.data = data
|
||||
action.name = name
|
||||
action.slug = slug
|
||||
action.filename = filename
|
||||
_sync_nsfw_from_tags(action, data)
|
||||
|
||||
# Check if cover image still exists
|
||||
if action.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], action.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
print(f"Image missing for {action.name}, clearing path.")
|
||||
action.image_path = None
|
||||
|
||||
flag_modified(action, "data")
|
||||
else:
|
||||
new_action = Action(
|
||||
action_id=action_id,
|
||||
slug=slug,
|
||||
filename=filename,
|
||||
name=name,
|
||||
data=data
|
||||
)
|
||||
_sync_nsfw_from_tags(new_action, data)
|
||||
db.session.add(new_action)
|
||||
except Exception as e:
|
||||
print(f"Error importing action {filename}: {e}")
|
||||
|
||||
# Remove actions that are no longer in the folder
|
||||
all_actions = Action.query.all()
|
||||
for action in all_actions:
|
||||
if action.action_id not in current_ids:
|
||||
db.session.delete(action)
|
||||
|
||||
db.session.commit()
|
||||
_sync_category('ACTIONS_DIR', Action, 'action_id', 'action_name')
|
||||
|
||||
def sync_styles():
|
||||
if not os.path.exists(current_app.config['STYLES_DIR']):
|
||||
return
|
||||
|
||||
current_ids = []
|
||||
|
||||
for filename in os.listdir(current_app.config['STYLES_DIR']):
|
||||
if filename.endswith('.json'):
|
||||
file_path = os.path.join(current_app.config['STYLES_DIR'], filename)
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
style_id = data.get('style_id') or filename.replace('.json', '')
|
||||
|
||||
current_ids.append(style_id)
|
||||
|
||||
# Generate URL-safe slug
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', style_id)
|
||||
|
||||
# Check if style already exists
|
||||
style = Style.query.filter_by(style_id=style_id).first()
|
||||
name = data.get('style_name', style_id.replace('_', ' ').title())
|
||||
|
||||
if style:
|
||||
style.data = data
|
||||
style.name = name
|
||||
style.slug = slug
|
||||
style.filename = filename
|
||||
_sync_nsfw_from_tags(style, data)
|
||||
|
||||
# Check if cover image still exists
|
||||
if style.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], style.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
print(f"Image missing for {style.name}, clearing path.")
|
||||
style.image_path = None
|
||||
|
||||
flag_modified(style, "data")
|
||||
else:
|
||||
new_style = Style(
|
||||
style_id=style_id,
|
||||
slug=slug,
|
||||
filename=filename,
|
||||
name=name,
|
||||
data=data
|
||||
)
|
||||
_sync_nsfw_from_tags(new_style, data)
|
||||
db.session.add(new_style)
|
||||
except Exception as e:
|
||||
print(f"Error importing style {filename}: {e}")
|
||||
|
||||
# Remove styles that are no longer in the folder
|
||||
all_styles = Style.query.all()
|
||||
for style in all_styles:
|
||||
if style.style_id not in current_ids:
|
||||
db.session.delete(style)
|
||||
|
||||
db.session.commit()
|
||||
_sync_category('STYLES_DIR', Style, 'style_id', 'style_name')
|
||||
|
||||
def sync_detailers():
|
||||
if not os.path.exists(current_app.config['DETAILERS_DIR']):
|
||||
return
|
||||
|
||||
current_ids = []
|
||||
|
||||
for filename in os.listdir(current_app.config['DETAILERS_DIR']):
|
||||
if filename.endswith('.json'):
|
||||
file_path = os.path.join(current_app.config['DETAILERS_DIR'], filename)
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
detailer_id = data.get('detailer_id') or filename.replace('.json', '')
|
||||
|
||||
current_ids.append(detailer_id)
|
||||
|
||||
# Generate URL-safe slug
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', detailer_id)
|
||||
|
||||
# Check if detailer already exists
|
||||
detailer = Detailer.query.filter_by(detailer_id=detailer_id).first()
|
||||
name = data.get('detailer_name', detailer_id.replace('_', ' ').title())
|
||||
|
||||
if detailer:
|
||||
detailer.data = data
|
||||
detailer.name = name
|
||||
detailer.slug = slug
|
||||
detailer.filename = filename
|
||||
_sync_nsfw_from_tags(detailer, data)
|
||||
|
||||
# Check if cover image still exists
|
||||
if detailer.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], detailer.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
print(f"Image missing for {detailer.name}, clearing path.")
|
||||
detailer.image_path = None
|
||||
|
||||
flag_modified(detailer, "data")
|
||||
else:
|
||||
new_detailer = Detailer(
|
||||
detailer_id=detailer_id,
|
||||
slug=slug,
|
||||
filename=filename,
|
||||
name=name,
|
||||
data=data
|
||||
)
|
||||
_sync_nsfw_from_tags(new_detailer, data)
|
||||
db.session.add(new_detailer)
|
||||
except Exception as e:
|
||||
print(f"Error importing detailer {filename}: {e}")
|
||||
|
||||
# Remove detailers that are no longer in the folder
|
||||
all_detailers = Detailer.query.all()
|
||||
for detailer in all_detailers:
|
||||
if detailer.detailer_id not in current_ids:
|
||||
db.session.delete(detailer)
|
||||
|
||||
db.session.commit()
|
||||
_sync_category('DETAILERS_DIR', Detailer, 'detailer_id', 'detailer_name')
|
||||
|
||||
def sync_scenes():
|
||||
if not os.path.exists(current_app.config['SCENES_DIR']):
|
||||
return
|
||||
|
||||
current_ids = []
|
||||
|
||||
for filename in os.listdir(current_app.config['SCENES_DIR']):
|
||||
if filename.endswith('.json'):
|
||||
file_path = os.path.join(current_app.config['SCENES_DIR'], filename)
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
scene_id = data.get('scene_id') or filename.replace('.json', '')
|
||||
|
||||
current_ids.append(scene_id)
|
||||
|
||||
# Generate URL-safe slug
|
||||
slug = re.sub(r'[^a-zA-Z0-9_]', '', scene_id)
|
||||
|
||||
# Check if scene already exists
|
||||
scene = Scene.query.filter_by(scene_id=scene_id).first()
|
||||
name = data.get('scene_name', scene_id.replace('_', ' ').title())
|
||||
|
||||
if scene:
|
||||
scene.data = data
|
||||
scene.name = name
|
||||
scene.slug = slug
|
||||
scene.filename = filename
|
||||
_sync_nsfw_from_tags(scene, data)
|
||||
|
||||
# Check if cover image still exists
|
||||
if scene.image_path:
|
||||
full_img_path = os.path.join(current_app.config['UPLOAD_FOLDER'], scene.image_path)
|
||||
if not os.path.exists(full_img_path):
|
||||
print(f"Image missing for {scene.name}, clearing path.")
|
||||
scene.image_path = None
|
||||
|
||||
flag_modified(scene, "data")
|
||||
else:
|
||||
new_scene = Scene(
|
||||
scene_id=scene_id,
|
||||
slug=slug,
|
||||
filename=filename,
|
||||
name=name,
|
||||
data=data
|
||||
)
|
||||
_sync_nsfw_from_tags(new_scene, data)
|
||||
db.session.add(new_scene)
|
||||
except Exception as e:
|
||||
print(f"Error importing scene {filename}: {e}")
|
||||
|
||||
# Remove scenes that are no longer in the folder
|
||||
all_scenes = Scene.query.all()
|
||||
for scene in all_scenes:
|
||||
if scene.scene_id not in current_ids:
|
||||
db.session.delete(scene)
|
||||
|
||||
db.session.commit()
|
||||
_sync_category('SCENES_DIR', Scene, 'scene_id', 'scene_name')
|
||||
|
||||
def _default_checkpoint_data(checkpoint_path, filename):
|
||||
"""Return template-default data for a checkpoint with no JSON file."""
|
||||
@@ -650,6 +344,7 @@ def _default_checkpoint_data(checkpoint_path, filename):
|
||||
"steps": 25,
|
||||
"cfg": 5,
|
||||
"sampler_name": "euler_ancestral",
|
||||
"scheduler": "normal",
|
||||
"vae": "integrated"
|
||||
}
|
||||
|
||||
@@ -669,7 +364,7 @@ def sync_checkpoints():
|
||||
if ckpt_path:
|
||||
json_data_by_path[ckpt_path] = data
|
||||
except Exception as e:
|
||||
print(f"Error reading checkpoint JSON {filename}: {e}")
|
||||
logger.error("Error reading checkpoint JSON %s: %s", filename, e)
|
||||
|
||||
current_ids = []
|
||||
dirs = [
|
||||
|
||||
Reference in New Issue
Block a user