Add background job queue system for generation
- Implements sequential job queue with background worker thread (_enqueue_job, _queue_worker) - All generate routes now return job_id instead of prompt_id; frontend polls /api/queue/<id>/status - Queue management UI in navbar with live badge, job list, pause/resume/remove controls - Fix: replaced url_for() calls inside finalize callbacks with direct string paths (url_for raises RuntimeError without request context in background threads) - Batch cover generation now uses two-phase pattern: queue all jobs upfront, then poll concurrently via Promise.all so page navigation doesn't interrupt the process - Strengths gallery sweep migrated to same two-phase pattern; sgStop() cancels queued jobs server-side - LoRA weight randomisation via lora_weight_min/lora_weight_max already present in _resolve_lora_weight Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aodhan Collins
@@ -292,173 +292,57 @@
|
||||
}
|
||||
});
|
||||
|
||||
// Generate a unique client ID
|
||||
const clientId = 'scene_detail_' + Math.random().toString(36).substring(2, 15);
|
||||
let currentJobId = null;
|
||||
|
||||
// ComfyUI WebSocket
|
||||
const socket = new WebSocket('{{ COMFYUI_WS_URL }}?clientId=' + clientId);
|
||||
|
||||
const nodeNames = {
|
||||
"3": "Sampling",
|
||||
"11": "Face Detailing",
|
||||
"13": "Hand Detailing",
|
||||
"4": "Loading Models",
|
||||
"16": "Character LoRA",
|
||||
"17": "Outfit LoRA",
|
||||
"18": "Action LoRA",
|
||||
"19": "Style/Detailer LoRA",
|
||||
"8": "Decoding Image",
|
||||
"9": "Saving Image"
|
||||
};
|
||||
|
||||
let currentPromptId = null;
|
||||
let resolveCompletion = null;
|
||||
|
||||
socket.addEventListener('message', (event) => {
|
||||
const msg = JSON.parse(event.data);
|
||||
|
||||
if (msg.type === 'status') {
|
||||
if (!currentPromptId) {
|
||||
const queueRemaining = msg.data.status.exec_info.queue_remaining;
|
||||
if (queueRemaining > 0) {
|
||||
progressLabel.textContent = `Queue position: ${queueRemaining}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (msg.type === 'progress') {
|
||||
if (msg.data.prompt_id !== currentPromptId) return;
|
||||
const value = msg.data.value;
|
||||
const max = msg.data.max;
|
||||
const percent = Math.round((value / max) * 100);
|
||||
progressBar.style.width = `${percent}%`;
|
||||
progressBar.textContent = `${percent}%`;
|
||||
}
|
||||
else if (msg.type === 'executing') {
|
||||
if (msg.data.prompt_id !== currentPromptId) return;
|
||||
|
||||
const nodeId = msg.data.node;
|
||||
if (nodeId === null) {
|
||||
// Execution finished via WebSocket
|
||||
console.log('Finished via WebSocket');
|
||||
if (resolveCompletion) resolveCompletion();
|
||||
} else {
|
||||
const nodeName = nodeNames[nodeId] || `Processing...`;
|
||||
progressLabel.textContent = nodeName;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
async function waitForCompletion(promptId) {
|
||||
return new Promise((resolve) => {
|
||||
const checkResolve = () => {
|
||||
clearInterval(pollInterval);
|
||||
resolve();
|
||||
};
|
||||
resolveCompletion = checkResolve;
|
||||
|
||||
// Fallback polling in case WebSocket is blocked (403)
|
||||
const pollInterval = setInterval(async () => {
|
||||
async function waitForJob(jobId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const poll = setInterval(async () => {
|
||||
try {
|
||||
const resp = await fetch(`/check_status/${promptId}`);
|
||||
const resp = await fetch(`/api/queue/${jobId}/status`);
|
||||
const data = await resp.json();
|
||||
if (data.status === 'finished') {
|
||||
console.log('Finished via Polling');
|
||||
checkResolve();
|
||||
}
|
||||
} catch (err) { console.error('Polling error:', err); }
|
||||
}, 2000);
|
||||
if (data.status === 'done') { clearInterval(poll); resolve(data); }
|
||||
else if (data.status === 'failed' || data.status === 'removed') { clearInterval(poll); reject(new Error(data.error || 'Job failed')); }
|
||||
else if (data.status === 'processing') progressLabel.textContent = 'Generating…';
|
||||
else progressLabel.textContent = 'Queued…';
|
||||
} catch (err) { console.error('Poll error:', err); }
|
||||
}, 1500);
|
||||
});
|
||||
}
|
||||
|
||||
form.addEventListener('submit', async (e) => {
|
||||
const submitter = e.submitter;
|
||||
if (!submitter || submitter.value !== 'preview') return;
|
||||
|
||||
e.preventDefault();
|
||||
|
||||
const formData = new FormData(form);
|
||||
formData.append('action', 'preview');
|
||||
formData.append('client_id', clientId);
|
||||
|
||||
progressContainer.classList.remove('d-none');
|
||||
progressBar.style.width = '0%';
|
||||
progressBar.textContent = '0%';
|
||||
progressLabel.textContent = 'Starting...';
|
||||
|
||||
progressBar.style.width = '100%'; progressBar.textContent = '';
|
||||
progressBar.classList.add('progress-bar-striped', 'progress-bar-animated');
|
||||
progressLabel.textContent = 'Queuing…';
|
||||
try {
|
||||
const response = await fetch(form.getAttribute('action'), {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
headers: { 'X-Requested-With': 'XMLHttpRequest' }
|
||||
method: 'POST', body: formData, headers: { 'X-Requested-With': 'XMLHttpRequest' }
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (data.error) {
|
||||
alert('Error: ' + data.error);
|
||||
progressContainer.classList.add('d-none');
|
||||
return;
|
||||
}
|
||||
|
||||
currentPromptId = data.prompt_id;
|
||||
progressLabel.textContent = 'Queued...';
|
||||
progressBar.style.width = '100%';
|
||||
progressBar.textContent = 'Queued';
|
||||
progressBar.classList.add('progress-bar-striped', 'progress-bar-animated');
|
||||
|
||||
// Wait for completion (WebSocket or Polling)
|
||||
await waitForCompletion(currentPromptId);
|
||||
|
||||
// Finalize
|
||||
finalizeGeneration(currentPromptId);
|
||||
currentPromptId = null;
|
||||
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
alert('Request failed');
|
||||
progressContainer.classList.add('d-none');
|
||||
}
|
||||
});
|
||||
|
||||
async function finalizeGeneration(promptId) {
|
||||
progressLabel.textContent = 'Saving image...';
|
||||
const url = `/scene/{{ scene.slug }}/finalize_generation/${promptId}`;
|
||||
const formData = new FormData();
|
||||
formData.append('action', 'preview');
|
||||
|
||||
try {
|
||||
const response = await fetch(url, { method: 'POST', body: formData });
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
// Update preview image
|
||||
previewImg.src = data.image_url;
|
||||
if (data.error) { alert('Error: ' + data.error); progressContainer.classList.add('d-none'); return; }
|
||||
currentJobId = data.job_id;
|
||||
const jobResult = await waitForJob(currentJobId);
|
||||
currentJobId = null;
|
||||
if (jobResult.result && jobResult.result.image_url) {
|
||||
previewImg.src = jobResult.result.image_url;
|
||||
if (previewCard) previewCard.classList.remove('d-none');
|
||||
|
||||
const replaceBtn = document.getElementById('replace-cover-btn');
|
||||
if (replaceBtn) {
|
||||
replaceBtn.disabled = false;
|
||||
const form = replaceBtn.closest('form');
|
||||
if (form) {
|
||||
form.action = `/scene/{{ scene.slug }}/replace_cover_from_preview`;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
alert('Save failed: ' + data.error);
|
||||
if (replaceBtn) replaceBtn.disabled = false;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
alert('Finalize request failed');
|
||||
} finally {
|
||||
progressContainer.classList.add('d-none');
|
||||
}
|
||||
}
|
||||
} catch (err) { console.error(err); alert('Generation failed: ' + err.message); }
|
||||
finally { progressContainer.classList.add('d-none'); progressBar.classList.remove('progress-bar-striped', 'progress-bar-animated'); }
|
||||
});
|
||||
|
||||
// Batch: Generate All Characters
|
||||
const allCharacters = [
|
||||
{% for char in characters %}{ slug: "{{ char.slug }}", name: {{ char.name | tojson }} },
|
||||
{% endfor %}
|
||||
];
|
||||
const finalizeBaseUrl = '/scene/{{ scene.slug }}/finalize_generation';
|
||||
|
||||
let stopBatch = false;
|
||||
const generateAllBtn = document.getElementById('generate-all-btn');
|
||||
@@ -494,54 +378,37 @@
|
||||
stopAllBtn.classList.remove('d-none');
|
||||
batchProgress.classList.remove('d-none');
|
||||
bootstrap.Tab.getOrCreateInstance(document.getElementById('previews-tab')).show();
|
||||
|
||||
for (let i = 0; i < allCharacters.length; i++) {
|
||||
if (stopBatch) break;
|
||||
const char = allCharacters[i];
|
||||
batchBar.style.width = `${Math.round((i / allCharacters.length) * 100)}%`;
|
||||
batchLabel.textContent = `Generating ${char.name} (${i + 1} / ${allCharacters.length})`;
|
||||
|
||||
const genForm = document.getElementById('generate-form');
|
||||
const fd = new FormData();
|
||||
genForm.querySelectorAll('input[name="include_field"]:checked').forEach(cb => fd.append('include_field', cb.value));
|
||||
fd.append('character_slug', char.slug);
|
||||
fd.append('action', 'preview');
|
||||
fd.append('client_id', clientId);
|
||||
|
||||
try {
|
||||
progressContainer.classList.remove('d-none');
|
||||
progressBar.style.width = '0%';
|
||||
progressBar.textContent = '0%';
|
||||
progressLabel.textContent = `${char.name}: Starting...`;
|
||||
|
||||
progressBar.style.width = '100%'; progressBar.textContent = '';
|
||||
progressBar.classList.add('progress-bar-striped', 'progress-bar-animated');
|
||||
progressLabel.textContent = `${char.name}: Queuing…`;
|
||||
const resp = await fetch(genForm.getAttribute('action'), {
|
||||
method: 'POST', body: fd, headers: { 'X-Requested-With': 'XMLHttpRequest' }
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (data.error) { console.error(`Error for ${char.name}:`, data.error); progressContainer.classList.add('d-none'); continue; }
|
||||
|
||||
currentPromptId = data.prompt_id;
|
||||
await waitForCompletion(currentPromptId);
|
||||
|
||||
progressLabel.textContent = 'Saving image...';
|
||||
const finalFD = new FormData();
|
||||
finalFD.append('action', 'preview');
|
||||
const finalResp = await fetch(`${finalizeBaseUrl}/${currentPromptId}`, { method: 'POST', body: finalFD });
|
||||
const finalData = await finalResp.json();
|
||||
if (finalData.success) {
|
||||
addToPreviewGallery(finalData.image_url, char.name);
|
||||
previewImg.src = finalData.image_url;
|
||||
currentJobId = data.job_id;
|
||||
const jobResult = await waitForJob(currentJobId);
|
||||
currentJobId = null;
|
||||
if (jobResult.result && jobResult.result.image_url) {
|
||||
addToPreviewGallery(jobResult.result.image_url, char.name);
|
||||
previewImg.src = jobResult.result.image_url;
|
||||
if (previewCard) previewCard.classList.remove('d-none');
|
||||
}
|
||||
currentPromptId = null;
|
||||
} catch (err) {
|
||||
console.error(`Failed for ${char.name}:`, err);
|
||||
currentPromptId = null;
|
||||
} finally {
|
||||
progressContainer.classList.add('d-none');
|
||||
}
|
||||
} catch (err) { console.error(`Failed for ${char.name}:`, err); currentJobId = null; }
|
||||
finally { progressContainer.classList.add('d-none'); progressBar.classList.remove('progress-bar-striped', 'progress-bar-animated'); }
|
||||
}
|
||||
|
||||
batchBar.style.width = '100%';
|
||||
batchLabel.textContent = stopBatch ? 'Stopped.' : 'Complete!';
|
||||
generateAllBtn.disabled = false;
|
||||
|
||||
@@ -102,71 +102,18 @@
|
||||
const sceneNameText = document.getElementById('current-scene-name');
|
||||
const stepProgressText = document.getElementById('current-step-progress');
|
||||
|
||||
const clientId = 'scenes_batch_' + Math.random().toString(36).substring(2, 15);
|
||||
const socket = new WebSocket('{{ COMFYUI_WS_URL }}?clientId=' + clientId);
|
||||
|
||||
const nodeNames = {
|
||||
"3": "Sampling",
|
||||
"11": "Face Detailing",
|
||||
"13": "Hand Detailing",
|
||||
"4": "Loading Models",
|
||||
"16": "Character LoRA",
|
||||
"17": "Outfit LoRA",
|
||||
"18": "Action LoRA",
|
||||
"19": "Style/Detailer LoRA",
|
||||
"8": "Decoding",
|
||||
"9": "Saving"
|
||||
};
|
||||
|
||||
let currentPromptId = null;
|
||||
let resolveGeneration = null;
|
||||
|
||||
socket.addEventListener('message', (event) => {
|
||||
const msg = JSON.parse(event.data);
|
||||
|
||||
if (msg.type === 'progress') {
|
||||
if (msg.data.prompt_id !== currentPromptId) return;
|
||||
const value = msg.data.value;
|
||||
const max = msg.data.max;
|
||||
const percent = Math.round((value / max) * 100);
|
||||
stepProgressText.textContent = `${percent}%`;
|
||||
taskProgressBar.style.width = `${percent}%`;
|
||||
taskProgressBar.textContent = `${percent}%`;
|
||||
taskProgressBar.classList.remove('progress-bar-striped', 'progress-bar-animated');
|
||||
}
|
||||
else if (msg.type === 'executing') {
|
||||
if (msg.data.prompt_id !== currentPromptId) return;
|
||||
const nodeId = msg.data.node;
|
||||
if (nodeId === null) {
|
||||
if (resolveGeneration) resolveGeneration();
|
||||
} else {
|
||||
nodeStatus.textContent = nodeNames[nodeId] || `Processing...`;
|
||||
stepProgressText.textContent = "";
|
||||
if (nodeId !== "3") {
|
||||
taskProgressBar.style.width = '100%';
|
||||
taskProgressBar.textContent = nodeNames[nodeId] || 'Processing...';
|
||||
taskProgressBar.classList.add('progress-bar-striped', 'progress-bar-animated');
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
async function waitForCompletion(promptId) {
|
||||
return new Promise((resolve) => {
|
||||
const checkResolve = () => {
|
||||
clearInterval(pollInterval);
|
||||
resolve();
|
||||
};
|
||||
resolveGeneration = checkResolve;
|
||||
const pollInterval = setInterval(async () => {
|
||||
async function waitForJob(jobId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const poll = setInterval(async () => {
|
||||
try {
|
||||
const resp = await fetch(`/check_status/${promptId}`);
|
||||
const resp = await fetch(`/api/queue/${jobId}/status`);
|
||||
const data = await resp.json();
|
||||
if (data.status === 'finished') {
|
||||
checkResolve();
|
||||
}
|
||||
if (data.status === 'done') { clearInterval(poll); resolve(data); }
|
||||
else if (data.status === 'failed' || data.status === 'removed') { clearInterval(poll); reject(new Error(data.error || 'Job failed')); }
|
||||
else if (data.status === 'processing') nodeStatus.textContent = 'Generating…';
|
||||
else nodeStatus.textContent = 'Queued…';
|
||||
} catch (err) {}
|
||||
}, 2000);
|
||||
}, 1500);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -183,66 +130,65 @@
|
||||
batchBtn.disabled = true;
|
||||
regenAllBtn.disabled = true;
|
||||
container.classList.remove('d-none');
|
||||
|
||||
let completed = 0;
|
||||
for (const scene of missing) {
|
||||
const percent = Math.round((completed / missing.length) * 100);
|
||||
progressBar.style.width = `${percent}%`;
|
||||
progressBar.textContent = `${percent}%`;
|
||||
statusText.textContent = `Batch Generating Scenes: ${completed + 1} / ${missing.length}`;
|
||||
sceneNameText.textContent = `Current: ${scene.name}`;
|
||||
nodeStatus.textContent = "Queuing...";
|
||||
|
||||
taskProgressBar.style.width = '100%';
|
||||
taskProgressBar.textContent = 'Queued';
|
||||
taskProgressBar.classList.add('progress-bar-striped', 'progress-bar-animated');
|
||||
|
||||
// Phase 1: Queue all jobs upfront
|
||||
progressBar.style.width = '100%';
|
||||
progressBar.textContent = '';
|
||||
progressBar.classList.add('progress-bar-striped', 'progress-bar-animated');
|
||||
nodeStatus.textContent = 'Queuing…';
|
||||
|
||||
const jobs = [];
|
||||
for (const scene of missing) {
|
||||
statusText.textContent = `Queuing ${jobs.length + 1} / ${missing.length}…`;
|
||||
try {
|
||||
const genResp = await fetch(`/scene/${scene.slug}/generate`, {
|
||||
method: 'POST',
|
||||
body: new URLSearchParams({
|
||||
'action': 'replace',
|
||||
'client_id': clientId,
|
||||
'character_slug': '__random__'
|
||||
}),
|
||||
body: new URLSearchParams({ action: 'replace', character_slug: '__random__' }),
|
||||
headers: { 'X-Requested-With': 'XMLHttpRequest' }
|
||||
});
|
||||
const genData = await genResp.json();
|
||||
currentPromptId = genData.prompt_id;
|
||||
if (genData.job_id) jobs.push({ item: scene, jobId: genData.job_id });
|
||||
} catch (err) {
|
||||
console.error(`Failed to queue ${scene.name}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
await waitForCompletion(currentPromptId);
|
||||
// Phase 2: Poll all concurrently
|
||||
progressBar.classList.remove('progress-bar-striped', 'progress-bar-animated');
|
||||
progressBar.style.width = '0%';
|
||||
progressBar.textContent = '0%';
|
||||
statusText.textContent = `0 / ${jobs.length} done`;
|
||||
|
||||
const finResp = await fetch(`/scene/${scene.slug}/finalize_generation/${currentPromptId}`, {
|
||||
method: 'POST',
|
||||
body: new URLSearchParams({ 'action': 'replace' })
|
||||
});
|
||||
const finData = await finResp.json();
|
||||
|
||||
if (finData.success) {
|
||||
const img = document.getElementById(`img-${scene.slug}`);
|
||||
const noImgSpan = document.getElementById(`no-img-${scene.slug}`);
|
||||
if (img) {
|
||||
img.src = finData.image_url;
|
||||
img.classList.remove('d-none');
|
||||
}
|
||||
let completed = 0;
|
||||
await Promise.all(jobs.map(async ({ item, jobId }) => {
|
||||
try {
|
||||
const jobResult = await waitForJob(jobId);
|
||||
if (jobResult.result && jobResult.result.image_url) {
|
||||
const img = document.getElementById(`img-${item.slug}`);
|
||||
const noImgSpan = document.getElementById(`no-img-${item.slug}`);
|
||||
if (img) { img.src = jobResult.result.image_url; img.classList.remove('d-none'); }
|
||||
if (noImgSpan) noImgSpan.classList.add('d-none');
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Failed for ${scene.name}:`, err);
|
||||
console.error(`Failed for ${item.name}:`, err);
|
||||
}
|
||||
completed++;
|
||||
}
|
||||
const pct = Math.round((completed / jobs.length) * 100);
|
||||
progressBar.style.width = `${pct}%`;
|
||||
progressBar.textContent = `${pct}%`;
|
||||
statusText.textContent = `${completed} / ${jobs.length} done`;
|
||||
}));
|
||||
|
||||
progressBar.style.width = '100%';
|
||||
progressBar.textContent = '100%';
|
||||
statusText.textContent = "Batch Scene Generation Complete!";
|
||||
sceneNameText.textContent = "";
|
||||
nodeStatus.textContent = "Done";
|
||||
stepProgressText.textContent = "";
|
||||
statusText.textContent = 'Batch Scene Generation Complete!';
|
||||
sceneNameText.textContent = '';
|
||||
nodeStatus.textContent = 'Done';
|
||||
stepProgressText.textContent = '';
|
||||
taskProgressBar.style.width = '0%';
|
||||
taskProgressBar.textContent = '';
|
||||
batchBtn.disabled = false;
|
||||
setTimeout(() => { container.classList.add('d-none'); }, 5000);
|
||||
setTimeout(() => container.classList.add('d-none'), 5000);
|
||||
}
|
||||
|
||||
batchBtn.addEventListener('click', async () => {
|
||||
|
||||
Reference in New Issue
Block a user