admin/cobalt
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

web: parallel queue item processing

Browse Source
This commit is contained in:
jj
2025-05-16 16:42:40 +00:00
parent 426c073d5f
commit 398681857b
8 changed files with 112 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
web/src/components/queue/ProcessingQueue.svelte
View File

@@ -8,6 +8,7 @@
import { queueVisible } from "$lib/state/queue-visibility";
import { currentTasks } from "$lib/state/task-manager/current-tasks";
import { clearQueue, queue as readableQueue } from "$lib/state/task-manager/queue";
import { getProgress } from "$lib/task-manager/queue";
import SectionHeading from "$components/misc/SectionHeading.svelte";
import PopoverContainer from "$components/misc/PopoverContainer.svelte";
@@ -21,23 +22,10 @@
$queueVisible = !$queueVisible;
};
const totalItemProgress = (completed: number, current: number, total: number) => {
return (completed * 100 + current) / total
}
$: queue = Object.entries($readableQueue);
$: totalProgress = queue.length ? queue.map(([, item]) => {
if (item.state === "done" || item.state === "error") {
return 100;
} else if (item.state === "running") {
return totalItemProgress(
item.completedWorkers.size,
$currentTasks[item.runningWorker]?.progress?.percentage || 0,
item.pipeline.length || 0
);
}
return 0;
return getProgress(item) * 100;
}).reduce((a, b) => a + b) / (100 * queue.length) : 0;
$: indeterminate = queue.length > 0 && totalProgress === 0;
@@ -93,16 +81,7 @@
<div id="processing-list" role="list" aria-labelledby="queue-title">
{#each queue as [id, item]}
<ProcessingQueueItem
{id}
info={item}
runningWorker={
item.state === "running" ? $currentTasks[item.runningWorker] : undefined
}
runningWorkerId={
item.state === "running" ? item.runningWorker : undefined
}
/>
<ProcessingQueueItem {id} info={item} />
{/each}
{#if queue.length === 0}
<ProcessingQueueStub />

94
web/src/components/queue/ProcessingQueueItem.svelte
View File

@@ -4,10 +4,11 @@
import { downloadFile } from "$lib/download";
import { removeItem } from "$lib/state/task-manager/queue";
import { savingHandler } from "$lib/api/saving-handler";
import { getProgress } from "$lib/task-manager/queue";
import { currentTasks } from "$lib/state/task-manager/current-tasks";
import type { CobaltQueueItem } from "$lib/types/queue";
import type { CobaltWorkerProgress } from "$lib/types/workers";
import type { CobaltCurrentTaskItem } from "$lib/types/task-manager";
import type { CobaltCurrentTasks } from "$lib/types/task-manager";
import ProgressBar from "$components/queue/ProgressBar.svelte";
@@ -30,8 +31,6 @@
export let id: string;
export let info: CobaltQueueItem;
export let runningWorker: CobaltCurrentTaskItem | undefined;
export let runningWorkerId: string | undefined;
let retrying = false;
@@ -52,41 +51,51 @@
}),
});
$: progress = runningWorker?.progress;
type StatusText = {
info: CobaltQueueItem;
runningWorker: CobaltCurrentTaskItem | undefined;
progress: CobaltWorkerProgress | undefined;
currentTasks: CobaltCurrentTasks;
retrying: boolean;
};
const generateStatusText = ({ info, runningWorker, progress, retrying }: StatusText) => {
const generateStatusText = ({ info, retrying, currentTasks }: StatusText) => {
switch (info.state) {
case "running":
if (runningWorker) {
const running = $t(`queue.state.running.${runningWorker.type}`);
const formattedSize = formatFileSize(progress?.size);
const progress = getProgress(info);
if (progress && progress.percentage) {
return `${running}: ${Math.floor(progress.percentage)}%, ${formattedSize}`;
}
else if (runningWorker && progress) {
if (progress.size > 0) {
return `${running}: ${formattedSize}`;
}
return running;
}
else if (runningWorker?.type) {
const starting = $t(`queue.state.starting.${runningWorker.type}`);
const runningWorkers = info.pipeline.filter(w => w.workerId in currentTasks);
const running = [...new Set(runningWorkers.map(task => $t(`queue.state.running.${task.worker}`)))].join(', ');
const progresses = runningWorkers.map(w => currentTasks[w.workerId])
.map(t => t.progress)
.filter(p => p);
if (info.pipeline.length > 1) {
const currentPipeline = info.completedWorkers.size + 1;
return `${starting} (${currentPipeline}/${info.pipeline.length})`;
}
return starting;
}
const totalSize = progresses.reduce((s, p) => s + (p?.size ?? 0), 0);
if (runningWorkers.length && totalSize > 0) {
const formattedSize = formatFileSize(totalSize);
return `${running}: ${Math.floor(progress * 100)}%, ${formattedSize}`;
}
const firstUnstarted = info.pipeline.find(w => {
if (info.completedWorkers.has(w.workerId))
return false;
const task = currentTasks[w.workerId];
if (!task || !task.progress?.percentage) {
return true;
}
});
if (firstUnstarted) {
const starting = $t(`queue.state.starting.${firstUnstarted.worker}`);
if (info.pipeline.length > 1) {
const currentPipeline = info.completedWorkers.size + 1;
return `${starting} (${currentPipeline}/${info.pipeline.length})`;
}
return starting;
}
return $t("queue.state.starting");
case "done":
@@ -100,6 +109,23 @@
}
};
const getWorkerProgress = (item: CobaltQueueItem, workerId: string): number | undefined => {
if (item.state === 'running' && item.completedWorkers.has(workerId)) {
return 100;
}
const workerIndex = item.pipeline.findIndex(w => w.workerId === workerId);
if (workerIndex === -1) {
return;
}
const worker = item.pipeline[workerIndex];
const task = $currentTasks[worker.workerId];
if (task?.progress) {
return task.progress.percentage;
}
}
/*
params are passed here because svelte will re-run
the function every time either of them is changed,
@@ -107,9 +133,8 @@
*/
$: statusText = generateStatusText({
info,
runningWorker,
progress,
retrying,
currentTasks: $currentTasks
});
</script>
@@ -126,11 +151,10 @@
{#if info.state === "running"}
<div class="progress-holder">
{#each info.pipeline as pipeline}
{#each info.pipeline as task}
<ProgressBar
percentage={progress?.percentage}
workerId={pipeline.workerId}
{runningWorkerId}
percentage={getWorkerProgress(info, task.workerId) || 0}
workerId={task.workerId}
completedWorkers={info.completedWorkers}
/>
{/each}

7
web/src/components/queue/ProgressBar.svelte
View File

@@ -3,22 +3,21 @@
export let percentage: number = 0;
export let workerId: string;
export let runningWorkerId: string | undefined;
export let completedWorkers: Set<string>;
</script>
<div class="file-progress">
{#if percentage && workerId === runningWorkerId}
{#if percentage}
<div
class="progress"
style="width: {Math.min(100, percentage || 0)}%"
style="width: {Math.min(100, percentage)}%"
></div>
{:else if completedWorkers.has(workerId)}
<div
class="progress"
style="width: 100%"
></div>
{:else if workerId === runningWorkerId}
{:else}
<Skeleton
height="6px"
width="100%"