web: parallel queue item processing

web/src/lib/state/task-manager/queue.ts

@@ -86,13 +86,12 @@ export function pipelineTaskDone(id: string, workerId: string, file: File) {
     schedule();
 }
 
-export function itemRunning(id: string, workerId: string) {
+export function itemRunning(id: string) {
     update(queueData => {
         const data = queueData[id] as CobaltQueueItemRunning;
 
         if (data) {
             data.state = 'running';
-            data.runningWorker = workerId;
             data.completedWorkers ??= new Set();
             data.pipelineResults ??= [];
         }

web/src/lib/task-manager/queue.ts

@@ -4,9 +4,11 @@ import { ffmpegMetadataArgs } from "$lib/util";
 import { createDialog } from "$lib/state/dialogs";
 import { addItem } from "$lib/state/task-manager/queue";
 import { openQueuePopover } from "$lib/state/queue-visibility";
+import { currentTasks } from "$lib/state/task-manager/current-tasks";
 
 import type { CobaltPipelineItem, CobaltPipelineResultFileType } from "$lib/types/workers";
 import type { CobaltLocalProcessingResponse, CobaltSaveRequestBody } from "$lib/types/api";
+import type { CobaltQueueItem } from "$lib/types/queue";
 
 export const getMediaType = (type: string) => {
     const kind = type.split('/')[0];
@@ -177,6 +179,7 @@ export const createSavePipeline = (info: CobaltLocalProcessingResponse, request:
         worker: workerType,
         workerId: crypto.randomUUID(),
         parentId,
+        dependsOn: pipeline.map(w => w.workerId),
         workerArgs: {
             files: [],
             ffargs,
@@ -200,3 +203,24 @@ export const createSavePipeline = (info: CobaltLocalProcessingResponse, request:
 
     openQueuePopover();
 }
+
+export const getProgress = (item: CobaltQueueItem): number => {
+    if (item.state === 'done' || item.state === 'error') {
+        return 1;
+    } else if (item.state === 'waiting') {
+        return 0;
+    }
+
+    const runningTasks = get(currentTasks);
+    let sum = 0;
+    for (const worker of item.pipeline) {
+        if (item.completedWorkers.has(worker.workerId)) {
+            sum += 1;
+        } else {
+            const task = runningTasks[worker.workerId];
+            sum += (task?.progress?.percentage || 0) / 100;
+        }
+    }
+
+    return sum / item.pipeline.length;
+}

web/src/lib/task-manager/scheduler.ts

@@ -11,11 +11,7 @@ const startPipeline = (pipelineItem: CobaltPipelineItem) => {
         parentId: pipelineItem.parentId,
     });
 
-    itemRunning(
-        pipelineItem.parentId,
-        pipelineItem.workerId,
-    );
-
+    itemRunning(pipelineItem.parentId);
     startWorker(pipelineItem);
 }
 
@@ -23,18 +19,9 @@ export const schedule = () => {
     const queueItems = get(queue);
     const ongoingTasks = get(currentTasks);
 
-    // TODO (?): task concurrency
-    if (Object.keys(ongoingTasks).length > 0) {
-        return;
-    }
-
     for (const task of Object.values(queueItems)) {
         if (task.state === "running") {
-            // if the running worker isn't completed, wait
-            // to be called again on worker completion
-            if (!task.completedWorkers.has(task.runningWorker)) {
-                break;
-            }
+            const finalWorker = task.pipeline[task.pipeline.length - 1];
 
             // if all workers are completed, then return the
             // the final file and go to the next task
@@ -44,7 +31,7 @@ export const schedule = () => {
                 if (finalFile) {
                     itemDone(task.id, finalFile);
                 } else {
-                    itemError(task.id, task.runningWorker, "queue.no_final_file");
+                    itemError(task.id, finalWorker.workerId, "queue.no_final_file");
                 }
 
                 continue;
@@ -53,10 +40,16 @@ export const schedule = () => {
             // if current worker is completed, but there are more workers,
             // then start the next one and wait to be called again
             for (const worker of task.pipeline) {
-                if (!task.completedWorkers.has(worker.workerId)) {
-                    startPipeline(worker);
+                if (task.completedWorkers.has(worker.workerId) || ongoingTasks[worker.workerId]) {
+                    continue;
+                }
+
+                const needsToWait = worker.dependsOn?.some(id => !task.completedWorkers.has(id));
+                if (needsToWait) {
                     break;
                 }
+
+                startPipeline(worker);
             }
 
             // break because we don't want to start next tasks before this one is done

web/src/lib/types/queue.ts

@@ -1,11 +1,8 @@
 import type { CobaltSaveRequestBody } from "$lib/types/api";
 import type { CobaltPipelineItem, CobaltPipelineResultFileType } from "$lib/types/workers";
 
-export type CobaltQueueItemState = "waiting" | "running" | "done" | "error";
-
-export type CobaltQueueBaseItem = {
+type CobaltQueueBaseItem = {
     id: string,
-    state: CobaltQueueItemState,
     pipeline: CobaltPipelineItem[],
     canRetry?: boolean,
     originalRequest?: CobaltSaveRequestBody,
@@ -14,28 +11,30 @@ export type CobaltQueueBaseItem = {
     mediaType: CobaltPipelineResultFileType,
 };
 
-export type CobaltQueueItemWaiting = CobaltQueueBaseItem & {
+type CobaltQueueItemWaiting = CobaltQueueBaseItem & {
     state: "waiting",
 };
 
 export type CobaltQueueItemRunning = CobaltQueueBaseItem & {
     state: "running",
-    runningWorker: string,
     completedWorkers: Set<string>,
     pipelineResults: File[],
 };
 
-export type CobaltQueueItemDone = CobaltQueueBaseItem & {
+type CobaltQueueItemDone = CobaltQueueBaseItem & {
     state: "done",
     resultFile: File,
 };
 
-export type CobaltQueueItemError = CobaltQueueBaseItem & {
+type CobaltQueueItemError = CobaltQueueBaseItem & {
     state: "error",
     errorCode: string,
 };
 
-export type CobaltQueueItem = CobaltQueueItemWaiting | CobaltQueueItemRunning | CobaltQueueItemDone | CobaltQueueItemError;
+export type CobaltQueueItem = CobaltQueueItemWaiting
+                            | CobaltQueueItemRunning
+                            | CobaltQueueItemDone
+                            | CobaltQueueItemError;
 
 export type CobaltQueue = {
     [id: string]: CobaltQueueItem,

web/src/lib/types/workers.ts

@@ -8,17 +8,18 @@ export type CobaltWorkerProgress = {
     percentage?: number,
     speed?: number,
     size: number,
-}
+};
 
 type CobaltFFmpegWorkerArgs = {
     files: File[],
     ffargs: string[],
     output: FileInfo,
-}
+};
 
 type CobaltPipelineItemBase = {
     workerId: string,
     parentId: string,
+    dependsOn?: string[],
 };
 
 type CobaltRemuxPipelineItem = CobaltPipelineItemBase & {