web: very early proof-of-concept of on-device image background removal

web/i18n/en/settings.json

@@ -113,9 +113,9 @@
 
     "advanced.data": "data management",
 
-    "advanced.duck": "local processing",
-    "advanced.duck.title": "process everything on device",
-    "advanced.duck.description": "very wip, may cause critical issues or not work at all. this toggle will probably be gone by release.",
+    "advanced.duck": "duck",
+    "advanced.duck.title": "enable new on-device features",
+    "advanced.duck.description": "very wip, WILL cause critical issues or not work at all. this toggle will be gone by release.",
 
     "processing.community": "community instances",
     "processing.enable_custom.title": "use a custom processing server",

web/i18n/en/tabs.json

@@ -4,5 +4,6 @@
     "updates": "updates",
     "donate": "donate",
     "about": "about",
-    "remux": "remux"
+    "remux": "remux",
+    "cutout": "cut out"
 }

web/package.json

@@ -28,6 +28,7 @@
         "@fontsource-variable/noto-sans-mono": "^5.0.20",
         "@fontsource/ibm-plex-mono": "^5.0.13",
         "@fontsource/redaction-10": "^5.0.2",
+        "@huggingface/transformers": "^3.2.4",
         "@imput/libav.js-remux-cli": "^5.5.6",
         "@imput/version-info": "workspace:^",
         "@sveltejs/adapter-static": "^3.0.6",

web/src/components/sidebar/Sidebar.svelte

@@ -1,4 +1,7 @@
 <script lang="ts">
+    import settings from "$lib/state/settings";
+
+    import { device } from "$lib/device";
     import { t } from "$lib/i18n/translations";
     import { defaultNavPage } from "$lib/subnav";
 
@@ -14,6 +17,8 @@
     import IconHeart from "@tabler/icons-svelte/IconHeart.svelte";
     import IconInfoCircle from "@tabler/icons-svelte/IconInfoCircle.svelte";
 
+    import IconCut from "@tabler/icons-svelte/IconCut.svelte";
+
     let screenWidth: number;
     let settingsLink = defaultNavPage("settings");
     let aboutLink = defaultNavPage("about");
@@ -30,6 +35,9 @@
     <div id="sidebar-tabs" role="tablist">
         <div id="sidebar-actions" class="sidebar-inner-container">
             <SidebarTab name="save" path="/" icon={IconDownload} />
+            {#if $settings.advanced.duck && !device.is.mobile && $settings.advanced.debug}
+                <SidebarTab name="cutout" path="/cutout" icon={IconCut} beta />
+            {/if}
             <SidebarTab name="remux" path="/remux" icon={IconRepeat} beta />
         </div>
         <div id="sidebar-info" class="sidebar-inner-container">

web/src/components/sidebar/SidebarTab.svelte

@@ -9,7 +9,7 @@
 
     export let beta = false;
 
-    const firstTabPage = ["save", "remux", "settings"];
+    const firstTabPage = ["save", "cutout", "remux", "settings"];
 
     let tab: HTMLElement;
 

web/src/lib/workers/removebg.ts

@@ -0,0 +1,56 @@
+import { AutoModel, AutoProcessor, RawImage } from "@huggingface/transformers";
+
+const models = {
+    light: {
+        id: "briaai/RMBG-1.4",
+        input: "input",
+        modelConfig: {
+            device: "wasm",
+            dtype: "fp32",
+        },
+        processorConfig: {},
+    },
+    heavy: {
+        id: "onnx-community/BiRefNet_lite",
+        input: "input_image",
+        modelConfig: {
+            device: "webgpu",
+            dtype: "fp16",
+        },
+        processorConfig: {},
+    }
+}
+
+export const removeImageBackground = async (file: File) => {
+    const image = await RawImage.fromBlob(new Blob([file]));
+
+    const model_type = "light";
+    const model = await AutoModel.from_pretrained(models[model_type].id, models[model_type].modelConfig);
+
+    const processor = await AutoProcessor.from_pretrained(models[model_type].id, models[model_type].processorConfig);
+
+    if (model && processor) {
+        const { pixel_values } = await processor(image);
+
+        const { output } = await model({ [models[model_type].input]: pixel_values });
+
+        const mask = await RawImage.fromTensor(output[0].mul(255).to('uint8')).resize(image.width, image.height);
+
+        const canvas = document.createElement('canvas');
+        canvas.width = image.width;
+        canvas.height = image.height;
+        const ctx = canvas.getContext('2d');
+
+        if (!ctx) return;
+
+        ctx.drawImage(image.toCanvas(), 0, 0);
+
+        const pixelData = ctx.getImageData(0, 0, image.width, image.height);
+        for (let i = 0; i < mask.data.length; ++i) {
+            pixelData.data[4 * i + 3] = mask.data[i];
+        }
+        ctx.putImageData(pixelData, 0, 0);
+
+        return canvas;
+    }
+}

web/src/routes/cutout/+page.svelte

@@ -0,0 +1,144 @@
+<script lang="ts">
+    import settings from "$lib/state/settings";
+
+    import { onMount } from "svelte";
+    import { goto } from "$app/navigation";
+
+    import { downloadFile } from "$lib/download";
+    import { removeImageBackground } from "$lib/workers/removebg";
+
+    import DropReceiver from "$components/misc/DropReceiver.svelte";
+    import FileReceiver from "$components/misc/FileReceiver.svelte";
+    import Skeleton from "$components/misc/Skeleton.svelte";
+
+    let imageContainer: HTMLElement;
+
+    let draggedOver = false;
+    let file: File | undefined;
+
+    let thinking = false;
+    let done = false;
+
+    let result: HTMLCanvasElement;
+
+    const processImage = async () => {
+        if (file) {
+            thinking = true;
+            const removedBackground = await removeImageBackground(file);
+            if (removedBackground) {
+                thinking = false;
+                done = true;
+                result = removedBackground;
+                imageContainer.append(removedBackground);
+            }
+        }
+    };
+
+    const exportImage = async () => {
+        result.toBlob(async (blob) => {
+            if (!blob || !file) return;
+            return await downloadFile({
+                file: new File([blob], `${file.name} (cutout).png`, {
+                    type: "image/png",
+                }),
+            });
+        }, "image/png");
+    };
+
+    onMount(() => {
+        if (!($settings.advanced.duck && $settings.advanced.debug)) {
+            goto("/", { replaceState: true });
+        }
+    });
+</script>
+
+<DropReceiver bind:file bind:draggedOver id="cutout-container">
+    {#if !thinking && !done}
+        <FileReceiver
+            bind:draggedOver
+            bind:file
+            acceptTypes={["image/*"]}
+            acceptExtensions={["jpg", "png", "webp"]}
+        />
+        <div class="subtext early-note">
+            this is a very early & basic proof-of-concept, nothing about this feature is final or complete. please don't share or talk about it.
+        </div>
+        {#if file}
+            <button on:click={processImage}>process imported stuff</button>
+        {/if}
+    {/if}
+
+    {#if thinking}
+        <div>thinking very hard rn...</div>
+    {/if}
+
+    {#if done}
+        <div>thought a lot, here's what i got:</div>
+    {/if}
+
+    {#if thinking || done}
+        <div id="image-preview" bind:this={imageContainer}>
+            {#if !done}
+                <Skeleton width="100%" height="100%" class="big" />
+            {/if}
+        </div>
+    {/if}
+
+    {#if done}
+        <div id="finished-actions">
+            <button
+                on:click={() => {
+                    done = false;
+                    file = undefined;
+                }}
+            >
+                start over
+            </button>
+            <button on:click={exportImage}>save the result</button>
+        </div>
+    {/if}
+</DropReceiver>
+
+<style>
+    :global(#cutout-container) {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        flex-direction: column;
+        width: 100%;
+        gap: 18px;
+    }
+
+    .early-note {
+        max-width: 400px;
+        text-align: center;
+    }
+
+    #image-preview {
+        background: var(--button);
+        display: flex;
+        flex-direction: column;
+        justify-content: center;
+        align-items: center;
+        width: 720px;
+        height: 480px;
+        overflow: hidden;
+        border-radius: var(--border-radius);
+        box-shadow: var(--button-box-shadow);
+    }
+
+    #finished-actions {
+        display: flex;
+        flex-direction: row;
+        gap: 6px;
+    }
+
+    button {
+        padding: 12px 24px;
+        border-radius: 200px;
+    }
+
+    :global(canvas) {
+        height: 100%;
+    }
+</style>