api: flatten code directories, better filenames, remove old files

api/src/processing/cookie/cookie.js

@@ -0,0 +1,37 @@
+import { strict as assert } from 'node:assert';
+
+export default class Cookie {
+    constructor(input) {
+        assert(typeof input === 'object');
+        this._values = {};
+        this.set(input)
+    }
+    set(values) {
+        Object.entries(values).forEach(
+            ([ key, value ]) => this._values[key] = value
+        )
+    }
+    unset(keys) {
+        for (const key of keys) delete this._values[key]
+    }
+    static fromString(str) {
+        const obj = {};
+
+        str.split('; ').forEach(cookie => {
+            const key = cookie.split('=')[0];
+            const value = cookie.split('=').splice(1).join('=');
+            obj[key] = value
+        })
+
+        return new Cookie(obj)
+    }
+    toString() {
+        return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ')
+    }
+    toJSON() {
+        return this.toString()
+    }
+    values() {
+        return Object.freeze({ ...this._values })
+    }
+}

api/src/processing/cookie/manager.js

@@ -0,0 +1,66 @@
+import Cookie from './cookie.js';
+import { readFile, writeFile } from 'fs/promises';
+import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
+import { env } from '../../config.js';
+
+const WRITE_INTERVAL = 60000,
+      cookiePath = env.cookiePath,
+      COUNTER = Symbol('counter');
+
+let cookies = {}, dirty = false, intervalId;
+
+const setup = async () => {
+    try {
+        if (!cookiePath) return;
+
+        cookies = await readFile(cookiePath, 'utf8');
+        cookies = JSON.parse(cookies);
+        intervalId = setInterval(writeChanges, WRITE_INTERVAL)
+    } catch { /* no cookies for you */ }
+}
+
+setup();
+
+function writeChanges() {
+    if (!dirty) return;
+    dirty = false;
+
+    writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => {
+        clearInterval(intervalId)
+    })
+}
+
+export function getCookie(service) {
+    if (!cookies[service] || !cookies[service].length) return;
+
+    let n;
+    if (cookies[service][COUNTER] === undefined) {
+        n = cookies[service][COUNTER] = 0
+    } else {
+        ++cookies[service][COUNTER]
+        n = (cookies[service][COUNTER] %= cookies[service].length)
+    }
+
+    const cookie = cookies[service][n];
+    if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie);
+
+    return cookies[service][n]
+}
+
+export function updateCookie(cookie, headers) {
+    if (!cookie) return;
+
+    const parsed = parseSetCookie(
+        splitCookiesString(headers.get('set-cookie')),
+        { decodeValues: false }
+    ), values = {}
+
+    cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name));
+    parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value);
+    updateCookieValues(cookie, values);
+}
+
+export function updateCookieValues(cookie, values) {
+    cookie.set(values);
+    if (Object.keys(values).length) dirty = true
+}

api/src/processing/create-filename.js

@@ -0,0 +1,78 @@
+export default function(f, template, isAudioOnly, isAudioMuted) {
+    let filename = '';
+
+    switch(template) {
+        default:
+        case "classic":
+            // youtube_MMK3L4W70g4_1920x1080_h264_mute.mp4
+            // youtube_MMK3L4W70g4_audio.mp3
+            filename += `${f.service}_${f.id}`;
+            if (!isAudioOnly) {
+                if (f.resolution) filename += `_${f.resolution}`;
+                if (f.youtubeFormat) filename += `_${f.youtubeFormat}`;
+                if (!isAudioMuted && f.youtubeDubName) filename += `_${f.youtubeDubName}`;
+                if (isAudioMuted) filename += '_mute';
+                filename += `.${f.extension}`
+            } else {
+                filename += `_audio`;
+                if (f.youtubeDubName) filename += `_${f.youtubeDubName}`;
+            }
+            break;
+        case "pretty":
+            // Loossemble (루셈블) - 'Sensitive' MV (1080p, h264, mute, youtube).mp4
+            // How secure is 256 bit security? - 3Blue1Brown (es, youtube).mp3
+            filename += `${f.title} `;
+            if (!isAudioOnly) {
+                filename += '('
+                if (f.qualityLabel) filename += `${f.qualityLabel}, `;
+                if (f.youtubeFormat) filename += `${f.youtubeFormat}, `;
+                if (!isAudioMuted && f.youtubeDubName) filename += `${f.youtubeDubName}, `;
+                if (isAudioMuted) filename += 'mute, ';
+                filename += `${f.service}`;
+                filename += ')';
+                filename += `.${f.extension}`
+            } else {
+                filename += `- ${f.author} (`;
+                if (f.youtubeDubName) filename += `${f.youtubeDubName}, `;
+                filename += `${f.service})`
+            }
+            break;
+        case "basic":
+            // Loossemble (루셈블) - 'Sensitive' MV (1080p, h264, ru).mp4
+            // How secure is 256 bit security? - 3Blue1Brown (es).mp3
+            filename += `${f.title} `;
+            if (!isAudioOnly) {
+                filename += '('
+                if (f.qualityLabel) filename += `${f.qualityLabel}, `;
+                if (f.youtubeFormat) filename += `${f.youtubeFormat}`;
+                if (!isAudioMuted && f.youtubeDubName) filename += `, ${f.youtubeDubName}`;
+                if (isAudioMuted) filename += ', mute';
+                filename += ')';
+                filename += `.${f.extension}`
+            } else {
+                filename += `- ${f.author}`;
+                if (f.youtubeDubName) filename += ` (${f.youtubeDubName})`;
+            }
+            break;
+        case "nerdy":
+            // Loossemble (루셈블) - 'Sensitive' MV (1080p, h264, ru, youtube, MMK3L4W70g4).mp4
+            // Loossemble (루셈블) - 'Sensitive' MV - Loossemble (ru, youtube, MMK3L4W70g4).mp4
+            filename += `${f.title} `;
+            if (!isAudioOnly) {
+                filename += '('
+                if (f.qualityLabel) filename += `${f.qualityLabel}, `;
+                if (f.youtubeFormat) filename += `${f.youtubeFormat}, `;
+                if (!isAudioMuted && f.youtubeDubName) filename += `${f.youtubeDubName}, `;
+                if (isAudioMuted) filename += 'mute, ';
+                filename += `${f.service}, ${f.id}`;
+                filename += ')'
+                filename += `.${f.extension}`
+            } else {
+                filename += `- ${f.author} (`;
+                if (f.youtubeDubName) filename += `${f.youtubeDubName}, `;
+                filename += `${f.service}, ${f.id})`
+            }
+            break;
+    }
+    return filename.replace(' ,', '').replace(', )', ')').replace(',)', ')')
+}

api/src/processing/match-action.js

@@ -0,0 +1,204 @@
+import { audioIgnore, services, supportedAudio } from "../config.js";
+import { createResponse } from "./request.js";
+import createFilename from "./create-filename.js";
+import { createStream } from "../stream/manage.js";
+
+export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif, requestIP) {
+    let action,
+        responseType = "stream",
+        defaultParams = {
+            u: r.urls,
+            headers: r.headers,
+            service: host,
+            filename: r.filenameAttributes ?
+                    createFilename(r.filenameAttributes, filenamePattern, isAudioOnly, isAudioMuted) : r.filename,
+            fileMetadata: !disableMetadata ? r.fileMetadata : false,
+            requestIP
+        },
+        params = {},
+        audioFormat = String(userFormat);
+
+    if (r.isPhoto) action = "photo";
+    else if (r.picker) action = "picker"
+    else if (r.isGif && toGif) action = "gif";
+    else if (isAudioMuted) action = "muteVideo";
+    else if (isAudioOnly) action = "audio";
+    else if (r.isM3U8) action = "m3u8";
+    else action = "video";
+
+    if (action === "picker" || action === "audio") {
+        if (!r.filenameAttributes) defaultParams.filename = r.audioFilename;
+        defaultParams.isAudioOnly = true;
+        defaultParams.audioFormat = audioFormat;
+    }
+    if (isAudioMuted && !r.filenameAttributes) {
+        defaultParams.filename = r.filename.replace('.', '_mute.')
+    }
+
+    switch (action) {
+        default:
+            return createResponse("error", {
+                code: "ErrorEmptyDownload"
+            });
+
+        case "photo":
+            responseType = "redirect";
+            break;
+
+        case "gif":
+            params = { type: "gif" }
+            break;
+
+        case "m3u8":
+            params = {
+                type: Array.isArray(r.urls) ? "render" : "remux"
+            }
+            break;
+
+        case "muteVideo":
+            let muteType = "mute";
+            if (Array.isArray(r.urls) && !r.isM3U8) {
+                muteType = "bridge";
+            }
+            params = {
+                type: muteType,
+                u: Array.isArray(r.urls) ? r.urls[0] : r.urls,
+                mute: true
+            }
+            if (host === "reddit" && r.typeId === "redirect")
+                responseType = "redirect";
+            break;
+
+        case "picker":
+            responseType = "picker";
+            switch (host) {
+                case "instagram":
+                case "twitter":
+                case "snapchat":
+                    params = { picker: r.picker };
+                    break;
+                case "tiktok":
+                    let audioStreamType = "render";
+                    if (r.bestAudio === "mp3" && (audioFormat === "mp3" || audioFormat === "best")) {
+                        audioFormat = "mp3";
+                        audioStreamType = "bridge"
+                    }
+                    params = {
+                        picker: r.picker,
+                        u: createStream({
+                            service: "tiktok",
+                            type: audioStreamType,
+                            u: r.urls,
+                            headers: r.headers,
+                            filename: r.audioFilename,
+                            isAudioOnly: true,
+                            audioFormat,
+                        }),
+                        copy: audioFormat === "best"
+                    }
+            }
+            break;
+
+        case "video":
+            switch (host) {
+                case "bilibili":
+                    params = { type: "render" };
+                    break;
+                case "youtube":
+                    params = { type: r.type };
+                    break;
+                case "reddit":
+                    responseType = r.typeId;
+                    params = { type: r.type };
+                    break;
+                case "vimeo":
+                    if (Array.isArray(r.urls)) {
+                        params = { type: "render" }
+                    } else {
+                        responseType = "redirect";
+                    }
+                    break;
+
+                case "twitter":
+                    if (r.type === "remux") {
+                        params = { type: r.type };
+                    } else {
+                        responseType = "redirect";
+                    }
+                    break;
+
+                case "vk":
+                case "tiktok":
+                    params = { type: "bridge" };
+                    break;
+
+                case "facebook":
+                case "vine":
+                case "instagram":
+                case "tumblr":
+                case "pinterest":
+                case "streamable":
+                case "snapchat":
+                case "loom":
+                    responseType = "redirect";
+                    break;
+            }
+            break;
+
+        case "audio":
+            if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) {
+                return createResponse("error", {
+                    code: "ErrorEmptyDownload"
+                })
+            }
+
+            let processType = "render",
+                copy = false;
+
+            if (!supportedAudio.includes(audioFormat)) {
+                audioFormat = "best"
+            }
+
+            const serviceBestAudio = r.bestAudio || services[host]["bestAudio"];
+            const isBestAudio = audioFormat === "best";
+            const isBestOrMp3 = isBestAudio || audioFormat === "mp3";
+            const isBestAudioDefined = isBestAudio && serviceBestAudio;
+            const isBestHostAudio = serviceBestAudio && (audioFormat === serviceBestAudio);
+
+            const isTumblrAudio = host === "tumblr" && !r.filename;
+            const isSoundCloud = host === "soundcloud";
+            const isTiktok = host === "tiktok";
+
+            if (isBestAudioDefined || isBestHostAudio) {
+                audioFormat = serviceBestAudio;
+                processType = "bridge";
+                if (isSoundCloud || (isTiktok && audioFormat === "m4a")) {
+                    processType = "render"
+                    copy = true
+                }
+            } else if (isBestAudio && !isSoundCloud) {
+                audioFormat = "m4a";
+                copy = true
+            }
+
+            if (isTumblrAudio && isBestOrMp3) {
+                audioFormat = "mp3";
+                processType = "bridge"
+            }
+
+            if (r.isM3U8 || host === "vimeo") {
+                copy = false;
+                processType = "render"
+            }
+
+            params = {
+                type: processType,
+                u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+                audioFormat: audioFormat,
+                copy: copy
+            }
+            break;
+    }
+
+    return createResponse(responseType, {...defaultParams, ...params})
+}

api/src/processing/match.js

@@ -0,0 +1,247 @@
+import { strict as assert } from "node:assert";
+
+import { env } from "../config.js";
+import { createResponse } from "../processing/request.js";
+
+import { testers } from "./service-patterns.js";
+import matchActionDecider from "./match-action.js";
+
+import bilibili from "./services/bilibili.js";
+import reddit from "./services/reddit.js";
+import twitter from "./services/twitter.js";
+import youtube from "./services/youtube.js";
+import vk from "./services/vk.js";
+import ok from "./services/ok.js";
+import tiktok from "./services/tiktok.js";
+import tumblr from "./services/tumblr.js";
+import vimeo from "./services/vimeo.js";
+import soundcloud from "./services/soundcloud.js";
+import instagram from "./services/instagram.js";
+import vine from "./services/vine.js";
+import pinterest from "./services/pinterest.js";
+import streamable from "./services/streamable.js";
+import twitch from "./services/twitch.js";
+import rutube from "./services/rutube.js";
+import dailymotion from "./services/dailymotion.js";
+import snapchat from "./services/snapchat.js";
+import loom from "./services/loom.js";
+import facebook from "./services/facebook.js";
+
+let freebind;
+
+export default async function(host, patternMatch, lang, obj) {
+    const { url } = obj;
+    assert(url instanceof URL);
+    let dispatcher, requestIP;
+
+    if (env.freebindCIDR) {
+        if (!freebind) {
+            freebind = await import('freebind');
+        }
+
+        requestIP = freebind.ip.random(env.freebindCIDR);
+        dispatcher = freebind.dispatcherFromIP(requestIP, { strict: false });
+    }
+
+    try {
+        let r,
+            isAudioOnly = !!obj.isAudioOnly,
+            disableMetadata = !!obj.disableMetadata;
+
+        if (!testers[host]) {
+            return createResponse("error", {
+                code: "ErrorUnsupported"
+            });
+        }
+        if (!(testers[host](patternMatch))) {
+            return createResponse("error", {
+                code: "ErrorBrokenLink",
+                context: {
+                    service: host
+                }
+            });
+        }
+
+        switch (host) {
+            case "twitter":
+                r = await twitter({
+                    id: patternMatch.id,
+                    index: patternMatch.index - 1,
+                    toGif: !!obj.twitterGif,
+                    dispatcher
+                });
+                break;
+            case "vk":
+                r = await vk({
+                    userId: patternMatch.userId,
+                    videoId: patternMatch.videoId,
+                    quality: obj.vQuality
+                });
+                break;
+            case "ok":
+                r = await ok({
+                    id: patternMatch.id,
+                    quality: obj.vQuality
+                });
+                break;
+            case "bilibili":
+                r = await bilibili(patternMatch);
+                break;
+            case "youtube":
+                let fetchInfo = {
+                    id: patternMatch.id.slice(0, 11),
+                    quality: obj.vQuality,
+                    format: obj.vCodec,
+                    isAudioOnly: isAudioOnly,
+                    isAudioMuted: obj.isAudioMuted,
+                    dubLang: obj.dubLang,
+                    dispatcher
+                }
+
+                if (url.hostname === 'music.youtube.com' || isAudioOnly === true) {
+                    fetchInfo.quality = "max";
+                    fetchInfo.format = "vp9";
+                    fetchInfo.isAudioOnly = true
+                }
+
+                r = await youtube(fetchInfo);
+                break;
+            case "reddit":
+                r = await reddit({
+                    sub: patternMatch.sub,
+                    id: patternMatch.id,
+                    user: patternMatch.user
+                });
+                break;
+            case "tiktok":
+                r = await tiktok({
+                    postId: patternMatch.postId,
+                    id: patternMatch.id,
+                    fullAudio: obj.isTTFullAudio,
+                    isAudioOnly: isAudioOnly,
+                    h265: obj.tiktokH265
+                });
+                break;
+            case "tumblr":
+                r = await tumblr({
+                    id: patternMatch.id,
+                    user: patternMatch.user,
+                    url
+                });
+                break;
+            case "vimeo":
+                r = await vimeo({
+                    id: patternMatch.id.slice(0, 11),
+                    password: patternMatch.password,
+                    quality: obj.vQuality,
+                    isAudioOnly: isAudioOnly
+                });
+                break;
+            case "soundcloud":
+                isAudioOnly = true;
+                r = await soundcloud({
+                    url,
+                    author: patternMatch.author,
+                    song: patternMatch.song,
+                    format: obj.aFormat,
+                    shortLink: patternMatch.shortLink || false,
+                    accessKey: patternMatch.accessKey || false
+                });
+                break;
+            case "instagram":
+                r = await instagram({
+                    ...patternMatch,
+                    quality: obj.vQuality,
+                    dispatcher
+                })
+                break;
+            case "vine":
+                r = await vine({
+                    id: patternMatch.id
+                });
+                break;
+            case "pinterest":
+                r = await pinterest({
+                    id: patternMatch.id,
+                    shortLink: patternMatch.shortLink || false
+                });
+                break;
+            case "streamable":
+                r = await streamable({
+                    id: patternMatch.id,
+                    quality: obj.vQuality,
+                    isAudioOnly: isAudioOnly,
+                });
+                break;
+            case "twitch":
+                r = await twitch({
+                    clipId: patternMatch.clip || false,
+                    quality: obj.vQuality,
+                    isAudioOnly: obj.isAudioOnly
+                });
+                break;
+            case "rutube":
+                r = await rutube({
+                    id: patternMatch.id,
+                    yappyId: patternMatch.yappyId,
+                    key: patternMatch.key,
+                    quality: obj.vQuality,
+                    isAudioOnly: isAudioOnly
+                });
+                break;
+            case "dailymotion":
+                r = await dailymotion(patternMatch);
+                break;
+            case "snapchat":
+                r = await snapchat({
+                    hostname: url.hostname,
+                    ...patternMatch
+                });
+                break;
+            case "loom":
+                r = await loom({
+                    id: patternMatch.id
+                });
+                break;
+            case "facebook":
+                r = await facebook({
+                    ...patternMatch
+                });
+                break;
+            default:
+                return createResponse("error", {
+                    code: "ErrorUnsupported"
+                });
+        }
+
+        if (r.isAudioOnly) isAudioOnly = true;
+        let isAudioMuted = isAudioOnly ? false : obj.isAudioMuted;
+
+        if (r.error && r.critical) {
+            return createResponse("critical", {
+                code: r.error
+            })
+        }
+
+        if (r.error) {
+            return createResponse("error", {
+                code: r.error,
+                context: response?.context
+            })
+        }
+
+        return matchActionDecider(
+            r, host, obj.aFormat, isAudioOnly,
+            lang, isAudioMuted, disableMetadata,
+            obj.filenamePattern, obj.twitterGif,
+            requestIP
+        )
+    } catch {
+        return createResponse("error", {
+            code: "ErrorBadFetch",
+            context: {
+                service: host
+            }
+        })
+    }
+}

api/src/processing/request.js

@@ -0,0 +1,174 @@
+import ipaddr from "ipaddr.js";
+
+import { normalizeURL } from "./url.js";
+import { createStream } from "../stream/manage.js";
+import { verifyLanguageCode } from "../misc/utils.js";
+
+const apiVar = {
+    allowed: {
+        vCodec: ["h264", "av1", "vp9"],
+        vQuality: ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"],
+        aFormat: ["best", "mp3", "ogg", "wav", "opus"],
+        filenamePattern: ["classic", "pretty", "basic", "nerdy"]
+    },
+    booleanOnly: [
+        "isAudioOnly",
+        "isTTFullAudio",
+        "isAudioMuted",
+        "dubLang",
+        "disableMetadata",
+        "twitterGif",
+        "tiktokH265"
+    ]
+}
+
+export function createResponse(responseType, responseData) {
+    const internalError = (code) => {
+        let error = code || "Internal Server Error";
+        return {
+            status: 500,
+            body: {
+                status: "error",
+                error: {
+                    code: code || "Internal Server Error",
+                },
+                text: error, // temporary backwards compatibility
+                critical: true
+            }
+        }
+    }
+
+    try {
+        let status = 200,
+            response = {};
+
+        switch(responseType) {
+            case "error":
+                status = 400;
+                break;
+
+            case "rate-limit":
+                status = 429;
+                break;
+        }
+
+        switch (responseType) {
+            case "error":
+                response = {
+                    error: {
+                        code: responseData.code,
+                        context: responseData?.context,
+                    },
+                    text: responseData.code, // temporary backwards compatibility
+                }
+                break;
+            case "success":
+            case "rate-limit":
+                response = {
+                    text: responseData.t,
+                }
+                break;
+
+            case "redirect":
+                response = {
+                    url: responseData.u,
+                }
+                break;
+
+            case "stream":
+                response = {
+                    url: createStream(responseData),
+                }
+                break;
+
+            case "picker":
+                let pickerType = "various",
+                    audio = false;
+
+                if (responseData.service === "tiktok") {
+                    audio = responseData.u;
+                    pickerType = "images";
+                }
+
+                response = {
+                    pickerType: pickerType,
+                    picker: responseData.picker,
+                    audio: audio,
+                }
+                break;
+            case "critical":
+                return internalError(responseData.code);
+            default:
+                throw "unreachable"
+        }
+        return {
+            status,
+            body: {
+                status: responseType,
+                ...response
+            }
+        }
+    } catch {
+        return internalError()
+    }
+}
+
+export function normalizeRequest(request) {
+    try {
+        let template = {
+            url: normalizeURL(decodeURIComponent(request.url)),
+            vCodec: "h264",
+            vQuality: "720",
+            aFormat: "mp3",
+            filenamePattern: "classic",
+            isAudioOnly: false,
+            isTTFullAudio: false,
+            isAudioMuted: false,
+            disableMetadata: false,
+            dubLang: false,
+            twitterGif: false,
+            tiktokH265: false
+        }
+
+        const requestKeys = Object.keys(request);
+        const templateKeys = Object.keys(template);
+
+        if (requestKeys.length > templateKeys.length + 1 || !request.url) {
+            return false;
+        }
+
+        for (const i in requestKeys) {
+            const key = requestKeys[i];
+            const item = request[key];
+
+            if (String(key) !== "url" && templateKeys.includes(key)) {
+                if (apiVar.booleanOnly.includes(key)) {
+                    template[key] = !!item;
+                } else if (apiVar.allowed[key] && apiVar.allowed[key].includes(item)) {
+                    template[key] = String(item)
+                }
+            }
+        }
+
+        if (template.dubLang)
+            template.dubLang = verifyLanguageCode(request.dubLang);
+
+        return template
+    } catch {
+        return false
+    }
+}
+
+export function getIP(req) {
+    const strippedIP = req.ip.replace(/^::ffff:/, '');
+    const ip = ipaddr.parse(strippedIP);
+    if (ip.kind() === 'ipv4') {
+        return strippedIP;
+    }
+
+    const prefix = 56;
+    const v6Bytes = ip.toByteArray();
+          v6Bytes.fill(0, prefix / 8);
+
+    return ipaddr.fromByteArray(v6Bytes).toString();
+}

api/src/processing/service-config.json

@@ -0,0 +1,142 @@
+{
+    "audioIgnore": ["vk", "ok", "loom"],
+    "hlsExceptions": ["dailymotion", "vimeo", "rutube"],
+    "config": {
+        "bilibili": {
+            "alias": "bilibili.com & bilibili.tv",
+            "patterns": [
+                "video/:comId", "_shortLink/:comShortLink",
+                "_tv/:lang/video/:tvId", "_tv/video/:tvId"
+            ],
+            "subdomains": ["m"],
+            "enabled": true
+        },
+        "reddit": {
+            "alias": "reddit videos & gifs",
+            "patterns": ["r/:sub/comments/:id/:title", "user/:user/comments/:id/:title"],
+            "subdomains": "*",
+            "enabled": true
+        },
+        "twitter": {
+            "alias": "twitter videos & voice",
+            "altDomains": ["x.com", "vxtwitter.com", "fixvx.com"],
+            "subdomains": ["mobile"],
+            "patterns": [
+                ":user/status/:id",
+                ":user/status/:id/video/:index",
+                ":user/status/:id/photo/:index",
+                ":user/status/:id/mediaviewer",
+                ":user/status/:id/mediaViewer"
+            ],
+            "enabled": true
+        },
+        "vk": {
+            "alias": "vk video & clips",
+            "patterns": ["video:userId_:videoId", "clip:userId_:videoId", "clips:duplicate?z=clip:userId_:videoId"],
+            "subdomains": ["m"],
+            "enabled": true
+        },
+        "ok": {
+            "alias": "ok video",
+            "tld": "ru",
+            "patterns": ["video/:id", "videoembed/:id"],
+            "enabled": true
+        },
+        "youtube": {
+            "alias": "youtube videos, shorts & music",
+            "patterns": ["watch?v=:id", "embed/:id", "watch/:id"],
+            "subdomains": ["music", "m"],
+            "bestAudio": "opus",
+            "enabled": true
+        },
+        "tumblr": {
+            "alias": "tumblr video & audio",
+            "patterns": ["post/:id", "blog/view/:user/:id", ":user/:id", ":user/:id/:trackingId"],
+            "subdomains": "*",
+            "enabled": true
+        },
+        "tiktok": {
+            "alias": "tiktok videos, photos & audio",
+            "patterns": [":user/video/:postId", ":id", "t/:id", ":user/photo/:postId", "v/:id.html"],
+            "subdomains": ["vt", "vm", "m"],
+            "enabled": true
+        },
+        "vimeo": {
+            "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"],
+            "subdomains": ["player"],
+            "bestAudio": "mp3",
+            "enabled": true
+        },
+        "soundcloud": {
+            "patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"],
+            "subdomains": ["on", "m"],
+            "enabled": true
+        },
+        "instagram": {
+            "alias": "instagram posts & reels",
+            "altDomains": ["ddinstagram.com"],
+            "patterns": [
+                "reels/:postId", ":username/reel/:postId", "reel/:postId", "p/:postId", ":username/p/:postId",
+                "tv/:postId", "stories/:username/:storyId"
+            ],
+            "enabled": true
+        },
+        "vine": {
+            "alias": "vine archive",
+            "tld": "co",
+            "patterns": ["v/:id"],
+            "enabled": true
+        },
+        "pinterest": {
+            "alias": "pinterest (all media)",
+            "patterns": ["pin/:id", "pin/:id/:garbage", "url_shortener/:shortLink"],
+            "enabled": true
+        },
+        "streamable": {
+            "alias": "streamable.com",
+            "patterns": [":id", "o/:id", "e/:id", "s/:id"],
+            "enabled": true
+        },
+        "twitch": {
+            "alias": "twitch clips",
+            "tld": "tv",
+            "patterns": [":channel/clip/:clip"],
+            "enabled": true
+        },
+        "rutube": {
+            "alias": "rutube videos",
+            "tld": "ru",
+            "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId", "video/private/:id?p=:key", "video/private/:id"],
+            "enabled": true
+        },
+        "dailymotion": {
+            "alias": "dailymotion videos",
+            "patterns": ["video/:id"],
+            "enabled": true
+        },
+        "snapchat": {
+            "alias": "snapchat stories & spotlights",
+            "subdomains": ["t", "story"],
+            "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId", "add/:username", "u/:username"],
+            "enabled": true
+        },
+        "loom": {
+            "alias": "loom videos",
+            "patterns": ["share/:id"],
+            "enabled": true
+        },
+        "facebook": {
+            "alias": "facebook videos",
+            "altDomains": ["fb.watch"],
+            "subdomains": ["web"],
+            "patterns": [
+                "_shortLink/:shortLink",
+                ":username/videos/:caption/:id",
+                ":username/videos/:id",
+                "reel/:id",
+                "share/:shareType/:id"
+            ],
+            "enabled": true
+        }
+    }
+}

api/src/processing/service-patterns.js

@@ -0,0 +1,73 @@
+export const testers = {
+    "bilibili": (patternMatch) =>
+        patternMatch.comId?.length <= 12 || patternMatch.comShortLink?.length <= 16
+        || patternMatch.tvId?.length <= 24,
+
+    "dailymotion": (patternMatch) => patternMatch.id?.length <= 32,
+
+    "instagram": (patternMatch) =>
+        patternMatch.postId?.length <= 12
+        || (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24),
+
+    "loom": (patternMatch) =>
+        patternMatch.id?.length <= 32,
+
+    "ok": (patternMatch) =>
+        patternMatch.id?.length <= 16,
+
+    "pinterest": (patternMatch) =>
+        patternMatch.id?.length <= 128 || patternMatch.shortLink?.length <= 32,
+
+    "reddit": (patternMatch) =>
+        (patternMatch.sub?.length <= 22 && patternMatch.id?.length <= 10)
+        || (patternMatch.user?.length <= 22 && patternMatch.id?.length <= 10),
+
+    "rutube": (patternMatch) =>
+        (patternMatch.id?.length === 32 && patternMatch.key?.length <= 32) ||
+        patternMatch.id?.length === 32 || patternMatch.yappyId?.length === 32,
+
+    "soundcloud": (patternMatch) =>
+        (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255)
+        || patternMatch.shortLink?.length <= 32,
+
+    "snapchat": (patternMatch) =>
+        (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255))
+        || patternMatch.spotlightId?.length <= 255
+        || patternMatch.shortLink?.length <= 16,
+
+    "streamable": (patternMatch) =>
+        patternMatch.id?.length === 6,
+
+    "tiktok": (patternMatch) =>
+        patternMatch.postId?.length <= 21 || patternMatch.id?.length <= 13,
+
+    "tumblr": (patternMatch) =>
+        patternMatch.id?.length < 21
+        || (patternMatch.id?.length < 21 && patternMatch.user?.length <= 32),
+
+    "twitch": (patternMatch) =>
+        patternMatch.channel && patternMatch.clip?.length <= 100,
+
+    "twitter": (patternMatch) =>
+        patternMatch.id?.length < 20,
+
+    "vimeo": (patternMatch) =>
+        patternMatch.id?.length <= 11
+        && (!patternMatch.password || patternMatch.password.length < 16),
+
+    "vine": (patternMatch) =>
+        patternMatch.id?.length <= 12,
+
+    "vk": (patternMatch) =>
+        patternMatch.userId?.length <= 10 && patternMatch.videoId?.length <= 10,
+
+    "youtube": (patternMatch) =>
+        patternMatch.id?.length <= 11,
+
+    "facebook": (patternMatch) =>
+        patternMatch.shortLink?.length <= 11
+        || patternMatch.username?.length <= 30
+        || patternMatch.caption?.length <= 255
+        || patternMatch.id?.length <= 20 && !patternMatch.shareType
+        || patternMatch.id?.length <= 20 && patternMatch.shareType?.length === 1,
+}

api/src/processing/services/bilibili.js

@@ -0,0 +1,105 @@
+import { genericUserAgent, env } from "../../config.js";
+
+// TO-DO: higher quality downloads (currently requires an account)
+
+function com_resolveShortlink(shortId) {
+    return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' })
+            .then(r => r.status > 300 && r.status < 400 && r.headers.get('location'))
+            .then(url => {
+                if (!url) return;
+                const path = new URL(url).pathname;
+                if (path.startsWith('/video/'))
+                    return path.split('/')[2];
+            })
+            .catch(() => {})
+}
+
+function getBest(content) {
+    return content?.filter(v => v.baseUrl || v.url)
+                .map(v => (v.baseUrl = v.baseUrl || v.url, v))
+                .reduce((a, b) => a?.bandwidth > b?.bandwidth ? a : b);
+}
+
+function extractBestQuality(dashData) {
+    const bestVideo = getBest(dashData.video),
+          bestAudio = getBest(dashData.audio);
+
+    if (!bestVideo || !bestAudio) return [];
+    return [ bestVideo, bestAudio ];
+}
+
+async function com_download(id) {
+    let html = await fetch(`https://bilibili.com/video/${id}`, {
+        headers: { "user-agent": genericUserAgent }
+    }).then(r => r.text()).catch(() => {});
+    if (!html) return { error: 'ErrorCouldntFetch' };
+
+    if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
+    if (streamData.data.timelength > env.durationLimit * 1000) {
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+    }
+
+    const [ video, audio ] = extractBestQuality(streamData.data.dash);
+    if (!video || !audio) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    return {
+        urls: [video.baseUrl, audio.baseUrl],
+        audioFilename: `bilibili_${id}_audio`,
+        filename: `bilibili_${id}_${video.width}x${video.height}.mp4`
+    };
+}
+
+async function tv_download(id) {
+    const url = new URL(
+        'https://api.bilibili.tv/intl/gateway/web/playurl'
+        + '?s_locale=en_US&platform=web&qn=64&type=0&device=wap'
+        + '&tf=0&spm_id=bstar-web.ugc-video-detail.0.0&from_spm_id='
+    );
+
+    url.searchParams.set('aid', id);
+
+    const { data } = await fetch(url).then(a => a.json());
+    if (!data?.playurl?.video) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    const [ video, audio ] = extractBestQuality({
+        video: data.playurl.video.map(s => s.video_resource)
+                                 .filter(s => s.codecs.includes('avc1')),
+        audio: data.playurl.audio_resource
+    });
+
+    if (!video || !audio) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    if (video.duration > env.durationLimit * 1000) {
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+    }
+
+    return {
+        urls: [video.url, audio.url],
+        audioFilename: `bilibili_tv_${id}_audio`,
+        filename: `bilibili_tv_${id}.mp4`
+    };
+}
+
+export default async function({ comId, tvId, comShortLink }) {
+    if (comShortLink) {
+        comId = await com_resolveShortlink(comShortLink);
+    }
+
+    if (comId) {
+        return com_download(comId);
+    } else if (tvId) {
+        return tv_download(tvId);
+    }
+
+    return { error: 'ErrorCouldntFetch' };
+}

api/src/processing/services/dailymotion.js

@@ -0,0 +1,107 @@
+import HLSParser from 'hls-parser';
+import { env } from '../../config.js';
+
+let _token;
+
+function getExp(token) {
+    return JSON.parse(
+        Buffer.from(token.split('.')[1], 'base64')
+    ).exp * 1000;
+}
+
+const getToken = async () => {
+    if (_token && getExp(_token) > new Date().getTime()) {
+        return _token;
+    }
+
+    const req = await fetch('https://graphql.api.dailymotion.com/oauth/token', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
+            'User-Agent': 'dailymotion/240213162706 CFNetwork/1492.0.1 Darwin/23.3.0',
+            'Authorization': 'Basic MGQyZDgyNjQwOWFmOWU3MmRiNWQ6ODcxNmJmYTVjYmEwMmUwMGJkYTVmYTg1NTliNDIwMzQ3NzIyYWMzYQ=='
+        },
+        body: 'traffic_segment=&grant_type=client_credentials'
+    }).then(r => r.json()).catch(() => {});
+
+    if (req.access_token) {
+        return _token = req.access_token;
+    }
+}
+
+export default async function({ id }) {
+    const token = await getToken();
+    if (!token) return { error: 'ErrorSomethingWentWrong' };
+
+    const req = await fetch('https://graphql.api.dailymotion.com/',
+        {
+            method: 'POST',
+            headers: {
+                'User-Agent': 'dailymotion/240213162706 CFNetwork/1492.0.1 Darwin/23.3.0',
+                Authorization: `Bearer ${token}`,
+                'Content-Type': 'application/json',
+                'X-DM-AppInfo-Version': '7.16.0_240213162706',
+                'X-DM-AppInfo-Type': 'iosapp',
+                'X-DM-AppInfo-Id': 'com.dailymotion.dailymotion'
+            },
+            body: JSON.stringify({
+                operationName: "Media",
+                query: `
+                    query Media($xid: String!, $password: String) {
+                        media(xid: $xid, password: $password) {
+                        __typename
+                            ... on Video {
+                                xid
+                                hlsURL
+                                duration
+                                title
+                                channel {
+                                    displayName
+                                }
+                            }
+                        }
+                    }
+                `,
+                variables: { xid: id }
+            })
+        }
+    ).then(r => r.status === 200 && r.json()).catch(() => {});
+
+    const media = req?.data?.media;
+
+    if (media?.__typename !== 'Video' || !media.hlsURL) {
+        return { error: 'ErrorEmptyDownload' }
+    }
+
+    if (media.duration > env.durationLimit) {
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+    }
+
+    const manifest = await fetch(media.hlsURL).then(r => r.text()).catch(() => {});
+    if (!manifest) return { error: 'ErrorSomethingWentWrong' };
+
+    const bestQuality = HLSParser.parse(manifest).variants
+                        .filter(v => v.codecs.includes('avc1'))
+                        .reduce((a, b) => a.bandwidth > b.bandwidth ? a : b);
+    if (!bestQuality) return { error: 'ErrorEmptyDownload' }
+
+    const fileMetadata = {
+        title: media.title,
+        artist: media.channel.displayName
+    }
+
+    return {
+        urls: bestQuality.uri,
+        isM3U8: true,
+        filenameAttributes: {
+            service: 'dailymotion',
+            id: media.xid,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
+            qualityLabel: `${bestQuality.resolution.height}p`,
+            extension: 'mp4'
+        },
+        fileMetadata
+    }
+}

api/src/processing/services/facebook.js

@@ -0,0 +1,56 @@
+import { genericUserAgent } from "../../config.js";
+
+const headers = {
+    'User-Agent': genericUserAgent,
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+}
+
+const resolveUrl = (url) => {
+    return fetch(url, { headers })
+        .then(r => {
+            if (r.headers.get('location')) {
+                return decodeURIComponent(r.headers.get('location'));
+            }
+            if (r.headers.get('link')) {
+                const linkMatch = r.headers.get('link').match(/<(.*?)\/>/);
+                return decodeURIComponent(linkMatch[1]);
+            }
+            return false;
+        })
+        .catch(() => false);
+}
+
+export default async function({ id, shareType, shortLink }) {
+    let url = `https://web.facebook.com/i/videos/${id}`;
+
+    if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
+
+    const html = await fetch(url, { headers })
+        .then(r => r.text())
+        .catch(() => false);
+
+    if (!html) return { error: 'ErrorCouldntFetch' };
+
+    const urls = [];
+    const hd = html.match('"browser_native_hd_url":(".*?")');
+    const sd = html.match('"browser_native_sd_url":(".*?")');
+
+    if (hd?.[1]) urls.push(JSON.parse(hd[1]));
+    if (sd?.[1]) urls.push(JSON.parse(sd[1]));
+
+    if (!urls.length) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    const baseFilename = `facebook_${id || shortLink}`;
+
+    return {
+        urls: urls[0],
+        filename: `${baseFilename}.mp4`,
+        audioFilename: `${baseFilename}_audio`,
+    };
+}

api/src/processing/services/instagram.js

@@ -0,0 +1,349 @@
+import { createStream } from "../../stream/manage.js";
+import { genericUserAgent } from "../../config.js";
+import { getCookie, updateCookie } from "../cookie/manager.js";
+
+const commonHeaders = {
+    "user-agent": genericUserAgent,
+    "sec-gpc": "1",
+    "sec-fetch-site": "same-origin",
+    "x-ig-app-id": "936619743392459"
+}
+const mobileHeaders = {
+    "x-ig-app-locale": "en_US",
+    "x-ig-device-locale": "en_US",
+    "x-ig-mapped-locale": "en_US",
+    "user-agent": "Instagram 275.0.0.27.98 Android (33/13; 280dpi; 720x1423; Xiaomi; Redmi 7; onclite; qcom; en_US; 458229237)",
+    "accept-language": "en-US",
+    "x-fb-http-engine": "Liger",
+    "x-fb-client-ip": "True",
+    "x-fb-server-cluster": "True",
+    "content-length": "0",
+}
+const embedHeaders = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "Accept-Language": "en-GB,en;q=0.9",
+    "Cache-Control": "max-age=0",
+    "Dnt": "1",
+    "Priority": "u=0, i",
+    "Sec-Ch-Ua": 'Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99',
+    "Sec-Ch-Ua-Mobile": "?0",
+    "Sec-Ch-Ua-Platform": "macOS",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+}
+
+const cachedDtsg = {
+    value: '',
+    expiry: 0
+}
+
+export default function(obj) {
+    const dispatcher = obj.dispatcher;
+
+    async function findDtsgId(cookie) {
+        try {
+            if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value;
+
+            const data = await fetch('https://www.instagram.com/', {
+                headers: {
+                    ...commonHeaders,
+                    cookie
+                },
+                dispatcher
+            }).then(r => r.text());
+
+            const token = data.match(/"dtsg":{"token":"(.*?)"/)[1];
+
+            cachedDtsg.value = token;
+            cachedDtsg.expiry = Date.now() + 86390000;
+
+            if (token) return token;
+            return false;
+        }
+        catch {}
+    }
+
+    async function request(url, cookie, method = 'GET', requestData) {
+        let headers = {
+            ...commonHeaders,
+            'x-ig-www-claim': cookie?._wwwClaim || '0',
+            'x-csrftoken': cookie?.values()?.csrftoken,
+            cookie
+        }
+        if (method === 'POST') {
+            headers['content-type'] = 'application/x-www-form-urlencoded';
+        }
+
+        const data = await fetch(url, {
+            method,
+            headers,
+            body: requestData && new URLSearchParams(requestData),
+            dispatcher
+        });
+
+        if (data.headers.get('X-Ig-Set-Www-Claim') && cookie)
+            cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim');
+
+        updateCookie(cookie, data.headers);
+        return data.json();
+    }
+    async function getMediaId(id, { cookie, token } = {}) {
+        const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/');
+        oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`);
+
+        const oembed = await fetch(oembedURL, {
+            headers: {
+                ...mobileHeaders,
+                ...( token && { authorization: `Bearer ${token}` } ),
+                cookie
+            },
+            dispatcher
+        }).then(r => r.json()).catch(() => {});
+
+        return oembed?.media_id;
+    }
+
+    async function requestMobileApi(mediaId, { cookie, token } = {}) {
+        const mediaInfo = await fetch(`https://i.instagram.com/api/v1/media/${mediaId}/info/`, {
+            headers: {
+                ...mobileHeaders,
+                ...( token && { authorization: `Bearer ${token}` } ),
+                cookie
+            },
+            dispatcher
+        }).then(r => r.json()).catch(() => {});
+
+        return mediaInfo?.items?.[0];
+    }
+    async function requestHTML(id, cookie) {
+        const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, {
+            headers: {
+                ...embedHeaders,
+                cookie
+            },
+            dispatcher
+        }).then(r => r.text()).catch(() => {});
+
+        let embedData = JSON.parse(data?.match(/"init",\[\],\[(.*?)\]\],/)[1]);
+
+        if (!embedData || !embedData?.contextJSON) return false;
+
+        embedData = JSON.parse(embedData.contextJSON);
+
+        return embedData;
+    }
+    async function requestGQL(id, cookie) {
+        let dtsgId;
+
+        if (cookie) {
+            dtsgId = await findDtsgId(cookie);
+        }
+        const url = new URL('https://www.instagram.com/api/graphql/');
+
+        const requestData = {
+            jazoest: '26406',
+            variables: JSON.stringify({
+            shortcode: id,
+            __relay_internal__pv__PolarisShareMenurelayprovider: false
+            }),
+            doc_id: '7153618348081770'
+        };
+        if (dtsgId) {
+            requestData.fb_dtsg = dtsgId;
+        }
+
+        return (await request(url, cookie, 'POST', requestData))
+                .data
+                ?.xdt_api__v1__media__shortcode__web_info
+                ?.items
+                ?.[0];
+    }
+
+    function extractOldPost(data, id) {
+        const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children;
+        if (sidecar) {
+            const picker = sidecar.edges.filter(e => e.node?.display_url)
+                .map(e => {
+                    const type = e.node?.is_video ? "video" : "photo";
+                    const url = type === "video" ? e.node?.video_url : e.node?.display_url;
+
+                    return {
+                        type, url,
+                        /* thumbnails have `Cross-Origin-Resource-Policy`
+                        ** set to `same-origin`, so we need to proxy them */
+                        thumb: createStream({
+                            service: "instagram",
+                            type: "default",
+                            u: e.node?.display_url,
+                            filename: "image.jpg"
+                        })
+                    }
+                });
+
+            if (picker.length) return { picker }
+        } else if (data?.gql_data?.shortcode_media?.video_url) {
+            return {
+                urls: data.gql_data.shortcode_media.video_url,
+                filename: `instagram_${id}.mp4`,
+                audioFilename: `instagram_${id}_audio`
+            }
+        } else if (data?.gql_data?.shortcode_media?.display_url) {
+            return {
+                urls: data.gql_data?.shortcode_media.display_url,
+                isPhoto: true
+            }
+        }
+    }
+
+    function extractNewPost(data, id) {
+        const carousel = data.carousel_media;
+        if (carousel) {
+            const picker = carousel.filter(e => e?.image_versions2)
+                .map(e => {
+                    const type = e.video_versions ? "video" : "photo";
+                    const imageUrl = e.image_versions2.candidates[0].url;
+                    
+                    let url = imageUrl;
+                    if (type === 'video') {
+                        const video = e.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a);
+                        url = video.url;
+                    }
+
+                    return {
+                        type, url,
+                        /* thumbnails have `Cross-Origin-Resource-Policy`
+                        ** set to `same-origin`, so we need to proxy them */
+                        thumb: createStream({
+                            service: "instagram",
+                            type: "default",
+                            u: imageUrl,
+                            filename: "image.jpg"
+                        })
+                    }
+                });
+
+            if (picker.length) return { picker }
+        } else if (data.video_versions) {
+            const video = data.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a)
+            return {
+                urls: video.url,
+                filename: `instagram_${id}.mp4`,
+                audioFilename: `instagram_${id}_audio`
+            }
+        } else if (data.image_versions2?.candidates) {
+            return {
+                urls: data.image_versions2.candidates[0].url,
+                isPhoto: true
+            }
+        }
+    }
+
+    async function getPost(id) {
+        let data, result;
+        try {
+            const cookie = getCookie('instagram');
+        
+            const bearer = getCookie('instagram_bearer');
+            const token = bearer?.values()?.token;
+
+            // get media_id for mobile api, three methods
+            let media_id = await getMediaId(id);
+            if (!media_id && token) media_id = await getMediaId(id, { token });
+            if (!media_id && cookie) media_id = await getMediaId(id, { cookie });
+
+            // mobile api (bearer)
+            if (media_id && token) data = await requestMobileApi(media_id, { token });
+
+            // mobile api (no cookie, cookie)
+            if (media_id && !data) data = await requestMobileApi(media_id);
+            if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie });
+
+            // html embed (no cookie, cookie)
+            if (!data) data = await requestHTML(id);
+            if (!data && cookie) data = await requestHTML(id, cookie);
+
+            // web app graphql api (no cookie, cookie)
+            if (!data) data = await requestGQL(id);
+            if (!data && cookie) data = await requestGQL(id, cookie);
+        } catch {}
+
+        if (!data) return { error: 'ErrorCouldntFetch' };
+
+        if (data?.gql_data) {
+            result = extractOldPost(data, id)
+        } else {
+            result = extractNewPost(data, id)
+        }
+
+        if (result) return result;
+        return { error: 'ErrorEmptyDownload' }
+    }
+
+    async function usernameToId(username, cookie) {
+        const url = new URL('https://www.instagram.com/api/v1/users/web_profile_info/');
+            url.searchParams.set('username', username);
+
+        try {
+            const data = await request(url, cookie);
+            return data?.data?.user?.id;
+        } catch {}
+    }
+
+    async function getStory(username, id) {
+        const cookie = getCookie('instagram');
+        if (!cookie) return { error: 'ErrorUnsupported' };
+
+        const userId = await usernameToId(username, cookie);
+        if (!userId) return { error: 'ErrorEmptyDownload' };
+        
+        const dtsgId = await findDtsgId(cookie);
+
+        const url = new URL('https://www.instagram.com/api/graphql/');
+        const requestData = {
+            fb_dtsg: dtsgId,
+            jazoest: '26438',
+            variables: JSON.stringify({
+                reel_ids_arr : [ userId ],
+            }),
+            server_timestamps: true,
+            doc_id: '25317500907894419'
+        };
+
+        let media;
+        try {
+            const data = (await request(url, cookie, 'POST', requestData));
+            media = data?.data?.xdt_api__v1__feed__reels_media?.reels_media?.find(m => m.id === userId);
+        } catch {}
+
+        const item = media.items.find(m => m.pk === id);
+        if (!item) return { error: 'ErrorEmptyDownload' };
+        
+        if (item.video_versions) {
+            const video = item.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a)
+            return {
+                urls: video.url,
+                filename: `instagram_${id}.mp4`,
+                audioFilename: `instagram_${id}_audio`
+            }
+        }
+
+        if (item.image_versions2?.candidates) {
+            return {
+                urls: item.image_versions2.candidates[0].url,
+                isPhoto: true
+            }
+        }
+
+        return { error: 'ErrorUnsupported' };
+    }
+
+    const { postId, storyId, username } = obj;
+    if (postId) return getPost(postId);
+    if (username && storyId) return getStory(username, storyId);
+
+    return { error: 'ErrorUnsupported' }
+}

api/src/processing/services/loom.js

@@ -0,0 +1,39 @@
+import { genericUserAgent } from "../../config.js";
+
+export default async function({ id }) {
+    const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, {
+        method: "POST",
+        headers: {
+            "user-agent": genericUserAgent,
+            origin: "https://www.loom.com",
+            referer: `https://www.loom.com/share/${id}`,
+            cookie: `loom_referral_video=${id};`,
+
+            "apollographql-client-name": "web",
+            "apollographql-client-version": "14c0b42",
+            "x-loom-request-source": "loom_web_14c0b42",
+        },
+        body: JSON.stringify({
+            force_original: false,
+            password: null,
+            anonID: null,
+            deviceID: null
+        })
+    })
+    .then(r => r.status === 200 ? r.json() : false)
+    .catch(() => {});
+
+    if (!gql) return { error: 'ErrorEmptyDownload' };
+
+    const videoUrl = gql?.url;
+
+    if (videoUrl?.includes('.mp4?')) {
+        return {
+            urls: videoUrl,
+            filename: `loom_${id}.mp4`,
+            audioFilename: `loom_${id}_audio`
+        }
+    }
+
+    return { error: 'ErrorEmptyDownload' }
+}

api/src/processing/services/ok.js

@@ -0,0 +1,65 @@
+import { genericUserAgent, env } from "../../config.js";
+import { cleanString } from "../../misc/utils.js";
+
+const resolutions = {
+    "ultra": "2160",
+    "quad": "1440",
+    "full": "1080",
+    "hd": "720",
+    "sd": "480",
+    "low": "360",
+    "lowest": "240",
+    "mobile": "144"
+}
+
+export default async function(o) {
+    let quality = o.quality === "max" ? "2160" : o.quality;
+
+    let html = await fetch(`https://ok.ru/video/${o.id}`, {
+        headers: { "user-agent": genericUserAgent }
+    }).then(r => r.text()).catch(() => {});
+
+    if (!html) return { error: 'ErrorCouldntFetch' };
+
+    let videoData = html.match(/<div data-module="OKVideo" .*? data-options="({.*?})"( .*?)>/)
+                        ?.[1]
+                        ?.replaceAll("&quot;", '"');
+
+    if (!videoData) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    videoData = JSON.parse(JSON.parse(videoData).flashvars.metadata);
+
+    if (videoData.provider !== "UPLOADED_ODKL")
+        return { error: 'ErrorUnsupported' };
+
+    if (videoData.movie.is_live)
+        return { error: 'ErrorLiveVideo' };
+
+    if (videoData.movie.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+
+    let videos = videoData.videos.filter(v => !v.disallowed);
+    let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];
+
+    let fileMetadata = {
+        title: cleanString(videoData.movie.title.trim()),
+        author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()),
+    }
+
+    if (bestVideo) return {
+        urls: bestVideo.url,
+        filenameAttributes: {
+            service: "ok",
+            id: o.id,
+            title: fileMetadata.title,
+            author: fileMetadata.author,
+            resolution: `${resolutions[bestVideo.name]}p`,
+            qualityLabel: `${resolutions[bestVideo.name]}p`,
+            extension: "mp4"
+        }
+    }
+
+    return { error: 'ErrorEmptyDownload' }
+}

api/src/processing/services/pinterest.js

@@ -0,0 +1,43 @@
+import { genericUserAgent } from "../../config.js";
+
+const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
+const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
+
+export default async function(o) {
+    let id = o.id;
+
+    if (!o.id && o.shortLink) {
+        id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" })
+                   .then(r => r.headers.get("location").split('pin/')[1].split('/')[0])
+                   .catch(() => {});
+    }
+    if (id.includes("--")) id = id.split("--")[1];
+    if (!id) return { error: 'ErrorCouldntFetch' };
+
+    let html = await fetch(`https://www.pinterest.com/pin/${id}/`, {
+        headers: { "user-agent": genericUserAgent }
+    }).then(r => r.text()).catch(() => {});
+
+    if (!html) return { error: 'ErrorCouldntFetch' };
+
+    let videoLink = [...html.matchAll(videoRegex)]
+                    .map(([, link]) => link)
+                    .find(a => a.endsWith('.mp4') && a.includes('720p'));
+
+    if (videoLink) return {
+        urls: videoLink,
+        filename: `pinterest_${o.id}.mp4`,
+        audioFilename: `pinterest_${o.id}_audio`
+    }
+
+    let imageLink = [...html.matchAll(imageRegex)]
+                    .map(([, link]) => link)
+                    .find(a => a.endsWith('.jpg') || a.endsWith('.gif'));
+
+    if (imageLink) return {
+        urls: imageLink,
+        isPhoto: true
+    }
+
+    return { error: 'ErrorEmptyDownload' };
+}

api/src/processing/services/reddit.js

@@ -0,0 +1,124 @@
+import { genericUserAgent, env } from "../../config.js";
+import { getCookie, updateCookieValues } from "../cookie/manager.js";
+
+async function getAccessToken() {
+    /* "cookie" in cookiefile needs to contain:
+     * client_id, client_secret, refresh_token
+     * e.g. client_id=bla; client_secret=bla; refresh_token=bla
+     *
+     * you can get these by making a reddit app and
+     * authenticating an account against reddit's oauth2 api
+     * see: https://github.com/reddit-archive/reddit/wiki/OAuth2
+     * 
+     * any additional cookie fields are managed by this code and you
+     * should not touch them unless you know what you're doing. **/
+    const cookie = await getCookie('reddit');
+    if (!cookie) return;
+
+    const values = cookie.values(),
+          needRefresh = !values.access_token
+                        || !values.expiry
+                        || Number(values.expiry) < new Date().getTime();
+    if (!needRefresh) return values.access_token;
+
+    const data = await fetch('https://www.reddit.com/api/v1/access_token', {
+        method: 'POST',
+        headers: {
+            'authorization': `Basic ${Buffer.from(
+                [values.client_id, values.client_secret].join(':')
+            ).toString('base64')}`,
+            'content-type': 'application/x-www-form-urlencoded',
+            'user-agent': genericUserAgent,
+            'accept': 'application/json'
+        },
+        body: `grant_type=refresh_token&refresh_token=${encodeURIComponent(values.refresh_token)}`
+    }).then(r => r.json()).catch(() => {});
+    if (!data) return;
+
+    const { access_token, refresh_token, expires_in } = data;
+    if (!access_token) return;
+
+    updateCookieValues(cookie, {
+        ...cookie.values(),
+        access_token, refresh_token,
+        expiry: new Date().getTime() + (expires_in * 1000),
+    });
+
+    return access_token;
+}
+
+export default async function(obj) {
+    let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
+
+    if (obj.user) {
+        url.pathname = `/user/${obj.user}/comments/${obj.id}.json`;
+    }
+
+    const accessToken = await getAccessToken();
+    if (accessToken) url.hostname = 'oauth.reddit.com';
+
+    let data = await fetch(
+        url, {
+            headers: {
+                'User-Agent': genericUserAgent,
+                accept: 'application/json',
+                authorization: accessToken && `Bearer ${accessToken}`
+            }
+        }
+    ).then(r => r.json()).catch(() => {});
+
+    if (!data || !Array.isArray(data)) return { error: 'ErrorCouldntFetch' };
+
+    data = data[0]?.data?.children[0]?.data;
+
+    if (data?.url?.endsWith('.gif')) return {
+        typeId: "redirect",
+        urls: data.url
+    }
+
+    if (!data.secure_media?.reddit_video)
+        return { error: 'ErrorEmptyDownload' };
+
+    if (data.secure_media?.reddit_video?.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+
+    let audio = false,
+        video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
+        audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`;
+
+    if (video.match('.mp4')) {
+        audioFileLink = `${video.split('_')[0]}_audio.mp4`
+    }
+
+    // test the existence of audio
+    await fetch(audioFileLink, { method: "HEAD" }).then(r => {
+        if (Number(r.status) === 200) {
+            audio = true
+        }
+    }).catch(() => {})
+
+    // fallback for videos with variable audio quality
+    if (!audio) {
+        audioFileLink = `${video.split('_')[0]}_AUDIO_128.mp4`
+        await fetch(audioFileLink, { method: "HEAD" }).then(r => {
+            if (Number(r.status) === 200) {
+                audio = true
+            }
+        }).catch(() => {})
+    }
+
+    let id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
+
+    if (!audio) return {
+        typeId: "redirect",
+        urls: video
+    }
+
+    return {
+        typeId: "stream",
+        type: "render",
+        urls: [video, audioFileLink],
+        audioFilename: `reddit_${id}_audio`,
+        filename: `reddit_${id}.mp4`
+    }
+}

api/src/processing/services/rutube.js

@@ -0,0 +1,80 @@
+import HLS from 'hls-parser';
+
+import { env } from "../../config.js";
+import { cleanString } from '../../misc/utils.js';
+
+async function requestJSON(url) {
+    try {
+        const r = await fetch(url);
+        return await r.json();
+    } catch {}
+}
+
+const delta = (a, b) => Math.abs(a - b);
+
+export default async function(obj) {
+    if (obj.yappyId) {
+        const yappy = await requestJSON(
+            `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15`
+        )
+        const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link;
+        if (!yappyURL) return { error: 'ErrorEmptyDownload' };
+
+        return {
+            urls: yappyURL,
+            filename: `rutube_yappy_${obj.yappyId}.mp4`,
+            audioFilename: `rutube_yappy_${obj.yappyId}_audio`
+        }
+    }
+
+    const quality = Number(obj.quality) || 9000;
+
+    const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`);
+    if (obj.key) requestURL.searchParams.set('p', obj.key);
+
+    const play = await requestJSON(requestURL);
+    if (!play) return { error: 'ErrorCouldntFetch' };
+
+    if (play.detail || !play.video_balancer) return { error: 'ErrorEmptyDownload' };
+    if (play.live_streams?.hls) return { error: 'ErrorLiveVideo' };
+
+    if (play.duration > env.durationLimit * 1000)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+
+    let m3u8 = await fetch(play.video_balancer.m3u8)
+                     .then(r => r.text())
+                     .catch(() => {});
+
+    if (!m3u8) return { error: 'ErrorCouldntFetch' };
+
+    m3u8 = HLS.parse(m3u8).variants;
+
+    const matchingQuality = m3u8.reduce((prev, next) => {
+        const diff = {
+            prev: delta(quality, prev.resolution.height),
+            next: delta(quality, next.resolution.height)
+        };
+
+        return diff.prev < diff.next ? prev : next;
+    });
+
+    const fileMetadata = {
+        title: cleanString(play.title.trim()),
+        artist: cleanString(play.author.name.trim()),
+    }
+
+    return {
+        urls: matchingQuality.uri,
+        isM3U8: true,
+        filenameAttributes: {
+            service: "rutube",
+            id: obj.id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+            resolution: `${matchingQuality.resolution.width}x${matchingQuality.resolution.height}`,
+            qualityLabel: `${matchingQuality.resolution.height}p`,
+            extension: "mp4"
+        },
+        fileMetadata: fileMetadata
+    }
+}

api/src/processing/services/snapchat.js

@@ -0,0 +1,96 @@
+import { genericUserAgent } from "../../config.js";
+import { getRedirectingURL } from "../../misc/utils.js";
+import { extract, normalizeURL } from "../url.js";
+
+const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="(https:\/\/cf-st\.sc-cdn\.net\/d\/[\w.?=]+&amp;uc=\d+)" as="video"\/>/;
+const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
+
+async function getSpotlight(id) {
+    const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {
+        return { error: 'ErrorCouldntFetch' };
+    }
+
+    const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
+    if (videoURL) {
+        return {
+            urls: videoURL,
+            filename: `snapchat_${id}.mp4`,
+            audioFilename: `snapchat_${id}_audio`
+        }
+    }
+}
+
+async function getStory(username, storyId) {
+    const html = await fetch(`https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {
+        return { error: 'ErrorCouldntFetch' };
+    }
+
+    const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
+    if (nextDataString) {
+        const data = JSON.parse(nextDataString);
+        const storyIdParam = data.query.profileParams[1];
+
+        if (storyIdParam && data.props.pageProps.story) {
+            const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
+            if (story) {
+                if (story.snapMediaType === 0) {
+                    return {
+                        urls: story.snapUrls.mediaUrl,
+                        isPhoto: true
+                    }
+                }
+
+                return {
+                    urls: story.snapUrls.mediaUrl,
+                    filename: `snapchat_${storyId}.mp4`,
+                    audioFilename: `snapchat_${storyId}_audio`
+                }
+            }
+        }
+
+        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        if (defaultStory) {
+            return {
+                picker: defaultStory.snapList.map((snap) => ({
+                    type: snap.snapMediaType === 0 ? 'photo' : 'video',
+                    url: snap.snapUrls.mediaUrl,
+                    thumb: snap.snapUrls.mediaPreviewUrl.value
+                }))
+            }
+        }
+    }
+}
+
+export default async function (obj) {
+    let params = obj;
+    if (obj.hostname === 't.snapchat.com' && obj.shortLink) {
+        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
+
+        if (!link?.startsWith('https://www.snapchat.com/')) {
+            return { error: 'ErrorCouldntFetch' };
+        }
+
+        const extractResult = extract(normalizeURL(link));
+        if (extractResult?.host !== 'snapchat') {
+            return { error: 'ErrorCouldntFetch' };
+        }
+
+        params = extractResult.patternMatch;
+    }
+
+    if (params.spotlightId) {
+        const result = await getSpotlight(params.spotlightId);
+        if (result) return result;
+    } else if (params.username) {
+        const result = await getStory(params.username, params.storyId);
+        if (result) return result;
+    }
+
+    return { error: 'ErrorCouldntFetch' };
+}

api/src/processing/services/soundcloud.js

@@ -0,0 +1,107 @@
+import { env } from "../../config.js";
+import { cleanString } from "../../misc/utils.js";
+
+const cachedID = {
+    version: '',
+    id: ''
+}
+
+async function findClientID() {
+    try {
+        let sc = await fetch('https://soundcloud.com/').then(r => r.text()).catch(() => {});
+        let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
+
+        if (cachedID.version === scVersion) return cachedID.id;
+
+        let scripts = sc.matchAll(/<script.+src="(.+)">/g);
+        let clientid;
+        for (let script of scripts) {
+            let url = script[1];
+
+            if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
+                return;
+            }
+
+            let scrf = await fetch(url).then(r => r.text()).catch(() => {});
+            let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
+
+            if (id && typeof id[0] === 'string') {
+                clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
+                break;
+            }
+        }
+        cachedID.version = scVersion;
+        cachedID.id = clientid;
+
+        return clientid;
+    } catch {}
+}
+
+export default async function(obj) {
+    let clientId = await findClientID();
+    if (!clientId) return { error: 'ErrorSoundCloudNoClientId' };
+
+    let link;
+    if (obj.url.hostname === 'on.soundcloud.com' && obj.shortLink) {
+        link = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`, { redirect: "manual" }).then(r => {
+            if (r.status === 302 && r.headers.get("location").startsWith("https://soundcloud.com/")) {
+                return r.headers.get("location").split('?', 1)[0]
+            }
+        }).catch(() => {});
+    }
+
+    if (!link && obj.author && obj.song) {
+        link = `https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`
+    }
+
+    if (!link) return { error: 'ErrorCouldntFetch' };
+
+    let json = await fetch(`https://api-v2.soundcloud.com/resolve?url=${link}&client_id=${clientId}`)
+                     .then(r => r.status === 200 ? r.json() : false)
+                     .catch(() => {});
+
+    if (!json) return { error: 'ErrorCouldntFetch' };
+
+    if (!json.media.transcodings) return { error: 'ErrorEmptyDownload' };
+
+    let bestAudio = "opus",
+        selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"),
+        mp3Media = json.media.transcodings.find(v => v.preset === "mp3_0_0");
+
+    // use mp3 if present if user prefers it or if opus isn't available
+    if (mp3Media && (obj.format === "mp3" || !selectedStream)) {
+        selectedStream = mp3Media;
+        bestAudio = "mp3"
+    }
+
+    let fileUrlBase = selectedStream.url;
+    let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
+
+    if (!fileUrl.startsWith("https://api-v2.soundcloud.com/media/soundcloud:tracks:"))
+        return { error: 'ErrorEmptyDownload' };
+
+    if (json.duration > env.durationLimit * 1000)
+        return { error: ['ErrorLengthAudioConvert', env.durationLimit / 60] };
+
+    let file = await fetch(fileUrl)
+                     .then(async r => (await r.json()).url)
+                     .catch(() => {});
+    if (!file) return { error: 'ErrorCouldntFetch' };
+
+    let fileMetadata = {
+        title: cleanString(json.title.trim()),
+        artist: cleanString(json.user.username.trim()),
+    }
+
+    return {
+        urls: file,
+        filenameAttributes: {
+            service: "soundcloud",
+            id: json.id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist
+        },
+        bestAudio,
+        fileMetadata
+    }
+}

api/src/processing/services/streamable.js

@@ -0,0 +1,22 @@
+export default async function(obj) {
+    let video = await fetch(`https://api.streamable.com/videos/${obj.id}`)
+                      .then(r => r.status === 200 ? r.json() : false)
+                      .catch(() => {});
+
+    if (!video) return { error: 'ErrorEmptyDownload' };
+
+    let best = video.files['mp4-mobile'];
+    if (video.files.mp4 && (obj.isAudioOnly || obj.quality === "max" || obj.quality >= 720)) {
+        best = video.files.mp4;
+    }
+
+    if (best) return {
+        urls: best.url,
+        filename: `streamable_${obj.id}_${best.width}x${best.height}.mp4`,
+        audioFilename: `streamable_${obj.id}_audio`,
+        fileMetadata: {
+            title: video.title
+        }
+    }
+    return { error: 'ErrorEmptyDownload' }
+}

api/src/processing/services/tiktok.js

@@ -0,0 +1,120 @@
+import { genericUserAgent } from "../../config.js";
+import { updateCookie } from "../cookie/manager.js";
+import { extract } from "../url.js";
+import Cookie from "../cookie/cookie.js";
+
+const shortDomain = "https://vt.tiktok.com/";
+
+export default async function(obj) {
+    const cookie = new Cookie({});
+    let postId = obj.postId;
+
+    if (!postId) {
+        let html = await fetch(`${shortDomain}${obj.id}`, {
+            redirect: "manual",
+            headers: {
+                "user-agent": genericUserAgent.split(' Chrome/1')[0]
+            }
+        }).then(r => r.text()).catch(() => {});
+
+        if (!html) return { error: 'ErrorCouldntFetch' };
+
+        if (html.startsWith('<a href="https://')) {
+            const extractedURL = html.split('<a href="')[1].split('?')[0];
+            const { patternMatch } = extract(extractedURL);
+            postId = patternMatch.postId
+        }
+    }
+    if (!postId) return { error: 'ErrorCantGetID' };
+
+    // should always be /video/, even for photos
+    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
+        headers: {
+            "user-agent": genericUserAgent,
+            cookie,
+        }
+    })
+    updateCookie(cookie, res.headers);
+
+    const html = await res.text();
+
+    let detail;
+    try {
+        const json = html
+            .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
+            .split('</script>')[0]
+        const data = JSON.parse(json)
+        detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+    } catch {
+        return { error: 'ErrorCouldntFetch' };
+    }
+
+    let video, videoFilename, audioFilename, audio, images,
+        filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`,
+        bestAudio = 'm4a';
+
+    images = detail.imagePost?.images;
+
+    let playAddr = detail.video.playAddr;
+    if (obj.h265) {
+        const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
+        playAddr = h265PlayAddr || playAddr
+    }
+
+    if (!obj.isAudioOnly && !images) {
+        video = playAddr;
+        videoFilename = `${filenameBase}.mp4`;
+    } else {
+        audio = playAddr;
+        audioFilename = `${filenameBase}_audio`;
+
+        if (obj.fullAudio || !audio) {
+            audio = detail.music.playUrl;
+            audioFilename += `_original`
+        }
+        if (audio.includes("mime_type=audio_mpeg")) bestAudio = 'mp3';
+    }
+
+    if (video) {
+        return {
+            urls: video,
+            filename: videoFilename,
+            headers: { cookie }
+        }
+    }
+
+    if (images && obj.isAudioOnly) {
+        return {
+            urls: audio,
+            audioFilename: audioFilename,
+            isAudioOnly: true,
+            bestAudio,
+            headers: { cookie }
+        }
+    }
+
+    if (images) {
+        let imageLinks = images
+            .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?")))
+            .map(url => ({ url }));
+
+        return {
+            picker: imageLinks,
+            urls: audio,
+            audioFilename: audioFilename,
+            isAudioOnly: true,
+            bestAudio,
+            headers: { cookie }
+        }
+    }
+
+    if (audio) {
+        return {
+            urls: audio,
+            audioFilename: audioFilename,
+            isAudioOnly: true,
+            bestAudio,
+            headers: { cookie }
+        }
+    }
+}

api/src/processing/services/tumblr.js

@@ -0,0 +1,70 @@
+import psl from "psl";
+
+const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z';
+const API_BASE = 'https://api-http2.tumblr.com';
+
+function request(domain, id) {
+    const url = new URL(`/v2/blog/${domain}/posts/${id}/permalink`, API_BASE);
+    url.searchParams.set('api_key', API_KEY);
+    url.searchParams.set('fields[blogs]', 'uuid,name,avatar,?description,?can_message,?can_be_followed,?is_adult,?reply_conditions,'
+                                            + '?theme,?title,?url,?is_blocked_from_primary,?placement_id,?primary,?updated,?followed,'
+                                            + '?ask,?can_subscribe,?paywall_access,?subscription_plan,?is_blogless_advertiser,?tumblrmart_accessories');
+
+    return fetch(url, {
+        headers: {
+            'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr',
+            'X-Version': 'iPhone/33.3/333010/17.3.1/tumblr'
+        }
+    }).then(a => a.json()).catch(() => {});
+}
+
+export default async function(input) {
+    let { subdomain } = psl.parse(input.url.hostname);
+
+    if (subdomain?.includes('.')) {
+        return { error: ['ErrorBrokenLink', 'tumblr'] }
+    } else if (subdomain === 'www' || subdomain === 'at') {
+        subdomain = undefined
+    }
+
+    const domain = `${subdomain ?? input.user}.tumblr.com`;
+    const data = await request(domain, input.id);
+
+    const element = data?.response?.timeline?.elements?.[0];
+    if (!element) return { error: 'ErrorEmptyDownload' };
+
+    const contents = [
+        ...element.content,
+        ...element?.trail?.map(t => t.content).flat()
+    ]
+
+    const audio = contents.find(c => c.type === 'audio');
+    if (audio && audio.provider === 'tumblr') {
+        const fileMetadata = {
+            title: audio?.title,
+            artist: audio?.artist
+        };
+
+        return {
+            urls: audio.media.url,
+            filenameAttributes: {
+                service: 'tumblr',
+                id: input.id,
+                title: fileMetadata.title,
+                author: fileMetadata.artist
+            },
+            isAudioOnly: true
+        }
+    }
+
+    const video = contents.find(c => c.type === 'video');
+    if (video && video.provider === 'tumblr') {
+        return {
+            urls: video.media.url,
+            filename: `tumblr_${input.id}.mp4`,
+            audioFilename: `tumblr_${input.id}_audio`
+        }
+    }
+
+    return { error: 'ErrorEmptyDownload' }
+}

api/src/processing/services/twitch.js

@@ -0,0 +1,87 @@
+import { env } from "../../config.js";
+import { cleanString } from '../../misc/utils.js';
+
+const gqlURL = "https://gql.twitch.tv/gql";
+const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" };
+
+export default async function (obj) {
+    let req_metadata = await fetch(gqlURL, {
+        method: "POST",
+        headers: clientIdHead,
+        body: JSON.stringify({
+            query: `{
+            clip(slug: "${obj.clipId}") {
+                broadcaster {
+                    login
+                }
+                createdAt
+                curator {
+                    login
+                }
+                durationSeconds
+                id
+                medium: thumbnailURL(width: 480, height: 272)
+                title
+                videoQualities {
+                    quality
+                    sourceURL
+                }
+            }
+        }`
+        })
+    }).then(r => r.status === 200 ? r.json() : false).catch(() => {});
+    if (!req_metadata) return { error: 'ErrorCouldntFetch' };
+
+    let clipMetadata = req_metadata.data.clip;
+
+    if (clipMetadata.durationSeconds > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+    if (!clipMetadata.videoQualities || !clipMetadata.broadcaster)
+        return { error: 'ErrorEmptyDownload' };
+
+    let req_token = await fetch(gqlURL, {
+        method: "POST",
+        headers: clientIdHead,
+        body: JSON.stringify([
+            {
+                "operationName": "VideoAccessToken_Clip",
+                "variables": {
+                    "slug": obj.clipId
+                },
+                "extensions": {
+                    "persistedQuery": {
+                        "version": 1,
+                        "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11"
+                    }
+                }
+            }
+        ])
+    }).then(r => r.status === 200 ? r.json() : false).catch(() => {});
+
+    if (!req_token) return { error: 'ErrorCouldntFetch' };
+
+    let formats = clipMetadata.videoQualities;
+    let format = formats.find(f => f.quality === obj.quality) || formats[0];
+
+    return {
+        type: "bridge",
+        urls: `${format.sourceURL}?${new URLSearchParams({
+            sig: req_token[0].data.clip.playbackAccessToken.signature,
+            token: req_token[0].data.clip.playbackAccessToken.value
+        })}`,
+        fileMetadata: {
+            title: cleanString(clipMetadata.title.trim()),
+            artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`,
+        },
+        filenameAttributes: {
+            service: "twitch",
+            id: clipMetadata.id,
+            title: cleanString(clipMetadata.title.trim()),
+            author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`,
+            qualityLabel: `${format.quality}p`,
+            extension: 'mp4'
+        },
+        filename: `twitchclip_${clipMetadata.id}_${format.quality}p.mp4`,
+        audioFilename: `twitchclip_${clipMetadata.id}_audio`
+    }
+}

api/src/processing/services/twitter.js

@@ -0,0 +1,210 @@
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+import { getCookie, updateCookie } from "../cookie/manager.js";
+
+const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';
+const tokenURL = 'https://api.x.com/1.1/guest/activate.json';
+
+const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});
+
+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});
+
+const commonHeaders = {
+    "user-agent": genericUserAgent,
+    "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
+    "x-twitter-client-language": "en",
+    "x-twitter-active-user": "yes",
+    "accept-language": "en"
+}
+
+// fix all videos affected by the container bug in twitter muxer (took them over two weeks to fix it????)
+const TWITTER_EPOCH = 1288834974657n;
+const badContainerStart = new Date(1701446400000);
+const badContainerEnd = new Date(1702605600000);
+
+function needsFixing(media) {
+    const representativeId = media.source_status_id_str ?? media.id_str;
+    const mediaTimestamp = new Date(
+        Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
+    );
+    return mediaTimestamp > badContainerStart && mediaTimestamp < badContainerEnd
+}
+
+function bestQuality(arr) {
+    return arr.filter(v => v.content_type === "video/mp4")
+        .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b)
+        .url
+}
+
+let _cachedToken;
+const getGuestToken = async (dispatcher, forceReload = false) => {
+    if (_cachedToken && !forceReload) {
+        return _cachedToken;
+    }
+
+    const tokenResponse = await fetch(tokenURL, {
+        method: 'POST',
+        headers: commonHeaders,
+        dispatcher
+    }).then(r => r.status === 200 && r.json()).catch(() => {})
+
+    if (tokenResponse?.guest_token) {
+        return _cachedToken = tokenResponse.guest_token
+    }
+}
+
+const requestTweet = async(dispatcher, tweetId, token, cookie) => {
+    const graphqlTweetURL = new URL(graphqlURL);
+
+    let headers = {
+        ...commonHeaders,
+        'content-type': 'application/json',
+        'x-guest-token': token,
+        cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}`
+    }
+
+    if (cookie) {
+        headers = {
+            ...commonHeaders,
+            'content-type': 'application/json',
+            'X-Twitter-Auth-Type': 'OAuth2Session',
+            'x-csrf-token': cookie.values().ct0,
+            cookie
+        }
+    }
+
+    graphqlTweetURL.searchParams.set('variables',
+        JSON.stringify({
+            tweetId,
+            withCommunity: false,
+            includePromotedContent: false,
+            withVoice: false
+        })
+    );
+    graphqlTweetURL.searchParams.set('features', tweetFeatures);
+    graphqlTweetURL.searchParams.set('fieldToggles', tweetFieldToggles);
+
+    let result = await fetch(graphqlTweetURL, { headers, dispatcher });
+    updateCookie(cookie, result.headers);
+
+    // we might have been missing the `ct0` cookie, retry
+    if (result.status === 403 && result.headers.get('set-cookie')) {
+        result = await fetch(graphqlTweetURL, {
+            headers: {
+                ...headers,
+                'x-csrf-token': cookie.values().ct0
+            },
+            dispatcher
+        });
+    }
+
+    return result
+}
+
+export default async function({ id, index, toGif, dispatcher }) {
+    const cookie = await getCookie('twitter');
+
+    let guestToken = await getGuestToken(dispatcher);
+    if (!guestToken) return { error: 'ErrorCouldntFetch' };
+
+    let tweet = await requestTweet(dispatcher, id, guestToken);
+
+    // get new token & retry if old one expired
+    if ([403, 429].includes(tweet.status)) {
+        guestToken = await getGuestToken(dispatcher, true);
+        tweet = await requestTweet(dispatcher, id, guestToken)
+    }
+
+    tweet = await tweet.json();
+
+    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+
+    if (tweetTypename === "TweetUnavailable") {
+        const reason = tweet?.data?.tweetResult?.result?.reason;
+        switch(reason) {
+            case "Protected":
+                return { error: 'ErrorTweetProtected' }
+            case "NsfwLoggedOut":
+                if (cookie) {
+                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+                    tweet = await tweet.json();
+                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+                } else return { error: 'ErrorTweetNSFW' }
+        }
+    }
+
+    if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) {
+        return { error: 'ErrorTweetUnavailable' }
+    }
+
+    let tweetResult = tweet.data.tweetResult.result,
+        baseTweet = tweetResult.legacy,
+        repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
+
+    if (tweetTypename === "TweetWithVisibilityResults") {
+        baseTweet = tweetResult.tweet.legacy;
+        repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
+    }
+
+    let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
+
+    // check if there's a video at given index (/video/<index>)
+    if (index >= 0 && index < media?.length) {
+        media = [media[index]]
+    }
+
+    switch (media?.length) {
+        case undefined:
+        case 0:
+            return { error: 'ErrorNoVideosInTweet' };
+        case 1:
+            if (media[0].type === "photo") {
+                return {
+                    type: "normal",
+                    isPhoto: true,
+                    urls: `${media[0].media_url_https}?name=4096x4096`
+                }
+            }
+
+            return {
+                type: needsFixing(media[0]) ? "remux" : "normal",
+                urls: bestQuality(media[0].video_info.variants),
+                filename: `twitter_${id}.mp4`,
+                audioFilename: `twitter_${id}_audio`,
+                isGif: media[0].type === "animated_gif"
+            }
+        default:
+            const picker = media.map((content, i) => {
+                if (content.type === "photo") {
+                    let url = `${content.media_url_https}?name=4096x4096`;
+                    return {
+                        type: "photo",
+                        url,
+                        thumb: url,
+                    }
+                }
+
+                let url = bestQuality(content.video_info.variants);
+                const shouldRenderGif = content.type === 'animated_gif' && toGif;
+
+                let type = "video";
+                if (shouldRenderGif) type = "gif";
+
+                if (needsFixing(content) || shouldRenderGif) {
+                    url = createStream({
+                        service: 'twitter',
+                        type: shouldRenderGif ? 'gif' : 'remux',
+                        u: url,
+                        filename: `twitter_${id}_${i + 1}.mp4`
+                    })
+                }
+
+                return {
+                    type,
+                    url,
+                    thumb: content.media_url_https
+                }
+            });
+            return { picker };
+    }
+}

api/src/processing/services/vimeo.js

@@ -0,0 +1,169 @@
+import { env } from "../../config.js";
+import { cleanString, merge } from '../../misc/utils.js';
+
+import HLS from "hls-parser";
+
+const resolutionMatch = {
+    "3840": 2160,
+    "2732": 1440,
+    "2560": 1440,
+    "2048": 1080,
+    "1920": 1080,
+    "1366": 720,
+    "1280": 720,
+    "960": 480,
+    "640": 360,
+    "426": 240
+}
+
+const requestApiInfo = (videoId, password) => {
+    if (password) {
+        videoId += `:${password}`
+    }
+
+    return fetch(
+        `https://api.vimeo.com/videos/${videoId}`,
+        {
+            headers: {
+                Accept: 'application/vnd.vimeo.*+json; version=3.4.2',
+                'User-Agent': 'Vimeo/10.19.0 (com.vimeo; build:101900.57.0; iOS 17.5.1) Alamofire/5.9.0 VimeoNetworking/5.0.0',
+                Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
+                'Accept-Language': 'en'
+            }
+        }
+    )
+    .then(a => a.json())
+    .catch(() => {});
+}
+
+const compareQuality = (rendition, requestedQuality) => {
+    const quality = parseInt(rendition);
+    return Math.abs(quality - requestedQuality);
+}
+
+const getDirectLink = (data, quality) => {
+    if (!data.files) return;
+
+    const match = data.files
+        .filter(f => f.rendition?.endsWith('p'))
+        .reduce((prev, next) => {
+            const delta = {
+                prev: compareQuality(prev.rendition, quality),
+                next: compareQuality(next.rendition, quality)
+            };
+
+            return delta.prev < delta.next ? prev : next;
+        });
+
+    if (!match) return;
+
+    return {
+        urls: match.link,
+        filenameAttributes: {
+            resolution: `${match.width}x${match.height}`,
+            qualityLabel: match.rendition,
+            extension: "mp4"
+        }
+    }
+}
+
+const getHLS = async (configURL, obj) => {
+    if (!configURL) return;
+
+    const api = await fetch(configURL)
+                    .then(r => r.json())
+                    .catch(() => {});
+    if (!api) return { error: 'ErrorCouldntFetch' };
+
+    if (api.video?.duration > env.durationLimit) {
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+    }
+
+    const urlMasterHLS = api.request?.files?.hls?.cdns?.akfire_interconnect_quic?.url;
+    if (!urlMasterHLS) return { error: 'ErrorCouldntFetch' }
+
+    const masterHLS = await fetch(urlMasterHLS)
+                            .then(r => r.text())
+                            .catch(() => {});
+
+    if (!masterHLS) return { error: 'ErrorCouldntFetch' };
+
+    const variants = HLS.parse(masterHLS)?.variants?.sort(
+        (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+    );
+    if (!variants || variants.length === 0) return { error: 'ErrorEmptyDownload' };
+
+    let bestQuality;
+
+    if (obj.quality < resolutionMatch[variants[0]?.resolution?.width]) {
+        bestQuality = variants.find(v =>
+            (obj.quality === resolutionMatch[v.resolution.width])
+        );
+    }
+
+    if (!bestQuality) bestQuality = variants[0];
+
+    const expandLink = (path) => {
+        return new URL(path, urlMasterHLS).toString();
+    };
+
+    let urls = expandLink(bestQuality.uri);
+
+    const audioPath = bestQuality?.audio[0]?.uri;
+    if (audioPath) {
+        urls = [
+            urls,
+            expandLink(audioPath)
+        ]
+    } else if (obj.isAudioOnly) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    return {
+        urls,
+        isM3U8: true,
+        filenameAttributes: {
+            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
+            qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
+            extension: "mp4"
+        }
+    }
+}
+
+export default async function(obj) {
+    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
+    if (quality < 240) quality = 240;
+    if (!quality || obj.isAudioOnly) quality = 9000;
+
+    const info = await requestApiInfo(obj.id, obj.password);
+    let response;
+
+    if (obj.isAudioOnly) {
+        response = await getHLS(info.config_url, { ...obj, quality });
+    }
+
+    if (!response) response = getDirectLink(info, quality);
+    if (!response) response = { error: 'ErrorEmptyDownload' };
+
+    if (response.error) {
+        return response;
+    }
+
+    const fileMetadata = {
+        title: cleanString(info.name),
+        artist: cleanString(info.user.name),
+    };
+
+    return merge(
+        {
+            fileMetadata,
+            filenameAttributes: {
+                service: "vimeo",
+                id: obj.id,
+                title: fileMetadata.title,
+                author: fileMetadata.artist,
+            }
+        },
+        response
+    );
+}

api/src/processing/services/vine.js

@@ -0,0 +1,15 @@
+export default async function(obj) {
+    let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`)
+                    .then(r => r.json())
+                    .catch(() => {});
+
+    if (!post) return { error: 'ErrorEmptyDownload' };
+
+    if (post.videoUrl) return {
+        urls: post.videoUrl.replace("http://", "https://"),
+        filename: `vine_${obj.id}.mp4`,
+        audioFilename: `vine_${obj.id}_audio`
+    }
+
+    return { error: 'ErrorEmptyDownload' }
+}

api/src/processing/services/vk.js

@@ -0,0 +1,55 @@
+import { genericUserAgent, env } from "../../config.js";
+import { cleanString } from "../../misc/utils.js";
+
+const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"];
+
+export default async function(o) {
+    let html, url, quality = o.quality === "max" ? 2160 : o.quality;
+
+    html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, {
+        headers: { "user-agent": genericUserAgent }
+    }).then(r => r.arrayBuffer()).catch(() => {});
+
+    if (!html) return { error: 'ErrorCouldntFetch' };
+
+    // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times
+    let decoder = new TextDecoder('windows-1251');
+    html = decoder.decode(html);
+
+    if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' };
+
+    let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
+
+    if (Number(js.mvData.is_active_live) !== 0) return { error: 'ErrorLiveVideo' };
+    if (js.mvData.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+
+    for (let i in resolutions) {
+        if (js.player.params[0][`url${resolutions[i]}`]) {
+            quality = resolutions[i];
+            break
+        }
+    }
+    if (Number(quality) > Number(o.quality)) quality = o.quality;
+
+    url = js.player.params[0][`url${quality}`];
+
+    let fileMetadata = {
+        title: cleanString(js.player.params[0].md_title.trim()),
+        author: cleanString(js.player.params[0].md_author.trim()),
+    }
+
+    if (url) return {
+        urls: url,
+        filenameAttributes: {
+            service: "vk",
+            id: `${o.userId}_${o.videoId}`,
+            title: fileMetadata.title,
+            author: fileMetadata.author,
+            resolution: `${quality}p`,
+            qualityLabel: `${quality}p`,
+            extension: "mp4"
+        }
+    }
+    return { error: 'ErrorEmptyDownload' }
+}

api/src/processing/services/youtube.js

@@ -0,0 +1,290 @@
+import { fetch } from "undici";
+
+import { Innertube, Session } from "youtubei.js";
+
+import { env } from "../../config.js";
+import { cleanString } from "../../misc/utils.js";
+import { getCookie, updateCookieValues } from "../cookie/manager.js";
+
+const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
+
+let innertube, lastRefreshedAt;
+
+const codecMatch = {
+    h264: {
+        videoCodec: "avc1",
+        audioCodec: "mp4a",
+        container: "mp4"
+    },
+    av1: {
+        videoCodec: "av01",
+        audioCodec: "mp4a",
+        container: "mp4"
+    },
+    vp9: {
+        videoCodec: "vp9",
+        audioCodec: "opus",
+        container: "webm"
+    }
+}
+
+const transformSessionData = (cookie) => {
+    if (!cookie)
+        return;
+
+    const values = { ...cookie.values() };
+    const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];
+
+    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
+        return;
+    }
+
+    if (values.expires) {
+        values.expiry_date = values.expires;
+        delete values.expires;
+    } else if (!values.expiry_date) {
+        return;
+    }
+
+    return values;
+}
+
+const cloneInnertube = async (customFetch) => {
+    const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
+    if (!innertube || shouldRefreshPlayer) {
+        innertube = await Innertube.create({
+            fetch: customFetch
+        });
+        lastRefreshedAt = +new Date();
+    }
+
+    const session = new Session(
+        innertube.session.context,
+        innertube.session.key,
+        innertube.session.api_version,
+        innertube.session.account_index,
+        innertube.session.player,
+        undefined,
+        customFetch ?? innertube.session.http.fetch,
+        innertube.session.cache
+    );
+
+    const cookie = getCookie('youtube_oauth');
+    const oauthData = transformSessionData(cookie);
+
+    if (!session.logged_in && oauthData) {
+        await session.oauth.init(oauthData);
+        session.logged_in = true;
+    }
+
+    if (session.logged_in) {
+        if (session.oauth.shouldRefreshToken()) {
+            await session.oauth.refreshAccessToken();
+        }
+
+        const cookieValues = cookie.values();
+        const oldExpiry = new Date(cookieValues.expiry_date);
+        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
+
+        if (oldExpiry.getTime() !== newExpiry.getTime()) {
+            updateCookieValues(cookie, {
+                ...session.oauth.client_id,
+                ...session.oauth.oauth2_tokens,
+                expiry_date: newExpiry.toISOString()
+            });
+        }
+    }
+
+    const yt = new Innertube(session);
+    return yt;
+}
+
+export default async function(o) {
+    let yt;
+    try {
+        yt = await cloneInnertube(
+            (input, init) => fetch(input, {
+                ...init,
+                dispatcher: o.dispatcher
+            })
+        );
+    } catch(e) {
+        if (e.message?.endsWith("decipher algorithm")) {
+                return { error: "ErrorYoutubeDecipher" }
+        } else throw e;
+    }
+
+    const quality = o.quality === "max" ? "9000" : o.quality;
+
+    let info, isDubbed,
+        format = o.format || "h264";
+
+    function qual(i) {
+        if (!i.quality_label) {
+            return;
+        }
+
+        return i.quality_label.split('p')[0].split('s')[0]
+    }
+
+    try {
+        info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS');
+    } catch(e) {
+        if (e?.message === 'This video is unavailable') {
+            return { error: 'ErrorCouldntFetch' };
+        } else {
+            return { error: 'ErrorCantConnectToServiceAPI' };
+        }
+    }
+
+    if (!info) return { error: 'ErrorCantConnectToServiceAPI' };
+
+    const playability = info.playability_status;
+    const basicInfo = info.basic_info;
+
+    if (playability.status === 'LOGIN_REQUIRED') {
+        if (playability.reason.endsWith('bot')) {
+            return { error: 'ErrorYTLogin' }
+        }
+        if (playability.reason.endsWith('age')) {
+            return { error: 'ErrorYTAgeRestrict' }
+        }
+    }
+    if (playability.status === "UNPLAYABLE" && playability.reason.endsWith('request limit.')) {
+        return { error: 'ErrorYTRateLimit' }
+    }
+
+    if (playability.status !== 'OK') return { error: 'ErrorYTUnavailable' };
+    if (basicInfo.is_live) return { error: 'ErrorLiveVideo' };
+
+    // return a critical error if returned video is "Video Not Available"
+    // or a similar stub by youtube
+    if (basicInfo.id !== o.id) {
+        return {
+            error: 'ErrorCantConnectToServiceAPI',
+            critical: true
+        }
+    }
+
+    let bestQuality, hasAudio;
+
+    const filterByCodec = (formats) =>
+        formats
+        .filter(e =>
+            e.mime_type.includes(codecMatch[format].videoCodec)
+            || e.mime_type.includes(codecMatch[format].audioCodec)
+        )
+        .sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
+
+    let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats);
+
+    if (adaptive_formats.length === 0 && format === "vp9") {
+        format = "h264"
+        adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats)
+    }
+
+    bestQuality = adaptive_formats.find(i => i.has_video && i.content_length);
+    hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length);
+
+    if (bestQuality) bestQuality = qual(bestQuality);
+
+    if ((!bestQuality && !o.isAudioOnly) || !hasAudio)
+        return { error: 'ErrorYTTryOtherCodec' };
+
+    if (basicInfo.duration > env.durationLimit)
+        return { error: ['ErrorLengthLimit', env.durationLimit / 60] };
+
+    const checkBestAudio = (i) => (i.has_audio && !i.has_video);
+
+    let audio = adaptive_formats.find(i =>
+        checkBestAudio(i) && i.is_original
+    );
+
+    if (o.dubLang) {
+        let dubbedAudio = adaptive_formats.find(i =>
+            checkBestAudio(i)
+            && i.language === o.dubLang
+            && i.audio_track
+        )
+
+        if (dubbedAudio) {
+            audio = dubbedAudio;
+            isDubbed = true;
+        }
+    }
+
+    if (!audio) {
+        audio = adaptive_formats.find(i => checkBestAudio(i));
+    }
+
+    let fileMetadata = {
+        title: cleanString(basicInfo.title.trim()),
+        artist: cleanString(basicInfo.author.replace("- Topic", "").trim()),
+    }
+
+    if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) {
+        let descItems = basicInfo.short_description.split("\n\n");
+        fileMetadata.album = descItems[2];
+        fileMetadata.copyright = descItems[3];
+        if (descItems[4].startsWith("Released on:")) {
+            fileMetadata.date = descItems[4].replace("Released on: ", '').trim()
+        }
+    }
+
+    let filenameAttributes = {
+        service: "youtube",
+        id: o.id,
+        title: fileMetadata.title,
+        author: fileMetadata.artist,
+        youtubeDubName: isDubbed ? o.dubLang : false
+    }
+
+    if (audio && o.isAudioOnly) return {
+        type: "render",
+        isAudioOnly: true,
+        urls: audio.decipher(yt.session.player),
+        filenameAttributes: filenameAttributes,
+        fileMetadata: fileMetadata,
+        bestAudio: format === "h264" ? "m4a" : "opus"
+    }
+
+    const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality,
+        checkSingle = i =>
+            qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec),
+        checkRender = i =>
+            qual(i) === matchingQuality && i.has_video && !i.has_audio;
+
+    let match, type, urls;
+
+    if (!o.isAudioOnly && !o.isAudioMuted && format === 'h264') {
+        match = info.streaming_data.formats.find(checkSingle);
+        type = "bridge";
+        urls = match?.decipher(yt.session.player);
+    }
+
+    const video = adaptive_formats.find(checkRender);
+
+    if (!match && video && audio) {
+        match = video;
+        type = "render";
+        urls = [
+            video.decipher(yt.session.player),
+            audio.decipher(yt.session.player)
+        ]
+    }
+
+    if (match) {
+        filenameAttributes.qualityLabel = match.quality_label;
+        filenameAttributes.resolution = `${match.width}x${match.height}`;
+        filenameAttributes.extension = codecMatch[format].container;
+        filenameAttributes.youtubeFormat = format;
+        return {
+            type,
+            urls,
+            filenameAttributes,
+            fileMetadata
+        }
+    }
+
+    return { error: 'ErrorYTTryOtherCodec' }
+}

api/src/processing/url.js

@@ -0,0 +1,183 @@
+import { services } from "../config.js";
+import { strict as assert } from "node:assert";
+import psl from "psl";
+
+function aliasURL(url) {
+    assert(url instanceof URL);
+
+    const host = psl.parse(url.hostname);
+    const parts = url.pathname.split('/');
+
+    switch (host.sld) {
+        case "youtube":
+            if (url.pathname.startsWith('/live/') || url.pathname.startsWith('/shorts/')) {
+                url.pathname = '/watch';
+                // parts := ['', 'live' || 'shorts', id, ...rest]
+                url.search = `?v=${encodeURIComponent(parts[2])}`
+            }
+            break;
+
+        case "youtu":
+            if (url.hostname === 'youtu.be' && parts.length >= 2) {
+                /* youtu.be urls can be weird, e.g. https://youtu.be/<id>//asdasd// still works
+                ** but we only care about the 1st segment of the path */
+                url = new URL(`https://youtube.com/watch?v=${
+                    encodeURIComponent(parts[1])
+                }`)
+            }
+            break;
+
+        case "pin":
+            if (url.hostname === 'pin.it' && parts.length === 2) {
+                url = new URL(`https://pinterest.com/url_shortener/${
+                    encodeURIComponent(parts[1])
+                }`)
+            }
+            break;
+
+        case "vxtwitter":
+        case "fixvx":
+        case "x":
+            if (services.twitter.altDomains.includes(url.hostname)) {
+                url.hostname = 'twitter.com'
+            }
+            break;
+
+        case "twitch":
+            if (url.hostname === 'clips.twitch.tv' && parts.length >= 2) {
+                url = new URL(`https://twitch.tv/_/clip/${parts[1]}`);
+            }
+            break;
+
+        case "bilibili":
+            if (host.tld === 'tv') {
+                url = new URL(`https://bilibili.com/_tv${url.pathname}`);
+            }
+            break;
+
+        case "b23":
+            if (url.hostname === 'b23.tv' && parts.length === 2) {
+                url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`)
+            }
+            break;
+
+        case "dai":
+            if (url.hostname === 'dai.ly' && parts.length === 2) {
+                url = new URL(`https://dailymotion.com/video/${parts[1]}`)
+            }
+
+        case "facebook":
+        case "fb":
+            if (url.searchParams.get('v')) {
+                url = new URL(`https://web.facebook.com/user/videos/${url.searchParams.get('v')}`)
+            }
+            if (url.hostname === 'fb.watch') {
+                url = new URL(`https://web.facebook.com/_shortLink/${parts[1]}`)
+            }
+            break;
+
+        case "ddinstagram":
+            if (services.instagram.altDomains.includes(host.domain) && [null, 'd', 'g'].includes(host.subdomain)) {
+                url.hostname = 'instagram.com';
+            }
+            break;
+    }
+
+    return url
+}
+
+function cleanURL(url) {
+    assert(url instanceof URL);
+    const host = psl.parse(url.hostname).sld;
+
+    let stripQuery = true;
+
+    const limitQuery = (param) => {
+        url.search = `?${param}=` + encodeURIComponent(url.searchParams.get(param));
+        stripQuery = false;
+    }
+
+    switch (host) {
+        case "pinterest":
+            url.hostname = 'pinterest.com';
+            break;
+        case "vk":
+            if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
+                limitQuery('z')
+            }
+            break;
+        case "youtube":
+            if (url.searchParams.get('v')) {
+                limitQuery('v')
+            }
+            break;
+        case "rutube":
+            if (url.searchParams.get('p')) {
+                limitQuery('p')
+            }
+            break;
+    }
+
+    if (stripQuery) {
+        url.search = ''
+    }
+
+    url.username = url.password = url.port = url.hash = ''
+
+    if (url.pathname.endsWith('/'))
+        url.pathname = url.pathname.slice(0, -1);
+
+    return url
+}
+
+function getHostIfValid(url) {
+    const host = psl.parse(url.hostname);
+    if (host.error) return;
+
+    const service = services[host.sld];
+    if (!service) return;
+    if ((service.tld ?? 'com') !== host.tld) return;
+
+    const anySubdomainAllowed = service.subdomains === '*';
+    const validSubdomain = [null, 'www', ...(service.subdomains ?? [])].includes(host.subdomain);
+    if (!validSubdomain && !anySubdomainAllowed) return;
+
+    return host.sld;
+}
+
+export function normalizeURL(url) {
+    return cleanURL(
+        aliasURL(
+            new URL(url.replace(/^https\/\//, 'https://'))
+        )
+    );
+}
+
+export function extract(url) {
+    if (!(url instanceof URL)) {
+        url = new URL(url);
+    }
+
+    const host = getHostIfValid(url);
+
+    if (!host || !services[host].enabled) {
+        return null;
+    }
+
+    let patternMatch;
+    for (const pattern of services[host].patterns) {
+        patternMatch = pattern.match(
+            url.pathname.substring(1) + url.search
+        );
+
+        if (patternMatch) {
+            break;
+        }
+    }
+
+    if (!patternMatch) {
+        return null;
+    }
+
+    return { host, patternMatch };
+}