api/services/youtube: force player_id

temporary workaround

.github/workflows/test-services.yml

@@ -30,7 +30,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-      - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }}
+      - run: pnpm i --frozen-lockfile
+      - run: node api/src/util/test run-tests-for ${{ matrix.service }}
         env:
-          API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
+          HTTP_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
           TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }}

README.md

@@ -29,6 +29,20 @@ cobalt is a media downloader that doesn't piss you off. it's friendly, efficient
 
 paste the link, get the file, move on. that simple, just how it should be.
 
+### sponsors
+<div align="center" markdown="1">
+    <sup>special thanks to Warp for sponsoring the development of cobalt</sup>
+    <br>
+    <a href="https://go.warp.dev/cobalt">
+        <img alt="Warp banner" width="400" src="https://raw.githubusercontent.com/warpdotdev/brand-assets/be7d584f98e62b1579fd2e9338d4c7318a732f1b/Github/Sponsor/Warp-Github-LG-03.png">
+    </a>
+
+### [Warp, built for coding with multiple AI agents](https://go.warp.dev/cobalt)
+</div>
+
+#### RoyaleHosting
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), and a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
+
 ### cobalt monorepo
 this monorepo includes source code for api, frontend, and related packages:
 - [api tree & readme](/api/)
@@ -53,9 +67,6 @@ same content can be downloaded via dev tools of any modern web browser.
 ### contributing
 if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
 
-### thank you
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
-
 ### licenses
 for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
 unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).

api/README.md

@@ -23,6 +23,7 @@ if the desired service isn't supported yet, feel free to create an appropriate i
 | instagram         | ✅            | ✅         | ✅         | ➖         | ➖              |
 | facebook          | ✅            | ❌         | ✅         | ➖         | ➖              |
 | loom              | ✅            | ❌         | ✅         | ✅         | ➖              |
+| newgrounds        | ✅            | ✅         | ✅         | ✅         | ✅              |
 | ok.ru             | ✅            | ❌         | ✅         | ✅         | ✅              |
 | pinterest         | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit            | ✅            | ✅         | ✅         | ❌         | ❌              |

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "11.2.2",
+    "version": "11.5",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -26,7 +26,6 @@
         "@datastructures-js/priority-queue": "^6.3.1",
         "@imput/psl": "^2.0.4",
         "@imput/version-info": "workspace:^",
-        "@imput/youtubei.js": "^14.0.0",
         "content-disposition-header": "0.6.0",
         "cors": "^2.8.5",
         "dotenv": "^16.0.1",
@@ -40,6 +39,7 @@
         "set-cookie-parser": "2.6.0",
         "undici": "^6.21.3",
         "url-pattern": "1.0.3",
+        "youtubei.js": "15.1.1",
         "zod": "^3.23.8"
     },
     "optionalDependencies": {

api/src/config.js

@@ -3,12 +3,12 @@ import { loadEnvs, validateEnvs } from "./core/env.js";
 
 const version = await getVersion();
 
+const canonicalEnv = Object.freeze(structuredClone(process.env));
 const env = loadEnvs();
 
 const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
 const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
 
-export const canonicalEnv = Object.freeze(structuredClone(process.env));
 export const setTunnelPort = (port) => env.tunnelPort = port;
 export const isCluster = env.instanceCount > 1;
 export const updateEnv = (newEnv) => {
@@ -18,6 +18,10 @@ export const updateEnv = (newEnv) => {
     newEnv.tunnelPort = env.tunnelPort;
 
     for (const key in env) {
+        if (key === 'subscribe') {
+            continue;
+        }
+
         if (String(env[key]) !== String(newEnv[key])) {
             changes.push(key);
         }
@@ -31,6 +35,7 @@ await validateEnvs(env);
 
 export {
     env,
+    canonicalEnv,
     genericUserAgent,
     cobaltUserAgent,
 }

api/src/core/api.js

@@ -1,7 +1,7 @@
 import cors from "cors";
 import http from "node:http";
 import rateLimit from "express-rate-limit";
-import { setGlobalDispatcher, ProxyAgent } from "undici";
+import { setGlobalDispatcher, EnvHttpProxyAgent } from "undici";
 import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
 
 import jwt from "../security/jwt.js";
@@ -337,10 +337,18 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     randomizeCiphers();
     setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
 
+    env.subscribe(['externalProxy', 'httpProxyValues'], () => {
+        // TODO: remove env.externalProxy in a future version
+        const options = {};
         if (env.externalProxy) {
-        setGlobalDispatcher(new ProxyAgent(env.externalProxy))
+            options.httpProxy = env.externalProxy;
         }
 
+        setGlobalDispatcher(
+            new EnvHttpProxyAgent(options)
+        );
+    });
+
     http.createServer(app).listen({
         port: env.apiPort,
         host: env.listenAddress,

api/src/core/env.js

@@ -1,4 +1,4 @@
-import { Constants } from "@imput/youtubei.js";
+import { Constants } from "youtubei.js";
 import { services } from "../processing/service-config.js";
 import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";
 
@@ -10,6 +10,34 @@ import { Green, Yellow } from "../misc/console-text.js";
 const forceLocalProcessingOptions = ["never", "session", "always"];
 const youtubeHlsOptions = ["never", "key", "always"];
 
+const httpProxyVariables = ["NO_PROXY", "HTTP_PROXY", "HTTPS_PROXY"].flatMap(
+    k => [ k, k.toLowerCase() ]
+);
+
+const changeCallbacks = {};
+
+const onEnvChanged = (changes) => {
+    for (const key of changes) {
+        if (changeCallbacks[key]) {
+            changeCallbacks[key].map(fn => {
+                try { fn() } catch {}
+            });
+        }
+    }
+}
+
+const subscribe = (keys, fn) => {
+    keys = [keys].flat();
+
+    for (const key of keys) {
+        if (key in currentEnv && key !== 'subscribe') {
+            changeCallbacks[key] ??= [];
+            changeCallbacks[key].push(fn);
+            fn();
+        } else throw `invalid env key ${key}`;
+    }
+}
+
 export const loadEnvs = (env = process.env) => {
     const allServices = new Set(Object.keys(services));
     const disabledServices = env.DISABLED_SERVICES?.split(',') || [];
@@ -19,6 +47,18 @@ export const loadEnvs = (env = process.env) => {
         }
     }));
 
+    // we need to copy the proxy envs (HTTP_PROXY, HTTPS_PROXY)
+    // back into process.env, so that EnvHttpProxyAgent can pick
+    // them up later
+    for (const key of httpProxyVariables) {
+        const value = env[key] ?? canonicalEnv[key];
+        if (value !== undefined) {
+            process.env[key] = env[key];
+        } else {
+            delete process.env[key];
+        }
+    }
+
     return {
         apiURL: env.API_URL || '',
         apiPort: env.API_PORT || 9000,
@@ -55,6 +95,9 @@ export const loadEnvs = (env = process.env) => {
 
         externalProxy: env.API_EXTERNAL_PROXY,
 
+        // used only for comparing against old values when envs are being updated
+        httpProxyValues: httpProxyVariables.map(k => String(env[k])).join(''),
+
         turnstileSitekey: env.TURNSTILE_SITEKEY,
         turnstileSecret: env.TURNSTILE_SECRET,
         jwtSecret: env.JWT_SECRET,
@@ -87,9 +130,13 @@ export const loadEnvs = (env = process.env) => {
 
         envFile: env.API_ENV_FILE,
         envRemoteReloadInterval: 300,
+
+        subscribe,
     };
 }
 
+let loggedProxyWarning = false;
+
 export const validateEnvs = async (env) => {
     if (env.sessionEnabled && env.jwtSecret.length < 16) {
         throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)");
@@ -127,6 +174,15 @@ export const validateEnvs = async (env) => {
         throw new Error('freebind is not available when external proxy is enabled')
     }
 
+    if (env.externalProxy && !loggedProxyWarning) {
+        console.error('API_EXTERNAL_PROXY is deprecated and will be removed in a future release.');
+        console.error('Use HTTP_PROXY or HTTPS_PROXY instead.');
+        console.error('You can read more about the new proxy variables in docs/api-env-variables.md\n');
+
+        // prevent the warning from being printed on every env validation
+        loggedProxyWarning = true;
+    }
+
     return env;
 }
 
@@ -170,11 +226,14 @@ const wrapReload = (contents) => {
             return;
         }
 
+        onEnvChanged(changes);
+
         console.log(`${Green('[✓]')} envs reloaded successfully!`);
         for (const key of changes) {
             const value = currentEnv[key];
             const isSecret = key.toLowerCase().includes('apikey')
-                          || key.toLowerCase().includes('secret');
+                          || key.toLowerCase().includes('secret')
+                          || key === 'httpProxyValues';
 
             if (!value) {
                 console.log(`    removed: ${key}`);

api/src/misc/utils.js

@@ -1,4 +1,4 @@
-import { request } from 'undici';
+import { request } from "undici";
 const redirectStatuses = new Set([301, 302, 303, 307, 308]);
 
 export async function getRedirectingURL(url, dispatcher, headers) {
@@ -8,18 +8,34 @@ export async function getRedirectingURL(url, dispatcher, headers) {
         headers,
         redirect: 'manual'
     };
+    const getParams = {
+        ...params,
+        method: 'GET',
+    };
 
-    let location = await request(url, params).then(r => {
+    const callback = (r) => {
         if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
             return r.headers['location'];
         }
-    }).catch(() => null);
-
-    location ??= await fetch(url, params).then(r => {
-        if (redirectStatuses.has(r.status) && r.headers.has('location')) {
-            return r.headers.get('location');
     }
-    }).catch(() => null);
+
+    /*
+        try request() with HEAD & GET,
+        then do the same with fetch
+        (fetch is required for shortened reddit links)
+    */
+
+    let location = await request(url, params)
+        .then(callback).catch(() => null);
+
+    location ??= await request(url, getParams)
+        .then(callback).catch(() => null);
+
+    location ??= await fetch(url, params)
+        .then(callback).catch(() => null);
+
+    location ??= await fetch(url, getParams)
+        .then(callback).catch(() => null);
 
     return location;
 }

api/src/processing/cookie/manager.js

@@ -13,6 +13,7 @@ const VALID_SERVICES = new Set([
     'reddit',
     'twitter',
     'youtube',
+    'vimeo_bearer',
 ]);
 
 const invalidCookies = {};

api/src/processing/match-action.js

@@ -180,6 +180,7 @@ export default function({
 
                 case "ok":
                 case "xiaohongshu":
+                case "newgrounds":
                     params = { type: "proxy" };
                     break;
 

api/src/processing/match.js

@@ -29,7 +29,7 @@ import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
 import xiaohongshu from "./services/xiaohongshu.js";
-import nicovideo from "./services/nicovideo.js";
+import newgrounds from "./services/newgrounds.js";
 
 let freebind;
 
@@ -269,10 +269,9 @@ export default async function({ host, patternMatch, params, authType }) {
                 });
                 break;
 
-            case "nicovideo":
-                r = await nicovideo({
+            case "newgrounds":
+                r = await newgrounds({
                     ...patternMatch,
-                    dispatcher,
                     quality: params.videoQuality,
                 });
                 break;
@@ -323,7 +322,7 @@ export default async function({ host, patternMatch, params, authType }) {
         let localProcessing = params.localProcessing;
         const lpEnv = env.forceLocalProcessing;
         const shouldForceLocal = lpEnv === "always" || (lpEnv === "session" && authType === "session");
-        const localDisabled = (!localProcessing || localProcessing === "none");
+        const localDisabled = (!localProcessing || localProcessing === "disabled");
 
         if (shouldForceLocal && localDisabled) {
             localProcessing = "preferred";

api/src/processing/service-config.js

@@ -7,6 +7,7 @@ export const services = {
     bilibili: {
         patterns: [
             "video/:comId",
+            "video/:comId?p=:partId",
             "_shortLink/:comShortLink",
             "_tv/:lang/video/:tvId",
             "_tv/video/:tvId"
@@ -60,11 +61,6 @@ export const services = {
     loom: {
         patterns: ["share/:id", "embed/:id"],
     },
-    nicovideo: {
-        patterns: ["watch/:id"],
-        tld: "jp",
-        subdomains: ["sp"],
-    },
     ok: {
         patterns: [
             "video/:id",
@@ -79,6 +75,12 @@ export const services = {
             "url_shortener/:shortLink"
         ],
     },
+    newgrounds: {
+        patterns: [
+            "portal/view/:id",
+            "audio/listen/:audioId",
+        ]
+    },
     reddit: {
         patterns: [
             "comments/:id",
@@ -164,6 +166,7 @@ export const services = {
     twitch: {
         patterns: [":channel/clip/:clip"],
         tld: "tv",
+        subdomains: ["clips", "www", "m"],
     },
     twitter: {
         patterns: [
@@ -206,7 +209,7 @@ export const services = {
         patterns: [
             "explore/:id?xsec_token=:token",
             "discovery/item/:id?xsec_token=:token",
-            "a/:shareId"
+            ":shareType/:shareId",
         ],
         altDomains: ["xhslink.com"],
     },

api/src/processing/service-patterns.js

@@ -1,53 +1,72 @@
 export const testers = {
     "bilibili": pattern =>
-        pattern.comId?.length <= 12 || pattern.comShortLink?.length <= 16
-        || pattern.tvId?.length <= 24,
+        (pattern.comId?.length <= 12 && pattern.partId?.length <= 3) ||
+        (pattern.comId?.length <= 12 && !pattern.partId) ||
+        pattern.comShortLink?.length <= 16 ||
+        pattern.tvId?.length <= 24,
+
+    "bsky": pattern =>
+        pattern.user?.length <= 128 && pattern.post?.length <= 128,
 
     "dailymotion": pattern => pattern.id?.length <= 32,
 
+    "facebook": pattern =>
+        pattern.shortLink?.length <= 11 ||
+        pattern.username?.length <= 30 ||
+        pattern.caption?.length <= 255 ||
+        pattern.id?.length <= 20 && !pattern.shareType ||
+        pattern.id?.length <= 20 && pattern.shareType?.length === 1,
+
     "instagram": pattern =>
-        pattern.postId?.length <= 48
-        || pattern.shareId?.length <= 16
-        || (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
+        pattern.postId?.length <= 48 ||
+        pattern.shareId?.length <= 16 ||
+        (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
 
     "loom": pattern =>
         pattern.id?.length <= 32,
 
+    "newgrounds": pattern =>
+        pattern.id?.length <= 12 ||
+        pattern.audioId?.length <= 12,
+
     "ok": pattern =>
         pattern.id?.length <= 16,
 
     "pinterest": pattern =>
-        pattern.id?.length <= 128 || pattern.shortLink?.length <= 32,
+        pattern.id?.length <= 128 ||
+        pattern.shortLink?.length <= 32,
 
     "reddit": pattern =>
-        pattern.id?.length <= 16 && !pattern.sub && !pattern.user
-        || (pattern.sub?.length <= 22 && pattern.id?.length <= 16)
-        || (pattern.user?.length <= 22 && pattern.id?.length <= 16)
-        || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16)
-        || (pattern.shortId?.length <= 16),
+        pattern.id?.length <= 16 && !pattern.sub && !pattern.user ||
+        (pattern.sub?.length <= 22 && pattern.id?.length <= 16) ||
+        (pattern.user?.length <= 22 && pattern.id?.length <= 16) ||
+        (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16) ||
+        (pattern.shortId?.length <= 16),
 
     "rutube": pattern =>
         (pattern.id?.length === 32 && pattern.key?.length <= 32) ||
-        pattern.id?.length === 32 || pattern.yappyId?.length === 32,
-
-    "soundcloud": pattern =>
-        (pattern.author?.length <= 255 && pattern.song?.length <= 255)
-        || pattern.shortLink?.length <= 32,
+        pattern.id?.length === 32 ||
+        pattern.yappyId?.length === 32,
 
     "snapchat": pattern =>
-        (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255))
-        || pattern.spotlightId?.length <= 255
-        || pattern.shortLink?.length <= 16,
+        (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255)) ||
+        pattern.spotlightId?.length <= 255 ||
+        pattern.shortLink?.length <= 16,
+
+    "soundcloud": pattern =>
+        (pattern.author?.length <= 255 && pattern.song?.length <= 255) ||
+        pattern.shortLink?.length <= 32,
 
     "streamable": pattern =>
         pattern.id?.length <= 6,
 
     "tiktok": pattern =>
-        pattern.postId?.length <= 21 || pattern.shortLink?.length <= 21,
+        pattern.postId?.length <= 21 ||
+        pattern.shortLink?.length <= 21,
 
     "tumblr": pattern =>
-        pattern.id?.length < 21
-        || (pattern.id?.length < 21 && pattern.user?.length <= 32),
+        pattern.id?.length < 21 ||
+        (pattern.id?.length < 21 && pattern.user?.length <= 32),
 
     "twitch": pattern =>
         pattern.channel && pattern.clip?.length <= 100,
@@ -56,29 +75,16 @@ export const testers = {
         pattern.id?.length < 20,
 
     "vimeo": pattern =>
-        pattern.id?.length <= 11
-        && (!pattern.password || pattern.password.length < 16),
+        pattern.id?.length <= 11 && (!pattern.password || pattern.password.length < 16),
 
     "vk": pattern =>
         (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
         (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
 
+    "xiaohongshu": pattern =>
+        pattern.id?.length <= 24 && pattern.token?.length <= 64 ||
+        pattern.shareId?.length <= 24 && pattern.shareType?.length === 1,
+
     "youtube": pattern =>
         pattern.id?.length <= 11,
-
-    "facebook": pattern =>
-        pattern.shortLink?.length <= 11
-        || pattern.username?.length <= 30
-        || pattern.caption?.length <= 255
-        || pattern.id?.length <= 20 && !pattern.shareType
-        || pattern.id?.length <= 20 && pattern.shareType?.length === 1,
-
-    "bsky": pattern =>
-        pattern.user?.length <= 128 && pattern.post?.length <= 128,
-
-    "xiaohongshu": pattern =>
-        pattern.id?.length <= 24 && pattern.token?.length <= 64
-        || pattern.shareId?.length <= 24,
-
-    "nicovideo": pattern => pattern.id?.length <= 12,
 }

api/src/processing/services/bilibili.js

@@ -17,8 +17,14 @@ function extractBestQuality(dashData) {
     return [ bestVideo, bestAudio ];
 }
 
-async function com_download(id) {
-    const html = await fetch(`https://bilibili.com/video/${id}`, {
+async function com_download(id, partId) {
+    const url = new URL(`https://bilibili.com/video/${id}`);
+
+    if (partId) {
+        url.searchParams.set('p', partId);
+    }
+
+    const html = await fetch(url, {
         headers: {
             "user-agent": genericUserAgent
         }
@@ -47,10 +53,15 @@ async function com_download(id) {
         return { error: "fetch.empty" };
     }
 
+    let filenameBase = `bilibili_${id}`;
+    if (partId) {
+        filenameBase += `_${partId}`;
+    }
+
     return {
         urls: [video.baseUrl, audio.baseUrl],
-        audioFilename: `bilibili_${id}_audio`,
-        filename: `bilibili_${id}_${video.width}x${video.height}.mp4`,
+        audioFilename: `${filenameBase}_audio`,
+        filename: `${filenameBase}_${video.width}x${video.height}.mp4`,
     };
 }
 
@@ -89,14 +100,14 @@ async function tv_download(id) {
     };
 }
 
-export default async function({ comId, tvId, comShortLink }) {
+export default async function({ comId, tvId, comShortLink, partId }) {
     if (comShortLink) {
         const patternMatch = await resolveRedirectingURL(`https://b23.tv/${comShortLink}`);
         comId = patternMatch?.comId;
     }
 
     if (comId) {
-        return com_download(comId);
+        return com_download(comId, partId);
     } else if (tvId) {
         return tv_download(tvId);
     }

api/src/processing/services/newgrounds.js

@@ -0,0 +1,103 @@
+import { genericUserAgent } from "../../config.js";
+
+const getVideo = async ({ id, quality }) => {
+    const json = await fetch(`https://www.newgrounds.com/portal/video/${id}`, {
+        headers: {
+            "User-Agent": genericUserAgent,
+            "X-Requested-With": "XMLHttpRequest", // required to get the JSON response
+        }
+    })
+    .then(r => r.json())
+    .catch(() => {});
+
+    if (!json) return { error: "fetch.empty" };
+
+    const videoSources = json.sources;
+    const videoQualities = Object.keys(videoSources);
+
+    if (videoQualities.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    const bestVideo = videoSources[videoQualities[0]]?.[0],
+          userQuality = quality === "2160" ? "4k" : `${quality}p`,
+          preferredVideo = videoSources[userQuality]?.[0],
+          video = preferredVideo || bestVideo,
+          videoQuality = preferredVideo ? userQuality : videoQualities[0];
+
+    if (!bestVideo || !video.type.includes("mp4")) {
+        return { error: "fetch.empty" };
+    }
+
+    const fileMetadata = {
+        title: decodeURIComponent(json.title),
+        artist: decodeURIComponent(json.author),
+    }
+
+    return {
+        urls: video.src,
+        filenameAttributes: {
+            service: "newgrounds",
+            id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+            extension: "mp4",
+            qualityLabel: videoQuality,
+            resolution: videoQuality,
+        },
+        fileMetadata,
+    }
+}
+
+const getMusic = async ({ id }) => {
+    const html = await fetch(`https://www.newgrounds.com/audio/listen/${id}`, {
+        headers: {
+            "User-Agent": genericUserAgent,
+        }
+    })
+    .then(r => r.text())
+    .catch(() => {});
+
+    if (!html) return { error: "fetch.fail" };
+
+    const params = JSON.parse(
+        `{${html.split(',"params":{')[1]?.split(',"images":')[0]}}`
+    );
+    if (!params) return { error: "fetch.empty" };
+
+    if (!params.name || !params.artist || !params.filename || !params.icon) {
+        return { error: "fetch.empty" };
+    }
+
+    const fileMetadata = {
+        title: decodeURIComponent(params.name),
+        artist: decodeURIComponent(params.artist),
+    }
+
+    return {
+        urls: params.filename,
+        filenameAttributes: {
+            service: "newgrounds",
+            id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+        },
+        fileMetadata,
+        cover:
+            params.icon.includes(".png?") || params.icon.includes(".jpg?")
+                ? params.icon
+                : undefined,
+        isAudioOnly: true,
+        bestAudio: "mp3",
+    }
+}
+
+export default function({ id, audioId, quality }) {
+    if (id) {
+        return getVideo({ id, quality });
+    } else if (audioId) {
+        return getMusic({ id: audioId });
+    }
+
+    return { error: "fetch.empty" };
+}

api/src/processing/services/nicovideo.js

@@ -1,120 +0,0 @@
-import { genericUserAgent } from "../../config.js";
-
-const genericHeaders = {
-    "user-agent": genericUserAgent,
-    "accept-language": "en-US,en;q=0.9",
-}
-
-const getVideoInfo = async (id, dispatcher, quality) => {
-    const html = await fetch(`https://www.nicovideo.jp/watch/${id}`, {
-        dispatcher,
-        headers: genericHeaders
-    }).then(r => r.text()).catch(() => {});
-
-    if (!(html.includes("accessRightKey")
-            || !(html.includes('<meta name="server-response" content="')))) {
-        return { error: "fetch.fail" };
-    }
-
-    const rawContent =
-        html.split('<meta name="server-response" content="')[1]
-        .split('" />')[0]
-        .replaceAll("&quot;", '"');
-
-    const data = JSON.parse(rawContent)?.data?.response;
-
-    if (!data) {
-        return { error: "fetch.fail" };
-    }
-
-    const audio = data.media?.domand?.audios.find(audio => audio.isAvailable);
-    const bestVideo = data.media?.domand?.videos.find(video => video.isAvailable);
-
-    const preferredVideo = data.media?.domand?.videos.find(video =>
-        video.isAvailable && video.label.split('p')[0] === quality
-    );
-
-    const video = preferredVideo || bestVideo;
-
-    return {
-        watchTrackId: data.client?.watchTrackId,
-        accessRightKey: data.media?.domand?.accessRightKey,
-
-        video,
-
-        outputs: [[video.id, audio.id]],
-
-        author: data.owner?.nickname,
-        title: data.video?.title,
-    }
-}
-
-const getHls = async (dispatcher, id, trackId, accessRightKey, outputs) => {
-    const response = await fetch(
-        `https://nvapi.nicovideo.jp/v1/watch/${id}/access-rights/hls?actionTrackId=${trackId}`,
-        {
-            method: "POST",
-            dispatcher,
-            headers: {
-                ...genericHeaders,
-                "content-type": "application/json",
-                "x-access-right-key": accessRightKey,
-                "x-frontend-id": "6",
-                "x-frontend-version": "0",
-                "x-request-with": "nicovideo",
-            },
-            body: JSON.stringify({
-                outputs,
-            })
-        }
-    ).then(r => r.json()).catch(() => {});
-
-    if (!response?.data?.contentUrl) return;
-    return response.data.contentUrl;
-}
-
-export default async function ({ id, dispatcher, quality }) {
-    const {
-        watchTrackId,
-        accessRightKey,
-        video,
-        outputs,
-        author,
-        title,
-        error
-    } = await getVideoInfo(id, dispatcher, quality);
-
-    if (error) {
-        return { error };
-    }
-
-    if (!watchTrackId || !accessRightKey || !outputs) {
-        return { error: "fetch.empty" };
-    }
-
-    const hlsUrl = await getHls(
-        dispatcher,
-        id,
-        watchTrackId,
-        accessRightKey,
-        outputs
-    );
-
-    if (!hlsUrl) {
-        return { error: "fetch.empty" };
-    }
-
-    return {
-        urls: hlsUrl,
-        isHLS: true,
-        filenameAttributes: {
-            service: "nicovideo",
-            id,
-            title,
-            author,
-            resolution: `${video.width}x${video.height}`,
-            qualityLabel: `${video.label}`,
-            extension: "mp4"
-        }
-    }
-}

api/src/processing/services/soundcloud.js

@@ -148,7 +148,14 @@ export default async function(obj) {
 
     let cover;
     if (json.artwork_url) {
-        cover = json.artwork_url.replace(/-large/, "-t1080x1080");
+        const coverUrl = json.artwork_url.replace(/-large/, "-t1080x1080");
+        const testCover = await fetch(coverUrl)
+            .then(r => r.status === 200)
+            .catch(() => {});
+
+        if (testCover) {
+            cover = coverUrl;
+        }
     }
 
     return {

api/src/processing/services/twitter.js

@@ -3,12 +3,12 @@ import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
 
-const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';
+const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail';
 const tokenURL = 'https://api.x.com/1.1/guest/activate.json';
 
-const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});
+const tweetFeatures = JSON.stringify({"rweb_video_screen_enabled":false,"payments_enabled":false,"rweb_xchat_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false});
 
-const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});
+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false});
 
 const commonHeaders = {
     "user-agent": genericUserAgent,
@@ -100,10 +100,14 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
 
     graphqlTweetURL.searchParams.set('variables',
         JSON.stringify({
-            tweetId,
-            withCommunity: false,
-            includePromotedContent: false,
-            withVoice: false
+            focalTweetId: tweetId,
+            with_rux_injections: false,
+            rankingMode: "Relevance",
+            includePromotedContent: true,
+            withCommunity: true,
+            withQuickPromoteEligibilityTweetFields: true,
+            withBirdwatchNotes: true,
+            withVoice: true
         })
     );
     graphqlTweetURL.searchParams.set('features', tweetFeatures);
@@ -129,24 +133,48 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     return result
 }
 
-const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {
-    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+const parseCard = (cardOuter) => {
+    const card = JSON.parse(
+      (cardOuter?.legacy?.binding_values[0].value
+       || cardOuter?.binding_values?.unified_card)?.string_value,
+    );
+
+    if (!["video_website", "image_website"].includes(card?.type)
+        || !card?.media_entities
+        || card?.component_objects?.media_1?.type !== "media") {
+      return;
+    }
+
+    const mediaId = card.component_objects?.media_1?.data?.id;
+    return [card.media_entities[mediaId]];
+};
+
+const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) => {
+    const addInsn = thread?.data?.threaded_conversation_with_injections_v2?.instructions?.find(
+        insn => insn.type === 'TimelineAddEntries'
+    );
+
+    const tweetResult = addInsn?.entries?.find(
+        entry => entry.entryId === `tweet-${id}`
+    )?.content?.itemContent?.tweet_results?.result;
+
+    let tweetTypename = tweetResult?.__typename;
 
     if (!tweetTypename) {
         return { error: "fetch.empty" }
     }
 
-    if (tweetTypename === "TweetUnavailable") {
-        const reason = tweet?.data?.tweetResult?.result?.reason;
-        switch(reason) {
-            case "Protected":
+    if (tweetTypename === "TweetUnavailable" || tweetTypename === "TweetTombstone") {
+        const reason = tweetResult?.result?.reason;
+        if (reason === 'Protected') {
             return { error: "content.post.private" };
-            case "NsfwLoggedOut":
-                if (cookie) {
-                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
-                    tweet = await tweet.json();
-                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-                } else return { error: "content.post.age" };
+        } else if (reason === "NsfwLoggedOut" || tweetResult?.tombstone?.text?.text?.startsWith('Age-restricted')) {
+            if (!cookie) {
+                return { error: "content.post.age" };
+            }
+
+            const tweet = await requestTweet(dispatcher, id, guestToken, cookie).then(t => t.json());
+            return extractGraphqlMedia(tweet, dispatcher, id, guestToken);
         }
     }
 
@@ -154,8 +182,7 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) =>
         return { error: "content.post.unavailable" }
     }
 
-    let tweetResult = tweet.data.tweetResult.result,
-        baseTweet = tweetResult.legacy,
+    let baseTweet = tweetResult.legacy,
         repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
 
     if (tweetTypename === "TweetWithVisibilityResults") {
@@ -164,81 +191,51 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) =>
     }
 
     if (tweetResult.card?.legacy?.binding_values?.length) {
-        const card = JSON.parse(tweetResult.card.legacy.binding_values[0].value?.string_value);
-
-        if (!["video_website", "image_website"].includes(card?.type) ||
-            !card?.media_entities ||
-            card?.component_objects?.media_1?.type !== "media") {
-            return;
-        }
-
-        const mediaId = card.component_objects?.media_1?.data?.id;
-        return [card.media_entities[mediaId]];
+        return parseCard(tweetResult.card);
     }
 
     return (repostedTweet?.media || baseTweet?.extended_entities?.media);
 }
 
-const testResponse = (result) => {
-    const contentLength = result.headers.get("content-length");
-
-    if (!contentLength || contentLength === '0') {
-        return false;
-    }
-
-    if (!result.headers.get("content-type").startsWith("application/json")) {
-        return false;
-    }
-
-    return true;
-}
-
 export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) {
     const cookie = await getCookie('twitter');
 
-    let syndication = false;
-
     let guestToken = await getGuestToken(dispatcher);
     if (!guestToken) return { error: "fetch.fail" };
 
-    // for now we assume that graphql api will come back after some time,
-    // so we try it first
-
     let tweet = await requestTweet(dispatcher, id, guestToken);
 
+    if ([403, 404, 429].includes(tweet.status)) {
         // get new token & retry if old one expired
         if ([403, 429].includes(tweet.status)) {
             guestToken = await getGuestToken(dispatcher, true);
-        if (cookie) {
-            tweet = await requestTweet(dispatcher, id, guestToken, cookie);
-        } else {
-            tweet = await requestTweet(dispatcher, id, guestToken);
         }
+        tweet = await requestTweet(dispatcher, id, guestToken, cookie);
     }
 
-    const testGraphql = testResponse(tweet);
+    let media;
+    try {
+        tweet = await tweet.json();
+        media = await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+    } catch {}
 
     // if graphql requests fail, then resort to tweet embed api
-    if (!testGraphql) {
-        syndication = true;
+    if (!media || 'error' in media) {
+        try {
             tweet = await requestSyndication(dispatcher, id);
-
-        const testSyndication = testResponse(tweet);
-
-        // if even syndication request failed, then cry out loud
-        if (!testSyndication) {
-            return { error: "fetch.fail" };
-        }
-    }
-
             tweet = await tweet.json();
 
-    let media =
-        syndication
-            ? tweet.mediaDetails
-            : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+            if (tweet?.card) {
+                media = parseCard(tweet.card);
+            }
+        } catch {}
 
-    if (!media) return { error: "fetch.empty" };
+        media = tweet?.mediaDetails ?? media;
+    }
+
+    if (!media || 'error' in media) {
+        return { error: media?.error || "fetch.empty" };
+    }
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {

api/src/processing/services/vimeo.js

@@ -1,6 +1,7 @@
 import HLS from "hls-parser";
 import { env } from "../../config.js";
 import { merge } from '../../misc/utils.js';
+import { getCookie } from "../cookie/manager.js";
 
 const resolutionMatch = {
     "3840": 2160,
@@ -17,30 +18,28 @@ const resolutionMatch = {
 
 const genericHeaders = {
     Accept: 'application/vnd.vimeo.*+json; version=3.4.10',
-    'User-Agent': 'Vimeo/11.13.0 (com.vimeo; build:250619.102023.0; iOS 18.5.0) Alamofire/5.9.0 VimeoNetworking/5.0.0',
-    Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
-    'Accept-Language': 'en-US,en;q=0.9',
+    'User-Agent': 'com.vimeo.android.videoapp (Google, Pixel 7a, google, Android 16/36 Version 11.8.1) Kotlin VimeoNetworking/3.12.0',
+    Authorization: 'Basic NzRmYTg5YjgxMWExY2JiNzUwZDg1MjhkMTYzZjQ4YWYyOGEyZGJlMTp4OGx2NFd3QnNvY1lkamI2UVZsdjdDYlNwSDUrdm50YzdNNThvWDcwN1JrenJGZC9tR1lReUNlRjRSVklZeWhYZVpRS0tBcU9YYzRoTGY2Z1dlVkJFYkdJc0dMRHpoZWFZbU0reDRqZ1dkZ1diZmdIdGUrNUM5RVBySlM0VG1qcw==',
+    'Accept-Language': 'en',
 }
 
 let bearer = '';
 
 const getBearer = async (refresh = false) => {
-    if (bearer && !refresh) return bearer;
+    const cookie = getCookie('vimeo_bearer')?.values?.()?.access_token;
+    if ((bearer || cookie) && !refresh) return bearer || cookie;
 
     const oauthResponse = await fetch(
-        `https://api.vimeo.com/oauth/authorize/client?sizes=216,288,300,360,640,960,1280,1920&cdm_type=fairplay`,
+        'https://api.vimeo.com/oauth/authorize/client',
         {
             method: 'POST',
-            body: JSON.stringify({
-                scope: 'public private purchased create edit delete interact upload stats',
+            body: new URLSearchParams({
+                scope: 'private public create edit delete interact upload purchased stats',
                 grant_type: 'client_credentials',
-                // device_identifier is a long ass base64 string of seemingly
-                // random data, but it doesn't seem to be required, so we just omit it lol
-                device_identifier: '',
-            }),
+            }).toString(),
             headers: {
                 ...genericHeaders,
-                'Content-Type': 'application/json',
+                'Content-Type': 'application/x-www-form-urlencoded',
             }
         }
     )

api/src/processing/services/xiaohongshu.js

@@ -6,13 +6,13 @@ const https = (url) => {
     return url.replace(/^http:/i, 'https:');
 }
 
-export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
+export default async function ({ id, token, shareType, shareId, h265, isAudioOnly, dispatcher }) {
     let noteId = id;
     let xsecToken = token;
 
     if (!noteId) {
         const patternMatch = await resolveRedirectingURL(
-            `https://xhslink.com/a/${shareId}`,
+            `https://xhslink.com/${shareType}/${shareId}`,
             dispatcher
         );
 

api/src/processing/services/youtube.js

@@ -1,7 +1,7 @@
 import HLS from "hls-parser";
 
 import { fetch } from "undici";
-import { Innertube, Session } from "@imput/youtubei.js";
+import { Innertube, Session } from "youtubei.js";
 
 import { env } from "../../config.js";
 import { getCookie } from "../cookie/manager.js";
@@ -66,6 +66,7 @@ const cloneInnertube = async (customFetch, useSession) => {
             cookie,
             po_token: useSession ? sessionTokens?.potoken : undefined,
             visitor_data: useSession ? sessionTokens?.visitor_data : undefined,
+            player_id : "0004de42",
         });
         lastRefreshedAt = +new Date();
     }
@@ -224,7 +225,7 @@ export default async function (o) {
 
     let info;
     try {
-        info = await yt.getBasicInfo(o.id, innertubeClient);
+        info = await yt.getBasicInfo(o.id, { client: innertubeClient });
     } catch (e) {
         if (e?.info) {
             let errorInfo;

api/src/processing/url.js

@@ -99,7 +99,7 @@ function aliasURL(url) {
 
         case "xhslink":
             if (url.hostname === 'xhslink.com' && parts.length === 3) {
-                url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
+                url = new URL(`https://www.xiaohongshu.com/${parts[1]}/${parts[2]}`);
             }
             break;
 
@@ -147,6 +147,7 @@ function cleanURL(url) {
                 limitQuery('v');
             }
             break;
+        case "bilibili":
         case "rutube":
             if (url.searchParams.get('p')) {
                 limitQuery('p');

api/src/stream/shared.js

@@ -19,6 +19,9 @@ const serviceHeaders = {
     },
     vk: {
         'user-agent': vkClientAgent
+    },
+    tiktok: {
+        referer: 'https://www.tiktok.com/',
     }
 }
 

api/src/util/test.js

@@ -4,7 +4,7 @@ import { env } from "../config.js";
 import { runTest } from "../misc/run-test.js";
 import { loadJSON } from "../misc/load-from-fs.js";
 import { Red, Bright } from "../misc/console-text.js";
-import { setGlobalDispatcher, ProxyAgent } from "undici";
+import { setGlobalDispatcher, EnvHttpProxyAgent, ProxyAgent } from "undici";
 import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 
 import { services } from "../processing/service-config.js";
@@ -69,9 +69,10 @@ const printHeader = (service, padLen) => {
     console.log(service + '='.repeat(50));
 }
 
-if (env.externalProxy) {
-    setGlobalDispatcher(new ProxyAgent(env.externalProxy));
-}
+// TODO: remove env.externalProxy in a future version
+setGlobalDispatcher(
+    new EnvHttpProxyAgent({ httpProxy: env.externalProxy || undefined })
+);
 
 env.streamLifespan = 10000;
 env.apiURL = 'http://x/';

api/src/util/tests/bilibili.json

@@ -56,5 +56,14 @@
             "code": 200,
             "status": "tunnel"
         }
+    },
+    {
+        "name": "bilibili.com link with part id",
+        "url": "https://www.bilibili.com/video/BV1uo4y1K72s?spm_id_from=333.788.videopod.episodes&p=6",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
     }
 ]

api/src/util/tests/newgrounds.json

@@ -0,0 +1,42 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://www.newgrounds.com/portal/view/938050",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (audio only)",
+        "url": "https://www.newgrounds.com/portal/view/938050",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (muted)",
+        "url": "https://www.newgrounds.com/portal/view/938050",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular music",
+        "url": "https://www.newgrounds.com/audio/listen/500476",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/soundcloud.json

@@ -86,6 +86,7 @@
     },
     {
         "name": "go+ song, should fail",
+        "canFail": true,
         "url": "https://soundcloud.com/dualipa/physical-feat-troye-sivan",
         "params": {},
         "expected": {

api/src/util/tests/twitch.json

@@ -29,5 +29,14 @@
             "code": 200,
             "status": "tunnel"
         }
+    },
+    {
+        "name": "clip (mobile subdomain)",
+        "url": "https://m.twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
     }
 ]

api/src/util/tests/xiaohongshu.json

@@ -1,7 +1,7 @@
 [
     {
         "name": "video (might have expired)",
-        "url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=",
+        "url": "https://www.xiaohongshu.com/explore/685e63e1000000000b02ee3b?xsec_token=ABN8EQJCDMPcFX9RRggeIPSHLIJ8zkGceFDyBewLGUz30=",
         "canFail": true,
         "params": {},
         "expected": {
@@ -11,7 +11,7 @@
     },
     {
         "name": "picker with multiple live photos (might have expired)",
-        "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=",
+        "url": "https://www.xiaohongshu.com/explore/687128a2000000001203d94c?xsec_token=CBlDi5QDXDWZu2uUmbUrpKwg8lEL3uC10mc59lGf43r9w=",
         "canFail": true,
         "params": {},
         "expected": {
@@ -21,7 +21,7 @@
     },
     {
         "name": "one photo (might have expired)",
-        "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA",
+        "url": "https://www.xiaohongshu.com/explore/64726b99000000000800e115?xsec_token=ABoD3qPHqVZolCfS-J8UP9QQaPXZ6Z6PVyODrhaiUg27U=",
         "canFail": true,
         "params": {},
         "expected": {
@@ -31,7 +31,7 @@
     },
     {
         "name": "short link (might have expired)",
-        "url": "https://xhslink.com/a/czn4z6c1tic4",
+        "url": "https://xhslink.com/m/2wAnaTkLRc1",
         "canFail": true,
         "params": {},
         "expected": {

docs/api-env-variables.md

@@ -21,9 +21,15 @@ this document is not final and will expand over time. feel free to improve it!
 | name                | default   | value example                         |
 |:--------------------|:----------|:--------------------------------------|
 | API_LISTEN_ADDRESS  | `0.0.0.0` | `127.0.0.1`                           |
-| API_EXTERNAL_PROXY  |           | `http://user:password@127.0.0.1:8080` |
 | FREEBIND_CIDR       |           | `2001:db8::/32`                       |
 
+#### undici proxy vars
+| name        | value example                         |
+|:------------|:--------------------------------------|
+| HTTP_PROXY  | `http://user:password@10.0.0.1:1337/` |
+| HTTPS_PROXY | `https://10.0.0.2:1337/`              |
+| NO_PROXY    | `localhost`                           |
+
 [*view details*](#networking)
 
 ### limit vars
@@ -123,10 +129,23 @@ defines the local address for the api instance. if you are using a docker contai
 
 the value is a local IP address.
 
-### API_EXTERNAL_PROXY
-URL of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only.
+### HTTP_PROXY, HTTPS_PROXY, NO_PROXY
+URL of the proxy that will be passed to [`EnvHttpProxyAgent`](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent) for proxying external requests. if some cobalt functionality breaks when using a proxy, please [make a new issue](https://github.com/imputnet/cobalt/issues) about it!
 
-if some feature breaks when using a proxy, please make a new issue about it!
+quoted from [undici docs](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent):
+> When `HTTP_PROXY` and `HTTPS_PROXY` are set, `HTTP_PROXY` is used for HTTP requests and `HTTPS_PROXY` is used for HTTPS requests. If only `HTTP_PROXY` is set, `HTTP_PROXY` is used for both HTTP and HTTPS requests. If only `HTTPS_PROXY` is set, it is only used for HTTPS requests.
+
+> `NO_PROXY` is a comma or space-separated list of hostnames that should not be proxied. The list may contain leading wildcard characters (`*`). If `NO_PROXY` is set, the EnvHttpProxyAgent() will bypass the proxy for requests to hosts that match the list. If `NO_PROXY` is set to `"*"`, the EnvHttpProxyAgent() will bypass the proxy for all requests.
+
+the value is a string:
+- `HTTP_PROXY`/`HTTPS_PROXY`: URL or hostname.
+- `NO_PROXY`: comma or space-separated list of hostnames.
+
+### API_EXTERNAL_PROXY (deprecated)
+> [!WARNING]
+> this env variable is deprecated and will be removed in a future release. please update your configuration to use `HTTP_PROXY` or `HTTPS_PROXY`, as mentioned above.
+
+URL of the proxy that will be passed to [`EnvHttpProxyAgent`](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent) and used for proxying external requests. HTTP(S) only.
 
 the value is a URL.
 

docs/examples/cookies.example.json

@@ -13,5 +13,8 @@
     ],
     "youtube": [
         "cookie=<replace_this>; b=<replace_this>"
+    ],
+    "vimeo": [
+        "access_token=<replace_this>"
     ]
 }

docs/run-an-instance.md

@@ -4,9 +4,9 @@ this tutorial will help you run your own cobalt processing instance. if your ins
 ## using docker compose and package from github (recommended)
 to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured.
 
-if you need help with installing docker, follow *only the first step* of these tutorials by digitalocean:
-- [how to install docker](https://www.digitalocean.com/community/tutorial-collections/how-to-install-and-use-docker)
-- [how to install docker compose](https://www.digitalocean.com/community/tutorial-collections/how-to-install-docker-compose)
+if you need help with installing docker, you can find more information here:
+- [how to install docker](https://docs.docker.com/engine/install/)
+- [how to install docker compose](https://docs.docker.com/compose/install/)
 
 ## how to run a cobalt docker package:
 1. create a folder for cobalt config file, something like this:

pnpm-lock.yaml

@@ -19,9 +19,6 @@ importers:
       '@imput/version-info':
         specifier: workspace:^
         version: link:../packages/version-info
-      '@imput/youtubei.js':
-        specifier: ^14.0.0
-        version: 14.0.0
       content-disposition-header:
         specifier: 0.6.0
         version: 0.6.0
@@ -61,6 +58,9 @@ importers:
       url-pattern:
         specifier: 1.0.3
         version: 1.0.3
+      youtubei.js:
+        specifier: 15.1.1
+        version: 15.1.1
       zod:
         specifier: ^3.23.8
         version: 3.23.8
@@ -563,9 +563,6 @@ packages:
   '@imput/psl@2.0.4':
     resolution: {integrity: sha512-vuy76JX78/DnJegLuJoLpMmw11JTA/9HvlIADg/f8dDVXyxbh0jnObL0q13h+WvlBO4Gk26Pu8sUa7/h0JGQig==}
 
-  '@imput/youtubei.js@14.0.0':
-    resolution: {integrity: sha512-YvTnh53URPlzsmMzqF/DFHZyR9HrpgoWYHzEOklx5OCkwk1/0F/CrO9gqArXw/1oI6GjaTS2CqBd1CzyFZB07A==}
-
   '@isaacs/cliui@8.0.2':
     resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
     engines: {node: '>=12'}
@@ -1879,6 +1876,7 @@ packages:
   source-map@0.8.0-beta.0:
     resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==}
     engines: {node: '>= 8'}
+    deprecated: The work that was done in this beta branch won't be included in future versions
 
   sprintf-js@1.0.3:
     resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
@@ -2188,6 +2186,9 @@ packages:
     resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
     engines: {node: '>=10'}
 
+  youtubei.js@15.1.1:
+    resolution: {integrity: sha512-fuEDj9Ky6cAQg93BrRVCbr+GTYNZQAIFZrx/a3oDRuGc3Mf5bS0dQfoYwwgjtSV7sgAKQEEdGtzRdBzOc8g72Q==}
+
   zimmerframe@1.1.2:
     resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==}
 
@@ -2428,13 +2429,6 @@ snapshots:
     dependencies:
       punycode: 2.3.1
 
-  '@imput/youtubei.js@14.0.0':
-    dependencies:
-      '@bufbuild/protobuf': 2.2.5
-      jintr: 3.3.1
-      tslib: 2.6.3
-      undici: 6.21.3
-
   '@isaacs/cliui@8.0.2':
     dependencies:
       string-width: 5.1.2
@@ -4050,6 +4044,12 @@ snapshots:
 
   yocto-queue@0.1.0: {}
 
+  youtubei.js@15.1.1:
+    dependencies:
+      '@bufbuild/protobuf': 2.2.5
+      jintr: 3.3.1
+      undici: 6.21.3
+
   zimmerframe@1.1.2: {}
 
   zod@3.23.8: {}

web/changelogs/11.2.md

@@ -51,6 +51,7 @@ by the way, we also made it possible to [choose any preferred media container](/
 - pinterest now returns an appropriate error when a pin is unavailable.
 - AI dubs on youtube are no longer accidentally selected as default tracks.
 - youtube HLS preference is now deprecated, but can be enabled on a self-hosted instance via `ENABLE_DEPRECATED_YOUTUBE_HLS` in env.
+- downloads from vk are now way faster.
 
 ## web app improvements
 - improved compatibility of local processing & related code with older browsers.

web/i18n/en/save.json

@@ -10,7 +10,7 @@
     "services.title": "supported services",
     "services.title_show": "show supported services",
     "services.title_hide": "hide supported services",
-    "services.disclaimer": "cobalt is not affiliated with any of the services listed above.",
+    "services.disclaimer": "support for a service does not imply affiliation, endorsement, or any form of support other than technical compatibility.",
 
     "tutorial.title": "how to save on ios?",
     "tutorial.intro": "to save media conveniently on ios, you'll need to use a companion siri shortcut from the share sheet.",

web/i18n/ru/save.json

@@ -10,7 +10,7 @@
     "services.title": "поддерживаемые сервисы",
     "services.title_show": "показать поддерживаемые сервисы",
     "services.title_hide": "скрыть поддерживаемые сервисы",
-    "services.disclaimer": "кобальт не аффилирован ни с одним из перечисленных выше сервисов.\n\nдеятельность meta platforms (владельца facebook и instagram) запрещена на территории РФ и признана экстремистской.",
+    "services.disclaimer": "поддержка сервиса не означает аффилированность, одобрение или любую другую форму поддержки, кроме технической совместимости.\n\nдеятельность владельца facebook и instagram запрещена на территории РФ и признана экстремистской.",
     "tutorial.step.1": "добавь команды-компаньоны:",
     "tutorial.step.2": "нажми кнопку \"поделиться\" в диалоге сохранения кобальта.",
     "tutorial.step.3": "выбери нужную команду в окне обмена.",

web/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@imput/cobalt-web",
-    "version": "11.2.3",
+    "version": "11.3",
     "type": "module",
     "private": true,
     "scripts": {