api: add initial draft support for nicovideo

.github/workflows/test-services.yml

@@ -30,8 +30,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-      - run: pnpm i --frozen-lockfile
+      - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }}
-      - run: node api/src/util/test run-tests-for ${{ matrix.service }}
         env:
-          HTTP_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
+          API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
           TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }}

README.md

@@ -29,20 +29,6 @@ cobalt is a media downloader that doesn't piss you off. it's friendly, efficient
 
 paste the link, get the file, move on. that simple, just how it should be.
 
-### sponsors
-<div align="center" markdown="1">
-    <sup>special thanks to Warp for sponsoring the development of cobalt</sup>
-    <br>
-    <a href="https://go.warp.dev/cobalt">
-        <img alt="Warp banner" width="400" src="https://raw.githubusercontent.com/warpdotdev/brand-assets/be7d584f98e62b1579fd2e9338d4c7318a732f1b/Github/Sponsor/Warp-Github-LG-03.png">
-    </a>
-
-### [Warp, built for coding with multiple AI agents](https://go.warp.dev/cobalt)
-</div>
-
-#### RoyaleHosting
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), and a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
-
 ### cobalt monorepo
 this monorepo includes source code for api, frontend, and related packages:
 - [api tree & readme](/api/)
@@ -67,6 +53,9 @@ same content can be downloaded via dev tools of any modern web browser.
 ### contributing
 if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
 
+### thank you
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
+
 ### licenses
 for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
 unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).

api/README.md

@@ -23,7 +23,6 @@ if the desired service isn't supported yet, feel free to create an appropriate i
 | instagram         | ✅            | ✅         | ✅         | ➖         | ➖              |
 | facebook          | ✅            | ❌         | ✅         | ➖         | ➖              |
 | loom              | ✅            | ❌         | ✅         | ✅         | ➖              |
-| newgrounds        | ✅            | ✅         | ✅         | ✅         | ✅              |
 | ok.ru             | ✅            | ❌         | ✅         | ✅         | ✅              |
 | pinterest         | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit            | ✅            | ✅         | ✅         | ❌         | ❌              |

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "11.5",
+    "version": "11.2.2",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -26,6 +26,7 @@
         "@datastructures-js/priority-queue": "^6.3.1",
         "@imput/psl": "^2.0.4",
         "@imput/version-info": "workspace:^",
+        "@imput/youtubei.js": "^14.0.0",
         "content-disposition-header": "0.6.0",
         "cors": "^2.8.5",
         "dotenv": "^16.0.1",
@@ -39,7 +40,6 @@
         "set-cookie-parser": "2.6.0",
         "undici": "^6.21.3",
         "url-pattern": "1.0.3",
-        "youtubei.js": "15.1.1",
         "zod": "^3.23.8"
     },
     "optionalDependencies": {

api/src/config.js

@@ -3,12 +3,12 @@ import { loadEnvs, validateEnvs } from "./core/env.js";
 
 const version = await getVersion();
 
-const canonicalEnv = Object.freeze(structuredClone(process.env));
 const env = loadEnvs();
 
 const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
 const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
 
+export const canonicalEnv = Object.freeze(structuredClone(process.env));
 export const setTunnelPort = (port) => env.tunnelPort = port;
 export const isCluster = env.instanceCount > 1;
 export const updateEnv = (newEnv) => {
@@ -18,10 +18,6 @@ export const updateEnv = (newEnv) => {
     newEnv.tunnelPort = env.tunnelPort;
 
     for (const key in env) {
-        if (key === 'subscribe') {
-            continue;
-        }
-
         if (String(env[key]) !== String(newEnv[key])) {
             changes.push(key);
         }
@@ -35,7 +31,6 @@ await validateEnvs(env);
 
 export {
     env,
-    canonicalEnv,
     genericUserAgent,
     cobaltUserAgent,
 }

api/src/core/api.js

@@ -1,7 +1,7 @@
 import cors from "cors";
 import http from "node:http";
 import rateLimit from "express-rate-limit";
-import { setGlobalDispatcher, EnvHttpProxyAgent } from "undici";
+import { setGlobalDispatcher, ProxyAgent } from "undici";
 import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
 
 import jwt from "../security/jwt.js";
@@ -337,18 +337,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     randomizeCiphers();
     setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
 
-    env.subscribe(['externalProxy', 'httpProxyValues'], () => {
-        // TODO: remove env.externalProxy in a future version
-        const options = {};
     if (env.externalProxy) {
-            options.httpProxy = env.externalProxy;
+        setGlobalDispatcher(new ProxyAgent(env.externalProxy))
     }
 
-        setGlobalDispatcher(
-            new EnvHttpProxyAgent(options)
-        );
-    });
-
     http.createServer(app).listen({
         port: env.apiPort,
         host: env.listenAddress,

api/src/core/env.js

@@ -1,4 +1,4 @@
-import { Constants } from "youtubei.js";
+import { Constants } from "@imput/youtubei.js";
 import { services } from "../processing/service-config.js";
 import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";
 
@@ -10,34 +10,6 @@ import { Green, Yellow } from "../misc/console-text.js";
 const forceLocalProcessingOptions = ["never", "session", "always"];
 const youtubeHlsOptions = ["never", "key", "always"];
 
-const httpProxyVariables = ["NO_PROXY", "HTTP_PROXY", "HTTPS_PROXY"].flatMap(
-    k => [ k, k.toLowerCase() ]
-);
-
-const changeCallbacks = {};
-
-const onEnvChanged = (changes) => {
-    for (const key of changes) {
-        if (changeCallbacks[key]) {
-            changeCallbacks[key].map(fn => {
-                try { fn() } catch {}
-            });
-        }
-    }
-}
-
-const subscribe = (keys, fn) => {
-    keys = [keys].flat();
-
-    for (const key of keys) {
-        if (key in currentEnv && key !== 'subscribe') {
-            changeCallbacks[key] ??= [];
-            changeCallbacks[key].push(fn);
-            fn();
-        } else throw `invalid env key ${key}`;
-    }
-}
-
 export const loadEnvs = (env = process.env) => {
     const allServices = new Set(Object.keys(services));
     const disabledServices = env.DISABLED_SERVICES?.split(',') || [];
@@ -47,18 +19,6 @@ export const loadEnvs = (env = process.env) => {
         }
     }));
 
-    // we need to copy the proxy envs (HTTP_PROXY, HTTPS_PROXY)
-    // back into process.env, so that EnvHttpProxyAgent can pick
-    // them up later
-    for (const key of httpProxyVariables) {
-        const value = env[key] ?? canonicalEnv[key];
-        if (value !== undefined) {
-            process.env[key] = env[key];
-        } else {
-            delete process.env[key];
-        }
-    }
-
     return {
         apiURL: env.API_URL || '',
         apiPort: env.API_PORT || 9000,
@@ -95,9 +55,6 @@ export const loadEnvs = (env = process.env) => {
 
         externalProxy: env.API_EXTERNAL_PROXY,
 
-        // used only for comparing against old values when envs are being updated
-        httpProxyValues: httpProxyVariables.map(k => String(env[k])).join(''),
-
         turnstileSitekey: env.TURNSTILE_SITEKEY,
         turnstileSecret: env.TURNSTILE_SECRET,
         jwtSecret: env.JWT_SECRET,
@@ -130,13 +87,9 @@ export const loadEnvs = (env = process.env) => {
 
         envFile: env.API_ENV_FILE,
         envRemoteReloadInterval: 300,
-
-        subscribe,
     };
 }
 
-let loggedProxyWarning = false;
-
 export const validateEnvs = async (env) => {
     if (env.sessionEnabled && env.jwtSecret.length < 16) {
         throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)");
@@ -174,15 +127,6 @@ export const validateEnvs = async (env) => {
         throw new Error('freebind is not available when external proxy is enabled')
     }
 
-    if (env.externalProxy && !loggedProxyWarning) {
-        console.error('API_EXTERNAL_PROXY is deprecated and will be removed in a future release.');
-        console.error('Use HTTP_PROXY or HTTPS_PROXY instead.');
-        console.error('You can read more about the new proxy variables in docs/api-env-variables.md\n');
-
-        // prevent the warning from being printed on every env validation
-        loggedProxyWarning = true;
-    }
-
     return env;
 }
 
@@ -226,14 +170,11 @@ const wrapReload = (contents) => {
             return;
         }
 
-        onEnvChanged(changes);
-
         console.log(`${Green('[✓]')} envs reloaded successfully!`);
         for (const key of changes) {
             const value = currentEnv[key];
             const isSecret = key.toLowerCase().includes('apikey')
-                          || key.toLowerCase().includes('secret')
+                          || key.toLowerCase().includes('secret');
-                          || key === 'httpProxyValues';
 
             if (!value) {
                 console.log(`    removed: ${key}`);

api/src/misc/utils.js

@@ -1,4 +1,4 @@
-import { request } from "undici";
+import { request } from 'undici';
 const redirectStatuses = new Set([301, 302, 303, 307, 308]);
 
 export async function getRedirectingURL(url, dispatcher, headers) {
@@ -8,34 +8,18 @@ export async function getRedirectingURL(url, dispatcher, headers) {
         headers,
         redirect: 'manual'
     };
-    const getParams = {
-        ...params,
-        method: 'GET',
-    };
 
-    const callback = (r) => {
+    let location = await request(url, params).then(r => {
         if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
             return r.headers['location'];
         }
+    }).catch(() => null);
+
+    location ??= await fetch(url, params).then(r => {
+        if (redirectStatuses.has(r.status) && r.headers.has('location')) {
+            return r.headers.get('location');
         }
-
+    }).catch(() => null);
-    /*
-        try request() with HEAD & GET,
-        then do the same with fetch
-        (fetch is required for shortened reddit links)
-    */
-
-    let location = await request(url, params)
-        .then(callback).catch(() => null);
-
-    location ??= await request(url, getParams)
-        .then(callback).catch(() => null);
-
-    location ??= await fetch(url, params)
-        .then(callback).catch(() => null);
-
-    location ??= await fetch(url, getParams)
-        .then(callback).catch(() => null);
 
     return location;
 }

api/src/processing/cookie/manager.js

@@ -13,7 +13,6 @@ const VALID_SERVICES = new Set([
     'reddit',
     'twitter',
     'youtube',
-    'vimeo_bearer',
 ]);
 
 const invalidCookies = {};

api/src/processing/match-action.js

@@ -180,7 +180,6 @@ export default function({
 
                 case "ok":
                 case "xiaohongshu":
-                case "newgrounds":
                     params = { type: "proxy" };
                     break;
 

api/src/processing/match.js

@@ -29,7 +29,7 @@ import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
 import xiaohongshu from "./services/xiaohongshu.js";
-import newgrounds from "./services/newgrounds.js";
+import nicovideo from "./services/nicovideo.js";
 
 let freebind;
 
@@ -269,9 +269,10 @@ export default async function({ host, patternMatch, params, authType }) {
                 });
                 break;
 
-            case "newgrounds":
+            case "nicovideo":
-                r = await newgrounds({
+                r = await nicovideo({
                     ...patternMatch,
+                    dispatcher,
                     quality: params.videoQuality,
                 });
                 break;
@@ -322,7 +323,7 @@ export default async function({ host, patternMatch, params, authType }) {
         let localProcessing = params.localProcessing;
         const lpEnv = env.forceLocalProcessing;
         const shouldForceLocal = lpEnv === "always" || (lpEnv === "session" && authType === "session");
-        const localDisabled = (!localProcessing || localProcessing === "disabled");
+        const localDisabled = (!localProcessing || localProcessing === "none");
 
         if (shouldForceLocal && localDisabled) {
             localProcessing = "preferred";

api/src/processing/service-config.js

@@ -7,7 +7,6 @@ export const services = {
     bilibili: {
         patterns: [
             "video/:comId",
-            "video/:comId?p=:partId",
             "_shortLink/:comShortLink",
             "_tv/:lang/video/:tvId",
             "_tv/video/:tvId"
@@ -61,6 +60,11 @@ export const services = {
     loom: {
         patterns: ["share/:id", "embed/:id"],
     },
+    nicovideo: {
+        patterns: ["watch/:id"],
+        tld: "jp",
+        subdomains: ["sp"],
+    },
     ok: {
         patterns: [
             "video/:id",
@@ -75,12 +79,6 @@ export const services = {
             "url_shortener/:shortLink"
         ],
     },
-    newgrounds: {
-        patterns: [
-            "portal/view/:id",
-            "audio/listen/:audioId",
-        ]
-    },
     reddit: {
         patterns: [
             "comments/:id",
@@ -166,7 +164,6 @@ export const services = {
     twitch: {
         patterns: [":channel/clip/:clip"],
         tld: "tv",
-        subdomains: ["clips", "www", "m"],
     },
     twitter: {
         patterns: [
@@ -209,7 +206,7 @@ export const services = {
         patterns: [
             "explore/:id?xsec_token=:token",
             "discovery/item/:id?xsec_token=:token",
-            ":shareType/:shareId",
+            "a/:shareId"
         ],
         altDomains: ["xhslink.com"],
     },

api/src/processing/service-patterns.js

@@ -1,72 +1,53 @@
 export const testers = {
     "bilibili": pattern =>
-        (pattern.comId?.length <= 12 && pattern.partId?.length <= 3) ||
+        pattern.comId?.length <= 12 || pattern.comShortLink?.length <= 16
-        (pattern.comId?.length <= 12 && !pattern.partId) ||
+        || pattern.tvId?.length <= 24,
-        pattern.comShortLink?.length <= 16 ||
-        pattern.tvId?.length <= 24,
-
-    "bsky": pattern =>
-        pattern.user?.length <= 128 && pattern.post?.length <= 128,
 
     "dailymotion": pattern => pattern.id?.length <= 32,
 
-    "facebook": pattern =>
-        pattern.shortLink?.length <= 11 ||
-        pattern.username?.length <= 30 ||
-        pattern.caption?.length <= 255 ||
-        pattern.id?.length <= 20 && !pattern.shareType ||
-        pattern.id?.length <= 20 && pattern.shareType?.length === 1,
-
     "instagram": pattern =>
-        pattern.postId?.length <= 48 ||
+        pattern.postId?.length <= 48
-        pattern.shareId?.length <= 16 ||
+        || pattern.shareId?.length <= 16
-        (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
+        || (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
 
     "loom": pattern =>
         pattern.id?.length <= 32,
 
-    "newgrounds": pattern =>
-        pattern.id?.length <= 12 ||
-        pattern.audioId?.length <= 12,
-
     "ok": pattern =>
         pattern.id?.length <= 16,
 
     "pinterest": pattern =>
-        pattern.id?.length <= 128 ||
+        pattern.id?.length <= 128 || pattern.shortLink?.length <= 32,
-        pattern.shortLink?.length <= 32,
 
     "reddit": pattern =>
-        pattern.id?.length <= 16 && !pattern.sub && !pattern.user ||
+        pattern.id?.length <= 16 && !pattern.sub && !pattern.user
-        (pattern.sub?.length <= 22 && pattern.id?.length <= 16) ||
+        || (pattern.sub?.length <= 22 && pattern.id?.length <= 16)
-        (pattern.user?.length <= 22 && pattern.id?.length <= 16) ||
+        || (pattern.user?.length <= 22 && pattern.id?.length <= 16)
-        (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16) ||
+        || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16)
-        (pattern.shortId?.length <= 16),
+        || (pattern.shortId?.length <= 16),
 
     "rutube": pattern =>
         (pattern.id?.length === 32 && pattern.key?.length <= 32) ||
-        pattern.id?.length === 32 ||
+        pattern.id?.length === 32 || pattern.yappyId?.length === 32,
-        pattern.yappyId?.length === 32,
-
-    "snapchat": pattern =>
-        (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255)) ||
-        pattern.spotlightId?.length <= 255 ||
-        pattern.shortLink?.length <= 16,
 
     "soundcloud": pattern =>
-        (pattern.author?.length <= 255 && pattern.song?.length <= 255) ||
+        (pattern.author?.length <= 255 && pattern.song?.length <= 255)
-        pattern.shortLink?.length <= 32,
+        || pattern.shortLink?.length <= 32,
+
+    "snapchat": pattern =>
+        (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255))
+        || pattern.spotlightId?.length <= 255
+        || pattern.shortLink?.length <= 16,
 
     "streamable": pattern =>
         pattern.id?.length <= 6,
 
     "tiktok": pattern =>
-        pattern.postId?.length <= 21 ||
+        pattern.postId?.length <= 21 || pattern.shortLink?.length <= 21,
-        pattern.shortLink?.length <= 21,
 
     "tumblr": pattern =>
-        pattern.id?.length < 21 ||
+        pattern.id?.length < 21
-        (pattern.id?.length < 21 && pattern.user?.length <= 32),
+        || (pattern.id?.length < 21 && pattern.user?.length <= 32),
 
     "twitch": pattern =>
         pattern.channel && pattern.clip?.length <= 100,
@@ -75,16 +56,29 @@ export const testers = {
         pattern.id?.length < 20,
 
     "vimeo": pattern =>
-        pattern.id?.length <= 11 && (!pattern.password || pattern.password.length < 16),
+        pattern.id?.length <= 11
+        && (!pattern.password || pattern.password.length < 16),
 
     "vk": pattern =>
         (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
         (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
 
-    "xiaohongshu": pattern =>
-        pattern.id?.length <= 24 && pattern.token?.length <= 64 ||
-        pattern.shareId?.length <= 24 && pattern.shareType?.length === 1,
-
     "youtube": pattern =>
         pattern.id?.length <= 11,
+
+    "facebook": pattern =>
+        pattern.shortLink?.length <= 11
+        || pattern.username?.length <= 30
+        || pattern.caption?.length <= 255
+        || pattern.id?.length <= 20 && !pattern.shareType
+        || pattern.id?.length <= 20 && pattern.shareType?.length === 1,
+
+    "bsky": pattern =>
+        pattern.user?.length <= 128 && pattern.post?.length <= 128,
+
+    "xiaohongshu": pattern =>
+        pattern.id?.length <= 24 && pattern.token?.length <= 64
+        || pattern.shareId?.length <= 24,
+
+    "nicovideo": pattern => pattern.id?.length <= 12,
 }

api/src/processing/services/bilibili.js

@@ -17,14 +17,8 @@ function extractBestQuality(dashData) {
     return [ bestVideo, bestAudio ];
 }
 
-async function com_download(id, partId) {
+async function com_download(id) {
-    const url = new URL(`https://bilibili.com/video/${id}`);
+    const html = await fetch(`https://bilibili.com/video/${id}`, {
-
-    if (partId) {
-        url.searchParams.set('p', partId);
-    }
-
-    const html = await fetch(url, {
         headers: {
             "user-agent": genericUserAgent
         }
@@ -53,15 +47,10 @@ async function com_download(id, partId) {
         return { error: "fetch.empty" };
     }
 
-    let filenameBase = `bilibili_${id}`;
-    if (partId) {
-        filenameBase += `_${partId}`;
-    }
-
     return {
         urls: [video.baseUrl, audio.baseUrl],
-        audioFilename: `${filenameBase}_audio`,
+        audioFilename: `bilibili_${id}_audio`,
-        filename: `${filenameBase}_${video.width}x${video.height}.mp4`,
+        filename: `bilibili_${id}_${video.width}x${video.height}.mp4`,
     };
 }
 
@@ -100,14 +89,14 @@ async function tv_download(id) {
     };
 }
 
-export default async function({ comId, tvId, comShortLink, partId }) {
+export default async function({ comId, tvId, comShortLink }) {
     if (comShortLink) {
         const patternMatch = await resolveRedirectingURL(`https://b23.tv/${comShortLink}`);
         comId = patternMatch?.comId;
     }
 
     if (comId) {
-        return com_download(comId, partId);
+        return com_download(comId);
     } else if (tvId) {
         return tv_download(tvId);
     }

api/src/processing/services/newgrounds.js

@@ -1,103 +0,0 @@
-import { genericUserAgent } from "../../config.js";
-
-const getVideo = async ({ id, quality }) => {
-    const json = await fetch(`https://www.newgrounds.com/portal/video/${id}`, {
-        headers: {
-            "User-Agent": genericUserAgent,
-            "X-Requested-With": "XMLHttpRequest", // required to get the JSON response
-        }
-    })
-    .then(r => r.json())
-    .catch(() => {});
-
-    if (!json) return { error: "fetch.empty" };
-
-    const videoSources = json.sources;
-    const videoQualities = Object.keys(videoSources);
-
-    if (videoQualities.length === 0) {
-        return { error: "fetch.empty" };
-    }
-
-    const bestVideo = videoSources[videoQualities[0]]?.[0],
-          userQuality = quality === "2160" ? "4k" : `${quality}p`,
-          preferredVideo = videoSources[userQuality]?.[0],
-          video = preferredVideo || bestVideo,
-          videoQuality = preferredVideo ? userQuality : videoQualities[0];
-
-    if (!bestVideo || !video.type.includes("mp4")) {
-        return { error: "fetch.empty" };
-    }
-
-    const fileMetadata = {
-        title: decodeURIComponent(json.title),
-        artist: decodeURIComponent(json.author),
-    }
-
-    return {
-        urls: video.src,
-        filenameAttributes: {
-            service: "newgrounds",
-            id,
-            title: fileMetadata.title,
-            author: fileMetadata.artist,
-            extension: "mp4",
-            qualityLabel: videoQuality,
-            resolution: videoQuality,
-        },
-        fileMetadata,
-    }
-}
-
-const getMusic = async ({ id }) => {
-    const html = await fetch(`https://www.newgrounds.com/audio/listen/${id}`, {
-        headers: {
-            "User-Agent": genericUserAgent,
-        }
-    })
-    .then(r => r.text())
-    .catch(() => {});
-
-    if (!html) return { error: "fetch.fail" };
-
-    const params = JSON.parse(
-        `{${html.split(',"params":{')[1]?.split(',"images":')[0]}}`
-    );
-    if (!params) return { error: "fetch.empty" };
-
-    if (!params.name || !params.artist || !params.filename || !params.icon) {
-        return { error: "fetch.empty" };
-    }
-
-    const fileMetadata = {
-        title: decodeURIComponent(params.name),
-        artist: decodeURIComponent(params.artist),
-    }
-
-    return {
-        urls: params.filename,
-        filenameAttributes: {
-            service: "newgrounds",
-            id,
-            title: fileMetadata.title,
-            author: fileMetadata.artist,
-        },
-        fileMetadata,
-        cover:
-            params.icon.includes(".png?") || params.icon.includes(".jpg?")
-                ? params.icon
-                : undefined,
-        isAudioOnly: true,
-        bestAudio: "mp3",
-    }
-}
-
-export default function({ id, audioId, quality }) {
-    if (id) {
-        return getVideo({ id, quality });
-    } else if (audioId) {
-        return getMusic({ id: audioId });
-    }
-
-    return { error: "fetch.empty" };
-}

api/src/processing/services/nicovideo.js

@@ -0,0 +1,120 @@
+import { genericUserAgent } from "../../config.js";
+
+const genericHeaders = {
+    "user-agent": genericUserAgent,
+    "accept-language": "en-US,en;q=0.9",
+}
+
+const getVideoInfo = async (id, dispatcher, quality) => {
+    const html = await fetch(`https://www.nicovideo.jp/watch/${id}`, {
+        dispatcher,
+        headers: genericHeaders
+    }).then(r => r.text()).catch(() => {});
+
+    if (!(html.includes("accessRightKey")
+            || !(html.includes('<meta name="server-response" content="')))) {
+        return { error: "fetch.fail" };
+    }
+
+    const rawContent =
+        html.split('<meta name="server-response" content="')[1]
+        .split('" />')[0]
+        .replaceAll("&quot;", '"');
+
+    const data = JSON.parse(rawContent)?.data?.response;
+
+    if (!data) {
+        return { error: "fetch.fail" };
+    }
+
+    const audio = data.media?.domand?.audios.find(audio => audio.isAvailable);
+    const bestVideo = data.media?.domand?.videos.find(video => video.isAvailable);
+
+    const preferredVideo = data.media?.domand?.videos.find(video =>
+        video.isAvailable && video.label.split('p')[0] === quality
+    );
+
+    const video = preferredVideo || bestVideo;
+
+    return {
+        watchTrackId: data.client?.watchTrackId,
+        accessRightKey: data.media?.domand?.accessRightKey,
+
+        video,
+
+        outputs: [[video.id, audio.id]],
+
+        author: data.owner?.nickname,
+        title: data.video?.title,
+    }
+}
+
+const getHls = async (dispatcher, id, trackId, accessRightKey, outputs) => {
+    const response = await fetch(
+        `https://nvapi.nicovideo.jp/v1/watch/${id}/access-rights/hls?actionTrackId=${trackId}`,
+        {
+            method: "POST",
+            dispatcher,
+            headers: {
+                ...genericHeaders,
+                "content-type": "application/json",
+                "x-access-right-key": accessRightKey,
+                "x-frontend-id": "6",
+                "x-frontend-version": "0",
+                "x-request-with": "nicovideo",
+            },
+            body: JSON.stringify({
+                outputs,
+            })
+        }
+    ).then(r => r.json()).catch(() => {});
+
+    if (!response?.data?.contentUrl) return;
+    return response.data.contentUrl;
+}
+
+export default async function ({ id, dispatcher, quality }) {
+    const {
+        watchTrackId,
+        accessRightKey,
+        video,
+        outputs,
+        author,
+        title,
+        error
+    } = await getVideoInfo(id, dispatcher, quality);
+
+    if (error) {
+        return { error };
+    }
+
+    if (!watchTrackId || !accessRightKey || !outputs) {
+        return { error: "fetch.empty" };
+    }
+
+    const hlsUrl = await getHls(
+        dispatcher,
+        id,
+        watchTrackId,
+        accessRightKey,
+        outputs
+    );
+
+    if (!hlsUrl) {
+        return { error: "fetch.empty" };
+    }
+
+    return {
+        urls: hlsUrl,
+        isHLS: true,
+        filenameAttributes: {
+            service: "nicovideo",
+            id,
+            title,
+            author,
+            resolution: `${video.width}x${video.height}`,
+            qualityLabel: `${video.label}`,
+            extension: "mp4"
+        }
+    }
+}

api/src/processing/services/soundcloud.js

@@ -148,14 +148,7 @@ export default async function(obj) {
 
     let cover;
     if (json.artwork_url) {
-        const coverUrl = json.artwork_url.replace(/-large/, "-t1080x1080");
+        cover = json.artwork_url.replace(/-large/, "-t1080x1080");
-        const testCover = await fetch(coverUrl)
-            .then(r => r.status === 200)
-            .catch(() => {});
-
-        if (testCover) {
-            cover = coverUrl;
-        }
     }
 
     return {

api/src/processing/services/twitter.js

@@ -3,12 +3,12 @@ import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
 
-const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail';
+const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';
 const tokenURL = 'https://api.x.com/1.1/guest/activate.json';
 
-const tweetFeatures = JSON.stringify({"rweb_video_screen_enabled":false,"payments_enabled":false,"rweb_xchat_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false});
+const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});
 
-const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false});
+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});
 
 const commonHeaders = {
     "user-agent": genericUserAgent,
@@ -100,14 +100,10 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
 
     graphqlTweetURL.searchParams.set('variables',
         JSON.stringify({
-            focalTweetId: tweetId,
+            tweetId,
-            with_rux_injections: false,
+            withCommunity: false,
-            rankingMode: "Relevance",
+            includePromotedContent: false,
-            includePromotedContent: true,
+            withVoice: false
-            withCommunity: true,
-            withQuickPromoteEligibilityTweetFields: true,
-            withBirdwatchNotes: true,
-            withVoice: true
         })
     );
     graphqlTweetURL.searchParams.set('features', tweetFeatures);
@@ -133,48 +129,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     return result
 }
 
-const parseCard = (cardOuter) => {
+const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {
-    const card = JSON.parse(
+    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-      (cardOuter?.legacy?.binding_values[0].value
-       || cardOuter?.binding_values?.unified_card)?.string_value,
-    );
-
-    if (!["video_website", "image_website"].includes(card?.type)
-        || !card?.media_entities
-        || card?.component_objects?.media_1?.type !== "media") {
-      return;
-    }
-
-    const mediaId = card.component_objects?.media_1?.data?.id;
-    return [card.media_entities[mediaId]];
-};
-
-const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) => {
-    const addInsn = thread?.data?.threaded_conversation_with_injections_v2?.instructions?.find(
-        insn => insn.type === 'TimelineAddEntries'
-    );
-
-    const tweetResult = addInsn?.entries?.find(
-        entry => entry.entryId === `tweet-${id}`
-    )?.content?.itemContent?.tweet_results?.result;
-
-    let tweetTypename = tweetResult?.__typename;
 
     if (!tweetTypename) {
         return { error: "fetch.empty" }
     }
 
-    if (tweetTypename === "TweetUnavailable" || tweetTypename === "TweetTombstone") {
+    if (tweetTypename === "TweetUnavailable") {
-        const reason = tweetResult?.result?.reason;
+        const reason = tweet?.data?.tweetResult?.result?.reason;
-        if (reason === 'Protected') {
+        switch(reason) {
+            case "Protected":
                 return { error: "content.post.private" };
-        } else if (reason === "NsfwLoggedOut" || tweetResult?.tombstone?.text?.text?.startsWith('Age-restricted')) {
+            case "NsfwLoggedOut":
-            if (!cookie) {
+                if (cookie) {
-                return { error: "content.post.age" };
+                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
-            }
+                    tweet = await tweet.json();
-
+                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-            const tweet = await requestTweet(dispatcher, id, guestToken, cookie).then(t => t.json());
+                } else return { error: "content.post.age" };
-            return extractGraphqlMedia(tweet, dispatcher, id, guestToken);
         }
     }
 
@@ -182,7 +154,8 @@ const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) =
         return { error: "content.post.unavailable" }
     }
 
-    let baseTweet = tweetResult.legacy,
+    let tweetResult = tweet.data.tweetResult.result,
+        baseTweet = tweetResult.legacy,
         repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
 
     if (tweetTypename === "TweetWithVisibilityResults") {
@@ -191,51 +164,81 @@ const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) =
     }
 
     if (tweetResult.card?.legacy?.binding_values?.length) {
-        return parseCard(tweetResult.card);
+        const card = JSON.parse(tweetResult.card.legacy.binding_values[0].value?.string_value);
+
+        if (!["video_website", "image_website"].includes(card?.type) ||
+            !card?.media_entities ||
+            card?.component_objects?.media_1?.type !== "media") {
+            return;
+        }
+
+        const mediaId = card.component_objects?.media_1?.data?.id;
+        return [card.media_entities[mediaId]];
     }
 
     return (repostedTweet?.media || baseTweet?.extended_entities?.media);
 }
 
+const testResponse = (result) => {
+    const contentLength = result.headers.get("content-length");
+
+    if (!contentLength || contentLength === '0') {
+        return false;
+    }
+
+    if (!result.headers.get("content-type").startsWith("application/json")) {
+        return false;
+    }
+
+    return true;
+}
+
 export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) {
     const cookie = await getCookie('twitter');
 
+    let syndication = false;
+
     let guestToken = await getGuestToken(dispatcher);
     if (!guestToken) return { error: "fetch.fail" };
 
+    // for now we assume that graphql api will come back after some time,
+    // so we try it first
+
     let tweet = await requestTweet(dispatcher, id, guestToken);
 
-    if ([403, 404, 429].includes(tweet.status)) {
     // get new token & retry if old one expired
     if ([403, 429].includes(tweet.status)) {
         guestToken = await getGuestToken(dispatcher, true);
-        }
+        if (cookie) {
             tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+        } else {
+            tweet = await requestTweet(dispatcher, id, guestToken);
+        }
     }
 
-    let media;
+    const testGraphql = testResponse(tweet);
-    try {
-        tweet = await tweet.json();
-        media = await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
-    } catch {}
 
     // if graphql requests fail, then resort to tweet embed api
-    if (!media || 'error' in media) {
+    if (!testGraphql) {
-        try {
+        syndication = true;
         tweet = await requestSyndication(dispatcher, id);
+
+        const testSyndication = testResponse(tweet);
+
+        // if even syndication request failed, then cry out loud
+        if (!testSyndication) {
+            return { error: "fetch.fail" };
+        }
+    }
+
     tweet = await tweet.json();
 
-            if (tweet?.card) {
+    let media =
-                media = parseCard(tweet.card);
+        syndication
-            }
+            ? tweet.mediaDetails
-        } catch {}
+            : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
 
-        media = tweet?.mediaDetails ?? media;
+    if (!media) return { error: "fetch.empty" };
-    }
-
-    if (!media || 'error' in media) {
-        return { error: media?.error || "fetch.empty" };
-    }
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {

api/src/processing/services/vimeo.js

@@ -1,7 +1,6 @@
 import HLS from "hls-parser";
 import { env } from "../../config.js";
 import { merge } from '../../misc/utils.js';
-import { getCookie } from "../cookie/manager.js";
 
 const resolutionMatch = {
     "3840": 2160,
@@ -18,28 +17,30 @@ const resolutionMatch = {
 
 const genericHeaders = {
     Accept: 'application/vnd.vimeo.*+json; version=3.4.10',
-    'User-Agent': 'com.vimeo.android.videoapp (Google, Pixel 7a, google, Android 16/36 Version 11.8.1) Kotlin VimeoNetworking/3.12.0',
+    'User-Agent': 'Vimeo/11.13.0 (com.vimeo; build:250619.102023.0; iOS 18.5.0) Alamofire/5.9.0 VimeoNetworking/5.0.0',
-    Authorization: 'Basic NzRmYTg5YjgxMWExY2JiNzUwZDg1MjhkMTYzZjQ4YWYyOGEyZGJlMTp4OGx2NFd3QnNvY1lkamI2UVZsdjdDYlNwSDUrdm50YzdNNThvWDcwN1JrenJGZC9tR1lReUNlRjRSVklZeWhYZVpRS0tBcU9YYzRoTGY2Z1dlVkJFYkdJc0dMRHpoZWFZbU0reDRqZ1dkZ1diZmdIdGUrNUM5RVBySlM0VG1qcw==',
+    Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
-    'Accept-Language': 'en',
+    'Accept-Language': 'en-US,en;q=0.9',
 }
 
 let bearer = '';
 
 const getBearer = async (refresh = false) => {
-    const cookie = getCookie('vimeo_bearer')?.values?.()?.access_token;
+    if (bearer && !refresh) return bearer;
-    if ((bearer || cookie) && !refresh) return bearer || cookie;
 
     const oauthResponse = await fetch(
-        'https://api.vimeo.com/oauth/authorize/client',
+        `https://api.vimeo.com/oauth/authorize/client?sizes=216,288,300,360,640,960,1280,1920&cdm_type=fairplay`,
         {
             method: 'POST',
-            body: new URLSearchParams({
+            body: JSON.stringify({
-                scope: 'private public create edit delete interact upload purchased stats',
+                scope: 'public private purchased create edit delete interact upload stats',
                 grant_type: 'client_credentials',
-            }).toString(),
+                // device_identifier is a long ass base64 string of seemingly
+                // random data, but it doesn't seem to be required, so we just omit it lol
+                device_identifier: '',
+            }),
             headers: {
                 ...genericHeaders,
-                'Content-Type': 'application/x-www-form-urlencoded',
+                'Content-Type': 'application/json',
             }
         }
     )

api/src/processing/services/xiaohongshu.js

@@ -6,13 +6,13 @@ const https = (url) => {
     return url.replace(/^http:/i, 'https:');
 }
 
-export default async function ({ id, token, shareType, shareId, h265, isAudioOnly, dispatcher }) {
+export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
     let noteId = id;
     let xsecToken = token;
 
     if (!noteId) {
         const patternMatch = await resolveRedirectingURL(
-            `https://xhslink.com/${shareType}/${shareId}`,
+            `https://xhslink.com/a/${shareId}`,
             dispatcher
         );
 

api/src/processing/services/youtube.js

@@ -1,7 +1,7 @@
 import HLS from "hls-parser";
 
 import { fetch } from "undici";
-import { Innertube, Session } from "youtubei.js";
+import { Innertube, Session } from "@imput/youtubei.js";
 
 import { env } from "../../config.js";
 import { getCookie } from "../cookie/manager.js";
@@ -66,7 +66,6 @@ const cloneInnertube = async (customFetch, useSession) => {
             cookie,
             po_token: useSession ? sessionTokens?.potoken : undefined,
             visitor_data: useSession ? sessionTokens?.visitor_data : undefined,
-            player_id : "0004de42",
         });
         lastRefreshedAt = +new Date();
     }
@@ -225,7 +224,7 @@ export default async function (o) {
 
     let info;
     try {
-        info = await yt.getBasicInfo(o.id, { client: innertubeClient });
+        info = await yt.getBasicInfo(o.id, innertubeClient);
     } catch (e) {
         if (e?.info) {
             let errorInfo;

api/src/processing/url.js

@@ -99,7 +99,7 @@ function aliasURL(url) {
 
         case "xhslink":
             if (url.hostname === 'xhslink.com' && parts.length === 3) {
-                url = new URL(`https://www.xiaohongshu.com/${parts[1]}/${parts[2]}`);
+                url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
             }
             break;
 
@@ -147,7 +147,6 @@ function cleanURL(url) {
                 limitQuery('v');
             }
             break;
-        case "bilibili":
         case "rutube":
             if (url.searchParams.get('p')) {
                 limitQuery('p');

api/src/stream/shared.js

@@ -19,9 +19,6 @@ const serviceHeaders = {
     },
     vk: {
         'user-agent': vkClientAgent
-    },
-    tiktok: {
-        referer: 'https://www.tiktok.com/',
     }
 }
 

api/src/util/test.js

@@ -4,7 +4,7 @@ import { env } from "../config.js";
 import { runTest } from "../misc/run-test.js";
 import { loadJSON } from "../misc/load-from-fs.js";
 import { Red, Bright } from "../misc/console-text.js";
-import { setGlobalDispatcher, EnvHttpProxyAgent, ProxyAgent } from "undici";
+import { setGlobalDispatcher, ProxyAgent } from "undici";
 import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 
 import { services } from "../processing/service-config.js";
@@ -69,10 +69,9 @@ const printHeader = (service, padLen) => {
     console.log(service + '='.repeat(50));
 }
 
-// TODO: remove env.externalProxy in a future version
+if (env.externalProxy) {
-setGlobalDispatcher(
+    setGlobalDispatcher(new ProxyAgent(env.externalProxy));
-    new EnvHttpProxyAgent({ httpProxy: env.externalProxy || undefined })
+}
-);
 
 env.streamLifespan = 10000;
 env.apiURL = 'http://x/';

api/src/util/tests/bilibili.json

@@ -56,14 +56,5 @@
             "code": 200,
             "status": "tunnel"
         }
-    },
-    {
-        "name": "bilibili.com link with part id",
-        "url": "https://www.bilibili.com/video/BV1uo4y1K72s?spm_id_from=333.788.videopod.episodes&p=6",
-        "params": {},
-        "expected": {
-            "code": 200,
-            "status": "tunnel"
-        }
     }
 ]

api/src/util/tests/newgrounds.json

@@ -1,42 +0,0 @@
-[
-    {
-        "name": "regular video",
-        "url": "https://www.newgrounds.com/portal/view/938050",
-        "params": {},
-        "expected": {
-            "code": 200,
-            "status": "tunnel"
-        }
-    },
-    {
-        "name": "regular video (audio only)",
-        "url": "https://www.newgrounds.com/portal/view/938050",
-        "params": {
-            "downloadMode": "audio"
-        },
-        "expected": {
-            "code": 200,
-            "status": "tunnel"
-        }
-    },
-    {
-        "name": "regular video (muted)",
-        "url": "https://www.newgrounds.com/portal/view/938050",
-        "params": {
-            "downloadMode": "mute"
-        },
-        "expected": {
-            "code": 200,
-            "status": "tunnel"
-        }
-    },
-    {
-        "name": "regular music",
-        "url": "https://www.newgrounds.com/audio/listen/500476",
-        "params": {},
-        "expected": {
-            "code": 200,
-            "status": "tunnel"
-        }
-    }
-]

api/src/util/tests/soundcloud.json

@@ -86,7 +86,6 @@
     },
     {
         "name": "go+ song, should fail",
-        "canFail": true,
         "url": "https://soundcloud.com/dualipa/physical-feat-troye-sivan",
         "params": {},
         "expected": {

api/src/util/tests/twitch.json

@@ -29,14 +29,5 @@
             "code": 200,
             "status": "tunnel"
         }
-    },
-    {
-        "name": "clip (mobile subdomain)",
-        "url": "https://m.twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
-        "params": {},
-        "expected": {
-            "code": 200,
-            "status": "redirect"
-        }
     }
 ]

api/src/util/tests/xiaohongshu.json

@@ -1,7 +1,7 @@
 [
     {
         "name": "video (might have expired)",
-        "url": "https://www.xiaohongshu.com/explore/685e63e1000000000b02ee3b?xsec_token=ABN8EQJCDMPcFX9RRggeIPSHLIJ8zkGceFDyBewLGUz30=",
+        "url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=",
         "canFail": true,
         "params": {},
         "expected": {
@@ -11,7 +11,7 @@
     },
     {
         "name": "picker with multiple live photos (might have expired)",
-        "url": "https://www.xiaohongshu.com/explore/687128a2000000001203d94c?xsec_token=CBlDi5QDXDWZu2uUmbUrpKwg8lEL3uC10mc59lGf43r9w=",
+        "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=",
         "canFail": true,
         "params": {},
         "expected": {
@@ -21,7 +21,7 @@
     },
     {
         "name": "one photo (might have expired)",
-        "url": "https://www.xiaohongshu.com/explore/64726b99000000000800e115?xsec_token=ABoD3qPHqVZolCfS-J8UP9QQaPXZ6Z6PVyODrhaiUg27U=",
+        "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA",
         "canFail": true,
         "params": {},
         "expected": {
@@ -31,7 +31,7 @@
     },
     {
         "name": "short link (might have expired)",
-        "url": "https://xhslink.com/m/2wAnaTkLRc1",
+        "url": "https://xhslink.com/a/czn4z6c1tic4",
         "canFail": true,
         "params": {},
         "expected": {

docs/api-env-variables.md

@@ -21,15 +21,9 @@ this document is not final and will expand over time. feel free to improve it!
 | name                | default   | value example                         |
 |:--------------------|:----------|:--------------------------------------|
 | API_LISTEN_ADDRESS  | `0.0.0.0` | `127.0.0.1`                           |
+| API_EXTERNAL_PROXY  |           | `http://user:password@127.0.0.1:8080` |
 | FREEBIND_CIDR       |           | `2001:db8::/32`                       |
 
-#### undici proxy vars
-| name        | value example                         |
-|:------------|:--------------------------------------|
-| HTTP_PROXY  | `http://user:password@10.0.0.1:1337/` |
-| HTTPS_PROXY | `https://10.0.0.2:1337/`              |
-| NO_PROXY    | `localhost`                           |
-
 [*view details*](#networking)
 
 ### limit vars
@@ -129,23 +123,10 @@ defines the local address for the api instance. if you are using a docker contai
 
 the value is a local IP address.
 
-### HTTP_PROXY, HTTPS_PROXY, NO_PROXY
+### API_EXTERNAL_PROXY
-URL of the proxy that will be passed to [`EnvHttpProxyAgent`](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent) for proxying external requests. if some cobalt functionality breaks when using a proxy, please [make a new issue](https://github.com/imputnet/cobalt/issues) about it!
+URL of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only.
 
-quoted from [undici docs](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent):
+if some feature breaks when using a proxy, please make a new issue about it!
-> When `HTTP_PROXY` and `HTTPS_PROXY` are set, `HTTP_PROXY` is used for HTTP requests and `HTTPS_PROXY` is used for HTTPS requests. If only `HTTP_PROXY` is set, `HTTP_PROXY` is used for both HTTP and HTTPS requests. If only `HTTPS_PROXY` is set, it is only used for HTTPS requests.
-
-> `NO_PROXY` is a comma or space-separated list of hostnames that should not be proxied. The list may contain leading wildcard characters (`*`). If `NO_PROXY` is set, the EnvHttpProxyAgent() will bypass the proxy for requests to hosts that match the list. If `NO_PROXY` is set to `"*"`, the EnvHttpProxyAgent() will bypass the proxy for all requests.
-
-the value is a string:
-- `HTTP_PROXY`/`HTTPS_PROXY`: URL or hostname.
-- `NO_PROXY`: comma or space-separated list of hostnames.
-
-### API_EXTERNAL_PROXY (deprecated)
-> [!WARNING]
-> this env variable is deprecated and will be removed in a future release. please update your configuration to use `HTTP_PROXY` or `HTTPS_PROXY`, as mentioned above.
-
-URL of the proxy that will be passed to [`EnvHttpProxyAgent`](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent) and used for proxying external requests. HTTP(S) only.
 
 the value is a URL.
 

docs/examples/cookies.example.json

@@ -13,8 +13,5 @@
     ],
     "youtube": [
         "cookie=<replace_this>; b=<replace_this>"
-    ],
-    "vimeo": [
-        "access_token=<replace_this>"
     ]
 }

docs/run-an-instance.md

@@ -4,9 +4,9 @@ this tutorial will help you run your own cobalt processing instance. if your ins
 ## using docker compose and package from github (recommended)
 to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured.
 
-if you need help with installing docker, you can find more information here:
+if you need help with installing docker, follow *only the first step* of these tutorials by digitalocean:
-- [how to install docker](https://docs.docker.com/engine/install/)
+- [how to install docker](https://www.digitalocean.com/community/tutorial-collections/how-to-install-and-use-docker)
-- [how to install docker compose](https://docs.docker.com/compose/install/)
+- [how to install docker compose](https://www.digitalocean.com/community/tutorial-collections/how-to-install-docker-compose)
 
 ## how to run a cobalt docker package:
 1. create a folder for cobalt config file, something like this:

pnpm-lock.yaml

@@ -19,6 +19,9 @@ importers:
       '@imput/version-info':
         specifier: workspace:^
         version: link:../packages/version-info
+      '@imput/youtubei.js':
+        specifier: ^14.0.0
+        version: 14.0.0
       content-disposition-header:
         specifier: 0.6.0
         version: 0.6.0
@@ -58,9 +61,6 @@ importers:
       url-pattern:
         specifier: 1.0.3
         version: 1.0.3
-      youtubei.js:
-        specifier: 15.1.1
-        version: 15.1.1
       zod:
         specifier: ^3.23.8
         version: 3.23.8
@@ -563,6 +563,9 @@ packages:
   '@imput/psl@2.0.4':
     resolution: {integrity: sha512-vuy76JX78/DnJegLuJoLpMmw11JTA/9HvlIADg/f8dDVXyxbh0jnObL0q13h+WvlBO4Gk26Pu8sUa7/h0JGQig==}
 
+  '@imput/youtubei.js@14.0.0':
+    resolution: {integrity: sha512-YvTnh53URPlzsmMzqF/DFHZyR9HrpgoWYHzEOklx5OCkwk1/0F/CrO9gqArXw/1oI6GjaTS2CqBd1CzyFZB07A==}
+
   '@isaacs/cliui@8.0.2':
     resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
     engines: {node: '>=12'}
@@ -1876,7 +1879,6 @@ packages:
   source-map@0.8.0-beta.0:
     resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==}
     engines: {node: '>= 8'}
-    deprecated: The work that was done in this beta branch won't be included in future versions
 
   sprintf-js@1.0.3:
     resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
@@ -2186,9 +2188,6 @@ packages:
     resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
     engines: {node: '>=10'}
 
-  youtubei.js@15.1.1:
-    resolution: {integrity: sha512-fuEDj9Ky6cAQg93BrRVCbr+GTYNZQAIFZrx/a3oDRuGc3Mf5bS0dQfoYwwgjtSV7sgAKQEEdGtzRdBzOc8g72Q==}
-
   zimmerframe@1.1.2:
     resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==}
 
@@ -2429,6 +2428,13 @@ snapshots:
     dependencies:
       punycode: 2.3.1
 
+  '@imput/youtubei.js@14.0.0':
+    dependencies:
+      '@bufbuild/protobuf': 2.2.5
+      jintr: 3.3.1
+      tslib: 2.6.3
+      undici: 6.21.3
+
   '@isaacs/cliui@8.0.2':
     dependencies:
       string-width: 5.1.2
@@ -4044,12 +4050,6 @@ snapshots:
 
   yocto-queue@0.1.0: {}
 
-  youtubei.js@15.1.1:
-    dependencies:
-      '@bufbuild/protobuf': 2.2.5
-      jintr: 3.3.1
-      undici: 6.21.3
-
   zimmerframe@1.1.2: {}
 
   zod@3.23.8: {}

web/changelogs/11.2.md

@@ -51,7 +51,6 @@ by the way, we also made it possible to [choose any preferred media container](/
 - pinterest now returns an appropriate error when a pin is unavailable.
 - AI dubs on youtube are no longer accidentally selected as default tracks.
 - youtube HLS preference is now deprecated, but can be enabled on a self-hosted instance via `ENABLE_DEPRECATED_YOUTUBE_HLS` in env.
-- downloads from vk are now way faster.
 
 ## web app improvements
 - improved compatibility of local processing & related code with older browsers.

web/i18n/en/save.json

@@ -10,7 +10,7 @@
     "services.title": "supported services",
     "services.title_show": "show supported services",
     "services.title_hide": "hide supported services",
-    "services.disclaimer": "support for a service does not imply affiliation, endorsement, or any form of support other than technical compatibility.",
+    "services.disclaimer": "cobalt is not affiliated with any of the services listed above.",
 
     "tutorial.title": "how to save on ios?",
     "tutorial.intro": "to save media conveniently on ios, you'll need to use a companion siri shortcut from the share sheet.",

web/i18n/ru/save.json

@@ -10,7 +10,7 @@
     "services.title": "поддерживаемые сервисы",
     "services.title_show": "показать поддерживаемые сервисы",
     "services.title_hide": "скрыть поддерживаемые сервисы",
-    "services.disclaimer": "поддержка сервиса не означает аффилированность, одобрение или любую другую форму поддержки, кроме технической совместимости.\n\nдеятельность владельца facebook и instagram запрещена на территории РФ и признана экстремистской.",
+    "services.disclaimer": "кобальт не аффилирован ни с одним из перечисленных выше сервисов.\n\nдеятельность meta platforms (владельца facebook и instagram) запрещена на территории РФ и признана экстремистской.",
     "tutorial.step.1": "добавь команды-компаньоны:",
     "tutorial.step.2": "нажми кнопку \"поделиться\" в диалоге сохранения кобальта.",
     "tutorial.step.3": "выбери нужную команду в окне обмена.",

web/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@imput/cobalt-web",
-    "version": "11.3",
+    "version": "11.2.3",
     "type": "module",
     "private": true,
     "scripts": {