merge: 10.6 updates

2025-01-21 13:34:19 +00:00
parent 192635f2ce 8d3db909d9
commit c4c47bdc27
25 changed files with 466 additions and 79 deletions
--- a/api/src/processing/match-action.js
+++ b/api/src/processing/match-action.js
@@ -15,7 +15,8 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
            filename: r.filenameAttributes ?
                    createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
            fileMetadata: !disableMetadata ? r.fileMetadata : false,
-            requestIP
+            requestIP,
+            originalRequest: r.originalRequest
        },
        params = {};

@@ -47,7 +48,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
            });

        case "photo":
-            responseType = "redirect";
+            params = { type: "proxy" };
            break;

        case "gif":
@@ -83,6 +84,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                case "twitter":
                case "snapchat":
                case "bsky":
+                case "xiaohongshu":
                    params = { picker: r.picker };
                    break;

@@ -143,6 +145,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                case "ok":
                case "vk":
                case "tiktok":
+                case "xiaohongshu":
                    params = { type: "proxy" };
                    break;

--- a/api/src/processing/match.js
+++ b/api/src/processing/match.js
@@ -28,6 +28,7 @@ import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
+import xiaohongshu from "./services/xiaohongshu.js";

 let freebind;

@@ -239,6 +240,15 @@ export default async function({ host, patternMatch, params }) {
                });
                break;

+            case "xiaohongshu":
+                r = await xiaohongshu({
+                    ...patternMatch,
+                    h265: params.tiktokH265,
+                    isAudioOnly,
+                    dispatcher,
+                });
+                break;
+
            default:
                return createResponse("error", {
                    code: "error.api.service.unsupported"
--- a/api/src/processing/service-config.js
+++ b/api/src/processing/service-config.js
@@ -166,6 +166,14 @@ export const services = {
        subdomains: ["m"],
        altDomains: ["vkvideo.ru", "vk.ru"],
    },
+    xiaohongshu: {
+        patterns: [
+            "explore/:id?xsec_token=:token",
+            "discovery/item/:id?xsec_token=:token",
+            "a/:shareId"
+        ],
+        altDomains: ["xhslink.com"],
+    },
    youtube: {
        patterns: [
            "watch?v=:id",
--- a/api/src/processing/service-patterns.js
+++ b/api/src/processing/service-patterns.js
@@ -71,4 +71,8 @@ export const testers = {

    "bsky": pattern =>
        pattern.user?.length <= 128 && pattern.post?.length <= 128,
+
+    "xiaohongshu": pattern =>
+        pattern.id?.length <= 24 && pattern.token?.length <= 64
+        || pattern.shareId?.length <= 12,
 }
--- a/api/src/processing/services/bluesky.js
+++ b/api/src/processing/services/bluesky.js
@@ -71,6 +71,24 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
    return { picker };
 }

+const extractGif = ({ url, filename }) => {
+    const gifUrl = new URL(url);
+
+    if (!gifUrl || gifUrl.hostname !== "media.tenor.com") {
+        return { error: "fetch.empty" };
+    }
+
+    // remove downscaling params from gif url
+    // such as "?hh=498&ww=498"
+    gifUrl.search = "";
+
+    return {
+        urls: gifUrl,
+        isPhoto: true,
+        filename: `${filename}.gif`,
+    }
+}
+
 export default async function ({ user, post, alwaysProxy, dispatcher }) {
    const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
    apiEndpoint.searchParams.set(
@@ -102,22 +120,37 @@ export default async function ({ user, post, alwaysProxy, dispatcher }) {
    const embedType = getPost?.thread?.post?.embed?.$type;
    const filename = `bluesky_${user}_${post}`;

-    if (embedType === "app.bsky.embed.video#view") {
-        return extractVideo({
-            media: getPost.thread?.post?.embed,
-            filename,
-        })
-    }
+    switch (embedType) {
+        case "app.bsky.embed.video#view":
+            return extractVideo({
+                media: getPost.thread?.post?.embed,
+                filename,
+            });

-    if (embedType === "app.bsky.embed.recordWithMedia#view") {
-        return extractVideo({
-            media: getPost.thread?.post?.embed?.media,
-            filename,
-        })
-    }
+        case "app.bsky.embed.images#view":
+            return extractImages({
+                getPost,
+                filename,
+                alwaysProxy
+            });

-    if (embedType === "app.bsky.embed.images#view") {
-        return extractImages({ getPost, filename, alwaysProxy });
+        case "app.bsky.embed.external#view":
+            return extractGif({
+                url: getPost?.thread?.post?.embed?.external?.uri,
+                filename,
+            });
+
+        case "app.bsky.embed.recordWithMedia#view":
+            if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") {
+                return extractGif({
+                    url: getPost?.thread?.post?.embed?.media?.external?.uri,
+                    filename,
+                });
+            }
+            return extractVideo({
+                media: getPost.thread?.post?.embed?.media,
+                filename,
+            });
    }

    return { error: "fetch.empty" };
--- a/api/src/processing/services/tiktok.js
+++ b/api/src/processing/services/tiktok.js
@@ -30,7 +30,7 @@ export default async function(obj) {
    if (!postId) return { error: "fetch.short_link" };

    // should always be /video/, even for photos
-    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
+    const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, {
        headers: {
            "user-agent": genericUserAgent,
            cookie,
--- a/api/src/processing/services/xiaohongshu.js
+++ b/api/src/processing/services/xiaohongshu.js
@@ -0,0 +1,116 @@
+import { extract, normalizeURL } from "../url.js";
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+import { getRedirectingURL } from "../../misc/utils.js";
+
+const https = (url) => {
+    return url.replace(/^http:/i, 'https:');
+}
+
+export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
+    let noteId = id;
+    let xsecToken = token;
+
+    if (!noteId) {
+        const extractedURL = await getRedirectingURL(
+            `https://xhslink.com/a/${shareId}`,
+            dispatcher
+        );
+
+        if (extractedURL) {
+            const { patternMatch } = extract(normalizeURL(extractedURL));
+
+            if (patternMatch) {
+                noteId = patternMatch.id;
+                xsecToken = patternMatch.token;
+            }
+        }
+    }
+
+    if (!noteId || !xsecToken) return { error: "fetch.short_link" };
+
+    const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, {
+        headers: {
+            "user-agent": genericUserAgent,
+        },
+        dispatcher,
+    });
+
+    const html = await res.text();
+
+    let note;
+    try {
+        const initialState = html
+            .split('<script>window.__INITIAL_STATE__=')[1]
+            .split('</script>')[0]
+            .replace(/:\s*undefined/g, ":null");
+
+        const data = JSON.parse(initialState);
+
+        const noteInfo = data?.note?.noteDetailMap;
+        if (!noteInfo) throw "no note detail map";
+
+        const currentNote = noteInfo[noteId];
+        if (!currentNote) throw "no current note in detail map";
+
+        note = currentNote.note;
+    } catch {}
+
+    if (!note) return { error: "fetch.empty" };
+
+    const video = note.video;
+    const images = note.imageList;
+
+    const filenameBase = `xiaohongshu_${noteId}`;
+
+    if (video) {
+        const videoFilename = `${filenameBase}.mp4`;
+        const audioFilename = `${filenameBase}_audio`;
+
+        let videoURL;
+
+        if (h265 && !isAudioOnly && video.consumer?.originVideoKey) {
+            videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`;
+        } else {
+            const h264Streams = video.media?.stream?.h264;
+
+            if (h264Streams?.length) {
+                videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl;
+            }
+        }
+
+        if (!videoURL) return { error: "fetch.empty" };
+
+        return {
+            urls: https(videoURL),
+            filename: videoFilename,
+            audioFilename: audioFilename,
+        }
+    }
+
+    if (!images || images.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    if (images.length === 1) {
+        return {
+            isPhoto: true,
+            urls: https(images[0].urlDefault),
+            filename: `${filenameBase}.jpg`,
+        }
+    }
+
+    const picker = images.map((image, i) => {
+        return {
+            type: "photo",
+            url: createStream({
+                service: "xiaohongshu",
+                type: "proxy",
+                url: https(image.urlDefault),
+                filename: `${filenameBase}_${i + 1}.jpg`,
+            })
+        }
+    });
+
+    return { picker };
+}
--- a/api/src/processing/services/youtube.js
+++ b/api/src/processing/services/youtube.js
@@ -149,7 +149,7 @@ export default async function (o) {
        useHLS = false;
    }

-    let innertubeClient = "ANDROID";
+    let innertubeClient = o.innertubeClient || "ANDROID";

    if (cookie) {
        useHLS = false;
@@ -240,12 +240,12 @@ export default async function (o) {
    const quality = o.quality === "max" ? 9000 : Number(o.quality);

    const normalizeQuality = res => {
-        const shortestSide = res.height > res.width ? res.width : res.height;
+        const shortestSide = Math.min(res.height, res.width);
        return videoQualities.find(qual => qual >= shortestSide);
    }

    let video, audio, dubbedLanguage,
-        codec = o.format || "h264";
+        codec = o.format || "h264", itag = o.itag;

    if (useHLS) {
        const hlsManifest = info.streaming_data.hls_manifest_url;
@@ -351,17 +351,21 @@ export default async function (o) {
            Number(b.bitrate) - Number(a.bitrate)
        ).forEach(format => {
            Object.keys(codecList).forEach(yCodec => {
+                const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag;
                const sorted = sorted_formats[yCodec];
                const goodFormat = checkFormat(format, yCodec);
                if (!goodFormat) return;

-                if (format.has_video) {
+                if (format.has_video && matchingItag('video')) {
                    sorted.video.push(format);
-                    if (!sorted.bestVideo) sorted.bestVideo = format;
+                    if (!sorted.bestVideo)
+                        sorted.bestVideo = format;
                }
-                if (format.has_audio) {
+
+                if (format.has_audio && matchingItag('audio')) {
                    sorted.audio.push(format);
-                    if (!sorted.bestAudio) sorted.bestAudio = format;
+                    if (!sorted.bestAudio)
+                        sorted.bestAudio = format;
                }
            })
        });
@@ -448,6 +452,18 @@ export default async function (o) {
        youtubeDubName: dubbedLanguage || false,
    }

+    itag = {
+        video: video?.itag,
+        audio: audio?.itag
+    };
+
+    const originalRequest = {
+        ...o,
+        dispatcher: undefined,
+        itag,
+        innertubeClient
+    };
+
    if (audio && o.isAudioOnly) {
        let bestAudio = codec === "h264" ? "m4a" : "opus";
        let urls = audio.url;
@@ -469,6 +485,7 @@ export default async function (o) {
            fileMetadata,
            bestAudio,
            isHLS: useHLS,
+            originalRequest
        }
    }

@@ -491,12 +508,12 @@ export default async function (o) {
            filenameAttributes.resolution = `${video.width}x${video.height}`;
            filenameAttributes.extension = codecList[codec].container;

-            video = video.url;
-            audio = audio.url;
-
            if (innertubeClient === "WEB" && innertube) {
                video = video.decipher(innertube.session.player);
                audio = audio.decipher(innertube.session.player);
+            } else {
+                video = video.url;
+                audio = audio.url;
            }
        }

@@ -512,6 +529,7 @@ export default async function (o) {
            filenameAttributes,
            fileMetadata,
            isHLS: useHLS,
+            originalRequest
        }
    }

--- a/api/src/processing/url.js
+++ b/api/src/processing/url.js
@@ -92,9 +92,14 @@ function aliasURL(url) {
                url.hostname = 'vk.com';
            }
            break;
+
+        case "xhslink":
+            if (url.hostname === 'xhslink.com' && parts.length === 3) {
+                url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
+            }
    }

-    return url
+    return url;
 }

 function cleanURL(url) {
@@ -114,36 +119,41 @@ function cleanURL(url) {
            break;
        case "vk":
            if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
-                limitQuery('z')
+                limitQuery('z');
            }
            break;
        case "youtube":
            if (url.searchParams.get('v')) {
-                limitQuery('v')
+                limitQuery('v');
            }
            break;
        case "rutube":
            if (url.searchParams.get('p')) {
-                limitQuery('p')
+                limitQuery('p');
            }
            break;
        case "twitter":
            if (url.searchParams.get('post_id')) {
-                limitQuery('post_id')
+                limitQuery('post_id');
+            }
+            break;
+        case "xiaohongshu":
+            if (url.searchParams.get('xsec_token')) {
+                limitQuery('xsec_token');
            }
            break;
    }

    if (stripQuery) {
-        url.search = ''
+        url.search = '';
    }

-    url.username = url.password = url.port = url.hash = ''
+    url.username = url.password = url.port = url.hash = '';

    if (url.pathname.endsWith('/'))
        url.pathname = url.pathname.slice(0, -1);

-    return url
+    return url;
 }

 function getHostIfValid(url) {