merge: 10.6 updates

api/src/processing/services/bluesky.js

@@ -71,6 +71,24 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
     return { picker };
 }
 
+const extractGif = ({ url, filename }) => {
+    const gifUrl = new URL(url);
+
+    if (!gifUrl || gifUrl.hostname !== "media.tenor.com") {
+        return { error: "fetch.empty" };
+    }
+
+    // remove downscaling params from gif url
+    // such as "?hh=498&ww=498"
+    gifUrl.search = "";
+
+    return {
+        urls: gifUrl,
+        isPhoto: true,
+        filename: `${filename}.gif`,
+    }
+}
+
 export default async function ({ user, post, alwaysProxy, dispatcher }) {
     const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
     apiEndpoint.searchParams.set(
@@ -102,22 +120,37 @@ export default async function ({ user, post, alwaysProxy, dispatcher }) {
     const embedType = getPost?.thread?.post?.embed?.$type;
     const filename = `bluesky_${user}_${post}`;
 
-    if (embedType === "app.bsky.embed.video#view") {
-        return extractVideo({
-            media: getPost.thread?.post?.embed,
-            filename,
-        })
-    }
+    switch (embedType) {
+        case "app.bsky.embed.video#view":
+            return extractVideo({
+                media: getPost.thread?.post?.embed,
+                filename,
+            });
 
-    if (embedType === "app.bsky.embed.recordWithMedia#view") {
-        return extractVideo({
-            media: getPost.thread?.post?.embed?.media,
-            filename,
-        })
-    }
+        case "app.bsky.embed.images#view":
+            return extractImages({
+                getPost,
+                filename,
+                alwaysProxy
+            });
 
-    if (embedType === "app.bsky.embed.images#view") {
-        return extractImages({ getPost, filename, alwaysProxy });
+        case "app.bsky.embed.external#view":
+            return extractGif({
+                url: getPost?.thread?.post?.embed?.external?.uri,
+                filename,
+            });
+
+        case "app.bsky.embed.recordWithMedia#view":
+            if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") {
+                return extractGif({
+                    url: getPost?.thread?.post?.embed?.media?.external?.uri,
+                    filename,
+                });
+            }
+            return extractVideo({
+                media: getPost.thread?.post?.embed?.media,
+                filename,
+            });
     }
 
     return { error: "fetch.empty" };

api/src/processing/services/tiktok.js

@@ -30,7 +30,7 @@ export default async function(obj) {
     if (!postId) return { error: "fetch.short_link" };
 
     // should always be /video/, even for photos
-    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
+    const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, {
         headers: {
             "user-agent": genericUserAgent,
             cookie,

api/src/processing/services/xiaohongshu.js

@@ -0,0 +1,116 @@
+import { extract, normalizeURL } from "../url.js";
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+import { getRedirectingURL } from "../../misc/utils.js";
+
+const https = (url) => {
+    return url.replace(/^http:/i, 'https:');
+}
+
+export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
+    let noteId = id;
+    let xsecToken = token;
+
+    if (!noteId) {
+        const extractedURL = await getRedirectingURL(
+            `https://xhslink.com/a/${shareId}`,
+            dispatcher
+        );
+
+        if (extractedURL) {
+            const { patternMatch } = extract(normalizeURL(extractedURL));
+
+            if (patternMatch) {
+                noteId = patternMatch.id;
+                xsecToken = patternMatch.token;
+            }
+        }
+    }
+
+    if (!noteId || !xsecToken) return { error: "fetch.short_link" };
+
+    const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, {
+        headers: {
+            "user-agent": genericUserAgent,
+        },
+        dispatcher,
+    });
+
+    const html = await res.text();
+
+    let note;
+    try {
+        const initialState = html
+            .split('<script>window.__INITIAL_STATE__=')[1]
+            .split('</script>')[0]
+            .replace(/:\s*undefined/g, ":null");
+
+        const data = JSON.parse(initialState);
+
+        const noteInfo = data?.note?.noteDetailMap;
+        if (!noteInfo) throw "no note detail map";
+
+        const currentNote = noteInfo[noteId];
+        if (!currentNote) throw "no current note in detail map";
+
+        note = currentNote.note;
+    } catch {}
+
+    if (!note) return { error: "fetch.empty" };
+
+    const video = note.video;
+    const images = note.imageList;
+
+    const filenameBase = `xiaohongshu_${noteId}`;
+
+    if (video) {
+        const videoFilename = `${filenameBase}.mp4`;
+        const audioFilename = `${filenameBase}_audio`;
+
+        let videoURL;
+
+        if (h265 && !isAudioOnly && video.consumer?.originVideoKey) {
+            videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`;
+        } else {
+            const h264Streams = video.media?.stream?.h264;
+
+            if (h264Streams?.length) {
+                videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl;
+            }
+        }
+
+        if (!videoURL) return { error: "fetch.empty" };
+
+        return {
+            urls: https(videoURL),
+            filename: videoFilename,
+            audioFilename: audioFilename,
+        }
+    }
+
+    if (!images || images.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    if (images.length === 1) {
+        return {
+            isPhoto: true,
+            urls: https(images[0].urlDefault),
+            filename: `${filenameBase}.jpg`,
+        }
+    }
+
+    const picker = images.map((image, i) => {
+        return {
+            type: "photo",
+            url: createStream({
+                service: "xiaohongshu",
+                type: "proxy",
+                url: https(image.urlDefault),
+                filename: `${filenameBase}_${i + 1}.jpg`,
+            })
+        }
+    });
+
+    return { picker };
+}

api/src/processing/services/youtube.js

@@ -149,7 +149,7 @@ export default async function (o) {
         useHLS = false;
     }
 
-    let innertubeClient = "ANDROID";
+    let innertubeClient = o.innertubeClient || "ANDROID";
 
     if (cookie) {
         useHLS = false;
@@ -240,12 +240,12 @@ export default async function (o) {
     const quality = o.quality === "max" ? 9000 : Number(o.quality);
 
     const normalizeQuality = res => {
-        const shortestSide = res.height > res.width ? res.width : res.height;
+        const shortestSide = Math.min(res.height, res.width);
         return videoQualities.find(qual => qual >= shortestSide);
     }
 
     let video, audio, dubbedLanguage,
-        codec = o.format || "h264";
+        codec = o.format || "h264", itag = o.itag;
 
     if (useHLS) {
         const hlsManifest = info.streaming_data.hls_manifest_url;
@@ -351,17 +351,21 @@ export default async function (o) {
             Number(b.bitrate) - Number(a.bitrate)
         ).forEach(format => {
             Object.keys(codecList).forEach(yCodec => {
+                const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag;
                 const sorted = sorted_formats[yCodec];
                 const goodFormat = checkFormat(format, yCodec);
                 if (!goodFormat) return;
 
-                if (format.has_video) {
+                if (format.has_video && matchingItag('video')) {
                     sorted.video.push(format);
-                    if (!sorted.bestVideo) sorted.bestVideo = format;
+                    if (!sorted.bestVideo)
+                        sorted.bestVideo = format;
                 }
-                if (format.has_audio) {
+
+                if (format.has_audio && matchingItag('audio')) {
                     sorted.audio.push(format);
-                    if (!sorted.bestAudio) sorted.bestAudio = format;
+                    if (!sorted.bestAudio)
+                        sorted.bestAudio = format;
                 }
             })
         });
@@ -448,6 +452,18 @@ export default async function (o) {
         youtubeDubName: dubbedLanguage || false,
     }
 
+    itag = {
+        video: video?.itag,
+        audio: audio?.itag
+    };
+
+    const originalRequest = {
+        ...o,
+        dispatcher: undefined,
+        itag,
+        innertubeClient
+    };
+
     if (audio && o.isAudioOnly) {
         let bestAudio = codec === "h264" ? "m4a" : "opus";
         let urls = audio.url;
@@ -469,6 +485,7 @@ export default async function (o) {
             fileMetadata,
             bestAudio,
             isHLS: useHLS,
+            originalRequest
         }
     }
 
@@ -491,12 +508,12 @@ export default async function (o) {
             filenameAttributes.resolution = `${video.width}x${video.height}`;
             filenameAttributes.extension = codecList[codec].container;
 
-            video = video.url;
-            audio = audio.url;
-
             if (innertubeClient === "WEB" && innertube) {
                 video = video.decipher(innertube.session.player);
                 audio = audio.decipher(innertube.session.player);
+            } else {
+                video = video.url;
+                audio = audio.url;
             }
         }
 
@@ -512,6 +529,7 @@ export default async function (o) {
             filenameAttributes,
             fileMetadata,
             isHLS: useHLS,
+            originalRequest
         }
     }