merge: updates from main

api/src/processing/services/pinterest.js

@@ -23,7 +23,7 @@ export default async function(o) {
 
     const videoLink = [...html.matchAll(videoRegex)]
                     .map(([, link]) => link)
-                    .find(a => a.endsWith('.mp4') && a.includes('720p'));
+                    .find(a => a.endsWith('.mp4'));
 
     if (videoLink) return {
         urls: videoLink,

api/src/processing/services/twitter.js

@@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000);
 
 function needsFixing(media) {
     const representativeId = media.source_status_id_str ?? media.id_str;
+
+    // syndication api doesn't have media ids in its response,
+    // so we just assume it's all good
+    if (!representativeId) return false;
+
     const mediaTimestamp = new Date(
         Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
     );
@@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => {
     }
 }
 
+const requestSyndication = async(dispatcher, tweetId) => {
+    // thank you
+    // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334
+    const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '');
+    const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result");
+
+    syndicationUrl.searchParams.set("id", tweetId);
+    syndicationUrl.searchParams.set("token", token(tweetId));
+
+    const result = await fetch(syndicationUrl, {
+        headers: {
+            "user-agent": genericUserAgent
+        },
+        dispatcher
+    });
+
+    return result;
+}
+
 const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     const graphqlTweetURL = new URL(graphqlURL);
 
@@ -87,36 +111,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     let result = await fetch(graphqlTweetURL, { headers, dispatcher });
     updateCookie(cookie, result.headers);
 
-    // we might have been missing the `ct0` cookie, retry
+    // we might have been missing the ct0 cookie, retry
     if (result.status === 403 && result.headers.get('set-cookie')) {
-        result = await fetch(graphqlTweetURL, {
-            headers: {
-                ...headers,
-                'x-csrf-token': cookie.values().ct0
-            },
-            dispatcher
-        });
+        const cookieValues = cookie?.values();
+        if (cookieValues?.ct0) {
+            result = await fetch(graphqlTweetURL, {
+                headers: {
+                    ...headers,
+                    'x-csrf-token': cookieValues.ct0
+                },
+                dispatcher
+            });
+        }
     }
 
     return result
 }
 
-export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
-    const cookie = await getCookie('twitter');
-
-    let guestToken = await getGuestToken(dispatcher);
-    if (!guestToken) return { error: "fetch.fail" };
-
-    let tweet = await requestTweet(dispatcher, id, guestToken);
-
-    // get new token & retry if old one expired
-    if ([403, 429].includes(tweet.status)) {
-        guestToken = await getGuestToken(dispatcher, true);
-        tweet = await requestTweet(dispatcher, id, guestToken)
-    }
-
-    tweet = await tweet.json();
-
+const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {
     let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
 
     if (!tweetTypename) {
@@ -127,13 +139,13 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         const reason = tweet?.data?.tweetResult?.result?.reason;
         switch(reason) {
             case "Protected":
-                return { error: "content.post.private" }
+                return { error: "content.post.private" };
             case "NsfwLoggedOut":
                 if (cookie) {
                     tweet = await requestTweet(dispatcher, id, guestToken, cookie);
                     tweet = await tweet.json();
                     tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-                } else return { error: "content.post.age" }
+                } else return { error: "content.post.age" };
         }
     }
 
@@ -150,7 +162,69 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
     }
 
-    let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
+    return (repostedTweet?.media || baseTweet?.extended_entities?.media);
+}
+
+const testResponse = (result) => {
+    const contentLength = result.headers.get("content-length");
+
+    if (!contentLength || contentLength === '0') {
+        return false;
+    }
+
+    if (!result.headers.get("content-type").startsWith("application/json")) {
+        return false;
+    }
+
+    return true;
+}
+
+export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
+    const cookie = await getCookie('twitter');
+
+    let syndication = false;
+
+    let guestToken = await getGuestToken(dispatcher);
+    if (!guestToken) return { error: "fetch.fail" };
+
+    // for now we assume that graphql api will come back after some time,
+    // so we try it first
+
+    let tweet = await requestTweet(dispatcher, id, guestToken);
+
+    // get new token & retry if old one expired
+    if ([403, 429].includes(tweet.status)) {
+        guestToken = await getGuestToken(dispatcher, true);
+        if (cookie) {
+            tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+        } else {
+            tweet = await requestTweet(dispatcher, id, guestToken);
+        }
+    }
+
+    const testGraphql = testResponse(tweet);
+
+    // if graphql requests fail, then resort to tweet embed api
+    if (!testGraphql) {
+        syndication = true;
+        tweet = await requestSyndication(dispatcher, id);
+
+        const testSyndication = testResponse(tweet);
+
+        // if even syndication request failed, then cry out loud
+        if (!testSyndication) {
+            return { error: "fetch.fail" };
+        }
+    }
+
+    tweet = await tweet.json();
+
+    let media =
+        syndication
+            ? tweet.mediaDetails
+            : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+
+    if (!media) return { error: "fetch.empty" };
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {
@@ -163,7 +237,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         service: "twitter",
         type: "proxy",
         url, filename,
-    })
+    });
 
     switch (media?.length) {
         case undefined:

api/src/processing/services/youtube.js

@@ -428,6 +428,10 @@ export default async function (o) {
         }
     }
 
+    if (video?.drm_families || audio?.drm_families) {
+        return { error: "youtube.drm" };
+    }
+
     const fileMetadata = {
         title: basicInfo.title.trim(),
         artist: basicInfo.author.replace("- Topic", "").trim()

api/src/util/tests/twitter.json

@@ -169,6 +169,15 @@
             "status": "tunnel"
         }
     },
+    {
+        "name": "gif",
+        "url": "https://x.com/thelastromances/status/1897839691212202479",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
     {
         "name": "inexistent post",
         "url": "https://twitter.com/test/status/9487653",