Add api/meow.js

* the android client now also needs a valid device_identifier, but already generated access_tokens work fine
* you can also get them from a rooted device by mitm'ing the device while starting vimeo for the first time or going into its data directory and searching for the token in shared_prefs/

.github/test.sh

@@ -18,7 +18,7 @@ test_api() {
          -X POST \
          -H "Accept: application/json" \
          -H "Content-Type: application/json" \
-         -d '{"url":"https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894"}')
+         -d '{"url":"https://garfield-69.tumblr.com/post/696499862852780032","alwaysProxy":true}')
 
     echo "API_RESPONSE=$API_RESPONSE"
     STATUS=$(echo "$API_RESPONSE" | jq -r .status)
@@ -46,6 +46,7 @@ setup_api() {
 }
 
 setup_web() {
+    pnpm run --prefix web check
     pnpm run --prefix web build
 }
 

.github/workflows/codeql.yml

@@ -0,0 +1,93 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches: [ "main", "7" ]
+  schedule:
+    - cron: '33 7 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: javascript-typescript
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker-develop.yml

@@ -1,4 +1,4 @@
-name: Build Docker development image
+name: Build development Docker image
 
 on:
   workflow_dispatch:

.github/workflows/docker-staging.yml

@@ -0,0 +1,55 @@
+name: Build staging Docker image
+
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get release metadata
+        id: release-meta
+        run: |
+            version=$(cat package.json | jq -r .version)
+            echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+            echo "version=$version" >> $GITHUB_OUTPUT
+            echo "major_version=$(echo "$version" | cut -d. -f1)" >> $GITHUB_OUTPUT
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          tags: type=raw,value=staging
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/docker.yml

@@ -1,4 +1,4 @@
-name: Build Docker image
+name: Build release Docker image
 
 on:
   workflow_dispatch:
@@ -51,7 +51,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test-services.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
       - id: checkServices
-        run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test-ci get-services)" >> "$GITHUB_OUTPUT"
+        run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test get-services)" >> "$GITHUB_OUTPUT"
 
   test-services:
     needs: check-services
@@ -30,4 +30,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-      - run: pnpm i --frozen-lockfile && node api/src/util/test-ci run-tests-for ${{ matrix.service }}
+      - run: pnpm i --frozen-lockfile
+      - run: node api/src/util/test run-tests-for ${{ matrix.service }}
+        env:
+          HTTP_PROXY: ${{ secrets.API_EXTERNAL_PROXY }}
+          TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }}

.github/workflows/test.yml

@@ -24,6 +24,8 @@ jobs:
           node-version: 'lts/*'
       - uses: pnpm/action-setup@v4
       - run: .github/test.sh web
+        env:
+          WEB_DEFAULT_API: https://api.dummy.example/
 
   test-api:
     name: api sanity check
@@ -31,4 +33,4 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
-      - run: .github/test.sh api
+      - run: .github/test.sh api

.gitignore

@@ -13,6 +13,7 @@ build
 .env.*
 !.env.example
 cookies.json
+keys.json
 
 # docker
 docker-compose.yml

CONTRIBUTING.md

@@ -4,7 +4,23 @@ if you're reading this, you are probably interested in contributing to cobalt, w
 this document serves as a guide to help you make contributions that we can merge into the cobalt codebase.
 
 ## translations
-currently, we are **not accepting** translations of cobalt. this is because we are making significant changes to the frontend, and the currently used localization structure is being completely reworked. if this changes, this document will be updated.
+we are currently accepting translations via the [i18n platform](https://i18n.imput.net).
+
+thank you for showing interest in making cobalt more accessible around the world, we really appreciate it! here are some guidelines for how a cobalt translation should look:
+
+- cobalt's writing style is informal. please do not use formal language, unless there is no other way to express the same idea of the original text in your language.
+- all cobalt text is written in lowercase. this is a stylistic choice, please do not capitalize translated sentences.
+- do not translate the name "cobalt", or "imput"
+- you can translate "meowbalt" into whatever your language's equivalent of _meow_ is (e.g. _miaubalt_ in German)
+- **please don't translate cobalt into languages which you are not experienced in.** we can use google translate ourselves, but we would prefer cobalt to be translated by humans, not computers.
+
+if your language does not exist on the translation platform yet, you can request to add it by adding it to any of cobalt's components (e.g. [here](https://i18n.imput.net/projects/cobalt/about/)).
+
+before translating a piece of text, check that no one has made a translation yet. pending translations are displayed in the **Suggestions** tab on the translate page. if someone already made a suggestion, and you think it's correct, you can upvote it! this helps us distinguish that a translation is correct.
+
+if no one has submitted a translation, or the submitted translation seems wrong to you, you can submit your translation by clicking the **Suggest** button for each individual string, which sends it off for human review. we will then check it to to ensure no malicious translations are submitted, and add it to cobalt.
+
+if any translation string's meaning seems unclear to you, please leave a comment on the *Comments* tab, and we will either add an explanation or a screenshot.
 
 ## adding features or support for services
 before putting in the effort to implement a feature, it's worth considering whether it would be appropriate to add it to cobalt. the cobalt api is built to assist people **only with downloading freely accessible content**. other functionality, such as:
@@ -22,9 +38,9 @@ when contributing code to cobalt, there are a few guidelines in place to ensure
 ### clean commit messages
 internally, we use a format similar to [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) - the first part signifies which part of the code you are changing (the *scope*), and the second part explains the change. for inspiration on how to write appropriate commit titles, you can take a look at the [commit history](https://github.com/imputnet/cobalt/commits/).
 
-the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `stream/internal: fix object not being handled properly`).
+the scope is not strictly defined, you can write whatever you find most fitting for the particular change. suppose you are changing a small part of a more significant part of the codebase. in that case, you can specify both the larger and smaller scopes in the commit message for clarity (e.g., if you were changing something in internal streams, the commit could be something like `api/stream: fix object not being handled properly`).
 
-if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/d2e5b6542f71f3809ba94d56c26f382b5cb62762)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
+if you think a change deserves further explanation, we encourage you to write a short explanation in the commit message ([example](https://github.com/imputnet/cobalt/commit/31be60484de8eaf63bba8a4f508e16438aa7ba6e)), which will save both you and us time having to enquire about the change, and you explaining the reason behind it.
 
 if your contribution has uninformative commit titles, you may be asked to interactively rebase your branch and amend each commit to include a meaningful title.
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20-bullseye-slim AS base
+FROM node:24-alpine AS base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 
@@ -7,8 +7,7 @@ WORKDIR /app
 COPY . /app
 
 RUN corepack enable
-RUN apt-get update && \
-    apt-get install -y python3 build-essential
+RUN apk add --no-cache python3 alpine-sdk
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
     pnpm install --prod --frozen-lockfile
@@ -18,8 +17,10 @@ RUN pnpm deploy --filter=@imput/cobalt-api --prod /prod/api
 FROM base AS api
 WORKDIR /app
 
-COPY --from=build /prod/api /app
-COPY --from=build /app/.git /app/.git
+COPY --from=build --chown=node:node /prod/api /app
+COPY --from=build --chown=node:node /app/.git /app/.git
+
+USER node
 
 EXPOSE 9000
-CMD [ "node", "src/cobalt" ]
+CMD [ "node", "src/cobalt" ]

README.md

@@ -14,109 +14,59 @@
         <a href="https://discord.gg/pQPt8HBUPu">
             💬 community discord server
         </a>
+        <br/>
         <a href="https://x.com/justusecobalt">
-            🐦 twitter/x
+            🐦 twitter
+        </a>
+        <a href="https://bsky.app/profile/cobalt.tools">
+            🦋 bluesky
         </a>
     </p>
     <br/>
 </div>
 
-cobalt is a media downloader that doesn't piss you off. it's fast, friendly, and doesn't have any bullshit that modern web is filled with: ***no ads, trackers, or paywalls***.
+cobalt is a media downloader that doesn't piss you off. it's friendly, efficient, and doesn't have ads, trackers, paywalls or other nonsense.
 
-paste the link, get the file, move on. it's that simple. just how it should be.
+paste the link, get the file, move on. that simple, just how it should be.
 
-### supported services
-this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀).
+### sponsors
+<div align="center" markdown="1">
+    <sup>special thanks to Warp for sponsoring the development of cobalt</sup>
+    <br>
+    <a href="https://go.warp.dev/cobalt">
+        <img alt="Warp banner" width="400" src="https://raw.githubusercontent.com/warpdotdev/brand-assets/be7d584f98e62b1579fd2e9338d4c7318a732f1b/Github/Sponsor/Warp-Github-LG-03.png">
+    </a>
 
-| service           | video + audio | only audio | only video | metadata | rich file names |
-| :--------         | :-----------: | :--------: | :--------: | :------: | :-------------: |
-| bilibili          | ✅            | ✅         | ✅         | ➖         | ➖              |
-| bluesky           | ✅            | ✅         | ✅         | ➖         | ➖              |
-| dailymotion       | ✅            | ✅         | ✅         | ✅         | ✅              |
-| instagram         | ✅            | ✅         | ✅         | ➖         | ➖              |
-| facebook          | ✅            | ❌         | ✅         | ➖         | ➖              |
-| loom              | ✅            | ❌         | ✅         | ✅         | ➖              |
-| ok.ru             | ✅            | ❌         | ✅         | ✅         | ✅              |
-| pinterest         | ✅            | ✅         | ✅         | ➖         | ➖              |
-| reddit            | ✅            | ✅         | ✅         | ❌         | ❌              |
-| rutube            | ✅            | ✅         | ✅         | ✅         | ✅              |
-| snapchat          | ✅            | ✅         | ✅         | ➖         | ➖              |
-| soundcloud        | ➖            | ✅         | ➖         | ✅         | ✅              |
-| streamable        | ✅            | ✅         | ✅         | ➖         | ➖              |
-| tiktok            | ✅            | ✅         | ✅         | ❌         | ❌              |
-| tumblr            | ✅            | ✅         | ✅         | ➖         | ➖              |
-| twitch clips      | ✅            | ✅         | ✅         | ✅         | ✅              |
-| twitter/x         | ✅            | ✅         | ✅         | ➖         | ➖              |
-| vimeo             | ✅            | ✅         | ✅         | ✅         | ✅              |
-| vine              | ✅            | ✅         | ✅         | ➖         | ➖              |
-| vk videos & clips | ✅            | ❌         | ✅         | ✅         | ✅              |
-| youtube           | ✅            | ✅         | ✅         | ✅         | ✅              |
+### [Warp, built for coding with multiple AI agents](https://go.warp.dev/cobalt)
+</div>
 
-| emoji   | meaning                 |
-| :-----: | :---------------------- |
-| ✅      | supported               |
-| ➖      | impossible/unreasonable |
-| ❌      | not supported           |
+#### RoyaleHosting
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), and a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
 
-### additional notes or features (per service)
-| service    | notes or features                                                                                                    |
-| :--------  | :-----                                                                                                               |
-| instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
-| facebook   | supports public accessible videos content only.                                                                      |
-| pinterest  | supports photos, gifs, videos and stories.                                                                           |
-| reddit     | supports gifs and videos.                                                                                            |
-| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
-| rutube     | supports yappy & private links.                                                                                      |
-| soundcloud | supports private links.                                                                                              |
-| tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
-| twitter/x  | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management.               |
-| vimeo      | audio downloads are only available for dash.                                                                         |
-| youtube    | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. |
+### cobalt monorepo
+this monorepo includes source code for api, frontend, and related packages:
+- [api tree & readme](/api/)
+- [web tree & readme](/web/)
+- [packages tree](/packages/)
 
-### partners
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), all main instances are currently hosted on their network :)
+it also includes documentation in the [docs tree](/docs/):
+- [how to run a cobalt instance](/docs/run-an-instance.md)
+- [how to protect a cobalt instance](/docs/protect-an-instance.md)
+- [cobalt api instance environment variables](/docs/api-env-variables.md)
+- [cobalt api documentation](/docs/api.md)
 
-### ethics and disclaimer
-cobalt is a tool for easing content downloads from internet and takes ***zero liability***. you are responsible for what you download, how you use and distribute that content. please be mindful when using content of others and always credit original creators. fair use and credits benefit everyone.
+### ethics
+cobalt is a tool that makes downloading public content easier. it takes **zero liability**.
+the end user is responsible for what they download, how they use and distribute that content.
+cobalt never caches any content, it [works like a fancy proxy](/api/src/stream/).
 
-cobalt is ***NOT*** a piracy tool and cannot be used as such. it can only download free, publicly accessible content. such content can be easily downloaded through any browser's dev tools. pressing one button is easier, so i made a convenient, ad-less tool for such repeated actions.
+cobalt is in no way a piracy tool and cannot be used as such.
+it can only download free & publicly accessible content.
+same content can be downloaded via dev tools of any modern web browser.
 
-### cobalt license
+### contributing
+if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
+
+### licenses
 for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
 unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).
-
-## acknowledgements
-### ffmpeg
-cobalt heavily relies on ffmpeg for converting and merging media files. it's an absolutely amazing piece of software offered for anyone for free, yet doesn't receive as much credit as it should.
-
-you can [support ffmpeg here](https://ffmpeg.org/donations.html)!
-
-#### ffmpeg-static
-we use [ffmpeg-static](https://github.com/eugeneware/ffmpeg-static) to get binaries for ffmpeg depending on the platform.
-
-you can support the developer via various methods listed on their github page! (linked above)
-
-### youtube.js
-cobalt relies on [youtube.js](https://github.com/LuanRT/YouTube.js) for interacting with the innertube api, it wouldn't have been possible without it.
-
-you can support the developer via various methods listed on their github page! (linked above)
-
-### many others
-cobalt also depends on:
-
-- [content-disposition-header](https://www.npmjs.com/package/content-disposition-header) to simplify the provision of `content-disposition` headers.
-- [cors](https://www.npmjs.com/package/cors) to manage cross-origin resource sharing within expressjs.
-- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables from the `.env` file.
-- [esbuild](https://www.npmjs.com/package/esbuild) to minify the frontend files.
-- [express](https://www.npmjs.com/package/express) as the backbone of cobalt servers.
-- [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) to rate limit api endpoints.
-- [hls-parser](https://www.npmjs.com/package/hls-parser) to parse `m3u8` playlists for certain services.
-- [ipaddr.js](https://www.npmjs.com/package/ipaddr.js) to parse ip addresses (for rate limiting).
-- [nanoid](https://www.npmjs.com/package/nanoid) to generate unique (temporary) identifiers for each requested stream.
-- [node-cache](https://www.npmjs.com/package/node-cache) to cache stream info in server ram for a limited amount of time.
-- [psl](https://www.npmjs.com/package/psl) as the domain name parser.
-- [set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser) to parse cookies that cobalt receives from certain services.
-- [undici](https://www.npmjs.com/package/undici) for making http requests.
-- [url-pattern](https://www.npmjs.com/package/url-pattern) to match provided links with supported patterns.
-
-...and many other packages that these packages rely on.

api/README.md

@@ -1,4 +1,65 @@
 # cobalt api
+this directory includes the source code for cobalt api. it's made with [express.js](https://www.npmjs.com/package/express) and love!
+
+## running your own instance
+if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
+we recommend to use docker compose unless you intend to run cobalt for developing/debugging purposes.
+
+## accessing the api
+there is currently no publicly available pre-hosted api.
+we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish to use the cobalt api.
+
+you can read [the api documentation here](/docs/api.md).
+
+## supported services
+this list is not final and keeps expanding over time!
+if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀).
+
+| service           | video + audio | only audio | only video | metadata | rich file names |
+| :--------         | :-----------: | :--------: | :--------: | :------: | :-------------: |
+| bilibili          | ✅            | ✅         | ✅         | ➖         | ➖              |
+| bluesky           | ✅            | ✅         | ✅         | ➖         | ➖              |
+| dailymotion       | ✅            | ✅         | ✅         | ✅         | ✅              |
+| instagram         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| facebook          | ✅            | ❌         | ✅         | ➖         | ➖              |
+| loom              | ✅            | ❌         | ✅         | ✅         | ➖              |
+| newgrounds        | ✅            | ✅         | ✅         | ✅         | ✅              |
+| ok.ru             | ✅            | ❌         | ✅         | ✅         | ✅              |
+| pinterest         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| reddit            | ✅            | ✅         | ✅         | ❌         | ❌              |
+| rutube            | ✅            | ✅         | ✅         | ✅         | ✅              |
+| snapchat          | ✅            | ✅         | ✅         | ➖         | ➖              |
+| soundcloud        | ➖            | ✅         | ➖         | ✅         | ✅              |
+| streamable        | ✅            | ✅         | ✅         | ➖         | ➖              |
+| tiktok            | ✅            | ✅         | ✅         | ❌         | ❌              |
+| tumblr            | ✅            | ✅         | ✅         | ➖         | ➖              |
+| twitch clips      | ✅            | ✅         | ✅         | ✅         | ✅              |
+| twitter/x         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| vimeo             | ✅            | ✅         | ✅         | ✅         | ✅              |
+| vk videos & clips | ✅            | ❌         | ✅         | ✅         | ✅              |
+| xiaohongshu       | ✅            | ✅         | ✅         | ➖         | ➖              |
+| youtube           | ✅            | ✅         | ✅         | ✅         | ✅              |
+
+| emoji   | meaning                 |
+| :-----: | :---------------------- |
+| ✅      | supported               |
+| ➖      | unreasonable/impossible |
+| ❌      | not supported           |
+
+### additional notes or features (per service)
+| service    | notes or features                                                                                                    |
+| :--------  | :-----                                                                                                               |
+| instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
+| facebook   | supports public accessible videos content only.                                                                      |
+| pinterest  | supports photos, gifs, videos and stories.                                                                           |
+| reddit     | supports gifs and videos.                                                                                            |
+| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
+| rutube     | supports yappy & private links.                                                                                      |
+| soundcloud | supports private links.                                                                                              |
+| tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
+| twitter/x  | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management.               |
+| vimeo      | audio downloads are only available for dash.                                                                         |
+| youtube    | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. |
 
 ## license
 cobalt api code is licensed under [AGPL-3.0](LICENSE).
@@ -9,14 +70,35 @@ as long as you:
 - provide a link to the license and indicate if changes to the code were made, and
 - release the code under the **same license**
 
-## running your own instance
-if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
-it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes.
+## open source acknowledgements
+### ffmpeg
+cobalt relies on ffmpeg for muxing and encoding media files. ffmpeg is absolutely spectacular and we're privileged to have the ability to use it for free, just like anyone else. we believe it should be way more recognized.
 
-## accessing the api
-currently, there is no publicly accessible main api. we plan on providing a public api for
-cobalt 10 in some form in the future. we recommend deploying your own instance if you wish
-to use the latest api. you can access [the documentation](/docs/api.md) for it here.
+you can [support ffmpeg here](https://ffmpeg.org/donations.html)!
 
-if you are looking for the documentation for the old (7.x) api, you can find
-it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md)
+### youtube.js
+cobalt relies on **[youtube.js](https://github.com/LuanRT/YouTube.js)** for interacting with youtube's innertube api, it wouldn't have been possible without this package.
+
+you can support the developer via various methods listed on their github page!
+(linked above)
+
+### many others
+cobalt-api also depends on:
+
+- **[content-disposition-header](https://www.npmjs.com/package/content-disposition-header)** to simplify the provision of `content-disposition` headers.
+- **[cors](https://www.npmjs.com/package/cors)** to manage cross-origin resource sharing within expressjs.
+- **[dotenv](https://www.npmjs.com/package/dotenv)** to load environment variables from the `.env` file.
+- **[express](https://www.npmjs.com/package/express)** as the backbone of cobalt servers.
+- **[express-rate-limit](https://www.npmjs.com/package/express-rate-limit)** to rate limit api endpoints.
+- **[ffmpeg-static](https://www.npmjs.com/package/ffmpeg-static)** to get binaries for ffmpeg depending on the platform.
+- **[hls-parser](https://www.npmjs.com/package/hls-parser)** to parse HLS playlists according to spec (very impressive stuff).
+- **[ipaddr.js](https://www.npmjs.com/package/ipaddr.js)** to parse ip addresses (used for rate limiting).
+- **[nanoid](https://www.npmjs.com/package/nanoid)** to generate unique identifiers for each requested tunnel.
+- **[set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser)** to parse cookies that cobalt receives from certain services.
+- **[undici](https://www.npmjs.com/package/undici)** for making http requests.
+- **[url-pattern](https://www.npmjs.com/package/url-pattern)** to match provided links with supported patterns.
+- **[zod](https://www.npmjs.com/package/zod)** to lock down the api request schema.
+- **[@datastructures-js/priority-queue](https://www.npmjs.com/package/@datastructures-js/priority-queue)** for sorting stream caches for future clean up (without redis).
+- **[@imput/psl](https://www.npmjs.com/package/@imput/psl)** as the domain name parser, our fork of [psl](https://www.npmjs.com/package/psl).
+
+...and many other packages that these packages rely on.

api/meow.js

@@ -0,0 +1 @@
+""

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "10.0.0",
+    "version": "11.5",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -10,9 +10,8 @@
     },
     "scripts": {
         "start": "node src/cobalt",
-        "setup": "node src/util/setup",
         "test": "node src/util/test",
-        "token:youtube": "node src/util/generate-youtube-tokens"
+        "token:jwt": "node src/util/generate-jwt-secret"
     },
     "repository": {
         "type": "git",
@@ -24,26 +23,28 @@
     },
     "homepage": "https://github.com/imputnet/cobalt#readme",
     "dependencies": {
+        "@datastructures-js/priority-queue": "^6.3.1",
+        "@imput/psl": "^2.0.4",
         "@imput/version-info": "workspace:^",
         "content-disposition-header": "0.6.0",
         "cors": "^2.8.5",
         "dotenv": "^16.0.1",
-        "esbuild": "^0.14.51",
-        "express": "^4.18.1",
-        "express-rate-limit": "^6.3.0",
+        "express": "^4.21.2",
+        "express-rate-limit": "^7.4.1",
         "ffmpeg-static": "^5.1.0",
         "hls-parser": "^0.10.7",
-        "ipaddr.js": "2.1.0",
-        "nanoid": "^4.0.2",
-        "node-cache": "^5.1.2",
-        "psl": "1.9.0",
+        "ipaddr.js": "2.2.0",
+        "mime": "^4.0.4",
+        "nanoid": "^5.0.9",
         "set-cookie-parser": "2.6.0",
-        "undici": "^5.19.1",
+        "undici": "^6.21.3",
         "url-pattern": "1.0.3",
-        "youtubei.js": "^10.3.0",
+        "youtubei.js": "15.1.1",
         "zod": "^3.23.8"
     },
     "optionalDependencies": {
-        "freebind": "^0.2.2"
+        "freebind": "^0.2.2",
+        "rate-limit-redis": "^4.2.0",
+        "redis": "^4.7.0"
     }
 }

api/src/cobalt.js

@@ -1,27 +1,37 @@
 import "dotenv/config";
 
 import express from "express";
+import cluster from "node:cluster";
 
-import path from 'path';
-import { fileURLToPath } from 'url';
+import path from "path";
+import { fileURLToPath } from "url";
 
-import { env } from "./config.js"
-import { Bright, Green, Red } from "./misc/console-text.js";
+import { env, isCluster } from "./config.js"
+import { Red } from "./misc/console-text.js";
+import { initCluster } from "./misc/cluster.js";
+import { setupEnvWatcher } from "./core/env.js";
 
 const app = express();
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename).slice(0, -4);
 
-app.disable('x-powered-by');
+app.disable("x-powered-by");
 
 if (env.apiURL) {
-    const { runAPI } = await import('./core/api.js');
-    runAPI(express, app, __dirname)
+    const { runAPI } = await import("./core/api.js");
+
+    if (isCluster) {
+       await initCluster();
+    }
+
+    if (env.envFile) {
+        setupEnvWatcher();
+    }
+
+    runAPI(express, app, __dirname, cluster.isPrimary);
 } else {
     console.log(
-        Red(`cobalt wasn't configured yet or configuration is invalid.\n`)
-        + Bright(`please run the setup script to fix this: `)
-        + Green(`npm run setup`)
+        Red("API_URL env variable is missing, cobalt api can't start.")
     )
 }

api/src/config.js

@@ -1,51 +1,41 @@
 import { getVersion } from "@imput/version-info";
-import { services } from "./processing/service-config.js";
+import { loadEnvs, validateEnvs } from "./core/env.js";
 
 const version = await getVersion();
 
-const disabledServices = process.env.DISABLED_SERVICES?.split(',') || [];
-const enabledServices = new Set(Object.keys(services).filter(e => {
-    if (!disabledServices.includes(e)) {
-        return e;
+const canonicalEnv = Object.freeze(structuredClone(process.env));
+const env = loadEnvs();
+
+const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
+const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
+
+export const setTunnelPort = (port) => env.tunnelPort = port;
+export const isCluster = env.instanceCount > 1;
+export const updateEnv = (newEnv) => {
+    const changes = [];
+
+    // tunnelPort is special and needs to get carried over here
+    newEnv.tunnelPort = env.tunnelPort;
+
+    for (const key in env) {
+        if (key === 'subscribe') {
+            continue;
+        }
+
+        if (String(env[key]) !== String(newEnv[key])) {
+            changes.push(key);
+        }
+        env[key] = newEnv[key];
     }
-}));
 
-const env = {
-    apiURL: process.env.API_URL || '',
-    apiPort: process.env.API_PORT || 9000,
-
-    listenAddress: process.env.API_LISTEN_ADDRESS,
-    freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
-
-    corsWildcard: process.env.CORS_WILDCARD !== '0',
-    corsURL: process.env.CORS_URL,
-
-    cookiePath: process.env.COOKIE_PATH,
-
-    rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
-    rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
-
-    durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
-    streamLifespan: 90,
-
-    processingPriority: process.platform !== 'win32'
-        && process.env.PROCESSING_PRIORITY
-        && parseInt(process.env.PROCESSING_PRIORITY),
-
-    externalProxy: process.env.API_EXTERNAL_PROXY,
-
-    turnstileSecret: process.env.TURNSTILE_SECRET,
-    jwtSecret: process.env.JWT_SECRET,
-    jwtLifetime: process.env.JWT_EXPIRY || 120,
-
-    enabledServices,
+    return changes;
 }
 
-const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
-const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
+await validateEnvs(env);
 
 export {
     env,
+    canonicalEnv,
     genericUserAgent,
     cobaltUserAgent,
 }

api/src/core/api.js

@@ -1,6 +1,7 @@
 import cors from "cors";
+import http from "node:http";
 import rateLimit from "express-rate-limit";
-import { setGlobalDispatcher, ProxyAgent } from "undici";
+import { setGlobalDispatcher, EnvHttpProxyAgent } from "undici";
 import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
 
 import jwt from "../security/jwt.js";
@@ -9,14 +10,19 @@ import match from "../processing/match.js";
 
 import { env } from "../config.js";
 import { extract } from "../processing/url.js";
-import { languageCode } from "../misc/utils.js";
 import { Bright, Cyan } from "../misc/console-text.js";
-import { generateHmac, generateSalt } from "../misc/crypto.js";
+import { hashHmac } from "../security/secrets.js";
+import { createStore } from "../store/redis-ratelimit.js";
 import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 import { verifyTurnstileToken } from "../security/turnstile.js";
 import { friendlyServiceName } from "../processing/service-alias.js";
-import { verifyStream, getInternalStream } from "../stream/manage.js";
+import { verifyStream } from "../stream/manage.js";
 import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
+import { setupTunnelHandler } from "./itunnel.js";
+
+import * as APIKeys from "../security/api-keys.js";
+import * as Cookies from "../processing/cookie/manager.js";
+import * as YouTubeSession from "../processing/helpers/youtube-session.js";
 
 const git = {
     branch: await getBranch(),
@@ -28,7 +34,6 @@ const version = await getVersion();
 
 const acceptRegex = /^application\/json(; charset=utf-8)?$/;
 
-const ipSalt = generateSalt();
 const corsConfig = env.corsWildcard ? {} : {
     origin: env.corsURL,
     optionsSuccessStatus: 200
@@ -39,55 +44,70 @@ const fail = (res, code, context) => {
     res.status(status).json(body);
 }
 
-export const runAPI = (express, app, __dirname) => {
+export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     const startTime = new Date();
     const startTimestamp = startTime.getTime();
 
-    const serverInfo = JSON.stringify({
-        cobalt: {
-            version: version,
-            url: env.apiURL,
-            startTime: `${startTimestamp}`,
-            durationLimit: env.durationLimit,
-            services: [...env.enabledServices].map(e => {
-                return friendlyServiceName(e);
-            }),
-        },
-        git,
-    })
+    const getServerInfo = () => {
+        return JSON.stringify({
+            cobalt: {
+                version: version,
+                url: env.apiURL,
+                startTime: `${startTimestamp}`,
+                turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
+                services: [...env.enabledServices].map(e => {
+                    return friendlyServiceName(e);
+                }),
+            },
+            git,
+        });
+    }
+
+    const serverInfo = getServerInfo();
+
+    const handleRateExceeded = (_, res) => {
+        const { body } = createResponse("error", {
+            code: "error.api.rate_exceeded",
+            context: {
+                limit: env.rateLimitWindow
+            }
+        });
+        return res.status(429).json(body);
+    };
+
+    const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url');
+
+    const sessionLimiter = rateLimit({
+        windowMs: env.sessionRateLimitWindow * 1000,
+        limit: env.sessionRateLimit,
+        standardHeaders: 'draft-6',
+        legacyHeaders: false,
+        keyGenerator,
+        store: await createStore('session'),
+        handler: handleRateExceeded
+    });
 
     const apiLimiter = rateLimit({
         windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
-        standardHeaders: true,
+        limit: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: 'draft-6',
         legacyHeaders: false,
-        keyGenerator: req => {
-            if (req.authorized) {
-                return generateHmac(req.header("Authorization"), ipSalt);
-            }
-            return generateHmac(getIP(req), ipSalt);
-        },
-        handler: (req, res) => {
-            const { status, body } = createResponse("error", {
-                code: "error.api.rate_exceeded",
-                context: {
-                    limit: env.rateLimitWindow
-                }
-            });
-            return res.status(status).json(body);
-        }
-    })
+        keyGenerator: req => req.rateLimitKey || keyGenerator(req),
+        store: await createStore('api'),
+        handler: handleRateExceeded
+    });
 
-    const apiLimiterStream = rateLimit({
-        windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
-        standardHeaders: true,
+    const apiTunnelLimiter = rateLimit({
+        windowMs: env.tunnelRateLimitWindow * 1000,
+        limit: env.tunnelRateLimitMax,
+        standardHeaders: 'draft-6',
         legacyHeaders: false,
-        keyGenerator: req => generateHmac(getIP(req), ipSalt),
-        handler: (req, res) => {
-            return res.sendStatus(429)
+        keyGenerator: req => keyGenerator(req),
+        store: await createStore('tunnel'),
+        handler: (_, res) => {
+            return res.sendStatus(429);
         }
-    })
+    });
 
     app.set('trust proxy', ['loopback', 'uniquelocal']);
 
@@ -102,11 +122,46 @@ export const runAPI = (express, app, __dirname) => {
         ...corsConfig,
     }));
 
-    app.post('/', apiLimiter);
-    app.use('/tunnel', apiLimiterStream);
+    app.post('/', (req, res, next) => {
+        if (!acceptRegex.test(req.header('Accept'))) {
+            return fail(res, "error.api.header.accept");
+        }
+        if (!acceptRegex.test(req.header('Content-Type'))) {
+            return fail(res, "error.api.header.content_type");
+        }
+        next();
+    });
 
     app.post('/', (req, res, next) => {
-        if (!env.turnstileSecret || !env.jwtSecret) {
+        if (!env.apiKeyURL) {
+            return next();
+        }
+
+        const { success, error } = APIKeys.validateAuthorization(req);
+        if (!success) {
+            // We call next() here if either if:
+            // a) we have user sessions enabled, meaning the request
+            //    will still need a Bearer token to not be rejected, or
+            // b) we do not require the user to be authenticated, and
+            //    so they can just make the request with the regular
+            //    rate limit configuration;
+            // otherwise, we reject the request.
+            if (
+                (env.sessionEnabled || !env.authRequired)
+                && ['missing', 'not_api_key'].includes(error)
+            ) {
+                return next();
+            }
+
+            return fail(res, `error.api.auth.key.${error}`);
+        }
+
+        req.authType = "key";
+        return next();
+    });
+
+    app.post('/', (req, res, next) => {
+        if (!env.sessionEnabled || req.rateLimitKey) {
             return next();
         }
 
@@ -116,34 +171,30 @@ export const runAPI = (express, app, __dirname) => {
                 return fail(res, "error.api.auth.jwt.missing");
             }
 
-            if (!authorization.startsWith("Bearer ") || authorization.length > 256) {
+            if (authorization.length >= 256) {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            const verifyJwt = jwt.verify(
-                authorization.split("Bearer ", 2)[1]
-            );
-
-            if (!verifyJwt) {
+            const [ type, token, ...rest ] = authorization.split(" ");
+            if (!token || type.toLowerCase() !== 'bearer' || rest.length) {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            if (!acceptRegex.test(req.header('Accept'))) {
-                return fail(res, "error.api.header.accept");
+            if (!jwt.verify(token, getIP(req, 32))) {
+                return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            if (!acceptRegex.test(req.header('Content-Type'))) {
-                return fail(res, "error.api.header.content_type");
-            }
-
-            req.authorized = true;
+            req.rateLimitKey = hashHmac(token, 'rate');
+            req.authType = "session";
         } catch {
             return fail(res, "error.api.generic");
         }
         next();
     });
 
+    app.post('/', apiLimiter);
     app.use('/', express.json({ limit: 1024 }));
+
     app.use('/', (err, _, res, next) => {
         if (err) {
             const { status, body } = createResponse("error", {
@@ -155,8 +206,8 @@ export const runAPI = (express, app, __dirname) => {
         next();
     });
 
-    app.post("/session", async (req, res) => {
-        if (!env.turnstileSecret || !env.jwtSecret) {
+    app.post("/session", sessionLimiter, async (req, res) => {
+        if (!env.sessionEnabled) {
             return fail(res, "error.api.auth.not_configured")
         }
 
@@ -176,7 +227,7 @@ export const runAPI = (express, app, __dirname) => {
         }
 
         try {
-            res.json(jwt.generate());
+            res.json(jwt.generate(getIP(req, 32)));
         } catch {
             return fail(res, "error.api.generic");
         }
@@ -184,26 +235,25 @@ export const runAPI = (express, app, __dirname) => {
 
     app.post('/', async (req, res) => {
         const request = req.body;
-        const lang = languageCode(req);
 
         if (!request.url) {
             return fail(res, "error.api.link.missing");
         }
 
-        if (request.youtubeDubBrowserLang) {
-            request.youtubeDubLang = lang;
-        }
-
         const { success, data: normalizedRequest } = await normalizeRequest(request);
         if (!success) {
             return fail(res, "error.api.invalid_body");
         }
 
-        const parsed = extract(normalizedRequest.url);
+        const parsed = extract(
+            normalizedRequest.url,
+            APIKeys.getAllowedServices(req.rateLimitKey),
+        );
 
         if (!parsed) {
             return fail(res, "error.api.link.invalid");
         }
+
         if ("error" in parsed) {
             let context;
             if (parsed?.context) {
@@ -217,15 +267,25 @@ export const runAPI = (express, app, __dirname) => {
                 host: parsed.host,
                 patternMatch: parsed.patternMatch,
                 params: normalizedRequest,
+                authType: req.authType ?? "none",
             });
 
             res.status(result.status).json(result.body);
         } catch {
             fail(res, "error.api.generic");
         }
-    })
+    });
 
-    app.get('/tunnel', (req, res) => {
+    app.use('/tunnel', cors({
+        methods: ['GET'],
+        exposedHeaders: [
+            'Estimated-Content-Length',
+            'Content-Disposition'
+        ],
+        ...corsConfig,
+    }));
+
+    app.get('/tunnel', apiTunnelLimiter, async (req, res) => {
         const id = String(req.query.id);
         const exp = String(req.query.exp);
         const sig = String(req.query.sig);
@@ -244,7 +304,7 @@ export const runAPI = (express, app, __dirname) => {
             return res.status(200).end();
         }
 
-        const streamInfo = verifyStream(id, sig, exp, sec, iv);
+        const streamInfo = await verifyStream(id, sig, exp, sec, iv);
         if (!streamInfo?.service) {
             return res.status(streamInfo.status).end();
         }
@@ -254,33 +314,11 @@ export const runAPI = (express, app, __dirname) => {
         }
 
         return stream(res, streamInfo);
-    })
-
-    app.get('/itunnel', (req, res) => {
-        if (!req.ip.endsWith('127.0.0.1')) {
-            return res.sendStatus(403);
-        }
-
-        if (String(req.query.id).length !== 21) {
-            return res.sendStatus(400);
-        }
-
-        const streamInfo = getInternalStream(req.query.id);
-        if (!streamInfo) {
-            return res.sendStatus(404);
-        }
-
-        streamInfo.headers = new Map([
-            ...(streamInfo.headers || []),
-            ...Object.entries(req.headers)
-        ]);
-
-        return stream(res, { type: 'internal', ...streamInfo });
-    })
+    });
 
     app.get('/', (_, res) => {
         res.type('json');
-        res.status(200).send(serverInfo);
+        res.status(200).send(env.envFile ? getServerInfo() : serverInfo);
     })
 
     app.get('/favicon.ico', (req, res) => {
@@ -299,28 +337,52 @@ export const runAPI = (express, app, __dirname) => {
     randomizeCiphers();
     setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
 
-    if (env.externalProxy) {
-        if (env.freebindCIDR) {
-            throw new Error('Freebind is not available when external proxy is enabled')
+    env.subscribe(['externalProxy', 'httpProxyValues'], () => {
+        // TODO: remove env.externalProxy in a future version
+        const options = {};
+        if (env.externalProxy) {
+            options.httpProxy = env.externalProxy;
         }
 
-        setGlobalDispatcher(new ProxyAgent(env.externalProxy))
-    }
+        setGlobalDispatcher(
+            new EnvHttpProxyAgent(options)
+        );
+    });
 
-    app.listen(env.apiPort, env.listenAddress, () => {
-        console.log(`\n` +
-            Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" +
+    http.createServer(app).listen({
+        port: env.apiPort,
+        host: env.listenAddress,
+        reusePort: env.instanceCount > 1 || undefined
+    }, () => {
+        if (isPrimary) {
+            console.log(`\n` +
+                Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" +
 
-            "~~~~~~\n" +
-            Bright("version: ") + version + "\n" +
-            Bright("commit: ") + git.commit + "\n" +
-            Bright("branch: ") + git.branch + "\n" +
-            Bright("remote: ") + git.remote + "\n" +
-            Bright("start time: ") + startTime.toUTCString() + "\n" +
-            "~~~~~~\n" +
+                "~~~~~~\n" +
+                Bright("version: ") + version + "\n" +
+                Bright("commit: ") + git.commit + "\n" +
+                Bright("branch: ") + git.branch + "\n" +
+                Bright("remote: ") + git.remote + "\n" +
+                Bright("start time: ") + startTime.toUTCString() + "\n" +
+                "~~~~~~\n" +
 
-            Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
-            Bright("port: ") + env.apiPort + "\n"
-        )
-    })
+                Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
+                Bright("port: ") + env.apiPort + "\n"
+            );
+        }
+
+        if (env.apiKeyURL) {
+            APIKeys.setup(env.apiKeyURL);
+        }
+
+        if (env.cookiePath) {
+            Cookies.setup(env.cookiePath);
+        }
+
+        if (env.ytSessionServer) {
+            YouTubeSession.setup();
+        }
+    });
+
+    setupTunnelHandler();
 }

api/src/core/env.js

@@ -0,0 +1,289 @@
+import { Constants } from "youtubei.js";
+import { services } from "../processing/service-config.js";
+import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";
+
+import { FileWatcher } from "../misc/file-watcher.js";
+import { isURL } from "../misc/utils.js";
+import * as cluster from "../misc/cluster.js";
+import { Green, Yellow } from "../misc/console-text.js";
+
+const forceLocalProcessingOptions = ["never", "session", "always"];
+const youtubeHlsOptions = ["never", "key", "always"];
+
+const httpProxyVariables = ["NO_PROXY", "HTTP_PROXY", "HTTPS_PROXY"].flatMap(
+    k => [ k, k.toLowerCase() ]
+);
+
+const changeCallbacks = {};
+
+const onEnvChanged = (changes) => {
+    for (const key of changes) {
+        if (changeCallbacks[key]) {
+            changeCallbacks[key].map(fn => {
+                try { fn() } catch {}
+            });
+        }
+    }
+}
+
+const subscribe = (keys, fn) => {
+    keys = [keys].flat();
+
+    for (const key of keys) {
+        if (key in currentEnv && key !== 'subscribe') {
+            changeCallbacks[key] ??= [];
+            changeCallbacks[key].push(fn);
+            fn();
+        } else throw `invalid env key ${key}`;
+    }
+}
+
+export const loadEnvs = (env = process.env) => {
+    const allServices = new Set(Object.keys(services));
+    const disabledServices = env.DISABLED_SERVICES?.split(',') || [];
+    const enabledServices = new Set(Object.keys(services).filter(e => {
+        if (!disabledServices.includes(e)) {
+            return e;
+        }
+    }));
+
+    // we need to copy the proxy envs (HTTP_PROXY, HTTPS_PROXY)
+    // back into process.env, so that EnvHttpProxyAgent can pick
+    // them up later
+    for (const key of httpProxyVariables) {
+        const value = env[key] ?? canonicalEnv[key];
+        if (value !== undefined) {
+            process.env[key] = env[key];
+        } else {
+            delete process.env[key];
+        }
+    }
+
+    return {
+        apiURL: env.API_URL || '',
+        apiPort: env.API_PORT || 9000,
+        tunnelPort: env.API_PORT || 9000,
+
+        listenAddress: env.API_LISTEN_ADDRESS,
+        freebindCIDR: process.platform === 'linux' && env.FREEBIND_CIDR,
+
+        corsWildcard: env.CORS_WILDCARD !== '0',
+        corsURL: env.CORS_URL,
+
+        cookiePath: env.COOKIE_PATH,
+
+        rateLimitWindow: (env.RATELIMIT_WINDOW && parseInt(env.RATELIMIT_WINDOW)) || 60,
+        rateLimitMax: (env.RATELIMIT_MAX && parseInt(env.RATELIMIT_MAX)) || 20,
+
+        tunnelRateLimitWindow: (env.TUNNEL_RATELIMIT_WINDOW && parseInt(env.TUNNEL_RATELIMIT_WINDOW)) || 60,
+        tunnelRateLimitMax: (env.TUNNEL_RATELIMIT_MAX && parseInt(env.TUNNEL_RATELIMIT_MAX)) || 40,
+
+        sessionRateLimitWindow: (env.SESSION_RATELIMIT_WINDOW && parseInt(env.SESSION_RATELIMIT_WINDOW)) || 60,
+        sessionRateLimit:
+            // backwards compatibility with SESSION_RATELIMIT
+            // till next major due to an error in docs
+            (env.SESSION_RATELIMIT_MAX && parseInt(env.SESSION_RATELIMIT_MAX))
+            || (env.SESSION_RATELIMIT && parseInt(env.SESSION_RATELIMIT))
+            || 10,
+
+        durationLimit: (env.DURATION_LIMIT && parseInt(env.DURATION_LIMIT)) || 10800,
+        streamLifespan: (env.TUNNEL_LIFESPAN && parseInt(env.TUNNEL_LIFESPAN)) || 90,
+
+        processingPriority: process.platform !== 'win32'
+            && env.PROCESSING_PRIORITY
+            && parseInt(env.PROCESSING_PRIORITY),
+
+        externalProxy: env.API_EXTERNAL_PROXY,
+
+        // used only for comparing against old values when envs are being updated
+        httpProxyValues: httpProxyVariables.map(k => String(env[k])).join(''),
+
+        turnstileSitekey: env.TURNSTILE_SITEKEY,
+        turnstileSecret: env.TURNSTILE_SECRET,
+        jwtSecret: env.JWT_SECRET,
+        jwtLifetime: env.JWT_EXPIRY || 120,
+
+        sessionEnabled: env.TURNSTILE_SITEKEY
+                            && env.TURNSTILE_SECRET
+                            && env.JWT_SECRET,
+
+        apiKeyURL: env.API_KEY_URL && new URL(env.API_KEY_URL),
+        authRequired: env.API_AUTH_REQUIRED === '1',
+        redisURL: env.API_REDIS_URL,
+        instanceCount: (env.API_INSTANCE_COUNT && parseInt(env.API_INSTANCE_COUNT)) || 1,
+        keyReloadInterval: 900,
+
+        allServices,
+        enabledServices,
+
+        customInnertubeClient: env.CUSTOM_INNERTUBE_CLIENT,
+        ytSessionServer: env.YOUTUBE_SESSION_SERVER,
+        ytSessionReloadInterval: 300,
+        ytSessionInnertubeClient: env.YOUTUBE_SESSION_INNERTUBE_CLIENT,
+        ytAllowBetterAudio: env.YOUTUBE_ALLOW_BETTER_AUDIO !== "0",
+
+        // "never" | "session" | "always"
+        forceLocalProcessing: env.FORCE_LOCAL_PROCESSING ?? "never",
+
+        // "never" | "key" | "always"
+        enableDeprecatedYoutubeHls: env.ENABLE_DEPRECATED_YOUTUBE_HLS ?? "never",
+
+        envFile: env.API_ENV_FILE,
+        envRemoteReloadInterval: 300,
+
+        subscribe,
+    };
+}
+
+let loggedProxyWarning = false;
+
+export const validateEnvs = async (env) => {
+    if (env.sessionEnabled && env.jwtSecret.length < 16) {
+        throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)");
+    }
+
+    if (env.instanceCount > 1 && !env.redisURL) {
+        throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
+    } else if (env.instanceCount > 1 && !await cluster.supportsReusePort()) {
+        console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
+        console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
+        console.error('(or other OS that supports it). for more info, see `reusePort` option on');
+        console.error('https://nodejs.org/api/net.html#serverlistenoptions-callback');
+        throw new Error('SO_REUSEPORT is not supported');
+    }
+
+    if (env.customInnertubeClient && !Constants.SUPPORTED_CLIENTS.includes(env.customInnertubeClient)) {
+        console.error("CUSTOM_INNERTUBE_CLIENT is invalid. Provided client is not supported.");
+        console.error(`Supported clients are: ${Constants.SUPPORTED_CLIENTS.join(', ')}\n`);
+        throw new Error("Invalid CUSTOM_INNERTUBE_CLIENT");
+    }
+
+    if (env.forceLocalProcessing && !forceLocalProcessingOptions.includes(env.forceLocalProcessing)) {
+        console.error("FORCE_LOCAL_PROCESSING is invalid.");
+        console.error(`Supported options are are: ${forceLocalProcessingOptions.join(', ')}\n`);
+        throw new Error("Invalid FORCE_LOCAL_PROCESSING");
+    }
+
+    if (env.enableDeprecatedYoutubeHls && !youtubeHlsOptions.includes(env.enableDeprecatedYoutubeHls)) {
+        console.error("ENABLE_DEPRECATED_YOUTUBE_HLS is invalid.");
+        console.error(`Supported options are are: ${youtubeHlsOptions.join(', ')}\n`);
+        throw new Error("Invalid ENABLE_DEPRECATED_YOUTUBE_HLS");
+    }
+
+    if (env.externalProxy && env.freebindCIDR) {
+        throw new Error('freebind is not available when external proxy is enabled')
+    }
+
+    if (env.externalProxy && !loggedProxyWarning) {
+        console.error('API_EXTERNAL_PROXY is deprecated and will be removed in a future release.');
+        console.error('Use HTTP_PROXY or HTTPS_PROXY instead.');
+        console.error('You can read more about the new proxy variables in docs/api-env-variables.md\n');
+
+        // prevent the warning from being printed on every env validation
+        loggedProxyWarning = true;
+    }
+
+    return env;
+}
+
+const reloadEnvs = async (contents) => {
+    const newEnvs = {};
+    const resolvedContents = await contents;
+
+    for (let line of resolvedContents.split('\n')) {
+        line = line.trim();
+        if (line === '') {
+            continue;
+        }
+
+        let [ key, value ] = line.split(/=(.+)?/);
+        if (key) {
+            if (value.match(/^['"]/) && value.match(/['"]$/)) {
+                value = JSON.parse(value);
+            }
+
+            newEnvs[key] = value || '';
+        }
+    }
+
+    const candidate = {
+        ...canonicalEnv,
+        ...newEnvs,
+    };
+
+    const parsed = await validateEnvs(
+        loadEnvs(candidate)
+    );
+
+    cluster.broadcast({ env_update: resolvedContents });
+    return updateEnv(parsed);
+}
+
+const wrapReload = (contents) => {
+    reloadEnvs(contents)
+    .then(changes => {
+        if (changes.length === 0) {
+            return;
+        }
+
+        onEnvChanged(changes);
+
+        console.log(`${Green('[✓]')} envs reloaded successfully!`);
+        for (const key of changes) {
+            const value = currentEnv[key];
+            const isSecret = key.toLowerCase().includes('apikey')
+                          || key.toLowerCase().includes('secret')
+                          || key === 'httpProxyValues';
+
+            if (!value) {
+                console.log(`    removed: ${key}`);
+            } else {
+                console.log(`    changed: ${key} -> ${isSecret ? '***' : value}`);
+            }
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    });
+}
+
+let watcher;
+const setupWatcherFromFile = (path) => {
+    const load = () => wrapReload(watcher.read());
+
+    if (isURL(path)) {
+        watcher = FileWatcher.fromFileProtocol(path);
+    } else {
+        watcher = new FileWatcher({ path });
+    }
+
+    watcher.on('file-updated', load);
+    load();
+}
+
+const setupWatcherFromFetch = (url) => {
+    const load = () => wrapReload(fetch(url).then(r => r.text()));
+    setInterval(load, currentEnv.envRemoteReloadInterval);
+    load();
+}
+
+export const setupEnvWatcher = () => {
+    if (cluster.isPrimary) {
+        const envFile = currentEnv.envFile;
+        const isFile = !isURL(envFile)
+                       || new URL(envFile).protocol === 'file:';
+
+        if (isFile) {
+            setupWatcherFromFile(envFile);
+        } else {
+            setupWatcherFromFetch(envFile);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('env_update' in message) {
+                reloadEnvs(message.env_update);
+            }
+        });
+    }
+}

api/src/core/itunnel.js

@@ -0,0 +1,61 @@
+import stream from "../stream/stream.js";
+import { getInternalTunnel } from "../stream/manage.js";
+import { setTunnelPort } from "../config.js";
+import { Green } from "../misc/console-text.js";
+import express from "express";
+
+const validateTunnel = (req, res) => {
+    if (!req.ip.endsWith('127.0.0.1')) {
+        res.sendStatus(403);
+        return;
+    }
+
+    if (String(req.query.id).length !== 21) {
+        res.sendStatus(400);
+        return;
+    }
+
+    const streamInfo = getInternalTunnel(req.query.id);
+    if (!streamInfo) {
+        res.sendStatus(404);
+        return;
+    }
+
+    return streamInfo;
+}
+
+const streamTunnel = (req, res) => {
+    const streamInfo = validateTunnel(req, res);
+    if (!streamInfo) {
+        return;
+    }
+
+    streamInfo.headers = new Map([
+        ...(streamInfo.headers || []),
+        ...Object.entries(req.headers)
+    ]);
+
+    return stream(res, { type: 'internal', data: streamInfo });
+}
+
+export const setupTunnelHandler = () => {
+    const tunnelHandler = express();
+
+    tunnelHandler.get('/itunnel', streamTunnel);
+
+    // fallback
+    tunnelHandler.use((_, res) => res.sendStatus(400));
+    // error handler
+    tunnelHandler.use((_, __, res, ____) => res.socket.end());
+
+
+    const server = tunnelHandler.listen({
+        port: 0,
+        host: '127.0.0.1',
+        exclusive: true
+    }, () => {
+        const { port } = server.address();
+        console.log(`${Green('[✓]')} internal tunnel handler running on 127.0.0.1:${port}`);
+        setTunnelPort(port);
+    });
+}

api/src/misc/cluster.js

@@ -0,0 +1,72 @@
+import cluster from "node:cluster";
+import net from "node:net";
+import { syncSecrets } from "../security/secrets.js";
+import { env, isCluster } from "../config.js";
+
+export { isPrimary, isWorker } from "node:cluster";
+
+export const supportsReusePort = async () => {
+    try {
+        await new Promise((resolve, reject) => {
+            const server = net.createServer().listen({ port: 0, reusePort: true });
+            server.on('listening', () => server.close(resolve));
+            server.on('error', (err) => (server.close(), reject(err)));
+        });
+
+        const [major, minor] = process.versions.node.split('.').map(Number);
+        return major > 23 || (major === 23 && minor >= 1);
+    } catch {
+        return false;
+    }
+}
+
+export const initCluster = async () => {
+    if (cluster.isPrimary) {
+        for (let i = 1; i < env.instanceCount; ++i) {
+            cluster.fork();
+        }
+    }
+
+    await syncSecrets();
+}
+
+export const broadcast = (message) => {
+    if (!isCluster || !cluster.isPrimary || !cluster.workers) {
+        return;
+    }
+
+    for (const worker of Object.values(cluster.workers)) {
+        worker.send(message);
+    }
+}
+
+export const send = (message) => {
+    if (!isCluster) {
+        return;
+    }
+
+    if (cluster.isPrimary) {
+        return broadcast(message);
+    } else {
+        return process.send(message);
+    }
+}
+
+export const waitFor = (key) => {
+    return new Promise(resolve => {
+        const listener = (message) => {
+            if (key in message) {
+                process.off('message', listener);
+                return resolve(message);
+            }
+        }
+
+        process.on('message', listener);
+    });
+}
+
+export const mainOnMessage = (cb) => {
+    for (const worker of Object.values(cluster.workers)) {
+        worker.on('message', cb);
+    }
+}

api/src/misc/console-text.js

@@ -1,16 +1,36 @@
-function t(color, tt) {
-    return color + tt + "\x1b[0m"
+const ANSI = {
+    RESET: "\x1b[0m",
+    BRIGHT: "\x1b[1m",
+    RED: "\x1b[31m",
+    GREEN: "\x1b[32m",
+    CYAN: "\x1b[36m",
+    YELLOW: "\x1b[93m"
 }
 
-export function Bright(tt) {
-    return t("\x1b[1m", tt)
+function wrap(color, text) {
+    if (!ANSI[color.toUpperCase()]) {
+        throw "invalid color";
+    }
+
+    return ANSI[color.toUpperCase()] + text + ANSI.RESET;
 }
-export function Red(tt) {
-    return t("\x1b[31m", tt)
+
+export function Bright(text) {
+    return wrap('bright', text);
 }
-export function Green(tt) {
-    return t("\x1b[32m", tt)
+
+export function Red(text) {
+    return wrap('red', text);
 }
-export function Cyan(tt) {
-    return t("\x1b[36m", tt)
+
+export function Green(text) {
+    return wrap('green', text);
+}
+
+export function Cyan(text) {
+    return wrap('cyan', text);
+}
+
+export function Yellow(text) {
+    return wrap('yellow', text);
 }

api/src/misc/crypto.js

@@ -1,15 +1,7 @@
-import { createHmac, createCipheriv, createDecipheriv, randomBytes } from "crypto";
+import { createCipheriv, createDecipheriv } from "crypto";
 
 const algorithm = "aes256";
 
-export function generateSalt() {
-    return randomBytes(64).toString('hex');
-}
-
-export function generateHmac(str, salt) {
-    return createHmac("sha256", salt).update(str).digest("base64url");
-}
-
 export function encryptStream(plaintext, iv, secret) {
     const buff = Buffer.from(JSON.stringify(plaintext));
     const key = Buffer.from(secret, "base64url");

api/src/misc/file-watcher.js

@@ -0,0 +1,43 @@
+import { EventEmitter } from 'node:events';
+import * as fs from 'node:fs/promises';
+
+export class FileWatcher extends EventEmitter {
+    #path;
+    #hasWatcher = false;
+    #lastChange = new Date().getTime();
+
+    constructor({ path, ...rest }) {
+        super(rest);
+        this.#path = path;
+    }
+
+    async #setupWatcher() {
+        if (this.#hasWatcher)
+            return;
+
+        this.#hasWatcher = true;
+        const watcher = fs.watch(this.#path);
+        for await (const _ of watcher) {
+            if (new Date() - this.#lastChange > 50) {
+                this.emit('file-updated');
+                this.#lastChange = new Date().getTime();
+            }
+        }
+    }
+
+    read() {
+        this.#setupWatcher();
+        return fs.readFile(this.#path, 'utf8');
+    }
+
+    static fromFileProtocol(url_) {
+        const url = new URL(url_);
+        if (url.protocol !== 'file:') {
+            return;
+        }
+
+        const pathname = url.pathname === '/' ? '' : url.pathname;
+        const file_path = decodeURIComponent(url.host + pathname);
+        return new this({ path: file_path });
+    }
+}

api/src/misc/language-codes.js

@@ -0,0 +1,54 @@
+// converted from this file https://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
+const iso639_1to2 = {
+    'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'sqi',
+    'am': 'amh', 'ar': 'ara', 'an': 'arg', 'hy': 'hye', 'as': 'asm',
+    'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze', 'ba': 'bak',
+    'bm': 'bam', 'eu': 'eus', 'be': 'bel', 'bn': 'ben', 'bi': 'bis',
+    'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'mya', 'ca': 'cat',
+    'ch': 'cha', 'ce': 'che', 'zh': 'zho', 'cu': 'chu', 'cv': 'chv',
+    'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'ces', 'da': 'dan',
+    'dv': 'div', 'nl': 'nld', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo',
+    'et': 'est', 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin',
+    'fr': 'fra', 'fy': 'fry', 'ff': 'ful', 'ka': 'kat', 'de': 'deu',
+    'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
+    'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb',
+    'hz': 'her', 'hi': 'hin', 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun',
+    'ig': 'ibo', 'is': 'isl', 'io': 'ido', 'ii': 'iii', 'iu': 'iku',
+    'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita',
+    'jv': 'jav', 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas',
+    'kr': 'kau', 'kk': 'kaz', 'km': 'khm', 'ki': 'kik', 'rw': 'kin',
+    'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
+    'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim',
+    'ln': 'lin', 'lt': 'lit', 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug',
+    'mk': 'mkd', 'mh': 'mah', 'ml': 'mal', 'mi': 'mri', 'mr': 'mar',
+    'ms': 'msa', 'mg': 'mlg', 'mt': 'mlt', 'mn': 'mon', 'na': 'nau',
+    'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep',
+    'nn': 'nno', 'nb': 'nob', 'no': 'nor', 'ny': 'nya', 'oc': 'oci',
+    'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss', 'pa': 'pan',
+    'fa': 'fas', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus',
+    'qu': 'que', 'rm': 'roh', 'ro': 'ron', 'rn': 'run', 'ru': 'rus',
+    'sg': 'sag', 'sa': 'san', 'si': 'sin', 'sk': 'slk', 'sl': 'slv',
+    'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
+    'st': 'sot', 'es': 'spa', 'sc': 'srd', 'sr': 'srp', 'ss': 'ssw',
+    'su': 'sun', 'sw': 'swa', 'sv': 'swe', 'ty': 'tah', 'ta': 'tam',
+    'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
+    'bo': 'bod', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso',
+    'tk': 'tuk', 'tr': 'tur', 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr',
+    'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie', 'vo': 'vol',
+    'cy': 'cym', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid',
+    'yo': 'yor', 'za': 'zha', 'zu': 'zul',
+}
+
+const iso639_2to1 = Object.fromEntries(
+    Object.entries(iso639_1to2).map(([k, v]) => [v, k])
+);
+
+const maps = {
+    2: iso639_1to2,
+    3: iso639_2to1,
+}
+
+export const convertLanguageCode = (code) => {
+    code = code?.split("-")[0]?.split("_")[0] || "";
+    return maps[code.length]?.[code.toLowerCase()] || null;
+}

api/src/misc/run-test.js

@@ -23,6 +23,15 @@ export async function runTest(url, params, expect) {
     if (expect.status !== result.body.status) {
         const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
         error.push(`status mismatch: ${detail}`);
+
+        if (result.body.status === 'error') {
+            error.push(`error code: ${result.body?.error?.code}`);
+        }
+    }
+
+    if (expect.errorCode && expect.errorCode !== result.body?.error?.code) {
+        const detail = `${expect.errorCode} (expected) != ${result.body.error.code} (actual)`
+        error.push(`error mismatch: ${detail}`);
     }
 
     if (expect.code !== result.status) {
@@ -41,4 +50,4 @@ export async function runTest(url, params, expect) {
     if (result.body.status === 'tunnel') {
         // TODO: stream testing
     }
-}
+}

api/src/misc/utils.js

@@ -1,55 +1,43 @@
-const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '='];
+import { request } from "undici";
+const redirectStatuses = new Set([301, 302, 303, 307, 308]);
 
-export function metadataManager(obj) {
-    const keys = Object.keys(obj);
-    const tags = [
-        "album",
-        "copyright",
-        "title",
-        "artist",
-        "track",
-        "date"
-    ]
-    let commands = []
+export async function getRedirectingURL(url, dispatcher, headers) {
+    const params = {
+        dispatcher,
+        method: 'HEAD',
+        headers,
+        redirect: 'manual'
+    };
+    const getParams = {
+        ...params,
+        method: 'GET',
+    };
 
-    for (const i in keys) {
-        if (tags.includes(keys[i]))
-            commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`)
+    const callback = (r) => {
+        if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
+            return r.headers['location'];
         }
-    return commands;
-}
+    }
 
-export function cleanString(string) {
-    for (const i in forbiddenCharsString) {
-        string = string.replaceAll("/", "_")
-                       .replaceAll(forbiddenCharsString[i], '')
-    }
-    return string;
-}
-export function verifyLanguageCode(code) {
-    const langCode = String(code.slice(0, 2).toLowerCase());
-    if (RegExp(/[a-z]{2}/).test(code)) {
-        return langCode
-    }
-    return "en"
-}
-export function languageCode(req) {
-    if (req.header('Accept-Language')) {
-        return verifyLanguageCode(req.header('Accept-Language'))
-    }
-    return "en"
-}
-export function cleanHTML(html) {
-    let clean = html.replace(/ {4}/g, '');
-    clean = clean.replace(/\n/g, '');
-    return clean
-}
+    /*
+        try request() with HEAD & GET,
+        then do the same with fetch
+        (fetch is required for shortened reddit links)
+    */
 
-export function getRedirectingURL(url) {
-    return fetch(url, { redirect: 'manual' }).then((r) => {
-        if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
-            return r.headers.get('location');
-    }).catch(() => null);
+    let location = await request(url, params)
+        .then(callback).catch(() => null);
+
+    location ??= await request(url, getParams)
+        .then(callback).catch(() => null);
+
+    location ??= await fetch(url, params)
+        .then(callback).catch(() => null);
+
+    location ??= await fetch(url, getParams)
+        .then(callback).catch(() => null);
+
+    return location;
 }
 
 export function merge(a, b) {
@@ -65,3 +53,27 @@ export function merge(a, b) {
 
     return a;
 }
+
+export function splitFilenameExtension(filename) {
+    const parts = filename.split('.');
+    const ext = parts.pop();
+
+    if (!parts.length) {
+        return [ ext, "" ]
+    } else {
+        return [ parts.join('.'), ext ]
+    }
+}
+
+export function zip(a, b) {
+    return a.map((value, i) => [ value, b[i] ]);
+}
+
+export function isURL(input) {
+    try {
+        new URL(input);
+        return true;
+    } catch {
+        return false;
+    }
+}

api/src/processing/cookie/cookie.js

@@ -4,16 +4,24 @@ export default class Cookie {
     constructor(input) {
         assert(typeof input === 'object');
         this._values = {};
-        this.set(input)
+
+        for (const [ k, v ] of Object.entries(input))
+            this.set(k, v);
     }
-    set(values) {
-        Object.entries(values).forEach(
-            ([ key, value ]) => this._values[key] = value
-        )
+
+    set(key, value) {
+        const old = this._values[key];
+        if (old === value)
+            return false;
+
+        this._values[key] = value;
+        return true;
     }
+
     unset(keys) {
         for (const key of keys) delete this._values[key]
     }
+
     static fromString(str) {
         const obj = {};
 
@@ -25,12 +33,15 @@ export default class Cookie {
 
         return new Cookie(obj)
     }
+
     toString() {
         return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ')
     }
+
     toJSON() {
         return this.toString()
     }
+
     values() {
         return Object.freeze({ ...this._values })
     }

api/src/processing/cookie/manager.js

@@ -1,50 +1,145 @@
 import Cookie from './cookie.js';
+
 import { readFile, writeFile } from 'fs/promises';
+import { Red, Green, Yellow } from '../../misc/console-text.js';
 import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
-import { env } from '../../config.js';
+import * as cluster from '../../misc/cluster.js';
+import { isCluster } from '../../config.js';
 
-const WRITE_INTERVAL = 60000,
-      cookiePath = env.cookiePath,
-      COUNTER = Symbol('counter');
+const WRITE_INTERVAL = 60000;
+const VALID_SERVICES = new Set([
+    'instagram',
+    'instagram_bearer',
+    'reddit',
+    'twitter',
+    'youtube',
+    'vimeo_bearer',
+]);
 
+const invalidCookies = {};
 let cookies = {}, dirty = false, intervalId;
 
-const setup = async () => {
-    try {
-        if (!cookiePath) return;
-
-        cookies = await readFile(cookiePath, 'utf8');
-        cookies = JSON.parse(cookies);
-        intervalId = setInterval(writeChanges, WRITE_INTERVAL)
-    } catch { /* no cookies for you */ }
-}
-
-setup();
-
-function writeChanges() {
+function writeChanges(cookiePath) {
     if (!dirty) return;
     dirty = false;
 
-    writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => {
-        clearInterval(intervalId)
+    const cookieData = JSON.stringify({ ...cookies, ...invalidCookies }, null, 4);
+    writeFile(cookiePath, cookieData).catch((e) => {
+        console.warn(`${Yellow('[!]')} failed writing updated cookies to storage`);
+        console.warn(e);
+        clearInterval(intervalId);
+        intervalId = null;
     })
 }
 
-export function getCookie(service) {
-    if (!cookies[service] || !cookies[service].length) return;
+const setupMain = async (cookiePath) => {
+    try {
+        cookies = await readFile(cookiePath, 'utf8');
+        cookies = JSON.parse(cookies);
+        for (const serviceName in cookies) {
+            if (!VALID_SERVICES.has(serviceName)) {
+                console.warn(`${Yellow('[!]')} ignoring unknown service in cookie file: ${serviceName}`);
+            } else if (!Array.isArray(cookies[serviceName])) {
+                console.warn(`${Yellow('[!]')} ${serviceName} in cookies file is not an array, ignoring it`);
+            } else if (cookies[serviceName].some(c => typeof c !== 'string')) {
+                console.warn(`${Yellow('[!]')} some cookie for ${serviceName} contains non-string value in cookies file`);
+            } else continue;
 
-    let n;
-    if (cookies[service][COUNTER] === undefined) {
-        n = cookies[service][COUNTER] = 0
-    } else {
-        ++cookies[service][COUNTER]
-        n = (cookies[service][COUNTER] %= cookies[service].length)
+            invalidCookies[serviceName] = cookies[serviceName];
+            delete cookies[serviceName];
+        }
+
+        if (!intervalId) {
+            intervalId = setInterval(() => writeChanges(cookiePath), WRITE_INTERVAL);
+        }
+
+        cluster.broadcast({ cookies });
+
+        console.log(`${Green('[✓]')} cookies loaded successfully!`);
+    } catch (e) {
+        console.error(`${Yellow('[!]')} failed to load cookies.`);
+        console.error('error:', e);
+    }
+}
+
+const setupWorker = async () => {
+    cookies = (await cluster.waitFor('cookies')).cookies;
+}
+
+export const loadFromFile = async (path) => {
+    if (cluster.isPrimary) {
+        await setupMain(path);
+    } else if (cluster.isWorker) {
+        await setupWorker();
     }
 
-    const cookie = cookies[service][n];
-    if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie);
+    dirty = false;
+}
 
-    return cookies[service][n]
+export const setup = async (path) => {
+    await loadFromFile(path);
+
+    if (isCluster) {
+        const messageHandler = (message) => {
+            if ('cookieUpdate' in message) {
+                const { cookieUpdate } = message;
+
+                if (cluster.isPrimary) {
+                    dirty = true;
+                    cluster.broadcast({ cookieUpdate });
+                }
+
+                const { service, idx, cookie } = cookieUpdate;
+                cookies[service][idx] = cookie;
+            }
+        }
+
+        if (cluster.isPrimary) {
+            cluster.mainOnMessage(messageHandler);
+        } else {
+            process.on('message', messageHandler);
+        }
+    }
+}
+
+export function getCookie(service) {
+    if (!VALID_SERVICES.has(service)) {
+        console.error(
+            `${Red('[!]')} ${service} not in allowed services list for cookies.`
+            + ' if adding a new cookie type, include it there.'
+        );
+        return;
+    }
+
+    if (!cookies[service] || !cookies[service].length) return;
+
+    const idx = Math.floor(Math.random() * cookies[service].length);
+
+    const cookie = cookies[service][idx];
+    if (typeof cookie === 'string') {
+        cookies[service][idx] = Cookie.fromString(cookie);
+    }
+
+    cookies[service][idx].meta = { service, idx };
+    return cookies[service][idx];
+}
+
+export function updateCookieValues(cookie, values) {
+    let changed = false;
+
+    for (const [ key, value ] of Object.entries(values)) {
+        changed = cookie.set(key, value) || changed;
+    }
+
+    if (changed && cookie.meta) {
+        dirty = true;
+        if (isCluster) {
+            const message = { cookieUpdate: { ...cookie.meta, cookie } };
+            cluster.send(message);
+        }
+    }
+
+    return changed;
 }
 
 export function updateCookie(cookie, headers) {
@@ -57,10 +152,6 @@ export function updateCookie(cookie, headers) {
 
     cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name));
     parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value);
+
     updateCookieValues(cookie, values);
 }
-
-export function updateCookieValues(cookie, values) {
-    cookie.set(values);
-    if (Object.keys(values).length) dirty = true
-}

api/src/processing/create-filename.js

@@ -1,3 +1,28 @@
+// characters that are disallowed on windows:
+// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
+const characterMap = {
+    '<':  '＜',
+    '>':  '＞',
+    ':':  '：',
+    '"':  '＂',
+    '/':  '／',
+    '\\': '＼',
+    '|': '｜',
+    '?': '？',
+    '*': '＊'
+};
+
+export const sanitizeString = (string) => {
+    // remove any potential control characters the string might contain
+    string = string.replace(/[\u0000-\u001F\u007F-\u009F]/g, "");
+
+    for (const [ char, replacement ] of Object.entries(characterMap)) {
+        string = string.replaceAll(char, replacement);
+    }
+
+    return string;
+}
+
 export default (f, style, isAudioOnly, isAudioMuted) => {
     let filename = '';
 
@@ -5,7 +30,11 @@ export default (f, style, isAudioOnly, isAudioMuted) => {
     let classicTags = [...infoBase];
     let basicTags = [];
 
-    const title = `${f.title} - ${f.author}`;
+    let title = sanitizeString(f.title);
+
+    if (f.author) {
+        title += ` - ${sanitizeString(f.author)}`;
+    }
 
     if (f.resolution) {
         classicTags.push(f.resolution);

api/src/processing/helpers/youtube-session.js

@@ -0,0 +1,81 @@
+import * as cluster from "../../misc/cluster.js";
+
+import { Agent } from "undici";
+import { env } from "../../config.js";
+import { Green, Yellow } from "../../misc/console-text.js";
+
+const defaultAgent = new Agent();
+
+let session;
+
+const validateSession = (sessionResponse) => {
+    if (!sessionResponse.potoken) {
+        throw "no poToken in session response";
+    }
+
+    if (!sessionResponse.visitor_data) {
+        throw "no visitor_data in session response";
+    }
+
+    if (!sessionResponse.updated) {
+        throw "no last update timestamp in session response";
+    }
+
+    // https://github.com/iv-org/youtube-trusted-session-generator/blob/c2dfe3f/potoken_generator/main.py#L25
+    if (sessionResponse.potoken.length < 160) {
+        console.error(`${Yellow('[!]')} poToken is too short and might not work (${new Date().toISOString()})`);
+    }
+}
+
+const updateSession = (newSession) => {
+    session = newSession;
+}
+
+const loadSession = async () => {
+    const sessionServerUrl = new URL(env.ytSessionServer);
+    sessionServerUrl.pathname = "/token";
+
+    const newSession = await fetch(
+        sessionServerUrl,
+        { dispatcher: defaultAgent }
+    ).then(a => a.json());
+
+    validateSession(newSession);
+
+    if (!session || session.updated < newSession?.updated) {
+        cluster.broadcast({ youtube_session: newSession });
+        updateSession(newSession);
+    }
+}
+
+const wrapLoad = (initial = false) => {
+    loadSession()
+    .then(() => {
+        if (initial) {
+            console.log(`${Green('[✓]')} poToken & visitor_data loaded successfully!`);
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading poToken & visitor_data at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+export const getYouTubeSession = () => {
+    return session;
+}
+
+export const setup = () => {
+    if (cluster.isPrimary) {
+        wrapLoad(true);
+        if (env.ytSessionReloadInterval > 0) {
+            setInterval(wrapLoad, env.ytSessionReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('youtube_session' in message) {
+                updateSession(message.youtube_session);
+            }
+        });
+    }
+}

api/src/processing/match-action.js

@@ -3,27 +3,48 @@ import createFilename from "./create-filename.js";
 import { createResponse } from "./request.js";
 import { audioIgnore } from "./service-config.js";
 import { createStream } from "../stream/manage.js";
+import { splitFilenameExtension } from "../misc/utils.js";
+import { convertLanguageCode } from "../misc/language-codes.js";
 
-export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disableMetadata, filenameStyle, twitterGif, requestIP, audioBitrate, alwaysProxy }) {
+const extraProcessingTypes = new Set(["merge", "remux", "mute", "audio", "gif"]);
+
+export default function({
+    r,
+    host,
+    audioFormat,
+    isAudioOnly,
+    isAudioMuted,
+    disableMetadata,
+    filenameStyle,
+    convertGif,
+    requestIP,
+    audioBitrate,
+    alwaysProxy,
+    localProcessing,
+}) {
     let action,
         responseType = "tunnel",
         defaultParams = {
-            u: r.urls,
+            url: r.urls,
             headers: r.headers,
             service: host,
             filename: r.filenameAttributes ?
                     createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
             fileMetadata: !disableMetadata ? r.fileMetadata : false,
-            requestIP
+            requestIP,
+            originalRequest: r.originalRequest,
+            subtitles: r.subtitles,
+            cover: !disableMetadata ? r.cover : false,
+            cropCover: !disableMetadata ? r.cropCover : false,
         },
         params = {};
 
     if (r.isPhoto) action = "photo";
     else if (r.picker) action = "picker"
-    else if (r.isGif && twitterGif) action = "gif";
+    else if (r.isGif && convertGif) action = "gif";
     else if (isAudioOnly) action = "audio";
     else if (isAudioMuted) action = "muteVideo";
-    else if (r.isM3U8) action = "m3u8";
+    else if (r.isHLS) action = "hls";
     else action = "video";
 
     if (action === "picker" || action === "audio") {
@@ -32,10 +53,11 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
     }
 
     if (action === "muteVideo" && isAudioMuted && !r.filenameAttributes) {
-        const parts = r.filename.split(".");
-        const ext = parts.pop();
-
-        defaultParams.filename = `${parts.join(".")}_mute.${ext}`;
+        const [ name, ext ] = splitFilenameExtension(r.filename);
+        defaultParams.filename = `${name}_mute.${ext}`;
+    } else if (action === "gif") {
+        const [ name ] = splitFilenameExtension(r.filename);
+        defaultParams.filename = `${name}.gif`;
     }
 
     switch (action) {
@@ -45,27 +67,29 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
             });
 
         case "photo":
-            responseType = "redirect";
+            params = { type: "proxy" };
             break;
 
         case "gif":
             params = { type: "gif" };
             break;
 
-        case "m3u8":
+        case "hls":
             params = {
-                type: Array.isArray(r.urls) ? "merge" : "remux"
+                type: Array.isArray(r.urls) ? "merge" : "remux",
+                isHLS: true,
             }
             break;
 
         case "muteVideo":
             let muteType = "mute";
-            if (Array.isArray(r.urls) && !r.isM3U8) {
+            if (Array.isArray(r.urls) && !r.isHLS) {
                 muteType = "proxy";
             }
             params = {
                 type: muteType,
-                u: Array.isArray(r.urls) ? r.urls[0] : r.urls
+                url: Array.isArray(r.urls) ? r.urls[0] : r.urls,
+                isHLS: r.isHLS
             }
             if (host === "reddit" && r.typeId === "redirect") {
                 responseType = "redirect";
@@ -79,6 +103,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                 case "twitter":
                 case "snapchat":
                 case "bsky":
+                case "xiaohongshu":
                     params = { picker: r.picker };
                     break;
 
@@ -90,14 +115,15 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                     }
                     params = {
                         picker: r.picker,
-                        u: createStream({
+                        url: createStream({
                             service: "tiktok",
                             type: audioStreamType,
-                            u: r.urls,
+                            url: r.urls,
                             headers: r.headers,
-                            filename: r.audioFilename,
+                            filename: `${r.audioFilename}.${audioFormat}`,
                             isAudioOnly: true,
                             audioFormat,
+                            audioBitrate
                         })
                     }
                     break;
@@ -121,7 +147,9 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
 
                 case "vimeo":
                     if (Array.isArray(r.urls)) {
-                        params = { type: "merge" }
+                        params = { type: "merge" };
+                    } else if (r.subtitles) {
+                        params = { type: "remux" };
                     } else {
                         responseType = "redirect";
                     }
@@ -135,19 +163,33 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                     }
                     break;
 
+                case "loom":
+                    if (r.subtitles) {
+                        params = { type: "remux" };
+                    } else {
+                        responseType = "redirect";
+                    }
+                    break;
+
                 case "vk":
                 case "tiktok":
+                    params = {
+                        type: r.subtitles ? "remux" : "proxy"
+                    };
+                    break;
+
+                case "ok":
+                case "xiaohongshu":
+                case "newgrounds":
                     params = { type: "proxy" };
                     break;
 
                 case "facebook":
-                case "vine":
                 case "instagram":
                 case "tumblr":
                 case "pinterest":
                 case "streamable":
                 case "snapchat":
-                case "loom":
                 case "twitch":
                     responseType = "redirect";
                     break;
@@ -155,9 +197,9 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
             break;
 
         case "audio":
-            if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) {
+            if (audioIgnore.has(host) || (host === "reddit" && r.typeId === "redirect")) {
                 return createResponse("error", {
-                    code: "error.api.fetch.empty"
+                    code: "error.api.service.audio_not_supported"
                 })
             }
 
@@ -181,18 +223,20 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                 }
             }
 
-            if (r.isM3U8 || host === "vimeo") {
+            if (r.isHLS || host === "vimeo") {
                 copy = false;
                 processType = "audio";
             }
 
             params = {
                 type: processType,
-                u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+                url: Array.isArray(r.urls) ? r.urls[1] : r.urls,
 
                 audioBitrate,
                 audioCopy: copy,
                 audioFormat,
+
+                isHLS: r.isHLS,
             }
             break;
     }
@@ -201,10 +245,39 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
         defaultParams.filename += `.${audioFormat}`;
     }
 
+    // alwaysProxy is set to true in match.js if localProcessing is forced
     if (alwaysProxy && responseType === "redirect") {
         responseType = "tunnel";
         params.type = "proxy";
     }
 
-    return createResponse(responseType, {...defaultParams, ...params})
+    // TODO: add support for HLS
+    // (very painful)
+    if (!params.isHLS && responseType !== "picker") {
+        const isPreferredWithExtra =
+            localProcessing === "preferred" && extraProcessingTypes.has(params.type);
+
+        if (localProcessing === "forced" || isPreferredWithExtra) {
+            responseType = "local-processing";
+        }
+    }
+
+    // extractors usually return ISO 639-1 language codes,
+    // but video players expect ISO 639-2, so we convert them here
+    const sublanguage = defaultParams.fileMetadata?.sublanguage;
+    if (sublanguage && sublanguage.length !== 3) {
+        const code = convertLanguageCode(sublanguage);
+        if (code) {
+            defaultParams.fileMetadata.sublanguage = code;
+        } else {
+            // if a language code couldn't be converted,
+            // then we don't want it at all
+            delete defaultParams.fileMetadata.sublanguage;
+        }
+    }
+
+    return createResponse(
+        responseType,
+        { ...defaultParams, ...params }
+    );
 }

api/src/processing/match.js

@@ -19,7 +19,6 @@ import tumblr from "./services/tumblr.js";
 import vimeo from "./services/vimeo.js";
 import soundcloud from "./services/soundcloud.js";
 import instagram from "./services/instagram.js";
-import vine from "./services/vine.js";
 import pinterest from "./services/pinterest.js";
 import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
@@ -29,10 +28,12 @@ import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
+import xiaohongshu from "./services/xiaohongshu.js";
+import newgrounds from "./services/newgrounds.js";
 
 let freebind;
 
-export default async function({ host, patternMatch, params }) {
+export default async function({ host, patternMatch, params, authType }) {
     const { url } = params;
     assert(url instanceof URL);
     let dispatcher, requestIP;
@@ -65,22 +66,36 @@ export default async function({ host, patternMatch, params }) {
             });
         }
 
+        // youtubeHLS will be fully removed in the future
+        let youtubeHLS = params.youtubeHLS;
+        const hlsEnv = env.enableDeprecatedYoutubeHls;
+
+        if (hlsEnv === "never" || (hlsEnv === "key" && authType !== "key")) {
+            youtubeHLS = false;
+        }
+
+        const subtitleLang =
+            params.subtitleLang !== "none" ? params.subtitleLang : undefined;
+
         switch (host) {
             case "twitter":
                 r = await twitter({
                     id: patternMatch.id,
                     index: patternMatch.index - 1,
-                    toGif: !!params.twitterGif,
+                    toGif: !!params.convertGif,
                     alwaysProxy: params.alwaysProxy,
-                    dispatcher
+                    dispatcher,
+                    subtitleLang
                 });
                 break;
 
             case "vk":
                 r = await vk({
-                    userId: patternMatch.userId,
+                    ownerId: patternMatch.ownerId,
                     videoId: patternMatch.videoId,
-                    quality: params.videoQuality
+                    accessKey: patternMatch.accessKey,
+                    quality: params.videoQuality,
+                    subtitleLang,
                 });
                 break;
 
@@ -97,20 +112,27 @@ export default async function({ host, patternMatch, params }) {
 
             case "youtube":
                 let fetchInfo = {
+                    dispatcher,
                     id: patternMatch.id.slice(0, 11),
                     quality: params.videoQuality,
-                    format: params.youtubeVideoCodec,
+                    codec: params.youtubeVideoCodec,
+                    container: params.youtubeVideoContainer,
                     isAudioOnly,
                     isAudioMuted,
                     dubLang: params.youtubeDubLang,
-                    dispatcher
+                    youtubeHLS,
+                    subtitleLang,
                 }
 
                 if (url.hostname === "music.youtube.com" || isAudioOnly) {
-                    fetchInfo.quality = "max";
-                    fetchInfo.format = "vp9";
+                    fetchInfo.quality = "1080";
+                    fetchInfo.codec = "vp9";
                     fetchInfo.isAudioOnly = true;
                     fetchInfo.isAudioMuted = false;
+
+                    if (env.ytAllowBetterAudio && params.youtubeBetterAudio) {
+                        fetchInfo.quality = "max";
+                    }
                 }
 
                 r = await youtube(fetchInfo);
@@ -118,20 +140,20 @@ export default async function({ host, patternMatch, params }) {
 
             case "reddit":
                 r = await reddit({
-                    sub: patternMatch.sub,
-                    id: patternMatch.id,
-                    user: patternMatch.user
+                    ...patternMatch,
+                    dispatcher,
                 });
                 break;
 
             case "tiktok":
                 r = await tiktok({
                     postId: patternMatch.postId,
-                    id: patternMatch.id,
+                    shortLink: patternMatch.shortLink,
                     fullAudio: params.tiktokFullAudio,
                     isAudioOnly,
-                    h265: params.tiktokH265,
+                    h265: params.allowH265,
                     alwaysProxy: params.alwaysProxy,
+                    subtitleLang,
                 });
                 break;
 
@@ -149,6 +171,7 @@ export default async function({ host, patternMatch, params }) {
                     password: patternMatch.password,
                     quality: params.videoQuality,
                     isAudioOnly,
+                    subtitleLang,
                 });
                 break;
 
@@ -156,12 +179,8 @@ export default async function({ host, patternMatch, params }) {
                 isAudioOnly = true;
                 isAudioMuted = false;
                 r = await soundcloud({
-                    url,
-                    author: patternMatch.author,
-                    song: patternMatch.song,
+                    ...patternMatch,
                     format: params.audioFormat,
-                    shortLink: patternMatch.shortLink || false,
-                    accessKey: patternMatch.accessKey || false
                 });
                 break;
 
@@ -174,12 +193,6 @@ export default async function({ host, patternMatch, params }) {
                 })
                 break;
 
-            case "vine":
-                r = await vine({
-                    id: patternMatch.id
-                });
-                break;
-
             case "pinterest":
                 r = await pinterest({
                     id: patternMatch.id,
@@ -210,6 +223,7 @@ export default async function({ host, patternMatch, params }) {
                     key: patternMatch.key,
                     quality: params.videoQuality,
                     isAudioOnly,
+                    subtitleLang,
                 });
                 break;
 
@@ -226,20 +240,39 @@ export default async function({ host, patternMatch, params }) {
 
             case "loom":
                 r = await loom({
-                    id: patternMatch.id
+                    id: patternMatch.id,
+                    subtitleLang,
                 });
                 break;
 
             case "facebook":
                 r = await facebook({
-                    ...patternMatch
+                    ...patternMatch,
+                    dispatcher
                 });
                 break;
 
             case "bsky":
                 r = await bluesky({
                     ...patternMatch,
-                    alwaysProxy: params.alwaysProxy
+                    alwaysProxy: params.alwaysProxy,
+                    dispatcher
+                });
+                break;
+
+            case "xiaohongshu":
+                r = await xiaohongshu({
+                    ...patternMatch,
+                    h265: params.allowH265,
+                    isAudioOnly,
+                    dispatcher,
+                });
+                break;
+
+            case "newgrounds":
+                r = await newgrounds({
+                    ...patternMatch,
+                    quality: params.videoQuality,
                 });
                 break;
 
@@ -265,7 +298,7 @@ export default async function({ host, patternMatch, params }) {
             switch(r.error) {
                 case "content.too_long":
                     context = {
-                        limit: env.durationLimit / 60,
+                        limit: parseFloat((env.durationLimit / 60).toFixed(2)),
                     }
                     break;
 
@@ -286,6 +319,15 @@ export default async function({ host, patternMatch, params }) {
             })
         }
 
+        let localProcessing = params.localProcessing;
+        const lpEnv = env.forceLocalProcessing;
+        const shouldForceLocal = lpEnv === "always" || (lpEnv === "session" && authType === "session");
+        const localDisabled = (!localProcessing || localProcessing === "disabled");
+
+        if (shouldForceLocal && localDisabled) {
+            localProcessing = "preferred";
+        }
+
         return matchAction({
             r,
             host,
@@ -294,10 +336,11 @@ export default async function({ host, patternMatch, params }) {
             isAudioMuted,
             disableMetadata: params.disableMetadata,
             filenameStyle: params.filenameStyle,
-            twitterGif: params.twitterGif,
+            convertGif: params.convertGif,
             requestIP,
             audioBitrate: params.audioBitrate,
-            alwaysProxy: params.alwaysProxy,
+            alwaysProxy: params.alwaysProxy || localProcessing === "forced",
+            localProcessing,
         })
     } catch {
         return createResponse("error", {

api/src/processing/request.js

@@ -1,7 +1,8 @@
+import mime from "mime";
 import ipaddr from "ipaddr.js";
 
-import { createStream } from "../stream/manage.js";
 import { apiSchema } from "./schema.js";
+import { createProxyTunnels, createStream } from "../stream/manage.js";
 
 export function createResponse(responseType, responseData) {
     const internalError = (code) => {
@@ -10,7 +11,7 @@ export function createResponse(responseType, responseData) {
             body: {
                 status: "error",
                 error: {
-                    code: code || "error.api.fetch.critical",
+                    code: code || "error.api.fetch.critical.core",
                 },
                 critical: true
             }
@@ -37,7 +38,7 @@ export function createResponse(responseType, responseData) {
 
             case "redirect":
                 response = {
-                    url: responseData?.u,
+                    url: responseData?.url,
                     filename: responseData?.filename
                 }
                 break;
@@ -49,10 +50,48 @@ export function createResponse(responseType, responseData) {
                 }
                 break;
 
+            case "local-processing":
+                response = {
+                    type: responseData?.type,
+                    service: responseData?.service,
+                    tunnel: createProxyTunnels(responseData),
+
+                    output: {
+                        type: mime.getType(responseData?.filename) || undefined,
+                        filename: responseData?.filename,
+                        metadata: responseData?.fileMetadata || undefined,
+                        subtitles: !!responseData?.subtitles || undefined,
+                    },
+
+                    audio: {
+                        copy: responseData?.audioCopy,
+                        format: responseData?.audioFormat,
+                        bitrate: responseData?.audioBitrate,
+                        cover: !!responseData?.cover || undefined,
+                        cropCover: !!responseData?.cropCover || undefined,
+                    },
+
+                    isHLS: responseData?.isHLS,
+                }
+
+                if (!response.audio.format) {
+                    if (response.type === "audio") {
+                        // audio response without a format is invalid
+                        return internalError();
+                    }
+                    delete response.audio;
+                }
+
+                if (!response.output.type || !response.output.filename) {
+                    // response without a type or filename is invalid
+                    return internalError();
+                }
+                break;
+
             case "picker":
                 response = {
                     picker: responseData?.picker,
-                    audio: responseData?.u,
+                    audio: responseData?.url,
                     audioFilename: responseData?.filename
                 }
                 break;
@@ -72,24 +111,28 @@ export function createResponse(responseType, responseData) {
             }
         }
     } catch {
-        return internalError()
+        return internalError();
     }
 }
 
 export function normalizeRequest(request) {
+    // TODO: remove after backwards compatibility period
+    if ("localProcessing" in request && typeof request.localProcessing === "boolean") {
+        request.localProcessing = request.localProcessing ? "preferred" : "disabled";
+    }
+
     return apiSchema.safeParseAsync(request).catch(() => (
         { success: false }
     ));
 }
 
-export function getIP(req) {
+export function getIP(req, prefix = 56) {
     const strippedIP = req.ip.replace(/^::ffff:/, '');
     const ip = ipaddr.parse(strippedIP);
     if (ip.kind() === 'ipv4') {
         return strippedIP;
     }
 
-    const prefix = 56;
     const v6Bytes = ip.toByteArray();
           v6Bytes.fill(0, prefix / 8);
 

api/src/processing/schema.js

@@ -1,7 +1,5 @@
 import { z } from "zod";
-
 import { normalizeURL } from "./url.js";
-import { verifyLanguageCode } from "../misc/utils.js";
 
 export const apiSchema = z.object({
     url: z.string()
@@ -22,26 +20,45 @@ export const apiSchema = z.object({
 
     filenameStyle: z.enum(
         ["classic", "pretty", "basic", "nerdy"]
-    ).default("classic"),
+    ).default("basic"),
 
     youtubeVideoCodec: z.enum(
         ["h264", "av1", "vp9"]
     ).default("h264"),
 
+    youtubeVideoContainer: z.enum(
+        ["auto", "mp4", "webm", "mkv"]
+    ).default("auto"),
+
     videoQuality: z.enum(
         ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"]
     ).default("1080"),
 
+    localProcessing: z.enum(
+        ["disabled", "preferred", "forced"]
+    ).default("disabled"),
+
     youtubeDubLang: z.string()
-                     .length(2)
-                     .transform(verifyLanguageCode)
+                     .min(2)
+                     .max(8)
+                     .regex(/^[0-9a-zA-Z\-]+$/)
+                     .optional(),
+
+    subtitleLang: z.string()
+                     .min(2)
+                     .max(8)
+                     .regex(/^[0-9a-zA-Z\-]+$/)
                      .optional(),
 
-    alwaysProxy: z.boolean().default(false),
     disableMetadata: z.boolean().default(false),
+
+    allowH265: z.boolean().default(false),
+    convertGif: z.boolean().default(true),
     tiktokFullAudio: z.boolean().default(false),
-    tiktokH265: z.boolean().default(false),
-    twitterGif: z.boolean().default(true),
-    youtubeDubBrowserLang: z.boolean().default(false),
+
+    alwaysProxy: z.boolean().default(false),
+
+    youtubeHLS: z.boolean().default(false),
+    youtubeBetterAudio: z.boolean().default(false),
 })
 .strict();

api/src/processing/service-alias.js

@@ -1,5 +1,6 @@
 const friendlyNames = {
     bsky: "bluesky",
+    twitch: "twitch clips"
 }
 
 export const friendlyServiceName = (service) => {

api/src/processing/service-config.js

@@ -1,12 +1,13 @@
 import UrlPattern from "url-pattern";
 
-export const audioIgnore = ["vk", "ok", "loom"];
-export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky"];
+export const audioIgnore = new Set(["vk", "ok", "loom"]);
+export const hlsExceptions = new Set(["dailymotion", "vimeo", "rutube", "bsky", "youtube"]);
 
 export const services = {
     bilibili: {
         patterns: [
             "video/:comId",
+            "video/:comId?p=:partId",
             "_shortLink/:comShortLink",
             "_tv/:lang/video/:tvId",
             "_tv/video/:tvId"
@@ -30,23 +31,35 @@ export const services = {
             "reel/:id",
             "share/:shareType/:id"
         ],
-        subdomains: ["web"],
+        subdomains: ["web", "m"],
         altDomains: ["fb.watch"],
     },
     instagram: {
         patterns: [
-            "reels/:postId",
-            ":username/reel/:postId",
-            "reel/:postId",
             "p/:postId",
-            ":username/p/:postId",
             "tv/:postId",
-            "stories/:username/:storyId"
+            "reel/:postId",
+            "reels/:postId",
+            "stories/:username/:storyId",
+
+            /*
+                share & username links use the same url pattern,
+                so we test the share pattern first, cuz id type is different.
+                however, if someone has the "share" username and the user
+                somehow gets a link of this ancient style, it's joever.
+            */
+
+            "share/:shareId",
+            "share/p/:shareId",
+            "share/reel/:shareId",
+
+            ":username/p/:postId",
+            ":username/reel/:postId",
         ],
         altDomains: ["ddinstagram.com"],
     },
     loom: {
-        patterns: ["share/:id"],
+        patterns: ["share/:id", "embed/:id"],
     },
     ok: {
         patterns: [
@@ -62,10 +75,31 @@ export const services = {
             "url_shortener/:shortLink"
         ],
     },
+    newgrounds: {
+        patterns: [
+            "portal/view/:id",
+            "audio/listen/:audioId",
+        ]
+    },
     reddit: {
         patterns: [
+            "comments/:id",
+
+            "r/:sub/comments/:id",
             "r/:sub/comments/:id/:title",
-            "user/:user/comments/:id/:title"
+            "r/:sub/comments/:id/comment/:commentId",
+
+            "user/:user/comments/:id",
+            "user/:user/comments/:id/:title",
+            "user/:user/comments/:id/comment/:commentId",
+
+            "r/u_:user/comments/:id",
+            "r/u_:user/comments/:id/:title",
+            "r/u_:user/comments/:id/comment/:commentId",
+
+            "r/:sub/s/:shareId",
+
+            "video/:shortId",
         ],
         subdomains: "*",
     },
@@ -89,6 +123,7 @@ export const services = {
             "add/:username",
             "u/:username",
             "t/:shortLink",
+            "o/:spotlightId",
         ],
         subdomains: ["t", "story"],
     },
@@ -111,12 +146,13 @@ export const services = {
     tiktok: {
         patterns: [
             ":user/video/:postId",
-            ":id",
-            "t/:id",
+            "i18n/share/video/:postId",
+            ":shortLink",
+            "t/:shortLink",
             ":user/photo/:postId",
-            "v/:id.html"
+            "v/:postId.html"
         ],
-        subdomains: ["vt", "vm", "m"],
+        subdomains: ["vt", "vm", "m", "t"],
     },
     tumblr: {
         patterns: [
@@ -130,6 +166,7 @@ export const services = {
     twitch: {
         patterns: [":channel/clip/:clip"],
         tld: "tv",
+        subdomains: ["clips", "www", "m"],
     },
     twitter: {
         patterns: [
@@ -137,37 +174,51 @@ export const services = {
             ":user/status/:id/video/:index",
             ":user/status/:id/photo/:index",
             ":user/status/:id/mediaviewer",
-            ":user/status/:id/mediaViewer"
+            ":user/status/:id/mediaViewer",
+            "i/bookmarks?post_id=:id"
         ],
         subdomains: ["mobile"],
         altDomains: ["x.com", "vxtwitter.com", "fixvx.com"],
     },
-    vine: {
-        patterns: ["v/:id"],
-        tld: "co",
-    },
     vimeo: {
         patterns: [
             ":id",
             "video/:id",
             ":id/:password",
-            "/channels/:user/:id"
+            "/channels/:user/:id",
+            "groups/:groupId/videos/:id"
         ],
         subdomains: ["player"],
     },
     vk: {
         patterns: [
-            "video:userId_:videoId",
-            "clip:userId_:videoId",
-            "clips:duplicate?z=clip:userId_:videoId"
+            "video:ownerId_:videoId",
+            "clip:ownerId_:videoId",
+            "video:ownerId_:videoId_:accessKey",
+            "clip:ownerId_:videoId_:accessKey",
+
+            // links with a duplicate author id and/or zipper query param
+            "clips:duplicateId",
+            "videos:duplicateId",
+            "search/video"
         ],
         subdomains: ["m"],
+        altDomains: ["vkvideo.ru", "vk.ru"],
+    },
+    xiaohongshu: {
+        patterns: [
+            "explore/:id?xsec_token=:token",
+            "discovery/item/:id?xsec_token=:token",
+            ":shareType/:shareId",
+        ],
+        altDomains: ["xhslink.com"],
     },
     youtube: {
         patterns: [
             "watch?v=:id",
             "embed/:id",
-            "watch/:id"
+            "watch/:id",
+            "v/:id"
         ],
         subdomains: ["music", "m"],
     }
@@ -176,7 +227,7 @@ export const services = {
 Object.values(services).forEach(service => {
     service.patterns = service.patterns.map(
         pattern => new UrlPattern(pattern, {
-            segmentValueCharset: UrlPattern.defaultOptions.segmentValueCharset + '@\\.'
+            segmentValueCharset: UrlPattern.defaultOptions.segmentValueCharset + '@\\.:'
         })
     )
 })

api/src/processing/service-patterns.js

@@ -1,49 +1,72 @@
 export const testers = {
     "bilibili": pattern =>
-        pattern.comId?.length <= 12 || pattern.comShortLink?.length <= 16
-        || pattern.tvId?.length <= 24,
+        (pattern.comId?.length <= 12 && pattern.partId?.length <= 3) ||
+        (pattern.comId?.length <= 12 && !pattern.partId) ||
+        pattern.comShortLink?.length <= 16 ||
+        pattern.tvId?.length <= 24,
+
+    "bsky": pattern =>
+        pattern.user?.length <= 128 && pattern.post?.length <= 128,
 
     "dailymotion": pattern => pattern.id?.length <= 32,
 
+    "facebook": pattern =>
+        pattern.shortLink?.length <= 11 ||
+        pattern.username?.length <= 30 ||
+        pattern.caption?.length <= 255 ||
+        pattern.id?.length <= 20 && !pattern.shareType ||
+        pattern.id?.length <= 20 && pattern.shareType?.length === 1,
+
     "instagram": pattern =>
-        pattern.postId?.length <= 12
-        || (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
+        pattern.postId?.length <= 48 ||
+        pattern.shareId?.length <= 16 ||
+        (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
 
     "loom": pattern =>
         pattern.id?.length <= 32,
 
+    "newgrounds": pattern =>
+        pattern.id?.length <= 12 ||
+        pattern.audioId?.length <= 12,
+
     "ok": pattern =>
         pattern.id?.length <= 16,
 
     "pinterest": pattern =>
-        pattern.id?.length <= 128 || pattern.shortLink?.length <= 32,
+        pattern.id?.length <= 128 ||
+        pattern.shortLink?.length <= 32,
 
     "reddit": pattern =>
-        (pattern.sub?.length <= 22 && pattern.id?.length <= 10)
-        || (pattern.user?.length <= 22 && pattern.id?.length <= 10),
+        pattern.id?.length <= 16 && !pattern.sub && !pattern.user ||
+        (pattern.sub?.length <= 22 && pattern.id?.length <= 16) ||
+        (pattern.user?.length <= 22 && pattern.id?.length <= 16) ||
+        (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16) ||
+        (pattern.shortId?.length <= 16),
 
     "rutube": pattern =>
         (pattern.id?.length === 32 && pattern.key?.length <= 32) ||
-        pattern.id?.length === 32 || pattern.yappyId?.length === 32,
-
-    "soundcloud": pattern =>
-        (pattern.author?.length <= 255 && pattern.song?.length <= 255)
-        || pattern.shortLink?.length <= 32,
+        pattern.id?.length === 32 ||
+        pattern.yappyId?.length === 32,
 
     "snapchat": pattern =>
-        (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255))
-        || pattern.spotlightId?.length <= 255
-        || pattern.shortLink?.length <= 16,
+        (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255)) ||
+        pattern.spotlightId?.length <= 255 ||
+        pattern.shortLink?.length <= 16,
+
+    "soundcloud": pattern =>
+        (pattern.author?.length <= 255 && pattern.song?.length <= 255) ||
+        pattern.shortLink?.length <= 32,
 
     "streamable": pattern =>
-        pattern.id?.length === 6,
+        pattern.id?.length <= 6,
 
     "tiktok": pattern =>
-        pattern.postId?.length <= 21 || pattern.id?.length <= 13,
+        pattern.postId?.length <= 21 ||
+        pattern.shortLink?.length <= 21,
 
     "tumblr": pattern =>
-        pattern.id?.length < 21
-        || (pattern.id?.length < 21 && pattern.user?.length <= 32),
+        pattern.id?.length < 21 ||
+        (pattern.id?.length < 21 && pattern.user?.length <= 32),
 
     "twitch": pattern =>
         pattern.channel && pattern.clip?.length <= 100,
@@ -52,25 +75,16 @@ export const testers = {
         pattern.id?.length < 20,
 
     "vimeo": pattern =>
-        pattern.id?.length <= 11
-        && (!pattern.password || pattern.password.length < 16),
-
-    "vine": pattern =>
-        pattern.id?.length <= 12,
+        pattern.id?.length <= 11 && (!pattern.password || pattern.password.length < 16),
 
     "vk": pattern =>
-        pattern.userId?.length <= 10 && pattern.videoId?.length <= 10,
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
+
+    "xiaohongshu": pattern =>
+        pattern.id?.length <= 24 && pattern.token?.length <= 64 ||
+        pattern.shareId?.length <= 24 && pattern.shareType?.length === 1,
 
     "youtube": pattern =>
         pattern.id?.length <= 11,
-
-    "facebook": pattern =>
-        pattern.shortLink?.length <= 11
-        || pattern.username?.length <= 30
-        || pattern.caption?.length <= 255
-        || pattern.id?.length <= 20 && !pattern.shareType
-        || pattern.id?.length <= 20 && pattern.shareType?.length === 1,
-
-    "bsky": pattern =>
-        pattern.user?.length <= 128 && pattern.post?.length <= 128,
 }

api/src/processing/services/bilibili.js

@@ -1,19 +1,8 @@
 import { genericUserAgent, env } from "../../config.js";
+import { resolveRedirectingURL } from "../url.js";
 
 // TO-DO: higher quality downloads (currently requires an account)
 
-function com_resolveShortlink(shortId) {
-    return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' })
-            .then(r => r.status > 300 && r.status < 400 && r.headers.get('location'))
-            .then(url => {
-                if (!url) return;
-                const path = new URL(url).pathname;
-                if (path.startsWith('/video/'))
-                    return path.split('/')[2];
-            })
-            .catch(() => {})
-}
-
 function getBest(content) {
     return content?.filter(v => v.baseUrl || v.url)
                 .map(v => (v.baseUrl = v.baseUrl || v.url, v))
@@ -28,8 +17,14 @@ function extractBestQuality(dashData) {
     return [ bestVideo, bestAudio ];
 }
 
-async function com_download(id) {
-    let html = await fetch(`https://bilibili.com/video/${id}`, {
+async function com_download(id, partId) {
+    const url = new URL(`https://bilibili.com/video/${id}`);
+
+    if (partId) {
+        url.searchParams.set('p', partId);
+    }
+
+    const html = await fetch(url, {
         headers: {
             "user-agent": genericUserAgent
         }
@@ -45,7 +40,10 @@ async function com_download(id) {
         return { error: "fetch.empty" };
     }
 
-    let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
+    const streamData = JSON.parse(
+        html.split('<script>window.__playinfo__=')[1].split('</script>')[0]
+    );
+
     if (streamData.data.timelength > env.durationLimit * 1000) {
         return { error: "content.too_long" };
     }
@@ -55,10 +53,15 @@ async function com_download(id) {
         return { error: "fetch.empty" };
     }
 
+    let filenameBase = `bilibili_${id}`;
+    if (partId) {
+        filenameBase += `_${partId}`;
+    }
+
     return {
         urls: [video.baseUrl, audio.baseUrl],
-        audioFilename: `bilibili_${id}_audio`,
-        filename: `bilibili_${id}_${video.width}x${video.height}.mp4`
+        audioFilename: `${filenameBase}_audio`,
+        filename: `${filenameBase}_${video.width}x${video.height}.mp4`,
     };
 }
 
@@ -97,13 +100,14 @@ async function tv_download(id) {
     };
 }
 
-export default async function({ comId, tvId, comShortLink }) {
+export default async function({ comId, tvId, comShortLink, partId }) {
     if (comShortLink) {
-        comId = await com_resolveShortlink(comShortLink);
+        const patternMatch = await resolveRedirectingURL(`https://b23.tv/${comShortLink}`);
+        comId = patternMatch?.comId;
     }
 
     if (comId) {
-        return com_download(comId);
+        return com_download(comId, partId);
     } else if (tvId) {
         return tv_download(tvId);
     }

api/src/processing/services/bluesky.js

@@ -2,12 +2,19 @@ import HLS from "hls-parser";
 import { cobaltUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 
-const extractVideo = async ({ getPost, filename }) => {
-    const urlMasterHLS = getPost?.thread?.post?.embed?.playlist;
-    if (!urlMasterHLS) return { error: "fetch.empty" };
-    if (!urlMasterHLS.startsWith("https://video.bsky.app/")) return { error: "fetch.empty" };
+const extractVideo = async ({ media, filename, dispatcher }) => {
+    let urlMasterHLS = media?.playlist;
 
-    const masterHLS = await fetch(urlMasterHLS)
+    if (!urlMasterHLS || !urlMasterHLS.startsWith("https://video.bsky.app/")) {
+        return { error: "fetch.empty" };
+    }
+
+    urlMasterHLS = urlMasterHLS.replace(
+        "video.bsky.app/watch/",
+        "video.cdn.bsky.app/hls/"
+    );
+
+    const masterHLS = await fetch(urlMasterHLS, { dispatcher })
         .then(r => {
             if (r.status !== 200) return;
             return r.text();
@@ -26,7 +33,7 @@ const extractVideo = async ({ getPost, filename }) => {
         urls: videoURL,
         filename: `${filename}.mp4`,
         audioFilename: `${filename}_audio`,
-        isM3U8: true,
+        isHLS: true,
     }
 }
 
@@ -48,7 +55,7 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
         let proxiedImage = createStream({
             service: "bluesky",
             type: "proxy",
-            u: url,
+            url,
             filename: `${filename}_${i + 1}.jpg`,
         });
 
@@ -64,7 +71,25 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
     return { picker };
 }
 
-export default async function ({ user, post, alwaysProxy }) {
+const extractGif = ({ url, filename }) => {
+    const gifUrl = new URL(url);
+
+    if (!gifUrl || gifUrl.hostname !== "media.tenor.com") {
+        return { error: "fetch.empty" };
+    }
+
+    // remove downscaling params from gif url
+    // such as "?hh=498&ww=498"
+    gifUrl.search = "";
+
+    return {
+        urls: gifUrl,
+        isPhoto: true,
+        filename: `${filename}.gif`,
+    }
+}
+
+export default async function ({ user, post, alwaysProxy, dispatcher }) {
     const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
     apiEndpoint.searchParams.set(
         "uri",
@@ -73,20 +98,59 @@ export default async function ({ user, post, alwaysProxy }) {
 
     const getPost = await fetch(apiEndpoint, {
         headers: {
-            "user-agent": cobaltUserAgent
-        }
+            "user-agent": cobaltUserAgent,
+        },
+        dispatcher
     }).then(r => r.json()).catch(() => {});
 
-    if (!getPost || getPost?.error) return { error: "fetch.empty" };
+    if (!getPost) return { error: "fetch.empty" };
+
+    if (getPost.error) {
+        switch (getPost.error) {
+            case "NotFound":
+            case "InternalServerError":
+                return { error: "content.post.unavailable" };
+            case "InvalidRequest":
+                return { error: "link.unsupported" };
+            default:
+                return { error: "content.post.unavailable" };
+        }
+    }
 
     const embedType = getPost?.thread?.post?.embed?.$type;
     const filename = `bluesky_${user}_${post}`;
 
-    if (embedType === "app.bsky.embed.video#view") {
-        return extractVideo({ getPost, filename });
-    }
-    if (embedType === "app.bsky.embed.images#view") {
-        return extractImages({ getPost, filename, alwaysProxy });
+    switch (embedType) {
+        case "app.bsky.embed.video#view":
+            return extractVideo({
+                media: getPost.thread?.post?.embed,
+                filename,
+            });
+
+        case "app.bsky.embed.images#view":
+            return extractImages({
+                getPost,
+                filename,
+                alwaysProxy
+            });
+
+        case "app.bsky.embed.external#view":
+            return extractGif({
+                url: getPost?.thread?.post?.embed?.external?.uri,
+                filename,
+            });
+
+        case "app.bsky.embed.recordWithMedia#view":
+            if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") {
+                return extractGif({
+                    url: getPost?.thread?.post?.embed?.media?.external?.uri,
+                    filename,
+                });
+            }
+            return extractVideo({
+                media: getPost.thread?.post?.embed?.media,
+                filename,
+            });
     }
 
     return { error: "fetch.empty" };

api/src/processing/services/dailymotion.js

@@ -92,7 +92,7 @@ export default async function({ id }) {
 
     return {
         urls: bestQuality.uri,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             service: 'dailymotion',
             id: media.xid,

api/src/processing/services/facebook.js

@@ -8,8 +8,8 @@ const headers = {
     'Sec-Fetch-Site': 'none',
 }
 
-const resolveUrl = (url) => {
-    return fetch(url, { headers })
+const resolveUrl = (url, dispatcher) => {
+    return fetch(url, { headers, dispatcher })
         .then(r => {
             if (r.headers.get('location')) {
                 return decodeURIComponent(r.headers.get('location'));
@@ -23,13 +23,13 @@ const resolveUrl = (url) => {
         .catch(() => false);
 }
 
-export default async function({ id, shareType, shortLink }) {
+export default async function({ id, shareType, shortLink, dispatcher }) {
     let url = `https://web.facebook.com/i/videos/${id}`;
 
     if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
-    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`, dispatcher);
 
-    const html = await fetch(url, { headers })
+    const html = await fetch(url, { headers, dispatcher })
         .then(r => r.text())
         .catch(() => false);
 

api/src/processing/services/instagram.js

@@ -1,3 +1,5 @@
+import { randomBytes } from "node:crypto";
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
@@ -8,6 +10,7 @@ const commonHeaders = {
     "sec-fetch-site": "same-origin",
     "x-ig-app-id": "936619743392459"
 }
+
 const mobileHeaders = {
     "x-ig-app-locale": "en_US",
     "x-ig-device-locale": "en_US",
@@ -19,6 +22,7 @@ const mobileHeaders = {
     "x-fb-server-cluster": "True",
     "content-length": "0",
 }
+
 const embedHeaders = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
     "Accept-Language": "en-GB,en;q=0.9",
@@ -33,7 +37,7 @@ const embedHeaders = {
     "Sec-Fetch-Site": "none",
     "Sec-Fetch-User": "?1",
     "Upgrade-Insecure-Requests": "1",
-    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "User-Agent": genericUserAgent,
 }
 
 const cachedDtsg = {
@@ -41,7 +45,17 @@ const cachedDtsg = {
     expiry: 0
 }
 
-export default function(obj) {
+const getNumberFromQuery = (name, data) => {
+    const s = data?.match(new RegExp(name + '=(\\d+)'))?.[1];
+    if (+s) return +s;
+}
+
+const getObjectFromEntries = (name, data) => {
+    const obj = data?.match(new RegExp('\\["' + name + '",.*?,({.*?}),\\d+\\]'))?.[1];
+    return obj && JSON.parse(obj);
+}
+
+export default function instagram(obj) {
     const dispatcher = obj.dispatcher;
 
     async function findDtsgId(cookie) {
@@ -91,6 +105,7 @@ export default function(obj) {
         updateCookie(cookie, data.headers);
         return data.json();
     }
+
     async function getMediaId(id, { cookie, token } = {}) {
         const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/');
         oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`);
@@ -119,6 +134,7 @@ export default function(obj) {
 
         return mediaInfo?.items?.[0];
     }
+
     async function requestHTML(id, cookie) {
         const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, {
             headers: {
@@ -136,40 +152,167 @@ export default function(obj) {
 
         return embedData;
     }
-    async function requestGQL(id, cookie) {
-        let dtsgId;
 
-        if (cookie) {
-            dtsgId = await findDtsgId(cookie);
-        }
-        const url = new URL('https://www.instagram.com/api/graphql/');
+    async function getGQLParams(id, cookie) {
+        const req = await fetch(`https://www.instagram.com/p/${id}/`, {
+            headers: {
+                ...embedHeaders,
+                cookie
+            },
+            dispatcher
+        });
 
-        const requestData = {
-            jazoest: '26406',
-            variables: JSON.stringify({
-            shortcode: id,
-            __relay_internal__pv__PolarisShareMenurelayprovider: false
-            }),
-            doc_id: '7153618348081770'
+        const html = await req.text();
+        const siteData = getObjectFromEntries('SiteData', html);
+        const polarisSiteData = getObjectFromEntries('PolarisSiteData', html);
+        const webConfig = getObjectFromEntries('DGWWebConfig', html);
+        const pushInfo = getObjectFromEntries('InstagramWebPushInfo', html);
+        const lsd = getObjectFromEntries('LSD', html)?.token || randomBytes(8).toString('base64url');
+        const csrf = getObjectFromEntries('InstagramSecurityConfig', html)?.csrf_token;
+
+        const anon_cookie = [
+            csrf && "csrftoken=" + csrf,
+            polarisSiteData?.device_id && "ig_did=" + polarisSiteData?.device_id,
+            "wd=1280x720",
+            "dpr=2",
+            polarisSiteData?.machine_id && "mid=" + polarisSiteData.machine_id,
+            "ig_nrcb=1"
+        ].filter(a => a).join('; ');
+
+        return {
+            headers: {
+                'x-ig-app-id': webConfig?.appId || '936619743392459',
+                'X-FB-LSD': lsd,
+                'X-CSRFToken': csrf,
+                'X-Bloks-Version-Id': getObjectFromEntries('WebBloksVersioningID', html)?.versioningID,
+                'x-asbd-id': 129477,
+                cookie: anon_cookie
+            },
+            body: {
+                __d: 'www',
+                __a: '1',
+                __s: '::' + Math.random().toString(36).substring(2).replace(/\d/g, '').slice(0, 6),
+                __hs: siteData?.haste_session || '20126.HYP:instagram_web_pkg.2.1...0',
+                __req: 'b',
+                __ccg: 'EXCELLENT',
+                __rev: pushInfo?.rollout_hash || '1019933358',
+                __hsi: siteData?.hsi || '7436540909012459023',
+                __dyn: randomBytes(154).toString('base64url'),
+                __csr: randomBytes(154).toString('base64url'),
+                __user: '0',
+                __comet_req: getNumberFromQuery('__comet_req', html) || '7',
+                av: '0',
+                dpr: '2',
+                lsd,
+                jazoest: getNumberFromQuery('jazoest', html) || Math.floor(Math.random() * 10000),
+                __spin_r: siteData?.__spin_r || '1019933358',
+                __spin_b: siteData?.__spin_b || 'trunk',
+                __spin_t: siteData?.__spin_t || Math.floor(new Date().getTime() / 1000),
+            }
         };
-        if (dtsgId) {
-            requestData.fb_dtsg = dtsgId;
+    }
+
+    async function requestGQL(id, cookie) {
+        const { headers, body } = await getGQLParams(id, cookie);
+
+        const req = await fetch('https://www.instagram.com/graphql/query', {
+            method: 'POST',
+            dispatcher,
+            headers: {
+                ...embedHeaders,
+                ...headers,
+                cookie,
+                'content-type': 'application/x-www-form-urlencoded',
+                'X-FB-Friendly-Name': 'PolarisPostActionLoadPostQueryQuery',
+            },
+            body: new URLSearchParams({
+                ...body,
+                fb_api_caller_class: 'RelayModern',
+                fb_api_req_friendly_name: 'PolarisPostActionLoadPostQueryQuery',
+                variables: JSON.stringify({
+                    shortcode: id,
+                    fetch_tagged_user_count: null,
+                    hoisted_comment_id: null,
+                    hoisted_reply_id: null
+                }),
+                server_timestamps: true,
+                doc_id: '8845758582119845'
+            }).toString()
+        });
+
+        return {
+            gql_data: await req.json()
+                        .then(r => r.data)
+                        .catch(() => null)
+        };
+    }
+
+    async function getErrorContext(id) {
+        try {
+            const { headers, body } = await getGQLParams(id);
+
+            const req = await fetch('https://www.instagram.com/ajax/bulk-route-definitions/', {
+                method: 'POST',
+                dispatcher,
+                headers: {
+                    ...embedHeaders,
+                    ...headers,
+                    'content-type': 'application/x-www-form-urlencoded',
+                    'X-Ig-D': 'www',
+                },
+                body: new URLSearchParams({
+                    'route_urls[0]': `/p/${id}/`,
+                    routing_namespace: 'igx_www',
+                    ...body
+                }).toString()
+            });
+
+            const response = await req.text();
+            if (response.includes('"tracePolicy":"polaris.privatePostPage"'))
+                return { error: 'content.post.private' };
+
+            const [, mediaId, mediaOwnerId] = response.match(
+                /"media_id":\s*?"(\d+)","media_owner_id":\s*?"(\d+)"/
+            ) || [];
+
+            if (mediaId && mediaOwnerId) {
+                const rulingURL = new URL('https://www.instagram.com/api/v1/web/get_ruling_for_media_content_logged_out');
+                rulingURL.searchParams.set('media_id', mediaId);
+                rulingURL.searchParams.set('owner_id', mediaOwnerId);
+
+                const rulingResponse = await fetch(rulingURL, {
+                    headers: {
+                        ...headers,
+                        ...commonHeaders
+                    },
+                    dispatcher,
+                }).then(a => a.json()).catch(() => ({}));
+
+                if (rulingResponse?.title?.includes('Restricted'))
+                    return { error: "content.post.age" };
+            }
+        } catch {
+            return { error: "fetch.fail" };
         }
 
-        return (await request(url, cookie, 'POST', requestData))
-                .data
-                ?.xdt_api__v1__media__shortcode__web_info
-                ?.items
-                ?.[0];
+        return { error: "fetch.empty" };
     }
 
     function extractOldPost(data, id, alwaysProxy) {
-        const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children;
+        const shortcodeMedia = data?.gql_data?.shortcode_media || data?.gql_data?.xdt_shortcode_media;
+        const sidecar = shortcodeMedia?.edge_sidecar_to_children;
+
         if (sidecar) {
             const picker = sidecar.edges.filter(e => e.node?.display_url)
                 .map((e, i) => {
-                    const type = e.node?.is_video ? "video" : "photo";
-                    const url = type === "video" ? e.node?.video_url : e.node?.display_url;
+                    const type = e.node?.is_video && e.node?.video_url ? "video" : "photo";
+
+                    let url;
+                    if (type === "video") {
+                        url = e.node?.video_url;
+                    } else if (type === "photo") {
+                        url = e.node?.display_url;
+                    }
 
                     let itemExt = type === "video" ? "mp4" : "jpg";
 
@@ -177,7 +320,7 @@ export default function(obj) {
                     if (alwaysProxy) proxyFile = createStream({
                         service: "instagram",
                         type: "proxy",
-                        u: url,
+                        url,
                         filename: `instagram_${id}_${i + 1}.${itemExt}`
                     });
 
@@ -189,23 +332,28 @@ export default function(obj) {
                         thumb: createStream({
                             service: "instagram",
                             type: "proxy",
-                            u: e.node?.display_url,
+                            url: e.node?.display_url,
                             filename: `instagram_${id}_${i + 1}.jpg`
                         })
                     }
                 });
 
             if (picker.length) return { picker }
-        } else if (data?.gql_data?.shortcode_media?.video_url) {
+        }
+
+        if (shortcodeMedia?.video_url) {
             return {
-                urls: data.gql_data.shortcode_media.video_url,
+                urls: shortcodeMedia.video_url,
                 filename: `instagram_${id}.mp4`,
                 audioFilename: `instagram_${id}_audio`
             }
-        } else if (data?.gql_data?.shortcode_media?.display_url) {
+        }
+
+        if (shortcodeMedia?.display_url) {
             return {
-                urls: data.gql_data?.shortcode_media.display_url,
-                isPhoto: true
+                urls: shortcodeMedia.display_url,
+                isPhoto: true,
+                filename: `instagram_${id}.jpg`,
             }
         }
     }
@@ -230,7 +378,7 @@ export default function(obj) {
                     if (alwaysProxy) proxyFile = createStream({
                         service: "instagram",
                         type: "proxy",
-                        u: url,
+                        url,
                         filename: `instagram_${id}_${i + 1}.${itemExt}`
                     });
 
@@ -242,7 +390,7 @@ export default function(obj) {
                         thumb: createStream({
                             service: "instagram",
                             type: "proxy",
-                            u: imageUrl,
+                            url: imageUrl,
                             filename: `instagram_${id}_${i + 1}.jpg`
                         })
                     }
@@ -266,6 +414,9 @@ export default function(obj) {
     }
 
     async function getPost(id, alwaysProxy) {
+        const hasData = (data) => data
+                                    && data.gql_data !== null
+                                    && data?.gql_data?.xdt_shortcode_media !== null;
         let data, result;
         try {
             const cookie = getCookie('instagram');
@@ -282,19 +433,21 @@ export default function(obj) {
             if (media_id && token) data = await requestMobileApi(media_id, { token });
 
             // mobile api (no cookie, cookie)
-            if (media_id && !data) data = await requestMobileApi(media_id);
-            if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie });
+            if (media_id && !hasData(data)) data = await requestMobileApi(media_id);
+            if (media_id && cookie && !hasData(data)) data = await requestMobileApi(media_id, { cookie });
 
             // html embed (no cookie, cookie)
-            if (!data) data = await requestHTML(id);
-            if (!data && cookie) data = await requestHTML(id, cookie);
+            if (!hasData(data)) data = await requestHTML(id);
+            if (!hasData(data) && cookie) data = await requestHTML(id, cookie);
 
             // web app graphql api (no cookie, cookie)
-            if (!data) data = await requestGQL(id);
-            if (!data && cookie) data = await requestGQL(id, cookie);
+            if (!hasData(data)) data = await requestGQL(id);
+            if (!hasData(data) && cookie) data = await requestGQL(id, cookie);
         } catch {}
 
-        if (!data) return { error: "fetch.fail" };
+        if (!hasData(data)) {
+            return getErrorContext(id);
+        }
 
         if (data?.gql_data) {
             result = extractOldPost(data, id, alwaysProxy)
@@ -357,14 +510,30 @@ export default function(obj) {
         if (item.image_versions2?.candidates) {
             return {
                 urls: item.image_versions2.candidates[0].url,
-                isPhoto: true
+                isPhoto: true,
+                filename: `instagram_${id}.jpg`,
             }
         }
 
         return { error: "link.unsupported" };
     }
 
-    const { postId, storyId, username, alwaysProxy } = obj;
+    const { postId, shareId, storyId, username, alwaysProxy } = obj;
+
+    if (shareId) {
+        return resolveRedirectingURL(
+            `https://www.instagram.com/share/${shareId}/`,
+            dispatcher,
+            // for some reason instagram decides to return HTML
+            // instead of a redirect when requesting with a normal
+            // browser user-agent
+            {'User-Agent': 'curl/7.88.1'}
+        ).then(match => instagram({
+            ...obj, ...match,
+            shareId: undefined
+        }));
+    }
+
     if (postId) return getPost(postId, alwaysProxy);
     if (username && storyId) return getStory(username, storyId);
 

api/src/processing/services/loom.js

@@ -1,18 +1,18 @@
 import { genericUserAgent } from "../../config.js";
 
-export default async function({ id }) {
+const craftHeaders = id => ({
+    "user-agent": genericUserAgent,
+    "content-type": "application/json",
+    origin: "https://www.loom.com",
+    referer: `https://www.loom.com/share/${id}`,
+    cookie: `loom_referral_video=${id};`,
+    "x-loom-request-source": "loom_web_be851af",
+});
+
+async function fromTranscodedURL(id) {
     const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, {
         method: "POST",
-        headers: {
-            "user-agent": genericUserAgent,
-            origin: "https://www.loom.com",
-            referer: `https://www.loom.com/share/${id}`,
-            cookie: `loom_referral_video=${id};`,
-
-            "apollographql-client-name": "web",
-            "apollographql-client-version": "14c0b42",
-            "x-loom-request-source": "loom_web_14c0b42",
-        },
+        headers: craftHeaders(id),
         body: JSON.stringify({
             force_original: false,
             password: null,
@@ -20,20 +20,89 @@ export default async function({ id }) {
             deviceID: null
         })
     })
-    .then(r => r.status === 200 ? r.json() : false)
+    .then(r => r.status === 200 && r.json())
     .catch(() => {});
 
-    if (!gql) return { error: "fetch.empty" };
+    if (gql?.url?.includes('.mp4?')) {
+        return gql.url;
+    }
+}
 
-    const videoUrl = gql?.url;
+async function fromRawURL(id) {
+    const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/raw-url`, {
+        method: "POST",
+        headers: craftHeaders(id),
+        body: JSON.stringify({
+            anonID: crypto.randomUUID(),
+            client_name: "web",
+            client_version: "be851af",
+            deviceID: null,
+            force_original: false,
+            password: null,
+            supported_mime_types: ["video/mp4"],
+        })
+    })
+    .then(r => r.status === 200 && r.json())
+    .catch(() => {});
 
-    if (videoUrl?.includes('.mp4?')) {
-        return {
-            urls: videoUrl,
-            filename: `loom_${id}.mp4`,
-            audioFilename: `loom_${id}_audio`
-        }
+    if (gql?.url?.includes('.mp4?')) {
+        return gql.url;
+    }
+}
+
+async function getTranscript(id) {
+    const gql = await fetch(`https://www.loom.com/graphql`, {
+        method: "POST",
+        headers: craftHeaders(id),
+        body: JSON.stringify({
+            operationName: "FetchVideoTranscriptForFetchTranscript",
+            variables: {
+                videoId: id,
+                password: null,
+            },
+            query: `
+                query FetchVideoTranscriptForFetchTranscript($videoId: ID!, $password: String) {
+                    fetchVideoTranscript(videoId: $videoId, password: $password) {
+                        ... on VideoTranscriptDetails {
+                        captions_source_url
+                        language
+                        __typename
+                        }
+                        ... on GenericError {
+                        message
+                        __typename
+                        }
+                        __typename
+                    }
+                }`,
+        })
+    })
+    .then(r => r.status === 200 && r.json())
+    .catch(() => {});
+
+    if (gql?.data?.fetchVideoTranscript?.captions_source_url?.includes('.vtt?')) {
+        return gql.data.fetchVideoTranscript.captions_source_url;
+    }
+}
+
+export default async function({ id, subtitleLang }) {
+    let url = await fromTranscodedURL(id);
+    url ??= await fromRawURL(id);
+
+    if (!url) {
+        return { error: "fetch.empty" }
     }
 
-    return { error: "fetch.empty" }
+    let subtitles;
+    if (subtitleLang) {
+        const transcript = await getTranscript(id);
+        if (transcript) subtitles = transcript;
+    }
+
+    return {
+        urls: url,
+        subtitles,
+        filename: `loom_${id}.mp4`,
+        audioFilename: `loom_${id}_audio`
+    }
 }

api/src/processing/services/newgrounds.js

@@ -0,0 +1,103 @@
+import { genericUserAgent } from "../../config.js";
+
+const getVideo = async ({ id, quality }) => {
+    const json = await fetch(`https://www.newgrounds.com/portal/video/${id}`, {
+        headers: {
+            "User-Agent": genericUserAgent,
+            "X-Requested-With": "XMLHttpRequest", // required to get the JSON response
+        }
+    })
+    .then(r => r.json())
+    .catch(() => {});
+
+    if (!json) return { error: "fetch.empty" };
+
+    const videoSources = json.sources;
+    const videoQualities = Object.keys(videoSources);
+
+    if (videoQualities.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    const bestVideo = videoSources[videoQualities[0]]?.[0],
+          userQuality = quality === "2160" ? "4k" : `${quality}p`,
+          preferredVideo = videoSources[userQuality]?.[0],
+          video = preferredVideo || bestVideo,
+          videoQuality = preferredVideo ? userQuality : videoQualities[0];
+
+    if (!bestVideo || !video.type.includes("mp4")) {
+        return { error: "fetch.empty" };
+    }
+
+    const fileMetadata = {
+        title: decodeURIComponent(json.title),
+        artist: decodeURIComponent(json.author),
+    }
+
+    return {
+        urls: video.src,
+        filenameAttributes: {
+            service: "newgrounds",
+            id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+            extension: "mp4",
+            qualityLabel: videoQuality,
+            resolution: videoQuality,
+        },
+        fileMetadata,
+    }
+}
+
+const getMusic = async ({ id }) => {
+    const html = await fetch(`https://www.newgrounds.com/audio/listen/${id}`, {
+        headers: {
+            "User-Agent": genericUserAgent,
+        }
+    })
+    .then(r => r.text())
+    .catch(() => {});
+
+    if (!html) return { error: "fetch.fail" };
+
+    const params = JSON.parse(
+        `{${html.split(',"params":{')[1]?.split(',"images":')[0]}}`
+    );
+    if (!params) return { error: "fetch.empty" };
+
+    if (!params.name || !params.artist || !params.filename || !params.icon) {
+        return { error: "fetch.empty" };
+    }
+
+    const fileMetadata = {
+        title: decodeURIComponent(params.name),
+        artist: decodeURIComponent(params.artist),
+    }
+
+    return {
+        urls: params.filename,
+        filenameAttributes: {
+            service: "newgrounds",
+            id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+        },
+        fileMetadata,
+        cover:
+            params.icon.includes(".png?") || params.icon.includes(".jpg?")
+                ? params.icon
+                : undefined,
+        isAudioOnly: true,
+        bestAudio: "mp3",
+    }
+}
+
+export default function({ id, audioId, quality }) {
+    if (id) {
+        return getVideo({ id, quality });
+    } else if (audioId) {
+        return getMusic({ id: audioId });
+    }
+
+    return { error: "fetch.empty" };
+}

api/src/processing/services/ok.js

@@ -1,5 +1,4 @@
 import { genericUserAgent, env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 const resolutions = {
     "ultra": "2160",
@@ -44,8 +43,8 @@ export default async function(o) {
     let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];
 
     let fileMetadata = {
-        title: cleanString(videoData.movie.title.trim()),
-        author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()),
+        title: videoData.movie.title.trim(),
+        author: (videoData.author?.name || videoData.compilationTitle)?.trim(),
     }
 
     if (bestVideo) return {

api/src/processing/services/pinterest.js

@@ -1,16 +1,18 @@
 import { genericUserAgent } from "../../config.js";
+import { resolveRedirectingURL } from "../url.js";
 
 const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
 const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
+const notFoundRegex = /"__typename"\s*:\s*"PinNotFound"/;
 
 export default async function(o) {
     let id = o.id;
 
     if (!o.id && o.shortLink) {
-        id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" })
-                   .then(r => r.headers.get("location").split('pin/')[1].split('/')[0])
-                   .catch(() => {});
+        const patternMatch = await resolveRedirectingURL(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`);
+        id = patternMatch?.id;
     }
+
     if (id.includes("--")) id = id.split("--")[1];
     if (!id) return { error: "fetch.fail" };
 
@@ -18,16 +20,20 @@ export default async function(o) {
         headers: { "user-agent": genericUserAgent }
     }).then(r => r.text()).catch(() => {});
 
+    const invalidPin = html.match(notFoundRegex);
+
+    if (invalidPin) return { error: "fetch.empty" };
+
     if (!html) return { error: "fetch.fail" };
 
     const videoLink = [...html.matchAll(videoRegex)]
                     .map(([, link]) => link)
-                    .find(a => a.endsWith('.mp4') && a.includes('720p'));
+                    .find(a => a.endsWith('.mp4'));
 
     if (videoLink) return {
         urls: videoLink,
-        filename: `pinterest_${o.id}.mp4`,
-        audioFilename: `pinterest_${o.id}_audio`
+        filename: `pinterest_${id}.mp4`,
+        audioFilename: `pinterest_${id}_audio`
     }
 
     const imageLink = [...html.matchAll(imageRegex)]
@@ -39,7 +45,7 @@ export default async function(o) {
     if (imageLink) return {
         urls: imageLink,
         isPhoto: true,
-        filename: `pinterest_${o.id}.${imageType}`
+        filename: `pinterest_${id}.${imageType}`
     }
 
     return { error: "fetch.empty" };

api/src/processing/services/reddit.js

@@ -1,3 +1,4 @@
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent, env } from "../../config.js";
 import { getCookie, updateCookieValues } from "../cookie/manager.js";
 
@@ -48,23 +49,36 @@ async function getAccessToken() {
 }
 
 export default async function(obj) {
-    let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
+    let params = obj;
+    const accessToken = await getAccessToken();
+    const headers = {
+        'user-agent': genericUserAgent,
+        authorization: accessToken && `Bearer ${accessToken}`,
+        accept: 'application/json'
+    };
 
-    if (obj.user) {
-        url.pathname = `/user/${obj.user}/comments/${obj.id}.json`;
+    if (params.shortId) {
+        params = await resolveRedirectingURL(
+            `https://www.reddit.com/video/${params.shortId}`,
+            obj.dispatcher, headers
+        );
     }
 
-    const accessToken = await getAccessToken();
+    if (!params.id && params.shareId) {
+        params = await resolveRedirectingURL(
+            `https://www.reddit.com/r/${params.sub}/s/${params.shareId}`,
+            obj.dispatcher, headers
+        );
+    }
+
+    if (!params?.id) return { error: "fetch.short_link" };
+
+    const url = new URL(`https://www.reddit.com/comments/${params.id}.json`);
+
     if (accessToken) url.hostname = 'oauth.reddit.com';
 
     let data = await fetch(
-        url, {
-            headers: {
-                'User-Agent': genericUserAgent,
-                accept: 'application/json',
-                authorization: accessToken && `Bearer ${accessToken}`
-            }
-        }
+        url, { headers }
     ).then(r => r.json()).catch(() => {});
 
     if (!data || !Array.isArray(data)) {
@@ -73,12 +87,17 @@ export default async function(obj) {
 
     data = data[0]?.data?.children[0]?.data;
 
-    const id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
+    let sourceId;
+    if (params.sub || params.user) {
+        sourceId = `${String(params.sub || params.user).toLowerCase()}_${params.id}`;
+    } else {
+        sourceId = params.id;
+    }
 
     if (data?.url?.endsWith('.gif')) return {
         typeId: "redirect",
         urls: data.url,
-        filename: `reddit_${id}.gif`,
+        filename: `reddit_${sourceId}.gif`,
     }
 
     if (!data.secure_media?.reddit_video)
@@ -87,8 +106,9 @@ export default async function(obj) {
     if (data.secure_media?.reddit_video?.duration > env.durationLimit)
         return { error: "content.too_long" };
 
+    const video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0];
+
     let audio = false,
-        video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
         audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`;
 
     if (video.match('.mp4')) {
@@ -121,7 +141,7 @@ export default async function(obj) {
         typeId: "tunnel",
         type: "merge",
         urls: [video, audioFileLink],
-        audioFilename: `reddit_${id}_audio`,
-        filename: `reddit_${id}.mp4`
+        audioFilename: `reddit_${sourceId}_audio`,
+        filename: `reddit_${sourceId}.mp4`
     }
 }

api/src/processing/services/rutube.js

@@ -1,7 +1,5 @@
 import HLS from "hls-parser";
-
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 async function requestJSON(url) {
     try {
@@ -35,6 +33,10 @@ export default async function(obj) {
     const play = await requestJSON(requestURL);
     if (!play) return { error: "fetch.fail" };
 
+    if (play.detail?.type === "blocking_rule") {
+        return { error: "content.video.region" };
+    }
+
     if (play.detail || !play.video_balancer) return { error: "fetch.empty" };
     if (play.live_streams?.hls) return { error: "content.video.live" };
 
@@ -59,13 +61,26 @@ export default async function(obj) {
     });
 
     const fileMetadata = {
-        title: cleanString(play.title.trim()),
-        artist: cleanString(play.author.name.trim()),
+        title: play.title.trim(),
+        artist: play.author.name.trim(),
+    }
+
+    let subtitles;
+    if (obj.subtitleLang && play.captions?.length) {
+        const subtitle = play.captions.find(
+            s => ["webvtt", "srt"].includes(s.format) && s.code.startsWith(obj.subtitleLang)
+        );
+
+        if (subtitle) {
+            subtitles = subtitle.file;
+            fileMetadata.sublanguage = obj.subtitleLang;
+        }
     }
 
     return {
         urls: matchingQuality.uri,
-        isM3U8: true,
+        subtitles,
+        isHLS: true,
         filenameAttributes: {
             service: "rutube",
             id: obj.id,

api/src/processing/services/snapchat.js

@@ -1,7 +1,6 @@
-import { extract, normalizeURL } from "../url.js";
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
-import { getRedirectingURL } from "../../misc/utils.js";
 
 const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="([^"]+)" as="video"\/>/;
 const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
@@ -41,9 +40,9 @@ async function getStory(username, storyId, alwaysProxy) {
     const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
     if (nextDataString) {
         const data = JSON.parse(nextDataString);
-        const storyIdParam = data.query.profileParams[1];
+        const storyIdParam = data?.query?.profileParams?.[1];
 
-        if (storyIdParam && data.props.pageProps.story) {
+        if (storyIdParam && data?.props?.pageProps?.story) {
             const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
             if (story) {
                 if (story.snapMediaType === 0) {
@@ -62,7 +61,7 @@ async function getStory(username, storyId, alwaysProxy) {
             }
         }
 
-        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        const defaultStory = data?.props?.pageProps?.curatedHighlights?.[0];
         if (defaultStory) {
             return {
                 picker: defaultStory.snapList.map(snap => {
@@ -73,7 +72,7 @@ async function getStory(username, storyId, alwaysProxy) {
                     const proxy = createStream({
                         service: "snapchat",
                         type: "proxy",
-                        u: snapUrl,
+                        url: snapUrl,
                         filename: `snapchat_${username}_${snap.timestampInSec.value}.${snapExt}`,
                     });
 
@@ -81,7 +80,7 @@ async function getStory(username, storyId, alwaysProxy) {
                     if (snapType === "video") thumbProxy = createStream({
                         service: "snapchat",
                         type: "proxy",
-                        u: snap.snapUrls.mediaPreviewUrl.value,
+                        url: snap.snapUrls.mediaPreviewUrl.value,
                     });
 
                     if (alwaysProxy) snapUrl = proxy;
@@ -100,24 +99,13 @@ async function getStory(username, storyId, alwaysProxy) {
 export default async function (obj) {
     let params = obj;
     if (obj.shortLink) {
-        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
-
-        if (!link?.startsWith('https://www.snapchat.com/')) {
-            return { error: "fetch.short_link" };
-        }
-
-        const extractResult = extract(normalizeURL(link));
-        if (extractResult?.host !== 'snapchat') {
-            return { error: "fetch.short_link" };
-        }
-
-        params = extractResult.patternMatch;
+        params = await resolveRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
     }
 
-    if (params.spotlightId) {
+    if (params?.spotlightId) {
         const result = await getSpotlight(params.spotlightId);
         if (result) return result;
-    } else if (params.username) {
+    } else if (params?.username) {
         const result = await getStory(params.username, params.storyId, obj.alwaysProxy);
         if (result) return result;
     }

api/src/processing/services/soundcloud.js

@@ -1,5 +1,5 @@
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
+import { resolveRedirectingURL } from "../url.js";
 
 const cachedID = {
     version: '',
@@ -8,22 +8,25 @@ const cachedID = {
 
 async function findClientID() {
     try {
-        let sc = await fetch('https://soundcloud.com/').then(r => r.text()).catch(() => {});
-        let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
+        const sc = await fetch('https://soundcloud.com/').then(r => r.text()).catch(() => {});
+        const scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
 
-        if (cachedID.version === scVersion) return cachedID.id;
+        if (cachedID.version === scVersion) {
+            return cachedID.id;
+        }
+
+        const scripts = sc.matchAll(/<script.+src="(.+)">/g);
 
-        let scripts = sc.matchAll(/<script.+src="(.+)">/g);
         let clientid;
         for (let script of scripts) {
-            let url = script[1];
+            const url = script[1];
 
             if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
                 return;
             }
 
-            let scrf = await fetch(url).then(r => r.text()).catch(() => {});
-            let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
+            const scrf = await fetch(url).then(r => r.text()).catch(() => {});
+            const id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
 
             if (id && typeof id[0] === 'string') {
                 clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
@@ -37,37 +40,79 @@ async function findClientID() {
     } catch {}
 }
 
+const findBestForPreset = (transcodings, preset) => {
+    let inferior;
+    for (const entry of transcodings) {
+        const protocol = entry?.format?.protocol;
+
+        if (entry.snipped || protocol?.includes('encrypted')) {
+            continue;
+        }
+
+        if (entry?.preset?.startsWith(`${preset}_`)) {
+            if (protocol === 'progressive') {
+                return entry;
+            }
+
+            inferior = entry;
+        }
+    }
+
+    return inferior;
+}
+
 export default async function(obj) {
-    let clientId = await findClientID();
+    const clientId = await findClientID();
     if (!clientId) return { error: "fetch.fail" };
 
     let link;
-    if (obj.url.hostname === 'on.soundcloud.com' && obj.shortLink) {
-        link = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`, { redirect: "manual" }).then(r => {
-            if (r.status === 302 && r.headers.get("location").startsWith("https://soundcloud.com/")) {
-                return r.headers.get("location").split('?', 1)[0]
-            }
-        }).catch(() => {});
+
+    if (obj.shortLink) {
+        obj = {
+            ...obj,
+            ...await resolveRedirectingURL(
+                `https://on.soundcloud.com/${obj.shortLink}`
+            )
+        }
     }
 
-    if (!link && obj.author && obj.song) {
-        link = `https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`
+    if (obj.author && obj.song) {
+        link = `https://soundcloud.com/${obj.author}/${obj.song}`;
+        if (obj.accessKey) {
+            link += `/s-${obj.accessKey}`;
+        }
     }
 
     if (!link && obj.shortLink) return { error: "fetch.short_link" };
     if (!link) return { error: "link.unsupported" };
 
-    let json = await fetch(`https://api-v2.soundcloud.com/resolve?url=${link}&client_id=${clientId}`)
-                     .then(r => r.status === 200 ? r.json() : false)
-                     .catch(() => {});
+    const resolveURL = new URL("https://api-v2.soundcloud.com/resolve");
+    resolveURL.searchParams.set("url", link);
+    resolveURL.searchParams.set("client_id", clientId);
 
+    const json = await fetch(resolveURL).then(r => r.json()).catch(() => {});
     if (!json) return { error: "fetch.fail" };
 
-    if (!json.media.transcodings) return { error: "fetch.empty" };
+    if (json.duration > env.durationLimit * 1000) {
+        return { error: "content.too_long" };
+    }
+
+    if (json.policy === "BLOCK") {
+        return { error: "content.region" };
+    }
+
+    if (json.policy === "SNIP") {
+        return { error: "content.paid" };
+    }
+
+    if (!json.media?.transcodings || !json.media?.transcodings.length === 0) {
+        return { error: "fetch.empty" };
+    }
 
     let bestAudio = "opus",
-        selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"),
-        mp3Media = json.media.transcodings.find(v => v.preset === "mp3_0_0");
+        selectedStream = findBestForPreset(json.media.transcodings, "opus");
+
+    const mp3Media = findBestForPreset(json.media.transcodings, "mp3");
 
     // use mp3 if present if user prefers it or if opus isn't available
     if (mp3Media && (obj.format === "mp3" || !selectedStream)) {
@@ -75,35 +120,54 @@ export default async function(obj) {
         bestAudio = "mp3"
     }
 
-    let fileUrlBase = selectedStream.url;
-    let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
-
-    if (!fileUrl.startsWith("https://api-v2.soundcloud.com/media/soundcloud:tracks:"))
+    if (!selectedStream) {
         return { error: "fetch.empty" };
-
-    if (json.duration > env.durationLimit * 1000) {
-        return { error: "content.too_long" };
     }
 
-    let file = await fetch(fileUrl)
-                     .then(async r => (await r.json()).url)
+    const fileUrl = new URL(selectedStream.url);
+    fileUrl.searchParams.set("client_id", clientId);
+    fileUrl.searchParams.set("track_authorization", json.track_authorization);
+
+    const file = await fetch(fileUrl)
+                     .then(async r => new URL((await r.json()).url))
                      .catch(() => {});
+
     if (!file) return { error: "fetch.empty" };
 
-    let fileMetadata = {
-        title: cleanString(json.title.trim()),
-        artist: cleanString(json.user.username.trim()),
+    const artist = json.user?.username?.trim();
+    const fileMetadata = {
+        title: json.title?.trim(),
+        album: json.publisher_metadata?.album_title?.trim(),
+        artist,
+        album_artist: artist,
+        composer: json.publisher_metadata?.writer_composer?.trim(),
+        genre: json.genre?.trim(),
+        date: json.display_date?.trim().slice(0, 10),
+        copyright: json.license?.trim(),
+    }
+
+    let cover;
+    if (json.artwork_url) {
+        const coverUrl = json.artwork_url.replace(/-large/, "-t1080x1080");
+        const testCover = await fetch(coverUrl)
+            .then(r => r.status === 200)
+            .catch(() => {});
+
+        if (testCover) {
+            cover = coverUrl;
+        }
     }
 
     return {
-        urls: file,
+        urls: file.toString(),
+        cover,
         filenameAttributes: {
             service: "soundcloud",
             id: json.id,
-            title: fileMetadata.title,
-            author: fileMetadata.artist
+            ...fileMetadata
         },
         bestAudio,
-        fileMetadata
+        fileMetadata,
+        isHLS: file.pathname.endsWith('.m3u8'),
     }
 }

api/src/processing/services/tiktok.js

@@ -1,9 +1,10 @@
 import Cookie from "../cookie/cookie.js";
 
-import { extract } from "../url.js";
+import { extract, normalizeURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { updateCookie } from "../cookie/manager.js";
 import { createStream } from "../../stream/manage.js";
+import { convertLanguageCode } from "../../misc/language-codes.js";
 
 const shortDomain = "https://vt.tiktok.com/";
 
@@ -12,7 +13,7 @@ export default async function(obj) {
     let postId = obj.postId;
 
     if (!postId) {
-        let html = await fetch(`${shortDomain}${obj.id}`, {
+        let html = await fetch(`${shortDomain}${obj.shortLink}`, {
             redirect: "manual",
             headers: {
                 "user-agent": genericUserAgent.split(' Chrome/1')[0]
@@ -23,14 +24,16 @@ export default async function(obj) {
 
         if (html.startsWith('<a href="https://')) {
             const extractedURL = html.split('<a href="')[1].split('?')[0];
-            const { patternMatch } = extract(extractedURL);
-            postId = patternMatch.postId
+            const { host, patternMatch } = extract(normalizeURL(extractedURL));
+            if (host === "tiktok") {
+                postId = patternMatch?.postId;
+            }
         }
     }
     if (!postId) return { error: "fetch.short_link" };
 
     // should always be /video/, even for photos
-    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
+    const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, {
         headers: {
             "user-agent": genericUserAgent,
             cookie,
@@ -44,20 +47,39 @@ export default async function(obj) {
     try {
         const json = html
             .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
-            .split('</script>')[0]
-        const data = JSON.parse(json)
-        detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+            .split('</script>')[0];
+
+        const data = JSON.parse(json);
+        const videoDetail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"];
+
+        if (!videoDetail) throw "no video detail found";
+
+        // status_deleted or etc
+        if (videoDetail.statusMsg) {
+            return { error: "content.post.unavailable"};
+        }
+
+        detail = videoDetail?.itemInfo?.itemStruct;
     } catch {
         return { error: "fetch.fail" };
     }
 
+    if (detail.isContentClassified) {
+        return { error: "content.post.age" };
+    }
+
+    if (!detail.author) {
+        return { error: "fetch.empty" };
+    }
+
     let video, videoFilename, audioFilename, audio, images,
-        filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`,
+        filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`,
         bestAudio; // will get defaulted to m4a later on in match-action
 
     images = detail.imagePost?.images;
 
-    let playAddr = detail.video.playAddr;
+    let playAddr = detail.video?.playAddr;
+
     if (obj.h265) {
         const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
         playAddr = h265PlayAddr || playAddr
@@ -78,8 +100,23 @@ export default async function(obj) {
     }
 
     if (video) {
+        let subtitles, fileMetadata;
+        if (obj.subtitleLang && detail?.video?.subtitleInfos?.length) {
+            const langCode = convertLanguageCode(obj.subtitleLang);
+            const subtitle = detail?.video?.subtitleInfos.find(
+                s => s.LanguageCodeName.startsWith(langCode) && s.Format === "webvtt"
+            )
+            if (subtitle) {
+                subtitles = subtitle.Url;
+                fileMetadata = {
+                    sublanguage: langCode,
+                }
+            }
+        }
         return {
             urls: video,
+            subtitles,
+            fileMetadata,
             filename: videoFilename,
             headers: { cookie }
         }
@@ -102,7 +139,7 @@ export default async function(obj) {
                 if (obj.alwaysProxy) url = createStream({
                     service: "tiktok",
                     type: "proxy",
-                    u: url,
+                    url,
                     filename: `${filenameBase}_photo_${i + 1}.jpg`
                 })
 
@@ -131,4 +168,6 @@ export default async function(obj) {
             headers: { cookie }
         }
     }
+
+    return { error: "fetch.empty" };
 }

api/src/processing/services/tumblr.js

@@ -1,4 +1,4 @@
-import psl from "psl";
+import psl from "@imput/psl";
 
 const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z';
 const API_BASE = 'https://api-http2.tumblr.com';

api/src/processing/services/twitch.js

@@ -1,5 +1,4 @@
 import { env } from "../../config.js";
-import { cleanString } from '../../misc/utils.js';
 
 const gqlURL = "https://gql.twitch.tv/gql";
 const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" };
@@ -73,13 +72,13 @@ export default async function (obj) {
             token: req_token[0].data.clip.playbackAccessToken.value
         })}`,
         fileMetadata: {
-            title: cleanString(clipMetadata.title.trim()),
+            title: clipMetadata.title.trim(),
             artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`,
         },
         filenameAttributes: {
             service: "twitch",
             id: clipMetadata.id,
-            title: cleanString(clipMetadata.title.trim()),
+            title: clipMetadata.title.trim(),
             author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`,
             qualityLabel: `${format.quality}p`,
             extension: 'mp4'

api/src/processing/services/twitter.js

@@ -1,13 +1,14 @@
+import HLS from "hls-parser";
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
 
-const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';
+const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail';
 const tokenURL = 'https://api.x.com/1.1/guest/activate.json';
 
-const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});
+const tweetFeatures = JSON.stringify({"rweb_video_screen_enabled":false,"payments_enabled":false,"rweb_xchat_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false});
 
-const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});
+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false});
 
 const commonHeaders = {
     "user-agent": genericUserAgent,
@@ -24,6 +25,11 @@ const badContainerEnd = new Date(1702605600000);
 
 function needsFixing(media) {
     const representativeId = media.source_status_id_str ?? media.id_str;
+
+    // syndication api doesn't have media ids in its response,
+    // so we just assume it's all good
+    if (!representativeId) return false;
+
     const mediaTimestamp = new Date(
         Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
     );
@@ -53,6 +59,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => {
     }
 }
 
+const requestSyndication = async(dispatcher, tweetId) => {
+    // thank you
+    // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334
+    const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '');
+    const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result");
+
+    syndicationUrl.searchParams.set("id", tweetId);
+    syndicationUrl.searchParams.set("token", token(tweetId));
+
+    const result = await fetch(syndicationUrl, {
+        headers: {
+            "user-agent": genericUserAgent
+        },
+        dispatcher
+    });
+
+    return result;
+}
+
 const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     const graphqlTweetURL = new URL(graphqlURL);
 
@@ -75,10 +100,14 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
 
     graphqlTweetURL.searchParams.set('variables',
         JSON.stringify({
-            tweetId,
-            withCommunity: false,
-            includePromotedContent: false,
-            withVoice: false
+            focalTweetId: tweetId,
+            with_rux_injections: false,
+            rankingMode: "Relevance",
+            includePromotedContent: true,
+            withCommunity: true,
+            withQuickPromoteEligibilityTweetFields: true,
+            withBirdwatchNotes: true,
+            withVoice: true
         })
     );
     graphqlTweetURL.searchParams.set('features', tweetFeatures);
@@ -87,53 +116,65 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     let result = await fetch(graphqlTweetURL, { headers, dispatcher });
     updateCookie(cookie, result.headers);
 
-    // we might have been missing the `ct0` cookie, retry
+    // we might have been missing the ct0 cookie, retry
     if (result.status === 403 && result.headers.get('set-cookie')) {
-        result = await fetch(graphqlTweetURL, {
-            headers: {
-                ...headers,
-                'x-csrf-token': cookie.values().ct0
-            },
-            dispatcher
-        });
+        const cookieValues = cookie?.values();
+        if (cookieValues?.ct0) {
+            result = await fetch(graphqlTweetURL, {
+                headers: {
+                    ...headers,
+                    'x-csrf-token': cookieValues.ct0
+                },
+                dispatcher
+            });
+        }
     }
 
     return result
 }
 
-export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
-    const cookie = await getCookie('twitter');
+const parseCard = (cardOuter) => {
+    const card = JSON.parse(
+      (cardOuter?.legacy?.binding_values[0].value
+       || cardOuter?.binding_values?.unified_card)?.string_value,
+    );
 
-    let guestToken = await getGuestToken(dispatcher);
-    if (!guestToken) return { error: "fetch.fail" };
-
-    let tweet = await requestTweet(dispatcher, id, guestToken);
-
-    // get new token & retry if old one expired
-    if ([403, 429].includes(tweet.status)) {
-        guestToken = await getGuestToken(dispatcher, true);
-        tweet = await requestTweet(dispatcher, id, guestToken)
+    if (!["video_website", "image_website"].includes(card?.type)
+        || !card?.media_entities
+        || card?.component_objects?.media_1?.type !== "media") {
+      return;
     }
 
-    tweet = await tweet.json();
+    const mediaId = card.component_objects?.media_1?.data?.id;
+    return [card.media_entities[mediaId]];
+};
 
-    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) => {
+    const addInsn = thread?.data?.threaded_conversation_with_injections_v2?.instructions?.find(
+        insn => insn.type === 'TimelineAddEntries'
+    );
+
+    const tweetResult = addInsn?.entries?.find(
+        entry => entry.entryId === `tweet-${id}`
+    )?.content?.itemContent?.tweet_results?.result;
+
+    let tweetTypename = tweetResult?.__typename;
 
     if (!tweetTypename) {
         return { error: "fetch.empty" }
     }
 
-    if (tweetTypename === "TweetUnavailable") {
-        const reason = tweet?.data?.tweetResult?.result?.reason;
-        switch(reason) {
-            case "Protected":
-                return { error: "content.post.private" }
-            case "NsfwLoggedOut":
-                if (cookie) {
-                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
-                    tweet = await tweet.json();
-                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-                } else return { error: "content.post.age" }
+    if (tweetTypename === "TweetUnavailable" || tweetTypename === "TweetTombstone") {
+        const reason = tweetResult?.result?.reason;
+        if (reason === 'Protected') {
+            return { error: "content.post.private" };
+        } else if (reason === "NsfwLoggedOut" || tweetResult?.tombstone?.text?.text?.startsWith('Age-restricted')) {
+            if (!cookie) {
+                return { error: "content.post.age" };
+            }
+
+            const tweet = await requestTweet(dispatcher, id, guestToken, cookie).then(t => t.json());
+            return extractGraphqlMedia(tweet, dispatcher, id, guestToken);
         }
     }
 
@@ -141,8 +182,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         return { error: "content.post.unavailable" }
     }
 
-    let tweetResult = tweet.data.tweetResult.result,
-        baseTweet = tweetResult.legacy,
+    let baseTweet = tweetResult.legacy,
         repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
 
     if (tweetTypename === "TweetWithVisibilityResults") {
@@ -150,7 +190,52 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
     }
 
-    let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
+    if (tweetResult.card?.legacy?.binding_values?.length) {
+        return parseCard(tweetResult.card);
+    }
+
+    return (repostedTweet?.media || baseTweet?.extended_entities?.media);
+}
+
+export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) {
+    const cookie = await getCookie('twitter');
+
+    let guestToken = await getGuestToken(dispatcher);
+    if (!guestToken) return { error: "fetch.fail" };
+
+    let tweet = await requestTweet(dispatcher, id, guestToken);
+
+    if ([403, 404, 429].includes(tweet.status)) {
+        // get new token & retry if old one expired
+        if ([403, 429].includes(tweet.status)) {
+            guestToken = await getGuestToken(dispatcher, true);
+        }
+        tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+    }
+
+    let media;
+    try {
+        tweet = await tweet.json();
+        media = await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+    } catch {}
+
+    // if graphql requests fail, then resort to tweet embed api
+    if (!media || 'error' in media) {
+        try {
+            tweet = await requestSyndication(dispatcher, id);
+            tweet = await tweet.json();
+
+            if (tweet?.card) {
+                media = parseCard(tweet.card);
+            }
+        } catch {}
+
+        media = tweet?.mediaDetails ?? media;
+    }
+
+    if (!media || 'error' in media) {
+        return { error: media?.error || "fetch.empty" };
+    }
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {
@@ -159,11 +244,35 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
 
     const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1];
 
-    const proxyMedia = (u, filename) => createStream({
+    const proxyMedia = (url, filename) => createStream({
         service: "twitter",
         type: "proxy",
-        u, filename,
-    })
+        url, filename,
+    });
+
+    const extractSubtitles = async (hlsUrl) => {
+        const mainHls = await fetch(hlsUrl).then(r => r.text()).catch(() => {});
+        if (!mainHls) return;
+
+        const subtitle = HLS.parse(mainHls)?.variants[0]?.subtitles?.find(
+            s => s.language.startsWith(subtitleLang)
+        );
+        if (!subtitle) return;
+
+        const subtitleUrl = new URL(subtitle.uri, hlsUrl).toString();
+        const subtitleHls = await fetch(subtitleUrl).then(r => r.text());
+        if (!subtitleHls) return;
+
+        const finalSubtitlePath = HLS.parse(subtitleHls)?.segments?.[0].uri;
+        if (!finalSubtitlePath) return;
+
+        const finalSubtitleUrl = new URL(finalSubtitlePath, hlsUrl).toString();
+
+        return {
+            url: finalSubtitleUrl,
+            language: subtitle.language,
+        };
+    }
 
     switch (media?.length) {
         case undefined:
@@ -172,21 +281,37 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
                 error: "fetch.empty"
             }
         case 1:
-            if (media[0].type === "photo") {
+            const mediaItem = media[0];
+            if (mediaItem.type === "photo") {
                 return {
                     type: "proxy",
                     isPhoto: true,
-                    filename: `twitter_${id}.${getFileExt(media[0].media_url_https)}`,
-                    urls: `${media[0].media_url_https}?name=4096x4096`
+                    filename: `twitter_${id}.${getFileExt(mediaItem.media_url_https)}`,
+                    urls: `${mediaItem.media_url_https}?name=4096x4096`
+                }
+            }
+
+            let subtitles;
+            let fileMetadata;
+            if (mediaItem.type === "video" && subtitleLang) {
+                const hlsVariant = mediaItem.video_info?.variants?.find(
+                    v => v.content_type === "application/x-mpegURL"
+                );
+                if (hlsVariant) {
+                    const { url, language } = await extractSubtitles(hlsVariant.url) || {};
+                    subtitles = url;
+                    if (language) fileMetadata = { sublanguage: language };
                 }
             }
 
             return {
-                type: needsFixing(media[0]) ? "remux" : "proxy",
-                urls: bestQuality(media[0].video_info.variants),
+                type: subtitles || needsFixing(mediaItem) ? "remux" : "proxy",
+                urls: bestQuality(mediaItem.video_info.variants),
                 filename: `twitter_${id}.mp4`,
                 audioFilename: `twitter_${id}_audio`,
-                isGif: media[0].type === "animated_gif"
+                isGif: mediaItem.type === "animated_gif",
+                subtitles,
+                fileMetadata,
             }
         default:
             const proxyThumb = (url, i) =>
@@ -208,7 +333,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
 
                 let url = bestQuality(content.video_info.variants);
                 const shouldRenderGif = content.type === "animated_gif" && toGif;
-                const videoFilename = `twitter_${id}_${i + 1}.mp4`;
+                const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`;
 
                 let type = "video";
                 if (shouldRenderGif) type = "gif";
@@ -217,7 +342,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
                     url = createStream({
                         service: "twitter",
                         type: shouldRenderGif ? "gif" : "remux",
-                        u: url,
+                        url,
                         filename: videoFilename,
                     })
                 } else if (alwaysProxy) {

api/src/processing/services/vimeo.js

@@ -1,7 +1,7 @@
 import HLS from "hls-parser";
-
 import { env } from "../../config.js";
-import { cleanString, merge } from '../../misc/utils.js';
+import { merge } from '../../misc/utils.js';
+import { getCookie } from "../cookie/manager.js";
 
 const resolutionMatch = {
     "3840": 2160,
@@ -16,7 +16,44 @@ const resolutionMatch = {
     "426": 240
 }
 
-const requestApiInfo = (videoId, password) => {
+const genericHeaders = {
+    Accept: 'application/vnd.vimeo.*+json; version=3.4.10',
+    'User-Agent': 'com.vimeo.android.videoapp (Google, Pixel 7a, google, Android 16/36 Version 11.8.1) Kotlin VimeoNetworking/3.12.0',
+    Authorization: 'Basic NzRmYTg5YjgxMWExY2JiNzUwZDg1MjhkMTYzZjQ4YWYyOGEyZGJlMTp4OGx2NFd3QnNvY1lkamI2UVZsdjdDYlNwSDUrdm50YzdNNThvWDcwN1JrenJGZC9tR1lReUNlRjRSVklZeWhYZVpRS0tBcU9YYzRoTGY2Z1dlVkJFYkdJc0dMRHpoZWFZbU0reDRqZ1dkZ1diZmdIdGUrNUM5RVBySlM0VG1qcw==',
+    'Accept-Language': 'en',
+}
+
+let bearer = '';
+
+const getBearer = async (refresh = false) => {
+    const cookie = getCookie('vimeo_bearer')?.values?.()?.access_token;
+    if ((bearer || cookie) && !refresh) return bearer || cookie;
+
+    const oauthResponse = await fetch(
+        'https://api.vimeo.com/oauth/authorize/client',
+        {
+            method: 'POST',
+            body: new URLSearchParams({
+                scope: 'private public create edit delete interact upload purchased stats',
+                grant_type: 'client_credentials',
+            }).toString(),
+            headers: {
+                ...genericHeaders,
+                'Content-Type': 'application/x-www-form-urlencoded',
+            }
+        }
+    )
+    .then(a => a.json())
+    .catch(() => {});
+
+    if (!oauthResponse || !oauthResponse.access_token) {
+        return;
+    }
+
+    return bearer = oauthResponse.access_token;
+}
+
+const requestApiInfo = (bearerToken, videoId, password) => {
     if (password) {
         videoId += `:${password}`
     }
@@ -25,10 +62,8 @@ const requestApiInfo = (videoId, password) => {
         `https://api.vimeo.com/videos/${videoId}`,
         {
             headers: {
-                Accept: 'application/vnd.vimeo.*+json; version=3.4.2',
-                'User-Agent': 'Vimeo/10.19.0 (com.vimeo; build:101900.57.0; iOS 17.5.1) Alamofire/5.9.0 VimeoNetworking/5.0.0',
-                Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
-                'Accept-Language': 'en'
+                ...genericHeaders,
+                Authorization: `Bearer ${bearerToken}`,
             }
         }
     )
@@ -41,7 +76,7 @@ const compareQuality = (rendition, requestedQuality) => {
     return Math.abs(quality - requestedQuality);
 }
 
-const getDirectLink = (data, quality) => {
+const getDirectLink = async (data, quality, subtitleLang) => {
     if (!data.files) return;
 
     const match = data.files
@@ -57,8 +92,23 @@ const getDirectLink = (data, quality) => {
 
     if (!match) return;
 
+    let subtitles;
+    if (subtitleLang && data.config_url) {
+        const config = await fetch(data.config_url)
+                    .then(r => r.json())
+                    .catch(() => {});
+
+        if (config && config.request?.text_tracks?.length) {
+            subtitles = config.request.text_tracks.find(
+                t => t.lang.startsWith(subtitleLang)
+            );
+            subtitles = new URL(subtitles.url, "https://player.vimeo.com/").toString();
+        }
+    }
+
     return {
         urls: match.link,
+        subtitles,
         filenameAttributes: {
             resolution: `${match.width}x${match.height}`,
             qualityLabel: match.rendition,
@@ -122,7 +172,7 @@ const getHLS = async (configURL, obj) => {
 
     return {
         urls,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
             qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
@@ -137,14 +187,33 @@ export default async function(obj) {
     if (quality < 240) quality = 240;
     if (!quality || obj.isAudioOnly) quality = 9000;
 
-    const info = await requestApiInfo(obj.id, obj.password);
+    const bearerToken = await getBearer();
+    if (!bearerToken) {
+        return { error: "fetch.fail" };
+    }
+
+    let info = await requestApiInfo(bearerToken, obj.id, obj.password);
     let response;
 
+    // auth error, try to refresh the token
+    if (info?.error_code === 8003) {
+        const newBearer = await getBearer(true);
+        if (!newBearer) {
+            return { error: "fetch.fail" };
+        }
+        info = await requestApiInfo(newBearer, obj.id, obj.password);
+    }
+
+    // if there's still no info, then return a generic error
+    if (!info || info.error_code) {
+        return { error: "fetch.empty" };
+    }
+
     if (obj.isAudioOnly) {
         response = await getHLS(info.config_url, { ...obj, quality });
     }
 
-    if (!response) response = getDirectLink(info, quality);
+    if (!response) response = await getDirectLink(info, quality, obj.subtitleLang);
     if (!response) response = { error: "fetch.empty" };
 
     if (response.error) {
@@ -152,10 +221,14 @@ export default async function(obj) {
     }
 
     const fileMetadata = {
-        title: cleanString(info.name),
-        artist: cleanString(info.user.name),
+        title: info.name,
+        artist: info.user.name,
     };
 
+    if (response.subtitles) {
+        fileMetadata.sublanguage = obj.subtitleLang;
+    }
+
     return merge(
         {
             fileMetadata,

api/src/processing/services/vine.js

@@ -1,15 +0,0 @@
-export default async function(obj) {
-    let post = await fetch(`https://archive.vine.co/posts/${obj.id}.json`)
-                    .then(r => r.json())
-                    .catch(() => {});
-
-    if (!post) return { error: "fetch.empty" };
-
-    if (post.videoUrl) return {
-        urls: post.videoUrl.replace("http://", "https://"),
-        filename: `vine_${obj.id}.mp4`,
-        audioFilename: `vine_${obj.id}_audio`
-    }
-
-    return { error: "fetch.empty" }
-}

api/src/processing/services/vk.js

@@ -1,63 +1,152 @@
-import { cleanString } from "../../misc/utils.js";
-import { genericUserAgent, env } from "../../config.js";
+import { env } from "../../config.js";
 
-const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"];
+const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"];
 
-export default async function(o) {
-    let html, url, quality = o.quality === "max" ? 2160 : o.quality;
+const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token";
+const apiUrl = "https://api.vk.com/method";
 
-    html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, {
-        headers: {
-            "user-agent": genericUserAgent
-        }
-    })
-    .then(r => r.arrayBuffer())
-    .catch(() => {});
+const clientId = "51552953";
+const clientSecret = "qgr0yWwXCrsxA1jnRtRX";
 
-    if (!html) return { error: "fetch.fail" };
+// used in stream/shared.js for accessing media files
+export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119";
 
-    // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times
-    let decoder = new TextDecoder('windows-1251');
-    html = decoder.decode(html);
+const cachedToken = {
+    token: "",
+    expiry: 0,
+    device_id: "",
+};
 
-    if (!html.includes(`{"lang":`)) return { error: "fetch.empty" };
-
-    let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
-
-    if (Number(js.mvData.is_active_live) !== 0) {
-        return { error: "content.video.live" };
+const getToken = async () => {
+    if (cachedToken.expiry - 10 > Math.floor(new Date().getTime() / 1000)) {
+        return cachedToken.token;
     }
 
-    if (js.mvData.duration > env.durationLimit) {
+    const randomDeviceId = crypto.randomUUID().toUpperCase();
+
+    const anonymOauth = new URL(oauthUrl);
+    anonymOauth.searchParams.set("client_id", clientId);
+    anonymOauth.searchParams.set("client_secret", clientSecret);
+    anonymOauth.searchParams.set("device_id", randomDeviceId);
+
+    const oauthResponse = await fetch(anonymOauth.toString(), {
+        headers: {
+            "user-agent": vkClientAgent,
+        }
+    }).then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
+
+    if (!oauthResponse) return;
+
+    if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") {
+        cachedToken.token = oauthResponse.token;
+        cachedToken.expiry = oauthResponse.expired_at;
+        cachedToken.device_id = randomDeviceId;
+    }
+
+    if (!cachedToken.token) return;
+
+    return cachedToken.token;
+}
+
+const getVideo = async (ownerId, videoId, accessKey) => {
+    const video = await fetch(`${apiUrl}/video.get`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/x-www-form-urlencoded; charset=utf-8",
+            "user-agent": vkClientAgent,
+        },
+        body: new URLSearchParams({
+            anonymous_token: cachedToken.token,
+            device_id: cachedToken.device_id,
+            lang: "en",
+            v: "5.244",
+            videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`
+        }).toString()
+    })
+    .then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
+
+    return video;
+}
+
+export default async function ({ ownerId, videoId, accessKey, quality, subtitleLang }) {
+    const token = await getToken();
+    if (!token) return { error: "fetch.fail" };
+
+    const videoGet = await getVideo(ownerId, videoId, accessKey);
+
+    if (!videoGet || !videoGet.response || videoGet.response.items.length !== 1) {
+        return { error: "fetch.empty" };
+    }
+
+    const video = videoGet.response.items[0];
+
+    if (video.restriction) {
+        const title = video.restriction.title;
+        if (title.endsWith("country") || title.endsWith("region.")) {
+            return { error: "content.video.region" };
+        }
+        if (title === "Processing video") {
+            return { error: "fetch.empty" };
+        }
+        return { error: "content.video.unavailable" };
+    }
+
+    if (!video.files || !video.duration) {
+        return { error: "fetch.fail" };
+    }
+
+    if (video.duration > env.durationLimit) {
         return { error: "content.too_long" };
     }
 
-    for (let i in resolutions) {
-        if (js.player.params[0][`url${resolutions[i]}`]) {
-            quality = resolutions[i];
+    const userQuality = quality === "max" ? resolutions[0] : quality;
+    let pickedQuality;
+
+    for (const resolution of resolutions) {
+        if (video.files[`mp4_${resolution}`] && +resolution <= +userQuality) {
+            pickedQuality = resolution;
             break
         }
     }
-    if (Number(quality) > Number(o.quality)) quality = o.quality;
 
-    url = js.player.params[0][`url${quality}`];
+    const url = video.files[`mp4_${pickedQuality}`];
 
-    let fileMetadata = {
-        title: cleanString(js.player.params[0].md_title.trim()),
-        author: cleanString(js.player.params[0].md_author.trim()),
+    if (!url) return { error: "fetch.fail" };
+
+    const fileMetadata = {
+        title: video.title.trim(),
     }
 
-    if (url) return {
+    let subtitles;
+    if (subtitleLang && video.subtitles?.length) {
+        const subtitle = video.subtitles.find(
+            s => s.title.endsWith(".vtt") && s.lang.startsWith(subtitleLang)
+        );
+        if (subtitle) {
+            subtitles = subtitle.url;
+            fileMetadata.sublanguage = subtitleLang;
+        }
+    }
+
+    return {
         urls: url,
+        subtitles,
+        fileMetadata,
         filenameAttributes: {
             service: "vk",
-            id: `${o.userId}_${o.videoId}`,
+            id: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`,
             title: fileMetadata.title,
-            author: fileMetadata.author,
-            resolution: `${quality}p`,
-            qualityLabel: `${quality}p`,
+            resolution: `${pickedQuality}p`,
+            qualityLabel: `${pickedQuality}p`,
             extension: "mp4"
         }
     }
-    return { error: "fetch.empty" }
 }

api/src/processing/services/xiaohongshu.js

@@ -0,0 +1,109 @@
+import { resolveRedirectingURL } from "../url.js";
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+
+const https = (url) => {
+    return url.replace(/^http:/i, 'https:');
+}
+
+export default async function ({ id, token, shareType, shareId, h265, isAudioOnly, dispatcher }) {
+    let noteId = id;
+    let xsecToken = token;
+
+    if (!noteId) {
+        const patternMatch = await resolveRedirectingURL(
+            `https://xhslink.com/${shareType}/${shareId}`,
+            dispatcher
+        );
+
+        noteId = patternMatch?.id;
+        xsecToken = patternMatch?.token;
+    }
+
+    if (!noteId || !xsecToken) return { error: "fetch.short_link" };
+
+    const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, {
+        headers: {
+            "user-agent": genericUserAgent,
+        },
+        dispatcher,
+    });
+
+    const html = await res.text();
+
+    let note;
+    try {
+        const initialState = html
+            .split('<script>window.__INITIAL_STATE__=')[1]
+            .split('</script>')[0]
+            .replace(/:\s*undefined/g, ":null");
+
+        const data = JSON.parse(initialState);
+
+        const noteInfo = data?.note?.noteDetailMap;
+        if (!noteInfo) throw "no note detail map";
+
+        const currentNote = noteInfo[noteId];
+        if (!currentNote) throw "no current note in detail map";
+
+        note = currentNote.note;
+    } catch {}
+
+    if (!note) return { error: "fetch.empty" };
+
+    const video = note.video;
+    const images = note.imageList;
+
+    const filenameBase = `xiaohongshu_${noteId}`;
+
+    if (video) {
+        const videoFilename = `${filenameBase}.mp4`;
+        const audioFilename = `${filenameBase}_audio`;
+
+        let videoURL;
+
+        if (h265 && !isAudioOnly && video.consumer?.originVideoKey) {
+            videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`;
+        } else {
+            const h264Streams = video.media?.stream?.h264;
+
+            if (h264Streams?.length) {
+                videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl;
+            }
+        }
+
+        if (!videoURL) return { error: "fetch.empty" };
+
+        return {
+            urls: https(videoURL),
+            filename: videoFilename,
+            audioFilename: audioFilename,
+        }
+    }
+
+    if (!images || images.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    if (images.length === 1) {
+        return {
+            isPhoto: true,
+            urls: https(images[0].urlDefault),
+            filename: `${filenameBase}.jpg`,
+        }
+    }
+
+    const picker = images.map((image, i) => {
+        return {
+            type: "photo",
+            url: createStream({
+                service: "xiaohongshu",
+                type: "proxy",
+                url: https(image.urlDefault),
+                filename: `${filenameBase}_${i + 1}.jpg`,
+            })
+        }
+    });
+
+    return { picker };
+}

api/src/processing/services/youtube.js

@@ -1,16 +1,17 @@
-import { fetch } from "undici";
+import HLS from "hls-parser";
 
+import { fetch } from "undici";
 import { Innertube, Session } from "youtubei.js";
 
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
-import { getCookie, updateCookieValues } from "../cookie/manager.js";
+import { getCookie } from "../cookie/manager.js";
+import { getYouTubeSession } from "../helpers/youtube-session.js";
 
 const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
 
 let innertube, lastRefreshedAt;
 
-const codecMatch = {
+const codecList = {
     h264: {
         videoCodec: "avc1",
         audioCodec: "mp4a",
@@ -18,8 +19,8 @@ const codecMatch = {
     },
     av1: {
         videoCodec: "av01",
-        audioCodec: "mp4a",
-        container: "mp4"
+        audioCodec: "opus",
+        container: "webm"
     },
     vp9: {
         videoCodec: "vp9",
@@ -28,117 +29,220 @@ const codecMatch = {
     }
 }
 
-const transformSessionData = (cookie) => {
-    if (!cookie)
-        return;
-
-    const values = { ...cookie.values() };
-    const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];
-
-    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
-        return;
+const hlsCodecList = {
+    h264: {
+        videoCodec: "avc1",
+        audioCodec: "mp4a",
+        container: "mp4"
+    },
+    vp9: {
+        videoCodec: "vp09",
+        audioCodec: "mp4a",
+        container: "webm"
     }
-
-    if (values.expires) {
-        values.expiry_date = values.expires;
-        delete values.expires;
-    } else if (!values.expiry_date) {
-        return;
-    }
-
-    return values;
 }
 
-const cloneInnertube = async (customFetch) => {
+const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDROID'];
+
+const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320];
+
+const cloneInnertube = async (customFetch, useSession) => {
     const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
+
+    const rawCookie = getCookie('youtube');
+    const cookie = rawCookie?.toString();
+
+    const sessionTokens = getYouTubeSession();
+    const retrieve_player = Boolean(sessionTokens || cookie);
+
+    if (useSession && env.ytSessionServer && !sessionTokens?.potoken) {
+        throw "no_session_tokens";
+    }
+
     if (!innertube || shouldRefreshPlayer) {
         innertube = await Innertube.create({
-            fetch: customFetch
+            fetch: customFetch,
+            retrieve_player,
+            cookie,
+            po_token: useSession ? sessionTokens?.potoken : undefined,
+            visitor_data: useSession ? sessionTokens?.visitor_data : undefined,
         });
         lastRefreshedAt = +new Date();
     }
 
     const session = new Session(
         innertube.session.context,
-        innertube.session.key,
+        innertube.session.api_key,
         innertube.session.api_version,
         innertube.session.account_index,
+        innertube.session.config_data,
         innertube.session.player,
-        undefined,
+        cookie,
         customFetch ?? innertube.session.http.fetch,
-        innertube.session.cache
+        innertube.session.cache,
+        sessionTokens?.potoken
     );
 
-    const cookie = getCookie('youtube_oauth');
-    const oauthData = transformSessionData(cookie);
-
-    if (!session.logged_in && oauthData) {
-        await session.oauth.init(oauthData);
-        session.logged_in = true;
-    }
-
-    if (session.logged_in) {
-        if (session.oauth.shouldRefreshToken()) {
-            await session.oauth.refreshAccessToken();
-        }
-
-        const cookieValues = cookie.values();
-        const oldExpiry = new Date(cookieValues.expiry_date);
-        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
-
-        if (oldExpiry.getTime() !== newExpiry.getTime()) {
-            updateCookieValues(cookie, {
-                ...session.oauth.client_id,
-                ...session.oauth.oauth2_tokens,
-                expiry_date: newExpiry.toISOString()
-            });
-        }
-    }
-
     const yt = new Innertube(session);
     return yt;
 }
 
-export default async function(o) {
+const getHlsVariants = async (hlsManifest, dispatcher) => {
+    if (!hlsManifest) {
+        return { error: "youtube.no_hls_streams" };
+    }
+
+    const fetchedHlsManifest =
+        await fetch(hlsManifest, { dispatcher })
+            .then(r => r.status === 200 ? r.text() : undefined)
+            .catch(() => {});
+
+    if (!fetchedHlsManifest) {
+        return { error: "youtube.no_hls_streams" };
+    }
+
+    const variants = HLS.parse(fetchedHlsManifest).variants.sort(
+        (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+    );
+
+    if (!variants || variants.length === 0) {
+        return { error: "youtube.no_hls_streams" };
+    }
+
+    return variants;
+}
+
+const getSubtitles = async (info, dispatcher, subtitleLang) => {
+    const preferredCap = info.captions.caption_tracks.find(caption =>
+        caption.kind !== 'asr' && caption.language_code.startsWith(subtitleLang)
+    );
+
+    const captionsUrl = preferredCap?.base_url;
+    if (!captionsUrl) return;
+
+    if (!captionsUrl.includes("exp=xpe")) {
+        let url = new URL(captionsUrl);
+        url.searchParams.set('fmt', 'vtt');
+
+        return {
+            url: url.toString(),
+            language: preferredCap.language_code,
+        }
+    }
+
+    // if we have exp=xpe in the url, then captions are
+    // locked down and can't be accessed without a yummy potoken,
+    // so instead we just use subtitles from HLS
+
+    const hlsVariants = await getHlsVariants(
+        info.streaming_data.hls_manifest_url,
+        dispatcher
+    );
+    if (hlsVariants?.error) return;
+
+    // all variants usually have the same set of subtitles
+    const hlsSubtitles = hlsVariants[0]?.subtitles;
+    if (!hlsSubtitles?.length) return;
+
+    const preferredHls = hlsSubtitles.find(
+        subtitle => subtitle.language.startsWith(subtitleLang)
+    );
+
+    if (!preferredHls) return;
+
+    const fetchedHlsSubs =
+        await fetch(preferredHls.uri, { dispatcher })
+            .then(r => r.status === 200 ? r.text() : undefined)
+            .catch(() => {});
+
+    const parsedSubs = HLS.parse(fetchedHlsSubs);
+    if (!parsedSubs) return;
+
+    return {
+        url: parsedSubs.segments[0]?.uri,
+        language: preferredHls.language,
+    }
+}
+
+export default async function (o) {
+    const quality = o.quality === "max" ? 9000 : Number(o.quality);
+
+    let useHLS = o.youtubeHLS;
+    let innertubeClient = o.innertubeClient || env.customInnertubeClient || "IOS";
+
+    // HLS playlists from the iOS client don't contain the av1 video format.
+    if (useHLS && o.codec === "av1") {
+        useHLS = false;
+    }
+
+    if (useHLS) {
+        innertubeClient = "IOS";
+    }
+
+    // iOS client doesn't have adaptive formats of resolution >1080p,
+    // so we use the WEB_EMBEDDED client instead for those cases
+    let useSession =
+        env.ytSessionServer && (
+            (
+                !useHLS
+                && innertubeClient === "IOS"
+                && (
+                    (quality > 1080 && o.codec !== "h264")
+                    || (quality > 1080 && o.codec !== "vp9")
+                )
+            )
+        );
+
+    // we can get subtitles reliably only from the iOS client
+    if (o.subtitleLang) {
+        innertubeClient = "IOS";
+        useSession = false;
+    }
+
+    if (useSession) {
+        innertubeClient = env.ytSessionInnertubeClient || "WEB_EMBEDDED";
+    }
+
     let yt;
     try {
         yt = await cloneInnertube(
             (input, init) => fetch(input, {
                 ...init,
                 dispatcher: o.dispatcher
-            })
+            }),
+            useSession
         );
-    } catch(e) {
-        if (e.message?.endsWith("decipher algorithm")) {
+    } catch (e) {
+        if (e === "no_session_tokens") {
+            return { error: "youtube.no_session_tokens" };
+        } else if (e.message?.endsWith("decipher algorithm")) {
             return { error: "youtube.decipher" }
         } else if (e.message?.includes("refresh access token")) {
             return { error: "youtube.token_expired" }
         } else throw e;
     }
 
-    const quality = o.quality === "max" ? "9000" : o.quality;
-
-    let info, isDubbed,
-        format = o.format || "h264";
-
-    function qual(i) {
-        if (!i.quality_label) {
-            return;
-        }
-
-        return i.quality_label.split('p')[0].split('s')[0]
-    }
-
+    let info;
     try {
-        info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS');
-    } catch(e) {
-        if (e?.info?.reason === "This video is private") {
-            return { error: "content.video.private" };
-        } else if (e?.message === "This video is unavailable") {
-            return { error: "content.video.unavailable" };
-        } else {
-            return { error: "fetch.fail" };
+        info = await yt.getBasicInfo(o.id, { client: innertubeClient });
+    } catch (e) {
+        if (e?.info) {
+            let errorInfo;
+            try { errorInfo = JSON.parse(e?.info); } catch {}
+
+            if (errorInfo?.reason === "This video is private") {
+                return { error: "content.video.private" };
+            }
+            if (["INVALID_ARGUMENT", "UNAUTHENTICATED"].includes(errorInfo?.error?.status)) {
+                return { error: "youtube.api_error" };
+            }
         }
+
+        if (e?.message === "This video is unavailable") {
+            return { error: "content.video.unavailable" };
+        }
+
+        return { error: "fetch.fail" };
     }
 
     if (!info) return { error: "fetch.fail" };
@@ -146,37 +250,47 @@ export default async function(o) {
     const playability = info.playability_status;
     const basicInfo = info.basic_info;
 
-    if (playability.status === "LOGIN_REQUIRED") {
-        if (playability.reason.endsWith("bot")) {
-            return { error: "youtube.login" }
-        }
-        if (playability.reason.endsWith("age")) {
-            return { error: "content.video.age" }
-        }
-        if (playability?.error_screen?.reason?.text === "Private video") {
-            return { error: "content.video.private" }
-        }
-    }
+    switch (playability.status) {
+        case "LOGIN_REQUIRED":
+            if (playability.reason.endsWith("bot")) {
+                return { error: "youtube.login" }
+            }
+            if (playability.reason.endsWith("age") || playability.reason.endsWith("inappropriate for some users.")) {
+                return { error: "content.video.age" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
 
-    if (playability.status === "UNPLAYABLE") {
-        if (playability?.reason?.endsWith("request limit.")) {
-            return { error: "fetch.rate" }
-        }
-        if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
-            return { error: "content.video.region" }
-        }
-        if (playability?.error_screen?.reason?.text === "Private video") {
-            return { error: "content.video.private" }
-        }
+        case "UNPLAYABLE":
+            if (playability?.reason?.endsWith("request limit.")) {
+                return { error: "fetch.rate" }
+            }
+            if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
+                return { error: "content.video.region" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
+
+        case "AGE_VERIFICATION_REQUIRED":
+            return { error: "content.video.age" };
     }
 
     if (playability.status !== "OK") {
         return { error: "content.video.unavailable" };
     }
+
     if (basicInfo.is_live) {
         return { error: "content.video.live" };
     }
 
+    if (basicInfo.duration > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+
     // return a critical error if returned video is "Video Not Available"
     // or a similar stub by youtube
     if (basicInfo.id !== o.id) {
@@ -186,126 +300,306 @@ export default async function(o) {
         }
     }
 
-    const filterByCodec = (formats) =>
-        formats
-        .filter(e =>
-            e.mime_type.includes(codecMatch[format].videoCodec)
-            || e.mime_type.includes(codecMatch[format].audioCodec)
-        )
-        .sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
-
-    let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats);
-
-    if (adaptive_formats.length === 0 && format === "vp9") {
-        format = "h264"
-        adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats)
+    const normalizeQuality = res => {
+        const shortestSide = Math.min(res.height, res.width);
+        return videoQualities.find(qual => qual >= shortestSide);
     }
 
-    let bestQuality;
+    let video, audio, subtitles, dubbedLanguage,
+        codec = o.codec || "h264", itag = o.itag;
 
-    const bestVideo = adaptive_formats.find(i => i.has_video && i.content_length);
-    const hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length);
+    if (useHLS) {
+        const variants = await getHlsVariants(
+            info.streaming_data.hls_manifest_url,
+            o.dispatcher
+        );
 
-    if (bestVideo) bestQuality = qual(bestVideo);
+        if (variants?.error) return variants;
 
-    if ((!bestQuality && !o.isAudioOnly) || !hasAudio)
-        return { error: "youtube.codec" };
+        const matchHlsCodec = codecs => (
+            codecs.includes(hlsCodecList[codec].videoCodec)
+        );
 
-    if (basicInfo.duration > env.durationLimit)
-        return { error: "content.too_long" };
+        const best = variants.find(i => matchHlsCodec(i.codecs));
 
-    const checkBestAudio = (i) => (i.has_audio && !i.has_video);
+        const preferred = variants.find(i =>
+            matchHlsCodec(i.codecs) && normalizeQuality(i.resolution) === quality
+        );
 
-    let audio = adaptive_formats.find(i =>
-        checkBestAudio(i) && i.is_original
-    );
+        let selected = preferred || best;
 
-    if (o.dubLang) {
-        let dubbedAudio = adaptive_formats.find(i =>
-            checkBestAudio(i)
-            && i.language === o.dubLang
-            && i.audio_track
-        )
+        if (!selected) {
+            codec = "h264";
+            selected = variants.find(i => matchHlsCodec(i.codecs));
+        }
 
-        if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
-            audio = dubbedAudio;
-            isDubbed = true;
+        if (!selected) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = selected.audio.find(i => i.isDefault);
+
+        // some videos (mainly those with AI dubs) don't have any tracks marked as default
+        // why? god knows, but we assume that a default track is marked as such in the title
+        if (!audio) {
+            audio = selected.audio.find(i => i.name.endsWith("original"));
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = selected.audio.find(i =>
+                i.language?.startsWith(o.dubLang)
+            );
+
+            if (dubbedAudio && !dubbedAudio.isDefault) {
+                dubbedLanguage = dubbedAudio.language;
+                audio = dubbedAudio;
+            }
+        }
+
+        selected.audio = [];
+        selected.subtitles = [];
+        video = selected;
+    } else {
+        // i miss typescript so bad
+        const sorted_formats = {
+            h264: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            vp9: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            av1: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+        }
+
+        const checkFormat = (format, pCodec) => format.content_length &&
+            (format.mime_type.includes(codecList[pCodec].videoCodec)
+                || format.mime_type.includes(codecList[pCodec].audioCodec));
+
+        // sort formats & weed out bad ones
+        info.streaming_data.adaptive_formats.sort((a, b) =>
+            Number(b.bitrate) - Number(a.bitrate)
+        ).forEach(format => {
+            Object.keys(codecList).forEach(yCodec => {
+                const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag;
+                const sorted = sorted_formats[yCodec];
+                const goodFormat = checkFormat(format, yCodec);
+                if (!goodFormat) return;
+
+                if (format.has_video && matchingItag('video')) {
+                    sorted.video.push(format);
+                    if (!sorted.bestVideo)
+                        sorted.bestVideo = format;
+                }
+
+                if (format.has_audio && matchingItag('audio')) {
+                    sorted.audio.push(format);
+                    if (!sorted.bestAudio)
+                        sorted.bestAudio = format;
+                }
+            })
+        });
+
+        const noBestMedia = () => {
+            const vid = sorted_formats[codec]?.bestVideo;
+            const aud = sorted_formats[codec]?.bestAudio;
+            return (!vid && !o.isAudioOnly) || (!aud && o.isAudioOnly)
+        };
+
+        if (noBestMedia()) {
+            if (codec === "av1") codec = "vp9";
+            else if (codec === "vp9") codec = "av1";
+
+            // if there's no higher quality fallback, then use h264
+            if (noBestMedia()) codec = "h264";
+        }
+
+        // if there's no proper combo of av1, vp9, or h264, then give up
+        if (noBestMedia()) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = sorted_formats[codec].bestAudio;
+
+        if (audio?.audio_track && !audio?.is_original) {
+            audio = sorted_formats[codec].audio.find(i =>
+                i?.is_original
+            );
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = sorted_formats[codec].audio.find(i =>
+                i.language?.startsWith(o.dubLang) && i.audio_track
+            );
+
+            if (dubbedAudio && !dubbedAudio?.is_original) {
+                audio = dubbedAudio;
+                dubbedLanguage = dubbedAudio.language;
+            }
+        }
+
+        if (!o.isAudioOnly) {
+            const qual = (i) => {
+                return normalizeQuality({
+                    width: i.width,
+                    height: i.height,
+                })
+            }
+
+            const bestQuality = qual(sorted_formats[codec].bestVideo);
+            const useBestQuality = quality >= bestQuality;
+
+            video = useBestQuality
+                ? sorted_formats[codec].bestVideo
+                : sorted_formats[codec].video.find(i => qual(i) === quality);
+
+            if (!video) video = sorted_formats[codec].bestVideo;
+        }
+
+        if (o.subtitleLang && !o.isAudioOnly && info.captions?.caption_tracks?.length) {
+            const videoSubtitles = await getSubtitles(info, o.dispatcher, o.subtitleLang);
+            if (videoSubtitles) {
+                subtitles = videoSubtitles;
+            }
         }
     }
 
-    if (!audio) {
-        audio = adaptive_formats.find(i => checkBestAudio(i));
+    if (video?.drm_families || audio?.drm_families) {
+        return { error: "youtube.drm" };
     }
 
-    let fileMetadata = {
-        title: cleanString(basicInfo.title.trim()),
-        artist: cleanString(basicInfo.author.replace("- Topic", "").trim()),
+    const fileMetadata = {
+        title: basicInfo.title.trim(),
+        artist: basicInfo.author.replace("- Topic", "").trim()
     }
 
     if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) {
-        let descItems = basicInfo.short_description.split("\n\n");
-        fileMetadata.album = descItems[2];
-        fileMetadata.copyright = descItems[3];
-        if (descItems[4].startsWith("Released on:")) {
-            fileMetadata.date = descItems[4].replace("Released on: ", '').trim()
+        const descItems = basicInfo.short_description.split("\n\n", 5);
+
+        if (descItems.length === 5) {
+            fileMetadata.album = descItems[2];
+            fileMetadata.copyright = descItems[3];
+            if (descItems[4].startsWith("Released on:")) {
+                fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
+            }
         }
     }
 
-    let filenameAttributes = {
+    if (subtitles) {
+        fileMetadata.sublanguage = subtitles.language;
+    }
+
+    const filenameAttributes = {
         service: "youtube",
         id: o.id,
         title: fileMetadata.title,
         author: fileMetadata.artist,
-        youtubeDubName: isDubbed ? o.dubLang : false
+        youtubeDubName: dubbedLanguage || false,
     }
 
-    if (audio && o.isAudioOnly) return {
-        type: "audio",
-        isAudioOnly: true,
-        urls: audio.decipher(yt.session.player),
-        filenameAttributes: filenameAttributes,
-        fileMetadata: fileMetadata,
-        bestAudio: format === "h264" ? "m4a" : "opus"
-    }
+    itag = {
+        video: video?.itag,
+        audio: audio?.itag
+    };
 
-    const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality,
-        checkSingle = i =>
-            qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec),
-        checkRender = i =>
-            qual(i) === matchingQuality && i.has_video && !i.has_audio;
+    const originalRequest = {
+        ...o,
+        dispatcher: undefined,
+        itag,
+        innertubeClient
+    };
 
-    let match, type, urls;
+    if (audio && o.isAudioOnly) {
+        let bestAudio = codec === "h264" ? "m4a" : "opus";
+        let urls = audio.url;
 
-    // prefer good premuxed videos if available
-    if (!o.isAudioOnly && !o.isAudioMuted && format === "h264" && bestVideo.fps <= 30) {
-        match = info.streaming_data.formats.find(checkSingle);
-        type = "proxy";
-        urls = match?.decipher(yt.session.player);
-    }
+        if (useHLS) {
+            bestAudio = "mp3";
+            urls = audio.uri;
+        }
 
-    const video = adaptive_formats.find(checkRender);
+        if (!clientsWithNoCipher.includes(innertubeClient) && innertube) {
+            urls = audio.decipher(innertube.session.player);
+        }
 
-    if (!match && video && audio) {
-        match = video;
-        type = "merge";
-        urls = [
-            video.decipher(yt.session.player),
-            audio.decipher(yt.session.player)
-        ]
-    }
+        let cover = `https://i.ytimg.com/vi/${o.id}/maxresdefault.jpg`;
+        const testMaxCover = await fetch(cover, { dispatcher: o.dispatcher })
+            .then(r => r.status === 200)
+            .catch(() => {});
+
+        if (!testMaxCover) {
+            cover = basicInfo.thumbnail?.[0]?.url;
+        }
 
-    if (match) {
-        filenameAttributes.qualityLabel = match.quality_label;
-        filenameAttributes.resolution = `${match.width}x${match.height}`;
-        filenameAttributes.extension = codecMatch[format].container;
-        filenameAttributes.youtubeFormat = format;
         return {
-            type,
+            type: "audio",
+            isAudioOnly: true,
             urls,
             filenameAttributes,
-            fileMetadata
+            fileMetadata,
+            bestAudio,
+            isHLS: useHLS,
+            originalRequest,
+
+            cover,
+            cropCover: basicInfo.author.endsWith("- Topic"),
         }
     }
 
-    return { error: "fetch.fail" }
+    if (video && audio) {
+        let resolution;
+
+        if (useHLS) {
+            resolution = normalizeQuality(video.resolution);
+            filenameAttributes.resolution = `${video.resolution.width}x${video.resolution.height}`;
+            filenameAttributes.extension = o.container === "auto" ? hlsCodecList[codec].container : o.container;
+
+            video = video.uri;
+            audio = audio.uri;
+        } else {
+            resolution = normalizeQuality({
+                width: video.width,
+                height: video.height,
+            });
+
+            filenameAttributes.resolution = `${video.width}x${video.height}`;
+            filenameAttributes.extension = o.container === "auto" ? codecList[codec].container : o.container;
+
+            if (!clientsWithNoCipher.includes(innertubeClient) && innertube) {
+                video = video.decipher(innertube.session.player);
+                audio = audio.decipher(innertube.session.player);
+            } else {
+                video = video.url;
+                audio = audio.url;
+            }
+        }
+
+        filenameAttributes.qualityLabel = `${resolution}p`;
+        filenameAttributes.youtubeFormat = codec;
+
+        return {
+            type: "merge",
+            urls: [
+                video,
+                audio,
+            ],
+            subtitles: subtitles?.url,
+            filenameAttributes,
+            fileMetadata,
+            isHLS: useHLS,
+            originalRequest
+        }
+    }
+
+    return { error: "youtube.no_matching_format" };
 }

api/src/processing/url.js

@@ -1,8 +1,9 @@
-import psl from "psl";
+import psl from "@imput/psl";
 import { strict as assert } from "node:assert";
 
 import { env } from "../config.js";
 import { services } from "./service-config.js";
+import { getRedirectingURL } from "../misc/utils.js";
 import { friendlyServiceName } from "./service-alias.js";
 
 function aliasURL(url) {
@@ -16,7 +17,7 @@ function aliasURL(url) {
             if (url.pathname.startsWith('/live/') || url.pathname.startsWith('/shorts/')) {
                 url.pathname = '/watch';
                 // parts := ['', 'live' || 'shorts', id, ...rest]
-                url.search = `?v=${encodeURIComponent(parts[2])}`
+                url.search = `?v=${encodeURIComponent(parts[2])}`;
             }
             break;
 
@@ -42,7 +43,7 @@ function aliasURL(url) {
         case "fixvx":
         case "x":
             if (services.twitter.altDomains.includes(url.hostname)) {
-                url.hostname = 'twitter.com'
+                url.hostname = 'twitter.com';
             }
             break;
 
@@ -60,23 +61,23 @@ function aliasURL(url) {
 
         case "b23":
             if (url.hostname === 'b23.tv' && parts.length === 2) {
-                url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`)
+                url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`);
             }
             break;
 
         case "dai":
             if (url.hostname === 'dai.ly' && parts.length === 2) {
-                url = new URL(`https://dailymotion.com/video/${parts[1]}`)
+                url = new URL(`https://dailymotion.com/video/${parts[1]}`);
             }
             break;
 
         case "facebook":
         case "fb":
             if (url.searchParams.get('v')) {
-                url = new URL(`https://web.facebook.com/user/videos/${url.searchParams.get('v')}`)
+                url = new URL(`https://web.facebook.com/user/videos/${url.searchParams.get('v')}`);
             }
             if (url.hostname === 'fb.watch') {
-                url = new URL(`https://web.facebook.com/_shortLink/${parts[1]}`)
+                url = new URL(`https://web.facebook.com/_shortLink/${parts[1]}`);
             }
             break;
 
@@ -85,9 +86,40 @@ function aliasURL(url) {
                 url.hostname = 'instagram.com';
             }
             break;
+
+        case "vk":
+        case "vkvideo":
+            if (services.vk.altDomains.includes(url.hostname)) {
+                url.hostname = 'vk.com';
+            }
+            if (url.searchParams.get('z')) {
+                url = new URL(`https://vk.com/${url.searchParams.get('z')}`);
+            }
+            break;
+
+        case "xhslink":
+            if (url.hostname === 'xhslink.com' && parts.length === 3) {
+                url = new URL(`https://www.xiaohongshu.com/${parts[1]}/${parts[2]}`);
+            }
+            break;
+
+        case "loom":
+            const idPart = parts[parts.length - 1];
+            if (idPart.length > 32) {
+                url.pathname = `/share/${idPart.slice(-32)}`;
+            }
+            break;
+
+        case "redd":
+            /* reddit short video links can be treated by changing https://v.redd.it/<id>
+            to https://reddit.com/video/<id>.*/
+            if (url.hostname === "v.redd.it" && parts.length === 2) {
+                url = new URL(`https://www.reddit.com/video/${parts[1]}`);
+            }
+            break;
     }
 
-    return url
+    return url;
 }
 
 function cleanURL(url) {
@@ -107,31 +139,42 @@ function cleanURL(url) {
             break;
         case "vk":
             if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
-                limitQuery('z')
+                limitQuery('z');
             }
             break;
         case "youtube":
             if (url.searchParams.get('v')) {
-                limitQuery('v')
+                limitQuery('v');
             }
             break;
+        case "bilibili":
         case "rutube":
             if (url.searchParams.get('p')) {
-                limitQuery('p')
+                limitQuery('p');
+            }
+            break;
+        case "twitter":
+            if (url.searchParams.get('post_id')) {
+                limitQuery('post_id');
+            }
+            break;
+        case "xiaohongshu":
+            if (url.searchParams.get('xsec_token')) {
+                limitQuery('xsec_token');
             }
             break;
     }
 
     if (stripQuery) {
-        url.search = ''
+        url.search = '';
     }
 
-    url.username = url.password = url.port = url.hash = ''
+    url.username = url.password = url.port = url.hash = '';
 
     if (url.pathname.endsWith('/'))
         url.pathname = url.pathname.slice(0, -1);
 
-    return url
+    return url;
 }
 
 function getHostIfValid(url) {
@@ -157,7 +200,7 @@ export function normalizeURL(url) {
     );
 }
 
-export function extract(url) {
+export function extract(url, enabledServices = env.enabledServices) {
     if (!(url instanceof URL)) {
         url = new URL(url);
     }
@@ -168,7 +211,12 @@ export function extract(url) {
         return { error: "link.invalid" };
     }
 
-    if (!env.enabledServices.has(host)) {
+    if (!enabledServices.has(host)) {
+        // show a different message when youtube is disabled on official instances
+        // as it only happens when shit hits the fan
+        if (new URL(env.apiURL).hostname.endsWith(".imput.net") && host === "youtube") {
+            return { error: "youtube.temporary_disabled" };
+        }
         return { error: "service.disabled" };
     }
 
@@ -194,3 +242,17 @@ export function extract(url) {
 
     return { host, patternMatch };
 }
+
+export async function resolveRedirectingURL(url, dispatcher, headers) {
+    const originalService = getHostIfValid(normalizeURL(url));
+    if (!originalService) return;
+
+    const canonicalURL = await getRedirectingURL(url, dispatcher, headers);
+    if (!canonicalURL) return;
+
+    const { host, patternMatch } = extract(normalizeURL(canonicalURL));
+
+    if (host === originalService) {
+        return patternMatch;
+    }
+}

api/src/security/api-keys.js

@@ -0,0 +1,266 @@
+import { env } from "../config.js";
+import { Green, Yellow } from "../misc/console-text.js";
+import ip from "ipaddr.js";
+import * as cluster from "../misc/cluster.js";
+import { FileWatcher } from "../misc/file-watcher.js";
+
+// this function is a modified variation of code
+// from https://stackoverflow.com/a/32402438/14855621
+const generateWildcardRegex = rule => {
+    var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$");
+}
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+let keys = {}, reader = null;
+
+const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit', 'allowedServices']);
+
+/* Expected format pseudotype:
+** type KeyFileContents = Record<
+**    UUIDv4String,
+**    {
+**        name?: string,
+**        limit?: number | "unlimited",
+**        ips?: CIDRString[],
+**        userAgents?: string[],
+**        allowedServices?: "all" | string[],
+**    }
+** >;
+*/
+
+const validateKeys = (input) => {
+    if (typeof input !== 'object' || input === null) {
+        throw "input is not an object";
+    }
+
+    if (Object.keys(input).some(x => !UUID_REGEX.test(x))) {
+        throw "key file contains invalid key(s)";
+    }
+
+    Object.values(input).forEach(details => {
+        if (typeof details !== 'object' || details === null) {
+            throw "some key(s) are incorrectly configured";
+        }
+
+        const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k));
+        if (unexpected_key) {
+            throw "detail object contains unexpected key: " + unexpected_key;
+        }
+
+        if (details.limit && details.limit !== 'unlimited') {
+            if (typeof details.limit !== 'number')
+                throw "detail object contains invalid limit (not a number)";
+            else if (details.limit < 1)
+                throw "detail object contains invalid limit (not a positive number)";
+        }
+
+        if (details.ips) {
+            if (!Array.isArray(details.ips))
+                throw "details object contains value for `ips` which is not an array";
+
+            const invalid_ip = details.ips.find(
+                addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr))
+            );
+
+            if (invalid_ip) {
+                throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip;
+            }
+        }
+
+        if (details.userAgents) {
+            if (!Array.isArray(details.userAgents))
+                throw "details object contains value for `userAgents` which is not an array";
+
+            const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string');
+            if (invalid_ua) {
+                throw "`userAgents` in details contains an invalid user agent: " + invalid_ua;
+            }
+        }
+
+        if (details.allowedServices) {
+            if (Array.isArray(details.allowedServices)) {
+                const invalid_services = details.allowedServices.some(
+                    service => !env.allServices.has(service)
+                );
+                if (invalid_services) {
+                    throw "`allowedServices` in details contains an invalid service";
+                }
+            } else if (details.allowedServices !== "all") {
+                throw "details object contains value for `allowedServices` which is not an array or `all`";
+            }
+        }
+    });
+}
+
+const formatKeys = (keyData) => {
+    const formatted = {};
+
+    for (let key in keyData) {
+        const data = keyData[key];
+        key = key.toLowerCase();
+
+        formatted[key] = {};
+
+        if (data.limit) {
+            if (data.limit === "unlimited") {
+                data.limit = Infinity;
+            }
+
+            formatted[key].limit = data.limit;
+        }
+
+        if (data.ips) {
+            formatted[key].ips = data.ips.map(addr => {
+                if (ip.isValid(addr)) {
+                    const parsed = ip.parse(addr);
+                    const range = parsed.kind() === 'ipv6' ? 128 : 32;
+                    return [ parsed, range ];
+                }
+
+                return ip.parseCIDR(addr);
+            });
+        }
+
+        if (data.userAgents) {
+            formatted[key].userAgents = data.userAgents.map(generateWildcardRegex);
+        }
+
+        if (data.allowedServices) {
+            if (Array.isArray(data.allowedServices)) {
+                formatted[key].allowedServices = new Set(data.allowedServices);
+            } else {
+                formatted[key].allowedServices = data.allowedServices;
+            }
+        }
+    }
+
+    return formatted;
+}
+
+const updateKeys = (newKeys) => {
+    validateKeys(newKeys);
+
+    cluster.broadcast({ api_keys: newKeys });
+
+    keys = formatKeys(newKeys);
+}
+
+const loadRemoteKeys = async (source) => {
+    updateKeys(
+        await fetch(source).then(a => a.json())
+    );
+}
+
+const loadLocalKeys = async () => {
+    updateKeys(
+        JSON.parse(await reader.read())
+    );
+}
+
+const wrapLoad = (url, initial = false) => {
+    let load = loadRemoteKeys.bind(null, url);
+
+    if (url.protocol === 'file:') {
+        if (initial) {
+            reader = FileWatcher.fromFileProtocol(url);
+            reader.on('file-updated', () => wrapLoad(url));
+        }
+
+        load = loadLocalKeys;
+    }
+
+    load().then(() => {
+        if (initial || reader) {
+            console.log(`${Green('[✓]')} api keys loaded successfully!`)
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+const err = (reason) => ({ success: false, error: reason });
+
+export const validateAuthorization = (req) => {
+    const authHeader = req.get('Authorization');
+
+    if (typeof authHeader !== 'string') {
+        return err("missing");
+    }
+
+    const [ authType, keyString ] = authHeader.split(' ', 2);
+    if (authType.toLowerCase() !== 'api-key') {
+        return err("not_api_key");
+    }
+
+    if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) {
+        return err("invalid");
+    }
+
+    const matchingKey = keys[keyString.toLowerCase()];
+    if (!matchingKey) {
+        return err("not_found");
+    }
+
+    if (matchingKey.ips) {
+        let addr;
+        try {
+            addr = ip.parse(req.ip);
+        } catch {
+            return err("invalid_ip");
+        }
+
+        const ip_allowed = matchingKey.ips.some(
+            ([ allowed, size ]) => {
+                return addr.kind() === allowed.kind()
+                        && addr.match(allowed, size);
+            }
+        );
+
+        if (!ip_allowed) {
+            return err("ip_not_allowed");
+        }
+    }
+
+    if (matchingKey.userAgents) {
+        const userAgent = req.get('User-Agent');
+        if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) {
+            return err("ua_not_allowed");
+        }
+    }
+
+    req.rateLimitKey = keyString.toLowerCase();
+    req.rateLimitMax = matchingKey.limit;
+
+    return { success: true };
+}
+
+export const setup = (url) => {
+    if (cluster.isPrimary) {
+        wrapLoad(url, true);
+        if (env.keyReloadInterval > 0 && url.protocol !== 'file:') {
+            setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('api_keys' in message) {
+                updateKeys(message.api_keys);
+            }
+        });
+    }
+}
+
+export const getAllowedServices = (key) => {
+    if (typeof key !== "string") return;
+
+    const allowedServices = keys[key.toLowerCase()]?.allowedServices;
+    if (!allowedServices) return;
+
+    if (allowedServices === "all") {
+        return env.allServices;
+    }
+    return allowedServices;
+}

api/src/security/jwt.js

@@ -6,12 +6,19 @@ import { env } from "../config.js";
 const toBase64URL = (b) => Buffer.from(b).toString("base64url");
 const fromBase64URL = (b) => Buffer.from(b, "base64url").toString();
 
-const makeHmac = (header, payload) =>
-    createHmac("sha256", env.jwtSecret)
-        .update(`${header}.${payload}`)
-        .digest("base64url");
+const makeHmac = (data) => {
+    return createHmac("sha256", env.jwtSecret)
+            .update(data)
+            .digest("base64url");
+}
 
-const generate = () => {
+const sign = (header, payload) =>
+        makeHmac(`${header}.${payload}`);
+
+const getIPHash = (ip) =>
+        makeHmac(ip).slice(0, 8);
+
+const generate = (ip) => {
     const exp = Math.floor(new Date().getTime() / 1000) + env.jwtLifetime;
 
     const header = toBase64URL(JSON.stringify({
@@ -21,10 +28,11 @@ const generate = () => {
 
     const payload = toBase64URL(JSON.stringify({
         jti: nanoid(8),
+        sub: getIPHash(ip),
         exp,
     }));
 
-    const signature = makeHmac(header, payload);
+    const signature = sign(header, payload);
 
     return {
         token: `${header}.${payload}.${signature}`,
@@ -32,7 +40,7 @@ const generate = () => {
     };
 }
 
-const verify = (jwt) => {
+const verify = (jwt, ip) => {
     const [header, payload, signature] = jwt.split(".", 3);
     const timestamp = Math.floor(new Date().getTime() / 1000);
 
@@ -40,17 +48,16 @@ const verify = (jwt) => {
         return false;
     }
 
-    const verifySignature = makeHmac(header, payload);
+    const verifySignature = sign(header, payload);
 
     if (verifySignature !== signature) {
         return false;
     }
 
-    if (timestamp >= JSON.parse(fromBase64URL(payload)).exp) {
-        return false;
-    }
+    const data = JSON.parse(fromBase64URL(payload));
 
-    return true;
+    return getIPHash(ip) === data.sub
+            && timestamp <= data.exp;
 }
 
 export default {

api/src/security/secrets.js

@@ -0,0 +1,62 @@
+import cluster from "node:cluster";
+import { createHmac, randomBytes } from "node:crypto";
+
+const generateSalt = () => {
+    if (cluster.isPrimary)
+        return randomBytes(64);
+
+    return null;
+}
+
+let rateSalt = generateSalt();
+let streamSalt = generateSalt();
+
+export const syncSecrets = () => {
+    return new Promise((resolve, reject) => {
+        if (cluster.isPrimary) {
+            let remaining = Object.values(cluster.workers).length;
+            const handleReady = (worker, m) => {
+                if (m.ready)
+                    worker.send({ rateSalt, streamSalt });
+
+                if (!--remaining)
+                    resolve();
+            }
+
+            for (const worker of Object.values(cluster.workers)) {
+                worker.once(
+                    'message',
+                    (m) => handleReady(worker, m)
+                );
+            }
+        } else if (cluster.isWorker) {
+            if (rateSalt || streamSalt)
+                return reject();
+
+            process.send({ ready: true });
+            process.once('message', (message) => {
+                if (rateSalt || streamSalt)
+                    return reject();
+
+                if (message.rateSalt && message.streamSalt) {
+                    streamSalt = Buffer.from(message.streamSalt);
+                    rateSalt = Buffer.from(message.rateSalt);
+                    resolve();
+                }
+            });
+        } else reject();
+    });
+}
+
+
+export const hashHmac = (value, type) => {
+    let salt;
+    if (type === 'rate')
+        salt = rateSalt;
+    else if (type === 'stream')
+        salt = streamSalt;
+    else
+        throw "unknown salt";
+
+    return createHmac("sha256", salt).update(value).digest();
+}

api/src/store/base-store.js

@@ -0,0 +1,48 @@
+const _stores = new Set();
+
+export class Store {
+    id;
+
+    constructor(name) {
+        name = name.toUpperCase();
+
+        if (_stores.has(name))
+            throw `${name} store already exists`;
+        _stores.add(name);
+
+        this.id = name;
+    }
+
+    async _has(_key) { await Promise.reject("needs implementation"); }
+    has(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        return this._has(key);
+    }
+
+    async _get(_key) { await Promise.reject("needs implementation"); }
+    async get(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        const val = await this._get(key);
+        if (val === null)
+            return null;
+
+        return val;
+    }
+
+    async _set(_key, _val, _exp_sec = -1) { await Promise.reject("needs implementation") }
+    set(key, val, exp_sec = -1) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        exp_sec = Math.round(exp_sec);
+
+        return this._set(key, val, exp_sec);
+    }
+};

api/src/store/memory-store.js

@@ -0,0 +1,77 @@
+import { MinPriorityQueue } from '@datastructures-js/priority-queue';
+import { Store } from './base-store.js';
+
+// minimum delay between sweeps to avoid repeatedly
+// sweeping entries close in proximity one by one.
+const MIN_THRESHOLD_MS = 2500;
+
+export default class MemoryStore extends Store {
+    #store = new Map();
+    #timeouts = new MinPriorityQueue/*<{ t: number, k: unknown }>*/((obj) => obj.t);
+    #nextSweep = { id: null, t: null };
+
+    constructor(name) {
+        super(name);
+    }
+
+    _has(key) {
+        return this.#store.has(key);
+    }
+
+    _get(key) {
+        const val = this.#store.get(key);
+
+        return val === undefined ? null : val;
+    }
+
+    _set(key, val, exp_sec = -1) {
+        if (this.#store.has(key)) {
+            this.#timeouts.remove(o => o.k === key);
+        }
+
+        if (exp_sec > 0) {
+            const exp = 1000 * exp_sec;
+            const timeout_at = +new Date() + exp;
+
+            this.#timeouts.enqueue({ k: key, t: timeout_at });
+        }
+
+        this.#store.set(key, val);
+        this.#reschedule();
+    }
+
+    #reschedule() {
+        const current_time = new Date().getTime();
+        const time = this.#timeouts.front()?.t;
+        if (!time) {
+            return;
+        } else if (time < current_time) {
+            return this.#sweepNow();
+        }
+
+        const sweep = this.#nextSweep;
+        if (sweep.id === null || sweep.t > time) {
+            if (sweep.id) {
+                clearTimeout(sweep.id);
+            }
+
+            sweep.t = time;
+            sweep.id = setTimeout(
+                () => this.#sweepNow(),
+                Math.max(MIN_THRESHOLD_MS, time - current_time)
+            );
+            sweep.id.unref();
+        }
+    }
+
+    #sweepNow() {
+        while (this.#timeouts.front()?.t < new Date().getTime()) {
+            const item = this.#timeouts.dequeue();
+            this.#store.delete(item.k);
+        }
+
+        this.#nextSweep.id = null;
+        this.#nextSweep.t = null;
+        this.#reschedule();
+    }
+}

api/src/store/redis-ratelimit.js

@@ -0,0 +1,19 @@
+import { env } from "../config.js";
+
+let client, redis, redisLimiter;
+
+export const createStore = async (name) => {
+    if (!env.redisURL) return;
+
+    if (!client) {
+        redis = await import('redis');
+        redisLimiter = await import('rate-limit-redis');
+        client = redis.createClient({ url: env.redisURL });
+        await client.connect();
+    }
+
+    return new redisLimiter.default({
+        prefix: `RL${name}_`,
+        sendCommand: (...args) => client.sendCommand(args),
+    });
+}

api/src/store/redis-store.js

@@ -0,0 +1,64 @@
+import { commandOptions, createClient } from "redis";
+import { env } from "../config.js";
+import { Store } from "./base-store.js";
+
+export default class RedisStore extends Store {
+    #client = createClient({
+        url: env.redisURL,
+    });
+    #connected;
+
+    constructor(name) {
+        super(name);
+        this.#connected = this.#client.connect();
+    }
+
+    #keyOf(key) {
+        return this.id + '_' + key;
+    }
+
+    async _has(key) {
+        await this.#connected;
+
+        return this.#client.hExists(key);
+    }
+
+    async _get(key) {
+        await this.#connected;
+
+        const valueType = await this.#client.get(this.#keyOf(key) + '_t');
+        const value = await this.#client.get(
+            commandOptions({ returnBuffers: true }),
+            this.#keyOf(key)
+        );
+
+        if (!value) {
+            return null;
+        }
+
+        if (valueType === 'b')
+            return value;
+        else
+            return JSON.parse(value);
+    }
+
+    async _set(key, val, exp_sec = -1) {
+        await this.#connected;
+
+        const options = exp_sec > 0 ? { EX: exp_sec } : undefined;
+
+        if (val instanceof Buffer) {
+            await this.#client.set(
+                this.#keyOf(key) + '_t',
+                'b',
+                options
+            );
+        }
+
+        await this.#client.set(
+            this.#keyOf(key),
+            val,
+            options
+        );
+    }
+}

api/src/store/store.js

@@ -0,0 +1,10 @@
+import { env } from '../config.js';
+
+let _export;
+if (env.redisURL) {
+    _export = await import('./redis-store.js');
+} else {
+    _export = await import('./memory-store.js');
+}
+
+export default _export.default;

api/src/stream/ffmpeg.js

@@ -0,0 +1,215 @@
+import ffmpeg from "ffmpeg-static";
+import { spawn } from "child_process";
+import { create as contentDisposition } from "content-disposition-header";
+
+import { env } from "../config.js";
+import { destroyInternalStream } from "./manage.js";
+import { hlsExceptions } from "../processing/service-config.js";
+import { closeResponse, pipe, estimateTunnelLength, estimateAudioMultiplier } from "./shared.js";
+
+const metadataTags = new Set([
+    "album",
+    "composer",
+    "genre",
+    "copyright",
+    "title",
+    "artist",
+    "album_artist",
+    "track",
+    "date",
+    "sublanguage"
+]);
+
+const convertMetadataToFFmpeg = (metadata) => {
+    const args = [];
+
+    for (const [ name, value ] of Object.entries(metadata)) {
+        if (metadataTags.has(name)) {
+            if (name === "sublanguage") {
+                args.push('-metadata:s:s:0', `language=${value}`);
+                continue;
+            }
+            args.push('-metadata', `${name}=${value.replace(/[\u0000-\u0009]/g, '')}`); // skipcq: JS-0004
+        } else {
+            throw `${name} metadata tag is not supported.`;
+        }
+    }
+
+    return args;
+}
+
+const killProcess = (p) => {
+    p?.kill('SIGTERM'); // ask the process to terminate itself gracefully
+
+    setTimeout(() => {
+        if (p?.exitCode === null)
+            p?.kill('SIGKILL'); // brutally murder the process if it didn't quit
+    }, 5000);
+}
+
+const getCommand = (args) => {
+    if (typeof env.processingPriority === 'number' && !isNaN(env.processingPriority)) {
+        return ['nice', ['-n', env.processingPriority.toString(), ffmpeg, ...args]]
+    }
+    return [ffmpeg, args]
+}
+
+const render = async (res, streamInfo, ffargs, estimateMultiplier) => {
+    let process;
+    const urls = Array.isArray(streamInfo.urls) ? streamInfo.urls : [streamInfo.urls];
+    const shutdown = () => (
+        killProcess(process),
+        closeResponse(res),
+        urls.map(destroyInternalStream)
+    );
+
+    try {
+        const args = [
+            '-loglevel', '-8',
+            ...ffargs,
+        ];
+
+        process = spawn(...getCommand(args), {
+            windowsHide: true,
+            stdio: [
+                'inherit', 'inherit', 'inherit',
+                'pipe'
+            ],
+        });
+
+        const [,,, muxOutput] = process.stdio;
+
+        res.setHeader('Connection', 'keep-alive');
+        res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename));
+
+        res.setHeader(
+            'Estimated-Content-Length',
+            await estimateTunnelLength(streamInfo, estimateMultiplier)
+        );
+
+        pipe(muxOutput, res, shutdown);
+
+        process.on('close', shutdown);
+        res.on('finish', shutdown);
+    } catch {
+        shutdown();
+    }
+}
+
+const remux = async (streamInfo, res) => {
+    const format = streamInfo.filename.split('.').pop();
+    const urls = Array.isArray(streamInfo.urls) ? streamInfo.urls : [streamInfo.urls];
+    const args = urls.flatMap(url => ['-i', url]);
+
+    // if the stream type is merge, we expect two URLs
+    if (streamInfo.type === 'merge' && urls.length !== 2) {
+        return closeResponse(res);
+    }
+
+    if (streamInfo.subtitles) {
+        args.push(
+            '-i', streamInfo.subtitles,
+            '-map', `${urls.length}:s`,
+            '-c:s', format === 'mp4' ? 'mov_text' : 'webvtt',
+        );
+    }
+
+    if (urls.length === 2) {
+        args.push(
+            '-map', '0:v',
+            '-map', '1:a',
+        );
+    } else {
+        args.push(
+            '-map', '0:v:0',
+            '-map', '0:a:0'
+        );
+    }
+
+    args.push(
+        '-c:v', 'copy',
+        ...(streamInfo.type === 'mute' ? ['-an'] : ['-c:a', 'copy'])
+    );
+
+    if (format === 'mp4') {
+        args.push('-movflags', 'faststart+frag_keyframe+empty_moov');
+    }
+
+    if (streamInfo.type !== 'mute' && streamInfo.isHLS && hlsExceptions.has(streamInfo.service)) {
+        if (streamInfo.service === 'youtube' && format === 'webm') {
+            args.push('-c:a', 'libopus');
+        } else {
+            args.push('-c:a', 'aac', '-bsf:a', 'aac_adtstoasc');
+        }
+    }
+
+    if (streamInfo.metadata) {
+        args.push(...convertMetadataToFFmpeg(streamInfo.metadata));
+    }
+
+    args.push('-f', format === 'mkv' ? 'matroska' : format, 'pipe:3');
+
+    await render(res, streamInfo, args);
+}
+
+const convertAudio = async (streamInfo, res) => {
+    const args = [
+        '-i', streamInfo.urls,
+        '-vn',
+        ...(streamInfo.audioCopy ? ['-c:a', 'copy'] : ['-b:a', `${streamInfo.audioBitrate}k`]),
+    ];
+
+    if (streamInfo.audioFormat === 'mp3' && streamInfo.audioBitrate === '8') {
+        args.push('-ar', '12000');
+    }
+
+    if (streamInfo.audioFormat === 'opus') {
+        args.push('-vbr', 'off');
+    }
+
+    if (streamInfo.audioFormat === 'mp4a') {
+        args.push('-movflags', 'frag_keyframe+empty_moov');
+    }
+
+    if (streamInfo.metadata) {
+        args.push(...convertMetadataToFFmpeg(streamInfo.metadata));
+    }
+
+    args.push(
+        '-f',
+        streamInfo.audioFormat === 'm4a' ? 'ipod' : streamInfo.audioFormat,
+        'pipe:3',
+    );
+
+    await render(
+        res,
+        streamInfo,
+        args,
+        estimateAudioMultiplier(streamInfo) * 1.1,
+    );
+}
+
+const convertGif = async (streamInfo, res) => {
+    const args = [
+        '-i', streamInfo.urls,
+
+        '-vf',
+        'scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse',
+        '-loop', '0',
+
+        '-f', 'gif', 'pipe:3',
+    ];
+
+    await render(
+        res,
+        streamInfo,
+        args,
+        60,
+    );
+}
+
+export default {
+    remux,
+    convertAudio,
+    convertGif,
+}

api/src/stream/internal-hls.js

@@ -1,5 +1,6 @@
 import HLS from "hls-parser";
 import { createInternalStream } from "./manage.js";
+import { request } from "undici";
 
 function getURL(url) {
     try {
@@ -16,15 +17,17 @@ function transformObject(streamInfo, hlsObject) {
 
     let fullUrl;
     if (getURL(hlsObject.uri)) {
-        fullUrl = hlsObject.uri;
+        fullUrl = new URL(hlsObject.uri);
     } else {
         fullUrl = new URL(hlsObject.uri, streamInfo.url);
     }
 
-    hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
+    if (fullUrl.hostname !== '127.0.0.1') {
+        hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
 
-    if (hlsObject.map) {
-        hlsObject.map = transformObject(streamInfo, hlsObject.map);
+        if (hlsObject.map) {
+            hlsObject.map = transformObject(streamInfo, hlsObject.map);
+        }
     }
 
     return hlsObject;
@@ -53,8 +56,11 @@ function transformMediaPlaylist(streamInfo, hlsPlaylist) {
 
 const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"];
 
-export function isHlsRequest (req) {
-    return HLS_MIME_TYPES.includes(req.headers['content-type']);
+export function isHlsResponse(req, streamInfo) {
+    return HLS_MIME_TYPES.includes(req.headers['content-type'])
+        // bluesky's cdn responds with wrong content-type for the hls playlist,
+        // so we enforce it here until they fix it
+        || (streamInfo.service === 'bsky' && streamInfo.url.endsWith('.m3u8'));
 }
 
 export async function handleHlsPlaylist(streamInfo, req, res) {
@@ -69,3 +75,67 @@ export async function handleHlsPlaylist(streamInfo, req, res) {
 
     res.send(hlsPlaylist);
 }
+
+async function getSegmentSize(url, config) {
+    const segmentResponse = await request(url, {
+        ...config,
+        throwOnError: true
+    });
+
+    if (segmentResponse.headers['content-length']) {
+        segmentResponse.body.dump();
+        return +segmentResponse.headers['content-length'];
+    }
+
+    // if the response does not have a content-length
+    // header, we have to compute it ourselves
+    let size = 0;
+
+    for await (const data of segmentResponse.body) {
+        size += data.length;
+    }
+
+    return size;
+}
+
+export async function probeInternalHLSTunnel(streamInfo) {
+    const { url, headers, dispatcher, signal } = streamInfo;
+
+    // remove all falsy headers
+    Object.keys(headers).forEach(key => {
+        if (!headers[key]) delete headers[key];
+    });
+
+    const config = { headers, dispatcher, signal, maxRedirections: 16 };
+
+    const manifestResponse = await fetch(url, config);
+
+    const manifest = HLS.parse(await manifestResponse.text());
+    if (manifest.segments.length === 0)
+        return -1;
+
+    const segmentSamples = await Promise.all(
+        Array(5).fill().map(async () => {
+            const manifestIdx = Math.floor(Math.random() * manifest.segments.length);
+            const randomSegment = manifest.segments[manifestIdx];
+            if (!randomSegment.uri)
+                throw "segment is missing URI";
+
+            let segmentUrl;
+
+            if (getURL(randomSegment.uri)) {
+                segmentUrl = new URL(randomSegment.uri);
+            } else {
+                segmentUrl = new URL(randomSegment.uri, streamInfo.url);
+            }
+
+            const segmentSize = await getSegmentSize(segmentUrl, config) / randomSegment.duration;
+            return segmentSize;
+        })
+    );
+
+    const averageBitrate = segmentSamples.reduce((a, b) => a + b) / segmentSamples.length;
+    const totalDuration = manifest.segments.reduce((acc, segment) => acc + segment.duration, 0);
+
+    return averageBitrate * totalDuration;
+}

api/src/stream/internal.js

@@ -1,13 +1,15 @@
 import { request } from "undici";
 import { Readable } from "node:stream";
 import { closeRequest, getHeaders, pipe } from "./shared.js";
-import { handleHlsPlaylist, isHlsRequest } from "./internal-hls.js";
+import { handleHlsPlaylist, isHlsResponse, probeInternalHLSTunnel } from "./internal-hls.js";
 
 const CHUNK_SIZE = BigInt(8e6); // 8 MB
 const min = (a, b) => a < b ? a : b;
 
+const serviceNeedsChunks = new Set(["youtube", "vk"]);
+
 async function* readChunks(streamInfo, size) {
-    let read = 0n;
+    let read = 0n, chunksSinceTransplant = 0;
     while (read < size) {
         if (streamInfo.controller.signal.aborted) {
             throw new Error("controller aborted");
@@ -15,13 +17,24 @@ async function* readChunks(streamInfo, size) {
 
         const chunk = await request(streamInfo.url, {
             headers: {
-                ...getHeaders('youtube'),
+                ...getHeaders(streamInfo.service),
                 Range: `bytes=${read}-${read + CHUNK_SIZE}`
             },
             dispatcher: streamInfo.dispatcher,
-            signal: streamInfo.controller.signal
+            signal: streamInfo.controller.signal,
+            maxRedirections: 4
         });
 
+        if (chunk.statusCode === 403 && chunksSinceTransplant >= 3 && streamInfo.transplant) {
+            chunksSinceTransplant = 0;
+            try {
+                await streamInfo.transplant(streamInfo.dispatcher);
+                continue;
+            } catch {}
+        }
+
+        chunksSinceTransplant++;
+
         const expected = min(CHUNK_SIZE, size - read);
         const received = BigInt(chunk.headers['content-length']);
 
@@ -37,19 +50,30 @@ async function* readChunks(streamInfo, size) {
     }
 }
 
-async function handleYoutubeStream(streamInfo, res) {
+async function handleChunkedStream(streamInfo, res) {
     const { signal } = streamInfo.controller;
     const cleanup = () => (res.end(), closeRequest(streamInfo.controller));
 
     try {
-        const req = await fetch(streamInfo.url, {
-            headers: getHeaders('youtube'),
-            method: 'HEAD',
-            dispatcher: streamInfo.dispatcher,
-            signal
-        });
+        let req, attempts = 3;
+        while (attempts--) {
+            req = await fetch(streamInfo.url, {
+                headers: getHeaders(streamInfo.service),
+                method: 'HEAD',
+                dispatcher: streamInfo.dispatcher,
+                signal
+            });
+
+            streamInfo.url = req.url;
+            if (req.status === 403 && streamInfo.transplant) {
+                try {
+                    await streamInfo.transplant(streamInfo.dispatcher);
+                } catch {
+                    break;
+                }
+            } else break;
+        }
 
-        streamInfo.url = req.url;
         const size = BigInt(req.headers.get('content-length'));
 
         if (req.status !== 200 || !size) {
@@ -83,7 +107,7 @@ async function handleGenericStream(streamInfo, res) {
     const cleanup = () => res.end();
 
     try {
-        const req = await request(streamInfo.url, {
+        const fileResponse = await request(streamInfo.url, {
             headers: {
                 ...Object.fromEntries(streamInfo.headers),
                 host: undefined
@@ -93,19 +117,25 @@ async function handleGenericStream(streamInfo, res) {
             maxRedirections: 16
         });
 
-        res.status(req.statusCode);
-        req.body.on('error', () => {});
+        res.status(fileResponse.statusCode);
+        fileResponse.body.on('error', () => {});
 
-        for (const [ name, value ] of Object.entries(req.headers))
-            res.setHeader(name, value)
+        const isHls = isHlsResponse(fileResponse, streamInfo);
 
-        if (req.statusCode < 200 || req.statusCode > 299)
+        for (const [ name, value ] of Object.entries(fileResponse.headers)) {
+            if (!isHls || name.toLowerCase() !== 'content-length') {
+                res.setHeader(name, value);
+            }
+        }
+
+        if (fileResponse.statusCode < 200 || fileResponse.statusCode > 299) {
             return cleanup();
+        }
 
-        if (isHlsRequest(req)) {
-            await handleHlsPlaylist(streamInfo, req, res);
+        if (isHls) {
+            await handleHlsPlaylist(streamInfo, fileResponse, res);
         } else {
-            pipe(req.body, res, cleanup);
+            pipe(fileResponse.body, res, cleanup);
         }
     } catch {
         closeRequest(streamInfo.controller);
@@ -114,9 +144,50 @@ async function handleGenericStream(streamInfo, res) {
 }
 
 export function internalStream(streamInfo, res) {
-    if (streamInfo.service === 'youtube') {
-        return handleYoutubeStream(streamInfo, res);
+    if (streamInfo.headers) {
+        streamInfo.headers.delete('icy-metadata');
+    }
+
+    if (serviceNeedsChunks.has(streamInfo.service) && !streamInfo.isHLS) {
+        return handleChunkedStream(streamInfo, res);
     }
 
     return handleGenericStream(streamInfo, res);
 }
+
+export async function probeInternalTunnel(streamInfo) {
+    try {
+        const signal = AbortSignal.timeout(3000);
+        const headers = {
+            ...Object.fromEntries(streamInfo.headers || []),
+            ...getHeaders(streamInfo.service),
+            host: undefined,
+            range: undefined
+        };
+
+        if (streamInfo.isHLS) {
+            return probeInternalHLSTunnel({
+                ...streamInfo,
+                signal,
+                headers
+            });
+        }
+
+        const response = await request(streamInfo.url, {
+            method: 'HEAD',
+            headers,
+            dispatcher: streamInfo.dispatcher,
+            signal,
+            maxRedirections: 16
+        });
+
+        if (response.statusCode !== 200)
+            throw "status is not 200 OK";
+
+        const size = +response.headers['content-length'];
+        if (isNaN(size))
+            throw "content-length is not a number";
+
+        return size;
+    } catch {}
+}

api/src/stream/manage.js

@@ -1,4 +1,4 @@
-import NodeCache from "node-cache";
+import Store from "../store/store.js";
 
 import { nanoid } from "nanoid";
 import { randomBytes } from "crypto";
@@ -7,34 +7,27 @@ import { setMaxListeners } from "node:events";
 
 import { env } from "../config.js";
 import { closeRequest } from "./shared.js";
-import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js";
+import { decryptStream, encryptStream } from "../misc/crypto.js";
+import { hashHmac } from "../security/secrets.js";
+import { zip } from "../misc/utils.js";
 
 // optional dependency
 const freebind = env.freebindCIDR && await import('freebind').catch(() => {});
 
-const streamCache = new NodeCache({
-    stdTTL: env.streamLifespan,
-    checkperiod: 10,
-    deleteOnExpire: true
-})
-
-streamCache.on("expired", (key) => {
-    streamCache.del(key);
-})
+const streamCache = new Store('streams');
 
 const internalStreamCache = new Map();
-const hmacSalt = randomBytes(64).toString('hex');
 
 export function createStream(obj) {
     const streamID = nanoid(),
         iv = randomBytes(16).toString('base64url'),
         secret = randomBytes(32).toString('base64url'),
         exp = new Date().getTime() + env.streamLifespan * 1000,
-        hmac = generateHmac(`${streamID},${exp},${iv},${secret}`, hmacSalt),
+        hmac = hashHmac(`${streamID},${exp},${iv},${secret}`, 'stream').toString('base64url'),
         streamData = {
             exp: exp,
             type: obj.type,
-            urls: obj.u,
+            urls: obj.url,
             service: obj.service,
             filename: obj.filename,
 
@@ -46,12 +39,22 @@ export function createStream(obj) {
             audioBitrate: obj.audioBitrate,
             audioCopy: !!obj.audioCopy,
             audioFormat: obj.audioFormat,
+
+            isHLS: obj.isHLS || false,
+            originalRequest: obj.originalRequest,
+
+            // url to a subtitle file
+            subtitles: obj.subtitles,
         };
 
+    // FIXME: this is now a Promise, but it is not awaited
+    //        here. it may happen that the stream is not
+    //        stored in the Store before it is requested.
     streamCache.set(
         streamID,
-        encryptStream(streamData, iv, secret)
-    )
+        encryptStream(streamData, iv, secret),
+        env.streamLifespan
+    );
 
     let streamLink = new URL('/tunnel', env.apiURL);
 
@@ -70,14 +73,73 @@ export function createStream(obj) {
     return streamLink.toString();
 }
 
-export function getInternalStream(id) {
+export function createProxyTunnels(info) {
+    const proxyTunnels = [];
+
+    let urls = info.url;
+
+    if (typeof urls === "string") {
+        urls = [urls];
+    }
+
+    const tunnelTemplate = {
+        type: "proxy",
+        headers: info?.headers,
+        requestIP: info?.requestIP,
+    }
+
+    for (const url of urls) {
+        proxyTunnels.push(
+            createStream({
+                ...tunnelTemplate,
+                url,
+                service: info?.service,
+                originalRequest: info?.originalRequest,
+            })
+        );
+    }
+
+    if (info.subtitles) {
+        proxyTunnels.push(
+            createStream({
+                ...tunnelTemplate,
+                url: info.subtitles,
+                service: `${info?.service}-subtitles`,
+            })
+        );
+    }
+
+    if (info.cover) {
+        proxyTunnels.push(
+            createStream({
+                ...tunnelTemplate,
+                url: info.cover,
+                service: `${info?.service}-cover`,
+            })
+        );
+    }
+
+    return proxyTunnels;
+}
+
+export function getInternalTunnel(id) {
     return internalStreamCache.get(id);
 }
 
-export function createInternalStream(url, obj = {}) {
+export function getInternalTunnelFromURL(url) {
+    url = new URL(url);
+    if (url.hostname !== '127.0.0.1') {
+        return;
+    }
+
+    const id = url.searchParams.get('id');
+    return getInternalTunnel(id);
+}
+
+export function createInternalStream(url, obj = {}, isSubtitles) {
     assert(typeof url === 'string');
 
-    let dispatcher;
+    let dispatcher = obj.dispatcher;
     if (obj.requestIP) {
         dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false })
     }
@@ -95,15 +157,20 @@ export function createInternalStream(url, obj = {}) {
         headers = new Map(Object.entries(obj.headers));
     }
 
+    // subtitles don't need special treatment unlike big media files
+    const service = isSubtitles ? `${obj.service}-subtitles` : obj.service;
+
     internalStreamCache.set(streamID, {
         url,
-        service: obj.service,
+        service,
         headers,
         controller,
-        dispatcher
+        dispatcher,
+        isHLS: obj.isHLS,
+        transplant: obj.transplant
     });
 
-    let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.apiPort}`);
+    let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.tunnelPort}`);
     streamLink.searchParams.set('id', streamID);
 
     const cleanup = () => {
@@ -116,23 +183,86 @@ export function createInternalStream(url, obj = {}) {
     return streamLink.toString();
 }
 
-export function destroyInternalStream(url) {
+function getInternalTunnelId(url) {
     url = new URL(url);
     if (url.hostname !== '127.0.0.1') {
         return;
     }
 
-    const id = url.searchParams.get('id');
+    return url.searchParams.get('id');
+}
+
+export function destroyInternalStream(url) {
+    const id = getInternalTunnelId(url);
 
     if (internalStreamCache.has(id)) {
-        closeRequest(getInternalStream(id)?.controller);
+        closeRequest(getInternalTunnel(id)?.controller);
         internalStreamCache.delete(id);
     }
 }
 
+const transplantInternalTunnels = function(tunnelUrls, transplantUrls) {
+    if (tunnelUrls.length !== transplantUrls.length) {
+        return;
+    }
+
+    for (const [ tun, url ] of zip(tunnelUrls, transplantUrls)) {
+        const id = getInternalTunnelId(tun);
+        const itunnel = getInternalTunnel(id);
+
+        if (!itunnel) continue;
+        itunnel.url = url;
+    }
+}
+
+const transplantTunnel = async function (dispatcher) {
+    if (this.pendingTransplant) {
+        await this.pendingTransplant;
+        return;
+    }
+
+    let finished;
+    this.pendingTransplant = new Promise(r => finished = r);
+
+    try {
+        const handler = await import(`../processing/services/${this.service}.js`);
+        const response = await handler.default({
+            ...this.originalRequest,
+            dispatcher
+        });
+
+        if (!response.urls) {
+            return;
+        }
+
+        response.urls = [response.urls].flat();
+        if (this.originalRequest.isAudioOnly && response.urls.length > 1) {
+            response.urls = [response.urls[1]];
+        } else if (this.originalRequest.isAudioMuted) {
+            response.urls = [response.urls[0]];
+        }
+
+        const tunnels = [this.urls].flat();
+        if (tunnels.length !== response.urls.length) {
+            return;
+        }
+
+        transplantInternalTunnels(tunnels, response.urls);
+    }
+    catch {}
+    finally {
+        finished();
+        delete this.pendingTransplant;
+    }
+}
+
 function wrapStream(streamInfo) {
     const url = streamInfo.urls;
 
+    if (streamInfo.originalRequest) {
+        streamInfo.transplant = transplantTunnel.bind(streamInfo);
+    }
+
     if (typeof url === 'string') {
         streamInfo.urls = createInternalStream(url, streamInfo);
     } else if (Array.isArray(url)) {
@@ -143,13 +273,21 @@ function wrapStream(streamInfo) {
         }
     } else throw 'invalid urls';
 
+    if (streamInfo.subtitles) {
+        streamInfo.subtitles = createInternalStream(
+            streamInfo.subtitles,
+            streamInfo,
+            /*isSubtitles=*/true
+        );
+    }
+
     return streamInfo;
 }
 
-export function verifyStream(id, hmac, exp, secret, iv) {
+export async function verifyStream(id, hmac, exp, secret, iv) {
     try {
-        const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt);
-        const cache = streamCache.get(id.toString());
+        const ghmac = hashHmac(`${id},${exp},${iv},${secret}`, 'stream').toString('base64url');
+        const cache = await streamCache.get(id.toString());
 
         if (ghmac !== String(hmac)) return { status: 401 };
         if (!cache) return { status: 404 };

api/src/stream/proxy.js

@@ -0,0 +1,43 @@
+import { Agent, request } from "undici";
+import { create as contentDisposition } from "content-disposition-header";
+
+import { destroyInternalStream } from "./manage.js";
+import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js";
+
+const defaultAgent = new Agent();
+
+export default async function (streamInfo, res) {
+    const abortController = new AbortController();
+    const shutdown = () => (
+        closeRequest(abortController),
+        closeResponse(res),
+        destroyInternalStream(streamInfo.urls)
+    );
+
+    try {
+        res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');
+        res.setHeader('Content-disposition', contentDisposition(streamInfo.filename));
+
+        const { body: stream, headers, statusCode } = await request(streamInfo.urls, {
+            headers: {
+                ...getHeaders(streamInfo.service),
+                Range: streamInfo.range
+            },
+            signal: abortController.signal,
+            maxRedirections: 16,
+            dispatcher: defaultAgent,
+        });
+
+        res.status(statusCode);
+
+        for (const headerName of ['accept-ranges', 'content-type', 'content-length']) {
+            if (headers[headerName]) {
+                res.setHeader(headerName, headers[headerName]);
+            }
+        }
+
+        pipe(stream, res, shutdown);
+    } catch {
+        shutdown();
+    }
+}

api/src/stream/shared.js

@@ -1,4 +1,7 @@
 import { genericUserAgent } from "../config.js";
+import { vkClientAgent } from "../processing/services/vk.js";
+import { getInternalTunnelFromURL } from "./manage.js";
+import { probeInternalTunnel } from "./internal.js";
 
 const defaultHeaders = {
     'user-agent': genericUserAgent
@@ -13,6 +16,12 @@ const serviceHeaders = {
         origin: 'https://www.youtube.com',
         referer: 'https://www.youtube.com',
         DNT: '?1'
+    },
+    vk: {
+        'user-agent': vkClientAgent
+    },
+    tiktok: {
+        referer: 'https://www.tiktok.com/',
     }
 }
 
@@ -43,3 +52,40 @@ export function pipe(from, to, done) {
 
     from.pipe(to);
 }
+
+export async function estimateTunnelLength(streamInfo, multiplier = 1.1) {
+    let urls = streamInfo.urls;
+    if (!Array.isArray(urls)) {
+        urls = [ urls ];
+    }
+
+    const internalTunnels = urls.map(getInternalTunnelFromURL);
+    if (internalTunnels.some(t => !t))
+        return -1;
+
+    const sizes = await Promise.all(internalTunnels.map(probeInternalTunnel));
+    const estimatedSize = sizes.reduce(
+        // if one of the sizes is missing, let's just make a very
+        // bold guess that it's the same size as the existing one
+        (acc, cur) => cur <= 0 ? acc * 2 : acc + cur,
+        0
+    );
+
+    if (isNaN(estimatedSize) || estimatedSize <= 0) {
+        return -1;
+    }
+
+    return Math.floor(estimatedSize * multiplier);
+}
+
+export function estimateAudioMultiplier(streamInfo) {
+    if (streamInfo.audioFormat === 'wav') {
+        return 1411 / 128;
+    }
+
+    if (streamInfo.audioCopy) {
+        return 1;
+    }
+
+    return streamInfo.audioBitrate / 128;
+}

api/src/stream/stream.js

@@ -1,4 +1,5 @@
-import stream from "./types.js";
+import proxy from "./proxy.js";
+import ffmpeg from "./ffmpeg.js";
 
 import { closeResponse } from "./shared.js";
 import { internalStream } from "./internal.js";
@@ -7,23 +8,21 @@ export default async function(res, streamInfo) {
     try {
         switch (streamInfo.type) {
             case "proxy":
-                return await stream.proxy(streamInfo, res);
+                return await proxy(streamInfo, res);
 
             case "internal":
-                return internalStream(streamInfo, res);
+                return await internalStream(streamInfo.data, res);
 
             case "merge":
-                return stream.merge(streamInfo, res);
-
             case "remux":
             case "mute":
-                return stream.remux(streamInfo, res);
+                return await ffmpeg.remux(streamInfo, res);
 
             case "audio":
-                return stream.convertAudio(streamInfo, res);
+                return await ffmpeg.convertAudio(streamInfo, res);
 
             case "gif":
-                return stream.convertGif(streamInfo, res);
+                return await ffmpeg.convertGif(streamInfo, res);
         }
 
         closeResponse(res);

api/src/stream/types.js

@@ -1,311 +0,0 @@
-import { request } from "undici";
-import ffmpeg from "ffmpeg-static";
-import { spawn } from "child_process";
-import { create as contentDisposition } from "content-disposition-header";
-
-import { env } from "../config.js";
-import { metadataManager } from "../misc/utils.js";
-import { destroyInternalStream } from "./manage.js";
-import { hlsExceptions } from "../processing/service-config.js";
-import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js";
-
-const ffmpegArgs = {
-    webm: ["-c:v", "copy", "-c:a", "copy"],
-    mp4: ["-c:v", "copy", "-c:a", "copy", "-movflags", "faststart+frag_keyframe+empty_moov"],
-    m4a: ["-movflags", "frag_keyframe+empty_moov"],
-    gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"]
-}
-
-const toRawHeaders = (headers) => {
-    return Object.entries(headers)
-                 .map(([key, value]) => `${key}: ${value}\r\n`)
-                 .join('');
-}
-
-const killProcess = (p) => {
-    p?.kill('SIGTERM'); // ask the process to terminate itself gracefully
-
-    setTimeout(() => {
-        if (p?.exitCode === null)
-            p?.kill('SIGKILL'); // brutally murder the process if it didn't quit
-    }, 5000);
-}
-
-const getCommand = (args) => {
-    if (typeof env.processingPriority === 'number' && !isNaN(env.processingPriority)) {
-        return ['nice', ['-n', env.processingPriority.toString(), ffmpeg, ...args]]
-    }
-    return [ffmpeg, args]
-}
-
-const proxy = async (streamInfo, res) => {
-    const abortController = new AbortController();
-    const shutdown = () => (
-        closeRequest(abortController),
-        closeResponse(res),
-        destroyInternalStream(streamInfo.urls)
-    );
-
-    try {
-        res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');
-        res.setHeader('Content-disposition', contentDisposition(streamInfo.filename));
-
-        const { body: stream, headers, statusCode } = await request(streamInfo.urls, {
-            headers: {
-                ...getHeaders(streamInfo.service),
-                Range: streamInfo.range
-            },
-            signal: abortController.signal,
-            maxRedirections: 16
-        });
-
-        res.status(statusCode);
-
-        for (const headerName of ['accept-ranges', 'content-type', 'content-length']) {
-            if (headers[headerName]) {
-                res.setHeader(headerName, headers[headerName]);
-            }
-        }
-
-        pipe(stream, res, shutdown);
-    } catch {
-        shutdown();
-    }
-}
-
-const merge = (streamInfo, res) => {
-    let process;
-    const shutdown = () => (
-        killProcess(process),
-        closeResponse(res),
-        streamInfo.urls.map(destroyInternalStream)
-    );
-
-    const headers = getHeaders(streamInfo.service);
-    const rawHeaders = toRawHeaders(headers);
-
-    try {
-        if (streamInfo.urls.length !== 2) return shutdown();
-
-        const format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1];
-
-        let args = [
-            '-loglevel', '-8',
-            '-headers', rawHeaders,
-            '-i', streamInfo.urls[0],
-            '-headers', rawHeaders,
-            '-i', streamInfo.urls[1],
-            '-map', '0:v',
-            '-map', '1:a',
-        ]
-
-        args = args.concat(ffmpegArgs[format]);
-
-        if (hlsExceptions.includes(streamInfo.service)) {
-            args.push('-bsf:a', 'aac_adtstoasc')
-        }
-
-        if (streamInfo.metadata) {
-            args = args.concat(metadataManager(streamInfo.metadata))
-        }
-
-        args.push('-f', format, 'pipe:3');
-
-        process = spawn(...getCommand(args), {
-            windowsHide: true,
-            stdio: [
-                'inherit', 'inherit', 'inherit',
-                'pipe'
-            ],
-        });
-
-        const [,,, muxOutput] = process.stdio;
-
-        res.setHeader('Connection', 'keep-alive');
-        res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename));
-
-        pipe(muxOutput, res, shutdown);
-
-        process.on('close', shutdown);
-        res.on('finish', shutdown);
-    } catch {
-        shutdown();
-    }
-}
-
-const remux = (streamInfo, res) => {
-    let process;
-    const shutdown = () => (
-        killProcess(process),
-        closeResponse(res),
-        destroyInternalStream(streamInfo.urls)
-    );
-
-    try {
-        let args = [
-            '-loglevel', '-8',
-            '-headers', toRawHeaders(getHeaders(streamInfo.service)),
-        ]
-
-        if (streamInfo.service === "twitter") {
-            args.push('-seekable', '0')
-        }
-
-        args.push(
-            '-i', streamInfo.urls,
-            '-c:v', 'copy',
-        )
-
-        if (streamInfo.type === "mute") {
-            args.push('-an');
-        }
-
-        if (hlsExceptions.includes(streamInfo.service)) {
-            if (streamInfo.type !== "mute") {
-                args.push('-c:a', 'aac')
-            }
-            args.push('-bsf:a', 'aac_adtstoasc');
-        }
-
-        let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1];
-        if (format === "mp4") {
-            args.push('-movflags', 'faststart+frag_keyframe+empty_moov')
-        }
-
-        args.push('-f', format, 'pipe:3');
-
-        process = spawn(...getCommand(args), {
-            windowsHide: true,
-            stdio: [
-                'inherit', 'inherit', 'inherit',
-                'pipe'
-            ],
-        });
-
-        const [,,, muxOutput] = process.stdio;
-
-        res.setHeader('Connection', 'keep-alive');
-        res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename));
-
-        pipe(muxOutput, res, shutdown);
-
-        process.on('close', shutdown);
-        res.on('finish', shutdown);
-    } catch {
-        shutdown();
-    }
-}
-
-const convertAudio = (streamInfo, res) => {
-    let process;
-    const shutdown = () => (
-        killProcess(process),
-        closeResponse(res),
-        destroyInternalStream(streamInfo.urls)
-    );
-
-    try {
-        let args = [
-            '-loglevel', '-8',
-            '-headers', toRawHeaders(getHeaders(streamInfo.service)),
-        ]
-
-        if (streamInfo.service === "twitter") {
-            args.push('-seekable', '0');
-        }
-
-        args.push(
-            '-i', streamInfo.urls,
-            '-vn'
-        )
-
-        if (streamInfo.audioCopy) {
-            args.push("-c:a", "copy")
-        } else {
-            args.push("-b:a", `${streamInfo.audioBitrate}k`)
-        }
-
-        if (streamInfo.audioFormat === "mp3" && streamInfo.audioBitrate === "8") {
-            args.push("-ar", "12000");
-        }
-
-        if (streamInfo.audioFormat === "opus") {
-            args.push("-vbr", "off")
-        }
-
-        if (ffmpegArgs[streamInfo.audioFormat]) {
-            args = args.concat(ffmpegArgs[streamInfo.audioFormat])
-        }
-
-        if (streamInfo.metadata) {
-            args = args.concat(metadataManager(streamInfo.metadata))
-        }
-
-        args.push('-f', streamInfo.audioFormat === "m4a" ? "ipod" : streamInfo.audioFormat, 'pipe:3');
-
-        process = spawn(...getCommand(args), {
-            windowsHide: true,
-            stdio: [
-                'inherit', 'inherit', 'inherit',
-                'pipe'
-            ],
-        });
-
-        const [,,, muxOutput] = process.stdio;
-
-        res.setHeader('Connection', 'keep-alive');
-        res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename));
-
-        pipe(muxOutput, res, shutdown);
-        res.on('finish', shutdown);
-    } catch {
-        shutdown();
-    }
-}
-
-const convertGif = (streamInfo, res) => {
-    let process;
-    const shutdown = () => (killProcess(process), closeResponse(res));
-
-    try {
-        let args = [
-            '-loglevel', '-8'
-        ]
-
-        if (streamInfo.service === "twitter") {
-            args.push('-seekable', '0')
-        }
-
-        args.push('-i', streamInfo.urls);
-        args = args.concat(ffmpegArgs.gif);
-        args.push('-f', "gif", 'pipe:3');
-
-        process = spawn(...getCommand(args), {
-            windowsHide: true,
-            stdio: [
-                'inherit', 'inherit', 'inherit',
-                'pipe'
-            ],
-        });
-
-        const [,,, muxOutput] = process.stdio;
-
-        res.setHeader('Connection', 'keep-alive');
-        res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename.split('.')[0] + ".gif"));
-
-        pipe(muxOutput, res, shutdown);
-
-        process.on('close', shutdown);
-        res.on('finish', shutdown);
-    } catch {
-        shutdown();
-    }
-}
-
-export default {
-    proxy,
-    merge,
-    remux,
-    convertAudio,
-    convertGif,
-}

api/src/util/generate-jwt-secret.js

@@ -0,0 +1,22 @@
+// run with `pnpm -r token:jwt`
+
+const makeSecureString = (length = 64) => {
+    const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-';
+    const out = [];
+
+    while (out.length < length) {
+        for (const byte of crypto.getRandomValues(new Uint8Array(length))) {
+            if (byte < alphabet.length) {
+                out.push(alphabet[byte]);
+            }
+
+            if (out.length === length) {
+                break;
+            }
+        }
+    }
+
+    return out.join('');
+}
+
+console.log(`JWT_SECRET: ${JSON.stringify(makeSecureString(64))}`)

api/src/util/generate-youtube-tokens.js

@@ -1,38 +0,0 @@
-import { Innertube } from 'youtubei.js';
-import { Red } from '../misc/console-text.js'
-
-const bail = (...msg) => {
-    console.error(...msg);
-    throw new Error(msg);
-};
-
-const tube = await Innertube.create();
-
-tube.session.once(
-    'auth-pending',
-    ({ verification_url, user_code }) => {
-        console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`);
-        console.log(`    By using this token, you are risking your Google account getting terminated.`);
-        console.log(`    You should ${Red('NOT')} use your personal account!`);
-        console.log();
-        console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`);
-    }
-);
-
-tube.session.once('auth-error', (err) => bail('An error occurred:', err));
-tube.session.once('auth', ({ credentials }) => {
-    if (!credentials.access_token) {
-        bail('something went wrong');
-    }
-
-    console.log(
-        'add this cookie to the youtube_oauth array in your cookies file:',
-        JSON.stringify(
-            Object.entries(credentials)
-                .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`)
-                .join('; ')
-        )
-    );
-});
-
-await tube.session.signIn();

api/src/util/setup.js

@@ -1,105 +0,0 @@
-import { existsSync, unlinkSync, appendFileSync } from "fs";
-import { createInterface } from "readline";
-import { Cyan, Bright } from "./misc/console-text.js";
-import { loadJSON } from "./misc/load-from-fs.js";
-import { execSync } from "child_process";
-
-const { version } = loadJSON("./package.json");
-
-let envPath = './.env';
-let q = `${Cyan('?')} \x1b[1m`;
-let ob = {};
-let rl = createInterface({ input: process.stdin, output: process.stdout });
-
-let final = () => {
-    if (existsSync(envPath)) unlinkSync(envPath);
-
-    for (let i in ob) {
-        appendFileSync(envPath, `${i}=${ob[i]}\n`)
-    }
-    console.log(Bright("\nAwesome! I've created a fresh .env file for you."));
-    console.log(`${Bright("Now I'll run")} ${Cyan("npm install")} ${Bright("to install all dependencies. It shouldn't take long.\n\n")}`);
-    execSync('npm install', { stdio: [0, 1, 2] });
-    console.log(`\n\n${Cyan("All done!\n")}`);
-    console.log(Bright("You can re-run this script at any time to update the configuration."));
-    console.log(Bright("\nYou're now ready to start cobalt. Simply run ") + Cyan("npm start") + Bright('!\nHave fun :)'));
-    rl.close()
-}
-
-console.log(
-    `${Cyan(`Hey, this is cobalt v.${version}!`)}\n${Bright("Let's start by creating a new ")}${Cyan(".env")}${Bright(" file. You can always change it later.")}`
-)
-
-function setup() {
-    console.log(Bright("\nWhat kind of server will this instance be?\nOptions: api, web."));
-
-    rl.question(q, r1 => {
-        switch (r1.toLowerCase()) {
-            case 'api':
-                console.log(Bright("\nCool! What's the domain this API instance will be running on? (localhost)\nExample: api.cobalt.tools"));
-
-                rl.question(q, apiURL => {
-                    ob.API_URL = `http://localhost:9000/`;
-                    ob.API_PORT = 9000;
-                    if (apiURL && apiURL !== "localhost") ob.API_URL = `https://${apiURL.toLowerCase()}/`;
-
-                    console.log(Bright("\nGreat! Now, what port will it be running on? (9000)"));
-
-                    rl.question(q, apiPort => {
-                        if (apiPort) ob.API_PORT = apiPort;
-                        if (apiPort && (apiURL === "localhost" || !apiURL)) ob.API_URL = `http://localhost:${apiPort}/`;
-
-                        console.log(Bright("\nWhat will your instance's name be? Usually it's something like eu-nl aka region-country. (local)"));
-
-                        rl.question(q, apiName => {
-                            ob.API_NAME = apiName.toLowerCase();
-                            if (!apiName || apiName === "local") ob.API_NAME = "local";
-
-                            console.log(Bright("\nOne last thing: would you like to enable CORS? It allows other websites and extensions to use your instance's API.\ny/n (n)"));
-
-                            rl.question(q, apiCors => {
-                                let answCors = apiCors.toLowerCase().trim();
-                                if (answCors !== "y" && answCors !== "yes") ob.CORS_WILDCARD = '0'
-                                final()
-                            })
-                        })
-                    });
-    
-                })
-                break;
-            case 'web':
-                console.log(Bright("\nAwesome! What's the domain this web app instance will be running on? (localhost)\nExample: cobalt.tools"));
-    
-                rl.question(q, webURL => {
-                    ob.WEB_URL = `http://localhost:9001/`;
-                    ob.WEB_PORT = 9001;
-                    if (webURL && webURL !== "localhost") ob.WEB_URL = `https://${webURL.toLowerCase()}/`;
-    
-                    console.log(
-                        Bright("\nGreat! Now, what port will it be running on? (9001)")
-                    )
-                    rl.question(q, webPort => {
-                        if (webPort) ob.WEB_PORT = webPort;
-                        if (webPort && (webURL === "localhost" || !webURL)) ob.WEB_URL = `http://localhost:${webPort}/`;
-
-                        console.log(
-                            Bright("\nOne last thing: what default API domain should be used? (api.cobalt.tools)\nIf it's hosted locally, make sure to include the port:") + Cyan(" localhost:9000")
-                        );
-
-                        rl.question(q, apiURL => {
-                            ob.API_URL = `https://${apiURL.toLowerCase()}/`;
-                            if (apiURL.includes(':')) ob.API_URL = `http://${apiURL.toLowerCase()}/`;
-                            if (!apiURL) ob.API_URL = "https://api.cobalt.tools/";
-                            final()
-                        })
-                    });
-    
-                });
-                break;
-            default:
-                console.log(Bright("\nThis is not an option. Try again."));
-                setup()
-        }
-    })
-}
-setup()

api/src/util/test-ci.js

@@ -1,82 +0,0 @@
-import { env } from "../config.js";
-import { runTest } from "../misc/run-test.js";
-import { loadJSON } from "../misc/load-from-fs.js";
-import { Red, Bright } from "../misc/console-text.js";
-import { randomizeCiphers } from "../misc/randomize-ciphers.js";
-
-import { services } from "../processing/service-config.js";
-
-const tests = loadJSON('./src/util/tests.json');
-
-// services that are known to frequently fail due to external
-// factors (e.g. rate limiting)
-const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube'])
-
-const action = process.argv[2];
-switch (action) {
-    case "get-services":
-        const fromConfig = Object.keys(services);
-
-        const missingTests = fromConfig.filter(
-            service => !tests[service] || tests[service].length === 0
-        );
-
-        if (missingTests.length) {
-            console.error('services have no tests:', missingTests);
-            console.log('[]');
-            process.exitCode = 1;
-            break;
-        }
-
-        console.log(JSON.stringify(fromConfig));
-        break;
-
-    case "run-tests-for":
-        const service = process.argv[3];
-        let failed = false;
-
-        if (!tests[service]) {
-            console.error('no such service:', service);
-        }
-
-        env.streamLifespan = 10000;
-        env.apiURL = 'http://x';
-        randomizeCiphers();
-
-        for (const test of tests[service]) {
-            const { name, url, params, expected } = test;
-            const canFail = test.canFail || finnicky.has(service);
-
-            try {
-                await runTest(url, params, expected);
-                console.log(`${service}/${name}: ok`);
-
-            } catch(e) {
-                failed = !canFail;
-
-                let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
-                if (canFail && process.env.GITHUB_ACTION) {
-                    console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
-                }
-
-                console.error(`${service}/${name}: ${failText}`);
-                const errorString = e.toString().split('\n');
-                let c = '┃';
-                errorString.forEach((line, index) => {
-                    line = line.replace('!=', Red('!='));
-
-                    if (index === errorString.length - 1) {
-                        c = '┗';
-                    }
-
-                    console.error(`   ${c}`, line);
-                });
-            }
-        }
-
-        process.exitCode = Number(failed);
-        break;
-    default:
-        console.error('invalid action:', action);
-        process.exitCode = 1;
-}

api/src/util/test.js

@@ -1,84 +1,136 @@
-import "dotenv/config";
+import path from "node:path";
+
+import { env } from "../config.js";
+import { runTest } from "../misc/run-test.js";
+import { loadJSON } from "../misc/load-from-fs.js";
+import { Red, Bright } from "../misc/console-text.js";
+import { setGlobalDispatcher, EnvHttpProxyAgent, ProxyAgent } from "undici";
+import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 
 import { services } from "../processing/service-config.js";
-import { extract } from "../processing/url.js";
-import match from "../processing/match.js";
-import { loadJSON } from "../misc/load-from-fs.js";
-import { normalizeRequest } from "../processing/request.js";
-import { env } from "../config.js";
 
-env.apiURL = 'http://localhost:9000'
-let tests = loadJSON('./src/util/tests.json');
+const getTestPath = service => path.join('./src/util/tests/', `./${service}.json`);
+const getTests = (service) => loadJSON(getTestPath(service));
 
-let noTest = [];
-let failed = [];
-let success = 0;
+// services that are known to frequently fail due to external
+// factors (e.g. rate limiting)
+const finnicky = new Set(
+    process.env.TEST_IGNORE_SERVICES
+    ? process.env.TEST_IGNORE_SERVICES.split(',')
+    : ['bilibili', 'instagram', 'facebook', 'youtube', 'vk', 'twitter', 'reddit']
+);
 
-function addToFail(service, testName, url, status, response) {
-    failed.push({
-        service: service,
-        name: testName,
-        url: url,
-        status: status,
-        response: response
-    })
-}
-for (let i in services) {
-    if (tests[i]) {
-        console.log(`\nRunning tests for ${i}...\n`)
-        for (let k = 0; k < tests[i].length; k++) {
-            let test = tests[i][k];
+const runTestsFor = async (service) => {
+    const tests = getTests(service);
+    let softFails = 0, fails = 0;
 
-            console.log(`Running test ${k+1}: ${test.name}`);
-            console.log('params:');
-            let params = {...{url: test.url}, ...test.params};
-            console.log(params);
-
-            let chck = await normalizeRequest(params);
-            if (chck.success) {
-                chck = chck.data;
-
-                const parsed = extract(chck.url);
-                if (parsed === null) {
-                    throw `Invalid URL: ${chck.url}`
-                }
-
-                let j = await match({
-                    host: parsed.host,
-                    patternMatch: parsed.patternMatch,
-                    params: chck,
-                });
-                console.log('\nReceived:');
-                console.log(j)
-                if (j.status === test.expected.code && j.body.status === test.expected.status) {
-                    console.log("\n✅ Success.\n");
-                    success++
-                } else {
-                    console.log(`\n❌ Fail. Expected: ${test.expected.code} & ${test.expected.status}, received: ${j.status} & ${j.body.status}\n`);
-                    addToFail(i, test.name, test.url, j.body.status, j)
-                }
-            } else {
-                console.log("\n❌ couldn't validate the request JSON.\n");
-                addToFail(i, test.name, test.url, "unknown", {})
-            }
-        }
-        console.log("\n\n")
-    } else {
-        console.warn(`No tests found for ${i}.`);
-        noTest.push(i)
+    if (!tests) {
+        throw "no such service: " + service;
     }
+
+    for (const test of tests) {
+        const { name, url, params, expected } = test;
+        const canFail = test.canFail || finnicky.has(service);
+
+        try {
+            await runTest(url, params, expected);
+            console.log(`${service}/${name}: ok`);
+
+        } catch (e) {
+            softFails += !canFail;
+            fails++;
+
+            let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
+            if (canFail && process.env.GITHUB_ACTION) {
+                console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
+            }
+
+            console.error(`${service}/${name}: ${failText}`);
+            const errorString = e.toString().split('\n');
+            let c = '┃';
+            errorString.forEach((line, index) => {
+                line = line.replace('!=', Red('!='));
+
+                if (index === errorString.length - 1) {
+                    c = '┗';
+                }
+
+                console.error(`   ${c}`, line);
+            });
+        }
+    }
+
+    return { fails, softFails };
 }
 
-console.log(`✅ ${success} tests succeeded.`);
-console.log(`❌ ${failed.length} tests failed.`);
-console.log(`❔ ${noTest.length} services weren't tested.`);
-
-if (failed.length > 0) {
-    console.log(`\nFailed tests:`);
-    console.log(failed)
+const printHeader = (service, padLen) => {
+    const padding = padLen - service.length;
+    service = service.padEnd(1 + service.length + padding, ' ');
+    console.log(service + '='.repeat(50));
 }
 
-if (noTest.length > 0) {
-    console.log(`\nMissing tests:`);
-    console.log(noTest)
+// TODO: remove env.externalProxy in a future version
+setGlobalDispatcher(
+    new EnvHttpProxyAgent({ httpProxy: env.externalProxy || undefined })
+);
+
+env.streamLifespan = 10000;
+env.apiURL = 'http://x/';
+randomizeCiphers();
+
+const action = process.argv[2];
+switch (action) {
+    case "get-services":
+        const fromConfig = Object.keys(services);
+
+        const missingTests = fromConfig.filter(
+            service => {
+                const tests = getTests(service);
+                return !tests || tests.length === 0
+            }
+        );
+
+        if (missingTests.length) {
+            console.error('services have no tests:', missingTests);
+            process.exitCode = 1;
+            break;
+        }
+
+        console.log(JSON.stringify(fromConfig));
+        break;
+
+    case "run-tests-for":
+
+        try {
+            const { softFails } = await runTestsFor(process.argv[3]);
+            process.exitCode = Number(!!softFails);
+        } catch (e) {
+            console.error(e);
+            process.exitCode = 1;
+            break;
+        }
+
+        break;
+    default:
+        const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0);
+        const failCounters = {};
+
+        for (const service in services) {
+            printHeader(service, maxHeaderLen);
+            const { fails, softFails } = await runTestsFor(service);
+            failCounters[service] = fails;
+            console.log();
+
+            if (!process.exitCode && softFails)
+                process.exitCode = 1;
+        }
+
+        console.log('='.repeat(50 + maxHeaderLen));
+        console.log(
+            Bright('total fails:'),
+            Object.values(failCounters).reduce((a, b) => a + b)
+        );
+        for (const [ service, fails ] of Object.entries(failCounters)) {
+            if (fails) console.log(`${Bright(service)} fails: ${fails}`);
+        }
 }

api/src/util/tests/bilibili.json

@@ -0,0 +1,69 @@
+[
+    {
+        "name": "1080p video",
+        "url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p video muted",
+        "url": "https://www.bilibili.com/video/BV18i4y1m7xV/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p vertical video",
+        "url": "https://www.bilibili.com/video/BV1uu411z7VV/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p vertical video muted",
+        "url": "https://www.bilibili.com/video/BV1uu411z7VV/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "b23.tv shortlink",
+        "url": "https://b23.tv/av32430100",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "bilibili.tv link",
+        "url": "https://www.bilibili.tv/en/video/4789599404426256",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "bilibili.com link with part id",
+        "url": "https://www.bilibili.com/video/BV1uo4y1K72s?spm_id_from=333.788.videopod.episodes&p=6",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/bsky.json

@@ -0,0 +1,96 @@
+[
+    {
+        "name": "horizontal video",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3giwtwp222m",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "horizontal video, recordWithMedia",
+        "url": "https://bsky.app/profile/did:plc:ywbm3iywnhzep3ckt6efhoh7/post/3l3wonhk23g2i",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (muted)",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (audio)",
+        "url": "https://bsky.app/profile/did:plc:oisofpd7lj26yvgiivf3lxsi/post/3l3jhpomhjk2m",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "single image",
+        "url": "https://bsky.app/profile/did:plc:k4a7d65fcyevbrnntjxh57go/post/3l33flpoygt26",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "gif with a quoted post",
+        "url": "https://bsky.app/profile/imlunahey.com/post/3lgajpn5dtk2t",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "gif alone in a post",
+        "url": "https://bsky.app/profile/imlunahey.com/post/3lgah3ovxnc2q",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "several images",
+        "url": "https://bsky.app/profile/did:plc:rai7s6su2sy22ss7skouedl7/post/3kzxuxbiul626",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "deleted post/invalid user",
+        "url": "https://bsky.app/profile/notreal.bsky.team/post/3l2udah76ch2c",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/dailymotion.json

@@ -0,0 +1,29 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://www.dailymotion.com/video/x8t1eho",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private video",
+        "url": "https://www.dailymotion.com/video/k41fZWpx2TaAORA2nok",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "dai.ly shortened link",
+        "url": "https://dai.ly/k41fZWpx2TaAORA2nok",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/facebook.json

@@ -0,0 +1,65 @@
+[
+    {
+        "name": "direct video with username and id",
+        "url": "https://web.facebook.com/100071784061914/videos/588631943886661/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "direct video with id as query param",
+        "url": "https://web.facebook.com/watch/?v=883839773514682&ref=sharing",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "direct video with caption",
+        "url": "https://web.facebook.com/wood57/videos/𝐒𝐞𝐛𝐚𝐬𝐤𝐨𝐦-𝐟𝐮𝐥𝐥/883839773514682",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shortlink video",
+        "url": "https://fb.watch/r1K6XHMfGT/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "reel video",
+        "url": "https://web.facebook.com/reel/730293269054758",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shared video link",
+        "url": "https://www.facebook.com/share/v/6EJK4Z8EAEAHtz8K/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shared video link v2",
+        "url": "https://web.facebook.com/share/r/JFZfPVgLkiJQmWrr/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }
+]

api/src/util/tests/instagram.json

@@ -0,0 +1,134 @@
+[
+    {
+        "name": "single photo post",
+        "url": "https://www.instagram.com/p/DFx6KVduFWy/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "various picker (photos + video)",
+        "url": "https://www.instagram.com/p/CvYrSgnsKjv/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "reel",
+        "url": "https://www.instagram.com/reel/DFQe23tOWKz/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "regular video",
+        "url": "https://www.instagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "reel (isAudioOnly)",
+        "url": "https://www.instagram.com/reel/DFQe23tOWKz/",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "reel (isAudioMuted)",
+        "url": "https://www.instagram.com/reel/DFQe23tOWKz/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent reel",
+        "url": "https://www.instagram.com/reel/XXXXXXXXXX/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "inexistent post",
+        "url": "https://www.instagram.com/p/XXXXXXXXXX/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "post info in an array (for whatever reason??)",
+        "url": "https://www.instagram.com/reel/CrVB9tatUDv/?igshid=blaBlABALALbLABULLSHIT==",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "prone to get rate limited",
+        "url": "https://www.instagram.com/reel/CrO-T7Qo6rq/?igshid=fuckYouNoTrackingIdForYou==",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "ddinstagram link",
+        "url": "https://ddinstagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "d.ddinstagram.com link",
+        "url": "https://d.ddinstagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "g.ddinstagram.com link",
+        "url": "https://g.ddinstagram.com/p/CmCVWoIr9OH/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "private instagram post",
+        "url": "https://www.instagram.com/p/C5_A1TQNPrYw4c2g9KAUTPUl8RVHqiAdAcOOSY0",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error",
+            "errorCode": "error.api.content.post.private"
+        }
+    }
+]

api/src/util/tests/loom.json

@@ -0,0 +1,60 @@
+[
+    {
+        "name": "1080p video",
+        "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "1080p video (muted)",
+        "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "1080p video (audio only)",
+        "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "video with no transcodedUrl",
+        "url": "https://www.loom.com/share/aa3d8b08bee74d05af5b42989e9f33e9",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "video with title in url",
+        "url": "https://www.loom.com/share/Meet-AI-workflows-aa3d8b08bee74d05af5b42989e9f33e9",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "video with title in url (2)",
+        "url": "https://www.loom.com/share/Unlocking-Incredible-Organizational-Velocity-with-Async-Video-4a2a8baf124c4390954dcbb46a58cfd7",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }
+]

api/src/util/tests/newgrounds.json

@@ -0,0 +1,42 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://www.newgrounds.com/portal/view/938050",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (audio only)",
+        "url": "https://www.newgrounds.com/portal/view/938050",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (muted)",
+        "url": "https://www.newgrounds.com/portal/view/938050",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular music",
+        "url": "https://www.newgrounds.com/audio/listen/500476",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/ok.json

@@ -0,0 +1,11 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://ok.ru/video/7204071410346",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/pinterest.json

@@ -0,0 +1,97 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://www.pinterest.com/pin/70437485604616/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "invalid link",
+        "url": "https://www.pinterest.com/pin/eeeeeee/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error",
+            "errorCode": "error.api.fetch.empty"
+        }
+    },
+    {
+        "name": "regular video (isAudioOnly)",
+        "url": "https://www.pinterest.com/pin/70437485604616/",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (isAudioMuted)",
+        "url": "https://www.pinterest.com/pin/70437485604616/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (.ca TLD)",
+        "url": "https://www.pinterest.ca/pin/70437485604616/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "story",
+        "url": "https://www.pinterest.com/pin/gadget-cool-products-amazon-product-technology-kitchen-gadgets--1084663891475263837/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "regular picture",
+        "url": "https://www.pinterest.com/pin/412994228343400946/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular picture (.ca TLD)",
+        "url": "https://www.pinterest.ca/pin/412994228343400946/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular gif",
+        "url": "https://www.pinterest.com/pin/643170390530326178/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular gif (.ca TLD)",
+        "url": "https://www.pinterest.ca/pin/643170390530326178/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/reddit.json

@@ -0,0 +1,78 @@
+[
+    {
+        "name": "video with audio",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video with audio (isAudioOnly)",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video with audio (isAudioMuted)",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/wup1fg/id_be_escaping_at_the_first_chance_i_got/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video without audio",
+        "url": "https://www.reddit.com/r/catvideos/comments/ftoeo7/luna_doesnt_want_to_be_bothered_while_shes_napping/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "actual gif, not looping video",
+        "url": "https://www.reddit.com/r/whenthe/comments/109wqy1/god_really_did_some_trolling/?utm_source=share&utm_medium=web2x&context=3",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "different audio link, live render",
+        "url": "https://www.reddit.com/r/TikTokCringe/comments/15hce91/asian_daddy_kink/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "shortened video link",
+        "url": "https://v.redd.it/ifg2emt5ck0e1",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "shortened video link (alternative)",
+        "url": "https://reddit.com/video/ifg2emt5ck0e1",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/rutube.json

@@ -0,0 +1,100 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://rutube.ru/video/b2f6c27649907c2fde0af411b03825eb/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (isAudioMuted)",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "russian region lock",
+        "url": "https://rutube.ru/video/b521653b4f71ece57b8ff54e57ca9b82/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "vertical video",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "yappy",
+        "url": "https://rutube.ru/yappy/c8c32bf7aee04412837656ea26c2b25b/",
+        "canFail": true,
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "shorts",
+        "url": "https://rutube.ru/shorts/935c1afafd0e7d52836d671967d53dac/",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (isAudioOnly)",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "vertical video (isAudioMuted)",
+        "url": "https://rutube.ru/video/18a281399b96f9184c647455a86f6724/",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private video",
+        "url": "https://rutube.ru/video/private/1161415be0e686214bb2a498165cab3e/?p=_IL1G8RSnKutunnTYwhZ5A",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "region locked video, should fail",
+        "canFail": true,
+        "url": "https://rutube.ru/video/e7ac82708cc22bd068a3bf6a7004d1b1/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/snapchat.json

@@ -0,0 +1,29 @@
+[
+    {
+        "name": "spotlight",
+        "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "shortlinked spotlight",
+        "url": "https://t.snapchat.com/4ZsiBLDi",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "story",
+        "url": "https://www.snapchat.com/add/bazerkmakane",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    }
+]

api/src/util/tests/soundcloud.json

@@ -0,0 +1,107 @@
+[
+    {
+        "name": "public song (best)",
+        "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
+        "params": {
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "public song (mp3, isAudioMuted)",
+        "url": "https://soundcloud.com/l2share77/loona-butterfly?utm_source=clipboard&utm_medium=text&utm_campaign=social_sharing",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private song",
+        "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private song (wav, isAudioMuted)",
+        "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "wav"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "private song (ogg, isAudioMuted, isAudioOnly)",
+        "url": "https://soundcloud.com/user-798052861/asdasdasdsdsd/s-9TqZ7edLJ90",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "ogg"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "on.soundcloud link",
+        "url": "https://on.soundcloud.com/XHLLKSXRQ5yyGDuD9",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "on.soundcloud link, different stream type",
+        "url": "https://on.soundcloud.com/AG4c",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "no opus audio, fallback to mp3",
+        "url": "https://soundcloud.com/frums/credits",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "go+ song, should fail",
+        "canFail": true,
+        "url": "https://soundcloud.com/dualipa/physical-feat-troye-sivan",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "region locked song, should fail",
+        "canFail": true,
+        "url": "https://soundcloud.com/gotye/somebody-2024-feat-kimbra",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/streamable.json

@@ -0,0 +1,51 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://streamable.com/p9cln4",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "embedded link",
+        "url": "https://streamable.com/e/rsmo56",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "regular video (isAudioOnly)",
+        "url": "https://streamable.com/p9cln4",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "regular video (isAudioMuted)",
+        "url": "https://streamable.com/p9cln4",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent video",
+        "url": "https://streamable.com/XXXXXX",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/tiktok.json

@@ -0,0 +1,47 @@
+[
+    {
+        "name": "long link video",
+        "url": "https://www.tiktok.com/@fatfatmillycat/video/7195741644585454894",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "images",
+        "url": "https://www.tiktok.com/@matryoshk4/video/7231234675476532526",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "long link inexistent",
+        "url": "https://www.tiktok.com/@blablabla/video/7120851458451417478",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "short link inexistent",
+        "url": "https://vt.tiktok.com/2p4ewa7/",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "age restricted video",
+        "url": "https://www.tiktok.com/@.kyle.films/video/7415757181145877793",
+        "params": {},
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    }
+]

api/src/util/tests/tumblr.json

@@ -0,0 +1,49 @@
+[
+    {
+        "name": "at.tumblr link",
+        "url": "https://at.tumblr.com/music/704177038274281472/n7x7pr7x4w2b",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "user subdomain link",
+        "url": "https://garfield-69.tumblr.com/post/696499862852780032",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "web app link",
+        "url": "https://www.tumblr.com/rongzhi/707729381162958848/english-added-by-me?source=share",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "tumblr audio",
+        "url": "https://www.tumblr.com/zedneon/737815079301562368/zedneon-ft-mr-sauceman-tech-n9ne-speed-of?source=share",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "tumblr video converted to audio",
+        "url": "https://garfield-69.tumblr.com/post/696499862852780032",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    }
+]

api/src/util/tests/twitch.json

@@ -0,0 +1,42 @@
+[
+    {
+        "name": "clip",
+        "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "clip (isAudioOnly)",
+        "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {
+            "downloadMode": "audio"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "clip (isAudioMuted)",
+        "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {
+            "downloadMode": "mute"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "clip (mobile subdomain)",
+        "url": "https://m.twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }
+]

api/src/util/tests/twitter.json

@@ -0,0 +1,230 @@
+[
+    {
+        "name": "regular video",
+        "url": "https://twitter.com/X/status/1697304622749086011",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "video with mobile web mediaviewer",
+        "url": "https://twitter.com/X/status/1697304622749086011/mediaViewer?currentTweet=1697304622749086011&currentTweetUser=X&currentTweet=1697304622749086011&currentTweetUser=X",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "embedded twitter video",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "mixed media (image + gif)",
+        "url": "https://twitter.com/sky_mj26/status/1807756010712428565",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "picker: mixed media (video + image)",
+        "url": "https://x.com/PopCrave/status/1682176754792955905",
+        "params": {
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "audio from embedded twitter video (mp3, isAudioOnly)",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "audio from embedded twitter video (best, isAudioOnly)",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "audio from embedded twitter video (ogg, isAudioOnly, isAudioMuted)",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "audio",
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "muted embedded twitter video",
+        "url": "https://twitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "mp3"
+        },
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "retweeted video",
+        "url": "https://twitter.com/schlizzawg/status/1869017025055793405",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "age restricted video",
+        "url": "https://x.com/XSpaces/status/1526955853743546372",
+        "params": {},
+        "canFail": true,
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "twitter voice + x.com link",
+        "url": "https://x.com/eggsaladscreams/status/1693089534886506756?s=46",
+        "params": {},
+        "canFail": true,
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "vxtwitter link",
+        "url": "https://vxtwitter.com/dustbin_nie/status/1624596567188717568?s=20",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "post with 1 image",
+        "url": "https://x.com/PopCrave/status/1815960083475423235",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "post with 4 images",
+        "url": "https://x.com/PopCrave/status/1877880433242771717",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    },
+    {
+        "name": "retweeted video, isAudioOnly",
+        "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg",
+        "params": {
+            "downloadMode": "mute",
+            "audioFormat": "mp3"
+        },
+        "canFail": true,
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "gif",
+        "url": "https://x.com/thelastromances/status/1897839691212202479",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "inexistent post",
+        "url": "https://twitter.com/test/status/9487653",
+        "params": {
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "post with no media content",
+        "url": "https://twitter.com/elonmusk/status/1604617643973124097?s=20",
+        "params": {
+            "audioFormat": "best"
+        },
+        "expected": {
+            "code": 400,
+            "status": "error"
+        }
+    },
+    {
+        "name": "bookmarked video",
+        "url": "https://twitter.com/i/bookmarks?post_id=1828099210220294314",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    },
+    {
+        "name": "bookmarked photo",
+        "url": "https://twitter.com/i/bookmarks?post_id=1887450602164396149",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "video in an ad card",
+        "url": "https://x.com/igorbrigadir/status/1611399816487084033?s=46",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }
+]