admin/cobalt
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'twitter-gif' into html-cleanup

Browse Source
This commit is contained in:
dumbmoron
2024-01-17 12:08:01 +01:00
committed by GitHub
parent fc9740545c 64f5f360c6
commit 2acbbadbcb
20 changed files with 230 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/modules/pageRender/page.js
View File

@@ -335,6 +335,16 @@ export default function(obj) {
padding: "no-margin"
}])
})
+ settingsCategory({
name: "twitter",
title: "twitter",
body: checkbox([{
action: "twitterGif",
name: t("SettingsTwitterGif"),
padding: "no-margin"
}])
+ explanation(t('SettingsTwitterGifDescription'))
})
+ settingsCategory({
name: "codec",
title: t('SettingsCodecSubtitle'),
@@ -576,7 +586,7 @@ export default function(obj) {
<div id="download-area">
<div id="top">
<div id="link-icon">${linkSVG}</div>
<input id="url-input-area" class="mono" type="text" autocomplete="off" spellcheck="false" maxlength="128" autocapitalize="off" placeholder="${t('LinkInput')}" aria-label="${t('AccessibilityInputArea')}" oninput="button()">
<input id="url-input-area" class="mono" type="text" autocomplete="off" spellcheck="false" maxlength="256" autocapitalize="off" placeholder="${t('LinkInput')}" aria-label="${t('AccessibilityInputArea')}" oninput="button()">
<button id="url-clear" onclick="clearInput()" style="display:none;">x</button>
<input id="download-button" class="mono dontRead" onclick="download(document.getElementById('url-input-area').value)" type="submit" value="" disabled aria-label="${t('AccessibilityDownloadButton')}">
</div>

53
src/modules/processing/match.js
View File

@@ -13,6 +13,7 @@ import reddit from "./services/reddit.js";
import twitter from "./services/twitter.js";
import youtube from "./services/youtube.js";
import vk from "./services/vk.js";
import ok from "./services/ok.js";
import tiktok from "./services/tiktok.js";
import tumblr from "./services/tumblr.js";
import vimeo from "./services/vimeo.js";
@@ -37,24 +38,31 @@ export default async function(host, patternMatch, url, lang, obj) {
case "twitter":
r = await twitter({
id: patternMatch.id,
index: patternMatch.index - 1
index: patternMatch.index - 1,
toGif: obj.twitterGif
});
break;
case "vk":
r = await vk({
userId: patternMatch["userId"],
videoId: patternMatch["videoId"],
userId: patternMatch.userId,
videoId: patternMatch.videoId,
quality: obj.vQuality
});
break;
case "ok":
r = await ok({
id: patternMatch.id,
quality: obj.vQuality
});
break;
case "bilibili":
r = await bilibili({
id: patternMatch["id"].slice(0, 12)
id: patternMatch.id.slice(0, 12)
});
break;
case "youtube":
let fetchInfo = {
id: patternMatch["id"].slice(0, 11),
id: patternMatch.id.slice(0, 11),
quality: obj.vQuality,
format: obj.vCodec,
isAudioOnly: isAudioOnly,
@@ -72,16 +80,16 @@ export default async function(host, patternMatch, url, lang, obj) {
break;
case "reddit":
r = await reddit({
sub: patternMatch["sub"],
id: patternMatch["id"]
sub: patternMatch.sub,
id: patternMatch.id
});
break;
case "douyin":
case "tiktok":
r = await tiktok({
host: host,
postId: patternMatch["postId"],
id: patternMatch["id"],
postId: patternMatch.postId,
id: patternMatch.id,
noWatermark: obj.isNoTTWatermark,
fullAudio: obj.isTTFullAudio,
isAudioOnly: isAudioOnly
@@ -96,7 +104,7 @@ export default async function(host, patternMatch, url, lang, obj) {
break;
case "vimeo":
r = await vimeo({
id: patternMatch["id"].slice(0, 11),
id: patternMatch.id.slice(0, 11),
quality: obj.vQuality,
isAudioOnly: isAudioOnly,
forceDash: isAudioOnly ? true : obj.vimeoDash
@@ -106,10 +114,10 @@ export default async function(host, patternMatch, url, lang, obj) {
isAudioOnly = true;
r = await soundcloud({
url,
author: patternMatch["author"],
song: patternMatch["song"],
shortLink: patternMatch["shortLink"] || false,
accessKey: patternMatch["accessKey"] || false
author: patternMatch.author,
song: patternMatch.song,
shortLink: patternMatch.shortLink || false,
accessKey: patternMatch.accessKey || false
});
break;
case "instagram":
@@ -120,31 +128,32 @@ export default async function(host, patternMatch, url, lang, obj) {
break;
case "vine":
r = await vine({
id: patternMatch["id"]
id: patternMatch.id
});
break;
case "pinterest":
r = await pinterest({
id: patternMatch["id"]
id: patternMatch.id,
shortLink: patternMatch.shortLink || false
});
break;
case "streamable":
r = await streamable({
id: patternMatch["id"],
id: patternMatch.id,
quality: obj.vQuality,
isAudioOnly: isAudioOnly,
});
break;
case "twitch":
r = await twitch({
clipId: patternMatch["clip"] || false,
clipId: patternMatch.clip || false,
quality: obj.vQuality,
isAudioOnly: obj.isAudioOnly
});
break;
case "rutube":
r = await rutube({
id: patternMatch["id"],
id: patternMatch.id,
quality: obj.vQuality,
isAudioOnly: isAudioOnly
});
@@ -166,7 +175,11 @@ export default async function(host, patternMatch, url, lang, obj) {
: loc(lang, r.error)
})
return matchActionDecider(r, host, obj.aFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, obj.filenamePattern)
return matchActionDecider(
r, host, obj.aFormat, isAudioOnly,
lang, isAudioMuted, disableMetadata,
obj.filenamePattern, obj.twitterGif
)
} catch (e) {
return apiJSON(0, { t: genericError(lang, host) })
}

9
src/modules/processing/matchActionDecider.js
View File

@@ -3,7 +3,7 @@ import { apiJSON } from "../sub/utils.js";
import loc from "../../localization/manager.js";
import createFilename from "./createFilename.js";
export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern) {
export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, disableMetadata, filenamePattern, toGif) {
let action,
responseType = 2,
defaultParams = {
@@ -14,13 +14,14 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
fileMetadata: !disableMetadata ? r.fileMetadata : false
},
params = {},
audioFormat = String(userFormat)
audioFormat = String(userFormat);
if (r.isPhoto) action = "photo";
else if (r.picker) action = "picker"
else if (isAudioMuted) action = "muteVideo";
else if (isAudioOnly) action = "audio";
else if (r.isM3U8) action = "singleM3U8";
else if (r.isGif && toGif) action = "gif";
else action = "video";
if (action === "picker" || action === "audio") {
@@ -39,6 +40,10 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
case "photo":
responseType = 1;
break;
case "gif":
params = { type: "gif" }
break;
case "singleM3U8":
params = { type: "remux" }

56
src/modules/processing/services/ok.js Normal file
View File

@@ -0,0 +1,56 @@
import { genericUserAgent, maxVideoDuration } from "../../config.js";
import { cleanString } from "../../sub/utils.js";
const resolutions = {
"ultra": "2160",
"quad": "1440",
"full": "1080",
"hd": "720",
"sd": "480",
"low": "360",
"lowest": "240",
"mobile": "144"
}
export default async function(o) {
let quality = o.quality === "max" ? "2160" : o.quality;
let html = await fetch(`https://ok.ru/video/${o.id}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes(`<div data-module="OKVideo" data-options="{`)) {
return { error: 'ErrorEmptyDownload' };
}
let videoData = html.split(`<div data-module="OKVideo" data-options="`)[1].split('" data-')[0].replaceAll("&quot;", '"');
videoData = JSON.parse(JSON.parse(videoData).flashvars.metadata);
if (videoData.provider !== "UPLOADED_ODKL") return { error: 'ErrorUnsupported' };
if (videoData.movie.is_live) return { error: 'ErrorLiveVideo' };
if (videoData.movie.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let videos = videoData.videos.filter(v => !v.disallowed);
let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];
let fileMetadata = {
title: cleanString(videoData.movie.title.trim()),
author: cleanString(videoData.author.name.trim()),
}
if (bestVideo) return {
urls: bestVideo.url,
filenameAttributes: {
service: "ok",
id: o.id,
title: fileMetadata.title,
author: fileMetadata.author,
resolution: `${resolutions[bestVideo.name]}p`,
qualityLabel: `${resolutions[bestVideo.name]}p`,
extension: "mp4"
}
}
return { error: 'ErrorEmptyDownload' }
}

47
src/modules/processing/services/pinterest.js
View File

@@ -1,29 +1,36 @@
import { maxVideoDuration } from "../../config.js";
import { genericUserAgent } from "../../config.js";
export default async function(obj) {
const pinId = obj.id.split('--').reverse()[0];
if (!(/^\d+$/.test(pinId))) return { error: 'ErrorCantGetID' };
let data = await fetch(`https://www.pinterest.com/resource/PinResource/get?data=${encodeURIComponent(JSON.stringify({
options: {
field_set_key: "unauth_react_main_pin",
id: pinId
}
}))}`).then((r) => { return r.json() }).catch(() => { return false });
if (!data) return { error: 'ErrorCouldntFetch' };
const videoLinkBase = {
"regular": "https://v1.pinimg.com/videos/mc/720p/",
"story": "https://v1.pinimg.com/videos/mc/720p/"
}
data = data["resource_response"]["data"];
export default async function(o) {
let id = o.id, type = "regular";
let video = null;
if (id.includes("--")) {
id = id.split("--")[1];
type = "story";
}
if (!o.id && o.shortLink) {
id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" }).then((r) => {
return r.headers.get("location").split('pin/')[1].split('/')[0]
}).catch(() => {});
}
if (!id) return { error: 'ErrorCouldntFetch' };
if (data.videos !== null) video = data.videos.video_list.V_720P;
else if (data.story_pin_data !== null) video = data.story_pin_data.pages[0].blocks[0].video.video_list.V_EXP7;
let html = await fetch(`https://www.pinterest.com/pin/${id}/`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!video) return { error: 'ErrorEmptyDownload' };
if (video.duration > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
if (!html) return { error: 'ErrorCouldntFetch' };
let videoLink = html.split(`"url":"${videoLinkBase[type]}`)[1]?.split('"')[0];
if (!html.includes(videoLink)) return { error: 'ErrorEmptyDownload' };
return {
urls: video.url,
filename: `pinterest_${pinId}.mp4`,
audioFilename: `pinterest_${pinId}_audio`
urls: `${videoLinkBase[type]}${videoLink}`,
filename: `pinterest_${o.id}.mp4`,
audioFilename: `pinterest_${o.id}_audio`
}
}

9
src/modules/processing/services/twitter.js
View File

@@ -72,7 +72,7 @@ const requestTweet = (tweetId, token) => {
})
}
export default async function({ id, index }) {
export default async function({ id, index, toGif }) {
let guestToken = await getGuestToken();
if (!guestToken) return { error: 'ErrorCouldntFetch' };
@@ -110,7 +110,8 @@ export default async function({ id, index }) {
type: needsFixing(media[0]) ? "remux" : "normal",
urls: bestQuality(media[0].video_info.variants),
filename: `twitter_${id}.mp4`,
audioFilename: `twitter_${id}_audio`
audioFilename: `twitter_${id}_audio`,
isGif: media[0].type === "animated_gif"
};
default:
const picker = media.map((video, i) => {
@@ -120,7 +121,9 @@ export default async function({ id, index }) {
service: 'twitter',
type: 'remux',
u: url,
filename: `twitter_${id}_${i + 1}.mp4`
filename: `twitter_${id}_${i + 1}.mp4`,
isGif: media[0].type === "animated_gif",
toGif: toGif ?? false
})
}
return {

3
src/modules/processing/services/vimeo.js
View File

@@ -4,6 +4,7 @@ import { cleanString } from '../../sub/utils.js';
const resolutionMatch = {
"3840": "2160",
"2732": "1440",
"2560": "1440",
"2048": "1080",
"1920": "1080",
"1366": "720",
@@ -63,7 +64,7 @@ export default async function(obj) {
if (!masterJSON) return { error: 'ErrorCouldntFetch' };
if (!masterJSON.video) return { error: 'ErrorEmptyDownload' };
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width)).filter(a => a['format'] === "mp42"),
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width)).filter(a => ["dash", "mp42"].includes(a['format'])),
bestVideo = masterJSON_Video[0];
if (Number(quality) < Number(resolutionMatch[bestVideo["width"]])) {
bestVideo = masterJSON_Video.find(i => resolutionMatch[i["width"]] === quality)

6
src/modules/processing/services/vk.js
View File

@@ -12,7 +12,7 @@ export default async function(o) {
if (!html) return { error: 'ErrorCouldntFetch' };
// decode cyrillic from windows-1251 because vk still uses apis from prehistoring times
// decode cyrillic from windows-1251 because vk still uses apis from prehistoric times
let decoder = new TextDecoder('windows-1251');
html = decoder.decode(html);
@@ -35,7 +35,7 @@ export default async function(o) {
let fileMetadata = {
title: cleanString(js.player.params[0].md_title.trim()),
artist: cleanString(js.player.params[0].md_author.trim()),
author: cleanString(js.player.params[0].md_author.trim()),
}
if (url) return {
@@ -44,7 +44,7 @@ export default async function(o) {
service: "vk",
id: `${o.userId}_${o.videoId}`,
title: fileMetadata.title,
author: fileMetadata.artist,
author: fileMetadata.author,
resolution: `${quality}p`,
qualityLabel: `${quality}p`,
extension: "mp4"

10
src/modules/processing/servicesConfig.json
View File

@@ -1,5 +1,5 @@
{
"audioIgnore": ["vk"],
"audioIgnore": ["vk", "ok"],
"config": {
"bilibili": {
"alias": "bilibili.com videos",
@@ -28,6 +28,12 @@
"patterns": ["video:userId_:videoId", "clip:userId_:videoId", "clips:duplicate?z=clip:userId_:videoId"],
"enabled": true
},
"ok": {
"alias": "ok video",
"tld": "ru",
"patterns": ["video/:id"],
"enabled": true
},
"youtube": {
"alias": "youtube videos, shorts & music",
"patterns": ["watch?v=:id", "embed/:id", "watch/:id"],
@@ -80,7 +86,7 @@
},
"pinterest": {
"alias": "pinterest videos & stories",
"patterns": ["pin/:id"],
"patterns": ["pin/:id", "url_shortener/:shortLink"],
"enabled": true
},
"streamable": {

5
src/modules/processing/servicesPatternTesters.js
View File

@@ -5,9 +5,12 @@ export const testers = {
"instagram": (patternMatch) =>
patternMatch.postId?.length <= 12
|| (patternMatch.username?.length <= 30 && patternMatch.storyId?.length <= 24),
"ok": (patternMatch) =>
patternMatch.id?.length <= 16,
"pinterest": (patternMatch) =>
patternMatch.id?.length <= 128,
patternMatch.id?.length <= 128 || patternMatch.shortLink?.length <= 32,
"reddit": (patternMatch) =>
patternMatch.sub?.length <= 22 && patternMatch.id?.length <= 10,

7
src/modules/processing/url.js
View File

@@ -25,6 +25,13 @@ export function aliasURL(url) {
}`)
}
break;
case "pin":
if (url.hostname === 'pin.it' && parts.length === 2) {
url = new URL(`https://pinterest.com/url_shortener/${
encodeURIComponent(parts[1])
}`)
}
break;
case "vxtwitter":
case "fixvx":

5
src/modules/stream/stream.js
View File

@@ -1,4 +1,4 @@
import { streamAudioOnly, streamDefault, streamLiveRender, streamVideoOnly } from "./types.js";
import { streamAudioOnly, streamDefault, streamLiveRender, streamVideoOnly, convertToGif } from "./types.js";
export default async function(res, streamInfo) {
try {
@@ -10,6 +10,9 @@ export default async function(res, streamInfo) {
case "render":
await streamLiveRender(streamInfo, res);
break;
case "gif":
convertToGif(streamInfo, res);
break;
case "remux":
case "mute":
streamVideoOnly(streamInfo, res);

37
src/modules/stream/types.js
View File

@@ -212,3 +212,40 @@ export function streamVideoOnly(streamInfo, res) {
shutdown();
}
}
export function convertToGif(streamInfo, res) {
let process;
const shutdown = () => (killProcess(process), closeResponse(res));
try {
let args = [
'-loglevel', '-8'
]
if (streamInfo.service === "twitter") {
args.push('-seekable', '0')
}
args.push('-i', streamInfo.urls)
args = args.concat(ffmpegArgs["gif"]);
args.push('-f', "gif", 'pipe:3');
process = spawn(ffmpeg, args, {
windowsHide: true,
stdio: [
'inherit', 'inherit', 'inherit',
'pipe'
],
});
const [,,, muxOutput] = process.stdio;
res.setHeader('Connection', 'keep-alive');
res.setHeader('Content-Disposition', contentDisposition(streamInfo.filename.split('.')[0] + ".gif"));
pipe(muxOutput, res, shutdown);
process.on('close', shutdown);
res.on('finish', shutdown);
} catch {
shutdown();
}
}

5
src/modules/sub/utils.js
View File

@@ -8,7 +8,7 @@ const apiVar = {
aFormat: ["best", "mp3", "ogg", "wav", "opus"],
filenamePattern: ["classic", "pretty", "basic", "nerdy"]
},
booleanOnly: ["isAudioOnly", "isNoTTWatermark", "isTTFullAudio", "isAudioMuted", "dubLang", "vimeoDash", "disableMetadata"]
booleanOnly: ["isAudioOnly", "isNoTTWatermark", "isTTFullAudio", "isAudioMuted", "dubLang", "vimeoDash", "disableMetadata", "twitterGif"]
}
const forbiddenChars = ['}', '{', '(', ')', '\\', '>', '<', '^', '*', '!', '~', ';', ':', ',', '`', '[', ']', '#', '$', '"', "'", "@", '=='];
const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '='];
@@ -84,7 +84,8 @@ export function checkJSONPost(obj) {
isAudioMuted: false,
disableMetadata: false,
dubLang: false,
vimeoDash: false
vimeoDash: false,
twitterGif: false
}
try {
let objKeys = Object.keys(obj);