umedia 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/lib/download.js +24 -9
- package/lib/providers/facebook.js +126 -0
- package/lib/providers/instagram.js +166 -44
- package/lib/providers/reddit.js +32 -6
- package/lib/providers/rumble.js +110 -0
- package/lib/providers/snapchat.js +106 -0
- package/lib/providers/threads.js +103 -0
- package/lib/providers/tumblr.js +153 -0
- package/lib/providers/x.js +190 -46
- package/lib/registry.js +29 -9
- package/package.json +1 -1
- package/lib/providers/ogembed.js +0 -111
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { fetchJson, fetchText, hostMatches } from "../util.js";
|
|
2
|
+
import { Codes, UMediaError } from "../errors.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Snapchat — Spotlight videos (preload link on the page) and public stories
|
|
6
|
+
* (the page's __NEXT_DATA__ snap list — ALL snaps, in order).
|
|
7
|
+
* Reference: cobalt snapchat service.
|
|
8
|
+
*/
|
|
9
|
+
export const name = "snapchat";
|
|
10
|
+
|
|
11
|
+
const SPOTLIGHT_PRELOAD = /<link data-react-helmet="true" rel="preload" href="([^"]+)" as="video"\/>/;
|
|
12
|
+
const NEXT_DATA = /<script id="__NEXT_DATA__" type="application\/json">({.+?})<\/script>/;
|
|
13
|
+
|
|
14
|
+
export function canHandle(url) {
|
|
15
|
+
try {
|
|
16
|
+
return hostMatches(new URL(String(url)).hostname, ["snapchat.com", "t.snapchat.com"]);
|
|
17
|
+
} catch { return false; }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export async function resolve(url, ctx = {}) {
|
|
21
|
+
const attempt = { provider: name, status: "skipped" };
|
|
22
|
+
ctx.attempts?.push(attempt);
|
|
23
|
+
const t0 = Date.now();
|
|
24
|
+
|
|
25
|
+
let target = String(url);
|
|
26
|
+
if (/t\.snapchat\.com\//i.test(target)) {
|
|
27
|
+
try {
|
|
28
|
+
const res = await fetch(target, { redirect: "follow", headers: { "user-agent": "Mozilla/5.0" } });
|
|
29
|
+
target = res.url || target;
|
|
30
|
+
} catch { /* keep original */ }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const media = [];
|
|
34
|
+
let title = null;
|
|
35
|
+
let author = null;
|
|
36
|
+
|
|
37
|
+
const spotlight = target.match(/\/spotlight\/([\w-]+)/i);
|
|
38
|
+
const story = target.match(/\/add\/([\w.-]+)(?:\/([\w-]+))?/i);
|
|
39
|
+
|
|
40
|
+
if (spotlight) {
|
|
41
|
+
const r = await fetchText(`https://www.snapchat.com/spotlight/${spotlight[1]}`, { headers: { "user-agent": "Mozilla/5.0" } }, 18_000).catch(() => null);
|
|
42
|
+
const v = r?.text?.match(SPOTLIGHT_PRELOAD)?.[1];
|
|
43
|
+
if (v && new URL(v).hostname.endsWith("sc-cdn.net")) {
|
|
44
|
+
media.push({
|
|
45
|
+
type: "video", index: 0, url: v, thumbnail: null,
|
|
46
|
+
mimeType: "video/mp4", width: null, height: null, duration: null, size: null,
|
|
47
|
+
quality: null, hasAudio: true, hasVideo: true, source: name, variants: [],
|
|
48
|
+
});
|
|
49
|
+
title = (r.text.match(/"title":"([^"]{1,120})"/) || r.text.match(/<title[^>]*>([^<]*)/i) || [])[1] || null;
|
|
50
|
+
}
|
|
51
|
+
} else if (story) {
|
|
52
|
+
const r = await fetchText(`https://www.snapchat.com/add/${story[1]}${story[2] ? "/" + story[2] : ""}`,
|
|
53
|
+
{ headers: { "user-agent": "Mozilla/5.0" } }, 18_000).catch(() => null);
|
|
54
|
+
const raw = r?.text?.match(NEXT_DATA)?.[1];
|
|
55
|
+
if (raw) {
|
|
56
|
+
const data = JSON.parse(raw);
|
|
57
|
+
author = story[1];
|
|
58
|
+
const snaps = data?.props?.pageProps?.story?.snapList
|
|
59
|
+
|| data?.props?.pageProps?.curatedHighlights?.[0]?.snapList
|
|
60
|
+
|| [];
|
|
61
|
+
snaps.forEach((snap) => {
|
|
62
|
+
const isPhoto = snap.snapMediaType === 0;
|
|
63
|
+
const u = snap.snapUrls?.mediaUrl;
|
|
64
|
+
if (!u) return;
|
|
65
|
+
media.push({
|
|
66
|
+
type: isPhoto ? "image" : "video", index: media.length, url: u,
|
|
67
|
+
thumbnail: snap.snapUrls?.mediaPreviewUrl?.value || u,
|
|
68
|
+
mimeType: isPhoto ? "image/jpeg" : "video/mp4",
|
|
69
|
+
width: null, height: null,
|
|
70
|
+
duration: snap.duration ? Math.round(snap.duration / 1000) : null,
|
|
71
|
+
size: null, quality: null,
|
|
72
|
+
hasAudio: !isPhoto, hasVideo: !isPhoto, source: name, variants: [],
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (!media.length) {
|
|
79
|
+
attempt.status = "failed";
|
|
80
|
+
attempt.error = "no snap media exposed (removed, private, or login-gated)";
|
|
81
|
+
throw new UMediaError(Codes.MEDIA_NOT_FOUND,
|
|
82
|
+
`Snapchat exposes no media for this URL (${attempt.error}) — the yt-dlp tier covers gated stories.`, { attempts: [attempt] });
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
attempt.status = "success";
|
|
86
|
+
attempt.latencyMs = Date.now() - t0;
|
|
87
|
+
return {
|
|
88
|
+
result: {
|
|
89
|
+
sourceUrl: url,
|
|
90
|
+
platform: name,
|
|
91
|
+
title: title || (author ? `Snapchat story @${author}` : null),
|
|
92
|
+
author,
|
|
93
|
+
thumbnail: media[0].thumbnail,
|
|
94
|
+
itemCount: media.length,
|
|
95
|
+
counts: { images: media.filter((m) => m.type === "image").length, videos: media.filter((m) => m.type === "video").length, audios: 0, other: 0 },
|
|
96
|
+
truncated: false,
|
|
97
|
+
media,
|
|
98
|
+
streamUrlsAvailable: true,
|
|
99
|
+
},
|
|
100
|
+
engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export async function search() {
|
|
105
|
+
return [];
|
|
106
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { fetchText, hostMatches } from "../util.js";
|
|
2
|
+
import { Codes, UMediaError } from "../errors.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Threads (Meta) — the share page's server-rendered state (`video_url` /
|
|
6
|
+
* `image_urls` in the post JSON) plus official og: meta. Best-effort like all
|
|
7
|
+
* Meta surfaces (login-gated on some networks); yt-dlp tier covers gated ones.
|
|
8
|
+
*/
|
|
9
|
+
export const name = "threads";
|
|
10
|
+
|
|
11
|
+
export function canHandle(url) {
|
|
12
|
+
try {
|
|
13
|
+
return hostMatches(new URL(String(url)).hostname, ["threads.net", "threads.com"]);
|
|
14
|
+
} catch { return false; }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function resolve(url, ctx = {}) {
|
|
18
|
+
const attempt = { provider: name, status: "skipped" };
|
|
19
|
+
ctx.attempts?.push(attempt);
|
|
20
|
+
const t0 = Date.now();
|
|
21
|
+
|
|
22
|
+
let html;
|
|
23
|
+
try {
|
|
24
|
+
const r = await fetchText(String(url), {
|
|
25
|
+
headers: {
|
|
26
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36",
|
|
27
|
+
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
28
|
+
"accept-language": "en-US,en;q=0.9",
|
|
29
|
+
},
|
|
30
|
+
}, 18_000);
|
|
31
|
+
html = r.text;
|
|
32
|
+
} catch (e) {
|
|
33
|
+
attempt.status = "failed";
|
|
34
|
+
attempt.error = e.message;
|
|
35
|
+
throw new UMediaError(Codes.PROVIDER_UNAVAILABLE, `Threads page not retrievable. ${e.message}`, { attempts: [attempt] });
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const un = (s) => { try { return JSON.parse(`"${s}"`); } catch { return s.replace(/\\\//g, "/"); } };
|
|
39
|
+
const media = [];
|
|
40
|
+
|
|
41
|
+
// post state: video_url + display_url (escaped JSON strings inside the SSR payload)
|
|
42
|
+
const video = html.match(/\\"video_url\\":\\"((?:[^"\\]|\\.)+?)\\"/) || html.match(/"video_url":"((?:[^"\\]|\\.)+?)"/);
|
|
43
|
+
const image = html.match(/\\"display_url\\":\\"((?:[^"\\]|\\.)+?)\\"/) || html.match(/"display_url":"((?:[^"\\]|\\.)+?)"/);
|
|
44
|
+
if (video?.[1]) {
|
|
45
|
+
const u = un(video[1]);
|
|
46
|
+
if (/^https?:/.test(u)) media.push({
|
|
47
|
+
type: "video", index: 0, url: u, thumbnail: image ? un(image[1]) : u,
|
|
48
|
+
mimeType: "video/mp4", width: null, height: null, duration: null, size: null,
|
|
49
|
+
quality: null, hasAudio: true, hasVideo: true, source: name, variants: [],
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
if (!media.length && image?.[1]) {
|
|
53
|
+
const u = un(image[1]);
|
|
54
|
+
if (/^https?:/.test(u)) media.push({
|
|
55
|
+
type: "image", index: 0, url: u, thumbnail: u,
|
|
56
|
+
mimeType: "image/jpeg", width: null, height: null, duration: null, size: null,
|
|
57
|
+
quality: null, hasAudio: false, hasVideo: false, source: name, variants: [],
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// og: meta fallback
|
|
62
|
+
if (!media.length) {
|
|
63
|
+
const ogv = (html.match(/property="og:video"[^>]+content="([^"]+)"/) || html.match(/content="([^"]+)"[^>]+property="og:video"/) || [])[1];
|
|
64
|
+
const ogi = (html.match(/property="og:image"[^>]+content="([^"]+)"/) || html.match(/content="([^"]+)"[^>]+property="og:image"/) || [])[1];
|
|
65
|
+
const u = ogv || ogi;
|
|
66
|
+
if (u && /^https?:/.test(u)) media.push({
|
|
67
|
+
type: ogv ? "video" : "image", index: 0, url: u.replace(/&/g, "&"), thumbnail: (ogi || u).replace(/&/g, "&"),
|
|
68
|
+
mimeType: ogv ? "video/mp4" : "image/jpeg", width: null, height: null,
|
|
69
|
+
duration: null, size: null, quality: null,
|
|
70
|
+
hasAudio: Boolean(ogv), hasVideo: Boolean(ogv), source: name, variants: [],
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (!media.length) {
|
|
75
|
+
attempt.status = "failed";
|
|
76
|
+
attempt.error = "no media in the public page (login-gated or removed)";
|
|
77
|
+
throw new UMediaError(Codes.MEDIA_NOT_FOUND,
|
|
78
|
+
`Threads exposes no media for this URL (${attempt.error}) — the yt-dlp tier covers gated posts.`, { attempts: [attempt] });
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const title = (html.match(/"caption":"((?:[^"\\]|\\.){1,200})"/) || html.match(/<title[^>]*>([^<]*)/i) || [])[1] || null;
|
|
82
|
+
attempt.status = "success";
|
|
83
|
+
attempt.latencyMs = Date.now() - t0;
|
|
84
|
+
return {
|
|
85
|
+
result: {
|
|
86
|
+
sourceUrl: url,
|
|
87
|
+
platform: name,
|
|
88
|
+
title: title ? un(title).slice(0, 200) : null,
|
|
89
|
+
author: (String(url).match(/@([\w.]+)/) || [])[1] || null,
|
|
90
|
+
thumbnail: media[0].thumbnail,
|
|
91
|
+
itemCount: media.length,
|
|
92
|
+
counts: { images: media.filter((m) => m.type === "image").length, videos: media.filter((m) => m.type === "video").length, audios: 0, other: 0 },
|
|
93
|
+
truncated: false,
|
|
94
|
+
media,
|
|
95
|
+
streamUrlsAvailable: true,
|
|
96
|
+
},
|
|
97
|
+
engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export async function search() {
|
|
102
|
+
return [];
|
|
103
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { fetchJson, hostMatches } from "../util.js";
|
|
2
|
+
import { Codes, UMediaError } from "../errors.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Tumblr — the public mobile API (api-http2.tumblr.com, the same endpoint the
|
|
6
|
+
* official iPhone app calls with its public api_key). Videos, audio, photos —
|
|
7
|
+
* including reblog trails. Reference: cobalt tumblr service.
|
|
8
|
+
*/
|
|
9
|
+
export const name = "tumblr";
|
|
10
|
+
|
|
11
|
+
const API_KEY = "jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z";
|
|
12
|
+
const API_BASE = "https://api-http2.tumblr.com";
|
|
13
|
+
const MOBILE = {
|
|
14
|
+
"user-agent": "Tumblr/iPhone/33.3/333010/17.3.1/tumblr",
|
|
15
|
+
"x-version": "iPhone/33.3/333010/17.3.1/tumblr",
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export function canHandle(url) {
|
|
19
|
+
try {
|
|
20
|
+
return hostMatches(new URL(String(url)).hostname, ["tumblr.com"]);
|
|
21
|
+
} catch { return false; }
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function parseUrl(url) {
|
|
25
|
+
const u = new URL(String(url));
|
|
26
|
+
const host = u.hostname.replace(/^www\./, "");
|
|
27
|
+
let m;
|
|
28
|
+
// https://{user}.tumblr.com/post/{id}/...
|
|
29
|
+
m = u.pathname.match(/^\/post\/(\d+)/);
|
|
30
|
+
if (m && host !== "tumblr.com") return { user: host.split(".")[0], id: m[1] };
|
|
31
|
+
// https://www.tumblr.com/{user}/{id} or /blog/view/{user}/{id}
|
|
32
|
+
m = u.pathname.match(/^\/(?:blog\/view\/)?([\w.-]+)\/(\d+)/);
|
|
33
|
+
if (m) return { user: m[1], id: m[2] };
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function resolve(url, ctx = {}) {
|
|
38
|
+
const attempt = { provider: name, status: "skipped" };
|
|
39
|
+
ctx.attempts?.push(attempt);
|
|
40
|
+
const t0 = Date.now();
|
|
41
|
+
|
|
42
|
+
const parsed = parseUrl(url);
|
|
43
|
+
if (!parsed) {
|
|
44
|
+
attempt.status = "failed";
|
|
45
|
+
throw new UMediaError(Codes.INVALID_URL, "Not a Tumblr post URL (expected {user}.tumblr.com/post/{id} or tumblr.com/{user}/{id})", { attempts: [attempt] });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
let data;
|
|
49
|
+
try {
|
|
50
|
+
data = await fetchJson(
|
|
51
|
+
`${API_BASE}/v2/blog/${encodeURIComponent(parsed.user)}.tumblr.com/posts/${parsed.id}/permalink?api_key=${API_KEY}`, { headers: MOBILE }, 20_000);
|
|
52
|
+
} catch (e) {
|
|
53
|
+
attempt.status = "failed";
|
|
54
|
+
attempt.error = e.message;
|
|
55
|
+
throw new UMediaError(Codes.PROVIDER_UNAVAILABLE, `Tumblr API unavailable. ${e.message}`, { attempts: [attempt] });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const element = data?.response?.timeline?.elements?.[0];
|
|
59
|
+
if (!element) {
|
|
60
|
+
attempt.status = "failed";
|
|
61
|
+
attempt.error = data?.meta?.msg || "post not found";
|
|
62
|
+
throw new UMediaError(Codes.MEDIA_NOT_FOUND, `Tumblr: ${attempt.error}`, { attempts: [attempt] });
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// contents: the post itself + any reblog trail, in order — ALL of it
|
|
66
|
+
const contents = [
|
|
67
|
+
...(element.content || []),
|
|
68
|
+
...(element.trail || []).flatMap((t) => t.content || []),
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
const media = [];
|
|
72
|
+
for (const c of contents) {
|
|
73
|
+
if (c.type === "video" && (c.provider === "tumblr" || !c.provider) && c.media?.url) {
|
|
74
|
+
media.push({
|
|
75
|
+
type: "video", index: media.length, url: c.media.url,
|
|
76
|
+
thumbnail: c.poster?.[0]?.url || null,
|
|
77
|
+
mimeType: "video/mp4", width: c.width ?? null, height: c.height ?? null,
|
|
78
|
+
duration: null, size: null, quality: null,
|
|
79
|
+
hasAudio: true, hasVideo: true, source: name, variants: [],
|
|
80
|
+
});
|
|
81
|
+
} else if (c.type === "audio" && c.media?.url) {
|
|
82
|
+
media.push({
|
|
83
|
+
type: "audio", index: media.length, url: c.media.url,
|
|
84
|
+
thumbnail: null, mimeType: "audio/mpeg", width: null, height: null,
|
|
85
|
+
duration: null, size: null, quality: null,
|
|
86
|
+
hasAudio: true, hasVideo: false, source: name, variants: [],
|
|
87
|
+
});
|
|
88
|
+
} else if (c.type === "image" || c.type === "photo") {
|
|
89
|
+
// image content carries media as an ARRAY of renditions — take the best
|
|
90
|
+
// one as the item, keep the rest as variants. never inflate items.
|
|
91
|
+
const items = (Array.isArray(c.media) ? c.media : (c.media?.url ? [c.media] : []))
|
|
92
|
+
.filter((mm) => mm?.url);
|
|
93
|
+
const uniq = [...new Map(items.map((mm) => [mm.url, mm])).values()]
|
|
94
|
+
.sort((a, b) => ((b.width || 0) * (b.height || 0)) - ((a.width || 0) * (a.height || 0)));
|
|
95
|
+
const best = uniq[0];
|
|
96
|
+
if (best) {
|
|
97
|
+
media.push({
|
|
98
|
+
type: "image", index: media.length, url: best.url,
|
|
99
|
+
thumbnail: best.url, mimeType: "image/jpeg",
|
|
100
|
+
width: best.width ?? null, height: best.height ?? null,
|
|
101
|
+
duration: null, size: null, quality: best.width ? `${best.width}px` : null,
|
|
102
|
+
hasAudio: false, hasVideo: false, source: name,
|
|
103
|
+
variants: uniq.map((x) => ({
|
|
104
|
+
height: x.height ?? null, width: x.width ?? null, url: x.url,
|
|
105
|
+
hasAudio: false, hasVideo: false, quality: x.width ? `${x.width}px` : null, mimeType: "image/jpeg",
|
|
106
|
+
})),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// reblog trails repeat media from earlier hops — dedupe by URL, keep first order
|
|
113
|
+
const seen = new Set();
|
|
114
|
+
const deduped = [];
|
|
115
|
+
for (const mm of media) {
|
|
116
|
+
if (seen.has(mm.url)) continue;
|
|
117
|
+
seen.add(mm.url);
|
|
118
|
+
deduped.push({ ...mm, index: deduped.length });
|
|
119
|
+
}
|
|
120
|
+
media.length = 0;
|
|
121
|
+
media.push(...deduped);
|
|
122
|
+
|
|
123
|
+
if (!media.length) {
|
|
124
|
+
attempt.status = "failed";
|
|
125
|
+
throw new UMediaError(Codes.MEDIA_NOT_FOUND, "Tumblr post contains no downloadable media (text/link post, or third-party embed — use the yt-dlp tier for YouTube/Vimeo embeds)", { attempts: [attempt] });
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
attempt.status = "success";
|
|
129
|
+
attempt.latencyMs = Date.now() - t0;
|
|
130
|
+
return {
|
|
131
|
+
result: {
|
|
132
|
+
sourceUrl: url,
|
|
133
|
+
platform: name,
|
|
134
|
+
title: (element.summary || element.let || "").slice(0, 200) || null,
|
|
135
|
+
author: element.blog_name || parsed.user,
|
|
136
|
+
thumbnail: media[0].thumbnail,
|
|
137
|
+
itemCount: media.length,
|
|
138
|
+
counts: {
|
|
139
|
+
images: media.filter((m) => m.type === "image").length,
|
|
140
|
+
videos: media.filter((m) => m.type === "video").length,
|
|
141
|
+
audios: media.filter((m) => m.type === "audio").length, other: 0,
|
|
142
|
+
},
|
|
143
|
+
truncated: false,
|
|
144
|
+
media,
|
|
145
|
+
streamUrlsAvailable: true,
|
|
146
|
+
},
|
|
147
|
+
engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export async function search() {
|
|
152
|
+
return [];
|
|
153
|
+
}
|
package/lib/providers/x.js
CHANGED
|
@@ -1,69 +1,213 @@
|
|
|
1
1
|
import { fetchJson, hostMatches } from "../util.js";
|
|
2
2
|
import { Codes, UMediaError } from "../errors.js";
|
|
3
3
|
|
|
4
|
-
/**
|
|
4
|
+
/**
|
|
5
|
+
* X / Twitter — two proven public routes, no login:
|
|
6
|
+
* 1. syndication embed API with the computed token (the route embed widgets use)
|
|
7
|
+
* 2. GraphQL TweetDetail with a guest token (the route the logged-out web uses)
|
|
8
|
+
* Both are the exact techniques from yt-dlp's twitter extractor and cobalt's
|
|
9
|
+
* twitter service. Photos, videos (best bitrate mp4), gifs.
|
|
10
|
+
*/
|
|
5
11
|
export const name = "x";
|
|
6
12
|
|
|
13
|
+
const BEARER = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
|
14
|
+
const GQL_FEATURES = JSON.stringify({
|
|
15
|
+
rweb_video_screen_enabled: false, payments_enabled: false, rweb_xchat_enabled: false,
|
|
16
|
+
profile_label_improvements_pcf_label_in_post_enabled: true, rweb_tipjar_consumption_enabled: true,
|
|
17
|
+
verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true,
|
|
18
|
+
responsive_web_graphql_timeline_navigation_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
19
|
+
premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true,
|
|
20
|
+
c9s_tweet_anatomy_moderator_badge_enabled: true, responsive_web_grok_analyze_button_fetch_trends_enabled: false,
|
|
21
|
+
responsive_web_grok_analyze_post_followups_enabled: true, responsive_web_jetfuel_frame: true,
|
|
22
|
+
responsive_web_grok_share_attachment_enabled: true, articles_preview_enabled: true,
|
|
23
|
+
responsive_web_edit_tweet_api_enabled: true, graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
24
|
+
view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true,
|
|
25
|
+
responsive_web_twitter_article_tweet_consumption_enabled: true, tweet_awards_web_tipping_enabled: false,
|
|
26
|
+
creator_subscriptions_quote_tweet_preview_enabled: false, freedom_of_speech_not_reach_fetch_enabled: true,
|
|
27
|
+
standardized_nudges_misinfo: true, tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
28
|
+
longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: true,
|
|
29
|
+
responsive_web_grok_image_annotation_enabled: true, responsive_web_grok_imagine_annotation_enabled: true,
|
|
30
|
+
responsive_web_grok_community_note_auto_translation_is_enabled: false, responsive_web_enhance_cards_enabled: false,
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
const GQL_HEADERS = {
|
|
34
|
+
authorization: `Bearer ${BEARER}`,
|
|
35
|
+
"x-twitter-client-language": "en",
|
|
36
|
+
"x-twitter-active-user": "yes",
|
|
37
|
+
"accept-language": "en",
|
|
38
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36",
|
|
39
|
+
};
|
|
40
|
+
|
|
7
41
|
export function canHandle(url) {
|
|
8
42
|
try {
|
|
9
43
|
return hostMatches(new URL(String(url)).hostname, ["twitter.com", "x.com"]);
|
|
10
44
|
} catch { return false; }
|
|
11
45
|
}
|
|
12
46
|
|
|
47
|
+
export function tweetId(url) {
|
|
48
|
+
const m = String(url).match(/\/status(?:es)?\/(\d{8,25})/);
|
|
49
|
+
return m ? m[1] : null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// the syndication embed token — computed from the tweet id (yt-dlp technique)
|
|
53
|
+
const syndicationToken = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, "");
|
|
54
|
+
|
|
55
|
+
let _guestToken = null;
|
|
56
|
+
async function guestToken() {
|
|
57
|
+
if (_guestToken) return _guestToken;
|
|
58
|
+
try {
|
|
59
|
+
const j = await fetchJson("https://api.x.com/1.1/guest/activate.json", {
|
|
60
|
+
method: "POST", headers: GQL_HEADERS,
|
|
61
|
+
}, 15_000);
|
|
62
|
+
if (j?.guest_token) return (_guestToken = j.guest_token);
|
|
63
|
+
} catch { /* syndication may still work */ }
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function mediaFromEntities(entities, media) {
|
|
68
|
+
for (const m of entities || []) {
|
|
69
|
+
const isPhoto = m.type === "photo";
|
|
70
|
+
const isVideo = m.type === "video" || m.type === "animated_gif";
|
|
71
|
+
if (isPhoto && m.media_url_https) {
|
|
72
|
+
media.push({
|
|
73
|
+
type: "image", index: media.length, url: m.media_url_https, thumbnail: m.media_url_https,
|
|
74
|
+
mimeType: "image/jpeg", width: m.original_info?.width ?? null, height: m.original_info?.height ?? null,
|
|
75
|
+
duration: null, size: null, quality: "original",
|
|
76
|
+
hasAudio: false, hasVideo: false, source: name, variants: [],
|
|
77
|
+
});
|
|
78
|
+
} else if (isVideo && m.video_info?.variants?.length) {
|
|
79
|
+
const mp4s = m.video_info.variants
|
|
80
|
+
.filter((v) => v.content_type === "video/mp4" && v.url)
|
|
81
|
+
.map((v) => ({
|
|
82
|
+
height: null, width: null,
|
|
83
|
+
// cobalt strips the flaky `tag` param from twitter mp4 urls
|
|
84
|
+
url: (() => { try { const u = new URL(v.url); u.searchParams.delete("tag"); return u.toString(); } catch { return v.url; } })(),
|
|
85
|
+
hasAudio: true, hasVideo: true,
|
|
86
|
+
quality: v.bitrate ? `${Math.round(v.bitrate / 1000)}kbps` : "mp4",
|
|
87
|
+
mimeType: "video/mp4", bitrate: v.bitrate ?? null,
|
|
88
|
+
}))
|
|
89
|
+
.sort((a, b) => (b.bitrate || 0) - (a.bitrate || 0));
|
|
90
|
+
const best = mp4s[0]
|
|
91
|
+
|| m.video_info.variants.filter((v) => v.url).map((v) => ({
|
|
92
|
+
height: null, width: null, url: v.url, hasAudio: true, hasVideo: true,
|
|
93
|
+
quality: null, mimeType: v.content_type || "application/x-mpegURL", bitrate: null,
|
|
94
|
+
}))[0];
|
|
95
|
+
if (best) {
|
|
96
|
+
media.push({
|
|
97
|
+
type: "video", index: media.length, url: best.url,
|
|
98
|
+
thumbnail: m.media_url_https || null,
|
|
99
|
+
mimeType: best.mimeType, width: m.original_info?.width ?? null, height: m.original_info?.height ?? null,
|
|
100
|
+
duration: m.video_info.duration_millis ? Math.round(m.video_info.duration_millis / 1000) : null,
|
|
101
|
+
size: null, quality: best.quality,
|
|
102
|
+
hasAudio: true, hasVideo: true, source: name, variants: mp4s,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
13
109
|
export async function resolve(url, ctx = {}) {
|
|
14
110
|
const attempt = { provider: name, status: "skipped" };
|
|
15
111
|
ctx.attempts?.push(attempt);
|
|
16
|
-
const m = String(url).match(/status\/(\d+)/);
|
|
17
|
-
if (!m) throw new UMediaError(Codes.INVALID_URL, `No status id in ${url}`);
|
|
18
112
|
const t0 = Date.now();
|
|
113
|
+
|
|
114
|
+
const id = tweetId(url);
|
|
115
|
+
if (!id) {
|
|
116
|
+
attempt.status = "failed";
|
|
117
|
+
throw new UMediaError(Codes.INVALID_URL, "Not a tweet URL (expected x.com/{user}/status/{id})", { attempts: [attempt] });
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const media = [];
|
|
121
|
+
let title = null;
|
|
122
|
+
let author = null;
|
|
123
|
+
|
|
124
|
+
// 1) GraphQL TweetDetail with guest token (fullest data)
|
|
19
125
|
try {
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
126
|
+
const token = await guestToken();
|
|
127
|
+
if (token) {
|
|
128
|
+
const u = new URL("https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail");
|
|
129
|
+
u.searchParams.set("variables", JSON.stringify({
|
|
130
|
+
focalTweetId: id, with_rux_injections: false, rankingMode: "Relevance",
|
|
131
|
+
includePromotedContent: true, withCommunity: true,
|
|
132
|
+
withQuickPromoteEligibilityTweetFields: true, withBirdwatchNotes: true, withVoice: true,
|
|
133
|
+
}));
|
|
134
|
+
u.searchParams.set("features", GQL_FEATURES);
|
|
135
|
+
u.searchParams.set("fieldToggles", JSON.stringify({ withArticleRichContentState: true, withArticlePlainText: false }));
|
|
136
|
+
const j = await fetchJson(u.toString(), {
|
|
137
|
+
headers: { ...GQL_HEADERS, "content-type": "application/json", "x-guest-token": token, cookie: `guest_id=v1%3A${token}` },
|
|
138
|
+
}, 20_000);
|
|
139
|
+
const entries = j?.data?.threaded_conversation_with_injections_v2?.instructions
|
|
140
|
+
?.find((i) => i.type === "TimelineAddEntries")?.entries || [];
|
|
141
|
+
const node = entries.find((e) => e.entryId === `tweet-${id}`)?.content?.itemContent?.tweet_results?.result;
|
|
142
|
+
let base = node?.legacy;
|
|
143
|
+
if (node?.__typename === "TweetWithVisibilityResults") base = node.tweet?.legacy;
|
|
144
|
+
const rep = base?.retweeted_status_result?.result?.legacy?.extended_entities
|
|
145
|
+
|| base?.retweeted_status_result?.result?.tweet?.legacy?.extended_entities;
|
|
146
|
+
if (base?.full_text) title = base.full_text;
|
|
147
|
+
author = node?.core?.user_results?.result?.legacy?.screen_name
|
|
148
|
+
|| node?.tweet?.core?.user_results?.result?.legacy?.screen_name || null;
|
|
149
|
+
mediaFromEntities(rep?.media || base?.extended_entities?.media, media);
|
|
44
150
|
}
|
|
45
|
-
if (!media.length) throw new UMediaError(Codes.MEDIA_NOT_FOUND, "Post has no media (or it is not public)");
|
|
46
|
-
attempt.status = "success";
|
|
47
|
-
attempt.latencyMs = Date.now() - t0;
|
|
48
|
-
return {
|
|
49
|
-
result: {
|
|
50
|
-
sourceUrl: url, platform: "x", title: (tw.text || "").slice(0, 120) || null,
|
|
51
|
-
author: tw.user?.screen_name ?? null, thumbnail: media[0].thumbnail ?? null,
|
|
52
|
-
itemCount: media.length,
|
|
53
|
-
counts: {
|
|
54
|
-
images: media.filter((x) => x.type === "image").length,
|
|
55
|
-
videos: media.filter((x) => x.type === "video").length, audios: 0, other: 0,
|
|
56
|
-
},
|
|
57
|
-
truncated: false, media,
|
|
58
|
-
},
|
|
59
|
-
engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
|
|
60
|
-
};
|
|
61
151
|
} catch (e) {
|
|
152
|
+
attempt.note = `graphql: ${String(e.message).slice(0, 60)}`;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// 2) syndication embed API with the computed token (embed-widget route)
|
|
156
|
+
if (!media.length) {
|
|
157
|
+
try {
|
|
158
|
+
const j = await fetchJson(
|
|
159
|
+
`https://cdn.syndication.twimg.com/tweet-result?id=${id}&token=${syndicationToken(id)}&lang=en`, {}, 15_000);
|
|
160
|
+
title = title || j?.text || null;
|
|
161
|
+
author = author || j?.user?.screen_name || null;
|
|
162
|
+
mediaFromEntities(j?.mediaDetails, media);
|
|
163
|
+
if (!media.length && j?.video?.variants) {
|
|
164
|
+
const mp4s = j.video.variants.filter((v) => v.type === "video/mp4").sort((a, b) => (b.bitrate || 0) - (a.bitrate || 0));
|
|
165
|
+
if (mp4s[0]) media.push({
|
|
166
|
+
type: "video", index: 0, url: mp4s[0].src, thumbnail: j.thumbnail?.image?.src || null,
|
|
167
|
+
mimeType: "video/mp4", width: null, height: null,
|
|
168
|
+
duration: j.video.duration_millis ? Math.round(j.video.duration_millis / 1000) : null,
|
|
169
|
+
size: null, quality: mp4s[0].bitrate ? `${Math.round(mp4s[0].bitrate / 1000)}kbps` : null,
|
|
170
|
+
hasAudio: true, hasVideo: true, source: name,
|
|
171
|
+
variants: mp4s.map((v) => ({ height: null, width: null, url: v.src, hasAudio: true, hasVideo: true, quality: v.bitrate ? `${Math.round(v.bitrate / 1000)}kbps` : null, mimeType: "video/mp4", bitrate: v.bitrate ?? null })),
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
} catch (e) {
|
|
175
|
+
attempt.note = `${attempt.note ? attempt.note + " · " : ""}syndication: ${String(e.message).slice(0, 60)}`;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
if (!media.length) {
|
|
62
180
|
attempt.status = "failed";
|
|
63
|
-
attempt.error =
|
|
64
|
-
|
|
65
|
-
|
|
181
|
+
attempt.error = j_err(attempt.note);
|
|
182
|
+
throw new UMediaError(Codes.MEDIA_NOT_FOUND,
|
|
183
|
+
`X exposes no media for this tweet (${attempt.error}) — protected/removed tweets stay inaccessible; the yt-dlp tier adds cookie support.`, { attempts: [attempt] });
|
|
66
184
|
}
|
|
185
|
+
|
|
186
|
+
attempt.status = "success";
|
|
187
|
+
attempt.latencyMs = Date.now() - t0;
|
|
188
|
+
return {
|
|
189
|
+
result: {
|
|
190
|
+
sourceUrl: url,
|
|
191
|
+
platform: name,
|
|
192
|
+
title: title ? title.slice(0, 200) : null,
|
|
193
|
+
author,
|
|
194
|
+
thumbnail: media[0].thumbnail,
|
|
195
|
+
itemCount: media.length,
|
|
196
|
+
counts: {
|
|
197
|
+
images: media.filter((m) => m.type === "image").length,
|
|
198
|
+
videos: media.filter((m) => m.type === "video").length,
|
|
199
|
+
audios: 0, other: 0,
|
|
200
|
+
},
|
|
201
|
+
truncated: false,
|
|
202
|
+
media,
|
|
203
|
+
streamUrlsAvailable: true,
|
|
204
|
+
},
|
|
205
|
+
engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function j_err(note) {
|
|
210
|
+
return (note || "no media in public routes").replace(/^.*?(graphql|syndication)/, "$1");
|
|
67
211
|
}
|
|
68
212
|
|
|
69
213
|
export async function search() {
|