umedia 0.3.0 → 0.4.1

package/lib/providers/snapchat.js

@@ -0,0 +1,177 @@
+import { fetchText, hostMatches } from "../util.js";
+import { Codes, UMediaError } from "../errors.js";
+
+/**
+ * Snapchat — Spotlight videos and public story snap lists.
+ * Primary: the page's __NEXT_DATA__ -> spotlightFeed.spotlightStories, matched
+ * by storyId (the exact structure yt-dlp's snapchat extractor reads) ->
+ * videoMetadata.contentUrl (direct sc-cdn mp4). Fallback: the preload video
+ * link (cobalt). Stories: pageProps.story.snapList — ALL snaps, in order.
+ */
+export const name = "snapchat";
+
+const NEXT_DATA = /<script id="__NEXT_DATA__" type="application\/json">({.+?})<\/script>/;
+const PRELOAD_VIDEO = /<link data-react-helmet="true" rel="preload" href="([^"]+)" as="video"\/>/;
+
+export function canHandle(url) {
+  try {
+    return hostMatches(new URL(String(url)).hostname, ["snapchat.com", "t.snapchat.com"]);
+  } catch { return false; }
+}
+
+export function spotlightId(url) {
+  const m = String(url).match(/\/spotlight\/([\w-]{10,120})/);
+  return m ? m[1] : null;
+}
+
+function nextData(html) {
+  const m = html.match(NEXT_DATA);
+  if (!m) return null;
+  try { return JSON.parse(m[1]); } catch { return null; }
+}
+
+function pushStorySnap(snap, media) {
+  const isPhoto = snap.snapMediaType === 0;
+  const u = snap.snapUrls?.mediaUrl;
+  if (!u) return;
+  media.push({
+    type: isPhoto ? "image" : "video", index: media.length, url: u,
+    thumbnail: snap.snapUrls?.mediaPreviewUrl?.value || u,
+    mimeType: isPhoto ? "image/jpeg" : "video/mp4",
+    width: null, height: null,
+    duration: snap.durationInMs ? Math.round(snap.durationInMs / 1000) : (snap.duration ? Math.round(snap.duration / 1000) : null),
+    size: null, quality: null,
+    hasAudio: !isPhoto, hasVideo: !isPhoto, source: name, variants: [],
+  });
+}
+
+export async function resolve(url, ctx = {}) {
+  const attempt = { provider: name, status: "skipped" };
+  ctx.attempts?.push(attempt);
+  const t0 = Date.now();
+
+  let target = String(url);
+  if (/t\.snapchat\.com\//i.test(target)) {
+    try {
+      const res = await fetch(target, { redirect: "follow", headers: { "user-agent": "Mozilla/5.0" } });
+      target = res.url || target;
+    } catch { /* keep original */ }
+  }
+
+  const media = [];
+  let title = null;
+  let author = null;
+  let duration = null;
+
+  const sid = spotlightId(target);
+  const story = target.match(/\/add\/([\w.-]+)(?:\/([\w-]+))?/i);
+
+  if (sid) {
+    const r = await fetchText(`https://www.snapchat.com/spotlight/${sid}`, { headers: { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" } }, 20_000).catch(() => null);
+    const html = r?.text || "";
+
+    // primary: __NEXT_DATA__ spotlight feed entry matched by storyId (yt-dlp structure)
+    const d = nextData(html);
+    const stories = d?.props?.pageProps?.spotlightFeed?.spotlightStories || [];
+    const entry = stories.find((s) => s?.story?.storyId?.value === sid);
+    const vm = entry?.metadata?.videoMetadata;
+    if (vm?.contentUrl) {
+      media.push({
+        type: "video", index: 0, url: vm.contentUrl,
+        thumbnail: vm.thumbnailUrl || null,
+        mimeType: "video/mp4",
+        width: vm.width ?? null, height: vm.height ?? null,
+        duration: vm.durationMs ? Math.round(vm.durationMs / 1000) : null,
+        size: null, quality: vm.height ? `${vm.height}p` : null,
+        hasAudio: true, hasVideo: true, source: name, variants: [],
+      });
+      title = vm.name || null;
+      author = vm.creator?.personCreator?.username || vm.creator?.username || null;
+      duration = vm.durationMs ? Math.round(vm.durationMs / 1000) : null;
+    }
+
+    // fallback: the preload video link (cobalt's route)
+    if (!media.length) {
+      const v = html.match(PRELOAD_VIDEO)?.[1];
+      if (v && /^https?:/.test(v) && new URL(v).hostname.endsWith("sc-cdn.net")) {
+        media.push({
+          type: "video", index: 0, url: v, thumbnail: null,
+          mimeType: "video/mp4", width: null, height: null,
+          duration: null, size: null, quality: null,
+          hasAudio: true, hasVideo: true, source: name, variants: [],
+        });
+      }
+    }
+  } else if (story) {
+    const r = await fetchText(`https://www.snapchat.com/add/${story[1]}${story[2] ? "/" + story[2] : ""}`,
+      { headers: { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" } }, 20_000).catch(() => null);
+    const d = nextData(r?.text || "");
+    const pp = d?.props?.pageProps || {};
+    author = story[1];
+
+    const activeStory = pp.story?.snapList || [];
+    const highlights = pp.spotlightHighlights || [];
+    const curated = pp.curatedHighlights || [];
+
+    if (story[2]) {
+      // a single snap (by snapId) or one highlight reel (by storyId/highlightId)
+      const wanted = story[2];
+      const allLists = [activeStory, ...curated.map((c) => c.snapList || []), ...highlights.map((h) => h.snapList || [])];
+      for (const list of allLists) {
+        const snap = list.find((s) => s.snapId?.value === wanted || s.snapId === wanted);
+        if (snap) { pushStorySnap(snap, media); break; }
+      }
+      if (!media.length) {
+        const hl = highlights.find((h) => h.storyId === wanted || h.highlightId === wanted || h.storyShareId === wanted);
+        if (hl) {
+          title = hl.storyTitle || null;
+          (hl.snapList || []).forEach((snap) => pushStorySnap(snap, media));
+        }
+      }
+    }
+    if (!media.length) {
+      // bare /add/{user}: the current story if it exists, else their public
+      // highlight reels — ALL snaps, in order, nothing silently dropped
+      const list = activeStory.length
+        ? activeStory
+        : (curated[0]?.snapList?.length ? curated[0].snapList : highlights.flatMap((h) => h.snapList || []));
+      if (!activeStory.length && highlights.length && !curated[0]?.snapList?.length) {
+        title = `${author} — ${highlights.length} public highlight${highlights.length > 1 ? "s" : ""}`;
+      }
+      list.forEach((snap) => pushStorySnap(snap, media));
+    }
+  }
+
+  if (!media.length) {
+    attempt.status = "failed";
+    attempt.error = "no snap media exposed (removed, private, or an expired spotlight)";
+    throw new UMediaError(Codes.MEDIA_NOT_FOUND,
+      `Snapchat exposes no media for this URL (${attempt.error}) — public spotlight links and stories work; removed snaps are gone from Snapchat itself.`, { attempts: [attempt] });
+  }
+
+  attempt.status = "success";
+  attempt.latencyMs = Date.now() - t0;
+  return {
+    result: {
+      sourceUrl: url,
+      platform: name,
+      title: title || (author ? `Snapchat story @${author}` : null),
+      author,
+      thumbnail: media[0].thumbnail,
+      itemCount: media.length,
+      counts: {
+        images: media.filter((m) => m.type === "image").length,
+        videos: media.filter((m) => m.type === "video").length,
+        audios: 0, other: 0,
+      },
+      truncated: false,
+      media,
+      streamUrlsAvailable: true,
+    },
+    engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
+  };
+}
+
+export async function search() {
+  return [];
+}

package/lib/providers/threads.js

@@ -0,0 +1,103 @@
+import { fetchText, hostMatches } from "../util.js";
+import { Codes, UMediaError } from "../errors.js";
+
+/**
+ * Threads (Meta) — the share page's server-rendered state (`video_url` /
+ * `image_urls` in the post JSON) plus official og: meta. Best-effort like all
+ * Meta surfaces (login-gated on some networks); yt-dlp tier covers gated ones.
+ */
+export const name = "threads";
+
+export function canHandle(url) {
+  try {
+    return hostMatches(new URL(String(url)).hostname, ["threads.net", "threads.com"]);
+  } catch { return false; }
+}
+
+export async function resolve(url, ctx = {}) {
+  const attempt = { provider: name, status: "skipped" };
+  ctx.attempts?.push(attempt);
+  const t0 = Date.now();
+
+  let html;
+  try {
+    const r = await fetchText(String(url), {
+      headers: {
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36",
+        accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "accept-language": "en-US,en;q=0.9",
+      },
+    }, 18_000);
+    html = r.text;
+  } catch (e) {
+    attempt.status = "failed";
+    attempt.error = e.message;
+    throw new UMediaError(Codes.PROVIDER_UNAVAILABLE, `Threads page not retrievable. ${e.message}`, { attempts: [attempt] });
+  }
+
+  const un = (s) => { try { return JSON.parse(`"${s}"`); } catch { return s.replace(/\\\//g, "/"); } };
+  const media = [];
+
+  // post state: video_url + display_url (escaped JSON strings inside the SSR payload)
+  const video = html.match(/\\"video_url\\":\\"((?:[^"\\]|\\.)+?)\\"/) || html.match(/"video_url":"((?:[^"\\]|\\.)+?)"/);
+  const image = html.match(/\\"display_url\\":\\"((?:[^"\\]|\\.)+?)\\"/) || html.match(/"display_url":"((?:[^"\\]|\\.)+?)"/);
+  if (video?.[1]) {
+    const u = un(video[1]);
+    if (/^https?:/.test(u)) media.push({
+      type: "video", index: 0, url: u, thumbnail: image ? un(image[1]) : u,
+      mimeType: "video/mp4", width: null, height: null, duration: null, size: null,
+      quality: null, hasAudio: true, hasVideo: true, source: name, variants: [],
+    });
+  }
+  if (!media.length && image?.[1]) {
+    const u = un(image[1]);
+    if (/^https?:/.test(u)) media.push({
+      type: "image", index: 0, url: u, thumbnail: u,
+      mimeType: "image/jpeg", width: null, height: null, duration: null, size: null,
+      quality: null, hasAudio: false, hasVideo: false, source: name, variants: [],
+    });
+  }
+
+  // og: meta fallback
+  if (!media.length) {
+    const ogv = (html.match(/property="og:video"[^>]+content="([^"]+)"/) || html.match(/content="([^"]+)"[^>]+property="og:video"/) || [])[1];
+    const ogi = (html.match(/property="og:image"[^>]+content="([^"]+)"/) || html.match(/content="([^"]+)"[^>]+property="og:image"/) || [])[1];
+    const u = ogv || ogi;
+    if (u && /^https?:/.test(u)) media.push({
+      type: ogv ? "video" : "image", index: 0, url: u.replace(/&/g, "&"), thumbnail: (ogi || u).replace(/&/g, "&"),
+      mimeType: ogv ? "video/mp4" : "image/jpeg", width: null, height: null,
+      duration: null, size: null, quality: null,
+      hasAudio: Boolean(ogv), hasVideo: Boolean(ogv), source: name, variants: [],
+    });
+  }
+
+  if (!media.length) {
+    attempt.status = "failed";
+    attempt.error = "no media in the public page (login-gated or removed)";
+    throw new UMediaError(Codes.MEDIA_NOT_FOUND,
+      `Threads exposes no media for this URL (${attempt.error}) — the yt-dlp tier covers gated posts.`, { attempts: [attempt] });
+  }
+
+  const title = (html.match(/"caption":"((?:[^"\\]|\\.){1,200})"/) || html.match(/<title[^>]*>([^<]*)/i) || [])[1] || null;
+  attempt.status = "success";
+  attempt.latencyMs = Date.now() - t0;
+  return {
+    result: {
+      sourceUrl: url,
+      platform: name,
+      title: title ? un(title).slice(0, 200) : null,
+      author: (String(url).match(/@([\w.]+)/) || [])[1] || null,
+      thumbnail: media[0].thumbnail,
+      itemCount: media.length,
+      counts: { images: media.filter((m) => m.type === "image").length, videos: media.filter((m) => m.type === "video").length, audios: 0, other: 0 },
+      truncated: false,
+      media,
+      streamUrlsAvailable: true,
+    },
+    engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
+  };
+}
+
+export async function search() {
+  return [];
+}

package/lib/providers/tumblr.js

@@ -0,0 +1,153 @@
+import { fetchJson, hostMatches } from "../util.js";
+import { Codes, UMediaError } from "../errors.js";
+
+/**
+ * Tumblr — the public mobile API (api-http2.tumblr.com, the same endpoint the
+ * official iPhone app calls with its public api_key). Videos, audio, photos —
+ * including reblog trails. Reference: cobalt tumblr service.
+ */
+export const name = "tumblr";
+
+const API_KEY = "jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z";
+const API_BASE = "https://api-http2.tumblr.com";
+const MOBILE = {
+  "user-agent": "Tumblr/iPhone/33.3/333010/17.3.1/tumblr",
+  "x-version": "iPhone/33.3/333010/17.3.1/tumblr",
+};
+
+export function canHandle(url) {
+  try {
+    return hostMatches(new URL(String(url)).hostname, ["tumblr.com"]);
+  } catch { return false; }
+}
+
+export function parseUrl(url) {
+  const u = new URL(String(url));
+  const host = u.hostname.replace(/^www\./, "");
+  let m;
+  // https://{user}.tumblr.com/post/{id}/...
+  m = u.pathname.match(/^\/post\/(\d+)/);
+  if (m && host !== "tumblr.com") return { user: host.split(".")[0], id: m[1] };
+  // https://www.tumblr.com/{user}/{id} or /blog/view/{user}/{id}
+  m = u.pathname.match(/^\/(?:blog\/view\/)?([\w.-]+)\/(\d+)/);
+  if (m) return { user: m[1], id: m[2] };
+  return null;
+}
+
+export async function resolve(url, ctx = {}) {
+  const attempt = { provider: name, status: "skipped" };
+  ctx.attempts?.push(attempt);
+  const t0 = Date.now();
+
+  const parsed = parseUrl(url);
+  if (!parsed) {
+    attempt.status = "failed";
+    throw new UMediaError(Codes.INVALID_URL, "Not a Tumblr post URL (expected {user}.tumblr.com/post/{id} or tumblr.com/{user}/{id})", { attempts: [attempt] });
+  }
+
+  let data;
+  try {
+    data = await fetchJson(
+      `${API_BASE}/v2/blog/${encodeURIComponent(parsed.user)}.tumblr.com/posts/${parsed.id}/permalink?api_key=${API_KEY}`, { headers: MOBILE }, 20_000);
+  } catch (e) {
+    attempt.status = "failed";
+    attempt.error = e.message;
+    throw new UMediaError(Codes.PROVIDER_UNAVAILABLE, `Tumblr API unavailable. ${e.message}`, { attempts: [attempt] });
+  }
+
+  const element = data?.response?.timeline?.elements?.[0];
+  if (!element) {
+    attempt.status = "failed";
+    attempt.error = data?.meta?.msg || "post not found";
+    throw new UMediaError(Codes.MEDIA_NOT_FOUND, `Tumblr: ${attempt.error}`, { attempts: [attempt] });
+  }
+
+  // contents: the post itself + any reblog trail, in order — ALL of it
+  const contents = [
+    ...(element.content || []),
+    ...(element.trail || []).flatMap((t) => t.content || []),
+  ];
+
+  const media = [];
+  for (const c of contents) {
+    if (c.type === "video" && (c.provider === "tumblr" || !c.provider) && c.media?.url) {
+      media.push({
+        type: "video", index: media.length, url: c.media.url,
+        thumbnail: c.poster?.[0]?.url || null,
+        mimeType: "video/mp4", width: c.width ?? null, height: c.height ?? null,
+        duration: null, size: null, quality: null,
+        hasAudio: true, hasVideo: true, source: name, variants: [],
+      });
+    } else if (c.type === "audio" && c.media?.url) {
+      media.push({
+        type: "audio", index: media.length, url: c.media.url,
+        thumbnail: null, mimeType: "audio/mpeg", width: null, height: null,
+        duration: null, size: null, quality: null,
+        hasAudio: true, hasVideo: false, source: name, variants: [],
+      });
+    } else if (c.type === "image" || c.type === "photo") {
+      // image content carries media as an ARRAY of renditions — take the best
+      // one as the item, keep the rest as variants. never inflate items.
+      const items = (Array.isArray(c.media) ? c.media : (c.media?.url ? [c.media] : []))
+        .filter((mm) => mm?.url);
+      const uniq = [...new Map(items.map((mm) => [mm.url, mm])).values()]
+        .sort((a, b) => ((b.width || 0) * (b.height || 0)) - ((a.width || 0) * (a.height || 0)));
+      const best = uniq[0];
+      if (best) {
+        media.push({
+          type: "image", index: media.length, url: best.url,
+          thumbnail: best.url, mimeType: "image/jpeg",
+          width: best.width ?? null, height: best.height ?? null,
+          duration: null, size: null, quality: best.width ? `${best.width}px` : null,
+          hasAudio: false, hasVideo: false, source: name,
+          variants: uniq.map((x) => ({
+            height: x.height ?? null, width: x.width ?? null, url: x.url,
+            hasAudio: false, hasVideo: false, quality: x.width ? `${x.width}px` : null, mimeType: "image/jpeg",
+          })),
+        });
+      }
+    }
+  }
+
+  // reblog trails repeat media from earlier hops — dedupe by URL, keep first order
+  const seen = new Set();
+  const deduped = [];
+  for (const mm of media) {
+    if (seen.has(mm.url)) continue;
+    seen.add(mm.url);
+    deduped.push({ ...mm, index: deduped.length });
+  }
+  media.length = 0;
+  media.push(...deduped);
+
+  if (!media.length) {
+    attempt.status = "failed";
+    throw new UMediaError(Codes.MEDIA_NOT_FOUND, "Tumblr post contains no downloadable media (text/link post, or third-party embed — use the yt-dlp tier for YouTube/Vimeo embeds)", { attempts: [attempt] });
+  }
+
+  attempt.status = "success";
+  attempt.latencyMs = Date.now() - t0;
+  return {
+    result: {
+      sourceUrl: url,
+      platform: name,
+      title: (element.summary || element.let || "").slice(0, 200) || null,
+      author: element.blog_name || parsed.user,
+      thumbnail: media[0].thumbnail,
+      itemCount: media.length,
+      counts: {
+        images: media.filter((m) => m.type === "image").length,
+        videos: media.filter((m) => m.type === "video").length,
+        audios: media.filter((m) => m.type === "audio").length, other: 0,
+      },
+      truncated: false,
+      media,
+      streamUrlsAvailable: true,
+    },
+    engine: { attempts: ctx.attempts || [attempt], finalProvider: name, totalLatencyMs: Date.now() - t0 },
+  };
+}
+
+export async function search() {
+  return [];
+}