webcrawl 1.2.4 → 1.2.7

package/dist/core/parse/youtube.js

@@ -0,0 +1,264 @@
+import { transcriptMarkdown } from "./media.js";
+import { getCachedPageAsync, setCachedPage } from "../engine/cache.js";
+const PLAYER_URL = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
+const MAX_ATTEMPTS = 3;
+const UA = "com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip";
+/** Tried in order; YouTube blocks some clients from datacenter IPs, so we fall through. */
+const CLIENTS = [
+    {
+        context: {
+            client: {
+                clientName: "ANDROID",
+                clientVersion: "20.10.38",
+                androidSdkVersion: 30,
+                hl: "en",
+            },
+        },
+        ua: UA,
+    },
+    {
+        context: {
+            client: {
+                clientName: "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
+                clientVersion: "2.0",
+                hl: "en",
+            },
+            thirdParty: { embedUrl: "https://www.youtube.com/" },
+        },
+        ua: "Mozilla/5.0 (SMART-TV; Linux; Tizen 6.5) AppleWebKit/537.36 (KHTML, like Gecko) 85.0.4183.93/6.5 TV Safari/537.36",
+    },
+    {
+        context: {
+            client: {
+                clientName: "WEB_EMBEDDED_PLAYER",
+                clientVersion: "1.20250310.01.00",
+                hl: "en",
+            },
+            thirdParty: { embedUrl: "https://www.google.com/" },
+        },
+        ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0 Safari/537.36",
+    },
+    {
+        context: {
+            client: {
+                clientName: "IOS",
+                clientVersion: "20.10.4",
+                deviceMake: "Apple",
+                deviceModel: "iPhone16,2",
+                osName: "iPhone",
+                osVersion: "18.3.2.22D82",
+                hl: "en",
+            },
+        },
+        ua: "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)",
+    },
+];
+/**
+ * YouTube refuses most cloud-hosting IP ranges. When YOUTUBE_RELAY_URL and YOUTUBE_RELAY_TOKEN are set,
+ * requests go through a relay on a network YouTube accepts; otherwise they go straight out.
+ */
+function youtubeFetch(url, init) {
+    const relay = process.env.YOUTUBE_RELAY_URL;
+    const token = process.env.YOUTUBE_RELAY_TOKEN;
+    if (!relay || !token)
+        return fetch(url, init);
+    return fetch(relay, {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+            url,
+            method: init.method,
+            headers: init.headers,
+            body: init.body,
+        }),
+        signal: init.signal,
+    });
+}
+export class YouTubeError extends Error {
+    status;
+    constructor(message, status = 422) {
+        super(message);
+        this.status = status;
+        this.name = "YouTubeError";
+    }
+}
+/** Returns the 11-character video id for watch, short, embed, shorts, live and youtu.be links. */
+export function youtubeVideoId(input) {
+    let u;
+    try {
+        u = new URL(input);
+    }
+    catch {
+        return undefined;
+    }
+    const host = u.hostname.replace(/^(www|m|music)\./, "");
+    const ok = (s) => s && /^[A-Za-z0-9_-]{11}$/.test(s) ? s : undefined;
+    if (host === "youtu.be")
+        return ok(u.pathname.slice(1).split("/")[0]);
+    if (host !== "youtube.com" && host !== "youtube-nocookie.com")
+        return undefined;
+    if (u.pathname === "/watch")
+        return ok(u.searchParams.get("v"));
+    const m = /^\/(?:embed|shorts|live|v)\/([^/?#]+)/.exec(u.pathname);
+    return m ? ok(m[1]) : undefined;
+}
+function decodeEntities(s) {
+    return s
+        .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
+        .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16)))
+        .replace(/"/g, '"')
+        .replace(/'/g, "'")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&amp;/g, "&");
+}
+/** Parses YouTube's timedtext XML (`<p t= d=>` in ms, optional `<s>` children). */
+export function parseTimedText(xml) {
+    const out = [];
+    for (const m of xml.matchAll(/<p\s([^>]*)>([\s\S]*?)<\/p>/g)) {
+        const t = /\bt="(\d+)"/.exec(m[1]);
+        if (!t)
+            continue;
+        const d = /\bd="(\d+)"/.exec(m[1]);
+        const text = decodeEntities(m[2].replace(/<[^>]+>/g, ""))
+            .replace(/\s+/g, " ")
+            .trim();
+        if (!text)
+            continue;
+        const start = Number(t[1]) / 1000;
+        out.push({
+            start,
+            end: Math.round((start + (d ? Number(d[1]) / 1000 : 0)) * 1000) / 1000,
+            text,
+        });
+    }
+    return out;
+}
+function pickTrack(tracks, language) {
+    const want = (language || "en").toLowerCase().split("-")[0];
+    const byLang = tracks.filter((t) => t.languageCode.toLowerCase().split("-")[0] === want);
+    const pool = byLang.length ? byLang : tracks;
+    // Human-written captions beat auto-generated ones.
+    return pool.find((t) => t.kind !== "asr") ?? pool[0];
+}
+export async function parseYouTube(videoId, options = {}) {
+    const started = performance.now();
+    const signal = options.signal ?? AbortSignal.timeout(25_000);
+    let player;
+    let ua = UA;
+    let lastReason = "";
+    // The relay's addresses are shared, so YouTube occasionally flags one with a bot check.
+    // A retry usually leaves from a different address, so retry only that case.
+    for (let attempt = 0; attempt < MAX_ATTEMPTS && !player; attempt++) {
+        if (attempt > 0 && !/not a bot/i.test(lastReason))
+            break;
+        if (attempt > 0)
+            await new Promise((r) => setTimeout(r, 400 * attempt));
+        for (const c of CLIENTS) {
+            try {
+                const res = await youtubeFetch(PLAYER_URL, {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json", "User-Agent": c.ua },
+                    body: JSON.stringify({
+                        context: c.context,
+                        videoId,
+                        contentCheckOk: true,
+                        racyCheckOk: true,
+                    }),
+                    signal,
+                });
+                if (!res.ok) {
+                    lastReason = `HTTP ${res.status}`;
+                    continue;
+                }
+                const p = (await res.json());
+                if (p.playabilityStatus?.status === "OK") {
+                    player = p;
+                    ua = c.ua;
+                    break;
+                }
+                lastReason =
+                    p.playabilityStatus?.reason ||
+                        p.playabilityStatus?.status?.toLowerCase() ||
+                        "unavailable";
+            }
+            catch (err) {
+                lastReason = err instanceof Error ? err.message : String(err);
+            }
+        }
+    }
+    if (!player)
+        throw new YouTubeError(`This video can't be read (${lastReason})`);
+    const tracks = player.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
+    if (!tracks.length)
+        throw new YouTubeError("This video has no captions, so there is no transcript to read");
+    const track = pickTrack(tracks, options.language);
+    const cap = await youtubeFetch(track.baseUrl, {
+        headers: { "User-Agent": ua },
+        signal,
+    });
+    if (!cap.ok)
+        throw new YouTubeError(`Could not load the captions (HTTP ${cap.status})`, 502);
+    const segments = parseTimedText(await cap.text());
+    if (!segments.length)
+        throw new YouTubeError("The captions for this video were empty");
+    const lengthSeconds = Number(player.videoDetails?.lengthSeconds);
+    const duration = Number.isFinite(lengthSeconds) && lengthSeconds > 0
+        ? lengthSeconds
+        : segments[segments.length - 1].end;
+    const warnings = [];
+    if (track.kind === "asr")
+        warnings.push("Captions are auto-generated by YouTube and may contain errors");
+    return {
+        title: player.videoDetails?.title || "",
+        markdown: `## Transcript\n\n${transcriptMarkdown(segments)}`,
+        meta: {
+            kind: "video",
+            mime: "video/youtube",
+            method: "transcript",
+            bytes: 0,
+            durationSeconds: duration,
+            transcriptProvider: "captions",
+            warnings,
+            processingMs: Math.round(performance.now() - started),
+        },
+    };
+}
+/** Same transcript for the same video and language, so successful reads are kept for 24 hours. Failures are never cached. */
+export async function parseYouTubeCached(videoId, options = {}) {
+    const key = `wc-parse://youtube:${videoId}/lang=${options.language ?? "default"}`;
+    if (!options.bypassCache) {
+        const hit = await getCachedPageAsync(key, "parse");
+        const meta = hit?.structuredData?.parse;
+        if (hit && meta) {
+            return {
+                title: hit.title,
+                markdown: hit.markdown,
+                meta,
+                contentHash: `youtube:${videoId}`,
+                cached: true,
+                cacheLayer: hit.cacheLayer || "postgres",
+            };
+        }
+    }
+    const parsed = await parseYouTube(videoId, options);
+    setCachedPage(key, "parse", {
+        title: parsed.title,
+        markdown: parsed.markdown,
+        links: [],
+        structuredData: {
+            parse: parsed.meta,
+        },
+        statusCode: 200,
+        contentHash: `youtube:${videoId}`,
+    });
+    return {
+        ...parsed,
+        contentHash: `youtube:${videoId}`,
+        cached: false,
+        cacheLayer: "miss",
+    };
+}

package/dist/core/ranker/decision.d.ts

@@ -0,0 +1,28 @@
+import type { DecisionModelId, SearchFilterContext, UserLocationContext } from "./types.js";
+/** What `decisionModel: "auto"` resolves to. Set WEBCRAWL_DECISION_MODEL to change it in prod. */
+export declare function defaultDecisionModel(): DecisionModelId;
+/**
+ * Stage 5: Vercel AI Gateway Decision Model Reranker (`baseten/mammoth-1` & `convaiinnovations/laya`).
+ * Scores all top-N candidates in ONE shared-prefix round-trip via `experimental_decide`.
+ */
+export declare function runDecisionModelReranker(query: string, candidates: Array<{
+    title: string;
+    url: string;
+    description: string;
+    highlights: string[];
+}>, modelOption: DecisionModelId, timeoutMs: number, userLocation?: UserLocationContext, filterContext?: SearchFilterContext): Promise<{
+    scores: Map<number, {
+        normalizedScore: number;
+        rawScore: number;
+        confidence: number;
+    }>;
+    modelUsed: string | null;
+} | null>;
+/**
+ * Full 5-Stage Document & Passage Ranking Pipeline:
+ * 1. Canonical URL Deduplication
+ * 2. Weighted Reciprocal Rank Fusion (W-RRF, k=60) + Multi-Engine Consensus
+ * 3. Fielded Okapi BM25F+ (title 3.5x, headings 2.2x, url 2.0x, description 1.5x, body 1.0x)
+ * 4. ColBERT-inspired Subword + Token MaxSim Passage Scoring & Highlight Extraction
+ * 5. Vercel AI Gateway Decision Model Reranking (`baseten/mammoth-1` / `convaiinnovations/laya`)
+ */

package/dist/core/ranker/decision.js

@@ -0,0 +1,151 @@
+import { createGateway } from "@ai-sdk/gateway";
+import { experimental_decide as decide } from "ai";
+import { resolveCountryDisplayName } from "./lexical.js";
+const DECISION_MODELS = [
+    "baseten/mammoth-1",
+    "baseten/elephant-1",
+    "convaiinnovations/laya",
+    "none",
+];
+/** What `decisionModel: "auto"` resolves to. Set WEBCRAWL_DECISION_MODEL to change it in prod. */
+export function defaultDecisionModel() {
+    const env = process.env.WEBCRAWL_DECISION_MODEL?.trim();
+    return env && DECISION_MODELS.includes(env) ? env : "baseten/mammoth-1";
+}
+/**
+ * Stage 5: Vercel AI Gateway Decision Model Reranker (`baseten/mammoth-1` & `convaiinnovations/laya`).
+ * Scores all top-N candidates in ONE shared-prefix round-trip via `experimental_decide`.
+ */
+export async function runDecisionModelReranker(query, candidates, modelOption, timeoutMs, userLocation, filterContext) {
+    // Operator switch: WEBCRAWL_DECISION_MODEL picks what "auto" means (or "none" for lexical-only),
+    // so a model price change can be absorbed without a deploy of new code.
+    if (modelOption === "auto")
+        modelOption = defaultDecisionModel();
+    if (modelOption === "none" || candidates.length === 0) {
+        return null;
+    }
+    const apiKey = process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+    if (!apiKey) {
+        return null;
+    }
+    // Primary model selection: `baseten/mammoth-1` (#1 on Jev Decision Index 71.24, ~350ms)
+    // or user-specified `convaiinnovations/laya` / `baseten/elephant-1`.
+    const primaryModel = modelOption === "auto" ? "baseten/mammoth-1" : modelOption;
+    const fallbackModels = primaryModel === "baseten/mammoth-1"
+        ? ["baseten/elephant-1", "convaiinnovations/laya"]
+        : ["baseten/mammoth-1", "baseten/elephant-1"];
+    const gw = createGateway({ apiKey });
+    const stateCandidates = candidates.map((c, idx) => ({
+        id: `doc_${idx}`,
+        title: c.title.slice(0, 180),
+        url: c.url.slice(0, 200),
+        snippet: (c.highlights[0] || c.description || "").slice(0, 360),
+    }));
+    const contextClauses = [];
+    if (filterContext?.category) {
+        contextClauses.push(`category="${filterContext.category}"`);
+    }
+    if (filterContext?.subcategory) {
+        const subStr = Array.isArray(filterContext.subcategory)
+            ? filterContext.subcategory.join(", ")
+            : filterContext.subcategory;
+        contextClauses.push(`subcategory="${subStr}"`);
+    }
+    if (userLocation?.city || userLocation?.region || userLocation?.country) {
+        const locParts = [
+            userLocation.city,
+            userLocation.region,
+            userLocation.countryName ||
+                resolveCountryDisplayName(userLocation.country),
+        ].filter(Boolean);
+        contextClauses.push(`userLocation="${locParts.join(", ")}"`);
+    }
+    if (filterContext?.customInstruction) {
+        contextClauses.push(`agentFilter="${filterContext.customInstruction}"`);
+    }
+    const contextSuffix = contextClauses.length > 0 ? ` (${contextClauses.join("; ")})` : "";
+    const questions = {};
+    for (let i = 0; i < stateCandidates.length; i++) {
+        questions[`doc_${i}`] = {
+            type: "score",
+            instructions: `Evaluate how relevant, authoritative, and aligned candidate doc_${i} is for the search query${contextSuffix}. Penalize entity homonym mismatches or wrong-region pages when userLocation or category is specified.`,
+            criteria: [
+                "Irrelevant, off-topic, wrong entity, wrong region, or spam",
+                "Tangentially related or generic syndication with weak match to the requested category/subcategory/location",
+                "Relevant and addresses a substantial part of the query and category/subcategory context",
+                "Highly relevant, authoritative, and directly satisfies the query, category, subcategory, and user location",
+            ],
+        };
+    }
+    try {
+        const decisionPromise = decide({
+            model: gw.decisionModel(primaryModel),
+            state: JSON.stringify({
+                query,
+                ...(filterContext?.category
+                    ? { category: filterContext.category }
+                    : {}),
+                ...(filterContext?.subcategory
+                    ? { subcategory: filterContext.subcategory }
+                    : {}),
+                ...(userLocation ? { userLocation } : {}),
+                candidates: stateCandidates,
+            }),
+            questions,
+            providerOptions: {
+                gateway: {
+                    models: fallbackModels,
+                },
+            },
+            abortSignal: AbortSignal.timeout(timeoutMs),
+        });
+        const result = await decisionPromise;
+        const scores = new Map();
+        for (let i = 0; i < stateCandidates.length; i++) {
+            const ans = result.answers?.[`doc_${i}`];
+            if (!ans) {
+                continue;
+            }
+            let rawScore = typeof ans.score === "number" ? ans.score : 0;
+            let maxProb = typeof ans.confidence === "number" ? ans.confidence : 0;
+            if (ans.probabilities) {
+                const probs = Array.isArray(ans.probabilities)
+                    ? ans.probabilities
+                    : [
+                        Number(ans.probabilities["0"] ?? 0),
+                        Number(ans.probabilities["1"] ?? 0),
+                        Number(ans.probabilities["2"] ?? 0),
+                        Number(ans.probabilities["3"] ?? 0),
+                    ];
+                const expected = 0 * (probs[0] || 0) +
+                    1 * (probs[1] || 0) +
+                    2 * (probs[2] || 0) +
+                    3 * (probs[3] || 0);
+                if (expected > 0 || rawScore === 0) {
+                    rawScore = expected;
+                }
+                maxProb = Math.max(maxProb, ...probs.filter((p) => Number.isFinite(p)));
+            }
+            const normalizedScore = Math.max(0, Math.min(1, rawScore / 3));
+            scores.set(i, {
+                normalizedScore,
+                rawScore: Number(rawScore.toFixed(4)),
+                confidence: Number((maxProb || 0.85).toFixed(4)),
+            });
+        }
+        const modelUsed = result.response?.modelId ||
+            primaryModel;
+        return { scores, modelUsed };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Full 5-Stage Document & Passage Ranking Pipeline:
+ * 1. Canonical URL Deduplication
+ * 2. Weighted Reciprocal Rank Fusion (W-RRF, k=60) + Multi-Engine Consensus
+ * 3. Fielded Okapi BM25F+ (title 3.5x, headings 2.2x, url 2.0x, description 1.5x, body 1.0x)
+ * 4. ColBERT-inspired Subword + Token MaxSim Passage Scoring & Highlight Extraction
+ * 5. Vercel AI Gateway Decision Model Reranking (`baseten/mammoth-1` / `convaiinnovations/laya`)
+ */

package/dist/core/ranker/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./types.js";
+export * from "./lexical.js";
+export * from "./decision.js";
+import type { RawCandidateDocument, RerankOptions, RerankResult } from "./types.js";
+export declare function rankDocuments(query: string, rawCandidates: RawCandidateDocument[], options?: RerankOptions): Promise<RerankResult>;