webcrawl 1.2.4 → 1.2.5

package/dist/core/parse/merge.js

@@ -0,0 +1,48 @@
+/** Splits the pages to read into fixed-size batches. */
+export function pdfBatches(pageCount, selected, size) {
+    const pages = selected?.length
+        ? [...new Set(selected)].filter((p) => p >= 1 && p <= pageCount).sort((a, b) => a - b)
+        : Array.from({ length: pageCount }, (_, i) => i + 1);
+    const out = [];
+    for (let i = 0; i < pages.length; i += size)
+        out.push(pages.slice(i, i + size));
+    return out;
+}
+/** Merges per-batch PDF results back into one document, in page order. */
+export function mergePdfChunks(chunks, info) {
+    const nums = (pick) => chunks.flatMap((c) => pick(c.meta) ?? []).sort((a, b) => a - b);
+    const skipped = chunks.flatMap((c) => c.meta.pagesSkipped ?? []).sort((a, b) => a.page - b.page);
+    const pagesNative = nums((m) => m.pagesNative);
+    const pagesOcr = nums((m) => m.pagesOcr);
+    const parts = chunks.map((c) => c.markdown).filter(Boolean);
+    // Single-page batches have no page marker; add one so the merged document is navigable.
+    const markdown = chunks
+        .map((c) => {
+        if (!c.markdown)
+            return "";
+        if (c.markdown.startsWith("<!-- page "))
+            return c.markdown;
+        const p = [...(c.meta.pagesNative ?? []), ...(c.meta.pagesOcr ?? [])].sort((a, b) => a - b)[0];
+        return parts.length > 1 && p ? `<!-- page ${p} -->\n\n${c.markdown}` : c.markdown;
+    })
+        .filter(Boolean)
+        .join("\n\n");
+    const first = chunks[0]?.meta;
+    return {
+        title: info.title || chunks.find((c) => c.title)?.title || "",
+        markdown,
+        meta: {
+            kind: "pdf",
+            mime: "application/pdf",
+            method: pagesOcr.length > pagesNative.length ? "ocr" : "native",
+            bytes: first?.bytes ?? 0,
+            pages: info.pageCount,
+            pagesNative,
+            pagesOcr,
+            pagesSkipped: skipped,
+            ocrProvider: chunks.map((c) => c.meta.ocrProvider).find(Boolean),
+            warnings: [...new Set(chunks.flatMap((c) => c.meta.warnings))],
+            processingMs: chunks.reduce((n, c) => n + c.meta.processingMs, 0),
+        },
+    };
+}

package/dist/core/parse/ocr.d.ts

@@ -0,0 +1,25 @@
+export interface OcrPage {
+    /** 1-indexed. */
+    page: number;
+    markdown: string;
+}
+export interface OcrResult {
+    provider: string;
+    pages: OcrPage[];
+}
+export interface OcrInput {
+    bytes: Uint8Array;
+    mime: string;
+    /** 1-indexed pages to read (PDF only). Omit for images or the whole file. */
+    pages?: number[];
+    signal?: AbortSignal;
+}
+export declare function ocrAvailable(): boolean;
+export declare function mistralOcr(input: OcrInput): Promise<OcrResult>;
+/** Splits "<<<PAGE n>>>" sections; keeps only pages that were asked for. */
+export declare function splitPageMarkers(text: string, wanted?: number[]): OcrPage[];
+export declare function geminiOcr(input: OcrInput): Promise<OcrResult>;
+/** Tries Mistral, then Gemini. Throws only if every configured provider fails. */
+export declare function runOcr(input: OcrInput): Promise<OcrResult & {
+    warnings: string[];
+}>;

package/dist/core/parse/ocr.js

@@ -0,0 +1,121 @@
+/**
+ * OCR for scanned PDF pages and images.
+ * Primary: Mistral OCR (MISTRAL_API_KEY). Fallback: Gemini Flash-Lite via AI Gateway.
+ * Each provider returns only the pages it actually read; callers report the rest as skipped.
+ */
+import { createGateway } from "@ai-sdk/gateway";
+import { generateText } from "ai";
+const MISTRAL_URL = "https://api.mistral.ai/v1/ocr";
+const DEFAULT_MISTRAL_MODEL = "mistral-ocr-latest";
+const DEFAULT_VISION_MODEL = "google/gemini-2.5-flash-lite";
+function gatewayKey() {
+    return process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+}
+export function ocrAvailable() {
+    return Boolean(process.env.MISTRAL_API_KEY || gatewayKey());
+}
+function toBase64(bytes) {
+    return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+}
+/** Mistral replaces figures with `![img-0.jpeg](img-0.jpeg)` placeholders we can't serve. */
+function stripPlaceholders(md) {
+    return md.replace(/!\[[^\]]*\]\((img-\d+\.\w+)\)\s*/g, "").trim();
+}
+export async function mistralOcr(input) {
+    const key = process.env.MISTRAL_API_KEY;
+    if (!key)
+        throw new Error("MISTRAL_API_KEY not set");
+    const isImage = input.mime.startsWith("image/");
+    const dataUrl = `data:${input.mime};base64,${toBase64(input.bytes)}`;
+    const body = {
+        model: process.env.WEBCRAWL_OCR_MODEL || DEFAULT_MISTRAL_MODEL,
+        document: isImage ? { type: "image_url", image_url: dataUrl } : { type: "document_url", document_url: dataUrl },
+        include_image_base64: false,
+    };
+    if (!isImage && input.pages?.length)
+        body.pages = input.pages.map((p) => p - 1);
+    const res = await fetch(MISTRAL_URL, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
+        body: JSON.stringify(body),
+        signal: input.signal ?? AbortSignal.timeout(120_000),
+    });
+    if (!res.ok) {
+        const detail = (await res.text().catch(() => "")).slice(0, 300);
+        throw new Error(`Mistral OCR ${res.status}: ${detail}`);
+    }
+    const json = (await res.json());
+    return {
+        provider: "mistral",
+        pages: (json.pages ?? []).map((p) => ({ page: p.index + 1, markdown: stripPlaceholders(p.markdown || "") })),
+    };
+}
+const PAGE_MARK = /<<<PAGE (\d+)>>>/g;
+/** Splits "<<<PAGE n>>>" sections; keeps only pages that were asked for. */
+export function splitPageMarkers(text, wanted) {
+    const marks = [...text.matchAll(PAGE_MARK)];
+    if (marks.length === 0) {
+        const md = text.trim();
+        return md && (!wanted || wanted.length === 1) ? [{ page: wanted?.[0] ?? 1, markdown: md }] : [];
+    }
+    const allowed = wanted ? new Set(wanted) : null;
+    const out = [];
+    marks.forEach((m, i) => {
+        const page = Number(m[1]);
+        const end = i + 1 < marks.length ? marks[i + 1].index : text.length;
+        const md = text.slice((m.index ?? 0) + m[0].length, end).trim();
+        if (md && (!allowed || allowed.has(page)))
+            out.push({ page, markdown: md });
+    });
+    return out;
+}
+export async function geminiOcr(input) {
+    const key = gatewayKey();
+    if (!key)
+        throw new Error("AI gateway credentials not set");
+    const gw = createGateway({ apiKey: key });
+    const isImage = input.mime.startsWith("image/");
+    const pageList = input.pages?.length ? input.pages.join(", ") : "all";
+    const instruction = isImage
+        ? "Transcribe all text in this image as GitHub-flavoured Markdown. Keep headings, lists and tables. Output only the transcription; output nothing if there is no text."
+        : `Transcribe pages ${pageList} of this PDF as GitHub-flavoured Markdown. Keep headings, lists and tables (as Markdown tables). ` +
+            "Start each page with a line <<<PAGE n>>> where n is the 1-based page number. Output only the transcription. Do not summarise or invent text.";
+    const { text } = await generateText({
+        model: gw(process.env.WEBCRAWL_VISION_MODEL || DEFAULT_VISION_MODEL),
+        maxOutputTokens: 32_000,
+        abortSignal: input.signal ?? AbortSignal.timeout(120_000),
+        messages: [
+            {
+                role: "user",
+                content: [
+                    { type: "file", data: input.bytes, mediaType: input.mime },
+                    { type: "text", text: instruction },
+                ],
+            },
+        ],
+    });
+    const pages = isImage ? (text.trim() ? [{ page: 1, markdown: text.trim() }] : []) : splitPageMarkers(text, input.pages);
+    return { provider: "gemini-flash-lite", pages };
+}
+/** Tries Mistral, then Gemini. Throws only if every configured provider fails. */
+export async function runOcr(input) {
+    const warnings = [];
+    const providers = [];
+    if (process.env.MISTRAL_API_KEY)
+        providers.push(() => mistralOcr(input));
+    if (gatewayKey())
+        providers.push(() => geminiOcr(input));
+    if (providers.length === 0)
+        throw new Error("No OCR provider configured");
+    let lastErr;
+    for (const run of providers) {
+        try {
+            return { ...(await run()), warnings };
+        }
+        catch (err) {
+            lastErr = err;
+            warnings.push(err instanceof Error ? err.message : String(err));
+        }
+    }
+    throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+}

package/dist/core/parse/office.d.ts

@@ -0,0 +1,28 @@
+/**
+ * DOCX / XLSX / PPTX -> Markdown, locally with officeparser (no network, no model).
+ * Sheets and slides get their own headings so a reader can tell where content came from.
+ */
+import type { FileKind } from "./detect.js";
+import type { ParsedFile } from "./types.js";
+type OfficeKind = Extract<FileKind, "docx" | "xlsx" | "pptx">;
+interface AstNode {
+    type: string;
+    metadata?: {
+        sheetName?: string;
+        slideNumber?: number;
+    };
+}
+/** Removes YAML front matter and `{#anchor}` heading ids officeparser adds. */
+export declare function cleanOfficeMarkdown(md: string): string;
+/**
+ * officeparser separates slides / sheets with `---` lines. Split on those so each
+ * section can be labelled with the slide number or sheet name from the AST.
+ */
+export declare function splitSections(md: string): string[];
+export declare function labelSections(kind: OfficeKind, sections: string[], nodes: AstNode[]): {
+    markdown: string;
+    units: number;
+    empty: string[];
+};
+export declare function parseOffice(bytes: Uint8Array, kind: OfficeKind, mime: string): Promise<ParsedFile>;
+export {};

package/dist/core/parse/office.js

@@ -0,0 +1,83 @@
+/** Removes YAML front matter and `{#anchor}` heading ids officeparser adds. */
+export function cleanOfficeMarkdown(md) {
+    return md
+        // Only strip a leading block that is really YAML (every line `key: value`); PPTX output
+        // also starts with `---`, and there the block is slide 1.
+        .replace(/^---\n((?:[\w:-]+:.*\n)+)---\n?/, "")
+        .replace(/^(#{1,6} .*?)[ \t]*\{#[^}]+\}[ \t]*$/gm, "$1")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/**
+ * officeparser separates slides / sheets with `---` lines. Split on those so each
+ * section can be labelled with the slide number or sheet name from the AST.
+ */
+export function splitSections(md) {
+    return md
+        .split(/^---\s*$/m)
+        .map((s) => s.trim())
+        .filter(Boolean);
+}
+export function labelSections(kind, sections, nodes) {
+    if (kind === "docx")
+        return { markdown: sections.join("\n\n---\n\n"), units: 1, empty: [] };
+    const unitType = kind === "xlsx" ? "sheet" : "slide";
+    const units = nodes.filter((n) => n.type === unitType);
+    const empty = [];
+    const out = [];
+    // Empty sheets/slides produce no section, so match non-empty sections to units in order.
+    let si = 0;
+    units.forEach((u, i) => {
+        const label = kind === "xlsx"
+            ? `## Sheet: ${u.metadata?.sheetName || `Sheet ${i + 1}`}`
+            : `<!-- slide ${u.metadata?.slideNumber ?? i + 1} -->`;
+        const isEmpty = kind === "xlsx" ? u.children?.length === 0 : false;
+        if (isEmpty) {
+            empty.push(u.metadata?.sheetName || `Sheet ${i + 1}`);
+            return;
+        }
+        const body = sections[si++];
+        if (body !== undefined)
+            out.push(`${label}\n\n${body}`);
+    });
+    // Anything left over (AST and Markdown disagree) is kept rather than dropped.
+    while (si < sections.length)
+        out.push(sections[si++]);
+    return { markdown: out.join("\n\n"), units: units.length || sections.length, empty };
+}
+export async function parseOffice(bytes, kind, mime) {
+    const started = performance.now();
+    const { OfficeConverter, OfficeParser } = await import("officeparser");
+    const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const parseConfig = { fileType: kind, ignoreNotes: false };
+    const [converted, ast] = await Promise.all([
+        OfficeConverter.convert(buf, "md", { parseConfig }),
+        OfficeParser.parseOffice(buf, parseConfig),
+    ]);
+    const cleaned = cleanOfficeMarkdown(String(converted.value ?? ""));
+    const nodes = (ast.content ?? []);
+    const { markdown, units, empty } = labelSections(kind, splitSections(cleaned), nodes);
+    const warnings = (converted.messages ?? [])
+        .filter((m) => m.type === "warning")
+        .map((m) => String(m.message ?? m))
+        .slice(0, 10);
+    if (empty.length)
+        warnings.push(`Empty sheets: ${empty.join(", ")}`);
+    const title = ast.metadata?.title?.trim() ||
+        markdown.match(/^#\s+(.+)$/m)?.[1]?.trim() ||
+        "";
+    return {
+        title,
+        markdown,
+        meta: {
+            kind,
+            mime,
+            method: "native",
+            bytes: bytes.byteLength,
+            // "pages" means sheets for XLSX and slides for PPTX; DOCX has no reliable page count.
+            ...(kind === "docx" ? {} : { pages: units }),
+            warnings,
+            processingMs: Math.round(performance.now() - started),
+        },
+    };
+}

package/dist/core/parse/pdf.d.ts

@@ -0,0 +1,27 @@
+import type { ParsedFile, PdfParseOptions, SkippedPage } from "./types.js";
+export interface PdfPlan {
+    pageCount: number;
+    /** 1-indexed pages to read natively. */
+    native: number[];
+    /** 1-indexed pages to OCR. */
+    ocr: number[];
+    skipped: SkippedPage[];
+}
+/**
+ * Decides which pages go where. Pure, so it can be unit-tested without a PDF.
+ */
+export declare function planPdfPages(input: {
+    pageCount: number;
+    needsOcr: number[];
+    options: PdfParseOptions;
+    ocrConfigured: boolean;
+}): PdfPlan;
+/** First real-looking h1/h2 (skipping arXiv/DOI stamps and page furniture), else first h3. */
+export declare function pickTitle(markdown: string): string;
+/** Page count and pages needing OCR, without extracting text (cheap: ~1 ms/page). */
+export declare function pdfPageInfo(bytes: Uint8Array): {
+    pageCount: number;
+    pagesNeedingOcr: number[];
+    title?: string;
+};
+export declare function parsePdf(bytes: Uint8Array, options?: PdfParseOptions): Promise<ParsedFile>;

package/dist/core/parse/pdf.js

@@ -0,0 +1,156 @@
+/**
+ * PDF to Markdown. Text pages are read locally with pdf-inspector (free, ~5 ms/page).
+ * Only pages it flags as scanned go to OCR. Anything not read is reported as skipped.
+ */
+import { createRequire } from "node:module";
+import { ocrAvailable, runOcr } from "./ocr.js";
+let inspector = null;
+function loadInspector() {
+    // Native addon: load with require so the bundler leaves it alone.
+    if (!inspector)
+        inspector = createRequire(import.meta.url)("@firecrawl/pdf-inspector");
+    return inspector;
+}
+const DEFAULT_MAX_OCR_PAGES = Number(process.env.WEBCRAWL_PARSE_MAX_OCR_PAGES) || 50;
+/**
+ * Decides which pages go where. Pure, so it can be unit-tested without a PDF.
+ */
+export function planPdfPages(input) {
+    const { pageCount, options } = input;
+    const mode = options.ocr ?? "auto";
+    const maxOcr = options.maxOcrPages ?? DEFAULT_MAX_OCR_PAGES;
+    const all = Array.from({ length: pageCount }, (_, i) => i + 1);
+    const selected = options.pages?.length
+        ? [...new Set(options.pages)].filter((p) => p >= 1 && p <= pageCount).sort((a, b) => a - b)
+        : all;
+    const flagged = new Set(input.needsOcr);
+    const native = [];
+    const wantOcr = [];
+    for (const p of selected) {
+        if (mode === "force" || (mode === "auto" && flagged.has(p)))
+            wantOcr.push(p);
+        else
+            native.push(p);
+    }
+    const skipped = [];
+    let ocr = [];
+    if (mode === "off") {
+        for (const p of selected)
+            if (flagged.has(p))
+                skipped.push({ page: p, reason: "ocr_disabled" });
+    }
+    else if (!input.ocrConfigured) {
+        for (const p of wantOcr)
+            skipped.push({ page: p, reason: "ocr_unavailable" });
+        // In force mode, still return whatever text the page has natively.
+        if (mode === "force")
+            native.push(...wantOcr.filter((p) => !flagged.has(p)));
+    }
+    else {
+        ocr = wantOcr.slice(0, maxOcr);
+        for (const p of wantOcr.slice(maxOcr))
+            skipped.push({ page: p, reason: "page_limit" });
+    }
+    native.sort((a, b) => a - b);
+    return { pageCount, native, ocr, skipped };
+}
+/** First real-looking h1/h2 (skipping arXiv/DOI stamps and page furniture), else first h3. */
+export function pickTitle(markdown) {
+    const clean = (s) => s.replace(/<\/?\w+>|[*_]/g, "").trim();
+    const usable = (s) => {
+        const t = clean(s);
+        return t.length >= 3 && t.length <= 160 && !/^(arxiv:|doi:|page \d|\d+$)/i.test(t);
+    };
+    const headings = [...markdown.matchAll(/^(#{1,3})\s+(.+)$/gm)].map((m) => ({ level: m[1].length, text: m[2] }));
+    const pick = headings.find((h) => h.level <= 2 && usable(h.text)) ?? headings.find((h) => usable(h.text));
+    return pick ? clean(pick.text) : "";
+}
+function toBuffer(bytes) {
+    return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+}
+/** Page count and pages needing OCR, without extracting text (cheap: ~1 ms/page). */
+export function pdfPageInfo(bytes) {
+    const info = loadInspector().detectPdf(toBuffer(bytes));
+    return { pageCount: info.pageCount, pagesNeedingOcr: info.pagesNeedingOcr, title: info.title?.trim() || undefined };
+}
+export async function parsePdf(bytes, options = {}) {
+    const started = Date.now();
+    const buf = toBuffer(bytes);
+    const pdf = loadInspector();
+    const warnings = [];
+    const info = pdf.detectPdf(buf);
+    const plan = planPdfPages({
+        pageCount: info.pageCount,
+        needsOcr: info.pagesNeedingOcr,
+        options,
+        ocrConfigured: ocrAvailable(),
+    });
+    if (info.hasEncodingIssues)
+        warnings.push("Some fonts have missing character maps; text may contain gaps.");
+    const byPage = new Map();
+    const nativeRead = [];
+    if (plan.native.length) {
+        // extractPagesMarkdown takes and returns 0-indexed pages; detectPdf reports 1-indexed ones.
+        const extracted = await pdf.extractPagesMarkdownAsync(buf, plan.native.map((p) => p - 1));
+        for (const raw of extracted.pages) {
+            const page = raw.page + 1;
+            const md = raw.markdown.trim();
+            if (md) {
+                byPage.set(page, md);
+                nativeRead.push(page);
+            }
+            else if (!plan.ocr.includes(page) && !plan.skipped.some((s) => s.page === page)) {
+                plan.skipped.push({ page, reason: "empty" });
+            }
+        }
+    }
+    const ocrRead = [];
+    let ocrProvider;
+    if (plan.ocr.length) {
+        try {
+            const result = await runOcr({ bytes: buf, mime: "application/pdf", pages: plan.ocr, signal: options.signal });
+            ocrProvider = result.provider;
+            warnings.push(...result.warnings.map((w) => `OCR fallback: ${w}`));
+            const wanted = new Set(plan.ocr);
+            for (const p of result.pages) {
+                if (!wanted.has(p.page) || !p.markdown)
+                    continue;
+                byPage.set(p.page, p.markdown);
+                ocrRead.push(p.page);
+            }
+            for (const p of plan.ocr)
+                if (!ocrRead.includes(p))
+                    plan.skipped.push({ page: p, reason: "ocr_failed" });
+        }
+        catch (err) {
+            warnings.push(`OCR failed: ${err instanceof Error ? err.message : String(err)}`);
+            for (const p of plan.ocr)
+                plan.skipped.push({ page: p, reason: "ocr_failed" });
+        }
+    }
+    const ordered = [...byPage.keys()].sort((a, b) => a - b);
+    const multi = ordered.length > 1;
+    const markdown = ordered
+        .map((p) => (multi ? `<!-- page ${p} -->\n\n${byPage.get(p)}` : byPage.get(p)))
+        .join("\n\n");
+    plan.skipped.sort((a, b) => a.page - b.page);
+    // Prefer document metadata, then the highest-level heading (first page banners are often ###).
+    const title = info.title?.trim() || pickTitle(markdown);
+    return {
+        title,
+        markdown,
+        meta: {
+            kind: "pdf",
+            mime: "application/pdf",
+            method: ocrRead.length > nativeRead.length ? "ocr" : "native",
+            bytes: buf.byteLength,
+            pages: info.pageCount,
+            pagesNative: nativeRead.sort((a, b) => a - b),
+            pagesOcr: ocrRead.sort((a, b) => a - b),
+            pagesSkipped: plan.skipped,
+            ocrProvider,
+            warnings,
+            processingMs: Date.now() - started,
+        },
+    };
+}

package/dist/core/parse/types.d.ts

@@ -0,0 +1,38 @@
+import type { FileKind } from "./detect.js";
+/** How the bulk of the content was produced. */
+export type ParseMethod = "native" | "ocr" | "vision" | "transcript";
+export interface SkippedPage {
+    page: number;
+    reason: "ocr_disabled" | "ocr_unavailable" | "ocr_failed" | "page_limit" | "empty";
+}
+/** Reported to the caller as `metadata.parse`, and used to charge credits. */
+export interface ParseMeta {
+    kind: FileKind;
+    mime: string;
+    method: ParseMethod;
+    bytes: number;
+    pages?: number;
+    pagesNative?: number[];
+    pagesOcr?: number[];
+    pagesSkipped?: SkippedPage[];
+    ocrProvider?: string;
+    durationSeconds?: number;
+    transcriptProvider?: string;
+    visionProvider?: string;
+    warnings: string[];
+    processingMs: number;
+}
+export interface ParsedFile {
+    title: string;
+    markdown: string;
+    meta: ParseMeta;
+}
+export interface PdfParseOptions {
+    /** "auto": OCR only flagged pages; "force": OCR every page; "off": never OCR. */
+    ocr?: "auto" | "force" | "off";
+    /** 1-indexed page numbers to include. */
+    pages?: number[];
+    /** Hard cap on pages sent to OCR in one request. */
+    maxOcrPages?: number;
+    signal?: AbortSignal;
+}

package/dist/core/parse/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/parse/youtube.d.ts

@@ -0,0 +1,30 @@
+/**
+ * YouTube link -> timestamped transcript from the video's own captions.
+ * No audio download and no model call: we read the caption track YouTube already publishes.
+ * Videos without captions return an error rather than an empty or invented transcript.
+ */
+import type { Segment } from "./media.js";
+import type { ParsedFile } from "./types.js";
+export declare class YouTubeError extends Error {
+    readonly status: number;
+    constructor(message: string, status?: number);
+}
+/** Returns the 11-character video id for watch, short, embed, shorts, live and youtu.be links. */
+export declare function youtubeVideoId(input: string): string | undefined;
+/** Parses YouTube's timedtext XML (`<p t= d=>` in ms, optional `<s>` children). */
+export declare function parseTimedText(xml: string): Segment[];
+export declare function parseYouTube(videoId: string, options?: {
+    language?: string;
+    signal?: AbortSignal;
+}): Promise<ParsedFile>;
+export interface YouTubeResult extends ParsedFile {
+    contentHash: string;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+}
+/** Same transcript for the same video and language, so successful reads are kept for 24 hours. Failures are never cached. */
+export declare function parseYouTubeCached(videoId: string, options?: {
+    language?: string;
+    signal?: AbortSignal;
+    bypassCache?: boolean;
+}): Promise<YouTubeResult>;