webcrawl 1.2.4 → 1.2.7

package/dist/core/parse/image.js

@@ -0,0 +1,100 @@
+/**
+ * Images -> a short description plus any text in the image, from Gemini Flash-Lite
+ * through AI Gateway. One call per image. Without credentials nothing is invented:
+ * the result is empty with a warning.
+ */
+import { createGateway } from "@ai-sdk/gateway";
+import { generateText } from "ai";
+const DEFAULT_VISION_MODEL = "google/gemini-2.5-flash-lite";
+export const VISION_PROVIDER = "gemini-flash-lite";
+function gatewayKey() {
+    return process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+}
+export function visionAvailable() {
+    return Boolean(gatewayKey());
+}
+const PROMPT = [
+    "Describe this image for someone who cannot see it, then transcribe any text in it.",
+    "Reply in exactly this format:",
+    "<<<DESCRIPTION>>>",
+    "Two to four plain sentences: what the image shows, and the key facts a reader needs (numbers in charts, labels in diagrams).",
+    "<<<TEXT>>>",
+    "All legible text in the image as GitHub-flavoured Markdown (keep tables as tables). Write NONE if there is no text.",
+    "Do not guess at text you cannot read.",
+].join("\n");
+/** Splits the model reply into description and text. Tolerates a missing TEXT marker. */
+export function splitVisionReply(reply) {
+    const d = reply.indexOf("<<<DESCRIPTION>>>");
+    const t = reply.indexOf("<<<TEXT>>>");
+    if (d === -1 && t === -1)
+        return { description: reply.trim(), text: "" };
+    const description = (t === -1 ? reply.slice(d + 17) : reply.slice(d === -1 ? 0 : d + 17, t)).trim();
+    let text = t === -1 ? "" : reply.slice(t + 10).trim();
+    if (/^none\.?$/i.test(text))
+        text = "";
+    return { description, text };
+}
+export function imageMarkdown(description, text) {
+    const parts = [];
+    if (description)
+        parts.push(`## Description\n\n${description}`);
+    if (text)
+        parts.push(`## Text in image\n\n${text}`);
+    return parts.join("\n\n");
+}
+export async function parseImage(bytes, mime, options = {}) {
+    const started = performance.now();
+    const warnings = [];
+    const base = {
+        kind: "image",
+        mime,
+        method: "vision",
+        bytes: bytes.byteLength,
+    };
+    const title = options.fileName?.split("/").pop()?.replace(/\.[a-z0-9]+$/i, "") || "";
+    const key = gatewayKey();
+    if (!key) {
+        warnings.push("Image description unavailable: no vision model configured");
+        return {
+            title,
+            markdown: "",
+            meta: { ...base, warnings, processingMs: Math.round(performance.now() - started) },
+        };
+    }
+    try {
+        const gw = createGateway({ apiKey: key });
+        const { text: reply } = await generateText({
+            model: gw(process.env.WEBCRAWL_VISION_MODEL || DEFAULT_VISION_MODEL),
+            maxOutputTokens: 4_000,
+            abortSignal: options.signal ?? AbortSignal.timeout(60_000),
+            messages: [
+                {
+                    role: "user",
+                    content: [
+                        { type: "image", image: bytes, mediaType: mime },
+                        { type: "text", text: PROMPT },
+                    ],
+                },
+            ],
+        });
+        const { description, text } = splitVisionReply(reply);
+        return {
+            title,
+            markdown: imageMarkdown(description, text),
+            meta: {
+                ...base,
+                visionProvider: VISION_PROVIDER,
+                warnings,
+                processingMs: Math.round(performance.now() - started),
+            },
+        };
+    }
+    catch (err) {
+        warnings.push(`Image description failed: ${err instanceof Error ? err.message : String(err)}`);
+        return {
+            title,
+            markdown: "",
+            meta: { ...base, warnings, processingMs: Math.round(performance.now() - started) },
+        };
+    }
+}

package/dist/core/parse/index.d.ts

@@ -0,0 +1,53 @@
+import { type FileKind } from "./detect.js";
+import type { ParsedFile, ParseMeta } from "./types.js";
+export { detectFileKind, BINARY_KINDS, type FileKind } from "./detect.js";
+export type { ParsedFile, ParseMeta } from "./types.js";
+export interface ParseFileOptions {
+    ocr?: "auto" | "force" | "off";
+    pages?: number[];
+    maxOcrPages?: number;
+    captions?: string;
+    language?: string;
+    describeVideo?: boolean;
+    bypassCache?: boolean;
+    signal?: AbortSignal;
+}
+export interface ParseFileInput {
+    bytes: Uint8Array;
+    contentType?: string;
+    fileName?: string;
+}
+export declare class FileTooLargeError extends Error {
+    readonly kind: FileKind;
+    readonly bytes: number;
+    readonly maxBytes: number;
+    constructor(kind: FileKind, bytes: number, maxBytes: number);
+}
+export declare class UnsupportedFileError extends Error {
+    readonly kind: FileKind;
+    constructor(kind: FileKind);
+}
+/** Converts parse metadata into billable usage. Cache hits are billed like the original parse. */
+export declare function parseUsageFromMeta(meta: ParseMeta): {
+    kind: FileKind;
+    method: import("./types.js").ParseMethod;
+    pagesNative: number | undefined;
+    pagesOcr: number | undefined;
+    durationSeconds: number | undefined;
+    transcribed: boolean;
+    transcriptProvider: string | undefined;
+    videoDescribed: boolean;
+    imageDescribed: boolean;
+};
+/** Public shape of `metadata.parse` in API responses. */
+export declare function publicParseMeta(r: ParseFileResult): Record<string, unknown>;
+export declare function isBinaryKind(kind: FileKind): boolean;
+export declare function contentHash(bytes: Uint8Array): string;
+/** Only options that change the output go into the cache key. */
+export declare function parseCacheKey(hash: string, kind: FileKind, o: ParseFileOptions): string;
+export interface ParseFileResult extends ParsedFile {
+    contentHash: string;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+}
+export declare function parseFile(input: ParseFileInput, options?: ParseFileOptions): Promise<ParseFileResult>;

package/dist/core/parse/index.js

@@ -0,0 +1,153 @@
+/**
+ * One entry point for binary files: detect the kind, check the size cap, route to the
+ * right parser, and cache by content hash (same bytes => same result, whatever the URL).
+ */
+import crypto from "node:crypto";
+import { getCachedPageAsync, setCachedPage } from "../engine/cache.js";
+import { BINARY_KINDS, detectFileKind } from "./detect.js";
+import { maxBytesFor } from "./limits.js";
+import { parseImage } from "./image.js";
+import { parseMedia } from "./media.js";
+import { parseOffice } from "./office.js";
+import { parsePdf } from "./pdf.js";
+export { detectFileKind, BINARY_KINDS } from "./detect.js";
+export class FileTooLargeError extends Error {
+    kind;
+    bytes;
+    maxBytes;
+    constructor(kind, bytes, maxBytes) {
+        super(`${kind.toUpperCase()} is ${(bytes / 1e6).toFixed(1)} MB; the limit is ${(maxBytes / 1e6).toFixed(0)} MB`);
+        this.kind = kind;
+        this.bytes = bytes;
+        this.maxBytes = maxBytes;
+        this.name = "FileTooLargeError";
+    }
+}
+export class UnsupportedFileError extends Error {
+    kind;
+    constructor(kind) {
+        super(`Not a binary file kind (${kind}); use the HTML/text pipeline`);
+        this.kind = kind;
+        this.name = "UnsupportedFileError";
+    }
+}
+/** Converts parse metadata into billable usage. Cache hits are billed like the original parse. */
+export function parseUsageFromMeta(meta) {
+    const isOffice = meta.kind === "docx" || meta.kind === "xlsx" || meta.kind === "pptx";
+    return {
+        kind: meta.kind,
+        method: meta.method,
+        pagesNative: meta.pagesNative?.length ?? (isOffice ? meta.pages ?? 1 : undefined),
+        pagesOcr: meta.pagesOcr?.length,
+        durationSeconds: meta.durationSeconds,
+        transcribed: Boolean(meta.transcriptProvider && meta.transcriptProvider !== "captions"),
+        transcriptProvider: meta.transcriptProvider,
+        videoDescribed: meta.kind === "video" && Boolean(meta.visionProvider),
+        imageDescribed: meta.kind === "image" && Boolean(meta.visionProvider),
+    };
+}
+/** Public shape of `metadata.parse` in API responses. */
+export function publicParseMeta(r) {
+    return { ...r.meta, contentHash: r.contentHash, cached: r.cached };
+}
+export function isBinaryKind(kind) {
+    return BINARY_KINDS.has(kind);
+}
+export function contentHash(bytes) {
+    return crypto.createHash("sha256").update(bytes).digest("hex");
+}
+/** Only options that change the output go into the cache key. */
+export function parseCacheKey(hash, kind, o) {
+    const parts = [];
+    if (kind === "pdf") {
+        parts.push(`ocr=${o.ocr ?? "auto"}`);
+        if (o.pages?.length)
+            parts.push(`pages=${o.pages.join(",")}`);
+        if (o.maxOcrPages)
+            parts.push(`max=${o.maxOcrPages}`);
+    }
+    if (kind === "audio" || kind === "video") {
+        if (o.captions)
+            parts.push(`cap=${crypto.createHash("sha256").update(o.captions).digest("hex").slice(0, 16)}`);
+        if (o.language)
+            parts.push(`lang=${o.language}`);
+        if (kind === "video" && o.describeVideo)
+            parts.push("describe");
+    }
+    return `wc-parse://${hash}/${parts.join("&") || "default"}`;
+}
+/** A result is worth caching only if nothing failed transiently. */
+function cacheable(meta) {
+    if (meta.pagesSkipped?.some((s) => s.reason === "ocr_failed" || s.reason === "ocr_unavailable"))
+        return false;
+    return !meta.warnings.some((w) => /failed|unavailable/i.test(w));
+}
+export async function parseFile(input, options = {}) {
+    const { kind, mime } = detectFileKind(input);
+    if (!isBinaryKind(kind))
+        throw new UnsupportedFileError(kind);
+    const max = maxBytesFor(kind);
+    if (input.bytes.byteLength > max)
+        throw new FileTooLargeError(kind, input.bytes.byteLength, max);
+    const hash = contentHash(input.bytes);
+    const key = parseCacheKey(hash, kind, options);
+    if (!options.bypassCache) {
+        const hit = await getCachedPageAsync(key, "parse");
+        const meta = hit?.structuredData?.parse;
+        if (hit && meta) {
+            return {
+                title: hit.title,
+                markdown: hit.markdown,
+                meta,
+                contentHash: hash,
+                cached: true,
+                cacheLayer: hit.cacheLayer || "postgres",
+            };
+        }
+    }
+    let parsed;
+    switch (kind) {
+        case "pdf":
+            parsed = await parsePdf(input.bytes, {
+                ocr: options.ocr,
+                pages: options.pages,
+                maxOcrPages: options.maxOcrPages,
+                signal: options.signal,
+            });
+            break;
+        case "docx":
+        case "xlsx":
+        case "pptx":
+            parsed = await parseOffice(input.bytes, kind, mime);
+            break;
+        case "image":
+            parsed = await parseImage(input.bytes, mime, { fileName: input.fileName, signal: options.signal });
+            break;
+        case "audio":
+        case "video":
+            parsed = await parseMedia(input.bytes, kind, mime, {
+                fileName: input.fileName,
+                captions: options.captions,
+                language: options.language,
+                describeVideo: options.describeVideo,
+                signal: options.signal,
+            });
+            break;
+        default:
+            throw new UnsupportedFileError(kind);
+    }
+    if (!parsed.title && input.fileName) {
+        parsed.title = input.fileName.split(/[?#]/)[0].split("/").pop() || "";
+    }
+    if (cacheable(parsed.meta) && parsed.markdown.length > 0) {
+        setCachedPage(key, "parse", {
+            title: parsed.title,
+            markdown: parsed.markdown,
+            links: [],
+            structuredData: { parse: parsed.meta },
+            statusCode: 200,
+            contentHash: hash,
+        });
+    }
+    return { ...parsed, contentHash: hash, cached: false, cacheLayer: "miss" };
+}

package/dist/core/parse/limits.d.ts

@@ -0,0 +1,12 @@
+import type { FileKind } from "./detect.js";
+/** Per-kind size caps. Override with WEBCRAWL_PARSE_MAX_{DOC,IMAGE,MEDIA}_MB. */
+export declare function maxBytesFor(kind: FileKind): number;
+/** Upper bound for any download before the kind is known. */
+export declare function maxDownloadBytes(): number;
+/** Jobs bigger than these run as a background workflow instead of inline. */
+export declare const ASYNC_THRESHOLDS: {
+    pdfPages: number;
+    mediaSeconds: number;
+    pdfBatchPages: number;
+    mediaChunkSeconds: number;
+};

package/dist/core/parse/limits.js

@@ -0,0 +1,29 @@
+const MB = 1024 * 1024;
+function envMb(name, fallbackMb) {
+    const raw = Number(process.env[name]);
+    return (Number.isFinite(raw) && raw > 0 ? raw : fallbackMb) * MB;
+}
+/** Per-kind size caps. Override with WEBCRAWL_PARSE_MAX_{DOC,IMAGE,MEDIA}_MB. */
+export function maxBytesFor(kind) {
+    switch (kind) {
+        case "image":
+            return envMb("WEBCRAWL_PARSE_MAX_IMAGE_MB", 20);
+        case "audio":
+        case "video":
+            // Groq's transcription upload limit.
+            return envMb("WEBCRAWL_PARSE_MAX_MEDIA_MB", 100);
+        default:
+            return envMb("WEBCRAWL_PARSE_MAX_DOC_MB", 50);
+    }
+}
+/** Upper bound for any download before the kind is known. */
+export function maxDownloadBytes() {
+    return Math.max(maxBytesFor("pdf"), maxBytesFor("image"), maxBytesFor("audio"));
+}
+/** Jobs bigger than these run as a background workflow instead of inline. */
+export const ASYNC_THRESHOLDS = {
+    pdfPages: Number(process.env.WEBCRAWL_PARSE_ASYNC_PAGES) || 50,
+    mediaSeconds: Number(process.env.WEBCRAWL_PARSE_ASYNC_SECONDS) || 600,
+    pdfBatchPages: 20,
+    mediaChunkSeconds: 600,
+};

package/dist/core/parse/media.d.ts

@@ -0,0 +1,42 @@
+import type { ParsedFile } from "./types.js";
+export interface Segment {
+    start: number;
+    end: number;
+    text: string;
+}
+export interface MediaParseOptions {
+    fileName?: string;
+    /** WebVTT or SRT text. Used instead of transcription when present. */
+    captions?: string;
+    /** BCP-47 / ISO-639-1 hint for Whisper, e.g. "en". */
+    language?: string;
+    /** Video only: add a short description of what is on screen. */
+    describeVideo?: boolean;
+    signal?: AbortSignal;
+}
+export declare function transcriptionAvailable(): boolean;
+/** Transcribes through the Vercel AI Gateway (no extra provider key needed). */
+export declare function gatewayTranscribe(bytes: Uint8Array, options?: {
+    language?: string;
+    signal?: AbortSignal;
+}): Promise<{
+    segments: Segment[];
+    duration: number;
+    language?: string;
+}>;
+/** 3725.4 -> "1:02:05"; 65 -> "1:05". */
+export declare function formatTimestamp(seconds: number): string;
+/** Parses WebVTT or SRT cues. Drops styling tags and repeated rolling-caption lines. */
+export declare function parseCaptions(text: string): Segment[];
+/** Groups segments into ~30 s paragraphs, each prefixed with its start time. */
+export declare function transcriptMarkdown(segments: Segment[], paragraphSeconds?: number): string;
+export declare function groqTranscribe(bytes: Uint8Array, mime: string, options?: {
+    fileName?: string;
+    language?: string;
+    signal?: AbortSignal;
+}): Promise<{
+    segments: Segment[];
+    duration: number;
+    language?: string;
+}>;
+export declare function parseMedia(bytes: Uint8Array, kind: "audio" | "video", mime: string, options?: MediaParseOptions): Promise<ParsedFile>;

package/dist/core/parse/media.js

@@ -0,0 +1,238 @@
+/**
+ * Audio / video -> timestamped transcript.
+ * 1. Captions first: if the caller passes captions (WebVTT or SRT), use them. No model call.
+ * 2. Otherwise Groq `whisper-large-v3-turbo` if GROQ_API_KEY is set, falling back to the AI Gateway
+ *    (`spacexai/grok-stt` by default, billed at 2 credits/min) when Groq is missing or fails.
+ * 3. Optional visual description for video via Gemini Flash-Lite (AI Gateway).
+ * Nothing is invented: without a provider the transcript is empty with a warning.
+ */
+import { createGateway } from "@ai-sdk/gateway";
+import { experimental_transcribe, generateText } from "ai";
+const GROQ_URL = "https://api.groq.com/openai/v1/audio/transcriptions";
+const DEFAULT_TRANSCRIBE_MODEL = "whisper-large-v3-turbo";
+const DEFAULT_VISION_MODEL = "google/gemini-2.5-flash-lite";
+/** Gemini inline video limit through the gateway; larger videos skip the description. */
+const MAX_VIDEO_DESCRIBE_BYTES = 20 * 1024 * 1024;
+export function transcriptionAvailable() {
+    return Boolean(process.env.GROQ_API_KEY || gatewayKey());
+}
+function gatewayKey() {
+    return process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+}
+/** ~$0.0017/min with segment timestamps; whisper-1 is $0.006/min. Override with WEBCRAWL_GATEWAY_TRANSCRIBE_MODEL. */
+const DEFAULT_GATEWAY_TRANSCRIBE_MODEL = "spacexai/grok-stt";
+/** Transcribes through the Vercel AI Gateway (no extra provider key needed). */
+export async function gatewayTranscribe(bytes, options = {}) {
+    const key = gatewayKey();
+    if (!key)
+        throw new Error("no transcription provider configured");
+    const gw = createGateway({ apiKey: key });
+    const r = await experimental_transcribe({
+        model: gw.transcription(process.env.WEBCRAWL_GATEWAY_TRANSCRIBE_MODEL || DEFAULT_GATEWAY_TRANSCRIBE_MODEL),
+        audio: bytes,
+        abortSignal: options.signal ?? AbortSignal.timeout(300_000),
+        ...(options.language ? { providerOptions: { openai: { language: options.language } } } : {}),
+    });
+    const segments = r.segments
+        .map((s) => ({ start: s.startSecond, end: s.endSecond, text: s.text.trim() }))
+        .filter((s) => s.text);
+    if (!segments.length && r.text.trim())
+        segments.push({ start: 0, end: r.durationInSeconds ?? 0, text: r.text.trim() });
+    return { segments, duration: r.durationInSeconds ?? segments[segments.length - 1]?.end ?? 0, language: r.language };
+}
+/** 3725.4 -> "1:02:05"; 65 -> "1:05". */
+export function formatTimestamp(seconds) {
+    const s = Math.max(0, Math.floor(seconds));
+    const h = Math.floor(s / 3600);
+    const m = Math.floor((s % 3600) / 60);
+    const sec = String(s % 60).padStart(2, "0");
+    return h > 0 ? `${h}:${String(m).padStart(2, "0")}:${sec}` : `${m}:${sec}`;
+}
+function parseClock(t) {
+    const parts = t.trim().replace(",", ".").split(":").map(Number);
+    return parts.reduce((acc, n) => acc * 60 + n, 0);
+}
+/** Parses WebVTT or SRT cues. Drops styling tags and repeated rolling-caption lines. */
+export function parseCaptions(text) {
+    const cues = [];
+    const blocks = text.replace(/\r/g, "").split(/\n{2,}/);
+    for (const block of blocks) {
+        const lines = block.split("\n");
+        const i = lines.findIndex((l) => l.includes("-->"));
+        if (i === -1)
+            continue;
+        const [a, b] = lines[i].split("-->");
+        const body = lines
+            .slice(i + 1)
+            .join(" ")
+            .replace(/<[^>]+>/g, "")
+            .replace(/\s+/g, " ")
+            .trim();
+        if (!body)
+            continue;
+        const start = parseClock(a);
+        const end = parseClock(b.trim().split(/\s/)[0]);
+        const prev = cues[cues.length - 1];
+        if (prev && prev.text === body) {
+            prev.end = end;
+            continue;
+        }
+        cues.push({ start, end, text: body });
+    }
+    return cues;
+}
+/** Groups segments into ~30 s paragraphs, each prefixed with its start time. */
+export function transcriptMarkdown(segments, paragraphSeconds = 30) {
+    const out = [];
+    let buf = [];
+    let bufStart = 0;
+    for (const seg of segments) {
+        if (buf.length === 0)
+            bufStart = seg.start;
+        buf.push(seg.text.trim());
+        if (seg.end - bufStart >= paragraphSeconds) {
+            out.push(`**[${formatTimestamp(bufStart)}]** ${buf.join(" ")}`);
+            buf = [];
+        }
+    }
+    if (buf.length)
+        out.push(`**[${formatTimestamp(bufStart)}]** ${buf.join(" ")}`);
+    return out.join("\n\n");
+}
+export async function groqTranscribe(bytes, mime, options = {}) {
+    const key = process.env.GROQ_API_KEY;
+    if (!key)
+        throw new Error("GROQ_API_KEY not set");
+    const form = new FormData();
+    const name = options.fileName?.split("/").pop() || `media.${mime.split("/")[1] || "bin"}`;
+    form.append("file", new Blob([bytes], { type: mime }), name);
+    form.append("model", process.env.WEBCRAWL_TRANSCRIBE_MODEL || DEFAULT_TRANSCRIBE_MODEL);
+    form.append("response_format", "verbose_json");
+    form.append("timestamp_granularities[]", "segment");
+    if (options.language)
+        form.append("language", options.language);
+    const res = await fetch(GROQ_URL, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${key}` },
+        body: form,
+        signal: options.signal ?? AbortSignal.timeout(300_000),
+    });
+    if (!res.ok) {
+        const detail = (await res.text().catch(() => "")).slice(0, 300);
+        throw new Error(`Groq transcription ${res.status}: ${detail}`);
+    }
+    const json = (await res.json());
+    const segments = (json.segments ?? []).map((s) => ({ start: s.start, end: s.end, text: s.text.trim() }));
+    if (segments.length === 0 && json.text?.trim()) {
+        segments.push({ start: 0, end: json.duration ?? 0, text: json.text.trim() });
+    }
+    const duration = json.duration ?? segments[segments.length - 1]?.end ?? 0;
+    return { segments, duration, language: json.language };
+}
+async function describeVideo(bytes, mime, signal) {
+    const key = gatewayKey();
+    if (!key)
+        throw new Error("no vision model configured");
+    if (bytes.byteLength > MAX_VIDEO_DESCRIBE_BYTES)
+        throw new Error("video too large to describe");
+    const gw = createGateway({ apiKey: key });
+    const { text } = await generateText({
+        model: gw(process.env.WEBCRAWL_VISION_MODEL || DEFAULT_VISION_MODEL),
+        maxOutputTokens: 1_500,
+        abortSignal: signal ?? AbortSignal.timeout(120_000),
+        // Low resolution is ~66 tokens/frame instead of ~258: plenty for a scene summary, ~4x cheaper.
+        providerOptions: { google: { mediaResolution: "MEDIA_RESOLUTION_LOW" } },
+        messages: [
+            {
+                role: "user",
+                content: [
+                    { type: "file", data: bytes, mediaType: mime },
+                    {
+                        type: "text",
+                        text: "Describe what is shown on screen in this video in 3-6 plain sentences: setting, people, on-screen text, slides or charts. Do not transcribe speech.",
+                    },
+                ],
+            },
+        ],
+    });
+    return text.trim();
+}
+export async function parseMedia(bytes, kind, mime, options = {}) {
+    const started = performance.now();
+    const warnings = [];
+    const title = options.fileName?.split("/").pop()?.replace(/\.[a-z0-9]+$/i, "") || "";
+    let segments = [];
+    let duration;
+    let provider;
+    if (options.captions?.trim()) {
+        segments = parseCaptions(options.captions);
+        if (segments.length) {
+            provider = "captions";
+            duration = segments[segments.length - 1].end;
+        }
+        else {
+            warnings.push("Captions were provided but contained no cues; transcribing instead");
+        }
+    }
+    if (!provider) {
+        if (!transcriptionAvailable()) {
+            warnings.push("Transcript unavailable: no transcription provider configured");
+        }
+        else {
+            // Groq turbo (if configured) is ~2.5x cheaper than gateway grok-stt; the gateway is billed 2 cr/min.
+            const attempts = [];
+            if (process.env.GROQ_API_KEY)
+                attempts.push({ name: "groq-whisper-turbo", run: () => groqTranscribe(bytes, mime, options) });
+            if (gatewayKey())
+                attempts.push({ name: "gateway-stt", run: () => gatewayTranscribe(bytes, options) });
+            const errors = [];
+            for (const a of attempts) {
+                try {
+                    const r = await a.run();
+                    segments = r.segments;
+                    duration = r.duration;
+                    provider = a.name;
+                    if (!segments.length)
+                        warnings.push("No speech detected");
+                    break;
+                }
+                catch (err) {
+                    errors.push(`${a.name}: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+            if (!provider)
+                warnings.push(`Transcription failed: ${errors.join("; ")}`);
+        }
+    }
+    const parts = [];
+    let visionProvider;
+    if (kind === "video" && options.describeVideo) {
+        try {
+            const desc = await describeVideo(bytes, mime, options.signal);
+            if (desc) {
+                parts.push(`## What's on screen\n\n${desc}`);
+                visionProvider = "gemini-flash-lite";
+            }
+        }
+        catch (err) {
+            warnings.push(`Video description skipped: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    if (segments.length)
+        parts.push(`## Transcript\n\n${transcriptMarkdown(segments)}`);
+    return {
+        title,
+        markdown: parts.join("\n\n"),
+        meta: {
+            kind,
+            mime,
+            method: "transcript",
+            bytes: bytes.byteLength,
+            ...(duration !== undefined ? { durationSeconds: Math.round(duration * 10) / 10 } : {}),
+            ...(provider ? { transcriptProvider: provider } : {}),
+            ...(visionProvider ? { visionProvider } : {}),
+            warnings,
+            processingMs: Math.round(performance.now() - started),
+        },
+    };
+}

package/dist/core/parse/merge.d.ts

@@ -0,0 +1,19 @@
+/** Pure helpers for the durable parse workflow: batch PDF pages, then merge results. */
+import type { ParseMeta } from "./types.js";
+export interface Chunk {
+    title: string;
+    markdown: string;
+    meta: ParseMeta;
+    cached: boolean;
+}
+/** Splits the pages to read into fixed-size batches. */
+export declare function pdfBatches(pageCount: number, selected: number[] | undefined, size: number): number[][];
+/** Merges per-batch PDF results back into one document, in page order. */
+export declare function mergePdfChunks(chunks: Chunk[], info: {
+    pageCount?: number;
+    title?: string;
+}): {
+    title: string;
+    markdown: string;
+    meta: ParseMeta;
+};