webcrawl 1.4.2 → 1.4.3

package/dist/cli.js

@@ -429,8 +429,12 @@ export async function main(argv, io, deps = {}) {
                     city: v.city,
                     scrapeContent: v.content,
                 });
+                if (res.success === false && !v.json) {
+                    io.err(`${formatSearch(res)}\n`);
+                    return 1;
+                }
                 emit(res, formatSearch(res));
-                return 0;
+                return res.success === false ? 1 : 0;
             }
             case "scrape": {
                 const res = await client.scrape({ url: need("url"), formats: ["markdown"], noCache: v["no-cache"] });

package/dist/core/engine/chrome.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Path to a real installed Chrome/Chromium, or null.
+ * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
+ * defaults (same locations `webcrawl doctor` checks).
+ */
+export declare function installedChromePath(): string | null;
+export declare function isLocalChromeAvailable(): boolean;
+/**
+ * Render `url` in local Chrome and return the hydrated HTML, or null when
+ * Chrome is missing, times out, or produces nothing.
+ */
+export declare function renderViaLocalChrome(url: string, opts?: {
+    timeoutMs?: number;
+}): Promise<string | null>;

package/dist/core/engine/chrome.js

@@ -0,0 +1,140 @@
+/**
+ * Local Chrome dump-dom renderer (JS pages the native fetch ladder cannot hydrate).
+ *
+ * Uses a user-installed Chrome/Chromium binary — the same one `webcrawl doctor`
+ * reports — with `--dump-dom` + virtual time so `document.write` / client-rendered
+ * content appears in the HTML. Playwright is not required; this is the zero-key
+ * local JS tier.
+ */
+import { spawn } from "node:child_process";
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+const DUMP_MAX_BYTES = 2 * 1024 * 1024;
+const DEFAULT_TIMEOUT_MS = 20_000;
+const VIRTUAL_TIME_MS = 10_000;
+/**
+ * Path to a real installed Chrome/Chromium, or null.
+ * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
+ * defaults (same locations `webcrawl doctor` checks).
+ */
+export function installedChromePath() {
+    const fromEnv = process.env.WEBCRAWL_CHROME_PATH?.trim() ||
+        process.env.INTERACT_CHROME_PATH?.trim() ||
+        process.env.CHROME_PATH?.trim();
+    if (fromEnv)
+        return fromEnv;
+    const candidates = [];
+    if (process.platform === "darwin") {
+        candidates.push("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", `${process.env.HOME || ""}/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`);
+    }
+    else if (process.platform === "win32") {
+        candidates.push(`${process.env["ProgramFiles"] || "C:\\Program Files"}\\Google\\Chrome\\Application\\chrome.exe`, `${process.env["ProgramFiles(x86)"] || "C:\\Program Files (x86)"}\\Google\\Chrome\\Application\\chrome.exe`);
+    }
+    else {
+        candidates.push("/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium", "/snap/bin/chromium");
+    }
+    for (const p of candidates) {
+        try {
+            if (p && existsSync(p))
+                return p;
+        }
+        catch {
+            /* not installed here */
+        }
+    }
+    return null;
+}
+export function isLocalChromeAvailable() {
+    return installedChromePath() !== null;
+}
+/**
+ * Render `url` in local Chrome and return the hydrated HTML, or null when
+ * Chrome is missing, times out, or produces nothing.
+ */
+export async function renderViaLocalChrome(url, opts = {}) {
+    const chrome = installedChromePath();
+    if (!chrome)
+        return null;
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    let profile = "";
+    try {
+        profile = mkdtempSync(join(tmpdir(), "wc-chrome-"));
+    }
+    catch {
+        return null;
+    }
+    const args = [
+        "--headless=new",
+        "--no-sandbox",
+        "--disable-gpu",
+        "--disable-dev-shm-usage",
+        "--no-first-run",
+        "--no-default-browser-check",
+        "--disable-crash-reporter",
+        "--disable-breakpad",
+        `--user-data-dir=${profile}`,
+        `--virtual-time-budget=${VIRTUAL_TIME_MS}`,
+        `--timeout=${timeoutMs}`,
+        "--dump-dom",
+        url,
+    ];
+    try {
+        return await new Promise((resolve) => {
+            let settled = false;
+            const done = (value) => {
+                if (settled)
+                    return;
+                settled = true;
+                resolve(value);
+            };
+            let child;
+            try {
+                child = spawn(chrome, args, { stdio: ["ignore", "pipe", "pipe"] });
+            }
+            catch {
+                done(null);
+                return;
+            }
+            const chunks = [];
+            let bytes = 0;
+            const timer = setTimeout(() => {
+                try {
+                    child.kill("SIGKILL");
+                }
+                catch {
+                    /* already gone */
+                }
+                done(null);
+            }, timeoutMs + 2_000);
+            child.stdout?.on("data", (c) => {
+                if (bytes >= DUMP_MAX_BYTES)
+                    return;
+                const take = c.subarray(0, DUMP_MAX_BYTES - bytes);
+                chunks.push(take);
+                bytes += take.length;
+            });
+            child.on("error", () => {
+                clearTimeout(timer);
+                done(null);
+            });
+            child.on("close", () => {
+                clearTimeout(timer);
+                if (bytes === 0) {
+                    done(null);
+                    return;
+                }
+                const text = Buffer.concat(chunks).toString("utf8");
+                done(text.trim().length > 0 ? text : null);
+            });
+        });
+    }
+    finally {
+        try {
+            rmSync(profile, { recursive: true, force: true });
+        }
+        catch {
+            /* leftover profile is in /tmp */
+        }
+    }
+}

package/dist/core/engine/crawler.d.ts

@@ -66,8 +66,9 @@ export declare function checkUrlsStatusBatch(urls: string[], timeoutMs?: number)
     ok: boolean;
 }>>;
 /**
- * Extracts structured `{ app_name, product_features }` (or custom JSON schema) from a URL.
- * Replaces `firecrawl.asyncExtract` in `website-extract.ts`.
+ * Extracts structured JSON from a URL.
+ * Honours `prompt` / `schema` when provided; otherwise falls back to
+ * `{ app_name, product_features }` from title and headings.
  */
 export declare function extractWebsiteData(url: string, options?: {
     prompt?: string;
@@ -76,9 +77,9 @@ export declare function extractWebsiteData(url: string, options?: {
     bypassCache?: boolean;
     /** Already-extracted content (e.g. a parsed PDF); skips the HTML fetch and cache. */
     markdown?: string;
-}): Promise<{
-    app_name: string;
-    product_features: string[];
+    /** Source HTML when the caller already fetched the page (quote cards, JSON-LD). */
+    html?: string;
+}): Promise<Record<string, unknown> & {
     cached?: boolean;
     cacheLayer?: "memory" | "disk" | "postgres" | "miss";
 }>;

package/dist/core/engine/crawler.js

@@ -5,6 +5,7 @@ import { escalateToSandboxOrBrowser } from "../sandbox.js";
 import { getCachedMapAsync, getCachedPageAsync, setCachedMap, setCachedPage, sha256, } from "./cache.js";
 import { fetchWithTlsImpersonation, getBrowserHeaders, isAntiBotChallenge, } from "./fetcher.js";
 import { decodeHtmlEntities, extractLinksFromHtml, fastHtmlToMarkdown, } from "./markdown.js";
+import { extractByPromptAndSchema, hasCustomExtractRequest, } from "../extract/structured.js";
 export async function mapWebsiteUrls(targetUrl, options = {}) {
     const start = performance.now();
     const limit = Math.max(1, options.limit ?? 5000);
@@ -490,11 +491,13 @@ export async function checkUrlsStatusBatch(urls, timeoutMs = 8000) {
     }));
 }
 /**
- * Extracts structured `{ app_name, product_features }` (or custom JSON schema) from a URL.
- * Replaces `firecrawl.asyncExtract` in `website-extract.ts`.
+ * Extracts structured JSON from a URL.
+ * Honours `prompt` / `schema` when provided; otherwise falls back to
+ * `{ app_name, product_features }` from title and headings.
  */
 export async function extractWebsiteData(url, options = {}) {
-    if (!options.bypassCache && !options.markdown) {
+    const custom = hasCustomExtractRequest(options.prompt, options.schema);
+    if (!custom && !options.bypassCache && !options.markdown) {
         const cachedPage = await getCachedPageAsync(url, "all");
         if (cachedPage?.structuredData &&
             typeof cachedPage.structuredData.app_name === "string" &&
@@ -509,6 +512,7 @@ export async function extractWebsiteData(url, options = {}) {
     }
     let title = "";
     let markdown = "";
+    let html = options.html ?? "";
     let links = [];
     const existingPage = !options.bypassCache && !options.markdown
         ? await getCachedPageAsync(url, "all")
@@ -532,16 +536,25 @@ export async function extractWebsiteData(url, options = {}) {
             headers: { location: r.headers.get("location") ?? "" },
             text: () => r.text(),
         })));
-        const html = await res.text();
+        html = await res.text();
         const parsed = fastHtmlToMarkdown(html, url);
         title = parsed.title;
         markdown = parsed.markdown;
         links = parsed.links;
     }
+    const cacheMeta = () => ({
+        cached: Boolean(existingPage),
+        cacheLayer: existingPage?.cacheLayer || "miss",
+    });
     // If OPENAI_API_KEY is available, use gpt-4o-mini for structured JSON extraction
     const apiKey = options.openaiApiKey || process.env.OPENAI_API_KEY;
     if (apiKey) {
         try {
+            const schemaHint = options.schema
+                ? `Return a JSON object matching this JSON Schema:\n${JSON.stringify(options.schema)}`
+                : options.prompt
+                    ? `Return JSON that satisfies this request: ${options.prompt}`
+                    : 'Extract the application/website name and a list of key product features as JSON: {"app_name": string, "product_features": string[]}.';
             const llmRes = await fetch("https://api.openai.com/v1/chat/completions", {
                 method: "POST",
                 headers: {
@@ -554,11 +567,11 @@ export async function extractWebsiteData(url, options = {}) {
                     messages: [
                         {
                             role: "system",
-                            content: 'Extract the application/website name and a list of key product features as JSON: {"app_name": string, "product_features": string[]}.',
+                            content: schemaHint,
                         },
                         {
                             role: "user",
-                            content: `URL: ${url}\nTitle: ${title}\n\nContent:\n${markdown.slice(0, 12000)}`,
+                            content: `${options.prompt ? `${options.prompt}\n\n` : ""}URL: ${url}\nTitle: ${title}\n\nContent:\n${markdown.slice(0, 12000)}`,
                         },
                     ],
                     temperature: 0,
@@ -570,11 +583,16 @@ export async function extractWebsiteData(url, options = {}) {
                 const content = json.choices?.[0]?.message?.content;
                 if (content) {
                     const parsed = JSON.parse(content);
-                    if (parsed.app_name && Array.isArray(parsed.product_features)) {
-                        const structuredResult = {
-                            app_name: String(parsed.app_name),
-                            product_features: parsed.product_features.map(String),
-                        };
+                    const usable = custom
+                        ? parsed && typeof parsed === "object" && !Array.isArray(parsed)
+                        : Boolean(parsed.app_name && Array.isArray(parsed.product_features));
+                    if (usable) {
+                        const structuredResult = custom
+                            ? parsed
+                            : {
+                                app_name: String(parsed.app_name),
+                                product_features: parsed.product_features.map(String),
+                            };
                         setCachedPage(url, "all", {
                             title,
                             markdown,
@@ -583,11 +601,7 @@ export async function extractWebsiteData(url, options = {}) {
                             statusCode: 200,
                             contentHash: sha256(markdown),
                         });
-                        return {
-                            ...structuredResult,
-                            cached: Boolean(existingPage),
-                            cacheLayer: existingPage?.cacheLayer || "miss",
-                        };
+                        return { ...structuredResult, ...cacheMeta() };
                     }
                 }
             }
@@ -596,6 +610,23 @@ export async function extractWebsiteData(url, options = {}) {
             // Fall through to deterministic heuristic extraction
         }
     }
+    if (custom) {
+        const shaped = extractByPromptAndSchema(html, markdown, options.prompt, options.schema);
+        if (shaped) {
+            setCachedPage(url, "all", {
+                title,
+                markdown,
+                links,
+                structuredData: shaped,
+                statusCode: 200,
+                contentHash: sha256(markdown),
+            });
+            return { ...shaped, ...cacheMeta() };
+        }
+        // Honour the caller's schema/prompt: never substitute the generic
+        // `{ app_name, product_features }` object when they asked for something else.
+        return { ...cacheMeta() };
+    }
     // Deterministic fast extraction from Title, Meta Description, and H2/H3/List headings (<5ms)
     const cleanAppName = title.split(/\s+[-|\u2014–:]\s+/)[0]?.trim() ||
         new URL(url).hostname.replace(/^www\./, "").split(".")[0];
@@ -640,8 +671,7 @@ export async function extractWebsiteData(url, options = {}) {
     });
     return {
         ...structuredResult,
-        cached: Boolean(existingPage),
-        cacheLayer: existingPage?.cacheLayer || "miss",
+        ...cacheMeta(),
     };
 }
 /**
@@ -694,6 +724,9 @@ export async function crawlWebsitePages(targetUrl, options = {}) {
 }
 /** True for markdown converted from a bot-check page (old cache entries). */
 export function isChallengeMarkdown(title, markdown) {
+    if (/blocked by network security/i.test(`${title}\n${markdown}`)) {
+        return true;
+    }
     const t = `${title}\n${markdown.slice(0, 600)}`.toLowerCase();
     return (t.includes("vercel security checkpoint") ||
         t.includes("we're verifying your browser") ||

package/dist/core/engine/fetcher.js

@@ -95,7 +95,13 @@ export function isAntiBotChallenge(html, status) {
     if (status === 403 || status === 429 || status === 503) {
         return true;
     }
-    const lower = html.slice(0, 6000).toLowerCase();
+    const raw = String(html ?? "");
+    // Reddit (and similar) hide this copy after tens of KB of CSS + a data-URI PNG.
+    if (/blocked by network security/i.test(raw)) {
+        return true;
+    }
+    // Head slice catches title/meta markers on ordinary challenge interstitials.
+    const lower = raw.slice(0, 8000).toLowerCase();
     if (lower.includes("cf-browser-verification") ||
         lower.includes("challenge-platform") ||
         lower.includes("just a moment...") ||
@@ -457,18 +463,31 @@ export function needsBrowserEscalation(status, html, markdown) {
     if (isAntiBotChallenge(html, status)) {
         return true;
     }
-    const lower = html.slice(0, 6000).toLowerCase();
+    const text = markdown.trim();
+    const lower = html.slice(0, 8000).toLowerCase();
+    // Empty or near-empty body: nothing to return until a renderer hydrates it.
+    if (text.length === 0) {
+        return true;
+    }
     // Empty SPA shell check (<div id="root"></div> or <div id="app"></div> with almost no text)
-    if (markdown.length < 120 &&
+    if (text.length < 200 &&
         (lower.includes('id="root"') ||
             lower.includes('id="app"') ||
-            lower.includes('id="__next"'))) {
+            lower.includes('id="__next"') ||
+            lower.includes('id="__nuxt"'))) {
+        return true;
+    }
+    // Header-only JS pages (quotes.toscrape.com/js): real content lives in a
+    // <script> as JSON + document.write / innerHTML, so markdown is just the chrome.
+    if (text.length < 200 &&
+        /<script[\s>]/i.test(html) &&
+        /document\.write|innerHTML\s*=/i.test(html)) {
         return true;
     }
     // Big-HTML/no-content SPA shell: a very large page (JS bundles / inline
     // state, e.g. StubHub category pages ~3MB) that yields almost no
     // extractable text is serving its real content client-side.
-    if (html.length > 250_000 && markdown.length < 500) {
+    if (html.length > 250_000 && text.length < 500) {
         return true;
     }
     return false;

package/dist/core/engine/index.d.ts

@@ -3,4 +3,5 @@ export * from "./markdown.js";
 export * from "./session.js";
 export * from "./fetcher.js";
 export * from "./render.js";
+export * from "./chrome.js";
 export * from "./crawler.js";

package/dist/core/engine/index.js

@@ -3,4 +3,5 @@ export * from "./markdown.js";
 export * from "./session.js";
 export * from "./fetcher.js";
 export * from "./render.js";
+export * from "./chrome.js";
 export * from "./crawler.js";

package/dist/core/extract/index.d.ts

@@ -6,3 +6,4 @@ export { dedupeHotelListings, extractHotelListingFromSerpResult, extractHotelLis
 export { dedupeBusinessListings, extractBusinessListingFromSerpResult, extractBusinessListingsFromHtml, type BusinessListing, } from "./business.js";
 export { cvss3BaseScore, cveSeverityLabel, extractCveIdsFromText, extractGhsaIdsFromText, looksLikeVulnId, mergeCveRecords, normalizeCveId, nvdUrl, parseNvdCve, parseNvdResponse, parseOsvVuln, severityFromScore, sortCveRecords, type CveAffected, type CveRecord, type CveSeverity, } from "./cve.js";
 export { extractBusinessListingsWithScaleDown, extractFlightOffersWithScaleDown, extractHotelListingsWithScaleDown, extractPropertyListingsWithScaleDown, } from "./scaledown.js";
+export { extractByPromptAndSchema, extractQuoteAuthorPairs, hasCustomExtractRequest, shapeQuoteAuthors, wantsQuoteAuthorExtract, type QuoteAuthor, } from "./structured.js";

package/dist/core/extract/index.js

@@ -6,3 +6,4 @@ export { dedupeHotelListings, extractHotelListingFromSerpResult, extractHotelLis
 export { dedupeBusinessListings, extractBusinessListingFromSerpResult, extractBusinessListingsFromHtml, } from "./business.js";
 export { cvss3BaseScore, cveSeverityLabel, extractCveIdsFromText, extractGhsaIdsFromText, looksLikeVulnId, mergeCveRecords, normalizeCveId, nvdUrl, parseNvdCve, parseNvdResponse, parseOsvVuln, severityFromScore, sortCveRecords, } from "./cve.js";
 export { extractBusinessListingsWithScaleDown, extractFlightOffersWithScaleDown, extractHotelListingsWithScaleDown, extractPropertyListingsWithScaleDown, } from "./scaledown.js";
+export { extractByPromptAndSchema, extractQuoteAuthorPairs, hasCustomExtractRequest, shapeQuoteAuthors, wantsQuoteAuthorExtract, } from "./structured.js";

package/dist/core/extract/structured.d.ts

@@ -0,0 +1,14 @@
+export interface QuoteAuthor {
+    text: string;
+    author: string;
+}
+export declare function hasCustomExtractRequest(prompt?: string, schema?: Record<string, unknown>): boolean;
+export declare function wantsQuoteAuthorExtract(prompt?: string, schema?: Record<string, unknown>): boolean;
+/** Pull quote/author pairs from HTML (cards, microdata, `var data = [...]`) and markdown. */
+export declare function extractQuoteAuthorPairs(html: string, markdown?: string): QuoteAuthor[];
+export declare function shapeQuoteAuthors(pairs: QuoteAuthor[], schema?: Record<string, unknown>): Record<string, unknown>;
+/**
+ * Deterministic extract when the caller passed a prompt and/or JSON schema.
+ * Returns null when this page/request has nothing this extractor can fill.
+ */
+export declare function extractByPromptAndSchema(html: string, markdown: string, prompt?: string, schema?: Record<string, unknown>): Record<string, unknown> | null;

package/dist/core/extract/structured.js

@@ -0,0 +1,144 @@
+/**
+ * Prompt/schema-aware structured extraction used by `extractWebsiteData`.
+ * Deterministic (no LLM): quote/author cards, embedded JSON, and schema shaping.
+ */
+import { decodeHtmlEntities } from "../engine/markdown.js";
+function schemaProperties(schema) {
+    if (!schema || typeof schema !== "object")
+        return {};
+    const props = schema.properties;
+    if (props && typeof props === "object" && !Array.isArray(props)) {
+        return props;
+    }
+    return schema;
+}
+export function hasCustomExtractRequest(prompt, schema) {
+    if (prompt && prompt.trim())
+        return true;
+    return Object.keys(schemaProperties(schema)).length > 0;
+}
+function blobForIntent(prompt, schema) {
+    return `${prompt ?? ""} ${Object.keys(schemaProperties(schema)).join(" ")}`.toLowerCase();
+}
+export function wantsQuoteAuthorExtract(prompt, schema) {
+    const blob = blobForIntent(prompt, schema);
+    if (/\bquotes?\b/.test(blob) && /\bauthors?\b/.test(blob))
+        return true;
+    const props = schemaProperties(schema);
+    if (props.quotes || props.quote || props.authors)
+        return true;
+    return false;
+}
+function cleanText(raw) {
+    return decodeHtmlEntities(raw.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim());
+}
+function pushPair(out, text, author) {
+    const t = cleanText(text).replace(/^["“]|["”]$/g, "").trim();
+    const a = cleanText(author);
+    if (t.length < 8 || a.length < 2)
+        return;
+    if (out.some((p) => p.text === t && p.author === a))
+        return;
+    out.push({ text: t, author: a });
+}
+/** Pull quote/author pairs from HTML (cards, microdata, `var data = [...]`) and markdown. */
+export function extractQuoteAuthorPairs(html, markdown = "") {
+    const out = [];
+    const dataAssign = html.match(/var\s+data\s*=\s*(\[[\s\S]*?\]);/);
+    if (dataAssign) {
+        try {
+            const parsed = JSON.parse(dataAssign[1]);
+            if (Array.isArray(parsed)) {
+                for (const row of parsed) {
+                    const author = typeof row.author === "string" ? row.author : row.author?.name ?? "";
+                    if (row.text && author)
+                        pushPair(out, row.text, author);
+                }
+            }
+        }
+        catch {
+            /* not JSON */
+        }
+    }
+    const cardRe = /<(?:div|figure|blockquote|li)[^>]*class=["'][^"']*\bquote\b[^"']*["'][^>]*>([\s\S]*?)<\/(?:div|figure|blockquote|li)>/gi;
+    let card;
+    while ((card = cardRe.exec(html)) !== null) {
+        const block = card[1];
+        const text = block.match(/<(?:span|div|p|q)[^>]*class=["'][^"']*\btext\b[^"']*["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
+            block.match(/itemprop=["']text["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
+            "";
+        const author = block.match(/<(?:small|span|div|p)[^>]*class=["'][^"']*\bauthor\b[^"']*["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
+            block.match(/itemprop=["']author["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
+            "";
+        if (text && author)
+            pushPair(out, text, author);
+    }
+    const microRe = /itemprop=["']text["'][^>]*>([\s\S]*?)<\/[^>]+>[\s\S]{0,400}?itemprop=["']author["'][^>]*>([\s\S]*?)<\//gi;
+    let micro;
+    while ((micro = microRe.exec(html)) !== null) {
+        pushPair(out, micro[1], micro[2]);
+    }
+    if (out.length === 0 && markdown) {
+        const mdRe = /[“"]([^”"]{12,400})[”"]\s*(?:\n+)?\s*(?:by\s+|\u2014\s*|\s+[-–—]\s+)([A-Z][^\n]{1,80})/g;
+        let md;
+        while ((md = mdRe.exec(markdown)) !== null) {
+            pushPair(out, md[1], md[2]);
+        }
+    }
+    return out;
+}
+function itemFieldNames(schema) {
+    const props = schemaProperties(schema);
+    const keys = Object.keys(props);
+    const arrayKey = keys.find((k) => {
+        const p = props[k];
+        if (!p || typeof p !== "object")
+            return /quotes?|items|results|entries/i.test(k);
+        const t = p.type;
+        return t === "array" || /quotes?|items|results|entries/i.test(k);
+    }) ||
+        (keys.length === 0 ? "quotes" : undefined);
+    const itemSchema = arrayKey && props[arrayKey] && typeof props[arrayKey] === "object"
+        ? props[arrayKey]
+        : undefined;
+    const itemProps = itemSchema?.items?.properties ?? {};
+    const itemKeys = Object.keys(itemProps);
+    const textKey = itemKeys.find((k) => /quote|text|content|body/i.test(k)) ?? "quote";
+    const authorKey = itemKeys.find((k) => /author|by|name/i.test(k)) ?? "author";
+    return { arrayKey: arrayKey ?? "quotes", textKey, authorKey };
+}
+export function shapeQuoteAuthors(pairs, schema) {
+    const props = schemaProperties(schema);
+    const keys = Object.keys(props);
+    const quoteArr = keys.find((k) => /quotes?/i.test(k) && props[k]?.type === "array");
+    const authorArr = keys.find((k) => /authors?/i.test(k) && props[k]?.type === "array");
+    const quoteArrItems = quoteArr && props[quoteArr]?.items?.type;
+    if (quoteArr && authorArr && quoteArrItems === "string") {
+        return {
+            [quoteArr]: pairs.map((p) => p.text),
+            [authorArr]: pairs.map((p) => p.author),
+        };
+    }
+    const { arrayKey, textKey, authorKey } = itemFieldNames(schema);
+    return {
+        [arrayKey]: pairs.map((p) => ({ [textKey]: p.text, [authorKey]: p.author })),
+    };
+}
+/**
+ * Deterministic extract when the caller passed a prompt and/or JSON schema.
+ * Returns null when this page/request has nothing this extractor can fill.
+ */
+export function extractByPromptAndSchema(html, markdown, prompt, schema) {
+    if (!hasCustomExtractRequest(prompt, schema))
+        return null;
+    if (wantsQuoteAuthorExtract(prompt, schema)) {
+        const pairs = extractQuoteAuthorPairs(html, markdown);
+        if (pairs.length > 0)
+            return shapeQuoteAuthors(pairs, schema);
+    }
+    const pairs = extractQuoteAuthorPairs(html, markdown);
+    if (pairs.length > 0 && /\bquotes?\b/i.test(blobForIntent(prompt, schema))) {
+        return shapeQuoteAuthors(pairs, schema);
+    }
+    return null;
+}

package/dist/core/local-engine.d.ts

@@ -37,7 +37,17 @@ export interface LocalEngineBackend {
  * `success: false` with no markdown, so callers never mistake the block page
  * for content. Exported for unit tests.
  */
-export declare function blockedScrapeResult(url: string, statusCode: number, tier: string, latencyMs: number): Record<string, unknown>;
+export declare function blockedScrapeResult(url: string, statusCode: number, tier: string, latencyMs: number, reason?: string): Record<string, unknown>;
+/** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
+export declare const MIN_CACHEABLE_MARKDOWN = 200;
+/** Attach source HTML when the caller asked for the `html` format. */
+export declare function withOptionalHtml(result: Record<string, unknown>, html: string | undefined, formats?: Array<"markdown" | "links" | "html">): Record<string, unknown>;
+/** Structured extract payload: custom prompt/schema → only those fields, no `target` sidecar. */
+export declare function extractOutputData(extracted: Record<string, unknown>, opts: {
+    prompt?: string;
+    schema?: Record<string, unknown>;
+    target?: unknown;
+}): Record<string, unknown>;
 /**
  * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
  * Matching is a case-insensitive substring of the URL. Exported for unit tests.