webcrawl 1.4.3 → 1.4.5

package/dist/core/extract/structured.js

@@ -1,16 +1,73 @@
 /**
  * Prompt/schema-aware structured extraction used by `extractWebsiteData`.
  * Deterministic (no LLM): quote/author cards, embedded JSON, and schema shaping.
+ *
+ * Nested objects/arrays follow a JSON-schema walk: shorthand
+ * `{items:[{quote,author}]}` is coerced to a real object/array schema,
+ * root arrays are wrapped as `{items: ...}`, and empty results are detectable.
  */
 import { decodeHtmlEntities } from "../engine/markdown.js";
-function schemaProperties(schema) {
+const CACHE_META_KEYS = new Set(["cached", "cacheLayer"]);
+function isPlainObject(v) {
+    return !!v && typeof v === "object" && !Array.isArray(v);
+}
+/** A bare property map or shorthand array becomes a JSON Schema. */
+export function coerceExtractSchema(node) {
+    if (Array.isArray(node)) {
+        const item = node.length > 0 ? coerceExtractSchema(node[0]) : { type: "object" };
+        return { type: "array", items: item };
+    }
+    if (typeof node === "string") {
+        const t = node.trim() || "string";
+        return { type: t };
+    }
+    if (!isPlainObject(node)) {
+        return { type: "string" };
+    }
+    if (typeof node.type === "string" || Array.isArray(node.type)) {
+        const out = { ...node };
+        if (isPlainObject(out.properties)) {
+            out.properties = Object.fromEntries(Object.entries(out.properties).map(([k, v]) => [k, coerceExtractSchema(v)]));
+        }
+        if (out.items !== undefined) {
+            out.items = coerceExtractSchema(out.items);
+        }
+        return out;
+    }
+    const properties = Object.fromEntries(Object.entries(node).map(([k, v]) => [k, coerceExtractSchema(v)]));
+    return {
+        type: "object",
+        properties,
+        required: Object.keys(properties),
+        additionalProperties: false,
+    };
+}
+/**
+ * Root-array wrap: a top-level array becomes
+ * `{ type: object, properties: { items: <array> } }`.
+ */
+export function normalizeExtractSchema(schema) {
     if (!schema || typeof schema !== "object")
+        return undefined;
+    let coerced = coerceExtractSchema(schema);
+    if (coerced.type === "array") {
+        coerced = {
+            type: "object",
+            properties: { items: coerced },
+            required: ["items"],
+            additionalProperties: false,
+        };
+    }
+    return coerced;
+}
+export function schemaProperties(schema) {
+    const normalized = normalizeExtractSchema(schema);
+    if (!normalized)
         return {};
-    const props = schema.properties;
-    if (props && typeof props === "object" && !Array.isArray(props)) {
+    const props = normalized.properties;
+    if (isPlainObject(props))
         return props;
-    }
-    return schema;
+    return normalized;
 }
 export function hasCustomExtractRequest(prompt, schema) {
     if (prompt && prompt.trim())
@@ -18,7 +75,58 @@ export function hasCustomExtractRequest(prompt, schema) {
     return Object.keys(schemaProperties(schema)).length > 0;
 }
 function blobForIntent(prompt, schema) {
-    return `${prompt ?? ""} ${Object.keys(schemaProperties(schema)).join(" ")}`.toLowerCase();
+    const keys = [];
+    const walk = (node) => {
+        if (Array.isArray(node)) {
+            for (const n of node)
+                walk(n);
+            return;
+        }
+        if (!isPlainObject(node))
+            return;
+        keys.push(...Object.keys(node));
+        if (isPlainObject(node.properties))
+            walk(node.properties);
+        if (node.items)
+            walk(node.items);
+    };
+    walk(schema);
+    walk(schemaProperties(schema));
+    return `${prompt ?? ""} ${keys.join(" ")}`.toLowerCase();
+}
+/** Nested object/array field names that look like quote+author records. */
+export function nestedQuoteAuthorFields(schema) {
+    const normalized = normalizeExtractSchema(schema);
+    if (!normalized)
+        return null;
+    const visit = (node, path) => {
+        if (node.type === "array" && node.items) {
+            const items = coerceExtractSchema(node.items);
+            const itemProps = isPlainObject(items.properties) ? items.properties : {};
+            const itemKeys = Object.keys(itemProps);
+            const textKey = itemKeys.find((k) => /quote|text|content|body/i.test(k));
+            const authorKey = itemKeys.find((k) => /author|by|name/i.test(k));
+            if (textKey && authorKey) {
+                return { arrayPath: path, textKey, authorKey };
+            }
+            if (items.type === "object" && isPlainObject(items.properties)) {
+                const nested = visit(items, path);
+                if (nested)
+                    return nested;
+            }
+        }
+        const props = isPlainObject(node.properties) ? node.properties : {};
+        for (const [key, value] of Object.entries(props)) {
+            if (!isPlainObject(value) && !Array.isArray(value))
+                continue;
+            const child = coerceExtractSchema(value);
+            const found = visit(child, [...path, key]);
+            if (found)
+                return found;
+        }
+        return null;
+    };
+    return visit(normalized, []);
 }
 export function wantsQuoteAuthorExtract(prompt, schema) {
     const blob = blobForIntent(prompt, schema);
@@ -27,8 +135,44 @@ export function wantsQuoteAuthorExtract(prompt, schema) {
     const props = schemaProperties(schema);
     if (props.quotes || props.quote || props.authors)
         return true;
+    if (nestedQuoteAuthorFields(schema))
+        return true;
     return false;
 }
+/** Strip script/style/template (including unclosed tags) before reading visible DOM. */
+export function stripNonContentHtml(html) {
+    return String(html ?? "")
+        .replace(/<!--[\s\S]*?-->/g, "")
+        .replace(/<(script|style|noscript|svg|canvas|template|iframe)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "")
+        .replace(/<(script|style|noscript|svg|canvas|template|iframe)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*/gi, "");
+}
+/** Inline `document.write("... " + d['text'] + ...)` leftovers, not real quotes. */
+export function isJsTemplateJunk(text) {
+    const t = String(text ?? "");
+    return (/\+\s*d\s*\[/i.test(t) ||
+        /['"]\s*\+\s*d/i.test(t) ||
+        /\bd\s*\[\s*['"]text['"]\s*\]/.test(t) ||
+        /\bd\s*\[\s*['"]author['"]\s*\]/.test(t) ||
+        /document\.write/i.test(t));
+}
+export function isEmptyExtractResult(data) {
+    if (!data || typeof data !== "object")
+        return true;
+    const keys = Object.keys(data).filter((k) => !CACHE_META_KEYS.has(k));
+    if (keys.length === 0)
+        return true;
+    return keys.every((k) => {
+        const v = data[k];
+        if (v == null || v === "")
+            return true;
+        if (Array.isArray(v))
+            return v.length === 0;
+        if (typeof v === "object" && !Array.isArray(v)) {
+            return isEmptyExtractResult(v);
+        }
+        return false;
+    });
+}
 function cleanText(raw) {
     return decodeHtmlEntities(raw.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim());
 }
@@ -37,6 +181,8 @@ function pushPair(out, text, author) {
     const a = cleanText(author);
     if (t.length < 8 || a.length < 2)
         return;
+    if (isJsTemplateJunk(t) || isJsTemplateJunk(a))
+        return;
     if (out.some((p) => p.text === t && p.author === a))
         return;
     out.push({ text: t, author: a });
@@ -44,7 +190,8 @@ function pushPair(out, text, author) {
 /** Pull quote/author pairs from HTML (cards, microdata, `var data = [...]`) and markdown. */
 export function extractQuoteAuthorPairs(html, markdown = "") {
     const out = [];
-    const dataAssign = html.match(/var\s+data\s*=\s*(\[[\s\S]*?\]);/);
+    const source = String(html ?? "");
+    const dataAssign = source.match(/var\s+data\s*=\s*(\[[\s\S]*?\]);/);
     if (dataAssign) {
         try {
             const parsed = JSON.parse(dataAssign[1]);
@@ -60,9 +207,10 @@ export function extractQuoteAuthorPairs(html, markdown = "") {
             /* not JSON */
         }
     }
+    const visible = stripNonContentHtml(source);
     const cardRe = /<(?:div|figure|blockquote|li)[^>]*class=["'][^"']*\bquote\b[^"']*["'][^>]*>([\s\S]*?)<\/(?:div|figure|blockquote|li)>/gi;
     let card;
-    while ((card = cardRe.exec(html)) !== null) {
+    while ((card = cardRe.exec(visible)) !== null) {
         const block = card[1];
         const text = block.match(/<(?:span|div|p|q)[^>]*class=["'][^"']*\btext\b[^"']*["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
             block.match(/itemprop=["']text["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
@@ -75,7 +223,7 @@ export function extractQuoteAuthorPairs(html, markdown = "") {
     }
     const microRe = /itemprop=["']text["'][^>]*>([\s\S]*?)<\/[^>]+>[\s\S]{0,400}?itemprop=["']author["'][^>]*>([\s\S]*?)<\//gi;
     let micro;
-    while ((micro = microRe.exec(html)) !== null) {
+    while ((micro = microRe.exec(visible)) !== null) {
         pushPair(out, micro[1], micro[2]);
     }
     if (out.length === 0 && markdown) {
@@ -88,6 +236,14 @@ export function extractQuoteAuthorPairs(html, markdown = "") {
     return out;
 }
 function itemFieldNames(schema) {
+    const nested = nestedQuoteAuthorFields(schema);
+    if (nested && nested.arrayPath.length > 0) {
+        return {
+            arrayKey: nested.arrayPath[nested.arrayPath.length - 1] || "items",
+            textKey: nested.textKey,
+            authorKey: nested.authorKey,
+        };
+    }
     const props = schemaProperties(schema);
     const keys = Object.keys(props);
     const arrayKey = keys.find((k) => {
@@ -96,8 +252,7 @@ function itemFieldNames(schema) {
             return /quotes?|items|results|entries/i.test(k);
         const t = p.type;
         return t === "array" || /quotes?|items|results|entries/i.test(k);
-    }) ||
-        (keys.length === 0 ? "quotes" : undefined);
+    }) || (keys.length === 0 ? "quotes" : undefined);
     const itemSchema = arrayKey && props[arrayKey] && typeof props[arrayKey] === "object"
         ? props[arrayKey]
         : undefined;
@@ -107,6 +262,22 @@ function itemFieldNames(schema) {
     const authorKey = itemKeys.find((k) => /author|by|name/i.test(k)) ?? "author";
     return { arrayKey: arrayKey ?? "quotes", textKey, authorKey };
 }
+function assignAtPath(root, path, value) {
+    if (path.length === 0) {
+        return isPlainObject(value) ? { ...root, ...value } : root;
+    }
+    let cursor = root;
+    for (let i = 0; i < path.length - 1; i++) {
+        const key = path[i];
+        const next = cursor[key];
+        if (!isPlainObject(next)) {
+            cursor[key] = {};
+        }
+        cursor = cursor[key];
+    }
+    cursor[path[path.length - 1]] = value;
+    return root;
+}
 export function shapeQuoteAuthors(pairs, schema) {
     const props = schemaProperties(schema);
     const keys = Object.keys(props);
@@ -119,10 +290,13 @@ export function shapeQuoteAuthors(pairs, schema) {
             [authorArr]: pairs.map((p) => p.author),
         };
     }
+    const nested = nestedQuoteAuthorFields(schema);
     const { arrayKey, textKey, authorKey } = itemFieldNames(schema);
-    return {
-        [arrayKey]: pairs.map((p) => ({ [textKey]: p.text, [authorKey]: p.author })),
-    };
+    const rows = pairs.map((p) => ({ [textKey]: p.text, [authorKey]: p.author }));
+    if (nested && nested.arrayPath.length > 0) {
+        return assignAtPath({}, nested.arrayPath, rows);
+    }
+    return { [arrayKey]: rows };
 }
 /**
  * Deterministic extract when the caller passed a prompt and/or JSON schema.
@@ -131,13 +305,11 @@ export function shapeQuoteAuthors(pairs, schema) {
 export function extractByPromptAndSchema(html, markdown, prompt, schema) {
     if (!hasCustomExtractRequest(prompt, schema))
         return null;
-    if (wantsQuoteAuthorExtract(prompt, schema)) {
-        const pairs = extractQuoteAuthorPairs(html, markdown);
-        if (pairs.length > 0)
-            return shapeQuoteAuthors(pairs, schema);
-    }
     const pairs = extractQuoteAuthorPairs(html, markdown);
-    if (pairs.length > 0 && /\bquotes?\b/i.test(blobForIntent(prompt, schema))) {
+    if (wantsQuoteAuthorExtract(prompt, schema) && pairs.length > 0) {
+        return shapeQuoteAuthors(pairs, schema);
+    }
+    if (pairs.length > 0 && /\b(quotes?|items|results|entries)\b/i.test(blobForIntent(prompt, schema))) {
         return shapeQuoteAuthors(pairs, schema);
     }
     return null;

package/dist/core/local-engine.d.ts

@@ -38,6 +38,12 @@ export interface LocalEngineBackend {
  * for content. Exported for unit tests.
  */
 export declare function blockedScrapeResult(url: string, statusCode: number, tier: string, latencyMs: number, reason?: string): Record<string, unknown>;
+/**
+ * JS page whose renderer failed. `success: false` so callers never treat the
+ * unrendered nav shell as content (v1.4.3 smoke test: 114 chars of nav with
+ * success:true after Chrome hung on --dump-dom).
+ */
+export declare function jsRenderFailedResult(url: string, statusCode: number, tier: string, latencyMs: number, reason: string): Record<string, unknown>;
 /** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
 export declare const MIN_CACHEABLE_MARKDOWN = 200;
 /** Attach source HTML when the caller asked for the `html` format. */

package/dist/core/local-engine.js

@@ -9,7 +9,7 @@
  * Every community improvement to extractors, parsers, or targets in `src/` immediately
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
-import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, isLocalChromeAvailable, mapWebsiteUrls, renderViaLocalChrome, setCachedPage, sha256, } from "./engine.js";
+import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, isLocalChromeAvailable, mapWebsiteUrls, renderViaLocalChromeDetailed, setCachedPage, sha256, } from "./engine.js";
 import { isDomainRenderNeeded, markDomainRenderNeeded, } from "./engine/session.js";
 import { isRenderProviderEnabled, renderViaProvider } from "./engine/render.js";
 import { fetchResource, isAntiBotChallenge, lastFetchTier, needsBrowserEscalation, } from "./engine/fetcher.js";
@@ -20,7 +20,7 @@ import { parseYouTubeCached, youtubeVideoId } from "./parse/youtube.js";
 import { rankDocuments } from "./ranker.js";
 import { getSearchCategoriesCatalog, searchWeb } from "./search.js";
 import { inspectTarget, listTargets, resolveTarget } from "./targets.js";
-import { hasCustomExtractRequest } from "./extract/structured.js";
+import { hasCustomExtractRequest, isEmptyExtractResult, } from "./extract/structured.js";
 /** Lowercased hostname of a URL (memo key for the Tier R render memo). */
 function safeDomainOf(url) {
     try {
@@ -53,6 +53,34 @@ export function blockedScrapeResult(url, statusCode, tier, latencyMs, reason) {
         evidence: { tier, blocked: true, statusCode, latencyMs, limitations: [error] },
     };
 }
+/**
+ * JS page whose renderer failed. `success: false` so callers never treat the
+ * unrendered nav shell as content (v1.4.3 smoke test: 114 chars of nav with
+ * success:true after Chrome hung on --dump-dom).
+ */
+export function jsRenderFailedResult(url, statusCode, tier, latencyMs, reason) {
+    return {
+        success: false,
+        mode: "local",
+        url,
+        blocked: false,
+        needsJs: true,
+        statusCode,
+        error: reason,
+        title: url,
+        markdown: "",
+        links: [],
+        data: { url, title: url, markdown: "", links: [] },
+        evidence: {
+            tier,
+            blocked: false,
+            needsJs: true,
+            statusCode,
+            latencyMs,
+            limitations: [reason],
+        },
+    };
+}
 /** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
 export const MIN_CACHEABLE_MARKDOWN = 200;
 /** Attach source HTML when the caller asked for the `html` format. */
@@ -82,33 +110,55 @@ function emptyPageError(statusCode) {
         "The site may be blocking automated clients or serving a client-rendered shell. " +
         "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.");
 }
+function defaultJsRenderError() {
+    return ("Page needs JavaScript rendering; local Chrome was not available or produced no content. " +
+        "Install Chrome (see `webcrawl doctor`), point WEBCRAWL_CHROME_PATH at chrome-headless-shell " +
+        "(`npx playwright install chromium-headless-shell`), run `webcrawl login` and retry with `--mode cloud`, " +
+        "or self-host a render provider via WEBCRAWL_RENDER_URL.");
+}
 /**
- * JS / challenge shell → render provider, then local Chrome dump-dom, then stealth.
- * Returns null when no renderer produced real content.
+ * JS / challenge shell → render provider, then local Chrome (CDP, then dump-dom),
+ * then stealth. Returns null when no renderer produced real content.
  */
 async function escalateJsRender(url) {
+    const errors = [];
     if (isRenderProviderEnabled()) {
         const rendered = await renderViaProvider(url);
         if (rendered && rendered.markdown.trim().length >= 40) {
             return {
-                title: rendered.title,
-                markdown: rendered.markdown,
-                html: "",
-                links: [],
-                tier: "render",
+                page: {
+                    title: rendered.title,
+                    markdown: rendered.markdown,
+                    html: "",
+                    links: [],
+                    tier: "render",
+                },
             };
         }
+        errors.push("render provider produced no content");
     }
     if (isLocalChromeAvailable()) {
-        const html = await renderViaLocalChrome(url);
-        if (html && html.length > 100) {
-            const parsed = fastHtmlToMarkdown(html, url);
+        const chrome = await renderViaLocalChromeDetailed(url);
+        if (chrome.html && chrome.html.length > 100) {
+            const parsed = fastHtmlToMarkdown(chrome.html, url);
             if (parsed.markdown.trim().length > 40 &&
-                !isAntiBotChallenge(html, 200) &&
+                !isAntiBotChallenge(chrome.html, 200) &&
                 !isChallengeMarkdown(parsed.title, parsed.markdown)) {
-                return { ...parsed, html, tier: "browser" };
+                return { page: { ...parsed, html: chrome.html, tier: "browser" } };
             }
+            errors.push(chrome.method
+                ? `${chrome.method} rendered HTML but markdown was still a shell`
+                : "local Chrome rendered HTML but markdown was still a shell");
+        }
+        else if (chrome.error) {
+            errors.push(chrome.error);
         }
+        else {
+            errors.push("local Chrome produced no content");
+        }
+    }
+    else {
+        errors.push("no local Chrome/Chromium binary");
     }
     try {
         const { stealthAvailable } = await import("./engine/stealth.js");
@@ -119,19 +169,22 @@ async function escalateJsRender(url) {
                 esc.markdown.trim().length > 40 &&
                 !isChallengeResult(esc.html, esc.markdown, esc.statusCode)) {
                 return {
-                    title: esc.title,
-                    markdown: esc.markdown,
-                    html: esc.html,
-                    links: esc.links,
-                    tier: esc.tier,
+                    page: {
+                        title: esc.title,
+                        markdown: esc.markdown,
+                        html: esc.html,
+                        links: esc.links,
+                        tier: esc.tier,
+                    },
                 };
             }
+            errors.push("stealth browser produced no content");
         }
     }
     catch {
         /* stealth optional */
     }
-    return null;
+    return { page: null, error: errors.filter(Boolean).join("; ") || defaultJsRenderError() };
 }
 /**
  * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
@@ -329,12 +382,15 @@ export function createLocalEngineClient() {
             const needsRender = needsBrowserEscalation(fetched.status, html, markdown);
             const nativeTier = nativeScrapeTier();
             //    JS shell / challenge: memoize the domain and escalate to a renderer
-            //    (hosted provider, local Chrome dump-dom, then stealth). Never cache
-            //    the unrendered shell.
+            //    (hosted provider, local Chrome CDP, then stealth). Never cache
+            //    the unrendered shell, and never report the shell as success:true.
             if (needsRender) {
                 await markDomainRenderNeeded(renderDomain);
                 // 403/429/503 is already a refusal — don't spend a Chrome launch on the block page.
-                const rendered = blocked && fetched.status >= 400 ? null : await escalateJsRender(targetUrl);
+                const escalated = blocked && fetched.status >= 400
+                    ? { page: null, error: undefined }
+                    : await escalateJsRender(targetUrl);
+                const rendered = escalated.page;
                 if (rendered) {
                     const renderHash = sha256(rendered.markdown);
                     if (rendered.markdown.trim().length >= MIN_CACHEABLE_MARKDOWN) {
@@ -369,43 +425,10 @@ export function createLocalEngineClient() {
                         },
                     }, rendered.html, p.formats);
                 }
-                if (blocked || !markdown.trim()) {
-                    return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start, blocked ? undefined : emptyPageError(fetched.status));
+                if (blocked) {
+                    return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start);
                 }
-                const targetInspection = await inspectTarget({
-                    url: targetUrl,
-                    html,
-                    markdown,
-                });
-                return withOptionalHtml({
-                    success: true,
-                    mode: "local",
-                    url: targetUrl,
-                    title: title || targetInspection.title || targetUrl,
-                    markdown: targetInspection.markdown || markdown,
-                    links,
-                    data: {
-                        url: targetUrl,
-                        title: title || targetInspection.title || targetUrl,
-                        markdown: targetInspection.markdown || markdown,
-                        links,
-                        structured: targetInspection.structured,
-                    },
-                    evidence: {
-                        ...targetInspection.evidence,
-                        tier: nativeTier,
-                        contentHash: sha256(markdown || html),
-                        latencyMs: Date.now() - start,
-                        limitations: [
-                            ...(Array.isArray(targetInspection.evidence?.limitations)
-                                ? targetInspection.evidence.limitations
-                                : []),
-                            "Page needs JS rendering; local Chrome was not available or produced no content. " +
-                                "Install Chrome (see `webcrawl doctor`), run `webcrawl login` and retry with `--mode cloud`, " +
-                                "or self-host a render provider via WEBCRAWL_RENDER_URL.",
-                        ],
-                    },
-                }, html, p.formats);
+                return jsRenderFailedResult(targetUrl, fetched.status, nativeTier, Date.now() - start, escalated.error || (!markdown.trim() ? emptyPageError(fetched.status) : defaultJsRenderError()));
             }
             if (blocked || !markdown.trim()) {
                 return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start, blocked ? undefined : emptyPageError(fetched.status));
@@ -557,22 +580,38 @@ export function createLocalEngineClient() {
                 formats: ["markdown", "html"],
                 noCache: true,
             }));
-            const markdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
-            const html = (scraped.data && scraped.data.html) || scraped.html || "";
+            let markdown = (scraped.data && scraped.data.markdown) || scraped.markdown || "";
+            let html = (scraped.data && scraped.data.html) || scraped.html || "";
             if (scraped.blocked || scraped.success === false) {
-                return {
-                    success: false,
-                    blocked: true,
-                    mode: "local",
-                    url: targetUrl,
-                    data: {},
-                    error: scraped.error || emptyPageError(200),
-                    evidence: {
-                        tier: lastFetchTier(),
-                        sourceUrl: targetUrl,
-                        limitations: [String(scraped.error || emptyPageError(200))],
-                    },
-                };
+                // Bot-blocked pages stay failures. A JS shell whose Chrome render failed
+                // can still yield structured data from JSON islands once script tags are
+                // stripped for card matching — never from `document.write` templates.
+                if (scraped.needsJs && scraped.blocked !== true) {
+                    try {
+                        const fetched = await fetchWithTlsImpersonation(targetUrl);
+                        html = fetched.html || html;
+                        const parsedShell = fastHtmlToMarkdown(html, targetUrl);
+                        markdown = parsedShell.markdown || markdown;
+                    }
+                    catch {
+                        /* keep whatever scrape returned */
+                    }
+                }
+                else {
+                    return {
+                        success: false,
+                        blocked: true,
+                        mode: "local",
+                        url: targetUrl,
+                        data: {},
+                        error: scraped.error || emptyPageError(200),
+                        evidence: {
+                            tier: lastFetchTier(),
+                            sourceUrl: targetUrl,
+                            limitations: [String(scraped.error || emptyPageError(200))],
+                        },
+                    };
+                }
             }
             const targetInspection = await inspectTarget({
                 url: targetUrl,
@@ -587,15 +626,35 @@ export function createLocalEngineClient() {
                 bypassCache: true,
             });
             const { cached: _c, cacheLayer: _l, ...extracted } = siteData;
+            const data = extractOutputData(extracted, {
+                prompt: p.prompt,
+                schema: p.schema,
+                target: targetInspection.structured,
+            });
+            if (hasCustomExtractRequest(p.prompt, p.schema) && isEmptyExtractResult(data)) {
+                const error = "Extraction produced no data matching the prompt/schema. " +
+                    "If this is a JS-rendered page, install Chrome (see `webcrawl doctor`) and retry; " +
+                    "empty objects are not reported as success.";
+                return {
+                    success: false,
+                    mode: "local",
+                    url: targetUrl,
+                    data: {},
+                    error,
+                    evidence: {
+                        ...targetInspection.evidence,
+                        limitations: [
+                            ...(targetInspection.evidence?.limitations ?? []),
+                            error,
+                        ],
+                    },
+                };
+            }
             return {
                 success: true,
                 mode: "local",
                 url: targetUrl,
-                data: extractOutputData(extracted, {
-                    prompt: p.prompt,
-                    schema: p.schema,
-                    target: targetInspection.structured,
-                }),
+                data,
                 evidence: targetInspection.evidence,
             };
         },

package/dist/core/search/catalog.d.ts

@@ -52,7 +52,7 @@ export interface SearchRequest {
     bypassCache?: boolean;
     /**
      * Pick one search provider and skip the default multi-engine pipeline.
-     * `auto` (default) uses the free engines and falls back to Jina when results are thin.
+     * `auto` (default) uses the free engines and falls back to a keyed provider when results are thin.
      */
     provider?: SearchProviderChoice;
     rankOptions?: {

package/dist/core/search/catalog.js

@@ -11,7 +11,7 @@ export const SEARCH_PROVIDER_ENV_KEYS = {
 export const SEARCH_CATEGORY_CATALOG = {
     general: {
         description: "Broad web search across general-purpose engines.",
-        engines: ["brave-native", "ddg-lite", "wikipedia"],
+        engines: ["brave-native", "ddg-lite", "wikipedia", "hn-algolia", "bing-rss"],
         localizeBy: "none",
         preferLocalDomains: false,
         subcategories: {