webcrawl 1.4.2 → 1.4.4

package/dist/core/search/providers.d.ts

@@ -14,8 +14,30 @@ export declare function isEntityQuery(q: string): boolean;
  * preventing any single slow engine from inflating tail latency.
  */
 export declare function fetchHedgedNativeFederation(rawQuery: string, localizedQuery: string, perEngineLimit: number, req: SearchRequest, userLocation: UserLocationContext, quorumTimeoutMs: number, targetSites: string[], onOffer: (offer: ProductOffer) => void, onCve?: (cve: CveRecord) => void): Promise<RawCandidateDocument[]>;
-/** Below this many unique URLs from the free engines, the Jina fallback runs. */
+/** Below this many unique URLs from the free engines, the keyed search fallback runs. */
 export declare const JINA_FALLBACK_MIN_URLS = 3;
+/** True when a scraped SERP is a bot-check / captcha rather than results. */
+export declare function isSerpChallengeHtml(html: string, status?: number): boolean;
+export declare const EMPTY_SEARCH_ERROR: string;
+/** Canonical URL for search dedupe (host lowercased, trailing slash / tracking params stripped). */
+export declare function canonicalSearchUrl(raw: string | undefined): string;
+/** Round-robin merge so one source (e.g. Wikipedia) cannot fill `-n` by itself. */
+export declare function mergeSearchCandidates(groups: RawCandidateDocument[][], limit: number): RawCandidateDocument[];
+export declare function withRetry<T>(fn: () => Promise<T>, opts?: {
+    retries?: number;
+    delayMs?: number;
+    isEmpty?: (v: T) => boolean;
+}): Promise<T>;
+/** Parse Bing's `format=rss` web-search feed into candidate docs. */
+export declare function parseBingWebRss(xml: string, limit: number): RawCandidateDocument[];
+/**
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty
+ * or thinner than `limit`. Wikipedia OpenSearch, HN Algolia, Bing RSS, and
+ * DuckDuckGo Instant Answer do not require a JS challenge or API key.
+ */
+export declare function fetchKeylessSearchFallbacks(query: string, limit: number): Promise<RawCandidateDocument[]>;
+/** DuckDuckGo Instant Answer API — keyless JSON, unrelated to the HTML SERP challenge. */
+export declare function fetchDuckDuckGoInstantAnswer(query: string, limit: number): Promise<RawCandidateDocument[]>;
 export declare function fetchJinaSearch(query: string, limit: number, req?: {
     country?: string;
     lang?: string;

package/dist/core/search/providers.js

@@ -89,11 +89,15 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         },
         wikipedia: {
             primary: false,
-            run: () => fetchWikipediaOpenSearch(localizedQuery, 2),
+            run: () => fetchWikipediaOpenSearch(localizedQuery, Math.min(perEngineLimit, 5)),
         },
         "hn-algolia": {
             primary: false,
-            run: () => fetchHackerNewsAlgolia(localizedQuery, 5),
+            run: () => fetchHackerNewsAlgolia(localizedQuery, perEngineLimit),
+        },
+        "bing-rss": {
+            primary: false,
+            run: () => fetchBingWebRss(localizedQuery, perEngineLimit),
         },
         arxiv: {
             primary: category === "academic",
@@ -159,14 +163,17 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
                 resolve();
                 return;
             }
-            // Early-exit quorum: at least 1 primary web engine + 2 total engines + enough candidates (and shopping engine finished when in shopping plan)
+            const uniqueCount = new Set(collected.map((c) => canonicalSearchUrl(c.url)).filter(Boolean)).size;
+            const fillTarget = Math.max(1, req.limit ?? 5);
+            // Early-exit quorum: at least 1 primary web engine + 2 total engines + enough
+            // *unique* URLs to fill `limit` (not just Wikipedia's 2 hits).
             if (completedShoppingEngine &&
                 completedNewsEngine &&
                 completedNvdEngine &&
                 completedArxivEngine &&
                 completedPrimaryEngines >= 1 &&
                 totalCompletedEngines >= 2 &&
-                collected.length >= perEngineLimit &&
+                uniqueCount >= fillTarget &&
                 !graceTimer) {
                 graceTimer = setTimeout(() => {
                     if (!settled) {
@@ -209,7 +216,7 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         }
     });
     // Fallback: the free engines are scraped and can be blocked or rate limited. When they
-    // come back thin, fill in from Jina (paid) so the caller still gets an answer.
+    // come back thin, fill in from a keyed search provider so the caller still gets an answer.
     const uniqueUrls = new Set(collected.map((c) => c.url).filter(Boolean));
     if (uniqueUrls.size < JINA_FALLBACK_MIN_URLS && process.env.JINA_API_KEY) {
         const t0 = performance.now();
@@ -219,10 +226,220 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         }
         collected.push(...jina);
     }
+    // HTML SERPs (DDG, Brave) are frequently challenge/429 pages. When they
+    // yield nothing *or come back thinner than `limit`*, fill from keyless APIs
+    // (Wikipedia, HN Algolia, Bing RSS, DDG Instant Answer) with retry/backoff.
+    const fillTarget = Math.max(1, req.limit ?? 5);
+    const uniqueAfter = new Set(collected.map((c) => canonicalSearchUrl(c.url)).filter(Boolean));
+    if (uniqueAfter.size < fillTarget && (!req.provider || req.provider === "auto")) {
+        const extras = await fetchKeylessSearchFallbacks(localizedQuery, Math.max(fillTarget, perEngineLimit));
+        collected.push(...extras);
+    }
     return collected;
 }
-/** Below this many unique URLs from the free engines, the Jina fallback runs. */
+/** Below this many unique URLs from the free engines, the keyed search fallback runs. */
 export const JINA_FALLBACK_MIN_URLS = 3;
+/** True when a scraped SERP is a bot-check / captcha rather than results. */
+export function isSerpChallengeHtml(html, status = 200) {
+    if (status === 403 || status === 429 || status === 503)
+        return true;
+    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
+    return (lower.includes("bots use duckduckgo") ||
+        lower.includes("anomaly-modal") ||
+        lower.includes("please complete the following challenge") ||
+        lower.includes("schedule a captcha") ||
+        lower.includes("flagged as being suspicious") ||
+        lower.includes("enable javascript to function"));
+}
+export const EMPTY_SEARCH_ERROR = "Keyless search engines returned no results (DuckDuckGo and Brave often serve a bot challenge). " +
+    "Tried Wikipedia, Hacker News, Bing RSS, and DuckDuckGo Instant Answer as well. " +
+    "Set BRAVE_SEARCH_API_KEY, SERPER_API_KEY, or JINA_API_KEY, or retry later.";
+/** Canonical URL for search dedupe (host lowercased, trailing slash / tracking params stripped). */
+export function canonicalSearchUrl(raw) {
+    if (!raw)
+        return "";
+    try {
+        const u = new URL(raw);
+        u.hash = "";
+        u.hostname = u.hostname.toLowerCase();
+        for (const key of [...u.searchParams.keys()]) {
+            if (/^(utm_|ref|fbclid|gclid|mc_)/i.test(key))
+                u.searchParams.delete(key);
+        }
+        let path = u.pathname.replace(/\/+$/, "") || "/";
+        u.pathname = path;
+        return u.toString();
+    }
+    catch {
+        return raw.replace(/\/+$/, "").toLowerCase();
+    }
+}
+/** Round-robin merge so one source (e.g. Wikipedia) cannot fill `-n` by itself. */
+export function mergeSearchCandidates(groups, limit) {
+    const seen = new Set();
+    const out = [];
+    const queues = groups.map((g) => [...g]);
+    let progressed = true;
+    while (out.length < limit && progressed) {
+        progressed = false;
+        for (const q of queues) {
+            while (q.length > 0) {
+                const item = q.shift();
+                const key = canonicalSearchUrl(item.url);
+                if (!key || seen.has(key))
+                    continue;
+                seen.add(key);
+                out.push(item);
+                progressed = true;
+                break;
+            }
+            if (out.length >= limit)
+                break;
+        }
+    }
+    return out;
+}
+export async function withRetry(fn, opts = {}) {
+    const retries = opts.retries ?? 1;
+    const delayMs = opts.delayMs ?? 250;
+    const isEmpty = opts.isEmpty ?? ((v) => (Array.isArray(v) ? v.length === 0 : !v));
+    let last = await fn();
+    for (let i = 0; i < retries && isEmpty(last); i++) {
+        await new Promise((r) => setTimeout(r, delayMs * (i + 1)));
+        last = await fn();
+    }
+    return last;
+}
+/** Parse Bing's `format=rss` web-search feed into candidate docs. */
+export function parseBingWebRss(xml, limit) {
+    const items = [];
+    const itemRe = /<item>([\s\S]*?)<\/item>/gi;
+    let match;
+    while ((match = itemRe.exec(xml)) !== null && items.length < limit) {
+        const block = match[1];
+        const title = decodeHtmlEntities((block.match(/<title>([\s\S]*?)<\/title>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .trim());
+        const link = (block.match(/<link>([\s\S]*?)<\/link>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .trim();
+        const description = decodeHtmlEntities((block.match(/<description>([\s\S]*?)<\/description>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/\s+/g, " ")
+            .trim());
+        if (!title || !link.startsWith("http"))
+            continue;
+        items.push({
+            title,
+            url: link,
+            description: description || title,
+            engine: "bing-rss",
+            engineRank: items.length + 1,
+        });
+    }
+    return items;
+}
+async function fetchBingWebRss(query, limit) {
+    try {
+        const rssUrl = `https://www.bing.com/search?format=rss&q=${encodeURIComponent(query)}`;
+        const res = await fetch(rssUrl, {
+            headers: {
+                Accept: "application/rss+xml, application/xml, text/xml;q=0.9, */*;q=0.8",
+                "User-Agent": "Mozilla/5.0 (compatible; WebCrawl/1.4; +https://webcrawl.app)",
+            },
+            signal: AbortSignal.timeout(4000),
+        });
+        if (!res.ok)
+            return [];
+        const xml = await res.text();
+        if (!xml.includes("<item>"))
+            return [];
+        return parseBingWebRss(xml, limit);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty
+ * or thinner than `limit`. Wikipedia OpenSearch, HN Algolia, Bing RSS, and
+ * DuckDuckGo Instant Answer do not require a JS challenge or API key.
+ */
+export async function fetchKeylessSearchFallbacks(query, limit) {
+    const per = Math.max(limit, 5);
+    const empty = (v) => v.length === 0;
+    const run = () => Promise.all([
+        withRetry(() => fetchHackerNewsAlgolia(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
+        withRetry(() => fetchWikipediaOpenSearch(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
+        withRetry(() => fetchBingWebRss(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
+        withRetry(() => fetchDuckDuckGoInstantAnswer(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
+    ]);
+    let groups = await run();
+    if (groups.every((g) => g.length === 0)) {
+        await new Promise((r) => setTimeout(r, 400));
+        groups = await run();
+    }
+    return mergeSearchCandidates(groups, Math.max(limit, 8));
+}
+/** DuckDuckGo Instant Answer API — keyless JSON, unrelated to the HTML SERP challenge. */
+export async function fetchDuckDuckGoInstantAnswer(query, limit) {
+    try {
+        const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_redirect=1&no_html=1&skip_disambig=1`;
+        const res = await fetch(url, {
+            headers: {
+                Accept: "application/json",
+                "User-Agent": "WebCrawl/1.4 (https://webcrawl.app; search@webcrawl.app)",
+            },
+            signal: AbortSignal.timeout(2500),
+        });
+        if (!res.ok)
+            return [];
+        const json = (await res.json());
+        const items = [];
+        const push = (title, href, description) => {
+            if (!href.startsWith("http") || items.length >= limit)
+                return;
+            if (items.some((i) => canonicalSearchUrl(i.url) === canonicalSearchUrl(href)))
+                return;
+            items.push({
+                title: title || href,
+                url: href,
+                description: description || title,
+                engine: "ddg-lite",
+                engineRank: items.length + 1,
+            });
+        };
+        if (json.AbstractURL && (json.Heading || json.AbstractText)) {
+            push(json.Heading || json.AbstractSource || json.AbstractURL, json.AbstractURL, json.AbstractText || "");
+        }
+        for (const row of json.Results ?? []) {
+            if (row.FirstURL && row.Text)
+                push(row.Text, row.FirstURL, row.Text);
+        }
+        const walk = (node) => {
+            if (items.length >= limit)
+                return;
+            if (Array.isArray(node)) {
+                for (const n of node)
+                    walk(n);
+                return;
+            }
+            if (!node || typeof node !== "object")
+                return;
+            const rec = node;
+            if (rec.FirstURL && rec.Text)
+                push(rec.Text, rec.FirstURL, rec.Text);
+            if (rec.Topics)
+                walk(rec.Topics);
+        };
+        walk(json.RelatedTopics);
+        return items;
+    }
+    catch {
+        return [];
+    }
+}
 export async function fetchJinaSearch(query, limit, req) {
     const key = process.env.JINA_API_KEY;
     if (!key) {
@@ -435,7 +652,7 @@ export async function fetchBraveNativeSerp(query, limit, userLocation) {
             : "";
         const url = `https://search.brave.com/search?q=${encodeURIComponent(query)}&source=web${countryParam}`;
         const res = await fetchWithTlsImpersonation(url, 2200);
-        if (res.status !== 200 || !res.html) {
+        if (res.status !== 200 || !res.html || isSerpChallengeHtml(res.html, res.status)) {
             return [];
         }
         const html = res.html;
@@ -544,17 +761,33 @@ async function fetchDuckDuckGoLiteOnly(query, limit, userLocation) {
         const countryCode = (userLocation?.country || "").toUpperCase();
         const kl = ddgRegion(countryCode, undefined);
         const klParam = kl ? `&kl=${encodeURIComponent(kl)}` : "";
-        const liteRes = await fetchWithTlsImpersonation(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${klParam}`, 2200);
-        if (liteRes && liteRes.html) {
-            const html = liteRes.html;
-            const linkMatches = Array.from(html.matchAll(/<a\s+[^>]*href=["']([^"']+)["'][^>]*class=["']result-link["'][^>]*>([\s\S]*?)<\/a>|<a\s+[^>]*class=["']result-link["'][^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi));
-            const snippetMatches = Array.from(html.matchAll(/<td\s+[^>]*class=["']result-snippet["'][^>]*>([\s\S]*?)<\/td>/gi));
-            for (let i = 0; i < linkMatches.length && items.length < limit; i++) {
-                const m = linkMatches[i];
-                const rawUrl = m[1] || m[3] || "";
-                const rawTitle = m[2] || m[4] || "";
-                const rawSnippet = snippetMatches[i]?.[1] || "";
-                addResult(rawTitle, rawUrl, rawSnippet);
+        const endpoints = [
+            `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${klParam}`,
+            `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}${klParam}`,
+        ];
+        for (const endpoint of endpoints) {
+            if (items.length >= limit)
+                break;
+            try {
+                const liteRes = await fetchWithTlsImpersonation(endpoint, 2200);
+                if (!liteRes?.html || isSerpChallengeHtml(liteRes.html, liteRes.status)) {
+                    continue;
+                }
+                const html = liteRes.html;
+                const linkMatches = Array.from(html.matchAll(/<a\s+[^>]*href=["']([^"']+)["'][^>]*class=["']result-link["'][^>]*>([\s\S]*?)<\/a>|<a\s+[^>]*class=["']result-link["'][^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi));
+                const snippetMatches = Array.from(html.matchAll(/<td\s+[^>]*class=["']result-snippet["'][^>]*>([\s\S]*?)<\/td>/gi));
+                for (let i = 0; i < linkMatches.length && items.length < limit; i++) {
+                    const m = linkMatches[i];
+                    const rawUrl = m[1] || m[3] || "";
+                    const rawTitle = m[2] || m[4] || "";
+                    const rawSnippet = snippetMatches[i]?.[1] || "";
+                    addResult(rawTitle, rawUrl, rawSnippet);
+                }
+                if (items.length > 0)
+                    break;
+            }
+            catch {
+                // try the next DDG HTML endpoint
             }
         }
     }
@@ -629,7 +862,7 @@ async function fetchWikipediaOpenSearch(query, limit) {
                 "User-Agent": "WebCrawl/1.0 (https://webcrawl.app; search@webcrawl.app)",
                 Accept: "application/json",
             },
-            signal: AbortSignal.timeout(1400),
+            signal: AbortSignal.timeout(2200),
         });
         if (!res.ok) {
             return [];
@@ -666,7 +899,7 @@ async function fetchHackerNewsAlgolia(query, limit) {
         const url = `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(cleanQuery)}&tags=story&hitsPerPage=${limit}`;
         const res = await fetch(url, {
             headers: { Accept: "application/json" },
-            signal: AbortSignal.timeout(1400),
+            signal: AbortSignal.timeout(2200),
         });
         if (!res.ok) {
             return [];

package/dist/core/targets/registry.js

@@ -62,7 +62,7 @@ export const pdfOfficeTargetPlugin = {
     name: "PDFs & Office Documents (DOCX, XLSX, PPTX)",
     category: "documents",
     returns: "Page-numbered Markdown, tables, native text extraction, automatic OCR on scanned pages, and skipped-page diagnostics",
-    requirements: "Built-in @firecrawl/pdf-inspector, mammoth, and officeparser; optional OCR provider for scanned pages",
+    requirements: "Built-in PDF inspector, mammoth, and officeparser; optional OCR provider for scanned pages",
     examples: [
         "npx webcrawl scrape https://arxiv.org/pdf/1706.03762.pdf",
     ],

package/dist/format.js

@@ -49,7 +49,11 @@ export function formatSearch(res) {
         })
             .join("\n")}`);
     }
-    return parts.length ? parts.join("\n\n") : "No results.";
+    if (parts.length)
+        return parts.join("\n\n");
+    if (res.success === false && str(res.error))
+        return `Error: ${str(res.error)}`;
+    return "No results.";
 }
 export function formatScrape(res) {
     if (res.success === false && str(res.error)) {

package/dist/setup.js

@@ -147,19 +147,23 @@ export function runDoctor(env = process.env, effectiveMode = "local", apiKeySour
     });
     // 3. Local Chrome / Stealth Browser availability
     const chromePaths = [
+        env.WEBCRAWL_CHROME_PATH,
+        env.INTERACT_CHROME_PATH,
         env.CHROME_PATH,
         "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
         "/usr/bin/google-chrome",
+        "/usr/bin/google-chrome-stable",
         "/usr/bin/chromium",
         "/usr/bin/chromium-browser",
+        "/snap/bin/chromium",
     ].filter(Boolean);
     const foundChrome = chromePaths.find((p) => existsSync(p));
     checks.push({
         name: "Local stealth browser",
         status: foundChrome ? "ok" : "optional",
         detail: foundChrome
-            ? `Found Chrome at ${foundChrome}`
-            : "No local Chrome binary detected; static/TLS fetch works out of the box, or run `webcrawl login` for cloud stealth browsers",
+            ? `Found Chrome at ${foundChrome} (used to render JS pages)`
+            : "No local Chrome binary detected; static pages still work. For JS pages install Chrome or `npx playwright install chromium-headless-shell`, or run `webcrawl login` for cloud stealth browsers",
     });
     // 4. Search providers
     const serper = Boolean(env.SERPER_API_KEY);
@@ -180,7 +184,7 @@ export function runDoctor(env = process.env, effectiveMode = "local", apiKeySour
     checks.push({
         name: "Document & Media parsers",
         status: "ok",
-        detail: "PDF (@firecrawl/pdf-inspector), Office (mammoth/officeparser), YouTube captions, NVD/OSV CVEs ready",
+        detail: "PDF inspector, Office (mammoth/officeparser), YouTube captions, NVD/OSV CVEs ready",
     });
     return { mode: effectiveMode, checks };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.4.2",
+  "version": "1.4.4",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.4.2",
-    "@webcrawl/curl-impersonate": "^1.4.2",
+    "@webcrawl/client": "^1.4.4",
+    "@webcrawl/curl-impersonate": "^1.4.4",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",