webcrawl 1.4.3 → 1.4.4

package/dist/core/search/providers.js

@@ -89,11 +89,15 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         },
         wikipedia: {
             primary: false,
-            run: () => fetchWikipediaOpenSearch(localizedQuery, 2),
+            run: () => fetchWikipediaOpenSearch(localizedQuery, Math.min(perEngineLimit, 5)),
         },
         "hn-algolia": {
             primary: false,
-            run: () => fetchHackerNewsAlgolia(localizedQuery, 5),
+            run: () => fetchHackerNewsAlgolia(localizedQuery, perEngineLimit),
+        },
+        "bing-rss": {
+            primary: false,
+            run: () => fetchBingWebRss(localizedQuery, perEngineLimit),
         },
         arxiv: {
             primary: category === "academic",
@@ -159,14 +163,17 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
                 resolve();
                 return;
             }
-            // Early-exit quorum: at least 1 primary web engine + 2 total engines + enough candidates (and shopping engine finished when in shopping plan)
+            const uniqueCount = new Set(collected.map((c) => canonicalSearchUrl(c.url)).filter(Boolean)).size;
+            const fillTarget = Math.max(1, req.limit ?? 5);
+            // Early-exit quorum: at least 1 primary web engine + 2 total engines + enough
+            // *unique* URLs to fill `limit` (not just Wikipedia's 2 hits).
             if (completedShoppingEngine &&
                 completedNewsEngine &&
                 completedNvdEngine &&
                 completedArxivEngine &&
                 completedPrimaryEngines >= 1 &&
                 totalCompletedEngines >= 2 &&
-                collected.length >= perEngineLimit &&
+                uniqueCount >= fillTarget &&
                 !graceTimer) {
                 graceTimer = setTimeout(() => {
                     if (!settled) {
@@ -209,7 +216,7 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         }
     });
     // Fallback: the free engines are scraped and can be blocked or rate limited. When they
-    // come back thin, fill in from Jina (paid) so the caller still gets an answer.
+    // come back thin, fill in from a keyed search provider so the caller still gets an answer.
     const uniqueUrls = new Set(collected.map((c) => c.url).filter(Boolean));
     if (uniqueUrls.size < JINA_FALLBACK_MIN_URLS && process.env.JINA_API_KEY) {
         const t0 = performance.now();
@@ -220,14 +227,17 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         collected.push(...jina);
     }
     // HTML SERPs (DDG, Brave) are frequently challenge/429 pages. When they
-    // yield nothing, fall back to keyless APIs that still work without a key.
-    if (collected.length === 0 && (!req.provider || req.provider === "auto")) {
-        const extras = await fetchKeylessSearchFallbacks(localizedQuery, perEngineLimit);
+    // yield nothing *or come back thinner than `limit`*, fill from keyless APIs
+    // (Wikipedia, HN Algolia, Bing RSS, DDG Instant Answer) with retry/backoff.
+    const fillTarget = Math.max(1, req.limit ?? 5);
+    const uniqueAfter = new Set(collected.map((c) => canonicalSearchUrl(c.url)).filter(Boolean));
+    if (uniqueAfter.size < fillTarget && (!req.provider || req.provider === "auto")) {
+        const extras = await fetchKeylessSearchFallbacks(localizedQuery, Math.max(fillTarget, perEngineLimit));
         collected.push(...extras);
     }
     return collected;
 }
-/** Below this many unique URLs from the free engines, the Jina fallback runs. */
+/** Below this many unique URLs from the free engines, the keyed search fallback runs. */
 export const JINA_FALLBACK_MIN_URLS = 3;
 /** True when a scraped SERP is a bot-check / captcha rather than results. */
 export function isSerpChallengeHtml(html, status = 200) {
@@ -242,8 +252,64 @@ export function isSerpChallengeHtml(html, status = 200) {
         lower.includes("enable javascript to function"));
 }
 export const EMPTY_SEARCH_ERROR = "Keyless search engines returned no results (DuckDuckGo and Brave often serve a bot challenge). " +
-    "Tried Wikipedia, Hacker News, and Bing RSS as well. " +
+    "Tried Wikipedia, Hacker News, Bing RSS, and DuckDuckGo Instant Answer as well. " +
     "Set BRAVE_SEARCH_API_KEY, SERPER_API_KEY, or JINA_API_KEY, or retry later.";
+/** Canonical URL for search dedupe (host lowercased, trailing slash / tracking params stripped). */
+export function canonicalSearchUrl(raw) {
+    if (!raw)
+        return "";
+    try {
+        const u = new URL(raw);
+        u.hash = "";
+        u.hostname = u.hostname.toLowerCase();
+        for (const key of [...u.searchParams.keys()]) {
+            if (/^(utm_|ref|fbclid|gclid|mc_)/i.test(key))
+                u.searchParams.delete(key);
+        }
+        let path = u.pathname.replace(/\/+$/, "") || "/";
+        u.pathname = path;
+        return u.toString();
+    }
+    catch {
+        return raw.replace(/\/+$/, "").toLowerCase();
+    }
+}
+/** Round-robin merge so one source (e.g. Wikipedia) cannot fill `-n` by itself. */
+export function mergeSearchCandidates(groups, limit) {
+    const seen = new Set();
+    const out = [];
+    const queues = groups.map((g) => [...g]);
+    let progressed = true;
+    while (out.length < limit && progressed) {
+        progressed = false;
+        for (const q of queues) {
+            while (q.length > 0) {
+                const item = q.shift();
+                const key = canonicalSearchUrl(item.url);
+                if (!key || seen.has(key))
+                    continue;
+                seen.add(key);
+                out.push(item);
+                progressed = true;
+                break;
+            }
+            if (out.length >= limit)
+                break;
+        }
+    }
+    return out;
+}
+export async function withRetry(fn, opts = {}) {
+    const retries = opts.retries ?? 1;
+    const delayMs = opts.delayMs ?? 250;
+    const isEmpty = opts.isEmpty ?? ((v) => (Array.isArray(v) ? v.length === 0 : !v));
+    let last = await fn();
+    for (let i = 0; i < retries && isEmpty(last); i++) {
+        await new Promise((r) => setTimeout(r, delayMs * (i + 1)));
+        last = await fn();
+    }
+    return last;
+}
 /** Parse Bing's `format=rss` web-search feed into candidate docs. */
 export function parseBingWebRss(xml, limit) {
     const items = [];
@@ -296,27 +362,83 @@ async function fetchBingWebRss(query, limit) {
     }
 }
 /**
- * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty.
- * Wikipedia OpenSearch, HN Algolia, and Bing RSS do not require a JS challenge.
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty
+ * or thinner than `limit`. Wikipedia OpenSearch, HN Algolia, Bing RSS, and
+ * DuckDuckGo Instant Answer do not require a JS challenge or API key.
  */
 export async function fetchKeylessSearchFallbacks(query, limit) {
     const per = Math.max(limit, 5);
-    const [hn, wiki, bing] = await Promise.all([
-        fetchHackerNewsAlgolia(query, per),
-        fetchWikipediaOpenSearch(query, per),
-        fetchBingWebRss(query, per),
+    const empty = (v) => v.length === 0;
+    const run = () => Promise.all([
+        withRetry(() => fetchHackerNewsAlgolia(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
+        withRetry(() => fetchWikipediaOpenSearch(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
+        withRetry(() => fetchBingWebRss(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
+        withRetry(() => fetchDuckDuckGoInstantAnswer(query, per), { retries: 1, delayMs: 280, isEmpty: empty }),
     ]);
-    const seen = new Set();
-    const out = [];
-    for (const item of [...hn, ...wiki, ...bing]) {
-        if (!item.url || seen.has(item.url))
-            continue;
-        seen.add(item.url);
-        out.push(item);
-        if (out.length >= Math.max(limit + 4, 8))
-            break;
+    let groups = await run();
+    if (groups.every((g) => g.length === 0)) {
+        await new Promise((r) => setTimeout(r, 400));
+        groups = await run();
+    }
+    return mergeSearchCandidates(groups, Math.max(limit, 8));
+}
+/** DuckDuckGo Instant Answer API — keyless JSON, unrelated to the HTML SERP challenge. */
+export async function fetchDuckDuckGoInstantAnswer(query, limit) {
+    try {
+        const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_redirect=1&no_html=1&skip_disambig=1`;
+        const res = await fetch(url, {
+            headers: {
+                Accept: "application/json",
+                "User-Agent": "WebCrawl/1.4 (https://webcrawl.app; search@webcrawl.app)",
+            },
+            signal: AbortSignal.timeout(2500),
+        });
+        if (!res.ok)
+            return [];
+        const json = (await res.json());
+        const items = [];
+        const push = (title, href, description) => {
+            if (!href.startsWith("http") || items.length >= limit)
+                return;
+            if (items.some((i) => canonicalSearchUrl(i.url) === canonicalSearchUrl(href)))
+                return;
+            items.push({
+                title: title || href,
+                url: href,
+                description: description || title,
+                engine: "ddg-lite",
+                engineRank: items.length + 1,
+            });
+        };
+        if (json.AbstractURL && (json.Heading || json.AbstractText)) {
+            push(json.Heading || json.AbstractSource || json.AbstractURL, json.AbstractURL, json.AbstractText || "");
+        }
+        for (const row of json.Results ?? []) {
+            if (row.FirstURL && row.Text)
+                push(row.Text, row.FirstURL, row.Text);
+        }
+        const walk = (node) => {
+            if (items.length >= limit)
+                return;
+            if (Array.isArray(node)) {
+                for (const n of node)
+                    walk(n);
+                return;
+            }
+            if (!node || typeof node !== "object")
+                return;
+            const rec = node;
+            if (rec.FirstURL && rec.Text)
+                push(rec.Text, rec.FirstURL, rec.Text);
+            if (rec.Topics)
+                walk(rec.Topics);
+        };
+        walk(json.RelatedTopics);
+        return items;
+    }
+    catch {
+        return [];
     }
-    return out;
 }
 export async function fetchJinaSearch(query, limit, req) {
     const key = process.env.JINA_API_KEY;
@@ -639,17 +761,33 @@ async function fetchDuckDuckGoLiteOnly(query, limit, userLocation) {
         const countryCode = (userLocation?.country || "").toUpperCase();
         const kl = ddgRegion(countryCode, undefined);
         const klParam = kl ? `&kl=${encodeURIComponent(kl)}` : "";
-        const liteRes = await fetchWithTlsImpersonation(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${klParam}`, 2200);
-        if (liteRes && liteRes.html && !isSerpChallengeHtml(liteRes.html, liteRes.status)) {
-            const html = liteRes.html;
-            const linkMatches = Array.from(html.matchAll(/<a\s+[^>]*href=["']([^"']+)["'][^>]*class=["']result-link["'][^>]*>([\s\S]*?)<\/a>|<a\s+[^>]*class=["']result-link["'][^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi));
-            const snippetMatches = Array.from(html.matchAll(/<td\s+[^>]*class=["']result-snippet["'][^>]*>([\s\S]*?)<\/td>/gi));
-            for (let i = 0; i < linkMatches.length && items.length < limit; i++) {
-                const m = linkMatches[i];
-                const rawUrl = m[1] || m[3] || "";
-                const rawTitle = m[2] || m[4] || "";
-                const rawSnippet = snippetMatches[i]?.[1] || "";
-                addResult(rawTitle, rawUrl, rawSnippet);
+        const endpoints = [
+            `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${klParam}`,
+            `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}${klParam}`,
+        ];
+        for (const endpoint of endpoints) {
+            if (items.length >= limit)
+                break;
+            try {
+                const liteRes = await fetchWithTlsImpersonation(endpoint, 2200);
+                if (!liteRes?.html || isSerpChallengeHtml(liteRes.html, liteRes.status)) {
+                    continue;
+                }
+                const html = liteRes.html;
+                const linkMatches = Array.from(html.matchAll(/<a\s+[^>]*href=["']([^"']+)["'][^>]*class=["']result-link["'][^>]*>([\s\S]*?)<\/a>|<a\s+[^>]*class=["']result-link["'][^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi));
+                const snippetMatches = Array.from(html.matchAll(/<td\s+[^>]*class=["']result-snippet["'][^>]*>([\s\S]*?)<\/td>/gi));
+                for (let i = 0; i < linkMatches.length && items.length < limit; i++) {
+                    const m = linkMatches[i];
+                    const rawUrl = m[1] || m[3] || "";
+                    const rawTitle = m[2] || m[4] || "";
+                    const rawSnippet = snippetMatches[i]?.[1] || "";
+                    addResult(rawTitle, rawUrl, rawSnippet);
+                }
+                if (items.length > 0)
+                    break;
+            }
+            catch {
+                // try the next DDG HTML endpoint
             }
         }
     }
@@ -724,7 +862,7 @@ async function fetchWikipediaOpenSearch(query, limit) {
                 "User-Agent": "WebCrawl/1.0 (https://webcrawl.app; search@webcrawl.app)",
                 Accept: "application/json",
             },
-            signal: AbortSignal.timeout(1400),
+            signal: AbortSignal.timeout(2200),
         });
         if (!res.ok) {
             return [];
@@ -761,7 +899,7 @@ async function fetchHackerNewsAlgolia(query, limit) {
         const url = `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(cleanQuery)}&tags=story&hitsPerPage=${limit}`;
         const res = await fetch(url, {
             headers: { Accept: "application/json" },
-            signal: AbortSignal.timeout(1400),
+            signal: AbortSignal.timeout(2200),
         });
         if (!res.ok) {
             return [];

package/dist/core/targets/registry.js

@@ -62,7 +62,7 @@ export const pdfOfficeTargetPlugin = {
     name: "PDFs & Office Documents (DOCX, XLSX, PPTX)",
     category: "documents",
     returns: "Page-numbered Markdown, tables, native text extraction, automatic OCR on scanned pages, and skipped-page diagnostics",
-    requirements: "Built-in @firecrawl/pdf-inspector, mammoth, and officeparser; optional OCR provider for scanned pages",
+    requirements: "Built-in PDF inspector, mammoth, and officeparser; optional OCR provider for scanned pages",
     examples: [
         "npx webcrawl scrape https://arxiv.org/pdf/1706.03762.pdf",
     ],

package/dist/setup.js

@@ -163,7 +163,7 @@ export function runDoctor(env = process.env, effectiveMode = "local", apiKeySour
         status: foundChrome ? "ok" : "optional",
         detail: foundChrome
             ? `Found Chrome at ${foundChrome} (used to render JS pages)`
-            : "No local Chrome binary detected; static/TLS fetch works out of the box, or run `webcrawl login` for cloud stealth browsers",
+            : "No local Chrome binary detected; static pages still work. For JS pages install Chrome or `npx playwright install chromium-headless-shell`, or run `webcrawl login` for cloud stealth browsers",
     });
     // 4. Search providers
     const serper = Boolean(env.SERPER_API_KEY);
@@ -184,7 +184,7 @@ export function runDoctor(env = process.env, effectiveMode = "local", apiKeySour
     checks.push({
         name: "Document & Media parsers",
         status: "ok",
-        detail: "PDF (@firecrawl/pdf-inspector), Office (mammoth/officeparser), YouTube captions, NVD/OSV CVEs ready",
+        detail: "PDF inspector, Office (mammoth/officeparser), YouTube captions, NVD/OSV CVEs ready",
     });
     return { mode: effectiveMode, checks };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.4.3",
+  "version": "1.4.4",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.4.3",
-    "@webcrawl/curl-impersonate": "^1.4.3",
+    "@webcrawl/client": "^1.4.4",
+    "@webcrawl/curl-impersonate": "^1.4.4",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",