webcrawl 1.4.1 → 1.4.3

package/dist/core/search/providers.js

@@ -219,10 +219,105 @@ export async function fetchHedgedNativeFederation(rawQuery, localizedQuery, perE
         }
         collected.push(...jina);
     }
+    // HTML SERPs (DDG, Brave) are frequently challenge/429 pages. When they
+    // yield nothing, fall back to keyless APIs that still work without a key.
+    if (collected.length === 0 && (!req.provider || req.provider === "auto")) {
+        const extras = await fetchKeylessSearchFallbacks(localizedQuery, perEngineLimit);
+        collected.push(...extras);
+    }
     return collected;
 }
 /** Below this many unique URLs from the free engines, the Jina fallback runs. */
 export const JINA_FALLBACK_MIN_URLS = 3;
+/** True when a scraped SERP is a bot-check / captcha rather than results. */
+export function isSerpChallengeHtml(html, status = 200) {
+    if (status === 403 || status === 429 || status === 503)
+        return true;
+    const lower = String(html ?? "").slice(0, 8000).toLowerCase();
+    return (lower.includes("bots use duckduckgo") ||
+        lower.includes("anomaly-modal") ||
+        lower.includes("please complete the following challenge") ||
+        lower.includes("schedule a captcha") ||
+        lower.includes("flagged as being suspicious") ||
+        lower.includes("enable javascript to function"));
+}
+export const EMPTY_SEARCH_ERROR = "Keyless search engines returned no results (DuckDuckGo and Brave often serve a bot challenge). " +
+    "Tried Wikipedia, Hacker News, and Bing RSS as well. " +
+    "Set BRAVE_SEARCH_API_KEY, SERPER_API_KEY, or JINA_API_KEY, or retry later.";
+/** Parse Bing's `format=rss` web-search feed into candidate docs. */
+export function parseBingWebRss(xml, limit) {
+    const items = [];
+    const itemRe = /<item>([\s\S]*?)<\/item>/gi;
+    let match;
+    while ((match = itemRe.exec(xml)) !== null && items.length < limit) {
+        const block = match[1];
+        const title = decodeHtmlEntities((block.match(/<title>([\s\S]*?)<\/title>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .trim());
+        const link = (block.match(/<link>([\s\S]*?)<\/link>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .trim();
+        const description = decodeHtmlEntities((block.match(/<description>([\s\S]*?)<\/description>/i)?.[1] || "")
+            .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/\s+/g, " ")
+            .trim());
+        if (!title || !link.startsWith("http"))
+            continue;
+        items.push({
+            title,
+            url: link,
+            description: description || title,
+            engine: "bing-rss",
+            engineRank: items.length + 1,
+        });
+    }
+    return items;
+}
+async function fetchBingWebRss(query, limit) {
+    try {
+        const rssUrl = `https://www.bing.com/search?format=rss&q=${encodeURIComponent(query)}`;
+        const res = await fetch(rssUrl, {
+            headers: {
+                Accept: "application/rss+xml, application/xml, text/xml;q=0.9, */*;q=0.8",
+                "User-Agent": "Mozilla/5.0 (compatible; WebCrawl/1.4; +https://webcrawl.app)",
+            },
+            signal: AbortSignal.timeout(4000),
+        });
+        if (!res.ok)
+            return [];
+        const xml = await res.text();
+        if (!xml.includes("<item>"))
+            return [];
+        return parseBingWebRss(xml, limit);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty.
+ * Wikipedia OpenSearch, HN Algolia, and Bing RSS do not require a JS challenge.
+ */
+export async function fetchKeylessSearchFallbacks(query, limit) {
+    const per = Math.max(limit, 5);
+    const [hn, wiki, bing] = await Promise.all([
+        fetchHackerNewsAlgolia(query, per),
+        fetchWikipediaOpenSearch(query, per),
+        fetchBingWebRss(query, per),
+    ]);
+    const seen = new Set();
+    const out = [];
+    for (const item of [...hn, ...wiki, ...bing]) {
+        if (!item.url || seen.has(item.url))
+            continue;
+        seen.add(item.url);
+        out.push(item);
+        if (out.length >= Math.max(limit + 4, 8))
+            break;
+    }
+    return out;
+}
 export async function fetchJinaSearch(query, limit, req) {
     const key = process.env.JINA_API_KEY;
     if (!key) {
@@ -435,7 +530,7 @@ export async function fetchBraveNativeSerp(query, limit, userLocation) {
             : "";
         const url = `https://search.brave.com/search?q=${encodeURIComponent(query)}&source=web${countryParam}`;
         const res = await fetchWithTlsImpersonation(url, 2200);
-        if (res.status !== 200 || !res.html) {
+        if (res.status !== 200 || !res.html || isSerpChallengeHtml(res.html, res.status)) {
             return [];
         }
         const html = res.html;
@@ -545,7 +640,7 @@ async function fetchDuckDuckGoLiteOnly(query, limit, userLocation) {
         const kl = ddgRegion(countryCode, undefined);
         const klParam = kl ? `&kl=${encodeURIComponent(kl)}` : "";
         const liteRes = await fetchWithTlsImpersonation(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${klParam}`, 2200);
-        if (liteRes && liteRes.html) {
+        if (liteRes && liteRes.html && !isSerpChallengeHtml(liteRes.html, liteRes.status)) {
             const html = liteRes.html;
             const linkMatches = Array.from(html.matchAll(/<a\s+[^>]*href=["']([^"']+)["'][^>]*class=["']result-link["'][^>]*>([\s\S]*?)<\/a>|<a\s+[^>]*class=["']result-link["'][^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi));
             const snippetMatches = Array.from(html.matchAll(/<td\s+[^>]*class=["']result-snippet["'][^>]*>([\s\S]*?)<\/td>/gi));

package/dist/format.js

@@ -49,9 +49,16 @@ export function formatSearch(res) {
         })
             .join("\n")}`);
     }
-    return parts.length ? parts.join("\n\n") : "No results.";
+    if (parts.length)
+        return parts.join("\n\n");
+    if (res.success === false && str(res.error))
+        return `Error: ${str(res.error)}`;
+    return "No results.";
 }
 export function formatScrape(res) {
+    if (res.success === false && str(res.error)) {
+        return `Error: ${str(res.error)}`;
+    }
     const d = isRec(res.data) ? res.data : res;
     return str(d.markdown) || str(d.content) || "(no content returned)";
 }

package/dist/setup.js

@@ -147,18 +147,22 @@ export function runDoctor(env = process.env, effectiveMode = "local", apiKeySour
     });
     // 3. Local Chrome / Stealth Browser availability
     const chromePaths = [
+        env.WEBCRAWL_CHROME_PATH,
+        env.INTERACT_CHROME_PATH,
         env.CHROME_PATH,
         "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
         "/usr/bin/google-chrome",
+        "/usr/bin/google-chrome-stable",
         "/usr/bin/chromium",
         "/usr/bin/chromium-browser",
+        "/snap/bin/chromium",
     ].filter(Boolean);
     const foundChrome = chromePaths.find((p) => existsSync(p));
     checks.push({
         name: "Local stealth browser",
         status: foundChrome ? "ok" : "optional",
         detail: foundChrome
-            ? `Found Chrome at ${foundChrome}`
+            ? `Found Chrome at ${foundChrome} (used to render JS pages)`
             : "No local Chrome binary detected; static/TLS fetch works out of the box, or run `webcrawl login` for cloud stealth browsers",
     });
     // 4. Search providers

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.4.1",
-    "@webcrawl/curl-impersonate": "^1.4.1",
+    "@webcrawl/client": "^1.4.3",
+    "@webcrawl/curl-impersonate": "^1.4.3",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",