webcrawl 1.4.0 → 1.4.2

package/dist/cli.js

@@ -434,8 +434,13 @@ export async function main(argv, io, deps = {}) {
             }
             case "scrape": {
                 const res = await client.scrape({ url: need("url"), formats: ["markdown"], noCache: v["no-cache"] });
+                // Blocked pages come back as success:false: report on stderr and exit 1, not empty content.
+                if (res.success === false && !v.json) {
+                    io.err(`${formatScrape(res)}\n`);
+                    return 1;
+                }
                 emit(res, formatScrape(res));
-                return 0;
+                return res.success === false ? 1 : 0;
             }
             case "map": {
                 const res = await client.map({ url: need("url"), search: v.filter, limit: int(v.limit, "--limit") });

package/dist/core/engine/safe-url.d.ts

@@ -2,6 +2,12 @@ export declare class UnsafeUrlError extends Error {
     constructor(message?: string);
 }
 export declare function isPrivateAddress(ip: string): boolean;
+/**
+ * True for 198.18.0.0/15, the benchmarking range that fake-IP proxies
+ * (Clash, Surge, sing-box, some VPN clients) hand out for every hostname.
+ */
+export declare function isFakeIpAddress(ip: string): boolean;
+export declare const FAKE_IP_HINT: string;
 export type HostResolver = (host: string) => Promise<string[]>;
 /**
  * Throws `UnsafeUrlError` unless `raw` is an http(s) URL whose host resolves

package/dist/core/engine/safe-url.js

@@ -35,6 +35,20 @@ export function isPrivateAddress(ip) {
     }
     return true;
 }
+/**
+ * True for 198.18.0.0/15, the benchmarking range that fake-IP proxies
+ * (Clash, Surge, sing-box, some VPN clients) hand out for every hostname.
+ */
+export function isFakeIpAddress(ip) {
+    const s = ip.toLowerCase().replace(/^::ffff:/, "");
+    if (isIP(s) !== 4)
+        return false;
+    const [a, b] = s.split(".").map(Number);
+    return a === 198 && (b === 18 || b === 19);
+}
+export const FAKE_IP_HINT = "That address isn't publicly reachable: it resolved to 198.18.x.x, which fake-IP proxies " +
+    "(Clash, Surge, sing-box, some VPNs) return for every site. Switch the proxy to redir-host/real-IP " +
+    "DNS mode, or set WEBCRAWL_ALLOW_PRIVATE_URLS=true for local runs you trust.";
 const defaultResolve = async (h) => (await lookup(h, { all: true })).map((r) => r.address);
 /** Local development escape hatch (never set in production). */
 function allowPrivate() {
@@ -67,8 +81,11 @@ export async function assertSafeUrl(raw, resolve = defaultResolve) {
     const addrs = isIP(host) ? [host] : await resolve(host).catch(() => []);
     if (addrs.length === 0)
         throw new UnsafeUrlError("We couldn't find that website");
-    if (addrs.some(isPrivateAddress))
-        throw new UnsafeUrlError();
+    if (addrs.some(isPrivateAddress)) {
+        // Every address in the fake-IP range means a local proxy rewrote DNS,
+        // not that the site is internal: say so and name the workaround.
+        throw new UnsafeUrlError(addrs.every(isFakeIpAddress) ? FAKE_IP_HINT : undefined);
+    }
     return u;
 }
 export const MAX_REDIRECTS = 8;

package/dist/core/local-engine.d.ts

@@ -32,4 +32,15 @@ export interface LocalEngineBackend {
     targets(): Promise<Record<string, unknown>>;
     usage(): Promise<Record<string, unknown>>;
 }
+/**
+ * Scrape result for a page the site refused (bot check, 403/429/503).
+ * `success: false` with no markdown, so callers never mistake the block page
+ * for content. Exported for unit tests.
+ */
+export declare function blockedScrapeResult(url: string, statusCode: number, tier: string, latencyMs: number): Record<string, unknown>;
+/**
+ * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
+ * Matching is a case-insensitive substring of the URL. Exported for unit tests.
+ */
+export declare function filterMappedLinks(links: string[], search?: string, limit?: number): string[];
 export declare function createLocalEngineClient(): LocalEngineBackend;

package/dist/core/local-engine.js

@@ -9,10 +9,10 @@
  * Every community improvement to extractors, parsers, or targets in `src/` immediately
  * powers both this local engine (`npx webcrawl`) and the hosted `webcrawl.app` API.
  */
-import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
+import { extractWebsiteData, fastHtmlToMarkdown, fetchWithTlsImpersonation, getCachedPageAsync, isChallengeMarkdown, mapWebsiteUrls, setCachedPage, sha256, } from "./engine.js";
 import { isDomainRenderNeeded, markDomainRenderNeeded, } from "./engine/session.js";
 import { isRenderProviderEnabled, renderViaProvider } from "./engine/render.js";
-import { fetchResource, lastFetchTier, needsBrowserEscalation } from "./engine/fetcher.js";
+import { fetchResource, isAntiBotChallenge, lastFetchTier, needsBrowserEscalation, } from "./engine/fetcher.js";
 import { parseFile, publicParseMeta } from "./parse/index.js";
 import { isBinaryContentType, looksLikeBinaryUrl } from "./parse/fetch.js";
 import { maxDownloadBytes } from "./parse/limits.js";
@@ -29,6 +29,42 @@ function safeDomainOf(url) {
         return "";
     }
 }
+/**
+ * Scrape result for a page the site refused (bot check, 403/429/503).
+ * `success: false` with no markdown, so callers never mistake the block page
+ * for content. Exported for unit tests.
+ */
+export function blockedScrapeResult(url, statusCode, tier, latencyMs) {
+    const error = `Blocked by the site's bot check (HTTP ${statusCode}). No content was returned. ` +
+        "Retry with `--mode cloud` (hosted stealth browser), or self-host a render provider via WEBCRAWL_RENDER_URL.";
+    return {
+        success: false,
+        mode: "local",
+        url,
+        blocked: true,
+        statusCode,
+        error,
+        title: url,
+        markdown: "",
+        links: [],
+        data: { url, title: url, markdown: "", links: [] },
+        evidence: { tier, blocked: true, statusCode, latencyMs, limitations: [error] },
+    };
+}
+/**
+ * Apply `map --filter` / `search` after discovery, then honour `-n` / `limit`.
+ * Matching is a case-insensitive substring of the URL. Exported for unit tests.
+ */
+export function filterMappedLinks(links, search, limit = 100) {
+    const cap = Math.max(1, limit);
+    const needle = search?.trim().toLowerCase();
+    const filtered = needle
+        ? links.filter((l) => l.toLowerCase().includes(needle))
+        : links;
+    return filtered.slice(0, cap);
+}
+/** How many URLs to discover before applying `--filter`, so a small `-n` cannot hide matches. */
+const MAP_FILTER_DISCOVER_LIMIT = 5000;
 export function createLocalEngineClient() {
     return {
         baseUrl: "local://engine",
@@ -124,7 +160,9 @@ export function createLocalEngineClient() {
             // 3. Check local cache first
             if (!p.noCache) {
                 const cached = await getCachedPageAsync(targetUrl, "none");
-                if (cached) {
+                // Entries written before blocked pages stopped being cached may hold a
+                // bot-check page: treat those as a miss and fetch again.
+                if (cached && !isChallengeMarkdown(cached.title, cached.markdown)) {
                     const targetInspection = await inspectTarget({
                         url: targetUrl,
                         markdown: cached.markdown,
@@ -199,13 +237,18 @@ export function createLocalEngineClient() {
             const fetched = await fetchWithTlsImpersonation(targetUrl);
             const { title, markdown, links } = fastHtmlToMarkdown(fetched.html, targetUrl);
             const contentHash = sha256(markdown || fetched.html);
-            setCachedPage(targetUrl, "none", {
-                title,
-                markdown,
-                links,
-                statusCode: fetched.status,
-                contentHash,
-            });
+            // A bot-check / 403 / 429 / 503 response is not the page: never cache it,
+            // or every retry for the next 24h would get the block page back.
+            const blocked = isAntiBotChallenge(fetched.html, fetched.status);
+            if (!blocked) {
+                setCachedPage(targetUrl, "none", {
+                    title,
+                    markdown,
+                    links,
+                    statusCode: fetched.status,
+                    contentHash,
+                });
+            }
             const targetInspection = await inspectTarget({
                 url: targetUrl,
                 html: fetched.html,
@@ -258,6 +301,9 @@ export function createLocalEngineClient() {
                     : lastFetchTier() === "proxy-impersonate"
                         ? "proxy-impersonate"
                         : "impit-tls";
+                if (blocked) {
+                    return blockedScrapeResult(targetUrl, fetched.status, nativeTier, Date.now() - start);
+                }
                 return {
                     success: true,
                     mode: "local",
@@ -369,13 +415,15 @@ export function createLocalEngineClient() {
         },
         async map(p) {
             const targetUrl = /^https?:\/\//i.test(p.url) ? p.url : `https://${p.url}`;
+            const limit = Math.max(1, p.limit ?? 100);
+            const needle = p.search?.trim();
+            // Discover first (widely when filtering), then apply search and limit so
+            // `map --filter webhooks -n 10` can still find /webhooks/* URLs that are
+            // not among the first 10 sitemap entries (#11).
             const mapped = await mapWebsiteUrls(targetUrl, {
-                limit: p.limit ?? 100,
+                limit: needle ? Math.max(limit, MAP_FILTER_DISCOVER_LIMIT) : limit,
             });
-            const needle = p.search?.trim().toLowerCase();
-            const links = needle
-                ? mapped.links.filter((l) => l.toLowerCase().includes(needle))
-                : mapped.links;
+            const links = filterMappedLinks(mapped.links, needle, limit);
             return {
                 success: true,
                 mode: "local",

package/dist/format.js

@@ -52,6 +52,9 @@ export function formatSearch(res) {
     return parts.length ? parts.join("\n\n") : "No results.";
 }
 export function formatScrape(res) {
+    if (res.success === false && str(res.error)) {
+        return `Error: ${str(res.error)}`;
+    }
     const d = isRec(res.data) ? res.data : res;
     return str(d.markdown) || str(d.content) || "(no content returned)";
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcrawl",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "Webcrawl Anything CLI & Core Engine — crawl, search, and extract structured evidence from any website, SPA, PDF, Office file, YouTube video, GitHub repo, ArXiv paper, or CVE",
   "type": "module",
   "license": "MIT",
@@ -127,8 +127,8 @@
     "@ai-sdk/gateway": "^4.0.123",
     "@firecrawl/pdf-inspector": "^1.25.2",
     "@vercel/sandbox": "^1.0.0",
-    "@webcrawl/client": "^1.4.0",
-    "@webcrawl/curl-impersonate": "^1.4.0",
+    "@webcrawl/client": "^1.4.2",
+    "@webcrawl/curl-impersonate": "^1.4.2",
     "ai": "^7.0.150",
     "impit": "^0.14.7",
     "mammoth": "^1.13.0",