webcrawl 1.2.4 → 1.2.7

package/dist/core/engine/path-scope.js

@@ -0,0 +1,88 @@
+/**
+ * Path scoping for map/crawl: restrict discovered URLs to a subpath and/or
+ * include/exclude path patterns. Patterns match the URL path (+ query):
+ *   "/docs"        prefix at a segment boundary ("/docs", "/docs/a", not "/docs-old")
+ *   "/blog/*"      `*` matches any characters
+ *   "/api/*\/v1$"  trailing `$` anchors the end
+ */
+function toRegExp(pattern) {
+    const p = pattern.trim();
+    const anchored = p.endsWith("$");
+    const raw = anchored ? p.slice(0, -1) : p;
+    const hasWildcard = raw.includes("*");
+    const body = raw
+        .replace(/[.+?^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, ".*");
+    if (anchored)
+        return new RegExp(`^${body}$`);
+    if (hasWildcard)
+        return new RegExp(`^${body}`);
+    // Plain prefix: match at a segment boundary.
+    const trimmed = body.replace(/\/+$/, "");
+    return new RegExp(`^${trimmed}(?:/|$|\\?)`);
+}
+function normalizePattern(p) {
+    let s = String(p ?? "").trim();
+    if (!s)
+        return null;
+    // Accept full URLs by taking their path.
+    if (/^https?:\/\//i.test(s)) {
+        try {
+            s = new URL(s).pathname;
+        }
+        catch {
+            return null;
+        }
+    }
+    if (!s.startsWith("/") && !s.startsWith("*"))
+        s = `/${s}`;
+    return s;
+}
+export function buildPathScope(startUrl, options = {}) {
+    const clean = (list) => (list ?? []).map(normalizePattern).filter((x) => !!x).slice(0, 100);
+    let includes = clean(options.includePaths);
+    const excludes = clean(options.excludePaths);
+    if (includes.length === 0 && options.restrictToSubpath !== false) {
+        try {
+            const path = new URL(startUrl).pathname.replace(/\/+$/, "");
+            if (path && path !== "")
+                includes = [path];
+        }
+        catch {
+            // ignore invalid start URL
+        }
+    }
+    const inc = includes.map(toRegExp);
+    const exc = excludes.map(toRegExp);
+    let startKey = "";
+    try {
+        const u = new URL(startUrl);
+        startKey = (u.pathname.replace(/\/+$/, "") || "/") + u.search;
+    }
+    catch {
+        // ignore
+    }
+    return {
+        includes,
+        excludes,
+        signature: `inc=${includes.join(",")}|exc=${excludes.join(",")}`,
+        allows(url) {
+            let key;
+            try {
+                const u = new URL(url);
+                key = (u.pathname || "/") + u.search;
+            }
+            catch {
+                return false;
+            }
+            // The start URL itself is always allowed.
+            if ((key.replace(/\/+$/, "") || "/") === startKey)
+                return true;
+            if (exc.some((r) => r.test(key)))
+                return false;
+            if (inc.length && !inc.some((r) => r.test(key)))
+                return false;
+            return true;
+        },
+    };
+}

package/dist/core/engine/safe-url.d.ts

@@ -0,0 +1,23 @@
+export declare class UnsafeUrlError extends Error {
+    constructor(message?: string);
+}
+export declare function isPrivateAddress(ip: string): boolean;
+export type HostResolver = (host: string) => Promise<string[]>;
+/**
+ * Throws `UnsafeUrlError` unless `raw` is an http(s) URL whose host resolves
+ * only to public addresses. Call it before every request and every redirect hop.
+ */
+export declare function assertSafeUrl(raw: string, resolve?: HostResolver): Promise<URL>;
+export declare const MAX_REDIRECTS = 8;
+export interface HopResponse {
+    status: number;
+    headers: Record<string, string>;
+}
+/**
+ * Runs `send` for `url` and follows redirects manually, validating every hop
+ * with `assertSafeUrl`. `send` must NOT follow redirects itself.
+ */
+export declare function followSafely<T extends HopResponse>(url: string, send: (hopUrl: string) => Promise<T>, resolve?: HostResolver): Promise<{
+    res: T;
+    finalUrl: string;
+}>;

package/dist/core/engine/safe-url.js

@@ -0,0 +1,92 @@
+import { isIP } from "node:net";
+import { lookup } from "node:dns/promises";
+export class UnsafeUrlError extends Error {
+    constructor(message = "That address isn't publicly reachable") {
+        super(message);
+        this.name = "UnsafeUrlError";
+    }
+}
+export function isPrivateAddress(ip) {
+    const v = isIP(ip);
+    if (v === 4) {
+        const [a, b] = ip.split(".").map(Number);
+        return (a === 0 ||
+            a === 10 ||
+            a === 127 ||
+            (a === 100 && b >= 64 && b <= 127) ||
+            (a === 169 && b === 254) ||
+            (a === 172 && b >= 16 && b <= 31) ||
+            (a === 192 && b === 168) ||
+            (a === 198 && (b === 18 || b === 19)) ||
+            a >= 224);
+    }
+    if (v === 6) {
+        const s = ip.toLowerCase();
+        if (s.startsWith("::ffff:"))
+            return isPrivateAddress(s.slice(7));
+        return (s === "::" ||
+            s === "::1" ||
+            s.startsWith("fc") ||
+            s.startsWith("fd") ||
+            s.startsWith("fe8") ||
+            s.startsWith("fe9") ||
+            s.startsWith("fea") ||
+            s.startsWith("feb"));
+    }
+    return true;
+}
+const defaultResolve = async (h) => (await lookup(h, { all: true })).map((r) => r.address);
+/** Local development escape hatch (never set in production). */
+function allowPrivate() {
+    return process.env.WEBCRAWL_ALLOW_PRIVATE_URLS === "true";
+}
+/**
+ * Throws `UnsafeUrlError` unless `raw` is an http(s) URL whose host resolves
+ * only to public addresses. Call it before every request and every redirect hop.
+ */
+export async function assertSafeUrl(raw, resolve = defaultResolve) {
+    let u;
+    try {
+        u = new URL(raw);
+    }
+    catch {
+        throw new UnsafeUrlError("Invalid URL");
+    }
+    if (u.protocol !== "http:" && u.protocol !== "https:") {
+        throw new UnsafeUrlError("Only http and https addresses are supported");
+    }
+    if (allowPrivate())
+        return u;
+    const host = u.hostname.replace(/^\[|\]$/g, "").toLowerCase();
+    if (host === "localhost" ||
+        host.endsWith(".localhost") ||
+        host.endsWith(".local") ||
+        host.endsWith(".internal")) {
+        throw new UnsafeUrlError();
+    }
+    const addrs = isIP(host) ? [host] : await resolve(host).catch(() => []);
+    if (addrs.length === 0)
+        throw new UnsafeUrlError("We couldn't find that website");
+    if (addrs.some(isPrivateAddress))
+        throw new UnsafeUrlError();
+    return u;
+}
+export const MAX_REDIRECTS = 8;
+/**
+ * Runs `send` for `url` and follows redirects manually, validating every hop
+ * with `assertSafeUrl`. `send` must NOT follow redirects itself.
+ */
+export async function followSafely(url, send, resolve) {
+    let current = url;
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+        await assertSafeUrl(current, resolve);
+        const res = await send(current);
+        const loc = res.headers["location"];
+        if (res.status >= 300 && res.status < 400 && loc) {
+            current = new URL(loc, current).toString();
+            continue;
+        }
+        return { res, finalUrl: current };
+    }
+    throw new UnsafeUrlError("Too many redirects");
+}

package/dist/core/engine/stealth.d.ts

@@ -0,0 +1,79 @@
+import { type RemoteProvider } from "../interact/remote.js";
+export interface StealthDoc {
+    url: string;
+    ok: boolean;
+    status?: number | null;
+    contentType?: string;
+    body?: string;
+    finalUrl?: string;
+    title?: string;
+    blocked?: boolean;
+    error?: string;
+}
+export interface StealthResult {
+    /** True when the entry page passed the bot check. */
+    solved: boolean;
+    entry: StealthDoc | null;
+    docs: StealthDoc[];
+    durationMs: number;
+    error?: string;
+    /** Set when a remote provider (e.g. Browser Use) hosted the browser. */
+    provider?: RemoteProvider;
+    /** Provider charge for the session in USD, when it reported one. */
+    providerCostUsd?: number;
+}
+export interface StealthItem {
+    url: string;
+    /** Read as a raw document (robots.txt, sitemap XML) instead of rendering it. */
+    raw?: boolean;
+}
+/**
+ * True when a real browser can run here: our own (sandbox in prod, local
+ * Playwright in dev) or a remote provider. A remote browser still needs the
+ * sandbox or local Playwright to run the step script against it.
+ */
+export declare function stealthAvailable(): Promise<boolean>;
+/** True when this fetched document is a bot check, not the real content. */
+export declare function isBlockedResponse(status: number | null | undefined, body: string): boolean;
+/**
+ * Opens `entryUrl` in a real browser, waits for the bot check to clear, then
+ * reads `items` in order through that session. One browser per call.
+ */
+export declare function stealthFetch(entryUrl: string, items: StealthItem[], opts?: {
+    budgetMs?: number;
+    solveMs?: number;
+    followSitemaps?: number;
+    /** Allow the paid remote provider. Off for unbilled (anonymous) runs. Default true. */
+    remote?: boolean;
+}): Promise<StealthResult>;
+/** Extracts `<loc>` entries from a sitemap or sitemap index. */
+export declare function parseSitemapLocs(xml: string): {
+    pages: string[];
+    sitemaps: string[];
+};
+/** Sitemap URLs declared in robots.txt. */
+export declare function sitemapsFromRobots(robots: string): string[];
+/**
+ * Finds a blocked site's pages through one stealth session: robots.txt and the
+ * usual sitemap paths, following robots-declared and child sitemaps in the same
+ * session. Also returns the rendered entry page so its links count when there
+ * is no sitemap.
+ */
+export declare function stealthDiscover(entryUrl: string, opts?: {
+    budgetMs?: number;
+    sitemapUrls?: string[];
+    remote?: boolean;
+}): Promise<{
+    solved: boolean;
+    sitemapPages: string[];
+    sitemapsRead: string[];
+    entry: StealthDoc | null;
+    durationMs: number;
+    error?: string;
+    providerCostUsd?: number;
+}>;
+/** Page URLs from every readable sitemap among `docs` (robots.txt and blocked docs are skipped). */
+export declare function collectSitemapPages(docs: StealthDoc[]): {
+    pages: string[];
+    sitemapsRead: string[];
+};

package/dist/core/engine/stealth.js

@@ -0,0 +1,155 @@
+/**
+ * Stealth mode: read pages a bot check refuses to plain HTTP clients.
+ *
+ * One real Chromium session (the interact browser: Vercel Sandbox in prod,
+ * local Playwright in dev) opens the entry page, waits until the check clears,
+ * then reads every requested URL with that cleared session. Raw documents
+ * (robots.txt, sitemaps) are read with fetch() from inside the page so they
+ * carry the same cookies, IP and TLS fingerprint as the browser.
+ *
+ * Never fabricates content: a page still showing a check comes back `blocked`.
+ */
+import { isAntiBotChallenge } from "./fetcher.js";
+import { remoteStealthProvider } from "../interact/remote.js";
+/**
+ * True when a real browser can run here: our own (sandbox in prod, local
+ * Playwright in dev) or a remote provider. A remote browser still needs the
+ * sandbox or local Playwright to run the step script against it.
+ */
+export async function stealthAvailable() {
+    if (process.env.WEBCRAWL_STEALTH === "false")
+        return false;
+    const { interactBrowserAvailable } = await import("../interact/driver.js");
+    return interactBrowserAvailable();
+}
+/** True when this fetched document is a bot check, not the real content. */
+export function isBlockedResponse(status, body) {
+    return isAntiBotChallenge(body ?? "", status ?? 200);
+}
+/**
+ * Opens `entryUrl` in a real browser, waits for the bot check to clear, then
+ * reads `items` in order through that session. One browser per call.
+ */
+export async function stealthFetch(entryUrl, items, opts = {}) {
+    const t0 = Date.now();
+    if (!(await stealthAvailable())) {
+        return { solved: false, entry: null, docs: [], durationMs: 0, error: "Stealth browser not configured" };
+    }
+    const { startBrowser, runBrowserStep, stopBrowser } = await import("../interact/driver.js");
+    const provider = opts.remote === false ? null : remoteStealthProvider();
+    const tag = provider ? { provider } : {};
+    let handle = null;
+    // Filled in by `finally` once the provider reports the session cost.
+    const out = {};
+    try {
+        const budgetMs = opts.budgetMs ?? 90_000;
+        handle = await startBrowser({ timeoutMs: Math.min(5 * 60_000, budgetMs + 90_000), remote: provider });
+        const res = await runBrowserStep(handle, {
+            type: "fetchMany",
+            entry: entryUrl,
+            items: items.slice(0, 40),
+            budgetMs,
+            solveMs: opts.solveMs ?? 20_000,
+            followSitemaps: opts.followSitemaps ?? 0,
+        });
+        if (!res.ok) {
+            out.result = { solved: false, entry: null, docs: [], durationMs: 0, error: res.error || "Stealth browser step failed", ...tag };
+        }
+        else {
+            const value = (res.data?.value ?? {});
+            out.result = {
+                solved: Boolean(value.solved),
+                entry: value.entry ?? null,
+                docs: Array.isArray(value.docs) ? value.docs : [],
+                durationMs: 0,
+                ...tag,
+            };
+        }
+    }
+    catch (err) {
+        out.result = { solved: false, entry: null, docs: [], durationMs: 0, error: err instanceof Error ? err.message : String(err), ...tag };
+    }
+    finally {
+        const cost = handle ? await stopBrowser(handle).catch(() => null) : null;
+        if (out.result && cost)
+            out.result.providerCostUsd = cost.costUsd;
+    }
+    const result = out.result;
+    result.durationMs = Date.now() - t0;
+    if (provider) {
+        console.info("[stealth] remote session", {
+            provider,
+            solved: result.solved,
+            docs: result.docs.length,
+            costUsd: result.providerCostUsd,
+            durationMs: result.durationMs,
+            error: result.error,
+        });
+    }
+    return result;
+}
+/** Extracts `<loc>` entries from a sitemap or sitemap index. */
+export function parseSitemapLocs(xml) {
+    const pages = [];
+    const sitemaps = [];
+    const isIndex = /<sitemapindex[\s>]/i.test(xml);
+    for (const m of xml.matchAll(/<loc>\s*(?:<!\[CDATA\[)?([^<\]]+?)(?:\]\]>)?\s*<\/loc>/gi)) {
+        const loc = m[1]
+            .trim()
+            .replace(/&amp;/g, "&")
+            .replace(/&lt;/g, "<")
+            .replace(/&gt;/g, ">")
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;|&apos;/g, "'");
+        if (isIndex || /\.xml(\.gz)?(\?|$)/i.test(loc))
+            sitemaps.push(loc);
+        else
+            pages.push(loc);
+    }
+    return { pages, sitemaps };
+}
+/** Sitemap URLs declared in robots.txt. */
+export function sitemapsFromRobots(robots) {
+    return Array.from(robots.matchAll(/^\s*Sitemap:\s*(\S+)/gim)).map((m) => m[1].trim());
+}
+/**
+ * Finds a blocked site's pages through one stealth session: robots.txt and the
+ * usual sitemap paths, following robots-declared and child sitemaps in the same
+ * session. Also returns the rendered entry page so its links count when there
+ * is no sitemap.
+ */
+export async function stealthDiscover(entryUrl, opts = {}) {
+    const origin = new URL(entryUrl).origin;
+    const first = await stealthFetch(entryUrl, [
+        ...(opts.sitemapUrls ?? []).map((url) => ({ url, raw: true })),
+        { url: `${origin}/robots.txt`, raw: true },
+        { url: `${origin}/sitemap.xml`, raw: true },
+        { url: `${origin}/sitemap_index.xml`, raw: true },
+        { url: `${origin}/sitemap-index.xml`, raw: true },
+    ], { budgetMs: opts.budgetMs ?? 75_000, followSitemaps: 12, remote: opts.remote });
+    const { pages, sitemapsRead } = collectSitemapPages(first.docs);
+    return {
+        solved: first.solved,
+        sitemapPages: pages,
+        sitemapsRead,
+        entry: first.entry,
+        durationMs: first.durationMs,
+        ...(first.error ? { error: first.error } : {}),
+        ...(first.providerCostUsd !== undefined ? { providerCostUsd: first.providerCostUsd } : {}),
+    };
+}
+/** Page URLs from every readable sitemap among `docs` (robots.txt and blocked docs are skipped). */
+export function collectSitemapPages(docs) {
+    const pages = new Set();
+    const sitemapsRead = [];
+    for (const doc of docs) {
+        if (!doc.ok || !doc.body || doc.blocked || (doc.status ?? 200) >= 400)
+            continue;
+        if (/\/robots\.txt(\?|$)/i.test(doc.url) || !doc.body.includes("<loc>"))
+            continue;
+        sitemapsRead.push(doc.url);
+        for (const u of parseSitemapLocs(doc.body).pages)
+            pages.add(u);
+    }
+    return { pages: Array.from(pages), sitemapsRead };
+}

package/dist/core/engine.d.ts

@@ -0,0 +1 @@
+export * from "./engine/index.js";

package/dist/core/engine.js

@@ -0,0 +1 @@
+export * from "./engine/index.js";

package/dist/core/extract/business.d.ts

@@ -0,0 +1,28 @@
+export interface BusinessListing {
+    name: string;
+    category?: string;
+    description?: string;
+    address?: string;
+    city?: string;
+    state?: string;
+    postcode?: string;
+    country?: string;
+    phone?: string;
+    email?: string;
+    website?: string;
+    rating?: number;
+    reviewCount?: number;
+    priceRange?: string;
+    openingHours?: string | string[];
+    foundedYear?: number;
+    employeeCount?: string;
+    headquarters?: string;
+    latitude?: number;
+    longitude?: number;
+    url: string;
+    sourceDomain: string;
+    extractedVia: "json-state" | "rsc-payload" | "json-ld" | "scaledown-extract-1" | "serp-snippet";
+}
+export declare function extractBusinessListingsFromHtml(html: string, pageUrl: string, countryCode?: string): BusinessListing[];
+export declare function extractBusinessListingFromSerpResult(title: string, description: string, url: string, countryCode?: string): BusinessListing | null;
+export declare function dedupeBusinessListings(businesses: BusinessListing[], maxItems?: number): BusinessListing[];