webcrawl 1.2.4 → 1.2.7

package/dist/core/engine/fetcher.d.ts

@@ -0,0 +1,66 @@
+export interface ScrapeOptions {
+    url: string;
+    formats?: Array<"markdown" | "html" | "links">;
+    retainImages?: "all" | "none";
+    removeSelectors?: string[];
+    render?: boolean;
+    timeout?: number;
+    bypassCache?: boolean;
+    cache?: boolean;
+}
+export interface ScrapeResult {
+    success: boolean;
+    url: string;
+    title: string;
+    markdown: string;
+    html?: string;
+    links?: string[];
+    statusCode: number;
+    contentHash: string;
+    tier: "cache" | "static" | "sandbox" | "cloudflare" | "browser";
+    latencyMs: number;
+    error?: string;
+}
+export declare function getBrowserHeaders(targetUrl: string, extraHeaders?: Record<string, string>): Record<string, string>;
+export declare function setDomainCookie(hostname: string, cookieHeader: string): void;
+export declare function isAntiBotChallenge(html: string, status: number): boolean;
+/**
+ * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)
+ * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.
+ * Falls back to standard `fetch` if `impit` is unavailable on the current platform.
+ * Every redirect hop is checked against the SSRF guard.
+ */
+export declare function fetchWithTlsImpersonation(url: string, timeoutMs?: number): Promise<{
+    status: number;
+    html: string;
+    headers: Record<string, string>;
+}>;
+export interface FetchedResource {
+    status: number;
+    headers: Record<string, string>;
+    /** Declared content type without parameters, lower-cased ("" when missing). */
+    contentType: string;
+    body: Uint8Array;
+    /** Final URL after redirects when the transport reports it. */
+    url: string;
+}
+export declare class ResourceTooLargeError extends Error {
+    readonly bytes: number;
+    readonly maxBytes: number;
+    constructor(bytes: number, maxBytes: number);
+}
+/** Largest download accepted for any file (the per-type caps in src/parse/limits.ts are tighter). */
+export declare const MAX_RESOURCE_BYTES: number;
+/**
+ * Binary-safe GET. Same TLS impersonation as `fetchWithTlsImpersonation`, but keeps
+ * the body as bytes so PDFs, Office files, images and media survive intact.
+ */
+export declare function fetchResource(url: string, options?: {
+    timeoutMs?: number;
+    maxBytes?: number;
+}): Promise<FetchedResource>;
+/**
+ * Detects whether an HTML response is an empty SPA shell or a Cloudflare/WAF challenge page
+ * that requires headless browser / Vercel Sandbox escalation.
+ */
+export declare function needsBrowserEscalation(status: number, html: string, markdown: string): boolean;

package/dist/core/engine/fetcher.js

@@ -0,0 +1,185 @@
+import { followSafely, UnsafeUrlError } from "./safe-url.js";
+const domainCookies = new Map();
+const BROWSER_USER_AGENTS = [
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+];
+export function getBrowserHeaders(targetUrl, extraHeaders) {
+    const ua = BROWSER_USER_AGENTS[Math.abs(hashCode(targetUrl)) % BROWSER_USER_AGENTS.length];
+    let hostCookie;
+    try {
+        const parsed = new URL(targetUrl);
+        hostCookie = domainCookies.get(parsed.hostname);
+    }
+    catch {
+        // ignore invalid URL
+    }
+    return {
+        "User-Agent": ua,
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Accept-Encoding": "gzip, deflate, br",
+        "Cache-Control": "no-cache",
+        Pragma: "no-cache",
+        "Sec-Ch-Ua": '"Chromium";v="136", "Google Chrome";v="136", "Not.A/Brand";v="99"',
+        "Sec-Ch-Ua-Mobile": "?0",
+        "Sec-Ch-Ua-Platform": '"macOS"',
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
+        "Upgrade-Insecure-Requests": "1",
+        ...(hostCookie ? { Cookie: hostCookie } : {}),
+        ...extraHeaders,
+    };
+}
+export function setDomainCookie(hostname, cookieHeader) {
+    if (hostname && cookieHeader) {
+        domainCookies.set(hostname, cookieHeader);
+    }
+}
+function hashCode(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        hash = (hash << 5) - hash + str.charCodeAt(i);
+        hash |= 0;
+    }
+    return hash;
+}
+export function isAntiBotChallenge(html, status) {
+    if (status === 403 || status === 429 || status === 503) {
+        return true;
+    }
+    const lower = html.slice(0, 6000).toLowerCase();
+    return (lower.includes("cf-browser-verification") ||
+        lower.includes("challenge-platform") ||
+        lower.includes("just a moment...") ||
+        lower.includes("vercel security checkpoint") ||
+        lower.includes("_vercel/security") ||
+        lower.includes("enable javascript and cookies to continue") ||
+        lower.includes("_cf_chl_opt"));
+}
+let sharedImpit = null;
+/** One request, redirects NOT followed (callers validate each hop via followSafely). */
+async function rawGet(url, reqHeaders, timeoutMs, maxBytes) {
+    const read = async (status, headers, getBuf) => {
+        const isRedirect = status >= 300 && status < 400 && !!headers["location"];
+        if (!isRedirect && maxBytes !== undefined)
+            checkDeclaredLength(headers, maxBytes);
+        return { status, headers, body: isRedirect ? new ArrayBuffer(0) : await getBuf() };
+    };
+    try {
+        if (!sharedImpit) {
+            const impitMod = await import("impit");
+            sharedImpit = new impitMod.Impit({
+                browser: "chrome",
+                ignoreTlsErrors: true,
+                timeout: timeoutMs,
+            });
+        }
+        const res = await sharedImpit.fetch(url, {
+            method: "GET",
+            headers: reqHeaders,
+            timeout: timeoutMs,
+            redirect: "manual",
+        });
+        return await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
+    }
+    catch (err) {
+        if (err instanceof ResourceTooLargeError || err instanceof UnsafeUrlError)
+            throw err;
+        const res = await fetch(url, {
+            headers: reqHeaders,
+            redirect: "manual",
+            signal: AbortSignal.timeout(timeoutMs),
+        });
+        return await read(res.status, headersToObject(res.headers), () => res.arrayBuffer());
+    }
+}
+/**
+ * Fetches a URL using `impit` (BoringSSL Chrome 136 TLS JA3/JA4 + HTTP/2 fingerprint impersonation)
+ * so Cloudflare Bot Management and Vercel Security Checkpoint treat requests as real Chrome.
+ * Falls back to standard `fetch` if `impit` is unavailable on the current platform.
+ * Every redirect hop is checked against the SSRF guard.
+ */
+export async function fetchWithTlsImpersonation(url, timeoutMs = 12_000) {
+    const { res, finalUrl } = await followSafely(url, (hop) => rawGet(hop, getBrowserHeaders(hop), timeoutMs));
+    if (res.headers["set-cookie"]) {
+        try {
+            setDomainCookie(new URL(finalUrl).hostname, res.headers["set-cookie"].split(";")[0]);
+        }
+        catch {
+            // ignore
+        }
+    }
+    return {
+        status: res.status,
+        html: new TextDecoder().decode(res.body),
+        headers: res.headers,
+    };
+}
+export class ResourceTooLargeError extends Error {
+    bytes;
+    maxBytes;
+    constructor(bytes, maxBytes) {
+        super(`File is ${(bytes / 1e6).toFixed(1)} MB; the limit is ${(maxBytes / 1e6).toFixed(0)} MB`);
+        this.bytes = bytes;
+        this.maxBytes = maxBytes;
+        this.name = "ResourceTooLargeError";
+    }
+}
+/** Largest download accepted for any file (the per-type caps in src/parse/limits.ts are tighter). */
+export const MAX_RESOURCE_BYTES = 100 * 1024 * 1024;
+function headersToObject(headers) {
+    const out = {};
+    const h = headers;
+    if (h && typeof h.forEach === "function") {
+        h.forEach((v, k) => {
+            out[String(k).toLowerCase()] = String(v);
+        });
+    }
+    return out;
+}
+function checkDeclaredLength(headers, maxBytes) {
+    const declared = Number(headers["content-length"]);
+    if (Number.isFinite(declared) && declared > maxBytes) {
+        throw new ResourceTooLargeError(declared, maxBytes);
+    }
+}
+/**
+ * Binary-safe GET. Same TLS impersonation as `fetchWithTlsImpersonation`, but keeps
+ * the body as bytes so PDFs, Office files, images and media survive intact.
+ */
+export async function fetchResource(url, options = {}) {
+    const timeoutMs = options.timeoutMs ?? 30_000;
+    const maxBytes = options.maxBytes ?? MAX_RESOURCE_BYTES;
+    const { res, finalUrl } = await followSafely(url, (hop) => rawGet(hop, { ...getBrowserHeaders(hop), Accept: "*/*" }, timeoutMs, maxBytes));
+    if (res.body.byteLength > maxBytes)
+        throw new ResourceTooLargeError(res.body.byteLength, maxBytes);
+    return {
+        status: res.status,
+        headers: res.headers,
+        contentType: (res.headers["content-type"] || "").split(";")[0].trim().toLowerCase(),
+        body: new Uint8Array(res.body),
+        url: finalUrl,
+    };
+}
+/**
+ * Detects whether an HTML response is an empty SPA shell or a Cloudflare/WAF challenge page
+ * that requires headless browser / Vercel Sandbox escalation.
+ */
+export function needsBrowserEscalation(status, html, markdown) {
+    if (isAntiBotChallenge(html, status)) {
+        return true;
+    }
+    const lower = html.slice(0, 6000).toLowerCase();
+    // Empty SPA shell check (<div id="root"></div> or <div id="app"></div> with almost no text)
+    if (markdown.length < 120 &&
+        (lower.includes('id="root"') ||
+            lower.includes('id="app"') ||
+            lower.includes('id="__next"'))) {
+        return true;
+    }
+    return false;
+}

package/dist/core/engine/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./cache.js";
+export * from "./markdown.js";
+export * from "./fetcher.js";
+export * from "./crawler.js";

package/dist/core/engine/index.js

@@ -0,0 +1,4 @@
+export * from "./cache.js";
+export * from "./markdown.js";
+export * from "./fetcher.js";
+export * from "./crawler.js";

package/dist/core/engine/markdown.d.ts

@@ -0,0 +1,33 @@
+export declare function decodeHtmlEntities(text: string): string;
+export interface MarkdownOptions {
+    retainImages?: "all" | "none";
+    /** Default true: prefer <main>/<article> and drop header/footer/nav/aside. */
+    onlyMainContent?: boolean;
+    /** Simple selectors (`tag`, `.class`, `#id`, `tag.class`). Keep only matching elements. */
+    includeTags?: string[];
+    /** Same selector syntax. Matching elements are removed. */
+    excludeTags?: string[];
+}
+export declare function removeElements(html: string, rawSelectors: string[]): string;
+export declare function keepElements(html: string, rawSelectors: string[]): string;
+/**
+ * Ultra-fast zero-DOM streaming regex/state-machine HTML-to-Markdown converter.
+ * Processes a 200KB HTML document in ~1.2ms (vs 45-120ms for JSDOM + Turndown).
+ */
+export declare function fastHtmlToMarkdown(html: string, baseUrl: string, options?: MarkdownOptions): {
+    title: string;
+    markdown: string;
+    links: string[];
+};
+/**
+ * Fast anchor href extractor for high-speed multi-hop BFS crawling.
+ */
+export declare function extractLinksFromHtml(html: string, pageUrl: string): string[];
+/**
+ * Discovers all internal URLs for a website:
+ * 1. Checks 24-hour cache first (unless `bypassCache: true`).
+ * 2. Probes `robots.txt`, `/sitemap.xml`, `/sitemap_index.xml`, and subpath `sitemap.xml` in parallel.
+ * 3. When `sitemap.xml` is missing (or incomplete), performs a full multi-hop concurrent BFS crawl
+ *    across the entire website (with automatic `@vercel/sandbox` / browser escalation if the entry page is an SPA),
+ *    and caches both the discovered site map and scraped pages for 24 hours.
+ */

package/dist/core/engine/markdown.js

@@ -0,0 +1,334 @@
+const HTML_ENTITIES = {
+    "&": "&",
+    "&lt;": "<",
+    "&gt;": ">",
+    "&quot;": '"',
+    "&#39;": "'",
+    "&apos;": "'",
+    "&nbsp;": " ",
+    "&ndash;": "–",
+    "&mdash;": "\u2014",
+    "&copy;": "©",
+    "&reg;": "®",
+    "&trade;": "™",
+    "&hellip;": "…",
+};
+export function decodeHtmlEntities(text) {
+    return text.replace(/&[a-zA-Z0-9#]+;/g, (entity) => {
+        if (HTML_ENTITIES[entity]) {
+            return HTML_ENTITIES[entity];
+        }
+        if (entity.startsWith("&#x") || entity.startsWith("&#X")) {
+            const code = parseInt(entity.slice(3, -1), 16);
+            return !Number.isNaN(code) ? String.fromCodePoint(code) : entity;
+        }
+        if (entity.startsWith("&#")) {
+            const code = parseInt(entity.slice(2, -1), 10);
+            return !Number.isNaN(code) ? String.fromCodePoint(code) : entity;
+        }
+        return entity;
+    });
+}
+const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]);
+function parseSelector(raw) {
+    const m = raw.trim().match(/^([a-z][a-z0-9-]*)?((?:[.#][\w-]+)*)$/i);
+    if (!m || (!m[1] && !m[2]))
+        return null;
+    const sel = { tag: m[1]?.toLowerCase(), classes: [] };
+    for (const part of m[2].match(/[.#][\w-]+/g) ?? []) {
+        if (part[0] === "#")
+            sel.id = part.slice(1);
+        else
+            sel.classes.push(part.slice(1));
+    }
+    return sel;
+}
+function openTagMatches(tagName, attrs, sel) {
+    if (sel.tag && sel.tag !== tagName)
+        return false;
+    if (sel.id) {
+        const id = attrs.match(/\bid=["']([^"']*)["']/i)?.[1];
+        if (id !== sel.id)
+            return false;
+    }
+    if (sel.classes.length) {
+        const cls = (attrs.match(/\bclass=["']([^"']*)["']/i)?.[1] ?? "").split(/\s+/);
+        if (!sel.classes.every((c) => cls.includes(c)))
+            return false;
+    }
+    return true;
+}
+/** Finds balanced `[start, end)` ranges of elements matching any selector (outermost only). */
+function findElementRanges(html, selectors) {
+    const ranges = [];
+    const openRe = /<([a-z][a-z0-9-]*)((?:[^>"']|"[^"]*"|'[^']*')*)>/gi;
+    let m;
+    let skipUntil = 0;
+    while ((m = openRe.exec(html)) !== null) {
+        if (m.index < skipUntil)
+            continue;
+        const tag = m[1].toLowerCase();
+        if (!selectors.some((sel) => openTagMatches(tag, m[2], sel)))
+            continue;
+        if (VOID_TAGS.has(tag) || m[2].trimEnd().endsWith("/")) {
+            ranges.push([m.index, m.index + m[0].length]);
+            skipUntil = m.index + m[0].length;
+            continue;
+        }
+        const re = new RegExp(`<(/?)${tag}(?=[\\s>/])(?:[^>"']|"[^"]*"|'[^']*')*>`, "gi");
+        re.lastIndex = m.index + m[0].length;
+        let depth = 1;
+        let end = html.length;
+        let t;
+        while ((t = re.exec(html)) !== null) {
+            depth += t[1] ? -1 : 1;
+            if (depth === 0) {
+                end = t.index + t[0].length;
+                break;
+            }
+        }
+        ranges.push([m.index, end]);
+        skipUntil = end;
+    }
+    return ranges;
+}
+export function removeElements(html, rawSelectors) {
+    const sels = rawSelectors.map(parseSelector).filter((x) => !!x);
+    if (!sels.length)
+        return html;
+    const ranges = findElementRanges(html, sels);
+    let out = "";
+    let pos = 0;
+    for (const [a, b] of ranges) {
+        out += html.slice(pos, a);
+        pos = b;
+    }
+    return out + html.slice(pos);
+}
+export function keepElements(html, rawSelectors) {
+    const sels = rawSelectors.map(parseSelector).filter((x) => !!x);
+    if (!sels.length)
+        return html;
+    const ranges = findElementRanges(html, sels);
+    if (!ranges.length)
+        return html;
+    return ranges.map(([a, b]) => html.slice(a, b)).join("\n");
+}
+/**
+ * Ultra-fast zero-DOM streaming regex/state-machine HTML-to-Markdown converter.
+ * Processes a 200KB HTML document in ~1.2ms (vs 45-120ms for JSDOM + Turndown).
+ */
+export function fastHtmlToMarkdown(html, baseUrl, options = {}) {
+    // 1. Extract <title> or og:title, plus meta description
+    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i) ||
+        html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i) ||
+        html.match(/<h1(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/h1>/i);
+    const rawTitle = titleMatch ? stripTags(titleMatch[1]).trim() : "";
+    const title = decodeHtmlEntities(rawTitle) || baseUrl;
+    const metaDescMatch = html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']/i) ||
+        html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']description["']/i) ||
+        html.match(/<meta[^>]+property=["']og:description["'][^>]+content=["']([^"']+)["']/i);
+    const metaDescription = metaDescMatch
+        ? decodeHtmlEntities(metaDescMatch[1].trim())
+        : "";
+    // 2. Extract all <a href="..."> links before stripping nav/header/footer
+    const linksSet = new Set();
+    const linkRegex = /<a\s+[^>]*href=["']([^"'#]+)["'][^>]*>/gi;
+    let linkMatch;
+    while ((linkMatch = linkRegex.exec(html)) !== null) {
+        const href = linkMatch[1].trim();
+        if (!href ||
+            href.startsWith("javascript:") ||
+            href.startsWith("mailto:") ||
+            href.startsWith("tel:")) {
+            continue;
+        }
+        try {
+            const resolved = new URL(href, baseUrl);
+            resolved.hash = "";
+            if (resolved.protocol === "http:" || resolved.protocol === "https:") {
+                linksSet.add(resolved.toString());
+            }
+        }
+        catch {
+            // ignore malformed URLs
+        }
+    }
+    // 3. Focus on <main>/<article> (unless onlyMainContent is false), else <body>
+    const onlyMain = options.onlyMainContent !== false;
+    let bodyHtml = html;
+    if (options.includeTags?.length) {
+        bodyHtml = keepElements(html, options.includeTags);
+    }
+    else {
+        const mainMatch = onlyMain
+            ? html.match(/<main[^>]*>([\s\S]*?)<\/main>/i) ||
+                html.match(/<article[^>]*>([\s\S]*?)<\/article>/i)
+            : null;
+        if (mainMatch && mainMatch[1].length > 400) {
+            bodyHtml = mainMatch[1];
+        }
+        else {
+            const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+            if (bodyMatch) {
+                bodyHtml = bodyMatch[1];
+            }
+        }
+    }
+    if (options.excludeTags?.length) {
+        bodyHtml = removeElements(bodyHtml, options.excludeTags);
+    }
+    // 4. Strip noise tags: script, style, noscript, svg, iframe, dialog, form, aria-hidden (+ nav chrome when onlyMainContent)
+    let cleaned = bodyHtml
+        .replace(/<!--[\s\S]*?-->/g, "")
+        .replace(/<(script|style|noscript|svg|canvas|template|iframe|dialog|form)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "");
+    if (onlyMain) {
+        cleaned = cleaned.replace(/<(header|footer|nav|aside)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "");
+    }
+    cleaned = cleaned.replace(/<([a-z0-9]+)\b(?:[^>"']|"[^"]*"|'[^']*')*\baria-hidden=["']true["'](?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "");
+    // 5. Preserve <pre><code> blocks via placeholders so whitespace isn't collapsed
+    const codeBlocks = [];
+    cleaned = cleaned.replace(/<pre[^>]*>\s*(?:<code[^>]*>)?([\s\S]*?)(?:<\/code>)?\s*<\/pre>/gi, (_full, codeContent) => {
+        const idx = codeBlocks.length;
+        const decoded = decodeHtmlEntities(stripTags(codeContent)).trim();
+        codeBlocks.push(`\n\`\`\`\n${decoded}\n\`\`\`\n`);
+        return `__WEBCRAWL_CODE_BLOCK_${idx}__`;
+    });
+    // 6. Images
+    if (options.retainImages === "none") {
+        cleaned = cleaned.replace(/<img[^>]*>/gi, "");
+    }
+    else {
+        cleaned = cleaned.replace(/<img\s+[^>]*>/gi, (imgTag) => {
+            const srcMatch = imgTag.match(/src=["']([^"']+)["']/i);
+            if (!srcMatch) {
+                return "";
+            }
+            const altMatch = imgTag.match(/alt=["']([^"']*)["']/i);
+            const alt = altMatch ? decodeHtmlEntities(altMatch[1].trim()) : "";
+            try {
+                const absSrc = new URL(srcMatch[1], baseUrl).toString();
+                return `![${alt}](${absSrc})`;
+            }
+            catch {
+                return "";
+            }
+        });
+    }
+    // 7. Links: <a href="...">text</a> -> [text](url)
+    cleaned = cleaned.replace(/<a\s+[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_full, href, inner) => {
+        const linkText = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        if (!linkText) {
+            return "";
+        }
+        if (href.startsWith("javascript:") || href.startsWith("#")) {
+            return linkText;
+        }
+        try {
+            const absUrl = new URL(href, baseUrl).toString();
+            return `[${linkText}](${absUrl})`;
+        }
+        catch {
+            return linkText;
+        }
+    });
+    // 8. Headings h1-h6 (handle > inside quoted attributes like Tailwind class="[&>*]:mb-0")
+    cleaned = cleaned.replace(/<h([1-6])(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/h\1>/gi, (_f, level, inner) => {
+        const hashes = "#".repeat(Number(level));
+        const text = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        return text ? `\n\n${hashes} ${text}\n\n` : "";
+    });
+    // 9. Inline formatting
+    cleaned = cleaned
+        .replace(/<(strong|b)(?:\s(?:[^>"']|"[^"]*"|'[^']*')*)?>([\s\S]*?)<\/\1>/gi, "**$2**")
+        .replace(/<(em|i)(?:\s(?:[^>"']|"[^"]*"|'[^']*')*)?>([\s\S]*?)<\/\1>/gi, "*$2*")
+        .replace(/<code(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/code>/gi, (_f, inner) => {
+        const text = decodeHtmlEntities(stripTags(inner)).trim();
+        return text ? `\`${text}\`` : "";
+    });
+    // 10. Lists and paragraphs and tables
+    cleaned = cleaned
+        .replace(/<li(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/li>/gi, (_f, inner) => {
+        const item = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        return item ? `\n- ${item}` : "";
+    })
+        .replace(/<tr(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/tr>/gi, (_f, row) => {
+        const cells = [];
+        row.replace(/<t[dh](?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/t[dh]>/gi, (_c, cell) => {
+            cells.push(decodeHtmlEntities(stripTags(cell)).replace(/\s+/g, " ").trim());
+            return "";
+        });
+        return cells.length > 0 ? `\n| ${cells.join(" | ")} |` : "";
+    })
+        .replace(/<blockquote(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/blockquote>/gi, (_f, inner) => {
+        const text = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        return text ? `\n\n> ${text}\n\n` : "";
+    })
+        .replace(/<br\s*\/?>/gi, "\n")
+        .replace(/<\/(p|div|section|article|ul|ol|table|dl)>/gi, "\n\n")
+        .replace(/<hr\s*\/?>/gi, "\n\n---\n\n");
+    // 11. Strip remaining HTML tags and decode entities
+    cleaned = decodeHtmlEntities(stripTags(cleaned));
+    // 12. Restore code blocks
+    cleaned = cleaned.replace(/__WEBCRAWL_CODE_BLOCK_(\d+)__/g, (_f, idx) => codeBlocks[Number(idx)] || "");
+    // 13. Normalize whitespace and prepend meta description if not already in body
+    let markdown = cleaned
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    if (metaDescription && !markdown.includes(metaDescription.slice(0, 40))) {
+        markdown = `> ${metaDescription}\n\n${markdown}`;
+    }
+    return {
+        title,
+        markdown,
+        links: Array.from(linksSet),
+    };
+}
+function stripTags(html) {
+    return html.replace(/<(?:[^>"']|"[^"]*"|'[^']*')+>/g, " ");
+}
+/**
+ * Fast anchor href extractor for high-speed multi-hop BFS crawling.
+ */
+export function extractLinksFromHtml(html, pageUrl) {
+    const links = [];
+    const hrefRegex = /<a\s+[^>]*href=["']([^"'#]+)["'][^>]*>/gi;
+    let match;
+    while ((match = hrefRegex.exec(html)) !== null) {
+        const rawHref = match[1].trim();
+        if (!rawHref ||
+            rawHref.startsWith("javascript:") ||
+            rawHref.startsWith("mailto:") ||
+            rawHref.startsWith("tel:")) {
+            continue;
+        }
+        try {
+            const resolved = new URL(decodeHtmlEntities(rawHref), pageUrl);
+            if (resolved.protocol === "http:" || resolved.protocol === "https:") {
+                resolved.hash = "";
+                links.push(resolved.toString());
+            }
+        }
+        catch {
+            // ignore invalid URL
+        }
+    }
+    return links;
+}
+/**
+ * Discovers all internal URLs for a website:
+ * 1. Checks 24-hour cache first (unless `bypassCache: true`).
+ * 2. Probes `robots.txt`, `/sitemap.xml`, `/sitemap_index.xml`, and subpath `sitemap.xml` in parallel.
+ * 3. When `sitemap.xml` is missing (or incomplete), performs a full multi-hop concurrent BFS crawl
+ *    across the entire website (with automatic `@vercel/sandbox` / browser escalation if the entry page is an SPA),
+ *    and caches both the discovered site map and scraped pages for 24 hours.
+ */

package/dist/core/engine/path-scope.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Path scoping for map/crawl: restrict discovered URLs to a subpath and/or
+ * include/exclude path patterns. Patterns match the URL path (+ query):
+ *   "/docs"        prefix at a segment boundary ("/docs", "/docs/a", not "/docs-old")
+ *   "/blog/*"      `*` matches any characters
+ *   "/api/*\/v1$"  trailing `$` anchors the end
+ */
+export interface PathScopeOptions {
+    includePaths?: string[];
+    excludePaths?: string[];
+    /** Default true: when the start URL has a path and no includePaths are given, stay under it. */
+    restrictToSubpath?: boolean;
+}
+export interface PathScope {
+    allows(url: string): boolean;
+    /** Stable string for cache keys. */
+    signature: string;
+    includes: string[];
+    excludes: string[];
+}
+export declare function buildPathScope(startUrl: string, options?: PathScopeOptions): PathScope;