webcrawl 1.2.4 → 1.2.7

package/dist/core/engine/crawl-policy.js

@@ -0,0 +1,134 @@
+/**
+ * Crawl politeness helpers: robots.txt rules, adaptive concurrency (AIMD) and retries.
+ * Pure and dependency-free so they are easy to test.
+ */
+const ROBOT_TOKEN = "webcrawl";
+function patternToRegExp(path) {
+    const anchored = path.endsWith("$");
+    const body = (anchored ? path.slice(0, -1) : path)
+        .replace(/[.+?^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, ".*");
+    return new RegExp(`^${body}${anchored ? "$" : ""}`);
+}
+/** Parses robots.txt. Our own group wins over `*`; longest matching rule wins; Allow beats Disallow on ties. */
+export function parseRobots(text) {
+    const groups = [];
+    let current = null;
+    let lastWasAgent = false;
+    for (const rawLine of (text ?? "").split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, "").trim();
+        if (!line)
+            continue;
+        const idx = line.indexOf(":");
+        if (idx < 0)
+            continue;
+        const key = line.slice(0, idx).trim().toLowerCase();
+        const value = line.slice(idx + 1).trim();
+        if (key === "user-agent") {
+            if (!current || !lastWasAgent) {
+                current = { agents: [], rules: [], delay: 0 };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+            lastWasAgent = true;
+            continue;
+        }
+        lastWasAgent = false;
+        if (!current)
+            continue;
+        if (key === "disallow" || key === "allow") {
+            if (!value && key === "disallow")
+                continue; // empty Disallow = allow all
+            current.rules.push({
+                allow: key === "allow",
+                pattern: patternToRegExp(value),
+                length: value.length,
+            });
+        }
+        else if (key === "crawl-delay") {
+            const n = Number(value);
+            if (Number.isFinite(n) && n > 0)
+                current.delay = Math.min(n, 10) * 1000;
+        }
+    }
+    const mine = groups.filter((g) => g.agents.some((a) => a !== "*" && ROBOT_TOKEN.includes(a)));
+    const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes("*"));
+    const rules = chosen.flatMap((g) => g.rules);
+    const crawlDelayMs = Math.max(0, ...chosen.map((g) => g.delay));
+    return {
+        crawlDelayMs,
+        allows(url) {
+            let path;
+            try {
+                const u = new URL(url);
+                path = u.pathname + u.search;
+            }
+            catch {
+                return true;
+            }
+            let best = null;
+            for (const r of rules) {
+                if (!r.pattern.test(path))
+                    continue;
+                if (!best || r.length > best.length || (r.length === best.length && r.allow))
+                    best = r;
+            }
+            return best ? best.allow : true;
+        },
+    };
+}
+/** Additive-increase / multiplicative-decrease concurrency controller. */
+export class AdaptiveConcurrency {
+    min;
+    max;
+    target;
+    constructor(min = 1, max = 16, initial = 4) {
+        this.min = min;
+        this.max = max;
+        this.target = Math.min(Math.max(initial, min), max);
+    }
+    get current() {
+        return this.target;
+    }
+    /** Feed the outcome of a finished wave. */
+    record(outcome) {
+        const total = outcome.ok + outcome.failed;
+        if (outcome.throttled || (total > 0 && outcome.failed / total > 0.3)) {
+            this.target = Math.max(this.min, Math.floor(this.target / 2));
+        }
+        else if (outcome.ok > 0) {
+            this.target = Math.min(this.max, this.target + 1);
+        }
+        return this.target;
+    }
+}
+export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export function backoffDelayMs(attempt, baseMs = 250, capMs = 4000, rand = Math.random) {
+    const exp = Math.min(capMs, baseMs * 2 ** attempt);
+    return Math.round(exp / 2 + rand() * (exp / 2));
+}
+export function isRetryableStatus(status) {
+    return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+/** Retries transient failures (network errors, 408/425/429/5xx) with jittered exponential backoff. */
+export async function withRetries(run, opts = {}) {
+    const retries = opts.retries ?? 2;
+    const wait = opts.sleepFn ?? sleep;
+    let lastErr;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            const res = await run();
+            if (!isRetryableStatus(res.status) || attempt === retries)
+                return res;
+        }
+        catch (err) {
+            if (err?.name === "UnsafeUrlError")
+                throw err;
+            lastErr = err;
+            if (attempt === retries)
+                throw err;
+        }
+        await wait(backoffDelayMs(attempt, opts.baseMs));
+    }
+    throw lastErr;
+}

package/dist/core/engine/crawler.d.ts

@@ -0,0 +1,151 @@
+export interface MapWebsiteOptions {
+    /** Only crawl the exact root/target domain (default: true when includeSubdomains is false and selectedDomains is empty) */
+    onlyRootDomain?: boolean;
+    /** Also crawl subdomains of the allowed domain(s), e.g. docs.example.com (default: false) */
+    includeSubdomains?: boolean;
+    /** Explicit allowlist of domains/subdomains (supports exact hosts like "docs.example.com" or wildcards like "*.example.com") */
+    selectedDomains?: string[];
+    /** Maximum number of discovered URLs to return (default: 5000) */
+    limit?: number;
+    /** Maximum number of HTML pages to fetch during multi-hop BFS crawl when sitemap is missing (default: 250) */
+    maxCrawlPages?: number;
+    /** Alias for maxCrawlPages */
+    maxCrawl?: number;
+    /** Maximum BFS hop depth from the entry URL (default: 8) */
+    maxDepth?: number;
+    /** Only keep URLs whose path matches one of these (prefix, `*` wildcards, trailing `$`). */
+    includePaths?: string[];
+    /** Drop URLs whose path matches any of these. Wins over includePaths. */
+    excludePaths?: string[];
+    /** Stay under the start URL's path when no includePaths are given (default: true). */
+    restrictToSubpath?: boolean;
+    /** Sitemaps to read first (e.g. a site whose sitemap is not at /sitemap.xml). */
+    sitemapUrls?: string[];
+    /** Overall map timeout in milliseconds (default: 25000) */
+    timeoutMs?: number;
+    /** Bypass 24-hour map cache (default: false) */
+    bypassCache?: boolean;
+    /** When a bot check blocks plain requests, find pages through a real browser (default: true) */
+    stealth?: boolean;
+    /** Time budget for the stealth browser session (default: 75000) */
+    stealthBudgetMs?: number;
+    /** Allow the paid remote stealth provider (default true). Off for unbilled runs. */
+    remoteStealth?: boolean;
+}
+export declare function mapWebsiteUrls(targetUrl: string, options?: MapWebsiteOptions): Promise<{
+    success: boolean;
+    links: string[];
+    latencyMs: number;
+    source: string;
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+    pagesCrawled?: number;
+    /** The site answered plain requests with a bot check. */
+    blocked?: boolean;
+    /** Pages were found through the stealth browser. */
+    stealth?: boolean;
+    /** What a remote stealth provider charged for discovery, in USD. */
+    stealthCostUsd?: number;
+    scope?: {
+        onlyRootDomain: boolean;
+        includeSubdomains: boolean;
+        selectedDomains: string[];
+        maxCrawlPages: number;
+        maxDepth: number;
+        includePaths: string[];
+        excludePaths: string[];
+    };
+}>;
+/**
+ * Ultra-fast HTTP status batch check for `websiteDeletedPages.ts`.
+ * Checks if URLs return 404/410 without downloading/rendering full markdown!
+ */
+export declare function checkUrlsStatusBatch(urls: string[], timeoutMs?: number): Promise<Array<{
+    url: string;
+    statusCode: number;
+    ok: boolean;
+}>>;
+/**
+ * Extracts structured `{ app_name, product_features }` (or custom JSON schema) from a URL.
+ * Replaces `firecrawl.asyncExtract` in `website-extract.ts`.
+ */
+export declare function extractWebsiteData(url: string, options?: {
+    prompt?: string;
+    schema?: Record<string, unknown>;
+    openaiApiKey?: string;
+    bypassCache?: boolean;
+    /** Already-extracted content (e.g. a parsed PDF); skips the HTML fetch and cache. */
+    markdown?: string;
+}): Promise<{
+    app_name: string;
+    product_features: string[];
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+}>;
+/**
+ * Full-website multi-page crawler (`POST /v1/crawl`) backed by the shared 3-tier
+ * L1 Memory -> L2 Disk -> L3 Postgres URL store (`crawled_pages`).
+ * Pages already stored in Postgres for this website are served directly without re-crawling!
+ */
+export declare function crawlWebsitePages(targetUrl: string, options?: {
+    maxPages?: number;
+    includeSubdomains?: boolean;
+    onlyRootDomain?: boolean;
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<{
+    success: boolean;
+    rootUrl: string;
+    pagesCount: number;
+    cachedPagesCount: number;
+    postgresHits: number;
+    latencyMs: number;
+    pages: CrawledPage[];
+}>;
+export interface CrawledPage {
+    url: string;
+    title: string;
+    markdown: string;
+    linksCount: number;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+    structuredData?: Record<string, unknown> | null;
+    statusCode?: number;
+    error?: string;
+    /** The site served a bot check instead of this page. */
+    blocked?: boolean;
+    /** Read through the stealth browser. */
+    stealth?: boolean;
+    /** Remote provider that hosted the stealth browser (billed with a surcharge). */
+    stealthProvider?: string;
+}
+/** True for markdown converted from a bot-check page (old cache entries). */
+export declare function isChallengeMarkdown(title: string, markdown: string): boolean;
+/**
+ * Re-reads blocked pages through one stealth browser session. Pages that come
+ * back clean are cached and returned as content; pages still blocked keep
+ * `blocked: true` and an honest error.
+ */
+export declare function crawlPagesStealth(entryUrl: string, urls: string[], options?: {
+    extractStructured?: boolean;
+    budgetMs?: number;
+    remote?: boolean;
+}): Promise<{
+    pages: CrawledPage[];
+    solved: boolean;
+    durationMs: number;
+    error?: string;
+    provider?: string;
+    providerCostUsd?: number;
+}>;
+/**
+ * Fetches one page for a crawl: cache first (unless bypassed), then a
+ * TLS-impersonated fetch. Never throws; failures come back with `error`.
+ * Used by `crawlWebsitePages` and by the durable crawl workflow (one call per page).
+ */
+export declare function crawlPage(pageUrl: string, options?: {
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<CrawledPage>;