webcrawl 1.2.3 → 1.2.5

package/dist/cli.js

@@ -241,8 +241,9 @@ async function resolveBackendClient(cfg, deps) {
         });
     }
     try {
-        const localEngineSpec = ["..", "..", "..", "src", "local-engine.js"].join("/");
-        const localMod = (await import(localEngineSpec));
+        const packagedSpec = ["./core/local-engine.js"].join("");
+        const fallbackSpec = ["..", "..", "..", "src", "local-engine.js"].join("/");
+        const localMod = (await import(packagedSpec).catch(() => import(fallbackSpec)));
         return localMod.createLocalEngineClient();
     }
     catch {

package/dist/core/engine/cache.d.ts

@@ -0,0 +1,70 @@
+interface ExternalCacheDb {
+    getCrawledPage(urlHash: string): Promise<{
+        title: string;
+        markdown: string;
+        links: string[];
+        structuredData?: Record<string, unknown> | null;
+        statusCode: number;
+        contentHash: string;
+        etag?: string;
+        lastModified?: string;
+        crawledAt: string | Date;
+        expiresAt: string | Date;
+    } | null | undefined>;
+    upsertCrawledPage(entry: Record<string, unknown>): Promise<unknown>;
+    getCrawledSite(cacheKeyHash: string): Promise<{
+        links: string[];
+        source: string;
+        pagesCrawled: number;
+        crawledAt: string | Date;
+        expiresAt: string | Date;
+    } | null | undefined>;
+    upsertCrawledSite(entry: Record<string, unknown>): Promise<unknown>;
+}
+/**
+ * Registers an external L3 cache database store (e.g., Supabase Postgres in webcrawl.app cloud)
+ * when the engine is consumed as an installed npm package (`webcrawl/engine/cache`).
+ */
+export declare function registerExternalCacheDb(db: ExternalCacheDb | null): void;
+export interface CacheEntry {
+    title: string;
+    markdown: string;
+    links: string[];
+    structuredData?: Record<string, unknown> | null;
+    statusCode: number;
+    contentHash: string;
+    etag?: string;
+    lastModified?: string;
+    cacheLayer?: "memory" | "disk" | "postgres";
+    cachedAt: number;
+    expiresAt: number;
+}
+export interface MapCacheEntry {
+    links: string[];
+    source: string;
+    pagesCrawled: number;
+    cacheLayer?: "memory" | "disk" | "postgres";
+    cachedAt: number;
+    expiresAt: number;
+}
+export declare const CACHE_TTL_MS: number;
+export declare function sha256(content: string): string;
+export declare function normalizeCacheUrl(rawUrl: string): {
+    normalizedUrl: string;
+    domain: string;
+};
+export declare function clearLocalCacheForTests(): void;
+export declare function getCachedPage(url: string, retainImages: string): CacheEntry | undefined;
+/**
+ * 3-tier read-through lookup: L1 Memory -> L2 /tmp Disk -> L3 Postgres `crawled_pages`.
+ * Promotes L3 Postgres hits back into L1 and L2 so repeat requests never re-crawl the target website.
+ */
+export declare function getCachedPageAsync(url: string, retainImages: string): Promise<CacheEntry | undefined>;
+export declare function setCachedPage(url: string, retainImages: string, entry: Omit<CacheEntry, "expiresAt" | "cachedAt">, ttlMs?: number): void;
+export declare function getCachedMap(cacheKey: string): MapCacheEntry | undefined;
+/**
+ * 3-tier read-through lookup for website maps/crawls: L1 Memory -> L2 Disk -> L3 Postgres `crawled_sites`.
+ */
+export declare function getCachedMapAsync(cacheKey: string): Promise<MapCacheEntry | undefined>;
+export declare function setCachedMap(cacheKey: string, entry: Omit<MapCacheEntry, "cachedAt" | "expiresAt">, ttlMs?: number, rootUrl?: string): void;
+export {};

package/dist/core/engine/cache.js

@@ -0,0 +1,297 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+let cachedDbPromise = null;
+/**
+ * Registers an external L3 cache database store (e.g., Supabase Postgres in webcrawl.app cloud)
+ * when the engine is consumed as an installed npm package (`webcrawl/engine/cache`).
+ */
+export function registerExternalCacheDb(db) {
+    cachedDbPromise = Promise.resolve(db);
+}
+function getOptionalDb() {
+    if (!cachedDbPromise) {
+        const dbModulePath = "../db/index.js";
+        cachedDbPromise = import(/* @vite-ignore */ dbModulePath)
+            .then((mod) => mod?.db ?? null)
+            .catch(() => null);
+    }
+    return cachedDbPromise;
+}
+// 24-hour cache (L1 in-memory + L2 /tmp disk + L3 Postgres shared URL/website store)
+export const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
+const MAX_CACHE_ENTRIES = 2500;
+const TMP_CACHE_DIR = "/tmp/webcrawl-cache-v1";
+const pageCache = new Map();
+const mapCache = new Map();
+try {
+    fs.mkdirSync(TMP_CACHE_DIR, { recursive: true });
+}
+catch {
+    // ignore if read-only filesystem
+}
+export function sha256(content) {
+    return crypto.createHash("sha256").update(content).digest("hex");
+}
+export function normalizeCacheUrl(rawUrl) {
+    try {
+        const u = new URL(rawUrl.trim());
+        u.hash = "";
+        for (const key of Array.from(u.searchParams.keys())) {
+            if (/^(utm_|fbclid|gclid|msclkid|_ga|_gl|ref$)/i.test(key)) {
+                u.searchParams.delete(key);
+            }
+        }
+        const normalizedUrl = u.toString().replace(/\/$/, "") || u.origin;
+        const domain = u.hostname.toLowerCase().replace(/^www\./, "");
+        return { normalizedUrl, domain };
+    }
+    catch {
+        return {
+            normalizedUrl: rawUrl.trim(),
+            domain: "unknown",
+        };
+    }
+}
+export function clearLocalCacheForTests() {
+    pageCache.clear();
+    mapCache.clear();
+    try {
+        if (fs.existsSync(TMP_CACHE_DIR)) {
+            for (const f of fs.readdirSync(TMP_CACHE_DIR)) {
+                try {
+                    fs.unlinkSync(path.join(TMP_CACHE_DIR, f));
+                }
+                catch {
+                    // ignore
+                }
+            }
+        }
+    }
+    catch {
+        // ignore
+    }
+}
+export function getCachedPage(url, retainImages) {
+    const { normalizedUrl } = normalizeCacheUrl(url);
+    const key = `${normalizedUrl}::${retainImages}`;
+    const entry = pageCache.get(key);
+    if (entry) {
+        if (Date.now() > entry.expiresAt) {
+            pageCache.delete(key);
+        }
+        else {
+            return { ...entry, cacheLayer: "memory" };
+        }
+    }
+    // Check L2 /tmp disk cache for 24-hour persistence across warm/cold workers
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `page_${sha256(key)}.json`);
+        if (fs.existsSync(diskPath)) {
+            const parsed = JSON.parse(fs.readFileSync(diskPath, "utf8"));
+            if (Date.now() <= parsed.expiresAt) {
+                pageCache.set(key, parsed);
+                return { ...parsed, cacheLayer: "disk" };
+            }
+            fs.unlinkSync(diskPath);
+        }
+    }
+    catch {
+        // ignore disk read errors
+    }
+    return undefined;
+}
+/**
+ * 3-tier read-through lookup: L1 Memory -> L2 /tmp Disk -> L3 Postgres `crawled_pages`.
+ * Promotes L3 Postgres hits back into L1 and L2 so repeat requests never re-crawl the target website.
+ */
+export async function getCachedPageAsync(url, retainImages) {
+    const localHit = getCachedPage(url, retainImages);
+    if (localHit) {
+        return localHit;
+    }
+    const { normalizedUrl } = normalizeCacheUrl(url);
+    const key = `${normalizedUrl}::${retainImages}`;
+    const urlHash = sha256(key);
+    try {
+        const db = await getOptionalDb();
+        const pgHit = await db?.getCrawledPage(urlHash);
+        if (pgHit) {
+            const cachedAt = new Date(pgHit.crawledAt).getTime();
+            const expiresAt = new Date(pgHit.expiresAt).getTime();
+            if (expiresAt > Date.now()) {
+                const promoted = {
+                    title: pgHit.title,
+                    markdown: pgHit.markdown,
+                    links: pgHit.links,
+                    structuredData: pgHit.structuredData ?? null,
+                    statusCode: pgHit.statusCode,
+                    contentHash: pgHit.contentHash,
+                    etag: pgHit.etag,
+                    lastModified: pgHit.lastModified,
+                    cacheLayer: "postgres",
+                    cachedAt,
+                    expiresAt,
+                };
+                pageCache.set(key, promoted);
+                try {
+                    const diskPath = path.join(TMP_CACHE_DIR, `page_${urlHash}.json`);
+                    fs.writeFileSync(diskPath, JSON.stringify(promoted));
+                }
+                catch {
+                    // ignore
+                }
+                return promoted;
+            }
+        }
+    }
+    catch {
+        // ignore DB lookup errors
+    }
+    return undefined;
+}
+export function setCachedPage(url, retainImages, entry, ttlMs = CACHE_TTL_MS) {
+    const { normalizedUrl, domain } = normalizeCacheUrl(url);
+    const key = `${normalizedUrl}::${retainImages}`;
+    const urlHash = sha256(key);
+    if (pageCache.size >= MAX_CACHE_ENTRIES) {
+        const oldestKey = pageCache.keys().next().value;
+        if (oldestKey) {
+            pageCache.delete(oldestKey);
+        }
+    }
+    const now = Date.now();
+    const fullEntry = {
+        ...entry,
+        cacheLayer: "memory",
+        cachedAt: now,
+        expiresAt: now + ttlMs,
+    };
+    pageCache.set(key, fullEntry);
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `page_${urlHash}.json`);
+        fs.writeFileSync(diskPath, JSON.stringify(fullEntry));
+    }
+    catch {
+        // ignore disk write errors
+    }
+    // Write-through to L3 Postgres `crawled_pages` store (when available)
+    void getOptionalDb()
+        .then((db) => db?.upsertCrawledPage({
+        urlHash,
+        normalizedUrl,
+        domain,
+        title: entry.title,
+        markdown: entry.markdown,
+        links: entry.links || [],
+        structuredData: entry.structuredData ?? null,
+        statusCode: entry.statusCode,
+        contentHash: entry.contentHash,
+        etag: entry.etag,
+        lastModified: entry.lastModified,
+        retainImages,
+        fetchCount: 1,
+        crawledAt: new Date(now).toISOString(),
+        expiresAt: new Date(now + ttlMs).toISOString(),
+    }))
+        .catch(() => { });
+}
+export function getCachedMap(cacheKey) {
+    const entry = mapCache.get(cacheKey);
+    if (entry) {
+        if (Date.now() > entry.expiresAt) {
+            mapCache.delete(cacheKey);
+        }
+        else {
+            return { ...entry, cacheLayer: "memory" };
+        }
+    }
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `map_${sha256(cacheKey)}.json`);
+        if (fs.existsSync(diskPath)) {
+            const parsed = JSON.parse(fs.readFileSync(diskPath, "utf8"));
+            if (Date.now() <= parsed.expiresAt) {
+                mapCache.set(cacheKey, parsed);
+                return { ...parsed, cacheLayer: "disk" };
+            }
+            fs.unlinkSync(diskPath);
+        }
+    }
+    catch {
+        // ignore
+    }
+    return undefined;
+}
+/**
+ * 3-tier read-through lookup for website maps/crawls: L1 Memory -> L2 Disk -> L3 Postgres `crawled_sites`.
+ */
+export async function getCachedMapAsync(cacheKey) {
+    const localHit = getCachedMap(cacheKey);
+    if (localHit) {
+        return localHit;
+    }
+    const cacheKeyHash = sha256(cacheKey);
+    try {
+        const db = await getOptionalDb();
+        const pgHit = await db?.getCrawledSite(cacheKeyHash);
+        if (pgHit) {
+            const cachedAt = new Date(pgHit.crawledAt).getTime();
+            const expiresAt = new Date(pgHit.expiresAt).getTime();
+            if (expiresAt > Date.now()) {
+                const promoted = {
+                    links: pgHit.links,
+                    source: pgHit.source,
+                    pagesCrawled: pgHit.pagesCrawled,
+                    cacheLayer: "postgres",
+                    cachedAt,
+                    expiresAt,
+                };
+                mapCache.set(cacheKey, promoted);
+                try {
+                    const diskPath = path.join(TMP_CACHE_DIR, `map_${cacheKeyHash}.json`);
+                    fs.writeFileSync(diskPath, JSON.stringify(promoted));
+                }
+                catch {
+                    // ignore
+                }
+                return promoted;
+            }
+        }
+    }
+    catch {
+        // ignore
+    }
+    return undefined;
+}
+export function setCachedMap(cacheKey, entry, ttlMs = CACHE_TTL_MS, rootUrl) {
+    const now = Date.now();
+    const cacheKeyHash = sha256(cacheKey);
+    const fullEntry = {
+        ...entry,
+        cacheLayer: "memory",
+        cachedAt: now,
+        expiresAt: now + ttlMs,
+    };
+    mapCache.set(cacheKey, fullEntry);
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `map_${cacheKeyHash}.json`);
+        fs.writeFileSync(diskPath, JSON.stringify(fullEntry));
+    }
+    catch {
+        // ignore
+    }
+    const rawRoot = rootUrl || cacheKey.split("::")[0] || "";
+    const { normalizedUrl, domain } = normalizeCacheUrl(rawRoot);
+    void getOptionalDb()
+        .then((db) => db?.upsertCrawledSite({
+        cacheKeyHash,
+        rootUrl: normalizedUrl,
+        domain,
+        links: entry.links || [],
+        source: entry.source,
+        pagesCrawled: entry.pagesCrawled,
+        crawledAt: new Date(now).toISOString(),
+        expiresAt: new Date(now + ttlMs).toISOString(),
+    }))
+        .catch(() => { });
+}

package/dist/core/engine/crawl-policy.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Crawl politeness helpers: robots.txt rules, adaptive concurrency (AIMD) and retries.
+ * Pure and dependency-free so they are easy to test.
+ */
+export interface RobotsRules {
+    allows(url: string): boolean;
+    /** Crawl-delay in ms for our agent (or `*`), 0 when unset. */
+    crawlDelayMs: number;
+}
+/** Parses robots.txt. Our own group wins over `*`; longest matching rule wins; Allow beats Disallow on ties. */
+export declare function parseRobots(text: string | null | undefined): RobotsRules;
+/** Additive-increase / multiplicative-decrease concurrency controller. */
+export declare class AdaptiveConcurrency {
+    private readonly min;
+    private readonly max;
+    private target;
+    constructor(min?: number, max?: number, initial?: number);
+    get current(): number;
+    /** Feed the outcome of a finished wave. */
+    record(outcome: {
+        ok: number;
+        failed: number;
+        throttled: boolean;
+    }): number;
+}
+export declare const sleep: (ms: number) => Promise<void>;
+export declare function backoffDelayMs(attempt: number, baseMs?: number, capMs?: number, rand?: () => number): number;
+export declare function isRetryableStatus(status: number): boolean;
+/** Retries transient failures (network errors, 408/425/429/5xx) with jittered exponential backoff. */
+export declare function withRetries<T extends {
+    status: number;
+}>(run: () => Promise<T>, opts?: {
+    retries?: number;
+    baseMs?: number;
+    sleepFn?: (ms: number) => Promise<void>;
+}): Promise<T>;

package/dist/core/engine/crawl-policy.js

@@ -0,0 +1,134 @@
+/**
+ * Crawl politeness helpers: robots.txt rules, adaptive concurrency (AIMD) and retries.
+ * Pure and dependency-free so they are easy to test.
+ */
+const ROBOT_TOKEN = "webcrawl";
+function patternToRegExp(path) {
+    const anchored = path.endsWith("$");
+    const body = (anchored ? path.slice(0, -1) : path)
+        .replace(/[.+?^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, ".*");
+    return new RegExp(`^${body}${anchored ? "$" : ""}`);
+}
+/** Parses robots.txt. Our own group wins over `*`; longest matching rule wins; Allow beats Disallow on ties. */
+export function parseRobots(text) {
+    const groups = [];
+    let current = null;
+    let lastWasAgent = false;
+    for (const rawLine of (text ?? "").split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, "").trim();
+        if (!line)
+            continue;
+        const idx = line.indexOf(":");
+        if (idx < 0)
+            continue;
+        const key = line.slice(0, idx).trim().toLowerCase();
+        const value = line.slice(idx + 1).trim();
+        if (key === "user-agent") {
+            if (!current || !lastWasAgent) {
+                current = { agents: [], rules: [], delay: 0 };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+            lastWasAgent = true;
+            continue;
+        }
+        lastWasAgent = false;
+        if (!current)
+            continue;
+        if (key === "disallow" || key === "allow") {
+            if (!value && key === "disallow")
+                continue; // empty Disallow = allow all
+            current.rules.push({
+                allow: key === "allow",
+                pattern: patternToRegExp(value),
+                length: value.length,
+            });
+        }
+        else if (key === "crawl-delay") {
+            const n = Number(value);
+            if (Number.isFinite(n) && n > 0)
+                current.delay = Math.min(n, 10) * 1000;
+        }
+    }
+    const mine = groups.filter((g) => g.agents.some((a) => a !== "*" && ROBOT_TOKEN.includes(a)));
+    const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes("*"));
+    const rules = chosen.flatMap((g) => g.rules);
+    const crawlDelayMs = Math.max(0, ...chosen.map((g) => g.delay));
+    return {
+        crawlDelayMs,
+        allows(url) {
+            let path;
+            try {
+                const u = new URL(url);
+                path = u.pathname + u.search;
+            }
+            catch {
+                return true;
+            }
+            let best = null;
+            for (const r of rules) {
+                if (!r.pattern.test(path))
+                    continue;
+                if (!best || r.length > best.length || (r.length === best.length && r.allow))
+                    best = r;
+            }
+            return best ? best.allow : true;
+        },
+    };
+}
+/** Additive-increase / multiplicative-decrease concurrency controller. */
+export class AdaptiveConcurrency {
+    min;
+    max;
+    target;
+    constructor(min = 1, max = 16, initial = 4) {
+        this.min = min;
+        this.max = max;
+        this.target = Math.min(Math.max(initial, min), max);
+    }
+    get current() {
+        return this.target;
+    }
+    /** Feed the outcome of a finished wave. */
+    record(outcome) {
+        const total = outcome.ok + outcome.failed;
+        if (outcome.throttled || (total > 0 && outcome.failed / total > 0.3)) {
+            this.target = Math.max(this.min, Math.floor(this.target / 2));
+        }
+        else if (outcome.ok > 0) {
+            this.target = Math.min(this.max, this.target + 1);
+        }
+        return this.target;
+    }
+}
+export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export function backoffDelayMs(attempt, baseMs = 250, capMs = 4000, rand = Math.random) {
+    const exp = Math.min(capMs, baseMs * 2 ** attempt);
+    return Math.round(exp / 2 + rand() * (exp / 2));
+}
+export function isRetryableStatus(status) {
+    return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+/** Retries transient failures (network errors, 408/425/429/5xx) with jittered exponential backoff. */
+export async function withRetries(run, opts = {}) {
+    const retries = opts.retries ?? 2;
+    const wait = opts.sleepFn ?? sleep;
+    let lastErr;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            const res = await run();
+            if (!isRetryableStatus(res.status) || attempt === retries)
+                return res;
+        }
+        catch (err) {
+            if (err?.name === "UnsafeUrlError")
+                throw err;
+            lastErr = err;
+            if (attempt === retries)
+                throw err;
+        }
+        await wait(backoffDelayMs(attempt, opts.baseMs));
+    }
+    throw lastErr;
+}