recker 1.0.93 → 1.0.94-next.83dffd9

package/dist/scrape/spider.js

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { performance } from 'node:perf_hooks';
 import { createClient } from '../core/client.js';
 import { ScrapeDocument } from './document.js';
@@ -39,10 +40,15 @@ const SEC_CH_PLATFORM_HINTS = [
 function getHostname(url) {
     return new URL(url).hostname;
 }
-function sleep(ms) {
+function sleep(ms, signal) {
     if (ms <= 0)
         return Promise.resolve();
-    return new Promise(resolve => setTimeout(resolve, ms));
+    if (signal?.aborted)
+        return Promise.resolve();
+    return new Promise(resolve => {
+        const timer = setTimeout(resolve, ms);
+        signal?.addEventListener('abort', () => { clearTimeout(timer); resolve(); }, { once: true });
+    });
 }
 function getRetryAfterDelay(response) {
     const retryAfter = response.headers.get('retry-after');
@@ -108,7 +114,8 @@ function shouldCrawl(url, baseHost, options) {
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
-        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
+        const hostname = parsed.hostname.replace(/^www\./, '');
+        if (options.sameDomain !== false && hostname !== baseHost) {
             return false;
         }
         const skipExtensions = [
@@ -188,7 +195,10 @@ export class Spider {
     baseHost = '';
     running = false;
     aborted = false;
+    abortController = new AbortController();
     pendingCount = 0;
+    domainRequestTimestamps = new Map();
+    contentHashes = new Map();
     blockedDomains = new Set();
     curlTransport = null;
     curlAvailable = false;
@@ -198,6 +208,33 @@ export class Spider {
     robotsData = null;
     sitemapValidation = null;
     robotsValidation = null;
+    async waitForDomainRateLimit(hostname) {
+        const limit = this.options.domainRateLimit?.maxPerSecond;
+        if (!limit || limit <= 0)
+            return;
+        const now = Date.now();
+        const window = 1000;
+        let timestamps = this.domainRequestTimestamps.get(hostname);
+        if (!timestamps) {
+            timestamps = [];
+            this.domainRequestTimestamps.set(hostname, timestamps);
+        }
+        while (timestamps.length > 0 && timestamps[0] <= now - window) {
+            timestamps.shift();
+        }
+        if (timestamps.length >= limit) {
+            const waitMs = timestamps[0] + window - now;
+            if (waitMs > 0)
+                await sleep(waitMs, this.abortController.signal);
+            if (this.aborted)
+                return;
+            const afterWait = Date.now();
+            while (timestamps.length > 0 && timestamps[0] <= afterWait - window) {
+                timestamps.shift();
+            }
+        }
+        timestamps.push(Date.now());
+    }
     toHeaderRecord(headers) {
         const headerRecord = {};
         headers.forEach((value, key) => {
@@ -239,11 +276,17 @@ export class Spider {
             exclude: options.exclude,
             include: options.include,
             onPage: options.onPage,
-            onPageWithHtml: options.onPageWithHtml,
             onCaptchaDetected: options.onCaptchaDetected,
+            onBlocked: options.onBlocked,
+            onError: options.onError,
+            onRetry: options.onRetry,
+            onRedirect: options.onRedirect,
             onProgress: options.onProgress,
             extract: extractSchema,
             parserOptions: options.parserOptions,
+            domainRateLimit: options.domainRateLimit,
+            deduplicateContent: options.deduplicateContent ?? false,
+            resume: options.resume ?? false,
         };
         if (options.proxy) {
             if (typeof options.proxy === 'string') {
@@ -281,22 +324,41 @@ export class Spider {
         const startTimestamp = Date.now();
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
-        this.baseHost = new URL(normalizedStart).hostname;
-        await this.crawlQueue.clear();
-        await this.crawlStorage.clear();
-        this._visitedCount = 0;
-        this._queueSize = 0;
-        this._resultCount = 0;
+        this.baseHost = new URL(normalizedStart).hostname.replace(/^www\./, '');
+        if (!this.options.resume) {
+            await this.crawlQueue.clear();
+            await this.crawlStorage.clear();
+            this._visitedCount = 0;
+            this._queueSize = 0;
+            this._resultCount = 0;
+            this.domainRequestTimestamps.clear();
+            this.contentHashes.clear();
+        }
+        else {
+            this._queueSize = await this.crawlQueue.size();
+            this._resultCount = await this.crawlStorage.getResultCount();
+        }
         this.running = true;
         this.aborted = false;
+        this.abortController = new AbortController();
         this.pendingCount = 0;
         this.sitemapUrls = [];
         this.sitemapUrlSet.clear();
         this.robotsData = null;
         this.sitemapValidation = null;
         this.robotsValidation = null;
-        this.blockedDomains.clear();
-        this.domainStates.clear();
+        if (!this.options.resume) {
+            this.blockedDomains.clear();
+            this.domainStates.clear();
+        }
+        if (this.options.resume && this.options.deduplicateContent) {
+            const existingResults = await this.crawlStorage.getResults();
+            for (const r of existingResults) {
+                if (r.contentHash) {
+                    this.contentHashes.set(r.contentHash, r.url);
+                }
+            }
+        }
         if (this.options.transport !== 'undici') {
             this.curlAvailable = await hasImpersonate();
             if (this.options.transport === 'auto' && this.curlAvailable && isProtectedDomain(this.baseHost)) {
@@ -312,10 +374,12 @@ export class Spider {
         const pending = new Map();
         const scheduleUrl = async (item) => {
             const normalized = normalizeUrl(item.url);
-            if (await this.crawlQueue.hasVisited(normalized))
-                return;
             if (pending.has(normalized))
                 return;
+            if (await this.crawlQueue.hasVisited(normalized))
+                return;
+            await this.crawlQueue.markVisited(normalized);
+            this._visitedCount++;
             if (item.depth > this.options.maxDepth)
                 return;
             if (this._resultCount + pending.size >= this.options.maxPages)
@@ -331,8 +395,6 @@ export class Spider {
                     return;
                 }
             }
-            await this.crawlQueue.markVisited(normalized);
-            this._visitedCount++;
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
                 .finally(() => {
@@ -341,16 +403,18 @@ export class Spider {
             });
             pending.set(normalized, promise);
         };
-        await scheduleUrl({ url: normalizedStart, depth: 0 });
-        if (this.options.useSitemap && this.sitemapUrls.length > 0) {
-            for (const sitemapUrl of this.sitemapUrls) {
-                try {
-                    const urlHost = new URL(sitemapUrl.loc).hostname;
-                    if (urlHost === this.baseHost) {
-                        await scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
+        if (!this.options.resume) {
+            await scheduleUrl({ url: normalizedStart, depth: 0 });
+            if (this.options.useSitemap && this.sitemapUrls.length > 0) {
+                for (const sitemapUrl of this.sitemapUrls) {
+                    try {
+                        const urlHost = new URL(sitemapUrl.loc).hostname;
+                        if (urlHost === this.baseHost) {
+                            await scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
+                        }
+                    }
+                    catch {
                     }
-                }
-                catch {
                 }
             }
         }
@@ -372,14 +436,33 @@ export class Spider {
         if (pending.size > 0) {
             await Promise.all(pending.values());
         }
+        while (!this.aborted && this._resultCount < this.options.maxPages) {
+            const remaining = await this.crawlQueue.size();
+            if (remaining === 0 && pending.size === 0)
+                break;
+            this._queueSize = remaining;
+            let nextItem = await this.crawlQueue.pop();
+            while (nextItem && !this.aborted) {
+                this._queueSize = Math.max(0, this._queueSize - 1);
+                if (this._resultCount + pending.size >= this.options.maxPages)
+                    break;
+                await scheduleUrl(nextItem);
+                nextItem = await this.crawlQueue.pop();
+            }
+            if (pending.size > 0) {
+                await Promise.all(pending.values());
+            }
+        }
         this.running = false;
         const pages = await this.crawlStorage.getResults();
         const errors = await this.crawlStorage.getErrors();
         const sitemapAnalysis = this.buildSitemapAnalysis(pages);
         const robotsAnalysis = this.buildRobotsAnalysis();
         const visited = this.crawlQueue instanceof InMemoryCrawlQueue
-            ? this.crawlQueue.getVisited()
+            ? new Set(this.crawlQueue.getVisited())
             : new Set(pages.map(r => r.url));
+        await this.crawlQueue.close?.();
+        await this.crawlStorage.close?.();
         return {
             startUrl: normalizedStart,
             pages,
@@ -571,6 +654,8 @@ export class Spider {
         let lastRetryAfterMs = 0;
         const proxyUrl = this.proxyAdapter ? await this.proxyAdapter.getProxy() : null;
         const executeRequest = async (useCurl) => {
+            if (this.aborted)
+                throw new Error('Crawl aborted');
             if (useCurl && this.curlTransport) {
                 const curlForRequest = proxyUrl
                     ? new CurlTransport(proxyUrl)
@@ -611,6 +696,10 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                signal: this.abortController.signal,
+                beforeRedirect: this.options.onRedirect
+                    ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
+                    : undefined,
             });
             const contentType = response.headers.get('content-type') || '';
             const shouldReadUndiciBody = !contentType ||
@@ -643,6 +732,7 @@ export class Spider {
             };
         };
         for (let attempt = 0; attempt < maxAttempts; attempt++) {
+            await this.waitForDomainRateLimit(hostname);
             await this.waitForDomainPenalty(hostname);
             const useCurl = this.shouldUseCurlForHost(hostname, hasCurl, forcedTransport);
             const transportForAttempt = useCurl ? 'curl' : 'undici';
@@ -732,7 +822,21 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
-                await sleep(waitMs);
+                if (this.options.onRetry) {
+                    await this.options.onRetry({
+                        url,
+                        attempt: attempt + 1,
+                        maxAttempts,
+                        reason: attemptReason,
+                        delay: waitMs,
+                        transport: forcedTransport ?? transportForAttempt,
+                        previousStatus: response.status,
+                        timings,
+                    });
+                }
+                if (this.aborted)
+                    break;
+                await sleep(waitMs, this.abortController.signal);
                 continue;
             }
             catch (error) {
@@ -763,8 +867,10 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.aborted)
+                    break;
                 const waitMs = this.getRetryWait(hostname, attempt + 1);
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
             }
         }
         if (lastResponse) {
@@ -867,7 +973,21 @@ export class Spider {
                 };
                 await this.crawlStorage.saveResult(nonHtmlResult);
                 this._resultCount++;
-                this.options.onPage?.(nonHtmlResult);
+                if (this.options.onPage) {
+                    let cachedDoc = null;
+                    await this.options.onPage({
+                        result: nonHtmlResult,
+                        html: html || undefined,
+                        document: html ? () => {
+                            if (cachedDoc)
+                                return Promise.resolve(cachedDoc);
+                            return ScrapeDocument.create(html, { baseUrl: item.url, parserOptions: this.options.parserOptions }).then(d => { cachedDoc = d; return d; });
+                        } : undefined,
+                    });
+                }
+                if (this.options.onBlocked && (detection.blocked || hasCaptcha)) {
+                    await this.options.onBlocked(nonHtmlResult);
+                }
                 return;
             }
             const doc = await ScrapeDocument.create(html, {
@@ -898,6 +1018,21 @@ export class Spider {
                 catch {
                 }
             }
+            let isDuplicate = false;
+            let duplicateOf;
+            let contentHash;
+            if (this.options.deduplicateContent) {
+                const bodyText = doc.text('body');
+                contentHash = createHash('md5').update(bodyText).digest('hex');
+                const existingUrl = this.contentHashes.get(contentHash);
+                if (existingUrl) {
+                    isDuplicate = true;
+                    duplicateOf = existingUrl;
+                }
+                else {
+                    this.contentHashes.set(contentHash, item.url);
+                }
+            }
             const result = {
                 url: item.url,
                 status,
@@ -926,42 +1061,52 @@ export class Spider {
                 timings,
                 fetchedAt,
                 extracted,
+                contentHash,
+                isDuplicate: isDuplicate || undefined,
+                duplicateOf,
             };
             await this.crawlStorage.saveResult(result);
             this._resultCount++;
-            this.options.onPage?.(result);
-            if (this.options.onPageWithHtml) {
-                await this.options.onPageWithHtml(result, html);
-            }
-            const candidates = [];
-            const candidateUrls = [];
-            for (const link of links) {
-                if (!link.href)
-                    continue;
-                const normalized = normalizeUrl(link.href);
-                if (!shouldCrawl(normalized, this.baseHost, this.options))
-                    continue;
-                candidateUrls.push(normalized);
-                candidates.push({ url: normalized, depth: item.depth + 1 });
+            if (this.options.onPage) {
+                await this.options.onPage({
+                    result,
+                    html,
+                    document: () => Promise.resolve(doc),
+                });
             }
-            if (candidates.length > 0) {
-                const visitedSet = this.crawlQueue.hasVisitedBatch
-                    ? await this.crawlQueue.hasVisitedBatch(candidateUrls)
-                    : new Set(await Promise.all(candidateUrls.map(async (u) => (await this.crawlQueue.hasVisited(u)) ? u : null)).then(r => r.filter(Boolean)));
-                const newItems = candidates.filter((_, i) => !visitedSet.has(candidateUrls[i]));
-                if (newItems.length > 0) {
-                    if (this.crawlQueue.pushBatch) {
-                        await this.crawlQueue.pushBatch(newItems);
-                    }
-                    else {
-                        for (const newItem of newItems)
-                            await this.crawlQueue.push(newItem);
+            if (!isDuplicate) {
+                const candidates = [];
+                const candidateUrls = [];
+                for (const link of links) {
+                    if (!link.href)
+                        continue;
+                    const normalized = normalizeUrl(link.href);
+                    if (!shouldCrawl(normalized, this.baseHost, this.options))
+                        continue;
+                    candidateUrls.push(normalized);
+                    candidates.push({ url: normalized, depth: item.depth + 1 });
+                }
+                if (candidates.length > 0) {
+                    const visitedSet = this.crawlQueue.hasVisitedBatch
+                        ? await this.crawlQueue.hasVisitedBatch(candidateUrls)
+                        : new Set(await Promise.all(candidateUrls.map(async (u) => (await this.crawlQueue.hasVisited(u)) ? u : null)).then(r => r.filter(Boolean)));
+                    const newItems = candidates.filter((_, i) => !visitedSet.has(candidateUrls[i]));
+                    if (newItems.length > 0) {
+                        if (this.crawlQueue.pushBatch) {
+                            await this.crawlQueue.pushBatch(newItems);
+                        }
+                        else {
+                            for (const newItem of newItems)
+                                await this.crawlQueue.push(newItem);
+                        }
+                        this._queueSize += newItems.length;
                     }
-                    this._queueSize += newItems.length;
                 }
             }
         }
         catch (error) {
+            if (this.aborted)
+                return;
             const errAttempts = typeof error?.attempts === 'number' ? error.attempts : 1;
             const errRetryCount = typeof error?.retryCount === 'number' ? error.retryCount : Math.max(0, errAttempts - 1);
             const errRetryAfterMs = typeof error?.retryAfterMs === 'number' ? error?.retryAfterMs : 0;
@@ -1026,7 +1171,12 @@ export class Spider {
             await this.crawlStorage.saveResult(errorResult);
             this._resultCount++;
             await this.crawlStorage.saveError({ url: item.url, error: message });
-            this.options.onPage?.(errorResult);
+            if (this.options.onPage) {
+                await this.options.onPage({ result: errorResult });
+            }
+            if (this.options.onError) {
+                await this.options.onError(errorResult);
+            }
         }
     }
     getOrCreateDomainState(hostname) {
@@ -1146,7 +1296,7 @@ export class Spider {
         const now = Date.now();
         const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
         if (delay > 0) {
-            await sleep(delay);
+            await sleep(delay, this.abortController.signal);
         }
     }
     registerDomainBlock(hostname) {
@@ -1217,6 +1367,7 @@ export class Spider {
     }
     abort() {
         this.aborted = true;
+        this.abortController.abort();
     }
     isRunning() {
         return this.running;

package/dist/scrape/sqlite-crawl-queue.d.ts

@@ -0,0 +1,24 @@
+import type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
+export declare class SqliteCrawlQueue implements CrawlQueueAdapter {
+    private db;
+    private stmts;
+    private constructor();
+    static create(opts?: {
+        dbPath?: string;
+    }): Promise<SqliteCrawlQueue>;
+    private ensureDb;
+    getDb(): any;
+    push(item: CrawlQueueItem): Promise<void>;
+    pushBatch(items: CrawlQueueItem[]): Promise<void>;
+    pop(): Promise<CrawlQueueItem | null>;
+    hasVisited(url: string): Promise<boolean>;
+    hasVisitedBatch(urls: string[]): Promise<Set<string>>;
+    markVisited(url: string): Promise<void>;
+    size(): Promise<number>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+    getVisitedSet(): Set<string>;
+    saveMetadata(key: string, value: string): void;
+    getMetadata(key: string): string | undefined;
+    getAllMetadata(): Record<string, string>;
+}

package/dist/scrape/sqlite-crawl-queue.js

@@ -0,0 +1,118 @@
+import * as path from 'node:path';
+import * as os from 'node:os';
+import * as crypto from 'node:crypto';
+export class SqliteCrawlQueue {
+    db;
+    stmts;
+    constructor() { }
+    static async create(opts) {
+        const instance = new SqliteCrawlQueue();
+        const dbPath = opts?.dbPath ?? path.join(os.tmpdir(), `recker-crawl-${crypto.randomUUID().slice(0, 8)}.db`);
+        await instance.ensureDb(dbPath);
+        return instance;
+    }
+    async ensureDb(dbPath) {
+        const BetterSqlite3 = (await import('better-sqlite3')).default;
+        this.db = new BetterSqlite3(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS queue (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        depth INTEGER NOT NULL,
+        priority INTEGER,
+        discovered_from TEXT
+      );
+      CREATE TABLE IF NOT EXISTS visited (url TEXT PRIMARY KEY);
+      CREATE TABLE IF NOT EXISTS crawl_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+      CREATE INDEX IF NOT EXISTS idx_queue_priority ON queue(priority ASC, id ASC);
+    `);
+        this.stmts = {
+            push: this.db.prepare('INSERT INTO queue (url, depth, priority, discovered_from) VALUES (?, ?, ?, ?)'),
+            pop: this.db.prepare('SELECT id, url, depth, priority, discovered_from FROM queue ORDER BY priority ASC NULLS LAST, id ASC LIMIT 1'),
+            deletePop: this.db.prepare('DELETE FROM queue WHERE id = ?'),
+            hasVisited: this.db.prepare('SELECT 1 FROM visited WHERE url = ?'),
+            markVisited: this.db.prepare('INSERT OR IGNORE INTO visited (url) VALUES (?)'),
+            size: this.db.prepare('SELECT COUNT(*) AS cnt FROM queue'),
+            clearQueue: this.db.prepare('DELETE FROM queue'),
+            clearVisited: this.db.prepare('DELETE FROM visited'),
+            clearMetadata: this.db.prepare('DELETE FROM crawl_metadata'),
+            allVisited: this.db.prepare('SELECT url FROM visited'),
+            saveMeta: this.db.prepare('INSERT OR REPLACE INTO crawl_metadata (key, value) VALUES (?, ?)'),
+            getMeta: this.db.prepare('SELECT value FROM crawl_metadata WHERE key = ?'),
+            allMeta: this.db.prepare('SELECT key, value FROM crawl_metadata'),
+        };
+    }
+    getDb() {
+        return this.db;
+    }
+    async push(item) {
+        this.stmts.push.run(item.url, item.depth, item.priority ?? null, item.discoveredFrom ?? null);
+    }
+    async pushBatch(items) {
+        const insert = this.db.transaction((rows) => {
+            for (const item of rows) {
+                this.stmts.push.run(item.url, item.depth, item.priority ?? null, item.discoveredFrom ?? null);
+            }
+        });
+        insert(items);
+    }
+    async pop() {
+        const row = this.stmts.pop.get();
+        if (!row)
+            return null;
+        this.stmts.deletePop.run(row.id);
+        return {
+            url: row.url,
+            depth: row.depth,
+            priority: row.priority ?? undefined,
+            discoveredFrom: row.discovered_from ?? undefined,
+        };
+    }
+    async hasVisited(url) {
+        return this.stmts.hasVisited.get(url) !== undefined;
+    }
+    async hasVisitedBatch(urls) {
+        const result = new Set();
+        for (const url of urls) {
+            if (this.stmts.hasVisited.get(url) !== undefined) {
+                result.add(url);
+            }
+        }
+        return result;
+    }
+    async markVisited(url) {
+        this.stmts.markVisited.run(url);
+    }
+    async size() {
+        const row = this.stmts.size.get();
+        return row.cnt;
+    }
+    async clear() {
+        this.stmts.clearQueue.run();
+        this.stmts.clearVisited.run();
+        this.stmts.clearMetadata.run();
+    }
+    async close() {
+        this.db.close();
+    }
+    getVisitedSet() {
+        const rows = this.stmts.allVisited.all();
+        return new Set(rows.map((r) => r.url));
+    }
+    saveMetadata(key, value) {
+        this.stmts.saveMeta.run(key, value);
+    }
+    getMetadata(key) {
+        const row = this.stmts.getMeta.get(key);
+        return row?.value;
+    }
+    getAllMetadata() {
+        const rows = this.stmts.allMeta.all();
+        const result = {};
+        for (const row of rows) {
+            result[row.key] = row.value;
+        }
+        return result;
+    }
+}

package/dist/scrape/sqlite-crawl-storage.d.ts

@@ -0,0 +1,26 @@
+import type { CrawlStorageAdapter } from './crawl-storage.js';
+import type { SpiderPageResult } from './spider.js';
+export declare class SqliteCrawlStorage implements CrawlStorageAdapter {
+    private db;
+    private ownsDb;
+    private stmts;
+    private constructor();
+    static create(opts?: {
+        dbPath?: string;
+        db?: any;
+    }): Promise<SqliteCrawlStorage>;
+    private init;
+    saveResult(result: SpiderPageResult): Promise<void>;
+    saveError(error: {
+        url: string;
+        error: string;
+    }): Promise<void>;
+    getResultCount(): Promise<number>;
+    getResults(): Promise<SpiderPageResult[]>;
+    getErrors(): Promise<Array<{
+        url: string;
+        error: string;
+    }>>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+}