recker 1.0.93 → 1.0.94-next.83dffd9

package/dist/browser/scrape/index.d.ts

@@ -3,10 +3,12 @@ export type { Options as ParserOptions } from './parser/index.js';
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
-export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
+export type { SpiderOptions, SpiderPageResult, SpiderPageEvent, SpiderProgress, SpiderResult, } from './spider.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
 export type { CrawlStorageAdapter } from './crawl-storage.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export type { ProxyAdapter } from './proxy-adapter.js';

package/dist/browser/scrape/index.js

@@ -3,6 +3,8 @@ export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/browser/scrape/spider.d.ts

@@ -1,3 +1,4 @@
+import { ScrapeDocument } from './document.js';
 import type { ExtractedLink, ExtractionSchema } from './types.js';
 import type { Options as ParserOptions } from './parser/index.js';
 import { type SitemapUrl } from '../seo/validators/sitemap.js';
@@ -31,8 +32,7 @@ export interface SpiderOptions {
     proxy?: string | string[] | ProxyAdapter;
     transport?: SpiderTransport;
     preferCurlFirst?: boolean;
-    onPage?: (result: SpiderPageResult) => void;
-    onPageWithHtml?: (result: SpiderPageResult, html: string) => void | Promise<void>;
+    onPage?: (event: SpiderPageEvent) => void | Promise<void>;
     onCaptchaDetected?: (result: {
         url: string;
         status: number;
@@ -40,9 +40,31 @@ export interface SpiderOptions {
         provider?: CaptchaProvider;
         usedCurl: boolean;
     }) => void | Promise<void>;
+    onBlocked?: (result: SpiderPageResult) => void | Promise<void>;
+    onError?: (result: SpiderPageResult) => void | Promise<void>;
+    onRetry?: (info: {
+        url: string;
+        attempt: number;
+        maxAttempts: number;
+        reason?: string;
+        delay: number;
+        transport: SpiderTransport;
+        previousStatus: number;
+        timings?: SpiderPageResult['timings'];
+    }) => void | Promise<void>;
+    onRedirect?: (info: {
+        from: string;
+        to: string;
+        status: number;
+    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;
+    domainRateLimit?: {
+        maxPerSecond?: number;
+    };
+    deduplicateContent?: boolean;
+    resume?: boolean;
     crawlQueue?: CrawlQueueAdapter;
     crawlStorage?: CrawlStorageAdapter;
 }
@@ -102,6 +124,14 @@ export interface SpiderPageResult {
         stylesheets: number;
     };
     extracted?: Record<string, unknown>;
+    contentHash?: string;
+    isDuplicate?: boolean;
+    duplicateOf?: string;
+}
+export interface SpiderPageEvent {
+    result: SpiderPageResult;
+    html?: string;
+    document?: () => Promise<ScrapeDocument>;
 }
 export interface SpiderProgress {
     crawled: number;
@@ -162,7 +192,10 @@ export declare class Spider {
     private baseHost;
     private running;
     private aborted;
+    private abortController;
     private pendingCount;
+    private domainRequestTimestamps;
+    private contentHashes;
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
@@ -172,6 +205,7 @@ export declare class Spider {
     private robotsData;
     private sitemapValidation;
     private robotsValidation;
+    private waitForDomainRateLimit;
     private toHeaderRecord;
     constructor(options?: SpiderOptions);
     crawl(startUrl: string): Promise<SpiderResult>;

package/dist/browser/scrape/spider.js

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { performance } from 'node:perf_hooks';
 import { createClient } from '../core/client.js';
 import { ScrapeDocument } from './document.js';
@@ -39,10 +40,15 @@ const SEC_CH_PLATFORM_HINTS = [
 function getHostname(url) {
     return new URL(url).hostname;
 }
-function sleep(ms) {
+function sleep(ms, signal) {
     if (ms <= 0)
         return Promise.resolve();
-    return new Promise(resolve => setTimeout(resolve, ms));
+    if (signal?.aborted)
+        return Promise.resolve();
+    return new Promise(resolve => {
+        const timer = setTimeout(resolve, ms);
+        signal?.addEventListener('abort', () => { clearTimeout(timer); resolve(); }, { once: true });
+    });
 }
 function getRetryAfterDelay(response) {
     const retryAfter = response.headers.get('retry-after');
@@ -108,7 +114,8 @@ function shouldCrawl(url, baseHost, options) {
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
-        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
+        const hostname = parsed.hostname.replace(/^www\./, '');
+        if (options.sameDomain !== false && hostname !== baseHost) {
             return false;
         }
         const skipExtensions = [
@@ -188,7 +195,10 @@ export class Spider {
     baseHost = '';
     running = false;
     aborted = false;
+    abortController = new AbortController();
     pendingCount = 0;
+    domainRequestTimestamps = new Map();
+    contentHashes = new Map();
     blockedDomains = new Set();
     curlTransport = null;
     curlAvailable = false;
@@ -198,6 +208,33 @@ export class Spider {
     robotsData = null;
     sitemapValidation = null;
     robotsValidation = null;
+    async waitForDomainRateLimit(hostname) {
+        const limit = this.options.domainRateLimit?.maxPerSecond;
+        if (!limit || limit <= 0)
+            return;
+        const now = Date.now();
+        const window = 1000;
+        let timestamps = this.domainRequestTimestamps.get(hostname);
+        if (!timestamps) {
+            timestamps = [];
+            this.domainRequestTimestamps.set(hostname, timestamps);
+        }
+        while (timestamps.length > 0 && timestamps[0] <= now - window) {
+            timestamps.shift();
+        }
+        if (timestamps.length >= limit) {
+            const waitMs = timestamps[0] + window - now;
+            if (waitMs > 0)
+                await sleep(waitMs, this.abortController.signal);
+            if (this.aborted)
+                return;
+            const afterWait = Date.now();
+            while (timestamps.length > 0 && timestamps[0] <= afterWait - window) {
+                timestamps.shift();
+            }
+        }
+        timestamps.push(Date.now());
+    }
     toHeaderRecord(headers) {
         const headerRecord = {};
         headers.forEach((value, key) => {
@@ -239,11 +276,17 @@ export class Spider {
             exclude: options.exclude,
             include: options.include,
             onPage: options.onPage,
-            onPageWithHtml: options.onPageWithHtml,
             onCaptchaDetected: options.onCaptchaDetected,
+            onBlocked: options.onBlocked,
+            onError: options.onError,
+            onRetry: options.onRetry,
+            onRedirect: options.onRedirect,
             onProgress: options.onProgress,
             extract: extractSchema,
             parserOptions: options.parserOptions,
+            domainRateLimit: options.domainRateLimit,
+            deduplicateContent: options.deduplicateContent ?? false,
+            resume: options.resume ?? false,
         };
         if (options.proxy) {
             if (typeof options.proxy === 'string') {
@@ -281,22 +324,41 @@ export class Spider {
         const startTimestamp = Date.now();
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
-        this.baseHost = new URL(normalizedStart).hostname;
-        await this.crawlQueue.clear();
-        await this.crawlStorage.clear();
-        this._visitedCount = 0;
-        this._queueSize = 0;
-        this._resultCount = 0;
+        this.baseHost = new URL(normalizedStart).hostname.replace(/^www\./, '');
+        if (!this.options.resume) {
+            await this.crawlQueue.clear();
+            await this.crawlStorage.clear();
+            this._visitedCount = 0;
+            this._queueSize = 0;
+            this._resultCount = 0;
+            this.domainRequestTimestamps.clear();
+            this.contentHashes.clear();
+        }
+        else {
+            this._queueSize = await this.crawlQueue.size();
+            this._resultCount = await this.crawlStorage.getResultCount();
+        }
         this.running = true;
         this.aborted = false;
+        this.abortController = new AbortController();
         this.pendingCount = 0;
         this.sitemapUrls = [];
         this.sitemapUrlSet.clear();
         this.robotsData = null;
         this.sitemapValidation = null;
         this.robotsValidation = null;
-        this.blockedDomains.clear();
-        this.domainStates.clear();
+        if (!this.options.resume) {
+            this.blockedDomains.clear();
+            this.domainStates.clear();
+        }
+        if (this.options.resume && this.options.deduplicateContent) {
+            const existingResults = await this.crawlStorage.getResults();
+            for (const r of existingResults) {
+                if (r.contentHash) {
+                    this.contentHashes.set(r.contentHash, r.url);
+                }
+            }
+        }
         if (this.options.transport !== 'undici') {
             this.curlAvailable = await hasImpersonate();
             if (this.options.transport === 'auto' && this.curlAvailable && isProtectedDomain(this.baseHost)) {
@@ -312,10 +374,12 @@ export class Spider {
         const pending = new Map();
         const scheduleUrl = async (item) => {
             const normalized = normalizeUrl(item.url);
-            if (await this.crawlQueue.hasVisited(normalized))
-                return;
             if (pending.has(normalized))
                 return;
+            if (await this.crawlQueue.hasVisited(normalized))
+                return;
+            await this.crawlQueue.markVisited(normalized);
+            this._visitedCount++;
             if (item.depth > this.options.maxDepth)
                 return;
             if (this._resultCount + pending.size >= this.options.maxPages)
@@ -331,8 +395,6 @@ export class Spider {
                     return;
                 }
             }
-            await this.crawlQueue.markVisited(normalized);
-            this._visitedCount++;
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
                 .finally(() => {
@@ -341,16 +403,18 @@ export class Spider {
             });
             pending.set(normalized, promise);
         };
-        await scheduleUrl({ url: normalizedStart, depth: 0 });
-        if (this.options.useSitemap && this.sitemapUrls.length > 0) {
-            for (const sitemapUrl of this.sitemapUrls) {
-                try {
-                    const urlHost = new URL(sitemapUrl.loc).hostname;
-                    if (urlHost === this.baseHost) {
-                        await scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
+        if (!this.options.resume) {
+            await scheduleUrl({ url: normalizedStart, depth: 0 });
+            if (this.options.useSitemap && this.sitemapUrls.length > 0) {
+                for (const sitemapUrl of this.sitemapUrls) {
+                    try {
+                        const urlHost = new URL(sitemapUrl.loc).hostname;
+                        if (urlHost === this.baseHost) {
+                            await scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
+                        }
+                    }
+                    catch {
                     }
-                }
-                catch {
                 }
             }
         }
@@ -372,14 +436,33 @@ export class Spider {
         if (pending.size > 0) {
             await Promise.all(pending.values());
         }
+        while (!this.aborted && this._resultCount < this.options.maxPages) {
+            const remaining = await this.crawlQueue.size();
+            if (remaining === 0 && pending.size === 0)
+                break;
+            this._queueSize = remaining;
+            let nextItem = await this.crawlQueue.pop();
+            while (nextItem && !this.aborted) {
+                this._queueSize = Math.max(0, this._queueSize - 1);
+                if (this._resultCount + pending.size >= this.options.maxPages)
+                    break;
+                await scheduleUrl(nextItem);
+                nextItem = await this.crawlQueue.pop();
+            }
+            if (pending.size > 0) {
+                await Promise.all(pending.values());
+            }
+        }
         this.running = false;
         const pages = await this.crawlStorage.getResults();
         const errors = await this.crawlStorage.getErrors();
         const sitemapAnalysis = this.buildSitemapAnalysis(pages);
         const robotsAnalysis = this.buildRobotsAnalysis();
         const visited = this.crawlQueue instanceof InMemoryCrawlQueue
-            ? this.crawlQueue.getVisited()
+            ? new Set(this.crawlQueue.getVisited())
             : new Set(pages.map(r => r.url));
+        await this.crawlQueue.close?.();
+        await this.crawlStorage.close?.();
         return {
             startUrl: normalizedStart,
             pages,
@@ -571,6 +654,8 @@ export class Spider {
         let lastRetryAfterMs = 0;
         const proxyUrl = this.proxyAdapter ? await this.proxyAdapter.getProxy() : null;
         const executeRequest = async (useCurl) => {
+            if (this.aborted)
+                throw new Error('Crawl aborted');
             if (useCurl && this.curlTransport) {
                 const curlForRequest = proxyUrl
                     ? new CurlTransport(proxyUrl)
@@ -611,6 +696,10 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                signal: this.abortController.signal,
+                beforeRedirect: this.options.onRedirect
+                    ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
+                    : undefined,
             });
             const contentType = response.headers.get('content-type') || '';
             const shouldReadUndiciBody = !contentType ||
@@ -643,6 +732,7 @@ export class Spider {
             };
         };
         for (let attempt = 0; attempt < maxAttempts; attempt++) {
+            await this.waitForDomainRateLimit(hostname);
             await this.waitForDomainPenalty(hostname);
             const useCurl = this.shouldUseCurlForHost(hostname, hasCurl, forcedTransport);
             const transportForAttempt = useCurl ? 'curl' : 'undici';
@@ -732,7 +822,21 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
-                await sleep(waitMs);
+                if (this.options.onRetry) {
+                    await this.options.onRetry({
+                        url,
+                        attempt: attempt + 1,
+                        maxAttempts,
+                        reason: attemptReason,
+                        delay: waitMs,
+                        transport: forcedTransport ?? transportForAttempt,
+                        previousStatus: response.status,
+                        timings,
+                    });
+                }
+                if (this.aborted)
+                    break;
+                await sleep(waitMs, this.abortController.signal);
                 continue;
             }
             catch (error) {
@@ -763,8 +867,10 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.aborted)
+                    break;
                 const waitMs = this.getRetryWait(hostname, attempt + 1);
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
             }
         }
         if (lastResponse) {
@@ -867,7 +973,21 @@ export class Spider {
                 };
                 await this.crawlStorage.saveResult(nonHtmlResult);
                 this._resultCount++;
-                this.options.onPage?.(nonHtmlResult);
+                if (this.options.onPage) {
+                    let cachedDoc = null;
+                    await this.options.onPage({
+                        result: nonHtmlResult,
+                        html: html || undefined,
+                        document: html ? () => {
+                            if (cachedDoc)
+                                return Promise.resolve(cachedDoc);
+                            return ScrapeDocument.create(html, { baseUrl: item.url, parserOptions: this.options.parserOptions }).then(d => { cachedDoc = d; return d; });
+                        } : undefined,
+                    });
+                }
+                if (this.options.onBlocked && (detection.blocked || hasCaptcha)) {
+                    await this.options.onBlocked(nonHtmlResult);
+                }
                 return;
             }
             const doc = await ScrapeDocument.create(html, {
@@ -898,6 +1018,21 @@ export class Spider {
                 catch {
                 }
             }
+            let isDuplicate = false;
+            let duplicateOf;
+            let contentHash;
+            if (this.options.deduplicateContent) {
+                const bodyText = doc.text('body');
+                contentHash = createHash('md5').update(bodyText).digest('hex');
+                const existingUrl = this.contentHashes.get(contentHash);
+                if (existingUrl) {
+                    isDuplicate = true;
+                    duplicateOf = existingUrl;
+                }
+                else {
+                    this.contentHashes.set(contentHash, item.url);
+                }
+            }
             const result = {
                 url: item.url,
                 status,
@@ -926,42 +1061,52 @@ export class Spider {
                 timings,
                 fetchedAt,
                 extracted,
+                contentHash,
+                isDuplicate: isDuplicate || undefined,
+                duplicateOf,
             };
             await this.crawlStorage.saveResult(result);
             this._resultCount++;
-            this.options.onPage?.(result);
-            if (this.options.onPageWithHtml) {
-                await this.options.onPageWithHtml(result, html);
-            }
-            const candidates = [];
-            const candidateUrls = [];
-            for (const link of links) {
-                if (!link.href)
-                    continue;
-                const normalized = normalizeUrl(link.href);
-                if (!shouldCrawl(normalized, this.baseHost, this.options))
-                    continue;
-                candidateUrls.push(normalized);
-                candidates.push({ url: normalized, depth: item.depth + 1 });
+            if (this.options.onPage) {
+                await this.options.onPage({
+                    result,
+                    html,
+                    document: () => Promise.resolve(doc),
+                });
             }
-            if (candidates.length > 0) {
-                const visitedSet = this.crawlQueue.hasVisitedBatch
-                    ? await this.crawlQueue.hasVisitedBatch(candidateUrls)
-                    : new Set(await Promise.all(candidateUrls.map(async (u) => (await this.crawlQueue.hasVisited(u)) ? u : null)).then(r => r.filter(Boolean)));
-                const newItems = candidates.filter((_, i) => !visitedSet.has(candidateUrls[i]));
-                if (newItems.length > 0) {
-                    if (this.crawlQueue.pushBatch) {
-                        await this.crawlQueue.pushBatch(newItems);
-                    }
-                    else {
-                        for (const newItem of newItems)
-                            await this.crawlQueue.push(newItem);
+            if (!isDuplicate) {
+                const candidates = [];
+                const candidateUrls = [];
+                for (const link of links) {
+                    if (!link.href)
+                        continue;
+                    const normalized = normalizeUrl(link.href);
+                    if (!shouldCrawl(normalized, this.baseHost, this.options))
+                        continue;
+                    candidateUrls.push(normalized);
+                    candidates.push({ url: normalized, depth: item.depth + 1 });
+                }
+                if (candidates.length > 0) {
+                    const visitedSet = this.crawlQueue.hasVisitedBatch
+                        ? await this.crawlQueue.hasVisitedBatch(candidateUrls)
+                        : new Set(await Promise.all(candidateUrls.map(async (u) => (await this.crawlQueue.hasVisited(u)) ? u : null)).then(r => r.filter(Boolean)));
+                    const newItems = candidates.filter((_, i) => !visitedSet.has(candidateUrls[i]));
+                    if (newItems.length > 0) {
+                        if (this.crawlQueue.pushBatch) {
+                            await this.crawlQueue.pushBatch(newItems);
+                        }
+                        else {
+                            for (const newItem of newItems)
+                                await this.crawlQueue.push(newItem);
+                        }
+                        this._queueSize += newItems.length;
                     }
-                    this._queueSize += newItems.length;
                 }
             }
         }
         catch (error) {
+            if (this.aborted)
+                return;
             const errAttempts = typeof error?.attempts === 'number' ? error.attempts : 1;
             const errRetryCount = typeof error?.retryCount === 'number' ? error.retryCount : Math.max(0, errAttempts - 1);
             const errRetryAfterMs = typeof error?.retryAfterMs === 'number' ? error?.retryAfterMs : 0;
@@ -1026,7 +1171,12 @@ export class Spider {
             await this.crawlStorage.saveResult(errorResult);
             this._resultCount++;
             await this.crawlStorage.saveError({ url: item.url, error: message });
-            this.options.onPage?.(errorResult);
+            if (this.options.onPage) {
+                await this.options.onPage({ result: errorResult });
+            }
+            if (this.options.onError) {
+                await this.options.onError(errorResult);
+            }
         }
     }
     getOrCreateDomainState(hostname) {
@@ -1146,7 +1296,7 @@ export class Spider {
         const now = Date.now();
         const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
         if (delay > 0) {
-            await sleep(delay);
+            await sleep(delay, this.abortController.signal);
         }
     }
     registerDomainBlock(hostname) {
@@ -1217,6 +1367,7 @@ export class Spider {
     }
     abort() {
         this.aborted = true;
+        this.abortController.abort();
     }
     isRunning() {
         return this.running;

package/dist/browser/scrape/sqlite-crawl-queue.d.ts

@@ -0,0 +1,24 @@
+import type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
+export declare class SqliteCrawlQueue implements CrawlQueueAdapter {
+    private db;
+    private stmts;
+    private constructor();
+    static create(opts?: {
+        dbPath?: string;
+    }): Promise<SqliteCrawlQueue>;
+    private ensureDb;
+    getDb(): any;
+    push(item: CrawlQueueItem): Promise<void>;
+    pushBatch(items: CrawlQueueItem[]): Promise<void>;
+    pop(): Promise<CrawlQueueItem | null>;
+    hasVisited(url: string): Promise<boolean>;
+    hasVisitedBatch(urls: string[]): Promise<Set<string>>;
+    markVisited(url: string): Promise<void>;
+    size(): Promise<number>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+    getVisitedSet(): Set<string>;
+    saveMetadata(key: string, value: string): void;
+    getMetadata(key: string): string | undefined;
+    getAllMetadata(): Record<string, string>;
+}