recker 1.0.93 → 1.0.94-next.fcefd58

package/dist/browser/scrape/index.d.ts

@@ -3,10 +3,12 @@ export type { Options as ParserOptions } from './parser/index.js';
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
-export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
+export type { SpiderOptions, SpiderPageResult, SpiderPageEvent, SpiderProgress, SpiderResult, } from './spider.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
 export type { CrawlStorageAdapter } from './crawl-storage.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export type { ProxyAdapter } from './proxy-adapter.js';

package/dist/browser/scrape/index.js

@@ -3,6 +3,8 @@ export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/browser/scrape/spider.d.ts

@@ -1,3 +1,4 @@
+import { ScrapeDocument } from './document.js';
 import type { ExtractedLink, ExtractionSchema } from './types.js';
 import type { Options as ParserOptions } from './parser/index.js';
 import { type SitemapUrl } from '../seo/validators/sitemap.js';
@@ -31,8 +32,7 @@ export interface SpiderOptions {
     proxy?: string | string[] | ProxyAdapter;
     transport?: SpiderTransport;
     preferCurlFirst?: boolean;
-    onPage?: (result: SpiderPageResult) => void;
-    onPageWithHtml?: (result: SpiderPageResult, html: string) => void | Promise<void>;
+    onPage?: (event: SpiderPageEvent) => void | Promise<void>;
     onCaptchaDetected?: (result: {
         url: string;
         status: number;
@@ -40,9 +40,31 @@ export interface SpiderOptions {
         provider?: CaptchaProvider;
         usedCurl: boolean;
     }) => void | Promise<void>;
+    onBlocked?: (result: SpiderPageResult) => void | Promise<void>;
+    onError?: (result: SpiderPageResult) => void | Promise<void>;
+    onRetry?: (info: {
+        url: string;
+        attempt: number;
+        maxAttempts: number;
+        reason?: string;
+        delay: number;
+        transport: SpiderTransport;
+        previousStatus: number;
+        timings?: SpiderPageResult['timings'];
+    }) => void | Promise<void>;
+    onRedirect?: (info: {
+        from: string;
+        to: string;
+        status: number;
+    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;
+    domainRateLimit?: {
+        maxPerSecond?: number;
+    };
+    deduplicateContent?: boolean;
+    resume?: boolean;
     crawlQueue?: CrawlQueueAdapter;
     crawlStorage?: CrawlStorageAdapter;
 }
@@ -102,6 +124,14 @@ export interface SpiderPageResult {
         stylesheets: number;
     };
     extracted?: Record<string, unknown>;
+    contentHash?: string;
+    isDuplicate?: boolean;
+    duplicateOf?: string;
+}
+export interface SpiderPageEvent {
+    result: SpiderPageResult;
+    html?: string;
+    document?: () => Promise<ScrapeDocument>;
 }
 export interface SpiderProgress {
     crawled: number;
@@ -163,6 +193,8 @@ export declare class Spider {
     private running;
     private aborted;
     private pendingCount;
+    private domainRequestTimestamps;
+    private contentHashes;
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
@@ -172,6 +204,7 @@ export declare class Spider {
     private robotsData;
     private sitemapValidation;
     private robotsValidation;
+    private waitForDomainRateLimit;
     private toHeaderRecord;
     constructor(options?: SpiderOptions);
     crawl(startUrl: string): Promise<SpiderResult>;

package/dist/browser/scrape/spider.js

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { performance } from 'node:perf_hooks';
 import { createClient } from '../core/client.js';
 import { ScrapeDocument } from './document.js';
@@ -189,6 +190,8 @@ export class Spider {
     running = false;
     aborted = false;
     pendingCount = 0;
+    domainRequestTimestamps = new Map();
+    contentHashes = new Map();
     blockedDomains = new Set();
     curlTransport = null;
     curlAvailable = false;
@@ -198,6 +201,31 @@ export class Spider {
     robotsData = null;
     sitemapValidation = null;
     robotsValidation = null;
+    async waitForDomainRateLimit(hostname) {
+        const limit = this.options.domainRateLimit?.maxPerSecond;
+        if (!limit || limit <= 0)
+            return;
+        const now = Date.now();
+        const window = 1000;
+        let timestamps = this.domainRequestTimestamps.get(hostname);
+        if (!timestamps) {
+            timestamps = [];
+            this.domainRequestTimestamps.set(hostname, timestamps);
+        }
+        while (timestamps.length > 0 && timestamps[0] <= now - window) {
+            timestamps.shift();
+        }
+        if (timestamps.length >= limit) {
+            const waitMs = timestamps[0] + window - now;
+            if (waitMs > 0)
+                await sleep(waitMs);
+            const afterWait = Date.now();
+            while (timestamps.length > 0 && timestamps[0] <= afterWait - window) {
+                timestamps.shift();
+            }
+        }
+        timestamps.push(Date.now());
+    }
     toHeaderRecord(headers) {
         const headerRecord = {};
         headers.forEach((value, key) => {
@@ -239,11 +267,17 @@ export class Spider {
             exclude: options.exclude,
             include: options.include,
             onPage: options.onPage,
-            onPageWithHtml: options.onPageWithHtml,
             onCaptchaDetected: options.onCaptchaDetected,
+            onBlocked: options.onBlocked,
+            onError: options.onError,
+            onRetry: options.onRetry,
+            onRedirect: options.onRedirect,
             onProgress: options.onProgress,
             extract: extractSchema,
             parserOptions: options.parserOptions,
+            domainRateLimit: options.domainRateLimit,
+            deduplicateContent: options.deduplicateContent ?? false,
+            resume: options.resume ?? false,
         };
         if (options.proxy) {
             if (typeof options.proxy === 'string') {
@@ -282,11 +316,19 @@ export class Spider {
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
         this.baseHost = new URL(normalizedStart).hostname;
-        await this.crawlQueue.clear();
-        await this.crawlStorage.clear();
-        this._visitedCount = 0;
-        this._queueSize = 0;
-        this._resultCount = 0;
+        if (!this.options.resume) {
+            await this.crawlQueue.clear();
+            await this.crawlStorage.clear();
+            this._visitedCount = 0;
+            this._queueSize = 0;
+            this._resultCount = 0;
+            this.domainRequestTimestamps.clear();
+            this.contentHashes.clear();
+        }
+        else {
+            this._queueSize = await this.crawlQueue.size();
+            this._resultCount = await this.crawlStorage.getResultCount();
+        }
         this.running = true;
         this.aborted = false;
         this.pendingCount = 0;
@@ -341,16 +383,18 @@ export class Spider {
             });
             pending.set(normalized, promise);
         };
-        await scheduleUrl({ url: normalizedStart, depth: 0 });
-        if (this.options.useSitemap && this.sitemapUrls.length > 0) {
-            for (const sitemapUrl of this.sitemapUrls) {
-                try {
-                    const urlHost = new URL(sitemapUrl.loc).hostname;
-                    if (urlHost === this.baseHost) {
-                        await scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
+        if (!this.options.resume) {
+            await scheduleUrl({ url: normalizedStart, depth: 0 });
+            if (this.options.useSitemap && this.sitemapUrls.length > 0) {
+                for (const sitemapUrl of this.sitemapUrls) {
+                    try {
+                        const urlHost = new URL(sitemapUrl.loc).hostname;
+                        if (urlHost === this.baseHost) {
+                            await scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
+                        }
+                    }
+                    catch {
                     }
-                }
-                catch {
                 }
             }
         }
@@ -611,6 +655,9 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                beforeRedirect: this.options.onRedirect
+                    ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
+                    : undefined,
             });
             const contentType = response.headers.get('content-type') || '';
             const shouldReadUndiciBody = !contentType ||
@@ -643,6 +690,7 @@ export class Spider {
             };
         };
         for (let attempt = 0; attempt < maxAttempts; attempt++) {
+            await this.waitForDomainRateLimit(hostname);
             await this.waitForDomainPenalty(hostname);
             const useCurl = this.shouldUseCurlForHost(hostname, hasCurl, forcedTransport);
             const transportForAttempt = useCurl ? 'curl' : 'undici';
@@ -732,6 +780,18 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.options.onRetry) {
+                    await this.options.onRetry({
+                        url,
+                        attempt: attempt + 1,
+                        maxAttempts,
+                        reason: attemptReason,
+                        delay: waitMs,
+                        transport: forcedTransport ?? transportForAttempt,
+                        previousStatus: response.status,
+                        timings,
+                    });
+                }
                 await sleep(waitMs);
                 continue;
             }
@@ -867,7 +927,21 @@ export class Spider {
                 };
                 await this.crawlStorage.saveResult(nonHtmlResult);
                 this._resultCount++;
-                this.options.onPage?.(nonHtmlResult);
+                if (this.options.onPage) {
+                    let cachedDoc = null;
+                    await this.options.onPage({
+                        result: nonHtmlResult,
+                        html: html || undefined,
+                        document: html ? () => {
+                            if (cachedDoc)
+                                return Promise.resolve(cachedDoc);
+                            return ScrapeDocument.create(html, { baseUrl: item.url, parserOptions: this.options.parserOptions }).then(d => { cachedDoc = d; return d; });
+                        } : undefined,
+                    });
+                }
+                if (this.options.onBlocked && (detection.blocked || hasCaptcha)) {
+                    await this.options.onBlocked(nonHtmlResult);
+                }
                 return;
             }
             const doc = await ScrapeDocument.create(html, {
@@ -898,6 +972,21 @@ export class Spider {
                 catch {
                 }
             }
+            let isDuplicate = false;
+            let duplicateOf;
+            let contentHash;
+            if (this.options.deduplicateContent) {
+                const bodyText = doc.text('body');
+                contentHash = createHash('md5').update(bodyText).digest('hex');
+                const existingUrl = this.contentHashes.get(contentHash);
+                if (existingUrl) {
+                    isDuplicate = true;
+                    duplicateOf = existingUrl;
+                }
+                else {
+                    this.contentHashes.set(contentHash, item.url);
+                }
+            }
             const result = {
                 url: item.url,
                 status,
@@ -926,38 +1015,46 @@ export class Spider {
                 timings,
                 fetchedAt,
                 extracted,
+                contentHash,
+                isDuplicate: isDuplicate || undefined,
+                duplicateOf,
             };
             await this.crawlStorage.saveResult(result);
             this._resultCount++;
-            this.options.onPage?.(result);
-            if (this.options.onPageWithHtml) {
-                await this.options.onPageWithHtml(result, html);
-            }
-            const candidates = [];
-            const candidateUrls = [];
-            for (const link of links) {
-                if (!link.href)
-                    continue;
-                const normalized = normalizeUrl(link.href);
-                if (!shouldCrawl(normalized, this.baseHost, this.options))
-                    continue;
-                candidateUrls.push(normalized);
-                candidates.push({ url: normalized, depth: item.depth + 1 });
+            if (this.options.onPage) {
+                await this.options.onPage({
+                    result,
+                    html,
+                    document: () => Promise.resolve(doc),
+                });
             }
-            if (candidates.length > 0) {
-                const visitedSet = this.crawlQueue.hasVisitedBatch
-                    ? await this.crawlQueue.hasVisitedBatch(candidateUrls)
-                    : new Set(await Promise.all(candidateUrls.map(async (u) => (await this.crawlQueue.hasVisited(u)) ? u : null)).then(r => r.filter(Boolean)));
-                const newItems = candidates.filter((_, i) => !visitedSet.has(candidateUrls[i]));
-                if (newItems.length > 0) {
-                    if (this.crawlQueue.pushBatch) {
-                        await this.crawlQueue.pushBatch(newItems);
-                    }
-                    else {
-                        for (const newItem of newItems)
-                            await this.crawlQueue.push(newItem);
+            if (!isDuplicate) {
+                const candidates = [];
+                const candidateUrls = [];
+                for (const link of links) {
+                    if (!link.href)
+                        continue;
+                    const normalized = normalizeUrl(link.href);
+                    if (!shouldCrawl(normalized, this.baseHost, this.options))
+                        continue;
+                    candidateUrls.push(normalized);
+                    candidates.push({ url: normalized, depth: item.depth + 1 });
+                }
+                if (candidates.length > 0) {
+                    const visitedSet = this.crawlQueue.hasVisitedBatch
+                        ? await this.crawlQueue.hasVisitedBatch(candidateUrls)
+                        : new Set(await Promise.all(candidateUrls.map(async (u) => (await this.crawlQueue.hasVisited(u)) ? u : null)).then(r => r.filter(Boolean)));
+                    const newItems = candidates.filter((_, i) => !visitedSet.has(candidateUrls[i]));
+                    if (newItems.length > 0) {
+                        if (this.crawlQueue.pushBatch) {
+                            await this.crawlQueue.pushBatch(newItems);
+                        }
+                        else {
+                            for (const newItem of newItems)
+                                await this.crawlQueue.push(newItem);
+                        }
+                        this._queueSize += newItems.length;
                     }
-                    this._queueSize += newItems.length;
                 }
             }
         }
@@ -1026,7 +1123,12 @@ export class Spider {
             await this.crawlStorage.saveResult(errorResult);
             this._resultCount++;
             await this.crawlStorage.saveError({ url: item.url, error: message });
-            this.options.onPage?.(errorResult);
+            if (this.options.onPage) {
+                await this.options.onPage({ result: errorResult });
+            }
+            if (this.options.onError) {
+                await this.options.onError(errorResult);
+            }
         }
     }
     getOrCreateDomainState(hostname) {

package/dist/browser/scrape/sqlite-crawl-queue.d.ts

@@ -0,0 +1,24 @@
+import type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
+export declare class SqliteCrawlQueue implements CrawlQueueAdapter {
+    private db;
+    private stmts;
+    private constructor();
+    static create(opts?: {
+        dbPath?: string;
+    }): Promise<SqliteCrawlQueue>;
+    private ensureDb;
+    getDb(): any;
+    push(item: CrawlQueueItem): Promise<void>;
+    pushBatch(items: CrawlQueueItem[]): Promise<void>;
+    pop(): Promise<CrawlQueueItem | null>;
+    hasVisited(url: string): Promise<boolean>;
+    hasVisitedBatch(urls: string[]): Promise<Set<string>>;
+    markVisited(url: string): Promise<void>;
+    size(): Promise<number>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+    getVisitedSet(): Set<string>;
+    saveMetadata(key: string, value: string): void;
+    getMetadata(key: string): string | undefined;
+    getAllMetadata(): Record<string, string>;
+}

package/dist/browser/scrape/sqlite-crawl-queue.js

@@ -0,0 +1,118 @@
+import * as path from 'node:path';
+import * as os from 'node:os';
+import * as crypto from 'node:crypto';
+export class SqliteCrawlQueue {
+    db;
+    stmts;
+    constructor() { }
+    static async create(opts) {
+        const instance = new SqliteCrawlQueue();
+        const dbPath = opts?.dbPath ?? path.join(os.tmpdir(), `recker-crawl-${crypto.randomUUID().slice(0, 8)}.db`);
+        await instance.ensureDb(dbPath);
+        return instance;
+    }
+    async ensureDb(dbPath) {
+        const BetterSqlite3 = (await import('better-sqlite3')).default;
+        this.db = new BetterSqlite3(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS queue (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        depth INTEGER NOT NULL,
+        priority INTEGER,
+        discovered_from TEXT
+      );
+      CREATE TABLE IF NOT EXISTS visited (url TEXT PRIMARY KEY);
+      CREATE TABLE IF NOT EXISTS crawl_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+      CREATE INDEX IF NOT EXISTS idx_queue_priority ON queue(priority ASC, id ASC);
+    `);
+        this.stmts = {
+            push: this.db.prepare('INSERT INTO queue (url, depth, priority, discovered_from) VALUES (?, ?, ?, ?)'),
+            pop: this.db.prepare('SELECT id, url, depth, priority, discovered_from FROM queue ORDER BY priority ASC NULLS LAST, id ASC LIMIT 1'),
+            deletePop: this.db.prepare('DELETE FROM queue WHERE id = ?'),
+            hasVisited: this.db.prepare('SELECT 1 FROM visited WHERE url = ?'),
+            markVisited: this.db.prepare('INSERT OR IGNORE INTO visited (url) VALUES (?)'),
+            size: this.db.prepare('SELECT COUNT(*) AS cnt FROM queue'),
+            clearQueue: this.db.prepare('DELETE FROM queue'),
+            clearVisited: this.db.prepare('DELETE FROM visited'),
+            clearMetadata: this.db.prepare('DELETE FROM crawl_metadata'),
+            allVisited: this.db.prepare('SELECT url FROM visited'),
+            saveMeta: this.db.prepare('INSERT OR REPLACE INTO crawl_metadata (key, value) VALUES (?, ?)'),
+            getMeta: this.db.prepare('SELECT value FROM crawl_metadata WHERE key = ?'),
+            allMeta: this.db.prepare('SELECT key, value FROM crawl_metadata'),
+        };
+    }
+    getDb() {
+        return this.db;
+    }
+    async push(item) {
+        this.stmts.push.run(item.url, item.depth, item.priority ?? null, item.discoveredFrom ?? null);
+    }
+    async pushBatch(items) {
+        const insert = this.db.transaction((rows) => {
+            for (const item of rows) {
+                this.stmts.push.run(item.url, item.depth, item.priority ?? null, item.discoveredFrom ?? null);
+            }
+        });
+        insert(items);
+    }
+    async pop() {
+        const row = this.stmts.pop.get();
+        if (!row)
+            return null;
+        this.stmts.deletePop.run(row.id);
+        return {
+            url: row.url,
+            depth: row.depth,
+            priority: row.priority ?? undefined,
+            discoveredFrom: row.discovered_from ?? undefined,
+        };
+    }
+    async hasVisited(url) {
+        return this.stmts.hasVisited.get(url) !== undefined;
+    }
+    async hasVisitedBatch(urls) {
+        const result = new Set();
+        for (const url of urls) {
+            if (this.stmts.hasVisited.get(url) !== undefined) {
+                result.add(url);
+            }
+        }
+        return result;
+    }
+    async markVisited(url) {
+        this.stmts.markVisited.run(url);
+    }
+    async size() {
+        const row = this.stmts.size.get();
+        return row.cnt;
+    }
+    async clear() {
+        this.stmts.clearQueue.run();
+        this.stmts.clearVisited.run();
+        this.stmts.clearMetadata.run();
+    }
+    async close() {
+        this.db.close();
+    }
+    getVisitedSet() {
+        const rows = this.stmts.allVisited.all();
+        return new Set(rows.map((r) => r.url));
+    }
+    saveMetadata(key, value) {
+        this.stmts.saveMeta.run(key, value);
+    }
+    getMetadata(key) {
+        const row = this.stmts.getMeta.get(key);
+        return row?.value;
+    }
+    getAllMetadata() {
+        const rows = this.stmts.allMeta.all();
+        const result = {};
+        for (const row of rows) {
+            result[row.key] = row.value;
+        }
+        return result;
+    }
+}

package/dist/browser/scrape/sqlite-crawl-storage.d.ts

@@ -0,0 +1,26 @@
+import type { CrawlStorageAdapter } from './crawl-storage.js';
+import type { SpiderPageResult } from './spider.js';
+export declare class SqliteCrawlStorage implements CrawlStorageAdapter {
+    private db;
+    private ownsDb;
+    private stmts;
+    private constructor();
+    static create(opts?: {
+        dbPath?: string;
+        db?: any;
+    }): Promise<SqliteCrawlStorage>;
+    private init;
+    saveResult(result: SpiderPageResult): Promise<void>;
+    saveError(error: {
+        url: string;
+        error: string;
+    }): Promise<void>;
+    getResultCount(): Promise<number>;
+    getResults(): Promise<SpiderPageResult[]>;
+    getErrors(): Promise<Array<{
+        url: string;
+        error: string;
+    }>>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/browser/scrape/sqlite-crawl-storage.js

@@ -0,0 +1,76 @@
+import * as path from 'node:path';
+import * as os from 'node:os';
+import * as crypto from 'node:crypto';
+export class SqliteCrawlStorage {
+    db;
+    ownsDb;
+    stmts;
+    constructor(db, ownsDb) {
+        this.db = db;
+        this.ownsDb = ownsDb;
+    }
+    static async create(opts) {
+        let db = opts?.db;
+        let ownsDb = false;
+        if (!db) {
+            const dbPath = opts?.dbPath ?? path.join(os.tmpdir(), `recker-storage-${crypto.randomUUID().slice(0, 8)}.db`);
+            const BetterSqlite3 = (await import('better-sqlite3')).default;
+            db = new BetterSqlite3(dbPath);
+            db.pragma('journal_mode = WAL');
+            ownsDb = true;
+        }
+        const instance = new SqliteCrawlStorage(db, ownsDb);
+        instance.init();
+        return instance;
+    }
+    init() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS results (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        status INTEGER NOT NULL,
+        data TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS errors (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        error TEXT NOT NULL
+      );
+    `);
+        this.stmts = {
+            saveResult: this.db.prepare('INSERT INTO results (url, status, data) VALUES (?, ?, ?)'),
+            saveError: this.db.prepare('INSERT INTO errors (url, error) VALUES (?, ?)'),
+            resultCount: this.db.prepare('SELECT COUNT(*) AS cnt FROM results'),
+            allResults: this.db.prepare('SELECT data FROM results'),
+            allErrors: this.db.prepare('SELECT url, error FROM errors'),
+            clearResults: this.db.prepare('DELETE FROM results'),
+            clearErrors: this.db.prepare('DELETE FROM errors'),
+        };
+    }
+    async saveResult(result) {
+        this.stmts.saveResult.run(result.url, result.status, JSON.stringify(result));
+    }
+    async saveError(error) {
+        this.stmts.saveError.run(error.url, error.error);
+    }
+    async getResultCount() {
+        const row = this.stmts.resultCount.get();
+        return row.cnt;
+    }
+    async getResults() {
+        const rows = this.stmts.allResults.all();
+        return rows.map((r) => JSON.parse(r.data));
+    }
+    async getErrors() {
+        return this.stmts.allErrors.all();
+    }
+    async clear() {
+        this.stmts.clearResults.run();
+        this.stmts.clearErrors.run();
+    }
+    async close() {
+        if (this.ownsDb) {
+            this.db.close();
+        }
+    }
+}

package/dist/browser/seo/seo-spider.d.ts

@@ -5,6 +5,8 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
+    onBlocked?: (result: SeoPageResult) => void | Promise<void>;
+    onError?: (result: SeoPageResult) => void | Promise<void>;
     focusCategories?: string[];
     focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }

package/dist/browser/seo/seo-spider.js

@@ -32,11 +32,25 @@ export class SeoSpider {
     }
     constructor(options = {}) {
         this.options = options;
+        const userOnPage = options.onPage;
         this.spider = new Spider({
             ...options,
-            onPageWithHtml: this.options.seo
-                ? async (pageResult, html) => {
-                    await this.analyzePageDuringCrawl(pageResult, html);
+            onPage: async (event) => {
+                if (this.options.seo && event.html) {
+                    await this.analyzePageDuringCrawl(event.result, event.html);
+                }
+                if (userOnPage) {
+                    await userOnPage(event);
+                }
+            },
+            onBlocked: this.options.onBlocked
+                ? async (pageResult) => {
+                    await this.options.onBlocked({ ...pageResult });
+                }
+                : undefined,
+            onError: this.options.onError
+                ? async (pageResult) => {
+                    await this.options.onError({ ...pageResult });
                 }
                 : undefined,
         });