recker 1.0.93 → 1.0.94-next.83dffd9

package/dist/browser/scrape/sqlite-crawl-queue.js

@@ -0,0 +1,118 @@
+import * as path from 'node:path';
+import * as os from 'node:os';
+import * as crypto from 'node:crypto';
+export class SqliteCrawlQueue {
+    db;
+    stmts;
+    constructor() { }
+    static async create(opts) {
+        const instance = new SqliteCrawlQueue();
+        const dbPath = opts?.dbPath ?? path.join(os.tmpdir(), `recker-crawl-${crypto.randomUUID().slice(0, 8)}.db`);
+        await instance.ensureDb(dbPath);
+        return instance;
+    }
+    async ensureDb(dbPath) {
+        const BetterSqlite3 = (await import('better-sqlite3')).default;
+        this.db = new BetterSqlite3(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS queue (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        depth INTEGER NOT NULL,
+        priority INTEGER,
+        discovered_from TEXT
+      );
+      CREATE TABLE IF NOT EXISTS visited (url TEXT PRIMARY KEY);
+      CREATE TABLE IF NOT EXISTS crawl_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+      CREATE INDEX IF NOT EXISTS idx_queue_priority ON queue(priority ASC, id ASC);
+    `);
+        this.stmts = {
+            push: this.db.prepare('INSERT INTO queue (url, depth, priority, discovered_from) VALUES (?, ?, ?, ?)'),
+            pop: this.db.prepare('SELECT id, url, depth, priority, discovered_from FROM queue ORDER BY priority ASC NULLS LAST, id ASC LIMIT 1'),
+            deletePop: this.db.prepare('DELETE FROM queue WHERE id = ?'),
+            hasVisited: this.db.prepare('SELECT 1 FROM visited WHERE url = ?'),
+            markVisited: this.db.prepare('INSERT OR IGNORE INTO visited (url) VALUES (?)'),
+            size: this.db.prepare('SELECT COUNT(*) AS cnt FROM queue'),
+            clearQueue: this.db.prepare('DELETE FROM queue'),
+            clearVisited: this.db.prepare('DELETE FROM visited'),
+            clearMetadata: this.db.prepare('DELETE FROM crawl_metadata'),
+            allVisited: this.db.prepare('SELECT url FROM visited'),
+            saveMeta: this.db.prepare('INSERT OR REPLACE INTO crawl_metadata (key, value) VALUES (?, ?)'),
+            getMeta: this.db.prepare('SELECT value FROM crawl_metadata WHERE key = ?'),
+            allMeta: this.db.prepare('SELECT key, value FROM crawl_metadata'),
+        };
+    }
+    getDb() {
+        return this.db;
+    }
+    async push(item) {
+        this.stmts.push.run(item.url, item.depth, item.priority ?? null, item.discoveredFrom ?? null);
+    }
+    async pushBatch(items) {
+        const insert = this.db.transaction((rows) => {
+            for (const item of rows) {
+                this.stmts.push.run(item.url, item.depth, item.priority ?? null, item.discoveredFrom ?? null);
+            }
+        });
+        insert(items);
+    }
+    async pop() {
+        const row = this.stmts.pop.get();
+        if (!row)
+            return null;
+        this.stmts.deletePop.run(row.id);
+        return {
+            url: row.url,
+            depth: row.depth,
+            priority: row.priority ?? undefined,
+            discoveredFrom: row.discovered_from ?? undefined,
+        };
+    }
+    async hasVisited(url) {
+        return this.stmts.hasVisited.get(url) !== undefined;
+    }
+    async hasVisitedBatch(urls) {
+        const result = new Set();
+        for (const url of urls) {
+            if (this.stmts.hasVisited.get(url) !== undefined) {
+                result.add(url);
+            }
+        }
+        return result;
+    }
+    async markVisited(url) {
+        this.stmts.markVisited.run(url);
+    }
+    async size() {
+        const row = this.stmts.size.get();
+        return row.cnt;
+    }
+    async clear() {
+        this.stmts.clearQueue.run();
+        this.stmts.clearVisited.run();
+        this.stmts.clearMetadata.run();
+    }
+    async close() {
+        this.db.close();
+    }
+    getVisitedSet() {
+        const rows = this.stmts.allVisited.all();
+        return new Set(rows.map((r) => r.url));
+    }
+    saveMetadata(key, value) {
+        this.stmts.saveMeta.run(key, value);
+    }
+    getMetadata(key) {
+        const row = this.stmts.getMeta.get(key);
+        return row?.value;
+    }
+    getAllMetadata() {
+        const rows = this.stmts.allMeta.all();
+        const result = {};
+        for (const row of rows) {
+            result[row.key] = row.value;
+        }
+        return result;
+    }
+}

package/dist/browser/scrape/sqlite-crawl-storage.d.ts

@@ -0,0 +1,26 @@
+import type { CrawlStorageAdapter } from './crawl-storage.js';
+import type { SpiderPageResult } from './spider.js';
+export declare class SqliteCrawlStorage implements CrawlStorageAdapter {
+    private db;
+    private ownsDb;
+    private stmts;
+    private constructor();
+    static create(opts?: {
+        dbPath?: string;
+        db?: any;
+    }): Promise<SqliteCrawlStorage>;
+    private init;
+    saveResult(result: SpiderPageResult): Promise<void>;
+    saveError(error: {
+        url: string;
+        error: string;
+    }): Promise<void>;
+    getResultCount(): Promise<number>;
+    getResults(): Promise<SpiderPageResult[]>;
+    getErrors(): Promise<Array<{
+        url: string;
+        error: string;
+    }>>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/browser/scrape/sqlite-crawl-storage.js

@@ -0,0 +1,76 @@
+import * as path from 'node:path';
+import * as os from 'node:os';
+import * as crypto from 'node:crypto';
+export class SqliteCrawlStorage {
+    db;
+    ownsDb;
+    stmts;
+    constructor(db, ownsDb) {
+        this.db = db;
+        this.ownsDb = ownsDb;
+    }
+    static async create(opts) {
+        let db = opts?.db;
+        let ownsDb = false;
+        if (!db) {
+            const dbPath = opts?.dbPath ?? path.join(os.tmpdir(), `recker-storage-${crypto.randomUUID().slice(0, 8)}.db`);
+            const BetterSqlite3 = (await import('better-sqlite3')).default;
+            db = new BetterSqlite3(dbPath);
+            db.pragma('journal_mode = WAL');
+            ownsDb = true;
+        }
+        const instance = new SqliteCrawlStorage(db, ownsDb);
+        instance.init();
+        return instance;
+    }
+    init() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS results (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        status INTEGER NOT NULL,
+        data TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS errors (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        error TEXT NOT NULL
+      );
+    `);
+        this.stmts = {
+            saveResult: this.db.prepare('INSERT INTO results (url, status, data) VALUES (?, ?, ?)'),
+            saveError: this.db.prepare('INSERT INTO errors (url, error) VALUES (?, ?)'),
+            resultCount: this.db.prepare('SELECT COUNT(*) AS cnt FROM results'),
+            allResults: this.db.prepare('SELECT data FROM results'),
+            allErrors: this.db.prepare('SELECT url, error FROM errors'),
+            clearResults: this.db.prepare('DELETE FROM results'),
+            clearErrors: this.db.prepare('DELETE FROM errors'),
+        };
+    }
+    async saveResult(result) {
+        this.stmts.saveResult.run(result.url, result.status, JSON.stringify(result));
+    }
+    async saveError(error) {
+        this.stmts.saveError.run(error.url, error.error);
+    }
+    async getResultCount() {
+        const row = this.stmts.resultCount.get();
+        return row.cnt;
+    }
+    async getResults() {
+        const rows = this.stmts.allResults.all();
+        return rows.map((r) => JSON.parse(r.data));
+    }
+    async getErrors() {
+        return this.stmts.allErrors.all();
+    }
+    async clear() {
+        this.stmts.clearResults.run();
+        this.stmts.clearErrors.run();
+    }
+    async close() {
+        if (this.ownsDb) {
+            this.db.close();
+        }
+    }
+}

package/dist/browser/seo/seo-spider.d.ts

@@ -5,6 +5,8 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
+    onBlocked?: (result: SeoPageResult) => void | Promise<void>;
+    onError?: (result: SeoPageResult) => void | Promise<void>;
     focusCategories?: string[];
     focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }

package/dist/browser/seo/seo-spider.js

@@ -32,11 +32,25 @@ export class SeoSpider {
     }
     constructor(options = {}) {
         this.options = options;
+        const userOnPage = options.onPage;
         this.spider = new Spider({
             ...options,
-            onPageWithHtml: this.options.seo
-                ? async (pageResult, html) => {
-                    await this.analyzePageDuringCrawl(pageResult, html);
+            onPage: async (event) => {
+                if (this.options.seo && event.html) {
+                    await this.analyzePageDuringCrawl(event.result, event.html);
+                }
+                if (userOnPage) {
+                    await userOnPage(event);
+                }
+            },
+            onBlocked: this.options.onBlocked
+                ? async (pageResult) => {
+                    await this.options.onBlocked({ ...pageResult });
+                }
+                : undefined,
+            onError: this.options.onError
+                ? async (pageResult) => {
+                    await this.options.onError({ ...pageResult });
                 }
                 : undefined,
         });

package/dist/cli/commands/spider-runner.js

@@ -102,7 +102,8 @@ export class SpiderRunner extends CommandEmitter {
                 extract,
                 include: include?.map(p => new RegExp(p)),
                 exclude: exclude?.map(p => new RegExp(p)),
-                onPage: (page) => {
+                onPage: (event) => {
+                    const page = event.result;
                     collectPageMetrics(page);
                     pages.push({
                         url: page.url,

package/dist/cli/index.js

@@ -51,9 +51,6 @@ async function main() {
     const { handleRequest } = await import('./handler.js');
     const { resolvePreset } = await import('./presets.js');
     const presets = await import('../presets/index.js');
-    import('../utils/binary-manager.js')
-        .then(({ ensureCurlImpersonate }) => ensureCurlImpersonate(console))
-        .catch(() => { });
     const version = await getVersion();
     function parseMixedArgs(args, initialClientOptions = {}) {
         const headers = { ...initialClientOptions.headers };

package/dist/cli/tui/jobs/spider-job.js

@@ -57,7 +57,8 @@ export class SpiderJob {
                         errors: 0,
                     });
                 },
-                onPage: (result) => {
+                onPage: (event) => {
+                    const result = event.result;
                     if (result.error) {
                         const currentProgress = this.job.progress;
                         this.manager.updateProgress(this.job.id, {

package/dist/mcp/prompts/index.js

@@ -546,18 +546,27 @@ ${targetType === 'ecommerce' ? `
 \`\`\`typescript
 import { Spider } from 'recker';
 
-const spider = new Spider('https://example.com', {
+const spider = new Spider({
   maxPages: 100,
   maxDepth: 3,
   respectRobotsTxt: true,
   delay: 1000,
+  onPage: async ({ result, html, document }) => {
+    console.log('Scraped:', result.url);
+    if (document) {
+      const doc = await document();
+      console.log('Title:', doc.selectFirst('title').text());
+    }
+  },
+  onBlocked: (result) => {
+    console.warn('Blocked:', result.url, result.security?.reason);
+  },
+  onError: (result) => {
+    console.error('Error:', result.url, result.error);
+  },
 });
 
-spider.on('page', ({ url, doc }) => {
-  console.log('Scraped:', url);
-});
-
-await spider.crawl();
+await spider.crawl('https://example.com');
 \`\`\`
 
 Please provide a complete workflow including:

package/dist/scrape/index.d.ts

@@ -3,10 +3,12 @@ export type { Options as ParserOptions } from './parser/index.js';
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
-export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
+export type { SpiderOptions, SpiderPageResult, SpiderPageEvent, SpiderProgress, SpiderResult, } from './spider.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
 export type { CrawlStorageAdapter } from './crawl-storage.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export type { ProxyAdapter } from './proxy-adapter.js';

package/dist/scrape/index.js

@@ -3,6 +3,8 @@ export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/scrape/spider.d.ts

@@ -1,3 +1,4 @@
+import { ScrapeDocument } from './document.js';
 import type { ExtractedLink, ExtractionSchema } from './types.js';
 import type { Options as ParserOptions } from './parser/index.js';
 import { type SitemapUrl } from '../seo/validators/sitemap.js';
@@ -31,8 +32,7 @@ export interface SpiderOptions {
     proxy?: string | string[] | ProxyAdapter;
     transport?: SpiderTransport;
     preferCurlFirst?: boolean;
-    onPage?: (result: SpiderPageResult) => void;
-    onPageWithHtml?: (result: SpiderPageResult, html: string) => void | Promise<void>;
+    onPage?: (event: SpiderPageEvent) => void | Promise<void>;
     onCaptchaDetected?: (result: {
         url: string;
         status: number;
@@ -40,9 +40,31 @@ export interface SpiderOptions {
         provider?: CaptchaProvider;
         usedCurl: boolean;
     }) => void | Promise<void>;
+    onBlocked?: (result: SpiderPageResult) => void | Promise<void>;
+    onError?: (result: SpiderPageResult) => void | Promise<void>;
+    onRetry?: (info: {
+        url: string;
+        attempt: number;
+        maxAttempts: number;
+        reason?: string;
+        delay: number;
+        transport: SpiderTransport;
+        previousStatus: number;
+        timings?: SpiderPageResult['timings'];
+    }) => void | Promise<void>;
+    onRedirect?: (info: {
+        from: string;
+        to: string;
+        status: number;
+    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;
+    domainRateLimit?: {
+        maxPerSecond?: number;
+    };
+    deduplicateContent?: boolean;
+    resume?: boolean;
     crawlQueue?: CrawlQueueAdapter;
     crawlStorage?: CrawlStorageAdapter;
 }
@@ -102,6 +124,14 @@ export interface SpiderPageResult {
         stylesheets: number;
     };
     extracted?: Record<string, unknown>;
+    contentHash?: string;
+    isDuplicate?: boolean;
+    duplicateOf?: string;
+}
+export interface SpiderPageEvent {
+    result: SpiderPageResult;
+    html?: string;
+    document?: () => Promise<ScrapeDocument>;
 }
 export interface SpiderProgress {
     crawled: number;
@@ -162,7 +192,10 @@ export declare class Spider {
     private baseHost;
     private running;
     private aborted;
+    private abortController;
     private pendingCount;
+    private domainRequestTimestamps;
+    private contentHashes;
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
@@ -172,6 +205,7 @@ export declare class Spider {
     private robotsData;
     private sitemapValidation;
     private robotsValidation;
+    private waitForDomainRateLimit;
     private toHeaderRecord;
     constructor(options?: SpiderOptions);
     crawl(startUrl: string): Promise<SpiderResult>;