recker 1.0.93 → 1.0.94

package/dist/scrape/sqlite-crawl-storage.js

@@ -0,0 +1,76 @@
+import * as path from 'node:path';
+import * as os from 'node:os';
+import * as crypto from 'node:crypto';
+export class SqliteCrawlStorage {
+    db;
+    ownsDb;
+    stmts;
+    constructor(db, ownsDb) {
+        this.db = db;
+        this.ownsDb = ownsDb;
+    }
+    static async create(opts) {
+        let db = opts?.db;
+        let ownsDb = false;
+        if (!db) {
+            const dbPath = opts?.dbPath ?? path.join(os.tmpdir(), `recker-storage-${crypto.randomUUID().slice(0, 8)}.db`);
+            const BetterSqlite3 = (await import('better-sqlite3')).default;
+            db = new BetterSqlite3(dbPath);
+            db.pragma('journal_mode = WAL');
+            ownsDb = true;
+        }
+        const instance = new SqliteCrawlStorage(db, ownsDb);
+        instance.init();
+        return instance;
+    }
+    init() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS results (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        status INTEGER NOT NULL,
+        data TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS errors (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        error TEXT NOT NULL
+      );
+    `);
+        this.stmts = {
+            saveResult: this.db.prepare('INSERT INTO results (url, status, data) VALUES (?, ?, ?)'),
+            saveError: this.db.prepare('INSERT INTO errors (url, error) VALUES (?, ?)'),
+            resultCount: this.db.prepare('SELECT COUNT(*) AS cnt FROM results'),
+            allResults: this.db.prepare('SELECT data FROM results'),
+            allErrors: this.db.prepare('SELECT url, error FROM errors'),
+            clearResults: this.db.prepare('DELETE FROM results'),
+            clearErrors: this.db.prepare('DELETE FROM errors'),
+        };
+    }
+    async saveResult(result) {
+        this.stmts.saveResult.run(result.url, result.status, JSON.stringify(result));
+    }
+    async saveError(error) {
+        this.stmts.saveError.run(error.url, error.error);
+    }
+    async getResultCount() {
+        const row = this.stmts.resultCount.get();
+        return row.cnt;
+    }
+    async getResults() {
+        const rows = this.stmts.allResults.all();
+        return rows.map((r) => JSON.parse(r.data));
+    }
+    async getErrors() {
+        return this.stmts.allErrors.all();
+    }
+    async clear() {
+        this.stmts.clearResults.run();
+        this.stmts.clearErrors.run();
+    }
+    async close() {
+        if (this.ownsDb) {
+            this.db.close();
+        }
+    }
+}

package/dist/seo/seo-spider.d.ts

@@ -5,6 +5,8 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
+    onBlocked?: (result: SeoPageResult) => void | Promise<void>;
+    onError?: (result: SeoPageResult) => void | Promise<void>;
     focusCategories?: string[];
     focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }

package/dist/seo/seo-spider.js

@@ -32,11 +32,25 @@ export class SeoSpider {
     }
     constructor(options = {}) {
         this.options = options;
+        const userOnPage = options.onPage;
         this.spider = new Spider({
             ...options,
-            onPageWithHtml: this.options.seo
-                ? async (pageResult, html) => {
-                    await this.analyzePageDuringCrawl(pageResult, html);
+            onPage: async (event) => {
+                if (this.options.seo && event.html) {
+                    await this.analyzePageDuringCrawl(event.result, event.html);
+                }
+                if (userOnPage) {
+                    await userOnPage(event);
+                }
+            },
+            onBlocked: this.options.onBlocked
+                ? async (pageResult) => {
+                    await this.options.onBlocked({ ...pageResult });
+                }
+                : undefined,
+            onError: this.options.onError
+                ? async (pageResult) => {
+                    await this.options.onError({ ...pageResult });
                 }
                 : undefined,
         });

package/dist/version.js

@@ -1,4 +1,4 @@
-const VERSION = '1.0.93';
+const VERSION = '1.0.94';
 let _version = null;
 export async function getVersion() {
     if (_version)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.93",
+  "version": "1.0.94",
   "description": "Multi-Protocol SDK for the AI Era - HTTP, WebSocket, DNS, FTP, SFTP, Telnet, HLS unified with AI providers and MCP tools",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -241,6 +241,7 @@
     "zod": "^4.3.6"
   },
   "peerDependencies": {
+    "better-sqlite3": "^11.0.0",
     "cardinal": "^2.1.0",
     "ioredis": "^5.0.0",
     "socks": "^2.8.0",
@@ -248,6 +249,9 @@
     "ws": "^8.0.0"
   },
   "peerDependenciesMeta": {
+    "better-sqlite3": {
+      "optional": true
+    },
     "cardinal": {
       "optional": true
     },
@@ -266,6 +270,7 @@
   },
   "devDependencies": {
     "@hapi/wreck": "^18.1.0",
+    "@types/better-sqlite3": "^7.6.13",
     "@types/he": "^1.2.3",
     "@types/needle": "^3.3.0",
     "@types/node": "^25.5.0",
@@ -274,6 +279,7 @@
     "@types/ws": "^8.18.1",
     "@vitest/coverage-v8": "^4.1.0",
     "axios": "^1.13.6",
+    "better-sqlite3": "^12.8.0",
     "cardinal": "^2.1.1",
     "cross-fetch": "^4.1.0",
     "domhandler": "^5.0.3",