recker 1.0.93-next.c2e60bf → 1.0.93-next.ecac5c4

package/dist/browser/scrape/spider.d.ts

@@ -40,23 +40,6 @@ export interface SpiderOptions {
         provider?: CaptchaProvider;
         usedCurl: boolean;
     }) => void | Promise<void>;
-    onBlocked?: (result: SpiderPageResult) => void | Promise<void>;
-    onError?: (result: SpiderPageResult) => void | Promise<void>;
-    onRetry?: (info: {
-        url: string;
-        attempt: number;
-        maxAttempts: number;
-        reason?: string;
-        delay: number;
-        transport: SpiderTransport;
-        previousStatus: number;
-        timings?: SpiderPageResult['timings'];
-    }) => void | Promise<void>;
-    onRedirect?: (info: {
-        from: string;
-        to: string;
-        status: number;
-    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;

package/dist/browser/scrape/spider.js

@@ -241,10 +241,6 @@ export class Spider {
             onPage: options.onPage,
             onPageWithHtml: options.onPageWithHtml,
             onCaptchaDetected: options.onCaptchaDetected,
-            onBlocked: options.onBlocked,
-            onError: options.onError,
-            onRetry: options.onRetry,
-            onRedirect: options.onRedirect,
             onProgress: options.onProgress,
             extract: extractSchema,
             parserOptions: options.parserOptions,
@@ -615,9 +611,6 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
-                beforeRedirect: this.options.onRedirect
-                    ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
-                    : undefined,
             });
             const contentType = response.headers.get('content-type') || '';
             const shouldReadUndiciBody = !contentType ||
@@ -739,18 +732,6 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
-                if (this.options.onRetry) {
-                    await this.options.onRetry({
-                        url,
-                        attempt: attempt + 1,
-                        maxAttempts,
-                        reason: attemptReason,
-                        delay: waitMs,
-                        transport: forcedTransport ?? transportForAttempt,
-                        previousStatus: response.status,
-                        timings,
-                    });
-                }
                 await sleep(waitMs);
                 continue;
             }
@@ -890,9 +871,6 @@ export class Spider {
                 if (this.options.onPageWithHtml && html) {
                     await this.options.onPageWithHtml(nonHtmlResult, html);
                 }
-                if (this.options.onBlocked && (detection.blocked || hasCaptcha)) {
-                    await this.options.onBlocked(nonHtmlResult);
-                }
                 return;
             }
             const doc = await ScrapeDocument.create(html, {
@@ -1052,9 +1030,6 @@ export class Spider {
             this._resultCount++;
             await this.crawlStorage.saveError({ url: item.url, error: message });
             this.options.onPage?.(errorResult);
-            if (this.options.onError) {
-                await this.options.onError(errorResult);
-            }
         }
     }
     getOrCreateDomainState(hostname) {

package/dist/browser/seo/seo-spider.d.ts

@@ -5,8 +5,6 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
-    onBlocked?: (result: SeoPageResult) => void | Promise<void>;
-    onError?: (result: SeoPageResult) => void | Promise<void>;
     focusCategories?: string[];
     focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }

package/dist/browser/seo/seo-spider.js

@@ -39,16 +39,6 @@ export class SeoSpider {
                     await this.analyzePageDuringCrawl(pageResult, html);
                 }
                 : undefined,
-            onBlocked: this.options.onBlocked
-                ? async (pageResult) => {
-                    await this.options.onBlocked({ ...pageResult });
-                }
-                : undefined,
-            onError: this.options.onError
-                ? async (pageResult) => {
-                    await this.options.onError({ ...pageResult });
-                }
-                : undefined,
         });
     }
     async analyzePageDuringCrawl(pageResult, html) {

package/dist/scrape/spider.d.ts

@@ -40,23 +40,6 @@ export interface SpiderOptions {
         provider?: CaptchaProvider;
         usedCurl: boolean;
     }) => void | Promise<void>;
-    onBlocked?: (result: SpiderPageResult) => void | Promise<void>;
-    onError?: (result: SpiderPageResult) => void | Promise<void>;
-    onRetry?: (info: {
-        url: string;
-        attempt: number;
-        maxAttempts: number;
-        reason?: string;
-        delay: number;
-        transport: SpiderTransport;
-        previousStatus: number;
-        timings?: SpiderPageResult['timings'];
-    }) => void | Promise<void>;
-    onRedirect?: (info: {
-        from: string;
-        to: string;
-        status: number;
-    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;

package/dist/scrape/spider.js

@@ -241,10 +241,6 @@ export class Spider {
             onPage: options.onPage,
             onPageWithHtml: options.onPageWithHtml,
             onCaptchaDetected: options.onCaptchaDetected,
-            onBlocked: options.onBlocked,
-            onError: options.onError,
-            onRetry: options.onRetry,
-            onRedirect: options.onRedirect,
             onProgress: options.onProgress,
             extract: extractSchema,
             parserOptions: options.parserOptions,
@@ -615,9 +611,6 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
-                beforeRedirect: this.options.onRedirect
-                    ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
-                    : undefined,
             });
             const contentType = response.headers.get('content-type') || '';
             const shouldReadUndiciBody = !contentType ||
@@ -739,18 +732,6 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
-                if (this.options.onRetry) {
-                    await this.options.onRetry({
-                        url,
-                        attempt: attempt + 1,
-                        maxAttempts,
-                        reason: attemptReason,
-                        delay: waitMs,
-                        transport: forcedTransport ?? transportForAttempt,
-                        previousStatus: response.status,
-                        timings,
-                    });
-                }
                 await sleep(waitMs);
                 continue;
             }
@@ -890,9 +871,6 @@ export class Spider {
                 if (this.options.onPageWithHtml && html) {
                     await this.options.onPageWithHtml(nonHtmlResult, html);
                 }
-                if (this.options.onBlocked && (detection.blocked || hasCaptcha)) {
-                    await this.options.onBlocked(nonHtmlResult);
-                }
                 return;
             }
             const doc = await ScrapeDocument.create(html, {
@@ -1052,9 +1030,6 @@ export class Spider {
             this._resultCount++;
             await this.crawlStorage.saveError({ url: item.url, error: message });
             this.options.onPage?.(errorResult);
-            if (this.options.onError) {
-                await this.options.onError(errorResult);
-            }
         }
     }
     getOrCreateDomainState(hostname) {

package/dist/seo/seo-spider.d.ts

@@ -5,8 +5,6 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
-    onBlocked?: (result: SeoPageResult) => void | Promise<void>;
-    onError?: (result: SeoPageResult) => void | Promise<void>;
     focusCategories?: string[];
     focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }

package/dist/seo/seo-spider.js

@@ -39,16 +39,6 @@ export class SeoSpider {
                     await this.analyzePageDuringCrawl(pageResult, html);
                 }
                 : undefined,
-            onBlocked: this.options.onBlocked
-                ? async (pageResult) => {
-                    await this.options.onBlocked({ ...pageResult });
-                }
-                : undefined,
-            onError: this.options.onError
-                ? async (pageResult) => {
-                    await this.options.onError({ ...pageResult });
-                }
-                : undefined,
         });
     }
     async analyzePageDuringCrawl(pageResult, html) {

package/dist/version.js

@@ -1,4 +1,4 @@
-const VERSION = '1.0.93-next.c2e60bf';
+const VERSION = '1.0.93-next.ecac5c4';
 let _version = null;
 export async function getVersion() {
     if (_version)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.93-next.c2e60bf",
+  "version": "1.0.93-next.ecac5c4",
   "description": "Multi-Protocol SDK for the AI Era - HTTP, WebSocket, DNS, FTP, SFTP, Telnet, HLS unified with AI providers and MCP tools",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",