recker 1.0.93-next.40856cc → 1.0.93-next.c2e60bf

package/dist/browser/scrape/spider.d.ts

@@ -40,6 +40,23 @@ export interface SpiderOptions {
         provider?: CaptchaProvider;
         usedCurl: boolean;
     }) => void | Promise<void>;
+    onBlocked?: (result: SpiderPageResult) => void | Promise<void>;
+    onError?: (result: SpiderPageResult) => void | Promise<void>;
+    onRetry?: (info: {
+        url: string;
+        attempt: number;
+        maxAttempts: number;
+        reason?: string;
+        delay: number;
+        transport: SpiderTransport;
+        previousStatus: number;
+        timings?: SpiderPageResult['timings'];
+    }) => void | Promise<void>;
+    onRedirect?: (info: {
+        from: string;
+        to: string;
+        status: number;
+    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;

package/dist/browser/scrape/spider.js

@@ -241,6 +241,10 @@ export class Spider {
             onPage: options.onPage,
             onPageWithHtml: options.onPageWithHtml,
             onCaptchaDetected: options.onCaptchaDetected,
+            onBlocked: options.onBlocked,
+            onError: options.onError,
+            onRetry: options.onRetry,
+            onRedirect: options.onRedirect,
             onProgress: options.onProgress,
             extract: extractSchema,
             parserOptions: options.parserOptions,
@@ -611,6 +615,9 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                beforeRedirect: this.options.onRedirect
+                    ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
+                    : undefined,
             });
             const contentType = response.headers.get('content-type') || '';
             const shouldReadUndiciBody = !contentType ||
@@ -732,6 +739,18 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.options.onRetry) {
+                    await this.options.onRetry({
+                        url,
+                        attempt: attempt + 1,
+                        maxAttempts,
+                        reason: attemptReason,
+                        delay: waitMs,
+                        transport: forcedTransport ?? transportForAttempt,
+                        previousStatus: response.status,
+                        timings,
+                    });
+                }
                 await sleep(waitMs);
                 continue;
             }
@@ -868,6 +887,12 @@ export class Spider {
                 await this.crawlStorage.saveResult(nonHtmlResult);
                 this._resultCount++;
                 this.options.onPage?.(nonHtmlResult);
+                if (this.options.onPageWithHtml && html) {
+                    await this.options.onPageWithHtml(nonHtmlResult, html);
+                }
+                if (this.options.onBlocked && (detection.blocked || hasCaptcha)) {
+                    await this.options.onBlocked(nonHtmlResult);
+                }
                 return;
             }
             const doc = await ScrapeDocument.create(html, {
@@ -1027,6 +1052,9 @@ export class Spider {
             this._resultCount++;
             await this.crawlStorage.saveError({ url: item.url, error: message });
             this.options.onPage?.(errorResult);
+            if (this.options.onError) {
+                await this.options.onError(errorResult);
+            }
         }
     }
     getOrCreateDomainState(hostname) {

package/dist/browser/seo/seo-spider.d.ts

@@ -5,6 +5,8 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
+    onBlocked?: (result: SeoPageResult) => void | Promise<void>;
+    onError?: (result: SeoPageResult) => void | Promise<void>;
     focusCategories?: string[];
     focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }

package/dist/browser/seo/seo-spider.js

@@ -39,6 +39,16 @@ export class SeoSpider {
                     await this.analyzePageDuringCrawl(pageResult, html);
                 }
                 : undefined,
+            onBlocked: this.options.onBlocked
+                ? async (pageResult) => {
+                    await this.options.onBlocked({ ...pageResult });
+                }
+                : undefined,
+            onError: this.options.onError
+                ? async (pageResult) => {
+                    await this.options.onError({ ...pageResult });
+                }
+                : undefined,
         });
     }
     async analyzePageDuringCrawl(pageResult, html) {

package/dist/scrape/spider.d.ts

@@ -40,6 +40,23 @@ export interface SpiderOptions {
         provider?: CaptchaProvider;
         usedCurl: boolean;
     }) => void | Promise<void>;
+    onBlocked?: (result: SpiderPageResult) => void | Promise<void>;
+    onError?: (result: SpiderPageResult) => void | Promise<void>;
+    onRetry?: (info: {
+        url: string;
+        attempt: number;
+        maxAttempts: number;
+        reason?: string;
+        delay: number;
+        transport: SpiderTransport;
+        previousStatus: number;
+        timings?: SpiderPageResult['timings'];
+    }) => void | Promise<void>;
+    onRedirect?: (info: {
+        from: string;
+        to: string;
+        status: number;
+    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;

package/dist/scrape/spider.js

@@ -241,6 +241,10 @@ export class Spider {
             onPage: options.onPage,
             onPageWithHtml: options.onPageWithHtml,
             onCaptchaDetected: options.onCaptchaDetected,
+            onBlocked: options.onBlocked,
+            onError: options.onError,
+            onRetry: options.onRetry,
+            onRedirect: options.onRedirect,
             onProgress: options.onProgress,
             extract: extractSchema,
             parserOptions: options.parserOptions,
@@ -611,6 +615,9 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                beforeRedirect: this.options.onRedirect
+                    ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
+                    : undefined,
             });
             const contentType = response.headers.get('content-type') || '';
             const shouldReadUndiciBody = !contentType ||
@@ -732,6 +739,18 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.options.onRetry) {
+                    await this.options.onRetry({
+                        url,
+                        attempt: attempt + 1,
+                        maxAttempts,
+                        reason: attemptReason,
+                        delay: waitMs,
+                        transport: forcedTransport ?? transportForAttempt,
+                        previousStatus: response.status,
+                        timings,
+                    });
+                }
                 await sleep(waitMs);
                 continue;
             }
@@ -868,6 +887,12 @@ export class Spider {
                 await this.crawlStorage.saveResult(nonHtmlResult);
                 this._resultCount++;
                 this.options.onPage?.(nonHtmlResult);
+                if (this.options.onPageWithHtml && html) {
+                    await this.options.onPageWithHtml(nonHtmlResult, html);
+                }
+                if (this.options.onBlocked && (detection.blocked || hasCaptcha)) {
+                    await this.options.onBlocked(nonHtmlResult);
+                }
                 return;
             }
             const doc = await ScrapeDocument.create(html, {
@@ -1027,6 +1052,9 @@ export class Spider {
             this._resultCount++;
             await this.crawlStorage.saveError({ url: item.url, error: message });
             this.options.onPage?.(errorResult);
+            if (this.options.onError) {
+                await this.options.onError(errorResult);
+            }
         }
     }
     getOrCreateDomainState(hostname) {

package/dist/seo/seo-spider.d.ts

@@ -5,6 +5,8 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
+    onBlocked?: (result: SeoPageResult) => void | Promise<void>;
+    onError?: (result: SeoPageResult) => void | Promise<void>;
     focusCategories?: string[];
     focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }

package/dist/seo/seo-spider.js

@@ -39,6 +39,16 @@ export class SeoSpider {
                     await this.analyzePageDuringCrawl(pageResult, html);
                 }
                 : undefined,
+            onBlocked: this.options.onBlocked
+                ? async (pageResult) => {
+                    await this.options.onBlocked({ ...pageResult });
+                }
+                : undefined,
+            onError: this.options.onError
+                ? async (pageResult) => {
+                    await this.options.onError({ ...pageResult });
+                }
+                : undefined,
         });
     }
     async analyzePageDuringCrawl(pageResult, html) {

package/dist/version.js

@@ -1,4 +1,4 @@
-const VERSION = '1.0.93-next.40856cc';
+const VERSION = '1.0.93-next.c2e60bf';
 let _version = null;
 export async function getVersion() {
     if (_version)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.93-next.40856cc",
+  "version": "1.0.93-next.c2e60bf",
   "description": "Multi-Protocol SDK for the AI Era - HTTP, WebSocket, DNS, FTP, SFTP, Telnet, HLS unified with AI providers and MCP tools",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",