recker 1.0.79 → 1.0.80-next.70ea84d

package/dist/browser/mini.d.ts

@@ -6,8 +6,8 @@ export declare class MiniRequestPromise<T = unknown> implements Promise<Response
     private readonly p;
     constructor(promise: Promise<Response>);
     get [Symbol.toStringTag](): string;
-    then<TResult1 = Response, TResult2 = never>(onfulfilled?: ((value: Response) => TResult1 | PromiseLike<TResult1>) | null, onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | null): Promise<TResult1 | TResult2>;
-    catch<TResult = never>(onrejected?: ((reason: any) => TResult | PromiseLike<TResult>) | null): Promise<Response | TResult>;
+    then<TResult1 = Response, TResult2 = never>(onfulfilled?: ((value: Response) => TResult1 | PromiseLike<TResult1>) | null, onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null): Promise<TResult1 | TResult2>;
+    catch<TResult = never>(onrejected?: ((reason: unknown) => TResult | PromiseLike<TResult>) | null): Promise<Response | TResult>;
     finally(onfinally?: (() => void) | null): Promise<Response>;
     json<R = T>(): Promise<R>;
     text(): Promise<string>;

package/dist/browser/plugins/proxy-rotator.d.ts

@@ -1,6 +1,6 @@
-import { Plugin } from '../types/index.js';
+import type { Plugin, ProxyOptions } from '../types/index.js';
 export interface ProxyRotatorOptions {
-    proxies: string[];
+    proxies: (string | ProxyOptions)[];
     strategy?: 'round-robin' | 'random';
     failover?: boolean;
 }

package/dist/browser/plugins/proxy-rotator.js

@@ -1,35 +1,13 @@
-import { ProxyAgent } from 'undici';
 export function proxyRotatorPlugin(options) {
-    const proxies = options.proxies.map(url => ({
-        url,
-        agent: new ProxyAgent(url),
-        failures: 0
-    }));
-    let index = 0;
-    const getNextProxy = () => {
-        if (proxies.length === 0)
-            return null;
-        let selected;
+    return (client) => {
         if (options.strategy === 'random') {
-            selected = proxies[Math.floor(Math.random() * proxies.length)];
+            const shuffled = [...options.proxies].sort(() => Math.random() - 0.5);
+            client.defaults = client.defaults ?? {};
+            client.defaults.proxy = shuffled;
         }
         else {
-            selected = proxies[index];
-            index = (index + 1) % proxies.length;
+            client.defaults = client.defaults ?? {};
+            client.defaults.proxy = options.proxies;
         }
-        return selected;
-    };
-    return (client) => {
-        client.beforeRequest((req) => {
-            const proxy = getNextProxy();
-            if (proxy) {
-                req._dispatcher = proxy.agent;
-                req._proxyUrl = proxy.url;
-            }
-        });
-        client.onError((err, req) => {
-            if (options.failover && req._proxyUrl) {
-            }
-        });
     };
 }

package/dist/browser/recker-mini.d.ts

@@ -2,14 +2,14 @@ import { Client, type ExtendedClientOptions } from '../core/client.js';
 import { type RequestPromise } from '../core/request-promise.js';
 import type { RequestOptions } from '../types/index.js';
 import { simulateNetwork } from '../plugins/network-simulation.js';
-export declare function get(url: string, options?: RequestOptions): RequestPromise;
-export declare function post(url: string, options?: RequestOptions): RequestPromise;
-export declare function put(url: string, options?: RequestOptions): RequestPromise;
-export declare function patch(url: string, options?: RequestOptions): RequestPromise;
-export declare function del(url: string, options?: RequestOptions): RequestPromise;
-export declare function head(url: string, options?: RequestOptions): RequestPromise;
-export declare function options(url: string, options?: RequestOptions): RequestPromise;
-export declare function purge(url: string, options?: RequestOptions): RequestPromise;
+export declare function get<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function post<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function put<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function patch<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function del<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function head<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function options<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function purge<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
 export declare function ws(url: string, protocols?: string | string[]): WebSocket;
 export declare const recker: {
     get: typeof get;

package/dist/browser/recker.d.ts

@@ -4,14 +4,17 @@ import type { RequestOptions } from '../types/index.js';
 import { analyzeSeo } from '../seo/analyzer.js';
 import { createAI } from '../ai/index.js';
 import { simulateNetwork } from '../plugins/network-simulation.js';
-export declare function get(url: string, options?: RequestOptions): RequestPromise;
-export declare function post(url: string, options?: RequestOptions): RequestPromise;
-export declare function put(url: string, options?: RequestOptions): RequestPromise;
-export declare function patch(url: string, options?: RequestOptions): RequestPromise;
-export declare function del(url: string, options?: RequestOptions): RequestPromise;
-export declare function head(url: string, options?: RequestOptions): RequestPromise;
-export declare function options(url: string, options?: RequestOptions): RequestPromise;
-export declare function purge(url: string, options?: RequestOptions): RequestPromise;
+export declare function get<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function post<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function post<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
+export declare function put<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function put<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
+export declare function patch<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function patch<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
+export declare function del<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function head<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function options<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function purge<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
 export declare function ws(url: string, protocols?: string | string[]): WebSocket;
 export declare const recker: {
     get: typeof get;

package/dist/browser/recker.js

@@ -5,6 +5,51 @@ import { createAI } from '../ai/index.js';
 import { harRecorder } from '../plugins/har-recorder.js';
 import { simulateNetwork } from '../plugins/network-simulation.js';
 let _defaultClient = null;
+const REQUEST_OPTIONS_HINTS = new Set([
+    'method',
+    'headers',
+    'body',
+    'json',
+    'form',
+    'xml',
+    'yaml',
+    'csv',
+    'timeout',
+    'params',
+    'searchParams',
+    'retry',
+    'hooks',
+    'throwHttpErrors',
+    'signal',
+    'http2',
+    'followRedirects',
+    'maxRedirects',
+    'beforeRedirect',
+    'maxResponseSize',
+    'useCurl',
+    'proxy',
+    'dns',
+    'tls',
+    'policySource',
+    'policyTags',
+    'traceId',
+    'correlationId',
+    'tenant',
+    'retryOn404',
+    'retryOn5xx',
+]);
+function isRequestOptions(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return false;
+    }
+    return Object.keys(value).some((key) => REQUEST_OPTIONS_HINTS.has(key));
+}
+function resolveRequestBodyAndOptions(bodyOrOptions, options) {
+    if (options === undefined && isRequestOptions(bodyOrOptions)) {
+        return { body: undefined, options: bodyOrOptions };
+    }
+    return { body: bodyOrOptions, options: options || {} };
+}
 function getDefaultClient() {
     if (!_defaultClient) {
         _defaultClient = createClient({
@@ -16,14 +61,17 @@ function getDefaultClient() {
 export function get(url, options) {
     return getDefaultClient().get(url, options);
 }
-export function post(url, options) {
-    return getDefaultClient().post(url, options);
+export function post(url, bodyOrOptions, options) {
+    const resolved = resolveRequestBodyAndOptions(bodyOrOptions, options);
+    return getDefaultClient().post(url, resolved.body, resolved.options);
 }
-export function put(url, options) {
-    return getDefaultClient().put(url, options);
+export function put(url, bodyOrOptions, options) {
+    const resolved = resolveRequestBodyAndOptions(bodyOrOptions, options);
+    return getDefaultClient().put(url, resolved.body, resolved.options);
 }
-export function patch(url, options) {
-    return getDefaultClient().patch(url, options);
+export function patch(url, bodyOrOptions, options) {
+    const resolved = resolveRequestBodyAndOptions(bodyOrOptions, options);
+    return getDefaultClient().patch(url, resolved.body, resolved.options);
 }
 export function del(url, options) {
     return getDefaultClient().delete(url, options);

package/dist/browser/scrape/document.js

@@ -9,11 +9,11 @@ export class ScrapeDocument {
         this.options = options || {};
     }
     static async create(html, options) {
-        const root = parse(html);
+        const root = parse(html, options?.parserOptions);
         return new ScrapeDocument(root, options);
     }
     static createSync(html, options) {
-        const root = parse(html);
+        const root = parse(html, options?.parserOptions);
         return new ScrapeDocument(root, options);
     }
     select(selector) {

package/dist/browser/scrape/element.js

@@ -16,8 +16,14 @@ export class ScrapeElement {
     }
     find(selector) {
         const found = [];
+        const seen = new Set();
         this.elements.forEach((el) => {
-            found.push(...el.querySelectorAll(selector));
+            el.querySelectorAll(selector).forEach((match) => {
+                if (!seen.has(match)) {
+                    seen.add(match);
+                    found.push(match);
+                }
+            });
         });
         return new ScrapeElement(found);
     }

package/dist/browser/scrape/parser/nodes/html.js

@@ -8,7 +8,7 @@ import Node from './node.js';
 import TextNode from './text.js';
 import NodeType from './type.js';
 function decode(val) {
-    return JSON.parse(JSON.stringify(he.decode(val)));
+    return he.decode(val);
 }
 const Htags = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup'];
 const Dtags = ['details', 'dialog', 'dd', 'div', 'dt'];

package/dist/browser/scrape/spider.d.ts

@@ -1,6 +1,9 @@
 import type { ExtractedLink, ExtractionSchema } from './types.js';
+import type { Options as ParserOptions } from './parser/index.js';
 import { type SitemapUrl } from '../seo/validators/sitemap.js';
+import { type CaptchaDetectionResult, type BlockDetectionResult } from '../utils/block-detector.js';
 export type SpiderTransport = 'auto' | 'undici' | 'curl';
+type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
     maxDepth?: number;
     maxPages?: number;
@@ -11,14 +14,31 @@ export interface SpiderOptions {
     exclude?: RegExp[];
     include?: RegExp[];
     userAgent?: string;
+    rotateUserAgent?: boolean;
+    randomizeHeaders?: boolean;
+    maxRetryAttempts?: number;
+    baseRetryDelayMs?: number;
+    maxRetryDelayMs?: number;
+    retryBackoffMultiplier?: number;
+    retryJitterMs?: number;
+    maxDomainBlockStrikes?: number;
     respectRobotsTxt?: boolean;
     useSitemap?: boolean;
     sitemapUrl?: string;
     transport?: SpiderTransport;
+    preferCurlFirst?: boolean;
     onPage?: (result: SpiderPageResult) => void;
     onPageWithHtml?: (result: SpiderPageResult, html: string) => void | Promise<void>;
+    onCaptchaDetected?: (result: {
+        url: string;
+        status: number;
+        confidence: number;
+        provider?: CaptchaProvider;
+        usedCurl: boolean;
+    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
+    parserOptions?: Partial<ParserOptions>;
 }
 export interface SpiderPageResult {
     url: string;
@@ -56,6 +76,25 @@ export interface SpiderPageResult {
         total?: number;
     };
     fetchedAt?: number;
+    security?: {
+        blocked: boolean;
+        reason?: BlockDetectionResult['reason'];
+        confidence?: number;
+        captchaDetected?: boolean;
+        captchaProvider?: CaptchaProvider;
+        attempts?: number;
+        retryCount?: number;
+        retryAfterMs?: number;
+        transport?: SpiderTransport;
+        blockedByCaptcha?: boolean;
+        lastStatus?: number;
+        lastTransport?: SpiderTransport;
+    };
+    resources?: {
+        images: number;
+        scripts: number;
+        stylesheets: number;
+    };
     extracted?: Record<string, unknown>;
 }
 export interface SpiderProgress {
@@ -118,6 +157,7 @@ export declare class Spider {
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
+    private domainStates;
     private sitemapUrls;
     private sitemapUrlSet;
     private robotsData;
@@ -132,8 +172,20 @@ export declare class Spider {
     private buildRobotsAnalysis;
     private fetchPage;
     private crawlPage;
+    private getOrCreateDomainState;
+    private recordTransportResult;
+    private isRetryableStatus;
+    private buildRequestHeaders;
+    private shouldUseCurlForHost;
+    private waitForDomainPenalty;
+    private registerDomainBlock;
+    private registerDomainSuccess;
+    private getCaptchaRetryMultiplier;
+    private registerDomainChallenge;
+    private getRetryWait;
     abort(): void;
     isRunning(): boolean;
     getProgress(): SpiderProgress;
 }
 export declare function spider(url: string, options?: SpiderOptions): Promise<SpiderResult>;
+export {};