recker 1.0.79 → 1.0.80-next.70ea84d

package/dist/raffel/client.js

@@ -0,0 +1,282 @@
+import { ReckerWebSocket } from '../websocket/client.js';
+import { RaffelError } from './types.js';
+class SimpleEmitter {
+    listeners = new Map();
+    on(event, listener) {
+        const set = this.listeners.get(event) ?? new Set();
+        set.add(listener);
+        this.listeners.set(event, set);
+        return this;
+    }
+    once(event, listener) {
+        const wrapped = (...args) => {
+            this.off(event, wrapped);
+            listener(...args);
+        };
+        return this.on(event, wrapped);
+    }
+    off(event, listener) {
+        const set = this.listeners.get(event);
+        if (set) {
+            set.delete(listener);
+            if (set.size === 0)
+                this.listeners.delete(event);
+        }
+        return this;
+    }
+    emit(event, ...args) {
+        const set = this.listeners.get(event);
+        if (!set)
+            return false;
+        for (const listener of [...set]) {
+            listener(...args);
+        }
+        return true;
+    }
+}
+export class RaffelClient extends SimpleEmitter {
+    ws;
+    pendingCalls = new Map();
+    subscribedChannels = new Map();
+    idCounter = 0;
+    defaultTimeout;
+    onEvent;
+    constructor(url, options = {}) {
+        super();
+        const { channels, channelHandlers, onEvent, defaultTimeout, ...wsOptions } = options;
+        this.defaultTimeout = defaultTimeout ?? 30_000;
+        this.onEvent = onEvent;
+        this.ws = new ReckerWebSocket(url, wsOptions);
+        if (channels) {
+            for (const ch of channels) {
+                this.subscribedChannels.set(ch, channelHandlers?.[ch] ?? null);
+            }
+        }
+        if (channelHandlers) {
+            for (const [ch, handler] of Object.entries(channelHandlers)) {
+                if (!this.subscribedChannels.has(ch)) {
+                    this.subscribedChannels.set(ch, handler);
+                }
+            }
+        }
+        this.ws.on('message', (msg) => this.handleMessage(msg));
+        this.ws.on('open', () => {
+            this.emit('raffel:connected');
+            for (const [channel] of this.subscribedChannels) {
+                this.sendChannelSubscribe(channel);
+            }
+        });
+        this.ws.on('close', (_code, _reason) => {
+            this.emit('raffel:disconnected');
+            for (const [id, pending] of this.pendingCalls) {
+                if (pending.timer)
+                    clearTimeout(pending.timer);
+                pending.reject(new Error(`Connection closed while waiting for response to ${id}`));
+            }
+            this.pendingCalls.clear();
+        });
+        this.ws.on('reconnecting', (attempt, delay) => {
+            this.emit('ws:reconnecting', attempt, delay);
+        });
+        this.ws.on('error', (err) => {
+            this.emit('ws:error', err);
+        });
+    }
+    async connect() {
+        return this.ws.connect();
+    }
+    close(code, reason) {
+        for (const [id, pending] of this.pendingCalls) {
+            if (pending.timer)
+                clearTimeout(pending.timer);
+            pending.reject(new Error(`Connection closed while waiting for response to ${id}`));
+        }
+        this.pendingCalls.clear();
+        this.ws.close(code, reason);
+    }
+    get isConnected() {
+        return this.ws.isConnected;
+    }
+    get raw() {
+        return this.ws;
+    }
+    async call(procedure, payload, options) {
+        const id = this.nextId();
+        const timeout = options?.timeout ?? this.defaultTimeout;
+        const envelope = {
+            id,
+            procedure,
+            type: 'request',
+            payload: payload ?? {},
+            metadata: {},
+        };
+        return new Promise((resolve, reject) => {
+            let timer = null;
+            const cleanup = () => {
+                this.pendingCalls.delete(id);
+                if (timer)
+                    clearTimeout(timer);
+                if (options?.signal) {
+                    options.signal.removeEventListener('abort', onAbort);
+                }
+            };
+            const onAbort = () => {
+                cleanup();
+                this.sendRaw({ id, type: 'cancel' });
+                reject(new Error(`Call to ${procedure} was aborted`));
+            };
+            if (timeout > 0) {
+                timer = setTimeout(() => {
+                    cleanup();
+                    reject(new Error(`Call to ${procedure} timed out after ${timeout}ms`));
+                }, timeout);
+            }
+            if (options?.signal) {
+                if (options.signal.aborted) {
+                    reject(new Error(`Call to ${procedure} was aborted`));
+                    return;
+                }
+                options.signal.addEventListener('abort', onAbort, { once: true });
+            }
+            this.pendingCalls.set(id, { resolve, reject, timer });
+            this.ws.sendJSON(envelope);
+        });
+    }
+    notify(procedure, payload) {
+        const envelope = {
+            id: this.nextId(),
+            procedure,
+            type: 'event',
+            payload: payload ?? {},
+            metadata: {},
+        };
+        this.ws.sendJSON(envelope);
+    }
+    subscribe(channel, handler) {
+        this.subscribedChannels.set(channel, handler ?? null);
+        if (this.ws.isConnected) {
+            this.sendChannelSubscribe(channel);
+        }
+    }
+    unsubscribe(channel) {
+        this.subscribedChannels.delete(channel);
+        if (this.ws.isConnected) {
+            const msg = {
+                id: this.nextId(),
+                type: 'unsubscribe',
+                channel,
+            };
+            this.ws.sendJSON(msg);
+        }
+    }
+    publish(channel, event, data) {
+        const msg = {
+            id: this.nextId(),
+            type: 'publish',
+            channel,
+            event,
+            data,
+        };
+        this.ws.sendJSON(msg);
+    }
+    cancel(id) {
+        const pending = this.pendingCalls.get(id);
+        if (pending) {
+            if (pending.timer)
+                clearTimeout(pending.timer);
+            pending.reject(new Error(`Call ${id} was cancelled`));
+            this.pendingCalls.delete(id);
+        }
+        this.sendRaw({ id, type: 'cancel' });
+    }
+    handleMessage(msg) {
+        if (typeof msg.data !== 'string')
+            return;
+        let parsed;
+        try {
+            parsed = JSON.parse(msg.data);
+        }
+        catch {
+            return;
+        }
+        if (parsed.channel) {
+            this.handleChannelMessage(parsed);
+        }
+        else if (parsed.procedure || parsed.type === 'cancel') {
+            this.handleEnvelope(parsed);
+        }
+        else {
+            this.emit('raffel:unknown', parsed);
+        }
+    }
+    handleEnvelope(envelope) {
+        const { id, type, procedure, payload } = envelope;
+        if (type === 'event' && procedure) {
+            this.onEvent?.(procedure, payload);
+            this.emit('raffel:event', procedure, payload);
+            return;
+        }
+        if (type === 'response' || type === 'error') {
+            const baseId = this.extractBaseId(id);
+            const pending = this.pendingCalls.get(baseId);
+            if (!pending)
+                return;
+            if (pending.timer)
+                clearTimeout(pending.timer);
+            this.pendingCalls.delete(baseId);
+            if (type === 'error') {
+                pending.reject(new RaffelError(payload, procedure));
+            }
+            else {
+                pending.resolve(payload);
+            }
+            return;
+        }
+    }
+    handleChannelMessage(msg) {
+        const { type, channel } = msg;
+        switch (type) {
+            case 'subscribed':
+                this.emit('raffel:channel:subscribed', channel, msg.members);
+                break;
+            case 'unsubscribed':
+                this.emit('raffel:channel:unsubscribed', channel);
+                break;
+            case 'event': {
+                const event = msg.event ?? '';
+                const data = msg.data;
+                const handler = this.subscribedChannels.get(channel);
+                if (handler)
+                    handler(event, data);
+                this.emit('raffel:channel:event', channel, event, data);
+                break;
+            }
+        }
+    }
+    nextId() {
+        return `req-${++this.idCounter}`;
+    }
+    extractBaseId(id) {
+        const suffixes = [':response', ':error'];
+        for (const suffix of suffixes) {
+            if (id.endsWith(suffix)) {
+                return id.slice(0, -suffix.length);
+            }
+        }
+        return id;
+    }
+    sendChannelSubscribe(channel) {
+        const msg = {
+            id: this.nextId(),
+            type: 'subscribe',
+            channel,
+        };
+        this.ws.sendJSON(msg);
+    }
+    sendRaw(data) {
+        this.ws.sendJSON(data);
+    }
+}
+export function createRaffelClient(url, options) {
+    return new RaffelClient(url, options);
+}

package/dist/raffel/index.d.ts

@@ -0,0 +1,2 @@
+export * from './client.js';
+export * from './types.js';

package/dist/raffel/index.js

@@ -0,0 +1,2 @@
+export * from './client.js';
+export * from './types.js';

package/dist/raffel/types.d.ts

@@ -0,0 +1,40 @@
+import type { WebSocketOptions } from '../websocket/client.js';
+export interface RaffelEnvelope {
+    id: string;
+    procedure?: string;
+    type: string;
+    payload?: unknown;
+    metadata?: Record<string, unknown>;
+}
+export interface RaffelChannelMessage {
+    id?: string;
+    type: string;
+    channel: string;
+    event?: string;
+    data?: unknown;
+    members?: string[];
+}
+export interface RaffelErrorPayload {
+    code: string;
+    status: number;
+    message: string;
+    details?: unknown;
+}
+export type ChannelEventHandler = (event: string, data: unknown) => void;
+export interface RaffelClientOptions extends WebSocketOptions {
+    channels?: string[];
+    channelHandlers?: Record<string, ChannelEventHandler>;
+    onEvent?: (procedure: string, payload: unknown) => void;
+    defaultTimeout?: number;
+}
+export interface RaffelCallOptions {
+    timeout?: number;
+    signal?: AbortSignal;
+}
+export declare class RaffelError extends Error {
+    code: string;
+    status: number;
+    details?: unknown;
+    procedure?: string;
+    constructor(payload: RaffelErrorPayload, procedure?: string);
+}

package/dist/raffel/types.js

@@ -0,0 +1,14 @@
+export class RaffelError extends Error {
+    code;
+    status;
+    details;
+    procedure;
+    constructor(payload, procedure) {
+        super(payload.message);
+        this.name = 'RaffelError';
+        this.code = payload.code;
+        this.status = payload.status;
+        this.details = payload.details;
+        this.procedure = procedure;
+    }
+}

package/dist/recker.d.ts

@@ -3,6 +3,8 @@ import { getVersion, getVersionInfo } from './version.js';
 import { type RequestPromise } from './core/request-promise.js';
 import type { RequestOptions } from './types/index.js';
 import { type WebSocketOptions, type ReckerWebSocket } from './websocket/client.js';
+import { type RaffelClient } from './raffel/client.js';
+import type { RaffelClientOptions } from './raffel/types.js';
 import { createWhois, type WhoisResult, type WhoisOptions } from './utils/whois.js';
 import { type DNSClientOptions, type DNSClient } from './dns/index.js';
 import type { AIClientConfig } from './types/ai.js';
@@ -11,13 +13,16 @@ import { VideoBuilder } from './video/builder.js';
 import { getExtractorName, listExtractors } from './extractors/index.js';
 import type { ExtractorResult } from './extractors/base.js';
 import { type GoogleSearchAdvancedOptions, type GoogleSearchResponse } from './search/index.js';
-export declare function get(url: string, options?: RequestOptions): RequestPromise;
-export declare function post(url: string, options?: RequestOptions): RequestPromise;
-export declare function put(url: string, options?: RequestOptions): RequestPromise;
-export declare function patch(url: string, options?: RequestOptions): RequestPromise;
-export declare function del(url: string, options?: RequestOptions): RequestPromise;
-export declare function head(url: string, options?: RequestOptions): RequestPromise;
-export declare function options(url: string, options?: RequestOptions): RequestPromise;
+export declare function get<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function post<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function post<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
+export declare function put<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function put<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
+export declare function patch<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function patch<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
+export declare function del<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function head<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
+export declare function options<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
 export declare function whois(query: string, options?: WhoisOptions): Promise<WhoisResult>;
 export declare function whoisAvailable(domain: string): Promise<boolean>;
 export declare function dns(hostname: string, type?: 'A' | 'AAAA' | 'MX' | 'TXT' | 'NS' | 'CNAME'): Promise<string[]>;
@@ -40,6 +45,7 @@ export declare const recker: {
     dns: typeof dns;
     dnsSecurity: typeof dnsSecurity;
     ws: typeof ws;
+    raffel: (url: string, options?: RaffelClientOptions) => RaffelClient;
     video: typeof video;
     isVideoSupported: typeof isVideoSupported;
     extractVideo: typeof extractVideo;

package/dist/recker.js

@@ -1,6 +1,7 @@
 import { createClient } from './core/client.js';
 import { getVersion, getVersionSync, getVersionInfo } from './version.js';
 import { createWebSocket } from './websocket/client.js';
+import { createRaffelClient } from './raffel/client.js';
 import { whois as whoisLookup, isDomainAvailable, createWhois } from './utils/whois.js';
 import { createDNS } from './dns/index.js';
 import { createAI } from './ai/index.js';
@@ -12,6 +13,53 @@ import { searchGoogleAdvanced } from './search/index.js';
 let _defaultClient = null;
 let _defaultDns = null;
 let _defaultAi = null;
+const REQUEST_OPTIONS_HINTS = new Set([
+    'method',
+    'headers',
+    'body',
+    'json',
+    'form',
+    'xml',
+    'yaml',
+    'csv',
+    'timeout',
+    'params',
+    'searchParams',
+    'retry',
+    'hooks',
+    'throwHttpErrors',
+    'signal',
+    'http2',
+    'followRedirects',
+    'maxRedirects',
+    'beforeRedirect',
+    'maxResponseSize',
+    'useCurl',
+    'baseUrl',
+    'base',
+    'proxy',
+    'dns',
+    'tls',
+    'policySource',
+    'policyTags',
+    'traceId',
+    'correlationId',
+    'tenant',
+    'retryOn404',
+    'retryOn5xx',
+]);
+function isRequestOptions(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return false;
+    }
+    return Object.keys(value).some((key) => REQUEST_OPTIONS_HINTS.has(key));
+}
+function resolveRequestBodyAndOptions(bodyOrOptions, options) {
+    if (options === undefined && isRequestOptions(bodyOrOptions)) {
+        return { body: undefined, options: bodyOrOptions };
+    }
+    return { body: bodyOrOptions, options: options || {} };
+}
 function getDefaultClient() {
     if (!_defaultClient) {
         _defaultClient = createClient();
@@ -33,14 +81,17 @@ function getDefaultAi() {
 export function get(url, options) {
     return getDefaultClient().get(url, options);
 }
-export function post(url, options) {
-    return getDefaultClient().post(url, options);
+export function post(url, bodyOrOptions, options) {
+    const resolved = resolveRequestBodyAndOptions(bodyOrOptions, options);
+    return getDefaultClient().post(url, resolved.body, resolved.options);
 }
-export function put(url, options) {
-    return getDefaultClient().put(url, options);
+export function put(url, bodyOrOptions, options) {
+    const resolved = resolveRequestBodyAndOptions(bodyOrOptions, options);
+    return getDefaultClient().put(url, resolved.body, resolved.options);
 }
-export function patch(url, options) {
-    return getDefaultClient().patch(url, options);
+export function patch(url, bodyOrOptions, options) {
+    const resolved = resolveRequestBodyAndOptions(bodyOrOptions, options);
+    return getDefaultClient().patch(url, resolved.body, resolved.options);
 }
 export function del(url, options) {
     return getDefaultClient().delete(url, options);
@@ -100,6 +151,7 @@ export const recker = {
     dns,
     dnsSecurity,
     ws,
+    raffel: (url, options) => createRaffelClient(url, options),
     video,
     isVideoSupported,
     extractVideo,

package/dist/scrape/document.js

@@ -9,11 +9,11 @@ export class ScrapeDocument {
         this.options = options || {};
     }
     static async create(html, options) {
-        const root = parse(html);
+        const root = parse(html, options?.parserOptions);
         return new ScrapeDocument(root, options);
     }
     static createSync(html, options) {
-        const root = parse(html);
+        const root = parse(html, options?.parserOptions);
         return new ScrapeDocument(root, options);
     }
     select(selector) {

package/dist/scrape/element.js

@@ -16,8 +16,14 @@ export class ScrapeElement {
     }
     find(selector) {
         const found = [];
+        const seen = new Set();
         this.elements.forEach((el) => {
-            found.push(...el.querySelectorAll(selector));
+            el.querySelectorAll(selector).forEach((match) => {
+                if (!seen.has(match)) {
+                    seen.add(match);
+                    found.push(match);
+                }
+            });
         });
         return new ScrapeElement(found);
     }

package/dist/scrape/parser/nodes/html.js

@@ -8,7 +8,7 @@ import Node from './node.js';
 import TextNode from './text.js';
 import NodeType from './type.js';
 function decode(val) {
-    return JSON.parse(JSON.stringify(he.decode(val)));
+    return he.decode(val);
 }
 const Htags = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup'];
 const Dtags = ['details', 'dialog', 'dd', 'div', 'dt'];

package/dist/scrape/spider.d.ts

@@ -1,6 +1,9 @@
 import type { ExtractedLink, ExtractionSchema } from './types.js';
+import type { Options as ParserOptions } from './parser/index.js';
 import { type SitemapUrl } from '../seo/validators/sitemap.js';
+import { type CaptchaDetectionResult, type BlockDetectionResult } from '../utils/block-detector.js';
 export type SpiderTransport = 'auto' | 'undici' | 'curl';
+type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
     maxDepth?: number;
     maxPages?: number;
@@ -11,14 +14,31 @@ export interface SpiderOptions {
     exclude?: RegExp[];
     include?: RegExp[];
     userAgent?: string;
+    rotateUserAgent?: boolean;
+    randomizeHeaders?: boolean;
+    maxRetryAttempts?: number;
+    baseRetryDelayMs?: number;
+    maxRetryDelayMs?: number;
+    retryBackoffMultiplier?: number;
+    retryJitterMs?: number;
+    maxDomainBlockStrikes?: number;
     respectRobotsTxt?: boolean;
     useSitemap?: boolean;
     sitemapUrl?: string;
     transport?: SpiderTransport;
+    preferCurlFirst?: boolean;
     onPage?: (result: SpiderPageResult) => void;
     onPageWithHtml?: (result: SpiderPageResult, html: string) => void | Promise<void>;
+    onCaptchaDetected?: (result: {
+        url: string;
+        status: number;
+        confidence: number;
+        provider?: CaptchaProvider;
+        usedCurl: boolean;
+    }) => void | Promise<void>;
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
+    parserOptions?: Partial<ParserOptions>;
 }
 export interface SpiderPageResult {
     url: string;
@@ -56,6 +76,25 @@ export interface SpiderPageResult {
         total?: number;
     };
     fetchedAt?: number;
+    security?: {
+        blocked: boolean;
+        reason?: BlockDetectionResult['reason'];
+        confidence?: number;
+        captchaDetected?: boolean;
+        captchaProvider?: CaptchaProvider;
+        attempts?: number;
+        retryCount?: number;
+        retryAfterMs?: number;
+        transport?: SpiderTransport;
+        blockedByCaptcha?: boolean;
+        lastStatus?: number;
+        lastTransport?: SpiderTransport;
+    };
+    resources?: {
+        images: number;
+        scripts: number;
+        stylesheets: number;
+    };
     extracted?: Record<string, unknown>;
 }
 export interface SpiderProgress {
@@ -118,6 +157,7 @@ export declare class Spider {
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
+    private domainStates;
     private sitemapUrls;
     private sitemapUrlSet;
     private robotsData;
@@ -132,8 +172,20 @@ export declare class Spider {
     private buildRobotsAnalysis;
     private fetchPage;
     private crawlPage;
+    private getOrCreateDomainState;
+    private recordTransportResult;
+    private isRetryableStatus;
+    private buildRequestHeaders;
+    private shouldUseCurlForHost;
+    private waitForDomainPenalty;
+    private registerDomainBlock;
+    private registerDomainSuccess;
+    private getCaptchaRetryMultiplier;
+    private registerDomainChallenge;
+    private getRetryWait;
     abort(): void;
     isRunning(): boolean;
     getProgress(): SpiderProgress;
 }
 export declare function spider(url: string, options?: SpiderOptions): Promise<SpiderResult>;
+export {};