recker 1.0.26 → 1.0.27

package/dist/browser/plugins/rate-limit.js

@@ -0,0 +1,162 @@
+export class RateLimitExceededError extends Error {
+    limit;
+    window;
+    key;
+    constructor(limit, window, key) {
+        super(`Rate limit exceeded for ${key}: ${limit} requests per ${window}ms`);
+        this.limit = limit;
+        this.window = window;
+        this.key = key;
+        this.name = 'RateLimitExceededError';
+    }
+}
+function parseRateLimitHeaders(headers) {
+    const result = {};
+    const remaining = headers.get('x-ratelimit-remaining') || headers.get('ratelimit-remaining');
+    if (remaining !== null) {
+        result.remaining = parseInt(remaining, 10);
+    }
+    const reset = headers.get('x-ratelimit-reset') || headers.get('ratelimit-reset');
+    if (reset !== null) {
+        const val = parseFloat(reset);
+        if (val < 1000000000) {
+            result.reset = Date.now() + (val * 1000);
+        }
+        else {
+            result.reset = val * 1000;
+        }
+    }
+    const retryAfter = headers.get('retry-after');
+    if (retryAfter !== null) {
+        if (/^\d+$/.test(retryAfter)) {
+            result.retryAfter = parseInt(retryAfter, 10) * 1000;
+        }
+        else {
+            const date = Date.parse(retryAfter);
+            if (!isNaN(date)) {
+                result.retryAfter = Math.max(0, date - Date.now());
+            }
+        }
+    }
+    return result;
+}
+export function rateLimitPlugin(options) {
+    const limit = options.limit;
+    const windowMs = options.window || 1000;
+    const strategy = options.strategy || 'queue';
+    const adaptive = options.adaptive || false;
+    const buckets = new Map();
+    const getKey = options.keyGenerator || ((req) => {
+        try {
+            return new URL(req.url).hostname;
+        }
+        catch {
+            return 'global';
+        }
+    });
+    const refillBucket = (bucket) => {
+        const now = Date.now();
+        if (now < bucket.blockedUntil) {
+            return;
+        }
+        const elapsed = now - bucket.lastRefill;
+        if (elapsed > windowMs) {
+            bucket.tokens = limit;
+            bucket.lastRefill = now;
+        }
+    };
+    const processQueue = (bucket) => {
+        const now = Date.now();
+        if (now < bucket.blockedUntil) {
+            const wait = bucket.blockedUntil - now;
+            setTimeout(() => processQueue(bucket), wait);
+            return;
+        }
+        refillBucket(bucket);
+        while (bucket.queue.length > 0 && bucket.tokens > 0) {
+            bucket.tokens--;
+            const next = bucket.queue.shift();
+            if (next)
+                next.resolve();
+        }
+        if (bucket.queue.length > 0) {
+            const timeToNextRefill = windowMs - (Date.now() - bucket.lastRefill);
+            setTimeout(() => processQueue(bucket), Math.max(0, timeToNextRefill));
+        }
+    };
+    return (client) => {
+        const middleware = async (req, next) => {
+            const key = getKey(req);
+            let bucket = buckets.get(key);
+            if (!bucket) {
+                bucket = {
+                    tokens: limit,
+                    lastRefill: Date.now(),
+                    queue: [],
+                    blockedUntil: 0
+                };
+                buckets.set(key, bucket);
+            }
+            const now = Date.now();
+            if (now < bucket.blockedUntil) {
+                bucket.tokens = 0;
+            }
+            else {
+                refillBucket(bucket);
+            }
+            if (bucket.tokens > 0) {
+                bucket.tokens--;
+                const response = await next(req);
+                if (adaptive) {
+                    adaptLimits(bucket, response);
+                }
+                return response;
+            }
+            if (strategy === 'throw') {
+                throw new RateLimitExceededError(limit, windowMs, key);
+            }
+            if (strategy === 'drop') {
+                throw new Error(`Request dropped due to rate limit (${key})`);
+            }
+            return new Promise((resolve, reject) => {
+                bucket.queue.push({
+                    resolve: async () => {
+                        try {
+                            const response = await next(req);
+                            if (adaptive) {
+                                adaptLimits(bucket, response);
+                            }
+                            resolve(response);
+                        }
+                        catch (err) {
+                            reject(err);
+                        }
+                    },
+                    reject
+                });
+                if (bucket.queue.length === 1) {
+                    const timeToNextRefill = windowMs - (Date.now() - bucket.lastRefill);
+                    setTimeout(() => processQueue(bucket), Math.max(0, timeToNextRefill));
+                }
+            });
+        };
+        function adaptLimits(bucket, response) {
+            const limits = parseRateLimitHeaders(response.headers);
+            let backoffTime = 0;
+            if (limits.retryAfter) {
+                backoffTime = limits.retryAfter;
+            }
+            else if (limits.remaining !== undefined && limits.remaining <= 0 && limits.reset) {
+                backoffTime = Math.max(0, limits.reset - Date.now());
+            }
+            if (backoffTime > 0) {
+                bucket.blockedUntil = Date.now() + backoffTime;
+                bucket.tokens = 0;
+                if (bucket.queue.length > 0) {
+                    setTimeout(() => processQueue(bucket), 0);
+                }
+            }
+        }
+        client.use(middleware);
+    };
+}

package/dist/browser/plugins/retry.d.ts

@@ -0,0 +1,14 @@
+import { Plugin } from '../types/index.js';
+export type BackoffStrategy = 'linear' | 'exponential' | 'decorrelated';
+export interface RetryOptions {
+    maxAttempts?: number;
+    delay?: number;
+    maxDelay?: number;
+    backoff?: BackoffStrategy;
+    jitter?: boolean;
+    statusCodes?: number[];
+    shouldRetry?: (error: unknown) => boolean;
+    onRetry?: (attempt: number, error: unknown, delay: number) => void;
+    respectRetryAfter?: boolean;
+}
+export declare function retryPlugin(options?: RetryOptions): Plugin;

package/dist/browser/plugins/retry.js

@@ -0,0 +1,116 @@
+import { HttpError, NetworkError, TimeoutError } from '../core/errors.js';
+function calculateDelay(attempt, baseDelay, maxDelay, strategy, useJitter) {
+    let calculatedDelay;
+    switch (strategy) {
+        case 'linear':
+            calculatedDelay = baseDelay * attempt;
+            break;
+        case 'exponential':
+            calculatedDelay = Math.pow(2, attempt - 1) * baseDelay;
+            break;
+        case 'decorrelated':
+            const prevDelay = attempt === 1 ? baseDelay : Math.pow(2, attempt - 2) * baseDelay;
+            calculatedDelay = Math.random() * (prevDelay * 3 - baseDelay) + baseDelay;
+            break;
+        default:
+            calculatedDelay = baseDelay * attempt;
+    }
+    calculatedDelay = Math.min(calculatedDelay, maxDelay);
+    if (useJitter) {
+        const jitterRange = calculatedDelay * 0.25;
+        const jitterAmount = (Math.random() * jitterRange * 2) - jitterRange;
+        calculatedDelay += jitterAmount;
+    }
+    return Math.max(0, Math.floor(calculatedDelay));
+}
+function parseRetryAfter(headerValue) {
+    if (!headerValue)
+        return undefined;
+    const seconds = parseInt(headerValue, 10);
+    if (!isNaN(seconds) && seconds >= 0) {
+        return seconds * 1000;
+    }
+    const date = Date.parse(headerValue);
+    if (!isNaN(date)) {
+        const delay = date - Date.now();
+        return delay > 0 ? delay : undefined;
+    }
+    return undefined;
+}
+export function retryPlugin(options = {}) {
+    const maxAttempts = options.maxAttempts || 3;
+    const baseDelay = options.delay || 1000;
+    const maxDelay = options.maxDelay || 30000;
+    const backoffStrategy = options.backoff || 'exponential';
+    const useJitter = options.jitter !== false;
+    const statusCodes = options.statusCodes || [408, 429, 500, 502, 503, 504];
+    const onRetry = options.onRetry;
+    const respectRetryAfter = options.respectRetryAfter !== false;
+    const defaultShouldRetry = (error) => {
+        if (error instanceof NetworkError)
+            return true;
+        if (error instanceof TimeoutError)
+            return true;
+        if (error instanceof HttpError) {
+            return statusCodes.includes(error.status);
+        }
+        if (error && typeof error === 'object' && 'code' in error) {
+            const code = error.code;
+            return code === 'ECONNRESET' || code === 'ETIMEDOUT' || code === 'ENOTFOUND';
+        }
+        return false;
+    };
+    const shouldRetry = options.shouldRetry || defaultShouldRetry;
+    return (client) => {
+        const middleware = async (req, next) => {
+            let attempt = 0;
+            while (true) {
+                try {
+                    attempt++;
+                    const res = await next(req);
+                    if (attempt < maxAttempts && !res.ok && statusCodes.includes(res.status)) {
+                        let delayMs;
+                        if (respectRetryAfter) {
+                            const retryAfterDelay = parseRetryAfter(res.headers.get('Retry-After'));
+                            delayMs = retryAfterDelay !== undefined
+                                ? Math.min(retryAfterDelay, maxDelay)
+                                : calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
+                        }
+                        else {
+                            delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
+                        }
+                        const err = new HttpError(res, req);
+                        if (onRetry) {
+                            onRetry(attempt, err, delayMs);
+                        }
+                        if (client.hooks?.onRetry) {
+                            for (const hook of client.hooks.onRetry) {
+                                await hook(err, attempt, delayMs, req);
+                            }
+                        }
+                        await new Promise(resolve => setTimeout(resolve, delayMs));
+                        continue;
+                    }
+                    return res;
+                }
+                catch (error) {
+                    if (attempt < maxAttempts && shouldRetry(error)) {
+                        const delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
+                        if (onRetry) {
+                            onRetry(attempt, error, delayMs);
+                        }
+                        if (client.hooks?.onRetry) {
+                            for (const hook of client.hooks.onRetry) {
+                                await hook(error, attempt, delayMs, req);
+                            }
+                        }
+                        await new Promise(resolve => setTimeout(resolve, delayMs));
+                        continue;
+                    }
+                    throw error;
+                }
+            }
+        };
+        client.use(middleware);
+    };
+}

package/dist/browser/plugins/scrape.d.ts

@@ -0,0 +1,21 @@
+import type { ReckerResponse } from '../types/index.js';
+import type { RequestPromise } from '../core/request-promise.js';
+import type { ScrapeOptions, ExtractionSchema, ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedTable, ExtractedScript, ExtractedStyle, LinkExtractionOptions, ImageExtractionOptions } from '../scrape/types.js';
+export interface ScrapePromise<T> extends Promise<T> {
+    scrape(options?: ScrapeOptions): Promise<import('../scrape/document.js').ScrapeDocument>;
+    links(options?: LinkExtractionOptions): Promise<ExtractedLink[]>;
+    images(options?: ImageExtractionOptions): Promise<ExtractedImage[]>;
+    meta(): Promise<ExtractedMeta>;
+    openGraph(): Promise<OpenGraphData>;
+    twitterCard(): Promise<TwitterCardData>;
+    jsonLd(): Promise<JsonLdData[]>;
+    forms(selector?: string): Promise<ExtractedForm[]>;
+    tables(selector?: string): Promise<ExtractedTable[]>;
+    scripts(): Promise<ExtractedScript[]>;
+    styles(): Promise<ExtractedStyle[]>;
+    extract<R extends Record<string, unknown>>(schema: ExtractionSchema): Promise<R>;
+}
+export declare function scrape<T extends ReckerResponse>(promise: RequestPromise<T> | Promise<T>): ScrapePromise<T>;
+export declare function parseHtml(html: string, options?: ScrapeOptions): Promise<import('../scrape/document.js').ScrapeDocument>;
+export declare function scrapeResponse(promise: Promise<ReckerResponse>, options?: ScrapeOptions): Promise<import('../scrape/document.js').ScrapeDocument>;
+export type { ScrapeOptions, ExtractionSchema, ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedFormField, ExtractedTable, ExtractedScript, ExtractedStyle, LinkExtractionOptions, ImageExtractionOptions, } from '../scrape/types.js';

package/dist/browser/plugins/scrape.js

@@ -0,0 +1,82 @@
+let ScrapeDocumentClass = null;
+async function getScrapeDocumentClass() {
+    if (!ScrapeDocumentClass) {
+        const module = await import('../scrape/document.js');
+        ScrapeDocumentClass = module.ScrapeDocument;
+    }
+    return ScrapeDocumentClass;
+}
+export function scrape(promise) {
+    const basePromise = Promise.resolve(promise);
+    const getDocument = async (options) => {
+        const ScrapeDoc = await getScrapeDocumentClass();
+        const response = await basePromise;
+        const html = await response.text();
+        return ScrapeDoc.create(html, {
+            baseUrl: options?.baseUrl || response.url,
+            ...options,
+        });
+    };
+    const enhanced = basePromise;
+    enhanced.scrape = async (options) => {
+        return getDocument(options);
+    };
+    enhanced.links = async (options) => {
+        const doc = await getDocument();
+        return doc.links(options);
+    };
+    enhanced.images = async (options) => {
+        const doc = await getDocument();
+        return doc.images(options);
+    };
+    enhanced.meta = async () => {
+        const doc = await getDocument();
+        return doc.meta();
+    };
+    enhanced.openGraph = async () => {
+        const doc = await getDocument();
+        return doc.openGraph();
+    };
+    enhanced.twitterCard = async () => {
+        const doc = await getDocument();
+        return doc.twitterCard();
+    };
+    enhanced.jsonLd = async () => {
+        const doc = await getDocument();
+        return doc.jsonLd();
+    };
+    enhanced.forms = async (selector) => {
+        const doc = await getDocument();
+        return doc.forms(selector);
+    };
+    enhanced.tables = async (selector) => {
+        const doc = await getDocument();
+        return doc.tables(selector);
+    };
+    enhanced.scripts = async () => {
+        const doc = await getDocument();
+        return doc.scripts();
+    };
+    enhanced.styles = async () => {
+        const doc = await getDocument();
+        return doc.styles();
+    };
+    enhanced.extract = async (schema) => {
+        const doc = await getDocument();
+        return doc.extract(schema);
+    };
+    return enhanced;
+}
+export async function parseHtml(html, options) {
+    const ScrapeDoc = await getScrapeDocumentClass();
+    return ScrapeDoc.create(html, options);
+}
+export async function scrapeResponse(promise, options) {
+    const ScrapeDoc = await getScrapeDocumentClass();
+    const response = await promise;
+    const html = await response.text();
+    return ScrapeDoc.create(html, {
+        baseUrl: options?.baseUrl || response.url,
+        ...options,
+    });
+}

package/dist/browser/plugins/server-timing.d.ts

@@ -0,0 +1,7 @@
+import { Plugin } from '../types/index.js';
+export interface ServerTiming {
+    name: string;
+    duration?: number;
+    description?: string;
+}
+export declare function serverTimingPlugin(): Plugin;

package/dist/browser/plugins/server-timing.js

@@ -0,0 +1,24 @@
+export function serverTimingPlugin() {
+    return (client) => {
+        client.afterResponse((req, res) => {
+            const header = res.headers.get('server-timing');
+            if (!header)
+                return;
+            const timings = header.split(',').map(entry => {
+                const parts = entry.split(';');
+                const name = parts[0].trim();
+                let duration;
+                let description;
+                for (let i = 1; i < parts.length; i++) {
+                    const [key, val] = parts[i].split('=').map(s => s.trim());
+                    if (key === 'dur')
+                        duration = parseFloat(val);
+                    if (key === 'desc')
+                        description = val?.replace(/"/g, '');
+                }
+                return { name, duration, description };
+            });
+            res.serverTimings = timings;
+        });
+    };
+}

package/dist/browser/plugins/soap.d.ts

@@ -0,0 +1,72 @@
+import type { Client } from '../core/client.js';
+import type { RequestOptions } from '../types/index.js';
+export interface XmlRpcValue {
+    type: 'int' | 'i4' | 'i8' | 'boolean' | 'string' | 'double' | 'dateTime.iso8601' | 'base64' | 'array' | 'struct' | 'nil';
+    value: unknown;
+}
+export interface XmlRpcResponse<T = unknown> {
+    success: boolean;
+    result?: T;
+    fault?: {
+        faultCode: number;
+        faultString: string;
+    };
+}
+export declare class XmlRpcClient {
+    private client;
+    private endpoint;
+    private requestOptions;
+    constructor(client: Client, options: {
+        endpoint: string;
+        requestOptions?: RequestOptions;
+    });
+    call<T = unknown>(method: string, params?: unknown[]): Promise<XmlRpcResponse<T>>;
+    private parseResponse;
+}
+export type SoapVersion = '1.1' | '1.2';
+export interface SoapOptions {
+    endpoint: string;
+    version?: SoapVersion;
+    namespace?: string;
+    namespacePrefix?: string;
+    wsdl?: string;
+    soapHeaders?: Record<string, unknown>;
+    requestOptions?: RequestOptions;
+}
+export interface SoapFault {
+    code: string;
+    string: string;
+    actor?: string;
+    detail?: unknown;
+}
+export interface SoapResponse<T = unknown> {
+    success: boolean;
+    result?: T;
+    fault?: SoapFault;
+    rawXml: string;
+}
+export declare class SoapClient {
+    private client;
+    private options;
+    constructor(client: Client, options: SoapOptions);
+    call<T = unknown>(method: string, params?: Record<string, unknown>, options?: {
+        soapHeaders?: Record<string, unknown>;
+        soapAction?: string;
+    }): Promise<SoapResponse<T>>;
+    getWsdl(): Promise<string | null>;
+    private objectToXml;
+    private parseResponse;
+    private parseXmlToObject;
+}
+export declare function createXmlRpcClient(client: Client, options: {
+    endpoint: string;
+    requestOptions?: RequestOptions;
+}): XmlRpcClient;
+export declare function createSoapClient(client: Client, options: SoapOptions): SoapClient;
+export declare function soap(): (client: Client) => void;
+declare module '../core/client.js' {
+    interface Client {
+        soap(options: SoapOptions): SoapClient;
+        xmlrpc(endpoint: string, requestOptions?: RequestOptions): XmlRpcClient;
+    }
+}