recker 1.0.72 → 1.0.75-next.2e5a94f

package/dist/browser/plugins/auth/aws-sigv4.d.ts

@@ -8,3 +8,4 @@ export interface AWSSignatureV4Options {
 }
 export declare function awsSignatureV4(options: AWSSignatureV4Options): Middleware;
 export declare function awsSignatureV4Plugin(options: AWSSignatureV4Options): Plugin;
+export declare function clearAwsSigV4PluginCache(): void;

package/dist/browser/plugins/auth/aws-sigv4.js

@@ -81,8 +81,25 @@ export function awsSignatureV4(options) {
         return next(newReq);
     };
 }
+const awsSigV4PluginCache = new Map();
 export function awsSignatureV4Plugin(options) {
-    return (client) => {
-        client.use(awsSignatureV4(options));
+    const cacheKey = [
+        options.accessKeyId,
+        options.secretAccessKey,
+        options.region,
+        options.service,
+        options.sessionToken || '',
+    ].join('|');
+    const cached = awsSigV4PluginCache.get(cacheKey);
+    if (cached)
+        return cached;
+    const safeOptions = { ...options };
+    const plugin = (client) => {
+        client.use(awsSignatureV4(safeOptions));
     };
+    awsSigV4PluginCache.set(cacheKey, plugin);
+    return plugin;
+}
+export function clearAwsSigV4PluginCache() {
+    awsSigV4PluginCache.clear();
 }

package/dist/browser/plugins/retry.js

@@ -1,4 +1,5 @@
-import { HttpError, NetworkError, TimeoutError } from '../core/errors.js';
+import { HttpError, NetworkError, TimeoutError, classifyTransportError } from '../core/errors.js';
+import { getRequestContext } from '../core-runtime/request-context.js';
 function calculateDelay(attempt, baseDelay, maxDelay, strategy, useJitter) {
     let calculatedDelay;
     switch (strategy) {
@@ -54,6 +55,13 @@ export function retryPlugin(options = {}) {
         if (error instanceof HttpError) {
             return statusCodes.includes(error.status);
         }
+        const classification = classifyTransportError(error);
+        if (classification && classification.canRetry === false) {
+            return false;
+        }
+        if (classification) {
+            return classification.canRetry;
+        }
         if (error && typeof error === 'object' && 'code' in error) {
             const code = error.code;
             return code === 'ECONNRESET' || code === 'ETIMEDOUT' || code === 'ENOTFOUND';
@@ -62,6 +70,24 @@ export function retryPlugin(options = {}) {
     };
     const shouldRetry = options.shouldRetry || defaultShouldRetry;
     return (client) => {
+        const emitRequestRetry = (error, attempt, delayMs, req) => {
+            const eventBus = client.runtimeEventBus;
+            if (!eventBus?.emit) {
+                return;
+            }
+            const context = getRequestContext(req);
+            if (!context) {
+                return;
+            }
+            const classification = classifyTransportError(error);
+            eventBus.emit('request:retry', {
+                context,
+                req,
+                attempt,
+                delayMs,
+                reason: classification?.reason || error?.message || 'retry requested'
+            });
+        };
         const middleware = async (req, next) => {
             let attempt = 0;
             while (true) {
@@ -80,6 +106,7 @@ export function retryPlugin(options = {}) {
                             delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
                         }
                         const err = new HttpError(res, req);
+                        emitRequestRetry(err, attempt, delayMs, req);
                         if (onRetry) {
                             onRetry(attempt, err, delayMs);
                         }
@@ -96,6 +123,7 @@ export function retryPlugin(options = {}) {
                 catch (error) {
                     if (attempt < maxAttempts && shouldRetry(error)) {
                         const delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
+                        emitRequestRetry(error, attempt, delayMs, req);
                         if (onRetry) {
                             onRetry(attempt, error, delayMs);
                         }

package/dist/browser/presets/aws.d.ts

@@ -10,6 +10,7 @@ export interface AWSPresetOptions {
 export type AWSService = 's3' | 'dynamodb' | 'lambda' | 'sqs' | 'sns' | 'ses' | 'secretsmanager' | 'ssm' | 'sts' | 'iam' | 'ec2' | 'ecs' | 'eks' | 'cloudwatch' | 'logs' | 'events' | 'kinesis' | 'firehose' | 'apigateway' | 'execute-api' | 'cognito-idp' | 'cognito-identity' | 'kms' | 'athena' | 'glue' | 'stepfunctions' | 'states' | 'bedrock' | 'bedrock-runtime';
 export declare function aws(options: AWSPresetOptions): ClientOptions;
 export declare function awsS3(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
+export declare function s3(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
 export declare function awsDynamoDB(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
 export declare function awsLambda(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
 export declare function awsSQS(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;

package/dist/browser/presets/aws.js

@@ -1,4 +1,62 @@
 import { awsSignatureV4Plugin } from '../plugins/auth.js';
+const S3_STORAGE_PRESET = {
+    timeout: {
+        lookup: 5000,
+        connect: 10000,
+        secureConnect: 10000,
+        response: 30000,
+        request: 60000,
+    },
+    http2: 'performance',
+    expectContinue: 2 * 1024 * 1024,
+    observability: false,
+    concurrency: {
+        agent: {
+            connections: 100,
+            pipelining: 10,
+            keepAlive: true,
+            keepAliveTimeout: 4000,
+            keepAliveMaxTimeout: 600000,
+            keepAliveTimeoutThreshold: 1000,
+            maxRequestsPerClient: 100,
+            clientTtl: null,
+            maxCachedSessions: 100,
+        },
+    },
+    retry: {
+        maxDelay: 30000,
+    },
+};
+function createAwsS3Preset(options) {
+    const baseConfig = aws({
+        ...options,
+        service: 's3',
+    });
+    const baseConcurrency = baseConfig.concurrency;
+    let concurrency;
+    if (typeof baseConcurrency === 'number') {
+        concurrency = baseConcurrency;
+    }
+    else {
+        const mergedConcurrency = {};
+        if (baseConcurrency && typeof baseConcurrency === 'object') {
+            Object.assign(mergedConcurrency, S3_STORAGE_PRESET.concurrency, baseConcurrency);
+        }
+        else {
+            Object.assign(mergedConcurrency, S3_STORAGE_PRESET.concurrency);
+        }
+        concurrency = mergedConcurrency;
+    }
+    return {
+        ...baseConfig,
+        ...S3_STORAGE_PRESET,
+        retry: {
+            ...baseConfig.retry,
+            ...S3_STORAGE_PRESET.retry,
+        },
+        concurrency,
+    };
+}
 function getServiceEndpoint(service, region) {
     const globalServices = ['iam', 'sts', 'cloudfront', 'route53'];
     if (globalServices.includes(service)) {
@@ -43,7 +101,10 @@ export function aws(options) {
     };
 }
 export function awsS3(options) {
-    return aws({ ...options, service: 's3' });
+    return createAwsS3Preset(options);
+}
+export function s3(options) {
+    return createAwsS3Preset(options);
 }
 export function awsDynamoDB(options) {
     return aws({ ...options, service: 'dynamodb' });

package/dist/browser/runner/request-runner.d.ts

@@ -3,6 +3,12 @@ export interface RunnerOptions {
     retries?: number;
     retryDelay?: number;
 }
+export interface RunnerRunOptions {
+    priority?: number;
+    retries?: number;
+    signal?: AbortSignal;
+    deadlineMs?: number;
+}
 export interface RequestTask<T = any> {
     id: string;
     fn: () => Promise<T>;
@@ -35,10 +41,14 @@ export declare class RequestRunner extends SimpleEmitter {
     private queue;
     private activeCount;
     private paused;
-    private results;
     private stats;
     private startTime;
     private pendingRetries;
+    private isCancelled;
+    private cancelReason;
+    private retryTimers;
+    private timeoutId?;
+    private abortUnsubscribe?;
     constructor(options?: RunnerOptions);
     add<T>(fn: () => Promise<T>, options?: {
         priority?: number;
@@ -47,13 +57,13 @@ export declare class RequestRunner extends SimpleEmitter {
         trackTotal?: boolean;
         resolve?: (value: T | Error) => void;
     }): void;
-    run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: {
-        priority?: number;
-        retries?: number;
-    }): Promise<RunnerResult<T>>;
+    run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: RunnerRunOptions): Promise<RunnerResult<T>>;
     private queueTask;
     private scheduleRetry;
     private processNext;
+    private cancelAll;
+    private resolveQueue;
+    private resolveTask;
     getProgress(): {
         total: number;
         completed: number;

package/dist/browser/runner/request-runner.js

@@ -1,3 +1,4 @@
+import { QueueCancelledError } from '../core/errors.js';
 class SimpleEmitter {
     listeners = new Map();
     on(event, listener) {
@@ -43,10 +44,14 @@ export class RequestRunner extends SimpleEmitter {
     queue = [];
     activeCount = 0;
     paused = false;
-    results = new Map();
     stats = { total: 0, successful: 0, failed: 0 };
     startTime = 0;
     pendingRetries = 0;
+    isCancelled = false;
+    cancelReason = new QueueCancelledError('Request runner cancelled');
+    retryTimers = new Map();
+    timeoutId;
+    abortUnsubscribe;
     constructor(options = {}) {
         super();
         this.concurrency = options.concurrency || 5;
@@ -67,48 +72,146 @@ export class RequestRunner extends SimpleEmitter {
         this.processNext();
     }
     async run(items, processor, options = {}) {
+        this.queue = [];
+        this.activeCount = 0;
+        this.pendingRetries = 0;
+        this.stats = { total: 0, successful: 0, failed: 0 };
+        this.isCancelled = false;
+        if (this.retryTimers.size > 0) {
+            for (const [, timer] of this.retryTimers) {
+                clearTimeout(timer);
+            }
+            this.retryTimers.clear();
+        }
+        if (this.timeoutId) {
+            clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        if (this.abortUnsubscribe) {
+            this.abortUnsubscribe();
+            this.abortUnsubscribe = undefined;
+        }
         this.startTime = Date.now();
         this.stats = { total: items.length, successful: 0, failed: 0 };
-        this.results.clear();
-        const promises = items.map((item, index) => {
-            return new Promise((resolve) => {
-                this.add(() => processor(item, index), {
-                    priority: options.priority,
-                    id: String(index),
-                    retries: options.retries,
-                    resolve,
-                    trackTotal: false
+        this.isCancelled = false;
+        this.cancelReason = new QueueCancelledError('Request runner cancelled', {
+            queueName: 'request-runner',
+            request: undefined
+        });
+        if (options.signal) {
+            const signal = options.signal;
+            if (signal.aborted) {
+                const reason = signal.reason instanceof Error
+                    ? signal.reason
+                    : new QueueCancelledError('Request runner signal was aborted', {
+                        queueName: 'request-runner',
+                        request: undefined
+                    });
+                this.cancelAll(reason);
+            }
+            else {
+                const handleAbort = () => {
+                    const reason = signal.reason instanceof Error
+                        ? signal.reason
+                        : new QueueCancelledError('Request runner signal was aborted', {
+                            queueName: 'request-runner',
+                            request: undefined
+                        });
+                    this.cancelAll(reason);
+                };
+                signal.addEventListener('abort', handleAbort, { once: true });
+                this.abortUnsubscribe = () => signal.removeEventListener('abort', handleAbort);
+            }
+        }
+        if (options.deadlineMs !== undefined) {
+            const deadline = options.deadlineMs;
+            if (deadline <= 0) {
+                this.cancelAll(new QueueCancelledError('Request runner deadline elapsed', {
+                    queueName: 'request-runner',
+                    request: undefined
+                }));
+            }
+            else {
+                this.timeoutId = setTimeout(() => {
+                    this.cancelAll(new QueueCancelledError('Request runner deadline exceeded', {
+                        queueName: 'request-runner',
+                        request: undefined
+                    }));
+                }, deadline);
+            }
+        }
+        try {
+            const promises = items.map((item, index) => {
+                return new Promise((resolve) => {
+                    this.add(() => processor(item, index), {
+                        priority: options.priority,
+                        id: String(index),
+                        retries: options.retries,
+                        resolve,
+                        trackTotal: false
+                    });
                 });
             });
-        });
-        const results = await Promise.all(promises);
-        return {
-            results,
-            stats: {
-                ...this.stats,
-                duration: Date.now() - this.startTime
+            const results = await Promise.all(promises);
+            return {
+                results,
+                stats: {
+                    ...this.stats,
+                    duration: Date.now() - this.startTime
+                }
+            };
+        }
+        finally {
+            if (this.timeoutId) {
+                clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
             }
-        };
+            if (this.abortUnsubscribe) {
+                this.abortUnsubscribe();
+                this.abortUnsubscribe = undefined;
+            }
+        }
     }
     queueTask(task) {
         this.queue.push(task);
         this.queue.sort((a, b) => b.priority - a.priority);
     }
     scheduleRetry(task, delay) {
-        this.pendingRetries++;
-        if (delay > 0) {
-            setTimeout(() => {
-                this.pendingRetries--;
-                this.queueTask(task);
-                this.processNext();
-            }, delay);
+        if (this.isCancelled) {
+            this.resolveTask(task, this.cancelReason);
             return;
         }
-        this.pendingRetries--;
-        this.queueTask(task);
-        this.processNext();
+        if (delay <= 0) {
+            this.queueTask(task);
+            this.processNext();
+            return;
+        }
+        const timerKey = `${task.id}-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+        const enqueueTask = () => {
+            this.retryTimers.delete(timerKey);
+            this.pendingRetries--;
+            if (this.isCancelled) {
+                return;
+            }
+            this.queueTask(task);
+            this.processNext();
+        };
+        this.retryTimers.set(timerKey, setTimeout(() => {
+            if (this.isCancelled) {
+                this.pendingRetries--;
+                this.retryTimers.delete(timerKey);
+                this.resolveTask(task, this.cancelReason);
+                return;
+            }
+            enqueueTask();
+        }, delay));
+        this.pendingRetries++;
     }
     async processNext() {
+        if (this.isCancelled) {
+            this.resolveQueue();
+            return;
+        }
         if (this.paused || this.activeCount >= this.concurrency || this.queue.length === 0) {
             return;
         }
@@ -124,6 +227,10 @@ export class RequestRunner extends SimpleEmitter {
             this.emit('taskComplete', { task, result });
         }
         catch (error) {
+            if (this.isCancelled) {
+                this.resolveTask(task, this.cancelReason);
+                return;
+            }
             const remaining = task.retries ?? 0;
             if (remaining > 0) {
                 task.retries = remaining - 1;
@@ -131,8 +238,7 @@ export class RequestRunner extends SimpleEmitter {
                 this.scheduleRetry(task, this.retryDelay);
             }
             else {
-                this.stats.failed++;
-                task.resolve?.(error);
+                this.resolveTask(task, error);
                 this.emit('taskError', { task, error });
             }
         }
@@ -145,6 +251,34 @@ export class RequestRunner extends SimpleEmitter {
             this.processNext();
         }
     }
+    cancelAll(reason) {
+        if (this.isCancelled) {
+            return;
+        }
+        this.isCancelled = true;
+        this.cancelReason = reason;
+        for (const [, timer] of this.retryTimers) {
+            clearTimeout(timer);
+        }
+        this.retryTimers.clear();
+        this.pendingRetries = 0;
+        this.resolveQueue();
+    }
+    resolveQueue() {
+        while (this.queue.length > 0) {
+            const task = this.queue.shift();
+            if (task) {
+                this.resolveTask(task, this.cancelReason);
+            }
+        }
+    }
+    resolveTask(task, error) {
+        if (!task.resolve) {
+            return;
+        }
+        this.stats.failed++;
+        task.resolve?.(error);
+    }
     getProgress() {
         const completed = this.stats.successful + this.stats.failed;
         return {

package/dist/browser/scrape/parser/nodes/html.d.ts

@@ -33,11 +33,16 @@ export default class HTMLElement extends Node {
     private voidTag;
     private _attrs?;
     private _rawAttrs?;
+    private _queryCache?;
     private _parseOptions;
     rawTagName: string;
     id: string;
     classList: DOMTokenList;
     nodeType: NodeType;
+    private get isSelectorCacheEnabled();
+    private getQueryCache;
+    private clearQueryCache;
+    invalidateSelectorCacheRecursively(): void;
     private quoteAttribute;
     constructor(tagName: string, keyAttrs: KeyAttributes, rawAttrs?: string, parentNode?: HTMLElement | null, range?: [number, number], voidTag?: VoidTag, _parseOptions?: Partial<Options>);
     removeChild(node: Node): this;
@@ -104,6 +109,7 @@ export interface Options {
         tags?: string[];
         closingSlash?: boolean;
     };
+    selectorCache?: boolean;
 }
 export declare function base_parse(data: string, options?: Partial<Options>): HTMLElement[];
 export declare function parse(data: string, options?: Partial<Options>): HTMLElement;