recker 1.0.72 → 1.0.75-next.2e5a94f

package/dist/presets/aws.js

@@ -1,4 +1,62 @@
 import { awsSignatureV4Plugin } from '../plugins/auth.js';
+const S3_STORAGE_PRESET = {
+    timeout: {
+        lookup: 5000,
+        connect: 10000,
+        secureConnect: 10000,
+        response: 30000,
+        request: 60000,
+    },
+    http2: 'performance',
+    expectContinue: 2 * 1024 * 1024,
+    observability: false,
+    concurrency: {
+        agent: {
+            connections: 100,
+            pipelining: 10,
+            keepAlive: true,
+            keepAliveTimeout: 4000,
+            keepAliveMaxTimeout: 600000,
+            keepAliveTimeoutThreshold: 1000,
+            maxRequestsPerClient: 100,
+            clientTtl: null,
+            maxCachedSessions: 100,
+        },
+    },
+    retry: {
+        maxDelay: 30000,
+    },
+};
+function createAwsS3Preset(options) {
+    const baseConfig = aws({
+        ...options,
+        service: 's3',
+    });
+    const baseConcurrency = baseConfig.concurrency;
+    let concurrency;
+    if (typeof baseConcurrency === 'number') {
+        concurrency = baseConcurrency;
+    }
+    else {
+        const mergedConcurrency = {};
+        if (baseConcurrency && typeof baseConcurrency === 'object') {
+            Object.assign(mergedConcurrency, S3_STORAGE_PRESET.concurrency, baseConcurrency);
+        }
+        else {
+            Object.assign(mergedConcurrency, S3_STORAGE_PRESET.concurrency);
+        }
+        concurrency = mergedConcurrency;
+    }
+    return {
+        ...baseConfig,
+        ...S3_STORAGE_PRESET,
+        retry: {
+            ...baseConfig.retry,
+            ...S3_STORAGE_PRESET.retry,
+        },
+        concurrency,
+    };
+}
 function getServiceEndpoint(service, region) {
     const globalServices = ['iam', 'sts', 'cloudfront', 'route53'];
     if (globalServices.includes(service)) {
@@ -43,7 +101,10 @@ export function aws(options) {
     };
 }
 export function awsS3(options) {
-    return aws({ ...options, service: 's3' });
+    return createAwsS3Preset(options);
+}
+export function s3(options) {
+    return createAwsS3Preset(options);
 }
 export function awsDynamoDB(options) {
     return aws({ ...options, service: 'dynamodb' });

package/dist/recker.d.ts

@@ -10,6 +10,7 @@ import { uploadFile } from './utils/upload.js';
 import { VideoBuilder } from './video/builder.js';
 import { getExtractorName, listExtractors } from './extractors/index.js';
 import type { ExtractorResult } from './extractors/base.js';
+import { type GoogleSearchAdvancedOptions, type GoogleSearchResponse } from './search/index.js';
 export declare function get(url: string, options?: RequestOptions): RequestPromise;
 export declare function post(url: string, options?: RequestOptions): RequestPromise;
 export declare function put(url: string, options?: RequestOptions): RequestPromise;
@@ -25,6 +26,7 @@ export declare function ws(url: string, options?: WebSocketOptions): ReckerWebSo
 export declare function video(url: string): VideoBuilder;
 export declare function isVideoSupported(url: string): Promise<boolean>;
 export declare function extractVideo(url: string): Promise<ExtractorResult>;
+export declare function searchGoogle(query: string, options?: GoogleSearchAdvancedOptions): Promise<GoogleSearchResponse>;
 export declare const recker: {
     get: typeof get;
     post: typeof post;
@@ -81,6 +83,7 @@ export declare const recker: {
         XVideosQuickies: typeof import("./extractors/xvideos.js").XVideosQuickiesExtractor;
         Generic: typeof import("./extractors/generic.js").GenericExtractor;
     };
+    searchGoogle: typeof searchGoogle;
     listExtractors: typeof listExtractors;
     getExtractorName: typeof getExtractorName;
     ai: {

package/dist/recker.js

@@ -8,6 +8,7 @@ import { harRecorder } from './plugins/har-recorder.js';
 import { uploadFile } from './utils/upload.js';
 import { createVideoBuilder } from './video/builder.js';
 import { extract, extractors, isSupported, getExtractorName, listExtractors } from './extractors/index.js';
+import { searchGoogleAdvanced } from './search/index.js';
 let _defaultClient = null;
 let _defaultDns = null;
 let _defaultAi = null;
@@ -74,6 +75,9 @@ export async function isVideoSupported(url) {
 export async function extractVideo(url) {
     return extract(url, getDefaultClient());
 }
+export function searchGoogle(query, options) {
+    return searchGoogleAdvanced(query, options);
+}
 const aiNamespace = {
     chat: (...args) => getDefaultAi().chat(...args),
     stream: (...args) => getDefaultAi().stream(...args),
@@ -100,6 +104,7 @@ export const recker = {
     isVideoSupported,
     extractVideo,
     extractors,
+    searchGoogle,
     listExtractors,
     getExtractorName,
     ai: aiNamespace,

package/dist/runner/request-runner.d.ts

@@ -3,6 +3,12 @@ export interface RunnerOptions {
     retries?: number;
     retryDelay?: number;
 }
+export interface RunnerRunOptions {
+    priority?: number;
+    retries?: number;
+    signal?: AbortSignal;
+    deadlineMs?: number;
+}
 export interface RequestTask<T = any> {
     id: string;
     fn: () => Promise<T>;
@@ -35,10 +41,14 @@ export declare class RequestRunner extends SimpleEmitter {
     private queue;
     private activeCount;
     private paused;
-    private results;
     private stats;
     private startTime;
     private pendingRetries;
+    private isCancelled;
+    private cancelReason;
+    private retryTimers;
+    private timeoutId?;
+    private abortUnsubscribe?;
     constructor(options?: RunnerOptions);
     add<T>(fn: () => Promise<T>, options?: {
         priority?: number;
@@ -47,13 +57,13 @@ export declare class RequestRunner extends SimpleEmitter {
         trackTotal?: boolean;
         resolve?: (value: T | Error) => void;
     }): void;
-    run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: {
-        priority?: number;
-        retries?: number;
-    }): Promise<RunnerResult<T>>;
+    run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: RunnerRunOptions): Promise<RunnerResult<T>>;
     private queueTask;
     private scheduleRetry;
     private processNext;
+    private cancelAll;
+    private resolveQueue;
+    private resolveTask;
     getProgress(): {
         total: number;
         completed: number;

package/dist/runner/request-runner.js

@@ -1,3 +1,4 @@
+import { QueueCancelledError } from '../core/errors.js';
 class SimpleEmitter {
     listeners = new Map();
     on(event, listener) {
@@ -43,10 +44,14 @@ export class RequestRunner extends SimpleEmitter {
     queue = [];
     activeCount = 0;
     paused = false;
-    results = new Map();
     stats = { total: 0, successful: 0, failed: 0 };
     startTime = 0;
     pendingRetries = 0;
+    isCancelled = false;
+    cancelReason = new QueueCancelledError('Request runner cancelled');
+    retryTimers = new Map();
+    timeoutId;
+    abortUnsubscribe;
     constructor(options = {}) {
         super();
         this.concurrency = options.concurrency || 5;
@@ -67,48 +72,146 @@ export class RequestRunner extends SimpleEmitter {
         this.processNext();
     }
     async run(items, processor, options = {}) {
+        this.queue = [];
+        this.activeCount = 0;
+        this.pendingRetries = 0;
+        this.stats = { total: 0, successful: 0, failed: 0 };
+        this.isCancelled = false;
+        if (this.retryTimers.size > 0) {
+            for (const [, timer] of this.retryTimers) {
+                clearTimeout(timer);
+            }
+            this.retryTimers.clear();
+        }
+        if (this.timeoutId) {
+            clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        if (this.abortUnsubscribe) {
+            this.abortUnsubscribe();
+            this.abortUnsubscribe = undefined;
+        }
         this.startTime = Date.now();
         this.stats = { total: items.length, successful: 0, failed: 0 };
-        this.results.clear();
-        const promises = items.map((item, index) => {
-            return new Promise((resolve) => {
-                this.add(() => processor(item, index), {
-                    priority: options.priority,
-                    id: String(index),
-                    retries: options.retries,
-                    resolve,
-                    trackTotal: false
+        this.isCancelled = false;
+        this.cancelReason = new QueueCancelledError('Request runner cancelled', {
+            queueName: 'request-runner',
+            request: undefined
+        });
+        if (options.signal) {
+            const signal = options.signal;
+            if (signal.aborted) {
+                const reason = signal.reason instanceof Error
+                    ? signal.reason
+                    : new QueueCancelledError('Request runner signal was aborted', {
+                        queueName: 'request-runner',
+                        request: undefined
+                    });
+                this.cancelAll(reason);
+            }
+            else {
+                const handleAbort = () => {
+                    const reason = signal.reason instanceof Error
+                        ? signal.reason
+                        : new QueueCancelledError('Request runner signal was aborted', {
+                            queueName: 'request-runner',
+                            request: undefined
+                        });
+                    this.cancelAll(reason);
+                };
+                signal.addEventListener('abort', handleAbort, { once: true });
+                this.abortUnsubscribe = () => signal.removeEventListener('abort', handleAbort);
+            }
+        }
+        if (options.deadlineMs !== undefined) {
+            const deadline = options.deadlineMs;
+            if (deadline <= 0) {
+                this.cancelAll(new QueueCancelledError('Request runner deadline elapsed', {
+                    queueName: 'request-runner',
+                    request: undefined
+                }));
+            }
+            else {
+                this.timeoutId = setTimeout(() => {
+                    this.cancelAll(new QueueCancelledError('Request runner deadline exceeded', {
+                        queueName: 'request-runner',
+                        request: undefined
+                    }));
+                }, deadline);
+            }
+        }
+        try {
+            const promises = items.map((item, index) => {
+                return new Promise((resolve) => {
+                    this.add(() => processor(item, index), {
+                        priority: options.priority,
+                        id: String(index),
+                        retries: options.retries,
+                        resolve,
+                        trackTotal: false
+                    });
                 });
             });
-        });
-        const results = await Promise.all(promises);
-        return {
-            results,
-            stats: {
-                ...this.stats,
-                duration: Date.now() - this.startTime
+            const results = await Promise.all(promises);
+            return {
+                results,
+                stats: {
+                    ...this.stats,
+                    duration: Date.now() - this.startTime
+                }
+            };
+        }
+        finally {
+            if (this.timeoutId) {
+                clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
             }
-        };
+            if (this.abortUnsubscribe) {
+                this.abortUnsubscribe();
+                this.abortUnsubscribe = undefined;
+            }
+        }
     }
     queueTask(task) {
         this.queue.push(task);
         this.queue.sort((a, b) => b.priority - a.priority);
     }
     scheduleRetry(task, delay) {
-        this.pendingRetries++;
-        if (delay > 0) {
-            setTimeout(() => {
-                this.pendingRetries--;
-                this.queueTask(task);
-                this.processNext();
-            }, delay);
+        if (this.isCancelled) {
+            this.resolveTask(task, this.cancelReason);
             return;
         }
-        this.pendingRetries--;
-        this.queueTask(task);
-        this.processNext();
+        if (delay <= 0) {
+            this.queueTask(task);
+            this.processNext();
+            return;
+        }
+        const timerKey = `${task.id}-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+        const enqueueTask = () => {
+            this.retryTimers.delete(timerKey);
+            this.pendingRetries--;
+            if (this.isCancelled) {
+                return;
+            }
+            this.queueTask(task);
+            this.processNext();
+        };
+        this.retryTimers.set(timerKey, setTimeout(() => {
+            if (this.isCancelled) {
+                this.pendingRetries--;
+                this.retryTimers.delete(timerKey);
+                this.resolveTask(task, this.cancelReason);
+                return;
+            }
+            enqueueTask();
+        }, delay));
+        this.pendingRetries++;
     }
     async processNext() {
+        if (this.isCancelled) {
+            this.resolveQueue();
+            return;
+        }
         if (this.paused || this.activeCount >= this.concurrency || this.queue.length === 0) {
             return;
         }
@@ -124,6 +227,10 @@ export class RequestRunner extends SimpleEmitter {
             this.emit('taskComplete', { task, result });
         }
         catch (error) {
+            if (this.isCancelled) {
+                this.resolveTask(task, this.cancelReason);
+                return;
+            }
             const remaining = task.retries ?? 0;
             if (remaining > 0) {
                 task.retries = remaining - 1;
@@ -131,8 +238,7 @@ export class RequestRunner extends SimpleEmitter {
                 this.scheduleRetry(task, this.retryDelay);
             }
             else {
-                this.stats.failed++;
-                task.resolve?.(error);
+                this.resolveTask(task, error);
                 this.emit('taskError', { task, error });
             }
         }
@@ -145,6 +251,34 @@ export class RequestRunner extends SimpleEmitter {
             this.processNext();
         }
     }
+    cancelAll(reason) {
+        if (this.isCancelled) {
+            return;
+        }
+        this.isCancelled = true;
+        this.cancelReason = reason;
+        for (const [, timer] of this.retryTimers) {
+            clearTimeout(timer);
+        }
+        this.retryTimers.clear();
+        this.pendingRetries = 0;
+        this.resolveQueue();
+    }
+    resolveQueue() {
+        while (this.queue.length > 0) {
+            const task = this.queue.shift();
+            if (task) {
+                this.resolveTask(task, this.cancelReason);
+            }
+        }
+    }
+    resolveTask(task, error) {
+        if (!task.resolve) {
+            return;
+        }
+        this.stats.failed++;
+        task.resolve?.(error);
+    }
     getProgress() {
         const completed = this.stats.successful + this.stats.failed;
         return {

package/dist/scrape/parser/nodes/html.d.ts

@@ -33,11 +33,16 @@ export default class HTMLElement extends Node {
     private voidTag;
     private _attrs?;
     private _rawAttrs?;
+    private _queryCache?;
     private _parseOptions;
     rawTagName: string;
     id: string;
     classList: DOMTokenList;
     nodeType: NodeType;
+    private get isSelectorCacheEnabled();
+    private getQueryCache;
+    private clearQueryCache;
+    invalidateSelectorCacheRecursively(): void;
     private quoteAttribute;
     constructor(tagName: string, keyAttrs: KeyAttributes, rawAttrs?: string, parentNode?: HTMLElement | null, range?: [number, number], voidTag?: VoidTag, _parseOptions?: Partial<Options>);
     removeChild(node: Node): this;
@@ -104,6 +109,7 @@ export interface Options {
         tags?: string[];
         closingSlash?: boolean;
     };
+    selectorCache?: boolean;
 }
 export declare function base_parse(data: string, options?: Partial<Options>): HTMLElement[];
 export declare function parse(data: string, options?: Partial<Options>): HTMLElement;

package/dist/scrape/parser/nodes/html.js

@@ -98,11 +98,31 @@ export default class HTMLElement extends Node {
     voidTag;
     _attrs;
     _rawAttrs;
+    _queryCache;
     _parseOptions;
     rawTagName;
     id;
     classList;
     nodeType = NodeType.ELEMENT_NODE;
+    get isSelectorCacheEnabled() {
+        return this._parseOptions?.selectorCache !== false;
+    }
+    getQueryCache() {
+        if (!this._queryCache) {
+            this._queryCache = new Map();
+        }
+        return this._queryCache;
+    }
+    clearQueryCache() {
+        this._queryCache = undefined;
+    }
+    invalidateSelectorCacheRecursively() {
+        let current = this;
+        while (current) {
+            current.clearQueryCache();
+            current = current.parentNode;
+        }
+    }
     quoteAttribute(attr) {
         if (attr == null) {
             return 'null';
@@ -144,6 +164,7 @@ export default class HTMLElement extends Node {
         this.childNodes = this.childNodes.filter((child) => {
             return child !== node;
         });
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     exchangeChild(oldNode, newNode) {
@@ -154,6 +175,7 @@ export default class HTMLElement extends Node {
             }
             return child;
         });
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     get tagName() {
@@ -182,6 +204,7 @@ export default class HTMLElement extends Node {
     set textContent(val) {
         const content = [new TextNode(val, this)];
         this.childNodes = content;
+        this.invalidateSelectorCacheRecursively();
     }
     get text() {
         return decode(this.rawText);
@@ -249,6 +272,7 @@ export default class HTMLElement extends Node {
         resetParent(nodes, this);
         resetParent(this.childNodes, null);
         this.childNodes = nodes;
+        this.invalidateSelectorCacheRecursively();
     }
     set_content(content, options = {}) {
         if (content instanceof Node) {
@@ -264,6 +288,7 @@ export default class HTMLElement extends Node {
         resetParent(this.childNodes, null);
         resetParent(content, this);
         this.childNodes = content;
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     replaceWith(...nodes) {
@@ -293,6 +318,7 @@ export default class HTMLElement extends Node {
             ...resetParent(content, parent),
             ...parent.childNodes.slice(idx + 1),
         ];
+        parent.invalidateSelectorCacheRecursively();
         return this;
     }
     get outerHTML() {
@@ -312,6 +338,7 @@ export default class HTMLElement extends Node {
                 }
             }
         }
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     get structure() {
@@ -357,6 +384,7 @@ export default class HTMLElement extends Node {
             this.childNodes[o++] = node;
         });
         this.childNodes.length = o;
+        this.invalidateSelectorCacheRecursively();
         const attrs = Object.keys(this.rawAttributes)
             .map((key) => {
             const val = this.rawAttributes[key];
@@ -368,16 +396,49 @@ export default class HTMLElement extends Node {
         return this;
     }
     querySelectorAll(selector) {
-        return selectAll(selector, this, {
+        if (this.isSelectorCacheEnabled) {
+            const cached = this.getQueryCache().get(selector);
+            if (cached?.all) {
+                return cached.all.slice();
+            }
+        }
+        const nodes = selectAll(selector, this, {
             xmlMode: false,
             adapter: Matcher,
         });
+        if (this.isSelectorCacheEnabled) {
+            const cacheEntry = this.getQueryCache().get(selector) || {};
+            cacheEntry.all = nodes;
+            if (cacheEntry.first === undefined) {
+                cacheEntry.first = nodes[0] || null;
+            }
+            this.getQueryCache().set(selector, cacheEntry);
+        }
+        return nodes;
     }
     querySelector(selector) {
-        return selectOne(selector, this, {
+        if (this.isSelectorCacheEnabled) {
+            const cached = this.getQueryCache().get(selector);
+            if (cached?.first !== undefined) {
+                return cached.first || null;
+            }
+            if (cached?.all) {
+                const first = cached.all[0] || null;
+                cached.first = first;
+                this.getQueryCache().set(selector, cached);
+                return first;
+            }
+        }
+        const result = selectOne(selector, this, {
             xmlMode: false,
             adapter: Matcher,
         });
+        if (this.isSelectorCacheEnabled) {
+            const cacheEntry = this.getQueryCache().get(selector) || {};
+            cacheEntry.first = result;
+            this.getQueryCache().set(selector, cacheEntry);
+        }
+        return result;
     }
     getElementsByTagName(tagName) {
         const upperCasedTagName = tagName.toUpperCase();
@@ -440,22 +501,6 @@ export default class HTMLElement extends Node {
         const mapChild = new Map();
         let el = this;
         let old = null;
-        function findOne(test, elems) {
-            let elem = null;
-            for (let i = 0, l = elems.length; i < l && !elem; i++) {
-                const el = elems[i];
-                if (test(el)) {
-                    elem = el;
-                }
-                else {
-                    const child = mapChild.get(el);
-                    if (child) {
-                        elem = findOne(test, [child]);
-                    }
-                }
-            }
-            return elem;
-        }
         while (el) {
             if (old)
                 mapChild.set(el, old);
@@ -545,6 +590,7 @@ export default class HTMLElement extends Node {
         if (key === 'id') {
             this.id = '';
         }
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     hasAttribute(key) {
@@ -580,6 +626,7 @@ export default class HTMLElement extends Node {
         if (key === 'id') {
             this.id = value;
         }
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     setAttributes(attributes) {
@@ -597,6 +644,7 @@ export default class HTMLElement extends Node {
             return `${name}=${this.quoteAttribute(String(val))}`;
         })
             .join(' ');
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     insertAdjacentHTML(where, html) {
@@ -625,11 +673,13 @@ export default class HTMLElement extends Node {
         const nodes = resolveInsertable(insertable, this._parseOptions);
         resetParent(nodes, this);
         this.childNodes.unshift(...nodes);
+        this.invalidateSelectorCacheRecursively();
     }
     append(...insertable) {
         const nodes = resolveInsertable(insertable, this._parseOptions);
         resetParent(nodes, this);
         this.childNodes.push(...nodes);
+        this.invalidateSelectorCacheRecursively();
     }
     before(...insertable) {
         if (!this.parentNode)
@@ -638,6 +688,7 @@ export default class HTMLElement extends Node {
         const siblings = this.parentNode.childNodes;
         resetParent(nodes, this.parentNode);
         siblings.splice(siblings.indexOf(this), 0, ...nodes);
+        this.parentNode.invalidateSelectorCacheRecursively();
     }
     after(...insertable) {
         if (!this.parentNode)
@@ -646,6 +697,7 @@ export default class HTMLElement extends Node {
         const siblings = this.parentNode.childNodes;
         resetParent(nodes, this.parentNode);
         siblings.splice(siblings.indexOf(this) + 1, 0, ...nodes);
+        this.parentNode.invalidateSelectorCacheRecursively();
     }
     get nextSibling() {
         if (this.parentNode) {

package/dist/scrape/parser/nodes/node.d.ts

@@ -12,6 +12,7 @@ export default abstract class Node {
     abstract clone(): Node;
     constructor(parentNode?: HTMLElement | null, range?: [number, number]);
     remove(): this;
+    invalidateSelectorCacheRecursively(): void;
     get innerText(): string;
     get textContent(): string;
     set textContent(val: string);

package/dist/scrape/parser/nodes/node.js

@@ -11,6 +11,9 @@ export default class Node {
     }
     remove() {
         if (this.parentNode) {
+            if (typeof this.parentNode.invalidateSelectorCacheRecursively === 'function') {
+                this.parentNode.invalidateSelectorCacheRecursively();
+            }
             const children = this.parentNode.childNodes;
             this.parentNode.childNodes = children.filter((child) => {
                 return this !== child;
@@ -19,6 +22,8 @@ export default class Node {
         }
         return this;
     }
+    invalidateSelectorCacheRecursively() {
+    }
     get innerText() {
         return this.rawText;
     }