recker 1.0.73 → 1.0.75-next.2e5a94f

package/dist/runner/request-runner.js

@@ -1,3 +1,4 @@
+import { QueueCancelledError } from '../core/errors.js';
 class SimpleEmitter {
     listeners = new Map();
     on(event, listener) {
@@ -43,10 +44,14 @@ export class RequestRunner extends SimpleEmitter {
     queue = [];
     activeCount = 0;
     paused = false;
-    results = new Map();
     stats = { total: 0, successful: 0, failed: 0 };
     startTime = 0;
     pendingRetries = 0;
+    isCancelled = false;
+    cancelReason = new QueueCancelledError('Request runner cancelled');
+    retryTimers = new Map();
+    timeoutId;
+    abortUnsubscribe;
     constructor(options = {}) {
         super();
         this.concurrency = options.concurrency || 5;
@@ -67,48 +72,146 @@ export class RequestRunner extends SimpleEmitter {
         this.processNext();
     }
     async run(items, processor, options = {}) {
+        this.queue = [];
+        this.activeCount = 0;
+        this.pendingRetries = 0;
+        this.stats = { total: 0, successful: 0, failed: 0 };
+        this.isCancelled = false;
+        if (this.retryTimers.size > 0) {
+            for (const [, timer] of this.retryTimers) {
+                clearTimeout(timer);
+            }
+            this.retryTimers.clear();
+        }
+        if (this.timeoutId) {
+            clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        if (this.abortUnsubscribe) {
+            this.abortUnsubscribe();
+            this.abortUnsubscribe = undefined;
+        }
         this.startTime = Date.now();
         this.stats = { total: items.length, successful: 0, failed: 0 };
-        this.results.clear();
-        const promises = items.map((item, index) => {
-            return new Promise((resolve) => {
-                this.add(() => processor(item, index), {
-                    priority: options.priority,
-                    id: String(index),
-                    retries: options.retries,
-                    resolve,
-                    trackTotal: false
+        this.isCancelled = false;
+        this.cancelReason = new QueueCancelledError('Request runner cancelled', {
+            queueName: 'request-runner',
+            request: undefined
+        });
+        if (options.signal) {
+            const signal = options.signal;
+            if (signal.aborted) {
+                const reason = signal.reason instanceof Error
+                    ? signal.reason
+                    : new QueueCancelledError('Request runner signal was aborted', {
+                        queueName: 'request-runner',
+                        request: undefined
+                    });
+                this.cancelAll(reason);
+            }
+            else {
+                const handleAbort = () => {
+                    const reason = signal.reason instanceof Error
+                        ? signal.reason
+                        : new QueueCancelledError('Request runner signal was aborted', {
+                            queueName: 'request-runner',
+                            request: undefined
+                        });
+                    this.cancelAll(reason);
+                };
+                signal.addEventListener('abort', handleAbort, { once: true });
+                this.abortUnsubscribe = () => signal.removeEventListener('abort', handleAbort);
+            }
+        }
+        if (options.deadlineMs !== undefined) {
+            const deadline = options.deadlineMs;
+            if (deadline <= 0) {
+                this.cancelAll(new QueueCancelledError('Request runner deadline elapsed', {
+                    queueName: 'request-runner',
+                    request: undefined
+                }));
+            }
+            else {
+                this.timeoutId = setTimeout(() => {
+                    this.cancelAll(new QueueCancelledError('Request runner deadline exceeded', {
+                        queueName: 'request-runner',
+                        request: undefined
+                    }));
+                }, deadline);
+            }
+        }
+        try {
+            const promises = items.map((item, index) => {
+                return new Promise((resolve) => {
+                    this.add(() => processor(item, index), {
+                        priority: options.priority,
+                        id: String(index),
+                        retries: options.retries,
+                        resolve,
+                        trackTotal: false
+                    });
                 });
             });
-        });
-        const results = await Promise.all(promises);
-        return {
-            results,
-            stats: {
-                ...this.stats,
-                duration: Date.now() - this.startTime
+            const results = await Promise.all(promises);
+            return {
+                results,
+                stats: {
+                    ...this.stats,
+                    duration: Date.now() - this.startTime
+                }
+            };
+        }
+        finally {
+            if (this.timeoutId) {
+                clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
             }
-        };
+            if (this.abortUnsubscribe) {
+                this.abortUnsubscribe();
+                this.abortUnsubscribe = undefined;
+            }
+        }
     }
     queueTask(task) {
         this.queue.push(task);
         this.queue.sort((a, b) => b.priority - a.priority);
     }
     scheduleRetry(task, delay) {
-        this.pendingRetries++;
-        if (delay > 0) {
-            setTimeout(() => {
-                this.pendingRetries--;
-                this.queueTask(task);
-                this.processNext();
-            }, delay);
+        if (this.isCancelled) {
+            this.resolveTask(task, this.cancelReason);
             return;
         }
-        this.pendingRetries--;
-        this.queueTask(task);
-        this.processNext();
+        if (delay <= 0) {
+            this.queueTask(task);
+            this.processNext();
+            return;
+        }
+        const timerKey = `${task.id}-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+        const enqueueTask = () => {
+            this.retryTimers.delete(timerKey);
+            this.pendingRetries--;
+            if (this.isCancelled) {
+                return;
+            }
+            this.queueTask(task);
+            this.processNext();
+        };
+        this.retryTimers.set(timerKey, setTimeout(() => {
+            if (this.isCancelled) {
+                this.pendingRetries--;
+                this.retryTimers.delete(timerKey);
+                this.resolveTask(task, this.cancelReason);
+                return;
+            }
+            enqueueTask();
+        }, delay));
+        this.pendingRetries++;
     }
     async processNext() {
+        if (this.isCancelled) {
+            this.resolveQueue();
+            return;
+        }
         if (this.paused || this.activeCount >= this.concurrency || this.queue.length === 0) {
             return;
         }
@@ -124,6 +227,10 @@ export class RequestRunner extends SimpleEmitter {
             this.emit('taskComplete', { task, result });
         }
         catch (error) {
+            if (this.isCancelled) {
+                this.resolveTask(task, this.cancelReason);
+                return;
+            }
             const remaining = task.retries ?? 0;
             if (remaining > 0) {
                 task.retries = remaining - 1;
@@ -131,8 +238,7 @@ export class RequestRunner extends SimpleEmitter {
                 this.scheduleRetry(task, this.retryDelay);
             }
             else {
-                this.stats.failed++;
-                task.resolve?.(error);
+                this.resolveTask(task, error);
                 this.emit('taskError', { task, error });
             }
         }
@@ -145,6 +251,34 @@ export class RequestRunner extends SimpleEmitter {
             this.processNext();
         }
     }
+    cancelAll(reason) {
+        if (this.isCancelled) {
+            return;
+        }
+        this.isCancelled = true;
+        this.cancelReason = reason;
+        for (const [, timer] of this.retryTimers) {
+            clearTimeout(timer);
+        }
+        this.retryTimers.clear();
+        this.pendingRetries = 0;
+        this.resolveQueue();
+    }
+    resolveQueue() {
+        while (this.queue.length > 0) {
+            const task = this.queue.shift();
+            if (task) {
+                this.resolveTask(task, this.cancelReason);
+            }
+        }
+    }
+    resolveTask(task, error) {
+        if (!task.resolve) {
+            return;
+        }
+        this.stats.failed++;
+        task.resolve?.(error);
+    }
     getProgress() {
         const completed = this.stats.successful + this.stats.failed;
         return {

package/dist/scrape/parser/nodes/html.d.ts

@@ -33,11 +33,16 @@ export default class HTMLElement extends Node {
     private voidTag;
     private _attrs?;
     private _rawAttrs?;
+    private _queryCache?;
     private _parseOptions;
     rawTagName: string;
     id: string;
     classList: DOMTokenList;
     nodeType: NodeType;
+    private get isSelectorCacheEnabled();
+    private getQueryCache;
+    private clearQueryCache;
+    invalidateSelectorCacheRecursively(): void;
     private quoteAttribute;
     constructor(tagName: string, keyAttrs: KeyAttributes, rawAttrs?: string, parentNode?: HTMLElement | null, range?: [number, number], voidTag?: VoidTag, _parseOptions?: Partial<Options>);
     removeChild(node: Node): this;
@@ -104,6 +109,7 @@ export interface Options {
         tags?: string[];
         closingSlash?: boolean;
     };
+    selectorCache?: boolean;
 }
 export declare function base_parse(data: string, options?: Partial<Options>): HTMLElement[];
 export declare function parse(data: string, options?: Partial<Options>): HTMLElement;

package/dist/scrape/parser/nodes/html.js

@@ -98,11 +98,31 @@ export default class HTMLElement extends Node {
     voidTag;
     _attrs;
     _rawAttrs;
+    _queryCache;
     _parseOptions;
     rawTagName;
     id;
     classList;
     nodeType = NodeType.ELEMENT_NODE;
+    get isSelectorCacheEnabled() {
+        return this._parseOptions?.selectorCache !== false;
+    }
+    getQueryCache() {
+        if (!this._queryCache) {
+            this._queryCache = new Map();
+        }
+        return this._queryCache;
+    }
+    clearQueryCache() {
+        this._queryCache = undefined;
+    }
+    invalidateSelectorCacheRecursively() {
+        let current = this;
+        while (current) {
+            current.clearQueryCache();
+            current = current.parentNode;
+        }
+    }
     quoteAttribute(attr) {
         if (attr == null) {
             return 'null';
@@ -144,6 +164,7 @@ export default class HTMLElement extends Node {
         this.childNodes = this.childNodes.filter((child) => {
             return child !== node;
         });
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     exchangeChild(oldNode, newNode) {
@@ -154,6 +175,7 @@ export default class HTMLElement extends Node {
             }
             return child;
         });
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     get tagName() {
@@ -182,6 +204,7 @@ export default class HTMLElement extends Node {
     set textContent(val) {
         const content = [new TextNode(val, this)];
         this.childNodes = content;
+        this.invalidateSelectorCacheRecursively();
     }
     get text() {
         return decode(this.rawText);
@@ -249,6 +272,7 @@ export default class HTMLElement extends Node {
         resetParent(nodes, this);
         resetParent(this.childNodes, null);
         this.childNodes = nodes;
+        this.invalidateSelectorCacheRecursively();
     }
     set_content(content, options = {}) {
         if (content instanceof Node) {
@@ -264,6 +288,7 @@ export default class HTMLElement extends Node {
         resetParent(this.childNodes, null);
         resetParent(content, this);
         this.childNodes = content;
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     replaceWith(...nodes) {
@@ -293,6 +318,7 @@ export default class HTMLElement extends Node {
             ...resetParent(content, parent),
             ...parent.childNodes.slice(idx + 1),
         ];
+        parent.invalidateSelectorCacheRecursively();
         return this;
     }
     get outerHTML() {
@@ -312,6 +338,7 @@ export default class HTMLElement extends Node {
                 }
             }
         }
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     get structure() {
@@ -357,6 +384,7 @@ export default class HTMLElement extends Node {
             this.childNodes[o++] = node;
         });
         this.childNodes.length = o;
+        this.invalidateSelectorCacheRecursively();
         const attrs = Object.keys(this.rawAttributes)
             .map((key) => {
             const val = this.rawAttributes[key];
@@ -368,16 +396,49 @@ export default class HTMLElement extends Node {
         return this;
     }
     querySelectorAll(selector) {
-        return selectAll(selector, this, {
+        if (this.isSelectorCacheEnabled) {
+            const cached = this.getQueryCache().get(selector);
+            if (cached?.all) {
+                return cached.all.slice();
+            }
+        }
+        const nodes = selectAll(selector, this, {
             xmlMode: false,
             adapter: Matcher,
         });
+        if (this.isSelectorCacheEnabled) {
+            const cacheEntry = this.getQueryCache().get(selector) || {};
+            cacheEntry.all = nodes;
+            if (cacheEntry.first === undefined) {
+                cacheEntry.first = nodes[0] || null;
+            }
+            this.getQueryCache().set(selector, cacheEntry);
+        }
+        return nodes;
     }
     querySelector(selector) {
-        return selectOne(selector, this, {
+        if (this.isSelectorCacheEnabled) {
+            const cached = this.getQueryCache().get(selector);
+            if (cached?.first !== undefined) {
+                return cached.first || null;
+            }
+            if (cached?.all) {
+                const first = cached.all[0] || null;
+                cached.first = first;
+                this.getQueryCache().set(selector, cached);
+                return first;
+            }
+        }
+        const result = selectOne(selector, this, {
             xmlMode: false,
             adapter: Matcher,
         });
+        if (this.isSelectorCacheEnabled) {
+            const cacheEntry = this.getQueryCache().get(selector) || {};
+            cacheEntry.first = result;
+            this.getQueryCache().set(selector, cacheEntry);
+        }
+        return result;
     }
     getElementsByTagName(tagName) {
         const upperCasedTagName = tagName.toUpperCase();
@@ -440,22 +501,6 @@ export default class HTMLElement extends Node {
         const mapChild = new Map();
         let el = this;
         let old = null;
-        function findOne(test, elems) {
-            let elem = null;
-            for (let i = 0, l = elems.length; i < l && !elem; i++) {
-                const el = elems[i];
-                if (test(el)) {
-                    elem = el;
-                }
-                else {
-                    const child = mapChild.get(el);
-                    if (child) {
-                        elem = findOne(test, [child]);
-                    }
-                }
-            }
-            return elem;
-        }
         while (el) {
             if (old)
                 mapChild.set(el, old);
@@ -545,6 +590,7 @@ export default class HTMLElement extends Node {
         if (key === 'id') {
             this.id = '';
         }
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     hasAttribute(key) {
@@ -580,6 +626,7 @@ export default class HTMLElement extends Node {
         if (key === 'id') {
             this.id = value;
         }
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     setAttributes(attributes) {
@@ -597,6 +644,7 @@ export default class HTMLElement extends Node {
             return `${name}=${this.quoteAttribute(String(val))}`;
         })
             .join(' ');
+        this.invalidateSelectorCacheRecursively();
         return this;
     }
     insertAdjacentHTML(where, html) {
@@ -625,11 +673,13 @@ export default class HTMLElement extends Node {
         const nodes = resolveInsertable(insertable, this._parseOptions);
         resetParent(nodes, this);
         this.childNodes.unshift(...nodes);
+        this.invalidateSelectorCacheRecursively();
     }
     append(...insertable) {
         const nodes = resolveInsertable(insertable, this._parseOptions);
         resetParent(nodes, this);
         this.childNodes.push(...nodes);
+        this.invalidateSelectorCacheRecursively();
     }
     before(...insertable) {
         if (!this.parentNode)
@@ -638,6 +688,7 @@ export default class HTMLElement extends Node {
         const siblings = this.parentNode.childNodes;
         resetParent(nodes, this.parentNode);
         siblings.splice(siblings.indexOf(this), 0, ...nodes);
+        this.parentNode.invalidateSelectorCacheRecursively();
     }
     after(...insertable) {
         if (!this.parentNode)
@@ -646,6 +697,7 @@ export default class HTMLElement extends Node {
         const siblings = this.parentNode.childNodes;
         resetParent(nodes, this.parentNode);
         siblings.splice(siblings.indexOf(this) + 1, 0, ...nodes);
+        this.parentNode.invalidateSelectorCacheRecursively();
     }
     get nextSibling() {
         if (this.parentNode) {

package/dist/scrape/parser/nodes/node.d.ts

@@ -12,6 +12,7 @@ export default abstract class Node {
     abstract clone(): Node;
     constructor(parentNode?: HTMLElement | null, range?: [number, number]);
     remove(): this;
+    invalidateSelectorCacheRecursively(): void;
     get innerText(): string;
     get textContent(): string;
     set textContent(val: string);

package/dist/scrape/parser/nodes/node.js

@@ -11,6 +11,9 @@ export default class Node {
     }
     remove() {
         if (this.parentNode) {
+            if (typeof this.parentNode.invalidateSelectorCacheRecursively === 'function') {
+                this.parentNode.invalidateSelectorCacheRecursively();
+            }
             const children = this.parentNode.childNodes;
             this.parentNode.childNodes = children.filter((child) => {
                 return this !== child;
@@ -19,6 +22,8 @@ export default class Node {
         }
         return this;
     }
+    invalidateSelectorCacheRecursively() {
+    }
     get innerText() {
         return this.rawText;
     }

package/dist/scrape/spider.d.ts

@@ -123,6 +123,7 @@ export declare class Spider {
     private robotsData;
     private sitemapValidation;
     private robotsValidation;
+    private toHeaderRecord;
     constructor(options?: SpiderOptions);
     crawl(startUrl: string): Promise<SpiderResult>;
     private fetchRobotsTxt;

package/dist/scrape/spider.js

@@ -76,9 +76,6 @@ function shouldCrawl(url, baseHost, options) {
         return false;
     }
 }
-function sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
 function parseExtractSelectors(selectors) {
     const schema = {};
     for (const sel of selectors) {
@@ -115,6 +112,13 @@ export class Spider {
     robotsData = null;
     sitemapValidation = null;
     robotsValidation = null;
+    toHeaderRecord(headers) {
+        const headerRecord = {};
+        headers.forEach((value, key) => {
+            headerRecord[key] = value;
+        });
+        return headerRecord;
+    }
     constructor(options = {}) {
         let extractSchema;
         if (options.extract) {
@@ -194,7 +198,7 @@ export class Spider {
             await this.fetchSitemaps(baseUrl);
         }
         const pending = new Map();
-        const scheduleUrl = (item, fromSitemap = false) => {
+        const scheduleUrl = (item) => {
             const normalized = normalizeUrl(item.url);
             if (this.visited.has(normalized))
                 return;
@@ -230,7 +234,7 @@ export class Spider {
                 try {
                     const urlHost = new URL(sitemapUrl.loc).hostname;
                     if (urlHost === this.baseHost) {
-                        scheduleUrl({ url: sitemapUrl.loc, depth: 1 }, true);
+                        scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
                     }
                 }
                 catch {
@@ -303,7 +307,7 @@ export class Spider {
             return {
                 status: response.status,
                 text: await response.text(),
-                headers: Object.fromEntries([...response.headers.entries()]),
+                headers: this.toHeaderRecord(response.headers),
             };
         };
         try {
@@ -351,40 +355,49 @@ export class Spider {
     }
     buildSitemapAnalysis() {
         const crawledUrls = new Set(this.results.map(r => normalizeUrl(r.url)));
-        const crawledFromSitemap = this.sitemapUrls.filter(u => crawledUrls.has(normalizeUrl(u.loc))).length;
+        const sitemapUrlSet = this.sitemapUrlSet.size > 0
+            ? this.sitemapUrlSet
+            : new Set(this.sitemapUrls.map((u) => normalizeUrl(u.loc)));
+        const crawledFromSitemap = Array.from(sitemapUrlSet)
+            .filter(url => crawledUrls.has(url))
+            .length;
         const linkedUrls = new Set();
-        for (const page of this.results) {
-            for (const link of page.links) {
-                if (link.href) {
-                    linkedUrls.add(normalizeUrl(link.href));
-                }
-            }
-        }
-        const orphanUrls = this.sitemapUrls
-            .filter(u => {
-            const normalized = normalizeUrl(u.loc);
-            return !linkedUrls.has(normalized) && crawledUrls.has(normalized);
-        })
-            .map(u => u.loc);
-        const missingFromSitemap = Array.from(crawledUrls)
-            .filter(url => !this.sitemapUrlSet.has(url));
-        const blockedBySitemapRobots = [];
+        const blockedBySitemapRobotsSet = new Set();
         if (this.robotsData) {
             for (const sitemapUrl of this.sitemapUrls) {
                 try {
+                    const normalized = normalizeUrl(sitemapUrl.loc);
                     const urlPath = new URL(sitemapUrl.loc).pathname;
                     if (!isPathAllowed(this.robotsData, urlPath, this.options.userAgent)) {
-                        blockedBySitemapRobots.push(sitemapUrl.loc);
+                        blockedBySitemapRobotsSet.add(normalized);
                     }
                 }
                 catch {
                 }
             }
         }
+        for (const page of this.results) {
+            for (const link of page.links) {
+                if (link.href) {
+                    linkedUrls.add(normalizeUrl(link.href));
+                }
+            }
+        }
+        const orphanUrlSet = new Set();
+        for (const u of this.sitemapUrls) {
+            const normalized = normalizeUrl(u.loc);
+            if (!linkedUrls.has(normalized) && !blockedBySitemapRobotsSet.has(normalized)) {
+                orphanUrlSet.add(normalized);
+            }
+        }
+        const orphanUrls = Array.from(orphanUrlSet);
+        const missingFromSitemap = Array.from(crawledUrls)
+            .filter(url => !sitemapUrlSet.has(url));
+        const blockedBySitemapRobots = Array.from(blockedBySitemapRobotsSet);
         return {
             found: this.sitemapUrls.length > 0,
-            url: this.sitemapValidation?.parseResult ? undefined : undefined,
-            totalUrls: this.sitemapUrls.length,
+            url: this.sitemapUrls[0]?.loc,
+            totalUrls: sitemapUrlSet.size,
             crawledFromSitemap,
             orphanUrls,
             missingFromSitemap,