recker 1.0.94 → 1.0.95-next.0a5359d

package/dist/browser/scrape/crawl-queue.d.ts

@@ -17,7 +17,11 @@ export interface CrawlQueueAdapter {
 }
 export declare class InMemoryCrawlQueue implements CrawlQueueAdapter {
     private queue;
+    private head;
+    private tail;
     private visited;
+    private mode;
+    constructor(mode?: 'fifo' | 'lifo');
     push(item: CrawlQueueItem): Promise<void>;
     pushBatch(items: CrawlQueueItem[]): Promise<void>;
     pop(): Promise<CrawlQueueItem | null>;

package/dist/browser/scrape/crawl-queue.js

@@ -1,14 +1,45 @@
 export class InMemoryCrawlQueue {
     queue = [];
+    head = 0;
+    tail = 0;
     visited = new Set();
+    mode;
+    constructor(mode = 'fifo') {
+        this.mode = mode;
+    }
     async push(item) {
-        this.queue.push(item);
+        this.queue[this.tail++] = item;
     }
     async pushBatch(items) {
-        this.queue.push(...items);
+        for (const item of items) {
+            this.queue[this.tail++] = item;
+        }
     }
     async pop() {
-        return this.queue.shift() ?? null;
+        if (this.mode === 'lifo') {
+            while (this.tail > this.head) {
+                this.tail--;
+                const item = this.queue[this.tail];
+                this.queue[this.tail] = undefined;
+                if (item)
+                    return item;
+            }
+            return null;
+        }
+        while (this.head < this.tail) {
+            const item = this.queue[this.head];
+            this.queue[this.head] = undefined;
+            this.head++;
+            if (item) {
+                if (this.head > 1024 && this.head > this.tail / 2) {
+                    this.queue = this.queue.slice(this.head);
+                    this.tail -= this.head;
+                    this.head = 0;
+                }
+                return item;
+            }
+        }
+        return null;
     }
     async hasVisited(url) {
         return this.visited.has(url);
@@ -25,10 +56,12 @@ export class InMemoryCrawlQueue {
         this.visited.add(url);
     }
     async size() {
-        return this.queue.length;
+        return this.tail - this.head;
     }
     async clear() {
         this.queue = [];
+        this.head = 0;
+        this.tail = 0;
         this.visited.clear();
     }
     async close() {

package/dist/browser/scrape/spider.d.ts

@@ -11,7 +11,7 @@ type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
     maxDepth?: number;
     maxPages?: number;
-    sameDomain?: boolean;
+    sameDomain?: boolean | 'exact' | 'subdomain';
     concurrency?: number;
     timeout?: number;
     delay?: number;
@@ -63,8 +63,14 @@ export interface SpiderOptions {
     domainRateLimit?: {
         maxPerSecond?: number;
     };
+    autoThrottle?: boolean | {
+        targetMs?: number;
+        minDelay?: number;
+        maxDelay?: number;
+    };
     deduplicateContent?: boolean;
     resume?: boolean;
+    strategy?: 'bfs' | 'dfs';
     crawlQueue?: CrawlQueueAdapter;
     crawlStorage?: CrawlStorageAdapter;
 }
@@ -190,11 +196,14 @@ export declare class Spider {
     private _queueSize;
     private _resultCount;
     private baseHost;
+    private baseRootDomain;
     private running;
     private aborted;
+    private abortController;
     private pendingCount;
     private domainRequestTimestamps;
     private contentHashes;
+    private domainAvgResponseTime;
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
@@ -223,6 +232,7 @@ export declare class Spider {
     private waitForDomainPenalty;
     private registerDomainBlock;
     private registerDomainSuccess;
+    private updateAutoThrottle;
     private getCaptchaRetryMultiplier;
     private registerDomainChallenge;
     private getRetryWait;

package/dist/browser/scrape/spider.js

@@ -40,10 +40,15 @@ const SEC_CH_PLATFORM_HINTS = [
 function getHostname(url) {
     return new URL(url).hostname;
 }
-function sleep(ms) {
+function sleep(ms, signal) {
     if (ms <= 0)
         return Promise.resolve();
-    return new Promise(resolve => setTimeout(resolve, ms));
+    if (signal?.aborted)
+        return Promise.resolve();
+    return new Promise(resolve => {
+        const timer = setTimeout(resolve, ms);
+        signal?.addEventListener('abort', () => { clearTimeout(timer); resolve(); }, { once: true });
+    });
 }
 function getRetryAfterDelay(response) {
     const retryAfter = response.headers.get('retry-after');
@@ -103,14 +108,30 @@ function normalizeUrl(urlStr) {
         return urlStr;
     }
 }
-function shouldCrawl(url, baseHost, options) {
+function getRootDomain(hostname) {
+    const parts = hostname.replace(/^www\./, '').split('.');
+    if (parts.length >= 3 && parts[parts.length - 2].length <= 3) {
+        return parts.slice(-3).join('.');
+    }
+    return parts.slice(-2).join('.');
+}
+function shouldCrawl(url, baseHost, options, baseRootDomain) {
     try {
         const parsed = new URL(url);
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
-        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
-            return false;
+        const hostname = parsed.hostname.replace(/^www\./, '');
+        const sameDomain = options.sameDomain;
+        if (sameDomain === 'subdomain') {
+            const pageRoot = getRootDomain(hostname);
+            const rootDomain = baseRootDomain ?? getRootDomain(baseHost);
+            if (pageRoot !== rootDomain)
+                return false;
+        }
+        else if (sameDomain !== false) {
+            if (hostname !== baseHost)
+                return false;
         }
         const skipExtensions = [
             '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
@@ -187,11 +208,14 @@ export class Spider {
     _queueSize = 0;
     _resultCount = 0;
     baseHost = '';
+    baseRootDomain = '';
     running = false;
     aborted = false;
+    abortController = new AbortController();
     pendingCount = 0;
     domainRequestTimestamps = new Map();
     contentHashes = new Map();
+    domainAvgResponseTime = new Map();
     blockedDomains = new Set();
     curlTransport = null;
     curlAvailable = false;
@@ -218,7 +242,9 @@ export class Spider {
         if (timestamps.length >= limit) {
             const waitMs = timestamps[0] + window - now;
             if (waitMs > 0)
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
+            if (this.aborted)
+                return;
             const afterWait = Date.now();
             while (timestamps.length > 0 && timestamps[0] <= afterWait - window) {
                 timestamps.shift();
@@ -276,8 +302,10 @@ export class Spider {
             extract: extractSchema,
             parserOptions: options.parserOptions,
             domainRateLimit: options.domainRateLimit,
+            autoThrottle: options.autoThrottle ?? false,
             deduplicateContent: options.deduplicateContent ?? false,
             resume: options.resume ?? false,
+            strategy: options.strategy ?? 'bfs',
         };
         if (options.proxy) {
             if (typeof options.proxy === 'string') {
@@ -307,7 +335,7 @@ export class Spider {
                 interval: this.options.delay,
             } : {}),
         });
-        this.crawlQueue = options.crawlQueue ?? new InMemoryCrawlQueue();
+        this.crawlQueue = options.crawlQueue ?? new InMemoryCrawlQueue(this.options.strategy === 'dfs' ? 'lifo' : 'fifo');
         this.crawlStorage = options.crawlStorage ?? new InMemoryCrawlStorage();
     }
     async crawl(startUrl) {
@@ -315,7 +343,8 @@ export class Spider {
         const startTimestamp = Date.now();
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
-        this.baseHost = new URL(normalizedStart).hostname;
+        this.baseHost = new URL(normalizedStart).hostname.replace(/^www\./, '');
+        this.baseRootDomain = getRootDomain(this.baseHost);
         if (!this.options.resume) {
             await this.crawlQueue.clear();
             await this.crawlStorage.clear();
@@ -324,6 +353,7 @@ export class Spider {
             this._resultCount = 0;
             this.domainRequestTimestamps.clear();
             this.contentHashes.clear();
+            this.domainAvgResponseTime.clear();
         }
         else {
             this._queueSize = await this.crawlQueue.size();
@@ -331,14 +361,25 @@ export class Spider {
         }
         this.running = true;
         this.aborted = false;
+        this.abortController = new AbortController();
         this.pendingCount = 0;
         this.sitemapUrls = [];
         this.sitemapUrlSet.clear();
         this.robotsData = null;
         this.sitemapValidation = null;
         this.robotsValidation = null;
-        this.blockedDomains.clear();
-        this.domainStates.clear();
+        if (!this.options.resume) {
+            this.blockedDomains.clear();
+            this.domainStates.clear();
+        }
+        if (this.options.resume && this.options.deduplicateContent) {
+            const existingResults = await this.crawlStorage.getResults();
+            for (const r of existingResults) {
+                if (r.contentHash) {
+                    this.contentHashes.set(r.contentHash, r.url);
+                }
+            }
+        }
         if (this.options.transport !== 'undici') {
             this.curlAvailable = await hasImpersonate();
             if (this.options.transport === 'auto' && this.curlAvailable && isProtectedDomain(this.baseHost)) {
@@ -354,10 +395,12 @@ export class Spider {
         const pending = new Map();
         const scheduleUrl = async (item) => {
             const normalized = normalizeUrl(item.url);
-            if (await this.crawlQueue.hasVisited(normalized))
-                return;
             if (pending.has(normalized))
                 return;
+            if (await this.crawlQueue.hasVisited(normalized))
+                return;
+            await this.crawlQueue.markVisited(normalized);
+            this._visitedCount++;
             if (item.depth > this.options.maxDepth)
                 return;
             if (this._resultCount + pending.size >= this.options.maxPages)
@@ -373,8 +416,6 @@ export class Spider {
                     return;
                 }
             }
-            await this.crawlQueue.markVisited(normalized);
-            this._visitedCount++;
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
                 .finally(() => {
@@ -416,14 +457,41 @@ export class Spider {
         if (pending.size > 0) {
             await Promise.all(pending.values());
         }
+        while (!this.aborted && this._resultCount < this.options.maxPages) {
+            const remaining = await this.crawlQueue.size();
+            if (remaining === 0 && pending.size === 0)
+                break;
+            this._queueSize = remaining;
+            let nextItem = await this.crawlQueue.pop();
+            while (nextItem && !this.aborted) {
+                this._queueSize = Math.max(0, this._queueSize - 1);
+                if (this._resultCount + pending.size >= this.options.maxPages)
+                    break;
+                await scheduleUrl(nextItem);
+                nextItem = await this.crawlQueue.pop();
+            }
+            if (pending.size > 0) {
+                await Promise.all(pending.values());
+            }
+        }
         this.running = false;
         const pages = await this.crawlStorage.getResults();
         const errors = await this.crawlStorage.getErrors();
         const sitemapAnalysis = this.buildSitemapAnalysis(pages);
         const robotsAnalysis = this.buildRobotsAnalysis();
         const visited = this.crawlQueue instanceof InMemoryCrawlQueue
-            ? this.crawlQueue.getVisited()
+            ? new Set(this.crawlQueue.getVisited())
             : new Set(pages.map(r => r.url));
+        await this.crawlQueue.close?.();
+        await this.crawlStorage.close?.();
+        for (const client of this.proxyClients.values()) {
+            if (typeof client.destroy === 'function') {
+                client.destroy();
+            }
+        }
+        this.proxyClients.clear();
+        await this.proxyAdapter?.close?.();
+        this.domainAvgResponseTime.clear();
         return {
             startUrl: normalizedStart,
             pages,
@@ -615,6 +683,8 @@ export class Spider {
         let lastRetryAfterMs = 0;
         const proxyUrl = this.proxyAdapter ? await this.proxyAdapter.getProxy() : null;
         const executeRequest = async (useCurl) => {
+            if (this.aborted)
+                throw new Error('Crawl aborted');
             if (useCurl && this.curlTransport) {
                 const curlForRequest = proxyUrl
                     ? new CurlTransport(proxyUrl)
@@ -655,6 +725,7 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                signal: this.abortController.signal,
                 beforeRedirect: this.options.onRedirect
                     ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
                     : undefined,
@@ -746,6 +817,9 @@ export class Spider {
                 const isHighQualitySuccess = response.status < 400 && !detection.blocked && !hasCaptchaSignal;
                 if (isHighQualitySuccess) {
                     this.registerDomainSuccess(hostname);
+                    if (timings?.total) {
+                        this.updateAutoThrottle(hostname, timings.total);
+                    }
                 }
                 if (!shouldRetry || attempt === maxAttempts - 1) {
                     if (proxyUrl && this.proxyAdapter?.reportResult) {
@@ -792,7 +866,9 @@ export class Spider {
                         timings,
                     });
                 }
-                await sleep(waitMs);
+                if (this.aborted)
+                    break;
+                await sleep(waitMs, this.abortController.signal);
                 continue;
             }
             catch (error) {
@@ -823,8 +899,10 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.aborted)
+                    break;
                 const waitMs = this.getRetryWait(hostname, attempt + 1);
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
             }
         }
         if (lastResponse) {
@@ -1035,7 +1113,7 @@ export class Spider {
                     if (!link.href)
                         continue;
                     const normalized = normalizeUrl(link.href);
-                    if (!shouldCrawl(normalized, this.baseHost, this.options))
+                    if (!shouldCrawl(normalized, this.baseHost, this.options, this.baseRootDomain))
                         continue;
                     candidateUrls.push(normalized);
                     candidates.push({ url: normalized, depth: item.depth + 1 });
@@ -1059,6 +1137,8 @@ export class Spider {
             }
         }
         catch (error) {
+            if (this.aborted)
+                return;
             const errAttempts = typeof error?.attempts === 'number' ? error.attempts : 1;
             const errRetryCount = typeof error?.retryCount === 'number' ? error.retryCount : Math.max(0, errAttempts - 1);
             const errRetryAfterMs = typeof error?.retryAfterMs === 'number' ? error?.retryAfterMs : 0;
@@ -1144,6 +1224,7 @@ export class Spider {
             consecutiveUndiciFailures: 0,
             lastTransport: 'undici',
             lastCaptchaConfidence: 0,
+            autoThrottleDelay: 0,
         };
         this.domainStates.set(hostname, next);
         return next;
@@ -1246,9 +1327,11 @@ export class Spider {
     async waitForDomainPenalty(hostname) {
         const state = this.getOrCreateDomainState(hostname);
         const now = Date.now();
-        const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
+        const penaltyDelay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
+        const throttleDelay = state.autoThrottleDelay ?? 0;
+        const delay = Math.max(penaltyDelay, throttleDelay);
         if (delay > 0) {
-            await sleep(delay);
+            await sleep(delay, this.abortController.signal);
         }
     }
     registerDomainBlock(hostname) {
@@ -1274,6 +1357,20 @@ export class Spider {
             state.lastCaptchaProvider = undefined;
         }
     }
+    updateAutoThrottle(hostname, responseTimeMs) {
+        const config = this.options.autoThrottle;
+        if (!config)
+            return;
+        const target = (typeof config === 'object' ? config.targetMs : undefined) ?? 500;
+        const minDelay = (typeof config === 'object' ? config.minDelay : undefined) ?? 50;
+        const maxDelay = (typeof config === 'object' ? config.maxDelay : undefined) ?? 30000;
+        const prev = this.domainAvgResponseTime.get(hostname) ?? responseTimeMs;
+        const avg = prev * 0.7 + responseTimeMs * 0.3;
+        this.domainAvgResponseTime.set(hostname, avg);
+        const ratio = avg / target;
+        const state = this.getOrCreateDomainState(hostname);
+        state.autoThrottleDelay = Math.max(minDelay, Math.min(maxDelay, Math.round(this.options.delay * ratio)));
+    }
     getCaptchaRetryMultiplier(provider) {
         if (!provider)
             return 1.2;
@@ -1319,6 +1416,7 @@ export class Spider {
     }
     abort() {
         this.aborted = true;
+        this.abortController.abort();
     }
     isRunning() {
         return this.running;

package/dist/browser/seo/seo-spider.js

@@ -56,7 +56,7 @@ export class SeoSpider {
         });
     }
     async analyzePageDuringCrawl(pageResult, html) {
-        if (pageResult.error || pageResult.status >= 400) {
+        if (pageResult.status >= 400 || !html) {
             const seoPage = { ...pageResult, seoReport: undefined };
             this.seoPages.push(seoPage);
             return;
@@ -141,7 +141,8 @@ export class SeoSpider {
                     results.humans.content = await res.text();
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 const res = await client.get(results.llms.url);
                 if (res.status === 200) {
@@ -149,7 +150,8 @@ export class SeoSpider {
                     results.llms.content = await res.text();
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 const res = await client.get(results.sitemap.url);
                 if (res.status === 200) {
@@ -159,7 +161,8 @@ export class SeoSpider {
                     results.sitemap.urlCount = urlMatches ? urlMatches.length : 0;
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 let res = await client.get(results.manifest.url);
                 if (res.status !== 200) {
@@ -181,7 +184,8 @@ export class SeoSpider {
                     }
                 }
             }
-            catch { }
+            catch {
+            }
             return results;
         }
         catch {

package/dist/browser/seo/validators/rss.js

@@ -25,7 +25,8 @@ export async function discoverFeeds(baseUrl, html) {
                     href = new URL(href, baseUrl).toString();
                     candidateUrls.add(href);
                 }
-                catch { }
+                catch {
+                }
             }
         }
     }
@@ -35,7 +36,8 @@ export async function discoverFeeds(baseUrl, html) {
                 const url = new URL(path, baseUrl).toString();
                 candidateUrls.add(url);
             }
-            catch { }
+            catch {
+            }
         }
     }
     const client = createClient({ timeout: 8000 });