recker 1.0.95 → 1.0.96

package/dist/browser/scrape/crawl-queue.d.ts

@@ -20,6 +20,8 @@ export declare class InMemoryCrawlQueue implements CrawlQueueAdapter {
     private head;
     private tail;
     private visited;
+    private mode;
+    constructor(mode?: 'fifo' | 'lifo');
     push(item: CrawlQueueItem): Promise<void>;
     pushBatch(items: CrawlQueueItem[]): Promise<void>;
     pop(): Promise<CrawlQueueItem | null>;

package/dist/browser/scrape/crawl-queue.js

@@ -3,6 +3,10 @@ export class InMemoryCrawlQueue {
     head = 0;
     tail = 0;
     visited = new Set();
+    mode;
+    constructor(mode = 'fifo') {
+        this.mode = mode;
+    }
     async push(item) {
         this.queue[this.tail++] = item;
     }
@@ -12,6 +16,16 @@ export class InMemoryCrawlQueue {
         }
     }
     async pop() {
+        if (this.mode === 'lifo') {
+            while (this.tail > this.head) {
+                this.tail--;
+                const item = this.queue[this.tail];
+                this.queue[this.tail] = undefined;
+                if (item)
+                    return item;
+            }
+            return null;
+        }
         while (this.head < this.tail) {
             const item = this.queue[this.head];
             this.queue[this.head] = undefined;

package/dist/browser/scrape/spider.d.ts

@@ -11,7 +11,7 @@ type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
     maxDepth?: number;
     maxPages?: number;
-    sameDomain?: boolean;
+    sameDomain?: boolean | 'exact' | 'subdomain';
     concurrency?: number;
     timeout?: number;
     delay?: number;
@@ -63,8 +63,14 @@ export interface SpiderOptions {
     domainRateLimit?: {
         maxPerSecond?: number;
     };
+    autoThrottle?: boolean | {
+        targetMs?: number;
+        minDelay?: number;
+        maxDelay?: number;
+    };
     deduplicateContent?: boolean;
     resume?: boolean;
+    strategy?: 'bfs' | 'dfs';
     crawlQueue?: CrawlQueueAdapter;
     crawlStorage?: CrawlStorageAdapter;
 }
@@ -190,12 +196,14 @@ export declare class Spider {
     private _queueSize;
     private _resultCount;
     private baseHost;
+    private baseRootDomain;
     private running;
     private aborted;
     private abortController;
     private pendingCount;
     private domainRequestTimestamps;
     private contentHashes;
+    private domainAvgResponseTime;
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
@@ -224,6 +232,7 @@ export declare class Spider {
     private waitForDomainPenalty;
     private registerDomainBlock;
     private registerDomainSuccess;
+    private updateAutoThrottle;
     private getCaptchaRetryMultiplier;
     private registerDomainChallenge;
     private getRetryWait;

package/dist/browser/scrape/spider.js

@@ -108,15 +108,30 @@ function normalizeUrl(urlStr) {
         return urlStr;
     }
 }
-function shouldCrawl(url, baseHost, options) {
+function getRootDomain(hostname) {
+    const parts = hostname.replace(/^www\./, '').split('.');
+    if (parts.length >= 3 && parts[parts.length - 2].length <= 3) {
+        return parts.slice(-3).join('.');
+    }
+    return parts.slice(-2).join('.');
+}
+function shouldCrawl(url, baseHost, options, baseRootDomain) {
     try {
         const parsed = new URL(url);
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
         const hostname = parsed.hostname.replace(/^www\./, '');
-        if (options.sameDomain !== false && hostname !== baseHost) {
-            return false;
+        const sameDomain = options.sameDomain;
+        if (sameDomain === 'subdomain') {
+            const pageRoot = getRootDomain(hostname);
+            const rootDomain = baseRootDomain ?? getRootDomain(baseHost);
+            if (pageRoot !== rootDomain)
+                return false;
+        }
+        else if (sameDomain !== false) {
+            if (hostname !== baseHost)
+                return false;
         }
         const skipExtensions = [
             '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
@@ -193,12 +208,14 @@ export class Spider {
     _queueSize = 0;
     _resultCount = 0;
     baseHost = '';
+    baseRootDomain = '';
     running = false;
     aborted = false;
     abortController = new AbortController();
     pendingCount = 0;
     domainRequestTimestamps = new Map();
     contentHashes = new Map();
+    domainAvgResponseTime = new Map();
     blockedDomains = new Set();
     curlTransport = null;
     curlAvailable = false;
@@ -285,8 +302,10 @@ export class Spider {
             extract: extractSchema,
             parserOptions: options.parserOptions,
             domainRateLimit: options.domainRateLimit,
+            autoThrottle: options.autoThrottle ?? false,
             deduplicateContent: options.deduplicateContent ?? false,
             resume: options.resume ?? false,
+            strategy: options.strategy ?? 'bfs',
         };
         if (options.proxy) {
             if (typeof options.proxy === 'string') {
@@ -316,7 +335,7 @@ export class Spider {
                 interval: this.options.delay,
             } : {}),
         });
-        this.crawlQueue = options.crawlQueue ?? new InMemoryCrawlQueue();
+        this.crawlQueue = options.crawlQueue ?? new InMemoryCrawlQueue(this.options.strategy === 'dfs' ? 'lifo' : 'fifo');
         this.crawlStorage = options.crawlStorage ?? new InMemoryCrawlStorage();
     }
     async crawl(startUrl) {
@@ -325,6 +344,7 @@ export class Spider {
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
         this.baseHost = new URL(normalizedStart).hostname.replace(/^www\./, '');
+        this.baseRootDomain = getRootDomain(this.baseHost);
         if (!this.options.resume) {
             await this.crawlQueue.clear();
             await this.crawlStorage.clear();
@@ -333,6 +353,7 @@ export class Spider {
             this._resultCount = 0;
             this.domainRequestTimestamps.clear();
             this.contentHashes.clear();
+            this.domainAvgResponseTime.clear();
         }
         else {
             this._queueSize = await this.crawlQueue.size();
@@ -463,6 +484,14 @@ export class Spider {
             : new Set(pages.map(r => r.url));
         await this.crawlQueue.close?.();
         await this.crawlStorage.close?.();
+        for (const client of this.proxyClients.values()) {
+            if (typeof client.destroy === 'function') {
+                client.destroy();
+            }
+        }
+        this.proxyClients.clear();
+        await this.proxyAdapter?.close?.();
+        this.domainAvgResponseTime.clear();
         return {
             startUrl: normalizedStart,
             pages,
@@ -788,6 +817,9 @@ export class Spider {
                 const isHighQualitySuccess = response.status < 400 && !detection.blocked && !hasCaptchaSignal;
                 if (isHighQualitySuccess) {
                     this.registerDomainSuccess(hostname);
+                    if (timings?.total) {
+                        this.updateAutoThrottle(hostname, timings.total);
+                    }
                 }
                 if (!shouldRetry || attempt === maxAttempts - 1) {
                     if (proxyUrl && this.proxyAdapter?.reportResult) {
@@ -1081,7 +1113,7 @@ export class Spider {
                     if (!link.href)
                         continue;
                     const normalized = normalizeUrl(link.href);
-                    if (!shouldCrawl(normalized, this.baseHost, this.options))
+                    if (!shouldCrawl(normalized, this.baseHost, this.options, this.baseRootDomain))
                         continue;
                     candidateUrls.push(normalized);
                     candidates.push({ url: normalized, depth: item.depth + 1 });
@@ -1192,6 +1224,7 @@ export class Spider {
             consecutiveUndiciFailures: 0,
             lastTransport: 'undici',
             lastCaptchaConfidence: 0,
+            autoThrottleDelay: 0,
         };
         this.domainStates.set(hostname, next);
         return next;
@@ -1294,7 +1327,9 @@ export class Spider {
     async waitForDomainPenalty(hostname) {
         const state = this.getOrCreateDomainState(hostname);
         const now = Date.now();
-        const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
+        const penaltyDelay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
+        const throttleDelay = state.autoThrottleDelay ?? 0;
+        const delay = Math.max(penaltyDelay, throttleDelay);
         if (delay > 0) {
             await sleep(delay, this.abortController.signal);
         }
@@ -1322,6 +1357,20 @@ export class Spider {
             state.lastCaptchaProvider = undefined;
         }
     }
+    updateAutoThrottle(hostname, responseTimeMs) {
+        const config = this.options.autoThrottle;
+        if (!config)
+            return;
+        const target = (typeof config === 'object' ? config.targetMs : undefined) ?? 500;
+        const minDelay = (typeof config === 'object' ? config.minDelay : undefined) ?? 50;
+        const maxDelay = (typeof config === 'object' ? config.maxDelay : undefined) ?? 30000;
+        const prev = this.domainAvgResponseTime.get(hostname) ?? responseTimeMs;
+        const avg = prev * 0.7 + responseTimeMs * 0.3;
+        this.domainAvgResponseTime.set(hostname, avg);
+        const ratio = avg / target;
+        const state = this.getOrCreateDomainState(hostname);
+        state.autoThrottleDelay = Math.max(minDelay, Math.min(maxDelay, Math.round(this.options.delay * ratio)));
+    }
     getCaptchaRetryMultiplier(provider) {
         if (!provider)
             return 1.2;

package/dist/browser/seo/seo-spider.js

@@ -141,7 +141,8 @@ export class SeoSpider {
                     results.humans.content = await res.text();
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 const res = await client.get(results.llms.url);
                 if (res.status === 200) {
@@ -149,7 +150,8 @@ export class SeoSpider {
                     results.llms.content = await res.text();
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 const res = await client.get(results.sitemap.url);
                 if (res.status === 200) {
@@ -159,7 +161,8 @@ export class SeoSpider {
                     results.sitemap.urlCount = urlMatches ? urlMatches.length : 0;
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 let res = await client.get(results.manifest.url);
                 if (res.status !== 200) {
@@ -181,7 +184,8 @@ export class SeoSpider {
                     }
                 }
             }
-            catch { }
+            catch {
+            }
             return results;
         }
         catch {

package/dist/browser/seo/validators/rss.js

@@ -25,7 +25,8 @@ export async function discoverFeeds(baseUrl, html) {
                     href = new URL(href, baseUrl).toString();
                     candidateUrls.add(href);
                 }
-                catch { }
+                catch {
+                }
             }
         }
     }
@@ -35,7 +36,8 @@ export async function discoverFeeds(baseUrl, html) {
                 const url = new URL(path, baseUrl).toString();
                 candidateUrls.add(url);
             }
-            catch { }
+            catch {
+            }
         }
     }
     const client = createClient({ timeout: 8000 });

package/dist/browser/utils/block-detector.js

@@ -526,18 +526,42 @@ export function detectBlock(response, body) {
     if (body) {
         const isLongBody = body.length > 100_000;
         const checkBody = isLongBody ? body.slice(0, 30_000) : body;
-        const challengeHint = /just a moment|attention required|access denied|checking your browser|security check|human verification|suspicious activity|captcha|recaptcha|hcaptcha|turnstile|datadome|cloudflare/i.test(checkBody.slice(0, 8000));
-        for (const { pattern, reason, confidence, description } of BLOCK_PATTERNS) {
-            if (isLongBody && response.status === 200 && confidence < 0.85 && !challengeHint) {
-                continue;
-            }
-            if (pattern.test(checkBody)) {
-                results.push({
-                    blocked: true,
-                    reason,
-                    confidence,
-                    description,
-                });
+        const lowerBody = checkBody.slice(0, 8000).toLowerCase();
+        const hasAnyChallengeKeyword = lowerBody.includes('captcha') ||
+            lowerBody.includes('cloudflare') ||
+            lowerBody.includes('datadome') ||
+            lowerBody.includes('blocked') ||
+            lowerBody.includes('denied') ||
+            lowerBody.includes('forbidden') ||
+            lowerBody.includes('too many') ||
+            lowerBody.includes('rate limit') ||
+            lowerBody.includes('checking your browser') ||
+            lowerBody.includes('just a moment') ||
+            lowerBody.includes('security check') ||
+            lowerBody.includes('human verification') ||
+            lowerBody.includes('perimeterx') ||
+            lowerBody.includes('incapsula') ||
+            lowerBody.includes('imperva') ||
+            lowerBody.includes('akamai') ||
+            lowerBody.includes('httpservice') ||
+            lowerBody.includes('enablejs') ||
+            lowerBody.includes('verify') ||
+            lowerBody.includes('suspicious') ||
+            lowerBody.includes('bot') ||
+            lowerBody.includes('access denied');
+        if (hasAnyChallengeKeyword || response.status !== 200) {
+            for (const { pattern, reason, confidence, description } of BLOCK_PATTERNS) {
+                if (isLongBody && response.status === 200 && confidence < 0.85 && !hasAnyChallengeKeyword) {
+                    continue;
+                }
+                if (pattern.test(checkBody)) {
+                    results.push({
+                        blocked: true,
+                        reason,
+                        confidence,
+                        description,
+                    });
+                }
             }
         }
         if (body.length < 5000 &&
@@ -603,8 +627,25 @@ export function detectCaptcha(response, body) {
     if (body) {
         const checkBody = body.length < 120_000 ? body : body.slice(0, 120_000);
         const isTinyBody = checkBody.length > 0 && checkBody.length < 12000;
+        const lowerSnippet = checkBody.slice(0, 8000).toLowerCase();
+        const hasAnyCaptchaKeyword = lowerSnippet.includes('captcha') ||
+            lowerSnippet.includes('recaptcha') ||
+            lowerSnippet.includes('hcaptcha') ||
+            lowerSnippet.includes('turnstile') ||
+            lowerSnippet.includes('challenge') ||
+            lowerSnippet.includes('cloudflare') ||
+            lowerSnippet.includes('datadome') ||
+            lowerSnippet.includes('perimeterx') ||
+            lowerSnippet.includes('funcaptcha') ||
+            lowerSnippet.includes('arkose') ||
+            lowerSnippet.includes('sitekey') ||
+            lowerSnippet.includes('just a moment') ||
+            lowerSnippet.includes('human verification');
         const hasHtmlTags = /<html|<head|<body|<script|<meta/i.test(checkBody);
-        const challengeTitleOrText = /just a moment|attention required|human verification|verify you are human|security check/i.test(checkBody);
+        const challengeTitleOrText = hasAnyCaptchaKeyword ||
+            lowerSnippet.includes('attention required') ||
+            lowerSnippet.includes('verify you are human') ||
+            lowerSnippet.includes('security check');
         if (isTinyBody && challengeTitleOrText && hasHtmlTags) {
             addMatch(matches, 'generic', 0.6, 'Tiny HTML response with challenge-like text');
         }
@@ -616,24 +657,38 @@ export function detectCaptcha(response, body) {
                 addMatch(matches, provider, confidence, description);
             }
         }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_SCRIPT_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+        if (hasAnyCaptchaKeyword || !(response.status >= 200 && response.status < 300)) {
+            for (const { pattern, provider, confidence, description } of CAPTCHA_SCRIPT_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
             }
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_FORM_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+            for (const { pattern, provider, confidence, description } of CAPTCHA_FORM_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
             }
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_DOM_MARKERS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+            for (const { pattern, provider, confidence, description } of CAPTCHA_DOM_MARKERS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
             }
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_NOSCRIPT_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+            for (const { pattern, provider, confidence, description } of CAPTCHA_NOSCRIPT_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
+            }
+            for (const { pattern, provider, confidence, description } of CAPTCHA_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
+            }
+            if (/<script[^>]*src=["'][^"']*(?:challenges\.cloudflare\.com|recaptcha\/api\.js|hcaptcha\.com\/1\/api\.js|challenges\.perimeterx\.)[^"']*["'][^>]*>/i.test(checkBody.slice(0, 120_000))) {
+                addMatch(matches, 'cloudflare', 0.6, 'Known challenge script reference found in HTML');
+            }
+            if ((response.status >= 500 && response.status < 600) &&
+                /cloudflare|challenge|verification|blocked|security/i.test(checkBody.slice(0, 3000))) {
+                addMatch(matches, 'generic', 0.2, 'Server challenge-like text on error response');
             }
         }
         if (response.status === 403 || response.status === 503) {
@@ -653,18 +708,6 @@ export function detectCaptcha(response, body) {
             /refresh|challenge|verify|captcha|javascript challenge|cloudflare|bot/i.test((location || ''))) {
             addMatch(matches, 'generic', 0.67, 'Redirect location indicates bot challenge flow');
         }
-        if (/<script[^>]*src=["'][^"']*(?:challenges\.cloudflare\.com|recaptcha\/api\.js|hcaptcha\.com\/1\/api\.js|challenges\.perimeterx\.)[^"']*["'][^>]*>/i.test(checkBody.slice(0, 120_000))) {
-            addMatch(matches, 'cloudflare', 0.6, 'Known challenge script reference found in HTML');
-        }
-        if ((response.status >= 500 && response.status < 600) &&
-            /cloudflare|challenge|verification|blocked|security/i.test(checkBody.slice(0, 3000))) {
-            addMatch(matches, 'generic', 0.2, 'Server challenge-like text on error response');
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
-            }
-        }
     }
     const result = scoreByProvider(matches);
     if (result.confidence >= 0.5 || hasCaptchaHeader) {
@@ -681,23 +724,33 @@ export function detectCaptcha(response, body) {
     };
 }
 export function isProtectedDomain(hostname) {
-    const protectedPatterns = [
-        /cloudflare/i,
-        /\.gov$/i,
-        /\.mil$/i,
-        /linkedin\.com$/i,
-        /twitter\.com$/i,
-        /x\.com$/i,
-        /instagram\.com$/i,
-        /facebook\.com$/i,
-        /amazon\./i,
-        /google\./i,
-        /microsoft\.com$/i,
-        /apple\.com$/i,
-        /netflix\.com$/i,
-        /spotify\.com$/i,
+    const h = hostname.toLowerCase();
+    const protectedDomains = [
+        'linkedin.com',
+        'twitter.com',
+        'x.com',
+        'instagram.com',
+        'facebook.com',
+        'amazon.com',
+        'amazon.co.uk',
+        'amazon.de',
+        'amazon.co.jp',
+        'google.com',
+        'microsoft.com',
+        'apple.com',
+        'netflix.com',
+        'spotify.com',
     ];
-    return protectedPatterns.some(p => p.test(hostname));
+    const protectedTlds = ['.gov', '.mil'];
+    for (const domain of protectedDomains) {
+        if (h === domain || h.endsWith('.' + domain))
+            return true;
+    }
+    for (const tld of protectedTlds) {
+        if (h.endsWith(tld))
+            return true;
+    }
+    return false;
 }
 export function isCloudflareChallenge(response, body) {
     const cfRay = response.headers.get('cf-ray');

package/dist/scrape/crawl-queue.d.ts

@@ -20,6 +20,8 @@ export declare class InMemoryCrawlQueue implements CrawlQueueAdapter {
     private head;
     private tail;
     private visited;
+    private mode;
+    constructor(mode?: 'fifo' | 'lifo');
     push(item: CrawlQueueItem): Promise<void>;
     pushBatch(items: CrawlQueueItem[]): Promise<void>;
     pop(): Promise<CrawlQueueItem | null>;

package/dist/scrape/crawl-queue.js

@@ -3,6 +3,10 @@ export class InMemoryCrawlQueue {
     head = 0;
     tail = 0;
     visited = new Set();
+    mode;
+    constructor(mode = 'fifo') {
+        this.mode = mode;
+    }
     async push(item) {
         this.queue[this.tail++] = item;
     }
@@ -12,6 +16,16 @@ export class InMemoryCrawlQueue {
         }
     }
     async pop() {
+        if (this.mode === 'lifo') {
+            while (this.tail > this.head) {
+                this.tail--;
+                const item = this.queue[this.tail];
+                this.queue[this.tail] = undefined;
+                if (item)
+                    return item;
+            }
+            return null;
+        }
         while (this.head < this.tail) {
             const item = this.queue[this.head];
             this.queue[this.head] = undefined;

package/dist/scrape/spider.d.ts

@@ -11,7 +11,7 @@ type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
     maxDepth?: number;
     maxPages?: number;
-    sameDomain?: boolean;
+    sameDomain?: boolean | 'exact' | 'subdomain';
     concurrency?: number;
     timeout?: number;
     delay?: number;
@@ -63,8 +63,14 @@ export interface SpiderOptions {
     domainRateLimit?: {
         maxPerSecond?: number;
     };
+    autoThrottle?: boolean | {
+        targetMs?: number;
+        minDelay?: number;
+        maxDelay?: number;
+    };
     deduplicateContent?: boolean;
     resume?: boolean;
+    strategy?: 'bfs' | 'dfs';
     crawlQueue?: CrawlQueueAdapter;
     crawlStorage?: CrawlStorageAdapter;
 }
@@ -190,12 +196,14 @@ export declare class Spider {
     private _queueSize;
     private _resultCount;
     private baseHost;
+    private baseRootDomain;
     private running;
     private aborted;
     private abortController;
     private pendingCount;
     private domainRequestTimestamps;
     private contentHashes;
+    private domainAvgResponseTime;
     private blockedDomains;
     private curlTransport;
     private curlAvailable;
@@ -224,6 +232,7 @@ export declare class Spider {
     private waitForDomainPenalty;
     private registerDomainBlock;
     private registerDomainSuccess;
+    private updateAutoThrottle;
     private getCaptchaRetryMultiplier;
     private registerDomainChallenge;
     private getRetryWait;

package/dist/scrape/spider.js

@@ -108,15 +108,30 @@ function normalizeUrl(urlStr) {
         return urlStr;
     }
 }
-function shouldCrawl(url, baseHost, options) {
+function getRootDomain(hostname) {
+    const parts = hostname.replace(/^www\./, '').split('.');
+    if (parts.length >= 3 && parts[parts.length - 2].length <= 3) {
+        return parts.slice(-3).join('.');
+    }
+    return parts.slice(-2).join('.');
+}
+function shouldCrawl(url, baseHost, options, baseRootDomain) {
     try {
         const parsed = new URL(url);
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
         const hostname = parsed.hostname.replace(/^www\./, '');
-        if (options.sameDomain !== false && hostname !== baseHost) {
-            return false;
+        const sameDomain = options.sameDomain;
+        if (sameDomain === 'subdomain') {
+            const pageRoot = getRootDomain(hostname);
+            const rootDomain = baseRootDomain ?? getRootDomain(baseHost);
+            if (pageRoot !== rootDomain)
+                return false;
+        }
+        else if (sameDomain !== false) {
+            if (hostname !== baseHost)
+                return false;
         }
         const skipExtensions = [
             '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
@@ -193,12 +208,14 @@ export class Spider {
     _queueSize = 0;
     _resultCount = 0;
     baseHost = '';
+    baseRootDomain = '';
     running = false;
     aborted = false;
     abortController = new AbortController();
     pendingCount = 0;
     domainRequestTimestamps = new Map();
     contentHashes = new Map();
+    domainAvgResponseTime = new Map();
     blockedDomains = new Set();
     curlTransport = null;
     curlAvailable = false;
@@ -285,8 +302,10 @@ export class Spider {
             extract: extractSchema,
             parserOptions: options.parserOptions,
             domainRateLimit: options.domainRateLimit,
+            autoThrottle: options.autoThrottle ?? false,
             deduplicateContent: options.deduplicateContent ?? false,
             resume: options.resume ?? false,
+            strategy: options.strategy ?? 'bfs',
         };
         if (options.proxy) {
             if (typeof options.proxy === 'string') {
@@ -316,7 +335,7 @@ export class Spider {
                 interval: this.options.delay,
             } : {}),
         });
-        this.crawlQueue = options.crawlQueue ?? new InMemoryCrawlQueue();
+        this.crawlQueue = options.crawlQueue ?? new InMemoryCrawlQueue(this.options.strategy === 'dfs' ? 'lifo' : 'fifo');
         this.crawlStorage = options.crawlStorage ?? new InMemoryCrawlStorage();
     }
     async crawl(startUrl) {
@@ -325,6 +344,7 @@ export class Spider {
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
         this.baseHost = new URL(normalizedStart).hostname.replace(/^www\./, '');
+        this.baseRootDomain = getRootDomain(this.baseHost);
         if (!this.options.resume) {
             await this.crawlQueue.clear();
             await this.crawlStorage.clear();
@@ -333,6 +353,7 @@ export class Spider {
             this._resultCount = 0;
             this.domainRequestTimestamps.clear();
             this.contentHashes.clear();
+            this.domainAvgResponseTime.clear();
         }
         else {
             this._queueSize = await this.crawlQueue.size();
@@ -463,6 +484,14 @@ export class Spider {
             : new Set(pages.map(r => r.url));
         await this.crawlQueue.close?.();
         await this.crawlStorage.close?.();
+        for (const client of this.proxyClients.values()) {
+            if (typeof client.destroy === 'function') {
+                client.destroy();
+            }
+        }
+        this.proxyClients.clear();
+        await this.proxyAdapter?.close?.();
+        this.domainAvgResponseTime.clear();
         return {
             startUrl: normalizedStart,
             pages,
@@ -788,6 +817,9 @@ export class Spider {
                 const isHighQualitySuccess = response.status < 400 && !detection.blocked && !hasCaptchaSignal;
                 if (isHighQualitySuccess) {
                     this.registerDomainSuccess(hostname);
+                    if (timings?.total) {
+                        this.updateAutoThrottle(hostname, timings.total);
+                    }
                 }
                 if (!shouldRetry || attempt === maxAttempts - 1) {
                     if (proxyUrl && this.proxyAdapter?.reportResult) {
@@ -1081,7 +1113,7 @@ export class Spider {
                     if (!link.href)
                         continue;
                     const normalized = normalizeUrl(link.href);
-                    if (!shouldCrawl(normalized, this.baseHost, this.options))
+                    if (!shouldCrawl(normalized, this.baseHost, this.options, this.baseRootDomain))
                         continue;
                     candidateUrls.push(normalized);
                     candidates.push({ url: normalized, depth: item.depth + 1 });
@@ -1192,6 +1224,7 @@ export class Spider {
             consecutiveUndiciFailures: 0,
             lastTransport: 'undici',
             lastCaptchaConfidence: 0,
+            autoThrottleDelay: 0,
         };
         this.domainStates.set(hostname, next);
         return next;
@@ -1294,7 +1327,9 @@ export class Spider {
     async waitForDomainPenalty(hostname) {
         const state = this.getOrCreateDomainState(hostname);
         const now = Date.now();
-        const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
+        const penaltyDelay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
+        const throttleDelay = state.autoThrottleDelay ?? 0;
+        const delay = Math.max(penaltyDelay, throttleDelay);
         if (delay > 0) {
             await sleep(delay, this.abortController.signal);
         }
@@ -1322,6 +1357,20 @@ export class Spider {
             state.lastCaptchaProvider = undefined;
         }
     }
+    updateAutoThrottle(hostname, responseTimeMs) {
+        const config = this.options.autoThrottle;
+        if (!config)
+            return;
+        const target = (typeof config === 'object' ? config.targetMs : undefined) ?? 500;
+        const minDelay = (typeof config === 'object' ? config.minDelay : undefined) ?? 50;
+        const maxDelay = (typeof config === 'object' ? config.maxDelay : undefined) ?? 30000;
+        const prev = this.domainAvgResponseTime.get(hostname) ?? responseTimeMs;
+        const avg = prev * 0.7 + responseTimeMs * 0.3;
+        this.domainAvgResponseTime.set(hostname, avg);
+        const ratio = avg / target;
+        const state = this.getOrCreateDomainState(hostname);
+        state.autoThrottleDelay = Math.max(minDelay, Math.min(maxDelay, Math.round(this.options.delay * ratio)));
+    }
     getCaptchaRetryMultiplier(provider) {
         if (!provider)
             return 1.2;

package/dist/seo/seo-spider.js

@@ -141,7 +141,8 @@ export class SeoSpider {
                     results.humans.content = await res.text();
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 const res = await client.get(results.llms.url);
                 if (res.status === 200) {
@@ -149,7 +150,8 @@ export class SeoSpider {
                     results.llms.content = await res.text();
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 const res = await client.get(results.sitemap.url);
                 if (res.status === 200) {
@@ -159,7 +161,8 @@ export class SeoSpider {
                     results.sitemap.urlCount = urlMatches ? urlMatches.length : 0;
                 }
             }
-            catch { }
+            catch {
+            }
             try {
                 let res = await client.get(results.manifest.url);
                 if (res.status !== 200) {
@@ -181,7 +184,8 @@ export class SeoSpider {
                     }
                 }
             }
-            catch { }
+            catch {
+            }
             return results;
         }
         catch {

package/dist/seo/validators/rss.js

@@ -25,7 +25,8 @@ export async function discoverFeeds(baseUrl, html) {
                     href = new URL(href, baseUrl).toString();
                     candidateUrls.add(href);
                 }
-                catch { }
+                catch {
+                }
             }
         }
     }
@@ -35,7 +36,8 @@ export async function discoverFeeds(baseUrl, html) {
                 const url = new URL(path, baseUrl).toString();
                 candidateUrls.add(url);
             }
-            catch { }
+            catch {
+            }
         }
     }
     const client = createClient({ timeout: 8000 });

package/dist/utils/block-detector.js

@@ -526,18 +526,42 @@ export function detectBlock(response, body) {
     if (body) {
         const isLongBody = body.length > 100_000;
         const checkBody = isLongBody ? body.slice(0, 30_000) : body;
-        const challengeHint = /just a moment|attention required|access denied|checking your browser|security check|human verification|suspicious activity|captcha|recaptcha|hcaptcha|turnstile|datadome|cloudflare/i.test(checkBody.slice(0, 8000));
-        for (const { pattern, reason, confidence, description } of BLOCK_PATTERNS) {
-            if (isLongBody && response.status === 200 && confidence < 0.85 && !challengeHint) {
-                continue;
-            }
-            if (pattern.test(checkBody)) {
-                results.push({
-                    blocked: true,
-                    reason,
-                    confidence,
-                    description,
-                });
+        const lowerBody = checkBody.slice(0, 8000).toLowerCase();
+        const hasAnyChallengeKeyword = lowerBody.includes('captcha') ||
+            lowerBody.includes('cloudflare') ||
+            lowerBody.includes('datadome') ||
+            lowerBody.includes('blocked') ||
+            lowerBody.includes('denied') ||
+            lowerBody.includes('forbidden') ||
+            lowerBody.includes('too many') ||
+            lowerBody.includes('rate limit') ||
+            lowerBody.includes('checking your browser') ||
+            lowerBody.includes('just a moment') ||
+            lowerBody.includes('security check') ||
+            lowerBody.includes('human verification') ||
+            lowerBody.includes('perimeterx') ||
+            lowerBody.includes('incapsula') ||
+            lowerBody.includes('imperva') ||
+            lowerBody.includes('akamai') ||
+            lowerBody.includes('httpservice') ||
+            lowerBody.includes('enablejs') ||
+            lowerBody.includes('verify') ||
+            lowerBody.includes('suspicious') ||
+            lowerBody.includes('bot') ||
+            lowerBody.includes('access denied');
+        if (hasAnyChallengeKeyword || response.status !== 200) {
+            for (const { pattern, reason, confidence, description } of BLOCK_PATTERNS) {
+                if (isLongBody && response.status === 200 && confidence < 0.85 && !hasAnyChallengeKeyword) {
+                    continue;
+                }
+                if (pattern.test(checkBody)) {
+                    results.push({
+                        blocked: true,
+                        reason,
+                        confidence,
+                        description,
+                    });
+                }
             }
         }
         if (body.length < 5000 &&
@@ -603,8 +627,25 @@ export function detectCaptcha(response, body) {
     if (body) {
         const checkBody = body.length < 120_000 ? body : body.slice(0, 120_000);
         const isTinyBody = checkBody.length > 0 && checkBody.length < 12000;
+        const lowerSnippet = checkBody.slice(0, 8000).toLowerCase();
+        const hasAnyCaptchaKeyword = lowerSnippet.includes('captcha') ||
+            lowerSnippet.includes('recaptcha') ||
+            lowerSnippet.includes('hcaptcha') ||
+            lowerSnippet.includes('turnstile') ||
+            lowerSnippet.includes('challenge') ||
+            lowerSnippet.includes('cloudflare') ||
+            lowerSnippet.includes('datadome') ||
+            lowerSnippet.includes('perimeterx') ||
+            lowerSnippet.includes('funcaptcha') ||
+            lowerSnippet.includes('arkose') ||
+            lowerSnippet.includes('sitekey') ||
+            lowerSnippet.includes('just a moment') ||
+            lowerSnippet.includes('human verification');
         const hasHtmlTags = /<html|<head|<body|<script|<meta/i.test(checkBody);
-        const challengeTitleOrText = /just a moment|attention required|human verification|verify you are human|security check/i.test(checkBody);
+        const challengeTitleOrText = hasAnyCaptchaKeyword ||
+            lowerSnippet.includes('attention required') ||
+            lowerSnippet.includes('verify you are human') ||
+            lowerSnippet.includes('security check');
         if (isTinyBody && challengeTitleOrText && hasHtmlTags) {
             addMatch(matches, 'generic', 0.6, 'Tiny HTML response with challenge-like text');
         }
@@ -616,24 +657,38 @@ export function detectCaptcha(response, body) {
                 addMatch(matches, provider, confidence, description);
             }
         }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_SCRIPT_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+        if (hasAnyCaptchaKeyword || !(response.status >= 200 && response.status < 300)) {
+            for (const { pattern, provider, confidence, description } of CAPTCHA_SCRIPT_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
             }
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_FORM_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+            for (const { pattern, provider, confidence, description } of CAPTCHA_FORM_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
             }
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_DOM_MARKERS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+            for (const { pattern, provider, confidence, description } of CAPTCHA_DOM_MARKERS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
             }
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_NOSCRIPT_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
+            for (const { pattern, provider, confidence, description } of CAPTCHA_NOSCRIPT_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
+            }
+            for (const { pattern, provider, confidence, description } of CAPTCHA_PATTERNS) {
+                if (pattern.test(checkBody)) {
+                    addMatch(matches, provider, confidence, description);
+                }
+            }
+            if (/<script[^>]*src=["'][^"']*(?:challenges\.cloudflare\.com|recaptcha\/api\.js|hcaptcha\.com\/1\/api\.js|challenges\.perimeterx\.)[^"']*["'][^>]*>/i.test(checkBody.slice(0, 120_000))) {
+                addMatch(matches, 'cloudflare', 0.6, 'Known challenge script reference found in HTML');
+            }
+            if ((response.status >= 500 && response.status < 600) &&
+                /cloudflare|challenge|verification|blocked|security/i.test(checkBody.slice(0, 3000))) {
+                addMatch(matches, 'generic', 0.2, 'Server challenge-like text on error response');
             }
         }
         if (response.status === 403 || response.status === 503) {
@@ -653,18 +708,6 @@ export function detectCaptcha(response, body) {
             /refresh|challenge|verify|captcha|javascript challenge|cloudflare|bot/i.test((location || ''))) {
             addMatch(matches, 'generic', 0.67, 'Redirect location indicates bot challenge flow');
         }
-        if (/<script[^>]*src=["'][^"']*(?:challenges\.cloudflare\.com|recaptcha\/api\.js|hcaptcha\.com\/1\/api\.js|challenges\.perimeterx\.)[^"']*["'][^>]*>/i.test(checkBody.slice(0, 120_000))) {
-            addMatch(matches, 'cloudflare', 0.6, 'Known challenge script reference found in HTML');
-        }
-        if ((response.status >= 500 && response.status < 600) &&
-            /cloudflare|challenge|verification|blocked|security/i.test(checkBody.slice(0, 3000))) {
-            addMatch(matches, 'generic', 0.2, 'Server challenge-like text on error response');
-        }
-        for (const { pattern, provider, confidence, description } of CAPTCHA_PATTERNS) {
-            if (pattern.test(checkBody)) {
-                addMatch(matches, provider, confidence, description);
-            }
-        }
     }
     const result = scoreByProvider(matches);
     if (result.confidence >= 0.5 || hasCaptchaHeader) {
@@ -681,23 +724,33 @@ export function detectCaptcha(response, body) {
     };
 }
 export function isProtectedDomain(hostname) {
-    const protectedPatterns = [
-        /cloudflare/i,
-        /\.gov$/i,
-        /\.mil$/i,
-        /linkedin\.com$/i,
-        /twitter\.com$/i,
-        /x\.com$/i,
-        /instagram\.com$/i,
-        /facebook\.com$/i,
-        /amazon\./i,
-        /google\./i,
-        /microsoft\.com$/i,
-        /apple\.com$/i,
-        /netflix\.com$/i,
-        /spotify\.com$/i,
+    const h = hostname.toLowerCase();
+    const protectedDomains = [
+        'linkedin.com',
+        'twitter.com',
+        'x.com',
+        'instagram.com',
+        'facebook.com',
+        'amazon.com',
+        'amazon.co.uk',
+        'amazon.de',
+        'amazon.co.jp',
+        'google.com',
+        'microsoft.com',
+        'apple.com',
+        'netflix.com',
+        'spotify.com',
     ];
-    return protectedPatterns.some(p => p.test(hostname));
+    const protectedTlds = ['.gov', '.mil'];
+    for (const domain of protectedDomains) {
+        if (h === domain || h.endsWith('.' + domain))
+            return true;
+    }
+    for (const tld of protectedTlds) {
+        if (h.endsWith(tld))
+            return true;
+    }
+    return false;
 }
 export function isCloudflareChallenge(response, body) {
     const cfRay = response.headers.get('cf-ray');

package/dist/version.js

@@ -1,4 +1,4 @@
-const VERSION = '1.0.95';
+const VERSION = '1.0.96';
 let _version = null;
 export async function getVersion() {
     if (_version)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.95",
+  "version": "1.0.96",
   "description": "Multi-Protocol SDK for the AI Era - HTTP, WebSocket, DNS, FTP, SFTP, Telnet, HLS unified with AI providers and MCP tools",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",