recker 1.0.94 → 1.0.95

package/dist/browser/scrape/crawl-queue.d.ts

@@ -17,6 +17,8 @@ export interface CrawlQueueAdapter {
 }
 export declare class InMemoryCrawlQueue implements CrawlQueueAdapter {
     private queue;
+    private head;
+    private tail;
     private visited;
     push(item: CrawlQueueItem): Promise<void>;
     pushBatch(items: CrawlQueueItem[]): Promise<void>;

package/dist/browser/scrape/crawl-queue.js

@@ -1,14 +1,31 @@
 export class InMemoryCrawlQueue {
     queue = [];
+    head = 0;
+    tail = 0;
     visited = new Set();
     async push(item) {
-        this.queue.push(item);
+        this.queue[this.tail++] = item;
     }
     async pushBatch(items) {
-        this.queue.push(...items);
+        for (const item of items) {
+            this.queue[this.tail++] = item;
+        }
     }
     async pop() {
-        return this.queue.shift() ?? null;
+        while (this.head < this.tail) {
+            const item = this.queue[this.head];
+            this.queue[this.head] = undefined;
+            this.head++;
+            if (item) {
+                if (this.head > 1024 && this.head > this.tail / 2) {
+                    this.queue = this.queue.slice(this.head);
+                    this.tail -= this.head;
+                    this.head = 0;
+                }
+                return item;
+            }
+        }
+        return null;
     }
     async hasVisited(url) {
         return this.visited.has(url);
@@ -25,10 +42,12 @@ export class InMemoryCrawlQueue {
         this.visited.add(url);
     }
     async size() {
-        return this.queue.length;
+        return this.tail - this.head;
     }
     async clear() {
         this.queue = [];
+        this.head = 0;
+        this.tail = 0;
         this.visited.clear();
     }
     async close() {

package/dist/browser/scrape/spider.d.ts

@@ -192,6 +192,7 @@ export declare class Spider {
     private baseHost;
     private running;
     private aborted;
+    private abortController;
     private pendingCount;
     private domainRequestTimestamps;
     private contentHashes;

package/dist/browser/scrape/spider.js

@@ -40,10 +40,15 @@ const SEC_CH_PLATFORM_HINTS = [
 function getHostname(url) {
     return new URL(url).hostname;
 }
-function sleep(ms) {
+function sleep(ms, signal) {
     if (ms <= 0)
         return Promise.resolve();
-    return new Promise(resolve => setTimeout(resolve, ms));
+    if (signal?.aborted)
+        return Promise.resolve();
+    return new Promise(resolve => {
+        const timer = setTimeout(resolve, ms);
+        signal?.addEventListener('abort', () => { clearTimeout(timer); resolve(); }, { once: true });
+    });
 }
 function getRetryAfterDelay(response) {
     const retryAfter = response.headers.get('retry-after');
@@ -109,7 +114,8 @@ function shouldCrawl(url, baseHost, options) {
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
-        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
+        const hostname = parsed.hostname.replace(/^www\./, '');
+        if (options.sameDomain !== false && hostname !== baseHost) {
             return false;
         }
         const skipExtensions = [
@@ -189,6 +195,7 @@ export class Spider {
     baseHost = '';
     running = false;
     aborted = false;
+    abortController = new AbortController();
     pendingCount = 0;
     domainRequestTimestamps = new Map();
     contentHashes = new Map();
@@ -218,7 +225,9 @@ export class Spider {
         if (timestamps.length >= limit) {
             const waitMs = timestamps[0] + window - now;
             if (waitMs > 0)
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
+            if (this.aborted)
+                return;
             const afterWait = Date.now();
             while (timestamps.length > 0 && timestamps[0] <= afterWait - window) {
                 timestamps.shift();
@@ -315,7 +324,7 @@ export class Spider {
         const startTimestamp = Date.now();
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
-        this.baseHost = new URL(normalizedStart).hostname;
+        this.baseHost = new URL(normalizedStart).hostname.replace(/^www\./, '');
         if (!this.options.resume) {
             await this.crawlQueue.clear();
             await this.crawlStorage.clear();
@@ -331,14 +340,25 @@ export class Spider {
         }
         this.running = true;
         this.aborted = false;
+        this.abortController = new AbortController();
         this.pendingCount = 0;
         this.sitemapUrls = [];
         this.sitemapUrlSet.clear();
         this.robotsData = null;
         this.sitemapValidation = null;
         this.robotsValidation = null;
-        this.blockedDomains.clear();
-        this.domainStates.clear();
+        if (!this.options.resume) {
+            this.blockedDomains.clear();
+            this.domainStates.clear();
+        }
+        if (this.options.resume && this.options.deduplicateContent) {
+            const existingResults = await this.crawlStorage.getResults();
+            for (const r of existingResults) {
+                if (r.contentHash) {
+                    this.contentHashes.set(r.contentHash, r.url);
+                }
+            }
+        }
         if (this.options.transport !== 'undici') {
             this.curlAvailable = await hasImpersonate();
             if (this.options.transport === 'auto' && this.curlAvailable && isProtectedDomain(this.baseHost)) {
@@ -354,10 +374,12 @@ export class Spider {
         const pending = new Map();
         const scheduleUrl = async (item) => {
             const normalized = normalizeUrl(item.url);
-            if (await this.crawlQueue.hasVisited(normalized))
-                return;
             if (pending.has(normalized))
                 return;
+            if (await this.crawlQueue.hasVisited(normalized))
+                return;
+            await this.crawlQueue.markVisited(normalized);
+            this._visitedCount++;
             if (item.depth > this.options.maxDepth)
                 return;
             if (this._resultCount + pending.size >= this.options.maxPages)
@@ -373,8 +395,6 @@ export class Spider {
                     return;
                 }
             }
-            await this.crawlQueue.markVisited(normalized);
-            this._visitedCount++;
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
                 .finally(() => {
@@ -416,14 +436,33 @@ export class Spider {
         if (pending.size > 0) {
             await Promise.all(pending.values());
         }
+        while (!this.aborted && this._resultCount < this.options.maxPages) {
+            const remaining = await this.crawlQueue.size();
+            if (remaining === 0 && pending.size === 0)
+                break;
+            this._queueSize = remaining;
+            let nextItem = await this.crawlQueue.pop();
+            while (nextItem && !this.aborted) {
+                this._queueSize = Math.max(0, this._queueSize - 1);
+                if (this._resultCount + pending.size >= this.options.maxPages)
+                    break;
+                await scheduleUrl(nextItem);
+                nextItem = await this.crawlQueue.pop();
+            }
+            if (pending.size > 0) {
+                await Promise.all(pending.values());
+            }
+        }
         this.running = false;
         const pages = await this.crawlStorage.getResults();
         const errors = await this.crawlStorage.getErrors();
         const sitemapAnalysis = this.buildSitemapAnalysis(pages);
         const robotsAnalysis = this.buildRobotsAnalysis();
         const visited = this.crawlQueue instanceof InMemoryCrawlQueue
-            ? this.crawlQueue.getVisited()
+            ? new Set(this.crawlQueue.getVisited())
             : new Set(pages.map(r => r.url));
+        await this.crawlQueue.close?.();
+        await this.crawlStorage.close?.();
         return {
             startUrl: normalizedStart,
             pages,
@@ -615,6 +654,8 @@ export class Spider {
         let lastRetryAfterMs = 0;
         const proxyUrl = this.proxyAdapter ? await this.proxyAdapter.getProxy() : null;
         const executeRequest = async (useCurl) => {
+            if (this.aborted)
+                throw new Error('Crawl aborted');
             if (useCurl && this.curlTransport) {
                 const curlForRequest = proxyUrl
                     ? new CurlTransport(proxyUrl)
@@ -655,6 +696,7 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                signal: this.abortController.signal,
                 beforeRedirect: this.options.onRedirect
                     ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
                     : undefined,
@@ -792,7 +834,9 @@ export class Spider {
                         timings,
                     });
                 }
-                await sleep(waitMs);
+                if (this.aborted)
+                    break;
+                await sleep(waitMs, this.abortController.signal);
                 continue;
             }
             catch (error) {
@@ -823,8 +867,10 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.aborted)
+                    break;
                 const waitMs = this.getRetryWait(hostname, attempt + 1);
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
             }
         }
         if (lastResponse) {
@@ -1059,6 +1105,8 @@ export class Spider {
             }
         }
         catch (error) {
+            if (this.aborted)
+                return;
             const errAttempts = typeof error?.attempts === 'number' ? error.attempts : 1;
             const errRetryCount = typeof error?.retryCount === 'number' ? error.retryCount : Math.max(0, errAttempts - 1);
             const errRetryAfterMs = typeof error?.retryAfterMs === 'number' ? error?.retryAfterMs : 0;
@@ -1248,7 +1296,7 @@ export class Spider {
         const now = Date.now();
         const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
         if (delay > 0) {
-            await sleep(delay);
+            await sleep(delay, this.abortController.signal);
         }
     }
     registerDomainBlock(hostname) {
@@ -1319,6 +1367,7 @@ export class Spider {
     }
     abort() {
         this.aborted = true;
+        this.abortController.abort();
     }
     isRunning() {
         return this.running;

package/dist/browser/seo/seo-spider.js

@@ -56,7 +56,7 @@ export class SeoSpider {
         });
     }
     async analyzePageDuringCrawl(pageResult, html) {
-        if (pageResult.error || pageResult.status >= 400) {
+        if (pageResult.status >= 400 || !html) {
             const seoPage = { ...pageResult, seoReport: undefined };
             this.seoPages.push(seoPage);
             return;

package/dist/browser/utils/block-detector.js

@@ -517,6 +517,12 @@ export function detectBlock(response, body) {
             description: 'DataDome headers detected',
         });
     }
+    if (response.status === 200 &&
+        results.length === 0 &&
+        !location &&
+        !body) {
+        return { blocked: false, confidence: 0 };
+    }
     if (body) {
         const isLongBody = body.length > 100_000;
         const checkBody = isLongBody ? body.slice(0, 30_000) : body;
@@ -588,6 +594,12 @@ export function detectCaptcha(response, body) {
             addMatch(matches, 'cloudflare', 0.7, 'Cloudflare challenge headers detected');
         }
     }
+    if (matches.length === 0 &&
+        response.status >= 200 && response.status < 300 &&
+        !location &&
+        !body) {
+        return { detected: false, confidence: 0 };
+    }
     if (body) {
         const checkBody = body.length < 120_000 ? body : body.slice(0, 120_000);
         const isTinyBody = checkBody.length > 0 && checkBody.length < 12000;

package/dist/scrape/crawl-queue.d.ts

@@ -17,6 +17,8 @@ export interface CrawlQueueAdapter {
 }
 export declare class InMemoryCrawlQueue implements CrawlQueueAdapter {
     private queue;
+    private head;
+    private tail;
     private visited;
     push(item: CrawlQueueItem): Promise<void>;
     pushBatch(items: CrawlQueueItem[]): Promise<void>;

package/dist/scrape/crawl-queue.js

@@ -1,14 +1,31 @@
 export class InMemoryCrawlQueue {
     queue = [];
+    head = 0;
+    tail = 0;
     visited = new Set();
     async push(item) {
-        this.queue.push(item);
+        this.queue[this.tail++] = item;
     }
     async pushBatch(items) {
-        this.queue.push(...items);
+        for (const item of items) {
+            this.queue[this.tail++] = item;
+        }
     }
     async pop() {
-        return this.queue.shift() ?? null;
+        while (this.head < this.tail) {
+            const item = this.queue[this.head];
+            this.queue[this.head] = undefined;
+            this.head++;
+            if (item) {
+                if (this.head > 1024 && this.head > this.tail / 2) {
+                    this.queue = this.queue.slice(this.head);
+                    this.tail -= this.head;
+                    this.head = 0;
+                }
+                return item;
+            }
+        }
+        return null;
     }
     async hasVisited(url) {
         return this.visited.has(url);
@@ -25,10 +42,12 @@ export class InMemoryCrawlQueue {
         this.visited.add(url);
     }
     async size() {
-        return this.queue.length;
+        return this.tail - this.head;
     }
     async clear() {
         this.queue = [];
+        this.head = 0;
+        this.tail = 0;
         this.visited.clear();
     }
     async close() {

package/dist/scrape/spider.d.ts

@@ -192,6 +192,7 @@ export declare class Spider {
     private baseHost;
     private running;
     private aborted;
+    private abortController;
     private pendingCount;
     private domainRequestTimestamps;
     private contentHashes;

package/dist/scrape/spider.js

@@ -40,10 +40,15 @@ const SEC_CH_PLATFORM_HINTS = [
 function getHostname(url) {
     return new URL(url).hostname;
 }
-function sleep(ms) {
+function sleep(ms, signal) {
     if (ms <= 0)
         return Promise.resolve();
-    return new Promise(resolve => setTimeout(resolve, ms));
+    if (signal?.aborted)
+        return Promise.resolve();
+    return new Promise(resolve => {
+        const timer = setTimeout(resolve, ms);
+        signal?.addEventListener('abort', () => { clearTimeout(timer); resolve(); }, { once: true });
+    });
 }
 function getRetryAfterDelay(response) {
     const retryAfter = response.headers.get('retry-after');
@@ -109,7 +114,8 @@ function shouldCrawl(url, baseHost, options) {
         if (!['http:', 'https:'].includes(parsed.protocol)) {
             return false;
         }
-        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
+        const hostname = parsed.hostname.replace(/^www\./, '');
+        if (options.sameDomain !== false && hostname !== baseHost) {
             return false;
         }
         const skipExtensions = [
@@ -189,6 +195,7 @@ export class Spider {
     baseHost = '';
     running = false;
     aborted = false;
+    abortController = new AbortController();
     pendingCount = 0;
     domainRequestTimestamps = new Map();
     contentHashes = new Map();
@@ -218,7 +225,9 @@ export class Spider {
         if (timestamps.length >= limit) {
             const waitMs = timestamps[0] + window - now;
             if (waitMs > 0)
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
+            if (this.aborted)
+                return;
             const afterWait = Date.now();
             while (timestamps.length > 0 && timestamps[0] <= afterWait - window) {
                 timestamps.shift();
@@ -315,7 +324,7 @@ export class Spider {
         const startTimestamp = Date.now();
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
-        this.baseHost = new URL(normalizedStart).hostname;
+        this.baseHost = new URL(normalizedStart).hostname.replace(/^www\./, '');
         if (!this.options.resume) {
             await this.crawlQueue.clear();
             await this.crawlStorage.clear();
@@ -331,14 +340,25 @@ export class Spider {
         }
         this.running = true;
         this.aborted = false;
+        this.abortController = new AbortController();
         this.pendingCount = 0;
         this.sitemapUrls = [];
         this.sitemapUrlSet.clear();
         this.robotsData = null;
         this.sitemapValidation = null;
         this.robotsValidation = null;
-        this.blockedDomains.clear();
-        this.domainStates.clear();
+        if (!this.options.resume) {
+            this.blockedDomains.clear();
+            this.domainStates.clear();
+        }
+        if (this.options.resume && this.options.deduplicateContent) {
+            const existingResults = await this.crawlStorage.getResults();
+            for (const r of existingResults) {
+                if (r.contentHash) {
+                    this.contentHashes.set(r.contentHash, r.url);
+                }
+            }
+        }
         if (this.options.transport !== 'undici') {
             this.curlAvailable = await hasImpersonate();
             if (this.options.transport === 'auto' && this.curlAvailable && isProtectedDomain(this.baseHost)) {
@@ -354,10 +374,12 @@ export class Spider {
         const pending = new Map();
         const scheduleUrl = async (item) => {
             const normalized = normalizeUrl(item.url);
-            if (await this.crawlQueue.hasVisited(normalized))
-                return;
             if (pending.has(normalized))
                 return;
+            if (await this.crawlQueue.hasVisited(normalized))
+                return;
+            await this.crawlQueue.markVisited(normalized);
+            this._visitedCount++;
             if (item.depth > this.options.maxDepth)
                 return;
             if (this._resultCount + pending.size >= this.options.maxPages)
@@ -373,8 +395,6 @@ export class Spider {
                     return;
                 }
             }
-            await this.crawlQueue.markVisited(normalized);
-            this._visitedCount++;
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
                 .finally(() => {
@@ -416,14 +436,33 @@ export class Spider {
         if (pending.size > 0) {
             await Promise.all(pending.values());
         }
+        while (!this.aborted && this._resultCount < this.options.maxPages) {
+            const remaining = await this.crawlQueue.size();
+            if (remaining === 0 && pending.size === 0)
+                break;
+            this._queueSize = remaining;
+            let nextItem = await this.crawlQueue.pop();
+            while (nextItem && !this.aborted) {
+                this._queueSize = Math.max(0, this._queueSize - 1);
+                if (this._resultCount + pending.size >= this.options.maxPages)
+                    break;
+                await scheduleUrl(nextItem);
+                nextItem = await this.crawlQueue.pop();
+            }
+            if (pending.size > 0) {
+                await Promise.all(pending.values());
+            }
+        }
         this.running = false;
         const pages = await this.crawlStorage.getResults();
         const errors = await this.crawlStorage.getErrors();
         const sitemapAnalysis = this.buildSitemapAnalysis(pages);
         const robotsAnalysis = this.buildRobotsAnalysis();
         const visited = this.crawlQueue instanceof InMemoryCrawlQueue
-            ? this.crawlQueue.getVisited()
+            ? new Set(this.crawlQueue.getVisited())
             : new Set(pages.map(r => r.url));
+        await this.crawlQueue.close?.();
+        await this.crawlStorage.close?.();
         return {
             startUrl: normalizedStart,
             pages,
@@ -615,6 +654,8 @@ export class Spider {
         let lastRetryAfterMs = 0;
         const proxyUrl = this.proxyAdapter ? await this.proxyAdapter.getProxy() : null;
         const executeRequest = async (useCurl) => {
+            if (this.aborted)
+                throw new Error('Crawl aborted');
             if (useCurl && this.curlTransport) {
                 const curlForRequest = proxyUrl
                     ? new CurlTransport(proxyUrl)
@@ -655,6 +696,7 @@ export class Spider {
             const clientForRequest = this.getClientForProxy(proxyUrl);
             const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
+                signal: this.abortController.signal,
                 beforeRedirect: this.options.onRedirect
                     ? (info) => { this.options.onRedirect({ from: info.from, to: info.to, status: info.status }); }
                     : undefined,
@@ -792,7 +834,9 @@ export class Spider {
                         timings,
                     });
                 }
-                await sleep(waitMs);
+                if (this.aborted)
+                    break;
+                await sleep(waitMs, this.abortController.signal);
                 continue;
             }
             catch (error) {
@@ -823,8 +867,10 @@ export class Spider {
                         forcedTransport = 'curl';
                     }
                 }
+                if (this.aborted)
+                    break;
                 const waitMs = this.getRetryWait(hostname, attempt + 1);
-                await sleep(waitMs);
+                await sleep(waitMs, this.abortController.signal);
             }
         }
         if (lastResponse) {
@@ -1059,6 +1105,8 @@ export class Spider {
             }
         }
         catch (error) {
+            if (this.aborted)
+                return;
             const errAttempts = typeof error?.attempts === 'number' ? error.attempts : 1;
             const errRetryCount = typeof error?.retryCount === 'number' ? error.retryCount : Math.max(0, errAttempts - 1);
             const errRetryAfterMs = typeof error?.retryAfterMs === 'number' ? error?.retryAfterMs : 0;
@@ -1248,7 +1296,7 @@ export class Spider {
         const now = Date.now();
         const delay = Math.max(state.penaltyUntil, state.challengeCooldownUntil) - now;
         if (delay > 0) {
-            await sleep(delay);
+            await sleep(delay, this.abortController.signal);
         }
     }
     registerDomainBlock(hostname) {
@@ -1319,6 +1367,7 @@ export class Spider {
     }
     abort() {
         this.aborted = true;
+        this.abortController.abort();
     }
     isRunning() {
         return this.running;

package/dist/seo/seo-spider.js

@@ -56,7 +56,7 @@ export class SeoSpider {
         });
     }
     async analyzePageDuringCrawl(pageResult, html) {
-        if (pageResult.error || pageResult.status >= 400) {
+        if (pageResult.status >= 400 || !html) {
             const seoPage = { ...pageResult, seoReport: undefined };
             this.seoPages.push(seoPage);
             return;

package/dist/utils/block-detector.js

@@ -517,6 +517,12 @@ export function detectBlock(response, body) {
             description: 'DataDome headers detected',
         });
     }
+    if (response.status === 200 &&
+        results.length === 0 &&
+        !location &&
+        !body) {
+        return { blocked: false, confidence: 0 };
+    }
     if (body) {
         const isLongBody = body.length > 100_000;
         const checkBody = isLongBody ? body.slice(0, 30_000) : body;
@@ -588,6 +594,12 @@ export function detectCaptcha(response, body) {
             addMatch(matches, 'cloudflare', 0.7, 'Cloudflare challenge headers detected');
         }
     }
+    if (matches.length === 0 &&
+        response.status >= 200 && response.status < 300 &&
+        !location &&
+        !body) {
+        return { detected: false, confidence: 0 };
+    }
     if (body) {
         const checkBody = body.length < 120_000 ? body : body.slice(0, 120_000);
         const isTinyBody = checkBody.length > 0 && checkBody.length < 12000;

package/dist/version.js

@@ -1,4 +1,4 @@
-const VERSION = '1.0.94';
+const VERSION = '1.0.95';
 let _version = null;
 export async function getVersion() {
     if (_version)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.94",
+  "version": "1.0.95",
   "description": "Multi-Protocol SDK for the AI Era - HTTP, WebSocket, DNS, FTP, SFTP, Telnet, HLS unified with AI providers and MCP tools",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",