recker 1.0.85 → 1.0.86-next.a24fa13

package/dist/browser/scrape/spider.js

@@ -9,6 +9,9 @@ import { hasImpersonate } from '../utils/binary-manager.js';
 import { CurlTransport } from '../transport/curl.js';
 import { HttpRequest } from '../core/request.js';
 import { getRandomUserAgent } from '../utils/user-agent.js';
+import { InMemoryCrawlQueue } from './crawl-queue.js';
+import { InMemoryCrawlStorage } from './crawl-storage.js';
+import { ListProxyAdapter } from './proxy-adapter.js';
 const FALLBACK_ACCEPT_LANGUAGES = [
     'en-US,en;q=0.9',
     'en-GB,en;q=0.9',
@@ -79,6 +82,9 @@ function normalizeUrl(urlStr) {
     try {
         const url = new URL(urlStr);
         url.hash = '';
+        if (url.hostname.startsWith('www.')) {
+            url.hostname = url.hostname.slice(4);
+        }
         const paramsToDelete = [];
         url.searchParams.forEach((_, key) => {
             if (TRACKING_PARAMS.has(key.toLowerCase())) {
@@ -172,10 +178,13 @@ export class Spider {
     options;
     client;
     pool;
-    visited = new Set();
-    queue = [];
-    results = [];
-    errors = [];
+    crawlQueue;
+    crawlStorage;
+    proxyAdapter = null;
+    proxyClients = new Map();
+    _visitedCount = 0;
+    _queueSize = 0;
+    _resultCount = 0;
     baseHost = '';
     running = false;
     aborted = false;
@@ -236,6 +245,17 @@ export class Spider {
             extract: extractSchema,
             parserOptions: options.parserOptions,
         };
+        if (options.proxy) {
+            if (typeof options.proxy === 'string') {
+                this.proxyAdapter = new ListProxyAdapter([options.proxy]);
+            }
+            else if (Array.isArray(options.proxy)) {
+                this.proxyAdapter = new ListProxyAdapter(options.proxy);
+            }
+            else {
+                this.proxyAdapter = options.proxy;
+            }
+        }
         this.client = createClient({
             baseUrl: 'http://localhost',
             timeout: this.options.timeout,
@@ -253,6 +273,8 @@ export class Spider {
                 interval: this.options.delay,
             } : {}),
         });
+        this.crawlQueue = options.crawlQueue ?? new InMemoryCrawlQueue();
+        this.crawlStorage = options.crawlStorage ?? new InMemoryCrawlStorage();
     }
     async crawl(startUrl) {
         const perfStart = performance.now();
@@ -260,10 +282,11 @@ export class Spider {
         const normalizedStart = normalizeUrl(startUrl);
         const baseUrl = new URL(normalizedStart).origin;
         this.baseHost = new URL(normalizedStart).hostname;
-        this.visited.clear();
-        this.queue = [];
-        this.results = [];
-        this.errors = [];
+        await this.crawlQueue.clear();
+        await this.crawlStorage.clear();
+        this._visitedCount = 0;
+        this._queueSize = 0;
+        this._resultCount = 0;
         this.running = true;
         this.aborted = false;
         this.pendingCount = 0;
@@ -287,15 +310,15 @@ export class Spider {
             await this.fetchSitemaps(baseUrl);
         }
         const pending = new Map();
-        const scheduleUrl = (item) => {
+        const scheduleUrl = async (item) => {
             const normalized = normalizeUrl(item.url);
-            if (this.visited.has(normalized))
+            if (await this.crawlQueue.hasVisited(normalized))
                 return;
             if (pending.has(normalized))
                 return;
             if (item.depth > this.options.maxDepth)
                 return;
-            if (this.results.length + pending.size >= this.options.maxPages)
+            if (this._resultCount + pending.size >= this.options.maxPages)
                 return;
             if (this.options.respectRobotsTxt && this.robotsData) {
                 try {
@@ -308,7 +331,8 @@ export class Spider {
                     return;
                 }
             }
-            this.visited.add(normalized);
+            await this.crawlQueue.markVisited(normalized);
+            this._visitedCount++;
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
                 .finally(() => {
@@ -317,27 +341,29 @@ export class Spider {
             });
             pending.set(normalized, promise);
         };
-        scheduleUrl({ url: normalizedStart, depth: 0 });
+        await scheduleUrl({ url: normalizedStart, depth: 0 });
         if (this.options.useSitemap && this.sitemapUrls.length > 0) {
             for (const sitemapUrl of this.sitemapUrls) {
                 try {
                     const urlHost = new URL(sitemapUrl.loc).hostname;
                     if (urlHost === this.baseHost) {
-                        scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
+                        await scheduleUrl({ url: sitemapUrl.loc, depth: 1 });
                     }
                 }
                 catch {
                 }
             }
         }
-        while ((pending.size > 0 || this.queue.length > 0)
+        while ((pending.size > 0 || this._queueSize > 0)
             && !this.aborted
-            && this.results.length < this.options.maxPages) {
-            while (this.queue.length > 0 && !this.aborted) {
-                const item = this.queue.shift();
-                if (this.results.length + pending.size >= this.options.maxPages)
+            && this._resultCount < this.options.maxPages) {
+            let nextItem = await this.crawlQueue.pop();
+            while (nextItem && !this.aborted) {
+                this._queueSize = Math.max(0, this._queueSize - 1);
+                if (this._resultCount + pending.size >= this.options.maxPages)
                     break;
-                scheduleUrl(item);
+                await scheduleUrl(nextItem);
+                nextItem = await this.crawlQueue.pop();
             }
             if (pending.size > 0) {
                 await Promise.race(pending.values());
@@ -347,16 +373,21 @@ export class Spider {
             await Promise.all(pending.values());
         }
         this.running = false;
-        const sitemapAnalysis = this.buildSitemapAnalysis();
+        const pages = await this.crawlStorage.getResults();
+        const errors = await this.crawlStorage.getErrors();
+        const sitemapAnalysis = this.buildSitemapAnalysis(pages);
         const robotsAnalysis = this.buildRobotsAnalysis();
+        const visited = this.crawlQueue instanceof InMemoryCrawlQueue
+            ? this.crawlQueue.getVisited()
+            : new Set(pages.map(r => r.url));
         return {
             startUrl: normalizedStart,
-            pages: this.results,
-            visited: this.visited,
+            pages,
+            visited,
             duration: Math.round(performance.now() - perfStart),
             startTime: startTimestamp,
             endTime: Date.now(),
-            errors: this.errors,
+            errors,
             sitemap: this.options.useSitemap ? sitemapAnalysis : undefined,
             robots: robotsAnalysis,
         };
@@ -444,8 +475,8 @@ export class Spider {
         catch (error) {
         }
     }
-    buildSitemapAnalysis() {
-        const crawledUrls = new Set(this.results.map(r => normalizeUrl(r.url)));
+    buildSitemapAnalysis(results) {
+        const crawledUrls = new Set(results.map(r => normalizeUrl(r.url)));
         const sitemapUrlSet = this.sitemapUrlSet.size > 0
             ? this.sitemapUrlSet
             : new Set(this.sitemapUrls.map((u) => normalizeUrl(u.loc)));
@@ -467,7 +498,7 @@ export class Spider {
                 }
             }
         }
-        for (const page of this.results) {
+        for (const page of results) {
             for (const link of page.links) {
                 if (link.href) {
                     linkedUrls.add(normalizeUrl(link.href));
@@ -508,6 +539,21 @@ export class Spider {
             issues: this.robotsValidation?.issues ?? [],
         };
     }
+    getClientForProxy(proxyUrl) {
+        if (!proxyUrl)
+            return this.client;
+        let proxied = this.proxyClients.get(proxyUrl);
+        if (!proxied) {
+            proxied = createClient({
+                baseUrl: 'http://localhost',
+                timeout: this.options.timeout,
+                headers: { 'User-Agent': this.options.userAgent },
+                proxy: proxyUrl,
+            });
+            this.proxyClients.set(proxyUrl, proxied);
+        }
+        return proxied;
+    }
     async fetchPage(url) {
         const hostname = getHostname(url);
         const maxAttempts = Math.max(1, this.options.maxRetryAttempts);
@@ -523,14 +569,18 @@ export class Spider {
         let forcedTransport = null;
         let attemptLog = [];
         let lastRetryAfterMs = 0;
+        const proxyUrl = this.proxyAdapter ? await this.proxyAdapter.getProxy() : null;
         const executeRequest = async (useCurl) => {
             if (useCurl && this.curlTransport) {
+                const curlForRequest = proxyUrl
+                    ? new CurlTransport(proxyUrl)
+                    : this.curlTransport;
                 const req = new HttpRequest(url, {
                     method: 'GET',
                     headers: this.buildRequestHeaders(url, true),
                 });
                 const requestStart = performance.now();
-                const response = await this.curlTransport.dispatch(req);
+                const response = await curlForRequest.dispatch(req);
                 const contentType = response.headers.get('content-type') || '';
                 const shouldReadCurlResponseBody = shouldReadResponseBody(response.status, contentType);
                 let body = '';
@@ -558,7 +608,8 @@ export class Spider {
                     timings,
                 };
             }
-            const response = await this.client.get(url, {
+            const clientForRequest = this.getClientForProxy(proxyUrl);
+            const response = await clientForRequest.get(url, {
                 headers: this.buildRequestHeaders(url, false),
             });
             const contentType = response.headers.get('content-type') || '';
@@ -649,6 +700,9 @@ export class Spider {
                     this.registerDomainSuccess(hostname);
                 }
                 if (!shouldRetry || attempt === maxAttempts - 1) {
+                    if (proxyUrl && this.proxyAdapter?.reportResult) {
+                        await this.proxyAdapter.reportResult(proxyUrl, isHighQualitySuccess);
+                    }
                     return {
                         response,
                         body,
@@ -752,6 +806,9 @@ export class Spider {
                     lastStatus: lastResponse.status,
                 };
             }
+            if (proxyUrl && this.proxyAdapter?.reportResult) {
+                await this.proxyAdapter.reportResult(proxyUrl, false);
+            }
             throw wrapped;
         }
         throw new Error(`Failed to fetch ${url}`);
@@ -760,10 +817,10 @@ export class Spider {
         const startTime = performance.now();
         const fetchedAt = Date.now();
         this.options.onProgress?.({
-            crawled: this.results.length,
-            queued: this.queue.length,
+            crawled: this._resultCount,
+            queued: this._queueSize,
             pending: this.pendingCount,
-            total: this.visited.size,
+            total: this._visitedCount,
             currentUrl: item.url,
             depth: item.depth,
         });
@@ -808,7 +865,8 @@ export class Spider {
                     security,
                     fetchedAt,
                 };
-                this.results.push(nonHtmlResult);
+                await this.crawlStorage.saveResult(nonHtmlResult);
+                this._resultCount++;
                 this.options.onPage?.(nonHtmlResult);
                 return;
             }
@@ -869,23 +927,38 @@ export class Spider {
                 fetchedAt,
                 extracted,
             };
-            this.results.push(result);
+            await this.crawlStorage.saveResult(result);
+            this._resultCount++;
             this.options.onPage?.(result);
             if (this.options.onPageWithHtml) {
                 await this.options.onPageWithHtml(result, html);
             }
+            const candidates = [];
+            const candidateUrls = [];
             for (const link of links) {
                 if (!link.href)
                     continue;
                 const normalized = normalizeUrl(link.href);
-                if (this.visited.has(normalized))
-                    continue;
                 if (!shouldCrawl(normalized, this.baseHost, this.options))
                     continue;
-                this.queue.push({
-                    url: normalized,
-                    depth: item.depth + 1,
-                });
+                candidateUrls.push(normalized);
+                candidates.push({ url: normalized, depth: item.depth + 1 });
+            }
+            if (candidates.length > 0) {
+                const visitedSet = this.crawlQueue.hasVisitedBatch
+                    ? await this.crawlQueue.hasVisitedBatch(candidateUrls)
+                    : new Set(await Promise.all(candidateUrls.map(async (u) => (await this.crawlQueue.hasVisited(u)) ? u : null)).then(r => r.filter(Boolean)));
+                const newItems = candidates.filter((_, i) => !visitedSet.has(candidateUrls[i]));
+                if (newItems.length > 0) {
+                    if (this.crawlQueue.pushBatch) {
+                        await this.crawlQueue.pushBatch(newItems);
+                    }
+                    else {
+                        for (const newItem of newItems)
+                            await this.crawlQueue.push(newItem);
+                    }
+                    this._queueSize += newItems.length;
+                }
             }
         }
         catch (error) {
@@ -950,8 +1023,9 @@ export class Spider {
                 security,
                 fetchedAt,
             };
-            this.results.push(errorResult);
-            this.errors.push({ url: item.url, error: message });
+            await this.crawlStorage.saveResult(errorResult);
+            this._resultCount++;
+            await this.crawlStorage.saveError({ url: item.url, error: message });
             this.options.onPage?.(errorResult);
         }
     }
@@ -1149,10 +1223,10 @@ export class Spider {
     }
     getProgress() {
         return {
-            crawled: this.results.length,
-            queued: this.queue.length,
+            crawled: this._resultCount,
+            queued: this._queueSize,
             pending: this.pendingCount,
-            total: this.visited.size,
+            total: this._visitedCount,
             currentUrl: '',
             depth: 0,
         };

package/dist/browser/transport/curl.js

@@ -41,6 +41,9 @@ function parseCurlTimings(payload) {
             case 'total':
                 values.total = parseCurlTimingValue(value);
                 break;
+            case 'pretransfer':
+                values.pretransfer = parseCurlTimingValue(value);
+                break;
             default:
                 break;
         }
@@ -143,7 +146,7 @@ export class CurlTransport {
                 '--compressed',
                 '--no-keepalive',
                 '--write-out',
-                `\\n${TIMING_MARKER} dns=%{time_namelookup} tcp=%{time_connect} tls=%{time_appconnect} ttfb=%{time_starttransfer} total=%{time_total}`,
+                `\\n${TIMING_MARKER} dns=%{time_namelookup} tcp=%{time_connect} tls=%{time_appconnect} pretransfer=%{time_pretransfer} ttfb=%{time_starttransfer} total=%{time_total}`,
             ];
             const proxy = this.getNextProxy();
             if (proxy) {
@@ -240,14 +243,55 @@ export class CurlTransport {
                     statusText,
                     headers,
                 });
-                const responseTimings = timings && Object.keys(timings).length > 0 ? {
-                    dns: timings.dns,
-                    tcp: timings.tcp,
-                    tls: timings.tls ? (timings.ttfb !== undefined ? timings.tls : timings.tls) : undefined,
-                    firstByte: timings.ttfb,
-                    content: timings.download,
-                    total: timings.total,
-                } : {};
+                const usedProxy = this.proxyList.length > 0;
+                let responseTimings = {};
+                if (timings && Object.keys(timings).length > 0) {
+                    const dns = timings.dns;
+                    const tcp = timings.tcp !== undefined && timings.dns !== undefined
+                        ? timings.tcp - timings.dns : undefined;
+                    const hasTls = timings.tls !== undefined && timings.tls > 0;
+                    const tls = hasTls && timings.tcp !== undefined
+                        ? timings.tls - timings.tcp : undefined;
+                    const content = timings.download;
+                    if (usedProxy && timings.pretransfer !== undefined) {
+                        const afterProxy = hasTls ? timings.tls : timings.tcp;
+                        const targetTls = afterProxy !== undefined
+                            ? timings.pretransfer - afterProxy : undefined;
+                        const proxyTotal = (dns ?? 0) + (tcp ?? 0) + (tls ?? 0);
+                        const connectionTime = proxyTotal + (targetTls ?? 0);
+                        const serverTime = timings.ttfb !== undefined
+                            ? timings.ttfb - connectionTime : undefined;
+                        responseTimings = {
+                            tls: targetTls,
+                            content,
+                            transferTime: content,
+                            proxyDns: dns,
+                            proxyTcp: tcp,
+                            proxyTls: tls,
+                            proxyTotal,
+                            firstByte: timings.ttfb,
+                            total: timings.total,
+                            connectionTime,
+                            serverTime,
+                        };
+                    }
+                    else {
+                        const connectionTime = (dns ?? 0) + (tcp ?? 0) + (tls ?? 0);
+                        const serverTime = timings.ttfb !== undefined
+                            ? timings.ttfb - connectionTime : undefined;
+                        responseTimings = {
+                            dns,
+                            tcp,
+                            tls,
+                            content,
+                            transferTime: content,
+                            firstByte: timings.ttfb,
+                            total: timings.total,
+                            connectionTime,
+                            serverTime,
+                        };
+                    }
+                }
                 resolve(new HttpResponse(nativeResponse, {
                     timings: responseTimings,
                     connection: { protocol: 'curl' },

package/dist/browser/transport/undici.js

@@ -6,6 +6,7 @@ import { AsyncLocalStorage } from 'async_hooks';
 import { channel } from 'node:diagnostics_channel';
 import { createLookupFunction } from '../utils/dns.js';
 import { AgentManager } from '../utils/agent-manager.js';
+import { getProxyForUrl } from '../utils/env-proxy.js';
 import { createProgressStream } from '../utils/progress.js';
 import { nodeToWebStream } from '../utils/streaming.js';
 import { ProtocolCache, getGlobalProtocolCache, normalizeProtocol } from '../utils/protocol-cache.js';
@@ -318,6 +319,9 @@ export class UndiciTransport {
                 return slot.agent;
             }
         }
+        const envProxy = getProxyForUrl(url);
+        if (envProxy)
+            return new ProxyAgent(envProxy);
         if (this.agentManager)
             return this.agentManager.getAgentForUrl(url);
         if (this.dnsAgent)

package/dist/browser/types/index.d.ts

@@ -54,6 +54,9 @@ export interface RequestOptions {
     followRedirects?: boolean;
     http2?: boolean;
     useCurl?: boolean;
+    queue?: boolean | {
+        metadata?: Record<string, unknown>;
+    };
 }
 export interface ReckerRequest {
     url: string;
@@ -77,6 +80,9 @@ export interface ReckerRequest {
     http2?: boolean;
     useCurl?: boolean;
     userAgent?: string;
+    queue?: boolean | {
+        metadata?: Record<string, unknown>;
+    };
     withHeader(name: string, value: string): ReckerRequest;
     _hooks?: {
         onDnsLookup?: (info: any) => void;
@@ -101,11 +107,18 @@ export interface ReckerRequest {
 export interface Timings {
     queuing?: number;
     dns?: number;
-    tls?: number;
     tcp?: number;
-    firstByte?: number;
+    tls?: number;
     content?: number;
+    firstByte?: number;
     total?: number;
+    connectionTime?: number;
+    serverTime?: number;
+    transferTime?: number;
+    proxyDns?: number;
+    proxyTcp?: number;
+    proxyTls?: number;
+    proxyTotal?: number;
 }
 export interface ConcurrencyConfig {
     max?: number;
@@ -213,6 +226,44 @@ export interface CacheStorage {
     set(key: string, value: CacheEntry, ttl: number): Promise<void>;
     delete(key: string): Promise<void>;
 }
+export interface QueueJob {
+    jobId: string;
+    url: string;
+    method: Method;
+    headers: Record<string, string>;
+    body: string | null;
+    bodyContentType?: string;
+    correlationId?: string;
+    traceId?: string;
+    tenant?: string;
+    policyTags?: string[];
+    createdAt: number;
+    metadata?: Record<string, unknown>;
+}
+export type QueueJobStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'unknown';
+export interface QueueJobStatusResult {
+    jobId: string;
+    status: QueueJobStatus;
+    result?: unknown;
+    error?: string;
+}
+export interface QueueAdapter {
+    enqueue(job: QueueJob): Promise<string>;
+    getStatus?(jobId: string): Promise<QueueJobStatusResult>;
+    close?(): Promise<void>;
+}
+export type QueueFilter = (req: ReckerRequest) => boolean;
+export interface QueueFilterConfig {
+    methods?: Method[];
+    urlPatterns?: (string | RegExp)[];
+    headerPresent?: string;
+}
+export interface QueueOptions {
+    adapter: QueueAdapter;
+    filter?: QueueFilter | QueueFilterConfig;
+    jobIdGenerator?: (req: ReckerRequest) => string;
+    defaultMetadata?: Record<string, unknown>;
+}
 export type CacheStrategy = 'cache-first' | 'network-only' | 'network-first' | 'stale-while-revalidate' | 'revalidate' | 'rfc-compliant';
 export interface PluginClient {
     beforeRequest(fn: (req: ReckerRequest) => ReckerRequest | void | Promise<ReckerRequest | void>): void;

package/dist/core/client.d.ts

@@ -1,4 +1,5 @@
 import { ClientOptions, Middleware, ReckerRequest, ReckerResponse, RequestOptions, CacheStorage, PageResult } from '../types/index.js';
+import type { QueueOptions } from '../types/index.js';
 import type { ClientAI, ClientOptionsWithAI } from '../types/ai-client.js';
 import { RequestPromise } from './request-promise.js';
 import { type RuntimeEventName, type RuntimeEventPayloads, type TypedEventBus } from '../core-runtime/typed-events.js';
@@ -27,6 +28,7 @@ export interface ExtendedClientOptions extends ClientOptions {
     retry?: RetryOptions;
     cache?: ClientCacheConfig;
     dedup?: DedupOptions;
+    queue?: QueueOptions;
 }
 export declare class Client {
     static get version(): string;

package/dist/core/client.js

@@ -17,6 +17,7 @@ import { paginate, streamPages } from '../plugins/pagination.js';
 import { retryPlugin } from '../plugins/retry.js';
 import { cachePlugin } from '../plugins/cache.js';
 import { dedupPlugin } from '../plugins/dedup.js';
+import { queuePlugin } from '../plugins/queue.js';
 import { createXSRFMiddleware } from '../plugins/xsrf.js';
 import { createCompressionMiddleware } from '../plugins/compression.js';
 import { serializeXML } from '../plugins/xml.js';
@@ -389,6 +390,13 @@ export class Client {
                 scope: 'request'
             });
         }
+        if (options.queue) {
+            registerPlugin(queuePlugin(options.queue), {
+                name: 'recker:queue',
+                priority: 135,
+                scope: 'request'
+            });
+        }
         if (options.plugins) {
             options.plugins.forEach((plugin, index) => {
                 const existingManifest = getPluginManifest(plugin);

package/dist/core/request.d.ts

@@ -20,6 +20,9 @@ export declare class HttpRequest implements ReckerRequest {
     readonly policyTags: string[];
     readonly policySource?: string;
     readonly traceId?: string;
+    readonly queue?: boolean | {
+        metadata?: Record<string, unknown>;
+    };
     constructor(url: string, options?: RequestOptions);
     withHeader(name: string, value: string): ReckerRequest;
     withBody(body: BodyInit): ReckerRequest;

package/dist/core/request.js

@@ -28,6 +28,7 @@ export class HttpRequest {
     policyTags;
     policySource;
     traceId;
+    queue;
     constructor(url, options = {}) {
         this.url = url;
         this.method = options.method || 'GET';
@@ -51,6 +52,7 @@ export class HttpRequest {
         this.policyTags = options.policyTags ?? [];
         this.policySource = options.policySource;
         this.traceId = options.traceId;
+        this.queue = options.queue;
     }
     withHeader(name, value) {
         const context = getRequestContext(this);
@@ -75,7 +77,8 @@ export class HttpRequest {
             tenant: this.tenant,
             policyTags: this.policyTags,
             policySource: this.policySource,
-            traceId: this.traceId
+            traceId: this.traceId,
+            queue: this.queue,
         });
         if (context) {
             return attachRequestContext(request, context);
@@ -103,7 +106,8 @@ export class HttpRequest {
             tenant: this.tenant,
             policyTags: this.policyTags,
             policySource: this.policySource,
-            traceId: this.traceId
+            traceId: this.traceId,
+            queue: this.queue,
         });
         if (context) {
             return attachRequestContext(request, context);

package/dist/index.d.ts

@@ -60,6 +60,8 @@ export * from './plugins/odata.js';
 export * from './plugins/http2-push.js';
 export * from './plugins/http3.js';
 export * from './cache/redis-storage.js';
+export * from './plugins/queue.js';
+export * from './queue/consumer.js';
 export * from './extractors/index.js';
 export * from './video/index.js';
 export * from './events/request-events.js';

package/dist/index.js

@@ -60,6 +60,8 @@ export * from './plugins/odata.js';
 export * from './plugins/http2-push.js';
 export * from './plugins/http3.js';
 export * from './cache/redis-storage.js';
+export * from './plugins/queue.js';
+export * from './queue/consumer.js';
 export * from './extractors/index.js';
 export * from './video/index.js';
 export * from './events/request-events.js';