recker 1.0.86 → 1.0.87

package/dist/plugins/queue.d.ts

@@ -0,0 +1,41 @@
+import { Plugin, ReckerResponse, QueueAdapter, QueueJob, QueueOptions } from '../types/index.js';
+declare class QueuedResponse<T = unknown> implements ReckerResponse<T> {
+    readonly jobId: string;
+    readonly raw: Response;
+    readonly timings: undefined;
+    readonly connection: undefined;
+    constructor(jobId: string);
+    get status(): number;
+    get statusText(): string;
+    get headers(): Headers;
+    get ok(): boolean;
+    get url(): string;
+    json<R = T>(): Promise<R>;
+    text(): Promise<string>;
+    cleanText(): Promise<string>;
+    blob(): Promise<Blob>;
+    read(): ReadableStream<Uint8Array<ArrayBuffer>> | null;
+    clone(): ReckerResponse<T>;
+    sse(): AsyncGenerator<never, void, unknown>;
+    download(): AsyncGenerator<never, void, unknown>;
+    [Symbol.asyncIterator](): AsyncGenerator<Uint8Array<ArrayBuffer>, void, unknown>;
+}
+export declare function queuePlugin(options: QueueOptions): Plugin;
+export declare class InMemoryQueueAdapter implements QueueAdapter {
+    private jobs;
+    enqueue(job: QueueJob): Promise<string>;
+    getStatus(jobId: string): Promise<{
+        jobId: string;
+        status: "unknown";
+    } | {
+        jobId: string;
+        status: "pending" | "processing" | "completed" | "failed";
+    }>;
+    getPendingJobs(): QueueJob[];
+    setJobStatus(jobId: string, status: string): void;
+    getJob(jobId: string): QueueJob | undefined;
+    clear(): void;
+    get size(): number;
+    close(): Promise<void>;
+}
+export { QueuedResponse };

package/dist/plugins/queue.js

@@ -0,0 +1,184 @@
+let jobSeq = 0;
+function generateJobId() {
+    jobSeq += 1;
+    return `qj-${Date.now().toString(36)}-${jobSeq.toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
+}
+function serializeRequest(req, jobId, metadata) {
+    const headers = {};
+    req.headers.forEach((value, key) => {
+        headers[key] = value;
+    });
+    let body = null;
+    const bodyContentType = headers['content-type'];
+    if (req.body !== null && req.body !== undefined) {
+        if (typeof req.body === 'string') {
+            body = req.body;
+        }
+        else if (typeof req.body === 'object') {
+            try {
+                body = JSON.stringify(req.body);
+            }
+            catch {
+                body = String(req.body);
+            }
+        }
+        else {
+            body = String(req.body);
+        }
+    }
+    return {
+        jobId,
+        url: req.url,
+        method: req.method,
+        headers,
+        body,
+        bodyContentType: bodyContentType || undefined,
+        correlationId: req.correlationId,
+        traceId: req.traceId,
+        tenant: req.tenant,
+        policyTags: req.policyTags?.length ? req.policyTags : undefined,
+        createdAt: Date.now(),
+        metadata: metadata && Object.keys(metadata).length > 0 ? metadata : undefined,
+    };
+}
+function compileFilter(config) {
+    return (req) => {
+        if (config.methods && !config.methods.includes(req.method)) {
+            return false;
+        }
+        if (config.urlPatterns) {
+            const matched = config.urlPatterns.some((pattern) => {
+                if (typeof pattern === 'string')
+                    return req.url.includes(pattern);
+                return pattern.test(req.url);
+            });
+            if (!matched)
+                return false;
+        }
+        if (config.headerPresent && !req.headers.has(config.headerPresent)) {
+            return false;
+        }
+        return true;
+    };
+}
+class QueuedResponse {
+    jobId;
+    raw;
+    timings = undefined;
+    connection = undefined;
+    constructor(jobId) {
+        this.jobId = jobId;
+        const responseBody = JSON.stringify({
+            queued: true,
+            jobId,
+            status: 'queued',
+        });
+        this.raw = new Response(responseBody, {
+            status: 202,
+            statusText: 'Accepted',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-Queue-Job-Id': jobId,
+                'X-Queue-Status': 'queued',
+            },
+        });
+    }
+    get status() { return 202; }
+    get statusText() { return 'Accepted'; }
+    get headers() { return this.raw.headers; }
+    get ok() { return true; }
+    get url() { return ''; }
+    json() { return this.raw.json(); }
+    text() { return this.raw.text(); }
+    async cleanText() { return this.raw.text(); }
+    blob() { return this.raw.blob(); }
+    read() { return this.raw.body; }
+    clone() {
+        return new QueuedResponse(this.jobId);
+    }
+    async *sse() { }
+    async *download() { }
+    async *[Symbol.asyncIterator]() {
+        if (!this.raw.body)
+            return;
+        const reader = this.raw.body.getReader();
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            yield value;
+        }
+    }
+}
+export function queuePlugin(options) {
+    const adapter = options.adapter;
+    const filter = typeof options.filter === 'function'
+        ? options.filter
+        : options.filter
+            ? compileFilter(options.filter)
+            : () => true;
+    const generateId = options.jobIdGenerator || generateJobId;
+    const queueMiddleware = async (req, next) => {
+        const perReq = req.queue;
+        if (perReq === false) {
+            return next(req);
+        }
+        const shouldQueue = perReq === true || typeof perReq === 'object' || filter(req);
+        if (!shouldQueue) {
+            return next(req);
+        }
+        const metadata = {
+            ...options.defaultMetadata,
+            ...(typeof perReq === 'object' && perReq !== null && 'metadata' in perReq
+                ? perReq.metadata
+                : undefined),
+        };
+        const jobId = generateId(req);
+        const job = serializeRequest(req, jobId, metadata);
+        const assignedId = await adapter.enqueue(job);
+        return new QueuedResponse(assignedId || jobId);
+    };
+    return (client) => {
+        client.use(queueMiddleware);
+    };
+}
+export class InMemoryQueueAdapter {
+    jobs = new Map();
+    async enqueue(job) {
+        this.jobs.set(job.jobId, { ...job, status: 'pending' });
+        return job.jobId;
+    }
+    async getStatus(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return { jobId, status: 'unknown' };
+        return { jobId, status: job.status };
+    }
+    getPendingJobs() {
+        return Array.from(this.jobs.values())
+            .filter((j) => j.status === 'pending')
+            .map(({ status, ...job }) => job);
+    }
+    setJobStatus(jobId, status) {
+        const job = this.jobs.get(jobId);
+        if (job)
+            job.status = status;
+    }
+    getJob(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return undefined;
+        const { status, ...rest } = job;
+        return rest;
+    }
+    clear() {
+        this.jobs.clear();
+    }
+    get size() {
+        return this.jobs.size;
+    }
+    async close() {
+        this.jobs.clear();
+    }
+}
+export { QueuedResponse };

package/dist/queue/consumer.d.ts

@@ -0,0 +1,17 @@
+import type { QueueJob, ReckerResponse } from '../types/index.js';
+export interface ExecuteQueuedRequestOptions {
+    client?: {
+        request(url: string, options?: any): Promise<ReckerResponse>;
+    };
+    clientOptions?: Record<string, unknown>;
+    timeout?: number;
+}
+export interface QueueExecutionResult {
+    jobId: string;
+    success: boolean;
+    response?: ReckerResponse;
+    error?: Error;
+    duration: number;
+}
+export declare function executeQueuedRequest(job: QueueJob, options?: ExecuteQueuedRequestOptions): Promise<QueueExecutionResult>;
+export declare function isValidQueueJob(value: unknown): value is QueueJob;

package/dist/queue/consumer.js

@@ -0,0 +1,48 @@
+export async function executeQueuedRequest(job, options = {}) {
+    const startTime = Date.now();
+    const resolvedClient = options.client ??
+        (await import('../core/client.js').then((m) => m.createClient(options.clientOptions || {})));
+    const client = resolvedClient;
+    try {
+        const requestOptions = {
+            method: job.method,
+            headers: { ...job.headers },
+            body: job.body,
+            correlationId: job.correlationId,
+            traceId: job.traceId,
+            tenant: job.tenant,
+            policyTags: job.policyTags,
+            throwHttpErrors: false,
+            queue: false,
+        };
+        if (options.timeout) {
+            requestOptions.timeout = options.timeout;
+        }
+        const response = await client.request(job.url, requestOptions);
+        return {
+            jobId: job.jobId,
+            success: response.ok,
+            response,
+            duration: Date.now() - startTime,
+        };
+    }
+    catch (error) {
+        return {
+            jobId: job.jobId,
+            success: false,
+            error: error instanceof Error ? error : new Error(String(error)),
+            duration: Date.now() - startTime,
+        };
+    }
+}
+export function isValidQueueJob(value) {
+    if (!value || typeof value !== 'object')
+        return false;
+    const obj = value;
+    return (typeof obj.jobId === 'string' &&
+        typeof obj.url === 'string' &&
+        typeof obj.method === 'string' &&
+        typeof obj.headers === 'object' &&
+        obj.headers !== null &&
+        typeof obj.createdAt === 'number');
+}

package/dist/scrape/crawl-queue.d.ts

@@ -0,0 +1,31 @@
+export interface CrawlQueueItem {
+    url: string;
+    depth: number;
+    priority?: number;
+    discoveredFrom?: string;
+}
+export interface CrawlQueueAdapter {
+    push(item: CrawlQueueItem): Promise<void>;
+    pushBatch?(items: CrawlQueueItem[]): Promise<void>;
+    pop(): Promise<CrawlQueueItem | null>;
+    hasVisited(url: string): Promise<boolean>;
+    hasVisitedBatch?(urls: string[]): Promise<Set<string>>;
+    markVisited(url: string): Promise<void>;
+    size(): Promise<number>;
+    clear(): Promise<void>;
+    close?(): Promise<void>;
+}
+export declare class InMemoryCrawlQueue implements CrawlQueueAdapter {
+    private queue;
+    private visited;
+    push(item: CrawlQueueItem): Promise<void>;
+    pushBatch(items: CrawlQueueItem[]): Promise<void>;
+    pop(): Promise<CrawlQueueItem | null>;
+    hasVisited(url: string): Promise<boolean>;
+    hasVisitedBatch(urls: string[]): Promise<Set<string>>;
+    markVisited(url: string): Promise<void>;
+    size(): Promise<number>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+    getVisited(): Set<string>;
+}

package/dist/scrape/crawl-queue.js

@@ -0,0 +1,40 @@
+export class InMemoryCrawlQueue {
+    queue = [];
+    visited = new Set();
+    async push(item) {
+        this.queue.push(item);
+    }
+    async pushBatch(items) {
+        this.queue.push(...items);
+    }
+    async pop() {
+        return this.queue.shift() ?? null;
+    }
+    async hasVisited(url) {
+        return this.visited.has(url);
+    }
+    async hasVisitedBatch(urls) {
+        const result = new Set();
+        for (const url of urls) {
+            if (this.visited.has(url))
+                result.add(url);
+        }
+        return result;
+    }
+    async markVisited(url) {
+        this.visited.add(url);
+    }
+    async size() {
+        return this.queue.length;
+    }
+    async clear() {
+        this.queue = [];
+        this.visited.clear();
+    }
+    async close() {
+        await this.clear();
+    }
+    getVisited() {
+        return this.visited;
+    }
+}

package/dist/scrape/crawl-storage.d.ts

@@ -0,0 +1,33 @@
+import type { SpiderPageResult } from './spider.js';
+export interface CrawlStorageAdapter {
+    saveResult(result: SpiderPageResult): Promise<void>;
+    saveError(error: {
+        url: string;
+        error: string;
+    }): Promise<void>;
+    getResultCount(): Promise<number>;
+    getResults(): Promise<SpiderPageResult[]>;
+    getErrors(): Promise<Array<{
+        url: string;
+        error: string;
+    }>>;
+    clear(): Promise<void>;
+    close?(): Promise<void>;
+}
+export declare class InMemoryCrawlStorage implements CrawlStorageAdapter {
+    private results;
+    private errors;
+    saveResult(result: SpiderPageResult): Promise<void>;
+    saveError(error: {
+        url: string;
+        error: string;
+    }): Promise<void>;
+    getResultCount(): Promise<number>;
+    getResults(): Promise<SpiderPageResult[]>;
+    getErrors(): Promise<Array<{
+        url: string;
+        error: string;
+    }>>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/scrape/crawl-storage.js

@@ -0,0 +1,26 @@
+export class InMemoryCrawlStorage {
+    results = [];
+    errors = [];
+    async saveResult(result) {
+        this.results.push(result);
+    }
+    async saveError(error) {
+        this.errors.push(error);
+    }
+    async getResultCount() {
+        return this.results.length;
+    }
+    async getResults() {
+        return this.results;
+    }
+    async getErrors() {
+        return this.errors;
+    }
+    async clear() {
+        this.results = [];
+        this.errors = [];
+    }
+    async close() {
+        await this.clear();
+    }
+}

package/dist/scrape/index.d.ts

@@ -4,5 +4,11 @@ export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
 export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
+export { InMemoryCrawlQueue } from './crawl-queue.js';
+export type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
+export { InMemoryCrawlStorage } from './crawl-storage.js';
+export type { CrawlStorageAdapter } from './crawl-storage.js';
+export { ListProxyAdapter } from './proxy-adapter.js';
+export type { ProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
 export type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedFormField, ExtractedTable, ExtractedScript, ExtractedStyle, ExtractionSchema, ExtractionSchemaField, ScrapeOptions, LinkExtractionOptions, ImageExtractionOptions, } from './types.js';

package/dist/scrape/index.js

@@ -2,4 +2,7 @@ export { parse as parseHtmlSync, HTMLElement, TextNode, CommentNode, Node, NodeT
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
+export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { ListProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/scrape/proxy-adapter.d.ts

@@ -0,0 +1,12 @@
+export interface ProxyAdapter {
+    getProxy(): Promise<string | null>;
+    reportResult?(proxy: string, success: boolean): Promise<void>;
+    close?(): Promise<void>;
+}
+export declare class ListProxyAdapter implements ProxyAdapter {
+    private readonly proxies;
+    private index;
+    constructor(proxies: string[]);
+    getProxy(): Promise<string>;
+    close(): Promise<void>;
+}

package/dist/scrape/proxy-adapter.js

@@ -0,0 +1,17 @@
+export class ListProxyAdapter {
+    proxies;
+    index = 0;
+    constructor(proxies) {
+        this.proxies = proxies;
+        if (proxies.length === 0) {
+            throw new Error('ListProxyAdapter requires at least one proxy');
+        }
+    }
+    async getProxy() {
+        const proxy = this.proxies[this.index % this.proxies.length];
+        this.index++;
+        return proxy;
+    }
+    async close() {
+    }
+}

package/dist/scrape/spider.d.ts

@@ -2,6 +2,9 @@ import type { ExtractedLink, ExtractionSchema } from './types.js';
 import type { Options as ParserOptions } from './parser/index.js';
 import { type SitemapUrl } from '../seo/validators/sitemap.js';
 import { type CaptchaDetectionResult, type BlockDetectionResult } from '../utils/block-detector.js';
+import { type CrawlQueueAdapter } from './crawl-queue.js';
+import { type CrawlStorageAdapter } from './crawl-storage.js';
+import { type ProxyAdapter } from './proxy-adapter.js';
 export type SpiderTransport = 'auto' | 'undici' | 'curl';
 type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
@@ -25,6 +28,7 @@ export interface SpiderOptions {
     respectRobotsTxt?: boolean;
     useSitemap?: boolean;
     sitemapUrl?: string;
+    proxy?: string | string[] | ProxyAdapter;
     transport?: SpiderTransport;
     preferCurlFirst?: boolean;
     onPage?: (result: SpiderPageResult) => void;
@@ -39,6 +43,8 @@ export interface SpiderOptions {
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;
+    crawlQueue?: CrawlQueueAdapter;
+    crawlStorage?: CrawlStorageAdapter;
 }
 export interface SpiderPageResult {
     url: string;
@@ -146,10 +152,13 @@ export declare class Spider {
     private options;
     private client;
     private pool;
-    private visited;
-    private queue;
-    private results;
-    private errors;
+    private crawlQueue;
+    private crawlStorage;
+    private proxyAdapter;
+    private proxyClients;
+    private _visitedCount;
+    private _queueSize;
+    private _resultCount;
     private baseHost;
     private running;
     private aborted;
@@ -170,6 +179,7 @@ export declare class Spider {
     private fetchSitemaps;
     private buildSitemapAnalysis;
     private buildRobotsAnalysis;
+    private getClientForProxy;
     private fetchPage;
     private crawlPage;
     private getOrCreateDomainState;