recker 1.0.85 → 1.0.86-next.a24fa13

package/dist/browser/plugins/queue.d.ts

@@ -0,0 +1,41 @@
+import { Plugin, ReckerResponse, QueueAdapter, QueueJob, QueueOptions } from '../types/index.js';
+declare class QueuedResponse<T = unknown> implements ReckerResponse<T> {
+    readonly jobId: string;
+    readonly raw: Response;
+    readonly timings: undefined;
+    readonly connection: undefined;
+    constructor(jobId: string);
+    get status(): number;
+    get statusText(): string;
+    get headers(): Headers;
+    get ok(): boolean;
+    get url(): string;
+    json<R = T>(): Promise<R>;
+    text(): Promise<string>;
+    cleanText(): Promise<string>;
+    blob(): Promise<Blob>;
+    read(): ReadableStream<Uint8Array<ArrayBuffer>> | null;
+    clone(): ReckerResponse<T>;
+    sse(): AsyncGenerator<never, void, unknown>;
+    download(): AsyncGenerator<never, void, unknown>;
+    [Symbol.asyncIterator](): AsyncGenerator<Uint8Array<ArrayBuffer>, void, unknown>;
+}
+export declare function queuePlugin(options: QueueOptions): Plugin;
+export declare class InMemoryQueueAdapter implements QueueAdapter {
+    private jobs;
+    enqueue(job: QueueJob): Promise<string>;
+    getStatus(jobId: string): Promise<{
+        jobId: string;
+        status: "unknown";
+    } | {
+        jobId: string;
+        status: "pending" | "processing" | "completed" | "failed";
+    }>;
+    getPendingJobs(): QueueJob[];
+    setJobStatus(jobId: string, status: string): void;
+    getJob(jobId: string): QueueJob | undefined;
+    clear(): void;
+    get size(): number;
+    close(): Promise<void>;
+}
+export { QueuedResponse };

package/dist/browser/plugins/queue.js

@@ -0,0 +1,184 @@
+let jobSeq = 0;
+function generateJobId() {
+    jobSeq += 1;
+    return `qj-${Date.now().toString(36)}-${jobSeq.toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
+}
+function serializeRequest(req, jobId, metadata) {
+    const headers = {};
+    req.headers.forEach((value, key) => {
+        headers[key] = value;
+    });
+    let body = null;
+    const bodyContentType = headers['content-type'];
+    if (req.body !== null && req.body !== undefined) {
+        if (typeof req.body === 'string') {
+            body = req.body;
+        }
+        else if (typeof req.body === 'object') {
+            try {
+                body = JSON.stringify(req.body);
+            }
+            catch {
+                body = String(req.body);
+            }
+        }
+        else {
+            body = String(req.body);
+        }
+    }
+    return {
+        jobId,
+        url: req.url,
+        method: req.method,
+        headers,
+        body,
+        bodyContentType: bodyContentType || undefined,
+        correlationId: req.correlationId,
+        traceId: req.traceId,
+        tenant: req.tenant,
+        policyTags: req.policyTags?.length ? req.policyTags : undefined,
+        createdAt: Date.now(),
+        metadata: metadata && Object.keys(metadata).length > 0 ? metadata : undefined,
+    };
+}
+function compileFilter(config) {
+    return (req) => {
+        if (config.methods && !config.methods.includes(req.method)) {
+            return false;
+        }
+        if (config.urlPatterns) {
+            const matched = config.urlPatterns.some((pattern) => {
+                if (typeof pattern === 'string')
+                    return req.url.includes(pattern);
+                return pattern.test(req.url);
+            });
+            if (!matched)
+                return false;
+        }
+        if (config.headerPresent && !req.headers.has(config.headerPresent)) {
+            return false;
+        }
+        return true;
+    };
+}
+class QueuedResponse {
+    jobId;
+    raw;
+    timings = undefined;
+    connection = undefined;
+    constructor(jobId) {
+        this.jobId = jobId;
+        const responseBody = JSON.stringify({
+            queued: true,
+            jobId,
+            status: 'queued',
+        });
+        this.raw = new Response(responseBody, {
+            status: 202,
+            statusText: 'Accepted',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-Queue-Job-Id': jobId,
+                'X-Queue-Status': 'queued',
+            },
+        });
+    }
+    get status() { return 202; }
+    get statusText() { return 'Accepted'; }
+    get headers() { return this.raw.headers; }
+    get ok() { return true; }
+    get url() { return ''; }
+    json() { return this.raw.json(); }
+    text() { return this.raw.text(); }
+    async cleanText() { return this.raw.text(); }
+    blob() { return this.raw.blob(); }
+    read() { return this.raw.body; }
+    clone() {
+        return new QueuedResponse(this.jobId);
+    }
+    async *sse() { }
+    async *download() { }
+    async *[Symbol.asyncIterator]() {
+        if (!this.raw.body)
+            return;
+        const reader = this.raw.body.getReader();
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            yield value;
+        }
+    }
+}
+export function queuePlugin(options) {
+    const adapter = options.adapter;
+    const filter = typeof options.filter === 'function'
+        ? options.filter
+        : options.filter
+            ? compileFilter(options.filter)
+            : () => true;
+    const generateId = options.jobIdGenerator || generateJobId;
+    const queueMiddleware = async (req, next) => {
+        const perReq = req.queue;
+        if (perReq === false) {
+            return next(req);
+        }
+        const shouldQueue = perReq === true || typeof perReq === 'object' || filter(req);
+        if (!shouldQueue) {
+            return next(req);
+        }
+        const metadata = {
+            ...options.defaultMetadata,
+            ...(typeof perReq === 'object' && perReq !== null && 'metadata' in perReq
+                ? perReq.metadata
+                : undefined),
+        };
+        const jobId = generateId(req);
+        const job = serializeRequest(req, jobId, metadata);
+        const assignedId = await adapter.enqueue(job);
+        return new QueuedResponse(assignedId || jobId);
+    };
+    return (client) => {
+        client.use(queueMiddleware);
+    };
+}
+export class InMemoryQueueAdapter {
+    jobs = new Map();
+    async enqueue(job) {
+        this.jobs.set(job.jobId, { ...job, status: 'pending' });
+        return job.jobId;
+    }
+    async getStatus(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return { jobId, status: 'unknown' };
+        return { jobId, status: job.status };
+    }
+    getPendingJobs() {
+        return Array.from(this.jobs.values())
+            .filter((j) => j.status === 'pending')
+            .map(({ status, ...job }) => job);
+    }
+    setJobStatus(jobId, status) {
+        const job = this.jobs.get(jobId);
+        if (job)
+            job.status = status;
+    }
+    getJob(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return undefined;
+        const { status, ...rest } = job;
+        return rest;
+    }
+    clear() {
+        this.jobs.clear();
+    }
+    get size() {
+        return this.jobs.size;
+    }
+    async close() {
+        this.jobs.clear();
+    }
+}
+export { QueuedResponse };

package/dist/browser/recker.d.ts

@@ -4,6 +4,7 @@ import type { RequestOptions } from '../types/index.js';
 import { analyzeSeo } from '../seo/analyzer.js';
 import { createAI } from '../ai/index.js';
 import { simulateNetwork } from '../plugins/network-simulation.js';
+import type { RaffelClientOptions, RaffelClient } from 'raffel/client/browser';
 export declare function get<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
 export declare function post<T = unknown>(url: string, options?: RequestOptions): RequestPromise<T>;
 export declare function post<T = unknown>(url: string, body?: unknown, options?: RequestOptions): RequestPromise<T>;
@@ -26,6 +27,7 @@ export declare const recker: {
     options: typeof options;
     purge: typeof purge;
     ws: typeof ws;
+    raffel: (url: string, options?: Omit<RaffelClientOptions, "url">) => RaffelClient;
     seo: typeof analyzeSeo;
     ai: typeof createAI;
     har: import("../plugins/har-recorder.js").HarRecorder;

package/dist/browser/recker.js

@@ -4,6 +4,7 @@ import { analyzeSeo } from '../seo/analyzer.js';
 import { createAI } from '../ai/index.js';
 import { harRecorder } from '../plugins/har-recorder.js';
 import { simulateNetwork } from '../plugins/network-simulation.js';
+import { createRaffelClient } from 'raffel/client/browser';
 let _defaultClient = null;
 const REQUEST_OPTIONS_HINTS = new Set([
     'method',
@@ -98,6 +99,7 @@ export const recker = {
     options,
     purge,
     ws,
+    raffel: (url, options) => createRaffelClient({ ...options, url }),
     seo: analyzeSeo,
     ai: createAI,
     har: harRecorder,

package/dist/browser/scrape/crawl-queue.d.ts

@@ -0,0 +1,31 @@
+export interface CrawlQueueItem {
+    url: string;
+    depth: number;
+    priority?: number;
+    discoveredFrom?: string;
+}
+export interface CrawlQueueAdapter {
+    push(item: CrawlQueueItem): Promise<void>;
+    pushBatch?(items: CrawlQueueItem[]): Promise<void>;
+    pop(): Promise<CrawlQueueItem | null>;
+    hasVisited(url: string): Promise<boolean>;
+    hasVisitedBatch?(urls: string[]): Promise<Set<string>>;
+    markVisited(url: string): Promise<void>;
+    size(): Promise<number>;
+    clear(): Promise<void>;
+    close?(): Promise<void>;
+}
+export declare class InMemoryCrawlQueue implements CrawlQueueAdapter {
+    private queue;
+    private visited;
+    push(item: CrawlQueueItem): Promise<void>;
+    pushBatch(items: CrawlQueueItem[]): Promise<void>;
+    pop(): Promise<CrawlQueueItem | null>;
+    hasVisited(url: string): Promise<boolean>;
+    hasVisitedBatch(urls: string[]): Promise<Set<string>>;
+    markVisited(url: string): Promise<void>;
+    size(): Promise<number>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+    getVisited(): Set<string>;
+}

package/dist/browser/scrape/crawl-queue.js

@@ -0,0 +1,40 @@
+export class InMemoryCrawlQueue {
+    queue = [];
+    visited = new Set();
+    async push(item) {
+        this.queue.push(item);
+    }
+    async pushBatch(items) {
+        this.queue.push(...items);
+    }
+    async pop() {
+        return this.queue.shift() ?? null;
+    }
+    async hasVisited(url) {
+        return this.visited.has(url);
+    }
+    async hasVisitedBatch(urls) {
+        const result = new Set();
+        for (const url of urls) {
+            if (this.visited.has(url))
+                result.add(url);
+        }
+        return result;
+    }
+    async markVisited(url) {
+        this.visited.add(url);
+    }
+    async size() {
+        return this.queue.length;
+    }
+    async clear() {
+        this.queue = [];
+        this.visited.clear();
+    }
+    async close() {
+        await this.clear();
+    }
+    getVisited() {
+        return this.visited;
+    }
+}

package/dist/browser/scrape/crawl-storage.d.ts

@@ -0,0 +1,33 @@
+import type { SpiderPageResult } from './spider.js';
+export interface CrawlStorageAdapter {
+    saveResult(result: SpiderPageResult): Promise<void>;
+    saveError(error: {
+        url: string;
+        error: string;
+    }): Promise<void>;
+    getResultCount(): Promise<number>;
+    getResults(): Promise<SpiderPageResult[]>;
+    getErrors(): Promise<Array<{
+        url: string;
+        error: string;
+    }>>;
+    clear(): Promise<void>;
+    close?(): Promise<void>;
+}
+export declare class InMemoryCrawlStorage implements CrawlStorageAdapter {
+    private results;
+    private errors;
+    saveResult(result: SpiderPageResult): Promise<void>;
+    saveError(error: {
+        url: string;
+        error: string;
+    }): Promise<void>;
+    getResultCount(): Promise<number>;
+    getResults(): Promise<SpiderPageResult[]>;
+    getErrors(): Promise<Array<{
+        url: string;
+        error: string;
+    }>>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/browser/scrape/crawl-storage.js

@@ -0,0 +1,26 @@
+export class InMemoryCrawlStorage {
+    results = [];
+    errors = [];
+    async saveResult(result) {
+        this.results.push(result);
+    }
+    async saveError(error) {
+        this.errors.push(error);
+    }
+    async getResultCount() {
+        return this.results.length;
+    }
+    async getResults() {
+        return this.results;
+    }
+    async getErrors() {
+        return this.errors;
+    }
+    async clear() {
+        this.results = [];
+        this.errors = [];
+    }
+    async close() {
+        await this.clear();
+    }
+}

package/dist/browser/scrape/index.d.ts

@@ -4,5 +4,11 @@ export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
 export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
+export { InMemoryCrawlQueue } from './crawl-queue.js';
+export type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
+export { InMemoryCrawlStorage } from './crawl-storage.js';
+export type { CrawlStorageAdapter } from './crawl-storage.js';
+export { ListProxyAdapter } from './proxy-adapter.js';
+export type { ProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
 export type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedFormField, ExtractedTable, ExtractedScript, ExtractedStyle, ExtractionSchema, ExtractionSchemaField, ScrapeOptions, LinkExtractionOptions, ImageExtractionOptions, } from './types.js';

package/dist/browser/scrape/index.js

@@ -2,4 +2,7 @@ export { parse as parseHtmlSync, HTMLElement, TextNode, CommentNode, Node, NodeT
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
+export { InMemoryCrawlQueue } from './crawl-queue.js';
+export { InMemoryCrawlStorage } from './crawl-storage.js';
+export { ListProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/browser/scrape/proxy-adapter.d.ts

@@ -0,0 +1,12 @@
+export interface ProxyAdapter {
+    getProxy(): Promise<string | null>;
+    reportResult?(proxy: string, success: boolean): Promise<void>;
+    close?(): Promise<void>;
+}
+export declare class ListProxyAdapter implements ProxyAdapter {
+    private readonly proxies;
+    private index;
+    constructor(proxies: string[]);
+    getProxy(): Promise<string>;
+    close(): Promise<void>;
+}

package/dist/browser/scrape/proxy-adapter.js

@@ -0,0 +1,17 @@
+export class ListProxyAdapter {
+    proxies;
+    index = 0;
+    constructor(proxies) {
+        this.proxies = proxies;
+        if (proxies.length === 0) {
+            throw new Error('ListProxyAdapter requires at least one proxy');
+        }
+    }
+    async getProxy() {
+        const proxy = this.proxies[this.index % this.proxies.length];
+        this.index++;
+        return proxy;
+    }
+    async close() {
+    }
+}

package/dist/browser/scrape/spider.d.ts

@@ -2,6 +2,9 @@ import type { ExtractedLink, ExtractionSchema } from './types.js';
 import type { Options as ParserOptions } from './parser/index.js';
 import { type SitemapUrl } from '../seo/validators/sitemap.js';
 import { type CaptchaDetectionResult, type BlockDetectionResult } from '../utils/block-detector.js';
+import { type CrawlQueueAdapter } from './crawl-queue.js';
+import { type CrawlStorageAdapter } from './crawl-storage.js';
+import { type ProxyAdapter } from './proxy-adapter.js';
 export type SpiderTransport = 'auto' | 'undici' | 'curl';
 type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
@@ -25,6 +28,7 @@ export interface SpiderOptions {
     respectRobotsTxt?: boolean;
     useSitemap?: boolean;
     sitemapUrl?: string;
+    proxy?: string | string[] | ProxyAdapter;
     transport?: SpiderTransport;
     preferCurlFirst?: boolean;
     onPage?: (result: SpiderPageResult) => void;
@@ -39,6 +43,8 @@ export interface SpiderOptions {
     onProgress?: (progress: SpiderProgress) => void;
     extract?: string[] | ExtractionSchema;
     parserOptions?: Partial<ParserOptions>;
+    crawlQueue?: CrawlQueueAdapter;
+    crawlStorage?: CrawlStorageAdapter;
 }
 export interface SpiderPageResult {
     url: string;
@@ -146,10 +152,13 @@ export declare class Spider {
     private options;
     private client;
     private pool;
-    private visited;
-    private queue;
-    private results;
-    private errors;
+    private crawlQueue;
+    private crawlStorage;
+    private proxyAdapter;
+    private proxyClients;
+    private _visitedCount;
+    private _queueSize;
+    private _resultCount;
     private baseHost;
     private running;
     private aborted;
@@ -170,6 +179,7 @@ export declare class Spider {
     private fetchSitemaps;
     private buildSitemapAnalysis;
     private buildRobotsAnalysis;
+    private getClientForProxy;
     private fetchPage;
     private crawlPage;
     private getOrCreateDomainState;