recker 1.0.43 → 1.0.44

package/dist/browser/transport/curl.d.ts

@@ -0,0 +1,4 @@
+import { Transport, ReckerRequest, ReckerResponse } from '../types/index.js';
+export declare class CurlTransport implements Transport {
+    dispatch(req: ReckerRequest): Promise<ReckerResponse>;
+}

package/dist/browser/transport/curl.js

@@ -0,0 +1,101 @@
+import { spawn } from 'node:child_process';
+import { HttpResponse } from '../core/response.js';
+import { NetworkError } from '../core/errors.js';
+import { getCurlPath, hasImpersonate } from '../utils/binary-manager.js';
+export class CurlTransport {
+    async dispatch(req) {
+        return new Promise(async (resolve, reject) => {
+            const args = [
+                '-X', req.method,
+                req.url,
+                '-i',
+                '-s',
+                '--compressed',
+                '--no-keepalive'
+            ];
+            req.headers.forEach((val, key) => {
+                args.push('-H', `${key}: ${val}`);
+            });
+            if (req.body && typeof req.body === 'string') {
+                args.push('-d', req.body);
+            }
+            let command = process.env.RECKER_CURL_BIN;
+            if (!command) {
+                if (await hasImpersonate()) {
+                    command = getCurlPath();
+                }
+                else {
+                    command = 'curl';
+                }
+            }
+            const child = spawn(command, args);
+            const stdoutChunks = [];
+            const stderrChunks = [];
+            child.stdout.on('data', (chunk) => stdoutChunks.push(chunk));
+            child.stderr.on('data', (chunk) => stderrChunks.push(chunk));
+            child.on('error', (err) => {
+                reject(new NetworkError(`Failed to spawn curl: ${err.message}`, 'ERR_CURL_SPAWN', req));
+            });
+            child.on('close', (code) => {
+                if (code !== 0) {
+                    const stderr = Buffer.concat(stderrChunks).toString();
+                    reject(new NetworkError(`Curl exited with code ${code}: ${stderr}`, 'ERR_CURL_EXIT', req));
+                    return;
+                }
+                const fullOutput = Buffer.concat(stdoutChunks);
+                let headerEndIndex = fullOutput.indexOf('\r\n\r\n');
+                let offset = 4;
+                if (headerEndIndex === -1) {
+                    headerEndIndex = fullOutput.indexOf('\n\n');
+                    offset = 2;
+                }
+                if (headerEndIndex !== -1) {
+                    const firstLine = fullOutput.subarray(0, Math.min(20, headerEndIndex)).toString();
+                    if (firstLine.startsWith('HTTP/1.1 100') || firstLine.startsWith('HTTP/2 100')) {
+                        const nextStart = headerEndIndex + offset;
+                        const secondSplit = fullOutput.indexOf('\r\n\r\n', nextStart);
+                        if (secondSplit !== -1) {
+                            headerEndIndex = secondSplit;
+                            offset = 4;
+                        }
+                    }
+                }
+                if (headerEndIndex === -1) {
+                    const nativeResponse = new Response(fullOutput, { status: 200, statusText: 'OK' });
+                    resolve(new HttpResponse(nativeResponse, { connection: { protocol: 'curl' } }));
+                    return;
+                }
+                const headerBlock = fullOutput.subarray(0, headerEndIndex).toString();
+                const bodyBlock = fullOutput.subarray(headerEndIndex + offset);
+                const headerLines = headerBlock.split(/\r?\n/);
+                const statusLine = headerLines[0];
+                let status = 200;
+                let statusText = 'OK';
+                const statusMatch = statusLine.match(/HTTP\/[\d\.]+ (\d+) ?(.*)/);
+                if (statusMatch) {
+                    status = parseInt(statusMatch[1], 10);
+                    statusText = statusMatch[2] || '';
+                }
+                const headers = new Headers();
+                for (let i = 1; i < headerLines.length; i++) {
+                    const line = headerLines[i];
+                    const colon = line.indexOf(':');
+                    if (colon > 0) {
+                        const key = line.substring(0, colon).trim();
+                        const val = line.substring(colon + 1).trim();
+                        headers.append(key, val);
+                    }
+                }
+                const nativeResponse = new Response(bodyBlock, {
+                    status,
+                    statusText,
+                    headers
+                });
+                resolve(new HttpResponse(nativeResponse, {
+                    timings: {},
+                    connection: { protocol: 'curl' }
+                }));
+            });
+        });
+    }
+}

package/dist/browser/transport/undici.js

@@ -852,8 +852,7 @@ function wrapDownloadResponse(response, onProgress) {
     });
     return new Response(webBody, {
         status: response.statusCode,
-        statusText: String(response.statusCode),
-        headers: response.headers
+        headers: response.headers,
     });
 }
 function wrapUploadBody(body, onProgress, total) {

package/dist/browser/transport/worker.d.ts

@@ -0,0 +1,18 @@
+import type { ReckerRequest, ReckerResponse, Transport } from '../types/index.js';
+export interface WorkerTransportOptions {
+    poolSize?: number;
+}
+export declare class WorkerTransport implements Transport {
+    private options;
+    private workers;
+    private workerIndex;
+    private pendingRequests;
+    private workerUrl;
+    constructor(options?: WorkerTransportOptions);
+    static isSupported(): boolean;
+    private createWorker;
+    private getNextWorker;
+    dispatch(req: ReckerRequest): Promise<ReckerResponse>;
+    terminate(): void;
+    get pendingCount(): number;
+}

package/dist/browser/transport/worker.js

@@ -0,0 +1,278 @@
+const WORKER_SCRIPT = `
+self.onmessage = async (event) => {
+  const { id, method, url, headers, body, timeout } = event.data;
+
+  try {
+    const controller = new AbortController();
+    let timeoutId;
+
+    if (timeout) {
+      timeoutId = setTimeout(() => controller.abort(), timeout);
+    }
+
+    const start = performance.now();
+
+    const response = await fetch(url, {
+      method,
+      headers,
+      body,
+      signal: controller.signal,
+      keepalive: true
+    });
+
+    if (timeoutId) clearTimeout(timeoutId);
+
+    const totalTime = performance.now() - start;
+
+    // Read response body as ArrayBuffer for transferability
+    const arrayBuffer = await response.arrayBuffer();
+
+    // Serialize headers
+    const responseHeaders = {};
+    response.headers.forEach((value, key) => {
+      responseHeaders[key] = value;
+    });
+
+    self.postMessage({
+      id,
+      success: true,
+      status: response.status,
+      statusText: response.statusText,
+      headers: responseHeaders,
+      ok: response.ok,
+      url: response.url,
+      body: arrayBuffer,
+      timings: { total: totalTime, firstByte: totalTime }
+    }, [arrayBuffer]);
+
+  } catch (error) {
+    self.postMessage({
+      id,
+      success: false,
+      error: {
+        name: error.name,
+        message: error.message
+      }
+    });
+  }
+};
+`;
+export class WorkerTransport {
+    options;
+    workers = [];
+    workerIndex = 0;
+    pendingRequests = new Map();
+    workerUrl;
+    constructor(options = {}) {
+        this.options = options;
+        const poolSize = options.poolSize ?? (typeof navigator !== 'undefined' ? navigator.hardwareConcurrency : 4) ?? 4;
+        const blob = new Blob([WORKER_SCRIPT], { type: 'application/javascript' });
+        this.workerUrl = URL.createObjectURL(blob);
+        for (let i = 0; i < poolSize; i++) {
+            this.createWorker();
+        }
+    }
+    static isSupported() {
+        return typeof Worker !== 'undefined';
+    }
+    createWorker() {
+        const worker = new Worker(this.workerUrl);
+        worker.onmessage = (event) => {
+            const { id, success, error, ...response } = event.data;
+            const pending = this.pendingRequests.get(id);
+            if (!pending)
+                return;
+            this.pendingRequests.delete(id);
+            if (success) {
+                pending.resolve(new WorkerResponseWrapper(response));
+            }
+            else {
+                const err = new Error(error?.message || 'Worker request failed');
+                err.name = error?.name || 'WorkerError';
+                pending.reject(err);
+            }
+        };
+        worker.onerror = (event) => {
+            console.error('[WorkerTransport] Worker error:', event.message);
+        };
+        this.workers.push(worker);
+    }
+    getNextWorker() {
+        const worker = this.workers[this.workerIndex];
+        this.workerIndex = (this.workerIndex + 1) % this.workers.length;
+        return worker;
+    }
+    async dispatch(req) {
+        if (!WorkerTransport.isSupported()) {
+            throw new Error('Web Workers are not supported in this environment');
+        }
+        const id = crypto.randomUUID();
+        const worker = this.getNextWorker();
+        const timeoutMs = typeof req.timeout === 'number'
+            ? req.timeout
+            : req.timeout?.request;
+        const headers = {};
+        if (req.headers) {
+            if (req.headers instanceof Headers) {
+                req.headers.forEach((value, key) => {
+                    headers[key] = value;
+                });
+            }
+            else if (typeof req.headers === 'object') {
+                Object.assign(headers, req.headers);
+            }
+        }
+        let body;
+        if (req.body) {
+            if (typeof req.body === 'string') {
+                body = req.body;
+            }
+            else if (req.body instanceof ArrayBuffer) {
+                body = req.body;
+            }
+            else if (req.body instanceof Blob) {
+                body = await req.body.arrayBuffer();
+            }
+            else if (typeof req.body === 'object') {
+                body = JSON.stringify(req.body);
+            }
+        }
+        return new Promise((resolve, reject) => {
+            this.pendingRequests.set(id, { resolve, reject });
+            if (req.signal) {
+                req.signal.addEventListener('abort', () => {
+                    const pending = this.pendingRequests.get(id);
+                    if (pending) {
+                        this.pendingRequests.delete(id);
+                        const err = new Error('Request aborted');
+                        err.name = 'AbortError';
+                        pending.reject(err);
+                    }
+                });
+            }
+            worker.postMessage({
+                id,
+                method: req.method,
+                url: req.url,
+                headers,
+                body,
+                timeout: timeoutMs
+            });
+        });
+    }
+    terminate() {
+        for (const worker of this.workers) {
+            worker.terminate();
+        }
+        this.workers = [];
+        URL.revokeObjectURL(this.workerUrl);
+        for (const [id, pending] of this.pendingRequests) {
+            pending.reject(new Error('Transport terminated'));
+        }
+        this.pendingRequests.clear();
+    }
+    get pendingCount() {
+        return this.pendingRequests.size;
+    }
+}
+class WorkerResponseWrapper {
+    status;
+    statusText;
+    headers;
+    ok;
+    url;
+    timings;
+    connection = {};
+    raw;
+    bodyBuffer;
+    bodyUsed = false;
+    constructor(data) {
+        this.status = data.status ?? 0;
+        this.statusText = data.statusText ?? '';
+        this.ok = data.ok ?? false;
+        this.url = data.url ?? '';
+        this.timings = data.timings ?? { total: 0 };
+        this.bodyBuffer = data.body ?? new ArrayBuffer(0);
+        this.headers = new Headers();
+        if (data.headers) {
+            for (const [key, value] of Object.entries(data.headers)) {
+                this.headers.set(key, value);
+            }
+        }
+        this.raw = new Response(this.bodyBuffer, {
+            status: this.status,
+            statusText: this.statusText,
+            headers: this.headers,
+        });
+    }
+    checkBodyUsed() {
+        if (this.bodyUsed) {
+            throw new Error('Body has already been consumed');
+        }
+        this.bodyUsed = true;
+    }
+    async json() {
+        this.checkBodyUsed();
+        const text = new TextDecoder().decode(this.bodyBuffer);
+        return JSON.parse(text);
+    }
+    async text() {
+        this.checkBodyUsed();
+        return new TextDecoder().decode(this.bodyBuffer);
+    }
+    async blob() {
+        this.checkBodyUsed();
+        return new Blob([this.bodyBuffer]);
+    }
+    async cleanText() {
+        const text = await this.text();
+        return text.replace(/<[^>]*>?/gm, '');
+    }
+    read() {
+        if (this.bodyUsed)
+            return null;
+        this.bodyUsed = true;
+        const buffer = this.bodyBuffer;
+        return new ReadableStream({
+            start(controller) {
+                controller.enqueue(new Uint8Array(buffer));
+                controller.close();
+            }
+        });
+    }
+    clone() {
+        const clonedBuffer = this.bodyBuffer.slice(0);
+        const headersObj = {};
+        this.headers.forEach((value, key) => {
+            headersObj[key] = value;
+        });
+        return new WorkerResponseWrapper({
+            status: this.status,
+            statusText: this.statusText,
+            ok: this.ok,
+            url: this.url,
+            timings: this.timings,
+            body: clonedBuffer,
+            headers: headersObj,
+        });
+    }
+    async *sse() {
+        throw new Error('SSE is not supported in WorkerTransport. Use FetchTransport for SSE.');
+    }
+    async *download() {
+        const total = this.bodyBuffer.byteLength;
+        yield {
+            loaded: total,
+            transferred: total,
+            total,
+            percent: 100,
+            direction: 'download'
+        };
+    }
+    async *[Symbol.asyncIterator]() {
+        if (this.bodyUsed)
+            return;
+        this.bodyUsed = true;
+        yield new Uint8Array(this.bodyBuffer);
+    }
+}

package/dist/browser/types/index.d.ts

@@ -58,8 +58,9 @@ export interface ReckerRequest {
     maxRedirects?: number;
     followRedirects?: boolean;
     http2?: boolean;
+    useCurl?: boolean;
+    userAgent?: string;
     withHeader(name: string, value: string): ReckerRequest;
-    withBody(body: BodyInit): ReckerRequest;
     _hooks?: {
         onDnsLookup?: (info: any) => void;
         onTcpConnect?: (info: any) => void;
@@ -329,6 +330,8 @@ export interface ClientOptions {
         jitter?: boolean;
         statusCodes?: number[];
     };
+    useCurl?: boolean;
+    userAgent?: string;
 }
 export interface ReckerResponse<T = unknown> {
     status: number;

package/dist/browser/utils/binary-manager.d.ts

@@ -0,0 +1,4 @@
+export declare function getCurlBinName(): string;
+export declare function getCurlPath(): string;
+export declare function hasImpersonate(): Promise<boolean>;
+export declare function installCurlImpersonate(logger?: Console): Promise<void>;

package/dist/browser/utils/binary-manager.js

@@ -0,0 +1,72 @@
+import { promises as fs } from 'node:fs';
+import { join } from 'node:path';
+import { homedir, platform, arch } from 'node:os';
+import { spawn } from 'node:child_process';
+import { createWriteStream } from 'node:fs';
+import { Readable } from 'node:stream';
+import { finished } from 'node:stream/promises';
+const BIN_DIR = join(homedir(), '.recker', 'bin');
+const VERSION = 'v0.6.1';
+export function getCurlBinName() {
+    return 'curl-impersonate-chrome';
+}
+export function getCurlPath() {
+    return join(BIN_DIR, getCurlBinName());
+}
+export async function hasImpersonate() {
+    try {
+        await fs.access(getCurlPath());
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function getDownloadUrl() {
+    const p = platform();
+    const a = arch();
+    const baseUrl = `https://github.com/lwthiker/curl-impersonate/releases/download/${VERSION}`;
+    if (p === 'linux') {
+        if (a === 'x64')
+            return `${baseUrl}/curl-impersonate-${VERSION}.x86_64-linux-gnu.tar.gz`;
+        if (a === 'arm64')
+            return `${baseUrl}/curl-impersonate-${VERSION}.aarch64-linux-gnu.tar.gz`;
+    }
+    if (p === 'darwin') {
+        throw new Error('Auto-install not yet supported on macOS. Please install curl-impersonate manually.');
+    }
+    if (p === 'win32') {
+        throw new Error('Auto-install not yet supported on Windows.');
+    }
+    throw new Error(`Unsupported platform: ${p} ${a}`);
+}
+export async function installCurlImpersonate(logger = console) {
+    const url = getDownloadUrl();
+    const tarPath = join(BIN_DIR, 'curl-impersonate.tar.gz');
+    await fs.mkdir(BIN_DIR, { recursive: true });
+    logger.log(`Downloading curl-impersonate from ${url}...`);
+    const res = await fetch(url);
+    if (!res.ok)
+        throw new Error(`Failed to download: ${res.statusText}`);
+    if (!res.body)
+        throw new Error('Empty body');
+    const fileStream = createWriteStream(tarPath);
+    await finished(Readable.fromWeb(res.body).pipe(fileStream));
+    logger.log('Extracting...');
+    await new Promise((resolve, reject) => {
+        const tar = spawn('tar', ['-xzf', tarPath, '-C', BIN_DIR]);
+        tar.on('close', (code) => {
+            if (code === 0)
+                resolve();
+            else
+                reject(new Error(`Tar exited with code ${code}`));
+        });
+    });
+    await fs.unlink(tarPath);
+    if (await hasImpersonate()) {
+        logger.log(`✅ Installed to ${getCurlPath()}`);
+    }
+    else {
+        throw new Error('Installation failed: Binary not found after extraction');
+    }
+}

package/dist/browser/utils/user-agent.js

@@ -1,17 +1,6 @@
-import { readFileSync } from 'node:fs';
-import { join } from 'node:path';
-let RECKER_VERSION = '1.0.0';
-try {
-    const pkgPath = join(process.cwd(), 'package.json');
-    const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
-    if (pkg.name === 'recker' && pkg.version) {
-        RECKER_VERSION = pkg.version;
-    }
-}
-catch {
-}
+const VERSION = '0.0.0-dev';
 export function getDefaultUserAgent() {
-    return `recker/${RECKER_VERSION}`;
+    return `recker/${VERSION}`;
 }
 export const USER_AGENT_PRESETS = {
     chrome_windows: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',

package/dist/cache/indexed-db.d.ts

@@ -0,0 +1,10 @@
+import type { CacheStorage, CacheEntry } from '../types/index.js';
+export declare class IndexedDBStorage implements CacheStorage {
+    private dbPromise;
+    constructor(dbName?: string);
+    private openDB;
+    get(key: string): Promise<CacheEntry | undefined | null>;
+    set(key: string, value: CacheEntry, ttl: number): Promise<void>;
+    delete(key: string): Promise<void>;
+    clear(): Promise<void>;
+}

package/dist/cache/indexed-db.js

@@ -0,0 +1,88 @@
+const DB_NAME = 'recker-cache';
+const STORE_NAME = 'entries';
+const DB_VERSION = 1;
+export class IndexedDBStorage {
+    dbPromise;
+    constructor(dbName = DB_NAME) {
+        this.dbPromise = this.openDB(dbName);
+    }
+    openDB(dbName) {
+        return new Promise((resolve, reject) => {
+            if (typeof indexedDB === 'undefined') {
+                reject(new Error('IndexedDB is not supported in this environment'));
+                return;
+            }
+            const request = indexedDB.open(dbName, DB_VERSION);
+            request.onupgradeneeded = (event) => {
+                const db = event.target.result;
+                if (!db.objectStoreNames.contains(STORE_NAME)) {
+                    db.createObjectStore(STORE_NAME, { keyPath: 'key' });
+                }
+            };
+            request.onsuccess = (event) => {
+                resolve(event.target.result);
+            };
+            request.onerror = (event) => {
+                reject(event.target.error);
+            };
+        });
+    }
+    async get(key) {
+        const db = await this.dbPromise;
+        return new Promise((resolve, reject) => {
+            const transaction = db.transaction(STORE_NAME, 'readonly');
+            const store = transaction.objectStore(STORE_NAME);
+            const request = store.get(key);
+            request.onsuccess = () => {
+                const result = request.result;
+                if (!result) {
+                    resolve(null);
+                    return;
+                }
+                if (result.expiresAt && Date.now() > result.expiresAt) {
+                    this.delete(key).catch(() => { });
+                    resolve(null);
+                    return;
+                }
+                resolve(result.value);
+            };
+            request.onerror = () => reject(request.error);
+        });
+    }
+    async set(key, value, ttl) {
+        const db = await this.dbPromise;
+        return new Promise((resolve, reject) => {
+            const transaction = db.transaction(STORE_NAME, 'readwrite');
+            const store = transaction.objectStore(STORE_NAME);
+            const expiresAt = Date.now() + ttl;
+            const item = {
+                key,
+                value,
+                expiresAt
+            };
+            const request = store.put(item);
+            request.onsuccess = () => resolve();
+            request.onerror = () => reject(request.error);
+        });
+    }
+    async delete(key) {
+        const db = await this.dbPromise;
+        return new Promise((resolve, reject) => {
+            const transaction = db.transaction(STORE_NAME, 'readwrite');
+            const store = transaction.objectStore(STORE_NAME);
+            const request = store.delete(key);
+            request.onsuccess = () => resolve();
+            request.onerror = () => reject(request.error);
+        });
+    }
+    async clear() {
+        const db = await this.dbPromise;
+        return new Promise((resolve, reject) => {
+            const transaction = db.transaction(STORE_NAME, 'readwrite');
+            const store = transaction.objectStore(STORE_NAME);
+            const request = store.clear();
+            request.onsuccess = () => resolve();
+            request.onerror = () => reject(request.error);
+        });
+    }
+}

package/dist/cache/service-worker-cache.d.ts

@@ -0,0 +1,18 @@
+import type { CacheStorage, CacheEntry } from '../types/index.js';
+export declare class ServiceWorkerCache implements CacheStorage {
+    private cacheName;
+    private cachePromise;
+    constructor(options?: {
+        cacheName?: string;
+    });
+    static isSupported(): boolean;
+    private getCache;
+    private keyToUrl;
+    get(key: string): Promise<CacheEntry | undefined | null>;
+    set(key: string, value: CacheEntry, ttl: number): Promise<void>;
+    delete(key: string): Promise<void>;
+    clear(): Promise<void>;
+    keys(): Promise<string[]>;
+    size(): Promise<number>;
+    prune(): Promise<number>;
+}