recker 1.0.43 → 1.0.44

package/dist/transport/curl.js

@@ -0,0 +1,101 @@
+import { spawn } from 'node:child_process';
+import { HttpResponse } from '../core/response.js';
+import { NetworkError } from '../core/errors.js';
+import { getCurlPath, hasImpersonate } from '../utils/binary-manager.js';
+export class CurlTransport {
+    async dispatch(req) {
+        return new Promise(async (resolve, reject) => {
+            const args = [
+                '-X', req.method,
+                req.url,
+                '-i',
+                '-s',
+                '--compressed',
+                '--no-keepalive'
+            ];
+            req.headers.forEach((val, key) => {
+                args.push('-H', `${key}: ${val}`);
+            });
+            if (req.body && typeof req.body === 'string') {
+                args.push('-d', req.body);
+            }
+            let command = process.env.RECKER_CURL_BIN;
+            if (!command) {
+                if (await hasImpersonate()) {
+                    command = getCurlPath();
+                }
+                else {
+                    command = 'curl';
+                }
+            }
+            const child = spawn(command, args);
+            const stdoutChunks = [];
+            const stderrChunks = [];
+            child.stdout.on('data', (chunk) => stdoutChunks.push(chunk));
+            child.stderr.on('data', (chunk) => stderrChunks.push(chunk));
+            child.on('error', (err) => {
+                reject(new NetworkError(`Failed to spawn curl: ${err.message}`, 'ERR_CURL_SPAWN', req));
+            });
+            child.on('close', (code) => {
+                if (code !== 0) {
+                    const stderr = Buffer.concat(stderrChunks).toString();
+                    reject(new NetworkError(`Curl exited with code ${code}: ${stderr}`, 'ERR_CURL_EXIT', req));
+                    return;
+                }
+                const fullOutput = Buffer.concat(stdoutChunks);
+                let headerEndIndex = fullOutput.indexOf('\r\n\r\n');
+                let offset = 4;
+                if (headerEndIndex === -1) {
+                    headerEndIndex = fullOutput.indexOf('\n\n');
+                    offset = 2;
+                }
+                if (headerEndIndex !== -1) {
+                    const firstLine = fullOutput.subarray(0, Math.min(20, headerEndIndex)).toString();
+                    if (firstLine.startsWith('HTTP/1.1 100') || firstLine.startsWith('HTTP/2 100')) {
+                        const nextStart = headerEndIndex + offset;
+                        const secondSplit = fullOutput.indexOf('\r\n\r\n', nextStart);
+                        if (secondSplit !== -1) {
+                            headerEndIndex = secondSplit;
+                            offset = 4;
+                        }
+                    }
+                }
+                if (headerEndIndex === -1) {
+                    const nativeResponse = new Response(fullOutput, { status: 200, statusText: 'OK' });
+                    resolve(new HttpResponse(nativeResponse, { connection: { protocol: 'curl' } }));
+                    return;
+                }
+                const headerBlock = fullOutput.subarray(0, headerEndIndex).toString();
+                const bodyBlock = fullOutput.subarray(headerEndIndex + offset);
+                const headerLines = headerBlock.split(/\r?\n/);
+                const statusLine = headerLines[0];
+                let status = 200;
+                let statusText = 'OK';
+                const statusMatch = statusLine.match(/HTTP\/[\d\.]+ (\d+) ?(.*)/);
+                if (statusMatch) {
+                    status = parseInt(statusMatch[1], 10);
+                    statusText = statusMatch[2] || '';
+                }
+                const headers = new Headers();
+                for (let i = 1; i < headerLines.length; i++) {
+                    const line = headerLines[i];
+                    const colon = line.indexOf(':');
+                    if (colon > 0) {
+                        const key = line.substring(0, colon).trim();
+                        const val = line.substring(colon + 1).trim();
+                        headers.append(key, val);
+                    }
+                }
+                const nativeResponse = new Response(bodyBlock, {
+                    status,
+                    statusText,
+                    headers
+                });
+                resolve(new HttpResponse(nativeResponse, {
+                    timings: {},
+                    connection: { protocol: 'curl' }
+                }));
+            });
+        });
+    }
+}

package/dist/transport/udp.js

@@ -209,7 +209,6 @@ export const udp = {
                 headers: new Headers(),
                 body: new Uint8Array(data),
                 withHeader: () => ({ url: address }),
-                withBody: () => ({ url: address }),
             });
             return response;
         }

package/dist/transport/undici.js

@@ -852,8 +852,7 @@ function wrapDownloadResponse(response, onProgress) {
     });
     return new Response(webBody, {
         status: response.statusCode,
-        statusText: String(response.statusCode),
-        headers: response.headers
+        headers: response.headers,
     });
 }
 function wrapUploadBody(body, onProgress, total) {

package/dist/transport/worker.d.ts

@@ -0,0 +1,18 @@
+import type { ReckerRequest, ReckerResponse, Transport } from '../types/index.js';
+export interface WorkerTransportOptions {
+    poolSize?: number;
+}
+export declare class WorkerTransport implements Transport {
+    private options;
+    private workers;
+    private workerIndex;
+    private pendingRequests;
+    private workerUrl;
+    constructor(options?: WorkerTransportOptions);
+    static isSupported(): boolean;
+    private createWorker;
+    private getNextWorker;
+    dispatch(req: ReckerRequest): Promise<ReckerResponse>;
+    terminate(): void;
+    get pendingCount(): number;
+}

package/dist/transport/worker.js

@@ -0,0 +1,278 @@
+const WORKER_SCRIPT = `
+self.onmessage = async (event) => {
+  const { id, method, url, headers, body, timeout } = event.data;
+
+  try {
+    const controller = new AbortController();
+    let timeoutId;
+
+    if (timeout) {
+      timeoutId = setTimeout(() => controller.abort(), timeout);
+    }
+
+    const start = performance.now();
+
+    const response = await fetch(url, {
+      method,
+      headers,
+      body,
+      signal: controller.signal,
+      keepalive: true
+    });
+
+    if (timeoutId) clearTimeout(timeoutId);
+
+    const totalTime = performance.now() - start;
+
+    // Read response body as ArrayBuffer for transferability
+    const arrayBuffer = await response.arrayBuffer();
+
+    // Serialize headers
+    const responseHeaders = {};
+    response.headers.forEach((value, key) => {
+      responseHeaders[key] = value;
+    });
+
+    self.postMessage({
+      id,
+      success: true,
+      status: response.status,
+      statusText: response.statusText,
+      headers: responseHeaders,
+      ok: response.ok,
+      url: response.url,
+      body: arrayBuffer,
+      timings: { total: totalTime, firstByte: totalTime }
+    }, [arrayBuffer]);
+
+  } catch (error) {
+    self.postMessage({
+      id,
+      success: false,
+      error: {
+        name: error.name,
+        message: error.message
+      }
+    });
+  }
+};
+`;
+export class WorkerTransport {
+    options;
+    workers = [];
+    workerIndex = 0;
+    pendingRequests = new Map();
+    workerUrl;
+    constructor(options = {}) {
+        this.options = options;
+        const poolSize = options.poolSize ?? (typeof navigator !== 'undefined' ? navigator.hardwareConcurrency : 4) ?? 4;
+        const blob = new Blob([WORKER_SCRIPT], { type: 'application/javascript' });
+        this.workerUrl = URL.createObjectURL(blob);
+        for (let i = 0; i < poolSize; i++) {
+            this.createWorker();
+        }
+    }
+    static isSupported() {
+        return typeof Worker !== 'undefined';
+    }
+    createWorker() {
+        const worker = new Worker(this.workerUrl);
+        worker.onmessage = (event) => {
+            const { id, success, error, ...response } = event.data;
+            const pending = this.pendingRequests.get(id);
+            if (!pending)
+                return;
+            this.pendingRequests.delete(id);
+            if (success) {
+                pending.resolve(new WorkerResponseWrapper(response));
+            }
+            else {
+                const err = new Error(error?.message || 'Worker request failed');
+                err.name = error?.name || 'WorkerError';
+                pending.reject(err);
+            }
+        };
+        worker.onerror = (event) => {
+            console.error('[WorkerTransport] Worker error:', event.message);
+        };
+        this.workers.push(worker);
+    }
+    getNextWorker() {
+        const worker = this.workers[this.workerIndex];
+        this.workerIndex = (this.workerIndex + 1) % this.workers.length;
+        return worker;
+    }
+    async dispatch(req) {
+        if (!WorkerTransport.isSupported()) {
+            throw new Error('Web Workers are not supported in this environment');
+        }
+        const id = crypto.randomUUID();
+        const worker = this.getNextWorker();
+        const timeoutMs = typeof req.timeout === 'number'
+            ? req.timeout
+            : req.timeout?.request;
+        const headers = {};
+        if (req.headers) {
+            if (req.headers instanceof Headers) {
+                req.headers.forEach((value, key) => {
+                    headers[key] = value;
+                });
+            }
+            else if (typeof req.headers === 'object') {
+                Object.assign(headers, req.headers);
+            }
+        }
+        let body;
+        if (req.body) {
+            if (typeof req.body === 'string') {
+                body = req.body;
+            }
+            else if (req.body instanceof ArrayBuffer) {
+                body = req.body;
+            }
+            else if (req.body instanceof Blob) {
+                body = await req.body.arrayBuffer();
+            }
+            else if (typeof req.body === 'object') {
+                body = JSON.stringify(req.body);
+            }
+        }
+        return new Promise((resolve, reject) => {
+            this.pendingRequests.set(id, { resolve, reject });
+            if (req.signal) {
+                req.signal.addEventListener('abort', () => {
+                    const pending = this.pendingRequests.get(id);
+                    if (pending) {
+                        this.pendingRequests.delete(id);
+                        const err = new Error('Request aborted');
+                        err.name = 'AbortError';
+                        pending.reject(err);
+                    }
+                });
+            }
+            worker.postMessage({
+                id,
+                method: req.method,
+                url: req.url,
+                headers,
+                body,
+                timeout: timeoutMs
+            });
+        });
+    }
+    terminate() {
+        for (const worker of this.workers) {
+            worker.terminate();
+        }
+        this.workers = [];
+        URL.revokeObjectURL(this.workerUrl);
+        for (const [id, pending] of this.pendingRequests) {
+            pending.reject(new Error('Transport terminated'));
+        }
+        this.pendingRequests.clear();
+    }
+    get pendingCount() {
+        return this.pendingRequests.size;
+    }
+}
+class WorkerResponseWrapper {
+    status;
+    statusText;
+    headers;
+    ok;
+    url;
+    timings;
+    connection = {};
+    raw;
+    bodyBuffer;
+    bodyUsed = false;
+    constructor(data) {
+        this.status = data.status ?? 0;
+        this.statusText = data.statusText ?? '';
+        this.ok = data.ok ?? false;
+        this.url = data.url ?? '';
+        this.timings = data.timings ?? { total: 0 };
+        this.bodyBuffer = data.body ?? new ArrayBuffer(0);
+        this.headers = new Headers();
+        if (data.headers) {
+            for (const [key, value] of Object.entries(data.headers)) {
+                this.headers.set(key, value);
+            }
+        }
+        this.raw = new Response(this.bodyBuffer, {
+            status: this.status,
+            statusText: this.statusText,
+            headers: this.headers,
+        });
+    }
+    checkBodyUsed() {
+        if (this.bodyUsed) {
+            throw new Error('Body has already been consumed');
+        }
+        this.bodyUsed = true;
+    }
+    async json() {
+        this.checkBodyUsed();
+        const text = new TextDecoder().decode(this.bodyBuffer);
+        return JSON.parse(text);
+    }
+    async text() {
+        this.checkBodyUsed();
+        return new TextDecoder().decode(this.bodyBuffer);
+    }
+    async blob() {
+        this.checkBodyUsed();
+        return new Blob([this.bodyBuffer]);
+    }
+    async cleanText() {
+        const text = await this.text();
+        return text.replace(/<[^>]*>?/gm, '');
+    }
+    read() {
+        if (this.bodyUsed)
+            return null;
+        this.bodyUsed = true;
+        const buffer = this.bodyBuffer;
+        return new ReadableStream({
+            start(controller) {
+                controller.enqueue(new Uint8Array(buffer));
+                controller.close();
+            }
+        });
+    }
+    clone() {
+        const clonedBuffer = this.bodyBuffer.slice(0);
+        const headersObj = {};
+        this.headers.forEach((value, key) => {
+            headersObj[key] = value;
+        });
+        return new WorkerResponseWrapper({
+            status: this.status,
+            statusText: this.statusText,
+            ok: this.ok,
+            url: this.url,
+            timings: this.timings,
+            body: clonedBuffer,
+            headers: headersObj,
+        });
+    }
+    async *sse() {
+        throw new Error('SSE is not supported in WorkerTransport. Use FetchTransport for SSE.');
+    }
+    async *download() {
+        const total = this.bodyBuffer.byteLength;
+        yield {
+            loaded: total,
+            transferred: total,
+            total,
+            percent: 100,
+            direction: 'download'
+        };
+    }
+    async *[Symbol.asyncIterator]() {
+        if (this.bodyUsed)
+            return;
+        this.bodyUsed = true;
+        yield new Uint8Array(this.bodyBuffer);
+    }
+}

package/dist/types/index.d.ts

@@ -58,8 +58,9 @@ export interface ReckerRequest {
     maxRedirects?: number;
     followRedirects?: boolean;
     http2?: boolean;
+    useCurl?: boolean;
+    userAgent?: string;
     withHeader(name: string, value: string): ReckerRequest;
-    withBody(body: BodyInit): ReckerRequest;
     _hooks?: {
         onDnsLookup?: (info: any) => void;
         onTcpConnect?: (info: any) => void;
@@ -329,6 +330,8 @@ export interface ClientOptions {
         jitter?: boolean;
         statusCodes?: number[];
     };
+    useCurl?: boolean;
+    userAgent?: string;
 }
 export interface ReckerResponse<T = unknown> {
     status: number;

package/dist/utils/binary-manager.d.ts

@@ -0,0 +1,4 @@
+export declare function getCurlBinName(): string;
+export declare function getCurlPath(): string;
+export declare function hasImpersonate(): Promise<boolean>;
+export declare function installCurlImpersonate(logger?: Console): Promise<void>;

package/dist/utils/binary-manager.js

@@ -0,0 +1,72 @@
+import { promises as fs } from 'node:fs';
+import { join } from 'node:path';
+import { homedir, platform, arch } from 'node:os';
+import { spawn } from 'node:child_process';
+import { createWriteStream } from 'node:fs';
+import { Readable } from 'node:stream';
+import { finished } from 'node:stream/promises';
+const BIN_DIR = join(homedir(), '.recker', 'bin');
+const VERSION = 'v0.6.1';
+export function getCurlBinName() {
+    return 'curl-impersonate-chrome';
+}
+export function getCurlPath() {
+    return join(BIN_DIR, getCurlBinName());
+}
+export async function hasImpersonate() {
+    try {
+        await fs.access(getCurlPath());
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function getDownloadUrl() {
+    const p = platform();
+    const a = arch();
+    const baseUrl = `https://github.com/lwthiker/curl-impersonate/releases/download/${VERSION}`;
+    if (p === 'linux') {
+        if (a === 'x64')
+            return `${baseUrl}/curl-impersonate-${VERSION}.x86_64-linux-gnu.tar.gz`;
+        if (a === 'arm64')
+            return `${baseUrl}/curl-impersonate-${VERSION}.aarch64-linux-gnu.tar.gz`;
+    }
+    if (p === 'darwin') {
+        throw new Error('Auto-install not yet supported on macOS. Please install curl-impersonate manually.');
+    }
+    if (p === 'win32') {
+        throw new Error('Auto-install not yet supported on Windows.');
+    }
+    throw new Error(`Unsupported platform: ${p} ${a}`);
+}
+export async function installCurlImpersonate(logger = console) {
+    const url = getDownloadUrl();
+    const tarPath = join(BIN_DIR, 'curl-impersonate.tar.gz');
+    await fs.mkdir(BIN_DIR, { recursive: true });
+    logger.log(`Downloading curl-impersonate from ${url}...`);
+    const res = await fetch(url);
+    if (!res.ok)
+        throw new Error(`Failed to download: ${res.statusText}`);
+    if (!res.body)
+        throw new Error('Empty body');
+    const fileStream = createWriteStream(tarPath);
+    await finished(Readable.fromWeb(res.body).pipe(fileStream));
+    logger.log('Extracting...');
+    await new Promise((resolve, reject) => {
+        const tar = spawn('tar', ['-xzf', tarPath, '-C', BIN_DIR]);
+        tar.on('close', (code) => {
+            if (code === 0)
+                resolve();
+            else
+                reject(new Error(`Tar exited with code ${code}`));
+        });
+    });
+    await fs.unlink(tarPath);
+    if (await hasImpersonate()) {
+        logger.log(`✅ Installed to ${getCurlPath()}`);
+    }
+    else {
+        throw new Error('Installation failed: Binary not found after extraction');
+    }
+}

package/dist/utils/optional-require.d.ts

@@ -1,10 +1,10 @@
-interface DependencyInfo {
-    package: string;
-    submodule: string;
-    version?: string;
-    feature?: string;
-}
-export declare const OPTIONAL_DEPENDENCIES: Record<string, DependencyInfo>;
+export declare const OPTIONAL_DEPENDENCIES: {
+    readonly cardinal: {
+        readonly package: "cardinal";
+        readonly feature: "syntax highlighting in CLI";
+        readonly version: "2.x";
+    };
+};
 export declare class MissingDependencyError extends Error {
     readonly packageName: string;
     readonly submodule: string;
@@ -16,4 +16,3 @@ export declare function requireOptional<T = unknown>(packageName: string, submod
 export declare function isPackageAvailable(packageName: string): boolean;
 export declare function getInstallCommand(packages: string[]): string;
 export declare function requireOptionalMany<T extends Record<string, unknown>>(packages: string[], submodule: string): Promise<T>;
-export {};

package/dist/utils/optional-require.js

@@ -1,27 +1,8 @@
 export const OPTIONAL_DEPENDENCIES = {
     'cardinal': {
         package: 'cardinal',
-        submodule: 'recker/cli',
-        version: '^2.0.0',
-        feature: 'syntax highlighting',
-    },
-    'ssh2-sftp-client': {
-        package: 'ssh2-sftp-client',
-        submodule: 'recker/protocols/sftp',
-        version: '^11.0.0',
-        feature: 'SFTP client',
-    },
-    'cheerio': {
-        package: 'cheerio',
-        submodule: 'recker/scrape',
-        version: '^1.0.0',
-        feature: 'HTML parsing and scraping',
-    },
-    'ioredis': {
-        package: 'ioredis',
-        submodule: 'recker/cache',
-        version: '^5.0.0',
-        feature: 'Redis cache storage',
+        feature: 'syntax highlighting in CLI',
+        version: '2.x',
     },
 };
 export class MissingDependencyError extends Error {

package/dist/utils/upload.d.ts

@@ -1,4 +1,10 @@
 import { Readable } from 'node:stream';
+import type { Client } from '../core/client.js';
+export interface FileUploadOptions {
+    fieldName?: string;
+    headers?: Record<string, string>;
+}
+export declare function uploadFile(client: Client, url: string, filePath: string, options?: FileUploadOptions): Promise<import("../types/index.js").ReckerResponse<unknown>>;
 export interface UploadOptions {
     file: Readable | Buffer;
     chunkSize?: number;

package/dist/utils/upload.js

@@ -1,3 +1,14 @@
+import { openAsBlob } from 'node:fs';
+export async function uploadFile(client, url, filePath, options = {}) {
+    const blob = await openAsBlob(filePath);
+    const fieldName = options.fieldName || 'file';
+    return client.post(url, {
+        headers: options.headers,
+        form: {
+            [fieldName]: blob
+        }
+    });
+}
 export async function uploadParallel(options) {
     const chunkSize = options.chunkSize || 5 * 1024 * 1024;
     const concurrency = options.concurrency || 3;

package/dist/utils/user-agent.js

@@ -1,17 +1,6 @@
-import { readFileSync } from 'node:fs';
-import { join } from 'node:path';
-let RECKER_VERSION = '1.0.0';
-try {
-    const pkgPath = join(process.cwd(), 'package.json');
-    const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
-    if (pkg.name === 'recker' && pkg.version) {
-        RECKER_VERSION = pkg.version;
-    }
-}
-catch {
-}
+const VERSION = '0.0.0-dev';
 export function getDefaultUserAgent() {
-    return `recker/${RECKER_VERSION}`;
+    return `recker/${VERSION}`;
 }
 export const USER_AGENT_PRESETS = {
     chrome_windows: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',

package/dist/version.js

@@ -1,4 +1,4 @@
-const VERSION = '1.0.43';
+const VERSION = '1.0.44';
 let _version = null;
 export async function getVersion() {
     if (_version)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.43",
+  "version": "1.0.44",
   "description": "AI & DevX focused HTTP client for Node.js 18+",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -38,7 +38,6 @@
     "claude",
     "streaming",
     "scraping",
-    "cheerio",
     "html-parser"
   ],
   "author": "filipeforattini",
@@ -104,6 +103,14 @@
       "types": "./dist/scrape/index.d.ts",
       "import": "./dist/scrape/index.js"
     },
+    "./parser": {
+      "types": "./dist/scrape/parser/index.d.ts",
+      "import": "./dist/scrape/parser/index.js"
+    },
+    "./seo": {
+      "types": "./dist/seo/index.d.ts",
+      "import": "./dist/seo/index.js"
+    },
     "./utils/*": {
       "types": "./dist/utils/*.d.ts",
       "import": "./dist/utils/*.js"
@@ -153,9 +160,9 @@
   },
   "dependencies": {
     "@maxmind/geoip2-node": "^6.3.4",
-    "cheerio": "^1.0.0",
-    "commander": "^14.0.2",
+    "css-select": "^6.0.0",
     "fuse.js": "^7.1.0",
+    "he": "^1.2.0",
     "undici": "^7.16.0",
     "zod": "^4.1.13"
   },
@@ -177,6 +184,7 @@
   },
   "devDependencies": {
     "@hapi/wreck": "^18.1.0",
+    "@types/he": "^1.2.3",
     "@types/needle": "^3.3.0",
     "@types/node": "^24.10.1",
     "@types/ssh2-sftp-client": "^9.0.5",
@@ -185,8 +193,6 @@
     "@vitest/coverage-v8": "^3.2.4",
     "axios": "^1.13.2",
     "cardinal": "^2.1.1",
-    "cheerio": "^1.0.0",
-    "commander": "^14.0.0",
     "cross-fetch": "^4.1.0",
     "domhandler": "^5.0.3",
     "esbuild": "^0.24.2",