recker 1.0.43 → 1.0.44-next.084c2ef

package/dist/browser/seo/types.d.ts

@@ -0,0 +1,211 @@
+import { KeywordCloud } from './keywords.js';
+export type { KeywordCloud, KeywordItem } from './keywords.js';
+export type SeoStatus = 'pass' | 'warn' | 'fail' | 'info';
+export interface SeoCheckEvidence {
+    found?: string | number | string[];
+    expected?: string | number | string[];
+    location?: string;
+    issue?: string;
+    impact?: string;
+    example?: string;
+    learnMore?: string;
+}
+export interface SeoCheckResult {
+    name: string;
+    category: string;
+    status: SeoStatus;
+    message: string;
+    value?: string | number;
+    recommendation?: string;
+    evidence?: SeoCheckEvidence;
+}
+export interface HeadingInfo {
+    level: number;
+    text: string;
+    count: number;
+}
+export interface HeadingAnalysis {
+    structure: HeadingInfo[];
+    h1Count: number;
+    hasProperHierarchy: boolean;
+    issues: string[];
+}
+export interface ContentMetrics {
+    wordCount: number;
+    characterCount: number;
+    sentenceCount: number;
+    paragraphCount: number;
+    readingTimeMinutes: number;
+    avgWordsPerSentence: number;
+    avgParagraphLength: number;
+    listCount: number;
+    strongTagCount: number;
+    emTagCount: number;
+    fleschReadingEase?: number;
+    hasQuestionHeadings?: boolean;
+}
+export interface LinkAnalysis {
+    total: number;
+    internal: number;
+    external: number;
+    nofollow: number;
+    broken: number;
+    withoutText: number;
+    sponsoredLinks: number;
+    ugcLinks: number;
+    internalHttpLinks?: number;
+    internalHttpLinkUrls?: string[];
+}
+export interface ImageAnalysis {
+    total: number;
+    withAlt: number;
+    withoutAlt: number;
+    lazy: number;
+    missingDimensions: number;
+    modernFormats: number;
+    altTextLengths: number[];
+    imageAltTexts: string[];
+    imageFilenames: string[];
+    imagesWithAsyncDecoding: number;
+}
+export interface SocialMetaAnalysis {
+    openGraph: {
+        present: boolean;
+        hasTitle: boolean;
+        hasDescription: boolean;
+        hasImage: boolean;
+        hasUrl: boolean;
+        issues: string[];
+    };
+    twitterCard: {
+        present: boolean;
+        hasCard: boolean;
+        hasTitle: boolean;
+        hasDescription: boolean;
+        hasImage: boolean;
+        issues: string[];
+    };
+}
+export interface TechnicalSeo {
+    hasCanonical: boolean;
+    canonicalUrl?: string;
+    hasRobotsMeta: boolean;
+    robotsContent?: string[];
+    hasViewport: boolean;
+    hasCharset: boolean;
+    hasLang: boolean;
+    langValue?: string;
+}
+export interface SeoTiming {
+    ttfb?: number;
+    total?: number;
+    dns?: number;
+    tcp?: number;
+    tls?: number;
+    download?: number;
+}
+export interface SeoSummary {
+    totalChecks: number;
+    passed: number;
+    warnings: number;
+    errors: number;
+    infos: number;
+    passRate: number;
+    issuesByCategory: Record<string, {
+        passed: number;
+        warnings: number;
+        errors: number;
+    }>;
+    topIssues: Array<{
+        name: string;
+        message: string;
+        category: string;
+        severity: 'error' | 'warning';
+    }>;
+    quickWins: string[];
+    vitals: {
+        htmlSize?: number;
+        domElements?: number;
+        ttfb?: number;
+        totalTime?: number;
+        wordCount: number;
+        readingTime: number;
+        imageCount: number;
+        linkCount: number;
+    };
+    completeness: {
+        meta: number;
+        social: number;
+        technical: number;
+        content: number;
+        images: number;
+        links: number;
+    };
+}
+export interface SeoReport {
+    url: string;
+    timestamp: Date;
+    grade: string;
+    score: number;
+    summary: SeoSummary;
+    timing?: SeoTiming;
+    checks: SeoCheckResult[];
+    title?: {
+        text: string;
+        length: number;
+    };
+    metaDescription?: {
+        text: string;
+        length: number;
+    };
+    openGraph?: {
+        title?: string;
+        description?: string;
+        image?: string;
+        url?: string;
+        type?: string;
+        siteName?: string;
+    };
+    twitterCard?: {
+        card?: string;
+        title?: string;
+        description?: string;
+        image?: string;
+        site?: string;
+    };
+    structuredData: {
+        count: number;
+        types: string[];
+        items: Record<string, unknown>[];
+    };
+    content: ContentMetrics;
+    headings: HeadingAnalysis;
+    keywords: KeywordCloud;
+    links: LinkAnalysis;
+    images: ImageAnalysis;
+    social: SocialMetaAnalysis;
+    technical: TechnicalSeo;
+}
+export interface SeoAnalyzerOptions {
+    baseUrl?: string;
+    analyzeContent?: boolean;
+    checkBrokenLinks?: boolean;
+    responseHeaders?: Record<string, string | string[]>;
+}
+export interface ExtractedLink {
+    href: string;
+    text: string;
+    rel?: string;
+    target?: string;
+    title?: string;
+    type?: 'internal' | 'external' | 'anchor' | 'mailto' | 'tel';
+}
+export interface ExtractedImage {
+    src: string;
+    alt?: string;
+    title?: string;
+    width?: number;
+    height?: number;
+    srcset?: string;
+    loading?: 'lazy' | 'eager';
+}

package/dist/browser/seo/types.js

@@ -0,0 +1 @@
+export {};

package/dist/browser/transport/curl.d.ts

@@ -0,0 +1,4 @@
+import { Transport, ReckerRequest, ReckerResponse } from '../types/index.js';
+export declare class CurlTransport implements Transport {
+    dispatch(req: ReckerRequest): Promise<ReckerResponse>;
+}

package/dist/browser/transport/curl.js

@@ -0,0 +1,101 @@
+import { spawn } from 'node:child_process';
+import { HttpResponse } from '../core/response.js';
+import { NetworkError } from '../core/errors.js';
+import { getCurlPath, hasImpersonate } from '../utils/binary-manager.js';
+export class CurlTransport {
+    async dispatch(req) {
+        return new Promise(async (resolve, reject) => {
+            const args = [
+                '-X', req.method,
+                req.url,
+                '-i',
+                '-s',
+                '--compressed',
+                '--no-keepalive'
+            ];
+            req.headers.forEach((val, key) => {
+                args.push('-H', `${key}: ${val}`);
+            });
+            if (req.body && typeof req.body === 'string') {
+                args.push('-d', req.body);
+            }
+            let command = process.env.RECKER_CURL_BIN;
+            if (!command) {
+                if (await hasImpersonate()) {
+                    command = getCurlPath();
+                }
+                else {
+                    command = 'curl';
+                }
+            }
+            const child = spawn(command, args);
+            const stdoutChunks = [];
+            const stderrChunks = [];
+            child.stdout.on('data', (chunk) => stdoutChunks.push(chunk));
+            child.stderr.on('data', (chunk) => stderrChunks.push(chunk));
+            child.on('error', (err) => {
+                reject(new NetworkError(`Failed to spawn curl: ${err.message}`, 'ERR_CURL_SPAWN', req));
+            });
+            child.on('close', (code) => {
+                if (code !== 0) {
+                    const stderr = Buffer.concat(stderrChunks).toString();
+                    reject(new NetworkError(`Curl exited with code ${code}: ${stderr}`, 'ERR_CURL_EXIT', req));
+                    return;
+                }
+                const fullOutput = Buffer.concat(stdoutChunks);
+                let headerEndIndex = fullOutput.indexOf('\r\n\r\n');
+                let offset = 4;
+                if (headerEndIndex === -1) {
+                    headerEndIndex = fullOutput.indexOf('\n\n');
+                    offset = 2;
+                }
+                if (headerEndIndex !== -1) {
+                    const firstLine = fullOutput.subarray(0, Math.min(20, headerEndIndex)).toString();
+                    if (firstLine.startsWith('HTTP/1.1 100') || firstLine.startsWith('HTTP/2 100')) {
+                        const nextStart = headerEndIndex + offset;
+                        const secondSplit = fullOutput.indexOf('\r\n\r\n', nextStart);
+                        if (secondSplit !== -1) {
+                            headerEndIndex = secondSplit;
+                            offset = 4;
+                        }
+                    }
+                }
+                if (headerEndIndex === -1) {
+                    const nativeResponse = new Response(fullOutput, { status: 200, statusText: 'OK' });
+                    resolve(new HttpResponse(nativeResponse, { connection: { protocol: 'curl' } }));
+                    return;
+                }
+                const headerBlock = fullOutput.subarray(0, headerEndIndex).toString();
+                const bodyBlock = fullOutput.subarray(headerEndIndex + offset);
+                const headerLines = headerBlock.split(/\r?\n/);
+                const statusLine = headerLines[0];
+                let status = 200;
+                let statusText = 'OK';
+                const statusMatch = statusLine.match(/HTTP\/[\d\.]+ (\d+) ?(.*)/);
+                if (statusMatch) {
+                    status = parseInt(statusMatch[1], 10);
+                    statusText = statusMatch[2] || '';
+                }
+                const headers = new Headers();
+                for (let i = 1; i < headerLines.length; i++) {
+                    const line = headerLines[i];
+                    const colon = line.indexOf(':');
+                    if (colon > 0) {
+                        const key = line.substring(0, colon).trim();
+                        const val = line.substring(colon + 1).trim();
+                        headers.append(key, val);
+                    }
+                }
+                const nativeResponse = new Response(bodyBlock, {
+                    status,
+                    statusText,
+                    headers
+                });
+                resolve(new HttpResponse(nativeResponse, {
+                    timings: {},
+                    connection: { protocol: 'curl' }
+                }));
+            });
+        });
+    }
+}

package/dist/browser/transport/undici.js

@@ -852,8 +852,7 @@ function wrapDownloadResponse(response, onProgress) {
     });
     return new Response(webBody, {
         status: response.statusCode,
-        statusText: String(response.statusCode),
-        headers: response.headers
+        headers: response.headers,
     });
 }
 function wrapUploadBody(body, onProgress, total) {

package/dist/browser/transport/worker.d.ts

@@ -0,0 +1,18 @@
+import type { ReckerRequest, ReckerResponse, Transport } from '../types/index.js';
+export interface WorkerTransportOptions {
+    poolSize?: number;
+}
+export declare class WorkerTransport implements Transport {
+    private options;
+    private workers;
+    private workerIndex;
+    private pendingRequests;
+    private workerUrl;
+    constructor(options?: WorkerTransportOptions);
+    static isSupported(): boolean;
+    private createWorker;
+    private getNextWorker;
+    dispatch(req: ReckerRequest): Promise<ReckerResponse>;
+    terminate(): void;
+    get pendingCount(): number;
+}

package/dist/browser/transport/worker.js

@@ -0,0 +1,278 @@
+const WORKER_SCRIPT = `
+self.onmessage = async (event) => {
+  const { id, method, url, headers, body, timeout } = event.data;
+
+  try {
+    const controller = new AbortController();
+    let timeoutId;
+
+    if (timeout) {
+      timeoutId = setTimeout(() => controller.abort(), timeout);
+    }
+
+    const start = performance.now();
+
+    const response = await fetch(url, {
+      method,
+      headers,
+      body,
+      signal: controller.signal,
+      keepalive: true
+    });
+
+    if (timeoutId) clearTimeout(timeoutId);
+
+    const totalTime = performance.now() - start;
+
+    // Read response body as ArrayBuffer for transferability
+    const arrayBuffer = await response.arrayBuffer();
+
+    // Serialize headers
+    const responseHeaders = {};
+    response.headers.forEach((value, key) => {
+      responseHeaders[key] = value;
+    });
+
+    self.postMessage({
+      id,
+      success: true,
+      status: response.status,
+      statusText: response.statusText,
+      headers: responseHeaders,
+      ok: response.ok,
+      url: response.url,
+      body: arrayBuffer,
+      timings: { total: totalTime, firstByte: totalTime }
+    }, [arrayBuffer]);
+
+  } catch (error) {
+    self.postMessage({
+      id,
+      success: false,
+      error: {
+        name: error.name,
+        message: error.message
+      }
+    });
+  }
+};
+`;
+export class WorkerTransport {
+    options;
+    workers = [];
+    workerIndex = 0;
+    pendingRequests = new Map();
+    workerUrl;
+    constructor(options = {}) {
+        this.options = options;
+        const poolSize = options.poolSize ?? (typeof navigator !== 'undefined' ? navigator.hardwareConcurrency : 4) ?? 4;
+        const blob = new Blob([WORKER_SCRIPT], { type: 'application/javascript' });
+        this.workerUrl = URL.createObjectURL(blob);
+        for (let i = 0; i < poolSize; i++) {
+            this.createWorker();
+        }
+    }
+    static isSupported() {
+        return typeof Worker !== 'undefined';
+    }
+    createWorker() {
+        const worker = new Worker(this.workerUrl);
+        worker.onmessage = (event) => {
+            const { id, success, error, ...response } = event.data;
+            const pending = this.pendingRequests.get(id);
+            if (!pending)
+                return;
+            this.pendingRequests.delete(id);
+            if (success) {
+                pending.resolve(new WorkerResponseWrapper(response));
+            }
+            else {
+                const err = new Error(error?.message || 'Worker request failed');
+                err.name = error?.name || 'WorkerError';
+                pending.reject(err);
+            }
+        };
+        worker.onerror = (event) => {
+            console.error('[WorkerTransport] Worker error:', event.message);
+        };
+        this.workers.push(worker);
+    }
+    getNextWorker() {
+        const worker = this.workers[this.workerIndex];
+        this.workerIndex = (this.workerIndex + 1) % this.workers.length;
+        return worker;
+    }
+    async dispatch(req) {
+        if (!WorkerTransport.isSupported()) {
+            throw new Error('Web Workers are not supported in this environment');
+        }
+        const id = crypto.randomUUID();
+        const worker = this.getNextWorker();
+        const timeoutMs = typeof req.timeout === 'number'
+            ? req.timeout
+            : req.timeout?.request;
+        const headers = {};
+        if (req.headers) {
+            if (req.headers instanceof Headers) {
+                req.headers.forEach((value, key) => {
+                    headers[key] = value;
+                });
+            }
+            else if (typeof req.headers === 'object') {
+                Object.assign(headers, req.headers);
+            }
+        }
+        let body;
+        if (req.body) {
+            if (typeof req.body === 'string') {
+                body = req.body;
+            }
+            else if (req.body instanceof ArrayBuffer) {
+                body = req.body;
+            }
+            else if (req.body instanceof Blob) {
+                body = await req.body.arrayBuffer();
+            }
+            else if (typeof req.body === 'object') {
+                body = JSON.stringify(req.body);
+            }
+        }
+        return new Promise((resolve, reject) => {
+            this.pendingRequests.set(id, { resolve, reject });
+            if (req.signal) {
+                req.signal.addEventListener('abort', () => {
+                    const pending = this.pendingRequests.get(id);
+                    if (pending) {
+                        this.pendingRequests.delete(id);
+                        const err = new Error('Request aborted');
+                        err.name = 'AbortError';
+                        pending.reject(err);
+                    }
+                });
+            }
+            worker.postMessage({
+                id,
+                method: req.method,
+                url: req.url,
+                headers,
+                body,
+                timeout: timeoutMs
+            });
+        });
+    }
+    terminate() {
+        for (const worker of this.workers) {
+            worker.terminate();
+        }
+        this.workers = [];
+        URL.revokeObjectURL(this.workerUrl);
+        for (const [id, pending] of this.pendingRequests) {
+            pending.reject(new Error('Transport terminated'));
+        }
+        this.pendingRequests.clear();
+    }
+    get pendingCount() {
+        return this.pendingRequests.size;
+    }
+}
+class WorkerResponseWrapper {
+    status;
+    statusText;
+    headers;
+    ok;
+    url;
+    timings;
+    connection = {};
+    raw;
+    bodyBuffer;
+    bodyUsed = false;
+    constructor(data) {
+        this.status = data.status ?? 0;
+        this.statusText = data.statusText ?? '';
+        this.ok = data.ok ?? false;
+        this.url = data.url ?? '';
+        this.timings = data.timings ?? { total: 0 };
+        this.bodyBuffer = data.body ?? new ArrayBuffer(0);
+        this.headers = new Headers();
+        if (data.headers) {
+            for (const [key, value] of Object.entries(data.headers)) {
+                this.headers.set(key, value);
+            }
+        }
+        this.raw = new Response(this.bodyBuffer, {
+            status: this.status,
+            statusText: this.statusText,
+            headers: this.headers,
+        });
+    }
+    checkBodyUsed() {
+        if (this.bodyUsed) {
+            throw new Error('Body has already been consumed');
+        }
+        this.bodyUsed = true;
+    }
+    async json() {
+        this.checkBodyUsed();
+        const text = new TextDecoder().decode(this.bodyBuffer);
+        return JSON.parse(text);
+    }
+    async text() {
+        this.checkBodyUsed();
+        return new TextDecoder().decode(this.bodyBuffer);
+    }
+    async blob() {
+        this.checkBodyUsed();
+        return new Blob([this.bodyBuffer]);
+    }
+    async cleanText() {
+        const text = await this.text();
+        return text.replace(/<[^>]*>?/gm, '');
+    }
+    read() {
+        if (this.bodyUsed)
+            return null;
+        this.bodyUsed = true;
+        const buffer = this.bodyBuffer;
+        return new ReadableStream({
+            start(controller) {
+                controller.enqueue(new Uint8Array(buffer));
+                controller.close();
+            }
+        });
+    }
+    clone() {
+        const clonedBuffer = this.bodyBuffer.slice(0);
+        const headersObj = {};
+        this.headers.forEach((value, key) => {
+            headersObj[key] = value;
+        });
+        return new WorkerResponseWrapper({
+            status: this.status,
+            statusText: this.statusText,
+            ok: this.ok,
+            url: this.url,
+            timings: this.timings,
+            body: clonedBuffer,
+            headers: headersObj,
+        });
+    }
+    async *sse() {
+        throw new Error('SSE is not supported in WorkerTransport. Use FetchTransport for SSE.');
+    }
+    async *download() {
+        const total = this.bodyBuffer.byteLength;
+        yield {
+            loaded: total,
+            transferred: total,
+            total,
+            percent: 100,
+            direction: 'download'
+        };
+    }
+    async *[Symbol.asyncIterator]() {
+        if (this.bodyUsed)
+            return;
+        this.bodyUsed = true;
+        yield new Uint8Array(this.bodyBuffer);
+    }
+}

package/dist/browser/types/index.d.ts

@@ -58,8 +58,9 @@ export interface ReckerRequest {
     maxRedirects?: number;
     followRedirects?: boolean;
     http2?: boolean;
+    useCurl?: boolean;
+    userAgent?: string;
     withHeader(name: string, value: string): ReckerRequest;
-    withBody(body: BodyInit): ReckerRequest;
     _hooks?: {
         onDnsLookup?: (info: any) => void;
         onTcpConnect?: (info: any) => void;
@@ -329,6 +330,8 @@ export interface ClientOptions {
         jitter?: boolean;
         statusCodes?: number[];
     };
+    useCurl?: boolean;
+    userAgent?: string;
 }
 export interface ReckerResponse<T = unknown> {
     status: number;

package/dist/browser/utils/binary-manager.d.ts

@@ -0,0 +1,4 @@
+export declare function getCurlBinName(): string;
+export declare function getCurlPath(): string;
+export declare function hasImpersonate(): Promise<boolean>;
+export declare function installCurlImpersonate(logger?: Console): Promise<void>;