universal-parcel-scraper 0.0.0-stage → 0.2.0

package/dist/core/transport/text.js

@@ -0,0 +1,18 @@
+/** Small text helpers every adapter needs when projecting provider payloads. */
+/** Collapse whitespace, trim, and cap the length of a provider string; non-strings become ''. */
+export function clean(value, maxLength = 500) {
+    return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim().slice(0, maxLength) : '';
+}
+/** Like `clean`, for values that may arrive as numbers (ids, codes). */
+export function cleanScalar(value, maxLength = 500) {
+    return typeof value === 'number' && Number.isFinite(value) ? String(value) : clean(value, maxLength);
+}
+export function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/** Strip tags from an HTML fragment and collapse whitespace; for short provider strings only. */
+export function textFromHtml(value, maxLength = 500) {
+    return typeof value === 'string'
+        ? clean(value.replace(/<[^>]*>/g, ' ').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&#39;|&apos;/g, "'").replace(/&quot;/g, '"'), maxLength)
+        : '';
+}

package/dist/core/transport/trawl.d.ts

@@ -0,0 +1,80 @@
+/**
+ * One client for the private TRAWL browser service (a FlareSolverr-compatible
+ * API). Adapters that need a real browser to get past a challenge, or to
+ * capture an in-page API response, ask this client instead of building the
+ * request themselves.
+ *
+ * Two endpoints exist on the service: the native `/scrape` API (tiers,
+ * response capture, cookies, user agent) and the legacy `/v1` command API
+ * (`request.get`). Both are bounded like every other provider call.
+ */
+import { CarrierError, type CarrierErrorOptions } from '../errors/index.js';
+import { type JsonObject } from '../types.js';
+export interface TrawlScrapeRequest extends JsonObject {
+    url: string;
+    /** Skip the plain HTTP tier and go straight to a browser. */
+    skipHttp?: boolean;
+    /** 1 = HTTP client, 2 = headless browser, 3 = full browser with challenge solving. */
+    maxTier?: 1 | 2 | 3;
+    /** Milliseconds the service may spend before answering. */
+    maxTimeout?: number;
+    /** In-page API URLs whose responses the browser should capture. */
+    captureResponses?: string[];
+    settleTimeout?: number;
+}
+export interface TrawlCapturedResponse {
+    url: string;
+    status: number;
+    headers: Record<string, string>;
+    body: string | null;
+    truncated: boolean;
+    base64Encoded: boolean;
+    error: string | null;
+}
+export interface TrawlScrapeResponse {
+    url: string;
+    html: string;
+    cookies: JsonObject[];
+    userAgent: string | null;
+    tier: number;
+    statusCode: number;
+    capturedResponses: TrawlCapturedResponse[];
+    /** The whole decoded payload, for adapters that read fields not modelled here. */
+    raw: JsonObject;
+}
+export interface TrawlCallOptions {
+    /** Label used in errors and diagnostics, for example "TRAWL while fetching UPS". */
+    provider: string;
+    timeoutMs: number;
+    maxBytes?: number;
+    fetcher?: typeof fetch;
+    /** Cancel the service request and all readiness/recovery requests together. */
+    signal?: AbortSignal;
+    /** Require a solved browser tier (2 or 3) with HTTP 200. Default true. */
+    requireSolved?: boolean;
+}
+export declare class TrawlError extends CarrierError {
+    constructor(provider: string, message: string, options?: CarrierErrorOptions);
+}
+export declare function trawlEndpoint(configured: string): URL;
+/**
+ * Decode the several shapes TRAWL releases use for a captured body: a string,
+ * a byte array, a serialized Node Buffer, or an index-keyed object.
+ */
+export declare function trawlBody(value: JsonObject, maxBytes?: number): string;
+export declare class TrawlClient {
+    private readonly fetcher?;
+    readonly endpoint: URL;
+    constructor(configured: string, fetcher?: typeof fetch | undefined);
+    /** The client for `FLARESOLVERR_URL`, or null when no browser service is configured. */
+    static fromEnvironment(env?: Record<string, string | undefined>, fetcher?: typeof fetch): TrawlClient | null;
+    private withPath;
+    scrapeUrl(): URL;
+    commandUrl(): URL;
+    private requestFetcher;
+    /** Native API: load a page in a browser tier, optionally capturing in-page API responses. */
+    scrape(request: TrawlScrapeRequest, options: TrawlCallOptions): Promise<TrawlScrapeResponse>;
+    private scrapeOnce;
+    /** Legacy command API (`request.get`): returns the solved page HTML. */
+    solve(url: string, options: TrawlCallOptions): Promise<string>;
+}

package/dist/core/transport/trawl.js

@@ -0,0 +1,212 @@
+/**
+ * One client for the private TRAWL browser service (a FlareSolverr-compatible
+ * API). Adapters that need a real browser to get past a challenge, or to
+ * capture an in-page API response, ask this client instead of building the
+ * request themselves.
+ *
+ * Two endpoints exist on the service: the native `/scrape` API (tiers,
+ * response capture, cookies, user agent) and the legacy `/v1` command API
+ * (`request.get`). Both are bounded like every other provider call.
+ */
+import { CarrierError, UpstreamHttpError } from '../errors/index.js';
+import { decodeText, fetchBounded, parseJsonBytes } from './boundedFetch.js';
+import { clean } from './text.js';
+import { isRecord } from '../types.js';
+export class TrawlError extends CarrierError {
+    constructor(provider, message, options) {
+        super('transport', provider, message, options);
+        this.name = 'TrawlError';
+    }
+}
+const DEFAULT_MAX_BYTES = 10_000_000;
+const TRANSPORT_ALLOWANCE_MS = 15_000;
+export function trawlEndpoint(configured) {
+    let endpoint;
+    try {
+        endpoint = new URL(configured.trim());
+    }
+    catch (error) {
+        throw new TypeError('FLARESOLVERR_URL must be an HTTP(S) URL', { cause: error });
+    }
+    if (!['http:', 'https:'].includes(endpoint.protocol) || !endpoint.host) {
+        throw new TypeError('FLARESOLVERR_URL must be an HTTP(S) URL');
+    }
+    endpoint.pathname = endpoint.pathname.replace(/\/(?:v1|scrape)\/?$/, '').replace(/\/$/, '');
+    endpoint.search = '';
+    endpoint.hash = '';
+    return endpoint;
+}
+function captured(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter(isRecord).slice(0, 50).map((entry) => ({
+        url: typeof entry.url === 'string' ? entry.url : '',
+        status: Number.isInteger(entry.status) ? Number(entry.status) : 0,
+        headers: isRecord(entry.headers)
+            ? Object.fromEntries(Object.entries(entry.headers).filter((pair) => typeof pair[1] === 'string'))
+            : {},
+        body: typeof entry.body === 'string' ? entry.body : null,
+        truncated: entry.truncated === true,
+        base64Encoded: entry.base64Encoded === true,
+        error: typeof entry.error === 'string' ? entry.error : entry.error ? 'error' : null,
+    }));
+}
+/**
+ * Decode the several shapes TRAWL releases use for a captured body: a string,
+ * a byte array, a serialized Node Buffer, or an index-keyed object.
+ */
+export function trawlBody(value, maxBytes = DEFAULT_MAX_BYTES) {
+    if (typeof value.body === 'string')
+        return value.body;
+    const rawBody = value.body;
+    let bytes = null;
+    const byteLike = (entry) => Number.isInteger(entry) && Number(entry) >= 0 && Number(entry) <= 255;
+    if (Array.isArray(rawBody)) {
+        bytes = rawBody.length <= maxBytes && rawBody.every(byteLike) ? rawBody : null;
+    }
+    else if (isRecord(rawBody) && rawBody.type === 'Buffer' && Array.isArray(rawBody.data)) {
+        bytes = rawBody.data.length <= maxBytes && rawBody.data.every(byteLike) ? rawBody.data : null;
+    }
+    else if (isRecord(rawBody)) {
+        const entries = Object.entries(rawBody);
+        if (entries.length > 0 && entries.length <= maxBytes) {
+            entries.sort((left, right) => Number(left[0]) - Number(right[0]));
+            const sequential = entries.every(([key, entry], index) => String(index) === key && byteLike(entry));
+            if (sequential)
+                bytes = entries.map(([, entry]) => Number(entry));
+        }
+    }
+    return bytes ? decodeText(Uint8Array.from(bytes)) : '';
+}
+export class TrawlClient {
+    fetcher;
+    endpoint;
+    constructor(configured, fetcher) {
+        this.fetcher = fetcher;
+        this.endpoint = trawlEndpoint(configured);
+    }
+    /** The client for `FLARESOLVERR_URL`, or null when no browser service is configured. */
+    static fromEnvironment(env = process.env, fetcher) {
+        const configured = (env.FLARESOLVERR_URL ?? '').trim();
+        return configured ? new TrawlClient(configured, fetcher) : null;
+    }
+    withPath(segment) {
+        const url = new URL(this.endpoint);
+        // A bare origin normalizes its pathname back to "/", so strip it before appending.
+        url.pathname = `${url.pathname.replace(/\/$/, '')}/${segment}`;
+        return url;
+    }
+    scrapeUrl() {
+        return this.withPath('scrape');
+    }
+    commandUrl() {
+        return this.withPath('v1');
+    }
+    requestFetcher(options) {
+        const fetcher = options.fetcher ?? this.fetcher ?? fetch;
+        if (!options.signal)
+            return fetcher;
+        return (input, init) => {
+            options.signal.throwIfAborted();
+            return fetcher(input, { ...init, signal: AbortSignal.any([
+                    options.signal, ...(init?.signal ? [init.signal] : []),
+                ]) });
+        };
+    }
+    /** Native API: load a page in a browser tier, optionally capturing in-page API responses. */
+    async scrape(request, options) {
+        options.signal?.throwIfAborted();
+        const deadline = performance.now() + options.timeoutMs + TRANSPORT_ALLOWANCE_MS;
+        try {
+            return await this.scrapeOnce(request, options);
+        }
+        catch (error) {
+            options.signal?.throwIfAborted();
+            // Retry only an identified dead browser, never maintenance, challenges or arbitrary 500s.
+            if (!(error instanceof UpstreamHttpError) || error.status < 500
+                || !/Target page, context or browser has been closed/.test(error.diagnostics?.body_excerpt ?? '')
+                || deadline - performance.now() < TRANSPORT_ALLOWANCE_MS + 3_000)
+                throw error;
+            try {
+                const { bytes } = await fetchBounded(this.withPath('health'), {}, {
+                    provider: 'TRAWL readiness', timeoutMs: 2_000, maxBytes: 16_384,
+                    fetcher: this.requestFetcher(options),
+                });
+                options.signal?.throwIfAborted();
+                const health = parseJsonBytes(bytes, 'TRAWL readiness');
+                if (!isRecord(health) || health.status !== 'ok' || !isRecord(health.pool)
+                    || Number(health.pool.live) < 1 || Number(health.pool.available) < 1)
+                    throw error;
+            }
+            catch {
+                options.signal?.throwIfAborted();
+                throw error;
+            }
+            const remaining = Math.floor(deadline - performance.now() - TRANSPORT_ALLOWANCE_MS);
+            if (remaining < 1_000)
+                throw error;
+            return this.scrapeOnce({ ...request, maxTimeout: Math.min(request.maxTimeout ?? options.timeoutMs, remaining) }, { ...options, timeoutMs: remaining });
+        }
+    }
+    async scrapeOnce(request, options) {
+        options.signal?.throwIfAborted();
+        const { bytes } = await fetchBounded(this.scrapeUrl(), {
+            method: 'POST',
+            headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
+            body: JSON.stringify(request),
+        }, {
+            provider: options.provider,
+            timeoutMs: options.timeoutMs + TRANSPORT_ALLOWANCE_MS,
+            maxBytes: options.maxBytes ?? DEFAULT_MAX_BYTES,
+            fetcher: this.requestFetcher(options),
+        });
+        options.signal?.throwIfAborted();
+        const value = parseJsonBytes(bytes, options.provider);
+        if (!isRecord(value))
+            throw new TrawlError(options.provider, 'The browser service returned an invalid response');
+        if (value.error) {
+            throw new TrawlError(options.provider, clean(String(value.error), 200) || 'The browser service could not fetch the page');
+        }
+        const statusCode = Number(value.statusCode);
+        if (Number.isInteger(statusCode) && statusCode >= 400)
+            throw new UpstreamHttpError(options.provider, statusCode);
+        const tier = Number(value.tier);
+        if ((options.requireSolved ?? true) && (![2, 3].includes(tier) || statusCode !== 200)) {
+            throw new TrawlError(options.provider, 'The browser service did not solve the page');
+        }
+        if (typeof value.html !== 'string')
+            throw new TrawlError(options.provider, 'The browser service returned no page');
+        return {
+            url: typeof value.url === 'string' ? value.url : request.url,
+            html: value.html,
+            cookies: Array.isArray(value.cookies) ? value.cookies.filter(isRecord) : [],
+            userAgent: typeof value.userAgent === 'string' ? value.userAgent : null,
+            tier,
+            statusCode,
+            capturedResponses: captured(value.capturedResponses),
+            raw: value,
+        };
+    }
+    /** Legacy command API (`request.get`): returns the solved page HTML. */
+    async solve(url, options) {
+        options.signal?.throwIfAborted();
+        const { bytes } = await fetchBounded(this.commandUrl(), {
+            method: 'POST',
+            headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
+            body: JSON.stringify({ cmd: 'request.get', url, maxTimeout: options.timeoutMs }),
+        }, {
+            provider: options.provider,
+            timeoutMs: options.timeoutMs + TRANSPORT_ALLOWANCE_MS,
+            maxBytes: options.maxBytes ?? DEFAULT_MAX_BYTES,
+            fetcher: this.requestFetcher(options),
+        });
+        options.signal?.throwIfAborted();
+        const payload = parseJsonBytes(bytes, options.provider);
+        const solution = isRecord(payload) && isRecord(payload.solution) ? payload.solution : {};
+        if (isRecord(payload) && payload.status === 'ok' && [200, 302].includes(Number(solution.status))
+            && typeof solution.response === 'string') {
+            return solution.response;
+        }
+        throw new TrawlError(options.provider, 'The browser service did not solve the page');
+    }
+}

package/dist/core/transport/upstreamHttpDiagnostics.d.ts

@@ -0,0 +1,16 @@
+export interface UpstreamHttpDiagnostics {
+    content_type: string;
+    headers: Record<string, string>;
+    response_url: string;
+    status_text: string;
+    server?: string;
+    request_ids: Record<string, string>;
+    body_read: 'complete' | 'truncated' | 'timed_out' | 'unreadable' | 'empty' | 'skipped';
+    bytes_inspected: number;
+    /** Recognized signatures, not a definitive explanation of the rejection. */
+    body_signals: string[];
+    body_excerpt?: string;
+    error_code?: string;
+}
+/** Keep original response details; byte/time bounds prevent diagnostics delaying recovery. */
+export declare function readUpstreamHttpDiagnostics(response: Response): Promise<UpstreamHttpDiagnostics>;

package/dist/core/transport/upstreamHttpDiagnostics.js

@@ -0,0 +1,97 @@
+const MAX_BYTES = 8_192;
+const READ_TIMEOUT_MS = 200;
+const CONTENT_TYPES = new Set(['text/html', 'text/plain', 'application/json', 'application/problem+json', 'application/xml', 'text/xml']);
+const ERROR_CODES = new Set(['ACCESS_DENIED', 'FORBIDDEN', 'UNAUTHORIZED', 'RATE_LIMITED', 'TOO_MANY_REQUESTS',
+    'CAPTCHA_REQUIRED', 'CHALLENGE_REQUIRED', 'INVALID_TOKEN', 'TOKEN_EXPIRED']);
+function responseIds(headers) {
+    const ids = {};
+    for (const name of ['cf-ray', 'x-request-id', 'x-correlation-id', 'x-akamai-request-id']) {
+        const value = headers.get(name)?.trim();
+        if (value)
+            ids[name] = value.slice(0, 128);
+    }
+    return ids;
+}
+function recognizedCode(text) {
+    try {
+        const value = JSON.parse(text);
+        const record = (item) => item !== null && typeof item === 'object' && !Array.isArray(item) ? item : {};
+        const root = record(value);
+        const error = record(root.error);
+        for (const code of [root.code, root.errorCode, root.error, error.code]) {
+            if (typeof code === 'string' && ERROR_CODES.has(code.toUpperCase()))
+                return code.toUpperCase();
+        }
+    }
+    catch { /* HTML, truncated JSON and unknown formats have no recognized code. */ }
+    return undefined;
+}
+/** Keep original response details; byte/time bounds prevent diagnostics delaying recovery. */
+export async function readUpstreamHttpDiagnostics(response) {
+    const mediaType = response.headers.get('content-type')?.split(';', 1)[0].trim().toLowerCase();
+    const diagnostics = {
+        content_type: !mediaType ? 'missing' : CONTENT_TYPES.has(mediaType) ? mediaType : 'other',
+        headers: Object.fromEntries(response.headers), response_url: response.url, status_text: response.statusText,
+        request_ids: responseIds(response.headers), body_read: 'empty', bytes_inspected: 0, body_signals: [],
+        server: response.headers.get('server')?.slice(0, 128),
+    };
+    if (!response.body)
+        return diagnostics;
+    if (diagnostics.content_type === 'other') {
+        diagnostics.body_read = 'skipped';
+        void response.body.cancel().catch(() => { });
+        return diagnostics;
+    }
+    const reader = response.body.getReader();
+    const prefix = new Uint8Array(MAX_BYTES);
+    const expiresAt = performance.now() + READ_TIMEOUT_MS;
+    let timer;
+    const deadline = new Promise((resolve) => { timer = setTimeout(() => resolve(null), READ_TIMEOUT_MS); });
+    try {
+        while (diagnostics.bytes_inspected < MAX_BYTES) {
+            if (performance.now() >= expiresAt) {
+                diagnostics.body_read = 'timed_out';
+                break;
+            }
+            const part = await Promise.race([reader.read(), deadline]);
+            if (part === null) {
+                diagnostics.body_read = 'timed_out';
+                break;
+            }
+            if (part.done) {
+                diagnostics.body_read = diagnostics.bytes_inspected ? 'complete' : 'empty';
+                break;
+            }
+            const bytes = part.value.subarray(0, MAX_BYTES - diagnostics.bytes_inspected);
+            prefix.set(bytes, diagnostics.bytes_inspected);
+            diagnostics.bytes_inspected += bytes.byteLength;
+            if (diagnostics.bytes_inspected === MAX_BYTES)
+                diagnostics.body_read = 'truncated';
+        }
+    }
+    catch {
+        diagnostics.body_read = 'unreadable';
+    }
+    finally {
+        clearTimeout(timer);
+        // A stuck upstream cancellation must not delay the original HTTP error.
+        void reader.cancel().catch(() => { });
+        reader.releaseLock();
+    }
+    const text = new TextDecoder().decode(prefix.subarray(0, diagnostics.bytes_inspected));
+    if (text)
+        diagnostics.body_excerpt = text;
+    const signatures = [
+        ['cloudflare_challenge', /(?:\/cdn-cgi\/challenge-platform\/|\b_cf_chl_opt\b)/i],
+        ['datadome_challenge', /(?:geo\.captcha-delivery\.com|ct\.captcha-delivery\.com)/i],
+        ['perimeterx_challenge', /(?:\bpx-captcha\b|captcha\.px-cdn\.net)/i],
+        ['akamai_error_page', /errors\.edgesuite\.net\//i],
+        ['access_denied', /\baccess denied\b|\baccès refusé\b/i],
+        ['rate_limit_message', /\btoo many requests\b|\brate limit exceeded\b/i],
+        ['captcha_message', /\bcaptcha\b/i],
+    ];
+    diagnostics.body_signals = signatures.filter(([, pattern]) => pattern.test(text)).map(([name]) => name);
+    if (diagnostics.body_read === 'complete')
+        diagnostics.error_code = recognizedCode(text);
+    return diagnostics;
+}

package/dist/core/types.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Minimal JSON helpers shared by every core module and adapter. Kept here so
+ * the package never reaches into application code for them.
+ */
+export type JsonObject = Record<string, unknown>;
+export declare function isRecord(value: unknown): value is JsonObject;
+export declare function asRecord(value: unknown, message: string): JsonObject;
+export declare function errorMessage(error: unknown): string;

package/dist/core/types.js

@@ -0,0 +1,11 @@
+export function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+export function asRecord(value, message) {
+    if (!isRecord(value))
+        throw new TypeError(message);
+    return value;
+}
+export function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}