universal-parcel-scraper 0.0.0-stage → 0.2.0

package/dist/core/time/index.js

@@ -0,0 +1,243 @@
+/**
+ * Timestamp parsing with an explicit timezone policy.
+ *
+ * Providers disagree about time: some send offsets, some send naive wall-clock
+ * strings in a known zone, some send epoch milliseconds, some only a calendar
+ * day. Each helper below implements one policy and documents its limits so an
+ * adapter states which one applies instead of re-implementing the parsing.
+ * Every helper returns `null` rather than guessing.
+ */
+import { DateTime } from 'luxon';
+import { REGION_TOWNS } from '../../generated/regionTowns.js';
+import { clean } from '../transport/text.js';
+export const EXPLICIT_OFFSET_PATTERN = /(?:Z|[+-]\d{2}:?\d{2})$/i;
+function fromDateTime(parsed) {
+    const iso = parsed.isValid ? parsed.toISO({ suppressMilliseconds: true }) : null;
+    return iso ? { iso, timestamp: parsed.toMillis() } : null;
+}
+/**
+ * ISO strings that carry their own offset. Use for cross-border lanes where
+ * stamping a zone would be wrong; offset-less values are rejected.
+ */
+export function explicitOffsetTime(value, maxLength = 64) {
+    const raw = clean(value, maxLength);
+    if (!raw || !EXPLICIT_OFFSET_PATTERN.test(raw))
+        return null;
+    return fromDateTime(DateTime.fromISO(raw, { setZone: true }));
+}
+/**
+ * Naive wall-clock strings in one known zone. Use when the provider documents
+ * (or live checks show) a single backend zone; note regional caveats in the
+ * adapter.
+ */
+export function zonedTime(value, format, zone, options = {}) {
+    const raw = clean(value, options.maxLength ?? 64);
+    if (!raw)
+        return null;
+    return fromDateTime(DateTime.fromFormat(raw, format, { zone, locale: options.locale }));
+}
+/** ISO strings that may or may not carry an offset; offset-less values are read in `zone`. */
+export function isoTime(value, zone, maxLength = 64) {
+    const raw = clean(value, maxLength);
+    if (!raw)
+        return null;
+    const parsed = EXPLICIT_OFFSET_PATTERN.test(raw)
+        ? DateTime.fromISO(raw, { setZone: true })
+        : DateTime.fromISO(raw, { zone });
+    return fromDateTime(parsed);
+}
+/** The digits of a labeled instant read in UTC, or of an offset-less value as given. */
+function labeledDigits(value, maxLength) {
+    const raw = clean(value, maxLength);
+    if (!raw)
+        return null;
+    const labeled = EXPLICIT_OFFSET_PATTERN.test(raw)
+        ? DateTime.fromISO(raw, { setZone: true }).toUTC()
+        : DateTime.fromISO(raw, { zone: 'utc' });
+    return labeled.isValid ? labeled : null;
+}
+/**
+ * Wall-clock times a provider labels as UTC (or with an offset of its own)
+ * although they are the scan's local time: ParcelsApp and PostNL do this. The
+ * digits of the labeled instant, read in UTC, are re-read in `zone`.
+ * Offset-less values are read in `zone` directly.
+ */
+export function mislabeledLocalTime(value, zone, maxLength = 64) {
+    const labeled = labeledDigits(value, maxLength);
+    if (!labeled)
+        return null;
+    const { year, month, day, hour, minute, second, millisecond } = labeled;
+    return fromDateTime(DateTime.fromObject({ year, month, day, hour, minute, second, millisecond }, { zone }));
+}
+/** The wall clock `mislabeledLocalTime` re-reads, as an offset-less ISO string. */
+export function mislabeledWallTime(value, maxLength = 64) {
+    return labeledDigits(value, maxLength)?.toISO({ includeOffset: false }) ?? null;
+}
+/**
+ * One zone for several that keep the same clock at `wallIso`, an offset-less
+ * wall time: the first zone when every zone reads it as the same instant.
+ * Null when none is given, a zone or the time is invalid, the time carries an
+ * offset (every zone would agree), or the zones disagree at that moment, as
+ * Zurich and London always do.
+ */
+export function sharedClockZone(zones, wallIso) {
+    if (!zones.length || EXPLICIT_OFFSET_PATTERN.test(wallIso))
+        return null;
+    const instants = zones.map((zone) => DateTime.fromISO(wallIso, { zone }));
+    const first = instants[0].toMillis();
+    return instants.every((instant) => instant.isValid && instant.toMillis() === first) ? zones[0] : null;
+}
+// Some scanners run early; the host flags a scan only past the same hour.
+const CLOCK_SKEW_MS = 3_600_000;
+/**
+ * Settles guessed clocks against the feed itself, given newest first. A guess
+ * stands only if it is not after the lookup (`readAt`, give or take an hour of
+ * skew) and keeps the feed's order: not after the nearest newer scan with a
+ * known instant, not before the nearest older one. Each scan gets its known
+ * instant or its first guess that stands; null when none does.
+ */
+export function settleGuessedClocks(scans, readAt) {
+    const known = scans.map((scan) => scan.known ?? null);
+    const nearest = (from, step) => {
+        for (let index = from; index >= 0 && index < known.length; index += step) {
+            if (known[index])
+                return known[index].timestamp;
+        }
+        return undefined;
+    };
+    return scans.map((scan, index) => {
+        if (scan.known)
+            return scan.known;
+        const newer = nearest(index - 1, -1);
+        const older = nearest(index + 1, 1);
+        return scan.guesses?.find((guess) => Boolean(guess)
+            && guess.timestamp <= readAt + CLOCK_SKEW_MS
+            && (newer === undefined || guess.timestamp <= newer)
+            && (older === undefined || guess.timestamp >= older)) ?? null;
+    });
+}
+// Countries that keep one civil time. Spain and Portugal use their mainland
+// zone (their islands differ by an hour). Countries spanning several zones
+// (US, CA, BR, RU, AU, MX, ID...) are absent: they need a finer location.
+const COUNTRY_ZONES = {
+    AT: 'Europe/Vienna', BE: 'Europe/Brussels', BG: 'Europe/Sofia', CH: 'Europe/Zurich',
+    CY: 'Asia/Nicosia', CZ: 'Europe/Prague', DE: 'Europe/Berlin', DK: 'Europe/Copenhagen',
+    EE: 'Europe/Tallinn', ES: 'Europe/Madrid', FI: 'Europe/Helsinki', FR: 'Europe/Paris',
+    GB: 'Europe/London', GR: 'Europe/Athens', HR: 'Europe/Zagreb', HU: 'Europe/Budapest',
+    IE: 'Europe/Dublin', IS: 'Atlantic/Reykjavik', IT: 'Europe/Rome', LI: 'Europe/Vaduz',
+    LT: 'Europe/Vilnius', LU: 'Europe/Luxembourg', LV: 'Europe/Riga', MT: 'Europe/Malta',
+    NL: 'Europe/Amsterdam', NO: 'Europe/Oslo', PL: 'Europe/Warsaw', PT: 'Europe/Lisbon',
+    RO: 'Europe/Bucharest', SE: 'Europe/Stockholm', SI: 'Europe/Ljubljana', SK: 'Europe/Bratislava',
+    TR: 'Europe/Istanbul', AE: 'Asia/Dubai', IL: 'Asia/Jerusalem', IN: 'Asia/Kolkata',
+    CN: 'Asia/Shanghai', HK: 'Asia/Hong_Kong', JP: 'Asia/Tokyo', KR: 'Asia/Seoul',
+    MY: 'Asia/Kuala_Lumpur', PH: 'Asia/Manila', SG: 'Asia/Singapore', TH: 'Asia/Bangkok',
+    TW: 'Asia/Taipei', VN: 'Asia/Ho_Chi_Minh',
+};
+const ENGLISH_REGIONS = new Intl.DisplayNames(['en'], { type: 'region' });
+// Every region Intl names, by English name. Retired codes ("FX", "UK") repeat
+// a current region's name and are left out.
+const REGION_BY_NAME = new Map(Array.from({ length: 26 * 26 }, (_, index) => String.fromCharCode(65 + Math.floor(index / 26), 65 + (index % 26)))
+    .filter((code) => ENGLISH_REGIONS.of(code) !== code && new Intl.Locale(`und-${code}`).region === code)
+    .map((code) => [ENGLISH_REGIONS.of(code).toUpperCase(), code]));
+const REGION_CODES = new Set(REGION_BY_NAME.values());
+/**
+ * The ISO code of any country, whatever its clocks, from the code or its
+ * English name ("South Africa" is ZA, "United States" US); null otherwise.
+ */
+export function countryCode(country) {
+    if (typeof country !== 'string')
+        return null;
+    const value = country.trim().toUpperCase();
+    return REGION_CODES.has(value) ? value : REGION_BY_NAME.get(value) ?? null;
+}
+/** The zone of a single-zone country, from an ISO code or its English name; null otherwise. */
+export function countryTimeZone(country) {
+    const code = countryCode(country);
+    return code && Object.hasOwn(COUNTRY_ZONES, code) ? COUNTRY_ZONES[code] : null;
+}
+/** The single-zone country keeping this civil time ("Europe/Zurich" is Switzerland's); null for UTC. */
+export function timeZoneCountry(zone) {
+    const countries = Object.keys(COUNTRY_ZONES).filter((code) => COUNTRY_ZONES[code] === zone);
+    return countries.length === 1 ? countries[0] : null;
+}
+// US states and territories by postal code. A state with several zones keeps its
+// majority zone.
+const US_STATE_ZONES = {
+    AL: 'America/Chicago', AK: 'America/Anchorage', AZ: 'America/Phoenix', AR: 'America/Chicago',
+    CA: 'America/Los_Angeles', CO: 'America/Denver', CT: 'America/New_York', DE: 'America/New_York',
+    DC: 'America/New_York', FL: 'America/New_York', GA: 'America/New_York', HI: 'Pacific/Honolulu',
+    ID: 'America/Boise', IL: 'America/Chicago', IN: 'America/New_York', IA: 'America/Chicago',
+    KS: 'America/Chicago', KY: 'America/New_York', LA: 'America/Chicago', ME: 'America/New_York',
+    MD: 'America/New_York', MA: 'America/New_York', MI: 'America/Detroit', MN: 'America/Chicago',
+    MS: 'America/Chicago', MO: 'America/Chicago', MT: 'America/Denver', NE: 'America/Chicago',
+    NV: 'America/Los_Angeles', NH: 'America/New_York', NJ: 'America/New_York', NM: 'America/Denver',
+    NY: 'America/New_York', NC: 'America/New_York', ND: 'America/Chicago', OH: 'America/New_York',
+    OK: 'America/Chicago', OR: 'America/Los_Angeles', PA: 'America/New_York', RI: 'America/New_York',
+    SC: 'America/New_York', SD: 'America/Chicago', TN: 'America/Chicago', TX: 'America/Chicago',
+    UT: 'America/Denver', VT: 'America/New_York', VA: 'America/New_York', WA: 'America/Los_Angeles',
+    WV: 'America/New_York', WI: 'America/Chicago', WY: 'America/Denver', PR: 'America/Puerto_Rico',
+    GU: 'Pacific/Guam', VI: 'America/St_Thomas', AS: 'Pacific/Pago_Pago', MP: 'Pacific/Saipan',
+};
+/**
+ * The zone of a US state or territory from its postal code ("IL"): its
+ * majority zone when it has several; null otherwise.
+ */
+export function usStateTimeZone(state) {
+    if (typeof state !== 'string')
+        return null;
+    const code = state.trim().toUpperCase();
+    return Object.hasOwn(US_STATE_ZONES, code) ? US_STATE_ZONES[code] : null;
+}
+// Canadian provinces and territories by postal code, each in its majority zone.
+const CANADA_PROVINCE_ZONES = {
+    AB: 'America/Edmonton', BC: 'America/Vancouver', MB: 'America/Winnipeg', NB: 'America/Moncton',
+    NL: 'America/St_Johns', NS: 'America/Halifax', NT: 'America/Edmonton', NU: 'America/Iqaluit',
+    ON: 'America/Toronto', PE: 'America/Halifax', QC: 'America/Toronto', SK: 'America/Regina',
+    YT: 'America/Whitehorse',
+};
+/** The zone of a Canadian province or territory from its postal code ("ON"): its majority zone; null otherwise. */
+export function canadaProvinceTimeZone(province) {
+    if (typeof province !== 'string')
+        return null;
+    const code = province.trim().toUpperCase();
+    return Object.hasOwn(CANADA_PROVINCE_ZONES, code) ? CANADA_PROVINCE_ZONES[code] : null;
+}
+/** A town name as the region town lists key it: "St. John's" and "SAINT JOHNS" are both "stjohns". */
+export function townKey(name) {
+    return name.normalize('NFD').replace(/\p{Mn}/gu, '').toLowerCase()
+        .replace(/\bsaint\b/g, 'st').replace(/\bsainte\b/g, 'ste').replace(/\bmount\b/g, 'mt').replace(/\bfort\b/g, 'ft')
+        .replace(/[^a-z0-9]/g, '');
+}
+const REGION_TOWN_SETS = new Map(Object.entries(REGION_TOWNS).map(([code, towns]) => [code, new Set(towns.split('|'))]));
+/**
+ * Whether a town of this name lies in the US state or Canadian province whose
+ * code also names a single-clock country: Chicago is in Illinois, not Israel,
+ * and Koeln is not in Delaware. Only DE, IL, IN, MT, NL and SK have town lists
+ * (generated from GeoNames); false for any other code.
+ */
+export function regionHasTown(code, town) {
+    return REGION_TOWN_SETS.get(code.trim().toUpperCase())?.has(townKey(town)) ?? false;
+}
+/** Epoch milliseconds (numbers or numeric strings); zero and negatives are rejected. */
+export function epochMillisTime(value) {
+    const millis = typeof value === 'number' ? value
+        : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
+    if (!Number.isFinite(millis) || millis <= 0)
+        return null;
+    return fromDateTime(DateTime.fromMillis(millis, { zone: 'utc' }));
+}
+/** Epoch seconds, same rules as `epochMillisTime`. */
+export function epochSecondsTime(value) {
+    const seconds = typeof value === 'number' ? value
+        : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
+    if (!Number.isFinite(seconds) || seconds <= 0)
+        return null;
+    return epochMillisTime(seconds * 1000);
+}
+/** A local calendar day as `YYYY-MM-DD`, validated; never a timestamp. */
+export function calendarDay(year, month, day) {
+    if (![year, month, day].every(Number.isInteger))
+        return null;
+    const parsed = DateTime.fromObject({ year, month, day }, { zone: 'utc' });
+    return parsed.isValid ? parsed.toISODate() : null;
+}

package/dist/core/time/result.d.ts

@@ -0,0 +1,5 @@
+import type { CarrierResult } from '../result/index.js';
+export declare function resultTimezone(carrierId: string, result: CarrierResult): string;
+export declare function eventTimestamp(raw: unknown, assumedTimezone?: string): string | null;
+/** The newest instant of a result, under the same timezone policy as persisted events; 0 without one. */
+export declare function latestResultTime(result: CarrierResult, carrier: string): number;

package/dist/core/time/result.js

@@ -0,0 +1,55 @@
+/**
+ * How the host reads a carrier result's times: an offset is kept, and an
+ * offset-less wall time is read in the result's declared zone, else its
+ * carrier's catalog zone. Persisted events and the routing watermark share
+ * this policy, so a naive local clock is never compared as if it were UTC.
+ */
+import { DateTime, IANAZone } from 'luxon';
+import { carrierTimezone } from '../catalog/index.js';
+export function resultTimezone(carrierId, result) {
+    const declared = typeof result.timezone === 'string' ? result.timezone : '';
+    if (declared.length >= 1 && declared.length <= 64 && IANAZone.isValidZone(declared))
+        return declared;
+    try {
+        const configured = carrierTimezone(carrierId);
+        return IANAZone.isValidZone(configured) ? configured : 'UTC';
+    }
+    catch {
+        return 'UTC';
+    }
+}
+const EVENT_FORMATS = [
+    'yyyy-MM-dd HH:mm:ss',
+    'yyyy-MM-dd HH:mm',
+    'dd.MM.yyyy HH:mm:ss',
+    'dd.MM.yyyy HH:mm',
+    'dd/MM/yyyy HH:mm:ss',
+    'dd/MM/yyyy HH:mm',
+    'yyyy-MM-dd',
+    'dd.MM.yyyy',
+    'dd/MM/yyyy',
+];
+export function eventTimestamp(raw, assumedTimezone = 'UTC') {
+    if (typeof raw !== 'string' || !raw.trim())
+        return null;
+    const value = raw.trim();
+    let parsed = DateTime.fromISO(value, { setZone: true });
+    if (parsed.isValid) {
+        if (!/(?:Z|[+-]\d{2}:?\d{2})$/i.test(value)) {
+            parsed = DateTime.fromISO(value, { zone: assumedTimezone });
+        }
+        return parsed.toUTC().toISO({ suppressMilliseconds: true }) ?? null;
+    }
+    for (const format of EVENT_FORMATS) {
+        parsed = DateTime.fromFormat(value, format, { zone: assumedTimezone });
+        if (parsed.isValid)
+            return parsed.toUTC().toISO({ suppressMilliseconds: true }) ?? null;
+    }
+    return null;
+}
+/** The newest instant of a result, under the same timezone policy as persisted events; 0 without one. */
+export function latestResultTime(result, carrier) {
+    const timezone = resultTimezone(carrier, result);
+    return Math.max(0, ...[result.last_update, ...(result.events ?? []).map((event) => event.time)]
+        .map((time) => Date.parse(eventTimestamp(time, timezone) ?? '') || 0));
+}

package/dist/core/transport/boundedFetch.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Size- and time-bounded HTTP for provider requests. Every adapter reads
+ * provider responses through `fetchBounded`, which enforces a timeout, a
+ * byte cap, no caching, explicit redirect handling, and attaches bounded
+ * diagnostics to rejected responses. Errors are the shared taxonomy in
+ * core/errors.
+ */
+import { UpstreamHttpError, UpstreamNetworkError } from '../errors/index.js';
+export declare function fetchBounded(url: string | URL, init: RequestInit, options: {
+    provider: string;
+    timeoutMs?: number;
+    maxBytes?: number;
+    redirect?: RequestRedirect;
+    fetcher?: typeof fetch;
+    allowHttpError?: boolean;
+    /** Inspect specific non-success replies without bypassing other HTTP errors. */
+    allowHttpStatuses?: readonly number[];
+    /** One retry for replayable reads only; never retry parsing or validation. */
+    retryTransient?: boolean;
+}): Promise<{
+    response: Response;
+    bytes: Uint8Array;
+}>;
+export declare function decodeText(bytes: Uint8Array, encoding?: string): string;
+export declare function parseJsonBytes(bytes: Uint8Array, provider: string): unknown;
+export { UpstreamHttpError, UpstreamNetworkError };

package/dist/core/transport/boundedFetch.js

@@ -0,0 +1,152 @@
+/**
+ * Size- and time-bounded HTTP for provider requests. Every adapter reads
+ * provider responses through `fetchBounded`, which enforces a timeout, a
+ * byte cap, no caching, explicit redirect handling, and attaches bounded
+ * diagnostics to rejected responses. Errors are the shared taxonomy in
+ * core/errors.
+ */
+import { UpstreamHttpError, UpstreamNetworkError } from '../errors/index.js';
+import { readUpstreamHttpDiagnostics } from './upstreamHttpDiagnostics.js';
+const DEFAULT_MAX_BYTES = 2_000_000;
+const TRANSIENT_HTTP_STATUSES = new Set([429, 502, 503, 504]);
+const DEFAULT_RETRY_DELAY_MS = 1_000;
+const MAX_RETRY_DELAY_MS = 60_000;
+const MAX_DIAGNOSTIC_BODY = 8_192;
+function requestDiagnostics(url, init, timeoutMs) {
+    const body = typeof init.body === 'string' ? init.body
+        : init.body instanceof URLSearchParams ? init.body.toString() : undefined;
+    return { url: String(url), method: init.method ?? 'GET', headers: Object.fromEntries(new Headers(init.headers)),
+        body: body?.slice(0, MAX_DIAGNOSTIC_BODY), body_truncated: body === undefined ? undefined : body.length > MAX_DIAGNOSTIC_BODY,
+        timeout_ms: timeoutMs };
+}
+function retryDelay(header, status) {
+    // A rate limit without a retry window needs a later check, not another
+    // request one second later. Explicit, short Retry-After windows remain safe.
+    if (header === null)
+        return status === 429 ? null : DEFAULT_RETRY_DELAY_MS;
+    const value = header.trim();
+    const delay = /^\d+$/.test(value)
+        ? Number(value) * 1_000
+        : /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), /i.test(value)
+            ? Date.parse(value) - Date.now()
+            : NaN;
+    if (!Number.isFinite(delay) || delay > MAX_RETRY_DELAY_MS)
+        return null;
+    return Math.max(DEFAULT_RETRY_DELAY_MS, delay);
+}
+async function waitBeforeRetry(delayMs, signal) {
+    signal?.throwIfAborted();
+    await new Promise((resolve, reject) => {
+        const abort = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener('abort', abort);
+            reject(signal?.reason);
+        };
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', abort);
+            resolve();
+        }, delayMs);
+        signal?.addEventListener('abort', abort, { once: true });
+    });
+}
+async function cancelQuietly(body) {
+    try {
+        await body?.cancel();
+    }
+    catch {
+        // Cancellation is cleanup; it must not hide the response error.
+    }
+}
+export async function fetchBounded(url, init, options) {
+    const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+    let response;
+    for (let attempt = 0;; attempt += 1) {
+        init.signal?.throwIfAborted();
+        try {
+            const timeout = AbortSignal.timeout(Math.max(0, Math.floor(options.timeoutMs ?? 15_000)));
+            response = await (options.fetcher ?? fetch)(url, {
+                ...init,
+                cache: 'no-store',
+                redirect: options.redirect ?? 'error',
+                signal: init.signal ? AbortSignal.any([init.signal, timeout]) : timeout,
+            });
+        }
+        catch (error) {
+            if (options.retryTransient && attempt === 0 && !init.signal?.aborted) {
+                await waitBeforeRetry(DEFAULT_RETRY_DELAY_MS, init.signal);
+                continue;
+            }
+            throw new UpstreamNetworkError(options.provider, error, requestDiagnostics(url, init, options.timeoutMs ?? 15_000));
+        }
+        if (response.ok || options.allowHttpError || options.allowHttpStatuses?.includes(response.status))
+            break;
+        const delay = retryDelay(response.headers.get('retry-after'), response.status);
+        if (options.retryTransient && attempt === 0
+            && TRANSIENT_HTTP_STATUSES.has(response.status) && delay !== null) {
+            await cancelQuietly(response.body);
+            await waitBeforeRetry(delay, init.signal);
+            continue;
+        }
+        const retryHeader = response.headers.get('retry-after');
+        const retryAfterMs = retryHeader === null ? undefined : /^\d+$/.test(retryHeader.trim())
+            ? Number(retryHeader) * 1000 : Date.parse(retryHeader) - Date.now();
+        // Diagnostic failure must never replace the original HTTP status.
+        const diagnostics = await readUpstreamHttpDiagnostics(response).catch(() => undefined);
+        throw new UpstreamHttpError(options.provider, response.status, Number.isFinite(retryAfterMs) ? Math.max(0, retryAfterMs) : undefined, diagnostics, requestDiagnostics(url, init, options.timeoutMs ?? 15_000));
+    }
+    const contentLength = Number(response.headers.get('content-length'));
+    if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+        await cancelQuietly(response.body);
+        throw new Error(`${options.provider} returned an unexpectedly large response`);
+    }
+    if (!response.body)
+        return { response, bytes: new Uint8Array() };
+    const reader = response.body.getReader();
+    const chunks = [];
+    let length = 0;
+    try {
+        while (true) {
+            const { done, value } = await reader.read().catch((error) => {
+                throw new UpstreamNetworkError(options.provider, error, requestDiagnostics(url, init, options.timeoutMs ?? 15_000));
+            });
+            if (done)
+                break;
+            length += value.byteLength;
+            if (length > maxBytes) {
+                try {
+                    await reader.cancel();
+                }
+                catch {
+                    // Preserve the size error even if the upstream stream rejects cleanup.
+                }
+                throw new Error(`${options.provider} returned an unexpectedly large response`);
+            }
+            chunks.push(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    const bytes = new Uint8Array(length);
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return { response, bytes };
+}
+export function decodeText(bytes, encoding = 'utf-8') {
+    return new TextDecoder(encoding, { fatal: false }).decode(bytes);
+}
+export function parseJsonBytes(bytes, provider) {
+    try {
+        return JSON.parse(decodeText(bytes).replace(/^\uFEFF/, ''));
+    }
+    catch (error) {
+        throw Object.assign(new TypeError(`${provider} returned an invalid tracking response`, { cause: error }), {
+            provider, response_body: decodeText(bytes.subarray(0, MAX_DIAGNOSTIC_BODY)),
+            response_body_truncated: bytes.byteLength > MAX_DIAGNOSTIC_BODY,
+        });
+    }
+}
+export { UpstreamHttpError, UpstreamNetworkError };

package/dist/core/transport/browser.d.ts

@@ -0,0 +1,14 @@
+import type { Page } from 'playwright-core';
+import type { CarrierResult } from '../result/index.js';
+export interface UniversalBrowserOptions {
+    executablePath?: string;
+    timeoutMs?: number;
+}
+export declare function scrapeUniversalPage(options: UniversalBrowserOptions, spec: {
+    name: string;
+    url: string;
+    responseUrl: string;
+    submit?: (page: Page) => Promise<void>;
+    /** Explicit terminal replies from the lookup flow, including its submission endpoint. */
+    responseErrors?: Readonly<Record<string, (payload: unknown) => Error | undefined>>;
+}, parse: (payload: unknown) => CarrierResult): Promise<CarrierResult>;

package/dist/core/transport/browser.js

@@ -0,0 +1,113 @@
+import { loadChromium } from './optional.js';
+import { UpstreamHttpError } from '../errors/index.js';
+// A process handles many syncs concurrently. Do not spawn a browser per parcel
+// when a carrier outage sends an entire batch through the fallback chain.
+let busy = false;
+export async function scrapeUniversalPage(options, spec, parse) {
+    const executablePath = options.executablePath;
+    if (!executablePath)
+        throw new Error(`${spec.name} requires TRACKING_CHROMIUM_PATH`);
+    const timeoutMs = options.timeoutMs ?? 45_000;
+    if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 60_000)
+        throw new TypeError('Browser tracking timeout must be between 1 and 60000 ms');
+    if (busy)
+        throw new Error('Tracking browser is busy; retry on the next sync');
+    busy = true;
+    let browser;
+    let timer;
+    let expired = false;
+    try {
+        const deadline = Date.now() + timeoutMs;
+        // Do not pass the application environment (database/API secrets) to Chromium.
+        browser = await (await loadChromium(spec.name)).launch({ executablePath, headless: true,
+            args: ['--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage'], timeout: Math.min(timeoutMs, 10_000),
+            env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '/tmp', LANG: 'en_US.UTF-8' } });
+        const platform = process.platform === 'darwin' ? 'Macintosh; Intel Mac OS X 10_15_7' : 'X11; Linux x86_64';
+        const userAgent = `Mozilla/5.0 (${platform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${browser.version()} Safari/537.36`;
+        const remaining = Math.max(1, deadline - Date.now());
+        let received = 0;
+        let resolveHistory;
+        let rejectHistory;
+        const history = new Promise((resolve, reject) => {
+            resolveHistory = resolve;
+            rejectHistory = reject;
+            timer = setTimeout(() => { expired = true; reject(new Error(`${spec.name} did not return matching tracking history before the timeout`)); }, remaining);
+        });
+        // Attach the rejection handler before navigation can fail or time out.
+        const navigation = (async () => {
+            const context = await browser.newContext({ userAgent, locale: 'en-US', timezoneId: 'UTC',
+                viewport: { width: 1440, height: 1000 }, acceptDownloads: false, serviceWorkers: 'block' });
+            // Only the provider and its challenge resources are needed. This also keeps
+            // advertisements and provider scripts from reaching private network hosts.
+            const provider = new URL(spec.url).hostname.replace(/^www\./, '');
+            await context.route('**/*', async (route) => {
+                const url = new URL(route.request().url());
+                const allowed = url.protocol === 'https:' && (url.hostname === provider || url.hostname.endsWith(`.${provider}`)
+                    || url.hostname.endsWith('.challenges.cloudflare.com')
+                    || ['challenges.cloudflare.com', 'www.google.com', 'www.gstatic.com', 'www.recaptcha.net'].includes(url.hostname));
+                await (allowed ? route.continue() : route.abort());
+            });
+            const page = await context.newPage();
+            page.setDefaultTimeout(remaining);
+            page.on('response', async (response) => {
+                const url = response.url();
+                const inspect = Object.hasOwn(spec.responseErrors ?? {}, url) ? spec.responseErrors[url] : undefined;
+                if (expired || (url !== spec.responseUrl && !inspect))
+                    return;
+                if (response.status() === 429) {
+                    const raw = response.headers()['retry-after'];
+                    const delay = raw && /^\d+$/.test(raw) ? Number(raw) * 1000 : raw ? Date.parse(raw) - Date.now() : undefined;
+                    rejectHistory(new UpstreamHttpError(spec.name, 429, delay));
+                    return;
+                }
+                if (![200, 201].includes(response.status()))
+                    return;
+                if (++received > 20) {
+                    rejectHistory(new Error(`${spec.name} returned too many polling responses`));
+                    return;
+                }
+                try {
+                    if (Number(response.headers()['content-length'] ?? 0) > 2_000_000)
+                        return;
+                    const body = await response.body();
+                    if (expired || body.length > 2_000_000)
+                        return;
+                    const payload = JSON.parse(body.toString('utf8'));
+                    const error = inspect?.(payload);
+                    if (error) {
+                        rejectHistory(error);
+                        return;
+                    }
+                    if (url === spec.responseUrl)
+                        resolveHistory(parse(payload));
+                }
+                catch {
+                    // Initial polling, challenges and unrelated shipments are not history.
+                }
+            });
+            const response = await page.goto(spec.url, { waitUntil: 'domcontentloaded', timeout: remaining });
+            // Cloudflare initially responds with 403, then navigates after its
+            // automatic browser check. Let the bounded lookup wait for that reload.
+            const challenge = response?.status() === 403 && response.headers()['cf-mitigated'] === 'challenge';
+            if (!response)
+                throw new Error(`${spec.name} tracking page is unavailable`);
+            if (response.status() !== 200 && !challenge)
+                throw new UpstreamHttpError(spec.name, response.status());
+            if (spec.submit)
+                await spec.submit(page);
+            return history;
+        })();
+        return await Promise.race([navigation, history]);
+    }
+    finally {
+        expired = true;
+        if (timer)
+            clearTimeout(timer);
+        try {
+            await browser?.close();
+        }
+        finally {
+            busy = false;
+        }
+    }
+}

package/dist/core/transport/index.d.ts

@@ -0,0 +1,8 @@
+export { fetchBounded, decodeText, parseJsonBytes, UpstreamHttpError, UpstreamNetworkError } from './boundedFetch.js';
+export { readUpstreamHttpDiagnostics } from './upstreamHttpDiagnostics.js';
+export type { UpstreamHttpDiagnostics } from './upstreamHttpDiagnostics.js';
+export { TrawlClient, TrawlError, trawlBody, trawlEndpoint } from './trawl.js';
+export type { TrawlScrapeRequest, TrawlScrapeResponse, TrawlCapturedResponse, TrawlCallOptions } from './trawl.js';
+export { clean, cleanScalar, escapeRegExp, textFromHtml } from './text.js';
+export { scrapeUniversalPage } from './browser.js';
+export type { UniversalBrowserOptions } from './browser.js';

package/dist/core/transport/index.js

@@ -0,0 +1,5 @@
+export { fetchBounded, decodeText, parseJsonBytes, UpstreamHttpError, UpstreamNetworkError } from './boundedFetch.js';
+export { readUpstreamHttpDiagnostics } from './upstreamHttpDiagnostics.js';
+export { TrawlClient, TrawlError, trawlBody, trawlEndpoint } from './trawl.js';
+export { clean, cleanScalar, escapeRegExp, textFromHtml } from './text.js';
+export { scrapeUniversalPage } from './browser.js';

package/dist/core/transport/optional.d.ts

@@ -0,0 +1,2 @@
+export declare function loadChromium(provider: string): Promise<import("playwright-core").BrowserType<{}>>;
+export declare function loadSharp(provider: string): Promise<import("sharp").SharpConstructor>;

package/dist/core/transport/optional.js

@@ -0,0 +1,17 @@
+import { TransportError } from '../errors/index.js';
+export async function loadChromium(provider) {
+    try {
+        return (await import('playwright-core')).chromium;
+    }
+    catch (cause) {
+        throw new TransportError(provider, 'Install playwright-core to enable browser tracking', { cause });
+    }
+}
+export async function loadSharp(provider) {
+    try {
+        return (await import('sharp')).default;
+    }
+    catch (cause) {
+        throw new TransportError(provider, 'Install sharp to enable image challenges', { cause });
+    }
+}

package/dist/core/transport/text.d.ts

@@ -0,0 +1,8 @@
+/** Small text helpers every adapter needs when projecting provider payloads. */
+/** Collapse whitespace, trim, and cap the length of a provider string; non-strings become ''. */
+export declare function clean(value: unknown, maxLength?: number): string;
+/** Like `clean`, for values that may arrive as numbers (ids, codes). */
+export declare function cleanScalar(value: unknown, maxLength?: number): string;
+export declare function escapeRegExp(value: string): string;
+/** Strip tags from an HTML fragment and collapse whitespace; for short provider strings only. */
+export declare function textFromHtml(value: unknown, maxLength?: number): string;