universal-parcel-scraper 0.52.0-main.665 → 0.53.0-main.666

package/dist/carriers/spee-dee/adapter.d.ts

@@ -1,6 +1,7 @@
 import type { AdapterFactory, TrackingContext } from '../../core/adapter/index.js';
 import type { CarrierResult } from '../../core/result/index.js';
 import { type StepRecorder } from '../../core/telemetry/index.js';
+import type { TrawlClient } from '../../core/transport/trawl.js';
 export declare function speeDeeProgressUrl(raw: string): string;
 /**
  * Whether a lookup failed because the host never answered. A reply that
@@ -13,6 +14,7 @@ export declare class SpeeDeeTracker {
         fetcher?: typeof fetch;
         recorder?: StepRecorder;
         userAgent?: string;
+        trawl?: TrawlClient | null;
     });
     fetch(raw: string, context?: TrackingContext): Promise<CarrierResult>;
 }

package/dist/carriers/spee-dee/adapter.js

@@ -46,7 +46,7 @@ export class SpeeDeeTracker {
     }
     async fetch(raw, context = {}) {
         const number = normalizeSpeeDeeNumber(raw);
-        return runSteps({ carrier: 'spee-dee', budgetMs: context.budgetMs ?? 15_000, signal: context.signal,
+        return runSteps({ carrier: 'spee-dee', budgetMs: context.budgetMs ?? (this.options.trawl ? 30_000 : 15_000), signal: context.signal,
             recorder: this.options.recorder ?? NOOP_RECORDER }, [{ id: 'direct', run: async ({ signal, remainingMs }) => {
                     let fetched;
                     // Set once the status and headers arrive. A body that then breaks or
@@ -78,10 +78,38 @@ export class SpeeDeeTracker {
                     if (REDIRECTS.includes(fetched.response.status))
                         throw new SchemaError(PROVIDER, 'Spee-Dee tracking page moved');
                     return parseSpeeDee(decodeText(fetched.bytes), number);
-                } }]);
+                } }, {
+                id: 'trawl', enabled: Boolean(this.options.trawl), recovers: speeDeeUnanswered,
+                run: async ({ signal, remainingMs }) => {
+                    const url = speeDeeProgressUrl(number);
+                    // The remote network is the recovery; this page needs no browser or solver.
+                    const timeoutMs = Math.max(1, Math.min(REQUEST_TIMEOUT_MS, Math.floor(remainingMs)));
+                    let reply;
+                    try {
+                        reply = await this.options.trawl.scrape({ url, maxTier: 1, maxTimeout: timeoutMs }, {
+                            provider: PROVIDER, timeoutMs, signal: AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]),
+                            fetcher: this.options.fetcher,
+                            maxBytes: 4_000_000, requireSolved: false,
+                        });
+                    }
+                    catch (error) {
+                        if (error instanceof UpstreamHttpError && [404, 410].includes(error.status)) {
+                            throw new TransportError(PROVIDER, 'Spee-Dee remote tracking page is unavailable', { cause: error });
+                        }
+                        throw error;
+                    }
+                    if (reply.url !== url || reply.statusCode !== 200 || reply.tier !== 1) {
+                        throw new SchemaError(PROVIDER, 'Spee-Dee remote tracking page changed');
+                    }
+                    if (Buffer.byteLength(reply.html) > MAX_BYTES)
+                        throw new SchemaError(PROVIDER, 'Spee-Dee returned excessive tracking data');
+                    return parseSpeeDee(reply.html, number);
+                },
+            }]);
     }
 }
 export const adapter = (environment) => {
-    const tracker = new SpeeDeeTracker({ fetcher: environment.fetcher, recorder: environment.recorder, userAgent: environment.userAgent });
-    return { id: 'spee-dee', recordsSteps: true, steps: ['direct'], track: (input, context) => tracker.fetch(input.number, context) };
+    const tracker = new SpeeDeeTracker({ fetcher: environment.fetcher, recorder: environment.recorder,
+        userAgent: environment.userAgent, trawl: environment.trawl });
+    return { id: 'spee-dee', recordsSteps: true, steps: ['direct', 'trawl'], track: (input, context) => tracker.fetch(input.number, context) };
 };

package/dist/carriers/spee-dee/carrier.json

@@ -16,7 +16,8 @@
     "mode": "automatic",
     "adapter": "spee-dee",
     "steps": [
-      "direct"
+      "direct",
+      "trawl"
     ],
     "localClocks": true
   },

package/dist/core/transport/trawl.js

@@ -8,7 +8,7 @@
  * response capture, cookies, user agent) and the legacy `/v1` command API
  * (`request.get`). Both are bounded like every other provider call.
  */
-import { CarrierError, UpstreamHttpError } from '../errors/index.js';
+import { CarrierError, ChallengeError, UpstreamHttpError } from '../errors/index.js';
 import { decodeText, fetchBounded, parseJsonBytes } from './boundedFetch.js';
 import { cleanScalar } from './text.js';
 import { isRecord } from '../types.js';
@@ -26,6 +26,13 @@ const DEFAULT_MAX_BYTES = 10_000_000;
  */
 export const TRAWL_TRANSPORT_ALLOWANCE_MS = 15_000;
 const TRANSPORT_ALLOWANCE_MS = TRAWL_TRANSPORT_ALLOWANCE_MS;
+function pageHttpError(provider, status, headers) {
+    const retryHeader = isRecord(headers)
+        ? Object.entries(headers).find(([key]) => key.toLowerCase() === 'retry-after')?.[1] : undefined;
+    const retryAfterMs = typeof retryHeader !== 'string' ? undefined : /^\d+$/.test(retryHeader.trim())
+        ? Number(retryHeader) * 1_000 : Date.parse(retryHeader) - Date.now();
+    return new UpstreamHttpError(provider, status, Number.isFinite(retryAfterMs) ? Math.max(0, retryAfterMs) : undefined);
+}
 /**
  * End the call when its signal has aborted. A cancellation is rethrown as the
  * caller gave it. A deadline is a browser service that did not answer in
@@ -172,7 +179,7 @@ export class TrawlClient {
     }
     async scrapeOnce(request, options) {
         throwIfEnded(options);
-        const { bytes } = await fetchBounded(this.scrapeUrl(), {
+        const { bytes, response } = await fetchBounded(this.scrapeUrl(), {
             method: 'POST',
             headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
             body: JSON.stringify(request),
@@ -181,17 +188,43 @@ export class TrawlClient {
             timeoutMs: options.timeoutMs + TRANSPORT_ALLOWANCE_MS,
             maxBytes: options.maxBytes ?? DEFAULT_MAX_BYTES,
             fetcher: this.requestFetcher(options),
+            // An HTTP-only lookup has exactly one upstream outcome. TRAWL wraps its
+            // blocked responses in HTTP 500, including upstream 429 and HTTP 200 walls.
+            allowHttpStatuses: request.maxTier === 1 ? [500] : undefined,
         });
         throwIfEnded(options);
-        const value = parseJsonBytes(bytes, options.provider);
+        let value;
+        try {
+            value = parseJsonBytes(bytes, options.provider);
+        }
+        catch (error) {
+            if (response.status === 500)
+                throw new UpstreamHttpError(options.provider, 500);
+            throw error;
+        }
         if (!isRecord(value))
             throw new TrawlError(options.provider, 'The browser service returned an invalid response');
+        if (request.maxTier === 1 && value.error === 'Max tier reached without success'
+            && Array.isArray(value.timings) && value.timings.length === 1) {
+            const attempt = value.timings[0];
+            if (isRecord(attempt) && attempt.tier === 1 && ['blocked', 'needs-js', 'error'].includes(String(attempt.status))) {
+                if (Number.isInteger(attempt.statusCode) && Number(attempt.statusCode) >= 400) {
+                    throw pageHttpError(options.provider, Number(attempt.statusCode), attempt.responseHeaders);
+                }
+                if (attempt.statusCode === 200 && ['blocked', 'needs-js'].includes(String(attempt.status))) {
+                    throw new ChallengeError(options.provider, 'The tracking page requires browser verification');
+                }
+            }
+        }
+        if (response.status === 500)
+            throw new UpstreamHttpError(options.provider, 500);
         if (value.error) {
             throw new TrawlError(options.provider, cleanScalar(value.error, 200) || 'The browser service could not fetch the page');
         }
         const statusCode = Number(value.statusCode);
-        if (Number.isInteger(statusCode) && statusCode >= 400)
-            throw new UpstreamHttpError(options.provider, statusCode);
+        if (Number.isInteger(statusCode) && statusCode >= 400) {
+            throw pageHttpError(options.provider, statusCode, value.responseHeaders);
+        }
         const tier = Number(value.tier);
         if ((options.requireSolved ?? true) && (![2, 3].includes(tier) || statusCode !== 200)) {
             throw new TrawlError(options.provider, 'The browser service did not solve the page');

package/dist/server/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Universal Parcel Scraper",
-    "version": "0.52.0"
+    "version": "0.53.0"
   },
   "paths": {
     "/health": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "universal-parcel-scraper",
-  "version": "0.52.0-main.665",
+  "version": "0.53.0-main.666",
   "description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
   "type": "module",
   "license": "Apache-2.0",