universal-parcel-scraper 0.134.1-main.1120 → 0.135.0

package/data/catalog.json

@@ -5597,7 +5597,8 @@
     "tracking": {
       "mode": "automatic",
       "adapter": "dhl-ecommerce-pl",
-      "recognitionRank": 620
+      "recognitionRank": 620,
+      "localClocks": true
     },
     "canaryUrl": "https://mojdhl.pl/tracking",
     "trackingUrlTemplate": "https://mojdhl.pl/tracking?paczki={trackingNumber}",

package/dist/browser/scraper.js

@@ -4546,7 +4546,8 @@ var De = {
 		tracking: {
 			mode: "automatic",
 			adapter: "dhl-ecommerce-pl",
-			recognitionRank: 620
+			recognitionRank: 620,
+			localClocks: !0
 		},
 		canaryUrl: "https://mojdhl.pl/tracking",
 		trackingUrlTemplate: "https://mojdhl.pl/tracking?paczki={trackingNumber}",

package/dist/carriers/dhl-ecommerce-pl/adapter.d.ts

@@ -1,5 +1,7 @@
 import { type AdapterFactory, type TrackingContext } from '../../core/adapter/index.js';
 import { type StepRecorder } from '../../core/telemetry/index.js';
+import type { TrawlClient } from '../../core/transport/trawl.js';
+import type { CarrierResult } from '../../core/result/index.js';
 import { type DhlEcommercePlChallenge } from './parser.js';
 /**
  * The proof of work the page computes for every visitor before a lookup: the
@@ -13,7 +15,8 @@ export declare class DhlEcommercePlTracker {
         fetcher?: typeof fetch;
         recorder?: StepRecorder;
         userAgent?: string;
+        trawl?: TrawlClient | null;
     });
-    fetch(raw: string, context?: TrackingContext): Promise<import("../../index.js").CarrierResult>;
+    fetch(raw: string, context?: TrackingContext): Promise<CarrierResult>;
 }
 export declare const adapter: AdapterFactory;

package/dist/carriers/dhl-ecommerce-pl/adapter.js

@@ -1,10 +1,12 @@
 import { createHash } from 'node:crypto';
 import { accepted, recognizeFromLookup } from '../../core/adapter/index.js';
 import { BudgetExceededError, ChallengeError, IndeterminateError, UpstreamHttpError, UpstreamNetworkError } from '../../core/errors/index.js';
-import { runSteps } from '../../core/runner/index.js';
+import { recoverableByDefault, runSteps } from '../../core/runner/index.js';
 import { firstAttemptMs, networkRetryStep } from '../../core/runner/networkRetry.js';
 import { NOOP_RECORDER } from '../../core/telemetry/index.js';
 import { decodeText, fetchBounded, parseJsonBytes, userAgentOf } from '../../core/transport/index.js';
+import { isRecord } from '../../core/types.js';
+import { dhlPolandBrowser } from './browser.js';
 import { normalizeDhlEcommercePlNumber, parseDhlEcommercePl, parseDhlEcommercePlChallenge, parseDhlEcommercePlRejection } from './parser.js';
 const PROVIDER = 'DHL eCommerce Poland';
 // The guest routes behind mojdhl.pl. They return the current status and none
@@ -37,7 +39,9 @@ export class DhlEcommercePlTracker {
     }
     fetch(raw, context = {}) {
         const number = normalizeDhlEcommercePlNumber(raw);
-        const budgetMs = context.budgetMs ?? 15_000;
+        const budgetMs = context.budgetMs ?? (this.options.trawl ? 65_000 : 15_000);
+        let confirmed;
+        let waybill;
         const read = async (signal, remainingMs) => {
             signal.throwIfAborted();
             // One deadline covers the challenge, its solution and the lookup.
@@ -89,17 +93,36 @@ export class DhlEcommercePlTracker {
             // A 400 is the portal refusing the proof: nothing was looked up.
             if (response.status !== 200)
                 throw new IndeterminateError(PROVIDER, 'DHL eCommerce Poland returned an inconclusive tracking response');
-            return parseDhlEcommercePl(json(bytes), number);
+            const payload = json(bytes);
+            const result = parseDhlEcommercePl(payload, number);
+            // Parsing first establishes the whole submitted number and one shipment.
+            const entry = Array.isArray(payload) ? payload[0] : undefined;
+            const shipment = isRecord(entry) && Array.isArray(entry.shipments) ? entry.shipments[0] : undefined;
+            const alias = isRecord(shipment) ? shipment.shipmentNumber : undefined;
+            waybill = typeof alias === 'string' && /^\d{11}$/.test(alias) ? alias : /^\d{11}$/.test(number) ? number : undefined;
+            confirmed = result;
+            return result;
         };
         return runSteps({ carrier: 'dhl-ecommerce-pl', budgetMs, signal: context.signal,
             recorder: this.options.recorder ?? NOOP_RECORDER }, [
-            { id: 'direct', run: ({ signal, remainingMs }) => read(signal, firstAttemptMs(remainingMs)) },
+            { id: 'trawl', enabled: !!this.options.trawl, run: async ({ signal, remainingMs }) => {
+                    const started = performance.now();
+                    const summary = await read(signal, Math.min(15_000, firstAttemptMs(remainingMs)));
+                    if (!waybill)
+                        throw new IndeterminateError(PROVIDER, 'DHL Poland returned no waybill for its history page');
+                    const history = await dhlPolandBrowser(this.options.trawl, waybill, signal, remainingMs - (performance.now() - started) - 500);
+                    return { ...summary, ...history };
+                } },
+            { id: 'direct', recovers: error => confirmed !== undefined || recoverableByDefault(error),
+                run: async ({ signal, remainingMs }) => confirmed ?? read(signal, firstAttemptMs(remainingMs)) },
             networkRetryStep(({ signal, remainingMs }) => read(signal, remainingMs)),
         ]);
     }
 }
 export const adapter = (environment) => {
-    const tracker = new DhlEcommercePlTracker({ fetcher: environment.fetcher, recorder: environment.recorder, userAgent: environment.userAgent });
-    return { id: 'dhl-ecommerce-pl', recordsSteps: true, steps: ['direct', 'retry'], track: (input, context) => tracker.fetch(input.number, context),
-        recognize: (number, context) => recognizeFromLookup(() => tracker.fetch(number, context), () => accepted(() => normalizeDhlEcommercePlNumber(number))) };
+    const options = { fetcher: environment.fetcher, recorder: environment.recorder, userAgent: environment.userAgent };
+    const tracker = new DhlEcommercePlTracker({ ...options, trawl: environment.trawl });
+    const http = new DhlEcommercePlTracker(options);
+    return { id: 'dhl-ecommerce-pl', recordsSteps: true, steps: ['trawl', 'direct', 'retry'], track: (input, context) => tracker.fetch(input.number, context),
+        recognize: (number, context) => recognizeFromLookup(() => http.fetch(number, context), () => accepted(() => normalizeDhlEcommercePlNumber(number))) };
 };

package/dist/carriers/dhl-ecommerce-pl/browser.d.ts

@@ -0,0 +1,5 @@
+import type { CarrierResult } from '../../core/result/index.js';
+import type { TrawlClient } from '../../core/transport/trawl.js';
+/** Facility clocks remain local: international rows do not identify their zone. */
+export declare function parseDhlPolandHistory(payload: unknown, waybill: string): CarrierResult;
+export declare function dhlPolandBrowser(trawl: TrawlClient, waybill: string, signal: AbortSignal, remainingMs: number): Promise<CarrierResult>;

package/dist/carriers/dhl-ecommerce-pl/browser.js

@@ -0,0 +1,83 @@
+import { BudgetExceededError, ChallengeError, IndeterminateError, RateLimitedError, SchemaError, TransportError, UpstreamHttpError } from '../../core/errors/index.js';
+import { calendarDay } from '../../core/time/index.js';
+import { clean } from '../../core/transport/index.js';
+import { isRecord } from '../../core/types.js';
+const PROVIDER = 'DHL eCommerce Poland';
+const ENDPOINT = 'https://sprawdz.dhl.com.pl/';
+const MAX_BODY = 300_000;
+const STAGES = [
+    [/^(?:Delivered to customer\.|Przesyłka doręczona do Odbiorcy\.)$/i, 'delivered'],
+    [/^(?:With delivery courier|Przesyłka przekazana kurierowi do doręczenia)\.?$/i, 'out_for_delivery'],
+    [/^(?:The shipment is in transit to the delivery depot\.|Przesyłka w transporcie do oddziału doręczającego\.)$/i, 'in_transit'],
+    [/^(?:The shipment has arrives at sorting center\.|Przesyłka została przyjęta w centrum sortowania\.)$/i, 'in_transit'],
+    [/^(?:The shipment is processed at depot of origin country\.|Przesyłka jest obsługiwana w oddziale nadającym\.)$/i, 'in_transit'],
+    [/^(?:The shipment has been dropped off at depot by the sender\.|Przesyłka została nadana przez nadawcę w oddziale DHL\.)$/i, 'accepted'],
+    [/^(?:The shipment is handled in Sorting Center|Przesyłka jest obsługiwana w Centrum Sortowania)\.?$/i, 'in_transit'],
+    [/^(?:The shipment has reached DHL terminal\.|Przesyłka dotarła do oddziału nadawczego DHL Parcel\.)$/i, 'accepted'],
+];
+/** Facility clocks remain local: international rows do not identify their zone. */
+export function parseDhlPolandHistory(payload, waybill) {
+    if (isRecord(payload) && payload.challenge === true)
+        throw new ChallengeError(PROVIDER);
+    if (!/^\d{11}$/.test(waybill) || !isRecord(payload) || payload.number !== waybill
+        || !Array.isArray(payload.events) || payload.events.length > 500) {
+        throw new SchemaError(PROVIDER, 'DHL Poland returned a different or invalid history');
+    }
+    const events = payload.events.map((row) => {
+        if (!isRecord(row) || typeof row.date !== 'string' || typeof row.time !== 'string'
+            || typeof row.location !== 'string' || row.location.length > 200
+            || typeof row.description !== 'string' || row.description.length > 500) {
+            throw new SchemaError(PROVIDER, 'DHL Poland returned an invalid scan');
+        }
+        const date = /^(\d{2})\.(\d{2})\.(\d{4})$/.exec(row.date);
+        const day = date && calendarDay(Number(date[3]), Number(date[2]), Number(date[1]));
+        if (!day || !/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(row.time))
+            throw new SchemaError(PROVIDER, 'DHL Poland returned an invalid scan clock');
+        const description = clean(row.description, 500).replace(/\s*(?:The (?:shipment|parcel) (?:was )?received by|Przesyłka odebrana przez)\b[\s\S]*$/i, '').trim();
+        if (!description)
+            throw new SchemaError(PROVIDER, 'DHL Poland returned an empty scan');
+        const stage = STAGES.find(([pattern]) => pattern.test(description))?.[1];
+        const location = clean(row.location, 200);
+        return { local_time: `${day}T${row.time}:00`, description,
+            ...(location ? { location } : {}), ...(stage ? { stage, stage_source: 'carrier_map' } : {}) };
+    });
+    if (!events.length)
+        throw new IndeterminateError(PROVIDER, 'DHL Poland returned no scan history');
+    return { summary_only: false, events: events.slice(0, 100), last_update_local: events[0].local_time,
+        ...(events.length > 100 ? { history_truncated: true } : {}) };
+}
+export async function dhlPolandBrowser(trawl, waybill, signal, remainingMs) {
+    const timeoutMs = Math.floor(remainingMs - 2000);
+    if (timeoutMs < 1)
+        throw new BudgetExceededError(PROVIDER, remainingMs);
+    // Leave room for the carrier's already-confirmed summary if the service stalls.
+    const bounded = AbortSignal.any([signal, AbortSignal.timeout(Math.max(1, Math.floor(remainingMs - 500)))]);
+    const page = await trawl.scrape({ url: `${ENDPOINT}?sn=${waybill}&lang=en-gb`, skipHttp: true, maxTier: 3,
+        captureResponses: [ENDPOINT], maxTimeout: timeoutMs, settleTimeout: timeoutMs }, { provider: PROVIDER, timeoutMs, signal: bounded, requireSolved: false, captureWindow: 'last', maxBytes: 4 * MAX_BODY });
+    signal.throwIfAborted();
+    const responses = page.capturedResponses.filter(row => row.url === ENDPOINT);
+    const response = responses.at(-1);
+    const status = response?.status ?? page.statusCode;
+    if (status === 429)
+        throw new RateLimitedError(PROVIDER);
+    if ([401, 403, 428].includes(status))
+        throw new ChallengeError(PROVIDER);
+    if (status >= 400)
+        throw new UpstreamHttpError(PROVIDER, status);
+    if (!response)
+        throw new ChallengeError(PROVIDER, 'DHL Poland did not complete browser verification');
+    if (responses.length > 10 || response.status !== 200 || response.error || response.truncated || response.base64Encoded || !response.body) {
+        throw new TransportError(PROVIDER, 'DHL Poland returned an incomplete browser response');
+    }
+    if (response.trackingNumberSubmitted !== waybill || Buffer.byteLength(response.body) > MAX_BODY) {
+        throw new SchemaError(PROVIDER, 'DHL Poland returned unbound browser history');
+    }
+    let payload;
+    try {
+        payload = JSON.parse(response.body);
+    }
+    catch {
+        throw new SchemaError(PROVIDER, 'DHL Poland returned invalid browser history');
+    }
+    return parseDhlPolandHistory(payload, waybill);
+}

package/dist/carriers/dhl-ecommerce-pl/carrier.json

@@ -20,17 +20,21 @@
     "mode": "automatic",
     "adapter": "dhl-ecommerce-pl",
     "steps": [
+      "trawl",
       "direct",
       "retry"
     ],
     "recognition": {
       "rank": 620
-    }
+    },
+    "localClocks": true
   },
   "capabilities": [
     "eta",
     "delivered_at",
-    "sender_name"
+    "sender_name",
+    "history",
+    "location"
   ],
   "portal": {
     "url": "https://mojdhl.pl/tracking?paczki={trackingNumber}",
@@ -41,21 +45,24 @@
       "eta",
       "delivered_at",
       "sender_name",
-      "tracking_alias"
+      "tracking_alias",
+      "history",
+      "location",
+      "recipient_name"
     ],
     "scraped": [
       "status",
       "eta",
       "delivered_at",
-      "sender_name"
+      "sender_name",
+      "history",
+      "location"
     ],
     "discarded": [
-      "tracking_alias"
+      "tracking_alias",
+      "recipient_name"
     ],
-    "unavailable": [
-      "history",
-      "location"
-    ]
+    "unavailable": []
   },
   "links": [
     {

package/dist/carriers/dhl-ecommerce-pl/parser.js

@@ -66,7 +66,7 @@ export function parseDhlEcommercePl(payload, rawNumber) {
     if (!isRecord(shipment) || typeof shipment.status !== 'string' || !/^[A-Z][A-Z0-9_]{0,39}$/.test(shipment.status)) {
         throw new SchemaError(PROVIDER, 'DHL eCommerce Poland returned an invalid shipment status');
     }
-    // The sender is the name the portal shows; the waybill alias and the self-service links are not read.
+    // The sender is the name the portal shows; self-service links stay out of results.
     const sender = clean(shipment.sender, 200);
     const { description, stage } = dhlEcommercePlStatus(shipment.status, shipment.timelineStep, clean(shipment.title, 200) || clean(shipment.step, 200));
     // Only a receipt is timed, by the recipient or by the sender a return reached. The posting

package/dist/carriers/ecoscooting/parser.js

@@ -30,7 +30,7 @@ export function parseEcoscooting(payload, rawNumber) {
     // The same query error is returned for absent and unavailable orders. It
     // neither identifies a parcel nor proves that Ecoscooting does not know it.
     if (payload.success !== 'true')
-        throw new IndeterminateError('Ecoscooting', 'Ecoscooting could not complete the tracking query');
+        throw new IndeterminateError('Ecoscooting', 'Ecoscooting could not complete the tracking query', payload.errorMsg === 'query logisticOrder by mailNo error' ? { reason: 'portal_not_found' } : {});
     if (!isRecord(payload.packageParam) || payload.packageParam.trackingNumber !== number)
         throw new SchemaError('Ecoscooting', 'Ecoscooting returned a different parcel');
     if (!Array.isArray(payload.statuses) || payload.statuses.length > 500 || !payload.statuses.every(isRecord))

package/dist/data/catalog.json

@@ -5597,7 +5597,8 @@
         "tracking": {
             "mode": "automatic",
             "adapter": "dhl-ecommerce-pl",
-            "recognitionRank": 620
+            "recognitionRank": 620,
+            "localClocks": true
         },
         "canaryUrl": "https://mojdhl.pl/tracking",
         "trackingUrlTemplate": "https://mojdhl.pl/tracking?paczki={trackingNumber}",

package/dist/generated/catalog.d.ts

@@ -3813,6 +3813,7 @@ export declare const CARRIER_CATALOG: {
             readonly mode: "automatic";
             readonly adapter: "dhl-ecommerce-pl";
             readonly recognitionRank: 620;
+            readonly localClocks: true;
         };
         readonly canaryUrl: "https://mojdhl.pl/tracking";
         readonly trackingUrlTemplate: "https://mojdhl.pl/tracking?paczki={trackingNumber}";

package/dist/generated/catalog.js

@@ -5598,7 +5598,8 @@ export const CARRIER_CATALOG = {
         "tracking": {
             "mode": "automatic",
             "adapter": "dhl-ecommerce-pl",
-            "recognitionRank": 620
+            "recognitionRank": 620,
+            "localClocks": true
         },
         "canaryUrl": "https://mojdhl.pl/tracking",
         "trackingUrlTemplate": "https://mojdhl.pl/tracking?paczki={trackingNumber}",

package/dist/server/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Universal Parcel Scraper",
-    "version": "0.134.1"
+    "version": "0.135.0"
   },
   "paths": {
     "/health": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "universal-parcel-scraper",
-  "version": "0.134.1-main.1120",
+  "version": "0.135.0",
   "description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
   "type": "module",
   "license": "Apache-2.0",
@@ -107,6 +107,7 @@
     "eslint": "^10.12.0",
     "@eslint/js": "^10.0.1",
     "typescript-eslint": "^8.71.0",
-    "globals": "^17.13.0"
+    "globals": "^17.13.0",
+    "linkedom": "0.18.13"
   }
 }