universal-parcel-scraper 0.181.0 → 0.183.0-main.1321

package/README.md

@@ -13,7 +13,7 @@ The engine behind [Peek](https://peektracker.com) ([GitHub](https://github.com/p
 [![License: Apache-2.0](https://img.shields.io/badge/core-Apache--2.0-blue.svg)](LICENSE)
 
 <!-- GENERATED:summary -->
-**3,500+ carriers** through **256 dedicated adapters** and **5 universal fallbacks**
+**3,500+ carriers** through **260 dedicated adapters** and **5 universal fallbacks**
 
 <sub>494 catalog entries · 113 countries represented</sub>
 <!-- /GENERATED:summary -->
@@ -155,7 +155,7 @@ createTracker({ providers: ['ParcelsApp', 'Ship24', '17TRACK', 'Postal Ninja', '
 <!-- GENERATED:stages -->
 <img src="docs/assets/stages.svg" alt="DHL: Die Sendung wurde in das Zustellfahrzeug geladen.; Mondial Relay: En cours de livraison; Correios Brazil: Objeto saiu para entrega ao destinatário; Correos Express: EN REPARTO; Yamato Transport: 配達中; La Poste / Colissimo: DISTOU. All are filed under out_for_delivery." width="760">
 
-The carrier folders record 9,582 such statuses, each filed under one stage.
+The carrier folders record 9,604 such statuses, each filed under one stage.
 <!-- /GENERATED:stages -->
 
 Wording nobody recorded yet goes through a classifier that reads seven European languages.

package/data/catalog.json

@@ -7692,7 +7692,9 @@
     "timezone": "Australia/Sydney",
     "tracking": {
       "mode": "automatic",
-      "adapter": "universal"
+      "adapter": "couriersplease",
+      "recognitionRank": 961,
+      "localClocks": true
     },
     "canaryUrl": "https://www.couriersplease.com.au/tools-track",
     "trackingUrlTemplate": "https://www.couriersplease.com.au/tools-track?no={trackingNumber}",
@@ -10264,10 +10266,23 @@
     "timezone": "Asia/Kuala_Lumpur",
     "tracking": {
       "mode": "automatic",
-      "adapter": "universal"
+      "adapter": "gdex",
+      "recognitionRank": 957
     },
-    "canaryUrl": "https://my.gdexpress.com/",
+    "canaryUrl": "https://app.gdexpress.com/",
+    "trackingUrlTemplate": "https://gdexpress.com/tracking/?consignmentno={trackingNumber}",
+    "trackingSearchUrl": "https://gdexpress.com/our-business/delivery/tools-references/tracking/",
     "linkRules": [
+      {
+        "domains": [
+          "gdexpress.com",
+          "www.gdexpress.com"
+        ],
+        "pathPattern": "^/tracking/?$",
+        "params": [
+          "consignmentno"
+        ]
+      },
       {
         "domains": [
           "web2.gdexpress.com"
@@ -12685,7 +12700,8 @@
     "timezone": "Asia/Karachi",
     "tracking": {
       "mode": "automatic",
-      "adapter": "universal"
+      "adapter": "leopards-courier",
+      "recognitionRank": 965
     },
     "canaryUrl": "https://pk.leopardscourier.com/",
     "trackingSearchUrl": "https://pk.leopardscourier.com/",
@@ -15151,7 +15167,9 @@
     "timezone": "Asia/Jakarta",
     "tracking": {
       "mode": "automatic",
-      "adapter": "universal"
+      "adapter": "pos-indonesia",
+      "recognitionRank": 964,
+      "localClocks": true
     },
     "canaryUrl": "https://www.posindonesia.co.id/id/tracking",
     "trackingUrlTemplate": "https://www.posindonesia.co.id/id/tracking/{trackingNumber}",

package/data/coverage-summary.json

@@ -1,6 +1,6 @@
 {
   "catalog": 494,
-  "activeAdapters": 256,
+  "activeAdapters": 260,
   "countries": 113,
   "referenceCarriers": 100,
   "comparison": {

package/data/detection-golden.json

@@ -51340,6 +51340,17 @@
       "chronopost"
     ]
   },
+  {
+    "input": "EE205187475BN",
+    "carrier": "intl-post",
+    "confidence": "high",
+    "candidates": [
+      "intl-post"
+    ],
+    "asked": [
+      "chronopost"
+    ]
+  },
   {
     "input": "EE212938475ID",
     "carrier": "intl-post",
@@ -57016,6 +57027,14 @@
       "four-px"
     ]
   },
+  {
+    "input": "LP107221675JP",
+    "carrier": "japan-post",
+    "confidence": "high",
+    "candidates": [
+      "japan-post"
+    ]
+  },
   {
     "input": "LP157079155HK",
     "carrier": "hongkong-post",
@@ -58651,6 +58670,22 @@
       "tnt-it"
     ]
   },
+  {
+    "input": "MY37279142960",
+    "carrier": "gdex",
+    "confidence": "high",
+    "candidates": [
+      "gdex"
+    ]
+  },
+  {
+    "input": "MY37324153968",
+    "carrier": "gdex",
+    "confidence": "high",
+    "candidates": [
+      "gdex"
+    ]
+  },
   {
     "input": "MY40000807898",
     "carrier": "gdex",
@@ -59769,6 +59804,9 @@
     "confidence": "low",
     "candidates": [
       "pos-indonesia"
+    ],
+    "asked": [
+      "pos-indonesia"
     ]
   },
   {
@@ -59794,6 +59832,9 @@
     "confidence": "low",
     "candidates": [
       "pos-indonesia"
+    ],
+    "asked": [
+      "pos-indonesia"
     ]
   },
   {
@@ -59802,6 +59843,9 @@
     "confidence": "low",
     "candidates": [
       "pos-indonesia"
+    ],
+    "asked": [
+      "pos-indonesia"
     ]
   },
   {
@@ -59810,6 +59854,9 @@
     "confidence": "low",
     "candidates": [
       "pos-indonesia"
+    ],
+    "asked": [
+      "pos-indonesia"
     ]
   },
   {
@@ -67982,6 +68029,9 @@
     "confidence": "low",
     "candidates": [
       "pos-indonesia"
+    ],
+    "asked": [
+      "pos-indonesia"
     ]
   },
   {

package/dist/browser/scraper.js

@@ -6050,7 +6050,9 @@ var Ae = {
 		timezone: "Australia/Sydney",
 		tracking: {
 			mode: "automatic",
-			adapter: "universal"
+			adapter: "couriersplease",
+			recognitionRank: 961,
+			localClocks: !0
 		},
 		canaryUrl: "https://www.couriersplease.com.au/tools-track",
 		trackingUrlTemplate: "https://www.couriersplease.com.au/tools-track?no={trackingNumber}",
@@ -7951,10 +7953,17 @@ var Ae = {
 		timezone: "Asia/Kuala_Lumpur",
 		tracking: {
 			mode: "automatic",
-			adapter: "universal"
+			adapter: "gdex",
+			recognitionRank: 957
 		},
-		canaryUrl: "https://my.gdexpress.com/",
+		canaryUrl: "https://app.gdexpress.com/",
+		trackingUrlTemplate: "https://gdexpress.com/tracking/?consignmentno={trackingNumber}",
+		trackingSearchUrl: "https://gdexpress.com/our-business/delivery/tools-references/tracking/",
 		linkRules: [{
+			domains: ["gdexpress.com", "www.gdexpress.com"],
+			pathPattern: "^/tracking/?$",
+			params: ["consignmentno"]
+		}, {
 			domains: ["web2.gdexpress.com"],
 			pathPattern: "^/official/iframe/etracking(?:2|_v2)\\.php$",
 			params: ["capture"]
@@ -9749,7 +9758,8 @@ var Ae = {
 		timezone: "Asia/Karachi",
 		tracking: {
 			mode: "automatic",
-			adapter: "universal"
+			adapter: "leopards-courier",
+			recognitionRank: 965
 		},
 		canaryUrl: "https://pk.leopardscourier.com/",
 		trackingSearchUrl: "https://pk.leopardscourier.com/",
@@ -11567,7 +11577,9 @@ var Ae = {
 		timezone: "Asia/Jakarta",
 		tracking: {
 			mode: "automatic",
-			adapter: "universal"
+			adapter: "pos-indonesia",
+			recognitionRank: 964,
+			localClocks: !0
 		},
 		canaryUrl: "https://www.posindonesia.co.id/id/tracking",
 		trackingUrlTemplate: "https://www.posindonesia.co.id/id/tracking/{trackingNumber}",
@@ -21041,6 +21053,7 @@ var nl = {
 	"correo-argentino": 978,
 	"correos-de-mexico": 30,
 	"correos-express": 800,
+	couriersplease: 961,
 	"ctt-express": 790,
 	"cyprus-post": 60,
 	"dai-post": 62,
@@ -21074,6 +21087,7 @@ var nl = {
 	"four-px": 570,
 	foxpost: 200,
 	gati: 43,
+	gdex: 957,
 	geis: 90,
 	"geniki-taxydromiki": 140,
 	"geodis-pl": 994,
@@ -21100,6 +21114,7 @@ var nl = {
 	"kuehne-nagel": 435,
 	"la-poste": 1e3,
 	"latvijas-pasts": 100,
+	"leopards-courier": 965,
 	"lietuvos-pastas": 895,
 	logen: 48,
 	"loomis-express": 285,
@@ -21124,6 +21139,7 @@ var nl = {
 	paquetexpress: 35,
 	passport: 55,
 	"poczta-polska": 750,
+	"pos-indonesia": 964,
 	"post-luxembourg": 80,
 	"posta-slovenije": 130,
 	"posta-srbije": 110,

package/dist/carriers/couriersplease/adapter.d.ts

@@ -0,0 +1,17 @@
+import { type AdapterFactory, type TrackingContext } from '../../core/adapter/index.js';
+import type { CarrierResult } from '../../core/result/index.js';
+import { type StepRecorder } from '../../core/telemetry/index.js';
+/** The tracking page, which opens the session its lookup needs. */
+export declare const COURIERSPLEASE_PAGE = "https://www.couriersplease.com.au/tools-track";
+/** The lookup the tracking page makes for the numbers typed in. */
+export declare const COURIERSPLEASE_API = "https://www.couriersplease.com.au/api/track/locate";
+export declare class CouriersPleaseTracker {
+    private readonly options;
+    constructor(options?: {
+        fetcher?: typeof fetch;
+        recorder?: StepRecorder;
+        userAgent?: string;
+    });
+    fetch(raw: string, context?: TrackingContext): Promise<CarrierResult>;
+}
+export declare const adapter: AdapterFactory;

package/dist/carriers/couriersplease/adapter.js

@@ -0,0 +1,111 @@
+import { accepted, recognizeFromLookup } from '../../core/adapter/index.js';
+import { ChallengeError, IndeterminateError, RateLimitedError, SchemaError, TransportError } from '../../core/errors/index.js';
+import { runSteps } from '../../core/runner/index.js';
+import { firstAttemptMs, networkRetryStep } from '../../core/runner/networkRetry.js';
+import { NOOP_RECORDER } from '../../core/telemetry/index.js';
+import { decodeText, fetchBounded, userAgentOf } from '../../core/transport/index.js';
+import { COURIERSPLEASE_MAX_BYTES, couriersPleaseToken, normalizeCouriersPleaseNumber, parseCouriersPlease, PROVIDER } from './parser.js';
+/** The tracking page, which opens the session its lookup needs. */
+export const COURIERSPLEASE_PAGE = 'https://www.couriersplease.com.au/tools-track';
+/** The lookup the tracking page makes for the numbers typed in. */
+export const COURIERSPLEASE_API = 'https://www.couriersplease.com.au/api/track/locate';
+const ORIGIN = 'https://www.couriersplease.com.au';
+const PAGE_MAX_BYTES = 512 * 1024;
+const REDIRECTS = [301, 302, 303, 307, 308];
+const GONE = [404, 410];
+// Azure Front Door's own page when its firewall refuses a request: a 429 for
+// too many requests at once, a 403 for a blocked client.
+const BLOCKED = /<title>\s*Service unavailable\s*<\/title>|The request is blocked\./i;
+// Bot checks a CDN could put in place of the page or the reply.
+const CHALLENGE = /_cf_chl_opt|cf-turnstile|<title>\s*Just a moment|challenge-platform/i;
+// The session cookie the page sets beside its token.
+const SESSION = /^__Host-csrf=[A-Za-z0-9._~+/=-]{8,512}$/;
+function retryAfterMs(response) {
+    const value = response.headers.get('retry-after')?.trim() ?? '';
+    return /^\d{1,5}$/.test(value) ? Number(value) * 1000 : undefined;
+}
+/** A refusal before any reply: a rate limit, a firewall block or a moved route. */
+function refusal(response, body, stage) {
+    if (response.status === 429) {
+        throw new RateLimitedError(PROVIDER, retryAfterMs(response), `CouriersPlease rate limited its ${stage}`, { status: 429 });
+    }
+    if (REDIRECTS.includes(response.status) || GONE.includes(response.status)) {
+        throw new TransportError(PROVIDER, `CouriersPlease ${stage} is unavailable`, { status: response.status });
+    }
+    if (BLOCKED.test(body) || CHALLENGE.test(body))
+        throw new ChallengeError(PROVIDER, `CouriersPlease blocked its ${stage}`, { status: response.status });
+    throw new ChallengeError(PROVIDER, `CouriersPlease refused its ${stage}`, { status: response.status });
+}
+function json(body) {
+    try {
+        return JSON.parse(body);
+    }
+    catch (cause) {
+        if (BLOCKED.test(body.slice(0, 64_000)) || CHALLENGE.test(body.slice(0, 64_000))) {
+            throw new ChallengeError(PROVIDER, 'CouriersPlease answered with a block page', { cause });
+        }
+        throw new SchemaError(PROVIDER, 'CouriersPlease returned a reply that is not JSON', { cause });
+    }
+}
+export class CouriersPleaseTracker {
+    options;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    async fetch(raw, context = {}) {
+        const number = normalizeCouriersPleaseNumber(raw);
+        const userAgent = userAgentOf(this.options.userAgent);
+        const read = async (signal, timeoutMs) => {
+            // The page and the lookup share the attempt's time.
+            const deadline = performance.now() + timeoutMs;
+            const left = () => Math.max(1, Math.floor(deadline - performance.now()));
+            const page = await fetchBounded(COURIERSPLEASE_PAGE, { method: 'GET', signal, headers: {
+                    Accept: 'text/html,application/xhtml+xml', 'User-Agent': userAgent,
+                } }, { provider: PROVIDER, timeoutMs: left(), maxBytes: PAGE_MAX_BYTES, redirect: 'manual', fetcher: this.options.fetcher,
+                allowHttpStatuses: [...REDIRECTS, ...GONE, 403, 429] });
+            const html = decodeText(page.bytes);
+            if (page.response.status !== 200)
+                refusal(page.response, html.slice(0, 64_000), 'tracking page');
+            if (BLOCKED.test(html.slice(0, 64_000)) || CHALLENGE.test(html.slice(0, 64_000))) {
+                throw new ChallengeError(PROVIDER, 'CouriersPlease answered the tracking page with a block page');
+            }
+            // The lookup takes the page's token with the cookie it was minted for.
+            const token = couriersPleaseToken(html);
+            const cookie = page.response.headers.getSetCookie().map((header) => header.split(';', 1)[0].trim()).find((part) => SESSION.test(part));
+            if (!token || !cookie)
+                throw new SchemaError(PROVIDER, 'CouriersPlease tracking page opened no session');
+            const { response, bytes } = await fetchBounded(COURIERSPLEASE_API, { method: 'POST', signal, headers: {
+                    Accept: 'application/json', 'Content-Type': 'application/json', 'User-Agent': userAgent, Origin: ORIGIN,
+                    // The service answers only requests its own pages make, as browsers mark them.
+                    'Sec-Fetch-Site': 'same-origin', 'csrf-token': token, Cookie: cookie,
+                }, body: JSON.stringify({ trackingCode: number }) }, { provider: PROVIDER, timeoutMs: left(), maxBytes: COURIERSPLEASE_MAX_BYTES,
+                redirect: 'manual', fetcher: this.options.fetcher, allowHttpStatuses: [...REDIRECTS, ...GONE, 400, 403, 429] });
+            const body = decodeText(bytes);
+            if (response.status === 400 || response.status === 403) {
+                // The session the page just opened was refused: a fresh one is the only remedy.
+                if (/CSRF/i.test(body.slice(0, 4_000)))
+                    throw new IndeterminateError(PROVIDER, 'CouriersPlease refused the tracking page’s session');
+                if (response.status === 400 && !BLOCKED.test(body.slice(0, 4_000)))
+                    throw new IndeterminateError(PROVIDER, 'CouriersPlease did not take the lookup');
+            }
+            if (response.status !== 200)
+                refusal(response, body.slice(0, 64_000), 'tracking lookup');
+            return parseCouriersPlease(json(body), number);
+        };
+        // A page or lookup that fails to reach CouriersPlease, or hangs, gets one
+        // more try with a new session: the first gets half of the budget.
+        return runSteps({ carrier: 'couriersplease', budgetMs: context.budgetMs ?? 20_000, signal: context.signal,
+            recorder: this.options.recorder ?? NOOP_RECORDER }, [
+            { id: 'direct', run: ({ signal, remainingMs }) => read(signal, firstAttemptMs(remainingMs)) },
+            networkRetryStep(({ signal, remainingMs }) => read(signal, remainingMs)),
+        ]);
+    }
+}
+export const adapter = (environment) => {
+    const tracker = new CouriersPleaseTracker({ fetcher: environment.fetcher, recorder: environment.recorder, userAgent: environment.userAgent });
+    return {
+        id: 'couriersplease', recordsSteps: true, steps: ['direct', 'retry'],
+        track: (input, context) => tracker.fetch(input.number, context),
+        recognize: (number, context) => recognizeFromLookup(() => tracker.fetch(number, context), () => accepted(() => normalizeCouriersPleaseNumber(number))),
+    };
+};

package/dist/carriers/couriersplease/carrier.json

@@ -17,18 +17,62 @@
   "timezone": "Australia/Sydney",
   "tracking": {
     "mode": "automatic",
-    "adapter": "universal",
-    "steps": []
+    "adapter": "couriersplease",
+    "steps": [
+      "direct",
+      "retry"
+    ],
+    "recognition": {
+      "rank": 961
+    },
+    "localClocks": true
   },
-  "capabilities": [],
+  "capabilities": [
+    "history",
+    "location",
+    "provider_code",
+    "eta"
+  ],
   "portal": {
     "url": "https://www.couriersplease.com.au/tools-track?no={trackingNumber}",
     "searchUrl": "https://www.couriersplease.com.au/tools-track",
     "canaryUrl": "https://www.couriersplease.com.au/tools-track",
-    "shows": [],
-    "scraped": [],
-    "discarded": [],
-    "unavailable": []
+    "shows": [
+      "status",
+      "history",
+      "location",
+      "provider_code",
+      "eta",
+      "delivered_at",
+      "parcel_count",
+      "weight",
+      "volume",
+      "customer_reference",
+      "recipient_name",
+      "driver_name",
+      "depot_code"
+    ],
+    "scraped": [
+      "status",
+      "history",
+      "location",
+      "provider_code",
+      "eta"
+    ],
+    "discarded": [
+      "delivered_at",
+      "parcel_count",
+      "weight",
+      "volume",
+      "customer_reference",
+      "recipient_name",
+      "driver_name",
+      "depot_code"
+    ],
+    "unavailable": [
+      "service_name",
+      "offset_timestamps"
+    ]
   },
   "links": [
     {

package/dist/carriers/couriersplease/parser.d.ts

@@ -0,0 +1,22 @@
+import type { CarrierResult } from '../../core/result/index.js';
+export declare const PROVIDER = "CouriersPlease";
+export declare const COURIERSPLEASE_MAX_BYTES: number;
+/** A number for CouriersPlease's tracker, in upper case; anything else is not one it issues. */
+export declare function normalizeCouriersPleaseNumber(raw: string): string;
+/** The session token the tracking page puts in its `csrf-token` meta tag, or null. */
+export declare function couriersPleaseToken(html: string): string | null;
+/**
+ * A scan's wording without its markup. A line break becomes the end of a
+ * sentence, as on the page, and a sentence that points to a link, such as
+ * "Please contact our Customer Support Team here", is left out.
+ */
+export declare function couriersPleaseWording(raw: string): string;
+/**
+ * The reply of the tracking page's `/api/track/locate` for one number. Its one
+ * entry holds the consignment's details, when the number is a consignment, and
+ * each item's rows, the first naming the item. A consignment must be the
+ * number asked, or an item must: a label or coupon has no consignment details.
+ * An unknown number gets no rows ("No scan events found", or an empty list),
+ * which the page reports as not a valid tracking number.
+ */
+export declare function parseCouriersPlease(payload: unknown, raw: string): CarrierResult;