universal-parcel-scraper 0.158.0-main.1233 → 0.159.0-main.1237

package/README.md

@@ -13,7 +13,7 @@ The engine behind [Peek](https://peektracker.com) ([GitHub](https://github.com/p
 [![License: Apache-2.0](https://img.shields.io/badge/core-Apache--2.0-blue.svg)](LICENSE)
 
 <!-- GENERATED:summary -->
-**3,500+ carriers** through **218 dedicated adapters** and **5 universal fallbacks**
+**3,500+ carriers** through **221 dedicated adapters** and **5 universal fallbacks**
 
 <sub>358 catalog entries · 91 countries represented</sub>
 <!-- /GENERATED:summary -->
@@ -155,7 +155,7 @@ createTracker({ providers: ['ParcelsApp', 'Ship24', '17TRACK', 'Postal Ninja', '
 <!-- GENERATED:stages -->
 <img src="docs/assets/stages.svg" alt="DHL: Die Sendung wurde in das Zustellfahrzeug geladen.; Mondial Relay: En cours de livraison; Correios Brazil: Objeto saiu para entrega ao destinatário; Correos Express: EN REPARTO; Yamato Transport: 配達中; La Poste / Colissimo: DISTOU. All are filed under out_for_delivery." width="760">
 
-The carrier folders record 7,642 such statuses, each filed under one stage.
+The carrier folders record 7,762 such statuses, each filed under one stage.
 <!-- /GENERATED:stages -->
 
 Wording nobody recorded yet goes through a classifier that reads seven European languages.

package/data/catalog.json

@@ -8858,7 +8858,8 @@
     "timezone": "Asia/Kolkata",
     "tracking": {
       "mode": "automatic",
-      "adapter": "universal"
+      "adapter": "gati",
+      "recognitionRank": 43
     },
     "canaryUrl": "https://www.allcargologistics.com/track-shipment",
     "trackingSearchUrl": "https://www.allcargologistics.com/track-shipment",
@@ -13037,7 +13038,8 @@
     "timezone": "Asia/Kolkata",
     "tracking": {
       "mode": "automatic",
-      "adapter": "universal"
+      "adapter": "shadowfax",
+      "recognitionRank": 595
     },
     "canaryUrl": "https://www.shadowfax.in/track",
     "trackingUrlTemplate": "https://tracker.shadowfax.in/#/awb/{trackingNumber}",
@@ -14043,11 +14045,37 @@
     "timezone": "America/Sao_Paulo",
     "tracking": {
       "mode": "automatic",
-      "adapter": "universal"
+      "adapter": "total-express",
+      "recognitionRank": 975,
+      "localClocks": true
     },
-    "canaryUrl": "https://totalexpress.com.br/rastrear-um-produto/",
-    "linkRules": [],
-    "detectionRules": []
+    "canaryUrl": "https://totalconecta.totalexpress.com.br/rastreamento",
+    "trackingUrlTemplate": "https://totalconecta.totalexpress.com.br/rastreamento?codigo={trackingNumber}",
+    "trackingSearchUrl": "https://totalconecta.totalexpress.com.br/rastreamento",
+    "linkRules": [
+      {
+        "domains": [
+          "totalconecta.totalexpress.com.br"
+        ],
+        "path": "^/rastreamento/rastreamento/encomendas/([^/?#]+)/?$"
+      },
+      {
+        "domains": [
+          "totalconecta.totalexpress.com.br"
+        ],
+        "pathPattern": "^/rastreamento/?$",
+        "params": [
+          "codigo"
+        ],
+        "ownNumbersOnly": true
+      }
+    ],
+    "detectionRules": [
+      {
+        "pattern": "^(?:TX[A-Z]{2}|AMZB)\\d{9}TX$",
+        "confidence": "high"
+      }
+    ]
   },
   "trackon": {
     "displayName": "Trackon Couriers",

package/data/coverage-summary.json

@@ -1,6 +1,6 @@
 {
   "catalog": 358,
-  "activeAdapters": 218,
+  "activeAdapters": 221,
   "countries": 91,
   "referenceCarriers": 100,
   "comparison": {

package/data/detection-golden.json

@@ -10096,6 +10096,12 @@
       "dhl-ecommerce-pl"
     ]
   },
+  {
+    "input": "139815455",
+    "carrier": "unknown",
+    "confidence": "none",
+    "candidates": []
+  },
   {
     "input": "13CF4B49016730467J",
     "carrier": "zeleris",
@@ -23536,6 +23542,20 @@
       "dhl-ecommerce-pl"
     ]
   },
+  {
+    "input": "412933252",
+    "carrier": "unknown",
+    "confidence": "low",
+    "candidates": [
+      "tnt"
+    ],
+    "preferred": [
+      "tnt"
+    ],
+    "asked": [
+      "tnt"
+    ]
+  },
   {
     "input": "41505910017980",
     "carrier": "unknown",
@@ -37360,6 +37380,12 @@
       "brt"
     ]
   },
+  {
+    "input": "988397715",
+    "carrier": "unknown",
+    "confidence": "none",
+    "candidates": []
+  },
   {
     "input": "988C01701257F",
     "carrier": "unknown",
@@ -38894,6 +38920,14 @@
       "mtscn"
     ]
   },
+  {
+    "input": "AMZB664764467tx",
+    "carrier": "total-express",
+    "confidence": "high",
+    "candidates": [
+      "total-express"
+    ]
+  },
   {
     "input": "AN725955",
     "carrier": "unknown",
@@ -51474,6 +51508,20 @@
     "confidence": "low",
     "candidates": [
       "shadowfax"
+    ],
+    "asked": [
+      "shadowfax"
+    ]
+  },
+  {
+    "input": "R1858467302TIU",
+    "carrier": "unknown",
+    "confidence": "low",
+    "candidates": [
+      "shadowfax"
+    ],
+    "asked": [
+      "shadowfax"
     ]
   },
   {
@@ -53556,6 +53604,12 @@
       "shadowfax"
     ]
   },
+  {
+    "input": "SF1904798183E",
+    "carrier": "unknown",
+    "confidence": "none",
+    "candidates": []
+  },
   {
     "input": "SF211225399443FPL",
     "carrier": "shadowfax",
@@ -53572,6 +53626,22 @@
       "shadowfax"
     ]
   },
+  {
+    "input": "SF2917969609ARA",
+    "carrier": "shadowfax",
+    "confidence": "high",
+    "candidates": [
+      "shadowfax"
+    ]
+  },
+  {
+    "input": "SF3033213302ARA",
+    "carrier": "shadowfax",
+    "confidence": "high",
+    "candidates": [
+      "shadowfax"
+    ]
+  },
   {
     "input": "SF332188411FPL",
     "carrier": "shadowfax",
@@ -53596,6 +53666,28 @@
       "shadowfax"
     ]
   },
+  {
+    "input": "SF3667163853YWR",
+    "carrier": "shadowfax",
+    "confidence": "high",
+    "candidates": [
+      "shadowfax"
+    ]
+  },
+  {
+    "input": "SF3772148429KR",
+    "carrier": "unknown",
+    "confidence": "none",
+    "candidates": []
+  },
+  {
+    "input": "SF4073772682DEU",
+    "carrier": "shadowfax",
+    "confidence": "high",
+    "candidates": [
+      "shadowfax"
+    ]
+  },
   {
     "input": "SF6042542297687",
     "carrier": "sf-express",
@@ -54761,6 +54853,46 @@
       "tsjy"
     ]
   },
+  {
+    "input": "TXAQ243268174TX",
+    "carrier": "total-express",
+    "confidence": "high",
+    "candidates": [
+      "total-express"
+    ]
+  },
+  {
+    "input": "TXAQ402565477tx",
+    "carrier": "total-express",
+    "confidence": "high",
+    "candidates": [
+      "total-express"
+    ]
+  },
+  {
+    "input": "TXAQ549180149tx",
+    "carrier": "total-express",
+    "confidence": "high",
+    "candidates": [
+      "total-express"
+    ]
+  },
+  {
+    "input": "TXAQ589033943TX",
+    "carrier": "total-express",
+    "confidence": "high",
+    "candidates": [
+      "total-express"
+    ]
+  },
+  {
+    "input": "TXAS426107751tx",
+    "carrier": "total-express",
+    "confidence": "high",
+    "candidates": [
+      "total-express"
+    ]
+  },
   {
     "input": "TYPQYU501214190",
     "carrier": "unknown",

package/dist/browser/scraper.js

@@ -6903,7 +6903,8 @@ var Ae = {
 		timezone: "Asia/Kolkata",
 		tracking: {
 			mode: "automatic",
-			adapter: "universal"
+			adapter: "gati",
+			recognitionRank: 43
 		},
 		canaryUrl: "https://www.allcargologistics.com/track-shipment",
 		trackingSearchUrl: "https://www.allcargologistics.com/track-shipment",
@@ -10043,7 +10044,8 @@ var Ae = {
 		timezone: "Asia/Kolkata",
 		tracking: {
 			mode: "automatic",
-			adapter: "universal"
+			adapter: "shadowfax",
+			recognitionRank: 595
 		},
 		canaryUrl: "https://www.shadowfax.in/track",
 		trackingUrlTemplate: "https://tracker.shadowfax.in/#/awb/{trackingNumber}",
@@ -10762,11 +10764,26 @@ var Ae = {
 		timezone: "America/Sao_Paulo",
 		tracking: {
 			mode: "automatic",
-			adapter: "universal"
+			adapter: "total-express",
+			recognitionRank: 975,
+			localClocks: !0
 		},
-		canaryUrl: "https://totalexpress.com.br/rastrear-um-produto/",
-		linkRules: [],
-		detectionRules: []
+		canaryUrl: "https://totalconecta.totalexpress.com.br/rastreamento",
+		trackingUrlTemplate: "https://totalconecta.totalexpress.com.br/rastreamento?codigo={trackingNumber}",
+		trackingSearchUrl: "https://totalconecta.totalexpress.com.br/rastreamento",
+		linkRules: [{
+			domains: ["totalconecta.totalexpress.com.br"],
+			path: "^/rastreamento/rastreamento/encomendas/([^/?#]+)/?$"
+		}, {
+			domains: ["totalconecta.totalexpress.com.br"],
+			pathPattern: "^/rastreamento/?$",
+			params: ["codigo"],
+			ownNumbersOnly: !0
+		}],
+		detectionRules: [{
+			pattern: "^(?:TX[A-Z]{2}|AMZB)\\d{9}TX$",
+			confidence: "high"
+		}]
 	},
 	trackon: {
 		displayName: "Trackon Couriers",
@@ -11965,7 +11982,7 @@ var Ae = {
 	"tforce-freight": [],
 	thhy: ["thhy-1"],
 	toptrans: [],
-	"total-express": [],
+	"total-express": ["total-express-awb"],
 	trackon: [],
 	"trans-o-flex": [],
 	"trendyol-express": [],
@@ -17118,6 +17135,7 @@ var nl = {
 	"fan-courier": 330,
 	"four-px": 570,
 	foxpost: 200,
+	gati: 43,
 	geis: 90,
 	"geniki-taxydromiki": 140,
 	"geodis-pl": 994,
@@ -17178,6 +17196,7 @@ var nl = {
 	saia: 12,
 	sameday: 370,
 	seur: 870,
+	shadowfax: 595,
 	"slovak-parcel-service": 180,
 	"slovenska-posta": 170,
 	speedaf: 989,
@@ -17189,6 +17208,7 @@ var nl = {
 	thhy: 987,
 	tipsa: 680,
 	tnt: 880,
+	"total-express": 975,
 	tsjy: 992,
 	"ubi-smart-parcel": 45,
 	ukrposhta: 490,

package/dist/carriers/gati/adapter.d.ts

@@ -0,0 +1,20 @@
+import { type AdapterFactory, type TrackingContext } from '../../core/adapter/index.js';
+import type { CarrierResult } from '../../core/result/index.js';
+import { type StepRecorder } from '../../core/telemetry/index.js';
+/** The page token the tracking page on www.allcargologistics.com asks for when it loads. */
+export declare const GATI_TOKEN_URL = "https://admin.allcargologistics.com/api/frontend/token";
+/** The lookup the tracking page makes for the docket number typed in. */
+export declare const GATI_TRACKING_URL = "https://admin.allcargologistics.com/api/track-shipment";
+export declare class GatiTracker {
+    private readonly options;
+    constructor(options?: {
+        fetcher?: typeof fetch;
+        recorder?: StepRecorder;
+        userAgent?: string;
+    });
+    fetch(raw: string, context?: TrackingContext): Promise<CarrierResult>;
+    private read;
+    /** A page token, as the tracking page gets one on every visit; it lasts five minutes. */
+    private token;
+}
+export declare const adapter: AdapterFactory;

package/dist/carriers/gati/adapter.js

@@ -0,0 +1,105 @@
+import { accepted, recognizeFromLookup } from '../../core/adapter/index.js';
+import { ChallengeError, SchemaError, TransportError, UpstreamHttpError, UpstreamNetworkError } from '../../core/errors/index.js';
+import { runSteps } from '../../core/runner/index.js';
+import { NOOP_RECORDER } from '../../core/telemetry/index.js';
+import { decodeText, fetchBounded, userAgentOf } from '../../core/transport/index.js';
+import { isRecord } from '../../core/types.js';
+import { GATI_MAX_BYTES, normalizeGatiNumber, parseGati, PROVIDER } from './parser.js';
+/** The page token the tracking page on www.allcargologistics.com asks for when it loads. */
+export const GATI_TOKEN_URL = 'https://admin.allcargologistics.com/api/frontend/token';
+/** The lookup the tracking page makes for the docket number typed in. */
+export const GATI_TRACKING_URL = 'https://admin.allcargologistics.com/api/track-shipment';
+// The site answers the lookup only for its own pages.
+const ORIGIN = 'https://www.allcargologistics.com';
+const REDIRECTS = [301, 302, 303, 307, 308];
+// Bot checks a CDN or firewall could put in place of the reply.
+const CHALLENGE = /_cf_chl_opt|cf-turnstile|<title>\s*Just a moment|AwsWafIntegration|awswaf\.com|g-recaptcha|h-captcha/i;
+// The token is a number, a bar and forty letters and digits.
+const TOKEN = /^[\w|.~+/=-]{8,512}$/;
+function json(bytes) {
+    const body = decodeText(bytes);
+    try {
+        return JSON.parse(body);
+    }
+    catch (cause) {
+        if (CHALLENGE.test(body.slice(0, 64_000)))
+            throw new ChallengeError(PROVIDER, 'AllcargoGATI answered with a browser check', { cause });
+        throw new SchemaError(PROVIDER, 'AllcargoGATI returned a reply that is not JSON', { cause });
+    }
+}
+/**
+ * The same failure without the request details fetchBounded attaches: they
+ * hold the page token.
+ */
+function withoutToken(error) {
+    if (error instanceof UpstreamHttpError)
+        return new UpstreamHttpError(error.provider, error.status, error.retryAfterMs);
+    if (error instanceof UpstreamNetworkError)
+        return new UpstreamNetworkError(error.provider, error.cause);
+    return error;
+}
+export class GatiTracker {
+    options;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    async fetch(raw, context = {}) {
+        const number = normalizeGatiNumber(raw);
+        // One read with the whole budget: the site waits up to thirty seconds for
+        // the tracking service behind it, which some dockets never answer.
+        return runSteps({ carrier: 'gati', budgetMs: context.budgetMs ?? 20_000, signal: context.signal,
+            recorder: this.options.recorder ?? NOOP_RECORDER }, [
+            { id: 'direct', run: ({ signal, remainingMs }) => this.read(number, signal, remainingMs) },
+        ]);
+    }
+    async read(number, signal, budgetMs) {
+        const deadline = performance.now() + budgetMs;
+        const timeoutMs = () => Math.max(1, Math.floor(deadline - performance.now()));
+        const userAgent = userAgentOf(this.options.userAgent);
+        const token = await this.token(userAgent, signal, timeoutMs());
+        signal.throwIfAborted();
+        try {
+            const { response, bytes } = await fetchBounded(GATI_TRACKING_URL, { method: 'POST', signal, headers: {
+                    Accept: 'application/json', 'Content-Type': 'application/json', Authorization: `Bearer ${token}`, Origin: ORIGIN, 'User-Agent': userAgent,
+                }, body: JSON.stringify({ docketNo: number }) }, { provider: PROVIDER, timeoutMs: timeoutMs(), maxBytes: GATI_MAX_BYTES,
+                redirect: 'manual', fetcher: this.options.fetcher, allowHttpStatuses: [...REDIRECTS, 401, 403, 404, 410, 422] });
+            if (REDIRECTS.includes(response.status))
+                throw new TransportError(PROVIDER, 'AllcargoGATI redirected its tracking lookup');
+            if (response.status === 404 || response.status === 410)
+                throw new TransportError(PROVIDER, 'AllcargoGATI tracking lookup is unavailable');
+            // The token was just issued and the origin is the site's own, so a refusal is a changed gate.
+            if (response.status === 401 || response.status === 403) {
+                throw new ChallengeError(PROVIDER, 'AllcargoGATI refused the tracking page token', { status: response.status });
+            }
+            // A refused number is HTTP 422 with the reasons, which the parser reads.
+            return parseGati(json(bytes), number);
+        }
+        catch (error) {
+            throw withoutToken(error);
+        }
+    }
+    /** A page token, as the tracking page gets one on every visit; it lasts five minutes. */
+    async token(userAgent, signal, timeoutMs) {
+        const { response, bytes } = await fetchBounded(GATI_TOKEN_URL, { method: 'POST', signal, headers: {
+                Accept: 'application/json', 'Content-Type': 'application/json', Origin: ORIGIN, 'User-Agent': userAgent,
+            }, body: '{}' }, { provider: PROVIDER, timeoutMs, maxBytes: 64_000, redirect: 'manual', fetcher: this.options.fetcher,
+            allowHttpStatuses: [...REDIRECTS, 404, 410] });
+        if (REDIRECTS.includes(response.status) || response.status === 404 || response.status === 410) {
+            throw new TransportError(PROVIDER, 'AllcargoGATI tracking page token is unavailable');
+        }
+        const grant = json(bytes);
+        const token = isRecord(grant) && typeof grant.token === 'string' ? grant.token : '';
+        if (!TOKEN.test(token) || !isRecord(grant) || String(grant.type).toLowerCase() !== 'bearer') {
+            throw new SchemaError(PROVIDER, 'AllcargoGATI did not issue a tracking page token');
+        }
+        return token;
+    }
+}
+export const adapter = (environment) => {
+    const tracker = new GatiTracker({ fetcher: environment.fetcher, recorder: environment.recorder, userAgent: environment.userAgent });
+    return {
+        id: 'gati', recordsSteps: true, steps: ['direct'],
+        track: (input, context) => tracker.fetch(input.number, context),
+        recognize: (number, context) => recognizeFromLookup(() => tracker.fetch(number, context), () => accepted(() => normalizeGatiNumber(number))),
+    };
+};

package/dist/carriers/gati/carrier.json

@@ -22,17 +22,59 @@
   "timezone": "Asia/Kolkata",
   "tracking": {
     "mode": "automatic",
-    "adapter": "universal",
-    "steps": []
+    "adapter": "gati",
+    "steps": [
+      "direct"
+    ],
+    "recognition": {
+      "rank": 43
+    }
   },
-  "capabilities": [],
+  "capabilities": [
+    "history",
+    "location",
+    "provider_code",
+    "delivered_at",
+    "eta",
+    "weight"
+  ],
   "portal": {
     "searchUrl": "https://www.allcargologistics.com/track-shipment",
     "canaryUrl": "https://www.allcargologistics.com/track-shipment",
-    "shows": [],
-    "scraped": [],
-    "discarded": [],
-    "unavailable": []
+    "shows": [
+      "status",
+      "history",
+      "origin",
+      "departure_date",
+      "destination",
+      "delivered_at",
+      "eta",
+      "recipient_name",
+      "remarks",
+      "proof_of_delivery"
+    ],
+    "scraped": [
+      "status",
+      "history",
+      "location",
+      "provider_code",
+      "delivered_at",
+      "eta",
+      "weight"
+    ],
+    "discarded": [
+      "origin",
+      "departure_date",
+      "destination",
+      "recipient_name",
+      "remarks",
+      "proof_of_delivery"
+    ],
+    "unavailable": [
+      "offset_timestamps",
+      "pickup_point",
+      "delivery_partner"
+    ]
   },
   "links": [
     {

package/dist/carriers/gati/numbers.json

@@ -88,7 +88,7 @@
         "access": "full_page"
       },
       "context": {
-        "assessment": "candidate"
+        "assessment": "confirmed"
       },
       "expect": {
         "carrier": "unknown",
@@ -107,7 +107,7 @@
         "access": "full_page"
       },
       "context": {
-        "assessment": "candidate"
+        "assessment": "confirmed"
       },
       "expect": {
         "carrier": "unknown",
@@ -126,7 +126,7 @@
         "access": "full_page"
       },
       "context": {
-        "assessment": "candidate"
+        "assessment": "confirmed"
       },
       "expect": {
         "carrier": "unknown",
@@ -145,7 +145,7 @@
         "access": "full_page"
       },
       "context": {
-        "assessment": "candidate"
+        "assessment": "confirmed"
       },
       "expect": {
         "carrier": "unknown",
@@ -164,7 +164,7 @@
         "access": "full_page"
       },
       "context": {
-        "assessment": "candidate"
+        "assessment": "confirmed"
       },
       "expect": {
         "carrier": "unknown",
@@ -183,7 +183,7 @@
         "access": "full_page"
       },
       "context": {
-        "assessment": "candidate"
+        "assessment": "confirmed"
       },
       "expect": {
         "carrier": "unknown",
@@ -202,7 +202,7 @@
         "access": "full_page"
       },
       "context": {
-        "assessment": "candidate"
+        "assessment": "confirmed"
       },
       "expect": {
         "carrier": "unknown",
@@ -247,6 +247,66 @@
         "confidence": "none"
       },
       "note": "A customer review on TrackingMore's GATI Courier review page asks for delivery of this number."
+    },
+    {
+      "number": "412933252",
+      "role": "shipment",
+      "evidence": "public_shipment_report",
+      "source": {
+        "url": "https://www.trackingmore.com/review-gati-kwe.html",
+        "date": "2021-11-15",
+        "dateKind": "comment",
+        "access": "full_page"
+      },
+      "context": {
+        "assessment": "confirmed"
+      },
+      "expect": {
+        "carrier": "unknown",
+        "confidence": "low",
+        "candidates": [
+          "tnt"
+        ]
+      },
+      "note": "A customer review on TrackingMore's Gati KWE review page asks where the courier with this number is."
+    },
+    {
+      "number": "988397715",
+      "role": "shipment",
+      "evidence": "public_shipment_report",
+      "source": {
+        "url": "https://www.trackingmore.com/review-gati-kwe.html",
+        "date": "2021-09-03",
+        "dateKind": "comment",
+        "access": "full_page"
+      },
+      "context": {
+        "assessment": "candidate"
+      },
+      "expect": {
+        "carrier": "unknown",
+        "confidence": "none"
+      },
+      "note": "A customer review on TrackingMore's Gati KWE review page gives it as a Gati courier number."
+    },
+    {
+      "number": "139815455",
+      "role": "shipment",
+      "evidence": "public_shipment_report",
+      "source": {
+        "url": "https://www.trackingmore.com/review-gaticn.html",
+        "date": "2020-12-16",
+        "dateKind": "comment",
+        "access": "full_page"
+      },
+      "context": {
+        "assessment": "confirmed"
+      },
+      "expect": {
+        "carrier": "unknown",
+        "confidence": "none"
+      },
+      "note": "A customer review on TrackingMore's GATI Courier review page gives it as the docket of a shipment that arrived on time."
     }
   ]
 }