universal-parcel-scraper 0.142.0 → 0.143.0-main.1167

package/README.md

@@ -155,7 +155,7 @@ createTracker({ providers: ['ParcelsApp', 'Ship24', '17TRACK', 'Postal Ninja', '
 <!-- GENERATED:stages -->
 <img src="docs/assets/stages.svg" alt="DHL: Die Sendung wurde in das Zustellfahrzeug geladen.; Mondial Relay: En cours de livraison; Correios Brazil: Objeto saiu para entrega ao destinatário; Correos Express: EN REPARTO; Yamato Transport: 配達中; La Poste / Colissimo: DISTOU. All are filed under out_for_delivery." width="760">
 
-The carrier folders record 7,282 such statuses, each filed under one stage.
+The carrier folders record 7,309 such statuses, each filed under one stage.
 <!-- /GENERATED:stages -->
 
 Wording nobody recorded yet goes through a classifier that reads seven European languages.

package/data/detection-golden.json

@@ -1008,6 +1008,22 @@
       "nz-post"
     ]
   },
+  {
+    "input": "00340433660000960875",
+    "carrier": "dhl",
+    "confidence": "high",
+    "candidates": [
+      "dhl"
+    ]
+  },
+  {
+    "input": "00340433660000963159",
+    "carrier": "dhl",
+    "confidence": "high",
+    "candidates": [
+      "dhl"
+    ]
+  },
   {
     "input": "00340433982034779798",
     "carrier": "dhl",
@@ -45404,6 +45420,28 @@
       "an-post"
     ]
   },
+  {
+    "input": "LGG6221779",
+    "carrier": "unknown",
+    "confidence": "low",
+    "candidates": [
+      "australia-post"
+    ],
+    "asked": [
+      "australia-post"
+    ]
+  },
+  {
+    "input": "LGG6221781",
+    "carrier": "unknown",
+    "confidence": "low",
+    "candidates": [
+      "australia-post"
+    ],
+    "asked": [
+      "australia-post"
+    ]
+  },
   {
     "input": "LH 38304147",
     "carrier": "ontrac",

package/dist/carriers/dsv/adapter.d.ts

@@ -1,21 +1,37 @@
 import type { AdapterFactory, TrackingContext } from '../../core/adapter/index.js';
 import type { CarrierResult } from '../../core/result/index.js';
 import { type StepRecorder } from '../../core/telemetry/index.js';
+import { dsvSchenkerRoute } from './schenker.js';
 export declare const DSV_SEARCH_ENDPOINT = "https://mydsv.com/app/search/publicShipmentList";
 export declare const DSV_SHIPMENT_ENDPOINT = "https://mydsv.com/app/search/shipment/";
 export declare const DSV_STATUSES_ENDPOINT = "https://mydsv.com/app/search/shipment/new-statuses/";
+export declare const DSV_SCHENKER_ENDPOINT = "https://www.dsv.com/nges-portal/api/public/tracking-public";
+type SchenkerRoute = NonNullable<ReturnType<typeof dsvSchenkerRoute>>;
 export declare class DsvTracker {
     private readonly options;
     constructor(options?: {
         fetcher?: typeof fetch;
         recorder?: StepRecorder;
         userAgent?: string;
+        schenkerMs?: Readonly<Record<SchenkerRoute, number>>;
     });
     /**
      * One JSON read. The shipment's key opens its details, so a failed read by
      * key keeps neither the address nor the reply on its error.
      */
     private json;
+    /**
+     * One read from the former DB Schenker tracker. It answers every request
+     * with HTTP 429 and a proof-of-work puzzle; the same request with the
+     * solution gets the reply. The solution stays out of failure diagnostics.
+     */
+    private schenker;
+    /**
+     * The former DB Schenker tracker: an STT's details straight away, or a
+     * package's search, then the details of the one road shipment it finds.
+     */
+    private schenkerShipment;
     fetch(raw: string, context?: TrackingContext): Promise<CarrierResult>;
 }
 export declare const adapter: AdapterFactory;
+export {};

package/dist/carriers/dsv/adapter.js

@@ -1,13 +1,26 @@
-import { ChallengeError, SchemaError, TransportError, UpstreamHttpError, UpstreamNetworkError } from '../../core/errors/index.js';
+import { BudgetExceededError, ChallengeError, IndeterminateError, SchemaError, TransportError, UpstreamHttpError, UpstreamNetworkError } from '../../core/errors/index.js';
 import { runSteps } from '../../core/runner/index.js';
 import { firstAttemptMs, networkRetryStep } from '../../core/runner/networkRetry.js';
 import { NOOP_RECORDER } from '../../core/telemetry/index.js';
 import { decodeText, fetchBounded, parseJsonBytes, userAgentOf } from '../../core/transport/index.js';
+import { isRecord } from '../../core/types.js';
 import { confirmDsvShipment, DSV_MAX_BYTES, normalizeDsvNumber, parseDsvStatuses, pickDsvShipment, PROVIDER } from './parser.js';
+import { parseDsvPuzzles, solveDsvPuzzles } from './puzzle.js';
+import { dsvSchenkerRoute, dsvSchenkerSttId, notFound, parseDsvSchenkerShipment, pickDsvSchenkerShipment } from './schenker.js';
 // The services the shipment page at mydsv.com/track-shipment calls without an account.
 export const DSV_SEARCH_ENDPOINT = 'https://mydsv.com/app/search/publicShipmentList';
 export const DSV_SHIPMENT_ENDPOINT = 'https://mydsv.com/app/search/shipment/';
 export const DSV_STATUSES_ENDPOINT = 'https://mydsv.com/app/search/shipment/new-statuses/';
+// The former DB Schenker tracker's API, behind the page at www.dsv.com/mydsv/tracking-public.
+export const DSV_SCHENKER_ENDPOINT = 'https://www.dsv.com/nges-portal/api/public/tracking-public';
+const DEFAULT_BUDGET_MS = 15_000;
+// The former DB Schenker tracker's longest wait. It answers within a second,
+// but at times holds an STT's details for half a minute, and a package's
+// search for a quarter of a minute before them; a later reply is given up,
+// so the sources after it keep their time.
+const SCHENKER_MS = { stt: 34_000, sscc: 48_000 };
+// The tracker's answers when the number matches none of its shipments or reference patterns.
+const SCHENKER_NOT_FOUND = /^TRACKING-BADREQ-(?:SHIPMENT_NOT_FOUND|NO_MATCHING_PATTERN_FOUND)$/;
 // A web application firewall or CDN block page.
 const BLOCKED = /<title>\s*(?:Request Rejected|Access Denied|Just a moment|Attention Required)|The requested URL was rejected|Incapsula incident|cf-chl-|cf-turnstile|_cf_chl_opt/i;
 export class DsvTracker {
@@ -44,6 +57,95 @@ export class DsvTracker {
             throw new SchemaError(PROVIDER, 'myDSV returned invalid tracking JSON', byKey ? undefined : { cause });
         }
     }
+    /**
+     * One read from the former DB Schenker tracker. It answers every request
+     * with HTTP 429 and a proof-of-work puzzle; the same request with the
+     * solution gets the reply. The solution stays out of failure diagnostics.
+     */
+    async schenker(path, signal, deadline, budgetMs) {
+        const url = `${DSV_SCHENKER_ENDPOINT}${path}`;
+        const headers = { Accept: 'application/json', 'Accept-Language': 'en', 'User-Agent': userAgentOf(this.options.userAgent) };
+        const left = () => {
+            const ms = Math.floor(deadline - performance.now());
+            if (ms < 1)
+                throw new BudgetExceededError(PROVIDER, budgetMs);
+            return ms;
+        };
+        const request = async (extra, allow) => {
+            try {
+                return await fetchBounded(url, { signal, headers: { ...headers, ...extra } }, { provider: PROVIDER, timeoutMs: left(), maxBytes: DSV_MAX_BYTES, allowHttpStatuses: allow, fetcher: this.options.fetcher });
+            }
+            catch (error) {
+                if (error instanceof UpstreamHttpError)
+                    throw new UpstreamHttpError(PROVIDER, error.status, error.retryAfterMs);
+                if (error instanceof UpstreamNetworkError) {
+                    // A reply still pending when the step's time or the lookup's ends is a spent budget, not a failure to connect.
+                    const spent = signal.aborted ? signal.reason instanceof DOMException && signal.reason.name === 'TimeoutError'
+                        : deadline - performance.now() < 100;
+                    if (spent)
+                        throw new BudgetExceededError(PROVIDER, budgetMs);
+                    throw new UpstreamNetworkError(PROVIDER, new Error('DSV\'s former DB Schenker tracker did not answer'));
+                }
+                throw error;
+            }
+        };
+        const json = (bytes) => {
+            try {
+                return parseJsonBytes(bytes, PROVIDER);
+            }
+            catch {
+                if (BLOCKED.test(decodeText(bytes.subarray(0, 64_000))))
+                    throw new ChallengeError(PROVIDER, 'DSV answered with a block page');
+                throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned invalid tracking JSON');
+            }
+        };
+        const first = await request({}, [404, 429]);
+        // A missing route is not a missing shipment.
+        if (first.response.status === 404)
+            throw new TransportError(PROVIDER, 'DSV\'s former DB Schenker tracker is unavailable');
+        if (first.response.status !== 429)
+            return json(first.bytes);
+        const puzzle = first.response.headers.get('captcha-puzzle');
+        if (!puzzle)
+            throw new UpstreamHttpError(PROVIDER, 429);
+        const solution = await solveDsvPuzzles(parseDsvPuzzles(puzzle, PROVIDER), signal, deadline, budgetMs, PROVIDER);
+        signal.throwIfAborted();
+        const { response, bytes } = await request({ 'Captcha-Solution': solution }, [400, 404, 422, 429]);
+        if (response.status === 200)
+            return json(bytes);
+        if (response.status === 404)
+            notFound();
+        if (response.status === 400) {
+            let payload;
+            try {
+                payload = parseJsonBytes(bytes, PROVIDER);
+            }
+            catch {
+                payload = null;
+            }
+            if (isRecord(payload) && typeof payload.code === 'string' && SCHENKER_NOT_FOUND.test(payload.code))
+                notFound();
+            throw new IndeterminateError(PROVIDER, 'DSV\'s former DB Schenker tracker refused the lookup');
+        }
+        // Another puzzle, or a refusal of the solution.
+        if (response.status === 422 || response.headers.get('captcha-puzzle')) {
+            throw new ChallengeError(PROVIDER, 'DSV\'s former DB Schenker tracker refused the solved puzzle');
+        }
+        throw new UpstreamHttpError(PROVIDER, response.status);
+    }
+    /**
+     * The former DB Schenker tracker: an STT's details straight away, or a
+     * package's search, then the details of the one road shipment it finds.
+     */
+    async schenkerShipment(number, route, signal, remainingMs) {
+        const budgetMs = Math.min(remainingMs, (this.options.schenkerMs ?? SCHENKER_MS)[route]);
+        const deadline = performance.now() + budgetMs;
+        const read = (path) => this.schenker(path, signal, deadline, budgetMs);
+        const id = route === 'stt' ? dsvSchenkerSttId(number)
+            : pickDsvSchenkerShipment(await read(`/shipments?${new URLSearchParams({ query: number })}`), number);
+        // The page sends the id as it is, colons and commas included.
+        return parseDsvSchenkerShipment(await read(`/shipments/land/${id}`), number, id);
+    }
     async fetch(raw, context = {}) {
         const number = normalizeDsvNumber(raw);
         const search = `${DSV_SEARCH_ENDPOINT}?${new URLSearchParams({ q: number })}`;
@@ -61,18 +163,27 @@ export class DsvTracker {
             return parseDsvStatuses(await this.json(`${DSV_STATUSES_ENDPOINT}${key}`, signal, left(), true), shipment);
         };
         // A lookup that fails to reach myDSV, or hangs, gets one more try: the
-        // first gets half of the budget, so the retry fits in the rest.
-        return runSteps({ carrier: 'dsv', budgetMs: context.budgetMs ?? 15_000, signal: context.signal,
+        // first gets half of the budget, so the retry fits in the rest. An STT or
+        // a package's SSCC that myDSV does not know goes to the former DB Schenker
+        // tracker.
+        const route = dsvSchenkerRoute(number);
+        return runSteps({ carrier: 'dsv', budgetMs: context.budgetMs ?? DEFAULT_BUDGET_MS, signal: context.signal,
             recorder: this.options.recorder ?? NOOP_RECORDER }, [
             { id: 'direct', run: ({ signal, remainingMs }) => read(signal, firstAttemptMs(remainingMs)) },
             networkRetryStep(({ signal, remainingMs }) => read(signal, remainingMs)),
+            { id: 'schenker', recovers: error => route !== null && notInMyDsv(error),
+                run: ({ signal, remainingMs }) => this.schenkerShipment(number, route, signal, remainingMs) },
         ]);
     }
 }
+/** myDSV's search answered and does not know the number. */
+function notInMyDsv(error) {
+    return error instanceof IndeterminateError && error.reason === 'portal_not_found';
+}
 export const adapter = (environment) => {
     const tracker = new DsvTracker({ fetcher: environment.fetcher, recorder: environment.recorder, userAgent: environment.userAgent });
     return {
-        id: 'dsv', recordsSteps: true, steps: ['direct', 'retry'],
+        id: 'dsv', recordsSteps: true, steps: ['direct', 'retry', 'schenker'],
         track: (input, context) => tracker.fetch(input.number, context),
     };
 };

package/dist/carriers/dsv/carrier.json

@@ -25,7 +25,8 @@
     "adapter": "dsv",
     "steps": [
       "direct",
-      "retry"
+      "retry",
+      "schenker"
     ],
     "localClocks": true
   },

package/dist/carriers/dsv/numbers.json

@@ -125,6 +125,80 @@
         ]
       },
       "note": "Example value in the tool schema of an open-source DB Schenker tracking client; not a real shipment."
+    },
+    {
+      "number": "LGG6221781",
+      "role": "shipment",
+      "evidence": "public_shipment_report",
+      "source": {
+        "url": "https://www.parcelpanel.com/couriers/dbschenker/",
+        "access": "full_page"
+      },
+      "context": {
+        "assessment": "confirmed"
+      },
+      "expect": {
+        "carrier": "unknown",
+        "confidence": "low",
+        "candidates": [
+          "australia-post"
+        ]
+      },
+      "note": "ParcelPanel lists it as an example on its DB Schenker page. DSV's former DB Schenker tracker knows it as the STT of a road shipment."
+    },
+    {
+      "number": "LGG6221779",
+      "role": "shipment",
+      "evidence": "public_shipment_report",
+      "source": {
+        "url": "https://www.parcelpanel.com/couriers/dbschenker/",
+        "access": "full_page"
+      },
+      "context": {
+        "assessment": "confirmed"
+      },
+      "expect": {
+        "carrier": "unknown",
+        "confidence": "low",
+        "candidates": [
+          "australia-post"
+        ]
+      },
+      "note": "ParcelPanel lists it as an example on its DB Schenker page. DSV's former DB Schenker tracker knows it as the STT of a road shipment."
+    },
+    {
+      "number": "00340433660000960875",
+      "role": "shipment",
+      "evidence": "public_shipment_report",
+      "source": {
+        "url": "https://www.parcelpanel.com/couriers/dbschenker/",
+        "access": "full_page"
+      },
+      "context": {
+        "assessment": "confirmed"
+      },
+      "expect": {
+        "carrier": "dhl",
+        "confidence": "high"
+      },
+      "note": "ParcelPanel lists it as an example on its DB Schenker page. DSV's former DB Schenker tracker knows it as the SSCC of a road shipment's package; DHL's 0034043 SSCC rule claims it."
+    },
+    {
+      "number": "00340433660000963159",
+      "role": "shipment",
+      "evidence": "public_shipment_report",
+      "source": {
+        "url": "https://www.parcelpanel.com/couriers/dbschenker/",
+        "access": "full_page"
+      },
+      "context": {
+        "assessment": "confirmed"
+      },
+      "expect": {
+        "carrier": "dhl",
+        "confidence": "high"
+      },
+      "note": "ParcelPanel lists it as an example on its DB Schenker page. DSV's former DB Schenker tracker finds two road shipments for it; DHL's 0034043 SSCC rule claims it."
     }
   ]
 }

package/dist/carriers/dsv/puzzle.d.ts

@@ -0,0 +1,21 @@
+export interface DsvPuzzle {
+    /** The signed token the server checks the solution against. */
+    readonly jwt: string;
+    readonly seed: Buffer;
+    /** The bound a hash must stay under, as 32 little-endian bytes. */
+    readonly target: Buffer;
+    readonly expectedHashes: number;
+}
+/**
+ * The `Captcha-Puzzle` header: base64 of comma-separated signed tokens. Each
+ * token's `puzzle` is 64 bytes. The first 32 are the seed; bytes 13 and 14
+ * are the difficulty's exponent and mantissa, so a hash must stay under
+ * mantissa × 256^(exponent − 3), read little-endian.
+ */
+export declare function parseDsvPuzzles(header: string, provider: string): DsvPuzzle[];
+/**
+ * The nonce for each puzzle: eight little-endian bytes that, after the seed,
+ * give a double SHA-256 under the target. The answer is the `Captcha-Solution`
+ * header, base64 of the tokens with their nonces.
+ */
+export declare function solveDsvPuzzles(puzzles: readonly DsvPuzzle[], signal: AbortSignal, deadline: number, budgetMs: number, provider: string): Promise<string>;

package/dist/carriers/dsv/puzzle.js

@@ -0,0 +1,99 @@
+import { createHash } from 'node:crypto';
+import { BudgetExceededError, ChallengeError, IndeterminateError } from '../../core/errors/index.js';
+import { isRecord } from '../../core/types.js';
+const MAX_PUZZLES = 8;
+// The tracker's puzzles take about 6,600 hashes each. One that asks for a
+// thousand times more is not one a lookup can afford.
+const MAX_EXPECTED_HASHES = 2 ** 23;
+// Past this many times its expected work, a search would be a broken puzzle.
+const MAX_WORK_FACTOR = 64;
+const YIELD_EVERY = 2048;
+const BASE64URL = /^[A-Za-z0-9_-]+$/;
+function invalid(provider) {
+    return new ChallengeError(provider, `${provider} sent a proof-of-work puzzle the adapter cannot read`);
+}
+/**
+ * The `Captcha-Puzzle` header: base64 of comma-separated signed tokens. Each
+ * token's `puzzle` is 64 bytes. The first 32 are the seed; bytes 13 and 14
+ * are the difficulty's exponent and mantissa, so a hash must stay under
+ * mantissa × 256^(exponent − 3), read little-endian.
+ */
+export function parseDsvPuzzles(header, provider) {
+    const tokens = Buffer.from(header.trim(), 'base64').toString('latin1').split(',');
+    if (!tokens.length || tokens.length > MAX_PUZZLES)
+        throw invalid(provider);
+    return tokens.map((jwt) => {
+        const parts = jwt.split('.');
+        if (parts.length !== 3 || !parts.every(part => BASE64URL.test(part)) || jwt.length > 2000)
+            throw invalid(provider);
+        let payload;
+        try {
+            payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
+        }
+        catch {
+            throw invalid(provider);
+        }
+        if (!isRecord(payload) || typeof payload.puzzle !== 'string')
+            throw invalid(provider);
+        const puzzle = Buffer.from(payload.puzzle, 'base64');
+        if (puzzle.length !== 64)
+            throw invalid(provider);
+        const exponent = puzzle.readInt8(13);
+        const mantissa = puzzle.readInt8(14);
+        // The bound fits in 256 bits and leaves at least one hash to try.
+        if (mantissa < 1 || exponent < 3 || exponent > 34)
+            throw invalid(provider);
+        const target = Buffer.alloc(32);
+        target[exponent - 3] = mantissa;
+        const expectedHashes = 2 ** (256 - 8 * (exponent - 3)) / mantissa;
+        if (expectedHashes > MAX_EXPECTED_HASHES) {
+            throw new IndeterminateError(provider, `${provider} asked for more proof of work than a lookup can spend`, { reason: 'costly_puzzle' });
+        }
+        return { jwt, seed: puzzle.subarray(0, 32), target, expectedHashes };
+    });
+}
+/** Whether a hash, read as a little-endian number, is under the target. */
+function under(hash, target) {
+    for (let index = 31; index >= 0; index -= 1) {
+        if (hash[index] !== target[index])
+            return hash[index] < target[index];
+    }
+    return false;
+}
+/**
+ * The nonce for each puzzle: eight little-endian bytes that, after the seed,
+ * give a double SHA-256 under the target. The answer is the `Captcha-Solution`
+ * header, base64 of the tokens with their nonces.
+ */
+export async function solveDsvPuzzles(puzzles, signal, deadline, budgetMs, provider) {
+    const solutions = [];
+    const input = Buffer.alloc(40);
+    for (const puzzle of puzzles) {
+        puzzle.seed.copy(input, 0);
+        const limit = Math.ceil(puzzle.expectedHashes * MAX_WORK_FACTOR);
+        let found = false;
+        for (let nonce = 0; nonce < limit; nonce += 1) {
+            input.writeUInt32LE(nonce % 2 ** 32, 32);
+            input.writeUInt32LE(Math.floor(nonce / 2 ** 32), 36);
+            const hash = createHash('sha256').update(createHash('sha256').update(input).digest()).digest();
+            if (under(hash, puzzle.target)) {
+                solutions.push({ jwt: puzzle.jwt, solution: input.subarray(32).toString('base64') });
+                found = true;
+                break;
+            }
+            if (nonce % YIELD_EVERY === YIELD_EVERY - 1) {
+                // Let cancellation and deadline timers run during the search. The
+                // lookup's own timer ending it is the budget running out.
+                await new Promise((resolve) => setImmediate(resolve));
+                const reason = signal.reason;
+                if (performance.now() >= deadline || (reason instanceof DOMException && reason.name === 'TimeoutError')) {
+                    throw new BudgetExceededError(provider, budgetMs);
+                }
+                signal.throwIfAborted();
+            }
+        }
+        if (!found)
+            throw invalid(provider);
+    }
+    return Buffer.from(JSON.stringify(solutions)).toString('base64');
+}

package/dist/carriers/dsv/schenker.d.ts

@@ -0,0 +1,26 @@
+import type { CarrierResult } from '../../core/result/index.js';
+/**
+ * How the former DB Schenker tracker is asked for a number: an STT straight
+ * by its details, a package's SSCC through the search, any other number not
+ * at all.
+ */
+export declare function dsvSchenkerRoute(number: string): 'stt' | 'sscc' | null;
+/** The details id of a road STT, without the data sources the tracker adds itself. */
+export declare function dsvSchenkerSttId(number: string): string;
+/**
+ * The former DB Schenker tracker's search: the shipments whose STT, package,
+ * waybill or a reference matches the number. Only a road shipment is read,
+ * and only its id; whether the number is the shipment's own is settled by
+ * its details.
+ */
+export declare function pickDsvSchenkerShipment(payload: unknown, rawNumber: string): string;
+/** Neither of DSV's public trackers knows the number. */
+export declare function notFound(): never;
+/**
+ * A road shipment's details from the former DB Schenker tracker. The number
+ * must be one of the shipment's own ids. The history is the shipment's status
+ * rows: a code, English wording, a wall clock, a place and the reasons the
+ * page shows. The parties, their references, addresses and goods, the
+ * recipient's name on a delivery and the packages' own rows are not read.
+ */
+export declare function parseDsvSchenkerShipment(payload: unknown, rawNumber: string, id: string): CarrierResult;

package/dist/carriers/dsv/schenker.js

@@ -0,0 +1,206 @@
+import { DateTime } from 'luxon';
+import { isValidSscc } from '../../core/detection/index.js';
+import { returnedTrackingNumber } from '../../core/detection/returned.js';
+import { IndeterminateError, SchemaError } from '../../core/errors/index.js';
+import { languageStageStatus } from '../../core/status/index.js';
+import { clean, cleanScalar } from '../../core/transport/index.js';
+import { isRecord } from '../../core/types.js';
+import { normalizeDsvNumber, PROVIDER } from './parser.js';
+import { dsvStage } from './status.js';
+const MAX_SHIPMENTS = 100;
+const MAX_ROWS = 1000;
+const MAX_EVENTS = 100;
+const MAX_PACKAGES = 1000;
+// "LandStt:<STT>:<data sources>:LAND", the id the tracker's own detail request takes.
+const LAND_ID = /^[A-Za-z]{2,30}:[A-Z0-9]{3,40}:[A-Z]{2,10}(?:,[A-Z]{2,10}){0,5}:LAND$/;
+// A former DB Schenker road STT: three letters and seven or eight digits, or five letters and nine digits.
+const STT = /^(?:[A-Z]{3}\d{7,8}|[A-Z]{5}\d{9})$/;
+const CODE = /^[A-Z0-9]{2,10}$/;
+const CLOCK = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2})(?:\.\d{1,9})?(?:Z|[+-]\d{2}:?\d{2})?$/;
+// Postcodes and street numbers in a place name.
+const DIGITS = /\S*\d{3,}\S*/g;
+/**
+ * How the former DB Schenker tracker is asked for a number: an STT straight
+ * by its details, a package's SSCC through the search, any other number not
+ * at all.
+ */
+export function dsvSchenkerRoute(number) {
+    if (STT.test(number))
+        return 'stt';
+    return isValidSscc(/^\d{18}$/.test(number) ? `00${number}` : number) ? 'sscc' : null;
+}
+/** The details id of a road STT, without the data sources the tracker adds itself. */
+export function dsvSchenkerSttId(number) {
+    return `LandStt:${number}:LAND`;
+}
+/** The reply's id is the one asked for, or a road id of the same kind for an id asked without its data sources. */
+function answersId(asked, replied) {
+    if (typeof replied !== 'string' || replied === asked)
+        return true;
+    const [type, , sources] = asked.split(':');
+    return sources === 'LAND' && LAND_ID.test(replied) && replied.startsWith(`${type}:`);
+}
+/**
+ * The former DB Schenker tracker's search: the shipments whose STT, package,
+ * waybill or a reference matches the number. Only a road shipment is read,
+ * and only its id; whether the number is the shipment's own is settled by
+ * its details.
+ */
+export function pickDsvSchenkerShipment(payload, rawNumber) {
+    const number = normalizeDsvNumber(rawNumber);
+    if (!isRecord(payload) || !Array.isArray(payload.result) || payload.result.length > MAX_SHIPMENTS
+        || !payload.result.every(isRecord)) {
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker did not return a shipment list');
+    }
+    const entries = payload.result;
+    if (!entries.length)
+        notFound();
+    const own = entries.filter(entry => returnedTrackingNumber(entry.stt) === number);
+    const entry = own.length === 1 ? own[0] : entries.length === 1 ? entries[0] : null;
+    if (!entry)
+        throw new IndeterminateError(PROVIDER, 'DSV\'s former DB Schenker tracker found several shipments for the number', { reason: 'several_parcels' });
+    if (entry.transportMode !== 'LAND') {
+        throw new IndeterminateError(PROVIDER, 'DSV\'s former DB Schenker tracker found an air or ocean shipment, which the adapter does not read');
+    }
+    if (typeof entry.id !== 'string' || !LAND_ID.test(entry.id))
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned a shipment without its id');
+    return entry.id;
+}
+/** Neither of DSV's public trackers knows the number. */
+export function notFound() {
+    throw new IndeterminateError(PROVIDER, 'Neither of DSV\'s public trackers knows the number', { reason: 'portal_not_found' });
+}
+/**
+ * The ids the shipment carries itself: its STT, the STT it was booked under
+ * and its packages' SSCCs, with or without the 00 that precedes an SSCC in
+ * its barcode. Waybills and the parties' references are not among them.
+ */
+function ownIds(payload) {
+    const ids = new Set();
+    const references = isRecord(payload.references) ? payload.references : {};
+    for (const value of [payload.sttNumber, references.originalStt]) {
+        const id = returnedTrackingNumber(value);
+        if (id)
+            ids.add(id);
+    }
+    const packages = Array.isArray(payload.packages) ? payload.packages.slice(0, MAX_PACKAGES) : [];
+    for (const entry of packages) {
+        const id = isRecord(entry) ? returnedTrackingNumber(entry.id, true) : '';
+        if (!id)
+            continue;
+        ids.add(id);
+        if (/^\d{18}$/.test(id))
+            ids.add(`00${id}`);
+    }
+    return ids;
+}
+/** The row's clock as the page prints it: the wall clock as written, whatever offset follows it. */
+function clockOf(value) {
+    if (value === null || value === undefined)
+        return null;
+    const match = typeof value === 'string' ? CLOCK.exec(value.trim()) : null;
+    if (!match)
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned an invalid date');
+    // UTC only checks the calendar fields. No offset is assigned: some rows label a local clock Z.
+    const parsed = DateTime.fromFormat(`${match[1]} ${match[2]}`, 'yyyy-MM-dd HH:mm:ss', { zone: 'UTC' });
+    if (!parsed.isValid)
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned an invalid date');
+    return parsed.toFormat("yyyy-MM-dd'T'HH:mm:ss");
+}
+/** "Place, CC": the terminal's or the stop's town and its country, without postcodes. */
+function locationOf(value) {
+    if (!isRecord(value))
+        return '';
+    const place = clean(clean(value.name, 120).replace(DIGITS, ' ')).replace(/^[\s,/-]+|[\s,/-]+$/g, '');
+    const country = clean(value.countryCode, 2).toUpperCase();
+    return [place, /^[A-Z]{2}$/.test(country) ? country : ''].filter(Boolean).join(', ');
+}
+/** The reasons the page prints beside a row, such as why a delivery failed. */
+function reasonsOf(value) {
+    if (value === null || value === undefined)
+        return [];
+    if (!Array.isArray(value))
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned invalid reasons');
+    return value.slice(0, 10).map(reason => isRecord(reason) ? clean(reason.description, 200) || cleanScalar(reason.code, 10) : '')
+        .filter(Boolean);
+}
+/** The day of the agreed delivery, else of the estimated one. */
+function estimateOf(value) {
+    if (!isRecord(value))
+        return null;
+    for (const date of [value.agreed, value.estimated]) {
+        const clock = date === null || date === undefined ? null : clockOf(date);
+        if (clock)
+            return clock.slice(0, 10);
+    }
+    return null;
+}
+/**
+ * A road shipment's details from the former DB Schenker tracker. The number
+ * must be one of the shipment's own ids. The history is the shipment's status
+ * rows: a code, English wording, a wall clock, a place and the reasons the
+ * page shows. The parties, their references, addresses and goods, the
+ * recipient's name on a delivery and the packages' own rows are not read.
+ */
+export function parseDsvSchenkerShipment(payload, rawNumber, id) {
+    const number = normalizeDsvNumber(rawNumber);
+    if (!isRecord(payload) || !Array.isArray(payload.events) || typeof payload.sttNumber !== 'string'
+        || !(Array.isArray(payload.packages) || payload.packages === null || payload.packages === undefined)) {
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker did not return the shipment');
+    }
+    if (!answersId(id, payload.shipmentId)) {
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned another shipment');
+    }
+    if (!ownIds(payload).has(number)) {
+        throw new IndeterminateError(PROVIDER, 'The number matched only a reference of a former DB Schenker shipment', { reason: 'reference_only' });
+    }
+    if (payload.events.length > MAX_ROWS)
+        throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned an unexpectedly long history');
+    const scans = [];
+    const seen = new Set();
+    payload.events.forEach((row, index) => {
+        if (!isRecord(row))
+            throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned an invalid history row');
+        const code = cleanScalar(row.code, 40).toUpperCase();
+        if (!CODE.test(code))
+            throw new SchemaError(PROVIDER, 'DSV\'s former DB Schenker tracker returned a row without a code');
+        const time = clockOf(row.date);
+        if (!time)
+            return;
+        const wording = clean(row.comment, 300) || code;
+        const reasons = reasonsOf(row.reasons);
+        const description = reasons.length ? `${wording}: ${reasons.join(', ')}` : wording;
+        const location = locationOf(row.location);
+        const event = { local_time: time, description, provider_code: code, ...(location ? { location } : {}) };
+        const classified = dsvStage(code, wording);
+        if (classified)
+            Object.assign(event, { stage: classified.stage, stage_source: classified.source });
+        const identity = JSON.stringify(event);
+        if (seen.has(identity))
+            return;
+        seen.add(identity);
+        scans.push({ event, index });
+    });
+    if (!scans.length)
+        throw new IndeterminateError(PROVIDER, 'DSV\'s former DB Schenker tracker returned the shipment without a history', { reason: 'no_history' });
+    // Newest first. The reply runs oldest first, so rows at the same clock keep its order reversed.
+    scans.sort((left, right) => {
+        const a = left.event.local_time;
+        const b = right.event.local_time;
+        return b > a ? 1 : b < a ? -1 : right.index - left.index;
+    });
+    const events = scans.map(scan => scan.event);
+    const latest = events[0];
+    const stage = latest.stage;
+    const expected = stage === 'delivered' ? null : estimateOf(payload.deliveryDate);
+    return {
+        status: stage ? languageStageStatus(stage) : 'unknown',
+        ...(stage ? { current_stage: stage, current_stage_source: latest.stage_source } : {}),
+        last_status_text: latest.description,
+        last_update: null,
+        last_update_local: latest.local_time,
+        expected_delivery: expected,
+        events: events.slice(0, MAX_EVENTS),
+        ...(events.length > MAX_EVENTS ? { history_truncated: true } : {}),
+    };
+}

package/dist/carriers/dsv/status.js

@@ -2,7 +2,9 @@ import { classifyWording } from '../../core/status/index.js';
 /**
  * myDSV names each row of a shipment's history with a date type: the booking,
  * the pickup, the departure and arrival at each port and the handover at the
- * end. The same codes name the page's date filters.
+ * end. The same codes name the page's date filters. The former DB Schenker
+ * tracker names a road shipment's rows with three-letter event codes; the two
+ * sets share only DEP, a departure in both.
  */
 const CODES = {
     BOOKING: 'registered',
@@ -18,6 +20,36 @@ const CODES = {
     Z70: 'delivered',
     HANDOVER: 'delivered',
     ACTUAL_DELIVERY: 'delivered',
+    // Former DB Schenker road shipments.
+    ENT: 'registered',
+    COT: 'registered',
+    COL: 'accepted',
+    DET: 'accepted',
+    // Not collected: the pickup failed.
+    NCL: 'exception',
+    ENM: 'in_transit',
+    MAN: 'in_transit',
+    ARR: 'in_transit',
+    TIN: 'in_transit',
+    VLO: 'in_transit',
+    VUL: 'in_transit',
+    TLO: 'in_transit',
+    TUL: 'in_transit',
+    VTL: 'in_transit',
+    VTU: 'in_transit',
+    // Held back from the load it was planned on; a reason says why.
+    NLO: 'exception',
+    CCL: 'customs',
+    CCF: 'in_transit',
+    DOT: 'out_for_delivery',
+    NDL: 'failed_attempt',
+    DIS: 'ready_for_pickup',
+    DIF: 'ready_for_pickup',
+    DLV: 'delivered',
+    PUP: 'delivered',
+    PUF: 'delivered',
+    // The electronic proof of delivery, which follows the delivery.
+    POD: 'delivered',
 };
 // Dates DSV plans or a customer asked for: not scans.
 const ESTIMATES = new Map([

package/dist/carriers/dsv/statuses.json

@@ -17,6 +17,33 @@
     { "code": "ESTIMATED_PICKUP", "wording": "Estimated pickup", "language": "en", "confirmedBy": "live", "note": "An estimate, not a scan: left out of the history." },
     { "code": "REQUESTED_PICKUP", "wording": "Requested pickup", "language": "en", "confirmedBy": "live", "note": "The pickup the booking asked for, not a scan: left out of the history." },
     { "code": "ESTIMATED_DELIVERY", "wording": "Estimated delivery", "language": "en", "confirmedBy": "bundle", "note": "Read as the delivery estimate, not as a scan." },
-    { "code": "REQUESTED_DELIVERY", "wording": "Requested delivery", "language": "en", "confirmedBy": "bundle", "note": "The delivery the booking asked for, not a scan: left out of the history." }
+    { "code": "REQUESTED_DELIVERY", "wording": "Requested delivery", "language": "en", "confirmedBy": "bundle", "note": "The delivery the booking asked for, not a scan: left out of the history." },
+    { "code": "ENT", "wording": "Booked", "language": "en", "stage": "registered", "confirmedBy": "live", "note": "A former DB Schenker road shipment was booked." },
+    { "code": "COT", "wording": "Out for collection", "language": "en", "stage": "registered", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "COL", "wording": "Collected", "language": "en", "stage": "accepted", "confirmedBy": "live", "note": "Picked up from the shipper." },
+    { "code": "DET", "wording": "Delivered to terminal by shipper", "language": "en", "stage": "accepted", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "NCL", "wording": "Not Collected", "language": "en", "stage": "exception", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments. The pickup failed." },
+    { "code": "ENM", "wording": "Arrived", "language": "en", "stage": "in_transit", "confirmedBy": "live", "note": "Arrived at a terminal." },
+    { "code": "MAN", "wording": "Departed", "language": "en", "stage": "in_transit", "confirmedBy": "live", "note": "Left a terminal." },
+    { "code": "DEP", "wording": "Linehaul departed", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "ARR", "wording": "Linehaul arrived", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "TIN", "wording": "Terminal inventory", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "VLO", "wording": "Vessel Loaded", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "VUL", "wording": "Vessel Unloaded", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "TLO", "wording": "Train Loaded", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "TUL", "wording": "Train Unloaded", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "VTL", "wording": "Vessel/Train Loaded", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "VTU", "wording": "Vessel/Train Unloaded", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "NLO", "wording": "Not loaded", "language": "en", "stage": "exception", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments. The shipment missed its planned load; a reason says why." },
+    { "code": "CCL", "wording": "Customs clearance initiated", "language": "en", "stage": "customs", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "CCF", "wording": "Customs clearance finalised", "language": "en", "stage": "in_transit", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments. Cleared, and on its way again." },
+    { "code": "DOT", "wording": "Out for Delivery", "language": "en", "stage": "out_for_delivery", "confirmedBy": "live" },
+    { "code": "NDL", "wording": "Not delivered", "language": "en", "stage": "failed_attempt", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "DIS", "wording": "To consignee's disposal", "language": "en", "stage": "ready_for_pickup", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments. The shipment waits for the consignee to collect it." },
+    { "code": "DIF", "wording": "Ready for pick up", "language": "en", "stage": "ready_for_pickup", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "DLV", "wording": "Delivered", "language": "en", "stage": "delivered", "confirmedBy": "live" },
+    { "code": "PUP", "wording": "Picked up by consignee", "language": "en", "stage": "delivered", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "PUF", "wording": "Picked up at agent", "language": "en", "stage": "delivered", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments." },
+    { "code": "POD", "wording": "ePOD available", "language": "en", "stage": "delivered", "confirmedBy": "bundle", "note": "Named in the DSV Tracking app for former DB Schenker road shipments. The electronic proof of delivery follows the delivery." }
   ]
 }

package/dist/carriers/magyar-posta/adapter.d.ts

@@ -3,7 +3,8 @@ import type { CarrierResult } from '../../core/result/index.js';
 import { type StepRecorder } from '../../core/telemetry/index.js';
 import type { TrawlClient } from '../../core/transport/trawl.js';
 export { magyarPostaTrackingUrl, normalizeMagyarPostaNumber, parseMagyarPosta } from './parser.js';
-/** The search form's endpoint, and the result page it redirects to. */
+/** The tracker page, its search form's endpoint, and the result page the search redirects to. */
+export declare const MAGYAR_POSTA_TRACKER = "https://www.posta.hu/ikrnyomkovproxy/";
 export declare const MAGYAR_POSTA_SEARCH = "https://www.posta.hu/ikrnyomkovproxy/Search/Search";
 export declare const MAGYAR_POSTA_RESULT = "https://www.posta.hu/ikrnyomkovproxy/Result/Index";
 /**
@@ -13,6 +14,7 @@ export declare const MAGYAR_POSTA_RESULT = "https://www.posta.hu/ikrnyomkovproxy
  */
 export declare function magyarPostaBrowser(trawl: TrawlClient, raw: string, signal: AbortSignal, remainingMs: number): Promise<CarrierResult>;
 export declare class MagyarPostaTracker {
+    #private;
     private readonly options;
     constructor(options?: {
         trawl?: TrawlClient | null;

package/dist/carriers/magyar-posta/adapter.js

@@ -3,9 +3,14 @@ import { runSteps } from '../../core/runner/index.js';
 import { NOOP_RECORDER } from '../../core/telemetry/index.js';
 import { MAGYAR_POSTA_MAX_BYTES, magyarPostaTrackingUrl, normalizeMagyarPostaNumber, parseMagyarPosta, PROVIDER } from './parser.js';
 export { magyarPostaTrackingUrl, normalizeMagyarPostaNumber, parseMagyarPosta } from './parser.js';
-/** The search form's endpoint, and the result page it redirects to. */
+/** The tracker page, its search form's endpoint, and the result page the search redirects to. */
+export const MAGYAR_POSTA_TRACKER = 'https://www.posta.hu/ikrnyomkovproxy/';
 export const MAGYAR_POSTA_SEARCH = 'https://www.posta.hu/ikrnyomkovproxy/Search/Search';
 export const MAGYAR_POSTA_RESULT = 'https://www.posta.hu/ikrnyomkovproxy/Result/Index';
+/** The browser helper's record on the tracker page when Google refuses the audio challenge. */
+const REFUSED = 'Google refused the audio challenge';
+/** How long a refusal holds for this address, during which lookups do not open the browser. */
+const REFUSAL_MEMORY_MS = 10 * 60_000;
 /** A refused or failed page; a missing route is not a missing item. */
 function httpFailure(status, retryAfter) {
     if (status === 429)
@@ -31,6 +36,9 @@ export async function magyarPostaBrowser(trawl, raw, signal, remainingMs) {
     signal.throwIfAborted();
     if (page.capturedResponses.length > 10)
         throw new ChallengeError(PROVIDER, 'Magyar Posta returned no matching verified search');
+    if (page.capturedResponses.some(row => row.url === MAGYAR_POSTA_TRACKER && row.error === REFUSED)) {
+        throw new ChallengeError(PROVIDER, 'Google refused Magyar Posta\'s audio challenge', { reason: 'verification_refused' });
+    }
     const result = page.capturedResponses.filter(row => row.url === MAGYAR_POSTA_RESULT && row.trackingNumberSubmitted === number).at(-1);
     const search = page.capturedResponses.filter(row => row.url === MAGYAR_POSTA_SEARCH).at(-1);
     const response = result ?? search;
@@ -56,6 +64,7 @@ export async function magyarPostaBrowser(trawl, raw, signal, remainingMs) {
 }
 export class MagyarPostaTracker {
     options;
+    #refusedUntil = 0;
     constructor(options = {}) {
         this.options = options;
     }
@@ -67,7 +76,20 @@ export class MagyarPostaTracker {
             throw new ChallengeError(PROVIDER, 'Magyar Posta requires the browser tracking service');
         return runSteps({ carrier: 'magyar-posta', budgetMs: context.budgetMs ?? 45_000, signal: context.signal,
             recorder: this.options.recorder ?? NOOP_RECORDER }, [
-            { id: 'trawl', run: ({ signal, remainingMs }) => magyarPostaBrowser(trawl, number, signal, remainingMs) },
+            { id: 'trawl', run: async ({ signal, remainingMs }) => {
+                    // Another solve soon after a refusal would hold a browser for the same answer.
+                    if (Date.now() < this.#refusedUntil) {
+                        throw new ChallengeError(PROVIDER, 'Google refused Magyar Posta\'s audio challenge recently', { reason: 'verification_refused' });
+                    }
+                    try {
+                        return await magyarPostaBrowser(trawl, number, signal, remainingMs);
+                    }
+                    catch (error) {
+                        if (error instanceof ChallengeError && error.reason === 'verification_refused')
+                            this.#refusedUntil = Date.now() + REFUSAL_MEMORY_MS;
+                        throw error;
+                    }
+                } },
         ]);
     }
 }

package/dist/server/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Universal Parcel Scraper",
-    "version": "0.142.0"
+    "version": "0.143.0"
   },
   "paths": {
     "/health": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "universal-parcel-scraper",
-  "version": "0.142.0",
+  "version": "0.143.0-main.1167",
   "description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
   "type": "module",
   "license": "Apache-2.0",