universal-parcel-scraper 0.25.3-main.367 → 0.26.1-main.369

package/dist/carriers/australia-post/adapter.d.ts

@@ -9,6 +9,7 @@ export declare function australiaPostApiUrl(raw: string): string;
 export declare function parse(payload: unknown, trackingNumber: string): CarrierResult;
 export interface AustraliaPostTrackerOptions {
     trawl: TrawlClient | null;
+    fetcher?: typeof fetch;
     timeoutMs?: number;
     recorder?: StepRecorder;
 }

package/dist/carriers/australia-post/adapter.js

@@ -1,7 +1,7 @@
-import { BudgetExceededError, ChallengeError, InvalidInputError, NotFoundError, RateLimitedError, SchemaError, TransportError, UpstreamHttpError } from '../../core/errors/index.js';
-import { runSteps } from '../../core/runner/index.js';
+import { BudgetExceededError, ChallengeError, IndeterminateError, InvalidInputError, NotFoundError, RateLimitedError, SchemaError, TransportError, UpstreamHttpError } from '../../core/errors/index.js';
+import { recoverableByDefault, runSteps } from '../../core/runner/index.js';
 import { explicitOffsetTime, epochMillisTime } from '../../core/time/index.js';
-import { clean } from '../../core/transport/index.js';
+import { clean, fetchBounded } from '../../core/transport/index.js';
 import { isRecord } from '../../core/types.js';
 import { australiaPostStatus } from './status.js';
 const PROVIDER = 'Australia Post';
@@ -9,6 +9,14 @@ const DETAIL = 'https://auspost.com.au/mypost/track/details/';
 const API = 'https://digitalapi.auspost.com.au/shipments-gateway/v1/watchlist/shipments';
 const MAX_BYTES = 1_000_000;
 const TRANSPORT_ALLOWANCE_MS = 15_000;
+const DIRECT_TIMEOUT_MS = 10_000;
+// The gateway's bot protection passes the official Android app's HTTP client and
+// channel; other clients get a captcha reply. So does fetch's default `Accept-Language: *`.
+const APP_HEADERS = { Accept: 'application/json', 'Accept-Language': 'en-AU', AP_APP_ID: 'MYPOST', AP_CHANNEL_NAME: 'ANDROID',
+    'User-Agent': 'okhttp/4.12.0' };
+// fetch's `no-store` adds a browser reload's cache headers, which the gateway refuses
+// from this client. `no-cache` still takes every reply from the gateway.
+const asApp = (fetcher) => (input, init) => (fetcher ?? fetch)(input, { ...init, cache: 'no-cache' });
 export function normalizeAustraliaPostNumber(raw) {
     const number = raw.toUpperCase().replace(/[\s.-]/g, '');
     if (!/^(?=.*\d)[A-Z0-9]{10,34}$/.test(number)) {
@@ -32,6 +40,10 @@ export function parse(payload, trackingNumber) {
     if (entries.length !== 1)
         throw new SchemaError(PROVIDER, 'Australia Post returned a different or ambiguous lookup');
     const entry = entries[0];
+    // The gateway answers a reference it cannot process with an internal error, which says
+    // nothing about the parcel.
+    if (entry.status === 500 && entry.shipment === undefined)
+        throw new IndeterminateError(PROVIDER);
     if (entry.status === 400 && isRecord(entry.error) && entry.error.errorCode === 21
         && entry.error.error === 'Invalid Tracking ID' && entry.error.status === 'Failed'
         && entry.shipment === undefined)
@@ -122,12 +134,32 @@ export class AustraliaPostTracker {
         if (!Number.isFinite(budgetMs) || budgetMs <= 0 || budgetMs > 60_000)
             throw new TypeError('Australia Post budget must be between 1 and 60000 ms');
         context.signal?.throwIfAborted();
-        if (budgetMs <= TRANSPORT_ALLOWANCE_MS)
-            throw new BudgetExceededError(PROVIDER, budgetMs);
         const trawl = this.options.trawl;
-        if (!trawl)
-            throw new ChallengeError(PROVIDER, 'Australia Post requires the browser tracking service');
         return runSteps({ carrier: 'australia-post', budgetMs, signal: context.signal, recorder: this.options.recorder }, [{
+                id: 'direct', run: async ({ signal, remainingMs }) => {
+                    const { response, bytes } = await fetchBounded(australiaPostApiUrl(number), { signal, headers: APP_HEADERS }, {
+                        provider: PROVIDER, timeoutMs: Math.max(1, Math.min(DIRECT_TIMEOUT_MS, Math.floor(remainingMs))),
+                        maxBytes: MAX_BYTES, allowHttpStatuses: [401, 403, 429], fetcher: asApp(this.options.fetcher),
+                    });
+                    if (response.status === 429) {
+                        const retry = response.headers.get('retry-after');
+                        throw new RateLimitedError(PROVIDER, retry && /^\d+$/.test(retry) ? Number(retry) * 1000 : undefined);
+                    }
+                    if (response.status !== 200) {
+                        throw new ChallengeError(PROVIDER, trawl ? undefined : 'Australia Post requires the browser tracking service');
+                    }
+                    let payload;
+                    try {
+                        payload = JSON.parse(new TextDecoder().decode(bytes));
+                    }
+                    catch {
+                        throw new SchemaError(PROVIDER, 'Australia Post returned invalid tracking JSON');
+                    }
+                    return parse(payload, number);
+                },
+            }, {
+                // The browser reads the same gateway, so it cannot improve on an inconclusive entry.
+                enabled: trawl !== null, recovers: (error) => recoverableByDefault(error) && !(error instanceof IndeterminateError),
                 id: 'trawl', run: async ({ signal, remainingMs }) => {
                     const timeoutMs = Math.floor(remainingMs - TRANSPORT_ALLOWANCE_MS);
                     if (timeoutMs < 1)
@@ -170,6 +202,6 @@ export class AustraliaPostTracker {
     }
 }
 export const adapter = (environment) => {
-    const tracker = new AustraliaPostTracker({ trawl: environment.trawl, recorder: environment.recorder });
-    return { id: 'australia-post', recordsSteps: true, steps: ['trawl'], track: (input, context) => tracker.fetch(input.number, context) };
+    const tracker = new AustraliaPostTracker({ trawl: environment.trawl, fetcher: environment.fetcher, recorder: environment.recorder });
+    return { id: 'australia-post', recordsSteps: true, steps: ['direct', 'trawl'], track: (input, context) => tracker.fetch(input.number, context) };
 };

package/dist/carriers/australia-post/carrier.json

@@ -16,6 +16,7 @@
     "mode": "automatic",
     "adapter": "australia-post",
     "steps": [
+      "direct",
       "trawl"
     ]
   },

package/dist/carriers/ukrposhta/adapter.d.ts

@@ -1,11 +1,15 @@
 import type { AdapterFactory, TrackingContext } from '../../core/adapter/index.js';
 import { type StepRecorder } from '../../core/telemetry/index.js';
 export declare const UKRPOSHTA_API = "https://track.ukrposhta.ua/php/track_new.php";
+export declare const UKRPOSHTA_STATUS_API = "https://www.ukrposhta.ua/status-tracking/0.0.1/statuses";
 export declare function ukrposhtaTrackingUrl(number: string): string;
 export declare class UkrposhtaTracker {
     private readonly options;
     constructor(options?: {
         executablePath?: string | null;
+        token?: string | null;
+        userAgent?: string;
+        fetcher?: typeof fetch;
         recorder?: StepRecorder;
     });
     fetch(raw: string, context?: TrackingContext): Promise<import("../../index.js").CarrierResult>;

package/dist/carriers/ukrposhta/adapter.js

@@ -1,11 +1,51 @@
 import { withLocalBrowser } from '../../core/transport/localBrowser.js';
-import { BudgetExceededError, ChallengeError, SchemaError, TransportError, UpstreamHttpError } from '../../core/errors/index.js';
-import { runSteps } from '../../core/runner/index.js';
+import { isValidS10TrackingNumber } from '../../core/detection/s10.js';
+import { BudgetExceededError, ChallengeError, IndeterminateError, RateLimitedError, SchemaError, TransportError, UpstreamHttpError } from '../../core/errors/index.js';
+import { recoverableByDefault, runSteps } from '../../core/runner/index.js';
 import { NOOP_RECORDER } from '../../core/telemetry/index.js';
+import { fetchBounded } from '../../core/transport/boundedFetch.js';
 import { isRecord } from '../../core/types.js';
-import { normalizeUkrposhtaNumber, parseUkrposhtaHistory, parseUkrposhtaOverview } from './parser.js';
+import { normalizeUkrposhtaNumber, parseUkrposhtaHistory, parseUkrposhtaOverview, parseUkrposhtaStatuses } from './parser.js';
 export const UKRPOSHTA_API = 'https://track.ukrposhta.ua/php/track_new.php';
+export const UKRPOSHTA_STATUS_API = 'https://www.ukrposhta.ua/status-tracking/0.0.1/statuses';
 const MAX_BYTES = 1_000_000;
+const DIRECT_TIMEOUT_MS = 10_000;
+// The Android app's shared application bearer is deliberately distributed with this adapter.
+const APPLICATION_BEARER = 'c3e02b53-3b1d-386e-b676-141ffa054c57';
+/** The API's bearer, or null when none is configured or the value could not be a header. */
+function bearer(value) {
+    const token = value?.trim().replace(/^Bearer\s+/i, '') ?? '';
+    return /^[\w.~+/=-]{8,512}$/.test(token) ? token : null;
+}
+async function readStatuses(number, token, signal, timeoutMs, userAgent = 'Mozilla/5.0', fetcher) {
+    const url = new URL(UKRPOSHTA_STATUS_API);
+    url.searchParams.set('barcode', number);
+    url.searchParams.set('lang', 'en');
+    const { response, bytes } = await fetchBounded(url, { signal, headers: { Accept: 'application/json', Authorization: `Bearer ${token}`, 'User-Agent': userAgent } }, {
+        provider: 'ukrposhta', timeoutMs, maxBytes: MAX_BYTES, allowHttpStatuses: [401, 403, 404, 429], fetcher,
+    });
+    // The refusal of a credential is an HTML page; it says nothing about the parcel.
+    if ([401, 403].includes(response.status))
+        throw new ChallengeError('ukrposhta', 'Ukrposhta refused the tracking API credential');
+    if (response.status === 429) {
+        const retryAfter = response.headers.get('retry-after');
+        throw new RateLimitedError('ukrposhta', retryAfter && /^\d+$/.test(retryAfter) ? Number(retryAfter) * 1_000 : undefined);
+    }
+    let payload;
+    try {
+        payload = JSON.parse(new TextDecoder().decode(bytes));
+    }
+    catch (cause) {
+        throw new SchemaError('ukrposhta', 'Ukrposhta returned invalid tracking JSON', { cause });
+    }
+    if (response.status === 404) {
+        // The reply names no barcode, as on the portal: it cannot establish absence.
+        if (isRecord(payload) && payload.message === 'Shipment not found')
+            throw new IndeterminateError('ukrposhta');
+        throw new TransportError('ukrposhta', 'Ukrposhta tracking endpoint is unavailable', { status: 404 });
+    }
+    return parseUkrposhtaStatuses(payload, number);
+}
 export function ukrposhtaTrackingUrl(number) {
     const url = new URL('https://track.ukrposhta.ua/en/');
     url.searchParams.set('barcode', number);
@@ -97,14 +137,23 @@ export class UkrposhtaTracker {
         const budgetMs = context.budgetMs ?? 45_000;
         if (!Number.isFinite(budgetMs) || budgetMs <= 0 || budgetMs > 60_000)
             throw new TypeError('Ukrposhta budget must be between 1 and 60000 ms');
-        if (!this.options.executablePath)
-            throw new ChallengeError('ukrposhta', 'Ukrposhta requires a configured tracking browser');
+        const token = bearer(this.options.token);
+        const executablePath = this.options.executablePath;
+        if (!token && !executablePath)
+            throw new ChallengeError('ukrposhta', 'Ukrposhta requires a tracking API token or a configured tracking browser');
         return runSteps({ carrier: 'ukrposhta', budgetMs, signal: context.signal, recorder: this.options.recorder ?? NOOP_RECORDER }, [
-            { id: 'browser', run: ({ signal, remainingMs }) => localBrowser(number, this.options.executablePath, signal, Math.max(1, Math.floor(remainingMs))) },
+            { id: 'direct', enabled: token !== null,
+                run: ({ signal, remainingMs }) => readStatuses(number, token, signal, Math.max(1, Math.min(DIRECT_TIMEOUT_MS, Math.floor(remainingMs))), this.options.userAgent, this.options.fetcher) },
+            // The portal reads the same domestic records, so it cannot improve on an unknown
+            // domestic barcode. An international reference still gets its second source.
+            { id: 'browser', enabled: Boolean(executablePath),
+                recovers: error => recoverableByDefault(error) && (!(error instanceof IndeterminateError) || isValidS10TrackingNumber(number)),
+                run: ({ signal, remainingMs }) => localBrowser(number, executablePath, signal, Math.max(1, Math.floor(remainingMs))) },
         ]);
     }
 }
 export const adapter = environment => {
-    const tracker = new UkrposhtaTracker({ executablePath: environment.browserExecutablePath, recorder: environment.recorder });
-    return { id: 'ukrposhta', recordsSteps: true, steps: ['browser'], track: (input, context) => tracker.fetch(input.number, context) };
+    const tracker = new UkrposhtaTracker({ executablePath: environment.browserExecutablePath, token: environment.env.UKRPOSHTA_TRACKING_TOKEN?.trim() || APPLICATION_BEARER,
+        userAgent: environment.userAgent, fetcher: environment.fetcher, recorder: environment.recorder });
+    return { id: 'ukrposhta', recordsSteps: true, steps: ['direct', 'browser'], track: (input, context) => tracker.fetch(input.number, context) };
 };

package/dist/carriers/ukrposhta/carrier.json

@@ -16,6 +16,7 @@
     "mode": "automatic",
     "adapter": "ukrposhta",
     "steps": [
+      "direct",
       "browser"
     ],
     "localClocks": true

package/dist/carriers/ukrposhta/parser.d.ts

@@ -12,4 +12,9 @@ export interface UkrposhtaOverview {
     country: string;
 }
 export declare function parseUkrposhtaOverview(payload: unknown, requested: string): UkrposhtaOverview;
+/**
+ * The status-tracking API's reply to one barcode: a bare array of scans, each naming the
+ * barcode and its position in the history.
+ */
+export declare function parseUkrposhtaStatuses(payload: unknown, requested: string): CarrierResult;
 export declare function parseUkrposhtaHistory(payload: unknown, overview: UkrposhtaOverview): CarrierResult;

package/dist/carriers/ukrposhta/parser.js

@@ -1,7 +1,7 @@
 import { DateTime } from 'luxon';
 import { isValidS10TrackingNumber } from '../../core/detection/s10.js';
 import { ChallengeError, IndeterminateError, InvalidInputError, SchemaError } from '../../core/errors/index.js';
-import { clean } from '../../core/transport/text.js';
+import { clean, cleanScalar } from '../../core/transport/text.js';
 import { isRecord } from '../../core/types.js';
 import { ukrposhtaReturnCue, ukrposhtaStatus } from './status.js';
 const PROVIDER = 'ukrposhta';
@@ -60,6 +60,73 @@ export function parseUkrposhtaOverview(payload, requested) {
     return { number, count: Number(row.step), date: text(row.date, 'overview clock'), code: text(row.event, 'overview event code', true),
         label: text(row.eventName, 'overview status', true), location: text(row.name, 'overview location'), country: text(row.country, 'overview country') };
 }
+/** One result from current-first scans, following a return decision into its own leg. */
+function project(rows) {
+    const latest = rows[0];
+    // Both native histories are current-first. Keep that order across foreign
+    // wall clocks or missing dates instead of sorting them as UTC instants.
+    let returnLeg = false;
+    const events = new Array(rows.length);
+    for (let index = rows.length - 1; index >= 0; index--) {
+        const row = rows[index];
+        if (ukrposhtaReturnCue(row.label))
+            returnLeg = true;
+        let classified = ukrposhtaStatus(row.code);
+        if (row.reason === '65' || row.reason === '66')
+            classified = { status: 'exception', stage: 'exception' };
+        if (classified?.stage === 'delivered' && returnLeg)
+            classified = { status: 'exception', stage: 'returned' };
+        const local = ukrposhtaWallClock(row.date);
+        events[index] = {
+            description: row.label, provider_code: row.code,
+            ...(row.location || row.country ? { location: [row.location, row.country].filter(Boolean).join(', ') } : {}),
+            ...(local ? { local_time: local } : row.date ? { provider_time_text: row.date } : {}),
+            ...(classified ? { stage: classified.stage } : {}), ...(returnLeg ? { provider_leg: 'return' } : {}),
+        };
+    }
+    const current = events[0];
+    const stage = current.stage;
+    const status = stage === 'returned' || stage === 'exception' ? 'exception' : ukrposhtaStatus(latest.code)?.status ?? 'unknown';
+    const seen = new Set();
+    const deduplicated = events.filter(event => { const key = JSON.stringify(event); if (seen.has(key))
+        return false; seen.add(key); return true; });
+    return { status, ...(stage ? { current_stage: stage } : {}), last_status_text: latest.label, last_update: null, events: deduplicated.slice(0, 100) };
+}
+/**
+ * The status-tracking API's reply to one barcode: a bare array of scans, each naming the
+ * barcode and its position in the history.
+ */
+export function parseUkrposhtaStatuses(payload, requested) {
+    const number = normalizeUkrposhtaNumber(requested);
+    if (!Array.isArray(payload))
+        throw new SchemaError(PROVIDER, 'Ukrposhta returned an invalid tracking response');
+    if (!payload.length)
+        throw new IndeterminateError(PROVIDER);
+    if (payload.length > MAX_SCANS)
+        throw new SchemaError(PROVIDER, 'Ukrposhta returned excessive history');
+    const steps = new Set();
+    const rows = payload.map((raw) => {
+        if (!isRecord(raw))
+            throw new SchemaError(PROVIDER, 'Ukrposhta returned an invalid history row');
+        if (text(raw.barcode, 'history identity', true).toUpperCase() !== number)
+            throw new SchemaError(PROVIDER, 'Ukrposhta returned a different history identity');
+        if (!Number.isInteger(raw.step) || Number(raw.step) < 1 || steps.has(Number(raw.step)))
+            throw new SchemaError(PROVIDER, 'Ukrposhta returned an invalid history position');
+        steps.add(Number(raw.step));
+        if (!['string', 'number'].includes(typeof raw.event))
+            throw new SchemaError(PROVIDER, 'Ukrposhta returned an invalid scan code');
+        const reason = raw.eventReason_id == null ? '' : cleanScalar(raw.eventReason_id, 16);
+        if (raw.eventReason_id != null && !/^\d{1,9}$/.test(reason))
+            throw new SchemaError(PROVIDER, 'Ukrposhta returned an invalid scan reason code');
+        return { step: Number(raw.step), date: text(raw.date, 'scan clock'), code: cleanScalar(raw.event, 64),
+            label: text(raw.eventName, 'scan status', true), location: text(raw.name, 'scan location'), country: text(raw.country, 'scan country'),
+            reason };
+    });
+    if (rows.some(row => !row.code))
+        throw new SchemaError(PROVIDER, 'Ukrposhta omitted scan code');
+    // The API lists scans oldest first; the position, not the foreign wall clock, orders them.
+    return project(rows.sort((a, b) => b.step - a.step));
+}
 export function parseUkrposhtaHistory(payload, overview) {
     const data = envelope(payload);
     if (!isRecord(data.from_to) || !Array.isArray(data.result))
@@ -99,32 +166,5 @@ export function parseUkrposhtaHistory(payload, overview) {
     if (!sameClock || latest.code !== overview.code || latest.label !== overview.label
         || latest.location !== overview.location || latest.country !== overview.country)
         throw new IndeterminateError(PROVIDER, 'Ukrposhta overview and current history disagree');
-    // Both native histories are current-first. Keep that order across foreign
-    // wall clocks or missing dates instead of sorting them as UTC instants.
-    let returnLeg = false;
-    const events = new Array(rows.length);
-    for (let index = rows.length - 1; index >= 0; index--) {
-        const row = rows[index];
-        if (ukrposhtaReturnCue(row.label))
-            returnLeg = true;
-        let classified = ukrposhtaStatus(row.code);
-        if (row.reason === '65' || row.reason === '66')
-            classified = { status: 'exception', stage: 'exception' };
-        if (classified?.stage === 'delivered' && returnLeg)
-            classified = { status: 'exception', stage: 'returned' };
-        const local = ukrposhtaWallClock(row.date);
-        events[index] = {
-            description: row.label, provider_code: row.code,
-            ...(row.location || row.country ? { location: [row.location, row.country].filter(Boolean).join(', ') } : {}),
-            ...(local ? { local_time: local } : row.date ? { provider_time_text: row.date } : {}),
-            ...(classified ? { stage: classified.stage } : {}), ...(returnLeg ? { provider_leg: 'return' } : {}),
-        };
-    }
-    const current = events[0];
-    const stage = current.stage;
-    const status = stage === 'returned' || stage === 'exception' ? 'exception' : ukrposhtaStatus(latest.code)?.status ?? 'unknown';
-    const seen = new Set();
-    const deduplicated = events.filter(event => { const key = JSON.stringify(event); if (seen.has(key))
-        return false; seen.add(key); return true; });
-    return { status, ...(stage ? { current_stage: stage } : {}), last_status_text: latest.label, last_update: null, events: deduplicated.slice(0, 100) };
+    return project(rows);
 }

package/dist/carriers/ukrposhta/status.js

@@ -30,5 +30,5 @@ export function ukrposhtaStatus(code) {
 // "Returned to Sender" is the decision to start the journey back. The
 // separate "Return: Delivered to Sender" scan confirms its completion.
 export function ukrposhtaReturnCue(label) {
-    return /^Return: /i.test(label) || /^Returned to Sender(?:\s|\()/i.test(label);
+    return /^Return: /i.test(label) || /^Returned to Sender(?:\s|\()/i.test(label) || /^Повернення[.:]/i.test(label);
 }

package/dist/server/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Universal Parcel Scraper",
-    "version": "0.25.3"
+    "version": "0.26.1"
   },
   "paths": {
     "/health": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "universal-parcel-scraper",
-  "version": "0.25.3-main.367",
+  "version": "0.26.1-main.369",
   "description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
   "type": "module",
   "license": "Apache-2.0",