universal-parcel-scraper 0.55.0-main.708 → 0.56.0-main.723
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/data/catalog.json +42 -1
- package/data/coverage-summary.json +1 -1
- package/data/detection-golden.json +49 -2
- package/dist/browser/scraper.js +1048 -1005
- package/dist/carriers/chronopost/adapter.js +23 -16
- package/dist/carriers/dpd/adapter.d.ts +2 -2
- package/dist/carriers/dpd/adapter.js +29 -20
- package/dist/carriers/dpd-de/adapter.js +2 -3
- package/dist/carriers/dpd-de/app.d.ts +5 -2
- package/dist/carriers/dpd-de/app.js +10 -3
- package/dist/carriers/dpd-de/carrier.json +1 -1
- package/dist/carriers/dpd-de/numbers.json +26 -0
- package/dist/carriers/dpd-de/service.d.ts +42 -7
- package/dist/carriers/dpd-de/service.js +54 -26
- package/dist/carriers/dpd-fr/adapter.js +18 -16
- package/dist/carriers/dpd-pl/carrier.json +7 -0
- package/dist/carriers/emile/carrier.json +56 -0
- package/dist/carriers/emile/numbers.json +94 -0
- package/dist/carriers/emile/statuses.json +4 -0
- package/dist/carriers/four-px/numbers.json +5 -2
- package/dist/carriers/gls-ch/adapter.d.ts +4 -0
- package/dist/carriers/gls-ch/adapter.js +8 -3
- package/dist/carriers/gls-ch/status.js +2 -2
- package/dist/carriers/gls-ch/statuses.json +1 -1
- package/dist/carriers/gls-de/adapter.d.ts +4 -0
- package/dist/carriers/gls-de/adapter.js +8 -3
- package/dist/carriers/gls-de/statuses.json +1 -1
- package/dist/carriers/gls-fr/adapter.js +8 -4
- package/dist/carriers/la-poste/adapter.js +23 -11
- package/dist/carriers/poczta-polska/parser.js +22 -6
- package/dist/carriers/swiss-post/adapter.d.ts +1 -1
- package/dist/carriers/swiss-post/adapter.js +17 -8
- package/dist/carriers/ukrposhta/parser.js +19 -3
- package/dist/core/catalog/hints.js +8 -5
- package/dist/core/errors/index.d.ts +8 -1
- package/dist/core/errors/index.js +7 -0
- package/dist/core/runner/networkRetry.d.ts +15 -0
- package/dist/core/runner/networkRetry.js +51 -0
- package/dist/core/status/language.js +32 -6
- package/dist/data/catalog.json +42 -1
- package/dist/generated/catalog.d.ts +30 -2
- package/dist/generated/catalog.js +45 -2
- package/dist/generated/registry.js +1 -0
- package/dist/node.d.ts +1 -1
- package/dist/providers/parcelsapp/adapter.js +9 -3
- package/dist/server/openapi.json +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { accepted, recognizeFromLookup } from '../../core/adapter/index.js';
|
|
2
2
|
import { IndeterminateError } from '../../core/errors/index.js';
|
|
3
3
|
import { runSteps } from '../../core/runner/index.js';
|
|
4
|
+
import { firstAttemptMs, networkRetryStep } from '../../core/runner/networkRetry.js';
|
|
4
5
|
import { decodeText, fetchBounded, userAgentOf } from '../../core/transport/index.js';
|
|
5
6
|
import { CHRONOPOST_MAX_BYTES, CHRONOPOST_NAMESPACE, normalizeChronopostNumber, parseChronopostTrackingXml } from './parser.js';
|
|
6
7
|
// The official WSDL declares this read-only operation without account credentials.
|
|
@@ -12,28 +13,34 @@ export class ChronopostTracker {
|
|
|
12
13
|
}
|
|
13
14
|
fetch(raw, context = {}) {
|
|
14
15
|
const number = normalizeChronopostNumber(raw);
|
|
16
|
+
const read = async (signal, timeoutMs) => {
|
|
17
|
+
const { response, bytes } = await fetchBounded(ENDPOINT, {
|
|
18
|
+
method: 'POST', signal,
|
|
19
|
+
headers: { Accept: 'text/xml', 'Content-Type': 'text/xml; charset=utf-8', SOAPAction: '""',
|
|
20
|
+
'User-Agent': userAgentOf(this.options.userAgent) },
|
|
21
|
+
body: `<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:trk="${CHRONOPOST_NAMESPACE}">`
|
|
22
|
+
+ `<soap:Body><trk:trackSkybillV2><language>fr_FR</language><skybillNumber>${number}</skybillNumber>`
|
|
23
|
+
+ '</trk:trackSkybillV2></soap:Body></soap:Envelope>',
|
|
24
|
+
}, { provider: 'Chronopost tracking', timeoutMs: Math.max(1, Math.floor(timeoutMs)),
|
|
25
|
+
maxBytes: CHRONOPOST_MAX_BYTES, allowHttpStatuses: [404, 410], fetcher: this.options.fetcher });
|
|
26
|
+
// An HTTP route failure is not the service's positive unknown-parcel reply.
|
|
27
|
+
if ([404, 410].includes(response.status))
|
|
28
|
+
throw new IndeterminateError('Chronopost', 'Chronopost tracking endpoint is unavailable');
|
|
29
|
+
return parseChronopostTrackingXml(decodeText(bytes), number);
|
|
30
|
+
};
|
|
31
|
+
// A request that fails to reach Chronopost, or hangs, gets one more try:
|
|
32
|
+
// the first gets half of the budget, so the retry fits in the rest.
|
|
15
33
|
return runSteps({ carrier: 'chronopost', budgetMs: context.budgetMs ?? 15_000, signal: context.signal,
|
|
16
|
-
recorder: this.options.recorder }, [
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
'User-Agent': userAgentOf(this.options.userAgent) },
|
|
21
|
-
body: `<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:trk="${CHRONOPOST_NAMESPACE}">`
|
|
22
|
-
+ `<soap:Body><trk:trackSkybillV2><language>fr_FR</language><skybillNumber>${number}</skybillNumber>`
|
|
23
|
-
+ '</trk:trackSkybillV2></soap:Body></soap:Envelope>',
|
|
24
|
-
}, { provider: 'Chronopost tracking', timeoutMs: Math.max(1, Math.floor(remainingMs)),
|
|
25
|
-
maxBytes: CHRONOPOST_MAX_BYTES, allowHttpStatuses: [404, 410], fetcher: this.options.fetcher });
|
|
26
|
-
// An HTTP route failure is not the service's positive unknown-parcel reply.
|
|
27
|
-
if ([404, 410].includes(response.status))
|
|
28
|
-
throw new IndeterminateError('Chronopost', 'Chronopost tracking endpoint is unavailable');
|
|
29
|
-
return parseChronopostTrackingXml(decodeText(bytes), number);
|
|
30
|
-
} }]);
|
|
34
|
+
recorder: this.options.recorder }, [
|
|
35
|
+
{ id: 'direct', run: ({ signal, remainingMs }) => read(signal, firstAttemptMs(remainingMs)) },
|
|
36
|
+
networkRetryStep(({ signal, remainingMs }) => read(signal, remainingMs)),
|
|
37
|
+
]);
|
|
31
38
|
}
|
|
32
39
|
}
|
|
33
40
|
export const adapter = environment => {
|
|
34
41
|
const tracker = new ChronopostTracker(environment);
|
|
35
42
|
return {
|
|
36
|
-
id: 'chronopost', recordsSteps: true, steps: ['direct'],
|
|
43
|
+
id: 'chronopost', recordsSteps: true, steps: ['direct', 'retry'],
|
|
37
44
|
track: (input, context) => tracker.fetch(input.number, context),
|
|
38
45
|
recognize: (number, context) => recognizeFromLookup(() => tracker.fetch(number, context), () => accepted(() => normalizeChronopostNumber(number))),
|
|
39
46
|
};
|
|
@@ -51,6 +51,7 @@ export interface DPDParcelShops {
|
|
|
51
51
|
export interface DPDAppTier {
|
|
52
52
|
/**
|
|
53
53
|
* `leads` when the guest tier is still to come and needs time left. `postcode` is empty when none was given.
|
|
54
|
+
* `waiting`, given to a leading tier, is called when the lookup starts waiting for the tier's session to open.
|
|
54
55
|
* `stop` aborts when the lookup has its answer without the tier, which then reads nothing more.
|
|
55
56
|
*/
|
|
56
57
|
run(number: string, context: {
|
|
@@ -59,11 +60,10 @@ export interface DPDAppTier {
|
|
|
59
60
|
timeoutMs: number;
|
|
60
61
|
leads: boolean;
|
|
61
62
|
postcode: string;
|
|
63
|
+
waiting?: () => void;
|
|
62
64
|
}): Promise<CarrierResult>;
|
|
63
65
|
/** The failures of either tier the other may answer after. */
|
|
64
66
|
recovers(error: unknown): boolean;
|
|
65
|
-
/** Whether a lookup would first wait for the tier's session to open. */
|
|
66
|
-
opening?(): boolean;
|
|
67
67
|
}
|
|
68
68
|
export declare class DPDTracker {
|
|
69
69
|
#private;
|
|
@@ -632,35 +632,44 @@ export class DPDTracker {
|
|
|
632
632
|
let guest;
|
|
633
633
|
const guestReply = () => guest ??= this.apiFetch(trackingNumber, resolvedPostcode, lookup);
|
|
634
634
|
/**
|
|
635
|
-
* While the app's session opens, the guest tier is asked beside it
|
|
636
|
-
*
|
|
637
|
-
* not
|
|
638
|
-
*
|
|
639
|
-
*
|
|
640
|
-
*
|
|
635
|
+
* While the app's session opens, the guest tier is asked beside it, not
|
|
636
|
+
* while a host's store gives back a saved session. A parcel the guest tier
|
|
637
|
+
* does not know, or places in another country, the app does not answer
|
|
638
|
+
* either: the app stops waiting, reads nothing once its session opens, and
|
|
639
|
+
* the guest tier's step reports it. Otherwise the app keeps waiting, because
|
|
640
|
+
* the guest tier's reply without a postcode is a summary: one entry per
|
|
641
|
+
* milestone, without places, registration, repeated delivery attempts and
|
|
642
|
+
* failures, or the parcel's size. Its clocks and wording also differ from
|
|
643
|
+
* the app's scans, so a consumer that stored it would keep both versions of
|
|
644
|
+
* a scan once a later lookup reads the app. The reply serves the guest
|
|
645
|
+
* tier's step if the session does not open in time. A failed reply is
|
|
646
|
+
* asked again there.
|
|
641
647
|
*/
|
|
642
|
-
const besideGuest = (
|
|
648
|
+
const besideGuest = (stop) => {
|
|
643
649
|
const reply = guestReply();
|
|
644
650
|
const pending = new Promise(() => { });
|
|
645
|
-
return
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
651
|
+
return reply.then(() => pending, (error) => {
|
|
652
|
+
if (!app.recovers(error)) {
|
|
653
|
+
const stopped = new IndeterminateError('DPD Germany', 'DPD Germany app session is still opening', { reason: DPD_DE_SESSION_OPENING });
|
|
654
|
+
stop.abort(stopped);
|
|
655
|
+
throw stopped;
|
|
656
|
+
}
|
|
657
|
+
if (guest === reply)
|
|
658
|
+
guest = undefined;
|
|
659
|
+
return pending;
|
|
660
|
+
});
|
|
655
661
|
};
|
|
656
662
|
const appStep = (leads) => ({
|
|
657
663
|
id: 'app',
|
|
658
664
|
...(leads ? {} : { recovers: appRecovers }),
|
|
659
665
|
run: ({ signal, remainingMs }) => {
|
|
660
|
-
const opening = leads && app.opening?.() === true;
|
|
661
666
|
const stop = new AbortController();
|
|
662
|
-
const
|
|
663
|
-
|
|
667
|
+
const tier = { signal, stop: stop.signal, timeoutMs: remainingMs, leads, postcode: resolvedPostcode };
|
|
668
|
+
if (!leads)
|
|
669
|
+
return app.run(trackingNumber, tier);
|
|
670
|
+
let waiting;
|
|
671
|
+
const opening = new Promise((resolve) => { waiting = resolve; });
|
|
672
|
+
return Promise.race([app.run(trackingNumber, { ...tier, waiting }), opening.then(() => besideGuest(stop))]);
|
|
664
673
|
},
|
|
665
674
|
});
|
|
666
675
|
const result = await runSteps({
|
|
@@ -16,15 +16,14 @@ export const adapter = (environment) => {
|
|
|
16
16
|
recorder: environment.recorder,
|
|
17
17
|
userAgent: environment.userAgent,
|
|
18
18
|
app: {
|
|
19
|
-
run: (number, { signal, stop, timeoutMs, leads, postcode }) => app.track(number, {
|
|
20
|
-
signal, stop, timeoutMs, postcode, sessionWaitMs: leads ? Math.max(0, timeoutMs - GUEST_RESERVE_MS) : timeoutMs,
|
|
19
|
+
run: (number, { signal, stop, timeoutMs, leads, postcode, waiting }) => app.track(number, {
|
|
20
|
+
signal, stop, timeoutMs, postcode, waiting, sessionWaitMs: leads ? Math.max(0, timeoutMs - GUEST_RESERVE_MS) : timeoutMs,
|
|
21
21
|
}),
|
|
22
22
|
// Each service has its own host, limits and outages, so either answers when
|
|
23
23
|
// the other fails. Both read the same parcel: neither can place another
|
|
24
24
|
// country's delivery in Germany, nor find a parcel the guest API does not know.
|
|
25
25
|
recovers: error => carrierErrorKind(error) !== 'not_found'
|
|
26
26
|
&& !(error instanceof CarrierError && error.reason === DPD_DE_OTHER_COUNTRY),
|
|
27
|
-
opening: () => service.opening,
|
|
28
27
|
},
|
|
29
28
|
});
|
|
30
29
|
return {
|
|
@@ -25,8 +25,10 @@ export declare class DpdDeAppClient {
|
|
|
25
25
|
now?: () => number;
|
|
26
26
|
});
|
|
27
27
|
/**
|
|
28
|
-
* `sessionWaitMs`
|
|
29
|
-
*
|
|
28
|
+
* `sessionWaitMs` is how long from its start the lookup may wait for a session still opening, also
|
|
29
|
+
* for one that replaces a session the service refused; the whole `timeoutMs` by default. `waiting`
|
|
30
|
+
* is called when the lookup starts waiting for one. Once `stop` aborts, the lookup stops waiting,
|
|
31
|
+
* ends with its reason and reads nothing.
|
|
30
32
|
* DPD checks a `postcode` against the recipient's: a rejected one gets one lookup without it, and
|
|
31
33
|
* the result says which, as on the guest API.
|
|
32
34
|
*/
|
|
@@ -36,5 +38,6 @@ export declare class DpdDeAppClient {
|
|
|
36
38
|
sessionWaitMs?: number;
|
|
37
39
|
postcode?: string;
|
|
38
40
|
stop?: AbortSignal;
|
|
41
|
+
waiting?: () => void;
|
|
39
42
|
}): Promise<CarrierResult>;
|
|
40
43
|
}
|
|
@@ -195,8 +195,10 @@ export class DpdDeAppClient {
|
|
|
195
195
|
this.#service = options.service ?? new DpdAppService(options);
|
|
196
196
|
}
|
|
197
197
|
/**
|
|
198
|
-
* `sessionWaitMs`
|
|
199
|
-
*
|
|
198
|
+
* `sessionWaitMs` is how long from its start the lookup may wait for a session still opening, also
|
|
199
|
+
* for one that replaces a session the service refused; the whole `timeoutMs` by default. `waiting`
|
|
200
|
+
* is called when the lookup starts waiting for one. Once `stop` aborts, the lookup stops waiting,
|
|
201
|
+
* ends with its reason and reads nothing.
|
|
200
202
|
* DPD checks a `postcode` against the recipient's: a rejected one gets one lookup without it, and
|
|
201
203
|
* the result says which, as on the guest API.
|
|
202
204
|
*/
|
|
@@ -208,9 +210,12 @@ export class DpdDeAppClient {
|
|
|
208
210
|
throw new TypeError('DPD Germany app tracking takes a 5-digit postcode');
|
|
209
211
|
const deadline = performance.now() + options.timeoutMs;
|
|
210
212
|
const left = () => Math.max(1, Math.floor(deadline - performance.now()));
|
|
213
|
+
const waitEnds = Math.min(deadline, performance.now() + (options.sessionWaitMs ?? Infinity));
|
|
211
214
|
try {
|
|
212
215
|
for (let attempt = 0;; attempt += 1) {
|
|
213
|
-
const session = await this.#service.session(options.signal, Math.
|
|
216
|
+
const session = await this.#service.session(options.signal, Math.max(0, waitEnds - performance.now()), {
|
|
217
|
+
waiting: options.waiting, stop: options.stop,
|
|
218
|
+
});
|
|
214
219
|
options.stop?.throwIfAborted();
|
|
215
220
|
// Read-only: the parcel is neither added to the session nor redirected.
|
|
216
221
|
const tracking = (zip) => this.#service.call('getTrackingData', { SessionToken: session, ParcelNo: number, DeliveryZipCode: zip,
|
|
@@ -230,11 +235,13 @@ export class DpdDeAppClient {
|
|
|
230
235
|
data = await tracking('');
|
|
231
236
|
verified = false;
|
|
232
237
|
}
|
|
238
|
+
options.stop?.throwIfAborted();
|
|
233
239
|
const scans = await this.#service.call('getTrackingScanList', { SessionToken: session, ParcelNo: number,
|
|
234
240
|
DeliveryZipCode: verified ? postcode : '' }, options.signal, left());
|
|
235
241
|
const { result, shop } = parseDpdDeApp(data, scans, number);
|
|
236
242
|
// The scans name the shop without its address, which the shop's own record adds.
|
|
237
243
|
// Without it, the parcel is found all the same and its pickup point keeps the name alone.
|
|
244
|
+
options.stop?.throwIfAborted();
|
|
238
245
|
const address = shop ? (await this.#service.parcelShop(shop, { signal: options.signal, timeoutMs: left() }))?.address : undefined;
|
|
239
246
|
if (address)
|
|
240
247
|
result.pickup_point = `${result.pickup_point}\n${address}`;
|
|
@@ -296,6 +296,32 @@
|
|
|
296
296
|
]
|
|
297
297
|
},
|
|
298
298
|
"note": "Parcel number with its printed check character, from the German consumer forum. DPD Germany's tracking returns the fourteen-digit parcel's history."
|
|
299
|
+
},
|
|
300
|
+
{
|
|
301
|
+
"number": "13499302309359E",
|
|
302
|
+
"role": "shipment",
|
|
303
|
+
"evidence": "public_shipment_report",
|
|
304
|
+
"source": {
|
|
305
|
+
"url": "https://web.archive.org/web/20220128062637/https://tracking.dpd.de/status/de_DE/parcel/13499302309359E",
|
|
306
|
+
"date": "2022-01-28",
|
|
307
|
+
"access": "indexed_excerpt",
|
|
308
|
+
"dateKind": "capture"
|
|
309
|
+
},
|
|
310
|
+
"context": {
|
|
311
|
+
"assessment": "candidate"
|
|
312
|
+
},
|
|
313
|
+
"expect": {
|
|
314
|
+
"carrier": "unknown",
|
|
315
|
+
"confidence": "low",
|
|
316
|
+
"candidates": [
|
|
317
|
+
"dpd-pl",
|
|
318
|
+
"dpd",
|
|
319
|
+
"chronopost",
|
|
320
|
+
"dpd-de",
|
|
321
|
+
"dpd-uk"
|
|
322
|
+
]
|
|
323
|
+
},
|
|
324
|
+
"note": "Printed with its check character. Its 13 depot range is the one DPD Poland's own archived tracking pages show. Neither DPD Germany nor DPD Poland found it in October 2026."
|
|
299
325
|
}
|
|
300
326
|
]
|
|
301
327
|
}
|
|
@@ -47,6 +47,32 @@ export interface DpdSessionStore {
|
|
|
47
47
|
/** Saves a session, replacing what was saved for the same token. */
|
|
48
48
|
save(session: DpdSession): Promise<void>;
|
|
49
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* What a long-lived host learns of the sessions the service keeps, without their tokens: a
|
|
52
|
+
* session taken up from the store at start, and each opening once it ends, whatever began it.
|
|
53
|
+
*/
|
|
54
|
+
export interface DpdSessionEvent {
|
|
55
|
+
outcome: 'taken_up' | 'opened' | 'failed';
|
|
56
|
+
/**
|
|
57
|
+
* What began the opening: `start` when the host kept the service, `renewal` when the current
|
|
58
|
+
* session reached its renewal age, `retry` a quarter of an hour after a failed opening,
|
|
59
|
+
* `refused` when the service no longer accepted the session, and `lookup` when a lookup found
|
|
60
|
+
* none open. A session taken up is always `start`.
|
|
61
|
+
*/
|
|
62
|
+
trigger: 'start' | 'renewal' | 'retry' | 'refused' | 'lookup';
|
|
63
|
+
/** How long the opening took, or the store's load for a session taken up. */
|
|
64
|
+
durationMs: number;
|
|
65
|
+
/** For a session taken up: how long ago it opened. */
|
|
66
|
+
ageMs?: number;
|
|
67
|
+
/** For a failure: its error kind, such as `transport` or `schema`, else its class name. */
|
|
68
|
+
errorKind?: string;
|
|
69
|
+
}
|
|
70
|
+
/** What `keep` takes, as `warmDpdSession` does. */
|
|
71
|
+
interface KeepOptions {
|
|
72
|
+
store?: DpdSessionStore;
|
|
73
|
+
/** Told of each session taken up or opening that ends. It is called apart from lookups, and what it throws or rejects is ignored. */
|
|
74
|
+
onSession?: (event: DpdSessionEvent) => void | Promise<void>;
|
|
75
|
+
}
|
|
50
76
|
/**
|
|
51
77
|
* The service needs an anonymous device session, which it takes tens of seconds to open and then
|
|
52
78
|
* accepts for hours. One is kept per instance, and `sharedDpdAppService` gives the process one
|
|
@@ -69,7 +95,7 @@ export declare class DpdAppService {
|
|
|
69
95
|
signal: AbortSignal;
|
|
70
96
|
timeoutMs: number;
|
|
71
97
|
}): Promise<DpdParcelShop | undefined>;
|
|
72
|
-
/** Whether
|
|
98
|
+
/** Whether no session is open, as while one opens or a host's store gives back its sessions. */
|
|
73
99
|
get opening(): boolean;
|
|
74
100
|
/** Forgets a session the service refused, unless another lookup already replaced it. */
|
|
75
101
|
expire(session: string): void;
|
|
@@ -77,16 +103,22 @@ export declare class DpdAppService {
|
|
|
77
103
|
* Keeps a session open from now on, for a long-lived host: one opens at once if there is none,
|
|
78
104
|
* and the next before the current one lapses. Openings then run whether or not a lookup waits
|
|
79
105
|
* for them. With a store, the current session saved there is taken up again instead, and the
|
|
80
|
-
* sessions lookups no longer use are checked until the service refuses them.
|
|
81
|
-
* keep the process alive.
|
|
106
|
+
* sessions lookups no longer use are checked until the service refuses them. `onSession` hears
|
|
107
|
+
* of each session taken up or opening that ends. The timers do not keep the process alive.
|
|
108
|
+
* Only the first call counts.
|
|
82
109
|
*/
|
|
83
|
-
keep(
|
|
110
|
+
keep(options?: KeepOptions): void;
|
|
84
111
|
/**
|
|
85
112
|
* One session for every lookup. The opening runs while a lookup that asked for it is still
|
|
86
113
|
* running, even one that stopped waiting; each lookup waits until its signal or `waitMs` ends.
|
|
87
|
-
* A session past its renewal age answers at once while the next one opens.
|
|
114
|
+
* A session past its renewal age answers at once while the next one opens. `waiting` is called
|
|
115
|
+
* when the lookup starts waiting for an opening, once a host's store has given back its sessions.
|
|
116
|
+
* A lookup whose `stop` aborts stops waiting with its reason, and still holds the opening.
|
|
88
117
|
*/
|
|
89
|
-
session(signal: AbortSignal, waitMs: number
|
|
118
|
+
session(signal: AbortSignal, waitMs: number, options?: {
|
|
119
|
+
waiting?: () => void;
|
|
120
|
+
stop?: AbortSignal;
|
|
121
|
+
}): Promise<string>;
|
|
90
122
|
call(operation: string, fields: Fields, signal: AbortSignal, timeoutMs: number): Promise<XmlNode>;
|
|
91
123
|
}
|
|
92
124
|
/**
|
|
@@ -101,9 +133,12 @@ export declare function sharedDpdAppService(options?: Pick<AdapterEnvironment, '
|
|
|
101
133
|
* seconds an opening takes. For a long-lived host, with the `fetcher` and `userAgent` it gives
|
|
102
134
|
* `createTracker` or its adapter environment. With a `store`, a restart takes the saved session up
|
|
103
135
|
* again, and the sessions no longer used are checked every hour until the service refuses them,
|
|
104
|
-
* which the store records.
|
|
136
|
+
* which the store records. `onSession` hears of each session taken up and each opening that ends,
|
|
137
|
+
* so the host can tell how long openings take and when one fails. The timers do not keep the
|
|
138
|
+
* process alive.
|
|
105
139
|
*/
|
|
106
140
|
export declare function warmDpdSession(environment?: Pick<AdapterEnvironment, 'fetcher' | 'userAgent'>, options?: {
|
|
107
141
|
store?: DpdSessionStore;
|
|
142
|
+
onSession?: (event: DpdSessionEvent) => void | Promise<void>;
|
|
108
143
|
}): void;
|
|
109
144
|
export {};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash, randomBytes } from 'node:crypto';
|
|
2
|
-
import { ChallengeError, IndeterminateError, SchemaError, TransportError } from '../../core/errors/index.js';
|
|
2
|
+
import { carrierErrorKind, ChallengeError, errorTypeOf, IndeterminateError, SchemaError, TransportError } from '../../core/errors/index.js';
|
|
3
3
|
import { clean, decodeText, fetchBounded, userAgentOf } from '../../core/transport/index.js';
|
|
4
4
|
import { transportOf } from '../../core/transport/transportOf.js';
|
|
5
5
|
import { xmlDocument } from '../../core/transport/xml.js';
|
|
@@ -116,6 +116,9 @@ export class DpdAppService {
|
|
|
116
116
|
#kept = false;
|
|
117
117
|
#timer;
|
|
118
118
|
#store;
|
|
119
|
+
#onSession;
|
|
120
|
+
/** The service refused the last session, and none has opened since. */
|
|
121
|
+
#refused = false;
|
|
119
122
|
#loading = null;
|
|
120
123
|
#retired = new Map();
|
|
121
124
|
constructor(options = {}) {
|
|
@@ -150,7 +153,7 @@ export class DpdAppService {
|
|
|
150
153
|
return undefined;
|
|
151
154
|
}
|
|
152
155
|
}
|
|
153
|
-
/** Whether
|
|
156
|
+
/** Whether no session is open, as while one opens or a host's store gives back its sessions. */
|
|
154
157
|
get opening() {
|
|
155
158
|
return this.#session === null;
|
|
156
159
|
}
|
|
@@ -160,29 +163,40 @@ export class DpdAppService {
|
|
|
160
163
|
return;
|
|
161
164
|
this.#retire(this.#session, this.#now());
|
|
162
165
|
this.#session = null;
|
|
166
|
+
this.#refused = true;
|
|
163
167
|
}
|
|
164
168
|
/**
|
|
165
169
|
* Keeps a session open from now on, for a long-lived host: one opens at once if there is none,
|
|
166
170
|
* and the next before the current one lapses. Openings then run whether or not a lookup waits
|
|
167
171
|
* for them. With a store, the current session saved there is taken up again instead, and the
|
|
168
|
-
* sessions lookups no longer use are checked until the service refuses them.
|
|
169
|
-
* keep the process alive.
|
|
172
|
+
* sessions lookups no longer use are checked until the service refuses them. `onSession` hears
|
|
173
|
+
* of each session taken up or opening that ends. The timers do not keep the process alive.
|
|
174
|
+
* Only the first call counts.
|
|
170
175
|
*/
|
|
171
|
-
keep(
|
|
176
|
+
keep(options = {}) {
|
|
172
177
|
if (this.#kept)
|
|
173
178
|
return;
|
|
174
179
|
this.#kept = true;
|
|
180
|
+
const { store, onSession } = options;
|
|
175
181
|
this.#store = store;
|
|
182
|
+
this.#onSession = onSession;
|
|
176
183
|
if (!store)
|
|
177
|
-
return this.#renew();
|
|
184
|
+
return this.#renew('start');
|
|
178
185
|
this.#loading = this.#load(store).finally(() => {
|
|
179
186
|
this.#loading = null;
|
|
180
|
-
this.#renew();
|
|
187
|
+
this.#renew('start');
|
|
181
188
|
this.#follow();
|
|
182
189
|
});
|
|
183
190
|
}
|
|
191
|
+
/** Tells the host's observer, if there is one, apart from the lookup and whatever it does. */
|
|
192
|
+
#report(event) {
|
|
193
|
+
const observer = this.#onSession;
|
|
194
|
+
if (observer)
|
|
195
|
+
void Promise.resolve().then(() => observer(event)).catch(() => undefined);
|
|
196
|
+
}
|
|
184
197
|
/** Takes up the newest saved session while it is younger than the renewal age, and follows the others. */
|
|
185
198
|
async #load(store) {
|
|
199
|
+
const started = this.#now();
|
|
186
200
|
let timer;
|
|
187
201
|
const late = new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error('The store did not answer')), STORE_LOAD_MS); });
|
|
188
202
|
let saved;
|
|
@@ -202,8 +216,10 @@ export class DpdAppService {
|
|
|
202
216
|
&& session.refusedAt == null)
|
|
203
217
|
.sort((left, right) => right.openedAt - left.openedAt);
|
|
204
218
|
const [newest, ...older] = sessions;
|
|
205
|
-
if (newest && !this.#session && now - newest.openedAt < SESSION_RENEW_MS)
|
|
219
|
+
if (newest && !this.#session && now - newest.openedAt < SESSION_RENEW_MS) {
|
|
206
220
|
this.#session = { token: newest.token, openedAt: newest.openedAt };
|
|
221
|
+
this.#report({ outcome: 'taken_up', trigger: 'start', durationMs: Math.max(0, now - started), ageMs: now - newest.openedAt });
|
|
222
|
+
}
|
|
207
223
|
else if (newest)
|
|
208
224
|
older.unshift(newest);
|
|
209
225
|
for (const session of older)
|
|
@@ -270,24 +286,25 @@ export class DpdAppService {
|
|
|
270
286
|
}
|
|
271
287
|
}
|
|
272
288
|
/** Opens the next session if the current one is due, or waits until it is. */
|
|
273
|
-
#renew() {
|
|
289
|
+
#renew(trigger) {
|
|
274
290
|
if (this.#opening)
|
|
275
291
|
return;
|
|
276
292
|
const age = this.#session ? this.#now() - this.#session.openedAt : Infinity;
|
|
277
293
|
if (age < SESSION_RENEW_MS)
|
|
278
|
-
this.#schedule(SESSION_RENEW_MS - age);
|
|
294
|
+
this.#schedule(SESSION_RENEW_MS - age, 'renewal');
|
|
279
295
|
else
|
|
280
|
-
this.#opening = this.#open();
|
|
296
|
+
this.#opening = this.#open(trigger);
|
|
281
297
|
}
|
|
282
|
-
#schedule(ms) {
|
|
298
|
+
#schedule(ms, trigger) {
|
|
283
299
|
if (!this.#kept)
|
|
284
300
|
return;
|
|
285
301
|
clearTimeout(this.#timer);
|
|
286
|
-
this.#timer = setTimeout(() => this.#renew(), ms);
|
|
302
|
+
this.#timer = setTimeout(() => this.#renew(trigger), ms);
|
|
287
303
|
this.#timer.unref?.();
|
|
288
304
|
}
|
|
289
305
|
/** An opening, which a kept service finishes and otherwise ends when no lookup holds it. */
|
|
290
|
-
#open() {
|
|
306
|
+
#open(trigger) {
|
|
307
|
+
const started = this.#now();
|
|
291
308
|
const controller = new AbortController();
|
|
292
309
|
const holders = new Set();
|
|
293
310
|
const token = this.call('getSessionFullState', { SessionToken: '', DeviceData: {
|
|
@@ -299,6 +316,7 @@ export class DpdAppService {
|
|
|
299
316
|
if (this.#session)
|
|
300
317
|
this.#retire(this.#session);
|
|
301
318
|
this.#session = { token: value, openedAt: this.#now() };
|
|
319
|
+
this.#refused = false;
|
|
302
320
|
this.#save(this.#session);
|
|
303
321
|
return value;
|
|
304
322
|
});
|
|
@@ -310,7 +328,14 @@ export class DpdAppService {
|
|
|
310
328
|
controller.abort(new Error('No lookup is waiting for the DPD Germany session'));
|
|
311
329
|
};
|
|
312
330
|
const opening = { token, holders, release };
|
|
313
|
-
|
|
331
|
+
const durationMs = () => Math.max(0, this.#now() - started);
|
|
332
|
+
void token.then(() => {
|
|
333
|
+
this.#report({ outcome: 'opened', trigger, durationMs: durationMs() });
|
|
334
|
+
this.#schedule(SESSION_RENEW_MS, 'renewal');
|
|
335
|
+
}, (error) => {
|
|
336
|
+
this.#report({ outcome: 'failed', trigger, durationMs: durationMs(), errorKind: carrierErrorKind(error) ?? errorTypeOf(error) });
|
|
337
|
+
this.#schedule(SESSION_RETRY_MS, 'retry');
|
|
338
|
+
}).finally(() => {
|
|
314
339
|
if (this.#opening === opening)
|
|
315
340
|
this.#opening = null;
|
|
316
341
|
for (const holder of holders)
|
|
@@ -321,9 +346,11 @@ export class DpdAppService {
|
|
|
321
346
|
/**
|
|
322
347
|
* One session for every lookup. The opening runs while a lookup that asked for it is still
|
|
323
348
|
* running, even one that stopped waiting; each lookup waits until its signal or `waitMs` ends.
|
|
324
|
-
* A session past its renewal age answers at once while the next one opens.
|
|
349
|
+
* A session past its renewal age answers at once while the next one opens. `waiting` is called
|
|
350
|
+
* when the lookup starts waiting for an opening, once a host's store has given back its sessions.
|
|
351
|
+
* A lookup whose `stop` aborts stops waiting with its reason, and still holds the opening.
|
|
325
352
|
*/
|
|
326
|
-
async session(signal, waitMs) {
|
|
353
|
+
async session(signal, waitMs, options = {}) {
|
|
327
354
|
if (this.#loading)
|
|
328
355
|
await until(this.#loading, signal);
|
|
329
356
|
const current = this.#session;
|
|
@@ -331,7 +358,7 @@ export class DpdAppService {
|
|
|
331
358
|
if (current && !due)
|
|
332
359
|
return current.token;
|
|
333
360
|
signal.throwIfAborted();
|
|
334
|
-
const opening = this.#opening ??= this.#open();
|
|
361
|
+
const opening = this.#opening ??= this.#open(current ? 'renewal' : this.#refused ? 'refused' : 'lookup');
|
|
335
362
|
const { token, holders, release } = opening;
|
|
336
363
|
if (!holders.has(signal)) {
|
|
337
364
|
holders.add(signal);
|
|
@@ -339,21 +366,20 @@ export class DpdAppService {
|
|
|
339
366
|
}
|
|
340
367
|
if (current)
|
|
341
368
|
return current.token;
|
|
342
|
-
|
|
369
|
+
options.waiting?.();
|
|
370
|
+
const ended = options.stop ? AbortSignal.any([signal, options.stop]) : signal;
|
|
371
|
+
ended.throwIfAborted();
|
|
343
372
|
let timer;
|
|
344
|
-
const
|
|
345
|
-
leave = () => reject(signal.reason);
|
|
373
|
+
const late = new Promise((_resolve, reject) => {
|
|
346
374
|
timer = setTimeout(() => reject(new IndeterminateError('DPD Germany', 'DPD Germany app session is still opening', {
|
|
347
375
|
reason: DPD_DE_SESSION_OPENING,
|
|
348
376
|
})), Math.max(0, Math.min(waitMs, SESSION_OPEN_MS)));
|
|
349
377
|
});
|
|
350
|
-
signal.addEventListener('abort', leave, { once: true });
|
|
351
378
|
try {
|
|
352
|
-
return await Promise.race([token,
|
|
379
|
+
return await until(Promise.race([token, late]), ended);
|
|
353
380
|
}
|
|
354
381
|
finally {
|
|
355
382
|
clearTimeout(timer);
|
|
356
|
-
signal.removeEventListener('abort', leave);
|
|
357
383
|
}
|
|
358
384
|
}
|
|
359
385
|
async call(operation, fields, signal, timeoutMs) {
|
|
@@ -413,8 +439,10 @@ export function sharedDpdAppService(options = {}) {
|
|
|
413
439
|
* seconds an opening takes. For a long-lived host, with the `fetcher` and `userAgent` it gives
|
|
414
440
|
* `createTracker` or its adapter environment. With a `store`, a restart takes the saved session up
|
|
415
441
|
* again, and the sessions no longer used are checked every hour until the service refuses them,
|
|
416
|
-
* which the store records.
|
|
442
|
+
* which the store records. `onSession` hears of each session taken up and each opening that ends,
|
|
443
|
+
* so the host can tell how long openings take and when one fails. The timers do not keep the
|
|
444
|
+
* process alive.
|
|
417
445
|
*/
|
|
418
446
|
export function warmDpdSession(environment = {}, options = {}) {
|
|
419
|
-
sharedDpdAppService(environment).keep(options
|
|
447
|
+
sharedDpdAppService(environment).keep(options);
|
|
420
448
|
}
|
|
@@ -3,6 +3,7 @@ import { hasGs1CheckDigit } from '../../core/detection/numericChecksums.js';
|
|
|
3
3
|
import { ChallengeError, IndeterminateError, InvalidInputError, NotFoundError, SchemaError } from '../../core/errors/index.js';
|
|
4
4
|
import { eventPoint } from '../../core/result/index.js';
|
|
5
5
|
import { runSteps } from '../../core/runner/index.js';
|
|
6
|
+
import { firstAttemptMs, networkRetryStep } from '../../core/runner/networkRetry.js';
|
|
6
7
|
import { zonedTime } from '../../core/time/index.js';
|
|
7
8
|
import { TrawlClient, clean, decodeText, fetchBounded } from '../../core/transport/index.js';
|
|
8
9
|
import { isDeliveryStatusCell } from './identity.js';
|
|
@@ -235,24 +236,25 @@ export class DPDFranceTracker {
|
|
|
235
236
|
const number = normalizeDPDFranceTrackingNumber(rawTrackingNumber);
|
|
236
237
|
const url = dpdFranceTrackingUrl(number);
|
|
237
238
|
const trawl = this.browserService();
|
|
239
|
+
const direct = async (timeoutMs, signal) => {
|
|
240
|
+
try {
|
|
241
|
+
return await this.directGet(url, capped(this.directTimeoutMs, timeoutMs), signal);
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
// Without a browser service there is no second tier, so the
|
|
245
|
+
// challenge has to carry the operator's next step itself.
|
|
246
|
+
if (trawl || !(error instanceof DPDFranceChallengeError))
|
|
247
|
+
throw error;
|
|
248
|
+
throw new DPDFranceChallengeError('DPD France requires a browser challenge solver; configure FLARESOLVERR_URL');
|
|
249
|
+
}
|
|
250
|
+
};
|
|
238
251
|
const html = await runSteps({
|
|
239
252
|
carrier: 'dpd-fr', budgetMs: context.budgetMs ?? this.budgetMs, signal: context.signal, recorder: this.recorder,
|
|
240
253
|
}, [
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
return await this.directGet(url, capped(this.directTimeoutMs, remainingMs), signal);
|
|
246
|
-
}
|
|
247
|
-
catch (error) {
|
|
248
|
-
// Without a browser service there is no second tier, so the
|
|
249
|
-
// challenge has to carry the operator's next step itself.
|
|
250
|
-
if (trawl || !(error instanceof DPDFranceChallengeError))
|
|
251
|
-
throw error;
|
|
252
|
-
throw new DPDFranceChallengeError('DPD France requires a browser challenge solver; configure FLARESOLVERR_URL');
|
|
253
|
-
}
|
|
254
|
-
},
|
|
255
|
-
},
|
|
254
|
+
// A GET that fails to reach DPD, or hangs, is sent once more: the first
|
|
255
|
+
// gets at most half of what is left, so the second fits in the rest.
|
|
256
|
+
{ id: 'direct', run: ({ remainingMs, signal }) => direct(firstAttemptMs(remainingMs), signal) },
|
|
257
|
+
networkRetryStep(({ remainingMs, signal }) => direct(remainingMs, signal)),
|
|
256
258
|
{
|
|
257
259
|
id: 'trawl',
|
|
258
260
|
enabled: trawl !== null,
|
|
@@ -315,7 +317,7 @@ export const adapter = (environment) => {
|
|
|
315
317
|
id: 'dpd-fr',
|
|
316
318
|
// One direct HTML GET, then the browser service's native scrape API when
|
|
317
319
|
// Cloudflare challenges it.
|
|
318
|
-
steps: ['direct', 'trawl'],
|
|
320
|
+
steps: ['direct', 'retry', 'trawl'],
|
|
319
321
|
track: (input, context) => tracker.fetch(input.number, context),
|
|
320
322
|
};
|
|
321
323
|
};
|