universal-parcel-scraper 0.56.0-main.720 → 0.56.0-main.723

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,8 +50,9 @@ var e = {
50
50
  }
51
51
  }, l = class extends t {
52
52
  field;
53
+ carrier;
53
54
  constructor(e, t, n = `${e} tracking requires ${t}`, r) {
54
- super("input_required", e, n, r), this.name = "InputRequiredError", this.field = t;
55
+ super("input_required", e, n, r), this.name = "InputRequiredError", this.field = t, r?.carrier !== void 0 && (this.carrier = r.carrier);
55
56
  }
56
57
  }, u = class extends t {
57
58
  constructor(e, t = `${e} cannot look up this tracking number`, n) {
@@ -1,6 +1,7 @@
1
1
  import { accepted, recognizeFromLookup } from '../../core/adapter/index.js';
2
2
  import { IndeterminateError } from '../../core/errors/index.js';
3
3
  import { runSteps } from '../../core/runner/index.js';
4
+ import { firstAttemptMs, networkRetryStep } from '../../core/runner/networkRetry.js';
4
5
  import { decodeText, fetchBounded, userAgentOf } from '../../core/transport/index.js';
5
6
  import { CHRONOPOST_MAX_BYTES, CHRONOPOST_NAMESPACE, normalizeChronopostNumber, parseChronopostTrackingXml } from './parser.js';
6
7
  // The official WSDL declares this read-only operation without account credentials.
@@ -12,28 +13,34 @@ export class ChronopostTracker {
12
13
  }
13
14
  fetch(raw, context = {}) {
14
15
  const number = normalizeChronopostNumber(raw);
16
+ const read = async (signal, timeoutMs) => {
17
+ const { response, bytes } = await fetchBounded(ENDPOINT, {
18
+ method: 'POST', signal,
19
+ headers: { Accept: 'text/xml', 'Content-Type': 'text/xml; charset=utf-8', SOAPAction: '""',
20
+ 'User-Agent': userAgentOf(this.options.userAgent) },
21
+ body: `<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:trk="${CHRONOPOST_NAMESPACE}">`
22
+ + `<soap:Body><trk:trackSkybillV2><language>fr_FR</language><skybillNumber>${number}</skybillNumber>`
23
+ + '</trk:trackSkybillV2></soap:Body></soap:Envelope>',
24
+ }, { provider: 'Chronopost tracking', timeoutMs: Math.max(1, Math.floor(timeoutMs)),
25
+ maxBytes: CHRONOPOST_MAX_BYTES, allowHttpStatuses: [404, 410], fetcher: this.options.fetcher });
26
+ // An HTTP route failure is not the service's positive unknown-parcel reply.
27
+ if ([404, 410].includes(response.status))
28
+ throw new IndeterminateError('Chronopost', 'Chronopost tracking endpoint is unavailable');
29
+ return parseChronopostTrackingXml(decodeText(bytes), number);
30
+ };
31
+ // A request that fails to reach Chronopost, or hangs, gets one more try:
32
+ // the first gets half of the budget, so the retry fits in the rest.
15
33
  return runSteps({ carrier: 'chronopost', budgetMs: context.budgetMs ?? 15_000, signal: context.signal,
16
- recorder: this.options.recorder }, [{ id: 'direct', run: async ({ signal, remainingMs }) => {
17
- const { response, bytes } = await fetchBounded(ENDPOINT, {
18
- method: 'POST', signal,
19
- headers: { Accept: 'text/xml', 'Content-Type': 'text/xml; charset=utf-8', SOAPAction: '""',
20
- 'User-Agent': userAgentOf(this.options.userAgent) },
21
- body: `<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:trk="${CHRONOPOST_NAMESPACE}">`
22
- + `<soap:Body><trk:trackSkybillV2><language>fr_FR</language><skybillNumber>${number}</skybillNumber>`
23
- + '</trk:trackSkybillV2></soap:Body></soap:Envelope>',
24
- }, { provider: 'Chronopost tracking', timeoutMs: Math.max(1, Math.floor(remainingMs)),
25
- maxBytes: CHRONOPOST_MAX_BYTES, allowHttpStatuses: [404, 410], fetcher: this.options.fetcher });
26
- // An HTTP route failure is not the service's positive unknown-parcel reply.
27
- if ([404, 410].includes(response.status))
28
- throw new IndeterminateError('Chronopost', 'Chronopost tracking endpoint is unavailable');
29
- return parseChronopostTrackingXml(decodeText(bytes), number);
30
- } }]);
34
+ recorder: this.options.recorder }, [
35
+ { id: 'direct', run: ({ signal, remainingMs }) => read(signal, firstAttemptMs(remainingMs)) },
36
+ networkRetryStep(({ signal, remainingMs }) => read(signal, remainingMs)),
37
+ ]);
31
38
  }
32
39
  }
33
40
  export const adapter = environment => {
34
41
  const tracker = new ChronopostTracker(environment);
35
42
  return {
36
- id: 'chronopost', recordsSteps: true, steps: ['direct'],
43
+ id: 'chronopost', recordsSteps: true, steps: ['direct', 'retry'],
37
44
  track: (input, context) => tracker.fetch(input.number, context),
38
45
  recognize: (number, context) => recognizeFromLookup(() => tracker.fetch(number, context), () => accepted(() => normalizeChronopostNumber(number))),
39
46
  };
@@ -47,6 +47,32 @@ export interface DpdSessionStore {
47
47
  /** Saves a session, replacing what was saved for the same token. */
48
48
  save(session: DpdSession): Promise<void>;
49
49
  }
50
+ /**
51
+ * What a long-lived host learns of the sessions the service keeps, without their tokens: a
52
+ * session taken up from the store at start, and each opening once it ends, whatever began it.
53
+ */
54
+ export interface DpdSessionEvent {
55
+ outcome: 'taken_up' | 'opened' | 'failed';
56
+ /**
57
+ * What began the opening: `start` when the host kept the service, `renewal` when the current
58
+ * session reached its renewal age, `retry` a quarter of an hour after a failed opening,
59
+ * `refused` when the service no longer accepted the session, and `lookup` when a lookup found
60
+ * none open. A session taken up is always `start`.
61
+ */
62
+ trigger: 'start' | 'renewal' | 'retry' | 'refused' | 'lookup';
63
+ /** How long the opening took, or the store's load for a session taken up. */
64
+ durationMs: number;
65
+ /** For a session taken up: how long ago it opened. */
66
+ ageMs?: number;
67
+ /** For a failure: its error kind, such as `transport` or `schema`, else its class name. */
68
+ errorKind?: string;
69
+ }
70
+ /** What `keep` takes, as `warmDpdSession` does. */
71
+ interface KeepOptions {
72
+ store?: DpdSessionStore;
73
+ /** Told of each session taken up or opening that ends. It is called apart from lookups, and what it throws or rejects is ignored. */
74
+ onSession?: (event: DpdSessionEvent) => void | Promise<void>;
75
+ }
50
76
  /**
51
77
  * The service needs an anonymous device session, which it takes tens of seconds to open and then
52
78
  * accepts for hours. One is kept per instance, and `sharedDpdAppService` gives the process one
@@ -77,10 +103,11 @@ export declare class DpdAppService {
77
103
  * Keeps a session open from now on, for a long-lived host: one opens at once if there is none,
78
104
  * and the next before the current one lapses. Openings then run whether or not a lookup waits
79
105
  * for them. With a store, the current session saved there is taken up again instead, and the
80
- * sessions lookups no longer use are checked until the service refuses them. The timers do not
81
- * keep the process alive.
106
+ * sessions lookups no longer use are checked until the service refuses them. `onSession` hears
107
+ * of each session taken up or opening that ends. The timers do not keep the process alive.
108
+ * Only the first call counts.
82
109
  */
83
- keep(store?: DpdSessionStore): void;
110
+ keep(options?: KeepOptions): void;
84
111
  /**
85
112
  * One session for every lookup. The opening runs while a lookup that asked for it is still
86
113
  * running, even one that stopped waiting; each lookup waits until its signal or `waitMs` ends.
@@ -106,9 +133,12 @@ export declare function sharedDpdAppService(options?: Pick<AdapterEnvironment, '
106
133
  * seconds an opening takes. For a long-lived host, with the `fetcher` and `userAgent` it gives
107
134
  * `createTracker` or its adapter environment. With a `store`, a restart takes the saved session up
108
135
  * again, and the sessions no longer used are checked every hour until the service refuses them,
109
- * which the store records. The timers do not keep the process alive.
136
+ * which the store records. `onSession` hears of each session taken up and each opening that ends,
137
+ * so the host can tell how long openings take and when one fails. The timers do not keep the
138
+ * process alive.
110
139
  */
111
140
  export declare function warmDpdSession(environment?: Pick<AdapterEnvironment, 'fetcher' | 'userAgent'>, options?: {
112
141
  store?: DpdSessionStore;
142
+ onSession?: (event: DpdSessionEvent) => void | Promise<void>;
113
143
  }): void;
114
144
  export {};
@@ -1,5 +1,5 @@
1
1
  import { createHash, randomBytes } from 'node:crypto';
2
- import { ChallengeError, IndeterminateError, SchemaError, TransportError } from '../../core/errors/index.js';
2
+ import { carrierErrorKind, ChallengeError, errorTypeOf, IndeterminateError, SchemaError, TransportError } from '../../core/errors/index.js';
3
3
  import { clean, decodeText, fetchBounded, userAgentOf } from '../../core/transport/index.js';
4
4
  import { transportOf } from '../../core/transport/transportOf.js';
5
5
  import { xmlDocument } from '../../core/transport/xml.js';
@@ -116,6 +116,9 @@ export class DpdAppService {
116
116
  #kept = false;
117
117
  #timer;
118
118
  #store;
119
+ #onSession;
120
+ /** The service refused the last session, and none has opened since. */
121
+ #refused = false;
119
122
  #loading = null;
120
123
  #retired = new Map();
121
124
  constructor(options = {}) {
@@ -160,29 +163,40 @@ export class DpdAppService {
160
163
  return;
161
164
  this.#retire(this.#session, this.#now());
162
165
  this.#session = null;
166
+ this.#refused = true;
163
167
  }
164
168
  /**
165
169
  * Keeps a session open from now on, for a long-lived host: one opens at once if there is none,
166
170
  * and the next before the current one lapses. Openings then run whether or not a lookup waits
167
171
  * for them. With a store, the current session saved there is taken up again instead, and the
168
- * sessions lookups no longer use are checked until the service refuses them. The timers do not
169
- * keep the process alive.
172
+ * sessions lookups no longer use are checked until the service refuses them. `onSession` hears
173
+ * of each session taken up or opening that ends. The timers do not keep the process alive.
174
+ * Only the first call counts.
170
175
  */
171
- keep(store) {
176
+ keep(options = {}) {
172
177
  if (this.#kept)
173
178
  return;
174
179
  this.#kept = true;
180
+ const { store, onSession } = options;
175
181
  this.#store = store;
182
+ this.#onSession = onSession;
176
183
  if (!store)
177
- return this.#renew();
184
+ return this.#renew('start');
178
185
  this.#loading = this.#load(store).finally(() => {
179
186
  this.#loading = null;
180
- this.#renew();
187
+ this.#renew('start');
181
188
  this.#follow();
182
189
  });
183
190
  }
191
+ /** Tells the host's observer, if there is one, apart from the lookup and whatever it does. */
192
+ #report(event) {
193
+ const observer = this.#onSession;
194
+ if (observer)
195
+ void Promise.resolve().then(() => observer(event)).catch(() => undefined);
196
+ }
184
197
  /** Takes up the newest saved session while it is younger than the renewal age, and follows the others. */
185
198
  async #load(store) {
199
+ const started = this.#now();
186
200
  let timer;
187
201
  const late = new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error('The store did not answer')), STORE_LOAD_MS); });
188
202
  let saved;
@@ -202,8 +216,10 @@ export class DpdAppService {
202
216
  && session.refusedAt == null)
203
217
  .sort((left, right) => right.openedAt - left.openedAt);
204
218
  const [newest, ...older] = sessions;
205
- if (newest && !this.#session && now - newest.openedAt < SESSION_RENEW_MS)
219
+ if (newest && !this.#session && now - newest.openedAt < SESSION_RENEW_MS) {
206
220
  this.#session = { token: newest.token, openedAt: newest.openedAt };
221
+ this.#report({ outcome: 'taken_up', trigger: 'start', durationMs: Math.max(0, now - started), ageMs: now - newest.openedAt });
222
+ }
207
223
  else if (newest)
208
224
  older.unshift(newest);
209
225
  for (const session of older)
@@ -270,24 +286,25 @@ export class DpdAppService {
270
286
  }
271
287
  }
272
288
  /** Opens the next session if the current one is due, or waits until it is. */
273
- #renew() {
289
+ #renew(trigger) {
274
290
  if (this.#opening)
275
291
  return;
276
292
  const age = this.#session ? this.#now() - this.#session.openedAt : Infinity;
277
293
  if (age < SESSION_RENEW_MS)
278
- this.#schedule(SESSION_RENEW_MS - age);
294
+ this.#schedule(SESSION_RENEW_MS - age, 'renewal');
279
295
  else
280
- this.#opening = this.#open();
296
+ this.#opening = this.#open(trigger);
281
297
  }
282
- #schedule(ms) {
298
+ #schedule(ms, trigger) {
283
299
  if (!this.#kept)
284
300
  return;
285
301
  clearTimeout(this.#timer);
286
- this.#timer = setTimeout(() => this.#renew(), ms);
302
+ this.#timer = setTimeout(() => this.#renew(trigger), ms);
287
303
  this.#timer.unref?.();
288
304
  }
289
305
  /** An opening, which a kept service finishes and otherwise ends when no lookup holds it. */
290
- #open() {
306
+ #open(trigger) {
307
+ const started = this.#now();
291
308
  const controller = new AbortController();
292
309
  const holders = new Set();
293
310
  const token = this.call('getSessionFullState', { SessionToken: '', DeviceData: {
@@ -299,6 +316,7 @@ export class DpdAppService {
299
316
  if (this.#session)
300
317
  this.#retire(this.#session);
301
318
  this.#session = { token: value, openedAt: this.#now() };
319
+ this.#refused = false;
302
320
  this.#save(this.#session);
303
321
  return value;
304
322
  });
@@ -310,7 +328,14 @@ export class DpdAppService {
310
328
  controller.abort(new Error('No lookup is waiting for the DPD Germany session'));
311
329
  };
312
330
  const opening = { token, holders, release };
313
- void token.then(() => this.#schedule(SESSION_RENEW_MS), () => this.#schedule(SESSION_RETRY_MS)).finally(() => {
331
+ const durationMs = () => Math.max(0, this.#now() - started);
332
+ void token.then(() => {
333
+ this.#report({ outcome: 'opened', trigger, durationMs: durationMs() });
334
+ this.#schedule(SESSION_RENEW_MS, 'renewal');
335
+ }, (error) => {
336
+ this.#report({ outcome: 'failed', trigger, durationMs: durationMs(), errorKind: carrierErrorKind(error) ?? errorTypeOf(error) });
337
+ this.#schedule(SESSION_RETRY_MS, 'retry');
338
+ }).finally(() => {
314
339
  if (this.#opening === opening)
315
340
  this.#opening = null;
316
341
  for (const holder of holders)
@@ -333,7 +358,7 @@ export class DpdAppService {
333
358
  if (current && !due)
334
359
  return current.token;
335
360
  signal.throwIfAborted();
336
- const opening = this.#opening ??= this.#open();
361
+ const opening = this.#opening ??= this.#open(current ? 'renewal' : this.#refused ? 'refused' : 'lookup');
337
362
  const { token, holders, release } = opening;
338
363
  if (!holders.has(signal)) {
339
364
  holders.add(signal);
@@ -414,8 +439,10 @@ export function sharedDpdAppService(options = {}) {
414
439
  * seconds an opening takes. For a long-lived host, with the `fetcher` and `userAgent` it gives
415
440
  * `createTracker` or its adapter environment. With a `store`, a restart takes the saved session up
416
441
  * again, and the sessions no longer used are checked every hour until the service refuses them,
417
- * which the store records. The timers do not keep the process alive.
442
+ * which the store records. `onSession` hears of each session taken up and each opening that ends,
443
+ * so the host can tell how long openings take and when one fails. The timers do not keep the
444
+ * process alive.
418
445
  */
419
446
  export function warmDpdSession(environment = {}, options = {}) {
420
- sharedDpdAppService(environment).keep(options.store);
447
+ sharedDpdAppService(environment).keep(options);
421
448
  }
@@ -3,6 +3,7 @@ import { hasGs1CheckDigit } from '../../core/detection/numericChecksums.js';
3
3
  import { ChallengeError, IndeterminateError, InvalidInputError, NotFoundError, SchemaError } from '../../core/errors/index.js';
4
4
  import { eventPoint } from '../../core/result/index.js';
5
5
  import { runSteps } from '../../core/runner/index.js';
6
+ import { firstAttemptMs, networkRetryStep } from '../../core/runner/networkRetry.js';
6
7
  import { zonedTime } from '../../core/time/index.js';
7
8
  import { TrawlClient, clean, decodeText, fetchBounded } from '../../core/transport/index.js';
8
9
  import { isDeliveryStatusCell } from './identity.js';
@@ -235,24 +236,25 @@ export class DPDFranceTracker {
235
236
  const number = normalizeDPDFranceTrackingNumber(rawTrackingNumber);
236
237
  const url = dpdFranceTrackingUrl(number);
237
238
  const trawl = this.browserService();
239
+ const direct = async (timeoutMs, signal) => {
240
+ try {
241
+ return await this.directGet(url, capped(this.directTimeoutMs, timeoutMs), signal);
242
+ }
243
+ catch (error) {
244
+ // Without a browser service there is no second tier, so the
245
+ // challenge has to carry the operator's next step itself.
246
+ if (trawl || !(error instanceof DPDFranceChallengeError))
247
+ throw error;
248
+ throw new DPDFranceChallengeError('DPD France requires a browser challenge solver; configure FLARESOLVERR_URL');
249
+ }
250
+ };
238
251
  const html = await runSteps({
239
252
  carrier: 'dpd-fr', budgetMs: context.budgetMs ?? this.budgetMs, signal: context.signal, recorder: this.recorder,
240
253
  }, [
241
- {
242
- id: 'direct',
243
- run: async ({ remainingMs, signal }) => {
244
- try {
245
- return await this.directGet(url, capped(this.directTimeoutMs, remainingMs), signal);
246
- }
247
- catch (error) {
248
- // Without a browser service there is no second tier, so the
249
- // challenge has to carry the operator's next step itself.
250
- if (trawl || !(error instanceof DPDFranceChallengeError))
251
- throw error;
252
- throw new DPDFranceChallengeError('DPD France requires a browser challenge solver; configure FLARESOLVERR_URL');
253
- }
254
- },
255
- },
254
+ // A GET that fails to reach DPD, or hangs, is sent once more: the first
255
+ // gets at most half of what is left, so the second fits in the rest.
256
+ { id: 'direct', run: ({ remainingMs, signal }) => direct(firstAttemptMs(remainingMs), signal) },
257
+ networkRetryStep(({ remainingMs, signal }) => direct(remainingMs, signal)),
256
258
  {
257
259
  id: 'trawl',
258
260
  enabled: trawl !== null,
@@ -315,7 +317,7 @@ export const adapter = (environment) => {
315
317
  id: 'dpd-fr',
316
318
  // One direct HTML GET, then the browser service's native scrape API when
317
319
  // Cloudflare challenges it.
318
- steps: ['direct', 'trawl'],
320
+ steps: ['direct', 'retry', 'trawl'],
319
321
  track: (input, context) => tracker.fetch(input.number, context),
320
322
  };
321
323
  };
@@ -39,6 +39,10 @@ export declare class GLSSwitzerlandTracker {
39
39
  * a 400 or 403 can be a challenge, so it stays a failure.
40
40
  */
41
41
  recognizes(rawTrackingNumber: string, context?: TrackingContext): Promise<boolean>;
42
+ /**
43
+ * A request that fails to reach GLS, or hangs, is sent once more: the first gets at most half of
44
+ * what is left, so the second fits in the rest.
45
+ */
42
46
  private request;
43
47
  }
44
48
  /** Keep the verified overview's delivery owner when the postcode detail omits it. */
@@ -3,6 +3,7 @@ import { DateTime } from 'luxon';
3
3
  import { lookupBudget } from '../../core/adapter/index.js';
4
4
  import { DELIVERY_POSTCODE, deliveryPostcodeText } from '../../core/catalog/postcode.js';
5
5
  import { InputRequiredError, InvalidInputError, NotFoundError, SchemaError, UpstreamHttpError } from '../../core/errors/index.js';
6
+ import { withNetworkRetry } from '../../core/runner/networkRetry.js';
6
7
  import { cleanScalar, fetchBounded, parseJsonBytes, userAgentOf } from '../../core/transport/index.js';
7
8
  import { isRecord } from '../../core/types.js';
8
9
  import { classifyDescription, GLS_STATUSES, statusCode } from './status.js';
@@ -403,17 +404,21 @@ export class GLSSwitzerlandTracker {
403
404
  throw error;
404
405
  }
405
406
  }
407
+ /**
408
+ * A request that fails to reach GLS, or hangs, is sent once more: the first gets at most half of
409
+ * what is left, so the second fits in the rest.
410
+ */
406
411
  async request(url, budget) {
407
- const { response, bytes } = await fetchBounded(url, {
412
+ const { response, bytes } = await withNetworkRetry(budget, (timeoutMs) => fetchBounded(url, {
408
413
  signal: budget.signal,
409
414
  headers: pageHeaders(this.#userAgent),
410
415
  }, {
411
416
  provider: 'GLS Switzerland tracking',
412
- timeoutMs: Math.min(this.timeoutMs, budget.remainingMs()),
417
+ timeoutMs: Math.min(this.timeoutMs, timeoutMs),
413
418
  maxBytes: MAX_RESPONSE_BYTES,
414
419
  allowHttpError: true,
415
420
  ...(this.#fetcher ? { fetcher: this.#fetcher } : {}),
416
- });
421
+ }));
417
422
  if ([400, 403, 404].includes(response.status))
418
423
  throw new GLSSwitzerlandTrackingError();
419
424
  if (!response.ok)
@@ -24,6 +24,10 @@ export declare class GLSGermanyTracker {
24
24
  * only after this returns true; a provider failure must stay a failure.
25
25
  */
26
26
  recognizes(rawTrackingNumber: string, context?: TrackingContext): Promise<boolean>;
27
+ /**
28
+ * A request that fails to reach GLS, or hangs, is sent once more: the first gets at most half of
29
+ * what is left, so the second fits in the rest.
30
+ */
27
31
  private request;
28
32
  }
29
33
  export declare const adapter: AdapterFactory;
@@ -1,5 +1,6 @@
1
1
  import { lookupBudget } from '../../core/adapter/index.js';
2
2
  import { NotFoundError, SchemaError, UpstreamHttpError } from '../../core/errors/index.js';
3
+ import { withNetworkRetry } from '../../core/runner/networkRetry.js';
3
4
  import { cleanScalar, fetchBounded, parseJsonBytes, userAgentOf } from '../../core/transport/index.js';
4
5
  import { isRecord } from '../../core/types.js';
5
6
  import { glsSwitzerlandDetailApiUrl, glsDeliveryReference, glsSwitzerlandOverviewApiUrl, GLSSwitzerlandTrackingError, normalizeGLSSwitzerlandPostcode, normalizeGLSSwitzerlandTrackingNumber, parseGLSSwitzerlandTrackingResponse, selectGLSParcel, } from '../gls-ch/adapter.js';
@@ -88,17 +89,21 @@ export class GLSGermanyTracker {
88
89
  throw error;
89
90
  }
90
91
  }
92
+ /**
93
+ * A request that fails to reach GLS, or hangs, is sent once more: the first gets at most half of
94
+ * what is left, so the second fits in the rest.
95
+ */
91
96
  async request(url, budget) {
92
- const { response, bytes } = await fetchBounded(url, {
97
+ const { response, bytes } = await withNetworkRetry(budget, (timeoutMs) => fetchBounded(url, {
93
98
  signal: budget.signal,
94
99
  headers: { Accept: 'application/json', Referer: 'https://gls-group.eu/EU/en/parcel-tracking', 'User-Agent': this.#userAgent },
95
100
  }, {
96
101
  provider: PROVIDER,
97
- timeoutMs: Math.min(this.timeoutMs, budget.remainingMs()),
102
+ timeoutMs: Math.min(this.timeoutMs, timeoutMs),
98
103
  maxBytes: 1_000_000,
99
104
  allowHttpError: true,
100
105
  ...(this.#fetcher ? { fetcher: this.#fetcher } : {}),
101
- });
106
+ }));
102
107
  // A challenge, invalid postcode, rate limit or outage must not become "not found".
103
108
  if (response.status === 404) {
104
109
  const payload = parseJsonBytes(bytes, PROVIDER);
@@ -13,6 +13,7 @@ import { DateTime } from 'luxon';
13
13
  import { accepted, lookupBudget, recognizeFromLookup } from '../../core/adapter/index.js';
14
14
  import { isValidGlsParcelNumber } from '../../core/detection/index.js';
15
15
  import { InvalidInputError, SchemaError, TransportError, UpstreamHttpError } from '../../core/errors/index.js';
16
+ import { withNetworkRetry } from '../../core/runner/networkRetry.js';
16
17
  import { EXPLICIT_OFFSET_PATTERN } from '../../core/time/index.js';
17
18
  import { clean, cleanScalar, decodeText, fetchBounded, parseJsonBytes, userAgentOf } from '../../core/transport/index.js';
18
19
  import { isRecord } from '../../core/types.js';
@@ -280,7 +281,8 @@ export class GLSFranceTracker {
280
281
  async fetch(trackingNumber, context = {}, { pickupPoint = true } = {}) {
281
282
  const normalized = normalizeGLSFranceTrackingNumber(trackingNumber);
282
283
  const printed = printedNumber(trackingNumber);
283
- const budget = lookupBudget(context, this.timeoutMs);
284
+ // Room for a request and its retry.
285
+ const budget = lookupBudget(context, 2 * this.timeoutMs);
284
286
  let parsed;
285
287
  try {
286
288
  parsed = await this.lookup(normalized, normalized, budget);
@@ -335,16 +337,18 @@ export class GLSFranceTracker {
335
337
  };
336
338
  }
337
339
  async lookup(code, normalized, budget) {
338
- const { response, bytes } = await fetchBounded(`${TRACKING_API}/${encodeURIComponent(code)}`, {
340
+ // A request that fails to reach GLS, or hangs, is sent once more: the first
341
+ // gets at most half of what is left, so the second fits in the rest.
342
+ const { response, bytes } = await withNetworkRetry(budget, (timeoutMs) => fetchBounded(`${TRACKING_API}/${encodeURIComponent(code)}`, {
339
343
  signal: budget.signal,
340
344
  headers: this.#headers(),
341
345
  }, {
342
346
  provider: 'GLS France tracking',
343
- timeoutMs: Math.min(this.timeoutMs, budget.remainingMs()),
347
+ timeoutMs: Math.min(this.timeoutMs, timeoutMs),
344
348
  maxBytes: MAX_RESPONSE_BYTES,
345
349
  fetcher: this.#fetcher,
346
350
  allowHttpStatuses: [404, 410],
347
- });
351
+ }));
348
352
  budget.signal.throwIfAborted();
349
353
  if ([404, 410].includes(response.status)) {
350
354
  // The native negative names the complete submitted code. An absent API
@@ -3,6 +3,7 @@ import { carrierIdFromPartner } from '../../core/catalog/hints.js';
3
3
  import { dpdParcelNumber } from '../../core/detection/dpd.js';
4
4
  import { CarrierError, InvalidInputError, SchemaError, UpstreamHttpError } from '../../core/errors/index.js';
5
5
  import { runSteps } from '../../core/runner/index.js';
6
+ import { firstAttemptMs, isNetworkFailure } from '../../core/runner/networkRetry.js';
6
7
  import { isoTime } from '../../core/time/index.js';
7
8
  import { clean, decodeText, fetchBounded, parseJsonBytes, userAgentOf } from '../../core/transport/index.js';
8
9
  import { isRecord } from '../../core/types.js';
@@ -20,6 +21,9 @@ import { eventStage, eventStatus } from './status.js';
20
21
  // - Production HTTP 403s carried La Poste's "Site indisponible - Incident en
21
22
  // cours" page and immediately following checks succeeded, so a 403 is
22
23
  // retried up to three times inside the original deadline (see README.md).
24
+ // - A request that hangs would otherwise take the whole budget, so a network
25
+ // failure or timeout is retried once too, and an attempt gets at most half
26
+ // of what is left while that retry remains.
23
27
  // - The tracking page links the point holding a parcel to La Poste's locator,
24
28
  // `localiser.laposte.fr/{idPoint}`, which redirects to the point's page. That
25
29
  // page carries the point's record, address included, as JSON in
@@ -273,7 +277,7 @@ export class LaPosteTracker {
273
277
  const normalized = normalizeLaPosteTrackingNumber(trackingNumber);
274
278
  const budgetMs = context.budgetMs ?? this.timeoutMs;
275
279
  const deadline = performance.now() + budgetMs;
276
- const request = async (remainingMs, signal) => {
280
+ const request = async (timeoutMs, signal) => {
277
281
  const { bytes } = await fetchBounded(laPosteTrackingApiUrl(normalized), {
278
282
  signal,
279
283
  headers: {
@@ -284,7 +288,7 @@ export class LaPosteTracker {
284
288
  },
285
289
  }, {
286
290
  provider: 'La Poste tracking',
287
- timeoutMs: Math.max(1, Math.floor(Math.min(this.timeoutMs, remainingMs))),
291
+ timeoutMs: Math.max(1, Math.floor(Math.min(this.timeoutMs, timeoutMs))),
288
292
  maxBytes: MAX_RESPONSE_BYTES,
289
293
  fetcher: this.fetcher,
290
294
  });
@@ -302,21 +306,28 @@ export class LaPosteTracker {
302
306
  // later, is answered normally. It is a hiccup of that one request, not
303
307
  // maintenance, so the page is retried like any other 403: three immediate
304
308
  // retries sharing the original deadline. A real incident still fails every
305
- // attempt within seconds and reaches the router. Do not retry other HTTP or
306
- // parsing failures. Once the deadline is spent the retry is refused, so the
307
- // caller still sees the provider's own rejection rather than a budget error.
308
- const retriable = (error) => error instanceof UpstreamHttpError
309
- && error.status === 403
310
- && deadline - performance.now() >= 1;
309
+ // attempt within seconds and reaches the router. A request that fails to
310
+ // reach La Poste, or hangs until its timeout, gets one retry as well. Until
311
+ // that retry is spent, an attempt gets at most half of what is left, so a
312
+ // hang leaves it time. Do not retry other HTTP or parsing failures, nor a
313
+ // not-found. Once the deadline is spent the retry is refused, so the caller
314
+ // still sees the provider's own rejection rather than a budget error.
315
+ let networkRetried = false;
316
+ const retriable = (error) => deadline - performance.now() >= 1
317
+ && ((error instanceof UpstreamHttpError && error.status === 403) || (!networkRetried && isNetworkFailure(error)));
311
318
  const retry = {
312
319
  id: 'retry',
313
320
  recovers: retriable,
314
- run: ({ remainingMs, signal }) => request(remainingMs, signal),
321
+ run: ({ remainingMs, signal, previousError }) => {
322
+ if (isNetworkFailure(previousError))
323
+ networkRetried = true;
324
+ return request(networkRetried ? remainingMs : firstAttemptMs(remainingMs), signal);
325
+ },
315
326
  };
316
327
  return await runSteps({
317
328
  carrier: 'la-poste', budgetMs, signal: context.signal, recorder: this.recorder,
318
329
  }, [
319
- { id: 'direct', run: ({ remainingMs, signal }) => request(remainingMs, signal) },
330
+ { id: 'direct', run: ({ remainingMs, signal }) => request(firstAttemptMs(remainingMs), signal) },
320
331
  retry,
321
332
  { ...retry },
322
333
  { ...retry },
@@ -369,7 +380,8 @@ export const adapter = (environment) => {
369
380
  return {
370
381
  id: 'la-poste',
371
382
  // One keyless request, then up to three immediate retries of the same
372
- // request after a transient HTTP 403, inside the original deadline.
383
+ // request after a transient HTTP 403, and one after a network failure or
384
+ // timeout, inside the original deadline.
373
385
  steps: ['direct', 'retry'],
374
386
  track: (input, context) => tracker.fetch(input.number, context),
375
387
  recognize: (number, context) => recognizeFromLookup(() => tracker.fetch(number, context), () => accepted(() => normalizeLaPosteTrackingNumber(number))),
@@ -21,7 +21,7 @@ export declare function parseSwissPostShipment(item: JsonObject, rawEvents: unkn
21
21
  export declare class SwissPostTracker {
22
22
  #private;
23
23
  constructor(options?: SwissPostOptions);
24
- readJson(fetcher: typeof fetch, url: string, init?: RequestInit, budget?: LookupBudget): Promise<unknown>;
24
+ readJson(fetcher: typeof fetch, url: string, init?: RequestInit, budget?: LookupBudget, timeoutMs?: number): Promise<unknown>;
25
25
  /** Cached for the process: the table is large, static, and shared by every lookup. */
26
26
  loadTranslations(fetcher: typeof fetch, budget?: LookupBudget): Promise<Record<string, string>>;
27
27
  fetch(trackingNumber: string, context?: TrackingContext): Promise<CarrierResult>;
@@ -4,6 +4,7 @@ import { CookieJar } from 'tough-cookie';
4
4
  import { lookupBudget } from '../../core/adapter/index.js';
5
5
  import { NotFoundError, SchemaError } from '../../core/errors/index.js';
6
6
  import { countryCode } from '../../core/time/index.js';
7
+ import { withNetworkRetry } from '../../core/runner/networkRetry.js';
7
8
  import { fetchBounded, parseJsonBytes } from '../../core/transport/index.js';
8
9
  import { isRecord } from '../../core/types.js';
9
10
  import { FALLBACK_EVENT_LABELS, GLOBAL_STATUS_STAGE, STAGE_STATUS, STATUS_MAP, swissPostEventStage, } from './status.js';
@@ -266,14 +267,22 @@ export class SwissPostTracker {
266
267
  constructor(options = {}) {
267
268
  this.#fetcher = options.fetcher;
268
269
  }
269
- async readJson(fetcher, url, init = {}, budget) {
270
+ async readJson(fetcher, url, init = {}, budget, timeoutMs = REQUEST_TIMEOUT_MS) {
270
271
  const { bytes } = await fetchBounded(url, budget ? { ...init, signal: budget.signal } : init, {
271
272
  provider: 'Swiss Post tracking',
272
- timeoutMs: Math.min(REQUEST_TIMEOUT_MS, budget?.remainingMs() ?? REQUEST_TIMEOUT_MS),
273
+ timeoutMs: Math.min(REQUEST_TIMEOUT_MS, timeoutMs, budget?.remainingMs() ?? REQUEST_TIMEOUT_MS),
273
274
  fetcher,
274
275
  });
275
276
  return parseJsonBytes(bytes, PROVIDER);
276
277
  }
278
+ /**
279
+ * A call of the lookup's chain, sent once more if it fails to reach Swiss Post or hangs: the
280
+ * first gets at most half of what is left, so the second fits in the rest. The search's POST
281
+ * only files the number under the throwaway user, so it is sent again like a read.
282
+ */
283
+ #readRetrying(fetcher, url, init, budget) {
284
+ return withNetworkRetry(budget, (timeoutMs) => this.readJson(fetcher, url, init, budget, timeoutMs));
285
+ }
277
286
  /** Cached for the process: the table is large, static, and shared by every lookup. */
278
287
  async loadTranslations(fetcher, budget) {
279
288
  if (this.#translations)
@@ -301,11 +310,11 @@ export class SwissPostTracker {
301
310
  async fetch(trackingNumber, context = {}) {
302
311
  const budget = lookupBudget(context, DEFAULT_BUDGET_MS);
303
312
  const fetcher = makeFetchCookie(this.#fetcher ?? fetch, new CookieJar());
304
- const userResult = await fetchBounded(`${API_BASE}/user`, { headers: HEADERS, signal: budget.signal }, {
313
+ const userResult = await withNetworkRetry(budget, (timeoutMs) => fetchBounded(`${API_BASE}/user`, { headers: HEADERS, signal: budget.signal }, {
305
314
  provider: 'Swiss Post tracking',
306
- timeoutMs: Math.min(REQUEST_TIMEOUT_MS, budget.remainingMs()),
315
+ timeoutMs: Math.min(REQUEST_TIMEOUT_MS, timeoutMs),
307
316
  fetcher,
308
- });
317
+ }));
309
318
  const userPayload = parseJsonBytes(userResult.bytes, PROVIDER);
310
319
  if (!isRecord(userPayload))
311
320
  throw new SchemaError(PROVIDER, 'Swiss Post returned an invalid anonymous user response');
@@ -315,7 +324,7 @@ export class SwissPostTracker {
315
324
  const csrf = userResult.response.headers.get('x-csrf-token') ?? '';
316
325
  const headers = { ...HEADERS, 'x-csrf-token': csrf };
317
326
  const query = new URLSearchParams({ userId });
318
- const historyPayload = await this.readJson(fetcher, `${API_BASE}/history?${query}`, {
327
+ const historyPayload = await this.#readRetrying(fetcher, `${API_BASE}/history?${query}`, {
319
328
  method: 'POST',
320
329
  headers: { ...headers, 'Content-Type': 'application/json' },
321
330
  body: JSON.stringify({ searchQuery: trackingNumber }),
@@ -323,7 +332,7 @@ export class SwissPostTracker {
323
332
  const hash = isRecord(historyPayload) ? text(historyPayload.hash) : '';
324
333
  if (!hash)
325
334
  throw new SchemaError(PROVIDER, 'Swiss Post did not return a shipment search identifier');
326
- const items = await this.readJson(fetcher, `${API_BASE}/history/not-included/${encodeURIComponent(hash)}?${query}`, { headers }, budget);
335
+ const items = await this.#readRetrying(fetcher, `${API_BASE}/history/not-included/${encodeURIComponent(hash)}?${query}`, { headers }, budget);
327
336
  if (!Array.isArray(items)) {
328
337
  throw new SchemaError(PROVIDER, 'Swiss Post returned an invalid shipment response');
329
338
  }
@@ -348,7 +357,7 @@ export class SwissPostTracker {
348
357
  let events = [];
349
358
  if (identity) {
350
359
  try {
351
- const payload = await this.readJson(fetcher, `${API_BASE}/shipment/id/${encodeURIComponent(identity)}/events`, { headers }, budget);
360
+ const payload = await this.#readRetrying(fetcher, `${API_BASE}/shipment/id/${encodeURIComponent(identity)}/events`, { headers }, budget);
352
361
  if (Array.isArray(payload))
353
362
  events = payload;
354
363
  }
@@ -57,7 +57,14 @@ export declare class SchemaError extends CarrierError {
57
57
  }
58
58
  export declare class InputRequiredError extends CarrierError {
59
59
  readonly field: string;
60
- constructor(provider: string, field: string, message?: string, options?: CarrierErrorOptions);
60
+ /**
61
+ * The catalog id of the carrier that asks for the input, when the source names one: a universal
62
+ * provider relaying a carrier's postcode gate. Absent when the source names none, or none that maps.
63
+ */
64
+ readonly carrier?: string;
65
+ constructor(provider: string, field: string, message?: string, options?: CarrierErrorOptions & {
66
+ carrier?: string;
67
+ });
61
68
  }
62
69
  /**
63
70
  * The adapter rejected the number or credential before any request: the format
@@ -89,10 +89,17 @@ export class SchemaError extends CarrierError {
89
89
  }
90
90
  export class InputRequiredError extends CarrierError {
91
91
  field;
92
+ /**
93
+ * The catalog id of the carrier that asks for the input, when the source names one: a universal
94
+ * provider relaying a carrier's postcode gate. Absent when the source names none, or none that maps.
95
+ */
96
+ carrier;
92
97
  constructor(provider, field, message = `${provider} tracking requires ${field}`, options) {
93
98
  super('input_required', provider, message, options);
94
99
  this.name = 'InputRequiredError';
95
100
  this.field = field;
101
+ if (options?.carrier !== undefined)
102
+ this.carrier = options.carrier;
96
103
  }
97
104
  }
98
105
  /**
@@ -0,0 +1,15 @@
1
+ import type { StepContext, StepSpec } from './index.js';
2
+ /** Whether an error is, or wraps, a request that did not reach the carrier, including one that timed out. */
3
+ export declare function isNetworkFailure(error: unknown): boolean;
4
+ /** The time a first attempt's request gets out of what is left: half, so a retry still fits. */
5
+ export declare function firstAttemptMs(remainingMs: number): number;
6
+ /** A step that runs the read again after the step before it failed to reach the carrier. */
7
+ export declare function networkRetryStep<T>(run: (context: StepContext) => Promise<T>): StepSpec<T>;
8
+ /**
9
+ * `attempt` with the time its request may take, then once more after a network failure or a
10
+ * timeout, while the budget has time left and its signal has not ended.
11
+ */
12
+ export declare function withNetworkRetry<T>(budget: {
13
+ signal?: AbortSignal;
14
+ remainingMs: () => number;
15
+ }, attempt: (timeoutMs: number) => Promise<T>): Promise<T>;
@@ -0,0 +1,51 @@
1
+ /**
2
+ * One more try at a carrier's own read after a network failure or a timeout, inside the lookup's
3
+ * budget. The first attempt's request gets at most half of the time left, so a request that hangs
4
+ * leaves time for the second, which gets the rest. Only a failure to reach the carrier is tried
5
+ * again: an HTTP status, a reply that does not parse and a definite answer stand, and so does any
6
+ * failure once the lookup's own signal has ended. Reads only: a request that changes something, or
7
+ * one bound to a session the failure may have spent, is not replayed.
8
+ *
9
+ * Steps run by `runSteps` add `networkRetryStep` after the first and give it `firstAttemptMs` of
10
+ * what is left, so the retry is recorded as a step of its own. A read outside the runner wraps its
11
+ * request in `withNetworkRetry`.
12
+ */
13
+ import { UpstreamNetworkError } from '../errors/index.js';
14
+ const MAX_CAUSE_DEPTH = 8;
15
+ /** Whether an error is, or wraps, a request that did not reach the carrier, including one that timed out. */
16
+ export function isNetworkFailure(error) {
17
+ const seen = new Set();
18
+ let current = error;
19
+ for (let depth = 0; depth < MAX_CAUSE_DEPTH && current instanceof Error && !seen.has(current); depth += 1) {
20
+ seen.add(current);
21
+ if (current instanceof UpstreamNetworkError)
22
+ return true;
23
+ current = current.cause;
24
+ }
25
+ return false;
26
+ }
27
+ /** The time a first attempt's request gets out of what is left: half, so a retry still fits. */
28
+ export function firstAttemptMs(remainingMs) {
29
+ return Math.max(1, Math.floor(remainingMs / 2));
30
+ }
31
+ /** A step that runs the read again after the step before it failed to reach the carrier. */
32
+ export function networkRetryStep(run) {
33
+ return { id: 'retry', recovers: isNetworkFailure, run };
34
+ }
35
+ /**
36
+ * `attempt` with the time its request may take, then once more after a network failure or a
37
+ * timeout, while the budget has time left and its signal has not ended.
38
+ */
39
+ export async function withNetworkRetry(budget, attempt) {
40
+ try {
41
+ return await attempt(firstAttemptMs(budget.remainingMs()));
42
+ }
43
+ catch (error) {
44
+ if (!isNetworkFailure(error) || budget.signal?.aborted)
45
+ throw error;
46
+ const left = budget.remainingMs();
47
+ if (left <= 1)
48
+ throw error;
49
+ return await attempt(left);
50
+ }
51
+ }
package/dist/node.d.ts CHANGED
@@ -8,5 +8,5 @@ export { REGISTRY } from './generated/registry.js';
8
8
  export { UniversalTracker, UniversalTrackingError } from './providers/universal.js';
9
9
  export { TrackingCaptureError, SeventeenTrackLookupError, SeventeenTrackNoHistoryError, SeventeenTrackVerificationError } from './providers/universal.js';
10
10
  export { amazonShippingEligibility } from './carriers/amazon-shipping/eligibility.js';
11
- export { warmDpdSession, type DpdSession, type DpdSessionStore } from './carriers/dpd-de/service.js';
11
+ export { warmDpdSession, type DpdSession, type DpdSessionEvent, type DpdSessionStore } from './carriers/dpd-de/service.js';
12
12
  export { createTrackingServer, type TrackingServerOptions } from './server/index.js';
@@ -250,9 +250,15 @@ function parseHistory(payload, trackingNumber, timezone = null) {
250
250
  markReturnLeg(scans);
251
251
  latest.check();
252
252
  if (!events.length) {
253
- const fields = payload.states.flatMap((raw) => isRecord(raw) && Array.isArray(raw.require_fields) ? raw.require_fields : []);
254
- if (fields.some((field) => isRecord(field) && field.name === 'zipcode')) {
255
- throw new InputRequiredError(SOURCE, 'postcode', 'ParcelsApp requires a valid delivery postcode or further recipient information');
253
+ const gates = payload.states.filter((raw) => isRecord(raw)
254
+ && Array.isArray(raw.require_fields) && raw.require_fields.some((field) => isRecord(field) && field.name === 'zipcode'));
255
+ if (gates.length) {
256
+ // A gate names the carrier that asks for the postcode, as a scan does. It is
257
+ // given only when every gate names the same one and it maps to the catalog.
258
+ const [name, ...others] = gates.map((gate) => stateCarrierName(payload, gate));
259
+ const carrier = typeof name === 'string' && others.every((other) => other === name)
260
+ ? universalCarrierHints([name], number).discovered_carrier : undefined;
261
+ throw new InputRequiredError(SOURCE, 'postcode', 'ParcelsApp requires a valid delivery postcode or further recipient information', carrier ? { carrier } : undefined);
256
262
  }
257
263
  // An empty history is the NO_DATA answer; states that don't parse are not an answer.
258
264
  if (!payload.states.length)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "universal-parcel-scraper",
3
- "version": "0.56.0-main.720",
3
+ "version": "0.56.0-main.723",
4
4
  "description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",