universal-parcel-scraper 0.852.4-main.4549 → 0.852.6-main.4553

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -163,7 +163,7 @@ the command line's `track --source ParcelsApp` does the same.
163
163
  <!-- GENERATED:stages -->
164
164
  <img src="docs/assets/stages.svg" alt="DHL: Die Sendung wurde in das Zustellfahrzeug geladen.; Mondial Relay: En cours de livraison; Correios Brazil: Objeto saiu para entrega ao destinatário; Correos Express: EN REPARTO; Yamato Transport: 配達中; La Poste / Colissimo: DISTOU. All are filed under out_for_delivery." width="760">
165
165
 
166
- The carrier folders record 17,838 such statuses, each filed under one stage.
166
+ The carrier folders record 17,839 such statuses, each filed under one stage.
167
167
  <!-- /GENERATED:stages -->
168
168
 
169
169
  Wording nobody recorded yet goes through a classifier that reads seven European languages.
@@ -7,6 +7,7 @@
7
7
  {"wording": "Delivery Attempted", "stage": "failed_attempt", "language": "en", "confirmedBy": "portal", "note": "A step label in the script of Roadie's live tracker."},
8
8
  {"wording": "Transferred to Carrier", "stage": "in_transit", "language": "en", "confirmedBy": "portal", "note": "A step label in the script of Roadie's live tracker. Roadie handed the delivery to another carrier; the page then shows that carrier's tracking number and, for UPS, a link to it."},
9
9
  {"wording": "Delayed", "stage": "exception", "language": "en", "confirmedBy": "portal", "note": "A step label in the script of Roadie's live tracker."},
10
- {"wording": "Canceled", "stage": "exception", "language": "en", "confirmedBy": "portal", "note": "A step label in the script of Roadie's live tracker."}
10
+ {"wording": "Canceled", "stage": "exception", "language": "en", "confirmedBy": "portal", "note": "A step label in the script of Roadie's live tracker."},
11
+ {"wording": "Expired", "stage": "exception", "language": "en", "confirmedBy": "portal", "note": "A status label in the script of Roadie's live tracker, for a delivery whose state is expired. The page shows it with an alert and no steps, as it shows Canceled."}
11
12
  ]
12
13
  }
@@ -16,8 +16,8 @@
16
16
  {"code": "CO_C_CR", "wording": "已交承运商", "language": "zh", "english": "Handed to the carrier", "stage": "in_transit", "confirmedBy": "live", "note": "Handed to the carrier for the international leg. The English is a plain rendering."},
17
17
  {"code": "CO_C_CR", "wording": "航空公司", "language": "zh", "english": "With the airline", "stage": "in_transit", "confirmedBy": "live", "note": "With the airline. The English is a plain rendering."},
18
18
  {"code": "CO_C_CR", "wording": "飞机起飞", "language": "zh", "english": "Flight departed", "stage": "in_transit", "confirmedBy": "live", "note": "The flight took off. The English is a plain rendering."},
19
- {"code": "CO_C_CR", "wording": "飞机进港", "language": "zh", "english": "Plane has landed", "stage": "in_transit", "confirmedBy": "live", "note": "The flight arrived. The English is ParcelsApp's for China Post's same label."},
20
- {"code": "CO_C_CR", "wording": "到达寄达地", "language": "zh", "english": "Arrive at the place of destination", "stage": "in_transit", "confirmedBy": "live", "note": "Arrived in the destination country. The English is ParcelsApp's for China Post's same label."},
19
+ {"code": "CO_C_CR", "wording": "飞机进港", "language": "zh", "english": "Flight arrived", "stage": "in_transit", "confirmedBy": "live", "note": "The flight arrived. The English is a plain rendering."},
20
+ {"code": "CO_C_CR", "wording": "到达寄达地", "language": "zh", "english": "Arrived in the destination country", "stage": "in_transit", "confirmedBy": "live", "note": "Arrived in the destination country. The English is a plain rendering."},
21
21
  {"code": "CO_C_CR", "wording": "送交境外进口海关", "language": "zh", "english": "Presented to import customs abroad", "stage": "customs", "confirmedBy": "live", "note": "Presented to the destination country's import customs. The English is a plain rendering."},
22
22
  {"code": "CO_C_CR", "wording": "到达寄达地处理中心", "language": "zh", "english": "Arrival at processing center in the destination country", "stage": "in_transit", "confirmedBy": "live", "note": "Arrived at a processing centre in the destination country. The English is a plain rendering."},
23
23
  {"code": "CO_C_CR", "wording": "Goods leave the operation center", "language": "en", "stage": "in_transit", "confirmedBy": "live", "note": "The destination network's parcel leaves its operation centre."},
@@ -3,6 +3,7 @@ import { BudgetExceededError } from '../errors/index.js';
3
3
  import { normalizeCarrierResult } from '../result/index.js';
4
4
  import { dueScans } from '../result/due.js';
5
5
  import { readLateRegistrations } from '../result/registration.js';
6
+ import { withoutRepeatedScans } from '../result/repeats.js';
6
7
  import { runSteps } from '../runner/index.js';
7
8
  import { NOOP_RECORDER } from '../telemetry/index.js';
8
9
  import { dateLocalClocks } from '../../places/clocks.js';
@@ -29,7 +30,7 @@ export async function trackCarrier(carrier, typed, options) {
29
30
  recorder: options.recorder ?? NOOP_RECORDER }, [{ id: registered.steps[0] ?? 'direct',
30
31
  run: ({ signal, remainingMs }) => registered.track(input, ...(options.signal || options.budgetMs !== undefined ? [{ signal, budgetMs: context.budgetMs === undefined ? undefined : Math.min(context.budgetMs, remainingMs) }] : [])) }]);
31
32
  const answer = normalizeCarrierResult(result);
32
- return readLateRegistrations(dateLocalClocks(carrier, restoreLostLetters(registered.plannedScans ? dueScans(answer, carrier) : answer)));
33
+ return readLateRegistrations(withoutRepeatedScans(dateLocalClocks(carrier, restoreLostLetters(registered.plannedScans ? dueScans(answer, carrier) : answer))));
33
34
  }
34
35
  catch (error) {
35
36
  // A caller's cancellation is the caller's reason, not a carrier failure.
@@ -1,4 +1,16 @@
1
+ /**
2
+ * A registration a carrier writes after the parcel has moved, read as no step back.
3
+ * Every adapter's and provider's result goes through it before it leaves the package.
4
+ */
5
+ import { type Stage } from '../../generated/catalog.js';
1
6
  import type { CarrierResult } from './index.js';
7
+ /** The stage a scan is read in, as the consumer reads it: declared, else by its wording. Null when no map or rule read it. */
8
+ export declare function reading(event: unknown): Stage | null;
9
+ /**
10
+ * Each scan's instant, or, in a history with none, its wall clock. A scan with neither,
11
+ * or with a wall clock among instants, is not ordered.
12
+ */
13
+ export declare function clocks(events: readonly unknown[]): (number | null)[];
2
14
  /**
3
15
  * A registration (a label created, data received, a pre-advice) dated after a scan that moved
4
16
  * the parcel, such as a partner's label or a line's order row written once the parcel is on its
@@ -11,7 +11,7 @@ const stages = new Set(STAGES);
11
11
  const MOVED = new Set(['accepted', 'in_transit', 'out_for_delivery', 'failed_attempt', 'ready_for_pickup', 'delivered', 'returned']);
12
12
  const WALL_CLOCK = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?$/;
13
13
  /** The stage a scan is read in, as the consumer reads it: declared, else by its wording. Null when no map or rule read it. */
14
- function reading(event) {
14
+ export function reading(event) {
15
15
  if (!isRecord(event))
16
16
  return null;
17
17
  const declared = typeof event.stage === 'string' && stages.has(event.stage) ? event.stage : undefined;
@@ -24,7 +24,7 @@ function reading(event) {
24
24
  * Each scan's instant, or, in a history with none, its wall clock. A scan with neither,
25
25
  * or with a wall clock among instants, is not ordered.
26
26
  */
27
- function clocks(events) {
27
+ export function clocks(events) {
28
28
  const field = (event, key) => isRecord(event) && typeof event[key] === 'string' ? event[key] : '';
29
29
  const instants = events.map(event => explicitOffsetTime(field(event, 'time'))?.timestamp ?? null);
30
30
  if (instants.some(at => at !== null))
@@ -0,0 +1,8 @@
1
+ import type { CarrierResult } from './index.js';
2
+ /**
3
+ * A scan with an earlier scan's wording, place and stage, at most ten minutes after it, is left
4
+ * out: a gateway that records one departure several times, or a failed attempt written once by
5
+ * each of a carrier's systems, is one step. A scan of another stage between them keeps the
6
+ * repeat, as a second round after a failed attempt. Scans without a wording or a clock stay.
7
+ */
8
+ export declare function withoutRepeatedScans(result: CarrierResult): CarrierResult;
@@ -0,0 +1,41 @@
1
+ import { isRecord } from '../types.js';
2
+ import { clocks, reading } from './registration.js';
3
+ /** How long after a scan the same wording at the same place still repeats it. */
4
+ const REPEAT_MS = 10 * 60_000;
5
+ const text = (value) => typeof value === 'string' ? value.trim().replace(/\s+/g, ' ').toLowerCase() : '';
6
+ /**
7
+ * A scan with an earlier scan's wording, place and stage, at most ten minutes after it, is left
8
+ * out: a gateway that records one departure several times, or a failed attempt written once by
9
+ * each of a carrier's systems, is one step. A scan of another stage between them keeps the
10
+ * repeat, as a second round after a failed attempt. Scans without a wording or a clock stay.
11
+ */
12
+ export function withoutRepeatedScans(result) {
13
+ const events = Array.isArray(result.events) ? result.events : [];
14
+ if (events.length < 2)
15
+ return result;
16
+ const at = clocks(events);
17
+ const read = events.map(reading);
18
+ // Scans dated alike in the carrier's order (`tie_order`), so the first record of a step is the one kept.
19
+ const tie = (index) => { const event = events[index]; return isRecord(event) && typeof event.tie_order === 'number' ? event.tie_order : 0; };
20
+ const order = events.flatMap((_, index) => at[index] === null ? [] : [index]).sort((left, right) => at[left] - at[right] || tie(left) - tie(right));
21
+ // The newest kept scan of each wording, place and stage, and the stages of the scans kept since.
22
+ const kept = new Map();
23
+ const repeats = new Set();
24
+ for (const index of order) {
25
+ const event = events[index];
26
+ const wording = isRecord(event) ? text(event.description) : '';
27
+ const key = isRecord(event) && wording ? `${wording}\n${text(event.location)}\n${text(event.stage)}` : '';
28
+ const earlier = key ? kept.get(key) : undefined;
29
+ if (earlier && at[index] - earlier.at <= REPEAT_MS && earlier.since.every(stage => stage === null || stage === read[index])) {
30
+ repeats.add(index);
31
+ continue;
32
+ }
33
+ for (const scan of kept.values())
34
+ scan.since.push(read[index] ?? null);
35
+ if (key)
36
+ kept.set(key, { at: at[index], since: [] });
37
+ }
38
+ if (!repeats.size)
39
+ return result;
40
+ return { ...result, events: events.filter((_, index) => !repeats.has(index)) };
41
+ }
@@ -3,6 +3,7 @@ import { isPlaceholderNumber } from '../core/detection/placeholder.js';
3
3
  import { USPS_ROUTING_BARCODE, uspsPackageIdentifier } from '../core/detection/usps.js';
4
4
  import { BudgetExceededError, IndeterminateError, InvalidInputError, NoHistoryError } from '../core/errors/index.js';
5
5
  import { readLateRegistrations } from '../core/result/registration.js';
6
+ import { withoutRepeatedScans } from '../core/result/repeats.js';
6
7
  import { NOOP_RECORDER } from '../core/telemetry/index.js';
7
8
  import { TrawlClient } from '../core/transport/index.js';
8
9
  import { adapter as parcelsAppAdapter } from './parcelsapp/adapter.js';
@@ -115,7 +116,7 @@ export class UniversalTracker {
115
116
  if (source !== 'UPU' && result.events?.length && !result.events.some((event) => event.time)) {
116
117
  throw new IndeterminateError(source, 'No dated tracking events', { reason: 'undated_history' });
117
118
  }
118
- return readLateRegistrations(result);
119
+ return readLateRegistrations(withoutRepeatedScans(result));
119
120
  }
120
121
  /** One provider adapter, built from this tracker's environment. */
121
122
  provider(source) {
@@ -2,7 +2,7 @@
2
2
  "openapi": "3.1.0",
3
3
  "info": {
4
4
  "title": "Universal Parcel Scraper",
5
- "version": "0.852.4-main.4549"
5
+ "version": "0.852.6-main.4553"
6
6
  },
7
7
  "paths": {
8
8
  "/health": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "universal-parcel-scraper",
3
- "version": "0.852.4-main.4549",
3
+ "version": "0.852.6-main.4553",
4
4
  "description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",