universal-parcel-scraper 0.3.0-main.303 → 0.3.0-main.308
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -165
- package/dist/app.d.ts +4 -3
- package/dist/app.js +4 -3
- package/dist/browser/scraper.js +2 -1
- package/dist/carriers/dpd/app.d.ts +2 -0
- package/dist/carriers/dpd/app.js +5 -0
- package/dist/carriers/india-post/adapter.js +67 -7
- package/dist/carriers/india-post/app.d.ts +2 -0
- package/dist/carriers/india-post/app.js +4 -0
- package/dist/carriers/india-post/status.js +2 -0
- package/dist/carriers/india-post/statuses.json +1 -0
- package/dist/core/catalog/eventIdentity.d.ts +8 -0
- package/dist/core/catalog/eventIdentity.js +7 -0
- package/dist/core/catalog/hints.d.ts +9 -1
- package/dist/core/catalog/hints.js +16 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -18,44 +18,39 @@ The engine behind [Peek](https://github.com/plhery/delivery-tracker), the open-s
|
|
|
18
18
|
<sub>105 carriers in the catalog · 58 countries represented</sub>
|
|
19
19
|
<!-- /GENERATED:summary -->
|
|
20
20
|
|
|
21
|
-
[Try it](#try-it) · [Ways to run it](#ways-to-run-it) · [
|
|
21
|
+
[Try it](#try-it) · [Ways to run it](#ways-to-run-it) · [Benchmark](#benchmark) · [Carriers](carriers/) · [Add a carrier](CONTRIBUTING.md)
|
|
22
22
|
|
|
23
23
|
<img src="docs/assets/terminal.svg" width="840" alt="A terminal session: the detect command names UPS as the carrier of a tracking number, then the track command prints a parcel's scans with their stages">
|
|
24
24
|
|
|
25
25
|
</div>
|
|
26
26
|
|
|
27
|
-
Give it a tracking number. It
|
|
28
|
-
|
|
29
|
-
a Poczta Polska parcel look the same to your code.
|
|
27
|
+
Give it a tracking number. It finds the carrier, fetches the history from the carrier's own
|
|
28
|
+
site and returns the same JSON for every carrier. No account, no API key.
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
rest it can ask the universal trackers such as 17TRACK and Ship24, which is where the big
|
|
34
|
-
number above comes from. Those stay off until you switch them on, and each one you enable
|
|
35
|
-
sees the numbers you send it.
|
|
30
|
+
It is the tracking engine of [Peek](https://github.com/plhery/delivery-tracker), usable on
|
|
31
|
+
its own as a command, a Node library or an HTTP server.
|
|
36
32
|
|
|
37
|
-
|
|
38
|
-
and was pulled out so it can be used on its own. It runs as a command-line tool and as a Node
|
|
39
|
-
library, and it can serve the same lookups over HTTP.
|
|
33
|
+
## Benchmark
|
|
40
34
|
|
|
41
|
-
|
|
35
|
+
<!-- GENERATED:coverage -->
|
|
36
|
+
Benchmarked against 100 popular carriers, it returns tracking history for **70**. The best single aggregator returns 50.
|
|
42
37
|
|
|
43
|
-
|
|
38
|
+
<img src="docs/assets/coverage.svg" alt="Carriers with tracking history: This project, all fallbacks enabled 70, This project, dedicated adapters alone 51, ParcelsApp 50, Postal Ninja 41, 17TRACK 40, Ship24 38, UPU 11." width="760">
|
|
39
|
+
<!-- /GENERATED:coverage -->
|
|
44
40
|
|
|
45
|
-
|
|
46
|
-
npx universal-parcel-scraper detect 1Z999AA10123456784
|
|
47
|
-
```
|
|
41
|
+
[Method and results per carrier](providers/COVERAGE.md)
|
|
48
42
|
|
|
49
|
-
|
|
50
|
-
|
|
43
|
+
## Try it
|
|
44
|
+
|
|
45
|
+
Needs Node.js 24 or newer.
|
|
51
46
|
|
|
52
47
|
```sh
|
|
53
|
-
npx universal-parcel-scraper
|
|
48
|
+
npx universal-parcel-scraper detect 1Z999AA10123456784 # offline: names the carrier
|
|
49
|
+
npx universal-parcel-scraper track YOUR_TRACKING_NUMBER # asks the carrier
|
|
54
50
|
```
|
|
55
51
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
or the link from the shipping email (`--tracking-url`).
|
|
52
|
+
Add `--carrier dhl` when a number fits several carriers. Some carriers also want
|
|
53
|
+
`--postcode` or `--tracking-url`.
|
|
59
54
|
|
|
60
55
|
## Ways to run it
|
|
61
56
|
|
|
@@ -65,23 +60,15 @@ or the link from the shipping email (`--tracking-url`).
|
|
|
65
60
|
npm install -g universal-parcel-scraper
|
|
66
61
|
```
|
|
67
62
|
|
|
68
|
-
The command is `parcel-scraper`:
|
|
69
|
-
|
|
70
63
|
```sh
|
|
71
64
|
parcel-scraper detect <number, link or text> # which carrier is this? no network
|
|
72
65
|
parcel-scraper track <number> # the parcel's history
|
|
73
66
|
parcel-scraper recognize <number> # ask the possible carriers which one knows it
|
|
74
67
|
parcel-scraper carriers # the catalog, with the ids --carrier accepts
|
|
75
|
-
parcel-scraper serve # the HTTP API
|
|
76
|
-
```
|
|
77
|
-
|
|
78
|
-
Everything prints JSON, so it pipes:
|
|
79
|
-
|
|
80
|
-
```sh
|
|
81
|
-
parcel-scraper track "$NUMBER" | jq -r '.result.current_stage'
|
|
68
|
+
parcel-scraper serve # the HTTP API
|
|
82
69
|
```
|
|
83
70
|
|
|
84
|
-
`parcel-scraper --help` lists the environment variables.
|
|
71
|
+
Output is JSON, so it pipes into `jq`. `parcel-scraper --help` lists the environment variables.
|
|
85
72
|
|
|
86
73
|
### Node
|
|
87
74
|
|
|
@@ -97,10 +84,8 @@ for (const scan of result.events) {
|
|
|
97
84
|
}
|
|
98
85
|
```
|
|
99
86
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
returns a history, `track()` throws a `TrackingError` carrying those attempts and a hint about
|
|
103
|
-
the cause. [examples/node.mjs](examples/node.mjs) is a runnable version.
|
|
87
|
+
`source` names who answered and `attempts` lists what was tried. With no history, `track()`
|
|
88
|
+
throws a `TrackingError` carrying both. [Runnable example](examples/node.mjs).
|
|
104
89
|
|
|
105
90
|
### Browser
|
|
106
91
|
|
|
@@ -111,143 +96,65 @@ parseTrackingInput('1Z999AA10123456784');
|
|
|
111
96
|
// { carrier: 'ups', confidence: 'high', candidates: ['ups'], … }
|
|
112
97
|
```
|
|
113
98
|
|
|
114
|
-
|
|
115
|
-
and name the carrier as someone types.
|
|
116
|
-
|
|
117
|
-
### HTTP
|
|
99
|
+
Detection has no Node imports and makes no requests.
|
|
118
100
|
|
|
119
|
-
|
|
101
|
+
### HTTP and Docker
|
|
120
102
|
|
|
121
103
|
```sh
|
|
122
104
|
parcel-scraper serve --port 8080
|
|
123
105
|
```
|
|
124
106
|
|
|
125
107
|
```sh
|
|
126
|
-
|
|
127
|
-
-H 'Content-Type: application/json' \
|
|
128
|
-
-d '{"number":"1Z999AA10123456784","carrier":"ups"}'
|
|
108
|
+
docker run --rm -p 127.0.0.1:8080:8080 ghcr.io/plhery/universal-parcel-scraper:0.3.0
|
|
129
109
|
```
|
|
130
110
|
|
|
131
|
-
`/v1/detect`, `/v1/recognize` and `/v1/carriers` sit next to it, and the
|
|
132
|
-
[OpenAPI file](server/openapi.json) describes them all. The server caches recent answers in
|
|
133
|
-
memory and rate-limits callers. It has no database and never polls on its own.
|
|
134
|
-
|
|
135
|
-
Set `SCRAPER_TOKEN` to require a bearer token. `SCRAPER_DEMO_PAGE=true` serves a small page
|
|
136
|
-
at `/` for trying numbers by hand. Behind a reverse proxy, set `SCRAPER_TRUSTED_PROXIES` so
|
|
137
|
-
the rate limit counts each caller and not the proxy. The other settings are in
|
|
138
|
-
[.env.example](.env.example).
|
|
139
|
-
|
|
140
|
-
### Docker
|
|
141
|
-
|
|
142
111
|
```sh
|
|
143
|
-
|
|
112
|
+
curl http://127.0.0.1:8080/v1/track \
|
|
113
|
+
-H 'Content-Type: application/json' \
|
|
114
|
+
-d '{"number":"1Z999AA10123456784","carrier":"ups"}'
|
|
144
115
|
```
|
|
145
116
|
|
|
146
|
-
The
|
|
117
|
+
The server caches answers in memory and rate-limits callers. It has no database and never
|
|
118
|
+
polls. The image ships Chromium. Routes are in the [OpenAPI file](server/openapi.json) and
|
|
119
|
+
settings, such as `SCRAPER_TOKEN` and `SCRAPER_TRUSTED_PROXIES`, in [.env.example](.env.example).
|
|
147
120
|
|
|
148
121
|
## What you can build with it
|
|
149
122
|
|
|
150
|
-
-
|
|
151
|
-
|
|
152
|
-
-
|
|
153
|
-
|
|
154
|
-
-
|
|
155
|
-
site. Any backend that speaks HTTP can call the server.
|
|
156
|
-
- **Tracking numbers pulled out of shipping emails.** `detect` takes pasted text and carrier
|
|
157
|
-
links as well as bare numbers.
|
|
158
|
-
- **A carrier field that fills itself in**, with the browser import.
|
|
159
|
-
- **A cron job** that pings you when `current_stage` turns `delivered`.
|
|
123
|
+
- A parcel-tracking app, like [Peek](https://github.com/plhery/delivery-tracker).
|
|
124
|
+
- A [Home Assistant sensor](examples/home-assistant.yaml) for the parcel you are waiting on.
|
|
125
|
+
- Order status inside a shop or help desk, from any backend that speaks HTTP.
|
|
126
|
+
- Tracking numbers pulled out of shipping emails: `detect` reads pasted text and links.
|
|
127
|
+
- A cron job that pings you when `current_stage` turns `delivered`.
|
|
160
128
|
|
|
161
|
-
|
|
162
|
-
are yours to do.
|
|
129
|
+
It answers one lookup at a time. Storing parcels and polling are yours.
|
|
163
130
|
|
|
164
131
|
## How it works
|
|
165
132
|
|
|
166
|
-
<
|
|
167
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/how-it-works-dark.svg">
|
|
168
|
-
<img src="docs/assets/how-it-works-light.svg" width="840" alt="An input is detected offline and fetched by the carrier's dedicated adapter, or by a fallback you enabled when that finds no history. Each scan's wording is filed under a stage, and the result is one timeline.">
|
|
169
|
-
</picture>
|
|
170
|
-
|
|
171
|
-
Each adapter uses plain HTTP wherever the carrier's site allows it. Some sites only answer a
|
|
172
|
-
real browser and others need image processing, so those adapters can drive a local Chromium
|
|
173
|
-
or the optional [TRAWL browser service](trawl/README.md). Install what your carriers need:
|
|
133
|
+
<img src="docs/assets/how-it-works.svg" width="840" alt="An input is detected offline and fetched by the carrier's dedicated adapter, or by a fallback you enabled when that finds no history. Each scan's wording is filed under a stage, and the result is one timeline.">
|
|
174
134
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
Then pass `chromiumPath` or `trawlUrl` to `createTracker()`. The CLI and the server read
|
|
180
|
-
`TRACKING_CHROMIUM_PATH` and `FLARESOLVERR_URL`.
|
|
135
|
+
**Fetching.** Adapters use plain HTTP where the site allows it. For sites that only answer a
|
|
136
|
+
browser, install `playwright-core sharp onnxruntime-web` and pass `chromiumPath`, or run the
|
|
137
|
+
[TRAWL browser service](trawl/README.md) and pass `trawlUrl`. Pass `userAgent` to name your
|
|
138
|
+
install to carriers.
|
|
181
139
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
When the adapter finds no history, the lookup can move on to an aggregator. Only UPU is on
|
|
186
|
-
by default, for the postal numbers it can serve. The commercial ones are opt-in, and each
|
|
187
|
-
one you enable receives the tracking number:
|
|
140
|
+
**Fallbacks.** When an adapter finds no history, the lookup moves to the universal trackers
|
|
141
|
+
you enabled. Only UPU is on by default. Each one you enable receives the tracking number.
|
|
188
142
|
|
|
189
143
|
```js
|
|
190
144
|
createTracker({ providers: ['ParcelsApp', 'Ship24', '17TRACK', 'Postal Ninja', 'UPU'] });
|
|
191
145
|
```
|
|
192
146
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
Carriers describe the same moment in their own words, or with a bare code. Each carrier
|
|
196
|
-
folder keeps a `statuses.json` with the codes and wordings seen from that carrier and the
|
|
197
|
-
stage each one means. Every entry also says how it was confirmed, by a live reply or the
|
|
198
|
-
carrier's own documentation for instance.
|
|
147
|
+
**Stages.** Carriers word the same moment differently. Every scan is filed under one stage:
|
|
199
148
|
|
|
200
149
|
<!-- GENERATED:stages -->
|
|
201
|
-
<
|
|
202
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/stages-dark.svg">
|
|
203
|
-
<img src="docs/assets/stages-light.svg" alt="DHL: Die Sendung wurde in das Zustellfahrzeug geladen.; Mondial Relay: En cours de livraison; Correios Brazil: Objeto saiu para entrega ao destinatário; Correos Express: EN REPARTO; Yamato Transport: 配達中; La Poste / Colissimo: DISTOU. All are filed under out_for_delivery." width="760">
|
|
204
|
-
</picture>
|
|
150
|
+
<img src="docs/assets/stages.svg" alt="DHL: Die Sendung wurde in das Zustellfahrzeug geladen.; Mondial Relay: En cours de livraison; Correios Brazil: Objeto saiu para entrega ao destinatário; Correos Express: EN REPARTO; Yamato Transport: 配達中; La Poste / Colissimo: DISTOU. All are filed under out_for_delivery." width="760">
|
|
205
151
|
|
|
206
|
-
The folders
|
|
152
|
+
The carrier folders record 1,672 such statuses, each filed under one stage.
|
|
207
153
|
<!-- /GENERATED:stages -->
|
|
208
154
|
|
|
209
|
-
|
|
210
|
-
German, Italian, Spanish, Portuguese and Polish. Each scan in the result carries a
|
|
211
|
-
`stage_source` saying which of the two decided.
|
|
212
|
-
|
|
155
|
+
Wording nobody recorded yet goes through a classifier that reads seven European languages.
|
|
213
156
|
[ARCHITECTURE.md](ARCHITECTURE.md) has the rest.
|
|
214
157
|
|
|
215
|
-
## How much can it track?
|
|
216
|
-
|
|
217
|
-
The number at the top of this page is reach: with the fallbacks on, a lookup can go to the
|
|
218
|
-
largest aggregator, and [reach.json](docs/reach.json) records how many carriers each one
|
|
219
|
-
says it follows. Whether a real parcel comes back with its history is a different question,
|
|
220
|
-
so each source was run on its own against the same **100-carrier reference set**. The top
|
|
221
|
-
bar counts a carrier when any source below it returned a history.
|
|
222
|
-
|
|
223
|
-
<!-- GENERATED:coverage -->
|
|
224
|
-
<picture>
|
|
225
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/coverage-dark.svg">
|
|
226
|
-
<img src="docs/assets/coverage-light.svg" alt="Carriers with tracking history: This project, all fallbacks enabled 70, This project, dedicated adapters alone 51, ParcelsApp 50, Postal Ninja 41, 17TRACK 40, Ship24 38, UPU 11." width="760">
|
|
227
|
-
</picture>
|
|
228
|
-
|
|
229
|
-
<details>
|
|
230
|
-
<summary>The same numbers as a table</summary>
|
|
231
|
-
|
|
232
|
-
| Source | Carriers with history |
|
|
233
|
-
| --- | ---: |
|
|
234
|
-
| **Universal Parcel Scraper, all fallbacks enabled** | **70 / 100** |
|
|
235
|
-
| Dedicated adapters alone | 51 / 100 |
|
|
236
|
-
| ParcelsApp | 50 / 100 |
|
|
237
|
-
| Postal Ninja | 41 / 100 |
|
|
238
|
-
| 17TRACK | 40 / 100 |
|
|
239
|
-
| Ship24 | 38 / 100 |
|
|
240
|
-
| UPU | 11 / 100 |
|
|
241
|
-
|
|
242
|
-
</details>
|
|
243
|
-
<!-- /GENERATED:coverage -->
|
|
244
|
-
|
|
245
|
-
The numbers come from [coverage.json](providers/coverage.json): the recorded outcome for each
|
|
246
|
-
carrier's comparison reference, partial histories included and alternate samples left out.
|
|
247
|
-
The set is curated. Read it as a comparison between sources, not as a market-share ranking
|
|
248
|
-
or a promise about today, because parcels expire and carrier sites change.
|
|
249
|
-
[Carrier-by-carrier results and limitations](providers/COVERAGE.md).
|
|
250
|
-
|
|
251
158
|
## Next to a hosted tracking API
|
|
252
159
|
|
|
253
160
|
| | Universal Parcel Scraper | Hosted tracking API |
|
|
@@ -255,38 +162,25 @@ or a promise about today, because parcels expire and carrier sites change.
|
|
|
255
162
|
| Account | None | Sign-up and an API key |
|
|
256
163
|
| Cost | Your own compute | A plan or a per-shipment price |
|
|
257
164
|
| Who sees the tracking number | The carrier, plus any fallback you enable | The vendor, then the carrier |
|
|
258
|
-
| Carriers | Dedicated adapters for the [catalog](carriers/), fallbacks for the rest | One large catalog |
|
|
259
165
|
| Updates | You poll | Webhooks |
|
|
260
166
|
| When a carrier changes its site | The adapter breaks until it is fixed here | The vendor deals with it |
|
|
261
|
-
| Hosting | Yours | Theirs |
|
|
262
|
-
|
|
263
|
-
If you want webhooks and nothing to run, a hosted API is the better choice. This project is
|
|
264
|
-
for when the numbers should stay on your side, or when a price per parcel makes no sense for
|
|
265
|
-
what you are building. Vendor catalog totals are not comparable with the reference test
|
|
266
|
-
above.
|
|
267
167
|
|
|
268
168
|
## Privacy and limits
|
|
269
169
|
|
|
270
|
-
A lookup sends the
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
This is scraping, so things break. Carriers redesign their sites and turn away automated
|
|
277
|
-
requests. A few need more than the number, such as a postcode or the link from the shipping
|
|
278
|
-
email, and the ones that require an account stay limited. Keep to the catalog's refresh
|
|
279
|
-
limits and to the retry advice that comes with an error. A scan time with no known UTC offset
|
|
280
|
-
is returned as the carrier wrote it, with no zone guessed.
|
|
170
|
+
- A lookup sends the number to the carrier, with the postcode or tracking link when the
|
|
171
|
+
carrier requires one. Enabled fallbacks receive the number too.
|
|
172
|
+
- No telemetry. Server logs hold route names and outcomes, never parcel inputs.
|
|
173
|
+
- This is scraping: carriers change their sites and turn away automated requests. Keep to the
|
|
174
|
+
catalog's refresh limits and to the retry advice in errors.
|
|
175
|
+
- A scan time with no known UTC offset is returned as the carrier wrote it.
|
|
281
176
|
|
|
282
177
|
## Contributing
|
|
283
178
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
rest. Report security issues through [SECURITY.md](SECURITY.md).
|
|
179
|
+
Each carrier lives in its own folder with synthetic fixtures and offline tests.
|
|
180
|
+
`npm run carrier:new` scaffolds a new one. See [CONTRIBUTING.md](CONTRIBUTING.md) and
|
|
181
|
+
[SECURITY.md](SECURITY.md).
|
|
288
182
|
|
|
289
183
|
## License
|
|
290
184
|
|
|
291
|
-
|
|
292
|
-
[
|
|
185
|
+
[Apache-2.0](LICENSE). The optional TRAWL image is [AGPL-3.0](trawl/LICENSE). Credits are in
|
|
186
|
+
[NOTICE](NOTICE).
|
package/dist/app.d.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Helpers shaped for the parcel app this package was extracted from: its
|
|
3
3
|
* parcel view, carrier-name hints for provider results, the clock helpers
|
|
4
|
-
* its sync uses, and the country a scan's
|
|
5
|
-
* that app and carry no stability promise;
|
|
6
|
-
* other entry points.
|
|
4
|
+
* its sync uses, carrier scan-identity policies, and the country a scan's
|
|
5
|
+
* location names. They change with that app and carry no stability promise;
|
|
6
|
+
* the tracking contract is the other entry points.
|
|
7
7
|
*/
|
|
8
8
|
export * from './core/catalog/parcel.js';
|
|
9
9
|
export * from './core/catalog/hints.js';
|
|
10
10
|
export * from './core/time/result.js';
|
|
11
|
+
export { sameInstantIdentityPolicy, type SameInstantIdentityPolicy } from './core/catalog/eventIdentity.js';
|
|
11
12
|
export { universalCarrierHints } from './providers/shared/hints.js';
|
|
12
13
|
export { recognitionAskedCarriers } from './core/catalog/recognition.js';
|
|
13
14
|
export { countryFlag, countryName, trackingLocationCountry, trackingPlace, type TrackingPlace } from './places/trackingLocation.js';
|
package/dist/app.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Helpers shaped for the parcel app this package was extracted from: its
|
|
3
3
|
* parcel view, carrier-name hints for provider results, the clock helpers
|
|
4
|
-
* its sync uses, and the country a scan's
|
|
5
|
-
* that app and carry no stability promise;
|
|
6
|
-
* other entry points.
|
|
4
|
+
* its sync uses, carrier scan-identity policies, and the country a scan's
|
|
5
|
+
* location names. They change with that app and carry no stability promise;
|
|
6
|
+
* the tracking contract is the other entry points.
|
|
7
7
|
*/
|
|
8
8
|
export * from './core/catalog/parcel.js';
|
|
9
9
|
export * from './core/catalog/hints.js';
|
|
10
10
|
export * from './core/time/result.js';
|
|
11
|
+
export { sameInstantIdentityPolicy } from './core/catalog/eventIdentity.js';
|
|
11
12
|
export { universalCarrierHints } from './providers/shared/hints.js';
|
|
12
13
|
export { recognitionAskedCarriers } from './core/catalog/recognition.js';
|
|
13
14
|
export { countryFlag, countryName, trackingLocationCountry, trackingPlace } from './places/trackingLocation.js';
|
package/dist/browser/scraper.js
CHANGED
|
@@ -7503,7 +7503,8 @@ var Wo = (e) => e.toLowerCase().replace(/[^a-z0-9]/g, ""), Go = [
|
|
|
7503
7503
|
dhlecommerce: "dhl-ecommerce",
|
|
7504
7504
|
cainiao: "aliexpress",
|
|
7505
7505
|
postnl: "spring-gds",
|
|
7506
|
-
asendiausa: "asendia"
|
|
7506
|
+
asendiausa: "asendia",
|
|
7507
|
+
finlandpost: "posti"
|
|
7507
7508
|
};
|
|
7508
7509
|
[
|
|
7509
7510
|
...Object.entries(_).filter(([e]) => e !== "unknown").map(([, e]) => Wo(e.displayName)),
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
// Verified and postcode-free replies reword the same scans at the same instants.
|
|
2
|
+
// A universal provider can also have stored those scans while DPD was unavailable.
|
|
3
|
+
export const sameInstantIdentityPolicy = {
|
|
4
|
+
sourceCarrierId: 'dpd', storedSources: ['dpd', 'unknown'], requireProviderCode: false,
|
|
5
|
+
};
|
|
@@ -5,7 +5,7 @@ import { lookupBudget } from '../../core/adapter/index.js';
|
|
|
5
5
|
import { ChallengeError, IndeterminateError, InvalidInputError, NotFoundError, SchemaError } from '../../core/errors/index.js';
|
|
6
6
|
import { eventPoint } from '../../core/result/index.js';
|
|
7
7
|
import { isValidS10TrackingNumber } from '../../core/detection/index.js';
|
|
8
|
-
import { isoTime } from '../../core/time/index.js';
|
|
8
|
+
import { isoTime, mislabeledLocalTime } from '../../core/time/index.js';
|
|
9
9
|
import { cleanScalar, decodeText, fetchBounded, parseJsonBytes, UpstreamHttpError, } from '../../core/transport/index.js';
|
|
10
10
|
import { isRecord } from '../../core/types.js';
|
|
11
11
|
import { classifyIndiaPostEvent } from './status.js';
|
|
@@ -87,6 +87,60 @@ function eventText(raw) {
|
|
|
87
87
|
return EVENT_TEXT[code] ?? code.toLowerCase().split('_')
|
|
88
88
|
.map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(' ');
|
|
89
89
|
}
|
|
90
|
+
// India Post's code for a flight leaving, in `event_type`. The wording beside
|
|
91
|
+
// it changes: one row read "Aircraft Departure", then "UPLIFT".
|
|
92
|
+
const TAKE_OFF_CODE = 'AircraftTakeOff';
|
|
93
|
+
// The label a take-off's wall clock arrives under; any other offset is kept.
|
|
94
|
+
const UTC_LABEL = /(?:Z|[+-]00:?00)$/i;
|
|
95
|
+
// The zones of the airports a take-off row may name, by IATA code.
|
|
96
|
+
const AIRPORT_ZONES = {
|
|
97
|
+
BOM: 'Asia/Kolkata', DEL: 'Asia/Kolkata', MAA: 'Asia/Kolkata', CCU: 'Asia/Kolkata',
|
|
98
|
+
BLR: 'Asia/Kolkata', HYD: 'Asia/Kolkata', COK: 'Asia/Kolkata', AMD: 'Asia/Kolkata',
|
|
99
|
+
FRA: 'Europe/Berlin', MUC: 'Europe/Berlin', LEJ: 'Europe/Berlin', CGN: 'Europe/Berlin',
|
|
100
|
+
LHR: 'Europe/London', CDG: 'Europe/Paris', AMS: 'Europe/Amsterdam', ZRH: 'Europe/Zurich',
|
|
101
|
+
VIE: 'Europe/Vienna', BRU: 'Europe/Brussels', LGG: 'Europe/Brussels', FCO: 'Europe/Rome',
|
|
102
|
+
MXP: 'Europe/Rome', MAD: 'Europe/Madrid', CPH: 'Europe/Copenhagen', ARN: 'Europe/Stockholm',
|
|
103
|
+
HEL: 'Europe/Helsinki', IST: 'Europe/Istanbul',
|
|
104
|
+
DXB: 'Asia/Dubai', AUH: 'Asia/Dubai', DOH: 'Asia/Qatar', SIN: 'Asia/Singapore',
|
|
105
|
+
HKG: 'Asia/Hong_Kong', BKK: 'Asia/Bangkok', NRT: 'Asia/Tokyo', ICN: 'Asia/Seoul',
|
|
106
|
+
PVG: 'Asia/Shanghai', KUL: 'Asia/Kuala_Lumpur', CMB: 'Asia/Colombo', DAC: 'Asia/Dhaka',
|
|
107
|
+
KTM: 'Asia/Kathmandu',
|
|
108
|
+
JFK: 'America/New_York', EWR: 'America/New_York', ORD: 'America/Chicago',
|
|
109
|
+
LAX: 'America/Los_Angeles', SFO: 'America/Los_Angeles', YYZ: 'America/Toronto',
|
|
110
|
+
SYD: 'Australia/Sydney', MEL: 'Australia/Melbourne',
|
|
111
|
+
};
|
|
112
|
+
// Only verified names are expanded; other airport codes stay as the carrier wrote them.
|
|
113
|
+
const AIRPORT_PLACES = {
|
|
114
|
+
BOM: { name: 'Mumbai Airport', country: 'India' },
|
|
115
|
+
DEL: { name: 'Delhi Airport', country: 'India' },
|
|
116
|
+
FRA: { name: 'Frankfurt Airport', country: 'Germany' },
|
|
117
|
+
CDG: { name: 'Paris Charles de Gaulle Airport', country: 'France' },
|
|
118
|
+
};
|
|
119
|
+
function departureAirport(office) {
|
|
120
|
+
return /^Office - ([A-Z]{3})\b/.exec(office)?.[1];
|
|
121
|
+
}
|
|
122
|
+
/** Flight remarks have a narrow format; unrelated remarks can contain recipient details. */
|
|
123
|
+
function flightDescription(remarks, office) {
|
|
124
|
+
const match = /^Flight No:\s*([A-Z0-9]{2}\d{1,4}[A-Z]?)\s*\(From ([A-Z]{3}) To ([A-Z]{3})\)$/.exec(clean(remarks, 500));
|
|
125
|
+
if (!match || match[2] !== departureAirport(office))
|
|
126
|
+
return 'Aircraft Departure';
|
|
127
|
+
const airport = (code) => AIRPORT_PLACES[code] ? `${AIRPORT_PLACES[code].name} (${code})` : code;
|
|
128
|
+
return `Flight ${match[1]} departed: ${airport(match[2])} → ${airport(match[3])}`;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* The zone a take-off row's clock is kept in. Its `tracked_at` is the
|
|
132
|
+
* departure airport's wall clock under a UTC label: a take-off was seen
|
|
133
|
+
* recorded hours before its labelled time. The office names the airport first,
|
|
134
|
+
* as in "Office - DEL 00000000". Null for every other row, for an airport
|
|
135
|
+
* outside the table and for a value not labelled UTC: those are read as any
|
|
136
|
+
* row is.
|
|
137
|
+
*/
|
|
138
|
+
function takeOffZone(providerCode, office, trackedAt) {
|
|
139
|
+
if (providerCode !== TAKE_OFF_CODE || !UTC_LABEL.test(trackedAt))
|
|
140
|
+
return null;
|
|
141
|
+
const airport = departureAirport(office);
|
|
142
|
+
return airport ? AIRPORT_ZONES[airport] ?? null : null;
|
|
143
|
+
}
|
|
90
144
|
/**
|
|
91
145
|
* The office's point from MySpeedPost's pincode directory. It is looked up by
|
|
92
146
|
* pincode, not by office: "KOLKATA FOREIGN LCAO 900056" comes back as an office
|
|
@@ -128,18 +182,24 @@ export function parseIndiaPostTrackingHtml(html, trackingNumber) {
|
|
|
128
182
|
const parsed = [];
|
|
129
183
|
const seen = new Set();
|
|
130
184
|
events.slice(0, 500).forEach((rawEvent, index) => {
|
|
185
|
+
const office = clean(rawEvent.office, 120);
|
|
186
|
+
const providerCode = clean(rawEvent.event_type, 100);
|
|
131
187
|
// tracked_at is ISO; offset-less values are read as Asia/Kolkata, the zone
|
|
132
|
-
// every India Post office stamps.
|
|
133
|
-
const
|
|
134
|
-
const
|
|
188
|
+
// every India Post office stamps. A take-off is read on its airport's clock.
|
|
189
|
+
const zone = takeOffZone(providerCode, office, clean(rawEvent.tracked_at, 100));
|
|
190
|
+
const time = zone
|
|
191
|
+
? mislabeledLocalTime(rawEvent.tracked_at, zone, 100)
|
|
192
|
+
: isoTime(rawEvent.tracked_at, 'Asia/Kolkata', 100);
|
|
193
|
+
const takeOff = providerCode === TAKE_OFF_CODE;
|
|
194
|
+
const description = takeOff ? flightDescription(rawEvent.remarks, office) : eventText(clean(rawEvent.event));
|
|
135
195
|
if (!time || !description)
|
|
136
196
|
return;
|
|
137
|
-
const office = clean(rawEvent.office, 120);
|
|
138
197
|
const pincode = /^\d{6}$/.test(clean(rawEvent.pincode, 6))
|
|
139
198
|
? clean(rawEvent.pincode, 6)
|
|
140
199
|
: '';
|
|
141
|
-
const providerCode = clean(rawEvent.event_type, 100);
|
|
142
200
|
const point = officePoint(rawEvent, office);
|
|
201
|
+
const airportCode = takeOff ? departureAirport(office) : undefined;
|
|
202
|
+
const airport = airportCode ? AIRPORT_PLACES[airportCode] : undefined;
|
|
143
203
|
const identity = JSON.stringify([time.iso, description, office, pincode, providerCode]);
|
|
144
204
|
if (seen.has(identity))
|
|
145
205
|
return;
|
|
@@ -151,7 +211,7 @@ export function parseIndiaPostTrackingHtml(html, trackingNumber) {
|
|
|
151
211
|
// deliberately never retained.
|
|
152
212
|
event: {
|
|
153
213
|
time: time.iso,
|
|
154
|
-
location: [office, pincode].filter(Boolean).join(' '),
|
|
214
|
+
location: airport ? `${airport.name} (${airportCode}), ${airport.country}` : [office, pincode].filter(Boolean).join(' '),
|
|
155
215
|
description,
|
|
156
216
|
stage: classified.stage,
|
|
157
217
|
...(providerCode ? { provider_code: providerCode } : {}),
|
|
@@ -11,6 +11,8 @@ function includesAny(value, candidates) {
|
|
|
11
11
|
* `in_transit` stage.
|
|
12
12
|
*/
|
|
13
13
|
export function classifyIndiaPostEvent(...values) {
|
|
14
|
+
if (statusKey(values[0]) === 'aircrafttakeoff')
|
|
15
|
+
return { status: 'in_transit', stage: 'in_transit' };
|
|
14
16
|
const key = values.map(statusKey).filter(Boolean).join(' ');
|
|
15
17
|
if (includesAny(key, [
|
|
16
18
|
'returntosender',
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"carrier": "india-post",
|
|
3
3
|
"entries": [
|
|
4
|
+
{ "code": "AircraftTakeOff", "wording": "Aircraft Departure / UPLIFT", "language": "en", "stage": "in_transit", "confirmedBy": "live", "firstSeen": "2026-10-04", "note": "Departure from the airport named by the office; flight remarks name the route, not a destination arrival." },
|
|
4
5
|
{ "code": "ItemBooked", "wording": "Item Booked", "language": "en", "stage": "accepted", "confirmedBy": "fixture", "firstSeen": "2026-09-01", "note": "Substring key `itembooked`; `articlebooked` and `bookingconfirmed` map the same way." },
|
|
5
6
|
{ "code": "ItemDispatched", "wording": "Item Dispatched", "language": "en", "stage": "in_transit", "confirmedBy": "fixture", "firstSeen": "2026-09-01", "note": "Substring key `itemdispatched`." },
|
|
6
7
|
{ "wording": "Item Bagged", "language": "en", "stage": "in_transit", "confirmedBy": "prior-art", "firstSeen": "2026-09-01", "note": "Substring key `itembagged`." },
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Carrier evidence the parcel app uses when updating a stored scan in place. */
|
|
2
|
+
export interface SameInstantIdentityPolicy {
|
|
3
|
+
readonly sourceCarrierId: string;
|
|
4
|
+
readonly storedSources: readonly string[];
|
|
5
|
+
readonly requireProviderCode: boolean;
|
|
6
|
+
}
|
|
7
|
+
/** Unlisted sources cannot identify a reworded scan by its instant alone. */
|
|
8
|
+
export declare function sameInstantIdentityPolicy(sourceCarrierId: string): SameInstantIdentityPolicy | undefined;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { sameInstantIdentityPolicy as dpd } from '../../carriers/dpd/app.js';
|
|
2
|
+
import { sameInstantIdentityPolicy as indiaPost } from '../../carriers/india-post/app.js';
|
|
3
|
+
const policies = new Map([dpd, indiaPost].map((policy) => [policy.sourceCarrierId, policy]));
|
|
4
|
+
/** Unlisted sources cannot identify a reworded scan by its instant alone. */
|
|
5
|
+
export function sameInstantIdentityPolicy(sourceCarrierId) {
|
|
6
|
+
return policies.get(sourceCarrierId);
|
|
7
|
+
}
|
|
@@ -8,9 +8,17 @@ export declare function carrierIdFromName(name: string): string | undefined;
|
|
|
8
8
|
/**
|
|
9
9
|
* Whether a reported carrier name is already in the catalog: a carrier or alias,
|
|
10
10
|
* another network of a brand that has several ("DHL Express", "GLS Italy"), or
|
|
11
|
-
* one of those followed by a country.
|
|
11
|
+
* one of those followed by a country. A name it does not know is a carrier new
|
|
12
|
+
* to us or a feed (`isCarrierFeedName`).
|
|
12
13
|
*/
|
|
13
14
|
export declare function isKnownCarrierName(name: string): boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Whether a reported name is the postal union's shared feed ("UPU", "Universal
|
|
17
|
+
* Postal Union"). The feed relays every postal operator's scans and names no
|
|
18
|
+
* carrier: it has no catalog id and is not a known carrier name, so a reply
|
|
19
|
+
* that names only the feed says nothing about whose parcel it is.
|
|
20
|
+
*/
|
|
21
|
+
export declare function isCarrierFeedName(name: string): boolean;
|
|
14
22
|
/**
|
|
15
23
|
* The single clock of the country after a known carrier or brand network
|
|
16
24
|
* ("DPD UK", "GLS Italy", "DHL Parcel Netherlands"), for scans that carry no
|
|
@@ -6,11 +6,15 @@ export { carrierBrand };
|
|
|
6
6
|
const key = (name) => name.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
7
7
|
// These brands have several regional/service adapters.
|
|
8
8
|
const AMBIGUOUS_BRANDS = ['dhl', 'dpd', 'gls', 'hermes', 'post'];
|
|
9
|
+
// An alias also gives the name its carrier's clock, which moves the stored
|
|
10
|
+
// instants, and so the event ids, of past scans under that name.
|
|
9
11
|
const NAME_ALIASES = {
|
|
10
12
|
ups: 'ups', swisspost: 'swiss-post', laposte: 'la-poste', colissimo: 'la-poste',
|
|
11
13
|
dhlecommerce: 'dhl-ecommerce', cainiao: 'aliexpress', postnl: 'spring-gds',
|
|
12
|
-
asendiausa: 'asendia',
|
|
14
|
+
asendiausa: 'asendia', finlandpost: 'posti',
|
|
13
15
|
};
|
|
16
|
+
// The postal union's shared feed, which aggregators list among a parcel's carriers.
|
|
17
|
+
const FEED_NAMES = ['upu', 'universalpostalunion'];
|
|
14
18
|
const CATALOG_NAMES = new Set([
|
|
15
19
|
...Object.entries(CARRIER_DEFINITIONS).filter(([id]) => id !== 'unknown')
|
|
16
20
|
.map(([, definition]) => key(definition.displayName)),
|
|
@@ -81,12 +85,22 @@ export function carrierIdFromName(name) {
|
|
|
81
85
|
/**
|
|
82
86
|
* Whether a reported carrier name is already in the catalog: a carrier or alias,
|
|
83
87
|
* another network of a brand that has several ("DHL Express", "GLS Italy"), or
|
|
84
|
-
* one of those followed by a country.
|
|
88
|
+
* one of those followed by a country. A name it does not know is a carrier new
|
|
89
|
+
* to us or a feed (`isCarrierFeedName`).
|
|
85
90
|
*/
|
|
86
91
|
export function isKnownCarrierName(name) {
|
|
87
92
|
const qualified = countryQualified(name);
|
|
88
93
|
return [key(name), ...(qualified ? [key(qualified.base)] : [])].some((candidate) => CATALOG_NAMES.has(candidate) || NETWORK_BRANDS.some((brand) => candidate.startsWith(brand)));
|
|
89
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Whether a reported name is the postal union's shared feed ("UPU", "Universal
|
|
97
|
+
* Postal Union"). The feed relays every postal operator's scans and names no
|
|
98
|
+
* carrier: it has no catalog id and is not a known carrier name, so a reply
|
|
99
|
+
* that names only the feed says nothing about whose parcel it is.
|
|
100
|
+
*/
|
|
101
|
+
export function isCarrierFeedName(name) {
|
|
102
|
+
return FEED_NAMES.includes(key(name));
|
|
103
|
+
}
|
|
90
104
|
/**
|
|
91
105
|
* The single clock of the country after a known carrier or brand network
|
|
92
106
|
* ("DPD UK", "GLS Italy", "DHL Parcel Netherlands"), for scans that carry no
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "universal-parcel-scraper",
|
|
3
|
-
"version": "0.3.0-main.
|
|
3
|
+
"version": "0.3.0-main.308",
|
|
4
4
|
"description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|