universal-parcel-scraper 0.2.0 → 0.2.1-main.296
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -71
- package/data/carrier.schema.json +1 -1
- package/dist/carriers/colis-prive/adapter.js +0 -0
- package/dist/carriers/dachser/adapter.js +6 -4
- package/dist/carriers/dpd/adapter.js +2 -0
- package/dist/carriers/mrw/adapter.js +4 -2
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +32 -18
- package/dist/core/catalog/urls.js +1 -0
- package/dist/facade/index.js +12 -4
- package/dist/providers/postal-ninja/adapter.js +3 -2
- package/dist/providers/ship24/adapter.js +8 -4
- package/package.json +9 -7
package/README.md
CHANGED
|
@@ -1,124 +1,257 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
|
|
3
|
+
<img src="docs/assets/logo.svg" width="88" alt="">
|
|
4
|
+
|
|
3
5
|
# Universal Parcel Scraper
|
|
4
6
|
|
|
5
|
-
**
|
|
7
|
+
**Parcel tracking that asks the carrier directly, from your own machine.**
|
|
6
8
|
|
|
9
|
+
[](https://www.npmjs.com/package/universal-parcel-scraper)
|
|
7
10
|
[](https://github.com/plhery/universal-parcel-scraper/actions/workflows/ci.yml)
|
|
8
11
|
[](LICENSE)
|
|
9
12
|
|
|
10
|
-
Detect a carrier, fetch its tracking history, and get consistent stages and scan times.
|
|
11
|
-
A TypeScript library, a CLI, and a small HTTP server. No tracking-service account required.
|
|
12
|
-
|
|
13
13
|
<!-- GENERATED:summary -->
|
|
14
14
|
**105 carriers · 85 active dedicated adapters · 58 countries represented**
|
|
15
15
|
<!-- /GENERATED:summary -->
|
|
16
16
|
|
|
17
|
+
[Try it](#try-it) · [Ways to run it](#ways-to-run-it) · [Coverage](#how-much-can-it-track) · [Carriers](carriers/) · [Add a carrier](CONTRIBUTING.md)
|
|
18
|
+
|
|
19
|
+
<img src="docs/assets/demo.svg" width="840" alt="The track command printing a parcel's history as JSON, next to a timeline drawn from it">
|
|
20
|
+
|
|
17
21
|
</div>
|
|
18
22
|
|
|
19
|
-
|
|
20
|
-
|
|
23
|
+
Give it a tracking number. It works out which carrier the number belongs to and fetches the
|
|
24
|
+
history from that carrier's own website. What comes back has one shape, so a UPS parcel and
|
|
25
|
+
a Poczta Polska parcel look the same to your code.
|
|
26
|
+
|
|
27
|
+
There is no account to open and no API key. Every carrier publishes tracking in its own way,
|
|
28
|
+
and hosted tracking APIs smooth that over for a fee while seeing every number you look up.
|
|
29
|
+
Here the same work is open code: one adapter per carrier, each in [its own folder](carriers/)
|
|
30
|
+
with a README on how that site is read.
|
|
21
31
|
|
|
22
|
-
##
|
|
32
|
+
## Try it
|
|
23
33
|
|
|
24
|
-
Node.js 24 or newer
|
|
34
|
+
You need Node.js 24 or newer.
|
|
25
35
|
|
|
26
36
|
```sh
|
|
27
|
-
|
|
28
|
-
npx parcel-scraper detect 1Z999AA10123456784
|
|
29
|
-
npx parcel-scraper track YOUR_TRACKING_NUMBER --carrier ups
|
|
37
|
+
npx universal-parcel-scraper detect 1Z999AA10123456784
|
|
30
38
|
```
|
|
31
39
|
|
|
32
|
-
|
|
33
|
-
|
|
40
|
+
Detection is offline. It checks the number against each carrier's formats and checksums and
|
|
41
|
+
answers `ups` without sending anything anywhere.
|
|
34
42
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
number: process.env.PARCEL_NUMBER,
|
|
38
|
-
carrier: 'ups',
|
|
39
|
-
});
|
|
40
|
-
console.log(carrier, source, result.current_stage, result.events);
|
|
43
|
+
```sh
|
|
44
|
+
npx universal-parcel-scraper track YOUR_TRACKING_NUMBER
|
|
41
45
|
```
|
|
42
46
|
|
|
43
|
-
|
|
47
|
+
Tracking does go out, to the carrier that owns the number. If a number fits several carriers,
|
|
48
|
+
name one with `--carrier dhl`. A few carriers also want the delivery postcode (`--postcode`)
|
|
49
|
+
or the link from the shipping email (`--tracking-url`).
|
|
44
50
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
51
|
+
## Ways to run it
|
|
52
|
+
|
|
53
|
+
### Command line
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
npm install -g universal-parcel-scraper
|
|
48
57
|
```
|
|
49
58
|
|
|
50
|
-
|
|
51
|
-
optional [TRAWL browser service](trawl/README.md). Install the transports you need:
|
|
59
|
+
The command is `parcel-scraper`:
|
|
52
60
|
|
|
53
61
|
```sh
|
|
54
|
-
|
|
62
|
+
parcel-scraper detect <number, link or text> # which carrier is this? no network
|
|
63
|
+
parcel-scraper track <number> # the parcel's history
|
|
64
|
+
parcel-scraper recognize <number> # ask the possible carriers which one knows it
|
|
65
|
+
parcel-scraper carriers # the catalog, with the ids --carrier accepts
|
|
66
|
+
parcel-scraper serve # the HTTP API described below
|
|
55
67
|
```
|
|
56
68
|
|
|
57
|
-
|
|
58
|
-
`createTracker({ providers: ['ParcelsApp', 'Ship24', '17TRACK', 'Postal Ninja', 'UPU'] })`.
|
|
59
|
-
The default is direct adapters plus UPU for eligible postal numbers.
|
|
69
|
+
Everything prints JSON, so it pipes:
|
|
60
70
|
|
|
61
|
-
|
|
71
|
+
```sh
|
|
72
|
+
parcel-scraper track "$NUMBER" | jq -r '.result.current_stage'
|
|
73
|
+
```
|
|
62
74
|
|
|
63
|
-
|
|
64
|
-
actually returned. Here is the recorded comparison on our **100-carrier reference set**:
|
|
75
|
+
`parcel-scraper --help` lists the environment variables.
|
|
65
76
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
+
### Node
|
|
78
|
+
|
|
79
|
+
```js
|
|
80
|
+
import { createTracker } from 'universal-parcel-scraper/node';
|
|
81
|
+
|
|
82
|
+
const tracker = createTracker();
|
|
83
|
+
const { carrier, source, result } = await tracker.track({ number: process.env.PARCEL_NUMBER });
|
|
84
|
+
|
|
85
|
+
console.log(carrier, result.current_stage);
|
|
86
|
+
for (const scan of result.events) {
|
|
87
|
+
console.log(scan.time, scan.location, scan.description, scan.stage);
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Each scan keeps the carrier's wording and gains a `stage` from [one shared list](data/stages.json).
|
|
92
|
+
`source` names who answered, and `attempts` lists everything that was tried. When nothing
|
|
93
|
+
returns a history, `track()` throws a `TrackingError` carrying those attempts and a hint about
|
|
94
|
+
the cause. [examples/node.mjs](examples/node.mjs) is a runnable version.
|
|
95
|
+
|
|
96
|
+
### Browser
|
|
97
|
+
|
|
98
|
+
```js
|
|
99
|
+
import { parseTrackingInput } from 'universal-parcel-scraper';
|
|
100
|
+
|
|
101
|
+
parseTrackingInput('1Z999AA10123456784');
|
|
102
|
+
// { carrier: 'ups', confidence: 'high', candidates: ['ups'], … }
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The root import has no Node imports and makes no requests, so it can sit behind a form field
|
|
106
|
+
and name the carrier as someone types.
|
|
77
107
|
|
|
78
|
-
|
|
79
|
-
the comparison reference of each carrier, including partial histories; alternate samples
|
|
80
|
-
are excluded. This curated set is not a global market-share ranking or a live availability
|
|
81
|
-
promise. Old parcels expire, formats differ, and browser challenges change.
|
|
82
|
-
[Full results and limitations](providers/COVERAGE.md).
|
|
108
|
+
### HTTP
|
|
83
109
|
|
|
84
|
-
|
|
85
|
-
gives you the retrieval code, control over which sources receive a number, and no per-lookup
|
|
86
|
-
subscription. Catalog totals from vendors are not comparable to this reference test.
|
|
110
|
+
For anything that is not JavaScript:
|
|
87
111
|
|
|
88
|
-
|
|
112
|
+
```sh
|
|
113
|
+
parcel-scraper serve --port 8080
|
|
114
|
+
```
|
|
89
115
|
|
|
90
116
|
```sh
|
|
91
|
-
npx parcel-scraper serve --port 8080
|
|
92
117
|
curl http://127.0.0.1:8080/v1/track \
|
|
93
118
|
-H 'Content-Type: application/json' \
|
|
94
119
|
-d '{"number":"1Z999AA10123456784","carrier":"ups"}'
|
|
95
120
|
```
|
|
96
121
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
122
|
+
`/v1/detect`, `/v1/recognize` and `/v1/carriers` sit next to it, and the
|
|
123
|
+
[OpenAPI file](server/openapi.json) describes them all. The server caches recent answers in
|
|
124
|
+
memory and rate-limits callers. It has no database and never polls on its own.
|
|
125
|
+
|
|
126
|
+
Set `SCRAPER_TOKEN` to require a bearer token. `SCRAPER_DEMO_PAGE=true` serves a small page
|
|
127
|
+
at `/` for trying numbers by hand. The other settings are in [.env.example](.env.example).
|
|
101
128
|
|
|
102
|
-
|
|
129
|
+
### Docker
|
|
103
130
|
|
|
104
131
|
```sh
|
|
105
132
|
docker run --rm -p 127.0.0.1:8080:8080 ghcr.io/plhery/universal-parcel-scraper:0.2.0
|
|
106
133
|
```
|
|
107
134
|
|
|
108
|
-
|
|
109
|
-
|
|
135
|
+
The image runs `serve` and ships Chromium, for the carriers that only answer a real browser.
|
|
136
|
+
|
|
137
|
+
## What you can build with it
|
|
138
|
+
|
|
139
|
+
- **A tracking screen in your own app.** [Delivery Tracker](https://github.com/plhery/delivery-tracker),
|
|
140
|
+
an open-source iPhone app and PWA, runs on this package.
|
|
141
|
+
- **A Home Assistant sensor** for the parcel you are waiting on.
|
|
142
|
+
[The config is in examples](examples/home-assistant.yaml).
|
|
143
|
+
- **Order status inside a shop or help desk**, so customers are not sent off to the carrier's
|
|
144
|
+
site. Any backend that speaks HTTP can call the server.
|
|
145
|
+
- **Tracking numbers pulled out of shipping emails.** `detect` takes pasted text and carrier
|
|
146
|
+
links as well as bare numbers.
|
|
147
|
+
- **A carrier field that fills itself in**, with the browser import.
|
|
148
|
+
- **A cron job** that pings you when `current_stage` turns `delivered`.
|
|
110
149
|
|
|
111
|
-
|
|
150
|
+
The library answers one lookup at a time. Storing parcels and deciding when to check again
|
|
151
|
+
are yours to do.
|
|
112
152
|
|
|
113
|
-
|
|
114
|
-
carrier. Enabled commercial fallbacks receive the number; ParcelsApp also receives a supplied
|
|
115
|
-
postcode. A configured TRAWL instance handles the requested pages. Browser pages can load the
|
|
116
|
-
carrier's or provider's challenge resources. The library has no telemetry destination;
|
|
117
|
-
HTTP logs contain route names and outcomes, never parcel inputs.
|
|
153
|
+
## How it works
|
|
118
154
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
155
|
+
<picture>
|
|
156
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/how-it-works-dark.svg">
|
|
157
|
+
<img src="docs/assets/how-it-works-light.svg" width="840" alt="An input is detected offline and tracked by the carrier's dedicated adapter. With no history, the lookup moves to the fallbacks you enabled. Either way the result is one timeline.">
|
|
158
|
+
</picture>
|
|
159
|
+
|
|
160
|
+
Each adapter uses plain HTTP wherever the carrier's site allows it. Some sites only answer a
|
|
161
|
+
real browser and others need image processing, so those adapters can drive a local Chromium
|
|
162
|
+
or the optional [TRAWL browser service](trawl/README.md). Install what your carriers need:
|
|
163
|
+
|
|
164
|
+
```sh
|
|
165
|
+
npm install playwright-core sharp onnxruntime-web
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Then pass `chromiumPath` or `trawlUrl` to `createTracker()`. The CLI and the server read
|
|
169
|
+
`TRACKING_CHROMIUM_PATH` and `FLARESOLVERR_URL`.
|
|
170
|
+
|
|
171
|
+
When the adapter finds no history, the lookup can move on to an aggregator. Only UPU is on
|
|
172
|
+
by default, for the postal numbers it can serve. The commercial ones are opt-in, and each
|
|
173
|
+
one you enable receives the tracking number:
|
|
174
|
+
|
|
175
|
+
```js
|
|
176
|
+
createTracker({ providers: ['ParcelsApp', 'Ship24', '17TRACK', 'Postal Ninja', 'UPU'] });
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
[ARCHITECTURE.md](ARCHITECTURE.md) has the rest.
|
|
180
|
+
|
|
181
|
+
## How much can it track?
|
|
182
|
+
|
|
183
|
+
A carrier count says what the catalog knows about. Whether a real parcel comes back with its
|
|
184
|
+
history is a different question, so each source was run on its own against the same
|
|
185
|
+
**100-carrier reference set**. The top bar counts a carrier when any source below it returned
|
|
186
|
+
a history.
|
|
187
|
+
|
|
188
|
+
<!-- GENERATED:coverage -->
|
|
189
|
+
<picture>
|
|
190
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/coverage-dark.svg">
|
|
191
|
+
<img src="docs/assets/coverage-light.svg" alt="Carriers with tracking history: This project, all fallbacks enabled 70, This project, dedicated adapters alone 51, ParcelsApp 50, Postal Ninja 41, 17TRACK 40, Ship24 38, UPU 11." width="760">
|
|
192
|
+
</picture>
|
|
193
|
+
|
|
194
|
+
<details>
|
|
195
|
+
<summary>The same numbers as a table</summary>
|
|
196
|
+
|
|
197
|
+
| Source | Carriers with history |
|
|
198
|
+
| --- | ---: |
|
|
199
|
+
| **Universal Parcel Scraper, all fallbacks enabled** | **70 / 100** |
|
|
200
|
+
| Dedicated adapters alone | 51 / 100 |
|
|
201
|
+
| ParcelsApp | 50 / 100 |
|
|
202
|
+
| Postal Ninja | 41 / 100 |
|
|
203
|
+
| 17TRACK | 40 / 100 |
|
|
204
|
+
| Ship24 | 38 / 100 |
|
|
205
|
+
| UPU | 11 / 100 |
|
|
206
|
+
|
|
207
|
+
</details>
|
|
208
|
+
<!-- /GENERATED:coverage -->
|
|
122
209
|
|
|
123
|
-
The
|
|
124
|
-
|
|
210
|
+
The numbers come from [coverage.json](providers/coverage.json): the recorded outcome for each
|
|
211
|
+
carrier's comparison reference, partial histories included and alternate samples left out.
|
|
212
|
+
The set is curated. Read it as a comparison between sources, not as a market-share ranking
|
|
213
|
+
or a promise about today, because parcels expire and carrier sites change.
|
|
214
|
+
[Carrier-by-carrier results and limitations](providers/COVERAGE.md).
|
|
215
|
+
|
|
216
|
+
## Next to a hosted tracking API
|
|
217
|
+
|
|
218
|
+
| | Universal Parcel Scraper | Hosted tracking API |
|
|
219
|
+
| --- | --- | --- |
|
|
220
|
+
| Account | None | Sign-up and an API key |
|
|
221
|
+
| Cost | Your own compute | A plan or a per-shipment price |
|
|
222
|
+
| Who sees the tracking number | The carrier, plus any fallback you enable | The vendor, then the carrier |
|
|
223
|
+
| Carriers | The [catalog](carriers/) in this repository | A much larger catalog |
|
|
224
|
+
| Updates | You poll | Webhooks |
|
|
225
|
+
| When a carrier changes its site | The adapter breaks until it is fixed here | The vendor deals with it |
|
|
226
|
+
| Hosting | Yours | Theirs |
|
|
227
|
+
|
|
228
|
+
If you want the widest catalog and nothing to run, a hosted API is the better choice. This
|
|
229
|
+
project is for when the numbers should stay on your side, or when a price per parcel makes
|
|
230
|
+
no sense for what you are building. Vendor catalog totals are not comparable with the
|
|
231
|
+
reference test above.
|
|
232
|
+
|
|
233
|
+
## Privacy and limits
|
|
234
|
+
|
|
235
|
+
A lookup sends the tracking number to the carrier, along with the postcode or tracking link
|
|
236
|
+
when that carrier requires one. Fallbacks you enabled receive the number too, and ParcelsApp
|
|
237
|
+
also gets a postcode if you supplied it. Pages opened in a browser, locally or through TRAWL,
|
|
238
|
+
can load the carrier's or provider's challenge scripts. The library has no telemetry, and
|
|
239
|
+
the server's logs hold route names and outcomes, never parcel inputs.
|
|
240
|
+
|
|
241
|
+
This is scraping, so things break. Carriers redesign their sites and turn away automated
|
|
242
|
+
requests. A few need more than the number, such as a postcode or the link from the shipping
|
|
243
|
+
email, and the ones that require an account stay limited. Keep to the catalog's refresh
|
|
244
|
+
limits and to the retry advice that comes with an error. A scan time with no known UTC offset
|
|
245
|
+
is returned as the carrier wrote it, with no zone guessed.
|
|
246
|
+
|
|
247
|
+
## Contributing
|
|
248
|
+
|
|
249
|
+
Adapters break when carriers change their sites. Each carrier lives in its own folder with
|
|
250
|
+
synthetic fixtures and offline tests, so a fix stays local to that folder. For a new carrier,
|
|
251
|
+
`npm run carrier:new` scaffolds the folder and [CONTRIBUTING.md](CONTRIBUTING.md) covers the
|
|
252
|
+
rest. Report security issues through [SECURITY.md](SECURITY.md).
|
|
253
|
+
|
|
254
|
+
## License
|
|
255
|
+
|
|
256
|
+
The core is [Apache-2.0](LICENSE). The optional TRAWL image is a derivative under
|
|
257
|
+
[AGPL-3.0](trawl/LICENSE). Data and model credits are in [NOTICE](NOTICE).
|
package/data/carrier.schema.json
CHANGED
|
@@ -205,7 +205,7 @@
|
|
|
205
205
|
"minLength": 1
|
|
206
206
|
},
|
|
207
207
|
"canaryUrl": {
|
|
208
|
-
"description": "Public, credential-free HTTPS URL the
|
|
208
|
+
"description": "Public, credential-free HTTPS URL the weekly canary probes. Required for automatic carriers.",
|
|
209
209
|
"type": "string"
|
|
210
210
|
},
|
|
211
211
|
"shows": {
|
|
Binary file
|
|
@@ -136,10 +136,12 @@ export class DachserTracker {
|
|
|
136
136
|
catch {
|
|
137
137
|
throw new UpstreamHttpError('Dachser tracking', response.status);
|
|
138
138
|
}
|
|
139
|
-
// An unknown shipment/access tuple
|
|
140
|
-
//
|
|
141
|
-
//
|
|
142
|
-
//
|
|
139
|
+
// An unknown shipment/access tuple reaches a null result in Dachser's
|
|
140
|
+
// public endpoint and surfaces as a JSON 500 whose message names it.
|
|
141
|
+
// ERR_APP_500 is the endpoint's catch-all for any unhandled exception and
|
|
142
|
+
// the message is the only field that says which one, so the `message: null`
|
|
143
|
+
// reply the same tuple also gets stays an operational failure, like every
|
|
144
|
+
// unrelated 500, rather than a false not-found.
|
|
143
145
|
const errorMessage = plainText(isRecord(errorPayload) ? errorPayload.message ?? '' : '');
|
|
144
146
|
if (response.status === 500
|
|
145
147
|
&& isRecord(errorPayload)
|
|
@@ -797,6 +797,8 @@ export const adapter = (environment) => {
|
|
|
797
797
|
fetcher: environment.fetcher,
|
|
798
798
|
trawl: environment.trawl,
|
|
799
799
|
recorder: environment.recorder,
|
|
800
|
+
// A host can follow a rotated key without waiting for a release.
|
|
801
|
+
firebaseApiKey: environment.env.DPD_FIREBASE_API_KEY?.trim() || undefined,
|
|
800
802
|
});
|
|
801
803
|
return {
|
|
802
804
|
id: 'dpd',
|
|
@@ -51,8 +51,10 @@ export class MrwTracker {
|
|
|
51
51
|
const deadline = performance.now() + remainingMs;
|
|
52
52
|
const jar = new Map();
|
|
53
53
|
let requests = 0;
|
|
54
|
+
// The step's signal ends at the budget too, and its timer can fire just before the deadline reads as passed.
|
|
55
|
+
const spent = () => performance.now() >= deadline || (signal.aborted && !context.signal?.aborted);
|
|
54
56
|
const ensureTime = () => {
|
|
55
|
-
if (
|
|
57
|
+
if (spent())
|
|
56
58
|
throw new BudgetExceededError('MRW', budgetMs);
|
|
57
59
|
signal.throwIfAborted();
|
|
58
60
|
};
|
|
@@ -72,7 +74,7 @@ export class MrwTracker {
|
|
|
72
74
|
}, this.options.paceMs ?? PACE_MS);
|
|
73
75
|
signal.addEventListener('abort', abort, { once: true });
|
|
74
76
|
}).catch(error => {
|
|
75
|
-
if (
|
|
77
|
+
if (spent())
|
|
76
78
|
throw new BudgetExceededError('MRW', budgetMs, { cause: error });
|
|
77
79
|
throw error;
|
|
78
80
|
});
|
package/dist/cli/index.d.ts
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
export declare function main(argv?: string[], env?: NodeJS.ProcessEnv): Promise<number>;
|
|
3
|
+
/** What a failed command prints: a rejected input, setting or listener explains itself; anything else stays generic. */
|
|
4
|
+
export declare function failureText(error: unknown): string;
|
package/dist/cli/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { pathToFileURL } from 'node:url';
|
|
3
3
|
import { realpathSync } from 'node:fs';
|
|
4
|
+
import { InputRequiredError } from '../core/errors/index.js';
|
|
4
5
|
import { createTracker, TrackingError } from '../facade/index.js';
|
|
5
6
|
import { CARRIER_CATALOG } from '../generated/catalog.js';
|
|
6
7
|
import { createTrackingServer } from '../server/index.js';
|
|
@@ -21,6 +22,8 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
|
|
|
21
22
|
console.log(help);
|
|
22
23
|
return 0;
|
|
23
24
|
}
|
|
25
|
+
if (!['detect', 'recognize', 'track', 'carriers', 'serve'].includes(command))
|
|
26
|
+
throw new TypeError('Unknown command; use --help');
|
|
24
27
|
const values = {};
|
|
25
28
|
const positional = [];
|
|
26
29
|
const allowed = command === 'track' ? ['carrier', 'postcode', 'tracking-url'] : command === 'serve' ? ['host', 'port'] : [];
|
|
@@ -32,19 +35,30 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
|
|
|
32
35
|
}
|
|
33
36
|
const name = argument.slice(2);
|
|
34
37
|
if (!allowed.includes(name) || !args[i + 1] || args[i + 1].startsWith('--'))
|
|
35
|
-
throw new TypeError('Unknown or incomplete command option');
|
|
38
|
+
throw new TypeError('Unknown or incomplete command option; use --help');
|
|
36
39
|
values[name] = args[++i];
|
|
37
40
|
}
|
|
38
|
-
const providers = env.SCRAPER_PROVIDERS === undefined ? undefined
|
|
39
|
-
: env.SCRAPER_PROVIDERS.split(',').map(value => value.trim()).filter(Boolean);
|
|
40
|
-
const options = { providers, trawlUrl: env.FLARESOLVERR_URL, chromiumPath: env.TRACKING_CHROMIUM_PATH, env };
|
|
41
|
-
const tracker = createTracker(options);
|
|
42
41
|
const print = (value) => console.log(JSON.stringify(value, null, 2));
|
|
43
|
-
|
|
42
|
+
const input = positional.join(' ');
|
|
43
|
+
if (command === 'carriers' || command === 'serve') {
|
|
44
|
+
if (input)
|
|
45
|
+
throw new TypeError('Unexpected argument; use --help');
|
|
46
|
+
}
|
|
47
|
+
else if (!input)
|
|
48
|
+
throw new TypeError('Supply a tracking input; use --help');
|
|
49
|
+
// The catalog and detection are offline: the transport settings below cannot break them.
|
|
50
|
+
if (command === 'carriers') {
|
|
44
51
|
print(CARRIER_CATALOG);
|
|
45
52
|
return 0;
|
|
46
53
|
}
|
|
47
|
-
if (command === '
|
|
54
|
+
if (command === 'detect') {
|
|
55
|
+
print(createTracker().detect(input));
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
const providers = env.SCRAPER_PROVIDERS === undefined ? undefined
|
|
59
|
+
: env.SCRAPER_PROVIDERS.split(',').map(value => value.trim()).filter(Boolean);
|
|
60
|
+
const options = { providers, trawlUrl: env.FLARESOLVERR_URL, chromiumPath: env.TRACKING_CHROMIUM_PATH, env };
|
|
61
|
+
if (command === 'serve') {
|
|
48
62
|
const port = Number(values.port ?? env.PORT ?? 8080);
|
|
49
63
|
if (!Number.isInteger(port) || port < 1 || port > 65535)
|
|
50
64
|
throw new TypeError('Invalid port');
|
|
@@ -57,26 +71,26 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
|
|
|
57
71
|
process.once(signal, () => server.close());
|
|
58
72
|
return 0;
|
|
59
73
|
}
|
|
60
|
-
const
|
|
61
|
-
if (
|
|
62
|
-
throw new TypeError('Supply a tracking input');
|
|
63
|
-
if (command === 'detect')
|
|
64
|
-
print(tracker.detect(input));
|
|
65
|
-
else if (command === 'recognize')
|
|
74
|
+
const tracker = createTracker(options);
|
|
75
|
+
if (command === 'recognize')
|
|
66
76
|
print(await tracker.recognize(input));
|
|
67
|
-
else if (command === 'track')
|
|
68
|
-
print(await tracker.track({ number: input, carrier: values.carrier, postcode: values.postcode, trackingUrl: values['tracking-url'] }));
|
|
69
77
|
else
|
|
70
|
-
|
|
78
|
+
print(await tracker.track({ number: input, carrier: values.carrier, postcode: values.postcode, trackingUrl: values['tracking-url'] }));
|
|
71
79
|
return 0;
|
|
72
80
|
}
|
|
81
|
+
/** What a failed command prints: a rejected input, setting or listener explains itself; anything else stays generic. */
|
|
82
|
+
export function failureText(error) {
|
|
83
|
+
if (error instanceof TrackingError)
|
|
84
|
+
return JSON.stringify({ error: error.message, attempts: error.attempts, hint: error.hint });
|
|
85
|
+
return error instanceof TypeError || error instanceof RangeError || error instanceof InputRequiredError
|
|
86
|
+
|| (error instanceof Error && 'syscall' in error) ? error.message : 'Tracking could not be completed';
|
|
87
|
+
}
|
|
73
88
|
if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) {
|
|
74
89
|
try {
|
|
75
90
|
process.exitCode = await main();
|
|
76
91
|
}
|
|
77
92
|
catch (error) {
|
|
78
|
-
console.error(
|
|
79
|
-
: error instanceof TypeError ? 'Invalid input or command; use --help' : 'Tracking could not be completed');
|
|
93
|
+
console.error(failureText(error));
|
|
80
94
|
process.exitCode = 1;
|
|
81
95
|
}
|
|
82
96
|
}
|
|
@@ -5,6 +5,7 @@ const PLANZER_ACCESS_KEY = /^[A-Za-z0-9_-]{32,256}$/;
|
|
|
5
5
|
const DACHSER_HOST = 'customeriberia.dachser.com';
|
|
6
6
|
const DACHSER_PAGE_PATH = '/customerarea/utilidades/seguimiento-publico/detalle';
|
|
7
7
|
const CAPABILITY_VALUE = /^[A-Za-z0-9_-]{4,256}$/;
|
|
8
|
+
// eslint-disable-next-line no-control-regex -- control characters are what this rejects
|
|
8
9
|
const CONTROL_CHARACTER = /[\x00-\x1f\x7f]/;
|
|
9
10
|
const ALLOWED_QUERY_KEYS = new Set([
|
|
10
11
|
'hash',
|
package/dist/facade/index.js
CHANGED
|
@@ -98,9 +98,12 @@ export function createTracker(options = {}) {
|
|
|
98
98
|
throw new TypeError('Invalid tracking number');
|
|
99
99
|
const number = normalizeTrackingNumber(raw);
|
|
100
100
|
const ms = budget(context.budgetMs, 10_000);
|
|
101
|
+
// The budget cancels what is still in flight and ends the wait: carriers that have not
|
|
102
|
+
// answered by then are reported as unanswered. Only the caller's own signal rejects.
|
|
101
103
|
const signal = context.signal ? AbortSignal.any([context.signal, AbortSignal.timeout(ms)]) : AbortSignal.timeout(ms);
|
|
102
|
-
const candidates = recognitionCandidates(number).filter(candidate => registry.for(candidate.carrier)?.recognize);
|
|
103
|
-
const
|
|
104
|
+
const candidates = recognitionCandidates(number).filter(candidate => typeof registry.for(candidate.carrier)?.recognize === 'function');
|
|
105
|
+
const ask = () => lookupSignals.run(signal, () => recognizeAll(candidates, carrier => registry.for(carrier).recognize(number, { signal, budgetMs: ms }), ms));
|
|
106
|
+
const outcomes = await (context.signal ? bounded(ask, context.signal) : ask());
|
|
104
107
|
return { ...settleRecognition(outcomes), asked: candidates.map(candidate => candidate.carrier),
|
|
105
108
|
unanswered: outcomes.filter(outcome => outcome.status === 'failed').map(outcome => outcome.carrier) };
|
|
106
109
|
}
|
|
@@ -127,8 +130,13 @@ export function createTracker(options = {}) {
|
|
|
127
130
|
throw new InputRequiredError('Tracking', 'carrier', 'Choose a carrier for this number');
|
|
128
131
|
}
|
|
129
132
|
catch (error) {
|
|
130
|
-
if (signal
|
|
133
|
+
if (context.signal?.aborted)
|
|
134
|
+
throw context.signal.reason;
|
|
135
|
+
if (error instanceof InputRequiredError)
|
|
131
136
|
throw error;
|
|
137
|
+
// Recognition used the whole budget: no source was asked, and that is the failure to report.
|
|
138
|
+
if (signal.aborted)
|
|
139
|
+
throw new TrackingError([], failureHint(new BudgetExceededError('Tracking', ms)));
|
|
132
140
|
}
|
|
133
141
|
}
|
|
134
142
|
if (input.postcode != null && typeof input.postcode !== 'string')
|
|
@@ -159,7 +167,7 @@ export function createTracker(options = {}) {
|
|
|
159
167
|
const adapter = registry.for(carrier);
|
|
160
168
|
const directInput = { number, postcode: fields.dpdPostcode, trackingUrl: fields.trackingUrl };
|
|
161
169
|
const directBudget = () => Math.min(remaining(), 60_000);
|
|
162
|
-
let result = adapter ? await attempt(carrier, () => adapter.steps.length > 1
|
|
170
|
+
let result = adapter ? await attempt(carrier, () => adapter.recordsSteps || adapter.steps.length > 1
|
|
163
171
|
? adapter.track(directInput, { signal, budgetMs: directBudget() })
|
|
164
172
|
: runSteps({ carrier, budgetMs: directBudget(), signal, recorder: environment.recorder }, [
|
|
165
173
|
{ id: adapter.steps[0] ?? 'direct', run: ({ signal: stepSignal, remainingMs }) => adapter.track(directInput, { signal: stepSignal, budgetMs: remainingMs }) },
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* opens the normal results page for its verified handle and captures the full
|
|
6
6
|
* `/track/get` response. Local Chromium without TRAWL retains compact widget
|
|
7
7
|
* capture. The main entry form has a separate verification gate.
|
|
8
|
-
* The provider is disabled by default;
|
|
9
|
-
*
|
|
8
|
+
* The provider is disabled by default; selecting `Postal Ninja` in `providers`
|
|
9
|
+
* puts it in the chain before 17TRACK.
|
|
10
10
|
*/
|
|
11
11
|
import { DateTime } from 'luxon';
|
|
12
12
|
import { ChallengeError, NoHistoryError, SchemaError } from '../../core/errors/index.js';
|
|
@@ -109,6 +109,7 @@ export class PostalNinjaTracker {
|
|
|
109
109
|
&& payload.track.state === 'NO_INFO' && !payload.inProgress) {
|
|
110
110
|
return new NoHistoryError(SOURCE, 'Postal Ninja has no available tracking history');
|
|
111
111
|
}
|
|
112
|
+
return undefined;
|
|
112
113
|
};
|
|
113
114
|
return runSteps({ carrier: SOURCE, budgetMs: timeoutMs, recorder: this.options.recorder }, [{
|
|
114
115
|
id: 'trawl', enabled: Boolean(this.options.trawl),
|
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
* Two tiers: `direct` is one signed anonymous JSON POST per lookup (see
|
|
6
6
|
* http.ts), `browser` loads the public tracking page in a local Chromium
|
|
7
7
|
* session and reads the same API response from it. The browser tier only runs
|
|
8
|
-
* when the direct tier failed for a reason a
|
|
9
|
-
* or a server outage is reported as it is, so
|
|
10
|
-
* amplified into a second request, and a number
|
|
11
|
-
* stays unknown to the page that asks the same
|
|
8
|
+
* when a Chromium is configured and the direct tier failed for a reason a
|
|
9
|
+
* browser can repair: a rate limit or a server outage is reported as it is, so
|
|
10
|
+
* the router's backoff is not amplified into a second request, and a number
|
|
11
|
+
* the aggregator does not know stays unknown to the page that asks the same
|
|
12
|
+
* API.
|
|
12
13
|
*/
|
|
13
14
|
import { DateTime } from 'luxon';
|
|
14
15
|
import { carrierTimezone } from '../../core/catalog/index.js';
|
|
@@ -114,6 +115,9 @@ export class Ship24Tracker {
|
|
|
114
115
|
},
|
|
115
116
|
{
|
|
116
117
|
id: 'browser',
|
|
118
|
+
// Without a Chromium this tier could only replace the direct failure
|
|
119
|
+
// with its own configuration error. Alone, it still runs to report it.
|
|
120
|
+
enabled: http === null || Boolean(this.options.executablePath),
|
|
117
121
|
recovers: browserCanRecover,
|
|
118
122
|
run: async ({ remainingMs }) => ({
|
|
119
123
|
...await scrapeUniversalPage({ executablePath: this.options.executablePath, timeoutMs: Math.max(1, Math.floor(remainingMs)) }, {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "universal-parcel-scraper",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1-main.296",
|
|
4
4
|
"description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -27,15 +27,15 @@
|
|
|
27
27
|
"exports": {
|
|
28
28
|
".": {
|
|
29
29
|
"types": "./dist/index.d.ts",
|
|
30
|
-
"
|
|
30
|
+
"default": "./dist/index.js"
|
|
31
31
|
},
|
|
32
32
|
"./node": {
|
|
33
33
|
"types": "./dist/node.d.ts",
|
|
34
|
-
"
|
|
34
|
+
"default": "./dist/node.js"
|
|
35
35
|
},
|
|
36
36
|
"./places": {
|
|
37
37
|
"types": "./dist/places/index.d.ts",
|
|
38
|
-
"
|
|
38
|
+
"default": "./dist/places/index.js"
|
|
39
39
|
},
|
|
40
40
|
"./data/*": "./data/*"
|
|
41
41
|
},
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"build": "node scripts/build.mjs",
|
|
55
55
|
"prepare": "npm run build",
|
|
56
56
|
"typecheck": "tsc --noEmit",
|
|
57
|
-
"lint": "eslint . --max-warnings=0",
|
|
57
|
+
"lint": "eslint . --max-warnings=0 --pass-on-unpruned-suppressions",
|
|
58
58
|
"test": "vitest run",
|
|
59
59
|
"test:scripts": "node --test trawl/*.test.mjs scripts/*.test.mjs",
|
|
60
60
|
"generate": "node scripts/generate-catalog.mjs && node scripts/generate-registry.mjs --strict && node scripts/generate-region-towns.mjs && node scripts/coverage-tables.mjs && node scripts/generate-readme.mjs",
|
|
@@ -98,7 +98,9 @@
|
|
|
98
98
|
"@types/node": "^26.6.3",
|
|
99
99
|
"@types/luxon": "^3.7.5",
|
|
100
100
|
"ajv": "^8.20.0",
|
|
101
|
-
"eslint": "^
|
|
102
|
-
"@
|
|
101
|
+
"eslint": "^10.12.0",
|
|
102
|
+
"@eslint/js": "^10.0.1",
|
|
103
|
+
"typescript-eslint": "^8.71.0",
|
|
104
|
+
"globals": "^17.13.0"
|
|
103
105
|
}
|
|
104
106
|
}
|