universal-parcel-scraper 0.76.0-main.933 → 0.76.0-main.934
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -132,10 +132,10 @@ export declare function sharedDpdAppService(options?: Pick<AdapterEnvironment, '
|
|
|
132
132
|
* background, and keeps one open for the life of the process, so that no lookup waits the tens of
|
|
133
133
|
* seconds an opening takes. For a long-lived host, with the `fetcher` and `userAgent` it gives
|
|
134
134
|
* `createTracker` or its adapter environment. With a `store`, a restart takes the saved session up
|
|
135
|
-
* again, and the sessions no longer used are checked every hour
|
|
136
|
-
* which the store records. `onSession` hears of each session taken up and
|
|
137
|
-
* so the host can tell how long openings take and when one fails. The
|
|
138
|
-
* process alive.
|
|
135
|
+
* again, and the sessions no longer used are checked every hour, also across restarts, until the
|
|
136
|
+
* service refuses them, which the store records. `onSession` hears of each session taken up and
|
|
137
|
+
* each opening that ends, so the host can tell how long openings take and when one fails. The
|
|
138
|
+
* timers do not keep the process alive.
|
|
139
139
|
*/
|
|
140
140
|
export declare function warmDpdSession(environment?: Pick<AdapterEnvironment, 'fetcher' | 'userAgent'>, options?: {
|
|
141
141
|
store?: DpdSessionStore;
|
|
@@ -16,9 +16,9 @@ const MAX_BYTES = 2_000_000;
|
|
|
16
16
|
/** Opening a session has taken up to 75 seconds. */
|
|
17
17
|
const SESSION_OPEN_MS = 120_000;
|
|
18
18
|
/**
|
|
19
|
-
* The service has accepted a session for
|
|
20
|
-
* eight, and one left idle for four
|
|
21
|
-
* replaces it once open.
|
|
19
|
+
* The service has accepted a session for nearly twenty-seven hours, used once an hour after its
|
|
20
|
+
* first eight, and one left idle for four, and refused another from twenty-seven hours old. From
|
|
21
|
+
* eight hours old the next one opens beside it, and replaces it once open.
|
|
22
22
|
*/
|
|
23
23
|
const SESSION_RENEW_MS = 8 * 3_600_000;
|
|
24
24
|
/** A kept session that failed to open is tried again after this long. */
|
|
@@ -26,8 +26,9 @@ const SESSION_RETRY_MS = 15 * 60_000;
|
|
|
26
26
|
/** A host's store has this long to give back its sessions before the service opens one. */
|
|
27
27
|
const STORE_LOAD_MS = 5_000;
|
|
28
28
|
/**
|
|
29
|
-
* With a store, a session lookups no longer use is checked
|
|
30
|
-
*
|
|
29
|
+
* With a store, a session lookups no longer use is checked once it was last asked about this
|
|
30
|
+
* long ago, counting a check saved before a restart, until the service refuses it twice in a row:
|
|
31
|
+
* while slow, it can refuse a session it accepts again later.
|
|
31
32
|
*/
|
|
32
33
|
const CHECK_EVERY_MS = 3_600_000;
|
|
33
34
|
/** And for at most this long after it opened. */
|
|
@@ -224,7 +225,7 @@ export class DpdAppService {
|
|
|
224
225
|
else if (newest)
|
|
225
226
|
older.unshift(newest);
|
|
226
227
|
for (const session of older)
|
|
227
|
-
this.#retired.set(session.token, { session: { ...session } });
|
|
228
|
+
this.#retired.set(session.token, { session: { ...session }, askedAt: session.checkedAt ?? session.openedAt });
|
|
228
229
|
}
|
|
229
230
|
/** Saves a session's state in the host's store, if there is one, without waiting for it. */
|
|
230
231
|
#save(session) {
|
|
@@ -234,18 +235,22 @@ export class DpdAppService {
|
|
|
234
235
|
}
|
|
235
236
|
/** Follows a session lookups no longer use, when a store keeps what its checks find. */
|
|
236
237
|
#retire(session, refusedAt) {
|
|
237
|
-
if (this.#store && !this.#retired.has(session.token))
|
|
238
|
-
this.#retired.set(session.token, { session: { ...session }, refusedSince: refusedAt });
|
|
238
|
+
if (this.#store && !this.#retired.has(session.token)) {
|
|
239
|
+
this.#retired.set(session.token, { session: { ...session }, askedAt: this.#now(), refusedSince: refusedAt });
|
|
240
|
+
}
|
|
239
241
|
}
|
|
240
|
-
/** Checks the sessions lookups no longer use
|
|
242
|
+
/** Checks the sessions lookups no longer use when the first of them is due. */
|
|
241
243
|
#follow() {
|
|
242
|
-
|
|
244
|
+
let due = this.#now() + CHECK_EVERY_MS;
|
|
245
|
+
for (const retired of this.#retired.values())
|
|
246
|
+
due = Math.min(due, retired.askedAt + CHECK_EVERY_MS);
|
|
247
|
+
const timer = setTimeout(() => void this.#check().finally(() => this.#follow()), Math.max(0, due - this.#now()));
|
|
243
248
|
timer.unref?.();
|
|
244
249
|
}
|
|
245
250
|
/**
|
|
246
|
-
* Asks the service once for each followed session whether it still accepts it. A
|
|
247
|
-
* confirmed by the next check is saved as the time it began; a reply that is not about
|
|
248
|
-
* session says nothing.
|
|
251
|
+
* Asks the service once for each followed session that is due whether it still accepts it. A
|
|
252
|
+
* refusal confirmed by the next check is saved as the time it began; a reply that is not about
|
|
253
|
+
* the session says nothing.
|
|
249
254
|
*/
|
|
250
255
|
async #check() {
|
|
251
256
|
let first = true;
|
|
@@ -255,9 +260,12 @@ export class DpdAppService {
|
|
|
255
260
|
this.#retired.delete(token);
|
|
256
261
|
continue;
|
|
257
262
|
}
|
|
263
|
+
if (now - retired.askedAt < CHECK_EVERY_MS)
|
|
264
|
+
continue;
|
|
258
265
|
if (!first)
|
|
259
266
|
await new Promise((resolve) => { setTimeout(resolve, CHECK_GAP_MS).unref?.(); });
|
|
260
267
|
first = false;
|
|
268
|
+
retired.askedAt = this.#now();
|
|
261
269
|
let accepted;
|
|
262
270
|
try {
|
|
263
271
|
await this.call('getParcelShopByID', { SessionToken: token, ParcelShopID: '0', PudoID: CHECK_PUDO_ID, ParcelShopOnly: 'false' }, AbortSignal.timeout(CHECK_TIMEOUT_MS), CHECK_TIMEOUT_MS);
|
|
@@ -439,10 +447,10 @@ export function sharedDpdAppService(options = {}) {
|
|
|
439
447
|
* background, and keeps one open for the life of the process, so that no lookup waits the tens of
|
|
440
448
|
* seconds an opening takes. For a long-lived host, with the `fetcher` and `userAgent` it gives
|
|
441
449
|
* `createTracker` or its adapter environment. With a `store`, a restart takes the saved session up
|
|
442
|
-
* again, and the sessions no longer used are checked every hour
|
|
443
|
-
* which the store records. `onSession` hears of each session taken up and
|
|
444
|
-
* so the host can tell how long openings take and when one fails. The
|
|
445
|
-
* process alive.
|
|
450
|
+
* again, and the sessions no longer used are checked every hour, also across restarts, until the
|
|
451
|
+
* service refuses them, which the store records. `onSession` hears of each session taken up and
|
|
452
|
+
* each opening that ends, so the host can tell how long openings take and when one fails. The
|
|
453
|
+
* timers do not keep the process alive.
|
|
446
454
|
*/
|
|
447
455
|
export function warmDpdSession(environment = {}, options = {}) {
|
|
448
456
|
sharedDpdAppService(environment).keep(options);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "universal-parcel-scraper",
|
|
3
|
-
"version": "0.76.0-main.
|
|
3
|
+
"version": "0.76.0-main.934",
|
|
4
4
|
"description": "Self-hosted parcel tracking: carrier detection, dedicated scrapers and optional universal providers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|