pmtiles-swarm 0.72.0 → 0.72.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/package.json +1 -1
- package/src/api.js +10 -5
- package/src/auth.js +4 -1
- package/src/bake-jobs.js +2 -0
- package/src/catalog.js +1 -1
- package/src/config.js +5 -5
- package/src/cutlines.js +9 -2
- package/src/engines/composite.js +13 -2
- package/src/engines/libtorrent.js +6 -1
- package/src/engines/webtorrent.js +4 -0
- package/src/feed.js +1 -1
- package/src/hooks.js +8 -3
- package/src/incomplete.js +8 -2
- package/src/library.js +3 -0
- package/src/mutable.js +6 -0
- package/src/origin.js +0 -1
- package/src/pixels.js +5 -1
- package/src/pmtiles-probe.js +6 -0
- package/src/pmtiles-scan.js +2 -2
- package/src/pmtiles-write.js +16 -4
- package/src/prewarm.js +5 -1
- package/src/publisher.js +3 -1
- package/src/rate-limits.js +4 -1
- package/src/savepath.js +1 -0
- package/src/seeding.js +4 -1
- package/src/sources.js +12 -6
- package/src/stack-cache.js +2 -1
- package/src/stacks.js +0 -3
- package/src/subscriptions.js +2 -0
- package/src/tilejson.js +0 -1
- package/src/torrent-create.js +0 -1
- package/src/traffic-stats.js +8 -7
- package/src/watch.js +2 -1
- package/tools/tile-bench.mjs +18 -3
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,29 @@
|
|
|
7
7
|
### 🐞 Bug fixes
|
|
8
8
|
- _...Add new stuff here..._
|
|
9
9
|
|
|
10
|
+
## 0.72.1
|
|
11
|
+
### 🐞 Bug fixes
|
|
12
|
+
- **A settings save could set something that was not a setting.** `saveConfig` rejected a key it
|
|
13
|
+
did not recognise by asking `key in DEFAULTS`, and `in` walks the prototype chain: every object
|
|
14
|
+
inherits `__proto__`, `constructor` and `toString`, so all three passed as known settings. The
|
|
15
|
+
assignment underneath then did what those names mean rather than what a setting means -
|
|
16
|
+
`config.__proto__ = {...}` replaces the running config's prototype - and the result was written
|
|
17
|
+
to the config file. `Object.hasOwn` asks the question that was meant.
|
|
18
|
+
|
|
19
|
+
Reachable only by an admin token, which is a token that can change any setting anyway, so this
|
|
20
|
+
is a gate that was not doing its job rather than a way in. Found by reading through what
|
|
21
|
+
`security/detect-object-injection` had to say, which is what it is switched on for.
|
|
22
|
+
- **A date placeholder that was nearly right took exponentially long to reject.** `expandTemplate`
|
|
23
|
+
read a `{...}` group as a date pattern by testing it against `^[YMDymd]+([-_./ ]?[YMDymd]+)*$`.
|
|
24
|
+
With the separator optional, a run of field letters can be divided between the group and the `+`
|
|
25
|
+
in front of it in every possible way, so a group that is all field letters and one character that
|
|
26
|
+
cannot match has to try all of them before saying no: 24 characters took 100ms, and each two
|
|
27
|
+
after that doubled it.
|
|
28
|
+
|
|
29
|
+
Requiring the separator inside the group accepts exactly the same set of patterns - a run can
|
|
30
|
+
only be matched one way now - and rejects the near miss immediately. The template is config
|
|
31
|
+
rather than anything a stranger sends, so this was a source of surprise rather than a way in.
|
|
32
|
+
|
|
10
33
|
## 0.72.0
|
|
11
34
|
### ✨ Features and improvements
|
|
12
35
|
- **An unfinished export is picked up when the node starts.** The checkpoint was always there;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pmtiles-swarm",
|
|
3
|
-
"version": "0.72.
|
|
3
|
+
"version": "0.72.1",
|
|
4
4
|
"description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/api.js
CHANGED
|
@@ -84,10 +84,11 @@ function route(handler) {
|
|
|
84
84
|
* Progress has its own route already: `runningAdds()` feeds `/api/adds`, the
|
|
85
85
|
* console polls it, and `DELETE /api/adds` cancels the ones that can be.
|
|
86
86
|
* @param {object} res - The response to answer.
|
|
87
|
-
* @param {
|
|
88
|
-
* @param {
|
|
89
|
-
* @param {
|
|
90
|
-
* @param {string}
|
|
87
|
+
* @param {object} options - The add itself, and what to say about it.
|
|
88
|
+
* @param {Function} options.start - Called with `{onValidated}`; returns the add's promise.
|
|
89
|
+
* @param {object} options.accepted - Fields describing the source, for the 202 body.
|
|
90
|
+
* @param {string} options.message - What the 202 tells the caller is now happening.
|
|
91
|
+
* @param {string} options.what - Prefixed log tag and source, for a failure nobody is waiting on.
|
|
91
92
|
* @returns {Promise<void>} - Resolves once the response has been sent.
|
|
92
93
|
*/
|
|
93
94
|
async function acceptAdd(res, { start, accepted, message, what }) {
|
|
@@ -154,7 +155,6 @@ async function acceptAdd(res, { start, accepted, message, what }) {
|
|
|
154
155
|
* worse, because it makes the shape of the fragment depend on what happened to
|
|
155
156
|
* be available, so every reader has to handle both anyway. One shape,
|
|
156
157
|
* `URLSearchParams` reads it, and a magnet survives the round trip exactly.
|
|
157
|
-
*
|
|
158
158
|
* @param {string} url - The URL a style points at.
|
|
159
159
|
* @param {object} handles - `{torrent, magnet}`, either of which may be absent.
|
|
160
160
|
* @returns {string} - The URL, with a fragment if there is anything to put in one.
|
|
@@ -205,6 +205,11 @@ function sourceUrlFor(category, newest, base) {
|
|
|
205
205
|
return withSwarmHandles(url, { torrent, magnet });
|
|
206
206
|
}
|
|
207
207
|
|
|
208
|
+
/**
|
|
209
|
+
* Builds the HTTP surface over everything the node has already started.
|
|
210
|
+
* @param {object} parts - The node's live pieces.
|
|
211
|
+
* @returns {object} - The Express app.
|
|
212
|
+
*/
|
|
208
213
|
export function createApp({
|
|
209
214
|
library,
|
|
210
215
|
catalog,
|
package/src/auth.js
CHANGED
|
@@ -338,7 +338,10 @@ export function createAuth(config) {
|
|
|
338
338
|
/** Whether any credential is configured. */
|
|
339
339
|
enabled,
|
|
340
340
|
|
|
341
|
-
/**
|
|
341
|
+
/**
|
|
342
|
+
* Whether a password login is possible, as opposed to only a token.
|
|
343
|
+
* @returns {boolean} - True when a password or a hash is set.
|
|
344
|
+
*/
|
|
342
345
|
get passwordLoginEnabled() {
|
|
343
346
|
return Boolean(settings().password || settings().passwordHash);
|
|
344
347
|
},
|
package/src/bake-jobs.js
CHANGED
|
@@ -58,6 +58,7 @@ export function workDirFor(job, config = {}) {
|
|
|
58
58
|
/** A bake that has finished is kept this long, so the console can report it. */
|
|
59
59
|
const KEEP_FINISHED_MS = 10 * 60 * 1000;
|
|
60
60
|
|
|
61
|
+
/** The exports a node is running, and the ones it has just finished. */
|
|
61
62
|
export class BakeManager {
|
|
62
63
|
#library;
|
|
63
64
|
#tiles;
|
|
@@ -357,6 +358,7 @@ export class BakeManager {
|
|
|
357
358
|
* @param {string} workDir - Where the unfinished work lives.
|
|
358
359
|
* @param {string} destination - Where the archive goes.
|
|
359
360
|
* @param {string} format - The output format.
|
|
361
|
+
* @param {object} options - `signal` to stop early, and how often to checkpoint.
|
|
360
362
|
* @returns {Promise<void>} - Resolves when the file is written.
|
|
361
363
|
*/
|
|
362
364
|
async #merge(
|
package/src/catalog.js
CHANGED
|
@@ -8,7 +8,6 @@ import path from 'node:path';
|
|
|
8
8
|
* dozens, the whole thing is human-readable and hand-editable, and it avoids a
|
|
9
9
|
* native dependency in a project that already asks a lot of the install.
|
|
10
10
|
* Writes go through a temp file and a rename, so a crash cannot truncate it.
|
|
11
|
-
*
|
|
12
11
|
* @typedef {object} CatalogEntry
|
|
13
12
|
* @property {string} infoHash - Hex v1 infohash. The catalog's primary key.
|
|
14
13
|
* @property {string} name - Archive filename.
|
|
@@ -99,6 +98,7 @@ export function newerFirst(a, b) {
|
|
|
99
98
|
return String(b.createdAt ?? '').localeCompare(String(a.createdAt ?? ''));
|
|
100
99
|
}
|
|
101
100
|
|
|
101
|
+
/** What the node knows about every archive it holds. */
|
|
102
102
|
export class Catalog {
|
|
103
103
|
#file;
|
|
104
104
|
#entries = new Map();
|
package/src/config.js
CHANGED
|
@@ -538,16 +538,13 @@ function merge(base, override) {
|
|
|
538
538
|
// `libtorrent`, which would otherwise alter the defaults themselves.
|
|
539
539
|
const out = {};
|
|
540
540
|
for (const [key, value] of Object.entries(base ?? {})) {
|
|
541
|
-
// eslint-disable-next-line security/detect-object-injection -- keys come from DEFAULTS
|
|
542
541
|
out[key] = clone(value);
|
|
543
542
|
}
|
|
544
543
|
for (const [key, value] of Object.entries(override ?? {})) {
|
|
545
544
|
if (value === undefined) continue;
|
|
546
|
-
// eslint-disable-next-line security/detect-object-injection -- keys come from a config file the operator controls
|
|
547
545
|
out[key] =
|
|
548
546
|
value && typeof value === 'object' && !Array.isArray(value)
|
|
549
|
-
?
|
|
550
|
-
merge(base[key] ?? {}, value)
|
|
547
|
+
? merge(base[key] ?? {}, value)
|
|
551
548
|
: value;
|
|
552
549
|
}
|
|
553
550
|
return out;
|
|
@@ -926,7 +923,10 @@ export async function saveConfig(config, updates, configPath) {
|
|
|
926
923
|
);
|
|
927
924
|
|
|
928
925
|
for (const [key] of changing) {
|
|
929
|
-
|
|
926
|
+
// `in` would walk the prototype chain, which says yes to `__proto__` and
|
|
927
|
+
// to `constructor` — and the assignment below then swaps the config's
|
|
928
|
+
// prototype rather than setting a setting.
|
|
929
|
+
if (!Object.hasOwn(DEFAULTS, key)) {
|
|
930
930
|
throw new Error(`unknown setting: ${key}`);
|
|
931
931
|
}
|
|
932
932
|
if (guarded.has(key)) {
|
package/src/cutlines.js
CHANGED
|
@@ -14,6 +14,7 @@ import { fromBounds, fromGeoJSON } from './cutline.js';
|
|
|
14
14
|
/** Where they live under the data directory. */
|
|
15
15
|
const DIRECTORY = 'cutlines';
|
|
16
16
|
|
|
17
|
+
/** Loads the cutline files once and hands out prepared shapes. */
|
|
17
18
|
export class CutlineStore {
|
|
18
19
|
#dir;
|
|
19
20
|
#shapes = new Map();
|
|
@@ -26,7 +27,10 @@ export class CutlineStore {
|
|
|
26
27
|
this.#dir = path.join(dataDir ?? './data', DIRECTORY);
|
|
27
28
|
}
|
|
28
29
|
|
|
29
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* Where cutlines are read from.
|
|
32
|
+
* @returns {string} - The directory.
|
|
33
|
+
*/
|
|
30
34
|
get directory() {
|
|
31
35
|
return this.#dir;
|
|
32
36
|
}
|
|
@@ -56,7 +60,10 @@ export class CutlineStore {
|
|
|
56
60
|
}
|
|
57
61
|
}
|
|
58
62
|
|
|
59
|
-
/**
|
|
63
|
+
/**
|
|
64
|
+
* Every cutline that loaded.
|
|
65
|
+
* @returns {string[]} - Their names.
|
|
66
|
+
*/
|
|
60
67
|
list() {
|
|
61
68
|
return [...this.#shapes.keys()].sort();
|
|
62
69
|
}
|
package/src/engines/composite.js
CHANGED
|
@@ -53,12 +53,18 @@ export class CompositeEngine {
|
|
|
53
53
|
this.marksIncomplete = primary.marksIncomplete ?? false;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* The engine that owns the data.
|
|
58
|
+
* @returns {object} - The primary.
|
|
59
|
+
*/
|
|
57
60
|
get primary() {
|
|
58
61
|
return this.#primary;
|
|
59
62
|
}
|
|
60
63
|
|
|
61
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* The engines that only seed.
|
|
66
|
+
* @returns {object[]} - The secondaries.
|
|
67
|
+
*/
|
|
62
68
|
get secondaries() {
|
|
63
69
|
return this.#secondaries;
|
|
64
70
|
}
|
|
@@ -342,6 +348,10 @@ export class CompositeEngine {
|
|
|
342
348
|
return { ...(engines[0] ?? { state: 'unknown' }), engines };
|
|
343
349
|
}
|
|
344
350
|
|
|
351
|
+
/**
|
|
352
|
+
* Every torrent, gathered from every engine.
|
|
353
|
+
* @returns {Promise<object[]>} - Normalised torrents.
|
|
354
|
+
*/
|
|
345
355
|
async list() {
|
|
346
356
|
if (this.#stopping) return [];
|
|
347
357
|
const primary = await this.#primary.list();
|
|
@@ -490,6 +500,7 @@ export class CompositeEngine {
|
|
|
490
500
|
* present at all, since it is the only one that can produce a hybrid v1+v2
|
|
491
501
|
* torrent — and a hybrid serves v1 and v2 clients alike, so having it seed
|
|
492
502
|
* rather than lead is no reason to make a lesser torrent.
|
|
503
|
+
* @param {string} filePath - What to build it from.
|
|
493
504
|
* @param {object} options - Creation options.
|
|
494
505
|
* @returns {Promise<object>} - The created torrent.
|
|
495
506
|
*/
|
|
@@ -477,6 +477,10 @@ export class LibtorrentEngine {
|
|
|
477
477
|
}
|
|
478
478
|
}
|
|
479
479
|
|
|
480
|
+
/**
|
|
481
|
+
* Every torrent the sidecar holds.
|
|
482
|
+
* @returns {Promise<object[]>} - Normalised torrents.
|
|
483
|
+
*/
|
|
480
484
|
async list() {
|
|
481
485
|
// A node that is shutting down still has a console polling it and a sweep
|
|
482
486
|
// or two in flight. Answering "the sidecar exited" to each of them fills
|
|
@@ -759,6 +763,7 @@ export class LibtorrentEngine {
|
|
|
759
763
|
/**
|
|
760
764
|
* Persists resume data, so the next start skips re-hashing the store.
|
|
761
765
|
* @param {string} [infoHash] - One torrent, or all when omitted.
|
|
766
|
+
* @param {object} [options] - `timeoutMs` to override the default deadline.
|
|
762
767
|
* @returns {Promise<{written: number, asked: number}>} - How many torrents
|
|
763
768
|
* were told to write resume data, and how many actually did before the
|
|
764
769
|
* deadline.
|
|
@@ -807,7 +812,7 @@ export class LibtorrentEngine {
|
|
|
807
812
|
* @param {string} op - Operation name.
|
|
808
813
|
* @param {object} params - Operation parameters.
|
|
809
814
|
* @param {number} [timeoutMs] - How long to wait. Default 60s.
|
|
810
|
-
* @returns {Promise<
|
|
815
|
+
* @returns {Promise<unknown>} - Whatever the sidecar answered.
|
|
811
816
|
*/
|
|
812
817
|
async #call(op, params, timeoutMs = 60000) {
|
|
813
818
|
if (op !== 'shutdown') await this.connect();
|
|
@@ -357,6 +357,10 @@ export class WebTorrentSeedEngine {
|
|
|
357
357
|
};
|
|
358
358
|
}
|
|
359
359
|
|
|
360
|
+
/**
|
|
361
|
+
* Every torrent the client holds.
|
|
362
|
+
* @returns {Promise<object[]>} - Normalised torrents.
|
|
363
|
+
*/
|
|
360
364
|
async list() {
|
|
361
365
|
if (!this.#client) return [];
|
|
362
366
|
return this.#client.torrents.map((torrent) => this.#normalise(torrent));
|
package/src/feed.js
CHANGED
|
@@ -212,7 +212,6 @@ export function formatBytes(bytes) {
|
|
|
212
212
|
* has none to give: planetiler writes that section after every tile, so on a
|
|
213
213
|
* planet archive it is the very end of the file. An entry summarised from a
|
|
214
214
|
* feed therefore stays due for warming, which is what fills the rest in.
|
|
215
|
-
*
|
|
216
215
|
* @param {string} block - The item XML.
|
|
217
216
|
* @returns {object | undefined} - The summary, or undefined if absent.
|
|
218
217
|
*/
|
|
@@ -344,6 +343,7 @@ function isoDate(value) {
|
|
|
344
343
|
* @returns {string | undefined} - Decoded text, if found.
|
|
345
344
|
*/
|
|
346
345
|
function tag(block, name) {
|
|
346
|
+
// eslint-disable-next-line security/detect-non-literal-regexp -- name is one of this module's own element names
|
|
347
347
|
const pattern = new RegExp(`<${name}\\b[^>]*>([\\s\\S]*?)</${name}>`, 'i');
|
|
348
348
|
const match = pattern.exec(block);
|
|
349
349
|
if (!match) return undefined;
|
package/src/hooks.js
CHANGED
|
@@ -33,7 +33,6 @@ const TAIL_LINES = 20;
|
|
|
33
33
|
* %N name %I infohash %F content path
|
|
34
34
|
* %L category %G categories %D save path
|
|
35
35
|
* %Z size %C file count
|
|
36
|
-
*
|
|
37
36
|
* @param {string} argument - An argument possibly containing placeholders.
|
|
38
37
|
* @param {object} entry - The catalog entry that finished.
|
|
39
38
|
* @returns {string} - The argument with placeholders replaced.
|
|
@@ -96,7 +95,10 @@ export class ProgramHooks {
|
|
|
96
95
|
this.#config = config;
|
|
97
96
|
}
|
|
98
97
|
|
|
99
|
-
/**
|
|
98
|
+
/**
|
|
99
|
+
* Whether either command is configured.
|
|
100
|
+
* @returns {boolean} - True when armed.
|
|
101
|
+
*/
|
|
100
102
|
get enabled() {
|
|
101
103
|
return Boolean(
|
|
102
104
|
this.#config.onComplete?.command || this.#config.onAdded?.command,
|
|
@@ -129,7 +131,10 @@ export class ProgramHooks {
|
|
|
129
131
|
}
|
|
130
132
|
}
|
|
131
133
|
|
|
132
|
-
/**
|
|
134
|
+
/**
|
|
135
|
+
* Stops watching.
|
|
136
|
+
* @returns {void}
|
|
137
|
+
*/
|
|
133
138
|
stop() {
|
|
134
139
|
if (this.#timer) clearInterval(this.#timer);
|
|
135
140
|
this.#timer = undefined;
|
package/src/incomplete.js
CHANGED
|
@@ -147,7 +147,10 @@ export class CompletionWatcher {
|
|
|
147
147
|
this.#config = config;
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
-
/**
|
|
150
|
+
/**
|
|
151
|
+
* Whether marking is switched on at all.
|
|
152
|
+
* @returns {boolean} - True when armed.
|
|
153
|
+
*/
|
|
151
154
|
get enabled() {
|
|
152
155
|
return suffixFor(this.#config) !== '';
|
|
153
156
|
}
|
|
@@ -168,7 +171,10 @@ export class CompletionWatcher {
|
|
|
168
171
|
this.#timer.unref?.();
|
|
169
172
|
}
|
|
170
173
|
|
|
171
|
-
/**
|
|
174
|
+
/**
|
|
175
|
+
* Stops watching.
|
|
176
|
+
* @returns {void}
|
|
177
|
+
*/
|
|
172
178
|
stop() {
|
|
173
179
|
if (this.#timer) clearInterval(this.#timer);
|
|
174
180
|
this.#timer = undefined;
|
package/src/library.js
CHANGED
|
@@ -102,6 +102,7 @@ async function newestMtime(dir) {
|
|
|
102
102
|
return newest;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
+
/** The archives this node holds, and the engine seeding them. */
|
|
105
106
|
export class Library {
|
|
106
107
|
#catalog;
|
|
107
108
|
#engine;
|
|
@@ -1587,6 +1588,7 @@ export class Library {
|
|
|
1587
1588
|
* @param {string} magnet - Its magnet URI.
|
|
1588
1589
|
* @param {object} engine - The engine it came from.
|
|
1589
1590
|
* @param {object} options - Adopt options.
|
|
1591
|
+
* @param {boolean} [readable] - Whether the file can be opened from here.
|
|
1590
1592
|
* @returns {Promise<object>} - The catalog entry.
|
|
1591
1593
|
*/
|
|
1592
1594
|
async #adoptInPlace(torrent, magnet, engine, options, readable = true) {
|
|
@@ -2285,6 +2287,7 @@ export class Library {
|
|
|
2285
2287
|
* above has something to read while this is still running.
|
|
2286
2288
|
* @param {object[]} entries - Catalog entries, newest first.
|
|
2287
2289
|
* @param {{restored: number, failed: number}} tally - Mutated as it goes.
|
|
2290
|
+
* @param {object} handed - What the caller was handed, for the seeding check.
|
|
2288
2291
|
* @returns {Promise<{restored: number, failed: number}>} - That tally.
|
|
2289
2292
|
*/
|
|
2290
2293
|
async #restoreEach(entries, tally, handed) {
|
package/src/mutable.js
CHANGED
|
@@ -105,6 +105,12 @@ export function trackersFromMagnet(magnet) {
|
|
|
105
105
|
return new URLSearchParams(magnet.slice('magnet:?'.length)).getAll('tr');
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
+
/**
|
|
109
|
+
* A magnet URI for a mutable series.
|
|
110
|
+
* @param {string|Uint8Array} publicKey - The series key, hex or bytes.
|
|
111
|
+
* @param {object} [options] - `infoHash` for the current build, `trackers` to announce.
|
|
112
|
+
* @returns {string} - The magnet URI.
|
|
113
|
+
*/
|
|
108
114
|
export function mutableMagnet(publicKey, options = {}) {
|
|
109
115
|
// Buffer.from(string) would read hex as UTF-8 and produce a 64-byte key, so
|
|
110
116
|
// the two forms have to be told apart rather than coerced.
|
package/src/origin.js
CHANGED
|
@@ -190,7 +190,6 @@ async function contentLooksDifferent(entry) {
|
|
|
190
190
|
|
|
191
191
|
const fields = ['tileCount', 'minZoom', 'maxZoom', 'format', 'specVersion'];
|
|
192
192
|
for (const field of fields) {
|
|
193
|
-
// eslint-disable-next-line security/detect-object-injection -- field comes from the constant list above
|
|
194
193
|
if (stored[field] !== undefined && stored[field] !== fresh[field]) {
|
|
195
194
|
return true;
|
|
196
195
|
}
|
package/src/pixels.js
CHANGED
|
@@ -17,6 +17,7 @@ const here = path.dirname(fileURLToPath(import.meta.url));
|
|
|
17
17
|
/** How long to wait for a worker to answer before giving up on it. */
|
|
18
18
|
const REPLY_TIMEOUT_MS = 120000;
|
|
19
19
|
|
|
20
|
+
/** A pool of threads for the pixel work, so a merge never blocks the server. */
|
|
20
21
|
export class PixelWorker {
|
|
21
22
|
#workers = [];
|
|
22
23
|
#pending = new Map();
|
|
@@ -75,7 +76,10 @@ export class PixelWorker {
|
|
|
75
76
|
}
|
|
76
77
|
}
|
|
77
78
|
|
|
78
|
-
/**
|
|
79
|
+
/**
|
|
80
|
+
* How many threads this pool holds.
|
|
81
|
+
* @returns {number} - The count.
|
|
82
|
+
*/
|
|
79
83
|
get size() {
|
|
80
84
|
return this.#workers.length;
|
|
81
85
|
}
|
package/src/pmtiles-probe.js
CHANGED
|
@@ -108,6 +108,12 @@ export function customEncodingFactors(metadata) {
|
|
|
108
108
|
*/
|
|
109
109
|
export const SUMMARY_VERSION = 3;
|
|
110
110
|
|
|
111
|
+
/**
|
|
112
|
+
* What the catalog keeps about an archive, from its header and metadata.
|
|
113
|
+
* @param {object} header - A parsed PMTiles v3 header.
|
|
114
|
+
* @param {object} [metadata] - The archive's own metadata document.
|
|
115
|
+
* @returns {object} - The summary, at `SUMMARY_VERSION`.
|
|
116
|
+
*/
|
|
111
117
|
export function summarize(header, metadata = {}) {
|
|
112
118
|
const type = TILE_TYPES[header.tileType] ?? TILE_TYPES[0];
|
|
113
119
|
|
package/src/pmtiles-scan.js
CHANGED
|
@@ -125,7 +125,7 @@ async function openArchive(archive) {
|
|
|
125
125
|
* per-tile yield across a planet is the slowest part of the loop.
|
|
126
126
|
* @param {string|object} archive - Path to the `.pmtiles`, or a byte source.
|
|
127
127
|
* @param {object} [options] - `signal` to stop early.
|
|
128
|
-
* @
|
|
128
|
+
* @yields {number[]} - Chunks of ascending tile ids.
|
|
129
129
|
*/
|
|
130
130
|
export async function* scanTileIds(archive, options = {}) {
|
|
131
131
|
const { source, header, root, owned } = await openArchive(archive);
|
|
@@ -190,7 +190,7 @@ export async function* scanTileIds(archive, options = {}) {
|
|
|
190
190
|
* holding every tile id of every source in memory at once.
|
|
191
191
|
* @param {Array<string|object>} archives - Paths, or byte sources.
|
|
192
192
|
* @param {object} [options] - `signal` to stop early.
|
|
193
|
-
* @
|
|
193
|
+
* @yields {number} - Ascending tile ids, each once.
|
|
194
194
|
*/
|
|
195
195
|
export async function* unionOfTileIds(archives, options = {}) {
|
|
196
196
|
const readers = archives.map((archive) => ({
|
package/src/pmtiles-write.js
CHANGED
|
@@ -274,12 +274,18 @@ export class PMTilesWriter {
|
|
|
274
274
|
return new PMTilesWriter({ ...options, tempPath, handle });
|
|
275
275
|
}
|
|
276
276
|
|
|
277
|
-
/**
|
|
277
|
+
/**
|
|
278
|
+
* How many tiles have been offered, including duplicates.
|
|
279
|
+
* @returns {number} - The count.
|
|
280
|
+
*/
|
|
278
281
|
get addressedTiles() {
|
|
279
282
|
return this.#addressed;
|
|
280
283
|
}
|
|
281
284
|
|
|
282
|
-
/**
|
|
285
|
+
/**
|
|
286
|
+
* How many entries the directories will hold.
|
|
287
|
+
* @returns {number} - The count.
|
|
288
|
+
*/
|
|
283
289
|
get tileEntries() {
|
|
284
290
|
return this.#entries.length;
|
|
285
291
|
}
|
|
@@ -296,12 +302,18 @@ export class PMTilesWriter {
|
|
|
296
302
|
return this.#entries;
|
|
297
303
|
}
|
|
298
304
|
|
|
299
|
-
/**
|
|
305
|
+
/**
|
|
306
|
+
* Tile bytes buffered so far.
|
|
307
|
+
* @returns {number} - The byte count.
|
|
308
|
+
*/
|
|
300
309
|
get dataBytes() {
|
|
301
310
|
return this.#offset;
|
|
302
311
|
}
|
|
303
312
|
|
|
304
|
-
/**
|
|
313
|
+
/**
|
|
314
|
+
* Whether the tiles have arrived in order.
|
|
315
|
+
* @returns {boolean} - True while they have.
|
|
316
|
+
*/
|
|
305
317
|
get clustered() {
|
|
306
318
|
return this.#clustered;
|
|
307
319
|
}
|
package/src/prewarm.js
CHANGED
|
@@ -71,6 +71,7 @@ function tooEarly(error) {
|
|
|
71
71
|
return /metadata has not arrived|not held here/i.test(error?.message ?? '');
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
/** Pulls each archive's header and root directory before anything asks for a tile. */
|
|
74
75
|
export class HeadWarmer {
|
|
75
76
|
#tiles;
|
|
76
77
|
#catalog;
|
|
@@ -411,7 +412,10 @@ export class HeadWarmer {
|
|
|
411
412
|
this.#timer.unref?.();
|
|
412
413
|
}
|
|
413
414
|
|
|
414
|
-
/**
|
|
415
|
+
/**
|
|
416
|
+
* Stops warming.
|
|
417
|
+
* @returns {void}
|
|
418
|
+
*/
|
|
415
419
|
stop() {
|
|
416
420
|
if (this.#timer) clearInterval(this.#timer);
|
|
417
421
|
this.#timer = undefined;
|
package/src/publisher.js
CHANGED
|
@@ -68,6 +68,7 @@ const REROLL_AFTER = 2;
|
|
|
68
68
|
*/
|
|
69
69
|
const MINIMUM_NODES = 8;
|
|
70
70
|
|
|
71
|
+
/** Publishes the catalog under a BEP 46 key, and keeps it published. */
|
|
71
72
|
export class MutablePublisher {
|
|
72
73
|
#catalog;
|
|
73
74
|
#dht;
|
|
@@ -250,6 +251,7 @@ export class MutablePublisher {
|
|
|
250
251
|
/**
|
|
251
252
|
* Waits for the DHT to bootstrap, then publishes.
|
|
252
253
|
* @param {number} readyMs - How long to wait before going ahead regardless.
|
|
254
|
+
* @param {number} retryMs - How long to wait before trying again.
|
|
253
255
|
* @returns {Promise<void>} - Resolves once the first attempt is done.
|
|
254
256
|
*/
|
|
255
257
|
async #firstPublish(readyMs, retryMs) {
|
|
@@ -375,7 +377,7 @@ export class MutablePublisher {
|
|
|
375
377
|
* holding the original would save a table belonging to a socket that has
|
|
376
378
|
* been closed.
|
|
377
379
|
* @param {Function} save - Receives the live DHT.
|
|
378
|
-
* @returns {Promise
|
|
380
|
+
* @returns {Promise<unknown>} - Whatever `save` returns, or 0 with no socket.
|
|
379
381
|
*/
|
|
380
382
|
async saveTable(save) {
|
|
381
383
|
if (!this.#dht) return 0;
|
package/src/rate-limits.js
CHANGED
package/src/savepath.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
const MAX_SEGMENT = 120;
|
|
12
12
|
|
|
13
13
|
/** Reserved on Windows whatever the extension. */
|
|
14
|
+
// eslint-disable-next-line security/detect-unsafe-regex -- the alternatives share no prefix, so there is nothing to backtrack over
|
|
14
15
|
const RESERVED = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$/i;
|
|
15
16
|
|
|
16
17
|
/** Separators, control characters, and the ones Windows refuses. */
|
package/src/seeding.js
CHANGED
|
@@ -198,7 +198,10 @@ export class SeedingLimits {
|
|
|
198
198
|
this.#timer.unref?.();
|
|
199
199
|
}
|
|
200
200
|
|
|
201
|
-
/**
|
|
201
|
+
/**
|
|
202
|
+
* Stops the sweep.
|
|
203
|
+
* @returns {void}
|
|
204
|
+
*/
|
|
202
205
|
stop() {
|
|
203
206
|
if (this.#timer) clearInterval(this.#timer);
|
|
204
207
|
this.#timer = undefined;
|
package/src/sources.js
CHANGED
|
@@ -32,8 +32,13 @@ export function expandTemplate(template, date) {
|
|
|
32
32
|
|
|
33
33
|
return String(template).replace(/\{([^{}]*)\}/g, (whole, body) => {
|
|
34
34
|
// Separators an upstream might put between the fields. Anything else means
|
|
35
|
-
// this is not a date at all.
|
|
36
|
-
|
|
35
|
+
// this is not a date at all. The separator is required inside the group
|
|
36
|
+
// rather than optional: optional, a run of field letters can be divided
|
|
37
|
+
// between the group and the `+` in front of it in exponentially many ways,
|
|
38
|
+
// and something that is nearly a date but does not match takes that long
|
|
39
|
+
// to rule out.
|
|
40
|
+
// eslint-disable-next-line security/detect-unsafe-regex -- a run of field letters can only be divided one way now the separator is required
|
|
41
|
+
if (!/^[YMDymd]+(?:[-_./ ][YMDymd]+)*$/.test(body)) return whole;
|
|
37
42
|
|
|
38
43
|
return body.replace(/([YMDymd])\1*|[-_./ ]/g, (run) => {
|
|
39
44
|
const field = run[0].toLowerCase();
|
|
@@ -220,15 +225,16 @@ export class ScheduledSourceManager {
|
|
|
220
225
|
#running = false;
|
|
221
226
|
#lastRun = new Map();
|
|
222
227
|
|
|
228
|
+
/** Reads the clock. Injectable so a long import can be simulated. */
|
|
229
|
+
#now;
|
|
230
|
+
|
|
223
231
|
/**
|
|
224
232
|
* Creates the manager.
|
|
225
233
|
* @param {import('./library.js').Library} library - Where imports go.
|
|
226
234
|
* @param {import('./catalog.js').Catalog} catalog - Used to skip what we already have.
|
|
227
235
|
* @param {object} config - Resolved configuration.
|
|
236
|
+
* @param {object} [options] - `now` reads the clock.
|
|
228
237
|
*/
|
|
229
|
-
/** Reads the clock. Injectable so a long import can be simulated. */
|
|
230
|
-
#now;
|
|
231
|
-
|
|
232
238
|
constructor(library, catalog, config, { now = () => new Date() } = {}) {
|
|
233
239
|
this.#library = library;
|
|
234
240
|
this.#catalog = catalog;
|
|
@@ -558,6 +564,7 @@ export class ScheduledSourceManager {
|
|
|
558
564
|
let pattern = ARCHIVE_PATTERN;
|
|
559
565
|
if (source.match) {
|
|
560
566
|
try {
|
|
567
|
+
// eslint-disable-next-line security/detect-non-literal-regexp -- the pattern is the operator's own source config
|
|
561
568
|
pattern = new RegExp(source.match, 'i');
|
|
562
569
|
} catch (error) {
|
|
563
570
|
throw new Error(
|
|
@@ -629,7 +636,6 @@ export class ScheduledSourceManager {
|
|
|
629
636
|
*
|
|
630
637
|
* The dated file stays the real one either way, so it remains seedable under
|
|
631
638
|
* its own torrent while consumers reference a fixed path.
|
|
632
|
-
*
|
|
633
639
|
* @param {object} source - The source definition.
|
|
634
640
|
* @param {object} entry - The freshly imported entry.
|
|
635
641
|
* @returns {Promise<void>} - Resolves once linked, or logs and continues.
|
package/src/stack-cache.js
CHANGED
|
@@ -28,6 +28,7 @@ import path from 'node:path';
|
|
|
28
28
|
*/
|
|
29
29
|
const SHARD = 2;
|
|
30
30
|
|
|
31
|
+
/** Merged stack tiles kept on disk, bounded by total size. */
|
|
31
32
|
export class StackCache {
|
|
32
33
|
#dir;
|
|
33
34
|
#maxBytes;
|
|
@@ -177,7 +178,7 @@ export class StackCache {
|
|
|
177
178
|
* would issue its own reads to every source underneath.
|
|
178
179
|
* @param {string} key - What is being produced.
|
|
179
180
|
* @param {Function} work - Produces the tile.
|
|
180
|
-
* @returns {Promise<
|
|
181
|
+
* @returns {Promise<unknown>} - What `work` returned.
|
|
181
182
|
*/
|
|
182
183
|
async once(key, work) {
|
|
183
184
|
const running = this.#inFlight.get(key);
|
package/src/stacks.js
CHANGED
|
@@ -21,7 +21,6 @@ import { BLEND_MODES, isBlendMode } from './rgba.js';
|
|
|
21
21
|
* catalog entries, and serving a tile by handing back the bytes of whichever
|
|
22
22
|
* source has it. No pixel is decoded, which is what lets this ship before the
|
|
23
23
|
* codec question is settled.
|
|
24
|
-
*
|
|
25
24
|
* @typedef {object} StackSource
|
|
26
25
|
* @property {string} [category] - Resolve to the newest build in this category.
|
|
27
26
|
* @property {string} [archive] - Or pin one infohash. Exactly one of the two.
|
|
@@ -39,7 +38,6 @@ import { BLEND_MODES, isBlendMode } from './rgba.js';
|
|
|
39
38
|
* @property {number} [heightAdjustment] - Metres, added after masking.
|
|
40
39
|
* @property {number} [opacity] - 0-1, scales source alpha. RGBA only.
|
|
41
40
|
* @property {string} [blend] - Blend operator. RGBA only.
|
|
42
|
-
*
|
|
43
41
|
* @typedef {object} Stack
|
|
44
42
|
* @property {string} id - URL segment.
|
|
45
43
|
* @property {string} [title] - Shown in the console and in TileJSON.
|
|
@@ -304,7 +302,6 @@ export function resolveStack(stack, resolvers) {
|
|
|
304
302
|
* stopped.
|
|
305
303
|
* category An older build in a category with others. Removing it
|
|
306
304
|
* changes nothing the stack can see.
|
|
307
|
-
*
|
|
308
305
|
* @param {Stack[]} stacks - Every stack.
|
|
309
306
|
* @param {object} entry - The catalog entry about to go.
|
|
310
307
|
* @param {Function} categoryInfo - Name to `{ count, newest }` infohash.
|
package/src/subscriptions.js
CHANGED
|
@@ -141,6 +141,7 @@ export class SubscriptionManager {
|
|
|
141
141
|
// appetites, e.g. only taking Europe extracts.
|
|
142
142
|
if (
|
|
143
143
|
subscription.filter &&
|
|
144
|
+
// eslint-disable-next-line security/detect-non-literal-regexp -- the filter is this node's own subscription config
|
|
144
145
|
!new RegExp(subscription.filter, 'i').test(item.title)
|
|
145
146
|
) {
|
|
146
147
|
continue;
|
|
@@ -233,6 +234,7 @@ export class SubscriptionManager {
|
|
|
233
234
|
const archives = (document.archives ?? []).filter(
|
|
234
235
|
(archive) =>
|
|
235
236
|
!subscription.filter ||
|
|
237
|
+
// eslint-disable-next-line security/detect-non-literal-regexp -- as above
|
|
236
238
|
new RegExp(subscription.filter, 'i').test(archive.name ?? ''),
|
|
237
239
|
);
|
|
238
240
|
|
package/src/tilejson.js
CHANGED
|
@@ -9,7 +9,6 @@ import { TileStore } from './tiles.js';
|
|
|
9
9
|
* just a saving: a node in cache mode holds almost none of the archive, so
|
|
10
10
|
* reading the header to answer a TileJSON request would mean pulling pieces out
|
|
11
11
|
* of the swarm before a map has asked for a single tile.
|
|
12
|
-
*
|
|
13
12
|
* @see https://github.com/mapbox/tilejson-spec/tree/master/3.0.0
|
|
14
13
|
*/
|
|
15
14
|
|
package/src/torrent-create.js
CHANGED
|
@@ -247,7 +247,6 @@ export async function createTorrentFromUrl(url, options = {}) {
|
|
|
247
247
|
* Used to decide whether a file already under its final name is the archive
|
|
248
248
|
* being asked for. A HEAD is enough and costs nothing next to the alternative,
|
|
249
249
|
* which is transferring the whole thing a second time to find out.
|
|
250
|
-
*
|
|
251
250
|
* @param {string} url - The archive's URL.
|
|
252
251
|
* @param {AbortSignal} [signal] - Cancels the probe.
|
|
253
252
|
* @returns {Promise<number>} - Length, or 0.
|
package/src/traffic-stats.js
CHANGED
|
@@ -37,7 +37,6 @@ const PRUNE_EVERY_MS = 10 * 60 * 1000;
|
|
|
37
37
|
* A week of 15-second samples is 40,000 points and a chart a thousand pixels
|
|
38
38
|
* wide; sending all of them wastes the transfer and the browser's time to draw
|
|
39
39
|
* something no eye can resolve. Averaging into buckets keeps the shape.
|
|
40
|
-
*
|
|
41
40
|
* @param {number} from - Start of the window, unix seconds.
|
|
42
41
|
* @param {number} to - End of the window, unix seconds.
|
|
43
42
|
* @param {number} buckets - How many points are wanted.
|
|
@@ -116,7 +115,6 @@ export class TrafficStats {
|
|
|
116
115
|
* Zero rows are written as readily as busy ones: a gap in the series would
|
|
117
116
|
* be indistinguishable from the node having been switched off, and "this
|
|
118
117
|
* archive did nothing all week" is a real answer somebody wants.
|
|
119
|
-
*
|
|
120
118
|
* @returns {Promise<number>} - How many rows were written.
|
|
121
119
|
*/
|
|
122
120
|
async sample() {
|
|
@@ -163,7 +161,6 @@ export class TrafficStats {
|
|
|
163
161
|
* Averaged into buckets rather than returned raw — see bucketSeconds. The
|
|
164
162
|
* timestamp of a bucket is its start, so a caller plotting them gets evenly
|
|
165
163
|
* spaced points without having to know the sample interval.
|
|
166
|
-
*
|
|
167
164
|
* @param {object} [options] - Options.
|
|
168
165
|
* @param {string} [options.infoHash] - One archive, or every one summed.
|
|
169
166
|
* @param {number} [options.hours] - How far back to read.
|
|
@@ -226,7 +223,6 @@ export class TrafficStats {
|
|
|
226
223
|
* sampled over. Approximate by construction -- it assumes each sample held
|
|
227
224
|
* until the next -- and the right shape of approximate: it cannot drift from
|
|
228
225
|
* the graph above it, because it is the same numbers.
|
|
229
|
-
*
|
|
230
226
|
* @param {object} [options] - Options.
|
|
231
227
|
* @param {number} [options.hours] - How far back to read.
|
|
232
228
|
* @returns {object[]} - `{infoHash, down, up, samples}`, busiest first.
|
|
@@ -254,7 +250,10 @@ export class TrafficStats {
|
|
|
254
250
|
}));
|
|
255
251
|
}
|
|
256
252
|
|
|
257
|
-
/**
|
|
253
|
+
/**
|
|
254
|
+
* Starts sampling and pruning.
|
|
255
|
+
* @returns {void}
|
|
256
|
+
*/
|
|
258
257
|
start() {
|
|
259
258
|
if (this.#timer) return;
|
|
260
259
|
const tick = () =>
|
|
@@ -276,7 +275,10 @@ export class TrafficStats {
|
|
|
276
275
|
this.#pruneTimer.unref?.();
|
|
277
276
|
}
|
|
278
277
|
|
|
279
|
-
/**
|
|
278
|
+
/**
|
|
279
|
+
* Stops sampling.
|
|
280
|
+
* @returns {void}
|
|
281
|
+
*/
|
|
280
282
|
stop() {
|
|
281
283
|
if (this.#timer) clearInterval(this.#timer);
|
|
282
284
|
if (this.#pruneTimer) clearInterval(this.#pruneTimer);
|
|
@@ -292,7 +294,6 @@ export class TrafficStats {
|
|
|
292
294
|
* resume data, the DHT node cache. The config directory is the operator's:
|
|
293
295
|
* hand-edited, diffed, copied between nodes, and the thing you reach for when a
|
|
294
296
|
* node will not start. A database that grows on its own does not belong there.
|
|
295
|
-
*
|
|
296
297
|
* @param {object} config - Resolved configuration.
|
|
297
298
|
* @returns {Promise<object>} - An open DatabaseSync.
|
|
298
299
|
*/
|
package/src/watch.js
CHANGED
|
@@ -22,7 +22,6 @@ import { retains, retire } from './retention.js';
|
|
|
22
22
|
* `monthly-*.pmtiles` is what they mean. Every other character is escaped, so
|
|
23
23
|
* a name containing regex punctuation matches itself rather than becoming a
|
|
24
24
|
* pattern by accident.
|
|
25
|
-
*
|
|
26
25
|
* @param {string} pattern - A glob, matched against the basename.
|
|
27
26
|
* @returns {RegExp} - Anchored, case-insensitive.
|
|
28
27
|
*/
|
|
@@ -35,9 +34,11 @@ export function globToRegExp(pattern) {
|
|
|
35
34
|
return character.replace(/[.+^${}()|[\]\\]/g, '\\$&');
|
|
36
35
|
})
|
|
37
36
|
.join('');
|
|
37
|
+
// eslint-disable-next-line security/detect-non-literal-regexp -- body is the escaped glob built just above
|
|
38
38
|
return new RegExp(`^${body}$`, 'i');
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
/** Watches folders and imports the archives that appear in them. */
|
|
41
42
|
export class WatchManager {
|
|
42
43
|
#library;
|
|
43
44
|
#watchers = [];
|
package/tools/tile-bench.mjs
CHANGED
|
@@ -305,7 +305,12 @@ function twoModes(values) {
|
|
|
305
305
|
};
|
|
306
306
|
}
|
|
307
307
|
|
|
308
|
-
/**
|
|
308
|
+
/**
|
|
309
|
+
* A compact picture of where the latencies actually fell.
|
|
310
|
+
* @param {number[]} values - Latencies, in milliseconds.
|
|
311
|
+
* @param {number} [width] - Widest bar, in characters.
|
|
312
|
+
* @returns {string[]} - One line per bucket.
|
|
313
|
+
*/
|
|
309
314
|
function histogram(values, width = 34) {
|
|
310
315
|
if (values.length === 0) return [];
|
|
311
316
|
const top = Math.max(...values);
|
|
@@ -323,7 +328,12 @@ function histogram(values, width = 34) {
|
|
|
323
328
|
});
|
|
324
329
|
}
|
|
325
330
|
|
|
326
|
-
/**
|
|
331
|
+
/**
|
|
332
|
+
* The value at a percentile of a sorted copy.
|
|
333
|
+
* @param {number[]} values - Latencies, in milliseconds.
|
|
334
|
+
* @param {number} fraction - Where to look, 0 to 1.
|
|
335
|
+
* @returns {number} - The value there, or 0 when there are none.
|
|
336
|
+
*/
|
|
327
337
|
function percentile(values, fraction) {
|
|
328
338
|
if (values.length === 0) return 0;
|
|
329
339
|
const sorted = [...values].sort((one, two) => one - two);
|
|
@@ -427,7 +437,12 @@ function report(sides) {
|
|
|
427
437
|
void width;
|
|
428
438
|
}
|
|
429
439
|
|
|
430
|
-
/**
|
|
440
|
+
/**
|
|
441
|
+
* How much slower the second is than the first, as a readable ratio.
|
|
442
|
+
* @param {number} one - A's median, in milliseconds.
|
|
443
|
+
* @param {number} two - B's median, in milliseconds.
|
|
444
|
+
* @returns {string} - A phrase for the summary, or nothing when either is missing.
|
|
445
|
+
*/
|
|
431
446
|
function ratio(one, two) {
|
|
432
447
|
if (!one || !two) return '';
|
|
433
448
|
const factor = two / one;
|