pmtiles-swarm 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/docs/running-as-a-service.md +128 -1
- package/docs/serving-tiles.md +48 -0
- package/package.json +2 -1
- package/src/config.js +42 -0
- package/src/index.js +60 -0
- package/src/mutable.js +16 -2
- package/src/publisher.js +202 -0
- package/src/tilejson.js +8 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,79 @@
|
|
|
7
7
|
### 🐞 Bug fixes
|
|
8
8
|
- _...Add new stuff here..._
|
|
9
9
|
|
|
10
|
+
## 0.13.0
|
|
11
|
+
### ✨ Features and improvements
|
|
12
|
+
- **The publisher's DHT socket is bound explicitly, on a configurable port.** It was left to bind
|
|
13
|
+
implicitly on its first send, which works but takes an unpredictable ephemeral port and reports
|
|
14
|
+
nothing — so there was no way to forward it, and no way to tell which one it had. `mutable.dhtPort`
|
|
15
|
+
now sets it (`0`, ephemeral, by default) and the port is logged at startup.
|
|
16
|
+
|
|
17
|
+
Publishing needs no forward either way: a put is outbound, and the replies come back on the same
|
|
18
|
+
socket the way any UDP client's do. Pinning and forwarding one makes this a *reachable* DHT node
|
|
19
|
+
instead, which earns a better routing table and contributes back — worth having on a node that
|
|
20
|
+
runs continuously.
|
|
21
|
+
|
|
22
|
+
It must not collide with an engine's port. Each seeding engine runs a DHT of its own, so a node
|
|
23
|
+
with both has three UDP participants and only this one is placed here; two sockets cannot hold
|
|
24
|
+
one port and the node would fail to start.
|
|
25
|
+
|
|
26
|
+
### 📚 Documentation
|
|
27
|
+
- **[docs/running-as-a-service.md](docs/running-as-a-service.md) covers the publisher key**:
|
|
28
|
+
generating it as the service account so ownership is right, `chmod 400` because nothing ever
|
|
29
|
+
writes it back, and why it must be backed up off the machine — losing it breaks every style
|
|
30
|
+
pointing at that public key, permanently, with no reissue. Also what happens under HA config
|
|
31
|
+
sync: the configuration replicates to the standby and the key does not, so the standby logs
|
|
32
|
+
`not publishing: ENOENT` and serves on, which is the intended outcome rather than a fault. And
|
|
33
|
+
how to confirm it works, including that `nodes: 0` in the log means nobody stored the record
|
|
34
|
+
however healthy the rest of the line looks.
|
|
35
|
+
- **The ports table lists the DHT port**, which had only been described in the publisher section —
|
|
36
|
+
not where anyone looks when deciding what to forward.
|
|
37
|
+
- **Why publishing does not reuse an engine's DHT** is recorded in
|
|
38
|
+
[src/publisher.js](src/publisher.js), since it will be asked again. libtorrent's is unreachable:
|
|
39
|
+
the 2.x Python bindings expose neither `dht_put_item` nor `dht_get_item`, though the alerts are
|
|
40
|
+
bound, so the C++ side supports BEP 44 and there is simply no method to call. WebTorrent's *is*
|
|
41
|
+
`bittorrent-dht` and could be reused to save a socket; that is a deliberate choice rather than an
|
|
42
|
+
oversight, taken to keep one code path that behaves the same whichever engine is configured.
|
|
43
|
+
|
|
44
|
+
## 0.12.0
|
|
45
|
+
### ✨ Features and improvements
|
|
46
|
+
- **A category can now be addressed without a server at all.** A node that builds can publish a
|
|
47
|
+
signed DHT record (BEP 46) naming whichever archive is currently newest in each category, so a
|
|
48
|
+
style can point at a magnet that never goes stale:
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
magnet:?xs=urn:btpk:<public key>&s=openmaptiles&dn=…&ws=…
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
No infohash in it, which is the whole point — an infohash is what goes stale on the next build,
|
|
55
|
+
and it is why the fragment convention added in 0.11.0 could not be used for `/latest/` URLs. The
|
|
56
|
+
salt is the category name, so **one keypair addresses every category** rather than needing one
|
|
57
|
+
each.
|
|
58
|
+
|
|
59
|
+
Turn it on with `mutable.publish` and a key from the new **`pmtiles-swarm publisher-key`**
|
|
60
|
+
command. Off by default.
|
|
61
|
+
|
|
62
|
+
**Only the node that builds needs the key.** Serving nodes receive the public half on the catalog
|
|
63
|
+
entry, through the same sync that already carries `magnet` and `webSeeds`, and assemble the
|
|
64
|
+
identical magnet from it — there is nothing secret in one. Ten nodes behind a balancer hand out
|
|
65
|
+
the same string and none of them can publish. Run exactly one publisher: two under one key would
|
|
66
|
+
fight over the sequence number.
|
|
67
|
+
|
|
68
|
+
It is a **signing key rather than a credential**. Whoever holds it can tell your subscribers that
|
|
69
|
+
any archive is the current build, signed, and clients will believe it.
|
|
70
|
+
|
|
71
|
+
Records expire from the DHT after roughly two hours, so the node republishes on a timer
|
|
72
|
+
(`republishSeconds`, default 1800). That timer is the feature, not an optimisation — without it
|
|
73
|
+
a record published once works all afternoon and quietly stops resolving by evening. A category
|
|
74
|
+
whose put fails does not stop the others.
|
|
75
|
+
|
|
76
|
+
**`bittorrent-dht` is now a direct dependency** rather than reached for through webtorrent's
|
|
77
|
+
client, so publishing works on a node running the libtorrent engine alone.
|
|
78
|
+
- **The TileJSON's `torrent.mutable` block carries the magnet**, built from the public key, so no
|
|
79
|
+
consumer has to know how to assemble one. `mutableMagnet()` also accepts a hex key now — which
|
|
80
|
+
is all a serving node has — and carries `ws=` web seeds, so a client with no peers can still
|
|
81
|
+
range-read the archive over HTTP.
|
|
82
|
+
|
|
10
83
|
## 0.11.0
|
|
11
84
|
### ✨ Features and improvements
|
|
12
85
|
- **The magnet can travel in the TileJSON URL's fragment**, and the console will build that string
|
|
@@ -354,6 +354,116 @@ What the running service actually has:
|
|
|
354
354
|
systemctl show -p ReadWritePaths -p UMask -p SupplementaryGroups pmtiles-swarm
|
|
355
355
|
```
|
|
356
356
|
|
|
357
|
+
## The publisher key
|
|
358
|
+
|
|
359
|
+
Only needed if this node publishes BEP 46 records — the signed DHT entries that
|
|
360
|
+
let a style point at a category rather than at a build that goes stale. Skip
|
|
361
|
+
this section entirely if it does not build archives.
|
|
362
|
+
|
|
363
|
+
**Generate it as the service account**, so the file is owned by the user that
|
|
364
|
+
has to read it:
|
|
365
|
+
|
|
366
|
+
```sh
|
|
367
|
+
sudo -u pmtiles-swarm -H /var/lib/pmtiles-swarm/node_modules/.bin/pmtiles-swarm publisher-key > /etc/pmtiles-swarm/publisher.pem
|
|
368
|
+
sudo chown pmtiles-swarm:pmtiles-swarm /etc/pmtiles-swarm/publisher.pem
|
|
369
|
+
sudo chmod 400 /etc/pmtiles-swarm/publisher.pem
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
The PEM goes to stdout and the public key to stderr, so the redirect above
|
|
373
|
+
captures only the key material and you still see the public half on the
|
|
374
|
+
terminal. Write that public key down — it is what subscribers point at, and it
|
|
375
|
+
is the one part you will want later.
|
|
376
|
+
|
|
377
|
+
`400` rather than `600`: the service only ever reads this. Nothing in the
|
|
378
|
+
product writes it back, so removing write permission costs nothing and means a
|
|
379
|
+
compromised process cannot quietly replace it.
|
|
380
|
+
|
|
381
|
+
Then in `/etc/pmtiles-swarm/swarm.config.json`:
|
|
382
|
+
|
|
383
|
+
```json
|
|
384
|
+
{
|
|
385
|
+
"mutable": {
|
|
386
|
+
"publish": true,
|
|
387
|
+
"keyPath": "/etc/pmtiles-swarm/publisher.pem"
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
```
|
|
391
|
+
|
|
392
|
+
`/etc/pmtiles-swarm` is already in `ReadWritePaths` because the console rewrites
|
|
393
|
+
the configuration there, so the unit needs no change.
|
|
394
|
+
|
|
395
|
+
### Back it up, off this machine
|
|
396
|
+
|
|
397
|
+
Losing this file breaks **every style pointing at that public key, permanently**
|
|
398
|
+
— there is no recovery, no reissue, and no way to prove to a subscriber that a
|
|
399
|
+
new key is you. It is not like an API key you can rotate.
|
|
400
|
+
|
|
401
|
+
Back it up somewhere that is not this disk, and treat the backup as seriously as
|
|
402
|
+
the original: whoever holds it can publish a signed record telling your
|
|
403
|
+
subscribers that any archive is the current build, and clients will believe it
|
|
404
|
+
because the signature checks out.
|
|
405
|
+
|
|
406
|
+
### Exactly one publisher
|
|
407
|
+
|
|
408
|
+
Two nodes publishing under one key fight over the sequence number, each
|
|
409
|
+
overwriting the other's claim about what is current. So the PEM belongs on the
|
|
410
|
+
build node and nowhere else.
|
|
411
|
+
|
|
412
|
+
**This matters if you run HA config sync.** The configuration will replicate to
|
|
413
|
+
the standby, `publish: true` and all — but the PEM will not, because you are
|
|
414
|
+
not going to copy it. The standby then logs
|
|
415
|
+
|
|
416
|
+
```
|
|
417
|
+
[mutable] not publishing: ENOENT: no such file or directory
|
|
418
|
+
```
|
|
419
|
+
|
|
420
|
+
on every start and carries on serving normally. That is the intended outcome
|
|
421
|
+
rather than a fault: the config syncing is harmless, and the key not syncing is
|
|
422
|
+
the point. If the noise bothers you, set `mutable.publish` to `false` in the
|
|
423
|
+
standby's config after the sync.
|
|
424
|
+
|
|
425
|
+
### It does not need a port forwarded
|
|
426
|
+
|
|
427
|
+
The DHT socket is separate from the seeding engine's, and by default takes an
|
|
428
|
+
ephemeral UDP port (`mutable.dhtPort: 0`). That is enough to publish: a put is
|
|
429
|
+
outbound — find nodes, then send — and the replies come back on the same
|
|
430
|
+
socket the way any UDP client's do, which NAT handles without help.
|
|
431
|
+
|
|
432
|
+
Setting a fixed port and forwarding it makes this a *reachable* DHT node, which
|
|
433
|
+
means better lookups and contributing back to the network. Worth doing if this
|
|
434
|
+
node is long-lived, but nothing here requires it.
|
|
435
|
+
|
|
436
|
+
**Do not reuse the libtorrent engine's port.** That engine runs a DHT of its own
|
|
437
|
+
on `libtorrent.listen`, and two sockets cannot hold one port — the node would
|
|
438
|
+
fail to start.
|
|
439
|
+
|
|
440
|
+
The port in use is reported at startup:
|
|
441
|
+
|
|
442
|
+
```
|
|
443
|
+
[mutable] DHT on UDP 63213
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
### Confirming it works
|
|
447
|
+
|
|
448
|
+
```sh
|
|
449
|
+
journalctl -u pmtiles-swarm | grep mutable
|
|
450
|
+
```
|
|
451
|
+
|
|
452
|
+
A healthy publisher says which categories it is announcing and under which key
|
|
453
|
+
at startup, then one line per category whenever a build moves:
|
|
454
|
+
|
|
455
|
+
```
|
|
456
|
+
[mutable] publishing 3 categories as 7680dc95248eb807… every 30m
|
|
457
|
+
[mutable] openmaptiles -> 4813a0e68e4b (seq 1786108931, 8 nodes)
|
|
458
|
+
```
|
|
459
|
+
|
|
460
|
+
`nodes` is how many DHT peers stored the record. **Zero means nobody did**, and
|
|
461
|
+
the record does not exist however healthy the log looks otherwise — check that
|
|
462
|
+
UDP is not blocked and that the DHT is reachable.
|
|
463
|
+
|
|
464
|
+
The first publish waits about fifteen seconds after start, because a put into a
|
|
465
|
+
DHT that has not finished bootstrapping reaches nobody.
|
|
466
|
+
|
|
357
467
|
## The sidecar
|
|
358
468
|
|
|
359
469
|
The libtorrent engine runs Python as a child process, so the service user needs
|
|
@@ -373,16 +483,33 @@ rather than exiting. Name the interpreter explicitly when the service user's
|
|
|
373
483
|
|
|
374
484
|
## Ports
|
|
375
485
|
|
|
376
|
-
|
|
486
|
+
Five listeners, and only the peer ports want a firewall rule. See
|
|
377
487
|
[ports and reachability](engines.md#ports-and-reachability) for the detail.
|
|
378
488
|
|
|
379
489
|
| | |
|
|
380
490
|
| --- | --- |
|
|
381
491
|
| `libtorrent.listen` — 6881, TCP and UDP | forward it |
|
|
382
492
|
| `webtorrent.clientOptions.torrentPort` — pin it, or it changes every start | forward it |
|
|
493
|
+
| `mutable.dhtPort` — UDP, ephemeral by default | optional; see below |
|
|
383
494
|
| `port` — 8090 | your proxy or CDN |
|
|
384
495
|
| `adminPort` — 8091, bound to `127.0.0.1` | nothing; that is the point |
|
|
385
496
|
|
|
497
|
+
`mutable.dhtPort` is the odd one. Publishing works without any forward, because
|
|
498
|
+
a put is outbound and the replies come back on the same socket the way any UDP
|
|
499
|
+
client's do. Pin it and forward it only if you want this to be a *reachable*
|
|
500
|
+
DHT node — which earns a better routing table and contributes back, and is
|
|
501
|
+
worth having on a node that runs continuously.
|
|
502
|
+
|
|
503
|
+
Pin it to something free: **not** `libtorrent.listen`, which is a DHT of its
|
|
504
|
+
own, and not `torrentPort`. `6883` sits clear of both.
|
|
505
|
+
|
|
506
|
+
```json
|
|
507
|
+
{ "mutable": { "dhtPort": 6883 } }
|
|
508
|
+
```
|
|
509
|
+
|
|
510
|
+
Note that each engine runs its own DHT as well, so a node with both engines has
|
|
511
|
+
three UDP participants. Only this one is yours to place.
|
|
512
|
+
|
|
386
513
|
## Updating
|
|
387
514
|
|
|
388
515
|
```sh
|
package/docs/serving-tiles.md
CHANGED
|
@@ -226,6 +226,54 @@ nothing publishes those records yet.
|
|
|
226
226
|
For an immutable `/archives/<infohash>/tiles.json` URL the question does not
|
|
227
227
|
arise: both halves name the same fixed archive.
|
|
228
228
|
|
|
229
|
+
### A fragment that survives a rebuild
|
|
230
|
+
|
|
231
|
+
The caveat above — a pinned infohash going stale — is what BEP 46 fixes. A node
|
|
232
|
+
that publishes signs a DHT record naming whichever infohash is current, and the
|
|
233
|
+
magnet then names the **category** rather than a build:
|
|
234
|
+
|
|
235
|
+
```
|
|
236
|
+
magnet:?xs=urn:btpk:<public key>&s=openmaptiles&dn=…&ws=…
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
No infohash anywhere, so nothing to go stale. A client resolves the record over
|
|
240
|
+
the DHT and joins whatever is current.
|
|
241
|
+
|
|
242
|
+
Turn it on with a key on the node that builds:
|
|
243
|
+
|
|
244
|
+
```sh
|
|
245
|
+
pmtiles-swarm publisher-key > /etc/pmtiles-swarm/publisher.pem
|
|
246
|
+
chmod 600 /etc/pmtiles-swarm/publisher.pem
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
```json
|
|
250
|
+
{ "mutable": { "publish": true, "keyPath": "/etc/pmtiles-swarm/publisher.pem" } }
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
The magnet then appears in every TileJSON as `torrent.mutable.magnet`, and the
|
|
254
|
+
console's copy button uses it for category URLs.
|
|
255
|
+
|
|
256
|
+
**Only the node that builds needs the key.** Serving nodes receive the public
|
|
257
|
+
half on the catalog entry, through the same subscription sync that carries
|
|
258
|
+
`magnet` and `webSeeds`, and assemble the identical magnet from it — there is
|
|
259
|
+
nothing secret in one. Ten nodes behind a balancer hand out the same string and
|
|
260
|
+
none of them can publish.
|
|
261
|
+
|
|
262
|
+
**Run exactly one publisher.** Two nodes publishing under one key would fight
|
|
263
|
+
over the sequence number, each overwriting the other's claim about what is
|
|
264
|
+
current.
|
|
265
|
+
|
|
266
|
+
**It is a signing key, not a credential.** Whoever holds it can tell your
|
|
267
|
+
subscribers that any archive is the current build, signed, and they will believe
|
|
268
|
+
it. Treat it the way you would a code-signing key: lose it and every style
|
|
269
|
+
pointing at that public key breaks permanently.
|
|
270
|
+
|
|
271
|
+
**Records expire after roughly two hours**, so the node republishes on a timer
|
|
272
|
+
(`republishSeconds`, default 1800). That timer is not an optimisation — without
|
|
273
|
+
it a record published once works all afternoon and stops resolving by evening.
|
|
274
|
+
If the publisher is offline longer than that, the DHT path goes quiet until it
|
|
275
|
+
returns; the HTTP TileJSON URL is unaffected.
|
|
276
|
+
|
|
229
277
|
## The `torrent` block
|
|
230
278
|
|
|
231
279
|
TileJSON documents from this server carry a non-standard `torrent` member:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pmtiles-swarm",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@maplibre/maplibre-gl-inspect": "^1.8.2",
|
|
34
34
|
"bencode": "^4.0.1",
|
|
35
|
+
"bittorrent-dht": "^11.0.12",
|
|
35
36
|
"chokidar": "^5.0.0",
|
|
36
37
|
"create-torrent": "^6.1.0",
|
|
37
38
|
"express": "^5.2.1",
|
package/src/config.js
CHANGED
|
@@ -397,6 +397,48 @@ const DEFAULTS = {
|
|
|
397
397
|
* instead sidesteps the question entirely.
|
|
398
398
|
*/
|
|
399
399
|
trustProxy: false,
|
|
400
|
+
/**
|
|
401
|
+
* Announcing the current build of each category over the DHT (BEP 46).
|
|
402
|
+
*
|
|
403
|
+
* A category is the only stable handle this system has — every archive is
|
|
404
|
+
* addressed by its infohash, so a style pointing at one goes stale on the
|
|
405
|
+
* next build. `/latest/<category>/` fixes that with a server; this fixes it
|
|
406
|
+
* without one, as a signed DHT record naming whichever infohash is current.
|
|
407
|
+
*
|
|
408
|
+
* **Only the node that builds needs this.** Serving nodes carry the public
|
|
409
|
+
* half on the catalog entry and hand it out in the TileJSON; publishing is
|
|
410
|
+
* the only thing the secret is used for. Two nodes publishing under one key
|
|
411
|
+
* would fight over the sequence number, so run exactly one publisher.
|
|
412
|
+
*
|
|
413
|
+
* The key is a signing key, not a credential: whoever holds it can tell your
|
|
414
|
+
* subscribers that any archive is the current build, signed. Treat it the way
|
|
415
|
+
* you would a code-signing key. Generate one with
|
|
416
|
+
* `pmtiles-swarm publisher-key`.
|
|
417
|
+
*
|
|
418
|
+
* Records expire from the DHT after roughly two hours, so this republishes on
|
|
419
|
+
* a timer. Nothing else keeps them alive.
|
|
420
|
+
*/
|
|
421
|
+
mutable: {
|
|
422
|
+
/** Publish records. Off unless a key is configured and this is set. */
|
|
423
|
+
publish: false,
|
|
424
|
+
/** PEM file holding the ed25519 keypair. Never leaves the publisher. */
|
|
425
|
+
keyPath: undefined,
|
|
426
|
+
/** How often to republish, in seconds. */
|
|
427
|
+
republishSeconds: 1800,
|
|
428
|
+
/**
|
|
429
|
+
* UDP port for this node's DHT socket. 0 takes an ephemeral one.
|
|
430
|
+
*
|
|
431
|
+
* Publishing does not need a forwarded port: a put is outbound, and the
|
|
432
|
+
* replies come back on the same socket the way any UDP client's do, which
|
|
433
|
+
* NAT handles. Setting a fixed port and forwarding it makes this a
|
|
434
|
+
* reachable DHT node, which gives better lookups and contributes back —
|
|
435
|
+
* but nothing here requires it.
|
|
436
|
+
*
|
|
437
|
+
* Do not reuse the libtorrent engine's port. That engine runs a DHT of its
|
|
438
|
+
* own, and two sockets cannot hold one port.
|
|
439
|
+
*/
|
|
440
|
+
dhtPort: 0,
|
|
441
|
+
},
|
|
400
442
|
/**
|
|
401
443
|
* What this node has served, at `GET /api/stats`.
|
|
402
444
|
*
|
package/src/index.js
CHANGED
|
@@ -109,6 +109,7 @@ async function main() {
|
|
|
109
109
|
Usage:
|
|
110
110
|
pmtiles-swarm [--config FILE] start the node
|
|
111
111
|
pmtiles-swarm status [--config FILE] ask a running node what it is doing
|
|
112
|
+
pmtiles-swarm publisher-key print a new BEP 46 signing key
|
|
112
113
|
|
|
113
114
|
--config, -c path to a JSON config file
|
|
114
115
|
--port, -p override the listen port
|
|
@@ -135,6 +136,20 @@ PMTILES_SWARM_PUBLIC_URL
|
|
|
135
136
|
return;
|
|
136
137
|
}
|
|
137
138
|
|
|
139
|
+
if (positionals[0] === 'publisher-key') {
|
|
140
|
+
const { generatePublisherKey, publisherKeyToPem } = await import('./mutable.js');
|
|
141
|
+
const key = generatePublisherKey();
|
|
142
|
+
// The PEM on stdout so it can be redirected to a file; everything else on
|
|
143
|
+
// stderr so that redirect stays clean.
|
|
144
|
+
process.stdout.write(publisherKeyToPem(key));
|
|
145
|
+
console.error('');
|
|
146
|
+
console.error(`public key: ${Buffer.from(key.publicKey).toString('hex')}`);
|
|
147
|
+
console.error('Save the PEM where only this node can read it, and point');
|
|
148
|
+
console.error('mutable.keyPath at it. It signs what your subscribers');
|
|
149
|
+
console.error('believe is the current build, so treat it as a signing key.');
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
|
|
138
153
|
if (positionals.length > 0) {
|
|
139
154
|
console.error(`unknown command: ${positionals[0]}`);
|
|
140
155
|
console.error('try: pmtiles-swarm status');
|
|
@@ -245,6 +260,51 @@ PMTILES_SWARM_PUBLIC_URL
|
|
|
245
260
|
? null
|
|
246
261
|
: new TileStats({ recent: config.tileStats?.recent });
|
|
247
262
|
|
|
263
|
+
// Announcing the current build of each category over the DHT. Only ever on
|
|
264
|
+
// the node that builds: the key signs what subscribers believe is current,
|
|
265
|
+
// and two publishers under one key would fight over the sequence number.
|
|
266
|
+
let publisher;
|
|
267
|
+
if (config.mutable?.publish && config.mutable?.keyPath) {
|
|
268
|
+
try {
|
|
269
|
+
const [{ publisherKeyFromPem }, { MutablePublisher }, DHT] = await Promise.all([
|
|
270
|
+
import('./mutable.js'),
|
|
271
|
+
import('./publisher.js'),
|
|
272
|
+
import('bittorrent-dht').then((m) => m.default),
|
|
273
|
+
]);
|
|
274
|
+
const pem = await fs.readFile(config.mutable.keyPath, 'utf8');
|
|
275
|
+
const dht = new DHT();
|
|
276
|
+
// Bound explicitly rather than left to bind implicitly on first send,
|
|
277
|
+
// so the port is predictable and can be forwarded if you want this to
|
|
278
|
+
// be a reachable DHT node. 0 takes an ephemeral one, which is all
|
|
279
|
+
// publishing needs.
|
|
280
|
+
await new Promise((resolve, reject) => {
|
|
281
|
+
dht.once('error', reject);
|
|
282
|
+
dht.listen(config.mutable.dhtPort ?? 0, resolve);
|
|
283
|
+
});
|
|
284
|
+
console.log(`[mutable] DHT on UDP ${dht.address().port}`);
|
|
285
|
+
publisher = new MutablePublisher({
|
|
286
|
+
catalog,
|
|
287
|
+
dht,
|
|
288
|
+
key: publisherKeyFromPem(pem),
|
|
289
|
+
intervalMs: (config.mutable.republishSeconds ?? 1800) * 1000,
|
|
290
|
+
});
|
|
291
|
+
stoppers.unshift({
|
|
292
|
+
label: 'mutable publisher',
|
|
293
|
+
stop: () => {
|
|
294
|
+
publisher.stop();
|
|
295
|
+
dht.destroy();
|
|
296
|
+
},
|
|
297
|
+
ms: 2000,
|
|
298
|
+
});
|
|
299
|
+
publisher.start();
|
|
300
|
+
} catch (error) {
|
|
301
|
+
// Never fatal: a node that cannot publish should still serve. Loudly
|
|
302
|
+
// reported, because the failure is otherwise invisible until a
|
|
303
|
+
// subscriber's style quietly stops resolving.
|
|
304
|
+
console.error(`[mutable] not publishing: ${error.message}`);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
248
308
|
const tiles = new TileStore({ catalog, engine, config });
|
|
249
309
|
library.attachTiles(tiles);
|
|
250
310
|
const warm = new WarmRunner(tiles);
|
package/src/mutable.js
CHANGED
|
@@ -82,21 +82,35 @@ function rawPublicKey(publicKey) {
|
|
|
82
82
|
*
|
|
83
83
|
* Note this carries no infohash: `xs=urn:btpk:` names the public key, and the
|
|
84
84
|
* client resolves it through the DHT to whatever infohash is current.
|
|
85
|
-
* @param {Uint8Array} publicKey - Raw 32-byte public key.
|
|
85
|
+
* @param {Uint8Array | string} publicKey - Raw 32-byte public key, or its hex form.
|
|
86
|
+
* A serving node has only the hex, off the catalog entry, and must be able to
|
|
87
|
+
* build this string without ever seeing the raw key or the private half.
|
|
86
88
|
* @param {object} [options] - Extra magnet parameters.
|
|
87
89
|
* @param {string} [options.name] - Display name for the archive.
|
|
88
90
|
* @param {string[]} [options.trackers] - Tracker announce URLs.
|
|
91
|
+
* @param {string[]} [options.webSeeds] - BEP 19 web seeds.
|
|
89
92
|
* @param {string} [options.salt] - Salt, when one key publishes several archives.
|
|
90
93
|
* @returns {string} - A BEP 46 magnet URI.
|
|
91
94
|
*/
|
|
92
95
|
export function mutableMagnet(publicKey, options = {}) {
|
|
93
|
-
|
|
96
|
+
// Buffer.from(string) would read hex as UTF-8 and produce a 64-byte key, so
|
|
97
|
+
// the two forms have to be told apart rather than coerced.
|
|
98
|
+
const hex =
|
|
99
|
+
typeof publicKey === 'string'
|
|
100
|
+
? publicKey.toLowerCase()
|
|
101
|
+
: Buffer.from(publicKey).toString('hex');
|
|
94
102
|
const parts = [`magnet:?xs=urn:btpk:${hex}`];
|
|
95
103
|
if (options.name) parts.push(`dn=${encodeURIComponent(options.name)}`);
|
|
96
104
|
if (options.salt) parts.push(`s=${encodeURIComponent(options.salt)}`);
|
|
97
105
|
for (const tracker of options.trackers ?? []) {
|
|
98
106
|
parts.push(`tr=${encodeURIComponent(tracker)}`);
|
|
99
107
|
}
|
|
108
|
+
// Carried because it is what makes a magnet useful with no peers at all: a
|
|
109
|
+
// client can range-read the archive over HTTP and still be correct, which is
|
|
110
|
+
// the difference between a slow first paint and a blank map.
|
|
111
|
+
for (const seed of options.webSeeds ?? []) {
|
|
112
|
+
parts.push(`ws=${encodeURIComponent(seed)}`);
|
|
113
|
+
}
|
|
100
114
|
return parts.join('&');
|
|
101
115
|
}
|
|
102
116
|
|
package/src/publisher.js
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Announcing the current build of a category over the DHT (BEP 46).
|
|
3
|
+
*
|
|
4
|
+
* A category is the only stable handle this system has. Every archive is
|
|
5
|
+
* addressed by its infohash, which is what makes a tile immutable and leaves a
|
|
6
|
+
* style with nothing to point at that survives a rebuild. `/latest/<category>/`
|
|
7
|
+
* solves that with a server; this solves it without one — a signed DHT record,
|
|
8
|
+
* addressed by public key, naming whichever infohash is current.
|
|
9
|
+
*
|
|
10
|
+
* Only the node that builds needs the private key. Serving nodes carry the
|
|
11
|
+
* public half on the catalog entry and hand it out in the TileJSON, and
|
|
12
|
+
* publishing is the only operation the secret is used for. Two nodes
|
|
13
|
+
* publishing under one key would fight over `seq`, so this is deliberately a
|
|
14
|
+
* single-publisher design.
|
|
15
|
+
*
|
|
16
|
+
* The salt is the category name, so one keypair addresses every category
|
|
17
|
+
* rather than needing one each.
|
|
18
|
+
*
|
|
19
|
+
* The part that rots quietly: DHT nodes drop mutable items after roughly two
|
|
20
|
+
* hours. A record published once works all afternoon and is gone by evening,
|
|
21
|
+
* so republishing on a timer is not an optimisation here — it is the feature.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/*
|
|
25
|
+
* Why this uses bittorrent-dht rather than an engine's own DHT.
|
|
26
|
+
*
|
|
27
|
+
* Both seeding engines run a DHT already, so a third one looks redundant.
|
|
28
|
+
* It is not, for two different reasons:
|
|
29
|
+
*
|
|
30
|
+
* libtorrent - its DHT lives in the Python sidecar, and the 2.x Python
|
|
31
|
+
* bindings do not expose dht_put_item or dht_get_item at all.
|
|
32
|
+
* The alerts are bound (dht_mutable_item_alert, dht_put_alert)
|
|
33
|
+
* so the C++ side supports BEP 44, but there is no method to
|
|
34
|
+
* start one. Checked against 2.0.13; worth re-checking if a
|
|
35
|
+
* later binding adds them, because a sidecar op would then be
|
|
36
|
+
* the tidier answer for a libtorrent-only node.
|
|
37
|
+
*
|
|
38
|
+
* webtorrent - its DHT *is* bittorrent-dht, reachable as `client.dht`, so
|
|
39
|
+
* reusing it is possible and would save a socket. Not done,
|
|
40
|
+
* to keep one code path that behaves the same whichever
|
|
41
|
+
* engine is configured.
|
|
42
|
+
*
|
|
43
|
+
* The dependency itself costs nothing: webtorrent already depends on the same
|
|
44
|
+
* version, and npm dedupes them to one install. Declaring it directly only
|
|
45
|
+
* removes the reliance on a transitive.
|
|
46
|
+
*/
|
|
47
|
+
|
|
48
|
+
import { mutableMagnet, publishInfoHash } from './mutable.js';
|
|
49
|
+
|
|
50
|
+
/** Republish well inside the ~2h a DHT keeps an item. */
|
|
51
|
+
const DEFAULT_INTERVAL_MS = 30 * 60 * 1000;
|
|
52
|
+
|
|
53
|
+
/** Grace period before the first publish, for the DHT to find peers. */
|
|
54
|
+
const DEFAULT_READY_MS = 15_000;
|
|
55
|
+
|
|
56
|
+
export class MutablePublisher {
|
|
57
|
+
#catalog;
|
|
58
|
+
#dht;
|
|
59
|
+
#key;
|
|
60
|
+
#intervalMs;
|
|
61
|
+
#timer = null;
|
|
62
|
+
#stopped = false;
|
|
63
|
+
#log;
|
|
64
|
+
/** Last published infohash per category, so an unchanged build stays quiet. */
|
|
65
|
+
#published = new Map();
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* @param {object} options - Wiring.
|
|
69
|
+
* @param {object} options.catalog - Catalog to read categories from.
|
|
70
|
+
* @param {object} options.dht - A bittorrent-dht instance.
|
|
71
|
+
* @param {object} options.key - Keypair from `publisherKeyFromPem`.
|
|
72
|
+
* @param {number} [options.intervalMs] - Republish interval.
|
|
73
|
+
* @param {Function} [options.log] - Where to report.
|
|
74
|
+
*/
|
|
75
|
+
constructor({ catalog, dht, key, intervalMs, log = console.log }) {
|
|
76
|
+
this.#catalog = catalog;
|
|
77
|
+
this.#dht = dht;
|
|
78
|
+
this.#key = key;
|
|
79
|
+
this.#intervalMs = intervalMs ?? DEFAULT_INTERVAL_MS;
|
|
80
|
+
this.#log = log;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** @returns {string} - The public key, hex, which is safe to publish. */
|
|
84
|
+
get publicKeyHex() {
|
|
85
|
+
return Buffer.from(this.#key.publicKey).toString('hex');
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The newest archive in each category — what a record points at.
|
|
90
|
+
* @returns {Map<string, object>} - Category to catalog entry.
|
|
91
|
+
*/
|
|
92
|
+
#current() {
|
|
93
|
+
const newest = new Map();
|
|
94
|
+
for (const category of this.#catalog.categories()) {
|
|
95
|
+
// byCategory() is newest first, the same rule /latest/<category>/ uses,
|
|
96
|
+
// so the record and the endpoint cannot disagree about what is current.
|
|
97
|
+
const [entry] = this.#catalog.byCategory(category);
|
|
98
|
+
if (entry) newest.set(category, entry);
|
|
99
|
+
}
|
|
100
|
+
return newest;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Publishes every category once.
|
|
105
|
+
* @param {object} [options] - Behaviour.
|
|
106
|
+
* @param {boolean} [options.force] - Log even when nothing changed.
|
|
107
|
+
* @returns {Promise<object[]>} - What was published.
|
|
108
|
+
*/
|
|
109
|
+
async publishAll(options = {}) {
|
|
110
|
+
const done = [];
|
|
111
|
+
for (const [category, entry] of this.#current()) {
|
|
112
|
+
if (this.#stopped) break;
|
|
113
|
+
// Republished even when unchanged: the record expires whether or not the
|
|
114
|
+
// build has moved. `force` only decides whether it is worth saying.
|
|
115
|
+
const changed = this.#published.get(category) !== entry.infoHash;
|
|
116
|
+
try {
|
|
117
|
+
const result = await publishInfoHash(this.#dht, this.#key, entry.infoHash, {
|
|
118
|
+
salt: category,
|
|
119
|
+
});
|
|
120
|
+
this.#published.set(category, entry.infoHash);
|
|
121
|
+
done.push({ category, infoHash: entry.infoHash, ...result });
|
|
122
|
+
|
|
123
|
+
// Stamped on the entry so the TileJSON's torrent block carries the
|
|
124
|
+
// identity without any endpoint needing to know a publisher exists.
|
|
125
|
+
await this.#catalog.put({
|
|
126
|
+
infoHash: entry.infoHash,
|
|
127
|
+
mutable: { publicKey: this.publicKeyHex, salt: category, seq: result.seq },
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
if (changed || options.force) {
|
|
131
|
+
this.#log(
|
|
132
|
+
`[mutable] ${category} -> ${entry.infoHash.slice(0, 12)} ` +
|
|
133
|
+
`(seq ${result.seq}, ${result.nodes} nodes)`,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
} catch (error) {
|
|
137
|
+
// One category failing must not stop the rest.
|
|
138
|
+
this.#log(`[mutable] ${category} failed: ${error.message}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return done;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* The magnet a style should point at for a category.
|
|
146
|
+
*
|
|
147
|
+
* Public key only — there is nothing secret in it, which is why every
|
|
148
|
+
* serving node can hand it out and none of them can publish.
|
|
149
|
+
* @param {string} category - Which category.
|
|
150
|
+
* @param {object} [entry] - Newest entry, for the name and web seeds.
|
|
151
|
+
* @returns {string} - A BEP 46 magnet URI.
|
|
152
|
+
*/
|
|
153
|
+
magnetFor(category, entry) {
|
|
154
|
+
return mutableMagnet(this.#key.publicKey, {
|
|
155
|
+
salt: category,
|
|
156
|
+
name: entry?.name,
|
|
157
|
+
webSeeds: entry?.webSeeds,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Starts publishing, and keeps republishing before the records expire.
|
|
163
|
+
* @param {object} [options] - Timing.
|
|
164
|
+
* @param {number} [options.readyMs] - Grace before the first publish.
|
|
165
|
+
* @returns {void}
|
|
166
|
+
*/
|
|
167
|
+
start(options = {}) {
|
|
168
|
+
// A put into a DHT that has not found peers yet reaches nobody, and
|
|
169
|
+
// bootstrapping is the first thing a fresh node does. Waiting costs one
|
|
170
|
+
// interval of staleness at worst and makes the first publish mean
|
|
171
|
+
// something.
|
|
172
|
+
const first = setTimeout(() => {
|
|
173
|
+
if (this.#stopped) return;
|
|
174
|
+
this.publishAll({ force: true }).catch((error) =>
|
|
175
|
+
this.#log(`[mutable] first publish failed: ${error.message}`),
|
|
176
|
+
);
|
|
177
|
+
}, options.readyMs ?? DEFAULT_READY_MS);
|
|
178
|
+
first.unref?.();
|
|
179
|
+
|
|
180
|
+
this.#timer = setInterval(() => {
|
|
181
|
+
this.publishAll().catch((error) =>
|
|
182
|
+
this.#log(`[mutable] republish failed: ${error.message}`),
|
|
183
|
+
);
|
|
184
|
+
}, this.#intervalMs);
|
|
185
|
+
this.#timer.unref?.();
|
|
186
|
+
|
|
187
|
+
this.#log(
|
|
188
|
+
`[mutable] publishing ${this.#catalog.categories().length} categories as ` +
|
|
189
|
+
`${this.publicKeyHex.slice(0, 16)}… every ${Math.round(this.#intervalMs / 60000)}m`,
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Stops republishing.
|
|
195
|
+
* @returns {void}
|
|
196
|
+
*/
|
|
197
|
+
stop() {
|
|
198
|
+
this.#stopped = true;
|
|
199
|
+
if (this.#timer) clearInterval(this.#timer);
|
|
200
|
+
this.#timer = null;
|
|
201
|
+
}
|
|
202
|
+
}
|
package/src/tilejson.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { mutableMagnet } from './mutable.js';
|
|
1
2
|
import { TileStore } from './tiles.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -95,6 +96,13 @@ function buildTorrentBlock(entry, root) {
|
|
|
95
96
|
publicKey: entry.mutable.publicKey,
|
|
96
97
|
salt: entry.mutable.salt,
|
|
97
98
|
seq: entry.mutable.seq,
|
|
99
|
+
// Built here so a consumer does not have to know how to assemble one,
|
|
100
|
+
// and buildable by any node, because it contains only the public half.
|
|
101
|
+
magnet: mutableMagnet(entry.mutable.publicKey, {
|
|
102
|
+
salt: entry.mutable.salt,
|
|
103
|
+
name: entry.name,
|
|
104
|
+
webSeeds: entry.webSeeds,
|
|
105
|
+
}),
|
|
98
106
|
};
|
|
99
107
|
}
|
|
100
108
|
return block;
|