pmtiles-swarm 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/LICENSE +28 -0
- package/NOTICE.md +47 -0
- package/README.md +172 -0
- package/docs/architecture-diagram.md +287 -0
- package/docs/engines.md +140 -0
- package/docs/publishing.md +387 -0
- package/docs/serving-tiles.md +218 -0
- package/docs/subscribing.md +187 -0
- package/package.json +56 -0
- package/src/api.js +472 -0
- package/src/catalog.js +159 -0
- package/src/config.js +231 -0
- package/src/engines/libtorrent.js +384 -0
- package/src/engines/qbittorrent.js +320 -0
- package/src/engines/types.js +59 -0
- package/src/engines/webtorrent.js +264 -0
- package/src/feed.js +226 -0
- package/src/file-source.js +67 -0
- package/src/index.js +177 -0
- package/src/library.js +567 -0
- package/src/mutable.js +197 -0
- package/src/origin.js +205 -0
- package/src/pmtiles-probe.js +94 -0
- package/src/read-engine.js +175 -0
- package/src/sources.js +245 -0
- package/src/subscriptions.js +158 -0
- package/src/tilejson.js +112 -0
- package/src/tiles.js +284 -0
- package/src/torrent-create.js +250 -0
- package/src/warm.js +251 -0
- package/src/watch.js +98 -0
- package/src/web/index.html +254 -0
package/src/mutable.js
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Updatable torrents, via BEP 46 (Updating Torrents Via DHT Mutable Items).
|
|
3
|
+
*
|
|
4
|
+
* A torrent's infohash is a hash of its content, so a rebuilt archive is always
|
|
5
|
+
* a new torrent — there is no such thing as editing one in place. BEP 46 works
|
|
6
|
+
* around that with a level of indirection: you publish an ed25519-signed record
|
|
7
|
+
* into the DHT whose value names the *current* infohash, and hand out the
|
|
8
|
+
* public key instead of the infohash. Subscribers resolve the key whenever they
|
|
9
|
+
* check, and follow the archive across rebuilds.
|
|
10
|
+
*
|
|
11
|
+
* This is the decentralised sibling of the RSS feed. The feed is easier to
|
|
12
|
+
* consume (qBittorrent reads RSS today, with no new software) but needs a
|
|
13
|
+
* server that stays up; the DHT record needs no server but has to be
|
|
14
|
+
* republished periodically or it expires. Publishing both costs little.
|
|
15
|
+
*
|
|
16
|
+
* Built directly on bittorrent-dht's BEP 44 put/get rather than through
|
|
17
|
+
* WebTorrent's high-level API, which does not expose them — and independent of
|
|
18
|
+
* the seeding engine, since qBittorrent's WebUI has no way to publish these.
|
|
19
|
+
*
|
|
20
|
+
* BEP 44: https://www.bittorrent.org/beps/bep_0044.html
|
|
21
|
+
* BEP 46: https://www.bittorrent.org/beps/bep_0046.html
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import crypto from 'node:crypto';
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* An ed25519 keypair identifying a mutable torrent, in the raw 32-byte form
|
|
28
|
+
* BEP 44 expects.
|
|
29
|
+
* @typedef {object} PublisherKey
|
|
30
|
+
* @property {Uint8Array} publicKey - 32-byte raw public key. This is the stable identity you publish.
|
|
31
|
+
* @property {crypto.KeyObject} privateKey - Signing key. Keep it secret; whoever holds it controls the feed.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Generates a publishing identity.
|
|
36
|
+
*
|
|
37
|
+
* The public key is the permanent address of the archive: it does not change
|
|
38
|
+
* when the archive is rebuilt, which is the entire point. Back up the private
|
|
39
|
+
* key — losing it means subscribers can never be moved forward again.
|
|
40
|
+
* @returns {PublisherKey} - A fresh keypair.
|
|
41
|
+
*/
|
|
42
|
+
export function generatePublisherKey() {
|
|
43
|
+
const { publicKey, privateKey } = crypto.generateKeyPairSync('ed25519');
|
|
44
|
+
return { publicKey: rawPublicKey(publicKey), privateKey };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Restores a keypair from a stored private key.
|
|
49
|
+
* @param {string} pem - PKCS#8 PEM of the ed25519 private key.
|
|
50
|
+
* @returns {PublisherKey} - The keypair.
|
|
51
|
+
*/
|
|
52
|
+
export function publisherKeyFromPem(pem) {
|
|
53
|
+
const privateKey = crypto.createPrivateKey(pem);
|
|
54
|
+
const publicKey = crypto.createPublicKey(privateKey);
|
|
55
|
+
return { publicKey: rawPublicKey(publicKey), privateKey };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Serialises a private key for storage.
|
|
60
|
+
* @param {PublisherKey} key - The keypair.
|
|
61
|
+
* @returns {string} - PKCS#8 PEM.
|
|
62
|
+
*/
|
|
63
|
+
export function publisherKeyToPem(key) {
|
|
64
|
+
return key.privateKey.export({ type: 'pkcs8', format: 'pem' }).toString();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Extracts the raw 32 bytes of an ed25519 public key.
|
|
69
|
+
*
|
|
70
|
+
* Node only exports ed25519 keys as DER or JWK, and BEP 44 wants the bare
|
|
71
|
+
* bytes; JWK's base64url 'x' is the cleanest route to them.
|
|
72
|
+
* @param {crypto.KeyObject} publicKey - The key object.
|
|
73
|
+
* @returns {Uint8Array} - 32 raw bytes.
|
|
74
|
+
*/
|
|
75
|
+
function rawPublicKey(publicKey) {
|
|
76
|
+
const jwk = publicKey.export({ format: 'jwk' });
|
|
77
|
+
return new Uint8Array(Buffer.from(jwk.x, 'base64url'));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The magnet URI subscribers use to follow an archive across rebuilds.
|
|
82
|
+
*
|
|
83
|
+
* Note this carries no infohash: `xs=urn:btpk:` names the public key, and the
|
|
84
|
+
* client resolves it through the DHT to whatever infohash is current.
|
|
85
|
+
* @param {Uint8Array} publicKey - Raw 32-byte public key.
|
|
86
|
+
* @param {object} [options] - Extra magnet parameters.
|
|
87
|
+
* @param {string} [options.name] - Display name for the archive.
|
|
88
|
+
* @param {string[]} [options.trackers] - Tracker announce URLs.
|
|
89
|
+
* @param {string} [options.salt] - Salt, when one key publishes several archives.
|
|
90
|
+
* @returns {string} - A BEP 46 magnet URI.
|
|
91
|
+
*/
|
|
92
|
+
export function mutableMagnet(publicKey, options = {}) {
|
|
93
|
+
const hex = Buffer.from(publicKey).toString('hex');
|
|
94
|
+
const parts = [`magnet:?xs=urn:btpk:${hex}`];
|
|
95
|
+
if (options.name) parts.push(`dn=${encodeURIComponent(options.name)}`);
|
|
96
|
+
if (options.salt) parts.push(`s=${encodeURIComponent(options.salt)}`);
|
|
97
|
+
for (const tracker of options.trackers ?? []) {
|
|
98
|
+
parts.push(`tr=${encodeURIComponent(tracker)}`);
|
|
99
|
+
}
|
|
100
|
+
return parts.join('&');
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Publishes the infohash an archive currently resolves to.
|
|
105
|
+
*
|
|
106
|
+
* `seq` must increase on every update — the DHT rejects a record whose sequence
|
|
107
|
+
* number is not greater than the one it already holds, which is what stops a
|
|
108
|
+
* replayed old record from rolling subscribers backwards.
|
|
109
|
+
* @param {object} dht - A bittorrent-dht Client.
|
|
110
|
+
* @param {PublisherKey} key - The publishing identity.
|
|
111
|
+
* @param {string} infoHash - Hex infohash the key should now point at.
|
|
112
|
+
* @param {object} [options] - Publishing options.
|
|
113
|
+
* @param {number} [options.seq] - Sequence number. Defaults to seconds since the epoch, which is monotonic and needs no stored state.
|
|
114
|
+
* @param {string} [options.salt] - Salt, when one key publishes several archives.
|
|
115
|
+
* @returns {Promise<{hash: string, seq: number, nodes: number}>} - Where it landed.
|
|
116
|
+
*/
|
|
117
|
+
export function publishInfoHash(dht, key, infoHash, options = {}) {
|
|
118
|
+
const seq = options.seq ?? Math.floor(Date.now() / 1000);
|
|
119
|
+
const value = { ih: Buffer.from(infoHash, 'hex') };
|
|
120
|
+
|
|
121
|
+
const request = {
|
|
122
|
+
k: Buffer.from(key.publicKey),
|
|
123
|
+
seq,
|
|
124
|
+
v: value,
|
|
125
|
+
/**
|
|
126
|
+
* Signs the buffer bittorrent-dht assembles from salt, seq and value.
|
|
127
|
+
* @param {Buffer} buffer - The canonical bytes to sign.
|
|
128
|
+
* @returns {Buffer} - A 64-byte ed25519 signature.
|
|
129
|
+
*/
|
|
130
|
+
sign: (buffer) => crypto.sign(null, buffer, key.privateKey),
|
|
131
|
+
};
|
|
132
|
+
if (options.salt) request.salt = Buffer.from(options.salt);
|
|
133
|
+
|
|
134
|
+
return new Promise((resolve, reject) => {
|
|
135
|
+
dht.put(request, (error, hash, nodes) => {
|
|
136
|
+
if (error) {
|
|
137
|
+
reject(
|
|
138
|
+
new Error(`failed to publish mutable record: ${error.message}`, {
|
|
139
|
+
cause: error,
|
|
140
|
+
}),
|
|
141
|
+
);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
resolve({ hash: Buffer.from(hash).toString('hex'), seq, nodes });
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Resolves a public key to the infohash it currently names.
|
|
151
|
+
* @param {object} dht - A bittorrent-dht Client.
|
|
152
|
+
* @param {Uint8Array | string} publicKey - Raw or hex public key.
|
|
153
|
+
* @param {object} [options] - Lookup options.
|
|
154
|
+
* @param {string} [options.salt] - Salt used when publishing.
|
|
155
|
+
* @returns {Promise<{infoHash: string, seq: number} | null>} - The current target, or null if nothing is published.
|
|
156
|
+
*/
|
|
157
|
+
export function resolveInfoHash(dht, publicKey, options = {}) {
|
|
158
|
+
const raw =
|
|
159
|
+
typeof publicKey === 'string'
|
|
160
|
+
? Buffer.from(publicKey, 'hex')
|
|
161
|
+
: Buffer.from(publicKey);
|
|
162
|
+
const target = crypto.createHash('sha1');
|
|
163
|
+
target.update(raw);
|
|
164
|
+
if (options.salt) target.update(Buffer.from(options.salt));
|
|
165
|
+
|
|
166
|
+
return new Promise((resolve, reject) => {
|
|
167
|
+
dht.get(target.digest(), (error, result) => {
|
|
168
|
+
if (error) {
|
|
169
|
+
reject(
|
|
170
|
+
new Error(`failed to resolve mutable record: ${error.message}`, {
|
|
171
|
+
cause: error,
|
|
172
|
+
}),
|
|
173
|
+
);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (!result?.v) {
|
|
177
|
+
resolve(null);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
const ih = result.v.ih ?? result.v;
|
|
181
|
+
resolve({
|
|
182
|
+
infoHash: Buffer.from(ih).toString('hex'),
|
|
183
|
+
seq: result.seq ?? 0,
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Extracts the public key from a BEP 46 magnet URI.
|
|
191
|
+
* @param {string} magnet - A magnet URI.
|
|
192
|
+
* @returns {string | null} - Hex public key, or null if it is not a mutable magnet.
|
|
193
|
+
*/
|
|
194
|
+
export function publicKeyFromMagnet(magnet) {
|
|
195
|
+
const match = /xs=urn:btpk:([a-f0-9]{64})/i.exec(magnet ?? '');
|
|
196
|
+
return match ? match[1].toLowerCase() : null;
|
|
197
|
+
}
|
package/src/origin.js
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watching the source an archive was built from.
|
|
3
|
+
*
|
|
4
|
+
* A torrent describes a fixed set of bytes. If the file it was built from is
|
|
5
|
+
* later replaced — a nightly build overwriting `planet-latest.pmtiles`, say —
|
|
6
|
+
* the torrent does not become invalid, but it stops describing what the source
|
|
7
|
+
* now holds, and any web seed pointing at that source becomes actively harmful:
|
|
8
|
+
* peers fetch from it, fail hash verification, and eventually ban it.
|
|
9
|
+
*
|
|
10
|
+
* Detecting that is cheap. A HEAD request comparing ETag, Last-Modified and
|
|
11
|
+
* length catches essentially every real update for an HTTP origin, and a stat
|
|
12
|
+
* does the same for a local file. Neither needs the archive re-read.
|
|
13
|
+
*
|
|
14
|
+
* What to do about it is a judgement call, so this only reports. Rebuilding
|
|
15
|
+
* means re-hashing — and for a remote archive, re-downloading — which is not
|
|
16
|
+
* something to start behind an operator's back.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import fs from 'node:fs/promises';
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A fingerprint of the source, cheap enough to take repeatedly.
|
|
23
|
+
* @typedef {object} OriginFingerprint
|
|
24
|
+
* @property {string} type - 'http' or 'file'.
|
|
25
|
+
* @property {string} location - URL or path.
|
|
26
|
+
* @property {number} [size] - Byte length, when known.
|
|
27
|
+
* @property {string} [etag] - HTTP ETag.
|
|
28
|
+
* @property {string} [lastModified] - HTTP Last-Modified, or file mtime as ISO.
|
|
29
|
+
* @property {string} checkedAt - ISO timestamp of this check.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Fingerprints an archive's source without reading its contents.
|
|
34
|
+
* @param {object} source - Catalog source: {type, location}.
|
|
35
|
+
* @returns {Promise<OriginFingerprint | null>} - The fingerprint, or null if it cannot be taken.
|
|
36
|
+
*/
|
|
37
|
+
export async function fingerprintOrigin(source) {
|
|
38
|
+
if (!source?.location) return null;
|
|
39
|
+
const checkedAt = new Date().toISOString();
|
|
40
|
+
|
|
41
|
+
if (source.type === 'http') {
|
|
42
|
+
// HEAD first; some origins (notably S3-compatible ones behind a CDN) do
|
|
43
|
+
// not answer HEAD, in which case a ranged GET of one byte gets the same
|
|
44
|
+
// headers for the same negligible cost.
|
|
45
|
+
let response = await fetch(source.location, { method: 'HEAD' }).catch(
|
|
46
|
+
() => null,
|
|
47
|
+
);
|
|
48
|
+
if (!response?.ok) {
|
|
49
|
+
response = await fetch(source.location, {
|
|
50
|
+
headers: { range: 'bytes=0-0' },
|
|
51
|
+
}).catch(() => null);
|
|
52
|
+
}
|
|
53
|
+
if (!response?.ok) return null;
|
|
54
|
+
|
|
55
|
+
// A ranged response reports the range length, so prefer content-range.
|
|
56
|
+
const contentRange = response.headers.get('content-range');
|
|
57
|
+
const total = contentRange
|
|
58
|
+
? Number(contentRange.split('/')[1])
|
|
59
|
+
: Number(response.headers.get('content-length') ?? 0);
|
|
60
|
+
|
|
61
|
+
return {
|
|
62
|
+
type: 'http',
|
|
63
|
+
location: source.location,
|
|
64
|
+
size: Number.isFinite(total) && total > 0 ? total : undefined,
|
|
65
|
+
etag: response.headers.get('etag') ?? undefined,
|
|
66
|
+
lastModified: response.headers.get('last-modified') ?? undefined,
|
|
67
|
+
checkedAt,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (source.type === 'file') {
|
|
72
|
+
const stat = await fs.stat(source.location).catch(() => null);
|
|
73
|
+
if (!stat) return null;
|
|
74
|
+
return {
|
|
75
|
+
type: 'file',
|
|
76
|
+
location: source.location,
|
|
77
|
+
size: stat.size,
|
|
78
|
+
lastModified: new Date(stat.mtimeMs).toISOString(),
|
|
79
|
+
checkedAt,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Adopted torrents and magnets have no origin to watch.
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The result of comparing a fresh fingerprint against a stored one.
|
|
89
|
+
* @typedef {object} OriginCheck
|
|
90
|
+
* @property {string} infoHash - The archive checked.
|
|
91
|
+
* @property {'unchanged' | 'changed' | 'missing' | 'unknown'} status - What was found.
|
|
92
|
+
* @property {string} [reason] - Which validator differed.
|
|
93
|
+
* @property {OriginFingerprint} [fingerprint] - The fresh fingerprint.
|
|
94
|
+
*/
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Compares a stored fingerprint against the source as it is now.
|
|
98
|
+
*
|
|
99
|
+
* Any single differing validator is treated as a change. A false positive costs
|
|
100
|
+
* a warning; a false negative means continuing to advertise a web seed that
|
|
101
|
+
* serves bytes failing hash verification, so this errs toward reporting.
|
|
102
|
+
* @param {object} entry - The catalog entry.
|
|
103
|
+
* @returns {Promise<OriginCheck>} - What was found.
|
|
104
|
+
*/
|
|
105
|
+
export async function checkOrigin(entry) {
|
|
106
|
+
const stored = entry.origin;
|
|
107
|
+
if (!stored) {
|
|
108
|
+
// Nothing recorded to compare against; take a baseline for next time.
|
|
109
|
+
const fingerprint = await fingerprintOrigin(entry.source);
|
|
110
|
+
return fingerprint
|
|
111
|
+
? { infoHash: entry.infoHash, status: 'unknown', fingerprint }
|
|
112
|
+
: { infoHash: entry.infoHash, status: 'unknown' };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const fresh = await fingerprintOrigin(entry.source);
|
|
116
|
+
if (!fresh) {
|
|
117
|
+
return {
|
|
118
|
+
infoHash: entry.infoHash,
|
|
119
|
+
status: 'missing',
|
|
120
|
+
reason: `source is no longer reachable: ${entry.source?.location}`,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const differences = [];
|
|
125
|
+
if (stored.etag && fresh.etag && stored.etag !== fresh.etag) {
|
|
126
|
+
differences.push(`etag ${stored.etag} -> ${fresh.etag}`);
|
|
127
|
+
}
|
|
128
|
+
if (
|
|
129
|
+
stored.lastModified &&
|
|
130
|
+
fresh.lastModified &&
|
|
131
|
+
stored.lastModified !== fresh.lastModified
|
|
132
|
+
) {
|
|
133
|
+
differences.push(`last-modified ${stored.lastModified} -> ${fresh.lastModified}`);
|
|
134
|
+
}
|
|
135
|
+
if (stored.size && fresh.size && stored.size !== fresh.size) {
|
|
136
|
+
differences.push(`size ${stored.size} -> ${fresh.size}`);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (differences.length === 0) {
|
|
140
|
+
return { infoHash: entry.infoHash, status: 'unchanged', fingerprint: fresh };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// A modification time that moves while size and ETag stay put is weak
|
|
144
|
+
// evidence: a touch, a restored backup, or a re-upload of identical bytes all
|
|
145
|
+
// do it. Confirm against the archive's own header before calling it a change,
|
|
146
|
+
// because the consequence of being wrong is re-hashing — or re-downloading —
|
|
147
|
+
// the whole archive.
|
|
148
|
+
const mtimeOnly =
|
|
149
|
+
differences.length === 1 && differences[0].startsWith('last-modified');
|
|
150
|
+
if (mtimeOnly && entry.pmtiles) {
|
|
151
|
+
const confirmed = await contentLooksDifferent(entry);
|
|
152
|
+
if (confirmed === false) {
|
|
153
|
+
return {
|
|
154
|
+
infoHash: entry.infoHash,
|
|
155
|
+
status: 'unchanged',
|
|
156
|
+
reason: 'modification time moved but the archive header is identical',
|
|
157
|
+
fingerprint: fresh,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return {
|
|
163
|
+
infoHash: entry.infoHash,
|
|
164
|
+
status: 'changed',
|
|
165
|
+
reason: differences.join('; '),
|
|
166
|
+
fingerprint: fresh,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Re-reads the archive's header to see whether its structure actually changed.
|
|
172
|
+
*
|
|
173
|
+
* Reads only the header and root directory, so it costs a few kilobytes even
|
|
174
|
+
* against a multi-terabyte archive. A rebuilt archive essentially always shifts
|
|
175
|
+
* these offsets; identical ones mean the bytes are almost certainly the same.
|
|
176
|
+
* @param {object} entry - The catalog entry, carrying the stored summary.
|
|
177
|
+
* @returns {Promise<boolean | null>} - True if different, false if identical, null if undeterminable.
|
|
178
|
+
*/
|
|
179
|
+
async function contentLooksDifferent(entry) {
|
|
180
|
+
try {
|
|
181
|
+
const { probePMTiles } = await import('./pmtiles-probe.js');
|
|
182
|
+
const fresh = await probePMTiles(entry.source.location);
|
|
183
|
+
const stored = entry.pmtiles;
|
|
184
|
+
|
|
185
|
+
const fields = [
|
|
186
|
+
'tileCount',
|
|
187
|
+
'minZoom',
|
|
188
|
+
'maxZoom',
|
|
189
|
+
'format',
|
|
190
|
+
'specVersion',
|
|
191
|
+
];
|
|
192
|
+
for (const field of fields) {
|
|
193
|
+
// eslint-disable-next-line security/detect-object-injection -- field comes from the constant list above
|
|
194
|
+
if (stored[field] !== undefined && stored[field] !== fresh[field]) {
|
|
195
|
+
return true;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const sameBounds =
|
|
199
|
+
JSON.stringify(stored.bounds ?? []) === JSON.stringify(fresh.bounds ?? []);
|
|
200
|
+
return !sameBounds;
|
|
201
|
+
} catch {
|
|
202
|
+
// Cannot tell; let the caller fall back to the validator comparison.
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { FetchSource, PMTiles } from 'pmtiles';
|
|
2
|
+
import { NodeFileSource } from './file-source.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Reads the facts about an archive that a subscriber needs in order to decide
|
|
6
|
+
* whether they want it: coverage, zoom range, tile type, size.
|
|
7
|
+
*
|
|
8
|
+
* This is what makes a map RSS feed more useful than a generic torrent feed —
|
|
9
|
+
* an item can say "raster webp, z0-14, covering Switzerland, 36 GiB" instead of
|
|
10
|
+
* just a filename, so a consumer can filter before committing to a download.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** Maps PMTiles tile type numbers onto names and content types. */
|
|
14
|
+
const TILE_TYPES = {
|
|
15
|
+
0: { format: 'unknown', contentType: 'application/octet-stream' },
|
|
16
|
+
1: { format: 'pbf', contentType: 'application/x-protobuf' },
|
|
17
|
+
2: { format: 'png', contentType: 'image/png' },
|
|
18
|
+
3: { format: 'jpeg', contentType: 'image/jpeg' },
|
|
19
|
+
4: { format: 'webp', contentType: 'image/webp' },
|
|
20
|
+
5: { format: 'avif', contentType: 'image/avif' },
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Summary of an archive, as stored in the catalog and published in the feed.
|
|
25
|
+
* @typedef {object} PMTilesSummary
|
|
26
|
+
* @property {number} specVersion - PMTiles spec version.
|
|
27
|
+
* @property {string} format - Tile format: pbf, png, jpeg, webp, avif.
|
|
28
|
+
* @property {string} contentType - Matching content type.
|
|
29
|
+
* @property {number} minZoom - Lowest zoom present.
|
|
30
|
+
* @property {number} maxZoom - Highest zoom present.
|
|
31
|
+
* @property {number[]} bounds - [minLon, minLat, maxLon, maxLat].
|
|
32
|
+
* @property {number[]} center - [lon, lat, zoom].
|
|
33
|
+
* @property {number} tileCount - Addressed tile count.
|
|
34
|
+
* @property {boolean} clustered - Whether tiles are stored in Hilbert order.
|
|
35
|
+
* @property {string} [name] - Name from the archive metadata.
|
|
36
|
+
* @property {string} [description] - Description from the archive metadata.
|
|
37
|
+
* @property {string} [attribution] - Attribution from the archive metadata.
|
|
38
|
+
* @property {object[]} [vectorLayers] - Vector layer definitions, for pbf archives.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Reads an archive's header and metadata.
|
|
43
|
+
*
|
|
44
|
+
* Only the header and directory are read, not the tile data, so this is cheap
|
|
45
|
+
* even for a multi-terabyte archive — and it works against an HTTP URL without
|
|
46
|
+
* downloading it.
|
|
47
|
+
* @param {string} location - Local path or http(s) URL.
|
|
48
|
+
* @returns {Promise<PMTilesSummary>} - The summary.
|
|
49
|
+
*/
|
|
50
|
+
export async function probePMTiles(location) {
|
|
51
|
+
const isHttp = /^https?:\/\//i.test(location);
|
|
52
|
+
const source = isHttp ? new FetchSource(location) : new NodeFileSource(location);
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
const archive = new PMTiles(source);
|
|
56
|
+
const header = await archive.getHeader();
|
|
57
|
+
const metadata = (await archive.getMetadata()) ?? {};
|
|
58
|
+
|
|
59
|
+
const type = TILE_TYPES[header.tileType] ?? TILE_TYPES[0];
|
|
60
|
+
|
|
61
|
+
// An archive with no bounds set reports all zeroes; treat that as global
|
|
62
|
+
// rather than as a point at null island.
|
|
63
|
+
const hasBounds = !(
|
|
64
|
+
header.minLon === 0 &&
|
|
65
|
+
header.minLat === 0 &&
|
|
66
|
+
header.maxLon === 0 &&
|
|
67
|
+
header.maxLat === 0
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
return {
|
|
71
|
+
specVersion: header.specVersion,
|
|
72
|
+
format: type.format,
|
|
73
|
+
contentType: type.contentType,
|
|
74
|
+
minZoom: header.minZoom,
|
|
75
|
+
maxZoom: header.maxZoom,
|
|
76
|
+
bounds: hasBounds
|
|
77
|
+
? [header.minLon, header.minLat, header.maxLon, header.maxLat]
|
|
78
|
+
: [-180, -85.051129, 180, 85.051129],
|
|
79
|
+
center: [
|
|
80
|
+
header.centerLon,
|
|
81
|
+
header.centerLat,
|
|
82
|
+
header.centerZoom || Math.round(header.maxZoom / 2),
|
|
83
|
+
],
|
|
84
|
+
tileCount: header.numAddressedTiles,
|
|
85
|
+
clustered: header.clustered,
|
|
86
|
+
name: metadata.name,
|
|
87
|
+
description: metadata.description,
|
|
88
|
+
attribution: metadata.attribution,
|
|
89
|
+
vectorLayers: metadata.vector_layers,
|
|
90
|
+
};
|
|
91
|
+
} finally {
|
|
92
|
+
source.close?.();
|
|
93
|
+
}
|
|
94
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bridges the seeding engine onto the read engine pmtiles-torrent expects.
|
|
3
|
+
*
|
|
4
|
+
* The two interfaces answer different questions about the same torrent —
|
|
5
|
+
* "how is this doing in the swarm" versus "give me these bytes" — and both are
|
|
6
|
+
* satisfied by one client. Running a second client for the read side would mean
|
|
7
|
+
* two sessions on two ports contending for the same save path, so the bridge
|
|
8
|
+
* always reuses what the seeding engine already has open.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/** libtorrent piece priorities. 0 means "do not fetch". */
|
|
12
|
+
const LT_PRIORITY = { critical: 7, high: 4, normal: 1 };
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* A TorrentEngine over the libtorrent sidecar the seed engine already runs.
|
|
16
|
+
*
|
|
17
|
+
* pmtiles-torrent ships its own LibtorrentEngine, but that one spawns and owns
|
|
18
|
+
* a sidecar. Here the sidecar belongs to the seeding engine and is already
|
|
19
|
+
* holding the torrent, so this speaks to it through the seed engine rather than
|
|
20
|
+
* starting a rival session.
|
|
21
|
+
* @implements {import('pmtiles-torrent').TorrentEngine}
|
|
22
|
+
*/
|
|
23
|
+
export class LibtorrentReadEngine {
|
|
24
|
+
#engine;
|
|
25
|
+
#infoHash;
|
|
26
|
+
#info = null;
|
|
27
|
+
#pending = null;
|
|
28
|
+
#pieceTimeoutMs;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* @param {object} engine - The seeding LibtorrentEngine.
|
|
32
|
+
* @param {string} infoHash - Torrent to read from.
|
|
33
|
+
* @param {object} [options] - Per-piece timeout.
|
|
34
|
+
*/
|
|
35
|
+
constructor(engine, infoHash, options = {}) {
|
|
36
|
+
this.#engine = engine;
|
|
37
|
+
this.#infoHash = infoHash;
|
|
38
|
+
this.#pieceTimeoutMs = options.pieceTimeoutMs ?? 120000;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A stable key available before metadata arrives.
|
|
43
|
+
* @returns {string} - Cache key for PMTiles.
|
|
44
|
+
*/
|
|
45
|
+
get key() {
|
|
46
|
+
return `torrent:${this.#infoHash}`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Resolves once torrent metadata is available.
|
|
51
|
+
* @returns {Promise<object>} - Piece geometry and file extent.
|
|
52
|
+
*/
|
|
53
|
+
ready() {
|
|
54
|
+
// Idempotent by contract, and PMTiles calls it on every read.
|
|
55
|
+
this.#pending ??= (async () => {
|
|
56
|
+
const info = await this.#engine.info(this.#infoHash);
|
|
57
|
+
this.#info = info;
|
|
58
|
+
return info;
|
|
59
|
+
})().catch((error) => {
|
|
60
|
+
this.#pending = null;
|
|
61
|
+
throw error;
|
|
62
|
+
});
|
|
63
|
+
return this.#pending;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Reads a byte range from the archive.
|
|
68
|
+
*
|
|
69
|
+
* The source above only ever asks for whole pieces clipped to the file, so a
|
|
70
|
+
* range that straddles a piece boundary is a caller bug and is reported
|
|
71
|
+
* rather than quietly stitched together.
|
|
72
|
+
* @param {number} offset - Byte offset into the archive file.
|
|
73
|
+
* @param {number} length - Byte count.
|
|
74
|
+
* @param {object} [options] - Abort signal and priority.
|
|
75
|
+
* @returns {Promise<Uint8Array>} - Exactly `length` bytes.
|
|
76
|
+
*/
|
|
77
|
+
async readRange(offset, length, options = {}) {
|
|
78
|
+
const info = this.#info ?? (await this.ready());
|
|
79
|
+
const globalStart = info.fileOffset + offset;
|
|
80
|
+
const first = Math.floor(globalStart / info.pieceLength);
|
|
81
|
+
const last = Math.floor((globalStart + length - 1) / info.pieceLength);
|
|
82
|
+
if (first !== last) {
|
|
83
|
+
throw new Error(
|
|
84
|
+
`read of ${length}B at ${offset} spans pieces ${first}-${last}; ` +
|
|
85
|
+
'the source is expected to split reads on piece boundaries',
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (options.signal?.aborted) throw abortError();
|
|
90
|
+
const piece = await this.#engine.readPiece(this.#infoHash, first, {
|
|
91
|
+
deadlineMs: 0,
|
|
92
|
+
timeoutMs: this.#pieceTimeoutMs,
|
|
93
|
+
});
|
|
94
|
+
if (options.signal?.aborted) throw abortError();
|
|
95
|
+
|
|
96
|
+
const within = globalStart - first * info.pieceLength;
|
|
97
|
+
const slice = piece.subarray(within, within + length);
|
|
98
|
+
if (slice.byteLength !== length) {
|
|
99
|
+
throw new Error(
|
|
100
|
+
`short read: wanted ${length}B at ${offset}, got ${slice.byteLength}B`,
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
return slice;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Raises a range's priority so it downloads in the background.
|
|
108
|
+
* @param {number} offset - Byte offset into the archive file.
|
|
109
|
+
* @param {number} length - Byte count.
|
|
110
|
+
* @param {string} priority - critical, high or normal.
|
|
111
|
+
* @returns {void}
|
|
112
|
+
*/
|
|
113
|
+
hint(offset, length, priority) {
|
|
114
|
+
const range = this.#pieceRange(offset, length);
|
|
115
|
+
if (!range) return;
|
|
116
|
+
this.#engine
|
|
117
|
+
.setPriority(
|
|
118
|
+
this.#infoHash,
|
|
119
|
+
range.first,
|
|
120
|
+
range.last,
|
|
121
|
+
LT_PRIORITY[priority] ?? 1,
|
|
122
|
+
)
|
|
123
|
+
.catch(() => {});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Drops a range back to priority 0, so it stops competing for bandwidth.
|
|
128
|
+
* @param {number} offset - Byte offset into the archive file.
|
|
129
|
+
* @param {number} length - Byte count.
|
|
130
|
+
* @returns {void}
|
|
131
|
+
*/
|
|
132
|
+
unhint(offset, length) {
|
|
133
|
+
const range = this.#pieceRange(offset, length);
|
|
134
|
+
if (!range) return;
|
|
135
|
+
this.#engine
|
|
136
|
+
.setPriority(this.#infoHash, range.first, range.last, 0)
|
|
137
|
+
.catch(() => {});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Releases the reader. The sidecar belongs to the seeding engine and keeps
|
|
142
|
+
* running — the torrent is still being seeded.
|
|
143
|
+
* @returns {void}
|
|
144
|
+
*/
|
|
145
|
+
destroy() {
|
|
146
|
+
this.#pending = null;
|
|
147
|
+
this.#info = null;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Maps a file-relative byte range onto torrent-global piece indices.
|
|
152
|
+
* @param {number} offset - Byte offset into the archive file.
|
|
153
|
+
* @param {number} length - Byte count.
|
|
154
|
+
* @returns {{first: number, last: number} | null} - Piece range, or null before metadata.
|
|
155
|
+
*/
|
|
156
|
+
#pieceRange(offset, length) {
|
|
157
|
+
const info = this.#info;
|
|
158
|
+
if (!info || length <= 0) return null;
|
|
159
|
+
const start = info.fileOffset + offset;
|
|
160
|
+
return {
|
|
161
|
+
first: Math.floor(start / info.pieceLength),
|
|
162
|
+
last: Math.floor((start + length - 1) / info.pieceLength),
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Builds the abort rejection PMTiles and the source layer expect.
|
|
169
|
+
* @returns {Error} - An AbortError.
|
|
170
|
+
*/
|
|
171
|
+
function abortError() {
|
|
172
|
+
const error = new Error('The operation was aborted.');
|
|
173
|
+
error.name = 'AbortError';
|
|
174
|
+
return error;
|
|
175
|
+
}
|