pmtiles-swarm 0.32.1 → 0.32.3

package/CHANGELOG.md

@@ -7,6 +7,45 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.32.3
+### ✨ Features and improvements
+- _...Add new stuff here..._
+
+### 🐞 Bug fixes
+- **Requires pmtiles-torrent 0.5.2, which deletes a torrent's resume file with its data.** Resume
+  data was outliving the data it described: deleting an archive to re-fetch it left the record of
+  the old complete file behind, so the re-add handed libtorrent a description of a finished archive
+  against a path holding a fresh partial one. It answered `fastresume_rejected` and rechecked, and
+  until that settled nothing was verified — bytes arriving at full speed against a verified-piece
+  count stuck at 1, and every tile read told the piece it wanted was not in the slot list. Only when
+  the data goes too: a removal that keeps the files is how a pause is expressed for an engine with
+  no pause of its own, and discarding resume data there would turn every pause into a full re-hash.
+
+- _...Add new stuff here..._
+
+## 0.32.2
+### ✨ Features and improvements
+- _...Add new stuff here..._
+
+### 🐞 Bug fixes
+- **The head warmer no longer skips every archive it was built for.** It decided an archive was done
+  by `Boolean(entry.pmtiles?.format)` — "a summary that names a format is one a header was read
+  for". That is not what a format means. A feed carries format, zoom range and bounds precisely so a
+  subscriber can judge a 698 GiB download before starting one, so a subscribed archive arrives fully
+  summarised before a byte of it exists locally. Every one of them was therefore retired on the spot,
+  and retired silently: the runner logged "every archive has been summarised; nothing to warm" and
+  meant it. The archives most in need of their header were the only ones never offered one, and the
+  visible symptom was a readable TileJSON — served from that same feed summary, touching no bytes —
+  beside tile reads that failed and previews that came up empty.
+
+  Entries now record where the summary came from: `summarySource: 'header'` where an archive's own
+  header answered, `'feed'` where a subscription was told. An entry written before this has neither,
+  and is treated as unread — the self-healing direction, since a local archive re-reads its own
+  header off local disk for nothing, while assuming the opposite would leave every existing
+  subscription stuck exactly as it is.
+
+- _...Add new stuff here..._
+
 ## 0.32.1
 ### ✨ Features and improvements
 - _...Add new stuff here..._

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.32.1",
+  "version": "0.32.3",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",
@@ -46,7 +46,7 @@
     "maplibre-gl": "^6.2.0",
     "parse-torrent": "^11.0.24",
     "pmtiles": "^4.4.1",
-    "pmtiles-torrent": "^0.5.1",
+    "pmtiles-torrent": "^0.5.2",
     "webtorrent": "^3.0.21"
   },
   "engines": {

package/src/api.js

@@ -2168,6 +2168,7 @@ export function createApp({
           entry = await catalog.put({
             infoHash: entry.infoHash,
             pmtiles: summary,
+            summarySource: 'header',
           });
         } catch (error) {
           // The content had its say. Record it so this is answered from the
@@ -2383,6 +2384,7 @@ export function createApp({
           entry = await catalog.put({
             infoHash: entry.infoHash,
             pmtiles: summary,
+            summarySource: 'header',
           });
         }
       }

package/src/library.js

@@ -204,6 +204,10 @@ export class Library {
       // The torrent names the file, so the save path is its parent directory.
       savePath: path.dirname(absolute),
       pmtiles: summary,
+      // Read out of the archive, not taken from anybody's word for it. The
+      // prewarmer needs the difference: a summary that arrived in a feed says
+      // nothing about whether the header is on this disk. See prewarm.due().
+      summarySource: 'header',
       kind: identified.kind,
       sparse: options.sparse,
       md5: created.md5,
@@ -747,6 +751,10 @@ export class Library {
       buildDate: options.buildDate,
       savePath,
       pmtiles: summary,
+      // Read out of the archive, not taken from anybody's word for it. The
+      // prewarmer needs the difference: a summary that arrived in a feed says
+      // nothing about whether the header is on this disk. See prewarm.due().
+      summarySource: 'header',
       kind: identified.kind,
       sparse: options.sparse,
       md5: created.md5,
@@ -1212,6 +1220,10 @@ export class Library {
       magnet,
       webSeeds: [],
       pmtiles: summary,
+      // Read out of the archive, not taken from anybody's word for it. The
+      // prewarmer needs the difference: a summary that arrived in a feed says
+      // nothing about whether the header is on this disk. See prewarm.due().
+      summarySource: 'header',
       kind: guessKind(torrent.name ?? ''),
       mode: 'mirror',
       // The engine's own account of it. It is on disk under its real name

package/src/prewarm.js

@@ -157,12 +157,23 @@ export class HeadWarmer {
     const kind = entry.kind ?? guessKind(entry.name ?? '');
     if (kind !== 'pmtiles') return false;
 
-    // A summary that names a format is one a header was actually read for.
-    // Anything else — an empty object, or one left behind by a read that raced
-    // its deadline — is not an answer, and treating it as one retired the
-    // archive permanently: no logs, no retries, nothing to explain the silence.
+    // A summary is only an answer about *this disk* if a header on this disk
+    // produced it. `format` was standing in for that and does not mean it.
+    //
+    // The feed carries format, zoom range and bounds precisely so a subscriber
+    // can judge a 698 GiB download before starting one -- so a subscribed
+    // archive arrives fully summarised, before a byte of it exists here. Read
+    // as "a header was read", that retired every subscribed archive on the spot
+    // and did it silently, since as far as this was concerned there was nothing
+    // to do. The archives most in need of their header were the only ones never
+    // offered one, and the log said "nothing to warm".
+    //
+    // An entry from before this was recorded has no source, and is treated as
+    // unread. That is the self-healing direction: a local archive re-reads its
+    // own header off local disk and costs nothing, where the other way round
+    // leaves every existing subscription stuck exactly as it was.
     const summary = entry.pmtiles;
-    const read = Boolean(summary?.format);
+    const read = entry.summarySource === 'header' && Boolean(summary?.format);
     if (read && (summary.format !== 'pbf' || summary.vectorLayers)) {
       return false;
     }
@@ -257,6 +268,9 @@ export class HeadWarmer {
       const stored = await this.#catalog.put({
         infoHash: entry.infoHash,
         pmtiles: { ...entry.pmtiles, ...summary },
+        // The whole point of this pass: from here on the entry's summary is
+        // one the header answered for, so due() stops choosing it.
+        summarySource: 'header',
       });
       // Said accurately, because the two halves arrive separately and the
       // difference matters: the header is at byte zero, while the JSON

package/src/subscriptions.js

@@ -419,6 +419,10 @@ export class SubscriptionManager {
       // with what the archive itself says; this only means the wait is not
       // spent unservable.
       pmtiles: item.pmtiles,
+      // Told, not read. The feed carries this so a subscriber can judge a
+      // 698 GiB download before starting one, which means it is populated
+      // before a single byte exists here.
+      summarySource: 'feed',
     };
 
     // The .torrent is preferred where there is one: it carries the trackers