pmtiles-swarm 0.55.0 → 0.55.2

package/CHANGELOG.md

@@ -7,6 +7,31 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.55.2
+### 🐞 Bug fixes
+- **The stale-summary re-read never ran for a category URL.** 0.55.1 added it to
+  `/archives/<infohash>/tiles.json` and nowhere else, which missed exactly the archives that matter:
+  `/latest/<category>/tiles.json` is the URL a style points at, so a category consumed through the
+  documented path was the one place a summary stayed stale for ever. An archive only healed if
+  somebody happened to ask for it by infohash.
+
+## 0.55.1
+### 🐞 Bug fixes
+- **0.55.0 reached new archives only.** `encoding` sits in the metadata of archives some nodes have
+  been serving for months, the prober learned to read it, and nothing changed — because a summary is
+  written into the catalog once and never questioned. Every path that re-read one was gated on the
+  summary being *absent*, and these summaries were present, merely old.
+
+  The summary now carries a `summaryVersion`, and one written by an older prober is re-read once, in
+  the background, on the next TileJSON request for that archive. Rate-limited to once a minute per
+  archive, as the vector-layer backfill beside it already was, and written back on any read that
+  produced something newer — the old early return threw away everything except vector layers, which
+  would have discarded the encoding it went to fetch.
+
+  This is general, not a fix for one field: raise `SUMMARY_VERSION` whenever the prober learns to
+  read something new, and every archive already in the catalog picks it up on its own. Nothing is
+  needed on an upgrade beyond asking for the archive's TileJSON, which anything using it does anyway.
+
 ## 0.55.0
 ### ✨ Features and improvements
 - **The TileJSON now carries a `raster-dem` archive's `encoding`**, read out of the archive's own

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.55.0",
+  "version": "0.55.2",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",

package/src/api.js

@@ -33,6 +33,7 @@ import {
 } from './sources.js';
 import { limitFor, remaining } from './seeding.js';
 import { buildTileJson, extensionMatches } from './tilejson.js';
+import { SUMMARY_VERSION } from './pmtiles-probe.js';
 import { TileReadError } from './tiles.js';
 
 const here = path.dirname(fileURLToPath(import.meta.url));
@@ -363,9 +364,18 @@ export function createApp({
    * @returns {boolean} - True to attempt a read now.
    */
   const metadataRetries = new Map();
-  const needsVectorLayers = (summary, infoHash) => {
-    if (!summary || summary.format !== 'pbf' || summary.vectorLayers)
-      return false;
+  const needsReread = (summary, infoHash) => {
+    if (!summary) return false;
+    // Two reasons to go back to the archive. The first is the one this began
+    // as: a vector archive whose layers have not been read yet. The second is
+    // the general case it turned out to be — a summary written by an older
+    // prober, missing whatever that prober did not know to look for. Without
+    // it, adding a field to the summary reaches new archives only, and every
+    // archive already in the catalog keeps its hole until somebody removes and
+    // re-adds it by hand.
+    const stale = summary.summaryVersion !== SUMMARY_VERSION;
+    const missingLayers = summary.format === 'pbf' && !summary.vectorLayers;
+    if (!stale && !missingLayers) return false;
     const last = metadataRetries.get(infoHash) ?? 0;
     if (Date.now() - last < 60000) return false;
     metadataRetries.set(infoHash, Date.now());
@@ -388,13 +398,18 @@ export function createApp({
     // .torrent endpoints work without one. Enriching a summary is the least
     // important thing here and must never be what takes a reply down.
     if (typeof tiles?.summarize !== 'function') return;
-    if (!needsVectorLayers(entry.pmtiles, entry.infoHash)) return;
+    if (!needsReread(entry.pmtiles, entry.infoHash)) return;
 
     const timeoutMs = config.tiles?.metadataTimeoutMs ?? 120000;
     tiles
       .summarize(entry.infoHash, { timeoutMs })
       .then(async (summary) => {
-        if (!summary.vectorLayers) return;
+        // Written back whenever the read produced something newer than what
+        // was stored. Returning early unless vector layers turned up was right
+        // when layers were the only thing this looked for; it would now throw
+        // away the encoding it went to fetch.
+        const fresh = summary.summaryVersion !== entry.pmtiles?.summaryVersion;
+        if (!summary.vectorLayers && !fresh) return;
         await catalog.put({
           infoHash: entry.infoHash,
           pmtiles: { ...entry.pmtiles, ...summary },
@@ -2259,6 +2274,12 @@ export function createApp({
         });
       }
 
+      // The same background re-read the per-archive route does. Left out
+      // here, it missed exactly the archives that matter most: /latest/ is the
+      // URL a style points at, so a category anyone actually consumes through
+      // the documented path was the one place a stale summary never healed.
+      startMetadataBackfill(entry);
+
       res.setHeader('access-control-allow-origin', '*');
       // Short-lived, and the only mutable document in the system: everything
       // it points at is content-addressed and cached for a year, and this is

package/src/pmtiles-probe.js

@@ -110,6 +110,24 @@ export function customEncodingFactors(metadata) {
  * @param {string} location - Local path or http(s) URL.
  * @returns {Promise<PMTilesSummary>} - The summary.
  */
+/**
+ * What this prober reads, as a number that goes up when that changes.
+ *
+ * A summary is stored in the catalog and never read again, which is right --
+ * re-reading a header out of the swarm is not free, and the answer does not
+ * change for a given infohash. It goes wrong the moment the prober learns to
+ * read something new: every archive probed before that keeps a summary with a
+ * hole in it, for ever, and the only way out was to remove and re-add it.
+ *
+ * `encoding` was the case that made this obvious. It sat in the metadata of
+ * archives this node had been serving for months, and adding the code to read
+ * it changed nothing at all, because nothing ever asked again.
+ *
+ * So: stamp what the prober knew at the time, and re-read once when that is
+ * behind. Raise this whenever a field is added to the summary below.
+ */
+export const SUMMARY_VERSION = 2;
+
 export function summarize(header, metadata = {}) {
   const type = TILE_TYPES[header.tileType] ?? TILE_TYPES[0];
 
@@ -123,6 +141,7 @@ export function summarize(header, metadata = {}) {
   );
 
   return {
+    summaryVersion: SUMMARY_VERSION,
     specVersion: header.specVersion,
     format: type.format,
     contentType: type.contentType,