pmtiles-swarm 0.4.6 → 0.5.0

package/CHANGELOG.md

@@ -7,6 +7,29 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.5.0
+### ✨ Features and improvements
+- **The head of a newly joined archive is read without waiting to be asked.** A PMTiles archive
+  is useless until its 127-byte header has been read: it names where the root directory and the
+  JSON metadata live, and reading it raises both to a high piece priority — so the head of the
+  file arrives out of order instead of whenever a download happens to reach byte zero. That
+  machinery existed and was spec-correct, but only ran when something read the archive, and the
+  backfill that would have followed up began by requiring a summary to already exist. A freshly
+  joined archive has none, and the one thing that would have created one was the TileJSON route,
+  which is exactly what fails without a header. So an archive being mirrored stayed unservable
+  for hours while the few kilobytes that would have made it servable sat at position zero.
+
+  It now reads one archive's head at a time — several at once turn a queue of archives into a
+  queue of stalled reads competing for the same bandwidth — with the long metadata timeout
+  rather than the interactive one, backing off between attempts, because a young archive having
+  no peer that holds its first piece is ordinary rather than exceptional. It comes back for
+  vector layers separately, since a writer may put the JSON metadata after every tile and one
+  read routinely gets the header and not the metadata.
+
+  New under `tiles`: `prewarm` (default true), `prewarmIntervalSeconds` (30) and
+  `prewarmBackoffSeconds` (120). Turn it off on a node that distributes archives but never
+  serves tiles from them.
+
 ## 0.4.6
 ### ✨ Features and improvements
 - **A feed's categories are a list, and are called categories.** The console offered a single

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.4.6",
+  "version": "0.5.0",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",

package/src/config.js

@@ -444,6 +444,29 @@ const DEFAULTS = {
      * like a hang rather than like "not yet".
      */
     headerTimeoutMs: 12000,
+    /**
+     * Read the head of a newly joined archive without waiting to be asked.
+     *
+     * A PMTiles archive is useless until its header has been read: it names
+     * where the root directory and the JSON metadata are, and reading it also
+     * raises those to a high piece priority, so the head of the file arrives
+     * out of order rather than whenever a download happens to reach it. Left
+     * to the first request, an archive being mirrored is unservable for hours
+     * while the bytes that would make it servable sit at position zero.
+     *
+     * Off makes sense on a node that only distributes and never serves tiles.
+     */
+    prewarm: true,
+    /** How often to look for an archive whose head has not been read. */
+    prewarmIntervalSeconds: 30,
+    /**
+     * How long to leave an archive alone after a failed read.
+     *
+     * Failure here is ordinary rather than exceptional — a brand-new archive
+     * may have no peer holding its first piece yet — so this is a retry
+     * interval, not an error budget.
+     */
+    prewarmBackoffSeconds: 120,
     /**
      * What a missing tile answers with: true for 404, false for 204.
      *

package/src/index.js

@@ -19,6 +19,7 @@ import { closeServer, installSignalHandlers, runStoppers } from './shutdown.js';
 import { ScheduledSourceManager } from './sources.js';
 import { SubscriptionManager } from './subscriptions.js';
 import { TileStore } from './tiles.js';
+import { HeadWarmer } from './prewarm.js';
 import { WarmRunner } from './warm.js';
 import { WatchManager } from './watch.js';
 
@@ -216,6 +217,12 @@ PMTILES_SWARM_PUBLIC_URL
   library.attachTiles(tiles);
   const warm = new WarmRunner(tiles);
 
+  // Reads the head of anything joined but not yet understood — the header,
+  // then the root directory and metadata it points at. Without this an archive
+  // being mirrored is unservable until the download happens to reach byte
+  // zero, and the request that would have read it times out long before.
+  const headWarmer = new HeadWarmer(tiles, catalog, config);
+
   // Restarting one subsystem, rather than the process, for the settings that
   // only that subsystem reads. Each stops and starts from the live config, so
   // nothing here has to know what changed — only what to rebuild.
@@ -337,6 +344,11 @@ PMTILES_SWARM_PUBLIC_URL
   }
   hooks.start();
   completion.start();
+  // The header is 127 bytes and names where the root directory and the JSON
+  // metadata live (PMTiles v3 spec, fields at offsets 8 and 24). Reading it
+  // raises both to a high piece priority, so an archive becomes servable in
+  // seconds rather than at whatever hour the download reaches byte zero.
+  headWarmer.start();
 
   // Watch the sources archives were built from. A changed source does not
   // invalidate its torrent, but it does mean any web seed pointing there will
@@ -375,6 +387,7 @@ PMTILES_SWARM_PUBLIC_URL
       completion.stop();
       subscriptions.stop();
       warm.stop();
+      headWarmer.stop();
     }, ms: 1000 },
     { label: 'watchers', stop: () => watch.stop() },
     {

package/src/prewarm.js

@@ -0,0 +1,152 @@
+/**
+ * Reading the head of a newly joined archive, before anybody asks for it.
+ *
+ * A PMTiles archive is useless until its header has been read: the header is
+ * the first 127 bytes and names where the root directory and the JSON metadata
+ * live, and without it there is no TileJSON, no vector layers, and a preview
+ * that renders black. Reading it also *prioritises* what it found — the source
+ * hints the root directory as critical and the metadata as high — so the head
+ * of the file arrives out of order rather than whenever the download reaches
+ * it.
+ *
+ * None of that happened on its own. The read is on the interactive path, so it
+ * ran when somebody opened the archive; and the backfill that follows it up
+ * required a summary to already exist, which is precisely what a freshly joined
+ * archive does not have. So the first request paid for the header, and if it
+ * timed out first — which against a 72 GiB mirror with no web seed it does —
+ * nothing ever tried again.
+ *
+ * A mirror gets there eventually by downloading everything. The point of doing
+ * it deliberately is that "eventually" is hours, and the archive is servable in
+ * the first few seconds if the right few kilobytes are asked for first.
+ */
+
+/** How long to leave an archive alone after a failed attempt. */
+const DEFAULT_BACKOFF_MS = 120000;
+
+export class HeadWarmer {
+  #tiles;
+  #catalog;
+  #config;
+  #now;
+  #timer;
+  #tried = new Map();
+  #running = false;
+
+  /**
+   * @param {object} tiles - The tile store, for `summarize`.
+   * @param {object} catalog - Where the summary is written.
+   * @param {object} config - Resolved configuration.
+   * @param {Function} [now] - Clock, for testing.
+   */
+  constructor(tiles, catalog, config, now = () => Date.now()) {
+    this.#tiles = tiles;
+    this.#catalog = catalog;
+    this.#config = config;
+    this.#now = now;
+  }
+
+  /** @returns {boolean} - Whether this node can and should warm anything. */
+  get enabled() {
+    return (
+      this.#config.tiles?.prewarm !== false &&
+      typeof this.#tiles?.summarize === 'function'
+    );
+  }
+
+  /**
+   * Whether an archive is worth reading the head of right now.
+   *
+   * Deliberately narrow: an archive that has been summarised is done, unless it
+   * is vector and its layers are still missing — that section sits at the far
+   * end of the file and routinely arrives later than the header.
+   * @param {object} entry - A catalog entry.
+   * @returns {boolean} - True to attempt a read.
+   */
+  due(entry) {
+    // Only PMTiles has a head worth reading. A .osm.pbf from a feed is not an
+    // archive this can say anything about.
+    if (entry.kind && entry.kind !== 'pmtiles') return false;
+
+    const summary = entry.pmtiles;
+    if (summary && (summary.format !== 'pbf' || summary.vectorLayers)) {
+      return false;
+    }
+
+    const last = this.#tried.get(entry.infoHash) ?? 0;
+    const backoff = (this.#config.tiles?.prewarmBackoffSeconds ?? 120) * 1000;
+    return this.#now() - last >= (backoff || DEFAULT_BACKOFF_MS);
+  }
+
+  /**
+   * Reads the head of one archive that needs it.
+   *
+   * One per pass on purpose. Each read is a byte range fetched out of a swarm
+   * that may have no peer holding it yet, and starting several at once turns a
+   * queue of archives into a queue of stalled reads competing for the same
+   * bandwidth — the same mistake as prefetching leaf directories eagerly, which
+   * measured four times slower than not bothering.
+   * @returns {Promise<object|null>} - The entry warmed, or null.
+   */
+  async sweep() {
+    if (!this.enabled || this.#running) return null;
+
+    const entry = this.#catalog.list().find((candidate) => this.due(candidate));
+    if (!entry) return null;
+
+    this.#running = true;
+    this.#tried.set(entry.infoHash, this.#now());
+    try {
+      // The long timeout, not the interactive one: this is a byte range from
+      // an archive nobody has asked for a piece of yet.
+      const summary = await this.#tiles.summarize(entry.infoHash, {
+        timeoutMs: this.#config.tiles?.metadataTimeoutMs ?? 120000,
+      });
+
+      const stored = await this.#catalog.put({
+        infoHash: entry.infoHash,
+        pmtiles: { ...entry.pmtiles, ...summary },
+      });
+      console.log(
+        `[warm] read the head of ${entry.name}` +
+          (summary.vectorLayers
+            ? ` (${summary.vectorLayers.length} vector layers)`
+            : ''),
+      );
+      return stored;
+    } catch (error) {
+      // Ordinary while an archive is young — the piece holding the header may
+      // not exist anywhere reachable yet. Worth saying once per attempt, since
+      // it is the answer to "why is my preview blank", and the backoff keeps
+      // it from becoming noise.
+      console.warn(`[warm] ${entry.name}: ${error.message}`);
+      return null;
+    } finally {
+      this.#running = false;
+    }
+  }
+
+  /**
+   * Starts warming, and keeps at it.
+   * @returns {void}
+   */
+  start() {
+    if (!this.enabled) return;
+    const seconds = this.#config.tiles?.prewarmIntervalSeconds ?? 30;
+    if (seconds <= 0) return;
+
+    const run = () =>
+      this.sweep().catch((error) =>
+        console.error(`[warm] sweep failed: ${error.message}`),
+      );
+    run();
+    this.#timer = setInterval(run, seconds * 1000);
+    this.#timer.unref?.();
+  }
+
+  /** Stops warming. @returns {void} */
+  stop() {
+    if (this.#timer) clearInterval(this.#timer);
+    this.#timer = undefined;
+  }
+}