pmtiles-swarm 0.5.3 → 0.5.5

package/CHANGELOG.md

@@ -7,6 +7,33 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.5.5
+### 🐞 Bug fixes
+- **An archive could retire itself from head-warming and never say so.** The test for "already
+  read" was `summary.format !== 'pbf'`, and any stored summary *without* a format satisfies
+  that — so it counted as read, not vector, and therefore finished. A read that raced its
+  deadline leaves exactly such a summary behind, and from then on the archive was permanently
+  ineligible: no attempts, no log lines, and nothing to explain the silence. A summary now only
+  counts as an answer if it actually names a format.
+- **A read that never settles no longer disables warming for good.** The flag that stops two
+  reads running at once was held for the life of the process by a read that never returned, and
+  every later pass then returned at its first line — silently, for every archive. It is
+  abandoned after three times the metadata timeout, with a line saying so.
+
+## 0.5.4
+### ✨ Features and improvements
+- **The head-warm waits sensibly at both ends.** It used to read at the instant the node
+  started — when an archive joined by magnet has no metainfo, the engine has no peers, and the
+  attempt is certain to find nothing — and then leave a flat two minutes between every try
+  afterwards. Neither suited what was actually being waited for. The first pass now comes ten
+  seconds in, and the wait after an attempt that did not finish starts at fifteen seconds and
+  doubles to a ten-minute ceiling: seconds early on, when what is missing is usually a peer or a
+  piece already in flight, and minutes later on, when it is one piece at the far end of an
+  archive nobody has finished downloading.
+
+  New under `tiles`: `prewarmInitialDelaySeconds` (10) and `prewarmMaxBackoffSeconds` (600).
+  `prewarmBackoffSeconds` is now the *first* wait rather than every wait, and defaults to 15.
+
 ## 0.5.3
 ### 🐞 Bug fixes
 - **A "high" priority hint asked for nothing at all.** libtorrent's scale runs 0 to 7 and **4 is

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.5.3",
+  "version": "0.5.5",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",

package/src/config.js

@@ -460,13 +460,28 @@ const DEFAULTS = {
     /** How often to look for an archive whose head has not been read. */
     prewarmIntervalSeconds: 30,
     /**
-     * How long to leave an archive alone after a failed read.
+     * How long to let the node settle before the first attempt.
      *
-     * Failure here is ordinary rather than exceptional — a brand-new archive
-     * may have no peer holding its first piece yet — so this is a retry
-     * interval, not an error budget.
+     * At the moment a node starts, an archive joined by magnet has no metainfo
+     * and the engine has no peers, so a read attempted immediately is certain
+     * to find nothing.
      */
-    prewarmBackoffSeconds: 120,
+    prewarmInitialDelaySeconds: 10,
+    /**
+     * The first wait after an attempt that did not finish the job, doubling up
+     * to `prewarmMaxBackoffSeconds`.
+     *
+     * Not finishing is ordinary rather than exceptional here, so this is a
+     * retry interval and not an error budget — and it doubles because the two
+     * ends of the problem want different answers. Just after a start, what is
+     * being waited for is usually seconds away: a peer, a connection, a piece
+     * already in flight. Ten attempts later it is one piece at the far end of
+     * an archive nobody has finished, and asking every fifteen seconds
+     * achieves nothing but log lines.
+     */
+    prewarmBackoffSeconds: 15,
+    /** Where the doubling stops. */
+    prewarmMaxBackoffSeconds: 600,
     /**
      * What a missing tile answers with: true for 404, false for 204.
      *

package/src/prewarm.js

@@ -21,8 +21,14 @@
  * the first few seconds if the right few kilobytes are asked for first.
  */
 
-/** How long to leave an archive alone after a failed attempt. */
-const DEFAULT_BACKOFF_MS = 120000;
+/** The first wait after an attempt that did not finish the job. */
+const DEFAULT_BACKOFF_SECONDS = 15;
+
+/** Where the doubling stops. */
+const DEFAULT_MAX_BACKOFF_SECONDS = 600;
+
+/** How long to let the node settle before the first attempt. */
+const DEFAULT_INITIAL_DELAY_SECONDS = 10;
 
 /**
  * Whether a failure means "not yet" rather than "not working".
@@ -49,8 +55,10 @@ export class HeadWarmer {
   #now;
   #timer;
   #tried = new Map();
+  #attempts = new Map();
   #waiting = new Set();
   #running = false;
+  #runningSince = 0;
 
   /**
    * @param {object} tiles - The tile store, for `summarize`.
@@ -87,14 +95,41 @@ export class HeadWarmer {
     // archive this can say anything about.
     if (entry.kind && entry.kind !== 'pmtiles') return false;
 
+    // A summary that names a format is one a header was actually read for.
+    // Anything else — an empty object, or one left behind by a read that raced
+    // its deadline — is not an answer, and treating it as one retired the
+    // archive permanently: no logs, no retries, nothing to explain the silence.
     const summary = entry.pmtiles;
-    if (summary && (summary.format !== 'pbf' || summary.vectorLayers)) {
+    const read = Boolean(summary?.format);
+    if (read && (summary.format !== 'pbf' || summary.vectorLayers)) {
       return false;
     }
 
     const last = this.#tried.get(entry.infoHash) ?? 0;
-    const backoff = (this.#config.tiles?.prewarmBackoffSeconds ?? 120) * 1000;
-    return this.#now() - last >= (backoff || DEFAULT_BACKOFF_MS);
+    return this.#now() - last >= this.#backoffFor(entry.infoHash);
+  }
+
+  /**
+   * How long to wait before trying this archive again.
+   *
+   * Doubling, from a short first wait to a long ceiling. A flat interval is
+   * wrong at both ends: right after a node starts, the thing being waited for
+   * is usually a few seconds away — a peer, a connection, a piece already in
+   * flight — so a two-minute wait wastes most of it. Ten attempts later the
+   * thing being waited for is a single piece at the far end of an archive
+   * nobody has finished downloading, and asking every two minutes achieves
+   * nothing but log lines.
+   * @param {string} infoHash - The archive.
+   * @returns {number} - Milliseconds to wait.
+   */
+  #backoffFor(infoHash) {
+    const base =
+      (this.#config.tiles?.prewarmBackoffSeconds ?? DEFAULT_BACKOFF_SECONDS) * 1000;
+    const ceiling =
+      (this.#config.tiles?.prewarmMaxBackoffSeconds ?? DEFAULT_MAX_BACKOFF_SECONDS) *
+      1000;
+    const attempts = this.#attempts.get(infoHash) ?? 0;
+    return Math.min(base * 2 ** Math.max(0, attempts - 1), ceiling);
   }
 
   /**
@@ -108,12 +143,27 @@ export class HeadWarmer {
    * @returns {Promise<object|null>} - The entry warmed, or null.
    */
   async sweep() {
-    if (!this.enabled || this.#running) return null;
+    if (!this.enabled) return null;
+
+    if (this.#running) {
+      // A read that never settles would otherwise hold this flag for the life
+      // of the process and stop every archive from ever being warmed again —
+      // silently, since nothing logs a pass that returns at the first line.
+      const stuck = (this.#config.tiles?.metadataTimeoutMs ?? 120000) * 3;
+      if (this.#now() - this.#runningSince < stuck) return null;
+      console.warn(
+        `[warm] a read has been running for ${Math.round(
+          (this.#now() - this.#runningSince) / 1000,
+        )}s and is being abandoned`,
+      );
+      this.#running = false;
+    }
 
     const entry = this.#catalog.list().find((candidate) => this.due(candidate));
     if (!entry) return null;
 
     this.#running = true;
+    this.#runningSince = this.#now();
     try {
       // The long timeout, not the interactive one: this is a byte range from
       // an archive nobody has asked for a piece of yet.
@@ -123,6 +173,14 @@ export class HeadWarmer {
 
       this.#tried.set(entry.infoHash, this.#now());
       this.#waiting.delete(entry.infoHash);
+      // Counted even on success, because a read that got the header and not
+      // the metadata has not finished and will be back. The count only matters
+      // while an archive is still due, and one that is done is never asked
+      // about again.
+      this.#attempts.set(
+        entry.infoHash,
+        (this.#attempts.get(entry.infoHash) ?? 0) + 1,
+      );
 
       const stored = await this.#catalog.put({
         infoHash: entry.infoHash,
@@ -169,6 +227,10 @@ export class HeadWarmer {
       // is my preview blank", and the backoff keeps it from becoming noise.
       this.#tried.set(entry.infoHash, this.#now());
       this.#waiting.delete(entry.infoHash);
+      this.#attempts.set(
+        entry.infoHash,
+        (this.#attempts.get(entry.infoHash) ?? 0) + 1,
+      );
       console.warn(`[warm] ${entry.name}: ${error.message}`);
       return null;
     } finally {
@@ -189,7 +251,17 @@ export class HeadWarmer {
       this.sweep().catch((error) =>
         console.error(`[warm] sweep failed: ${error.message}`),
       );
-    run();
+
+    // Not immediately. At the moment a node starts, an archive joined by
+    // magnet has no metainfo, the engine has no peers, and nothing can be read
+    // from anywhere — so the first pass is guaranteed to find nothing and
+    // exists only to say so.
+    const delay =
+      (this.#config.tiles?.prewarmInitialDelaySeconds ??
+        DEFAULT_INITIAL_DELAY_SECONDS) * 1000;
+    const first = setTimeout(run, Math.max(0, delay));
+    first.unref?.();
+
     this.#timer = setInterval(run, seconds * 1000);
     this.#timer.unref?.();
   }