pmtiles-swarm 0.14.0 → 0.14.2

package/CHANGELOG.md

@@ -7,6 +7,42 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.14.2
+### 🐞 Bug fixes
+- **Waiting for the DHT's `ready` event was not enough.** It fires when the bootstrap lookup
+  *finishes*, whether or not that lookup found anything — so a node with no working UDP path
+  reports itself ready and then fails every put with "No nodes to query", and the warning added in
+  0.14.1 never fired because nothing had gone wrong by its measure.
+
+  It now waits for the routing table to have something in it, and says how empty it is when it
+  gives up:
+
+  ```
+  [mutable] DHT ready with 42 nodes
+  [mutable] the DHT found no peers in 60s. Publishing will fail until it does — check that
+            outbound UDP is not blocked, and that the bootstrap hosts resolve
+  ```
+
+  Per-category failures carry the count too, because "No nodes to query" with an empty table is a
+  network problem and the same message with a populated one is not.
+
+## 0.14.1
+### 🐞 Bug fixes
+- **The publisher waited a fixed fifteen seconds for the DHT and then published into an empty
+  routing table**, so on a real node every category failed at once:
+
+  ```
+  [mutable] openmaptiles failed: failed to publish mutable record: No nodes to query
+  ```
+
+  A fixed delay was a bet on how long bootstrapping takes. It now waits for the DHT's own `ready`
+  event instead, which fires in a couple of seconds on a healthy network, and only gives up after
+  a minute — saying so when it does, because a DHT that never bootstraps means outbound UDP is not
+  getting out and every later failure is a consequence of that.
+- **A failed attempt no longer waits out the whole republish interval.** Nothing published meant
+  half an hour of advertising a public key that resolves to nothing before trying again. It now
+  retries after 30 seconds, backing off towards the interval.
+
 ## 0.14.0
 ### ✨ Features and improvements
 - **The Categories page hands you the URL a style should actually use.** A new **For a style** row

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.14.0",
+  "version": "0.14.2",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",

package/src/publisher.js

@@ -50,8 +50,11 @@ import { mutableMagnet, publishInfoHash } from './mutable.js';
 /** Republish well inside the ~2h a DHT keeps an item. */
 const DEFAULT_INTERVAL_MS = 30 * 60 * 1000;
 
-/** Grace period before the first publish, for the DHT to find peers. */
-const DEFAULT_READY_MS = 15_000;
+/** How long to wait for the DHT to bootstrap before publishing anyway. */
+const DEFAULT_READY_MS = 60_000;
+
+/** First retry delay after an attempt where nothing was published. */
+const RETRY_MS = 30_000;
 
 export class MutablePublisher {
   #catalog;
@@ -59,6 +62,7 @@ export class MutablePublisher {
   #key;
   #intervalMs;
   #timer = null;
+  #retryTimer = null;
   #stopped = false;
   #log;
   /** Last published infohash per category, so an unchanged build stays quiet. */
@@ -134,10 +138,19 @@ export class MutablePublisher {
           );
         }
       } catch (error) {
-        // One category failing must not stop the rest.
-        this.#log(`[mutable] ${category} failed: ${error.message}`);
+        // One category failing must not stop the rest. The node count goes in
+        // the line because "No nodes to query" with an empty table is a
+        // network problem and the same message with a full one is not.
+        const nodes = this.#nodeCount();
+        this.#log(
+          `[mutable] ${category} failed: ${error.message}` +
+            (nodes === null ? '' : ` (${nodes} DHT nodes known)`),
+        );
       }
     }
+    if (done.length === 0 && this.#current().size > 0) {
+      this.#scheduleRetry(options.retryDelayMs ?? RETRY_MS);
+    }
     return done;
   }
 
@@ -165,17 +178,14 @@ export class MutablePublisher {
    * @returns {void}
    */
   start(options = {}) {
-    // A put into a DHT that has not found peers yet reaches nobody, and
-    // bootstrapping is the first thing a fresh node does. Waiting costs one
-    // interval of staleness at worst and makes the first publish mean
-    // something.
-    const first = setTimeout(() => {
-      if (this.#stopped) return;
-      this.publishAll({ force: true }).catch((error) =>
-        this.#log(`[mutable] first publish failed: ${error.message}`),
-      );
-    }, options.readyMs ?? DEFAULT_READY_MS);
-    first.unref?.();
+    // Waited for rather than guessed at. A put into a DHT whose routing table
+    // is still empty fails with "No nodes to query", and a fixed delay is a
+    // bet on how long bootstrapping takes -- one this lost in the field, where
+    // fifteen seconds was not enough and every category failed on the first
+    // attempt.
+    this.#firstPublish(options.readyMs ?? DEFAULT_READY_MS).catch((error) =>
+      this.#log(`[mutable] first publish failed: ${error.message}`),
+    );
 
     this.#timer = setInterval(() => {
       this.publishAll().catch((error) =>
@@ -190,6 +200,98 @@ export class MutablePublisher {
     );
   }
 
+  /**
+   * Waits for the DHT to bootstrap, then publishes.
+   * @param {number} readyMs - How long to wait before going ahead regardless.
+   * @returns {Promise<void>} - Resolves once the first attempt is done.
+   */
+  async #firstPublish(readyMs) {
+    const nodes = await this.#whenDhtReady(readyMs);
+    if (this.#stopped) return;
+    if (nodes === 0) {
+      // Named rather than left to be inferred. A routing table that is still
+      // empty after a minute means the bootstrap queries are not being
+      // answered, and every "No nodes to query" after this is that same fact
+      // reported once per category.
+      this.#log(
+        `[mutable] the DHT found no peers in ${Math.round(readyMs / 1000)}s. Publishing ` +
+          'will fail until it does — check that outbound UDP is not blocked, and that ' +
+          'the bootstrap hosts resolve',
+      );
+    } else if (nodes) {
+      this.#log(`[mutable] DHT ready with ${nodes} nodes`);
+    }
+    await this.publishAll({ force: true });
+  }
+
+  /**
+   * Waits for the DHT to have somewhere to send a query.
+   *
+   * `ready` alone is not enough: it fires when the bootstrap lookup finishes,
+   * whether or not that lookup found anything, so a node with no UDP path
+   * reports itself ready and then fails every put with "No nodes to query".
+   * What matters is the size of the routing table.
+   * @param {number} timeoutMs - How long to wait.
+   * @returns {Promise<number | null>} - Nodes found, or null if unknowable.
+   */
+  async #whenDhtReady(timeoutMs) {
+    const deadline = Date.now() + timeoutMs;
+
+    if (typeof this.#dht?.once === 'function' && !this.#dht.ready) {
+      await new Promise((resolve) => {
+        const timer = setTimeout(resolve, timeoutMs);
+        timer.unref?.();
+        this.#dht.once('ready', () => {
+          clearTimeout(timer);
+          resolve();
+        });
+      });
+    }
+
+    let count = this.#nodeCount();
+    while (count === 0 && Date.now() < deadline && !this.#stopped) {
+      await new Promise((resolve) => {
+        const timer = setTimeout(resolve, 1000);
+        timer.unref?.();
+      });
+      count = this.#nodeCount();
+    }
+    return count;
+  }
+
+  /**
+   * How many nodes the DHT currently knows, where it can say.
+   * @returns {number | null} - The count, or null for a stand-in that has none.
+   */
+  #nodeCount() {
+    const count = this.#dht?.nodes?.count?.();
+    if (typeof count === 'number') return count;
+    const listed = this.#dht?.toJSON?.().nodes?.length;
+    return typeof listed === 'number' ? listed : null;
+  }
+
+  /**
+   * Tries again sooner than the republish interval.
+   *
+   * An attempt where nothing published usually means the DHT is not ready yet,
+   * and waiting out a thirty-minute interval to discover that again is half an
+   * hour of a node advertising a key that resolves to nothing.
+   * @param {number} delayMs - How long to wait first.
+   * @returns {void}
+   */
+  #scheduleRetry(delayMs) {
+    if (this.#stopped || this.#retryTimer) return;
+    const timer = setTimeout(() => {
+      this.#retryTimer = null;
+      if (this.#stopped) return;
+      this.publishAll({ retryDelayMs: Math.min(delayMs * 2, this.#intervalMs) }).catch(
+        (error) => this.#log(`[mutable] retry failed: ${error.message}`),
+      );
+    }, delayMs);
+    timer.unref?.();
+    this.#retryTimer = timer;
+  }
+
   /**
    * Stops republishing.
    * @returns {void}
@@ -197,6 +299,8 @@ export class MutablePublisher {
   stop() {
     this.#stopped = true;
     if (this.#timer) clearInterval(this.#timer);
+    if (this.#retryTimer) clearTimeout(this.#retryTimer);
     this.#timer = null;
+    this.#retryTimer = null;
   }
 }