pmtiles-swarm 0.5.3 → 0.5.4

package/CHANGELOG.md

@@ -7,6 +7,20 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.5.4
+### ✨ Features and improvements
+- **The head-warm waits sensibly at both ends.** It used to read at the instant the node
+  started — when an archive joined by magnet has no metainfo, the engine has no peers, and the
+  attempt is certain to find nothing — and then leave a flat two minutes between every try
+  afterwards. Neither suited what was actually being waited for. The first pass now comes ten
+  seconds in, and the wait after an attempt that did not finish starts at fifteen seconds and
+  doubles to a ten-minute ceiling: seconds early on, when what is missing is usually a peer or a
+  piece already in flight, and minutes later on, when it is one piece at the far end of an
+  archive nobody has finished downloading.
+
+  New under `tiles`: `prewarmInitialDelaySeconds` (10) and `prewarmMaxBackoffSeconds` (600).
+  `prewarmBackoffSeconds` is now the *first* wait rather than every wait, and defaults to 15.
+
 ## 0.5.3
 ### 🐞 Bug fixes
 - **A "high" priority hint asked for nothing at all.** libtorrent's scale runs 0 to 7 and **4 is

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.5.3",
+  "version": "0.5.4",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",

package/src/config.js

@@ -460,13 +460,28 @@ const DEFAULTS = {
     /** How often to look for an archive whose head has not been read. */
     prewarmIntervalSeconds: 30,
     /**
-     * How long to leave an archive alone after a failed read.
+     * How long to let the node settle before the first attempt.
      *
-     * Failure here is ordinary rather than exceptional — a brand-new archive
-     * may have no peer holding its first piece yet — so this is a retry
-     * interval, not an error budget.
+     * At the moment a node starts, an archive joined by magnet has no metainfo
+     * and the engine has no peers, so a read attempted immediately is certain
+     * to find nothing.
      */
-    prewarmBackoffSeconds: 120,
+    prewarmInitialDelaySeconds: 10,
+    /**
+     * The first wait after an attempt that did not finish the job, doubling up
+     * to `prewarmMaxBackoffSeconds`.
+     *
+     * Not finishing is ordinary rather than exceptional here, so this is a
+     * retry interval and not an error budget — and it doubles because the two
+     * ends of the problem want different answers. Just after a start, what is
+     * being waited for is usually seconds away: a peer, a connection, a piece
+     * already in flight. Ten attempts later it is one piece at the far end of
+     * an archive nobody has finished, and asking every fifteen seconds
+     * achieves nothing but log lines.
+     */
+    prewarmBackoffSeconds: 15,
+    /** Where the doubling stops. */
+    prewarmMaxBackoffSeconds: 600,
     /**
      * What a missing tile answers with: true for 404, false for 204.
      *

package/src/prewarm.js

@@ -21,8 +21,14 @@
  * the first few seconds if the right few kilobytes are asked for first.
  */
 
-/** How long to leave an archive alone after a failed attempt. */
-const DEFAULT_BACKOFF_MS = 120000;
+/** The first wait after an attempt that did not finish the job. */
+const DEFAULT_BACKOFF_SECONDS = 15;
+
+/** Where the doubling stops. */
+const DEFAULT_MAX_BACKOFF_SECONDS = 600;
+
+/** How long to let the node settle before the first attempt. */
+const DEFAULT_INITIAL_DELAY_SECONDS = 10;
 
 /**
  * Whether a failure means "not yet" rather than "not working".
@@ -49,6 +55,7 @@ export class HeadWarmer {
   #now;
   #timer;
   #tried = new Map();
+  #attempts = new Map();
   #waiting = new Set();
   #running = false;
 
@@ -93,8 +100,30 @@ export class HeadWarmer {
     }
 
     const last = this.#tried.get(entry.infoHash) ?? 0;
-    const backoff = (this.#config.tiles?.prewarmBackoffSeconds ?? 120) * 1000;
-    return this.#now() - last >= (backoff || DEFAULT_BACKOFF_MS);
+    return this.#now() - last >= this.#backoffFor(entry.infoHash);
+  }
+
+  /**
+   * How long to wait before trying this archive again.
+   *
+   * Doubling, from a short first wait to a long ceiling. A flat interval is
+   * wrong at both ends: right after a node starts, the thing being waited for
+   * is usually a few seconds away — a peer, a connection, a piece already in
+   * flight — so a two-minute wait wastes most of it. Ten attempts later the
+   * thing being waited for is a single piece at the far end of an archive
+   * nobody has finished downloading, and asking every two minutes achieves
+   * nothing but log lines.
+   * @param {string} infoHash - The archive.
+   * @returns {number} - Milliseconds to wait.
+   */
+  #backoffFor(infoHash) {
+    const base =
+      (this.#config.tiles?.prewarmBackoffSeconds ?? DEFAULT_BACKOFF_SECONDS) * 1000;
+    const ceiling =
+      (this.#config.tiles?.prewarmMaxBackoffSeconds ?? DEFAULT_MAX_BACKOFF_SECONDS) *
+      1000;
+    const attempts = this.#attempts.get(infoHash) ?? 0;
+    return Math.min(base * 2 ** Math.max(0, attempts - 1), ceiling);
   }
 
   /**
@@ -123,6 +152,14 @@ export class HeadWarmer {
 
       this.#tried.set(entry.infoHash, this.#now());
       this.#waiting.delete(entry.infoHash);
+      // Counted even on success, because a read that got the header and not
+      // the metadata has not finished and will be back. The count only matters
+      // while an archive is still due, and one that is done is never asked
+      // about again.
+      this.#attempts.set(
+        entry.infoHash,
+        (this.#attempts.get(entry.infoHash) ?? 0) + 1,
+      );
 
       const stored = await this.#catalog.put({
         infoHash: entry.infoHash,
@@ -169,6 +206,10 @@ export class HeadWarmer {
       // is my preview blank", and the backoff keeps it from becoming noise.
       this.#tried.set(entry.infoHash, this.#now());
       this.#waiting.delete(entry.infoHash);
+      this.#attempts.set(
+        entry.infoHash,
+        (this.#attempts.get(entry.infoHash) ?? 0) + 1,
+      );
       console.warn(`[warm] ${entry.name}: ${error.message}`);
       return null;
     } finally {
@@ -189,7 +230,17 @@ export class HeadWarmer {
       this.sweep().catch((error) =>
         console.error(`[warm] sweep failed: ${error.message}`),
       );
-    run();
+
+    // Not immediately. At the moment a node starts, an archive joined by
+    // magnet has no metainfo, the engine has no peers, and nothing can be read
+    // from anywhere — so the first pass is guaranteed to find nothing and
+    // exists only to say so.
+    const delay =
+      (this.#config.tiles?.prewarmInitialDelaySeconds ??
+        DEFAULT_INITIAL_DELAY_SECONDS) * 1000;
+    const first = setTimeout(run, Math.max(0, delay));
+    first.unref?.();
+
     this.#timer = setInterval(run, seconds * 1000);
     this.#timer.unref?.();
   }