pmtiles-swarm 0.35.4 → 0.36.0

package/CHANGELOG.md

@@ -7,6 +7,48 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.36.0
+### ✨ Features and improvements
+- **Hashing an archive now happens in a process of its own.** Building the torrent for a 698 GiB
+  archive ran inside the libtorrent sidecar, competing with the session for the disk and for
+  Python's interpreter lock while every archive on the node was being served from that same disk.
+  It also could not be stopped: libtorrent's hashing never checks for interruption, and the sidecar
+  cannot be ended to end a hash because it holds the session and every torrent seeding from it. A
+  build started by a misclick ran its full six hours.
+
+  It is now `libtorrent_sidecar.py --create`, started per hash, holding no session and no port.
+  Killing it costs the hash and nothing else, and hashing only ever reads, so the archive is
+  untouched. It reports the piece it has reached as it goes, so a caller can draw a real fraction
+  rather than "hashing 698 GiB · 3m".
+
+  Requires pmtiles-torrent 0.8.0. Also picks up 0.7.5, which stops an archive that is hashing its
+  store from reporting itself as "paused" — libtorrent hashes one store at a time and flags every
+  torrent queued behind it as paused, so a library busy verifying itself read as one somebody had
+  stopped.
+
+### 🐞 Bug fixes
+
+## 0.35.5
+### ✨ Features and improvements
+
+### 🐞 Bug fixes
+- **Requires pmtiles-torrent 0.7.4, which stops archives dropping out of the engine a few more with
+  every restart.** Seen here as `[restore] <archive>: mismatching info-hash`, beginning with one
+  archive and reaching eighteen of twenty. An archive that failed this way was never handed to the
+  engine at all, so the console showed it at 0% with no state, a recheck answered `no such
+  torrent`, and its data sat complete on the disk the whole time — the preview rendered from it
+  perfectly well.
+
+  The sidecar was writing resume data under the wrong torrent's name: saving it was the last thing
+  still popping libtorrent's alert queue on its own thread while the alert pump popped on another,
+  and the pump's next pop freed the batch that loop was reading. `add` then refuses such a file
+  with "mismatching info-hash". 0.7.2 did not introduce it but made the pump pop far more often,
+  which is why it appeared immediately after that upgrade.
+
+  Restarting on this version is the whole recovery: an add refused over resume data is retried
+  without it, and the recheck finds every byte already on disk. Nothing is downloaded again,
+  though rechecking a large archive is not quick.
+
 ## 0.35.4
 ### ✨ Features and improvements
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.35.4",
+  "version": "0.36.0",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",
@@ -46,7 +46,7 @@
     "maplibre-gl": "^6.2.0",
     "parse-torrent": "^11.0.24",
     "pmtiles": "^4.4.1",
-    "pmtiles-torrent": "^0.7.2",
+    "pmtiles-torrent": "^0.8.0",
     "webtorrent": "^3.0.21"
   },
   "engines": {

package/src/engines/libtorrent.js

@@ -491,13 +491,23 @@ export class LibtorrentEngine {
 
   /**
    * Creates a torrent from a local file.
+   *
+   * In a process of its own rather than over the pipe. libtorrent's hashing
+   * never checks for interruption, so a hash running inside the sidecar could
+   * not be stopped -- and the sidecar itself cannot be ended to stop one,
+   * because it holds the session and every torrent seeding from it. A 698 GiB
+   * build started by a misclick therefore ran its full six hours, saturating
+   * the disk the rest of the library was being served from.
+   *
+   * A process started for one hash can simply be killed. Hashing only reads,
+   * so nothing is left half-written, and the archive is untouched.
    * @param {string} filePath - The file to hash.
-   * @param {object} [options] - Piece length, trackers, web seeds, format.
+   * @param {object} [options] - Piece length, trackers, web seeds, format,
+   *   `signal` to cancel with, and `onProgress({piece, pieces})`.
    * @returns {Promise<object>} - The torrent file and what it describes.
    */
   async createTorrent(filePath, options = {}) {
-    const result = await this.#call(
-      'create',
+    const result = await this.#hashApart(
       {
         path: filePath,
         pieceLength: options.pieceLength,
@@ -508,8 +518,7 @@ export class LibtorrentEngine {
         createdBy: options.createdBy,
         format: options.format ?? 'hybrid',
       },
-      // Hashing a large archive takes as long as it takes.
-      options.timeoutMs ?? 6 * 60 * 60 * 1000,
+      options,
     );
     return {
       ...result,
@@ -517,6 +526,122 @@ export class LibtorrentEngine {
     };
   }
 
+  /**
+   * Runs one `--create` to completion, or until it is no longer wanted.
+   * @param {object} params - What to hash and how.
+   * @param {object} options - signal, onProgress, timeoutMs.
+   * @returns {Promise<object>} - The sidecar's result object.
+   */
+  #hashApart(params, options) {
+    return new Promise((resolve, reject) => {
+      const script = this.#options.script ?? resolveSidecar();
+      const child = spawn(this.#options.python, [script, '--create'], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+      });
+
+      let settled = false;
+      let result;
+      let failure;
+      let pending = '';
+      let stderr = '';
+
+      /**
+       * Ends this hash once, whatever ends it.
+       * @param {Error} [error] - Why, if it failed.
+       */
+      const finish = (error) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        options.signal?.removeEventListener('abort', cancel);
+        if (error) reject(error);
+        else resolve(result);
+      };
+
+      const cancel = () => {
+        // The hash cannot be asked to stop, so it is ended. Nothing is lost:
+        // hashing reads and this process holds nothing else.
+        child.kill();
+        // Said the same way however it was cancelled. An AbortController with
+        // no reason gives "This operation was aborted", which in a log next to
+        // an archive name explains nothing; callers that care which kind of
+        // stop this was read `signal.reason`, which is carried as the cause.
+        finish(
+          new Error('hashing was cancelled', { cause: options.signal?.reason }),
+        );
+      };
+
+      // Hashing a large archive takes as long as it takes, but not forever:
+      // a hash that has stopped reporting is stuck, and holding the add open
+      // for six hours to discover that helps nobody.
+      const timer = setTimeout(
+        () => {
+          child.kill();
+          finish(new Error('hashing timed out'));
+        },
+        options.timeoutMs ?? 6 * 60 * 60 * 1000,
+      );
+      timer.unref?.();
+
+      if (options.signal?.aborted) return cancel();
+      options.signal?.addEventListener('abort', cancel, { once: true });
+
+      child.stdout.setEncoding('utf8');
+      child.stdout.on('data', (chunk) => {
+        pending += chunk;
+        const lines = pending.split('\n');
+        pending = lines.pop() ?? '';
+        for (const line of lines) {
+          if (!line.trim()) continue;
+          let message;
+          try {
+            message = JSON.parse(line);
+          } catch {
+            // A line that is not ours. Python's own output on the way to a
+            // crash arrives here, and is worth keeping for the error.
+            stderr += `${line}\n`;
+            continue;
+          }
+          if (message.event === 'progress') {
+            options.onProgress?.({
+              piece: message.piece,
+              pieces: message.pieces,
+            });
+          } else if (message.ok) {
+            result = message.result;
+          } else if (message.ok === false) {
+            failure = new Error(message.error ?? 'hashing failed');
+          }
+        }
+      });
+
+      child.stderr.setEncoding('utf8');
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk;
+      });
+
+      child.on('error', (error) =>
+        finish(new Error(`could not start the hasher: ${error.message}`)),
+      );
+
+      child.on('close', (code, signal) => {
+        if (failure) return finish(failure);
+        if (result) return finish();
+        const why = signal
+          ? `killed by ${signal}`
+          : `exited with code ${code}`;
+        finish(
+          new Error(
+            `hashing produced nothing (${why})${stderr ? `: ${stderr.trim()}` : ''}`,
+          ),
+        );
+      });
+
+      child.stdin.on('error', () => {});
+      child.stdin.end(JSON.stringify(params));
+    });
+  }
+
   /**
    * Reads one piece, promoted ahead of the normal picker.
    *