pmtiles-swarm 0.41.1 → 0.41.2

package/CHANGELOG.md

@@ -7,6 +7,31 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.41.2
+### ✨ Features and improvements
+
+### 🐞 Bug fixes
+- **An archive that finished downloading during an unclean stop no longer comes back at 0% for ever.**
+  A downloaded archive is written under a marker — `planet.pmtiles.incomplete` — and renamed the
+  instant it is whole. The rename and the catalog entry recording it are two steps, so a stop between
+  them leaves an archive that is finished on disk and unfinished in the record.
+
+  That disagreement was permanent, and worse than it looks. Restore re-added the entry with the marker
+  attached, so the engine opened a filename nothing was at any more, found no data, and began
+  downloading an archive the node already held. The sweep that would have noticed takes the engine's
+  word over the disk's whenever the engine has one — and the engine's word was now 0%. Rechecking did
+  not help either, since it hashes the marked name, which is the wrong file to look at. No number of
+  restarts recovered it.
+
+  Restore now checks the disk before handing over anything recorded as unfinished: marked file gone
+  and the real one whole means the rename happened and the record of it did not, so the entry is
+  corrected and the archive handed over as the complete thing it is. A genuine partial download is
+  left exactly as it was — claiming otherwise would offer peers an archive this node cannot serve.
+
+  This is why the archives that arrive from a feed or a URL were the ones that sat at 0%: an archive
+  built here is recorded complete the moment it is registered, having just been read end to end, so it
+  never carries a marker to disagree about.
+
 ## 0.41.1
 ### ✨ Features and improvements
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.41.1",
+  "version": "0.41.2",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",

package/src/library.js

@@ -1819,6 +1819,72 @@ export class Library {
     return queued;
   }
 
+  /**
+   * Reconciles what the catalog says about an archive with what is on disk.
+   *
+   * An archive downloaded here is written under a marker — `planet.pmtiles`
+   * becomes `planet.pmtiles.incomplete` — and renamed the instant it is whole.
+   * The rename and the catalog entry that records it are two steps, so a stop
+   * between them leaves an archive that is finished on disk and unfinished in
+   * the record.
+   *
+   * That disagreement does not settle itself, and it is worse than it looks.
+   * Restore re-adds an entry it believes incomplete with the marker attached,
+   * so the engine opens `planet.pmtiles.incomplete` — a name nothing is at any
+   * more — finds no data, and begins downloading a 128 GiB archive this node
+   * already holds. The sweep that would notice takes the engine's word over
+   * the disk's whenever the engine has one, and the engine's word is now 0%.
+   * Rechecking does not help either: it hashes the marked name, which is the
+   * wrong file to be looking at. Nothing recovers, through any number of
+   * restarts.
+   *
+   * Only archives that were downloaded can be in this state — one built here
+   * is recorded complete the moment it is registered, having just been read
+   * end to end — which is why the archives that come from a feed or a URL are
+   * the ones that sit at 0%.
+   * @param {object} entry - The catalog entry restore is about to hand over.
+   * @returns {Promise<object>} - That entry, or the corrected one.
+   */
+  async #reconcileMarker(entry) {
+    if (entry.complete !== false || !entry.savePath || !entry.name) {
+      return entry;
+    }
+
+    const marked = onDiskPath(entry, this.#config);
+    const real = path.join(entry.savePath, entry.name);
+    // Markers turned off, so there is no second name to disagree with.
+    if (!marked || marked === real) return entry;
+
+    // Still under the marker, which is what an unfinished download should look
+    // like. Nothing to correct, whatever its progress.
+    const partial = await fs.stat(marked).then(
+      () => true,
+      () => false,
+    );
+    if (partial) return entry;
+
+    // The marked name is gone and the real one is whole: the rename happened
+    // and the record of it did not.
+    const whole = await alreadyComplete({
+      savePath: entry.savePath,
+      name: entry.name,
+      size: entry.size,
+    });
+    if (!whole) return entry;
+
+    console.warn(
+      `[complete] ${entry.name} is whole under its own name but was recorded ` +
+        'as unfinished — a stop between the rename and the record. Handing it ' +
+        'to the engine as the complete archive it is.',
+    );
+    return (
+      (await this.#catalog.put({
+        infoHash: entry.infoHash,
+        complete: true,
+      })) ?? { ...entry, complete: true }
+    );
+  }
+
   /**
    * Checks that what restore just claimed is actually being seeded.
    *
@@ -1960,8 +2026,9 @@ export class Library {
     }, 15_000);
     progress.unref?.();
 
+    const handed = [];
     try {
-      await this.#restoreEach(entries, tally);
+      await this.#restoreEach(entries, tally, handed);
     } finally {
       clearInterval(progress);
     }
@@ -1971,7 +2038,7 @@ export class Library {
     // reports on a library half restored. Never allowed to fail the restore —
     // this is a report about seeding, and a node that could not produce it is
     // still a node that restored what it could.
-    await this.#verifySeeding(entries).catch((error) =>
+    await this.#verifySeeding(handed).catch((error) =>
       console.warn(
         `[seeding] could not check what is being seeded: ${error.message}`,
       ),
@@ -1989,7 +2056,7 @@ export class Library {
    * @param {{restored: number, failed: number}} tally - Mutated as it goes.
    * @returns {Promise<{restored: number, failed: number}>} - That tally.
    */
-  async #restoreEach(entries, tally) {
+  async #restoreEach(entries, tally, handed) {
     for (const entry of entries) {
       // An engine that cannot open its port will fail every one of these, each
       // after its own timeout. Stopping at the first is the difference between
@@ -2024,11 +2091,18 @@ export class Library {
           }
         }
 
+        // Before the add, because the add is what acts on the disagreement:
+        // an entry wrongly recorded as unfinished is handed to the engine
+        // under a filename nothing is at, and the engine starts downloading
+        // an archive that is already here.
+        const settled = await this.#reconcileMarker(entry);
+
         // Through the same path as every other re-add, so an archive stored
         // with no trackers is repaired here too. Restoring used to build its
         // own add and skip that, which is why an archive that could not find a
         // peer stayed unable to find one across every restart.
-        await this.#readd(entry);
+        await this.#readd(settled);
+        handed.push(settled);
         tally.restored++;
       } catch (error) {
         tally.failed++;