pmtiles-swarm 0.70.0 → 0.71.0

package/CHANGELOG.md

@@ -7,6 +7,33 @@
 ### 🐞 Bug fixes
 - _...Add new stuff here..._
 
+## 0.71.0
+### ✨ Features and improvements
+- **The preview comes back to where it was opened from.** The link out of a preview said
+  "console" and went to the archive list, whatever had been previewed - so previewing a stack and
+  coming back meant finding the Stacks tab again. The console now names its view in the address
+  and honours one it is given, and a preview links to the view it belongs to.
+
+- **A category offers terrain where its newest build is terrain.** Archives and stacks got the
+  second button; categories did not, on either page. Same rule and the same reason: the raster
+  keeps the plain name because it is the view that shows a hole.
+
+### 🐞 Bug fixes
+- **A stopped export resumed; a restarted service did not.** Both were called "stopping keeps the
+  work", and only one of them did. **Stop export** cancels the job, which writes a checkpoint. A
+  service restart tells it nothing at all: the process is torn down, and whatever had been merged
+  since the last checkpoint was merged again.
+
+  Two things were wrong. Nothing asked a running export to stop when the node did, so exports are
+  now a shutdown step of their own - cancelled early in the sequence, before the pieces they read
+  through are taken away, and waited for so each writes its checkpoint.
+
+  And a checkpoint fired on a tile count alone, every five thousand. That is the wrong measure for
+  a bake merging slowly, which can run for an hour without reaching it - an export that had done a
+  few hundred tiles had never checkpointed at all, so a restart lost everything it had done. There
+  is a clock now as well, thirty seconds, whichever comes first. A checkpoint costs about eleven
+  milliseconds.
+
 ## 0.70.0
 ### ✨ Features and improvements
 - **An export works on the disk the archive is going to.** It worked under the data directory and

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmtiles-swarm",
-  "version": "0.70.0",
+  "version": "0.71.0",
   "description": "BitTorrent distribution for PMTiles map archives: create torrents, watch folders, publish and subscribe to RSS feeds, and seed through qBittorrent or an embedded client",
   "type": "module",
   "main": "src/index.js",

package/src/bake-jobs.js

@@ -108,6 +108,38 @@ export class BakeManager {
     return true;
   }
 
+  /**
+   * Stops every running bake and waits for each to write its checkpoint.
+   *
+   * For a service stopping. Without this a restart kills a bake where it
+   * stands: the merging half is cancellable and keeps its work, but only if
+   * something tells it to stop, and a process being torn down does not. What
+   * was merged since the last checkpoint would be merged again.
+   * @param {object} [options] - `timeoutMs` to stop waiting.
+   * @returns {Promise<number>} - How many were stopped.
+   */
+  async stopAll(options = {}) {
+    const running = [...this.#jobs.values()].filter((job) => !job.finishedAt);
+    if (running.length === 0) return 0;
+
+    for (const job of running) {
+      job.cancelling = true;
+      job.controller.abort();
+    }
+
+    // Bounded, because a shutdown that waits for ever is a shutdown that gets
+    // killed harder. A checkpoint is milliseconds; what takes time is the tile
+    // in flight noticing it has been abandoned.
+    const timeout = new Promise((resolve) =>
+      setTimeout(resolve, options.timeoutMs ?? 10000).unref?.(),
+    );
+    await Promise.race([
+      Promise.all(running.map((job) => job.promise?.catch(() => {}))),
+      timeout,
+    ]);
+    return running.length;
+  }
+
   /**
    * Starts a bake, and answers as soon as it is running rather than when it
    * finishes.

package/src/bake.js

@@ -25,6 +25,17 @@ import { Compression, PMTilesWriter, TileType } from './pmtiles-write.js';
 /** Tiles merged between checkpoints. */
 const DEFAULT_CHECKPOINT_EVERY = 5000;
 
+/**
+ * Seconds between checkpoints, however few tiles have been done.
+ *
+ * A count alone is the wrong measure. A bake merging slowly -- a big tile, a
+ * cache-mode source, a stack with several layers -- can run for an hour
+ * without reaching five thousand, and a process killed before its first
+ * checkpoint has nothing to resume from at all. A checkpoint costs about
+ * eleven milliseconds, so doing one on the clock is close to free.
+ */
+const DEFAULT_CHECKPOINT_SECONDS = 30;
+
 /** Tiles merged at once. */
 const DEFAULT_CONCURRENCY = 4;
 
@@ -454,6 +465,7 @@ export async function bakeStack(options) {
     header = {},
     deduplicate = true,
     checkpointEvery = DEFAULT_CHECKPOINT_EVERY,
+    checkpointSeconds = DEFAULT_CHECKPOINT_SECONDS,
     pauseMs = 0,
     concurrency = DEFAULT_CONCURRENCY,
   } = options;
@@ -486,6 +498,7 @@ export async function bakeStack(options) {
   let skipped = found?.skipped ?? 0;
   let lastTileId = found?.lastTileId ?? -1;
   let sinceCheckpoint = 0;
+  let checkpointedAt = Date.now();
   let persisted = found?.entries.length ?? 0;
 
   /**
@@ -508,6 +521,7 @@ export async function bakeStack(options) {
       persisted,
     );
     sinceCheckpoint = 0;
+    checkpointedAt = Date.now();
   };
 
   try {
@@ -563,7 +577,13 @@ export async function bakeStack(options) {
       }
 
       batch = [];
-      if (sinceCheckpoint >= checkpointEvery) await checkpoint();
+      // Whichever comes first. The count keeps a fast bake from checkpointing
+      // constantly; the clock keeps a slow one from never checkpointing at
+      // all.
+      const due =
+        sinceCheckpoint >= checkpointEvery ||
+        Date.now() - checkpointedAt >= checkpointSeconds * 1000;
+      if (due) await checkpoint();
       signal?.throwIfAborted();
 
       // Handing time back, where the operator asked for that. A bake on a node

package/src/index.js

@@ -474,6 +474,22 @@ PMTILES_SWARM_PUBLIC_URL
     cutlines,
   });
 
+  // Early in the sequence, so an export is told to stop before the pieces it
+  // reads through are taken away. Its checkpoint is the hours already spent.
+  stoppers.unshift({
+    label: 'stack exports',
+    stop: async () => {
+      const stopped = await bakes.stopAll();
+      if (stopped > 0) {
+        console.log(
+          `[shutdown] stopped ${stopped} stack export(s); each kept its ` +
+            'checkpoint, so exporting again carries on from there',
+        );
+      }
+    },
+    ms: 12000,
+  });
+
   // Reads the head of anything joined but not yet understood — the header,
   // then the root directory and metadata it points at. Without this an archive
   // being mirrored is unservable until the download happens to reach byte

package/src/web/index.html

@@ -4019,7 +4019,12 @@ Every piece is hashed against the ` +
                     ? `<div class="links">
                          ${copyable(ends.tileJson, 'TileJSON')}
                          ${copyable(ends.xyz, 'XYZ')}
-                         ${link(ends.preview, 'preview')}
+                         ${
+                           drawsAsTerrain(newest)
+                             ? `${link(`${ends.preview}?raw=1`, 'preview')}
+                                ${link(ends.preview, 'terrain')}`
+                             : link(ends.preview, 'preview')
+                         }
                          ${link(ends.torrent, '.torrent', true)}
                          ${copyable(ends.magnet, 'magnet', true)}
                          ${copyable(ends.feed, 'RSS')}
@@ -6995,17 +7000,19 @@ Every piece is hashed against the ` +
       }
 
       // ── Tabs ──────────────────────────────────────────────────────────────
+      /** Views a URL may name, so a link can come back to the right one. */
+      const TABS = ['archives', 'categories', 'stacks', 'traffic', 'settings'];
+
       const showTab = (name) => {
-        for (const tab of [
-          'archives',
-          'categories',
-          'stacks',
-          'traffic',
-          'settings',
-        ]) {
+        for (const tab of TABS) {
           $(`view-${tab}`).hidden = tab !== name;
           $(`tab-${tab}`).classList.toggle('on', tab === name);
         }
+        // Written into the address, so a page that sends somebody here can say
+        // where to land -- a preview of a stack belongs back at Stacks, not at
+        // whatever the console opens on. `replaceState` rather than assigning
+        // the hash, which would scroll and add a history entry per tab.
+        history.replaceState(null, '', name === 'archives' ? '#' : `#${name}`);
         if (name === 'settings') loadSettings().catch((e) => toast(e.message));
         if (name === 'categories') loadCategories().catch((e) => toast(e.message));
         if (name === 'stacks') loadStacks().catch((e) => toast(e.message));
@@ -7014,6 +7021,13 @@ Every piece is hashed against the ` +
           loadSwarmTraffic();
         }
       };
+      // A view named in the address wins over the one this opens on. Ignored
+      // when it names nothing, so an old bookmark is not a blank console.
+      const tabFromUrl = () => {
+        const named = location.hash.replace('#', '');
+        return TABS.includes(named) ? named : null;
+      };
+
       // The footer's year, set from the clock rather than typed into a file
       // nobody will remember to edit. The version beside it arrives with the
       // first status, and reads "…" until then.
@@ -8312,6 +8326,10 @@ Every piece is hashed against the ` +
           // An unreachable server is reported by refresh() in the status line.
         }
         refresh();
+        // After the archives have been asked for, so the tab that opens is the
+        // one asked for and not the one this happens to start on.
+        const named = tabFromUrl();
+        if (named && named !== 'archives') showTab(named);
       }
 
       start();

package/src/web/preview.html

@@ -69,7 +69,7 @@
       <code id="endpoint"></code>
       <span class="links">
         <a id="mode" href="#" hidden></a>
-        <a href="/">← console</a>
+        <a id="back" href="/">← console</a>
       </span>
     </header>
     <div id="note" hidden></div>
@@ -96,6 +96,18 @@
       );
       const $ = (id) => document.getElementById(id);
 
+      // Back to the view this was opened from rather than to whatever the
+      // console opens on. The path already says which: a stack was reached
+      // from Stacks, a category from Categories, an archive from the list.
+      const back = $('back');
+      if (back) {
+        back.href = location.pathname.startsWith('/stacks/')
+          ? '/#stacks'
+          : location.pathname.startsWith('/latest/')
+            ? '/#categories'
+            : '/';
+      }
+
       const fail = (message) => {
         document.body.innerHTML = `<div class="banner">${message}</div>`;
       };