pmtiles-swarm 0.2.0

package/src/warm.js

@@ -0,0 +1,251 @@
+/**
+ * Pre-fetching a region so a node is useful the moment it enters rotation.
+ *
+ * A cache-mode node is slow exactly once per region: the first request pulls
+ * the pieces that region's tiles live in, and everything afterwards is served
+ * from what it now holds. Behind a load balancer that first request is paid
+ * separately by every node, which is the one real cost of scaling the serving
+ * tier horizontally.
+ *
+ * Warming moves that cost off the request path. Point it at the area you
+ * actually serve, wait, then add the node to the pool.
+ *
+ * Note the pieces do not have to come from the internet. Every node in the
+ * serving tier is a peer in the same swarm, so a node warming a region that a
+ * sibling already holds fetches it from that sibling — usually over the LAN,
+ * and far faster than from the original seed.
+ */
+
+/** Hard ceiling on a single job, so a careless bbox cannot run forever. */
+const DEFAULT_MAX_TILES = 5000;
+
+/**
+ * Converts longitude to a tile column.
+ * @param {number} lon - Longitude in degrees.
+ * @param {number} z - Zoom level.
+ * @returns {number} - Tile column.
+ */
+function lonToTileX(lon, z) {
+  return Math.floor(((lon + 180) / 360) * 2 ** z);
+}
+
+/**
+ * Converts latitude to a tile row.
+ * @param {number} lat - Latitude in degrees.
+ * @param {number} z - Zoom level.
+ * @returns {number} - Tile row.
+ */
+function latToTileY(lat, z) {
+  const clamped = Math.max(-85.05112878, Math.min(85.05112878, lat));
+  const radians = (clamped * Math.PI) / 180;
+  return Math.floor(
+    ((1 - Math.log(Math.tan(radians) + 1 / Math.cos(radians)) / Math.PI) / 2) *
+      2 ** z,
+  );
+}
+
+/**
+ * Enumerates the tiles covering a bounding box across a zoom range.
+ * @param {number[]} bounds - [west, south, east, north].
+ * @param {number} minZoom - Lowest zoom, inclusive.
+ * @param {number} maxZoom - Highest zoom, inclusive.
+ * @param {number} limit - Stop after this many tiles.
+ * @yields {{z: number, x: number, y: number}} - Each tile.
+ */
+export function* tilesInBounds(bounds, minZoom, maxZoom, limit) {
+  const [west, south, east, north] = bounds;
+  let produced = 0;
+  for (let z = minZoom; z <= maxZoom; z++) {
+    const span = 2 ** z;
+    const clamp = (value) => Math.max(0, Math.min(span - 1, value));
+    const x0 = clamp(lonToTileX(west, z));
+    const x1 = clamp(lonToTileX(east, z));
+    // Tile rows run north to south, so the northern edge is the lower index.
+    const y0 = clamp(latToTileY(north, z));
+    const y1 = clamp(latToTileY(south, z));
+    for (let x = x0; x <= x1; x++) {
+      for (let y = y0; y <= y1; y++) {
+        if (produced++ >= limit) return;
+        yield { z, x, y };
+      }
+    }
+  }
+}
+
+/**
+ * Counts the tiles a warm would visit, without fetching any.
+ * @param {number[]} bounds - [west, south, east, north].
+ * @param {number} minZoom - Lowest zoom.
+ * @param {number} maxZoom - Highest zoom.
+ * @param {number} limit - Ceiling.
+ * @returns {number} - Tile count, capped at the limit.
+ */
+export function countTiles(bounds, minZoom, maxZoom, limit) {
+  let total = 0;
+  for (const _tile of tilesInBounds(bounds, minZoom, maxZoom, limit)) total++;
+  return total;
+}
+
+/**
+ * Runs and tracks warming jobs, one per archive.
+ */
+export class WarmRunner {
+  #tiles;
+  #jobs = new Map();
+
+  /**
+   * @param {import('./tiles.js').TileStore} tiles - The tile reader.
+   */
+  constructor(tiles) {
+    this.#tiles = tiles;
+  }
+
+  /**
+   * Starts warming an archive.
+   * @param {object} entry - Catalog entry.
+   * @param {object} [options] - Bounds, zoom range, concurrency and ceiling.
+   * @returns {object} - The job state.
+   */
+  start(entry, options = {}) {
+    const existing = this.#jobs.get(entry.infoHash);
+    if (existing && existing.state === 'running') {
+      const error = new Error('a warm is already running for this archive');
+      error.status = 409;
+      throw error;
+    }
+
+    const summary = entry.pmtiles ?? {};
+    const bounds = options.bounds ??
+      summary.bounds ?? [-180, -85.051129, 180, 85.051129];
+    const minZoom = options.minZoom ?? summary.minZoom ?? 0;
+    // Warming every zoom to the archive's maximum is almost never what is
+    // wanted — the tile count quadruples per level — so stop a few levels
+    // short unless asked otherwise.
+    const maxZoom = Math.min(
+      options.maxZoom ?? Math.min(summary.maxZoom ?? 6, minZoom + 6),
+      summary.maxZoom ?? 22,
+    );
+    const limit = options.maxTiles ?? DEFAULT_MAX_TILES;
+
+    const controller = new AbortController();
+    const job = {
+      infoHash: entry.infoHash,
+      state: 'running',
+      bounds,
+      minZoom,
+      maxZoom,
+      total: countTiles(bounds, minZoom, maxZoom, limit),
+      done: 0,
+      hits: 0,
+      misses: 0,
+      errors: 0,
+      startedAt: new Date().toISOString(),
+      finishedAt: null,
+      error: null,
+      cancel: () => controller.abort(),
+    };
+    this.#jobs.set(entry.infoHash, job);
+
+    this.#run(job, controller.signal, options.concurrency ?? 4, limit).catch(
+      (error) => {
+        job.state = 'failed';
+        job.error = error.message;
+        job.finishedAt = new Date().toISOString();
+      },
+    );
+    return job;
+  }
+
+  /**
+   * Reports a job's progress.
+   * @param {string} infoHash - Which archive.
+   * @returns {object | null} - The job, or null if never warmed.
+   */
+  get(infoHash) {
+    const job = this.#jobs.get(infoHash);
+    return job ? publicView(job) : null;
+  }
+
+  /**
+   * Cancels a running job.
+   * @param {string} infoHash - Which archive.
+   * @returns {boolean} - Whether anything was cancelled.
+   */
+  cancel(infoHash) {
+    const job = this.#jobs.get(infoHash);
+    if (!job || job.state !== 'running') return false;
+    job.cancel();
+    return true;
+  }
+
+  /**
+   * Cancels every running job.
+   * @returns {void}
+   */
+  stop() {
+    for (const job of this.#jobs.values()) {
+      if (job.state === 'running') job.cancel();
+    }
+  }
+
+  /**
+   * Fetches the job's tiles, a few at a time.
+   * @param {object} job - The job to run.
+   * @param {AbortSignal} signal - Cancellation.
+   * @param {number} concurrency - Simultaneous reads.
+   * @param {number} limit - Tile ceiling.
+   * @returns {Promise<void>} - Resolves once finished.
+   */
+  async #run(job, signal, concurrency, limit) {
+    const queue = tilesInBounds(job.bounds, job.minZoom, job.maxZoom, limit);
+
+    /**
+     * Pulls tiles off the shared iterator until it runs dry.
+     * @returns {Promise<void>} - Resolves when the iterator is exhausted.
+     */
+    const worker = async () => {
+      for (const tile of queue) {
+        if (signal.aborted) return;
+        try {
+          const result = await this.#tiles.getTile(
+            job.infoHash,
+            tile.z,
+            tile.x,
+            tile.y,
+            { signal },
+          );
+          if (result) job.hits++;
+          else job.misses++;
+        } catch (error) {
+          if (error.name === 'AbortError') return;
+          job.errors++;
+          // One unreadable tile should not abandon the region — a sparse
+          // archive throws for all sorts of reasons. But an archive that has
+          // never once succeeded is not going to start, and grinding through
+          // thousands of tiles to prove it wastes the swarm's time.
+          if (job.errors > 25 && job.hits === 0 && job.misses === 0) {
+            throw error;
+          }
+        }
+        job.done++;
+      }
+    };
+
+    await Promise.all(
+      Array.from({ length: Math.max(1, concurrency) }, () => worker()),
+    );
+
+    job.state = signal.aborted ? 'cancelled' : 'complete';
+    job.finishedAt = new Date().toISOString();
+  }
+}
+
+/**
+ * Strips internals from a job before it goes over the wire.
+ * @param {object} job - The internal job.
+ * @returns {object} - A serialisable view.
+ */
+function publicView(job) {
+  const { cancel: _cancel, ...rest } = job;
+  return rest;
+}

package/src/watch.js

@@ -0,0 +1,98 @@
+import chokidar from 'chokidar';
+
+/**
+ * Watches folders for new PMTiles archives and imports them automatically.
+ *
+ * The subtlety is that a map build writes its output over minutes or hours, and
+ * hashing a half-written archive produces a torrent for bytes that no longer
+ * exist. chokidar's awaitWriteFinish handles that: nothing is imported until
+ * the file has stopped changing for a sustained period. The default here is
+ * deliberately generous, because a stalled network copy can pause for a long
+ * time mid-file.
+ */
+export class WatchManager {
+  #library;
+  #watchers = [];
+  #importing = new Set();
+
+  /**
+   * Creates the manager.
+   * @param {import('./library.js').Library} library - Where imports go.
+   */
+  constructor(library) {
+    this.#library = library;
+  }
+
+  /**
+   * Starts watching the configured folders.
+   * @param {object[]} folders - Entries of {path, category, webSeedBase, stabilitySeconds}.
+   * @returns {void}
+   */
+  start(folders = []) {
+    for (const folder of folders) {
+      const stability = (folder.stabilitySeconds ?? 30) * 1000;
+      const watcher = chokidar.watch(folder.path, {
+        ignoreInitial: false,
+        depth: folder.recursive === false ? 0 : undefined,
+        awaitWriteFinish: {
+          stabilityThreshold: stability,
+          pollInterval: 1000,
+        },
+      });
+
+      watcher.on('add', (file) => {
+        if (!/\.pmtiles$/i.test(file)) return;
+        this.#import(file, folder);
+      });
+      watcher.on('error', (error) => {
+        console.error(`[watch] ${folder.path}: ${error.message}`);
+      });
+
+      this.#watchers.push(watcher);
+      console.log(
+        `[watch] watching ${folder.path}${folder.category ? ` as "${folder.category}"` : ''}`,
+      );
+    }
+  }
+
+  /**
+   * Imports one archive, guarding against overlapping imports of the same file.
+   * @param {string} file - Path to the archive.
+   * @param {object} folder - The watch-folder configuration.
+   * @returns {Promise<void>} - Resolves once imported or skipped.
+   */
+  async #import(file, folder) {
+    if (this.#importing.has(file)) return;
+    this.#importing.add(file);
+    try {
+      // A web seed makes a brand-new archive usable before anyone has it, so
+      // publish one whenever the folder is also served over HTTP.
+      const webSeeds = folder.webSeedBase
+        ? [
+            `${folder.webSeedBase.replace(/\/$/, '')}/${encodeURIComponent(
+              file.split(/[\\/]/).pop(),
+            )}`,
+          ]
+        : [];
+
+      const entry = await this.#library.addLocalArchive(file, {
+        category: folder.category,
+        webSeeds,
+      });
+      console.log(`[watch] imported ${entry.name} (${entry.infoHash})`);
+    } catch (error) {
+      console.error(`[watch] failed to import ${file}: ${error.message}`);
+    } finally {
+      this.#importing.delete(file);
+    }
+  }
+
+  /**
+   * Stops all watchers.
+   * @returns {Promise<void>} - Resolves once closed.
+   */
+  async stop() {
+    await Promise.all(this.#watchers.map((watcher) => watcher.close()));
+    this.#watchers = [];
+  }
+}

package/src/web/index.html

@@ -0,0 +1,254 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>pmtiles-swarm</title>
+    <style>
+      :root {
+        color-scheme: light dark;
+        --bg: #fff;
+        --fg: #111;
+        --muted: #666;
+        --line: #ddd;
+        --accent: #2d6cdf;
+        --ok: #1a7f37;
+        --warn: #9a6700;
+        --bad: #cf222e;
+      }
+      @media (prefers-color-scheme: dark) {
+        :root {
+          --bg: #14161a;
+          --fg: #e8e8e8;
+          --muted: #9aa0a6;
+          --line: #2c3038;
+          --accent: #6ea8ff;
+          --ok: #3fb950;
+          --warn: #d29922;
+          --bad: #f85149;
+        }
+      }
+      * { box-sizing: border-box; }
+      body {
+        margin: 0;
+        background: var(--bg);
+        color: var(--fg);
+        font: 14px/1.5 system-ui, -apple-system, Segoe UI, sans-serif;
+      }
+      header {
+        display: flex;
+        align-items: baseline;
+        gap: 1rem;
+        padding: 1rem 1.25rem;
+        border-bottom: 1px solid var(--line);
+        flex-wrap: wrap;
+      }
+      h1 { font-size: 1.05rem; margin: 0; font-weight: 600; }
+      .status { color: var(--muted); font-size: 0.85rem; }
+      .status b { color: var(--fg); font-weight: 600; }
+      main { padding: 1.25rem; }
+      .bar { display: flex; gap: 0.5rem; margin-bottom: 1rem; flex-wrap: wrap; }
+      input, select, button {
+        font: inherit;
+        padding: 0.4rem 0.6rem;
+        border: 1px solid var(--line);
+        border-radius: 6px;
+        background: var(--bg);
+        color: var(--fg);
+      }
+      input { min-width: 22rem; flex: 1; }
+      button { cursor: pointer; }
+      button.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
+      table { width: 100%; border-collapse: collapse; }
+      th, td {
+        text-align: left;
+        padding: 0.5rem 0.6rem;
+        border-bottom: 1px solid var(--line);
+        white-space: nowrap;
+      }
+      th { font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.03em; color: var(--muted); font-weight: 600; }
+      td.name { white-space: normal; min-width: 18rem; }
+      .sub { color: var(--muted); font-size: 0.8rem; }
+      .num { text-align: right; font-variant-numeric: tabular-nums; }
+      .pill { display: inline-block; padding: 0.05rem 0.45rem; border-radius: 999px; font-size: 0.75rem; border: 1px solid var(--line); }
+      .seeding { color: var(--ok); border-color: currentColor; }
+      .downloading { color: var(--accent); border-color: currentColor; }
+      .stalled, .checking, .paused { color: var(--warn); border-color: currentColor; }
+      .error { color: var(--bad); border-color: currentColor; }
+      .track { width: 90px; height: 6px; border-radius: 3px; background: var(--line); overflow: hidden; }
+      .fill { height: 100%; background: var(--accent); }
+      a { color: var(--accent); }
+      .empty { color: var(--muted); padding: 2rem 0; }
+      .wrap { overflow-x: auto; }
+    </style>
+  </head>
+  <body>
+    <header>
+      <h1>pmtiles-swarm</h1>
+      <div class="status" id="status">connecting…</div>
+    </header>
+    <main>
+      <div class="bar">
+        <input
+          id="source"
+          placeholder="local .pmtiles path, https:// URL, magnet:… or .torrent URL"
+        />
+        <input id="category" placeholder="category (optional)" style="min-width: 10rem" />
+        <button class="primary" id="add">Add</button>
+        <button id="adopt" title="Import torrents the engine already holds">Adopt existing</button>
+        <button id="refresh-feeds" title="Poll subscribed feeds now">Poll feeds</button>
+        <a id="feed-link" href="/feed.xml" target="_blank" rel="noopener">feed.xml</a>
+      </div>
+      <div class="wrap">
+        <table>
+          <thead>
+            <tr>
+              <th>Name</th><th>Size</th><th>Map</th><th>State</th>
+              <th>Progress</th><th class="num">Seeds</th><th class="num">Peers</th>
+              <th class="num">Down</th><th class="num">Up</th><th class="num">Ratio</th><th>Export</th>
+            </tr>
+          </thead>
+          <tbody id="rows"></tbody>
+        </table>
+      </div>
+      <div class="empty" id="empty" hidden>
+        Nothing yet. Add an archive above, or click <b>Adopt existing</b> to pull
+        in what your torrent client already seeds.
+      </div>
+    </main>
+
+    <script type="module">
+      const bytes = (n) => {
+        if (!n) return '—';
+        const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
+        let v = n, i = 0;
+        while (v >= 1024 && i < u.length - 1) { v /= 1024; i++; }
+        return `${v.toFixed(i ? 1 : 0)} ${u[i]}`;
+      };
+      const rate = (n) => (n ? `${bytes(n)}/s` : '—');
+
+      async function api(path, options) {
+        const response = await fetch(path, options);
+        if (!response.ok) {
+          const body = await response.json().catch(() => ({}));
+          throw new Error(body.error ?? `${response.status} ${response.statusText}`);
+        }
+        return response.status === 204 ? null : response.json();
+      }
+
+      function mapCell(entry) {
+        const m = entry.pmtiles;
+        if (!m) return '<span class="sub">—</span>';
+        return `${m.format} <span class="sub">z${m.minZoom}–${m.maxZoom}</span>`;
+      }
+
+      function row(entry) {
+        const s = entry.status;
+        const pct = Math.round((s?.progress ?? 0) * 100);
+        const state = s?.state ?? 'unknown';
+        return `<tr>
+          <td class="name">${entry.name}
+            <div class="sub">${entry.category ? entry.category + ' · ' : ''}${entry.infoHash.slice(0, 12)}…</div>
+          </td>
+          <td>${bytes(entry.size)}</td>
+          <td>${mapCell(entry)}</td>
+          <td><span class="pill ${state}">${state}</span></td>
+          <td><div class="track"><div class="fill" style="width:${pct}%"></div></div>
+            <span class="sub">${pct}%</span></td>
+          <td class="num">${s?.seeds ?? '—'}</td>
+          <td class="num">${s?.peers ?? '—'}</td>
+          <td class="num">${rate(s?.downloadSpeed)}</td>
+          <td class="num">${rate(s?.uploadSpeed)}</td>
+          <td class="num">${s ? (s.ratio ?? 0).toFixed(2) : '—'}</td>
+          <td>
+            <a href="/api/torrents/${entry.infoHash}/file">.torrent</a> ·
+            <a href="#" data-magnet="${entry.infoHash}">magnet</a> ·
+            <a href="#" data-remove="${entry.infoHash}">remove</a>
+          </td>
+        </tr>`;
+      }
+
+      async function refresh() {
+        try {
+          const [status, torrents] = await Promise.all([
+            api('/api/status'),
+            api('/api/torrents'),
+          ]);
+          document.getElementById('status').innerHTML =
+            `engine <b>${status.engine.name}</b> ${status.engine.ok ? 'ok' : '<span style="color:var(--bad)">unavailable</span>'} · ` +
+            `<b>${status.archives}</b> archives · ` +
+            `${status.categories.length} categories · ${status.subscriptions.length} feeds`;
+          document.getElementById('rows').innerHTML = torrents.map(row).join('');
+          document.getElementById('empty').hidden = torrents.length > 0;
+        } catch (error) {
+          document.getElementById('status').textContent = `error: ${error.message}`;
+        }
+      }
+
+      document.getElementById('add').onclick = async () => {
+        const value = document.getElementById('source').value.trim();
+        if (!value) return;
+        const category = document.getElementById('category').value.trim() || undefined;
+        // Route by shape: the server accepts each of these on its own key.
+        const body = value.startsWith('magnet:')
+          ? { magnet: value, category }
+          : /\.torrent(\?|$)/i.test(value)
+            ? { torrentUrl: value, category }
+            : /^https?:\/\//i.test(value)
+              ? { url: value, category }
+              : { path: value, category };
+        try {
+          await api('/api/torrents', {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify(body),
+          });
+          document.getElementById('source').value = '';
+          refresh();
+        } catch (error) {
+          alert(`Could not add: ${error.message}`);
+        }
+      };
+
+      document.getElementById('adopt').onclick = async () => {
+        try {
+          const result = await api('/api/adopt', { method: 'POST' });
+          alert(`Adopted ${result.added} archive(s).`);
+          refresh();
+        } catch (error) {
+          alert(`Adopt failed: ${error.message}`);
+        }
+      };
+
+      document.getElementById('refresh-feeds').onclick = async () => {
+        try {
+          const result = await api('/api/subscriptions/refresh', { method: 'POST' });
+          alert(`Added ${result.added} archive(s) from feeds.`);
+          refresh();
+        } catch (error) {
+          alert(`Feed poll failed: ${error.message}`);
+        }
+      };
+
+      document.getElementById('rows').onclick = async (event) => {
+        const magnet = event.target.dataset?.magnet;
+        const remove = event.target.dataset?.remove;
+        if (magnet) {
+          event.preventDefault();
+          const uri = await fetch(`/api/torrents/${magnet}/magnet`).then((r) => r.text());
+          await navigator.clipboard.writeText(uri).catch(() => {});
+          prompt('Magnet link (copied to clipboard):', uri);
+        }
+        if (remove) {
+          event.preventDefault();
+          if (!confirm('Remove this archive from the catalog?')) return;
+          await api(`/api/torrents/${remove}`, { method: 'DELETE' });
+          refresh();
+        }
+      };
+
+      refresh();
+      setInterval(refresh, 3000);
+    </script>
+  </body>
+</html>