pmtiles-swarm 0.2.0

package/src/sources.js

@@ -0,0 +1,245 @@
+/**
+ * Scheduled sources: upstreams that publish a new archive on a schedule, at a
+ * URL that encodes the date.
+ *
+ * This is a different shape from the origin checking in `origin.js`. That
+ * watches one fixed URL for its content changing. An upstream like
+ * `https://build.protomaps.com/20260806.pmtiles` never changes any given URL —
+ * it publishes a *new* one every day, and yesterday's stays exactly as it was.
+ * Watching for change would never fire; what is needed is to work out today's
+ * URL and see whether it exists yet.
+ *
+ * Each build therefore becomes its own archive with its own torrent and its own
+ * lifetime, which is what you want: old builds stay seedable for as long as
+ * anyone still wants them.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+/**
+ * Expands date placeholders in a template.
+ *
+ * Supported: {YYYYMMDD} {YYYY-MM-DD} {YYYY} {MM} {DD}
+ * @param {string} template - The template string.
+ * @param {Date} date - The date to substitute.
+ * @returns {string} - The expanded string.
+ */
+export function expandTemplate(template, date) {
+  const yyyy = String(date.getUTCFullYear());
+  const mm = String(date.getUTCMonth() + 1).padStart(2, '0');
+  const dd = String(date.getUTCDate()).padStart(2, '0');
+
+  return template
+    .replaceAll('{YYYYMMDD}', `${yyyy}${mm}${dd}`)
+    .replaceAll('{YYYY-MM-DD}', `${yyyy}-${mm}-${dd}`)
+    .replaceAll('{YYYY}', yyyy)
+    .replaceAll('{MM}', mm)
+    .replaceAll('{DD}', dd);
+}
+
+/**
+ * The dates a source should currently be looking for, newest first.
+ *
+ * `offsetDays` handles upstreams that publish yesterday's build (protomaps
+ * does). `lookbackDays` covers the case where the daemon was down, or the
+ * upstream published late — without it, one missed poll loses that build
+ * permanently.
+ * @param {object} source - The source definition.
+ * @param {Date} [now] - Override the current time, for testing.
+ * @returns {Date[]} - Candidate dates.
+ */
+export function candidateDates(source, now = new Date()) {
+  const offset = source.offsetDays ?? 0;
+  const lookback = Math.max(0, source.lookbackDays ?? 3);
+  const dates = [];
+  for (let back = 0; back <= lookback; back++) {
+    const date = new Date(now);
+    date.setUTCDate(date.getUTCDate() + offset - back);
+    dates.push(date);
+  }
+  return dates;
+}
+
+/**
+ * Polls scheduled sources and imports whatever has appeared.
+ */
+export class ScheduledSourceManager {
+  #library;
+  #catalog;
+  #config;
+  #timer;
+  #running = false;
+
+  /**
+   * Creates the manager.
+   * @param {import('./library.js').Library} library - Where imports go.
+   * @param {import('./catalog.js').Catalog} catalog - Used to skip what we already have.
+   * @param {object} config - Resolved configuration.
+   */
+  constructor(library, catalog, config) {
+    this.#library = library;
+    this.#catalog = catalog;
+    this.#config = config;
+  }
+
+  /**
+   * Starts polling.
+   * @returns {void}
+   */
+  start() {
+    const sources = this.#config.sources ?? [];
+    if (sources.length === 0) return;
+
+    const intervalMs = (this.#config.sourceCheckIntervalHours ?? 6) * 3600 * 1000;
+    this.poll().catch((error) =>
+      console.error(`[source] initial poll failed: ${error.message}`),
+    );
+    this.#timer = setInterval(() => {
+      this.poll().catch((error) =>
+        console.error(`[source] poll failed: ${error.message}`),
+      );
+    }, intervalMs);
+    this.#timer.unref?.();
+
+    console.log(
+      `[source] following ${sources.length} scheduled source(s) every ${intervalMs / 3600000}h`,
+    );
+  }
+
+  /**
+   * Polls every configured source once.
+   * @returns {Promise<object[]>} - Entries imported this pass.
+   */
+  async poll() {
+    // Downloading a planet archive takes hours; a poll landing on top of one
+    // already in progress would start it again.
+    if (this.#running) {
+      console.log('[source] poll already in progress, skipping');
+      return [];
+    }
+    this.#running = true;
+    try {
+      const imported = [];
+      for (const source of this.#config.sources ?? []) {
+        imported.push(...(await this.#pollSource(source)));
+      }
+      return imported;
+    } finally {
+      this.#running = false;
+    }
+  }
+
+  /**
+   * Polls one source across its candidate dates.
+   * @param {object} source - The source definition.
+   * @returns {Promise<object[]>} - Entries imported.
+   */
+  async #pollSource(source) {
+    if (!source.url) {
+      console.error(`[source] ${source.name ?? 'unnamed'}: no url template`);
+      return [];
+    }
+
+    const imported = [];
+    for (const date of candidateDates(source)) {
+      const url = expandTemplate(source.url, date);
+
+      // Already have it: this is the common case on every poll after the first.
+      if (this.#catalog.findBySource(url)) continue;
+
+      const exists = await this.#exists(url);
+      if (!exists) continue;
+
+      const filename = source.filename
+        ? expandTemplate(source.filename, date)
+        : undefined;
+
+      console.log(`[source] ${source.name ?? url}: found ${url}, importing`);
+      try {
+        const entry = await this.#library.addRemoteArchive(url, {
+          name: filename,
+          category: source.category,
+          savePath: source.savePath,
+          trackers: source.trackers,
+          pieceLength: source.pieceLength,
+          retain: source.retain !== false,
+          comment: source.comment
+            ? `${source.comment} ${expandTemplate('{YYYY-MM-DD}', date)}`
+            : undefined,
+        });
+        imported.push(entry);
+        console.log(
+          `[source] ${source.name ?? url}: imported ${entry.name} (${entry.infoHash})`,
+        );
+
+        if (source.latestLink) {
+          await this.#linkLatest(source, entry);
+        }
+      } catch (error) {
+        console.error(`[source] ${url}: ${error.message}`);
+      }
+    }
+    return imported;
+  }
+
+  /**
+   * Does this URL exist yet?
+   *
+   * A build that has not been published returns 404, which is the normal case
+   * for most of the day — so this must not be noisy.
+   * @param {string} url - The URL to test.
+   * @returns {Promise<boolean>} - True if it is there.
+   */
+  async #exists(url) {
+    try {
+      const response = await fetch(url, { method: 'HEAD' });
+      if (response.ok) return true;
+      // Some origins do not answer HEAD; a one-byte range asks the same
+      // question without transferring anything.
+      if (response.status === 405 || response.status === 501) {
+        const ranged = await fetch(url, { headers: { range: 'bytes=0-0' } });
+        return ranged.ok;
+      }
+      return false;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Points a stable "latest" name at the newest build.
+   *
+   * A symlink keeps the dated file as the real one, so it stays seedable under
+   * its own torrent while consumers can still reference a fixed path.
+   * @param {object} source - The source definition.
+   * @param {object} entry - The freshly imported entry.
+   * @returns {Promise<void>} - Resolves once linked, or logs and continues.
+   */
+  async #linkLatest(source, entry) {
+    const target = entry.retainedAt ?? path.join(entry.savePath, entry.name);
+    const link = path.isAbsolute(source.latestLink)
+      ? source.latestLink
+      : path.join(path.dirname(target), source.latestLink);
+
+    try {
+      await fs.rm(link, { force: true });
+      await fs.symlink(target, link);
+      console.log(`[source] latest -> ${path.basename(target)}`);
+    } catch (error) {
+      // Windows needs elevation or developer mode for symlinks; not fatal.
+      console.warn(`[source] could not update ${link}: ${error.message}`);
+    }
+  }
+
+  /**
+   * Stops polling.
+   * @returns {void}
+   */
+  stop() {
+    if (this.#timer !== undefined) {
+      clearInterval(this.#timer);
+      this.#timer = undefined;
+    }
+  }
+}

package/src/subscriptions.js

@@ -0,0 +1,158 @@
+import { parseFeed } from './feed.js';
+
+/**
+ * Follows other nodes' feeds and picks up what they publish.
+ *
+ * Two modes, and the difference matters:
+ *
+ *   mirror — join the torrent and download the whole archive. The node becomes
+ *            a full seeder, adding redundancy to the swarm. Costs the archive's
+ *            full size in disk.
+ *
+ *   cache  — record the torrent but download nothing. A tile server reads byte
+ *            ranges from it on demand through pmtiles-torrent, so disk use
+ *            tracks what is actually viewed rather than what exists. The node
+ *            still seeds the pieces it has picked up along the way.
+ *
+ * Cache mode is what makes a 700 GiB planet archive usable on a small server.
+ */
+export class SubscriptionManager {
+  #library;
+  #config;
+  #timer;
+  #seen = new Set();
+
+  /**
+   * Creates the manager.
+   * @param {import('./library.js').Library} library - Where new torrents go.
+   * @param {object} config - Resolved configuration.
+   */
+  constructor(library, config) {
+    this.#library = library;
+    this.#config = config;
+  }
+
+  /**
+   * Starts polling the configured feeds.
+   * @returns {void}
+   */
+  start() {
+    const feeds = this.#config.subscriptions ?? [];
+    if (feeds.length === 0) return;
+
+    const intervalMs = (this.#config.subscriptionIntervalSeconds ?? 900) * 1000;
+    // Poll once at startup, then on the interval.
+    this.refresh().catch((error) =>
+      console.error(`[feed] initial refresh failed: ${error.message}`),
+    );
+    this.#timer = setInterval(() => {
+      this.refresh().catch((error) =>
+        console.error(`[feed] refresh failed: ${error.message}`),
+      );
+    }, intervalMs);
+    this.#timer.unref?.();
+
+    console.log(
+      `[feed] following ${feeds.length} feed(s) every ${intervalMs / 1000}s`,
+    );
+  }
+
+  /**
+   * Polls every configured feed once.
+   * @returns {Promise<object[]>} - The entries added this pass.
+   */
+  async refresh() {
+    const added = [];
+    for (const subscription of this.#config.subscriptions ?? []) {
+      try {
+        added.push(...(await this.#poll(subscription)));
+      } catch (error) {
+        console.error(`[feed] ${subscription.url}: ${error.message}`);
+      }
+    }
+    return added;
+  }
+
+  /**
+   * Polls one feed.
+   * @param {object} subscription - Entry of {url, mode, category, filter}.
+   * @returns {Promise<object[]>} - Entries added from this feed.
+   */
+  async #poll(subscription) {
+    const response = await fetch(subscription.url, {
+      headers: { accept: 'application/rss+xml, application/xml, text/xml' },
+    });
+    if (!response.ok) {
+      throw new Error(`${response.status} ${response.statusText}`);
+    }
+
+    const items = parseFeed(await response.text());
+    const added = [];
+
+    for (const item of items) {
+      // A regex filter lets one feed serve several consumers with different
+      // appetites, e.g. only taking Europe extracts.
+      if (subscription.filter && !new RegExp(subscription.filter, 'i').test(item.title)) {
+        continue;
+      }
+      const marker = item.infoHash ?? item.magnet ?? item.torrentUrl;
+      if (this.#seen.has(marker)) continue;
+      this.#seen.add(marker);
+
+      try {
+        const entry = await this.#add(item, subscription);
+        if (entry) {
+          added.push(entry);
+          console.log(
+            `[feed] ${subscription.mode ?? 'cache'} ${entry.name} from ${subscription.url}`,
+          );
+        }
+      } catch (error) {
+        console.error(`[feed] could not add "${item.title}": ${error.message}`);
+      }
+    }
+    return added;
+  }
+
+  /**
+   * Adds one feed item to the library.
+   * @param {import('./feed.js').FeedItem} item - The item.
+   * @param {object} subscription - The subscription it came from.
+   * @returns {Promise<object | null>} - The catalog entry.
+   */
+  async #add(item, subscription) {
+    const options = {
+      category: subscription.category ?? item.category,
+      savePath: subscription.savePath,
+      // Cache mode joins the swarm without pulling the whole archive; the
+      // pieces it does hold are still served to other peers.
+      paused: (subscription.mode ?? 'cache') === 'cache',
+    };
+
+    if (item.torrentUrl) {
+      const response = await fetch(item.torrentUrl);
+      if (!response.ok) {
+        throw new Error(
+          `torrent fetch failed: ${response.status} ${response.statusText}`,
+        );
+      }
+      const torrentFile = new Uint8Array(await response.arrayBuffer());
+      return this.#library.addExistingTorrent({ torrentFile }, options);
+    }
+    if (item.magnet) {
+      return this.#library.addExistingTorrent({ magnet: item.magnet }, options);
+    }
+    return null;
+  }
+
+  /**
+   * Stops polling.
+   * @returns {void}
+   */
+  stop() {
+    if (this.#timer !== undefined) {
+      clearInterval(this.#timer);
+      this.#timer = undefined;
+    }
+  }
+}

package/src/tilejson.js

@@ -0,0 +1,112 @@
+import { TileStore } from './tiles.js';
+
+/**
+ * Builds TileJSON for an archive.
+ *
+ * Everything here comes from the catalog rather than from the archive itself,
+ * because the probe already recorded it when the archive was added. That is not
+ * just a saving: a node in cache mode holds almost none of the archive, so
+ * reading the header to answer a TileJSON request would mean pulling pieces out
+ * of the swarm before a map has asked for a single tile.
+ *
+ * @see https://github.com/mapbox/tilejson-spec/tree/master/3.0.0
+ */
+
+/** Tile format to the extension used in the tile URL template. */
+const URL_EXTENSION = {
+  pbf: 'pbf',
+  png: 'png',
+  jpeg: 'jpg',
+  webp: 'webp',
+  avif: 'avif',
+  mlt: 'mlt',
+};
+
+/**
+ * Builds the TileJSON document for one catalog entry.
+ * @param {object} entry - Catalog entry.
+ * @param {string} baseUrl - Public base URL, without a trailing slash.
+ * @returns {object} - A TileJSON 3.0.0 document.
+ */
+export function buildTileJson(entry, baseUrl) {
+  const summary = entry.pmtiles ?? {};
+  const extension = URL_EXTENSION[summary.format] ?? 'bin';
+  const root = `${baseUrl}/archives/${entry.infoHash}`;
+
+  const doc = {
+    tilejson: '3.0.0',
+    scheme: 'xyz',
+    tiles: [`${root}/{z}/{x}/{y}.${extension}`],
+    name: summary.name ?? entry.name,
+    minzoom: summary.minZoom ?? 0,
+    maxzoom: summary.maxZoom ?? 14,
+    bounds: summary.bounds ?? [-180, -85.051129, 180, 85.051129],
+    center: summary.center,
+    // The infohash is a content hash, so it doubles as a version: any change to
+    // the archive produces a different one, and the tile URLs change with it.
+    version: `1.0.0+${entry.infoHash.slice(0, 12)}`,
+    torrent: buildTorrentBlock(entry, root),
+  };
+
+  if (summary.description) doc.description = summary.description;
+  if (summary.attribution) doc.attribution = summary.attribution;
+  if (summary.vectorLayers) doc.vector_layers = summary.vectorLayers;
+  if (summary.format === 'pbf') doc.format = 'pbf';
+  else if (summary.format) doc.format = summary.format;
+
+  // A TileJSON consumer that ignores unknown members sees a perfectly ordinary
+  // document; dropping empty keys keeps it that way.
+  for (const [key, value] of Object.entries(doc)) {
+    if (value === undefined) delete doc[key];
+  }
+  return doc;
+}
+
+/**
+ * Builds the non-standard `torrent` member.
+ *
+ * This is the progressive-enhancement hook. TileJSON's spec allows unknown
+ * members and MapLibre's style-spec permits arbitrary source properties, so a
+ * plain client — maplibre-gl-js, Leaflet, anything — ignores this entirely and
+ * fetches tiles over HTTP as usual. A torrent-aware client reads it, joins the
+ * swarm, and serves the same tiles from pieces instead, falling back to the
+ * HTTP URLs whenever the swarm cannot answer.
+ *
+ * One URL works for both, which is the property that makes it worth having: the
+ * style does not have to know which kind of client will load it.
+ * @param {object} entry - Catalog entry.
+ * @param {string} root - This archive's URL root.
+ * @returns {object} - The torrent block.
+ */
+function buildTorrentBlock(entry, root) {
+  const block = {
+    infohash: entry.infoHash,
+    magnet: entry.magnet,
+    torrent: `${root}/archive.torrent`,
+    name: entry.name,
+    size: entry.size,
+  };
+  if (entry.webSeeds?.length) block.webseeds = entry.webSeeds;
+  // A mutable torrent is addressed by public key rather than by infohash, so a
+  // client that understands BEP 46 can follow updates instead of pinning to the
+  // version this document was generated from.
+  if (entry.mutable?.publicKey) {
+    block.mutable = {
+      publicKey: entry.mutable.publicKey,
+      salt: entry.mutable.salt,
+      seq: entry.mutable.seq,
+    };
+  }
+  return block;
+}
+
+/**
+ * Checks a requested extension against what the archive actually holds.
+ * @param {object} entry - Catalog entry.
+ * @param {string} extension - Requested extension, without a dot.
+ * @returns {boolean} - Whether it matches.
+ */
+export function extensionMatches(entry, extension) {
+  const accepted = TileStore.extensionsFor(entry.pmtiles?.format);
+  return accepted.includes(extension.toLowerCase());
+}