pmtiles-swarm 0.2.0

package/src/tiles.js

@@ -0,0 +1,284 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import zlib from 'node:zlib';
+import { PMTiles, SharedPromiseCache } from 'pmtiles';
+import { TorrentSource } from 'pmtiles-torrent';
+import { NodeFileSource } from './file-source.js';
+import { LibtorrentReadEngine } from './read-engine.js';
+
+/**
+ * Serves tiles out of the archives this node distributes.
+ *
+ * The point of doing this here rather than in a separate tile server is that
+ * the archive is already in the swarm. A node in cache mode holds almost none
+ * of a 72 GiB archive but can still answer for any tile in it, pulling the few
+ * pieces that tile lives in and keeping them for the next request. A node
+ * mirroring the archive reads its local copy directly and never involves the
+ * swarm at all. Same URL either way.
+ */
+
+/** Formats that are already compressed; gzipping them again wastes CPU. */
+const PRECOMPRESSED = new Set(['png', 'jpeg', 'webp', 'avif']);
+
+/** Request extensions accepted for each archive format. */
+const EXTENSIONS = {
+  pbf: ['pbf', 'mvt'],
+  png: ['png'],
+  jpeg: ['jpg', 'jpeg'],
+  webp: ['webp'],
+  avif: ['avif'],
+  mlt: ['mlt'],
+};
+
+/**
+ * An archive that could not be opened for reading, with the reason.
+ */
+export class TileReadError extends Error {
+  /**
+   * @param {string} message - What went wrong.
+   * @param {number} status - HTTP status this maps onto.
+   */
+  constructor(message, status = 500) {
+    super(message);
+    this.name = 'TileReadError';
+    this.status = status;
+  }
+}
+
+/**
+ * Opens archives on demand and reads tiles out of them.
+ */
+export class TileStore {
+  #catalog;
+  #engine;
+  #config;
+  #open = new Map();
+  // One header and directory cache across every archive. Entries are keyed by
+  // the source's key, so sharing it bounds total memory rather than letting
+  // each archive keep its own hundred entries.
+  #directoryCache;
+
+  /**
+   * @param {object} deps - Catalog, seeding engine and config.
+   */
+  constructor({ catalog, engine, config }) {
+    this.#catalog = catalog;
+    this.#engine = engine;
+    this.#config = config;
+    this.#directoryCache = new SharedPromiseCache(
+      config.tiles?.directoryCacheEntries ?? 200,
+    );
+  }
+
+  /**
+   * The extensions that map onto an archive's tile format.
+   * @param {string} format - Format name from the probe.
+   * @returns {string[]} - Accepted extensions.
+   */
+  static extensionsFor(format) {
+    return EXTENSIONS[format] ?? [];
+  }
+
+  /**
+   * Reads one tile.
+   * @param {string} infoHash - Which archive.
+   * @param {number} z - Zoom.
+   * @param {number} x - Column.
+   * @param {number} y - Row.
+   * @param {object} [options] - Abort signal.
+   * @returns {Promise<{data: Buffer, encoding?: string} | null>} - The tile, or null when absent.
+   */
+  async getTile(infoHash, z, x, y, options = {}) {
+    const entry = this.#catalog.get(infoHash);
+    if (!entry) throw new TileReadError('unknown archive', 404);
+
+    const handle = await this.#acquire(entry);
+    const tile = await handle.archive.getZxy(z, x, y, options.signal);
+    // A missing tile is not an error: sparse coverage is normal, and the
+    // caller turns this into a 204.
+    if (!tile?.data) return null;
+
+    const data = Buffer.from(tile.data);
+    const format = entry.pmtiles?.format;
+    if (!format || PRECOMPRESSED.has(format)) return { data };
+
+    // PMTiles decompresses tile data on the way out, so vector tiles arrive
+    // here as raw protobuf. Compressing them again is worth it — they are text
+    // heavy and typically a third of the size gzipped.
+    const gzipped = await new Promise((resolve, reject) =>
+      zlib.gzip(data, (error, out) => (error ? reject(error) : resolve(out))),
+    );
+    return { data: gzipped, encoding: 'gzip' };
+  }
+
+  /**
+   * Reports how an archive is currently being read, for diagnostics.
+   * @param {string} infoHash - Which archive.
+   * @returns {object | null} - Mode and stats, or null when not open.
+   */
+  status(infoHash) {
+    const handle = this.#open.get(infoHash);
+    if (!handle) return null;
+    return {
+      mode: handle.mode,
+      openedAt: handle.openedAt,
+      stats: handle.source?.stats,
+    };
+  }
+
+  /**
+   * Closes every open archive.
+   * @returns {Promise<void>} - Resolves once closed.
+   */
+  async close() {
+    const handles = [...this.#open.values()];
+    this.#open.clear();
+    for (const handle of handles) await this.#release(handle);
+  }
+
+  /**
+   * Gets an open archive, opening it if needed and evicting the least recently
+   * used one when over budget.
+   * @param {object} entry - Catalog entry.
+   * @returns {Promise<object>} - The open handle.
+   */
+  async #acquire(entry) {
+    const existing = this.#open.get(entry.infoHash);
+    if (existing) {
+      // Re-inserting moves it to the end, so the first key is always the least
+      // recently used.
+      this.#open.delete(entry.infoHash);
+      this.#open.set(entry.infoHash, existing);
+      return existing;
+    }
+
+    const handle = await this.#openArchive(entry);
+    this.#open.set(entry.infoHash, handle);
+
+    const limit = this.#config.tiles?.maxOpenArchives ?? 16;
+    while (this.#open.size > limit) {
+      const [oldest, victim] = this.#open.entries().next().value;
+      this.#open.delete(oldest);
+      await this.#release(victim);
+    }
+    return handle;
+  }
+
+  /**
+   * Opens an archive, choosing between the local file and the swarm.
+   * @param {object} entry - Catalog entry.
+   * @returns {Promise<object>} - The open handle.
+   */
+  async #openArchive(entry) {
+    const local = await this.#completeLocalPath(entry);
+    if (local) {
+      const source = new NodeFileSource(local);
+      return {
+        mode: 'local',
+        source,
+        archive: new PMTiles(source, this.#directoryCache),
+        openedAt: new Date().toISOString(),
+        close: () => source.close(),
+      };
+    }
+
+    const engine = await this.#readEngine(entry);
+    const source = new TorrentSource(engine, {
+      cacheBytes: this.#config.tiles?.pieceCacheBytes,
+      hydrateIdleMs: this.#config.tiles?.hydrateIdleMs,
+    });
+    return {
+      mode: 'swarm',
+      source,
+      archive: new PMTiles(source, this.#directoryCache),
+      openedAt: new Date().toISOString(),
+      // destroy() takes the engine down with it, which for both bridges means
+      // dropping this reader without disturbing what the node is seeding.
+      close: () => source.destroy(),
+    };
+  }
+
+  /**
+   * Returns the archive's path when this node holds a complete copy.
+   *
+   * Size alone cannot answer this: both engines preallocate the full file, so a
+   * torrent one piece in already looks the right size on disk. The engine's own
+   * progress is the only trustworthy signal.
+   * @param {object} entry - Catalog entry.
+   * @returns {Promise<string | null>} - Path, or null if incomplete.
+   */
+  async #completeLocalPath(entry) {
+    if (!entry.savePath) return null;
+    const status = await this.#engine.get(entry.infoHash).catch(() => null);
+    if (status && status.progress < 1) return null;
+
+    // No status at all means the engine does not know this torrent — the file
+    // may still be a plain local archive that was added and never seeded.
+    const file = path.join(entry.savePath, entry.name);
+    try {
+      const stat = await fs.stat(file);
+      if (!stat.isFile()) return null;
+      if (status === null && stat.size !== entry.size) return null;
+      return file;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Builds the read engine for an archive this node does not fully hold.
+   * @param {object} entry - Catalog entry.
+   * @returns {Promise<object>} - A pmtiles-torrent TorrentEngine.
+   */
+  async #readEngine(entry) {
+    switch (this.#engine.name) {
+      case 'libtorrent':
+        return new LibtorrentReadEngine(this.#engine, entry.infoHash, {
+          pieceTimeoutMs: this.#config.tiles?.pieceTimeoutMs,
+        });
+
+      case 'webtorrent': {
+        const { WebTorrentEngine } = await import('pmtiles-torrent/webtorrent');
+        const client = this.#engine.client;
+        if (!client) {
+          throw new TileReadError(
+            'the webtorrent engine is not connected yet',
+            503,
+          );
+        }
+        // Sharing the seeding client is the whole point: one peer pool, one
+        // port, one DHT node, and the pieces this fetches count towards what
+        // the node seeds back.
+        return new WebTorrentEngine(entry.torrentPath ?? entry.magnet, {
+          client,
+          path: entry.savePath,
+          readyTimeoutMs: this.#config.tiles?.readyTimeoutMs,
+        });
+      }
+
+      default:
+        // qBittorrent's WebUI has per-file priorities but nothing per piece and
+        // no way to read one back, so there is no honest way to serve a tile
+        // from an archive it holds only part of.
+        throw new TileReadError(
+          `the ${this.#engine.name} engine cannot read pieces on demand, and this ` +
+            'node does not hold a complete copy of the archive. Mirror it, or ' +
+            'run the libtorrent or webtorrent engine.',
+          501,
+        );
+    }
+  }
+
+  /**
+   * Closes one handle, swallowing errors so eviction cannot fail a request.
+   * @param {object} handle - The handle to close.
+   * @returns {Promise<void>} - Resolves once closed.
+   */
+  async #release(handle) {
+    try {
+      await handle.close();
+    } catch {
+      // Nothing useful to do: the handle is being dropped either way.
+    }
+  }
+}

package/src/torrent-create.js

@@ -0,0 +1,250 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+
+/**
+ * Turning a PMTiles archive into a torrent.
+ *
+ * Two things here are specific to distributing maps rather than generic files.
+ *
+ * Piece length: creation tools size pieces for whole-file downloads, which for
+ * a multi-hundred-gigabyte archive means 16 MiB or more. That is a poor fit for
+ * a tile server, where a cold tile costs a whole piece; 4 MiB is the default
+ * here, trading a larger hash list for a quarter of the read amplification.
+ *
+ * Web seeds: a brand-new archive has no peers, which normally makes it useless
+ * until someone finishes downloading it. BEP 19 lets the torrent name an HTTP
+ * or S3 URL as a fallback source, so it works from the moment it is published
+ * and simply gets cheaper as peers appear. If the archive already lives on a
+ * web server, always pass its URL.
+ */
+
+/**
+ * Options for creating a torrent.
+ * @typedef {object} CreateTorrentOptions
+ * @property {string} [name] - Torrent name. Defaults to the filename.
+ * @property {number} [pieceLength] - Piece size in bytes; must be a power of two.
+ * @property {string[]} [trackers] - Announce URLs.
+ * @property {string[]} [webSeeds] - BEP 19 url-list entries.
+ * @property {string} [comment] - Free-text comment.
+ * @property {boolean} [private] - Mark the torrent private (no DHT/PEX).
+ */
+
+/**
+ * The result of creating a torrent.
+ * @typedef {object} CreatedTorrent
+ * @property {Uint8Array} torrentFile - Raw .torrent bytes.
+ * @property {string} infoHash - Hex v1 infohash.
+ * @property {string} magnet - Magnet URI.
+ * @property {string} name - Torrent name.
+ * @property {number} size - Total bytes.
+ * @property {number} pieceLength - Piece size used.
+ * @property {number} pieceCount - Number of pieces.
+ */
+
+/**
+ * Creates a torrent from a local archive.
+ * @param {string} filePath - Path to the .pmtiles file.
+ * @param {CreateTorrentOptions} [options] - Creation options.
+ * @returns {Promise<CreatedTorrent>} - The created torrent.
+ */
+export async function createTorrentFromFile(filePath, options = {}) {
+  const stat = await fs.stat(filePath);
+  if (!stat.isFile() || stat.size === 0) {
+    throw new Error(`not a usable file: ${filePath}`);
+  }
+  return buildTorrent(filePath, path.basename(filePath), stat.size, options);
+}
+
+/**
+ * Creates a torrent from a remote archive.
+ *
+ * Piece hashes are computed over content, so there is no way to do this without
+ * reading every byte. What differs is whether those bytes are kept:
+ *
+ *   retain (default) — they are written to `retainPath` as they arrive, so the
+ *     node holds a complete copy and becomes a real seeder the moment the
+ *     torrent is published. Costs the archive's full size in disk.
+ *
+ *   discard — they are streamed past the hasher and dropped. Costs bandwidth
+ *     and time but no disk, and leaves the node unable to seed what it just
+ *     published. That is only viable because the origin URL is registered as a
+ *     web seed, so peers fetch over HTTP until someone mirrors it. Reasonable
+ *     for publishing something you already host; a poor default, because a
+ *     torrent nobody seeds is just HTTP with extra steps.
+ *
+ * Either way the origin becomes a web seed, since by definition it serves the
+ * exact bytes being hashed.
+ * @param {string} url - HTTP(S) URL of the archive.
+ * @param {CreateTorrentOptions & {retainPath?: string, onProgress?: Function}} [options] - Creation options. Omit retainPath to discard.
+ * @returns {Promise<CreatedTorrent & {retainedAt?: string}>} - The created torrent.
+ */
+export async function createTorrentFromUrl(url, options = {}) {
+  const name = options.name ?? path.basename(new URL(url).pathname);
+  // The origin is, by construction, a valid web seed for these exact bytes.
+  const webSeeds = [...new Set([...(options.webSeeds ?? []), url])];
+
+  if (options.retainPath) {
+    // Download first, then hash from disk. Two passes over local storage, but
+    // only one trip over the network — which is the expensive part — and it
+    // leaves a seedable copy behind.
+    const target = path.join(options.retainPath, name);
+    await downloadTo(url, target, options.onProgress);
+    const created = await createTorrentFromFile(target, {
+      ...options,
+      name,
+      webSeeds,
+    });
+    return { ...created, retainedAt: target };
+  }
+
+  const response = await fetch(url);
+  if (!response.ok || !response.body) {
+    throw new Error(
+      `could not read ${url}: ${response.status} ${response.statusText}`,
+    );
+  }
+  const size = Number(response.headers.get('content-length') ?? 0);
+  const stream = Readable.fromWeb(response.body);
+  return buildTorrent(stream, name, size, { ...options, webSeeds });
+}
+
+/**
+ * Streams a URL to a file, reporting progress.
+ * @param {string} url - Source URL.
+ * @param {string} target - Destination path.
+ * @param {Function} [onProgress] - Called with {received, total}.
+ * @returns {Promise<number>} - Bytes written.
+ */
+async function downloadTo(url, target, onProgress) {
+  const { createWriteStream } = await import('node:fs');
+  const { pipeline } = await import('node:stream/promises');
+
+  const response = await fetch(url);
+  if (!response.ok || !response.body) {
+    throw new Error(
+      `could not read ${url}: ${response.status} ${response.statusText}`,
+    );
+  }
+  const total = Number(response.headers.get('content-length') ?? 0);
+
+  await fs.mkdir(path.dirname(target), { recursive: true });
+
+  let received = 0;
+  let lastReport = 0;
+  const source = Readable.fromWeb(response.body);
+  source.on('data', (chunk) => {
+    received += chunk.length;
+    // Report at most once a second; a multi-hour download should not produce
+    // millions of log lines.
+    const now = Date.now();
+    if (onProgress && now - lastReport > 1000) {
+      lastReport = now;
+      onProgress({ received, total });
+    }
+  });
+
+  await pipeline(source, createWriteStream(target));
+  onProgress?.({ received, total, done: true });
+  return received;
+}
+
+/**
+ * Runs create-torrent and normalises its output.
+ * @param {string | import('node:stream').Readable} input - File path or stream.
+ * @param {string} name - Torrent name.
+ * @param {number} size - Total bytes, for reporting.
+ * @param {CreateTorrentOptions} options - Creation options.
+ * @returns {Promise<CreatedTorrent>} - The created torrent.
+ */
+async function buildTorrent(input, name, size, options) {
+  const [{ default: createTorrent }, { default: parseTorrent }] =
+    await Promise.all([import('create-torrent'), import('parse-torrent')]);
+
+  const pieceLength = options.pieceLength ?? 4 * 1024 * 1024;
+  if ((pieceLength & (pieceLength - 1)) !== 0) {
+    throw new Error(`pieceLength must be a power of two, got ${pieceLength}`);
+  }
+
+  // create-torrent wants a stream to carry a name and length.
+  if (typeof input !== 'string') {
+    input.name = name;
+    if (size) input.length = size;
+  }
+
+  const torrentFile = await new Promise((resolve, reject) => {
+    createTorrent(
+      input,
+      {
+        name: options.name ?? name,
+        pieceLength,
+        announceList: toAnnounceList(options.trackers ?? []),
+        urlList: options.webSeeds ?? [],
+        comment: options.comment,
+        private: options.private ?? false,
+        createdBy: 'pmtiles-swarm',
+      },
+      (error, buffer) => (error ? reject(error) : resolve(buffer)),
+    );
+  });
+
+  const parsed = await parseTorrent(torrentFile);
+  return {
+    torrentFile: new Uint8Array(torrentFile),
+    infoHash: parsed.infoHash,
+    magnet: buildMagnet(parsed, options),
+    name: parsed.name,
+    size: parsed.length ?? size,
+    pieceLength: parsed.pieceLength,
+    pieceCount: parsed.pieces?.length ?? 0,
+  };
+}
+
+/**
+ * Normalises trackers into BEP 12 announce tiers.
+ *
+ * A tier is a group of trackers tried together before falling back to the next
+ * one, which is how you pair the UDP and HTTP endpoints of the same tracker
+ * without announcing to both. Config accepts either form:
+ *
+ *   "udp://a:1337/announce"                      one tracker, its own tier
+ *   ["udp://b:6969/announce", "http://b:6969/…"] one tier, two endpoints
+ *
+ * This mirrors mktorrent, where `-a x` is a tier and `-a x,y` groups them.
+ * @param {Array<string | string[]>} trackers - Configured trackers.
+ * @returns {string[][]} - Announce tiers.
+ */
+function toAnnounceList(trackers) {
+  return trackers
+    .map((tier) => (Array.isArray(tier) ? tier : [tier]))
+    .filter((tier) => tier.length > 0);
+}
+
+/**
+ * Flattens announce tiers for magnet `tr=` parameters, which have no notion of
+ * tiers.
+ * @param {Array<string | string[]>} trackers - Configured trackers.
+ * @returns {string[]} - A flat list.
+ */
+function flattenTrackers(trackers) {
+  return trackers.flatMap((tier) => (Array.isArray(tier) ? tier : [tier]));
+}
+
+/**
+ * Builds a magnet URI carrying trackers and web seeds, so a magnet alone is
+ * enough to fetch the archive even with no peers.
+ * @param {object} parsed - A parse-torrent result.
+ * @param {CreateTorrentOptions} options - Creation options.
+ * @returns {string} - The magnet URI.
+ */
+function buildMagnet(parsed, options) {
+  const parts = [`magnet:?xt=urn:btih:${parsed.infoHash}`];
+  if (parsed.name) parts.push(`dn=${encodeURIComponent(parsed.name)}`);
+  for (const tracker of flattenTrackers(options.trackers ?? [])) {
+    parts.push(`tr=${encodeURIComponent(tracker)}`);
+  }
+  for (const seed of options.webSeeds ?? []) {
+    parts.push(`ws=${encodeURIComponent(seed)}`);
+  }
+  return parts.join('&');
+}