pmtiles-swarm 0.2.0

package/src/config.js

@@ -0,0 +1,231 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+/**
+ * Defaults. Every one of these can be overridden in the config file, and a few
+ * of the deployment-specific ones by environment variables.
+ */
+const DEFAULTS = {
+  port: 8090,
+  host: '0.0.0.0',
+  /** Where the catalog, generated .torrent files and keys live. */
+  dataDir: './data',
+  /** Which SeedEngine to use: 'webtorrent' or 'qbittorrent'. */
+  engine: 'webtorrent',
+  qbittorrent: {
+    url: 'http://127.0.0.1:8080',
+    username: undefined,
+    password: undefined,
+  },
+  webtorrent: {
+    savePath: './data/torrents-data',
+  },
+  /**
+   * Piece length for torrents we create. 4 MiB is a deliberate compromise:
+   * tools default much higher for large files, which is fine for whole-file
+   * downloads but terrible for the random access a tile server does, since
+   * every cold tile costs a whole piece. Smaller pieces cut that, at the cost
+   * of a larger hash list that peers must transfer before any tile is served.
+   *
+   * Overridable per source and per request, so an archive nobody will read
+   * randomly can keep the larger pieces its size would normally suggest.
+   *
+   * Note this has little bearing on load imposed on network equipment: peers
+   * request data in 16 KiB blocks whatever the piece size, so packet volume is
+   * unchanged. What strains a consumer router is the number of simultaneous
+   * connections holding NAT table entries — see maxConnections.
+   */
+  pieceLength: 4 * 1024 * 1024,
+  /**
+   * Cap on simultaneous peer connections.
+   *
+   * This is the setting that decides how hard the node leans on a router:
+   * every peer is a NAT table entry, and cheap consumer hardware starts
+   * dropping or stalling connections well before a server would. Lower it if
+   * the network misbehaves while seeding.
+   */
+  maxConnections: 100,
+  /** Trackers baked into every torrent we create. */
+  trackers: [
+    'udp://tracker.opentrackr.org:1337/announce',
+    'udp://tracker.torrent.eu.org:451/announce',
+  ],
+  /**
+   * Copy every .torrent we create into this directory as well.
+   *
+   * Most clients, qBittorrent included, can watch a folder and add whatever
+   * appears in it. For the single job of "start seeding this", that is simpler
+   * and more robust than an API, and it works when the client shares a disk but
+   * is not reachable over HTTP.
+   */
+  torrentDropDir: undefined,
+  /** Rights statement for the RSS channel. */
+  feedCopyright: undefined,
+  /**
+   * Most items to include in a feed, newest first. Zero means no limit.
+   *
+   * Choose it against how often subscribers poll, not how tidy the feed looks:
+   * a feed holding a single item is only safe if everyone polls more often than
+   * you publish. A consumer that was down overnight would otherwise miss that
+   * build entirely, with nothing to indicate it had.
+   */
+  feedMaxItems: 50,
+  /** Scheduled upstreams that publish a new archive per date. See sources.js. */
+  sources: [],
+  /** How often to poll scheduled sources, in hours. */
+  sourceCheckIntervalHours: 6,
+  /** Public base URL, used to build absolute links in the RSS feed and TileJSON. */
+  publicUrl: undefined,
+  /**
+   * Trust X-Forwarded-* headers, for running behind a reverse proxy or CDN.
+   *
+   * Takes anything Express accepts: `true`, a hop count, or a subnet list such
+   * as "loopback, 10.0.0.0/8". Off by default, because trusting these headers
+   * from an untrusted client lets it claim any protocol or address it likes.
+   *
+   * Set it when a proxy terminates TLS, or the TileJSON will advertise http://
+   * tile URLs that browsers block as mixed content. Setting `publicUrl`
+   * instead sidesteps the question entirely.
+   */
+  trustProxy: false,
+  /**
+   * Tile serving: a TileJSON endpoint and z/x/y tiles per archive.
+   *
+   * A node holding a complete copy reads its local file. A node in cache mode
+   * reads through the swarm, pulling only the pieces a requested tile lives in
+   * — which is what lets a machine with 10 GiB free serve a 700 GiB planet.
+   */
+  tiles: {
+    /**
+     * Open archives kept alive at once. Each holds a file descriptor or a
+     * torrent reader plus its piece cache, so this bounds both.
+     */
+    maxOpenArchives: 16,
+    /** Header and directory cache entries, shared across every archive. */
+    directoryCacheEntries: 200,
+    /**
+     * Byte budget for the piece cache of one swarm-read archive. Left unset it
+     * is sized from the torrent's piece length, which is the safer default: a
+     * fixed budget is a trap with 16 MiB pieces, since 64 MiB holds only four.
+     */
+    pieceCacheBytes: undefined,
+    /** How long no read must be in flight before background hydration resumes. */
+    hydrateIdleMs: undefined,
+    /** How long to wait for one piece before giving up on a tile. */
+    pieceTimeoutMs: 120000,
+    /** How long to wait for torrent metadata when opening an archive. */
+    readyTimeoutMs: 60000,
+  },
+  /** Folders scanned for new archives: [{ path, category, webSeedBase }]. */
+  watch: [],
+  /** Feeds to follow: [{ url, mode, category }] where mode is 'mirror' or 'cache'. */
+  subscriptions: [],
+  /**
+   * How often to re-check whether the sources archives were built from have
+   * changed, in seconds. Zero disables it. A check is one HEAD request or stat
+   * per archive, so this is cheap; it defaults off only because a node that
+   * merely joins other people's torrents has nothing to check.
+   */
+  originCheckIntervalSeconds: 0,
+  /**
+   * Rebuild an archive automatically when its source changes.
+   *
+   * Off by default, and guarded when on, because a rebuild re-hashes the
+   * archive and for a remote source re-downloads it — potentially hours of
+   * transfer started by nobody. Enable it for local build outputs, where the
+   * cost is a local read; think harder before enabling it for http sources.
+   */
+  autoRebuild: {
+    enabled: false,
+    /** Source types eligible. 'http' means re-downloading the whole archive. */
+    sources: ['file'],
+    /** Skip anything larger than this. Zero disables the cap. */
+    maxBytes: 50 * 1024 * 1024 * 1024,
+    /** The source must be unchanged for this long before rebuilding. */
+    stabilitySeconds: 300,
+  },
+  /** How often to poll subscribed feeds, in seconds. */
+  subscriptionIntervalSeconds: 900,
+  /** Republish interval for BEP 46 records, in seconds. DHT items expire. */
+  republishIntervalSeconds: 3600,
+};
+
+/**
+ * Merges a config object over the defaults, one level deep so nested engine
+ * settings can be partially overridden.
+ * @param {object} base - Defaults.
+ * @param {object} override - User values.
+ * @returns {object} - The merged config.
+ */
+function merge(base, override) {
+  const out = { ...base };
+  for (const [key, value] of Object.entries(override ?? {})) {
+    if (value === undefined) continue;
+    // eslint-disable-next-line security/detect-object-injection -- keys come from a config file the operator controls
+    out[key] =
+      value && typeof value === 'object' && !Array.isArray(value)
+        ? // eslint-disable-next-line security/detect-object-injection -- as above
+          merge(base[key] ?? {}, value)
+        : value;
+  }
+  return out;
+}
+
+/**
+ * Loads configuration from a JSON file, applying defaults and environment
+ * overrides. A missing file is fine — the defaults run.
+ * @param {string} [configPath] - Path to a JSON config file.
+ * @returns {Promise<object>} - The resolved configuration.
+ */
+export async function loadConfig(configPath) {
+  let fileConfig = {};
+  if (configPath) {
+    try {
+      fileConfig = JSON.parse(await fs.readFile(configPath, 'utf8'));
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw new Error(
+          `could not read config ${configPath}: ${error.message}`,
+          { cause: error },
+        );
+      }
+    }
+  }
+
+  const config = merge(DEFAULTS, fileConfig);
+
+  // Environment overrides, for containerised deployments.
+  if (process.env.PMTILES_SWARM_PORT) {
+    config.port = Number(process.env.PMTILES_SWARM_PORT);
+  }
+  if (process.env.PMTILES_SWARM_DATA_DIR) {
+    config.dataDir = process.env.PMTILES_SWARM_DATA_DIR;
+  }
+  if (process.env.PMTILES_SWARM_ENGINE) {
+    config.engine = process.env.PMTILES_SWARM_ENGINE;
+  }
+  if (process.env.PMTILES_SWARM_QBT_URL) {
+    config.qbittorrent.url = process.env.PMTILES_SWARM_QBT_URL;
+  }
+  if (process.env.PMTILES_SWARM_QBT_USERNAME) {
+    config.qbittorrent.username = process.env.PMTILES_SWARM_QBT_USERNAME;
+  }
+  if (process.env.PMTILES_SWARM_QBT_PASSWORD) {
+    config.qbittorrent.password = process.env.PMTILES_SWARM_QBT_PASSWORD;
+  }
+  if (process.env.PMTILES_SWARM_PUBLIC_URL) {
+    config.publicUrl = process.env.PMTILES_SWARM_PUBLIC_URL;
+  }
+
+  // Resolve paths relative to the config file, so a config can be moved as a
+  // unit with the data it points at.
+  const base = configPath ? path.dirname(path.resolve(configPath)) : process.cwd();
+  config.dataDir = path.resolve(base, config.dataDir);
+  config.webtorrent.savePath = path.resolve(base, config.webtorrent.savePath);
+  config.watch = config.watch.map((entry) => ({
+    ...entry,
+    path: path.resolve(base, entry.path),
+  }));
+
+  return config;
+}

package/src/engines/libtorrent.js

@@ -0,0 +1,384 @@
+import { spawn } from 'node:child_process';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+
+/**
+ * Locates the libtorrent sidecar script.
+ *
+ * It lives in pmtiles-torrent rather than here. The two projects used to carry
+ * a copy each, which drifted — the read side grew `info` and `priority` ops
+ * that this copy never got. Since pmtiles-torrent is now a dependency and
+ * ships the script, there is one file again, and the read and seed sides
+ * cannot disagree about the protocol they speak over the same pipe.
+ * @returns {string} - Absolute path to the sidecar script.
+ */
+function resolveSidecar() {
+  const require = createRequire(import.meta.url);
+  try {
+    return path.join(
+      path.dirname(require.resolve('pmtiles-torrent/package.json')),
+      'sidecar',
+      'libtorrent_sidecar.py',
+    );
+  } catch (error) {
+    throw new Error(
+      'cannot locate the libtorrent sidecar: pmtiles-torrent is not resolvable. ' +
+        `Run npm install, or pass libtorrent.script to point at it. (${error.message})`,
+    );
+  }
+}
+
+/**
+ * A SeedEngine backed by libtorrent, through a sidecar process.
+ *
+ * libtorrent is the only one of the three engines that offers what large-scale
+ * map distribution actually wants: BitTorrent v2 and hybrid torrents, resume
+ * data so a restart does not re-hash the store, piece-level control for
+ * on-demand reads, and seeding that holds up at multi-terabyte scale.
+ *
+ * It reaches it through a child process rather than a native binding because
+ * Node has no maintained libtorrent binding — the packages on npm are
+ * abandoned 2022 stubs, and the one live fork exposes neither piece deadlines
+ * nor v2. A sidecar also keeps the install honest: one distro package rather
+ * than a C++ toolchain plus Boost.
+ *
+ * The protocol is line-delimited JSON, so this class is unchanged if the other
+ * end is later replaced by a real N-API addon.
+ */
+export class LibtorrentEngine {
+  #options;
+  #child = null;
+  #pending = new Map();
+  #nextId = 1;
+  #buffer = '';
+  #ready = null;
+  #version = null;
+
+  /**
+   * Creates the engine.
+   * @param {object} options - Engine options.
+   * @param {string} options.savePath - Default directory for torrent data.
+   * @param {string} [options.resumeDir] - Where resume data is kept.
+   * @param {string} [options.python] - Python executable. Default 'python3'.
+   * @param {string} [options.script] - Override the sidecar script path.
+   * @param {string} [options.listen] - Listen interfaces, e.g. '0.0.0.0:6881'.
+   * @param {number} [options.startTimeoutMs] - How long to wait for the sidecar. Default 20s.
+   */
+  constructor(options) {
+    if (!options?.savePath) {
+      throw new Error('libtorrent engine requires a savePath');
+    }
+    this.name = 'libtorrent';
+    this.#options = { python: 'python3', startTimeoutMs: 20000, ...options };
+  }
+
+  /** @returns {string | null} - libtorrent version, once connected. */
+  get version() {
+    return this.#version;
+  }
+
+  /**
+   * Starts the sidecar and waits for it to report readiness.
+   * @returns {Promise<void>} - Resolves once usable.
+   */
+  connect() {
+    if (this.#ready) return this.#ready;
+
+    this.#ready = new Promise((resolve, reject) => {
+      const script = this.#options.script ?? resolveSidecar();
+
+      const child = spawn(this.#options.python, [script], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        env: {
+          ...process.env,
+          SIDECAR_SETTINGS: JSON.stringify({
+            listen: this.#options.listen,
+            resumeDir: this.#options.resumeDir,
+            maxConnections: this.#options.maxConnections,
+            dht: this.#options.dht,
+            uploadLimit: this.#options.uploadLimit,
+            downloadLimit: this.#options.downloadLimit,
+          }),
+        },
+      });
+      this.#child = child;
+
+      const timer = setTimeout(() => {
+        reject(
+          new Error(
+            `libtorrent sidecar did not start within ${this.#options.startTimeoutMs}ms`,
+          ),
+        );
+        child.kill();
+      }, this.#options.startTimeoutMs);
+
+      child.stdout.setEncoding('utf8');
+      child.stdout.on('data', (chunk) => {
+        this.#buffer += chunk;
+        let newline;
+        while ((newline = this.#buffer.indexOf('\n')) >= 0) {
+          const line = this.#buffer.slice(0, newline).trim();
+          this.#buffer = this.#buffer.slice(newline + 1);
+          if (!line) continue;
+
+          let message;
+          try {
+            message = JSON.parse(line);
+          } catch {
+            console.error(`[libtorrent] unparseable output: ${line}`);
+            continue;
+          }
+
+          if (message.event === 'ready') {
+            clearTimeout(timer);
+            this.#version = message.libtorrent;
+            resolve();
+            continue;
+          }
+          this.#settle(message);
+        }
+      });
+
+      // The sidecar reports missing bindings and tracebacks on stderr; those
+      // are the errors an operator most needs to see.
+      child.stderr.setEncoding('utf8');
+      child.stderr.on('data', (text) => {
+        for (const line of text.split('\n')) {
+          if (line.trim()) console.error(`[libtorrent] ${line}`);
+        }
+      });
+
+      child.on('error', (error) => {
+        clearTimeout(timer);
+        reject(
+          new Error(
+            `could not start ${this.#options.python}: ${error.message}. ` +
+              'Install python3 and libtorrent (apt install python3-libtorrent).',
+            { cause: error },
+          ),
+        );
+      });
+
+      child.on('exit', (code) => {
+        clearTimeout(timer);
+        this.#child = null;
+        const error = new Error(`libtorrent sidecar exited (code ${code})`);
+        for (const { reject: fail } of this.#pending.values()) fail(error);
+        this.#pending.clear();
+        reject(error);
+      });
+    });
+
+    return this.#ready;
+  }
+
+  /**
+   * Adds a torrent.
+   * @param {import('./types.js').AddRequest} request - What to add.
+   * @returns {Promise<string>} - The infohash.
+   */
+  async add(request) {
+    const result = await this.#call('add', {
+      torrentFile: request.torrentFile
+        ? Buffer.from(request.torrentFile).toString('base64')
+        : undefined,
+      magnet: request.magnet,
+      savePath: request.savePath ?? this.#options.savePath,
+      mode: request.mode,
+      paused: request.paused,
+    });
+    return result.infoHash;
+  }
+
+  /**
+   * Removes a torrent.
+   * @param {string} infoHash - The torrent to remove.
+   * @param {{deleteData?: boolean}} [options] - Whether to delete data too.
+   * @returns {Promise<void>} - Resolves once removed.
+   */
+  async remove(infoHash, options = {}) {
+    await this.#call('remove', {
+      infoHash,
+      deleteData: Boolean(options.deleteData),
+    });
+  }
+
+  /**
+   * Lists every torrent in the session.
+   * @returns {Promise<import('./types.js').TorrentStatus[]>} - Statuses.
+   */
+  async list() {
+    return this.#call('list', {});
+  }
+
+  /**
+   * One torrent's state.
+   * @param {string} infoHash - The torrent to look up.
+   * @returns {Promise<import('./types.js').TorrentStatus | null>} - Its status.
+   */
+  async get(infoHash) {
+    return this.#call('get', { infoHash });
+  }
+
+  /**
+   * Per-peer detail.
+   * @param {string} infoHash - The torrent to inspect.
+   * @returns {Promise<object[]>} - One entry per peer.
+   */
+  async peers(infoHash) {
+    return this.#call('peers', { infoHash });
+  }
+
+  /**
+   * Creates a torrent from a local file.
+   *
+   * Defaults to hybrid v1+v2, which is the capability that justifies this
+   * engine: v2 gives per-file merkle trees with 16 KiB leaf blocks, so a peer
+   * can verify a small block without holding the whole hash list, while the v1
+   * half keeps every existing client working.
+   * @param {string} filePath - Path to the archive.
+   * @param {object} [options] - Piece length, trackers, web seeds, format.
+   * @returns {Promise<object>} - The created torrent, torrentFile as bytes.
+   */
+  async createTorrent(filePath, options = {}) {
+    const result = await this.#call(
+      'create',
+      {
+        path: filePath,
+        pieceLength: options.pieceLength,
+        trackers: options.trackers ?? [],
+        webSeeds: options.webSeeds ?? [],
+        comment: options.comment,
+        format: options.format ?? 'hybrid',
+      },
+      // Hashing a large archive takes as long as it takes.
+      options.timeoutMs ?? 6 * 60 * 60 * 1000,
+    );
+    return {
+      ...result,
+      torrentFile: new Uint8Array(Buffer.from(result.torrentFile, 'base64')),
+    };
+  }
+
+  /**
+   * Reads one piece, promoted ahead of the normal picker.
+   *
+   * This is the primitive on-demand tile serving wants, and the reason
+   * qBittorrent cannot do cache mode properly — its WebUI has no equivalent.
+   * @param {string} infoHash - The torrent.
+   * @param {number} piece - Piece index.
+   * @param {object} [options] - Deadline and timeout.
+   * @returns {Promise<Uint8Array>} - The piece contents.
+   */
+  async readPiece(infoHash, piece, options = {}) {
+    const result = await this.#call(
+      'read_piece',
+      {
+        infoHash,
+        piece,
+        deadlineMs: options.deadlineMs,
+        timeoutMs: options.timeoutMs,
+      },
+      (options.timeoutMs ?? 60000) + 5000,
+    );
+    return new Uint8Array(Buffer.from(result.data, 'base64'));
+  }
+
+  /**
+   * Reports the piece geometry a reader needs to map byte ranges onto pieces.
+   *
+   * Note the two coordinate systems: `pieceLength` and `numPieces` describe the
+   * torrent's global byte space, while `fileOffset` locates the archive inside
+   * it. A single-file torrent has a zero offset; a multi-file one does not, and
+   * getting that wrong reads the neighbouring file.
+   * @param {string} infoHash - The torrent.
+   * @param {number} [fileIndex] - Which file in a multi-file torrent.
+   * @returns {Promise<object>} - {infoHash, pieceLength, numPieces, fileLength, fileOffset, name}.
+   */
+  async info(infoHash, fileIndex = 0) {
+    return this.#call('info', { infoHash, fileIndex });
+  }
+
+  /**
+   * Sets the download priority of a piece range.
+   *
+   * Zero means "do not fetch", which is how cache mode avoids pulling an entire
+   * archive while still seeding what it holds.
+   * @param {string} infoHash - The torrent.
+   * @param {number} first - First piece index, inclusive.
+   * @param {number} last - Last piece index, inclusive.
+   * @param {number} priority - libtorrent piece priority, 0 to 7.
+   * @returns {Promise<void>} - Resolves once applied.
+   */
+  async setPriority(infoHash, first, last, priority) {
+    await this.#call('set_priority', { infoHash, first, last, priority });
+  }
+
+  /**
+   * Persists resume data, so the next start skips re-hashing the store.
+   * @param {string} [infoHash] - One torrent, or all when omitted.
+   * @returns {Promise<void>} - Resolves once saved.
+   */
+  async saveResume(infoHash) {
+    await this.#call('save_resume', { infoHash });
+  }
+
+  /**
+   * Saves resume data and stops the sidecar.
+   * @returns {Promise<void>} - Resolves once stopped.
+   */
+  async destroy() {
+    if (!this.#child) return;
+    await this.#call('shutdown', {}, 15000).catch(() => {});
+    this.#child?.kill();
+    this.#child = null;
+    this.#ready = null;
+  }
+
+  /**
+   * Sends a request and waits for its reply.
+   * @param {string} op - Operation name.
+   * @param {object} params - Operation parameters.
+   * @param {number} [timeoutMs] - How long to wait. Default 60s.
+   * @returns {Promise<any>} - The result.
+   */
+  async #call(op, params, timeoutMs = 60000) {
+    if (op !== 'shutdown') await this.connect();
+    const child = this.#child;
+    if (!child) throw new Error('libtorrent sidecar is not running');
+
+    const id = this.#nextId++;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.#pending.delete(id);
+        reject(new Error(`libtorrent ${op} timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
+
+      this.#pending.set(id, {
+        resolve: (value) => {
+          clearTimeout(timer);
+          resolve(value);
+        },
+        reject: (error) => {
+          clearTimeout(timer);
+          reject(error);
+        },
+      });
+
+      child.stdin.write(`${JSON.stringify({ id, op, params })}\n`);
+    });
+  }
+
+  /**
+   * Routes a reply to whoever is waiting for it.
+   * @param {object} message - A decoded reply.
+   * @returns {void}
+   */
+  #settle(message) {
+    const waiter = this.#pending.get(message.id);
+    if (!waiter) return;
+    this.#pending.delete(message.id);
+    if (message.ok) waiter.resolve(message.result);
+    else waiter.reject(new Error(message.error ?? 'unknown sidecar error'));
+  }
+}