pmtiles-swarm 0.2.0

package/src/feed.js

@@ -0,0 +1,226 @@
+/**
+ * RSS publishing and subscribing.
+ *
+ * The feed is deliberately plain RSS 2.0 with a torrent enclosure, because
+ * that is what qBittorrent's built-in RSS auto-downloader already understands
+ * — an operator can subscribe to a pmtiles-swarm feed today, with no new
+ * software, and have new archives download automatically.
+ *
+ * On top of that it carries a small namespaced extension describing the map:
+ * coverage, zoom range and tile format. That is the part a generic torrent feed
+ * cannot offer, and it is what lets a subscriber decide whether it wants a
+ * 72 GiB download before starting one.
+ */
+
+const PMTILES_NS = 'https://github.com/TechIdiots-LLC/pmtiles-swarm/ns/1.0';
+
+/**
+ * Escapes text for XML content.
+ * @param {unknown} value - The value to escape.
+ * @returns {string} - Escaped text.
+ */
+function xml(value) {
+  return String(value ?? '')
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}
+
+/**
+ * Renders the catalog as an RSS 2.0 feed.
+ * @param {object[]} entries - Catalog entries to publish.
+ * @param {object} options - Feed options.
+ * @param {string} options.title - Feed title.
+ * @param {string} options.baseUrl - Public base URL for building links.
+ * @param {string} [options.description] - Feed description.
+ * @param {string} [options.copyright] - Rights statement for the channel.
+ * @param {string} [options.category] - Category this feed covers.
+ * @param {number} [options.maxItems] - Keep only this many newest items. Zero or absent means all.
+ * @returns {string} - The feed XML.
+ */
+export function renderFeed(entries, options) {
+  const self = options.category
+    ? `${options.baseUrl}/feed/${encodeURIComponent(options.category)}.xml`
+    : `${options.baseUrl}/feed.xml`;
+
+  // Entries arrive newest first, so a cap keeps the most recent builds. Set it
+  // with a subscriber's poll interval in mind: a feed holding one item is only
+  // safe if every subscriber polls more often than you publish, or a consumer
+  // that was down overnight silently misses a build.
+  const shown =
+    options.maxItems > 0 ? entries.slice(0, options.maxItems) : entries;
+
+  const items = shown
+    .map((entry) => renderItem(entry, options.baseUrl))
+    .join('\n');
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:pmtiles="${PMTILES_NS}">
+  <channel>
+    <title>${xml(options.title)}</title>
+    <link>${xml(options.baseUrl)}</link>
+    <description>${xml(options.description ?? 'PMTiles map archives distributed over BitTorrent')}</description>
+${options.copyright ? `    <copyright>${xml(options.copyright)}</copyright>\n` : ''}    <generator>pmtiles-swarm</generator>
+    <lastBuildDate>${new Date().toUTCString()}</lastBuildDate>
+    <atom:link href="${xml(self)}" rel="self" type="application/rss+xml"/>
+${items}
+  </channel>
+</rss>
+`;
+}
+
+/**
+ * Renders one catalog entry as a feed item.
+ * @param {object} entry - The catalog entry.
+ * @param {string} baseUrl - Public base URL.
+ * @returns {string} - The item XML.
+ */
+function renderItem(entry, baseUrl) {
+  const torrentUrl = `${baseUrl}/api/torrents/${entry.infoHash}/file`;
+  const map = entry.pmtiles;
+
+  const mapFields = map
+    ? [
+        `      <pmtiles:format>${xml(map.format)}</pmtiles:format>`,
+        `      <pmtiles:minzoom>${xml(map.minZoom)}</pmtiles:minzoom>`,
+        `      <pmtiles:maxzoom>${xml(map.maxZoom)}</pmtiles:maxzoom>`,
+        `      <pmtiles:bounds>${xml((map.bounds ?? []).join(','))}</pmtiles:bounds>`,
+        map.tileCount
+          ? `      <pmtiles:tiles>${xml(map.tileCount)}</pmtiles:tiles>`
+          : '',
+        map.attribution
+          ? `      <pmtiles:attribution>${xml(map.attribution)}</pmtiles:attribution>`
+          : '',
+      ]
+        .filter(Boolean)
+        .join('\n')
+    : '';
+
+  const summary = map
+    ? `${map.format} tiles, zoom ${map.minZoom}-${map.maxZoom}, ${formatBytes(entry.size)}`
+    : formatBytes(entry.size);
+
+  return `    <item>
+      <title>${xml(entry.name)}</title>
+      <link>${xml(torrentUrl)}</link>
+      <guid isPermaLink="false">${xml(entry.infoHash)}</guid>
+      <pubDate>${new Date(entry.createdAt).toUTCString()}</pubDate>
+      <description>${xml(map?.description ?? summary)}</description>
+${entry.category ? `      <category>${xml(entry.category)}</category>` : ''}
+      <enclosure url="${xml(torrentUrl)}" length="${xml(entry.size)}" type="application/x-bittorrent"/>
+      <pmtiles:infohash>${xml(entry.infoHash)}</pmtiles:infohash>
+      <pmtiles:magnet>${xml(entry.magnet)}</pmtiles:magnet>
+${mapFields}
+    </item>`;
+}
+
+/**
+ * Formats a byte count for human-readable descriptions.
+ * @param {number} bytes - The size.
+ * @returns {string} - A short label.
+ */
+export function formatBytes(bytes) {
+  if (!bytes) return 'unknown size';
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit++;
+  }
+  return `${value.toFixed(unit === 0 ? 0 : 2)} ${units[unit]}`;
+}
+
+/**
+ * One item pulled from a subscribed feed.
+ * @typedef {object} FeedItem
+ * @property {string} title - Item title.
+ * @property {string} [infoHash] - Infohash, when the feed states it.
+ * @property {string} [magnet] - Magnet URI, when present.
+ * @property {string} [torrentUrl] - URL of a .torrent enclosure.
+ * @property {string} [category] - Item category.
+ */
+
+/**
+ * Parses a subscribed feed.
+ *
+ * Written against the shape of RSS rather than with a full XML parser: feeds
+ * in this ecosystem are simple, and this keeps the dependency list short. It
+ * reads any RSS feed carrying torrent enclosures or magnets, not only ours.
+ * @param {string} body - The feed XML.
+ * @returns {FeedItem[]} - The items found.
+ */
+export function parseFeed(body) {
+  const items = [];
+  const itemPattern = /<item[\s>][\s\S]*?<\/item>/gi;
+
+  for (const [block] of body.matchAll(itemPattern)) {
+    const title = tag(block, 'title');
+    const enclosure = /<enclosure\b[^>]*\burl=["']([^"']+)["'][^>]*>/i.exec(
+      block,
+    );
+    const enclosureType =
+      /<enclosure\b[^>]*\btype=["']([^"']+)["'][^>]*>/i.exec(block)?.[1] ?? '';
+    const link = tag(block, 'link');
+    const magnetTag = tag(block, 'pmtiles:magnet');
+
+    // A magnet can arrive in its own element, as the link, or as the enclosure.
+    const candidates = [magnetTag, link, enclosure?.[1]].filter(Boolean);
+    const magnet = candidates.find((value) => value.startsWith('magnet:'));
+
+    // Only treat an enclosure as a torrent if it says so or ends in .torrent;
+    // otherwise it may be an image or an audio file.
+    const torrentUrl = [enclosure?.[1], link]
+      .filter(Boolean)
+      .find(
+        (value) =>
+          !value.startsWith('magnet:') &&
+          (/bittorrent/i.test(enclosureType) ||
+            /\.torrent(\?|$)/i.test(value) ||
+            /\/file$/i.test(value)),
+      );
+
+    if (!magnet && !torrentUrl) continue;
+
+    items.push({
+      title: title ?? 'untitled',
+      infoHash: tag(block, 'pmtiles:infohash')?.toLowerCase(),
+      magnet,
+      torrentUrl,
+      category: tag(block, 'category'),
+    });
+  }
+  return items;
+}
+
+/**
+ * Reads the text content of the first matching element.
+ * @param {string} block - The XML fragment to search.
+ * @param {string} name - Element name.
+ * @returns {string | undefined} - Decoded text, if found.
+ */
+function tag(block, name) {
+  const pattern = new RegExp(
+    `<${name}\\b[^>]*>([\\s\\S]*?)</${name}>`,
+    'i',
+  );
+  const match = pattern.exec(block);
+  if (!match) return undefined;
+  return decode(match[1].trim());
+}
+
+/**
+ * Decodes CDATA and the XML entities a feed may carry.
+ * @param {string} value - Raw element content.
+ * @returns {string} - Decoded text.
+ */
+function decode(value) {
+  return value
+    .replace(/^<!\[CDATA\[([\s\S]*?)\]\]>$/, '$1')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&amp;/g, '&');
+}

package/src/file-source.js

@@ -0,0 +1,67 @@
+import fs from 'node:fs';
+
+/**
+ * A PMTiles source reading a local file through a descriptor.
+ *
+ * Used both by the prober, which only touches the header and directories, and
+ * by the tile server when this node holds a complete copy of an archive. A
+ * complete local file needs no swarm involvement at all: reading it directly is
+ * both faster and simpler than routing through a torrent engine that would only
+ * hand back pieces it already has on disk.
+ */
+export class NodeFileSource {
+  #fd;
+  #path;
+
+  /**
+   * Opens the file.
+   * @param {string} filePath - Path to the archive.
+   */
+  constructor(filePath) {
+    this.#path = filePath;
+    this.#fd = fs.openSync(filePath, 'r');
+  }
+
+  /**
+   * A stable key for PMTiles' internal caching.
+   * @returns {string} - The file path.
+   */
+  getKey() {
+    return this.#path;
+  }
+
+  /**
+   * Reads a byte range.
+   * @param {number} offset - Byte offset.
+   * @param {number} length - Byte count.
+   * @returns {Promise<{data: ArrayBuffer}>} - The bytes.
+   */
+  async getBytes(offset, length) {
+    const buffer = Buffer.alloc(length);
+    const bytesRead = await new Promise((resolve, reject) => {
+      fs.read(this.#fd, buffer, 0, length, offset, (error, read) =>
+        error ? reject(error) : resolve(read),
+      );
+    });
+    // A short read at the end of the file is normal: PMTiles over-reads the
+    // header. Hand back only what exists.
+    const slice = buffer.subarray(0, bytesRead);
+    return {
+      data: slice.buffer.slice(
+        slice.byteOffset,
+        slice.byteOffset + slice.byteLength,
+      ),
+    };
+  }
+
+  /**
+   * Closes the descriptor.
+   * @returns {void}
+   */
+  close() {
+    if (this.#fd !== undefined) {
+      fs.closeSync(this.#fd);
+      this.#fd = undefined;
+    }
+  }
+}

package/src/index.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+import { parseArgs } from 'node:util';
+import { createApp } from './api.js';
+import { Catalog } from './catalog.js';
+import { loadConfig } from './config.js';
+import { LibtorrentEngine } from './engines/libtorrent.js';
+import { QBittorrentEngine } from './engines/qbittorrent.js';
+import { WebTorrentSeedEngine } from './engines/webtorrent.js';
+import { Library } from './library.js';
+import { ScheduledSourceManager } from './sources.js';
+import { SubscriptionManager } from './subscriptions.js';
+import { TileStore } from './tiles.js';
+import { WarmRunner } from './warm.js';
+import { WatchManager } from './watch.js';
+
+/**
+ * Builds the seeding engine named by the config.
+ * @param {object} config - Resolved configuration.
+ * @returns {import('./engines/types.js').SeedEngine} - The engine.
+ */
+function createEngine(config) {
+  switch (config.engine) {
+    case 'qbittorrent':
+      return new QBittorrentEngine(config.qbittorrent);
+    case 'libtorrent':
+      return new LibtorrentEngine({
+        savePath: config.libtorrent?.savePath ?? config.webtorrent.savePath,
+        resumeDir: config.libtorrent?.resumeDir,
+        python: config.libtorrent?.python,
+        maxConnections: config.maxConnections,
+        listen: config.libtorrent?.listen,
+      });
+    case 'webtorrent':
+      return new WebTorrentSeedEngine({
+        savePath: config.webtorrent.savePath,
+        clientOptions: config.webtorrent.clientOptions,
+        maxConnections: config.maxConnections,
+      });
+    default:
+      throw new Error(
+        `unknown engine "${config.engine}"; expected libtorrent, qbittorrent or webtorrent`,
+      );
+  }
+}
+
+/**
+ * Starts the daemon.
+ * @returns {Promise<void>} - Resolves once listening.
+ */
+async function main() {
+  const { values } = parseArgs({
+    options: {
+      config: { type: 'string', short: 'c' },
+      port: { type: 'string', short: 'p' },
+      help: { type: 'boolean', short: 'h' },
+    },
+    allowPositionals: false,
+  });
+
+  if (values.help) {
+    console.log(`pmtiles-swarm — BitTorrent distribution for PMTiles archives
+
+  --config, -c   path to a JSON config file
+  --port,   -p   override the listen port
+  --help,   -h   this message
+
+Environment: PMTILES_SWARM_PORT, PMTILES_SWARM_DATA_DIR, PMTILES_SWARM_ENGINE,
+PMTILES_SWARM_QBT_URL, PMTILES_SWARM_QBT_USERNAME, PMTILES_SWARM_QBT_PASSWORD,
+PMTILES_SWARM_PUBLIC_URL
+`);
+    return;
+  }
+
+  const config = await loadConfig(values.config);
+  if (values.port) config.port = Number(values.port);
+
+  await fs.mkdir(config.dataDir, { recursive: true });
+  if (config.engine === 'webtorrent') {
+    await fs.mkdir(config.webtorrent.savePath, { recursive: true });
+  }
+
+  const catalog = new Catalog(config.dataDir);
+  await catalog.load();
+
+  const engine = createEngine(config);
+  try {
+    await engine.connect();
+    console.log(`[engine] ${engine.name} ready`);
+  } catch (error) {
+    // A dead engine should not stop the daemon: the catalog and feed still
+    // work, and the engine may come back.
+    console.error(`[engine] ${engine.name} unavailable: ${error.message}`);
+  }
+
+  const library = new Library({ catalog, engine, config });
+  const subscriptions = new SubscriptionManager(library, config);
+  const watch = new WatchManager(library);
+  const sources = new ScheduledSourceManager(library, catalog, config);
+
+  const tiles = new TileStore({ catalog, engine, config });
+  const warm = new WarmRunner(tiles);
+
+  const app = createApp({
+    library,
+    catalog,
+    engine,
+    subscriptions,
+    tiles,
+    warm,
+    config,
+  });
+  const server = app.listen(config.port, config.host, () => {
+    console.log(
+      `[http] listening on http://${config.host}:${config.port} (${catalog.list().length} archives)`,
+    );
+  });
+
+  watch.start(config.watch);
+  subscriptions.start();
+  sources.start();
+
+  // Watch the sources archives were built from. A changed source does not
+  // invalidate its torrent, but it does mean any web seed pointing there will
+  // fail hash verification for every peer that tries it.
+  let originTimer;
+  if (config.originCheckIntervalSeconds > 0) {
+    const intervalMs = config.originCheckIntervalSeconds * 1000;
+    const runCheck = () =>
+      library
+        .checkAllOrigins()
+        .then((changed) => {
+          if (changed.length > 0) {
+            console.warn(
+              `[origin] ${changed.length} archive(s) no longer match their source`,
+            );
+          }
+        })
+        .catch((error) =>
+          console.error(`[origin] check failed: ${error.message}`),
+        );
+    runCheck();
+    originTimer = setInterval(runCheck, intervalMs);
+    originTimer.unref?.();
+    console.log(
+      `[origin] checking sources every ${config.originCheckIntervalSeconds}s`,
+    );
+  }
+
+  /**
+   * Shuts everything down in order, so trackers see a clean stop.
+   * @param {string} signal - The signal received.
+   * @returns {Promise<void>} - Resolves once stopped.
+   */
+  const shutdown = async (signal) => {
+    console.log(`\n[shutdown] ${signal}`);
+    if (originTimer) clearInterval(originTimer);
+    sources.stop();
+    subscriptions.stop();
+    await watch.stop().catch(() => {});
+    await new Promise((resolve) => server.close(resolve));
+    warm.stop();
+    // Before the engine, so readers let go of their torrents while the client
+    // that owns them is still alive.
+    await tiles.close().catch(() => {});
+    await engine.destroy().catch(() => {});
+    process.exit(0);
+  };
+
+  process.on('SIGINT', () => shutdown('SIGINT'));
+  process.on('SIGTERM', () => shutdown('SIGTERM'));
+}
+
+main().catch((error) => {
+  console.error(error.stack ?? error.message);
+  process.exit(1);
+});