total-recall-brain 3.28.2 → 3.30.0

package/src/core/plugin-peers.mjs

@@ -0,0 +1,163 @@
+/**
+ * Peer-to-peer plugin discovery over the Total Recall mesh.
+ *
+ * There is no central catalog. Each node serves the plugins its owner chose to
+ * share (GET /api/mesh/plugins, see routes/plugins-mesh.mjs) and this module
+ * asks every online peer directly. What comes back is reported per peer
+ * exactly as observed — reachable, offline, not configured, running a version
+ * without plugin sharing, or failing — never smoothed over.
+ */
+import { listEnrichedMeshNodes, isMeshAvailable } from './mesh.mjs';
+import { getMeshSyncAuthorization } from './mesh-auth.mjs';
+import { throttledFetch } from './throttled-fetch.mjs';
+
+const LIST_TIMEOUT_MS = 5_000;
+const BUNDLE_TIMEOUT_MS = 20_000;
+const MAX_BUNDLE_RESPONSE_BYTES = 8 * 1024 * 1024; // base64 of the 5 MB bundle cap, plus envelope
+
+function serverPort() {
+  const value = Number(process.env.TR_SERVER_PORT || process.env.PORT || 3000);
+  return Number.isInteger(value) && value > 0 && value <= 65535 ? value : 3000;
+}
+
+/** Build a peer URL, refusing anything outside the mesh CGNAT range (100.64.0.0/10). */
+export function meshPeerUrl(ip, route, port = serverPort()) {
+  const octets = String(ip || '').split('.').map(Number);
+  if (
+    octets.length !== 4 ||
+    octets.some((part) => !Number.isInteger(part) || part < 0 || part > 255) ||
+    octets[0] !== 100 || octets[1] < 64 || octets[1] > 127
+  ) {
+    throw new Error(`Peer address ${ip} is outside the mesh range`);
+  }
+  if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('Invalid peer brain port');
+  return `http://${ip}:${port}${route}`;
+}
+
+function describeHttpFailure(status) {
+  if (status === 404) return { status: 'unsupported', error: 'Peer does not serve plugins (older Total Recall version)' };
+  if (status === 401) return { status: 'error', error: 'Peer rejected the mesh sync credential' };
+  if (status === 403) return { status: 'error', error: 'Peer refused the request source address' };
+  if (status === 503) return { status: 'not_configured', error: 'Mesh sync is not configured on the peer' };
+  return { status: 'error', error: `Peer returned HTTP ${status}` };
+}
+
+async function readJsonOrNull(res) {
+  const type = String(res.headers?.get?.('content-type') || '');
+  if (type && !type.includes('json')) return null;
+  try {
+    const body = await res.json();
+    return body && typeof body === 'object' ? body : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Ask every mesh peer which plugins it shares.
+ * @param {{ peers?: object[], fetchImpl?: Function, authorization?: string|null, meshAvailable?: boolean }} [deps]
+ */
+export async function listPeerPlugins(deps = {}) {
+  const meshAvailable = deps.meshAvailable ?? isMeshAvailable();
+  if (!meshAvailable) {
+    return { mesh: { available: false, configured: false }, peers: [] };
+  }
+
+  const peers = (deps.peers ?? listEnrichedMeshNodes()).filter((p) => !p.self);
+  let authorization = deps.authorization;
+  if (authorization === undefined) {
+    try {
+      authorization = await getMeshSyncAuthorization();
+    } catch {
+      authorization = null;
+    }
+  }
+  const doFetch = deps.fetchImpl || throttledFetch;
+
+  const results = await Promise.all(peers.map(async (peer) => {
+    const base = { hostname: peer.hostname, ip: peer.ip, online: !!peer.online, os: peer.os || null, plugins: [] };
+    if (!peer.online) return { ...base, status: 'offline' };
+    if (!authorization) {
+      return { ...base, status: 'not_configured', error: 'TR_MESH_SYNC_TOKEN is not set on this node' };
+    }
+    try {
+      const res = await doFetch(meshPeerUrl(peer.ip, '/api/mesh/plugins', peer.brain_port || serverPort()), { headers: { Authorization: authorization } }, LIST_TIMEOUT_MS);
+      if (!res.ok) return { ...base, ...describeHttpFailure(res.status) };
+      // An older brain has no such route and its SPA fallback answers with the
+      // dashboard's index.html — that is "does not serve plugins", not a parse error.
+      const body = await readJsonOrNull(res);
+      if (!body) return { ...base, ...describeHttpFailure(404) };
+      const plugins = Array.isArray(body?.plugins) ? body.plugins : [];
+      return {
+        ...base,
+        status: 'ok',
+        plugins: plugins
+          .filter((p) => p && typeof p.id === 'string' && typeof p.sha256 === 'string')
+          .map((p) => ({
+            id: p.id,
+            name: String(p.name || p.id),
+            version: String(p.version || ''),
+            description: String(p.description || ''),
+            use_cases: Array.isArray(p.use_cases) ? p.use_cases.map(String) : [],
+            sha256: p.sha256,
+            file_count: Number(p.file_count) || 0,
+            size_bytes: Number(p.size_bytes) || 0
+          }))
+      };
+    } catch (err) {
+      const msg = err?.message || String(err);
+      return {
+        ...base,
+        status: 'unreachable',
+        error: msg === 'fetch failed' ? `No Total Recall server answered at ${peer.ip}:${peer.brain_port || serverPort()}` : msg
+      };
+    }
+  }));
+
+  results.sort((a, b) => String(a.hostname).localeCompare(String(b.hostname)));
+  return { mesh: { available: true, configured: !!authorization }, peers: results };
+}
+
+/** Parse `peer:<hostname>/<id>`. */
+export function parsePeerSource(source) {
+  const m = /^peer:([^/\s]+)\/([a-z][a-z0-9-]{1,63})$/.exec(String(source || '').trim());
+  return m ? { hostname: m[1], id: m[2] } : null;
+}
+
+function sameHost(a, b) {
+  const norm = (h) => String(h || '').toLowerCase().replace(/\.$/, '').split('.')[0];
+  return norm(a) === norm(b);
+}
+
+/**
+ * Fetch one plugin bundle from a named peer. The advertised hash comes from the
+ * peer's own listing; plugin-bundle.decodeBundle() verifies the bytes against it.
+ * @returns {Promise<{ bundle: object, advertised: object, peer: object }>}
+ */
+export async function fetchPeerBundle(hostname, id, deps = {}) {
+  const peers = deps.peers ?? listEnrichedMeshNodes();
+  const peer = peers.find((p) => !p.self && (sameHost(p.hostname, hostname) || p.ip === hostname));
+  if (!peer) throw new Error(`No mesh peer named '${hostname}'`);
+  if (!peer.online) throw new Error(`Mesh peer '${peer.hostname}' is offline`);
+
+  const authorization = deps.authorization ?? await getMeshSyncAuthorization();
+  const doFetch = deps.fetchImpl || throttledFetch;
+  const headers = { Authorization: authorization };
+
+  const listRes = await doFetch(meshPeerUrl(peer.ip, '/api/mesh/plugins', peer.brain_port || serverPort()), { headers }, LIST_TIMEOUT_MS);
+  if (!listRes.ok) throw new Error(describeHttpFailure(listRes.status).error);
+  const listing = await readJsonOrNull(listRes);
+  if (!listing) throw new Error(describeHttpFailure(404).error);
+  const advertised = (listing?.plugins || []).find((p) => p.id === id);
+  if (!advertised) throw new Error(`Peer '${peer.hostname}' does not share plugin '${id}'`);
+
+  const res = await doFetch(
+    meshPeerUrl(peer.ip, `/api/mesh/plugins/${encodeURIComponent(id)}/bundle`, peer.brain_port || serverPort()),
+    { headers },
+    BUNDLE_TIMEOUT_MS
+  );
+  if (!res.ok) throw new Error(describeHttpFailure(res.status).error);
+  const text = await res.text();
+  if (text.length > MAX_BUNDLE_RESPONSE_BYTES) throw new Error('Peer bundle exceeds the size limit');
+  return { bundle: JSON.parse(text), advertised, peer };
+}

package/src/core/plugin-public.mjs

@@ -0,0 +1,85 @@
+/** Direct, hash-pinned plugin exchange between Total Recall users. */
+import dns from 'node:dns/promises';
+import https from 'node:https';
+import net from 'node:net';
+
+const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+const SOURCE_PATH = /^\/api\/public\/plugins\/([a-z][a-z0-9-]{1,63})\/bundle$/;
+
+function publicIpv4(address) {
+  if (net.isIP(address) !== 4) return false;
+  const [a, b, c] = address.split('.').map(Number);
+  if (a === 0 || a === 10 || a === 127 || a >= 224) return false;
+  if (a === 100 && b >= 64 && b <= 127) return false;
+  if (a === 169 && b === 254) return false;
+  if (a === 172 && b >= 16 && b <= 31) return false;
+  if (a === 192 && (b === 168 || b === 0 || (b === 88 && c === 99))) return false;
+  if (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100))) return false;
+  if (a === 203 && b === 0 && c === 113) return false;
+  return true;
+}
+
+export function parsePublicPluginSource(source) {
+  let url;
+  try { url = new URL(source); } catch { return null; }
+  const match = SOURCE_PATH.exec(url.pathname);
+  if (!match) return null;
+  if (url.protocol !== 'https:' || url.username || url.password || url.search || url.port) {
+    throw new Error('Plugin share links require HTTPS with no credentials, query, or custom port');
+  }
+  const hash = /^#sha256=([a-f0-9]{64})$/.exec(url.hash);
+  if (!hash) throw new Error('Plugin share link is missing its SHA-256 content pin');
+  return { url, id: match[1], sha256: hash[1] };
+}
+
+export function publicPluginShareUrl(id, sha256, base = process.env.TR_PUBLIC_BASE_URL) {
+  if (!base) return null;
+  const origin = new URL(base);
+  if (origin.protocol !== 'https:' || origin.username || origin.password || origin.search || origin.hash || origin.pathname !== '/' || origin.port) {
+    throw new Error('TR_PUBLIC_BASE_URL must be a public HTTPS origin without a path or custom port');
+  }
+  if (!/^[a-z][a-z0-9-]{1,63}$/.test(id) || !/^[a-f0-9]{64}$/.test(sha256)) throw new Error('Invalid plugin share link data');
+  return `${origin.origin}/api/public/plugins/${id}/bundle#sha256=${sha256}`;
+}
+
+export async function fetchPublicBundle(source, deps = {}) {
+  const parsed = parsePublicPluginSource(source);
+  if (!parsed) throw new Error('Invalid Total Recall plugin share link');
+  const host = parsed.url.hostname;
+  const addresses = net.isIP(host) ? [{ address: host }] : await (deps.lookup || dns.lookup)(host, { all: true, family: 4 });
+  const selected = addresses.find(({ address }) => publicIpv4(address));
+  if (!selected || addresses.some(({ address }) => !publicIpv4(address))) {
+    throw new Error('Plugin share host must resolve only to public IPv4 addresses');
+  }
+  const request = deps.request || https.request;
+  const body = await new Promise((resolve, reject) => {
+    const req = request(parsed.url, {
+      method: 'GET',
+      timeout: 20_000,
+      lookup: (_host, _options, callback) => callback(null, selected.address, 4),
+      headers: { Accept: 'application/json' }
+    }, (res) => {
+      if (res.statusCode !== 200) {
+        res.resume();
+        reject(new Error(`Plugin share returned HTTP ${res.statusCode}`));
+        return;
+      }
+      const chunks = [];
+      let size = 0;
+      res.on('data', (chunk) => {
+        size += chunk.length;
+        if (size > MAX_RESPONSE_BYTES) {
+          req.destroy(new Error('Plugin bundle exceeds the size limit'));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+      res.on('error', reject);
+    });
+    req.on('timeout', () => req.destroy(new Error('Plugin share timed out')));
+    req.on('error', reject);
+    req.end();
+  });
+  return { bundle: JSON.parse(body), id: parsed.id, sha256: parsed.sha256 };
+}

package/src/core/plugin-runner-child.mjs

@@ -0,0 +1,24 @@
+// Child-process entry for plugin-runner.mjs. Imports one plugin CLI handler and
+// invokes it with the argv shape `total-recall <command> [sub] [...args]` would
+// give it, then exits. Runs in its own process so a plugin that calls
+// process.exit(), throws asynchronously, or leaks handles cannot affect the
+// brain server.
+import { pathToFileURL } from 'node:url';
+
+const [handlerPath, ...pluginArgv] = process.argv.slice(2);
+
+try {
+  const mod = await import(pathToFileURL(handlerPath).href);
+  const argv = ['node', 'total-recall', ...pluginArgv];
+  if (typeof mod.run === 'function') {
+    await mod.run(argv);
+  } else if (typeof mod.default === 'function') {
+    await mod.default(argv.slice(3));
+  } else {
+    console.error(`Plugin handler ${handlerPath} exports neither run() nor a default function`);
+    process.exitCode = 1;
+  }
+} catch (err) {
+  console.error(err?.stack || String(err));
+  process.exitCode = 1;
+}

package/src/core/plugin-runner.mjs

@@ -0,0 +1,102 @@
+/**
+ * Run a plugin's CLI handler in a separate Node process.
+ *
+ * The dashboard runner used to import() handlers into the brain server and
+ * swap console.log process-wide to capture output: a plugin calling
+ * process.exit() stopped the server, and concurrent runs interleaved output.
+ */
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { PACKAGE_ROOT } from './plugin-loader.mjs';
+
+const CHILD = path.join(path.dirname(fileURLToPath(import.meta.url)), 'plugin-runner-child.mjs');
+export const DEFAULT_TIMEOUT_MS = 30_000;
+export const MAX_OUTPUT_BYTES = 256 * 1024;
+
+/**
+ * @param {object} plugin  A discovered plugin ({ id, dir, manifest }).
+ * @param {{ subcommand?: string, args?: string[], cwd?: string, timeoutMs?: number }} [options]
+ * @returns {Promise<{ ok: boolean, exitCode: number|null, signal: string|null, timedOut: boolean, truncated: boolean, output: string, durationMs: number }>}
+ */
+export function runPluginCommand(plugin, options = {}) {
+  const { subcommand = '', args = [], cwd = process.cwd(), timeoutMs = DEFAULT_TIMEOUT_MS } = options;
+  const handlerRel = plugin?.manifest?.cli?.handler;
+  if (!handlerRel) return Promise.reject(new Error(`Plugin ${plugin?.id} does not declare a CLI handler`));
+
+  const handler = path.resolve(plugin.dir, handlerRel);
+  const rel = path.relative(fs.realpathSync(plugin.dir), fs.existsSync(handler) ? fs.realpathSync(handler) : handler);
+  if (rel.startsWith('..') || path.isAbsolute(rel)) {
+    return Promise.reject(new Error(`Plugin ${plugin.id} handler resolves outside the plugin directory`));
+  }
+  if (!fs.existsSync(handler)) return Promise.reject(new Error(`Plugin CLI handler file not found: ${handler}`));
+
+  const command = plugin.manifest.cli.command || plugin.id;
+  const safeArgs = (Array.isArray(args) ? args : []).map(String);
+  const childArgv = [CHILD, handler, command, ...(subcommand ? [String(subcommand)] : []), ...safeArgs];
+
+  return new Promise((resolve) => {
+    const started = Date.now();
+    const chunks = [];
+    let bytes = 0;
+    let truncated = false;
+    let timedOut = false;
+    let settled = false;
+
+    const child = spawn(process.execPath, childArgv, {
+      cwd,
+      env: {
+        ...process.env,
+        TR_PACKAGE_ROOT: PACKAGE_ROOT,
+        TR_PLUGIN_ID: plugin.id,
+        TR_PLUGIN_DIR: plugin.dir,
+        FORCE_COLOR: '0',
+        NO_COLOR: '1'
+      },
+      stdio: ['ignore', 'pipe', 'pipe']
+    });
+
+    const collect = (buf) => {
+      if (truncated) return;
+      if (bytes + buf.length > MAX_OUTPUT_BYTES) {
+        chunks.push(buf.subarray(0, MAX_OUTPUT_BYTES - bytes));
+        bytes = MAX_OUTPUT_BYTES;
+        truncated = true;
+        return;
+      }
+      chunks.push(buf);
+      bytes += buf.length;
+    };
+    child.stdout.on('data', collect);
+    child.stderr.on('data', collect);
+
+    const timer = setTimeout(() => {
+      timedOut = true;
+      child.kill('SIGKILL');
+    }, timeoutMs);
+
+    const finish = (exitCode, signal, spawnError) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      let output = Buffer.concat(chunks).toString('utf8');
+      if (spawnError) output += `${output ? '\n' : ''}[runner] ${spawnError.message}`;
+      if (truncated) output += `\n[runner] output truncated at ${MAX_OUTPUT_BYTES} bytes`;
+      if (timedOut) output += `\n[runner] killed after ${timeoutMs} ms`;
+      resolve({
+        ok: !spawnError && !timedOut && exitCode === 0,
+        exitCode,
+        signal,
+        timedOut,
+        truncated,
+        output,
+        durationMs: Date.now() - started
+      });
+    };
+
+    // spawn reports a missing binary / bad cwd as an 'error' event, not a throw.
+    child.on('error', (err) => finish(null, null, err));
+    child.on('close', (code, signal) => finish(code, signal));
+  });
+}