talon-agent 5.2.2 → 5.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "5.2.2",
+  "version": "5.3.0",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/src/app.ts

@@ -27,6 +27,7 @@ import {
 } from "./core/background/cron/scheduler.js";
 import { shutdownTriggers } from "./core/background/triggers/index.js";
 import { shutdownAgents } from "./core/agents/index.js";
+import { stopBackupScheduler } from "./core/backup/index.js";
 import { pruneSettledTriggers } from "./storage/triggers.js";
 import { startWatchdog, stopWatchdog } from "./util/watchdog.js";
 import {
@@ -77,11 +78,48 @@ import {
 // the successor will run is importable at all. Nothing has booted yet,
 // so this is both the strongest and the last harmless place to say so.
 // See core/update/self-update.ts for what a failure does instead.
+//
+// This stays ahead of the staged restore below: the smoke run must not
+// touch state, and a restore is the single most destructive thing this
+// file does.
 if (process.argv.includes(BOOT_SMOKE_FLAG)) {
   console.log(BOOT_SMOKE_OK);
   process.exit(0);
 }
 
+/**
+ * A `/backup restore <id>` from chat writes ~/.talon/restore-pending.json
+ * and restarts. It is applied HERE, before anything else: every store
+ * below opens the database, and the database is one of the files this is
+ * about to replace. The pre-restore checkpoint inside needs the current
+ * database, so the handle is closed between the two — which is why the
+ * composition root, and not core/backup, owns that call.
+ *
+ * Never throws: a failed restore still boots the daemon (with the reason
+ * in the log and the request deleted, so the next boot is normal).
+ */
+async function applyStagedRestore(): Promise<string | null> {
+  const { applyPendingRestore, readRestorePending } =
+    await import("./core/backup/index.js");
+  if (!(await readRestorePending())) return null;
+  const { loadConfig } = await import("./core/config/index.js");
+  const { resolveBackupSettings } = await import("./core/backup/plan.js");
+  const { closeDatabase } = await import("./storage/db.js");
+  const report = await applyPendingRestore({
+    settings: resolveBackupSettings(loadConfig().backup),
+    beforeApply: closeDatabase,
+  });
+  if (!report) return null;
+  return (
+    `♻️ Restored snapshot ${report.id}` +
+    (report.checkpointId
+      ? ` (previous state saved as checkpoint ${report.checkpointId})`
+      : "")
+  );
+}
+
+const restoreReport = await bootPhase("staged restore", applyStagedRestore);
+
 const { config } = await bootPhase("bootstrap", () => bootstrap());
 
 // Record this process as the daemon. The gateway port is appended once
@@ -131,6 +169,36 @@ onBackendChange((holder, newBackend, info) => {
 
 // ── Graceful shutdown ────────────────────────────────────────────────────────
 
+/**
+ * Arm the backup scheduler. Here rather than in bootstrap because this is
+ * where the process lifecycle lives — `stopBackupScheduler` is two screens
+ * down in `gracefulShutdown`, and the two belong together. Failure
+ * notifications go to `backup.notifyChatId` when set, otherwise to the
+ * admin, over the same route as the plan alerts.
+ */
+async function startBackups(): Promise<void> {
+  const { initBackup } = await import("./core/backup/index.js");
+  const { resolveBackupSettings } = await import("./core/backup/plan.js");
+  const notifyChatId = config.backup?.notifyChatId;
+  await initBackup({
+    settings: resolveBackupSettings(config.backup),
+    notify: notifyChatId
+      ? async (text: string) => {
+          const { resolveFrontendIdAmong } =
+            await import("./core/frontend-runtime/routing.js");
+          const name = resolveFrontendIdAmong(
+            notifyChatId,
+            frontends.map((frontend) => frontend.name),
+          );
+          const target =
+            frontends.find((frontend) => frontend.name === name) ??
+            frontends[0];
+          if (target) await target.sendMessage(Number(notifyChatId), text);
+        }
+      : undefined,
+  });
+}
+
 let shuttingDown = false;
 let triggerPruneTimer: ReturnType<typeof setInterval> | null = null;
 
@@ -230,6 +298,7 @@ async function gracefulShutdown(signal: string): Promise<void> {
     });
   }
   await shutdownStep("fuse layer", unmountNamespaceFs);
+  await shutdownStep("backup scheduler", stopBackupScheduler);
   await shutdownStep("pulse timer", stopPulseTimer);
   await shutdownStep("heartbeat", async () => {
     stopHeartbeatTimer();
@@ -298,6 +367,7 @@ async function main(): Promise<void> {
   await Promise.all(frontends.map((frontend) => frontend.init()));
   log("bot", "Starting Talon...");
 
+  await startBackups();
   if (config.pulse) startPulseTimer(config.pulseIntervalMs);
   if (config.heartbeat) startHeartbeatTimer(config.heartbeatIntervalMinutes);
   startWatchdog(config.workspace);
@@ -345,6 +415,14 @@ async function main(): Promise<void> {
   // Phase 0 accounting (docs/ts-migration-plan.md): the boot is over the
   // moment the frontends are listening, so the totals are folded into the
   // metrics store here, from the same uptime figure the log line prints.
+  // A restore applied at boot happened before any frontend existed, so the
+  // operator hears about it here, on the first channel that can carry it.
+  if (restoreReport) {
+    const { notifyAdmin } =
+      await import("./core/frontend-runtime/admin-notify.js");
+    await notifyAdmin(restoreReport);
+  }
+
   const bootMs = Math.round(process.uptime() * 1000);
   recordBootMetrics(bootMs);
   startResourceSampler();

package/src/backend/codex/mcp-config.ts

@@ -29,7 +29,7 @@ import {
   pluginHubUrl,
   hubPluginServerNames,
 } from "../../core/mcp-hub/index.js";
-import type { ToolExclusionConfig } from "../../core/tools/ops/mcp-env.js";
+import type { ToolExclusionConfig } from "../../core/tools/mcp-env.js";
 import { frontendsForChat } from "../runtime/frontends.js";
 
 /**

package/src/backend/openai-agents/mcp-pool.ts

@@ -30,7 +30,7 @@ import {
   pluginHubUrl,
   hubPluginServerNames,
 } from "../../core/mcp-hub/index.js";
-import type { ToolExclusionConfig } from "../../core/tools/ops/mcp-env.js";
+import type { ToolExclusionConfig } from "../../core/tools/mcp-env.js";
 import { frontendsForChat } from "../runtime/frontends.js";
 import { log, logWarn } from "../../util/log.js";
 

package/src/backend/runtime/index.ts

@@ -37,7 +37,7 @@
  *   - SDK-specific event types — those live in each backend.
  *   - Session storage — that's `src/storage/sessions.ts`.
  *   - MCP server registration — backend-specific transport details
- *     (the spawn/env contract they share is `core/tools/ops/mcp-env.ts`).
+ *     (the spawn/env contract they share is `core/tools/mcp-env.ts`).
  */
 
 export { captureDeliveredText } from "./turn/delivered-text.js";

package/src/cli/commands/backup.ts

@@ -0,0 +1,396 @@
+/**
+ * `talon backup` — snapshots and checkpoints from the outside.
+ *
+ * Two execution modes, chosen per subcommand:
+ *
+ *   - Through the daemon (`now`, `targets`, `status`) when one is
+ *     running. The daemon has the plugins loaded, so it is the only
+ *     process that can see the remote targets — and routing through it
+ *     also means one writer, not two racing `VACUUM INTO`s.
+ *   - In this process (`list`, `show`, `pin`, `unpin`, `prune`, and
+ *     `now` with no daemon) — they read the manifests on disk and the
+ *     index in talon.db, both of which are safe to touch concurrently.
+ *
+ * `restore` refuses while the daemon is running, full stop: it replaces
+ * the database and the memory under a live process. Stop Talon first, or
+ * use `/backup restore <id>` from chat, which stages the request and
+ * lets the next boot apply it before anything opens the database.
+ *
+ * Lives in `commands/` because that is where docs/structure.md's worklist
+ * item 8 puts every CLI command; `src/cli/` itself is at its file ceiling.
+ */
+
+import pc from "picocolors";
+import { createInterface } from "node:readline/promises";
+import { getFrontends, loadConfig } from "../../core/config/index.js";
+import { findRunningInstance } from "../../core/daemon/discovery.js";
+import {
+  collectBackupStatus,
+  formatBackupStatus,
+  formatBytes,
+  formatRelative,
+  isSnapshotId,
+  listSnapshots,
+  discoverTargets,
+  readManifest,
+  restoreSnapshot,
+  setSnapshotPinned,
+  type BackupTarget,
+  type SnapshotSummary,
+} from "../../core/backup/index.js";
+import { resolveBackupSettings } from "../../core/backup/plan.js";
+import { buildSnapshot } from "../../core/backup/snapshot.js";
+import { pruneLocal } from "../../core/backup/store.js";
+import { fetchGateway } from "../daemon-api.js";
+
+const USAGE = `
+  ${pc.bold("talon backup")} — snapshots and checkpoints
+
+    ${pc.cyan("now")} [--checkpoint <label>] [--pin]   take a snapshot now
+    ${pc.cyan("list")} [--remote <targetId>]           list snapshots
+    ${pc.cyan("show")} <id>                            everything about one snapshot
+    ${pc.cyan("pin")} <id>  |  ${pc.cyan("unpin")} <id>          keep past retention, or release
+    ${pc.cyan("restore")} <id> [--from <target>] [--yes]  restore (daemon must be stopped)
+    ${pc.cyan("prune")}                                apply the local retention policy
+    ${pc.cyan("targets")}                              remote targets and their readiness
+    ${pc.cyan("status")}                               schedule, sizes, targets
+`;
+
+type Flags = { values: string[]; flags: Map<string, string | true> };
+
+/** `--from drive --yes x` → { values: ["x"], flags: { from: "drive", yes: true } } */
+function parseArgs(argv: readonly string[]): Flags {
+  const values: string[] = [];
+  const flags = new Map<string, string | true>();
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (!arg.startsWith("--")) {
+      values.push(arg);
+      continue;
+    }
+    const name = arg.slice(2);
+    const next = argv[i + 1];
+    if (next && !next.startsWith("--")) {
+      flags.set(name, next);
+      i++;
+    } else {
+      flags.set(name, true);
+    }
+  }
+  return { values, flags };
+}
+
+async function daemonPort(): Promise<number | null> {
+  const instance = await findRunningInstance();
+  return instance?.port ?? null;
+}
+
+/** Run one action inside the daemon. Returns null when there is no daemon. */
+async function viaDaemon(
+  body: Record<string, unknown>,
+): Promise<{ ok: boolean; text?: string; error?: string } | null> {
+  const port = await daemonPort();
+  if (port === null) return null;
+  // A snapshot of a large workspace can take minutes; the default CLI
+  // budget is three seconds.
+  return (await fetchGateway(
+    port,
+    "/action",
+    {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(body),
+    },
+    15 * 60_000,
+  )) as { ok: boolean; text?: string; error?: string };
+}
+
+function renderResult(result: {
+  ok: boolean;
+  text?: string;
+  error?: string;
+}): void {
+  if (result.ok) console.log(`  ${result.text ?? "Done."}\n`);
+  else console.log(`  ${pc.red("●")} ${result.error ?? "Failed."}\n`);
+}
+
+function renderRow(snapshot: SnapshotSummary): string {
+  const marks = [
+    snapshot.kind === "checkpoint"
+      ? pc.magenta("checkpoint")
+      : pc.dim("backup"),
+    snapshot.pinned ? pc.yellow("pinned") : "",
+    snapshot.local ? "" : pc.dim("(remote only)"),
+  ].filter(Boolean);
+  const remote = Object.entries(snapshot.remote)
+    .map(([id, entry]) =>
+      entry.status === "uploaded"
+        ? pc.green(id)
+        : pc.yellow(`${id}:${entry.status}`),
+    )
+    .join(" ");
+  return (
+    `  ${pc.bold(snapshot.id)}  ${marks.join(" ")}  ${formatBytes(snapshot.sizeBytes).padStart(9)}` +
+    `  ${pc.dim(formatRelative(snapshot.createdAt))}` +
+    (snapshot.label ? `  ${snapshot.label}` : "") +
+    (remote ? `  → ${remote}` : "")
+  );
+}
+
+// ── Subcommands ─────────────────────────────────────────────────────────────
+
+async function backupNow(flags: Flags): Promise<void> {
+  const label =
+    typeof flags.flags.get("checkpoint") === "string"
+      ? String(flags.flags.get("checkpoint"))
+      : undefined;
+  const pin = flags.flags.get("pin") === true;
+  const remote = await viaDaemon({ action: "backup.now", label, pin });
+  if (remote) {
+    renderResult(remote);
+    return;
+  }
+  console.log(
+    `  ${pc.dim("Talon is not running — taking a local snapshot (no remote upload).")}`,
+  );
+  const settings = resolveBackupSettings(loadConfig().backup);
+  const manifest = await buildSnapshot({
+    kind: label ? "checkpoint" : "backup",
+    label,
+    pinned: pin,
+    settings,
+  });
+  await pruneLocal(settings.keepLocal);
+  console.log(
+    `  ${pc.green("●")} ${manifest.id} — ${manifest.parts.length} part(s), ` +
+      `${formatBytes(manifest.sizeBytes)}\n`,
+  );
+}
+
+async function backupList(flags: Flags): Promise<void> {
+  const target = flags.flags.get("remote");
+  const snapshots = await listSnapshots();
+  const filtered =
+    typeof target === "string"
+      ? snapshots.filter(
+          (snapshot) => snapshot.remote[target]?.status === "uploaded",
+        )
+      : snapshots;
+  if (filtered.length === 0) {
+    console.log(`  ${pc.dim("No snapshots.")}\n`);
+    return;
+  }
+  console.log("");
+  for (const snapshot of filtered) console.log(renderRow(snapshot));
+  const bytes = filtered.reduce((sum, snapshot) => sum + snapshot.sizeBytes, 0);
+  console.log(
+    `\n  ${pc.dim(`${filtered.length} snapshot(s), ${formatBytes(bytes)}`)}\n`,
+  );
+}
+
+async function backupShow(id: string): Promise<void> {
+  const manifest = await readManifest(id);
+  if (!manifest) {
+    console.log(`  ${pc.red("●")} No snapshot ${id} on this machine.\n`);
+    return;
+  }
+  console.log(
+    `\n  ${pc.bold(manifest.id)}  ${manifest.kind}${manifest.pinned ? pc.yellow(" pinned") : ""}`,
+  );
+  if (manifest.label) console.log(`  ${manifest.label}`);
+  console.log(
+    `  taken     ${new Date(manifest.createdAt).toISOString()} (${formatRelative(manifest.createdAt)})`,
+  );
+  console.log(
+    `  host      ${manifest.host}  talon ${manifest.talonVersion}${manifest.gitHead ? ` @ ${manifest.gitHead}` : ""}`,
+  );
+  console.log(`  size      ${formatBytes(manifest.sizeBytes)}`);
+  console.log(
+    `  parts     ${manifest.parts.map((part) => `${part.name} (${formatBytes(part.bytes)})`).join(", ")}`,
+  );
+  console.log(`  includes  ${manifest.includes.join(", ")}`);
+  if (manifest.extras?.length) {
+    console.log(
+      `  extras    ${manifest.extras.map((extra) => `extra/${extra.n} ← ${extra.source}`).join(", ")}`,
+    );
+  }
+  for (const [targetId, state] of Object.entries(manifest.remote)) {
+    console.log(
+      `  remote    ${targetId}: ${state.status}${state.error ? ` — ${state.error}` : ""}`,
+    );
+  }
+  console.log("");
+}
+
+async function backupPin(id: string, pinned: boolean): Promise<void> {
+  const ok = await setSnapshotPinned(id, pinned);
+  console.log(
+    ok
+      ? `  ${pc.green("●")} ${id} is ${pinned ? "pinned — retention will never prune it" : "unpinned"}\n`
+      : `  ${pc.red("●")} No snapshot ${id}\n`,
+  );
+}
+
+async function backupPrune(): Promise<void> {
+  const settings = resolveBackupSettings(loadConfig().backup);
+  const removed = await pruneLocal(settings.keepLocal);
+  console.log(
+    removed.length === 0
+      ? `  ${pc.dim(`Nothing to prune (keepLocal=${settings.keepLocal}).`)}\n`
+      : `  ${pc.green("●")} Pruned ${removed.length}: ${removed.join(", ")}\n`,
+  );
+}
+
+async function backupTargets(): Promise<void> {
+  const result = await viaDaemon({ action: "backup.status" });
+  if (!result) {
+    console.log(
+      `  ${pc.yellow("●")} Talon is not running — remote targets live in the daemon's plugins.\n`,
+    );
+    return;
+  }
+  renderResult(result);
+}
+
+async function backupStatus(): Promise<void> {
+  const result = await viaDaemon({ action: "backup.status" });
+  if (result) {
+    renderResult(result);
+    return;
+  }
+  const status = await collectBackupStatus({ withTargets: false });
+  console.log(
+    `\n${formatBackupStatus(status)
+      .split("\n")
+      .map((line) => `  ${line}`)
+      .join("\n")}\n`,
+  );
+}
+
+async function confirm(question: string): Promise<boolean> {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  try {
+    return (await rl.question(question)).trim().toLowerCase() === "yes";
+  } finally {
+    rl.close();
+  }
+}
+
+async function backupRestore(id: string, flags: Flags): Promise<void> {
+  if (!isSnapshotId(id)) {
+    console.log(`  ${pc.red("●")} "${id}" is not a snapshot id.\n`);
+    return;
+  }
+  const instance = await findRunningInstance();
+  if (instance) {
+    console.log(
+      `  ${pc.red("●")} Talon is running (PID ${instance.pid}). A restore replaces the\n` +
+        `    database and memory underneath it. Run ${pc.cyan("talon stop")} first, or use\n` +
+        `    ${pc.cyan(`/backup restore ${id}`)} from chat — that stages the restore and applies\n` +
+        `    it during the next boot.\n`,
+    );
+    process.exitCode = 1;
+    return;
+  }
+  const manifest = await readManifest(id);
+  const from = flags.flags.get("from");
+  if (!manifest && typeof from !== "string") {
+    console.log(
+      `  ${pc.red("●")} No snapshot ${id} on this machine (use --from <target> to fetch it).\n`,
+    );
+    process.exitCode = 1;
+    return;
+  }
+  if (flags.flags.get("yes") !== true) {
+    console.log(
+      `\n  This replaces config, prompts, keys, sessions, the database and memory\n` +
+        `  with the contents of ${pc.bold(id)}${manifest?.label ? ` ("${manifest.label}")` : ""}.\n` +
+        `  A pinned checkpoint of the current state is taken first.\n`,
+    );
+    if (!(await confirm(`  Type ${pc.bold("yes")} to continue: `))) {
+      console.log(`  ${pc.dim("Cancelled.")}\n`);
+      return;
+    }
+  }
+  const settings = resolveBackupSettings(loadConfig().backup);
+  const target =
+    typeof from === "string" ? await resolveTarget(from) : undefined;
+  const report = await restoreSnapshot({ id, settings, target });
+  const written = Object.entries(report.written)
+    .map(([root, count]) => `${root} (${count})`)
+    .join(", ");
+  console.log(`\n  ${pc.green("●")} Restored ${report.id}`);
+  if (report.checkpointId) {
+    console.log(
+      `  Previous state saved as ${pc.bold(report.checkpointId)} (pinned).`,
+    );
+  }
+  console.log(`  Written: ${written || "nothing"}`);
+  console.log(`  Removed: ${report.removed} file(s)`);
+  console.log(
+    `  Database: ${report.databaseReplaced ? "replaced" : "left in place"}\n`,
+  );
+}
+
+/**
+ * Find a target by id. Targets are plugins, and a restore runs with the
+ * daemon stopped, so this process loads them itself — the same call
+ * bootstrap makes, minus the frontends.
+ */
+async function resolveTarget(id: string): Promise<BackupTarget | undefined> {
+  const config = loadConfig();
+  const { loadPlugins } = await import("../../core/plugin/index.js");
+  await loadPlugins(config.plugins, getFrontends(config));
+  const targets = await discoverTargets();
+  const target = targets.find((candidate) => candidate.id === id);
+  if (!target) {
+    console.log(
+      `  ${pc.red("●")} No backup target "${id}"` +
+        (targets.length > 0
+          ? ` — loaded: ${targets.map((t) => t.id).join(", ")}\n`
+          : " — no target plugins are configured.\n"),
+    );
+  }
+  return target;
+}
+
+// ── Entry ───────────────────────────────────────────────────────────────────
+
+export async function runBackupCommand(argv: readonly string[]): Promise<void> {
+  const { values, flags } = parseArgs(argv);
+  const [subcommand, id] = values;
+  const args: Flags = { values, flags };
+  switch (subcommand) {
+    case "now":
+      await backupNow(args);
+      return;
+    case "list":
+      await backupList(args);
+      return;
+    case "show":
+      if (!id) return void console.log(USAGE);
+      await backupShow(id);
+      return;
+    case "pin":
+    case "unpin":
+      if (!id) return void console.log(USAGE);
+      await backupPin(id, subcommand === "pin");
+      return;
+    case "restore":
+      if (!id) return void console.log(USAGE);
+      await backupRestore(id, args);
+      return;
+    case "prune":
+      await backupPrune();
+      return;
+    case "targets":
+      await backupTargets();
+      return;
+    case "status":
+    case undefined:
+      await backupStatus();
+      return;
+    default:
+      console.log(USAGE);
+  }
+}

package/src/cli/events.ts

@@ -44,6 +44,20 @@ function describe(event: TalonEvent): string {
       return `${event.agentId} "${event.label}" → ${event.state} (${event.durationMs}ms)`;
     case "agent.message":
       return `${event.from} → ${event.to} (${event.kind})`;
+    case "backup.started":
+      return `${event.kind} (${event.trigger})`;
+    case "backup.completed":
+      return (
+        `${event.snapshotId} ${event.kind} ${event.parts} part(s) ` +
+        `${(event.sizeBytes / 1024 / 1024).toFixed(1)}MB (${event.durationMs}ms)`
+      );
+    case "backup.failed":
+      return `${event.trigger} failed (streak ${event.consecutiveFailures}): ${event.error}`;
+    case "backup.uploaded":
+      return (
+        `${event.snapshotId} → ${event.targetId} ` +
+        `${(event.bytes / 1024 / 1024).toFixed(1)}MB${event.deduplicated ? " (deduplicated)" : ""}`
+      );
   }
 }