talon-agent 5.2.1 → 5.3.0

package/src/core/backup/upload.ts

@@ -0,0 +1,172 @@
+/**
+ * Getting a snapshot off this machine — and keeping the remote tidy.
+ *
+ * Targets run in parallel (they are independent networks) but the parts
+ * of one snapshot go up sequentially, manifest last. That order is the
+ * completeness marker: a remote snapshot with no manifest.json is an
+ * interrupted upload, and nothing will ever mistake it for a restorable
+ * backup.
+ *
+ * One target failing is not a failed backup. The snapshot is already on
+ * local disk by the time we get here, so a target that errors records
+ * `failed` with its reason and the run carries on — the next run retries
+ * it. The manifest keeps the per-target state alongside the parts, so a
+ * snapshot restored onto a new machine still knows where its copies are.
+ */
+
+import { bus } from "../bus/index.js";
+import { log, logWarn } from "../../util/log.js";
+import { dirs } from "../../util/paths.js";
+import {
+  deleteBackupRemote,
+  recordBackupRemote,
+} from "../../storage/backup/index.js";
+import {
+  partPath,
+  reindexSnapshot,
+  selectPrunable,
+  writeManifest,
+} from "./store.js";
+import type { BackupTarget } from "./targets.js";
+import type { Manifest, RemoteState } from "./types.js";
+
+function recordState(id: string, targetId: string, state: RemoteState): void {
+  recordBackupRemote({
+    backupId: id,
+    targetId,
+    status: state.status,
+    remoteId: state.remoteId,
+    uploadedAt: state.uploadedAt,
+    error: state.error,
+  });
+}
+
+/** Push every part, then the manifest. Throws with the target's own words. */
+async function sendSnapshot(
+  target: BackupTarget,
+  manifest: Manifest,
+  home: string,
+): Promise<{ state: RemoteState; deduplicated: boolean }> {
+  let deduplicated = manifest.parts.length > 0;
+  for (const part of manifest.parts) {
+    const result = await target.upload(
+      manifest.id,
+      { ...part, path: partPath(manifest.id, part.name, home) },
+      manifest,
+    );
+    if (!result.deduplicated) deduplicated = false;
+  }
+  const { remoteId } = await target.uploadManifest(manifest.id, manifest);
+  return {
+    state: { status: "uploaded", remoteId, uploadedAt: Date.now() },
+    deduplicated,
+  };
+}
+
+/**
+ * Upload one snapshot to every target. Returns the manifest with its
+ * `remote` map filled in; it is rewritten on disk and reindexed so the
+ * status surfaces can answer without asking the network.
+ */
+export async function uploadSnapshot(
+  manifest: Manifest,
+  targets: readonly BackupTarget[],
+  home: string = dirs.root,
+): Promise<Manifest> {
+  if (targets.length === 0) return manifest;
+  await Promise.all(
+    targets.map(async (target) => {
+      if (!target.ready) {
+        const state: RemoteState = {
+          status: "pending",
+          error: target.detail ?? "target not ready",
+        };
+        manifest.remote[target.id] = state;
+        recordState(manifest.id, target.id, state);
+        logWarn("backup", `Target ${target.id} not ready: ${state.error}`);
+        return;
+      }
+      try {
+        const { state, deduplicated } = await sendSnapshot(
+          target,
+          manifest,
+          home,
+        );
+        manifest.remote[target.id] = state;
+        recordState(manifest.id, target.id, state);
+        bus.publish({
+          type: "backup.uploaded",
+          snapshotId: manifest.id,
+          targetId: target.id,
+          bytes: manifest.sizeBytes,
+          deduplicated,
+        });
+        log(
+          "backup",
+          `Uploaded ${manifest.id} to ${target.id}${deduplicated ? " (deduplicated)" : ""}`,
+        );
+      } catch (err) {
+        const state: RemoteState = {
+          status: "failed",
+          error: err instanceof Error ? err.message : String(err),
+        };
+        manifest.remote[target.id] = state;
+        recordState(manifest.id, target.id, state);
+        logWarn("backup", `Upload to ${target.id} failed: ${state.error}`);
+      }
+    }),
+  );
+  await writeManifest(manifest, home);
+  reindexSnapshot(manifest);
+  return manifest;
+}
+
+/**
+ * Apply the remote retention policy on each target. Same rule as local:
+ * the newest `keep` unpinned snapshots survive, pinned ones always do.
+ * A target that cannot list is skipped — deleting on a partial listing is
+ * how a retention pass turns into data loss.
+ */
+export async function pruneRemote(
+  targets: readonly BackupTarget[],
+  keep: number,
+): Promise<void> {
+  for (const target of targets) {
+    if (!target.ready) continue;
+    let snapshots;
+    try {
+      snapshots = await target.list();
+    } catch (err) {
+      logWarn(
+        "backup",
+        `Cannot list ${target.id}, skipping remote prune: ${String(err)}`,
+      );
+      continue;
+    }
+    const doomed = selectPrunable(
+      snapshots.map((entry) => ({
+        id: entry.snapshotId,
+        createdAt: entry.manifest?.createdAt ?? 0,
+        pinned: entry.manifest?.pinned === true,
+      })),
+      keep,
+    );
+    for (const victim of doomed) {
+      try {
+        await target.remove(victim.id);
+        deleteBackupRemote(victim.id, target.id);
+      } catch (err) {
+        logWarn(
+          "backup",
+          `Could not delete ${victim.id} from ${target.id}: ${String(err)}`,
+        );
+      }
+    }
+    if (doomed.length > 0) {
+      log(
+        "backup",
+        `Pruned ${doomed.length} snapshot(s) from ${target.id} (keepRemote=${keep})`,
+      );
+    }
+  }
+}

package/src/core/bus/events.ts

@@ -15,6 +15,9 @@
  *     `agent.spawned` when an isolated sub-agent run starts, `agent.settled`
  *     when it reaches a terminal state, `agent.message` when a note or a
  *     report crosses between an agent and its parent.
+ *   - `backup.*` — the snapshot lifecycle, published by `core/backup`:
+ *     `backup.started` / `backup.completed` / `backup.failed` per run, and
+ *     `backup.uploaded` once per target a snapshot reaches.
  *   - `turn.*` — the chat-domain moments inside a turn that other
  *     subsystems key off: `turn.started` fires once the warp is bound and
  *     the backend is about to run (never for a no-model refusal);
@@ -91,6 +94,43 @@ interface AgentMessageEvent {
   readonly kind: "message" | "result";
 }
 
+/** A snapshot run began. `trigger` is what asked for it, never content. */
+interface BackupStartedEvent {
+  readonly type: "backup.started";
+  readonly kind: "backup" | "checkpoint";
+  /** "schedule", "manual", "pre-update", "pre-restore", … */
+  readonly trigger: string;
+}
+
+/** A snapshot was written and indexed. */
+interface BackupCompletedEvent {
+  readonly type: "backup.completed";
+  /** Not `id`: the bus stamps every published event with its own numeric id. */
+  readonly snapshotId: string;
+  readonly kind: "backup" | "checkpoint";
+  readonly sizeBytes: number;
+  readonly parts: number;
+  readonly durationMs: number;
+}
+
+/** A snapshot run failed. `error` is the reason, never a path's contents. */
+interface BackupFailedEvent {
+  readonly type: "backup.failed";
+  readonly trigger: string;
+  readonly error: string;
+  readonly consecutiveFailures: number;
+}
+
+/** One snapshot reached one remote target. */
+interface BackupUploadedEvent {
+  readonly type: "backup.uploaded";
+  readonly snapshotId: string;
+  readonly targetId: string;
+  readonly bytes: number;
+  /** True when the target already held every part (content-addressed reuse). */
+  readonly deduplicated: boolean;
+}
+
 export type TalonEvent =
   | TaskStartedEvent
   | TaskSettledEvent
@@ -98,7 +138,11 @@ export type TalonEvent =
   | TurnCompletedEvent
   | AgentSpawnedEvent
   | AgentSettledEvent
-  | AgentMessageEvent;
+  | AgentMessageEvent
+  | BackupStartedEvent
+  | BackupCompletedEvent
+  | BackupFailedEvent
+  | BackupUploadedEvent;
 
 export type TalonEventType = TalonEvent["type"];
 

package/src/core/config/index.ts

@@ -6,6 +6,7 @@ import { hardenTalonPermissions } from "./harden.js";
 import { setTimezone } from "../../util/time.js";
 import { BACKEND_IDS } from "../agent-runtime/model-ref.js";
 import { REASONING_LEVEL_ORDER } from "../models/reasoning-levels.js";
+import { DEFAULT_BACKUP_SETTINGS } from "../backup/plan.js";
 import {
   assembleSystemPrompt,
   joinSystemPromptParts,
@@ -458,6 +459,57 @@ const configSchema = z.object({
         .default(15 * 60 * 1000),
     })
     .optional(),
+  /**
+   * Backups & checkpoints (docs/backups.md). Talon's only safety net, so
+   * it is on by default: every `intervalHours` it writes a snapshot of
+   * the identity, state, database and memory under ~/.talon/backups/,
+   * keeps `keepLocal` of them, and uploads to whatever remote targets
+   * are registered (`targets: []` keeps everything local).
+   *
+   *   - `workspaceInclude` — the workspace is mostly bulk that can be
+   *     refetched; this is the subset that IS the agent.
+   *   - `extraPaths` — absolute or `~/…` paths outside ~/.talon worth
+   *     carrying along (a Claude Code memory directory, say).
+   *   - `checkpointBeforeUpdate` — pinned checkpoint before `/update`,
+   *     so a bad update is one restore away from undone.
+   *   - `notifyChatId` — where failures are reported; falls back to the
+   *     admin chat.
+   */
+  backup: z
+    .object({
+      enabled: z.boolean().default(DEFAULT_BACKUP_SETTINGS.enabled),
+      intervalHours: z
+        .number()
+        .int()
+        .min(1)
+        .max(168)
+        .default(DEFAULT_BACKUP_SETTINGS.intervalHours),
+      keepLocal: z
+        .number()
+        .int()
+        .min(1)
+        .max(1000)
+        .default(DEFAULT_BACKUP_SETTINGS.keepLocal),
+      keepRemote: z
+        .number()
+        .int()
+        .min(1)
+        .max(1000)
+        .default(DEFAULT_BACKUP_SETTINGS.keepRemote),
+      includePalace: z.boolean().default(DEFAULT_BACKUP_SETTINGS.includePalace),
+      workspaceInclude: z
+        .array(z.string().min(1))
+        .default([...DEFAULT_BACKUP_SETTINGS.workspaceInclude]),
+      extraPaths: z.array(z.string().min(1)).default([]),
+      /** Unset = every registered target; `[]` = local only. */
+      targets: z.array(z.string().min(1)).optional(),
+      checkpointBeforeUpdate: z
+        .boolean()
+        .default(DEFAULT_BACKUP_SETTINGS.checkpointBeforeUpdate),
+      notifyChatId: z.string().optional(),
+    })
+    .strict()
+    .optional(),
   braveApiKey: z.string().optional(),
   /**
    * Codex-specific OpenAI API key. Prefer this, CODEX_API_KEY, or

package/src/core/daemon/handoff.ts

@@ -0,0 +1,192 @@
+/**
+ * Handoff watcher — the process that proves a `/restart` or `/update`
+ * actually landed.
+ *
+ * The outgoing daemon spawns its successor and then calls
+ * `process.exit()`. Until 2026-09-18 that was the whole handoff, which
+ * means nothing in the system knew whether the successor had come up.
+ * On that day one didn't: it was spawned, lived about twenty seconds,
+ * never bound its gateway, and died — and because the dying parent was
+ * the only party to the handoff, Talon simply stayed down until a human
+ * noticed forty-five minutes later.
+ *
+ * So the handoff gets a witness. `spawnSuccessor()` (./respawn.ts)
+ * starts this watcher detached, sharing the successor's respawn.log fd.
+ * It polls identity-verified discovery (./discovery.ts — `app: "talon"`,
+ * `mode: "daemon"`, matching pid) until the successor answers /health or
+ * the window closes. Only a /health answer counts: discovery will also
+ * report a daemon whose pid is merely alive, and "the process exists" is
+ * exactly the claim that was false for 20 seconds on 2026-09-18. If the
+ * successor never serves, the watcher starts the daemon exactly
+ * the way `talon start` does (./control.ts — same spawn, same boot
+ * verification) and says why in the log. The watcher is tiny on purpose:
+ * `src/index.ts` dispatches its subcommand before the app graph loads,
+ * so it costs a bare runtime and these three modules.
+ */
+
+import { dirname, resolve } from "node:path";
+import { log, logError, logWarn } from "../../util/log.js";
+import { startDaemon, type StartOutcome } from "./control.js";
+import { findRunningInstance, type RunningInstance } from "./discovery.js";
+import { isProcessAlive } from "./pidfile.js";
+
+/** Hidden subcommand: `talon _handoff-watch <successor-pid>`. */
+export const HANDOFF_WATCH_SUBCOMMAND = "_handoff-watch";
+
+/**
+ * How long a successor gets to answer /health. Generous on purpose: a
+ * cold boot with every plugin and MCP server takes ~10s on the reference
+ * host, and `startDaemon()` itself waits 30s before calling a boot late.
+ */
+const HANDOFF_WINDOW_MS = 90_000;
+const POLL_MS = 500;
+
+export type HandoffOutcome =
+  | { ok: true; via: "successor" | "restart"; pid: number; port?: number }
+  | { ok: false; reason: string };
+
+export type WatchHandoffOptions = {
+  /** The pid `spawnSuccessor()` created. */
+  childPid: number;
+  /** Repo/package root, for the `talon start` equivalent. */
+  pkgRoot: string;
+  windowMs?: number;
+  pollMs?: number;
+  pidfilePath?: string;
+  /** Injection seams for tests. */
+  find?: typeof findRunningInstance;
+  alive?: (pid: number) => boolean;
+  start?: typeof startDaemon;
+  sleep?: (ms: number) => Promise<void>;
+};
+
+function defaultSleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+/** Why the wait ended without a live daemon. */
+type WaitFailure = "successor-exited" | "window-expired";
+
+/** Discovery found a process; only a /health answer proves it serves. */
+function isServing(instance: RunningInstance | null): boolean {
+  return instance?.health !== undefined;
+}
+
+/**
+ * Poll until a serving daemon answers, the successor process
+ * disappears, or the window closes. A daemon that isn't our child still
+ * counts: the goal is a live Talon, not a particular pid.
+ */
+async function awaitDaemon(
+  opts: WatchHandoffOptions,
+): Promise<RunningInstance | WaitFailure> {
+  const find = opts.find ?? findRunningInstance;
+  const alive = opts.alive ?? isProcessAlive;
+  const sleep = opts.sleep ?? defaultSleep;
+  const pollMs = opts.pollMs ?? POLL_MS;
+  const deadline = Date.now() + (opts.windowMs ?? HANDOFF_WINDOW_MS);
+
+  while (Date.now() < deadline) {
+    const instance = await find(opts.pidfilePath);
+    if (instance && isServing(instance)) return instance;
+    if (!alive(opts.childPid)) return "successor-exited";
+    await sleep(pollMs);
+  }
+  return "window-expired";
+}
+
+const FAILURE_DETAIL: Record<WaitFailure, string> = {
+  "successor-exited": "the successor exited before serving /health",
+  "window-expired": "the successor never answered /health in time",
+};
+
+function describeStart(outcome: StartOutcome): string {
+  if (outcome.ok) return `started (pid ${outcome.pid})`;
+  if (outcome.reason === "already-running") {
+    return `already running (pid ${outcome.instance.pid})`;
+  }
+  if (outcome.reason === "boot-timeout") return "spawned but not yet healthy";
+  return `${outcome.reason}${outcome.detail ? `: ${outcome.detail}` : ""}`;
+}
+
+function toOutcome(started: StartOutcome, why: string): HandoffOutcome {
+  if (started.ok) {
+    return { ok: true, via: "restart", pid: started.pid, port: started.port };
+  }
+  if (started.reason === "already-running") {
+    const inst = started.instance;
+    // `talon start` refuses while a pid is alive — right, since a second
+    // daemon would fight the first for Telegram's getUpdates. But an
+    // alive pid that has never served /health is the failure, not the
+    // recovery, so it is reported as one.
+    if (!isServing(inst)) {
+      return {
+        ok: false,
+        reason: `${why}; pid ${inst.pid} is alive but not serving — kill it and run \`talon start\``,
+      };
+    }
+    return { ok: true, via: "restart", pid: inst.pid, port: inst.port };
+  }
+  return { ok: false, reason: `${why}; restart ${describeStart(started)}` };
+}
+
+/**
+ * Verify the handoff, and repair it if it failed. Never throws: this
+ * process exists only to make the outcome known.
+ */
+export async function watchHandoff(
+  opts: WatchHandoffOptions,
+): Promise<HandoffOutcome> {
+  const result = await awaitDaemon(opts);
+  if (typeof result !== "string") {
+    const via = result.pid === opts.childPid ? "successor" : "restart";
+    log(
+      "shutdown",
+      `Handoff verified — daemon pid ${result.pid} serving on ` +
+        `:${result.port ?? "?"} (${via})`,
+    );
+    return { ok: true, via, pid: result.pid, port: result.port };
+  }
+
+  const why = FAILURE_DETAIL[result];
+  logWarn(
+    "shutdown",
+    `Handoff failed — ${why}; starting Talon the way \`talon start\` does`,
+  );
+  const start = opts.start ?? startDaemon;
+  const started = await start({
+    pkgRoot: opts.pkgRoot,
+    pidfilePath: opts.pidfilePath,
+  });
+  const outcome = toOutcome(started, why);
+  if (outcome.ok)
+    log("shutdown", `Handoff recovered — ${describeStart(started)}`);
+  else logError("shutdown", `Handoff unrecoverable — ${outcome.reason}`);
+  return outcome;
+}
+
+/** src/core/daemon/ → the package root. */
+function packageRoot(): string {
+  const here = import.meta.dirname ?? process.cwd();
+  return here.includes("$bunfs") || here.includes("~BUN")
+    ? dirname(process.execPath)
+    : resolve(here, "..", "..", "..");
+}
+
+/** Entry point for `talon _handoff-watch <pid>` (src/index.ts). */
+export async function runHandoffWatch(argv: readonly string[]): Promise<void> {
+  const childPid = Number.parseInt(argv[0] ?? "", 10);
+  if (!Number.isInteger(childPid) || childPid <= 0) {
+    logError("shutdown", `Handoff watcher got no successor pid (${argv[0]})`);
+    process.exitCode = 2;
+    return;
+  }
+  log("shutdown", `Handoff watcher armed for pid ${childPid}`);
+  try {
+    const outcome = await watchHandoff({ childPid, pkgRoot: packageRoot() });
+    process.exitCode = outcome.ok ? 0 : 1;
+  } catch (err) {
+    logError("shutdown", "Handoff watcher crashed", err);
+    process.exitCode = 1;
+  }
+}