talon-agent 5.2.2 → 5.4.0

package/src/core/backup/store.ts

@@ -0,0 +1,312 @@
+/**
+ * The local snapshot store — layout on disk, manifest I/O, retention,
+ * and the SQLite index that the listing surfaces read.
+ *
+ * Layout (relative to the Talon home):
+ *
+ *   backups/<id>/manifest.json          what this snapshot is
+ *   backups/<id>/state.tar.zst          identity + state + database
+ *   backups/<id>/palace-<hash12>.tar.zst  the memory palace, when present
+ *
+ * The manifest on disk is authoritative. SQLite is a cache so `/backup`
+ * and `talon backup list` answer without opening a file per snapshot —
+ * `reconcileIndex` rebuilds rows from the directories on every boot, and
+ * a row whose directory has gone is kept rather than deleted, because
+ * the snapshot may still exist on a remote target.
+ */
+
+import { randomBytes } from "node:crypto";
+import {
+  link,
+  copyFile,
+  mkdir,
+  readdir,
+  readFile,
+  rm,
+  stat,
+} from "node:fs/promises";
+import { join, resolve } from "node:path";
+import writeFileAtomic from "write-file-atomic";
+import { dirs } from "../../util/paths.js";
+import { log, logWarn } from "../../util/log.js";
+import {
+  backupIds,
+  deleteBackup,
+  getBackup,
+  listBackupRemotes,
+  listBackups,
+  pinBackup,
+  recordBackup,
+  updateBackupManifest,
+} from "../../storage/backup/index.js";
+import type {
+  Manifest,
+  RemoteState,
+  SnapshotKind,
+  SnapshotSummary,
+} from "./types.js";
+
+/** `20260918T233400Z-a1b2c3` — chronological and filesystem-safe. */
+const SNAPSHOT_ID_RE = /^\d{8}T\d{6}Z-[0-9a-f]{6}$/;
+const MANIFEST_NAME = "manifest.json";
+export const STATE_PART = "state.tar.zst";
+
+function backupsRoot(home: string = dirs.root): string {
+  return join(home, "backups");
+}
+
+/** True for a well-formed snapshot id. Ids reach us from chat and CLI. */
+export function isSnapshotId(id: string): boolean {
+  return SNAPSHOT_ID_RE.test(id);
+}
+
+export function snapshotDir(id: string, home: string = dirs.root): string {
+  return join(backupsRoot(home), id);
+}
+
+/** Mint a new id. Sorts chronologically; the suffix separates same-second runs. */
+export function newSnapshotId(
+  now: Date = new Date(),
+  suffix: string = randomBytes(3).toString("hex"),
+): string {
+  const stamp = now
+    .toISOString()
+    .replace(/[-:]/g, "")
+    .replace(/\.\d{3}Z$/, "Z");
+  return `${stamp}-${suffix}`;
+}
+
+// ── Manifest I/O ────────────────────────────────────────────────────────────
+
+export async function readManifest(
+  id: string,
+  home: string = dirs.root,
+): Promise<Manifest | null> {
+  if (!isSnapshotId(id)) return null;
+  try {
+    const body = await readFile(
+      join(snapshotDir(id, home), MANIFEST_NAME),
+      "utf8",
+    );
+    const parsed = JSON.parse(body) as Manifest;
+    return parsed.id === id ? parsed : null;
+  } catch {
+    return null; // absent or unreadable — the caller reports "not found"
+  }
+}
+
+export async function writeManifest(
+  manifest: Manifest,
+  home: string = dirs.root,
+): Promise<void> {
+  const dir = snapshotDir(manifest.id, home);
+  await mkdir(dir, { recursive: true });
+  await writeFileAtomic(
+    join(dir, MANIFEST_NAME),
+    JSON.stringify(manifest, null, 2) + "\n",
+  );
+}
+
+/** Every snapshot directory that holds a readable manifest, newest first. */
+export async function listLocalManifests(
+  home: string = dirs.root,
+): Promise<Manifest[]> {
+  let names: string[] = [];
+  try {
+    names = await readdir(backupsRoot(home));
+  } catch {
+    return []; // no backups taken yet
+  }
+  const manifests: Manifest[] = [];
+  for (const name of names.filter(isSnapshotId)) {
+    const manifest = await readManifest(name, home);
+    if (manifest) manifests.push(manifest);
+  }
+  return manifests.sort((a, b) => b.createdAt - a.createdAt);
+}
+
+/**
+ * Hard-link `source` to `dest`, copying when the link cannot be made
+ * (different filesystem, a host that refuses links). Content-addressed
+ * parts are identical bytes, so a link is free and a copy is correct.
+ */
+export async function linkOrCopy(source: string, dest: string): Promise<void> {
+  try {
+    await link(source, dest);
+  } catch {
+    await copyFile(source, dest);
+  }
+}
+
+// ── SQLite index ────────────────────────────────────────────────────────────
+
+function toRecord(manifest: Manifest) {
+  return {
+    id: manifest.id,
+    kind: manifest.kind,
+    label: manifest.label,
+    pinned: manifest.pinned,
+    createdAt: manifest.createdAt,
+    sizeBytes: manifest.sizeBytes,
+    manifestJson: JSON.stringify(manifest),
+  };
+}
+
+/** Record (or refresh) one snapshot in the index. */
+export function indexSnapshot(manifest: Manifest): void {
+  recordBackup(toRecord(manifest));
+}
+
+/** Replace the indexed manifest after it changed (pin, upload, prune). */
+export function reindexSnapshot(manifest: Manifest): void {
+  updateBackupManifest(
+    manifest.id,
+    JSON.stringify(manifest),
+    manifest.pinned,
+    manifest.sizeBytes,
+  );
+}
+
+/**
+ * Bring the index in line with the directories on disk. Snapshots present
+ * on disk but missing from the index are added — that is how a restored
+ * (or hand-copied) backup directory becomes visible, and how the snapshot
+ * a staged restore took before the database was swapped comes back.
+ */
+export async function reconcileIndex(
+  home: string = dirs.root,
+): Promise<number> {
+  const known = backupIds();
+  let added = 0;
+  for (const manifest of await listLocalManifests(home)) {
+    if (known.has(manifest.id)) continue;
+    indexSnapshot(manifest);
+    added += 1;
+  }
+  if (added > 0) log("backup", `Indexed ${added} snapshot(s) found on disk`);
+  return added;
+}
+
+/** The listing every surface renders, newest first. */
+export async function listSnapshots(
+  home: string = dirs.root,
+): Promise<SnapshotSummary[]> {
+  const remotes = listBackupRemotes();
+  const summaries: SnapshotSummary[] = [];
+  for (const record of listBackups()) {
+    const remote: Record<string, RemoteState> = {};
+    for (const row of remotes.filter((r) => r.backupId === record.id)) {
+      remote[row.targetId] = {
+        status: row.status as RemoteState["status"],
+        remoteId: row.remoteId,
+        uploadedAt: row.uploadedAt,
+        error: row.error,
+      };
+    }
+    summaries.push({
+      id: record.id,
+      kind: record.kind as SnapshotKind,
+      label: record.label,
+      pinned: record.pinned,
+      createdAt: record.createdAt,
+      sizeBytes: record.sizeBytes,
+      local: await exists(join(snapshotDir(record.id, home), MANIFEST_NAME)),
+      remote,
+    });
+  }
+  return summaries;
+}
+
+async function exists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Pin or unpin a snapshot, on disk and in the index. Returns false if unknown. */
+export async function setSnapshotPinned(
+  id: string,
+  pinned: boolean,
+  home: string = dirs.root,
+): Promise<boolean> {
+  const manifest = await readManifest(id, home);
+  if (manifest) {
+    manifest.pinned = pinned;
+    await writeManifest(manifest, home);
+    if (getBackup(id)) reindexSnapshot(manifest);
+    else indexSnapshot(manifest);
+    return true;
+  }
+  // Remote-only snapshot: the index row is all we have.
+  return pinBackup(id, pinned);
+}
+
+// ── Retention ───────────────────────────────────────────────────────────────
+
+/**
+ * Which snapshots to drop so that at most `keep` unpinned ones remain.
+ * Pure, and the order is the one the caller sees: newest first, pinned
+ * snapshots kept unconditionally and not counted against the budget —
+ * pinning is the user saying "this one outlives the policy".
+ */
+export function selectPrunable<
+  T extends { id: string; createdAt: number; pinned: boolean },
+>(snapshots: readonly T[], keep: number): T[] {
+  const ordered = [...snapshots].sort((a, b) => b.createdAt - a.createdAt);
+  const doomed: T[] = [];
+  let kept = 0;
+  for (const snapshot of ordered) {
+    if (snapshot.pinned) continue;
+    kept += 1;
+    if (kept > keep) doomed.push(snapshot);
+  }
+  return doomed;
+}
+
+/** Delete a snapshot directory and its index rows. */
+async function removeSnapshot(
+  id: string,
+  home: string = dirs.root,
+): Promise<void> {
+  if (!isSnapshotId(id)) return;
+  await rm(snapshotDir(id, home), { recursive: true, force: true });
+  deleteBackup(id);
+}
+
+/** Apply the local retention policy. Returns the ids removed. */
+export async function pruneLocal(
+  keep: number,
+  home: string = dirs.root,
+): Promise<string[]> {
+  const doomed = selectPrunable(await listLocalManifests(home), keep);
+  const removed: string[] = [];
+  for (const manifest of doomed) {
+    try {
+      await removeSnapshot(manifest.id, home);
+      removed.push(manifest.id);
+    } catch (err) {
+      logWarn("backup", `Could not prune ${manifest.id}: ${String(err)}`);
+    }
+  }
+  if (removed.length > 0) {
+    log(
+      "backup",
+      `Pruned ${removed.length} local snapshot(s) (keepLocal=${keep})`,
+    );
+  }
+  return removed;
+}
+
+/** Absolute path of one part inside a snapshot directory. */
+export function partPath(
+  id: string,
+  name: string,
+  home: string = dirs.root,
+): string {
+  const dir = snapshotDir(id, home);
+  const abs = resolve(dir, name);
+  return abs.startsWith(dir) ? abs : join(dir, "invalid-part");
+}

package/src/core/backup/targets.ts

@@ -0,0 +1,281 @@
+/**
+ * Remote targets — the core half of the backup plugin protocol.
+ *
+ * A target is any loaded plugin that answers `backup.target.describe`.
+ * Core never learns what a target is made of: it hands over an absolute
+ * path and a manifest and gets an opaque remote id back, so Drive, an
+ * S3 bucket or an rclone remote are the same object here. The wire
+ * vocabulary (see docs/backups.md) is fixed and shared with the plugins
+ * that implement it:
+ *
+ *   backup.target.describe         → { id, name, ready, detail? }
+ *   backup.target.upload           → { remoteId, deduplicated? }
+ *   backup.target.upload_manifest  → { remoteId }        (always last)
+ *   backup.target.list             → { snapshots: [...] }
+ *   backup.target.delete           → ok
+ *   backup.target.download         → ok
+ *
+ * `chatId` is the string `"system"`: these calls belong to the daemon,
+ * not to a conversation. The manifest is uploaded last on purpose — its
+ * presence on the remote is what marks a snapshot complete, so an upload
+ * interrupted halfway can never be mistaken for a restorable backup.
+ */
+
+import { handlePluginActionIn, pluginsWithActions } from "../plugin/index.js";
+import { logWarn } from "../../util/log.js";
+import { TalonError } from "../errors.js";
+import type { ActionResult } from "../types.js";
+import type { Manifest, SnapshotPart } from "./types.js";
+
+/** The daemon's chat id for plugin actions that belong to no conversation. */
+const SYSTEM_CHAT = "system";
+
+/** A part as the upload call describes it: metadata plus where to read it. */
+type UploadPart = SnapshotPart & { path: string };
+
+/** One snapshot as a target reports it. */
+type RemoteSnapshot = {
+  snapshotId: string;
+  manifest: Manifest;
+  parts: Array<{ name: string; remoteId?: string; bytes: number }>;
+};
+
+export interface BackupTarget {
+  readonly id: string;
+  readonly name: string;
+  /** False when the target is configured but cannot accept uploads yet. */
+  readonly ready: boolean;
+  /** Why it is not ready, in the target's own words. */
+  readonly detail?: string;
+  upload(
+    snapshotId: string,
+    part: UploadPart,
+    manifest: Manifest,
+  ): Promise<{ remoteId: string; deduplicated?: boolean }>;
+  uploadManifest(
+    snapshotId: string,
+    manifest: Manifest,
+  ): Promise<{ remoteId: string }>;
+  list(): Promise<RemoteSnapshot[]>;
+  remove(snapshotId: string): Promise<void>;
+  download(
+    snapshotId: string,
+    partName: string,
+    destPath: string,
+  ): Promise<void>;
+}
+
+/** Sends one protocol body to one plugin. Replaced wholesale in tests. */
+type TargetDispatch = (
+  plugin: string,
+  body: Record<string, unknown>,
+) => Promise<ActionResult | null>;
+
+export type TargetDeps = {
+  /** Plugins to ask, in load order. */
+  plugins: () => string[];
+  dispatch: TargetDispatch;
+};
+
+/**
+ * Both members are wrapped rather than referenced directly, so the plugin
+ * module's exports are read when a target is actually used and not while
+ * this module is being evaluated. Backup reaches a lot of the tree — via
+ * the scheduler, the gateway imports it transitively — so a module-scope
+ * read here means every test that partially mocks `core/plugin` has to
+ * know to include these two exports or fail at import time, nowhere near
+ * anything it was testing.
+ */
+const defaultDeps: TargetDeps = {
+  plugins: () => pluginsWithActions(),
+  dispatch: (plugin, body) => handlePluginActionIn(plugin, body, SYSTEM_CHAT),
+};
+
+function targetError(
+  target: string,
+  action: string,
+  detail: string,
+): TalonError {
+  return new TalonError(`${target}: ${action} failed — ${detail}`, {
+    reason: "unknown",
+  });
+}
+
+/** Unwrap `{ ok, data }`, turning every failure shape into one error. */
+function dataOf(
+  result: ActionResult | null,
+  target: string,
+  action: string,
+): Record<string, unknown> {
+  if (!result) throw targetError(target, action, "plugin did not answer");
+  if (!result.ok) {
+    throw targetError(target, action, String(result.error ?? "unknown error"));
+  }
+  const data = result.data;
+  return typeof data === "object" && data !== null
+    ? (data as Record<string, unknown>)
+    : {};
+}
+
+/** A target backed by one plugin. */
+class PluginTarget implements BackupTarget {
+  constructor(
+    private readonly plugin: string,
+    private readonly deps: TargetDeps,
+    readonly id: string,
+    readonly name: string,
+    readonly ready: boolean,
+    readonly detail?: string,
+  ) {}
+
+  private send(body: Record<string, unknown>): Promise<ActionResult | null> {
+    return this.deps.dispatch(this.plugin, body);
+  }
+
+  async upload(
+    snapshotId: string,
+    part: UploadPart,
+    manifest: Manifest,
+  ): Promise<{ remoteId: string; deduplicated?: boolean }> {
+    const data = dataOf(
+      await this.send({
+        action: "backup.target.upload",
+        snapshotId,
+        part,
+        manifest,
+      }),
+      this.id,
+      "upload",
+    );
+    const remoteId = typeof data.remoteId === "string" ? data.remoteId : "";
+    if (!remoteId) throw targetError(this.id, "upload", "no remoteId returned");
+    return { remoteId, deduplicated: data.deduplicated === true };
+  }
+
+  async uploadManifest(
+    snapshotId: string,
+    manifest: Manifest,
+  ): Promise<{ remoteId: string }> {
+    const data = dataOf(
+      await this.send({
+        action: "backup.target.upload_manifest",
+        snapshotId,
+        manifest,
+      }),
+      this.id,
+      "upload_manifest",
+    );
+    const remoteId = typeof data.remoteId === "string" ? data.remoteId : "";
+    if (!remoteId) {
+      throw targetError(this.id, "upload_manifest", "no remoteId returned");
+    }
+    return { remoteId };
+  }
+
+  async list(): Promise<RemoteSnapshot[]> {
+    const data = dataOf(
+      await this.send({ action: "backup.target.list" }),
+      this.id,
+      "list",
+    );
+    const snapshots = Array.isArray(data.snapshots) ? data.snapshots : [];
+    return snapshots.flatMap((entry) => {
+      const row = entry as Partial<RemoteSnapshot>;
+      if (typeof row.snapshotId !== "string" || !row.manifest) return [];
+      return [
+        {
+          snapshotId: row.snapshotId,
+          manifest: row.manifest,
+          parts: Array.isArray(row.parts) ? row.parts : [],
+        },
+      ];
+    });
+  }
+
+  async remove(snapshotId: string): Promise<void> {
+    dataOf(
+      await this.send({ action: "backup.target.delete", snapshotId }),
+      this.id,
+      "delete",
+    );
+  }
+
+  async download(
+    snapshotId: string,
+    partName: string,
+    destPath: string,
+  ): Promise<void> {
+    dataOf(
+      await this.send({
+        action: "backup.target.download",
+        snapshotId,
+        part: { name: partName },
+        destPath,
+      }),
+      this.id,
+      "download",
+    );
+  }
+}
+
+/**
+ * Ask every loaded plugin whether it is a backup target. A plugin that is
+ * not one answers null and is skipped; one that answers badly is logged
+ * and skipped, because a misbehaving plugin must not stop the backup that
+ * the other targets (and the local store) can still complete.
+ */
+export async function discoverTargets(
+  deps: TargetDeps = defaultDeps,
+): Promise<BackupTarget[]> {
+  const targets: BackupTarget[] = [];
+  for (const plugin of deps.plugins()) {
+    let result: ActionResult | null;
+    try {
+      result = await deps.dispatch(plugin, {
+        action: "backup.target.describe",
+      });
+    } catch (err) {
+      logWarn(
+        "backup",
+        `Plugin ${plugin} failed to describe itself: ${String(err)}`,
+      );
+      continue;
+    }
+    if (!result) continue; // not a backup target
+    if (!result.ok) {
+      logWarn(
+        "backup",
+        `Plugin ${plugin} describe error: ${String(result.error)}`,
+      );
+      continue;
+    }
+    const data = (result.data ?? {}) as Record<string, unknown>;
+    const id = typeof data.id === "string" ? data.id : "";
+    if (!id) {
+      logWarn("backup", `Plugin ${plugin} answered describe without an id`);
+      continue;
+    }
+    targets.push(
+      new PluginTarget(
+        plugin,
+        deps,
+        id,
+        typeof data.name === "string" ? data.name : id,
+        data.ready === true,
+        typeof data.detail === "string" ? data.detail : undefined,
+      ),
+    );
+  }
+  return targets;
+}
+
+/** The targets this run should use: config order wins, unknown ids are dropped. */
+export function selectTargets(
+  available: readonly BackupTarget[],
+  configured: readonly string[] | undefined,
+): BackupTarget[] {
+  if (configured === undefined) return [...available];
+  return configured.flatMap((id) =>
+    available.filter((target) => target.id === id),
+  );
+}

package/src/core/backup/types.ts

@@ -0,0 +1,96 @@
+/**
+ * The backup vocabulary — the shapes every other module in this subsystem
+ * and every surface above it speaks.
+ *
+ * `Manifest` is the contract with the future: it is written next to the
+ * parts on disk, uploaded last to every remote target (its presence is
+ * what marks a remote snapshot complete), and it is the source of truth
+ * for a restore. The SQLite index is a cache over these files, never the
+ * other way round — a snapshot whose row was lost is still restorable,
+ * a row whose directory is gone is not.
+ *
+ * `BackupSettings` is declared structurally rather than derived from the
+ * zod schema in core/config: the config layer imports this subsystem for
+ * its defaults, so the type may not travel back the other way.
+ */
+
+/** Scheduled and pruned, or deliberate and kept. */
+export type SnapshotKind = "backup" | "checkpoint";
+
+/** One compressed file inside a snapshot directory. */
+export type SnapshotPart = {
+  /** File name within the snapshot directory (also the remote object name). */
+  name: string;
+  bytes: number;
+  sha256: string;
+  /**
+   * The part's name encodes its content hash, so an identical part in an
+   * older snapshot is the same bytes: the local store hard-links it and
+   * targets may skip the upload entirely.
+   */
+  contentAddressed?: boolean;
+};
+
+/** Per-target upload state, mirrored into the `backup_remotes` table. */
+export type RemoteState = {
+  status: "pending" | "uploaded" | "failed";
+  remoteId?: string;
+  uploadedAt?: number;
+  error?: string;
+};
+
+/** Where an `extra/<n>/…` subtree came from, so restore can put it back. */
+type ExtraMapping = { n: number; source: string };
+
+export type Manifest = {
+  schema: 1;
+  id: string;
+  kind: SnapshotKind;
+  label?: string;
+  pinned: boolean;
+  /** Epoch ms. */
+  createdAt: number;
+  host: string;
+  talonVersion: string;
+  gitHead?: string;
+  parts: SnapshotPart[];
+  /** Archive-relative roots this snapshot covers — what a restore replaces. */
+  includes: string[];
+  /** Human-readable exclusion rules, recorded so an old snapshot explains itself. */
+  excludes: string[];
+  /** `extra/<n>` → absolute source path. */
+  extras?: ExtraMapping[];
+  /** Tree fingerprint of the palace part, for content-addressed reuse. */
+  palaceHash?: string;
+  /** Total bytes of all parts. */
+  sizeBytes: number;
+  remote: Record<string, RemoteState>;
+};
+
+/** A snapshot as the listing surfaces show it. */
+export type SnapshotSummary = {
+  id: string;
+  kind: SnapshotKind;
+  label?: string;
+  pinned: boolean;
+  createdAt: number;
+  sizeBytes: number;
+  /** False when the index has a row but the directory is gone (remote-only). */
+  local: boolean;
+  remote: Record<string, RemoteState>;
+};
+
+/** `config.backup`, with every default already applied. */
+export type BackupSettings = {
+  enabled: boolean;
+  intervalHours: number;
+  keepLocal: number;
+  keepRemote: number;
+  includePalace: boolean;
+  workspaceInclude: readonly string[];
+  extraPaths: readonly string[];
+  /** Unset = every registered target; `[]` = local only. */
+  targets?: readonly string[];
+  checkpointBeforeUpdate: boolean;
+  notifyChatId?: string;
+};