talon-agent 5.2.2 → 5.4.0

package/src/storage/backup/repo.ts

@@ -0,0 +1,164 @@
+/**
+ * Snapshot-index repository — executes the statements in
+ * sql/backups.sql against the `backups` and `backup_remotes` tables.
+ * No SQL text lives here and none lives above: `storage/backup/index.ts`
+ * holds the domain API, this module owns statement execution and the
+ * row↔domain mapping.
+ *
+ * The manifest travels as an opaque JSON string. Storage sits below the
+ * engine, so it must not know the shape core/backup gives it — it stores
+ * the bytes and the few columns the listing surfaces sort and filter on.
+ */
+
+import { getDatabase } from "../db.js";
+import { backupsSql } from "../sql/statements.generated.js";
+
+/** One snapshot as the index holds it. */
+export type BackupRecord = {
+  id: string;
+  kind: string;
+  label?: string;
+  pinned: boolean;
+  /** Epoch ms. */
+  createdAt: number;
+  sizeBytes: number;
+  /** The manifest.json body, verbatim. */
+  manifestJson: string;
+};
+
+/** One target's state for one snapshot. */
+export type BackupRemoteRecord = {
+  backupId: string;
+  targetId: string;
+  status: string;
+  remoteId?: string;
+  uploadedAt?: number;
+  error?: string;
+};
+
+type Row = {
+  id: string;
+  kind: string;
+  label: string | null;
+  pinned: number;
+  created_at: number;
+  size_bytes: number;
+  manifest_json: string;
+};
+
+type RemoteRow = {
+  backup_id: string;
+  target_id: string;
+  status: string;
+  remote_id: string | null;
+  uploaded_at: number | null;
+  error: string | null;
+};
+
+function toRecord(row: Row): BackupRecord {
+  return {
+    id: row.id,
+    kind: row.kind,
+    label: row.label ?? undefined,
+    pinned: row.pinned === 1,
+    createdAt: row.created_at,
+    sizeBytes: row.size_bytes,
+    manifestJson: row.manifest_json,
+  };
+}
+
+function toRemote(row: RemoteRow): BackupRemoteRecord {
+  return {
+    backupId: row.backup_id,
+    targetId: row.target_id,
+    status: row.status,
+    remoteId: row.remote_id ?? undefined,
+    uploadedAt: row.uploaded_at ?? undefined,
+    error: row.error ?? undefined,
+  };
+}
+
+export function upsert(record: BackupRecord): void {
+  getDatabase()
+    .prepare(backupsSql.upsert)
+    .run(
+      record.id,
+      record.kind,
+      record.label ?? null,
+      record.pinned ? 1 : 0,
+      record.createdAt,
+      record.sizeBytes,
+      record.manifestJson,
+    );
+}
+
+export function get(id: string): BackupRecord | undefined {
+  const row = getDatabase().prepare(backupsSql.get).get(id) as Row | undefined;
+  return row ? toRecord(row) : undefined;
+}
+
+/** Newest first. */
+export function all(): BackupRecord[] {
+  return (getDatabase().prepare(backupsSql.all).all() as Row[]).map(toRecord);
+}
+
+export function ids(): string[] {
+  return (getDatabase().prepare(backupsSql.ids).all() as { id: string }[]).map(
+    (row) => row.id,
+  );
+}
+
+export function setPinned(id: string, pinned: boolean): boolean {
+  const result = getDatabase()
+    .prepare(backupsSql.setPinned)
+    .run(pinned ? 1 : 0, id) as { changes?: number };
+  return (result.changes ?? 0) > 0;
+}
+
+export function setManifest(
+  id: string,
+  manifestJson: string,
+  pinned: boolean,
+  sizeBytes: number,
+): void {
+  getDatabase()
+    .prepare(backupsSql.setManifest)
+    .run(manifestJson, pinned ? 1 : 0, sizeBytes, id);
+}
+
+export function remove(id: string): void {
+  getDatabase().prepare(backupsSql.remove).run(id);
+}
+
+export function upsertRemote(record: BackupRemoteRecord): void {
+  getDatabase()
+    .prepare(backupsSql.upsertRemote)
+    .run(
+      record.backupId,
+      record.targetId,
+      record.status,
+      record.remoteId ?? null,
+      record.uploadedAt ?? null,
+      record.error ?? null,
+    );
+}
+
+export function remotesAll(): BackupRemoteRecord[] {
+  return (
+    getDatabase().prepare(backupsSql.remotesAll).all() as RemoteRow[]
+  ).map(toRemote);
+}
+
+export function remotesFor(backupId: string): BackupRemoteRecord[] {
+  return (
+    getDatabase().prepare(backupsSql.remotesFor).all(backupId) as RemoteRow[]
+  ).map(toRemote);
+}
+
+export function removeRemotes(backupId: string): void {
+  getDatabase().prepare(backupsSql.removeRemotes).run(backupId);
+}
+
+export function removeRemote(backupId: string, targetId: string): void {
+  getDatabase().prepare(backupsSql.removeRemote).run(backupId, targetId);
+}

package/src/storage/db.ts

@@ -216,6 +216,26 @@ export function flushDatabase(): void {
   }
 }
 
+/**
+ * Write a transactionally consistent copy of the database to `destPath`.
+ *
+ * The backup subsystem's only way in: a snapshot may not copy
+ * `data/talon.db` from disk, because in WAL mode the committed pages
+ * live partly in the sidecar and a byte-wise copy of the main file is a
+ * corrupt (or silently stale) database. `VACUUM INTO` asks SQLite for
+ * the copy instead — one file, no WAL, checkpointed and compacted,
+ * consistent as of the moment it runs, with concurrent readers and
+ * writers untouched.
+ *
+ * `destPath` must not exist (SQLite refuses to overwrite); its parent
+ * directory is created if needed.
+ */
+export function snapshotDatabase(destPath: string): void {
+  const database = getDatabase();
+  mkdirSync(dirname(destPath), { recursive: true });
+  database.prepare(dbSql.vacuumInto).run(destPath);
+}
+
 export function closeDatabase(): void {
   if (!db) return;
   try {

package/src/storage/sql/backups.sql

@@ -0,0 +1,46 @@
+-- Statements for the snapshot index (see repositories in
+-- storage/backup/repo.ts for parameter order and row↔domain mapping).
+
+-- name: upsert
+INSERT OR REPLACE INTO backups
+  (id, kind, label, pinned, created_at, size_bytes, manifest_json)
+VALUES (?, ?, ?, ?, ?, ?, ?)
+
+-- name: get
+SELECT id, kind, label, pinned, created_at, size_bytes, manifest_json
+FROM backups WHERE id = ?
+
+-- name: all
+SELECT id, kind, label, pinned, created_at, size_bytes, manifest_json
+FROM backups ORDER BY created_at DESC
+
+-- name: ids
+SELECT id FROM backups
+
+-- name: setPinned
+UPDATE backups SET pinned = ? WHERE id = ?
+
+-- name: setManifest
+UPDATE backups SET manifest_json = ?, pinned = ?, size_bytes = ? WHERE id = ?
+
+-- name: remove
+DELETE FROM backups WHERE id = ?
+
+-- name: upsertRemote
+INSERT OR REPLACE INTO backup_remotes
+  (backup_id, target_id, status, remote_id, uploaded_at, error)
+VALUES (?, ?, ?, ?, ?, ?)
+
+-- name: remotesAll
+SELECT backup_id, target_id, status, remote_id, uploaded_at, error
+FROM backup_remotes
+
+-- name: remotesFor
+SELECT backup_id, target_id, status, remote_id, uploaded_at, error
+FROM backup_remotes WHERE backup_id = ?
+
+-- name: removeRemotes
+DELETE FROM backup_remotes WHERE backup_id = ?
+
+-- name: removeRemote
+DELETE FROM backup_remotes WHERE backup_id = ? AND target_id = ?

package/src/storage/sql/db.sql

@@ -30,3 +30,11 @@ ALTER TABLE history_messages ADD COLUMN attachments TEXT
 -- Column reconciliation for databases that shipped before the cache-age
 -- signal existed. Fresh databases get the column via schema.sql.
 ALTER TABLE sessions ADD COLUMN last_turn_ended_at INTEGER
+
+-- name: vacuumInto
+-- Transactionally consistent copy of the whole database into a new file,
+-- produced by SQLite itself (storage/db.ts snapshotDatabase). A backup
+-- must never copy a live .db byte-wise: the WAL holds committed pages the
+-- main file does not, so the copy would be a corrupt database or an old
+-- one. Verified to accept a bound path on node:sqlite and bun:sqlite.
+VACUUM INTO ?

package/src/storage/sql/schema.sql

@@ -362,3 +362,33 @@ CREATE TABLE IF NOT EXISTS whatsapp_messages (
 CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_msg ON whatsapp_messages(msg_id);
 CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_wa_id ON whatsapp_messages(wa_id);
 CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_time ON whatsapp_messages(timestamp);
+
+-- Snapshot index: the listing/status view over ~/.talon/backups/. The
+-- manifest.json next to the parts on disk stays the source of truth for
+-- a restore (a database that needs restoring cannot also be the record
+-- of how), so these rows are a cache — dropped rows are re-derived from
+-- the directories on the next boot, and a row whose directory is gone is
+-- kept because the snapshot may still exist on a remote target.
+CREATE TABLE IF NOT EXISTS backups (
+  id            TEXT PRIMARY KEY,
+  kind          TEXT    NOT NULL,
+  label         TEXT,
+  pinned        INTEGER NOT NULL DEFAULT 0,
+  created_at    INTEGER NOT NULL,
+  size_bytes    INTEGER NOT NULL DEFAULT 0,
+  manifest_json TEXT    NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_backups_created ON backups(created_at DESC);
+
+-- Per-target upload state for one snapshot. Separate from the manifest
+-- copy so retention can ask "what is on Drive?" without opening a file
+-- per snapshot, and so a failed upload's error survives a restart.
+CREATE TABLE IF NOT EXISTS backup_remotes (
+  backup_id   TEXT    NOT NULL,
+  target_id   TEXT    NOT NULL,
+  status      TEXT    NOT NULL,
+  remote_id   TEXT,
+  uploaded_at INTEGER,
+  error       TEXT,
+  PRIMARY KEY (backup_id, target_id)
+);

package/src/storage/sql/statements.generated.ts

@@ -366,7 +366,60 @@ CREATE TABLE IF NOT EXISTS whatsapp_messages (
 );
 CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_msg ON whatsapp_messages(msg_id);
 CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_wa_id ON whatsapp_messages(wa_id);
-CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_time ON whatsapp_messages(timestamp);`;
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_time ON whatsapp_messages(timestamp);
+
+-- Snapshot index: the listing/status view over ~/.talon/backups/. The
+-- manifest.json next to the parts on disk stays the source of truth for
+-- a restore (a database that needs restoring cannot also be the record
+-- of how), so these rows are a cache — dropped rows are re-derived from
+-- the directories on the next boot, and a row whose directory is gone is
+-- kept because the snapshot may still exist on a remote target.
+CREATE TABLE IF NOT EXISTS backups (
+  id            TEXT PRIMARY KEY,
+  kind          TEXT    NOT NULL,
+  label         TEXT,
+  pinned        INTEGER NOT NULL DEFAULT 0,
+  created_at    INTEGER NOT NULL,
+  size_bytes    INTEGER NOT NULL DEFAULT 0,
+  manifest_json TEXT    NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_backups_created ON backups(created_at DESC);
+
+-- Per-target upload state for one snapshot. Separate from the manifest
+-- copy so retention can ask "what is on Drive?" without opening a file
+-- per snapshot, and so a failed upload's error survives a restart.
+CREATE TABLE IF NOT EXISTS backup_remotes (
+  backup_id   TEXT    NOT NULL,
+  target_id   TEXT    NOT NULL,
+  status      TEXT    NOT NULL,
+  remote_id   TEXT,
+  uploaded_at INTEGER,
+  error       TEXT,
+  PRIMARY KEY (backup_id, target_id)
+);`;
+
+export const backupsSql = {
+  upsert: `INSERT OR REPLACE INTO backups
+  (id, kind, label, pinned, created_at, size_bytes, manifest_json)
+VALUES (?, ?, ?, ?, ?, ?, ?)`,
+  get: `SELECT id, kind, label, pinned, created_at, size_bytes, manifest_json
+FROM backups WHERE id = ?`,
+  all: `SELECT id, kind, label, pinned, created_at, size_bytes, manifest_json
+FROM backups ORDER BY created_at DESC`,
+  ids: `SELECT id FROM backups`,
+  setPinned: `UPDATE backups SET pinned = ? WHERE id = ?`,
+  setManifest: `UPDATE backups SET manifest_json = ?, pinned = ?, size_bytes = ? WHERE id = ?`,
+  remove: `DELETE FROM backups WHERE id = ?`,
+  upsertRemote: `INSERT OR REPLACE INTO backup_remotes
+  (backup_id, target_id, status, remote_id, uploaded_at, error)
+VALUES (?, ?, ?, ?, ?, ?)`,
+  remotesAll: `SELECT backup_id, target_id, status, remote_id, uploaded_at, error
+FROM backup_remotes`,
+  remotesFor: `SELECT backup_id, target_id, status, remote_id, uploaded_at, error
+FROM backup_remotes WHERE backup_id = ?`,
+  removeRemotes: `DELETE FROM backup_remotes WHERE backup_id = ?`,
+  removeRemote: `DELETE FROM backup_remotes WHERE backup_id = ? AND target_id = ?`,
+} as const;
 
 export const chatSettingsSql = {
   upsert: `INSERT OR REPLACE INTO chat_settings (chat_id, settings) VALUES (?, ?)`,
@@ -421,6 +474,12 @@ ALTER TABLE history_messages ADD COLUMN attachments TEXT`,
   addSessionsLastTurnEndedAtColumn: `-- Column reconciliation for databases that shipped before the cache-age
 -- signal existed. Fresh databases get the column via schema.sql.
 ALTER TABLE sessions ADD COLUMN last_turn_ended_at INTEGER`,
+  vacuumInto: `-- Transactionally consistent copy of the whole database into a new file,
+-- produced by SQLite itself (storage/db.ts snapshotDatabase). A backup
+-- must never copy a live .db byte-wise: the WAL holds committed pages the
+-- main file does not, so the copy would be a corrupt database or an old
+-- one. Verified to accept a bound path on node:sqlite and bun:sqlite.
+VACUUM INTO ?`,
 } as const;
 
 export const goalsSql = {

package/src/util/log.ts

@@ -67,6 +67,7 @@ export type LogComponent =
   | "playwright"
   | "memory"
   | "stickers"
+  | "backup"
   | "backend-controller";
 
 const LOG_FILE = files.log;