talon-agent 5.2.1 → 5.3.0

package/src/storage/sql/statements.generated.ts

@@ -366,7 +366,60 @@ CREATE TABLE IF NOT EXISTS whatsapp_messages (
 );
 CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_msg ON whatsapp_messages(msg_id);
 CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_wa_id ON whatsapp_messages(wa_id);
-CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_time ON whatsapp_messages(timestamp);`;
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_time ON whatsapp_messages(timestamp);
+
+-- Snapshot index: the listing/status view over ~/.talon/backups/. The
+-- manifest.json next to the parts on disk stays the source of truth for
+-- a restore (a database that needs restoring cannot also be the record
+-- of how), so these rows are a cache — dropped rows are re-derived from
+-- the directories on the next boot, and a row whose directory is gone is
+-- kept because the snapshot may still exist on a remote target.
+CREATE TABLE IF NOT EXISTS backups (
+  id            TEXT PRIMARY KEY,
+  kind          TEXT    NOT NULL,
+  label         TEXT,
+  pinned        INTEGER NOT NULL DEFAULT 0,
+  created_at    INTEGER NOT NULL,
+  size_bytes    INTEGER NOT NULL DEFAULT 0,
+  manifest_json TEXT    NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_backups_created ON backups(created_at DESC);
+
+-- Per-target upload state for one snapshot. Separate from the manifest
+-- copy so retention can ask "what is on Drive?" without opening a file
+-- per snapshot, and so a failed upload's error survives a restart.
+CREATE TABLE IF NOT EXISTS backup_remotes (
+  backup_id   TEXT    NOT NULL,
+  target_id   TEXT    NOT NULL,
+  status      TEXT    NOT NULL,
+  remote_id   TEXT,
+  uploaded_at INTEGER,
+  error       TEXT,
+  PRIMARY KEY (backup_id, target_id)
+);`;
+
+export const backupsSql = {
+  upsert: `INSERT OR REPLACE INTO backups
+  (id, kind, label, pinned, created_at, size_bytes, manifest_json)
+VALUES (?, ?, ?, ?, ?, ?, ?)`,
+  get: `SELECT id, kind, label, pinned, created_at, size_bytes, manifest_json
+FROM backups WHERE id = ?`,
+  all: `SELECT id, kind, label, pinned, created_at, size_bytes, manifest_json
+FROM backups ORDER BY created_at DESC`,
+  ids: `SELECT id FROM backups`,
+  setPinned: `UPDATE backups SET pinned = ? WHERE id = ?`,
+  setManifest: `UPDATE backups SET manifest_json = ?, pinned = ?, size_bytes = ? WHERE id = ?`,
+  remove: `DELETE FROM backups WHERE id = ?`,
+  upsertRemote: `INSERT OR REPLACE INTO backup_remotes
+  (backup_id, target_id, status, remote_id, uploaded_at, error)
+VALUES (?, ?, ?, ?, ?, ?)`,
+  remotesAll: `SELECT backup_id, target_id, status, remote_id, uploaded_at, error
+FROM backup_remotes`,
+  remotesFor: `SELECT backup_id, target_id, status, remote_id, uploaded_at, error
+FROM backup_remotes WHERE backup_id = ?`,
+  removeRemotes: `DELETE FROM backup_remotes WHERE backup_id = ?`,
+  removeRemote: `DELETE FROM backup_remotes WHERE backup_id = ? AND target_id = ?`,
+} as const;
 
 export const chatSettingsSql = {
   upsert: `INSERT OR REPLACE INTO chat_settings (chat_id, settings) VALUES (?, ?)`,
@@ -421,6 +474,12 @@ ALTER TABLE history_messages ADD COLUMN attachments TEXT`,
   addSessionsLastTurnEndedAtColumn: `-- Column reconciliation for databases that shipped before the cache-age
 -- signal existed. Fresh databases get the column via schema.sql.
 ALTER TABLE sessions ADD COLUMN last_turn_ended_at INTEGER`,
+  vacuumInto: `-- Transactionally consistent copy of the whole database into a new file,
+-- produced by SQLite itself (storage/db.ts snapshotDatabase). A backup
+-- must never copy a live .db byte-wise: the WAL holds committed pages the
+-- main file does not, so the copy would be a corrupt database or an old
+-- one. Verified to accept a bound path on node:sqlite and bun:sqlite.
+VACUUM INTO ?`,
 } as const;
 
 export const goalsSql = {

package/src/util/log.ts

@@ -16,9 +16,10 @@ import {
   statSync,
   renameSync,
   unlinkSync,
-  createWriteStream,
+  openSync,
+  writeSync,
+  closeSync,
 } from "node:fs";
-import { Writable } from "node:stream";
 import { dirs, files } from "./paths.js";
 
 export type LogComponent =
@@ -66,6 +67,7 @@ export type LogComponent =
   | "playwright"
   | "memory"
   | "stickers"
+  | "backup"
   | "backend-controller";
 
 const LOG_FILE = files.log;
@@ -81,20 +83,29 @@ if (!existsSync(dirs.root)) {
 
 // Rotate log file on startup if it exceeds 10MB
 const MAX_LOG_SIZE = 10 * 1024 * 1024;
-try {
-  if (existsSync(LOG_FILE) && statSync(LOG_FILE).size > MAX_LOG_SIZE) {
-    const rotated = `${LOG_FILE}.old`;
+
+/**
+ * Move `path` aside to `path.old` when it has outgrown the cap. Used at
+ * import time for talon.log and at handoff time for respawn.log, so both
+ * files follow the same one-generation rule. Never throws.
+ */
+function rotateIfLarge(path: string): void {
+  try {
+    if (!existsSync(path) || statSync(path).size <= MAX_LOG_SIZE) return;
+    const rotated = `${path}.old`;
     try {
       unlinkSync(rotated);
     } catch {
-      /* ignore */
+      /* no previous generation */
     }
-    renameSync(LOG_FILE, rotated);
+    renameSync(path, rotated);
+  } catch {
+    /* a log file we cannot rotate is still a log file we can append to */
   }
-} catch {
-  /* ignore */
 }
 
+rotateIfLarge(LOG_FILE);
+
 // Suppress console output for terminal frontend (stdout belongs to the REPL)
 let quiet = process.env.TALON_QUIET === "1";
 if (!quiet) {
@@ -149,9 +160,21 @@ const SINK_RETRY_MS = 30_000;
 /** Ceiling for the doubling backoff between reopen attempts. */
 const SINK_MAX_RETRY_MS = 5 * 60_000;
 
+/**
+ * Where a {@link ResilientFileSink} puts its lines. Writing is
+ * synchronous by contract: a line that `write()` returned from is on the
+ * fd, not in a userspace queue. See the class doc for why that matters.
+ */
+export type SyncLogTarget = {
+  /** Append one already-serialized line. Throws on failure. */
+  write(line: string): void;
+  /** Release the underlying handle. Never throws. */
+  close(): void;
+};
+
 export type ResilientFileSinkOptions = {
-  /** Opens the underlying file stream. Injection seam for tests. */
-  open?: (path: string) => Writable;
+  /** Opens the underlying file. Injection seam for tests. */
+  open?: (path: string) => SyncLogTarget;
   /** Where the pause/resume notices go. Defaults to the console sink. */
   notify?: (level: "warn" | "info", message: string) => void;
   /** First backoff step (default 30s). */
@@ -161,20 +184,34 @@ export type ResilientFileSinkOptions = {
 };
 
 /**
- * A log file destination that cannot take the process down.
+ * A log file destination that cannot take the process down, and cannot
+ * lose the last thing the process said.
  *
- * A bare `createWriteStream` handed to `pino.multistream` is a loaded
- * gun: when the disk fills (ENOSPC), or the file is unlinked under a
- * rotation (EBADF), or the fd goes bad (EIO), the stream emits `error`.
- * With no listener that is an uncaught exception — and it fires from
+ * Two failures shaped this class, both on 2026-09-18.
+ *
+ * First, a bare `createWriteStream` handed to `pino.multistream` is a
+ * loaded gun: when the disk fills (ENOSPC), or the file is unlinked
+ * under a rotation (EBADF), or the fd goes bad (EIO), the stream emits
+ * `error`. With no listener that is an uncaught exception — raised from
  * inside a log call, so the crash handler's own `logError` runs on a
- * logger that is already broken. That is how a full disk killed the
- * daemon on 2026-09-18: the process aborted mid-`/update` with no
- * shutdown, no pidfile cleanup, and no successor.
+ * logger that is already broken. A full disk killed the daemon that
+ * way: no shutdown, no pidfile cleanup, no successor.
+ *
+ * Second — and this is why the writes below are synchronous — a
+ * `createWriteStream` buffers in userspace and drains on later ticks,
+ * while every terminal log line Talon writes is immediately followed by
+ * `process.exit()`: "State saved", "Respawn child started", "Timeout
+ * exceeded, forcing exit", "Fatal startup error". `process.exit()`
+ * discards whatever is still queued, so precisely the lines that explain
+ * a failed handoff were the ones that never reached the file. Measured
+ * under Bun 1.3.9: three lines logged and then `process.exit(0)` produced
+ * an empty (not even created) log file. A sink that writes with
+ * `writeSync` has nothing to flush and nothing to lose — no flush step to
+ * remember at any exit site, which is the only version of this that stays
+ * fixed.
  *
- * This sink owns the file stream instead of exposing it. pino only ever
- * sees this object, which never emits `error` and never blocks:
- *   - write failures pause file logging and destroy the broken stream,
+ * pino only ever sees this object, which never throws and never blocks:
+ *   - write failures pause file logging and close the broken handle,
  *   - lines written while paused are DROPPED and counted (never
  *     buffered — the failure mode here is "no space", so growing a
  *     buffer is the last thing to do),
@@ -184,26 +221,21 @@ export type ResilientFileSinkOptions = {
  * The console sink keeps working throughout, and carries the two
  * notices.
  */
-export class ResilientFileSink extends Writable {
+export class ResilientFileSink {
   private readonly path: string;
-  private readonly openStream: (path: string) => Writable;
+  private readonly openTarget: (path: string) => SyncLogTarget;
   private readonly notify: (level: "warn" | "info", message: string) => void;
   private readonly baseRetryMs: number;
   private readonly maxRetryMs: number;
-  private inner: Writable | null = null;
+  private target: SyncLogTarget | null = null;
   private retryMs: number;
   private retryTimer: ReturnType<typeof setTimeout> | null = null;
   private droppedWhileDown = 0;
   private down = false;
 
   constructor(path: string, opts: ResilientFileSinkOptions = {}) {
-    // decodeStrings:false keeps pino's serialized lines as strings;
-    // autoDestroy:false means a downstream failure can never tear this
-    // object down — it is the only thing standing between a broken file
-    // and the process.
-    super({ decodeStrings: false, autoDestroy: false });
     this.path = path;
-    this.openStream = opts.open ?? openLogFile;
+    this.openTarget = opts.open ?? openSyncLogFile;
     this.notify = opts.notify ?? notifyViaConsoleSink;
     this.baseRetryMs = opts.retryMs ?? SINK_RETRY_MS;
     this.maxRetryMs = opts.maxRetryMs ?? SINK_MAX_RETRY_MS;
@@ -221,63 +253,41 @@ export class ResilientFileSink extends Writable {
     return this.down;
   }
 
-  override _write(
-    chunk: unknown,
-    _encoding: BufferEncoding,
-    callback: (error?: Error | null) => void,
-  ): void {
-    const inner = this.inner;
-    if (inner === null) {
+  /** pino.multistream's entire contract: one serialized line in. */
+  write(line: string): void {
+    const target = this.target;
+    if (target === null) {
       this.droppedWhileDown++;
-    } else {
-      try {
-        inner.write(chunk as string, (err) => {
-          if (!err) this.markHealthy();
-        });
-      } catch (err) {
-        // Synchronous throw (write-after-destroy on a stream we have not
-        // been told about yet) — same handling as an `error` event.
-        this.fail(err);
-      }
+      return;
     }
-    // Always report success: pino must never see this sink fail, and
-    // backpressure here would stall whoever called log().
-    callback();
-  }
-
-  override _final(callback: (error?: Error | null) => void): void {
-    this.clearRetry();
     try {
-      this.inner?.end();
-    } catch {
-      /* going away anyway */
+      target.write(line);
+    } catch (err) {
+      this.droppedWhileDown++;
+      this.fail(err);
+      return;
     }
-    callback();
+    this.markHealthy();
   }
 
-  override _destroy(
-    _err: Error | null,
-    callback: (error?: Error | null) => void,
-  ): void {
+  /** No-op: every write already reached the fd. Part of pino's shape. */
+  flushSync(): void {}
+
+  /** Close the file and stop retrying. Part of pino's shape. */
+  end(): void {
     this.clearRetry();
     this.detachInner();
-    callback(null);
   }
 
   private openInner(): void {
     try {
-      const inner = this.openStream(this.path);
-      inner.on("error", (err: Error) => {
-        // Ignore errors from a stream we have already given up on.
-        if (this.inner === inner) this.fail(err);
-      });
-      this.inner = inner;
+      this.target = this.openTarget(this.path);
     } catch (err) {
       this.fail(err);
     }
   }
 
-  /** Give up on the current stream and arm a reopen. Never throws. */
+  /** Give up on the current handle and arm a reopen. Never throws. */
   private fail(err: unknown): void {
     const firstFailure = !this.down;
     this.detachInner();
@@ -311,15 +321,11 @@ export class ResilientFileSink extends Writable {
   }
 
   private detachInner(): void {
-    const inner = this.inner;
-    this.inner = null;
-    if (inner === null) return;
+    const target = this.target;
+    this.target = null;
+    if (target === null) return;
     try {
-      inner.removeAllListeners("error");
-      // destroy() can surface one last error — swallow it here rather
-      // than let it reach process-level uncaughtException.
-      inner.on("error", () => {});
-      inner.destroy();
+      target.close();
     } catch {
       /* best effort */
     }
@@ -358,9 +364,51 @@ export class ResilientFileSink extends Writable {
   }
 }
 
-function openLogFile(path: string): Writable {
+/** Append `line` to `fd`, looping over a short write. Throws on failure. */
+function appendLine(fd: number, line: string): void {
+  const buf = Buffer.from(line, "utf-8");
+  let offset = 0;
+  while (offset < buf.length) {
+    offset += writeSync(fd, buf, offset, buf.length - offset);
+  }
+}
+
+/** The production {@link SyncLogTarget}: an appended fd, written with writeSync. */
+export function openSyncLogFile(path: string): SyncLogTarget {
   // 0600: turns and tool output land here — same sensitivity as history.
-  return createWriteStream(path, { flags: "a", mode: 0o600 });
+  const fd = openSync(path, "a", 0o600);
+  return {
+    write: (line) => appendLine(fd, line),
+    close: () => {
+      try {
+        closeSync(fd);
+      } catch {
+        /* already gone */
+      }
+    },
+  };
+}
+
+/**
+ * Open ~/.talon/respawn.log for a successor's stdout+stderr.
+ *
+ * A `/restart` or `/update` handoff spawns the next daemon detached; with
+ * `stdio: "ignore"` a successor that dies before its logger exists — a
+ * broken import after a dependency install, a fatal bind, a runtime that
+ * aborts — leaves no trace anywhere, which is exactly how the 2026-09-18
+ * handoff vanished. Handing the child this fd makes that visible. Same
+ * one-generation rotation as talon.log.
+ *
+ * Returns null when the file cannot be opened; the caller falls back to
+ * nothing, never to a failed handoff.
+ */
+export function openRespawnLog(path: string = files.respawnLog): number | null {
+  try {
+    rotateIfLarge(path);
+    return openSync(path, "a", 0o600);
+  } catch {
+    return null;
+  }
 }
 
 function notifyViaConsoleSink(level: "warn" | "info", message: string): void {

package/src/util/paths.ts

@@ -102,6 +102,11 @@ export const files = {
   config: resolve(TALON_ROOT, "config.json"),
   /** Structured log: ~/.talon/talon.log */
   log: resolve(TALON_ROOT, "talon.log"),
+  /**
+   * Successor stdout+stderr during a `/restart` or `/update` handoff:
+   * ~/.talon/respawn.log. See core/daemon/respawn.ts.
+   */
+  respawnLog: resolve(TALON_ROOT, "respawn.log"),
   /** Legacy JSON session store (imported into talon.db on first boot) */
   sessions: resolve(TALON_ROOT, "data", "sessions.json"),
   /** SQLite database (history, sessions, chat settings, media index): ~/.talon/data/talon.db */