talon-agent 5.2.0 → 5.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "5.2.0",
+  "version": "5.2.1",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/src/app.ts

@@ -30,6 +30,11 @@ import { shutdownAgents } from "./core/agents/index.js";
 import { pruneSettledTriggers } from "./storage/triggers.js";
 import { startWatchdog, stopWatchdog } from "./util/watchdog.js";
 import { spawnSuccessor } from "./core/daemon/respawn.js";
+import {
+  crashCleanup,
+  crashStep,
+  handleUncaughtException,
+} from "./core/daemon/crash.js";
 import { log, logError, logWarn } from "./util/log.js";
 import { bootPhase, bootReport } from "./core/daemon/boot-timer.js";
 import {
@@ -114,6 +119,10 @@ onBackendChange((holder, newBackend, info) => {
 let shuttingDown = false;
 let triggerPruneTimer: ReturnType<typeof setInterval> | null = null;
 
+// The composition root owns the SQLite handle, so it hands the crash
+// path its checkpoint (see core/daemon/crash.ts).
+const crashHooks = { flushDatabase };
+
 const SHUTDOWN_TIMEOUT_MS = 15_000;
 const DRAIN_TIMEOUT_MS = 5_000;
 
@@ -127,7 +136,11 @@ async function shutdownStep(name: string, fn: () => unknown): Promise<void> {
   try {
     await fn();
   } catch (err) {
-    logError("shutdown", `${name} failed`, err);
+    // Even the report is best-effort: a shutdown triggered by a full
+    // disk must not die inside its own error path.
+    crashStep("shutdown report", () =>
+      logError("shutdown", `${name} failed`, err),
+    );
   }
 }
 
@@ -138,15 +151,18 @@ async function gracefulShutdown(signal: string): Promise<void> {
 
   const deadlineAt = Date.now() + SHUTDOWN_TIMEOUT_MS;
   const forceTimer = setTimeout(() => {
-    logError("shutdown", "Timeout exceeded, forcing exit");
-    // Hand off even on the forced path. A restart must survive a
-    // subsystem that won't stop (a wedged FUSE unmount, an MCP server
-    // ignoring SIGTERM, a backend child that never acks): without this
-    // the timeout exits without a successor and `/restart` silently
-    // takes the daemon down for good. The successor may briefly race
-    // the long-poll we failed to release, but grammy retries the 409 —
-    // a few seconds of overlap beats staying down.
-    spawnSuccessor();
+    // Cleanup first, report second. Handing off matters most here: a
+    // restart must survive a subsystem that won't stop (a wedged FUSE
+    // unmount, an MCP server ignoring SIGTERM, a backend child that
+    // never acks), and it must also survive a logger that can't write —
+    // logging first is what cost us the successor on 2026-09-18. The
+    // successor may briefly race the long-poll we failed to release,
+    // but grammy retries the 409 — a few seconds of overlap beats
+    // staying down.
+    crashCleanup(crashHooks);
+    crashStep("timeout report", () =>
+      logError("shutdown", "Timeout exceeded, forcing exit"),
+    );
     process.exit(1);
   }, SHUTDOWN_TIMEOUT_MS);
   forceTimer.unref();
@@ -225,37 +241,29 @@ async function gracefulShutdown(signal: string): Promise<void> {
     const { shutdownHub } = await import("./core/mcp-hub/index.js");
     await shutdownHub();
   });
-  flushDatabase();
+  // Each tail step stands alone: a full disk can make any of them throw,
+  // and none of them may cost us the ones that follow.
+  crashStep("database flush", () => flushDatabase());
   // Guarded removal: only clear the record if it still names us. A
   // successor that raced ahead and wrote its own pid here must not be
   // orphaned (the bug that made `talon restart` spawn duplicate daemons).
-  removePidRecordIfOwnedBy(process.pid);
-  log("shutdown", "State saved");
+  crashStep("pid record removal", () => removePidRecordIfOwnedBy(process.pid));
+  crashStep("shutdown report", () => log("shutdown", "State saved"));
   // Hand off last: the frontends are stopped, so the successor binds
   // Telegram's long-poll only after we have released it. No-op unless
   // /restart or /update armed a respawn.
-  spawnSuccessor();
+  crashStep("respawn handoff", () => spawnSuccessor());
   process.exit(0);
 }
 
 process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
 process.on("SIGINT", () => gracefulShutdown("SIGINT"));
 
-process.on("uncaughtException", (err) => {
-  // EPIPE errors from network sockets (e.g. Telegram MTProto) are transient —
-  // gramjs will reconnect; crashing the process here is wrong.
-  if ((err as NodeJS.ErrnoException).code === "EPIPE") {
-    logWarn("bot", `Suppressed transient EPIPE error: ${err.message}`);
-    return;
-  }
-  logError("bot", "Uncaught exception", err);
-  flushDatabase();
-  // Same pid-guarded removal as the graceful path — a crashed daemon
-  // must not leave a record that makes `talon status` chase a dead or
-  // recycled pid.
-  removePidRecordIfOwnedBy(process.pid);
-  process.exit(1);
-});
+// Cleanup runs before the crash is reported (and the EPIPE suppression
+// is unchanged) — see core/daemon/crash.ts for why the order matters.
+process.on("uncaughtException", (err) =>
+  handleUncaughtException(err, crashHooks),
+);
 
 process.on("unhandledRejection", (reason) => {
   logWarn(
@@ -333,8 +341,9 @@ async function main(): Promise<void> {
 }
 
 main().catch((err) => {
-  logError("bot", "Fatal startup error", err);
-  flushDatabase();
-  removePidRecordIfOwnedBy(process.pid);
+  crashCleanup(crashHooks);
+  crashStep("startup report", () =>
+    logError("bot", "Fatal startup error", err),
+  );
   process.exit(1);
 });

package/src/core/daemon/crash.ts

@@ -0,0 +1,82 @@
+/**
+ * Crash-path cleanup — what has to happen when the daemon goes down
+ * abnormally: an uncaught exception, the forced exit after a shutdown
+ * timeout, or a fatal startup error.
+ *
+ * Ordering is the whole point. The old handler logged first and cleaned
+ * up afterwards, which works right up until logging is the thing that
+ * broke. On 2026-09-18 the root disk filled; the log file's write stream
+ * emitted ENOSPC with nobody listening, the uncaught-exception handler
+ * called `logError` as its first act, threw inside itself, and Node
+ * aborted the process — no database checkpoint, a stale pidfile left
+ * behind, and (the expensive part) an armed `/update` handoff that never
+ * spawned its successor. The daemon simply vanished.
+ *
+ * So: the essentials first, each in its own try/catch, logging last and
+ * strictly best-effort. `src/util/log.ts` now keeps a broken log file
+ * from throwing at all — this is the second line of defence, for every
+ * other way logging can fail while the machine is sick.
+ */
+
+import { logError, logWarn } from "../../util/log.js";
+import { removePidRecordIfOwnedBy } from "./pidfile.js";
+import { spawnSuccessor } from "./respawn.js";
+
+/**
+ * Steps the crash path can only get from the composition root.
+ * `flushDatabase` is injected because the SQLite handle stays inside
+ * `storage/` (.dependency-cruiser: db-handle-stays-in-storage).
+ */
+export type CrashHooks = {
+  flushDatabase: () => void;
+};
+
+/**
+ * Run one crash-path step. Never throws, and never uses the logger —
+ * a broken logger is precisely the case this path exists for, so the
+ * console is the fallback of last resort.
+ */
+export function crashStep(name: string, fn: () => void): void {
+  try {
+    fn();
+  } catch (err) {
+    try {
+      console.error(`[talon] crash cleanup: ${name} failed`, err);
+    } catch {
+      /* stdio is gone too — nothing left to try */
+    }
+  }
+}
+
+/**
+ * The non-logging essentials, in the order that matters:
+ *   1. drop the pid record, so `talon status` stops chasing a dead pid
+ *      (before the successor writes its own — the guarded removal then
+ *      cannot possibly orphan it),
+ *   2. hand off to the successor if `/restart` or `/update` armed one
+ *      (a no-op otherwise, see respawn.ts),
+ *   3. checkpoint the database.
+ */
+export function crashCleanup(hooks: CrashHooks): void {
+  crashStep("pid record removal", () => removePidRecordIfOwnedBy(process.pid));
+  crashStep("respawn handoff", () => spawnSuccessor());
+  crashStep("database flush", hooks.flushDatabase);
+}
+
+/**
+ * `process.on("uncaughtException")` body. Cleanup happens before the
+ * crash is reported, never after.
+ */
+export function handleUncaughtException(err: Error, hooks: CrashHooks): void {
+  // EPIPE errors from network sockets (e.g. Telegram MTProto) are transient —
+  // gramjs will reconnect; crashing the process here is wrong.
+  if ((err as NodeJS.ErrnoException).code === "EPIPE") {
+    crashStep("EPIPE notice", () =>
+      logWarn("bot", `Suppressed transient EPIPE error: ${err.message}`),
+    );
+    return;
+  }
+  crashCleanup(hooks);
+  crashStep("crash report", () => logError("bot", "Uncaught exception", err));
+  process.exit(1);
+}

package/src/util/log.ts

@@ -18,6 +18,7 @@ import {
   unlinkSync,
   createWriteStream,
 } from "node:fs";
+import { Writable } from "node:stream";
 import { dirs, files } from "./paths.js";
 
 export type LogComponent =
@@ -127,23 +128,249 @@ const streams: pino.StreamEntry[] = [];
 
 // Console output (disabled in quiet mode), pretty-printed.
 if (!quiet) {
-  streams.push({
-    level: "trace",
-    stream: prettyStream({
-      colorize: true,
-      ignore: "pid,hostname",
-      translateTime: "HH:MM:ss",
-    }),
+  const consoleStream = prettyStream({
+    colorize: true,
+    ignore: "pid,hostname",
+    translateTime: "HH:MM:ss",
   });
+  // pino-pretty writes through a SonicBoom on fd 1 whose own error
+  // handler removes itself after the first non-EPIPE failure
+  // (pino-pretty/lib/utils/build-safe-sonic-boom.js). With stdout
+  // redirected to a file on a full disk that leaves the pipeline's next
+  // error unhandled — i.e. an uncaught exception raised from inside a
+  // log call. One permanent listener closes that door; a dead console
+  // is not worth a dead daemon.
+  consoleStream.on("error", () => {});
+  streams.push({ level: "trace", stream: consoleStream });
+}
+
+/** How long to wait before the first attempt to reopen a failed log file. */
+const SINK_RETRY_MS = 30_000;
+/** Ceiling for the doubling backoff between reopen attempts. */
+const SINK_MAX_RETRY_MS = 5 * 60_000;
+
+export type ResilientFileSinkOptions = {
+  /** Opens the underlying file stream. Injection seam for tests. */
+  open?: (path: string) => Writable;
+  /** Where the pause/resume notices go. Defaults to the console sink. */
+  notify?: (level: "warn" | "info", message: string) => void;
+  /** First backoff step (default 30s). */
+  retryMs?: number;
+  /** Backoff ceiling (default 5 min). */
+  maxRetryMs?: number;
+};
+
+/**
+ * A log file destination that cannot take the process down.
+ *
+ * A bare `createWriteStream` handed to `pino.multistream` is a loaded
+ * gun: when the disk fills (ENOSPC), or the file is unlinked under a
+ * rotation (EBADF), or the fd goes bad (EIO), the stream emits `error`.
+ * With no listener that is an uncaught exception — and it fires from
+ * inside a log call, so the crash handler's own `logError` runs on a
+ * logger that is already broken. That is how a full disk killed the
+ * daemon on 2026-09-18: the process aborted mid-`/update` with no
+ * shutdown, no pidfile cleanup, and no successor.
+ *
+ * This sink owns the file stream instead of exposing it. pino only ever
+ * sees this object, which never emits `error` and never blocks:
+ *   - write failures pause file logging and destroy the broken stream,
+ *   - lines written while paused are DROPPED and counted (never
+ *     buffered — the failure mode here is "no space", so growing a
+ *     buffer is the last thing to do),
+ *   - an unref'd timer retries the open on a 30s → 5min backoff,
+ *   - the first write to land again resumes logging and reports how
+ *     many lines were lost.
+ * The console sink keeps working throughout, and carries the two
+ * notices.
+ */
+export class ResilientFileSink extends Writable {
+  private readonly path: string;
+  private readonly openStream: (path: string) => Writable;
+  private readonly notify: (level: "warn" | "info", message: string) => void;
+  private readonly baseRetryMs: number;
+  private readonly maxRetryMs: number;
+  private inner: Writable | null = null;
+  private retryMs: number;
+  private retryTimer: ReturnType<typeof setTimeout> | null = null;
+  private droppedWhileDown = 0;
+  private down = false;
+
+  constructor(path: string, opts: ResilientFileSinkOptions = {}) {
+    // decodeStrings:false keeps pino's serialized lines as strings;
+    // autoDestroy:false means a downstream failure can never tear this
+    // object down — it is the only thing standing between a broken file
+    // and the process.
+    super({ decodeStrings: false, autoDestroy: false });
+    this.path = path;
+    this.openStream = opts.open ?? openLogFile;
+    this.notify = opts.notify ?? notifyViaConsoleSink;
+    this.baseRetryMs = opts.retryMs ?? SINK_RETRY_MS;
+    this.maxRetryMs = opts.maxRetryMs ?? SINK_MAX_RETRY_MS;
+    this.retryMs = this.baseRetryMs;
+    this.openInner();
+  }
+
+  /** Lines discarded since the sink went down; cleared on recovery. */
+  get dropped(): number {
+    return this.droppedWhileDown;
+  }
+
+  /** True while file logging is paused (the console sink still runs). */
+  get isDown(): boolean {
+    return this.down;
+  }
+
+  override _write(
+    chunk: unknown,
+    _encoding: BufferEncoding,
+    callback: (error?: Error | null) => void,
+  ): void {
+    const inner = this.inner;
+    if (inner === null) {
+      this.droppedWhileDown++;
+    } else {
+      try {
+        inner.write(chunk as string, (err) => {
+          if (!err) this.markHealthy();
+        });
+      } catch (err) {
+        // Synchronous throw (write-after-destroy on a stream we have not
+        // been told about yet) — same handling as an `error` event.
+        this.fail(err);
+      }
+    }
+    // Always report success: pino must never see this sink fail, and
+    // backpressure here would stall whoever called log().
+    callback();
+  }
+
+  override _final(callback: (error?: Error | null) => void): void {
+    this.clearRetry();
+    try {
+      this.inner?.end();
+    } catch {
+      /* going away anyway */
+    }
+    callback();
+  }
+
+  override _destroy(
+    _err: Error | null,
+    callback: (error?: Error | null) => void,
+  ): void {
+    this.clearRetry();
+    this.detachInner();
+    callback(null);
+  }
+
+  private openInner(): void {
+    try {
+      const inner = this.openStream(this.path);
+      inner.on("error", (err: Error) => {
+        // Ignore errors from a stream we have already given up on.
+        if (this.inner === inner) this.fail(err);
+      });
+      this.inner = inner;
+    } catch (err) {
+      this.fail(err);
+    }
+  }
+
+  /** Give up on the current stream and arm a reopen. Never throws. */
+  private fail(err: unknown): void {
+    const firstFailure = !this.down;
+    this.detachInner();
+    this.down = true;
+    const delay = this.retryMs;
+    this.retryMs = Math.min(this.retryMs * 2, this.maxRetryMs);
+    this.scheduleRetry(delay);
+    if (!firstFailure) return; // a failed reopen is not news
+    const code =
+      (err as NodeJS.ErrnoException | undefined)?.code ??
+      (err instanceof Error ? err.message : String(err));
+    this.emitNotice(
+      "warn",
+      `Log file sink failed (${code}) — file logging paused, ` +
+        `retrying in ${Math.round(delay / 1000)}s`,
+    );
+  }
+
+  /** A write landed: the file is usable again. */
+  private markHealthy(): void {
+    if (!this.down) return;
+    this.down = false;
+    this.retryMs = this.baseRetryMs;
+    const dropped = this.droppedWhileDown;
+    this.droppedWhileDown = 0;
+    this.emitNotice(
+      "info",
+      `Log file sink recovered — file logging resumed ` +
+        `(${dropped} line(s) dropped while it was down)`,
+    );
+  }
+
+  private detachInner(): void {
+    const inner = this.inner;
+    this.inner = null;
+    if (inner === null) return;
+    try {
+      inner.removeAllListeners("error");
+      // destroy() can surface one last error — swallow it here rather
+      // than let it reach process-level uncaughtException.
+      inner.on("error", () => {});
+      inner.destroy();
+    } catch {
+      /* best effort */
+    }
+  }
+
+  private scheduleRetry(delay: number): void {
+    this.clearRetry();
+    const timer = setTimeout(() => {
+      this.retryTimer = null;
+      this.openInner();
+    }, delay);
+    // A paused log sink must not hold the event loop open.
+    timer.unref();
+    this.retryTimer = timer;
+  }
+
+  private clearRetry(): void {
+    if (this.retryTimer === null) return;
+    clearTimeout(this.retryTimer);
+    this.retryTimer = null;
+  }
+
+  /**
+   * Notices are deferred: `fail()` can run inside pino's multistream
+   * write loop, and logging re-entrantly from there would scramble the
+   * metadata pino hangs off the stream for the rest of that loop.
+   */
+  private emitNotice(level: "warn" | "info", message: string): void {
+    queueMicrotask(() => {
+      try {
+        this.notify(level, message);
+      } catch {
+        /* the notice is the least important thing here */
+      }
+    });
+  }
+}
+
+function openLogFile(path: string): Writable {
+  // 0600: turns and tool output land here — same sensitivity as history.
+  return createWriteStream(path, { flags: "a", mode: 0o600 });
+}
+
+function notifyViaConsoleSink(level: "warn" | "info", message: string): void {
+  if (level === "warn") logWarn("file", message);
+  else log("file", message);
 }
 
 // JSON file output (always active outside test runs).
 if (!IS_VITEST) {
-  streams.push({
-    level: "trace",
-    // 0600: turns and tool output land here — same sensitivity as history.
-    stream: createWriteStream(LOG_FILE, { flags: "a", mode: 0o600 }),
-  });
+  streams.push({ level: "trace", stream: new ResilientFileSink(LOG_FILE) });
 }
 
 const logger =
@@ -151,8 +378,24 @@ const logger =
     ? pino({ level: "trace" }, pino.multistream(streams))
     : pino({ level: "silent" });
 
+/**
+ * Emit one record. A logger that throws is worse than a silent one: the
+ * throw lands on whoever called log(), which at shutdown is a signal
+ * handler or a crash handler, and a throw there takes the daemon down —
+ * pino-pretty's SonicBoom, for one, throws "SonicBoom destroyed"
+ * synchronously on every write once a failure has destroyed it. Nothing
+ * a sink does may escape this module.
+ */
+function emit(write: () => void): void {
+  try {
+    write();
+  } catch {
+    /* a broken logger must never become a broken daemon */
+  }
+}
+
 export function log(component: LogComponent, message: string): void {
-  logger.info({ component }, message);
+  emit(() => logger.info({ component }, message));
 }
 
 export function logError(
@@ -164,20 +407,22 @@ export function logError(
     // Capture both the concise message (for log consumers that look at `err`)
     // and the full stack (for diagnostics). pino-pretty renders the `stack`
     // field on its own line; JSON consumers can read either field.
-    logger.error({ component, err: err.message, stack: err.stack }, message);
+    emit(() =>
+      logger.error({ component, err: err.message, stack: err.stack }, message),
+    );
   } else if (err !== undefined) {
-    logger.error({ component, err: String(err) }, message);
+    emit(() => logger.error({ component, err: String(err) }, message));
   } else {
-    logger.error({ component }, message);
+    emit(() => logger.error({ component }, message));
   }
 }
 
 export function logWarn(component: LogComponent, message: string): void {
-  logger.warn({ component }, message);
+  emit(() => logger.warn({ component }, message));
 }
 
 export function logDebug(component: LogComponent, message: string): void {
-  logger.debug({ component }, message);
+  emit(() => logger.debug({ component }, message));
 }
 
 // Expose logger to plugins running in the same process