talon-agent 5.18.2 → 5.19.0

package/src/frontend/health/delivery.ts

@@ -0,0 +1,115 @@
+/**
+ * Reply delivery health — the worst silent failure is a turn that ran but
+ * whose answer never reached the chat. The model sees a failed tool call;
+ * the person who asked sees nothing, and so does the operator.
+ *
+ * Each frontend wraps its action handler with `trackDeliveries`. The
+ * reply actions (`send_message`, `send_message_with_buttons`, `reply_to`)
+ * are counted per chat: `DELIVERY_ALERT_AFTER` failures in a row for one
+ * chat raise `delivery.<frontend>` with that chat and the error, and the
+ * next success to it resolves the alert once no chat is still failing.
+ */
+
+import type { ActionResult } from "../../core/types.js";
+import {
+  raiseAlert,
+  resolveAlert,
+} from "../../core/frontend-runtime/alerts.js";
+import { log, logWarn, type LogComponent } from "../../util/log.js";
+import { errorText } from "./outage.js";
+
+/** Consecutive failed replies to one chat before the operator hears of it. */
+const DELIVERY_ALERT_AFTER = 3;
+
+const REPLY_ACTIONS = new Set([
+  "send_message",
+  "send_message_with_buttons",
+  "reply_to",
+]);
+
+export type DeliveryTracker = {
+  failed(chat: string | number, err: unknown): void;
+  delivered(chat: string | number): void;
+};
+
+/**
+ * `label` is the operator-facing platform name ("Telegram"); the alert key
+ * is `delivery.<frontend>`.
+ */
+export function createDeliveryTracker(
+  frontend: string,
+  label: string,
+  component: LogComponent,
+): DeliveryTracker {
+  const key = `delivery.${frontend}`;
+  const streaks = new Map<string, number>();
+  /** Chats that crossed the threshold and have not delivered since. */
+  const alerted = new Set<string>();
+
+  return {
+    failed(chat, err) {
+      const id = String(chat);
+      const streak = (streaks.get(id) ?? 0) + 1;
+      streaks.set(id, streak);
+      const error = errorText(err);
+      logWarn(
+        component,
+        `delivery.fail frontend=${frontend} chat=${id} streak=${streak} err=${error}`,
+      );
+      if (streak < DELIVERY_ALERT_AFTER) return;
+      alerted.add(id);
+      raiseAlert(
+        key,
+        `${label} replies to chat ${id} have failed ${streak} times in a row: ${error}. ` +
+          "Answers are not reaching that chat.",
+      );
+    },
+    delivered(chat) {
+      const id = String(chat);
+      const streak = streaks.get(id);
+      if (streak === undefined) return;
+      streaks.delete(id);
+      log(
+        component,
+        `delivery.recovered frontend=${frontend} chat=${id} after_failures=${streak}`,
+      );
+      if (alerted.delete(id) && alerted.size === 0) {
+        resolveAlert(key, `${label} replies are being delivered again.`);
+      }
+    },
+  };
+}
+
+type ActionHandler = (
+  body: Record<string, unknown>,
+  chatId: number,
+) => Promise<ActionResult | null>;
+
+/**
+ * Wrap a frontend action handler so its reply actions feed `tracker`.
+ * Results and thrown errors pass through untouched. `chatOf` names the
+ * destination when an action can target a chat other than `chatId`.
+ */
+export function trackDeliveries(
+  tracker: DeliveryTracker,
+  handler: ActionHandler,
+  chatOf: (body: Record<string, unknown>, chatId: number) => string | number = (
+    _body,
+    chatId,
+  ) => chatId,
+): ActionHandler {
+  return async (body, chatId) => {
+    if (!REPLY_ACTIONS.has(body.action as string)) return handler(body, chatId);
+    const chat = chatOf(body, chatId);
+    let result: ActionResult | null;
+    try {
+      result = await handler(body, chatId);
+    } catch (err) {
+      tracker.failed(chat, err);
+      throw err;
+    }
+    if (result?.ok) tracker.delivered(chat);
+    else if (result) tracker.failed(chat, result.error ?? "unknown error");
+    return result;
+  };
+}

package/src/frontend/health/outage.ts

@@ -0,0 +1,116 @@
+/**
+ * Connection outages — the line between "a reconnect failed" (log it) and
+ * "this chat channel is down" (tell the operator).
+ *
+ * A frontend reports every failed poll / reconnect with `fail` and every
+ * healthy one with `ok`. The first failure arms a timer; if nothing healthy
+ * arrives before `thresholdMs`, the alert is raised with the latest error.
+ * The timer — not the next failure — decides, because a link that dies
+ * silently (a gateway that never reconnects) produces no further events.
+ * `ok` ends the outage and, when an alert went out, sends the recovery.
+ */
+
+import {
+  raiseAlert,
+  resolveAlert,
+  type AlertSeverity,
+} from "../../core/frontend-runtime/alerts.js";
+
+export type OutageOptions = {
+  /** Stable alert key, e.g. "telegram.polling". */
+  key: string;
+  /** How long failures must persist before the operator hears of it. */
+  thresholdMs: number;
+  severity?: AlertSeverity;
+  /** Operator text for the raise, given the latest error and minutes down. */
+  describe: (lastError: string, downMin: number) => string;
+  /** Operator text for the recovery notice. */
+  recovered: string;
+};
+
+export type Outage = {
+  /** Record a failure. Returns the streak so the caller can log it. */
+  fail(err: unknown): { attempt: number; downMs: number };
+  /** Record a healthy round-trip. Returns the outage it ended, if any. */
+  ok(): { attempts: number; downMs: number } | null;
+  /** Raise immediately — for failures that will not heal on their own. */
+  raiseNow(message: string, severity?: AlertSeverity): void;
+  /** True between the first failure and the next `ok`. */
+  readonly down: boolean;
+  /** Forget the outage without resolving it — shutdown. */
+  dispose(): void;
+};
+
+const MAX_ERROR_CHARS = 200;
+
+/**
+ * One-line error text fit for an alert or a log line: the message only,
+ * bounded, with bot tokens and webhook signatures masked.
+ */
+export function errorText(err: unknown): string {
+  const raw = err instanceof Error ? err.message : String(err);
+  const text = raw
+    .replace(/bot\d+:[\w-]+/g, "bot<redacted>")
+    .replace(/([?&]sig=)[^&\s]+/g, "$1<redacted>")
+    .replace(/\s+/g, " ")
+    .trim();
+  return text.length > MAX_ERROR_CHARS
+    ? `${text.slice(0, MAX_ERROR_CHARS)}…`
+    : text;
+}
+
+export function createOutage(opts: OutageOptions): Outage {
+  let since = 0;
+  let attempts = 0;
+  let lastError = "";
+  let raised = false;
+  let timer: ReturnType<typeof setTimeout> | null = null;
+
+  const clearTimer = (): void => {
+    if (timer) clearTimeout(timer);
+    timer = null;
+  };
+  const raise = (message: string, severity?: AlertSeverity): void => {
+    raised = true;
+    raiseAlert(opts.key, message, { severity: severity ?? opts.severity });
+  };
+
+  return {
+    get down() {
+      return attempts > 0;
+    },
+    fail(err) {
+      const now = Date.now();
+      if (attempts === 0) {
+        since = now;
+        timer = setTimeout(() => {
+          timer = null;
+          const mins = Math.max(1, Math.round((Date.now() - since) / 60_000));
+          raise(opts.describe(lastError, mins));
+        }, opts.thresholdMs);
+        timer.unref?.();
+      }
+      attempts++;
+      lastError = errorText(err);
+      return { attempt: attempts, downMs: now - since };
+    },
+    ok() {
+      if (attempts === 0 && !raised) return null;
+      const ended = { attempts, downMs: attempts ? Date.now() - since : 0 };
+      clearTimer();
+      attempts = 0;
+      if (raised) resolveAlert(opts.key, opts.recovered);
+      raised = false;
+      return ended;
+    },
+    raiseNow(message, severity) {
+      clearTimer();
+      raise(message, severity);
+    },
+    dispose() {
+      clearTimer();
+      attempts = 0;
+      raised = false;
+    },
+  };
+}

package/src/frontend/native/bridge/routes/chats.ts

@@ -132,11 +132,9 @@ export function chatRoutes(
       const startedAt = Date.now();
       try {
         const attachment = await h.upload(filename, contentType, req);
-        // Uploads are the one client action whose failure used to be visible
-        // only in the app: the route answered in JSON and logged nothing, so
-        // "attaching a file doesn't work — check the logs" had nothing to
-        // read. Both outcomes are logged now, with the size and duration
-        // that distinguish a rejected upload from a dropped connection.
+        // Log both outcomes, with the size and duration that tell a rejected
+        // upload from a dropped connection — the JSON answer is otherwise
+        // only visible in the app.
         log(
           "native",
           `upload ok: ${attachment.name} (${attachment.size} bytes, ${attachment.mimeType}) in ${Date.now() - startedAt}ms`,

package/src/frontend/native/bridge/server.ts

@@ -22,9 +22,14 @@ import {
 } from "node:http";
 import { createServer as createTlsServer } from "node:https";
 import { createHash, timingSafeEqual } from "node:crypto";
-import { createReadStream } from "node:fs";
+import { createReadStream, type ReadStream } from "node:fs";
 import { stat } from "node:fs/promises";
-import { log, logError, logDebug } from "../../../util/log.js";
+import { log, logError, logDebug, logWarn } from "../../../util/log.js";
+import {
+  raiseAlert,
+  resolveAlert,
+} from "../../../core/frontend-runtime/alerts.js";
+import { errorText } from "../../health/outage.js";
 import {
   formatFingerprint,
   isLoopbackHost,
@@ -63,6 +68,14 @@ type StreamSession = {
 };
 
 const SSE_PING_MS = 25_000;
+/**
+ * Unsent bytes a stream may hold before it counts as dead. A client that
+ * stops reading without closing (phone asleep, network switch) otherwise
+ * buffers every broadcast in memory until TCP gives up on it, minutes later.
+ * Far past anything a live client falls behind by; evicted, it reconnects
+ * and gets a fresh `hello`.
+ */
+const SSE_MAX_BACKLOG_BYTES = 16 * 1024 * 1024;
 const MAX_BODY_BYTES = 256 * 1024;
 const PORT_FALLBACKS = 5;
 
@@ -94,6 +107,15 @@ export const DEFAULT_BRIDGE_TIMEOUTS: BridgeTimeouts = {
   checkIntervalMs: 30_000,
 };
 
+/**
+ * `pipe` never closes its source when the destination goes away, so a
+ * client that hangs up mid-download (app backgrounded, image scrolled
+ * away) would leave the paused read stream holding its fd forever.
+ */
+function releaseOnClose(res: ServerResponse, stream: ReadStream): void {
+  res.once("close", () => stream.destroy());
+}
+
 export class BridgeServer {
   private server: Server | null = null;
   /**
@@ -225,14 +247,14 @@ export class BridgeServer {
    *
    * Device commands are not public: their params carry one-time transfer
    * tokens, exec command lines, remote paths, and — on the chunked fallback —
-   * whole base64 file bodies. Broadcasting them handed every connected client
-   * another device's secrets and relied on each client discarding what wasn't
-   * addressed to it, which is courtesy, not enforcement.
+   * whole base64 file bodies. Broadcasting them would hand every connected
+   * client another device's secrets and rely on each client discarding what
+   * isn't addressed to it, which is courtesy, not enforcement.
    *
    * A claim is an ADDRESS, not a credential: any client holding the bridge
    * token could claim any id, and the bridge token is (still) the only trust
-   * boundary here. What this buys is that a device no longer passively
-   * receives traffic meant for its peers.
+   * boundary here. What this buys is that a device does not passively
+   * receive traffic meant for its peers.
    *
    * Clients that claimed nothing are the fallback audience, and only when the
    * target claimed nothing either: a companion build that predates the claim
@@ -265,12 +287,24 @@ export class BridgeServer {
 
   private write(targets: Iterable<ServerResponse>, event: BridgeEvent): void {
     const payload = `data: ${JSON.stringify(event)}\n\n`;
-    for (const res of targets) {
-      try {
-        res.write(payload);
-      } catch {
-        // Write on a half-closed socket — the 'close' handler will evict it.
-      }
+    for (const res of targets) this.send(res, payload);
+  }
+
+  /** Write one frame to a stream, evicting it if its backlog never drains. */
+  private send(res: ServerResponse, frame: string): void {
+    if (res.writableLength > SSE_MAX_BACKLOG_BYTES) {
+      this.clients.delete(res);
+      logWarn(
+        "native",
+        `Dropped an SSE client that stopped reading (${res.writableLength} bytes unsent)`,
+      );
+      res.destroy();
+      return;
+    }
+    try {
+      res.write(frame);
+    } catch {
+      // Write on a half-closed socket — the 'close' handler will evict it.
     }
   }
 
@@ -281,7 +315,7 @@ export class BridgeServer {
       loopback: isLoopbackHost(this.opts.host),
       allowWeakToken: this.opts.allowWeakToken,
     });
-    this.tlsIdentity = this.opts.tls ? await this.opts.tls() : null;
+    this.tlsIdentity = this.opts.tls ? await this.loadTls(this.opts.tls) : null;
     const onRequest = (req: IncomingMessage, res: ServerResponse): void => {
       this.handle(req, res).catch((err) => {
         logError("native", "Bridge request handler threw", err);
@@ -312,29 +346,70 @@ export class BridgeServer {
       : createServer(serverOpts, onRequest);
 
     this.pingTimer = setInterval(() => {
-      for (const res of this.clients.keys()) {
-        try {
-          res.write(": ping\n\n");
-        } catch {
-          /* evicted on close */
-        }
-      }
+      for (const res of this.clients.keys()) this.send(res, ": ping\n\n");
     }, SSE_PING_MS);
     this.pingTimer.unref?.();
     this.unsubscribeRevocations = this.opts.credentials?.authority.onRevoked(
       (ids) => this.dropCredentialSessions(ids),
     );
 
+    return this.bind(server);
+  }
+
+  /**
+   * The TLS identity, or a thrown boot failure the operator hears about:
+   * without it no companion app can connect.
+   */
+  private async loadTls(
+    load: () => Promise<BridgeTlsIdentity>,
+  ): Promise<BridgeTlsIdentity> {
+    try {
+      const identity = await load();
+      resolveAlert(
+        "bridge.tls",
+        "The client bridge TLS certificate loads again.",
+      );
+      return identity;
+    } catch (err) {
+      logError("native", `bridge.tls.fail err=${errorText(err)}`, err);
+      raiseAlert(
+        "bridge.tls",
+        `The client bridge could not load its TLS certificate: ${errorText(err)}. Companion apps cannot connect.`,
+        { severity: "critical" },
+      );
+      throw err;
+    }
+  }
+
+  /**
+   * Listen on the configured port, stepping up to PORT_FALLBACKS ports past
+   * it when one is taken. A bind that fails for good raises `bridge.listen`
+   * — the frontend has no surface at all without it.
+   */
+  private bind(server: Server): Promise<number> {
     return new Promise<number>((resolve, reject) => {
       let attempt = 0;
       const tryPort = (p: number): void => {
         server.once("error", (err: NodeJS.ErrnoException) => {
           if (err.code === "EADDRINUSE" && attempt < PORT_FALLBACKS) {
             attempt++;
+            logWarn(
+              "native",
+              `bridge.listen port=${p} in use — trying port=${p + 1} attempt=${attempt}/${PORT_FALLBACKS}`,
+            );
             server.removeAllListeners("error");
             server.removeAllListeners("listening");
             tryPort(p + 1);
           } else {
+            logError(
+              "native",
+              `bridge.listen.fail host=${this.opts.host} port=${p} attempt=${attempt} err=${errorText(err)}`,
+            );
+            raiseAlert(
+              "bridge.listen",
+              `The client bridge could not listen on ${this.opts.host}:${p}: ${errorText(err)}. Companion apps cannot connect.`,
+              { severity: "critical" },
+            );
             reject(err);
           }
         });
@@ -362,6 +437,10 @@ export class BridgeServer {
               `Bridge certificate fingerprint ${formatFingerprint(this.tlsIdentity.fingerprint)}`,
             );
           }
+          resolveAlert(
+            "bridge.listen",
+            "The client bridge is listening again.",
+          );
           resolve(this.port);
         });
       };
@@ -507,6 +586,7 @@ export class BridgeServer {
     });
     const stream = createReadStream(file.path);
     stream.on("error", () => res.destroy());
+    releaseOnClose(res, stream);
     stream.pipe(res);
   }
 
@@ -532,6 +612,7 @@ export class BridgeServer {
         if (!res.headersSent) res.writeHead(500);
         res.end();
       });
+      releaseOnClose(res, stream);
       stream.pipe(res);
     } catch {
       return this.json(res, 404, { ok: false, error: "No such media" });
@@ -599,6 +680,10 @@ export class BridgeServer {
     const ms = Math.round(max * (0.9 + Math.random() * 0.2));
     const timer = setTimeout(() => {
       logDebug("native", "bridge.sse event=max_lifetime reason=expired");
+      // Out of the fan-out before end(): a stream still flushing a backlog
+      // stays open until it drains, and a broadcast in that window is a
+      // write after end — an unhandled 'error' that takes the daemon down.
+      this.clients.delete(res);
       res.end();
     }, ms);
     timer.unref?.();

package/src/frontend/native/index.ts

@@ -191,7 +191,7 @@ export function createNativeFrontend(
       await mesh.load();
       unregisterMeshTransport = registerMeshTransport(runtime, server);
       // Mesh tool actions (list_devices / get_device_location) are shared
-      // gateway actions now — no native-only cases here.
+      // gateway actions — no native-only cases here.
       gateway.registerFrontendHandler(
         "native",
         createNativeActionHandler({

package/src/frontend/native/media/media.ts

@@ -20,13 +20,17 @@ import type { NativeRuntime } from "../runtime.js";
  */
 export const MAX_UPLOAD_BYTES = 512 * 1024 * 1024;
 
-/** Register a file for serving and return its short media id. */
+/** Register a file for serving and return its short media id (stable per
+ *  path for the life of the daemon). */
 export function registerMedia(
   runtime: NativeRuntime,
   filePath: string,
 ): string {
+  const known = runtime.mediaIds.get(filePath);
+  if (known) return known;
   const id = `m${runtime.nextId().toString(36)}`;
   runtime.media.set(id, filePath);
+  runtime.mediaIds.set(filePath, id);
   return id;
 }
 

package/src/frontend/native/runtime.ts

@@ -1,11 +1,9 @@
 /**
- * Native frontend runtime — the state every bridge module shares.
- *
- * `createNativeFrontend` used to hold all of this as closure variables with
- * every handler nested inside it. It is now one explicit object, constructed
- * once, that each module (chat-wire, context, emit, turn, …) takes as its
- * first parameter. The runtime carries state plus two primitives — the
- * message-id minter and the broadcast sink; the modules own the behaviour.
+ * Native frontend runtime — the state every bridge module shares: one
+ * object, constructed once, that each module (chat-wire, context, emit,
+ * turn, …) takes as its first parameter. It carries state plus two
+ * primitives — the message-id minter and the broadcast sink; the modules
+ * own the behaviour.
  */
 
 import type { TalonConfig } from "../../core/config/index.js";
@@ -55,6 +53,12 @@ export type NativeRuntime = {
    * frontends behave).
    */
   readonly media: Map<string, string>;
+  /**
+   * `media` reversed (path → id). Every history page re-registers its
+   * attachments, so without this each fetch — and every client reconnect
+   * re-fetches — grew `media` by one entry per attachment, forever.
+   */
+  readonly mediaIds: Map<string, string>;
   /**
    * Uploads this daemon run has accepted, keyed by their media id. `/send`
    * resolves a client's attachment references through here rather than
@@ -110,6 +114,7 @@ export function createNativeRuntime(
     chats: new NativeChats(),
     mesh: getMeshService(),
     media: new Map(),
+    mediaIds: new Map(),
     uploads: new Map(),
     contextByChat: new Map(),
     liveTurns: new Map(),

package/src/frontend/native/surface/handlers.ts

@@ -79,7 +79,7 @@ function resolveAttachments(
     }
   }
   // A dropped reference is how "I attached a file and the agent never saw it"
-  // happens, and it used to be entirely silent. The usual cause is a media id
+  // happens, so it must not be silent. The usual cause is a media id
   // minted by a previous daemon run — `runtime.uploads` is per-run, so a
   // client that staged a file before a restart and sent it after points at
   // nothing.

package/src/frontend/native/surface/memory.ts

@@ -5,7 +5,7 @@
  *
  * Read-only on purpose: a bridge client can ask what Talon remembers and
  * why, and nothing more. Asserting, superseding and dropping stay with
- * the write path (rollout PR 6), so a paired phone can never quietly
+ * the write path, so a paired phone can never quietly
  * rewrite the operator's memory.
  *
  * The one piece of policy here is the limit cap: whatever a client asks

package/src/frontend/native/surface/models.ts

@@ -35,9 +35,9 @@ export async function listModels(
 ): Promise<{ active: string; models: ModelOption[] }> {
   const { config } = runtime;
   // Resolve the chat's *own* backend so the model list tracks whatever
-  // backend the chat is currently bound to (fixes the list staying on the
-  // previous backend's models after a switch). Fall back to the global
-  // default backend when there's no chat / the pool isn't ready yet.
+  // backend the chat is currently bound to, not the one before a switch.
+  // Fall back to the global default backend when there's no chat / the
+  // pool isn't ready yet.
   let backendId: string = config.backend;
   let active = config.model;
   if (chatId) {

package/src/frontend/native/surface/settings.ts

@@ -186,19 +186,31 @@ export function applyConfigUpdate(
  * Merge a partial update into talon.json on disk, preserving everything
  * else. Shared by every bridge surface that persists config changes
  * (settings sync here, plugin toggles in extensions.ts).
+ *
+ * An existing file that can't be read or isn't a JSON object throws and
+ * is left untouched: merging into `{}` would replace the operator's whole
+ * config (tokens included) with just this patch.
  */
 export function persistConfigPatch(update: Record<string, unknown>): void {
   const file = pathFiles.config;
   let current: Record<string, unknown> = {};
-  try {
-    if (existsSync(file)) {
-      current = JSON.parse(readFileSync(file, "utf-8")) as Record<
-        string,
-        unknown
-      >;
+  if (existsSync(file)) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(readFileSync(file, "utf-8"));
+    } catch (err) {
+      throw new Error(
+        `Refusing to write ${file}: cannot read the existing file ` +
+          `(${err instanceof Error ? err.message : err}). It was left untouched.`,
+      );
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      throw new Error(
+        `Refusing to write ${file}: the top level is not a JSON object. ` +
+          `It was left untouched.`,
+      );
     }
-  } catch {
-    /* corrupt/absent — start from empty */
+    current = parsed as Record<string, unknown>;
   }
   for (const [k, v] of Object.entries(update)) {
     if (v === undefined) delete current[k];

package/src/frontend/native/turn/context.ts

@@ -88,14 +88,12 @@ export async function refreshContext(
 /**
  * Fill the context cache for chats restored at startup.
  *
- * The readout is served from an in-memory map that was only ever written at
- * turn end, so after a restart every existing chat reported no context at
- * all — the header chip vanished until that chat ran another turn, which
- * read as "context usage doesn't save". The numbers themselves are
- * persisted with the session; only the cache was cold. This re-reads them
- * for the most recently active chats (bounded, and on the cheap path that
- * never touches the backend pool); anything it skips or can't resolve is
- * filled in the moment the chat is opened.
+ * The readout is served from an in-memory map otherwise written only at
+ * turn end, so after a restart every chat would show no context until its
+ * next turn, though the numbers are persisted with the session. This
+ * re-reads them for the most recently active chats (bounded, and on the
+ * cheap path that never touches the backend pool); anything it skips or
+ * can't resolve is filled in the moment the chat is opened.
  */
 const CONTEXT_WARM_LIMIT = 40;
 

package/src/frontend/native/turn/turn-meta.ts

@@ -2,11 +2,8 @@
  * Turn-meta sidecar — thin native-frontend façade over the shared
  * storage/turn-meta store. It owns the TurnMeta *shape* (tool calls,
  * duration, token usage — the companion app's tool timeline and stats
- * footer); the storage layer treats each record as opaque JSON.
- *
- * The data used to live in a debounced JSON file under the data dir;
- * it now rides the `turn_meta` SQLite table shared by every frontend.
- * SQLite commits on every write, so the old flush timer is gone.
+ * footer); the storage layer treats each record as opaque JSON in the
+ * `turn_meta` SQLite table shared by every frontend.
  */
 
 import {