talon-agent 5.18.2 → 5.19.1

package/src/core/weaver/weaver.ts

@@ -41,6 +41,19 @@ import type { Thread, ThreadSnapshot } from "./thread.js";
 import { startTurnCpu } from "./turn-cpu.js";
 import { startTypingLoop } from "./typing-loop.js";
 import { resolveWarp } from "./warp-resolver.js";
+import {
+  createTurnTrace,
+  logTurnEnd,
+  logTurnFailure,
+  logTurnSettled,
+  logTurnStart,
+  type TurnTrace,
+} from "./turn-log.js";
+import {
+  closeTurnScope,
+  createTurnScope,
+  runInTurnScope,
+} from "../../util/logging/turn-scope.js";
 
 export type WeaverDeps = {
   /**
@@ -84,6 +97,10 @@ export class Weaver {
     const chat = this.deps.getBackend(params.chatId).chat;
     const interrupt = chat?.interruptChatTurn?.bind(chat);
     const lifecycle = { started: false, killed: false, enqueuedAt: Date.now() };
+    // The turn id is minted at enqueue so a queued turn's wait is already
+    // attributable; the log scope goes live when the turn starts running.
+    const scope = createTurnScope(params.chatId);
+    const trace = createTurnTrace(scope.turnId, params, thread.inFlightCount);
     // Registered before enqueueing so a turn waiting in its chat's FIFO is
     // visible as `queued` in the task table, not invisible until it runs.
     const task = taskTable.enqueue({
@@ -99,7 +116,11 @@ export class Weaver {
           }
         : {}),
     });
-    return thread.enqueue(() => this.run(thread, params, task, lifecycle));
+    return thread.enqueue(() =>
+      runInTurnScope(scope, () =>
+        this.run(thread, params, task, lifecycle, trace),
+      ).finally(() => closeTurnScope(scope)),
+    );
   }
 
   /** Number of turns currently running (not queued) across all chats. */
@@ -121,21 +142,28 @@ export class Weaver {
     params: ExecuteParams,
     task: TaskHandle,
     lifecycle: { started: boolean; killed: boolean; enqueuedAt: number },
+    trace: TurnTrace,
   ): Promise<ExecuteResult> {
     if (lifecycle.killed) {
       // Killed while queued — the turn never reaches the backend. The
       // caller still gets a resolved (empty) result; nothing is delivered
       // to the chat, which is the point of the kill.
       task.fail(new Error("killed while queued"));
+      logTurnEnd(trace, "aborted", { reason: "killed-while-queued" });
       return this.emptyResult("Turn killed before it started.", params);
     }
     lifecycle.started = true;
+    trace.startedAt = Date.now();
     this.activeCount++;
     task.start();
     try {
-      const result = await this.executeInner(thread, params, task, {
-        queueWait: Date.now() - lifecycle.enqueuedAt,
-      });
+      const result = await this.executeInner(
+        thread,
+        params,
+        task,
+        { queueWait: Date.now() - lifecycle.enqueuedAt },
+        trace,
+      );
       const usage = {
         inputTokens: result.inputTokens,
         outputTokens: result.outputTokens,
@@ -152,9 +180,11 @@ export class Weaver {
       } else {
         task.succeed(usage);
       }
+      logTurnSettled(trace, lifecycle.killed, usage);
       return result;
     } catch (err) {
       task.fail(err);
+      logTurnFailure(trace, err, lifecycle.killed);
       if (lifecycle.killed) {
         // The backend didn't manage a clean interrupt-completion (some
         // SDK versions surface an interrupted turn as an error result).
@@ -168,6 +198,7 @@ export class Weaver {
       throw err;
     } finally {
       this.activeCount--;
+      trace.tools.close();
     }
   }
 
@@ -176,6 +207,7 @@ export class Weaver {
     params: ExecuteParams,
     task: TaskHandle,
     phases: Partial<Record<TurnPhase, number>>,
+    trace: TurnTrace,
   ): Promise<ExecuteResult> {
     const { context } = this.deps;
     const backend = this.deps.getBackend(params.chatId);
@@ -189,7 +221,12 @@ export class Weaver {
       reqId,
     });
     phases.warpResolve = Date.now() - warpStartedAt;
+    logTurnStart(trace, {
+      backendId: warp.backendId,
+      model: warp.ok ? warp.ref.id : undefined,
+    });
     if (!warp.ok) {
+      trace.refused = "no-model";
       await deliverRefusal(params, warp.message, "no-model");
       return this.emptyResult(warp.message, params);
     }
@@ -202,6 +239,7 @@ export class Weaver {
         "dispatcher",
         `[${reqId}] guest-scoped turn refused chat=${params.chatId}: backend "${backend.id}" cannot enforce the guest tool scope`,
       );
+      trace.refused = "guest-scope";
       await deliverRefusal(params, GUEST_BACKEND_REFUSAL, "guest-scope");
       return this.emptyResult(GUEST_BACKEND_REFUSAL, params);
     }
@@ -270,7 +308,12 @@ export class Weaver {
       // same path `phases.stream` is, so both cover one population.
       const stopCpu = startTurnCpu();
       const streamStartedAt = Date.now();
-      const agentResult = await carryTurnEvents(stream, params.onEvent, timing);
+      const agentResult = await carryTurnEvents(
+        stream,
+        params.onEvent,
+        timing,
+        trace.tools,
+      );
       phases.stream = Date.now() - streamStartedAt;
       stopCpu();
       phases.delivery = timing.deliveryMs;

package/src/frontend/discord/actions/index.ts

@@ -28,6 +28,12 @@ import { messagingHandlers, restoreScheduledMessages } from "./messaging.js";
 import { mediaHandlers } from "./media.js";
 import { chatInfoHandlers } from "./chat-info.js";
 import type { DiscordActionContext, DiscordActionHandlers } from "./types.js";
+import {
+  createDeliveryTracker,
+  trackDeliveries,
+} from "../../health/delivery.js";
+
+const delivery = createDeliveryTracker("discord", "Discord", "discord");
 
 // Null-prototype so a request `action` of "toString" / "constructor" / etc.
 // can't resolve an inherited Object.prototype method via `handlers[action]`.
@@ -44,7 +50,7 @@ export function createDiscordActionHandler(client: Client, gateway: Gateway) {
   // shutdown — the timers died with the process, the store didn't.
   restoreScheduledMessages(client, scheduledMessages);
 
-  return async (
+  const dispatch = async (
     body: Record<string, unknown>,
     chatId: number,
   ): Promise<ActionResult | null> => {
@@ -74,4 +80,5 @@ export function createDiscordActionHandler(client: Client, gateway: Gateway) {
     };
     return handler(body, chatId, ctx);
   };
+  return trackDeliveries(delivery, dispatch);
 }

package/src/frontend/discord/diagnostics.ts

@@ -1,12 +1,19 @@
 /**
- * Client diagnostics — the error/warn/disconnect/rate-limit listeners that
- * only log. discord.js handles reconnects and retries itself; these exist
- * so a silent failure mode (a fatal close code, an approaching IP ban) is
- * visible in the daemon log.
+ * Client diagnostics — the error/warn/disconnect/rate-limit listeners.
+ * discord.js handles reconnects and retries itself; these exist so a
+ * silent failure mode (a fatal close code, an approaching IP ban) is
+ * visible in the daemon log, and so the operator hears when the gateway
+ * stays down: `discord.gateway` is raised once a shard has been
+ * disconnected for `GATEWAY_OUTAGE_MS` (at once, critically, for a close
+ * code discord.js will never recover from) and resolved on ready/resume.
  */
 
 import { type Client, Events } from "discord.js";
-import { logError, logWarn } from "../../util/log.js";
+import { log, logError, logWarn } from "../../util/log.js";
+import { createOutage, errorText, type Outage } from "../health/outage.js";
+
+/** A gateway down this long reaches the operator. */
+const GATEWAY_OUTAGE_MS = 5 * 60_000;
 
 function shardDisconnectLabel(code: number | undefined): string {
   return code === 4004
@@ -18,7 +25,76 @@ function shardDisconnectLabel(code: number | undefined): string {
         : `code=${code}`;
 }
 
-export function bindClientDiagnostics(client: Client): void {
+function createGatewayOutage(thresholdMs: number): Outage {
+  return createOutage({
+    key: "discord.gateway",
+    thresholdMs,
+    describe: (err, mins) =>
+      `The Discord gateway has been disconnected for ${mins} min: ${err}. Messages are not being received.`,
+    recovered: "The Discord gateway is connected again.",
+  });
+}
+
+/**
+ * Gateway connection health: which shards are down, and the outage that
+ * spans them. Reconnect attempts, errors and recoveries are logged with
+ * the shard, attempt number and time down.
+ */
+function bindGatewayHealth(client: Client, thresholdMs: number): void {
+  const outage = createGatewayOutage(thresholdMs);
+  const down = new Set<number>();
+
+  const lost = (shardId: number, event: string, err: unknown): void => {
+    down.add(shardId);
+    const { attempt, downMs } = outage.fail(err);
+    logWarn(
+      "discord",
+      `gateway.${event} shard=${shardId} attempt=${attempt} down_ms=${downMs} err=${errorText(err)}`,
+    );
+  };
+  const back = (shardId: number, event: string): void => {
+    down.delete(shardId);
+    if (down.size > 0) return;
+    const ended = outage.ok();
+    if (ended) {
+      log(
+        "discord",
+        `gateway.${event} shard=${shardId} failed_attempts=${ended.attempts} down_ms=${ended.downMs}`,
+      );
+    }
+  };
+
+  client.on(Events.ShardReconnecting, (shardId) =>
+    lost(shardId, "reconnecting", "connection closed, reconnecting"),
+  );
+  client.on(Events.ShardError, (err, shardId) => {
+    // An error on a live shard is followed by a close if it matters; only
+    // an error during an outage is part of it (its text is the useful one).
+    if (down.has(shardId)) lost(shardId, "error", err);
+  });
+  client.on(Events.ShardDisconnect, (event, shardId) => {
+    const code = event?.code;
+    lost(shardId, "disconnect", shardDisconnectLabel(code));
+    if (code === 4004 || code === 4013 || code === 4014) {
+      outage.raiseNow(
+        `The Discord gateway closed and will not reconnect: ${shardDisconnectLabel(code)}. ` +
+          "Messages are not being received until this is fixed and Talon restarts.",
+        "critical",
+      );
+    }
+  });
+  client.on(Events.Invalidated, () =>
+    lost(0, "invalidated", "session invalidated"),
+  );
+  client.on(Events.ShardReady, (shardId) => back(shardId, "ready"));
+  client.on(Events.ShardResume, (shardId) => back(shardId, "resumed"));
+}
+
+export function bindClientDiagnostics(
+  client: Client,
+  gatewayOutageMs = GATEWAY_OUTAGE_MS,
+): void {
+  bindGatewayHealth(client, gatewayOutageMs);
   client.on("error", (err) => {
     logError("discord", "Client error", err);
   });

package/src/frontend/discord/handlers/index.ts

@@ -11,8 +11,6 @@
  *   - `delivery` — the agent run + reply pipeline (processAndReply)
  *   - `queue`    — per-chat debounce queue
  *   - `messages` — the messageCreate handler (handleMessage)
- *
- * Re-exports the same public surface the old single-file module exposed.
  */
 
 export { registerDiscordChat, lookupDiscordChat } from "./registry.js";

package/src/frontend/discord/middleware.ts

@@ -1,17 +1,11 @@
 /**
- * Discord middleware — wires up message events to handlers.ts.
+ * Discord middleware — wires up message events to handlers/.
  *
- * Equivalent to src/frontend/telegram/middleware.ts. We attach a single
- * messageCreate listener that filters out bots/system messages and delegates
- * to handleMessage. We also push every message into the in-memory history
- * buffer so /admin commands and /status reflect real activity.
- *
- * Discord-specific behavior:
- *  - We don't have a separate "my_chat_member" event; instead, when the bot
- *    is removed from a guild we get `guildDelete`. The handler in index.ts
- *    handles that to revoke access.
- *  - Every guild message gets the chat registered for pulse so periodic
- *    check-ins work (DMs are excluded because we always respond).
+ * A single messageCreate listener filters out bots/system messages and
+ * delegates to handleMessage. Every message is also pushed into the
+ * in-memory history buffer so /admin commands and /status reflect real
+ * activity, and every guild message registers its chat for pulse (DMs are
+ * excluded because we always respond).
  */
 
 import type { Client, Message } from "discord.js";
@@ -24,7 +18,7 @@ import { handleMessage, getSenderName } from "./handlers/index.js";
 
 export function registerMiddleware(client: Client, config: TalonConfig): void {
   client.on("messageCreate", (msg: Message) => {
-    // Ignore self/bots/system here too — handlers.ts will check again, but
+    // Ignore self/bots/system here too — handleMessage checks again, but
     // we don't even want to record those in history.
     if (msg.author.bot || msg.system) return;
     if (msg.author.id === client.user?.id) return;

package/src/frontend/discord/runtime.ts

@@ -1,9 +1,7 @@
 /**
  * Discord frontend runtime — the state every module of this frontend shares.
  *
- * `createDiscordFrontend` used to hold the client and config as closure
- * variables with every gateway-event handler nested inside `init`. It is
- * now one explicit object, constructed once, that each module (ready,
+ * One explicit object, constructed once, that each module (ready,
  * guild-policy, diagnostics, outbound, connection) takes as its first
  * parameter. The runtime carries state only; the modules own the behaviour.
  */

package/src/frontend/health/delivery.ts

@@ -0,0 +1,115 @@
+/**
+ * Reply delivery health — the worst silent failure is a turn that ran but
+ * whose answer never reached the chat. The model sees a failed tool call;
+ * the person who asked sees nothing, and so does the operator.
+ *
+ * Each frontend wraps its action handler with `trackDeliveries`. The
+ * reply actions (`send_message`, `send_message_with_buttons`, `reply_to`)
+ * are counted per chat: `DELIVERY_ALERT_AFTER` failures in a row for one
+ * chat raise `delivery.<frontend>` with that chat and the error, and the
+ * next success to it resolves the alert once no chat is still failing.
+ */
+
+import type { ActionResult } from "../../core/types.js";
+import {
+  raiseAlert,
+  resolveAlert,
+} from "../../core/frontend-runtime/alerts.js";
+import { log, logWarn, type LogComponent } from "../../util/log.js";
+import { errorText } from "./outage.js";
+
+/** Consecutive failed replies to one chat before the operator hears of it. */
+const DELIVERY_ALERT_AFTER = 3;
+
+const REPLY_ACTIONS = new Set([
+  "send_message",
+  "send_message_with_buttons",
+  "reply_to",
+]);
+
+export type DeliveryTracker = {
+  failed(chat: string | number, err: unknown): void;
+  delivered(chat: string | number): void;
+};
+
+/**
+ * `label` is the operator-facing platform name ("Telegram"); the alert key
+ * is `delivery.<frontend>`.
+ */
+export function createDeliveryTracker(
+  frontend: string,
+  label: string,
+  component: LogComponent,
+): DeliveryTracker {
+  const key = `delivery.${frontend}`;
+  const streaks = new Map<string, number>();
+  /** Chats that crossed the threshold and have not delivered since. */
+  const alerted = new Set<string>();
+
+  return {
+    failed(chat, err) {
+      const id = String(chat);
+      const streak = (streaks.get(id) ?? 0) + 1;
+      streaks.set(id, streak);
+      const error = errorText(err);
+      logWarn(
+        component,
+        `delivery.fail frontend=${frontend} chat=${id} streak=${streak} err=${error}`,
+      );
+      if (streak < DELIVERY_ALERT_AFTER) return;
+      alerted.add(id);
+      raiseAlert(
+        key,
+        `${label} replies to chat ${id} have failed ${streak} times in a row: ${error}. ` +
+          "Answers are not reaching that chat.",
+      );
+    },
+    delivered(chat) {
+      const id = String(chat);
+      const streak = streaks.get(id);
+      if (streak === undefined) return;
+      streaks.delete(id);
+      log(
+        component,
+        `delivery.recovered frontend=${frontend} chat=${id} after_failures=${streak}`,
+      );
+      if (alerted.delete(id) && alerted.size === 0) {
+        resolveAlert(key, `${label} replies are being delivered again.`);
+      }
+    },
+  };
+}
+
+type ActionHandler = (
+  body: Record<string, unknown>,
+  chatId: number,
+) => Promise<ActionResult | null>;
+
+/**
+ * Wrap a frontend action handler so its reply actions feed `tracker`.
+ * Results and thrown errors pass through untouched. `chatOf` names the
+ * destination when an action can target a chat other than `chatId`.
+ */
+export function trackDeliveries(
+  tracker: DeliveryTracker,
+  handler: ActionHandler,
+  chatOf: (body: Record<string, unknown>, chatId: number) => string | number = (
+    _body,
+    chatId,
+  ) => chatId,
+): ActionHandler {
+  return async (body, chatId) => {
+    if (!REPLY_ACTIONS.has(body.action as string)) return handler(body, chatId);
+    const chat = chatOf(body, chatId);
+    let result: ActionResult | null;
+    try {
+      result = await handler(body, chatId);
+    } catch (err) {
+      tracker.failed(chat, err);
+      throw err;
+    }
+    if (result?.ok) tracker.delivered(chat);
+    else if (result) tracker.failed(chat, result.error ?? "unknown error");
+    return result;
+  };
+}

package/src/frontend/health/outage.ts

@@ -0,0 +1,116 @@
+/**
+ * Connection outages — the line between "a reconnect failed" (log it) and
+ * "this chat channel is down" (tell the operator).
+ *
+ * A frontend reports every failed poll / reconnect with `fail` and every
+ * healthy one with `ok`. The first failure arms a timer; if nothing healthy
+ * arrives before `thresholdMs`, the alert is raised with the latest error.
+ * The timer — not the next failure — decides, because a link that dies
+ * silently (a gateway that never reconnects) produces no further events.
+ * `ok` ends the outage and, when an alert went out, sends the recovery.
+ */
+
+import {
+  raiseAlert,
+  resolveAlert,
+  type AlertSeverity,
+} from "../../core/frontend-runtime/alerts.js";
+
+export type OutageOptions = {
+  /** Stable alert key, e.g. "telegram.polling". */
+  key: string;
+  /** How long failures must persist before the operator hears of it. */
+  thresholdMs: number;
+  severity?: AlertSeverity;
+  /** Operator text for the raise, given the latest error and minutes down. */
+  describe: (lastError: string, downMin: number) => string;
+  /** Operator text for the recovery notice. */
+  recovered: string;
+};
+
+export type Outage = {
+  /** Record a failure. Returns the streak so the caller can log it. */
+  fail(err: unknown): { attempt: number; downMs: number };
+  /** Record a healthy round-trip. Returns the outage it ended, if any. */
+  ok(): { attempts: number; downMs: number } | null;
+  /** Raise immediately — for failures that will not heal on their own. */
+  raiseNow(message: string, severity?: AlertSeverity): void;
+  /** True between the first failure and the next `ok`. */
+  readonly down: boolean;
+  /** Forget the outage without resolving it — shutdown. */
+  dispose(): void;
+};
+
+const MAX_ERROR_CHARS = 200;
+
+/**
+ * One-line error text fit for an alert or a log line: the message only,
+ * bounded, with bot tokens and webhook signatures masked.
+ */
+export function errorText(err: unknown): string {
+  const raw = err instanceof Error ? err.message : String(err);
+  const text = raw
+    .replace(/bot\d+:[\w-]+/g, "bot<redacted>")
+    .replace(/([?&]sig=)[^&\s]+/g, "$1<redacted>")
+    .replace(/\s+/g, " ")
+    .trim();
+  return text.length > MAX_ERROR_CHARS
+    ? `${text.slice(0, MAX_ERROR_CHARS)}…`
+    : text;
+}
+
+export function createOutage(opts: OutageOptions): Outage {
+  let since = 0;
+  let attempts = 0;
+  let lastError = "";
+  let raised = false;
+  let timer: ReturnType<typeof setTimeout> | null = null;
+
+  const clearTimer = (): void => {
+    if (timer) clearTimeout(timer);
+    timer = null;
+  };
+  const raise = (message: string, severity?: AlertSeverity): void => {
+    raised = true;
+    raiseAlert(opts.key, message, { severity: severity ?? opts.severity });
+  };
+
+  return {
+    get down() {
+      return attempts > 0;
+    },
+    fail(err) {
+      const now = Date.now();
+      if (attempts === 0) {
+        since = now;
+        timer = setTimeout(() => {
+          timer = null;
+          const mins = Math.max(1, Math.round((Date.now() - since) / 60_000));
+          raise(opts.describe(lastError, mins));
+        }, opts.thresholdMs);
+        timer.unref?.();
+      }
+      attempts++;
+      lastError = errorText(err);
+      return { attempt: attempts, downMs: now - since };
+    },
+    ok() {
+      if (attempts === 0 && !raised) return null;
+      const ended = { attempts, downMs: attempts ? Date.now() - since : 0 };
+      clearTimer();
+      attempts = 0;
+      if (raised) resolveAlert(opts.key, opts.recovered);
+      raised = false;
+      return ended;
+    },
+    raiseNow(message, severity) {
+      clearTimer();
+      raise(message, severity);
+    },
+    dispose() {
+      clearTimer();
+      attempts = 0;
+      raised = false;
+    },
+  };
+}

package/src/frontend/native/bridge/routes/chats.ts

@@ -132,11 +132,9 @@ export function chatRoutes(
       const startedAt = Date.now();
       try {
         const attachment = await h.upload(filename, contentType, req);
-        // Uploads are the one client action whose failure used to be visible
-        // only in the app: the route answered in JSON and logged nothing, so
-        // "attaching a file doesn't work — check the logs" had nothing to
-        // read. Both outcomes are logged now, with the size and duration
-        // that distinguish a rejected upload from a dropped connection.
+        // Log both outcomes, with the size and duration that tell a rejected
+        // upload from a dropped connection — the JSON answer is otherwise
+        // only visible in the app.
         log(
           "native",
           `upload ok: ${attachment.name} (${attachment.size} bytes, ${attachment.mimeType}) in ${Date.now() - startedAt}ms`,