talon-agent 5.18.2 → 5.19.0

package/src/core/engine/gateway-actions/agents/report.ts

@@ -16,6 +16,7 @@ import {
   agentIdFromContextLabel,
   agentRegistry,
   deliverMessage,
+  deliverToAgent,
   describeParent,
 } from "../../../agents/index.js";
 import { logError } from "../../../../util/log.js";
@@ -86,6 +87,67 @@ export const agentReportHandlers: SharedActionHandlers = {
     };
   },
 
+  list_peers: (_body, _chatId, _backend, chatKey) => {
+    const record = callingAgent(chatKey);
+    if (!record) return notAnAgent("list_peers");
+    const peers = agentRegistry.peersOf(record.id);
+    if (peers.length === 0) {
+      return {
+        ok: true,
+        text:
+          "No peers — you are the only agent your parent has running. " +
+          "Report to your parent as usual.",
+      };
+    }
+    const rendered = peers
+      .map(
+        (peer) =>
+          `- ${peer.label} [${peer.state}]\n  ID: ${peer.id}\n  Working on: ${peer.brief.slice(0, 160).replace(/\s+/g, " ")}…`,
+      )
+      .join("\n");
+    return {
+      ok: true,
+      text: `${peers.length} peer(s) running alongside you:\n\n${rendered}`,
+    };
+  },
+
+  message_peer: (body, _chatId, _backend, chatKey) => {
+    const record = callingAgent(chatKey);
+    if (!record) return notAnAgent("message_peer");
+    const id = String(body.agent_id ?? "").trim();
+    if (!id) return { ok: false, error: "Missing agent_id" };
+    const text = String(body.text ?? "").trim();
+    if (!text) return { ok: false, error: "Missing text" };
+    // Peers only. An agent may address a sibling, never an arbitrary id:
+    // resolving through peersOf is what stops one swarm reaching into
+    // another, and it means a wrong id fails closed rather than delivering.
+    const peer = agentRegistry
+      .peersOf(record.id)
+      .find((candidate) => candidate.id === id);
+    if (!peer) {
+      return {
+        ok: false,
+        error:
+          `No peer "${id}". You can only message agents spawned by the same ` +
+          `parent as you — call list_peers to see them.`,
+      };
+    }
+    if (!deliverToAgent(record.id, id, text)) {
+      return {
+        ok: false,
+        error:
+          `Could not deliver to "${peer.label}" (${id}): it has already ` +
+          `settled, or its inbox is full.`,
+      };
+    }
+    return {
+      ok: true,
+      text:
+        `Queued for peer "${peer.label}" (${id}). It reads its inbox at its ` +
+        `own milestones, so this is not an interrupt.`,
+    };
+  },
+
   check_inbox: (_body, _chatId, _backend, chatKey) => {
     const record = callingAgent(chatKey);
     if (!record) return notAnAgent("check_inbox");

package/src/core/engine/gateway-actions/history.ts

@@ -19,10 +19,8 @@ export const historyHandlers: SharedActionHandlers = {
     const limit = Math.min(100, Number(body.limit ?? 30));
     const cid = chatKey;
     // The tool schema advertises `offset_id` and `before` for paging back;
-    // this fallback used to ignore both and hand back the same newest
-    // window no matter what the model asked, so "go further back" was a
-    // silent no-op on frontends without a platform history API (WhatsApp,
-    // native).
+    // ignoring them here would make "go further back" a silent no-op on
+    // frontends without a platform history API (WhatsApp, native).
     const offsetId = Number(body.offset_id);
     if (Number.isFinite(offsetId) && offsetId > 0) {
       return { ok: true, text: getFormattedBefore(cid, offsetId, limit) };

package/src/core/engine/gateway-actions/native/exec.ts

@@ -61,23 +61,20 @@ async function bash(
     }
   }
 
-  const result = await (async () => {
-    if (background === true) {
-      if (active) {
-        return {
-          ok: false,
-          text:
-            "background:true runs on the daemon host only. On a teleported device, " +
-            "background it in-shell instead: `cmd > /tmp/out.log 2>&1 &`, then poll " +
-            "the log with read.",
-        };
-      }
-      return bashBackground(cmd, dir);
+  if (background === true) {
+    if (active) {
+      return {
+        ok: false,
+        text:
+          "background:true runs on the daemon host only. On a teleported device, " +
+          "background it in-shell instead: `cmd > /tmp/out.log 2>&1 &`, then poll " +
+          "the log with read.",
+      };
     }
-    if (active) return bashTeleported(chatId, active.deviceId, cmd, timeoutMs);
-    return bashLocal(cmd, dir, timeoutMs);
-  })();
-  return result;
+    return bashBackground(cmd, dir);
+  }
+  if (active) return bashTeleported(chatId, active.deviceId, cmd, timeoutMs);
+  return bashLocal(cmd, dir, timeoutMs);
 }
 
 function bashLocal(

package/src/core/engine/gateway.ts

@@ -14,8 +14,10 @@ import { classify } from "../errors.js";
 import { getActiveCount } from "./dispatcher.js";
 import { Loom, getActiveLoom, type ContextRegistry } from "../weaver/index.js";
 import { getHealthStatus } from "../../util/watchdog.js";
+import { activeAlerts } from "../frontend-runtime/alerts.js";
 import { getActiveSessionCount } from "../../storage/sessions.js";
 import { log, logError, logDebug } from "../../util/log.js";
+import { runInChatTurnScope } from "../../util/logging/turn-scope.js";
 import {
   handleSharedAction,
   handleChatFreeAction,
@@ -36,6 +38,38 @@ import type { FrontendActionHandler } from "../types.js";
 import type { Backend } from "../agent-runtime/capabilities.js";
 import { resolveOwnerFrontendId } from "../frontend-runtime/routing.js";
 
+/** Serialized size of an action result; -1 when it can't be serialized. */
+function actionResultBytes(result: unknown): number {
+  try {
+    return Buffer.byteLength(JSON.stringify(result) ?? "");
+  } catch {
+    return -1;
+  }
+}
+
+/** `tool.action name=… chat=… ms=… ok=… bytes=… [err=…]` for one bridge action. */
+function logActionOutcome(
+  body: Record<string, unknown>,
+  chat: string,
+  ms: number,
+  result: unknown,
+): void {
+  const action = typeof body.action === "string" ? body.action : "?";
+  const r = result as { ok?: unknown; error?: unknown } | null | undefined;
+  const failed = !!r && typeof r === "object" && r.ok === false;
+  const line =
+    `tool.action name=${action} chat=${chat || "-"} ms=${ms} ` +
+    `ok=${!failed} bytes=${actionResultBytes(result)}`;
+  if (!failed) {
+    logDebug("gateway", line);
+    return;
+  }
+  const err = String(r.error ?? "")
+    .replace(/\s+/g, " ")
+    .slice(0, 200);
+  log("gateway", `${line} err=${err}`);
+}
+
 // ── Retry helper (stateless — standalone export) ─────────────────────────────
 
 /**
@@ -290,7 +324,31 @@ export class Gateway {
     return null;
   }
 
-  private async handleAction(body: Record<string, unknown>): Promise<unknown> {
+  /**
+   * Run one bridge action inside the log scope of the turn that issued it
+   * (the MCP subprocess reaches us over HTTP, a fresh async root), so the
+   * action's own lines carry `turn=<id>`, and close it with one
+   * `tool.action` summary line — debug on success, info with the error
+   * text when the action answered `ok:false`.
+   */
+  private handleAction(body: Record<string, unknown>): Promise<unknown> {
+    const rawChatId = body._chatId ? String(body._chatId) : "";
+    const numericId = Number(rawChatId);
+    const chatKeys = [
+      rawChatId,
+      rawChatId !== "" && !isNaN(numericId)
+        ? this.loom.stringIdForNumeric(numericId)
+        : null,
+    ];
+    return runInChatTurnScope(chatKeys, async () => {
+      const t0 = Date.now();
+      const result = await this.routeAction(body);
+      logActionOutcome(body, rawChatId, Date.now() - t0, result);
+      return result;
+    });
+  }
+
+  private async routeAction(body: Record<string, unknown>): Promise<unknown> {
     // Route by _chatId from the MCP subprocess request.
     // _chatId may be a string (Teams: "teams_chat_19:...") or numeric string
     // (Telegram: "123456"). The context map is keyed by numeric chatId, so
@@ -492,6 +550,7 @@ export class Gateway {
         w.msSinceLastMessage < 60000
           ? "just now"
           : `${Math.round(w.msSinceLastMessage / 60000)}m ago`,
+      alerts: activeAlerts(),
     };
   }
 

package/src/core/engine/turn-health.ts

@@ -0,0 +1,222 @@
+/**
+ * Turn health — turns the per-turn outcomes the dispatcher sees into
+ * operator alerts.
+ *
+ * Every chat turn on every backend settles through `execute`, so this is
+ * the one place that can tell "one bad turn" from "this chat is broken"
+ * or "this backend is down":
+ *
+ *   - `backend.auth.<backend>` / `backend.quota.<backend>` — the backend
+ *     lost its login or ran out of quota. Raised on the first such
+ *     failure: nothing but a human (a re-login, a plan change, waiting
+ *     out a window the operator should know about) clears it.
+ *   - `turn.failing.<chatId>` — the chat's last 3 turns failed.
+ *   - `backend.failing.<backend>` — the backend failed turns in 3
+ *     different chats within 10 minutes.
+ *   - `dispatcher.stuck` — the watchdog saw work arrive and nothing
+ *     finish or make progress for 10 minutes (see util/watchdog.ts).
+ *
+ * Auth and quota failures carry their own alert, so they do not also
+ * count toward the two failure streaks. Every alert resolves on the next
+ * successful turn on the same chat / backend. A stopped turn is neither.
+ */
+
+import { AgentRunError } from "../agent-runtime/events.js";
+import { TalonError } from "../errors.js";
+import { raiseAlert, resolveAlert } from "../frontend-runtime/alerts.js";
+import { taskTable } from "../tasks/index.js";
+import { log, logWarn } from "../../util/log.js";
+import type { StuckLoopListener } from "../../util/watchdog.js";
+import { faultText } from "./fault-text.js";
+
+const CHAT_STREAK_THRESHOLD = 3;
+const BACKEND_CHAT_THRESHOLD = 3;
+const BACKEND_WINDOW_MS = 10 * 60_000;
+
+/**
+ * Login / credential failures across backends: Claude Code's "Not logged
+ * in · Please run /login", an expired OAuth token, Codex's refresh-token
+ * 401, Antigravity's "not authenticated", a bare 401/403.
+ */
+const AUTH_RE =
+  /not logged in|please run \/login|not (?:yet )?authenticated|authentication (?:required|failed|error)|unauthori[sz]ed|invalid (?:api[ _-]?key|x-api-key|bearer|token|credentials)|(?:oauth |access )?token (?:has )?expired|login expired|refresh_token_invalidated|failed to refresh token|permission_error|\b40[13]\b/i;
+
+/** Subscription / credit exhaustion — a limit a short retry won't clear. */
+const QUOTA_RE =
+  /usage limit|usage exhausted|you['’]ve hit your .{0,40}limit|out of (?:extra usage|credits)|insufficient_quota|quota (?:exceeded|exhausted)|exceeded your current quota|credit balance is too low|no remaining credits/i;
+
+type TurnFault = "auth" | "quota" | "error";
+
+export type TurnBinding = {
+  chatId: string;
+  backendId: string;
+  model?: string;
+  source?: string;
+  durationMs: number;
+};
+
+/** chatId → consecutive failed turns. */
+const chatStreaks = new Map<string, number>();
+/** backend → chatId → time of that chat's latest failure. */
+const backendFailures = new Map<string, Map<string, number>>();
+
+/** What kind of failure `err` is — null when it is not a fault at all. */
+function classifyTurnFault(err: unknown): TurnFault | null {
+  if (err instanceof TalonError) {
+    if (err.reason === "stopped") return null;
+    if (err.reason === "auth") return "auth";
+    if (err.reason === "usage_limit") return "quota";
+  }
+  if (err instanceof AgentRunError) {
+    if (err.kind === "aborted") return null;
+    if (err.kind === "auth") return "auth";
+  }
+  if (err instanceof Error && err.name === "AbortError") return null;
+  const text =
+    err instanceof AgentRunError
+      ? `${err.message} ${err.raw ?? ""}`
+      : err instanceof Error
+        ? err.message
+        : String(err);
+  if (QUOTA_RE.test(text)) return "quota";
+  if (AUTH_RE.test(text)) return "auth";
+  if (err instanceof AgentRunError && err.kind === "rate_limit") {
+    // A non-retryable rate-limit event is a usage limit, not a 429.
+    return err.retryable ? "error" : "quota";
+  }
+  return "error";
+}
+
+function describe(b: TurnBinding): string {
+  return b.model ? `${b.backendId}/${b.model}` : b.backendId;
+}
+
+/** Record a turn that completed. Clears every alert it can vouch for. */
+export function noteTurnSucceeded(b: TurnBinding): void {
+  const streak = chatStreaks.get(b.chatId);
+  if (streak) {
+    chatStreaks.delete(b.chatId);
+    log(
+      "dispatcher",
+      `turn.recovered chat=${b.chatId} backend=${b.backendId} after_failures=${streak} ms=${b.durationMs}`,
+    );
+    resolveAlert(
+      `turn.failing.${b.chatId}`,
+      `Chat ${b.chatId} is answering again (${describe(b)}).`,
+    );
+  }
+  if (backendFailures.delete(b.backendId)) {
+    resolveAlert(
+      `backend.failing.${b.backendId}`,
+      `The ${b.backendId} backend is completing turns again.`,
+    );
+  }
+  resolveAlert(
+    `backend.auth.${b.backendId}`,
+    `The ${b.backendId} backend is signed in again.`,
+  );
+  resolveAlert(
+    `backend.quota.${b.backendId}`,
+    `The ${b.backendId} backend has quota again.`,
+  );
+}
+
+/** Record a turn that threw. Raises once a threshold is crossed. */
+export function noteTurnFailed(b: TurnBinding, err: unknown): void {
+  const fault = classifyTurnFault(err);
+  if (fault === null) return;
+  const detail = faultText(err);
+  const prior = chatStreaks.get(b.chatId) ?? 0;
+  const count = fault === "error" ? prior + 1 : prior;
+  logWarn(
+    "dispatcher",
+    `turn.fail chat=${b.chatId} backend=${b.backendId} model=${b.model ?? "?"} ` +
+      `source=${b.source ?? "?"} cause=${fault} streak=${count} ` +
+      `ms=${b.durationMs} error="${detail}"`,
+  );
+  if (fault === "auth") {
+    raiseAlert(
+      `backend.auth.${b.backendId}`,
+      `The ${b.backendId} backend is not signed in: ${detail}. Every chat on it fails until it is re-authenticated.`,
+      { severity: "critical" },
+    );
+    return;
+  }
+  if (fault === "quota") {
+    raiseAlert(
+      `backend.quota.${b.backendId}`,
+      `The ${b.backendId} backend is out of quota: ${detail}. Chats on it fail until the limit resets.`,
+    );
+    return;
+  }
+  chatStreaks.set(b.chatId, count);
+  if (count >= CHAT_STREAK_THRESHOLD) {
+    raiseAlert(
+      `turn.failing.${b.chatId}`,
+      `Chat ${b.chatId} has failed its last ${count} turns on ${describe(b)}: ${detail}`,
+    );
+  }
+  noteBackendFailure(b, detail);
+}
+
+function noteBackendFailure(b: TurnBinding, detail: string): void {
+  const now = Date.now();
+  let chats = backendFailures.get(b.backendId);
+  if (!chats) {
+    chats = new Map();
+    backendFailures.set(b.backendId, chats);
+  }
+  chats.set(b.chatId, now);
+  for (const [chatId, at] of chats) {
+    if (now - at > BACKEND_WINDOW_MS) chats.delete(chatId);
+  }
+  if (chats.size >= BACKEND_CHAT_THRESHOLD) {
+    raiseAlert(
+      `backend.failing.${b.backendId}`,
+      `The ${b.backendId} backend failed turns in ${chats.size} chats in the last 10 min. Last error: ${detail}`,
+    );
+  }
+}
+
+/** The turns in flight, for the stuck-loop alert: what is wedged where. */
+function describeTurnQueue(now = Date.now()): string {
+  const turns = taskTable
+    .list()
+    .filter(
+      (t) =>
+        t.kind === "turn" && (t.state === "running" || t.state === "queued"),
+    );
+  if (turns.length === 0) return "No turn is running or queued.";
+  const running = turns
+    .filter((t) => t.state === "running")
+    .map((t) => {
+      const mins = Math.round((now - (t.startedAt ?? t.queuedAt)) / 60_000);
+      const on = t.backendId
+        ? ` on ${t.backendId}${t.model ? `/${t.model}` : ""}`
+        : "";
+      return `chat ${t.chatId ?? "?"}${on} (${mins} min)`;
+    });
+  const queued = turns.length - running.length;
+  const parts = [
+    running.length ? `Running: ${running.join(", ")}.` : "Nothing is running.",
+  ];
+  if (queued) parts.push(`${queued} turn(s) queued.`);
+  return parts.join(" ");
+}
+
+/** Watchdog hook: the stuck message loop as an operator alert. */
+export const stuckLoopAlert: StuckLoopListener = {
+  onStuck: ({ pendingMins, silentMins }) =>
+    raiseAlert(
+      "dispatcher.stuck",
+      `The message loop looks stuck: a message received ${pendingMins} min ago is unprocessed and no turn has made progress for ${silentMins} min. ${describeTurnQueue()}`,
+    ),
+  onRecovered: () =>
+    resolveAlert("dispatcher.stuck", "The message loop is processing again."),
+};
+
+/** Test seam: forget every streak. */
+export function resetTurnHealthForTest(): void {
+  chatStreaks.clear();
+  backendFailures.clear();
+}

package/src/core/errors.ts

@@ -247,8 +247,8 @@ export function classify(err: unknown): TalonError {
   }
 
   // 408 Request Timeout — a transient deadline like any other, but it
-  // is neither 4xx-terminal nor 5xx, so it used to fall through to
-  // `unknown`/non-retryable and strand the request.
+  // is neither 4xx-terminal nor 5xx, so without this branch it falls
+  // through to `unknown`/non-retryable and strands the request.
   if (status === 408) {
     return new TalonError(msg, {
       reason: "network",

package/src/core/frontend-runtime/admin-notify.ts

@@ -10,7 +10,7 @@
  *
  * First consumer: WhatsApp pairing. When WhatsApp unlinks the device,
  * recovery needs a human to type a pairing code into the phone — a code
- * that previously only ever appeared in the daemon log, which nobody
+ * that would otherwise only appear in the daemon log, which nobody
  * watches. Alerts about a dead frontend must travel over a LIVE one.
  */
 

package/src/core/frontend-runtime/alerts.ts

@@ -0,0 +1,130 @@
+/**
+ * Operator alerts — "something is wrong" (and "it's fine again") on the
+ * admin's chat, deduplicated so a flapping fault can't flood it.
+ *
+ * Every alert has a stable key naming the fault ("telegram.polling",
+ * "backend.auth", "disk.low"). The first raise delivers; repeats inside
+ * the cooldown only count, and the next delivery after it says how many
+ * were folded in. `resolveAlert` sends one recovery notice for a key that
+ * was delivered, and nothing for one that never was. Delivery rides
+ * `notifyAdmin`, so alerts degrade to a log line when no frontend is up.
+ *
+ * Two exceptions to the cooldown: an escalation (warn → error → critical)
+ * always delivers, and a delivery that failed (no notifier wired, the
+ * admin's frontend down) doesn't count, so the next raise tries again.
+ */
+
+import { log, logWarn } from "../../util/log.js";
+import { notifyAdmin } from "./admin-notify.js";
+
+export type AlertSeverity = "warn" | "error" | "critical";
+
+type ActiveAlert = {
+  severity: AlertSeverity;
+  message: string;
+  firstAt: number;
+  lastSentAt: number;
+  /** Raises folded into the cooldown since the last delivery. */
+  suppressed: number;
+};
+
+const DEFAULT_COOLDOWN_MS = 30 * 60_000;
+const ICON: Record<AlertSeverity, string> = {
+  warn: "⚠️",
+  error: "🔴",
+  critical: "🚨",
+};
+const RANK: Record<AlertSeverity, number> = { warn: 0, error: 1, critical: 2 };
+
+const active = new Map<string, ActiveAlert>();
+let cooldownMs = DEFAULT_COOLDOWN_MS;
+let enabled = true;
+let send: (text: string) => Promise<unknown> = notifyAdmin;
+
+/** Apply operator settings (config `alerts`). */
+export function configureAlerts(opts: {
+  enabled?: boolean;
+  cooldownMs?: number;
+}): void {
+  if (opts.enabled !== undefined) enabled = opts.enabled;
+  if (opts.cooldownMs !== undefined && opts.cooldownMs >= 0)
+    cooldownMs = opts.cooldownMs;
+}
+
+/**
+ * Report a fault. Always logged; delivered to the admin unless the same
+ * key was delivered within the cooldown. Never throws.
+ */
+export function raiseAlert(
+  key: string,
+  message: string,
+  opts: { severity?: AlertSeverity } = {},
+): void {
+  const severity = opts.severity ?? "error";
+  const now = Date.now();
+  logWarn("alert", `[${severity}] ${key}: ${message}`);
+  const prior = active.get(key);
+  const escalated =
+    prior !== undefined && RANK[severity] > RANK[prior.severity];
+  if (prior && !escalated && now - prior.lastSentAt < cooldownMs) {
+    prior.suppressed++;
+    prior.message = message;
+    return;
+  }
+  const folded = prior?.suppressed ?? 0;
+  const entry: ActiveAlert = {
+    severity,
+    message,
+    firstAt: prior?.firstAt ?? now,
+    lastSentAt: now,
+    suppressed: 0,
+  };
+  active.set(key, entry);
+  if (!enabled) return;
+  const repeat = folded > 0 ? `\n(+${folded} more since the last alert)` : "";
+  const undelivered = (): void => {
+    // Nobody heard it: don't let the cooldown swallow the next raise.
+    if (active.get(key) === entry) entry.lastSentAt = 0;
+  };
+  void send(`${ICON[severity]} ${message}${repeat}`).then((ok) => {
+    if (ok === false) undelivered();
+  }, undelivered);
+}
+
+/** Clear a fault; announces recovery only if its alert was delivered. */
+export function resolveAlert(key: string, message?: string): void {
+  const prior = active.get(key);
+  if (!prior) return;
+  active.delete(key);
+  const mins = Math.max(1, Math.round((Date.now() - prior.firstAt) / 60_000));
+  log("alert", `resolved ${key} after ${mins} min`);
+  if (!enabled) return;
+  void send(`✅ ${message ?? `Recovered: ${key}`} (after ${mins} min)`).catch(
+    () => {},
+  );
+}
+
+/** Keys currently raised — for status surfaces and doctor. */
+export function activeAlerts(): ReadonlyArray<{
+  key: string;
+  severity: AlertSeverity;
+  message: string;
+  since: number;
+}> {
+  return [...active].map(([key, a]) => ({
+    key,
+    severity: a.severity,
+    message: a.message,
+    since: a.firstAt,
+  }));
+}
+
+/** Test seam: reset state and swap the delivery function. */
+export function resetAlertsForTest(
+  deliver: (text: string) => Promise<unknown> = notifyAdmin,
+): void {
+  active.clear();
+  cooldownMs = DEFAULT_COOLDOWN_MS;
+  enabled = true;
+  send = deliver;
+}