talon-agent 3.34.0 → 3.35.0

package/src/backend/remote-server/turn.ts

@@ -69,8 +69,22 @@ export interface RunRemoteTurnInputs {
   onStreamDelta?: (accumulated: string, phase?: "thinking" | "text") => void;
   onTextBlock?: (text: string) => Promise<void>;
   onToolUse?: (toolName: string, input: Record<string, unknown>) => void;
+  /**
+   * Fires when the backend is stopped underneath the turn (hot-swap or
+   * shutdown). Its `reason` is the error the turn rejects with.
+   */
+  stopSignal?: AbortSignal;
 }
 
+/** Poll cadence for the headless question/permission watchdog. */
+const WATCHDOG_INTERVAL_MS = 350;
+/**
+ * Consecutive poll failures before the watchdog gives up. A server that
+ * refuses three polls in a row is gone; polling on would only log
+ * `fetch failed` every 350ms until the turn is torn down.
+ */
+const WATCHDOG_MAX_CONSECUTIVE_FAILURES = 3;
+
 /**
  * Run one turn end-to-end.
  *
@@ -105,6 +119,7 @@ export async function runRemoteTurn(
     onStreamDelta,
     onTextBlock,
     onToolUse,
+    stopSignal,
   } = inputs;
   const sessionClient = oc as unknown as RemoteSessionClient;
 
@@ -112,6 +127,10 @@ export async function runRemoteTurn(
   // `message.part.updated` events can fire immediately after promptAsync
   // returns, so the iterator must already be alive.
   const sseAbort = new AbortController();
+  // A backend stop ends the SSE loop and the watchdog the same way a
+  // finished turn does; the turn itself rejects via `rejectWhenStopped`.
+  const onStop = (): void => sseAbort.abort();
+  stopSignal?.addEventListener("abort", onStop, { once: true });
   const sseDone = subscribeToTurnEvents({
     label,
     oc,
@@ -144,35 +163,30 @@ export async function runRemoteTurn(
         label,
       ),
     ]);
-  const questionWatchdog = (async () => {
-    while (!sseAbort.signal.aborted) {
-      try {
-        await settlePending();
-      } catch (err) {
-        logWarn(
-          "agent",
-          `[${chatId}] question watchdog failed: ${errMsg(err)}`,
-        );
-      }
-      await sleep(350, sseAbort.signal);
-    }
-  })();
+  const questionWatchdog = runQuestionWatchdog({
+    settlePending,
+    signal: sseAbort.signal,
+    chatId,
+  });
 
   try {
     // Fire and forget — promptAsync returns immediately. The await below
     // is on the SSE close event. Per-prompt overrides hide sibling chats'
     // MCP tools while session permissions independently deny execution.
     await awaitRemoteTurn(
-      (async () => {
-        await oc.session.promptAsync({
-          sessionID: sessionId,
-          parts: [{ type: "text", text: prompt }],
-          model: { providerID, modelID },
-          system: systemPrompt,
-          ...(toolOverrides ? { tools: toolOverrides } : {}),
-        });
-        await sseDone;
-      })(),
+      rejectWhenStopped(
+        (async () => {
+          await oc.session.promptAsync({
+            sessionID: sessionId,
+            parts: [{ type: "text", text: prompt }],
+            model: { providerID, modelID },
+            system: systemPrompt,
+            ...(toolOverrides ? { tools: toolOverrides } : {}),
+          });
+          await sseDone;
+        })(),
+        stopSignal,
+      ),
       { client: oc, sessionId, chatId, label },
     );
 
@@ -214,6 +228,7 @@ export async function runRemoteTurn(
     }
     throw err;
   } finally {
+    stopSignal?.removeEventListener("abort", onStop);
     sseAbort.abort();
     // A dead SSE socket may ignore the local abort flag until another event
     // arrives. Bound cleanup so a timed-out turn cannot wedge its caller in
@@ -228,6 +243,61 @@ export async function runRemoteTurn(
   }
 }
 
+/**
+ * Race the turn against a backend stop. A stop rejects with the signal's
+ * `reason` (a `RemoteServerStoppedError`) the moment it fires, instead of
+ * waiting for an SSE socket that will never close on its own.
+ */
+function rejectWhenStopped<T>(
+  turn: Promise<T>,
+  stopSignal: AbortSignal | undefined,
+): Promise<T> {
+  if (!stopSignal) return turn;
+  if (stopSignal.aborted) return Promise.reject(stopSignal.reason);
+  return new Promise<T>((resolve, reject) => {
+    const onAbort = (): void => reject(stopSignal.reason);
+    stopSignal.addEventListener("abort", onAbort, { once: true });
+    turn.then(resolve, reject).finally(() => {
+      stopSignal.removeEventListener("abort", onAbort);
+    });
+  });
+}
+
+/**
+ * Poll the pending question/permission lists until the turn's signal
+ * fires. Gives up after {@link WATCHDOG_MAX_CONSECUTIVE_FAILURES} failed
+ * polls in a row — a server that stopped answering is not coming back
+ * within this turn, and the finally-block settle runs once more anyway.
+ */
+async function runQuestionWatchdog(inputs: {
+  settlePending: () => Promise<unknown>;
+  signal: AbortSignal;
+  chatId: string;
+}): Promise<void> {
+  const { settlePending, signal, chatId } = inputs;
+  let consecutiveFailures = 0;
+  while (!signal.aborted) {
+    try {
+      await settlePending();
+      consecutiveFailures = 0;
+    } catch (err) {
+      consecutiveFailures += 1;
+      logWarn(
+        "agent",
+        `[${chatId}] question watchdog failed (${consecutiveFailures}/${WATCHDOG_MAX_CONSECUTIVE_FAILURES}): ${errMsg(err)}`,
+      );
+      if (consecutiveFailures >= WATCHDOG_MAX_CONSECUTIVE_FAILURES) {
+        logWarn(
+          "agent",
+          `[${chatId}] question watchdog stopped: server unreachable`,
+        );
+        return;
+      }
+    }
+    await sleep(WATCHDOG_INTERVAL_MS, signal);
+  }
+}
+
 interface SubscribeInputs {
   label: string;
   oc: RemoteTurnClient;

package/src/backend/shared/cache-telemetry.ts

@@ -249,10 +249,23 @@ function estimateTokens(text: string): number {
   return Math.ceil(text.length / 4);
 }
 
+/**
+ * (label, model) pairs already warned about this process. A recurring
+ * caller (heartbeat, dream) builds the same prompt on every run, so the
+ * first warning says everything the rest would.
+ */
+const warnedCacheMinimums = new Set<string>();
+
+/** Test seam. */
+export function resetCacheMinimumWarnings(): void {
+  warnedCacheMinimums.clear();
+}
+
 /**
  * Warn when a prompt is too small to be cacheable on its model. No-op when
  * the model's floor is unknown or the prompt clears it. `label` names the
- * caller (e.g. `"dream"`) so the warning points somewhere.
+ * caller (e.g. `"dream"`) so the warning points somewhere. Warns once per
+ * (label, model) per process.
  */
 export function warnIfBelowCacheMinimum(
   label: string,
@@ -263,6 +276,9 @@ export function warnIfBelowCacheMinimum(
   if (min === undefined) return;
   const estimated = estimateTokens(prompt);
   if (estimated >= min) return;
+  const key = `${label}\0${model}`;
+  if (warnedCacheMinimums.has(key)) return;
+  warnedCacheMinimums.add(key);
   logWarn(
     "agent",
     `[${label}] prompt ~${estimated} tokens is below ${model}'s ${min}-token ` +

package/src/backend/shared/handler-to-events.ts

@@ -23,6 +23,7 @@ import {
 import { classify } from "../../core/errors.js";
 import type { ChatRunParams } from "../../core/agent-runtime/capabilities.js";
 import type { QueryParams, QueryResult } from "./handler-types.js";
+import { buildResultEvents } from "./result-events.js";
 
 const SENTINEL = Symbol("handler-to-events:sentinel");
 
@@ -205,21 +206,15 @@ export async function* handlerToEvents(
     return;
   }
 
-  const usage = {
-    inputTokens: result.inputTokens,
-    outputTokens: result.outputTokens,
-    cacheRead: result.cacheRead,
-    cacheWrite: result.cacheWrite,
-    modelId: params.model.id,
-  };
-  yield { type: "usage", usage };
-  yield {
-    type: "completed",
-    result: {
-      text: result.text,
-      durationMs: result.durationMs,
-      usage,
-      modelId: params.model.id,
+  yield* buildResultEvents({
+    text: result.text,
+    durationMs: result.durationMs,
+    usage: {
+      inputTokens: result.inputTokens,
+      outputTokens: result.outputTokens,
+      cacheRead: result.cacheRead,
+      cacheWrite: result.cacheWrite,
     },
-  };
+    modelId: params.model.id,
+  });
 }

package/src/backend/shared/index.ts

@@ -24,6 +24,8 @@
  *   - `stream-state` — backend-agnostic accumulator for stream loops.
  *   - `turn-interrupt` — user-driven mid-turn interrupt registry (the
  *     shared `ChatBackend.interruptChatTurn` for callback backends).
+ *   - `turn-phases` — the post-stream phases (accounting, session name,
+ *     trailing-prose contract, result tail) every handler runs.
  *
  * What's NOT here (intentionally):
  *   - SDK-specific event types — those live in each backend.
@@ -34,11 +36,6 @@
 
 export { captureDeliveredText } from "./delivered-text.js";
 
-export {
-  FLOW_VIOLATION_MAX_RETRIES,
-  detectFlowViolation,
-} from "./flow-violation.js";
-
 export { registerTurnInterrupt } from "./turn-interrupt.js";
 
 export { formatUserPrompt } from "./prompt-format.js";
@@ -49,8 +46,6 @@ export {
   buildFirstTurnReminder,
 } from "./delivery-contract.js";
 
-export { extractSessionName } from "../../util/session-name.js";
-
 export { summarizeUsage } from "./usage.js";
 
 // Only what is consumed THROUGH the barrel. Everything else in
@@ -66,8 +61,6 @@ export {
 
 export { prepareSystemPrompt, appendBackendSuffix } from "./system-prompt.js";
 
-export { classifyRetry } from "./model-retry.js";
-
 export {
   createStreamState,
   appendText,
@@ -88,11 +81,17 @@ export {
 
 export { sleep } from "./sleep.js";
 
-export {
-  recordToolCall,
-  recordTurnMetrics,
-  recordFailedTurnAccounting,
-  recordFlowViolation,
-} from "./metrics.js";
+export { recordToolCall } from "./metrics.js";
 
 export { applyRetryDecision } from "./handle-retry.js";
+
+export {
+  accountTurn,
+  accountFailedTurn,
+  nameSessionFromFirstMessage,
+  enforceTrailingProse,
+  finishCallbackTurn,
+  turnUsageSnapshot,
+} from "./turn-phases.js";
+
+export { buildResultEvents } from "./result-events.js";

package/src/backend/shared/result-events.ts

@@ -0,0 +1,30 @@
+/**
+ * The `usage` + `completed` pair that closes every successful
+ * `runChatTurn` stream, whether the backend emits events natively
+ * (Claude SDK) or through `handlerToEvents`. Kept free of storage and
+ * logging imports so the adapter stays a pure event translator.
+ */
+
+import type { AgentEvent } from "../../core/agent-runtime/events.js";
+import type { TokenUsageSnapshot } from "./usage.js";
+
+export function buildResultEvents(inputs: {
+  text: string;
+  durationMs: number;
+  usage: TokenUsageSnapshot;
+  modelId: string;
+}): [AgentEvent, AgentEvent] {
+  const usage = { ...inputs.usage, modelId: inputs.modelId };
+  return [
+    { type: "usage", usage },
+    {
+      type: "completed",
+      result: {
+        text: inputs.text,
+        durationMs: inputs.durationMs,
+        usage,
+        modelId: inputs.modelId,
+      },
+    },
+  ];
+}

package/src/backend/shared/turn-phases.ts

@@ -0,0 +1,277 @@
+/**
+ * The phases every chat turn runs once its SDK stream loop has ended.
+ *
+ * Each backend handler owns its options build, its `for await` loop and
+ * its event translation — everything that depends on the SDK's shape.
+ * What comes after is the same four steps in every backend:
+ *
+ *   1. `accountTurn` — per-turn metrics, session id, session usage
+ *      (or `accountFailedTurn` on the terminal-failure path).
+ *   2. `nameSessionFromFirstMessage` — the session title.
+ *   3. `enforceTrailingProse` — the tool-only delivery contract and the
+ *      flow-violation re-prompt decision.
+ *   4. `finishCallbackTurn` — the delivery summary log lines and the
+ *      `QueryResult` (the event-stream tail is `result-events.ts`).
+ *
+ * The functions take the stream state and explicit config bits and return
+ * explicit results; the handler decides what to do with a retry decision
+ * because recursion is the handler's own entry point.
+ */
+
+import {
+  getSession,
+  recordUsage,
+  setSessionId,
+  setSessionName,
+} from "../../storage/sessions.js";
+import { log } from "../../util/log.js";
+import { extractSessionName } from "../../util/session-name.js";
+import { traceMessage } from "../../util/trace.js";
+import {
+  FLOW_VIOLATION_MAX_RETRIES,
+  detectFlowViolation,
+  type FlowViolationResult,
+} from "./flow-violation.js";
+import type { QueryResult } from "./handler-types.js";
+import {
+  recordFailedTurnAccounting,
+  recordFlowViolation,
+  recordTurnMetrics,
+} from "./metrics.js";
+import type { StreamState } from "./stream-state.js";
+import { summarizeUsage, type TokenUsageSnapshot } from "./usage.js";
+
+// ── Usage snapshot ──────────────────────────────────────────────────────────
+
+/** The slice of a stream state the post-loop phases read. */
+export type TurnUsageState = Pick<
+  StreamState,
+  | "sdkInputTokens"
+  | "sdkOutputTokens"
+  | "sdkCacheRead"
+  | "sdkCacheWrite"
+  | "toolCalls"
+  | "numApiCalls"
+  | "contextTokens"
+  | "contextWindow"
+>;
+
+/** The turn's token totals as the metrics + session layers consume them. */
+export function turnUsageSnapshot(state: TurnUsageState): TokenUsageSnapshot {
+  return {
+    inputTokens: state.sdkInputTokens,
+    outputTokens: state.sdkOutputTokens,
+    cacheRead: state.sdkCacheRead,
+    cacheWrite: state.sdkCacheWrite,
+  };
+}
+
+// ── Phase 1: accounting ─────────────────────────────────────────────────────
+
+/** Context-fill fields persisted alongside the token totals. */
+type TurnContextUsage = {
+  contextTokens?: number;
+  contextWindow?: number;
+  numApiCalls?: number;
+  costUsd?: number;
+};
+
+export type AccountTurnInputs = {
+  chatId: string;
+  /** Backend id — the `backend.<id>.*` metric dimension. */
+  backend: string;
+  state: TurnUsageState;
+  durationMs: number;
+  model: string;
+  /** Provider session id to persist; skipped when absent or unchanged. */
+  sessionId?: string;
+  /** True when the turn ended in a delivered failure (Codex `turn.failed`). */
+  failed?: boolean;
+  /** Override for backends that count tool calls outside the stream state. */
+  toolCalls?: number;
+  /**
+   * Context-fill fields for `recordUsage`. Only backends whose stream
+   * reports them pass this; the others leave the session's context
+   * display untouched (zeroed) as they always have.
+   */
+  context?: TurnContextUsage;
+};
+
+/**
+ * Record the per-turn metric rollup, persist the provider session id and
+ * fold the turn's usage into the session. Each write is independent, so
+ * the order here is not load-bearing.
+ */
+export function accountTurn(inputs: AccountTurnInputs): void {
+  const { chatId, state, durationMs } = inputs;
+  const usage = turnUsageSnapshot(state);
+  recordTurnMetrics({
+    chatId,
+    backend: inputs.backend,
+    durationMs,
+    toolCalls: inputs.toolCalls ?? state.toolCalls,
+    apiCalls: state.numApiCalls,
+    ...(inputs.failed !== undefined ? { failed: inputs.failed } : {}),
+    usage,
+  });
+  persistSessionId(chatId, inputs.sessionId);
+  recordUsage(chatId, {
+    ...usage,
+    durationMs,
+    model: inputs.model,
+    ...inputs.context,
+  });
+}
+
+function persistSessionId(chatId: string, sessionId: string | undefined): void {
+  if (!sessionId) return;
+  if (getSession(chatId).sessionId === sessionId) return;
+  setSessionId(chatId, sessionId);
+}
+
+export type AccountFailedTurnInputs = {
+  chatId: string;
+  backend: string;
+  state: TurnUsageState;
+  durationMs: number;
+  model: string;
+  /** Overrides for backends that count outside the stream state. */
+  toolCalls?: number;
+  apiCalls?: number;
+  usage?: TokenUsageSnapshot;
+};
+
+/**
+ * Terminal-failure accounting from the stream state — the tokens a
+ * failed turn burned still count. Not for retry paths: the recursive
+ * attempt accounts for itself.
+ */
+export function accountFailedTurn(inputs: AccountFailedTurnInputs): void {
+  const { state } = inputs;
+  recordFailedTurnAccounting({
+    backend: inputs.backend,
+    chatId: inputs.chatId,
+    durationMs: inputs.durationMs,
+    toolCalls: inputs.toolCalls ?? state.toolCalls,
+    apiCalls: inputs.apiCalls ?? state.numApiCalls,
+    model: inputs.model,
+    usage: inputs.usage ?? turnUsageSnapshot(state),
+    contextTokens: state.contextTokens,
+    contextWindow: state.contextWindow,
+  });
+}
+
+// ── Phase 2: session name ───────────────────────────────────────────────────
+
+/**
+ * Title the session from the user's first message. Skipped on retries,
+ * whose `text` is a synthetic reminder rather than what the user said.
+ */
+export function nameSessionFromFirstMessage(inputs: {
+  chatId: string;
+  text: string;
+  previousTurns: number;
+  isRetry?: boolean;
+}): void {
+  if (inputs.previousTurns !== 0 || inputs.isRetry) return;
+  const name = extractSessionName(inputs.text);
+  if (name) setSessionName(inputs.chatId, name);
+}
+
+// ── Phase 3: trailing-prose contract ────────────────────────────────────────
+
+export type TrailingProseInputs = {
+  chatId: string;
+  state: Pick<
+    StreamState,
+    "lastTrailingText" | "turnTerminated" | "deliveredTextNorms" | "toolCalls"
+  >;
+  /** Synthetic flow-violation retries already spent on this message. */
+  flowRetries: number;
+  /** Frontend-aware reminder; omit for the default telegram-shaped text. */
+  reminder?: string;
+};
+
+/**
+ * Apply the tool-only delivery contract to a finished turn: detect a
+ * flow violation, count it, log it, and say whether the handler should
+ * re-prompt with `reminder`. Callers gate this on the contract actually
+ * being in force (a messaging frontend with delivery tools registered).
+ */
+export function enforceTrailingProse(
+  inputs: TrailingProseInputs,
+): FlowViolationResult {
+  const { chatId, state, flowRetries } = inputs;
+  const violation = detectFlowViolation({
+    trailingText: state.lastTrailingText,
+    turnTerminated: state.turnTerminated,
+    deliveredTextNorms: state.deliveredTextNorms,
+    toolCalls: state.toolCalls,
+    retried: flowRetries > 0,
+    retryCount: flowRetries,
+    maxRetries: FLOW_VIOLATION_MAX_RETRIES,
+    ...(inputs.reminder !== undefined ? { reminder: inputs.reminder } : {}),
+  });
+  if (!violation.violated) return violation;
+
+  recordFlowViolation(
+    chatId,
+    violation.shouldRetry ? "retried" : "cap_exhausted",
+  );
+  log(
+    "agent",
+    `[${chatId}] flow violation: ${violation.reason}. ${
+      violation.shouldRetry
+        ? "Re-prompting with reminder."
+        : `Retry cap (${FLOW_VIOLATION_MAX_RETRIES}) exhausted — accepting silent drop.`
+    }`,
+  );
+  return violation;
+}
+
+// ── Phase 4: result ─────────────────────────────────────────────────────────
+
+export type FinishCallbackTurnInputs = {
+  chatId: string;
+  state: TurnUsageState &
+    Pick<StreamState, "turnTerminated" | "deliveredTextNorms">;
+  responseText: string;
+  durationMs: number;
+  setupMs: number;
+  turnMs: number;
+  /** `routeDelivery`'s decision, or a backend's own route (`silent`). */
+  delivery: { route: string; chars: number };
+  /** Extra `key=value` diagnostics appended to the summary line. */
+  detail?: string;
+};
+
+/**
+ * The end-of-turn log lines and trace for a callback-shaped handler, and
+ * the `QueryResult` it returns.
+ */
+export function finishCallbackTurn(
+  inputs: FinishCallbackTurnInputs,
+): QueryResult {
+  const { chatId, state, responseText, durationMs, delivery } = inputs;
+  const usage = turnUsageSnapshot(state);
+  log(
+    "agent",
+    `[${chatId}] delivery: ${delivery.route} (${delivery.chars} chars)`,
+  );
+  log(
+    "agent",
+    `[${chatId}] -> (${summarizeUsage(usage, {
+      durationMs,
+      toolCalls: state.toolCalls,
+    })} terminator=${state.turnTerminated ? "yes" : "no"} ` +
+      `delivered=${state.deliveredTextNorms.length} ` +
+      `respLen=${responseText.length} ` +
+      `setup=${inputs.setupMs}ms turn=${inputs.turnMs}ms` +
+      `${inputs.detail ? ` ${inputs.detail}` : ""})`,
+  );
+  traceMessage(chatId, "out", responseText, {
+    durationMs,
+    toolCalls: state.toolCalls,
+  });
+  return { text: responseText, durationMs, ...usage };
+}

package/src/bootstrap.ts

@@ -28,6 +28,7 @@ import { initPulse, resetPulseTimer } from "./core/background/pulse.js";
 import { initCron } from "./core/background/cron.js";
 import { initPlanAlerts } from "./core/background/plan-alerts.js";
 import { setAdminNotifier } from "./core/notify.js";
+import { startAuthExpiryMonitor } from "./core/auth/expiry-monitor.js";
 import {
   initTriggers,
   resumeAfterRestart as resumeTriggersAfterRestart,
@@ -486,16 +487,7 @@ export async function initBackendAndDispatcher(
   // first consumer is WhatsApp pairing: codes must travel over a LIVE
   // frontend, not the dead one's log). Same delivery route as the plan
   // alerts above.
-  if (config.adminUserId) {
-    const adminChatId = config.adminUserId;
-    setAdminNotifier(async (text: string) =>
-      resolveFrontendByNumericId(
-        adminChatId,
-        String(adminChatId),
-        frontends,
-      ).sendMessage(adminChatId, text),
-    );
-  }
+  wireAdminNotifier(config, frontends);
 
   // Soul — initialize the identity kernel singleton from config so the prompt
   // injection / dream hooks see the right enabled state. Off by default; a
@@ -586,3 +578,25 @@ export async function initBackendAndDispatcher(
 
   return { backend };
 }
+
+/**
+ * Wire the admin notification seam to the admin's frontend, and start the
+ * login-expiry monitor that rides it: the CLIs' "N days to log in again"
+ * banner, delivered to the admin instead of a terminal nobody is watching
+ * (/auth then completes the sign-in from the chat).
+ */
+function wireAdminNotifier(
+  config: TalonConfig,
+  frontends: Parameters<typeof resolveFrontendByNumericId>[2],
+): void {
+  if (!config.adminUserId) return;
+  const adminChatId = config.adminUserId;
+  setAdminNotifier(async (text: string) =>
+    resolveFrontendByNumericId(
+      adminChatId,
+      String(adminChatId),
+      frontends,
+    ).sendMessage(adminChatId, text),
+  );
+  startAuthExpiryMonitor();
+}