talon-agent 3.22.0 → 3.22.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "3.22.0",
+  "version": "3.22.1",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/src/app.ts

@@ -9,7 +9,7 @@
 import { getFrontends } from "./util/config.js";
 import { startUploadCleanup, stopUploadCleanup } from "./util/workspace.js";
 import { flushDatabase } from "./storage/db.js";
-import { getActiveCount } from "./core/engine/dispatcher.js";
+import { getActiveCount, stopAllTurns } from "./core/engine/dispatcher.js";
 import { startPulseTimer, stopPulseTimer } from "./core/background/pulse.js";
 import { stopPlanAlerts } from "./core/background/plan-alerts.js";
 import {
@@ -121,6 +121,7 @@ async function gracefulShutdown(signal: string): Promise<void> {
   shuttingDown = true;
   log("shutdown", `${signal} received, shutting down gracefully...`);
 
+  const deadlineAt = Date.now() + SHUTDOWN_TIMEOUT_MS;
   const forceTimer = setTimeout(() => {
     logError("shutdown", "Timeout exceeded, forcing exit");
     // Hand off even on the forced path. A restart must survive a
@@ -135,12 +136,17 @@ async function gracefulShutdown(signal: string): Promise<void> {
   }, SHUTDOWN_TIMEOUT_MS);
   forceTimer.unref();
 
-  // Drain in-flight queries: poll instead of a fixed sleep, so an idle
-  // daemon exits immediately and a busy one gets the full budget.
+  // Drain in-flight queries. A turn can legitimately run for minutes, so a
+  // drain that only waits can never succeed against one — ask every running
+  // turn to abort first, then poll for the aborts to settle so backends can
+  // flush partial state before the process exits.
   if (getActiveCount() > 0) {
+    const aborted = stopAllTurns();
     log(
       "shutdown",
-      `Waiting for ${getActiveCount()} in-flight queries to drain...`,
+      `Waiting for ${getActiveCount()} in-flight queries to drain` +
+        (aborted > 0 ? ` (abort requested for ${aborted})` : "") +
+        `...`,
     );
     const deadline = Date.now() + DRAIN_TIMEOUT_MS;
     while (getActiveCount() > 0 && Date.now() < deadline) {
@@ -177,7 +183,11 @@ async function gracefulShutdown(signal: string): Promise<void> {
   await shutdownStep("pulse timer", stopPulseTimer);
   await shutdownStep("heartbeat", async () => {
     stopHeartbeatTimer();
-    await awaitHeartbeat();
+    // Cap the wait by what's left of the force-timer budget (minus margin
+    // for the steps below). The default 10s wait plus a full 5s drain used
+    // to consume the entire 15s budget, so a slow heartbeat tripped the
+    // forced exit even though teardown was proceeding normally.
+    await awaitHeartbeat(Math.max(0, deadlineAt - Date.now() - 3_000));
   });
   await shutdownStep("cron timer", stopCronTimer);
   await shutdownStep("plan alerts", stopPlanAlerts);

package/src/backend/claude-sdk/handler.ts

@@ -67,8 +67,9 @@ import {
   recordFailedTurnAccounting,
   recordFlowViolation,
   formatTurnCache,
-  exceedsLookbackWindow,
-  estimateTurnBlocks,
+  crossTurnVerdict,
+  priorLookbackOverflow,
+  noteLookbackRisk,
   CACHE_LOOKBACK_BLOCKS,
 } from "../shared/index.js";
 
@@ -594,16 +595,24 @@ export async function* runChatTurn(
 
   // The aggregate `cache=NN%` can't distinguish a turn that reused the
   // previous turn's prefix from one that re-wrote it — see
-  // shared/cache-telemetry.ts. Append the cross-turn verdict when the
-  // provider gave us per-request usage to derive it from.
-  if (state.cacheStats && exceedsLookbackWindow(state.toolCalls)) {
-    logWarn(
-      "agent",
-      `[${chatId}] turn emitted ~${estimateTurnBlocks(state.toolCalls)} content ` +
-        `blocks (> ${CACHE_LOOKBACK_BLOCKS} lookback) — the next turn's cache ` +
-        `breakpoint may not find this turn's prefix`,
-    );
+  // shared/cache-telemetry.ts. A lookback overflow only *predicts* a miss,
+  // so warn when this turn's verdict proves the previous turn's overflow
+  // cost a prefix re-write, then record this turn's overflow for the next.
+  if (state.cacheStats) {
+    const overflow = priorLookbackOverflow(chatId);
+    if (
+      overflow !== undefined &&
+      crossTurnVerdict(state.cacheStats) === "miss"
+    ) {
+      logWarn(
+        "agent",
+        `[${chatId}] previous turn emitted ~${overflow} content blocks ` +
+          `(> ${CACHE_LOOKBACK_BLOCKS} lookback) and this turn's prefix ` +
+          `missed — that turn's cache write was likely never read`,
+      );
+    }
   }
+  noteLookbackRisk(chatId, state.toolCalls);
 
   log(
     "agent",

package/src/backend/remote-server/mcp.ts

@@ -29,7 +29,7 @@
  * registrations run concurrently through the hub; later turns skip them.
  */
 
-import { log, logWarn } from "../../util/log.js";
+import { log, logDebug, logWarn } from "../../util/log.js";
 import {
   talonHubUrl,
   pluginHubUrl,
@@ -62,6 +62,9 @@ export const TALON_PLUGIN_MCP_SERVER_NAME = "talon-plugin";
  */
 const SLOW_MCP_REGISTRATION_MS = 1000;
 
+/** Servers whose registration failure was already warned about (see below). */
+const warnedMcpRegistrationFailures = new Set<string>();
+
 /**
  * OpenCode's `/experimental/tool/ids` endpoint omits dynamically registered
  * MCP tools despite its API description. Synthesize Talon's known MCP ids so
@@ -253,6 +256,7 @@ export async function ensurePluginMcpServers<TClient extends RemoteAgentClient>(
         state.registeredMcpServers.add(serverName);
         state.registeredMcpTools.set(serverName, toolNames);
         byPlugin.set(name, serverName);
+        warnedMcpRegistrationFailures.delete(serverName);
         const ms = Date.now() - startedAt;
         log(
           "agent",
@@ -261,7 +265,16 @@ export async function ensurePluginMcpServers<TClient extends RemoteAgentClient>(
         );
         return serverName;
       } catch (err) {
-        logWarn(
+        // Warn once per server, then debug: registration re-runs at the
+        // head of every turn, so a dead backing service (an offline
+        // browser endpoint, say) would otherwise emit one warning per
+        // turn for the whole outage. Success clears the latch so the
+        // next outage warns again.
+        const level = warnedMcpRegistrationFailures.has(serverName)
+          ? logDebug
+          : logWarn;
+        warnedMcpRegistrationFailures.add(serverName);
+        level(
           "agent",
           `Plugin MCP registration failed for ${serverName}: ${errMsg(err)}`,
         );

package/src/backend/shared/cache-telemetry.ts

@@ -154,6 +154,48 @@ export function exceedsLookbackWindow(toolCalls: number): boolean {
   return estimateTurnBlocks(toolCalls) > CACHE_LOOKBACK_BLOCKS;
 }
 
+/**
+ * Per-chat estimated block count of the last turn that overflowed the
+ * lookback window. Overflow is only a *prediction* of a cache miss — the
+ * proof is the NEXT turn's cross-turn verdict, so the overflow is recorded
+ * here and the warning waits for that verdict instead of firing on every
+ * tool-heavy turn. Bounded like `lastToolSets` below.
+ */
+const lookbackOverflows = new Map<string, number>();
+
+/**
+ * Record whether this turn plausibly overflowed the lookback window, so the
+ * next turn can attribute a cross-turn miss to it.
+ */
+export function noteLookbackRisk(chatId: string, toolCalls: number): void {
+  if (!exceedsLookbackWindow(toolCalls)) {
+    lookbackOverflows.delete(chatId);
+    return;
+  }
+  if (
+    lookbackOverflows.size >= MAX_TRACKED_CHATS &&
+    !lookbackOverflows.has(chatId)
+  ) {
+    const oldest = lookbackOverflows.keys().next().value;
+    if (oldest !== undefined) lookbackOverflows.delete(oldest);
+  }
+  lookbackOverflows.set(chatId, estimateTurnBlocks(toolCalls));
+}
+
+/**
+ * Estimated block count of the chat's previous turn IF it overflowed the
+ * lookback window, else undefined. Read this before `noteLookbackRisk`
+ * records the current turn.
+ */
+export function priorLookbackOverflow(chatId: string): number | undefined {
+  return lookbackOverflows.get(chatId);
+}
+
+/** Drop all recorded overflows (tests / explicit reset). */
+export function resetLookbackRisk(): void {
+  lookbackOverflows.clear();
+}
+
 // ── Cacheable minimum ──────────────────────────────────────────────────────
 
 /**

package/src/backend/shared/index.ts

@@ -73,8 +73,9 @@ export {
 // barrel discipline the prompt/ barrel was just trimmed to.
 export {
   formatTurnCache,
-  estimateTurnBlocks,
-  exceedsLookbackWindow,
+  crossTurnVerdict,
+  priorLookbackOverflow,
+  noteLookbackRisk,
   CACHE_LOOKBACK_BLOCKS,
 } from "./cache-telemetry.js";
 

package/src/core/engine/dispatcher.ts

@@ -66,6 +66,16 @@ export function stopCurrentTurn(chatId: string): KillOutcome {
   return taskTable.killRunningTurn(chatId);
 }
 
+/**
+ * Request an abort of every chat's running turn. The shutdown drain calls
+ * this before polling `getActiveCount` — a turn can legitimately run for
+ * minutes, so a drain that only waits can never succeed against one.
+ * Returns the number of kill requests issued.
+ */
+export function stopAllTurns(): number {
+  return taskTable.killAllRunningTurns();
+}
+
 /**
  * Execute an AI query with full lifecycle management.
  * Same-chat queries are serialized (FIFO) to avoid session conflicts.

package/src/core/errors.ts

@@ -33,6 +33,17 @@ type ErrorReason =
 const USAGE_LIMIT_RE =
   /you['’]ve hit your .{0,40}limit|you['’]re out of extra usage|claude ai usage limit reached|usage limit reached/i;
 
+/**
+ * Ceiling on how long a failed attempt may have run and still earn the
+ * frontend queues' blind retry. `retryable` means "a short pause may clear
+ * it", which holds for a 429 or a dropped socket but not for an attempt
+ * that already burned minutes before failing (e.g. the 600s remote turn
+ * deadline — its rejection is `name: "TimeoutError"`, classified as
+ * transient network). Turns serialize per chat, so retrying such an
+ * attempt doubles the stall for every message queued behind it.
+ */
+export const RETRY_ELAPSED_CAP_MS = 120_000;
+
 // ── TalonError class ────────────────────────────────────────────────────────
 
 export class TalonError extends Error {

package/src/core/mcp-hub/children.ts

@@ -62,6 +62,31 @@ type ChildEntry = {
 const children = new Map<string, ChildEntry>();
 const inflight = new Map<string, Promise<ChildHandle>>();
 
+/**
+ * Negative cache for spawn failures. A child whose backing service is down
+ * (e.g. playwright-tools with its browser endpoint offline) dies at the
+ * connect handshake, and without this every single turn re-paid the
+ * spawn+handshake (~600ms) and re-logged the failure for the whole outage.
+ * Failures back off exponentially; the first attempt after the window
+ * clears the entry on success, so recovery costs one turn.
+ */
+type SpawnFailure = { at: number; count: number; error: unknown };
+const spawnFailures = new Map<string, SpawnFailure>();
+const FAILURE_BACKOFF_BASE_MS = 30_000;
+const FAILURE_BACKOFF_MAX_MS = 10 * 60_000;
+
+function failureBackoffMs(count: number): number {
+  return Math.min(
+    FAILURE_BACKOFF_BASE_MS * 2 ** (count - 1),
+    FAILURE_BACKOFF_MAX_MS,
+  );
+}
+
+/** Test seam: forget recorded spawn failures. */
+export function resetSpawnFailures(): void {
+  spawnFailures.clear();
+}
+
 /** Idle TTL for hub children; tunable for tests / tight deployments. */
 function idleTtlMs(): number {
   const raw = Number(process.env.TALON_MCP_HUB_IDLE_MS);
@@ -168,9 +193,28 @@ export function acquireChild(
   const pending = inflight.get(key);
   if (pending) return pending;
 
+  const failure = spawnFailures.get(key);
+  if (failure && Date.now() - failure.at < failureBackoffMs(failure.count)) {
+    return Promise.reject(
+      failure.error instanceof Error
+        ? failure.error
+        : new Error(String(failure.error)),
+    );
+  }
+
   const promise = (async () => {
     try {
-      return await spawnChild(key, spec());
+      const handle = await spawnChild(key, spec());
+      spawnFailures.delete(key);
+      return handle;
+    } catch (err) {
+      const prior = spawnFailures.get(key);
+      spawnFailures.set(key, {
+        at: Date.now(),
+        count: (prior?.count ?? 0) + 1,
+        error: err,
+      });
+      throw err;
     } finally {
       inflight.delete(key);
     }

package/src/core/tasks/table.ts

@@ -122,6 +122,20 @@ export class TaskTable {
     return { ok: true };
   }
 
+  /**
+   * Request an abort of every running turn, regardless of chat — the
+   * shutdown drain's lever. Returns the number of kill requests issued.
+   */
+  killAllRunningTurns(): number {
+    let killed = 0;
+    for (const [id, task] of this.live) {
+      if (task.record.kind === "turn" && task.record.state === "running") {
+        if (this.kill(id).ok) killed++;
+      }
+    }
+    return killed;
+  }
+
   /**
    * Request an abort for the turn currently running in one chat. Queued turns
    * are deliberately ignored: `/stop` means "stop what is happening now",

package/src/frontend/discord/handlers/queue.ts

@@ -4,10 +4,15 @@
  * notification.
  */
 
-import { classify, friendlyMessage } from "../../../core/errors.js";
+import {
+  classify,
+  friendlyMessage,
+  RETRY_ELAPSED_CAP_MS,
+} from "../../../core/errors.js";
 import {
   recordMessageProcessed,
   recordMessageReceived,
+  recordMessageSettled,
   recordError,
 } from "../../../util/watchdog.js";
 import { appendDailyLog } from "../../../storage/daily-log.js";
@@ -82,6 +87,7 @@ async function flushQueue(chatId: string): Promise<void> {
       chatTitle: last.chatTitle,
     });
 
+  const startedAt = Date.now();
   try {
     await runTurn();
     recordMessageProcessed();
@@ -94,7 +100,19 @@ async function flushQueue(chatId: string): Promise<void> {
     );
     recordError(classified.message);
 
-    if (classified.retryable) {
+    // Retry once for transient errors — but only when the failed attempt
+    // was actually brief. An attempt that already ran for minutes (a remote
+    // turn deadline) won't be saved by a 2s pause, and turns serialize per
+    // chat, so the blind retry used to double a 10-minute stall for
+    // everything queued behind it.
+    const attemptMs = Date.now() - startedAt;
+    if (classified.retryable && attemptMs >= RETRY_ELAPSED_CAP_MS) {
+      log(
+        "bot",
+        `[${chatId}] Not retrying ${classified.reason}: the attempt already ran ${Math.round(attemptMs / 1000)}s`,
+      );
+    }
+    if (classified.retryable && attemptMs < RETRY_ELAPSED_CAP_MS) {
       const delayMs = classified.retryAfterMs ?? 2000;
       log(
         "bot",
@@ -108,6 +126,7 @@ async function flushQueue(chatId: string): Promise<void> {
       } catch (retryErr) {
         const retryClassified = classify(retryErr);
         logError("bot", `[${chatId}] Retry failed: ${retryClassified.message}`);
+        recordMessageSettled();
         // Error-recovery send must never throw — if the network is fully down
         // the queue handler would otherwise propagate up and stall future
         // messages. Best-effort: notify if we can, log + move on otherwise.
@@ -126,6 +145,7 @@ async function flushQueue(chatId: string): Promise<void> {
         return;
       }
     }
+    recordMessageSettled();
     try {
       await sendChunked(
         last.channel,

package/src/frontend/telegram/actions/shared.ts

@@ -11,9 +11,31 @@ import { TELEGRAM_MAX_TEXT } from "./types.js";
 
 export function replyParams(
   body: Record<string, unknown>,
-): { message_id: number } | undefined {
-  const replyTo = toPositiveId(body.reply_to ?? body.reply_to_message_id);
-  return replyTo !== undefined ? { message_id: replyTo } : undefined;
+): ReplyParams | undefined {
+  return replyParamsFor(
+    toPositiveId(body.reply_to ?? body.reply_to_message_id),
+  );
+}
+
+export type ReplyParams = {
+  message_id: number;
+  allow_sending_without_reply: true;
+};
+
+/**
+ * Build `reply_parameters` for an outbound send. Always allows sending
+ * without the reply: the target message may have been deleted by the time
+ * we send (fast-moving groups, cleanup bots), and without this flag
+ * Telegram 400s the whole send — every formatting-level fallback then
+ * fails identically and the message is lost. A send that arrives
+ * un-linked beats one that never arrives.
+ */
+export function replyParamsFor(
+  replyTo: number | undefined,
+): ReplyParams | undefined {
+  return replyTo !== undefined && replyTo > 0
+    ? { message_id: replyTo, allow_sending_without_reply: true }
+    : undefined;
 }
 
 /** Delivery modifiers shared by every outbound send. */
@@ -125,7 +147,7 @@ export async function sendText(
         chatId,
         { markdown: text },
         {
-          reply_parameters: replyTo ? { message_id: replyTo } : undefined,
+          reply_parameters: replyParamsFor(replyTo),
           reply_markup: replyMarkup,
           ...opts,
         },
@@ -140,7 +162,7 @@ export async function sendText(
   try {
     const sent = await bot.api.sendMessage(chatId, html, {
       parse_mode: "HTML",
-      reply_parameters: replyTo ? { message_id: replyTo } : undefined,
+      reply_parameters: replyParamsFor(replyTo),
       reply_markup: replyMarkup,
       ...opts,
     });
@@ -151,7 +173,7 @@ export async function sendText(
       `Legacy HTML send failed; retrying as plain text (chat=${chatId}): ${err instanceof Error ? err.message : err}`,
     );
     const sent = await bot.api.sendMessage(chatId, text, {
-      reply_parameters: replyTo ? { message_id: replyTo } : undefined,
+      reply_parameters: replyParamsFor(replyTo),
       reply_markup: replyMarkup,
       ...opts,
     });

package/src/frontend/telegram/handlers/delivery.ts

@@ -10,7 +10,7 @@ import { toolInputToRecord } from "../../../core/agent-runtime/events.js";
 import { appendDailyLogResponse } from "../../../storage/daily-log.js";
 import { stripMcpPrefix } from "../../../core/tools/index.js";
 import { logWarn } from "../../../util/log.js";
-import { sendText } from "../actions/shared.js";
+import { replyParamsFor, sendText } from "../actions/shared.js";
 import { ambientThreadId } from "../topics.js";
 import { trackDmUser } from "./access.js";
 
@@ -22,7 +22,7 @@ export async function sendHtml(
 ): Promise<number> {
   const params = {
     parse_mode: "HTML" as const,
-    reply_parameters: replyToId ? { message_id: replyToId } : undefined,
+    reply_parameters: replyParamsFor(replyToId),
     message_thread_id: ambientThreadId(chatId),
   };
   try {
@@ -40,7 +40,7 @@ export async function sendHtml(
       plain = plain.replace(/<[^>]*>/g, "");
     } while (plain !== prev);
     const sent = await bot.api.sendMessage(chatId, plain, {
-      reply_parameters: replyToId ? { message_id: replyToId } : undefined,
+      reply_parameters: replyParamsFor(replyToId),
       message_thread_id: ambientThreadId(chatId),
     });
     return sent.message_id;

package/src/frontend/telegram/handlers/queue.ts

@@ -9,13 +9,18 @@
 import type { Bot } from "grammy";
 import type { TalonConfig } from "../../../util/config.js";
 import { escapeHtml } from "../formatting.js";
-import { classify, friendlyMessage } from "../../../core/errors.js";
+import {
+  classify,
+  friendlyMessage,
+  RETRY_ELAPSED_CAP_MS,
+} from "../../../core/errors.js";
 import {
   getRecentHistory,
   type HistoryMessage,
 } from "../../../storage/history.js";
 import {
   recordMessageProcessed,
+  recordMessageSettled,
   recordMessageReceived,
   recordError,
 } from "../../../util/watchdog.js";
@@ -176,6 +181,7 @@ async function flushQueue(chatId: string): Promise<void> {
       chatTitle: last.chatTitle,
     });
 
+  const startedAt = Date.now();
   try {
     await runTurn();
     lastHandledMessageIdByChat.set(chatId, last.messageId);
@@ -190,8 +196,20 @@ async function flushQueue(chatId: string): Promise<void> {
     );
     recordError(classified.message);
 
-    // Retry once for transient errors (rate_limit, overloaded, network)
-    if (classified.retryable) {
+    // Retry once for transient errors (rate_limit, overloaded, network) —
+    // but only when the failed attempt was actually brief. An attempt that
+    // already ran for minutes (a remote turn deadline, a slow collapse)
+    // won't be saved by a 2s pause, and turns serialize per chat, so the
+    // blind retry used to double a 10-minute stall for everything queued
+    // behind it.
+    const attemptMs = Date.now() - startedAt;
+    if (classified.retryable && attemptMs >= RETRY_ELAPSED_CAP_MS) {
+      log(
+        "bot",
+        `[${chatId}] Not retrying ${classified.reason}: the attempt already ran ${Math.round(attemptMs / 1000)}s`,
+      );
+    }
+    if (classified.retryable && attemptMs < RETRY_ELAPSED_CAP_MS) {
       const delayMs = classified.retryAfterMs ?? 2000;
       log(
         "bot",
@@ -209,6 +227,7 @@ async function flushQueue(chatId: string): Promise<void> {
           "bot",
           `[${chatId}] [${chatType}] Retry failed: ${retryClassified.message}`,
         );
+        recordMessageSettled();
         await sendHtml(
           bot,
           numericChatId,
@@ -219,6 +238,7 @@ async function flushQueue(chatId: string): Promise<void> {
       }
     }
 
+    recordMessageSettled();
     await sendHtml(
       bot,
       numericChatId,

package/src/util/watchdog.ts

@@ -30,6 +30,18 @@ export function recordMessageProcessed(): void {
   resetStuckWarnBackoff();
 }
 
+/**
+ * Record that a message's turn settled in failure. The loop is moving —
+ * the error was reported and the queue slot freed — so advance the stuck
+ * clock without counting a success. Without this, one failed turn leaves
+ * `lastProcessedAt` stale forever and the watchdog reports a wedged loop
+ * (and an unhealthy bridge) on a bot that is actually idle.
+ */
+export function recordMessageSettled(): void {
+  lastProcessedAt = Date.now();
+  resetStuckWarnBackoff();
+}
+
 /**
  * Test-only: reset the activity clocks to "just processed, nothing pending"
  * at the CURRENT Date.now(). Fake-timer suites need this because each test