talon-agent 5.18.2 → 5.19.0

package/src/frontend/native/turn/turn.ts

@@ -102,10 +102,8 @@ export function liveTurnEvents(runtime: NativeRuntime): BridgeEvent[] {
 
 /**
  * Safety net: any tool the backend announced but never resolved (a crash can
- * eat a tool_result; callback backends historically never emitted one —
- * handler-to-events now pairs each tool_call with an immediate synthetic
- * result) gets a synthetic result at turn end — a spinner the app opened on
- * phase:"call" must always see a phase:"result".
+ * eat a tool_result) gets a synthetic result at turn end — a spinner the app
+ * opened on phase:"call" must always see a phase:"result".
  */
 function flushOpenTools(
   runtime: NativeRuntime,
@@ -312,14 +310,14 @@ async function runTurn(
     // clients. Any late tool spinner has already been flushed above.
     runtime.liveTurns.delete(entry.id);
     // Flush a queued follow-up as a fresh turn: "once it's done it will
-    // send". Deferred so we don't re-enter runTurn inside its own finally.
+    // send". Started synchronously so the chat never looks idle in between —
+    // a /send landing in a deferred gap would start its own turn and run
+    // ahead of the message that was queued first.
     const queued = takeQueued(runtime, entry);
     if (queued) {
-      setImmediate(() =>
-        startTurn(runtime, entry, queued.text, {
-          attachments: queued.attachments,
-        }),
-      );
+      startTurn(runtime, entry, queued.text, {
+        attachments: queued.attachments,
+      });
     }
   }
 }

package/src/frontend/presentation/format.ts

@@ -1,10 +1,8 @@
 /**
  * Frontend-agnostic display formatters shared by all chat frontends.
  *
- * These were previously duplicated per-frontend (the old
- * telegram/helpers/format.ts, discord/helpers.ts) and had started to drift —
- * parseInterval accepted days on Discord but not Telegram. One definition
- * here keeps them in lockstep; frontends import this module directly.
+ * One definition here keeps the frontends in lockstep — per-frontend copies
+ * drift (parseInterval once accepted days on Discord but not Telegram).
  */
 
 import { resolveModel } from "../../core/models/catalog.js";

package/src/frontend/presentation/session-status.ts

@@ -1,12 +1,8 @@
 /**
  * Shared /reset and /status logic for chat frontends.
  *
- * Telegram and Discord previously each carried a full copy of the session
- * reset sequence and the /status data-gathering pipeline, and the copies had
- * drifted (Discord's /status no longer re-fetched the context window for the
- * model that actually served the session). All the frontend-agnostic work
- * lives here now; frontends only render `SessionStatusData` in their native
- * markup.
+ * All the frontend-agnostic work lives here so per-frontend copies can't
+ * drift; frontends only render `SessionStatusData` in their native markup.
  */
 
 import type { TalonConfig } from "../../core/config/index.js";

package/src/frontend/teams/actions.ts

@@ -7,6 +7,9 @@ import type { Gateway } from "../../core/engine/gateway.js";
 import { buildAdaptiveCard, splitTeamsMessage } from "./formatting.js";
 import { log, logError } from "../../util/log.js";
 import { proxyFetch } from "./proxy-fetch.js";
+import { createDeliveryTracker, trackDeliveries } from "../health/delivery.js";
+
+const delivery = createDeliveryTracker("teams", "Teams", "teams");
 
 /**
  * POST an Adaptive Card to the Power Automate workflow webhook URL.
@@ -32,7 +35,10 @@ export function createTeamsActionHandler(
   webhookUrl: string,
   gateway: Gateway,
 ): FrontendActionHandler {
-  return async (body, chatId): Promise<ActionResult | null> => {
+  const dispatch: FrontendActionHandler = async (
+    body,
+    chatId,
+  ): Promise<ActionResult | null> => {
     const action = body.action as string;
 
     switch (action) {
@@ -100,6 +106,7 @@ export function createTeamsActionHandler(
         return null;
     }
   };
+  return trackDeliveries(delivery, dispatch);
 }
 
 export { postToTeams };

package/src/frontend/teams/graph.ts

@@ -302,7 +302,6 @@ export class GraphClient {
  * Initialize the Graph client — loads stored tokens or runs device code flow.
  */
 export async function initGraphClient(): Promise<GraphClient> {
-  // Clear old tokens that used channel scopes
   const stored = loadTokens();
 
   if (stored && stored.refreshToken) {

package/src/frontend/teams/index.ts

@@ -76,9 +76,7 @@ export function createTeamsFrontend(
       const graphClient = await initGraphClient();
       runtime.graphClient = graphClient;
 
-      // Get our own user ID (to filter out our own messages)
       const me = await graphClient.getMe();
-      runtime.myUserId = me.id;
       log("teams", `Authenticated as: ${me.displayName} (${me.id})`);
 
       const chatId = await resolveChatId(runtime, graphClient, me.id);
@@ -99,8 +97,7 @@ export function createTeamsFrontend(
 
       // The receive side is a timer on the runtime, not a loop to sit in:
       // once the first poll is done the frontend is listening, and
-      // start() is finished. (It used to park on a promise that never
-      // resolved, which made the boot end at shutdown.)
+      // start() is finished.
       await startPolling(runtime, chatId);
     },
 

package/src/frontend/teams/poll.ts

@@ -6,6 +6,7 @@
  */
 
 import { log, logError } from "../../util/log.js";
+import { errorText } from "../health/outage.js";
 import { handleSlashCommand } from "./commands.js";
 import type { ChatMessage } from "./graph.js";
 import type { TeamsRuntime } from "./runtime.js";
@@ -51,13 +52,47 @@ async function handleMessage(
   runTurn(runtime, msg, talonChatId);
 }
 
+/**
+ * Fetch the newest messages, feeding the poll outage: failures are logged
+ * with their attempt number and time down, and the first success after
+ * them logs the recovery. Returns null when the fetch failed.
+ */
+async function fetchMessages(
+  runtime: TeamsRuntime,
+  graph: NonNullable<TeamsRuntime["graphClient"]>,
+  chatId: string,
+): Promise<ChatMessage[] | null> {
+  let messages: ChatMessage[];
+  try {
+    messages = await graph.getChatMessages(chatId, 20);
+  } catch (err) {
+    const { attempt, downMs } = runtime.pollOutage.fail(err);
+    logError(
+      "teams",
+      `Poll error: poll.fail chat=${chatId} attempt=${attempt} down_ms=${downMs} ` +
+        `next_poll_ms=${runtime.pollIntervalMs} err=${errorText(err)}`,
+    );
+    return null;
+  }
+  const ended = runtime.pollOutage.ok();
+  if (ended) {
+    log(
+      "teams",
+      `poll.recovered chat=${chatId} failed_attempts=${ended.attempts} down_ms=${ended.downMs}`,
+    );
+  }
+  return messages;
+}
+
 async function poll(runtime: TeamsRuntime, chatId: string): Promise<void> {
   if (runtime.polling) return;
   runtime.polling = true;
+  let current: ChatMessage | null = null;
 
   try {
     if (!runtime.graphClient) return;
-    const messages = await runtime.graphClient.getChatMessages(chatId, 20);
+    const messages = await fetchMessages(runtime, runtime.graphClient, chatId);
+    if (!messages) return;
     const newMessages = selectNewMessages(messages, runtime.lastSeenMessageId);
 
     if (newMessages.length > 0) {
@@ -69,12 +104,14 @@ async function poll(runtime: TeamsRuntime, chatId: string): Promise<void> {
       if (!msg.text.trim()) continue;
       if (msg.edited) continue;
       if (isBotEcho(runtime, msg)) continue;
+      current = msg;
       await handleMessage(runtime, msg);
     }
   } catch (err) {
     logError(
       "teams",
-      `Poll error: ${err instanceof Error ? err.message : err}`,
+      `Poll error: message=${current?.id ?? "?"} chat=${current?.chatId ?? chatId} ` +
+        `${err instanceof Error ? err.message : err}`,
     );
   } finally {
     runtime.polling = false;
@@ -94,6 +131,7 @@ export async function startPolling(
 }
 
 export function stopPolling(runtime: TeamsRuntime): void {
+  runtime.pollOutage.dispose();
   if (runtime.pollTimer) {
     clearInterval(runtime.pollTimer);
     runtime.pollTimer = null;

package/src/frontend/teams/runtime.ts

@@ -1,15 +1,17 @@
 /**
  * Teams frontend runtime — the state every module shares.
  *
- * `createTeamsFrontend` used to hold all of this as closure variables, with
- * the poll loop and the slash commands nested inside `.start`. It is now one
- * explicit object, constructed once, that each module (chat-discovery,
+ * One explicit object, constructed once, that each module (chat-discovery,
  * poll, commands, turn, outbound) takes as its first parameter.
  */
 
 import type { TalonConfig } from "../../core/config/index.js";
 import type { Gateway } from "../../core/engine/gateway.js";
 import type { GraphClient } from "./graph.js";
+import { createOutage, type Outage } from "../health/outage.js";
+
+/** Graph polling must fail this long before `teams.poll` is raised. */
+const POLL_OUTAGE_MS = 10 * 60_000;
 
 export type TeamsRuntime = {
   readonly config: TalonConfig;
@@ -25,9 +27,10 @@ export type TeamsRuntime = {
   pollTimer: ReturnType<typeof setInterval> | null;
   /** Newest message id already handled — the poll loop cuts at it. */
   lastSeenMessageId: string | null;
-  myUserId: string | null;
   /** Re-entrancy guard: a slow poll never overlaps the next tick. */
   polling: boolean;
+  /** Failed Graph fetches, and the `teams.poll` alert they raise. */
+  readonly pollOutage: Outage;
 };
 
 export function createTeamsRuntime(
@@ -45,7 +48,13 @@ export function createTeamsRuntime(
     graphClient: null,
     pollTimer: null,
     lastSeenMessageId: null,
-    myUserId: null,
     polling: false,
+    pollOutage: createOutage({
+      key: "teams.poll",
+      thresholdMs: POLL_OUTAGE_MS,
+      describe: (err, mins) =>
+        `Teams polling has failed for ${mins} min: ${err}. Messages are not being received.`,
+      recovered: "Teams polling is working again.",
+    }),
   };
 }

package/src/frontend/telegram/actions/index.ts

@@ -24,6 +24,8 @@ import { mediaHandlers } from "./media.js";
 import { chatInfoHandlers } from "./chat-info.js";
 import { moderationHandlers } from "./moderation/index.js";
 import type { TelegramActionContext, TelegramActionHandlers } from "./types.js";
+import { trackDeliveries } from "../../health/delivery.js";
+import { telegramDelivery } from "./send.js";
 
 export { sendText } from "./send.js";
 
@@ -58,7 +60,7 @@ export function createTelegramActionHandler(
   // shutdown — the timers died with the process, the store didn't.
   restoreScheduledMessages(bot, ctx.scheduledMessages);
 
-  return async (
+  const dispatch = async (
     body: Record<string, unknown>,
     chatId: number,
   ): Promise<ActionResult | null> => {
@@ -67,4 +69,5 @@ export function createTelegramActionHandler(
     if (!handler) return null; // not a Telegram action
     return handler(body, chatId, ctx);
   };
+  return trackDeliveries(telegramDelivery, dispatch);
 }

package/src/frontend/telegram/actions/send.ts

@@ -14,8 +14,16 @@ import {
   noteRichMessageFailure,
   richMessagesAvailable,
 } from "./rich-messages.js";
+import { createDeliveryTracker } from "../../health/delivery.js";
 import { recordOutgoingText } from "./outgoing-log.js";
 
+/** Reply delivery streaks — shared by the reply actions and text blocks. */
+export const telegramDelivery = createDeliveryTracker(
+  "telegram",
+  "Telegram",
+  "bot",
+);
+
 export function replyParams(
   body: Record<string, unknown>,
 ): ReplyParams | undefined {

package/src/frontend/telegram/handlers/context.ts

@@ -147,17 +147,28 @@ export function getForwardContext(msg: {
   return `[Forwarded from ${from}]\n`;
 }
 
+/**
+ * Deadline for one whole download (getFile + body). Media handlers run in
+ * grammY's update loop, which handles updates one at a time, so a download
+ * stalled on a dead socket holds every later message behind it.
+ */
+const DOWNLOAD_TIMEOUT_MS = 120_000;
+
 export async function downloadTelegramFile(
   bot: Bot,
   config: TalonConfig,
   fileId: string,
   fileName: string,
 ): Promise<string> {
-  const file = await bot.api.getFile(fileId);
+  const signal = AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS);
+  const file = await bot.api.getFile(
+    fileId,
+    signal as Parameters<Bot["api"]["getFile"]>[1],
+  );
   if (!file.file_path) throw new Error("Could not get file path from Telegram");
 
   const url = `https://api.telegram.org/file/bot${config.botToken}/${file.file_path}`;
-  const resp = await fetch(url);
+  const resp = await fetch(url, { signal });
   if (!resp.ok) throw new Error(`Download failed: ${resp.status}`);
 
   // Guard against excessively large files (50MB limit)

package/src/frontend/telegram/handlers/delivery.ts

@@ -10,7 +10,7 @@ import { toolInputToRecord } from "../../../core/agent-runtime/events.js";
 import { appendDailyLogResponse } from "../../../storage/daily-log.js";
 import { stripMcpPrefix } from "../../../core/tools/index.js";
 import { logWarn } from "../../../util/log.js";
-import { replyParamsFor, sendText } from "../actions/send.js";
+import { replyParamsFor, sendText, telegramDelivery } from "../actions/send.js";
 import { ambientThreadId } from "../topics.js";
 import { isAdminInGroup, trackDmUser } from "./access.js";
 
@@ -90,10 +90,7 @@ function createStreamCallbacks(
   state: StreamState,
   chatTitle?: string,
 ) {
-  const onStreamDelta = async (
-    accumulated: string,
-    _phase?: "thinking" | "text",
-  ) => {
+  const onStreamDelta = async (accumulated: string) => {
     // Skip if drafts not supported or not ready
     if (draftsSupported === false || !state.started || state.editing) return;
     if (accumulated.length - state.lastSentLength < 40) return;
@@ -122,7 +119,13 @@ function createStreamCallbacks(
   };
 
   const onTextBlock = async (text: string) => {
-    await sendText(bot, chatId, text, _replyToId);
+    try {
+      await sendText(bot, chatId, text, _replyToId);
+    } catch (err) {
+      telegramDelivery.failed(chatId, err);
+      throw err;
+    }
+    telegramDelivery.delivered(chatId);
     appendDailyLogResponse("Talon", text, { chatTitle });
     state.lastSentLength = 0;
     state.sentTextBlock = true;
@@ -223,12 +226,12 @@ export async function processAndReply(
             textAccum += event.text;
             // Fire-and-forget: draft edits are throttled + self-mutexed
             // (`state.editing`), so we must NOT block stream consumption
-            // on them — same non-awaited semantics the old bridge had.
-            void onStreamDelta(textAccum, "text");
+            // on them.
+            void onStreamDelta(textAccum);
             break;
           case "reasoning":
             thinkingAccum += event.text;
-            void onStreamDelta(thinkingAccum, "thinking");
+            void onStreamDelta(thinkingAccum);
             break;
           case "assistant_message":
             // Keep the running total monotonic so a following

package/src/frontend/telegram/handlers/index.ts

@@ -9,8 +9,6 @@
  *   - `delivery` — HTML send, streaming drafts, the agent run pipeline
  *   - `queue`    — per-chat debounce queue + per-user rate limiting
  *   - `messages` — per-message-type handlers (text/photo/voice/…/callback)
- *
- * Re-exports the same public surface the old single-file module exposed.
  */
 
 export {

package/src/frontend/telegram/index.ts

@@ -6,7 +6,7 @@
  * core gateway so MCP tool calls route to Telegram API.
  */
 
-import { Bot, InputFile, API_CONSTANTS } from "grammy";
+import { Bot, GrammyError, InputFile, API_CONSTANTS } from "grammy";
 import { autoRetry } from "@grammyjs/auto-retry";
 import { apiThrottler } from "@grammyjs/transformer-throttler";
 import {
@@ -17,6 +17,7 @@ import type { ContextManager } from "../../core/types.js";
 import type { Gateway } from "../../core/engine/gateway.js";
 import { runUntilStopped } from "../../core/frontend-runtime/run-loop.js";
 import { pollDeadline } from "./polling/poll-deadline.js";
+import { pollHealth } from "./polling/poll-health.js";
 import { createTelegramActionHandler, sendText } from "./actions/index.js";
 import { ambientThreadId } from "./topics.js";
 import { initUserClient, disconnectUserClient } from "./userbot.js";
@@ -48,6 +49,12 @@ export type TelegramFrontend = {
   stop: () => Promise<void>;
 };
 
+/** The slice of a grammY BotError's context the error log reads. */
+type UpdateContext = {
+  update?: { update_id?: number };
+  chat?: { id?: number };
+};
+
 // ── Access ──────────────────────────────────────────────────────────────────
 
 /**
@@ -68,6 +75,30 @@ function applyAccessControl(config: TalonConfig): void {
   setAllowedGroups(config.allowedGroups);
 }
 
+/**
+ * The bot's last-resort middleware error handler: log which update in which
+ * chat failed, and exit when Telegram says the token itself is bad.
+ */
+function onBotError(err: unknown): void {
+  const ctx = (err as { ctx?: UpdateContext } | null)?.ctx;
+  logError(
+    "bot",
+    `Unhandled bot error update=${ctx?.update?.update_id ?? "?"} chat=${ctx?.chat?.id ?? "?"}`,
+    err,
+  );
+  // Judge the token by Telegram's error code, never the message text:
+  // a handler's ordinary 400 ("message to edit not found", "chat not
+  // found") must not take the whole daemon down.
+  const cause = (err as { error?: unknown } | null)?.error ?? err;
+  if (
+    cause instanceof GrammyError &&
+    (cause.error_code === 401 || cause.error_code === 404)
+  ) {
+    logError("bot", "Bot token appears invalid — shutting down");
+    process.exit(1);
+  }
+}
+
 // ── Factory ─────────────────────────────────────────────────────────────────
 
 export function createTelegramFrontend(
@@ -80,6 +111,8 @@ export function createTelegramFrontend(
   bot.api.config.use(pollDeadline());
   bot.api.config.use(apiThrottler());
   bot.api.config.use(autoRetry({ maxRetryAttempts: 3, maxDelaySeconds: 60 }));
+  // Outermost: judges each poll by the result grammY finally sees.
+  bot.api.config.use(pollHealth());
 
   const context: ContextManager = {
     acquire: (chatId: number, stringId?: string) =>
@@ -149,14 +182,7 @@ export function createTelegramFrontend(
     },
 
     async start() {
-      bot.catch((err: unknown) => {
-        const msg = err instanceof Error ? err.message : String(err);
-        logError("bot", "Unhandled bot error", err);
-        if (/unauthorized|401|not found|404/i.test(msg)) {
-          logError("bot", "Bot token appears invalid — shutting down");
-          process.exit(1);
-        }
-      });
+      bot.catch(onBotError);
       // Beyond grammY's defaults: `chat_join_request` feeds the moderation
       // tool's pending-join cache (inert unless the bot admins an
       // approval-gated chat).

package/src/frontend/telegram/polling/poll-health.ts

@@ -0,0 +1,110 @@
+/**
+ * Long-poll health — the operator hears when the bot stops receiving.
+ *
+ * grammY retries a failed getUpdates forever (3s apart, or Telegram's
+ * `retry_after`) and logs nothing outside its debug channel, so a bot cut
+ * off from api.telegram.org used to look exactly like a quiet day. This
+ * transformer watches every getUpdates grammY makes:
+ *
+ *   - each failure is logged with its attempt number, the time down so far,
+ *     grammY's backoff and the error; failures that persist for
+ *     `POLL_OUTAGE_MS` raise `telegram.polling`, and the next successful
+ *     poll resolves it;
+ *   - 409 Conflict (another process polling the same token) and 401 (token
+ *     rejected) end grammY's polling loop outright, so they alert at once
+ *     and critically — nothing will retry them.
+ *
+ * Installed outermost, so it sees what grammY sees: one result per poll,
+ * after the deadline and auto-retry layers have had their turn.
+ */
+
+import { GrammyError, type Transformer } from "grammy";
+import {
+  raiseAlert,
+  resolveAlert,
+} from "../../../core/frontend-runtime/alerts.js";
+import { log, logError, logWarn } from "../../../util/log.js";
+import { createOutage, errorText } from "../../health/outage.js";
+
+/** Failed polls must persist this long before `telegram.polling` fires. */
+const POLL_OUTAGE_MS = 5 * 60_000;
+/** grammY's pause between failed getUpdates calls (bot.js handlePollingError). */
+const GRAMMY_RETRY_MS = 3_000;
+
+function backoffMs(err: unknown): number {
+  if (err instanceof GrammyError && err.error_code === 429) {
+    return (err.parameters.retry_after ?? 3) * 1000;
+  }
+  return GRAMMY_RETRY_MS;
+}
+
+export function pollHealth(thresholdMs = POLL_OUTAGE_MS): Transformer {
+  const outage = createOutage({
+    key: "telegram.polling",
+    thresholdMs,
+    describe: (err, mins) =>
+      `Telegram polling has failed for ${mins} min: ${err}. Messages are not being received.`,
+    recovered: "Telegram polling is working again.",
+  });
+  let conflict = false;
+
+  const onFatal = (err: GrammyError): void => {
+    const detail = errorText(err);
+    logError(
+      "bot",
+      `telegram.poll.stopped code=${err.error_code} err=${detail}`,
+    );
+    if (err.error_code === 409) {
+      conflict = true;
+      raiseAlert(
+        "telegram.conflict",
+        `Another process is polling this Telegram bot token (${detail}). ` +
+          "Talon has stopped receiving Telegram messages: stop the other instance, then restart Talon.",
+        { severity: "critical" },
+      );
+    } else {
+      outage.raiseNow(
+        `Telegram rejected the bot token (${detail}). Talon has stopped receiving Telegram messages.`,
+        "critical",
+      );
+    }
+  };
+
+  return async (prev, method, payload, signal) => {
+    if (method !== "getUpdates") return prev(method, payload, signal);
+    try {
+      const res = await prev(method, payload, signal);
+      const ended = outage.ok();
+      if (ended) {
+        log(
+          "bot",
+          `telegram.poll.recovered failed_attempts=${ended.attempts} down_ms=${ended.downMs}`,
+        );
+      }
+      if (conflict) {
+        conflict = false;
+        resolveAlert(
+          "telegram.conflict",
+          "Telegram polling resumed — no other process is polling this bot token.",
+        );
+      }
+      return res;
+    } catch (err) {
+      // bot.stop() cancelling the in-flight poll is a shutdown, not a fault.
+      if (signal?.aborted) throw err;
+      if (
+        err instanceof GrammyError &&
+        (err.error_code === 409 || err.error_code === 401)
+      ) {
+        onFatal(err);
+        throw err;
+      }
+      const { attempt, downMs } = outage.fail(err);
+      logWarn(
+        "bot",
+        `telegram.poll.fail attempt=${attempt} down_ms=${downMs} backoff_ms=${backoffMs(err)} err=${errorText(err)}`,
+      );
+      throw err;
+    }
+  };
+}