talon-agent 5.18.2 → 5.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +2 -1
  2. package/package.json +2 -2
  3. package/prompts/system/agent-brief.md +20 -3
  4. package/src/app.ts +13 -0
  5. package/src/backend/claude-sdk/handler.ts +4 -4
  6. package/src/backend/claude-sdk/mcp-ready.ts +16 -2
  7. package/src/backend/claude-sdk/one-shot.ts +32 -2
  8. package/src/backend/claude-sdk/stream.ts +2 -2
  9. package/src/backend/codex/auth.ts +1 -1
  10. package/src/backend/codex/handler/message.ts +11 -11
  11. package/src/backend/codex/init.ts +4 -9
  12. package/src/backend/codex/mcp-config.ts +1 -2
  13. package/src/backend/codex/oauth-incompat.ts +8 -4
  14. package/src/backend/codex/one-shot.ts +1 -1
  15. package/src/backend/openai-agents/builtins.ts +55 -27
  16. package/src/backend/openai-agents/factory.ts +3 -3
  17. package/src/backend/openai-agents/handler/message.ts +3 -5
  18. package/src/backend/openai-agents/mcp-pool.ts +10 -27
  19. package/src/backend/remote-server/chat-turn.ts +6 -6
  20. package/src/backend/remote-server/events.ts +1 -5
  21. package/src/backend/remote-server/index.ts +0 -1
  22. package/src/backend/remote-server/messages.ts +3 -7
  23. package/src/backend/remote-server/one-shot.ts +1 -3
  24. package/src/backend/remote-server/session-helpers.ts +1 -4
  25. package/src/backend/remote-server/sse-stream.ts +8 -9
  26. package/src/backend/runtime/metrics.ts +7 -13
  27. package/src/backend/runtime/sleep.ts +1 -2
  28. package/src/backend/runtime/turn/handle-retry.ts +48 -2
  29. package/src/backend/runtime/turn/handler-to-events.ts +3 -4
  30. package/src/bootstrap.ts +9 -1
  31. package/src/cli/doctor.ts +3 -0
  32. package/src/cli/index.ts +8 -10
  33. package/src/cli/logs.ts +148 -9
  34. package/src/cli/setup.ts +9 -11
  35. package/src/cli/status.ts +29 -0
  36. package/src/core/agent-runtime/README.md +5 -19
  37. package/src/core/agent-runtime/events.ts +3 -39
  38. package/src/core/agent-runtime/model-ref.ts +0 -8
  39. package/src/core/agents/registry.ts +65 -2
  40. package/src/core/auth/expiry-monitor.ts +9 -1
  41. package/src/core/auth/login-flow.ts +9 -1
  42. package/src/core/auth/status.ts +31 -3
  43. package/src/core/background/cron/scheduler.ts +25 -5
  44. package/src/core/background/dream/index.ts +29 -10
  45. package/src/core/background/failure-backoff.ts +30 -0
  46. package/src/core/background/heartbeat/agent.ts +2 -27
  47. package/src/core/background/heartbeat/index.ts +0 -2
  48. package/src/core/background/heartbeat/scheduler.ts +21 -13
  49. package/src/core/background/heartbeat/state.ts +10 -2
  50. package/src/core/background/isolated-agent.ts +6 -2
  51. package/src/core/background/pulse/pulse.ts +9 -0
  52. package/src/core/background/triggers/exit.ts +54 -0
  53. package/src/core/background/triggers/index.ts +1 -3
  54. package/src/core/background/triggers/resume.ts +2 -4
  55. package/src/core/backup/archive/tar.ts +14 -4
  56. package/src/core/backup/plan.ts +1 -0
  57. package/src/core/backup/restore.ts +21 -13
  58. package/src/core/backup/scheduler.ts +28 -9
  59. package/src/core/backup/snapshot.ts +43 -10
  60. package/src/core/backup/store.ts +7 -19
  61. package/src/core/backup/targets.ts +69 -17
  62. package/src/core/config/index.ts +14 -0
  63. package/src/core/daemon/crash-marker.ts +141 -0
  64. package/src/core/daemon/crash.ts +9 -2
  65. package/src/core/daemon/handoff.ts +15 -0
  66. package/src/core/daemon/health-alerts.ts +297 -0
  67. package/src/core/daemon/log-reader.ts +289 -0
  68. package/src/core/doctor/index.ts +18 -2
  69. package/src/core/doctor/logs.ts +124 -0
  70. package/src/core/doctor/types.ts +1 -1
  71. package/src/core/engine/backend-controller/index.ts +1 -13
  72. package/src/core/engine/backend-router/router.ts +1 -1
  73. package/src/core/engine/dispatcher.ts +55 -2
  74. package/src/core/engine/fault-text.ts +40 -0
  75. package/src/core/engine/gateway-actions/agents/index.ts +3 -2
  76. package/src/core/engine/gateway-actions/agents/report.ts +62 -0
  77. package/src/core/engine/gateway-actions/history.ts +2 -4
  78. package/src/core/engine/gateway-actions/mesh.ts +26 -14
  79. package/src/core/engine/gateway-actions/native/exec.ts +13 -16
  80. package/src/core/engine/gateway-actions/native/teleport.ts +1 -1
  81. package/src/core/engine/gateway.ts +60 -1
  82. package/src/core/engine/turn-health.ts +222 -0
  83. package/src/core/errors.ts +2 -2
  84. package/src/core/frontend-runtime/admin-notify.ts +1 -1
  85. package/src/core/frontend-runtime/alerts.ts +130 -0
  86. package/src/core/mcp-hub/children.ts +78 -29
  87. package/src/core/mcp-hub/index.ts +21 -18
  88. package/src/core/mcp-hub/proxy-server.ts +8 -4
  89. package/src/core/mcp-hub/talon-server.ts +5 -12
  90. package/src/core/mesh/credentials/store.ts +16 -1
  91. package/src/core/mesh/devices/registry.ts +1 -17
  92. package/src/core/mesh/devices/service.ts +49 -17
  93. package/src/core/mesh/devices/teleport.ts +14 -2
  94. package/src/core/mesh/links/node-binaries.ts +13 -6
  95. package/src/core/mesh/persist.ts +22 -10
  96. package/src/core/mesh/transfers/device-files.ts +5 -23
  97. package/src/core/mesh/transfers/transfers.ts +16 -3
  98. package/src/core/models/active-model.ts +2 -55
  99. package/src/core/plugin/actions.ts +19 -20
  100. package/src/core/plugin/builtins.ts +80 -90
  101. package/src/core/plugin/index.ts +1 -4
  102. package/src/core/plugin/loader.ts +25 -33
  103. package/src/core/plugin/mcp.ts +3 -5
  104. package/src/core/plugin/registry.ts +19 -35
  105. package/src/core/plugin/types.ts +2 -5
  106. package/src/core/prompt/assemble.ts +15 -3
  107. package/src/core/scripts/lua.ts +6 -2
  108. package/src/core/tasks/table.ts +8 -2
  109. package/src/core/tools/bridge.ts +2 -4
  110. package/src/core/tools/chat/cross-send.ts +1 -1
  111. package/src/core/tools/chat/messaging.ts +1 -1
  112. package/src/core/tools/index.ts +2 -2
  113. package/src/core/tools/mcp-env.ts +2 -59
  114. package/src/core/tools/ops/agents.ts +22 -1
  115. package/src/core/tools/schemas.ts +4 -9
  116. package/src/core/vfs/fusefs.ts +0 -5
  117. package/src/core/vfs/index.ts +9 -2
  118. package/src/core/vfs/mounts/diagnostics.ts +109 -0
  119. package/src/core/vfs/mounts/proc.ts +17 -1
  120. package/src/core/vfs/workspace.ts +7 -3
  121. package/src/core/weaver/shuttle.ts +8 -1
  122. package/src/core/weaver/turn-log.ts +320 -0
  123. package/src/core/weaver/weaver.ts +48 -5
  124. package/src/frontend/discord/actions/index.ts +8 -1
  125. package/src/frontend/discord/diagnostics.ts +82 -6
  126. package/src/frontend/discord/handlers/index.ts +0 -2
  127. package/src/frontend/discord/middleware.ts +7 -13
  128. package/src/frontend/discord/runtime.ts +1 -3
  129. package/src/frontend/health/delivery.ts +115 -0
  130. package/src/frontend/health/outage.ts +116 -0
  131. package/src/frontend/native/bridge/routes/chats.ts +3 -5
  132. package/src/frontend/native/bridge/server.ts +106 -21
  133. package/src/frontend/native/index.ts +1 -1
  134. package/src/frontend/native/media/media.ts +5 -1
  135. package/src/frontend/native/runtime.ts +12 -7
  136. package/src/frontend/native/surface/handlers.ts +1 -1
  137. package/src/frontend/native/surface/memory.ts +1 -1
  138. package/src/frontend/native/surface/models.ts +3 -3
  139. package/src/frontend/native/surface/settings.ts +20 -8
  140. package/src/frontend/native/turn/context.ts +6 -8
  141. package/src/frontend/native/turn/turn-meta.ts +2 -5
  142. package/src/frontend/native/turn/turn.ts +8 -10
  143. package/src/frontend/presentation/format.ts +2 -4
  144. package/src/frontend/presentation/session-status.ts +2 -6
  145. package/src/frontend/teams/actions.ts +8 -1
  146. package/src/frontend/teams/graph.ts +0 -1
  147. package/src/frontend/teams/index.ts +1 -4
  148. package/src/frontend/teams/poll.ts +40 -2
  149. package/src/frontend/teams/runtime.ts +14 -5
  150. package/src/frontend/telegram/actions/index.ts +4 -1
  151. package/src/frontend/telegram/actions/send.ts +8 -0
  152. package/src/frontend/telegram/handlers/context.ts +13 -2
  153. package/src/frontend/telegram/handlers/delivery.ts +12 -9
  154. package/src/frontend/telegram/handlers/index.ts +0 -2
  155. package/src/frontend/telegram/index.ts +35 -9
  156. package/src/frontend/telegram/polling/poll-health.ts +110 -0
  157. package/src/frontend/telegram/userbot.ts +100 -36
  158. package/src/frontend/terminal/builtins/session.ts +2 -2
  159. package/src/frontend/terminal/index.ts +1 -3
  160. package/src/frontend/terminal/renderer.ts +2 -18
  161. package/src/frontend/whatsapp/actions/index.ts +12 -1
  162. package/src/frontend/whatsapp/actions/messaging.ts +7 -2
  163. package/src/frontend/whatsapp/connection/connection.ts +29 -9
  164. package/src/frontend/whatsapp/connection/health.ts +89 -0
  165. package/src/frontend/whatsapp/connection/identity.ts +4 -4
  166. package/src/frontend/whatsapp/runtime.ts +11 -5
  167. package/src/native/blake3.ts +28 -2
  168. package/src/native/fusefs.ts +23 -5
  169. package/src/native/registry.ts +1 -1
  170. package/src/native/warden.ts +33 -5
  171. package/src/plugins/github/index.ts +0 -1
  172. package/src/plugins/mempalace/index.ts +9 -3
  173. package/src/plugins/playwright/index.ts +2 -4
  174. package/src/plugins/playwright/provision.ts +12 -4
  175. package/src/storage/chat-settings.ts +6 -1
  176. package/src/storage/cron.ts +29 -4
  177. package/src/storage/daily-log.ts +43 -47
  178. package/src/storage/db.ts +61 -33
  179. package/src/storage/history.ts +6 -1
  180. package/src/storage/journal.ts +9 -2
  181. package/src/storage/kv.ts +19 -6
  182. package/src/storage/media-index.ts +28 -6
  183. package/src/storage/repositories/chat-settings-repo.ts +8 -2
  184. package/src/storage/repositories/sessions-repo.ts +10 -3
  185. package/src/storage/scripts.ts +24 -13
  186. package/src/storage/sessions.ts +11 -2
  187. package/src/storage/skills.ts +21 -2
  188. package/src/storage/stickers.ts +17 -3
  189. package/src/storage/triggers.ts +8 -3
  190. package/src/storage/turn-meta.ts +25 -7
  191. package/src/util/log.ts +189 -6
  192. package/src/util/logging/turn-scope.ts +85 -0
  193. package/src/util/time.ts +3 -3
  194. package/src/util/watchdog.ts +30 -0
  195. package/src/core/engine/backend-controller/legacy.ts +0 -111
@@ -24,6 +24,8 @@ import { mediaHandlers } from "./media.js";
24
24
  import { chatInfoHandlers } from "./chat-info.js";
25
25
  import { moderationHandlers } from "./moderation/index.js";
26
26
  import type { TelegramActionContext, TelegramActionHandlers } from "./types.js";
27
+ import { trackDeliveries } from "../../health/delivery.js";
28
+ import { telegramDelivery } from "./send.js";
27
29
 
28
30
  export { sendText } from "./send.js";
29
31
 
@@ -58,7 +60,7 @@ export function createTelegramActionHandler(
58
60
  // shutdown — the timers died with the process, the store didn't.
59
61
  restoreScheduledMessages(bot, ctx.scheduledMessages);
60
62
 
61
- return async (
63
+ const dispatch = async (
62
64
  body: Record<string, unknown>,
63
65
  chatId: number,
64
66
  ): Promise<ActionResult | null> => {
@@ -67,4 +69,5 @@ export function createTelegramActionHandler(
67
69
  if (!handler) return null; // not a Telegram action
68
70
  return handler(body, chatId, ctx);
69
71
  };
72
+ return trackDeliveries(telegramDelivery, dispatch);
70
73
  }
@@ -14,8 +14,16 @@ import {
14
14
  noteRichMessageFailure,
15
15
  richMessagesAvailable,
16
16
  } from "./rich-messages.js";
17
+ import { createDeliveryTracker } from "../../health/delivery.js";
17
18
  import { recordOutgoingText } from "./outgoing-log.js";
18
19
 
20
+ /** Reply delivery streaks — shared by the reply actions and text blocks. */
21
+ export const telegramDelivery = createDeliveryTracker(
22
+ "telegram",
23
+ "Telegram",
24
+ "bot",
25
+ );
26
+
19
27
  export function replyParams(
20
28
  body: Record<string, unknown>,
21
29
  ): ReplyParams | undefined {
@@ -147,17 +147,28 @@ export function getForwardContext(msg: {
147
147
  return `[Forwarded from ${from}]\n`;
148
148
  }
149
149
 
150
+ /**
151
+ * Deadline for one whole download (getFile + body). Media handlers run in
152
+ * grammY's update loop, which handles updates one at a time, so a download
153
+ * stalled on a dead socket holds every later message behind it.
154
+ */
155
+ const DOWNLOAD_TIMEOUT_MS = 120_000;
156
+
150
157
  export async function downloadTelegramFile(
151
158
  bot: Bot,
152
159
  config: TalonConfig,
153
160
  fileId: string,
154
161
  fileName: string,
155
162
  ): Promise<string> {
156
- const file = await bot.api.getFile(fileId);
163
+ const signal = AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS);
164
+ const file = await bot.api.getFile(
165
+ fileId,
166
+ signal as Parameters<Bot["api"]["getFile"]>[1],
167
+ );
157
168
  if (!file.file_path) throw new Error("Could not get file path from Telegram");
158
169
 
159
170
  const url = `https://api.telegram.org/file/bot${config.botToken}/${file.file_path}`;
160
- const resp = await fetch(url);
171
+ const resp = await fetch(url, { signal });
161
172
  if (!resp.ok) throw new Error(`Download failed: ${resp.status}`);
162
173
 
163
174
  // Guard against excessively large files (50MB limit)
@@ -10,7 +10,7 @@ import { toolInputToRecord } from "../../../core/agent-runtime/events.js";
10
10
  import { appendDailyLogResponse } from "../../../storage/daily-log.js";
11
11
  import { stripMcpPrefix } from "../../../core/tools/index.js";
12
12
  import { logWarn } from "../../../util/log.js";
13
- import { replyParamsFor, sendText } from "../actions/send.js";
13
+ import { replyParamsFor, sendText, telegramDelivery } from "../actions/send.js";
14
14
  import { ambientThreadId } from "../topics.js";
15
15
  import { isAdminInGroup, trackDmUser } from "./access.js";
16
16
 
@@ -90,10 +90,7 @@ function createStreamCallbacks(
90
90
  state: StreamState,
91
91
  chatTitle?: string,
92
92
  ) {
93
- const onStreamDelta = async (
94
- accumulated: string,
95
- _phase?: "thinking" | "text",
96
- ) => {
93
+ const onStreamDelta = async (accumulated: string) => {
97
94
  // Skip if drafts not supported or not ready
98
95
  if (draftsSupported === false || !state.started || state.editing) return;
99
96
  if (accumulated.length - state.lastSentLength < 40) return;
@@ -122,7 +119,13 @@ function createStreamCallbacks(
122
119
  };
123
120
 
124
121
  const onTextBlock = async (text: string) => {
125
- await sendText(bot, chatId, text, _replyToId);
122
+ try {
123
+ await sendText(bot, chatId, text, _replyToId);
124
+ } catch (err) {
125
+ telegramDelivery.failed(chatId, err);
126
+ throw err;
127
+ }
128
+ telegramDelivery.delivered(chatId);
126
129
  appendDailyLogResponse("Talon", text, { chatTitle });
127
130
  state.lastSentLength = 0;
128
131
  state.sentTextBlock = true;
@@ -223,12 +226,12 @@ export async function processAndReply(
223
226
  textAccum += event.text;
224
227
  // Fire-and-forget: draft edits are throttled + self-mutexed
225
228
  // (`state.editing`), so we must NOT block stream consumption
226
- // on them — same non-awaited semantics the old bridge had.
227
- void onStreamDelta(textAccum, "text");
229
+ // on them.
230
+ void onStreamDelta(textAccum);
228
231
  break;
229
232
  case "reasoning":
230
233
  thinkingAccum += event.text;
231
- void onStreamDelta(thinkingAccum, "thinking");
234
+ void onStreamDelta(thinkingAccum);
232
235
  break;
233
236
  case "assistant_message":
234
237
  // Keep the running total monotonic so a following
@@ -9,8 +9,6 @@
9
9
  * - `delivery` — HTML send, streaming drafts, the agent run pipeline
10
10
  * - `queue` — per-chat debounce queue + per-user rate limiting
11
11
  * - `messages` — per-message-type handlers (text/photo/voice/…/callback)
12
- *
13
- * Re-exports the same public surface the old single-file module exposed.
14
12
  */
15
13
 
16
14
  export {
@@ -6,7 +6,7 @@
6
6
  * core gateway so MCP tool calls route to Telegram API.
7
7
  */
8
8
 
9
- import { Bot, InputFile, API_CONSTANTS } from "grammy";
9
+ import { Bot, GrammyError, InputFile, API_CONSTANTS } from "grammy";
10
10
  import { autoRetry } from "@grammyjs/auto-retry";
11
11
  import { apiThrottler } from "@grammyjs/transformer-throttler";
12
12
  import {
@@ -17,6 +17,7 @@ import type { ContextManager } from "../../core/types.js";
17
17
  import type { Gateway } from "../../core/engine/gateway.js";
18
18
  import { runUntilStopped } from "../../core/frontend-runtime/run-loop.js";
19
19
  import { pollDeadline } from "./polling/poll-deadline.js";
20
+ import { pollHealth } from "./polling/poll-health.js";
20
21
  import { createTelegramActionHandler, sendText } from "./actions/index.js";
21
22
  import { ambientThreadId } from "./topics.js";
22
23
  import { initUserClient, disconnectUserClient } from "./userbot.js";
@@ -48,6 +49,12 @@ export type TelegramFrontend = {
48
49
  stop: () => Promise<void>;
49
50
  };
50
51
 
52
+ /** The slice of a grammY BotError's context the error log reads. */
53
+ type UpdateContext = {
54
+ update?: { update_id?: number };
55
+ chat?: { id?: number };
56
+ };
57
+
51
58
  // ── Access ──────────────────────────────────────────────────────────────────
52
59
 
53
60
  /**
@@ -68,6 +75,30 @@ function applyAccessControl(config: TalonConfig): void {
68
75
  setAllowedGroups(config.allowedGroups);
69
76
  }
70
77
 
78
+ /**
79
+ * The bot's last-resort middleware error handler: log which update in which
80
+ * chat failed, and exit when Telegram says the token itself is bad.
81
+ */
82
+ function onBotError(err: unknown): void {
83
+ const ctx = (err as { ctx?: UpdateContext } | null)?.ctx;
84
+ logError(
85
+ "bot",
86
+ `Unhandled bot error update=${ctx?.update?.update_id ?? "?"} chat=${ctx?.chat?.id ?? "?"}`,
87
+ err,
88
+ );
89
+ // Judge the token by Telegram's error code, never the message text:
90
+ // a handler's ordinary 400 ("message to edit not found", "chat not
91
+ // found") must not take the whole daemon down.
92
+ const cause = (err as { error?: unknown } | null)?.error ?? err;
93
+ if (
94
+ cause instanceof GrammyError &&
95
+ (cause.error_code === 401 || cause.error_code === 404)
96
+ ) {
97
+ logError("bot", "Bot token appears invalid — shutting down");
98
+ process.exit(1);
99
+ }
100
+ }
101
+
71
102
  // ── Factory ─────────────────────────────────────────────────────────────────
72
103
 
73
104
  export function createTelegramFrontend(
@@ -80,6 +111,8 @@ export function createTelegramFrontend(
80
111
  bot.api.config.use(pollDeadline());
81
112
  bot.api.config.use(apiThrottler());
82
113
  bot.api.config.use(autoRetry({ maxRetryAttempts: 3, maxDelaySeconds: 60 }));
114
+ // Outermost: judges each poll by the result grammY finally sees.
115
+ bot.api.config.use(pollHealth());
83
116
 
84
117
  const context: ContextManager = {
85
118
  acquire: (chatId: number, stringId?: string) =>
@@ -149,14 +182,7 @@ export function createTelegramFrontend(
149
182
  },
150
183
 
151
184
  async start() {
152
- bot.catch((err: unknown) => {
153
- const msg = err instanceof Error ? err.message : String(err);
154
- logError("bot", "Unhandled bot error", err);
155
- if (/unauthorized|401|not found|404/i.test(msg)) {
156
- logError("bot", "Bot token appears invalid — shutting down");
157
- process.exit(1);
158
- }
159
- });
185
+ bot.catch(onBotError);
160
186
  // Beyond grammY's defaults: `chat_join_request` feeds the moderation
161
187
  // tool's pending-join cache (inert unless the bot admins an
162
188
  // approval-gated chat).
@@ -0,0 +1,110 @@
1
+ /**
2
+ * Long-poll health — the operator hears when the bot stops receiving.
3
+ *
4
+ * grammY retries a failed getUpdates forever (3s apart, or Telegram's
5
+ * `retry_after`) and logs nothing outside its debug channel, so a bot cut
6
+ * off from api.telegram.org used to look exactly like a quiet day. This
7
+ * transformer watches every getUpdates grammY makes:
8
+ *
9
+ * - each failure is logged with its attempt number, the time down so far,
10
+ * grammY's backoff and the error; failures that persist for
11
+ * `POLL_OUTAGE_MS` raise `telegram.polling`, and the next successful
12
+ * poll resolves it;
13
+ * - 409 Conflict (another process polling the same token) and 401 (token
14
+ * rejected) end grammY's polling loop outright, so they alert at once
15
+ * and critically — nothing will retry them.
16
+ *
17
+ * Installed outermost, so it sees what grammY sees: one result per poll,
18
+ * after the deadline and auto-retry layers have had their turn.
19
+ */
20
+
21
+ import { GrammyError, type Transformer } from "grammy";
22
+ import {
23
+ raiseAlert,
24
+ resolveAlert,
25
+ } from "../../../core/frontend-runtime/alerts.js";
26
+ import { log, logError, logWarn } from "../../../util/log.js";
27
+ import { createOutage, errorText } from "../../health/outage.js";
28
+
29
+ /** Failed polls must persist this long before `telegram.polling` fires. */
30
+ const POLL_OUTAGE_MS = 5 * 60_000;
31
+ /** grammY's pause between failed getUpdates calls (bot.js handlePollingError). */
32
+ const GRAMMY_RETRY_MS = 3_000;
33
+
34
+ function backoffMs(err: unknown): number {
35
+ if (err instanceof GrammyError && err.error_code === 429) {
36
+ return (err.parameters.retry_after ?? 3) * 1000;
37
+ }
38
+ return GRAMMY_RETRY_MS;
39
+ }
40
+
41
+ export function pollHealth(thresholdMs = POLL_OUTAGE_MS): Transformer {
42
+ const outage = createOutage({
43
+ key: "telegram.polling",
44
+ thresholdMs,
45
+ describe: (err, mins) =>
46
+ `Telegram polling has failed for ${mins} min: ${err}. Messages are not being received.`,
47
+ recovered: "Telegram polling is working again.",
48
+ });
49
+ let conflict = false;
50
+
51
+ const onFatal = (err: GrammyError): void => {
52
+ const detail = errorText(err);
53
+ logError(
54
+ "bot",
55
+ `telegram.poll.stopped code=${err.error_code} err=${detail}`,
56
+ );
57
+ if (err.error_code === 409) {
58
+ conflict = true;
59
+ raiseAlert(
60
+ "telegram.conflict",
61
+ `Another process is polling this Telegram bot token (${detail}). ` +
62
+ "Talon has stopped receiving Telegram messages: stop the other instance, then restart Talon.",
63
+ { severity: "critical" },
64
+ );
65
+ } else {
66
+ outage.raiseNow(
67
+ `Telegram rejected the bot token (${detail}). Talon has stopped receiving Telegram messages.`,
68
+ "critical",
69
+ );
70
+ }
71
+ };
72
+
73
+ return async (prev, method, payload, signal) => {
74
+ if (method !== "getUpdates") return prev(method, payload, signal);
75
+ try {
76
+ const res = await prev(method, payload, signal);
77
+ const ended = outage.ok();
78
+ if (ended) {
79
+ log(
80
+ "bot",
81
+ `telegram.poll.recovered failed_attempts=${ended.attempts} down_ms=${ended.downMs}`,
82
+ );
83
+ }
84
+ if (conflict) {
85
+ conflict = false;
86
+ resolveAlert(
87
+ "telegram.conflict",
88
+ "Telegram polling resumed — no other process is polling this bot token.",
89
+ );
90
+ }
91
+ return res;
92
+ } catch (err) {
93
+ // bot.stop() cancelling the in-flight poll is a shutdown, not a fault.
94
+ if (signal?.aborted) throw err;
95
+ if (
96
+ err instanceof GrammyError &&
97
+ (err.error_code === 409 || err.error_code === 401)
98
+ ) {
99
+ onFatal(err);
100
+ throw err;
101
+ }
102
+ const { attempt, downMs } = outage.fail(err);
103
+ logWarn(
104
+ "bot",
105
+ `telegram.poll.fail attempt=${attempt} down_ms=${downMs} backoff_ms=${backoffMs(err)} err=${errorText(err)}`,
106
+ );
107
+ throw err;
108
+ }
109
+ };
110
+ }
@@ -17,6 +17,7 @@ import writeFileAtomic from "write-file-atomic";
17
17
  import { log, logError, logWarn } from "../../util/log.js";
18
18
  import { dirs, files } from "../../util/paths.js";
19
19
  import { formatSmartTimestamp } from "../../util/time.js";
20
+ import { createOutage, errorText } from "../health/outage.js";
20
21
 
21
22
  const SESSION_FILE = files.userSession;
22
23
 
@@ -152,6 +153,39 @@ export function withTimeout<T>(
152
153
  return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
153
154
  }
154
155
 
156
+ /** How long the monitor may fail to reconnect before the operator hears. */
157
+ const USERBOT_OUTAGE_MS = 15 * 60 * 1000;
158
+
159
+ const outage = createOutage({
160
+ key: "telegram.userbot",
161
+ thresholdMs: USERBOT_OUTAGE_MS,
162
+ severity: "warn",
163
+ describe: (err, mins) =>
164
+ `The Telegram user client has failed to reconnect for ${mins} min: ${err}. ` +
165
+ "Full history search and member lookups are unavailable.",
166
+ recovered: "The Telegram user client is reconnected.",
167
+ });
168
+
169
+ /** One failed monitor step, logged with its place in the outage. */
170
+ function noteFailure(step: string, err: unknown): void {
171
+ const { attempt, downMs } = outage.fail(err);
172
+ logWarn(
173
+ "userbot",
174
+ `userbot.reconnect.fail step=${step} attempt=${attempt} down_ms=${downMs} ` +
175
+ `next_check_ms=${CHECK_INTERVAL_MS} err=${errorText(err)}`,
176
+ );
177
+ }
178
+
179
+ /** A healthy monitor step; logs the end of an outage when there was one. */
180
+ function noteHealthy(via: string): void {
181
+ const ended = outage.ok();
182
+ if (!ended) return;
183
+ log(
184
+ "userbot",
185
+ `userbot.reconnect.ok via=${via} failed_attempts=${ended.attempts} down_ms=${ended.downMs}`,
186
+ );
187
+ }
188
+
155
189
  let reconnecting = false;
156
190
  // Bumped by stopConnectionMonitor so a tick whose awaits straddle a shutdown
157
191
  // can tell that its own client was torn down under it — the liveness probe
@@ -164,8 +198,19 @@ function startConnectionMonitor(): void {
164
198
  const gen = monitorGeneration;
165
199
  reconnectTimer = setInterval(async () => {
166
200
  if (gen !== monitorGeneration) return;
167
- if (!client) return;
168
201
  if (reconnecting) return; // prevent overlapping reconnect attempts
202
+ if (!client) {
203
+ // A failed re-init leaves no client. Retry it every tick; bailing
204
+ // here left the userbot dead until the daemon restarted.
205
+ if (!storedApiId || !storedApiHash) return;
206
+ reconnecting = true;
207
+ try {
208
+ await reinitClient();
209
+ } finally {
210
+ reconnecting = false;
211
+ }
212
+ return;
213
+ }
169
214
 
170
215
  // GramJS keeps `client.connected === true` even after the underlying
171
216
  // socket has silently died, so a real round-trip is the only reliable
@@ -179,6 +224,7 @@ function startConnectionMonitor(): void {
179
224
  RECONNECT_STEP_TIMEOUT_MS,
180
225
  "liveness probe",
181
226
  );
227
+ noteHealthy("probe");
182
228
  return; // genuinely alive
183
229
  } catch {
184
230
  if (gen !== monitorGeneration) return; // monitor stopped mid-probe
@@ -203,55 +249,73 @@ function startConnectionMonitor(): void {
203
249
  )
204
250
  ) {
205
251
  log("userbot", "Reconnected successfully.");
252
+ noteHealthy("reconnect");
206
253
  } else {
207
254
  logWarn("userbot", "Reconnected but not authorized.");
255
+ noteFailure("reconnect", "connected but not authorized");
208
256
  }
209
257
  } catch (err) {
210
258
  if (gen !== monitorGeneration) return; // stopped while reconnecting
211
259
  logError("userbot", "Reconnect failed", err);
212
- // Try a full re-init on next check
213
- if (storedApiId && storedApiHash) {
214
- try {
215
- // Drop the wedged client without awaiting it — its socket may be
216
- // the thing that's hung.
217
- const stale = client;
218
- client = null;
219
- void stale?.disconnect().catch(() => {});
220
- let sessionString = "";
221
- if (existsSync(SESSION_FILE)) {
222
- sessionString = readFileSync(SESSION_FILE, "utf-8").trim();
223
- }
224
- const session = new StringSession(sessionString);
225
- client = new TelegramClient(session, storedApiId, storedApiHash, {
226
- connectionRetries: 5,
227
- });
228
- await withTimeout(
229
- client.connect(),
230
- RECONNECT_STEP_TIMEOUT_MS,
231
- "re-init connect",
232
- );
233
- if (
234
- await withTimeout(
235
- client.isUserAuthorized(),
236
- RECONNECT_STEP_TIMEOUT_MS,
237
- "re-init auth check",
238
- )
239
- ) {
240
- log("userbot", "Full re-init reconnect succeeded.");
241
- }
242
- } catch (retryErr) {
243
- logError("userbot", "Full re-init reconnect failed", retryErr);
244
- client = null;
245
- }
246
- }
260
+ if (storedApiId && storedApiHash) await reinitClient();
261
+ else noteFailure("reconnect", err);
247
262
  } finally {
248
263
  reconnecting = false;
249
264
  }
250
265
  }, CHECK_INTERVAL_MS);
251
266
  }
252
267
 
268
+ /**
269
+ * Replace the client with a fresh one from the saved session. On failure
270
+ * the slot is left empty (and the failed client closed); the monitor's
271
+ * next tick tries again.
272
+ */
273
+ async function reinitClient(): Promise<void> {
274
+ // Drop the wedged client without awaiting it — its socket may be the
275
+ // thing that's hung.
276
+ const stale = client;
277
+ client = null;
278
+ void stale?.disconnect().catch(() => {});
279
+ let fresh: TelegramClient | null = null;
280
+ try {
281
+ let sessionString = "";
282
+ if (existsSync(SESSION_FILE)) {
283
+ sessionString = readFileSync(SESSION_FILE, "utf-8").trim();
284
+ }
285
+ const session = new StringSession(sessionString);
286
+ fresh = new TelegramClient(session, storedApiId, storedApiHash, {
287
+ connectionRetries: 5,
288
+ });
289
+ client = fresh;
290
+ await withTimeout(
291
+ fresh.connect(),
292
+ RECONNECT_STEP_TIMEOUT_MS,
293
+ "re-init connect",
294
+ );
295
+ if (
296
+ await withTimeout(
297
+ fresh.isUserAuthorized(),
298
+ RECONNECT_STEP_TIMEOUT_MS,
299
+ "re-init auth check",
300
+ )
301
+ ) {
302
+ log("userbot", "Full re-init reconnect succeeded.");
303
+ noteHealthy("re-init");
304
+ } else {
305
+ noteFailure("re-init", "connected but not authorized");
306
+ }
307
+ } catch (retryErr) {
308
+ logError("userbot", "Full re-init reconnect failed", retryErr);
309
+ noteFailure("re-init", retryErr);
310
+ if (client === fresh) client = null;
311
+ // Its connect may still be retrying in the background.
312
+ void fresh?.disconnect().catch(() => {});
313
+ }
314
+ }
315
+
253
316
  function stopConnectionMonitor(): void {
254
317
  monitorGeneration++;
318
+ outage.dispose();
255
319
  if (reconnectTimer) {
256
320
  clearInterval(reconnectTimer);
257
321
  reconnectTimer = null;
@@ -1,7 +1,7 @@
1
1
  /** `/reset`, `/resume`, `/rename` — session lifecycle. */
2
2
 
3
3
  import pc from "picocolors";
4
- import { formatTimeAgo } from "../renderer.js";
4
+ import { formatRelativeAge } from "../../../util/time.js";
5
5
  import { isTerminalChatId } from "../../../core/frontend-runtime/chat-id.js";
6
6
  import { resolveModel as coreResolveModel } from "../../../core/models/catalog.js";
7
7
  import {
@@ -49,7 +49,7 @@ export const resumeCommand: Command = {
49
49
  ? `"${s.info.sessionName}"`
50
50
  : pc.dim("(unnamed)");
51
51
  const turns = `${s.info.turns} turn${s.info.turns !== 1 ? "s" : ""}`;
52
- const ago = formatTimeAgo(s.info.lastActive);
52
+ const ago = formatRelativeAge(s.info.lastActive);
53
53
  const model = s.info.lastModel
54
54
  ? (coreResolveModel(s.info.lastModel)?.displayName ?? s.info.lastModel)
55
55
  : "";
@@ -8,9 +8,7 @@
8
8
  * - Renderer NEVER touches readline. Only this file does.
9
9
  *
10
10
  * `start()` returns once that prompt is drawn — the frontend lifecycle
11
- * contract (core/frontend-runtime/capabilities.ts). It used to park on a
12
- * promise that never resolved, which is why the composition root had to
13
- * special-case the terminal and start it without awaiting.
11
+ * contract (core/frontend-runtime/capabilities.ts).
14
12
  */
15
13
 
16
14
  import pc from "picocolors";
@@ -7,7 +7,7 @@
7
7
  */
8
8
 
9
9
  import pc from "picocolors";
10
- import { formatUsd } from "../presentation/format.js";
10
+ import { formatTokenCount, formatUsd } from "../presentation/format.js";
11
11
 
12
12
  // ── Types ────────────────────────────────────────────────────────────────────
13
13
 
@@ -70,22 +70,6 @@ export function wrap(text: string, indent: number, maxWidth: number): string {
70
70
  .join("\n");
71
71
  }
72
72
 
73
- export function formatTimeAgo(ts: number): string {
74
- const sec = Math.floor((Date.now() - ts) / 1000);
75
- if (sec < 60) return "just now";
76
- const min = Math.floor(sec / 60);
77
- if (min < 60) return `${min}m ago`;
78
- const hr = Math.floor(min / 60);
79
- if (hr < 24) return `${hr}h ago`;
80
- return `${Math.floor(hr / 24)}d ago`;
81
- }
82
-
83
- function fmtTok(n: number): string {
84
- if (n < 1000) return String(n);
85
- if (n < 1_000_000) return `${(n / 1000).toFixed(1)}k`;
86
- return `${(n / 1_000_000).toFixed(1)}M`;
87
- }
88
-
89
73
  export function extractToolDetail(
90
74
  input: Record<string, unknown>,
91
75
  maxLen: number,
@@ -189,7 +173,7 @@ export function createRenderer(cols?: number, displayName = "Talon"): Renderer {
189
173
  if (info.sessionName) p.push(`"${info.sessionName}"`);
190
174
  p.push(
191
175
  `${info.turns} turn${info.turns !== 1 ? "s" : ""}`,
192
- `${fmtTok(info.inputTokens + info.outputTokens)} tok`,
176
+ `${formatTokenCount(info.inputTokens + info.outputTokens)} tok`,
193
177
  );
194
178
  if (typeof info.cacheHitPct === "number") {
195
179
  p.push(`${info.cacheHitPct}% cache`);
@@ -33,6 +33,16 @@ import { mediaHandlers } from "./media.js";
33
33
  import { messagingHandlers } from "./messaging.js";
34
34
  import { moderationHandlers } from "./moderation.js";
35
35
  import type { WhatsAppActionContext, WhatsAppActionHandlers } from "./types.js";
36
+ import {
37
+ createDeliveryTracker,
38
+ trackDeliveries,
39
+ } from "../../health/delivery.js";
40
+
41
+ const delivery = createDeliveryTracker("whatsapp", "WhatsApp", "whatsapp");
42
+
43
+ /** A cross-send names its destination; everything else goes to `chatId`. */
44
+ const destination = (body: Record<string, unknown>, chatId: number) =>
45
+ body.target !== undefined ? String(body.target) : chatId;
36
46
 
37
47
  // Null-prototype so a request `action` of "toString" / "constructor" can't
38
48
  // resolve an inherited Object.prototype method via `handlers[action]`.
@@ -62,7 +72,7 @@ export function createWhatsAppActionHandler(
62
72
  ) {
63
73
  const scheduledMessages = new Map<string, ReturnType<typeof setTimeout>>();
64
74
 
65
- return async (
75
+ const dispatch = async (
66
76
  body: Record<string, unknown>,
67
77
  chatId: number,
68
78
  ): Promise<ActionResult | null> => {
@@ -118,4 +128,5 @@ export function createWhatsAppActionHandler(
118
128
  ? { ...result, chat_id: chat.chatId }
119
129
  : result;
120
130
  };
131
+ return trackDeliveries(delivery, dispatch, destination);
121
132
  }
@@ -17,6 +17,9 @@ import { tryAction } from "./try-action.js";
17
17
  import { resolveQuoted, sendContent, sendText, boundedSend } from "./send.js";
18
18
  import type { WhatsAppActionHandlers } from "./types.js";
19
19
 
20
+ /** The tool schema's documented ceiling (1-86400), as telegram/discord clamp. */
21
+ const MAX_DELAY_SEC = 24 * 60 * 60;
22
+
20
23
  /** WhatsApp keeps a pin for 24h, 7d, or 30d — no indefinite option. */
21
24
  const PIN_DURATIONS = [86_400, 604_800, 2_592_000] as const;
22
25
 
@@ -229,16 +232,18 @@ export const messagingHandlers: WhatsAppActionHandlers = {
229
232
  schedule_message: (body, _chatId, ctx) =>
230
233
  tryAction("schedule_message", async () => {
231
234
  const text = String(body.text ?? "");
232
- const seconds = Number(body.delay_seconds ?? 0);
235
+ const requested = Number(body.delay_seconds ?? 0);
233
236
  if (!text.trim()) {
234
237
  return { ok: false, error: "schedule_message: text is required" };
235
238
  }
236
- if (!Number.isFinite(seconds) || seconds <= 0) {
239
+ if (!Number.isFinite(requested) || requested <= 0) {
237
240
  return {
238
241
  ok: false,
239
242
  error: "schedule_message: delay_seconds must be a positive number",
240
243
  };
241
244
  }
245
+ // Past setTimeout's 2^31-1 ms (~24.8 days) the timer fires at once.
246
+ const seconds = Math.min(MAX_DELAY_SEC, requested);
242
247
  const id = randomUUID();
243
248
  const chat = ctx.chat!;
244
249
  const timer = setTimeout(() => {