talon-agent 5.18.2 → 5.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. package/README.md +2 -1
  2. package/package.json +1 -1
  3. package/prompts/system/agent-brief.md +20 -3
  4. package/src/app.ts +13 -0
  5. package/src/backend/claude-sdk/handler.ts +4 -4
  6. package/src/backend/claude-sdk/mcp-ready.ts +16 -2
  7. package/src/backend/claude-sdk/stream.ts +2 -2
  8. package/src/backend/codex/auth.ts +1 -1
  9. package/src/backend/codex/handler/message.ts +11 -11
  10. package/src/backend/codex/init.ts +4 -9
  11. package/src/backend/codex/mcp-config.ts +1 -2
  12. package/src/backend/codex/oauth-incompat.ts +8 -4
  13. package/src/backend/codex/one-shot.ts +1 -1
  14. package/src/backend/openai-agents/builtins.ts +55 -27
  15. package/src/backend/openai-agents/factory.ts +3 -3
  16. package/src/backend/openai-agents/handler/message.ts +3 -5
  17. package/src/backend/openai-agents/mcp-pool.ts +10 -27
  18. package/src/backend/remote-server/chat-turn.ts +6 -6
  19. package/src/backend/remote-server/events.ts +1 -5
  20. package/src/backend/remote-server/index.ts +0 -1
  21. package/src/backend/remote-server/messages.ts +3 -7
  22. package/src/backend/remote-server/one-shot.ts +1 -3
  23. package/src/backend/remote-server/session-helpers.ts +1 -4
  24. package/src/backend/remote-server/sse-stream.ts +8 -9
  25. package/src/backend/runtime/metrics.ts +7 -13
  26. package/src/backend/runtime/sleep.ts +1 -2
  27. package/src/backend/runtime/turn/handle-retry.ts +48 -2
  28. package/src/backend/runtime/turn/handler-to-events.ts +3 -4
  29. package/src/bootstrap.ts +9 -1
  30. package/src/cli/doctor.ts +3 -0
  31. package/src/cli/index.ts +8 -10
  32. package/src/cli/logs.ts +148 -9
  33. package/src/cli/setup.ts +9 -11
  34. package/src/cli/status.ts +29 -0
  35. package/src/core/agent-runtime/README.md +5 -19
  36. package/src/core/agent-runtime/events.ts +3 -39
  37. package/src/core/agent-runtime/model-ref.ts +0 -8
  38. package/src/core/agents/registry.ts +49 -2
  39. package/src/core/auth/expiry-monitor.ts +9 -1
  40. package/src/core/auth/login-flow.ts +9 -1
  41. package/src/core/auth/status.ts +31 -3
  42. package/src/core/background/cron/scheduler.ts +25 -5
  43. package/src/core/background/dream/index.ts +29 -10
  44. package/src/core/background/failure-backoff.ts +30 -0
  45. package/src/core/background/heartbeat/agent.ts +2 -27
  46. package/src/core/background/heartbeat/index.ts +0 -2
  47. package/src/core/background/heartbeat/scheduler.ts +21 -13
  48. package/src/core/background/heartbeat/state.ts +10 -2
  49. package/src/core/background/isolated-agent.ts +6 -2
  50. package/src/core/background/pulse/pulse.ts +9 -0
  51. package/src/core/background/triggers/exit.ts +54 -0
  52. package/src/core/background/triggers/index.ts +1 -3
  53. package/src/core/background/triggers/resume.ts +2 -4
  54. package/src/core/backup/archive/tar.ts +14 -4
  55. package/src/core/backup/restore.ts +21 -13
  56. package/src/core/backup/scheduler.ts +28 -9
  57. package/src/core/backup/snapshot.ts +43 -10
  58. package/src/core/backup/store.ts +7 -19
  59. package/src/core/backup/targets.ts +69 -17
  60. package/src/core/config/index.ts +14 -0
  61. package/src/core/daemon/crash-marker.ts +141 -0
  62. package/src/core/daemon/crash.ts +9 -2
  63. package/src/core/daemon/handoff.ts +15 -0
  64. package/src/core/daemon/health-alerts.ts +297 -0
  65. package/src/core/daemon/log-reader.ts +289 -0
  66. package/src/core/doctor/index.ts +18 -2
  67. package/src/core/doctor/logs.ts +124 -0
  68. package/src/core/doctor/types.ts +1 -1
  69. package/src/core/engine/backend-controller/index.ts +1 -13
  70. package/src/core/engine/backend-router/router.ts +1 -1
  71. package/src/core/engine/dispatcher.ts +55 -2
  72. package/src/core/engine/fault-text.ts +40 -0
  73. package/src/core/engine/gateway-actions/agents/index.ts +3 -2
  74. package/src/core/engine/gateway-actions/agents/report.ts +62 -0
  75. package/src/core/engine/gateway-actions/history.ts +2 -4
  76. package/src/core/engine/gateway-actions/native/exec.ts +13 -16
  77. package/src/core/engine/gateway.ts +60 -1
  78. package/src/core/engine/turn-health.ts +222 -0
  79. package/src/core/errors.ts +2 -2
  80. package/src/core/frontend-runtime/admin-notify.ts +1 -1
  81. package/src/core/frontend-runtime/alerts.ts +130 -0
  82. package/src/core/mcp-hub/children.ts +78 -29
  83. package/src/core/mcp-hub/index.ts +21 -18
  84. package/src/core/mcp-hub/proxy-server.ts +8 -4
  85. package/src/core/mcp-hub/talon-server.ts +5 -12
  86. package/src/core/mesh/credentials/store.ts +16 -1
  87. package/src/core/mesh/devices/service.ts +26 -15
  88. package/src/core/mesh/devices/teleport.ts +14 -2
  89. package/src/core/mesh/links/node-binaries.ts +13 -6
  90. package/src/core/mesh/persist.ts +22 -10
  91. package/src/core/mesh/transfers/device-files.ts +5 -23
  92. package/src/core/mesh/transfers/transfers.ts +16 -3
  93. package/src/core/models/active-model.ts +2 -55
  94. package/src/core/plugin/actions.ts +19 -20
  95. package/src/core/plugin/builtins.ts +80 -90
  96. package/src/core/plugin/index.ts +1 -4
  97. package/src/core/plugin/loader.ts +25 -33
  98. package/src/core/plugin/mcp.ts +3 -5
  99. package/src/core/plugin/registry.ts +19 -35
  100. package/src/core/plugin/types.ts +2 -5
  101. package/src/core/prompt/assemble.ts +15 -3
  102. package/src/core/scripts/lua.ts +6 -2
  103. package/src/core/tasks/table.ts +8 -2
  104. package/src/core/tools/bridge.ts +2 -4
  105. package/src/core/tools/chat/cross-send.ts +1 -1
  106. package/src/core/tools/chat/messaging.ts +1 -1
  107. package/src/core/tools/index.ts +2 -2
  108. package/src/core/tools/mcp-env.ts +2 -59
  109. package/src/core/tools/ops/agents.ts +22 -1
  110. package/src/core/tools/schemas.ts +4 -9
  111. package/src/core/vfs/fusefs.ts +0 -5
  112. package/src/core/vfs/index.ts +9 -2
  113. package/src/core/vfs/mounts/diagnostics.ts +109 -0
  114. package/src/core/vfs/mounts/proc.ts +17 -1
  115. package/src/core/vfs/workspace.ts +7 -3
  116. package/src/core/weaver/shuttle.ts +8 -1
  117. package/src/core/weaver/turn-log.ts +320 -0
  118. package/src/core/weaver/weaver.ts +48 -5
  119. package/src/frontend/discord/actions/index.ts +8 -1
  120. package/src/frontend/discord/diagnostics.ts +82 -6
  121. package/src/frontend/discord/handlers/index.ts +0 -2
  122. package/src/frontend/discord/middleware.ts +7 -13
  123. package/src/frontend/discord/runtime.ts +1 -3
  124. package/src/frontend/health/delivery.ts +115 -0
  125. package/src/frontend/health/outage.ts +116 -0
  126. package/src/frontend/native/bridge/routes/chats.ts +3 -5
  127. package/src/frontend/native/bridge/server.ts +106 -21
  128. package/src/frontend/native/index.ts +1 -1
  129. package/src/frontend/native/media/media.ts +5 -1
  130. package/src/frontend/native/runtime.ts +12 -7
  131. package/src/frontend/native/surface/handlers.ts +1 -1
  132. package/src/frontend/native/surface/memory.ts +1 -1
  133. package/src/frontend/native/surface/models.ts +3 -3
  134. package/src/frontend/native/surface/settings.ts +20 -8
  135. package/src/frontend/native/turn/context.ts +6 -8
  136. package/src/frontend/native/turn/turn-meta.ts +2 -5
  137. package/src/frontend/native/turn/turn.ts +8 -10
  138. package/src/frontend/presentation/format.ts +2 -4
  139. package/src/frontend/presentation/session-status.ts +2 -6
  140. package/src/frontend/teams/actions.ts +8 -1
  141. package/src/frontend/teams/graph.ts +0 -1
  142. package/src/frontend/teams/index.ts +1 -4
  143. package/src/frontend/teams/poll.ts +40 -2
  144. package/src/frontend/teams/runtime.ts +14 -5
  145. package/src/frontend/telegram/actions/index.ts +4 -1
  146. package/src/frontend/telegram/actions/send.ts +8 -0
  147. package/src/frontend/telegram/handlers/context.ts +13 -2
  148. package/src/frontend/telegram/handlers/delivery.ts +12 -9
  149. package/src/frontend/telegram/handlers/index.ts +0 -2
  150. package/src/frontend/telegram/index.ts +35 -9
  151. package/src/frontend/telegram/polling/poll-health.ts +110 -0
  152. package/src/frontend/telegram/userbot.ts +100 -36
  153. package/src/frontend/terminal/builtins/session.ts +2 -2
  154. package/src/frontend/terminal/index.ts +1 -3
  155. package/src/frontend/terminal/renderer.ts +2 -18
  156. package/src/frontend/whatsapp/actions/index.ts +12 -1
  157. package/src/frontend/whatsapp/actions/messaging.ts +7 -2
  158. package/src/frontend/whatsapp/connection/connection.ts +29 -9
  159. package/src/frontend/whatsapp/connection/health.ts +89 -0
  160. package/src/frontend/whatsapp/connection/identity.ts +4 -4
  161. package/src/frontend/whatsapp/runtime.ts +11 -5
  162. package/src/native/blake3.ts +28 -2
  163. package/src/native/fusefs.ts +23 -5
  164. package/src/native/registry.ts +1 -1
  165. package/src/native/warden.ts +33 -5
  166. package/src/plugins/github/index.ts +0 -1
  167. package/src/plugins/mempalace/index.ts +9 -3
  168. package/src/plugins/playwright/index.ts +2 -4
  169. package/src/plugins/playwright/provision.ts +12 -4
  170. package/src/storage/chat-settings.ts +6 -1
  171. package/src/storage/cron.ts +29 -4
  172. package/src/storage/daily-log.ts +43 -47
  173. package/src/storage/db.ts +61 -33
  174. package/src/storage/history.ts +6 -1
  175. package/src/storage/journal.ts +9 -2
  176. package/src/storage/kv.ts +19 -6
  177. package/src/storage/media-index.ts +28 -6
  178. package/src/storage/repositories/chat-settings-repo.ts +8 -2
  179. package/src/storage/repositories/sessions-repo.ts +10 -3
  180. package/src/storage/scripts.ts +24 -13
  181. package/src/storage/sessions.ts +11 -2
  182. package/src/storage/skills.ts +21 -2
  183. package/src/storage/stickers.ts +17 -3
  184. package/src/storage/triggers.ts +8 -3
  185. package/src/storage/turn-meta.ts +25 -7
  186. package/src/util/log.ts +189 -6
  187. package/src/util/logging/turn-scope.ts +85 -0
  188. package/src/util/time.ts +3 -3
  189. package/src/util/watchdog.ts +30 -0
  190. package/src/core/engine/backend-controller/legacy.ts +0 -111
@@ -27,6 +27,7 @@ import {
27
27
  type AgentResult,
28
28
  } from "../agent-runtime/events.js";
29
29
  import { recordTurnActivity } from "../../util/watchdog.js";
30
+ import type { TurnToolLog } from "./turn-log.js";
30
31
 
31
32
  export type EventSink = (event: AgentEvent) => void | Promise<void>;
32
33
 
@@ -45,12 +46,14 @@ export const startShuttleTiming = (): ShuttleTiming => ({ deliveryMs: 0 });
45
46
  /**
46
47
  * Pump the stream to completion. Returns the `completed` event's
47
48
  * result (if the backend emitted one); throws `AgentRunError` when the
48
- * stream terminates with an `error` event.
49
+ * stream terminates with an `error` event. `tools`, when given, sees
50
+ * every tool call/result pair for the turn's `tool.call` log lines.
49
51
  */
50
52
  export async function carryTurnEvents(
51
53
  stream: AsyncIterable<AgentEvent>,
52
54
  onEvent?: EventSink,
53
55
  timing?: ShuttleTiming,
56
+ tools?: TurnToolLog,
54
57
  ): Promise<AgentResult | undefined> {
55
58
  let agentResult: AgentResult | undefined;
56
59
  const sink: EventSink | undefined =
@@ -73,6 +76,10 @@ export async function carryTurnEvents(
73
76
  }
74
77
  if (event.type === "completed") {
75
78
  agentResult = event.result;
79
+ } else if (event.type === "tool_call") {
80
+ tools?.onCall(event);
81
+ } else if (event.type === "tool_result") {
82
+ tools?.onResult(event);
76
83
  }
77
84
 
78
85
  if (event.type === "assistant_message" && event.deliveryAck) {
@@ -0,0 +1,320 @@
1
+ /**
2
+ * Turn log — the greppable lifecycle lines for one turn, so talon.log
3
+ * alone can answer "what happened in that conversation":
4
+ *
5
+ * turn.queued turn=… chat=… trigger=… depth=… (behind a running turn)
6
+ * turn.start turn=… chat=… frontend=… backend=… model=… trigger=… queue=… wait_ms=…
7
+ * tool.start turn=… chat=… name=… id=… args=<≤80> (debug)
8
+ * tool.slow turn=… chat=… name=… id=… running_ms=… (still running at 60s)
9
+ * tool.call turn=… chat=… name=… ms=… ok=… [bytes=…] [err=…]
10
+ * turn.error turn=… chat=… class=auth|quota|timeout|tool|backend|unknown msg=…
11
+ * turn.end turn=… chat=… … ms=… outcome=ok|error|aborted|timeout|refused tools=N in_tokens=…
12
+ *
13
+ * Every other line written during the turn carries `turn=<id>` too, via
14
+ * the log context (util/logging/turn-scope.ts). Free text (`msg=`, `err=`) is
15
+ * always the LAST field, collapsed to one line and capped, so the
16
+ * key=value fields before it stay machine-parseable.
17
+ */
18
+
19
+ import { AgentRunError, type AgentEvent } from "../agent-runtime/events.js";
20
+ import { classify, type TalonError } from "../errors.js";
21
+ import { resolveOwnerFrontendId } from "../frontend-runtime/routing.js";
22
+ import { log, logDebug, logWarn } from "../../util/log.js";
23
+
24
+ type TurnOutcome = "ok" | "error" | "aborted" | "timeout" | "refused";
25
+
26
+ type TurnErrorClass =
27
+ "auth" | "quota" | "timeout" | "tool" | "backend" | "aborted" | "unknown";
28
+
29
+ /** A tool still running at this age gets a `tool.slow` line. */
30
+ const SLOW_TOOL_MS = 60_000;
31
+ /** Cap for free-text fields (error messages). */
32
+ const MAX_TEXT = 200;
33
+ /** Cap for the debug-level tool args preview. */
34
+ const MAX_ARGS_PREVIEW = 80;
35
+
36
+ /** One line, capped — safe to put at the end of a key=value log line. */
37
+ function oneLine(text: string, max = MAX_TEXT): string {
38
+ const flat = text.replace(/\s+/g, " ").trim();
39
+ return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
40
+ }
41
+
42
+ function errorText(err: unknown): string {
43
+ return oneLine(err instanceof Error ? err.message : String(err));
44
+ }
45
+
46
+ /** Serialized size of a tool result, or undefined when there is none. */
47
+ function resultBytes(result: unknown): number | undefined {
48
+ if (result === undefined) return undefined;
49
+ if (typeof result === "string") return Buffer.byteLength(result);
50
+ try {
51
+ return Buffer.byteLength(JSON.stringify(result) ?? "");
52
+ } catch {
53
+ return undefined;
54
+ }
55
+ }
56
+
57
+ function argsPreview(input: unknown): string {
58
+ try {
59
+ return oneLine(JSON.stringify(input) ?? "", MAX_ARGS_PREVIEW);
60
+ } catch {
61
+ return "[unserializable]";
62
+ }
63
+ }
64
+
65
+ type OpenToolCall = {
66
+ name: string;
67
+ startedAt: number;
68
+ slowTimer: ReturnType<typeof setTimeout>;
69
+ };
70
+
71
+ /**
72
+ * Per-turn tool-call ledger, fed by the Shuttle from the canonical
73
+ * `tool_call` / `tool_result` events — the one place every backend's
74
+ * tools (SDK built-ins and Talon's MCP tools alike) pass through.
75
+ */
76
+ export class TurnToolLog {
77
+ private readonly turnId: string;
78
+ private readonly chatId: string;
79
+ private readonly slowMs: number;
80
+ private readonly open = new Map<string, OpenToolCall>();
81
+ private calls = 0;
82
+
83
+ constructor(turnId: string, chatId: string, slowMs = SLOW_TOOL_MS) {
84
+ this.turnId = turnId;
85
+ this.chatId = chatId;
86
+ this.slowMs = slowMs;
87
+ }
88
+
89
+ /** Tool calls seen this turn. */
90
+ get count(): number {
91
+ return this.calls;
92
+ }
93
+
94
+ onCall(event: Extract<AgentEvent, { type: "tool_call" }>): void {
95
+ this.calls++;
96
+ const prefix = `turn=${this.turnId} chat=${this.chatId} name=${event.name} id=${event.id}`;
97
+ logDebug(
98
+ "dispatcher",
99
+ `tool.start ${prefix} args=${argsPreview(event.input)}`,
100
+ );
101
+ const startedAt = Date.now();
102
+ const slowTimer = setTimeout(() => {
103
+ log(
104
+ "dispatcher",
105
+ `tool.slow ${prefix} running_ms=${Date.now() - startedAt}`,
106
+ );
107
+ }, this.slowMs);
108
+ // A tool that outlives its turn must not hold the process open.
109
+ slowTimer.unref?.();
110
+ const previous = this.open.get(event.id);
111
+ if (previous) clearTimeout(previous.slowTimer);
112
+ this.open.set(event.id, { name: event.name, startedAt, slowTimer });
113
+ }
114
+
115
+ onResult(event: Extract<AgentEvent, { type: "tool_result" }>): void {
116
+ const call = this.open.get(event.id);
117
+ if (call) {
118
+ clearTimeout(call.slowTimer);
119
+ this.open.delete(event.id);
120
+ }
121
+ const ms = call ? Date.now() - call.startedAt : 0;
122
+ const bytes = resultBytes(event.result);
123
+ const line =
124
+ `tool.call turn=${this.turnId} chat=${this.chatId} name=${event.name} ms=${ms} ` +
125
+ `ok=${event.error === undefined}` +
126
+ (bytes === undefined ? "" : ` bytes=${bytes}`);
127
+ if (event.error === undefined) log("dispatcher", line);
128
+ else logWarn("dispatcher", `${line} err=${oneLine(event.error)}`);
129
+ }
130
+
131
+ /** Turn settled: stop the slow-tool timers, note calls that never resolved. */
132
+ close(): void {
133
+ for (const [id, call] of this.open) {
134
+ clearTimeout(call.slowTimer);
135
+ logDebug(
136
+ "dispatcher",
137
+ `tool.unsettled turn=${this.turnId} chat=${this.chatId} name=${call.name} id=${id} ms=${Date.now() - call.startedAt}`,
138
+ );
139
+ }
140
+ this.open.clear();
141
+ }
142
+ }
143
+
144
+ /** What one turn's lifecycle lines report — filled in as the turn resolves. */
145
+ export type TurnTrace = {
146
+ readonly turnId: string;
147
+ readonly chatId: string;
148
+ readonly trigger: string;
149
+ readonly frontend: string;
150
+ /** Turns queued or running ahead of this one when it was enqueued. */
151
+ readonly queuedBehind: number;
152
+ readonly enqueuedAt: number;
153
+ readonly tools: TurnToolLog;
154
+ startedAt?: number;
155
+ backend?: string;
156
+ model?: string;
157
+ /** Why the turn was answered with a refusal instead of a backend run. */
158
+ refused?: string;
159
+ };
160
+
161
+ function head(trace: TurnTrace): string {
162
+ return `turn=${trace.turnId} chat=${trace.chatId}`;
163
+ }
164
+
165
+ /**
166
+ * The trace for a turn being enqueued. `queuedBehind` is how many turns
167
+ * the chat's FIFO already holds; when non-zero the turn has to wait, and
168
+ * that is logged now — the wait is otherwise invisible until it starts.
169
+ */
170
+ export function createTurnTrace(
171
+ turnId: string,
172
+ params: { chatId: string; source: string },
173
+ queuedBehind: number,
174
+ ): TurnTrace {
175
+ const trace: TurnTrace = {
176
+ turnId,
177
+ chatId: params.chatId,
178
+ trigger: params.source,
179
+ frontend:
180
+ resolveOwnerFrontendId(params.chatId, { includeNonMessaging: true }) ??
181
+ "unknown",
182
+ queuedBehind,
183
+ enqueuedAt: Date.now(),
184
+ tools: new TurnToolLog(turnId, params.chatId),
185
+ };
186
+ if (queuedBehind > 0) logTurnQueued(trace);
187
+ return trace;
188
+ }
189
+
190
+ /** A turn is waiting behind others in its chat's FIFO. */
191
+ function logTurnQueued(trace: TurnTrace): void {
192
+ log(
193
+ "dispatcher",
194
+ `turn.queued ${head(trace)} trigger=${trace.trigger} depth=${trace.queuedBehind}`,
195
+ );
196
+ }
197
+
198
+ /**
199
+ * The turn resolved its model and is about to reach the backend (or
200
+ * refuse): bind the resolution onto the trace and write `turn.start`.
201
+ */
202
+ export function logTurnStart(
203
+ trace: TurnTrace,
204
+ warp: { backendId: string; model?: string },
205
+ ): void {
206
+ trace.backend = warp.backendId;
207
+ trace.model = warp.model;
208
+ const waitMs = (trace.startedAt ?? trace.enqueuedAt) - trace.enqueuedAt;
209
+ log(
210
+ "dispatcher",
211
+ `turn.start ${head(trace)} frontend=${trace.frontend} ` +
212
+ `backend=${trace.backend ?? "?"} model=${trace.model ?? "none"} ` +
213
+ `trigger=${trace.trigger} queue=${trace.queuedBehind} wait_ms=${waitMs}`,
214
+ );
215
+ }
216
+
217
+ type TurnUsage = {
218
+ inputTokens: number;
219
+ outputTokens: number;
220
+ cacheRead: number;
221
+ cacheWrite: number;
222
+ };
223
+
224
+ /**
225
+ * The turn returned normally: `ok`, a refusal, or a kill the backend
226
+ * honoured with a clean early completion.
227
+ */
228
+ export function logTurnSettled(
229
+ trace: TurnTrace,
230
+ killed: boolean,
231
+ usage: TurnUsage,
232
+ ): void {
233
+ if (killed) logTurnEnd(trace, "aborted", { usage, reason: "killed" });
234
+ else if (trace.refused)
235
+ logTurnEnd(trace, "refused", { usage, reason: trace.refused });
236
+ else logTurnEnd(trace, "ok", { usage });
237
+ }
238
+
239
+ /** The single closing line of every turn, whatever its outcome. */
240
+ export function logTurnEnd(
241
+ trace: TurnTrace,
242
+ outcome: TurnOutcome,
243
+ opts: { usage?: TurnUsage; reason?: string } = {},
244
+ ): void {
245
+ const ms = Date.now() - (trace.startedAt ?? trace.enqueuedAt);
246
+ const usage = opts.usage;
247
+ const line =
248
+ `turn.end ${head(trace)} backend=${trace.backend ?? "?"} ` +
249
+ `model=${trace.model ?? "none"} trigger=${trace.trigger} ms=${ms} ` +
250
+ `outcome=${outcome} tools=${trace.tools.count}` +
251
+ (usage
252
+ ? ` in_tokens=${usage.inputTokens} out_tokens=${usage.outputTokens} ` +
253
+ `cache_read=${usage.cacheRead} cache_write=${usage.cacheWrite}`
254
+ : "") +
255
+ (opts.reason ? ` reason=${opts.reason}` : "");
256
+ log("dispatcher", line);
257
+ }
258
+
259
+ const AGENT_KIND_CLASS: Record<AgentRunError["kind"], TurnErrorClass> = {
260
+ context_overflow: "backend",
261
+ rate_limit: "quota",
262
+ overload: "backend",
263
+ session_expired: "backend",
264
+ auth: "auth",
265
+ model_unsupported: "backend",
266
+ tool_failure: "tool",
267
+ subprocess_exit: "backend",
268
+ timeout: "timeout",
269
+ aborted: "aborted",
270
+ // Re-read through classify(): the message may still say more.
271
+ unknown: "unknown",
272
+ };
273
+
274
+ const REASON_CLASS: Record<TalonError["reason"], TurnErrorClass> = {
275
+ rate_limit: "quota",
276
+ usage_limit: "quota",
277
+ overloaded: "backend",
278
+ network: "backend",
279
+ auth: "auth",
280
+ forbidden: "auth",
281
+ context_length: "backend",
282
+ session_expired: "backend",
283
+ bad_request: "backend",
284
+ telegram_api: "unknown",
285
+ stopped: "aborted",
286
+ unknown: "unknown",
287
+ };
288
+
289
+ /** Bucket a turn failure into the coarse class `turn.error` reports. */
290
+ export function classifyTurnError(err: unknown): TurnErrorClass {
291
+ if (err instanceof AgentRunError && err.kind !== "unknown") {
292
+ return AGENT_KIND_CLASS[err.kind];
293
+ }
294
+ if (err instanceof Error && err.name === "TimeoutError") return "timeout";
295
+ return REASON_CLASS[classify(err).reason];
296
+ }
297
+
298
+ /**
299
+ * A turn threw. A kill the user asked for closes as `aborted` with no
300
+ * `turn.error` (nothing went wrong); anything else gets the classified
301
+ * `turn.error` line, then `turn.end`.
302
+ */
303
+ export function logTurnFailure(
304
+ trace: TurnTrace,
305
+ err: unknown,
306
+ killed: boolean,
307
+ ): void {
308
+ if (killed) {
309
+ logTurnEnd(trace, "aborted", { reason: "killed" });
310
+ return;
311
+ }
312
+ const cls = classifyTurnError(err);
313
+ logWarn(
314
+ "dispatcher",
315
+ `turn.error ${head(trace)} backend=${trace.backend ?? "?"} class=${cls} msg=${errorText(err)}`,
316
+ );
317
+ const outcome: TurnOutcome =
318
+ cls === "timeout" ? "timeout" : cls === "aborted" ? "aborted" : "error";
319
+ logTurnEnd(trace, outcome);
320
+ }
@@ -41,6 +41,19 @@ import type { Thread, ThreadSnapshot } from "./thread.js";
41
41
  import { startTurnCpu } from "./turn-cpu.js";
42
42
  import { startTypingLoop } from "./typing-loop.js";
43
43
  import { resolveWarp } from "./warp-resolver.js";
44
+ import {
45
+ createTurnTrace,
46
+ logTurnEnd,
47
+ logTurnFailure,
48
+ logTurnSettled,
49
+ logTurnStart,
50
+ type TurnTrace,
51
+ } from "./turn-log.js";
52
+ import {
53
+ closeTurnScope,
54
+ createTurnScope,
55
+ runInTurnScope,
56
+ } from "../../util/logging/turn-scope.js";
44
57
 
45
58
  export type WeaverDeps = {
46
59
  /**
@@ -84,6 +97,10 @@ export class Weaver {
84
97
  const chat = this.deps.getBackend(params.chatId).chat;
85
98
  const interrupt = chat?.interruptChatTurn?.bind(chat);
86
99
  const lifecycle = { started: false, killed: false, enqueuedAt: Date.now() };
100
+ // The turn id is minted at enqueue so a queued turn's wait is already
101
+ // attributable; the log scope goes live when the turn starts running.
102
+ const scope = createTurnScope(params.chatId);
103
+ const trace = createTurnTrace(scope.turnId, params, thread.inFlightCount);
87
104
  // Registered before enqueueing so a turn waiting in its chat's FIFO is
88
105
  // visible as `queued` in the task table, not invisible until it runs.
89
106
  const task = taskTable.enqueue({
@@ -99,7 +116,11 @@ export class Weaver {
99
116
  }
100
117
  : {}),
101
118
  });
102
- return thread.enqueue(() => this.run(thread, params, task, lifecycle));
119
+ return thread.enqueue(() =>
120
+ runInTurnScope(scope, () =>
121
+ this.run(thread, params, task, lifecycle, trace),
122
+ ).finally(() => closeTurnScope(scope)),
123
+ );
103
124
  }
104
125
 
105
126
  /** Number of turns currently running (not queued) across all chats. */
@@ -121,21 +142,28 @@ export class Weaver {
121
142
  params: ExecuteParams,
122
143
  task: TaskHandle,
123
144
  lifecycle: { started: boolean; killed: boolean; enqueuedAt: number },
145
+ trace: TurnTrace,
124
146
  ): Promise<ExecuteResult> {
125
147
  if (lifecycle.killed) {
126
148
  // Killed while queued — the turn never reaches the backend. The
127
149
  // caller still gets a resolved (empty) result; nothing is delivered
128
150
  // to the chat, which is the point of the kill.
129
151
  task.fail(new Error("killed while queued"));
152
+ logTurnEnd(trace, "aborted", { reason: "killed-while-queued" });
130
153
  return this.emptyResult("Turn killed before it started.", params);
131
154
  }
132
155
  lifecycle.started = true;
156
+ trace.startedAt = Date.now();
133
157
  this.activeCount++;
134
158
  task.start();
135
159
  try {
136
- const result = await this.executeInner(thread, params, task, {
137
- queueWait: Date.now() - lifecycle.enqueuedAt,
138
- });
160
+ const result = await this.executeInner(
161
+ thread,
162
+ params,
163
+ task,
164
+ { queueWait: Date.now() - lifecycle.enqueuedAt },
165
+ trace,
166
+ );
139
167
  const usage = {
140
168
  inputTokens: result.inputTokens,
141
169
  outputTokens: result.outputTokens,
@@ -152,9 +180,11 @@ export class Weaver {
152
180
  } else {
153
181
  task.succeed(usage);
154
182
  }
183
+ logTurnSettled(trace, lifecycle.killed, usage);
155
184
  return result;
156
185
  } catch (err) {
157
186
  task.fail(err);
187
+ logTurnFailure(trace, err, lifecycle.killed);
158
188
  if (lifecycle.killed) {
159
189
  // The backend didn't manage a clean interrupt-completion (some
160
190
  // SDK versions surface an interrupted turn as an error result).
@@ -168,6 +198,7 @@ export class Weaver {
168
198
  throw err;
169
199
  } finally {
170
200
  this.activeCount--;
201
+ trace.tools.close();
171
202
  }
172
203
  }
173
204
 
@@ -176,6 +207,7 @@ export class Weaver {
176
207
  params: ExecuteParams,
177
208
  task: TaskHandle,
178
209
  phases: Partial<Record<TurnPhase, number>>,
210
+ trace: TurnTrace,
179
211
  ): Promise<ExecuteResult> {
180
212
  const { context } = this.deps;
181
213
  const backend = this.deps.getBackend(params.chatId);
@@ -189,7 +221,12 @@ export class Weaver {
189
221
  reqId,
190
222
  });
191
223
  phases.warpResolve = Date.now() - warpStartedAt;
224
+ logTurnStart(trace, {
225
+ backendId: warp.backendId,
226
+ model: warp.ok ? warp.ref.id : undefined,
227
+ });
192
228
  if (!warp.ok) {
229
+ trace.refused = "no-model";
193
230
  await deliverRefusal(params, warp.message, "no-model");
194
231
  return this.emptyResult(warp.message, params);
195
232
  }
@@ -202,6 +239,7 @@ export class Weaver {
202
239
  "dispatcher",
203
240
  `[${reqId}] guest-scoped turn refused chat=${params.chatId}: backend "${backend.id}" cannot enforce the guest tool scope`,
204
241
  );
242
+ trace.refused = "guest-scope";
205
243
  await deliverRefusal(params, GUEST_BACKEND_REFUSAL, "guest-scope");
206
244
  return this.emptyResult(GUEST_BACKEND_REFUSAL, params);
207
245
  }
@@ -270,7 +308,12 @@ export class Weaver {
270
308
  // same path `phases.stream` is, so both cover one population.
271
309
  const stopCpu = startTurnCpu();
272
310
  const streamStartedAt = Date.now();
273
- const agentResult = await carryTurnEvents(stream, params.onEvent, timing);
311
+ const agentResult = await carryTurnEvents(
312
+ stream,
313
+ params.onEvent,
314
+ timing,
315
+ trace.tools,
316
+ );
274
317
  phases.stream = Date.now() - streamStartedAt;
275
318
  stopCpu();
276
319
  phases.delivery = timing.deliveryMs;
@@ -28,6 +28,12 @@ import { messagingHandlers, restoreScheduledMessages } from "./messaging.js";
28
28
  import { mediaHandlers } from "./media.js";
29
29
  import { chatInfoHandlers } from "./chat-info.js";
30
30
  import type { DiscordActionContext, DiscordActionHandlers } from "./types.js";
31
+ import {
32
+ createDeliveryTracker,
33
+ trackDeliveries,
34
+ } from "../../health/delivery.js";
35
+
36
+ const delivery = createDeliveryTracker("discord", "Discord", "discord");
31
37
 
32
38
  // Null-prototype so a request `action` of "toString" / "constructor" / etc.
33
39
  // can't resolve an inherited Object.prototype method via `handlers[action]`.
@@ -44,7 +50,7 @@ export function createDiscordActionHandler(client: Client, gateway: Gateway) {
44
50
  // shutdown — the timers died with the process, the store didn't.
45
51
  restoreScheduledMessages(client, scheduledMessages);
46
52
 
47
- return async (
53
+ const dispatch = async (
48
54
  body: Record<string, unknown>,
49
55
  chatId: number,
50
56
  ): Promise<ActionResult | null> => {
@@ -74,4 +80,5 @@ export function createDiscordActionHandler(client: Client, gateway: Gateway) {
74
80
  };
75
81
  return handler(body, chatId, ctx);
76
82
  };
83
+ return trackDeliveries(delivery, dispatch);
77
84
  }
@@ -1,12 +1,19 @@
1
1
  /**
2
- * Client diagnostics — the error/warn/disconnect/rate-limit listeners that
3
- * only log. discord.js handles reconnects and retries itself; these exist
4
- * so a silent failure mode (a fatal close code, an approaching IP ban) is
5
- * visible in the daemon log.
2
+ * Client diagnostics — the error/warn/disconnect/rate-limit listeners.
3
+ * discord.js handles reconnects and retries itself; these exist so a
4
+ * silent failure mode (a fatal close code, an approaching IP ban) is
5
+ * visible in the daemon log, and so the operator hears when the gateway
6
+ * stays down: `discord.gateway` is raised once a shard has been
7
+ * disconnected for `GATEWAY_OUTAGE_MS` (at once, critically, for a close
8
+ * code discord.js will never recover from) and resolved on ready/resume.
6
9
  */
7
10
 
8
11
  import { type Client, Events } from "discord.js";
9
- import { logError, logWarn } from "../../util/log.js";
12
+ import { log, logError, logWarn } from "../../util/log.js";
13
+ import { createOutage, errorText, type Outage } from "../health/outage.js";
14
+
15
+ /** A gateway down this long reaches the operator. */
16
+ const GATEWAY_OUTAGE_MS = 5 * 60_000;
10
17
 
11
18
  function shardDisconnectLabel(code: number | undefined): string {
12
19
  return code === 4004
@@ -18,7 +25,76 @@ function shardDisconnectLabel(code: number | undefined): string {
18
25
  : `code=${code}`;
19
26
  }
20
27
 
21
- export function bindClientDiagnostics(client: Client): void {
28
+ function createGatewayOutage(thresholdMs: number): Outage {
29
+ return createOutage({
30
+ key: "discord.gateway",
31
+ thresholdMs,
32
+ describe: (err, mins) =>
33
+ `The Discord gateway has been disconnected for ${mins} min: ${err}. Messages are not being received.`,
34
+ recovered: "The Discord gateway is connected again.",
35
+ });
36
+ }
37
+
38
+ /**
39
+ * Gateway connection health: which shards are down, and the outage that
40
+ * spans them. Reconnect attempts, errors and recoveries are logged with
41
+ * the shard, attempt number and time down.
42
+ */
43
+ function bindGatewayHealth(client: Client, thresholdMs: number): void {
44
+ const outage = createGatewayOutage(thresholdMs);
45
+ const down = new Set<number>();
46
+
47
+ const lost = (shardId: number, event: string, err: unknown): void => {
48
+ down.add(shardId);
49
+ const { attempt, downMs } = outage.fail(err);
50
+ logWarn(
51
+ "discord",
52
+ `gateway.${event} shard=${shardId} attempt=${attempt} down_ms=${downMs} err=${errorText(err)}`,
53
+ );
54
+ };
55
+ const back = (shardId: number, event: string): void => {
56
+ down.delete(shardId);
57
+ if (down.size > 0) return;
58
+ const ended = outage.ok();
59
+ if (ended) {
60
+ log(
61
+ "discord",
62
+ `gateway.${event} shard=${shardId} failed_attempts=${ended.attempts} down_ms=${ended.downMs}`,
63
+ );
64
+ }
65
+ };
66
+
67
+ client.on(Events.ShardReconnecting, (shardId) =>
68
+ lost(shardId, "reconnecting", "connection closed, reconnecting"),
69
+ );
70
+ client.on(Events.ShardError, (err, shardId) => {
71
+ // An error on a live shard is followed by a close if it matters; only
72
+ // an error during an outage is part of it (its text is the useful one).
73
+ if (down.has(shardId)) lost(shardId, "error", err);
74
+ });
75
+ client.on(Events.ShardDisconnect, (event, shardId) => {
76
+ const code = event?.code;
77
+ lost(shardId, "disconnect", shardDisconnectLabel(code));
78
+ if (code === 4004 || code === 4013 || code === 4014) {
79
+ outage.raiseNow(
80
+ `The Discord gateway closed and will not reconnect: ${shardDisconnectLabel(code)}. ` +
81
+ "Messages are not being received until this is fixed and Talon restarts.",
82
+ "critical",
83
+ );
84
+ }
85
+ });
86
+ client.on(Events.Invalidated, () =>
87
+ lost(0, "invalidated", "session invalidated"),
88
+ );
89
+ client.on(Events.ShardReady, (shardId) => back(shardId, "ready"));
90
+ client.on(Events.ShardResume, (shardId) => back(shardId, "resumed"));
91
+ }
92
+
93
+ export function bindClientDiagnostics(
94
+ client: Client,
95
+ gatewayOutageMs = GATEWAY_OUTAGE_MS,
96
+ ): void {
97
+ bindGatewayHealth(client, gatewayOutageMs);
22
98
  client.on("error", (err) => {
23
99
  logError("discord", "Client error", err);
24
100
  });
@@ -11,8 +11,6 @@
11
11
  * - `delivery` — the agent run + reply pipeline (processAndReply)
12
12
  * - `queue` — per-chat debounce queue
13
13
  * - `messages` — the messageCreate handler (handleMessage)
14
- *
15
- * Re-exports the same public surface the old single-file module exposed.
16
14
  */
17
15
 
18
16
  export { registerDiscordChat, lookupDiscordChat } from "./registry.js";
@@ -1,17 +1,11 @@
1
1
  /**
2
- * Discord middleware — wires up message events to handlers.ts.
2
+ * Discord middleware — wires up message events to handlers/.
3
3
  *
4
- * Equivalent to src/frontend/telegram/middleware.ts. We attach a single
5
- * messageCreate listener that filters out bots/system messages and delegates
6
- * to handleMessage. We also push every message into the in-memory history
7
- * buffer so /admin commands and /status reflect real activity.
8
- *
9
- * Discord-specific behavior:
10
- * - We don't have a separate "my_chat_member" event; instead, when the bot
11
- * is removed from a guild we get `guildDelete`. The handler in index.ts
12
- * handles that to revoke access.
13
- * - Every guild message gets the chat registered for pulse so periodic
14
- * check-ins work (DMs are excluded because we always respond).
4
+ * A single messageCreate listener filters out bots/system messages and
5
+ * delegates to handleMessage. Every message is also pushed into the
6
+ * in-memory history buffer so /admin commands and /status reflect real
7
+ * activity, and every guild message registers its chat for pulse (DMs are
8
+ * excluded because we always respond).
15
9
  */
16
10
 
17
11
  import type { Client, Message } from "discord.js";
@@ -24,7 +18,7 @@ import { handleMessage, getSenderName } from "./handlers/index.js";
24
18
 
25
19
  export function registerMiddleware(client: Client, config: TalonConfig): void {
26
20
  client.on("messageCreate", (msg: Message) => {
27
- // Ignore self/bots/system here too — handlers.ts will check again, but
21
+ // Ignore self/bots/system here too — handleMessage checks again, but
28
22
  // we don't even want to record those in history.
29
23
  if (msg.author.bot || msg.system) return;
30
24
  if (msg.author.id === client.user?.id) return;
@@ -1,9 +1,7 @@
1
1
  /**
2
2
  * Discord frontend runtime — the state every module of this frontend shares.
3
3
  *
4
- * `createDiscordFrontend` used to hold the client and config as closure
5
- * variables with every gateway-event handler nested inside `init`. It is
6
- * now one explicit object, constructed once, that each module (ready,
4
+ * One explicit object, constructed once, that each module (ready,
7
5
  * guild-policy, diagnostics, outbound, connection) takes as its first
8
6
  * parameter. The runtime carries state only; the modules own the behaviour.
9
7
  */