talon-agent 5.18.2 → 5.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +2 -1
  2. package/package.json +2 -2
  3. package/prompts/system/agent-brief.md +20 -3
  4. package/src/app.ts +13 -0
  5. package/src/backend/claude-sdk/handler.ts +4 -4
  6. package/src/backend/claude-sdk/mcp-ready.ts +16 -2
  7. package/src/backend/claude-sdk/one-shot.ts +32 -2
  8. package/src/backend/claude-sdk/stream.ts +2 -2
  9. package/src/backend/codex/auth.ts +1 -1
  10. package/src/backend/codex/handler/message.ts +11 -11
  11. package/src/backend/codex/init.ts +4 -9
  12. package/src/backend/codex/mcp-config.ts +1 -2
  13. package/src/backend/codex/oauth-incompat.ts +8 -4
  14. package/src/backend/codex/one-shot.ts +1 -1
  15. package/src/backend/openai-agents/builtins.ts +55 -27
  16. package/src/backend/openai-agents/factory.ts +3 -3
  17. package/src/backend/openai-agents/handler/message.ts +3 -5
  18. package/src/backend/openai-agents/mcp-pool.ts +10 -27
  19. package/src/backend/remote-server/chat-turn.ts +6 -6
  20. package/src/backend/remote-server/events.ts +1 -5
  21. package/src/backend/remote-server/index.ts +0 -1
  22. package/src/backend/remote-server/messages.ts +3 -7
  23. package/src/backend/remote-server/one-shot.ts +1 -3
  24. package/src/backend/remote-server/session-helpers.ts +1 -4
  25. package/src/backend/remote-server/sse-stream.ts +8 -9
  26. package/src/backend/runtime/metrics.ts +7 -13
  27. package/src/backend/runtime/sleep.ts +1 -2
  28. package/src/backend/runtime/turn/handle-retry.ts +48 -2
  29. package/src/backend/runtime/turn/handler-to-events.ts +3 -4
  30. package/src/bootstrap.ts +9 -1
  31. package/src/cli/doctor.ts +3 -0
  32. package/src/cli/index.ts +8 -10
  33. package/src/cli/logs.ts +148 -9
  34. package/src/cli/setup.ts +9 -11
  35. package/src/cli/status.ts +29 -0
  36. package/src/core/agent-runtime/README.md +5 -19
  37. package/src/core/agent-runtime/events.ts +3 -39
  38. package/src/core/agent-runtime/model-ref.ts +0 -8
  39. package/src/core/agents/registry.ts +65 -2
  40. package/src/core/auth/expiry-monitor.ts +9 -1
  41. package/src/core/auth/login-flow.ts +9 -1
  42. package/src/core/auth/status.ts +31 -3
  43. package/src/core/background/cron/scheduler.ts +25 -5
  44. package/src/core/background/dream/index.ts +29 -10
  45. package/src/core/background/failure-backoff.ts +30 -0
  46. package/src/core/background/heartbeat/agent.ts +2 -27
  47. package/src/core/background/heartbeat/index.ts +0 -2
  48. package/src/core/background/heartbeat/scheduler.ts +21 -13
  49. package/src/core/background/heartbeat/state.ts +10 -2
  50. package/src/core/background/isolated-agent.ts +6 -2
  51. package/src/core/background/pulse/pulse.ts +9 -0
  52. package/src/core/background/triggers/exit.ts +54 -0
  53. package/src/core/background/triggers/index.ts +1 -3
  54. package/src/core/background/triggers/resume.ts +2 -4
  55. package/src/core/backup/archive/tar.ts +14 -4
  56. package/src/core/backup/plan.ts +1 -0
  57. package/src/core/backup/restore.ts +21 -13
  58. package/src/core/backup/scheduler.ts +28 -9
  59. package/src/core/backup/snapshot.ts +43 -10
  60. package/src/core/backup/store.ts +7 -19
  61. package/src/core/backup/targets.ts +69 -17
  62. package/src/core/config/index.ts +14 -0
  63. package/src/core/daemon/crash-marker.ts +141 -0
  64. package/src/core/daemon/crash.ts +9 -2
  65. package/src/core/daemon/handoff.ts +15 -0
  66. package/src/core/daemon/health-alerts.ts +297 -0
  67. package/src/core/daemon/log-reader.ts +289 -0
  68. package/src/core/doctor/index.ts +18 -2
  69. package/src/core/doctor/logs.ts +124 -0
  70. package/src/core/doctor/types.ts +1 -1
  71. package/src/core/engine/backend-controller/index.ts +1 -13
  72. package/src/core/engine/backend-router/router.ts +1 -1
  73. package/src/core/engine/dispatcher.ts +55 -2
  74. package/src/core/engine/fault-text.ts +40 -0
  75. package/src/core/engine/gateway-actions/agents/index.ts +3 -2
  76. package/src/core/engine/gateway-actions/agents/report.ts +62 -0
  77. package/src/core/engine/gateway-actions/history.ts +2 -4
  78. package/src/core/engine/gateway-actions/mesh.ts +26 -14
  79. package/src/core/engine/gateway-actions/native/exec.ts +13 -16
  80. package/src/core/engine/gateway-actions/native/teleport.ts +1 -1
  81. package/src/core/engine/gateway.ts +60 -1
  82. package/src/core/engine/turn-health.ts +222 -0
  83. package/src/core/errors.ts +2 -2
  84. package/src/core/frontend-runtime/admin-notify.ts +1 -1
  85. package/src/core/frontend-runtime/alerts.ts +130 -0
  86. package/src/core/mcp-hub/children.ts +78 -29
  87. package/src/core/mcp-hub/index.ts +21 -18
  88. package/src/core/mcp-hub/proxy-server.ts +8 -4
  89. package/src/core/mcp-hub/talon-server.ts +5 -12
  90. package/src/core/mesh/credentials/store.ts +16 -1
  91. package/src/core/mesh/devices/registry.ts +1 -17
  92. package/src/core/mesh/devices/service.ts +49 -17
  93. package/src/core/mesh/devices/teleport.ts +14 -2
  94. package/src/core/mesh/links/node-binaries.ts +13 -6
  95. package/src/core/mesh/persist.ts +22 -10
  96. package/src/core/mesh/transfers/device-files.ts +5 -23
  97. package/src/core/mesh/transfers/transfers.ts +16 -3
  98. package/src/core/models/active-model.ts +2 -55
  99. package/src/core/plugin/actions.ts +19 -20
  100. package/src/core/plugin/builtins.ts +80 -90
  101. package/src/core/plugin/index.ts +1 -4
  102. package/src/core/plugin/loader.ts +25 -33
  103. package/src/core/plugin/mcp.ts +3 -5
  104. package/src/core/plugin/registry.ts +19 -35
  105. package/src/core/plugin/types.ts +2 -5
  106. package/src/core/prompt/assemble.ts +15 -3
  107. package/src/core/scripts/lua.ts +6 -2
  108. package/src/core/tasks/table.ts +8 -2
  109. package/src/core/tools/bridge.ts +2 -4
  110. package/src/core/tools/chat/cross-send.ts +1 -1
  111. package/src/core/tools/chat/messaging.ts +1 -1
  112. package/src/core/tools/index.ts +2 -2
  113. package/src/core/tools/mcp-env.ts +2 -59
  114. package/src/core/tools/ops/agents.ts +22 -1
  115. package/src/core/tools/schemas.ts +4 -9
  116. package/src/core/vfs/fusefs.ts +0 -5
  117. package/src/core/vfs/index.ts +9 -2
  118. package/src/core/vfs/mounts/diagnostics.ts +109 -0
  119. package/src/core/vfs/mounts/proc.ts +17 -1
  120. package/src/core/vfs/workspace.ts +7 -3
  121. package/src/core/weaver/shuttle.ts +8 -1
  122. package/src/core/weaver/turn-log.ts +320 -0
  123. package/src/core/weaver/weaver.ts +48 -5
  124. package/src/frontend/discord/actions/index.ts +8 -1
  125. package/src/frontend/discord/diagnostics.ts +82 -6
  126. package/src/frontend/discord/handlers/index.ts +0 -2
  127. package/src/frontend/discord/middleware.ts +7 -13
  128. package/src/frontend/discord/runtime.ts +1 -3
  129. package/src/frontend/health/delivery.ts +115 -0
  130. package/src/frontend/health/outage.ts +116 -0
  131. package/src/frontend/native/bridge/routes/chats.ts +3 -5
  132. package/src/frontend/native/bridge/server.ts +106 -21
  133. package/src/frontend/native/index.ts +1 -1
  134. package/src/frontend/native/media/media.ts +5 -1
  135. package/src/frontend/native/runtime.ts +12 -7
  136. package/src/frontend/native/surface/handlers.ts +1 -1
  137. package/src/frontend/native/surface/memory.ts +1 -1
  138. package/src/frontend/native/surface/models.ts +3 -3
  139. package/src/frontend/native/surface/settings.ts +20 -8
  140. package/src/frontend/native/turn/context.ts +6 -8
  141. package/src/frontend/native/turn/turn-meta.ts +2 -5
  142. package/src/frontend/native/turn/turn.ts +8 -10
  143. package/src/frontend/presentation/format.ts +2 -4
  144. package/src/frontend/presentation/session-status.ts +2 -6
  145. package/src/frontend/teams/actions.ts +8 -1
  146. package/src/frontend/teams/graph.ts +0 -1
  147. package/src/frontend/teams/index.ts +1 -4
  148. package/src/frontend/teams/poll.ts +40 -2
  149. package/src/frontend/teams/runtime.ts +14 -5
  150. package/src/frontend/telegram/actions/index.ts +4 -1
  151. package/src/frontend/telegram/actions/send.ts +8 -0
  152. package/src/frontend/telegram/handlers/context.ts +13 -2
  153. package/src/frontend/telegram/handlers/delivery.ts +12 -9
  154. package/src/frontend/telegram/handlers/index.ts +0 -2
  155. package/src/frontend/telegram/index.ts +35 -9
  156. package/src/frontend/telegram/polling/poll-health.ts +110 -0
  157. package/src/frontend/telegram/userbot.ts +100 -36
  158. package/src/frontend/terminal/builtins/session.ts +2 -2
  159. package/src/frontend/terminal/index.ts +1 -3
  160. package/src/frontend/terminal/renderer.ts +2 -18
  161. package/src/frontend/whatsapp/actions/index.ts +12 -1
  162. package/src/frontend/whatsapp/actions/messaging.ts +7 -2
  163. package/src/frontend/whatsapp/connection/connection.ts +29 -9
  164. package/src/frontend/whatsapp/connection/health.ts +89 -0
  165. package/src/frontend/whatsapp/connection/identity.ts +4 -4
  166. package/src/frontend/whatsapp/runtime.ts +11 -5
  167. package/src/native/blake3.ts +28 -2
  168. package/src/native/fusefs.ts +23 -5
  169. package/src/native/registry.ts +1 -1
  170. package/src/native/warden.ts +33 -5
  171. package/src/plugins/github/index.ts +0 -1
  172. package/src/plugins/mempalace/index.ts +9 -3
  173. package/src/plugins/playwright/index.ts +2 -4
  174. package/src/plugins/playwright/provision.ts +12 -4
  175. package/src/storage/chat-settings.ts +6 -1
  176. package/src/storage/cron.ts +29 -4
  177. package/src/storage/daily-log.ts +43 -47
  178. package/src/storage/db.ts +61 -33
  179. package/src/storage/history.ts +6 -1
  180. package/src/storage/journal.ts +9 -2
  181. package/src/storage/kv.ts +19 -6
  182. package/src/storage/media-index.ts +28 -6
  183. package/src/storage/repositories/chat-settings-repo.ts +8 -2
  184. package/src/storage/repositories/sessions-repo.ts +10 -3
  185. package/src/storage/scripts.ts +24 -13
  186. package/src/storage/sessions.ts +11 -2
  187. package/src/storage/skills.ts +21 -2
  188. package/src/storage/stickers.ts +17 -3
  189. package/src/storage/triggers.ts +8 -3
  190. package/src/storage/turn-meta.ts +25 -7
  191. package/src/util/log.ts +189 -6
  192. package/src/util/logging/turn-scope.ts +85 -0
  193. package/src/util/time.ts +3 -3
  194. package/src/util/watchdog.ts +30 -0
  195. package/src/core/engine/backend-controller/legacy.ts +0 -111
@@ -5,12 +5,12 @@
5
5
  * Every backend (Claude SDK, Codex, Kilo, OpenCode, OpenAI Agents)
6
6
  * translates its SDK's native event stream into `AgentEvent`s. Core
7
7
  * renderers (Telegram dispatch, terminal output, heartbeat log,
8
- * dream log, `/status`, tests) consume `AgentEvent`s. Backends no
9
- * longer render markdown logs themselves and core no longer parses
8
+ * dream log, `/status`, tests) consume `AgentEvent`s. Backends don't
9
+ * render markdown logs themselves and core never parses
10
10
  * backend-specific output.
11
11
  *
12
12
  * The shared wrapper `backend/runtime/turn/handler-to-events.ts` converts
13
- * each backend's existing callback-driven `handleMessage` into the
13
+ * each backend's callback-driven `handleMessage` into the
14
14
  * canonical sequence: `run_started → text_delta* →
15
15
  * assistant_message* → tool_call* → usage → completed`. Backends
16
16
  * with richer SDKs can emit events directly without the wrapper.
@@ -146,27 +146,6 @@ export type AgentEvent =
146
146
  | { type: "error"; error: AgentError }
147
147
  | { type: "completed"; result?: AgentResult };
148
148
 
149
- /**
150
- * Type-narrowing helper. Saves callers from writing
151
- * `event.type === "completed"` in two places when they need both the
152
- * narrowing and a boolean expression.
153
- */
154
- export function isAgentEventOf<K extends AgentEvent["type"]>(
155
- event: AgentEvent,
156
- kind: K,
157
- ): event is Extract<AgentEvent, { type: K }> {
158
- return event.type === kind;
159
- }
160
-
161
- /**
162
- * Whether this event is a stream terminator — `completed` (success)
163
- * or `error` (failure). Useful for stream consumers that want to
164
- * release a typing indicator or close a log section on either.
165
- */
166
- export function isAgentRunTerminator(event: AgentEvent): boolean {
167
- return event.type === "completed" || event.type === "error";
168
- }
169
-
170
149
  /**
171
150
  * Error thrown when an `AgentEvent` stream terminates with an `error`
172
151
  * event. The dispatcher consumes the canonical event stream directly
@@ -234,21 +213,6 @@ export function emptyUsage(): UsageSnapshot {
234
213
  };
235
214
  }
236
215
 
237
- /**
238
- * Accumulate two usage snapshots. Pure — caller passes both, gets a
239
- * new object back. Used by stream consumers that aggregate per-event
240
- * usage into a final figure for `/status`.
241
- */
242
- export function addUsage(a: UsageSnapshot, b: UsageSnapshot): UsageSnapshot {
243
- return {
244
- inputTokens: a.inputTokens + b.inputTokens,
245
- outputTokens: a.outputTokens + b.outputTokens,
246
- cacheRead: a.cacheRead + b.cacheRead,
247
- cacheWrite: a.cacheWrite + b.cacheWrite,
248
- modelId: b.modelId ?? a.modelId,
249
- };
250
- }
251
-
252
216
  /**
253
217
  * The `core/errors.ts` reasons that map to a specific `AgentErrorKind`.
254
218
  * Anything not listed collapses to `unknown`. `ErrorReason` is
@@ -102,14 +102,6 @@ export interface ModelRef {
102
102
  unavailableReason?: string;
103
103
  }
104
104
 
105
- /**
106
- * Equality on identity only — two refs are the same run if they
107
- * point at the same backend + id.
108
- */
109
- export function sameModelRef(a: ModelRef, b: ModelRef): boolean {
110
- return a.backend === b.backend && a.id === b.id;
111
- }
112
-
113
105
  /**
114
106
  * Bare-minimum constructor for tests and adapters that don't yet
115
107
  * carry rich metadata. Real catalog code should populate the
@@ -25,6 +25,7 @@ import type { TaskUsage } from "../tasks/types.js";
25
25
  import type { AgentSettledEvent, AgentSpawnedEvent } from "../bus/events.js";
26
26
  import type { ReasoningEffortLevel } from "../types.js";
27
27
  import { bus } from "../bus/index.js";
28
+ import { logWarn } from "../../util/log.js";
28
29
 
29
30
  /** Settled agents kept for status queries after they leave the live map. */
30
31
  const DEFAULT_HISTORY_LIMIT = 100;
@@ -87,6 +88,23 @@ interface LiveAgent {
87
88
  killRequested: boolean;
88
89
  }
89
90
 
91
+ /**
92
+ * Whether two agents were spawned by the same parent.
93
+ *
94
+ * Compared structurally rather than by reference: records come from separate
95
+ * snapshots, so the parent objects are equal in value and never identical.
96
+ * The `kind` check is what stops a chat-parented agent matching an
97
+ * agent-parented one whose id happens to equal a chat key.
98
+ */
99
+ function sameParent(a: AgentParent, b: AgentParent): boolean {
100
+ if (a.kind !== b.kind) return false;
101
+ return a.kind === "chat" && b.kind === "chat"
102
+ ? a.chatId === b.chatId
103
+ : a.kind === "agent" && b.kind === "agent"
104
+ ? a.agentId === b.agentId
105
+ : false;
106
+ }
107
+
90
108
  function snapshot(entry: LiveAgent): AgentRecord {
91
109
  return {
92
110
  ...entry.record,
@@ -198,6 +216,22 @@ export class AgentRegistry {
198
216
  const entry = this.live.get(id);
199
217
  if (!entry || entry.record.state !== "queued") return;
200
218
  entry.abort = binding.abort;
219
+ // A kill can arrive while the agent is still `queued` — before any abort
220
+ // handle exists, so requestKill's `entry.abort?.abort()` was a no-op that
221
+ // only set the flag. When the handle finally binds here, honour that
222
+ // pending kill; otherwise the fresh, un-aborted controller lets the run
223
+ // proceed and the kill is silently lost. The runner checks the signal
224
+ // right after start() and settles the run as "killed".
225
+ if (entry.killRequested) {
226
+ try {
227
+ binding.abort.abort();
228
+ } catch (err) {
229
+ logWarn(
230
+ "agents",
231
+ `Abort hook threw on start agent=${id}: ${err instanceof Error ? err.message : String(err)}`,
232
+ );
233
+ }
234
+ }
201
235
  entry.record.model = binding.model;
202
236
  entry.record.state = "running";
203
237
  entry.record.startedAt = Date.now();
@@ -279,8 +313,13 @@ export class AgentRegistry {
279
313
  entry.killRequested = true;
280
314
  try {
281
315
  entry.abort?.abort();
282
- } catch {
283
- // An abort hook must not be able to break the kill path.
316
+ } catch (err) {
317
+ // An abort hook must not be able to break the kill path — but a
318
+ // throwing one may leave the agent running, so say so.
319
+ logWarn(
320
+ "agents",
321
+ `Abort hook threw agent=${id}: ${err instanceof Error ? err.message : String(err)}`,
322
+ );
284
323
  }
285
324
  }
286
325
  return true;
@@ -351,6 +390,30 @@ export class AgentRegistry {
351
390
  return children.filter((child) => this.live.has(child));
352
391
  }
353
392
 
393
+ /**
394
+ * An agent's live **peers** — the other agents sharing its parent.
395
+ *
396
+ * This is the addressing scope for agent-to-agent messaging, and it is
397
+ * deliberately narrower than "everything under the same chat". A swarm is
398
+ * a set of siblings spawned for one job, so siblings are the useful unit;
399
+ * widening to the whole chat tree would let an agent reach a cousin from an
400
+ * unrelated piece of work it knows nothing about.
401
+ *
402
+ * Live only: a settled agent has no mailbox to deliver into, and offering
403
+ * it as a peer would only produce a delivery failure one call later.
404
+ */
405
+ peersOf(id: string): AgentRecord[] {
406
+ const self = this.get(id);
407
+ if (!self) return [];
408
+ const peers: AgentRecord[] = [];
409
+ for (const entry of this.live.values()) {
410
+ const record = snapshot(entry);
411
+ if (record.id === id) continue;
412
+ if (sameParent(record.parent, self.parent)) peers.push(record);
413
+ }
414
+ return peers.sort((a, b) => a.createdAt - b.createdAt);
415
+ }
416
+
354
417
  /** Live agents plus the bounded settled ring, oldest first. */
355
418
  list(): AgentRecord[] {
356
419
  const records = [...this.history];
@@ -10,6 +10,7 @@
10
10
  */
11
11
 
12
12
  import { notifyAdmin } from "../frontend-runtime/admin-notify.js";
13
+ import { logWarn } from "../../util/log.js";
13
14
  import {
14
15
  daysUntil,
15
16
  PROVIDER_LABELS,
@@ -75,7 +76,14 @@ export function resetAuthExpiryAnnouncements(): void {
75
76
 
76
77
  export function startAuthExpiryMonitor(): () => void {
77
78
  const tick = (): void => {
78
- void runAuthExpiryCheck().catch(() => {});
79
+ void runAuthExpiryCheck().catch((err: unknown) => {
80
+ // The alert is already marked announced, so a failed send is not
81
+ // retried until the status changes — the log is its only trace.
82
+ logWarn(
83
+ "notify",
84
+ `auth expiry check failed: ${err instanceof Error ? err.message : String(err)}`,
85
+ );
86
+ });
79
87
  };
80
88
  // First check shortly after boot so a lapsed login is surfaced right away.
81
89
  const first = setTimeout(tick, 30_000);
@@ -160,7 +160,15 @@ export function startLogin(
160
160
  new Error(outcome.ok ? "no prompt" : outcome.detail || outcome.reason),
161
161
  );
162
162
  if (tmpHome)
163
- await rm(tmpHome, { recursive: true, force: true }).catch(() => {});
163
+ await rm(tmpHome, { recursive: true, force: true }).catch(
164
+ (err: unknown) =>
165
+ // The scratch HOME can hold a fresh credential copy — a leftover
166
+ // is worth knowing about.
167
+ logWarn(
168
+ "notify",
169
+ `${provider} login cleanup failed dir=${tmpHome}: ${err instanceof Error ? err.message : String(err)}`,
170
+ ),
171
+ );
164
172
  resolveDone(outcome);
165
173
  };
166
174
 
@@ -15,6 +15,7 @@
15
15
  import { readFile, stat } from "node:fs/promises";
16
16
  import { homedir } from "node:os";
17
17
  import { join } from "node:path";
18
+ import { logWarn } from "../../util/log.js";
18
19
 
19
20
  export type AuthProvider = "claude" | "codex";
20
21
 
@@ -64,6 +65,19 @@ export function clearProviderExpired(provider: AuthProvider): void {
64
65
  reportedExpired.delete(provider);
65
66
  }
66
67
 
68
+ /**
69
+ * A credentials file that exists but isn't JSON reads as "not signed
70
+ * in" — indistinguishable from a missing login unless logged. Nothing
71
+ * from the file goes into the line: JSON.parse's message quotes the
72
+ * input, and the input is a token.
73
+ */
74
+ function warnUnparseable(provider: AuthProvider): void {
75
+ logWarn(
76
+ "notify",
77
+ `${provider} credentials file is not valid JSON — reporting not signed in`,
78
+ );
79
+ }
80
+
67
81
  export function parseClaudeCredentials(raw: string): ProviderAuthStatus {
68
82
  const base: ProviderAuthStatus = {
69
83
  provider: "claude",
@@ -81,6 +95,7 @@ export function parseClaudeCredentials(raw: string): ProviderAuthStatus {
81
95
  try {
82
96
  parsed = JSON.parse(raw);
83
97
  } catch {
98
+ warnUnparseable("claude");
84
99
  return base;
85
100
  }
86
101
  const oauth = parsed.claudeAiOauth;
@@ -115,6 +130,7 @@ export function parseCodexAuth(raw: string): ProviderAuthStatus {
115
130
  try {
116
131
  parsed = JSON.parse(raw);
117
132
  } catch {
133
+ warnUnparseable("codex");
118
134
  return base;
119
135
  }
120
136
  const apiKey =
@@ -136,11 +152,23 @@ export function parseCodexAuth(raw: string): ProviderAuthStatus {
136
152
  };
137
153
  }
138
154
 
139
- async function readOrEmpty(path: string): Promise<string | undefined> {
155
+ async function readOrEmpty(
156
+ provider: AuthProvider,
157
+ path: string,
158
+ ): Promise<string | undefined> {
140
159
  try {
141
160
  await stat(path);
142
161
  return await readFile(path, "utf8");
143
- } catch {
162
+ } catch (err) {
163
+ // Missing is the normal signed-out case; anything else (EACCES,
164
+ // EISDIR) is a fault that would otherwise read as signed out too.
165
+ const code = (err as NodeJS.ErrnoException).code;
166
+ if (code !== "ENOENT") {
167
+ logWarn(
168
+ "notify",
169
+ `${provider} credentials unreadable code=${code ?? "?"} — reporting not signed in`,
170
+ );
171
+ }
144
172
  return undefined;
145
173
  }
146
174
  }
@@ -151,7 +179,7 @@ export async function readProviderStatus(
151
179
  ): Promise<ProviderAuthStatus> {
152
180
  const path =
153
181
  provider === "claude" ? claudeCredentialsPath(env) : codexAuthPath(env);
154
- const raw = await readOrEmpty(path);
182
+ const raw = await readOrEmpty(provider, path);
155
183
  const status =
156
184
  raw === undefined
157
185
  ? { provider, loggedIn: false, expired: true }
@@ -34,6 +34,8 @@ import {
34
34
  } from "../../../storage/cron.js";
35
35
  import { appendDailyLog } from "../../../storage/daily-log.js";
36
36
  import { log, logError, logWarn } from "../../../util/log.js";
37
+ import { raiseAlert, resolveAlert } from "../../frontend-runtime/alerts.js";
38
+ import { faultText } from "../../engine/fault-text.js";
37
39
  import { numericChatIdFor } from "../../frontend-runtime/chat-id.js";
38
40
  import {
39
41
  chooseBackend,
@@ -156,8 +158,12 @@ async function runCronTick(): Promise<void> {
156
158
  pruneJobHealth(new Set(jobs.map((j) => j.id)));
157
159
 
158
160
  let loadShed = false;
159
- for (const job of jobs) {
160
- if (!job.enabled) continue;
161
+ for (const listed of jobs) {
162
+ // Re-read: each run is awaited, so a slow job lets the next tick start
163
+ // and run later jobs before this loop reaches them. The listing's copy
164
+ // would carry the pre-run lastRunAt and fire such a job a second time.
165
+ const job = getCronJob(listed.id);
166
+ if (!job?.enabled) continue;
161
167
  // Expiry takes priority over dueness: a job past its end time is disabled
162
168
  // and skipped even if this minute would otherwise match.
163
169
  if (expireIfPast(job, nowMs)) continue;
@@ -201,6 +207,10 @@ async function runScheduled(job: CronJob): Promise<void> {
201
207
  durationMs: Date.now() - startedAt,
202
208
  };
203
209
  recordJobSuccess(job.id, Date.now(), JOB_HEALTH);
210
+ resolveAlert(
211
+ `cron.job.${job.id}`,
212
+ `Cron job "${job.name}" is running again.`,
213
+ );
204
214
  recordCronRun(job.id, outcome);
205
215
  appendDailyLog(
206
216
  "Cron",
@@ -223,12 +233,23 @@ async function runScheduled(job: CronJob): Promise<void> {
223
233
  lastError: err instanceof Error ? err.message : String(err),
224
234
  lastDurationMs: Date.now() - startedAt,
225
235
  });
226
- logError("cron", `Job "${job.name}" [${job.id}] failed`, err);
236
+ logError(
237
+ "cron",
238
+ `Job "${job.name}" [${job.id}] failed chat=${job.chatId} type=${job.type} ms=${Date.now() - startedAt}`,
239
+ err,
240
+ );
227
241
  const cooldown = recordJobFailure(job.id, Date.now(), JOB_HEALTH);
228
242
  if (cooldown !== null) {
243
+ const mins = Math.round(cooldown / 60_000);
229
244
  logWarn(
230
245
  "cron",
231
- `Breaker opened for "${job.name}" [${job.id}] — cooling down ~${Math.round(cooldown / 60_000)}min`,
246
+ `Breaker opened for "${job.name}" [${job.id}] — cooling down ~${mins}min`,
247
+ );
248
+ // The breaker opens at JOB_HEALTH.threshold consecutive failures —
249
+ // the job is now paused, which the operator should hear about.
250
+ raiseAlert(
251
+ `cron.job.${job.id}`,
252
+ `Cron job "${job.name}" failed ${JOB_HEALTH.threshold} runs in a row: ${faultText(err)}. Paused for ~${mins} min.`,
232
253
  );
233
254
  }
234
255
  } finally {
@@ -461,7 +482,6 @@ function isCronDue(job: CronJob, now: Date, windowStartMs: number): boolean {
461
482
  export const _cronInternals = {
462
483
  isDue,
463
484
  isCronDue,
464
- MAX_TICK_LOOKBACK_MS,
465
485
  };
466
486
 
467
487
  /**
@@ -24,6 +24,7 @@ import type { Backend } from "../../agent-runtime/capabilities.js";
24
24
  import { taskTable } from "../../tasks/index.js";
25
25
  import { resolveBackgroundEffort } from "../effort.js";
26
26
  import { FailureBackoff } from "../failure-backoff.js";
27
+ import { faultText } from "../../engine/fault-text.js";
27
28
 
28
29
  // ── Types ────────────────────────────────────────────────────────────────────
29
30
 
@@ -61,7 +62,10 @@ let dreaming = false; // in-process guard (one dream at a time)
61
62
  * every message (observed live: a model outage produced 403 identical
62
63
  * consolidation failures). Exported for tests.
63
64
  */
64
- export const dreamFailureBackoff = new FailureBackoff();
65
+ export const dreamFailureBackoff = new FailureBackoff({
66
+ key: "dream.failing",
67
+ label: "Dream (memory consolidation)",
68
+ });
65
69
  let configRef: {
66
70
  model?: string;
67
71
  dreamModel?: string;
@@ -141,6 +145,16 @@ export async function forceDream(): Promise<void> {
141
145
  await executeDream("forced");
142
146
  }
143
147
 
148
+ /** Arm the failure backoff so maybeStartDream stays quiet for a while. */
149
+ function backOff(err: unknown): void {
150
+ const until = dreamFailureBackoff.fail(err);
151
+ logWarn(
152
+ "dream",
153
+ `dream.backoff failures=${dreamFailureBackoff.failures} ` +
154
+ `until=${new Date(until).toISOString()} error="${faultText(err)}"`,
155
+ );
156
+ }
157
+
144
158
  /** Shared dream execution — claims lock, runs agent, releases lock. */
145
159
  async function executeDream(trigger: "auto" | "forced"): Promise<void> {
146
160
  const state = readDreamState();
@@ -153,8 +167,13 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
153
167
 
154
168
  try {
155
169
  const dreamLogPath = await runDreamAgent(state?.last_run ?? 0);
156
- writeDreamState({ last_run: Date.now(), status: "idle" });
157
- dreamFailureBackoff.succeed();
170
+ // A run whose last_run didn't stick still looks overdue, and
171
+ // maybeStartDream would run it all again on the very next message.
172
+ if (writeDreamState({ last_run: Date.now(), status: "idle" })) {
173
+ dreamFailureBackoff.succeed();
174
+ } else {
175
+ backOff(new Error("dream state was not persisted"));
176
+ }
158
177
  log(
159
178
  "dream",
160
179
  `Memory consolidation complete (${trigger}), log: ${dreamLogPath}`,
@@ -164,12 +183,7 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
164
183
  writeDreamState({ last_run: state?.last_run ?? 0, status: "idle" });
165
184
  // A failed dream keeps the old last_run, so it would re-fire on the very
166
185
  // next invocation — back off instead (forceDream bypasses the window).
167
- const until = dreamFailureBackoff.fail(err);
168
- logWarn(
169
- "dream",
170
- `Backing off until ${new Date(until).toISOString()} ` +
171
- `after ${dreamFailureBackoff.failures} consecutive failure(s)`,
172
- );
186
+ backOff(err);
173
187
  if (trigger === "forced") throw err;
174
188
  } finally {
175
189
  dreaming = false;
@@ -427,10 +441,15 @@ function readDreamState(): DreamState | null {
427
441
  return parsed;
428
442
  }
429
443
 
430
- function writeDreamState(state: DreamState): void {
444
+ /**
445
+ * Persist the state. Returns whether last_run actually landed: kvSet logs
446
+ * and swallows a failed write (full disk), so the store is read back.
447
+ */
448
+ function writeDreamState(state: DreamState): boolean {
431
449
  const enriched: DreamState = {
432
450
  ...state,
433
451
  last_run_at: new Date(state.last_run).toISOString(),
434
452
  };
435
453
  kvSet(DREAM_STATE_KEY, enriched);
454
+ return readDreamState()?.last_run === state.last_run;
436
455
  }
@@ -15,8 +15,16 @@
15
15
  *
16
16
  * Backoff state is in-memory only: a process restart forgets it, which is
17
17
  * fine — the first post-restart attempt either works or re-arms the window.
18
+ *
19
+ * A holder built with an `alert` also tells the operator: once the streak
20
+ * reaches its threshold (2 by default — one failure is a blip, a second in
21
+ * a row after the backoff is a broken component) it raises `alert.key`
22
+ * with the error, and the next success resolves it.
18
23
  */
19
24
 
25
+ import { faultText } from "../engine/fault-text.js";
26
+ import { raiseAlert, resolveAlert } from "../frontend-runtime/alerts.js";
27
+
20
28
  const FAILURE_BACKOFF_BASE_MS = 5 * 60 * 1000;
21
29
  const FAILURE_BACKOFF_MAX_MS = 60 * 60 * 1000;
22
30
  /** Safety margin added past a parsed limit-reset time (clock skew, rollout). */
@@ -70,6 +78,16 @@ export function failureBackoffUntil(
70
78
  return now + exp;
71
79
  }
72
80
 
81
+ /** Operator alert a backoff holder raises while its component keeps failing. */
82
+ type BackoffAlert = {
83
+ /** Stable alert key, e.g. "heartbeat.failing". */
84
+ key: string;
85
+ /** What failed, for the message: "The heartbeat", "Dream (memory consolidation)". */
86
+ label: string;
87
+ /** Consecutive failures before raising. Default 2. */
88
+ threshold?: number;
89
+ };
90
+
73
91
  /**
74
92
  * Per-component backoff holder. Cadence checks gate on `active()`; run
75
93
  * completions call `succeed()` / `fail(err)`.
@@ -78,6 +96,8 @@ export class FailureBackoff {
78
96
  private consecutiveFailures = 0;
79
97
  private until = 0;
80
98
 
99
+ constructor(private readonly alert?: BackoffAlert) {}
100
+
81
101
  /** True while inside the backoff window — the caller should skip firing. */
82
102
  active(now = Date.now()): boolean {
83
103
  return now < this.until;
@@ -87,6 +107,14 @@ export class FailureBackoff {
87
107
  fail(err: unknown, now = Date.now()): number {
88
108
  this.consecutiveFailures += 1;
89
109
  this.until = failureBackoffUntil(err, this.consecutiveFailures, now);
110
+ const alert = this.alert;
111
+ if (alert && this.consecutiveFailures >= (alert.threshold ?? 2)) {
112
+ const retryAt = new Date(this.until).toISOString().slice(11, 16);
113
+ raiseAlert(
114
+ alert.key,
115
+ `${alert.label} has failed ${this.consecutiveFailures} times in a row: ${faultText(err)}. Next attempt after ${retryAt} UTC.`,
116
+ );
117
+ }
90
118
  return this.until;
91
119
  }
92
120
 
@@ -94,6 +122,8 @@ export class FailureBackoff {
94
122
  succeed(): void {
95
123
  this.consecutiveFailures = 0;
96
124
  this.until = 0;
125
+ if (this.alert)
126
+ resolveAlert(this.alert.key, `${this.alert.label} is running again.`);
97
127
  }
98
128
 
99
129
  get failures(): number {
@@ -22,6 +22,7 @@ import {
22
22
  resolveRoutedModel,
23
23
  } from "../../engine/backend-router/index.js";
24
24
  import { resolveBackgroundEffort } from "../effort.js";
25
+ import { raceWithTimeout } from "../isolated-agent.js";
25
26
  import { hb } from "./state.js";
26
27
 
27
28
  const DEFAULT_HEARTBEAT_TIMEOUT_MS = 10 * 60 * 1000; // 10-minute soft cap
@@ -88,7 +89,7 @@ export function buildHeartbeatSystemPrompt(): string {
88
89
  /**
89
90
  * Render the open-goal listing for the heartbeat prompt. Cross-chat by design:
90
91
  * the heartbeat is a global agent, so it sees every chat's open goals (with
91
- * chat ids for routing updates back). Exported for tests.
92
+ * chat ids for routing updates back).
92
93
  */
93
94
  function renderGoalsBlock(): { text: string; count: number } {
94
95
  let text = "(no open goals)";
@@ -486,32 +487,6 @@ export async function runHeartbeatAgent(
486
487
  return heartbeatLogFile;
487
488
  }
488
489
 
489
- /**
490
- * Race a promise against a timeout. Returns the promise's resolved value, or
491
- * the sentinel `"timed_out"` if the timeout fires first.
492
- *
493
- * NOTE: if `p` rejects before the timeout fires, that rejection propagates —
494
- * callers that need a never-throwing race should `.catch()` the input promise
495
- * themselves.
496
- */
497
- async function raceWithTimeout<T>(
498
- p: Promise<T>,
499
- ms: number,
500
- ): Promise<T | "timed_out"> {
501
- let t: ReturnType<typeof setTimeout> | null = null;
502
- try {
503
- return await Promise.race([
504
- p,
505
- new Promise<"timed_out">((resolve) => {
506
- t = setTimeout(() => resolve("timed_out"), ms);
507
- t.unref();
508
- }),
509
- ]);
510
- } finally {
511
- if (t) clearTimeout(t);
512
- }
513
- }
514
-
515
490
  // ── Logging helpers ─────────────────────────────────────────────────────────
516
491
 
517
492
  async function createHeartbeatLogFile(): Promise<string> {
@@ -9,8 +9,6 @@
9
9
  * - `state` — shared run-guard/timers/config + persisted-state I/O
10
10
  * - `agent` — prompt building + the one-shot agent run + log helpers
11
11
  * - `scheduler` — init, the cadence timer, run guard, force/await/status API
12
- *
13
- * Re-exports the same public surface the old single-file module exposed.
14
12
  */
15
13
 
16
14
  export {
@@ -5,6 +5,7 @@
5
5
  */
6
6
 
7
7
  import { log, logError, logWarn } from "../../../util/log.js";
8
+ import { faultText } from "../../engine/fault-text.js";
8
9
  import {
9
10
  catchupRunCount,
10
11
  missedRunCount,
@@ -147,6 +148,16 @@ export function getHeartbeatStatus(): HeartbeatState | null {
147
148
  return readHeartbeatState();
148
149
  }
149
150
 
151
+ /** Arm the failure backoff so the due check stops re-firing for a while. */
152
+ function backOff(err: unknown): void {
153
+ const until = hb.failureBackoff.fail(err);
154
+ logWarn(
155
+ "heartbeat",
156
+ `heartbeat.backoff failures=${hb.failureBackoff.failures} ` +
157
+ `until=${new Date(until).toISOString()} error="${faultText(err)}"`,
158
+ );
159
+ }
160
+
150
161
  async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
151
162
  if (hb.running) return;
152
163
 
@@ -175,13 +186,16 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
175
186
  previousRunCount + 1,
176
187
  );
177
188
  // Only update last_run and increment run_count on success
178
- writeHeartbeatState({
189
+ const persisted = writeHeartbeatState({
179
190
  last_run: Date.now(),
180
191
  last_started: now,
181
192
  status: "idle",
182
193
  run_count: previousRunCount + 1,
183
194
  });
184
- hb.failureBackoff.succeed();
195
+ // A run whose last_run didn't stick looks overdue to the next due
196
+ // check, which would run the whole agent again every minute.
197
+ if (persisted) hb.failureBackoff.succeed();
198
+ else backOff(new Error("heartbeat state was not persisted"));
185
199
  log(
186
200
  "heartbeat",
187
201
  `Heartbeat #${previousRunCount + 1} complete (${trigger}), log: ${heartbeatLogPath}`,
@@ -197,23 +211,17 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
197
211
  // re-triggering against the same `last_run` forever. Non-timeout errors
198
212
  // retry from the previous successful run (no budget consumed).
199
213
  const isTimeout = err instanceof HeartbeatTimeoutError;
200
- writeHeartbeatState({
214
+ const persisted = writeHeartbeatState({
201
215
  last_run: isTimeout ? Date.now() : previousLastRun,
202
216
  last_started: now,
203
217
  status: "idle",
204
218
  run_count: isTimeout ? previousRunCount + 1 : previousRunCount,
205
219
  });
206
220
  // Timeouts advance last_run (budget consumed), so the cadence itself
207
- // spaces the next attempt. Every other failure retries against the same
208
- // last_run — back off so the due check doesn't hammer it every minute.
209
- if (!isTimeout) {
210
- const until = hb.failureBackoff.fail(err);
211
- logWarn(
212
- "heartbeat",
213
- `Backing off until ${new Date(until).toISOString()} ` +
214
- `after ${hb.failureBackoff.failures} consecutive failure(s)`,
215
- );
216
- }
221
+ // spaces the next attempt — if that write landed. Every other failure
222
+ // retries against the same last_run — back off so the due check
223
+ // doesn't hammer it every minute.
224
+ if (!isTimeout || !persisted) backOff(err);
217
225
  if (trigger === "forced") throw err;
218
226
  } finally {
219
227
  hb.running = false;