talon-agent 5.18.2 → 5.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +2 -1
  2. package/package.json +2 -2
  3. package/prompts/system/agent-brief.md +20 -3
  4. package/src/app.ts +13 -0
  5. package/src/backend/claude-sdk/handler.ts +4 -4
  6. package/src/backend/claude-sdk/mcp-ready.ts +16 -2
  7. package/src/backend/claude-sdk/one-shot.ts +32 -2
  8. package/src/backend/claude-sdk/stream.ts +2 -2
  9. package/src/backend/codex/auth.ts +1 -1
  10. package/src/backend/codex/handler/message.ts +11 -11
  11. package/src/backend/codex/init.ts +4 -9
  12. package/src/backend/codex/mcp-config.ts +1 -2
  13. package/src/backend/codex/oauth-incompat.ts +8 -4
  14. package/src/backend/codex/one-shot.ts +1 -1
  15. package/src/backend/openai-agents/builtins.ts +55 -27
  16. package/src/backend/openai-agents/factory.ts +3 -3
  17. package/src/backend/openai-agents/handler/message.ts +3 -5
  18. package/src/backend/openai-agents/mcp-pool.ts +10 -27
  19. package/src/backend/remote-server/chat-turn.ts +6 -6
  20. package/src/backend/remote-server/events.ts +1 -5
  21. package/src/backend/remote-server/index.ts +0 -1
  22. package/src/backend/remote-server/messages.ts +3 -7
  23. package/src/backend/remote-server/one-shot.ts +1 -3
  24. package/src/backend/remote-server/session-helpers.ts +1 -4
  25. package/src/backend/remote-server/sse-stream.ts +8 -9
  26. package/src/backend/runtime/metrics.ts +7 -13
  27. package/src/backend/runtime/sleep.ts +1 -2
  28. package/src/backend/runtime/turn/handle-retry.ts +48 -2
  29. package/src/backend/runtime/turn/handler-to-events.ts +3 -4
  30. package/src/bootstrap.ts +9 -1
  31. package/src/cli/doctor.ts +3 -0
  32. package/src/cli/index.ts +8 -10
  33. package/src/cli/logs.ts +148 -9
  34. package/src/cli/setup.ts +9 -11
  35. package/src/cli/status.ts +29 -0
  36. package/src/core/agent-runtime/README.md +5 -19
  37. package/src/core/agent-runtime/events.ts +3 -39
  38. package/src/core/agent-runtime/model-ref.ts +0 -8
  39. package/src/core/agents/registry.ts +65 -2
  40. package/src/core/auth/expiry-monitor.ts +9 -1
  41. package/src/core/auth/login-flow.ts +9 -1
  42. package/src/core/auth/status.ts +31 -3
  43. package/src/core/background/cron/scheduler.ts +25 -5
  44. package/src/core/background/dream/index.ts +29 -10
  45. package/src/core/background/failure-backoff.ts +30 -0
  46. package/src/core/background/heartbeat/agent.ts +2 -27
  47. package/src/core/background/heartbeat/index.ts +0 -2
  48. package/src/core/background/heartbeat/scheduler.ts +21 -13
  49. package/src/core/background/heartbeat/state.ts +10 -2
  50. package/src/core/background/isolated-agent.ts +6 -2
  51. package/src/core/background/pulse/pulse.ts +9 -0
  52. package/src/core/background/triggers/exit.ts +54 -0
  53. package/src/core/background/triggers/index.ts +1 -3
  54. package/src/core/background/triggers/resume.ts +2 -4
  55. package/src/core/backup/archive/tar.ts +14 -4
  56. package/src/core/backup/plan.ts +1 -0
  57. package/src/core/backup/restore.ts +21 -13
  58. package/src/core/backup/scheduler.ts +28 -9
  59. package/src/core/backup/snapshot.ts +43 -10
  60. package/src/core/backup/store.ts +7 -19
  61. package/src/core/backup/targets.ts +69 -17
  62. package/src/core/config/index.ts +14 -0
  63. package/src/core/daemon/crash-marker.ts +141 -0
  64. package/src/core/daemon/crash.ts +9 -2
  65. package/src/core/daemon/handoff.ts +15 -0
  66. package/src/core/daemon/health-alerts.ts +297 -0
  67. package/src/core/daemon/log-reader.ts +289 -0
  68. package/src/core/doctor/index.ts +18 -2
  69. package/src/core/doctor/logs.ts +124 -0
  70. package/src/core/doctor/types.ts +1 -1
  71. package/src/core/engine/backend-controller/index.ts +1 -13
  72. package/src/core/engine/backend-router/router.ts +1 -1
  73. package/src/core/engine/dispatcher.ts +55 -2
  74. package/src/core/engine/fault-text.ts +40 -0
  75. package/src/core/engine/gateway-actions/agents/index.ts +3 -2
  76. package/src/core/engine/gateway-actions/agents/report.ts +62 -0
  77. package/src/core/engine/gateway-actions/history.ts +2 -4
  78. package/src/core/engine/gateway-actions/mesh.ts +26 -14
  79. package/src/core/engine/gateway-actions/native/exec.ts +13 -16
  80. package/src/core/engine/gateway-actions/native/teleport.ts +1 -1
  81. package/src/core/engine/gateway.ts +60 -1
  82. package/src/core/engine/turn-health.ts +222 -0
  83. package/src/core/errors.ts +2 -2
  84. package/src/core/frontend-runtime/admin-notify.ts +1 -1
  85. package/src/core/frontend-runtime/alerts.ts +130 -0
  86. package/src/core/mcp-hub/children.ts +78 -29
  87. package/src/core/mcp-hub/index.ts +21 -18
  88. package/src/core/mcp-hub/proxy-server.ts +8 -4
  89. package/src/core/mcp-hub/talon-server.ts +5 -12
  90. package/src/core/mesh/credentials/store.ts +16 -1
  91. package/src/core/mesh/devices/registry.ts +1 -17
  92. package/src/core/mesh/devices/service.ts +49 -17
  93. package/src/core/mesh/devices/teleport.ts +14 -2
  94. package/src/core/mesh/links/node-binaries.ts +13 -6
  95. package/src/core/mesh/persist.ts +22 -10
  96. package/src/core/mesh/transfers/device-files.ts +5 -23
  97. package/src/core/mesh/transfers/transfers.ts +16 -3
  98. package/src/core/models/active-model.ts +2 -55
  99. package/src/core/plugin/actions.ts +19 -20
  100. package/src/core/plugin/builtins.ts +80 -90
  101. package/src/core/plugin/index.ts +1 -4
  102. package/src/core/plugin/loader.ts +25 -33
  103. package/src/core/plugin/mcp.ts +3 -5
  104. package/src/core/plugin/registry.ts +19 -35
  105. package/src/core/plugin/types.ts +2 -5
  106. package/src/core/prompt/assemble.ts +15 -3
  107. package/src/core/scripts/lua.ts +6 -2
  108. package/src/core/tasks/table.ts +8 -2
  109. package/src/core/tools/bridge.ts +2 -4
  110. package/src/core/tools/chat/cross-send.ts +1 -1
  111. package/src/core/tools/chat/messaging.ts +1 -1
  112. package/src/core/tools/index.ts +2 -2
  113. package/src/core/tools/mcp-env.ts +2 -59
  114. package/src/core/tools/ops/agents.ts +22 -1
  115. package/src/core/tools/schemas.ts +4 -9
  116. package/src/core/vfs/fusefs.ts +0 -5
  117. package/src/core/vfs/index.ts +9 -2
  118. package/src/core/vfs/mounts/diagnostics.ts +109 -0
  119. package/src/core/vfs/mounts/proc.ts +17 -1
  120. package/src/core/vfs/workspace.ts +7 -3
  121. package/src/core/weaver/shuttle.ts +8 -1
  122. package/src/core/weaver/turn-log.ts +320 -0
  123. package/src/core/weaver/weaver.ts +48 -5
  124. package/src/frontend/discord/actions/index.ts +8 -1
  125. package/src/frontend/discord/diagnostics.ts +82 -6
  126. package/src/frontend/discord/handlers/index.ts +0 -2
  127. package/src/frontend/discord/middleware.ts +7 -13
  128. package/src/frontend/discord/runtime.ts +1 -3
  129. package/src/frontend/health/delivery.ts +115 -0
  130. package/src/frontend/health/outage.ts +116 -0
  131. package/src/frontend/native/bridge/routes/chats.ts +3 -5
  132. package/src/frontend/native/bridge/server.ts +106 -21
  133. package/src/frontend/native/index.ts +1 -1
  134. package/src/frontend/native/media/media.ts +5 -1
  135. package/src/frontend/native/runtime.ts +12 -7
  136. package/src/frontend/native/surface/handlers.ts +1 -1
  137. package/src/frontend/native/surface/memory.ts +1 -1
  138. package/src/frontend/native/surface/models.ts +3 -3
  139. package/src/frontend/native/surface/settings.ts +20 -8
  140. package/src/frontend/native/turn/context.ts +6 -8
  141. package/src/frontend/native/turn/turn-meta.ts +2 -5
  142. package/src/frontend/native/turn/turn.ts +8 -10
  143. package/src/frontend/presentation/format.ts +2 -4
  144. package/src/frontend/presentation/session-status.ts +2 -6
  145. package/src/frontend/teams/actions.ts +8 -1
  146. package/src/frontend/teams/graph.ts +0 -1
  147. package/src/frontend/teams/index.ts +1 -4
  148. package/src/frontend/teams/poll.ts +40 -2
  149. package/src/frontend/teams/runtime.ts +14 -5
  150. package/src/frontend/telegram/actions/index.ts +4 -1
  151. package/src/frontend/telegram/actions/send.ts +8 -0
  152. package/src/frontend/telegram/handlers/context.ts +13 -2
  153. package/src/frontend/telegram/handlers/delivery.ts +12 -9
  154. package/src/frontend/telegram/handlers/index.ts +0 -2
  155. package/src/frontend/telegram/index.ts +35 -9
  156. package/src/frontend/telegram/polling/poll-health.ts +110 -0
  157. package/src/frontend/telegram/userbot.ts +100 -36
  158. package/src/frontend/terminal/builtins/session.ts +2 -2
  159. package/src/frontend/terminal/index.ts +1 -3
  160. package/src/frontend/terminal/renderer.ts +2 -18
  161. package/src/frontend/whatsapp/actions/index.ts +12 -1
  162. package/src/frontend/whatsapp/actions/messaging.ts +7 -2
  163. package/src/frontend/whatsapp/connection/connection.ts +29 -9
  164. package/src/frontend/whatsapp/connection/health.ts +89 -0
  165. package/src/frontend/whatsapp/connection/identity.ts +4 -4
  166. package/src/frontend/whatsapp/runtime.ts +11 -5
  167. package/src/native/blake3.ts +28 -2
  168. package/src/native/fusefs.ts +23 -5
  169. package/src/native/registry.ts +1 -1
  170. package/src/native/warden.ts +33 -5
  171. package/src/plugins/github/index.ts +0 -1
  172. package/src/plugins/mempalace/index.ts +9 -3
  173. package/src/plugins/playwright/index.ts +2 -4
  174. package/src/plugins/playwright/provision.ts +12 -4
  175. package/src/storage/chat-settings.ts +6 -1
  176. package/src/storage/cron.ts +29 -4
  177. package/src/storage/daily-log.ts +43 -47
  178. package/src/storage/db.ts +61 -33
  179. package/src/storage/history.ts +6 -1
  180. package/src/storage/journal.ts +9 -2
  181. package/src/storage/kv.ts +19 -6
  182. package/src/storage/media-index.ts +28 -6
  183. package/src/storage/repositories/chat-settings-repo.ts +8 -2
  184. package/src/storage/repositories/sessions-repo.ts +10 -3
  185. package/src/storage/scripts.ts +24 -13
  186. package/src/storage/sessions.ts +11 -2
  187. package/src/storage/skills.ts +21 -2
  188. package/src/storage/stickers.ts +17 -3
  189. package/src/storage/triggers.ts +8 -3
  190. package/src/storage/turn-meta.ts +25 -7
  191. package/src/util/log.ts +189 -6
  192. package/src/util/logging/turn-scope.ts +85 -0
  193. package/src/util/time.ts +3 -3
  194. package/src/util/watchdog.ts +30 -0
  195. package/src/core/engine/backend-controller/legacy.ts +0 -111
@@ -100,7 +100,10 @@ export const hb: {
100
100
  intervalMinutesRef: 60,
101
101
  config: null,
102
102
  logFileSequence: 0,
103
- failureBackoff: new FailureBackoff(),
103
+ failureBackoff: new FailureBackoff({
104
+ key: "heartbeat.failing",
105
+ label: "The heartbeat",
106
+ }),
104
107
  };
105
108
 
106
109
  // ── State-file I/O ───────────────────────────────────────────────────────────
@@ -160,7 +163,11 @@ export function readHeartbeatState(): HeartbeatState | null {
160
163
  return normalizeHeartbeatState(kvGet(HEARTBEAT_STATE_KEY));
161
164
  }
162
165
 
163
- export function writeHeartbeatState(state: HeartbeatState): void {
166
+ /**
167
+ * Persist the state. Returns whether last_run actually landed: kvSet logs
168
+ * and swallows a failed write (full disk), so the store is read back.
169
+ */
170
+ export function writeHeartbeatState(state: HeartbeatState): boolean {
164
171
  // Re-derive last_run_at from last_run so the persisted ISO stamp can
165
172
  // never drift from the millisecond field; omit it on the sentinel
166
173
  // last_run === 0 (never-run) to match the pre-SQLite file format.
@@ -172,4 +179,5 @@ export function writeHeartbeatState(state: HeartbeatState): void {
172
179
  : {}),
173
180
  };
174
181
  kvSet(HEARTBEAT_STATE_KEY, enriched);
182
+ return readHeartbeatState()?.last_run === state.last_run;
175
183
  }
@@ -50,8 +50,12 @@ export interface IsolatedRunOptions {
50
50
  readonly logCategory?: LogComponent;
51
51
  }
52
52
 
53
- /** Resolves to the value, or the string "timed_out" if `ms` elapses first. */
54
- async function raceWithTimeout<T>(
53
+ /**
54
+ * Resolves to the value, or the string "timed_out" if `ms` elapses first.
55
+ * A rejection of `promise` inside the window propagates — `.catch()` it
56
+ * first for a never-throwing race.
57
+ */
58
+ export async function raceWithTimeout<T>(
55
59
  promise: Promise<T>,
56
60
  ms: number,
57
61
  ): Promise<T | "timed_out"> {
@@ -27,6 +27,8 @@ import { formatSmartTimestamp } from "../../../util/time.js";
27
27
  let timer: ReturnType<typeof setInterval> | null = null;
28
28
  const registeredChats = new Set<string>();
29
29
  const lastCheckMessageId = new Map<string, number>();
30
+ /** Chats whose pulse turn is still queued or running. */
31
+ const inFlight = new Set<string>();
30
32
 
31
33
  const DEFAULT_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
32
34
  let activeIntervalMs = DEFAULT_INTERVAL_MS;
@@ -126,6 +128,10 @@ async function runPulse(): Promise<void> {
126
128
  }
127
129
 
128
130
  async function pulseChat(chatId: string): Promise<void> {
131
+ // The pulse turn queues behind the chat's other turns, so it can outlast
132
+ // the interval. Until it settles the checkpoint hasn't moved, and the next
133
+ // tick would dispatch the same unread messages a second time.
134
+ if (inFlight.has(chatId)) return;
129
135
  const numericChatId = parseInt(chatId, 10);
130
136
  if (isNaN(numericChatId)) {
131
137
  logError("pulse", `Invalid chatId: ${chatId}`);
@@ -157,6 +163,7 @@ async function pulseChat(chatId: string): Promise<void> {
157
163
  })
158
164
  .join("\n");
159
165
 
166
+ inFlight.add(chatId);
160
167
  try {
161
168
  const prompt =
162
169
  `[System: Pulse check — ${unread.length} new message(s) since last check. ` +
@@ -179,5 +186,7 @@ async function pulseChat(chatId: string): Promise<void> {
179
186
  } catch (err) {
180
187
  logError("pulse", `Chat ${chatId} failed`, err);
181
188
  // Don't update lastCheckMessageId — messages will be retried next pulse
189
+ } finally {
190
+ inFlight.delete(chatId);
182
191
  }
183
192
  }
@@ -25,6 +25,54 @@ import {
25
25
  WARDEN_GRACE_SLACK_MS,
26
26
  } from "./state.js";
27
27
  import { bufferAsPayload, fireWake } from "./output.js";
28
+ import { raiseAlert, resolveAlert } from "../../frontend-runtime/alerts.js";
29
+ import { faultText } from "../../engine/fault-text.js";
30
+
31
+ /** Exit codes that mean the command itself could not run. */
32
+ const CANNOT_RUN_CODES: Readonly<Record<number, string>> = {
33
+ 126: "command not executable",
34
+ 127: "command not found",
35
+ };
36
+
37
+ /**
38
+ * Tell the operator about a trigger that broke rather than finished: its
39
+ * command could not run (exit 126/127) or the process died on a signal
40
+ * nobody sent it (a crash, the OOM killer — Talon's own kills set the
41
+ * status before the exit lands, so they never reach `errored`). An
42
+ * ordinary non-zero exit is the script's own verdict and already reaches
43
+ * the chat as a wake. A clean fire clears an earlier alert for the id.
44
+ */
45
+ function alertOnExit(
46
+ t: Trigger,
47
+ status: TriggerStatus,
48
+ code: number | null,
49
+ signal: NodeJS.Signals | null,
50
+ output: string[],
51
+ ): void {
52
+ const key = `trigger.${t.id}`;
53
+ if (status === "fired") {
54
+ resolveAlert(key, `Trigger "${t.name}" is firing normally again.`);
55
+ return;
56
+ }
57
+ if (status !== "errored") return;
58
+ const cause =
59
+ signal !== null
60
+ ? `crashed (${signal})`
61
+ : code !== null && CANNOT_RUN_CODES[code]
62
+ ? `could not run its command (exit ${code}: ${CANNOT_RUN_CODES[code]})`
63
+ : null;
64
+ if (!cause) return;
65
+ const last = output.at(-1);
66
+ log(
67
+ "triggers",
68
+ `trigger.broken id=${t.id} name="${t.name}" code=${code} signal=${signal} cause="${cause}"`,
69
+ );
70
+ raiseAlert(
71
+ key,
72
+ `Trigger "${t.name}" [${t.id}] ${cause}${last ? `: ${faultText(last)}` : "."}`,
73
+ { severity: "warn" },
74
+ );
75
+ }
28
76
 
29
77
  export function handleTimeout(trigger: Trigger): void {
30
78
  timeouts.delete(trigger.id);
@@ -177,6 +225,7 @@ export async function finalizeExit(
177
225
  "triggers",
178
226
  `Exited "${t.name}" [${id}] code=${code} signal=${signal} → ${status}`,
179
227
  );
228
+ alertOnExit(t, status, code, signal, buffered);
180
229
 
181
230
  appendDailyLog(
182
231
  "Triggers",
@@ -211,6 +260,11 @@ export async function finalizeExit(
211
260
 
212
261
  export function failTrigger(t: Trigger, message: string): void {
213
262
  logError("triggers", `Failed to spawn ${t.id}: ${message}`);
263
+ raiseAlert(
264
+ `trigger.${t.id}`,
265
+ `Trigger "${t.name}" [${t.id}] failed to start: ${faultText(message)}`,
266
+ { severity: "warn" },
267
+ );
214
268
  updateTrigger(t.id, {
215
269
  status: "errored",
216
270
  lastError: message,
@@ -15,7 +15,7 @@
15
15
  * Knows nothing about backend or frontend — dependencies are injected.
16
16
  */
17
17
 
18
- import { children, timeouts, logStreams, wardened } from "./state.js";
18
+ import { children, timeouts } from "./state.js";
19
19
  import { commandForLanguage } from "./command.js";
20
20
  import { handleStdoutLine } from "./output.js";
21
21
  import { handleTimeout, finalizeExit } from "./exit.js";
@@ -35,8 +35,6 @@ export { resumeAfterRestart } from "./resume.js";
35
35
  export const _internals = {
36
36
  children,
37
37
  timeouts,
38
- logStreams,
39
- wardened,
40
38
  handleStdoutLine,
41
39
  handleTimeout,
42
40
  finalizeExit,
@@ -49,7 +49,7 @@ export async function resumeAfterRestart(): Promise<void> {
49
49
  continue;
50
50
  }
51
51
  // Late death notice. Two cases earn one:
52
- // - never fired at all (the old rule) — the chat heard nothing
52
+ // - never fired at all — the chat heard nothing
53
53
  // from this trigger, so its termination is news; and
54
54
  // - killed by THIS restart — either the crash path
55
55
  // (recoverInterrupted stamped RESTART_KILL_ERROR) or the clean
@@ -58,9 +58,7 @@ export async function resumeAfterRestart(): Promise<void> {
58
58
  // backend pool was already going away). Even a multi-fire
59
59
  // watcher that signalled mid-run was still an active promise
60
60
  // when the process died, and without this wake the chat never
61
- // learns its watcher is gone. (Previously gated on
62
- // lastFireAt === undefined alone, which silently dropped
63
- // exactly those watchers.)
61
+ // learns its watcher is gone.
64
62
  // Triggers that exited on their own already fired their terminal
65
63
  // wake (lastFireAt set, no marker) — they stay silent here.
66
64
  if (
@@ -22,8 +22,8 @@
22
22
  * `~/.ssh/authorized_keys` is not a safety net.
23
23
  */
24
24
 
25
- import { createReadStream, createWriteStream } from "node:fs";
26
- import { mkdir, symlink, utimes } from "node:fs/promises";
25
+ import { createWriteStream } from "node:fs";
26
+ import { mkdir, open, symlink, utimes } from "node:fs/promises";
27
27
  import { once } from "node:events";
28
28
  import { dirname, isAbsolute, relative, resolve } from "node:path";
29
29
  import type { Writable } from "node:stream";
@@ -262,6 +262,9 @@ export class TarWriter {
262
262
  * Stream a file from disk. `size` is the length recorded in the header:
263
263
  * a file that changes under us is truncated or zero-padded to it, because
264
264
  * a tar whose payload length disagrees with its header is unreadable.
265
+ * The source is opened before the header goes out, so a file that
266
+ * vanished or became unreadable since it was listed throws with nothing
267
+ * written — the archive is still whole and the caller may carry on.
265
268
  */
266
269
  async addFile(
267
270
  path: string,
@@ -270,9 +273,16 @@ export class TarWriter {
270
273
  mtime: number,
271
274
  size: number,
272
275
  ): Promise<void> {
273
- await this.writeHeaders({ path, type: "file", mode, mtime, size });
276
+ const handle = await open(source, "r");
277
+ try {
278
+ await this.writeHeaders({ path, type: "file", mode, mtime, size });
279
+ } catch (err) {
280
+ await handle.close();
281
+ throw err;
282
+ }
274
283
  let written = 0;
275
- const stream = createReadStream(source);
284
+ // Owns the handle from here: closed when the stream ends or is destroyed.
285
+ const stream = handle.createReadStream();
276
286
  for await (const chunk of stream) {
277
287
  const buf = chunk as Buffer;
278
288
  const room = size - written;
@@ -75,6 +75,7 @@ export const HOME_INCLUDES: readonly string[] = [
75
75
  "google",
76
76
  "plugins",
77
77
  "mesh-devices.json",
78
+ "mesh-credentials.json",
78
79
  "mesh-history.json",
79
80
  "mesh-locations.json",
80
81
  "teleport-state.json",
@@ -63,13 +63,19 @@ import {
63
63
  rewriteConfigForClone,
64
64
  type CloneTarget,
65
65
  } from "./sources/relocate.js";
66
- import { isSnapshotId, partPath, readManifest, snapshotDir } from "./store.js";
66
+ import { pathExists } from "./sources/sessions.js";
67
+ import {
68
+ DB_MEMBER,
69
+ isSnapshotId,
70
+ partPath,
71
+ readManifest,
72
+ snapshotDir,
73
+ } from "./store.js";
67
74
  import type { BackupTarget } from "./targets.js";
68
75
  import type { BackupSettings, Manifest, SnapshotPart } from "./types.js";
69
76
 
70
77
  /** A staged request older than this is stale and ignored. */
71
78
  export const RESTORE_PENDING_MAX_AGE_MS = 10 * 60_000;
72
- const DB_MEMBER = "db/talon.db";
73
79
 
74
80
  export type RestorePending = {
75
81
  id: string;
@@ -165,15 +171,6 @@ export async function readRestorePending(
165
171
 
166
172
  // ── Parts ───────────────────────────────────────────────────────────────────
167
173
 
168
- async function isLocal(path: string): Promise<boolean> {
169
- try {
170
- await stat(path);
171
- return true;
172
- } catch {
173
- return false;
174
- }
175
- }
176
-
177
174
  /** The parts of this snapshot that are not on local disk. */
178
175
  async function missingParts(
179
176
  manifest: Manifest,
@@ -181,7 +178,7 @@ async function missingParts(
181
178
  ): Promise<SnapshotPart[]> {
182
179
  const missing: SnapshotPart[] = [];
183
180
  for (const part of manifest.parts) {
184
- if (!(await isLocal(partPath(manifest.id, part.name, home)))) {
181
+ if (!(await pathExists(partPath(manifest.id, part.name, home)))) {
185
182
  missing.push(part);
186
183
  }
187
184
  }
@@ -218,7 +215,18 @@ async function ensureParts(
218
215
  const path = partPath(manifest.id, part.name, home);
219
216
  log("backup", `Downloading ${part.name} from ${target.id}…`);
220
217
  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
221
- await target.download(manifest.id, part.name, path);
218
+ // Into a side file, renamed only once complete: a download cut short
219
+ // under the part's own name would count as present next time, and
220
+ // every later restore would fail its checksum instead of refetching.
221
+ const partial = `${path}.partial`;
222
+ await rm(partial, { force: true });
223
+ try {
224
+ await target.download(manifest.id, part.name, partial);
225
+ await rename(partial, path);
226
+ } catch (err) {
227
+ await rm(partial, { force: true });
228
+ throw err;
229
+ }
222
230
  }
223
231
  return manifest.parts.filter((part) => !skipped.has(part.name));
224
232
  }
@@ -26,6 +26,8 @@
26
26
  import { FailureBackoff } from "../background/failure-backoff.js";
27
27
  import { TalonError } from "../errors.js";
28
28
  import { notifyAdmin } from "../frontend-runtime/admin-notify.js";
29
+ import { raiseAlert, resolveAlert } from "../frontend-runtime/alerts.js";
30
+ import { faultText } from "../engine/fault-text.js";
29
31
  import { bus } from "../bus/index.js";
30
32
  import { log, logError } from "../../util/log.js";
31
33
  import { dirs } from "../../util/paths.js";
@@ -117,6 +119,7 @@ export function firstRunDelayMs(
117
119
  // ── Runs ────────────────────────────────────────────────────────────────────
118
120
 
119
121
  const backoff = new FailureBackoff();
122
+ const BACKUP_ALERT = "backup.failing";
120
123
 
121
124
  async function executeRun(request: RunRequest): Promise<Manifest> {
122
125
  const settings = state.settings;
@@ -163,6 +166,7 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
163
166
  }
164
167
  }
165
168
  backoff.succeed();
169
+ resolveAlert(BACKUP_ALERT, "Backups are succeeding again.");
166
170
  return manifest;
167
171
  } catch (err) {
168
172
  const message = err instanceof Error ? err.message : String(err);
@@ -177,21 +181,36 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
177
181
  logError("backup", `Run failed (${request.trigger})`, err);
178
182
  // Once per streak: the second identical failure tells the admin nothing
179
183
  // the first one did not.
180
- if (backoff.failures === 1) {
181
- await state
182
- .notify(
183
- `⚠️ Backup failed: ${message}\nRetrying after ${new Date(until).toISOString().slice(11, 16)} UTC.`,
184
- )
185
- .catch(() => {
186
- /* the notifier logs its own failures */
187
- });
188
- }
184
+ if (backoff.failures === 1) await reportFailure(message, until);
189
185
  throw err;
190
186
  } finally {
191
187
  state.running = false;
192
188
  }
193
189
  }
194
190
 
191
+ /**
192
+ * Tell the admin a backup streak started. The default route is an operator
193
+ * alert (`backup.failing`), so the recovery is announced too; a deployment
194
+ * that routes backup notices to its own chat (`backup.notifyChatId`) keeps
195
+ * getting the plain notice there.
196
+ */
197
+ async function reportFailure(message: string, until: number): Promise<void> {
198
+ const retryAt = new Date(until).toISOString().slice(11, 16);
199
+ if (state.notify === notifyAdmin) {
200
+ raiseAlert(
201
+ BACKUP_ALERT,
202
+ `Backup failed: ${faultText(message, 300)}. Retrying after ${retryAt} UTC.`,
203
+ { severity: "warn" },
204
+ );
205
+ return;
206
+ }
207
+ await state
208
+ .notify(`⚠️ Backup failed: ${message}\nRetrying after ${retryAt} UTC.`)
209
+ .catch(() => {
210
+ /* the notifier logs its own failures */
211
+ });
212
+ }
213
+
195
214
  /**
196
215
  * Take a snapshot. Requests queue behind whatever is already running, so
197
216
  * this resolves with THIS request's snapshot, not someone else's.
@@ -72,6 +72,7 @@ import {
72
72
  type SourceContext,
73
73
  } from "./sources/sessions.js";
74
74
  import {
75
+ DB_MEMBER,
75
76
  STATE_PART,
76
77
  indexSnapshot,
77
78
  linkOrCopy,
@@ -88,8 +89,6 @@ import type {
88
89
  SnapshotPart,
89
90
  } from "./types.js";
90
91
 
91
- /** Where the database copy lands inside the archive. */
92
- const DB_MEMBER = "db/talon.db";
93
92
  /** The part that holds WhatsApp auth and the userbot session. */
94
93
  const LOGINS_PART = "logins.tar.zst";
95
94
  /** How a clone reinstalls fetched plugins (see sources/plugins.ts). */
@@ -120,10 +119,18 @@ export type BuildOptions = {
120
119
 
121
120
  // ── Archive writing ─────────────────────────────────────────────────────────
122
121
 
122
+ /**
123
+ * Open failures that mean "this file is gone or locked since the walk":
124
+ * traces and backend transcripts churn while a snapshot runs, and one of
125
+ * them disappearing must not cost the whole backup (see collectTree).
126
+ */
127
+ const SKIPPABLE_OPEN_ERRORS = new Set(["ENOENT", "EACCES", "EPERM"]);
128
+
123
129
  async function addEntries(
124
130
  writer: TarWriter,
125
131
  entries: readonly SourceEntry[],
126
132
  ): Promise<void> {
133
+ const skipped: string[] = [];
127
134
  for (const entry of entries) {
128
135
  if (entry.type === "dir") {
129
136
  await writer.addDirectory(entry.archivePath, entry.mode, entry.mtime);
@@ -135,15 +142,27 @@ async function addEntries(
135
142
  entry.mtime,
136
143
  );
137
144
  } else {
138
- await writer.addFile(
139
- entry.archivePath,
140
- entry.source,
141
- entry.mode,
142
- entry.mtime,
143
- entry.size,
144
- );
145
+ try {
146
+ await writer.addFile(
147
+ entry.archivePath,
148
+ entry.source,
149
+ entry.mode,
150
+ entry.mtime,
151
+ entry.size,
152
+ );
153
+ } catch (err) {
154
+ const code = (err as NodeJS.ErrnoException).code ?? "";
155
+ if (!SKIPPABLE_OPEN_ERRORS.has(code)) throw err;
156
+ skipped.push(`${entry.source} (${code})`);
157
+ }
145
158
  }
146
159
  }
160
+ if (skipped.length > 0) {
161
+ logWarn(
162
+ "backup",
163
+ `Skipped ${skipped.length} file(s) that vanished or became unreadable mid-snapshot; first: ${skipped[0]}`,
164
+ );
165
+ }
147
166
  }
148
167
 
149
168
  /** A part's file name: `.enc` marks one written through the encryptor. */
@@ -171,6 +190,11 @@ async function writePart(
171
190
  const flushed = passphrase
172
191
  ? pipeline(compressor, await createEncryptor(passphrase), tap, out)
173
192
  : pipeline(compressor, tap, out);
193
+ // A sink failure (ENOSPC) rejects this while `fill` is still writing, and
194
+ // `fill` then throws the same error — so the await below is never
195
+ // reached. Observe it here, or it surfaces as an unhandled rejection
196
+ // that kills the CLI before the cleanup runs.
197
+ flushed.catch(() => {});
174
198
  try {
175
199
  const writer = new TarWriter(compressor);
176
200
  await fill(writer);
@@ -312,11 +336,20 @@ async function palaceFingerprint(
312
336
  const files: TreeFile[] = [];
313
337
  for (const entry of entries) {
314
338
  if (entry.type !== "file") continue;
339
+ let sha256: string;
340
+ try {
341
+ sha256 = await sha256File(entry.source);
342
+ } catch (err) {
343
+ // Gone since the walk: the part will skip it too (see addEntries).
344
+ const code = (err as NodeJS.ErrnoException).code ?? "";
345
+ if (SKIPPABLE_OPEN_ERRORS.has(code)) continue;
346
+ throw err;
347
+ }
315
348
  files.push({
316
349
  path: entry.archivePath,
317
350
  size: entry.size,
318
351
  mtime: entry.mtime,
319
- sha256: await sha256File(entry.source),
352
+ sha256,
320
353
  });
321
354
  }
322
355
  return treeHash(files);
@@ -17,15 +17,7 @@
17
17
  */
18
18
 
19
19
  import { randomBytes } from "node:crypto";
20
- import {
21
- link,
22
- copyFile,
23
- mkdir,
24
- readdir,
25
- readFile,
26
- rm,
27
- stat,
28
- } from "node:fs/promises";
20
+ import { link, copyFile, mkdir, readdir, readFile, rm } from "node:fs/promises";
29
21
  import { join, resolve } from "node:path";
30
22
  import writeFileAtomic from "write-file-atomic";
31
23
  import { dirs } from "../../util/paths.js";
@@ -40,6 +32,7 @@ import {
40
32
  recordBackup,
41
33
  updateBackupManifest,
42
34
  } from "../../storage/backup/index.js";
35
+ import { pathExists } from "./sources/sessions.js";
43
36
  import type {
44
37
  Manifest,
45
38
  RemoteState,
@@ -51,6 +44,8 @@ import type {
51
44
  const SNAPSHOT_ID_RE = /^\d{8}T\d{6}Z-[0-9a-f]{6}$/;
52
45
  const MANIFEST_NAME = "manifest.json";
53
46
  export const STATE_PART = "state.tar.zst";
47
+ /** Where the database copy lands inside the state part. */
48
+ export const DB_MEMBER = "db/talon.db";
54
49
 
55
50
  function backupsRoot(home: string = dirs.root): string {
56
51
  return join(home, "backups");
@@ -212,22 +207,15 @@ export async function listSnapshots(
212
207
  pinned: record.pinned,
213
208
  createdAt: record.createdAt,
214
209
  sizeBytes: record.sizeBytes,
215
- local: await exists(join(snapshotDir(record.id, home), MANIFEST_NAME)),
210
+ local: await pathExists(
211
+ join(snapshotDir(record.id, home), MANIFEST_NAME),
212
+ ),
216
213
  remote,
217
214
  });
218
215
  }
219
216
  return summaries;
220
217
  }
221
218
 
222
- async function exists(path: string): Promise<boolean> {
223
- try {
224
- await stat(path);
225
- return true;
226
- } catch {
227
- return false;
228
- }
229
- }
230
-
231
219
  /** Pin or unpin a snapshot, on disk and in the index. Returns false if unknown. */
232
220
  export async function setSnapshotPinned(
233
221
  id: string,
@@ -30,6 +30,17 @@ import type { Manifest, SnapshotPart } from "./types.js";
30
30
  /** The daemon's chat id for plugin actions that belong to no conversation. */
31
31
  const SYSTEM_CHAT = "system";
32
32
 
33
+ /**
34
+ * How long a target may take to answer. A plugin that never answers would
35
+ * otherwise hold the backup queue forever — every later snapshot, and the
36
+ * pre-update checkpoint `/update` waits on, queued behind it. Part uploads
37
+ * get a base allowance plus time for the bytes at a slow uplink's pace.
38
+ */
39
+ const CONTROL_DEADLINE_MS = 10 * 60_000;
40
+ const TRANSFER_BASE_MS = 30 * 60_000;
41
+ /** 64 KiB/s — well under any link a remote backup is usable over. */
42
+ const TRANSFER_FLOOR_BYTES_PER_MS = 64;
43
+
33
44
  /** A part as the upload call describes it: metadata plus where to read it. */
34
45
  type UploadPart = SnapshotPart & { path: string };
35
46
 
@@ -101,6 +112,37 @@ function targetError(
101
112
  });
102
113
  }
103
114
 
115
+ /** setTimeout's ceiling; a longer delay would fire at once. */
116
+ const MAX_TIMER_MS = 2 ** 31 - 1;
117
+
118
+ /**
119
+ * Send one body, failing with `target`'s name once `deadlineMs` passes.
120
+ * A null deadline sends it unbounded.
121
+ */
122
+ async function dispatchWithin(
123
+ deps: TargetDeps,
124
+ plugin: string,
125
+ body: Record<string, unknown>,
126
+ target: string,
127
+ deadlineMs: number | null,
128
+ ): Promise<ActionResult | null> {
129
+ if (deadlineMs === null) return deps.dispatch(plugin, body);
130
+ const ms = Math.min(deadlineMs, MAX_TIMER_MS);
131
+ let timer: ReturnType<typeof setTimeout> | undefined;
132
+ const deadline = new Promise<never>((_, reject) => {
133
+ timer = setTimeout(() => {
134
+ const action = String(body.action).replace(/^backup\.target\./, "");
135
+ const minutes = Math.round(ms / 60_000);
136
+ reject(targetError(target, action, `no answer within ${minutes} min`));
137
+ }, ms);
138
+ });
139
+ try {
140
+ return await Promise.race([deps.dispatch(plugin, body), deadline]);
141
+ } finally {
142
+ clearTimeout(timer);
143
+ }
144
+ }
145
+
104
146
  /** Unwrap `{ ok, data }`, turning every failure shape into one error. */
105
147
  function dataOf(
106
148
  result: ActionResult | null,
@@ -128,8 +170,11 @@ class PluginTarget implements BackupTarget {
128
170
  readonly detail?: string,
129
171
  ) {}
130
172
 
131
- private send(body: Record<string, unknown>): Promise<ActionResult | null> {
132
- return this.deps.dispatch(this.plugin, body);
173
+ private send(
174
+ body: Record<string, unknown>,
175
+ deadlineMs: number | null = CONTROL_DEADLINE_MS,
176
+ ): Promise<ActionResult | null> {
177
+ return dispatchWithin(this.deps, this.plugin, body, this.id, deadlineMs);
133
178
  }
134
179
 
135
180
  async upload(
@@ -138,12 +183,10 @@ class PluginTarget implements BackupTarget {
138
183
  manifest: Manifest,
139
184
  ): Promise<{ remoteId: string; deduplicated?: boolean }> {
140
185
  const data = dataOf(
141
- await this.send({
142
- action: "backup.target.upload",
143
- snapshotId,
144
- part,
145
- manifest,
146
- }),
186
+ await this.send(
187
+ { action: "backup.target.upload", snapshotId, part, manifest },
188
+ TRANSFER_BASE_MS + part.bytes / TRANSFER_FLOOR_BYTES_PER_MS,
189
+ ),
147
190
  this.id,
148
191
  "upload",
149
192
  );
@@ -206,12 +249,17 @@ class PluginTarget implements BackupTarget {
206
249
  destPath: string,
207
250
  ): Promise<void> {
208
251
  dataOf(
209
- await this.send({
210
- action: "backup.target.download",
211
- snapshotId,
212
- part: { name: partName },
213
- destPath,
214
- }),
252
+ // Unbounded: only an interactive restore downloads, the size is not
253
+ // known here, and the operator can interrupt it.
254
+ await this.send(
255
+ {
256
+ action: "backup.target.download",
257
+ snapshotId,
258
+ part: { name: partName },
259
+ destPath,
260
+ },
261
+ null,
262
+ ),
215
263
  this.id,
216
264
  "download",
217
265
  );
@@ -231,9 +279,13 @@ export async function discoverTargets(
231
279
  for (const plugin of deps.plugins()) {
232
280
  let result: ActionResult | null;
233
281
  try {
234
- result = await deps.dispatch(plugin, {
235
- action: "backup.target.describe",
236
- });
282
+ result = await dispatchWithin(
283
+ deps,
284
+ plugin,
285
+ { action: "backup.target.describe" },
286
+ plugin,
287
+ CONTROL_DEADLINE_MS,
288
+ );
237
289
  } catch (err) {
238
290
  logWarn(
239
291
  "backup",