talon-agent 5.18.1 → 5.19.0

package/src/core/background/dream/index.ts

@@ -24,6 +24,7 @@ import type { Backend } from "../../agent-runtime/capabilities.js";
 import { taskTable } from "../../tasks/index.js";
 import { resolveBackgroundEffort } from "../effort.js";
 import { FailureBackoff } from "../failure-backoff.js";
+import { faultText } from "../../engine/fault-text.js";
 
 // ── Types ────────────────────────────────────────────────────────────────────
 
@@ -61,7 +62,10 @@ let dreaming = false; // in-process guard (one dream at a time)
  * every message (observed live: a model outage produced 403 identical
  * consolidation failures). Exported for tests.
  */
-export const dreamFailureBackoff = new FailureBackoff();
+export const dreamFailureBackoff = new FailureBackoff({
+  key: "dream.failing",
+  label: "Dream (memory consolidation)",
+});
 let configRef: {
   model?: string;
   dreamModel?: string;
@@ -141,6 +145,16 @@ export async function forceDream(): Promise<void> {
   await executeDream("forced");
 }
 
+/** Arm the failure backoff so maybeStartDream stays quiet for a while. */
+function backOff(err: unknown): void {
+  const until = dreamFailureBackoff.fail(err);
+  logWarn(
+    "dream",
+    `dream.backoff failures=${dreamFailureBackoff.failures} ` +
+      `until=${new Date(until).toISOString()} error="${faultText(err)}"`,
+  );
+}
+
 /** Shared dream execution — claims lock, runs agent, releases lock. */
 async function executeDream(trigger: "auto" | "forced"): Promise<void> {
   const state = readDreamState();
@@ -153,8 +167,13 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
 
   try {
     const dreamLogPath = await runDreamAgent(state?.last_run ?? 0);
-    writeDreamState({ last_run: Date.now(), status: "idle" });
-    dreamFailureBackoff.succeed();
+    // A run whose last_run didn't stick still looks overdue, and
+    // maybeStartDream would run it all again on the very next message.
+    if (writeDreamState({ last_run: Date.now(), status: "idle" })) {
+      dreamFailureBackoff.succeed();
+    } else {
+      backOff(new Error("dream state was not persisted"));
+    }
     log(
       "dream",
       `Memory consolidation complete (${trigger}), log: ${dreamLogPath}`,
@@ -164,12 +183,7 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
     writeDreamState({ last_run: state?.last_run ?? 0, status: "idle" });
     // A failed dream keeps the old last_run, so it would re-fire on the very
     // next invocation — back off instead (forceDream bypasses the window).
-    const until = dreamFailureBackoff.fail(err);
-    logWarn(
-      "dream",
-      `Backing off until ${new Date(until).toISOString()} ` +
-        `after ${dreamFailureBackoff.failures} consecutive failure(s)`,
-    );
+    backOff(err);
     if (trigger === "forced") throw err;
   } finally {
     dreaming = false;
@@ -427,10 +441,15 @@ function readDreamState(): DreamState | null {
   return parsed;
 }
 
-function writeDreamState(state: DreamState): void {
+/**
+ * Persist the state. Returns whether last_run actually landed: kvSet logs
+ * and swallows a failed write (full disk), so the store is read back.
+ */
+function writeDreamState(state: DreamState): boolean {
   const enriched: DreamState = {
     ...state,
     last_run_at: new Date(state.last_run).toISOString(),
   };
   kvSet(DREAM_STATE_KEY, enriched);
+  return readDreamState()?.last_run === state.last_run;
 }

package/src/core/background/failure-backoff.ts

@@ -15,8 +15,16 @@
  *
  * Backoff state is in-memory only: a process restart forgets it, which is
  * fine — the first post-restart attempt either works or re-arms the window.
+ *
+ * A holder built with an `alert` also tells the operator: once the streak
+ * reaches its threshold (2 by default — one failure is a blip, a second in
+ * a row after the backoff is a broken component) it raises `alert.key`
+ * with the error, and the next success resolves it.
  */
 
+import { faultText } from "../engine/fault-text.js";
+import { raiseAlert, resolveAlert } from "../frontend-runtime/alerts.js";
+
 const FAILURE_BACKOFF_BASE_MS = 5 * 60 * 1000;
 const FAILURE_BACKOFF_MAX_MS = 60 * 60 * 1000;
 /** Safety margin added past a parsed limit-reset time (clock skew, rollout). */
@@ -70,6 +78,16 @@ export function failureBackoffUntil(
   return now + exp;
 }
 
+/** Operator alert a backoff holder raises while its component keeps failing. */
+type BackoffAlert = {
+  /** Stable alert key, e.g. "heartbeat.failing". */
+  key: string;
+  /** What failed, for the message: "The heartbeat", "Dream (memory consolidation)". */
+  label: string;
+  /** Consecutive failures before raising. Default 2. */
+  threshold?: number;
+};
+
 /**
  * Per-component backoff holder. Cadence checks gate on `active()`; run
  * completions call `succeed()` / `fail(err)`.
@@ -78,6 +96,8 @@ export class FailureBackoff {
   private consecutiveFailures = 0;
   private until = 0;
 
+  constructor(private readonly alert?: BackoffAlert) {}
+
   /** True while inside the backoff window — the caller should skip firing. */
   active(now = Date.now()): boolean {
     return now < this.until;
@@ -87,6 +107,14 @@ export class FailureBackoff {
   fail(err: unknown, now = Date.now()): number {
     this.consecutiveFailures += 1;
     this.until = failureBackoffUntil(err, this.consecutiveFailures, now);
+    const alert = this.alert;
+    if (alert && this.consecutiveFailures >= (alert.threshold ?? 2)) {
+      const retryAt = new Date(this.until).toISOString().slice(11, 16);
+      raiseAlert(
+        alert.key,
+        `${alert.label} has failed ${this.consecutiveFailures} times in a row: ${faultText(err)}. Next attempt after ${retryAt} UTC.`,
+      );
+    }
     return this.until;
   }
 
@@ -94,6 +122,8 @@ export class FailureBackoff {
   succeed(): void {
     this.consecutiveFailures = 0;
     this.until = 0;
+    if (this.alert)
+      resolveAlert(this.alert.key, `${this.alert.label} is running again.`);
   }
 
   get failures(): number {

package/src/core/background/heartbeat/agent.ts

@@ -22,6 +22,7 @@ import {
   resolveRoutedModel,
 } from "../../engine/backend-router/index.js";
 import { resolveBackgroundEffort } from "../effort.js";
+import { raceWithTimeout } from "../isolated-agent.js";
 import { hb } from "./state.js";
 
 const DEFAULT_HEARTBEAT_TIMEOUT_MS = 10 * 60 * 1000; // 10-minute soft cap
@@ -88,7 +89,7 @@ export function buildHeartbeatSystemPrompt(): string {
 /**
  * Render the open-goal listing for the heartbeat prompt. Cross-chat by design:
  * the heartbeat is a global agent, so it sees every chat's open goals (with
- * chat ids for routing updates back). Exported for tests.
+ * chat ids for routing updates back).
  */
 function renderGoalsBlock(): { text: string; count: number } {
   let text = "(no open goals)";
@@ -486,32 +487,6 @@ export async function runHeartbeatAgent(
   return heartbeatLogFile;
 }
 
-/**
- * Race a promise against a timeout. Returns the promise's resolved value, or
- * the sentinel `"timed_out"` if the timeout fires first.
- *
- * NOTE: if `p` rejects before the timeout fires, that rejection propagates —
- * callers that need a never-throwing race should `.catch()` the input promise
- * themselves.
- */
-async function raceWithTimeout<T>(
-  p: Promise<T>,
-  ms: number,
-): Promise<T | "timed_out"> {
-  let t: ReturnType<typeof setTimeout> | null = null;
-  try {
-    return await Promise.race([
-      p,
-      new Promise<"timed_out">((resolve) => {
-        t = setTimeout(() => resolve("timed_out"), ms);
-        t.unref();
-      }),
-    ]);
-  } finally {
-    if (t) clearTimeout(t);
-  }
-}
-
 // ── Logging helpers ─────────────────────────────────────────────────────────
 
 async function createHeartbeatLogFile(): Promise<string> {

package/src/core/background/heartbeat/index.ts

@@ -9,8 +9,6 @@
  *   - `state`     — shared run-guard/timers/config + persisted-state I/O
  *   - `agent`     — prompt building + the one-shot agent run + log helpers
  *   - `scheduler` — init, the cadence timer, run guard, force/await/status API
- *
- * Re-exports the same public surface the old single-file module exposed.
  */
 
 export {

package/src/core/background/heartbeat/scheduler.ts

@@ -5,6 +5,7 @@
  */
 
 import { log, logError, logWarn } from "../../../util/log.js";
+import { faultText } from "../../engine/fault-text.js";
 import {
   catchupRunCount,
   missedRunCount,
@@ -147,6 +148,16 @@ export function getHeartbeatStatus(): HeartbeatState | null {
   return readHeartbeatState();
 }
 
+/** Arm the failure backoff so the due check stops re-firing for a while. */
+function backOff(err: unknown): void {
+  const until = hb.failureBackoff.fail(err);
+  logWarn(
+    "heartbeat",
+    `heartbeat.backoff failures=${hb.failureBackoff.failures} ` +
+      `until=${new Date(until).toISOString()} error="${faultText(err)}"`,
+  );
+}
+
 async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
   if (hb.running) return;
 
@@ -175,13 +186,16 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
         previousRunCount + 1,
       );
       // Only update last_run and increment run_count on success
-      writeHeartbeatState({
+      const persisted = writeHeartbeatState({
         last_run: Date.now(),
         last_started: now,
         status: "idle",
         run_count: previousRunCount + 1,
       });
-      hb.failureBackoff.succeed();
+      // A run whose last_run didn't stick looks overdue to the next due
+      // check, which would run the whole agent again every minute.
+      if (persisted) hb.failureBackoff.succeed();
+      else backOff(new Error("heartbeat state was not persisted"));
       log(
         "heartbeat",
         `Heartbeat #${previousRunCount + 1} complete (${trigger}), log: ${heartbeatLogPath}`,
@@ -197,23 +211,17 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
       // re-triggering against the same `last_run` forever. Non-timeout errors
       // retry from the previous successful run (no budget consumed).
       const isTimeout = err instanceof HeartbeatTimeoutError;
-      writeHeartbeatState({
+      const persisted = writeHeartbeatState({
         last_run: isTimeout ? Date.now() : previousLastRun,
         last_started: now,
         status: "idle",
         run_count: isTimeout ? previousRunCount + 1 : previousRunCount,
       });
       // Timeouts advance last_run (budget consumed), so the cadence itself
-      // spaces the next attempt. Every other failure retries against the same
-      // last_run — back off so the due check doesn't hammer it every minute.
-      if (!isTimeout) {
-        const until = hb.failureBackoff.fail(err);
-        logWarn(
-          "heartbeat",
-          `Backing off until ${new Date(until).toISOString()} ` +
-            `after ${hb.failureBackoff.failures} consecutive failure(s)`,
-        );
-      }
+      // spaces the next attempt — if that write landed. Every other failure
+      // retries against the same last_run — back off so the due check
+      // doesn't hammer it every minute.
+      if (!isTimeout || !persisted) backOff(err);
       if (trigger === "forced") throw err;
     } finally {
       hb.running = false;

package/src/core/background/heartbeat/state.ts

@@ -100,7 +100,10 @@ export const hb: {
   intervalMinutesRef: 60,
   config: null,
   logFileSequence: 0,
-  failureBackoff: new FailureBackoff(),
+  failureBackoff: new FailureBackoff({
+    key: "heartbeat.failing",
+    label: "The heartbeat",
+  }),
 };
 
 // ── State-file I/O ───────────────────────────────────────────────────────────
@@ -160,7 +163,11 @@ export function readHeartbeatState(): HeartbeatState | null {
   return normalizeHeartbeatState(kvGet(HEARTBEAT_STATE_KEY));
 }
 
-export function writeHeartbeatState(state: HeartbeatState): void {
+/**
+ * Persist the state. Returns whether last_run actually landed: kvSet logs
+ * and swallows a failed write (full disk), so the store is read back.
+ */
+export function writeHeartbeatState(state: HeartbeatState): boolean {
   // Re-derive last_run_at from last_run so the persisted ISO stamp can
   // never drift from the millisecond field; omit it on the sentinel
   // last_run === 0 (never-run) to match the pre-SQLite file format.
@@ -172,4 +179,5 @@ export function writeHeartbeatState(state: HeartbeatState): void {
       : {}),
   };
   kvSet(HEARTBEAT_STATE_KEY, enriched);
+  return readHeartbeatState()?.last_run === state.last_run;
 }

package/src/core/background/isolated-agent.ts

@@ -50,8 +50,12 @@ export interface IsolatedRunOptions {
   readonly logCategory?: LogComponent;
 }
 
-/** Resolves to the value, or the string "timed_out" if `ms` elapses first. */
-async function raceWithTimeout<T>(
+/**
+ * Resolves to the value, or the string "timed_out" if `ms` elapses first.
+ * A rejection of `promise` inside the window propagates — `.catch()` it
+ * first for a never-throwing race.
+ */
+export async function raceWithTimeout<T>(
   promise: Promise<T>,
   ms: number,
 ): Promise<T | "timed_out"> {

package/src/core/background/pulse/pulse.ts

@@ -27,6 +27,8 @@ import { formatSmartTimestamp } from "../../../util/time.js";
 let timer: ReturnType<typeof setInterval> | null = null;
 const registeredChats = new Set<string>();
 const lastCheckMessageId = new Map<string, number>();
+/** Chats whose pulse turn is still queued or running. */
+const inFlight = new Set<string>();
 
 const DEFAULT_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
 let activeIntervalMs = DEFAULT_INTERVAL_MS;
@@ -126,6 +128,10 @@ async function runPulse(): Promise<void> {
 }
 
 async function pulseChat(chatId: string): Promise<void> {
+  // The pulse turn queues behind the chat's other turns, so it can outlast
+  // the interval. Until it settles the checkpoint hasn't moved, and the next
+  // tick would dispatch the same unread messages a second time.
+  if (inFlight.has(chatId)) return;
   const numericChatId = parseInt(chatId, 10);
   if (isNaN(numericChatId)) {
     logError("pulse", `Invalid chatId: ${chatId}`);
@@ -157,6 +163,7 @@ async function pulseChat(chatId: string): Promise<void> {
     })
     .join("\n");
 
+  inFlight.add(chatId);
   try {
     const prompt =
       `[System: Pulse check — ${unread.length} new message(s) since last check. ` +
@@ -179,5 +186,7 @@ async function pulseChat(chatId: string): Promise<void> {
   } catch (err) {
     logError("pulse", `Chat ${chatId} failed`, err);
     // Don't update lastCheckMessageId — messages will be retried next pulse
+  } finally {
+    inFlight.delete(chatId);
   }
 }

package/src/core/background/triggers/exit.ts

@@ -25,6 +25,54 @@ import {
   WARDEN_GRACE_SLACK_MS,
 } from "./state.js";
 import { bufferAsPayload, fireWake } from "./output.js";
+import { raiseAlert, resolveAlert } from "../../frontend-runtime/alerts.js";
+import { faultText } from "../../engine/fault-text.js";
+
+/** Exit codes that mean the command itself could not run. */
+const CANNOT_RUN_CODES: Readonly<Record<number, string>> = {
+  126: "command not executable",
+  127: "command not found",
+};
+
+/**
+ * Tell the operator about a trigger that broke rather than finished: its
+ * command could not run (exit 126/127) or the process died on a signal
+ * nobody sent it (a crash, the OOM killer — Talon's own kills set the
+ * status before the exit lands, so they never reach `errored`). An
+ * ordinary non-zero exit is the script's own verdict and already reaches
+ * the chat as a wake. A clean fire clears an earlier alert for the id.
+ */
+function alertOnExit(
+  t: Trigger,
+  status: TriggerStatus,
+  code: number | null,
+  signal: NodeJS.Signals | null,
+  output: string[],
+): void {
+  const key = `trigger.${t.id}`;
+  if (status === "fired") {
+    resolveAlert(key, `Trigger "${t.name}" is firing normally again.`);
+    return;
+  }
+  if (status !== "errored") return;
+  const cause =
+    signal !== null
+      ? `crashed (${signal})`
+      : code !== null && CANNOT_RUN_CODES[code]
+        ? `could not run its command (exit ${code}: ${CANNOT_RUN_CODES[code]})`
+        : null;
+  if (!cause) return;
+  const last = output.at(-1);
+  log(
+    "triggers",
+    `trigger.broken id=${t.id} name="${t.name}" code=${code} signal=${signal} cause="${cause}"`,
+  );
+  raiseAlert(
+    key,
+    `Trigger "${t.name}" [${t.id}] ${cause}${last ? `: ${faultText(last)}` : "."}`,
+    { severity: "warn" },
+  );
+}
 
 export function handleTimeout(trigger: Trigger): void {
   timeouts.delete(trigger.id);
@@ -177,6 +225,7 @@ export async function finalizeExit(
     "triggers",
     `Exited "${t.name}" [${id}] code=${code} signal=${signal} → ${status}`,
   );
+  alertOnExit(t, status, code, signal, buffered);
 
   appendDailyLog(
     "Triggers",
@@ -211,6 +260,11 @@ export async function finalizeExit(
 
 export function failTrigger(t: Trigger, message: string): void {
   logError("triggers", `Failed to spawn ${t.id}: ${message}`);
+  raiseAlert(
+    `trigger.${t.id}`,
+    `Trigger "${t.name}" [${t.id}] failed to start: ${faultText(message)}`,
+    { severity: "warn" },
+  );
   updateTrigger(t.id, {
     status: "errored",
     lastError: message,

package/src/core/background/triggers/index.ts

@@ -15,7 +15,7 @@
  * Knows nothing about backend or frontend — dependencies are injected.
  */
 
-import { children, timeouts, logStreams, wardened } from "./state.js";
+import { children, timeouts } from "./state.js";
 import { commandForLanguage } from "./command.js";
 import { handleStdoutLine } from "./output.js";
 import { handleTimeout, finalizeExit } from "./exit.js";
@@ -35,8 +35,6 @@ export { resumeAfterRestart } from "./resume.js";
 export const _internals = {
   children,
   timeouts,
-  logStreams,
-  wardened,
   handleStdoutLine,
   handleTimeout,
   finalizeExit,

package/src/core/background/triggers/resume.ts

@@ -49,7 +49,7 @@ export async function resumeAfterRestart(): Promise<void> {
       continue;
     }
     // Late death notice. Two cases earn one:
-    //   - never fired at all (the old rule) — the chat heard nothing
+    //   - never fired at all — the chat heard nothing
     //     from this trigger, so its termination is news; and
     //   - killed by THIS restart — either the crash path
     //     (recoverInterrupted stamped RESTART_KILL_ERROR) or the clean
@@ -58,9 +58,7 @@ export async function resumeAfterRestart(): Promise<void> {
     //     backend pool was already going away). Even a multi-fire
     //     watcher that signalled mid-run was still an active promise
     //     when the process died, and without this wake the chat never
-    //     learns its watcher is gone. (Previously gated on
-    //     lastFireAt === undefined alone, which silently dropped
-    //     exactly those watchers.)
+    //     learns its watcher is gone.
     // Triggers that exited on their own already fired their terminal
     // wake (lastFireAt set, no marker) — they stay silent here.
     if (

package/src/core/backup/archive/tar.ts

@@ -22,8 +22,8 @@
  * `~/.ssh/authorized_keys` is not a safety net.
  */
 
-import { createReadStream, createWriteStream } from "node:fs";
-import { mkdir, symlink, utimes } from "node:fs/promises";
+import { createWriteStream } from "node:fs";
+import { mkdir, open, symlink, utimes } from "node:fs/promises";
 import { once } from "node:events";
 import { dirname, isAbsolute, relative, resolve } from "node:path";
 import type { Writable } from "node:stream";
@@ -262,6 +262,9 @@ export class TarWriter {
    * Stream a file from disk. `size` is the length recorded in the header:
    * a file that changes under us is truncated or zero-padded to it, because
    * a tar whose payload length disagrees with its header is unreadable.
+   * The source is opened before the header goes out, so a file that
+   * vanished or became unreadable since it was listed throws with nothing
+   * written — the archive is still whole and the caller may carry on.
    */
   async addFile(
     path: string,
@@ -270,9 +273,16 @@ export class TarWriter {
     mtime: number,
     size: number,
   ): Promise<void> {
-    await this.writeHeaders({ path, type: "file", mode, mtime, size });
+    const handle = await open(source, "r");
+    try {
+      await this.writeHeaders({ path, type: "file", mode, mtime, size });
+    } catch (err) {
+      await handle.close();
+      throw err;
+    }
     let written = 0;
-    const stream = createReadStream(source);
+    // Owns the handle from here: closed when the stream ends or is destroyed.
+    const stream = handle.createReadStream();
     for await (const chunk of stream) {
       const buf = chunk as Buffer;
       const room = size - written;

package/src/core/backup/restore.ts

@@ -63,13 +63,19 @@ import {
   rewriteConfigForClone,
   type CloneTarget,
 } from "./sources/relocate.js";
-import { isSnapshotId, partPath, readManifest, snapshotDir } from "./store.js";
+import { pathExists } from "./sources/sessions.js";
+import {
+  DB_MEMBER,
+  isSnapshotId,
+  partPath,
+  readManifest,
+  snapshotDir,
+} from "./store.js";
 import type { BackupTarget } from "./targets.js";
 import type { BackupSettings, Manifest, SnapshotPart } from "./types.js";
 
 /** A staged request older than this is stale and ignored. */
 export const RESTORE_PENDING_MAX_AGE_MS = 10 * 60_000;
-const DB_MEMBER = "db/talon.db";
 
 export type RestorePending = {
   id: string;
@@ -165,15 +171,6 @@ export async function readRestorePending(
 
 // ── Parts ───────────────────────────────────────────────────────────────────
 
-async function isLocal(path: string): Promise<boolean> {
-  try {
-    await stat(path);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 /** The parts of this snapshot that are not on local disk. */
 async function missingParts(
   manifest: Manifest,
@@ -181,7 +178,7 @@ async function missingParts(
 ): Promise<SnapshotPart[]> {
   const missing: SnapshotPart[] = [];
   for (const part of manifest.parts) {
-    if (!(await isLocal(partPath(manifest.id, part.name, home)))) {
+    if (!(await pathExists(partPath(manifest.id, part.name, home)))) {
       missing.push(part);
     }
   }
@@ -218,7 +215,18 @@ async function ensureParts(
     const path = partPath(manifest.id, part.name, home);
     log("backup", `Downloading ${part.name} from ${target.id}…`);
     await mkdir(dirname(path), { recursive: true, mode: 0o700 });
-    await target.download(manifest.id, part.name, path);
+    // Into a side file, renamed only once complete: a download cut short
+    // under the part's own name would count as present next time, and
+    // every later restore would fail its checksum instead of refetching.
+    const partial = `${path}.partial`;
+    await rm(partial, { force: true });
+    try {
+      await target.download(manifest.id, part.name, partial);
+      await rename(partial, path);
+    } catch (err) {
+      await rm(partial, { force: true });
+      throw err;
+    }
   }
   return manifest.parts.filter((part) => !skipped.has(part.name));
 }

package/src/core/backup/scheduler.ts

@@ -26,6 +26,8 @@
 import { FailureBackoff } from "../background/failure-backoff.js";
 import { TalonError } from "../errors.js";
 import { notifyAdmin } from "../frontend-runtime/admin-notify.js";
+import { raiseAlert, resolveAlert } from "../frontend-runtime/alerts.js";
+import { faultText } from "../engine/fault-text.js";
 import { bus } from "../bus/index.js";
 import { log, logError } from "../../util/log.js";
 import { dirs } from "../../util/paths.js";
@@ -117,6 +119,7 @@ export function firstRunDelayMs(
 // ── Runs ────────────────────────────────────────────────────────────────────
 
 const backoff = new FailureBackoff();
+const BACKUP_ALERT = "backup.failing";
 
 async function executeRun(request: RunRequest): Promise<Manifest> {
   const settings = state.settings;
@@ -163,6 +166,7 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
       }
     }
     backoff.succeed();
+    resolveAlert(BACKUP_ALERT, "Backups are succeeding again.");
     return manifest;
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
@@ -177,21 +181,36 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
     logError("backup", `Run failed (${request.trigger})`, err);
     // Once per streak: the second identical failure tells the admin nothing
     // the first one did not.
-    if (backoff.failures === 1) {
-      await state
-        .notify(
-          `⚠️ Backup failed: ${message}\nRetrying after ${new Date(until).toISOString().slice(11, 16)} UTC.`,
-        )
-        .catch(() => {
-          /* the notifier logs its own failures */
-        });
-    }
+    if (backoff.failures === 1) await reportFailure(message, until);
     throw err;
   } finally {
     state.running = false;
   }
 }
 
+/**
+ * Tell the admin a backup streak started. The default route is an operator
+ * alert (`backup.failing`), so the recovery is announced too; a deployment
+ * that routes backup notices to its own chat (`backup.notifyChatId`) keeps
+ * getting the plain notice there.
+ */
+async function reportFailure(message: string, until: number): Promise<void> {
+  const retryAt = new Date(until).toISOString().slice(11, 16);
+  if (state.notify === notifyAdmin) {
+    raiseAlert(
+      BACKUP_ALERT,
+      `Backup failed: ${faultText(message, 300)}. Retrying after ${retryAt} UTC.`,
+      { severity: "warn" },
+    );
+    return;
+  }
+  await state
+    .notify(`⚠️ Backup failed: ${message}\nRetrying after ${retryAt} UTC.`)
+    .catch(() => {
+      /* the notifier logs its own failures */
+    });
+}
+
 /**
  * Take a snapshot. Requests queue behind whatever is already running, so
  * this resolves with THIS request's snapshot, not someone else's.