talon-agent 5.18.2 → 5.19.0

package/src/core/background/triggers/exit.ts

@@ -25,6 +25,54 @@ import {
   WARDEN_GRACE_SLACK_MS,
 } from "./state.js";
 import { bufferAsPayload, fireWake } from "./output.js";
+import { raiseAlert, resolveAlert } from "../../frontend-runtime/alerts.js";
+import { faultText } from "../../engine/fault-text.js";
+
+/** Exit codes that mean the command itself could not run. */
+const CANNOT_RUN_CODES: Readonly<Record<number, string>> = {
+  126: "command not executable",
+  127: "command not found",
+};
+
+/**
+ * Tell the operator about a trigger that broke rather than finished: its
+ * command could not run (exit 126/127) or the process died on a signal
+ * nobody sent it (a crash, the OOM killer — Talon's own kills set the
+ * status before the exit lands, so they never reach `errored`). An
+ * ordinary non-zero exit is the script's own verdict and already reaches
+ * the chat as a wake. A clean fire clears an earlier alert for the id.
+ */
+function alertOnExit(
+  t: Trigger,
+  status: TriggerStatus,
+  code: number | null,
+  signal: NodeJS.Signals | null,
+  output: string[],
+): void {
+  const key = `trigger.${t.id}`;
+  if (status === "fired") {
+    resolveAlert(key, `Trigger "${t.name}" is firing normally again.`);
+    return;
+  }
+  if (status !== "errored") return;
+  const cause =
+    signal !== null
+      ? `crashed (${signal})`
+      : code !== null && CANNOT_RUN_CODES[code]
+        ? `could not run its command (exit ${code}: ${CANNOT_RUN_CODES[code]})`
+        : null;
+  if (!cause) return;
+  const last = output.at(-1);
+  log(
+    "triggers",
+    `trigger.broken id=${t.id} name="${t.name}" code=${code} signal=${signal} cause="${cause}"`,
+  );
+  raiseAlert(
+    key,
+    `Trigger "${t.name}" [${t.id}] ${cause}${last ? `: ${faultText(last)}` : "."}`,
+    { severity: "warn" },
+  );
+}
 
 export function handleTimeout(trigger: Trigger): void {
   timeouts.delete(trigger.id);
@@ -177,6 +225,7 @@ export async function finalizeExit(
     "triggers",
     `Exited "${t.name}" [${id}] code=${code} signal=${signal} → ${status}`,
   );
+  alertOnExit(t, status, code, signal, buffered);
 
   appendDailyLog(
     "Triggers",
@@ -211,6 +260,11 @@ export async function finalizeExit(
 
 export function failTrigger(t: Trigger, message: string): void {
   logError("triggers", `Failed to spawn ${t.id}: ${message}`);
+  raiseAlert(
+    `trigger.${t.id}`,
+    `Trigger "${t.name}" [${t.id}] failed to start: ${faultText(message)}`,
+    { severity: "warn" },
+  );
   updateTrigger(t.id, {
     status: "errored",
     lastError: message,

package/src/core/background/triggers/index.ts

@@ -15,7 +15,7 @@
  * Knows nothing about backend or frontend — dependencies are injected.
  */
 
-import { children, timeouts, logStreams, wardened } from "./state.js";
+import { children, timeouts } from "./state.js";
 import { commandForLanguage } from "./command.js";
 import { handleStdoutLine } from "./output.js";
 import { handleTimeout, finalizeExit } from "./exit.js";
@@ -35,8 +35,6 @@ export { resumeAfterRestart } from "./resume.js";
 export const _internals = {
   children,
   timeouts,
-  logStreams,
-  wardened,
   handleStdoutLine,
   handleTimeout,
   finalizeExit,

package/src/core/background/triggers/resume.ts

@@ -49,7 +49,7 @@ export async function resumeAfterRestart(): Promise<void> {
       continue;
     }
     // Late death notice. Two cases earn one:
-    //   - never fired at all (the old rule) — the chat heard nothing
+    //   - never fired at all — the chat heard nothing
     //     from this trigger, so its termination is news; and
     //   - killed by THIS restart — either the crash path
     //     (recoverInterrupted stamped RESTART_KILL_ERROR) or the clean
@@ -58,9 +58,7 @@ export async function resumeAfterRestart(): Promise<void> {
     //     backend pool was already going away). Even a multi-fire
     //     watcher that signalled mid-run was still an active promise
     //     when the process died, and without this wake the chat never
-    //     learns its watcher is gone. (Previously gated on
-    //     lastFireAt === undefined alone, which silently dropped
-    //     exactly those watchers.)
+    //     learns its watcher is gone.
     // Triggers that exited on their own already fired their terminal
     // wake (lastFireAt set, no marker) — they stay silent here.
     if (

package/src/core/backup/archive/tar.ts

@@ -22,8 +22,8 @@
  * `~/.ssh/authorized_keys` is not a safety net.
  */
 
-import { createReadStream, createWriteStream } from "node:fs";
-import { mkdir, symlink, utimes } from "node:fs/promises";
+import { createWriteStream } from "node:fs";
+import { mkdir, open, symlink, utimes } from "node:fs/promises";
 import { once } from "node:events";
 import { dirname, isAbsolute, relative, resolve } from "node:path";
 import type { Writable } from "node:stream";
@@ -262,6 +262,9 @@ export class TarWriter {
    * Stream a file from disk. `size` is the length recorded in the header:
    * a file that changes under us is truncated or zero-padded to it, because
    * a tar whose payload length disagrees with its header is unreadable.
+   * The source is opened before the header goes out, so a file that
+   * vanished or became unreadable since it was listed throws with nothing
+   * written — the archive is still whole and the caller may carry on.
    */
   async addFile(
     path: string,
@@ -270,9 +273,16 @@ export class TarWriter {
     mtime: number,
     size: number,
   ): Promise<void> {
-    await this.writeHeaders({ path, type: "file", mode, mtime, size });
+    const handle = await open(source, "r");
+    try {
+      await this.writeHeaders({ path, type: "file", mode, mtime, size });
+    } catch (err) {
+      await handle.close();
+      throw err;
+    }
     let written = 0;
-    const stream = createReadStream(source);
+    // Owns the handle from here: closed when the stream ends or is destroyed.
+    const stream = handle.createReadStream();
     for await (const chunk of stream) {
       const buf = chunk as Buffer;
       const room = size - written;

package/src/core/backup/restore.ts

@@ -63,13 +63,19 @@ import {
   rewriteConfigForClone,
   type CloneTarget,
 } from "./sources/relocate.js";
-import { isSnapshotId, partPath, readManifest, snapshotDir } from "./store.js";
+import { pathExists } from "./sources/sessions.js";
+import {
+  DB_MEMBER,
+  isSnapshotId,
+  partPath,
+  readManifest,
+  snapshotDir,
+} from "./store.js";
 import type { BackupTarget } from "./targets.js";
 import type { BackupSettings, Manifest, SnapshotPart } from "./types.js";
 
 /** A staged request older than this is stale and ignored. */
 export const RESTORE_PENDING_MAX_AGE_MS = 10 * 60_000;
-const DB_MEMBER = "db/talon.db";
 
 export type RestorePending = {
   id: string;
@@ -165,15 +171,6 @@ export async function readRestorePending(
 
 // ── Parts ───────────────────────────────────────────────────────────────────
 
-async function isLocal(path: string): Promise<boolean> {
-  try {
-    await stat(path);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 /** The parts of this snapshot that are not on local disk. */
 async function missingParts(
   manifest: Manifest,
@@ -181,7 +178,7 @@ async function missingParts(
 ): Promise<SnapshotPart[]> {
   const missing: SnapshotPart[] = [];
   for (const part of manifest.parts) {
-    if (!(await isLocal(partPath(manifest.id, part.name, home)))) {
+    if (!(await pathExists(partPath(manifest.id, part.name, home)))) {
       missing.push(part);
     }
   }
@@ -218,7 +215,18 @@ async function ensureParts(
     const path = partPath(manifest.id, part.name, home);
     log("backup", `Downloading ${part.name} from ${target.id}…`);
     await mkdir(dirname(path), { recursive: true, mode: 0o700 });
-    await target.download(manifest.id, part.name, path);
+    // Into a side file, renamed only once complete: a download cut short
+    // under the part's own name would count as present next time, and
+    // every later restore would fail its checksum instead of refetching.
+    const partial = `${path}.partial`;
+    await rm(partial, { force: true });
+    try {
+      await target.download(manifest.id, part.name, partial);
+      await rename(partial, path);
+    } catch (err) {
+      await rm(partial, { force: true });
+      throw err;
+    }
   }
   return manifest.parts.filter((part) => !skipped.has(part.name));
 }

package/src/core/backup/scheduler.ts

@@ -26,6 +26,8 @@
 import { FailureBackoff } from "../background/failure-backoff.js";
 import { TalonError } from "../errors.js";
 import { notifyAdmin } from "../frontend-runtime/admin-notify.js";
+import { raiseAlert, resolveAlert } from "../frontend-runtime/alerts.js";
+import { faultText } from "../engine/fault-text.js";
 import { bus } from "../bus/index.js";
 import { log, logError } from "../../util/log.js";
 import { dirs } from "../../util/paths.js";
@@ -117,6 +119,7 @@ export function firstRunDelayMs(
 // ── Runs ────────────────────────────────────────────────────────────────────
 
 const backoff = new FailureBackoff();
+const BACKUP_ALERT = "backup.failing";
 
 async function executeRun(request: RunRequest): Promise<Manifest> {
   const settings = state.settings;
@@ -163,6 +166,7 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
       }
     }
     backoff.succeed();
+    resolveAlert(BACKUP_ALERT, "Backups are succeeding again.");
     return manifest;
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
@@ -177,21 +181,36 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
     logError("backup", `Run failed (${request.trigger})`, err);
     // Once per streak: the second identical failure tells the admin nothing
     // the first one did not.
-    if (backoff.failures === 1) {
-      await state
-        .notify(
-          `⚠️ Backup failed: ${message}\nRetrying after ${new Date(until).toISOString().slice(11, 16)} UTC.`,
-        )
-        .catch(() => {
-          /* the notifier logs its own failures */
-        });
-    }
+    if (backoff.failures === 1) await reportFailure(message, until);
     throw err;
   } finally {
     state.running = false;
   }
 }
 
+/**
+ * Tell the admin a backup streak started. The default route is an operator
+ * alert (`backup.failing`), so the recovery is announced too; a deployment
+ * that routes backup notices to its own chat (`backup.notifyChatId`) keeps
+ * getting the plain notice there.
+ */
+async function reportFailure(message: string, until: number): Promise<void> {
+  const retryAt = new Date(until).toISOString().slice(11, 16);
+  if (state.notify === notifyAdmin) {
+    raiseAlert(
+      BACKUP_ALERT,
+      `Backup failed: ${faultText(message, 300)}. Retrying after ${retryAt} UTC.`,
+      { severity: "warn" },
+    );
+    return;
+  }
+  await state
+    .notify(`⚠️ Backup failed: ${message}\nRetrying after ${retryAt} UTC.`)
+    .catch(() => {
+      /* the notifier logs its own failures */
+    });
+}
+
 /**
  * Take a snapshot. Requests queue behind whatever is already running, so
  * this resolves with THIS request's snapshot, not someone else's.

package/src/core/backup/snapshot.ts

@@ -72,6 +72,7 @@ import {
   type SourceContext,
 } from "./sources/sessions.js";
 import {
+  DB_MEMBER,
   STATE_PART,
   indexSnapshot,
   linkOrCopy,
@@ -88,8 +89,6 @@ import type {
   SnapshotPart,
 } from "./types.js";
 
-/** Where the database copy lands inside the archive. */
-const DB_MEMBER = "db/talon.db";
 /** The part that holds WhatsApp auth and the userbot session. */
 const LOGINS_PART = "logins.tar.zst";
 /** How a clone reinstalls fetched plugins (see sources/plugins.ts). */
@@ -120,10 +119,18 @@ export type BuildOptions = {
 
 // ── Archive writing ─────────────────────────────────────────────────────────
 
+/**
+ * Open failures that mean "this file is gone or locked since the walk":
+ * traces and backend transcripts churn while a snapshot runs, and one of
+ * them disappearing must not cost the whole backup (see collectTree).
+ */
+const SKIPPABLE_OPEN_ERRORS = new Set(["ENOENT", "EACCES", "EPERM"]);
+
 async function addEntries(
   writer: TarWriter,
   entries: readonly SourceEntry[],
 ): Promise<void> {
+  const skipped: string[] = [];
   for (const entry of entries) {
     if (entry.type === "dir") {
       await writer.addDirectory(entry.archivePath, entry.mode, entry.mtime);
@@ -135,15 +142,27 @@ async function addEntries(
         entry.mtime,
       );
     } else {
-      await writer.addFile(
-        entry.archivePath,
-        entry.source,
-        entry.mode,
-        entry.mtime,
-        entry.size,
-      );
+      try {
+        await writer.addFile(
+          entry.archivePath,
+          entry.source,
+          entry.mode,
+          entry.mtime,
+          entry.size,
+        );
+      } catch (err) {
+        const code = (err as NodeJS.ErrnoException).code ?? "";
+        if (!SKIPPABLE_OPEN_ERRORS.has(code)) throw err;
+        skipped.push(`${entry.source} (${code})`);
+      }
     }
   }
+  if (skipped.length > 0) {
+    logWarn(
+      "backup",
+      `Skipped ${skipped.length} file(s) that vanished or became unreadable mid-snapshot; first: ${skipped[0]}`,
+    );
+  }
 }
 
 /** A part's file name: `.enc` marks one written through the encryptor. */
@@ -171,6 +190,11 @@ async function writePart(
   const flushed = passphrase
     ? pipeline(compressor, await createEncryptor(passphrase), tap, out)
     : pipeline(compressor, tap, out);
+  // A sink failure (ENOSPC) rejects this while `fill` is still writing, and
+  // `fill` then throws the same error — so the await below is never
+  // reached. Observe it here, or it surfaces as an unhandled rejection
+  // that kills the CLI before the cleanup runs.
+  flushed.catch(() => {});
   try {
     const writer = new TarWriter(compressor);
     await fill(writer);
@@ -312,11 +336,20 @@ async function palaceFingerprint(
   const files: TreeFile[] = [];
   for (const entry of entries) {
     if (entry.type !== "file") continue;
+    let sha256: string;
+    try {
+      sha256 = await sha256File(entry.source);
+    } catch (err) {
+      // Gone since the walk: the part will skip it too (see addEntries).
+      const code = (err as NodeJS.ErrnoException).code ?? "";
+      if (SKIPPABLE_OPEN_ERRORS.has(code)) continue;
+      throw err;
+    }
     files.push({
       path: entry.archivePath,
       size: entry.size,
       mtime: entry.mtime,
-      sha256: await sha256File(entry.source),
+      sha256,
     });
   }
   return treeHash(files);

package/src/core/backup/store.ts

@@ -17,15 +17,7 @@
  */
 
 import { randomBytes } from "node:crypto";
-import {
-  link,
-  copyFile,
-  mkdir,
-  readdir,
-  readFile,
-  rm,
-  stat,
-} from "node:fs/promises";
+import { link, copyFile, mkdir, readdir, readFile, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import writeFileAtomic from "write-file-atomic";
 import { dirs } from "../../util/paths.js";
@@ -40,6 +32,7 @@ import {
   recordBackup,
   updateBackupManifest,
 } from "../../storage/backup/index.js";
+import { pathExists } from "./sources/sessions.js";
 import type {
   Manifest,
   RemoteState,
@@ -51,6 +44,8 @@ import type {
 const SNAPSHOT_ID_RE = /^\d{8}T\d{6}Z-[0-9a-f]{6}$/;
 const MANIFEST_NAME = "manifest.json";
 export const STATE_PART = "state.tar.zst";
+/** Where the database copy lands inside the state part. */
+export const DB_MEMBER = "db/talon.db";
 
 function backupsRoot(home: string = dirs.root): string {
   return join(home, "backups");
@@ -212,22 +207,15 @@ export async function listSnapshots(
       pinned: record.pinned,
       createdAt: record.createdAt,
       sizeBytes: record.sizeBytes,
-      local: await exists(join(snapshotDir(record.id, home), MANIFEST_NAME)),
+      local: await pathExists(
+        join(snapshotDir(record.id, home), MANIFEST_NAME),
+      ),
       remote,
     });
   }
   return summaries;
 }
 
-async function exists(path: string): Promise<boolean> {
-  try {
-    await stat(path);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 /** Pin or unpin a snapshot, on disk and in the index. Returns false if unknown. */
 export async function setSnapshotPinned(
   id: string,

package/src/core/backup/targets.ts

@@ -30,6 +30,17 @@ import type { Manifest, SnapshotPart } from "./types.js";
 /** The daemon's chat id for plugin actions that belong to no conversation. */
 const SYSTEM_CHAT = "system";
 
+/**
+ * How long a target may take to answer. A plugin that never answers would
+ * otherwise hold the backup queue forever — every later snapshot, and the
+ * pre-update checkpoint `/update` waits on, queued behind it. Part uploads
+ * get a base allowance plus time for the bytes at a slow uplink's pace.
+ */
+const CONTROL_DEADLINE_MS = 10 * 60_000;
+const TRANSFER_BASE_MS = 30 * 60_000;
+/** 64 KiB/s — well under any link a remote backup is usable over. */
+const TRANSFER_FLOOR_BYTES_PER_MS = 64;
+
 /** A part as the upload call describes it: metadata plus where to read it. */
 type UploadPart = SnapshotPart & { path: string };
 
@@ -101,6 +112,37 @@ function targetError(
   });
 }
 
+/** setTimeout's ceiling; a longer delay would fire at once. */
+const MAX_TIMER_MS = 2 ** 31 - 1;
+
+/**
+ * Send one body, failing with `target`'s name once `deadlineMs` passes.
+ * A null deadline sends it unbounded.
+ */
+async function dispatchWithin(
+  deps: TargetDeps,
+  plugin: string,
+  body: Record<string, unknown>,
+  target: string,
+  deadlineMs: number | null,
+): Promise<ActionResult | null> {
+  if (deadlineMs === null) return deps.dispatch(plugin, body);
+  const ms = Math.min(deadlineMs, MAX_TIMER_MS);
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      const action = String(body.action).replace(/^backup\.target\./, "");
+      const minutes = Math.round(ms / 60_000);
+      reject(targetError(target, action, `no answer within ${minutes} min`));
+    }, ms);
+  });
+  try {
+    return await Promise.race([deps.dispatch(plugin, body), deadline]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 /** Unwrap `{ ok, data }`, turning every failure shape into one error. */
 function dataOf(
   result: ActionResult | null,
@@ -128,8 +170,11 @@ class PluginTarget implements BackupTarget {
     readonly detail?: string,
   ) {}
 
-  private send(body: Record<string, unknown>): Promise<ActionResult | null> {
-    return this.deps.dispatch(this.plugin, body);
+  private send(
+    body: Record<string, unknown>,
+    deadlineMs: number | null = CONTROL_DEADLINE_MS,
+  ): Promise<ActionResult | null> {
+    return dispatchWithin(this.deps, this.plugin, body, this.id, deadlineMs);
   }
 
   async upload(
@@ -138,12 +183,10 @@ class PluginTarget implements BackupTarget {
     manifest: Manifest,
   ): Promise<{ remoteId: string; deduplicated?: boolean }> {
     const data = dataOf(
-      await this.send({
-        action: "backup.target.upload",
-        snapshotId,
-        part,
-        manifest,
-      }),
+      await this.send(
+        { action: "backup.target.upload", snapshotId, part, manifest },
+        TRANSFER_BASE_MS + part.bytes / TRANSFER_FLOOR_BYTES_PER_MS,
+      ),
       this.id,
       "upload",
     );
@@ -206,12 +249,17 @@ class PluginTarget implements BackupTarget {
     destPath: string,
   ): Promise<void> {
     dataOf(
-      await this.send({
-        action: "backup.target.download",
-        snapshotId,
-        part: { name: partName },
-        destPath,
-      }),
+      // Unbounded: only an interactive restore downloads, the size is not
+      // known here, and the operator can interrupt it.
+      await this.send(
+        {
+          action: "backup.target.download",
+          snapshotId,
+          part: { name: partName },
+          destPath,
+        },
+        null,
+      ),
       this.id,
       "download",
     );
@@ -231,9 +279,13 @@ export async function discoverTargets(
   for (const plugin of deps.plugins()) {
     let result: ActionResult | null;
     try {
-      result = await deps.dispatch(plugin, {
-        action: "backup.target.describe",
-      });
+      result = await dispatchWithin(
+        deps,
+        plugin,
+        { action: "backup.target.describe" },
+        plugin,
+        CONTROL_DEADLINE_MS,
+      );
     } catch (err) {
       logWarn(
         "backup",

package/src/core/config/index.ts

@@ -478,6 +478,20 @@ const configSchema = z.object({
   planAlertThreshold: z.number().int().min(1).max(100).default(80),
   /** Chat that receives plan warnings. Defaults to `adminUserId`. */
   planAlertChatId: z.string().optional(),
+  /**
+   * Operator alerts (core/frontend-runtime/alerts.ts): faults that need a
+   * human — a full disk, a crash, an error spike, a dead frontend — sent
+   * to the admin chat, once per fault per `cooldownMinutes`, with a
+   * recovery notice when it clears. `enabled: false` keeps them in the
+   * log and in `talon status` only.
+   */
+  alerts: z
+    .object({
+      enabled: z.boolean().default(true),
+      cooldownMinutes: z.number().int().min(0).max(1440).default(30),
+    })
+    .strict()
+    .optional(),
   /** Background memory-consolidation (dream) runs. Mirrors `pulse`/`heartbeat`. */
   dream: z.boolean().default(true),
   /**