talon-agent 5.18.1 → 5.19.0

package/src/core/backup/snapshot.ts

@@ -72,6 +72,7 @@ import {
   type SourceContext,
 } from "./sources/sessions.js";
 import {
+  DB_MEMBER,
   STATE_PART,
   indexSnapshot,
   linkOrCopy,
@@ -88,8 +89,6 @@ import type {
   SnapshotPart,
 } from "./types.js";
 
-/** Where the database copy lands inside the archive. */
-const DB_MEMBER = "db/talon.db";
 /** The part that holds WhatsApp auth and the userbot session. */
 const LOGINS_PART = "logins.tar.zst";
 /** How a clone reinstalls fetched plugins (see sources/plugins.ts). */
@@ -120,10 +119,18 @@ export type BuildOptions = {
 
 // ── Archive writing ─────────────────────────────────────────────────────────
 
+/**
+ * Open failures that mean "this file is gone or locked since the walk":
+ * traces and backend transcripts churn while a snapshot runs, and one of
+ * them disappearing must not cost the whole backup (see collectTree).
+ */
+const SKIPPABLE_OPEN_ERRORS = new Set(["ENOENT", "EACCES", "EPERM"]);
+
 async function addEntries(
   writer: TarWriter,
   entries: readonly SourceEntry[],
 ): Promise<void> {
+  const skipped: string[] = [];
   for (const entry of entries) {
     if (entry.type === "dir") {
       await writer.addDirectory(entry.archivePath, entry.mode, entry.mtime);
@@ -135,15 +142,27 @@ async function addEntries(
         entry.mtime,
       );
     } else {
-      await writer.addFile(
-        entry.archivePath,
-        entry.source,
-        entry.mode,
-        entry.mtime,
-        entry.size,
-      );
+      try {
+        await writer.addFile(
+          entry.archivePath,
+          entry.source,
+          entry.mode,
+          entry.mtime,
+          entry.size,
+        );
+      } catch (err) {
+        const code = (err as NodeJS.ErrnoException).code ?? "";
+        if (!SKIPPABLE_OPEN_ERRORS.has(code)) throw err;
+        skipped.push(`${entry.source} (${code})`);
+      }
     }
   }
+  if (skipped.length > 0) {
+    logWarn(
+      "backup",
+      `Skipped ${skipped.length} file(s) that vanished or became unreadable mid-snapshot; first: ${skipped[0]}`,
+    );
+  }
 }
 
 /** A part's file name: `.enc` marks one written through the encryptor. */
@@ -171,6 +190,11 @@ async function writePart(
   const flushed = passphrase
     ? pipeline(compressor, await createEncryptor(passphrase), tap, out)
     : pipeline(compressor, tap, out);
+  // A sink failure (ENOSPC) rejects this while `fill` is still writing, and
+  // `fill` then throws the same error — so the await below is never
+  // reached. Observe it here, or it surfaces as an unhandled rejection
+  // that kills the CLI before the cleanup runs.
+  flushed.catch(() => {});
   try {
     const writer = new TarWriter(compressor);
     await fill(writer);
@@ -312,11 +336,20 @@ async function palaceFingerprint(
   const files: TreeFile[] = [];
   for (const entry of entries) {
     if (entry.type !== "file") continue;
+    let sha256: string;
+    try {
+      sha256 = await sha256File(entry.source);
+    } catch (err) {
+      // Gone since the walk: the part will skip it too (see addEntries).
+      const code = (err as NodeJS.ErrnoException).code ?? "";
+      if (SKIPPABLE_OPEN_ERRORS.has(code)) continue;
+      throw err;
+    }
     files.push({
       path: entry.archivePath,
       size: entry.size,
       mtime: entry.mtime,
-      sha256: await sha256File(entry.source),
+      sha256,
     });
   }
   return treeHash(files);

package/src/core/backup/store.ts

@@ -17,15 +17,7 @@
  */
 
 import { randomBytes } from "node:crypto";
-import {
-  link,
-  copyFile,
-  mkdir,
-  readdir,
-  readFile,
-  rm,
-  stat,
-} from "node:fs/promises";
+import { link, copyFile, mkdir, readdir, readFile, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import writeFileAtomic from "write-file-atomic";
 import { dirs } from "../../util/paths.js";
@@ -40,6 +32,7 @@ import {
   recordBackup,
   updateBackupManifest,
 } from "../../storage/backup/index.js";
+import { pathExists } from "./sources/sessions.js";
 import type {
   Manifest,
   RemoteState,
@@ -51,6 +44,8 @@ import type {
 const SNAPSHOT_ID_RE = /^\d{8}T\d{6}Z-[0-9a-f]{6}$/;
 const MANIFEST_NAME = "manifest.json";
 export const STATE_PART = "state.tar.zst";
+/** Where the database copy lands inside the state part. */
+export const DB_MEMBER = "db/talon.db";
 
 function backupsRoot(home: string = dirs.root): string {
   return join(home, "backups");
@@ -212,22 +207,15 @@ export async function listSnapshots(
       pinned: record.pinned,
       createdAt: record.createdAt,
       sizeBytes: record.sizeBytes,
-      local: await exists(join(snapshotDir(record.id, home), MANIFEST_NAME)),
+      local: await pathExists(
+        join(snapshotDir(record.id, home), MANIFEST_NAME),
+      ),
       remote,
     });
   }
   return summaries;
 }
 
-async function exists(path: string): Promise<boolean> {
-  try {
-    await stat(path);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 /** Pin or unpin a snapshot, on disk and in the index. Returns false if unknown. */
 export async function setSnapshotPinned(
   id: string,

package/src/core/backup/targets.ts

@@ -30,6 +30,17 @@ import type { Manifest, SnapshotPart } from "./types.js";
 /** The daemon's chat id for plugin actions that belong to no conversation. */
 const SYSTEM_CHAT = "system";
 
+/**
+ * How long a target may take to answer. A plugin that never answers would
+ * otherwise hold the backup queue forever — every later snapshot, and the
+ * pre-update checkpoint `/update` waits on, queued behind it. Part uploads
+ * get a base allowance plus time for the bytes at a slow uplink's pace.
+ */
+const CONTROL_DEADLINE_MS = 10 * 60_000;
+const TRANSFER_BASE_MS = 30 * 60_000;
+/** 64 KiB/s — well under any link a remote backup is usable over. */
+const TRANSFER_FLOOR_BYTES_PER_MS = 64;
+
 /** A part as the upload call describes it: metadata plus where to read it. */
 type UploadPart = SnapshotPart & { path: string };
 
@@ -101,6 +112,37 @@ function targetError(
   });
 }
 
+/** setTimeout's ceiling; a longer delay would fire at once. */
+const MAX_TIMER_MS = 2 ** 31 - 1;
+
+/**
+ * Send one body, failing with `target`'s name once `deadlineMs` passes.
+ * A null deadline sends it unbounded.
+ */
+async function dispatchWithin(
+  deps: TargetDeps,
+  plugin: string,
+  body: Record<string, unknown>,
+  target: string,
+  deadlineMs: number | null,
+): Promise<ActionResult | null> {
+  if (deadlineMs === null) return deps.dispatch(plugin, body);
+  const ms = Math.min(deadlineMs, MAX_TIMER_MS);
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      const action = String(body.action).replace(/^backup\.target\./, "");
+      const minutes = Math.round(ms / 60_000);
+      reject(targetError(target, action, `no answer within ${minutes} min`));
+    }, ms);
+  });
+  try {
+    return await Promise.race([deps.dispatch(plugin, body), deadline]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 /** Unwrap `{ ok, data }`, turning every failure shape into one error. */
 function dataOf(
   result: ActionResult | null,
@@ -128,8 +170,11 @@ class PluginTarget implements BackupTarget {
     readonly detail?: string,
   ) {}
 
-  private send(body: Record<string, unknown>): Promise<ActionResult | null> {
-    return this.deps.dispatch(this.plugin, body);
+  private send(
+    body: Record<string, unknown>,
+    deadlineMs: number | null = CONTROL_DEADLINE_MS,
+  ): Promise<ActionResult | null> {
+    return dispatchWithin(this.deps, this.plugin, body, this.id, deadlineMs);
   }
 
   async upload(
@@ -138,12 +183,10 @@ class PluginTarget implements BackupTarget {
     manifest: Manifest,
   ): Promise<{ remoteId: string; deduplicated?: boolean }> {
     const data = dataOf(
-      await this.send({
-        action: "backup.target.upload",
-        snapshotId,
-        part,
-        manifest,
-      }),
+      await this.send(
+        { action: "backup.target.upload", snapshotId, part, manifest },
+        TRANSFER_BASE_MS + part.bytes / TRANSFER_FLOOR_BYTES_PER_MS,
+      ),
       this.id,
       "upload",
     );
@@ -206,12 +249,17 @@ class PluginTarget implements BackupTarget {
     destPath: string,
   ): Promise<void> {
     dataOf(
-      await this.send({
-        action: "backup.target.download",
-        snapshotId,
-        part: { name: partName },
-        destPath,
-      }),
+      // Unbounded: only an interactive restore downloads, the size is not
+      // known here, and the operator can interrupt it.
+      await this.send(
+        {
+          action: "backup.target.download",
+          snapshotId,
+          part: { name: partName },
+          destPath,
+        },
+        null,
+      ),
       this.id,
       "download",
     );
@@ -231,9 +279,13 @@ export async function discoverTargets(
   for (const plugin of deps.plugins()) {
     let result: ActionResult | null;
     try {
-      result = await deps.dispatch(plugin, {
-        action: "backup.target.describe",
-      });
+      result = await dispatchWithin(
+        deps,
+        plugin,
+        { action: "backup.target.describe" },
+        plugin,
+        CONTROL_DEADLINE_MS,
+      );
     } catch (err) {
       logWarn(
         "backup",

package/src/core/config/index.ts

@@ -478,6 +478,20 @@ const configSchema = z.object({
   planAlertThreshold: z.number().int().min(1).max(100).default(80),
   /** Chat that receives plan warnings. Defaults to `adminUserId`. */
   planAlertChatId: z.string().optional(),
+  /**
+   * Operator alerts (core/frontend-runtime/alerts.ts): faults that need a
+   * human — a full disk, a crash, an error spike, a dead frontend — sent
+   * to the admin chat, once per fault per `cooldownMinutes`, with a
+   * recovery notice when it clears. `enabled: false` keeps them in the
+   * log and in `talon status` only.
+   */
+  alerts: z
+    .object({
+      enabled: z.boolean().default(true),
+      cooldownMinutes: z.number().int().min(0).max(1440).default(30),
+    })
+    .strict()
+    .optional(),
   /** Background memory-consolidation (dream) runs. Mirrors `pulse`/`heartbeat`. */
   dream: z.boolean().default(true),
   /**

package/src/core/daemon/crash-marker.ts

@@ -0,0 +1,141 @@
+/**
+ * Crash marker — how a daemon that died tells the next one.
+ *
+ * A dying process can't reliably message anyone: the uncaught-exception
+ * path runs `process.exit(1)` within the same tick, and an async send to
+ * Telegram never leaves. So the crash path writes one small file,
+ * synchronously, and the next boot — once the admin notifier and the
+ * frontends are up — turns it into a `daemon.crash` alert and deletes it.
+ *
+ * Three writers: the uncaught-exception handler (./crash.ts), the fatal
+ * startup path (app.ts), and the handoff watcher (./handoff.ts) when a
+ * `/restart` successor never served. Last write wins; the file is tiny on
+ * purpose and never grows. This module imports nothing from core so the
+ * handoff watcher's bare runtime can carry it.
+ */
+
+import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { dirs } from "../../util/paths.js";
+
+export type CrashKind = "uncaught" | "startup" | "handoff";
+
+export type CrashMarker = {
+  kind: CrashKind;
+  /** ISO time of the crash. */
+  at: string;
+  message: string;
+  /** The first few stack lines, frames only. */
+  stack: string[];
+  pid: number;
+};
+
+const STACK_LINES = 5;
+const MAX_MESSAGE = 500;
+
+function crashMarkerPath(): string {
+  return resolve(dirs.data, "last-crash.json");
+}
+
+/** Error → message plus its top frames, bounded. */
+function describe(err: unknown): { message: string; stack: string[] } {
+  if (!(err instanceof Error)) {
+    return { message: String(err).slice(0, MAX_MESSAGE), stack: [] };
+  }
+  const stack = (err.stack ?? "")
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.startsWith("at "))
+    .slice(0, STACK_LINES);
+  const name = err.name && err.name !== "Error" ? `${err.name}: ` : "";
+  return { message: `${name}${err.message}`.slice(0, MAX_MESSAGE), stack };
+}
+
+/**
+ * Record a crash. Synchronous by design — the caller is about to exit.
+ * Throws on failure (a full disk); crash-path callers wrap it in
+ * `crashStep`.
+ */
+export function writeCrashMarker(
+  kind: CrashKind,
+  err: unknown,
+  opts: { path?: string; keepExisting?: boolean } = {},
+): void {
+  const marker: CrashMarker = {
+    kind,
+    at: new Date().toISOString(),
+    ...describe(err),
+    pid: process.pid,
+  };
+  // keepExisting: the handoff watcher must not clobber the more specific
+  // marker a successor that crashed on its own already left.
+  writeFileSync(opts.path ?? crashMarkerPath(), JSON.stringify(marker), {
+    mode: 0o600,
+    flag: opts.keepExisting ? "wx" : "w",
+  });
+}
+
+/**
+ * Read and delete the marker. Null when there is none; a corrupt marker
+ * is deleted and reported as null. Never throws.
+ */
+export function takeCrashMarker(
+  path: string = crashMarkerPath(),
+): CrashMarker | null {
+  let raw: string;
+  try {
+    raw = readFileSync(path, "utf-8");
+  } catch {
+    return null;
+  }
+  try {
+    unlinkSync(path);
+  } catch {
+    /* a marker we can't delete would re-announce next boot — acceptable */
+  }
+  try {
+    const parsed = JSON.parse(raw) as Partial<CrashMarker>;
+    if (typeof parsed.at !== "string" || typeof parsed.message !== "string")
+      return null;
+    return {
+      kind: parsed.kind ?? "uncaught",
+      at: parsed.at,
+      message: parsed.message,
+      stack: Array.isArray(parsed.stack) ? parsed.stack : [],
+      pid: typeof parsed.pid === "number" ? parsed.pid : 0,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Cross-boot throttle. Under systemd's `Restart=on-failure` a crash loop
+ * boots a fresh process every few seconds, and each would otherwise send
+ * its own alert — the in-process cooldown can't see across restarts.
+ * Returns how many crashes were held back since the last announcement
+ * (fold them into the message), or null when this one is inside the
+ * cooldown and should only be counted. Never throws.
+ */
+export function claimCrashAnnouncement(
+  cooldownMs: number,
+  path: string = resolve(dirs.data, "crash-announced.json"),
+  now: number = Date.now(),
+): number | null {
+  let prior = { at: 0, held: 0 };
+  try {
+    prior = { ...prior, ...JSON.parse(readFileSync(path, "utf-8")) };
+  } catch {
+    /* first crash, or an unreadable ledger — announce */
+  }
+  const inCooldown = now - prior.at < cooldownMs;
+  const next = inCooldown
+    ? { at: prior.at, held: prior.held + 1 }
+    : { at: now, held: 0 };
+  try {
+    writeFileSync(path, JSON.stringify(next), { mode: 0o600 });
+  } catch {
+    /* worst case the next crash announces too */
+  }
+  return inCooldown ? null : prior.held;
+}

package/src/core/daemon/crash.ts

@@ -19,6 +19,8 @@
  */
 
 import { logError, logWarn } from "../../util/log.js";
+import { writeCrashMarker } from "./crash-marker.js";
+import { noteUnhandledRejection } from "./health-alerts.js";
 import { removePidRecordIfOwnedBy } from "./pidfile.js";
 import { spawnSuccessor } from "./respawn.js";
 
@@ -65,7 +67,9 @@ export function crashCleanup(hooks: CrashHooks): void {
 
 /**
  * `process.on("uncaughtException")` body. Cleanup happens before the
- * crash is reported, never after.
+ * crash is reported, never after. The crash marker sits between the two:
+ * it is how the operator hears about this crash (the next boot announces
+ * it), but it is not worth a pidfile or a successor.
  */
 export function handleUncaughtException(err: Error, hooks: CrashHooks): void {
   // EPIPE errors from network sockets (e.g. Telegram MTProto) are transient —
@@ -77,6 +81,7 @@ export function handleUncaughtException(err: Error, hooks: CrashHooks): void {
     return;
   }
   crashCleanup(hooks);
+  crashStep("crash marker", () => writeCrashMarker("uncaught", err));
   crashStep("crash report", () => logError("bot", "Uncaught exception", err));
   process.exit(1);
 }
@@ -86,7 +91,8 @@ export function handleUncaughtException(err: Error, hooks: CrashHooks): void {
  * the stack: a bare "Unhandled rejection: ENOSPC: no space left on device,
  * write" says nothing about which code path forgot its `.catch()`. Async fs
  * errors carry `path`/`syscall` rather than useful frames, so those ride
- * along in the message too.
+ * along in the message too. Repeats raise `daemon.unhandled`
+ * (./health-alerts.ts).
  */
 export function handleUnhandledRejection(reason: unknown): void {
   crashStep("rejection report", () => {
@@ -102,4 +108,5 @@ export function handleUnhandledRejection(reason: unknown): void {
       reason,
     );
   });
+  crashStep("rejection alarm", () => noteUnhandledRejection(reason));
 }

package/src/core/daemon/handoff.ts

@@ -27,6 +27,7 @@
 import { dirname, resolve } from "node:path";
 import { log, logError, logWarn } from "../../util/log.js";
 import { startDaemon, type StartOutcome } from "./control.js";
+import { writeCrashMarker } from "./crash-marker.js";
 import { findRunningInstance, type RunningInstance } from "./discovery.js";
 import { isProcessAlive } from "./pidfile.js";
 
@@ -130,6 +131,19 @@ function toOutcome(started: StartOutcome, why: string): HandoffOutcome {
   return { ok: false, reason: `${why}; restart ${describeStart(started)}` };
 }
 
+/**
+ * Leave a crash marker so whichever daemon comes up next tells the
+ * operator the restart failed. A successor that crashed on its own left
+ * a more specific marker already; that one is kept.
+ */
+function markHandoffFailure(why: string): void {
+  try {
+    writeCrashMarker("handoff", why, { keepExisting: true });
+  } catch {
+    /* EEXIST (the successor's own marker) or a full disk — nothing to add */
+  }
+}
+
 /**
  * Verify the handoff, and repair it if it failed. Never throws: this
  * process exists only to make the outcome known.
@@ -153,6 +167,7 @@ export async function watchHandoff(
     "shutdown",
     `Handoff failed — ${why}; starting Talon the way \`talon start\` does`,
   );
+  markHandoffFailure(why);
   const start = opts.start ?? startDaemon;
   const started = await start({
     pkgRoot: opts.pkgRoot,