spinuptui 1.3.0-alpha.3 → 1.3.0-alpha.5

package/src/lib/schedulerTick.ts

@@ -34,9 +34,9 @@ import {
   type TickRecord,
   type TickRun,
 } from "./schedulerJobs.ts"
-import { jobEnv, notify, runJob, runProcess, selfCommand } from "./schedulerRun.ts"
+import { BLOCKED_EXIT, jobEnv, notify, runJob, runProcess, selfCommand } from "./schedulerRun.ts"
 import { commitAndPush, decideClaim, gitPull, readClaim, stationName } from "./schedulerGit.ts"
-import { pagerProblem, sendHeartbeat, tickHeartbeat } from "./heartbeat.ts"
+import { pagerProblem, sendHeartbeat, stillRunningHeartbeat, tickHeartbeat } from "./heartbeat.ts"
 
 const LOG_MAX_BYTES = 1_000_000
 const MODULE_JOB_TIMEOUT_MS = 10 * 60_000
@@ -48,7 +48,7 @@ export type TickResult = ({ ok: true } & TickRecord) | { ok: true; skipped: "loc
 export interface ModuleJobState {
   last_run?: string
   last_scheduled?: string
-  last_status?: "ok" | "failed"
+  last_status?: "ok" | "failed" | "blocked"
   failing_since?: string
   last_output?: string
 }
@@ -56,7 +56,7 @@ export interface ModuleJobState {
 export interface ModuleJobDeps {
   stateDir: string
   now: Date
-  run(decl: ScheduledJobDecl): Promise<{ ok: boolean; output: string }>
+  run(decl: ScheduledJobDecl): Promise<{ ok: boolean; output: string; blocked?: boolean }>
   page(text: string): Promise<string>
 }
 
@@ -91,13 +91,14 @@ export async function runModuleJobs(decls: ScheduledJobDecl[], deps: ModuleJobDe
     // aligned to ticks instead of drifting by each run's duration.
     const started = deps.now
     const r = await deps.run(decl)
-    const status = r.ok ? "ok" : "failed"
+    const status = r.ok ? "ok" : r.blocked ? "blocked" : "failed"
     const tail = r.output.trim().split("\n").slice(-10).join("\n").slice(-600)
     let notified: string | undefined
-    if (status === "failed" && state.last_status !== "failed") {
-      notified = await deps.page(`⚠️ ${decl.id} failed (spinuptui ${decl.command.join(" ")})\n\n${tail || "(no output)"}`)
-      state.failing_since = fmt(started)
-    } else if (status === "ok" && state.last_status === "failed") {
+    const wasOk = !state.last_status || state.last_status === "ok"
+    if (status !== "ok" && status !== state.last_status) {
+      notified = await deps.page(`${status === "blocked" ? "🚧" : "⚠️"} ${decl.id} ${status} (spinuptui ${decl.command.join(" ")})\n\n${tail || "(no output)"}`)
+      if (wasOk) state.failing_since = fmt(started)
+    } else if (status === "ok" && !wasOk) {
       notified = await deps.page(`✅ ${decl.id} is working again (failing since ${state.failing_since ?? "?"})`)
       delete state.failing_since
     }
@@ -250,7 +251,12 @@ export async function tick(opts: TickOptions): Promise<TickResult> {
   const dir = schedulerStateDir()
   mkdirSync(dir, { recursive: true })
   const lock = tryLock(join(dir, "tick.lock"))
-  if (!lock) return { ok: true, skipped: "locked", message: "A previous tick is still running." }
+  if (!lock) {
+    // The heartbeat can't wait for a long job to finish: Kuma would call a busy
+    // station dead. The holder is alive, so this tick says so.
+    if (opts.via === "dispatcher") await recordStillRunning()
+    return { ok: true, skipped: "locked", message: "A previous tick is still running." }
+  }
   const startedAt = Date.now()
   const now = opts.now ?? new Date()
   const ran: TickRun[] = []
@@ -265,7 +271,7 @@ export async function tick(opts: TickOptions): Promise<TickResult> {
           now,
           run: async (decl) => {
             const p = await runProcess([...selfCommand(), ...decl.command], { cwd: dir, env: jobEnv(), timeoutMs: MODULE_JOB_TIMEOUT_MS })
-            return { ok: p.code === 0, output: p.output }
+            return { ok: p.code === 0, output: p.output, blocked: p.code === BLOCKED_EXIT }
           },
           page: (text) => notify(text),
         })
@@ -300,8 +306,7 @@ export async function tick(opts: TickOptions): Promise<TickResult> {
         record.heartbeat = { ok: r.ok, status: r.status, ...(r.error ? { error: r.error } : {}) }
         if (!r.ok) record.errors.push(`heartbeat: Kuma didn't take the ping (${r.error})`)
       }
-      writeFileSync(`${lastTickPath()}.tmp`, JSON.stringify(record, null, 2) + "\n")
-      renameSync(`${lastTickPath()}.tmp`, lastTickPath())
+      writeLastTick(record)
     }
     if (ran.length || errors.length) appendTickLog(record)
     return { ok: true, ...record }
@@ -310,6 +315,23 @@ export async function tick(opts: TickOptions): Promise<TickResult> {
   }
 }
 
+function writeLastTick(record: TickRecord): void {
+  writeFileSync(`${lastTickPath()}.tmp`, JSON.stringify(record, null, 2) + "\n")
+  renameSync(`${lastTickPath()}.tmp`, lastTickPath())
+}
+
+async function recordStillRunning(): Promise<void> {
+  const record: TickRecord = { at: new Date().toISOString(), via: "dispatcher", durationMs: 0, ran: [], errors: [], stillRunning: true }
+  const hb = loadConfig().scheduler?.heartbeat
+  if (hb) {
+    const t = stillRunningHeartbeat(pagerProblem())
+    const r = await sendHeartbeat(hb.url, t.status, t.msg, 0)
+    record.heartbeat = { ok: r.ok, status: r.status, ...(r.error ? { error: r.error } : {}) }
+    if (!r.ok) record.errors.push(`heartbeat: Kuma didn't take the ping (${r.error})`)
+  }
+  writeLastTick(record)
+}
+
 function appendTickLog(record: TickRecord): void {
   try {
     const path = tickLogPath()

package/src/lib/sentinelAttach.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, test } from "bun:test"
+import { applyAttach, planAttach } from "./sentinelAttach.ts"
+import type { KumaMonitor } from "./uptimeKuma.ts"
+
+const m = (id: number, name: string, over: Partial<KumaMonitor> = {}): KumaMonitor => ({ id, name, type: "http", ...over })
+const sites = ["a.example.com", "b.example.com", "c.example.com"]
+
+describe("planAttach", () => {
+  test("matches by the naming convention", () => {
+    const p = planAttach([m(1, "a.example.com"), m(2, "a.example.com load", { type: "push" }), m(3, "a.example.com php-fatal"), m(4, "a.example.com front page"), m(5, "a.example.com cache-bypass"), m(6, "a.example.com redis")], sites, {})
+    expect(p.exact.map((x) => x.field).sort()).toEqual(["bypassId", "fatalId", "fingerprintId", "healthId", "pushId", "redisId"])
+    expect(p.unmonitored).toEqual(["b.example.com", "c.example.com"])
+  })
+
+  test("a site with extra domains counts once", () => {
+    const p = planAttach([], ["a.example.com", "www.a.example.com"], {}, ["a.example.com"])
+    expect(p.unmonitored).toEqual(["a.example.com"])
+  })
+
+  test("a hand-made monitor on a fleet host is only likely", () => {
+    const p = planAttach([m(1, "Client homepage", { url: "https://www.b.example.com/" })], sites, {})
+    expect(p.exact).toEqual([])
+    expect(p.likely).toMatchObject([{ domain: "b.example.com", field: "healthId", monitorId: 1 }])
+  })
+
+  test("ignores strangers, and what's already attached", () => {
+    const p = planAttach([m(1, "a.example.com"), m(9, "My router", { url: "https://192.0.2.1" })], sites, { "a.example.com": { healthId: 1 } })
+    expect(p.already).toBe(1)
+    expect(p.exact).toEqual([])
+    expect(p.strangers).toEqual(["My router"])
+  })
+})
+
+describe("applyAttach", () => {
+  test("keeps the token Kuma already holds, and never overwrites", () => {
+    const monitors = [m(2, "a.example.com load", { type: "push", pushToken: "live-token" }), m(1, "a.example.com")]
+    const r = applyAttach({ "a.example.com": { healthId: 7 } }, [{ domain: "a.example.com", field: "pushId", monitorId: 2, name: "" }, { domain: "a.example.com", field: "healthId", monitorId: 1, name: "" }], monitors)
+    expect(r["a.example.com"]).toEqual({ healthId: 7, pushId: 2, pushToken: "live-token" })
+  })
+
+  test("rebuilds a fingerprint's keyword", () => {
+    const r = applyAttach({}, [{ domain: "a.example.com", field: "fingerprintId", monitorId: 4, name: "" }], [m(4, "x", { keyword: "home", interval: 600 })])
+    expect(r["a.example.com"]?.fingerprint).toMatchObject({ keyword: "home", kind: "attached", interval: 600 })
+  })
+})

package/src/lib/sentinelAttach.ts

@@ -0,0 +1,98 @@
+// `sentinel attach`: adopts an existing Kuma's monitors, because a fresh install
+// connected to a Kuma full of them still sees none (`incidents` reads the local
+// registry). The registry is rebuilt from what Kuma already knows.
+//
+// Read-only toward Kuma and the servers: it never creates, renames or edits a
+// monitor, and never generates a push token, because the server crons are already
+// pushing to the existing ones. Re-running only fills gaps.
+
+import type { KumaMonitorRef } from "../config.ts"
+import type { KumaMonitor } from "./uptimeKuma.ts"
+
+// SpinupTUI's own naming convention, longest suffix first.
+const SUFFIXES: [string, keyof KumaMonitorRef, (keyof KumaMonitorRef)?][] = [
+  [" php-fatal", "fatalId", "fatalToken"],
+  [" cache-bypass", "bypassId"],
+  [" front page", "fingerprintId"],
+  [" redis", "redisId", "redisToken"],
+  [" load", "pushId", "pushToken"],
+]
+
+export interface AttachMatch {
+  domain: string
+  field: keyof KumaMonitorRef
+  monitorId: number
+  name: string
+}
+
+export interface AttachPlan {
+  exact: AttachMatch[]
+  likely: AttachMatch[]
+  unmonitored: string[]
+  strangers: string[]
+  already: number
+}
+
+function nameMatch(m: KumaMonitor): { domain: string; field: keyof KumaMonitorRef; token?: keyof KumaMonitorRef } {
+  for (const [suffix, field, token] of SUFFIXES) if (m.name.endsWith(suffix)) return { domain: m.name.slice(0, -suffix.length), field, token }
+  return { domain: m.name, field: "healthId" }
+}
+
+function urlHost(url: string | undefined): string | null {
+  try {
+    return url ? new URL(url).hostname.toLowerCase() : null
+  } catch {
+    return null
+  }
+}
+
+// `domains` is every name a monitor might use; `primary` is one per site, so a
+// site with extra domains counts once when listing what has no monitor.
+export function planAttach(monitors: KumaMonitor[], domains: string[], registry: Record<string, KumaMonitorRef>, primary: string[] = domains): AttachPlan {
+  const sites = new Set(domains)
+  const has = (domain: string, field: keyof KumaMonitorRef) => typeof registry[domain]?.[field] === "number"
+  const taken = new Set(Object.values(registry).flatMap((r) => Object.entries(r).filter(([k, v]) => k.endsWith("Id") && typeof v === "number").map(([, v]) => v as number)))
+  const plan: AttachPlan = { exact: [], likely: [], unmonitored: [], strangers: [], already: 0 }
+  const covered = new Set(Object.keys(registry).filter((d) => registry[d]?.healthId))
+  const leftover: KumaMonitor[] = []
+  for (const m of monitors) {
+    if (taken.has(m.id)) {
+      plan.already++
+      continue
+    }
+    const { domain, field } = nameMatch(m)
+    if (sites.has(domain) && !has(domain, field)) {
+      plan.exact.push({ domain, field, monitorId: m.id, name: m.name })
+      if (field === "healthId") covered.add(domain)
+    } else leftover.push(m)
+  }
+  for (const m of leftover) {
+    const host = urlHost(m.url)
+    const domain = host && (sites.has(host) ? host : sites.has(host.replace(/^www\./, "")) ? host.replace(/^www\./, "") : null)
+    if (domain && m.type === "http" && !covered.has(domain)) {
+      plan.likely.push({ domain, field: "healthId", monitorId: m.id, name: m.name })
+      covered.add(domain)
+    } else plan.strangers.push(m.name)
+  }
+  plan.unmonitored = primary.filter((d) => !covered.has(d) && !registry[d]).sort()
+  return plan
+}
+
+// The registry after attaching: existing entries are never overwritten, and each
+// push monitor keeps the token Kuma already holds.
+export function applyAttach(registry: Record<string, KumaMonitorRef>, matches: AttachMatch[], monitors: KumaMonitor[]): Record<string, KumaMonitorRef> {
+  const byId = new Map(monitors.map((m) => [m.id, m]))
+  const out: Record<string, KumaMonitorRef> = structuredClone(registry)
+  for (const a of matches) {
+    const ref = (out[a.domain] ??= {}) as Record<string, unknown>
+    if (typeof ref[a.field] === "number") continue
+    ref[a.field] = a.monitorId
+    const m = byId.get(a.monitorId)
+    const token = SUFFIXES.find(([, field]) => field === a.field)?.[2]
+    if (token && m?.pushToken) ref[token] = m.pushToken
+    if (a.field === "fingerprintId" && typeof m?.keyword === "string") {
+      ref.fingerprint = { keyword: m.keyword, kind: "attached", detail: "", interval: Number(m.interval) || 300, derivedAt: new Date().toISOString() }
+    }
+  }
+  return out
+}

package/src/lib/sentinelDigest.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, test } from "bun:test"
+import { aptTargets, composeDigest, type DigestData } from "./sentinelDigest.ts"
+import type { ExpectResult } from "./sentinelExpect.ts"
+import type { Flags, Incident } from "./sentinelState.ts"
+
+const when = new Date("2026-10-30T12:00:00")
+const flag = (id: string, text = "LOW"): ExpectResult => ({ id, label: id, state: "flag", text })
+const data = (over: Partial<DigestData> = {}): DigestData => ({ when, health: [{ name: "Station", ok: true, detail: "ticking" }], ticks: [], incidents: [], results: [], aptTotal: 0, apt: [], notes: [], ...over })
+
+describe("the header reacts to what's new", () => {
+  test("all clear is green and silent", () => {
+    const d = composeDigest(data(), {})
+    expect(d.text.startsWith("✅ Fleet OK")).toBe(true)
+    expect(d.silent).toBe(true)
+  })
+
+  test("a new flag makes a sound, and records the day", () => {
+    const d = composeDigest(data({ results: [flag("a")] }), {})
+    expect(d.text.startsWith("⚠️ 1 new")).toBe(true)
+    expect(d.silent).toBe(false)
+    expect(d.text).toContain("New:")
+    expect(d.flags.a!.firstSeen).toBe("2026-10-30")
+  })
+
+  test("yesterday's flag is ongoing, with its age, and stays quiet", () => {
+    const flags: Flags = { a: { firstSeen: "2026-10-18", text: "a: LOW" } }
+    const d = composeDigest(data({ results: [flag("a")] }), flags)
+    expect(d.text.startsWith("✅ Fleet OK")).toBe(true)
+    expect(d.silent).toBe(true)
+    expect(d.text).toContain("Ongoing:")
+    expect(d.text).toContain("(12 days)")
+  })
+
+  test("an acknowledged flag becomes a count, and is never hidden", () => {
+    const d = composeDigest(data({ results: [flag("a")] }), { a: { firstSeen: "2026-10-18", text: "x", acked: true } })
+    expect(d.text).toContain("1 acknowledged flag still set")
+    expect(d.text).not.toContain("Ongoing:")
+  })
+
+  test("a flag that clears starts over if it comes back", () => {
+    const d = composeDigest(data({ results: [{ id: "a", label: "a", state: "ok", text: "fine" }] }), { a: { firstSeen: "2026-10-18", text: "x" } })
+    expect(d.flags).toEqual({})
+  })
+
+  test("a check that couldn't run keeps its flag's age", () => {
+    const d = composeDigest(data({ results: [{ id: "a", label: "a", state: "unverified", text: "no count" }] }), { a: { firstSeen: "2026-10-18", text: "x" } })
+    expect(d.flags.a!.firstSeen).toBe("2026-10-18")
+  })
+
+  test("a failing health check makes a sound", () => {
+    const d = composeDigest(data({ health: [{ name: "Agent", ok: false, detail: "login lapsed" }] }), {})
+    expect(d.text.startsWith("⚠️ Needs attention")).toBe(true)
+    expect(d.silent).toBe(false)
+    expect(d.text).toContain("Agent: ⚠️ login lapsed")
+  })
+})
+
+test("a stuck incident is named", () => {
+  const i: Incident = { id: "old", key: "k", server: null, state: "waiting_agent", createdAt: new Date(when.getTime() - 2 * 86_400_000).toISOString(), events: [{ domain: "a.example.com", kind: "health", time: "t", message: "", monitorId: 1 }], down: [], lastDownAt: "", attempts: 0 }
+  const d = composeDigest(data({ incidents: [i] }), {})
+  expect(d.silent).toBe(false)
+  expect(d.text).toContain("old: waiting_agent")
+})
+
+test("activity counts, and recurrences by site", () => {
+  const base: Incident = { id: "x", key: "k", server: null, state: "reported", createdAt: when.toISOString(), events: [{ domain: "a.example.com", kind: "health", time: "t", message: "", monitorId: 1 }], down: [], lastDownAt: "", attempts: 1 }
+  const d = composeDigest(data({ incidents: [base, { ...base, id: "y", state: "recurrence" }, { ...base, id: "z", state: "recurrence" }], ticks: [{ at: "", ok: true, events: 0, incidents: 0, retries: 1 }, { at: "", ok: false, events: 0, incidents: 0, retries: 2 }] }), {})
+  expect(d.text).toContain("2 checks · 1 incident · 1 report · 2 recurrences · 1 blind · 1 cold start")
+  expect(d.text).toContain("a.example.com ×2")
+})
+
+test("server updates collapse to a count, naming only the exceptions", () => {
+  const apt: ExpectResult[] = [{ id: "apt-a", label: "a", state: "ok", text: "" }, { id: "apt-b", label: "b", state: "unverified", text: "" }, flag("apt-c", "STALE")]
+  const d = composeDigest(data({ apt, aptTotal: 3 }), {})
+  expect(d.text).toContain("Server updates: 1 of 3 normal, unverified: b")
+  expect(d.text).toContain("apt-c: STALE")
+})
+
+test("the update check tries one site per server, the one that worked last", () => {
+  const fleet = { domains: { "a.example.com": { server: "w1", site: "a.example.com" }, "b.example.com": { server: "w1", site: "b.example.com" }, "www.b.example.com": { server: "w1", site: "b.example.com" }, "w2.example.com": { server: "w2", site: "w2.example.com", vanity: true }, "c.example.com": { server: "w2", site: "c.example.com" } } }
+  expect(aptTargets(fleet, { w1: "b.example.com" }, new Set(["c.example.com"]))).toEqual([{ server: "w1", domain: "b.example.com" }, { server: "w2", domain: "c.example.com" }])
+})

package/src/lib/sentinelDigest.ts

@@ -0,0 +1,261 @@
+// The daily digest: the place a slow-moving problem gets said out loud. It always
+// sends, and it's silent unless something is new, because a header that's amber
+// every day stops being read. A flag is under New on the day it trips, then under
+// Ongoing with its age (docs/2026-10-30_sentinel-spec.md, "The digest").
+
+import type { AppConfig } from "../config.ts"
+import type { SpinupWPClientLike } from "../api/client.ts"
+import { schedulerModule } from "../modules/scheduler.ts"
+import { boardLogin } from "./beaconBoard.ts"
+import { listenerHealth } from "./beaconListenInstall.ts"
+import { agentCommand, AUTH_FAIL_RE, jobEnv, runProcess } from "./schedulerRun.ts"
+import { APT_COMMAND, appendTrends, loadExpectations, readApt, runExpectation, validateExpectation, type ExpectDeps, type ExpectResult } from "./sentinelExpect.ts"
+import { isWaiting, loadFleet, loadIncidents, loadPause, readTickLog, saveFleet, saveFlags, readJson, writeJson, sentinelDir, type Fleet, type Flags, type Incident, type SentinelSettings, type TickLine } from "./sentinelState.ts"
+import { refreshFleetFrom } from "./sentinelTick.ts"
+import { execSshCommand } from "./sshExec.ts"
+import { hasIncidentMonitors } from "./cliIncidents.ts"
+import { withKuma } from "./uptimeKuma.ts"
+import { join } from "node:path"
+
+const CHECK_TIMEOUT_MS = 30_000
+const CONCURRENCY = 4
+const AGENT_PROBE_MS = 60_000
+const DAY_MS = 24 * 3_600_000
+const STUCK_MS = DAY_MS
+
+export interface HealthCheck {
+  name: string
+  ok: boolean
+  detail: string
+}
+
+export interface DigestData {
+  when: Date
+  health: HealthCheck[]
+  ticks: TickLine[]
+  incidents: Incident[]
+  results: ExpectResult[]
+  aptTotal: number
+  apt: ExpectResult[]
+  notes: string[]
+  paused?: string
+}
+
+export interface Digest {
+  text: string
+  silent: boolean
+  flags: Flags
+}
+
+const localDay = (d: Date) => d.toLocaleDateString("en-CA")
+const dayCount = (from: string, to: string) => Math.round((Date.parse(to) - Date.parse(from)) / DAY_MS)
+const plural = (n: number, s: string) => `${n} ${s}${n === 1 ? "" : "s"}`
+
+export function composeDigest(d: DigestData, flags: Flags): Digest {
+  const today = localDay(d.when)
+  const since = d.when.getTime() - DAY_MS
+  const checks = [...d.results, ...d.apt]
+  const next: Flags = {}
+  const fresh: string[] = []
+  const ongoing: string[] = []
+  let acked = 0
+  for (const r of checks) {
+    if (r.state !== "flag") continue
+    const line = `${r.label}: ${r.text}`
+    const prior = flags[r.id]
+    next[r.id] = { firstSeen: prior?.firstSeen ?? today, text: line, ...(prior?.acked ? { acked: true } : {}) }
+    if (!prior || prior.firstSeen === today) fresh.push(line)
+    else if (prior.acked) acked++
+    else ongoing.push(`${line} (${plural(dayCount(prior.firstSeen, today), "day")})`)
+  }
+  // A check that couldn't run says nothing either way, so its flag keeps its age.
+  for (const r of checks) if (r.state === "unverified" && flags[r.id]) next[r.id] = flags[r.id]!
+
+  const recent = d.incidents.filter((i) => Date.parse(i.createdAt) >= since)
+  const stuck = d.incidents.filter((i) => isWaiting(i) && d.when.getTime() - Date.parse(i.createdAt) > STUCK_MS)
+  const failing = d.health.filter((h) => !h.ok)
+  const reports = recent.filter((i) => i.state === "reported").length
+  const recurrences = recent.filter((i) => i.state === "recurrence")
+  const blind = d.ticks.filter((t) => !t.ok).length
+  const retried = d.ticks.filter((t) => t.ok && t.retries > 0).length
+
+  const date = d.when.toLocaleDateString("en-US", { weekday: "short", month: "short", day: "numeric" })
+  const attention = fresh.length > 0 || failing.length > 0 || stuck.length > 0
+  const header = !attention ? `✅ Fleet OK — ${date}` : fresh.length ? `⚠️ ${fresh.length} new — ${date}` : `⚠️ Needs attention — ${date}`
+
+  const lines = [header, ""]
+  if (d.paused) lines.push(`Paused until ${d.paused}.`, "")
+  lines.push(d.health.map((h) => `${h.name}: ${h.ok ? h.detail : `⚠️ ${h.detail}`}`).join("\n"))
+  lines.push(`Last 24h: ${plural(d.ticks.length, "check")} · ${plural(recent.length - recurrences.length, "incident")} · ${plural(reports, "report")} · ${plural(recurrences.length, "recurrence")} · ${blind} blind${retried ? ` · ${retried} cold start${retried === 1 ? "" : "s"}` : ""}`)
+  if (recurrences.length) {
+    const bySite = new Map<string, number>()
+    for (const i of recurrences) bySite.set(i.events[0]!.domain, (bySite.get(i.events[0]!.domain) ?? 0) + 1)
+    lines.push(`Recurring: ${[...bySite].map(([s, n]) => `${s} ×${n}`).join(", ")}`)
+  }
+  if (stuck.length) lines.push("", "Stuck:", ...stuck.map((i) => `  ${i.id}: ${i.state}${i.why ? ` (${i.why})` : ""}`))
+  if (fresh.length) lines.push("", "New:", ...fresh.map((l) => `  ${l}`))
+  if (ongoing.length) lines.push("", "Ongoing:", ...ongoing.map((l) => `  ${l}`))
+  if (acked) lines.push("", `${plural(acked, "acknowledged flag")} still set.`)
+  const quiet = d.results.filter((r) => r.state !== "flag")
+  if (quiet.length) lines.push("", "Checks:", ...quiet.map((r) => `  ${r.label}: ${r.state === "unverified" ? `unverified (${r.text})` : r.text}`))
+  if (d.aptTotal) {
+    const unverified = d.apt.filter((r) => r.state === "unverified")
+    const normal = d.apt.filter((r) => r.state === "ok").length
+    lines.push("", `Server updates: ${normal} of ${d.aptTotal} normal${unverified.length ? `, unverified: ${unverified.map((r) => r.label).join(", ")}` : ""}`)
+  }
+  if (d.notes.length) lines.push("", ...d.notes)
+  return { text: lines.join("\n"), silent: !attention, flags: next }
+}
+
+// ---- gathering --------------------------------------------------------------------------------
+
+async function bounded<T>(fn: () => Promise<T>, fallback: T, ms = CHECK_TIMEOUT_MS): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined
+  try {
+    return await Promise.race([fn(), new Promise<T>((r) => (timer = setTimeout(() => r(fallback), ms)))])
+  } catch {
+    return fallback
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+async function mapLimit<T, R>(items: T[], limit: number, fn: (x: T) => Promise<R>): Promise<R[]> {
+  const out: R[] = new Array(items.length)
+  let next = 0
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, async () => {
+      while (next < items.length) {
+        const i = next++
+        out[i] = await fn(items[i]!)
+      }
+    }),
+  )
+  return out
+}
+
+export function expectDeps(cfg: AppConfig, client: SpinupWPClientLike): ExpectDeps {
+  return {
+    exec: async (domain, command, server) => {
+      const r = await execSshCommand(domain, command, client, cfg, { server: server ?? null })
+      return r.ok ? r.stdout : null
+    },
+    status: async (url) => {
+      try {
+        return (await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(20_000) })).status
+      } catch {
+        return null
+      }
+    },
+  }
+}
+
+// One attempt per server per day, on the site that worked last: a server with no
+// reachable site reads "unverified", and fail2ban never hears about it.
+const aptSitesPath = () => join(sentinelDir(), "apt-sites.json")
+
+export function aptTargets(fleet: Fleet, lastGood: Record<string, string>, monitored: Set<string>): { server: string; domain: string }[] {
+  const byServer = new Map<string, string[]>()
+  for (const [domain, e] of Object.entries(fleet.domains)) {
+    if (domain !== e.site) continue
+    byServer.set(e.server, [...(byServer.get(e.server) ?? []), domain])
+  }
+  return [...byServer].sort().map(([server, domains]) => {
+    const rank = (d: string) => (d === lastGood[server] ? 0 : monitored.has(d) ? 1 : fleet.domains[d]?.vanity ? 2 : 3)
+    return { server, domain: [...domains].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))[0]! }
+  })
+}
+
+async function checkApt(fleet: Fleet, cfg: AppConfig, deps: ExpectDeps, dryRun: boolean): Promise<ExpectResult[]> {
+  const lastGood = readJson<Record<string, string>>(aptSitesPath(), {})
+  const monitored = new Set(Object.entries(cfg.kumaMonitors).filter(([, r]) => hasIncidentMonitors(r)).map(([d]) => d))
+  const results = await mapLimit(aptTargets(fleet, lastGood, monitored), CONCURRENCY, async ({ server, domain }) => {
+    const out = await bounded(() => deps.exec(domain, APT_COMMAND), null)
+    const read = readApt(out)
+    if (read) lastGood[server] = domain
+    const base = { id: `apt-${server}`, label: server }
+    if (!read) return { ...base, state: "unverified" as const, text: "probe failed" }
+    return read.flag ? { ...base, state: "flag" as const, text: read.flag } : { ...base, state: "ok" as const, text: "updating" }
+  })
+  if (!dryRun) writeJson(aptSitesPath(), lastGood)
+  return results
+}
+
+async function probeAgent(s: SentinelSettings): Promise<HealthCheck> {
+  const env = jobEnv()
+  const cmd = agentCommand({ agent: s.agent!, prompt: "Reply with exactly: AUTH_OK", workdir: s.workspace }, env)
+  if (typeof cmd === "string") return { name: "Agent", ok: false, detail: cmd }
+  const r = await runProcess(cmd, { cwd: sentinelDir(), env, timeoutMs: AGENT_PROBE_MS }).catch((e: Error) => ({ code: 1, output: e.message, timedOut: false }))
+  if (r.output.includes("AUTH_OK")) return { name: "Agent", ok: true, detail: "OK" }
+  return { name: "Agent", ok: false, detail: AUTH_FAIL_RE.test(r.output) ? `the ${s.agent} login has lapsed` : `unverified (probe ${r.timedOut ? "timed out" : `exited ${r.code}`})` }
+}
+
+export async function gatherDigest(s: SentinelSettings, cfg: AppConfig, client: SpinupWPClientLike, opts: { dryRun?: boolean; now?: Date } = {}): Promise<DigestData> {
+  const when = opts.now ?? new Date()
+  const notes: string[] = []
+  let fleet = loadFleet()
+  try {
+    fleet = await refreshFleetFrom(client, when)
+    if (!opts.dryRun) saveFleet(fleet)
+  } catch (err) {
+    notes.push(`Couldn't refresh the fleet list: ${(err as Error).message}`)
+  }
+
+  const health: HealthCheck[] = []
+  const sched = await schedulerModule.status()
+  const bad = sched.checks.find((c) => !c.ok && (c.name === "last tick" || c.name === "dispatcher" || c.name === "heartbeat"))
+  health.push({ name: "Station", ok: !bad, detail: bad ? `${bad.name}: ${bad.detail}` : "ticking" })
+  if (s.agent) health.push(await probeAgent(s))
+  if (s.reportTo) {
+    const b = await bounded(() => boardLogin(), { ok: false, detail: "timed out" })
+    health.push({ name: "Board", ok: b.ok, detail: b.ok ? "OK" : b.detail })
+  }
+  const listener = listenerHealth()
+  if (listener) health.push({ name: "Listener", ok: listener.ok, detail: listener.ok ? "OK" : listener.detail })
+  if (cfg.uptimeKuma) {
+    const ids = await bounded(async () => new Set((await withKuma(cfg.uptimeKuma!, (k) => k.getMonitors())).result.map((m) => m.id)), null)
+    if (!ids) health.push({ name: "Kuma", ok: false, detail: "couldn't list monitors" })
+    else {
+      const gone = Object.entries(cfg.kumaMonitors)
+        .filter(([, r]) => Object.entries(r).some(([k, v]) => k.endsWith("Id") && typeof v === "number" && !ids.has(v)))
+        .map(([d]) => d)
+      health.push({ name: "Kuma", ok: gone.length === 0, detail: gone.length ? `monitors deleted for ${gone.join(", ")}` : "OK" })
+    }
+  }
+
+  const deps = expectDeps(cfg, client)
+  const today = when.getTime()
+  const active = loadExpectations().filter((e) => {
+    if (validateExpectation(e).length) {
+      notes.push(`Skipped invalid check ${e.id}: ${validateExpectation(e).join("; ")}`)
+      return false
+    }
+    if (e.expires && Date.parse(`${e.expires}T23:59:59`) < today) {
+      notes.push(`Check ${e.id} expired on ${e.expires} and no longer runs.`)
+      return false
+    }
+    return true
+  })
+  const results = await mapLimit(active, CONCURRENCY, (e) =>
+    bounded(() => runExpectation(e, deps), { id: e.id, label: e.label ?? e.id, state: "unverified" as const, text: "timed out" }, 90_000),
+  )
+  const apt = await checkApt(fleet, cfg, deps, !!opts.dryRun)
+  const pause = loadPause()
+  return {
+    when,
+    health,
+    ticks: readTickLog(when.getTime() - DAY_MS),
+    incidents: loadIncidents().filter((i) => !i.closedAt || Date.parse(i.createdAt) > when.getTime() - DAY_MS || isWaiting(i)),
+    results,
+    aptTotal: apt.length,
+    apt,
+    notes,
+    ...(pause && Date.parse(pause.until) > when.getTime() ? { paused: new Date(pause.until).toLocaleString() } : {}),
+  }
+}
+
+export function recordDigest(digest: Digest, data: DigestData): void {
+  saveFlags(digest.flags)
+  appendTrends(data.results, localDay(new Date(data.when.getTime() - DAY_MS)))
+}
+