spinuptui 1.3.0-alpha.4 → 1.3.0-alpha.5

package/src/lib/sentinelWorker.ts

@@ -0,0 +1,291 @@
+// The diagnosis worker: everything slow, outside the tick. A tick starts it when
+// work is queued, and it takes the oldest incident at a time until none are left,
+// then exits. It isn't a scheduled unit, and only one runs at a time.
+//
+// The worker, not the agent, saves the report and posts it. So a missing write
+// permission can't lose a report, and nothing the agent read can make it publish
+// (docs/2026-10-30_sentinel-spec.md, "Diagnosis").
+
+import { existsSync, mkdirSync, writeFileSync } from "node:fs"
+import { dirname, join, relative } from "node:path"
+import type { AppConfig } from "../config.ts"
+import { SpinupWPClient, type SpinupWPClientLike } from "../api/client.ts"
+import { moduleWorkDir } from "../modules/contract.ts"
+import { postToBoard } from "./beaconBoard.ts"
+import { resolveSiblings } from "./cliSiblings.ts"
+import { tryLock } from "./flock.ts"
+import { prepareWorkDir } from "./home.ts"
+import { commitAndPush } from "./schedulerGit.ts"
+import { gitRootOf } from "./schedulerJobs.ts"
+import { agentCommand, jobEnv, outcomeOf, runProcess, type ProcResult } from "./schedulerRun.ts"
+import { diagnoseButton, startedToday, type Page } from "./sentinelIncidents.ts"
+import { diagnosisPrompt, parseAgentReport } from "./sentinelPrompt.ts"
+import { loadIncident, loadIncidents, loadTickState, saveIncident, saveTickState, transcriptsDir, workerLockPath, type Incident, type SentinelSettings } from "./sentinelState.ts"
+import { MAX_ATTEMPTS, nextRetryAt, sendPage } from "./sentinelTick.ts"
+import { runSiteDoctor } from "./siteDoctor.ts"
+import { execSshCommand } from "./sshExec.ts"
+
+const SNAPSHOT = "uptime; free -m | head -3; df -h / | tail -1"
+
+export interface WorkerDeps {
+  triage(i: Incident): Promise<string>
+  agent(prompt: string, s: SentinelSettings): Promise<ProcResult | string>
+  page(p: Page): Promise<string>
+  post(report: string, s: SentinelSettings): Promise<{ ok: true; id: string; url: string } | { ok: false; message: string }>
+  commit(s: SentinelSettings, message: string): Promise<void>
+  now(): Date
+}
+
+// ---- triage ---------------------------------------------------------------------------
+
+// What a person would look at first. Deterministic and needing no agent, so an
+// install without one still gets a useful write-up. Each step can fail on its own.
+//
+// It never uses `siblings --probe`: one SSH attempt per co-tenant, on every
+// incident, is how a station gets itself banned by fail2ban.
+export async function triage(i: Incident, cfg: AppConfig, client: SpinupWPClientLike): Promise<string> {
+  const domains = [...new Set(i.events.map((e) => e.domain))]
+  const out: string[] = []
+  const events = i.events.map((e) => `- ${e.time} UTC: ${e.domain} ${e.kind}${e.message ? `: ${e.message}` : ""}`)
+  out.push(`### Events\n\n${events.join("\n")}`)
+
+  const first = domains[0]!
+  try {
+    const sib = await resolveSiblings(first, client, cfg, {})
+    if (sib.ok) {
+      const sites = sib.sites.map((s) => `- ${s.domain}${s.isVanity ? " (vanity host)" : ""}${s.monitored ? "" : " (not monitored)"}`)
+      out.push(`### Server\n\n${sib.server.name}: ${sib.server.size ?? "unknown size"}, Ubuntu ${sib.server.ubuntuVersion ?? "?"}\n\n${sites.join("\n")}`)
+    } else out.push(`### Server\n\nCouldn't look it up: ${sib.message}`)
+  } catch (err) {
+    out.push(`### Server\n\nCouldn't look it up: ${(err as Error).message}`)
+  }
+
+  const doctor: string[] = []
+  for (const d of domains.slice(0, 5)) {
+    try {
+      const r = await runSiteDoctor({ url: `https://${d}/` })
+      doctor.push(`- ${d}: ${r.verdict}. ${r.summary}`)
+    } catch (err) {
+      doctor.push(`- ${d}: the check failed: ${(err as Error).message}`)
+    }
+  }
+  out.push(`### HTTP\n\n${doctor.join("\n")}`)
+
+  try {
+    const r = await execSshCommand(first, SNAPSHOT, client, cfg)
+    if (r.ok) out.push(`### Load on the box\n\n\`\`\`\n${r.stdout.trim()}\n\`\`\``)
+  } catch {
+    // SSH isn't reachable: the rest stands without it.
+  }
+  return out.join("\n\n")
+}
+
+// ---- saving and posting -----------------------------------------------------------------
+
+const SAFE_SITE = /^[a-z0-9][a-z0-9.-]*\.[a-z0-9-]+$/i
+const safeSlug = (s: string) => s.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60)
+
+// Files the report under `site` and returns its path relative to the workspace.
+// `site` and `slug` came from the agent, so neither gets near a path unchecked.
+export function saveReport(workspace: string, site: string, slug: string, body: string, now: Date): string {
+  const day = now.toISOString().slice(0, 10)
+  const dir = join("sites", site, "incidents")
+  mkdirSync(join(workspace, dir), { recursive: true })
+  let name = `${day}-${slug}.md`
+  for (let n = 2; existsSync(join(workspace, dir, name)); n++) name = `${day}-${slug}-${n}.md`
+  writeFileSync(join(workspace, dir, name), body)
+  return join(dir, name)
+}
+
+function pointTo(workspace: string, paged: string, report: string, now: Date): void {
+  const dir = join("sites", paged, "incidents")
+  mkdirSync(join(workspace, dir), { recursive: true })
+  const rel = relative(dir, report)
+  writeFileSync(join(workspace, dir, `${now.toISOString().slice(0, 10)}-see-${report.split("/")[1]}.md`), `See [the report under ${report.split("/")[1]}](${rel}).\n`)
+}
+
+function templateReport(i: Incident, evidence: string): string {
+  const e = i.events[0]!
+  const t = new Date(`${e.time.replace(" ", "T").slice(0, 19)}Z`)
+  const p = (n: number) => String(n).padStart(2, "0")
+  const when = `${p(t.getUTCMonth() + 1)}/${p(t.getUTCDate())}/${t.getUTCFullYear()} (${p(t.getUTCHours())}:${p(t.getUTCMinutes())})`
+  return `# ${when} - ${e.domain} - ${e.kind}\n\n## Evidence\n\n${evidence}\n`
+}
+
+function finish(i: Incident, s: SentinelSettings, deps: WorkerDeps, report: string, site: string, slug: string, summary: string): Page {
+  const now = deps.now()
+  const path = saveReport(s.workspace, site, slug, report, now)
+  const paged = i.events[0]!.domain
+  if (site !== paged) pointTo(s.workspace, paged, path, now)
+  i.state = "reported"
+  i.site = site
+  i.reportPath = path
+  i.reportedAt = now.toISOString()
+  i.summary = summary
+  saveIncident(i)
+  return { text: `🧾 ${paged}: ${summary}\n\nReport: ${path}`, silent: false }
+}
+
+export async function postReport(i: Incident, report: string, s: SentinelSettings, deps: WorkerDeps): Promise<string | null> {
+  if (!s.reportTo) return null
+  i.post = "posting"
+  saveIncident(i)
+  const r = await deps.post(report, s)
+  if (r.ok) {
+    i.post = "posted"
+    i.postId = r.id
+    i.postUrl = r.url
+    delete i.postError
+    saveIncident(i)
+    return null
+  }
+  i.post = "failed"
+  i.postError = r.message
+  saveIncident(i)
+  return r.message
+}
+
+// ---- one incident ---------------------------------------------------------------------------
+
+type Step = "next" | "stop"
+
+async function diagnose(i: Incident, s: SentinelSettings, deps: WorkerDeps): Promise<Step> {
+  i.state = "diagnosing"
+  i.attempts++
+  i.startedAt = deps.now().toISOString()
+  saveIncident(i)
+  i.evidence ??= await deps.triage(i).catch((err: Error) => `Triage failed: ${err.message}`)
+  saveIncident(i)
+
+  const paged = i.events[0]!.domain
+  const deliver = async (page: Page, report: string) => {
+    const failed = await postReport(i, report, s, deps)
+    await deps.commit(s, `sentinel: ${i.id}`)
+    await deps.page(failed ? { ...page, text: `${page.text}\n\n⚠️ Couldn't post it to the board: ${failed}\nRetry with: spinuptui sentinel post ${i.id}` } : page)
+  }
+
+  if (!s.agent) {
+    const report = templateReport(i, i.evidence)
+    await deliver(finish(i, s, deps, report, paged, `${i.events[0]!.kind}-${i.id.slice(9, 13)}`, `${i.events[0]!.kind} is down; the evidence is saved.`), report)
+    return "next"
+  }
+
+  const r = await deps.agent(diagnosisPrompt(i, s.workspace, deps.now()), s)
+  if (typeof r === "string") {
+    i.state = "blocked"
+    i.why = r
+    saveIncident(i)
+    await deps.page({ text: `🚧 ${paged} wasn't diagnosed: ${r}.`, silent: false, buttons: [[diagnoseButton(i.id, "Run again")]] })
+    return "stop"
+  }
+
+  const transcript = () => {
+    mkdirSync(transcriptsDir(), { recursive: true })
+    writeFileSync(join(transcriptsDir(), `${i.id}-${i.attempts}.log`), r.output, { mode: 0o600 })
+  }
+  const status = outcomeOf(true, r.code, r.timedOut, r.output)
+
+  if (status === "auth") {
+    i.state = "waiting_agent"
+    i.attempts--
+    i.retryAfter = nextRetryAt(deps.now())
+    saveIncident(i)
+    // Once per lapse, however many incidents queue behind it.
+    const ts = loadTickState()
+    if (!ts.authPagedAt) {
+      saveTickState({ ...ts, authPagedAt: deps.now().toISOString() })
+      const waiting = loadIncidents().filter((x) => x.state === "waiting_agent" || x.state === "queued").length
+      const how = s.agent === "claude" ? "run `claude` there, then /login" : "run `codex login` there"
+      await deps.page({ text: `🚨 The ${s.agent} login on this station has lapsed, so ${waiting} incident(s) are waiting. Fix: ${how}. They'll start on their own once it's back.`, silent: false })
+    }
+    return "stop"
+  }
+  const ts = loadTickState()
+  if (ts.authPagedAt) saveTickState({ ...ts, authPagedAt: undefined })
+
+  const parsed = status === "ok" || status === "blocked" ? parseAgentReport(r.output) : null
+  if (!parsed) {
+    transcript()
+    const why = status === "timeout" ? `timed out after ${s.timeoutMin} minutes` : status === "failed" ? `the agent exited ${r.code}` : "its answer wasn't in the report format"
+    if (i.attempts < MAX_ATTEMPTS) i.state = "queued"
+    else i.state = "failed"
+    i.why = why
+    saveIncident(i)
+    if (i.state === "failed") await deps.page({ text: `❌ ${paged} wasn't diagnosed: ${why}. The transcript is in Sentinel's state folder.`, silent: false, buttons: [[diagnoseButton(i.id, "Run again")]] })
+    return "next"
+  }
+
+  if (parsed.status === "blocked") {
+    i.state = "blocked"
+    i.why = parsed.why
+    i.summary = parsed.summary
+    saveIncident(i)
+    await deps.page({ text: `🚧 ${paged} couldn't be diagnosed: ${parsed.why}`, silent: false, buttons: [[diagnoseButton(i.id, "Run again")]] })
+    return "next"
+  }
+  if (parsed.status === "duplicate") {
+    i.state = "reported"
+    i.reportPath = parsed.path
+    i.reportedAt = deps.now().toISOString()
+    i.summary = parsed.summary
+    saveIncident(i)
+    await deps.page({ text: `🧾 ${paged} is already documented: ${parsed.path}`, silent: true })
+    return "next"
+  }
+  const site = parsed.site && SAFE_SITE.test(parsed.site) ? parsed.site : paged
+  const slug = safeSlug(parsed.slug ?? "") || `${i.events[0]!.kind}-${i.id.slice(9, 13)}`
+  await deliver(finish(i, s, deps, parsed.body, site, slug, parsed.summary), parsed.body)
+  return "next"
+}
+
+// ---- the loop ---------------------------------------------------------------------------------
+
+export function realWorkerDeps(cfg: AppConfig, s: SentinelSettings): WorkerDeps {
+  const client = new SpinupWPClient(cfg)
+  return {
+    triage: (i) => triage(i, cfg, client),
+    agent: async (prompt, set) => {
+      const env = jobEnv()
+      const cmd = agentCommand({ agent: set.agent!, prompt, workdir: set.workspace, allow: set.allow, model: set.model ?? undefined }, env)
+      if (typeof cmd === "string") return cmd
+      return runProcess(cmd, { cwd: set.workspace, env, timeoutMs: set.timeoutMin * 60_000 })
+    },
+    page: (p) => sendPage(cfg, p, s.channel),
+    post: async (report) => {
+      const board = cfg.beacon?.boards?.[s.reportTo!]
+      if (!board) return { ok: false, message: `there's no Beacon board named "${s.reportTo}"` }
+      return postToBoard(board, report)
+    },
+    commit: async (set, message) => {
+      // Only Sentinel's own folder. A workspace elsewhere belongs to its owner.
+      const work = moduleWorkDir("sentinel")
+      const root = gitRootOf(work)
+      if (set.workspace === work && root) await commitAndPush(root, [relative(root, work)], message)
+    },
+    now: () => new Date(),
+  }
+}
+
+export async function runWorker(s: SentinelSettings, deps: WorkerDeps): Promise<{ ran: number; locked?: boolean }> {
+  const lock = tryLock(workerLockPath())
+  if (!lock) return { ran: 0, locked: true }
+  prepareWorkDir(s.workspace)
+  mkdirSync(dirname(transcriptsDir()), { recursive: true })
+  let ran = 0
+  try {
+    for (;;) {
+      const all = loadIncidents()
+      if (startedToday(all, deps.now()) >= s.maxDiagnosesPerDay) break
+      const next = all.find((i) => i.state === "queued")
+      if (!next) break
+      // The file, not this list: a tap may have changed it since.
+      const i = loadIncident(next.id)
+      if (!i || i.state !== "queued") continue
+      ran++
+      if ((await diagnose(i, s, deps)) === "stop") break
+    }
+  } finally {
+    lock.release()
+  }
+  return { ran }
+}

package/src/modules/beacon.ts

@@ -37,6 +37,9 @@ export const beaconModule: ModuleDefinition = {
       ok: beacon.defaultChannel !== null,
       detail: beacon.defaultChannel ?? "none — senders must pass --to",
     })
+    for (const [name, b] of Object.entries(loadConfig().beacon?.boards ?? {})) {
+      checks.push({ name: `board ${name}`, ok: true, detail: `Basecamp project ${b.projectId}, team only` })
+    }
     const listener = listenerHealth()
     checks.push(listener ? { name: "listener", ...listener } : { name: "listener", ok: true, detail: "not installed: the pager sends but doesn't answer" })
     const last = lastSendLogEntry()

package/src/modules/index.ts

@@ -4,18 +4,13 @@
 import type { ModuleDefinition, UpcomingModule } from "./contract.ts"
 import { beaconModule } from "./beacon.ts"
 import { schedulerModule } from "./scheduler.ts"
+import { sentinelModule } from "./sentinel.ts"
 
-export const MODULES: ModuleDefinition[] = [beaconModule, schedulerModule]
+export const MODULES: ModuleDefinition[] = [beaconModule, schedulerModule, sentinelModule]
 
 // Outcome copy only — the same lines feed the landing page. An entry moves to
 // MODULES (and out of here) when the module ships.
 export const UPCOMING_MODULES: UpcomingModule[] = [
-  {
-    id: "sentinel",
-    name: "Sentinel",
-    summary: "Watches the fleet, works out what actually broke, and writes it up.",
-    dependsOn: ["beacon", "scheduler"],
-  },
   {
     id: "samurai",
     name: "Samurai",

package/src/modules/scheduler.ts

@@ -11,6 +11,7 @@ import { fmtDate, nextRunText, parseDate, parseWhen } from "../lib/schedule.ts"
 import { findSource, listSources, loadJobs, loadState, readLastTick, type Source } from "../lib/schedulerJobs.ts"
 import { gitPull, readClaim, stationName } from "../lib/schedulerGit.ts"
 import { addPeriod, extendJob } from "../lib/schedulerExtend.ts"
+import { jobEnv, runAgainButton, selfCommand, spawnDetached } from "../lib/schedulerRun.ts"
 import { moduleStateDir, moduleWorkDir, type BeaconButton, type BeaconReply, type ModuleCheck, type ModuleDefinition, type ModuleStatus } from "./contract.ts"
 
 // Three missed ticks.
@@ -53,7 +54,10 @@ export const schedulerModule: ModuleDefinition = {
   workDir: () => moduleWorkDir("scheduler"),
   jobs: [],
   commands: [{ command: "jobs", describe: "this station's scheduled jobs and when they expire", run: () => jobsReply() }],
-  actions: [{ action: "scheduler.extend", run: (payload) => extendReply(payload) }],
+  actions: [
+    { action: "scheduler.extend", run: (payload) => extendReply(payload) },
+    { action: "scheduler.run", run: (payload) => runAgainReply(payload) },
+  ],
   async status(): Promise<ModuleStatus> {
     const cfg = loadConfig()
     const d = cfg.scheduler?.dispatcher
@@ -211,7 +215,7 @@ export async function jobsReply(opts: { now?: Date; pull?: boolean } = {}): Prom
   const now = opts.now ?? new Date()
   const sources = listSources(loadConfig()).filter((s) => existsSync(s.path))
   const multi = sources.length > 1
-  const rows: { block: string; days: number; button?: BeaconButton }[] = []
+  const rows: { block: string; days: number; buttons: BeaconButton[] }[] = []
   let hidden = 0
   const problems: string[] = []
   for (const s of sources) {
@@ -244,11 +248,13 @@ export async function jobsReply(opts: { now?: Date; pull?: boolean } = {}): Prom
       const nextText = !h.when && /^\d{4}-/.test(next) ? ` · next ${shortTime(next, now)}` : ""
       const block = [`${soon ? "⚠️ " : ""}<b>${escapeHtml(name)}</b>`, expiry, escapeHtml(`${kind} · ${shortSchedule(h, now)}${nextText}`)].join("\n")
       const extendable = soon && !h.when
-      rows.push({
-        block,
-        days,
-        ...(extendable ? { button: { text: `Extend ${name} 1 year`, action: "scheduler.extend", payload: { source: s.name, id: h.id!, from: h.expires! } } } : {}),
-      })
+      const status = loadState(s, h.id!).last_status
+      const again = !!status && RUN_AGAIN_STATUSES.has(status) && next !== "expired"
+      const buttons = [
+        ...(extendable ? [{ text: `Extend ${name} 1 year`, action: "scheduler.extend", payload: { source: s.name, id: h.id!, from: h.expires! } }] : []),
+        ...(again ? [runAgainButton(s.name, h.id!)] : []),
+      ]
+      rows.push({ block: again ? `${block}\nLast run: ${escapeHtml(status!)}` : block, days, buttons })
     }
   }
   rows.sort((a, b) => a.days - b.days)
@@ -258,10 +264,27 @@ export async function jobsReply(opts: { now?: Date; pull?: boolean } = {}): Prom
   if (hidden) tail.push(`${hidden} finished or long-expired not shown.`)
   for (const p of problems) tail.push(`⚠️ ${escapeHtml(p)}`)
   const text = [head, ...rows.map((r) => r.block), ...(tail.length ? [tail.join("\n")] : [])].join("\n\n")
-  const buttons = rows.filter((r) => r.button).map((r) => [r.button!])
+  const buttons = rows.flatMap((r) => r.buttons.map((b) => [b]))
   return { text, html: true, ...(buttons.length ? { buttons } : {}) }
 }
 
+const RUN_AGAIN_STATUSES = new Set(["failed", "blocked", "timeout", "error"])
+
+// The Run again button. A job can take half an hour and the listener answers one
+// update at a time, so it's started apart and pages its result itself. It runs
+// through `scheduler run`, the same runner with the same allow list and sandbox.
+export async function runAgainReply(payload: Record<string, string>): Promise<BeaconReply> {
+  const { source, id } = payload
+  if (!source || !id) return { text: "That button is missing its job; send /jobs again." }
+  const s = findSource(loadConfig(), source)
+  if (!s) return { text: `There's no Scheduler source named "${source}" on this station any more.` }
+  if (!loadJobs(s).some((j) => j.header.id === id)) return { text: `Job ${id} isn't in ${source} any more.` }
+  // `scheduler run` needs the tick lock. Buttons work once, so offer a fresh one.
+  if (tickRunning()) return { text: `A tick is running, so ${id} can't start yet.`, buttons: [[runAgainButton(source, id)]] }
+  spawnDetached([...selfCommand(), "scheduler", "run", id, "--source", source], { cwd: moduleStateDir("scheduler"), env: jobEnv() })
+  return { text: `▶️ Running ${id} again. Its result will page here.` }
+}
+
 // The "Extend 1 year" button: a year past the expiry the reply showed, so a
 // stale button can't stack a second year onto one that was already extended.
 export async function extendReply(payload: Record<string, string>, now = new Date()): Promise<BeaconReply> {

package/src/modules/sentinel.ts

@@ -0,0 +1,90 @@
+// Sentinel's module declaration: watches the fleet, works out what broke, writes
+// it up. Its status is cheap and sends nothing: Scheduler asks every tick.
+
+import { existsSync } from "node:fs"
+import { listProfiles, loadConfig } from "../config.ts"
+import { beaconFromConfig, escapeHtml } from "../lib/beacon.ts"
+import { hasIncidentMonitors } from "../lib/cliIncidents.ts"
+import { waitingRows } from "../lib/sentinelReply.ts"
+import { isWaiting, loadIncident, loadIncidents, loadPause, loadTickState, saveIncident, sentinelSettings } from "../lib/sentinelState.ts"
+import { moduleStateDir, moduleWorkDir, type BeaconReply, type ModuleCheck, type ModuleDefinition, type ModuleStatus } from "./contract.ts"
+
+const TICK_STALE_MS = 15 * 60_000
+const STUCK_MS = 24 * 60 * 60_000
+
+export const sentinelModule: ModuleDefinition = {
+  id: "sentinel",
+  name: "Sentinel",
+  summary: "Watches the fleet, works out what actually broke, and writes it up.",
+  dependsOn: ["beacon", "scheduler"],
+  configKey: "sentinel",
+  stateDir: () => moduleStateDir("sentinel"),
+  workDir: () => moduleWorkDir("sentinel"),
+  jobs: [
+    { id: "sentinel.tick", every: "5m", command: ["sentinel", "tick"] },
+    { id: "sentinel.digest", at: "08:00", command: ["sentinel", "digest"] },
+  ],
+  commands: [{ command: "incidents", describe: "open incidents and anything waiting on you", run: () => incidentsReply() }],
+  actions: [{ action: "sentinel.diagnose", run: (payload) => diagnoseReply(payload) }],
+  async status(): Promise<ModuleStatus> {
+    const cfg = loadConfig()
+    const s = sentinelSettings(cfg)
+    if (!s.enabled) {
+      return { id: "sentinel", state: "not_configured", summary: "Not set up", checks: [], remedy: "spinuptui sentinel setup --plan" }
+    }
+    const checks: ModuleCheck[] = []
+    const domains = Object.entries(cfg.kumaMonitors).filter(([, r]) => hasIncidentMonitors(r)).length
+    checks.push({ name: "watching", ok: domains > 0, detail: domains ? `${domains} domain${domains === 1 ? "" : "s"}` : "no monitors registered: run spinuptui sentinel attach --plan" })
+    const t = loadTickState()
+    const age = t.lastTickAt ? Date.now() - Date.parse(t.lastTickAt) : null
+    const stale = age === null || age > TICK_STALE_MS
+    checks.push({
+      name: "last tick",
+      ok: !t.blindSince && !stale,
+      detail: t.blindSince ? `blind since ${t.blindSince}: ${t.lastError}` : age === null ? "none yet" : `${Math.round(age / 60_000)} min ago${t.lastError ? ` (${t.lastError})` : ""}`,
+    })
+    const incidents = loadIncidents()
+    const now = Date.now()
+    const waiting = incidents.filter(isWaiting)
+    const stuck = waiting.filter((i) => now - Date.parse(i.createdAt) > STUCK_MS)
+    const count = (state: string) => waiting.filter((i) => i.state === state).length
+    const queue = [["queued", count("queued")], ["diagnosing", count("diagnosing")], ["waiting on login", count("waiting_agent")], ["blocked", count("blocked")], ["failed", count("failed")]].filter(([, n]) => n)
+    checks.push({ name: "queue", ok: stuck.length === 0, detail: queue.length ? queue.map(([k, n]) => `${n} ${k}`).join(", ") : "empty" })
+    const agentBin = s.agent ? Bun.which(s.agent, { PATH: `${process.env.HOME}/.local/bin:${process.env.HOME}/.bun/bin:/opt/homebrew/bin:/usr/local/bin:${process.env.PATH}` }) : null
+    checks.push({ name: "agent", ok: !s.agent || !!agentBin, detail: !s.agent ? "none: incidents get a write-up of the evidence, not a diagnosis" : agentBin ? s.agent : `${s.agent} isn't on the PATH` })
+    checks.push({ name: "beacon", ok: beaconFromConfig(cfg).channels.length > 0, detail: s.channel ?? "default channel" })
+    if (s.reportTo) checks.push({ name: "board", ok: !!cfg.beacon?.boards?.[s.reportTo], detail: cfg.beacon?.boards?.[s.reportTo] ? s.reportTo : `no Beacon board named ${s.reportTo}` })
+    const pause = loadPause()
+    const paused = pause && Date.parse(pause.until) > now
+    checks.push({ name: "mode", ok: true, detail: paused ? `paused until ${new Date(pause.until).toLocaleString()}` : s.shadow ? "shadow: pages nothing, runs no agent" : "live" })
+    checks.push({ name: "workspace", ok: existsSync(s.workspace) || s.workspace === moduleWorkDir("sentinel"), detail: s.workspace })
+    const others = listProfiles().filter((p) => !p.active).length
+    if (others) checks.push({ name: "accounts", ok: true, detail: `watching the active account only; ${others} other account${others === 1 ? " is" : "s are"} not covered` })
+    const failing = checks.find((c) => !c.ok)
+    return {
+      id: "sentinel",
+      state: failing ? "degraded" : "ok",
+      summary: failing ? `${failing.name}: ${failing.detail}` : paused ? "paused" : s.shadow ? `shadow, ${domains} domains` : `watching ${domains} domain${domains === 1 ? "" : "s"}`,
+      checks,
+      ...(failing ? { remedy: failing.name === "watching" ? "spinuptui sentinel attach --plan" : "spinuptui sentinel status" } : {}),
+    }
+  },
+}
+
+export async function incidentsReply(): Promise<BeaconReply> {
+  const rows = waitingRows(loadIncidents())
+  if (!rows.length) return { text: "No open incidents." }
+  const text = [`<b>Incidents</b> (${rows.length})`, ...rows.map((r) => escapeHtml(r.line))].join("\n\n")
+  const buttons = rows.filter((r) => r.button).map((r) => [r.button!])
+  return { text, html: true, ...(buttons.length ? { buttons } : {}) }
+}
+
+// A tap only queues: the next tick starts the worker, so it can't run anything
+// the tick wouldn't, and it can't add permissions.
+export async function diagnoseReply(payload: Record<string, string>): Promise<BeaconReply> {
+  const i = payload.id ? loadIncident(payload.id) : null
+  if (!i) return { text: "That incident is gone; send /incidents again." }
+  if (!["recurrence", "blocked", "failed"].includes(i.state)) return { text: `${i.id} is ${i.state} already.` }
+  saveIncident({ ...i, state: "queued", attempts: 0, why: undefined })
+  return { text: `▶️ Queued ${i.events[0]!.domain}. The next tick starts it, within 5 minutes.` }
+}

package/src/ui/App.tsx

@@ -43,6 +43,7 @@ import { DnsRecords } from "./views/DnsRecords.tsx"
 import { KumaSite } from "./views/KumaSite.tsx"
 import { Accounts } from "./views/Accounts.tsx"
 import { Modules } from "./views/Modules.tsx"
+import { SentinelOverlay } from "./views/SentinelSetup.tsx"
 import { BeaconSetup } from "./views/BeaconSetup.tsx"
 import { SchedulerInstall } from "./views/SchedulerInstall.tsx"
 import { HomeSync } from "./views/HomeSync.tsx"
@@ -99,6 +100,7 @@ export function App() {
     store.kumaSite !== null ||
     store.beaconSetupOpen ||
     store.schedulerInstallOpen ||
+    store.sentinelOverlay !== null ||
     store.homeSyncOpen ||
     store.heartbeatSetupOpen ||
     store.listenerSetupOpen ||
@@ -224,6 +226,7 @@ export function App() {
     // Beacon's guided setup (Modules tab) owns the keyboard while open.
     if (store.beaconSetupOpen) return
     if (store.schedulerInstallOpen) return
+    if (store.sentinelOverlay) return
     if (store.homeSyncOpen) return
     if (store.heartbeatSetupOpen) return
     if (store.listenerSetupOpen) return
@@ -317,6 +320,7 @@ export function App() {
       {store.kumaSite && <KumaSite />}
       {store.beaconSetupOpen && <BeaconSetup />}
       {store.schedulerInstallOpen && <SchedulerInstall />}
+      {store.sentinelOverlay && <SentinelOverlay />}
       {store.homeSyncOpen && <HomeSync />}
       {store.heartbeatSetupOpen && <HeartbeatSetup />}
       {store.listenerSetupOpen && <ListenerSetup />}

package/src/ui/store.tsx

@@ -751,6 +751,8 @@ interface StoreValue extends DataState {
   // Scheduler's install plan + confirm overlay, opened from the Modules tab.
   schedulerInstallOpen: boolean
   setSchedulerInstallOpen: (v: boolean) => void
+  sentinelOverlay: "setup" | "digest" | null
+  setSentinelOverlay: (v: "setup" | "digest" | null) => void
   // The SpinupTUI folder: where it is, and sharing it across machines (Modules tab).
   homeSyncOpen: boolean
   setHomeSyncOpen: (v: boolean) => void
@@ -1157,6 +1159,7 @@ export function StoreProvider({ children }: { children: ReactNode }) {
   const [discoverOpen, setDiscoverOpen] = useState(false)
   const [beaconSetupOpen, setBeaconSetupOpen] = useState(false)
   const [schedulerInstallOpen, setSchedulerInstallOpen] = useState(false)
+  const [sentinelOverlay, setSentinelOverlay] = useState<"setup" | "digest" | null>(null)
   const [homeSyncOpen, setHomeSyncOpen] = useState(false)
   const [heartbeatSetupOpen, setHeartbeatSetupOpen] = useState(false)
   const [listenerSetupOpen, setListenerSetupOpen] = useState(false)
@@ -4620,6 +4623,8 @@ export function StoreProvider({ children }: { children: ReactNode }) {
     setBeaconSetupOpen,
     schedulerInstallOpen,
     setSchedulerInstallOpen,
+    sentinelOverlay,
+    setSentinelOverlay,
     homeSyncOpen,
     setHomeSyncOpen,
     heartbeatSetupOpen,