spinuptui 1.3.0-alpha.3 → 1.3.0-alpha.5

package/src/lib/sentinelWorker.ts

@@ -0,0 +1,291 @@
+// The diagnosis worker: everything slow, outside the tick. A tick starts it when
+// work is queued, and it takes the oldest incident at a time until none are left,
+// then exits. It isn't a scheduled unit, and only one runs at a time.
+//
+// The worker, not the agent, saves the report and posts it. So a missing write
+// permission can't lose a report, and nothing the agent read can make it publish
+// (docs/2026-10-30_sentinel-spec.md, "Diagnosis").
+
+import { existsSync, mkdirSync, writeFileSync } from "node:fs"
+import { dirname, join, relative } from "node:path"
+import type { AppConfig } from "../config.ts"
+import { SpinupWPClient, type SpinupWPClientLike } from "../api/client.ts"
+import { moduleWorkDir } from "../modules/contract.ts"
+import { postToBoard } from "./beaconBoard.ts"
+import { resolveSiblings } from "./cliSiblings.ts"
+import { tryLock } from "./flock.ts"
+import { prepareWorkDir } from "./home.ts"
+import { commitAndPush } from "./schedulerGit.ts"
+import { gitRootOf } from "./schedulerJobs.ts"
+import { agentCommand, jobEnv, outcomeOf, runProcess, type ProcResult } from "./schedulerRun.ts"
+import { diagnoseButton, startedToday, type Page } from "./sentinelIncidents.ts"
+import { diagnosisPrompt, parseAgentReport } from "./sentinelPrompt.ts"
+import { loadIncident, loadIncidents, loadTickState, saveIncident, saveTickState, transcriptsDir, workerLockPath, type Incident, type SentinelSettings } from "./sentinelState.ts"
+import { MAX_ATTEMPTS, nextRetryAt, sendPage } from "./sentinelTick.ts"
+import { runSiteDoctor } from "./siteDoctor.ts"
+import { execSshCommand } from "./sshExec.ts"
+
+const SNAPSHOT = "uptime; free -m | head -3; df -h / | tail -1"
+
+export interface WorkerDeps {
+  triage(i: Incident): Promise<string>
+  agent(prompt: string, s: SentinelSettings): Promise<ProcResult | string>
+  page(p: Page): Promise<string>
+  post(report: string, s: SentinelSettings): Promise<{ ok: true; id: string; url: string } | { ok: false; message: string }>
+  commit(s: SentinelSettings, message: string): Promise<void>
+  now(): Date
+}
+
+// ---- triage ---------------------------------------------------------------------------
+
+// What a person would look at first. Deterministic and needing no agent, so an
+// install without one still gets a useful write-up. Each step can fail on its own.
+//
+// It never uses `siblings --probe`: one SSH attempt per co-tenant, on every
+// incident, is how a station gets itself banned by fail2ban.
+export async function triage(i: Incident, cfg: AppConfig, client: SpinupWPClientLike): Promise<string> {
+  const domains = [...new Set(i.events.map((e) => e.domain))]
+  const out: string[] = []
+  const events = i.events.map((e) => `- ${e.time} UTC: ${e.domain} ${e.kind}${e.message ? `: ${e.message}` : ""}`)
+  out.push(`### Events\n\n${events.join("\n")}`)
+
+  const first = domains[0]!
+  try {
+    const sib = await resolveSiblings(first, client, cfg, {})
+    if (sib.ok) {
+      const sites = sib.sites.map((s) => `- ${s.domain}${s.isVanity ? " (vanity host)" : ""}${s.monitored ? "" : " (not monitored)"}`)
+      out.push(`### Server\n\n${sib.server.name}: ${sib.server.size ?? "unknown size"}, Ubuntu ${sib.server.ubuntuVersion ?? "?"}\n\n${sites.join("\n")}`)
+    } else out.push(`### Server\n\nCouldn't look it up: ${sib.message}`)
+  } catch (err) {
+    out.push(`### Server\n\nCouldn't look it up: ${(err as Error).message}`)
+  }
+
+  const doctor: string[] = []
+  for (const d of domains.slice(0, 5)) {
+    try {
+      const r = await runSiteDoctor({ url: `https://${d}/` })
+      doctor.push(`- ${d}: ${r.verdict}. ${r.summary}`)
+    } catch (err) {
+      doctor.push(`- ${d}: the check failed: ${(err as Error).message}`)
+    }
+  }
+  out.push(`### HTTP\n\n${doctor.join("\n")}`)
+
+  try {
+    const r = await execSshCommand(first, SNAPSHOT, client, cfg)
+    if (r.ok) out.push(`### Load on the box\n\n\`\`\`\n${r.stdout.trim()}\n\`\`\``)
+  } catch {
+    // SSH isn't reachable: the rest stands without it.
+  }
+  return out.join("\n\n")
+}
+
+// ---- saving and posting -----------------------------------------------------------------
+
+const SAFE_SITE = /^[a-z0-9][a-z0-9.-]*\.[a-z0-9-]+$/i
+const safeSlug = (s: string) => s.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60)
+
+// Files the report under `site` and returns its path relative to the workspace.
+// `site` and `slug` came from the agent, so neither gets near a path unchecked.
+export function saveReport(workspace: string, site: string, slug: string, body: string, now: Date): string {
+  const day = now.toISOString().slice(0, 10)
+  const dir = join("sites", site, "incidents")
+  mkdirSync(join(workspace, dir), { recursive: true })
+  let name = `${day}-${slug}.md`
+  for (let n = 2; existsSync(join(workspace, dir, name)); n++) name = `${day}-${slug}-${n}.md`
+  writeFileSync(join(workspace, dir, name), body)
+  return join(dir, name)
+}
+
+function pointTo(workspace: string, paged: string, report: string, now: Date): void {
+  const dir = join("sites", paged, "incidents")
+  mkdirSync(join(workspace, dir), { recursive: true })
+  const rel = relative(dir, report)
+  writeFileSync(join(workspace, dir, `${now.toISOString().slice(0, 10)}-see-${report.split("/")[1]}.md`), `See [the report under ${report.split("/")[1]}](${rel}).\n`)
+}
+
+function templateReport(i: Incident, evidence: string): string {
+  const e = i.events[0]!
+  const t = new Date(`${e.time.replace(" ", "T").slice(0, 19)}Z`)
+  const p = (n: number) => String(n).padStart(2, "0")
+  const when = `${p(t.getUTCMonth() + 1)}/${p(t.getUTCDate())}/${t.getUTCFullYear()} (${p(t.getUTCHours())}:${p(t.getUTCMinutes())})`
+  return `# ${when} - ${e.domain} - ${e.kind}\n\n## Evidence\n\n${evidence}\n`
+}
+
+function finish(i: Incident, s: SentinelSettings, deps: WorkerDeps, report: string, site: string, slug: string, summary: string): Page {
+  const now = deps.now()
+  const path = saveReport(s.workspace, site, slug, report, now)
+  const paged = i.events[0]!.domain
+  if (site !== paged) pointTo(s.workspace, paged, path, now)
+  i.state = "reported"
+  i.site = site
+  i.reportPath = path
+  i.reportedAt = now.toISOString()
+  i.summary = summary
+  saveIncident(i)
+  return { text: `🧾 ${paged}: ${summary}\n\nReport: ${path}`, silent: false }
+}
+
+export async function postReport(i: Incident, report: string, s: SentinelSettings, deps: WorkerDeps): Promise<string | null> {
+  if (!s.reportTo) return null
+  i.post = "posting"
+  saveIncident(i)
+  const r = await deps.post(report, s)
+  if (r.ok) {
+    i.post = "posted"
+    i.postId = r.id
+    i.postUrl = r.url
+    delete i.postError
+    saveIncident(i)
+    return null
+  }
+  i.post = "failed"
+  i.postError = r.message
+  saveIncident(i)
+  return r.message
+}
+
+// ---- one incident ---------------------------------------------------------------------------
+
+type Step = "next" | "stop"
+
+async function diagnose(i: Incident, s: SentinelSettings, deps: WorkerDeps): Promise<Step> {
+  i.state = "diagnosing"
+  i.attempts++
+  i.startedAt = deps.now().toISOString()
+  saveIncident(i)
+  i.evidence ??= await deps.triage(i).catch((err: Error) => `Triage failed: ${err.message}`)
+  saveIncident(i)
+
+  const paged = i.events[0]!.domain
+  const deliver = async (page: Page, report: string) => {
+    const failed = await postReport(i, report, s, deps)
+    await deps.commit(s, `sentinel: ${i.id}`)
+    await deps.page(failed ? { ...page, text: `${page.text}\n\n⚠️ Couldn't post it to the board: ${failed}\nRetry with: spinuptui sentinel post ${i.id}` } : page)
+  }
+
+  if (!s.agent) {
+    const report = templateReport(i, i.evidence)
+    await deliver(finish(i, s, deps, report, paged, `${i.events[0]!.kind}-${i.id.slice(9, 13)}`, `${i.events[0]!.kind} is down; the evidence is saved.`), report)
+    return "next"
+  }
+
+  const r = await deps.agent(diagnosisPrompt(i, s.workspace, deps.now()), s)
+  if (typeof r === "string") {
+    i.state = "blocked"
+    i.why = r
+    saveIncident(i)
+    await deps.page({ text: `🚧 ${paged} wasn't diagnosed: ${r}.`, silent: false, buttons: [[diagnoseButton(i.id, "Run again")]] })
+    return "stop"
+  }
+
+  const transcript = () => {
+    mkdirSync(transcriptsDir(), { recursive: true })
+    writeFileSync(join(transcriptsDir(), `${i.id}-${i.attempts}.log`), r.output, { mode: 0o600 })
+  }
+  const status = outcomeOf(true, r.code, r.timedOut, r.output)
+
+  if (status === "auth") {
+    i.state = "waiting_agent"
+    i.attempts--
+    i.retryAfter = nextRetryAt(deps.now())
+    saveIncident(i)
+    // Once per lapse, however many incidents queue behind it.
+    const ts = loadTickState()
+    if (!ts.authPagedAt) {
+      saveTickState({ ...ts, authPagedAt: deps.now().toISOString() })
+      const waiting = loadIncidents().filter((x) => x.state === "waiting_agent" || x.state === "queued").length
+      const how = s.agent === "claude" ? "run `claude` there, then /login" : "run `codex login` there"
+      await deps.page({ text: `🚨 The ${s.agent} login on this station has lapsed, so ${waiting} incident(s) are waiting. Fix: ${how}. They'll start on their own once it's back.`, silent: false })
+    }
+    return "stop"
+  }
+  const ts = loadTickState()
+  if (ts.authPagedAt) saveTickState({ ...ts, authPagedAt: undefined })
+
+  const parsed = status === "ok" || status === "blocked" ? parseAgentReport(r.output) : null
+  if (!parsed) {
+    transcript()
+    const why = status === "timeout" ? `timed out after ${s.timeoutMin} minutes` : status === "failed" ? `the agent exited ${r.code}` : "its answer wasn't in the report format"
+    if (i.attempts < MAX_ATTEMPTS) i.state = "queued"
+    else i.state = "failed"
+    i.why = why
+    saveIncident(i)
+    if (i.state === "failed") await deps.page({ text: `❌ ${paged} wasn't diagnosed: ${why}. The transcript is in Sentinel's state folder.`, silent: false, buttons: [[diagnoseButton(i.id, "Run again")]] })
+    return "next"
+  }
+
+  if (parsed.status === "blocked") {
+    i.state = "blocked"
+    i.why = parsed.why
+    i.summary = parsed.summary
+    saveIncident(i)
+    await deps.page({ text: `🚧 ${paged} couldn't be diagnosed: ${parsed.why}`, silent: false, buttons: [[diagnoseButton(i.id, "Run again")]] })
+    return "next"
+  }
+  if (parsed.status === "duplicate") {
+    i.state = "reported"
+    i.reportPath = parsed.path
+    i.reportedAt = deps.now().toISOString()
+    i.summary = parsed.summary
+    saveIncident(i)
+    await deps.page({ text: `🧾 ${paged} is already documented: ${parsed.path}`, silent: true })
+    return "next"
+  }
+  const site = parsed.site && SAFE_SITE.test(parsed.site) ? parsed.site : paged
+  const slug = safeSlug(parsed.slug ?? "") || `${i.events[0]!.kind}-${i.id.slice(9, 13)}`
+  await deliver(finish(i, s, deps, parsed.body, site, slug, parsed.summary), parsed.body)
+  return "next"
+}
+
+// ---- the loop ---------------------------------------------------------------------------------
+
+export function realWorkerDeps(cfg: AppConfig, s: SentinelSettings): WorkerDeps {
+  const client = new SpinupWPClient(cfg)
+  return {
+    triage: (i) => triage(i, cfg, client),
+    agent: async (prompt, set) => {
+      const env = jobEnv()
+      const cmd = agentCommand({ agent: set.agent!, prompt, workdir: set.workspace, allow: set.allow, model: set.model ?? undefined }, env)
+      if (typeof cmd === "string") return cmd
+      return runProcess(cmd, { cwd: set.workspace, env, timeoutMs: set.timeoutMin * 60_000 })
+    },
+    page: (p) => sendPage(cfg, p, s.channel),
+    post: async (report) => {
+      const board = cfg.beacon?.boards?.[s.reportTo!]
+      if (!board) return { ok: false, message: `there's no Beacon board named "${s.reportTo}"` }
+      return postToBoard(board, report)
+    },
+    commit: async (set, message) => {
+      // Only Sentinel's own folder. A workspace elsewhere belongs to its owner.
+      const work = moduleWorkDir("sentinel")
+      const root = gitRootOf(work)
+      if (set.workspace === work && root) await commitAndPush(root, [relative(root, work)], message)
+    },
+    now: () => new Date(),
+  }
+}
+
+export async function runWorker(s: SentinelSettings, deps: WorkerDeps): Promise<{ ran: number; locked?: boolean }> {
+  const lock = tryLock(workerLockPath())
+  if (!lock) return { ran: 0, locked: true }
+  prepareWorkDir(s.workspace)
+  mkdirSync(dirname(transcriptsDir()), { recursive: true })
+  let ran = 0
+  try {
+    for (;;) {
+      const all = loadIncidents()
+      if (startedToday(all, deps.now()) >= s.maxDiagnosesPerDay) break
+      const next = all.find((i) => i.state === "queued")
+      if (!next) break
+      // The file, not this list: a tap may have changed it since.
+      const i = loadIncident(next.id)
+      if (!i || i.state !== "queued") continue
+      ran++
+      if ((await diagnose(i, s, deps)) === "stop") break
+    }
+  } finally {
+    lock.release()
+  }
+  return { ran }
+}

package/src/modules/beacon.ts

@@ -2,6 +2,7 @@
 
 import { loadConfig } from "../config.ts"
 import { beaconFromConfig, lastSendLogEntry, maskChatId, maskToken } from "../lib/beacon.ts"
+import { listenerHealth } from "../lib/beaconListenInstall.ts"
 import { moduleStateDir, moduleWorkDir, type ModuleCheck, type ModuleDefinition, type ModuleStatus } from "./contract.ts"
 
 export const beaconModule: ModuleDefinition = {
@@ -12,7 +13,9 @@ export const beaconModule: ModuleDefinition = {
   configKey: "beacon",
   stateDir: () => moduleStateDir("beacon"),
   workDir: () => moduleWorkDir("beacon"),
-  jobs: [],
+  // The listener's alive check: exits 1 while an installed listener isn't
+  // polling, so Scheduler pages once when it stops and once when it's back.
+  jobs: [{ id: "beacon.listener", every: "5m", command: ["beacon", "listener", "check"] }],
   async status(): Promise<ModuleStatus> {
     const beacon = beaconFromConfig(loadConfig())
     if (beacon.channels.length === 0) {
@@ -34,6 +37,11 @@ export const beaconModule: ModuleDefinition = {
       ok: beacon.defaultChannel !== null,
       detail: beacon.defaultChannel ?? "none — senders must pass --to",
     })
+    for (const [name, b] of Object.entries(loadConfig().beacon?.boards ?? {})) {
+      checks.push({ name: `board ${name}`, ok: true, detail: `Basecamp project ${b.projectId}, team only` })
+    }
+    const listener = listenerHealth()
+    checks.push(listener ? { name: "listener", ...listener } : { name: "listener", ok: true, detail: "not installed: the pager sends but doesn't answer" })
     const last = lastSendLogEntry()
     checks.push({
       name: "last send",

package/src/modules/contract.ts

@@ -16,6 +16,10 @@
 // - Recurring work is declared in `jobs` for Scheduler to run; a module never
 //   installs an OS scheduler unit of its own.
 // - status() is computed on demand, is cheap, and never sends anything.
+// - What a module answers on the operator's pager is declared in `commands`
+//   (a /command) and `actions` (a button tap). Beacon's listener receives,
+//   checks identity, records what it handled and routes; it knows nothing about
+//   what a command or button means.
 
 import { homedir } from "node:os"
 import { join } from "node:path"
@@ -31,6 +35,34 @@ export interface ScheduledJobDecl {
   command: string[]
 }
 
+// A reply on the operator channel: plain text, or Telegram HTML when `html` is
+// set (escape anything variable with escapeHtml), with optional rows of inline
+// buttons. A button names one of the module's own actions and the
+// payload to hand it; Beacon stores the payload and gives Telegram a short
+// one-use token instead, so a button works once, whatever its payload's size.
+export interface BeaconReply {
+  text: string
+  html?: boolean
+  buttons?: BeaconButton[][]
+}
+
+export interface BeaconButton {
+  text: string
+  action: string // e.g. "scheduler.extend"
+  payload: Record<string, string>
+}
+
+export interface BeaconCommandDecl {
+  command: string // "jobs", answered as /jobs
+  describe: string // one line for /help
+  run(): Promise<BeaconReply>
+}
+
+export interface BeaconActionDecl {
+  action: string // "scheduler.extend"
+  run(payload: Record<string, string>): Promise<BeaconReply>
+}
+
 export interface ModuleCheck {
   name: string
   ok: boolean
@@ -57,6 +89,8 @@ export interface ModuleDefinition {
   // The user's work. Created on first use, never at install.
   workDir(): string
   jobs: ScheduledJobDecl[]
+  commands?: BeaconCommandDecl[]
+  actions?: BeaconActionDecl[]
   status(): Promise<ModuleStatus>
 }
 

package/src/modules/index.ts

@@ -4,18 +4,13 @@
 import type { ModuleDefinition, UpcomingModule } from "./contract.ts"
 import { beaconModule } from "./beacon.ts"
 import { schedulerModule } from "./scheduler.ts"
+import { sentinelModule } from "./sentinel.ts"
 
-export const MODULES: ModuleDefinition[] = [beaconModule, schedulerModule]
+export const MODULES: ModuleDefinition[] = [beaconModule, schedulerModule, sentinelModule]
 
 // Outcome copy only — the same lines feed the landing page. An entry moves to
 // MODULES (and out of here) when the module ships.
 export const UPCOMING_MODULES: UpcomingModule[] = [
-  {
-    id: "sentinel",
-    name: "Sentinel",
-    summary: "Watches the fleet, works out what actually broke, and writes it up.",
-    dependsOn: ["beacon", "scheduler"],
-  },
   {
     id: "samurai",
     name: "Samurai",

package/src/modules/scheduler.ts

@@ -6,11 +6,13 @@ import { existsSync } from "node:fs"
 import { join } from "node:path"
 import { tryLock } from "../lib/flock.ts"
 import { loadConfig } from "../config.ts"
-import { beaconFromConfig } from "../lib/beacon.ts"
-import { nextRunText, parseWhen } from "../lib/schedule.ts"
-import { listSources, loadJobs, loadState, readLastTick, type Source } from "../lib/schedulerJobs.ts"
-import { readClaim, stationName } from "../lib/schedulerGit.ts"
-import { moduleStateDir, moduleWorkDir, type ModuleCheck, type ModuleDefinition, type ModuleStatus } from "./contract.ts"
+import { beaconFromConfig, escapeHtml } from "../lib/beacon.ts"
+import { fmtDate, nextRunText, parseDate, parseWhen } from "../lib/schedule.ts"
+import { findSource, listSources, loadJobs, loadState, readLastTick, type Source } from "../lib/schedulerJobs.ts"
+import { gitPull, readClaim, stationName } from "../lib/schedulerGit.ts"
+import { addPeriod, extendJob } from "../lib/schedulerExtend.ts"
+import { jobEnv, runAgainButton, selfCommand, spawnDetached } from "../lib/schedulerRun.ts"
+import { moduleStateDir, moduleWorkDir, type BeaconButton, type BeaconReply, type ModuleCheck, type ModuleDefinition, type ModuleStatus } from "./contract.ts"
 
 // Three missed ticks.
 export const TICK_STALE_MS = 15 * 60_000
@@ -51,6 +53,11 @@ export const schedulerModule: ModuleDefinition = {
   stateDir: () => moduleStateDir("scheduler"),
   workDir: () => moduleWorkDir("scheduler"),
   jobs: [],
+  commands: [{ command: "jobs", describe: "this station's scheduled jobs and when they expire", run: () => jobsReply() }],
+  actions: [
+    { action: "scheduler.extend", run: (payload) => extendReply(payload) },
+    { action: "scheduler.run", run: (payload) => runAgainReply(payload) },
+  ],
   async status(): Promise<ModuleStatus> {
     const cfg = loadConfig()
     const d = cfg.scheduler?.dispatcher
@@ -110,7 +117,7 @@ export const schedulerModule: ModuleDefinition = {
       name: "heartbeat",
       ok: !hb || !lastHb || lastHb.ok,
       detail: !hb
-        ? "not set: nothing outside this machine notices if it stops (spinuptui scheduler heartbeat set <kuma-push-url>)"
+        ? `not set: nothing outside this machine notices if it stops (${cfg.uptimeKuma ? "h on this card, or spinuptui scheduler heartbeat setup --plan" : "h on this card, or spinuptui scheduler heartbeat set <kuma-push-url>"})`
         : !lastHb
           ? `${new URL(hb.url).host}: first ping on the next dispatcher tick`
           : lastHb.ok
@@ -157,3 +164,137 @@ export function upcomingRuns(limit = 5): { source: string; id: string; next: str
   // Fixed times sort before "every N min", which has no single next time.
   return out.sort((a, b) => (a.next.startsWith("every") ? 1 : 0) - (b.next.startsWith("every") ? 1 : 0) || a.next.localeCompare(b.next)).slice(0, limit)
 }
+
+// ---- /jobs on the pager ------------------------------------------------------------
+
+// A job this close to expiring (or this recently expired) gets an Extend button.
+export const EXPIRING_SOON_DAYS = 14
+const DAY_MS = 24 * 60 * 60_000
+
+function daysUntil(expires: string, now: Date): number {
+  const today = new Date(now.getFullYear(), now.getMonth(), now.getDate())
+  return Math.round((parseDate(expires).getTime() - today.getTime()) / DAY_MS)
+}
+
+function inDays(n: number): string {
+  if (n === 0) return "today"
+  if (n === 1) return "tomorrow"
+  if (n > 0) return `in ${n} days`
+  return n === -1 ? "yesterday" : `${-n} days ago`
+}
+
+const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
+
+// "Nov 6", with the year only when it isn't this year's.
+export function shortDate(d: Date, now: Date): string {
+  const md = `${MONTHS[d.getMonth()]} ${d.getDate()}`
+  return d.getFullYear() === now.getFullYear() ? md : `${md}, ${d.getFullYear()}`
+}
+
+// "tonight 22:00", "tomorrow 08:00", "Nov 6 08:00".
+export function shortTime(value: string, now: Date): string {
+  const d = parseWhen(value)
+  const time = value.trim().split(" ")[1]!
+  const days = Math.round((new Date(d.getFullYear(), d.getMonth(), d.getDate()).getTime() - new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime()) / DAY_MS)
+  if (days === 0) return `${d.getHours() >= 18 ? "tonight" : "today"} ${time}`
+  if (days === 1) return `tomorrow ${time}`
+  return `${shortDate(d, now)} ${time}`
+}
+
+// The schedule as it reads aloud: "daily 22:00", "every 30m", "once, Nov 6 09:00".
+function shortSchedule(h: Record<string, string>, now: Date): string {
+  if (h.when) return `once, ${shortTime(h.when, now)}`
+  return /^\d/.test(h.every!) ? `every ${h.every}` : h.every!
+}
+
+// The station's jobs, soonest expiry first, in short lines that fit a phone:
+// the name alone in bold, then when it expires, then how it runs. Finished
+// one-shots and long-expired jobs are counted, not listed, so the reply stays
+// short as history builds up.
+export async function jobsReply(opts: { now?: Date; pull?: boolean } = {}): Promise<BeaconReply> {
+  const now = opts.now ?? new Date()
+  const sources = listSources(loadConfig()).filter((s) => existsSync(s.path))
+  const multi = sources.length > 1
+  const rows: { block: string; days: number; buttons: BeaconButton[] }[] = []
+  let hidden = 0
+  const problems: string[] = []
+  for (const s of sources) {
+    if (s.git && opts.pull !== false) {
+      const e = await gitPull(s.path)
+      if (e) problems.push(`${s.name}: ${e}`)
+    }
+    for (const j of loadJobs(s)) {
+      const h = j.header
+      if (j.problems.length) {
+        problems.push(`${s.name}/${h.id ?? j.name} is invalid: ${j.problems.join("; ")}`)
+        continue
+      }
+      let next: string
+      try {
+        next = nextRunText(h, loadState(s, h.id!), now)
+      } catch {
+        next = "?"
+      }
+      const days = daysUntil(h.expires!, now)
+      if (next === "done" || (next === "expired" && days < -EXPIRING_SOON_DAYS)) {
+        hidden++
+        continue
+      }
+      const name = multi ? `${s.name}/${h.id}` : h.id!
+      const soon = days <= EXPIRING_SOON_DAYS
+      const kind = h.kind === "agent" ? `agent/${h.agent || "claude"}` : h.kind!
+      const date = shortDate(parseDate(h.expires!), now)
+      const expiry = next === "expired" ? `Expired ${date} (${inDays(days)})` : `Expires ${date}${soon ? ` (${inDays(days)})` : ""}`
+      const nextText = !h.when && /^\d{4}-/.test(next) ? ` · next ${shortTime(next, now)}` : ""
+      const block = [`${soon ? "⚠️ " : ""}<b>${escapeHtml(name)}</b>`, expiry, escapeHtml(`${kind} · ${shortSchedule(h, now)}${nextText}`)].join("\n")
+      const extendable = soon && !h.when
+      const status = loadState(s, h.id!).last_status
+      const again = !!status && RUN_AGAIN_STATUSES.has(status) && next !== "expired"
+      const buttons = [
+        ...(extendable ? [{ text: `Extend ${name} 1 year`, action: "scheduler.extend", payload: { source: s.name, id: h.id!, from: h.expires! } }] : []),
+        ...(again ? [runAgainButton(s.name, h.id!)] : []),
+      ]
+      rows.push({ block: again ? `${block}\nLast run: ${escapeHtml(status!)}` : block, days, buttons })
+    }
+  }
+  rows.sort((a, b) => a.days - b.days)
+  const station = escapeHtml(stationName())
+  const head = rows.length ? `<b>Scheduled jobs on ${station}</b> (${rows.length})` : `No scheduled jobs on ${station}.`
+  const tail: string[] = []
+  if (hidden) tail.push(`${hidden} finished or long-expired not shown.`)
+  for (const p of problems) tail.push(`⚠️ ${escapeHtml(p)}`)
+  const text = [head, ...rows.map((r) => r.block), ...(tail.length ? [tail.join("\n")] : [])].join("\n\n")
+  const buttons = rows.flatMap((r) => r.buttons.map((b) => [b]))
+  return { text, html: true, ...(buttons.length ? { buttons } : {}) }
+}
+
+const RUN_AGAIN_STATUSES = new Set(["failed", "blocked", "timeout", "error"])
+
+// The Run again button. A job can take half an hour and the listener answers one
+// update at a time, so it's started apart and pages its result itself. It runs
+// through `scheduler run`, the same runner with the same allow list and sandbox.
+export async function runAgainReply(payload: Record<string, string>): Promise<BeaconReply> {
+  const { source, id } = payload
+  if (!source || !id) return { text: "That button is missing its job; send /jobs again." }
+  const s = findSource(loadConfig(), source)
+  if (!s) return { text: `There's no Scheduler source named "${source}" on this station any more.` }
+  if (!loadJobs(s).some((j) => j.header.id === id)) return { text: `Job ${id} isn't in ${source} any more.` }
+  // `scheduler run` needs the tick lock. Buttons work once, so offer a fresh one.
+  if (tickRunning()) return { text: `A tick is running, so ${id} can't start yet.`, buttons: [[runAgainButton(source, id)]] }
+  spawnDetached([...selfCommand(), "scheduler", "run", id, "--source", source], { cwd: moduleStateDir("scheduler"), env: jobEnv() })
+  return { text: `▶️ Running ${id} again. Its result will page here.` }
+}
+
+// The "Extend 1 year" button: a year past the expiry the reply showed, so a
+// stale button can't stack a second year onto one that was already extended.
+export async function extendReply(payload: Record<string, string>, now = new Date()): Promise<BeaconReply> {
+  const { source, id, from } = payload
+  if (!source || !id || !from) return { text: "That button is missing its job; send /jobs again." }
+  const s = findSource(loadConfig(), source)
+  if (!s) return { text: `There's no Scheduler source named "${source}" on this station any more.` }
+  const today = new Date(now.getFullYear(), now.getMonth(), now.getDate())
+  const base = parseDate(from) < today ? today : parseDate(from)
+  const r = await extendJob(s, id, { date: fmtDate(addPeriod(base, "1y")) }, { now })
+  if (!r.ok) return { text: `Couldn't extend ${id}: ${r.message}${r.remedy ? `\n${r.remedy}` : ""}` }
+  return { text: `✅ Extended ${id}: it now expires ${r.to} (was ${r.from})${r.pushed ? ", pushed" : ""}.` }
+}

package/src/modules/sentinel.ts

@@ -0,0 +1,90 @@
+// Sentinel's module declaration: watches the fleet, works out what broke, writes
+// it up. Its status is cheap and sends nothing: Scheduler asks every tick.
+
+import { existsSync } from "node:fs"
+import { listProfiles, loadConfig } from "../config.ts"
+import { beaconFromConfig, escapeHtml } from "../lib/beacon.ts"
+import { hasIncidentMonitors } from "../lib/cliIncidents.ts"
+import { waitingRows } from "../lib/sentinelReply.ts"
+import { isWaiting, loadIncident, loadIncidents, loadPause, loadTickState, saveIncident, sentinelSettings } from "../lib/sentinelState.ts"
+import { moduleStateDir, moduleWorkDir, type BeaconReply, type ModuleCheck, type ModuleDefinition, type ModuleStatus } from "./contract.ts"
+
+const TICK_STALE_MS = 15 * 60_000
+const STUCK_MS = 24 * 60 * 60_000
+
+export const sentinelModule: ModuleDefinition = {
+  id: "sentinel",
+  name: "Sentinel",
+  summary: "Watches the fleet, works out what actually broke, and writes it up.",
+  dependsOn: ["beacon", "scheduler"],
+  configKey: "sentinel",
+  stateDir: () => moduleStateDir("sentinel"),
+  workDir: () => moduleWorkDir("sentinel"),
+  jobs: [
+    { id: "sentinel.tick", every: "5m", command: ["sentinel", "tick"] },
+    { id: "sentinel.digest", at: "08:00", command: ["sentinel", "digest"] },
+  ],
+  commands: [{ command: "incidents", describe: "open incidents and anything waiting on you", run: () => incidentsReply() }],
+  actions: [{ action: "sentinel.diagnose", run: (payload) => diagnoseReply(payload) }],
+  async status(): Promise<ModuleStatus> {
+    const cfg = loadConfig()
+    const s = sentinelSettings(cfg)
+    if (!s.enabled) {
+      return { id: "sentinel", state: "not_configured", summary: "Not set up", checks: [], remedy: "spinuptui sentinel setup --plan" }
+    }
+    const checks: ModuleCheck[] = []
+    const domains = Object.entries(cfg.kumaMonitors).filter(([, r]) => hasIncidentMonitors(r)).length
+    checks.push({ name: "watching", ok: domains > 0, detail: domains ? `${domains} domain${domains === 1 ? "" : "s"}` : "no monitors registered: run spinuptui sentinel attach --plan" })
+    const t = loadTickState()
+    const age = t.lastTickAt ? Date.now() - Date.parse(t.lastTickAt) : null
+    const stale = age === null || age > TICK_STALE_MS
+    checks.push({
+      name: "last tick",
+      ok: !t.blindSince && !stale,
+      detail: t.blindSince ? `blind since ${t.blindSince}: ${t.lastError}` : age === null ? "none yet" : `${Math.round(age / 60_000)} min ago${t.lastError ? ` (${t.lastError})` : ""}`,
+    })
+    const incidents = loadIncidents()
+    const now = Date.now()
+    const waiting = incidents.filter(isWaiting)
+    const stuck = waiting.filter((i) => now - Date.parse(i.createdAt) > STUCK_MS)
+    const count = (state: string) => waiting.filter((i) => i.state === state).length
+    const queue = [["queued", count("queued")], ["diagnosing", count("diagnosing")], ["waiting on login", count("waiting_agent")], ["blocked", count("blocked")], ["failed", count("failed")]].filter(([, n]) => n)
+    checks.push({ name: "queue", ok: stuck.length === 0, detail: queue.length ? queue.map(([k, n]) => `${n} ${k}`).join(", ") : "empty" })
+    const agentBin = s.agent ? Bun.which(s.agent, { PATH: `${process.env.HOME}/.local/bin:${process.env.HOME}/.bun/bin:/opt/homebrew/bin:/usr/local/bin:${process.env.PATH}` }) : null
+    checks.push({ name: "agent", ok: !s.agent || !!agentBin, detail: !s.agent ? "none: incidents get a write-up of the evidence, not a diagnosis" : agentBin ? s.agent : `${s.agent} isn't on the PATH` })
+    checks.push({ name: "beacon", ok: beaconFromConfig(cfg).channels.length > 0, detail: s.channel ?? "default channel" })
+    if (s.reportTo) checks.push({ name: "board", ok: !!cfg.beacon?.boards?.[s.reportTo], detail: cfg.beacon?.boards?.[s.reportTo] ? s.reportTo : `no Beacon board named ${s.reportTo}` })
+    const pause = loadPause()
+    const paused = pause && Date.parse(pause.until) > now
+    checks.push({ name: "mode", ok: true, detail: paused ? `paused until ${new Date(pause.until).toLocaleString()}` : s.shadow ? "shadow: pages nothing, runs no agent" : "live" })
+    checks.push({ name: "workspace", ok: existsSync(s.workspace) || s.workspace === moduleWorkDir("sentinel"), detail: s.workspace })
+    const others = listProfiles().filter((p) => !p.active).length
+    if (others) checks.push({ name: "accounts", ok: true, detail: `watching the active account only; ${others} other account${others === 1 ? " is" : "s are"} not covered` })
+    const failing = checks.find((c) => !c.ok)
+    return {
+      id: "sentinel",
+      state: failing ? "degraded" : "ok",
+      summary: failing ? `${failing.name}: ${failing.detail}` : paused ? "paused" : s.shadow ? `shadow, ${domains} domains` : `watching ${domains} domain${domains === 1 ? "" : "s"}`,
+      checks,
+      ...(failing ? { remedy: failing.name === "watching" ? "spinuptui sentinel attach --plan" : "spinuptui sentinel status" } : {}),
+    }
+  },
+}
+
+export async function incidentsReply(): Promise<BeaconReply> {
+  const rows = waitingRows(loadIncidents())
+  if (!rows.length) return { text: "No open incidents." }
+  const text = [`<b>Incidents</b> (${rows.length})`, ...rows.map((r) => escapeHtml(r.line))].join("\n\n")
+  const buttons = rows.filter((r) => r.button).map((r) => [r.button!])
+  return { text, html: true, ...(buttons.length ? { buttons } : {}) }
+}
+
+// A tap only queues: the next tick starts the worker, so it can't run anything
+// the tick wouldn't, and it can't add permissions.
+export async function diagnoseReply(payload: Record<string, string>): Promise<BeaconReply> {
+  const i = payload.id ? loadIncident(payload.id) : null
+  if (!i) return { text: "That incident is gone; send /incidents again." }
+  if (!["recurrence", "blocked", "failed"].includes(i.state)) return { text: `${i.id} is ${i.state} already.` }
+  saveIncident({ ...i, state: "queued", attempts: 0, why: undefined })
+  return { text: `▶️ Queued ${i.events[0]!.domain}. The next tick starts it, within 5 minutes.` }
+}