spinuptui 1.3.0-alpha.4 → 1.3.0-alpha.5

package/src/lib/cliBeacon.ts

@@ -20,6 +20,7 @@ import {
   setupChannelName,
   verifyBotToken,
 } from "./beaconSetup.ts"
+import { boardLogin } from "./beaconBoard.ts"
 import { confirm, PromptCancelled, readLine } from "./prompt.ts"
 import { listen, routesFrom } from "./beaconListen.ts"
 import {
@@ -37,7 +38,7 @@ import { beaconModule } from "../modules/beacon.ts"
 import { allModuleStatuses, UPCOMING_MODULES } from "../modules/index.ts"
 import type { ModuleStatus } from "../modules/contract.ts"
 
-const VALUE_FLAGS = new Set(["--to", "--env-file", "--bin"])
+const VALUE_FLAGS = new Set(["--to", "--env-file", "--bin", "--project", "--mention"])
 
 function parse(list: string[]): { positionals: string[]; flags: Map<string, string>; rest: string | null } {
   const dash = list.indexOf("--")
@@ -71,6 +72,8 @@ const USAGE = `Usage:
   spinuptui beacon setup [<name>]   (guided, in a terminal)
   spinuptui beacon send [--to <channel>] [--silent] [--json] [<message> | -]
   spinuptui beacon add telegram <name> [--env-file <path>] [--default] [--no-test]
+  spinuptui beacon add basecamp <name> --project <id> [--mention "<person>"]
+                          a team message board: the \`basecamp\` CLI must be logged in
   spinuptui beacon remove <name>
   spinuptui beacon test [--to <channel>] [--json]
   spinuptui beacon status [--json]
@@ -138,8 +141,22 @@ async function test(flags: Map<string, string>, json: boolean): Promise<number>
 
 const NAME_RE = /^[a-z0-9][a-z0-9-]{0,31}$/
 
+async function addBoard(name: string, flags: Map<string, string>, json: boolean): Promise<number> {
+  const projectId = flags.get("--project") ?? ""
+  if (!/^\d+$/.test(projectId)) return fail(json, "usage", `--project needs the Basecamp project's numeric id.\n${USAGE}`)
+  const login = await boardLogin()
+  if (!login.ok) return fail(json, "basecamp_login", `Can't reach Basecamp: ${login.detail}.`)
+  const mention = flags.get("--mention")
+  const stored: BeaconStored = loadConfig().beacon ?? {}
+  await saveConfig({ beacon: { ...stored, boards: { ...(stored.boards ?? {}), [name]: { type: "basecamp", projectId, audience: "team", ...(mention ? { mention } : {}) } } } })
+  if (json) console.log(JSON.stringify({ ok: true, board: name, projectId }))
+  else console.log(`Added team board "${name}" (Basecamp project ${projectId}), posting as ${login.detail}.`)
+  return 0
+}
+
 async function add(pos: string[], flags: Map<string, string>, json: boolean): Promise<number> {
   const [type, name] = pos
+  if (type === "basecamp" && name && NAME_RE.test(name)) return addBoard(name, flags, json)
   if (type !== "telegram" || !name) return fail(json, "usage", USAGE)
   if (!NAME_RE.test(name)) return fail(json, "usage", `Channel names are lowercase letters, digits and dashes (got "${name}").`)
   if (name === "env") return fail(json, "usage", `"env" is reserved for the channel defined by SPINUPTUI_BEACON_* environment variables.`)
@@ -267,6 +284,13 @@ async function remove(pos: string[], json: boolean): Promise<number> {
   const name = pos[0]
   if (!name) return fail(json, "usage", USAGE)
   const stored: BeaconStored = loadConfig().beacon ?? {}
+  if (stored.boards?.[name]) {
+    const { [name]: _board, ...boards } = stored.boards
+    await saveConfig({ beacon: { ...stored, boards } })
+    if (json) console.log(JSON.stringify({ ok: true, removed: name }))
+    else console.log(`Removed board "${name}".`)
+    return 0
+  }
   if (!stored.channels?.[name]) {
     const resolved = pickChannel(beaconFromConfig(loadConfig()), name)
     if (!("ok" in resolved) && resolved.env) {

package/src/lib/cliIncidents.ts

@@ -18,10 +18,14 @@ export interface IncidentEvent {
   status: IncidentStatus
   time: string
   message: string
+  monitorId: number
 }
 
 export type IncidentsResult =
-  | { ok: true; windowHours: number; events: IncidentEvent[] }
+  // failedMonitors: monitors whose history couldn't be read this time, so a
+  // caller doesn't mistake their silence for "no events". kumaAlerted: with
+  // `wiring`, the monitors Kuma itself sends a notification for.
+  | { ok: true; windowHours: number; events: IncidentEvent[]; failedMonitors: string[]; kumaAlerted?: number[] }
   | { ok: false; reason: "kuma_not_configured" | "no_monitors_registered" | "kuma_error"; message: string }
 
 const STATUS_LABEL: Record<number, IncidentStatus> = { 0: "down", 1: "up", 2: "pending", 3: "maintenance" }
@@ -50,7 +54,7 @@ interface Target {
 
 export async function resolveIncidents(
   cfg: AppConfig,
-  opts: { domain?: string; hours: number },
+  opts: { domain?: string; hours: number; wiring?: boolean },
 ): Promise<IncidentsResult> {
   if (!cfg.uptimeKuma) {
     return {
@@ -83,11 +87,13 @@ export async function resolveIncidents(
 
   try {
     const { result } = await withKuma(cfg.uptimeKuma, async (kuma) => {
+      const failedMonitors: string[] = []
       const perTarget = await Promise.all(
         targets.map(async (t) => {
           try {
             return { t, beats: await kuma.getMonitorBeats(t.id, opts.hours) }
           } catch {
+            failedMonitors.push(`${t.domain} ${t.kind}`)
             return { t, beats: [] as KumaBeat[] }
           }
         }),
@@ -115,17 +121,18 @@ export async function resolveIncidents(
           const looksLikeDomainList = domainList.every((d) => /^[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)+$/.test(d))
           if (t.kind === "fatal" && status === "down" && looksLikeDomainList) {
             for (const domain of domainList) {
-              events.push({ domain, kind: t.kind, status, time: b.time, message })
+              events.push({ domain, kind: t.kind, status, time: b.time, message, monitorId: t.id })
             }
             continue
           }
-          events.push({ domain: t.domain, kind: t.kind, status, time: b.time, message })
+          events.push({ domain: t.domain, kind: t.kind, status, time: b.time, message, monitorId: t.id })
         }
       }
       events.sort((a, b) => a.time.localeCompare(b.time))
-      return events
+      const kumaAlerted = opts.wiring ? (await kuma.getMonitors()).filter((m) => Object.keys((m.notificationIDList as object | null) ?? {}).length > 0).map((m) => m.id) : undefined
+      return { events, failedMonitors, kumaAlerted }
     })
-    return { ok: true, windowHours: opts.hours, events: result }
+    return { ok: true, windowHours: opts.hours, ...result }
   } catch (err) {
     const message = err instanceof KumaError ? err.message : `Unexpected error: ${(err as Error).message}`
     return { ok: false, reason: "kuma_error", message }

package/src/lib/cliSentinel.ts

@@ -0,0 +1,451 @@
+// `spinuptui sentinel …` — the CLI over Sentinel. Headless like every module
+// command: --json prints one result object on stdout, progress goes to stderr,
+// failures use the { ok: false, reason, message, remedy? } shape and exit 1.
+
+import { loadConfig, saveConfig, type SentinelStored } from "../config.ts"
+import { SpinupWPClient } from "../api/client.ts"
+import { moduleWorkDir } from "../modules/contract.ts"
+import { sentinelModule } from "../modules/sentinel.ts"
+import { BLOCKED_EXIT } from "./schedulerRun.ts"
+import { postToBoard } from "./beaconBoard.ts"
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+import { applyAttach, planAttach, type AttachPlan } from "./sentinelAttach.ts"
+import { composeDigest, expectDeps, gatherDigest, recordDigest } from "./sentinelDigest.ts"
+import { EXPECT_TYPES, loadExpectations, runExpectation, saveExpectations, validateExpectation, type Expectation } from "./sentinelExpect.ts"
+import { describeIncident } from "./sentinelReply.ts"
+import { clearPause, deleteIncident, loadFlags, loadIncident, loadIncidents, loadPause, loadSeen, loadTickState, logShadow, saveFlags, saveIncident, savePause, saveSeen, saveTickState, sentinelSettings, type Seen } from "./sentinelState.ts"
+import { eventMs } from "./sentinelIncidents.ts"
+import { realDeps, sendPage, sentinelTick } from "./sentinelTick.ts"
+import { realWorkerDeps, runWorker } from "./sentinelWorker.ts"
+import { withKuma, type KumaMonitor } from "./uptimeKuma.ts"
+
+const VALUE_FLAGS = new Set(["--for", "--since", "--agent", "--workspace", "--report-to", "--type", "--domain", "--server", "--match", "--min", "--threshold", "--host", "--paths", "--log-glob", "--url", "--expect", "--label", "--note", "--expires"])
+
+const USAGE = `Usage:
+  spinuptui sentinel setup [--plan | --apply] [--shadow] [--agent claude|codex|none]
+                           [--workspace <dir>] [--report-to <board>]
+  spinuptui sentinel attach --plan | --apply [--include-likely]
+                           adopt an existing Kuma's monitors; read-only toward Kuma
+  spinuptui sentinel status [--json]
+  spinuptui sentinel tick [--dry-run]          one check of Kuma (Scheduler runs this)
+  spinuptui sentinel digest [--dry-run]        the daily digest (Scheduler runs this)
+  spinuptui sentinel list [--open] [--json]
+  spinuptui sentinel show <incident>
+  spinuptui sentinel diagnose <incident>       queue it; the next tick starts it
+  spinuptui sentinel post <incident>           post its report to the board
+  spinuptui sentinel pause --for 2h            stop acting on events, up to 7d
+  spinuptui sentinel resume
+  spinuptui sentinel go-live --since "YYYY-MM-DD HH:MM:SS"
+                           leave shadow mode; events after that UTC time are new
+  spinuptui sentinel expect list | check <id> | remove <id> | ack <id>
+  spinuptui sentinel expect add <id> --type ${EXPECT_TYPES.join("|")} [--domain <d>] [--label <l>] [--note <n>] [--expires YYYY-MM-DD]
+      log-count:  --match <text> [--min N] [--trend]
+      load-shape: --threshold N
+      cache-hit:  --host <canonical host> --paths /,/?one=param
+      warmer:     --log-glob <path>
+      status:     --url <url> --expect <code>`
+
+interface Parsed {
+  positionals: string[]
+  flags: Map<string, string>
+}
+
+function parse(list: string[]): Parsed {
+  const positionals: string[] = []
+  const flags = new Map<string, string>()
+  for (let i = 0; i < list.length; i++) {
+    const a = list[i]!
+    if (!a.startsWith("--")) positionals.push(a)
+    else if (VALUE_FLAGS.has(a)) flags.set(a, list[++i] ?? "")
+    else flags.set(a, "")
+  }
+  return { positionals, flags }
+}
+
+function fail(json: boolean, reason: string, message: string, remedy?: string): number {
+  if (json) console.log(JSON.stringify({ ok: false, reason, message, ...(remedy ? { remedy } : {}) }))
+  else {
+    console.error(message)
+    if (remedy) console.error(remedy)
+  }
+  return 1
+}
+
+const say = (line: string) => process.stderr.write(`→ ${line}\n`)
+const out = (json: boolean, value: object, text: string) => console.log(json ? JSON.stringify(value) : text)
+
+export async function runSentinel(argv: string[]): Promise<number> {
+  const p = parse(argv)
+  const json = p.flags.has("--json")
+  const [sub, ...rest] = p.positionals
+  switch (sub) {
+    case "setup":
+      return setup(p, json)
+    case "attach":
+      return attach(p, json)
+    case "status":
+      return status(json)
+    case "tick":
+      return tickCmd(p, json)
+    case "worker":
+      return workerCmd()
+    case "digest":
+      return digestCmd(p, json)
+    case "list":
+      return list(p, json)
+    case "show":
+      return show(rest)
+    case "diagnose":
+      return diagnose(rest, json)
+    case "post":
+      return post(rest, p, json)
+    case "pause":
+      return pause(p, json)
+    case "resume":
+      return resume(json)
+    case "go-live":
+      return goLive(p, json)
+    case "expect":
+      return expect(rest, p, json)
+  }
+  return fail(json, "usage", USAGE)
+}
+
+// ---- setup --------------------------------------------------------------------------
+
+interface SetupPlan {
+  blockers: string[]
+  next: SentinelStored
+  notes: string[]
+  watching: number
+}
+
+export function planSetup(p: { shadow?: boolean; agent?: string; workspace?: string; reportTo?: string }): SetupPlan {
+  const cfg = loadConfig()
+  const blockers: string[] = []
+  const notes: string[] = []
+  if (!cfg.beacon?.channels || !Object.keys(cfg.beacon.channels).length) blockers.push("Beacon has no channel: spinuptui beacon setup")
+  if (!cfg.scheduler?.dispatcher) blockers.push("Scheduler's dispatcher isn't installed: spinuptui scheduler install --plan")
+  if (!cfg.uptimeKuma) blockers.push("No Uptime Kuma connection: press M on a site")
+  const watching = Object.keys(cfg.kumaMonitors).length
+  if (!watching) notes.push("No monitors are registered yet: run spinuptui sentinel attach --plan.")
+  const found = (["claude", "codex"] as const).find((a) => Bun.which(a))
+  const agent = p.agent === "none" ? null : ((p.agent as "claude" | "codex" | undefined) ?? cfg.sentinel?.diagnosis?.agent ?? found ?? null)
+  if (agent === "codex") notes.push("Codex is unproven here: its read-only sandbox blocks the network that ssh-exec needs.")
+  if (!agent) notes.push("No agent: incidents get a write-up of the evidence, not a diagnosis.")
+  const reportTo = p.reportTo ?? cfg.sentinel?.reportTo ?? null
+  if (reportTo && !cfg.beacon?.boards?.[reportTo]) blockers.push(`No Beacon board named "${reportTo}": spinuptui beacon add basecamp ${reportTo} --project <id>`)
+  const next: SentinelStored = {
+    ...cfg.sentinel,
+    enabled: true,
+    mode: p.shadow ? "shadow" : (cfg.sentinel?.mode ?? "live"),
+    reportTo,
+    ...(p.workspace ? { workspace: p.workspace } : {}),
+    diagnosis: { ...cfg.sentinel?.diagnosis, agent },
+  }
+  return { blockers, next, notes, watching }
+}
+
+function setupText(plan: SetupPlan): string {
+  const n = plan.next
+  return [
+    `Sentinel would watch ${plan.watching} domain(s), every 5 minutes, with a digest at 08:00, as Scheduler jobs.`,
+    `  mode:      ${n.mode}`,
+    `  agent:     ${n.diagnosis?.agent ?? "none"}`,
+    `  workspace: ${n.workspace ?? join("~/SpinupTUI", "sentinel")}`,
+    `  board:     ${n.reportTo ?? "none"}`,
+    ...plan.notes.map((x) => `Note: ${x}`),
+    ...plan.blockers.map((x) => `Blocked: ${x}`),
+  ].join("\n")
+}
+
+// Saves the settings and proves them with one real tick. The TUI uses this too.
+export async function applySetup(next: SentinelStored) {
+  await saveConfig({ sentinel: next })
+  return tick(false)
+}
+
+async function setup(p: Parsed, json: boolean): Promise<number> {
+  const plan = planSetup({ shadow: p.flags.has("--shadow"), agent: p.flags.get("--agent"), workspace: p.flags.get("--workspace"), reportTo: p.flags.get("--report-to") })
+  if (!p.flags.has("--apply")) {
+    if (!p.flags.has("--plan") && !json) console.log("Nothing applied. Review this, then run with --apply.\n")
+    out(json, { ok: true, ...plan }, setupText(plan))
+    return 0
+  }
+  if (plan.blockers.length) return fail(json, "not_ready", plan.blockers.join("\n"))
+  say("Running one real tick")
+  const r = await applySetup(plan.next)
+  if (!r.ok) return fail(json, r.reason, r.message)
+  out(json, r, r.firstRun ? "Sentinel is on: it read Kuma and starts from now." : `Sentinel is on: ${r.events} new event(s).`)
+  return 0
+}
+
+// ---- attach ---------------------------------------------------------------------------
+
+function attachText(a: AttachPlan, includeLikely: boolean): string {
+  const lines = [`${a.exact.length} monitor(s) match by name and would attach; ${a.already} already attached.`]
+  if (a.likely.length) lines.push(`${a.likely.length} likely match(es)${includeLikely ? "" : ", attached only with --include-likely"}:`, ...a.likely.map((m) => `  ${m.name} → ${m.domain}`))
+  if (a.unmonitored.length) lines.push(`${a.unmonitored.length} site(s) have no monitor: ${a.unmonitored.slice(0, 10).join(", ")}${a.unmonitored.length > 10 ? ", …" : ""}`)
+  if (a.strangers.length) lines.push(`${a.strangers.length} Kuma monitor(s) match no site and are ignored.`)
+  return lines.join("\n")
+}
+
+// What attach would do: reads Kuma and the fleet, changes nothing.
+export async function loadAttachPlan(): Promise<{ plan: AttachPlan; monitors: KumaMonitor[] }> {
+  const cfg = loadConfig()
+  const monitors = (await withKuma(cfg.uptimeKuma!, (k) => k.getMonitors())).result
+  const sites = await new SpinupWPClient(cfg).listSites()
+  const domains = sites.flatMap((s) => [s.domain, ...(s.additional_domains ?? []).map((d) => d.domain)])
+  return { plan: planAttach(monitors, domains, cfg.kumaMonitors, sites.map((s) => s.domain)), monitors }
+}
+
+export async function saveAttach(plan: AttachPlan, monitors: KumaMonitor[], includeLikely: boolean): Promise<number> {
+  const matches = [...plan.exact, ...(includeLikely ? plan.likely : [])]
+  await saveConfig({ kumaMonitors: applyAttach(loadConfig().kumaMonitors, matches, monitors) })
+  return matches.length
+}
+
+async function attach(p: Parsed, json: boolean): Promise<number> {
+  if (!loadConfig().uptimeKuma) return fail(json, "kuma_not_configured", "No Uptime Kuma connection configured in SpinupTUI.")
+  const includeLikely = p.flags.has("--include-likely")
+  let loaded: Awaited<ReturnType<typeof loadAttachPlan>>
+  try {
+    loaded = await loadAttachPlan()
+  } catch (err) {
+    return fail(json, "kuma_error", (err as Error).message)
+  }
+  const { plan, monitors } = loaded
+  if (!p.flags.has("--apply")) {
+    out(json, { ok: true, ...plan }, attachText(plan, includeLikely))
+    return 0
+  }
+  const attached = await saveAttach(plan, monitors, includeLikely)
+  out(json, { ok: true, attached }, `Attached ${attached} monitor(s). Nothing was changed in Kuma.`)
+  return 0
+}
+
+// ---- ticks ----------------------------------------------------------------------------
+
+async function tick(dryRun: boolean) {
+  const cfg = loadConfig()
+  const client = new SpinupWPClient(cfg)
+  return sentinelTick(sentinelSettings(cfg), realDeps(cfg, client), { dryRun })
+}
+
+async function tickCmd(p: Parsed, json: boolean): Promise<number> {
+  const s = sentinelSettings(loadConfig())
+  const dryRun = p.flags.has("--dry-run")
+  if (!s.enabled && !dryRun) return fail(json, "not_configured", "Sentinel isn't set up.", "spinuptui sentinel setup --plan")
+  const r = await tick(dryRun)
+  console.log(JSON.stringify(r))
+  if (r.ok) return 0
+  return r.needsPerson ? BLOCKED_EXIT : 1
+}
+
+async function workerCmd(): Promise<number> {
+  const cfg = loadConfig()
+  const s = sentinelSettings(cfg)
+  if (!s.enabled || s.shadow) return 0
+  await runWorker(s, realWorkerDeps(cfg, s))
+  return 0
+}
+
+async function digestCmd(p: Parsed, json: boolean): Promise<number> {
+  const cfg = loadConfig()
+  const s = sentinelSettings(cfg)
+  const dryRun = p.flags.has("--dry-run")
+  if (!s.enabled && !dryRun) return fail(json, "not_configured", "Sentinel isn't set up.", "spinuptui sentinel setup --plan")
+  const data = await gatherDigest(s, cfg, new SpinupWPClient(cfg), { dryRun })
+  const digest = composeDigest(data, loadFlags())
+  if (dryRun) {
+    console.log(digest.text)
+    return 0
+  }
+  if (s.shadow) {
+    logShadow(`would send the digest:\n${digest.text}`)
+    return 0
+  }
+  recordDigest(digest, data)
+  const sent = await sendPage(cfg, { text: digest.text, silent: digest.silent }, s.channel)
+  if (sent === "sent") return 0
+  console.error(`The digest wasn't sent: ${sent}`)
+  return 1
+}
+
+// ---- incidents ------------------------------------------------------------------------
+
+function list(p: Parsed, json: boolean): number {
+  const open = p.flags.has("--open")
+  const rows = loadIncidents().filter((i) => !open || !i.closedAt)
+  if (json) console.log(JSON.stringify({ ok: true, incidents: rows }))
+  else if (!rows.length) console.log("No incidents.")
+  else for (const i of rows) console.log(`${i.id}  ${describeIncident(i)}`)
+  return 0
+}
+
+function show(pos: string[]): number {
+  const i = pos[0] ? loadIncident(pos[0]) : null
+  if (!i) return fail(false, "unknown_incident", `No incident "${pos[0] ?? ""}".`)
+  const { evidence, ...rest } = i
+  console.log(JSON.stringify(rest, null, 2))
+  if (evidence) console.log(`\n${evidence}`)
+  return 0
+}
+
+function diagnose(pos: string[], json: boolean): number {
+  const i = pos[0] ? loadIncident(pos[0]) : null
+  if (!i) return fail(json, "unknown_incident", `No incident "${pos[0] ?? ""}".`)
+  saveIncident({ ...i, state: "queued", attempts: 0, why: undefined })
+  out(json, { ok: true, id: i.id }, `Queued ${i.id}. The next tick starts it.`)
+  return 0
+}
+
+async function post(pos: string[], p: Parsed, json: boolean): Promise<number> {
+  const cfg = loadConfig()
+  const s = sentinelSettings(cfg)
+  const i = pos[0] ? loadIncident(pos[0]) : null
+  if (!i) return fail(json, "unknown_incident", `No incident "${pos[0] ?? ""}".`)
+  if (!i.reportPath) return fail(json, "no_report", `${i.id} has no report yet.`)
+  if (i.post === "posted") return fail(json, "already_posted", `Already posted: ${i.postUrl}`)
+  const board = s.reportTo ? cfg.beacon?.boards?.[s.reportTo] : undefined
+  if (!board) return fail(json, "no_board", "No board is configured.", "spinuptui sentinel setup --report-to <board> --apply")
+  saveIncident({ ...i, post: "posting" })
+  const r = await postToBoard(board, readFileSync(join(s.workspace, i.reportPath), "utf8"))
+  if (!r.ok) {
+    saveIncident({ ...i, post: "failed", postError: r.message })
+    return fail(json, r.reason, r.message)
+  }
+  saveIncident({ ...i, post: "posted", postId: r.id, postUrl: r.url, postError: undefined })
+  out(json, { ok: true, url: r.url }, `Posted: ${r.url}`)
+  return 0
+}
+
+// ---- pause and go-live -------------------------------------------------------------------
+
+const MAX_PAUSE_MS = 7 * 24 * 3_600_000
+
+export function pauseMs(value: string): number | null {
+  const m = /^(\d+)([mhd])$/.exec(value.trim())
+  if (!m || !+m[1]!) return null
+  const ms = +m[1]! * { m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2] as "m" | "h" | "d"]
+  return ms <= MAX_PAUSE_MS ? ms : null
+}
+
+function pause(p: Parsed, json: boolean): number {
+  const ms = pauseMs(p.flags.get("--for") ?? "")
+  if (ms === null) return fail(json, "usage", "--for is required, up to 7d: 30m, 2h, 3d.")
+  const until = new Date(Date.now() + ms)
+  savePause({ until: until.toISOString() })
+  out(json, { ok: true, until: until.toISOString() }, `Sentinel is paused until ${until.toLocaleString()}. It resumes by itself.`)
+  return 0
+}
+
+async function resume(json: boolean): Promise<number> {
+  const was = loadPause()
+  clearPause()
+  out(json, { ok: true, wasPaused: !!was }, was ? `Sentinel is watching again. ${was.events ?? 0} event(s) passed during the pause.` : "Sentinel wasn't paused.")
+  return 0
+}
+
+// Leaves shadow mode at the exact point the old watcher stopped: everything up to
+// `since` was its, everything after is Sentinel's. Shadow records go, and any
+// event after that time is forgotten so the next tick treats it as new.
+async function goLive(p: Parsed, json: boolean): Promise<number> {
+  const since = p.flags.get("--since") ?? ""
+  if (!/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/.test(since)) return fail(json, "usage", '--since "YYYY-MM-DD HH:MM:SS" is required, in UTC: the old watcher\'s lastEventTime.')
+  const cfg = loadConfig()
+  if (!cfg.sentinel?.enabled) return fail(json, "not_configured", "Sentinel isn't set up.", "spinuptui sentinel setup --plan")
+  if (cfg.sentinel.mode !== "shadow") return fail(json, "not_shadow", "Sentinel is already live; go-live only leaves shadow mode.")
+  const cutoff = eventMs(since)
+  for (const i of loadIncidents()) if (i.state === "shadow" || i.refersTo) deleteIncident(i.id)
+  const seen: Seen = {}
+  for (const [k, t] of Object.entries(loadSeen())) if (eventMs(t) <= cutoff) seen[k] = t
+  saveSeen(seen)
+  const ts = loadTickState()
+  saveTickState({ ...ts, started: true, lastCleanAt: new Date(cutoff).toISOString() })
+  await saveConfig({ sentinel: { ...cfg.sentinel, mode: "live" } })
+  out(json, { ok: true, since }, `Sentinel is live. Events after ${since} UTC are its to handle.`)
+  return 0
+}
+
+// ---- expectations ---------------------------------------------------------------------------
+
+function buildExpectation(id: string, f: Map<string, string>): Expectation {
+  const base = {
+    id,
+    ...(f.get("--domain") ? { domain: f.get("--domain") } : {}),
+    ...(f.get("--server") ? { server: f.get("--server") } : {}),
+    ...(f.get("--label") ? { label: f.get("--label") } : {}),
+    ...(f.get("--note") ? { note: f.get("--note") } : {}),
+    ...(f.get("--expires") ? { expires: f.get("--expires") } : {}),
+  }
+  const n = (name: string) => (f.has(name) ? Number(f.get(name)) : undefined)
+  switch (f.get("--type")) {
+    case "log-count":
+      return { ...base, type: "log-count", match: f.get("--match") ?? "", ...(f.has("--min") ? { min: n("--min") } : {}), ...(f.has("--trend") ? { trend: true } : {}) } as Expectation
+    case "load-shape":
+      return { ...base, type: "load-shape", threshold: n("--threshold") ?? 0 } as Expectation
+    case "cache-hit":
+      return { ...base, type: "cache-hit", host: f.get("--host") ?? "", paths: (f.get("--paths") ?? "").split(",").filter(Boolean) } as Expectation
+    case "warmer":
+      return { ...base, type: "warmer", logGlob: f.get("--log-glob") ?? "" } as Expectation
+    default:
+      return { ...base, type: f.get("--type") ?? "", url: f.get("--url") ?? "", expect: n("--expect") ?? 0 } as Expectation
+  }
+}
+
+async function expect(pos: string[], p: Parsed, json: boolean): Promise<number> {
+  const [action, id] = pos
+  const all = loadExpectations()
+  switch (action) {
+    case "list":
+      if (json) console.log(JSON.stringify({ ok: true, expectations: all }))
+      else if (!all.length) console.log("No expectations yet.")
+      else for (const e of all) console.log(`${e.id}  ${e.type}  ${e.domain ?? ("url" in e ? e.url : "")}${e.expires ? `  expires ${e.expires}` : ""}${e.label ? `  (${e.label})` : ""}`)
+      return 0
+    case "add": {
+      if (!id) return fail(json, "usage", USAGE)
+      const e = buildExpectation(id, p.flags)
+      const problems = validateExpectation(e)
+      if (all.some((x) => x.id === id)) problems.push(`there's already an expectation named ${id}`)
+      if (problems.length) return fail(json, "invalid", problems.join("\n"))
+      saveExpectations([...all, e])
+      out(json, { ok: true, id }, `Added ${id}. It runs with the next digest; to try it now: spinuptui sentinel expect check ${id}. Saved in ${moduleWorkDir("sentinel")}.`)
+      return 0
+    }
+    case "remove":
+      if (!all.some((x) => x.id === id)) return fail(json, "unknown_expectation", `No expectation "${id ?? ""}".`)
+      saveExpectations(all.filter((x) => x.id !== id))
+      out(json, { ok: true, id }, `Removed ${id}.`)
+      return 0
+    case "check": {
+      const e = all.find((x) => x.id === id)
+      if (!e) return fail(json, "unknown_expectation", `No expectation "${id ?? ""}".`)
+      const cfg = loadConfig()
+      const r = await runExpectation(e, expectDeps(cfg, new SpinupWPClient(cfg)))
+      out(json, { ok: true, ...r }, `${r.label}: ${r.state} — ${r.text}`)
+      return 0
+    }
+    case "ack": {
+      const flags = loadFlags()
+      if (!id || !flags[id]) return fail(json, "unknown_flag", `No flag is set for "${id ?? ""}".`)
+      saveFlags({ ...flags, [id]: { ...flags[id]!, acked: true } })
+      out(json, { ok: true, id }, `Acknowledged ${id}. It stays counted in the digest until it clears.`)
+      return 0
+    }
+  }
+  return fail(json, "usage", USAGE)
+}
+
+async function status(json: boolean): Promise<number> {
+  const s = await sentinelModule.status()
+  const newest = Object.values(loadSeen()).sort().pop() ?? null
+  if (json) console.log(JSON.stringify({ ok: true, lastEventTime: newest, ...s }))
+  else {
+    console.log(`Sentinel: ${s.state} — ${s.summary}`)
+    for (const c of s.checks) console.log(`  ${c.ok ? "✓" : "✗"} ${c.name}: ${c.detail}`)
+    if (s.remedy) console.log(`  → ${s.remedy}`)
+  }
+  return 0
+}

package/src/lib/heartbeat.ts

@@ -80,6 +80,11 @@ export function tickHeartbeat(ranCount: number, errors: string[], pager: string
   return { status: "up", msg: errors.length ? `tick ok, ${what}; ${errors.length} problem${errors.length === 1 ? "" : "s"}: ${errors[0]}` : `tick ok, ${what}` }
 }
 
+// What a tick that found the previous one still running tells Kuma.
+export function stillRunningHeartbeat(pager: string | null): { status: "up" | "down"; msg: string } {
+  return pager ? { status: "down", msg: `Station is up, but ${pager}` } : { status: "up", msg: "tick still running" }
+}
+
 // ---- setup through SpinupTUI's own Kuma login --------------------------------------
 //
 // With a Kuma login saved (the one site monitoring uses), SpinupTUI creates the

package/src/lib/schedule.ts

@@ -80,6 +80,7 @@ export function declSchedule(decl: { every?: string; at?: string }): Schedule {
 }
 
 export interface ScheduleState {
+  last_status?: string
   last_run?: string
   last_scheduled?: string
   retry_after?: string
@@ -143,7 +144,9 @@ export function nextRunText(job: DueInput, state: ScheduleState, now: Date): str
   if (job.expires && dayStart(now) > parseDate(job.expires).getTime()) return "expired"
   if (job.when) {
     const when = parseWhen(job.when)
-    return state.last_scheduled === fmt(when) ? "done" : fmt(when)
+    if (state.last_scheduled !== fmt(when)) return fmt(when)
+    // It ran but couldn't do its work, so it stays visibly unfinished.
+    return state.last_status === "blocked" ? "blocked" : "done"
   }
   const sched = job.schedule ?? parseEvery(job.every!)
   if (sched.type === "interval") return `every ${sched.minutes} min`