spinuptui 1.3.0-alpha.3 → 1.3.0-alpha.5

package/src/lib/sentinelExpect.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, test } from "bun:test"
+import { APT_COMMAND, cacheProbeCommand, loadShapeCommand, logCountCommand, readApt, runExpectation, validateExpectation, warmerCommand, type ExpectDeps, type Expectation } from "./sentinelExpect.ts"
+import { classifySshCommand } from "./sshExecPolicy.ts"
+
+const answering = (stdout: string | null, status: number | null = 200): ExpectDeps => ({ exec: async () => stdout, status: async () => status })
+
+describe("every remote command", () => {
+  test("gets past the ssh-exec policy", () => {
+    const commands = [logCountCommand("SomeCrawler/1.1"), logCountCommand("/old_tax/"), loadShapeCommand(8), cacheProbeCommand("www.example.net", "/?gbraid=probe"), warmerCommand("~/files/cache-warm/logs/warm-*.log"), APT_COMMAND]
+    for (const c of commands) expect([c, classifySshCommand(c).decision]).toEqual([c, "allow"])
+  })
+})
+
+describe("validation", () => {
+  const valid = (e: unknown) => validateExpectation(e as Expectation)
+
+  test("accepts each type", () => {
+    expect(valid({ id: "a-b", type: "log-count", domain: "example.com", match: "Bot/1.1", min: 10 })).toEqual([])
+    expect(valid({ id: "a-b", type: "load-shape", domain: "example.com", threshold: 8 })).toEqual([])
+    expect(valid({ id: "a-b", type: "cache-hit", domain: "example.com", host: "www.example.com", paths: ["/", "/?a=b"] })).toEqual([])
+    expect(valid({ id: "a-b", type: "warmer", domain: "example.com", logGlob: "~/files/logs/warm-*.log" })).toEqual([])
+    expect(valid({ id: "a-b", type: "status", url: "https://example.com/x", expect: 410 })).toEqual([])
+  })
+
+  test("keeps shell out of what reaches a remote command", () => {
+    expect(valid({ id: "a-b", type: "log-count", domain: "example.com", match: 'x"; rm -rf ~; "' }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "log-count", domain: "example.com", match: "$(id)" }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "cache-hit", domain: "example.com", host: "h.example.com", paths: ["/?a=1&b=2"] }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "warmer", domain: "example.com", logGlob: "x; id" }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "cache-hit", domain: "example.com", host: "$(id)", paths: ["/"] }).length).toBe(1)
+  })
+})
+
+describe("running a check", () => {
+  const count: Expectation = { id: "c", type: "log-count", domain: "example.com", match: "Bot", min: 100, trend: true, label: "Bot" }
+
+  test("log-count", async () => {
+    expect(await runExpectation(count, answering("150\n"))).toMatchObject({ state: "ok", trend: "150" })
+    expect(await runExpectation(count, answering("0\n"))).toMatchObject({ state: "flag", text: "LOW: 0 yesterday (expected ≥100)" })
+  })
+
+  test("a failed probe is unverified, never a pass or a flag", async () => {
+    expect((await runExpectation(count, answering(null))).state).toBe("unverified")
+    expect((await runExpectation(count, answering(""))).state).toBe("unverified")
+    expect((await runExpectation({ id: "l", type: "load-shape", domain: "e.com", threshold: 8 }, answering(""))).state).toBe("unverified")
+  })
+
+  test("load-shape flags a latch, not a spike", async () => {
+    const e: Expectation = { id: "l", type: "load-shape", domain: "e.com", threshold: 8 }
+    expect(await runExpectation(e, answering("9.5 2 2\n"))).toMatchObject({ state: "ok", text: "load max 9.5, 2 over 8, longest 20 min" })
+    expect((await runExpectation(e, answering("9.5 3 3\n"))).state).toBe("flag")
+  })
+
+  test("cache-hit", async () => {
+    const e: Expectation = { id: "h", type: "cache-hit", domain: "e.com", host: "www.e.com", paths: ["/", "/a"] }
+    expect((await runExpectation(e, answering("HIT\n"))).state).toBe("ok")
+    expect(await runExpectation(e, answering("MISS\n"))).toMatchObject({ state: "flag" })
+    expect((await runExpectation(e, answering(null))).state).toBe("unverified")
+  })
+
+  test("warmer", async () => {
+    const e: Expectation = { id: "w", type: "warmer", domain: "e.com", logGlob: "~/w-*.log" }
+    expect((await runExpectation(e, answering("12\n"))).state).toBe("ok")
+    expect(await runExpectation(e, answering("0\n"))).toMatchObject({ state: "flag", text: "WARMER IDLE: 0 queued" })
+  })
+
+  test("status", async () => {
+    const e: Expectation = { id: "s", type: "status", url: "https://e.com/x", expect: 410 }
+    expect((await runExpectation(e, answering(null, 410))).state).toBe("ok")
+    expect((await runExpectation(e, answering(null, 200))).state).toBe("flag")
+    expect((await runExpectation(e, answering(null, null))).state).toBe("unverified")
+  })
+})
+
+describe("readApt", () => {
+  const now = 1_800_000_000
+  test("healthy", () => expect(readApt(`${now - 3600} ${now} inactive 0`)).toEqual({ flag: null }))
+  test("stale", () => expect(readApt(`${now - 10 * 86_400} ${now} inactive 0`)!.flag).toContain("STALE: last successful update 10d ago"))
+  test("no stamp", () => expect(readApt(`none ${now} inactive 0`)!.flag).toContain("no successful update"))
+  test("stuck", () => expect(readApt(`${now} ${now} activating ${now - 3 * 86_400}`)!.flag).toContain("STUCK: apt-daily running 3d"))
+  test("unreadable", () => expect(readApt(null)).toBeNull())
+})

package/src/lib/sentinelExpect.ts

@@ -0,0 +1,197 @@
+// Expectation checks: what uptime monitoring can't see. A crawler that stopped
+// arriving, a cache that stopped hitting, a box that latches under load. They run
+// with the digest, read-only through ssh-exec, and report on the previous full
+// day: at 08:00 today's log is only a few hours old and would read as a collapse.
+//
+// A failed probe is "unverified", never a pass and never a flag. Commands avoid a
+// bare `>` and multi-parameter URLs, which the ssh-exec policy reads as a redirect
+// and a shell operator (docs/2026-10-30_sentinel-spec.md, "Expectations").
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"
+import { dirname } from "node:path"
+import { parseDate } from "./schedule.ts"
+import { expectationsPath, trendsPath, writeJson } from "./sentinelState.ts"
+
+interface Base {
+  id: string
+  domain?: string
+  server?: string // when the domain exists on more than one server
+  label?: string
+  note?: string
+  expires?: string
+}
+
+export type Expectation = Base &
+  (
+    | { type: "log-count"; domain: string; match: string; min?: number; trend?: boolean }
+    | { type: "load-shape"; domain: string; threshold: number }
+    | { type: "cache-hit"; domain: string; host: string; paths: string[] }
+    | { type: "warmer"; domain: string; logGlob: string }
+    | { type: "status"; url: string; expect: number }
+  )
+
+export const EXPECT_TYPES = ["log-count", "load-shape", "cache-hit", "warmer", "status"] as const
+const ID_RE = /^[a-z0-9][a-z0-9-]{1,79}$/
+const DOMAIN_RE = /^[a-z0-9.-]+$/i
+const LATCHED_SAMPLES = 3
+// `sar` writes a sample every 10 minutes.
+const SAMPLE_MIN = 10
+
+export function loadExpectations(): Expectation[] {
+  try {
+    return JSON.parse(readFileSync(expectationsPath(), "utf8")) as Expectation[]
+  } catch {
+    return []
+  }
+}
+
+export const saveExpectations = (list: Expectation[]) => writeJson(expectationsPath(), list)
+
+// Everything here ends up inside a remote shell string, so each value is held to
+// what it needs to be rather than escaped and hoped for.
+export function validateExpectation(e: Expectation): string[] {
+  const p: string[] = []
+  if (!ID_RE.test(e.id ?? "")) p.push("id must be lowercase letters, digits and hyphens")
+  if (!(EXPECT_TYPES as readonly string[]).includes(e.type)) p.push(`type must be one of ${EXPECT_TYPES.join(", ")}`)
+  if (e.expires) {
+    try {
+      parseDate(e.expires)
+    } catch (err) {
+      p.push((err as Error).message)
+    }
+  }
+  if (e.type !== "status" && !DOMAIN_RE.test(e.domain ?? "")) p.push("domain is required")
+  if (e.type === "log-count") {
+    if (!e.match) p.push("match is required")
+    if (/["$`\\\n>]/.test(e.match ?? "")) p.push('match can\'t contain " $ ` \\ or >')
+    if (e.min !== undefined && !(e.min >= 0)) p.push("min must be a number")
+  }
+  if (e.type === "load-shape" && !(e.threshold > 0)) p.push("threshold must be a positive number")
+  if (e.type === "cache-hit") {
+    if (!DOMAIN_RE.test(e.host ?? "")) p.push("host is required: the site's canonical host")
+    if (!e.paths?.length) p.push("paths is required")
+    for (const path of e.paths ?? []) if (!/^\/[\w./?=%-]*$/.test(path) || path.includes("&")) p.push(`${path}: a path may carry one query parameter and no shell characters`)
+  }
+  if (e.type === "warmer" && !/^[\w./~*-]+$/.test(e.logGlob ?? "")) p.push("logGlob is required, as a plain path with * wildcards")
+  if (e.type === "status") {
+    if (!/^https?:\/\//.test(e.url ?? "")) p.push("url is required")
+    if (!(e.expect >= 100)) p.push("expect is required: the status code")
+  }
+  return p
+}
+
+// ---- the remote commands ----------------------------------------------------------------
+
+// grep -c exits 1 on a zero count, so callers read stdout rather than the exit code.
+export const logCountCommand = (match: string) => `zcat ~/logs/access.log-$(date +%Y%m%d).gz 2>/dev/null | grep -cF "${match}"`
+
+// `!(x < T)` and `if (mx < x)` rather than `x >= T`: the guard reads a bare > as a
+// redirect and denies the whole command.
+export const loadShapeCommand = (threshold: number) =>
+  String.raw`LC_ALL=C sar -q -f /var/log/sysstat/sa$(date -d yesterday +%d) 2>/dev/null | awk -v T=${threshold} "\$4 ~ /^[0-9.]+\$/ {n++; if (!(\$4+0 < T)) {run++; over++; if (mxrun < run) mxrun=run} else {run=0}; if (mx < \$4+0) mx=\$4} END {printf \"%s %d %d\\n\", (mx==\"\"?0:mx), over+0, mxrun+0}"`
+
+// No `-o /dev/null`: the guard reads an output flag as a redirect, so the body
+// falls through and grep picks the header out of it.
+export const cacheProbeCommand = (host: string, path: string) =>
+  String.raw`for i in 1 2; do curl -s -D - --max-time 15 --resolve ${host}:443:127.0.0.1 "https://${host}${path}"; done | grep -i "^fastcgi-cache:" | tail -1 | awk "{print \$2}"`
+
+export const warmerCommand = (glob: string) => `L=$(ls -t ${glob} 2>/dev/null | head -1); [ -n "$L" ] && grep -hoE "starting: [0-9]+ URLs queued" "$L" | grep -oE "[0-9]+" | head -1`
+
+export const APT_COMMAND =
+  'echo $(stat -c %Y /var/lib/apt/periodic/update-success-stamp 2>/dev/null || echo none) $(date +%s) $(systemctl show apt-daily.service -p ActiveState --value) $(date -d "$(systemctl show apt-daily.service -p StateChangeTimestamp --value)" +%s 2>/dev/null || echo 0)'
+
+// ---- running one ---------------------------------------------------------------------------
+
+export type ExpectState = "ok" | "flag" | "unverified"
+
+export interface ExpectResult {
+  id: string
+  label: string
+  state: ExpectState
+  text: string
+  trend?: string
+}
+
+export interface ExpectDeps {
+  // stdout, or null when the probe itself failed.
+  exec(domain: string, command: string, server?: string): Promise<string | null>
+  status(url: string): Promise<number | null>
+}
+
+const labelOf = (e: Expectation) => e.label ?? e.id
+const num = (s: string | null) => (s !== null && /^\d+$/.test(s.trim()) ? Number(s.trim()) : null)
+
+export async function runExpectation(e: Expectation, deps: ExpectDeps): Promise<ExpectResult> {
+  const r = (state: ExpectState, text: string, trend?: string): ExpectResult => ({ id: e.id, label: labelOf(e), state, text, ...(trend !== undefined ? { trend } : {}) })
+  switch (e.type) {
+    case "log-count": {
+      const n = num(await deps.exec(e.domain, logCountCommand(e.match), e.server))
+      if (n === null) return r("unverified", "no count returned")
+      const trend = e.trend ? String(n) : undefined
+      if (e.min !== undefined && n < e.min) return r("flag", `LOW: ${n} yesterday (expected ≥${e.min})`, trend)
+      return r("ok", String(n), trend)
+    }
+    case "load-shape": {
+      const out = (await deps.exec(e.domain, loadShapeCommand(e.threshold), e.server))?.trim().split(/\s+/)
+      if (out?.length !== 3 || !/^[\d.]+$/.test(out[0]!) || !/^\d+$/.test(out[1]!) || !/^\d+$/.test(out[2]!)) return r("unverified", "no sar data")
+      const [max, over, run] = [out[0]!, Number(out[1]), Number(out[2])]
+      const shape = `load max ${max}, ${over} over ${e.threshold}, longest ${run * SAMPLE_MIN} min`
+      return run >= LATCHED_SAMPLES ? r("flag", `LATCHED: ${shape}`) : r("ok", shape)
+    }
+    case "cache-hit": {
+      const bad: string[] = []
+      let probes = 0
+      for (const path of e.paths) {
+        const got = (await deps.exec(e.domain, cacheProbeCommand(e.host, path), e.server))?.trim()
+        if (got) probes++
+        if (got?.toUpperCase() !== "HIT") bad.push(`${path}=${got || "no answer"}`)
+      }
+      if (!probes) return r("unverified", "no answer from any path")
+      return bad.length ? r("flag", `CACHE REGRESSED: ${bad.join(", ")}`) : r("ok", `${e.paths.length} path(s) HIT`)
+    }
+    case "warmer": {
+      const n = num(await deps.exec(e.domain, warmerCommand(e.logGlob), e.server))
+      if (n === null) return r("unverified", "no warmer log found")
+      return n === 0 ? r("flag", "WARMER IDLE: 0 queued") : r("ok", `${n} queued`)
+    }
+    case "status": {
+      const code = await deps.status(e.url)
+      if (code === null) return r("unverified", "no response")
+      return code === e.expect ? r("ok", String(code)) : r("flag", `${e.url} returned ${code} (expected ${e.expect})`)
+    }
+  }
+}
+
+export function appendTrends(results: ExpectResult[], date: string): void {
+  const rows = results.filter((r) => r.trend !== undefined).map((r) => `${date},${r.id},${r.trend}`)
+  if (!rows.length) return
+  const path = trendsPath()
+  mkdirSync(dirname(path), { recursive: true })
+  if (!existsSync(path)) appendFileSync(path, "date,id,value\n")
+  appendFileSync(path, rows.join("\n") + "\n")
+}
+
+// ---- apt freshness, for every server -----------------------------------------------------------
+
+const APT_STALE_DAYS = 3
+const APT_STUCK_HOURS = 6
+
+// Catches a server whose updates quietly stopped: a daily `apt-get update` that
+// hung and held the lock for over a year once, with the sites up and nothing
+// paging. null means the output didn't parse.
+export function readApt(output: string | null): { flag: string | null } | null {
+  const [stamp, nowS, state, since] = (output ?? "").trim().split(/\s+/)
+  if (!/^\d+$/.test(nowS ?? "")) return null
+  const now = Number(nowS)
+  const flags: string[] = []
+  if (!stamp || stamp === "none") flags.push("STALE: no successful update on record")
+  else {
+    const days = Math.floor((now - Number(stamp)) / 86_400)
+    if (days > APT_STALE_DAYS) flags.push(`STALE: last successful update ${days}d ago`)
+  }
+  if (state === "activating" && /^[1-9]\d*$/.test(since ?? "")) {
+    const hours = Math.floor((now - Number(since)) / 3600)
+    if (hours > APT_STUCK_HOURS) flags.push(`STUCK: apt-daily running ${hours > 48 ? `${Math.floor(hours / 24)}d` : `${hours}h`}`)
+  }
+  return { flag: flags.length ? flags.join("; ") : null }
+}

package/src/lib/sentinelIncidents.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, test } from "bun:test"
+import type { IncidentEvent, IncidentKind, IncidentStatus } from "./cliIncidents.ts"
+import { applyEvents, closeIncidents, eventMs, markSeen, newEvents, startedToday, windowHours, type ApplyInput } from "./sentinelIncidents.ts"
+import { seenKey, type Fleet, type Incident } from "./sentinelState.ts"
+
+const fleet: Fleet = { domains: { "a.example.com": { server: "web1", site: "a.example.com" }, "b.example.com": { server: "web1", site: "b.example.com" }, "c.example.com": { server: "web2", site: "c.example.com" } } }
+const now = new Date("2026-10-30T14:00:00Z")
+
+let nextId = 1
+function ev(domain: string, status: IncidentStatus = "down", kind: IncidentKind = "health", time = "2026-10-30 13:58:00.000", monitorId?: number): IncidentEvent {
+  return { domain, kind, status, time, message: "", monitorId: monitorId ?? nextId++ }
+}
+
+const apply = (over: Partial<ApplyInput> & { events: IncidentEvent[] }) =>
+  applyEvents({ incidents: [], fleet, now, recurrenceHours: 6, hasAgent: true, shadow: false, kumaAlerted: new Set(), ...over })
+
+describe("the seen-set", () => {
+  test("finds an old event a newer one would have hidden from a timestamp", () => {
+    const newer = ev("a.example.com", "down", "health", "2026-10-30 13:50:00.000", 1)
+    const older = ev("b.example.com", "down", "health", "2026-10-30 13:40:00.000", 2)
+    const seen = markSeen({}, [newer], now)
+    expect(newEvents([older, newer], seen)).toEqual([older])
+  })
+
+  test("two monitors can share a timestamp", () => {
+    const a = ev("a.example.com", "down", "health", "2026-10-30 13:50:00.000", 1)
+    const b = ev("a.example.com", "down", "redis", "2026-10-30 13:50:00.000", 2)
+    expect(newEvents([a, b], markSeen({}, [a], now))).toEqual([b])
+  })
+
+  test("drops what's past the longest window", () => {
+    const old = ev("a.example.com", "down", "health", "2026-10-20 13:50:00.000", 1)
+    expect(markSeen({ [seenKey(old)]: old.time }, [], now)).toEqual({})
+  })
+
+  test("the window reaches back to the last clean tick, within limits", () => {
+    expect(windowHours(undefined, now)).toBe(1)
+    expect(windowHours("2026-10-30T13:55:00Z", now)).toBe(2)
+    expect(windowHours("2026-10-29T13:55:00Z", now)).toBe(26)
+    expect(windowHours("2026-09-01T00:00:00Z", now)).toBe(168)
+  })
+
+  test("reads Kuma's zoneless times as UTC", () => {
+    expect(eventMs("2026-10-30 13:34:05.123")).toBe(Date.parse("2026-10-30T13:34:05.123Z"))
+  })
+})
+
+describe("grouping", () => {
+  test("down events on one server are one incident, with one page", () => {
+    const r = apply({ events: [ev("a.example.com"), ev("b.example.com"), ev("c.example.com")] })
+    expect(r.created.map((i) => i.key).sort()).toEqual(["web1", "web2"])
+    expect(r.pages.length).toBe(2)
+    expect(r.pages[0]!.text).toContain("a.example.com, b.example.com")
+  })
+
+  test("a domain outside the fleet is its own incident, and asks for a refresh", () => {
+    const r = apply({ events: [ev("other.example.org")] })
+    expect(r.created[0]!.key).toBe("other.example.org")
+    expect(r.unknownDomains).toEqual(["other.example.org"])
+  })
+
+  test("joining an open incident pages once silently, and queues nothing new", () => {
+    const first = apply({ events: [ev("a.example.com")] })
+    const later = apply({ incidents: first.created, events: [ev("b.example.com")] })
+    expect(later.created).toEqual([])
+    expect(later.pages).toMatchObject([{ silent: true }])
+    expect(later.pages[0]!.text).toContain("joined")
+    const again = apply({ incidents: first.created, events: [ev("b.example.com", "down", "redis")] })
+    expect(again.pages).toEqual([])
+  })
+
+  test("the first page is silent only when Kuma alerted for every monitor", () => {
+    const a = ev("a.example.com", "down", "health", undefined, 7)
+    const b = ev("b.example.com", "down", "health", undefined, 8)
+    expect(apply({ events: [a, b], kumaAlerted: new Set([7, 8]) }).pages[0]!.silent).toBe(true)
+    expect(apply({ events: [a, b], kumaAlerted: new Set([7]) }).pages[0]!.silent).toBe(false)
+  })
+
+  test("without an agent the page says so", () => {
+    expect(apply({ events: [ev("a.example.com")], hasAgent: false }).pages[0]!.text).toContain("no agent")
+  })
+})
+
+describe("closing", () => {
+  test("every monitor back, then 30 quiet minutes", () => {
+    const down = ev("a.example.com", "down", "health", undefined, 1)
+    const [i] = apply({ events: [down] }).created
+    expect(closeIncidents([i!], new Date(now.getTime() + 3_600_000))).toEqual([])
+    apply({ incidents: [i!], events: [ev("a.example.com", "up", "health", "2026-10-30 14:00:00.000", 1)] })
+    expect(i!.down).toEqual([])
+    expect(closeIncidents([i!], new Date(now.getTime() + 60_000))).toEqual([])
+    expect(closeIncidents([i!], new Date(now.getTime() + 31 * 60_000)).length).toBe(1)
+  })
+
+  test("a monitor that never comes back can't hold it open for ever", () => {
+    const [i] = apply({ events: [ev("a.example.com")] }).created
+    expect(closeIncidents([i!], new Date(now.getTime() + 8 * 24 * 3_600_000)).length).toBe(1)
+  })
+})
+
+describe("recurrence", () => {
+  const reported = (at: Date): Incident => {
+    const [i] = apply({ events: [ev("a.example.com")], now: at }).created
+    return { ...i!, state: "reported", reportedAt: at.toISOString(), reportPath: "sites/a/incidents/x.md", closedAt: at.toISOString() }
+  }
+
+  test("a report in the window means no new diagnosis, and a way to ask for one", () => {
+    const r = apply({ incidents: [reported(new Date(now.getTime() - 3_600_000))], events: [ev("a.example.com")] })
+    expect(r.created[0]!.state).toBe("recurrence")
+    expect(r.pages[0]).toMatchObject({ silent: true })
+    expect(r.pages[0]!.text).toContain("sites/a/incidents/x.md")
+    expect(r.pages[0]!.buttons![0]![0]).toMatchObject({ action: "sentinel.diagnose" })
+  })
+
+  test("outside the window, or for another kind, it's a new incident", () => {
+    const old = reported(new Date(now.getTime() - 7 * 3_600_000))
+    expect(apply({ incidents: [old], events: [ev("a.example.com")] }).created[0]!.state).toBe("queued")
+    const recent = reported(new Date(now.getTime() - 3_600_000))
+    expect(apply({ incidents: [recent], events: [ev("a.example.com", "down", "redis")] }).created[0]!.state).toBe("queued")
+  })
+
+  test("a failed diagnosis doesn't cover anything", () => {
+    const failed: Incident = { ...reported(new Date(now.getTime() - 3_600_000)), state: "failed" }
+    expect(apply({ incidents: [failed], events: [ev("a.example.com")] }).created[0]!.state).toBe("queued")
+  })
+
+  test("the third in 24 hours makes a sound", () => {
+    const base = reported(new Date(now.getTime() - 3_600_000))
+    const rec = (n: number): Incident => ({ ...base, id: `r${n}`, state: "recurrence", refersTo: base.id, createdAt: new Date(now.getTime() - n * 3_600_000).toISOString() })
+    expect(apply({ incidents: [base, rec(1)], events: [ev("a.example.com")] }).pages[0]!.silent).toBe(true)
+    expect(apply({ incidents: [base, rec(1), rec(2)], events: [ev("a.example.com")] }).pages[0]!.silent).toBe(false)
+  })
+})
+
+describe("shadow mode", () => {
+  test("records what would have been queued", () => {
+    expect(apply({ events: [ev("a.example.com")], shadow: true }).created[0]!.state).toBe("shadow")
+  })
+})
+
+test("the cap counts today's starts", () => {
+  const i = apply({ events: [ev("a.example.com")] }).created[0]!
+  expect(startedToday([{ ...i, startedAt: now.toISOString() }, i], now)).toBe(1)
+  expect(startedToday([{ ...i, startedAt: "2026-10-01T00:00:00Z" }], now)).toBe(0)
+})

package/src/lib/sentinelIncidents.ts

@@ -0,0 +1,194 @@
+// What a tick decides, with no I/O: which events are new, how they group into
+// incidents, what's a recurrence, and what to page. The tick feeds it state and
+// carries out what it returns (docs/2026-10-30_sentinel-spec.md, "Incidents").
+
+import type { IncidentEvent as KumaEvent } from "./cliIncidents.ts"
+import { seenKey, SEEN_KEEP_DAYS, type Fleet, type Incident, type Seen } from "./sentinelState.ts"
+import type { BeaconButton } from "../modules/contract.ts"
+
+const HOUR = 3_600_000
+export const CLOSE_QUIET_MS = 30 * 60_000
+const CLOSE_ANYWAY_MS = 7 * 24 * HOUR
+const LOUD_RECURRENCE = 3
+
+export interface Page {
+  text: string
+  silent: boolean
+  buttons?: BeaconButton[][]
+}
+
+// A Kuma time ("2026-10-30 13:34:05.123") is UTC with no zone marker.
+export const eventMs = (time: string) => Date.parse(`${time.replace(" ", "T").slice(0, 23)}Z`)
+
+// The events not handled yet. A seen-set rather than a timestamp: a monitor whose
+// history failed to read last time can still deliver an event older than one
+// already handled, and two events can share a timestamp.
+export function newEvents(events: KumaEvent[], seen: Seen): KumaEvent[] {
+  return events.filter((e) => !seen[seenKey(e)])
+}
+
+export function markSeen(seen: Seen, events: KumaEvent[], now: Date): Seen {
+  const out = { ...seen }
+  for (const e of events) out[seenKey(e)] = e.time
+  const cutoff = now.getTime() - SEEN_KEEP_DAYS * 24 * HOUR
+  for (const [k, t] of Object.entries(out)) if (eventMs(t) < cutoff) delete out[k]
+  return out
+}
+
+// How far back to read: from the last tick that read every monitor.
+export function windowHours(lastCleanAt: string | undefined, now: Date): number {
+  if (!lastCleanAt) return 1
+  const hours = Math.ceil((now.getTime() - Date.parse(lastCleanAt)) / HOUR) + 1
+  return Math.min(Math.max(hours, 1), 168)
+}
+
+const stamp = (d: Date) => d.toISOString().slice(0, 16).replace(/[-:]/g, "").replace("T", "-")
+const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")
+
+export interface ApplyInput {
+  incidents: Incident[]
+  events: KumaEvent[]
+  fleet: Fleet
+  now: Date
+  recurrenceHours: number
+  hasAgent: boolean
+  shadow: boolean
+  kumaAlerted: Set<number>
+}
+
+export interface ApplyOutput {
+  changed: Incident[]
+  created: Incident[]
+  pages: Page[]
+  unknownDomains: string[]
+}
+
+function covering(all: Incident[], e: KumaEvent, sinceMs: number): Incident | null {
+  const live = new Set(["reported", "queued", "diagnosing", "waiting_agent", "shadow"])
+  for (const i of [...all].reverse()) {
+    if (!live.has(i.state)) continue
+    if (Date.parse(i.reportedAt ?? i.createdAt) < sinceMs) continue
+    if (i.events.some((x) => x.domain === e.domain && x.kind === e.kind)) return i
+  }
+  return null
+}
+
+const domainsOf = (i: Incident) => [...new Set(i.events.map((e) => e.domain))]
+const kindsOf = (i: Incident) => [...new Set(i.events.map((e) => e.kind))]
+
+export function diagnoseButton(id: string, text = "Diagnose anyway"): BeaconButton {
+  return { text, action: "sentinel.diagnose", payload: { id } }
+}
+
+function newIncidentPage(i: Incident, hasAgent: boolean, kumaAlerted: Set<number>): Page {
+  const where = i.server ? ` on ${i.server}` : ""
+  const next = hasAgent ? "diagnosing" : "writing it up (no agent configured)"
+  return {
+    text: `🔴 ${domainsOf(i).join(", ")} down (${kindsOf(i).join(", ")})${where}: ${next}.`,
+    // Kuma already made a sound for this; Sentinel's job is the "why" that follows.
+    silent: i.events.every((e) => kumaAlerted.has(e.monitorId)),
+  }
+}
+
+function recurrencePage(i: Incident, earlier: Incident, count: number): Page {
+  const first = i.events[0]!
+  return {
+    text: `🔁 ${first.domain} ${first.kind} again${count > 1 ? ` (${count} in 24h)` : ""}. Covered by ${earlier.reportPath ?? earlier.id}, so no new diagnosis.`,
+    silent: count < LOUD_RECURRENCE,
+    buttons: [[diagnoseButton(i.id)]],
+  }
+}
+
+// Applies new events to the incidents. A down event joins its server's open
+// incident, or starts one, unless a recent report already covers it. An up event
+// clears its monitor.
+export function applyEvents(input: ApplyInput): ApplyOutput {
+  const { fleet, now, recurrenceHours, hasAgent, shadow, kumaAlerted } = input
+  const all = [...input.incidents]
+  const changed = new Map<string, Incident>()
+  const created: Incident[] = []
+  const fresh = new Set<string>()
+  const unknownDomains = new Set<string>()
+  const pages: Page[] = []
+  const nowIso = now.toISOString()
+
+  for (const e of [...input.events].sort((a, b) => a.time.localeCompare(b.time))) {
+    if (e.status === "up") {
+      for (const i of all) {
+        if (!i.closedAt && i.down.includes(e.monitorId)) {
+          i.down = i.down.filter((m) => m !== e.monitorId)
+          changed.set(i.id, i)
+        }
+      }
+      continue
+    }
+    if (e.status !== "down") continue
+    const entry = fleet.domains[e.domain]
+    if (!entry) unknownDomains.add(e.domain)
+    const key = entry?.server ?? e.domain
+    const event = { domain: e.domain, kind: e.kind, time: e.time, message: e.message, monitorId: e.monitorId }
+    const open = all.find((i) => !i.closedAt && i.key === key)
+    if (open) {
+      const known = open.events.some((x) => x.domain === e.domain)
+      open.events.push(event)
+      if (!open.down.includes(e.monitorId)) open.down.push(e.monitorId)
+      open.lastDownAt = nowIso
+      changed.set(open.id, open)
+      if (!known && !fresh.has(open.id)) pages.push({ text: `➕ ${e.domain} (${e.kind}) joined the incident on ${key}.`, silent: true })
+      continue
+    }
+    const earlier = covering(all, e, now.getTime() - recurrenceHours * HOUR)
+    const base = `${stamp(now)}-${slug(key)}`
+    let id = base
+    for (let n = 2; all.some((i) => i.id === id); n++) id = `${base}-${n}`
+    const incident: Incident = {
+      id,
+      key,
+      server: entry?.server ?? null,
+      state: earlier ? "recurrence" : shadow ? "shadow" : "queued",
+      createdAt: nowIso,
+      events: [event],
+      down: [e.monitorId],
+      lastDownAt: nowIso,
+      attempts: 0,
+      ...(earlier ? { refersTo: earlier.id } : {}),
+    }
+    all.push(incident)
+    created.push(incident)
+    changed.set(id, incident)
+    fresh.add(id)
+  }
+
+  for (const i of created) {
+    if (i.state === "recurrence") {
+      const earlier = all.find((x) => x.id === i.refersTo)!
+      const domain = i.events[0]!.domain
+      const count = all.filter((x) => x.state === "recurrence" && Date.parse(x.createdAt) > now.getTime() - 24 * HOUR && x.events.some((y) => y.domain === domain)).length
+      pages.push(recurrencePage(i, earlier, count))
+    } else {
+      pages.push(newIncidentPage(i, hasAgent, kumaAlerted))
+    }
+  }
+  return { changed: [...changed.values()], created, pages, unknownDomains: [...unknownDomains] }
+}
+
+// Closes what's over: every monitor back and quiet for 30 minutes. A monitor that
+// never reports back (deleted in Kuma, say) can't hold an incident open forever.
+export function closeIncidents(incidents: Incident[], now: Date): Incident[] {
+  const out: Incident[] = []
+  for (const i of incidents) {
+    if (i.closedAt) continue
+    const quiet = now.getTime() - Date.parse(i.lastDownAt)
+    if ((i.down.length === 0 && quiet >= CLOSE_QUIET_MS) || quiet >= CLOSE_ANYWAY_MS) {
+      i.closedAt = now.toISOString()
+      out.push(i)
+    }
+  }
+  return out
+}
+
+// How many diagnoses have started today, for the daily cap.
+export function startedToday(incidents: Incident[], now: Date): number {
+  const day = now.toLocaleDateString("en-CA")
+  return incidents.filter((i) => i.startedAt && new Date(i.startedAt).toLocaleDateString("en-CA") === day).length
+}