spinuptui 1.3.0-alpha.4 → 1.3.0-alpha.5

package/src/lib/sentinelDigest.ts

@@ -0,0 +1,261 @@
+// The daily digest: the place a slow-moving problem gets said out loud. It always
+// sends, and it's silent unless something is new, because a header that's amber
+// every day stops being read. A flag is under New on the day it trips, then under
+// Ongoing with its age (docs/2026-10-30_sentinel-spec.md, "The digest").
+
+import type { AppConfig } from "../config.ts"
+import type { SpinupWPClientLike } from "../api/client.ts"
+import { schedulerModule } from "../modules/scheduler.ts"
+import { boardLogin } from "./beaconBoard.ts"
+import { listenerHealth } from "./beaconListenInstall.ts"
+import { agentCommand, AUTH_FAIL_RE, jobEnv, runProcess } from "./schedulerRun.ts"
+import { APT_COMMAND, appendTrends, loadExpectations, readApt, runExpectation, validateExpectation, type ExpectDeps, type ExpectResult } from "./sentinelExpect.ts"
+import { isWaiting, loadFleet, loadIncidents, loadPause, readTickLog, saveFleet, saveFlags, readJson, writeJson, sentinelDir, type Fleet, type Flags, type Incident, type SentinelSettings, type TickLine } from "./sentinelState.ts"
+import { refreshFleetFrom } from "./sentinelTick.ts"
+import { execSshCommand } from "./sshExec.ts"
+import { hasIncidentMonitors } from "./cliIncidents.ts"
+import { withKuma } from "./uptimeKuma.ts"
+import { join } from "node:path"
+
+const CHECK_TIMEOUT_MS = 30_000
+const CONCURRENCY = 4
+const AGENT_PROBE_MS = 60_000
+const DAY_MS = 24 * 3_600_000
+const STUCK_MS = DAY_MS
+
+export interface HealthCheck {
+  name: string
+  ok: boolean
+  detail: string
+}
+
+export interface DigestData {
+  when: Date
+  health: HealthCheck[]
+  ticks: TickLine[]
+  incidents: Incident[]
+  results: ExpectResult[]
+  aptTotal: number
+  apt: ExpectResult[]
+  notes: string[]
+  paused?: string
+}
+
+export interface Digest {
+  text: string
+  silent: boolean
+  flags: Flags
+}
+
+const localDay = (d: Date) => d.toLocaleDateString("en-CA")
+const dayCount = (from: string, to: string) => Math.round((Date.parse(to) - Date.parse(from)) / DAY_MS)
+const plural = (n: number, s: string) => `${n} ${s}${n === 1 ? "" : "s"}`
+
+export function composeDigest(d: DigestData, flags: Flags): Digest {
+  const today = localDay(d.when)
+  const since = d.when.getTime() - DAY_MS
+  const checks = [...d.results, ...d.apt]
+  const next: Flags = {}
+  const fresh: string[] = []
+  const ongoing: string[] = []
+  let acked = 0
+  for (const r of checks) {
+    if (r.state !== "flag") continue
+    const line = `${r.label}: ${r.text}`
+    const prior = flags[r.id]
+    next[r.id] = { firstSeen: prior?.firstSeen ?? today, text: line, ...(prior?.acked ? { acked: true } : {}) }
+    if (!prior || prior.firstSeen === today) fresh.push(line)
+    else if (prior.acked) acked++
+    else ongoing.push(`${line} (${plural(dayCount(prior.firstSeen, today), "day")})`)
+  }
+  // A check that couldn't run says nothing either way, so its flag keeps its age.
+  for (const r of checks) if (r.state === "unverified" && flags[r.id]) next[r.id] = flags[r.id]!
+
+  const recent = d.incidents.filter((i) => Date.parse(i.createdAt) >= since)
+  const stuck = d.incidents.filter((i) => isWaiting(i) && d.when.getTime() - Date.parse(i.createdAt) > STUCK_MS)
+  const failing = d.health.filter((h) => !h.ok)
+  const reports = recent.filter((i) => i.state === "reported").length
+  const recurrences = recent.filter((i) => i.state === "recurrence")
+  const blind = d.ticks.filter((t) => !t.ok).length
+  const retried = d.ticks.filter((t) => t.ok && t.retries > 0).length
+
+  const date = d.when.toLocaleDateString("en-US", { weekday: "short", month: "short", day: "numeric" })
+  const attention = fresh.length > 0 || failing.length > 0 || stuck.length > 0
+  const header = !attention ? `✅ Fleet OK — ${date}` : fresh.length ? `⚠️ ${fresh.length} new — ${date}` : `⚠️ Needs attention — ${date}`
+
+  const lines = [header, ""]
+  if (d.paused) lines.push(`Paused until ${d.paused}.`, "")
+  lines.push(d.health.map((h) => `${h.name}: ${h.ok ? h.detail : `⚠️ ${h.detail}`}`).join("\n"))
+  lines.push(`Last 24h: ${plural(d.ticks.length, "check")} · ${plural(recent.length - recurrences.length, "incident")} · ${plural(reports, "report")} · ${plural(recurrences.length, "recurrence")} · ${blind} blind${retried ? ` · ${retried} cold start${retried === 1 ? "" : "s"}` : ""}`)
+  if (recurrences.length) {
+    const bySite = new Map<string, number>()
+    for (const i of recurrences) bySite.set(i.events[0]!.domain, (bySite.get(i.events[0]!.domain) ?? 0) + 1)
+    lines.push(`Recurring: ${[...bySite].map(([s, n]) => `${s} ×${n}`).join(", ")}`)
+  }
+  if (stuck.length) lines.push("", "Stuck:", ...stuck.map((i) => `  ${i.id}: ${i.state}${i.why ? ` (${i.why})` : ""}`))
+  if (fresh.length) lines.push("", "New:", ...fresh.map((l) => `  ${l}`))
+  if (ongoing.length) lines.push("", "Ongoing:", ...ongoing.map((l) => `  ${l}`))
+  if (acked) lines.push("", `${plural(acked, "acknowledged flag")} still set.`)
+  const quiet = d.results.filter((r) => r.state !== "flag")
+  if (quiet.length) lines.push("", "Checks:", ...quiet.map((r) => `  ${r.label}: ${r.state === "unverified" ? `unverified (${r.text})` : r.text}`))
+  if (d.aptTotal) {
+    const unverified = d.apt.filter((r) => r.state === "unverified")
+    const normal = d.apt.filter((r) => r.state === "ok").length
+    lines.push("", `Server updates: ${normal} of ${d.aptTotal} normal${unverified.length ? `, unverified: ${unverified.map((r) => r.label).join(", ")}` : ""}`)
+  }
+  if (d.notes.length) lines.push("", ...d.notes)
+  return { text: lines.join("\n"), silent: !attention, flags: next }
+}
+
+// ---- gathering --------------------------------------------------------------------------------
+
+async function bounded<T>(fn: () => Promise<T>, fallback: T, ms = CHECK_TIMEOUT_MS): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined
+  try {
+    return await Promise.race([fn(), new Promise<T>((r) => (timer = setTimeout(() => r(fallback), ms)))])
+  } catch {
+    return fallback
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+async function mapLimit<T, R>(items: T[], limit: number, fn: (x: T) => Promise<R>): Promise<R[]> {
+  const out: R[] = new Array(items.length)
+  let next = 0
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, async () => {
+      while (next < items.length) {
+        const i = next++
+        out[i] = await fn(items[i]!)
+      }
+    }),
+  )
+  return out
+}
+
+export function expectDeps(cfg: AppConfig, client: SpinupWPClientLike): ExpectDeps {
+  return {
+    exec: async (domain, command, server) => {
+      const r = await execSshCommand(domain, command, client, cfg, { server: server ?? null })
+      return r.ok ? r.stdout : null
+    },
+    status: async (url) => {
+      try {
+        return (await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(20_000) })).status
+      } catch {
+        return null
+      }
+    },
+  }
+}
+
+// One attempt per server per day, on the site that worked last: a server with no
+// reachable site reads "unverified", and fail2ban never hears about it.
+const aptSitesPath = () => join(sentinelDir(), "apt-sites.json")
+
+export function aptTargets(fleet: Fleet, lastGood: Record<string, string>, monitored: Set<string>): { server: string; domain: string }[] {
+  const byServer = new Map<string, string[]>()
+  for (const [domain, e] of Object.entries(fleet.domains)) {
+    if (domain !== e.site) continue
+    byServer.set(e.server, [...(byServer.get(e.server) ?? []), domain])
+  }
+  return [...byServer].sort().map(([server, domains]) => {
+    const rank = (d: string) => (d === lastGood[server] ? 0 : monitored.has(d) ? 1 : fleet.domains[d]?.vanity ? 2 : 3)
+    return { server, domain: [...domains].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))[0]! }
+  })
+}
+
+async function checkApt(fleet: Fleet, cfg: AppConfig, deps: ExpectDeps, dryRun: boolean): Promise<ExpectResult[]> {
+  const lastGood = readJson<Record<string, string>>(aptSitesPath(), {})
+  const monitored = new Set(Object.entries(cfg.kumaMonitors).filter(([, r]) => hasIncidentMonitors(r)).map(([d]) => d))
+  const results = await mapLimit(aptTargets(fleet, lastGood, monitored), CONCURRENCY, async ({ server, domain }) => {
+    const out = await bounded(() => deps.exec(domain, APT_COMMAND), null)
+    const read = readApt(out)
+    if (read) lastGood[server] = domain
+    const base = { id: `apt-${server}`, label: server }
+    if (!read) return { ...base, state: "unverified" as const, text: "probe failed" }
+    return read.flag ? { ...base, state: "flag" as const, text: read.flag } : { ...base, state: "ok" as const, text: "updating" }
+  })
+  if (!dryRun) writeJson(aptSitesPath(), lastGood)
+  return results
+}
+
+async function probeAgent(s: SentinelSettings): Promise<HealthCheck> {
+  const env = jobEnv()
+  const cmd = agentCommand({ agent: s.agent!, prompt: "Reply with exactly: AUTH_OK", workdir: s.workspace }, env)
+  if (typeof cmd === "string") return { name: "Agent", ok: false, detail: cmd }
+  const r = await runProcess(cmd, { cwd: sentinelDir(), env, timeoutMs: AGENT_PROBE_MS }).catch((e: Error) => ({ code: 1, output: e.message, timedOut: false }))
+  if (r.output.includes("AUTH_OK")) return { name: "Agent", ok: true, detail: "OK" }
+  return { name: "Agent", ok: false, detail: AUTH_FAIL_RE.test(r.output) ? `the ${s.agent} login has lapsed` : `unverified (probe ${r.timedOut ? "timed out" : `exited ${r.code}`})` }
+}
+
+export async function gatherDigest(s: SentinelSettings, cfg: AppConfig, client: SpinupWPClientLike, opts: { dryRun?: boolean; now?: Date } = {}): Promise<DigestData> {
+  const when = opts.now ?? new Date()
+  const notes: string[] = []
+  let fleet = loadFleet()
+  try {
+    fleet = await refreshFleetFrom(client, when)
+    if (!opts.dryRun) saveFleet(fleet)
+  } catch (err) {
+    notes.push(`Couldn't refresh the fleet list: ${(err as Error).message}`)
+  }
+
+  const health: HealthCheck[] = []
+  const sched = await schedulerModule.status()
+  const bad = sched.checks.find((c) => !c.ok && (c.name === "last tick" || c.name === "dispatcher" || c.name === "heartbeat"))
+  health.push({ name: "Station", ok: !bad, detail: bad ? `${bad.name}: ${bad.detail}` : "ticking" })
+  if (s.agent) health.push(await probeAgent(s))
+  if (s.reportTo) {
+    const b = await bounded(() => boardLogin(), { ok: false, detail: "timed out" })
+    health.push({ name: "Board", ok: b.ok, detail: b.ok ? "OK" : b.detail })
+  }
+  const listener = listenerHealth()
+  if (listener) health.push({ name: "Listener", ok: listener.ok, detail: listener.ok ? "OK" : listener.detail })
+  if (cfg.uptimeKuma) {
+    const ids = await bounded(async () => new Set((await withKuma(cfg.uptimeKuma!, (k) => k.getMonitors())).result.map((m) => m.id)), null)
+    if (!ids) health.push({ name: "Kuma", ok: false, detail: "couldn't list monitors" })
+    else {
+      const gone = Object.entries(cfg.kumaMonitors)
+        .filter(([, r]) => Object.entries(r).some(([k, v]) => k.endsWith("Id") && typeof v === "number" && !ids.has(v)))
+        .map(([d]) => d)
+      health.push({ name: "Kuma", ok: gone.length === 0, detail: gone.length ? `monitors deleted for ${gone.join(", ")}` : "OK" })
+    }
+  }
+
+  const deps = expectDeps(cfg, client)
+  const today = when.getTime()
+  const active = loadExpectations().filter((e) => {
+    if (validateExpectation(e).length) {
+      notes.push(`Skipped invalid check ${e.id}: ${validateExpectation(e).join("; ")}`)
+      return false
+    }
+    if (e.expires && Date.parse(`${e.expires}T23:59:59`) < today) {
+      notes.push(`Check ${e.id} expired on ${e.expires} and no longer runs.`)
+      return false
+    }
+    return true
+  })
+  const results = await mapLimit(active, CONCURRENCY, (e) =>
+    bounded(() => runExpectation(e, deps), { id: e.id, label: e.label ?? e.id, state: "unverified" as const, text: "timed out" }, 90_000),
+  )
+  const apt = await checkApt(fleet, cfg, deps, !!opts.dryRun)
+  const pause = loadPause()
+  return {
+    when,
+    health,
+    ticks: readTickLog(when.getTime() - DAY_MS),
+    incidents: loadIncidents().filter((i) => !i.closedAt || Date.parse(i.createdAt) > when.getTime() - DAY_MS || isWaiting(i)),
+    results,
+    aptTotal: apt.length,
+    apt,
+    notes,
+    ...(pause && Date.parse(pause.until) > when.getTime() ? { paused: new Date(pause.until).toLocaleString() } : {}),
+  }
+}
+
+export function recordDigest(digest: Digest, data: DigestData): void {
+  saveFlags(digest.flags)
+  appendTrends(data.results, localDay(new Date(data.when.getTime() - DAY_MS)))
+}
+

package/src/lib/sentinelExpect.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, test } from "bun:test"
+import { APT_COMMAND, cacheProbeCommand, loadShapeCommand, logCountCommand, readApt, runExpectation, validateExpectation, warmerCommand, type ExpectDeps, type Expectation } from "./sentinelExpect.ts"
+import { classifySshCommand } from "./sshExecPolicy.ts"
+
+const answering = (stdout: string | null, status: number | null = 200): ExpectDeps => ({ exec: async () => stdout, status: async () => status })
+
+describe("every remote command", () => {
+  test("gets past the ssh-exec policy", () => {
+    const commands = [logCountCommand("SomeCrawler/1.1"), logCountCommand("/old_tax/"), loadShapeCommand(8), cacheProbeCommand("www.example.net", "/?gbraid=probe"), warmerCommand("~/files/cache-warm/logs/warm-*.log"), APT_COMMAND]
+    for (const c of commands) expect([c, classifySshCommand(c).decision]).toEqual([c, "allow"])
+  })
+})
+
+describe("validation", () => {
+  const valid = (e: unknown) => validateExpectation(e as Expectation)
+
+  test("accepts each type", () => {
+    expect(valid({ id: "a-b", type: "log-count", domain: "example.com", match: "Bot/1.1", min: 10 })).toEqual([])
+    expect(valid({ id: "a-b", type: "load-shape", domain: "example.com", threshold: 8 })).toEqual([])
+    expect(valid({ id: "a-b", type: "cache-hit", domain: "example.com", host: "www.example.com", paths: ["/", "/?a=b"] })).toEqual([])
+    expect(valid({ id: "a-b", type: "warmer", domain: "example.com", logGlob: "~/files/logs/warm-*.log" })).toEqual([])
+    expect(valid({ id: "a-b", type: "status", url: "https://example.com/x", expect: 410 })).toEqual([])
+  })
+
+  test("keeps shell out of what reaches a remote command", () => {
+    expect(valid({ id: "a-b", type: "log-count", domain: "example.com", match: 'x"; rm -rf ~; "' }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "log-count", domain: "example.com", match: "$(id)" }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "cache-hit", domain: "example.com", host: "h.example.com", paths: ["/?a=1&b=2"] }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "warmer", domain: "example.com", logGlob: "x; id" }).length).toBe(1)
+    expect(valid({ id: "a-b", type: "cache-hit", domain: "example.com", host: "$(id)", paths: ["/"] }).length).toBe(1)
+  })
+})
+
+describe("running a check", () => {
+  const count: Expectation = { id: "c", type: "log-count", domain: "example.com", match: "Bot", min: 100, trend: true, label: "Bot" }
+
+  test("log-count", async () => {
+    expect(await runExpectation(count, answering("150\n"))).toMatchObject({ state: "ok", trend: "150" })
+    expect(await runExpectation(count, answering("0\n"))).toMatchObject({ state: "flag", text: "LOW: 0 yesterday (expected ≥100)" })
+  })
+
+  test("a failed probe is unverified, never a pass or a flag", async () => {
+    expect((await runExpectation(count, answering(null))).state).toBe("unverified")
+    expect((await runExpectation(count, answering(""))).state).toBe("unverified")
+    expect((await runExpectation({ id: "l", type: "load-shape", domain: "e.com", threshold: 8 }, answering(""))).state).toBe("unverified")
+  })
+
+  test("load-shape flags a latch, not a spike", async () => {
+    const e: Expectation = { id: "l", type: "load-shape", domain: "e.com", threshold: 8 }
+    expect(await runExpectation(e, answering("9.5 2 2\n"))).toMatchObject({ state: "ok", text: "load max 9.5, 2 over 8, longest 20 min" })
+    expect((await runExpectation(e, answering("9.5 3 3\n"))).state).toBe("flag")
+  })
+
+  test("cache-hit", async () => {
+    const e: Expectation = { id: "h", type: "cache-hit", domain: "e.com", host: "www.e.com", paths: ["/", "/a"] }
+    expect((await runExpectation(e, answering("HIT\n"))).state).toBe("ok")
+    expect(await runExpectation(e, answering("MISS\n"))).toMatchObject({ state: "flag" })
+    expect((await runExpectation(e, answering(null))).state).toBe("unverified")
+  })
+
+  test("warmer", async () => {
+    const e: Expectation = { id: "w", type: "warmer", domain: "e.com", logGlob: "~/w-*.log" }
+    expect((await runExpectation(e, answering("12\n"))).state).toBe("ok")
+    expect(await runExpectation(e, answering("0\n"))).toMatchObject({ state: "flag", text: "WARMER IDLE: 0 queued" })
+  })
+
+  test("status", async () => {
+    const e: Expectation = { id: "s", type: "status", url: "https://e.com/x", expect: 410 }
+    expect((await runExpectation(e, answering(null, 410))).state).toBe("ok")
+    expect((await runExpectation(e, answering(null, 200))).state).toBe("flag")
+    expect((await runExpectation(e, answering(null, null))).state).toBe("unverified")
+  })
+})
+
+describe("readApt", () => {
+  const now = 1_800_000_000
+  test("healthy", () => expect(readApt(`${now - 3600} ${now} inactive 0`)).toEqual({ flag: null }))
+  test("stale", () => expect(readApt(`${now - 10 * 86_400} ${now} inactive 0`)!.flag).toContain("STALE: last successful update 10d ago"))
+  test("no stamp", () => expect(readApt(`none ${now} inactive 0`)!.flag).toContain("no successful update"))
+  test("stuck", () => expect(readApt(`${now} ${now} activating ${now - 3 * 86_400}`)!.flag).toContain("STUCK: apt-daily running 3d"))
+  test("unreadable", () => expect(readApt(null)).toBeNull())
+})

package/src/lib/sentinelExpect.ts

@@ -0,0 +1,197 @@
+// Expectation checks: what uptime monitoring can't see. A crawler that stopped
+// arriving, a cache that stopped hitting, a box that latches under load. They run
+// with the digest, read-only through ssh-exec, and report on the previous full
+// day: at 08:00 today's log is only a few hours old and would read as a collapse.
+//
+// A failed probe is "unverified", never a pass and never a flag. Commands avoid a
+// bare `>` and multi-parameter URLs, which the ssh-exec policy reads as a redirect
+// and a shell operator (docs/2026-10-30_sentinel-spec.md, "Expectations").
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"
+import { dirname } from "node:path"
+import { parseDate } from "./schedule.ts"
+import { expectationsPath, trendsPath, writeJson } from "./sentinelState.ts"
+
+interface Base {
+  id: string
+  domain?: string
+  server?: string // when the domain exists on more than one server
+  label?: string
+  note?: string
+  expires?: string
+}
+
+export type Expectation = Base &
+  (
+    | { type: "log-count"; domain: string; match: string; min?: number; trend?: boolean }
+    | { type: "load-shape"; domain: string; threshold: number }
+    | { type: "cache-hit"; domain: string; host: string; paths: string[] }
+    | { type: "warmer"; domain: string; logGlob: string }
+    | { type: "status"; url: string; expect: number }
+  )
+
+export const EXPECT_TYPES = ["log-count", "load-shape", "cache-hit", "warmer", "status"] as const
+const ID_RE = /^[a-z0-9][a-z0-9-]{1,79}$/
+const DOMAIN_RE = /^[a-z0-9.-]+$/i
+const LATCHED_SAMPLES = 3
+// `sar` writes a sample every 10 minutes.
+const SAMPLE_MIN = 10
+
+export function loadExpectations(): Expectation[] {
+  try {
+    return JSON.parse(readFileSync(expectationsPath(), "utf8")) as Expectation[]
+  } catch {
+    return []
+  }
+}
+
+export const saveExpectations = (list: Expectation[]) => writeJson(expectationsPath(), list)
+
+// Everything here ends up inside a remote shell string, so each value is held to
+// what it needs to be rather than escaped and hoped for.
+export function validateExpectation(e: Expectation): string[] {
+  const p: string[] = []
+  if (!ID_RE.test(e.id ?? "")) p.push("id must be lowercase letters, digits and hyphens")
+  if (!(EXPECT_TYPES as readonly string[]).includes(e.type)) p.push(`type must be one of ${EXPECT_TYPES.join(", ")}`)
+  if (e.expires) {
+    try {
+      parseDate(e.expires)
+    } catch (err) {
+      p.push((err as Error).message)
+    }
+  }
+  if (e.type !== "status" && !DOMAIN_RE.test(e.domain ?? "")) p.push("domain is required")
+  if (e.type === "log-count") {
+    if (!e.match) p.push("match is required")
+    if (/["$`\\\n>]/.test(e.match ?? "")) p.push('match can\'t contain " $ ` \\ or >')
+    if (e.min !== undefined && !(e.min >= 0)) p.push("min must be a number")
+  }
+  if (e.type === "load-shape" && !(e.threshold > 0)) p.push("threshold must be a positive number")
+  if (e.type === "cache-hit") {
+    if (!DOMAIN_RE.test(e.host ?? "")) p.push("host is required: the site's canonical host")
+    if (!e.paths?.length) p.push("paths is required")
+    for (const path of e.paths ?? []) if (!/^\/[\w./?=%-]*$/.test(path) || path.includes("&")) p.push(`${path}: a path may carry one query parameter and no shell characters`)
+  }
+  if (e.type === "warmer" && !/^[\w./~*-]+$/.test(e.logGlob ?? "")) p.push("logGlob is required, as a plain path with * wildcards")
+  if (e.type === "status") {
+    if (!/^https?:\/\//.test(e.url ?? "")) p.push("url is required")
+    if (!(e.expect >= 100)) p.push("expect is required: the status code")
+  }
+  return p
+}
+
+// ---- the remote commands ----------------------------------------------------------------
+
+// grep -c exits 1 on a zero count, so callers read stdout rather than the exit code.
+export const logCountCommand = (match: string) => `zcat ~/logs/access.log-$(date +%Y%m%d).gz 2>/dev/null | grep -cF "${match}"`
+
+// `!(x < T)` and `if (mx < x)` rather than `x >= T`: the guard reads a bare > as a
+// redirect and denies the whole command.
+export const loadShapeCommand = (threshold: number) =>
+  String.raw`LC_ALL=C sar -q -f /var/log/sysstat/sa$(date -d yesterday +%d) 2>/dev/null | awk -v T=${threshold} "\$4 ~ /^[0-9.]+\$/ {n++; if (!(\$4+0 < T)) {run++; over++; if (mxrun < run) mxrun=run} else {run=0}; if (mx < \$4+0) mx=\$4} END {printf \"%s %d %d\\n\", (mx==\"\"?0:mx), over+0, mxrun+0}"`
+
+// No `-o /dev/null`: the guard reads an output flag as a redirect, so the body
+// falls through and grep picks the header out of it.
+export const cacheProbeCommand = (host: string, path: string) =>
+  String.raw`for i in 1 2; do curl -s -D - --max-time 15 --resolve ${host}:443:127.0.0.1 "https://${host}${path}"; done | grep -i "^fastcgi-cache:" | tail -1 | awk "{print \$2}"`
+
+export const warmerCommand = (glob: string) => `L=$(ls -t ${glob} 2>/dev/null | head -1); [ -n "$L" ] && grep -hoE "starting: [0-9]+ URLs queued" "$L" | grep -oE "[0-9]+" | head -1`
+
+export const APT_COMMAND =
+  'echo $(stat -c %Y /var/lib/apt/periodic/update-success-stamp 2>/dev/null || echo none) $(date +%s) $(systemctl show apt-daily.service -p ActiveState --value) $(date -d "$(systemctl show apt-daily.service -p StateChangeTimestamp --value)" +%s 2>/dev/null || echo 0)'
+
+// ---- running one ---------------------------------------------------------------------------
+
+export type ExpectState = "ok" | "flag" | "unverified"
+
+export interface ExpectResult {
+  id: string
+  label: string
+  state: ExpectState
+  text: string
+  trend?: string
+}
+
+export interface ExpectDeps {
+  // stdout, or null when the probe itself failed.
+  exec(domain: string, command: string, server?: string): Promise<string | null>
+  status(url: string): Promise<number | null>
+}
+
+const labelOf = (e: Expectation) => e.label ?? e.id
+const num = (s: string | null) => (s !== null && /^\d+$/.test(s.trim()) ? Number(s.trim()) : null)
+
+export async function runExpectation(e: Expectation, deps: ExpectDeps): Promise<ExpectResult> {
+  const r = (state: ExpectState, text: string, trend?: string): ExpectResult => ({ id: e.id, label: labelOf(e), state, text, ...(trend !== undefined ? { trend } : {}) })
+  switch (e.type) {
+    case "log-count": {
+      const n = num(await deps.exec(e.domain, logCountCommand(e.match), e.server))
+      if (n === null) return r("unverified", "no count returned")
+      const trend = e.trend ? String(n) : undefined
+      if (e.min !== undefined && n < e.min) return r("flag", `LOW: ${n} yesterday (expected ≥${e.min})`, trend)
+      return r("ok", String(n), trend)
+    }
+    case "load-shape": {
+      const out = (await deps.exec(e.domain, loadShapeCommand(e.threshold), e.server))?.trim().split(/\s+/)
+      if (out?.length !== 3 || !/^[\d.]+$/.test(out[0]!) || !/^\d+$/.test(out[1]!) || !/^\d+$/.test(out[2]!)) return r("unverified", "no sar data")
+      const [max, over, run] = [out[0]!, Number(out[1]), Number(out[2])]
+      const shape = `load max ${max}, ${over} over ${e.threshold}, longest ${run * SAMPLE_MIN} min`
+      return run >= LATCHED_SAMPLES ? r("flag", `LATCHED: ${shape}`) : r("ok", shape)
+    }
+    case "cache-hit": {
+      const bad: string[] = []
+      let probes = 0
+      for (const path of e.paths) {
+        const got = (await deps.exec(e.domain, cacheProbeCommand(e.host, path), e.server))?.trim()
+        if (got) probes++
+        if (got?.toUpperCase() !== "HIT") bad.push(`${path}=${got || "no answer"}`)
+      }
+      if (!probes) return r("unverified", "no answer from any path")
+      return bad.length ? r("flag", `CACHE REGRESSED: ${bad.join(", ")}`) : r("ok", `${e.paths.length} path(s) HIT`)
+    }
+    case "warmer": {
+      const n = num(await deps.exec(e.domain, warmerCommand(e.logGlob), e.server))
+      if (n === null) return r("unverified", "no warmer log found")
+      return n === 0 ? r("flag", "WARMER IDLE: 0 queued") : r("ok", `${n} queued`)
+    }
+    case "status": {
+      const code = await deps.status(e.url)
+      if (code === null) return r("unverified", "no response")
+      return code === e.expect ? r("ok", String(code)) : r("flag", `${e.url} returned ${code} (expected ${e.expect})`)
+    }
+  }
+}
+
+export function appendTrends(results: ExpectResult[], date: string): void {
+  const rows = results.filter((r) => r.trend !== undefined).map((r) => `${date},${r.id},${r.trend}`)
+  if (!rows.length) return
+  const path = trendsPath()
+  mkdirSync(dirname(path), { recursive: true })
+  if (!existsSync(path)) appendFileSync(path, "date,id,value\n")
+  appendFileSync(path, rows.join("\n") + "\n")
+}
+
+// ---- apt freshness, for every server -----------------------------------------------------------
+
+const APT_STALE_DAYS = 3
+const APT_STUCK_HOURS = 6
+
+// Catches a server whose updates quietly stopped: a daily `apt-get update` that
+// hung and held the lock for over a year once, with the sites up and nothing
+// paging. null means the output didn't parse.
+export function readApt(output: string | null): { flag: string | null } | null {
+  const [stamp, nowS, state, since] = (output ?? "").trim().split(/\s+/)
+  if (!/^\d+$/.test(nowS ?? "")) return null
+  const now = Number(nowS)
+  const flags: string[] = []
+  if (!stamp || stamp === "none") flags.push("STALE: no successful update on record")
+  else {
+    const days = Math.floor((now - Number(stamp)) / 86_400)
+    if (days > APT_STALE_DAYS) flags.push(`STALE: last successful update ${days}d ago`)
+  }
+  if (state === "activating" && /^[1-9]\d*$/.test(since ?? "")) {
+    const hours = Math.floor((now - Number(since)) / 3600)
+    if (hours > APT_STUCK_HOURS) flags.push(`STUCK: apt-daily running ${hours > 48 ? `${Math.floor(hours / 24)}d` : `${hours}h`}`)
+  }
+  return { flag: flags.length ? flags.join("; ") : null }
+}