spinuptui 0.20.0 → 0.21.0

package/src/lib/cliIncidents.ts

@@ -0,0 +1,126 @@
+// `spinuptui incidents <domain>` / `spinuptui incidents --all` — surfaces
+// Uptime Kuma's down/up incident history for the fleet SpinupTUI itself
+// manages monitoring for, for external tooling (e.g. an incident-diagnostics
+// agent polling on its own schedule). Scoped to config.kumaMonitors, not
+// Kuma's full live monitor list — a monitor added by hand in Kuma with no
+// corresponding SpinupWP site would have nowhere actionable for a caller to
+// go, so it's deliberately excluded.
+
+import type { AppConfig, KumaMonitorRef } from "../config.ts"
+import { withKuma, KumaError, type KumaBeat } from "./uptimeKuma.ts"
+
+export type IncidentKind = "health" | "push" | "redis" | "fingerprint" | "fatal" | "bypass"
+export type IncidentStatus = "down" | "up" | "pending" | "maintenance"
+
+export interface IncidentEvent {
+  domain: string
+  kind: IncidentKind
+  status: IncidentStatus
+  time: string
+  message: string
+}
+
+export type IncidentsResult =
+  | { ok: true; windowHours: number; events: IncidentEvent[] }
+  | { ok: false; reason: "kuma_not_configured" | "no_monitors_registered" | "kuma_error"; message: string }
+
+const STATUS_LABEL: Record<number, IncidentStatus> = { 0: "down", 1: "up", 2: "pending", 3: "maintenance" }
+
+const KIND_FIELDS: Array<[keyof KumaMonitorRef, IncidentKind]> = [
+  ["healthId", "health"],
+  ["pushId", "push"],
+  ["redisId", "redis"],
+  ["fingerprintId", "fingerprint"],
+  ["fatalId", "fatal"],
+  ["bypassId", "bypass"],
+]
+
+interface Target {
+  domain: string
+  kind: IncidentKind
+  id: number
+}
+
+export async function resolveIncidents(
+  cfg: AppConfig,
+  opts: { domain?: string; hours: number },
+): Promise<IncidentsResult> {
+  if (!cfg.uptimeKuma) {
+    return {
+      ok: false,
+      reason: "kuma_not_configured",
+      message: "No Uptime Kuma connection configured in SpinupTUI.",
+    }
+  }
+
+  const domains = opts.domain ? [opts.domain] : Object.keys(cfg.kumaMonitors)
+  const targets: Target[] = []
+  for (const domain of domains) {
+    const ref = cfg.kumaMonitors[domain]
+    if (!ref) continue
+    for (const [field, kind] of KIND_FIELDS) {
+      const id = ref[field]
+      if (typeof id === "number") targets.push({ domain, kind, id })
+    }
+  }
+
+  if (targets.length === 0) {
+    return {
+      ok: false,
+      reason: "no_monitors_registered",
+      message: opts.domain
+        ? `No Kuma monitors registered for "${opts.domain}" in SpinupTUI.`
+        : "No Kuma monitors registered for any site in SpinupTUI.",
+    }
+  }
+
+  try {
+    const { result } = await withKuma(cfg.uptimeKuma, async (kuma) => {
+      const perTarget = await Promise.all(
+        targets.map(async (t) => {
+          try {
+            return { t, beats: await kuma.getMonitorBeats(t.id, opts.hours) }
+          } catch {
+            return { t, beats: [] as KumaBeat[] }
+          }
+        }),
+      )
+      const events: IncidentEvent[] = []
+      for (const { t, beats } of perTarget) {
+        for (const b of beats) {
+          if (!b.important) continue
+          const status = STATUS_LABEL[b.status] ?? "pending"
+          const message = b.msg || ""
+          // The fatal sentinel is registered under the server's vanity domain,
+          // not the site that's actually broken — a down beat pushed BY OUR
+          // OWN cron carries the real, comma-separated affected domain(s) as
+          // msg instead. But Kuma can ALSO generate its own synthetic down
+          // beats on this same push monitor (e.g. "No heartbeat in the time
+          // window" if the cron itself stops beating) — those are an
+          // infrastructure signal about the server, not a parseable domain
+          // list, and must NOT be mistaken for one (confirmed live: this
+          // exact message has no commas and isn't a hostname). Only messages
+          // that parse cleanly as a comma-separated list of hostname-shaped
+          // tokens get split into per-domain events; anything else — Kuma's
+          // own wording included — stays tagged to the vanity domain like
+          // every other kind.
+          const domainList = message.split(",").map((d) => d.trim())
+          const looksLikeDomainList = domainList.every((d) => /^[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)+$/.test(d))
+          if (t.kind === "fatal" && status === "down" && looksLikeDomainList) {
+            for (const domain of domainList) {
+              events.push({ domain, kind: t.kind, status, time: b.time, message })
+            }
+            continue
+          }
+          events.push({ domain: t.domain, kind: t.kind, status, time: b.time, message })
+        }
+      }
+      events.sort((a, b) => a.time.localeCompare(b.time))
+      return events
+    })
+    return { ok: true, windowHours: opts.hours, events: result }
+  } catch (err) {
+    const message = err instanceof KumaError ? err.message : `Unexpected error: ${(err as Error).message}`
+    return { ok: false, reason: "kuma_error", message }
+  }
+}

package/src/lib/cliSsh.ts

@@ -0,0 +1,161 @@
+// `spinuptui ssh <domain>` — non-interactive SSH access lookup for external
+// tooling (e.g. an incident-diagnostics agent that's handed only a domain).
+// Resolves domain -> site -> server -> SSH target, then runs a live
+// connectivity probe (BatchMode, same as probeSite) so the result is a
+// trustworthy go/no-go rather than a guess.
+
+import type { AppConfig } from "../config.ts"
+import type { SpinupWPClientLike } from "../api/client.ts"
+import { ApiError } from "../api/client.ts"
+import { resolveSiteSshTarget, SSH_OPTS } from "./probe.ts"
+
+export type SshAccessReason =
+  | "no_token"
+  | "token_rejected"
+  | "site_not_found"
+  | "multiple_matches"
+  | "server_has_no_ip"
+  | "permission_denied"
+  | "connection_failed"
+  | "ssh_error"
+  | "api_error"
+
+export interface SshAccessCandidate {
+  siteId: number
+  serverId: number
+  server: string
+}
+
+export type SshAccessResult =
+  | { ok: true; domain: string; primaryDomain: string; sshTarget: string; port: number | null; server: string }
+  | {
+      ok: false
+      domain: string
+      reason: SshAccessReason
+      message: string
+      remedy?: string
+      candidates?: SshAccessCandidate[]
+    }
+
+const GRANT_KEY_REMEDY =
+  "Run `spinuptui`, select this site in the Browser view, and press K to grant this device's key (sudo must be connected on the server first — press S)."
+
+export async function resolveSshAccess(
+  domain: string,
+  client: SpinupWPClientLike,
+  cfg: AppConfig,
+): Promise<SshAccessResult> {
+  if (!cfg.token) {
+    return { ok: false, domain, reason: "no_token", message: "No API token configured. Run `spinuptui login`." }
+  }
+
+  let site
+  let server
+  try {
+    const sites = await client.listSites()
+    const matches = sites.filter(
+      (s) => s.domain === domain || s.additional_domains?.some((a) => a.domain === domain),
+    )
+    if (matches.length === 0) {
+      return {
+        ok: false,
+        domain,
+        reason: "site_not_found",
+        message: `No site matching "${domain}" found in this SpinupWP account.`,
+      }
+    }
+    if (matches.length > 1) {
+      const candidates: SshAccessCandidate[] = await Promise.all(
+        matches.map(async (s) => {
+          const srv = await client.getServer(s.server_id)
+          return { siteId: s.id, serverId: s.server_id, server: srv.name }
+        }),
+      )
+      return {
+        ok: false,
+        domain,
+        reason: "multiple_matches",
+        message: `"${domain}" matches ${matches.length} sites in this account — cannot pick one automatically.`,
+        candidates,
+      }
+    }
+    site = matches[0]
+    server = await client.getServer(site.server_id)
+  } catch (err) {
+    if (err instanceof ApiError && err.status === 401) {
+      return {
+        ok: false,
+        domain,
+        reason: "token_rejected",
+        message: "Saved API token was rejected (401) — it may be expired or revoked.",
+        remedy: "Run `spinuptui login` to save a fresh token.",
+      }
+    }
+    const message = err instanceof ApiError ? err.message : `Unexpected error: ${(err as Error).message}`
+    return { ok: false, domain, reason: "api_error", message }
+  }
+
+  if (!server.ip_address) {
+    return {
+      ok: false,
+      domain,
+      reason: "server_has_no_ip",
+      message: `Server "${server.name}" has no IP address on file.`,
+    }
+  }
+
+  const target = resolveSiteSshTarget(site, server, cfg.sshUser)
+  const port = server.ssh_port && server.ssh_port !== 22 ? server.ssh_port : null
+  const portOpt = port ? ["-p", String(port)] : []
+
+  let proc: ReturnType<typeof Bun.spawn>
+  try {
+    proc = Bun.spawn(["ssh", ...SSH_OPTS, ...portOpt, target, "true"], {
+      stdout: "pipe",
+      stderr: "pipe",
+      stdin: "ignore",
+    })
+  } catch (err) {
+    return { ok: false, domain, reason: "ssh_error", message: `Failed to launch ssh: ${(err as Error).message}` }
+  }
+
+  const timeout = setTimeout(() => {
+    try {
+      proc.kill()
+    } catch {
+      /* already gone */
+    }
+  }, 15000)
+
+  const exitCode = await proc.exited
+  clearTimeout(timeout)
+  const stderr = await new Response(proc.stderr as ReadableStream<Uint8Array>).text()
+
+  if (exitCode === 0) {
+    return { ok: true, domain, primaryDomain: site.domain, sshTarget: target, port, server: server.name }
+  }
+
+  if (/permission denied/i.test(stderr)) {
+    return {
+      ok: false,
+      domain,
+      reason: "permission_denied",
+      message: stderr.trim().split("\n").slice(-2).join(" ") || "Permission denied.",
+      remedy: GRANT_KEY_REMEDY,
+    }
+  }
+  if (/(connection timed out|operation timed out|connection refused|could not resolve)/i.test(stderr)) {
+    return {
+      ok: false,
+      domain,
+      reason: "connection_failed",
+      message: stderr.trim().split("\n").slice(-2).join(" ") || `ssh exited with code ${exitCode}`,
+    }
+  }
+  return {
+    ok: false,
+    domain,
+    reason: "ssh_error",
+    message: stderr.trim().split("\n").slice(-2).join(" ") || `ssh exited with code ${exitCode}`,
+  }
+}

package/src/lib/probe.ts

@@ -60,7 +60,7 @@ export function resolveSiteSshTarget(site: Site, server: Server | undefined, ssh
   return `${user}@${ip}`
 }
 
-const SSH_OPTS = [
+export const SSH_OPTS = [
   "-o", "BatchMode=yes",
   "-o", "ConnectTimeout=7",
   "-o", "StrictHostKeyChecking=accept-new",

package/src/lib/ssh.ts

@@ -444,6 +444,52 @@ async function runSudoSiteOp(
   return { ok: true, target: `${siteUser}@${ip}` }
 }
 
+export type SudoServerResult = { ok: true } | { ok: false; error: string }
+
+// Same probe → maybe-password → `sudo -S -p '' bash -s` idiom as runSudoSiteOp,
+// but scoped to the SERVER as root — no Site/site_user involved. Used for ops
+// that need to read/write across every site's directory (e.g. the fatal-log
+// monitor's cron, which globs /sites/* — a regular site user can't even list
+// that directory, confirmed live).
+export async function runSudoServerOp(
+  server: Server,
+  opts: { sudoUser: string; sudoPassword: string },
+  build: () => string,
+  marker: string,
+): Promise<SudoServerResult> {
+  const ip = server.ip_address
+  if (!ip) return { ok: false, error: "Server has no IP address." }
+  const target = `${opts.sudoUser}@${ip}`
+  const port = server.ssh_port ?? null
+
+  const probe = await runSshStdin(target, port, "sudo -n true 2>&1", null, 15000)
+  let needPassword = false
+  if (probe.code === 0) {
+    needPassword = false
+  } else if (probe.code === 255 || probe.code === -1) {
+    return { ok: false, error: lastErrorLine(probe.stdout + "\n" + probe.stderr, "Couldn't connect over SSH — is your key on the server for this user?") }
+  } else {
+    const out = (probe.stdout + " " + probe.stderr).toLowerCase()
+    if (/not in the sudoers|not allowed to run|may not run|unknown user|is not allowed/.test(out)) {
+      return { ok: false, error: `${opts.sudoUser} can't run sudo on this server.` }
+    }
+    needPassword = true
+    if (!opts.sudoPassword) return { ok: false, error: "A sudo password is required for this server." }
+  }
+
+  const script = build()
+  const payload = needPassword ? `${opts.sudoPassword}\n${script}\n` : `${script}\n`
+  const res = await runSshStdin(target, port, "sudo -S -p '' bash -s", payload, 30000)
+  if (res.code !== 0 || !res.stdout.includes(marker)) {
+    const out = (res.stderr + " " + res.stdout).toLowerCase()
+    if (/incorrect password|sorry, try again|authentication failure/.test(out)) {
+      return { ok: false, error: "Sudo password was rejected." }
+    }
+    return { ok: false, error: lastErrorLine(res.stderr || res.stdout, `The remote command failed (ssh exit ${res.code}).`) }
+  }
+  return { ok: true }
+}
+
 // Append the given public keys to a site user's authorized_keys, via the server's
 // sudo user. Idempotent (re-running never duplicates a line). The caller resolves
 // which keys to deploy (machine key via ensureSpinupKey, personal via listPersonalKeys).

package/src/lib/uptimeKuma.ts

@@ -41,6 +41,7 @@ export interface KumaBeat {
   time: string
   msg?: string
   ping?: number | null
+  important?: boolean // true on a status-transition row (Kuma's own "important events" flag)
 }
 
 // A notification provider as configured in Kuma (Settings → Notifications) —
@@ -403,6 +404,27 @@ export function redisPushMonitorPayload(name: string, pushToken: string): Record
   }
 }
 
+// The server-wide PHP-fatal sentinel: one per server (like the Redis sentinel),
+// fed by a root-level cron that greps every site's error.log/debug.log for new
+// "PHP Fatal error" lines — catches PHP dying behind a still-warm page cache,
+// which the plain up/down and fingerprint monitors sleep straight through.
+// Unlike every other monitor kind, a down beat's `msg` carries a comma-separated
+// list of the actual site domain(s) that fataled (this monitor's own domain is
+// just the server's vanity anchor, not the site that's actually broken).
+export function fatalPushMonitorPayload(name: string, pushToken: string): Record<string, unknown> {
+  return {
+    type: "push",
+    name,
+    pushToken,
+    description: "Server-wide PHP fatal sentinel: a root cron greps every site's error/debug log each minute for new PHP fatals and reports up/down. `msg` on a down beat lists the affected domain(s).",
+    interval: 60,
+    retryInterval: 60,
+    maxretries: 2,
+    upsideDown: false,
+    accepted_statuscodes: ["200-299"],
+  }
+}
+
 // The front-page fingerprint monitor (site monitoring Phase 1): a `keyword`
 // monitor asserting the front page still carries the template-identity string
 // calibration derived (lib/siteFingerprint.ts). Catches "the page cache is
@@ -453,6 +475,53 @@ export async function registerFingerprintMonitor(
   return kuma.addMonitor(fingerprintMonitorPayload(name, url, opts.keyword, opts.interval))
 }
 
+// The cache-bypass monitor (site monitoring blind-spot fix, part 2): a plain
+// http monitor on the front page, but with a Cookie header that forces
+// SpinupWP's nginx to bypass the page cache — the same bypass siteDoctor.ts
+// already uses and has verified live. Unlike the health/fingerprint monitors,
+// this actually exercises PHP on every check, so it's deliberately opt-in
+// (not automatic) and meant for a much looser interval than either of those.
+export function bypassMonitorPayload(name: string, url: string, interval: number): Record<string, unknown> {
+  return {
+    type: "http",
+    name,
+    url,
+    method: "GET",
+    interval,
+    retryInterval: 60,
+    resendInterval: 0,
+    maxretries: 2,
+    timeout: 48,
+    accepted_statuscodes: ["200-299"],
+    expiryNotification: false, // the plain health monitor already owns cert expiry
+    ignoreTls: false,
+    upsideDown: false,
+    maxredirects: 10,
+    headers: JSON.stringify({ Cookie: "wordpress_no_cache=1" }),
+  }
+}
+
+// Adopt-or-EDIT-or-create, same philosophy as registerFingerprintMonitor:
+// recalibrating the interval updates the existing row in place so history and
+// notification wiring survive.
+export async function registerBypassMonitor(
+  kuma: KumaClient,
+  domain: string,
+  opts: { proto: "http" | "https"; interval: number; knownId?: number | null },
+): Promise<number> {
+  const name = `${domain} cache-bypass`
+  const url = `${opts.proto}://${domain}/`
+  const monitors = await kuma.getMonitors()
+  const byId = new Map(monitors.map((m) => [m.id, m]))
+  const existingId = (opts.knownId != null && byId.has(opts.knownId) ? opts.knownId : undefined) ?? monitors.find((m) => m.name === name)?.id
+  if (existingId != null) {
+    const full = await kuma.getMonitor(existingId)
+    await kuma.editMonitor({ ...full, type: "http", url, interval: opts.interval, headers: JSON.stringify({ Cookie: "wordpress_no_cache=1" }) })
+    return existingId
+  }
+  return kuma.addMonitor(bypassMonitorPayload(name, url, opts.interval))
+}
+
 // Adopt-or-create the monitors for one domain — THE single implementation shared
 // by the vanity wizard's monitor step and the `m` overlay's add/repair, so the
 // two paths cannot drift. Rules:
@@ -464,8 +533,24 @@ export async function registerFingerprintMonitor(
 export async function registerMonitors(
   kuma: KumaClient,
   domain: string,
-  opts: { proto: "http" | "https"; healthzPath: string; wantPush: boolean; wantRedis?: boolean; known?: KumaMonitorRef | null; pushToken?: string | null },
-): Promise<{ healthId: number; pushId?: number; pushToken?: string; redisId?: number; redisToken?: string }> {
+  opts: {
+    proto: "http" | "https"
+    healthzPath: string
+    wantPush: boolean
+    wantRedis?: boolean
+    wantFatal?: boolean
+    known?: KumaMonitorRef | null
+    pushToken?: string | null
+  },
+): Promise<{
+  healthId: number
+  pushId?: number
+  pushToken?: string
+  redisId?: number
+  redisToken?: string
+  fatalId?: number
+  fatalToken?: string
+}> {
   const monitors = await kuma.getMonitors()
   const byId = new Map(monitors.map((m) => [m.id, m]))
   const byName = new Map(monitors.map((m) => [m.name, m]))
@@ -482,7 +567,12 @@ export async function registerMonitors(
   const existingRedis = opts.wantRedis && live(known.redisId) == null ? byName.get(`${domain} redis`) : undefined
   if (existingRedis?.pushToken) redisToken = existingRedis.pushToken
 
-  const [healthId, pushId, redisId] = await Promise.all([
+  // Same independent-token story as Redis — its own push URL, its own cron line.
+  let fatalToken = known.fatalToken ?? genPushToken()
+  const existingFatal = opts.wantFatal && live(known.fatalId) == null ? byName.get(`${domain} php-fatal`) : undefined
+  if (existingFatal?.pushToken) fatalToken = existingFatal.pushToken
+
+  const [healthId, pushId, redisId, fatalId] = await Promise.all([
     (async () => live(known.healthId) ?? byName.get(domain)?.id ?? (await kuma.addMonitor(healthMonitorPayload(domain, `${opts.proto}://${domain}${opts.healthzPath}`))))(),
     (async () => {
       if (!opts.wantPush) return undefined
@@ -492,6 +582,10 @@ export async function registerMonitors(
       if (!opts.wantRedis) return undefined
       return live(known.redisId) ?? existingRedis?.id ?? (await kuma.addMonitor(redisPushMonitorPayload(`${domain} redis`, redisToken)))
     })(),
+    (async () => {
+      if (!opts.wantFatal) return undefined
+      return live(known.fatalId) ?? existingFatal?.id ?? (await kuma.addMonitor(fatalPushMonitorPayload(`${domain} php-fatal`, fatalToken)))
+    })(),
   ])
   return {
     healthId,
@@ -499,6 +593,8 @@ export async function registerMonitors(
     pushToken: opts.wantPush ? pushToken : undefined,
     redisId,
     redisToken: opts.wantRedis ? redisToken : undefined,
+    fatalId,
+    fatalToken: opts.wantFatal ? fatalToken : undefined,
   }
 }
 

package/src/lib/vanitySite.ts

@@ -6,8 +6,10 @@
 // docs/vanity-site/index.php is the design reference; keep the two in sync.
 
 import { resolve4 } from "node:dns/promises"
+import type { Server } from "../api/types.ts"
 import { SSH_OPTS, sshPort, runProcess, meaningfulError, remoteDocRoot } from "./dbBackup.ts"
 import { pushUrl, LOAD_PUSH_SCALE } from "./uptimeKuma.ts"
+import { runSudoServerOp } from "./ssh.ts"
 
 // The vanity convention: the site living at the server's own hostname. Every
 // feature that special-cases vanity sites (full monitoring, page re-seed) keys
@@ -295,3 +297,75 @@ export async function seedVanityPushCron(t: PushCronTarget): Promise<{ ok: boole
   if (r.code !== 0) return { ok: false, error: meaningfulError(r.stderr, "Couldn't install the heartbeat cron over SSH.") }
   return { ok: true }
 }
+
+const FATAL_CRON_MARKER = "spinup-kuma-fatal"
+const FATAL_SCRIPT_PATH = "/root/spinup-fatal-check.sh"
+const FATAL_INSTALLED_MARKER = "===FATALCRON_INSTALLED"
+
+// Server-wide PHP-fatal sentinel (site monitoring blind-spot fix): SpinupWP's
+// page cache keeps serving `/` even while PHP is fataling on anything the
+// cache doesn't already have — the plain up/down and fingerprint monitors
+// never touch broken PHP as long as the cache stays warm (verified against a
+// real 2026-07-07 incident that sat undetected for 54 of its 68 minutes).
+//
+// Runs as ROOT (via sudo) because it globs every site on the box: a site's
+// own user can't even list `/sites` (confirmed live), let alone read a
+// sibling site's logs once logrotate recreates them `640 root adm`. One
+// monitor covers the whole server rather than one per site, to avoid Kuma
+// "monitor bloat" — a down beat's `msg` carries the actual affected
+// domain(s), since THIS monitor's own name is just the server's vanity
+// anchor, not the site that's actually broken.
+//
+// A PHP fatal can land in either logs/error.log (nginx) or logs/debug.log
+// (WordPress's own log, populated when WP_DEBUG_LOG is on) depending on that
+// site's config — verified live: the real incident above is in debug.log,
+// not error.log. Both are checked for every site.
+const FATAL_CHECK_SCRIPT = `#!/bin/bash
+STATE_DIR=/root/.spinup-fatal-state
+mkdir -p "$STATE_DIR"
+FOUND=""
+for site_dir in /sites/*/; do
+  domain=$(basename "$site_dir")
+  for logname in error.log debug.log; do
+    logfile="\${site_dir}logs/\${logname}"
+    [ -f "$logfile" ] || continue
+    state_file="$STATE_DIR/\${domain}__\${logname}.offset"
+    prev=$(cat "$state_file" 2>/dev/null || echo 0)
+    cur=$(stat -c %s "$logfile" 2>/dev/null || echo 0)
+    [ "$cur" -lt "$prev" ] && prev=0
+    if [ "$cur" -gt "$prev" ] && tail -c +"$((prev + 1))" "$logfile" | grep -q "PHP Fatal error"; then
+      FOUND="\${FOUND:+$FOUND,}$domain"
+    fi
+    echo "$cur" > "$state_file"
+  done
+done
+if [ -n "$FOUND" ]; then
+  curl -fsS --max-time 20 "__PUSH_URL__?status=down&msg=$FOUND" >/dev/null 2>&1
+else
+  curl -fsS --max-time 20 "__PUSH_URL__?status=up&msg=fatal-check-ok" >/dev/null 2>&1
+fi
+`
+
+// Adopt-or-refresh: writes the check script (push URL baked in) and installs
+// the one-line-per-minute crontab entry that invokes it, in ROOT's crontab.
+// Idempotent, same strip-marked-line-then-append idiom as seedVanityPushCron.
+export async function seedFatalMonitorCron(
+  server: Server,
+  opts: { sudoUser: string; sudoPassword: string; kumaUrl: string; pushToken: string },
+): Promise<{ ok: boolean; error?: string }> {
+  const push = pushUrl(opts.kumaUrl, opts.pushToken)
+  const script = FATAL_CHECK_SCRIPT.replace(/__PUSH_URL__/g, push)
+  const scriptB64 = Buffer.from(script, "utf8").toString("base64")
+  return runSudoServerOp(
+    server,
+    { sudoUser: opts.sudoUser, sudoPassword: opts.sudoPassword },
+    () =>
+      [
+        `printf '%s' '${scriptB64}' | base64 -d > ${FATAL_SCRIPT_PATH}`,
+        `chmod +x ${FATAL_SCRIPT_PATH}`,
+        `( crontab -l 2>/dev/null | grep -v -e '${FATAL_CRON_MARKER}' ; echo '* * * * * ${FATAL_SCRIPT_PATH} >/dev/null 2>&1 # ${FATAL_CRON_MARKER}' ) | crontab -`,
+        `echo ${FATAL_INSTALLED_MARKER}`,
+      ].join("\n"),
+    FATAL_INSTALLED_MARKER,
+  )
+}

package/src/ui/Header.tsx

@@ -28,7 +28,13 @@ const SUBTITLES: Record<Route, string> = {
 
 export function Header() {
   const { route, servers, sites, loading, lastUpdated, updateInfo, newServerJob, vanityJob, cloneJob, cloneServer, kumaStatus } = useStore()
-  const kumaDown = [...kumaStatus.values()].filter((s) => s.up === false).length
+  // Name the domain(s) when there are few enough to fit the header row; fall
+  // back to a bare count once there are too many to list safely. Deliberately
+  // still just the plain up/down signal (kumaStatus.up) — not expanding what
+  // counts as "down" here, only how it's displayed.
+  const downDomains = [...kumaStatus.entries()].filter(([, s]) => s.up === false).map(([domain]) => domain)
+  const kumaDown = downDomains.length
+  const kumaDownLabel = kumaDown === 0 ? "" : kumaDown <= 2 ? `${downDomains.map((d) => truncate(d, 24)).join(", ")} down` : `${kumaDown} monitors down`
   const updateReady = updateInfo?.updateAvailable ?? false
   const building = isNewServerInFlight(newServerJob)
   // The sshkey step is the one that waits on the USER (add your key, confirm in
@@ -103,7 +109,9 @@ export function Header() {
           </box>
         )}
         {/* Uptime Kuma: only bad news earns header space — silence when all up. */}
-        {kumaDown > 0 && <text content={`  ▼ ${kumaDown} monitor${kumaDown === 1 ? "" : "s"} down  `} fg={theme.bad} style={{ flexShrink: 0 }} wrapMode="none" />}
+        {kumaDown > 0 && (
+          <text content={`  ▼ ${kumaDownLabel} — 4 to find ${kumaDown === 1 ? "it" : "them"}  `} fg={theme.bad} style={{ flexShrink: 0 }} wrapMode="none" />
+        )}
         {loading && <Spinner interval={100} />}
         <text content={`  ${servers.length} servers · ${sites.length} sites  `} fg={theme.textDim} style={{ flexShrink: 0 }} />
         <text content={lastUpdated ? `updated ${timeAgo(lastUpdated.toISOString())}` : "loading…"} fg={theme.textFaint} style={{ flexShrink: 0 }} />