rvlib-shipkit 0.38.0 → 1.0.0

package/src/core/check-runner.ts

@@ -0,0 +1,796 @@
+// THE RUNNER — one DAG over every check this repo has, whatever declared it
+// and whichever body it carries. There is no second engine: `shipkit ci` and
+// `shipkit check` differ by which checks they ASK for, never by how one runs.
+//
+// What it does, in the order a reader meets it:
+// - a check with no `deps` is a root, one with `deps` waits for them to PASS.
+//   A filtered out dep counts as satisfied, a failed or skipped dep skips its
+//   dependents transitively with a note saying why. Deps double as the race
+//   guard between two checks that touch one file.
+// - `off` is handled ONCE, by name, before anything is scheduled: the check
+//   never runs, it prints its skip line with the reason, and its row reports
+//   `skip` so no surface reads it as a green nobody measured.
+// - a REPLAY comes first, from the ledger (content keyed) or from the world
+//   cache (TTL). A replayed check runs nothing and says so on its first line.
+// - a `cmd` body is a subprocess: its whole output is streamed to a log file,
+//   its wire lines are parsed back, the exit code is truth, and it is killed
+//   with its whole process TREE at its ceiling (120s by default, whatever a
+//   build step's floor resolution gives it).
+// - a `run` body runs IN PROCESS through check.ts runCheckBody, which tags its
+//   lines with the check id and catches a throw as that check's failure. It
+//   cannot be killed, so a hanging one hangs the gate: that is the price of
+//   in-process, and `cmd` is the answer for anything that can hang. Its one
+//   parameter hangs off ONE pass per call (core/check-pass.ts), which is where
+//   a shared scan is memoised, and what it read goes to the ledger with it.
+// - `blocking: false` is demoted at the SOURCE: the FAIL lines are rewritten
+//   to warnings before anything reads them, and the real outcome rides beside
+//   them on the result, so the exit code means exactly "something that blocks
+//   you failed" and a page can still show the red.
+// - ONE tap fires per check, with what the wire lines cannot say: the real
+//   status, whether it replayed, whether it can block, what it cost.
+import { closeSync, mkdirSync, openSync, readdirSync, rmSync, writeSync } from 'fs'
+import { basename, join } from 'path'
+import { shareRun } from './attach.ts'
+import { CheckCtx } from './check-ctx.ts'
+import { CheckPass } from './check-pass.ts'
+import type { ReadSet } from './read-set.ts'
+import { RepoView } from '../ledger/repo-view.ts'
+import { parallelCap } from './parallel.ts'
+import { machinePool, SLOT_HELD_ENV, takeSlot, type SlotPool } from './slots.ts'
+import { killTree, spawnWithTimeout } from './spawn-timeout.ts'
+import { formatLine, parseWireOutput, type Line } from './wire.ts'
+import { offLine, type WhenDecl } from './when.ts'
+import { runCheckBody, type CheckDef, type CheckEnv } from '../check.ts'
+import { Ledger, stampCached } from '../ledger/ledger.ts'
+import { attachedStamp } from './outcome-words.ts'
+import type { RunLog, RunStatus } from '../ledger/run-log.ts'
+import { liveStateOf } from '../ledger/live-runs.ts'
+import { CHECK_ID_ENV } from '../ledger/suite-log.ts'
+import { rememberWorld, replayWorld, type WorldCache } from '../ledger/world-cache.ts'
+import { extractProblems } from '../problems/extract.ts'
+import { feedsProblemsPane, type Population, type Problem } from '../problems/problem-line.ts'
+
+/** the ceiling a check takes when it names none. A build step's floor
+ *  resolution hands it a longer one: a whole tool chain is not one question
+ *  about the world. */
+export const CHECK_TIMEOUT_MS = 120_000
+
+/** epoch ms that never goes backwards inside one process, whole numbers so a
+ *  row's `at - ms` is its start exactly: the wall clock can step back (an NTP
+ *  correction), and a check stamped before an earlier one ended makes the
+ *  record show more checks at once than ever ran */
+const clock = (): number => Math.floor(performance.timeOrigin + performance.now())
+
+export type CheckBody = Pick<CheckDef, 'run' | 'cmd' | 'env' | 'cwd'>
+
+/** one check, resolved: everything the runner needs and nothing it does not.
+ *  ledger/gate-checks.ts builds these from the one enumeration, so a surface
+ *  can never list a different set than the one that runs. */
+export type RunnableCheck = {
+    id: string
+    title: string
+    /** the fix pointer floor, appended after `docsBase` when a failing body
+     *  attached no precise pointer of its own */
+    owner: string
+    /** null = it never said, which the runner treats as fatal and the caller
+     *  reports as a defect */
+    blocking: boolean | null
+    /** a build step: the gate runs it, the health pass does not */
+    build: boolean
+    when: WhenDecl
+    deps: string[]
+    group: string | null
+    timeoutMs: number
+    /** the declared world TTL in ms, 0 when this check is not a world probe */
+    ttlMs: number
+    /** the rotation half of a world key (a secret hash, a config switch) */
+    rotation?: string
+    /** it rewrites the repo unless frozen, so frozen is part of its key */
+    writes?: boolean
+    body: CheckBody
+    /** an idempotent repair exists, so a fix pointer advertises it */
+    fix?: unknown
+    /** what a failing subprocess points the reader at */
+    pointer: string | null
+    /** how it is remembered (problems/problem-line.ts populationOf). Absent
+     *  reads a declared TTL as world and anything else as not */
+    population?: Population
+}
+
+export type CheckResult = {
+    id: string
+    /** the REAL outcome. A failure that cannot block still says fail here */
+    status: RunStatus
+    /** it failed and cannot block: red on its own page, never on the gate */
+    advisory: boolean
+    replayed: boolean
+    ms: number
+    exitCode: number | null
+    /** the check's own wire lines, demoted already when it cannot block */
+    lines: Line[]
+    /** its file anchored findings, for the problems pane. Empty for a world
+     *  check, whatever it printed (problems/problem-line.ts owns the rule) */
+    problems: Problem[]
+    /** the headline a failure summary reads */
+    lastLine: string
+    /** why it did not run (a dead dep, a bail, an interrupt) */
+    note: string
+    logPath: string | null
+}
+
+/** one check finishing, with the facts the wire lines drop. ONE tap: the TUI,
+ *  the hub stream and the run record all read this. */
+export type CheckEvent = {
+    id: string
+    /** the name the wire and the TUI know it by (its title) */
+    name: string
+    status: RunStatus
+    ms: number
+    replayed: boolean
+    blocking: boolean | null
+    advisory: boolean
+    lines: Line[]
+    /** file anchored findings of this verdict, measured or replayed. The
+     *  runner always sets it, absent reads as none */
+    problems?: Problem[]
+    logPath?: string | null
+    lastLine?: string
+    note?: string
+    /** it measured nothing here: another run measuring the same key answered it (core/attach.ts) */
+    attached?: boolean
+}
+
+/** which question is being asked. `gate` is `shipkit ci` (is my change
+ *  good), `health` is `shipkit check` (is this repo healthy), and the only
+ *  thing it changes is WHICH checks are asked. */
+export type Audience = 'gate' | 'health'
+
+export type RunChecksOpts = {
+    checks: RunnableCheck[]
+    root: string
+    config: CheckEnv['config']
+    audience: Audience
+    /** `release` checks are in at the full tier and out at the quick one */
+    fullTier: boolean
+    /** where a `cmd` check's whole output is streamed */
+    logDir: string
+    /** the world answers' store, null = nothing is remembered by TTL */
+    cache: WorldCache | null
+    /** the run's ledger, null = every check is measured */
+    ledger: Ledger | null
+    /** the run record, written per check, never read */
+    log: RunLog | null
+    /** extra environment for every `cmd` check of this run */
+    childEnv?: Record<string, string | undefined>
+    bail?: boolean
+    maxParallel?: number
+    /** the machine pool every measured check takes a slot from (core/slots.ts).
+     *  Omitted: this machine's pool. null: no pool, the in process cap alone */
+    pool?: SlotPool | null
+    skipGroups?: string[]
+    onlyIds?: string[]
+    withIds?: string[]
+    /** run only the checks that can BLOCK. A check declaring blocking:false
+     *  cannot change the outcome, so a release does not spend a third-party
+     *  API call to learn that a DNS provider is reachable. */
+    onlyBlocking?: boolean
+    /** the BUILD steps alone. The TUI's tasks panel owns this: it renders the
+     *  rest in its own panel, and running them twice per refresh proves the
+     *  same thing twice. */
+    buildOnly?: boolean
+    /** the gate asks the PROACTIVE checks too (`release.requireHealthy`): a
+     *  repo that wants a missing cred to stop its publish asks for exactly
+     *  those probes, and the everyday opt-out must not drop them silently. */
+    includeProactive?: boolean
+    /** ids this caller does not need. A sub-unit release skips the steps that
+     *  gate the whole repo, since the check's own gate already ran. */
+    exclude?: string[]
+    print?: (line: string) => void
+    color?: boolean
+    onCheck?: (e: CheckEvent) => void
+}
+
+export type RunChecksSummary = {
+    /** nothing that can block you failed */
+    ok: boolean
+    interrupted: boolean
+    /** every check that was asked for, in declaration order */
+    results: CheckResult[]
+    /** checks this run did not ask about, with the reason */
+    parked: string[]
+    /** checks that really failed and cannot block */
+    advisoryFailures: string[]
+    /** checks that never declared whether they block */
+    undeclared: string[]
+    /** a registration nobody can act on: the run printed these and stopped */
+    errors: string[]
+}
+
+const stripAnsi = (s: string): string => s.replace(/\x1b\[[0-9;]*m/g, '').replace(/\r/g, '')
+
+// only-mode keeps the transitive dep closure — running `--only typecheck`
+// with a lint race-guard dep must still run lint first
+function closure(checks: RunnableCheck[], ids: string[]): Set<string> {
+    const byId = new Map(checks.map((u) => [u.id, u]))
+    const keep = new Set<string>()
+    const visit = (id: string): void => {
+        if (keep.has(id)) return
+        keep.add(id)
+        for (const d of byId.get(id)?.deps ?? []) visit(d)
+    }
+    for (const id of ids) visit(id)
+    return keep
+}
+
+export type Selection = {
+    /** what this run measures */
+    selected: RunnableCheck[]
+    /** turned off by name, with the reason: never run, always reported */
+    off: RunnableCheck[]
+    /** not asked about by this run, and why */
+    parked: { id: string; text: string }[]
+    /** asked for, and it never said whether it blocks */
+    undeclared: string[]
+    errors: string[]
+}
+
+/** what this run asks about. The audience decides the shape of the question,
+ *  `--only` and `--with` name checks past any filter, and OFF is the one word
+ *  no flag lifts. */
+export function selectChecks(o: Pick<RunChecksOpts, 'checks' | 'audience' | 'fullTier' | 'skipGroups' | 'onlyIds' | 'withIds' | 'onlyBlocking' | 'buildOnly' | 'exclude' | 'includeProactive'>): Selection {
+    const errors: string[] = []
+    const ids = new Set(o.checks.map((u) => u.id))
+    if (ids.size !== o.checks.length) errors.push('duplicate check ids')
+    for (const u of o.checks) {
+        if ((u.body.cmd == null) === (u.body.run == null)) errors.push(`check ${u.id}: exactly one body required, \`run\` (in process) or \`cmd\` (argv)`)
+        for (const d of u.deps) if (!ids.has(d)) errors.push(`check ${u.id}: unknown dep '${d}'`)
+    }
+    // `--only` and `--with` address a check by the name an operator last saw
+    // it under: its id, its title, or the file a probe lives in
+    const byName = new Map<string, string>()
+    for (const u of o.checks) {
+        byName.set(u.id.toLowerCase(), u.id)
+        byName.set(u.title.toLowerCase(), u.id)
+        if (u.owner !== '') byName.set(u.owner.toLowerCase(), u.id)
+    }
+    // `--only` also takes a GROUP, every check declaring that `group`, so a
+    // family of checks runs without a hand kept list. A check name wins over
+    // a group of the same spelling
+    const byGroup = new Map<string, string[]>()
+    for (const u of o.checks) if (u.group != null) byGroup.set(u.group.toLowerCase(), [...(byGroup.get(u.group.toLowerCase()) ?? []), u.id])
+    const resolve = (raw: string, groups: boolean): string[] | null => {
+        const id = byName.get(raw.toLowerCase())
+        if (id != null) return [id]
+        return groups ? byGroup.get(raw.toLowerCase()) ?? null : null
+    }
+    const named = new Set<string>()
+    const onlyIds: string[] = []
+    for (const [raw, groups] of [...(o.onlyIds ?? []).map((r) => [r, true] as const), ...(o.withIds ?? []).map((r) => [r, false] as const)]) {
+        const ids = resolve(raw, groups)
+        if (ids == null) errors.push(`unknown check id${groups ? ' or group' : ''} '${raw}'`)
+        else for (const id of ids) { named.add(id); if (groups) onlyIds.push(id) }
+    }
+    // a dep cycle must die loud at select time, not hang the loop
+    const state = new Map<string, 0 | 1>()
+    const byId = new Map(o.checks.map((u) => [u.id, u]))
+    const walk = (id: string, trail: string[]): void => {
+        if (state.get(id) === 1) return
+        if (state.get(id) === 0) { errors.push(`dep cycle: ${[...trail, id].join(' → ')}`); return }
+        state.set(id, 0)
+        for (const d of byId.get(id)?.deps ?? []) walk(d, [...trail, id])
+        state.set(id, 1)
+    }
+    for (const u of o.checks) walk(u.id, [])
+    if (errors.length > 0) return { selected: [], off: [], parked: [], undeclared: [], errors }
+
+    const skipGroups = new Set(o.skipGroups ?? [])
+    const excluded = new Set(o.exclude ?? [])
+    const off: RunnableCheck[] = []
+    const parked: { id: string; text: string }[] = []
+    const undeclared: string[] = []
+    let selected: RunnableCheck[] = []
+    const nonBlocking = new Map<string, RunnableCheck>()
+    const take = (u: RunnableCheck): void => {
+        // a check that never said whether it blocks is a defect, not a
+        // setting, and it is named whether or not this run measures it
+        if (u.blocking == null) undeclared.push(u.id)
+        selected.push(u)
+    }
+    for (const u of o.checks) {
+        // IS IT THIS RUN'S BUSINESS AT ALL? The health pass never builds (a
+        // repo's test suite and its type checker answer "is my change good",
+        // which is the gate's question), the TUI's build panel asks for
+        // nothing else, and a sub-unit release skips what its own gate ran.
+        // A check out of scope here is not even reported as off: it is not part
+        // of this question.
+        if (!named.has(u.id)) {
+            if (o.audience === 'health' && u.build) continue
+            if (o.buildOnly === true && !u.build) continue
+            if (excluded.has(u.id)) { parked.push({ id: u.id, text: `${u.id} (not part of this release, its own gate covers it)` }); continue }
+        }
+        // OFF is the one word that means never, past every other flag: a
+        // switch a filter can lift is a suggestion
+        if (u.when.at === 'off') { off.push(u); continue }
+        if (named.has(u.id)) { take(u); continue }
+        if (u.group != null && skipGroups.has(u.group)) { parked.push({ id: u.id, text: `${u.id} (--skip ${u.group})` }); continue }
+        if (o.audience === 'gate') {
+            if (u.when.at === 'health' && o.includeProactive !== true) { parked.push({ id: u.id, text: `${u.id} (health)` }); continue }
+            if (u.when.at === 'release' && !o.fullTier) { parked.push({ id: u.id, text: `${u.id} (release)` }); continue }
+        }
+        if (o.onlyBlocking === true && u.blocking !== true) { if (u.blocking == null) undeclared.push(u.id); nonBlocking.set(u.id, u); continue }
+        take(u)
+    }
+    // a non blocking check cannot change the outcome on its own, but one a
+    // selected check waits on decides whether that check may run at all:
+    // dropping it read as "dep satisfied", so a suite whose prerequisite only
+    // warns ran without it and went red
+    for (let i = 0; i < selected.length; i++)
+        for (const d of selected[i]!.deps) {
+            const dep = nonBlocking.get(d)
+            if (dep != null) { nonBlocking.delete(d); selected.push(dep) }
+        }
+    if (o.onlyIds != null && o.onlyIds.length > 0) {
+        const keep = closure(o.checks, onlyIds)
+        selected = selected.filter((u) => keep.has(u.id))
+    }
+    return { selected, off, parked, undeclared, errors }
+}
+
+/** THIS run's log dir. It used to be `logDir` itself, wiped at the top of
+ *  every run: a second agent starting a gate deleted the log files the first
+ *  was still streaming into, so the surviving FAIL line pointed at a path
+ *  that no longer existed. Now each run owns `run-<pid>-<started>/`.
+ *
+ *  Old run dirs are swept on the way in, and only ones nothing can be using:
+ *  the pid must not be running AND the dir must be older than the grace
+ *  window, so a recycled pid cannot cost a live run its logs. */
+function openRunDir(base: string): string {
+    // the pid plus the millisecond separates two PROCESSES; the counter
+    // separates two runs inside one (the TUI can start a second before the
+    // clock ticks), so no run ever writes into another's dir
+    const dir = join(base, `run-${process.pid}-${Date.now()}-${runSeq++}`)
+    mkdirSync(dir, { recursive: true })
+    const cutoff = Date.now() - RUN_DIR_KEEP_MS
+    try {
+        for (const name of readdirSync(base)) {
+            // run dirs AND the per-run release survey files, which a killed
+            // run leaves behind: both carry the pid and the start time, so one
+            // sweep covers both rather than one path leaking forever
+            const m = /^(?:run|releases)-(\d+)-(\d+)(?:-\d+)?(?:\.json(?:\.lock)?)?$/.exec(name)
+            if (m == null || name === basename(dir)) continue
+            if (Number(m[2]) > cutoff) continue
+            try { process.kill(Number(m[1]), 0); continue } catch (e) { if ((e as { code?: string }).code === 'EPERM') continue }
+            rmSync(join(base, name), { recursive: true, force: true })
+        }
+    } catch {
+        // an unreadable log base is a housekeeping miss, never a gate failure
+    }
+    return dir
+}
+
+/** how long a finished run's logs stay readable, and the pid-reuse guard */
+const RUN_DIR_KEEP_MS = 6 * 60 * 60 * 1000
+let runSeq = 0
+
+type Live = RunnableCheck & CheckResult & {
+    startedAt: number
+    /** when its body stopped, stamped BEFORE its slot goes back: a row must
+     *  never outlast the slot, or another process starting on it overlaps */
+    endedAt: number
+    /** the body never reached its subject, so there is no answer to remember */
+    unmeasured: boolean
+    /** its answer came from another run measuring the same key */
+    attached: boolean
+    /** what an in process body read, null when it reached the world or ran
+     *  as a subprocess */
+    reads: ReadSet | null
+    /** it printed nothing of its own, so its one line is the headline below */
+    synthetic: boolean
+    /** what the headline carries after the id: a failure's last line, or the
+     *  reason it never ran */
+    tail: string
+}
+
+export async function runChecks(o: RunChecksOpts): Promise<RunChecksSummary> {
+    const print = o.print ?? ((l: string) => console.log(l))
+    const color = o.color ?? (process.stdout.isTTY && process.env.NO_COLOR == null)
+    const emit = (l: Line): void => print(formatLine(l, color))
+    const sel = selectChecks(o)
+    const advisoryFailures: string[] = []
+    if (sel.errors.length > 0) {
+        for (const e of sel.errors) emit({ kind: 'fail', text: e })
+        return { ok: false, interrupted: false, results: [], parked: [], advisoryFailures, undeclared: [], errors: sel.errors }
+    }
+    // ONE pass for every body of this call: its scans run once, and its file
+    // list is the ledger's, so a listing and its validation read one snapshot
+    const pass = new CheckPass(o.root, o.config, () => o.ledger?.view() ?? RepoView.open(o.root))
+    const results: CheckResult[] = []
+    const report = (u: RunnableCheck, r: CheckResult, at?: number, attached = false): void => {
+        results.push(r)
+        o.onCheck?.({
+            id: u.id, name: u.title, status: r.status, ms: r.ms, replayed: r.replayed,
+            blocking: u.blocking, advisory: r.advisory, lines: r.lines, problems: r.problems, logPath: r.logPath, lastLine: r.lastLine, note: r.note, attached,
+        })
+        // an attached check is filed as a replay: it measured nothing, and a
+        // row carrying its wait would read as a check running in this process
+        o.log?.add({ id: u.id, status: r.status, ms: r.ms, replayed: r.replayed || attached, lines: r.lines, at })
+    }
+
+    // OFF, in ONE place: skipped by name, never run, never stored, and the
+    // reason printed where the verdict would have been
+    for (const u of sel.off) {
+        const line: Line = { kind: 'skip', text: offLine(u.id, u.when) }
+        emit(line)
+        report(u, { id: u.id, status: 'skip', advisory: false, replayed: false, ms: 0, exitCode: null, lines: [line], problems: [], lastLine: line.text, note: 'off', logPath: null })
+    }
+    // `--only` that leaves nothing to run asked a question nobody answers, and
+    // a green over zero checks reads exactly like a clean repo
+    if ((o.onlyIds ?? []).length > 0 && sel.selected.length === 0) {
+        const text = `--only ${o.onlyIds!.join(', ')} leaves nothing to run${sel.off.length > 0 ? `: ${sel.off.map((u) => u.id).join(', ')} is off` : ''}`
+        emit({ kind: 'fail', text })
+        return { ok: false, interrupted: false, results, parked: [], advisoryFailures, undeclared: [], errors: [text] }
+    }
+
+    const runDir = openRunDir(o.logDir)
+    const checks: Live[] = sel.selected.map((u) => ({
+        ...u,
+        status: 'skip', advisory: false, replayed: false, ms: 0, exitCode: null, lines: [], problems: [], lastLine: '', note: '', startedAt: 0, endedAt: 0, unmeasured: false, attached: false, reads: null, synthetic: false, tail: '',
+        logPath: u.body.cmd != null ? join(runDir, `${u.id.replace(/[^a-zA-Z0-9-]/g, '_')}.log`) : null,
+    }))
+    const selectedIds = new Set(checks.map((u) => u.id))
+    const byId = new Map(checks.map((u) => [u.id, u]))
+    const queued = new Set(checks.map((u) => u.id))
+    const done = new Map<string, Live>()
+    const running = new Map<string, Promise<void>>()
+    const cap = o.maxParallel ?? parallelCap()
+    const pool = o.pool === undefined ? machinePool() : o.pool
+    let slotWaitSaid = false
+    let interrupted = false
+    let bailed = false
+    const procs = new Map<string, ReturnType<typeof Bun.spawn>>()
+    const killed = new Map<string, string>()
+    const killAll = (reason: string): void => {
+        for (const [id, proc] of procs) {
+            killed.set(id, reason)
+            const tree = killTree(proc, 'SIGTERM')
+            setTimeout(() => { killTree(proc, 'SIGKILL', tree) }, 2000)
+        }
+    }
+
+    const depState = (u: Live): 'ready' | 'blocked' | 'dead' => {
+        for (const d of u.deps) {
+            if (!selectedIds.has(d)) continue // filtered dep = satisfied
+            const dep = done.get(d)
+            if (dep == null) return 'blocked'
+            if (dep.status !== 'pass') { u.note = `dep ${d} ${dep.status}`; return 'dead' }
+        }
+        return 'ready'
+    }
+
+    /** what a check is remembered under while nothing on disk describes it: a
+     *  TTL says how long the WORLD keeps, never that the probe was rewritten
+     *  or that a different shipkit measured it. */
+    const worldKey = (u: RunnableCheck): string => {
+        let code = ''
+        try { code = o.ledger?.layer.keyFor(u.id).codeKey ?? '' } catch { code = '' }
+        // a check that writes answers a different question when frozen
+        const frozen = u.writes === true ? `\0frozen=${process.env['SHIPKIT_FROZEN'] === '1'}` : ''
+        return `tc=${o.ledger?.layer.toolchainId ?? ''}\0code=${code}\0${u.rotation ?? ''}${frozen}`
+    }
+
+    // a non-blocking failure becomes a WARNING here, at the source: printing
+    // FAIL and exiting 0 would be the tool lying about its own severity
+    const demote = (u: Live): void => {
+        if (u.blocking !== false || u.status !== 'fail') return
+        u.advisory = true
+        if (!advisoryFailures.includes(u.id)) advisoryFailures.push(u.id)
+        u.lines = u.lines.map((l) => (l.kind === 'fail' ? { ...l, kind: 'warn' as const } : l.kind === 'fix-fail' ? { ...l, kind: 'fix-warn' as const } : l))
+        u.problems = u.problems.map((p) => ({ ...p, severity: 'warning' as const }))
+    }
+
+    /** what this check contributes to the problems pane, from what it printed */
+    const feeds = (u: Live): boolean => feedsProblemsPane(u.population ?? (u.ttlMs > 0 ? 'world' : 'neither'))
+    const findings = (u: Live, lines: Line[], output?: string): Problem[] => {
+        if (!feeds(u)) return []
+        return extractProblems({ lines, output }, { root: o.root, cwd: u.body.cwd ?? o.root, check: u.id })
+    }
+
+    const finish = (u: Live): void => {
+        const finishedAt = u.endedAt > 0 ? u.endedAt : clock()
+        u.ms = u.replayed || u.startedAt === 0 ? 0 : finishedAt - u.startedAt
+        done.set(u.id, u)
+        if (u.lines.length === 0) headlineOf(u)
+        demote(u)
+        // the duration is rendered at PRINT time and never stored: a wall time
+        // baked into a remembered line comes back on a replay as if this run
+        // had measured it (ledger/ledger.ts rule 5)
+        const head = u.lines[0]
+        const shown = u.synthetic && !u.replayed && head != null
+            ? [{ ...head, text: `${u.id} (${(u.ms / 1000).toFixed(1)}s)${u.tail}` }, ...u.lines.slice(1)]
+            : u.lines
+        for (const l of shown) emit(l)
+        if (u.status === 'fail' && !u.advisory && u.logPath != null)
+            emit({ kind: 'fix-fail', text: `full log: ${u.logPath}` })
+        report(u, { id: u.id, status: u.status, advisory: u.advisory, replayed: u.replayed, ms: u.ms, exitCode: u.exitCode, lines: u.lines, problems: u.problems, lastLine: u.lastLine, note: u.note, logPath: u.logPath }, finishedAt, u.attached)
+        if (u.status === 'fail' && !u.advisory && o.bail === true && !bailed) { bailed = true; killAll('bail') }
+    }
+
+    /** a stored answer, re-emitted without running anything */
+    const replay = (u: Live): boolean => {
+        const hit = o.ledger?.replay(u.id)
+        if (hit != null) {
+            u.replayed = true
+            u.lines = stampCached(hit.lines, hit.measuredMs)
+            u.lastLine = hit.lastLine ?? ''
+            u.status = hit.status
+            // a replay stands for the verdict it replays, findings included:
+            // the inputs have not moved, so what was wrong is still wrong
+            u.problems = !feeds(u) ? [] : hit.problems ?? findings(u, hit.lines)
+            return true
+        }
+        if (o.cache == null || u.ttlMs <= 0) return false
+        const stored = replayWorld(o.cache, u.id, worldKey(u), u.ttlMs)
+        if (stored == null) return false
+        u.replayed = true
+        u.lines = stored
+        u.problems = findings(u, stored)
+        // a replayed WORLD negative keeps its severity: it is still the
+        // answer, and demoting it to a pass would hide a real finding
+        u.status = stored.some((l) => l.kind === 'warn' || l.kind === 'fail') ? 'warn' : 'pass'
+        return true
+    }
+
+    const measure = async (u: Live): Promise<void> => {
+        const t = new CheckCtx(pass)
+        if (u.body.run != null) {
+            // check.ts owns the semantics: a throwing body still fails with its
+            // id, every line is tagged, and a failure carries a fix pointer
+            await runCheckBody({ id: u.id, title: u.title, owner: u.owner, run: u.body.run, fix: u.fix }, t)
+            u.reads = t.untracked == null ? t.reads.snapshot() : null
+            u.lines = t.lines
+            u.problems = findings(u, t.lines)
+            u.unmeasured = t.unmeasured
+            const last = t.lines.filter((l) => l.kind === 'fail').at(-1) ?? t.lines.at(-1)
+            u.lastLine = last?.text.slice(0, 200) ?? ''
+            u.status = t.failed ? 'fail' : t.warned ? 'warn' : 'pass'
+            u.exitCode = t.failed ? 1 : 0
+            return
+        }
+        const cmd = u.body.cmd!
+        // the log opens only once the child actually spawned (a missing binary
+        // must not leave a dangling open on the log dir), and every write is
+        // synchronous: when measure returns the whole log is on disk and no
+        // open is still pending against a dir the caller may remove next
+        let fd: number | null = null
+        let logProblem: string | null = null
+        const logWrite = (text: string): void => {
+            if (fd == null) return
+            try { writeSync(fd, text) } catch (e) { logProblem ??= (e as Error).message; try { closeSync(fd) } catch { /* already gone */ } fd = null }
+        }
+        const r = await spawnWithTimeout(cmd, {
+            timeoutMs: u.timeoutMs,
+            // a child runs on this check's slot and takes none of its own
+            env: { ...process.env, ...o.childEnv, ...u.body.env, [SLOT_HELD_ENV]: '1', [CHECK_ID_ENV]: u.id },
+            cwd: u.body.cwd ?? o.root,
+            onSpawn: (proc) => {
+                try { fd = openSync(u.logPath!, 'a') } catch (e) { logProblem = (e as Error).message }
+                procs.set(u.id, proc)
+            },
+            onChunk: (text) => {
+                logWrite(text)
+                for (const raw of text.split(/\r?\n/)) {
+                    const s = stripAnsi(raw).trim()
+                    if (s !== '') u.lastLine = s.slice(0, 200)
+                }
+            },
+        })
+        procs.delete(u.id)
+        // TS narrows `fd` to its initializer across the await (the assignment
+        // happens in the onSpawn closure), so widen explicitly
+        const open = fd as number | null
+        if (open != null) try { closeSync(open) } catch { /* the write path already reported it */ }
+        if (r.spawnFailed) { u.lastLine = r.out; u.status = 'fail'; u.exitCode = -1; return }
+        const killReason = killed.get(u.id)
+        if (killReason != null) { u.note = `killed (${killReason})`; u.status = 'skip'; u.exitCode = r.code; return }
+        // a check REPORTS, in wire lines. A build step runs a TOOL, whose
+        // output is a log rather than a report, so its verdict is the one
+        // line the headline draws and the rest is in the file
+        u.lines = u.build ? [] : parseWireOutput(r.out)
+        if (logProblem != null) u.lines.push({ kind: 'warn', text: `${u.id}: its log ${u.logPath} was not written: ${logProblem}` })
+        u.exitCode = r.code
+        const failed = u.lines.some((l) => l.kind === 'fail')
+        if (r.timedOut) {
+            const tail = u.lastLine !== '' ? ` — ${u.lastLine}` : ''
+            u.lastLine = `timeout: ${u.id} killed at its ${(u.timeoutMs / 1000).toFixed(1)}s ceiling, raise its timeoutMs${tail}`
+        }
+        // exit code is truth: a non-zero exit with no FAIL line (a crash, non
+        // wire output, a timeout) must still fail loud. A BUILD step says it
+        // in its headline instead, which is the line that carries the wall
+        // time and the log path
+        if (r.code !== 0 && !failed && !u.build) {
+            u.lines.push({ kind: 'fail', text: `${u.id} exited ${r.code}${r.timedOut ? ` (timeout after ${u.timeoutMs}ms)` : ''}${u.lastLine === '' ? '' : ` — ${u.lastLine.slice(0, 160)}`}` })
+            if (u.pointer != null) u.lines.push({ kind: 'fix-fail', text: u.pointer })
+        }
+        u.status = r.code !== 0 || failed ? 'fail' : u.lines.some((l) => l.kind === 'warn' || l.kind === 'fix-warn') ? 'warn' : 'pass'
+        u.problems = findings(u, u.lines, r.out)
+        const lastFail = u.lines.filter((l) => l.kind === 'fail').at(-1)
+        if (lastFail != null) u.lastLine = lastFail.text.slice(0, 200)
+    }
+
+    /** a check that printed nothing of its own still has to report that it
+     *  ran: the verdict, its name, and (at print time) what it cost */
+    const headlineOf = (u: Live): void => {
+        u.synthetic = true
+        u.tail = u.status === 'fail' ? (u.lastLine === '' ? '' : ` — ${u.lastLine}`) : u.status === 'skip' ? (u.note === '' ? '' : ` (${u.note})`) : ''
+        u.lines = [{ kind: u.status === 'fail' ? 'fail' : u.status === 'skip' ? 'skip' : 'ok', text: `${u.id}${u.tail}` }]
+    }
+
+    /** what an attached asker adopts: the verdict and the lines BEFORE this
+     *  run's own demotion, which the asker applies from its own declaration */
+    type Answer = Pick<Live, 'status' | 'lines' | 'problems' | 'lastLine' | 'exitCode' | 'logPath' | 'synthetic' | 'tail'> & { measuredMs: number }
+    const answerOf = (u: Live): Answer => ({
+        status: u.status, lines: u.lines, problems: u.problems, lastLine: u.lastLine, exitCode: u.exitCode, logPath: u.logPath,
+        synthetic: u.synthetic, tail: u.tail, measuredMs: (u.endedAt > 0 ? u.endedAt : clock()) - u.startedAt,
+    })
+    // only a GREEN is ever stored. A pass is one whatever the check blocks. A
+    // WARN from a check that cannot block is not: it may be a red about the
+    // world, and a content key has no TTL, so storing it would pin it until an
+    // unrelated edit moved the tree
+    const storable = (u: Live): u is Live & { status: 'pass' | 'warn' } => u.status === 'pass' || (u.status === 'warn' && u.blocking !== false)
+
+    const adopt = (u: Live, a: Answer, pid: number): void => {
+        const stamp = ` ${attachedStamp(pid, a.measuredMs)}`
+        u.status = a.status
+        u.lastLine = a.lastLine
+        u.exitCode = a.exitCode
+        // the full log lives in the run that measured
+        u.logPath = a.logPath
+        u.attached = true
+        const head = a.lines[0]
+        u.lines = head == null ? [] : [{ ...head, text: `${head.text}${stamp}` }, ...a.lines.slice(1)]
+        u.problems = feeds(u) ? a.problems ?? [] : []
+        if (o.ledger != null && storable(u))
+            o.ledger.record(u.id, { status: u.status, lines: a.lines, lastLine: a.lastLine, problems: u.problems, measuredMs: a.measuredMs })
+    }
+
+    /** ONE measurement per in flight question: an asker whose exact key
+     *  another run is measuring right now waits for that answer instead of
+     *  running the tool a second time (core/attach.ts) */
+    const measureShared = async (u: Live): Promise<void> => {
+        const slot = o.ledger?.attachSlot(u.id) ?? null
+        if (slot == null) return await measureAndKeep(u)
+        const r = await shareRun<Answer>(slot, {
+            askedAt: Date.now(),
+            label: u.id,
+            print,
+            // past the holder's own ceiling, so a live holder always answers first
+            maxWaitMs: u.timeoutMs + 30_000,
+            abort: () => interrupted || bailed,
+        }, async () => {
+            await measureAndKeep(u)
+            return { value: answerOf(u), share: !killed.has(u.id) && !u.unmeasured }
+        })
+        if (r.kind === 'attached') { o.ledger?.noteAttached(); adopt(u, r.value, r.pid) }
+        else if (r.kind === 'aborted') { u.status = 'skip'; u.note = bailed ? 'bailed' : 'interrupted'; u.lines = [] }
+        else if (r.kind === 'overlapped') emit({ kind: 'warn', text: `${u.id} ran again beside another run: ${r.warn}` })
+    }
+
+    const measureAndKeep = async (u: Live): Promise<void> => {
+        // a machine slot for the body, and only the body: a replay or an
+        // attached wait runs nothing, so it holds none
+        const slot = pool == null ? null : await takeSlot(pool, { repo: o.config.name, check: u.id }, {
+            abort: () => interrupted || bailed,
+            onWait: (held) => {
+                if (slotWaitSaid) return
+                slotWaitSaid = true
+                const who = held.slice(0, 3).map((h) => `${h.repo}/${h.check} pid ${h.pid}`).join(', ')
+                emit({ kind: 'ok', text: `waiting for a machine slot: ${held.length} of ${pool.size} held${who === '' ? '' : ` (${who}${held.length > 3 ? ', …' : ''})`}` })
+            },
+        })
+        if (pool != null && slot == null) { u.status = 'skip'; u.note = bailed ? 'bailed' : 'interrupted'; u.lines = []; u.unmeasured = true; return }
+        // its time starts when it runs, so a row never counts the queue
+        u.startedAt = clock()
+        try { await measure(u) } finally { u.endedAt = clock(); slot?.release() }
+        // the headline is built BEFORE anything is remembered: a check
+        // whose result is stored empty replays as a line nobody can tell
+        // from a measurement
+        if (u.lines.length === 0) headlineOf(u)
+        // what may be remembered, and for how long, is ONE policy
+        // (ledger/world-cache.ts): a clean answer keeps for the declared
+        // TTL, a measured negative only long enough to stop a re-ask storm
+        if (o.cache != null && u.ttlMs > 0) {
+            const clean = u.status === 'pass'
+            u.lines = rememberWorld(o.cache, u.id, worldKey(u), u.ttlMs, {
+                lines: u.lines,
+                clean,
+                unmeasured: u.unmeasured || u.lines.some((l) => l.kind === 'skip'),
+                keepNegative: u.blocking === false,
+            })
+        }
+        if (o.ledger != null && storable(u))
+            o.ledger.record(u.id, { status: u.status, lines: u.lines, lastLine: u.lastLine, problems: u.problems, measuredMs: clock() - u.startedAt }, u.reads)
+    }
+
+    const start = (u: Live): void => {
+        queued.delete(u.id)
+        u.startedAt = clock()
+        const p = (async (): Promise<void> => {
+            if (replay(u)) return finish(u)
+            await measureShared(u)
+            finish(u)
+        })().finally(() => running.delete(u.id))
+        running.set(u.id, p)
+    }
+
+    // what is running, what finished and what waits on what, for the surfaces
+    // that show a run in flight. Written once per turn of the loop, after the
+    // starts, and once more marked ended when the runner returns
+    const snapshot = (ended: number | null) => liveStateOf({
+        cap,
+        total: checks.length,
+        finished: done,
+        running: running.keys(),
+        startedAt: (id) => byId.get(id)?.startedAt ?? 0,
+        queued,
+        depsOf: (id) => byId.get(id)?.deps ?? [],
+        selected: selectedIds,
+        ended,
+    })
+
+    const onSigint = (): void => { interrupted = true; killAll('interrupt') }
+    process.on('SIGINT', onSigint)
+    try {
+        while (queued.size > 0 || running.size > 0) {
+            if (interrupted || bailed) {
+                for (const id of [...queued]) {
+                    const u = byId.get(id)!
+                    u.note = bailed ? 'bailed' : 'interrupted'
+                    queued.delete(id)
+                    done.set(id, u)
+                }
+                await Promise.all(running.values())
+                break
+            }
+            let started = false
+            for (const id of [...queued]) {
+                if (running.size >= cap) break
+                const u = byId.get(id)!
+                const st = depState(u)
+                if (st === 'dead') { queued.delete(id); finish(u); started = true }
+                else if (st === 'ready') { start(u); started = true }
+            }
+            o.log?.live(snapshot(null))
+            if (running.size > 0) await Promise.race(running.values())
+            else if (!started && queued.size > 0) {
+                // nothing running, nothing startable: impossible unless a bug
+                for (const id of [...queued]) emit({ kind: 'fail', text: `${id}: scheduler stuck (unsatisfiable deps?)` })
+                break
+            }
+        }
+    } finally {
+        process.off('SIGINT', onSigint)
+        // the runner is the one scope that knows the run is over, a throw
+        // included: the row says so, and the flush deletes it
+        o.log?.live(snapshot(Date.now()))
+    }
+
+    // bail and interrupt leftovers never finished: they are visible, never a
+    // fake pass
+    for (const u of checks) if (!results.some((r) => r.id === u.id)) report(u, { id: u.id, status: 'skip', advisory: false, replayed: false, ms: 0, exitCode: null, lines: [], problems: [], lastLine: u.lastLine, note: u.note, logPath: u.logPath })
+    const order = new Map(o.checks.map((u, i) => [u.id, i]))
+    results.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0))
+    const blocked = results.filter((r) => r.status === 'fail' && !r.advisory)
+    const skipped = results.filter((r) => r.status === 'skip' && r.note !== '' && r.note !== 'off')
+    if (blocked.length > 0) emit({ kind: 'fail', text: `${blocked.length}/${results.length} failed: ${blocked.map((r) => r.id).join(', ')}` })
+    if (skipped.length > 0) emit({ kind: 'note', text: `skip: ${skipped.map((r) => `${r.id} (${r.note})`).join(', ')}` })
+    if (blocked.length === 0 && !interrupted && results.length > 0) emit({ kind: 'ok', text: `all ${results.filter((r) => r.status !== 'skip').length} check(s) green` })
+    return {
+        ok: blocked.length === 0 && !interrupted,
+        interrupted,
+        results,
+        parked: sel.parked.map((p) => p.text),
+        advisoryFailures,
+        undeclared: sel.undeclared,
+        errors: [],
+    }
+}