rvlib-shipkit 0.38.1 → 1.0.0

package/src/suite/suite-cli.ts

@@ -6,17 +6,32 @@
 // `Ran N tests` so the gate can still report what ran.
 //
 // usage: suite-cli [--budget-ms N] [--slow file]... [--serial file]... [--declared-in <where>] -- <bun test args>
-import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs'
-import { tmpdir } from 'os'
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'
+import { availableParallelism, loadavg, tmpdir } from 'os'
 import { join } from 'path'
-import { FULL_TESTS_ENV } from '../ledger/gate-units.ts'
-import { confirmOver, crashedWorkers, DEFAULT_BUDGET_MS, judgeSuites, normSuitePath, parseSuiteFlags, planPhases } from './judge-suites.ts'
+import { FULL_TESTS_ENV } from '../ledger/gate-checks.ts'
+import { CHECK_ID_ENV, recordSuites, type SuitePhase } from '../ledger/suite-log.ts'
+import { killTree } from '../core/spawn-timeout.ts'
+import { parallelCap } from '../core/parallel.ts'
+import { poolDir, poolOccupancy } from '../core/slots.ts'
+import { keepTimings, seedTimings, timingsHome } from './kept-timings.ts'
+import { aloneRunsNeeded, bunfigTest, readPhaseCpu, SUITE_CPU_ENV, type AloneRun, type Confirmed, confirmOver, crashedWorkers, type MachineLoad, WAIT_FACTOR, DEFAULT_BUDGET_MS, FULL_CEILING_MS, failedSuites, failLines, judgeSuites, normSuitePath, parseSuiteFlags, planPhases, QUICK_CEILING_MS, poolWorkers } from './judge-suites.ts'
 
-type Phase = { code: number; signal: string | null; out: string; timings: Record<string, number> }
+type Phase = { code: number; signal: string | null; stalled: boolean; out: string; timings: Record<string, number>; cpuMs: number | null }
 
-async function runPhase(args: string[], timingsFile: string | null, echo = true): Promise<Phase> {
+async function runPhase(args: string[], timingsFile: string | null, ceilingMs: number, echo = true, cpuFile: string | null = null): Promise<Phase> {
     const timing = timingsFile != null ? [`--timings=${timingsFile}`, '--update-timings'] : []
-    const proc = Bun.spawn(['bun', 'test', ...args, ...timing], { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' })
+    // the per suite CPU figure rides in a preload of the runner's own, beside
+    // the repo's (a command line preload adds to bunfig's, never replaces it)
+    const cpu = cpuFile != null ? ['--preload', CPU_PRELOAD] : []
+    const env = cpuFile != null ? { ...process.env, [SUITE_CPU_ENV]: cpuFile } : process.env
+    const proc = Bun.spawn(['bun', 'test', ...cpu, ...timing, ...args], { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore', env })
+    // the workers are children of `bun test`, and a stuck one is the process
+    // to reach, so the whole tree goes
+    let stalled = false
+    let tree: number[] = []
+    const ceiling = setTimeout(() => { stalled = true; tree = killTree(proc, 'SIGTERM') }, ceilingMs)
+    const killer = setTimeout(() => { killTree(proc, 'SIGKILL', tree) }, ceilingMs + 2000)
     let out = ''
     const pump = async (stream: ReadableStream<Uint8Array>, sink: NodeJS.WriteStream): Promise<void> => {
         const dec = new TextDecoder()
@@ -28,20 +43,41 @@ async function runPhase(args: string[], timingsFile: string | null, echo = true)
     }
     await Promise.all([pump(proc.stdout, process.stdout), pump(proc.stderr, process.stderr)])
     const code = await proc.exited
+    clearTimeout(ceiling)
+    clearTimeout(killer)
     let timings: Record<string, number> = {}
     try { if (timingsFile != null && existsSync(timingsFile)) timings = (JSON.parse(readFileSync(timingsFile, 'utf8')) as { files?: Record<string, number> }).files ?? {} }
     catch { timings = {} }
-    return { code, signal: proc.signalCode ?? null, out, timings }
+    // wait4 rusage: the process plus every child it reaped, which is where
+    // bun test's workers and a suite's own subprocesses land
+    const usage = proc.resourceUsage()
+    const cpuMs = usage != null ? Number(usage.cpuTime.user + usage.cpuTime.system) / 1000 : null
+    return { code, signal: stalled ? null : proc.signalCode ?? null, stalled, out, timings, cpuMs }
 }
 
+const CPU_PRELOAD = join(import.meta.dir, 'suite-cpu-preload.ts')
+
 const secs = (ms: number): string => `${(ms / 1000).toFixed(1)}s`
+
+const machineLoad = (): MachineLoad => ({ load1: loadavg()[0] ?? 0, cores: availableParallelism() })
+
+const overWords = (c: Confirmed): string =>
+    c.over === 'cpu' ? `${c.file} ${secs(c.ms)} of CPU (${secs(c.wallMs)} wall)`
+        : c.over === 'wait' ? `${c.file} ${secs(c.wallMs)} wall on an idle machine, more than ${WAIT_FACTOR}x the budget, with ${secs(c.cpuMs ?? 0)} of CPU: it waits (a timer, a sleep, a poll)`
+            : `${c.file} ${secs(c.wallMs)} wall (no CPU figure)`
+
+/** the suite FILES a phase names: a flag is not a file, and the repo's own
+ *  `bun test` arguments are not files this runner put there */
+const phaseFiles = (phase: string[], rawArgs: string[]): string[] => phase.filter((a) => !a.startsWith('-') && !rawArgs.includes(a))
 const lastCount = (out: string, re: RegExp): number => Number([...out.matchAll(re)].at(-1)?.[1] ?? 0)
 
 export async function suiteCli(argv: string[]): Promise<number> {
+    const t0 = Date.now()
     const parsed = parseSuiteFlags(argv)
     if (typeof parsed === 'string') { console.log(`FAIL: suite runner: ${parsed}`); return 2 }
-    const { tests, rawArgs, declaredIn } = parsed
+    const { tests, rawArgs, declaredIn, unit } = parsed
     const tier = process.env[FULL_TESTS_ENV] === '1' ? 'full' : 'quick'
+    const ceilingMs = parsed.ceilingMs ?? (tier === 'full' ? FULL_CEILING_MS : QUICK_CEILING_MS)
     const budget = tests.budgetMs ?? DEFAULT_BUDGET_MS
 
     // a declaration naming a gone suite would silently stop excluding anything
@@ -58,14 +94,44 @@ export async function suiteCli(argv: string[]): Promise<number> {
     // gates, and sharing a run measured slower than sequential and timed out
     const parallel = helpText.includes('--parallel') && tier === 'quick'
     const timed = helpText.includes('--update-timings')
-    const plan = planPhases({ tests, tier, rawArgs, parallel })
+    // bun drops these the moment any ignore reaches its command line
+    const bunfig = bunfigTest(existsSync('bunfig.toml') ? readFileSync('bunfig.toml', 'utf8') : null)
+    if (bunfig.error != null) { console.log(`FAIL: ${bunfig.error}`); return 1 }
+    // the pool is read, never taken: this process runs on its check's slot
+    const cores = availableParallelism()
+    const held = parallel ? poolOccupancy({ dir: poolDir(), size: parallelCap() }).held.length : 0
+    const workers = parallel ? poolWorkers({ cores, size: parallelCap(), held }) : undefined
+    const plan = planPhases({ tests, tier, rawArgs, parallel, noIsolate: helpText.includes('--no-isolate'), configIgnores: bunfig.ignores, workers })
     const dir = mkdtempSync(join(tmpdir(), 'shipkit-suite-'))
     const notes: string[] = []
     if (!helpText.includes('--parallel')) notes.push(`skip: parallel suites (bun ${Bun.version} has no --parallel)`)
     if (!timed) notes.push(`skip: suite budget (bun ${Bun.version} writes no --timings)`)
     try {
-        const p1 = await runPhase(plan.parallel, timed ? join(dir, 'parallel.json') : null)
-        const p2 = plan.serial != null ? await runPhase(plan.serial, timed ? join(dir, 'serial.json') : null) : null
+        const setupMs = Date.now() - t0
+        // the parallel phase reads the durations the last run kept, so bun
+        // starts the slowest files first, and keeps what it measured for the next
+        const kept = timed && parallel ? timingsHome(process.cwd(), unit) : null
+        if (kept != null) seedTimings(kept, join(dir, 'parallel.json'))
+        const cpuFile = timed && parallel ? join(dir, 'cpu.tsv') : null
+        const p1At = Date.now()
+        const p1 = await runPhase(plan.parallel, timed ? join(dir, 'parallel.json') : null, ceilingMs, true, cpuFile)
+        const p1Ms = Date.now() - p1At
+        // bun merges what it read into what it writes, so a suite deleted since
+        // the last run is judged and kept only if the workers say it ran here
+        // or it is still on disk. On disk alone is the floor: a suite whose
+        // line went missing is never dropped from the judgment
+        const phaseCpu = cpuFile != null && existsSync(cpuFile) ? readPhaseCpu(readFileSync(cpuFile, 'utf8'), process.cwd()) : { ran: [], cpu: {} }
+        if (kept != null) {
+            const ranHere = new Set(phaseCpu.ran)
+            p1.timings = Object.fromEntries(Object.entries(p1.timings).filter(([f]) => (ranHere.has(normSuitePath(f)) || existsSync(f))))
+            if (!p1.stalled && p1.signal == null) {
+                writeFileSync(join(dir, 'parallel.json'), JSON.stringify({ version: 1, files: p1.timings }))
+                keepTimings(join(dir, 'parallel.json'), kept)
+            }
+        }
+        const p2At = Date.now()
+        const p2 = plan.serial != null ? await runPhase(plan.serial, timed ? join(dir, 'serial.json') : null, ceilingMs) : null
+        const p2Ms = p2 == null ? null : Date.now() - p2At
         const phases = [p1, ...(p2 != null ? [p2] : [])]
         const ran = phases.reduce((n, p) => n + lastCount(p.out, /Ran (\d+) tests?/g), 0)
         const failedTests = phases.reduce((n, p) => n + lastCount(p.out, /^\s*(\d+) fail$/gm), 0)
@@ -73,38 +139,95 @@ export async function suiteCli(argv: string[]): Promise<number> {
         const judged = judgeSuites({ timings, tests, tier })
         const slowest = judged.slowest.length > 0 ? ` (slowest: ${judged.slowest.map((t) => `${t.file} ${secs(t.ms)}`).join(', ')})` : ''
 
-        // a parallel timing carries every other suite's load, so a suspect is
-        // timed again alone, one process, before it can fail the task
-        let over = judged.over
-        if (over.length > 0) {
-            const alone = await runPhase([...rawArgs, ...over.map((t) => t.file)], join(dir, 'alone.json'), false)
-            over = confirmOver(over, alone.timings, budget)
+        // a parallel timing carries every other suite's load, so a suspect
+        // runs again alone, one process per suspect so its CPU is its own,
+        // before it can fail the task. One at a time: four alone runs side by
+        // side each burned about 40% more CPU than one alone (8 cores)
+        const aloneRuns: Record<string, AloneRun> = {}
+        const load = machineLoad()
+        const parallelCpu = phaseCpu.cpu
+        const { rerun, cleared: clearedInPhase } = aloneRunsNeeded(judged.over, parallelCpu, budget, load)
+        const aloneAt = Date.now()
+        for (const [i, t] of rerun.entries()) {
+            const run = await runPhase([...rawArgs, t.file], join(dir, `alone-${i}.json`), ceilingMs, false)
+            const wallMs = Object.entries(run.timings).find(([f]) => normSuitePath(f) === t.file)?.[1]
+            if (wallMs != null) aloneRuns[t.file] = { wallMs, cpuMs: run.cpuMs }
         }
+        const aloneMs = Date.now() - aloneAt
+        const { over, cleared } = rerun.length > 0 ? confirmOver(rerun, aloneRuns, budget, load) : { over: [], cleared: [] }
+        for (const c of clearedInPhase) notes.push(`note: ${c.file} took ${secs(c.wallMs)} in the parallel run and burned ${secs(c.cpuMs ?? 0)} of CPU there, plus ${secs(parallelCpu[c.file]?.priorMs ?? 0)} a lone run loads first, inside the ${budget}ms budget, so it did not run again alone`)
+        for (const c of cleared) notes.push(`note: ${c.file} took ${secs(c.wallMs)} alone but burned ${c.cpuMs != null ? secs(c.cpuMs) : '?'} of CPU, inside the ${budget}ms budget (load ${load.load1.toFixed(1)} on ${load.cores} cores)`)
 
-        const killed = crashedWorkers(p1.out).map((c) => `killed by ${c.signal} while running ${c.file} (out of memory, or killed from outside)`)
+        // a ceiling kill makes bun report every worker it held as crashed, and
+        // those are the suites that never finished, not an outside kill
+        const stalledIn = (name: string, p: Phase | null, files: string[]): string | null =>
+            p?.stalled === true
+                ? `${name} ran past its ${secs(ceilingMs)} ceiling and was killed${files.length > 0 ? ` with ${files.join(', ')} still running` : ''}: a test worker stopped answering, most likely stuck in a synchronous call, which no per test timeout can interrupt`
+                : null
+        const crashed = crashedWorkers(p1.out)
+        const killed = p1.stalled ? [] : crashed.map((c) => `killed by ${c.signal} while running ${c.file} (out of memory, or killed from outside)`)
+        const stalls = [stalledIn('the parallel phase', p1, crashed.map((c) => c.file)), stalledIn('the serial phase', p2, phaseFiles(plan.serial ?? [], rawArgs).filter((a) => p2?.timings[normSuitePath(a)] == null))]
+        for (const line of stalls) if (line != null) killed.unshift(line)
         if (p1.signal != null) killed.push(`killed by ${p1.signal} during the parallel run (out of memory, or killed from outside)`)
         if (p2?.signal != null) {
-            const unfinished = (plan.serial ?? []).filter((a) => !a.startsWith('-') && !rawArgs.includes(a) && p2.timings[normSuitePath(a)] == null)
+            const unfinished = phaseFiles(plan.serial ?? [], rawArgs).filter((a) => p2.timings[normSuitePath(a)] == null)
             killed.push(`killed by ${p2.signal} while running ${unfinished.join(' or ')} (out of memory, or killed from outside)`)
         }
+        // the record behind the hub's test files to fix: every suite this run
+        // timed, with the alone timing where one was taken, and red when a
+        // test in it failed or its worker died
+        const red = new Set([...phases.flatMap((p) => failedSuites(p.out)), ...crashedWorkers(p1.out).map((c) => c.file)])
+        const at = Date.now()
+        const alone = new Map([...over, ...cleared, ...clearedInPhase].map((t) => [t.file, t.ms]))
+        const why = failLines(phases.map((p) => p.out).join('\n'))
+        for (const c of crashedWorkers(p1.out)) if (!why.has(c.file)) why.set(c.file, `its worker crashed: ${c.signal}`)
+        // which phase timed each file, and the CPU spent on it: an alone
+        // rerun, the parallel phase's own figure, or a phase that ran that one file
+        const phaseOf = (f: string): SuitePhase => (p2 != null && Object.keys(p2.timings).some((k) => normSuitePath(k) === f) ? 'serial' : parallel ? 'parallel' : 'sequential')
+        const soloCpu = (p: Phase | null): number | null => (p != null && Object.keys(p.timings).length === 1 ? p.cpuMs : null)
+        const cpuOf = (f: string): number | null => aloneRuns[f]?.cpuMs ?? (phaseOf(f) === 'serial' ? soloCpu(p2) : parallelCpu[f]?.cpuMs ?? soloCpu(p1))
+        const check = process.env[CHECK_ID_ENV] ?? ''
+        const serialFiles = p2 == null ? 0 : Object.keys(p2.timings).length
+        const unrecorded = recordSuites(process.cwd(), Object.entries(timings).map(([file, ms]) => {
+            const f = normSuitePath(file)
+            return {
+                at, file: f, ms: alone.get(f) ?? ms, status: red.has(f) ? 'fail' as const : 'pass' as const, tier, budgetMs: budget, check, unit,
+                cpuMs: cpuOf(f), phase: phaseOf(f), phaseMs: ms, failLine: red.has(f) ? why.get(f) ?? '' : '',
+            }
+        }), {
+            at, check, unit, tier, wallMs: Date.now() - t0, setupMs, workers: parallel ? workers ?? 0 : 0, files: Object.keys(timings).length,
+            parallelMs: p1Ms, parallelCpuMs: p1.cpuMs, serialMs: p2Ms, serialCpuMs: p2?.cpuMs ?? null, serialFiles,
+            aloneMs, aloneFiles: rerun.length, load1: load.load1, cores: load.cores,
+        })
+        if (unrecorded != null) notes.push(unrecorded)
         // never blocking: a suite that got faster must not stop a release
         const stale = judged.noLongerSlow.map((t) => `WARN ${declaredIn}.slow declares ${t.file}, which ran in ${secs(t.ms)}, under half the ${budget}ms budget: remove the declaration`)
 
         for (const n of notes) console.log(n)
         for (const s of stale) console.log(s)
+        // the root of a repo with units leaves every unit dir to its own check,
+        // so finding no file of its own is a finding, not a failure
+        if (unit === 'root' && ran === 0 && failedTests === 0 && killed.length === 0 && /\b0 test files matching\b|did not match any test files/.test(p1.out)) {
+            for (const n of notes) console.log(n)
+            console.log('ok: no test file outside the units: each unit runs its own in test:<unit id>')
+            return 0
+        }
+        // a suite that runs nothing proves nothing, and a green would say it did
+        const vacuous = ran === 0 && failedTests === 0 && killed.length === 0
         const fails: string[] = [
             ...killed.map((k) => `FAIL: ${k}`),
-            ...(failedTests > 0 || (phases.some((p) => p.code !== 0 && p.signal == null) && killed.length === 0)
+            ...(vacuous ? ['FAIL: ran 0 tests: a suite with no test in it proves nothing, write a real one'] : []),
+            ...(!vacuous && (failedTests > 0 || (phases.some((p) => p.code !== 0 && p.signal == null) && killed.length === 0))
                 ? [`FAIL: ${failedTests} test(s) failed, Ran ${ran} tests${slowest}`] : []),
             ...(over.length > 0
-                ? [`FAIL: ${over.length} suite(s) over the ${budget}ms budget: ${over.map((t) => `${t.file} ${secs(t.ms)}`).join(', ')} (timed alone). Make each fast, or declare it in ${declaredIn}.slow with what it catches and why it cannot be fast`]
+                ? [`FAIL: ${over.length} suite(s) over the ${budget}ms budget: ${over.map(overWords).join(', ')} (run alone, load ${load.load1.toFixed(1)} on ${load.cores} cores). Make each fast, or declare it in ${declaredIn}.slow with what it catches and why it cannot be fast`]
                 : []),
         ]
         // the most actionable failure prints LAST, where the gate reads its headline
         for (const f of fails.slice(1).reverse()) console.log(f)
         if (fails.length > 0) { console.log(fails[0]); return 1 }
-        const serialCount = plan.serial != null ? plan.serial.length - rawArgs.length : 0
-        console.log(`ok: Ran ${ran} tests across ${Object.keys(timings).length || '?'} suites, ${parallel ? 'parallel' : 'one at a time'}${serialCount > 0 ? ` then ${serialCount} serial` : ''}, ${tier} tier${slowest}`)
+        const serialCount = plan.serial != null ? phaseFiles(plan.serial, rawArgs).length : 0
+        console.log(`ok: Ran ${ran} tests across ${Object.keys(timings).length || '?'} suites, ${parallel ? `parallel on ${workers} workers` : 'one at a time'}${serialCount > 0 ? ` then ${serialCount} serial` : ''}, ${tier} tier${slowest}`)
         return 0
     } finally {
         rmSync(dir, { recursive: true, force: true })

package/src/suite/suite-cpu-preload.ts

@@ -0,0 +1,92 @@
+// THE CPU EACH SUITE BURNED INSIDE THE PARALLEL PHASE, preloaded into every
+// worker by the suite runner. A suite over the budget in wall time is run again
+// alone to learn its CPU, one after the other, and on a loaded machine nearly
+// every gate heavy suite goes over in wall time: those re-runs were most of the
+// test check's wall time. A suite whose CPU in the parallel phase already
+// bounds its alone run under the budget needs no re-run.
+//
+// One line per suite: `<file>\t<cpu ms>\t<prior ms>`. `cpu` is the worker's own
+// CPU plus that of every child it reaped, from the end of the previous suite in
+// this worker to the end of this one, so the file's import and its hooks count.
+// `prior` is ALL the worker burned before this suite began: its start, and the
+// load, top level and tests of every earlier suite. A lone run can only need
+// what this worker computed by the end of the suite (a module an earlier suite
+// loaded, a memo an earlier test filled), so cpu plus prior bounds it from
+// above, and sharing the machine only raises CPU.
+//
+// A child still alive, or dead and not yet reaped, when the suite ends has CPU
+// nobody counted yet, so that suite's line says `-` and it runs alone if it is
+// a suspect. The lines also say which suites ran at all.
+//
+// It needs `--parallel`: without it bun runs a preload's hooks once for the
+// whole run, not once per suite. Children come from getrusage(RUSAGE_CHILDREN)
+// through bun:ffi: Bun's own resourceUsage() is the process alone, and a gate
+// heavy suite burns most of its CPU in git. A runtime where either call cannot
+// be made records nothing, and every suspect then runs alone as before.
+import { appendFileSync, readdirSync, readFileSync } from 'fs'
+import { afterAll } from 'bun:test'
+import { SUITE_CPU_ENV } from './judge-suites.ts'
+
+const RUSAGE_SELF = 0
+const RUSAGE_CHILDREN = -1
+
+type Probes = { cpu: () => number; liveChildren: () => number }
+
+/** user plus system ms of this process and of its reaped children, and how
+ *  many children it has that are alive or not yet reaped. null when the
+ *  platform does not answer both */
+function probes(): Probes | null {
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        const { dlopen, FFIType, ptr } = require('bun:ffi') as typeof import('bun:ffi')
+        const darwin = process.platform === 'darwin'
+        const libc = darwin ? '/usr/lib/libSystem.B.dylib' : process.platform === 'linux' ? 'libc.so.6' : null
+        if (libc == null) return null
+        const lib = dlopen(libc, {
+            getrusage: { args: [FFIType.i32, FFIType.ptr], returns: FFIType.i32 },
+            ...(darwin ? { proc_listchildpids: { args: [FFIType.i32, FFIType.ptr, FFIType.i32], returns: FFIType.i32 } } : {}),
+        })
+        // struct rusage opens with two struct timeval (64 bit seconds, then
+        // microseconds), user then system, on both platforms
+        const buf = new BigInt64Array(18)
+        const view = new DataView(buf.buffer)
+        const tv = (o: number): number => Number(view.getBigInt64(o, true)) * 1000 + view.getInt32(o + 8, true) / 1000
+        const one = (who: number): number => {
+            if (lib.symbols.getrusage(who, ptr(buf)) !== 0) throw new Error('getrusage failed')
+            return tv(0) + tv(16)
+        }
+        const pids = new Int32Array(1024)
+        const listChildren = (lib.symbols as Record<string, (...a: unknown[]) => unknown>)['proc_listchildpids']
+        const liveChildren = darwin
+            ? (): number => Number(listChildren!(process.pid, ptr(pids), pids.byteLength))
+            // a zombie is still listed with this pid as its parent
+            : (): number => {
+                let n = 0
+                for (const d of readdirSync('/proc')) {
+                    if (!/^\d+$/.test(d)) continue
+                    try { if (readFileSync(`/proc/${d}/stat`, 'utf8').replace(/^.*\) /s, '').split(' ')[1] === String(process.pid)) n++ } catch { /* gone */ }
+                }
+                return n
+            }
+        liveChildren()
+        return { cpu: () => one(RUSAGE_SELF) + one(RUSAGE_CHILDREN), liveChildren }
+    } catch {
+        return null
+    }
+}
+
+const out = process.env[SUITE_CPU_ENV]
+const p = out != null && out !== '' ? probes() : null
+if (out != null && p != null) {
+    let ended = p.cpu()
+    afterAll(() => {
+        const now = p.cpu()
+        // every suite gets a line, since the line is also how the runner knows
+        // it ran; a suite whose figure cannot be trusted says `-`
+        try {
+            const figure = p.liveChildren() === 0 ? `${Math.round(now - ended)}\t${Math.round(ended)}` : '-\t-'
+            appendFileSync(out, `${Bun.main}\t${figure}\n`)
+        } catch { /* no line: the suite runs alone if it is a suspect */ }
+        ended = now
+    })
+}

package/src/suite/test-units.ts

@@ -0,0 +1,243 @@
+// WHICH TEST CHECK RUNS WHICH TEST FILE. Every `*.test.ts` in the repo is
+// discovered, each one belongs to the unit whose directory holds it
+// (the root takes the rest), and each unit's check is `test` for the root and
+// `test:<unit id>` for a sub-unit. A file that no check runs is an ORPHAN.
+//
+// Pure: every read goes through the reader it is handed, so the gate hands it
+// the disk and a check hands it its context. The constraint a reader breaks
+// first: bun REPLACES the bunfig ignores with the command line ones, and the
+// suite runner puts them back (judge-suites.ts planPhases), so what runs is
+// always the UNION modelled here.
+import { basename, posix } from 'path'
+import { rawName, routesToShipkit, slotBody } from '../core/canonical-scripts.ts'
+import { parseJsonc } from '../ts/jsonc.ts'
+import { bunfigTest, normSuitePath, parseBunTestScript, type BunfigTest, type TestsConfig } from './judge-suites.ts'
+
+export { bunfigTest, type BunfigTest }
+
+/** bun's own discovery pattern, as a glob a listing takes */
+export const TEST_GLOB = '**/*{.test,.spec,_test_,_spec_}.{js,ts,jsx,tsx}'
+
+/** bun's own discovery pattern: `**{.test,.spec,_test_,_spec_}.{js,ts,jsx,tsx}` */
+export const isTestFile = (rel: string): boolean => /(\.test|\.spec|_test_|_spec_)\.(js|ts|jsx|tsx)$/.test(rel)
+
+/** the check id a unit's tests run under */
+export const testCheckId = (unitId: string): string => (unitId === 'root' ? 'test' : `test:${unitId}`)
+
+/** `bun test` flags that take the NEXT token as their value */
+const VALUE_FLAGS = new Set(['--timeout', '--rerun-each', '-t', '--test-name-pattern', '--preload', '--coverage-reporter', '--coverage-dir', '--reporter', '--reporter-outfile', '--max-concurrency', '--shard', '--timings', '--path-ignore-patterns', '--seed', '--retry', '--cwd', '--config', '-c', '--env-file', '--tsconfig-override'])
+
+/** what one test check hands `bun test`, as far as discovery is concerned */
+export type TestRun = {
+    /** the `bun test` arguments, or null when the body is not a plain `bun test`
+     *  and nothing can say which files it runs */
+    args: string[] | null
+    bunfig: BunfigTest
+    /** ignores the gate adds (the sub-unit dirs, for the root), relative to the run's cwd */
+    extraIgnores: string[]
+}
+
+/** does this run reach this file (relative to the run's cwd) ? */
+export function runReaches(file: string, run: TestRun): { runs: true } | { runs: false; ignoredBy: string } | { runs: false; outside: string } | null {
+    if (run.args == null) return null
+    const f = normSuitePath(file)
+    const cli: string[] = []
+    const filters: string[] = []
+    for (let i = 0; i < run.args.length; i++) {
+        const a = run.args[i]!
+        if (a.startsWith('--path-ignore-patterns=')) { cli.push(a.slice('--path-ignore-patterns='.length)); continue }
+        if (a === '--path-ignore-patterns') { cli.push(run.args[++i] ?? ''); continue }
+        if (VALUE_FLAGS.has(a)) { i++; continue }
+        if (!a.startsWith('-')) filters.push(a)
+    }
+    for (const g of [...run.bunfig.ignores, ...cli, ...run.extraIgnores]) if (g !== '' && new Bun.Glob(g).match(f)) return { runs: false, ignoredBy: g }
+    if (run.bunfig.root != null && run.bunfig.root !== '.' && !f.startsWith(`${run.bunfig.root}/`))
+        return { runs: false, outside: `bunfig.toml [test] root = "${run.bunfig.root}"` }
+    if (filters.length > 0 && !filters.some((p) => (p.startsWith('./') || p.startsWith('/') ? f.startsWith(normSuitePath(p)) : f.includes(p))))
+        return { runs: false, outside: `the \`bun test\` arguments name only ${filters.join(', ')}` }
+    return { runs: true }
+}
+
+/** a sub-unit as its generated checks see it: declared, never scanned.
+ *  `releases` false = a `release: false` unit, which only scopes checks */
+export type TestUnitDecl = { id: string; rel: string; releases: boolean; tests?: TestsConfig; tsconfig?: string; deps?: string[] }
+
+/** the sub-units a repo declares, read straight off `units[]`. A unit whose
+ *  RELEASE resolution fails (a missing npm flag, a bad tag) still gets its
+ *  tests run: a packaging defect is rule SK010's to report, not a reason to
+ *  stop testing the code */
+export function declaredTestUnits(units: { dir?: unknown; id?: string; tests?: TestsConfig; tsconfig?: string; release?: boolean; deps?: string[] }[] | undefined): TestUnitDecl[] {
+    const out: TestUnitDecl[] = []
+    const seen = new Set<string>(['root'])
+    for (const d of units ?? []) {
+        if (typeof d.dir !== 'string' || d.dir === '') continue
+        const rel = normSuitePath(d.dir).replace(/\/+$/, '')
+        const id = d.id ?? basename(rel)
+        if (seen.has(id)) continue
+        seen.add(id)
+        out.push({ id, rel, releases: d.release !== false, ...(d.tests == null ? {} : { tests: d.tests }), ...(d.tsconfig == null ? {} : { tsconfig: d.tsconfig }), ...(d.deps == null ? {} : { deps: d.deps }) })
+    }
+    return out
+}
+
+type Pkg = { name?: string; scripts?: Record<string, string>; dependencies?: Record<string, string>; devDependencies?: Record<string, string>; peerDependencies?: Record<string, string>; optionalDependencies?: Record<string, string> }
+const parsePkg = (text: string | null): Pkg | null => {
+    if (text == null) return null
+    try { return JSON.parse(text) as Pkg } catch { return null }
+}
+
+/** the script body a sub-unit's check runs: its own `test:raw`, else a `test`
+ *  that does not route to shipkit, else a plain `bun test`. A unit's own
+ *  preload and bunfig apply because the check runs in the unit's dir */
+export function unitTestBody(pkgText: string | null): { script: string | null; body: string } {
+    const scripts = parsePkg(pkgText)?.scripts ?? {}
+    const raw = scripts[rawName('test')]
+    if (raw != null) return { script: rawName('test'), body: raw }
+    const t = scripts['test']
+    if (t != null && !routesToShipkit(t)) return { script: 'test', body: t }
+    return { script: null, body: 'bun test' }
+}
+
+/** the sibling units a unit's tests can read: the ones its package.json names
+ *  in any dependency block, and the ones its tsconfig `paths` point into (a
+ *  bench resolving the libs at their source folders names them nowhere else).
+ *  Their dirs belong to its inputs. Derived, never hand kept, and over
+ *  declaring is safe where under declaring is a stale green */
+export function siblingUnits(units: TestUnitDecl[], read: (rel: string) => string | null): Map<string, string[]> {
+    const pkgs = new Map(units.map((u) => [u.id, parsePkg(read(posix.join(u.rel, 'package.json')))]))
+    const byName = new Map<string, TestUnitDecl>()
+    for (const u of units) { const n = pkgs.get(u.id)?.name; if (n != null) byName.set(n, u) }
+    const out = new Map<string, string[]>()
+    for (const u of units) {
+        const p = pkgs.get(u.id)
+        const names = new Set([p?.dependencies, p?.devDependencies, p?.peerDependencies, p?.optionalDependencies].flatMap((b) => Object.keys(b ?? {})))
+        const tsconfig = unitTsconfig(u)
+        const targets = pathTargets(read(tsconfig)).map((t) => posix.normalize(posix.join(posix.dirname(tsconfig), t)))
+        out.set(u.id, units.filter((s) => s.id !== u.id && ([...names].some((n) => byName.get(n) === s) || targets.some((t) => under(t, s.rel)))).map((s) => s.rel))
+    }
+    return out
+}
+
+/** the unit's own tsconfig, repo-relative: the declared one, else `<dir>/tsconfig.json` */
+export const unitTsconfig = (u: TestUnitDecl): string => (u.tsconfig != null ? normSuitePath(u.tsconfig) : posix.join(u.rel, 'tsconfig.json'))
+
+/** every target of a tsconfig `compilerOptions.paths`, as written */
+function pathTargets(text: string | null): string[] {
+    if (text == null) return []
+    try {
+        const paths = (parseJsonc(text) as { compilerOptions?: { paths?: Record<string, unknown> } } | null)?.compilerOptions?.paths ?? {}
+        return Object.values(paths).flatMap((v) => (Array.isArray(v) ? v.filter((x): x is string => typeof x === 'string') : []))
+    } catch { return [] }
+}
+
+/** the root run's ignores for the sub-unit dirs: each has a check of its own,
+ *  run in its own dir with its own preload */
+export const unitIgnores = (units: TestUnitDecl[]): string[] => units.map((u) => `${u.rel}/**`)
+
+const under = (file: string, rel: string): boolean => rel === '' || file === rel || file.startsWith(`${rel}/`)
+
+export type Orphan = { file: string; why: string }
+export type TestPlan = {
+    /** check id → the repo-relative files it runs */
+    runs: Map<string, string[]>
+    /** a file a bunfig ignore keeps out, on purpose */
+    ignored: { file: string; pattern: string; checkId: string }[]
+    /** a file no check runs */
+    orphans: Orphan[]
+    /** checks whose body is not a plain `bun test`, so what they run is unknowable */
+    opaque: string[]
+    /** the files of those checks, each with the check it belongs to */
+    unread: { file: string; checkId: string }[]
+}
+
+/** which check runs each discovered test file. `rootRun` null = the repo has
+ *  no test slot at all (unfilled, or turned off), so its files run nowhere */
+export function planTests(o: {
+    files: string[]
+    units: TestUnitDecl[]
+    rootRun: TestRun | null
+    rootWhy?: string
+    unitRun: (u: TestUnitDecl) => TestRun
+}): TestPlan {
+    const runs = new Map<string, string[]>([['test', []], ...o.units.map((u) => [testCheckId(u.id), []] as [string, string[]])])
+    const ignored: TestPlan['ignored'] = []
+    const orphans: Orphan[] = []
+    const opaque = new Set<string>()
+    const unread: TestPlan['unread'] = []
+    for (const file of o.files.filter(isTestFile).map(normSuitePath).sort()) {
+        // the deepest unit wins, so a unit nested in another still owns its own
+        const unit = o.units.filter((u) => under(file, u.rel)).sort((a, b) => b.rel.length - a.rel.length)[0]
+        const checkId = unit == null ? 'test' : testCheckId(unit.id)
+        const run = unit == null ? o.rootRun : o.unitRun(unit)
+        if (run == null) { orphans.push({ file, why: o.rootWhy ?? 'the repo has no test check' }); continue }
+        const rel = unit == null ? file : file.slice(unit.rel.length + 1)
+        const r = runReaches(rel, run)
+        if (r == null) { opaque.add(checkId); unread.push({ file, checkId }); continue }
+        if (r.runs) runs.get(checkId)!.push(file)
+        else if ('ignoredBy' in r) ignored.push({ file, pattern: r.ignoredBy, checkId })
+        else orphans.push({ file, why: `${checkId} never reaches it: ${r.outside}` })
+    }
+    return { runs, ignored, orphans, opaque: [...opaque], unread }
+}
+
+/** the whole plan for a repo, from what its config and files say. `read`
+ *  answers a repo-relative path, null when absent */
+export function testPlanOf(o: {
+    config: { defaults?: Record<string, unknown>; units?: { dir?: unknown; id?: string; tests?: TestsConfig; tsconfig?: string; release?: boolean; deps?: string[] }[] }
+    files: string[]
+    read: (rel: string) => string | null
+}): TestPlan & { units: TestUnitDecl[] } {
+    const units = declaredTestUnits(o.config.units)
+    let scripts: Record<string, string> = {}
+    try { scripts = (JSON.parse(o.read('package.json') ?? '{}') as { scripts?: Record<string, string> }).scripts ?? {} } catch { scripts = {} }
+    const slot = slotBody('test', o.config.defaults?.['test'] as { cmd?: unknown; logic?: unknown } | false | undefined, scripts)
+    const bunfig = (dir: string): BunfigTest => bunfigTest(o.read(dir === '' ? 'bunfig.toml' : posix.join(dir, 'bunfig.toml')))
+    const rootRun: TestRun | null = slot.kind === 'override' ? { args: null, bunfig: bunfig(''), extraIgnores: [] }
+        : slot.kind === 'script' ? { args: parseBunTestScript(slot.body), bunfig: bunfig(''), extraIgnores: unitIgnores(units) }
+            : null
+    const rootWhy = slot.kind === 'off' ? '`defaults.test: false` turns the root test check off, so nothing runs it'
+        : slot.kind === 'routed-empty' ? "the `test` script routes to shipkit and there is no 'test:raw' body, so the root test check runs nothing"
+            : 'no `test` or `test:raw` script, so the root has no test check'
+    const plan = planTests({
+        files: o.files,
+        units,
+        rootRun,
+        rootWhy,
+        unitRun: (u) => ({ args: parseBunTestScript(unitTestBody(o.read(posix.join(u.rel, 'package.json'))).body), bunfig: bunfig(u.rel), extraIgnores: [] }),
+    })
+    return { ...plan, units }
+}
+
+/** every suite declared slow, repo relative: the root's own, then each
+ *  unit's under its dir. The runs page and the tests collection read the
+ *  same list, so a file is slow on both or on neither */
+export function declaredSlowFiles(c: { tests?: TestsConfig; units?: { dir?: unknown; tests?: TestsConfig }[] }): string[] {
+    const own = (c.tests?.slow ?? []).map((s) => normSuitePath(s.file))
+    const units = (c.units ?? []).flatMap((u) => (typeof u.dir === 'string' && u.dir !== '' ? (u.tests?.slow ?? []).map((s) => posix.join(normSuitePath(u.dir as string), normSuitePath(s.file))) : []))
+    return [...own, ...units]
+}
+
+/** the root's declared inputs with every sub-unit dir cut out: a path that
+ *  holds a unit becomes the entries beside it, derived from the listing. The
+ *  root check never runs a unit's tests, and what it imports from one comes
+ *  back through its code closure */
+export function inputsOutsideUnits(inputs: string[], unitRels: string[], files: string[]): string[] {
+    const out = new Set<string>()
+    for (const raw of inputs) {
+        const input = raw === '.' ? '' : normSuitePath(raw).replace(/\/+$/, '')
+        const inside = unitRels.filter((u) => under(u, input) && u !== input)
+        if (unitRels.includes(input)) continue
+        if (inside.length === 0) { out.add(raw); continue }
+        for (const f of files) {
+            if (!under(f, input) || inside.some((u) => under(f, u))) continue
+            // the shallowest prefix of f that holds no unit
+            const parts = f.split('/')
+            const skip = input === '' ? 0 : input.split('/').length
+            for (let i = skip + 1; i <= parts.length; i++) {
+                const p = parts.slice(0, i).join('/')
+                if (!inside.some((u) => under(u, p))) { out.add(p); break }
+            }
+        }
+    }
+    return [...out].sort()
+}