rvlib-shipkit 0.37.0 → 0.38.0

package/src/suite/suite-cli.ts

@@ -0,0 +1,114 @@
+#!/usr/bin/env bun
+// the floor `test` slot for a plain `bun test` script: quick tier suites in
+// parallel workers, the full tier one at a time, the declared serial ones after,
+// and each quick tier suite timed against the budget.
+// The LAST stdout line is the task headline the gate prints, and it keeps bun's
+// `Ran N tests` so the gate can still report what ran.
+//
+// usage: suite-cli [--budget-ms N] [--slow file]... [--serial file]... [--declared-in <where>] -- <bun test args>
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import { FULL_TESTS_ENV } from '../ledger/gate-units.ts'
+import { confirmOver, crashedWorkers, DEFAULT_BUDGET_MS, judgeSuites, normSuitePath, parseSuiteFlags, planPhases } from './judge-suites.ts'
+
+type Phase = { code: number; signal: string | null; out: string; timings: Record<string, number> }
+
+async function runPhase(args: string[], timingsFile: string | null, echo = true): Promise<Phase> {
+    const timing = timingsFile != null ? [`--timings=${timingsFile}`, '--update-timings'] : []
+    const proc = Bun.spawn(['bun', 'test', ...args, ...timing], { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' })
+    let out = ''
+    const pump = async (stream: ReadableStream<Uint8Array>, sink: NodeJS.WriteStream): Promise<void> => {
+        const dec = new TextDecoder()
+        for await (const chunk of stream) {
+            const text = dec.decode(chunk)
+            out += text
+            if (echo) sink.write(text)
+        }
+    }
+    await Promise.all([pump(proc.stdout, process.stdout), pump(proc.stderr, process.stderr)])
+    const code = await proc.exited
+    let timings: Record<string, number> = {}
+    try { if (timingsFile != null && existsSync(timingsFile)) timings = (JSON.parse(readFileSync(timingsFile, 'utf8')) as { files?: Record<string, number> }).files ?? {} }
+    catch { timings = {} }
+    return { code, signal: proc.signalCode ?? null, out, timings }
+}
+
+const secs = (ms: number): string => `${(ms / 1000).toFixed(1)}s`
+const lastCount = (out: string, re: RegExp): number => Number([...out.matchAll(re)].at(-1)?.[1] ?? 0)
+
+export async function suiteCli(argv: string[]): Promise<number> {
+    const parsed = parseSuiteFlags(argv)
+    if (typeof parsed === 'string') { console.log(`FAIL: suite runner: ${parsed}`); return 2 }
+    const { tests, rawArgs, declaredIn } = parsed
+    const tier = process.env[FULL_TESTS_ENV] === '1' ? 'full' : 'quick'
+    const budget = tests.budgetMs ?? DEFAULT_BUDGET_MS
+
+    // a declaration naming a gone suite would silently stop excluding anything
+    const gone = [...(tests.slow ?? []).map((s) => ['slow', s.file]), ...(tests.serial ?? []).map((s) => ['serial', s.file])]
+        .filter(([, f]) => !existsSync(f!))
+    if (gone.length > 0) {
+        for (const [list, f] of gone) console.log(`FAIL: ${declaredIn}.${list} names ${normSuitePath(f!)}, which does not exist`)
+        return 1
+    }
+
+    const help = Bun.spawnSync(['bun', 'test', '--help'], { stdout: 'pipe', stderr: 'pipe' })
+    const helpText = `${help.stdout.toString()}${help.stderr.toString()}`
+    // the full tier runs one suite at a time: its suites spawn git and whole
+    // gates, and sharing a run measured slower than sequential and timed out
+    const parallel = helpText.includes('--parallel') && tier === 'quick'
+    const timed = helpText.includes('--update-timings')
+    const plan = planPhases({ tests, tier, rawArgs, parallel })
+    const dir = mkdtempSync(join(tmpdir(), 'shipkit-suite-'))
+    const notes: string[] = []
+    if (!helpText.includes('--parallel')) notes.push(`skip: parallel suites (bun ${Bun.version} has no --parallel)`)
+    if (!timed) notes.push(`skip: suite budget (bun ${Bun.version} writes no --timings)`)
+    try {
+        const p1 = await runPhase(plan.parallel, timed ? join(dir, 'parallel.json') : null)
+        const p2 = plan.serial != null ? await runPhase(plan.serial, timed ? join(dir, 'serial.json') : null) : null
+        const phases = [p1, ...(p2 != null ? [p2] : [])]
+        const ran = phases.reduce((n, p) => n + lastCount(p.out, /Ran (\d+) tests?/g), 0)
+        const failedTests = phases.reduce((n, p) => n + lastCount(p.out, /^\s*(\d+) fail$/gm), 0)
+        const timings = Object.assign({}, ...phases.map((p) => p.timings)) as Record<string, number>
+        const judged = judgeSuites({ timings, tests, tier })
+        const slowest = judged.slowest.length > 0 ? ` (slowest: ${judged.slowest.map((t) => `${t.file} ${secs(t.ms)}`).join(', ')})` : ''
+
+        // a parallel timing carries every other suite's load, so a suspect is
+        // timed again alone, one process, before it can fail the task
+        let over = judged.over
+        if (over.length > 0) {
+            const alone = await runPhase([...rawArgs, ...over.map((t) => t.file)], join(dir, 'alone.json'), false)
+            over = confirmOver(over, alone.timings, budget)
+        }
+
+        const killed = crashedWorkers(p1.out).map((c) => `killed by ${c.signal} while running ${c.file} (out of memory, or killed from outside)`)
+        if (p1.signal != null) killed.push(`killed by ${p1.signal} during the parallel run (out of memory, or killed from outside)`)
+        if (p2?.signal != null) {
+            const unfinished = (plan.serial ?? []).filter((a) => !a.startsWith('-') && !rawArgs.includes(a) && p2.timings[normSuitePath(a)] == null)
+            killed.push(`killed by ${p2.signal} while running ${unfinished.join(' or ')} (out of memory, or killed from outside)`)
+        }
+        // never blocking: a suite that got faster must not stop a release
+        const stale = judged.noLongerSlow.map((t) => `WARN ${declaredIn}.slow declares ${t.file}, which ran in ${secs(t.ms)}, under half the ${budget}ms budget: remove the declaration`)
+
+        for (const n of notes) console.log(n)
+        for (const s of stale) console.log(s)
+        const fails: string[] = [
+            ...killed.map((k) => `FAIL: ${k}`),
+            ...(failedTests > 0 || (phases.some((p) => p.code !== 0 && p.signal == null) && killed.length === 0)
+                ? [`FAIL: ${failedTests} test(s) failed, Ran ${ran} tests${slowest}`] : []),
+            ...(over.length > 0
+                ? [`FAIL: ${over.length} suite(s) over the ${budget}ms budget: ${over.map((t) => `${t.file} ${secs(t.ms)}`).join(', ')} (timed alone). Make each fast, or declare it in ${declaredIn}.slow with what it catches and why it cannot be fast`]
+                : []),
+        ]
+        // the most actionable failure prints LAST, where the gate reads its headline
+        for (const f of fails.slice(1).reverse()) console.log(f)
+        if (fails.length > 0) { console.log(fails[0]); return 1 }
+        const serialCount = plan.serial != null ? plan.serial.length - rawArgs.length : 0
+        console.log(`ok: Ran ${ran} tests across ${Object.keys(timings).length || '?'} suites, ${parallel ? 'parallel' : 'one at a time'}${serialCount > 0 ? ` then ${serialCount} serial` : ''}, ${tier} tier${slowest}`)
+        return 0
+    } finally {
+        rmSync(dir, { recursive: true, force: true })
+    }
+}
+
+if (import.meta.main) process.exit(await suiteCli(process.argv.slice(2)))