rvlib-shipkit 1.8.0 → 1.9.0

package/src/suite/judge-suites.ts

@@ -41,7 +41,8 @@ export const DEFAULT_BUDGET_MS = 2000
  *  5s default while the preload said otherwise. */
 export const FULL_TIMEOUT_MS = 120_000
 /** the wall ceiling of one `bun test` phase. bun's per test timeout never
- *  fires on a worker stuck in a SYNCHRONOUS call, and bun's spawnSync has been
+ *  fires on a worker stuck in a SYNCHRONOUS call, or in `expect(p).resolves`
+ *  or `.rejects` on a promise that never settles, and bun's spawnSync has been
  *  seen spinning on a child that already exited (planPhases avoids the known
  *  trigger), so without this a hang lasts until something outside kills it. The quick
  *  tier answers in seconds, so two minutes is far from any real run. */
@@ -182,14 +183,16 @@ export function poolWorkers(p: { cores: number; size: number; held: number }): n
 
 type Timed = { file: string; ms: number }
 
-export function judgeSuites(p: { timings: Record<string, number>; tests: TestsConfig; tier: 'quick' | 'full' }): { over: Timed[]; noLongerSlow: Timed[]; slowest: Timed[] } {
+export function judgeSuites(p: { timings: Record<string, number>; tests: TestsConfig; tier: 'quick' | 'full'; red?: string[] }): { over: Timed[]; noLongerSlow: Timed[]; slowest: Timed[] } {
     const budget = p.tests.budgetMs ?? DEFAULT_BUDGET_MS
     const slow = new Set((p.tests.slow ?? []).map((s) => normSuitePath(s.file)))
+    const red = new Set((p.red ?? []).map(normSuitePath))
     const timed = Object.entries(p.timings).map(([file, ms]) => ({ file: normSuitePath(file), ms })).sort((a, b) => b.ms - a.ms)
     return {
         // the budget guards the tier every edit waits on. The full tier is
-        // where slow suites belong, and it only reports
-        over: p.tier === 'quick' ? timed.filter((t) => t.ms > budget && !slow.has(t.file)) : [],
+        // where slow suites belong, and it only reports. A red suite took
+        // the time of its failure (a timeout, a crash), which its red explains
+        over: p.tier === 'quick' ? timed.filter((t) => t.ms > budget && !slow.has(t.file) && !red.has(t.file)) : [],
         // half the budget, so a declaration does not flap on timing noise
         noLongerSlow: timed.filter((t) => slow.has(t.file) && t.ms < budget / 2),
         slowest: timed.slice(0, 3),
@@ -253,6 +256,20 @@ export function confirmOver(suspects: Timed[], alone: Record<string, AloneRun>,
     return { over: judged.filter((c) => c.over != null), cleared: judged.filter((c) => c.over == null) }
 }
 
+/** a lone run over the budget on CPU runs once more. Under load the
+ *  scheduler can spill a lone run onto the efficiency cores, where the same
+ *  retired instructions cost up to 4x the CPU time (measured on Apple
+ *  silicon), and a spill only ever raises the figure. A wait or a wall
+ *  verdict is not CPU, so it keeps its one run */
+export const retryAlone = (t: Timed, run: AloneRun, budgetMs: number, load: MachineLoad): boolean =>
+    confirmOver([t], { [t.file]: run }, budgetMs, load).over[0]?.over === 'cpu'
+
+/** the run charged less, as confirmOver charges it: the smaller of CPU and wall */
+export const smallerRun = (a: AloneRun, b: AloneRun): AloneRun => {
+    const charge = (r: AloneRun): number => Math.min(r.cpuMs ?? r.wallMs, r.wallMs)
+    return charge(b) < charge(a) ? b : a
+}
+
 const secs = (ms: number): string => `${(ms / 1000).toFixed(1)}s`
 
 /** the line a confirmed suite fails with */
@@ -358,6 +375,37 @@ export function failLines(output: string): Map<string, string> {
 
 const clip = (s: string): string => (s.length > 300 ? `${s.slice(0, 299)}…` : s)
 
+/** the clause the headline carries for the gate line: the command phase one
+ *  ran, and the flags the runner added to the script's own. Per file ignores
+ *  stay out, the slow and serial counts already say what they leave out */
+export function ranAs(phase: string[], rawArgs: string[]): string {
+    const shown = phase.filter((a) => !a.startsWith('--path-ignore-patterns'))
+    const added = shown.filter((a) => !rawArgs.includes(a))
+    return `as \`${['bun', 'test', ...shown].join(' ')}\`${added.length > 0 ? ` (added ${added.join(' ')})` : ''}`
+}
+
+/** the suites bun lists under `Interrupted while still running:` when a
+ *  phase is killed, one indented `<file> (<n>s)` line each */
+export function interruptedSuites(output: string): string[] {
+    const out: string[] = []
+    let inBlock = false
+    for (const line of output.split('\n')) {
+        if (/^Interrupted while still running:\s*$/.test(line)) { inBlock = true; continue }
+        if (!inBlock) continue
+        const m = /^\s+(\S+)\s+\(\d+(?:\.\d+)?m?s\)\s*$/.exec(line)
+        if (m == null) { inBlock = false; continue }
+        out.push(normSuitePath(m[1]!))
+    }
+    return out
+}
+
+/** the line a phase killed at its ceiling fails with. Measured on bun 1.4.3:
+ *  a plain await of a promise that never settles fails at the per test
+ *  timeout, while `expect(p).resolves` or `.rejects` on one blocks the worker
+ *  past it, as a synchronous loop does */
+export const stallWords = (name: string, ceilingMs: number, files: string[]): string =>
+    `${name} ran past its ${secs(ceilingMs)} ceiling and was killed${files.length > 0 ? ` with ${files.join(', ')} still running` : ''}: a test blocked its worker past the per test timeout, either a synchronous loop or expect(...).resolves or .rejects on a promise that never settles`
+
 /** the suites bun names as a crashed parallel worker */
 export function crashedWorkers(output: string): { file: string; signal: string }[] {
     return [...output.matchAll(/^\s*✗\s+(\S+)\s+\(worker crashed: (SIG[A-Z0-9]+)\)\s*$/gm)].map((m) => ({ file: normSuitePath(m[1]!), signal: m[2]! }))

package/src/suite/suite-cli.ts

@@ -20,7 +20,7 @@ import { SUITE_READS_ENV, SUITE_ROOT_ENV, SUITE_SELECT_ENV } from './reads-env.t
 import { changedKeys, dropStaticEdges, extras, isBunSuite, envFilesOf, foldReads, loadErrorModules, loadState, preloadsOf, saveState, selectStatePath, selectSuites, snapshot, suiteFiles, taintedSuites, walkGraph, type GraphEntry, type Selection, type SelectState } from './select.ts'
 import { gitRoot } from '../core/git-root.ts'
 import { loadConfig } from '../ts/config.ts'
-import { aloneRunsNeeded, bunfigTest, type BunfigTest, readPhaseCpu, SUITE_CPU_ENV, type AloneRun, type Confirmed, confirmOver, crashedWorkers, overWords, DEFAULT_BUDGET_MS, FULL_CEILING_MS, failedSuites, failLines, judgeSuites, normSuitePath, parseSuiteFlags, planPhases, QUICK_CEILING_MS, poolWorkers } from './judge-suites.ts'
+import { aloneRunsNeeded, bunfigTest, type BunfigTest, readPhaseCpu, SUITE_CPU_ENV, type AloneRun, type Confirmed, confirmOver, crashedWorkers, interruptedSuites, overWords, ranAs, retryAlone, smallerRun, stallWords, DEFAULT_BUDGET_MS, FULL_CEILING_MS, failedSuites, failLines, judgeSuites, normSuitePath, parseSuiteFlags, planPhases, QUICK_CEILING_MS, poolWorkers } from './judge-suites.ts'
 
 type Phase = { code: number; signal: string | null; stalled: boolean; out: string; timings: Record<string, number>; cpuMs: number | null }
 
@@ -194,7 +194,9 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         const ran = phases.reduce((n, p) => n + lastCount(p.out, /Ran (\d+) tests?/g), 0)
         const failedTests = phases.reduce((n, p) => n + lastCount(p.out, /^\s*(\d+) fail$/gm), 0)
         const timings = Object.assign({}, ...phases.map((p) => p.timings)) as Record<string, number>
-        const judged = judgeSuites({ timings, tests, tier })
+        const crashed = crashedWorkers(p1.out)
+        const red = new Set([...phases.flatMap((p) => failedSuites(p.out)), ...crashed.map((c) => c.file)])
+        const judged = judgeSuites({ timings, tests, tier, red: [...red] })
         const slowest = judged.slowest.length > 0 ? ` (slowest: ${judged.slowest.map((t) => `${t.file} ${secs(t.ms)}`).join(', ')})` : ''
 
         // a parallel timing carries every other suite's load, so a suspect
@@ -205,10 +207,20 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         const parallelCpu = phaseCpu.cpu
         const { rerun, cleared: clearedInPhase } = aloneRunsNeeded(judged.over, parallelCpu, budget, load)
         const aloneAt = Date.now()
-        for (const [i, t] of rerun.entries()) {
-            const run = await runPhase([...plan.alone, t.file], join(dir, `alone-${i}.json`), ceilingMs, tmp, false, null, rec)
+        const runAlone = async (t: { file: string }, name: string): Promise<AloneRun | null> => {
+            const run = await runPhase([...plan.alone, t.file], join(dir, `${name}.json`), ceilingMs, tmp, false, null, rec)
             const wallMs = Object.entries(run.timings).find(([f]) => normSuitePath(f) === t.file)?.[1]
-            if (wallMs != null) aloneRuns[t.file] = { wallMs, cpuMs: run.cpuMs }
+            return wallMs == null ? null : { wallMs, cpuMs: run.cpuMs }
+        }
+        for (const [i, t] of rerun.entries()) {
+            const first = await runAlone(t, `alone-${i}`)
+            if (first == null) continue
+            aloneRuns[t.file] = first
+            if (!retryAlone(t, first, budget, load)) continue
+            const second = await runAlone(t, `alone-${i}-again`)
+            if (second == null) continue
+            aloneRuns[t.file] = smallerRun(first, second)
+            notes.push(`note: ${t.file} ran alone twice, ${secs(first.cpuMs ?? first.wallMs)} then ${secs(second.cpuMs ?? second.wallMs)} of CPU, and the smaller stands: a lone run the scheduler spills onto slower cores burns more CPU for the same work`)
         }
         const aloneMs = Date.now() - aloneAt
         const { over, cleared } = rerun.length > 0 ? confirmOver(rerun, aloneRuns, budget, load) : { over: [], cleared: [] }
@@ -218,12 +230,9 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         // a ceiling kill makes bun report every worker it held as crashed, and
         // those are the suites that never finished, not an outside kill
         const stalledIn = (name: string, p: Phase | null, files: string[]): string | null =>
-            p?.stalled === true
-                ? `${name} ran past its ${secs(ceilingMs)} ceiling and was killed${files.length > 0 ? ` with ${files.join(', ')} still running` : ''}: a test worker stopped answering, most likely stuck in a synchronous call, which no per test timeout can interrupt`
-                : null
-        const crashed = crashedWorkers(p1.out)
+            p?.stalled === true ? stallWords(name, ceilingMs, [...new Set([...files, ...interruptedSuites(p.out)])]) : null
         const killed = p1.stalled ? [] : crashed.map((c) => `killed by ${c.signal} while running ${c.file} (out of memory, or killed from outside)`)
-        const stalls = [stalledIn('the parallel phase', p1, crashed.map((c) => c.file)), stalledIn('the serial phase', p2, phaseFiles(serialArgs ?? [], rawArgs).filter((a) => p2?.timings[normSuitePath(a)] == null))]
+        const stalls = [stalledIn('the parallel phase', p1, crashed.map((c) => c.file)), stalledIn('the serial phase', p2, phaseFiles(serialArgs ?? [], rawArgs).filter((a) => p2?.timings[normSuitePath(a)] == null).map(normSuitePath))]
         for (const line of stalls) if (line != null) killed.unshift(line)
         if (p1.signal != null) killed.push(`killed by ${p1.signal} during the parallel run (out of memory, or killed from outside)`)
         if (p2?.signal != null) {
@@ -233,7 +242,6 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         // the record behind the hub's test files to fix: every suite this run
         // timed, with the alone timing where one was taken, and red when a
         // test in it failed or its worker died
-        const red = new Set([...phases.flatMap((p) => failedSuites(p.out)), ...crashedWorkers(p1.out).map((c) => c.file)])
         const overSet = new Set(over.map((o) => o.file))
         ctx.complete = killed.length === 0
         ctx.out = phases.map((p) => p.out).join('\n')
@@ -241,7 +249,7 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         const at = Date.now()
         const alone = new Map([...over, ...cleared, ...clearedInPhase].map((t) => [t.file, t.ms]))
         const why = failLines(phases.map((p) => p.out).join('\n'))
-        for (const c of crashedWorkers(p1.out)) if (!why.has(c.file)) why.set(c.file, `its worker crashed: ${c.signal}`)
+        for (const c of crashed) if (!why.has(c.file)) why.set(c.file, `its worker crashed: ${c.signal}`)
         // which phase timed each file, and the CPU spent on it: an alone
         // rerun, the parallel phase's own figure, or a phase that ran that one file
         const phaseOf = (f: string): SuitePhase => (p2 != null && Object.keys(p2.timings).some((k) => normSuitePath(k) === f) ? 'serial' : parallel ? 'parallel' : 'sequential')
@@ -289,7 +297,7 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         if (fails.length > 0) { console.log(fails[0]); return 1 }
         const serialCount = serialArgs != null ? phaseFiles(serialArgs, rawArgs).length : 0
         const chosen = ctx.sel?.picked != null ? `, the ${ctx.sel.picked.length} of ${ctx.sel.total} a change reaches` : ''
-        console.log(`ok: Ran ${ran} tests across ${Object.keys(timings).length || '?'} suites${chosen}, ${parallel ? `parallel on ${workers} workers` : 'one at a time'}${serialCount > 0 ? ` then ${serialCount} serial` : ''}, ${tier} tier${slowest}`)
+        console.log(`ok: Ran ${ran} tests across ${Object.keys(timings).length || '?'} suites${chosen}, ${parallel ? `parallel on ${workers} workers` : 'one at a time'}${serialCount > 0 ? ` then ${serialCount} serial` : ''}, ${tier} tier, ${ranAs(plan.parallel, rawArgs)}${slowest}`)
         return 0
     } finally {
         rmSync(dir, { recursive: true, force: true })

package/src/ts/typecheck-cache.ts

@@ -0,0 +1,11 @@
+// the gate's typecheck keeps its own incremental cache, one file per project.
+// a tsconfig's tsBuildInfoFile is shared by every tsc that runs that config (a
+// watcher task, a raw script), and a verdict read off a cache another process
+// writes is not a verdict on this tree. no fs here: it only names paths.
+import { basename, dirname, join } from 'path'
+
+export const gateBuildInfoArgs = (tsconfig: string): string[] => [
+    '--incremental',
+    '--tsBuildInfoFile',
+    join(dirname(tsconfig), 'node_modules/.cache/shipkit', `${basename(tsconfig, '.json')}.tsbuildinfo`),
+]

package/src/ts/typecheck-cli.ts

@@ -10,6 +10,7 @@
 // who fixes the one error tsc showed and re-runs into a second one has been
 // told the truth twice and believed it once.
 import { resolveTypecheckProjects } from 'rvlib-shipkit/src/ts/tsconfig-projects.ts'
+import { gateBuildInfoArgs } from 'rvlib-shipkit/src/ts/typecheck-cache.ts'
 
 export function typecheckCli(argv: string[]): number {
     const [tsc, tsconfig] = argv
@@ -30,7 +31,7 @@ export function typecheckCli(argv: string[]): number {
 
     let worst = 0
     for (const p of resolved.projects) {
-        const r = Bun.spawnSync([tsc, '--noEmit', '-p', p], { stdout: 'inherit', stderr: 'inherit' })
+        const r = Bun.spawnSync([tsc, '--noEmit', '-p', p, ...gateBuildInfoArgs(p)], { stdout: 'inherit', stderr: 'inherit' })
         if (r.exitCode !== 0) {
             console.log(`FAIL: ${p}`)
             worst = r.exitCode ?? 1