rvlib-shipkit 1.7.0 → 1.8.0

package/src/serve/run-lanes.ts

@@ -0,0 +1,134 @@
+// THE RUNS OF ONE HUB: who attaches, who runs beside whom, who waits.
+//
+//   - a run with a key identical to one in flight attaches to it: the lines it
+//     printed so far, then the rest, then its exit code. To a queued one always
+//     (it reads the tree after both asks), to a running one only while the tree
+//     stamp equals the one taken at its start
+//   - a run that writes the tree (a repair pass) waits for every run ahead of
+//     it, and every later run waits for it
+//   - every other run starts at once: the machine slot pool each check body
+//     takes bounds the CPU, never this queue
+//
+// the one constraint a reader breaks first: the stamp is taken when a run
+// STARTS, before its body reads anything, so a tree that moves during the run
+// can only make a later asker run on its own, never adopt a stale verdict.
+import { createHash } from 'crypto'
+import { statSync } from 'fs'
+import { join } from 'path'
+import { gitBin } from '../core/git-bin.ts'
+
+export type LaneAsk = {
+    /** what makes two runs the same run, null: never shared */
+    key: string | null
+    /** the run rewrites tracked files (a repair pass) */
+    writes: boolean
+    print: (line: string) => void
+    /** no note lines: a --json run prints the json and nothing else */
+    quiet?: boolean
+}
+
+type Entry = {
+    key: string | null
+    writes: boolean
+    running: boolean
+    stamp: string | null
+    lines: string[]
+    listeners: Set<(line: string) => void>
+    done: Promise<number>
+}
+
+export const ATTACHED_NOTE = 'note: an identical run is in flight on this tree, so this one takes its lines and its verdict'
+export const WAIT_NOTE = 'note: a run that rewrites files is in flight, so this one starts when it ends'
+
+export class RunLanes {
+    private entries: Entry[] = []
+    constructor(private readonly stampOf: () => Promise<string | null> | string | null) {}
+
+    /** runs in flight, queued ones included */
+    get size(): number {
+        return this.entries.length
+    }
+
+    async submit(ask: LaneAsk, body: (print: (line: string) => void) => Promise<number>): Promise<number> {
+        if (ask.key != null) {
+            const same = this.entries.find((e) => e.key === ask.key)
+            if (same != null && (!same.running || (same.stamp != null && same.stamp === (await this.stampOf())))) return this.attach(same, ask.print)
+        }
+        const ahead = this.entries.filter((e) => ask.writes || e.writes).map((e) => e.done.catch(() => 1))
+        let finish!: (code: number) => void
+        let fail!: (e: unknown) => void
+        const entry: Entry = {
+            key: ask.key,
+            writes: ask.writes,
+            running: false,
+            stamp: null,
+            lines: [],
+            listeners: new Set([ask.print]),
+            done: new Promise<number>((res, rej) => { finish = res; fail = rej }),
+        }
+        // the asker gets the throw; an attacher, if any, its own rejection
+        entry.done.catch(() => {})
+        this.entries.push(entry)
+        const tee = (line: string): void => {
+            entry.lines.push(line)
+            for (const l of entry.listeners) l(line)
+        }
+        try {
+            if (ahead.length > 0) {
+                if (ask.quiet !== true) tee(WAIT_NOTE)
+                await Promise.all(ahead)
+            }
+            entry.stamp = ask.key == null ? null : await this.stampOf()
+            entry.running = true
+            const code = await body(tee)
+            finish(code)
+            return code
+        } catch (e) {
+            fail(e)
+            throw e
+        } finally {
+            this.entries = this.entries.filter((x) => x !== entry)
+        }
+    }
+
+    private attach(e: Entry, print: (line: string) => void): Promise<number> {
+        print(ATTACHED_NOTE)
+        for (const l of e.lines) print(l)
+        e.listeners.add(print)
+        return e.done.finally(() => e.listeners.delete(print))
+    }
+}
+
+/** the key of a run: its kind and its options, whoever asked */
+export function runKey(kind: string, opts: Record<string, unknown>): string {
+    const { invokedBy: _by, ...rest } = opts
+    const sorted = Object.fromEntries(Object.entries(rest).filter(([, v]) => v !== undefined).sort(([a], [b]) => (a < b ? -1 : 1)))
+    return `${kind} ${JSON.stringify(sorted)}`
+}
+
+/** a run with its repair pass on: `fix` and `fixRules` default to on */
+export const writesTree = (opts: Record<string, unknown>): boolean => opts.fix !== false || opts.fixRules !== false
+
+/** HEAD, what git status lists, and the stat of each listed path, hashed:
+ *  equal stamps mean no tracked or untracked file moved in between. Null
+ *  outside git, where nothing attaches */
+export function treeStamp(root: string): string | null {
+    const git = (args: string[]): string | null => {
+        const p = Bun.spawnSync([gitBin(), '--no-optional-locks', ...args], { cwd: root, stdout: 'pipe', stderr: 'pipe' })
+        return p.exitCode === 0 ? p.stdout.toString() : null
+    }
+    const head = git(['rev-parse', '-q', '--verify', 'HEAD']) ?? 'none'
+    const status = git(['status', '--porcelain=v1', '-z', '-uall', '--no-renames', '--ignore-submodules'])
+    if (status == null) return null
+    const h = createHash('sha256').update(head).update('\0').update(status)
+    for (const entry of status.split('\0')) {
+        if (entry.length < 4) continue
+        try {
+            const st = statSync(join(root, entry.slice(3)))
+            h.update(`\0${st.size}:${st.mtimeMs}:${st.ctimeMs}`)
+        } catch {
+            h.update('\0gone')
+        }
+    }
+    return h.digest('hex')
+}

package/src/suite/judge-suites.ts

@@ -24,6 +24,8 @@ export type TestsConfig = {
     slow?: SlowSuite[]
     /** suites that cannot share a run: they run one at a time after the parallel phase */
     serial?: SerialSuite[]
+    /** true: the quick tier runs only the suites a change since the last run can reach (their import closure and what its modules read at runtime) and the ones that did not pass then. Off by default: every suite runs every time. The full tier always runs every suite */
+    select?: boolean
 }
 
 export const DEFAULT_BUDGET_MS = 2000
@@ -46,10 +48,10 @@ export const FULL_TIMEOUT_MS = 120_000
 export const QUICK_CEILING_MS = 120_000
 /** the full tier runs every suite one after the other, about three minutes */
 export const FULL_CEILING_MS = 1_800_000
-const TESTS_KEYS = ['budgetMs', 'slow', 'serial']
+const TESTS_KEYS = ['budgetMs', 'slow', 'serial', 'select']
 const SLOW_KEYS = ['file', 'catches', 'whyNotFast']
 const SERIAL_KEYS = ['file', 'why']
-const OWN_FLAGS = ['--budget-ms', '--slow', '--serial', '--declared-in', '--ceiling-ms', '--unit']
+const OWN_FLAGS = ['--budget-ms', '--slow', '--serial', '--declared-in', '--ceiling-ms', '--unit', '--select']
 
 export const normSuitePath = (file: string): string => posix.normalize(file.replace(/\\/g, '/')).replace(/^\.\//, '')
 
@@ -84,13 +86,14 @@ export function suiteFlags(t: TestsConfig | undefined): string[] {
         ...(t.budgetMs != null ? ['--budget-ms', String(t.budgetMs)] : []),
         ...(t.slow ?? []).flatMap((s) => ['--slow', s.file]),
         ...(t.serial ?? []).flatMap((s) => ['--serial', s.file]),
+        ...(t.select === true ? ['--select', 'on'] : []),
     ]
 }
 
 /** the inverse of suiteFlags, plus where the declarations live (printed in
  *  every fix line) and the `bun test` arguments after `--` */
 export function parseSuiteFlags(argv: string[]): { tests: TestsConfig; rawArgs: string[]; declaredIn: string; ceilingMs: number | null; unit: string } | string {
-    const tests: Required<TestsConfig> = { budgetMs: DEFAULT_BUDGET_MS, slow: [], serial: [] }
+    const tests: Required<TestsConfig> = { budgetMs: DEFAULT_BUDGET_MS, slow: [], serial: [], select: false }
     let declaredIn = 'repo.config.ts tests'
     let ceilingMs: number | null = null
     let unit = ''
@@ -109,6 +112,7 @@ export function parseSuiteFlags(argv: string[]): { tests: TestsConfig; rawArgs:
         else if (flag === '--serial') tests.serial.push({ file: value, why: '(declared)' })
         else if (flag === '--ceiling-ms') ceilingMs = Number(value)
         else if (flag === '--unit') unit = value
+        else if (flag === '--select') { if (value !== 'off' && value !== 'on') return '--select takes on or off'; tests.select = value === 'on' }
         else declaredIn = value
     }
     if (!(tests.budgetMs > 0)) return '--budget-ms must be a positive number'
@@ -116,6 +120,8 @@ export function parseSuiteFlags(argv: string[]): { tests: TestsConfig; rawArgs:
     return { tests, rawArgs: argv.slice(i), declaredIn, ceilingMs, unit }
 }
 
+const PARALLEL_COUNT = /^--parallel=([1-9]\d*)$/
+
 /** what `bun test` receives in each phase. The serial phase names its files
  *  explicitly, because bun reports no tests when a path is both named and ignored.
  *
@@ -124,8 +130,15 @@ export function parseSuiteFlags(argv: string[]): { tests: TestsConfig; rawArgs:
  *  did. `--parallel` implies per file isolation, and under it a worker's
  *  spawnSync can spin forever on a child that already exited (bun issue
  *  34069), which hung one full run in eight. A repo that passes `--isolate`
- *  itself keeps it. */
-export function planPhases(p: { tests: TestsConfig; tier: 'quick' | 'full'; rawArgs: string[]; parallel: boolean; noIsolate?: boolean; configIgnores?: string[]; workers?: number }): { parallel: string[]; serial: string[] | null } {
+ *  itself keeps it.
+ *
+ *  `workers` (the pool share) is a ceiling under the repo's own
+ *  `--parallel=<n>`: the phase takes the smaller, as the one flag on its line,
+ *  since bun reads the last `--parallel` it is given. No other line names a
+ *  worker count: the serial phase, a phase one that is not parallel (the full
+ *  tier) and `alone` (what a suspect's rerun puts before its file) each run
+ *  one suite at a time, so the repo's `--parallel` stays off them. */
+export function planPhases(p: { tests: TestsConfig; tier: 'quick' | 'full'; rawArgs: string[]; parallel: boolean; noIsolate?: boolean; configIgnores?: string[]; workers?: number }): { parallel: string[]; serial: string[] | null; alone: string[]; workers: number | null } {
     const slow = (p.tests.slow ?? []).map((s) => normSuitePath(s.file))
     const serial = (p.tests.serial ?? []).map((s) => normSuitePath(s.file))
     const own = [...new Set([...(p.tier === 'quick' ? slow : []), ...serial])]
@@ -139,9 +152,15 @@ export function planPhases(p: { tests: TestsConfig; tier: 'quick' | 'full'; rawA
     const named = p.rawArgs.some((a) => a === '--timeout' || a.startsWith('--timeout='))
     const args = p.tier === 'full' && !named ? [`--timeout=${FULL_TIMEOUT_MS}`, ...p.rawArgs] : p.rawArgs
     const isolated = p.rawArgs.some((a) => a === '--isolate' || a === '--no-isolate')
+    const repoCount = p.rawArgs.map((a) => PARALLEL_COUNT.exec(a)?.[1]).filter((n) => n != null).map(Number).at(-1)
+    const bounds = [p.workers, repoCount].filter((n) => n != null)
+    const workers = p.parallel && bounds.length > 0 ? Math.min(...bounds) : null
+    const solo = args.filter((a) => a !== '--parallel' && !PARALLEL_COUNT.test(a))
     return {
-        parallel: [...args, ...(p.parallel ? [p.workers != null ? `--parallel=${p.workers}` : '--parallel'] : []), ...(p.parallel && p.noIsolate === true && !isolated ? ['--no-isolate'] : []), ...ignored.map((f) => `--path-ignore-patterns=${f}`)],
-        serial: serialRun.length > 0 ? [...args, ...serialRun] : null,
+        parallel: [...solo, ...(p.parallel ? [workers != null ? `--parallel=${workers}` : '--parallel'] : []), ...(p.parallel && p.noIsolate === true && !isolated ? ['--no-isolate'] : []), ...ignored.map((f) => `--path-ignore-patterns=${f}`)],
+        serial: serialRun.length > 0 ? [...solo, ...serialRun] : null,
+        alone: solo,
+        workers,
     }
 }
 
@@ -348,6 +367,7 @@ export function testsConfigProblems(t: TestsConfig): string[] {
     const problems: string[] = []
     for (const k of Object.keys(t)) if (!TESTS_KEYS.includes(k)) problems.push(`tests.${k} is not a tests field (fields: ${TESTS_KEYS.join(', ')})`)
     if (t.budgetMs != null && !(typeof t.budgetMs === 'number' && t.budgetMs > 0)) problems.push('tests.budgetMs must be a positive number of milliseconds')
+    if (t.select != null && typeof t.select !== 'boolean') problems.push('tests.select must be true or false')
     const lists: [string, { file: string }[], string[], [string, string][]][] = [
         ['slow', t.slow ?? [], SLOW_KEYS, [['catches', 'what it catches that no fast test can'], ['whyNotFast', 'why it cannot be made fast']]],
         ['serial', t.serial ?? [], SERIAL_KEYS, [['why', 'what breaks when it shares a run']]],

package/src/suite/reads-env.ts

@@ -0,0 +1,8 @@
+// the variables that hand the read recorder (suite-reads-preload.ts) its job:
+// where to write, the root it charges reads under, and, in a spawned bun
+// child, the module that spawned it
+export const SUITE_READS_ENV = 'SHIPKIT_SUITE_READS'
+export const SUITE_ROOT_ENV = 'SHIPKIT_SUITE_ROOT'
+export const SUITE_OWNER_ENV = 'SHIPKIT_SUITE_READS_OWNER'
+/** `0` runs every suite in the quick tier and records nothing, `1` selects even where tests.select is not true */
+export const SUITE_SELECT_ENV = 'SHIPKIT_TEST_SELECT'