rvlib-shipkit 1.7.0 → 1.9.0

package/src/suite/suite-reads-preload.ts

@@ -0,0 +1,335 @@
+// WHAT EACH MODULE READ, preloaded into every test worker and every bun
+// process a suite spawns. The quick tier runs only the suites whose reads
+// changed since the last green run (select.ts), and a module's static imports
+// are only part of what it reads: a suite also reads files by path, lists
+// dirs, globs and spawns. This preload records each of those, attributed to
+// every module of the repo on the call stack. A memo filled by one suite and
+// used by the next is still charged to the module that holds it, and every
+// suite whose import closure holds that module depends on the read.
+//
+// Lines, appended to the file SHIPKIT_SUITE_READS names: `<kind>\t<module>\t<path>`.
+//   r  a file read, probed or stat'ed (its content, or its absence, matters)
+//   d  a dir listed (its names matter)
+//   g  a tree walked by a glob (every name under it matters)
+//   o  a process spawned that reads what nobody records (git or a shell run
+//      in the repo): the module is opaque, and every suite that reaches it
+//      runs every time
+//   l  a repo module loaded at runtime from a call this module was on the
+//      stack of (a require or an import() in a function body)
+//   L  a repo module loaded while this module itself was being loaded
+// A read with no repo frame on its stack is charged to `-`, which every suite
+// depends on. A bun child records under the module that spawned it.
+//
+// node:fs is replaced through bun:test's mock.module, the one hook that also
+// reaches a named ESM import, and it updates modules imported before it.
+import { mock, afterAll } from 'bun:test'
+import * as fsLive from 'node:fs'
+import * as fspLive from 'node:fs/promises'
+import * as cpLive from 'node:child_process'
+import { appendFileSync, realpathSync } from 'node:fs'
+import { isAbsolute, join, relative, resolve, sep } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { SUITE_OWNER_ENV, SUITE_READS_ENV, SUITE_ROOT_ENV } from './reads-env.ts'
+
+const out = process.env[SUITE_READS_ENV]
+const root = process.env[SUITE_ROOT_ENV]
+const owner = process.env[SUITE_OWNER_ENV]
+const SELF = import.meta.path
+
+if (out != null && out !== '' && root != null && root !== '') install(out, resolve(root))
+
+function install(outFile: string, rootDir: string): void {
+    const realFs = { ...fsLive } as Record<string, unknown>
+    const realFsp = { ...fspLive } as Record<string, unknown>
+    const realCp = { ...cpLive } as Record<string, unknown>
+    // the root as git prints it is a realpath, and a path a module names can
+    // go through a symlink to it (macOS: /var is /private/var), so each
+    // spelling of the root counts
+    const real = ((): string => { try { return realpathSync(rootDir) } catch { return rootDir } })()
+    const roots = [...new Set([rootDir, real, ...[rootDir, real].filter((r) => r.startsWith('/private/')).map((r) => r.slice('/private'.length))])]
+    const seen = new Set<string>()
+    let buf: string[] = []
+
+    /** repo-relative posix path, null outside the repo, in an install, or in git's own dir */
+    const rel = (p: unknown): string | null => {
+        let s: string
+        if (typeof p === 'string') s = p
+        else if (p instanceof URL) { if (p.protocol !== 'file:') return null; s = fileURLToPath(p) }
+        else if (p instanceof Uint8Array) s = Buffer.from(p).toString()
+        else return null
+        if (s.startsWith('file:')) { try { s = fileURLToPath(s) } catch { return null } }
+        const abs = isAbsolute(s) ? resolve(s) : resolve(process.cwd(), s)
+        const base = roots.find((r) => abs === r || abs.startsWith(r.endsWith(sep) ? r : `${r}${sep}`))
+        if (base == null) return null
+        const r = relative(base, abs).split(sep).join('/')
+        if (r === '' ) return '.'
+        if (r === 'node_modules' || r.startsWith('node_modules/') || r.includes('/node_modules/')) return null
+        return r
+    }
+    /** the innermost repo module on the stack, outside this file */
+    // a read is charged to EVERY repo module on the stack: the one that read
+    // it, the ones that hold a memo of what it read, and the one that asked. A
+    // cache a preload installs is asked through the module that holds its
+    // hook, and every suite reusing the memo calls that module again
+    const callers = (): string[] => {
+        if (owner != null && owner !== '') return [owner]
+        const limit = Error.stackTraceLimit
+        Error.stackTraceLimit = 80
+        const stack = new Error().stack ?? ''
+        Error.stackTraceLimit = limit
+        const out: string[] = []
+        for (const line of stack.split('\n')) {
+            const m = /\(?((?:\/|[A-Za-z]:\\)[^():]+):\d+:\d+\)?\s*$/.exec(line)
+            const file = m?.[1]
+            if (file == null || file === SELF) continue
+            const r = rel(file)
+            if (r != null && r !== '.' && !out.includes(r)) out.push(r)
+        }
+        return out.length > 0 ? out : ['-']
+    }
+    /** the module a child records under: the innermost one that spawned it */
+    const caller = (): string => callers()[0]!
+    /** a read through a symlink reads its target too */
+    const target = (path: string): string | null => {
+        try {
+            const real = rel((realFs['realpathSync'] as (p: string) => string)(join(rootDir, path)))
+            return real != null && real !== path ? real : null
+        } catch {
+            return null
+        }
+    }
+    const note = (kind: string, path: string | null, mods: string | string[] = callers()): void => {
+        if (path == null) return
+        if (kind !== 'o') {
+            const t = target(path)
+            if (t != null) note(kind, t, mods)
+        }
+        for (const mod of typeof mods === 'string' ? [mods] : mods) {
+            const line = `${kind}\t${mod}\t${path}`
+            if (seen.has(line)) continue
+            seen.add(line)
+            buf.push(line)
+        }
+    }
+    const flush = (): void => {
+        if (buf.length === 0) return
+        try { appendFileSync(outFile, `${buf.join('\n')}\n`) } catch { /* no record: the next full run fills it */ }
+        buf = []
+    }
+
+    const READS = ['readFileSync', 'readFile', 'existsSync', 'exists', 'statSync', 'stat', 'lstatSync', 'lstat', 'accessSync', 'access', 'openSync', 'open', 'createReadStream', 'readlinkSync', 'readlink', 'realpathSync', 'realpath', 'opendirSync', 'opendir', 'copyFileSync', 'copyFile', 'watchFile', 'statfsSync', 'statfs']
+    // a copy or a watch of a dir takes the whole tree under it
+    const TREES = ['cpSync', 'cp', 'watch']
+    const LISTS = ['readdirSync', 'readdir']
+    const GLOBS = ['globSync', 'glob']
+    const wrap = (real: Record<string, unknown>, kind: string, names: string[]): Record<string, unknown> => {
+        const shim: Record<string, unknown> = {}
+        for (const n of names) {
+            const f = real[n]
+            if (typeof f !== 'function') continue
+            const g = function (this: unknown, ...a: unknown[]): unknown {
+                if (kind === 'g') note('g', globRoot(a[0], a[1]))
+                else if (kind === 't') {
+                    const r = rel(a[0])
+                    note(r != null && exists(join(rootDir, r), false) ? 'r' : 'g', r)
+                }
+                else {
+                    const r = rel(a[0])
+                    // a recursive listing is a walk of the whole tree
+                    const recursive = kind === 'd' && typeof a[1] === 'object' && a[1] != null && (a[1] as { recursive?: unknown }).recursive === true
+                    note(recursive ? 'g' : n.startsWith('opendir') ? 'd' : kind, r)
+                }
+                return (f as (...x: unknown[]) => unknown).apply(this, a)
+            }
+            Object.assign(g, f)
+            shim[n] = g
+        }
+        return shim
+    }
+    const globRoot = (pattern: unknown, opts: unknown): string | null => {
+        const pats = Array.isArray(pattern) ? pattern : [pattern]
+        const cwd = typeof opts === 'object' && opts != null && typeof (opts as { cwd?: unknown }).cwd === 'string' ? (opts as { cwd: string }).cwd : process.cwd()
+        const roots = pats.filter((p): p is string => typeof p === 'string').map((p) => {
+            const lit = p.split('/').filter((_, i, a) => !/[*?[{]/.test(a.slice(0, i + 1).join('/'))).join('/')
+            return rel(join(cwd, lit))
+        })
+        return roots.length === 1 ? roots[0]! : rel(cwd)
+    }
+    const fsShim: Record<string, unknown> = { ...realFs, ...wrap(realFs, 'r', READS), ...wrap(realFs, 'd', LISTS), ...wrap(realFs, 'g', GLOBS), ...wrap(realFs, 't', TREES) }
+    const fspShim: Record<string, unknown> = { ...realFsp, ...wrap(realFsp, 'r', READS), ...wrap(realFsp, 'd', LISTS), ...wrap(realFsp, 'g', GLOBS), ...wrap(realFsp, 't', TREES) }
+    fsShim['promises'] = fspShim
+    mock.module('node:fs', () => ({ ...fsShim, default: fsShim }))
+    mock.module('fs', () => ({ ...fsShim, default: fsShim }))
+    mock.module('node:fs/promises', () => ({ ...fspShim, default: fspShim }))
+    mock.module('fs/promises', () => ({ ...fspShim, default: fspShim }))
+
+    // a bun child runs this same preload and records under the spawning module.
+    // Any other child reads what nobody records
+    /** a path that stands there (a file, or a dir when `dir`) */
+    const exists = (abs: string, dir: boolean): boolean => {
+        try {
+            const st = (realFs['statSync'] as (p: string) => { isFile(): boolean; isDirectory(): boolean })(abs)
+            return st.isFile() || (dir && st.isDirectory())
+        } catch { return false }
+    }
+    const SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'fish', 'env', 'xargs', 'nice', 'time', 'timeout'])
+    // tools that read only the files their arguments name, nothing they find through them
+    const FILE_TOOLS = new Set(['true', 'false', 'echo', 'sleep', 'kill', 'rm', 'mkdir', 'touch', 'cwebp', 'dwebp'])
+    const isBun = (bin: unknown): boolean => typeof bin === 'string' && (bin === 'bun' || bin === process.execPath || /(^|\/)bun$/.test(bin))
+    const childEnv = (env: unknown): Record<string, string | undefined> => {
+        const base = (typeof env === 'object' && env != null ? env : process.env) as Record<string, string | undefined>
+        const opts = base['BUN_OPTIONS'] ?? ''
+        return {
+            ...base,
+            [SUITE_READS_ENV]: outFile,
+            [SUITE_ROOT_ENV]: rootDir,
+            [SUITE_OWNER_ENV]: caller(),
+            BUN_OPTIONS: opts.includes(SELF) ? opts : `${opts} --preload=${SELF}`.trim(),
+        }
+    }
+    // a child that is not bun reads what nobody records. It is trusted only
+    // when it runs outside the repo and names nothing in it (git in a temp
+    // repo); a tool known to read only the files it names (cwebp) records
+    // them; any other child in or about the repo makes its module opaque
+    const spawned = (cmd: unknown, cwdOpt: unknown): void => {
+        const argv = (Array.isArray(cmd) ? cmd : typeof cmd === 'string' ? [cmd] : []).map(String)
+        if (isBun(argv[0])) return
+        const tool = (argv[0] ?? '?').split('/').pop() ?? '?'
+        const cwd = typeof cwdOpt === 'string' ? resolve(cwdOpt) : cwdOpt instanceof URL ? fileURLToPath(cwdOpt) : process.cwd()
+        const shell = SHELLS.has(tool)
+        const words = argv.slice(1).flatMap((a) => (shell ? a.split(/[\s;&|<>()'"`=]+/) : [a.replace(/^--?[\w-]+=/, '')])).filter((w) => w !== '' && !w.startsWith('-'))
+        const inRepo: string[] = []
+        for (const w of words) {
+            const abs = isAbsolute(w) ? w : join(cwd, w)
+            if (!isAbsolute(w) && !w.includes('/') && !exists(abs, true)) continue
+            const r = rel(abs)
+            if (r != null) inRepo.push(r)
+        }
+        // a tool known to read only the files it names records them; any other
+        // tool is trusted only far from the repo: run outside it, naming nothing in it
+        if (!shell && FILE_TOOLS.has(tool)) {
+            for (const r of inRepo) note(exists(join(rootDir, r), false) ? 'r' : 'g', r)
+            return
+        }
+        if (rel(cwd) != null || inRepo.length > 0) note('o', tool)
+    }
+    const realSpawn = Bun.spawn
+    const realSpawnSync = Bun.spawnSync
+    const viaBun = (real: typeof Bun.spawn | typeof Bun.spawnSync) => function (this: unknown, ...a: unknown[]): unknown {
+        const first = a[0]
+        const cmd = Array.isArray(first) ? first : (first as { cmd?: unknown } | undefined)?.cmd
+        spawned(cmd, ((Array.isArray(first) ? a[1] : first) as { cwd?: unknown } | undefined)?.cwd)
+        if (Array.isArray(first)) {
+            const opts = (a[1] ?? {}) as { env?: unknown }
+            return (real as (...x: unknown[]) => unknown).call(this, first, { ...opts, env: childEnv(opts.env) })
+        }
+        const opts = (first ?? {}) as { env?: unknown }
+        return (real as (...x: unknown[]) => unknown).call(this, { ...opts, env: childEnv(opts.env) })
+    }
+    Bun.spawn = viaBun(realSpawn) as typeof Bun.spawn
+    Bun.spawnSync = viaBun(realSpawnSync) as typeof Bun.spawnSync
+    const cpWrap = (n: string, cmdAt: (a: unknown[]) => unknown, optsAt: number | ((a: unknown[]) => number)): unknown => {
+        const f = realCp[n]
+        if (typeof f !== 'function') return f
+        return function (this: unknown, ...a: unknown[]): unknown {
+            const cmd = cmdAt(a)
+            const i = typeof optsAt === 'number' ? optsAt : optsAt(a)
+            const opts = (typeof a[i] === 'object' && a[i] != null && !Array.isArray(a[i]) ? a[i] : {}) as { env?: unknown; cwd?: unknown }
+            if (n === 'exec' || n === 'execSync') spawned(['sh', '-c', cmd], opts.cwd)
+            else spawned([cmd, ...(Array.isArray(a[1]) ? a[1] : [])], opts.cwd)
+            const args = [...a]
+            while (args.length < i) args.push(undefined)
+            args[i] = { ...opts, env: childEnv(opts.env) }
+            return (f as (...x: unknown[]) => unknown).apply(this, args)
+        }
+    }
+    const withArgs = (a: unknown[]): number => (Array.isArray(a[1]) ? 2 : 1)
+    const cpShim: Record<string, unknown> = {
+        ...realCp,
+        spawn: cpWrap('spawn', (a) => a[0], withArgs),
+        spawnSync: cpWrap('spawnSync', (a) => a[0], withArgs),
+        execFile: cpWrap('execFile', (a) => a[0], withArgs),
+        execFileSync: cpWrap('execFileSync', (a) => a[0], withArgs),
+        exec: cpWrap('exec', (a) => a[0], 1),
+        execSync: cpWrap('execSync', (a) => a[0], 1),
+        fork: cpWrap('fork', () => 'bun', withArgs),
+    }
+    mock.module('node:child_process', () => ({ ...cpShim, default: cpShim }))
+    mock.module('child_process', () => ({ ...cpShim, default: cpShim }))
+    const realShell = Bun.$
+    Bun.$ = new Proxy(realShell, { apply: (t, self, a) => { note('o', 'sh'); return Reflect.apply(t, self, a) } })
+
+    // a file:// fetch is a read; a Worker runs code no preload of ours sees
+    const realFetch = globalThis.fetch
+    globalThis.fetch = Object.assign(function (this: unknown, input: unknown, init?: unknown): unknown {
+        const url = typeof input === 'string' ? input : input instanceof URL ? input.href : (input as { url?: unknown } | null)?.url
+        if (typeof url === 'string' && url.startsWith('file:')) note('r', rel(url))
+        return (realFetch as (...x: unknown[]) => unknown).call(this, input, init)
+    }, realFetch) as typeof fetch
+    for (const name of ['Worker', 'SharedWorker']) {
+        const Real = (globalThis as Record<string, unknown>)[name] as (new (...a: unknown[]) => object) | undefined
+        if (typeof Real !== 'function') continue
+        const Wrapped = function (this: unknown, ...a: unknown[]): object {
+            note('o', name.toLowerCase())
+            return new Real(...a)
+        } as unknown as typeof Real
+        Wrapped.prototype = Real.prototype
+        ;(globalThis as Record<string, unknown>)[name] = Wrapped
+    }
+
+    const realFile = Bun.file
+    Bun.file = function (this: unknown, p: unknown, o?: unknown): unknown {
+        note('r', rel(p))
+        return (realFile as (...x: unknown[]) => unknown).call(this, p, o)
+    } as typeof Bun.file
+    // a Glob keeps no pattern a caller can read, so each one remembers its own:
+    // the walked tree is the cwd joined with the pattern's literal head
+    const RealGlob = Bun.Glob
+    const patterns = new WeakMap<object, string>()
+    const Glob = function (this: unknown, pattern: string): unknown {
+        const g = new RealGlob(pattern)
+        patterns.set(g, pattern)
+        return g
+    } as unknown as typeof Bun.Glob
+    Glob.prototype = RealGlob.prototype
+    Bun.Glob = Glob
+    const G = RealGlob.prototype as unknown as Record<string, (...a: unknown[]) => unknown>
+    for (const n of ['scan', 'scanSync']) {
+        const f = G[n]
+        if (typeof f !== 'function') continue
+        G[n] = function (this: object, opts?: unknown): unknown {
+            const cwd = typeof opts === 'string' ? opts : typeof opts === 'object' && opts != null && typeof (opts as { cwd?: unknown }).cwd === 'string' ? (opts as { cwd: string }).cwd : process.cwd()
+            note('g', globRoot(patterns.get(this) ?? '**', { cwd }))
+            return f.call(this, opts)
+        }
+    }
+
+    // every import resolved at runtime, as an edge from its importer and from
+    // every module on the stack: a module a computed import loads is in no
+    // static closure, and the edge charges it wherever the load was asked
+    // from, whichever suite ran first. A key already seen asks nothing again
+    const edges = new Set<string>()
+    const imported = (spec: string, importer: string | undefined): void => {
+        if (importer == null || importer === '' || spec.startsWith('node:') || spec.startsWith('bun:')) return
+        const from = rel(importer)
+        if (from == null || from === '.') return
+        const key = `${from}\0${spec}\0${new Error().stack ?? ''}`
+        if (edges.has(key)) return
+        edges.add(key)
+        const bare = spec.replace(/\?.*$/, '')
+        const base = isAbsolute(bare) ? bare : bare.startsWith('.') ? join(importer, '..', bare) : join(rootDir, bare)
+        const abs = [base, `${base}.ts`, `${base}.tsx`, join(base, 'index.ts')].find((x) => exists(x, false))
+        if (abs == null) return
+        const to = rel(abs)
+        // charged like a read: a lazy load inside a hook a preload installed
+        // is reached through the modules that called the hook
+        if (to == null || to === from) return
+        const asked = callers().filter((c) => c !== from && c !== '-')
+        if (asked.length === 0) note('L', to, from)
+        else note('l', to, [from, ...asked])
+    }
+    Bun.plugin({ name: 'shipkit-suite-reads', setup(b) { b.onResolve({ filter: /.*/ }, (a) => { imported(a.path, a.importer); return undefined }) } })
+
+    try { afterAll(flush) } catch { /* not under bun test: the exit hook below records it */ }
+    process.on('exit', flush)
+}

package/src/ts/typecheck-cache.ts

@@ -0,0 +1,11 @@
+// the gate's typecheck keeps its own incremental cache, one file per project.
+// a tsconfig's tsBuildInfoFile is shared by every tsc that runs that config (a
+// watcher task, a raw script), and a verdict read off a cache another process
+// writes is not a verdict on this tree. no fs here: it only names paths.
+import { basename, dirname, join } from 'path'
+
+export const gateBuildInfoArgs = (tsconfig: string): string[] => [
+    '--incremental',
+    '--tsBuildInfoFile',
+    join(dirname(tsconfig), 'node_modules/.cache/shipkit', `${basename(tsconfig, '.json')}.tsbuildinfo`),
+]

package/src/ts/typecheck-cli.ts

@@ -10,6 +10,7 @@
 // who fixes the one error tsc showed and re-runs into a second one has been
 // told the truth twice and believed it once.
 import { resolveTypecheckProjects } from 'rvlib-shipkit/src/ts/tsconfig-projects.ts'
+import { gateBuildInfoArgs } from 'rvlib-shipkit/src/ts/typecheck-cache.ts'
 
 export function typecheckCli(argv: string[]): number {
     const [tsc, tsconfig] = argv
@@ -30,7 +31,7 @@ export function typecheckCli(argv: string[]): number {
 
     let worst = 0
     for (const p of resolved.projects) {
-        const r = Bun.spawnSync([tsc, '--noEmit', '-p', p], { stdout: 'inherit', stderr: 'inherit' })
+        const r = Bun.spawnSync([tsc, '--noEmit', '-p', p, ...gateBuildInfoArgs(p)], { stdout: 'inherit', stderr: 'inherit' })
         if (r.exitCode !== 0) {
             console.log(`FAIL: ${p}`)
             worst = r.exitCode ?? 1