rvlib-shipkit 1.8.0 → 2.0.0

package/src/suite/suite-cli.ts

@@ -20,7 +20,7 @@ import { SUITE_READS_ENV, SUITE_ROOT_ENV, SUITE_SELECT_ENV } from './reads-env.t
 import { changedKeys, dropStaticEdges, extras, isBunSuite, envFilesOf, foldReads, loadErrorModules, loadState, preloadsOf, saveState, selectStatePath, selectSuites, snapshot, suiteFiles, taintedSuites, walkGraph, type GraphEntry, type Selection, type SelectState } from './select.ts'
 import { gitRoot } from '../core/git-root.ts'
 import { loadConfig } from '../ts/config.ts'
-import { aloneRunsNeeded, bunfigTest, type BunfigTest, readPhaseCpu, SUITE_CPU_ENV, type AloneRun, type Confirmed, confirmOver, crashedWorkers, overWords, DEFAULT_BUDGET_MS, FULL_CEILING_MS, failedSuites, failLines, judgeSuites, normSuitePath, parseSuiteFlags, planPhases, QUICK_CEILING_MS, poolWorkers } from './judge-suites.ts'
+import { aloneRunsNeeded, bunfigTest, type BunfigTest, readPhaseCpu, SUITE_CPU_ENV, type AloneRun, type Confirmed, confirmOver, crashedWorkers, interruptedSuites, overWords, ranAs, retryAlone, smallerRun, stallWords, DEFAULT_BUDGET_MS, FULL_CEILING_MS, failedSuites, failLines, judgeSuites, normSuitePath, parseSuiteFlags, planPhases, QUICK_CEILING_MS, poolWorkers } from './judge-suites.ts'
 
 type Phase = { code: number; signal: string | null; stalled: boolean; out: string; timings: Record<string, number>; cpuMs: number | null }
 
@@ -57,7 +57,10 @@ async function runPhase(args: string[], timingsFile: string | null, ceilingMs: n
             if (echo) sink.write(text)
         }
     }
-    await Promise.all([pump(proc.stdout, process.stdout), pump(proc.stderr, process.stderr)])
+    // bun reports on stderr, and it is echoed on stdout: two pipes reach the
+    // gate in either order, so bun's `Ran 0 tests` could land after the FAIL
+    // printed below and become the headline. One stream keeps the order
+    await Promise.all([pump(proc.stdout, process.stdout), pump(proc.stderr, process.stdout)])
     const code = await proc.exited
     clearTimeout(ceiling)
     clearTimeout(killer)
@@ -194,7 +197,9 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         const ran = phases.reduce((n, p) => n + lastCount(p.out, /Ran (\d+) tests?/g), 0)
         const failedTests = phases.reduce((n, p) => n + lastCount(p.out, /^\s*(\d+) fail$/gm), 0)
         const timings = Object.assign({}, ...phases.map((p) => p.timings)) as Record<string, number>
-        const judged = judgeSuites({ timings, tests, tier })
+        const crashed = crashedWorkers(p1.out)
+        const red = new Set([...phases.flatMap((p) => failedSuites(p.out)), ...crashed.map((c) => c.file)])
+        const judged = judgeSuites({ timings, tests, tier, red: [...red] })
         const slowest = judged.slowest.length > 0 ? ` (slowest: ${judged.slowest.map((t) => `${t.file} ${secs(t.ms)}`).join(', ')})` : ''
 
         // a parallel timing carries every other suite's load, so a suspect
@@ -205,25 +210,32 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         const parallelCpu = phaseCpu.cpu
         const { rerun, cleared: clearedInPhase } = aloneRunsNeeded(judged.over, parallelCpu, budget, load)
         const aloneAt = Date.now()
-        for (const [i, t] of rerun.entries()) {
-            const run = await runPhase([...plan.alone, t.file], join(dir, `alone-${i}.json`), ceilingMs, tmp, false, null, rec)
+        const runAlone = async (t: { file: string }, name: string): Promise<AloneRun | null> => {
+            const run = await runPhase([...plan.alone, t.file], join(dir, `${name}.json`), ceilingMs, tmp, false, null, rec)
             const wallMs = Object.entries(run.timings).find(([f]) => normSuitePath(f) === t.file)?.[1]
-            if (wallMs != null) aloneRuns[t.file] = { wallMs, cpuMs: run.cpuMs }
+            return wallMs == null ? null : { wallMs, cpuMs: run.cpuMs }
+        }
+        for (const [i, t] of rerun.entries()) {
+            const first = await runAlone(t, `alone-${i}`)
+            if (first == null) continue
+            aloneRuns[t.file] = first
+            if (!retryAlone(t, first, budget, load)) continue
+            const second = await runAlone(t, `alone-${i}-again`)
+            if (second == null) continue
+            aloneRuns[t.file] = smallerRun(first, second)
+            notes.push(`note: ${t.file} ran alone twice, ${secs(first.cpuMs ?? first.wallMs)} then ${secs(second.cpuMs ?? second.wallMs)} of CPU, and the smaller stands: a lone run the scheduler spills onto slower cores burns more CPU for the same work`)
         }
         const aloneMs = Date.now() - aloneAt
         const { over, cleared } = rerun.length > 0 ? confirmOver(rerun, aloneRuns, budget, load) : { over: [], cleared: [] }
-        for (const c of clearedInPhase) notes.push(`note: ${c.file} took ${secs(c.wallMs)} in the parallel run and burned ${secs(c.cpuMs ?? 0)} of CPU there, plus ${secs(parallelCpu[c.file]?.priorMs ?? 0)} a lone run loads first, inside the ${budget}ms budget, so it did not run again alone`)
+        for (const c of clearedInPhase) notes.push(`note: ${c.file} took ${secs(c.wallMs)} in the parallel run but burned ${secs(c.ms)} of its own CPU there, inside the ${budget}ms budget (load ${load.load1.toFixed(1)} on ${load.cores} cores), so it did not run again alone`)
         for (const c of cleared) notes.push(`note: ${c.file} took ${secs(c.wallMs)} alone but burned ${c.cpuMs != null ? secs(c.cpuMs) : '?'} of CPU, inside the ${budget}ms budget (load ${load.load1.toFixed(1)} on ${load.cores} cores)`)
 
         // a ceiling kill makes bun report every worker it held as crashed, and
         // those are the suites that never finished, not an outside kill
         const stalledIn = (name: string, p: Phase | null, files: string[]): string | null =>
-            p?.stalled === true
-                ? `${name} ran past its ${secs(ceilingMs)} ceiling and was killed${files.length > 0 ? ` with ${files.join(', ')} still running` : ''}: a test worker stopped answering, most likely stuck in a synchronous call, which no per test timeout can interrupt`
-                : null
-        const crashed = crashedWorkers(p1.out)
+            p?.stalled === true ? stallWords(name, ceilingMs, [...new Set([...files, ...interruptedSuites(p.out)])]) : null
         const killed = p1.stalled ? [] : crashed.map((c) => `killed by ${c.signal} while running ${c.file} (out of memory, or killed from outside)`)
-        const stalls = [stalledIn('the parallel phase', p1, crashed.map((c) => c.file)), stalledIn('the serial phase', p2, phaseFiles(serialArgs ?? [], rawArgs).filter((a) => p2?.timings[normSuitePath(a)] == null))]
+        const stalls = [stalledIn('the parallel phase', p1, crashed.map((c) => c.file)), stalledIn('the serial phase', p2, phaseFiles(serialArgs ?? [], rawArgs).filter((a) => p2?.timings[normSuitePath(a)] == null).map(normSuitePath))]
         for (const line of stalls) if (line != null) killed.unshift(line)
         if (p1.signal != null) killed.push(`killed by ${p1.signal} during the parallel run (out of memory, or killed from outside)`)
         if (p2?.signal != null) {
@@ -233,7 +245,6 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         // the record behind the hub's test files to fix: every suite this run
         // timed, with the alone timing where one was taken, and red when a
         // test in it failed or its worker died
-        const red = new Set([...phases.flatMap((p) => failedSuites(p.out)), ...crashedWorkers(p1.out).map((c) => c.file)])
         const overSet = new Set(over.map((o) => o.file))
         ctx.complete = killed.length === 0
         ctx.out = phases.map((p) => p.out).join('\n')
@@ -241,12 +252,12 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         const at = Date.now()
         const alone = new Map([...over, ...cleared, ...clearedInPhase].map((t) => [t.file, t.ms]))
         const why = failLines(phases.map((p) => p.out).join('\n'))
-        for (const c of crashedWorkers(p1.out)) if (!why.has(c.file)) why.set(c.file, `its worker crashed: ${c.signal}`)
+        for (const c of crashed) if (!why.has(c.file)) why.set(c.file, `its worker crashed: ${c.signal}`)
         // which phase timed each file, and the CPU spent on it: an alone
         // rerun, the parallel phase's own figure, or a phase that ran that one file
         const phaseOf = (f: string): SuitePhase => (p2 != null && Object.keys(p2.timings).some((k) => normSuitePath(k) === f) ? 'serial' : parallel ? 'parallel' : 'sequential')
         const soloCpu = (p: Phase | null): number | null => (p != null && Object.keys(p.timings).length === 1 ? p.cpuMs : null)
-        const cpuOf = (f: string): number | null => aloneRuns[f]?.cpuMs ?? (phaseOf(f) === 'serial' ? soloCpu(p2) : parallelCpu[f]?.cpuMs ?? soloCpu(p1))
+        const cpuOf = (f: string): number | null => aloneRuns[f]?.cpuMs ?? (phaseOf(f) === 'serial' ? soloCpu(p2) : parallelCpu[f] ?? soloCpu(p1))
         const check = process.env[CHECK_ID_ENV] ?? ''
         const serialFiles = p2 == null ? 0 : Object.keys(p2.timings).length
         const unrecorded = recordSuites(process.cwd(), Object.entries(timings).map(([file, ms]) => {
@@ -289,7 +300,7 @@ async function runSuites(argv: string[], ctx: SelectCtx): Promise<number> {
         if (fails.length > 0) { console.log(fails[0]); return 1 }
         const serialCount = serialArgs != null ? phaseFiles(serialArgs, rawArgs).length : 0
         const chosen = ctx.sel?.picked != null ? `, the ${ctx.sel.picked.length} of ${ctx.sel.total} a change reaches` : ''
-        console.log(`ok: Ran ${ran} tests across ${Object.keys(timings).length || '?'} suites${chosen}, ${parallel ? `parallel on ${workers} workers` : 'one at a time'}${serialCount > 0 ? ` then ${serialCount} serial` : ''}, ${tier} tier${slowest}`)
+        console.log(`ok: Ran ${ran} tests across ${Object.keys(timings).length || '?'} suites${chosen}, ${parallel ? `parallel on ${workers} workers` : 'one at a time'}${serialCount > 0 ? ` then ${serialCount} serial` : ''}, ${tier} tier, ${ranAs(plan.parallel, rawArgs)}${slowest}`)
         return 0
     } finally {
         rmSync(dir, { recursive: true, force: true })

package/src/suite/suite-cpu-preload.ts

@@ -2,17 +2,13 @@
 // worker by the suite runner. A suite over the budget in wall time is run again
 // alone to learn its CPU, one after the other, and on a loaded machine nearly
 // every gate heavy suite goes over in wall time: those re-runs were most of the
-// test check's wall time. A suite whose CPU in the parallel phase already
-// bounds its alone run under the budget needs no re-run.
+// test check's wall time. A suite whose own CPU in the parallel phase is inside
+// the budget needs no re-run.
 //
-// One line per suite: `<file>\t<cpu ms>\t<prior ms>`. `cpu` is the worker's own
-// CPU plus that of every child it reaped, from the end of the previous suite in
-// this worker to the end of this one, so the file's import and its hooks count.
-// `prior` is ALL the worker burned before this suite began: its start, and the
-// load, top level and tests of every earlier suite. A lone run can only need
-// what this worker computed by the end of the suite (a module an earlier suite
-// loaded, a memo an earlier test filled), so cpu plus prior bounds it from
-// above, and sharing the machine only raises CPU.
+// One line per suite: `<file>\t<cpu ms>`, the worker's own CPU plus that of
+// every child it reaped, from the end of the previous suite in this worker to
+// the end of this one, so the file's import, its hooks and its tests count. What
+// the worker burned before that (its start, earlier suites) is not this suite's.
 //
 // A child still alive, or dead and not yet reaped, when the suite ends has CPU
 // nobody counted yet, so that suite's line says `-` and it runs alone if it is
@@ -84,7 +80,7 @@ if (out != null && p != null) {
         // every suite gets a line, since the line is also how the runner knows
         // it ran; a suite whose figure cannot be trusted says `-`
         try {
-            const figure = p.liveChildren() === 0 ? `${Math.round(now - ended)}\t${Math.round(ended)}` : '-\t-'
+            const figure = p.liveChildren() === 0 ? `${Math.round(now - ended)}` : '-'
             appendFileSync(out, `${Bun.main}\t${figure}\n`)
         } catch { /* no line: the suite runs alone if it is a suspect */ }
         ended = now

package/src/suite/test-units.ts

@@ -11,7 +11,8 @@
 import { basename, posix } from 'path'
 import { rawName, routesToShipkit, slotBody } from '../core/canonical-scripts.ts'
 import { parseJsonc } from '../ts/jsonc.ts'
-import { bunfigTest, normSuitePath, parseBunTestScript, type BunfigTest, type TestsConfig } from './judge-suites.ts'
+import { repoOff, type OffKeys } from '../core/when.ts'
+import { bunfigTest, inDotDir, normSuitePath, parseBunTestScript, type BunfigTest, type TestsConfig } from './judge-suites.ts'
 
 export { bunfigTest, type BunfigTest }
 
@@ -41,6 +42,7 @@ export type TestRun = {
 export function runReaches(file: string, run: TestRun): { runs: true } | { runs: false; ignoredBy: string } | { runs: false; outside: string } | null {
     if (run.args == null) return null
     const f = normSuitePath(file)
+    if (inDotDir(f)) return { runs: false, ignoredBy: 'a dot dir, which bun never discovers' }
     const cli: string[] = []
     const filters: string[] = []
     for (let i = 0; i < run.args.length; i++) {
@@ -183,19 +185,20 @@ export function planTests(o: {
 /** the whole plan for a repo, from what its config and files say. `read`
  *  answers a repo-relative path, null when absent */
 export function testPlanOf(o: {
-    config: { defaults?: Record<string, unknown>; units?: { dir?: unknown; id?: string; tests?: TestsConfig; tsconfig?: string; release?: boolean; deps?: string[] }[] }
+    config: OffKeys & { units?: { dir?: unknown; id?: string; tests?: TestsConfig; tsconfig?: string; release?: boolean; deps?: string[] }[] }
     files: string[]
     read: (rel: string) => string | null
 }): TestPlan & { units: TestUnitDecl[] } {
     const units = declaredTestUnits(o.config.units)
     let scripts: Record<string, string> = {}
     try { scripts = (JSON.parse(o.read('package.json') ?? '{}') as { scripts?: Record<string, string> }).scripts ?? {} } catch { scripts = {} }
-    const slot = slotBody('test', o.config.defaults?.['test'] as { cmd?: unknown; logic?: unknown } | false | undefined, scripts)
+    const off = repoOff(o.config, 'test')
+    const slot = slotBody('test', off != null ? false : o.config.defaults?.['test'] as { cmd?: unknown; logic?: unknown } | undefined, scripts)
     const bunfig = (dir: string): BunfigTest => bunfigTest(o.read(dir === '' ? 'bunfig.toml' : posix.join(dir, 'bunfig.toml')))
     const rootRun: TestRun | null = slot.kind === 'override' ? { args: null, bunfig: bunfig(''), extraIgnores: [] }
         : slot.kind === 'script' ? { args: parseBunTestScript(slot.body), bunfig: bunfig(''), extraIgnores: unitIgnores(units) }
             : null
-    const rootWhy = slot.kind === 'off' ? '`defaults.test: false` turns the root test check off, so nothing runs it'
+    const rootWhy = slot.kind === 'off' ? `the repo turns the root test check off (${off?.reason ?? 'off'}), so nothing runs it`
         : slot.kind === 'routed-empty' ? "the `test` script routes to shipkit and there is no 'test:raw' body, so the root test check runs nothing"
             : 'no `test` or `test:raw` script, so the root has no test check'
     const plan = planTests({

package/src/ts/config.ts

@@ -10,9 +10,13 @@
 //   2. tsconfig "paths" keys ("src/*" → root 'src') when roots are not set
 //   3. fallback: ['src', 'scripts']
 // skip entries are repo-relative dir prefixes; the defaults below always apply.
+// The repo config's `vendored` dirs (core/vendored.ts) are third party code:
+// never crawled, never a package, never rewritten.
 import { existsSync, readdirSync, readFileSync, statSync } from 'fs'
 import { join } from 'path'
 import { gitRoot } from '../core/git-root.ts'
+import { isVendored } from '../core/vendored.ts'
+import { readVendored } from '../core/vendored-read.ts'
 
 export type RvtPackage = {
    name: string // package.json "name" — the import root for files inside
@@ -24,6 +28,7 @@ export type RvtConfig = {
    roots: string[] // repo-relative dirs to crawl / allowed specifier roots
    skip: string[] // repo-relative dir prefixes to never touch
    packages: RvtPackage[] // discovered sub-packages, longest dir first
+   vendored?: string[] // repo-relative third party dirs (repo.config.ts `vendored`), skipped as exact prefixes
 }
 
 /** never source, at any depth */
@@ -83,9 +88,9 @@ function expandRootGlobs(root: string, roots: string[]): string[] {
 
 // sub-package discovery: any dir under the roots with a named package.json is
 // its own import root (files inside address each other via the package name)
-function discoverPackages(root: string, roots: string[], skip: string[]): RvtPackage[] {
+function discoverPackages(root: string, roots: string[], skip: string[], vendored: string[]): RvtPackage[] {
    const found: RvtPackage[] = []
-   const cfg = { root, roots, skip, packages: [] }
+   const cfg = { root, roots, skip, packages: [], vendored }
    const walk = (rel: string): void => {
       const abs = join(root, rel)
       let entries
@@ -128,9 +133,10 @@ export function loadConfig(cwd: string = process.cwd()): RvtConfig {
    }
    if (roots.length === 0) roots = rootsFromTsconfig(root)
    if (roots.length === 0) roots = ['src', 'scripts'].filter((r) => existsSync(join(root, r)))
-   roots = expandRootGlobs(root, roots)
+   const vendored = readVendored(root)
+   roots = expandRootGlobs(root, roots).filter((r) => !isVendored(r, vendored))
    const mergedSkip = [...DEFAULT_SKIP, ...skip]
-   const packages = discoverPackages(root, roots, mergedSkip)
+   const packages = discoverPackages(root, roots, mergedSkip, vendored)
    // a package named like a root (or another package) hijacks that specifier
    // prefix — resolution would silently diverge from tsc/bun; be loud
    const seen = new Set<string>(roots)
@@ -138,7 +144,7 @@ export function loadConfig(cwd: string = process.cwd()): RvtConfig {
       if (seen.has(p.name)) console.error(`rvt WARN: package name '${p.name}' (${p.dir}) collides with a root or another package — its specifiers may resolve wrong`)
       seen.add(p.name)
    }
-   return { root, roots, skip: mergedSkip, packages }
+   return { root, roots, skip: mergedSkip, packages, vendored }
 }
 
 // innermost package containing a repo-relative path, or null
@@ -169,6 +175,7 @@ export const toPosix = (p: string): string => p.replace(/\\/g, '/')
 // package root is output, `wiki/build/<lib>/src` is source.
 export function isSkipped(relPath: string, cfg: RvtConfig, packageDirs: readonly string[] = []): boolean {
    const n = toPosix(relPath)
+   if (isVendored(n, cfg.vendored ?? [])) return true
    const segs = n.split('/')
    return cfg.skip.some((s) => {
       if (s.includes('/')) return n === s || n.startsWith(`${s}/`)

package/src/ts/imports.ts

@@ -14,6 +14,7 @@ import { readdirSync, readFileSync, writeFileSync, existsSync, type Dirent } fro
 import { dirname, extname, isAbsolute, join, relative, resolve } from 'path'
 import { canonicalSpecifier, isSkipped, loadConfig, parseJsonc, resolvePackageSpecifier, toPosix, type RvtConfig } from 'rvlib-shipkit/src/ts/config.ts'
 import { templateLineMask } from 'rvlib-shipkit/src/ts/source-mask.ts'
+import { isVendored } from 'rvlib-shipkit/src/core/vendored.ts'
 
 const ASSET_EXTENSIONS = new Set([
    '.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp', '.ico',
@@ -341,7 +342,8 @@ export function runImports(opts: { save?: boolean; files?: string[]; cfg?: RvtCo
       for (const e of result.errors) run.errors.push({ file: toPosix(relative(cfg.root, abs)), ...e })
    }
    if (opts.files != null && opts.files.length > 0) {
-      for (const fp of opts.files) process1(fp)
+      // a vendored file named by the on-save hook is still third party code
+      for (const fp of opts.files) if (!isVendored(toPosix(relative(cfg.root, resolve(cfg.root, fp))), cfg.vendored ?? [])) process1(fp)
    } else {
       for (const r of cfg.roots) {
          const abs = join(cfg.root, r)

package/src/ts/typecheck-cache.ts

@@ -0,0 +1,11 @@
+// the gate's typecheck keeps its own incremental cache, one file per project.
+// a tsconfig's tsBuildInfoFile is shared by every tsc that runs that config (a
+// watcher task, a raw script), and a verdict read off a cache another process
+// writes is not a verdict on this tree. no fs here: it only names paths.
+import { basename, dirname, join } from 'path'
+
+export const gateBuildInfoArgs = (tsconfig: string): string[] => [
+    '--incremental',
+    '--tsBuildInfoFile',
+    join(dirname(tsconfig), 'node_modules/.cache/shipkit', `${basename(tsconfig, '.json')}.tsbuildinfo`),
+]

package/src/ts/typecheck-cli.ts

@@ -10,6 +10,7 @@
 // who fixes the one error tsc showed and re-runs into a second one has been
 // told the truth twice and believed it once.
 import { resolveTypecheckProjects } from 'rvlib-shipkit/src/ts/tsconfig-projects.ts'
+import { gateBuildInfoArgs } from 'rvlib-shipkit/src/ts/typecheck-cache.ts'
 
 export function typecheckCli(argv: string[]): number {
     const [tsc, tsconfig] = argv
@@ -30,7 +31,7 @@ export function typecheckCli(argv: string[]): number {
 
     let worst = 0
     for (const p of resolved.projects) {
-        const r = Bun.spawnSync([tsc, '--noEmit', '-p', p], { stdout: 'inherit', stderr: 'inherit' })
+        const r = Bun.spawnSync([tsc, '--noEmit', '-p', p, ...gateBuildInfoArgs(p)], { stdout: 'inherit', stderr: 'inherit' })
         if (r.exitCode !== 0) {
             console.log(`FAIL: ${p}`)
             worst = r.exitCode ?? 1

package/src/tui/build-steps-panel.ts

@@ -36,7 +36,7 @@ export function buildStepsPanel(root: string, config: RepoConfig): TuiPanelDef {
             const { tasks, disabled, unfilled } = await allTasks(config, root)
             rows = [
                 ...tasks.map((t): TuiRow => ({ key: t.id, label: t.id, status: 'pending', detail: [] })),
-                ...disabled.map((id): TuiRow => ({ key: id, label: id, status: 'skip', detail: ['skip: disabled by repo.config defaults'] })),
+                ...disabled.map((id): TuiRow => ({ key: id, label: id, status: 'skip', detail: ['skip: turned off in repo.config.ts'] })),
                 ...unfilled.map((u): TuiRow => ({ key: u.id, label: u.id, status: 'skip', detail: [`skip: ${u.reason}`] })),
             ]
             const byKey = new Map(rows.map((r, i) => [r.key, i]))

package/src/wiki/scan-md.ts

@@ -11,9 +11,11 @@
 // Depth = 0-1 BFS from the index (eager edge keeps depth, lazy adds 1);
 // unreachable files stay in the graph with no depth (lint-md.ts turns those,
 // plus dangling refs, into failures — the graph itself is a viewer).
+// The enclosing repo's vendored dirs (core/vendored.ts) are never scanned.
 import { existsSync, readdirSync, readFileSync, statSync } from 'fs'
 import { basename, dirname, join, relative, resolve } from 'path'
 import type { MdEdge, MdGraph, MdNode } from 'rvlib-shipkit/src/wiki/md-graph.ts'
+import { vendoredAround } from 'rvlib-shipkit/src/core/vendored-read.ts'
 
 export type ScanMdConfig = {
     root: string // absolute dir to scan
@@ -22,7 +24,7 @@ export type ScanMdConfig = {
     boundary?: string // absolute dir enclosing root (a repo): refs leaving root but staying here resolve on disk
 }
 
-export type MdDanglingRef = { from: string; token: string } // an @/→/[](…) ref whose target is not in the scan
+export type MdDanglingRef = { from: string; token: string; line: number } // an @/→/[](…) ref whose target is not in the scan, and its 1-based line in `from`
 export type MdScan = {
     graph: MdGraph
     index: string | null // resolved entrypoint(s), ' + '-joined when both defaults exist; null = none found
@@ -35,12 +37,14 @@ export function scanMdGraph(cfg: ScanMdConfig): MdGraph {
 
 export function scanMd(cfg: ScanMdConfig): MdScan {
     const skip = new Set(['node_modules', '.git', 'dist', 'tmp', ...(cfg.skipDirs ?? [])])
+    const repo = vendoredAround(cfg.root)
+    const vendored = new Set((repo?.dirs ?? []).map((d) => join(repo!.root, d)))
     const files: string[] = []
     const walk = (rel: string) => {
         for (const e of readdirSync(join(cfg.root, rel))) {
             const r = rel === '' ? e : `${rel}/${e}`
             if (statSync(join(cfg.root, r)).isDirectory()) {
-                if (!skip.has(e) && !e.startsWith('.')) walk(r)
+                if (!skip.has(e) && !e.startsWith('.') && !vendored.has(join(cfg.root, r))) walk(r)
             } else if (e.endsWith('.md')) files.push(r)
         }
     }
@@ -84,19 +88,22 @@ export function scanMd(cfg: ScanMdConfig): MdScan {
         const raw = readFileSync(join(cfg.root, rel), 'utf8')
         const body = raw.startsWith('---\n') ? raw.slice(4).replace(/^[\s\S]*?^---\s*$/m, '') : raw
         const spans: string[] = []
-        const prose = body.replace(/^```[\s\S]*?^```\s*$/gm, '').replace(/`([^`\n]+)`/g, (_, s: string) => { spans.push(s); return ' ' })
+        // fences blanked to their newlines, so an offset in prose still names its line
+        const prose = body.replace(/^```[\s\S]*?^```\s*$/gm, (f) => f.replace(/[^\n]/g, '')).replace(/`([^`\n]+)`/g, (_, s: string) => { spans.push(s); return ' ' })
+        const above = raw.slice(0, raw.length - body.length).split('\n').length - 1
+        const lineOf = (m: RegExpMatchArray): number => above + prose.slice(0, m.index! + m[0].length - 1).split('\n').length
         const out: MdEdge[] = []
         // strict: @/→/[](…) are deliberate wiki refs — a miss is a dangling ref.
         // Backtick spans mention paths casually (other repos, deleted examples) — resolve or ignore, never fail.
-        const push = (token: string, eager: boolean, strict: boolean) => {
+        const push = (token: string, eager: boolean, strict: boolean, line = 0) => {
             const hit = resolveRef(rel, token)
-            if (hit == null) { if (strict) dangling.push({ from: rel, token }); return }
+            if (hit == null) { if (strict) dangling.push({ from: rel, token, line }); return }
             for (const to of hit.to) if (to !== rel) out.push({ from: rel, to, eager })
         }
-        for (const m of prose.matchAll(/(?:^|[\s(])@((?:\.{1,2}\/)*[\w][\w./-]*\.md)/gm)) push(m[1]!, true, true)
-        for (const m of prose.matchAll(/→\s*([\w][\w./-]*\.md)\b/g)) push(m[1]!, false, true)
-        for (const m of prose.matchAll(/\]\(([^)\s:]+\.md)\)/g)) push(m[1]!, false, true)
-        for (const m of prose.matchAll(/\]\(([^)\s:]*\/)\)/g)) push(m[1]!, false, true)
+        for (const m of prose.matchAll(/(?:^|[\s(])@((?:\.{1,2}\/)*[\w][\w./-]*\.md)/gm)) push(m[1]!, true, true, lineOf(m))
+        for (const m of prose.matchAll(/→\s*([\w][\w./-]*\.md)\b/g)) push(m[1]!, false, true, lineOf(m))
+        for (const m of prose.matchAll(/\]\(([^)\s:]+\.md)\)/g)) push(m[1]!, false, true, lineOf(m))
+        for (const m of prose.matchAll(/\]\(([^)\s:]*\/)\)/g)) push(m[1]!, false, true, lineOf(m))
         for (const s of spans) for (const m of s.matchAll(/[\w][\w./-]*\.md/g)) push(m[0], false, false)
         for (const s of spans) for (const m of s.matchAll(/(?:^|\s)([\w.][\w./-]*\/)(?=$|[\s<*])/g)) push(m[1]!, false, false)
         return out

package/src/checks/SK009-commit-ai-tells.check.ts

@@ -1,131 +0,0 @@
-// SK009 commit AI tells (identity/how-we-commit.md): the commit-msg hook is
-// what refuses AI attribution, in every repo, so this check verifies the hook
-// is ARMED: shipkit's own hooksPath, or a foreign one whose commit-msg chains
-// the shipkit gate. It never scans history: what is pushed is accepted as it
-// is, and only a new commit can still be stopped. A declared-public repo also
-// needs the tracked attribution silencer, and its ref NAMES are checked (a
-// FABLE-checkpoint tag outs the agent like a trailer); a branch nobody pushed
-// is named "local only" and warns.
-import { createHash } from 'crypto'
-import { readFileSync } from 'fs'
-import { isAbsolute, join } from 'path'
-import { nestedIn } from '../core/nesting.ts'
-import { gitBin } from '../core/git-bin.ts'
-import { fileStamp } from '../core/file-stamp.ts'
-import { CONFIG_FILE } from '../core/check-pass.ts'
-import { LAYOUT } from '../core/layout.ts'
-import { check, type CheckEnv } from '../check.ts'
-import { declaredVisibility } from '../core/repo-facts.ts'
-import { hookBinInvocation } from '../hooks/run-hooks.ts'
-
-// the documented silencer: .claude/settings.json attribution.commit/.pr both
-// empty — the harness then never writes the trailer or the PR footer
-const attributionSilenced = (root: string): boolean => {
-    try {
-        const s = JSON.parse(readFileSync(join(root, '.claude', 'settings.json'), 'utf8')) as { attribution?: { commit?: string; pr?: string } }
-        return s.attribution != null && s.attribution.commit === '' && s.attribution.pr === ''
-    } catch {
-        return false
-    }
-}
-
-const readOrNull = (abs: string): string | null => {
-    try {
-        return readFileSync(abs, 'utf8')
-    } catch {
-        return null
-    }
-}
-
-/** a commit-msg body that runs the shipkit gate on the message: a live line
- *  (never a comment) whose failure is not swallowed by `|| true` */
-const chainsShipkit = (body: string): boolean =>
-    body.split('\n').some((l) => !l.trim().startsWith('#') && /\bhooks\s+commit-msg\b/.test(l) && !/\|\|\s*(?:true|:)\b/.test(l))
-
-/** the chain line to paste into a foreign hook: a clone with no install skips
- *  the gate with a note, rather than failing every commit */
-export const guardedChain = (bin: string): string => {
-    const cli = bin.replace(/^bun\s+/, '')
-    return `if [ -f ${cli} ]; then ${bin} hooks commit-msg "$1" || exit 1; else echo "commit-msg: rvlib-shipkit not installed (run bun install), attribution gate skipped" >&2; fi`
-}
-
-// ref NAMES tell too. bare match only for names with no other meaning;
-// vendor and codename words (cursor, openai, gemini, opus, fable, …) name
-// legitimate branches and features, they only match in a checkpoint/session shape
-const REF_TELL = /\b(?:claude|chatgpt|copilot|aider|devin)\b|(?:^|[-_/])(?:fable|sonnet|haiku|opus|gemini|codex)[-_](?:checkpoint|session|ckpt)/i
-
-const gitOut = (root: string, args: string[]): { code: number; out: string } => {
-    const p = Bun.spawnSync([gitBin(), ...args], { cwd: root, stdout: 'pipe', stderr: 'pipe' })
-    return { code: p.exitCode, out: p.stdout.toString() }
-}
-
-/** what the answer depends on: the declared visibility, the hooksPath and the
- *  commit-msg file it points at, and for a public repo every ref NAME and the
- *  silencer. A commit moves none of them */
-export function historyKey(env: CheckEnv): string {
-    const nested = nestedIn(env.root)
-    if (nested != null) return `nested in ${nested}`
-    const declared = declaredVisibility(env.config)
-    const hooksPath = gitOut(env.root, ['config', 'core.hooksPath'])
-    const dir = hooksPath.code === 0 ? hooksPath.out.trim() : ''
-    const hook = `hooks=${dir}|msg=${dir === '' ? 'none' : fileStamp(isAbsolute(dir) ? join(dir, 'commit-msg') : join(env.root, dir, 'commit-msg'))}`
-    if (declared !== 'public') return `declared ${declared}|${hook}`
-    const refs = gitOut(env.root, ['for-each-ref', '--format=%(refname)', 'refs/tags', 'refs/heads', 'refs/remotes']).out
-    const digest = createHash('sha256').update(refs).digest('hex').slice(0, 16)
-    return `public|${hook}|refs=${digest}|settings=${fileStamp(join(env.root, '.claude', 'settings.json'))}`
-}
-
-export const commitAiTells = check({
-    id: 'SK009',
-    blocking: true,
-    title: 'commit ai tells',
-    description: 'the commit-msg hook that refuses AI attribution is armed, and no ref name tells',
-    owner: 'identity/how-we-commit.md',
-    runs: { onPreCommit: false, onCi: true, onFull: true, onRelease: true, onHealth: true, onSave: 'moved', onClock: false },
-    // git config and ref names, which no tree id describes, so they are the key
-    cache: { by: 'files', inputs: [CONFIG_FILE], key: historyKey },
-    fix: 'none',
-    applies: 'always',
-    run: async (c) => {
-        const nested = nestedIn(c.root)
-        if (nested != null) return void c.lines.push({ kind: 'skip', text: `commit gates not checked, nested in ${nested}: the parent repo owns them` })
-        const top = await c.exec(['git', 'rev-parse', '--show-toplevel'], { cwd: c.root })
-        if (top.code !== 0) return void c.lines.push({ kind: 'skip', text: 'not a git repo: no commit to guard yet' })
-
-        const hp = await c.exec(['git', 'config', 'core.hooksPath'], { cwd: c.root })
-        const dir = hp.code === 0 ? hp.out.trim() : ''
-        const chain = guardedChain(hookBinInvocation(c.root))
-        if (dir === '' && process.env['CI'] != null && process.env['CI'] !== '') {
-            c.lines.push({ kind: 'skip', text: 'commit-msg hook not armed in this CI checkout: a CI run makes no commit, and the fix pass arms hooks on a machine only' })
-        } else if (dir === '') {
-            c.fail('the commit-msg hook is not armed, so nothing refuses an AI attribution trailer: run `shipkit hooks install` (`shipkit ci` arms it too)')
-        } else {
-            const body = isAbsolute(dir) ? readOrNull(join(dir, 'commit-msg')) : c.read(`${dir}/commit-msg`)
-            const own = dir === LAYOUT.hooks
-            if (body == null)
-                c.fail(`core.hooksPath=${dir} has no ${dir}/commit-msg, so nothing refuses an AI attribution trailer: ${own ? 'run `shipkit hooks install`' : `add one that runs \`${chain}\``}`)
-            else if (!chainsShipkit(body))
-                c.fail(`core.hooksPath=${dir}: its commit-msg (${body.split('\n').find((l) => l.trim() !== '' && !l.startsWith('#'))?.trim().slice(0, 80) ?? 'empty'}) does not run the shipkit gate, so nothing refuses AI attribution: add \`${chain}\` above its last line`)
-            else c.ok(`the commit-msg hook refuses AI attribution (${own ? dir : `${dir}, chained to the shipkit gate`})`)
-        }
-
-        if (declaredVisibility(c.config) !== 'public') return
-        if (!attributionSilenced(c.root))
-            c.fail(`.claude/settings.json does not silence AI attribution: track it with {"attribution": {"commit": "", "pr": ""}} so no agent's harness ever writes the trailer or the PR footer`)
-        const refs = await c.exec(['git', 'for-each-ref', '--format=%(refname)', 'refs/tags', 'refs/heads', 'refs/remotes'], { cwd: c.root, timeoutMs: 30_000 })
-        if (refs.code !== 0) return void c.lines.push({ kind: 'skip', text: `ref names unverifiable: git for-each-ref exit ${refs.code}` })
-        const all = refs.out.split('\n').map((r) => r.trim()).filter((r) => r !== '')
-        const remote = new Set(all.filter((r) => r.startsWith('refs/remotes/')).map((r) => r.split('/').slice(3).join('/')))
-        const tells = all.filter((r) => REF_TELL.test(r.replace(/^refs\/(?:heads|tags|remotes)\//, '')))
-        const shown = (r: string): string => r.replace(/^refs\/(?:heads|tags|remotes)\//, '')
-        const local = tells.filter((r) => r.startsWith('refs/heads/') && !remote.has(shown(r)))
-        const out = tells.filter((r) => !local.includes(r) && !(r.startsWith('refs/heads/') && remote.has(shown(r))))
-        const list = (rs: string[], suffix = ''): string => rs.slice(0, 5).map((r) => `${shown(r)}${suffix}`).join(' ') + (rs.length > 5 ? ' …' : '')
-        if (out.length > 0)
-            c.fail(`${out.length} ref name(s) carry an AI tell in a declared-public repo (${list(out)}): rename or delete them, a pushed one on the remote too, then \`git fetch --prune\` so a remote ref deleted there leaves this clone`)
-        if (local.length > 0)
-            c.warn(`${local.length} branch name(s) carry an AI tell (${list(local, ' (local only)')}): never push them under this name, rename or delete them`)
-    },
-})
-
-export default commitAiTells

package/src/checks/SK014-prose-ste.check.ts

@@ -1,78 +0,0 @@
-// SK014 — the STE prose standard, in the gate (identity/writing-voice.md).
-//
-// Opt-in by declaration: a repo with no `prose.include` is SKIPPED, not
-// failed. STE is house doctrine for prose an agent writes, and a rule that
-// turned every existing repo red the day it shipped would be turned off
-// rather than obeyed.
-//
-// The tier split is the whole design. `fix` and `hard` hits BLOCK: they are
-// mechanical, and a machine reporting them is never guessing. `soft` hits
-// (sentence length, passive voice, modals, semicolons, …) need a human read,
-// so they are a note, never an attention item, and only a declared
-// `baseline` makes them blocking, as a ratchet that can only go down.
-//
-// It carries NO `fix()` on purpose. `shipkit ci` runs `--fix-rules` by
-// default, so a rule with a fixer rewrites tracked files unattended, and
-// prose is not repo shape: a wrong package.json line is a config, a wrong
-// sentence is content. The rewrite lives behind an explicit `shipkit prose
-// --fix` that a human ran and can read the diff of.
-import { classSummary, proseLines, runProseCheck, type ProseReport } from '../prose/run-prose.ts'
-import { tierOf, type ProseClass } from '../prose/ste.ts'
-import { check } from '../check.ts'
-
-const OWNER = 'identity/writing-voice.md'
-
-export const proseSte = check({
-    id: 'SK014',
-    blocking: true,
-    title: 'STE prose',
-    description: 'house prose passes the mechanical STE rules',
-    owner: OWNER,
-    // keyed on what it reads: each include glob's listing and every file in
-    // scope, so an edit outside the declared prose never reruns it
-    runs: { onPreCommit: false, onCi: false, onFull: true, onRelease: true, onHealth: true, onSave: 'moved', onClock: false },
-    cache: { by: 'reads' },
-    fix: 'none',
-    applies: 'always',
-    run: (c) => {
-        const cfg = c.config.prose
-        if (cfg == null || (cfg.include ?? []).length === 0) {
-            c.ok(`no prose scope declared — add \`prose: { include: [...] }\` to repo.config.ts to gate house prose on STE`)
-            return
-        }
-        const r = runProseCheck({ c, config: cfg })
-        // a scope that resolves to zero files reads exactly like a clean repo,
-        // and it is the opposite: the repo asked for the gate and the gate
-        // inspects nothing. A green here is coverage nobody has.
-        if (r.files === 0) {
-            c.warn(
-                `prose scope matches no file: ${(cfg.include ?? []).join(', ')} — declared and inspecting nothing`,
-                'fix the globs in repo.config.ts prose.include (or its exempt list), or drop the key to opt out',
-            )
-            return
-        }
-        const blocking = r.byTier.fix + r.byTier.hard
-        for (const l of proseLines(r, { cap: 12, tiers: ['fix', 'hard'] })) c.note(l)
-        if (blocking > 0)
-            c.fail(
-                `${blocking} STE violation(s) across ${r.byFile.length}/${r.files} file(s) — ${classSummary(r)}`,
-                r.fixable > 0 ? `shipkit prose --fix rewrites ${r.fixable} of them` : undefined,
-            )
-        const soft = softSummary(r)
-        if (cfg.baseline != null && r.byTier.soft > cfg.baseline)
-            c.fail(`${r.byTier.soft} soft hits (${soft}), above the declared baseline of ${cfg.baseline}: the ratchet only goes down`)
-        else if (r.byTier.soft > 0)
-            c.note(`${r.byTier.soft} soft hit(s) need a human read (${soft})${cfg.baseline != null ? `, within the baseline of ${cfg.baseline}` : ''}: \`shipkit prose\` lists them`)
-        if (!c.failed && blocking === 0) c.ok(`${r.files} file(s) STE-clean on the mechanical rules`)
-    }
-})
-
-/** every soft class with its count, most first */
-const softSummary = (r: ProseReport): string =>
-    Object.entries(r.byClass)
-        .filter(([cls]) => tierOf(cls as ProseClass) === 'soft')
-        .sort((a, b) => b[1] - a[1])
-        .map(([cls, n]) => `${cls} ${n}`)
-        .join(', ')
-
-export default proseSte