rvlib-shipkit 0.33.0 → 0.34.0

package/src/ledger/git-key.ts

@@ -0,0 +1,330 @@
+// THE KEY. What question did this unit answer, and is that still the same
+// question? Nothing in this file caches anything: it produces the identity a
+// later ledger will store results under, and it refuses to produce one
+// whenever the answer would be a guess.
+//
+// THREE POPULATIONS, declared at registration, never inferred:
+//
+//   CONTENT  declares `inputs` — the paths it reads. Keyed on their git object
+//            ids, plus the unit's own code (module graph), plus the toolchain
+//            (which content-hashes the gate binary, so editing shipkit moves
+//            every key in every repo), plus argv and declared env. No TTL: a
+//            content key is either the same question or a different one.
+//   WORLD    declares `ttl` — it reads the network, the keychain, $PATH, a git
+//            remote or the clock. Nothing on disk describes its answer, so it
+//            keeps the existing clean-pass TTL and its `cacheKey`, unchanged.
+//   NEITHER  declares nothing. Never cached. This is the DEFAULT, and it is
+//            the whole safety story: forgetting to declare costs speed, never
+//            correctness, so the burn-down list below is a to-do rather than a
+//            hazard. Declaring BOTH is a registration failure, loud, because
+//            the two answers contradict each other.
+//
+// CROSS WORKTREE. Every component here is content addressed or a version
+// string; not one of them is an absolute path. Two worktrees of one repo
+// holding the same content and the same install therefore produce the same
+// key, and a worktree whose node_modules differs produces a different one.
+// Measured on this disk: main and two worktrees, main's install id
+// e55a7fd4ae252e5a against fd0e2908d222fe2e shared by both worktrees, which
+// differ from main by two top-level packages. That is what makes a shared
+// ledger safe rather than a stale green in a checkout nobody looked at.
+import { createHash } from 'crypto'
+import { loadConfig, type RvtConfig } from '../ts/config.ts'
+import { importClosure } from '../ts/source-graph.ts'
+import { GitScanError, hashFiles, normalizeInput, scanPaths, type ScanResult } from './git-scan.ts'
+import { toolchainIdentity } from './toolchain-identity.ts'
+
+/** the key format. Bumped when a component is added or its rendering changes,
+ *  so an old ledger can never answer a new question. */
+export const KEY_VERSION = 1
+
+export type Population = 'content' | 'world' | 'none'
+
+/** what one gate unit (a task, a rule, a discovered check) declares about its
+ *  own cacheability. `gate-units.ts` builds these from the live registries. */
+export type UnitDecl = {
+    /** the id an operator types: a task id, a rule id, a check name */
+    id: string
+    kind: 'task' | 'rule' | 'check'
+    /** CONTENT: repo-relative paths this unit reads. Presence declares the population. */
+    inputs?: string[]
+    /** WORLD: clean-pass duration, e.g. '24h'. Presence declares the population. */
+    ttl?: string
+    /** repo-relative entry files of the unit's OWN code — the module-graph
+     *  roots. A discovered check gets its own file for free. */
+    code?: string[]
+    /** the argv a subprocess unit runs */
+    argv?: string[]
+    /** env var NAMES whose values belong in the key */
+    env?: string[]
+    /** anything else that identifies the question (a rule's `cacheKey`) */
+    extra?: string
+    /** a DECIDED never-cacheable, with the reason. Not the same thing as an
+     *  undeclared unit: this one has been looked at and cannot be described
+     *  (it reads git history, a path outside the repo, the wall clock), so it
+     *  leaves the burn-down list. Declaring it beside inputs or ttl is a
+     *  registration failure, because those say the opposite. */
+    never?: string
+}
+
+export type UnitKey = {
+    id: string
+    kind: UnitDecl['kind']
+    population: Population
+    /** the content key, or null for world/none/refused */
+    key: string | null
+    /** why there is no content key. Always set when key is null. */
+    reason: string | null
+    /** every component that went into the key, in key order — this is what a
+     *  human reads when a key moved and nobody knows why */
+    components: string[]
+    /** the input paths this unit was keyed on, with their object ids */
+    inputOids: [string, string][]
+    /** the identity of the unit's OWN code, module graph included. Set for
+     *  every unit that has code, WORLD ones too: a TTL'd probe whose source
+     *  changed must not replay its previous answer, and this is the value that
+     *  busts it. null when the unit has no code entry or git could not hash it. */
+    codeKey: string | null
+}
+
+export type Registration = {
+    /** loud, blocking: a unit that declared two contradictory populations */
+    errors: string[]
+    /** the default-deny burn-down, one line per undeclared unit */
+    burnDown: string[]
+    /** every unit that was classified */
+    units: number
+    /** units in a cacheable population (content or world) */
+    declared: number
+}
+
+/** which population a declaration puts a unit in, before any git work.
+ *  `inputs` on a `ttl` unit is a registration FAIL: a unit cannot both be
+ *  described by files on disk and be un-describable by them. */
+export function classify(d: UnitDecl): { population: Population; error?: string; reason?: string; decided?: boolean } {
+    const hasInputs = (d.inputs ?? []).length > 0
+    const hasTtl = d.ttl != null && d.ttl !== ''
+    const hasNever = d.never != null && d.never !== ''
+    if (hasInputs && hasTtl)
+        return {
+            population: 'none',
+            error: `${d.kind} '${d.id}' declares BOTH inputs (CONTENT) and ttl (WORLD) — a unit reads files or it reads the world, never both`,
+        }
+    if (hasNever && (hasInputs || hasTtl))
+        return {
+            population: 'none',
+            error: `${d.kind} '${d.id}' declares never:'${d.never}' AND ${hasInputs ? 'inputs' : 'ttl'} — never means nothing can describe this unit, which the other declaration contradicts`,
+        }
+    if (hasInputs) return { population: 'content' }
+    if (hasTtl) return { population: 'world' }
+    if (hasNever) return { population: 'none', reason: d.never, decided: true }
+    return { population: 'none', reason: 'declares neither inputs nor ttl' }
+}
+
+export function validateRegistration(decls: UnitDecl[]): Registration {
+    const errors: string[] = []
+    const burnDown: string[] = []
+    let declared = 0
+    for (const d of decls) {
+        const c = classify(d)
+        if (c.error != null) errors.push(c.error)
+        // a DECIDED never leaves the list: the burn-down is what nobody has
+        // looked at yet, and mixing the two turns it into wallpaper
+        else if (c.population !== 'none') declared++
+        else if (c.decided !== true) burnDown.push(`${d.kind} ${d.id} — ${c.reason ?? 'undeclared'}`)
+        // a declaration that cannot even be turned into a repo-relative path
+        // is a registration bug, not a cache miss: it would key the unit on a
+        // path in another repo, or on the repo root
+        for (const raw of [...(d.inputs ?? []), ...(d.code ?? [])]) {
+            try {
+                normalizeInput(raw)
+            } catch (e) {
+                errors.push(`${d.kind} '${d.id}': ${e instanceof Error ? e.message : String(e)}`)
+            }
+        }
+    }
+    return { errors, burnDown, units: decls.length, declared }
+}
+
+export type KeyLayerOpts = {
+    /** the cli entry actually driving this run — toolchain identity needs to
+     *  know whether the gate binary lives inside the repo being gated */
+    cliEntry: string
+    /** module-graph config; loaded from the repo when omitted */
+    cfg?: RvtConfig
+}
+
+/**
+ * one scan, N keys. Build it once per run: the git work is shared across every
+ * unit, and building two of them in one run would pay for the scan twice and
+ * could straddle an edit.
+ */
+export class KeyLayer {
+    private constructor(
+        readonly root: string,
+        readonly decls: UnitDecl[],
+        readonly registration: Registration,
+        private readonly scan: ScanResult | null,
+        private readonly scanError: string | null,
+        private readonly closureByUnit: Map<string, string[]>,
+        private readonly codeOids: Map<string, string>,
+        private readonly toolchain: { id: string; uncacheable: string[] },
+        private readonly scanPathsUsed: string[],
+        /** wall time of the WHOLE layer: toolchain identity, the module-graph
+         *  closures, the git scan and the closure hashing. Reported rather
+         *  than the scan alone, because the scan alone is the flattering
+         *  number and the caller pays all of it. */
+        readonly openMs: number,
+    ) {}
+
+    /** wall time the GIT half alone cost, ms — `openMs` is what a run pays */
+    get scanMs(): number {
+        return this.scan?.ms ?? 0
+    }
+
+    get failure(): string | null {
+        return this.scanError
+    }
+
+    static open(root: string, decls: UnitDecl[], o: KeyLayerOpts): KeyLayer {
+        const started = performance.now()
+        const registration = validateRegistration(decls)
+        const cfg = o.cfg ?? loadConfig(root)
+        // a gate binary that cannot be resolved is not a crash, it is a run
+        // whose program nobody can name — and the only safe answer to that is
+        // no key at all
+        let toolchain: { id: string; uncacheable: string[] }
+        try {
+            toolchain = toolchainIdentity(root, o.cliEntry)
+        } catch (e) {
+            toolchain = { id: 'unresolved', uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
+        }
+
+        // the unit's OWN code, resolved through the same module graph the
+        // `cycles` task walks. A check that imports a shared helper carries
+        // that helper, so fixing the helper moves the check's key.
+        const closureByUnit = new Map<string, string[]>()
+        // computed for EVERY unit with code, not only the content ones: a
+        // world unit's TTL cache must break when the probe itself is rewritten
+        for (const d of decls) {
+            const entries = d.code ?? []
+            if (entries.length > 0) closureByUnit.set(d.id, importClosure(cfg, entries))
+        }
+
+        // ONE scan for the declared inputs plus shipkit's own source: those
+        // are mostly directories, mostly unchanged, and git's stat cache
+        // answers them without reading a byte.
+        const paths = new Set<string>()
+        for (const d of decls) {
+            if (classify(d).population !== 'content') continue
+            for (const p of d.inputs ?? []) paths.add(p)
+        }
+
+        // the closure is INDIVIDUAL FILES belonging to no declared path, so it
+        // goes through one `hash-object --stdin-paths` instead of one `git
+        // add` spawn per file (measured: 15ms for the whole set)
+        const closureFiles = new Set<string>()
+        for (const files of closureByUnit.values()) for (const f of files) closureFiles.add(f)
+
+        let scan: ScanResult | null = null
+        let codeOids = new Map<string, string>()
+        let scanError: string | null = null
+        try {
+            scan = scanPaths(root, [...paths])
+            codeOids = hashFiles(root, [...closureFiles]).oids
+        } catch (e) {
+            // a scan that failed may have left a STALE tree behind, so nothing
+            // downstream may key on it. Loud, and the whole run goes uncached.
+            scanError = e instanceof GitScanError ? e.message : String(e)
+        }
+        return new KeyLayer(root, decls, registration, scan, scanError, closureByUnit, codeOids, toolchain, [...paths].sort(), performance.now() - started)
+    }
+
+    /** the key for one unit, or the reason there is none. */
+    keyFor(id: string): UnitKey {
+        const d = this.decls.find((u) => u.id === id)
+        if (d == null) throw new Error(`keyFor: no unit '${id}' in this layer`)
+        const c = classify(d)
+        const own = (this.closureByUnit.get(d.id) ?? []).slice().sort()
+        const ownReadable = own.every((p) => this.codeOids.get(p) != null)
+        const codeLines = own.map((p) => `code:${p}=${this.codeOids.get(p)!}`)
+        const base: UnitKey = {
+            id: d.id, kind: d.kind, population: c.population, key: null, reason: null, components: [], inputOids: [],
+            codeKey: own.length === 0 || !ownReadable ? null : createHash('sha256').update(codeLines.join('\n')).digest('hex').slice(0, 16),
+        }
+        if (c.error != null) return { ...base, reason: c.error }
+        if (c.population === 'world') return { ...base, reason: `world unit — TTL ${d.ttl}, no content key` }
+        if (c.population === 'none') return { ...base, reason: c.reason ?? 'undeclared' }
+        if (this.scanError != null) return { ...base, reason: `git scan failed, nothing may be keyed: ${this.scanError}` }
+        const scan = this.scan!
+        if (this.toolchain.uncacheable.length > 0)
+            return { ...base, reason: `toolchain not describable: ${this.toolchain.uncacheable.join('; ')}` }
+
+        let inputs: string[]
+        try {
+            inputs = (d.inputs ?? []).map(normalizeInput).sort()
+        } catch (e) {
+            return { ...base, reason: e instanceof Error ? e.message : String(e) }
+        }
+        const stale = inputs.filter((p) => scan.missing.includes(p))
+        if (stale.length > 0) return { ...base, reason: `declared input(s) not on disk: ${stale.join(', ')}` }
+        const hidden = inputs.flatMap((p) => scan.excluded.get(p) ?? [])
+        const inputOids: [string, string][] = inputs.map((p) => [p, scan.oidByPath.get(p) ?? ''])
+        if (inputOids.some(([, oid]) => oid === '')) return { ...base, reason: 'git could not name one of the declared inputs' }
+
+        if (!ownReadable) return { ...base, reason: `git could not hash this unit's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
+        const components: string[] = [
+            `v=${KEY_VERSION}`,
+            `unit=${d.kind}:${d.id}`,
+            `bun=${Bun.version}`,
+            `toolchain=${this.toolchain.id}`,
+            `argv=${JSON.stringify(d.argv ?? [])}`,
+            ...(d.env ?? []).slice().sort().map((n) => `env:${n}=${process.env[n] ?? '<unset>'}`),
+            ...inputOids.map(([p, oid]) => `in:${p}=${oid}`),
+            ...codeLines,
+            ...(d.extra != null ? [`extra=${d.extra}`] : []),
+            // ignored content nobody keyed on is part of the key's MEANING:
+            // two runs that excluded different things answered different
+            // questions, and a silently narrowed key is the false green
+            ...hidden.sort().map((p) => `excluded=${p}`),
+        ]
+        const key = createHash('sha256').update(components.join('\n')).digest('hex').slice(0, 32)
+        return { ...base, key, components, inputOids }
+    }
+
+    keys(): UnitKey[] {
+        return this.decls.map((d) => this.keyFor(d.id))
+    }
+
+    /**
+     * POST RUN RE HASH. Re-scan exactly what this layer keyed on and report
+     * every path whose object id moved while the units were running.
+     *
+     * This is the answer to the poisoning class: a unit reads a file, an agent
+     * in the next window edits it, and the result gets stored under a key that
+     * describes the file as it was BEFORE the edit. Any drift here means the
+     * run's results describe a tree that no longer exists, and the caller
+     * discards them rather than storing a lie. Measured at ~140ms, which is
+     * the price of never having to explain a hit that was wrong.
+     */
+    rescan(): { drifted: string[]; error: string | null; ms: number } {
+        if (this.scanError != null) return { drifted: [], error: this.scanError, ms: 0 }
+        const started = performance.now()
+        try {
+            const after = scanPaths(this.root, this.scanPathsUsed)
+            const drifted: string[] = []
+            for (const [p, oid] of this.scan!.oidByPath) if (after.oidByPath.get(p) !== oid) drifted.push(p)
+            for (const [p] of after.oidByPath) if (!this.scan!.oidByPath.has(p)) drifted.push(p)
+            // the unit's own code drifts the same way a declared input does: an
+            // agent fixing a check WHILE that check runs would otherwise store
+            // the old verdict under the new code's key
+            const files = [...this.codeOids.keys()]
+            if (files.length > 0) {
+                const now = hashFiles(this.root, files)
+                for (const [p, oid] of this.codeOids) if (now.oids.get(p) !== oid) drifted.push(p)
+            }
+            return { drifted: [...new Set(drifted)].sort(), error: null, ms: performance.now() - started }
+        } catch (e) {
+            return { drifted: [], error: e instanceof GitScanError ? e.message : String(e), ms: performance.now() - started }
+        }
+    }
+}

package/src/ledger/git-scan.ts

@@ -0,0 +1,314 @@
+// WHAT IS ON DISK RIGHT NOW, as git object ids, using git's own index.
+//
+// A cache key needs the identity of the files a unit reads, INCLUDING files
+// that are untracked or modified. `git hash-object` per file is O(files) and
+// re-reads everything; git's index already carries a stat cache that answers
+// "unchanged" without reading the blob. So: a scratch index of our own, per
+// path adds, one `write-tree`, one `cat-file --batch-check`.
+//
+// FOUR THINGS THIS FILE EXISTS TO GET RIGHT, each reproduced before it was
+// written:
+//
+//  1. PER PATH ADDS, EXIT CODE CHECKED. `git add -A -- a.txt missing/` exits
+//     128, stages NOTHING, and `write-tree` then hands back the tree of the
+//     LAST successful write. Measured on a scratch repo: the batched form
+//     returned HEAD's tree while a.txt was modified on disk, so every key
+//     derived from it described a file nobody had. One add per path turns that
+//     into one loud failure on one path.
+//  2. A MISSING DECLARED PATH IS NOT A GIT ERROR. A repo without `tests/` is
+//     normal, and hard-failing the gate on it would be wrong, so a path that
+//     does not exist never reaches `git add`: it comes back in `missing` and
+//     the caller refuses to cache that unit. That keeps rule 1's hard failure
+//     for what it is actually about, git going wrong on a path that IS there.
+//  3. ONLY AN IGNORED PATH IS FORCE ADDED. `git add -Af` on a broad path force
+//     adds node_modules and costs 17.7s; the ignored paths that genuinely hold
+//     declared inputs here (.rv-social/patreon/, .rv-assets/out/) cost 76ms.
+//     One `ls-files --others -i --directory` names them, and node_modules is
+//     refused force-add at any depth.
+//  4. THE SCRATCH INDEX IS SHARED, SO IT IS CONTENDED. Measured: 10 parallel
+//     `git add` on one index file, 9 fail with `Unable to create …lock`. Three
+//     agents in one checkout is the normal case here, so lock contention is
+//     retried and only a NON lock failure is fatal.
+import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'fs'
+import { isAbsolute, join, normalize } from 'path'
+
+/** what a scan measured. Every OID is a git object id of the CURRENT disk
+ *  content, tracked or not. */
+export type ScanResult = {
+    /** root tree of the scratch index after the adds */
+    treeOid: string
+    /** repo-relative path → git object id (tree for a dir, blob for a file) */
+    oidByPath: Map<string, string>
+    /** declared paths that are not on disk — the caller must not cache them */
+    missing: string[]
+    /** repo-relative path → ignored content under it that is NOT in the key
+     *  (node_modules only, by policy: toolchain-identity.ts owns installs) */
+    excluded: Map<string, string[]>
+    /** wall time of the whole scan */
+    ms: number
+}
+
+export class GitScanError extends Error {}
+
+/** the scratch index FORMAT. Bump it whenever what the scan stages changes.
+ *  The index is persistent, and anything staged into it stays there until
+ *  something stages over it: a run that force-added a path a later version no
+ *  longer touches would keep serving that path's content forever, frozen at
+ *  the moment of the bad run. A version stamp beside the file turns that into
+ *  one discarded index instead of a permanently wrong key. */
+export const SCAN_INDEX_VERSION = 2
+
+const LOCK_RE = /Unable to create '.*\.lock'|index\.lock/i
+const RETRIES = 12
+
+const git = (root: string, args: string[], indexFile: string): { code: number; out: string } => {
+    const p = Bun.spawnSync(['git', ...args], {
+        cwd: root,
+        stdout: 'pipe',
+        stderr: 'pipe',
+        env: { ...process.env, GIT_INDEX_FILE: indexFile },
+    })
+    return { code: p.exitCode, out: `${p.stdout.toString()}${p.stderr.toString()}` }
+}
+
+/** one git call that MUST succeed. Lock contention is retried (three agents in
+ *  one checkout is normal); anything else throws with git's own words. */
+function gitOrThrow(root: string, args: string[], indexFile: string): string {
+    let last = { code: -1, out: '' }
+    for (let attempt = 0; attempt < RETRIES; attempt++) {
+        last = git(root, args, indexFile)
+        if (last.code === 0) return last.out
+        if (!LOCK_RE.test(last.out)) break
+        Bun.sleepSync(10 + attempt * 10)
+    }
+    throw new GitScanError(`git ${args.join(' ')} exited ${last.code}: ${last.out.trim().split('\n').slice(0, 3).join(' / ')}`)
+}
+
+/** the THREE index files a scan uses, and why it is three and not one.
+ *
+ *  SCRATCH is where the plain adds go, and it is COPIED FRESH from the repo's
+ *  real index on every scan. Keeping it across runs was the obvious design and
+ *  it was wrong twice, both measured. Anything staged into an index stays
+ *  staged, so a path force-added by one run rode in every later run's tree,
+ *  which made the whole-tree id a function of this machine's scan history and
+ *  reproducible on no other checkout. And a file that stopped being tracked
+ *  (`git rm --cached`) stayed in the scratch index, so the tree still carried
+ *  it and a rule keyed on tracked-ness saw nothing change. The copy costs
+ *  about 1ms and the real index is already warm, so the whole reset plus
+ *  `git add -- .` plus `write-tree` measures 30 to 40ms, FASTER than the
+ *  persistent index it replaces.
+ *
+ *  IGNORED is a separate index for the force-adds, so a path one unit named
+ *  can never leak into the whole-tree id another unit is keyed on. Without the
+ *  split, one unit's key changed depending on which OTHER units were in the
+ *  run, which is exactly the silently narrowed key the excluded list exists to
+ *  prevent. It stays warm across runs: an entry left in it can only affect the
+ *  path it belongs to, never the root tree.
+ *
+ *  REAL is read, never written, and only to answer "does git ignore this".
+ *  `ls-files --others` means UNTRACKED, so asking an index we force-add into
+ *  reports a force-added path as no longer ignored, and the next run then
+ *  plain-adds it, which exits 1 and takes the scan down. */
+export function indexPaths(root: string): { scratch: string; ignored: string; real: string } {
+    const p = Bun.spawnSync(['git', 'rev-parse', '--absolute-git-dir'], { cwd: root, stdout: 'pipe', stderr: 'pipe' })
+    if (p.exitCode !== 0) throw new GitScanError(`not a git repository: ${root} (${p.stderr.toString().trim()})`)
+    const gitDir = p.stdout.toString().trim()
+    const dir = join(gitDir, 'shipkit')
+    mkdirSync(dir, { recursive: true })
+    const real = join(gitDir, 'index')
+    const ignored = join(dir, 'scan-ignored.idx')
+    const stamp = join(dir, 'scan.idx.version')
+    const seen = ((): string => { try { return readFileSync(stamp, 'utf8').trim() } catch { return '' } })()
+    if (seen !== String(SCAN_INDEX_VERSION)) {
+        try { rmSync(ignored, { force: true }) } catch {}
+        try { writeFileSync(stamp, `${SCAN_INDEX_VERSION}\n`) } catch {}
+    }
+    // per-process scratch: it is rewritten from `real` every scan anyway, so
+    // sharing one file across concurrent runners buys no warmth and only adds
+    // lock contention
+    const scratch = join(dir, `scan-${process.pid}.idx`)
+    // a run that was killed leaves its file behind, and pids are reused, so
+    // sweep anything an hour old rather than trusting the pid to be free
+    try {
+        const cutoff = Date.now() - 3_600_000
+        for (const f of readdirSync(dir))
+            if (/^scan-\d+\.idx(\.tmp)?$/.test(f) && statSync(join(dir, f)).mtimeMs < cutoff) rmSync(join(dir, f), { force: true })
+    } catch {}
+    // tmp + rename: a half-copied index is a corrupt index, and git's error
+    // for that is not the one the retry loop is meant to absorb
+    try {
+        rmSync(scratch, { force: true })
+        if (existsSync(real)) {
+            const tmp = `${scratch}.tmp`
+            copyFileSync(real, tmp)
+            renameSync(tmp, scratch)
+        }
+    } catch {
+        // no seed is survivable: the adds are just colder
+    }
+    return { scratch, ignored, real }
+}
+
+/** the whole work tree as one input. A rule that walks the repo (SK001's test
+ *  file probe, SK010's `git ls-files`) genuinely reads all of it, and saying so
+ *  is more honest than naming three of the files it happens to open. Its object
+ *  id is the scan's root tree, so it moves on any edit anywhere. */
+export const WHOLE_TREE = '.'
+
+/** repo-relative posix, no './', no trailing slash. Throws on anything that
+ *  escapes the repo — a key component outside the tree is a key nobody else
+ *  can reproduce. */
+export function normalizeInput(p: string): string {
+    const n = normalize(p).replace(/\\/g, '/').replace(/\/+$/, '').replace(/^\.\//, '')
+    if (n === '.' || n === '') return WHOLE_TREE
+    if (isAbsolute(p) || n === '..' || n.startsWith('../'))
+        throw new GitScanError(`input path '${p}' is outside the repo — declare a path inside it, or '.' for the whole work tree`)
+    return n
+}
+
+/** drop paths already covered by an ancestor in the same set: adding `wiki`
+ *  stages `wiki/build/...` too, and a second add of the child is pure cost. */
+export function dropCovered(paths: string[]): string[] {
+    if (paths.includes(WHOLE_TREE)) return [WHOLE_TREE]
+    const sorted = [...new Set(paths)].sort()
+    const kept: string[] = []
+    for (const p of sorted) if (!kept.some((k) => p === k || p.startsWith(`${k}/`))) kept.push(p)
+    return kept
+}
+
+const underNodeModules = (p: string): boolean => p.split('/').includes('node_modules')
+
+/**
+ * stage `paths` into the scratch index and read back one git object id each.
+ *
+ * Throws `GitScanError` on any git failure that is not lock contention. The
+ * caller's contract on a throw is absolute: produce NO key. A tree that may be
+ * stale describes files nobody has, and a key built on it is a false green
+ * waiting for its first hit.
+ */
+export function scanPaths(root: string, rawPaths: string[]): ScanResult {
+    const started = performance.now()
+    const { scratch: indexFile, ignored: ignoredIndex, real: realIndex } = indexPaths(root)
+    const wanted = [...new Set(rawPaths.map(normalizeInput))].sort()
+    const missing = wanted.filter((p) => !existsSync(join(root, p)))
+    const present = wanted.filter((p) => existsSync(join(root, p)))
+    const toAdd = dropCovered(present)
+    const named = wanted.filter((p) => p !== WHOLE_TREE)
+
+    // ignored content under what we are about to stage. One call, and it also
+    // answers "is this declared path itself ignored": `--directory` collapses
+    // a dir whose ENTIRE content is ignored, so the answer comes back as the
+    // dir with a trailing slash.
+    const ignored = toAdd.length === 0
+        ? []
+        : gitOrThrow(root, ['ls-files', '--others', '-i', '--exclude-standard', '--directory', '--', ...toAdd], realIndex)
+            .split('\n')
+            .map((l) => l.trim().replace(/\/+$/, ''))
+            .filter((l) => l !== '')
+    // ONLY a path a declaration NAMES is force-added. Ignored content that
+    // merely sits under a declared path is reported and left out.
+    //
+    // This is the whole of defense 3, and the wide reading of it is a disaster
+    // measured twice: with `.` declared (which every repo-walking rule
+    // declares), "ignored content under a declared path" is every ignored path
+    // in the repo, so one run force-staged two entire worktrees, the log dir,
+    // the private model dumps and the prompt corpora, for 680ms. Naming the
+    // path is what makes it an input.
+    //
+    // Installs are refused on top of that, at any depth AND as an ancestor:
+    // `--directory` collapses a dir whose entire content is ignored, so
+    // force-adding that one entry stages the node_modules inside it.
+    const declared = new Set(wanted)
+    const denied = ignored.filter(underNodeModules)
+    const forceAdd = ignored.filter((p) => declared.has(p) && !underNodeModules(p) && !denied.some((d) => d.startsWith(`${p}/`)))
+    // a path git itself calls ignored must NOT get the plain add: `git add --
+    // <ignored>` exits 1 and takes the whole scan down with it
+    const ignoredSet = new Set(ignored)
+    for (const p of toAdd) if (!ignoredSet.has(p)) gitOrThrow(root, ['add', '--', p], indexFile)
+    // force-adds land in their OWN index, so a path one unit named can never
+    // ride inside the whole-tree id another unit is keyed on
+    for (const p of forceAdd) gitOrThrow(root, ['add', '-Af', '--', p], ignoredIndex)
+
+    const treeOid = gitOrThrow(root, ['write-tree'], indexFile).trim()
+    const oidByPath = new Map<string, string>()
+    // the work tree's own id IS the tree the adds just produced — there is no
+    // `<tree>:` path to ask git for
+    if (wanted.includes(WHOLE_TREE)) oidByPath.set(WHOLE_TREE, treeOid)
+    const forceSet = new Set(forceAdd)
+    const plain = named.filter((p) => !forceSet.has(p))
+    for (const [tree, paths, index] of [[treeOid, plain, indexFile], [null, forceAdd, ignoredIndex]] as const) {
+        if (paths.length === 0) continue
+        const root_ = tree ?? gitOrThrow(root, ['write-tree'], index).trim()
+        const query = paths.map((p) => `${root_}:${p}`).join('\n')
+        const p = Bun.spawnSync(['git', 'cat-file', '--batch-check'], {
+            cwd: root,
+            stdin: Buffer.from(`${query}\n`),
+            stdout: 'pipe',
+            stderr: 'pipe',
+            env: { ...process.env, GIT_INDEX_FILE: index },
+        })
+        if (p.exitCode !== 0) throw new GitScanError(`git cat-file --batch-check exited ${p.exitCode}: ${p.stderr.toString().trim()}`)
+        const lines = p.stdout.toString().split('\n').filter((l) => l !== '')
+        if (lines.length !== paths.length)
+            throw new GitScanError(`git cat-file --batch-check answered ${lines.length} of ${paths.length} paths`)
+        paths.forEach((path, i) => {
+            const oid = lines[i]!.split(' ')[0] ?? ''
+            // 'missing' / 'ambiguous' come back in place of an id — a path git
+            // cannot name is a path we cannot key on
+            if (/^[0-9a-f]{40}$/.test(oid)) oidByPath.set(path, oid)
+            else if (!missing.includes(path)) missing.push(path)
+        })
+    }
+
+    const excluded = new Map<string, string[]>()
+    for (const d of ignored) {
+        // EVERY owner that covers it, not the first one found, and membership
+        // must not depend on what OTHER units declared. `.` sorts first and
+        // covers everything, so a `find` here handed the whole exclusion list
+        // to the whole-tree unit and left every narrower unit claiming it was
+        // blind to nothing. Filtering force-added paths out globally had the
+        // same shape: one unit naming `ign` deleted `excluded=ign` from a
+        // whole-tree unit that is still blind to it. Measured both ways, one
+        // unit producing two keys for one unchanged tree, and the component
+        // that vanished is the one that exists to stop a silently narrowed key.
+        for (const w of wanted) {
+            if (!(w === WHOLE_TREE || d === w || d.startsWith(`${w}/`))) continue
+            // the one owner that is NOT blind: it named the path itself, so it
+            // got the content through the force-add
+            if (d === w && forceSet.has(d)) continue
+            excluded.set(w, [...(excluded.get(w) ?? []), d])
+        }
+    }
+    return { treeOid, oidByPath, missing: [...new Set(missing)].sort(), excluded, ms: performance.now() - started }
+}
+
+/**
+ * blob ids for a list of FILES, one process, no index, no lock.
+ *
+ * The scratch index is the right tool for a declared input, which is usually a
+ * directory and usually unchanged (git's stat cache answers without reading).
+ * It is the wrong tool for the module-graph closure: those are individual
+ * files, they belong to no declared path, and staging each one would cost a
+ * `git add` spawn per file. `hash-object --stdin-paths` reads them all in one
+ * call and returns exactly what a blob of that content would be.
+ *
+ * A file that is not on disk is skipped and reported, never guessed at.
+ */
+export function hashFiles(root: string, files: string[]): { oids: Map<string, string>; missing: string[] } {
+    const oids = new Map<string, string>()
+    const wanted = [...new Set(files.map(normalizeInput))].sort()
+    const missing = wanted.filter((f) => !existsSync(join(root, f)))
+    const present = wanted.filter((f) => existsSync(join(root, f)))
+    if (present.length === 0) return { oids, missing }
+    const p = Bun.spawnSync(['git', 'hash-object', '--stdin-paths'], {
+        cwd: root,
+        stdin: Buffer.from(`${present.join('\n')}\n`),
+        stdout: 'pipe',
+        stderr: 'pipe',
+    })
+    if (p.exitCode !== 0) throw new GitScanError(`git hash-object exited ${p.exitCode}: ${p.stderr.toString().trim().split('\n').slice(0, 3).join(' / ')}`)
+    const lines = p.stdout.toString().split('\n').filter((l) => l !== '')
+    if (lines.length !== present.length) throw new GitScanError(`git hash-object answered ${lines.length} of ${present.length} files`)
+    present.forEach((f, i) => oids.set(f, lines[i]!.trim()))
+    return { oids, missing }
+}