rvlib-shipkit 0.33.0 → 0.35.0

package/src/ledger/git-key.ts

@@ -0,0 +1,381 @@
+// THE KEY. What question did this unit answer, and is that still the same
+// question? Nothing in this file caches anything: it produces the identity a
+// later ledger will store results under, and it refuses to produce one
+// whenever the answer would be a guess.
+//
+// THREE POPULATIONS, declared at registration, never inferred:
+//
+//   CONTENT  declares `inputs` — the paths it reads. Keyed on their git object
+//            ids, plus the unit's own code (module graph), plus the toolchain
+//            (which content-hashes the gate binary, so editing shipkit moves
+//            every key in every repo), plus argv and declared env. No TTL: a
+//            content key is either the same question or a different one.
+//   WORLD    declares `ttl` — it reads the network, the keychain, $PATH, a git
+//            remote or the clock. Nothing on disk describes its answer, so it
+//            keeps the existing clean-pass TTL and its `cacheKey`, unchanged.
+//   NEITHER  declares nothing. Never cached. This is the DEFAULT, and it is
+//            the whole safety story: forgetting to declare costs speed, never
+//            correctness, so the burn-down list below is a to-do rather than a
+//            hazard. Declaring BOTH is a registration failure, loud, because
+//            the two answers contradict each other.
+//
+// CROSS WORKTREE. Every component here is content addressed or a version
+// string; not one of them is an absolute path. Two worktrees of one repo
+// holding the same content and the same install therefore produce the same
+// key, and a worktree whose node_modules differs produces a different one.
+// Measured on this disk: main and two worktrees, main's install id
+// e55a7fd4ae252e5a against fd0e2908d222fe2e shared by both worktrees, which
+// differ from main by two top-level packages. That is what makes a shared
+// ledger safe rather than a stale green in a checkout nobody looked at.
+import { createHash } from 'crypto'
+import { join } from 'path'
+import { loadConfig, type RvtConfig } from '../ts/config.ts'
+import { importClosure } from '../ts/source-graph.ts'
+import { GitScanError, hashFiles, normalizeInput, scanPaths, WHOLE_TREE, type ScanResult } from './git-scan.ts'
+import { installIdentity, toolchainIdentity } from './toolchain-identity.ts'
+
+/** the key format. Bumped when a component is added or its rendering changes,
+ *  so an old ledger can never answer a new question. */
+export const KEY_VERSION = 2
+
+export type Population = 'content' | 'world' | 'none'
+
+/** what one gate unit (a task, a rule, a discovered check) declares about its
+ *  own cacheability. `gate-units.ts` builds these from the live registries. */
+export type UnitDecl = {
+    /** the id an operator types: a task id, a rule id, a check name */
+    id: string
+    kind: 'task' | 'rule' | 'check'
+    /** CONTENT: repo-relative paths this unit reads. Presence declares the population. */
+    inputs?: string[]
+    /** WORLD: clean-pass duration, e.g. '24h'. Presence declares the population. */
+    ttl?: string
+    /** repo-relative entry files of the unit's OWN code — the module-graph
+     *  roots. A discovered check gets its own file for free. */
+    code?: string[]
+    /** the argv a subprocess unit runs */
+    argv?: string[]
+    /** env var NAMES whose values belong in the key */
+    env?: string[]
+    /** anything else that identifies the question (a rule's `cacheKey`) */
+    extra?: string
+    /** a DECIDED never-cacheable, with the reason. Not the same thing as an
+     *  undeclared unit: this one has been looked at and cannot be described
+     *  (it reads git history, a path outside the repo, the wall clock), so it
+     *  leaves the burn-down list. Declaring it beside inputs or ttl is a
+     *  registration failure, because those say the opposite. */
+    never?: string
+}
+
+export type UnitKey = {
+    id: string
+    kind: UnitDecl['kind']
+    population: Population
+    /** the content key, or null for world/none/refused */
+    key: string | null
+    /** why there is no content key. Always set when key is null. */
+    reason: string | null
+    /** every component that went into the key, in key order — this is what a
+     *  human reads when a key moved and nobody knows why */
+    components: string[]
+    /** the input paths this unit was keyed on, with their object ids */
+    inputOids: [string, string][]
+    /** the identity of the unit's OWN code, module graph included. Set for
+     *  every unit that has code, WORLD ones too: a TTL'd probe whose source
+     *  changed must not replay its previous answer, and this is the value that
+     *  busts it. null when the unit has no code entry or git could not hash it. */
+    codeKey: string | null
+}
+
+export type Registration = {
+    /** loud, blocking: a unit that declared two contradictory populations */
+    errors: string[]
+    /** the default-deny burn-down, one line per undeclared unit */
+    burnDown: string[]
+    /** every unit that was classified */
+    units: number
+    /** units in a cacheable population (content or world) */
+    declared: number
+}
+
+/** which population a declaration puts a unit in, before any git work.
+ *  `inputs` on a `ttl` unit is a registration FAIL: a unit cannot both be
+ *  described by files on disk and be un-describable by them. */
+export function classify(d: UnitDecl): { population: Population; error?: string; reason?: string; decided?: boolean } {
+    const hasInputs = (d.inputs ?? []).length > 0
+    const hasTtl = d.ttl != null && d.ttl !== ''
+    const hasNever = d.never != null && d.never !== ''
+    if (hasInputs && hasTtl)
+        return {
+            population: 'none',
+            error: `${d.kind} '${d.id}' declares BOTH inputs (CONTENT) and ttl (WORLD) — a unit reads files or it reads the world, never both`,
+        }
+    if (hasNever && (hasInputs || hasTtl))
+        return {
+            population: 'none',
+            error: `${d.kind} '${d.id}' declares never:'${d.never}' AND ${hasInputs ? 'inputs' : 'ttl'} — never means nothing can describe this unit, which the other declaration contradicts`,
+        }
+    if (hasInputs) return { population: 'content' }
+    if (hasTtl) return { population: 'world' }
+    if (hasNever) return { population: 'none', reason: d.never, decided: true }
+    return { population: 'none', reason: 'declares neither inputs nor ttl' }
+}
+
+export function validateRegistration(decls: UnitDecl[]): Registration {
+    const errors: string[] = []
+    const burnDown: string[] = []
+    let declared = 0
+    for (const d of decls) {
+        const c = classify(d)
+        if (c.error != null) errors.push(c.error)
+        // a DECIDED never leaves the list: the burn-down is what nobody has
+        // looked at yet, and mixing the two turns it into wallpaper
+        else if (c.population !== 'none') declared++
+        else if (c.decided !== true) burnDown.push(`${d.kind} ${d.id} — ${c.reason ?? 'undeclared'}`)
+        // a declaration that cannot even be turned into a repo-relative path
+        // is a registration bug, not a cache miss: it would key the unit on a
+        // path in another repo, or on the repo root
+        for (const raw of [...(d.inputs ?? []), ...(d.code ?? [])]) {
+            try {
+                normalizeInput(raw)
+            } catch (e) {
+                errors.push(`${d.kind} '${d.id}': ${e instanceof Error ? e.message : String(e)}`)
+            }
+        }
+    }
+    return { errors, burnDown, units: decls.length, declared }
+}
+
+export type KeyLayerOpts = {
+    /** the cli entry actually driving this run — toolchain identity needs to
+     *  know whether the gate binary lives inside the repo being gated */
+    cliEntry: string
+    /** module-graph config; loaded from the repo when omitted */
+    cfg?: RvtConfig
+}
+
+/**
+ * one scan, N keys. Build it once per run: the git work is shared across every
+ * unit, and building two of them in one run would pay for the scan twice and
+ * could straddle an edit.
+ */
+export class KeyLayer {
+    private constructor(
+        readonly root: string,
+        readonly decls: UnitDecl[],
+        readonly registration: Registration,
+        private readonly scan: ScanResult | null,
+        private readonly scanError: string | null,
+        private readonly closureByUnit: Map<string, string[]>,
+        private readonly codeOids: Map<string, string>,
+        private readonly toolchain: { id: string; uncacheable: string[] },
+        private readonly scanPathsUsed: string[],
+        /** wall time of the WHOLE layer: toolchain identity, the module-graph
+         *  closures, the git scan and the closure hashing. Reported rather
+         *  than the scan alone, because the scan alone is the flattering
+         *  number and the caller pays all of it. */
+        readonly openMs: number,
+    ) {}
+
+    private readonly installIds = new Map<string, string>()
+
+    /** the identity of the program this run is driving: bun version, the gate
+     *  binary's whole source, and every installed package. A WORLD unit needs
+     *  it too, because a TTL'd answer measured by a different shipkit is a
+     *  different answer, and the TTL alone cannot see that. */
+    get toolchainId(): string {
+        return this.toolchain.id
+    }
+
+    /** wall time the GIT half alone cost, ms — `openMs` is what a run pays */
+    get scanMs(): number {
+        return this.scan?.ms ?? 0
+    }
+
+    get failure(): string | null {
+        return this.scanError
+    }
+
+    static open(root: string, decls: UnitDecl[], o: KeyLayerOpts): KeyLayer {
+        const started = performance.now()
+        const registration = validateRegistration(decls)
+        const cfg = o.cfg ?? loadConfig(root)
+        // a gate binary that cannot be resolved is not a crash, it is a run
+        // whose program nobody can name — and the only safe answer to that is
+        // no key at all
+        let toolchain: { id: string; uncacheable: string[] }
+        try {
+            toolchain = toolchainIdentity(root, o.cliEntry)
+        } catch (e) {
+            toolchain = { id: 'unresolved', uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
+        }
+
+        // the unit's OWN code, resolved through the same module graph the
+        // `cycles` task walks. A check that imports a shared helper carries
+        // that helper, so fixing the helper moves the check's key.
+        const closureByUnit = new Map<string, string[]>()
+        // computed for EVERY unit with code, not only the content ones: a
+        // world unit's TTL cache must break when the probe itself is rewritten
+        for (const d of decls) {
+            const entries = d.code ?? []
+            if (entries.length > 0) closureByUnit.set(d.id, importClosure(cfg, entries))
+        }
+
+        // ONE scan for the declared inputs plus shipkit's own source: those
+        // are mostly directories, mostly unchanged, and git's stat cache
+        // answers them without reading a byte.
+        const paths = new Set<string>()
+        for (const d of decls) {
+            if (classify(d).population !== 'content') continue
+            for (const p of d.inputs ?? []) paths.add(p)
+        }
+
+        // the closure is INDIVIDUAL FILES belonging to no declared path, so it
+        // goes through one `hash-object --stdin-paths` instead of one `git
+        // add` spawn per file (measured: 15ms for the whole set)
+        const closureFiles = new Set<string>()
+        for (const files of closureByUnit.values()) for (const f of files) closureFiles.add(f)
+
+        let scan: ScanResult | null = null
+        let codeOids = new Map<string, string>()
+        let scanError: string | null = null
+        try {
+            scan = scanPaths(root, [...paths])
+            codeOids = hashFiles(root, [...closureFiles]).oids
+        } catch (e) {
+            // a scan that failed may have left a STALE tree behind, so nothing
+            // downstream may key on it. Loud, and the whole run goes uncached.
+            scanError = e instanceof GitScanError ? e.message : String(e)
+        }
+        return new KeyLayer(root, decls, registration, scan, scanError, closureByUnit, codeOids, toolchain, [...paths].sort(), performance.now() - started)
+    }
+
+    /** the key for one unit, or the reason there is none. */
+    keyFor(id: string): UnitKey {
+        const d = this.decls.find((u) => u.id === id)
+        if (d == null) throw new Error(`keyFor: no unit '${id}' in this layer`)
+        const c = classify(d)
+        const own = (this.closureByUnit.get(d.id) ?? []).slice().sort()
+        const ownReadable = own.every((p) => this.codeOids.get(p) != null)
+        const codeLines = own.map((p) => `code:${p}=${this.codeOids.get(p)!}`)
+        const base: UnitKey = {
+            id: d.id, kind: d.kind, population: c.population, key: null, reason: null, components: [], inputOids: [],
+            codeKey: own.length === 0 || !ownReadable ? null : createHash('sha256').update(codeLines.join('\n')).digest('hex').slice(0, 16),
+        }
+        if (c.error != null) return { ...base, reason: c.error }
+        if (c.population === 'world') return { ...base, reason: `world unit — TTL ${d.ttl}, no content key` }
+        if (c.population === 'none') return { ...base, reason: c.reason ?? 'undeclared' }
+        if (this.scanError != null) return { ...base, reason: `git scan failed, nothing may be keyed: ${this.scanError}` }
+        const scan = this.scan!
+        if (this.toolchain.uncacheable.length > 0)
+            return { ...base, reason: `toolchain not describable: ${this.toolchain.uncacheable.join('; ')}` }
+
+        let inputs: string[]
+        try {
+            inputs = (d.inputs ?? []).map(normalizeInput).sort()
+        } catch (e) {
+            return { ...base, reason: e instanceof Error ? e.message : String(e) }
+        }
+        const stale = inputs.filter((p) => scan.missing.includes(p))
+        if (stale.length > 0) return { ...base, reason: `declared input(s) not on disk: ${stale.join(', ')}` }
+        const hidden = inputs.flatMap((p) => scan.excluded.get(p) ?? [])
+        const inputOids: [string, string][] = inputs.map((p) => [p, scan.oidByPath.get(p) ?? ''])
+        if (inputOids.some(([, oid]) => oid === '')) return { ...base, reason: 'git could not name one of the declared inputs' }
+
+        if (!ownReadable) return { ...base, reason: `git could not hash this unit's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
+        const components: string[] = [
+            `v=${KEY_VERSION}`,
+            `unit=${d.kind}:${d.id}`,
+            `bun=${Bun.version}`,
+            // report-only mode changes what a self-maintaining rule DOES, and
+            // a TTL cannot see a mode. A frozen answer replayed into a normal
+            // run, or the reverse, is an answer to the other question.
+            `frozen=${process.env['SHIPKIT_FROZEN'] === '1'}`,
+            `toolchain=${this.toolchain.id}`,
+            `argv=${JSON.stringify(d.argv ?? [])}`,
+            ...(d.env ?? []).slice().sort().map((n) => `env:${n}=${process.env[n] ?? '<unset>'}`),
+            ...inputOids.map(([p, oid]) => `in:${p}=${oid}`),
+            ...codeLines,
+            ...(d.extra != null ? [`extra=${d.extra}`] : []),
+            // ignored content nobody keyed on is part of the key's MEANING:
+            // two runs that excluded different things answered different
+            // questions, and a silently narrowed key is the false green.
+            //
+            // An excluded INSTALL carries its contents rather than only its
+            // name. `test:libs` declares `wiki/build`, every sub-package's
+            // `node_modules` sits inside it and is ignored, and the toolchain
+            // component describes the ROOT install only: without this, a lib
+            // whose own dependencies moved replayed a green measured against
+            // the previous ones, and no path in the key could say so.
+            ...hidden.sort().map((p) => (p.split('/').at(-1) === 'node_modules' ? `excluded=${p}#${this.installOf(p)}` : `excluded=${p}`)),
+        ]
+        const key = createHash('sha256').update(components.join('\n')).digest('hex').slice(0, 32)
+        return { ...base, key, components, inputOids }
+    }
+
+    /** an excluded install's identity, once per layer: several units can name
+     *  the same one, and hashing a `node_modules` twice in a run is pure cost. */
+    private installOf(rel: string): string {
+        let id = this.installIds.get(rel)
+        if (id == null) { id = installIdentity(join(this.root, rel)); this.installIds.set(rel, id) }
+        return id
+    }
+
+    keys(): UnitKey[] {
+        return this.decls.map((d) => this.keyFor(d.id))
+    }
+
+    /** which units READ one of these paths. Answers the only question a
+     *  post-run re-hash has: whose result now describes a tree that no longer
+     *  exists. A unit declaring the whole work tree reads everything, so any
+     *  drift at all touches it. */
+    unitsTouchedBy(drifted: string[]): Set<string> {
+        const hit = new Set<string>()
+        if (drifted.length === 0) return hit
+        for (const d of this.decls) {
+            // the unit's own source counts exactly like a declared input: an
+            // agent fixing a check WHILE it runs must not store the old
+            // verdict under the new code's key
+            if ((this.closureByUnit.get(d.id) ?? []).some((f) => drifted.includes(f))) { hit.add(d.id); continue }
+            for (const raw of d.inputs ?? []) {
+                let p: string
+                try { p = normalizeInput(raw) } catch { hit.add(d.id); break }
+                if (p === WHOLE_TREE || drifted.some((x) => x === p || x.startsWith(`${p}/`))) { hit.add(d.id); break }
+            }
+        }
+        return hit
+    }
+
+    /**
+     * POST RUN RE HASH. Re-scan exactly what this layer keyed on and report
+     * every path whose object id moved while the units were running.
+     *
+     * This is the answer to the poisoning class: a unit reads a file, an agent
+     * in the next window edits it, and the result gets stored under a key that
+     * describes the file as it was BEFORE the edit. Any drift here means the
+     * run's results describe a tree that no longer exists, and the caller
+     * discards them rather than storing a lie. Measured at ~140ms, which is
+     * the price of never having to explain a hit that was wrong.
+     */
+    rescan(): { drifted: string[]; error: string | null; ms: number } {
+        if (this.scanError != null) return { drifted: [], error: this.scanError, ms: 0 }
+        const started = performance.now()
+        try {
+            const after = scanPaths(this.root, this.scanPathsUsed)
+            const drifted: string[] = []
+            for (const [p, oid] of this.scan!.oidByPath) if (after.oidByPath.get(p) !== oid) drifted.push(p)
+            for (const [p] of after.oidByPath) if (!this.scan!.oidByPath.has(p)) drifted.push(p)
+            // the unit's own code drifts the same way a declared input does: an
+            // agent fixing a check WHILE that check runs would otherwise store
+            // the old verdict under the new code's key
+            const files = [...this.codeOids.keys()]
+            if (files.length > 0) {
+                const now = hashFiles(this.root, files)
+                for (const [p, oid] of this.codeOids) if (now.oids.get(p) !== oid) drifted.push(p)
+            }
+            return { drifted: [...new Set(drifted)].sort(), error: null, ms: performance.now() - started }
+        } catch (e) {
+            return { drifted: [], error: e instanceof GitScanError ? e.message : String(e), ms: performance.now() - started }
+        }
+    }
+}