rvlib-shipkit 0.38.0 → 1.0.0

package/src/ledger/git-key.ts

@@ -1,22 +1,30 @@
-// THE KEY. What question did this unit answer, and is that still the same
+// THE KEY. What question did this check answer, and is that still the same
 // question? Nothing in this file caches anything: it produces the identity a
 // later ledger will store results under, and it refuses to produce one
 // whenever the answer would be a guess.
 //
-// THREE POPULATIONS, declared at registration, never inferred:
+// FOUR POPULATIONS, set at registration, never inferred from source:
 //
 //   CONTENT  declares `inputs` — the paths it reads. Keyed on their git object
-//            ids, plus the unit's own code (module graph), plus the toolchain
+//            ids, plus the check's own code (module graph), plus the toolchain
 //            (which content-hashes the gate binary, so editing shipkit moves
 //            every key in every repo), plus argv and declared env. No TTL: a
 //            content key is either the same question or a different one.
 //   WORLD    declares `ttl` — it reads the network, the keychain, $PATH, a git
 //            remote or the clock. Nothing on disk describes its answer, so it
 //            keeps the existing clean-pass TTL and its `cacheKey`, unchanged.
-//   NEITHER  declares nothing. Never cached. This is the DEFAULT, and it is
-//            the whole safety story: forgetting to declare costs speed, never
-//            correctness, so the burn-down list below is a to-do rather than a
-//            hazard. It holds only while nothing declares ON a unit's behalf:
+//   READS    a `run` check file that declares none of the three: the default
+//            for that shape. Its body reads through the check context, which
+//            records every read, so the key here is only the question (check,
+//            code, toolchain, tier) and the ledger stores the recorded read set
+//            beside the answer, re-checking it before any replay
+//            (ledger/ledger.ts). One whose code reaches the disk or a
+//            process around the context stays NEITHER (core/raw-access.ts).
+//   NEITHER  a `cmd` check or a legacy shape that declares nothing, or any
+//            check that says `never`. Never cached. For those it is the
+//            DEFAULT, and it is the whole safety story: forgetting to declare
+//            costs speed, never correctness, so the burn-down list below is a
+//            to-do rather than a hazard. It holds only while nothing declares ON a check's behalf:
 //            the gate floor gives `['.']` to the four slots tracked source
 //            decides and deliberately leaves `test` here (core/floor.ts says
 //            why). Declaring BOTH is a registration failure, loud, because the
@@ -39,40 +47,48 @@ import { installIdentity, toolchainIdentity } from './toolchain-identity.ts'
 
 /** the key format. Bumped when a component is added or its rendering changes,
  *  so an old ledger can never answer a new question. */
-export const KEY_VERSION = 2
+export const KEY_VERSION = 3
 
-export type Population = 'content' | 'world' | 'none'
+export type Population = 'content' | 'world' | 'reads' | 'none'
 
-/** what one gate unit (a task, a rule, a discovered check) declares about its
- *  own cacheability. `gate-units.ts` builds these from the live registries. */
-export type UnitDecl = {
-    /** the id an operator types: a task id, a rule id, a check name */
+/** what one gate check declares about its own cacheability.
+ *  `gate-checks.ts` builds these from the one enumeration. */
+export type CheckDecl = {
+    /** the id an operator types, unique in the repo */
     id: string
-    kind: 'task' | 'rule' | 'check'
-    /** CONTENT: repo-relative paths this unit reads. Presence declares the population. */
+    /** CONTENT: repo-relative paths this check reads. Presence declares the population. */
     inputs?: string[]
     /** WORLD: clean-pass duration, e.g. '24h'. Presence declares the population. */
     ttl?: string
-    /** repo-relative entry files of the unit's OWN code — the module-graph
+    /** repo-relative entry files of the check's OWN code — the module-graph
      *  roots. A discovered check gets its own file for free. */
     code?: string[]
-    /** the argv a subprocess unit runs */
+    /** the argv a subprocess check runs */
     argv?: string[]
     /** env var NAMES whose values belong in the key */
     env?: string[]
     /** anything else that identifies the question (a rule's `cacheKey`) */
     extra?: string
     /** a DECIDED never-cacheable, with the reason. Not the same thing as an
-     *  undeclared unit: this one has been looked at and cannot be described
+     *  undeclared check: this one has been looked at and cannot be described
      *  (it reads git history, a path outside the repo, the wall clock), so it
      *  leaves the burn-down list. Declaring it beside inputs or ttl is a
      *  registration failure, because those say the opposite. */
     never?: string
+    /** it rewrites the repo unless the run is frozen, so frozen is part of its
+     *  question (check.ts `writes`) */
+    writes?: boolean
+    /** READS: an in process body whose reads the check context records. Set
+     *  by the composer for a `run` check file declaring no other population,
+     *  never by a repo, and ignored beside `inputs`, `ttl` or `never` */
+    reads?: boolean
+    /** why an undeclared `run` check stays uncached: the first module its body
+     *  can reach that touches the disk or a process around the context */
+    uncacheable?: string
 }
 
-export type UnitKey = {
+export type CheckKey = {
     id: string
-    kind: UnitDecl['kind']
     population: Population
     /** the content key, or null for world/none/refused */
     key: string | null
@@ -81,50 +97,52 @@ export type UnitKey = {
     /** every component that went into the key, in key order — this is what a
      *  human reads when a key moved and nobody knows why */
     components: string[]
-    /** the input paths this unit was keyed on, with their object ids */
+    /** the input paths this check was keyed on, with their object ids */
     inputOids: [string, string][]
-    /** the identity of the unit's OWN code, module graph included. Set for
-     *  every unit that has code, WORLD ones too: a TTL'd probe whose source
+    /** the identity of the check's OWN code, module graph included. Set for
+     *  every check that has code, WORLD ones too: a TTL'd probe whose source
      *  changed must not replay its previous answer, and this is the value that
-     *  busts it. null when the unit has no code entry or git could not hash it. */
+     *  busts it. null when the check has no code entry or git could not hash it. */
     codeKey: string | null
 }
 
 export type Registration = {
-    /** loud, blocking: a unit that declared two contradictory populations */
+    /** loud, blocking: a check that declared two contradictory populations */
     errors: string[]
-    /** the default-deny burn-down, one line per undeclared unit */
+    /** the default-deny burn-down, one line per undeclared check */
     burnDown: string[]
-    /** every unit that was classified */
-    units: number
-    /** units in a cacheable population (content or world) */
+    /** every check that was classified */
+    checks: number
+    /** checks in a cacheable population (content or world) */
     declared: number
 }
 
-/** which population a declaration puts a unit in, before any git work.
- *  `inputs` on a `ttl` unit is a registration FAIL: a unit cannot both be
+/** which population a declaration puts a check in, before any git work.
+ *  `inputs` on a `ttl` check is a registration FAIL: a check cannot both be
  *  described by files on disk and be un-describable by them. */
-export function classify(d: UnitDecl): { population: Population; error?: string; reason?: string; decided?: boolean } {
+export function classify(d: CheckDecl): { population: Population; error?: string; reason?: string; decided?: boolean } {
     const hasInputs = (d.inputs ?? []).length > 0
     const hasTtl = d.ttl != null && d.ttl !== ''
     const hasNever = d.never != null && d.never !== ''
     if (hasInputs && hasTtl)
         return {
             population: 'none',
-            error: `${d.kind} '${d.id}' declares BOTH inputs (CONTENT) and ttl (WORLD) — a unit reads files or it reads the world, never both`,
+            error: `check '${d.id}' declares BOTH inputs (CONTENT) and ttl (WORLD) — a check reads files or it reads the world, never both`,
         }
     if (hasNever && (hasInputs || hasTtl))
         return {
             population: 'none',
-            error: `${d.kind} '${d.id}' declares never:'${d.never}' AND ${hasInputs ? 'inputs' : 'ttl'} — never means nothing can describe this unit, which the other declaration contradicts`,
+            error: `check '${d.id}' declares never:'${d.never}' AND ${hasInputs ? 'inputs' : 'ttl'} — never means nothing can describe this check, which the other declaration contradicts`,
         }
     if (hasInputs) return { population: 'content' }
     if (hasTtl) return { population: 'world' }
     if (hasNever) return { population: 'none', reason: d.never, decided: true }
+    if (d.reads === true) return { population: 'reads' }
+    if (d.uncacheable != null) return { population: 'none', reason: d.uncacheable }
     return { population: 'none', reason: 'declares neither inputs nor ttl' }
 }
 
-export function validateRegistration(decls: UnitDecl[]): Registration {
+export function validateRegistration(decls: CheckDecl[]): Registration {
     const errors: string[] = []
     const burnDown: string[] = []
     let declared = 0
@@ -134,19 +152,19 @@ export function validateRegistration(decls: UnitDecl[]): Registration {
         // a DECIDED never leaves the list: the burn-down is what nobody has
         // looked at yet, and mixing the two turns it into wallpaper
         else if (c.population !== 'none') declared++
-        else if (c.decided !== true) burnDown.push(`${d.kind} ${d.id} — ${c.reason ?? 'undeclared'}`)
+        else if (c.decided !== true) burnDown.push(`${d.id} — ${c.reason ?? 'undeclared'}`)
         // a declaration that cannot even be turned into a repo-relative path
-        // is a registration bug, not a cache miss: it would key the unit on a
+        // is a registration bug, not a cache miss: it would key the check on a
         // path in another repo, or on the repo root
         for (const raw of [...(d.inputs ?? []), ...(d.code ?? [])]) {
             try {
                 normalizeInput(raw)
             } catch (e) {
-                errors.push(`${d.kind} '${d.id}': ${e instanceof Error ? e.message : String(e)}`)
+                errors.push(`check '${d.id}': ${e instanceof Error ? e.message : String(e)}`)
             }
         }
     }
-    return { errors, burnDown, units: decls.length, declared }
+    return { errors, burnDown, checks: decls.length, declared }
 }
 
 export type KeyLayerOpts = {
@@ -159,17 +177,17 @@ export type KeyLayerOpts = {
 
 /**
  * one scan, N keys. Build it once per run: the git work is shared across every
- * unit, and building two of them in one run would pay for the scan twice and
+ * check, and building two of them in one run would pay for the scan twice and
  * could straddle an edit.
  */
 export class KeyLayer {
     private constructor(
         readonly root: string,
-        readonly decls: UnitDecl[],
+        readonly decls: CheckDecl[],
         readonly registration: Registration,
         private readonly scan: ScanResult | null,
         private readonly scanError: string | null,
-        private readonly closureByUnit: Map<string, string[]>,
+        private readonly closureByCheck: Map<string, string[]>,
         private readonly codeOids: Map<string, string>,
         private readonly toolchain: { id: string; uncacheable: string[] },
         private readonly scanPathsUsed: string[],
@@ -183,7 +201,7 @@ export class KeyLayer {
     private readonly installIds = new Map<string, string>()
 
     /** the identity of the program this run is driving: bun version, the gate
-     *  binary's whole source, and every installed package. A WORLD unit needs
+     *  binary's whole source, and every installed package. A WORLD check needs
      *  it too, because a TTL'd answer measured by a different shipkit is a
      *  different answer, and the TTL alone cannot see that. */
     get toolchainId(): string {
@@ -199,7 +217,7 @@ export class KeyLayer {
         return this.scanError
     }
 
-    static open(root: string, decls: UnitDecl[], o: KeyLayerOpts): KeyLayer {
+    static open(root: string, decls: CheckDecl[], o: KeyLayerOpts): KeyLayer {
         const started = performance.now()
         const registration = validateRegistration(decls)
         const cfg = o.cfg ?? loadConfig(root)
@@ -213,15 +231,15 @@ export class KeyLayer {
             toolchain = { id: 'unresolved', uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
         }
 
-        // the unit's OWN code, resolved through the same module graph the
+        // the check's OWN code, resolved through the same module graph the
         // `cycles` task walks. A check that imports a shared helper carries
         // that helper, so fixing the helper moves the check's key.
-        const closureByUnit = new Map<string, string[]>()
-        // computed for EVERY unit with code, not only the content ones: a
-        // world unit's TTL cache must break when the probe itself is rewritten
+        const closureByCheck = new Map<string, string[]>()
+        // computed for EVERY check with code, not only the content ones: a
+        // world check's TTL cache must break when the probe itself is rewritten
         for (const d of decls) {
             const entries = d.code ?? []
-            if (entries.length > 0) closureByUnit.set(d.id, importClosure(cfg, entries))
+            if (entries.length > 0) closureByCheck.set(d.id, importClosure(cfg, entries))
         }
 
         // ONE scan for the declared inputs plus shipkit's own source: those
@@ -237,7 +255,7 @@ export class KeyLayer {
         // goes through one `hash-object --stdin-paths` instead of one `git
         // add` spawn per file (measured: 15ms for the whole set)
         const closureFiles = new Set<string>()
-        for (const files of closureByUnit.values()) for (const f of files) closureFiles.add(f)
+        for (const files of closureByCheck.values()) for (const f of files) closureFiles.add(f)
 
         let scan: ScanResult | null = null
         let codeOids = new Map<string, string>()
@@ -250,28 +268,45 @@ export class KeyLayer {
             // downstream may key on it. Loud, and the whole run goes uncached.
             scanError = e instanceof GitScanError ? e.message : String(e)
         }
-        return new KeyLayer(root, decls, registration, scan, scanError, closureByUnit, codeOids, toolchain, [...paths].sort(), performance.now() - started)
+        return new KeyLayer(root, decls, registration, scan, scanError, closureByCheck, codeOids, toolchain, [...paths].sort(), performance.now() - started)
     }
 
-    /** the key for one unit, or the reason there is none. */
-    keyFor(id: string): UnitKey {
+    /** the key for one check, or the reason there is none. */
+    keyFor(id: string): CheckKey {
         const d = this.decls.find((u) => u.id === id)
-        if (d == null) throw new Error(`keyFor: no unit '${id}' in this layer`)
+        if (d == null) throw new Error(`keyFor: no check '${id}' in this layer`)
         const c = classify(d)
-        const own = (this.closureByUnit.get(d.id) ?? []).slice().sort()
+        const own = (this.closureByCheck.get(d.id) ?? []).slice().sort()
         const ownReadable = own.every((p) => this.codeOids.get(p) != null)
         const codeLines = own.map((p) => `code:${p}=${this.codeOids.get(p)!}`)
-        const base: UnitKey = {
-            id: d.id, kind: d.kind, population: c.population, key: null, reason: null, components: [], inputOids: [],
+        const base: CheckKey = {
+            id: d.id, population: c.population, key: null, reason: null, components: [], inputOids: [],
             codeKey: own.length === 0 || !ownReadable ? null : createHash('sha256').update(codeLines.join('\n')).digest('hex').slice(0, 16),
         }
         if (c.error != null) return { ...base, reason: c.error }
-        if (c.population === 'world') return { ...base, reason: `world unit — TTL ${d.ttl}, no content key` }
+        if (c.population === 'world') return { ...base, reason: `world check — TTL ${d.ttl}, no content key` }
         if (c.population === 'none') return { ...base, reason: c.reason ?? 'undeclared' }
         if (this.scanError != null) return { ...base, reason: `git scan failed, nothing may be keyed: ${this.scanError}` }
         const scan = this.scan!
         if (this.toolchain.uncacheable.length > 0)
             return { ...base, reason: `toolchain not describable: ${this.toolchain.uncacheable.join('; ')}` }
+        // the question alone: what it read is stored with each answer and
+        // re-checked before a replay, so no read belongs in this key
+        if (c.population === 'reads') {
+            // a built-in has no code entry: the toolchain id hashes its source
+            if (!ownReadable) return { ...base, reason: `git could not hash this check's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
+            const components = [
+                `v=${KEY_VERSION}`,
+                `check=${d.id}`,
+                `bun=${Bun.version}`,
+                ...(d.writes === true ? [`frozen=${process.env['SHIPKIT_FROZEN'] === '1'}`] : []),
+                `toolchain=${this.toolchain.id}`,
+                'reads=recorded',
+                ...codeLines,
+                ...(d.extra != null ? [`extra=${d.extra}`] : []),
+            ]
+            return { ...base, key: createHash('sha256').update(components.join('\n')).digest('hex').slice(0, 32), components }
+        }
 
         let inputs: string[]
         try {
@@ -285,15 +320,17 @@ export class KeyLayer {
         const inputOids: [string, string][] = inputs.map((p) => [p, scan.oidByPath.get(p) ?? ''])
         if (inputOids.some(([, oid]) => oid === '')) return { ...base, reason: 'git could not name one of the declared inputs' }
 
-        if (!ownReadable) return { ...base, reason: `git could not hash this unit's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
+        if (!ownReadable) return { ...base, reason: `git could not hash this check's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
         const components: string[] = [
             `v=${KEY_VERSION}`,
-            `unit=${d.kind}:${d.id}`,
+            `check=${d.id}`,
             `bun=${Bun.version}`,
-            // report-only mode changes what a self-maintaining rule DOES, and
-            // a TTL cannot see a mode. A frozen answer replayed into a normal
-            // run, or the reverse, is an answer to the other question.
-            `frozen=${process.env['SHIPKIT_FROZEN'] === '1'}`,
+            // report-only mode changes what a check that WRITES does, so for
+            // that check a frozen answer replayed into a normal run, or the
+            // reverse, answers the other question. Every other check does the
+            // same thing either way, and keying it on the mode only made a
+            // frozen run and a normal one measure it twice.
+            ...(d.writes === true ? [`frozen=${process.env['SHIPKIT_FROZEN'] === '1'}`] : []),
             `toolchain=${this.toolchain.id}`,
             `argv=${JSON.stringify(d.argv ?? [])}`,
             ...(d.env ?? []).slice().sort().map((n) => `env:${n}=${process.env[n] ?? '<unset>'}`),
@@ -305,7 +342,7 @@ export class KeyLayer {
             // questions, and a silently narrowed key is the false green.
             //
             // An excluded INSTALL carries its contents rather than only its
-            // name. `test:libs` declares `wiki/build`, every sub-package's
+            // name. A unit's test check declares its dir, and the unit's
             // `node_modules` sits inside it and is ignored, and the toolchain
             // component describes the ROOT install only: without this, a lib
             // whose own dependencies moved replayed a green measured against
@@ -316,7 +353,7 @@ export class KeyLayer {
         return { ...base, key, components, inputOids }
     }
 
-    /** an excluded install's identity, once per layer: several units can name
+    /** an excluded install's identity, once per layer: several checks can name
      *  the same one, and hashing a `node_modules` twice in a run is pure cost. */
     private installOf(rel: string): string {
         let id = this.installIds.get(rel)
@@ -324,22 +361,22 @@ export class KeyLayer {
         return id
     }
 
-    keys(): UnitKey[] {
+    keys(): CheckKey[] {
         return this.decls.map((d) => this.keyFor(d.id))
     }
 
-    /** which units READ one of these paths. Answers the only question a
+    /** which checks READ one of these paths. Answers the only question a
      *  post-run re-hash has: whose result now describes a tree that no longer
-     *  exists. A unit declaring the whole work tree reads everything, so any
+     *  exists. A check declaring the whole work tree reads everything, so any
      *  drift at all touches it. */
     unitsTouchedBy(drifted: string[]): Set<string> {
         const hit = new Set<string>()
         if (drifted.length === 0) return hit
         for (const d of this.decls) {
-            // the unit's own source counts exactly like a declared input: an
+            // the check's own source counts exactly like a declared input: an
             // agent fixing a check WHILE it runs must not store the old
             // verdict under the new code's key
-            if ((this.closureByUnit.get(d.id) ?? []).some((f) => drifted.includes(f))) { hit.add(d.id); continue }
+            if ((this.closureByCheck.get(d.id) ?? []).some((f) => drifted.includes(f))) { hit.add(d.id); continue }
             for (const raw of d.inputs ?? []) {
                 let p: string
                 try { p = normalizeInput(raw) } catch { hit.add(d.id); break }
@@ -351,24 +388,26 @@ export class KeyLayer {
 
     /**
      * POST RUN RE HASH. Re-scan exactly what this layer keyed on and report
-     * every path whose object id moved while the units were running.
+     * every path whose object id moved while the checks were running.
      *
-     * This is the answer to the poisoning class: a unit reads a file, an agent
+     * This is the answer to the poisoning class: a check reads a file, an agent
      * in the next window edits it, and the result gets stored under a key that
      * describes the file as it was BEFORE the edit. Any drift here means the
      * run's results describe a tree that no longer exists, and the caller
-     * discards them rather than storing a lie. Measured at ~140ms, which is
-     * the price of never having to explain a hit that was wrong.
+     * discards them rather than storing a lie. It reuses the first scan's
+     * ignored answer, and an unchanged root tree answers every path without
+     * asking git per path: about 60ms on this repo when nothing moved, which
+     * is the price of never having to explain a hit that was wrong.
      */
     rescan(): { drifted: string[]; error: string | null; ms: number } {
         if (this.scanError != null) return { drifted: [], error: this.scanError, ms: 0 }
         const started = performance.now()
         try {
-            const after = scanPaths(this.root, this.scanPathsUsed)
+            const after = scanPaths(this.root, this.scanPathsUsed, this.scan!)
             const drifted: string[] = []
             for (const [p, oid] of this.scan!.oidByPath) if (after.oidByPath.get(p) !== oid) drifted.push(p)
             for (const [p] of after.oidByPath) if (!this.scan!.oidByPath.has(p)) drifted.push(p)
-            // the unit's own code drifts the same way a declared input does: an
+            // the check's own code drifts the same way a declared input does: an
             // agent fixing a check WHILE that check runs would otherwise store
             // the old verdict under the new code's key
             const files = [...this.codeOids.keys()]