rvlib-shipkit 1.4.0 → 1.6.0

package/src/ledger/git-key.ts

@@ -6,10 +6,10 @@
 // FOUR POPULATIONS, set at registration, never inferred from source:
 //
 //   CONTENT  declares `inputs` — the paths it reads. Keyed on their git object
-//            ids, plus the check's own code (module graph), plus the toolchain
-//            (which content-hashes the gate binary, so editing shipkit moves
-//            every key in every repo), plus argv and declared env. No TTL: a
-//            content key is either the same question or a different one.
+//            ids, plus the check's own code (module graph), plus the runner
+//            and the install (toolchain-identity.ts), plus argv and declared
+//            env. No TTL: a content key is either the same question or a
+//            different one.
 //   WORLD    declares `ttl` — it reads the network, the keychain, $PATH, a git
 //            remote or the clock. Nothing on disk describes its answer, so it
 //            keeps the existing clean-pass TTL and its `cacheKey`, unchanged.
@@ -24,11 +24,11 @@
 //            check that says `never`. Never cached. For those it is the
 //            DEFAULT, and it is the whole safety story: forgetting to declare
 //            costs speed, never correctness, so the burn-down list below is a
-//            to-do rather than a hazard. It holds only while nothing declares ON a check's behalf:
-//            the gate floor gives `['.']` to the four slots tracked source
-//            decides and deliberately leaves `test` here (core/floor.ts says
-//            why). Declaring BOTH is a registration failure, loud, because the
-//            two answers contradict each other.
+//            to-do rather than a hazard. A TASK that declares nothing is not
+//            here: the composer keys it on the whole work tree plus the repo's
+//            ignoredInputs (ledger/gate-checks.ts). Declaring BOTH is a
+//            registration failure, loud, because the two answers contradict
+//            each other.
 //
 // CROSS WORKTREE. Every component here is content addressed or a version
 // string; not one of them is an absolute path. Two worktrees of one repo
@@ -39,15 +39,31 @@
 // differ from main by two top-level packages. That is what makes a shared
 // ledger safe rather than a stale green in a checkout nobody looked at.
 import { createHash } from 'crypto'
-import { join } from 'path'
+import { existsSync, readdirSync, readFileSync, type Dirent } from 'fs'
+import { isAbsolute, join, relative } from 'path'
+import { gitBin } from '../core/git-bin.ts'
 import { loadConfig, type RvtConfig } from '../ts/config.ts'
-import { importClosure } from '../ts/source-graph.ts'
+import { extractSpecs, importClosure } from '../ts/source-graph.ts'
 import { GitScanError, hashFiles, normalizeInput, scanPaths, WHOLE_TREE, type ScanResult } from './git-scan.ts'
-import { installIdentity, toolchainIdentity } from './toolchain-identity.ts'
+import { installIdentity, toolchainIdentity, type LibSource } from './toolchain-identity.ts'
 
 /** the key format. Bumped when a component is added or its rendering changes,
  *  so an old ledger can never answer a new question. */
-export const KEY_VERSION = 3
+export const KEY_VERSION = 4
+
+/** `dir/**` names a dir a task only LISTS: keyed on the names under it */
+export const isNamesInput = (p: string): boolean => p.endsWith('/**')
+
+let nodeMemo: string | null = null
+/** the node a task's tools run under, from PATH: a runtime upgrade is a
+ *  different program, like a bun upgrade */
+function nodeVersion(): string {
+    if (nodeMemo == null) {
+        const node = Bun.which('node')
+        nodeMemo = node == null ? 'none' : Bun.spawnSync([node, '--version'], { stdout: 'pipe', stderr: 'pipe' }).stdout.toString().trim() || 'unknown'
+    }
+    return nodeMemo
+}
 
 export type Population = 'content' | 'world' | 'reads' | 'none'
 
@@ -63,6 +79,10 @@ export type CheckDecl = {
     /** repo-relative entry files of the check's OWN code — the module-graph
      *  roots. A discovered check gets its own file for free. */
     code?: string[]
+    /** absolute entry files of the check's own code INSIDE the toolkit: a
+     *  built-in check's file. A task whose argv runs a toolkit module gets
+     *  that module's closure without declaring it */
+    libCode?: string[]
     /** the argv a subprocess check runs */
     argv?: string[]
     /** env var NAMES whose values belong in the key */
@@ -85,6 +105,9 @@ export type CheckDecl = {
     /** why an undeclared `run` check stays uncached: the first module its body
      *  can reach that touches the disk or a process around the context */
     uncacheable?: string
+    /** the check's cache declaration contradicts its body (`reads` on a body
+     *  that touches the disk around its context): a registration failure */
+    declError?: string
 }
 
 export type CheckKey = {
@@ -99,10 +122,14 @@ export type CheckKey = {
     components: string[]
     /** the input paths this check was keyed on, with their object ids */
     inputOids: [string, string][]
-    /** the identity of the check's OWN code, module graph included. Set for
-     *  every check that has code, WORLD ones too: a TTL'd probe whose source
-     *  changed must not replay its previous answer, and this is the value that
-     *  busts it. null when the check has no code entry or git could not hash it. */
+    /** the toolkit files behind its `lib=` and `libused=` components,
+     *  package-relative: what `shipkit keys --why` lists */
+    libFiles?: string[]
+    /** the identity of the check's OWN code, module graph included, in the
+     *  repo and in the toolkit. Set for every check that has code, WORLD ones
+     *  too: a TTL'd probe whose source changed must not replay its previous
+     *  answer, and this is the value that busts it. null when the check has no
+     *  code entry or its code could not be hashed. */
     codeKey: string | null
 }
 
@@ -121,6 +148,7 @@ export type Registration = {
  *  `inputs` on a `ttl` check is a registration FAIL: a check cannot both be
  *  described by files on disk and be un-describable by them. */
 export function classify(d: CheckDecl): { population: Population; error?: string; reason?: string; decided?: boolean } {
+    if (d.declError != null) return { population: 'none', error: `check '${d.id}': ${d.declError}` }
     const hasInputs = (d.inputs ?? []).length > 0
     const hasTtl = d.ttl != null && d.ttl !== ''
     const hasNever = d.never != null && d.never !== ''
@@ -192,7 +220,7 @@ export class KeyLayer {
         private readonly scanError: string | null,
         private readonly closureByCheck: Map<string, string[]>,
         private readonly codeOids: Map<string, string>,
-        private readonly toolchain: { id: string; uncacheable: string[] },
+        private readonly toolchain: { id: string; install: string; lib: LibSource | null; uncacheable: string[] },
         private readonly scanPathsUsed: string[],
         private readonly scope: ReadonlySet<string> | null,
         /** wall time of the WHOLE layer: toolchain identity, the module-graph
@@ -204,12 +232,11 @@ export class KeyLayer {
 
     private readonly installIds = new Map<string, string>()
 
-    /** the identity of the program this run is driving: bun version, the gate
-     *  binary's whole source, and every installed package. A WORLD check needs
-     *  it too, because a TTL'd answer measured by a different shipkit is a
-     *  different answer, and the TTL alone cannot see that. */
+    /** the runner and the install a WORLD answer is remembered under: a TTL'd
+     *  answer measured by a different shipkit is a different answer, and the
+     *  TTL alone cannot see that. The probe's own code rides in `codeKey` */
     get toolchainId(): string {
-        return this.toolchain.id
+        return `${this.toolchain.id}:${this.toolchain.install}`
     }
 
     /** wall time the GIT half alone cost, ms — `openMs` is what a run pays */
@@ -228,11 +255,11 @@ export class KeyLayer {
         // a gate binary that cannot be resolved is not a crash, it is a run
         // whose program nobody can name — and the only safe answer to that is
         // no key at all
-        let toolchain: { id: string; uncacheable: string[] }
+        let toolchain: { id: string; install: string; lib: LibSource | null; uncacheable: string[] }
         try {
-            toolchain = toolchainIdentity(root, o.cliEntry)
+            toolchain = { ...toolchainIdentity(root, o.cliEntry), uncacheable: [] }
         } catch (e) {
-            toolchain = { id: 'unresolved', uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
+            toolchain = { id: 'unresolved', install: 'unresolved', lib: null, uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
         }
 
         // the check's OWN code, resolved through the same module graph the
@@ -253,7 +280,7 @@ export class KeyLayer {
         const paths = new Set<string>()
         for (const d of scoped) {
             if (classify(d).population !== 'content') continue
-            for (const p of d.inputs ?? []) paths.add(p)
+            for (const p of d.inputs ?? []) if (!isNamesInput(p)) paths.add(p)
         }
 
         // the closure is INDIVIDUAL FILES belonging to no declared path, so it
@@ -283,9 +310,12 @@ export class KeyLayer {
         const own = (this.closureByCheck.get(d.id) ?? []).slice().sort()
         const ownReadable = own.every((p) => this.codeOids.get(p) != null)
         const codeLines = own.map((p) => `code:${p}=${this.codeOids.get(p)!}`)
+        const lib = this.libLines(d)
         const base: CheckKey = {
-            id: d.id, population: c.population, key: null, reason: null, components: [], inputOids: [],
-            codeKey: own.length === 0 || !ownReadable ? null : createHash('sha256').update(codeLines.join('\n')).digest('hex').slice(0, 16),
+            id: d.id, population: c.population, key: null, reason: null, components: [], inputOids: [], libFiles: this.libFilesOf(d),
+            codeKey: typeof lib === 'string' || (own.length === 0 || !ownReadable) && lib.length === 0
+                ? null
+                : createHash('sha256').update([...(ownReadable ? codeLines : []), ...lib].join('\n')).digest('hex').slice(0, 16),
         }
         if (c.error != null) return { ...base, reason: c.error }
         if (this.scope != null && !this.scope.has(d.id)) return { ...base, reason: 'not one of the checks this run executes, so nothing was scanned for it' }
@@ -295,10 +325,10 @@ export class KeyLayer {
         const scan = this.scan!
         if (this.toolchain.uncacheable.length > 0)
             return { ...base, reason: `toolchain not describable: ${this.toolchain.uncacheable.join('; ')}` }
+        if (typeof lib === 'string') return { ...base, reason: lib }
         // the question alone: what it read is stored with each answer and
         // re-checked before a replay, so no read belongs in this key
         if (c.population === 'reads') {
-            // a built-in has no code entry: the toolchain id hashes its source
             if (!ownReadable) return { ...base, reason: `git could not hash this check's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
             const components = [
                 `v=${KEY_VERSION}`,
@@ -308,14 +338,17 @@ export class KeyLayer {
                 `toolchain=${this.toolchain.id}`,
                 'reads=recorded',
                 ...codeLines,
+                ...lib,
                 ...(d.extra != null ? [`extra=${d.extra}`] : []),
             ]
             return { ...base, key: createHash('sha256').update(components.join('\n')).digest('hex').slice(0, 32), components }
         }
 
         let inputs: string[]
+        let names: string[]
         try {
-            inputs = (d.inputs ?? []).map(normalizeInput).sort()
+            inputs = (d.inputs ?? []).filter((p) => !isNamesInput(p)).map(normalizeInput).sort()
+            names = (d.inputs ?? []).filter(isNamesInput).map((p) => normalizeInput(p.slice(0, -3))).sort()
         } catch (e) {
             return { ...base, reason: e instanceof Error ? e.message : String(e) }
         }
@@ -337,10 +370,17 @@ export class KeyLayer {
             // frozen run and a normal one measure it twice.
             ...(d.writes === true ? [`frozen=${process.env['SHIPKIT_FROZEN'] === '1'}`] : []),
             `toolchain=${this.toolchain.id}`,
+            // the repo's own tools run here, and what is installed is part of
+            // what they are
+            `install=${this.toolchain.install}`,
             `argv=${JSON.stringify(d.argv ?? [])}`,
             ...(d.env ?? []).slice().sort().map((n) => `env:${n}=${process.env[n] ?? '<unset>'}`),
             ...inputOids.map(([p, oid]) => `in:${p}=${oid}`),
+            // a dir the task only lists: the names under it, never its bytes
+            ...names.map((p) => `names:${p}=${this.namesOf(p)}`),
+            `node=${nodeVersion()}`,
             ...codeLines,
+            ...lib,
             ...(d.extra != null ? [`extra=${d.extra}`] : []),
             // ignored content nobody keyed on is part of the key's MEANING:
             // two runs that excluded different things answered different
@@ -358,6 +398,88 @@ export class KeyLayer {
         return { ...base, key, components, inputOids }
     }
 
+    /** the toolkit code a check's answer comes from, as key lines, or why it
+     *  cannot be keyed. A built-in names its file, a task running a toolkit
+     *  module names it in its argv. Everything else runs the REPO's code, and
+     *  in a repo that does not host the toolkit that code imports it by
+     *  package name, through an install the install id leaves the toolkit out
+     *  of: those carry the closure of every toolkit module the repo imports */
+    private libLines(d: CheckDecl): string[] | string {
+        const lib = this.toolchain.lib
+        if (lib == null) return []
+        const entries = [...(d.libCode ?? []), ...(d.argv ?? []).filter((a) => isAbsolute(a) && lib.owns(a))]
+        try {
+            const used = d.libCode == null ? this.libUsed() : []
+            if (typeof used === 'string') return used
+            return [...(entries.length > 0 ? [`lib=${lib.closureKey(entries)}`] : []), ...used]
+        } catch (e) {
+            return `the toolkit source this check runs could not be read: ${e instanceof Error ? e.message : String(e)}`
+        }
+    }
+
+    private libUsedMemo: string[] | string | null = null
+    private libUsedEntries: string[] = []
+
+    /** the files behind libLines, for a reader; empty when there are none */
+    private libFilesOf(d: CheckDecl): string[] {
+        const lib = this.toolchain.lib
+        if (lib == null) return []
+        const entries = [...(d.libCode ?? []), ...(d.argv ?? []).filter((a) => isAbsolute(a) && lib.owns(a)), ...(d.libCode == null ? this.libUsedEntries : [])]
+        try { return [...new Set(entries.flatMap((e) => lib.closureOf(e)))].sort() } catch { return [] }
+    }
+
+    /** the closure of every toolkit module this repo's own files import by
+     *  package name, found with one `git grep` over tracked and untracked
+     *  files. Empty in the repo that hosts the toolkit, whose own resolver
+     *  follows those imports into the code closure */
+    private libUsed(): string[] | string {
+        if (this.libUsedMemo != null) return this.libUsedMemo
+        const lib = this.toolchain.lib
+        if (lib == null || lib.name == null || !relative(this.root, lib.pkgRoot).startsWith('..')) return (this.libUsedMemo = [])
+        const p = Bun.spawnSync([gitBin(), 'grep', '-l', '--untracked', '-F', '-e', `${lib.name}/`, '--', '*.ts', '*.tsx', '*.mts', '*.cts', '*.js', '*.mjs'], { cwd: this.root, stdout: 'pipe', stderr: 'pipe' })
+        // exit 1 is "no match", anything above it is git failing
+        if (p.exitCode > 1) return (this.libUsedMemo = `git grep could not list the files importing ${lib.name}: ${p.stderr.toString().trim().split('\n')[0]}`)
+        const entries: string[] = []
+        for (const f of p.stdout.toString().split('\n').filter((l) => l !== '')) {
+            let text = ''
+            try { text = readFileSync(join(this.root, f), 'utf8') } catch { continue }
+            for (const spec of [...extractSpecs(text, { dynamic: true }), ...[...text.matchAll(/import\.meta\.resolve\(\s*['"]([^'"\n]+)['"]\s*\)/g)].map((m) => m[1]!)]) {
+                const abs = lib.fileOf(spec)
+                if (abs != null) entries.push(abs)
+            }
+        }
+        this.libUsedEntries = [...new Set(entries)].sort()
+        return (this.libUsedMemo = entries.length === 0 ? [] : [`libused=${lib.closureKey(this.libUsedEntries)}`])
+    }
+
+    private readonly namesMemo = new Map<string, string>()
+
+    /** every relative path under a dir, sorted, as one hash: what a task that
+     *  lists the dir without reading its files sees. `absent` for no dir */
+    private namesOf(rel: string): string {
+        let h = this.namesMemo.get(rel)
+        if (h == null) {
+            const abs = join(this.root, rel)
+            if (!existsSync(abs)) h = 'absent'
+            else {
+                const all: string[] = []
+                const walk = (d: string, prefix: string): void => {
+                    let entries: Dirent[] = []
+                    try { entries = readdirSync(d, { withFileTypes: true }) } catch { return }
+                    for (const e of entries) {
+                        const r = `${prefix}${e.name}`
+                        all.push(e.isDirectory() ? `${r}/` : r)
+                        if (e.isDirectory()) walk(join(d, e.name), `${r}/`)
+                    }
+                }
+                walk(abs, '')
+                h = createHash('sha256').update(all.sort().join('\n')).digest('hex').slice(0, 16)
+            }
+            this.namesMemo.set(rel, h)
+        }
+        return h
+    }
+
     /** an excluded install's identity, once per layer: several checks can name
      *  the same one, and hashing a `node_modules` twice in a run is pure cost. */
     private installOf(rel: string): string {

package/src/ledger/git-scan.ts

@@ -290,10 +290,13 @@ export function scanPaths(root: string, rawPaths: string[], prior?: ScanResult,
     // force-adding that one entry stages the node_modules inside it.
     const declared = new Set(wanted)
     const denied = ignored.filter(underNodeModules)
-    const forceAdd = ignored.filter((p) => declared.has(p) && !underNodeModules(p) && !denied.some((d) => d.startsWith(`${p}/`)))
+    // a declared path INSIDE a wholly ignored dir comes back as that dir, so
+    // it is named by the collapsed ancestor rather than by itself
+    const underIgnored = present.filter((p) => p !== WHOLE_TREE && !ignored.includes(p) && ignored.some((d) => p.startsWith(`${d}/`)))
+    const forceAdd = [...ignored, ...underIgnored].filter((p) => declared.has(p) && !underNodeModules(p) && !denied.some((d) => d.startsWith(`${p}/`)))
     // a path git itself calls ignored must NOT get the plain add: `git add --
     // <ignored>` exits 1 and takes the whole scan down with it
-    const ignoredSet = new Set(ignored)
+    const ignoredSet = new Set([...ignored, ...underIgnored])
     const plainAdds = toAdd.filter((p) => !ignoredSet.has(p))
     if (plainAdds.length > 0) gitOrThrow(root, ['add', '--', ...plainAdds], indexFile)
     // force-adds land in their OWN index, so a path one check named can never

package/src/ledger/ledger.ts

@@ -9,15 +9,17 @@
 //
 //  1. DEFAULT DENY. A check that declared nothing is never replayed and never
 //     stored. Forgetting to declare costs speed, never correctness, which is
-//     what makes the burn-down a to-do rather than a hazard.
+//     what makes the burn-down a to-do rather than a hazard. A task that
+//     declares nothing is keyed on the work tree instead (gate-checks.ts).
 //  2. POST RUN RE HASH. Results are BUFFERED while the gate runs and written
 //     only after the inputs are hashed a second time. A check whose inputs
 //     moved between its run and that second hash answered about a tree that no
 //     longer exists, so its result is discarded rather than stored under a key
 //     that describes the file as it was BEFORE the edit. Three agents in one
 //     checkout is the normal case here, so that window is not theoretical.
-//  3. THE GATE BINARY IS IN THE KEY. Editing shipkit moves every key in every
-//     repo (ledger/toolchain-identity.ts content-hashes the cli tree).
+//  3. THE GATE CODE IS IN THE KEY. Editing the shipkit code that runs a check
+//     or stores its answer moves that key in every repo, and an edit no answer
+//     depends on moves none (ledger/toolchain-identity.ts).
 //  4. RELEASE NEVER READS IT. `bump` and `release` pass read:false, not
 //     configurably. Every remaining hole is then capped at "an agent saw a
 //     stale local green", never "a stale green authorised a publish".
@@ -41,7 +43,7 @@
 import { join } from 'path'
 import type { RvtConfig } from '../ts/config.ts'
 import type { Line } from '../core/wire.ts'
-import { KeyLayer, type CheckDecl, type CheckKey } from './git-key.ts'
+import { classify, KeyLayer, type CheckDecl, type CheckKey } from './git-key.ts'
 import { shipkitRoot } from './db.ts'
 import { Answers, answersDb, FRESH_ENV, type AnswerRef, type AnswerWrite } from './store.ts'
 import type { Problem } from '../problems/problem-line.ts'
@@ -81,8 +83,20 @@ export type CheckOutcome = {
     measuredMs: number
 }
 
-/** what a READS answer is stored as: the verdict and what it read */
-type Stored = CheckOutcome & { reads?: ReadSet }
+/** what an answer is stored as: the verdict, what it read (READS), and the
+ *  key components it was stored under, so a later miss can say what moved */
+type Stored = CheckOutcome & { reads?: ReadSet; components?: string[] }
+
+/** why a check did not come from the cache this run, one per check that ran
+ *  or cannot be cached. The verdict prints them grouped, so a re-run is never
+ *  unexplained */
+export type RanWhy = {
+    /** `moved`: its key or a read changed. `first`: nothing stored for it.
+     *  `not kept`: it ran and its answer cannot be stored. `never`: no key can
+     *  describe it. `off`: this run reads no cache */
+    group: 'moved' | 'first' | 'not kept' | 'never' | 'off'
+    why: string
+}
 
 export type LedgerOpts = {
     /** the cli entry driving this run (toolchain identity needs it) */
@@ -128,6 +142,8 @@ export class Ledger {
     private misses = 0
     private unkeyed = 0
     private joined = 0
+    private readonly ranWhy = new Map<string, RanWhy>()
+    private fresh = false
 
     private constructor(
         readonly layer: KeyLayer,
@@ -144,7 +160,9 @@ export class Ledger {
         const layer = KeyLayer.open(root, decls, { cliEntry: o.cliEntry, cfg: o.cfg, scope: o.scope })
         const fresh = o.fresh === true || process.env[FRESH_ENV] != null
         const reading = (o.read ?? ledgerReadEnabled()) && !fresh
-        return new Ledger(layer, new Answers(answersDb(root)), join(shipkitRoot(root), 'inflight'), reading, o.now ?? Date.now)
+        const l = new Ledger(layer, new Answers(answersDb(root)), join(shipkitRoot(root), 'inflight'), reading, o.now ?? Date.now)
+        l.fresh = fresh
+        return l
     }
 
     /** the key for a check, computed once per run and remembered: `keyFor`
@@ -156,7 +174,7 @@ export class Ledger {
         return k
     }
 
-    private read(k: CheckKey): { payload: Stored } | null {
+    private read(k: CheckKey): { payload: Stored; at: number } | null {
         return this.answers.read<Stored>(NS, k.id, k.key!)
     }
 
@@ -168,16 +186,27 @@ export class Ledger {
 
     /** the stored answer for this check, or null when it must run. Counts what
      *  it was asked, so the run can report hit/miss/uncacheable honestly. */
-    replay(id: string): CheckOutcome | null {
+    /** drop the stored answer for a check's CURRENT key: the cache audit found
+     *  that a fresh run disagrees with it, and a fresh failure is never stored
+     *  in its place, so the stale green would keep being served */
+    forget(id: string): void {
+        const k = this.key(id)
+        if (k.key != null) this.answers.drop(NS, k.id, k.key)
+    }
+
+    replay(id: string): (CheckOutcome & { at: number }) | null {
         const k = this.key(id)
-        if (k.key == null) { this.unkeyed++; return null }
-        if (!this.reading) { this.misses++; return null }
+        if (k.key == null) { this.unkeyed++; this.ranWhy.set(id, { group: 'never', why: this.neverWhy(k) }); return null }
+        if (!this.reading) { this.misses++; this.ranWhy.set(id, { group: 'off', why: this.fresh ? '--fresh' : 'cache reads off' }); return null }
         const e = this.read(k)
-        if (e == null || !Array.isArray(e.payload.lines)) { this.misses++; return null }
-        if (k.population === 'reads' && (e.payload.reads == null || this.driftOf(e.payload.reads, () => this.view()) != null)) { this.misses++; return null }
+        if (e == null || !Array.isArray(e.payload.lines)) { this.misses++; this.ranWhy.set(id, this.movedWhy(k)); return null }
+        if (k.population === 'reads') {
+            const drift = e.payload.reads == null ? 'its read set was not stored' : this.driftOf(e.payload.reads, () => this.view())
+            if (drift != null) { this.misses++; this.ranWhy.set(id, { group: 'moved', why: drift }); return null }
+        }
         this.hits++
         this.replayed.push({ ns: NS, name: k.id, key: k.key })
-        return e.payload
+        return { ...e.payload, at: e.at }
     }
 
     /** the stored read set of a READS check, as the last store wrote it */
@@ -212,23 +241,70 @@ export class Ledger {
 
     /** a check that missed the store and then adopted another run's in
      *  flight answer: counted apart, so the header never claims it measured */
-    noteAttached(): void {
+    noteAttached(id: string): void {
         if (this.misses > 0) this.misses--
         this.joined++
+        this.ranWhy.delete(id)
+    }
+
+    /** the lines that say why every check that did not replay ran, one per
+     *  reason, so a re-run is never unexplained. A run that reads no cache
+     *  says so in its header instead. `quiet`: a check its author decided is
+     *  never cached, which nobody acts on (--verbose prints it) */
+    explain(): { line: Line; quiet: boolean }[] {
+        const order: RanWhy['group'][] = ['moved', 'first', 'not kept', 'never']
+        const groups = new Map<string, { w: RanWhy; ids: string[] }>()
+        for (const [id, w] of this.ranWhy) {
+            if (w.group === 'off') continue
+            const k = `${order.indexOf(w.group)}\0${w.why}`
+            const g = groups.get(k) ?? { w, ids: [] }
+            g.ids.push(id)
+            groups.set(k, g)
+        }
+        return [...groups.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([, { w, ids }]) => {
+            const who = listShort(ids, 5)
+            const text = w.group === 'never' ? `never cached: ${w.why} (${who})`
+                : w.group === 'not kept' ? `ran, not kept: ${w.why} (${who})`
+                : `ran: ${w.why} (${who})`
+            return { line: { kind: 'ok' as const, text }, quiet: w.group === 'never' && w.why.startsWith('never: ') }
+        })
     }
 
     /** buffer what a check just measured. Nothing reaches the store until
      *  commit() has re-hashed the inputs (rule 2). A READS check with no read
      *  set (it reached the world) is never stored (rule 7). */
-    record(id: string, outcome: CheckOutcome, reads?: ReadSet | null): void {
+    record(id: string, outcome: CheckOutcome, reads?: ReadSet | null, notKept?: string | null): void {
         const k = this.key(id)
         if (k.key == null) return
         if (k.population === 'reads') {
-            if (reads == null) return
-            this.buffered.set(id, { ...outcome, reads })
+            if (reads == null) { this.ranWhy.set(id, { group: 'not kept', why: notKept ?? 'it read outside what the cache can see' }); return }
+            this.buffered.set(id, { ...outcome, reads, components: k.components })
             return
         }
-        this.buffered.set(id, outcome)
+        this.buffered.set(id, { ...outcome, components: k.components })
+    }
+
+    /** why a check has no key: its own words when it decided `never`, the
+     *  key layer's otherwise */
+    private neverWhy(k: CheckKey): string {
+        const d = this.layer.decls.find((x) => x.id === k.id)
+        return d != null && classify(d).decided === true ? `never: ${d.never}` : k.reason ?? 'undeclared'
+    }
+
+    /** what moved since this check's last stored answer, by diffing the key
+     *  components it was stored under against this run's */
+    private movedWhy(k: CheckKey): RanWhy {
+        const prev = this.answers.latest<Stored>(NS, k.id)
+        if (prev == null) return { group: 'first', why: 'nothing stored for it yet' }
+        const before = prev.payload.components
+        if (before == null) return { group: 'moved', why: 'its key changed' }
+        const moved = movedParts(before, k.components)
+        return moved.length === 0 ? { group: 'first', why: 'nothing stored for this key yet' } : { group: 'moved', why: `${listShort(moved, 3)} changed` }
+    }
+
+    /** every check that did not come from the cache, and why */
+    get why(): ReadonlyMap<string, RanWhy> {
+        return this.ranWhy
     }
 
     /** re-hash, then write what still describes the tree on disk. */
@@ -295,6 +371,45 @@ export class Ledger {
     }
 }
 
+/** the key components that differ between two keys of one check, in words */
+export function movedParts(before: string[], after: string[]): string[] {
+    const split = (c: string): [string, string] => {
+        // several excluded paths share the name: each one is its own component
+        if (c.startsWith('excluded=')) return [c, '']
+        const i = c.indexOf('=')
+        return i < 0 ? [c, ''] : [c.slice(0, i), c.slice(i + 1)]
+    }
+    const a = new Map(before.map(split))
+    const b = new Map(after.map(split))
+    const names = [...new Set([...a.keys(), ...b.keys()])].filter((n) => a.get(n) !== b.get(n))
+    return [...new Set(names.map(componentWords))]
+}
+
+/** one key component's name as a reader says it */
+function componentWords(name: string): string {
+    if (name === 'v') return "shipkit's key format"
+    if (name === 'bun') return 'the bun version'
+    if (name === 'frozen') return 'frozen mode'
+    if (name === 'toolchain') return "shipkit's gate code"
+    if (name === 'install') return 'node_modules'
+    if (name === 'argv') return 'its command'
+    if (name === 'lib') return 'its shipkit code'
+    if (name === 'libused') return 'the shipkit modules this repo imports'
+    if (name === 'extra') return 'its settings'
+    if (name === 'node') return 'the node version'
+    if (name.startsWith('names:')) return `the names under ${name.slice(6)}`
+    if (name.startsWith('env:')) return `$${name.slice(4)}`
+    if (name === 'in:.') return 'the work tree'
+    if (name.startsWith('in:') || name.startsWith('code:')) return name.slice(name.indexOf(':') + 1)
+    if (name.startsWith('excluded')) return 'ignored files under its inputs'
+    return name
+}
+
+/** `a, b, c`, and `a, b, +3 more` past `max` */
+function listShort(items: string[], max: number): string {
+    return items.length <= max ? items.join(', ') : `${items.slice(0, max - 1).join(', ')}, +${items.length - max + 1} more`
+}
+
 /** where askers of one question meet while one of them measures it: a file
  *  name per (ns, id, key), hashed so any id is a safe name */
 export function inflightSlot(dir: string, ns: string, id: string, key: string): string {
@@ -305,6 +420,6 @@ export function inflightSlot(dir: string, ns: string, id: string, key: string):
 /** stamp a cached check's lines. The FIRST line carries the word, because
  *  that is the line an agent reads, and a cached answer that renders
  *  identically to one that ran is the one thing this design refuses to ship. */
-export function stampCached(lines: Line[], measuredMs: number): Line[] {
-    return lines.map((l, i) => (i === 0 ? { ...l, text: `${l.text} ${cachedStamp(measuredMs)}` } : l))
+export function stampCached(lines: Line[], measuredMs: number, at: number, now: number = Date.now()): Line[] {
+    return lines.map((l, i) => (i === 0 ? { ...l, text: `${l.text} ${cachedStamp(measuredMs, now - at)}` } : l))
 }

package/src/ledger/store.ts

@@ -164,6 +164,20 @@ export class Answers {
         })
     }
 
+    /** the newest answer of one check under ANY key: what a miss diffs its
+     *  own key against, to say what moved */
+    latest<T>(ns: string, name: string): Stored<T> | null {
+        const row = this.with(READ_BUSY_MS, (h) => h.query('select v, key, at, keeps, payload from answers where ns = ? and name = ? order by at desc limit 1').get(ns, name) as (Row & { key: string }) | null)
+        if (row == null) return null
+        try {
+            const payload = JSON.parse(row.payload) as T
+            if (Number(row.v) !== STORE_VERSION || typeof row.at !== 'number' || payload == null) return null
+            return { v: Number(row.v), key: row.key, at: row.at, keeps: row.keeps ?? undefined, payload }
+        } catch {
+            return null
+        }
+    }
+
     drop(ns: string, name: string, key: string): void {
         this.with(BUSY_MS, (h) => h.query('delete from answers where ns = ? and name = ? and key = ?').run(ns, name, key))
     }