rvlib-shipkit 1.1.1 → 1.3.0

package/src/home/via-hub.ts

@@ -0,0 +1,142 @@
+// `shipkit ci` and `shipkit check` on a machine with the home: the run happens
+// in the repo's hub, which runs in the home, and the terminal only streams its
+// lines and takes its exit code. One runner per repo, so a terminal gate, an
+// agent's gate and the hub's own share one queue and one problems pane. The
+// run itself is a detached gate job, so a daemon that restarts mid gate cuts
+// the stream and never the gate: the terminal follows the rest from the job's
+// folder and still exits with the gate's code.
+//
+// the one constraint a reader breaks first: a process the home itself started
+// (a check's child, a script calling shipkit) runs in process. Sending it back
+// would queue it behind the very run that spawned it.
+import { basename } from 'path'
+import { isLinkedWorktree } from '../core/git-root.ts'
+import { SLOT_HELD_ENV } from '../core/slots.ts'
+import { homeRepo } from '../serve/project-gutter.ts'
+import { daemonUp, daemonUrl, hubBase, WORKER_ENV } from './home-paths.ts'
+import { homeWanted, startHome } from './home-start.ts'
+import { RUN_EXIT, RUN_JOB } from './home-words.ts'
+import { followJob, type FollowOpts } from '../core/job-follow.ts'
+
+type Env = Record<string, string | undefined>
+const set = (v: string | undefined): boolean => v != null && v !== ''
+
+/** why this run stays in process, or null when it goes to the home */
+export function runsHere(root: string, env: Env = process.env, wanted: (env: Env) => boolean = homeWanted): string | null {
+    if (set(env[WORKER_ENV])) return 'inside the home'
+    if (set(env[SLOT_HELD_ENV])) return 'inside a check'
+    if (set(env.SHIPKIT_NO_HOME)) return 'SHIPKIT_NO_HOME'
+    if (set(env.CI)) return 'CI'
+    if (!wanted(env)) return 'no home on this machine'
+    if (isLinkedWorktree(root)) return 'a linked worktree'
+    return null
+}
+
+/** the run through the repo's hub in the home: its exit code, or null when it
+ *  runs here (a note says why when that is a surprise). A home that cannot be
+ *  reached is a loud failure, never a quiet run elsewhere */
+export async function runViaHub(root: string, kind: 'ci' | 'check', opts: Record<string, unknown>, io: RunIo): Promise<number | null> {
+    if (runsHere(root) != null) return null
+    const name = basename(homeRepo(root))
+    try {
+        if (!(await daemonUp())) await startHome()
+    } catch (e) {
+        io.printErr(`FAIL: ${(e as Error).message}. On this machine a gate runs in the home: SHIPKIT_NO_HOME=1 runs this one in process`)
+        return 1
+    }
+    // what the terminal knows and the hub does not: its tier and whether it is a terminal
+    const full = process.env.SHIPKIT_TEST_FULL === '1'
+    const sent = {
+        ...opts,
+        ...(full ? (kind === 'ci' ? { tests: opts.tests ?? 'full' } : { tier: opts.tier ?? 'full' }) : {}),
+        color: process.stdout.isTTY === true && process.env.NO_COLOR == null,
+    }
+    // the home waits for a hub still loading before it answers
+    const post = (): Promise<Response> => fetch(`${daemonUrl()}${hubBase(name)}/api/run`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ kind, opts: sent }) })
+    return followRun({ name, jobRoot: homeRepo(root), post, up: () => daemonUp(), io })
+}
+
+type RunIo = { print: (l: string) => void; printErr: (l: string) => void }
+
+/** how many times a run the daemon dropped before it started is sent again */
+const RESENDS = 3
+/** how long a terminal waits for a restarted daemon to answer again */
+const BACK_MS = 60_000
+
+/** one terminal run through the daemon: its lines, then its exit code. The
+ *  stream carries the gate job's id once it starts (RUN_JOB), so a stream the
+ *  daemon cuts (it restarted, or died) is not the end: the rest of the job is
+ *  followed from its folder, from the line the stream had reached. A cut
+ *  before the job started lost nothing but a place in the queue, and the run
+ *  is sent again. Null = not one of the daemon's repos, so it runs here */
+export async function followRun(o: {
+    name: string
+    jobRoot: string
+    post: () => Promise<Response>
+    up: () => Promise<boolean>
+    io: RunIo
+    sleep?: (ms: number) => Promise<void>
+    follow?: FollowOpts
+}): Promise<number | null> {
+    const { name, io } = o
+    const sleep = o.sleep ?? Bun.sleep
+    for (let sent = 0; ; sent++) {
+        let r: Response | null = null
+        try { r = await o.post() } catch { r = null }
+        if (r != null) {
+            // not one of the home's repos: nothing to send it to
+            if (r.status === 404) return null
+            if (r.status === 409) {
+                io.printErr(`note: the home could not load ${name}'s hub (${await r.text()}), so this gate runs in process`)
+                return null
+            }
+            if (!r.ok || r.body == null) {
+                io.printErr(`FAIL: the home refused ${name}'s run: ${r.status} ${await r.text()}`)
+                return 1
+            }
+            let code: number | null = null
+            let job: string | null = null
+            let jobLines = 0
+            let partial = ''
+            const put = (l: string): void => {
+                if (l.startsWith(RUN_EXIT)) code = Number(l.slice(RUN_EXIT.length))
+                else if (l.startsWith(RUN_JOB)) job = l.slice(RUN_JOB.length)
+                else {
+                    if (job != null) jobLines++
+                    io.print(l)
+                }
+            }
+            const decoder = new TextDecoder()
+            try {
+                for await (const chunk of r.body) {
+                    const lines = (partial + decoder.decode(chunk, { stream: true })).split('\n')
+                    partial = lines.pop() ?? ''
+                    for (const l of lines) put(l)
+                }
+                if (partial !== '') put(partial)
+            } catch { /* the daemon went away mid stream: what it sent is kept */ }
+            if (code != null) {
+                if (Number.isInteger(code)) return code
+                io.printErr(`FAIL: ${name}'s hub ended the run on a broken exit code`)
+                return 1
+            }
+            if (job != null) {
+                io.printErr(`note: the daemon went away mid gate, so the rest of gate job ${job} is read from its folder`)
+                return (await followJob(o.jobRoot, job, { line: io.print }, { ...o.follow, skipLines: jobLines })) ?? 1
+            }
+        }
+        if (sent >= RESENDS) {
+            io.printErr(`FAIL: ${name}'s hub closed the run before it started, ${RESENDS + 1} times: run it again`)
+            return 1
+        }
+        io.printErr(`note: the daemon went away before this gate started, so it is sent again once the daemon is back`)
+        const t0 = Date.now()
+        while (!(await o.up())) {
+            if (Date.now() - t0 > BACK_MS) {
+                io.printErr(`FAIL: the daemon did not come back within ${BACK_MS / 1000}s: shipkit pm status says why`)
+                return 1
+            }
+            await sleep(250)
+        }
+    }
+}

package/src/home/widget-text.ts

@@ -0,0 +1,119 @@
+// a widget's html as console lines, for `shipkit overview`: the html the
+// daemon rendered for the page, tags stripped, a table's cells aligned in
+// columns, a block element on its own line, and a `.wbar` meter (its width
+// in `--p:NN%`) drawn as `█████░░░░░`. It reads, never runs: no script, no
+// style, no layout, so a widget that draws only with css reads as its text.
+
+const BLOCK = new Set(['div', 'p', 'section', 'article', 'header', 'footer', 'ul', 'ol', 'li', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'pre', 'hr', 'dl', 'dt', 'dd', 'figure', 'figcaption', 'blockquote'])
+const SKIP = new Set(['script', 'style', 'template', 'svg'])
+const BAR_CELLS = 10
+
+const ENTITY: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', middot: '·', deg: '°', hellip: '…', times: '×' }
+
+export function decodeEntities(s: string): string {
+    return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
+        if (e[0] === '#') {
+            const n = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1))
+            return Number.isFinite(n) ? String.fromCodePoint(n) : m
+        }
+        return ENTITY[e.toLowerCase()] ?? m
+    })
+}
+
+/** a meter of BAR_CELLS cells, `p` in 0..100 */
+export function barText(p: number): string {
+    const on = Math.round((Math.min(100, Math.max(0, p)) / 100) * BAR_CELLS)
+    return '█'.repeat(on) + '░'.repeat(BAR_CELLS - on)
+}
+
+type Block = { kind: 'line'; text: string } | { kind: 'table'; rows: string[][] }
+
+const attr = (attrs: string, name: string): string | null => {
+    const m = new RegExp(`\\b${name}\\s*=\\s*("([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i').exec(attrs)
+    return m == null ? null : (m[2] ?? m[3] ?? m[4] ?? '')
+}
+
+const squash = (s: string): string => s.replace(/\s+/g, ' ').trim()
+
+/** a table's rows as aligned lines: each column as wide as its widest cell */
+export function alignRows(rows: string[][]): string[] {
+    const w: number[] = []
+    for (const r of rows) r.forEach((c, i) => { w[i] = Math.max(w[i] ?? 0, c.length) })
+    return rows.map((r) => r.map((c, i) => (i === r.length - 1 ? c : c.padEnd(w[i]!))).join('  ').trimEnd())
+}
+
+/** a widget's leading `<header>`: the words it puts on its card's title
+ *  line (state, age), apart from the body drawn under that line */
+export function splitWidgetHead(html: string): { head: string | null; body: string } {
+    const m = /^\s*<header\b[^>]*>([\s\S]*?)<\/header>/i.exec(html)
+    return m == null ? { head: null, body: html } : { head: m[1]!, body: html.slice(m[0].length) }
+}
+
+/** the html as lines, no empty one */
+export function widgetText(html: string): string[] {
+    const blocks: Block[] = []
+    let line = ''
+    let table: string[][] | null = null
+    let cell: string | null = null
+    let skip: string | null = null
+    const put = (s: string): void => {
+        if (cell != null) cell += s
+        else line += s
+    }
+    const flush = (): void => {
+        for (const part of line.split('\n')) if (squash(part) !== '') blocks.push({ kind: 'line', text: squash(part) })
+        line = ''
+    }
+    const endCell = (): void => {
+        if (cell != null && table != null) table.at(-1)?.push(squash(cell))
+        cell = null
+    }
+    const tokens = /<!--[\s\S]*?-->|<(\/?)([a-zA-Z][a-zA-Z0-9-]*)([^>]*)>|([^<]+)|</g
+    for (const m of html.matchAll(tokens)) {
+        const [whole, close, rawTag, attrs = '', text] = m
+        if (text != null || whole === '<') {
+            if (skip == null) put(decodeEntities(text ?? '<'))
+            continue
+        }
+        if (rawTag == null) continue
+        const tag = rawTag.toLowerCase()
+        if (skip != null) {
+            if (close === '/' && tag === skip) skip = null
+            continue
+        }
+        if (SKIP.has(tag) && close !== '/' && !attrs.trim().endsWith('/')) {
+            skip = tag
+            continue
+        }
+        if (tag === 'table') {
+            endCell()
+            if (close === '/') {
+                if (table != null) blocks.push({ kind: 'table', rows: table.filter((r) => r.some((c) => c !== '')) })
+                table = null
+            } else {
+                flush()
+                table = []
+            }
+        } else if (tag === 'tr' && table != null) {
+            endCell()
+            if (close !== '/') table.push([])
+        } else if ((tag === 'td' || tag === 'th') && table != null) {
+            endCell()
+            if (close !== '/') {
+                if (table.length === 0) table.push([])
+                cell = ''
+            }
+        } else if (tag === 'br') put(cell != null ? ' ' : '\n')
+        else if (BLOCK.has(tag)) {
+            if (cell != null) put(' ')
+            else flush()
+        } else if (close !== '/' && /\bwbar\b/.test(attr(attrs, 'class') ?? '')) {
+            const p = /--p\s*:\s*(-?\d+(?:\.\d+)?)%/.exec(attr(attrs, 'style') ?? '')
+            if (p != null) put(` ${barText(Number(p[1]))} `)
+        }
+    }
+    endCell()
+    if (table != null) blocks.push({ kind: 'table', rows: table })
+    flush()
+    return blocks.flatMap((b) => (b.kind === 'line' ? [b.text] : alignRows(b.rows)))
+}

package/src/home/widgets.ts

@@ -0,0 +1,102 @@
+// the widgets of every house repo (`widget({...})` in a `.shipkit.ts` file),
+// rendered by the daemon on each one's own clock and kept as html the page
+// reads at /api/home/widgets.json. The files are read again every
+// WIDGET_RESCAN_MS, a walk of every repo that costs a few tens of ms, and an
+// unchanged file is never imported twice.
+//
+// the one constraint a reader breaks first: a render runs in the daemon, so
+// it is bounded (WIDGET_RENDER_MS) and never awaited by the daemon's tick.
+import { scanGlobalUnits, type LoadedWidget, type RepoRef } from './global-units.ts'
+
+export const WIDGET_EVERY_MS = 5_000
+export const WIDGET_RENDER_MS = 5_000
+export const WIDGET_RESCAN_MS = 15_000
+
+export type WidgetView = {
+    repo: string
+    id: string
+    title: string
+    file: string
+    /** the last html it rendered, null before its first render */
+    html: string | null
+    /** why its last render failed, or null */
+    error: string | null
+    /** epoch ms of its last render */
+    at: number | null
+}
+
+type Slot = { w: LoadedWidget; view: WidgetView; due: number; busy: boolean }
+
+export type WidgetsDeps = {
+    now?: () => number
+    scan?: (repos: RepoRef[]) => Promise<{ widgets: LoadedWidget[]; errors: string[] }>
+}
+
+const key = (w: LoadedWidget): string => `${w.repo}/${w.def.id}`
+
+export class Widgets {
+    private slots = new Map<string, Slot>()
+    private scannedAt = -Infinity
+    private scanning = false
+    /** the unit files that would not load */
+    errors: string[] = []
+
+    constructor(private d: WidgetsDeps = {}) {}
+
+    private now(): number {
+        return (this.d.now ?? Date.now)()
+    }
+
+    /** one look: read the files again when due, render every widget due. Never
+     *  waits on a render */
+    async tick(repos: RepoRef[]): Promise<void> {
+        const now = this.now()
+        if (!this.scanning && now - this.scannedAt >= WIDGET_RESCAN_MS) {
+            this.scanning = true
+            try {
+                const u = await (this.d.scan ?? scanGlobalUnits)(repos)
+                this.scannedAt = now
+                this.errors = u.errors
+                const next = new Map<string, Slot>()
+                for (const w of u.widgets) {
+                    const old = this.slots.get(key(w))
+                    // a new module is a new render: it shows on the next tick
+                    next.set(key(w), old != null && old.w.def === w.def ? old : {
+                        w,
+                        due: now,
+                        busy: false,
+                        view: { repo: w.repo, id: w.def.id, title: w.def.title ?? w.def.id, file: w.file, html: old?.view.html ?? null, error: null, at: old?.view.at ?? null },
+                    })
+                }
+                this.slots = next
+            } finally {
+                this.scanning = false
+            }
+        }
+        for (const s of this.slots.values()) if (!s.busy && this.now() >= s.due) void this.render(s)
+    }
+
+    /** one render, bounded, its result or its error kept */
+    async render(s: Slot): Promise<void> {
+        s.busy = true
+        let timer: ReturnType<typeof setTimeout> | undefined
+        try {
+            const html = await Promise.race([
+                Promise.resolve().then(() => s.w.def.render()),
+                new Promise<never>((_, no) => { timer = setTimeout(() => no(new Error(`render took more than ${WIDGET_RENDER_MS / 1000}s`)), WIDGET_RENDER_MS) }),
+            ])
+            if (typeof html !== 'string') throw new Error(`render returned ${typeof html}, not an html string`)
+            s.view = { ...s.view, html, error: null, at: this.now() }
+        } catch (e) {
+            s.view = { ...s.view, error: (e as Error).message, at: this.now() }
+        } finally {
+            clearTimeout(timer)
+            s.due = this.now() + (s.w.def.everyMs ?? WIDGET_EVERY_MS)
+            s.busy = false
+        }
+    }
+
+    views(): WidgetView[] {
+        return [...this.slots.values()].map((s) => s.view)
+    }
+}

package/src/hooks/run-hooks.ts

@@ -43,7 +43,10 @@ export type HooksConfig = {
      *  hooks/leak-guard.ts */
     leakAllow?: LeakAllowEntry[]
     /** staged-path prefixes that can never enter git (local state like
-     *  '.comfy-ts/'). A gitignore line is advisory and editable. This gate is not. */
+     *  '.comfy-ts/'). A gitignore line is advisory and editable. This gate is not.
+     *  A repo declaring `visibility: 'private'` still commits
+     *  '.comfy-ts/drafts/' through a broader prefix. Public and undeclared
+     *  repos block it. */
     blockedPaths?: string[]
     /** words the commit-message language gate never counts as French
      *  (proper nouns that carry accents without being a language choice) */
@@ -191,6 +194,20 @@ function waivedLines(waived: LeakWaiver[]): string[] {
     )
 }
 
+const COMFY_DRAFTS = '.comfy-ts/drafts/'
+
+/** the staged files the `blocked-paths` gate refuses. A repo DECLARED private
+ *  (never by default) commits its comfy drafts through a
+ *  broader block: a draft names private loras, which a private tree may hold.
+ *  A prefix naming the drafts folder itself keeps them blocked. */
+export function blockedStaged(staged: string[], config: RepoConfig): string[] {
+    const prefixes = config.hooks?.blockedPaths ?? []
+    const open = draftsOpen(config)
+    return staged.filter((f) => prefixes.some((p) => f.startsWith(p) && !(open && f.startsWith(COMFY_DRAFTS) && !p.startsWith(COMFY_DRAFTS))))
+}
+
+const draftsOpen = (c: RepoConfig): boolean => c.visibility === 'private'
+
 /** the pre-commit gate: built-in leak guard, then the repo's own commands.
  *  A commit takes ~1s (build/shipkit.md): the gate times itself, and going
  *  over budget WARNs naming the fix, so drift back to slow commits is loud. */
@@ -226,7 +243,7 @@ export async function runPreCommit(root: string, config: RepoConfig): Promise<Ga
     if ((h.blockedPaths ?? []).length > 0) {
         const names = Bun.spawnSync([gitBin(), 'diff', '--cached', '--name-only'], { cwd: root })
         const staged = names.stdout.toString().split('\n').filter(Boolean)
-        const blocked = staged.filter((f) => h.blockedPaths!.some((p) => f.startsWith(p)))
+        const blocked = blockedStaged(staged, config)
         out.push(blocked.length === 0
             ? { id: 'blocked-paths', ok: true, lines: [`ok: blocked-paths clean (${h.blockedPaths!.join(', ')})`] }
             : {
@@ -327,7 +344,7 @@ export function listGates(config: RepoConfig, root = process.cwd()): string[] {
     const leakAllowState = waivers === 0 ? '' : `, ${(h.leakAllow ?? []).length} repo waiver(s)${globalAllowFile() == null ? '' : ' + ~/.shipkit/private/leak-allow.json'}`
     return [
         `pre-commit  leak-guard${leakState} — staged added lines + paths vs ${kw} + ~/.shipkit/private/banned-keywords.txt${leakAllowState}`,
-        ...((h.blockedPaths ?? []).length > 0 ? [`pre-commit  blocked-paths — nothing under ${h.blockedPaths!.join(', ')} enters git`] : []),
+        ...((h.blockedPaths ?? []).length > 0 ? [`pre-commit  blocked-paths — nothing under ${h.blockedPaths!.join(', ')} enters git${draftsOpen(config) && blockedStaged([COMFY_DRAFTS], config).length === 0 && h.blockedPaths!.some((p) => COMFY_DRAFTS.startsWith(p)) ? `, except ${COMFY_DRAFTS} (declared private)` : ''}`] : []),
         `pre-commit  root-identity${off.has('root-identity') ? ' (DISABLED by config)' : ''} — the repo's own name in package.json + repo.config.ts, and scratch files`,
         ...(h.preCommit ?? []).map((c) => `pre-commit  ${c.id} — ${c.cmd.join(' ')}`),
         `commit-msg  commit-message${off.has('commit-message') ? ' (DISABLED by config)' : ''} — dash punctuation + AI attribution + French${off.has('commit-language') ? ' (commit-language DISABLED by config)' : ''}`,

package/src/ledger/check-stats.ts

@@ -1,6 +1,6 @@
 // What a check costs, per check, from the two things this repo remembers: the
-// store (one file per key, greens only, `at` plus `measuredMs`) and the runs
-// run record beside it (every outcome, red included).
+// answers (one row per key, greens only, `at` plus `measuredMs`) and the run
+// record beside them (every outcome, red included).
 //
 // The store is HISTORY here and the db is the record. They overlap for as
 // long as both describe the same run, so a stored entry is counted only when
@@ -14,11 +14,9 @@
 // rules and tasks only, and a surface reading it alone shows every check as
 // never measured however often it runs. The record is the only source that
 // measures a check, and the store is only ever the history of the other two.
-import { readdirSync, readFileSync, statSync } from 'fs'
-import { join } from 'path'
 import type { RunRow, RunStatus } from './run-log.ts'
 import { readRuns, runsPath, runsStamp } from './run-log.ts'
-import { storeRoot } from './store.ts'
+import { answersDb, answersStamp, listAnswers } from './store.ts'
 
 /** one measurement read back off the store */
 export type StoreSample = { key: string; at: number; ms: number; status: RunStatus }
@@ -96,60 +94,36 @@ export function mergeCheckStats(o: { runs: RunRow[]; store: StoreSample[] }): Ma
     return out
 }
 
-/** the store, read back. A check's file name is the slug of its id
- *  (ledger/store.ts entryPath), so the caller's check list decides which
- *  prefixes are read: deriving the id back OUT of a slug cannot work, since
- *  both `:` and `/` slug to `_`. */
-export function scanCheckStore(store: string, checks: { id: string }[]): StoreSample[] {
-    const byPrefix = new Map<string, string>()
-    for (const u of checks) byPrefix.set(storeSlug(u.id), u.id)
-    const dir = join(store, 'checks')
+/** the answers, read back: the rows of the checks the caller lists, since a
+ *  row of a check that no longer exists is no sample of anything */
+export function scanCheckStore(db: string, checks: { id: string }[]): StoreSample[] {
+    const ids = new Set(checks.map((c) => c.id))
     const out: StoreSample[] = []
-    let files: string[] = []
-    try { files = readdirSync(dir) } catch { return out }
-    for (const f of files) {
-        if (!f.endsWith('.json')) continue
-        // `<slug>.<12 hex of the key>.json`
-        const key = byPrefix.get(f.replace(/\.[0-9a-f]{12}\.json$/, ''))
-        if (key == null) continue
+    for (const r of listAnswers(db, 'checks')) {
+        if (!ids.has(r.name)) continue
         try {
-            const e = JSON.parse(readFileSync(join(dir, f), 'utf8')) as { at?: number; payload?: { measuredMs?: number; status?: RunStatus } }
-            const ms = e.payload?.measuredMs
-            if (typeof e.at !== 'number' || typeof ms !== 'number') continue
-            out.push({ key, at: e.at, ms, status: e.payload?.status === 'warn' ? 'warn' : 'pass' })
-        } catch { /* a half written or foreign entry reads as no sample */ }
+            const p = JSON.parse(r.payload) as { measuredMs?: number; status?: RunStatus }
+            if (typeof p.measuredMs !== 'number') continue
+            out.push({ key: r.name, at: r.at, ms: p.measuredMs, status: p.status === 'warn' ? 'warn' : 'pass' })
+        } catch { /* a row that does not parse is no sample */ }
     }
     return out
 }
 
-/** ledger/store.ts entryPath's file name rule, which this module must agree
- *  with byte for byte or it reads nothing */
-const storeSlug = (s: string): string => s.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 60)
-
 type Cached = { stamp: string; stats: Map<string, CheckStat> }
 const cache = new Map<string, Cached>()
 
 /** the per check numbers for a repo, cached on what the two sources look like
- *  from the outside: the store dir's mtime, plus what the run record holds.
- *  The store is thousands of small files and the hub asks on a ten second
- *  poll. The record's own FILE cannot answer for it: in WAL mode a commit
- *  lands in the sidecar and the db's mtime does not move. */
+ *  from the outside: a count and newest stamp of each, since the hub asks on
+ *  a ten second poll. The db's own FILE cannot answer for it: in WAL mode a
+ *  commit lands in the sidecar and the db's mtime does not move. */
 export function checkStatsFor(root: string, checks: { id: string }[], o?: { fresh?: boolean }): Map<string, CheckStat> {
-    const store = storeRoot(root)
+    const db = answersDb(root)
     const record = runsPath(root)
-    const stamp = `${checks.length}:${stampOf(join(store, 'checks'))}:${runsStamp(record)}`
+    const stamp = `${checks.length}:${answersStamp(db, 'checks')}:${runsStamp(record)}`
     const hit = cache.get(root)
     if (o?.fresh !== true && hit != null && hit.stamp === stamp) return hit.stats
-    const stats = mergeCheckStats({ runs: readRuns(record), store: scanCheckStore(store, checks) })
+    const stats = mergeCheckStats({ runs: readRuns(record), store: scanCheckStore(db, checks) })
     cache.set(root, { stamp, stats })
     return stats
 }
-
-const stampOf = (p: string): string => {
-    try {
-        const s = statSync(p)
-        return `${s.mtimeMs}/${s.size}`
-    } catch {
-        return 'none'
-    }
-}

package/src/ledger/db.ts

@@ -7,25 +7,39 @@
 // during the rewrite.
 //
 // THREE PROPERTIES, and every one is load bearing:
-//  1. NEVER READ BY THE GATE. Nothing stored here can change what a check does
-//     or what it replays, so a corrupt, locked or missing db costs a run its
-//     HISTORY and never its verdict. Every failure path out of this module is
-//     a null the caller turns into one warn line.
-//  2. NOT THE LEDGER. The keyed store beside it (`ledger/store.ts`) is still
-//     one immutable json file per key, which is what a replay reads. This db
-//     is the reporting record: what every check cost on every run, red runs
-//     included, and what each check last printed.
+//  1. A FAILURE NEVER DECIDES A VERDICT. The run record tables are never read
+//     by the gate at all. The `answers` table is (ledger/store.ts), and every
+//     way it can fail (corrupt, locked, missing, a row that does not parse)
+//     reads as "no answer", so the check is MEASURED: never a replay, never a
+//     red. Every failure path out of this module is a null.
+//  2. TWO HALVES. `answers` holds what a replay reads, keyed by the question.
+//     Every other table is the reporting record: what every check cost on
+//     every run, red runs included, and what each check last printed.
 //  3. SEVERAL WRITERS AT ONCE is the normal case (three agents in one
 //     checkout). WAL, a busy timeout, and one immediate transaction per flush
 //     is what makes six gates land every row whole.
 //
-// WHERE. `<git-common-dir>/shipkit/shipkit.sqlite`, beside the store and
-// outside the work tree, so nothing it writes can dirty a diff or reach a
-// commit. Every linked worktree of the repo shares it, the same as the store.
+// WHERE. `<git-common-dir>/shipkit/shipkit.sqlite`, outside the work tree, so
+// nothing it writes can dirty a diff or reach a commit. Every linked worktree
+// of the repo shares it, which is safe for answers because no key component
+// is a path (ledger/toolchain-identity.ts).
 import { Database } from 'bun:sqlite'
 import { existsSync, mkdirSync } from 'fs'
 import { dirname, join } from 'path'
-import { shipkitRoot } from './store.ts'
+import { gitRoot } from '../core/git-root.ts'
+import { LAYOUT } from '../core/layout.ts'
+
+/** the one dir everything this repo remembers hangs off.
+ *
+ *  `--git-common-dir` rather than `--git-dir`: a linked worktree has its own
+ *  git dir but shares the common one, and that sharing is safe because every
+ *  key component is content or a version string. A tree that is not a git
+ *  repo at all still gets one, under `.shipkit/cache/`, because a scaffold in
+ *  progress must not crash on it. */
+export function shipkitRoot(root: string): string {
+    const dir = gitRoot(root)?.commonDir
+    return dir == null ? join(root, LAYOUT.cache) : join(dir, 'shipkit')
+}
 
 export const DB_FILE = 'shipkit.sqlite'
 
@@ -65,8 +79,12 @@ export const DB_FILE = 'shipkit.sqlite'
  *  again, so an older writer still inserts and every older row stays.
  *
  *  v9 adds `phase_ms` to `suites`: the file's wall time in the phase that ran
- *  it, which `ms` is not once an alone rerun judged it. Same shape again. */
-export const SCHEMA_VERSION = 9
+ *  it, which `ms` is not once an alone rerun judged it. Same shape again.
+ *
+ *  v10 adds `answers`, the remembered verdicts a replay reads (ledger/store.ts),
+ *  which were one json file per key before. A new table, so every older row
+ *  stays, and an older build that opens the file never touches it. */
+export const SCHEMA_VERSION = 10
 
 /** how long a writer waits on another process's transaction before it gives
  *  up and the run says so. Long enough for a whole gate's flush, short enough
@@ -158,6 +176,18 @@ create table if not exists suite_runs (
     cores integer default null
 );
 create index if not exists suite_runs_check on suite_runs (check_id, at);
+create table if not exists answers (
+    ns text not null,
+    name text not null,
+    key text not null,
+    v integer not null,
+    at integer not null,
+    keeps integer default null,
+    used integer not null,
+    payload text not null,
+    primary key (ns, name, key)
+);
+create index if not exists answers_used on answers (used);
 create table if not exists live (
     run_id text primary key,
     kind text not null,
@@ -205,7 +235,7 @@ export function openDb(path: string, o: { create?: boolean; busyMs?: number } =
  *  switch is not attempted at all), retried briefly, and never fatal: a db
  *  left on the rollback journal still writes every row correctly under the
  *  busy timeout, it only serialises readers against the writer. */
-function ensureWal(db: Database, busyMs: number): void {
+export function ensureWal(db: Database, busyMs: number): void {
     const mode = (): string => String((db.query('pragma journal_mode').get() as { journal_mode?: string } | null)?.journal_mode ?? '')
     const deadline = Date.now() + Math.min(busyMs, 2_000)
     for (;;) {
@@ -233,8 +263,8 @@ function migrate(db: Database): void {
             // the pre-v2 tables carry a `kind` column and keys of the form
             // `<kind>:<id>`, which no reader speaks any more. v2 to v3 only
             // adds tables, v3 to v4 renames one, v4 to v5 adds one and v5 to
-            // v6 and every step to v9 add columns (v8 a table too), so
-            // their rows stay
+            // v6 and every step to v9 add columns (v8 a table too), and v10
+            // adds a table, so their rows stay
             if (was < 2) db.exec('drop table if exists runs; drop table if exists unit_last;')
             else if (was < 4 && db.query("select 1 from sqlite_master where type = 'table' and name = 'unit_last'").get() != null) db.exec('alter table unit_last rename to check_last; alter table check_last rename column unit to id; drop index if exists runs_unit;')
             const suites = db.query("select name from pragma_table_info('suites')").all() as { name: string }[]
@@ -253,3 +283,25 @@ function migrate(db: Database): void {
         throw e
     }
 }
+
+/** give the file back its free pages once more than half of it is free and
+ *  that is worth it: a DELETE frees pages for reuse but never shrinks the
+ *  file. VACUUM holds the write lock while it runs: a gate that asks then
+ *  measures instead of waiting. The daemon's sweep and `check --fix stores-under-cap` (house repo)
+ *  call it. True when it ran */
+export function compactDb(path: string, o: { minFreeBytes?: number } = {}): boolean {
+    const opened = openDb(path, { create: false })
+    if (opened == null) return false
+    try {
+        const n = (q: string): number => Number(Object.values(opened.db.query(q).get() as Record<string, number>)[0] ?? 0)
+        const pages = n('pragma page_count')
+        const free = n('pragma freelist_count')
+        if (free * 2 < pages || free * n('pragma page_size') < (o.minFreeBytes ?? 8 * 1024 * 1024)) return false
+        opened.db.exec('vacuum')
+        return true
+    } catch {
+        return false
+    } finally {
+        try { opened.db.close() } catch { /* already gone */ }
+    }
+}