rvlib-shipkit 1.1.0 → 1.2.0

package/src/ledger/ledger.ts

@@ -2,7 +2,7 @@
 // stored under the identity of the question it answered, replayed only when
 // that identity is bit-for-bit the same question again.
 //
-// ledger/git-key.ts owns the identity, ledger/store.ts owns the file. This
+// ledger/git-key.ts owns the identity, ledger/store.ts owns the table. This
 // module owns the POLICY, and the policy is where a cache turns into a false
 // green, so every rule below exists because a specific failure was reproduced
 // during the design pass:
@@ -38,11 +38,12 @@
 //     on disk. The same set is re-checked at commit, so an edit landing
 //     between the read and the store throws the answer away (rule 2). A body
 //     that reached the world (exec, fetch, tool) stores nothing.
-import { dirname, join } from 'path'
+import { join } from 'path'
 import type { RvtConfig } from '../ts/config.ts'
 import type { Line } from '../core/wire.ts'
 import { KeyLayer, type CheckDecl, type CheckKey } from './git-key.ts'
-import { entryPath, FRESH_ENV, KEEP_MS, readStored, storeRoot, sweepStore, writeStored } from './store.ts'
+import { shipkitRoot } from './db.ts'
+import { Answers, answersDb, FRESH_ENV, type AnswerRef, type AnswerWrite } from './store.ts'
 import type { Problem } from '../problems/problem-line.ts'
 import { cachedStamp, outcomeCounts } from '../core/outcome-words.ts'
 import { readSetDrift, type ReadSet } from '../core/read-set.ts'
@@ -119,6 +120,8 @@ export function ledgerReadEnabled(): boolean {
 
 export class Ledger {
     private readonly buffered = new Map<string, Stored>()
+    /** answers replayed this run, stamped as used in the commit's one write */
+    private readonly replayed: AnswerRef[] = []
     private viewMemo: RepoView | null = null
     private readonly keys = new Map<string, CheckKey>()
     private hits = 0
@@ -128,7 +131,10 @@ export class Ledger {
 
     private constructor(
         readonly layer: KeyLayer,
-        private readonly store: string,
+        /** the repo's answers, one handle for the run, closed by commit() */
+        private readonly answers: Answers,
+        /** the attach slots dir (core/attach.ts) */
+        private readonly inflight: string,
         /** replays are allowed */
         readonly reading: boolean,
         private readonly now: () => number,
@@ -138,7 +144,7 @@ export class Ledger {
         const layer = KeyLayer.open(root, decls, { cliEntry: o.cliEntry, cfg: o.cfg, scope: o.scope })
         const fresh = o.fresh === true || process.env[FRESH_ENV] != null
         const reading = (o.read ?? ledgerReadEnabled()) && !fresh
-        return new Ledger(layer, storeRoot(root), reading, o.now ?? Date.now)
+        return new Ledger(layer, new Answers(answersDb(root)), join(shipkitRoot(root), 'inflight'), reading, o.now ?? Date.now)
     }
 
     /** the key for a check, computed once per run and remembered: `keyFor`
@@ -150,8 +156,8 @@ export class Ledger {
         return k
     }
 
-    private path(k: CheckKey): string {
-        return entryPath(this.store, NS, k.id, k.key!)
+    private read(k: CheckKey): { payload: Stored } | null {
+        return this.answers.read<Stored>(NS, k.id, k.key!)
     }
 
     /** the file list this run lists and validates against, taken once */
@@ -166,10 +172,11 @@ export class Ledger {
         const k = this.key(id)
         if (k.key == null) { this.unkeyed++; return null }
         if (!this.reading) { this.misses++; return null }
-        const e = readStored<Stored>(this.path(k), k.key)
+        const e = this.read(k)
         if (e == null || !Array.isArray(e.payload.lines)) { this.misses++; return null }
         if (k.population === 'reads' && (e.payload.reads == null || this.driftOf(e.payload.reads, () => this.view()) != null)) { this.misses++; return null }
         this.hits++
+        this.replayed.push({ ns: NS, name: k.id, key: k.key })
         return e.payload
     }
 
@@ -177,7 +184,7 @@ export class Ledger {
     storedReads(id: string): ReadSet | null {
         const k = this.key(id)
         if (k.key == null || k.population !== 'reads') return null
-        return readStored<Stored>(this.path(k), k.key)?.payload.reads ?? null
+        return this.read(k)?.payload.reads ?? null
     }
 
     /** the view is asked for only when a glob needs it: a read set with no
@@ -200,7 +207,7 @@ export class Ledger {
         // a READS key names the question and not the tree, so two runs over
         // different trees share it: only one over this tree may answer
         if (k.population === 'reads') return null
-        return k.key == null ? null : entryPath(join(dirname(this.store), 'inflight'), NS, k.id, k.key).replace(/\.json$/, '')
+        return k.key == null ? null : inflightSlot(this.inflight, NS, k.id, k.key)
     }
 
     /** a check that missed the store and then adopted another run's in
@@ -227,13 +234,21 @@ export class Ledger {
     /** re-hash, then write what still describes the tree on disk. */
     commit(): CommitReport {
         const empty: CommitReport = { stored: [], discarded: [], drifted: [], error: null, ms: 0 }
-        if (this.buffered.size === 0) return empty
+        if (this.buffered.size === 0) {
+            this.answers.write([], { touched: this.replayed.splice(0), now: this.now() })
+            this.answers.close()
+            return empty
+        }
         const started = performance.now()
         const d = this.layer.rescan()
-        if (d.error != null) return { ...empty, discarded: [...this.buffered.keys()].sort(), error: d.error, ms: performance.now() - started }
+        if (d.error != null) {
+            this.answers.close()
+            return { ...empty, discarded: [...this.buffered.keys()].sort(), error: d.error, ms: performance.now() - started }
+        }
         const touched = this.layer.unitsTouchedBy(d.drifted)
         const stored: string[] = []
         const discarded: string[] = []
+        const rows: AnswerWrite[] = []
         const drifted = [...d.drifted]
         // a fresh list, never this run's: the question is whether the tree
         // still is what the body read
@@ -246,10 +261,14 @@ export class Ledger {
                 if (moved != null) { discarded.push(id); drifted.push(moved); continue }
             }
             const k = this.key(id)
-            writeStored(this.path(k), k.key!, outcome, this.now())
+            rows.push({ ns: NS, name: k.id, key: k.key!, payload: outcome, at: this.now() })
             stored.push(id)
         }
-        sweepStore(this.store, NS, this.now(), KEEP_MS)
+        // one transaction for every answer and every replay stamp. A write
+        // that fails stores nothing, which costs the next run a measurement
+        if (!this.answers.write(rows, { touched: this.replayed.splice(0), now: this.now() })) stored.length = 0
+        this.answers.sweep(this.now())
+        this.answers.close()
         return { stored: stored.sort(), discarded: discarded.sort(), drifted, error: null, ms: performance.now() - started }
     }
 
@@ -276,6 +295,13 @@ export class Ledger {
     }
 }
 
+/** where askers of one question meet while one of them measures it: a file
+ *  name per (ns, id, key), hashed so any id is a safe name */
+export function inflightSlot(dir: string, ns: string, id: string, key: string): string {
+    const h = new Bun.CryptoHasher('sha256').update(`${ns}\0${id}\0${key}`).digest('hex').slice(0, 12)
+    return join(dir, ns, `${id.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 60)}.${h}`)
+}
+
 /** stamp a cached check's lines. The FIRST line carries the word, because
  *  that is the line an agent reads, and a cached answer that renders
  *  identically to one that ran is the one thing this design refuses to ship. */

package/src/ledger/run-log.ts

@@ -23,10 +23,16 @@ import { existsSync } from 'fs'
 import type { Line } from '../core/wire.ts'
 import { BUSY_MS, dbPath, INSERT_RUN, openDb, UPSERT_INVOCATION, UPSERT_LAST, UPSERT_LIVE } from './db.ts'
 import { LIVE_BEAT_MS, liveRowDead, type LiveRow, type LiveState } from './live-runs.ts'
-import { KEEP_MS } from './store.ts'
 
 export type RunStatus = 'pass' | 'warn' | 'fail' | 'skip'
 
+/** how long a row of the record survives */
+export const KEEP_MS = 30 * 24 * 60 * 60 * 1000
+
+/** the most rows each record table keeps, newest first. A busy repo writes
+ *  tens of thousands of run rows a day, so age alone is no cap */
+export const MAX_ROWS = 200_000
+
 /** one check, one run. `ms` is what it MEASURED: a replay measured nothing and
  *  says 0, so the two can never be averaged together by accident. */
 export type RunRow = {
@@ -48,9 +54,7 @@ export const runsPath = dbPath
 
 /** what wrote a run: the command (`ci` answers the gate's question, `check`
  *  the health pass's), who started it (a terminal, the hub's buttons, the
- *  hub's daily timer), and whether it was narrowed to named checks. A
- *  narrowed health pass is not a health pass: the scheduler reads the last
- *  WHOLE one. */
+ *  plan), and whether it was narrowed to named checks. */
 export type InvocationKind = 'ci' | 'check'
 export type InvokedBy = 'cli' | 'hub' | 'schedule'
 export type Invocation = { kind: InvocationKind; label: string; by: InvokedBy; narrowed: boolean }
@@ -228,12 +232,38 @@ const unwritable = (path: string): Line[] => [
 
 /** the age sweep, over every table that grows per run. A live row nobody
  *  writes anymore is a run lost mid way, and it goes here too */
-const sweepAll = (db: Database, now: number, cutoff: number): void => {
+const sweepAll = (db: Database, now: number, cutoff: number, max?: number): void => {
     sweepDeadLive(db, now)
     db.query('delete from runs where at < ?').run(cutoff)
     db.query('delete from invocations where ended < ?').run(cutoff)
     db.query('delete from suites where at < ?').run(cutoff)
     db.query('delete from suite_runs where at < ?').run(cutoff)
+    if (max == null) return
+    for (const [table, col] of RECORD_TABLES) db.query(`delete from ${table} where rowid in (select rowid from ${table} order by ${col} desc limit -1 offset ?)`).run(max)
+}
+
+/** every record table and the column its age is read from */
+const RECORD_TABLES = [['runs', 'at'], ['invocations', 'ended'], ['suites', 'at'], ['suite_runs', 'at']] as const
+
+/** the biggest record table's row count and the oldest row, null when the db
+ *  cannot be read */
+export function recordUse(path: string): { count: number; oldest: number | null } | null {
+    const opened = openDb(path, { create: false })
+    if (opened == null) return null
+    try {
+        let count = 0
+        let oldest: number | null = null
+        for (const [table, col] of RECORD_TABLES) {
+            const r = opened.db.query(`select count(*) as n, min(${col}) as o from ${table}`).get() as { n: number; o: number | null }
+            count = Math.max(count, Number(r.n))
+            if (r.o != null) oldest = oldest == null ? r.o : Math.min(oldest, r.o)
+        }
+        return { count, oldest }
+    } catch {
+        return null
+    } finally {
+        try { opened.db.close() } catch { /* already gone */ }
+    }
 }
 
 /** every invocation on record that ended at or after `since`, oldest
@@ -324,31 +354,36 @@ export function readLastStatusReadonly(path: string): { id: string; status: stri
     }
 }
 
-/** the rows that finished at or after `since`, oldest first: the recent
- *  window a live chart draws, without reading a fortnight of history */
-export function readRunsSince(path: string, since: number): RunRow[] {
+/** every check's last result and how many runs in a row ended on that same
+ *  status: what the planner hands a check's `every` lambda. `{}` when the
+ *  record cannot be read, so a plan look never crashes on it */
+export function readLastResults(path: string): Record<string, { status: RunStatus; at: number; streak: number }> {
     const opened = openDb(path, { create: false })
-    if (opened == null) return []
+    if (opened == null) return {}
+    const out: Record<string, { status: RunStatus; at: number; streak: number }> = {}
     try {
-        const rows = opened.db.query(`${RUN_ROWS} where r.at >= ? order by r.at asc, r.rowid asc`).all(since) as RawRow[]
-        return rows.map(runRow)
+        const rows = opened.db.query(`select l.id as id, l.status as status, l.at as at,
+            (select count(*) from runs r where r.id = l.id and r.at > coalesce((select max(r2.at) from runs r2 where r2.id = l.id and r2.status != l.status), 0)) as streak
+            from runs l where l.rowid = (select r3.rowid from runs r3 where r3.id = l.id order by r3.at desc, r3.rowid desc limit 1)`).all() as { id: string; status: string; at: number; streak: number }[]
+        for (const r of rows) out[r.id] = { status: r.status as RunStatus, at: Number(r.at), streak: Math.max(1, Number(r.streak)) }
     } catch {
-        return []
+        return {}
     } finally {
         try { opened.db.close() } catch { /* already gone */ }
     }
+    return out
 }
 
-/** when the last WHOLE health pass started, whoever ran it. null = none on
- *  record, which is what makes a fresh hub run one at once. */
-export function lastHealthPassAt(path: string): number | null {
+/** the rows that finished at or after `since`, oldest first: the recent
+ *  window a live chart draws, without reading a fortnight of history */
+export function readRunsSince(path: string, since: number): RunRow[] {
     const opened = openDb(path, { create: false })
-    if (opened == null) return null
+    if (opened == null) return []
     try {
-        const r = opened.db.query("select max(started) as at from invocations where kind = 'check' and narrowed = 0").get() as { at: number | null } | null
-        return r?.at == null ? null : Number(r.at)
+        const rows = opened.db.query(`${RUN_ROWS} where r.at >= ? order by r.at asc, r.rowid asc`).all(since) as RawRow[]
+        return rows.map(runRow)
     } catch {
-        return null
+        return []
     } finally {
         try { opened.db.close() } catch { /* already gone */ }
     }
@@ -417,13 +452,13 @@ export function runsStamp(path: string): string {
     }
 }
 
-/** age based housekeeping, the same window the store keeps. A DELETE, so a row
- *  written while it runs is kept: the file rewrite this replaced lost them. */
-export function sweepRuns(path: string, now: number, keepMs = KEEP_MS): void {
+/** the record's cap, enforced: by age, then by count per table. A DELETE, so
+ *  a row written while it runs is kept */
+export function sweepRuns(path: string, now: number, keepMs = KEEP_MS, max = MAX_ROWS): void {
     const opened = openDb(path, { create: false })
     if (opened == null) return
     try {
-        sweepAll(opened.db, now, now - keepMs)
+        sweepAll(opened.db, now, now - keepMs, max)
     } catch {
         // housekeeping never fails a run
     } finally {