rvlib-shipkit 1.1.0 → 1.2.0

package/src/ledger/store.ts

@@ -1,52 +1,34 @@
-// ONE STORE for every ANSWER this toolkit remembers between runs, in one
-// shape, under one root, busted by one flag. What a run COST lives beside it
-// in `ledger/db.ts`, which nothing about a verdict ever reads.
+// ONE STORE for every ANSWER this toolkit remembers between runs: the
+// `answers` table of the repo's shipkit.sqlite (ledger/db.ts owns the file).
+// What differs between populations (the gate's checks, the world probes) is
+// the namespace and the KEY, never the storage.
 //
-// There used to be two: `tmp/.shipkit-cache` for the check orchestrator and
-// `tmp/.check-cache` for the page checks, each with its own bust env var, each
-// per worktree. Two stores means two staleness windows over one question, and
-// the stamps in this repo's own history went BACKWARDS between checkouts
-// because of it. This module is the single mechanism; what differs between
-// populations is the KEY, never the storage.
+// THE RULE THAT KEEPS A VERDICT HONEST: a db error is "no answer". Corrupt,
+// locked past a short wait, missing, a row whose envelope does not match: the
+// check is MEASURED. Never a replay of something unreadable, and never a red
+// because the store failed. A write that fails is a slower next run, nothing
+// more.
 //
-// THE SHAPE, unchanged from the check cache slice 2 landed and proved:
-// one immutable file per key, named by the key, written tmp + rename. No
-// lock, no merge, no read-modify-write, and no prune of another runner's
-// entries. Three agents in one checkout is the normal case, so the only safe
-// aggregate is no aggregate: a run with a different key writes a DIFFERENT
-// file, a run with the same key replaces its own whole answer or none of it,
-// and housekeeping sweeps by AGE, which needs no knowledge of what the
-// current run knows about.
-//
-// WHERE. `<git-common-dir>/shipkit/store/`, which is SHARED by every linked
-// worktree of the repo. That is safe only because no key component is a path:
-// the toolchain identity content-hashes the gate binary and every installed
-// package, so two worktrees carrying different installs produce different
-// keys and simply miss each other (ledger/toolchain-identity.ts owns the
-// measurement). It is also outside the work tree, so nothing here can dirty a
-// diff or reach a commit.
-//
-// The store dir holds keyed entries and NOTHING else: every file under it is
-// one whole json document, so a reader can walk it and parse all of it. The
-// reporting record beside it (the run db) lives one level up, in
-// `shipkitRoot`, because it is not keyed and is not a json document.
-import { mkdirSync, readdirSync, readFileSync, rmSync, statSync } from 'fs'
-import { dirname, join } from 'path'
-import { gitRoot } from '../core/git-root.ts'
-import { atomicWrite } from '../core/locked-write.ts'
-import { LAYOUT } from '../core/layout.ts'
-
-/** the storage FORMAT. Bumped when the envelope changes, so an old file can
- *  never be handed to a new reader. The KEY carries everything about the
- *  question; this is only about the file around it. */
+// A row is keyed by (ns, name, key) and replaced whole when the same question
+// is answered again, so a run with a different key writes a DIFFERENT row and
+// nothing is merged or read modified written. `used` is the last write or
+// replay, which is what the sweep prunes on: an answer nobody replayed for
+// KEEP_MS is a question nobody asks any more.
+import type { Database } from 'bun:sqlite'
+import { readdirSync, readFileSync, rmSync, statSync } from 'fs'
+import { join } from 'path'
+import { BUSY_MS, dbPath, openDb, shipkitRoot } from './db.ts'
+
+/** the envelope FORMAT, stored per row. Bumped when the envelope changes, so
+ *  an old row can never be handed to a new reader. The KEY carries everything
+ *  about the question. */
 export const STORE_VERSION = 1
 
 /** the ONE env var that busts every remembered answer, content or TTL. */
 export const FRESH_ENV = 'SHIPKIT_FRESH'
 
-/** one remembered answer. `key` is stored as well as hashed into the file
- *  name, and re-checked on read: a name collision must miss, never serve a
- *  neighbour's answer. */
+/** one remembered answer. `key` is re-checked on read, so a row can only ever
+ *  answer the exact question it was stored for. */
 export type Stored<T> = {
     v: number
     key: string
@@ -60,79 +42,222 @@ export type Stored<T> = {
     payload: T
 }
 
-/** how long a superseded entry survives. Far past the longest TTL any check
- *  declares, so a live entry is never swept out from under a replay. */
-export const KEEP_MS = 30 * 24 * 60 * 60 * 1000
-
-/** the one dir everything this repo remembers hangs off.
- *
- *  `--git-common-dir` rather than `--git-dir`: a linked worktree has its own
- *  git dir but shares the common one, and slice 3 measured that sharing is
- *  SAFE here because every key component is content or a version string.
- *  A tree that is not a git repo at all still gets one, under `.shipkit/cache/`,
- *  because a scaffold in progress must not crash on it. */
-export function shipkitRoot(root: string): string {
-    const dir = gitRoot(root)?.commonDir
-    return dir == null ? join(root, LAYOUT.cache) : join(dir, 'shipkit')
-}
+/** how long an answer nobody wrote or replayed survives. Far past the longest
+ *  TTL any check declares, so a live answer is never swept out from under a
+ *  replay. */
+export const KEEP_MS = 14 * 24 * 60 * 60 * 1000
 
-/** where this repo's remembered ANSWERS live: keyed entries only, one whole
- *  json document per file, nothing else at any depth. */
-export const storeRoot = (root: string): string => join(shipkitRoot(root), 'store')
+/** the most rows one repo keeps. A gate asks about 70 questions, so this is
+ *  dozens of distinct trees; past it the least recently used go first. */
+export const MAX_ANSWERS = 5_000
 
-const slug = (s: string): string => s.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 60)
+/** the import takes json answers this young only: every edit of shipkit moves
+ *  every key, so an older answer is one no gate asks for again */
+export const IMPORT_MAX_AGE_MS = 24 * 60 * 60 * 1000
 
-/** where ONE answer lives. The key is in the file NAME, so a run whose key
- *  moved writes a new file instead of rewriting the old one, and two runners
- *  with different keys never collide. `ns` keeps populations apart so a sweep
- *  or a manual `rm` can be scoped. */
-export function entryPath(store: string, ns: string, name: string, key: string): string {
-    const h = new Bun.CryptoHasher('sha256').update(`${ns}\0${name}\0${key}`).digest('hex').slice(0, 12)
-    return join(store, slug(ns), `${slug(name)}.${h}.json`)
-}
+/** a read waits this long on a writer, then measures. A replay that waits
+ *  longer than the check would take is no replay. */
+export const READ_BUSY_MS = 250
 
-/** a missing or half-written file reads as "no answer", which is the safe
- *  side: the check runs. tmp + rename means half-written cannot happen from a
- *  shipkit write, so this only ever catches a truncated file from something
- *  else, and treats it the same way. */
-export function readStored<T>(path: string, key: string): Stored<T> | null {
-    try {
-        const e = JSON.parse(readFileSync(path, 'utf8')) as Partial<Stored<T>>
-        if (e.v !== STORE_VERSION || e.key !== key || typeof e.at !== 'number' || e.payload == null) return null
-        return e as Stored<T>
-    } catch { return null }
+/** where this repo's answers live: the one db file */
+export const answersDb = (root: string): string => dbPath(root)
+
+/** the one dir the pre-db store used: one json file per key. Read once by
+ *  importLegacyStore, then deleted. */
+export const legacyStoreDir = (root: string): string => join(shipkitRoot(root), 'store')
+
+/** one session over a repo's answers: ONE handle, opened on first use and
+ *  closed by the owner (a gate's ledger, one world probe). Never shared across
+ *  sessions: a handle outliving its run can serve pages of a file something
+ *  replaced underneath it. A handle that fails once is dropped, and the rest
+ *  of the session measures. */
+export class Answers {
+    private h: Database | null | undefined
+
+    constructor(readonly db: string) {}
+
+    private handle(busyMs: number): Database | null {
+        if (this.h === undefined) this.h = openDb(this.db, { busyMs })?.db ?? null
+        if (this.h == null) return null
+        try {
+            this.h.exec(`pragma busy_timeout = ${Math.max(0, Math.round(busyMs))}`)
+            return this.h
+        } catch {
+            this.close()
+            return null
+        }
+    }
+
+    private with<R>(busyMs: number, body: (h: Database) => R): R | null {
+        const h = this.handle(busyMs)
+        if (h == null) return null
+        try {
+            return body(h)
+        } catch {
+            this.close()
+            return null
+        }
+    }
+
+    close(): void {
+        try { this.h?.close() } catch { /* already gone */ }
+        this.h = null
+    }
+
+    /** the stored answer, or null when the check must run */
+    read<T>(ns: string, name: string, key: string): Stored<T> | null {
+        const row = this.with(READ_BUSY_MS, (h) => h.query('select v, at, keeps, payload from answers where ns = ? and name = ? and key = ?').get(ns, name, key) as Row | null)
+        if (row == null) return null
+        try {
+            const payload = JSON.parse(row.payload) as T
+            if (Number(row.v) !== STORE_VERSION || typeof row.at !== 'number' || payload == null) return null
+            return { v: Number(row.v), key, at: row.at, keeps: row.keeps ?? undefined, payload }
+        } catch {
+            return null
+        }
+    }
+
+    /** write answers and stamp replays, in ONE transaction. False when it
+     *  could not: the answers were already printed, and the next run
+     *  measures again. */
+    write(rows: AnswerWrite[], o: { touched?: AnswerRef[]; now?: number; keepNewer?: boolean } = {}): boolean {
+        if (rows.length === 0 && (o.touched ?? []).length === 0) return true
+        const now = o.now ?? Date.now()
+        return (
+            this.with(BUSY_MS, (h) => {
+                // the import never overwrites a row a gate already wrote
+                const put = h.query(`insert or ${o.keepNewer === true ? 'ignore' : 'replace'} into answers (ns, name, key, v, at, keeps, used, payload) values (?, ?, ?, ?, ?, ?, ?, ?)`)
+                const use = h.query('update answers set used = max(used, ?) where ns = ? and name = ? and key = ?')
+                h.transaction(() => {
+                    for (const r of rows) put.run(r.ns, r.name, r.key, STORE_VERSION, r.at, r.keeps ?? null, Math.max(r.at, now), JSON.stringify(r.payload))
+                    for (const t of o.touched ?? []) use.run(now, t.ns, t.name, t.key)
+                }).immediate()
+                return true
+            }) ?? false
+        )
+    }
+
+    drop(ns: string, name: string, key: string): void {
+        this.with(BUSY_MS, (h) => h.query('delete from answers where ns = ? and name = ? and key = ?').run(ns, name, key))
+    }
+
+    /** every answer of one population: what the hub's check stats read */
+    list(ns: string): { name: string; at: number; payload: string }[] {
+        return this.with(READ_BUSY_MS, (h) => h.query('select name, at, payload from answers where ns = ?').all(ns) as { name: string; at: number; payload: string }[]) ?? []
+    }
+
+    /** a cheap fingerprint of one population, so a reader can cache on it */
+    stamp(ns: string): string {
+        const r = this.with(READ_BUSY_MS, (h) => h.query('select count(*) as n, max(at) as last from answers where ns = ?').get(ns) as { n: number; last: number | null } | null)
+        return r == null ? 'none' : `${r.n}/${r.last ?? 0}`
+    }
+
+    use(): AnswersUse | null {
+        const r = this.with(READ_BUSY_MS, (h) => h.query('select count(*) as n, min(used) as oldest from answers').get() as { n: number; oldest: number | null } | null)
+        return r == null ? null : { count: Number(r.n), oldestUsed: r.oldest }
+    }
+
+    /** the cap, enforced: rows not written nor replayed for `keepMs` go, then
+     *  the least recently used past `max`. A DELETE, so a row written while it
+     *  runs is kept. How many went, null when the db could not be opened. */
+    sweep(now: number, o: { keepMs?: number; max?: number } = {}): number | null {
+        return this.with(BUSY_MS, (h) => {
+            let n = h.query('delete from answers where used < ?').run(now - (o.keepMs ?? KEEP_MS)).changes
+            n += h.query('delete from answers where rowid in (select rowid from answers order by used desc limit -1 offset ?)').run(o.max ?? MAX_ANSWERS).changes
+            return n
+        })
+    }
 }
 
-export function writeStored<T>(path: string, key: string, payload: T, at: number, keeps?: number): void {
+type Row = { v: number; at: number; keeps: number | null; payload: string }
+
+export type AnswerRef = { ns: string; name: string; key: string }
+export type AnswerWrite = AnswerRef & { payload: unknown; at: number; keeps?: number }
+export type AnswersUse = { count: number; oldestUsed: number | null }
+
+/** one call on a session of its own, closed after */
+export function withAnswers<R>(db: string, body: (a: Answers) => R): R {
+    const a = new Answers(db)
     try {
-        mkdirSync(dirname(path), { recursive: true })
-        atomicWrite(path, JSON.stringify({ v: STORE_VERSION, key, at, keeps, payload }))
-    } catch {
-        // a store that cannot be written is slow, never wrong: the answer was
-        // already printed, and the next run measures again
+        return body(a)
+    } finally {
+        a.close()
     }
 }
 
-export function dropStored(path: string): void {
-    try { rmSync(path, { force: true }) } catch { /* raced with another runner: it is gone either way */ }
+export const listAnswers = (db: string, ns: string): { name: string; at: number; payload: string }[] => withAnswers(db, (a) => a.list(ns))
+export const answersStamp = (db: string, ns: string): string => withAnswers(db, (a) => a.stamp(ns))
+export const answersUse = (db: string): AnswersUse | null => withAnswers(db, (a) => a.use())
+export const sweepAnswers = (db: string, now: number, o?: { keepMs?: number; max?: number }): number | null => withAnswers(db, (a) => a.sweep(now, o))
+
+/** the migration: the json answers of the pre-db store read into the table
+ *  once, then the dir deleted. An answer older than IMPORT_MAX_AGE_MS is not
+ *  worth a row, a file that does not parse is just gone, and a row keeps the
+ *  time it was written as its last use, so the sweep ages it from there. Yields between
+ *  batches, so a daemon importing tens of thousands of files keeps serving. */
+export async function importLegacyStore(root: string, now = Date.now(), o: { batch?: number } = {}): Promise<{ imported: number; files: number } | null> {
+    const dir = legacyStoreDir(root)
+    let namespaces: string[]
+    try { namespaces = readdirSync(dir) } catch { return null }
+    const answers = new Answers(answersDb(root))
+    let imported = 0
+    let files = 0
+    // the file name holds the check id only as a slug (`:` and `/` became
+    // `_`), and the table keys on the id, so each file's id is proved against
+    // the hash its own name carries
+    const resolved = new Map<string, string | null>()
+    for (const ns of namespaces) {
+        let names: string[] = []
+        try { names = readdirSync(join(dir, ns)) } catch { continue }
+        for (let i = 0; i < names.length; i += o.batch ?? 500) {
+            const rows: AnswerWrite[] = []
+            for (const f of names.slice(i, i + (o.batch ?? 500))) {
+                files++
+                // `<name slug>.<12 hex>.json`: the key is inside the file, the
+                // name only as a slug, which is what the table keys on too
+                const m = /^(.*)\.([0-9a-f]{12})\.json$/.exec(f)
+                if (m == null) continue
+                try {
+                    const p = join(dir, ns, f)
+                    if (statSync(p).mtimeMs < now - IMPORT_MAX_AGE_MS) continue
+                    const e = JSON.parse(readFileSync(p, 'utf8')) as Partial<Stored<unknown>>
+                    if (e.v !== STORE_VERSION || typeof e.key !== 'string' || typeof e.at !== 'number' || e.payload == null) continue
+                    // a spelling found for this slug is reused only when THIS file's
+                    // hash proves it: two ids can share a slug and a key
+                    const slugKey = `${ns}\0${m[1]!}`
+                    const known = resolved.get(slugKey)
+                    const name = known != null && legacyHash(ns, known, e.key) === m[2] ? known : unslug(ns, m[1]!, e.key, m[2]!)
+                    if (name == null) continue
+                    resolved.set(slugKey, name)
+                    rows.push({ ns, name, key: e.key, payload: e.payload, at: e.at, keeps: e.keeps })
+                } catch { /* a torn file is no answer */ }
+            }
+            if (rows.length > 0 && !answers.write(rows, { now: 0, keepNewer: true })) { answers.close(); return null }
+            imported += rows.length
+            await Bun.sleep(0)
+        }
+    }
+    answers.close()
+    try { rmSync(dir, { recursive: true, force: true }) } catch { return null }
+    return { imported, files }
 }
 
-/** age-based housekeeping. Sweeping by "the checks I know about" is what made
- *  the old prune wrong: a run narrowed with --only knows about one check and
- *  would delete the other sixty-five. */
-export function sweepStore(store: string, ns: string, now: number, keepMs = KEEP_MS): void {
-    const dir = join(store, slug(ns))
-    const cutoff = now - keepMs
-    try {
-        for (const f of readdirSync(dir)) {
-            // `.<name>.tmp-<pid>` is what atomicWrite lands beside its target,
-            // and a writer killed between the write and the rename leaves one
-            // behind. Sweeping only *.json accreted them without bound.
-            if (!f.endsWith('.json') && !/\.tmp-\d+$/.test(f)) continue
-            const p = join(dir, f)
-            try { if (statSync(p).mtimeMs < cutoff) rmSync(p, { force: true }) } catch { /* raced: gone either way */ }
+const legacyHash = (ns: string, name: string, key: string): string => new Bun.CryptoHasher('sha256').update(`${ns}\0${name}\0${key}`).digest('hex').slice(0, 12)
+
+/** the id a legacy slug stood for, found by hashing the likely spellings of
+ *  each `_` against the hash in the file name. Null past six underscores or a
+ *  truncated slug: that answer is re-measured, which costs speed only. */
+function unslug(ns: string, slug: string, key: string, hash: string): string | null {
+    const parts = slug.split('_')
+    if (parts.length > 7) return null
+    const seps = ['_', ':', '/', '@', ' ']
+    const walk = (i: number, acc: string): string | null => {
+        if (i === parts.length) return legacyHash(ns, acc, key) === hash ? acc : null
+        for (const s of seps) {
+            const r = walk(i + 1, `${acc}${s}${parts[i]}`)
+            if (r != null) return r
         }
-    } catch {
-        // no dir yet, or unreadable: housekeeping never fails a run
+        return null
     }
+    return walk(1, parts[0]!)
 }

package/src/ledger/world-cache.ts

@@ -4,7 +4,7 @@
 //
 // The gate runner remembers a check's `ttl` answer through here. Storage is
 // not this module's concern:
-// ledger/store.ts owns the file, and the CONTENT half of the three
+// ledger/store.ts owns the table, and the CONTENT half of the three
 // populations is keyed instead of timed (ledger/ledger.ts).
 //
 // AN ANSWER CACHES. THE ABSENCE OF ONE DOES NOT. That distinction, not
@@ -32,7 +32,7 @@
 // THIS REPO is fixed by a human and the next run goes green, so replaying its
 // red would be the cache lying about work already done.
 import { fmtStamp, type Line } from '../core/wire.ts'
-import { dropStored, entryPath, FRESH_ENV, readStored, writeStored } from './store.ts'
+import { FRESH_ENV, withAnswers } from './store.ts'
 import { CACHED, RAN } from '../core/outcome-words.ts'
 
 /** the longest a MEASURED NEGATIVE is remembered, whatever TTL was declared.
@@ -41,8 +41,8 @@ import { CACHED, RAN } from '../core/outcome-words.ts'
 export const NEGATIVE_TTL_CAP_MS = 60 * 60 * 1000
 
 export type WorldCache = {
-    /** the ONE store root (ledger/store.ts storeRoot) */
-    storeDir: string
+    /** the repo's db file (ledger/store.ts answersDb) */
+    db: string
     /** which namespace inside it (the repo name for the gate), so a sweep or
      *  a manual `rm` can be scoped */
     ns: string
@@ -53,7 +53,6 @@ export type WorldCache = {
     freshEnv?: string
 }
 
-const pathFor = (c: WorldCache, name: string, key: string): string => entryPath(c.storeDir, c.ns, name, key)
 
 /** the stored answer for this exact question, or null when it must be
  *  measured. An entry may keep for LESS than the declared TTL: a measured
@@ -62,10 +61,14 @@ const pathFor = (c: WorldCache, name: string, key: string): string => entryPath(
 export function replayWorld(c: WorldCache, name: string, key: string, ttlMs: number): Line[] | null {
     if (ttlMs <= 0) return null
     if (process.env[c.freshEnv ?? FRESH_ENV] != null) return null
-    const entry = readStored<Line[]>(pathFor(c, name, key), key)
-    if (entry == null) return null
-    const keeps = entry.keeps ?? ttlMs
     const now = c.now()
+    const entry = withAnswers(c.db, (a) => {
+        const e = a.read<Line[]>(c.ns, name, key)
+        if (e != null) a.write([], { touched: [{ ns: c.ns, name, key }], now })
+        return e
+    })
+    if (entry == null || !Array.isArray(entry.payload)) return null
+    const keeps = entry.keeps ?? ttlMs
     if (now - entry.at >= keeps || now < entry.at) return null
     // the stamp was baked in when the answer was MEASURED (below): a replay
     // prints the stored lines VERBATIM, so the live run and every later replay
@@ -86,26 +89,25 @@ export type Measured = {
 }
 
 /** what a measured answer leaves behind, and the lines to report for it. This
- *  check writes its OWN file and only its own, so a racing runner can lose
+ *  check writes its OWN row and only its own, so a racing runner can lose
  *  nothing. */
 export function rememberWorld(c: WorldCache, name: string, key: string, ttlMs: number, m: Measured): Line[] {
     if (ttlMs <= 0) return m.lines
     const cacheable = !m.unmeasured && (m.clean || m.keepNegative)
     const keepsFor = m.clean ? ttlMs : Math.min(ttlMs, NEGATIVE_TTL_CAP_MS)
-    const path = pathFor(c, name, key)
     if (!cacheable) {
         // an unmeasured run evicts whatever was stored: a later run must
         // re-probe rather than replay an answer recorded before the outage. A
-        // racing runner may have written a good answer into that same file a
+        // racing runner may have written a good answer into that same row a
         // moment ago and lose it here, which costs one re-probe and can never
         // produce a stale green, so it is the safe direction.
-        dropStored(path)
+        withAnswers(c.db, (a) => a.drop(c.ns, name, key))
         return m.lines
     }
     const now = c.now()
     const lines = m.lines.map((l, i): Line =>
         i === 0 ? { ...l, text: `${l.text} (${m.clean ? `${CACHED} at ${fmtStamp(now)} until` : `${RAN} at ${fmtStamp(now)}, kept until`} ${fmtStamp(now + keepsFor)})` } : l,
     )
-    writeStored(path, key, lines, now, keepsFor)
+    withAnswers(c.db, (a) => a.write([{ ns: c.ns, name, key, payload: lines, at: now, keeps: keepsFor }], { now }))
     return lines
 }