rvlib-shipkit 0.38.0 → 1.0.0

package/src/ledger/db.ts

@@ -0,0 +1,255 @@
+// THE ONE DB — the file the run record lives in, its schema, and the migration
+// that creates it. `ledger/run-log.ts` owns what a row MEANS; this module owns
+// the file around it.
+//
+// Sweeping is a delete by age, which no concurrent write can be caught
+// inside of: a size sweep that REWRITES a file loses whatever was appended
+// during the rewrite.
+//
+// THREE PROPERTIES, and every one is load bearing:
+//  1. NEVER READ BY THE GATE. Nothing stored here can change what a check does
+//     or what it replays, so a corrupt, locked or missing db costs a run its
+//     HISTORY and never its verdict. Every failure path out of this module is
+//     a null the caller turns into one warn line.
+//  2. NOT THE LEDGER. The keyed store beside it (`ledger/store.ts`) is still
+//     one immutable json file per key, which is what a replay reads. This db
+//     is the reporting record: what every check cost on every run, red runs
+//     included, and what each check last printed.
+//  3. SEVERAL WRITERS AT ONCE is the normal case (three agents in one
+//     checkout). WAL, a busy timeout, and one immediate transaction per flush
+//     is what makes six gates land every row whole.
+//
+// WHERE. `<git-common-dir>/shipkit/shipkit.sqlite`, beside the store and
+// outside the work tree, so nothing it writes can dirty a diff or reach a
+// commit. Every linked worktree of the repo shares it, the same as the store.
+import { Database } from 'bun:sqlite'
+import { existsSync, mkdirSync } from 'fs'
+import { dirname, join } from 'path'
+import { shipkitRoot } from './store.ts'
+
+export const DB_FILE = 'shipkit.sqlite'
+
+/** the schema this build speaks, stored in `pragma user_version`. Bumped when
+ *  a table changes, never when a row's meaning does: the migration is the one
+ *  place an older file is brought forward.
+ *
+ *  v2 dropped the `kind` column. `task`, `rule` and `check` stopped being a
+ *  thing a row could carry, and every row is filed under its bare id now. The rows written under the old scheme are DROPPED rather
+ *  than migrated: they are per checkout timings a fortnight deep at most, and
+ *  re-keying them would invent a mapping nobody can verify.
+ *
+ *  v3 added `invocations`, one row per command that wrote runs (what ran it,
+ *  its label, when it started and ended), and `suites`, one row per test
+ *  suite per run of the suite runner. Both are new tables, so a v2 file keeps
+ *  every row it holds.
+ *
+ *  v4 names the last lines table `check_last`, keyed by `id`. An older file's
+ *  table is renamed in place, so its rows stay.
+ *
+ *  v5 adds `live`, one row per run IN FLIGHT: what the runner is doing right
+ *  now, rewritten as checks start and finish and deleted when the runner
+ *  returns. A new table, so every older row stays.
+ *
+ *  v6 adds `check_id` to `suites`: which check ran the suite runner that
+ *  timed it (`test`, `test:<unit>`). An older row keeps '' there, and an older
+ *  build still inserts, because the column has a default.
+ *
+ *  v7 adds `unit` to `suites`: the unit whose test check timed it
+ *  (`root`, `rvlib-shipkit`). Same shape as v6: '' on an older row, a default
+ *  for an older writer.
+ *
+ *  v8 adds three columns to `suites` (`cpu_ms`, null where no CPU figure is
+ *  the file's own; `phase`, the runner phase that timed it; `fail_line`, the
+ *  first failure bun printed for it) and the `suite_runs` table, one row per
+ *  run of the suite runner: its wall time, its phases, its workers. Defaults
+ *  again, so an older writer still inserts and every older row stays.
+ *
+ *  v9 adds `phase_ms` to `suites`: the file's wall time in the phase that ran
+ *  it, which `ms` is not once an alone rerun judged it. Same shape again. */
+export const SCHEMA_VERSION = 9
+
+/** how long a writer waits on another process's transaction before it gives
+ *  up and the run says so. Long enough for a whole gate's flush, short enough
+ *  that nothing waits on a dashboard. */
+export const BUSY_MS = 5_000
+
+export const dbPath = (root: string): string => join(shipkitRoot(root), DB_FILE)
+
+export const INSERT_RUN = 'insert into runs (run_id, id, at, ms, status, replayed) values (?, ?, ?, ?, ?, ?)'
+
+export const UPSERT_INVOCATION =
+    'insert into invocations (run_id, kind, label, by, narrowed, started, ended) values (?, ?, ?, ?, ?, ?, ?)' +
+    ' on conflict(run_id) do update set ended = excluded.ended'
+
+export const UPSERT_LIVE =
+    'insert into live (run_id, kind, label, by, pid, started, updated, state) values (?, ?, ?, ?, ?, ?, ?, ?)' +
+    ' on conflict(run_id) do update set updated = excluded.updated, state = excluded.state'
+
+export const INSERT_SUITE = 'insert into suites (at, file, ms, status, tier, budget_ms, check_id, unit, cpu_ms, phase, phase_ms, fail_line) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
+
+export const INSERT_SUITE_RUN =
+    'insert into suite_runs (at, check_id, unit, tier, wall_ms, setup_ms, workers, files, parallel_ms, parallel_cpu_ms, serial_ms, serial_cpu_ms, serial_files, alone_ms, alone_files, load1, cores)' +
+    ' values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
+
+export const UPSERT_LAST =
+    'insert into check_last (id, at, status, ms, lines) values (?, ?, ?, ?, ?)' +
+    ' on conflict(id) do update set at = excluded.at, status = excluded.status, ms = excluded.ms, lines = excluded.lines'
+
+const SCHEMA = `
+create table if not exists runs (
+    run_id text not null,
+    id text not null,
+    at integer not null,
+    ms integer not null,
+    status text not null,
+    replayed integer not null
+);
+create index if not exists runs_at on runs (at);
+create index if not exists runs_id on runs (id, at);
+create table if not exists check_last (
+    id text primary key,
+    at integer not null,
+    status text not null,
+    ms integer not null,
+    lines text not null
+);
+create table if not exists invocations (
+    run_id text primary key,
+    kind text not null,
+    label text not null,
+    by text not null,
+    narrowed integer not null,
+    started integer not null,
+    ended integer not null
+);
+create index if not exists invocations_started on invocations (started);
+create table if not exists suites (
+    at integer not null,
+    file text not null,
+    ms integer not null,
+    status text not null,
+    tier text not null,
+    budget_ms integer not null,
+    check_id text not null default '',
+    unit text not null default '',
+    cpu_ms integer default null,
+    phase text not null default '',
+    phase_ms integer default null,
+    fail_line text not null default ''
+);
+create index if not exists suites_file on suites (file, at);
+create table if not exists suite_runs (
+    at integer not null,
+    check_id text not null,
+    unit text not null,
+    tier text not null,
+    wall_ms integer not null,
+    setup_ms integer not null,
+    workers integer not null,
+    files integer not null,
+    parallel_ms integer not null,
+    parallel_cpu_ms integer default null,
+    serial_ms integer default null,
+    serial_cpu_ms integer default null,
+    serial_files integer not null default 0,
+    alone_ms integer not null default 0,
+    alone_files integer not null default 0,
+    load1 real default null,
+    cores integer default null
+);
+create index if not exists suite_runs_check on suite_runs (check_id, at);
+create table if not exists live (
+    run_id text primary key,
+    kind text not null,
+    label text not null,
+    by text not null,
+    pid integer not null,
+    started integer not null,
+    updated integer not null,
+    state text not null
+);
+`
+
+export type Opened = { db: Database }
+
+/** open the record, create the schema if this is its first open, and hand back
+ *  a handle the caller CLOSES. Null for every failure there is: no path, no
+ *  db and `create: false`, a corrupt file, a locked one, a read only disk. */
+export function openDb(path: string, o: { create?: boolean; busyMs?: number } = {}): Opened | null {
+    if (path === '') return null
+    const create = o.create !== false
+    if (!create && !existsSync(path)) return null
+    let db: Database | null = null
+    try {
+        if (create) mkdirSync(dirname(path), { recursive: true })
+        db = new Database(path, { create, readwrite: true })
+        const busyMs = Math.max(0, Math.round(o.busyMs ?? BUSY_MS))
+        // the timeout goes on FIRST, so every ordinary statement waits on
+        // another process's transaction instead of failing
+        db.exec(`pragma busy_timeout = ${busyMs}`)
+        ensureWal(db, busyMs)
+        db.exec('pragma synchronous = NORMAL')
+        migrate(db)
+        return { db }
+    } catch {
+        try { db?.close() } catch { /* never opened, or already gone */ }
+        return null
+    }
+}
+
+/** WAL is what lets readers and a writer share the file. Switching INTO it
+ *  takes an exclusive lock that sqlite refuses WITHOUT consulting the busy
+ *  handler, so `busy_timeout` does not cover it: six processes opening a fresh
+ *  db together, one of them loses and throws `database is locked`. Reproduced.
+ *  So the mode is read first (after any one process has switched it, the
+ *  switch is not attempted at all), retried briefly, and never fatal: a db
+ *  left on the rollback journal still writes every row correctly under the
+ *  busy timeout, it only serialises readers against the writer. */
+function ensureWal(db: Database, busyMs: number): void {
+    const mode = (): string => String((db.query('pragma journal_mode').get() as { journal_mode?: string } | null)?.journal_mode ?? '')
+    const deadline = Date.now() + Math.min(busyMs, 2_000)
+    for (;;) {
+        if (mode() === 'wal') return
+        try {
+            db.exec('pragma journal_mode = WAL')
+            if (mode() === 'wal') return
+        } catch { /* another process holds the exclusive lock: look again */ }
+        if (Date.now() >= deadline) return
+        Bun.sleepSync(5 + Math.floor(Math.random() * 15))
+    }
+}
+
+const userVersion = (db: Database): number => Number((db.query('pragma user_version').get() as { user_version?: number } | null)?.user_version ?? 0)
+
+/** create the schema, once. The version is read a SECOND time inside the
+ *  write lock: two processes can open a fresh db at the same moment, and the
+ *  loser must find the schema already there rather than drop it. */
+function migrate(db: Database): void {
+    if (userVersion(db) >= SCHEMA_VERSION) return
+    db.exec('begin immediate')
+    try {
+        const was = userVersion(db)
+        if (was < SCHEMA_VERSION) {
+            // the pre-v2 tables carry a `kind` column and keys of the form
+            // `<kind>:<id>`, which no reader speaks any more. v2 to v3 only
+            // adds tables, v3 to v4 renames one, v4 to v5 adds one and v5 to
+            // v6 and every step to v9 add columns (v8 a table too), so
+            // their rows stay
+            if (was < 2) db.exec('drop table if exists runs; drop table if exists unit_last;')
+            else if (was < 4 && db.query("select 1 from sqlite_master where type = 'table' and name = 'unit_last'").get() != null) db.exec('alter table unit_last rename to check_last; alter table check_last rename column unit to id; drop index if exists runs_unit;')
+            const suites = db.query("select name from pragma_table_info('suites')").all() as { name: string }[]
+            if (suites.length > 0 && !suites.some((c) => c.name === 'check_id')) db.exec("alter table suites add column check_id text not null default ''")
+            if (suites.length > 0 && !suites.some((c) => c.name === 'unit')) db.exec("alter table suites add column unit text not null default ''")
+            if (suites.length > 0 && !suites.some((c) => c.name === 'cpu_ms')) db.exec('alter table suites add column cpu_ms integer default null')
+            if (suites.length > 0 && !suites.some((c) => c.name === 'phase')) db.exec("alter table suites add column phase text not null default ''")
+            if (suites.length > 0 && !suites.some((c) => c.name === 'phase_ms')) db.exec('alter table suites add column phase_ms integer default null')
+            if (suites.length > 0 && !suites.some((c) => c.name === 'fail_line')) db.exec("alter table suites add column fail_line text not null default ''")
+            db.exec(SCHEMA)
+            db.exec(`pragma user_version = ${SCHEMA_VERSION}`)
+        }
+        db.exec('commit')
+    } catch (e) {
+        try { db.exec('rollback') } catch { /* the transaction is already gone */ }
+        throw e
+    }
+}