rvlib-shipkit 1.1.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/CHANGELOG.md +130 -0
  2. package/README.md +1 -1
  3. package/package.json +3 -1
  4. package/src/assets/run-assets.ts +3 -8
  5. package/src/check.ts +40 -2
  6. package/src/checks/SK001-package-json-shape.check.ts +14 -3
  7. package/src/checks/SK002-contracts-fulfillness.check.ts +3 -3
  8. package/src/checks/SK004-gitignore-policy.check.ts +11 -3
  9. package/src/checks/SK005-rvlib-versions.check.ts +13 -1
  10. package/src/checks/SK006-npm-publish-safety.check.ts +5 -0
  11. package/src/checks/SK007-github-visibility.check.ts +9 -9
  12. package/src/checks/SK008-reviewer-agent.check.ts +14 -8
  13. package/src/checks/SK009-commit-ai-tells.check.ts +29 -5
  14. package/src/checks/SK013-one-way-in-policy.check.ts +10 -2
  15. package/src/checks/SK015-nested-repo.check.ts +5 -1
  16. package/src/checks/SK017-vscode-hub-task.check.ts +7 -0
  17. package/src/checks/SK022-machine-runs-the-pin.check.ts +47 -0
  18. package/src/checks/SK024-identity-colors.check.ts +54 -0
  19. package/src/checks/SK025-release-drift.check.ts +58 -0
  20. package/src/checks/SK026-asks-autoclose.check.ts +44 -0
  21. package/src/checks/SK027-agent-wiki-links.check.ts +28 -0
  22. package/src/checks/SK028-readme.check.ts +79 -0
  23. package/src/cli/cmd_Assets.ts +1 -1
  24. package/src/cli/cmd_Check.ts +26 -1
  25. package/src/cli/cmd_Ci.ts +23 -5
  26. package/src/cli/cmd_CrossRepoActivity.ts +116 -0
  27. package/src/cli/cmd_GlobalProcesses.ts +155 -0
  28. package/src/cli/cmd_Home.ts +170 -0
  29. package/src/cli/cmd_Keys.ts +17 -3
  30. package/src/cli/cmd_Overview.ts +98 -0
  31. package/src/cli/cmd_Planned.ts +117 -0
  32. package/src/cli.ts +30 -2
  33. package/src/collections/checks.ts +3 -2
  34. package/src/collections/providers.ts +1 -1
  35. package/src/collections/test-time.ts +10 -1
  36. package/src/comfy/comfy-panel.ts +1 -16
  37. package/src/core/agent-spawn.ts +46 -0
  38. package/src/core/archived.ts +82 -0
  39. package/src/core/attach.ts +14 -9
  40. package/src/core/canonical-scripts.ts +12 -3
  41. package/src/core/check-file-names.ts +8 -0
  42. package/src/core/check-runner.ts +80 -30
  43. package/src/core/discover-checks.ts +38 -18
  44. package/src/core/file-stamp.ts +20 -0
  45. package/src/{checks/SK021-github-login.check.ts → core/gh-login.ts} +45 -54
  46. package/src/core/identity-color.ts +203 -0
  47. package/src/core/identity.ts +91 -0
  48. package/src/core/job-follow.ts +102 -0
  49. package/src/core/job-run.ts +27 -0
  50. package/src/core/jobs.ts +87 -27
  51. package/src/core/legacy-shapes.ts +1 -14
  52. package/src/core/machine-db.ts +83 -0
  53. package/src/core/one-lib.ts +85 -0
  54. package/src/core/port-holders.ts +43 -0
  55. package/src/core/repo-facts.ts +14 -0
  56. package/src/core/retention.ts +222 -0
  57. package/src/core/retired-checks.ts +7 -0
  58. package/src/core/run-check.ts +12 -4
  59. package/src/core/run-ci.ts +46 -12
  60. package/src/core/run-fix.ts +7 -2
  61. package/src/core/run-fixers.ts +10 -6
  62. package/src/core/slots.ts +5 -0
  63. package/src/core/statement-end.ts +17 -0
  64. package/src/core/wire.ts +11 -0
  65. package/src/fleet/scan-fleet.ts +2 -1
  66. package/src/home/activity-shape.ts +135 -0
  67. package/src/home/activity.ts +249 -0
  68. package/src/home/attach.ts +95 -0
  69. package/src/home/autostart.ts +108 -0
  70. package/src/home/global-units.ts +87 -0
  71. package/src/home/gui-app/ShipkitGui.swift +159 -0
  72. package/src/home/gui-app.ts +99 -0
  73. package/src/home/handoff.ts +51 -0
  74. package/src/home/home-launchd.ts +104 -0
  75. package/src/home/home-paths.ts +88 -0
  76. package/src/home/home-routes.ts +235 -0
  77. package/src/home/home-start.ts +105 -0
  78. package/src/home/home-state.ts +43 -0
  79. package/src/home/home-words.ts +12 -0
  80. package/src/home/hubs.ts +286 -0
  81. package/src/home/lib-watch.ts +139 -0
  82. package/src/home/machine-stores.ts +115 -0
  83. package/src/home/open-app.ts +16 -0
  84. package/src/home/pm-config.ts +37 -0
  85. package/src/home/pm.ts +174 -0
  86. package/src/home/process-rows.ts +205 -0
  87. package/src/home/resources.ts +365 -0
  88. package/src/home/run-daemon.ts +137 -0
  89. package/src/home/run-pm.ts +150 -0
  90. package/src/home/run-unit-action.ts +25 -0
  91. package/src/home/run-web.ts +37 -0
  92. package/src/home/serve-daemon.ts +31 -0
  93. package/src/home/serve-web.ts +175 -0
  94. package/src/home/supervise.ts +361 -0
  95. package/src/home/via-hub.ts +142 -0
  96. package/src/home/widget-text.ts +119 -0
  97. package/src/home/widgets.ts +102 -0
  98. package/src/hooks/run-hooks.ts +20 -3
  99. package/src/ledger/check-stats.ts +19 -45
  100. package/src/ledger/db.ts +69 -17
  101. package/src/ledger/gate-checks.ts +22 -3
  102. package/src/ledger/git-scan.ts +31 -7
  103. package/src/ledger/house-runs.ts +43 -0
  104. package/src/ledger/ledger.ts +40 -14
  105. package/src/ledger/live-timeline.ts +4 -19
  106. package/src/ledger/run-log.ts +92 -28
  107. package/src/ledger/store.ts +245 -103
  108. package/src/ledger/world-cache.ts +64 -15
  109. package/src/plan/cadence.ts +253 -0
  110. package/src/plan/catch-up.ts +37 -0
  111. package/src/plan/plan-timeline.ts +99 -0
  112. package/src/plan/planner.ts +428 -0
  113. package/src/plan/repo-source.ts +212 -0
  114. package/src/problems/problem-line.ts +1 -1
  115. package/src/release/run-release.ts +9 -2
  116. package/src/release/unit-resolve.ts +8 -12
  117. package/src/repo-config.ts +61 -4
  118. package/src/serve/house-runs.ts +53 -0
  119. package/src/serve/hub-app/activity-st.ts +168 -0
  120. package/src/serve/hub-app/activity-view.tsx +218 -0
  121. package/src/serve/hub-app/activity.css +78 -0
  122. package/src/serve/hub-app/admin-svc.ts +41 -0
  123. package/src/serve/hub-app/admin-view.tsx +42 -0
  124. package/src/serve/hub-app/chart-view.ts +12 -5
  125. package/src/serve/hub-app/check-view.tsx +9 -9
  126. package/src/serve/hub-app/check-words.ts +3 -1
  127. package/src/serve/hub-app/checks-svc.ts +12 -8
  128. package/src/serve/hub-app/col-card.tsx +4 -4
  129. package/src/serve/hub-app/cols-svc.ts +92 -13
  130. package/src/serve/hub-app/comfy-svc.ts +12 -6
  131. package/src/serve/hub-app/comfy-view.tsx +3 -3
  132. package/src/serve/hub-app/controls.css +32 -0
  133. package/src/serve/hub-app/cycle-button.tsx +36 -0
  134. package/src/serve/hub-app/dock-anim.ts +25 -0
  135. package/src/serve/hub-app/dock-st.ts +329 -0
  136. package/src/serve/hub-app/dock-view.tsx +68 -0
  137. package/src/serve/hub-app/dock.css +35 -0
  138. package/src/serve/hub-app/floating.tsx +49 -0
  139. package/src/serve/hub-app/geometry-view.tsx +4 -4
  140. package/src/serve/hub-app/hash-route.ts +8 -2
  141. package/src/serve/hub-app/header-bar.tsx +102 -95
  142. package/src/serve/hub-app/health-svc.ts +3 -3
  143. package/src/serve/hub-app/help.tsx +10 -7
  144. package/src/serve/hub-app/home-st.ts +278 -0
  145. package/src/serve/hub-app/home-strip.css +113 -0
  146. package/src/serve/hub-app/home-strip.tsx +376 -0
  147. package/src/serve/hub-app/home.html +21 -0
  148. package/src/serve/hub-app/hub-app.tsx +221 -177
  149. package/src/serve/hub-app/hub-types.ts +0 -3
  150. package/src/serve/hub-app/hub.css +112 -52
  151. package/src/serve/hub-app/icons.tsx +34 -0
  152. package/src/serve/hub-app/index.html +7 -1
  153. package/src/serve/hub-app/keys.ts +60 -0
  154. package/src/serve/hub-app/layout-st.ts +55 -0
  155. package/src/serve/hub-app/live-svc.ts +35 -72
  156. package/src/serve/hub-app/machine-st.ts +333 -0
  157. package/src/serve/hub-app/md.ts +38 -5
  158. package/src/serve/hub-app/media-svc.ts +63 -44
  159. package/src/serve/hub-app/media-view.tsx +20 -20
  160. package/src/serve/hub-app/next-run.ts +24 -30
  161. package/src/serve/hub-app/palette-st.ts +92 -0
  162. package/src/serve/hub-app/palette-view.tsx +47 -0
  163. package/src/serve/hub-app/pill-row.tsx +65 -0
  164. package/src/serve/hub-app/planned-st.ts +107 -0
  165. package/src/serve/hub-app/planned-view.tsx +114 -0
  166. package/src/serve/hub-app/planned.css +42 -0
  167. package/src/serve/hub-app/processes-svc.ts +63 -0
  168. package/src/serve/hub-app/project-list.css +119 -0
  169. package/src/serve/hub-app/project-list.tsx +266 -0
  170. package/src/serve/hub-app/projects-svc.ts +253 -23
  171. package/src/serve/hub-app/prose-hits-st.ts +3 -3
  172. package/src/serve/hub-app/prose-hits-view.tsx +6 -6
  173. package/src/serve/hub-app/readme-svc.ts +38 -0
  174. package/src/serve/hub-app/readme-view.tsx +19 -0
  175. package/src/serve/hub-app/readme.css +11 -0
  176. package/src/serve/hub-app/repo-sections.tsx +61 -0
  177. package/src/serve/hub-app/repo-st.ts +340 -0
  178. package/src/serve/hub-app/runs-tabs.ts +1 -1
  179. package/src/serve/hub-app/screen-cache.ts +74 -0
  180. package/src/serve/hub-app/sidebar.tsx +84 -0
  181. package/src/serve/hub-app/test-view.tsx +4 -4
  182. package/src/serve/hub-app/tests-svc.ts +3 -3
  183. package/src/serve/hub-app/tests-view.tsx +41 -14
  184. package/src/serve/hub-app/timeline-chart.tsx +55 -39
  185. package/src/serve/hub-app/timeline-geometry.ts +40 -5
  186. package/src/serve/hub-app/timeline-svc.ts +7 -6
  187. package/src/serve/hub-app/timeline-view.tsx +33 -16
  188. package/src/serve/hub-app/tree-row.tsx +2 -2
  189. package/src/serve/hub-app/tree-svc.ts +2 -2
  190. package/src/serve/hub-app/wikis-view.tsx +10 -6
  191. package/src/serve/hub-runtime.ts +550 -0
  192. package/src/serve/other-repos.ts +8 -45
  193. package/src/serve/project-gutter-route.ts +121 -20
  194. package/src/serve/project-gutter.ts +33 -11
  195. package/src/serve/readme-route.ts +57 -0
  196. package/src/serve/repo-processes.ts +89 -0
  197. package/src/serve/run-serve.ts +110 -402
  198. package/src/serve/serve-hub.ts +155 -52
  199. package/src/serve/stale-restart.ts +5 -0
  200. package/src/serve/{gate-trigger.ts → tracked-watch.ts} +37 -65
  201. package/src/serve/warm-cell.ts +95 -0
  202. package/src/shipkit-files.ts +130 -0
  203. package/src/shipkit.ts +2 -1
  204. package/src/suite/kept-timings.ts +1 -1
  205. package/src/wiki/graph-positions.ts +1 -1
  206. package/src/checks/contracts.check.ts +0 -20
  207. package/src/serve/health-schedule.ts +0 -102
  208. package/src/serve/hub-app/hub-st.ts +0 -250
  209. package/src/serve/hub-app/project-gutter.css +0 -65
  210. package/src/serve/hub-app/project-gutter.tsx +0 -128
@@ -1,52 +1,35 @@
1
- // ONE STORE for every ANSWER this toolkit remembers between runs, in one
2
- // shape, under one root, busted by one flag. What a run COST lives beside it
3
- // in `ledger/db.ts`, which nothing about a verdict ever reads.
1
+ // ONE STORE for every ANSWER this toolkit remembers between runs: the
2
+ // `answers` table of the repo's shipkit.sqlite (ledger/db.ts owns the file).
3
+ // What differs between populations (the gate's checks, the world probes) is
4
+ // the namespace and the KEY, never the storage.
4
5
  //
5
- // There used to be two: `tmp/.shipkit-cache` for the check orchestrator and
6
- // `tmp/.check-cache` for the page checks, each with its own bust env var, each
7
- // per worktree. Two stores means two staleness windows over one question, and
8
- // the stamps in this repo's own history went BACKWARDS between checkouts
9
- // because of it. This module is the single mechanism; what differs between
10
- // populations is the KEY, never the storage.
6
+ // THE RULE THAT KEEPS A VERDICT HONEST: a db error is "no answer". Corrupt,
7
+ // locked past a short wait, missing, a row whose envelope does not match: the
8
+ // check is MEASURED. Never a replay of something unreadable, and never a red
9
+ // because the store failed. A write that fails is a slower next run, nothing
10
+ // more.
11
11
  //
12
- // THE SHAPE, unchanged from the check cache slice 2 landed and proved:
13
- // one immutable file per key, named by the key, written tmp + rename. No
14
- // lock, no merge, no read-modify-write, and no prune of another runner's
15
- // entries. Three agents in one checkout is the normal case, so the only safe
16
- // aggregate is no aggregate: a run with a different key writes a DIFFERENT
17
- // file, a run with the same key replaces its own whole answer or none of it,
18
- // and housekeeping sweeps by AGE, which needs no knowledge of what the
19
- // current run knows about.
20
- //
21
- // WHERE. `<git-common-dir>/shipkit/store/`, which is SHARED by every linked
22
- // worktree of the repo. That is safe only because no key component is a path:
23
- // the toolchain identity content-hashes the gate binary and every installed
24
- // package, so two worktrees carrying different installs produce different
25
- // keys and simply miss each other (ledger/toolchain-identity.ts owns the
26
- // measurement). It is also outside the work tree, so nothing here can dirty a
27
- // diff or reach a commit.
28
- //
29
- // The store dir holds keyed entries and NOTHING else: every file under it is
30
- // one whole json document, so a reader can walk it and parse all of it. The
31
- // reporting record beside it (the run db) lives one level up, in
32
- // `shipkitRoot`, because it is not keyed and is not a json document.
33
- import { mkdirSync, readdirSync, readFileSync, rmSync, statSync } from 'fs'
34
- import { dirname, join } from 'path'
35
- import { gitRoot } from '../core/git-root.ts'
36
- import { atomicWrite } from '../core/locked-write.ts'
37
- import { LAYOUT } from '../core/layout.ts'
38
-
39
- /** the storage FORMAT. Bumped when the envelope changes, so an old file can
40
- * never be handed to a new reader. The KEY carries everything about the
41
- * question; this is only about the file around it. */
12
+ // A row is keyed by (ns, name, key) and replaced whole when the same question
13
+ // is answered again, so a run with a different key writes a DIFFERENT row and
14
+ // nothing is merged or read modified written. `used` is the last write or
15
+ // replay, which is what the sweep prunes on: an answer nobody replayed for
16
+ // KEEP_MS is a question nobody asks any more.
17
+ import type { Database } from 'bun:sqlite'
18
+ import { readdirSync, readFileSync, rmSync, statSync } from 'fs'
19
+ import { join } from 'path'
20
+ import { FOREVER_MS } from '../core/wire.ts'
21
+ import { BUSY_MS, dbPath, openDb, shipkitRoot } from './db.ts'
22
+
23
+ /** the envelope FORMAT, stored per row. Bumped when the envelope changes, so
24
+ * an old row can never be handed to a new reader. The KEY carries everything
25
+ * about the question. */
42
26
  export const STORE_VERSION = 1
43
27
 
44
28
  /** the ONE env var that busts every remembered answer, content or TTL. */
45
29
  export const FRESH_ENV = 'SHIPKIT_FRESH'
46
30
 
47
- /** one remembered answer. `key` is stored as well as hashed into the file
48
- * name, and re-checked on read: a name collision must miss, never serve a
49
- * neighbour's answer. */
31
+ /** one remembered answer. `key` is re-checked on read, so a row can only ever
32
+ * answer the exact question it was stored for. */
50
33
  export type Stored<T> = {
51
34
  v: number
52
35
  key: string
@@ -60,79 +43,238 @@ export type Stored<T> = {
60
43
  payload: T
61
44
  }
62
45
 
63
- /** how long a superseded entry survives. Far past the longest TTL any check
64
- * declares, so a live entry is never swept out from under a replay. */
65
- export const KEEP_MS = 30 * 24 * 60 * 60 * 1000
66
-
67
- /** the one dir everything this repo remembers hangs off.
68
- *
69
- * `--git-common-dir` rather than `--git-dir`: a linked worktree has its own
70
- * git dir but shares the common one, and slice 3 measured that sharing is
71
- * SAFE here because every key component is content or a version string.
72
- * A tree that is not a git repo at all still gets one, under `.shipkit/cache/`,
73
- * because a scaffold in progress must not crash on it. */
74
- export function shipkitRoot(root: string): string {
75
- const dir = gitRoot(root)?.commonDir
76
- return dir == null ? join(root, LAYOUT.cache) : join(dir, 'shipkit')
77
- }
46
+ /** how long an answer nobody wrote or replayed survives. Far past the longest
47
+ * TTL any check declares, so a live answer is never swept out from under a
48
+ * replay. */
49
+ export const KEEP_MS = 14 * 24 * 60 * 60 * 1000
78
50
 
79
- /** where this repo's remembered ANSWERS live: keyed entries only, one whole
80
- * json document per file, nothing else at any depth. */
81
- export const storeRoot = (root: string): string => join(shipkitRoot(root), 'store')
51
+ /** the most rows one repo keeps. A gate asks about 70 questions, so this is
52
+ * dozens of distinct trees; past it the least recently used go first. */
53
+ export const MAX_ANSWERS = 5_000
82
54
 
83
- const slug = (s: string): string => s.replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 60)
55
+ /** the import takes json answers this young only: every edit of shipkit moves
56
+ * every key, so an older answer is one no gate asks for again */
57
+ export const IMPORT_MAX_AGE_MS = 24 * 60 * 60 * 1000
84
58
 
85
- /** where ONE answer lives. The key is in the file NAME, so a run whose key
86
- * moved writes a new file instead of rewriting the old one, and two runners
87
- * with different keys never collide. `ns` keeps populations apart so a sweep
88
- * or a manual `rm` can be scoped. */
89
- export function entryPath(store: string, ns: string, name: string, key: string): string {
90
- const h = new Bun.CryptoHasher('sha256').update(`${ns}\0${name}\0${key}`).digest('hex').slice(0, 12)
91
- return join(store, slug(ns), `${slug(name)}.${h}.json`)
92
- }
59
+ /** a read waits this long on a writer, then measures. A replay that waits
60
+ * longer than the check would take is no replay. */
61
+ export const READ_BUSY_MS = 250
93
62
 
94
- /** a missing or half-written file reads as "no answer", which is the safe
95
- * side: the check runs. tmp + rename means half-written cannot happen from a
96
- * shipkit write, so this only ever catches a truncated file from something
97
- * else, and treats it the same way. */
98
- export function readStored<T>(path: string, key: string): Stored<T> | null {
99
- try {
100
- const e = JSON.parse(readFileSync(path, 'utf8')) as Partial<Stored<T>>
101
- if (e.v !== STORE_VERSION || e.key !== key || typeof e.at !== 'number' || e.payload == null) return null
102
- return e as Stored<T>
103
- } catch { return null }
63
+ /** where this repo's answers live: the one db file */
64
+ export const answersDb = (root: string): string => dbPath(root)
65
+
66
+ /** the one dir the pre-db store used: one json file per key. Read once by
67
+ * importLegacyStore, then deleted. */
68
+ export const legacyStoreDir = (root: string): string => join(shipkitRoot(root), 'store')
69
+
70
+ /** one session over a repo's answers: ONE handle, opened on first use and
71
+ * closed by the owner (a gate's ledger, one world probe). Never shared across
72
+ * sessions: a handle outliving its run can serve pages of a file something
73
+ * replaced underneath it. A handle that fails once is dropped, and the rest
74
+ * of the session measures. */
75
+ export class Answers {
76
+ private h: Database | null | undefined
77
+
78
+ constructor(readonly db: string) {}
79
+
80
+ private handle(busyMs: number): Database | null {
81
+ if (this.h === undefined) this.h = openDb(this.db, { busyMs })?.db ?? null
82
+ if (this.h == null) return null
83
+ try {
84
+ this.h.exec(`pragma busy_timeout = ${Math.max(0, Math.round(busyMs))}`)
85
+ return this.h
86
+ } catch {
87
+ this.close()
88
+ return null
89
+ }
90
+ }
91
+
92
+ private with<R>(busyMs: number, body: (h: Database) => R): R | null {
93
+ const h = this.handle(busyMs)
94
+ if (h == null) return null
95
+ try {
96
+ return body(h)
97
+ } catch {
98
+ this.close()
99
+ return null
100
+ }
101
+ }
102
+
103
+ close(): void {
104
+ try { this.h?.close() } catch { /* already gone */ }
105
+ this.h = null
106
+ }
107
+
108
+ /** the stored answer, or null when the check must run */
109
+ read<T>(ns: string, name: string, key: string): Stored<T> | null {
110
+ const row = this.with(READ_BUSY_MS, (h) => h.query('select v, at, keeps, payload from answers where ns = ? and name = ? and key = ?').get(ns, name, key) as Row | null)
111
+ if (row == null) return null
112
+ try {
113
+ const payload = JSON.parse(row.payload) as T
114
+ if (Number(row.v) !== STORE_VERSION || typeof row.at !== 'number' || payload == null) return null
115
+ return { v: Number(row.v), key, at: row.at, keeps: row.keeps ?? undefined, payload }
116
+ } catch {
117
+ return null
118
+ }
119
+ }
120
+
121
+ /** write answers and stamp replays, in ONE transaction. False when it
122
+ * could not: the answers were already printed, and the next run
123
+ * measures again. */
124
+ write(rows: AnswerWrite[], o: { touched?: AnswerRef[]; now?: number; keepNewer?: boolean } = {}): boolean {
125
+ if (rows.length === 0 && (o.touched ?? []).length === 0) return true
126
+ const now = o.now ?? Date.now()
127
+ return (
128
+ this.with(BUSY_MS, (h) => {
129
+ // the import never overwrites a row a gate already wrote
130
+ const put = h.query(`insert or ${o.keepNewer === true ? 'ignore' : 'replace'} into answers (ns, name, key, v, at, keeps, used, payload) values (?, ?, ?, ?, ?, ?, ?, ?)`)
131
+ const use = h.query('update answers set used = max(used, ?) where ns = ? and name = ? and key = ?')
132
+ h.transaction(() => {
133
+ for (const r of rows) put.run(r.ns, r.name, r.key, STORE_VERSION, r.at, r.keeps ?? null, Math.max(r.at, now), JSON.stringify(r.payload))
134
+ for (const t of o.touched ?? []) use.run(now, t.ns, t.name, t.key)
135
+ }).immediate()
136
+ return true
137
+ }) ?? false
138
+ )
139
+ }
140
+
141
+ /** every stored answer of one check, newest first: what adopting an
142
+ * answer written under an older key shape reads */
143
+ readAll<T>(ns: string, name: string): Stored<T>[] {
144
+ const rows = this.with(READ_BUSY_MS, (h) => h.query('select v, key, at, keeps, payload from answers where ns = ? and name = ? order by at desc').all(ns, name) as (Row & { key: string })[]) ?? []
145
+ return rows.flatMap((row) => {
146
+ try {
147
+ const payload = JSON.parse(row.payload) as T
148
+ if (Number(row.v) !== STORE_VERSION || typeof row.at !== 'number' || payload == null) return []
149
+ return [{ v: Number(row.v), key: row.key, at: row.at, keeps: row.keeps ?? undefined, payload }]
150
+ } catch {
151
+ return []
152
+ }
153
+ })
154
+ }
155
+
156
+ drop(ns: string, name: string, key: string): void {
157
+ this.with(BUSY_MS, (h) => h.query('delete from answers where ns = ? and name = ? and key = ?').run(ns, name, key))
158
+ }
159
+
160
+ /** every answer of one population: what the hub's check stats read */
161
+ list(ns: string): { name: string; at: number; payload: string }[] {
162
+ return this.with(READ_BUSY_MS, (h) => h.query('select name, at, payload from answers where ns = ?').all(ns) as { name: string; at: number; payload: string }[]) ?? []
163
+ }
164
+
165
+ /** a cheap fingerprint of one population, so a reader can cache on it */
166
+ stamp(ns: string): string {
167
+ const r = this.with(READ_BUSY_MS, (h) => h.query('select count(*) as n, max(at) as last from answers where ns = ?').get(ns) as { n: number; last: number | null } | null)
168
+ return r == null ? 'none' : `${r.n}/${r.last ?? 0}`
169
+ }
170
+
171
+ use(): AnswersUse | null {
172
+ const r = this.with(READ_BUSY_MS, (h) => h.query('select count(*) as n, min(used) as oldest from answers').get() as { n: number; oldest: number | null } | null)
173
+ return r == null ? null : { count: Number(r.n), oldestUsed: r.oldest }
174
+ }
175
+
176
+ /** the cap, enforced: rows not written nor replayed for `keepMs` go, then
177
+ * the least recently used past `max`. A DELETE, so a row written while it
178
+ * runs is kept. How many went, null when the db could not be opened. */
179
+ sweep(now: number, o: { keepMs?: number; max?: number } = {}): number | null {
180
+ return this.with(BUSY_MS, (h) => {
181
+ // a forever answer is a decided fact: dropping it re-asks the world
182
+ let n = h.query('delete from answers where used < ? and coalesce(keeps, 0) < ?').run(now - (o.keepMs ?? KEEP_MS), FOREVER_MS).changes
183
+ n += h.query('delete from answers where rowid in (select rowid from answers where coalesce(keeps, 0) < ? order by used desc limit -1 offset ?)').run(FOREVER_MS, o.max ?? MAX_ANSWERS).changes
184
+ return n
185
+ })
186
+ }
104
187
  }
105
188
 
106
- export function writeStored<T>(path: string, key: string, payload: T, at: number, keeps?: number): void {
189
+ type Row = { v: number; at: number; keeps: number | null; payload: string }
190
+
191
+ export type AnswerRef = { ns: string; name: string; key: string }
192
+ export type AnswerWrite = AnswerRef & { payload: unknown; at: number; keeps?: number }
193
+ export type AnswersUse = { count: number; oldestUsed: number | null }
194
+
195
+ /** one call on a session of its own, closed after */
196
+ export function withAnswers<R>(db: string, body: (a: Answers) => R): R {
197
+ const a = new Answers(db)
107
198
  try {
108
- mkdirSync(dirname(path), { recursive: true })
109
- atomicWrite(path, JSON.stringify({ v: STORE_VERSION, key, at, keeps, payload }))
110
- } catch {
111
- // a store that cannot be written is slow, never wrong: the answer was
112
- // already printed, and the next run measures again
199
+ return body(a)
200
+ } finally {
201
+ a.close()
113
202
  }
114
203
  }
115
204
 
116
- export function dropStored(path: string): void {
117
- try { rmSync(path, { force: true }) } catch { /* raced with another runner: it is gone either way */ }
205
+ export const listAnswers = (db: string, ns: string): { name: string; at: number; payload: string }[] => withAnswers(db, (a) => a.list(ns))
206
+ export const answersStamp = (db: string, ns: string): string => withAnswers(db, (a) => a.stamp(ns))
207
+ export const answersUse = (db: string): AnswersUse | null => withAnswers(db, (a) => a.use())
208
+ export const sweepAnswers = (db: string, now: number, o?: { keepMs?: number; max?: number }): number | null => withAnswers(db, (a) => a.sweep(now, o))
209
+
210
+ /** the migration: the json answers of the pre-db store read into the table
211
+ * once, then the dir deleted. An answer older than IMPORT_MAX_AGE_MS is not
212
+ * worth a row, a file that does not parse is just gone, and a row keeps the
213
+ * time it was written as its last use, so the sweep ages it from there. Yields between
214
+ * batches, so a daemon importing tens of thousands of files keeps serving. */
215
+ export async function importLegacyStore(root: string, now = Date.now(), o: { batch?: number } = {}): Promise<{ imported: number; files: number } | null> {
216
+ const dir = legacyStoreDir(root)
217
+ let namespaces: string[]
218
+ try { namespaces = readdirSync(dir) } catch { return null }
219
+ const answers = new Answers(answersDb(root))
220
+ let imported = 0
221
+ let files = 0
222
+ // the file name holds the check id only as a slug (`:` and `/` became
223
+ // `_`), and the table keys on the id, so each file's id is proved against
224
+ // the hash its own name carries
225
+ const resolved = new Map<string, string | null>()
226
+ for (const ns of namespaces) {
227
+ let names: string[] = []
228
+ try { names = readdirSync(join(dir, ns)) } catch { continue }
229
+ for (let i = 0; i < names.length; i += o.batch ?? 500) {
230
+ const rows: AnswerWrite[] = []
231
+ for (const f of names.slice(i, i + (o.batch ?? 500))) {
232
+ files++
233
+ // `<name slug>.<12 hex>.json`: the key is inside the file, the
234
+ // name only as a slug, which is what the table keys on too
235
+ const m = /^(.*)\.([0-9a-f]{12})\.json$/.exec(f)
236
+ if (m == null) continue
237
+ try {
238
+ const p = join(dir, ns, f)
239
+ if (statSync(p).mtimeMs < now - IMPORT_MAX_AGE_MS) continue
240
+ const e = JSON.parse(readFileSync(p, 'utf8')) as Partial<Stored<unknown>>
241
+ if (e.v !== STORE_VERSION || typeof e.key !== 'string' || typeof e.at !== 'number' || e.payload == null) continue
242
+ // a spelling found for this slug is reused only when THIS file's
243
+ // hash proves it: two ids can share a slug and a key
244
+ const slugKey = `${ns}\0${m[1]!}`
245
+ const known = resolved.get(slugKey)
246
+ const name = known != null && legacyHash(ns, known, e.key) === m[2] ? known : unslug(ns, m[1]!, e.key, m[2]!)
247
+ if (name == null) continue
248
+ resolved.set(slugKey, name)
249
+ rows.push({ ns, name, key: e.key, payload: e.payload, at: e.at, keeps: e.keeps })
250
+ } catch { /* a torn file is no answer */ }
251
+ }
252
+ if (rows.length > 0 && !answers.write(rows, { now: 0, keepNewer: true })) { answers.close(); return null }
253
+ imported += rows.length
254
+ await Bun.sleep(0)
255
+ }
256
+ }
257
+ answers.close()
258
+ try { rmSync(dir, { recursive: true, force: true }) } catch { return null }
259
+ return { imported, files }
118
260
  }
119
261
 
120
- /** age-based housekeeping. Sweeping by "the checks I know about" is what made
121
- * the old prune wrong: a run narrowed with --only knows about one check and
122
- * would delete the other sixty-five. */
123
- export function sweepStore(store: string, ns: string, now: number, keepMs = KEEP_MS): void {
124
- const dir = join(store, slug(ns))
125
- const cutoff = now - keepMs
126
- try {
127
- for (const f of readdirSync(dir)) {
128
- // `.<name>.tmp-<pid>` is what atomicWrite lands beside its target,
129
- // and a writer killed between the write and the rename leaves one
130
- // behind. Sweeping only *.json accreted them without bound.
131
- if (!f.endsWith('.json') && !/\.tmp-\d+$/.test(f)) continue
132
- const p = join(dir, f)
133
- try { if (statSync(p).mtimeMs < cutoff) rmSync(p, { force: true }) } catch { /* raced: gone either way */ }
262
+ const legacyHash = (ns: string, name: string, key: string): string => new Bun.CryptoHasher('sha256').update(`${ns}\0${name}\0${key}`).digest('hex').slice(0, 12)
263
+
264
+ /** the id a legacy slug stood for, found by hashing the likely spellings of
265
+ * each `_` against the hash in the file name. Null past six underscores or a
266
+ * truncated slug: that answer is re-measured, which costs speed only. */
267
+ function unslug(ns: string, slug: string, key: string, hash: string): string | null {
268
+ const parts = slug.split('_')
269
+ if (parts.length > 7) return null
270
+ const seps = ['_', ':', '/', '@', ' ']
271
+ const walk = (i: number, acc: string): string | null => {
272
+ if (i === parts.length) return legacyHash(ns, acc, key) === hash ? acc : null
273
+ for (const s of seps) {
274
+ const r = walk(i + 1, `${acc}${s}${parts[i]}`)
275
+ if (r != null) return r
134
276
  }
135
- } catch {
136
- // no dir yet, or unreadable: housekeeping never fails a run
277
+ return null
137
278
  }
279
+ return walk(1, parts[0]!)
138
280
  }
@@ -4,7 +4,7 @@
4
4
  //
5
5
  // The gate runner remembers a check's `ttl` answer through here. Storage is
6
6
  // not this module's concern:
7
- // ledger/store.ts owns the file, and the CONTENT half of the three
7
+ // ledger/store.ts owns the table, and the CONTENT half of the three
8
8
  // populations is keyed instead of timed (ledger/ledger.ts).
9
9
  //
10
10
  // AN ANSWER CACHES. THE ABSENCE OF ONE DOES NOT. That distinction, not
@@ -31,8 +31,8 @@
31
31
  // A measured negative may be kept only for a check about the WORLD. One about
32
32
  // THIS REPO is fixed by a human and the next run goes green, so replaying its
33
33
  // red would be the cache lying about work already done.
34
- import { fmtStamp, type Line } from '../core/wire.ts'
35
- import { dropStored, entryPath, FRESH_ENV, readStored, writeStored } from './store.ts'
34
+ import { FOREVER_MS, fmtStamp, type Line } from '../core/wire.ts'
35
+ import { FRESH_ENV, withAnswers, type Answers, type Stored } from './store.ts'
36
36
  import { CACHED, RAN } from '../core/outcome-words.ts'
37
37
 
38
38
  /** the longest a MEASURED NEGATIVE is remembered, whatever TTL was declared.
@@ -40,9 +40,18 @@ import { CACHED, RAN } from '../core/outcome-words.ts'
40
40
  * enough that the world being fixed is noticed the same day. */
41
41
  export const NEGATIVE_TTL_CAP_MS = 60 * 60 * 1000
42
42
 
43
+ /** what a world answer is stored under. A timed answer follows the program
44
+ * that measured it (toolchain, probe code); a forever one is the declared
45
+ * value alone, or a shipkit release would ask every decided fact again */
46
+ export function worldKeyOf(k: { ttlMs: number; toolchain: string; code: string; rotation: string; frozen: boolean | null }): string {
47
+ const frozen = k.frozen == null ? '' : `\0frozen=${k.frozen}`
48
+ if (k.ttlMs >= FOREVER_MS) return `forever\0${k.rotation}${frozen}`
49
+ return `tc=${k.toolchain}\0code=${k.code}\0${k.rotation}${frozen}`
50
+ }
51
+
43
52
  export type WorldCache = {
44
- /** the ONE store root (ledger/store.ts storeRoot) */
45
- storeDir: string
53
+ /** the repo's db file (ledger/store.ts answersDb) */
54
+ db: string
46
55
  /** which namespace inside it (the repo name for the gate), so a sweep or
47
56
  * a manual `rm` can be scoped */
48
57
  ns: string
@@ -53,7 +62,31 @@ export type WorldCache = {
53
62
  freshEnv?: string
54
63
  }
55
64
 
56
- const pathFor = (c: WorldCache, name: string, key: string): string => entryPath(c.storeDir, c.ns, name, key)
65
+
66
+ const FOREVER_PREFIX = 'forever\0'
67
+ /** the timed key a check wrote before it declared `ttl: 'forever'`:
68
+ * `tc=<toolchain>\0code=<code>\0` then the same declared value */
69
+ const TIMED_HEAD = /^tc=[^\0]*\0code=[^\0]*\0/
70
+
71
+ /** a decided fact with no row under its forever key takes the newest CLEAN
72
+ * answer its timed key shape stored for the same declared value, once: the
73
+ * value it checks did not move, so the fact is still known. Written under
74
+ * the forever key; the timed rows age out with the store's sweep */
75
+ function adoptTimed(a: Answers, ns: string, name: string, key: string): Stored<Line[]> | null {
76
+ if (!key.startsWith(FOREVER_PREFIX)) return null
77
+ const rest = key.slice(FOREVER_PREFIX.length)
78
+ const old = a.readAll<Line[]>(ns, name).find((e) => {
79
+ const head = TIMED_HEAD.exec(e.key)
80
+ return head != null && e.key.slice(head[0].length) === rest && Array.isArray(e.payload) && e.payload.length > 0 && e.payload.every((l) => l.kind === 'ok' || l.kind === 'note')
81
+ })
82
+ if (old == null) return null
83
+ const payload = old.payload.map((l, i): Line => (i !== 0 ? l : { ...l, text: l.text.replace(new RegExp(`\\((${CACHED} at [^)]*?) until [^)]*\\)$`), '($1 until its declaration changes)') }))
84
+ a.write([{ ns, name, key, payload, at: old.at, keeps: FOREVER_MS }], { now: old.at })
85
+ return { ...old, key, keeps: FOREVER_MS, payload }
86
+ }
87
+
88
+ /** the answer for this key, or the one a forever key adopts */
89
+ const readOrAdopt = (a: Answers, ns: string, name: string, key: string): Stored<Line[]> | null => a.read<Line[]>(ns, name, key) ?? adoptTimed(a, ns, name, key)
57
90
 
58
91
  /** the stored answer for this exact question, or null when it must be
59
92
  * measured. An entry may keep for LESS than the declared TTL: a measured
@@ -62,10 +95,14 @@ const pathFor = (c: WorldCache, name: string, key: string): string => entryPath(
62
95
  export function replayWorld(c: WorldCache, name: string, key: string, ttlMs: number): Line[] | null {
63
96
  if (ttlMs <= 0) return null
64
97
  if (process.env[c.freshEnv ?? FRESH_ENV] != null) return null
65
- const entry = readStored<Line[]>(pathFor(c, name, key), key)
66
- if (entry == null) return null
67
- const keeps = entry.keeps ?? ttlMs
68
98
  const now = c.now()
99
+ const entry = withAnswers(c.db, (a) => {
100
+ const e = readOrAdopt(a, c.ns, name, key)
101
+ if (e != null) a.write([], { touched: [{ ns: c.ns, name, key }], now })
102
+ return e
103
+ })
104
+ if (entry == null || !Array.isArray(entry.payload)) return null
105
+ const keeps = entry.keeps ?? ttlMs
69
106
  if (now - entry.at >= keeps || now < entry.at) return null
70
107
  // the stamp was baked in when the answer was MEASURED (below): a replay
71
108
  // prints the stored lines VERBATIM, so the live run and every later replay
@@ -85,27 +122,39 @@ export type Measured = {
85
122
  keepNegative: boolean
86
123
  }
87
124
 
125
+ /** is an answer stored for this exact question, without stamping it used:
126
+ * what the plan asks of a forever check */
127
+ export function hasWorld(c: WorldCache, name: string, key: string, ttlMs: number): boolean {
128
+ if (ttlMs <= 0) return false
129
+ const now = c.now()
130
+ const entry = withAnswers(c.db, (a) => readOrAdopt(a, c.ns, name, key))
131
+ if (entry == null || !Array.isArray(entry.payload)) return false
132
+ const keeps = entry.keeps ?? ttlMs
133
+ return now - entry.at < keeps && now >= entry.at
134
+ }
135
+
88
136
  /** what a measured answer leaves behind, and the lines to report for it. This
89
- * check writes its OWN file and only its own, so a racing runner can lose
137
+ * check writes its OWN row and only its own, so a racing runner can lose
90
138
  * nothing. */
91
139
  export function rememberWorld(c: WorldCache, name: string, key: string, ttlMs: number, m: Measured): Line[] {
92
140
  if (ttlMs <= 0) return m.lines
93
141
  const cacheable = !m.unmeasured && (m.clean || m.keepNegative)
94
142
  const keepsFor = m.clean ? ttlMs : Math.min(ttlMs, NEGATIVE_TTL_CAP_MS)
95
- const path = pathFor(c, name, key)
96
143
  if (!cacheable) {
97
144
  // an unmeasured run evicts whatever was stored: a later run must
98
145
  // re-probe rather than replay an answer recorded before the outage. A
99
- // racing runner may have written a good answer into that same file a
146
+ // racing runner may have written a good answer into that same row a
100
147
  // moment ago and lose it here, which costs one re-probe and can never
101
148
  // produce a stale green, so it is the safe direction.
102
- dropStored(path)
149
+ withAnswers(c.db, (a) => a.drop(c.ns, name, key))
103
150
  return m.lines
104
151
  }
105
152
  const now = c.now()
106
153
  const lines = m.lines.map((l, i): Line =>
107
- i === 0 ? { ...l, text: `${l.text} (${m.clean ? `${CACHED} at ${fmtStamp(now)} until` : `${RAN} at ${fmtStamp(now)}, kept until`} ${fmtStamp(now + keepsFor)})` } : l,
154
+ i !== 0 ? l
155
+ : keepsFor >= FOREVER_MS ? { ...l, text: `${l.text} (${CACHED} at ${fmtStamp(now)} until its declaration changes)` }
156
+ : { ...l, text: `${l.text} (${m.clean ? `${CACHED} at ${fmtStamp(now)} until` : `${RAN} at ${fmtStamp(now)}, kept until`} ${fmtStamp(now + keepsFor)})` },
108
157
  )
109
- writeStored(path, key, lines, now, keepsFor)
158
+ withAnswers(c.db, (a) => a.write([{ ns: c.ns, name, key, payload: lines, at: now, keeps: keepsFor }], { now }))
110
159
  return lines
111
160
  }