simt-dev 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/CHANGELOG.md +83 -1
  2. package/README.md +46 -17
  3. package/agents/simt-agent.md +84 -0
  4. package/bin/simt.mjs +19 -6
  5. package/docs/decisions/ADR-009-skill-layout.md +1 -1
  6. package/docs/decisions/ADR-086-the-stable-prefix-comes-first-and-the-lock-records-both-digests.md +1 -1
  7. package/docs/decisions/ADR-100-a-staged-diff-is-the-last-gate-before-history-and-it-runs-inside-the-tool.md +94 -0
  8. package/docs/decisions/ADR-101-a-file-lease-belongs-to-one-session-and-the-pre-edit-hook-refuses-another.md +95 -0
  9. package/docs/decisions/ADR-102-the-two-queued-review-rows-are-closed-by-ruling.md +49 -0
  10. package/docs/decisions/ADR-103-co-projected-duplicates-merge-down.md +74 -0
  11. package/docs/decisions/ADR-104-the-pre-tool-use-policy-reads-the-command-the-way-a-shell-reads-it.md +76 -0
  12. package/docs/decisions/ADR-105-the-json-ceiling-shrinks-arrays-and-never-cuts-bytes.md +44 -0
  13. package/docs/decisions/ADR-106-the-harness-hook-runs-through-a-repo-shim.md +46 -0
  14. package/docs/decisions/ADR-107-an-install-can-name-one-harness-and-the-skipped-ones-are-named.md +51 -0
  15. package/docs/decisions/ADR-108-an-adoption-builds-the-index-and-a-command-refreshes-it-on-read.md +59 -0
  16. package/docs/decisions/ADR-109-simt-dev-is-its-own-consumer-and-the-guards-that-kept-it-from-being-one-are-retired.md +47 -0
  17. package/docs/decisions/ADR-110-the-lock-file-follows-the-manifest-name-and-the-old-name-is-read-until-the-next-write.md +43 -0
  18. package/docs/decisions/ADR-111-the-tool-ships-one-agent-rendered-per-harness-and-installs-it-with-init-and-a-command.md +75 -0
  19. package/docs/decisions/ADR-112-the-registry-is-what-simt-dev-ships-and-a-repo-owns-its-own-skills.md +81 -0
  20. package/docs/decisions/ADR-113-a-pre-push-that-carries-only-documentation-skips-the-test-gate-and-says-so.md +87 -0
  21. package/docs/decisions/ADR-114-the-bench-pairs-at-the-task-level-a-missing-cost-is-not-zero-and-an-unasked-cross-group-compare-is-refused.md +74 -0
  22. package/docs/decisions/ADR-115-the-index-stamps-its-schema-version-migrates-older-databases-in-place-and-serves-newer-ones-from-a-sidecar.md +69 -0
  23. package/docs/decisions/ADR-116-rust-support-reaches-the-example-path-the-indent-and-the-bench-runner.md +100 -0
  24. package/docs/decisions/README.md +53 -0
  25. package/package.json +4 -2
  26. package/rules/core.md +2 -2
  27. package/rules/mdc/preview-postflight.mdc +5 -6
  28. package/rules/mdc/test-gate.mdc +2 -0
  29. package/rules/stacks/android.md +10 -11
  30. package/rules/stacks/cloudflare-workers.md +1 -1
  31. package/rules/stacks/connectiq.md +2 -2
  32. package/rules/stacks/container.md +2 -0
  33. package/rules/stacks/dart.md +1 -1
  34. package/rules/stacks/deno.md +1 -1
  35. package/rules/stacks/dotnet.md +1 -1
  36. package/rules/stacks/elixir.md +1 -1
  37. package/rules/stacks/go.md +1 -1
  38. package/rules/stacks/jvm.md +4 -3
  39. package/rules/stacks/node.md +2 -2
  40. package/rules/stacks/python.md +7 -6
  41. package/rules/stacks/react-native.md +3 -5
  42. package/rules/stacks/react-web.md +9 -9
  43. package/rules/stacks/ruby.md +1 -1
  44. package/rules/stacks/rust.md +1 -1
  45. package/rules/stacks/swift.md +1 -1
  46. package/rules/stacks/typescript.md +2 -2
  47. package/rules/stacks/uv.md +3 -2
  48. package/skills/simt-commands/SKILL.md +70 -0
  49. package/src/advice/advice.mjs +7 -3
  50. package/src/agenthooks/decide.mjs +238 -15
  51. package/src/agenthooks/opencode.mjs +72 -0
  52. package/src/agenthooks/policy.mjs +81 -5
  53. package/src/agenthooks/prompt.mjs +6 -2
  54. package/src/agenthooks/render.mjs +186 -19
  55. package/src/agenthooks/shell.mjs +523 -0
  56. package/src/agenthooks/shim.mjs +55 -0
  57. package/src/agents/render.mjs +277 -0
  58. package/src/argv.mjs +62 -9
  59. package/src/bench/ablate.mjs +31 -14
  60. package/src/bench/mine.mjs +48 -1
  61. package/src/bench/probes.mjs +5 -6
  62. package/src/bench/report.mjs +243 -80
  63. package/src/bench/run.mjs +12 -11
  64. package/src/bench/variants.mjs +112 -40
  65. package/src/brief/assemble.mjs +6 -0
  66. package/src/ci/workflow.mjs +16 -7
  67. package/src/commands/agent.mjs +206 -0
  68. package/src/commands/agenthook.mjs +133 -17
  69. package/src/commands/bench.mjs +2 -1
  70. package/src/commands/ci.mjs +28 -3
  71. package/src/commands/compare.mjs +8 -7
  72. package/src/commands/completions.mjs +27 -0
  73. package/src/commands/decide.mjs +20 -6
  74. package/src/commands/doctor.mjs +367 -0
  75. package/src/commands/evidence.mjs +13 -3
  76. package/src/commands/explain.mjs +52 -0
  77. package/src/commands/gate.mjs +85 -19
  78. package/src/commands/graph.mjs +18 -5
  79. package/src/commands/help.mjs +96 -26
  80. package/src/commands/hooks.mjs +258 -67
  81. package/src/commands/ignore.mjs +152 -0
  82. package/src/commands/impact.mjs +4 -2
  83. package/src/commands/index.mjs +69 -102
  84. package/src/commands/init.mjs +239 -37
  85. package/src/commands/lease.mjs +304 -0
  86. package/src/commands/lessons.mjs +77 -9
  87. package/src/commands/map.mjs +33 -18
  88. package/src/commands/mcp.mjs +2 -2
  89. package/src/commands/pack.mjs +9 -0
  90. package/src/commands/plan.mjs +13 -1
  91. package/src/commands/plugin.mjs +17 -2
  92. package/src/commands/prbody.mjs +45 -0
  93. package/src/commands/repoindex.mjs +4 -0
  94. package/src/commands/rules.mjs +219 -105
  95. package/src/commands/shared.mjs +36 -5
  96. package/src/commands/skills.mjs +329 -206
  97. package/src/commands/skillscompile.mjs +8 -7
  98. package/src/commands/skillsinstall.mjs +112 -30
  99. package/src/commands/slice.mjs +712 -0
  100. package/src/commands/status.mjs +62 -0
  101. package/src/commands/sync.mjs +146 -36
  102. package/src/commands/tests.mjs +5 -2
  103. package/src/commands/undo.mjs +114 -0
  104. package/src/commands/upgrade.mjs +103 -9
  105. package/src/commands/why.mjs +10 -4
  106. package/src/compare/hooksstate.mjs +23 -1
  107. package/src/compare/learn.mjs +82 -57
  108. package/src/compare/parallel.mjs +50 -0
  109. package/src/compare/row.mjs +26 -1
  110. package/src/completions.mjs +196 -0
  111. package/src/decisions/dir.mjs +84 -0
  112. package/src/decisions/index.mjs +109 -11
  113. package/src/diff.mjs +254 -0
  114. package/src/distill/distill.mjs +3 -1
  115. package/src/ecosystem/node-runner.mjs +144 -0
  116. package/src/explain.mjs +270 -0
  117. package/src/federation/config.mjs +44 -0
  118. package/src/graph/load.mjs +52 -24
  119. package/src/hooks/capability.mjs +14 -1
  120. package/src/hooks/dispatcher.mjs +441 -9
  121. package/src/hooks/gates.mjs +14 -3
  122. package/src/hooks/resolve.mjs +23 -3
  123. package/src/hooks/shim.mjs +12 -7
  124. package/src/identity.mjs +67 -14
  125. package/src/ignore.mjs +133 -0
  126. package/src/index/extract.mjs +192 -11
  127. package/src/index/persist.mjs +372 -55
  128. package/src/index/store.mjs +263 -60
  129. package/src/index/tests.mjs +6 -35
  130. package/src/journal.mjs +545 -0
  131. package/src/kernel/card.mjs +37 -12
  132. package/src/kernel/checks.mjs +44 -9
  133. package/src/kernel/variants.mjs +27 -16
  134. package/src/lease.mjs +195 -0
  135. package/src/ledger/index.mjs +91 -25
  136. package/src/mcp/resources.mjs +5 -3
  137. package/src/mcp/server.mjs +86 -13
  138. package/src/mcp/tools.mjs +340 -264
  139. package/src/memory/decide.mjs +98 -38
  140. package/src/memory/lessons.mjs +180 -33
  141. package/src/name.mjs +9 -0
  142. package/src/output/index.mjs +82 -12
  143. package/src/package-version.mjs +49 -1
  144. package/src/packs/manage.mjs +13 -4
  145. package/src/plan/index.mjs +37 -8
  146. package/src/plan/template.mjs +27 -5
  147. package/src/plugins.mjs +33 -8
  148. package/src/prbody.mjs +253 -0
  149. package/src/projection/baselines.mjs +1 -1
  150. package/src/projection/disclosure.mjs +2 -2
  151. package/src/projection/loaders.mjs +41 -1
  152. package/src/projection/lock.mjs +21 -14
  153. package/src/projection/manifest.mjs +82 -21
  154. package/src/projection/markers.mjs +43 -3
  155. package/src/projection/mdc.mjs +34 -7
  156. package/src/projection/project.mjs +162 -18
  157. package/src/projection/rules.mjs +54 -3
  158. package/src/projection/shared.mjs +4 -1
  159. package/src/projection/surfaces.mjs +9 -1
  160. package/src/repo-root.mjs +69 -0
  161. package/src/resolve/body.mjs +4 -0
  162. package/src/resolve/fingerprint.mjs +5 -0
  163. package/src/resolve/local.mjs +58 -0
  164. package/src/review/critic.mjs +4 -2
  165. package/src/secrets.mjs +374 -0
  166. package/src/state.mjs +76 -0
  167. package/src/status.mjs +157 -0
  168. package/src/toolwords.mjs +5 -0
  169. package/src/tsconfig.mjs +39 -99
  170. package/templates/plan/tracker.json +1 -1
@@ -4,18 +4,80 @@
4
4
  // rows. This file is the format; store.mjs is the walk that fills it. They are
5
5
  // split because the walk and the format together crossed the review trigger.
6
6
 
7
- import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
7
+ import { createHash } from 'node:crypto';
8
+ import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
8
9
  import { dirname, join } from 'node:path';
9
10
 
10
- export const SQLITE_FILE = '.simt/index.sqlite';
11
- export const JSONL_FILE = '.simt/index.jsonl';
12
- export const SCHEMA_V = '3';
11
+ import { LOCAL_DIR } from '../state.mjs';
12
+
13
+ export const SQLITE_FILE = join(LOCAL_DIR, 'index.sqlite');
14
+ export const JSONL_FILE = join(LOCAL_DIR, 'index.jsonl');
15
+ const SQLITE_FILE_LEGACY = '.simt/index.sqlite';
16
+ const JSONL_FILE_LEGACY = '.simt/index.jsonl';
17
+ export const SCHEMA_V = '4';
18
+
19
+ // The FTS tables carry their tokenizer in their definition, so `if not exists`
20
+ // cannot migrate them: a v3 database would keep tokenizing new rows the v3 way
21
+ // (ASCII fold, kind and ref searchable). The 3 -> 4 migration below recreates
22
+ // them, which is the versioned replacement for the per-write wipe (Q44).
23
+ export const FTS_SCHEMA = 'create virtual table fts using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "unicode61 remove_diacritics 2")';
24
+ export const FTS_TRI_SCHEMA = 'create virtual table fts_tri using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "trigram")';
25
+
26
+ // A database written by a *newer* simt is never opened for write, and never
27
+ // read blind (review 0003 Q44): the mismatched canonical file stays with its
28
+ // owner, and this version reads and writes a sidecar named after itself, so
29
+ // two pinned versions stop rewriting one file past each other (the ADR-059
30
+ // flip-flop). An *older* database is read and written in place, and the
31
+ // migration brings it current.
32
+ export function indexFileFor(root, backend, version) {
33
+ const base = backend === 'sqlite' ? SQLITE_FILE : JSONL_FILE;
34
+ if (version === '' || version === SCHEMA_V) return join(root, base);
35
+ const dot = base.lastIndexOf('.');
36
+ return join(root, `${base.slice(0, dot)}.v${SCHEMA_V}${base.slice(dot)}`);
37
+ }
38
+
39
+ // The canonical file is opened for write when the stored version is this one
40
+ // or older - the ordered migration upgrades the older one in place. A newer
41
+ // version owns its file, and opening it here would demote the stamp.
42
+ function canonicalForWrite(version) {
43
+ const at = Number(version);
44
+ return Number.isInteger(at) && at <= Number(SCHEMA_V);
45
+ }
46
+
47
+ // Ordered schema migrations: each step upgrades a database written at `from`
48
+ // to `to`, applied in list order. A database at a version past the last step
49
+ // is not opened for write at all (indexFileFor routes it to a sidecar).
50
+ const MIGRATIONS = [
51
+ {
52
+ from: 3,
53
+ to: 4,
54
+ apply: (db) => {
55
+ db.exec('drop table if exists fts');
56
+ db.exec(FTS_SCHEMA);
57
+ db.exec('drop table if exists fts_tri');
58
+ db.exec(FTS_TRI_SCHEMA);
59
+ },
60
+ },
61
+ ];
62
+
63
+ /**
64
+ * Bring an open database up to the current schema and stamp it. `user_version`
65
+ * 0 means "written before the stamp existed": those databases carried the
66
+ * version in meta only, so every step at or below the current schema runs -
67
+ * each is idempotent on an already-migrated table set.
68
+ */
69
+ function migrateDb(db) {
70
+ const at = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
71
+ for (const step of MIGRATIONS) {
72
+ if (at === 0 || (at >= step.from && at < step.to)) step.apply(db);
73
+ }
74
+ db.exec(`pragma user_version = ${Number(SCHEMA_V)}`);
75
+ }
13
76
 
14
77
  const SCHEMA = `
15
78
  create table if not exists files(path text primary key, blob text, lang text, bytes int, pkg text);
16
79
  create table if not exists symbols(id integer primary key, path text, name text, kind text, line int, sig text, exported int);
17
80
  create table if not exists edges(src text, dst text, kind text, weight real);
18
- create virtual table if not exists fts using fts5(kind, ref, body);
19
81
  create table if not exists embeddings(path text primary key, blob text, model text, dims int, vec blob);
20
82
  create table if not exists meta(key text primary key, value text);
21
83
  `;
@@ -44,59 +106,229 @@ export function decodeVectorBlob(blob) {
44
106
  return new Float32Array(copy.buffer);
45
107
  }
46
108
 
47
- export async function writeSqlite(root, next, removed, symbols, edges, fts, embeddings = []) {
109
+ let tmpSeq = 0;
110
+
111
+ /** A per-writer temp file: pid plus a sequence, so in-process concurrent writers meet on no shared path. */
112
+ function tmpFor(file) {
113
+ tmpSeq += 1;
114
+ return `${file}.${process.pid}.${tmpSeq}.tmp`;
115
+ }
116
+
117
+ /**
118
+ * What the index was built from, so a reader can tell a current tree from an
119
+ * old one without re-reading any blob (review 0003 Q42).
120
+ *
121
+ * The digest covers the `git ls-files -s` listing the build consumed: same
122
+ * bytes in, same digest out. Stored beside `schema` in `meta`, written on every
123
+ * build including the no-change one, so a build that changed nothing still
124
+ * vouches for the tree it saw.
125
+ */
126
+ export function treeDigest(listed) {
127
+ const rows = [...(listed ?? [])]
128
+ .map((entry) => `${entry.blob ?? ''} ${entry.path ?? ''}`)
129
+ .sort()
130
+ .join('\n');
131
+ return createHash('sha1').update(rows, 'utf8').digest('hex');
132
+ }
133
+
134
+ export async function writeSqlite(root, next, removed, symbols, edges, fts, embeddings = [], tree = '') {
48
135
  const { DatabaseSync } = await import('node:sqlite');
49
- const file = join(root, SQLITE_FILE);
136
+ // The canonical file is written when it is this version's or older (the
137
+ // migration upgrades the older one in place, review 0003 Q44): a database at
138
+ // a newer version stays with its owner, and the build goes to the sidecar
139
+ // this version owns.
140
+ let file = join(root, SQLITE_FILE);
141
+ try {
142
+ await stat(file);
143
+ } catch {
144
+ file = join(root, SQLITE_FILE_LEGACY);
145
+ try {
146
+ await stat(file);
147
+ } catch {
148
+ file = null;
149
+ }
150
+ }
151
+ if (file) {
152
+ let version = SCHEMA_V;
153
+ try {
154
+ version = await storedVersion(file);
155
+ } catch {
156
+ version = SCHEMA_V; // unreadable: rebuild it in place, as a cold build would
157
+ }
158
+ file = canonicalForWrite(version) ? file : indexFileFor(root, 'sqlite', version);
159
+ } else {
160
+ file = join(root, SQLITE_FILE);
161
+ }
50
162
  await mkdir(dirname(file), { recursive: true });
51
- const db = new DatabaseSync(file);
163
+ // A cold build goes to a temp file that is renamed over the target: three
164
+ // concurrent cold runs used to interleave schema creation and leave a locked,
165
+ // table-less database behind (Q08). Each writer builds its own complete file
166
+ // and the rename is atomic, so the last writer wins with a whole index.
167
+ let cold = false;
52
168
  try {
169
+ await stat(file);
170
+ } catch {
171
+ cold = true;
172
+ }
173
+ const target = cold ? tmpFor(file) : file;
174
+ if (cold) {
175
+ try {
176
+ await stat(target);
177
+ const { unlink } = await import('node:fs/promises');
178
+ await unlink(target);
179
+ } catch {
180
+ // No stale temp file: build fresh.
181
+ }
182
+ }
183
+ const db = new DatabaseSync(target);
184
+ try {
185
+ db.exec('pragma journal_mode = WAL');
186
+ db.exec('pragma busy_timeout = 5000');
187
+ db.exec('pragma synchronous = normal');
53
188
  db.exec(SCHEMA);
54
- db.exec('begin');
189
+ db.exec('begin immediate');
190
+ migrateDb(db);
55
191
  const upsert = db.prepare(`insert into files(path, blob, lang, bytes, pkg) values (?, ?, ?, ?, ?)
56
192
  on conflict(path) do update set blob = excluded.blob, lang = excluded.lang, bytes = excluded.bytes, pkg = excluded.pkg`);
57
193
  const dropSymbols = db.prepare('delete from symbols where path = ?');
58
- const dropEdges = db.prepare('delete from edges where src = ? or dst = ?');
194
+ // An importer owns its outgoing edges: re-extracting a changed file drops
195
+ // only `src = ?`, so the unchanged importers' edges pointing into it survive
196
+ // (Q07). A removed file's incoming edges dangle, so they drop with it.
197
+ const dropEdges = db.prepare('delete from edges where src = ?');
198
+ const dropIncoming = db.prepare('delete from edges where dst = ?');
59
199
  const dropFts = db.prepare('delete from fts where ref = ?');
200
+ const dropTri = db.prepare('delete from fts_tri where ref = ?');
60
201
  const dropEmbed = db.prepare('delete from embeddings where path = ?');
61
202
  const dropFile = db.prepare('delete from files where path = ?');
62
203
  const addSymbol = db.prepare('insert into symbols(path, name, kind, line, sig, exported) values (?, ?, ?, ?, ?, ?)');
63
204
  const addEdge = db.prepare('insert into edges(src, dst, kind, weight) values (?, ?, ?, ?)');
64
205
  const addFts = db.prepare('insert into fts(kind, ref, body) values (?, ?, ?)');
206
+ const addTri = db.prepare('insert into fts_tri(kind, ref, body) values (?, ?, ?)');
65
207
  const upsertEmbed = db.prepare(`insert into embeddings(path, blob, model, dims, vec) values (?, ?, ?, ?, ?)
66
208
  on conflict(path) do update set blob = excluded.blob, model = excluded.model, dims = excluded.dims, vec = excluded.vec`);
67
209
  for (const row of next) {
68
210
  upsert.run(row.path, row.blob, row.lang, row.bytes, row.pkg);
69
211
  dropSymbols.run(row.path);
70
- dropEdges.run(row.path, row.path);
212
+ dropEdges.run(row.path);
71
213
  dropFts.run(row.path);
214
+ dropTri.run(row.path);
72
215
  dropEmbed.run(row.path);
73
216
  }
74
217
  for (const path of removed) {
75
218
  dropFile.run(path);
76
219
  dropSymbols.run(path);
77
- dropEdges.run(path, path);
220
+ dropEdges.run(path);
221
+ dropIncoming.run(path);
78
222
  dropFts.run(path);
223
+ dropTri.run(path);
79
224
  dropEmbed.run(path);
80
225
  }
81
226
  for (const row of symbols) addSymbol.run(row.path, row.name, row.kind, row.line, row.sig, row.exported);
82
227
  for (const row of edges) addEdge.run(row.src, row.dst, row.kind, row.weight);
83
- for (const row of fts) addFts.run(row.kind, row.ref, row.body);
228
+ for (const row of fts) {
229
+ addFts.run(row.kind, row.ref, row.body);
230
+ // The trigram table carries folded raw text for CJK-bearing sections
231
+ // only; an ASCII-only section is already reachable through `fts`.
232
+ if (row.raw) addTri.run(row.kind, row.ref, row.raw);
233
+ }
84
234
  for (const row of embeddings) {
85
235
  const vecBlob = Buffer.isBuffer(row.vec) || row.vec instanceof Uint8Array ? row.vec : encodeVectorBlob(row.vec);
86
236
  const dims = Number.isInteger(row.dims) ? row.dims : Math.floor(vecBlob.byteLength / 4);
87
237
  upsertEmbed.run(row.path, row.blob ?? '', row.model ?? '', dims, vecBlob);
88
238
  }
89
- db.prepare('insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value').run('schema', SCHEMA_V);
239
+ const upsertMeta = db.prepare(
240
+ 'insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value',
241
+ );
242
+ upsertMeta.run('schema', SCHEMA_V);
243
+ upsertMeta.run('tree', tree);
90
244
  db.exec('commit');
91
245
  } catch (error) {
92
246
  try { db.exec('rollback'); } catch { /* a failed begin has nothing to undo */ }
93
- throw error;
247
+ throw asIndexError(error);
248
+ } finally {
249
+ db.close();
250
+ }
251
+ if (cold) await rename(target, file);
252
+ }
253
+
254
+ /**
255
+ * The version a database declares: `PRAGMA user_version` when stamped, else the
256
+ * meta `schema` row databases written before the stamp carried it in. Throws
257
+ * when the file cannot be read as a database - the caller decides whether that
258
+ * is a cold start or a corrupt one.
259
+ */
260
+ async function storedVersion(file) {
261
+ const { DatabaseSync } = await import('node:sqlite');
262
+ const db = new DatabaseSync(file);
263
+ try {
264
+ const user = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
265
+ if (user > 0) return String(user);
266
+ return db.prepare("select value from meta where key = 'schema'").get()?.value ?? '';
94
267
  } finally {
95
268
  db.close();
96
269
  }
97
270
  }
98
271
 
99
- export async function writeJsonl(root, previous, next, removed, symbols, edges, fts, force, embeddings = []) {
272
+ /** A damaged index file is named, not silent (review 0003 Q44). */
273
+ function corruptIndexError(cause) {
274
+ const corrupt = new Error('the index file is corrupt; run the index build to replace it');
275
+ corrupt.code = 'INDEX_CORRUPT';
276
+ corrupt.cause = cause;
277
+ return corrupt;
278
+ }
279
+
280
+ /**
281
+ * A contended index write is a retryable race, not a broken repository: name it
282
+ * so the command maps its own remedy instead of blaming the git checkout (Q08).
283
+ */
284
+ function asIndexError(error) {
285
+ const code = error?.code ?? error?.errno;
286
+ if (code === 'SQLITE_BUSY' || code === 5 || /database is locked/i.test(error?.message ?? '')) {
287
+ const busy = new Error('another index run is writing; wait a moment and run again');
288
+ busy.code = 'INDEX_BUSY';
289
+ busy.cause = error;
290
+ return busy;
291
+ }
292
+ if (/file is not a database|database disk image|malformed|corrupt/i.test(error?.message ?? '')) {
293
+ return corruptIndexError(error);
294
+ }
295
+ return error;
296
+ }
297
+
298
+ /**
299
+ * The jsonl file this version writes to: the canonical file when it is this
300
+ * version's or unreadable, the sidecar when it belongs to another version
301
+ * (review 0003 Q44).
302
+ */
303
+ async function jsonlTargetFile(root) {
304
+ const base = join(root, JSONL_FILE);
305
+ let text = null;
306
+ try {
307
+ text = await readFile(base, 'utf8');
308
+ } catch {
309
+ try {
310
+ text = await readFile(join(root, JSONL_FILE_LEGACY), 'utf8');
311
+ } catch {
312
+ return base; // no file: cold build goes to the canonical path
313
+ }
314
+ }
315
+ try {
316
+ for (const line of text.split('\n')) {
317
+ if (line === '') continue;
318
+ const row = JSON.parse(line);
319
+ if (row.table !== 'meta' || row.key !== 'schema') continue;
320
+ const value = row.value ?? '';
321
+ // Older files are rewritten in place at the current schema; a newer one
322
+ // stays with its owner and the build goes to the sidecar.
323
+ return canonicalForWrite(value) ? base : indexFileFor(root, 'jsonl', value);
324
+ }
325
+ } catch {
326
+ return base; // unreadable: rebuild it in place
327
+ }
328
+ return base;
329
+ }
330
+
331
+ export async function writeJsonl(root, previous, next, removed, symbols, edges, fts, force, embeddings = [], tree = '') {
100
332
  const gone = new Set(removed);
101
333
  const replaced = new Map(next.map((row) => [row.path, row]));
102
334
  const drop = new Set([...replaced.keys(), ...gone]);
@@ -108,21 +340,26 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
108
340
  }
109
341
  for (const row of replaced.values()) files.push(row);
110
342
  files.sort(byPath);
111
- const keptSymbols = force ? [] : (await readJsonl(root, 'symbols')).filter((row) => !drop.has(row.path));
112
- const keptEdges = force ? [] : (await readJsonl(root, 'edges')).filter((row) => !drop.has(row.src) && !drop.has(row.dst));
113
- const keptFts = force ? [] : (await readJsonl(root, 'fts')).filter((row) => !drop.has(row.ref));
343
+ const keptSymbols = force ? [] : (await readJsonl(root, 'symbols')).rows.filter((row) => !drop.has(row.path));
344
+ // Edge ownership mirrors the sqlite backend: a changed file loses only the edges
345
+ // it owns (src), while a removed file also drops the edges that pointed at it.
346
+ const keptEdges = force ? [] : (await readJsonl(root, 'edges')).rows.filter((row) => !drop.has(row.src) && !gone.has(row.dst));
347
+ const keptFts = force ? [] : (await readJsonl(root, 'fts')).rows.filter((row) => !drop.has(row.ref));
114
348
  const newEmbedPaths = new Set(embeddings.map((row) => row.path));
115
349
  const keptEmbeddings = force
116
350
  ? []
117
- : (await readJsonl(root, 'embeddings')).filter((row) => !drop.has(row.path) && !newEmbedPaths.has(row.path));
351
+ : (await readJsonl(root, 'embeddings')).rows.filter((row) => !drop.has(row.path) && !newEmbedPaths.has(row.path));
118
352
  const allSymbols = [...keptSymbols, ...symbols].sort(bySymbol);
119
353
  const allEdges = [...keptEdges, ...edges].sort(byEdge);
120
354
  const allFts = [...keptFts, ...fts].sort((a, b) => cmp(a.ref, b.ref));
121
355
  const allEmbeddings = [...keptEmbeddings, ...embeddings].sort(byPath);
122
- const file = join(root, JSONL_FILE);
356
+ const file = await jsonlTargetFile(root);
123
357
  await mkdir(dirname(file), { recursive: true });
358
+ // Written atomically: a concurrent reader never sees a half-written file.
359
+ const tmp = tmpFor(file);
124
360
  const lines = [
125
361
  JSON.stringify({ table: 'meta', key: 'schema', value: SCHEMA_V }),
362
+ JSON.stringify({ table: 'meta', key: 'tree', value: tree }),
126
363
  ...files.map((row) => JSON.stringify({ table: 'files', path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg })),
127
364
  ...allSymbols.map((row) => JSON.stringify({ table: 'symbols', path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported })),
128
365
  ...allEdges.map((row) => JSON.stringify({ table: 'edges', src: row.src, dst: row.dst, kind: row.kind, weight: row.weight })),
@@ -140,63 +377,133 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
140
377
  });
141
378
  }),
142
379
  ];
143
- await writeFile(file, `${lines.join('\n')}\n`);
380
+ await writeFile(tmp, `${lines.join('\n')}\n`);
381
+ await rename(tmp, file);
144
382
  return { symbols: allSymbols.length, edges: allEdges.length, embeddings: allEmbeddings.length };
145
383
  }
146
384
 
147
385
  export async function readJsonl(root, table) {
386
+ // The pre-move path is a fallback for one release (review 0003 Q19).
148
387
  let text;
149
388
  try {
150
389
  text = await readFile(join(root, JSONL_FILE), 'utf8');
151
390
  } catch {
152
- return [];
391
+ try {
392
+ text = await readFile(join(root, JSONL_FILE_LEGACY), 'utf8');
393
+ } catch {
394
+ return { rows: [], error: null };
395
+ }
396
+ }
397
+ let schema = '';
398
+ try {
399
+ for (const line of text.split('\n')) {
400
+ if (line === '') continue;
401
+ const row = JSON.parse(line);
402
+ if (row.table === 'meta' && row.key === 'schema') schema = row.value ?? '';
403
+ }
404
+ } catch (error) {
405
+ return { rows: [], error: corruptIndexError(error) };
406
+ }
407
+ // A file a newer version owns is not read blind (review 0003 Q44): this
408
+ // version's sidecar is read instead, and a sidecar that has not been built
409
+ // yet is cold, not broken. An older file is read in place; the next write
410
+ // rewrites it at the current schema.
411
+ if (!canonicalForWrite(schema)) {
412
+ try {
413
+ text = await readFile(indexFileFor(root, 'jsonl', schema), 'utf8');
414
+ } catch {
415
+ return { rows: [], error: null };
416
+ }
153
417
  }
154
418
  const rows = [];
155
- for (const line of text.split('\n')) {
156
- if (line === '') continue;
157
- const row = JSON.parse(line);
158
- if (row.table !== table) continue;
159
- if (table === 'meta') rows.push({ key: row.key, value: row.value });
160
- else if (table === 'files') rows.push({ path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg });
161
- else if (table === 'symbols') rows.push({ path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported });
162
- else if (table === 'edges') rows.push({ src: row.src, dst: row.dst, kind: row.kind, weight: row.weight });
163
- else if (table === 'fts') rows.push({ kind: row.kind, ref: row.ref, body: row.body });
164
- else if (table === 'embeddings') {
165
- const vec = decodeVectorBlob(row.vec);
166
- rows.push({
167
- path: row.path,
168
- blob: row.blob ?? '',
169
- model: row.model ?? '',
170
- dims: Number.isInteger(row.dims) ? row.dims : vec.length,
171
- vec,
172
- });
419
+ try {
420
+ for (const line of text.split('\n')) {
421
+ if (line === '') continue;
422
+ const row = JSON.parse(line);
423
+ if (row.table !== table) continue;
424
+ if (table === 'meta') rows.push({ key: row.key, value: row.value });
425
+ else if (table === 'files') rows.push({ path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg });
426
+ else if (table === 'symbols') rows.push({ path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported });
427
+ else if (table === 'edges') rows.push({ src: row.src, dst: row.dst, kind: row.kind, weight: row.weight });
428
+ else if (table === 'fts') rows.push({ kind: row.kind, ref: row.ref, body: row.body });
429
+ else if (table === 'embeddings') {
430
+ const vec = decodeVectorBlob(row.vec);
431
+ rows.push({
432
+ path: row.path,
433
+ blob: row.blob ?? '',
434
+ model: row.model ?? '',
435
+ dims: Number.isInteger(row.dims) ? row.dims : vec.length,
436
+ vec,
437
+ });
438
+ }
173
439
  }
440
+ } catch (error) {
441
+ return { rows: [], error: corruptIndexError(error) };
174
442
  }
175
443
  if (table === 'symbols') rows.sort(bySymbol);
176
444
  else if (table === 'edges') rows.sort(byEdge);
177
445
  else if (table === 'fts') rows.sort((a, b) => cmp(a.ref, b.ref));
178
446
  else rows.sort(byPath);
179
- return rows;
447
+ return { rows, error: null };
448
+ }
449
+
450
+ /**
451
+ * The sqlite file a read opens: the canonical file when it is this version's
452
+ * or older, the sidecar when it belongs to a newer one (review 0003 Q44),
453
+ * null when no index for this version exists. Throws when a present file
454
+ * cannot be read as a database - corrupt, not cold.
455
+ */
456
+ async function resolveSqliteFile(root) {
457
+ // The pre-move path is a fallback for one release (review 0003 Q19).
458
+ let file = join(root, SQLITE_FILE);
459
+ try {
460
+ await stat(file);
461
+ } catch {
462
+ file = join(root, SQLITE_FILE_LEGACY);
463
+ try {
464
+ await stat(file);
465
+ } catch {
466
+ return null;
467
+ }
468
+ }
469
+ const version = await storedVersion(file);
470
+ // An older database is readable in place: its tables carry the same columns,
471
+ // and the next write migrates it. Only a newer one is routed to a sidecar.
472
+ if (canonicalForWrite(version)) return file;
473
+ const sidecar = indexFileFor(root, 'sqlite', version);
474
+ try {
475
+ await stat(sidecar);
476
+ return sidecar;
477
+ } catch {
478
+ return null; // this version has not built its sidecar yet: cold
479
+ }
180
480
  }
181
481
 
482
+ /**
483
+ * Read rows from the index. Returns `{ rows, error }`: a missing index is
484
+ * `{ rows: [], error: null }` (cold), a present index that cannot be read is
485
+ * an empty row set with a named error - "locked", "no such column" and
486
+ * "corrupt" used to all look like "no results" (review 0003 Q44).
487
+ */
182
488
  export async function readSqliteRows(root, sql, params = []) {
183
489
  let DatabaseSync;
184
490
  try {
185
491
  ({ DatabaseSync } = await import('node:sqlite'));
186
492
  } catch {
187
- return [];
493
+ return { rows: [], error: null };
188
494
  }
189
- const file = join(root, SQLITE_FILE);
495
+ let file;
190
496
  try {
191
- await stat(file);
192
- } catch {
193
- return [];
497
+ file = await resolveSqliteFile(root);
498
+ } catch (error) {
499
+ return { rows: [], error: asIndexError(error) };
194
500
  }
501
+ if (!file) return { rows: [], error: null };
195
502
  const db = new DatabaseSync(file);
196
503
  try {
197
- return db.prepare(sql).all(...params);
198
- } catch {
199
- return [];
504
+ return { rows: db.prepare(sql).all(...params), error: null };
505
+ } catch (error) {
506
+ return { rows: [], error: asIndexError(error) };
200
507
  } finally {
201
508
  db.close();
202
509
  }
@@ -204,21 +511,31 @@ export async function readSqliteRows(root, sql, params = []) {
204
511
 
205
512
  export async function readSchema(root, backend) {
206
513
  if (backend === 'sqlite') {
207
- const rows = await readSqliteRows(root, "select value from meta where key = 'schema'");
514
+ const { rows } = await readSqliteRows(root, "select value from meta where key = 'schema'");
208
515
  return rows[0]?.value ?? '';
209
516
  }
210
- const rows = await readJsonl(root, 'meta');
517
+ const { rows } = await readJsonl(root, 'meta');
211
518
  return rows.find((row) => row.key === 'schema')?.value ?? '';
212
519
  }
213
520
 
521
+ /** The tree the stored index was built from, or '' when no build has vouched for one. */
522
+ export async function readTreeDigest(root, backend) {
523
+ if (backend === 'sqlite') {
524
+ const { rows } = await readSqliteRows(root, "select value from meta where key = 'tree'");
525
+ return rows[0]?.value ?? '';
526
+ }
527
+ const { rows } = await readJsonl(root, 'meta');
528
+ return rows.find((row) => row.key === 'tree')?.value ?? '';
529
+ }
530
+
214
531
  export async function countSqlite(root) {
215
532
  const symbols = await readSqliteRows(root, 'select count(*) as n from symbols');
216
533
  const edges = await readSqliteRows(root, 'select count(*) as n from edges');
217
534
  const embeddings = await readSqliteRows(root, 'select count(*) as n from embeddings');
218
535
  return {
219
- symbols: Number(symbols[0]?.n ?? 0),
220
- edges: Number(edges[0]?.n ?? 0),
221
- embeddings: Number(embeddings[0]?.n ?? 0),
536
+ symbols: Number(symbols.rows[0]?.n ?? 0),
537
+ edges: Number(edges.rows[0]?.n ?? 0),
538
+ embeddings: Number(embeddings.rows[0]?.n ?? 0),
222
539
  };
223
540
  }
224
541