simt-dev 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +83 -1
- package/README.md +46 -17
- package/agents/simt-agent.md +84 -0
- package/bin/simt.mjs +19 -6
- package/docs/decisions/ADR-009-skill-layout.md +1 -1
- package/docs/decisions/ADR-086-the-stable-prefix-comes-first-and-the-lock-records-both-digests.md +1 -1
- package/docs/decisions/ADR-100-a-staged-diff-is-the-last-gate-before-history-and-it-runs-inside-the-tool.md +94 -0
- package/docs/decisions/ADR-101-a-file-lease-belongs-to-one-session-and-the-pre-edit-hook-refuses-another.md +95 -0
- package/docs/decisions/ADR-102-the-two-queued-review-rows-are-closed-by-ruling.md +49 -0
- package/docs/decisions/ADR-103-co-projected-duplicates-merge-down.md +74 -0
- package/docs/decisions/ADR-104-the-pre-tool-use-policy-reads-the-command-the-way-a-shell-reads-it.md +76 -0
- package/docs/decisions/ADR-105-the-json-ceiling-shrinks-arrays-and-never-cuts-bytes.md +44 -0
- package/docs/decisions/ADR-106-the-harness-hook-runs-through-a-repo-shim.md +46 -0
- package/docs/decisions/ADR-107-an-install-can-name-one-harness-and-the-skipped-ones-are-named.md +51 -0
- package/docs/decisions/ADR-108-an-adoption-builds-the-index-and-a-command-refreshes-it-on-read.md +59 -0
- package/docs/decisions/ADR-109-simt-dev-is-its-own-consumer-and-the-guards-that-kept-it-from-being-one-are-retired.md +47 -0
- package/docs/decisions/ADR-110-the-lock-file-follows-the-manifest-name-and-the-old-name-is-read-until-the-next-write.md +43 -0
- package/docs/decisions/ADR-111-the-tool-ships-one-agent-rendered-per-harness-and-installs-it-with-init-and-a-command.md +75 -0
- package/docs/decisions/ADR-112-the-registry-is-what-simt-dev-ships-and-a-repo-owns-its-own-skills.md +81 -0
- package/docs/decisions/ADR-113-a-pre-push-that-carries-only-documentation-skips-the-test-gate-and-says-so.md +87 -0
- package/docs/decisions/ADR-114-the-bench-pairs-at-the-task-level-a-missing-cost-is-not-zero-and-an-unasked-cross-group-compare-is-refused.md +74 -0
- package/docs/decisions/ADR-115-the-index-stamps-its-schema-version-migrates-older-databases-in-place-and-serves-newer-ones-from-a-sidecar.md +69 -0
- package/docs/decisions/ADR-116-rust-support-reaches-the-example-path-the-indent-and-the-bench-runner.md +100 -0
- package/docs/decisions/README.md +53 -0
- package/package.json +4 -2
- package/rules/core.md +2 -2
- package/rules/mdc/preview-postflight.mdc +5 -6
- package/rules/mdc/test-gate.mdc +2 -0
- package/rules/stacks/android.md +10 -11
- package/rules/stacks/cloudflare-workers.md +1 -1
- package/rules/stacks/connectiq.md +2 -2
- package/rules/stacks/container.md +2 -0
- package/rules/stacks/dart.md +1 -1
- package/rules/stacks/deno.md +1 -1
- package/rules/stacks/dotnet.md +1 -1
- package/rules/stacks/elixir.md +1 -1
- package/rules/stacks/go.md +1 -1
- package/rules/stacks/jvm.md +4 -3
- package/rules/stacks/node.md +2 -2
- package/rules/stacks/python.md +7 -6
- package/rules/stacks/react-native.md +3 -5
- package/rules/stacks/react-web.md +9 -9
- package/rules/stacks/ruby.md +1 -1
- package/rules/stacks/rust.md +1 -1
- package/rules/stacks/swift.md +1 -1
- package/rules/stacks/typescript.md +2 -2
- package/rules/stacks/uv.md +3 -2
- package/skills/simt-commands/SKILL.md +70 -0
- package/src/advice/advice.mjs +7 -3
- package/src/agenthooks/decide.mjs +238 -15
- package/src/agenthooks/opencode.mjs +72 -0
- package/src/agenthooks/policy.mjs +81 -5
- package/src/agenthooks/prompt.mjs +6 -2
- package/src/agenthooks/render.mjs +186 -19
- package/src/agenthooks/shell.mjs +523 -0
- package/src/agenthooks/shim.mjs +55 -0
- package/src/agents/render.mjs +277 -0
- package/src/argv.mjs +62 -9
- package/src/bench/ablate.mjs +31 -14
- package/src/bench/mine.mjs +48 -1
- package/src/bench/probes.mjs +5 -6
- package/src/bench/report.mjs +243 -80
- package/src/bench/run.mjs +12 -11
- package/src/bench/variants.mjs +112 -40
- package/src/brief/assemble.mjs +6 -0
- package/src/ci/workflow.mjs +16 -7
- package/src/commands/agent.mjs +206 -0
- package/src/commands/agenthook.mjs +133 -17
- package/src/commands/bench.mjs +2 -1
- package/src/commands/ci.mjs +28 -3
- package/src/commands/compare.mjs +8 -7
- package/src/commands/completions.mjs +27 -0
- package/src/commands/decide.mjs +20 -6
- package/src/commands/doctor.mjs +367 -0
- package/src/commands/evidence.mjs +13 -3
- package/src/commands/explain.mjs +52 -0
- package/src/commands/gate.mjs +85 -19
- package/src/commands/graph.mjs +18 -5
- package/src/commands/help.mjs +96 -26
- package/src/commands/hooks.mjs +258 -67
- package/src/commands/ignore.mjs +152 -0
- package/src/commands/impact.mjs +4 -2
- package/src/commands/index.mjs +69 -102
- package/src/commands/init.mjs +239 -37
- package/src/commands/lease.mjs +304 -0
- package/src/commands/lessons.mjs +77 -9
- package/src/commands/map.mjs +33 -18
- package/src/commands/mcp.mjs +2 -2
- package/src/commands/pack.mjs +9 -0
- package/src/commands/plan.mjs +13 -1
- package/src/commands/plugin.mjs +17 -2
- package/src/commands/prbody.mjs +45 -0
- package/src/commands/repoindex.mjs +4 -0
- package/src/commands/rules.mjs +219 -105
- package/src/commands/shared.mjs +36 -5
- package/src/commands/skills.mjs +329 -206
- package/src/commands/skillscompile.mjs +8 -7
- package/src/commands/skillsinstall.mjs +112 -30
- package/src/commands/slice.mjs +712 -0
- package/src/commands/status.mjs +62 -0
- package/src/commands/sync.mjs +146 -36
- package/src/commands/tests.mjs +5 -2
- package/src/commands/undo.mjs +114 -0
- package/src/commands/upgrade.mjs +103 -9
- package/src/commands/why.mjs +10 -4
- package/src/compare/hooksstate.mjs +23 -1
- package/src/compare/learn.mjs +82 -57
- package/src/compare/parallel.mjs +50 -0
- package/src/compare/row.mjs +26 -1
- package/src/completions.mjs +196 -0
- package/src/decisions/dir.mjs +84 -0
- package/src/decisions/index.mjs +109 -11
- package/src/diff.mjs +254 -0
- package/src/distill/distill.mjs +3 -1
- package/src/ecosystem/node-runner.mjs +144 -0
- package/src/explain.mjs +270 -0
- package/src/federation/config.mjs +44 -0
- package/src/graph/load.mjs +52 -24
- package/src/hooks/capability.mjs +14 -1
- package/src/hooks/dispatcher.mjs +441 -9
- package/src/hooks/gates.mjs +14 -3
- package/src/hooks/resolve.mjs +23 -3
- package/src/hooks/shim.mjs +12 -7
- package/src/identity.mjs +67 -14
- package/src/ignore.mjs +133 -0
- package/src/index/extract.mjs +192 -11
- package/src/index/persist.mjs +372 -55
- package/src/index/store.mjs +263 -60
- package/src/index/tests.mjs +6 -35
- package/src/journal.mjs +545 -0
- package/src/kernel/card.mjs +37 -12
- package/src/kernel/checks.mjs +44 -9
- package/src/kernel/variants.mjs +27 -16
- package/src/lease.mjs +195 -0
- package/src/ledger/index.mjs +91 -25
- package/src/mcp/resources.mjs +5 -3
- package/src/mcp/server.mjs +86 -13
- package/src/mcp/tools.mjs +340 -264
- package/src/memory/decide.mjs +98 -38
- package/src/memory/lessons.mjs +180 -33
- package/src/name.mjs +9 -0
- package/src/output/index.mjs +82 -12
- package/src/package-version.mjs +49 -1
- package/src/packs/manage.mjs +13 -4
- package/src/plan/index.mjs +37 -8
- package/src/plan/template.mjs +27 -5
- package/src/plugins.mjs +33 -8
- package/src/prbody.mjs +253 -0
- package/src/projection/baselines.mjs +1 -1
- package/src/projection/disclosure.mjs +2 -2
- package/src/projection/loaders.mjs +41 -1
- package/src/projection/lock.mjs +21 -14
- package/src/projection/manifest.mjs +82 -21
- package/src/projection/markers.mjs +43 -3
- package/src/projection/mdc.mjs +34 -7
- package/src/projection/project.mjs +162 -18
- package/src/projection/rules.mjs +54 -3
- package/src/projection/shared.mjs +4 -1
- package/src/projection/surfaces.mjs +9 -1
- package/src/repo-root.mjs +69 -0
- package/src/resolve/body.mjs +4 -0
- package/src/resolve/fingerprint.mjs +5 -0
- package/src/resolve/local.mjs +58 -0
- package/src/review/critic.mjs +4 -2
- package/src/secrets.mjs +374 -0
- package/src/state.mjs +76 -0
- package/src/status.mjs +157 -0
- package/src/toolwords.mjs +5 -0
- package/src/tsconfig.mjs +39 -99
- package/templates/plan/tracker.json +1 -1
package/src/index/persist.mjs
CHANGED
|
@@ -4,18 +4,80 @@
|
|
|
4
4
|
// rows. This file is the format; store.mjs is the walk that fills it. They are
|
|
5
5
|
// split because the walk and the format together crossed the review trigger.
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import { createHash } from 'node:crypto';
|
|
8
|
+
import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
|
|
8
9
|
import { dirname, join } from 'node:path';
|
|
9
10
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
export const
|
|
11
|
+
import { LOCAL_DIR } from '../state.mjs';
|
|
12
|
+
|
|
13
|
+
export const SQLITE_FILE = join(LOCAL_DIR, 'index.sqlite');
|
|
14
|
+
export const JSONL_FILE = join(LOCAL_DIR, 'index.jsonl');
|
|
15
|
+
const SQLITE_FILE_LEGACY = '.simt/index.sqlite';
|
|
16
|
+
const JSONL_FILE_LEGACY = '.simt/index.jsonl';
|
|
17
|
+
export const SCHEMA_V = '4';
|
|
18
|
+
|
|
19
|
+
// The FTS tables carry their tokenizer in their definition, so `if not exists`
|
|
20
|
+
// cannot migrate them: a v3 database would keep tokenizing new rows the v3 way
|
|
21
|
+
// (ASCII fold, kind and ref searchable). The 3 -> 4 migration below recreates
|
|
22
|
+
// them, which is the versioned replacement for the per-write wipe (Q44).
|
|
23
|
+
export const FTS_SCHEMA = 'create virtual table fts using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "unicode61 remove_diacritics 2")';
|
|
24
|
+
export const FTS_TRI_SCHEMA = 'create virtual table fts_tri using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "trigram")';
|
|
25
|
+
|
|
26
|
+
// A database written by a *newer* simt is never opened for write, and never
|
|
27
|
+
// read blind (review 0003 Q44): the mismatched canonical file stays with its
|
|
28
|
+
// owner, and this version reads and writes a sidecar named after itself, so
|
|
29
|
+
// two pinned versions stop rewriting one file past each other (the ADR-059
|
|
30
|
+
// flip-flop). An *older* database is read and written in place, and the
|
|
31
|
+
// migration brings it current.
|
|
32
|
+
export function indexFileFor(root, backend, version) {
|
|
33
|
+
const base = backend === 'sqlite' ? SQLITE_FILE : JSONL_FILE;
|
|
34
|
+
if (version === '' || version === SCHEMA_V) return join(root, base);
|
|
35
|
+
const dot = base.lastIndexOf('.');
|
|
36
|
+
return join(root, `${base.slice(0, dot)}.v${SCHEMA_V}${base.slice(dot)}`);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// The canonical file is opened for write when the stored version is this one
|
|
40
|
+
// or older - the ordered migration upgrades the older one in place. A newer
|
|
41
|
+
// version owns its file, and opening it here would demote the stamp.
|
|
42
|
+
function canonicalForWrite(version) {
|
|
43
|
+
const at = Number(version);
|
|
44
|
+
return Number.isInteger(at) && at <= Number(SCHEMA_V);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Ordered schema migrations: each step upgrades a database written at `from`
|
|
48
|
+
// to `to`, applied in list order. A database at a version past the last step
|
|
49
|
+
// is not opened for write at all (indexFileFor routes it to a sidecar).
|
|
50
|
+
const MIGRATIONS = [
|
|
51
|
+
{
|
|
52
|
+
from: 3,
|
|
53
|
+
to: 4,
|
|
54
|
+
apply: (db) => {
|
|
55
|
+
db.exec('drop table if exists fts');
|
|
56
|
+
db.exec(FTS_SCHEMA);
|
|
57
|
+
db.exec('drop table if exists fts_tri');
|
|
58
|
+
db.exec(FTS_TRI_SCHEMA);
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
];
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Bring an open database up to the current schema and stamp it. `user_version`
|
|
65
|
+
* 0 means "written before the stamp existed": those databases carried the
|
|
66
|
+
* version in meta only, so every step at or below the current schema runs -
|
|
67
|
+
* each is idempotent on an already-migrated table set.
|
|
68
|
+
*/
|
|
69
|
+
function migrateDb(db) {
|
|
70
|
+
const at = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
|
|
71
|
+
for (const step of MIGRATIONS) {
|
|
72
|
+
if (at === 0 || (at >= step.from && at < step.to)) step.apply(db);
|
|
73
|
+
}
|
|
74
|
+
db.exec(`pragma user_version = ${Number(SCHEMA_V)}`);
|
|
75
|
+
}
|
|
13
76
|
|
|
14
77
|
const SCHEMA = `
|
|
15
78
|
create table if not exists files(path text primary key, blob text, lang text, bytes int, pkg text);
|
|
16
79
|
create table if not exists symbols(id integer primary key, path text, name text, kind text, line int, sig text, exported int);
|
|
17
80
|
create table if not exists edges(src text, dst text, kind text, weight real);
|
|
18
|
-
create virtual table if not exists fts using fts5(kind, ref, body);
|
|
19
81
|
create table if not exists embeddings(path text primary key, blob text, model text, dims int, vec blob);
|
|
20
82
|
create table if not exists meta(key text primary key, value text);
|
|
21
83
|
`;
|
|
@@ -44,59 +106,229 @@ export function decodeVectorBlob(blob) {
|
|
|
44
106
|
return new Float32Array(copy.buffer);
|
|
45
107
|
}
|
|
46
108
|
|
|
47
|
-
|
|
109
|
+
let tmpSeq = 0;
|
|
110
|
+
|
|
111
|
+
/** A per-writer temp file: pid plus a sequence, so in-process concurrent writers meet on no shared path. */
|
|
112
|
+
function tmpFor(file) {
|
|
113
|
+
tmpSeq += 1;
|
|
114
|
+
return `${file}.${process.pid}.${tmpSeq}.tmp`;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* What the index was built from, so a reader can tell a current tree from an
|
|
119
|
+
* old one without re-reading any blob (review 0003 Q42).
|
|
120
|
+
*
|
|
121
|
+
* The digest covers the `git ls-files -s` listing the build consumed: same
|
|
122
|
+
* bytes in, same digest out. Stored beside `schema` in `meta`, written on every
|
|
123
|
+
* build including the no-change one, so a build that changed nothing still
|
|
124
|
+
* vouches for the tree it saw.
|
|
125
|
+
*/
|
|
126
|
+
export function treeDigest(listed) {
|
|
127
|
+
const rows = [...(listed ?? [])]
|
|
128
|
+
.map((entry) => `${entry.blob ?? ''} ${entry.path ?? ''}`)
|
|
129
|
+
.sort()
|
|
130
|
+
.join('\n');
|
|
131
|
+
return createHash('sha1').update(rows, 'utf8').digest('hex');
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export async function writeSqlite(root, next, removed, symbols, edges, fts, embeddings = [], tree = '') {
|
|
48
135
|
const { DatabaseSync } = await import('node:sqlite');
|
|
49
|
-
|
|
136
|
+
// The canonical file is written when it is this version's or older (the
|
|
137
|
+
// migration upgrades the older one in place, review 0003 Q44): a database at
|
|
138
|
+
// a newer version stays with its owner, and the build goes to the sidecar
|
|
139
|
+
// this version owns.
|
|
140
|
+
let file = join(root, SQLITE_FILE);
|
|
141
|
+
try {
|
|
142
|
+
await stat(file);
|
|
143
|
+
} catch {
|
|
144
|
+
file = join(root, SQLITE_FILE_LEGACY);
|
|
145
|
+
try {
|
|
146
|
+
await stat(file);
|
|
147
|
+
} catch {
|
|
148
|
+
file = null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (file) {
|
|
152
|
+
let version = SCHEMA_V;
|
|
153
|
+
try {
|
|
154
|
+
version = await storedVersion(file);
|
|
155
|
+
} catch {
|
|
156
|
+
version = SCHEMA_V; // unreadable: rebuild it in place, as a cold build would
|
|
157
|
+
}
|
|
158
|
+
file = canonicalForWrite(version) ? file : indexFileFor(root, 'sqlite', version);
|
|
159
|
+
} else {
|
|
160
|
+
file = join(root, SQLITE_FILE);
|
|
161
|
+
}
|
|
50
162
|
await mkdir(dirname(file), { recursive: true });
|
|
51
|
-
|
|
163
|
+
// A cold build goes to a temp file that is renamed over the target: three
|
|
164
|
+
// concurrent cold runs used to interleave schema creation and leave a locked,
|
|
165
|
+
// table-less database behind (Q08). Each writer builds its own complete file
|
|
166
|
+
// and the rename is atomic, so the last writer wins with a whole index.
|
|
167
|
+
let cold = false;
|
|
52
168
|
try {
|
|
169
|
+
await stat(file);
|
|
170
|
+
} catch {
|
|
171
|
+
cold = true;
|
|
172
|
+
}
|
|
173
|
+
const target = cold ? tmpFor(file) : file;
|
|
174
|
+
if (cold) {
|
|
175
|
+
try {
|
|
176
|
+
await stat(target);
|
|
177
|
+
const { unlink } = await import('node:fs/promises');
|
|
178
|
+
await unlink(target);
|
|
179
|
+
} catch {
|
|
180
|
+
// No stale temp file: build fresh.
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
const db = new DatabaseSync(target);
|
|
184
|
+
try {
|
|
185
|
+
db.exec('pragma journal_mode = WAL');
|
|
186
|
+
db.exec('pragma busy_timeout = 5000');
|
|
187
|
+
db.exec('pragma synchronous = normal');
|
|
53
188
|
db.exec(SCHEMA);
|
|
54
|
-
db.exec('begin');
|
|
189
|
+
db.exec('begin immediate');
|
|
190
|
+
migrateDb(db);
|
|
55
191
|
const upsert = db.prepare(`insert into files(path, blob, lang, bytes, pkg) values (?, ?, ?, ?, ?)
|
|
56
192
|
on conflict(path) do update set blob = excluded.blob, lang = excluded.lang, bytes = excluded.bytes, pkg = excluded.pkg`);
|
|
57
193
|
const dropSymbols = db.prepare('delete from symbols where path = ?');
|
|
58
|
-
|
|
194
|
+
// An importer owns its outgoing edges: re-extracting a changed file drops
|
|
195
|
+
// only `src = ?`, so the unchanged importers' edges pointing into it survive
|
|
196
|
+
// (Q07). A removed file's incoming edges dangle, so they drop with it.
|
|
197
|
+
const dropEdges = db.prepare('delete from edges where src = ?');
|
|
198
|
+
const dropIncoming = db.prepare('delete from edges where dst = ?');
|
|
59
199
|
const dropFts = db.prepare('delete from fts where ref = ?');
|
|
200
|
+
const dropTri = db.prepare('delete from fts_tri where ref = ?');
|
|
60
201
|
const dropEmbed = db.prepare('delete from embeddings where path = ?');
|
|
61
202
|
const dropFile = db.prepare('delete from files where path = ?');
|
|
62
203
|
const addSymbol = db.prepare('insert into symbols(path, name, kind, line, sig, exported) values (?, ?, ?, ?, ?, ?)');
|
|
63
204
|
const addEdge = db.prepare('insert into edges(src, dst, kind, weight) values (?, ?, ?, ?)');
|
|
64
205
|
const addFts = db.prepare('insert into fts(kind, ref, body) values (?, ?, ?)');
|
|
206
|
+
const addTri = db.prepare('insert into fts_tri(kind, ref, body) values (?, ?, ?)');
|
|
65
207
|
const upsertEmbed = db.prepare(`insert into embeddings(path, blob, model, dims, vec) values (?, ?, ?, ?, ?)
|
|
66
208
|
on conflict(path) do update set blob = excluded.blob, model = excluded.model, dims = excluded.dims, vec = excluded.vec`);
|
|
67
209
|
for (const row of next) {
|
|
68
210
|
upsert.run(row.path, row.blob, row.lang, row.bytes, row.pkg);
|
|
69
211
|
dropSymbols.run(row.path);
|
|
70
|
-
dropEdges.run(row.path
|
|
212
|
+
dropEdges.run(row.path);
|
|
71
213
|
dropFts.run(row.path);
|
|
214
|
+
dropTri.run(row.path);
|
|
72
215
|
dropEmbed.run(row.path);
|
|
73
216
|
}
|
|
74
217
|
for (const path of removed) {
|
|
75
218
|
dropFile.run(path);
|
|
76
219
|
dropSymbols.run(path);
|
|
77
|
-
dropEdges.run(path
|
|
220
|
+
dropEdges.run(path);
|
|
221
|
+
dropIncoming.run(path);
|
|
78
222
|
dropFts.run(path);
|
|
223
|
+
dropTri.run(path);
|
|
79
224
|
dropEmbed.run(path);
|
|
80
225
|
}
|
|
81
226
|
for (const row of symbols) addSymbol.run(row.path, row.name, row.kind, row.line, row.sig, row.exported);
|
|
82
227
|
for (const row of edges) addEdge.run(row.src, row.dst, row.kind, row.weight);
|
|
83
|
-
for (const row of fts)
|
|
228
|
+
for (const row of fts) {
|
|
229
|
+
addFts.run(row.kind, row.ref, row.body);
|
|
230
|
+
// The trigram table carries folded raw text for CJK-bearing sections
|
|
231
|
+
// only; an ASCII-only section is already reachable through `fts`.
|
|
232
|
+
if (row.raw) addTri.run(row.kind, row.ref, row.raw);
|
|
233
|
+
}
|
|
84
234
|
for (const row of embeddings) {
|
|
85
235
|
const vecBlob = Buffer.isBuffer(row.vec) || row.vec instanceof Uint8Array ? row.vec : encodeVectorBlob(row.vec);
|
|
86
236
|
const dims = Number.isInteger(row.dims) ? row.dims : Math.floor(vecBlob.byteLength / 4);
|
|
87
237
|
upsertEmbed.run(row.path, row.blob ?? '', row.model ?? '', dims, vecBlob);
|
|
88
238
|
}
|
|
89
|
-
|
|
239
|
+
const upsertMeta = db.prepare(
|
|
240
|
+
'insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value',
|
|
241
|
+
);
|
|
242
|
+
upsertMeta.run('schema', SCHEMA_V);
|
|
243
|
+
upsertMeta.run('tree', tree);
|
|
90
244
|
db.exec('commit');
|
|
91
245
|
} catch (error) {
|
|
92
246
|
try { db.exec('rollback'); } catch { /* a failed begin has nothing to undo */ }
|
|
93
|
-
throw error;
|
|
247
|
+
throw asIndexError(error);
|
|
248
|
+
} finally {
|
|
249
|
+
db.close();
|
|
250
|
+
}
|
|
251
|
+
if (cold) await rename(target, file);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* The version a database declares: `PRAGMA user_version` when stamped, else the
|
|
256
|
+
* meta `schema` row databases written before the stamp carried it in. Throws
|
|
257
|
+
* when the file cannot be read as a database - the caller decides whether that
|
|
258
|
+
* is a cold start or a corrupt one.
|
|
259
|
+
*/
|
|
260
|
+
async function storedVersion(file) {
|
|
261
|
+
const { DatabaseSync } = await import('node:sqlite');
|
|
262
|
+
const db = new DatabaseSync(file);
|
|
263
|
+
try {
|
|
264
|
+
const user = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
|
|
265
|
+
if (user > 0) return String(user);
|
|
266
|
+
return db.prepare("select value from meta where key = 'schema'").get()?.value ?? '';
|
|
94
267
|
} finally {
|
|
95
268
|
db.close();
|
|
96
269
|
}
|
|
97
270
|
}
|
|
98
271
|
|
|
99
|
-
|
|
272
|
+
/** A damaged index file is named, not silent (review 0003 Q44). */
|
|
273
|
+
function corruptIndexError(cause) {
|
|
274
|
+
const corrupt = new Error('the index file is corrupt; run the index build to replace it');
|
|
275
|
+
corrupt.code = 'INDEX_CORRUPT';
|
|
276
|
+
corrupt.cause = cause;
|
|
277
|
+
return corrupt;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* A contended index write is a retryable race, not a broken repository: name it
|
|
282
|
+
* so the command maps its own remedy instead of blaming the git checkout (Q08).
|
|
283
|
+
*/
|
|
284
|
+
function asIndexError(error) {
|
|
285
|
+
const code = error?.code ?? error?.errno;
|
|
286
|
+
if (code === 'SQLITE_BUSY' || code === 5 || /database is locked/i.test(error?.message ?? '')) {
|
|
287
|
+
const busy = new Error('another index run is writing; wait a moment and run again');
|
|
288
|
+
busy.code = 'INDEX_BUSY';
|
|
289
|
+
busy.cause = error;
|
|
290
|
+
return busy;
|
|
291
|
+
}
|
|
292
|
+
if (/file is not a database|database disk image|malformed|corrupt/i.test(error?.message ?? '')) {
|
|
293
|
+
return corruptIndexError(error);
|
|
294
|
+
}
|
|
295
|
+
return error;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* The jsonl file this version writes to: the canonical file when it is this
|
|
300
|
+
* version's or unreadable, the sidecar when it belongs to another version
|
|
301
|
+
* (review 0003 Q44).
|
|
302
|
+
*/
|
|
303
|
+
async function jsonlTargetFile(root) {
|
|
304
|
+
const base = join(root, JSONL_FILE);
|
|
305
|
+
let text = null;
|
|
306
|
+
try {
|
|
307
|
+
text = await readFile(base, 'utf8');
|
|
308
|
+
} catch {
|
|
309
|
+
try {
|
|
310
|
+
text = await readFile(join(root, JSONL_FILE_LEGACY), 'utf8');
|
|
311
|
+
} catch {
|
|
312
|
+
return base; // no file: cold build goes to the canonical path
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
try {
|
|
316
|
+
for (const line of text.split('\n')) {
|
|
317
|
+
if (line === '') continue;
|
|
318
|
+
const row = JSON.parse(line);
|
|
319
|
+
if (row.table !== 'meta' || row.key !== 'schema') continue;
|
|
320
|
+
const value = row.value ?? '';
|
|
321
|
+
// Older files are rewritten in place at the current schema; a newer one
|
|
322
|
+
// stays with its owner and the build goes to the sidecar.
|
|
323
|
+
return canonicalForWrite(value) ? base : indexFileFor(root, 'jsonl', value);
|
|
324
|
+
}
|
|
325
|
+
} catch {
|
|
326
|
+
return base; // unreadable: rebuild it in place
|
|
327
|
+
}
|
|
328
|
+
return base;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export async function writeJsonl(root, previous, next, removed, symbols, edges, fts, force, embeddings = [], tree = '') {
|
|
100
332
|
const gone = new Set(removed);
|
|
101
333
|
const replaced = new Map(next.map((row) => [row.path, row]));
|
|
102
334
|
const drop = new Set([...replaced.keys(), ...gone]);
|
|
@@ -108,21 +340,26 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
|
|
|
108
340
|
}
|
|
109
341
|
for (const row of replaced.values()) files.push(row);
|
|
110
342
|
files.sort(byPath);
|
|
111
|
-
const keptSymbols = force ? [] : (await readJsonl(root, 'symbols')).filter((row) => !drop.has(row.path));
|
|
112
|
-
|
|
113
|
-
|
|
343
|
+
const keptSymbols = force ? [] : (await readJsonl(root, 'symbols')).rows.filter((row) => !drop.has(row.path));
|
|
344
|
+
// Edge ownership mirrors the sqlite backend: a changed file loses only the edges
|
|
345
|
+
// it owns (src), while a removed file also drops the edges that pointed at it.
|
|
346
|
+
const keptEdges = force ? [] : (await readJsonl(root, 'edges')).rows.filter((row) => !drop.has(row.src) && !gone.has(row.dst));
|
|
347
|
+
const keptFts = force ? [] : (await readJsonl(root, 'fts')).rows.filter((row) => !drop.has(row.ref));
|
|
114
348
|
const newEmbedPaths = new Set(embeddings.map((row) => row.path));
|
|
115
349
|
const keptEmbeddings = force
|
|
116
350
|
? []
|
|
117
|
-
: (await readJsonl(root, 'embeddings')).filter((row) => !drop.has(row.path) && !newEmbedPaths.has(row.path));
|
|
351
|
+
: (await readJsonl(root, 'embeddings')).rows.filter((row) => !drop.has(row.path) && !newEmbedPaths.has(row.path));
|
|
118
352
|
const allSymbols = [...keptSymbols, ...symbols].sort(bySymbol);
|
|
119
353
|
const allEdges = [...keptEdges, ...edges].sort(byEdge);
|
|
120
354
|
const allFts = [...keptFts, ...fts].sort((a, b) => cmp(a.ref, b.ref));
|
|
121
355
|
const allEmbeddings = [...keptEmbeddings, ...embeddings].sort(byPath);
|
|
122
|
-
const file =
|
|
356
|
+
const file = await jsonlTargetFile(root);
|
|
123
357
|
await mkdir(dirname(file), { recursive: true });
|
|
358
|
+
// Written atomically: a concurrent reader never sees a half-written file.
|
|
359
|
+
const tmp = tmpFor(file);
|
|
124
360
|
const lines = [
|
|
125
361
|
JSON.stringify({ table: 'meta', key: 'schema', value: SCHEMA_V }),
|
|
362
|
+
JSON.stringify({ table: 'meta', key: 'tree', value: tree }),
|
|
126
363
|
...files.map((row) => JSON.stringify({ table: 'files', path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg })),
|
|
127
364
|
...allSymbols.map((row) => JSON.stringify({ table: 'symbols', path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported })),
|
|
128
365
|
...allEdges.map((row) => JSON.stringify({ table: 'edges', src: row.src, dst: row.dst, kind: row.kind, weight: row.weight })),
|
|
@@ -140,63 +377,133 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
|
|
|
140
377
|
});
|
|
141
378
|
}),
|
|
142
379
|
];
|
|
143
|
-
await writeFile(
|
|
380
|
+
await writeFile(tmp, `${lines.join('\n')}\n`);
|
|
381
|
+
await rename(tmp, file);
|
|
144
382
|
return { symbols: allSymbols.length, edges: allEdges.length, embeddings: allEmbeddings.length };
|
|
145
383
|
}
|
|
146
384
|
|
|
147
385
|
export async function readJsonl(root, table) {
|
|
386
|
+
// The pre-move path is a fallback for one release (review 0003 Q19).
|
|
148
387
|
let text;
|
|
149
388
|
try {
|
|
150
389
|
text = await readFile(join(root, JSONL_FILE), 'utf8');
|
|
151
390
|
} catch {
|
|
152
|
-
|
|
391
|
+
try {
|
|
392
|
+
text = await readFile(join(root, JSONL_FILE_LEGACY), 'utf8');
|
|
393
|
+
} catch {
|
|
394
|
+
return { rows: [], error: null };
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
let schema = '';
|
|
398
|
+
try {
|
|
399
|
+
for (const line of text.split('\n')) {
|
|
400
|
+
if (line === '') continue;
|
|
401
|
+
const row = JSON.parse(line);
|
|
402
|
+
if (row.table === 'meta' && row.key === 'schema') schema = row.value ?? '';
|
|
403
|
+
}
|
|
404
|
+
} catch (error) {
|
|
405
|
+
return { rows: [], error: corruptIndexError(error) };
|
|
406
|
+
}
|
|
407
|
+
// A file a newer version owns is not read blind (review 0003 Q44): this
|
|
408
|
+
// version's sidecar is read instead, and a sidecar that has not been built
|
|
409
|
+
// yet is cold, not broken. An older file is read in place; the next write
|
|
410
|
+
// rewrites it at the current schema.
|
|
411
|
+
if (!canonicalForWrite(schema)) {
|
|
412
|
+
try {
|
|
413
|
+
text = await readFile(indexFileFor(root, 'jsonl', schema), 'utf8');
|
|
414
|
+
} catch {
|
|
415
|
+
return { rows: [], error: null };
|
|
416
|
+
}
|
|
153
417
|
}
|
|
154
418
|
const rows = [];
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
419
|
+
try {
|
|
420
|
+
for (const line of text.split('\n')) {
|
|
421
|
+
if (line === '') continue;
|
|
422
|
+
const row = JSON.parse(line);
|
|
423
|
+
if (row.table !== table) continue;
|
|
424
|
+
if (table === 'meta') rows.push({ key: row.key, value: row.value });
|
|
425
|
+
else if (table === 'files') rows.push({ path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg });
|
|
426
|
+
else if (table === 'symbols') rows.push({ path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported });
|
|
427
|
+
else if (table === 'edges') rows.push({ src: row.src, dst: row.dst, kind: row.kind, weight: row.weight });
|
|
428
|
+
else if (table === 'fts') rows.push({ kind: row.kind, ref: row.ref, body: row.body });
|
|
429
|
+
else if (table === 'embeddings') {
|
|
430
|
+
const vec = decodeVectorBlob(row.vec);
|
|
431
|
+
rows.push({
|
|
432
|
+
path: row.path,
|
|
433
|
+
blob: row.blob ?? '',
|
|
434
|
+
model: row.model ?? '',
|
|
435
|
+
dims: Number.isInteger(row.dims) ? row.dims : vec.length,
|
|
436
|
+
vec,
|
|
437
|
+
});
|
|
438
|
+
}
|
|
173
439
|
}
|
|
440
|
+
} catch (error) {
|
|
441
|
+
return { rows: [], error: corruptIndexError(error) };
|
|
174
442
|
}
|
|
175
443
|
if (table === 'symbols') rows.sort(bySymbol);
|
|
176
444
|
else if (table === 'edges') rows.sort(byEdge);
|
|
177
445
|
else if (table === 'fts') rows.sort((a, b) => cmp(a.ref, b.ref));
|
|
178
446
|
else rows.sort(byPath);
|
|
179
|
-
return rows;
|
|
447
|
+
return { rows, error: null };
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* The sqlite file a read opens: the canonical file when it is this version's
|
|
452
|
+
* or older, the sidecar when it belongs to a newer one (review 0003 Q44),
|
|
453
|
+
* null when no index for this version exists. Throws when a present file
|
|
454
|
+
* cannot be read as a database - corrupt, not cold.
|
|
455
|
+
*/
|
|
456
|
+
async function resolveSqliteFile(root) {
|
|
457
|
+
// The pre-move path is a fallback for one release (review 0003 Q19).
|
|
458
|
+
let file = join(root, SQLITE_FILE);
|
|
459
|
+
try {
|
|
460
|
+
await stat(file);
|
|
461
|
+
} catch {
|
|
462
|
+
file = join(root, SQLITE_FILE_LEGACY);
|
|
463
|
+
try {
|
|
464
|
+
await stat(file);
|
|
465
|
+
} catch {
|
|
466
|
+
return null;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
const version = await storedVersion(file);
|
|
470
|
+
// An older database is readable in place: its tables carry the same columns,
|
|
471
|
+
// and the next write migrates it. Only a newer one is routed to a sidecar.
|
|
472
|
+
if (canonicalForWrite(version)) return file;
|
|
473
|
+
const sidecar = indexFileFor(root, 'sqlite', version);
|
|
474
|
+
try {
|
|
475
|
+
await stat(sidecar);
|
|
476
|
+
return sidecar;
|
|
477
|
+
} catch {
|
|
478
|
+
return null; // this version has not built its sidecar yet: cold
|
|
479
|
+
}
|
|
180
480
|
}
|
|
181
481
|
|
|
482
|
+
/**
|
|
483
|
+
* Read rows from the index. Returns `{ rows, error }`: a missing index is
|
|
484
|
+
* `{ rows: [], error: null }` (cold), a present index that cannot be read is
|
|
485
|
+
* an empty row set with a named error - "locked", "no such column" and
|
|
486
|
+
* "corrupt" used to all look like "no results" (review 0003 Q44).
|
|
487
|
+
*/
|
|
182
488
|
export async function readSqliteRows(root, sql, params = []) {
|
|
183
489
|
let DatabaseSync;
|
|
184
490
|
try {
|
|
185
491
|
({ DatabaseSync } = await import('node:sqlite'));
|
|
186
492
|
} catch {
|
|
187
|
-
return [];
|
|
493
|
+
return { rows: [], error: null };
|
|
188
494
|
}
|
|
189
|
-
|
|
495
|
+
let file;
|
|
190
496
|
try {
|
|
191
|
-
await
|
|
192
|
-
} catch {
|
|
193
|
-
return [];
|
|
497
|
+
file = await resolveSqliteFile(root);
|
|
498
|
+
} catch (error) {
|
|
499
|
+
return { rows: [], error: asIndexError(error) };
|
|
194
500
|
}
|
|
501
|
+
if (!file) return { rows: [], error: null };
|
|
195
502
|
const db = new DatabaseSync(file);
|
|
196
503
|
try {
|
|
197
|
-
return db.prepare(sql).all(...params);
|
|
198
|
-
} catch {
|
|
199
|
-
return [];
|
|
504
|
+
return { rows: db.prepare(sql).all(...params), error: null };
|
|
505
|
+
} catch (error) {
|
|
506
|
+
return { rows: [], error: asIndexError(error) };
|
|
200
507
|
} finally {
|
|
201
508
|
db.close();
|
|
202
509
|
}
|
|
@@ -204,21 +511,31 @@ export async function readSqliteRows(root, sql, params = []) {
|
|
|
204
511
|
|
|
205
512
|
export async function readSchema(root, backend) {
|
|
206
513
|
if (backend === 'sqlite') {
|
|
207
|
-
const rows = await readSqliteRows(root, "select value from meta where key = 'schema'");
|
|
514
|
+
const { rows } = await readSqliteRows(root, "select value from meta where key = 'schema'");
|
|
208
515
|
return rows[0]?.value ?? '';
|
|
209
516
|
}
|
|
210
|
-
const rows = await readJsonl(root, 'meta');
|
|
517
|
+
const { rows } = await readJsonl(root, 'meta');
|
|
211
518
|
return rows.find((row) => row.key === 'schema')?.value ?? '';
|
|
212
519
|
}
|
|
213
520
|
|
|
521
|
+
/** The tree the stored index was built from, or '' when no build has vouched for one. */
|
|
522
|
+
export async function readTreeDigest(root, backend) {
|
|
523
|
+
if (backend === 'sqlite') {
|
|
524
|
+
const { rows } = await readSqliteRows(root, "select value from meta where key = 'tree'");
|
|
525
|
+
return rows[0]?.value ?? '';
|
|
526
|
+
}
|
|
527
|
+
const { rows } = await readJsonl(root, 'meta');
|
|
528
|
+
return rows.find((row) => row.key === 'tree')?.value ?? '';
|
|
529
|
+
}
|
|
530
|
+
|
|
214
531
|
export async function countSqlite(root) {
|
|
215
532
|
const symbols = await readSqliteRows(root, 'select count(*) as n from symbols');
|
|
216
533
|
const edges = await readSqliteRows(root, 'select count(*) as n from edges');
|
|
217
534
|
const embeddings = await readSqliteRows(root, 'select count(*) as n from embeddings');
|
|
218
535
|
return {
|
|
219
|
-
symbols: Number(symbols[0]?.n ?? 0),
|
|
220
|
-
edges: Number(edges[0]?.n ?? 0),
|
|
221
|
-
embeddings: Number(embeddings[0]?.n ?? 0),
|
|
536
|
+
symbols: Number(symbols.rows[0]?.n ?? 0),
|
|
537
|
+
edges: Number(edges.rows[0]?.n ?? 0),
|
|
538
|
+
embeddings: Number(embeddings.rows[0]?.n ?? 0),
|
|
222
539
|
};
|
|
223
540
|
}
|
|
224
541
|
|