simt-dev 0.7.0 → 0.8.0

package/src/index/persist.mjs

@@ -9,19 +9,27 @@ import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 
 import { LOCAL_DIR } from '../state.mjs';
+import { cmp } from '../util/sort.mjs';
 
 export const SQLITE_FILE = join(LOCAL_DIR, 'index.sqlite');
 export const JSONL_FILE = join(LOCAL_DIR, 'index.jsonl');
 const SQLITE_FILE_LEGACY = '.simt/index.sqlite';
 const JSONL_FILE_LEGACY = '.simt/index.jsonl';
-export const SCHEMA_V = '4';
+export const SCHEMA_V = '5';
 
 // The FTS tables carry their tokenizer in their definition, so `if not exists`
 // cannot migrate them: a v3 database would keep tokenizing new rows the v3 way
-// (ASCII fold, kind and ref searchable). The 3 -> 4 migration below recreates
-// them, which is the versioned replacement for the per-write wipe (Q44).
-export const FTS_SCHEMA = 'create virtual table fts using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "unicode61 remove_diacritics 2")';
-export const FTS_TRI_SCHEMA = 'create virtual table fts_tri using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "trigram")';
+// (ASCII fold, kind and ref searchable), and a v4 `fts` table cannot grow the
+// node_id column the v5 layout needs. The preamble creates them only when they
+// are missing (a cold build); the 3 -> 4 and 4 -> 5 migrations below drop and
+// recreate them, which is the versioned replacement for the per-write wipe (Q44).
+export const FTS_SCHEMA = 'create virtual table if not exists fts using fts5(body, node_id UNINDEXED, kind UNINDEXED, tokenize = "unicode61 remove_diacritics 2")';
+export const FTS_TRI_SCHEMA = 'create virtual table if not exists fts_tri using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "trigram")';
+
+// The v4 `fts` shape. The 3 -> 4 step recreates the table with THIS shape so a
+// v3 database emerges looking exactly like a genuine v4 one - the 4 -> 5 step
+// then copies its `ref` column, which the v5 shape no longer has.
+const FTS_V4_SCHEMA = 'create virtual table fts using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "unicode61 remove_diacritics 2")';
 
 // A database written by a *newer* simt is never opened for write, and never
 // read blind (review 0003 Q44): the mismatched canonical file stays with its
@@ -53,11 +61,58 @@ const MIGRATIONS = [
     to: 4,
     apply: (db) => {
       db.exec('drop table if exists fts');
-      db.exec(FTS_SCHEMA);
+      db.exec(FTS_V4_SCHEMA);
       db.exec('drop table if exists fts_tri');
       db.exec(FTS_TRI_SCHEMA);
     },
   },
+  // v4 is per-kind tables (files/symbols/edges/embeddings); v5 is the entity
+  // layout (sources/nodes/edges/vectors) with the read shapes restored as SQL
+  // in the store. `edges` and `fts` are built under `_v5` names and swapped in
+  // after the copy, because the v4 tables of those names still hold data.
+  // fts_tri keeps its (kind, ref, body) shape - the CJK trigram path is
+  // untouched (Q43).
+  {
+    from: 4,
+    to: 5,
+    apply: (db) => {
+      db.exec(TABLES);
+      db.exec(EDGES_DDL.replaceAll('edges(', 'edges_v5('));
+      db.exec(FTS_SCHEMA.replaceAll('exists fts ', 'exists fts_v5 '));
+      db.exec(`insert into sources(kind, path, revision, lang, bytes, status, props)
+        select 'blob', path, blob, lang, bytes, 'indexed', json_object('pkg', coalesce(pkg, '')) from files`);
+      db.exec(`insert into nodes(key, kind, source_id)
+        select 'file:' || path, 'file', id from sources where kind = 'blob'`);
+      db.exec(`insert into nodes(key, kind, source_id, name, line, props)
+        select 'sym:' || s.path || '#' || s.name || '@' || coalesce(s.line, 0), 'symbol', src.id,
+               s.name, s.line, json_object('kind', coalesce(s.kind, 'symbol'), 'sig', coalesce(s.sig, ''), 'exported', coalesce(s.exported, 0))
+        from symbols s join sources src on src.kind = 'blob' and src.path = s.path`);
+      db.exec(`insert into edges_v5(src_id, kind, dst_key, dst_id, weight, owner_source_id, trust)
+        select sn.id, e.kind, e.dst, dn.id, e.weight, src.id, 'observed'
+        from edges e
+        join sources src on src.kind = 'blob' and src.path = e.src
+        join nodes sn on sn.source_id = src.id and sn.kind = 'file'
+        left join sources dst on dst.kind = 'blob' and dst.path = e.dst
+        left join nodes dn on dn.source_id = dst.id and dn.kind = 'file'`);
+      db.exec(`insert into vectors(node_id, model, dims, vec)
+        select n.id, em.model, em.dims, em.vec
+        from embeddings em
+        join sources s on s.kind = 'blob' and s.path = em.path
+        join nodes n on n.source_id = s.id and n.kind = 'file'`);
+      db.exec(`insert into fts_v5(body, node_id, kind)
+        select f.body, n.id, f.kind
+        from fts f
+        join sources s on s.kind = 'blob' and s.path = f.ref
+        join nodes n on n.source_id = s.id and n.kind = 'file'`);
+      db.exec('drop table files');
+      db.exec('drop table symbols');
+      db.exec('drop table embeddings');
+      db.exec('drop table edges');
+      db.exec('drop table fts');
+      db.exec('alter table edges_v5 rename to edges');
+      db.exec('alter table fts_v5 rename to fts');
+    },
+  },
 ];
 
 /**
@@ -67,21 +122,89 @@ const MIGRATIONS = [
  * each is idempotent on an already-migrated table set.
  */
 function migrateDb(db) {
-  const at = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
-  for (const step of MIGRATIONS) {
-    if (at === 0 || (at >= step.from && at < step.to)) step.apply(db);
+  let at = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
+  // `user_version 0` is ambiguous: a cold build has no legacy tables at all,
+  // and a pre-stamp database still holds the v3 shape. The 4 -> 5 step copies
+  // FROM the v4 tables (`files`, `symbols`, ...), so a cold build must not run
+  // it - the 3 -> 4 step would leave its `fts` in the v4 shape behind.
+  const legacy =
+    at !== 0 ||
+    db.prepare(`select count(*) as n from sqlite_master where type = 'table' and name in ('files', 'symbols', 'embeddings')`).get().n > 0;
+  if (legacy) {
+    for (const step of MIGRATIONS) {
+      if (at === 0 || (at >= step.from && at < step.to)) {
+        step.apply(db);
+        at = step.to; // the next step sees the version this one left behind
+      }
+    }
   }
   db.exec(`pragma user_version = ${Number(SCHEMA_V)}`);
 }
 
-const SCHEMA = `
-create table if not exists files(path text primary key, blob text, lang text, bytes int, pkg text);
-create table if not exists symbols(id integer primary key, path text, name text, kind text, line int, sig text, exported int);
-create table if not exists edges(src text, dst text, kind text, weight real);
-create table if not exists embeddings(path text primary key, blob text, model text, dims int, vec blob);
+const TABLES = `
+create table if not exists sources(
+  id integer primary key,
+  kind text not null default 'blob',
+  path text not null,
+  revision text,
+  lang text,
+  bytes integer,
+  status text not null default 'indexed',
+  props text,
+  unique(kind, path)
+);
+create table if not exists nodes(
+  id integer primary key,
+  key text not null unique,
+  kind text,
+  source_id integer,
+  name text,
+  line integer,
+  span_start integer,
+  span_end integer,
+  digest text,
+  trust text not null default 'observed',
+  props text
+);
+create table if not exists vectors(
+  node_id integer primary key,
+  model text,
+  dims integer,
+  vec blob
+);
 create table if not exists meta(key text primary key, value text);
 `;
 
+// The v5 edges table. Named `edges_v5` in the 4 -> 5 migration, because the
+// v4 database still holds an `edges` table of a different shape until the
+// copy below is done.
+const EDGES_DDL = `create table if not exists edges(
+  src_id integer,
+  kind text,
+  dst_key text,
+  dst_id integer,
+  weight real,
+  owner_source_id integer,
+  trust text,
+  props text,
+  primary key(src_id, kind, dst_key)
+);`;
+
+// Q76: the v4 layout had no index on the edge endpoints or the node owners -
+// the dependent, impact and coverage walks each paid a full scan per file.
+// Executed after `migrateDb`, not in the preamble: on a v3 database the v3
+// `edges` table survives `if not exists` until the 4 -> 5 step swaps in the
+// v5 shape, and `idx_edges_dst` needs the `dst_id` column that only the v5
+// table has.
+const INDEXES = `
+create index if not exists idx_edges_dst on edges(dst_id, kind);
+create index if not exists idx_edges_owner on edges(owner_source_id);
+create index if not exists idx_nodes_source on nodes(source_id);
+create index if not exists idx_nodes_kind_name on nodes(kind, name);
+`;
+
+const SCHEMA = `${TABLES}\n${EDGES_DDL}\n`;
+
 export function encodeVectorBlob(vec) {
   if (!Array.isArray(vec) && !(vec instanceof Float32Array)) return Buffer.alloc(0);
   const f32 = vec instanceof Float32Array ? vec : Float32Array.from(vec, (v) => Number(v) || 0);
@@ -186,55 +309,95 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts, embe
     db.exec('pragma busy_timeout = 5000');
     db.exec('pragma synchronous = normal');
     db.exec(SCHEMA);
+    db.exec(FTS_SCHEMA);
+    db.exec(FTS_TRI_SCHEMA);
     db.exec('begin immediate');
     migrateDb(db);
-    const upsert = db.prepare(`insert into files(path, blob, lang, bytes, pkg) values (?, ?, ?, ?, ?)
-      on conflict(path) do update set blob = excluded.blob, lang = excluded.lang, bytes = excluded.bytes, pkg = excluded.pkg`);
-    const dropSymbols = db.prepare('delete from symbols where path = ?');
-    // An importer owns its outgoing edges: re-extracting a changed file drops
-    // only `src = ?`, so the unchanged importers' edges pointing into it survive
-    // (Q07). A removed file's incoming edges dangle, so they drop with it.
-    const dropEdges = db.prepare('delete from edges where src = ?');
-    const dropIncoming = db.prepare('delete from edges where dst = ?');
-    const dropFts = db.prepare('delete from fts where ref = ?');
-    const dropTri = db.prepare('delete from fts_tri where ref = ?');
-    const dropEmbed = db.prepare('delete from embeddings where path = ?');
-    const dropFile = db.prepare('delete from files where path = ?');
-    const addSymbol = db.prepare('insert into symbols(path, name, kind, line, sig, exported) values (?, ?, ?, ?, ?, ?)');
-    const addEdge = db.prepare('insert into edges(src, dst, kind, weight) values (?, ?, ?, ?)');
-    const addFts = db.prepare('insert into fts(kind, ref, body) values (?, ?, ?)');
+    // The `edges` table is v5-shaped now on every path: the preamble created
+    // it on a cold build, the migration swapped it in on a legacy one.
+    db.exec(INDEXES);
+    // One source row per file, keyed (kind, path): the revision is the git blob
+    // sha, so an unchanged blob keeps its row, and the file's node, vectors,
+    // edges and fts rows all hang off it. `props` carries what the v5 columns
+    // do not name - the package directory, which the files read shape serves.
+    const upsertSource = db.prepare(`insert into sources(kind, path, revision, lang, bytes, status, props)
+      values('blob', ?, ?, ?, ?, 'indexed', ?)
+      on conflict(kind, path) do update set revision = excluded.revision, lang = excluded.lang,
+        bytes = excluded.bytes, props = excluded.props`);
+    const sourceIdOf = db.prepare(`select id from sources where kind = 'blob' and path = ?`);
+    // `prepare` compiles only the FIRST statement in a string (Q76): a
+    // multi-statement delete prepared as one ran only its head, and the
+    // surviving `file:` node collided with the next insert on `nodes.key`.
+    const dropDependents = [
+      db.prepare('delete from vectors where node_id in (select id from nodes where source_id = ?)'),
+      db.prepare('delete from fts where node_id in (select id from nodes where source_id = ?)'),
+      db.prepare('delete from edges where owner_source_id = ?'),
+      db.prepare('delete from nodes where source_id = ?'),
+    ];
+    const addFileNode = db.prepare(`insert into nodes(key, kind, source_id, trust) values('file:' || ?, 'file', ?, 'observed')`);
+    const addSymbol = db.prepare(`insert into nodes(key, kind, source_id, name, line, props)
+      values('sym:' || ? || '#' || ? || '@' || ?, 'symbol', ?, ?, ?, ?)`);
+    const addEdge = db.prepare(`insert into edges(src_id, kind, dst_key, dst_id, weight, owner_source_id, trust)
+      values (?, ?, ?, ?, ?, ?, 'observed')`);
+    const addFts = db.prepare('insert into fts(body, node_id, kind) values (?, ?, ?)');
     const addTri = db.prepare('insert into fts_tri(kind, ref, body) values (?, ?, ?)');
-    const upsertEmbed = db.prepare(`insert into embeddings(path, blob, model, dims, vec) values (?, ?, ?, ?, ?)
-      on conflict(path) do update set blob = excluded.blob, model = excluded.model, dims = excluded.dims, vec = excluded.vec`);
+    const upsertVector = db.prepare(`insert into vectors(node_id, model, dims, vec) values (?, ?, ?, ?)
+      on conflict(node_id) do update set model = excluded.model, dims = excluded.dims, vec = excluded.vec`);
+    const dropSource = [
+      db.prepare(`delete from vectors where node_id in (select id from nodes where source_id in (select id from sources where kind = 'blob' and path = ?))`),
+      db.prepare(`delete from fts where node_id in (select id from nodes where source_id in (select id from sources where kind = 'blob' and path = ?))`),
+      db.prepare(`delete from edges where owner_source_id in (select id from sources where kind = 'blob' and path = ?)`),
+      db.prepare('delete from edges where dst_key = ?'),
+      db.prepare(`delete from nodes where source_id in (select id from sources where kind = 'blob' and path = ?)`),
+      db.prepare(`delete from sources where kind = 'blob' and path = ?`),
+    ];
     for (const row of next) {
-      upsert.run(row.path, row.blob, row.lang, row.bytes, row.pkg);
-      dropSymbols.run(row.path);
-      dropEdges.run(row.path);
-      dropFts.run(row.path);
-      dropTri.run(row.path);
-      dropEmbed.run(row.path);
+      const sourceId = sourceIdOf.get(row.path)?.id;
+      if (sourceId !== undefined) for (const st of dropDependents) st.run(sourceId);
+      upsertSource.run(row.path, row.blob, row.lang, row.bytes, JSON.stringify({ pkg: row.pkg ?? '' }));
     }
     for (const path of removed) {
-      dropFile.run(path);
-      dropSymbols.run(path);
-      dropEdges.run(path);
-      dropIncoming.run(path);
-      dropFts.run(path);
-      dropTri.run(path);
-      dropEmbed.run(path);
+      for (const st of dropSource) st.run(path);
+      // fts_tri is still path-keyed: the trigram table keeps its v4 shape (Q43).
+      db.prepare('delete from fts_tri where ref = ?').run(path);
+    }
+    for (const row of next) {
+      addFileNode.run(row.path, sourceIdOf.get(row.path).id);
+    }
+    // The ids a changed file's rows hang off were just rebuilt; an unchanged
+    // file's rows keep their old ids. One query maps every path to both.
+    const ids = new Map();
+    for (const r of db.prepare(`select s.path, s.id as source_id, n.id as node_id
+      from sources s join nodes n on n.source_id = s.id and n.kind = 'file'
+      where s.kind = 'blob'`).all()) {
+      ids.set(r.path, { sourceId: r.source_id, nodeId: r.node_id });
+    }
+    for (const row of symbols) {
+      const entry = ids.get(row.path);
+      if (!entry) continue;
+      addSymbol.run(row.path, row.name, row.line, entry.sourceId, row.name, row.line,
+        JSON.stringify({ kind: row.kind ?? 'symbol', sig: row.sig ?? '', exported: row.exported ? 1 : 0 }));
+    }
+    for (const row of edges) {
+      const src = ids.get(row.src);
+      if (!src) continue;
+      const dst = ids.get(row.dst);
+      addEdge.run(src.nodeId, row.kind, row.dst, dst ? dst.nodeId : null, row.weight, src.sourceId);
     }
-    for (const row of symbols) addSymbol.run(row.path, row.name, row.kind, row.line, row.sig, row.exported);
-    for (const row of edges) addEdge.run(row.src, row.dst, row.kind, row.weight);
     for (const row of fts) {
-      addFts.run(row.kind, row.ref, row.body);
+      const entry = ids.get(row.ref);
+      if (!entry) continue;
+      addFts.run(row.body, entry.nodeId, row.kind);
       // The trigram table carries folded raw text for CJK-bearing sections
       // only; an ASCII-only section is already reachable through `fts`.
       if (row.raw) addTri.run(row.kind, row.ref, row.raw);
     }
     for (const row of embeddings) {
+      const entry = ids.get(row.path);
+      if (!entry) continue;
       const vecBlob = Buffer.isBuffer(row.vec) || row.vec instanceof Uint8Array ? row.vec : encodeVectorBlob(row.vec);
       const dims = Number.isInteger(row.dims) ? row.dims : Math.floor(vecBlob.byteLength / 4);
-      upsertEmbed.run(row.path, row.blob ?? '', row.model ?? '', dims, vecBlob);
+      upsertVector.run(entry.nodeId, row.model ?? '', dims, vecBlob);
     }
     const upsertMeta = db.prepare(
       'insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value',
@@ -529,13 +692,20 @@ export async function readTreeDigest(root, backend) {
 }
 
 export async function countSqlite(root) {
-  const symbols = await readSqliteRows(root, 'select count(*) as n from symbols');
-  const edges = await readSqliteRows(root, 'select count(*) as n from edges');
-  const embeddings = await readSqliteRows(root, 'select count(*) as n from embeddings');
+  // One open, three counts: the v4 layout paid a full open/close per metric,
+  // and a read that only wants the counts was buying three connections (Q76).
+  const { rows } = await readSqliteRows(
+    root,
+    `select
+      (select count(*) from nodes where kind = 'symbol') as symbols,
+      (select count(*) from edges) as edges,
+      (select count(*) from vectors) as embeddings`,
+  );
+  const row = rows[0] ?? {};
   return {
-    symbols: Number(symbols.rows[0]?.n ?? 0),
-    edges: Number(edges.rows[0]?.n ?? 0),
-    embeddings: Number(embeddings.rows[0]?.n ?? 0),
+    symbols: Number(row.symbols ?? 0),
+    edges: Number(row.edges ?? 0),
+    embeddings: Number(row.embeddings ?? 0),
   };
 }
 
@@ -551,6 +721,3 @@ function byEdge(a, b) {
   return cmp(a.src, b.src) || cmp(a.dst, b.dst) || cmp(a.kind, b.kind);
 }
 
-function cmp(a, b) {
-  return a < b ? -1 : a > b ? 1 : 0;
-}

package/src/index/store.mjs

@@ -8,10 +8,16 @@ import { execFile, spawn } from 'node:child_process';
 import { stat } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 
+import { git as gitRaw } from '../sys/git.mjs';
+
 import { readManifest } from '../projection/manifest.mjs';
 import { readLocalConfig } from '../local/config.mjs';
 import { localEmbed } from '../local/client.mjs';
 import { extract, resolveImports } from './extract.mjs';
+// Q70: the extension-to-language map is the records' `languages[]` (ADR-013). `canonical`
+// names the extensions that RESOLVE to a language (JavaScript claims `.ts` for extraction
+// but not for resolution), so a record's extraction and resolution cannot desynchronize.
+import { languageResolver } from '../ecosystem/records.mjs';
 import {
   FTS_SCHEMA,
   FTS_TRI_SCHEMA,
@@ -34,24 +40,32 @@ export { JSONL_FILE, SQLITE_FILE, decodeVectorBlob, encodeVectorBlob };
 
 const MARKERS = ['package.json', 'pyproject.toml', 'go.mod', 'Cargo.toml', 'pom.xml'];
 
-const LANG = {
-  js: 'javascript', mjs: 'javascript', cjs: 'javascript', jsx: 'javascript',
-  ts: 'typescript', tsx: 'typescript', kt: 'kotlin', kts: 'kotlin', java: 'java',
-  py: 'python', go: 'go', rs: 'rust', swift: 'swift', dart: 'dart', rb: 'ruby', cs: 'csharp',
-};
+// A tracked blob larger than this is recorded with its size and counted in
+// `skipped`, not streamed and tokenized: a 65 MiB file is indexed as a row, and
+// tokenizing it would buy nothing (Q76). The cap sits under that file on purpose
+// so the 65 MiB test exercises the skip, not the tokenize path.
+const BLOB_CAP = 16 * 1024 * 1024;
+
+const LANG = languageResolver();
 
 /** `sqlite` when this process can open a database and create an FTS5 table. */
+// Probed once per process: the check builds both FTS schemas in a memory
+// database, and every read path used to pay for that (Q76).
+let backendProbe;
 async function probeBackend() {
-  try {
-    const { DatabaseSync } = await import('node:sqlite');
-    const db = new DatabaseSync(':memory:');
-    db.exec(FTS_SCHEMA);
-    db.exec(FTS_TRI_SCHEMA);
-    db.close();
-    return 'sqlite';
-  } catch {
-    return 'jsonl';
-  }
+  backendProbe ??= (async () => {
+    try {
+      const { DatabaseSync } = await import('node:sqlite');
+      const db = new DatabaseSync(':memory:');
+      db.exec(FTS_SCHEMA);
+      db.exec(FTS_TRI_SCHEMA);
+      db.close();
+      return 'sqlite';
+    } catch {
+      return 'jsonl';
+    }
+  })();
+  return backendProbe;
 }
 
 export function ftsKindOf(filePath, lang = '') {
@@ -224,7 +238,10 @@ export async function reindex(root, opts = {}) {
     changed.push(entry);
   }
   const sizes = await blobSizes(root, changed.map((entry) => entry.blob));
-  const texts = await blobTexts(root, changed.map((entry) => entry.blob));
+  // Over-cap blobs are recorded with their size and counted in `skipped`, not
+  // streamed and tokenized: their row is written, their text is never read.
+  const textBlobs = changed.map((entry) => entry.blob).filter((blob) => (sizes.get(blob) ?? 0) <= BLOB_CAP);
+  const texts = await blobTexts(root, textBlobs);
   const adapter = await resolveAdapter(root, opts);
   const paths = new Set(listed.map((entry) => entry.path));
   const next = [];
@@ -383,7 +400,14 @@ function withReadError(rows, error) {
 export async function readFiles(root, opts = {}) {
   const backend = opts.backend ?? await probeBackend();
   if (backend === 'sqlite') {
-    const { rows, error } = await readSqliteRows(root, 'select path, blob, lang, bytes, pkg from files order by path');
+    // The v5 layout keys files as `sources` rows; the git blob sha is `revision`
+    // and the package directory rides in `props`, which the v4 `files` table
+    // named `pkg`.
+    const { rows, error } = await readSqliteRows(
+      root,
+      `select s.path as path, s.revision as blob, s.lang as lang, s.bytes as bytes,
+        json_extract(s.props, '$.pkg') as pkg from sources s where s.kind = 'blob' order by s.path`,
+    );
     return withReadError(rows, error);
   }
   const { rows, error } = await readJsonl(root, 'files');
@@ -394,7 +418,14 @@ export async function readFiles(root, opts = {}) {
 export async function readSymbols(root, opts = {}) {
   const backend = opts.backend ?? await probeBackend();
   if (backend === 'sqlite') {
-    const { rows, error } = await readSqliteRows(root, 'select path, name, kind, line, sig, exported from symbols order by path, line, name, kind');
+    const { rows, error } = await readSqliteRows(
+      root,
+      `select s.path as path, n.name as name, json_extract(n.props, '$.kind') as kind,
+        n.line as line, json_extract(n.props, '$.sig') as sig,
+        json_extract(n.props, '$.exported') as exported
+        from nodes n join sources s on s.id = n.source_id and s.kind = 'blob'
+        where n.kind = 'symbol' order by s.path, n.line, n.name, n.kind`,
+    );
     return withReadError(rows, error);
   }
   const { rows, error } = await readJsonl(root, 'symbols');
@@ -405,7 +436,14 @@ export async function readSymbols(root, opts = {}) {
 export async function readEdges(root, opts = {}) {
   const backend = opts.backend ?? await probeBackend();
   if (backend === 'sqlite') {
-    const { rows, error } = await readSqliteRows(root, 'select src, dst, kind, weight from edges order by src, dst, kind');
+    // Paths come back from the node key (`file:<path>`) and the stable `dst_key`:
+    // node ids are regenerated when a file changes, so a path is the only key
+    // that survives a reindex of one endpoint.
+    const { rows, error } = await readSqliteRows(
+      root,
+      `select substr(ns.key, 6) as src, e.dst_key as dst, e.kind as kind, e.weight as weight
+        from edges e join nodes ns on ns.id = e.src_id order by src, dst, kind`,
+    );
     return withReadError(rows, error);
   }
   const { rows, error } = await readJsonl(root, 'edges');
@@ -416,7 +454,11 @@ export async function readEdges(root, opts = {}) {
 export async function readFts(root, opts = {}) {
   const backend = opts.backend ?? await probeBackend();
   if (backend === 'sqlite') {
-    const { rows, error } = await readSqliteRows(root, 'select kind, ref, body from fts order by ref, kind, rowid');
+    const { rows, error } = await readSqliteRows(
+      root,
+      `select f.kind as kind, substr(n.key, 6) as ref, f.body as body
+        from fts f join nodes n on n.id = f.node_id order by ref, kind, f.rowid`,
+    );
     return withReadError(rows, error);
   }
   const { rows, error } = await readJsonl(root, 'fts');
@@ -427,7 +469,12 @@ export async function readFts(root, opts = {}) {
 export async function readEmbeddings(root, opts = {}) {
   const backend = opts.backend ?? await probeBackend();
   if (backend === 'sqlite') {
-    const { rows: raw, error } = await readSqliteRows(root, 'select path, blob, model, dims, vec from embeddings order by path');
+    const { rows: raw, error } = await readSqliteRows(
+      root,
+      `select substr(n.key, 6) as path, s.revision as blob, v.model as model, v.dims as dims, v.vec as vec
+        from vectors v join nodes n on n.id = v.node_id
+        join sources s on s.id = n.source_id and s.kind = 'blob' order by path`,
+    );
     const rows = raw.map((row) => {
       const vec = decodeVectorBlob(row.vec);
       return {
@@ -500,11 +547,15 @@ export async function searchFtsBm25(root, task, opts = {}) {
 async function ftsBranch(root, table, terms) {
   if (terms.length === 0) return { rows: [], error: null };
   const matchExpr = terms.map((t) => `"${t}"`).join(' OR ');
-  const { rows, error } = await readSqliteRows(
-    root,
-    `select kind, ref as path, -bm25(${table}) as score from ${table} where ${table} match ? order by score desc, path asc limit ?`,
-    [matchExpr, FTS_LIMIT],
-  );
+  // `fts` is the v5 node-keyed table: the path comes back through the node key.
+  // `fts_tri` keeps its v4 (kind, ref, body) shape for the CJK trigram path (Q43),
+  // so it is queried the old way.
+  const sql = table === 'fts'
+    ? `select f.kind as kind, substr(n.key, 6) as path, -bm25(f) as score
+       from fts f join nodes n on n.id = f.node_id
+       where f match ? order by score desc, path asc limit ?`
+    : `select kind, ref as path, -bm25(${table}) as score from ${table} where ${table} match ? order by score desc, path asc limit ?`;
+  const { rows, error } = await readSqliteRows(root, sql, [matchExpr, FTS_LIMIT]);
   return {
     rows: rows.map((row) => ({ kind: row.kind, path: row.path, score: Number(row.score) || 0 })),
     error,
@@ -590,20 +641,10 @@ export async function packageOf(root, filePath, cache) {
   }
 }
 
-function git(root, args, input) {
-  return new Promise((resolve, reject) => {
-    const child = execFile('git', args, { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
-      if (error) reject(Object.assign(error, { stdout, stderr }));
-      else resolve(stdout);
-    });
-    if (input !== undefined) {
-      child.stdin.end(input);
-    }
-  });
-}
+
 
 async function lsFiles(root) {
-  const stdout = await git(root, ['ls-files', '-s', '-z']);
+  const stdout = await gitRaw(root, ['ls-files', '-s', '-z']);
   const rows = [];
   for (const rec of stdout.split('\0')) {
     if (rec === '') continue;
@@ -738,7 +779,9 @@ async function blobSizes(root, blobs) {
   const unique = [...new Set(blobs)];
   const sizes = new Map();
   if (unique.length === 0) return sizes;
-  const stdout = await git(root, ['cat-file', '--batch-check=%(objectname) %(objectsize)'], `${unique.join('\n')}\n`);
+  const stdout = await gitRaw(root, ['cat-file', '--batch-check=%(objectname) %(objectsize)'], {
+    input: `${unique.join('\n')}\n`,
+  });
   for (const line of stdout.split('\n')) {
     if (line === '') continue;
     const sp = line.indexOf(' ');

package/src/index/tests.mjs

@@ -6,8 +6,9 @@
 // the tree is not a guess. Co-change is a different kind: files that land in
 // the same commit are not, by that fact, a test.
 
-import { isTestPath } from '../bench/mine.mjs';
+import { isTestPath } from '../testing/paths.mjs';
 import { extract, resolveImports } from './extract.mjs';
+import { cmp } from '../util/sort.mjs';
 
 export { scriptSpecifiers } from './extract.mjs';
 
@@ -251,11 +252,6 @@ function byEdge(a, b) {
   return cmp(a.kind, b.kind) || cmp(a.src, b.src) || cmp(a.dst, b.dst);
 }
 
-function cmp(a, b) {
-  if (a < b) return -1;
-  if (a > b) return 1;
-  return 0;
-}
 
 /** Imports in a file, resolved against the tracked tree. One path now: `extract`
  * reads scripts with `scriptSpecifiers`, so gate, impact and the index agree on