simt-dev 0.6.3 → 0.7.0

package/src/index/persist.mjs

@@ -4,18 +4,80 @@
 // rows. This file is the format; store.mjs is the walk that fills it. They are
 // split because the walk and the format together crossed the review trigger.
 
+import { createHash } from 'node:crypto';
 import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 
-export const SQLITE_FILE = '.simt/index.sqlite';
-export const JSONL_FILE = '.simt/index.jsonl';
-export const SCHEMA_V = '3';
+import { LOCAL_DIR } from '../state.mjs';
+
+export const SQLITE_FILE = join(LOCAL_DIR, 'index.sqlite');
+export const JSONL_FILE = join(LOCAL_DIR, 'index.jsonl');
+const SQLITE_FILE_LEGACY = '.simt/index.sqlite';
+const JSONL_FILE_LEGACY = '.simt/index.jsonl';
+export const SCHEMA_V = '4';
+
+// The FTS tables carry their tokenizer in their definition, so `if not exists`
+// cannot migrate them: a v3 database would keep tokenizing new rows the v3 way
+// (ASCII fold, kind and ref searchable). The 3 -> 4 migration below recreates
+// them, which is the versioned replacement for the per-write wipe (Q44).
+export const FTS_SCHEMA = 'create virtual table fts using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "unicode61 remove_diacritics 2")';
+export const FTS_TRI_SCHEMA = 'create virtual table fts_tri using fts5(kind UNINDEXED, ref UNINDEXED, body, tokenize = "trigram")';
+
+// A database written by a *newer* simt is never opened for write, and never
+// read blind (review 0003 Q44): the mismatched canonical file stays with its
+// owner, and this version reads and writes a sidecar named after itself, so
+// two pinned versions stop rewriting one file past each other (the ADR-059
+// flip-flop). An *older* database is read and written in place, and the
+// migration brings it current.
+export function indexFileFor(root, backend, version) {
+  const base = backend === 'sqlite' ? SQLITE_FILE : JSONL_FILE;
+  if (version === '' || version === SCHEMA_V) return join(root, base);
+  const dot = base.lastIndexOf('.');
+  return join(root, `${base.slice(0, dot)}.v${SCHEMA_V}${base.slice(dot)}`);
+}
+
+// The canonical file is opened for write when the stored version is this one
+// or older - the ordered migration upgrades the older one in place. A newer
+// version owns its file, and opening it here would demote the stamp.
+function canonicalForWrite(version) {
+  const at = Number(version);
+  return Number.isInteger(at) && at <= Number(SCHEMA_V);
+}
+
+// Ordered schema migrations: each step upgrades a database written at `from`
+// to `to`, applied in list order. A database at a version past the last step
+// is not opened for write at all (indexFileFor routes it to a sidecar).
+const MIGRATIONS = [
+  {
+    from: 3,
+    to: 4,
+    apply: (db) => {
+      db.exec('drop table if exists fts');
+      db.exec(FTS_SCHEMA);
+      db.exec('drop table if exists fts_tri');
+      db.exec(FTS_TRI_SCHEMA);
+    },
+  },
+];
+
+/**
+ * Bring an open database up to the current schema and stamp it. `user_version`
+ * 0 means "written before the stamp existed": those databases carried the
+ * version in meta only, so every step at or below the current schema runs -
+ * each is idempotent on an already-migrated table set.
+ */
+function migrateDb(db) {
+  const at = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
+  for (const step of MIGRATIONS) {
+    if (at === 0 || (at >= step.from && at < step.to)) step.apply(db);
+  }
+  db.exec(`pragma user_version = ${Number(SCHEMA_V)}`);
+}
 
 const SCHEMA = `
 create table if not exists files(path text primary key, blob text, lang text, bytes int, pkg text);
 create table if not exists symbols(id integer primary key, path text, name text, kind text, line int, sig text, exported int);
 create table if not exists edges(src text, dst text, kind text, weight real);
-create virtual table if not exists fts using fts5(kind, ref, body);
 create table if not exists embeddings(path text primary key, blob text, model text, dims int, vec blob);
 create table if not exists meta(key text primary key, value text);
 `;
@@ -52,9 +114,51 @@ function tmpFor(file) {
   return `${file}.${process.pid}.${tmpSeq}.tmp`;
 }
 
-export async function writeSqlite(root, next, removed, symbols, edges, fts, embeddings = []) {
+/**
+ * What the index was built from, so a reader can tell a current tree from an
+ * old one without re-reading any blob (review 0003 Q42).
+ *
+ * The digest covers the `git ls-files -s` listing the build consumed: same
+ * bytes in, same digest out. Stored beside `schema` in `meta`, written on every
+ * build including the no-change one, so a build that changed nothing still
+ * vouches for the tree it saw.
+ */
+export function treeDigest(listed) {
+  const rows = [...(listed ?? [])]
+    .map((entry) => `${entry.blob ?? ''} ${entry.path ?? ''}`)
+    .sort()
+    .join('\n');
+  return createHash('sha1').update(rows, 'utf8').digest('hex');
+}
+
+export async function writeSqlite(root, next, removed, symbols, edges, fts, embeddings = [], tree = '') {
   const { DatabaseSync } = await import('node:sqlite');
-  const file = join(root, SQLITE_FILE);
+  // The canonical file is written when it is this version's or older (the
+  // migration upgrades the older one in place, review 0003 Q44): a database at
+  // a newer version stays with its owner, and the build goes to the sidecar
+  // this version owns.
+  let file = join(root, SQLITE_FILE);
+  try {
+    await stat(file);
+  } catch {
+    file = join(root, SQLITE_FILE_LEGACY);
+    try {
+      await stat(file);
+    } catch {
+      file = null;
+    }
+  }
+  if (file) {
+    let version = SCHEMA_V;
+    try {
+      version = await storedVersion(file);
+    } catch {
+      version = SCHEMA_V; // unreadable: rebuild it in place, as a cold build would
+    }
+    file = canonicalForWrite(version) ? file : indexFileFor(root, 'sqlite', version);
+  } else {
+    file = join(root, SQLITE_FILE);
+  }
   await mkdir(dirname(file), { recursive: true });
   // A cold build goes to a temp file that is renamed over the target: three
   // concurrent cold runs used to interleave schema creation and leave a locked,
@@ -83,6 +187,7 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts, embe
     db.exec('pragma synchronous = normal');
     db.exec(SCHEMA);
     db.exec('begin immediate');
+    migrateDb(db);
     const upsert = db.prepare(`insert into files(path, blob, lang, bytes, pkg) values (?, ?, ?, ?, ?)
       on conflict(path) do update set blob = excluded.blob, lang = excluded.lang, bytes = excluded.bytes, pkg = excluded.pkg`);
     const dropSymbols = db.prepare('delete from symbols where path = ?');
@@ -92,11 +197,13 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts, embe
     const dropEdges = db.prepare('delete from edges where src = ?');
     const dropIncoming = db.prepare('delete from edges where dst = ?');
     const dropFts = db.prepare('delete from fts where ref = ?');
+    const dropTri = db.prepare('delete from fts_tri where ref = ?');
     const dropEmbed = db.prepare('delete from embeddings where path = ?');
     const dropFile = db.prepare('delete from files where path = ?');
     const addSymbol = db.prepare('insert into symbols(path, name, kind, line, sig, exported) values (?, ?, ?, ?, ?, ?)');
     const addEdge = db.prepare('insert into edges(src, dst, kind, weight) values (?, ?, ?, ?)');
     const addFts = db.prepare('insert into fts(kind, ref, body) values (?, ?, ?)');
+    const addTri = db.prepare('insert into fts_tri(kind, ref, body) values (?, ?, ?)');
     const upsertEmbed = db.prepare(`insert into embeddings(path, blob, model, dims, vec) values (?, ?, ?, ?, ?)
       on conflict(path) do update set blob = excluded.blob, model = excluded.model, dims = excluded.dims, vec = excluded.vec`);
     for (const row of next) {
@@ -104,6 +211,7 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts, embe
       dropSymbols.run(row.path);
       dropEdges.run(row.path);
       dropFts.run(row.path);
+      dropTri.run(row.path);
       dropEmbed.run(row.path);
     }
     for (const path of removed) {
@@ -112,17 +220,27 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts, embe
       dropEdges.run(path);
       dropIncoming.run(path);
       dropFts.run(path);
+      dropTri.run(path);
       dropEmbed.run(path);
     }
     for (const row of symbols) addSymbol.run(row.path, row.name, row.kind, row.line, row.sig, row.exported);
     for (const row of edges) addEdge.run(row.src, row.dst, row.kind, row.weight);
-    for (const row of fts) addFts.run(row.kind, row.ref, row.body);
+    for (const row of fts) {
+      addFts.run(row.kind, row.ref, row.body);
+      // The trigram table carries folded raw text for CJK-bearing sections
+      // only; an ASCII-only section is already reachable through `fts`.
+      if (row.raw) addTri.run(row.kind, row.ref, row.raw);
+    }
     for (const row of embeddings) {
       const vecBlob = Buffer.isBuffer(row.vec) || row.vec instanceof Uint8Array ? row.vec : encodeVectorBlob(row.vec);
       const dims = Number.isInteger(row.dims) ? row.dims : Math.floor(vecBlob.byteLength / 4);
       upsertEmbed.run(row.path, row.blob ?? '', row.model ?? '', dims, vecBlob);
     }
-    db.prepare('insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value').run('schema', SCHEMA_V);
+    const upsertMeta = db.prepare(
+      'insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value',
+    );
+    upsertMeta.run('schema', SCHEMA_V);
+    upsertMeta.run('tree', tree);
     db.exec('commit');
   } catch (error) {
     try { db.exec('rollback'); } catch { /* a failed begin has nothing to undo */ }
@@ -133,6 +251,32 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts, embe
   if (cold) await rename(target, file);
 }
 
+/**
+ * The version a database declares: `PRAGMA user_version` when stamped, else the
+ * meta `schema` row databases written before the stamp carried it in. Throws
+ * when the file cannot be read as a database - the caller decides whether that
+ * is a cold start or a corrupt one.
+ */
+async function storedVersion(file) {
+  const { DatabaseSync } = await import('node:sqlite');
+  const db = new DatabaseSync(file);
+  try {
+    const user = Number(db.prepare('pragma user_version').get()?.user_version ?? 0);
+    if (user > 0) return String(user);
+    return db.prepare("select value from meta where key = 'schema'").get()?.value ?? '';
+  } finally {
+    db.close();
+  }
+}
+
+/** A damaged index file is named, not silent (review 0003 Q44). */
+function corruptIndexError(cause) {
+  const corrupt = new Error('the index file is corrupt; run the index build to replace it');
+  corrupt.code = 'INDEX_CORRUPT';
+  corrupt.cause = cause;
+  return corrupt;
+}
+
 /**
  * A contended index write is a retryable race, not a broken repository: name it
  * so the command maps its own remedy instead of blaming the git checkout (Q08).
@@ -145,10 +289,46 @@ function asIndexError(error) {
     busy.cause = error;
     return busy;
   }
+  if (/file is not a database|database disk image|malformed|corrupt/i.test(error?.message ?? '')) {
+    return corruptIndexError(error);
+  }
   return error;
 }
 
-export async function writeJsonl(root, previous, next, removed, symbols, edges, fts, force, embeddings = []) {
+/**
+ * The jsonl file this version writes to: the canonical file when it is this
+ * version's or unreadable, the sidecar when it belongs to another version
+ * (review 0003 Q44).
+ */
+async function jsonlTargetFile(root) {
+  const base = join(root, JSONL_FILE);
+  let text = null;
+  try {
+    text = await readFile(base, 'utf8');
+  } catch {
+    try {
+      text = await readFile(join(root, JSONL_FILE_LEGACY), 'utf8');
+    } catch {
+      return base; // no file: cold build goes to the canonical path
+    }
+  }
+  try {
+    for (const line of text.split('\n')) {
+      if (line === '') continue;
+      const row = JSON.parse(line);
+      if (row.table !== 'meta' || row.key !== 'schema') continue;
+      const value = row.value ?? '';
+      // Older files are rewritten in place at the current schema; a newer one
+      // stays with its owner and the build goes to the sidecar.
+      return canonicalForWrite(value) ? base : indexFileFor(root, 'jsonl', value);
+    }
+  } catch {
+    return base; // unreadable: rebuild it in place
+  }
+  return base;
+}
+
+export async function writeJsonl(root, previous, next, removed, symbols, edges, fts, force, embeddings = [], tree = '') {
   const gone = new Set(removed);
   const replaced = new Map(next.map((row) => [row.path, row]));
   const drop = new Set([...replaced.keys(), ...gone]);
@@ -160,25 +340,26 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
   }
   for (const row of replaced.values()) files.push(row);
   files.sort(byPath);
-  const keptSymbols = force ? [] : (await readJsonl(root, 'symbols')).filter((row) => !drop.has(row.path));
+  const keptSymbols = force ? [] : (await readJsonl(root, 'symbols')).rows.filter((row) => !drop.has(row.path));
   // Edge ownership mirrors the sqlite backend: a changed file loses only the edges
   // it owns (src), while a removed file also drops the edges that pointed at it.
-  const keptEdges = force ? [] : (await readJsonl(root, 'edges')).filter((row) => !drop.has(row.src) && !gone.has(row.dst));
-  const keptFts = force ? [] : (await readJsonl(root, 'fts')).filter((row) => !drop.has(row.ref));
+  const keptEdges = force ? [] : (await readJsonl(root, 'edges')).rows.filter((row) => !drop.has(row.src) && !gone.has(row.dst));
+  const keptFts = force ? [] : (await readJsonl(root, 'fts')).rows.filter((row) => !drop.has(row.ref));
   const newEmbedPaths = new Set(embeddings.map((row) => row.path));
   const keptEmbeddings = force
     ? []
-    : (await readJsonl(root, 'embeddings')).filter((row) => !drop.has(row.path) && !newEmbedPaths.has(row.path));
+    : (await readJsonl(root, 'embeddings')).rows.filter((row) => !drop.has(row.path) && !newEmbedPaths.has(row.path));
   const allSymbols = [...keptSymbols, ...symbols].sort(bySymbol);
   const allEdges = [...keptEdges, ...edges].sort(byEdge);
   const allFts = [...keptFts, ...fts].sort((a, b) => cmp(a.ref, b.ref));
   const allEmbeddings = [...keptEmbeddings, ...embeddings].sort(byPath);
-  const file = join(root, JSONL_FILE);
+  const file = await jsonlTargetFile(root);
   await mkdir(dirname(file), { recursive: true });
   // Written atomically: a concurrent reader never sees a half-written file.
   const tmp = tmpFor(file);
   const lines = [
     JSON.stringify({ table: 'meta', key: 'schema', value: SCHEMA_V }),
+    JSON.stringify({ table: 'meta', key: 'tree', value: tree }),
     ...files.map((row) => JSON.stringify({ table: 'files', path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg })),
     ...allSymbols.map((row) => JSON.stringify({ table: 'symbols', path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported })),
     ...allEdges.map((row) => JSON.stringify({ table: 'edges', src: row.src, dst: row.dst, kind: row.kind, weight: row.weight })),
@@ -202,58 +383,127 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
 }
 
 export async function readJsonl(root, table) {
+  // The pre-move path is a fallback for one release (review 0003 Q19).
   let text;
   try {
     text = await readFile(join(root, JSONL_FILE), 'utf8');
   } catch {
-    return [];
+    try {
+      text = await readFile(join(root, JSONL_FILE_LEGACY), 'utf8');
+    } catch {
+      return { rows: [], error: null };
+    }
+  }
+  let schema = '';
+  try {
+    for (const line of text.split('\n')) {
+      if (line === '') continue;
+      const row = JSON.parse(line);
+      if (row.table === 'meta' && row.key === 'schema') schema = row.value ?? '';
+    }
+  } catch (error) {
+    return { rows: [], error: corruptIndexError(error) };
+  }
+  // A file a newer version owns is not read blind (review 0003 Q44): this
+  // version's sidecar is read instead, and a sidecar that has not been built
+  // yet is cold, not broken. An older file is read in place; the next write
+  // rewrites it at the current schema.
+  if (!canonicalForWrite(schema)) {
+    try {
+      text = await readFile(indexFileFor(root, 'jsonl', schema), 'utf8');
+    } catch {
+      return { rows: [], error: null };
+    }
   }
   const rows = [];
-  for (const line of text.split('\n')) {
-    if (line === '') continue;
-    const row = JSON.parse(line);
-    if (row.table !== table) continue;
-    if (table === 'meta') rows.push({ key: row.key, value: row.value });
-    else if (table === 'files') rows.push({ path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg });
-    else if (table === 'symbols') rows.push({ path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported });
-    else if (table === 'edges') rows.push({ src: row.src, dst: row.dst, kind: row.kind, weight: row.weight });
-    else if (table === 'fts') rows.push({ kind: row.kind, ref: row.ref, body: row.body });
-    else if (table === 'embeddings') {
-      const vec = decodeVectorBlob(row.vec);
-      rows.push({
-        path: row.path,
-        blob: row.blob ?? '',
-        model: row.model ?? '',
-        dims: Number.isInteger(row.dims) ? row.dims : vec.length,
-        vec,
-      });
+  try {
+    for (const line of text.split('\n')) {
+      if (line === '') continue;
+      const row = JSON.parse(line);
+      if (row.table !== table) continue;
+      if (table === 'meta') rows.push({ key: row.key, value: row.value });
+      else if (table === 'files') rows.push({ path: row.path, blob: row.blob, lang: row.lang, bytes: row.bytes, pkg: row.pkg });
+      else if (table === 'symbols') rows.push({ path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported });
+      else if (table === 'edges') rows.push({ src: row.src, dst: row.dst, kind: row.kind, weight: row.weight });
+      else if (table === 'fts') rows.push({ kind: row.kind, ref: row.ref, body: row.body });
+      else if (table === 'embeddings') {
+        const vec = decodeVectorBlob(row.vec);
+        rows.push({
+          path: row.path,
+          blob: row.blob ?? '',
+          model: row.model ?? '',
+          dims: Number.isInteger(row.dims) ? row.dims : vec.length,
+          vec,
+        });
+      }
     }
+  } catch (error) {
+    return { rows: [], error: corruptIndexError(error) };
   }
   if (table === 'symbols') rows.sort(bySymbol);
   else if (table === 'edges') rows.sort(byEdge);
   else if (table === 'fts') rows.sort((a, b) => cmp(a.ref, b.ref));
   else rows.sort(byPath);
-  return rows;
+  return { rows, error: null };
 }
 
+/**
+ * The sqlite file a read opens: the canonical file when it is this version's
+ * or older, the sidecar when it belongs to a newer one (review 0003 Q44),
+ * null when no index for this version exists. Throws when a present file
+ * cannot be read as a database - corrupt, not cold.
+ */
+async function resolveSqliteFile(root) {
+  // The pre-move path is a fallback for one release (review 0003 Q19).
+  let file = join(root, SQLITE_FILE);
+  try {
+    await stat(file);
+  } catch {
+    file = join(root, SQLITE_FILE_LEGACY);
+    try {
+      await stat(file);
+    } catch {
+      return null;
+    }
+  }
+  const version = await storedVersion(file);
+  // An older database is readable in place: its tables carry the same columns,
+  // and the next write migrates it. Only a newer one is routed to a sidecar.
+  if (canonicalForWrite(version)) return file;
+  const sidecar = indexFileFor(root, 'sqlite', version);
+  try {
+    await stat(sidecar);
+    return sidecar;
+  } catch {
+    return null; // this version has not built its sidecar yet: cold
+  }
+}
+
+/**
+ * Read rows from the index. Returns `{ rows, error }`: a missing index is
+ * `{ rows: [], error: null }` (cold), a present index that cannot be read is
+ * an empty row set with a named error - "locked", "no such column" and
+ * "corrupt" used to all look like "no results" (review 0003 Q44).
+ */
 export async function readSqliteRows(root, sql, params = []) {
   let DatabaseSync;
   try {
     ({ DatabaseSync } = await import('node:sqlite'));
   } catch {
-    return [];
+    return { rows: [], error: null };
   }
-  const file = join(root, SQLITE_FILE);
+  let file;
   try {
-    await stat(file);
-  } catch {
-    return [];
+    file = await resolveSqliteFile(root);
+  } catch (error) {
+    return { rows: [], error: asIndexError(error) };
   }
+  if (!file) return { rows: [], error: null };
   const db = new DatabaseSync(file);
   try {
-    return db.prepare(sql).all(...params);
-  } catch {
-    return [];
+    return { rows: db.prepare(sql).all(...params), error: null };
+  } catch (error) {
+    return { rows: [], error: asIndexError(error) };
   } finally {
     db.close();
   }
@@ -261,21 +511,31 @@ export async function readSqliteRows(root, sql, params = []) {
 
 export async function readSchema(root, backend) {
   if (backend === 'sqlite') {
-    const rows = await readSqliteRows(root, "select value from meta where key = 'schema'");
+    const { rows } = await readSqliteRows(root, "select value from meta where key = 'schema'");
     return rows[0]?.value ?? '';
   }
-  const rows = await readJsonl(root, 'meta');
+  const { rows } = await readJsonl(root, 'meta');
   return rows.find((row) => row.key === 'schema')?.value ?? '';
 }
 
+/** The tree the stored index was built from, or '' when no build has vouched for one. */
+export async function readTreeDigest(root, backend) {
+  if (backend === 'sqlite') {
+    const { rows } = await readSqliteRows(root, "select value from meta where key = 'tree'");
+    return rows[0]?.value ?? '';
+  }
+  const { rows } = await readJsonl(root, 'meta');
+  return rows.find((row) => row.key === 'tree')?.value ?? '';
+}
+
 export async function countSqlite(root) {
   const symbols = await readSqliteRows(root, 'select count(*) as n from symbols');
   const edges = await readSqliteRows(root, 'select count(*) as n from edges');
   const embeddings = await readSqliteRows(root, 'select count(*) as n from embeddings');
   return {
-    symbols: Number(symbols[0]?.n ?? 0),
-    edges: Number(edges[0]?.n ?? 0),
-    embeddings: Number(embeddings[0]?.n ?? 0),
+    symbols: Number(symbols.rows[0]?.n ?? 0),
+    edges: Number(edges.rows[0]?.n ?? 0),
+    embeddings: Number(embeddings.rows[0]?.n ?? 0),
   };
 }