simt-dev 0.3.1 → 0.5.0

package/src/index/map.mjs

@@ -1,19 +1,32 @@
-// Ranked repo map (R22, ADR-091).
+// Ranked repo map and hybrid retrieval (R22, R28, ADR-091).
 //
-// PageRank over import and call edges, personalised to the task. An edge is
-// followed in both directions: the file that imports a mentioned file is part of
-// the task, and so is the file it imports. One direction would hide one of them.
-// Iterations and the tie break are fixed so two runs of the same graph return
-// the same order. A random walk, or a tie broken by insertion order, would not.
+// PageRank over import and call edges, personalised to the task, fused with
+// FTS5 BM25 (`bm25()`) and resolver signals via reciprocal-rank fusion (RRF).
+// Optional local vector embeddings (stored as SQLite blobs via R37) are scored
+// by brute-force cosine similarity and stay off by default unless they raise
+// gold-file recall@10 by at least 10 percentage points (+0.10).
 
 import { readdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 
-import { resolveVaultDir } from '../bench/mine.mjs';
-import { readEdges, readFiles, readSymbols } from './store.mjs';
+import { isTestPath, resolveVaultDir } from '../bench/mine.mjs';
+import { readLocalConfig } from '../local/config.mjs';
+import { localEmbed } from '../local/client.mjs';
+import {
+  decodeVectorBlob,
+  readEdges,
+  readEmbeddings,
+  readFiles,
+  readFts,
+  readSymbols,
+  scoreFtsBm25,
+  searchFtsBm25,
+} from './store.mjs';
 
 export const ITERATIONS = 20;
 export const DAMPING = 0.85;
+export const RRF_K = 40;
+export const EMBED_RECALL_GAIN_BAR = 0.1;
 const TOP = 10;
 export const DEFAULT_BUDGET = 2048;
 
@@ -26,13 +39,106 @@ const STOP = new Set([
 ]);
 
 /**
- * Rank files for one task. No task text ranks the whole graph. A task ranks the
- * files it mentions and the files one import or call away from those.
+ * Reciprocal-rank fusion (RRF) over multiple ranked lists.
+ *
+ * Each list entry may be a path string or `{ path }`. Ties break by path ascending
+ * so fusion is strictly deterministic.
  *
- * @param {{files: {path: string}[], edges: {src: string, dst: string, kind: string, weight?: number}[], symbols: {path: string, name: string}[], task?: string}} input
+ * @param {{items: ({path: string}|string)[], weight?: number}[]} lists
+ * @param {number} [k]
  * @returns {{path: string, rank: number}[]}
  */
-export function rankFiles({ files, edges, symbols, task = '' }) {
+export function rrfFuse(lists, k = RRF_K) {
+  const fused = new Map();
+  const bestScore = new Map();
+  for (const { items = [], weight = 1 } of lists) {
+    if (!Array.isArray(items) || weight <= 0) continue;
+    for (let i = 0; i < items.length; i += 1) {
+      const entry = items[i];
+      const path = typeof entry === 'string' ? entry : entry?.path;
+      if (!path) continue;
+      fused.set(path, (fused.get(path) ?? 0) + weight / (k + i + 1));
+      if (typeof entry === 'object' && typeof entry?.score === 'number') {
+        bestScore.set(path, Math.max(bestScore.get(path) ?? 0, entry.score));
+      }
+    }
+  }
+  return [...fused.entries()]
+    .map(([path, rank]) => ({ path, rank, score: bestScore.get(path) ?? 0 }))
+    .sort((a, b) => (a.rank !== b.rank ? (a.rank < b.rank ? 1 : -1) : cmp(a.path, b.path)));
+}
+
+/**
+ * Brute-force cosine similarity between two numeric / Float32Array vectors.
+ */
+export function cosineSimilarity(a, b) {
+  if (!a || !b) return 0;
+  const len = Math.min(a.length ?? 0, b.length ?? 0);
+  if (len === 0) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < len; i += 1) {
+    const va = Number(a[i]) || 0;
+    const vb = Number(b[i]) || 0;
+    dot += va * vb;
+    normA += va * va;
+    normB += vb * vb;
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  if (denom === 0) return 0;
+  return dot / denom;
+}
+
+/**
+ * Rank stored vector embeddings against a query vector by brute-force cosine similarity.
+ *
+ * @param {{path: string, vec: Float32Array|number[]|Buffer}[]} embeddings
+ * @param {Float32Array|number[]|Buffer} queryVector
+ * @returns {{path: string, score: number}[]}
+ */
+export function rankByCosine(embeddings, queryVector) {
+  const qVec = queryVector instanceof Float32Array ? queryVector : decodeVectorBlob(queryVector);
+  if (!Array.isArray(embeddings) || embeddings.length === 0 || qVec.length === 0) return [];
+  const scored = [];
+  for (const row of embeddings) {
+    if (!row?.path) continue;
+    const vec = row.vec instanceof Float32Array ? row.vec : decodeVectorBlob(row.vec);
+    if (vec.length === 0) continue;
+    const sim = cosineSimilarity(vec, qVec);
+    if (sim > 0) scored.push({ path: row.path, score: sim });
+  }
+  return scored.sort((a, b) => (a.score !== b.score ? (a.score < b.score ? 1 : -1) : cmp(a.path, b.path)));
+}
+
+/**
+ * Rank files for one task. No task text ranks the whole graph. A task ranks the
+ * files it mentions and the files one import or call away from those, and when
+ * FTS / BM25 rows or optional vector embeddings are supplied, fuses BM25 +
+ * graph proximity + resolver signals via reciprocal-rank fusion (R28).
+ *
+ * @param {{
+ *   files: {path: string}[],
+ *   edges: {src: string, dst: string, kind: string, weight?: number}[],
+ *   symbols: {path: string, name: string}[],
+ *   fts?: {kind?: string, ref?: string, path?: string, body?: string}[],
+ *   bm25?: {path: string, score?: number}[],
+ *   embeddings?: {path: string, vec: Float32Array|number[]|Buffer}[],
+ *   queryVector?: Float32Array|number[]|Buffer|null,
+ *   task?: string
+ * }} input
+ * @returns {{path: string, rank: number, score: number}[]}
+ */
+export function rankFiles({
+  files,
+  edges,
+  symbols,
+  fts = undefined,
+  bm25 = undefined,
+  embeddings = undefined,
+  queryVector = null,
+  task = '',
+}) {
   const nodes = [...new Set(files.map((file) => file.path))].sort();
   const symbolsByPath = groupSymbols(symbols);
   const scores = scoreFiles(nodes, symbolsByPath, task);
@@ -43,9 +149,44 @@ export function rankFiles({ files, edges, symbols, task = '' }) {
   // A file the task mentions stays ahead of a file it does not. PageRank on
   // this graph otherwise returns the file every module imports, and the files
   // the task named never reach the top 10.
-  return kept
+  const baseRanked = kept
     .map((path, i) => ({ path, rank: rank[i], score: scores.get(path) ?? 0 }))
     .sort(byRank);
+
+  if (String(task ?? '').trim().length === 0) return baseRanked;
+
+  const knownPaths = new Set(nodes);
+  const rawBm25 = Array.isArray(bm25)
+    ? bm25
+    : Array.isArray(fts) && fts.length > 0
+      ? scoreFtsBm25(fts, task)
+      : [];
+  const bm25Ranked = rawBm25
+    .filter((row) => knownPaths.has(row.path) && !isTestPath(row.path))
+    .slice(0, 30);
+
+  const embedRanked =
+    queryVector && Array.isArray(embeddings) && embeddings.length > 0
+      ? rankByCosine(embeddings, queryVector)
+          .filter((row) => knownPaths.has(row.path) && !isTestPath(row.path))
+          .slice(0, 30)
+      : [];
+
+  if (bm25Ranked.length === 0 && embedRanked.length === 0) {
+    return baseRanked;
+  }
+
+  const resolverRanked = baseRanked.filter((row) => (row.score ?? 0) > 0);
+  const graphRanked = [...baseRanked].sort((a, b) => (a.rank !== b.rank ? (a.rank < b.rank ? 1 : -1) : cmp(a.path, b.path)));
+  const lists = [
+    { items: resolverRanked, weight: 2.0 },
+    { items: graphRanked, weight: 1.0 },
+    { items: bm25Ranked, weight: 1.0 },
+  ];
+  if (embedRanked.length > 0) {
+    lists.push({ items: embedRanked, weight: 1.5 });
+  }
+  return rrfFuse(lists, RRF_K);
 }
 
 /**
@@ -101,13 +242,47 @@ export function goldPaths(patch) {
   return [...new Set(out)].sort();
 }
 
+/**
+ * Embed a task query using the repository's configured `local.embed` endpoint.
+ *
+ * Returns `null` without opening a socket when `embed` is false, `local.embed`
+ * is unset, or a non-loopback URL is refused.
+ */
+export async function embedTaskQuery(root, task, { embed = false, allowRemote = false, fetchImpl = undefined } = {}) {
+  if (!embed || String(task ?? '').trim().length === 0) return null;
+  const localCfg = await readLocalConfig(root, { allowRemote });
+  if (!localCfg.embed || localCfg.embed.blocked) return null;
+  const res = await localEmbed(localCfg.embed, {
+    input: String(task).slice(0, 4096),
+    allowRemote,
+    fetchImpl,
+  });
+  if (!res.ok || !Array.isArray(res.embeddings?.[0]) || res.embeddings[0].length === 0) {
+    return null;
+  }
+  return Float32Array.from(res.embeddings[0], (v) => Number(v) || 0);
+}
+
 /**
  * Mean recall@10 over mined tasks whose gold patch is in the vault.
  * A missing vault is unmeasured, not a zero: the absence of a hidden patch is
  * not evidence the map missed the file (ADR-076).
  *
+ * When `embed: true` (or stored embeddings are provided), also evaluates recall@10
+ * with embeddings fused in and enforces the R28 acceptance gate: embeddings stay off
+ * by default unless they raise gold-file recall@10 by at least 10 percentage points
+ * (`EMBED_RECALL_GAIN_BAR = 0.1`).
+ *
  * @param {string} root
- * @param {{vaultRoot?: string, backend?: string}} [opts]
+ * @param {{
+ *   vaultRoot?: string,
+ *   backend?: string,
+ *   embed?: boolean,
+ *   allowRemote?: boolean,
+ *   fetchImpl?: typeof fetch,
+ *   embeddings?: {path: string, vec: Float32Array|number[]|Buffer}[],
+ *   taskVectors?: Record<string, Float32Array|number[]>|Map<string, Float32Array|number[]>
+ * }} [opts]
  */
 export async function measureRecall(root, opts = {}) {
   const tasksDir = join(root, '.simt-dev', 'bench', 'tasks');
@@ -121,9 +296,19 @@ export async function measureRecall(root, opts = {}) {
   if (files.length === 0) {
     return { measured: false, reason: 'no index', tasks: 0, missing: 0, recall: null };
   }
-  const [symbols, edges] = await Promise.all([readSymbols(root, opts), readEdges(root, opts)]);
+  const [symbols, edges, fts] = await Promise.all([
+    readSymbols(root, opts),
+    readEdges(root, opts),
+    readFts(root, opts),
+  ]);
+  const storedEmbeddings =
+    opts.embeddings ?? (opts.embed ? await readEmbeddings(root, opts) : []);
+  const checkEmbed = storedEmbeddings.length > 0 || opts.embed === true;
+
   const vault = opts.vaultRoot ?? resolveVaultDir(root);
   let sum = 0;
+  let embedSum = 0;
+  let embedEvaluated = 0;
   let tasks = 0;
   let missing = 0;
   for (const name of names) {
@@ -140,12 +325,61 @@ export async function measureRecall(root, opts = {}) {
       missing += 1;
       continue;
     }
-    const ranked = rankFiles({ files, edges, symbols, task: task.prompt ?? '' });
+    const prompt = task.prompt ?? '';
+    const bm25 = await searchFtsBm25(root, prompt, { ...opts, fts });
+    const ranked = rankFiles({ files, edges, symbols, fts, bm25, task: prompt });
     sum += recallAt10(ranked, gold);
     tasks += 1;
+
+    if (checkEmbed && storedEmbeddings.length > 0) {
+      const providedVec =
+        opts.taskVectors instanceof Map
+          ? opts.taskVectors.get(task.id)
+          : opts.taskVectors?.[task.id];
+      const queryVector =
+        providedVec ??
+        (await embedTaskQuery(root, prompt, {
+          embed: opts.embed === true,
+          allowRemote: opts.allowRemote === true,
+          fetchImpl: opts.fetchImpl,
+        }));
+      if (queryVector) {
+        const embedRanked = rankFiles({
+          files,
+          edges,
+          symbols,
+          fts,
+          bm25,
+          embeddings: storedEmbeddings,
+          queryVector,
+          task: prompt,
+        });
+        embedSum += recallAt10(embedRanked, gold);
+        embedEvaluated += 1;
+      } else {
+        embedSum += recallAt10(ranked, gold);
+      }
+    }
   }
   if (tasks === 0) return { measured: false, reason: 'no gold patches', tasks: 0, missing, recall: null };
-  return { measured: true, reason: '', tasks, missing, recall: sum / tasks };
+  const baseRecall = sum / tasks;
+  const embedRecall = checkEmbed && embedEvaluated > 0 ? embedSum / tasks : null;
+  const embedDelta = embedRecall === null ? null : embedRecall - baseRecall;
+  // R28 acceptance gate: embeddings stay off by default unless they raise
+  // gold-file recall@10 by at least 10 points (+0.10).
+  const embedAccepted = embedDelta !== null && embedDelta + 1e-9 >= EMBED_RECALL_GAIN_BAR;
+  return {
+    measured: true,
+    reason: '',
+    tasks,
+    missing,
+    recall: embedAccepted ? embedRecall : baseRecall,
+    baseRecall,
+    embedRecall,
+    embedDelta,
+    embedAccepted,
+    embeddingsDefault: 'off',
+  };
 }
 
 export function formatRecall(result) {

package/src/index/persist.mjs

@@ -9,17 +9,42 @@ import { dirname, join } from 'node:path';
 
 export const SQLITE_FILE = '.simt-dev/index.sqlite';
 export const JSONL_FILE = '.simt-dev/index.jsonl';
-export const SCHEMA_V = '2';
+export const SCHEMA_V = '3';
 
 const SCHEMA = `
 create table if not exists files(path text primary key, blob text, lang text, bytes int, pkg text);
 create table if not exists symbols(id integer primary key, path text, name text, kind text, line int, sig text, exported int);
 create table if not exists edges(src text, dst text, kind text, weight real);
 create virtual table if not exists fts using fts5(kind, ref, body);
+create table if not exists embeddings(path text primary key, blob text, model text, dims int, vec blob);
 create table if not exists meta(key text primary key, value text);
 `;
 
-export async function writeSqlite(root, next, removed, symbols, edges, fts) {
+export function encodeVectorBlob(vec) {
+  if (!Array.isArray(vec) && !(vec instanceof Float32Array)) return Buffer.alloc(0);
+  const f32 = vec instanceof Float32Array ? vec : Float32Array.from(vec, (v) => Number(v) || 0);
+  return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+
+export function decodeVectorBlob(blob) {
+  if (!blob) return new Float32Array(0);
+  if (blob instanceof Float32Array) return blob;
+  if (Array.isArray(blob)) return Float32Array.from(blob, (v) => Number(v) || 0);
+  const buf = Buffer.isBuffer(blob)
+    ? blob
+    : blob instanceof Uint8Array
+      ? Buffer.from(blob.buffer, blob.byteOffset, blob.byteLength)
+      : typeof blob === 'string'
+        ? Buffer.from(blob, 'base64')
+        : Buffer.alloc(0);
+  const count = Math.floor(buf.byteLength / 4);
+  if (count === 0) return new Float32Array(0);
+  const copy = new Uint8Array(count * 4);
+  copy.set(new Uint8Array(buf.buffer, buf.byteOffset, count * 4));
+  return new Float32Array(copy.buffer);
+}
+
+export async function writeSqlite(root, next, removed, symbols, edges, fts, embeddings = []) {
   const { DatabaseSync } = await import('node:sqlite');
   const file = join(root, SQLITE_FILE);
   await mkdir(dirname(file), { recursive: true });
@@ -32,25 +57,35 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts) {
     const dropSymbols = db.prepare('delete from symbols where path = ?');
     const dropEdges = db.prepare('delete from edges where src = ? or dst = ?');
     const dropFts = db.prepare('delete from fts where ref = ?');
+    const dropEmbed = db.prepare('delete from embeddings where path = ?');
     const dropFile = db.prepare('delete from files where path = ?');
     const addSymbol = db.prepare('insert into symbols(path, name, kind, line, sig, exported) values (?, ?, ?, ?, ?, ?)');
     const addEdge = db.prepare('insert into edges(src, dst, kind, weight) values (?, ?, ?, ?)');
     const addFts = db.prepare('insert into fts(kind, ref, body) values (?, ?, ?)');
+    const upsertEmbed = db.prepare(`insert into embeddings(path, blob, model, dims, vec) values (?, ?, ?, ?, ?)
+      on conflict(path) do update set blob = excluded.blob, model = excluded.model, dims = excluded.dims, vec = excluded.vec`);
     for (const row of next) {
       upsert.run(row.path, row.blob, row.lang, row.bytes, row.pkg);
       dropSymbols.run(row.path);
       dropEdges.run(row.path, row.path);
       dropFts.run(row.path);
+      dropEmbed.run(row.path);
     }
     for (const path of removed) {
       dropFile.run(path);
       dropSymbols.run(path);
       dropEdges.run(path, path);
       dropFts.run(path);
+      dropEmbed.run(path);
     }
     for (const row of symbols) addSymbol.run(row.path, row.name, row.kind, row.line, row.sig, row.exported);
     for (const row of edges) addEdge.run(row.src, row.dst, row.kind, row.weight);
     for (const row of fts) addFts.run(row.kind, row.ref, row.body);
+    for (const row of embeddings) {
+      const vecBlob = Buffer.isBuffer(row.vec) || row.vec instanceof Uint8Array ? row.vec : encodeVectorBlob(row.vec);
+      const dims = Number.isInteger(row.dims) ? row.dims : Math.floor(vecBlob.byteLength / 4);
+      upsertEmbed.run(row.path, row.blob ?? '', row.model ?? '', dims, vecBlob);
+    }
     db.prepare('insert into meta(key, value) values (?, ?) on conflict(key) do update set value = excluded.value').run('schema', SCHEMA_V);
     db.exec('commit');
   } catch (error) {
@@ -61,7 +96,7 @@ export async function writeSqlite(root, next, removed, symbols, edges, fts) {
   }
 }
 
-export async function writeJsonl(root, previous, next, removed, symbols, edges, fts, force) {
+export async function writeJsonl(root, previous, next, removed, symbols, edges, fts, force, embeddings = []) {
   const gone = new Set(removed);
   const replaced = new Map(next.map((row) => [row.path, row]));
   const drop = new Set([...replaced.keys(), ...gone]);
@@ -76,9 +111,14 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
   const keptSymbols = force ? [] : (await readJsonl(root, 'symbols')).filter((row) => !drop.has(row.path));
   const keptEdges = force ? [] : (await readJsonl(root, 'edges')).filter((row) => !drop.has(row.src) && !drop.has(row.dst));
   const keptFts = force ? [] : (await readJsonl(root, 'fts')).filter((row) => !drop.has(row.ref));
+  const newEmbedPaths = new Set(embeddings.map((row) => row.path));
+  const keptEmbeddings = force
+    ? []
+    : (await readJsonl(root, 'embeddings')).filter((row) => !drop.has(row.path) && !newEmbedPaths.has(row.path));
   const allSymbols = [...keptSymbols, ...symbols].sort(bySymbol);
   const allEdges = [...keptEdges, ...edges].sort(byEdge);
   const allFts = [...keptFts, ...fts].sort((a, b) => cmp(a.ref, b.ref));
+  const allEmbeddings = [...keptEmbeddings, ...embeddings].sort(byPath);
   const file = join(root, JSONL_FILE);
   await mkdir(dirname(file), { recursive: true });
   const lines = [
@@ -87,9 +127,21 @@ export async function writeJsonl(root, previous, next, removed, symbols, edges,
     ...allSymbols.map((row) => JSON.stringify({ table: 'symbols', path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported })),
     ...allEdges.map((row) => JSON.stringify({ table: 'edges', src: row.src, dst: row.dst, kind: row.kind, weight: row.weight })),
     ...allFts.map((row) => JSON.stringify({ table: 'fts', kind: row.kind, ref: row.ref, body: row.body })),
+    ...allEmbeddings.map((row) => {
+      const vecBlob = Buffer.isBuffer(row.vec) || row.vec instanceof Uint8Array ? Buffer.from(row.vec.buffer, row.vec.byteOffset, row.vec.byteLength) : encodeVectorBlob(row.vec);
+      const dims = Number.isInteger(row.dims) ? row.dims : Math.floor(vecBlob.byteLength / 4);
+      return JSON.stringify({
+        table: 'embeddings',
+        path: row.path,
+        blob: row.blob ?? '',
+        model: row.model ?? '',
+        dims,
+        vec: vecBlob.toString('base64'),
+      });
+    }),
   ];
   await writeFile(file, `${lines.join('\n')}\n`);
-  return { symbols: allSymbols.length, edges: allEdges.length };
+  return { symbols: allSymbols.length, edges: allEdges.length, embeddings: allEmbeddings.length };
 }
 
 export async function readJsonl(root, table) {
@@ -109,6 +161,16 @@ export async function readJsonl(root, table) {
     else if (table === 'symbols') rows.push({ path: row.path, name: row.name, kind: row.kind, line: row.line, sig: row.sig, exported: row.exported });
     else if (table === 'edges') rows.push({ src: row.src, dst: row.dst, kind: row.kind, weight: row.weight });
     else if (table === 'fts') rows.push({ kind: row.kind, ref: row.ref, body: row.body });
+    else if (table === 'embeddings') {
+      const vec = decodeVectorBlob(row.vec);
+      rows.push({
+        path: row.path,
+        blob: row.blob ?? '',
+        model: row.model ?? '',
+        dims: Number.isInteger(row.dims) ? row.dims : vec.length,
+        vec,
+      });
+    }
   }
   if (table === 'symbols') rows.sort(bySymbol);
   else if (table === 'edges') rows.sort(byEdge);
@@ -117,7 +179,7 @@ export async function readJsonl(root, table) {
   return rows;
 }
 
-export async function readSqliteRows(root, sql) {
+export async function readSqliteRows(root, sql, params = []) {
   let DatabaseSync;
   try {
     ({ DatabaseSync } = await import('node:sqlite'));
@@ -132,7 +194,7 @@ export async function readSqliteRows(root, sql) {
   }
   const db = new DatabaseSync(file);
   try {
-    return db.prepare(sql).all();
+    return db.prepare(sql).all(...params);
   } catch {
     return [];
   } finally {
@@ -152,7 +214,12 @@ export async function readSchema(root, backend) {
 export async function countSqlite(root) {
   const symbols = await readSqliteRows(root, 'select count(*) as n from symbols');
   const edges = await readSqliteRows(root, 'select count(*) as n from edges');
-  return { symbols: Number(symbols[0]?.n ?? 0), edges: Number(edges[0]?.n ?? 0) };
+  const embeddings = await readSqliteRows(root, 'select count(*) as n from embeddings');
+  return {
+    symbols: Number(symbols[0]?.n ?? 0),
+    edges: Number(edges[0]?.n ?? 0),
+    embeddings: Number(embeddings[0]?.n ?? 0),
+  };
 }
 
 function byPath(a, b) {