simt-dev 0.7.0 → 0.8.0

package/src/index/coverage.mjs

@@ -0,0 +1,58 @@
+// Which tests a change should run, from one place (Q77).
+//
+// Four callers used to answer this question four ways: `tests` walked the stored test
+// edges directly, `gate` walked the import graph transitively over a re-read worktree,
+// `impact` asked `dependents` which asked `selectTests`, and `brief` used the edges again.
+// The answers disagreed — `simt tests src/index/persist.mjs` named one test where `simt
+// gate` on the same file named dozens — and every disagreement was a reader trusting the
+// wrong one.
+//
+// The two walks are not the same question, so they are not merged into one answer:
+// `transitive` is the import closure over changed paths, and the direct walk is the test
+// edges the index stores. Each caller keeps its default; what changes is that there is
+// now one implementation of each.
+
+import { selectTests } from './gate.mjs';
+import { testEdges, testsFor } from './tests.mjs';
+
+/**
+ * Tests that exercise `paths`.
+ *
+ * @param {{files?: {path: string}[]|string[], imports?: {src: string, dst: string, kind?: string}[],
+ *   coverage?: string[], commits?: string[][]}} graph
+ * @param {Iterable<string>} paths
+ * @param {{transitive?: boolean, cochange?: boolean}} [options]
+ *   `transitive` (default true) is the import closure; `cochange` adds files that land in
+ *   the same commits, and is only ever on when a caller asked for it.
+ * @returns {string[]}
+ */
+export function coverage(graph, paths, { transitive = true, cochange = false } = {}) {
+  const changed = [...new Set([...paths].filter(Boolean))];
+  const imports = graph.imports ?? [];
+  if (transitive) {
+    const selected = selectTests(changed, imports);
+    if (!cochange || graph.files === undefined) return selected;
+    const edges = testEdges({
+      files: graph.files,
+      imports,
+      coverage: graph.coverage ?? [],
+      commits: graph.commits ?? [],
+    });
+    const found = new Set(selected);
+    for (const path of changed) {
+      for (const test of testsFor(path, edges, { cochange: true })) found.add(test);
+    }
+    return [...found].sort();
+  }
+  const edges = testEdges({
+    files: graph.files ?? [],
+    imports,
+    coverage: graph.coverage ?? [],
+    commits: graph.commits ?? [],
+  });
+  const found = new Set();
+  for (const path of changed) {
+    for (const test of testsFor(path, edges, { cochange })) found.add(test);
+  }
+  return [...found].sort();
+}

package/src/index/extract.mjs

@@ -4,21 +4,14 @@
 // line at a time: an export, a definition, an import. That is enough for a map
 // and an import graph, and it is the same result on every machine.
 
+import { cmp } from '../util/sort.mjs';
+// Q70: the extension table is the records' `languages[].exts` (ADR-013: content is data,
+// the engine is code). A new language lands in a record, not a row the extractor edits.
+import { languageExts } from '../ecosystem/records.mjs';
+
 const IDENT = '[A-Za-z_$][\\w$]*';
 
-const EXTS = {
-  javascript: ['.js', '.mjs', '.cjs', '.jsx', '.ts', '.tsx'],
-  typescript: ['.ts', '.tsx', '.js', '.mjs', '.jsx'],
-  kotlin: ['.kt', '.kts'],
-  java: ['.java'],
-  python: ['.py'],
-  go: ['.go'],
-  rust: ['.rs'],
-  swift: ['.swift'],
-  dart: ['.dart'],
-  ruby: ['.rb'],
-  csharp: ['.cs'],
-};
+const EXTS = languageExts();
 
 /**
  * Symbols and import specifiers in one blob.
@@ -261,9 +254,6 @@ function byEdge(a, b) {
   return cmp(a.src, b.src) || cmp(a.dst, b.dst) || cmp(a.kind, b.kind);
 }
 
-function cmp(a, b) {
-  return a < b ? -1 : a > b ? 1 : 0;
-}
 
 function sigOf(line) {
   return line.trim().slice(0, 200);

package/src/index/gate.mjs

@@ -8,7 +8,7 @@
 import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 
-import { isTestPath } from '../bench/mine.mjs';
+import { isTestPath } from '../testing/paths.mjs';
 import { langOf } from './store.mjs';
 import { importsOf } from './tests.mjs';
 

package/src/index/govern.mjs

@@ -6,7 +6,9 @@
 // and a lesson do the same with their own path list. Nothing here is stored: the
 // answer is the records on disk.
 
-import { parseDecisionFrontmatter, splitFrontmatter } from '../memory/decide.mjs';
+import { parseDecisionFrontmatter } from '../memory/decide.mjs';
+import { parseFrontmatter } from '../text/frontmatter.mjs';
+import { cmp } from '../util/sort.mjs';
 
 const SIBLING = /^\*{0,2}[A-Z][\w-]*(?:[ \t]+[\w-]+){0,3}:\*{0,2}(?:[ \t]|$)/;
 const HEADING = /^#{1,6}\s/;
@@ -18,7 +20,7 @@ const TOOL = /^(?:simt-dev|simt)\b/;
  */
 export function affectsTokens(text) {
   const fm = parseDecisionFrontmatter(text);
-  const { body } = splitFrontmatter(text);
+  const { body } = parseFrontmatter(text);
   const lines = String(body).split('\n');
   const start = lines.findIndex((line) => /^\s*\*{0,2}Affects:\*{0,2}/.test(line));
   const proseTokens = [];
@@ -131,7 +133,27 @@ export function scopeHits(file, { rules = [], skills = [], gates = [], lessons =
   return hits;
 }
 
-export function sortHits(hits) {
+/**
+ * What governs one file: decisions whose Affects line names it, then the rules, skills,
+ * gates and lessons whose path lists match, in one stable order.
+ *
+ * `why`, `impact` and the critic each assembled this themselves, with different corpora
+ * and - in `why`'s case - a hard-coded list of this repository's own rule directories
+ * (Q77). The corpus is now the caller's, so a receiving repository's directories are the
+ * ones read.
+ *
+ * @param {string} file
+ * @param {{records?: {id: string, title: string, text: string}[], ctx: {files: Set<string>,
+ *   commands: Set<string>, roots: string[]}, rules?: object[], skills?: object[],
+ *   gates?: object[], lessons?: object[], commands?: Record<string, string>}} corpus
+ */
+export function governs(file, corpus) {
+  const { records = [], ctx, ...scopes } = corpus;
+  const { hits } = decisionHits(records, file, ctx);
+  return sortHits([...hits, ...scopeHits(file, scopes)]);
+}
+
+function sortHits(hits) {
   const order = { decision: 0, rule: 1, skill: 2, gate: 3, lesson: 4 };
   return [...hits].sort((a, b) => (order[a.kind] - order[b.kind]) || cmp(a.id, b.id));
 }
@@ -218,7 +240,7 @@ function extension(file) {
 }
 
 function frontmatter(text) {
-  return splitFrontmatter(text).front ?? '';
+  return parseFrontmatter(text).front ?? '';
 }
 
 function quotedList(front, key) {
@@ -227,8 +249,3 @@ function quotedList(front, key) {
   return [...match[1].matchAll(/['"]([^'"]+)['"]/g)].map((item) => item[1]);
 }
 
-function cmp(a, b) {
-  if (a < b) return -1;
-  if (a > b) return 1;
-  return 0;
-}

package/src/index/impact.mjs

@@ -5,8 +5,9 @@
 // touches. The walk stops at a fixed count: an unbounded report is not an
 // answer a reader can use.
 
-import { isTestPath } from '../bench/mine.mjs';
-import { selectTests } from './gate.mjs';
+import { isTestPath } from '../testing/paths.mjs';
+import { coverage } from './coverage.mjs';
+import { cmp } from '../util/sort.mjs';
 
 const IMPACT_LIMIT = 64;
 
@@ -50,7 +51,7 @@ export function dependents(file, imports, limit = IMPACT_LIMIT) {
     }
   }
   importers.sort();
-  const tests = selectTests([file], imports).filter((path) => path !== file);
+  const tests = coverage({ imports }, [file]).filter((path) => path !== file);
   const shown = tests.length > limit ? tests.slice(0, limit) : tests;
   if (tests.length > limit) truncated = true;
   return { importers, tests: shown, truncated };
@@ -179,8 +180,3 @@ function glob(pattern, text) {
   return new RegExp(`^${body}$`).test(text);
 }
 
-function cmp(a, b) {
-  if (a < b) return -1;
-  if (a > b) return 1;
-  return 0;
-}

package/src/index/map.mjs

@@ -9,7 +9,8 @@
 import { readdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 
-import { isTestPath, resolveVaultDir } from '../bench/mine.mjs';
+import { isTestPath } from '../testing/paths.mjs';
+import { resolveVaultDir } from '../testing/vault.mjs';
 import { readLocalConfig } from '../local/config.mjs';
 import { localEmbed } from '../local/client.mjs';
 import {
@@ -22,6 +23,8 @@ import {
   scoreFtsBm25,
   searchFtsBm25,
 } from './store.mjs';
+import { cmp } from '../util/sort.mjs';
+import { INDEX_STOP as STOP } from '../text/tokens.mjs';
 
 export const ITERATIONS = 20;
 export const DAMPING = 0.85;
@@ -30,14 +33,6 @@ export const EMBED_RECALL_GAIN_BAR = 0.1;
 const TOP = 10;
 export const DEFAULT_BUDGET = 2048;
 
-const STOP = new Set([
-  'this', 'that', 'with', 'from', 'have', 'been', 'into', 'only', 'when', 'what',
-  'which', 'their', 'about', 'after', 'before', 'where', 'there', 'these', 'those',
-  'would', 'could', 'should', 'every', 'other', 'than', 'then', 'them', 'they',
-  'your', 'will', 'does', 'file', 'files', 'test', 'tests', 'line', 'lines', 'code',
-  'also', 'just', 'more', 'some', 'such', 'each', 'same', 'over', 'used', 'using',
-]);
-
 /**
  * Reciprocal-rank fusion (RRF) over multiple ranked lists.
  *
@@ -405,16 +400,25 @@ function pageRank(nodes, links, scores) {
   let rank = pers.slice();
   for (let iter = 0; iter < ITERATIONS; iter += 1) {
     const next = pers.map((value) => (1 - DAMPING) * value);
+    // Dangling nodes (no outgoing edges) spread their rank over the whole
+    // distribution. Accumulating their mass first makes that one pass over j
+    // instead of one pass per dangling node - the no-task rank of a 30k-file
+    // repo used to go quadratic here (Q76).
+    let danglingMass = 0;
     for (let i = 0; i < n; i += 1) {
       const outs = links[i];
       if (outs.length === 0) {
-        for (let j = 0; j < n; j += 1) next[j] += DAMPING * rank[i] * pers[j];
+        danglingMass += rank[i];
       } else {
         let sum = 0;
         for (const [, weight] of outs) sum += weight;
         for (const [j, weight] of outs) next[j] += (DAMPING * rank[i] * weight) / sum;
       }
     }
+    if (danglingMass > 0) {
+      const share = DAMPING * danglingMass;
+      for (let j = 0; j < n; j += 1) next[j] += share * pers[j];
+    }
     rank = next;
   }
   return rank;
@@ -620,8 +624,3 @@ function round6(value) {
   return Math.round(value * 1e6) / 1e6;
 }
 
-function cmp(a, b) {
-  if (a < b) return -1;
-  if (a > b) return 1;
-  return 0;
-}

package/src/index/open.mjs

@@ -0,0 +1,173 @@
+// The index as a handle (review 0003 Q76).
+//
+// `openIndex` returns a handle whose methods answer with pure data — rows and
+// counts, no exit codes, no formatted lines — and with a structured error,
+// `{ ok: false, error: { code, message } }`, when they cannot. The methods are
+// thin over the same primitives the commands use, so a handle and a command
+// cannot disagree about the same index.
+//
+// The graph loads once at open: files, symbols, edges. FTS bodies run in
+// SQLite (`search` delegates to `searchFtsBm25`) and stored vectors are not
+// loaded at all: a map without `--embed` does not touch them (Q76).
+
+import { coverage } from './coverage.mjs';
+import { governs } from './govern.mjs';
+import { dependents } from './impact.mjs';
+import { rankFiles } from './map.mjs';
+import { isIndexStale, readEdges, readFiles, readSymbols, reindex, searchFtsBm25 } from './store.mjs';
+
+const INDEX_CODES = {
+  BUILD: 'E_BUILD',
+  CLOSED: 'E_CLOSED',
+  NO_INDEX: 'E_NO_INDEX',
+  READ: 'E_READ',
+};
+
+function ok(data) {
+  return { ok: true, data };
+}
+
+function fail(code, message) {
+  return { ok: false, error: { code, message } };
+}
+
+/**
+ * The files, symbols and edges the index holds. The lazy projection of
+ * `loadGraph`: no FTS bodies, no vectors, because nothing on this handle
+ * scores them.
+ */
+async function loadGraphData(root) {
+  const [files, symbols, edges] = await Promise.all([
+    readFiles(root),
+    readSymbols(root),
+    readEdges(root),
+  ]);
+  return { files, symbols, edges };
+}
+
+/**
+ * Open the repository's index.
+ *
+ * @param {string} root
+ * @param {{build?: boolean, refresh?: boolean}} [options]
+ *   `build` (default true) refreshes the index from the git index before the
+ *   first read — incrementally, so a current repository pays for one
+ *   `git ls-files -s`, not a walk of the tree. `refresh` re-reads the graph on
+ *   every call instead of caching it at open.
+ * @returns {Promise<IndexHandle | {ok: false, error: {code: string, message: string}}>}
+ */
+export async function openIndex(root, { build = true, refresh = false } = {}) {
+  if (build) {
+    try {
+      await reindex(root);
+    } catch (error) {
+      return fail(INDEX_CODES.BUILD, `could not build the index (${error.message})`);
+    }
+  }
+
+  let closed = false;
+  let cached = null;
+
+  async function graph() {
+    if (closed) throw new Error(INDEX_CODES.CLOSED);
+    if (cached === null || refresh) cached = await loadGraphData(root);
+    return cached;
+  }
+
+  async function answered(fn) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (error.message === INDEX_CODES.CLOSED) return fail(INDEX_CODES.CLOSED, 'the index is closed');
+      return fail(INDEX_CODES.READ, error.message);
+    }
+  }
+
+  const handle = {
+    ok: true,
+
+    /** Freshness and row counts of the open index. */
+    status: () =>
+      answered(async () => {
+        const g = await graph();
+        if (g.files.length === 0) return fail(INDEX_CODES.NO_INDEX, 'the index is empty');
+        const stale = await isIndexStale(root);
+        return ok({ stale, counts: { files: g.files.length, symbols: g.symbols.length, edges: g.edges.length } });
+      }),
+
+    /**
+     * BM25 rows for a task string, best first.
+     * @param {string} task
+     * @param {{limit?: number}} [options]
+     * @returns {Promise<{ok: true, data: {path: string, kind: string, score: number}[]}|{ok: false, error: {code: string, message: string}}>}
+     */
+    search: (task, { limit } = {}) =>
+      answered(async () => {
+        if (closed) throw new Error(INDEX_CODES.CLOSED);
+        const rows = await searchFtsBm25(root, task);
+        if (rows.error) return fail(INDEX_CODES.READ, rows.error);
+        return ok(limit === undefined ? rows : rows.slice(0, limit));
+      }),
+
+    /**
+     * Rank the repository's files for a task string (R22): graph proximity
+     * fused with the BM25 rows.
+     * @param {string} task
+     * @param {{limit?: number, embed?: boolean, allowRemote?: boolean}} [options]
+     * @returns {Promise<{ok: true, data: {path: string, rank: number, score: number}[]}|{ok: false, error: {code: string, message: string}}>}
+     */
+    rank: (task, { limit, embed = false, allowRemote = false } = {}) =>
+      answered(async () => {
+        const g = await graph();
+        if (g.files.length === 0) return fail(INDEX_CODES.NO_INDEX, 'the index is empty');
+        const bm25 = String(task ?? '').trim().length > 0 ? await searchFtsBm25(root, task) : [];
+        if (bm25.error) return fail(INDEX_CODES.READ, bm25.error);
+        const ranked = rankFiles({ ...g, bm25, task });
+        return ok(limit === undefined ? ranked : ranked.slice(0, limit));
+      }),
+
+    /**
+     * The tests that exercise `paths` (Q77): the import closure when
+     * `transitive` (default true), the stored test edges otherwise.
+     * @param {Iterable<string>} paths
+     * @param {{transitive?: boolean, cochange?: boolean}} [options]
+     * @returns {Promise<{ok: true, data: string[]}|{ok: false, error: {code: string, message: string}}>}
+     */
+    coverage: (paths, options) =>
+      answered(async () => {
+        const g = await graph();
+        return ok(coverage({ files: g.files, imports: g.edges }, paths, options));
+      }),
+
+    /**
+     * Who a change to `file` reaches (Q77): the reverse import closure and
+     * the tests that import it.
+     * @param {string} file
+     * @param {{limit?: number}} [options]
+     * @returns {Promise<{ok: true, data: {importers: string[], tests: string[], truncated: boolean}}|{ok: false, error: {code: string, message: string}}>}
+     */
+    impact: (file, { limit } = {}) =>
+      answered(async () => {
+        const g = await graph();
+        return ok(dependents(file, g.edges, limit));
+      }),
+
+    /**
+     * What governs `file` (Q77): the records in `corpus` whose Affects line
+     * names it, plus the rules, skills, gates and lessons whose path lists
+     * match.
+     * @param {string} file
+     * @param {{records?: {id: string, title: string, text: string}[], ctx?: {files: Set<string>, commands: Set<string>, roots: string[]}}} [corpus]
+     * @returns {Promise<{ok: true, data: {kind: string, id: string, detail?: string}[]}|{ok: false, error: {code: string, message: string}}>}
+     */
+    governs: (file, corpus) => answered(() => ok(governs(file, corpus ?? {}))),
+
+    /** Release the handle. Calls after it answer `E_CLOSED`. */
+    close() {
+      closed = true;
+      cached = null;
+    },
+  };
+
+  return handle;
+}