scrumrun 2.1.0 → 2.1.1

package/lib/memory/index.js

@@ -12,8 +12,10 @@ const { containsSecret } = require("../security/secrets");
 const { canonicalWatchSnapshot, fileWatchSnapshot } = require("../runtime/canonical-snapshot");
 
 const CACHE_RELATIVE = path.join(".cache", "semantic-index.sqlite");
-const INDEX_SCHEMA_VERSION = 3;
+const INDEX_SCHEMA_VERSION = 4;
 const INACTIVE = new Set(["rejected", "invalidated", "deprecated", "archived"]);
+const SEARCH_BACKENDS = new Set(["auto", "fts5", "like"]);
+const MAX_SEARCH_TOKENS = 32;
 
 function indexPath(projectRoot) {
   return path.join(projectRoot, ".scrumrun", CACHE_RELATIVE);
@@ -23,6 +25,10 @@ function titleFromBody(body, fallback) {
   return ((String(body || "").match(/^#\s+([^\r\n]+)/m) || [])[1] || fallback).trim();
 }
 
+function normalizeSearchText(...values) {
+  return values.map((value) => String(value || "")).join(" ").normalize("NFKC").toLowerCase();
+}
+
 function canonicalArtifacts(scrumDir) {
   const repository = new ArtifactRepository(scrumDir);
   const artifacts = [];
@@ -104,7 +110,8 @@ function explicitEdges(artifact) {
   return [...unique.values()];
 }
 
-function createSchema(database) {
+function createSchema(database, { searchBackend = "auto" } = {}) {
+  if (!SEARCH_BACKENDS.has(searchBackend)) throw new Error(`Unknown semantic search backend: ${searchBackend}`);
   database.exec(`
     PRAGMA journal_mode = DELETE;
     PRAGMA synchronous = FULL;
@@ -121,6 +128,7 @@ function createSchema(database) {
       valid_until TEXT,
       last_verified_commit TEXT,
       review_trigger TEXT,
+      search_text TEXT NOT NULL,
       active INTEGER NOT NULL CHECK (active IN (0, 1))
     );
     CREATE TABLE edges (
@@ -143,7 +151,8 @@ function createSchema(database) {
       status TEXT NOT NULL CHECK (status IN ('current', 'orphaned')),
       language TEXT NOT NULL,
       exported INTEGER NOT NULL CHECK (exported IN (0, 1)),
-      remapped_from TEXT
+      remapped_from TEXT,
+      search_text TEXT NOT NULL
     );
     CREATE TABLE evidence_snapshots (
       artifact_id TEXT NOT NULL,
@@ -167,9 +176,18 @@ function createSchema(database) {
     CREATE INDEX edges_from ON edges(from_id);
     CREATE INDEX edges_to ON edges(to_id);
     CREATE INDEX artifacts_status ON artifacts(status);
-    CREATE VIRTUAL TABLE artifacts_fts USING fts5(id UNINDEXED, title, content, tokenize='unicode61');
-    CREATE VIRTUAL TABLE code_fts USING fts5(id UNINDEXED, name, qualified_name, path, tokenize='unicode61');
   `);
+  if (searchBackend === "like") return "like";
+  try {
+    database.exec(`
+      CREATE VIRTUAL TABLE artifacts_fts USING fts5(id UNINDEXED, title, content, tokenize='unicode61');
+      CREATE VIRTUAL TABLE code_fts USING fts5(id UNINDEXED, name, qualified_name, path, tokenize='unicode61');
+    `);
+    return "fts5";
+  } catch (error) {
+    if (searchBackend === "fts5" || !/no such module:\s*fts5/i.test(error.message)) throw error;
+    return "like";
+  }
 }
 
 function tableExists(database, name) {
@@ -342,7 +360,7 @@ function derivedInvalidations(projectRoot, artifacts, codeNodes, prior, evidence
   return { invalidations: [...invalidations.values()], subjects };
 }
 
-function rebuildIndex(projectRoot) {
+function rebuildIndex(projectRoot, { searchBackend = "auto" } = {}) {
   const scrumDir = path.join(projectRoot, ".scrumrun");
   if (!fs.existsSync(scrumDir)) throw new Error("Not a ScrumRun project: .scrumrun/ is missing.");
   const cacheDir = path.join(scrumDir, ".cache");
@@ -358,14 +376,19 @@ function rebuildIndex(projectRoot) {
   const evidence = evidenceSnapshots(projectRoot, artifacts);
   const derived = derivedInvalidations(projectRoot, artifacts, remapped.nodes, prior, evidence);
   let database;
+  let selectedSearchBackend;
   try {
     database = new DatabaseSync(temp);
-    createSchema(database);
-    const insertArtifact = database.prepare("INSERT INTO artifacts(id, kind, status, title, path, hash, content, valid_from, valid_until, last_verified_commit, review_trigger, active) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
-    const insertFts = database.prepare("INSERT INTO artifacts_fts(id, title, content) VALUES (?, ?, ?)");
+    selectedSearchBackend = createSchema(database, { searchBackend });
+    const insertArtifact = database.prepare("INSERT INTO artifacts(id, kind, status, title, path, hash, content, valid_from, valid_until, last_verified_commit, review_trigger, search_text, active) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+    const insertFts = selectedSearchBackend === "fts5"
+      ? database.prepare("INSERT INTO artifacts_fts(id, title, content) VALUES (?, ?, ?)")
+      : null;
     const insertEdge = database.prepare("INSERT OR IGNORE INTO edges(from_id, relation, to_id, evidence, confidence) VALUES (?, ?, ?, ?, ?)");
-    const insertCode = database.prepare("INSERT INTO code_nodes(id, kind, name, qualified_name, path, line, fingerprint, commit_hash, status, language, exported, remapped_from) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
-    const insertCodeFts = database.prepare("INSERT INTO code_fts(id, name, qualified_name, path) VALUES (?, ?, ?, ?)");
+    const insertCode = database.prepare("INSERT INTO code_nodes(id, kind, name, qualified_name, path, line, fingerprint, commit_hash, status, language, exported, remapped_from, search_text) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+    const insertCodeFts = selectedSearchBackend === "fts5"
+      ? database.prepare("INSERT INTO code_fts(id, name, qualified_name, path) VALUES (?, ?, ?, ?)")
+      : null;
     database.exec("BEGIN IMMEDIATE");
     try {
       for (const artifact of artifacts) {
@@ -381,14 +404,29 @@ function rebuildIndex(projectRoot) {
           artifact.record.valid_until || null,
           artifact.record.last_verified_commit || null,
           artifact.record.review_trigger || null,
+          normalizeSearchText(artifact.title, artifact.content),
           INACTIVE.has(artifact.record.status) ? 0 : 1
         );
-        insertFts.run(artifact.record.id, artifact.title, artifact.content);
+        if (insertFts) insertFts.run(artifact.record.id, artifact.title, artifact.content);
         for (const edge of explicitEdges(artifact)) insertEdge.run(artifact.record.id, edge.relation, edge.target, edge.evidence, "canonical");
       }
       for (const node of remapped.nodes) {
-        insertCode.run(node.id, node.kind, node.name, node.qualifiedName, node.path, node.line, node.fingerprint, node.commit || null, node.status, node.language, node.exported ? 1 : 0, node.remappedFrom || null);
-        insertCodeFts.run(node.id, node.name, node.qualifiedName, node.path);
+        insertCode.run(
+          node.id,
+          node.kind,
+          node.name,
+          node.qualifiedName,
+          node.path,
+          node.line,
+          node.fingerprint,
+          node.commit || null,
+          node.status,
+          node.language,
+          node.exported ? 1 : 0,
+          node.remappedFrom || null,
+          normalizeSearchText(node.name, node.qualifiedName, node.path)
+        );
+        if (insertCodeFts) insertCodeFts.run(node.id, node.name, node.qualifiedName, node.path);
       }
       for (const edge of snapshot.code.edges) insertEdge.run(edge.from, edge.relation, edge.to, edge.evidence, edge.confidence);
       for (const edge of prior.edges.filter((item) => remapped.orphanIds.has(item.from_id))) {
@@ -403,6 +441,7 @@ function rebuildIndex(projectRoot) {
       const metadata = database.prepare("INSERT INTO metadata(key, value) VALUES (?, ?)");
       metadata.run("schema_version", String(INDEX_SCHEMA_VERSION));
       metadata.run("method_version", "2.0.0");
+      metadata.run("search_backend", selectedSearchBackend);
       metadata.run("source_fingerprint", snapshot.fingerprint);
       metadata.run("source_watch_fingerprint", snapshot.watch.fingerprint);
       metadata.run("artifact_count", String(artifacts.length));
@@ -422,7 +461,24 @@ function rebuildIndex(projectRoot) {
     if (fs.existsSync(temp)) fs.rmSync(temp, { force: true });
     throw error;
   }
-  return { file: target, fingerprint: snapshot.fingerprint, watchFingerprint: snapshot.watch.fingerprint, artifacts: artifacts.length, codeNodes: remapped.nodes.length, warnings: [...snapshot.code.warnings, ...snapshot.watch.warnings] };
+  return {
+    file: target,
+    fingerprint: snapshot.fingerprint,
+    watchFingerprint: snapshot.watch.fingerprint,
+    searchBackend: selectedSearchBackend,
+    artifacts: artifacts.length,
+    codeNodes: remapped.nodes.length,
+    warnings: [...snapshot.code.warnings, ...snapshot.watch.warnings]
+  };
+}
+
+function runtimeSupportsFts5(database) {
+  try {
+    database.prepare("SELECT 1 FROM artifacts_fts LIMIT 1").get();
+    return true;
+  } catch {
+    return false;
+  }
 }
 
 function indexStatus(projectRoot) {
@@ -434,12 +490,20 @@ function indexStatus(projectRoot) {
     database = new DatabaseSync(file, { readOnly: true });
     const metadata = new Map(database.prepare("SELECT key, value FROM metadata").all().map((row) => [row.key, row.value]));
     const stored = metadata.get("source_fingerprint") || null;
-    if (metadata.get("schema_version") !== String(INDEX_SCHEMA_VERSION) || !metadata.get("source_watch_fingerprint")) {
+    const searchBackend = metadata.get("search_backend") || null;
+    if (
+      metadata.get("schema_version") !== String(INDEX_SCHEMA_VERSION)
+      || !metadata.get("source_watch_fingerprint")
+      || !["fts5", "like"].includes(searchBackend)
+    ) {
       return { exists: true, stale: true, file, stored, check: "schema", reason: `semantic index schema ${INDEX_SCHEMA_VERSION} rebuild required` };
     }
+    if (searchBackend === "fts5" && !runtimeSupportsFts5(database)) {
+      return { exists: true, stale: true, file, stored, searchBackend, check: "schema", reason: "semantic index requires unavailable FTS5; fallback rebuild required" };
+    }
     const watch = sourceWatchSnapshot(projectRoot);
     if (metadata.get("source_watch_fingerprint") === watch.fingerprint) {
-      return { exists: true, stale: false, file, stored, actual: stored, watch: watch.fingerprint, check: "metadata" };
+      return { exists: true, stale: false, file, stored, actual: stored, watch: watch.fingerprint, searchBackend, check: "metadata" };
     }
     const actual = sourceSnapshot(projectRoot, { watch }).fingerprint;
     return {
@@ -449,6 +513,7 @@ function indexStatus(projectRoot) {
       stored,
       actual,
       watch: watch.fingerprint,
+      searchBackend,
       check: "hash",
       reason: stored === actual ? "file metadata changed but semantic content is unchanged" : "canonical or source content changed"
     };
@@ -459,11 +524,40 @@ function indexStatus(projectRoot) {
   }
 }
 
+function searchTokens(query) {
+  return (String(query || "").normalize("NFKC").match(/[\p{L}\p{N}_-]+/gu) || []).slice(0, MAX_SEARCH_TOKENS);
+}
+
 function ftsQuery(query) {
-  const tokens = String(query || "").normalize("NFKC").match(/[\p{L}\p{N}_-]+/gu) || [];
+  const tokens = searchTokens(query);
   return tokens.map((token) => `"${token.replace(/"/g, "")}"`).join(" AND ");
 }
 
+function fallbackArtifactRows(database, tokens, activeClause, cap) {
+  if (!tokens.length) return [];
+  const clauses = tokens.map(() => "instr(a.search_text, ?) > 0").join(" AND ");
+  const parameters = tokens.map((token) => normalizeSearchText(token));
+  return database.prepare(`
+    SELECT a.id, a.kind, a.status, a.title, a.path, a.active, a.valid_until, a.review_trigger
+    FROM artifacts a
+    WHERE ${clauses} ${activeClause}
+    ORDER BY a.id LIMIT ?
+  `).all(...parameters, cap);
+}
+
+function fallbackCodeRows(database, tokens, cap) {
+  if (!tokens.length) return [];
+  const clauses = tokens.map(() => "instr(c.search_text, ?) > 0").join(" AND ");
+  const parameters = tokens.map((token) => normalizeSearchText(token));
+  return database.prepare(`
+    SELECT c.id, c.kind, c.name AS title, c.qualified_name, c.path, c.line, c.fingerprint,
+      c.commit_hash, c.status, c.language, c.remapped_from
+    FROM code_nodes c
+    WHERE ${clauses}
+    ORDER BY CASE c.status WHEN 'current' THEN 0 ELSE 1 END, c.id LIMIT ?
+  `).all(...parameters, cap);
+}
+
 function queryIndex(projectRoot, query, { includeInactive = false, limit = DEFAULT_CONTEXT_RESULTS, relationLimit = DEFAULT_RELATIONS_PER_RESULT, rebuild = true } = {}) {
   const text = String(query || "").trim();
   if (!text) throw new Error("A non-empty memory query is required.");
@@ -476,18 +570,23 @@ function queryIndex(projectRoot, query, { includeInactive = false, limit = DEFAU
   }
   const database = new DatabaseSync(indexPath(projectRoot), { readOnly: true });
   try {
+    const metadata = new Map(database.prepare("SELECT key, value FROM metadata").all().map((row) => [row.key, row.value]));
+    const searchBackend = metadata.get("search_backend");
     const cap = Math.max(1, Math.min(Number(limit) || DEFAULT_CONTEXT_RESULTS, MAX_CONTEXT_RESULTS));
     const relationCap = Math.max(1, Math.min(Number(relationLimit) || DEFAULT_RELATIONS_PER_RESULT, MAX_RELATIONS_PER_RESULT));
     const activeClause = includeInactive ? "" : "AND a.active = 1";
     const matches = new Map();
+    const tokens = searchTokens(text);
     const match = ftsQuery(text);
-    if (match) {
-      const rows = database.prepare(`
-        SELECT a.id, a.kind, a.status, a.title, a.path, a.active, a.valid_until, a.review_trigger, bm25(artifacts_fts) AS rank
-        FROM artifacts_fts JOIN artifacts a ON a.id = artifacts_fts.id
-        WHERE artifacts_fts MATCH ? ${activeClause}
-        ORDER BY rank, a.id LIMIT ?
-      `).all(match, cap);
+    if (tokens.length) {
+      const rows = searchBackend === "fts5"
+        ? database.prepare(`
+          SELECT a.id, a.kind, a.status, a.title, a.path, a.active, a.valid_until, a.review_trigger, bm25(artifacts_fts) AS rank
+          FROM artifacts_fts JOIN artifacts a ON a.id = artifacts_fts.id
+          WHERE artifacts_fts MATCH ? ${activeClause}
+          ORDER BY rank, a.id LIMIT ?
+        `).all(match, cap)
+        : fallbackArtifactRows(database, tokens, activeClause, cap);
       for (const row of rows) matches.set(`artifact:${row.id}`, { ...row, nodeType: "artifact", match: "content" });
     }
     const like = `%${text.toLowerCase()}%`;
@@ -504,14 +603,16 @@ function queryIndex(projectRoot, query, { includeInactive = false, limit = DEFAU
       if (existing) existing.match = "content+relation";
       else matches.set(key, { ...row, nodeType: "artifact", match: "relation" });
     }
-    if (match) {
-      const codeRows = database.prepare(`
-        SELECT c.id, c.kind, c.name AS title, c.qualified_name, c.path, c.line, c.fingerprint,
-          c.commit_hash, c.status, c.language, c.remapped_from, bm25(code_fts) AS rank
-        FROM code_fts JOIN code_nodes c ON c.id = code_fts.id
-        WHERE code_fts MATCH ?
-        ORDER BY CASE c.status WHEN 'current' THEN 0 ELSE 1 END, rank, c.id LIMIT ?
-      `).all(match, cap);
+    if (tokens.length) {
+      const codeRows = searchBackend === "fts5"
+        ? database.prepare(`
+          SELECT c.id, c.kind, c.name AS title, c.qualified_name, c.path, c.line, c.fingerprint,
+            c.commit_hash, c.status, c.language, c.remapped_from, bm25(code_fts) AS rank
+          FROM code_fts JOIN code_nodes c ON c.id = code_fts.id
+          WHERE code_fts MATCH ?
+          ORDER BY CASE c.status WHEN 'current' THEN 0 ELSE 1 END, rank, c.id LIMIT ?
+        `).all(match, cap)
+        : fallbackCodeRows(database, tokens, cap);
       for (const row of codeRows) matches.set(`code:${row.id}`, { ...row, nodeType: "code", match: "code" });
     }
     const codeRelationRows = database.prepare(`
@@ -575,7 +676,7 @@ function queryIndex(projectRoot, query, { includeInactive = false, limit = DEFAU
         relations
       };
     });
-    return { indexFile: indexPath(projectRoot), rebuilt, query: text, results };
+    return { indexFile: indexPath(projectRoot), rebuilt, searchBackend, query: text, results };
   } finally {
     database.close();
   }

package/lib/memory/service.js

@@ -15,6 +15,7 @@ const {
 } = require("../v2/artifacts");
 const { extractEvidence, renderBullets } = require("./markdown");
 const { assertNoSecret } = require("../security/secrets");
+const { assertCanonicalWrite } = require("../runtime/mutation-gateway");
 
 const MEMORY_KINDS = new Set(["knowledge", "decision", "insight", "dossier"]);
 const DEFAULT_STATUS = Object.freeze({ knowledge: "candidate", decision: "open", insight: "candidate", dossier: "active" });
@@ -203,6 +204,7 @@ function createMemoryUnlocked(projectRoot, kind, options = {}) {
     "",
     `- ${created}: ${record.status} — proposed by ${sourceType}.`
   ].join("\n");
+  assertCanonicalWrite(projectRoot, `create-${kind}`, [title, content, body]);
   repository.write(record, body);
   return repository.read(kind, record.id);
 }
@@ -267,6 +269,7 @@ function transitionMemoryUnlocked(projectRoot, kind, id, action, options = {}) {
   const parsed = parseArtifact(next);
   const errors = [...parsed.errors, ...validateArtifact(parsed.record, kind)];
   if (errors.length) throw new Error(`Memory transition validation failed: ${errors.join("; ")}`);
+  assertCanonicalWrite(projectRoot, `transition-${kind}`, [next]);
   atomicWrite(artifact.file, next);
   return repository.read(kind, id);
 }