sweet-search 2.5.2 → 2.5.4

package/core/search/search-read-semantic.js

@@ -30,10 +30,13 @@
  */
 
 import path from 'node:path';
+import fs from 'node:fs';
 import { CodebaseRepository } from '../infrastructure/codebase-repository.js';
-import { DB_PATHS, LATE_INTERACTION_CONFIG } from '../infrastructure/config/index.js';
+import { DB_PATHS, LATE_INTERACTION_CONFIG, PROJECT_ROOT } from '../infrastructure/config/index.js';
 import { applyPersistedLiModel } from '../infrastructure/init-config.js';
 import { readFile as readFileExact } from './search-read.js';
+import { withPinnedRead } from './search-reader-pin.js';
+import { emitToolIdentityAuto } from './cli-decoration.js';
 
 // Applies the user's persisted LI model exactly once per (projectRoot, env)
 // pair so encodeQuery/_getLateInteractionIndex below see the right variant.
@@ -62,6 +65,21 @@ const DEFAULTS = {
   lexicalWeight: 1.0,
   symbolWeight: 1.5,         // symbol-name hits are stronger evidence per-file
   maxsimWeight: 1.6,         // late interaction wins ties
+  // Demotion factors applied to the final re-rank score (after MaxSim re-rank).
+  // Stage 3 diagnosis (2026-05-13, PHASE6_REDO ss-semantic) found:
+  //  - chunks with null/unknown symbol metadata frequently win top-1 when
+  //    they're really file-header fragments or unnamed code blocks
+  //    (CPP-002, RB-001, C-005, PY-004 dev failures)
+  //  - tiny chunks (≤ 5 lines) inflate MaxSim by concentrating literal
+  //    token presence in a small window (RB-001 `module Sinatra`,
+  //    C-005 single-line `redisContext *redisConnectWithOptions(...)`).
+  // Multiplicative demotion at the final-rank stage is conservative: the
+  // chunk is still returned, just less likely to be top-1. Tunable; 0.85
+  // was chosen by inspecting per-failure score margins (typical wrong-vs-
+  // gold gap is 0.01-0.04, so 0.85 reliably flips the cases identified).
+  unsymboledDemote: 0.85,
+  smallChunkDemote: 0.85,
+  smallChunkMaxLines: 5,
 };
 
 const APPROX_CHARS_PER_TOKEN = 4;
@@ -70,36 +88,155 @@ const APPROX_CHARS_PER_TOKEN = 4;
 // Module-level lazy singletons
 // ---------------------------------------------------------------------------
 
-let _repo = null;
-function _getRepo() {
-  if (_repo === null) {
-    try { _repo = new CodebaseRepository(DB_PATHS.codebase); }
-    catch { _repo = false; }
+const RECONCILE_MANIFEST_FILENAME = 'reconcile-manifest.json';
+
+function _projectKey(projectRoot) {
+  return path.resolve(projectRoot || PROJECT_ROOT || process.cwd());
+}
+
+function _dataDirName() {
+  const dir = path.basename(path.dirname(DB_PATHS.codebase || ''));
+  return dir && dir !== '.' ? dir : '.sweet-search';
+}
+
+function _stateDirForProject(projectRoot) {
+  const root = _projectKey(projectRoot);
+  if (root === path.resolve(PROJECT_ROOT)) return path.dirname(DB_PATHS.codebase);
+  return path.join(root, _dataDirName());
+}
+
+function _codebasePathForProject(projectRoot, manifest = null) {
+  const descriptor = manifest?.vectors?.path || manifest?.vectors?.dbPath;
+  if (descriptor) {
+    return _resolveStatePath(projectRoot, descriptor);
+  }
+  return _defaultCodebasePathForProject(projectRoot);
+}
+
+function _defaultCodebasePathForProject(projectRoot) {
+  const root = _projectKey(projectRoot);
+  if (root === path.resolve(PROJECT_ROOT)) return DB_PATHS.codebase;
+  return path.join(_stateDirForProject(root), 'codebase.db');
+}
+
+function _readReconcileManifest(projectRoot) {
+  try {
+    const manifest = JSON.parse(
+      fs.readFileSync(path.join(_stateDirForProject(projectRoot), RECONCILE_MANIFEST_FILENAME), 'utf-8'),
+    );
+    return Number.isInteger(manifest?.epoch) ? manifest : null;
+  } catch {
+    return null;
+  }
+}
+
+function _resolveStatePath(projectRoot, filePath) {
+  if (!filePath) return null;
+  if (path.isAbsolute(filePath)) return filePath;
+  return path.join(_stateDirForProject(projectRoot), filePath);
+}
+
+function _lateInteractionIndexPath(projectRoot, manifest) {
+  const descriptor = manifest?.lateInteraction?.path
+    || manifest?.lateInteraction?.indexPath
+    || manifest?.lateInteraction?.manifest;
+  if (descriptor) {
+    const resolved = _resolveStatePath(projectRoot, descriptor);
+    const segmentDir = path.dirname(resolved);
+    return segmentDir.endsWith('.segments')
+      ? segmentDir.slice(0, -'.segments'.length)
+      : resolved;
+  }
+  const root = _projectKey(projectRoot);
+  if (root === path.resolve(PROJECT_ROOT)) return DB_PATHS.lateInteraction;
+  if (DB_PATHS.lateInteraction && fs.existsSync(DB_PATHS.lateInteraction)) {
+    return DB_PATHS.lateInteraction;
   }
-  return _repo || null;
+  return path.join(_stateDirForProject(root), path.basename(DB_PATHS.lateInteraction));
+}
+
+function _sourceStaleness(projectRoot, filePathRel) {
+  const manifest = _readReconcileManifest(projectRoot);
+  const publishedMs = Date.parse(manifest?.publishedAt || '');
+  if (!Number.isFinite(publishedMs)) return null;
+  try {
+    const abs = path.isAbsolute(filePathRel)
+      ? filePathRel
+      : path.resolve(projectRoot, filePathRel);
+    const stat = fs.statSync(abs);
+    if (stat.mtimeMs <= publishedMs) return null;
+    return {
+      stale: true,
+      indexEpoch: manifest.epoch,
+      indexPublishedAt: manifest.publishedAt,
+      sourceMtime: stat.mtime.toISOString(),
+      warning: 'source file is newer than the semantic index; spans were selected from stale index metadata and text was reread from disk',
+    };
+  } catch {
+    return null;
+  }
+}
+
+const _repos = new Map();
+function _getRepo(projectRoot) {
+  const key = _projectKey(projectRoot);
+  const manifest = _readReconcileManifest(projectRoot);
+  const dbPath = _codebasePathForProject(projectRoot, manifest);
+  const baseDbPath = _defaultCodebasePathForProject(projectRoot);
+  let entry = _repos.get(key);
+  if (!entry || entry.dbPath !== dbPath || entry.baseDbPath !== baseDbPath) {
+    entry?.repo?.close?.();
+    try {
+      entry = { dbPath, baseDbPath, repo: new CodebaseRepository(baseDbPath) };
+      _repos.set(key, entry);
+    } catch {
+      return null;
+    }
+  }
+  const repo = entry.repo;
+  repo.refreshManifestEpoch?.();
+  return repo;
 }
 
 let _liIndex = null;
 let _liInitPromise = null;
-async function _getLateInteractionIndex() {
-  if (_liIndex) return _liIndex;
+let _liProjectKey = null;
+let _liManifestEpoch = null;
+async function _getLateInteractionIndex(projectRoot) {
+  const projectKey = _projectKey(projectRoot);
+  const manifest = _readReconcileManifest(projectRoot);
+  const manifestEpoch = Number.isInteger(manifest?.epoch) ? manifest.epoch : null;
+  const samePin = _liProjectKey === projectKey && _liManifestEpoch === manifestEpoch;
+  if (_liIndex !== null && samePin) return _liIndex || null;
+  if (_liIndex !== null && !samePin) {
+    _liIndex = null;
+    _liInitPromise = null;
+  }
   if (_liInitPromise) return _liInitPromise;
   if (!LATE_INTERACTION_CONFIG?.enabled) return null;
   _liInitPromise = (async () => {
     try {
       const { LateInteractionIndex } = await import('../ranking/late-interaction-index.js');
-      const idx = new LateInteractionIndex({});
+      const idx = new LateInteractionIndex({
+        indexPath: _lateInteractionIndexPath(projectRoot, manifest),
+      });
       await idx.init();
       // If the index is empty (no segments, no docs), treat as unavailable —
       // saves a noisy warning later when scoreWithLateInteraction runs.
       if (!idx.documents || idx.documents.size === 0) {
         _liIndex = false;
+        _liProjectKey = projectKey;
+        _liManifestEpoch = manifestEpoch;
         return null;
       }
       _liIndex = idx;
+      _liProjectKey = projectKey;
+      _liManifestEpoch = manifestEpoch;
       return idx;
     } catch {
       _liIndex = false;
+      _liProjectKey = projectKey;
+      _liManifestEpoch = manifestEpoch;
       return null;
     } finally {
       _liInitPromise = null;
@@ -130,7 +267,26 @@ function _projectRelative(absOrRelPath, projectRoot) {
     ? absOrRelPath
     : path.resolve(root, absOrRelPath);
   const rel = path.relative(root, abs);
-  return rel.startsWith('..') || path.isAbsolute(rel) ? abs : rel;
+  const normalized = _normalizeRelativePath(rel);
+  if (normalized) return normalized;
+  try {
+    const realRel = path.relative(
+      fs.realpathSync.native(root),
+      fs.realpathSync.native(abs),
+    );
+    return _normalizeRelativePath(realRel) || abs;
+  } catch {
+    return abs;
+  }
+}
+
+function _normalizeRelativePath(rel) {
+  const normalized = rel.replace(/\\/g, '/').replace(/^\.\//, '');
+  if (!normalized || normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) {
+    return null;
+  }
+  if (path.isAbsolute(normalized)) return null;
+  return normalized;
 }
 
 function _parseMeta(rawMeta) {
@@ -176,7 +332,7 @@ function _escapeRegex(s) {
 // ---------------------------------------------------------------------------
 
 async function _loadFileChunks(filePathRel, projectRoot) {
-  const repo = _getRepo();
+  const repo = _getRepo(projectRoot);
   if (!repo) return { chunks: [], language: null };
   const rows = repo.getChunksByFilePath(filePathRel);
   if (rows.length === 0) return { chunks: [], language: null };
@@ -268,24 +424,38 @@ function _scoreSymbol(chunks, queryTerms, queryRaw) {
     const sym = (c.symbol || '').toLowerCase();
     if (!sym) continue;
     let s = 0;
-    if (sym && lowerRaw.includes(sym)) s += 2;            // raw query mentions the symbol
+    // Word-boundary match prevents short symbols from collecting +2 just for
+    // being a substring of an unrelated longer token in the query. Stage 3
+    // PHASE6_REDO diagnosis (2026-05-13) found two ss-semantic dev FAILs
+    // (JV-004 `Show getType` → `get` chunk got +2 from "get" ⊂ "gettype";
+    // LU-001 `trace _class metatable` → `class` chunk got +2 from "class"
+    // ⊂ "_class") where this substring-rule over-credited the wrong chunk.
+    // The word-boundary form still credits genuine mentions (e.g., "query"
+    // as a real query token still matches `query`-symbol chunks — ZG-001
+    // ambiguity is preserved). Structural rule, no per-language signal,
+    // no stopword growth.
+    const reBoundary = new RegExp(`(?:^|[^a-zA-Z0-9_])${_escapeRegex(sym)}(?=[^a-zA-Z0-9_]|$)`, 'i');
+    if (sym && reBoundary.test(lowerRaw)) s += 2;         // query mentions symbol as a word
     for (const t of queryTerms) {
       if (sym === t) s += 3;                              // exact name match
-      else if (sym.includes(t)) s += 1;                   // substring
+      else if (sym.includes(t)) s += 1;                   // substring (chunk symbol contains query token)
     }
     if (s > 0) scores.set(c.id, s);
   }
   return scores;
 }
 
-async function _scoreLateInteraction(chunks, query) {
+async function _scoreLateInteraction(chunks, query, projectRoot) {
   if (chunks.length === 0) return { scores: new Map(), ran: false };
-  const liIndex = await _getLateInteractionIndex();
+  const liIndex = await _getLateInteractionIndex(projectRoot);
   if (!liIndex) return { scores: new Map(), ran: false };
 
-  // Only score chunks whose IDs actually appear in the LI index.
+  // Only score chunks whose IDs actually appear in the LI index. Use the
+  // public availability API so alias pointers and live tombstone sidecars
+  // share the same visibility contract as normal search.
+  const available = liIndex.hasTokens(chunks.map(c => c.id));
   const candidates = chunks
-    .filter(c => liIndex.documents.has(c.id))
+    .filter(c => available.has(c.id))
     .map(c => ({ id: c.id, score: 0 }));
   if (candidates.length === 0) return { scores: new Map(), ran: false };
 
@@ -457,7 +627,7 @@ function _fallbackSpanFromText(fileText, totalLines, maxChars) {
  * @param {boolean} [req.verbose=false] - include timings + signal contributions
  * @returns {Promise<Object>}
  */
-export async function readSemantic(req) {
+async function _readSemanticUnpinned(req) {
   const t0 = performance.now();
   if (!req || !req.path) throw new Error('path is required');
   if (!req.query || !String(req.query).trim()) throw new Error('query is required');
@@ -465,6 +635,7 @@ export async function readSemantic(req) {
   const projectRoot = req.projectRoot || process.cwd();
   _ensurePersistedLiModelApplied(projectRoot);
   const filePathRel = _projectRelative(req.path, projectRoot);
+  const staleness = _sourceStaleness(projectRoot, filePathRel);
 
   const topK = req.topK ?? DEFAULTS.topK;
   const threshold = req.threshold ?? DEFAULTS.threshold;
@@ -493,6 +664,7 @@ export async function readSemantic(req) {
       spans: fallback.ok ? [_fallbackSpanFromRead(fallback, maxChars)] : [],
       charsReturned: fallback.ok ? Math.min((fallback.text || '').length, maxChars) : 0,
       approxTokensReturned: fallback.ok ? Math.ceil(Math.min((fallback.text || '').length, maxChars) / APPROX_CHARS_PER_TOKEN) : 0,
+      ...(staleness ? { staleness, warnings: [staleness.warning] } : {}),
       timings: { totalMs: +(performance.now() - t0).toFixed(2) },
     };
   }
@@ -514,7 +686,7 @@ export async function readSemantic(req) {
   const tLex1 = performance.now();
 
   const tLi0 = performance.now();
-  const { scores: maxsimScores, ran: liRan } = await _scoreLateInteraction(chunks, req.query);
+  const { scores: maxsimScores, ran: liRan } = await _scoreLateInteraction(chunks, req.query, projectRoot);
   const tLi1 = performance.now();
 
   // Threshold gate on MaxSim — drop chunks whose LI score is too low. This
@@ -550,6 +722,7 @@ export async function readSemantic(req) {
       charsReturned: Math.min(fileText.length, maxChars),
       approxTokensReturned: Math.ceil(Math.min(fileText.length, maxChars) / APPROX_CHARS_PER_TOKEN),
       signals: verbose ? { liRan, lexicalHits: 0, symbolHits: 0, maxsimHits: 0 } : undefined,
+      ...(staleness ? { staleness, warnings: [staleness.warning] } : {}),
       timings: verbose ? {
         loadMs: +(tLoad1 - tLoad0).toFixed(2),
         lexicalMs: +(tLex1 - tLex0).toFixed(2),
@@ -568,12 +741,39 @@ export async function readSemantic(req) {
   // Final re-rank: prefer late-interaction score when LI ran; otherwise the
   // RRF score is the authority. This mirrors the SOTA pattern (cheap candidate
   // pool → expensive LI re-rank on the survivors).
+  //
+  // Multiplicative score demotions on null/unknown-symbol chunks and on tiny
+  // chunks are applied here so the re-rank below sees the corrected score
+  // (Stage 3 PHASE6_REDO ss-semantic, 2026-05-13). Demotion is intentionally
+  // applied AFTER the MaxSim re-rank threshold gate above — chunks still
+  // survive into the result, they're just less likely to win top-1.
+  const unsymDemote = req.unsymboledDemote ?? DEFAULTS.unsymboledDemote;
+  const smallDemote = req.smallChunkDemote ?? DEFAULTS.smallChunkDemote;
+  const smallChunkMaxLines = req.smallChunkMaxLines ?? DEFAULTS.smallChunkMaxLines;
+
   const ranked = fusedTop
     .map(([id, fusedScore]) => {
       const c = idToChunk.get(id);
       if (!c) return null;
       const li = maxsimScores.get(id);
-      const finalScore = liRan && li != null ? li : fusedScore;
+      const baseScore = liRan && li != null ? li : fusedScore;
+      // Stage 3 PHASE6_REDO ss-semantic (2026-05-13): demote only the
+      // INTERSECTION of (null-or-unknown symbol) AND (≤ smallChunkMaxLines).
+      // Earlier OR-form regressed typescript-lib (interface declarations
+      // are legitimately small AND symboled; OR-rule demoted them too).
+      // The intersection targets exactly the RB-001 pattern — short
+      // unnamed code fragments that win MaxSim by concentrated literal
+      // tokens (e.g., 3-line `module Sinatra` decl beating the 24-line
+      // Base class body). Multiplicative composition gives 0.85*0.85=0.7225x
+      // when both conditions fire.
+      const symMeta = c.symbol;
+      const isUnsymboled = !symMeta || symMeta === 'unknown';
+      const chunkLines = c.endLine - c.startLine + 1;
+      const isSmall = chunkLines <= smallChunkMaxLines;
+      const demoteFactor = (isUnsymboled && isSmall)
+        ? unsymDemote * smallDemote
+        : 1;
+      const finalScore = baseScore * demoteFactor;
       return {
         id,
         symbol: c.symbol,
@@ -586,6 +786,8 @@ export async function readSemantic(req) {
           symbol: symbolScores.get(id) || 0,
           maxsim: liRan ? (maxsimScores.get(id) ?? null) : null,
           fused: fusedScore,
+          baseScore,
+          demoteFactor,
         },
       };
     })
@@ -607,6 +809,7 @@ export async function readSemantic(req) {
     spans,
     charsReturned: charsUsed,
     approxTokensReturned: Math.ceil(charsUsed / APPROX_CHARS_PER_TOKEN),
+    ...(staleness ? { staleness, warnings: [staleness.warning] } : {}),
     signals: verbose ? {
       liRan,
       lexicalHits: lexicalScores.size,
@@ -624,6 +827,21 @@ export async function readSemantic(req) {
   };
 }
 
+export async function readSemantic(req) {
+  const projectRoot = req?.projectRoot || process.cwd();
+  return withPinnedRead(
+    {
+      projectRoot,
+      meta: {
+        tool: 'read-semantic',
+        path: req?.path ?? null,
+        query: req?.query ? String(req.query).slice(0, 200) : null,
+      },
+    },
+    () => _readSemanticUnpinned({ ...req, projectRoot }),
+  );
+}
+
 // ---------------------------------------------------------------------------
 // Formatting
 // ---------------------------------------------------------------------------
@@ -640,6 +858,9 @@ export function formatReadSemanticResult(result, format = 'agent') {
     lines.push(`[error]`);
     return lines.join('\n');
   }
+  for (const warning of result.warnings || []) {
+    lines.push(`[warning] ${warning}`);
+  }
   for (const span of result.spans) {
     const label = span.symbols && span.symbols.length
       ? `${span.symbols.join(', ')} (lines ${span.startLine}-${span.endLine})`
@@ -663,10 +884,18 @@ function _parseArgs(args) {
   const positional = [];
   let format = 'agent';
   let topK; let threshold; let contextLines; let maxChars; let maxTokens; let verbose = false;
+  let plain = false; let noBanner = false;
   for (let i = 0; i < args.length; i++) {
     const a = args[i];
     if (a === '--json') format = 'json';
     else if (a === '--agent') format = 'agent';
+    else if (a === '--no-banner') noBanner = true;
+    else if (a === '--format' || a.startsWith('--format=')) {
+      const v = a === '--format' ? args[++i] : a.slice('--format='.length);
+      if (v === 'json' || v === 'agent') format = v;
+      else if (v === 'plain') plain = true;
+      else throw new Error(`unknown --format value: ${v}`);
+    }
     else if (a === '--verbose') verbose = true;
     else if (a === '--top' || a === '--top-k' || a === '-k') topK = +args[++i];
     else if (a === '--threshold') threshold = +args[++i];
@@ -677,7 +906,7 @@ function _parseArgs(args) {
     else if (a.startsWith('--')) throw new Error(`unknown flag: ${a}`);
     else positional.push(a);
   }
-  return { positional, format, topK, threshold, contextLines, maxChars, maxTokens, verbose };
+  return { positional, format, topK, threshold, contextLines, maxChars, maxTokens, verbose, plain, noBanner };
 }
 
 function _printHelp() {
@@ -694,6 +923,8 @@ function _printHelp() {
     '  --max-chars <n>     Hard cap on returned text (default: 8000)',
     '  --max-tokens <n>    Convenience cap (~chars/4)',
     '  --json              Emit JSON',
+    '  --format <fmt>      json | agent | plain (plain = no identity line)',
+    '  --no-banner         Suppress the identity line',
     '  --verbose           Include timings + per-signal scores',
     '',
   ].join('\n'));
@@ -719,6 +950,9 @@ export async function handleReadSemanticCli(args) {
     maxTokens: parsed.maxTokens,
     verbose: parsed.verbose,
   });
+  if (parsed.format !== 'json') {
+    emitToolIdentityAuto('read-semantic', `${file} · "${query}"`, { plain: parsed.plain, noBanner: parsed.noBanner });
+  }
   process.stdout.write(formatReadSemanticResult(result, parsed.format));
   if (parsed.format !== 'json') process.stdout.write('\n');
   process.exit(result.ok ? 0 : 1);
@@ -726,9 +960,12 @@ export async function handleReadSemanticCli(args) {
 
 // Test-only export — clears caches between unit tests.
 export function __resetReadSemanticCachesForTests() {
-  _repo = null;
+  for (const entry of _repos.values()) entry?.repo?.close?.();
+  _repos.clear();
   _liIndex = null;
   _liInitPromise = null;
+  _liProjectKey = null;
+  _liManifestEpoch = null;
   _encodeQueryFn = null;
   _appliedLiPerRoot.clear();
 }