sweet-search 2.5.2 → 2.5.4

package/core/search/search-pattern-chunks.js

@@ -59,9 +59,8 @@ export function resolveSearchSymbolFilter(options = {}) {
 export function buildChunkLocationMap(liIndex) {
   const map = new Map();
 
-  for (const [id, doc] of liIndex.documents) {
-    const meta = doc.metadata;
-    if (!meta?.file || meta.startLine == null || meta.endLine == null) continue;
+  const addInterval = (id, meta) => {
+    if (!meta?.file || meta.startLine == null || meta.endLine == null) return;
 
     let bucket = map.get(meta.file);
     if (!bucket) {
@@ -75,11 +74,24 @@ export function buildChunkLocationMap(liIndex) {
       type: meta.type || null,
       name: meta.name || null,
     });
+  };
+
+  for (const [id, doc] of liIndex.documents) {
+    addInterval(id, doc.metadata);
+  }
+
+  for (const [id, ptr] of liIndex.aliasPointers || []) {
+    addInterval(id, ptr.metadata);
   }
 
   // Sort each file's intervals by startLine for binary search
   for (const bucket of map.values()) {
     bucket.sort((a, b) => a.startLine - b.startLine);
+    let maxEnd = -Infinity;
+    for (const interval of bucket) {
+      maxEnd = Math.max(maxEnd, interval.endLine);
+      interval._maxEndSoFar = maxEnd;
+    }
   }
 
   return map;
@@ -90,6 +102,7 @@ export function findChunkIntervalForLine(intervals, lineNumber) {
 
   let lo = 0;
   let hi = intervals.length - 1;
+  let candidateIdx = -1;
 
   while (lo <= hi) {
     const mid = (lo + hi) >>> 1;
@@ -97,13 +110,38 @@ export function findChunkIntervalForLine(intervals, lineNumber) {
 
     if (lineNumber < iv.startLine) {
       hi = mid - 1;
-    } else if (lineNumber > iv.endLine) {
-      lo = mid + 1;
     } else {
-      return { interval: iv, index: mid };
+      candidateIdx = mid;
+      lo = mid + 1;
     }
   }
 
+  if (candidateIdx < 0) return null;
+
+  let best = null;
+  let bestIdx = -1;
+  for (let i = candidateIdx; i >= 0; i--) {
+    const iv = intervals[i];
+    if (iv.startLine > lineNumber) continue;
+    if (iv._maxEndSoFar != null && iv._maxEndSoFar < lineNumber) break;
+    if (iv.endLine < lineNumber) continue;
+
+    // Prefer the tightest containing interval; if spans tie, prefer the
+    // later-starting interval because it is usually the nested symbol chunk.
+    if (
+      !best ||
+      (iv.endLine - iv.startLine) < (best.endLine - best.startLine) ||
+      ((iv.endLine - iv.startLine) === (best.endLine - best.startLine) && iv.startLine > best.startLine)
+    ) {
+      best = iv;
+      bestIdx = i;
+    }
+  }
+
+  if (best) {
+    return { interval: best, index: bestIdx };
+  }
+
   return null;
 }
 
@@ -270,12 +308,18 @@ export function buildBareGrepResults(matches, options = {}) {
  * Rebuilds only when the LI index document count changes (re-index).
  */
 export function getChunkLocationMap() {
-  const currentSize = this.lateInteractionIndex.documents.size;
-  if (this._chunkLocationMap && this._chunkLocationMapSize === currentSize) {
+  const currentSize = this.lateInteractionIndex.documents.size
+    + (this.lateInteractionIndex.aliasPointers?.size || 0);
+  if (
+    this._chunkLocationMap
+    && this._chunkLocationMapSize === currentSize
+    && this._chunkLocationMapIndex === this.lateInteractionIndex
+  ) {
     return this._chunkLocationMap;
   }
   this._chunkLocationMap = buildChunkLocationMap(this.lateInteractionIndex);
   this._chunkLocationMapSize = currentSize;
+  this._chunkLocationMapIndex = this.lateInteractionIndex;
   return this._chunkLocationMap;
 }
 
@@ -314,6 +358,11 @@ export function getCodebaseChunkTypeMap(searcher) {
 
   for (const bucket of map.values()) {
     bucket.sort((a, b) => a.startLine - b.startLine);
+    let maxEnd = -Infinity;
+    for (const interval of bucket) {
+      maxEnd = Math.max(maxEnd, interval.endLine);
+      interval._maxEndSoFar = maxEnd;
+    }
   }
 
   searcher._codebaseChunkTypeMap = map;

package/core/search/search-pattern-planner.js

@@ -13,9 +13,10 @@
 import {
   extractLiteralClauses, runLiteralPrefilterClauses, querySparseGramCandidates,
   ensureSparseGramIndex,
+  sparseDeltaOverlayHasChanges, getSparseGramAllFilesWithOverlay,
   hasCaseInsensitiveRegexFlag, nativeGrepFilesWithMatches,
   nativeGrepFilesWithMatchesFixed, nativeGrepLines, nativeGrepFull,
-  getSparseGramAllFiles, queryAndGrepLines, queryAndGrepFull,
+  queryAndGrepLines, queryAndGrepFull,
   searchLines, searchFull, resolveSparseSymbolMask,
 } from './search-pattern-prefilter.js';
 import { CODE_FILE_EXTENSIONS } from '../infrastructure/constants.js';
@@ -30,10 +31,12 @@ import { _getRgCapabilities, runRipgrepFilesWithMatches, runRipgrepJson, normali
  * Native grep bypasses ripgrep, so we must normalize here.
  */
 function normalizeNativeMatches(matches, searchDir) {
-  for (const m of matches) {
-    m.file = normalizeSearchPath(searchDir, m.file);
+  const out = [];
+  for (const m of matches || []) {
+    const file = normalizeSearchPath(searchDir, m.file);
+    if (file) out.push({ ...m, file });
   }
-  return matches;
+  return out;
 }
 
 // Cached once at module load — passed to Rust for code extension filtering.
@@ -61,7 +64,8 @@ export async function generateRegexMatches(searcher, regex, searchDir, options =
   // Rust internally: tries gram narrowing → if eligible, greps candidates; if not, greps all files.
   // Eliminates the JS planner, separate getSparseGramAllFiles call, and multiple NAPI crossings.
   const useGramIndex = options.useGramIndex ?? options.gramIndex ?? true;
-  const canUseUnifiedSearch = !fixedString && globs.length === 0;
+  const hasSparseDeltaOverlay = sparseDeltaOverlayHasChanges(searcher, options);
+  const canUseUnifiedSearch = !fixedString && globs.length === 0 && !hasSparseDeltaOverlay;
   if (canUseUnifiedSearch) {
     const sparseGramIndex = ensureSparseGramIndex(searcher, options);
     if (sparseGramIndex) {
@@ -173,6 +177,62 @@ export async function generateRegexMatches(searcher, regex, searchDir, options =
   const gramCandidateFiles = gramLookupResult?.candidateFiles || 0;
   const gramTotalFiles = gramLookupResult?.totalFiles || 0;
   const gramSelectivity = gramTotalFiles > 0 ? gramCandidateFiles / gramTotalFiles : null;
+  const narrowedThreshold = options.narrowedJsonThreshold ?? 300;
+  const directJsonThreshold = options.directJsonFileThreshold ?? 4096;
+
+  if (gramLookupResult?.eligible === true && Array.isArray(gramLookupResult.files) && gramLookupResult.files.length === 0) {
+    return {
+      indexedMatches: [],
+      overlayMatches: [],
+      matchingFiles: [],
+      stats: {
+        nativeGrepUsed: false,
+        candidateGenTime_ms: Math.round(performance.now() - start),
+        grepTime_ms: 0,
+        literalFilterTime_ms: 0,
+        gramLookupTime_ms: Math.round(gramLookupTime),
+        filesConsidered: gramTotalFiles,
+        filesScanned: 0,
+        filesSkipped: 0,
+        dirtyOverlayFiles: 0,
+        candidateFilesBeforeFilter: 0,
+        candidateFilesAfterFilter: 0,
+        candidateReductionRatio: 0,
+        literalExtractionHit: literalPlan.clauses.length > 0,
+        literalExtractionSource: literalPlan.source,
+        gramLookupReason: gramLookupResult.reason || 'ok',
+        prefilterDiscarded: false,
+        prefilterDiscardedCount: 0,
+        denseGramsTouched: gramLookupResult.denseGramsTouched || 0,
+        sparseGramsTouched: gramLookupResult.sparseGramsTouched || 0,
+        gramFalsePositiveRatio: 0,
+        grepStrategy: 'none',
+        plannerRoute: 'empty_gram_candidates',
+        gramSelectivity,
+        plannerInputs: {
+          narrowedFileCount: 0,
+          gramCandidateFiles,
+          gramTotalFiles,
+          narrowedThreshold,
+          directJsonThreshold,
+          skipLiteralPrefilter: true,
+        },
+        symbolTypeFilter,
+        trackerLastIndex: null,
+        grepMatches: 0,
+        stageTiming: {
+          literalExtractionTime_ms: +literalExtractionTime.toFixed(3),
+          gramQueryTime_ms: +gramLookupTime.toFixed(3),
+          regexBuildTime_ms: 0,
+          literalPrefilterTime_ms: 0,
+          plannerTime_ms: 0,
+          grepVerifyTime_ms: 0,
+          napiOverheadTime_ms: 0,
+          resultMaterializationTime_ms: 0,
+        },
+      },
+    };
+  }
 
   const fileGramTooBroad = gramLookupResult?.eligible === false && gramLookupResult?.reason === 'too_broad';
 
@@ -190,7 +250,9 @@ export async function generateRegexMatches(searcher, regex, searchDir, options =
   const sparseForAllFiles = (!fixedString && globs.length === 0)
     ? ensureSparseGramIndex(searcher, options)
     : null;
-  const allIndexedFiles = sparseForAllFiles ? getSparseGramAllFiles(sparseForAllFiles) : null;
+  const allIndexedFiles = sparseForAllFiles
+    ? getSparseGramAllFilesWithOverlay(searcher, sparseForAllFiles, options)
+    : null;
   const canNativeGrepAll = Array.isArray(allIndexedFiles) && allIndexedFiles.length > 0;
 
   const skipLiteralPrefilter = gramTooBroad || gramSaysBroad || canNativeGrepAll;
@@ -247,9 +309,6 @@ export async function generateRegexMatches(searcher, regex, searchDir, options =
   // Cost-model query planner
   // ==========================================================================
 
-  const narrowedThreshold = options.narrowedJsonThreshold ?? 300;
-  const directJsonThreshold = options.directJsonFileThreshold ?? 4096;
-
   let plannerRoute;
   let grepStrategy;
 

package/core/search/search-pattern-prefilter.js

@@ -23,6 +23,7 @@ import {
   searchLines as _searchLines,
   searchFull as _searchFull,
 } from '../infrastructure/native-sparse-gram.js';
+import { applySparseDeltaOverlay, liveOverlayFiles, loadSparseDeltaOverlay } from './search-pattern-sparse-overlay.js';
 
 // Re-export for search-pattern.js (avoids circular import through native-sparse-gram.js)
 export const resolveSparseSymbolMask = _resolveSparseSymbolMask;
@@ -35,8 +36,13 @@ export const queryAndGrepLines = _queryAndGrepLines;
 export const queryAndGrepFull = _queryAndGrepFull;
 export const searchLines = _searchLines;
 export const searchFull = _searchFull;
-import { DB_PATHS, PROJECT_ROOT } from '../infrastructure/config/index.js';
-import { CODE_FILE_EXTENSIONS } from '../infrastructure/constants.js';
+export {
+  applySparseDeltaOverlay,
+  getSparseGramAllFilesWithOverlay,
+  loadSparseDeltaOverlay,
+  sparseDeltaOverlayHasChanges,
+} from './search-pattern-sparse-overlay.js';
+import { DB_PATHS } from '../infrastructure/config/index.js';
 import { isRipgrepCodePath, resolveSearchSymbolFilter } from './search-pattern-chunks.js';
 
 // =============================================================================
@@ -251,12 +257,18 @@ export function ensureSparseGramIndex(searcher, options = {}) {
   if (!searcher) return null;
   const useGramIndex = options.useGramIndex ?? options.gramIndex ?? true;
   if (!useGramIndex) return null;
-  if (searcher.sparseGramIndex) return searcher.sparseGramIndex;
-
   const indexPath = options.sparseGramIndexPath || searcher.sparseGramIndexPath || DB_PATHS.sparseGramIndex;
+  if (searcher.sparseGramIndex) {
+    if (!searcher._sparseGramLoadedPath || searcher._sparseGramLoadedPath === indexPath) {
+      return searcher.sparseGramIndex;
+    }
+    searcher.sparseGramIndex = null;
+    searcher._sparseGramLoadedPath = null;
+  }
   const loaded = loadSparseGramIndex(indexPath);
   if (loaded) {
     searcher.sparseGramIndex = loaded;
+    searcher._sparseGramLoadedPath = indexPath;
   }
   return loaded;
 }
@@ -289,6 +301,8 @@ export function querySparseGramCandidates(searcher, literalClauses, options = {}
   }
 
   try {
+    const symbolMask = _resolveSparseSymbolMask(resolveSearchSymbolFilter(options));
+    const overlay = loadSparseDeltaOverlay(searcher, options);
     const sparseGramIndex = ensureSparseGramIndex(searcher, options);
     if (!sparseGramIndex) {
       return {
@@ -305,7 +319,6 @@ export function querySparseGramCandidates(searcher, literalClauses, options = {}
 
     const maxCandidateFiles = options.maxGramCandidateFiles ?? 100000;
     const maxCandidateRatio = options.maxGramCandidateRatio ?? 1.0;
-    const symbolMask = _resolveSparseSymbolMask(resolveSearchSymbolFilter(options));
     const combined = new Set();
     let totalFiles = 0;
     let gramsUsed = 0;
@@ -346,13 +359,22 @@ export function querySparseGramCandidates(searcher, literalClauses, options = {}
       gramsUsed += result.gramsUsed || 0;
       denseGramsTouched += result.denseGramsTouched || 0;
       sparseGramsTouched += result.sparseGramsTouched || 0;
-      const clauseFiles = Array.isArray(result.files)
+      const baseClauseFiles = Array.isArray(result.files)
         ? result.files.filter(isRipgrepCodePath)
         : [];
+      const clauseFiles = applySparseDeltaOverlay(
+        baseClauseFiles,
+        overlay,
+        symbolMask || 0,
+        searcher?.projectRoot || options.projectRoot,
+        clause,
+        sparseGramIndex,
+      );
+      const clauseTotalFiles = (result.totalFiles || 0) + liveOverlayFiles(overlay, symbolMask || 0).length;
+      totalFiles = Math.max(totalFiles, clauseTotalFiles);
       if (
-        clauseFiles.length === 0 ||
         clauseFiles.length > maxCandidateFiles ||
-        (result.totalFiles > 0 && (clauseFiles.length / result.totalFiles) > maxCandidateRatio)
+        (clauseTotalFiles > 0 && (clauseFiles.length / clauseTotalFiles) > maxCandidateRatio)
       ) {
         return {
           eligible: false,
@@ -370,7 +392,6 @@ export function querySparseGramCandidates(searcher, literalClauses, options = {}
 
     const files = [...combined];
     if (
-      files.length === 0 ||
       files.length > maxCandidateFiles ||
       (totalFiles > 0 && (files.length / totalFiles) > maxCandidateRatio)
     ) {
@@ -409,4 +430,3 @@ export function querySparseGramCandidates(searcher, literalClauses, options = {}
     };
   }
 }
-

package/core/search/search-pattern-ripgrep.js

@@ -10,7 +10,7 @@
  */
 
 import { spawn, execFileSync } from 'child_process';
-import { existsSync, readdirSync } from 'fs';
+import { existsSync, realpathSync, readdirSync } from 'fs';
 import { StringDecoder } from 'string_decoder';
 import path from 'path';
 import { RIPGREP_CODE_TYPE_GLOB } from '../infrastructure/constants.js';
@@ -134,7 +134,28 @@ export function normalizeSearchPath(searchDir, filePath) {
   const relative = path.isAbsolute(filePath)
     ? path.relative(searchDir, filePath)
     : filePath;
-  return relative.replace(/\\/g, '/').replace(/^\.\//, '');
+  const normalized = normalizeRelativeSearchPath(relative);
+  if (normalized) return normalized;
+  if (path.isAbsolute(filePath)) {
+    try {
+      const realRelative = path.relative(
+        realpathSync.native(searchDir),
+        realpathSync.native(filePath),
+      );
+      return normalizeRelativeSearchPath(realRelative);
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+
+function normalizeRelativeSearchPath(relative) {
+  const normalized = relative.replace(/\\/g, '/').replace(/^\.\//, '');
+  if (!normalized || normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) {
+    return null;
+  }
+  return normalized;
 }
 
 export function chunkRipgrepFiles(files) {
@@ -197,6 +218,10 @@ async function executeRipgrep({
       outputMode === 'json' ? '--json' : '--files-with-matches',
       '--type-add', RIPGREP_CODE_TYPE,
       '--type', 'code',
+      // Suppress I/O error messages (e.g. a candidate file deleted during the
+      // reconcile window). ripgrep still exits 2 on such errors but stderr
+      // stays empty; pattern-syntax errors still surface on stderr.
+      '--no-messages',
     ];
 
     if (maxCount > 0) args.push('--max-count', String(maxCount));
@@ -238,7 +263,11 @@ async function executeRipgrep({
     proc.stderr.on('data', (chunk) => { stderr += chunk; });
 
     proc.on('close', (code) => {
-      if (code !== 0 && code !== 1) {
+      // code 2 with empty stderr = benign I/O error (a candidate file vanished
+      // mid-flight under concurrent reconcile); use whatever matched rather
+      // than failing the whole query. Real errors (bad regex) keep stderr.
+      const benignIoError = code === 2 && stderr.trim() === '';
+      if (code !== 0 && code !== 1 && !benignIoError) {
         reject(new Error(`ripgrep failed (code ${code}): ${stderr.slice(0, 200)}`));
         return;
       }
@@ -371,6 +400,10 @@ async function executeRipgrepStreaming({
       '--json',
       '--type-add', RIPGREP_CODE_TYPE,
       '--type', 'code',
+      // Suppress I/O error messages (e.g. a candidate file deleted during the
+      // reconcile window). ripgrep still exits 2 on such errors but stderr
+      // stays empty; pattern-syntax errors still surface on stderr.
+      '--no-messages',
     ];
 
     if (maxCount > 0) args.push('--max-count', String(maxCount));
@@ -472,7 +505,10 @@ async function executeRipgrepStreaming({
         }
       }
 
-      if (killed || code === 0 || code === 1) {
+      // code 2 with empty stderr = benign I/O error (a candidate file vanished
+      // mid-flight under concurrent reconcile); resolve with what matched.
+      const benignIoError = code === 2 && stderr.trim() === '';
+      if (killed || code === 0 || code === 1 || benignIoError) {
         resolve(matches);
         return;
       }

package/core/search/search-pattern-sparse-overlay.js

@@ -0,0 +1,256 @@
+import fs from 'fs';
+import path from 'path';
+import {
+  extractSparseGramRequiredGrams,
+  getSparseGramAllFiles as _getSparseGramAllFiles,
+  resolveSparseSymbolMask as _resolveSparseSymbolMask,
+} from '../infrastructure/native-sparse-gram.js';
+import { resolveLatestSparseGramDeltaRecords } from '../infrastructure/sparse-gram-delta-reader.js';
+import { DB_PATHS, PROJECT_ROOT } from '../infrastructure/config/index.js';
+import { isRipgrepCodePath, resolveSearchSymbolFilter } from './search-pattern-chunks.js';
+
+const RECONCILE_MANIFEST_FILENAME = 'reconcile-manifest.json';
+
+function sparseGramIndexPath(searcher, options = {}) {
+  return options.sparseGramIndexPath || searcher?.sparseGramIndexPath || DB_PATHS.sparseGramIndex;
+}
+
+function normalizeDeltaPath(filePath, projectRoot = PROJECT_ROOT) {
+  if (!filePath || typeof filePath !== 'string') return null;
+  let normalized = filePath;
+  if (path.isAbsolute(normalized)) {
+    normalized = relativeInsideRoot(projectRoot, normalized);
+    return normalized || null;
+  }
+  normalized = normalized.replace(/\\/g, '/').replace(/^\.\//, '');
+  return normalized && !normalized.startsWith('../') ? normalized : null;
+}
+
+function relativeInsideRoot(projectRoot, absolutePath) {
+  const root = path.resolve(projectRoot);
+  const candidate = path.resolve(absolutePath);
+  const lexical = safeRelative(root, candidate);
+  if (lexical !== null) return lexical;
+
+  const rootReal = realpathOrNull(root);
+  const candidateReal = materializedRealpath(candidate);
+  if (!rootReal || !candidateReal) return null;
+  return safeRelative(rootReal, candidateReal);
+}
+
+function safeRelative(root, candidate) {
+  const rel = path.relative(root, candidate).replace(/\\/g, '/').replace(/^\.\//, '');
+  if (!rel || rel.startsWith('../') || path.isAbsolute(rel)) return null;
+  return rel;
+}
+
+function realpathOrNull(filePath) {
+  try {
+    return fs.realpathSync.native(filePath);
+  } catch {
+    return null;
+  }
+}
+
+function materializedRealpath(filePath) {
+  let current = filePath;
+  const rest = [];
+  while (current && current !== path.dirname(current)) {
+    const real = realpathOrNull(current);
+    if (real) return rest.length > 0 ? path.join(real, ...rest.reverse()) : real;
+    rest.push(path.basename(current));
+    current = path.dirname(current);
+  }
+  const rootReal = realpathOrNull(current);
+  return rootReal && rest.length > 0 ? path.join(rootReal, ...rest.reverse()) : rootReal;
+}
+
+function sparseManifestStateDirs(searcher, options = {}, indexPath) {
+  return [
+    options.manifestStateDir,
+    searcher?._manifestStateDir,
+    indexPath ? path.dirname(indexPath) : null,
+  ].filter((dir, idx, dirs) => typeof dir === 'string' && dir && dirs.indexOf(dir) === idx);
+}
+
+// Negative cache for stateDirs known to lack reconcile-manifest.json.
+// 1s TTL bounds staleness when reconcile later starts publishing.
+const _sparseManifestAbsentAt = new Map();
+const SPARSE_MANIFEST_ABSENT_TTL_MS = 1000;
+
+export function _resetSparseManifestAbsentCache() {
+  _sparseManifestAbsentAt.clear();
+}
+
+function readSparseManifest(searcher, options, indexPath) {
+  const dirs = sparseManifestStateDirs(searcher, options, indexPath);
+  const now = Date.now();
+  for (const dir of dirs) {
+    const absentAt = _sparseManifestAbsentAt.get(dir);
+    if (absentAt !== undefined && now - absentAt < SPARSE_MANIFEST_ABSENT_TTL_MS) {
+      continue;
+    }
+    const manifest = readSparseManifestFromDir(dir);
+    if (manifest) {
+      _sparseManifestAbsentAt.delete(dir);
+      return manifest;
+    }
+    _sparseManifestAbsentAt.set(dir, now);
+  }
+  return null;
+}
+
+function readSparseManifestFromDir(dir) {
+  try {
+    const manifestPath = path.join(dir, RECONCILE_MANIFEST_FILENAME);
+    if (!fs.existsSync(manifestPath)) return null;
+    const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+    const epoch = manifest?.sparseGram?.epoch ?? manifest?.epoch;
+    return {
+      epoch: Number.isInteger(epoch) ? epoch : null,
+      weightsId: typeof manifest?.sparseGram?.weightsId === 'string'
+        ? manifest.sparseGram.weightsId
+        : null,
+      deltas: Array.isArray(manifest?.sparseGram?.deltas)
+        ? manifest.sparseGram.deltas.filter((entry) => typeof entry === 'string')
+        : null,
+      stateDir: dir,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function resolveDeltaSegments(segments, stateDir) {
+  if (!Array.isArray(segments)) return null;
+  const out = [];
+  for (const segment of segments) {
+    if (path.isAbsolute(segment)) {
+      out.push(segment);
+    } else {
+      if (stateDir) out.push(path.join(stateDir, segment));
+      out.push(segment);
+    }
+  }
+  return [...new Set(out)];
+}
+
+function sparseManifestEpoch(searcher, options, manifestInfo) {
+  if (Number.isInteger(options.manifestEpoch)) return options.manifestEpoch;
+  if (Number.isInteger(searcher?.manifestEpoch)) return searcher.manifestEpoch;
+  const repoEpoch = searcher?.codebaseRepo?.getManifestEpoch?.();
+  if (Number.isInteger(repoEpoch)) return repoEpoch;
+  const graphEpoch = searcher?.graphSearch?.getManifestEpoch?.();
+  if (Number.isInteger(graphEpoch)) return graphEpoch;
+  return manifestInfo?.epoch ?? null;
+}
+
+function sparseWeightsId(searcher, options, manifestInfo) {
+  if (typeof options.sparseGramWeightsId === 'string') return options.sparseGramWeightsId;
+  if (typeof searcher?.sparseGramWeightsId === 'string') return searcher.sparseGramWeightsId;
+  return manifestInfo?.weightsId ?? null;
+}
+
+function sparseDeltaSegments(searcher, options, manifestInfo) {
+  if (Array.isArray(options.sparseGramDeltas)) return options.sparseGramDeltas;
+  if (Array.isArray(searcher?.sparseGramDeltas)) return searcher.sparseGramDeltas;
+  return Array.isArray(manifestInfo?.deltas) ? manifestInfo.deltas : null;
+}
+
+function normalizeRecordGrams(grams) {
+  if (!Array.isArray(grams) || grams.length === 0) return null;
+  const out = new Set();
+  for (const entry of grams) {
+    const gram = Array.isArray(entry) ? entry[0] : entry;
+    if (typeof gram === 'string' && gram.length > 0) out.add(gram);
+  }
+  return out.size > 0 ? out : null;
+}
+
+function recordMatchesClause(record, literals, sparseGramIndex) {
+  if (!Array.isArray(literals)) return true;
+  if (!record.grams) return true;
+  const required = extractSparseGramRequiredGrams(sparseGramIndex, literals);
+  if (!required) return true;
+  if (!required.eligible || !Array.isArray(required.grams) || required.grams.length === 0) return false;
+  return required.grams.every((gram) => record.grams.has(gram));
+}
+
+export function loadSparseDeltaOverlay(searcher, options = {}) {
+  const indexPath = sparseGramIndexPath(searcher, options);
+  if (!indexPath) return null;
+  const manifestInfo = readSparseManifest(searcher, options, indexPath);
+  const maxEpoch = sparseManifestEpoch(searcher, options, manifestInfo);
+  const manifestSegments = sparseDeltaSegments(searcher, options, manifestInfo);
+  if (!Array.isArray(manifestSegments)) return null;
+  const segments = resolveDeltaSegments(
+    manifestSegments,
+    manifestInfo?.stateDir || sparseManifestStateDirs(searcher, options, indexPath)[0],
+  );
+  if (!Array.isArray(segments) || segments.length === 0) return null;
+  const latest = resolveLatestSparseGramDeltaRecords(indexPath, {
+    ...(Number.isInteger(maxEpoch) ? { maxEpoch } : {}),
+    segments,
+  });
+  if (latest.size === 0) return null;
+
+  const hidden = new Set();
+  const live = [];
+  const projectRoot = searcher?.projectRoot || options.projectRoot || PROJECT_ROOT;
+  const expectedWeightsId = sparseWeightsId(searcher, options, manifestInfo);
+  for (const { record } of latest.values()) {
+    if (expectedWeightsId && record.weightsId !== expectedWeightsId) continue;
+    const filePath = normalizeDeltaPath(record.filePath, projectRoot);
+    if (!filePath) continue;
+    hidden.add(filePath);
+    if (!record.deleted) {
+      live.push({
+        filePath,
+        symbolMask: Number.isInteger(record.symbolMask) ? record.symbolMask : 0,
+        grams: normalizeRecordGrams(record.grams),
+      });
+    }
+  }
+  if (hidden.size === 0 && live.length === 0) return null;
+  return { hidden, live, maxEpoch };
+}
+
+export function sparseDeltaOverlayHasChanges(searcher, options = {}) {
+  const overlay = loadSparseDeltaOverlay(searcher, options);
+  return !!overlay && (overlay.hidden.size > 0 || overlay.live.length > 0);
+}
+
+export function liveOverlayFiles(overlay, symbolMask = 0, literals = null, sparseGramIndex = null) {
+  if (!overlay) return [];
+  const out = [];
+  for (const record of overlay.live) {
+    if (!isRipgrepCodePath(record.filePath)) continue;
+    if (symbolMask && record.symbolMask && (record.symbolMask & symbolMask) === 0) continue;
+    if (!recordMatchesClause(record, literals, sparseGramIndex)) continue;
+    out.push(record.filePath);
+  }
+  return out;
+}
+
+export function applySparseDeltaOverlay(files, overlay, symbolMask = 0, projectRoot = PROJECT_ROOT, literals = null, sparseGramIndex = null) {
+  if (!overlay) return Array.isArray(files) ? files : [];
+  const merged = new Set();
+  for (const file of Array.isArray(files) ? files : []) {
+    const normalized = normalizeDeltaPath(file, projectRoot);
+    if (normalized && !overlay.hidden.has(normalized) && isRipgrepCodePath(normalized)) {
+      merged.add(normalized);
+    }
+  }
+  for (const file of liveOverlayFiles(overlay, symbolMask, literals, sparseGramIndex)) {
+    merged.add(file);
+  }
+  return [...merged];
+}
+
+export function getSparseGramAllFilesWithOverlay(searcher, sparseGramIndex, options = {}) {
+  const baseFiles = _getSparseGramAllFiles(sparseGramIndex);
+  if (!Array.isArray(baseFiles)) return baseFiles;
+  const symbolMask = _resolveSparseSymbolMask(resolveSearchSymbolFilter(options));
+  const projectRoot = searcher?.projectRoot || options.projectRoot || PROJECT_ROOT;
+  return applySparseDeltaOverlay(baseFiles, loadSparseDeltaOverlay(searcher, options), symbolMask || 0, projectRoot);
+}