sweet-search 2.5.2 → 2.5.4

package/core/indexing/indexer-manifest.js

@@ -0,0 +1,66 @@
+import path from 'path';
+import { DB_PATHS } from '../infrastructure/config/index.js';
+import {
+  buildNextManifest,
+  readManifest,
+  writeManifest,
+  zeroManifest,
+} from '../incremental-indexing/infrastructure/manifest.mjs';
+import { FALLBACK_WEIGHTS_ID } from '../incremental-indexing/infrastructure/sparse-gram-delta.mjs';
+
+function basename(filePath) {
+  return path.basename(filePath);
+}
+
+function sparseGramWeightsIdFromResult(result) {
+  if (typeof result?.weightsId === 'string' && result.weightsId) return result.weightsId;
+  if (typeof result?.weights_id === 'string' && result.weights_id) return result.weights_id;
+  if (result?.usedFallbackWeights) return FALLBACK_WEIGHTS_ID;
+  return null;
+}
+
+export function defaultIndexerManifestPaths() {
+  const liBase = basename(DB_PATHS.lateInteraction);
+  return {
+    codeGraph: basename(DB_PATHS.codeGraph),
+    vectors: basename(DB_PATHS.codebase),
+    hnsw: basename(DB_PATHS.hnswIndex),
+    hnswStale: basename(DB_PATHS.hnswIndex) + '.stale.bin',
+    binaryHnsw: basename(DB_PATHS.binaryHnswIndex),
+    liManifest: `${liBase}.segments/manifest.json`,
+    sparseBase: basename(DB_PATHS.sparseGramIndex),
+  };
+}
+
+export function defaultIndexerStateDir() {
+  return path.dirname(DB_PATHS.codebase);
+}
+
+export function publishIndexerManifest(options = {}) {
+  const stateDir = options.stateDir || defaultIndexerStateDir();
+  const defaultManifest = zeroManifest(defaultIndexerManifestPaths());
+  const previous = readManifest(stateDir) || defaultManifest;
+  const epoch = Number.isInteger(options.epoch) ? options.epoch : (previous.epoch ?? 0) + 1;
+  const sparseWeightsId = sparseGramWeightsIdFromResult(options.sparseGramResult);
+  const defaultTiers = {
+    codeGraph: defaultManifest.codeGraph,
+    vectors: defaultManifest.vectors,
+    hnsw: defaultManifest.hnsw,
+    binaryHnsw: defaultManifest.binaryHnsw,
+    lateInteraction: defaultManifest.lateInteraction,
+    sparseGram: {
+      ...defaultManifest.sparseGram,
+      ...(sparseWeightsId ? { weightsId: sparseWeightsId } : {}),
+    },
+  };
+  const tiers = {};
+  for (const [tier, descriptor] of Object.entries(defaultTiers)) {
+    tiers[tier] = { ...descriptor, ...(options.tiers?.[tier] || {}) };
+  }
+  const manifest = buildNextManifest(previous, {
+    epoch,
+    tiers,
+  });
+  writeManifest(stateDir, manifest);
+  return manifest;
+}

package/core/indexing/indexer-phases.js

@@ -16,12 +16,14 @@ import { runDedupPhase, formatDedupSummary } from './dedup/dedup-phase.js';
 import { DEDUP_CONFIG } from '../infrastructure/config/index.js';
 import { incrementalUpdateHNSW, buildHNSWIndex, buildLateInteractionIndex, buildQuantizedArtifactsPhase } from './indexer-ann.js';
 import { buildSparseGramArtifact } from './indexer-sparse-gram.js';
+import { publishIndexerManifest } from './indexer-manifest.js';
+import { contentHashSync } from '../incremental-indexing/infrastructure/hashing.mjs';
 import {
   configureLocalModelRuntime,
   resetLocalModelRuntime,
 } from '../embedding/embedding-local-model.js';
 import { isNativeInferenceAvailable } from '../infrastructure/native-inference.js';
-import { teardownAllModels, initIndexGpuPool, teardownIndexGpuPool, warmupQueryCpuModels, GPU_ARMING_MIN_FILES } from './model-pool.js';
+import { teardownAllModels, initIndexGpuPool, teardownIndexGpuPool, warmupQueryCpuModels, GPU_ARMING_MIN_FILES, isIndexAcceleratorAvailable } from './model-pool.js';
 import {
   configureLateInteractionRuntime,
   resetLateInteractionRuntime,
@@ -423,34 +425,47 @@ export async function buildVectorsAndArtifactsPhase(options = {}) {
   }
 
   // The embedding worker pool uses ORT INT8 CPU in each worker. It must only
-  // be active when the query-time encoder ALSO uses ORT INT8 CPU, otherwise
+  // be active when the index-time encoder ALSO uses ORT INT8 CPU, otherwise
   // the stored index and the query vectors live in different embedding spaces
-  // (gencodesearchnet 83% → 58% MRR regression). Two cases where that's true:
+  // (gencodesearchnet 83% → 58% MRR regression — queries are always ORT INT8
+  // CPU). Three cases where index-time embed is ORT INT8 CPU:
   //   1. Native inference isn't available at all (pre-native hosts).
-  //   2. SWEET_SEARCH_EMBED_USE_CPU=1 — the user opted into CPU embed on
+  //   2. No usable accelerator (Metal/CoreML/CUDA) — even if the native addon
+  //      is installed, the native model is never loaded on a no-accelerator
+  //      host (see model-pool.initIndexGpuPool), so embed dispatch falls to
+  //      ORT INT8. Running the pool here makes that path multi-threaded
+  //      instead of inline.
+  //   3. SWEET_SEARCH_EMBED_USE_CPU=1 — the user opted into CPU embed on
   //      both sides (index + query), so pool ORT embed matches dispatcher
-  //      ORT embed. This is the "ORT embed on CPU ‖ native LI on Metal"
+  //      ORT embed. This is the "ORT embed on CPU ‖ native LI on accelerator"
   //      pipeline that maximises index throughput by running embed and LI
   //      on different devices.
   //
   // The historical `!shouldParallelLI` gate existed for the all-CPU era where
   // pool workers and parallel LI both wanted CPU and fought. In the CPU-embed
-  // + Metal-LI world, that conflict goes away — pool workers do ORT on CPU
-  // cores, the main thread drives Metal LI dispatches (negligible CPU), no
-  // contention. So when `SWEET_SEARCH_EMBED_USE_CPU=1` we lift the gate and
-  // let the pool run alongside parallel LI.
+  // + accelerator-LI world, that conflict goes away — pool workers do ORT on
+  // CPU cores, the main thread drives accelerator LI dispatches (negligible
+  // CPU), no contention. So when `SWEET_SEARCH_EMBED_USE_CPU=1` (and LI is on
+  // a real accelerator) we lift the gate and let the pool run alongside
+  // parallel LI. On a no-accelerator host LI is also on ORT CPU, so the gate
+  // stays in force and pool + parallel LI take turns rather than contend.
   const forceEmbedCpu = process.env.SWEET_SEARCH_EMBED_USE_CPU === '1';
-  const queryTimeEmbedIsCpu = !isNativeInferenceAvailable() || forceEmbedCpu;
-  // When LI is on Metal (native), pool + parallelLI is safe — the LI driver
-  // is just dispatching commands, not competing for CPU cores.
-  const liOnMetal = isNativeInferenceAvailable() && !noLateInteraction;
-  const allowPoolWithParallelLi = forceEmbedCpu && liOnMetal;
+  const indexTimeEmbedIsCpu = !isNativeInferenceAvailable()
+    || !isIndexAcceleratorAvailable()
+    || forceEmbedCpu;
+  // LI runs on a native accelerator only when one is actually armed. When it
+  // is, pool + parallelLI is safe — the LI driver is just dispatching GPU
+  // commands, not competing for CPU cores.
+  const liOnAccelerator = isNativeInferenceAvailable()
+    && isIndexAcceleratorAvailable()
+    && !noLateInteraction;
+  const allowPoolWithParallelLi = forceEmbedCpu && liOnAccelerator;
   const useEmbeddingPool = !dryRun
     && filesToIndex.length > 0
     && EMBEDDING_CONFIG.provider === 'local'
     && resourcePlan.useWorkerPool
     && (!shouldParallelLI || allowPoolWithParallelLi)
-    && queryTimeEmbedIsCpu;
+    && indexTimeEmbedIsCpu;
 
   if (!dryRun && EMBEDDING_CONFIG.provider === 'local' && filesToIndex.length > 0) {
     configureLocalModelRuntime({ intraOpThreads: embeddingThreads });
@@ -500,15 +515,26 @@ export async function buildVectorsAndArtifactsPhase(options = {}) {
   // run a dummy forward pass to compile Metal pipelines / CoreML variants
   // / BLAS threads.
   //
+  // No-accelerator skip: a host with no usable Metal / CoreML / CUDA
+  // accelerator indexes on the optimized ORT INT8 CPU path and never arms
+  // candle/native. `isIndexAcceleratorAvailable()` gates this even when the
+  // optional native addon is installed (e.g. Linux + the CUDA package but a
+  // failed/absent CUDA runtime, or SWEET_SEARCH_CUDA=0) — the JS layer is the
+  // authoritative selector; we never lean on Rust degrading loadWithDevice()
+  // to CPU. Skipping arming also skips the teardown/CPU-rewarm lifecycle in
+  // the `finally` below, so a CPU-only full reindex simply runs on ORT CPU.
+  //
   // Small-changeset skip: incremental runs with fewer than
   // GPU_ARMING_MIN_FILES files keep the ORT CPU path. The GPU load +
   // warmup + teardown + CPU rewarm round-trip costs 5–15s on M3 class
   // hardware and would dwarf the actual work (<1s per file on CPU).
-  // Full reindex always arms the GPU regardless of file count.
+  // Full reindex always arms the GPU regardless of file count — but only
+  // when an accelerator exists.
   const shouldArmGpu = !dryRun
     && filesToIndex.length > 0
     && EMBEDDING_CONFIG.provider === 'local'
     && isNativeInferenceAvailable()
+    && isIndexAcceleratorAvailable()
     && (fullReindex || filesToIndex.length >= GPU_ARMING_MIN_FILES);
 
   if (shouldArmGpu) {
@@ -531,6 +557,8 @@ export async function buildVectorsAndArtifactsPhase(options = {}) {
     }
   } else if (!dryRun && filesToIndex.length > 0 && filesToIndex.length < GPU_ARMING_MIN_FILES) {
     log(`Small changeset (${filesToIndex.length} < ${GPU_ARMING_MIN_FILES} files) — using ORT CPU`, 'dim');
+  } else if (!dryRun && filesToIndex.length > 0 && !isIndexAcceleratorAvailable()) {
+    log('No inference accelerator detected — indexing on ORT INT8 CPU', 'dim');
   }
 
   try {
@@ -696,7 +724,7 @@ export async function buildVectorsAndArtifactsPhase(options = {}) {
 }
 
 export async function updateIncrementalStatePhase(options = {}) {
-  const { dryRun, fullReindex, incrementalInfo, allFiles, vectorStats, graphStats } = options;
+  const { dryRun, fullReindex, incrementalInfo, allFiles, vectorStats, graphStats, manifestStateDir, sparseGramResult } = options;
 
   if (dryRun) return;
 
@@ -709,18 +737,18 @@ export async function updateIncrementalStatePhase(options = {}) {
     log('\nIncremental state updated', 'green');
   } else if (fullReindex) {
     const hashes = {};
-    const crypto = await import('crypto');
     for (const file of allFiles) {
       try {
         const fullPath = path.join(PROJECT_ROOT, file);
         const [content, stat] = await Promise.all([
-          fs.readFile(fullPath, 'utf-8'),
-          fs.stat(fullPath).catch(() => null),
+          fs.readFile(fullPath),
+          fs.stat(fullPath, { bigint: true }).catch(() => null),
         ]);
         hashes[file] = {
-          hash: crypto.createHash('sha256').update(content).digest('hex').slice(0, 16),
-          size: stat?.size ?? null,
-          mtime_ns: stat ? String(BigInt(Math.round(stat.mtimeMs)) * 1000000n) : null,
+          hash: contentHashSync(content),
+          size: stat ? stat.size.toString() : null,
+          mtime_ns: stat ? stat.mtimeNs.toString() : null,
+          inode: stat ? stat.ino.toString() : null,
         };
       } catch (e) { /* skip */ }
     }
@@ -731,6 +759,11 @@ export async function updateIncrementalStatePhase(options = {}) {
     });
     log('\nIncremental state saved', 'green');
   }
+  publishIndexerManifest({
+    ...(manifestStateDir ? { stateDir: manifestStateDir } : {}),
+    ...(sparseGramResult ? { sparseGramResult } : {}),
+  });
+  log('Reconcile manifest published', 'green');
 }
 
 export function printSummaryPhase(options) {

package/core/indexing/indexer-sparse-gram.js

@@ -9,6 +9,8 @@ import {
   hasNativeSparseGramSupport,
   resolveSparseSymbolMask,
 } from '../infrastructure/native-sparse-gram.js';
+import { contentHash } from '../incremental-indexing/infrastructure/hashing.mjs';
+import { FALLBACK_WEIGHTS_ID } from '../incremental-indexing/infrastructure/sparse-gram-delta.mjs';
 import { atomicSwapDatabase, log } from './indexer-utils.js';
 
 async function unlinkIfExists(filePath) {
@@ -28,18 +30,7 @@ async function collectFileSymbolMasks(codeFiles) {
   const db = new Database(DB_PATHS.codebase, { readonly: true });
 
   try {
-    const rows = db.prepare('SELECT file_path, metadata FROM vectors').iterate();
-    for (const row of rows) {
-      if (!masks.has(row.file_path)) continue;
-      try {
-        const metadata = JSON.parse(row.metadata || '{}');
-        const typeMask = resolveSparseSymbolMask(metadata.type);
-        if (typeMask === 0) continue;
-        masks.set(row.file_path, masks.get(row.file_path) | typeMask);
-      } catch {
-        // Ignore malformed metadata rows; sparse gram build is best effort.
-      }
-    }
+    collectFileSymbolMasksFromDb(db, codeFiles, masks);
   } finally {
     db.close();
   }
@@ -47,6 +38,48 @@ async function collectFileSymbolMasks(codeFiles) {
   return codeFiles.map((filePath) => masks.get(filePath) || 0);
 }
 
+async function resolveSparseGramWeightsId(result, artifactPath) {
+  if (typeof result?.weightsId === 'string' && result.weightsId) {
+    return result.weightsId;
+  }
+  if (typeof result?.weights_id === 'string' && result.weights_id) {
+    return result.weights_id;
+  }
+  if (result?.usedFallbackWeights) return FALLBACK_WEIGHTS_ID;
+  const artifactBytes = await fs.readFile(artifactPath);
+  return `corpus-bigram-v1-${await contentHash(artifactBytes)}`;
+}
+
+function collectFileSymbolMasksFromDb(db, codeFiles, existingMasks = null) {
+  const masks = existingMasks || new Map(codeFiles.map((filePath) => [filePath, 0]));
+  const liveSql = liveVectorSql(db);
+  const rows = db.prepare(`SELECT file_path, metadata FROM vectors WHERE ${liveSql}`).iterate();
+  for (const row of rows) {
+    if (!masks.has(row.file_path)) continue;
+    try {
+      const metadata = JSON.parse(row.metadata || '{}');
+      const typeMask = resolveSparseSymbolMask(metadata.type);
+      if (typeMask === 0) continue;
+      masks.set(row.file_path, masks.get(row.file_path) | typeMask);
+    } catch {
+      // Ignore malformed metadata rows; sparse gram build is best effort.
+    }
+  }
+  return codeFiles.map((filePath) => masks.get(filePath) || 0);
+}
+
+function hasVectorColumn(db, column) {
+  try {
+    return db.prepare('PRAGMA table_info(vectors)').all().some((col) => col.name === column);
+  } catch (_err) {
+    return false;
+  }
+}
+
+function liveVectorSql(db) {
+  return hasVectorColumn(db, 'epoch_retired') ? 'epoch_retired IS NULL' : '1=1';
+}
+
 export async function buildSparseGramArtifact(allFiles, dryRun) {
   if (dryRun) {
     log('DRY RUN: Skipping sparse gram artifact build', 'magenta');
@@ -85,14 +118,22 @@ export async function buildSparseGramArtifact(allFiles, dryRun) {
       fileSymbolMasks,
       outputPath: stagedPath,
     });
+    const weightsId = await resolveSparseGramWeightsId(result, stagedPath);
     await atomicSwapDatabase(stagedPath, DB_PATHS.sparseGramIndex);
     log(
       `Sparse gram artifact promoted (${result.filesIndexed} files, ${result.grams} grams, ${result.postings} postings)`,
       'green'
     );
-    return result;
+    return { ...result, weightsId };
   } catch (err) {
     await unlinkIfExists(stagedPath);
     throw err;
   }
 }
+
+export const __TEST__ = {
+  collectFileSymbolMasks,
+  collectFileSymbolMasksFromDb,
+  liveVectorSql,
+  resolveSparseGramWeightsId,
+};

package/core/indexing/indexer-utils.js

@@ -5,11 +5,21 @@
 
 import fs from 'fs/promises';
 import { existsSync } from 'fs';
-import { spawn } from 'child_process';
 import path from 'path';
 import fg from 'fast-glob';
 
-import { PROJECT_ROOT, setQuietMode as setGlobalQuietMode, loadProjectConfig, AGENTIC_GITIGNORE_ALLOWLIST } from '../infrastructure/config/index.js';
+import { PROJECT_ROOT, setQuietMode as setGlobalQuietMode } from '../infrastructure/config/index.js';
+import { createAdmissionPolicy } from './admission-policy.js';
+
+// `.gitignore` alignment now lives in gitignore-filter.js (shared with the
+// incremental admission policy). Re-exported here so existing
+// `import { ... } from indexer-utils` / barrel call sites keep working.
+export {
+  toPosixPath,
+  isGitignoreAllowlistedAgenticPath,
+  getGitIgnoredPathSet,
+  applyGitignoreAlignment,
+} from './gitignore-filter.js';
 
 const glob = fg.glob || fg;
 
@@ -205,10 +215,6 @@ export function stripWslUncPrefix(filePath) {
   return filePath;
 }
 
-export function toPosixPath(filePath) {
-  return filePath.replace(/\\/g, '/');
-}
-
 export async function readFilesFromStdin() {
   return new Promise((resolve, reject) => {
     let data = '';
@@ -278,201 +284,6 @@ export async function readFilesFromStdin() {
   });
 }
 
-// =============================================================================
-// GITIGNORE ALIGNMENT
-// =============================================================================
-
-export function isGitignoreAllowlistedAgenticPath(relativePath) {
-  const normalized = toPosixPath(relativePath).replace(/^\.\//, '');
-  const basename = path.posix.basename(normalized);
-
-  if (AGENTIC_GITIGNORE_ALLOWLIST.files.includes(basename)) {
-    return true;
-  }
-
-  if (AGENTIC_GITIGNORE_ALLOWLIST.filePrefixes.some(prefix => basename.startsWith(prefix))) {
-    return true;
-  }
-
-  return AGENTIC_GITIGNORE_ALLOWLIST.directories.some(dirPrefix =>
-    normalized.startsWith(dirPrefix) || normalized.includes(`/${dirPrefix}`)
-  );
-}
-
-/**
- * Run `git check-ignore` on a single batch of paths.
- * Returns a Set of ignored paths, or null on fatal error.
- */
-function checkIgnoreBatch(batch, reportError) {
-  return new Promise((resolve) => {
-    const ignoredChunks = [];
-    let settled = false;
-
-    const git = spawn('git', ['check-ignore', '-z', '--stdin'], { cwd: PROJECT_ROOT });
-
-    git.stdout.on('data', chunk => ignoredChunks.push(chunk));
-    git.stderr.on('data', () => {}); // Suppress — batched caller handles partial failures
-
-    git.on('error', (err) => {
-      if (settled) return;
-      settled = true;
-      reportError(`WARN: Unable to run git check-ignore (${err.message})`);
-      resolve(null);
-    });
-
-    git.on('close', (code) => {
-      if (settled) return;
-      settled = true;
-
-      // code 0 = some ignored, code 1 = none ignored, both valid.
-      // code 128 = fatal (e.g. path beyond symlink) — still use partial stdout.
-      if (code !== 0 && code !== 1 && ignoredChunks.length === 0) {
-        resolve(null);
-        return;
-      }
-
-      const ignored = Buffer.concat(ignoredChunks)
-        .toString('utf8')
-        .split('\0')
-        .filter(Boolean)
-        .map(toPosixPath);
-
-      resolve(ignored);
-    });
-
-    const stdinPayload = `${batch.map(toPosixPath).join('\0')}\0`;
-    git.stdin.on('error', () => {}); // Suppress EPIPE if git exits early
-    git.stdin.end(stdinPayload);
-  });
-}
-
-const CHECK_IGNORE_BATCH_SIZE = 5000;
-
-/**
- * Find directory components that are symlinks, so we can filter out paths
- * that traverse them (git check-ignore fatals on "beyond a symbolic link").
- */
-async function findSymlinkDirs(paths) {
-  const checked = new Map();
-  const symlinkPrefixes = [];
-
-  for (const p of paths) {
-    const parts = p.split('/');
-    let dir = '';
-    for (let i = 0; i < parts.length - 1; i++) {
-      dir = dir ? `${dir}/${parts[i]}` : parts[i];
-      if (checked.has(dir)) continue;
-      try {
-        const stat = await fs.lstat(path.join(PROJECT_ROOT, dir));
-        const isLink = stat.isSymbolicLink();
-        checked.set(dir, isLink);
-        if (isLink) symlinkPrefixes.push(dir + '/');
-      } catch {
-        checked.set(dir, false);
-      }
-    }
-  }
-
-  return symlinkPrefixes;
-}
-
-export async function getGitIgnoredPathSet(paths, options = {}) {
-  const silent = options.silent ?? false;
-  const reportError = silent ? () => {} : logError;
-
-  if (paths.length === 0) {
-    return new Set();
-  }
-
-  const ignored = new Set();
-
-  // Pre-filter paths that traverse symlinks — git check-ignore fatals on these.
-  // Files beyond symlinks are also checked: if the symlink dir itself is ignored,
-  // all files under it are treated as ignored too.
-  const symlinkPrefixes = await findSymlinkDirs(paths);
-  let safePaths = paths;
-  if (symlinkPrefixes.length > 0) {
-    // Check if the symlink directories themselves are ignored
-    const symlinkDirs = symlinkPrefixes.map(p => p.slice(0, -1)); // remove trailing /
-    const symlinkIgnored = await checkIgnoreBatch(symlinkDirs, reportError);
-    const ignoredSymlinks = new Set(symlinkIgnored || []);
-
-    safePaths = [];
-    for (const p of paths) {
-      const matchedPrefix = symlinkPrefixes.find(prefix => p.startsWith(prefix));
-      if (matchedPrefix) {
-        // Path traverses a symlink — check if symlink dir is gitignored
-        const dir = matchedPrefix.slice(0, -1);
-        if (ignoredSymlinks.has(dir)) {
-          ignored.add(toPosixPath(p)); // inherit parent's ignored status
-        }
-        // Either way, skip git check-ignore (would fatal)
-      } else {
-        safePaths.push(p);
-      }
-    }
-  }
-
-  let failedBatches = 0;
-
-  for (let i = 0; i < safePaths.length; i += CHECK_IGNORE_BATCH_SIZE) {
-    const batch = safePaths.slice(i, i + CHECK_IGNORE_BATCH_SIZE);
-    const result = await checkIgnoreBatch(batch, reportError);
-    if (result) {
-      for (const p of result) ignored.add(p);
-    } else {
-      failedBatches++;
-    }
-  }
-
-  const totalBatches = Math.ceil(safePaths.length / CHECK_IGNORE_BATCH_SIZE);
-  if (failedBatches === totalBatches && totalBatches > 0) {
-    reportError('WARN: git check-ignore failed on all batches — gitignore filtering disabled');
-    return null;
-  }
-
-  return ignored;
-}
-
-export async function applyGitignoreAlignment(files, respectGitignore, options = {}) {
-  if (!respectGitignore || !existsSync(path.join(PROJECT_ROOT, '.git'))) {
-    return { files, gitignored: 0 };
-  }
-
-  const bypassGitignore = new Set();
-  const candidates = [];
-  for (const file of files) {
-    if (isGitignoreAllowlistedAgenticPath(file)) {
-      bypassGitignore.add(file);
-    } else {
-      candidates.push(file);
-    }
-  }
-
-  const ignoredSet = await getGitIgnoredPathSet(candidates, options);
-  if (!ignoredSet) {
-    return { files, gitignored: 0 };
-  }
-
-  const kept = [];
-  let gitignored = 0;
-  for (const file of files) {
-    if (bypassGitignore.has(file)) {
-      kept.push(file);
-      continue;
-    }
-
-    const normalized = toPosixPath(file);
-    if (ignoredSet.has(normalized)) {
-      gitignored++;
-      continue;
-    }
-    kept.push(file);
-  }
-
-  return { files: kept, gitignored };
-}
-
 // =============================================================================
 // FILE DISCOVERY
 // =============================================================================
@@ -487,19 +298,26 @@ export async function discoverFiles(options = {}) {
 
   writeLog('\n━━━ Discovering Files ━━━', 'bright');
 
-  const projectConfig = loadProjectConfig(projectRoot);
-  const respectGitignore = projectConfig.respectGitignore !== false;
-  const maxFileSize = projectConfig.maxFileSize || (1 * 1024 * 1024);
-
-  const discovered = await glob(projectConfig.include, {
-    ignore: projectConfig.exclude,
+  // Single shared admission policy — the same include allowlist / deny-list /
+  // `.sweet-search-ignore` / `.gitignore` / size gates the incremental
+  // maintainer uses, so a fresh full index and an incrementally-maintained
+  // index admit exactly the same files.
+  const policy = createAdmissionPolicy({ projectRoot });
+  const maxFileSize = policy.maxFileSize;
+
+  // Enumerate via the include globs (with the exclude globs pruning big dirs
+  // during traversal), then apply the policy's shape gate so `.sweet-search-ignore`
+  // is honoured here too — the one rule full discovery did not previously apply.
+  const discovered = await glob(policy.includeGlobs, {
+    ignore: policy.excludeGlobs,
     cwd: projectRoot,
     absolute: false,
     onlyFiles: true,
     dot: true,
   });
+  const shaped = discovered.filter((rel) => policy.admitsShape(rel));
 
-  const { files: allFiles, gitignored } = await applyGitignoreAlignment(discovered, respectGitignore, { silent });
+  const { files: allFiles, gitignored } = await policy.applyGitignore(shaped, { silent });
 
   const files = [];
   let oversized = 0;

package/core/indexing/indexing-file-policy.js

@@ -17,6 +17,28 @@ function normalizePath(p) {
   return p.replace(/\\/g, '/');
 }
 
+function chunkPath(chunk) {
+  return firstSafeRelativePath(
+    chunk?.metadata?.relative_path,
+    chunk?.metadata?.path,
+    chunk?.metadata?.file_path,
+    chunk?.file,
+    chunk?.metadata?.file,
+  ) || '';
+}
+
+function firstSafeRelativePath(...candidates) {
+  for (const candidate of candidates) {
+    if (typeof candidate !== 'string') continue;
+    const normalized = candidate.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/');
+    if (!normalized || normalized === '.' || normalized.startsWith('/')) continue;
+    if (/^[A-Za-z]:\//.test(normalized)) continue;
+    if (normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) continue;
+    return normalized;
+  }
+  return null;
+}
+
 const _excludesByRoot = new Map();
 
 function getExcludes(projectRoot) {
@@ -93,24 +115,26 @@ export function applyIndexingChunkPolicy(chunks, options = {}) {
   const fileFirstReason = new Map();
 
   for (const chunk of chunks) {
-    if (!chunk?.file) continue;
-    if (fileFirstReason.has(chunk.file)) continue;
+    const file = chunkPath(chunk);
+    if (!file) continue;
+    if (fileFirstReason.has(file)) continue;
 
     let reason = null;
-    if (isExcludedByConfig(chunk.file, projectRoot)) {
+    if (isExcludedByConfig(file, projectRoot)) {
       reason = 'excluded';
     } else {
       const text = chunk.text || chunk.content || '';
       if (chunkLooksGenerated(text)) reason = 'generated';
     }
-    fileFirstReason.set(chunk.file, reason);
+    fileFirstReason.set(file, reason);
   }
 
   const kept = [];
   const skipped = [];
   const stats = emptyStats();
   for (const chunk of chunks) {
-    const reason = chunk?.file ? fileFirstReason.get(chunk.file) : null;
+    const file = chunkPath(chunk);
+    const reason = file ? fileFirstReason.get(file) : null;
     if (reason) {
       skipped.push(chunk);
       stats[reason]++;
@@ -119,8 +143,8 @@ export function applyIndexingChunkPolicy(chunks, options = {}) {
       kept.push(chunk);
     }
   }
-  stats.skippedFiles = new Set(skipped.map((c) => c.file).filter(Boolean)).size;
-  stats.keptFiles = new Set(kept.map((c) => c.file).filter(Boolean)).size;
+  stats.skippedFiles = new Set(skipped.map(chunkPath).filter(Boolean)).size;
+  stats.keptFiles = new Set(kept.map(chunkPath).filter(Boolean)).size;
   return { kept, skipped, stats };
 }
 
@@ -136,5 +160,6 @@ function emptyStats() {
 
 export const _internals = {
   GENERATED_MARKERS,
+  chunkPath,
   resetCache,
 };