sweet-search 2.6.8 → 2.6.9

package/core/graph/graph-extractor.js

@@ -2351,7 +2351,28 @@ function resolveThrows(exceptionName, entityByName) {
  * Insert entities and relationships into database
  * Uses better-sqlite3 (sync API, no .free() needed)
  */
-export function insertGraph(db, entities, relationships, hasFts5 = false) {
+/**
+ * Rebuild + optimize the external-content FTS5 mirrors from the entities
+ * table. 'rebuild' reconstructs the whole FTS index from current content, so
+ * only the LAST rebuild before any FTS read matters — batch loops should
+ * pass { syncFts: false } to insertGraph and call this once at the end.
+ */
+export function rebuildGraphFts(db) {
+  try {
+    db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('rebuild')`);
+    db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('rebuild')`);
+    console.log('  FTS5 indexes rebuilt (porter + trigram)');
+
+    // Best-effort post-build compaction for faster reads.
+    db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('optimize')`);
+    db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('optimize')`);
+    console.log('  FTS5 indexes optimized (segments merged)');
+  } catch (err) {
+    // FTS5 rebuild/optimize failed, ignore
+  }
+}
+
+export function insertGraph(db, entities, relationships, hasFts5 = false, { syncFts = true } = {}) {
   // Insert entities with HCGS hierarchy support
   // Includes signature_hash for collision-proof backup/restore
   const entityStmt = db.prepare(`
@@ -2481,19 +2502,8 @@ export function insertGraph(db, entities, relationships, hasFts5 = false) {
   // }
 
   // Rebuild FTS indexes if available
-  if (hasFts5) {
-    try {
-      db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('rebuild')`);
-      db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('rebuild')`);
-      console.log('  FTS5 indexes rebuilt (porter + trigram)');
-
-      // Best-effort post-build compaction for faster reads.
-      db.exec(`INSERT INTO entities_fts(entities_fts) VALUES('optimize')`);
-      db.exec(`INSERT INTO entities_trigram(entities_trigram) VALUES('optimize')`);
-      console.log('  FTS5 indexes optimized (segments merged)');
-    } catch (err) {
-      // FTS5 rebuild/optimize failed, ignore
-    }
+  if (hasFts5 && syncFts) {
+    rebuildGraphFts(db);
   }
 }
 

package/core/graph/hcgs-generator.js

@@ -428,9 +428,10 @@ async function generateSummariesForEntities(entityIds, options = {}) {
     const dbWrite = new Database(dbPath);
     const stmt = dbWrite.prepare(`UPDATE entities SET summary = NULL WHERE id = ?`);
 
-    for (const entity of entities) {
-      stmt.run(entity.id);
-    }
+    const clearSummaries = dbWrite.transaction((rows) => {
+      for (const entity of rows) stmt.run(entity.id);
+    });
+    clearSummaries(entities);
 
     dbWrite.close();
 

package/core/incremental-indexing/application/production-reconciler.mjs

@@ -34,6 +34,7 @@ import { floatToBinary, normalizedFloatToInt8, truncateForHNSW } from '../../inf
 import { extractSparseGramDeltaRecord } from '../../infrastructure/native-sparse-gram.js';
 import { migrateEntitiesSchema, migrateRelationshipsSchema } from '../infrastructure/schema-migrations.mjs';
 import { readMaintenanceState as readMaintenanceStateFromArtifacts } from '../infrastructure/maintenance-state-reader.mjs';
+import { prepareCached } from '../../infrastructure/db-utils.js';
 
 const DIRTY_QUEUE = 'index-maintainer-queue.jsonl';
 const PROCESSING_QUEUE = 'index-maintainer-queue.processing.jsonl';
@@ -270,9 +271,9 @@ async function enrichChunksFromGraph(chunks, stateDir, tickCtx = null) {
     ownConn = true;
   }
   try {
-    const entityStmt = db.prepare('SELECT type, name, start_line, end_line FROM entities WHERE file_path = ? AND epoch_retired IS NULL ORDER BY start_line ASC');
-    const fileEntityStmt = db.prepare('SELECT id FROM entities WHERE file_path = ? AND logical_entity_id = ? AND epoch_retired IS NULL ORDER BY epoch_written DESC LIMIT 1');
-    const importStmt = db.prepare("SELECT DISTINCT target_name FROM relationships WHERE source_id = ? AND type IN ('imports', 'plainImport') AND epoch_retired IS NULL ORDER BY target_name");
+    const entityStmt = prepareCached(db, 'SELECT type, name, start_line, end_line FROM entities WHERE file_path = ? AND epoch_retired IS NULL ORDER BY start_line ASC');
+    const fileEntityStmt = prepareCached(db, 'SELECT id FROM entities WHERE file_path = ? AND logical_entity_id = ? AND epoch_retired IS NULL ORDER BY epoch_written DESC LIMIT 1');
+    const importStmt = prepareCached(db, "SELECT DISTINCT target_name FROM relationships WHERE source_id = ? AND type IN ('imports', 'plainImport') AND epoch_retired IS NULL ORDER BY target_name");
     for (const chunk of chunks) {
       const file = chunk.file || chunk.metadata?.relative_path;
       const symbol = chunk.metadata?.symbol;
@@ -376,6 +377,23 @@ class ProductionReconcileAdapter {
     // when the flag is on; null when disabled.
     this._cutoffCache = null;
     this._cutoffDirty = false;
+    // merkle-state.json is written exactly once per tick (persistManifest);
+    // every other reader gets one shared parse. `undefined` = not read yet,
+    // `null` = missing/corrupt file.
+    this._merkleCache = undefined;
+  }
+
+  /**
+   * Shared read of merkle-state.json — parsed once per adapter lifetime and
+   * invalidated by persistManifest (the only writer). All consumers treat the
+   * object as read-only.
+   * @returns {object|null}
+   */
+  _merkleRead() {
+    if (this._merkleCache === undefined) {
+      this._merkleCache = readJson(path.join(this.stateDir, MERKLE_STATE), null);
+    }
+    return this._merkleCache;
   }
 
   progress(phase) {
@@ -697,7 +715,7 @@ class ProductionReconcileAdapter {
     // dropped if it was never indexed, and retired if it was (so the index
     // converges to a fresh full rebuild). Existence + shape + size are sync;
     // gitignore is ONE batched check over the admissible candidates.
-    const merkle = readJson(path.join(this.stateDir, MERKLE_STATE), { files: {} }).files || {};
+    const merkle = (this._merkleRead() ?? { files: {} }).files || {};
     const info = rels.map((rel) => {
       const abs = path.join(this.projectRoot, rel);
       const exists = fs.existsSync(abs);
@@ -736,7 +754,7 @@ class ProductionReconcileAdapter {
   async hashFile(file) {
     const rel = typeof file === 'string' ? file : file.path;
     const abs = path.join(this.projectRoot, rel);
-    const merkle = readJson(path.join(this.stateDir, MERKLE_STATE), { files: {} });
+    const merkle = this._merkleRead() ?? { files: {} };
     // Deleted on disk, or flagged inadmissible by readDirtySet (a previously
     // indexed file that became excluded/oversized/gitignored): retire it so all
     // tiers tombstone and the merkle entry is dropped.
@@ -784,7 +802,7 @@ class ProductionReconcileAdapter {
       ownConn = true;
     }
     try {
-      const oldRows = db.prepare('SELECT rowid, id, logical_entity_id, signature_hash FROM entities WHERE file_path = ? AND epoch_retired IS NULL').all(rel);
+      const oldRows = prepareCached(db, 'SELECT rowid, id, logical_entity_id, signature_hash FROM entities WHERE file_path = ? AND epoch_retired IS NULL').all(rel);
       const oldByLogical = new Map(oldRows.map((r) => [r.logical_entity_id || r.id, r]));
       const oldIds = oldRows.map((r) => r.id);
       const extractor = new GraphExtractor();
@@ -811,7 +829,7 @@ class ProductionReconcileAdapter {
       let tombstone = 0;
       const liveIdFor = new Map();
       const tx = db.transaction(() => {
-        const retireEntity = db.prepare('UPDATE entities SET epoch_retired = ?, stale_since = COALESCE(stale_since, ?) WHERE id = ? AND epoch_retired IS NULL');
+        const retireEntity = prepareCached(db, 'UPDATE entities SET epoch_retired = ?, stale_since = COALESCE(stale_since, ?) WHERE id = ? AND epoch_retired IS NULL');
         const retireRel = oldIds.length > 0
           ? db.prepare(`UPDATE relationships SET epoch_retired = ? WHERE source_id IN (${oldIds.map(() => '?').join(',')}) AND epoch_retired IS NULL`)
           : null;
@@ -913,7 +931,7 @@ class ProductionReconcileAdapter {
           // tier write. The merkle still records the new content hash so the
           // file is not re-queued forever.
           const prevTouched = this.touched.get(rel) || {};
-          const prevChunkIds = readJson(path.join(this.stateDir, MERKLE_STATE), { files: {} }).files?.[rel]?.chunkIds || prevTouched.chunkIds || [];
+          const prevChunkIds = (this._merkleRead() ?? { files: {} }).files?.[rel]?.chunkIds || prevTouched.chunkIds || [];
           this.touched.set(rel, { ...prevTouched, hash: hashes, chunkIds: prevChunkIds, content: hashes.content });
           if (ctx) ctx.persistedFiles.add(rel);
           return {
@@ -1170,7 +1188,7 @@ class ProductionReconcileAdapter {
    * @returns {number}
    */
   _publishedEpoch() {
-    const merkle = readJson(path.join(this.stateDir, MERKLE_STATE), {});
+    const merkle = this._merkleRead() ?? {};
     return Number.isInteger(merkle?.epoch) ? merkle.epoch : -1;
   }
 
@@ -1225,6 +1243,8 @@ class ProductionReconcileAdapter {
     merkle.epoch = manifest.epoch;
     merkle.stats = { ...(merkle.stats || {}), totalFiles: Object.keys(merkle.files).length };
     safeWriteJson(merklePath, merkle);
+    // Invalidate the shared read cache — this is the one merkle writer.
+    this._merkleCache = undefined;
     // E.6: persist the updated chunk-cutoff cache once per tick (after the
     // merkle advances). Best-effort; a failure only costs a redundant re-embed.
     if (this._cutoffCache && this._cutoffDirty) {

package/core/incremental-indexing/domain/encoder-input.mjs

@@ -192,13 +192,17 @@ function firstSafeRelativePath(...candidates) {
  * carried inside the encoder-input dependency registry; see
  * `core/incremental-indexing/domain/encoder-deps.mjs`).
  *
+ * Pass `{ includeDedup: false }` when the caller never persists the dedup
+ * fingerprint (the full-indexing insert path) to skip that hash entirely.
+ *
  * @param {object} chunk
- * @returns {{chunk_text_hash:string, embedding_input_hash:string, li_input_hash:string, metadata_fingerprint:string, dedup_fingerprint:string}}
+ * @param {{includeDedup?: boolean}} [opts]
+ * @returns {{chunk_text_hash:string, embedding_input_hash:string, li_input_hash:string, metadata_fingerprint:string, dedup_fingerprint?:string}}
  */
-export function chunkInputHashes(chunk) {
+export function chunkInputHashes(chunk, { includeDedup = true } = {}) {
   const text = chunk?.content || chunk?.text || '';
   const meta = chunk?.metadata || {};
-  return {
+  const hashes = {
     chunk_text_hash: contentHashSync(text),
     embedding_input_hash: denseInputHash(chunk),
     li_input_hash: liInputHash(chunk),
@@ -214,8 +218,9 @@ export function chunkInputHashes(chunk) {
       policy_embed: EMBED_TEXT_POLICY_VERSION,
       policy_li: LI_INPUT_POLICY_VERSION,
     })),
-    dedup_fingerprint: dedupFingerprint(chunk),
   };
+  if (includeDedup) hashes.dedup_fingerprint = dedupFingerprint(chunk);
+  return hashes;
 }
 
 /**

package/core/indexing/ast-chunker.js

@@ -151,12 +151,12 @@ function buildEmbeddingText({ variant: variantOverride, content, relativePath, l
   const pathLine = relativePath ? `# ${relativePath}` : null;
   const parentLine = hierarchyInfo?.parentSymbol
     ? `# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}` : null;
-  // Ruby method chunks keep metadata anchors, but omit method-name header
-  // text so top-level Ruby snippets stay aligned with the pre-method-boundary
-  // embedding surface.
-  const isRubyMethodChunk = language === 'ruby'
-    && (chunkType === 'method' || chunkType === 'singleton_method');
-  const symbolLine = (symbol && symbol !== 'unknown' && !isRubyMethodChunk)
+  // Ruby method chunks keep metadata anchors, but omit method-name header
+  // text so top-level Ruby snippets stay aligned with the pre-method-boundary
+  // embedding surface.
+  const isRubyMethodChunk = language === 'ruby'
+    && (chunkType === 'method' || chunkType === 'singleton_method');
+  const symbolLine = (symbol && symbol !== 'unknown' && !isRubyMethodChunk)
     ? `# ${chunkType}: ${symbol}` : null;
   const langLine = (language && language !== 'text')
     ? `# Language: ${language}` : null;
@@ -293,10 +293,10 @@ function buildLiText({ content, relativePath, language, chunkType, symbol, hiera
   if (hierarchyInfo?.parentSymbol) {
     lines.push(`# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}`);
   }
-  // Mirror the Ruby method header carve-out used by embedding_text.
-  const isRubyMethodChunk = language === 'ruby'
-    && (chunkType === 'method' || chunkType === 'singleton_method');
-  if (symbol && symbol !== 'unknown' && !isRubyMethodChunk) {
+  // Mirror the Ruby method header carve-out used by embedding_text.
+  const isRubyMethodChunk = language === 'ruby'
+    && (chunkType === 'method' || chunkType === 'singleton_method');
+  if (symbol && symbol !== 'unknown' && !isRubyMethodChunk) {
     lines.push(`# ${chunkType}: ${symbol}`);
   }
   // Mirror embedding_text: surface sibling symbol names so the LI MaxSim
@@ -391,27 +391,27 @@ export class ASTChunker {
     const tsChunks = await provider.parseFileToChunks(content, langInfo.id);
     if (!tsChunks || tsChunks.length === 0) return null;
 
-    return tsChunks.map(chunk => {
-      const isTopLevelRubyMethod = langInfo.id === 'ruby'
-        && (chunk.type === 'method' || chunk.type === 'singleton_method')
-        && !chunk.parentSymbol;
-
-      return this.buildChunk(
-        chunk.text, filePath, langInfo.id,
-        isTopLevelRubyMethod ? 'code' : chunk.type,
-        isTopLevelRubyMethod ? null : chunk.name,
-        chunk.startLine, chunk.endLine,
-        {
-          chunkId: chunk.chunkId,
-          parentChunkId: chunk.parentChunkId,
-          parentSymbol: chunk.parentSymbol,
-          parentType: chunk.parentType,
-          signature: chunk.signature || null,
-          additionalSymbols: chunk.additionalSymbols || null,
-        }
-      );
-    });
-  }
+    return tsChunks.map(chunk => {
+      const isTopLevelRubyMethod = langInfo.id === 'ruby'
+        && (chunk.type === 'method' || chunk.type === 'singleton_method')
+        && !chunk.parentSymbol;
+
+      return this.buildChunk(
+        chunk.text, filePath, langInfo.id,
+        isTopLevelRubyMethod ? 'code' : chunk.type,
+        isTopLevelRubyMethod ? null : chunk.name,
+        chunk.startLine, chunk.endLine,
+        {
+          chunkId: chunk.chunkId,
+          parentChunkId: chunk.parentChunkId,
+          parentSymbol: chunk.parentSymbol,
+          parentType: chunk.parentType,
+          signature: chunk.signature || null,
+          additionalSymbols: chunk.additionalSymbols || null,
+        }
+      );
+    });
+  }
 
   parseBraceBasedFile(filePath, content, language, patterns, comment, multiLine) {
     const chunks = [];
@@ -942,6 +942,7 @@ export class ASTChunker {
     // Production embedding text. With the default variant (current) this
     // is byte-identical to shipped v6.2; under a research-only variant
     // (SWEET_SEARCH_EMBED_TEXT_VARIANT) it produces the experimental form.
+    const activeVariant = getEmbedTextVariant();
     const embedding_text = buildEmbeddingText({
       content,
       relativePath,
@@ -954,17 +955,19 @@ export class ASTChunker {
     // Research isolation: always build the shipped (current-variant)
     // form alongside, so the LI stage can read a canonical input even
     // when an R1 experiment is active on embedding_text. In production
-    // (variant=current) this is the same content as embedding_text and
-    // pickLiInput's preference for li_greedy_text changes nothing.
-    const li_greedy_text = buildEmbeddingText({
-      variant: 'current',
-      content,
-      relativePath,
-      language,
-      chunkType,
-      symbol,
-      hierarchyInfo,
-    });
+    // (variant=current) it IS the embedding_text — reuse the string
+    // instead of rebuilding it.
+    const li_greedy_text = activeVariant === 'current'
+      ? embedding_text
+      : buildEmbeddingText({
+        variant: 'current',
+        content,
+        relativePath,
+        language,
+        chunkType,
+        symbol,
+        hierarchyInfo,
+      });
 
     // Build LI text via the v6.2 builder (language-conditioned path
     // policy). See buildLiText() docstring for rationale.
@@ -1034,26 +1037,26 @@ export class ASTChunker {
     const parts = [];
     parts.push(`# ${chunk.metadata.path}`);
 
-    const isRubyMethodChunk = chunk.metadata.language === 'ruby'
-      && (chunk.metadata.chunk_type === 'method'
-        || chunk.metadata.chunk_type === 'singleton_method');
-    const hasOnlySelfScope = scopeChain
-      && scopeChain.length === 1
-      && scopeChain[0] === chunk.metadata.symbol
-      && !chunk.metadata.parent_symbol;
-
-    if (scopeChain && scopeChain.length > 0 && !(isRubyMethodChunk && hasOnlySelfScope)) {
-      parts.push(`# Scope: ${scopeChain.join(' > ')}`);
-    } else if (chunk.metadata.parent_symbol) {
-      // Preserve cAST parent context when no scope chain from code graph
-      parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
-    }
-    // Keep Ruby method metadata, but avoid injecting the method name into
-    // the production embedding/LI text. Top-level Ruby method chunks also
-    // skip self-only scope above.
-    if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown' && !isRubyMethodChunk) {
-      parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
-    }
+    const isRubyMethodChunk = chunk.metadata.language === 'ruby'
+      && (chunk.metadata.chunk_type === 'method'
+        || chunk.metadata.chunk_type === 'singleton_method');
+    const hasOnlySelfScope = scopeChain
+      && scopeChain.length === 1
+      && scopeChain[0] === chunk.metadata.symbol
+      && !chunk.metadata.parent_symbol;
+
+    if (scopeChain && scopeChain.length > 0 && !(isRubyMethodChunk && hasOnlySelfScope)) {
+      parts.push(`# Scope: ${scopeChain.join(' > ')}`);
+    } else if (chunk.metadata.parent_symbol) {
+      // Preserve cAST parent context when no scope chain from code graph
+      parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
+    }
+    // Keep Ruby method metadata, but avoid injecting the method name into
+    // the production embedding/LI text. Top-level Ruby method chunks also
+    // skip self-only scope above.
+    if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown' && !isRubyMethodChunk) {
+      parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
+    }
 
     if (chunk.metadata.additional_symbols && chunk.metadata.additional_symbols.length > 0) {
       parts.push(`# Additional: ${chunk.metadata.additional_symbols.join(', ')}`);

package/core/indexing/indexer-build.js

@@ -9,7 +9,7 @@ import { existsSync } from 'fs';
 import path from 'path';
 
 import { DB_PATHS, EMBEDDING_CONFIG, PROJECT_ROOT } from '../infrastructure/config/index.js';
-import { GraphExtractor, createGraphSchema, insertGraph } from '../graph/graph-extractor.js';
+import { GraphExtractor, createGraphSchema, insertGraph, rebuildGraphFts } from '../graph/graph-extractor.js';
 import { resolveRelationshipTargets } from '../graph/relationship-resolver.js';
 import { populatePageRankColumn } from '../graph/structural-pagerank.js';
 import { getEmbeddings, getModelInfo } from '../embedding/embedding-service.js';
@@ -153,6 +153,7 @@ export async function buildCodeGraph(files, dryRun = false) {
   let errors = 0;
   let totalEntities = 0;
   let totalRelationships = 0;
+  let graphFlushed = false;
 
   for (let i = 0; i < files.length; i++) {
     try {
@@ -170,10 +171,14 @@ export async function buildCodeGraph(files, dryRun = false) {
       errors++;
     }
 
-    // Flush batch every GRAPH_BATCH_SIZE files or at the end
+    // Flush batch every GRAPH_BATCH_SIZE files or at the end. FTS5 sync is
+    // deferred to ONE rebuild after the loop — 'rebuild' reconstructs the
+    // whole index from the entities table, so per-batch rebuilds were
+    // O(entities × batches) work discarded by the next batch.
     if ((i + 1) % GRAPH_BATCH_SIZE === 0 || i === files.length - 1) {
       if (entityBatch.length > 0 || relBatch.length > 0) {
-        insertGraph(db, entityBatch, relBatch, hasFts5);
+        insertGraph(db, entityBatch, relBatch, hasFts5, { syncFts: false });
+        graphFlushed = true;
         totalEntities += entityBatch.length;
         totalRelationships += relBatch.length;
         entityBatch = [];
@@ -186,6 +191,13 @@ export async function buildCodeGraph(files, dryRun = false) {
     }
   }
 
+  // Single FTS5 rebuild at the exact point the last per-batch rebuild used to
+  // run (before relationship resolution, whose entity updates were never
+  // reflected in FTS).
+  if (graphFlushed && hasFts5) {
+    rebuildGraphFts(db);
+  }
+
   log(`\n✓ Extracted ${totalEntities} entities, ${totalRelationships} relationships`, 'green');
   if (errors > 0) {
     log(`⚠ ${errors} files had errors`, 'yellow');
@@ -360,7 +372,9 @@ function annotateChunksForVectorInsert(chunks) {
       const idx = indices[i];
       annotations[idx] = {
         ...ids[i],
-        hashes: chunkInputHashes(chunks[idx]),
+        // Full indexing never persists dedup_fingerprint (only the reconcile
+        // delta writer does) — skip that hash.
+        hashes: chunkInputHashes(chunks[idx], { includeDedup: false }),
       };
     }
   }

package/core/indexing/indexer-phases.js

@@ -10,7 +10,7 @@ import path from 'path';
 import { DB_PATHS, PROJECT_ROOT, EMBEDDING_CONFIG, HCGS_CONFIG } from '../infrastructure/config/index.js';
 import { getChangedFiles, updateState, getStats as getIncrementalStats, updatePhaseProgress, markPhaseComplete, clearPhaseProgress } from './incremental-tracker.js';
 import { backupSummaries, restoreSummaries, markForRegeneration } from '../graph/summary-manager.js';
-import { colors, log, logProgress, logError, discoverFiles, readFilesFromStdin, atomicSwapDatabase } from './indexer-utils.js';
+import { colors, log, logProgress, logError, discoverFiles, readFilesFromStdin, atomicSwapDatabase, shouldStreamVectors } from './indexer-utils.js';
 import { buildCodeGraph, buildVectorIndex, chunkFiles } from './indexer-build.js';
 import { runDedupPhase, formatDedupSummary } from './dedup/dedup-phase.js';
 import { DEDUP_CONFIG } from '../infrastructure/config/index.js';
@@ -420,15 +420,24 @@ export async function buildVectorsAndArtifactsPhase(options = {}) {
   // model. For large full rebuilds we instead spill chunks to disk and embed/LI
   // in bounded windows (see streaming-vectors.js) so peak heap is O(window).
   //
-  // Gated by file count so small repos + incremental runs keep the original
-  // in-memory path byte-for-byte (benchmark indexes unaffected). Auto-selected,
-  // no opt-in flag; SWEET_SEARCH_STREAM_VECTORS=0 forces the legacy path and
-  // SWEET_SEARCH_STREAM_MIN_FILES tunes the threshold.
-  const streamMinFiles = Number(process.env.SWEET_SEARCH_STREAM_MIN_FILES) || 5000;
-  const useStreaming = !dryRun
-    && fullReindex
-    && filesToIndex.length >= streamMinFiles
-    && process.env.SWEET_SEARCH_STREAM_VECTORS !== '0';
+  // Gated by file count OR total admitted source bytes (see shouldStreamVectors
+  // in indexer-utils.js) so small repos + incremental runs keep the original
+  // in-memory path byte-for-byte (benchmark indexes unaffected). The byte
+  // trigger catches few-files-huge-bytes repos (amalgamations, vendored blobs)
+  // that OOM the in-memory path while staying under the file gate.
+  // Auto-selected, no opt-in flag; SWEET_SEARCH_STREAM_VECTORS=0 forces the
+  // legacy path, SWEET_SEARCH_STREAM_MIN_FILES / SWEET_SEARCH_STREAM_MIN_BYTES
+  // tune the thresholds.
+  const streamingDecision = await shouldStreamVectors({ filesToIndex, dryRun, fullReindex });
+  const useStreaming = streamingDecision.useStreaming;
+  if (streamingDecision.reason === 'bytes') {
+    log(
+      `  Streaming vectors: ${filesToIndex.length} files total ` +
+      `${(streamingDecision.totalBytes / 1048576).toFixed(0)}+ MB >= ` +
+      `${(streamingDecision.thresholdBytes / 1048576).toFixed(0)} MB (bounded memory)`,
+      'dim'
+    );
+  }
 
   // The in-memory path pre-chunks up front so both vector + LI encoders share
   // one chunk list. The streaming path does its own windowed chunking + dedup,
@@ -814,20 +823,32 @@ export async function updateIncrementalStatePhase(options = {}) {
     log('\nIncremental state updated', 'green');
   } else if (fullReindex) {
     const hashes = {};
-    for (const file of allFiles) {
-      try {
-        const fullPath = path.join(PROJECT_ROOT, file);
-        const [content, stat] = await Promise.all([
-          fs.readFile(fullPath),
-          fs.stat(fullPath, { bigint: true }).catch(() => null),
-        ]);
-        hashes[file] = {
-          hash: contentHashSync(content),
-          size: stat ? stat.size.toString() : null,
-          mtime_ns: stat ? stat.mtimeNs.toString() : null,
-          inode: stat ? stat.ino.toString() : null,
-        };
-      } catch (e) { /* skip */ }
+    // Batched read+stat (order-preserving assignment — key insertion order,
+    // and therefore the serialized state file, matches the sequential loop).
+    // Mirrors the H6 batching in incremental-tracker.getChangedFiles.
+    const HASH_BATCH = 100;
+    for (let i = 0; i < allFiles.length; i += HASH_BATCH) {
+      const batch = allFiles.slice(i, i + HASH_BATCH);
+      const results = await Promise.all(batch.map(async (file) => {
+        try {
+          const fullPath = path.join(PROJECT_ROOT, file);
+          const [content, stat] = await Promise.all([
+            fs.readFile(fullPath),
+            fs.stat(fullPath, { bigint: true }).catch(() => null),
+          ]);
+          return {
+            hash: contentHashSync(content),
+            size: stat ? stat.size.toString() : null,
+            mtime_ns: stat ? stat.mtimeNs.toString() : null,
+            inode: stat ? stat.ino.toString() : null,
+          };
+        } catch (e) {
+          return null; // skip
+        }
+      }));
+      for (let j = 0; j < batch.length; j++) {
+        if (results[j]) hashes[batch[j]] = results[j];
+      }
     }
     await updateState(hashes, {
       totalChunks: vectorStats.chunks,

package/core/indexing/indexer-utils.js

@@ -395,16 +395,22 @@ export async function discoverFiles(options = {}) {
 
   const files = [];
   let oversized = 0;
-  for (const file of allFiles) {
-    try {
-      const stat = await fs.stat(path.join(projectRoot, file));
+  // Stat in batches (order-preserving) instead of one serialized await per
+  // file; results are consumed in the original allFiles order.
+  const STAT_BATCH = 200;
+  for (let i = 0; i < allFiles.length; i += STAT_BATCH) {
+    const batch = allFiles.slice(i, i + STAT_BATCH);
+    const stats = await Promise.all(
+      batch.map((file) => fs.stat(path.join(projectRoot, file)).catch(() => null)),
+    );
+    for (let j = 0; j < batch.length; j++) {
+      const stat = stats[j];
+      if (!stat) continue; // File disappeared between glob and stat
       if (stat.size > maxFileSize) {
         oversized++;
       } else {
-        files.push(file);
+        files.push(batch[j]);
       }
-    } catch {
-      // File disappeared between glob and stat
     }
   }
 
@@ -426,3 +432,87 @@ export async function discoverFiles(options = {}) {
 
   return files;
 }
+
+// =============================================================================
+// STREAMING-VECTORS GATE
+// =============================================================================
+
+/**
+ * Sum on-disk sizes of `files` (relative to `projectRoot`), stopping as soon
+ * as the running total crosses `stopAt`. The streaming gate only needs to
+ * know whether the total crosses the threshold, not the exact figure, so on
+ * large-byte repos this exits after a fraction of the stats.
+ */
+export async function sumFileSizesUpTo(files, stopAt, projectRoot = PROJECT_ROOT) {
+  let total = 0;
+  for (const file of files) {
+    try {
+      const stat = await fs.stat(path.isAbsolute(file) ? file : path.join(projectRoot, file));
+      total += stat.size;
+      if (total >= stopAt) return total;
+    } catch {
+      // File disappeared between discovery and this gate — skip it.
+    }
+  }
+  return total;
+}
+
+/**
+ * Decide whether a full rebuild should take the bounded-memory streaming
+ * vectors path (see streaming-vectors.js) instead of the in-memory path.
+ *
+ * Two independent triggers, either is sufficient:
+ *   - file count ≥ SWEET_SEARCH_STREAM_MIN_FILES (default 5000)
+ *   - total admitted source bytes ≥ SWEET_SEARCH_STREAM_MIN_BYTES
+ *     (default 512 MB; set to 0 to disable the byte trigger)
+ *
+ * The byte trigger exists because peak heap on the in-memory path scales
+ * with the chunk corpus (≈ source bytes), not file count: a repo with
+ * few-but-huge files (amalgamations, vendored/generated blobs, extreme
+ * duplication) can OOM the default ~4 GB heap while staying far under the
+ * file gate.
+ *
+ * The 512 MB default is deliberately conservative so the trigger only fires
+ * where the in-memory path would fail outright, never where it would merely
+ * be tight. The one MEASURED failure at this scale is libsql (596 MB
+ * admitted source; the in-memory path needed a 9.6+ GB heap); the wider
+ * crash zone is ESTIMATED — not measured — to start around ~200 MB of
+ * admitted source on a default ~4 GB heap. A repo the byte trigger newly
+ * moves to streaming was therefore not getting a usable in-memory index at
+ * all, so streaming strictly improves on a crash; every repo below the
+ * threshold keeps the byte-for-byte-identical in-memory path and identical
+ * retrieval behaviour.
+ *
+ * Sizes are only stat'd when the (free) count trigger hasn't already fired,
+ * so the byte check costs at most one stat per file on sub-threshold repos.
+ * The re-stat duplicates work discoverFiles already did for its size cap —
+ * accepted deliberately: threading sizes through would change discoverFiles'
+ * public return shape, and ≤5000 extra stats on a full rebuild is noise next
+ * to chunking + embedding.
+ *
+ * @returns {Promise<{useStreaming: boolean, reason?: 'files'|'bytes', totalBytes?: number, thresholdBytes?: number}>}
+ */
+export async function shouldStreamVectors({ filesToIndex, dryRun, fullReindex, projectRoot = PROJECT_ROOT, env = process.env }) {
+  if (dryRun || !fullReindex || env.SWEET_SEARCH_STREAM_VECTORS === '0') {
+    return { useStreaming: false };
+  }
+  const streamMinFiles = Number(env.SWEET_SEARCH_STREAM_MIN_FILES) || 5000;
+  if (filesToIndex.length >= streamMinFiles) {
+    return { useStreaming: true, reason: 'files' };
+  }
+  // Unset/empty/invalid → 512 MB default; an explicit 0 (or negative)
+  // disables the byte trigger alone, leaving the count trigger active.
+  const rawMinBytes = env.SWEET_SEARCH_STREAM_MIN_BYTES;
+  const parsedMinBytes = (rawMinBytes === undefined || rawMinBytes === '') ? NaN : Number(rawMinBytes);
+  const streamMinBytes = Number.isFinite(parsedMinBytes)
+    ? (parsedMinBytes > 0 ? parsedMinBytes : Infinity)
+    : 512 * 1024 * 1024;
+  if (!Number.isFinite(streamMinBytes)) {
+    return { useStreaming: false };
+  }
+  const totalBytes = await sumFileSizesUpTo(filesToIndex, streamMinBytes, projectRoot);
+  if (totalBytes >= streamMinBytes) {
+    return { useStreaming: true, reason: 'bytes', totalBytes, thresholdBytes: streamMinBytes };
+  }
+  return { useStreaming: false };
+}