sweet-search 2.6.7 → 2.6.9

package/core/indexing/indexing-file-policy.js

@@ -8,7 +8,7 @@
  */
 
 import { existsSync, readFileSync } from 'fs';
-import { minimatch } from 'minimatch';
+import { Minimatch } from 'minimatch';
 import { loadProjectConfig } from '../infrastructure/config/index.js';
 
 const MM_OPTS = { dot: true, nocase: false };
@@ -54,6 +54,31 @@ function getExcludes(projectRoot) {
 function resetCache() {
   _excludesByRoot.clear();
   _cachedExtraPatterns = null;
+  _excludeMatchersByRoot.clear();
+  _cachedExtraMatchers = null;
+}
+
+// Precompiled Minimatch instances — `minimatch(p, g, opts)` recompiles the
+// glob to a regex on every call; `new Minimatch(g, opts).match(p)` is the
+// documented equivalent with the compile amortized.
+const _excludeMatchersByRoot = new Map();
+
+function getExcludeMatchers(projectRoot) {
+  const key = projectRoot || '__cwd__';
+  let cached = _excludeMatchersByRoot.get(key);
+  if (cached) return cached;
+  cached = getExcludes(projectRoot)
+    .filter((g) => typeof g === 'string')
+    .map((g) => new Minimatch(g, MM_OPTS));
+  _excludeMatchersByRoot.set(key, cached);
+  return cached;
+}
+
+let _cachedExtraMatchers = null;
+function getExtraMatchers() {
+  if (_cachedExtraMatchers !== null) return _cachedExtraMatchers;
+  _cachedExtraMatchers = loadExtraPatternsFromFile().map((g) => new Minimatch(g, MM_OPTS));
+  return _cachedExtraMatchers;
 }
 
 const GENERATED_MARKERS = [
@@ -87,13 +112,11 @@ function loadExtraPatternsFromFile() {
 export function isExcludedByConfig(filePath, projectRoot) {
   if (!filePath) return false;
   const p = normalizePath(filePath);
-  const excludes = getExcludes(projectRoot);
-  for (const g of excludes) {
-    if (typeof g === 'string' && minimatch(p, g, MM_OPTS)) return true;
+  for (const m of getExcludeMatchers(projectRoot)) {
+    if (m.match(p)) return true;
   }
-  const extras = loadExtraPatternsFromFile();
-  for (const g of extras) {
-    if (minimatch(p, g, MM_OPTS)) return true;
+  for (const m of getExtraMatchers()) {
+    if (m.match(p)) return true;
   }
   return false;
 }

package/core/infrastructure/code-graph-repository.js

@@ -10,7 +10,7 @@
 
 import Database from 'better-sqlite3';
 import { existsSync, statSync } from 'fs';
-import { applyReadPragmas } from './db-utils.js';
+import { applyReadPragmas, prepareCached } from './db-utils.js';
 import { readAdjacentManifest, resolveManifestCodeGraphPath, sqlAliasPrefix } from './code-graph-visibility.js';
 
 export class CodeGraphRepository {
@@ -72,7 +72,7 @@ export class CodeGraphRepository {
   }
 
   _hasColumns(db, table, columns) {
-    const rows = db.prepare(`PRAGMA table_info(${table})`).all();
+    const rows = prepareCached(db, `PRAGMA table_info(${table})`).all();
     const names = new Set(rows.map((row) => row.name));
     return columns.every((column) => names.has(column));
   }
@@ -138,7 +138,7 @@ export class CodeGraphRepository {
     const db = this._open();
     if (!db) return null;
     try {
-      const row = db.prepare(`
+      const row = prepareCached(db, `
         SELECT id, name, type, start_line, end_line, parent_class
         FROM entities
         WHERE file_path = ?
@@ -207,7 +207,7 @@ export class CodeGraphRepository {
       // in the chunk range (catches entities like a multi-chunk struct whose
       // declaration starts in this chunk but body extends beyond — e.g. gin
       // Engine struct at gin.go:92-189 split across chunks 92-133 and 134-189).
-      const containedRows = db.prepare(`
+      const containedRows = prepareCached(db, `
         SELECT name, type, start_line, end_line FROM entities
         WHERE file_path = ? AND start_line >= ? AND end_line <= ? AND ${this._entityVisibilitySql(db)}
         ORDER BY (end_line - start_line) DESC
@@ -219,7 +219,7 @@ export class CodeGraphRepository {
           return { name: row.name, type: row.type, startLine: row.start_line, endLine: row.end_line };
         }
       }
-      const startsInRows = db.prepare(`
+      const startsInRows = prepareCached(db, `
         SELECT name, type, start_line, end_line FROM entities
         WHERE file_path = ? AND start_line >= ? AND start_line <= ? AND ${this._entityVisibilitySql(db)}
         ORDER BY (end_line - start_line) DESC
@@ -245,7 +245,7 @@ export class CodeGraphRepository {
     const db = this._open();
     if (!db) return null;
     try {
-      const row = db.prepare(`
+      const row = prepareCached(db, `
         SELECT id, name, type, start_line, end_line, parent_class
         FROM entities
         WHERE file_path = ?
@@ -289,7 +289,7 @@ export class CodeGraphRepository {
     const db = this._open();
     if (!db) return [];
     try {
-      const rows = db.prepare(`
+      const rows = prepareCached(db, `
         SELECT id, name, type, start_line, end_line, parent_class
         FROM entities
         WHERE file_path = ?
@@ -322,7 +322,7 @@ export class CodeGraphRepository {
     const db = this._open();
     if (!db) return null;
     try {
-      const row = db.prepare(`
+      const row = prepareCached(db, `
         SELECT id, name, type, file_path, start_line, end_line, parent_class
         FROM entities
         WHERE id = ? AND ${this._entityVisibilitySql(db)}
@@ -379,7 +379,7 @@ export class CodeGraphRepository {
       const args = types
         ? [...this._entityVisibilityParams(db), sourceId, ...this._relationshipVisibilityParams(db), ...types, limit]
         : [...this._entityVisibilityParams(db), sourceId, ...this._relationshipVisibilityParams(db), limit];
-      const rows = db.prepare(baseSql).all(...args);
+      const rows = prepareCached(db, baseSql).all(...args);
       return rows.map(r => ({
         type: r.rel_type,
         targetName: r.target_name,
@@ -445,7 +445,7 @@ export class CodeGraphRepository {
       const args = excludeFile
         ? [...uniq, ...types, ...this._entityVisibilityParams(db), excludeFile, limit]
         : [...uniq, ...types, ...this._entityVisibilityParams(db), limit];
-      const rows = db.prepare(sql).all(...args);
+      const rows = prepareCached(db, sql).all(...args);
       // De-dup by name+type, keeping the first (smallest body).
       const seen = new Set();
       const out = [];
@@ -493,7 +493,7 @@ export class CodeGraphRepository {
         ORDER BY (end_line - start_line) ASC
         LIMIT ?
       `;
-      const rows = db.prepare(sql).all(...uniq, ...types, ...this._entityVisibilityParams(db), limit);
+      const rows = prepareCached(db, sql).all(...uniq, ...types, ...this._entityVisibilityParams(db), limit);
       return rows.map(row => ({
         id: row.id,
         name: row.name,
@@ -549,7 +549,7 @@ export class CodeGraphRepository {
         ORDER BY (end_line - start_line) ASC
         LIMIT ?
       `;
-      const rows = db.prepare(sql).all(...uniq, ...exclude, ...this._entityVisibilityParams(db), limit);
+      const rows = prepareCached(db, sql).all(...uniq, ...exclude, ...this._entityVisibilityParams(db), limit);
       return rows.map(row => ({
         id: row.id,
         name: row.name,
@@ -598,7 +598,7 @@ export class CodeGraphRepository {
           AND ${this._entityVisibilitySql(db)}
         GROUP BY lname
       `;
-      const rows = db.prepare(sql).all(...uniq, ...exclude, ...this._entityVisibilityParams(db));
+      const rows = prepareCached(db, sql).all(...uniq, ...exclude, ...this._entityVisibilityParams(db));
       const map = new Map();
       for (const r of rows) map.set(r.lname, r.count);
       // Names with no row are absent — caller treats absent as 0.
@@ -640,7 +640,7 @@ export class CodeGraphRepository {
       const args = types
         ? [...this._entityVisibilityParams(db), targetId, ...this._relationshipVisibilityParams(db), ...types, limit]
         : [...this._entityVisibilityParams(db), targetId, ...this._relationshipVisibilityParams(db), limit];
-      const rows = db.prepare(baseSql).all(...args);
+      const rows = prepareCached(db, baseSql).all(...args);
       return rows.map(r => ({
         type: r.rel_type,
         contextLine: r.context_line || null,
@@ -689,7 +689,7 @@ export class CodeGraphRepository {
       return Number.isFinite(n) && n > 0 ? n : 12;
     })();
     try {
-      const rows = db.prepare(`
+      const rows = prepareCached(db, `
         SELECT r.target_name, COUNT(*) as cnt
         FROM relationships r
         LEFT JOIN entities e ON e.id = r.target_id AND ${this._entityVisibilitySql(db, 'e')}
@@ -798,7 +798,7 @@ export class CodeGraphRepository {
     if (!db) return null;
     try {
       if (this._hasColumns(db, 'entities', ['epoch_written', 'epoch_retired'])) {
-        const visible = db.prepare(`
+        const visible = prepareCached(db, `
           SELECT 1 FROM entities
           WHERE file_path = ? AND ${this._entityVisibilitySql(db)}
           LIMIT 1
@@ -827,10 +827,10 @@ export class CodeGraphRepository {
         const staleArgs = this._manifestEpoch !== null
           ? [filePath, this._manifestEpoch, this._manifestEpoch]
           : [filePath];
-        const staleRow = db.prepare(staleSql).get(...staleArgs);
+        const staleRow = prepareCached(db, staleSql).get(...staleArgs);
         return staleRow ? { staleSince: staleRow.stale_since || null } : null;
       }
-      const row = db.prepare(`
+      const row = prepareCached(db, `
         SELECT stale_since, MIN(ROWID) as first_row
         FROM entities
         WHERE file_path = ?

package/core/infrastructure/config/search.js

@@ -67,7 +67,7 @@ export const FILE_PATTERNS = {
     '**/*.{cr,vala,hx,pas,nix,vim}',       // Crystal / Vala / Haxe / Pascal / Nix / Vim
     '**/*.{elm,sol,tla,rdl,el,ejs}',       // Elm / Solidity / TLA+ / SystemRDL / Emacs Lisp / EJS
     '**/*.{ql,qll}',                       // CodeQL
-    '**/*.{zeek,bro}',                     // Zeek
+    '**/*.{zeek,bro,bif,pac,pac2,spicy,evt,hlt}', // Zeek + BiF + BinPAC + Spicy DSLs
     '**/*.{tcl,tk}',                       // Tcl
     '**/*.astro',                          // Astro (SFC)
     // GPU shaders
@@ -83,6 +83,23 @@ export const FILE_PATTERNS = {
     '**/*.php',                             // PHP
     '**/*.{swift,m,mm}',                   // Apple
     '**/*.{lua,zig,nim,ex,exs,dart}',      // Other
+    // ── Task-bench coverage audit (2026-07): additional source / DSL / template / build ──
+    '**/*.{pyx,pxd,pxi}',                   // Cython
+    '**/*.{sbt,sc}',                        // Scala build / worksheets
+    '**/*.{rake,gemspec,podspec,ru}',      // Ruby build/manifest DSLs
+    '**/*.{coffee,litcoffee}',             // CoffeeScript
+    '**/*.{pcss,styl}',                     // PostCSS / Stylus
+    '**/*.{twig,liquid,njk,hbs,handlebars,mustache,gohtml,eex,heex,leex}', // HTML templating
+    '**/*.{pug,jade,haml,slim}',           // Indentation templating
+    '**/*.{jinja,jinja2,j2,tera,tmpl,tpl,gotmpl}', // Text templating
+    '**/*.{props,targets,proj,vbproj,fsproj,resx,nuspec}', // .NET / MSBuild
+    '**/*.{m4,ac,am}',                     // Autotools
+    '**/*.{awk,sed}',                      // Text-processing scripts
+    '**/*.ipynb',                          // Jupyter notebooks (generic fixed-window chunking)
+    '**/*.in',                             // Autoconf/config templates (Makefile.in, *.py.in)
+    '**/*.sln',                            // Visual Studio solution
+    '**/go.mod', '**/go.work',             // Go module / workspace manifests
+    '**/Rakefile', '**/Gemfile',           // Ruby (extensionless)
     '**/*.{sh,bash,zsh,fish,ps1}',         // Shell
     '**/*.sql',                             // SQL
     '**/*.proto',                           // Protobuf

package/core/infrastructure/db-utils.js

@@ -17,6 +17,35 @@ import Database from 'better-sqlite3';
  */
 export const SAFE_IN_CLAUSE_BATCH = 999;
 
+// Per-connection prepared-statement cache. Statements belong to their
+// Database object, so the WeakMap self-invalidates when a connection object
+// is dropped (closed connections are never passed back in by callers).
+// Only use for .get/.all/.run statements — never .iterate (a cached statement
+// mid-iteration cannot be re-entered).
+const _stmtCaches = new WeakMap();
+
+/**
+ * db.prepare with a per-connection cache — compiles each distinct SQL string
+ * once per Database object instead of once per call.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} sql
+ * @returns {import('better-sqlite3').Statement}
+ */
+export function prepareCached(db, sql) {
+  let cache = _stmtCaches.get(db);
+  if (!cache) {
+    cache = new Map();
+    _stmtCaches.set(db, cache);
+  }
+  let stmt = cache.get(sql);
+  if (!stmt) {
+    stmt = db.prepare(sql);
+    cache.set(sql, stmt);
+  }
+  return stmt;
+}
+
 /**
  * Apply read-path PRAGMA optimizations to a read-only database connection.
  *

package/core/infrastructure/language-patterns/maps.js

@@ -155,6 +155,9 @@ export const EXTENSION_MAP = {
   '.vim': 'vim',
   '.tcl': 'tcl', '.tk': 'tcl',
   '.zeek': 'zeek', '.bro': 'zeek',
+  // Zeek build-time DSLs: BiF declarations, BinPAC + Spicy parser sources (generic chunking)
+  '.bif': 'zeek', '.pac': 'zeek', '.pac2': 'zeek',
+  '.spicy': 'spicy', '.evt': 'spicy', '.hlt': 'spicy',
 
   // Scientific / legacy (lowercase keys cover the uppercase .F/.F90/.S forms)
   '.f': 'fortran', '.for': 'fortran', '.f90': 'fortran', '.f95': 'fortran',
@@ -183,6 +186,37 @@ export const EXTENSION_MAP = {
   '.ninja': 'ninja',
   '.bzl': 'starlark', '.star': 'starlark',
 
+  // ── Task-bench coverage audit (2026-07): additional source / DSL / template / build ──
+  // Cython (Python-like syntax → reuse the wired python chunker)
+  '.pyx': 'python', '.pxd': 'python', '.pxi': 'python',
+  // Scala build definitions / worksheets (→ scala grammar)
+  '.sbt': 'scala', '.sc': 'scala',
+  // Ruby build & manifest DSLs (→ ruby grammar)
+  '.rake': 'ruby', '.gemspec': 'ruby', '.podspec': 'ruby', '.ru': 'ruby',
+  // CoffeeScript (no bundled grammar → generic chunking)
+  '.coffee': 'coffeescript', '.litcoffee': 'coffeescript',
+  // Style preprocessors
+  '.pcss': 'css', '.styl': 'stylus',
+  // HTML-based templating (reuse the html SFC path, like .vue/.svelte)
+  '.twig': 'html', '.liquid': 'html', '.njk': 'html', '.hbs': 'html',
+  '.handlebars': 'html', '.mustache': 'html', '.gohtml': 'html',
+  '.eex': 'html', '.heex': 'html', '.leex': 'html',
+  // Indentation-based templating (generic chunking; not HTML syntax)
+  '.pug': 'pug', '.jade': 'pug', '.haml': 'haml', '.slim': 'slim',
+  // Polymorphic text templating (any target text → generic chunking)
+  '.jinja': 'jinja', '.jinja2': 'jinja', '.j2': 'jinja',
+  '.tera': 'jinja', '.tmpl': 'jinja', '.tpl': 'jinja', '.gotmpl': 'jinja',
+  // .NET / MSBuild project & resource files (XML dialects → xml grammar)
+  '.props': 'xml', '.targets': 'xml', '.proj': 'xml',
+  '.vbproj': 'xml', '.fsproj': 'xml', '.resx': 'xml', '.nuspec': 'xml',
+  // Autotools / text-processing script sources (generic chunking)
+  '.m4': 'm4', '.ac': 'm4', '.am': 'makefile',
+  '.awk': 'awk', '.sed': 'sed',
+  // Jupyter notebooks — chunker-less id → parseGenericFile (fixed-window, lossless)
+  // over the raw JSON. Not JSON-AST-chunked (that would emit one giant cells chunk).
+  // Image-heavy notebooks self-filter via the admission maxFileSize (1MB) gate.
+  '.ipynb': 'jupyter',
+
   // Document formats (dispatched to DocumentChunker in ast-chunker.js)
   '.md': 'markdown', '.mdx': 'markdown', '.markdown': 'markdown',
   '.rst': 'rst',
@@ -201,4 +235,7 @@ export const FILENAME_MAP = {
   Earthfile: 'earthfile',
   justfile: 'just',
   Justfile: 'just',
+  // Ruby extensionless build files (→ ruby grammar)
+  Rakefile: 'ruby',
+  Gemfile: 'ruby',
 };

package/core/infrastructure/simd-distance.js

@@ -98,6 +98,23 @@ async function initWasm() {
 // Eager non-blocking init
 initWasm().catch(() => {});
 
+// WASM memory views are cached; both modules declare fixed-size memory (no
+// memory.grow anywhere), so the buffer identity check only fires if that
+// ever changes. Same pattern as _wasmPerTokenLayout.
+function wasmMemView() {
+  if (wasmMem === null || wasmMem.buffer !== wasmExports.memory.buffer) {
+    wasmMem = new Uint8Array(wasmExports.memory.buffer);
+  }
+  return wasmMem;
+}
+
+function maxsimMemView() {
+  if (maxsimMem === null || maxsimMem.buffer !== maxsimExports.memory.buffer) {
+    maxsimMem = new Uint8Array(maxsimExports.memory.buffer);
+  }
+  return maxsimMem;
+}
+
 // =============================================================================
 // POPCOUNT LUT (JS fallback)
 // =============================================================================
@@ -115,10 +132,9 @@ export function wasmHammingDistance(a, b) {
   if (wasmExports) {
     const aPtr = DATA_OFFSET;
     const bPtr = DATA_OFFSET + a.length;
-    // Re-acquire view in case memory grew
-    wasmMem = new Uint8Array(wasmExports.memory.buffer);
-    wasmMem.set(a, aPtr);
-    wasmMem.set(b, bPtr);
+    const mem = wasmMemView();
+    mem.set(a, aPtr);
+    mem.set(b, bPtr);
     return wasmExports.hamming_distance(aPtr, bPtr, a.length);
   }
   // JS fallback
@@ -137,9 +153,9 @@ export function wasmInt8Cosine(a, b) {
   if (wasmExports) {
     const aPtr = DATA_OFFSET;
     const bPtr = DATA_OFFSET + a.length;
-    wasmMem = new Uint8Array(wasmExports.memory.buffer);
-    wasmMem.set(new Uint8Array(a.buffer, a.byteOffset, a.byteLength), aPtr);
-    wasmMem.set(new Uint8Array(b.buffer, b.byteOffset, b.byteLength), bPtr);
+    const mem = wasmMemView();
+    mem.set(new Uint8Array(a.buffer, a.byteOffset, a.byteLength), aPtr);
+    mem.set(new Uint8Array(b.buffer, b.byteOffset, b.byteLength), bPtr);
 
     const dot = wasmExports.int8_dot(aPtr, bPtr, a.length);
     const normA = wasmExports.int8_norm_sq(aPtr, a.length);
@@ -193,18 +209,18 @@ export function wasmInt8BatchDot(query, candidates) {
     const alignedScoresPtr = (scoresPtr + 3) & ~3;
     const needed = alignedScoresPtr + count * 4;
 
-    wasmMem = new Uint8Array(wasmExports.memory.buffer);
-    if (needed > wasmMem.length) {
+    const mem = wasmMemView();
+    if (needed > mem.length) {
       return _jsInt8BatchDot(query, candidates);
     }
 
     // Copy query once
-    wasmMem.set(new Uint8Array(query.buffer, query.byteOffset, query.byteLength), queryPtr);
+    mem.set(new Uint8Array(query.buffer, query.byteOffset, query.byteLength), queryPtr);
 
     // Copy all candidates as contiguous slab
     for (let i = 0; i < count; i++) {
       const v = candidates[i];
-      wasmMem.set(new Uint8Array(v.buffer, v.byteOffset, v.byteLength), slabPtr + i * dim);
+      mem.set(new Uint8Array(v.buffer, v.byteOffset, v.byteLength), slabPtr + i * dim);
     }
 
     // Single WASM call: scores all candidates internally, writes to output buffer
@@ -267,9 +283,9 @@ export function wasmInt8Dot(a, b) {
   if (wasmExports) {
     const aPtr = DATA_OFFSET;
     const bPtr = DATA_OFFSET + a.length;
-    wasmMem = new Uint8Array(wasmExports.memory.buffer);
-    wasmMem.set(new Uint8Array(a.buffer, a.byteOffset, a.byteLength), aPtr);
-    wasmMem.set(new Uint8Array(b.buffer, b.byteOffset, b.byteLength), bPtr);
+    const mem = wasmMemView();
+    mem.set(new Uint8Array(a.buffer, a.byteOffset, a.byteLength), aPtr);
+    mem.set(new Uint8Array(b.buffer, b.byteOffset, b.byteLength), bPtr);
     return wasmExports.int8_dot(aPtr, bPtr, a.length);
   }
   // JS fallback
@@ -290,9 +306,9 @@ export function wasmAsymmetricDistance(docBinary, queryInt4, queryNormScaled) {
   if (wasmExports) {
     const docPtr = DATA_OFFSET;
     const queryPtr = DATA_OFFSET + docBinary.length;
-    wasmMem = new Uint8Array(wasmExports.memory.buffer);
-    wasmMem.set(docBinary, docPtr);
-    wasmMem.set(new Uint8Array(queryInt4.buffer, queryInt4.byteOffset, queryInt4.byteLength), queryPtr);
+    const mem = wasmMemView();
+    mem.set(docBinary, docPtr);
+    mem.set(new Uint8Array(queryInt4.buffer, queryInt4.byteOffset, queryInt4.byteLength), queryPtr);
     return wasmExports.asymmetric_distance(docPtr, queryPtr, queryInt4.length, queryNormScaled);
   }
   // JS fallback
@@ -336,9 +352,7 @@ export function wasmMaxSimF32(queryFlat, docFlat, numQ, numD, dim) {
   if (!maxsimExports?.maxsim_f32) return null;
   const exports = maxsimExports;
 
-  const mem = maxsimExports
-    ? new Uint8Array(maxsimExports.memory.buffer)
-    : (wasmMem = new Uint8Array(wasmExports.memory.buffer));
+  const mem = maxsimMemView();
 
   const qBytes = numQ * dim * 4;
   const dBytes = numD * dim * 4;
@@ -375,7 +389,7 @@ export function wasmMaxSimDequant(queryFlat, docInt8, numQ, numD, dim, min, scal
   const qBytes = numQ * dim * 4; // float32
   const dBytes = numD * dim;     // int8 (4x smaller!)
 
-  const mem = new Uint8Array(maxsimExports.memory.buffer);
+  const mem = maxsimMemView();
   if (qBytes + dBytes + 1024 > mem.length) return null;
 
   const qPtr = DATA_OFFSET;

package/core/infrastructure/tombstone-bitmap-reader.js

@@ -49,8 +49,10 @@ export function loadBitmap(filePath) {
   if (raw.length < expectedLength) {
     throw new Error(`loadBitmap: ${filePath} truncated payload (${raw.length} bytes, expected ${expectedLength})`);
   }
+  // View, not copy: `raw` is otherwise unreferenced, so this only keeps the
+  // 64-byte header alive alongside the payload bytes.
   const payload = raw.subarray(HEADER_RESERVED, HEADER_RESERVED + payloadByteLength(capacity));
-  return { capacity, payload: Buffer.from(payload) };
+  return { capacity, payload };
 }
 
 export function saveBitmap(filePath, bitmap) {

package/core/ranking/file-kind-ranking.js

@@ -533,11 +533,23 @@ function envFloat(name, dflt) {
   return Number.isFinite(n) && n > 0 ? n : dflt;
 }
 
+// Pure function of `preferred` (static ENTITY_KIND_KEYWORDS table) — memoize
+// the last kind so the per-result demotion loop reuses one Set.
+let _kindKeywordSetKey;
+let _kindKeywordSet;
+function kindKeywordSet(preferred) {
+  if (preferred !== _kindKeywordSetKey) {
+    _kindKeywordSetKey = preferred;
+    _kindKeywordSet = new Set((ENTITY_KIND_KEYWORDS[preferred] || []).map(normalizeType));
+  }
+  return _kindKeywordSet;
+}
+
 function entityKindMultiplier(r, preferred, opts = {}) {
   if (!preferred) return 1;
   const kindBoost = envFloat('SWEET_SEARCH_KIND_BOOST', 1.10);
   const kindDemote = envFloat('SWEET_SEARCH_KIND_DEMOTE', 0.90);
-  const wantSet = new Set((ENTITY_KIND_KEYWORDS[preferred] || []).map(normalizeType));
+  const wantSet = kindKeywordSet(preferred);
   const inferred = resolveEntityKindInfo(r, opts)?.type || '';
   const recorded = normalizeType(resolveResultType(r));
   const type = recorded && recorded !== 'code' && recorded !== 'chunk' ? recorded : normalizeType(inferred);
@@ -550,7 +562,7 @@ function namePrecisionMultiplier(r, preferred, nameHintsLower, opts = {}) {
   if (!preferred || nameHintsLower.size === 0) return 1;
   const exactBoost = envFloat('SWEET_SEARCH_NAME_EXACT_BOOST', 1.10);
   const substrBoost = envFloat('SWEET_SEARCH_NAME_SUBSTR_BOOST', 1.03);
-  const wantSet = new Set((ENTITY_KIND_KEYWORDS[preferred] || []).map(normalizeType));
+  const wantSet = kindKeywordSet(preferred);
   const entityInfo = resolveEntityKindInfo(r, opts);
   const recorded = normalizeType(resolveResultType(r));
   const type = recorded && recorded !== 'code' && recorded !== 'chunk'

package/core/ranking/late-interaction-index.js

@@ -12,6 +12,7 @@
 
 import fs from 'fs/promises';
 import { existsSync, createWriteStream, createReadStream, statSync } from 'fs';
+import readline from 'readline';
 import path from 'path';
 import { DB_PATHS, LATE_INTERACTION_CONFIG } from '../infrastructure/config/index.js';
 import { wasmMaxSimF32, wasmMaxSimDequantPerToken, wasmMaxSimDequant4Bit, nativeMaxSimBatch, nativeMaxSimBatchPerToken, nativeMaxSimBatch4Bit, initWasm, isNativeMaxSimAvailable, isNativePerTokenAvailable, isNative4BitAvailable } from '../infrastructure/simd-distance.js';
@@ -938,33 +939,90 @@ export class LateInteractionIndex {
   }
 
   async _saveAliasSidecar(indexPath = this.indexPath) {
+    const sidecarPath = this._aliasSidecarPath(indexPath);
     if (this.aliasPointers.size === 0) {
       // Remove any stale sidecar from a previous build.
-      try { await fs.unlink(this._aliasSidecarPath(indexPath)); } catch (_e) { /* not present */ }
+      try { await fs.unlink(sidecarPath); } catch (_e) { /* not present */ }
       return;
     }
-    const entries = [];
-    for (const [aliasId, ptr] of this.aliasPointers) {
-      entries.push({
-        aliasId,
-        exemplarId: ptr.exemplarId,
-        clusterId: ptr.clusterId,
-        metadata: ptr.metadata || {},
-      });
+    // NDJSON (version 2): one header line, then one alias pointer per line,
+    // flushed through a write stream in bounded batches. The previous
+    // monolithic JSON.stringify of the whole payload hits V8's ~512 MB string
+    // ceiling ("Invalid string length") on extreme-dedup repos, and the
+    // matching readFile on load had the same ceiling.
+    const ws = createWriteStream(sidecarPath, { encoding: 'utf8' });
+    const finished = new Promise((resolve, reject) => {
+      ws.on('error', reject);
+      ws.on('finish', resolve);
+    });
+    const flush = (str) => new Promise((resolve, reject) => {
+      ws.write(str, (err) => (err ? reject(err) : resolve()));
+    });
+    try {
+      let lines = [JSON.stringify({ version: 2, count: this.aliasPointers.size })];
+      for (const [aliasId, ptr] of this.aliasPointers) {
+        lines.push(JSON.stringify({
+          aliasId,
+          exemplarId: ptr.exemplarId,
+          clusterId: ptr.clusterId,
+          metadata: ptr.metadata || {},
+        }));
+        if (lines.length >= 20000) {
+          await flush(lines.join('\n') + '\n');
+          lines = [];
+        }
+      }
+      if (lines.length > 0) await flush(lines.join('\n') + '\n');
+      ws.end();
+      await finished;
+    } catch (err) {
+      finished.catch(() => { /* surfaced via throw below */ });
+      ws.destroy();
+      throw err;
     }
-    const payload = { version: 1, count: entries.length, aliases: entries };
-    await fs.writeFile(this._aliasSidecarPath(indexPath), JSON.stringify(payload));
   }
 
   async _loadAliasSidecar(indexPath = this.indexPath) {
     const p = this._aliasSidecarPath(indexPath);
     if (!existsSync(p)) return;
+    // Streamed line-by-line so no single string approaches V8's ~512 MB
+    // ceiling. Two on-disk formats:
+    //   v1 — one JSON.stringify'd { version: 1, aliases: [...] } line
+    //   v2 — NDJSON: { version: 2, count } header, then one alias per line
+    //
+    // The input stream is destroyed in `finally`: early returns abandon the
+    // readline iterator, and rl.close() does NOT destroy its input — an
+    // fs read stream only auto-closes on 'end'/'error', so without the
+    // explicit destroy every early return leaks an fd (fatal only in
+    // long-lived processes like the daemon, but a leak everywhere).
+    const input = createReadStream(p, { encoding: 'utf8' });
+    const rl = readline.createInterface({ input, crlfDelay: Infinity });
     try {
-      const raw = await fs.readFile(p, 'utf-8');
-      const payload = JSON.parse(raw);
-      if (!payload || !Array.isArray(payload.aliases)) return;
       this.aliasPointers.clear();
-      for (const { aliasId, exemplarId, clusterId, metadata } of payload.aliases) {
+      let first = true;
+      let expected = 0;
+      let parsed = 0;
+      for await (const line of rl) {
+        if (!line) continue;
+        if (first) {
+          first = false;
+          const head = JSON.parse(line);
+          if (head && Array.isArray(head.aliases)) {
+            // v1 monolithic payload — the whole sidecar is this one line,
+            // so JSON.parse succeeding IS the integrity check (any
+            // truncation makes it unparseable).
+            for (const { aliasId, exemplarId, clusterId, metadata } of head.aliases) {
+              if (!this.documents.has(exemplarId)) continue; // orphan guard, see below
+              this.aliasPointers.set(aliasId, { exemplarId, clusterId, metadata: metadata || {} });
+            }
+            return;
+          }
+          if (!head || head.version !== 2 || !Number.isFinite(head.count)) return;
+          expected = head.count;
+          continue;
+        }
+        const { aliasId, exemplarId, clusterId, metadata } = JSON.parse(line);
+        parsed++;
         // Orphan guard: drop aliases whose exemplar is no longer in documents.
         // Happens if the file containing the exemplar was removed between
         // save and load (incremental re-index removed the exemplar file
@@ -972,8 +1030,21 @@ export class LateInteractionIndex {
         if (!this.documents.has(exemplarId)) continue;
         this.aliasPointers.set(aliasId, { exemplarId, clusterId, metadata: metadata || {} });
       }
+      // Truncation guard: NDJSON has no whole-file parse to fail, so a file
+      // cut exactly at a line boundary (crash mid-save, disk full) is valid
+      // prefix NDJSON and would otherwise load a silent subset. The header's
+      // `count` restores v1's all-or-nothing semantics. Compared against
+      // PARSED lines, not kept aliases — the orphan guard intentionally
+      // drops entries and must not trip this.
+      if (parsed !== expected) {
+        this.aliasPointers.clear();
+      }
     } catch (_e) {
       // Malformed sidecar — treat as absent; aliases will be skipped at query time.
+      this.aliasPointers.clear();
+    } finally {
+      rl.close();
+      input.destroy();
     }
   }