sweet-search 2.4.2 → 2.5.2

package/core/indexing/ast-chunker.js

@@ -20,6 +20,276 @@ const MIN_CONTENT_LENGTH = 30;
 const MAX_PEEK_LINES = 3;
 const DEFAULT_MAX_REGEX_LINE_LENGTH = 4000;
 
+// =============================================================================
+// Embedding-text cap — RESEARCH / ABLATION INFRASTRUCTURE
+// =============================================================================
+//
+// Production default: 2000, byte-identical to shipped v6.2.
+//
+// Set SWEET_SEARCH_EMBED_TEXT_CAP=N to raise/lower the slice ceiling on
+// `embedding_text`, `li_text`, `li_greedy_text`, and the enriched form.
+// The May-2026 chunk-overflow audit found 80.5% of overflowing chunks are
+// header-pushed (raw content ≤2000, headers push the embedding text over),
+// motivating an A/B vs slightly-larger caps. See eval/results/
+// chunk-overflow-audit.md for the audit and eval/run_overflow_ablation.sh
+// for the wiring. Does NOT alter the chunker max size — that lives behind
+// SWEET_SEARCH_CHUNK_HEADER_OVERHEAD in tree-sitter-provider.js.
+function getEmbedTextCap() {
+  const v = parseInt(process.env.SWEET_SEARCH_EMBED_TEXT_CAP || '', 10);
+  return Number.isFinite(v) && v >= 500 ? v : MAX_CHUNK_SIZE;
+}
+
+/**
+ * Strip a trailing `_<hex>` slug from a path's final basename, before
+ * its extension. Used by the Java family (Java/PHP/C#/Kotlin/Scala)
+ * per the v6.2 language-conditioned policy.
+ *
+ *   javascript/Semantic-UI/Table_366c13.js -> javascript/Semantic-UI/Table.js
+ *   src/Button.tsx (no slug) -> src/Button.tsx (unchanged)
+ *
+ * Exported for reuse in chunk-builder.js (document chunks) and tests.
+ */
+export function normalizePathSlug(relativePath) {
+  if (!relativePath) return relativePath;
+  return relativePath.replace(/_[0-9a-f]{6,}(\.[a-zA-Z0-9]+)$/, '$1');
+}
+
+// Languages that share Java's slug-stripped-path li_text policy.
+// (At LI input routing time these will all be sent to chunk.li_text;
+// see LI_TEXT_LANGUAGES in core/indexing/indexer-ann.js, which imports
+// this set so the policy stays consistent in one place.)
+export const JAVA_FAMILY = new Set([
+  'java', 'php',
+  'csharp', 'c#',
+  'kotlin', 'scala',
+]);
+
+// =============================================================================
+// Embedding-text variant switch — RESEARCH / ABLATION INFRASTRUCTURE
+// =============================================================================
+//
+// Production behavior is fixed at variant=current and is byte-identical to
+// shipped v6.2. None of the alternative variants below are recommended for
+// production; an isolated R1 ablation in May 2026 found no variant beats
+// shipped on total MRR@10 (see docs/JS_CHUNK_BLEEDING_ANALYSIS.md for the
+// full table). The switch and helpers are kept ONLY so future R1
+// experiments can be re-run without re-implementing the scaffolding.
+//
+// Default: SWEET_SEARCH_EMBED_TEXT_VARIANT unset → 'current' →
+//          embedding_text + LI input both byte-identical to shipped.
+//
+// Available experiment variants (research use only):
+//   current           shipped form (production default)
+//   no_path           drop the # path line
+//   normalized_path   slug-strip the trailing _<hex> in path
+//   no_language       drop # Language: line
+//   parent_only       path + parent only, drop function + language
+//   enriched          identical to current here; also runs enrichment
+//   code_breadcrumb   compact `# Parent.Symbol` or `# Parent::symbol`
+//   signature         current + `# Signature: <multi-line-sig>` line
+//                     between the symbol line and the language line.
+//                     Signature is AST-extracted by tree-sitter (decl
+//                     header up to body, whitespace-collapsed, capped
+//                     at MAX_SIGNATURE_LENGTH). When no signature is
+//                     available (regex fallback path, non-boundary
+//                     chunks, or merged sibling buffers) this variant
+//                     is byte-identical to `current`.
+//   signature_rbphp   like `signature` but only for Ruby and PHP — the
+//                     two languages where the May-2026 R1 ablation
+//                     showed signatures helped (PHP +0.79 MRR@10,
+//                     Ruby +0.58); other languages route to the same
+//                     output as `current`. Motivated by per-language
+//                     pareto front (matching v6.2 LI routing pattern).
+//
+// LI isolation: when a non-current variant is active, `chunk.li_greedy_text`
+// still carries the shipped (variant=current) form, and pickLiInput() for
+// non-Python/Java languages reads it — so an R1 experiment never
+// contaminates the LI input. See enrichEmbeddingText() and pickLiInput().
+const ENRICHMENT_VARIANTS = new Set(['current', 'enriched']);
+
+function getEmbedTextVariant() {
+  const v = (process.env.SWEET_SEARCH_EMBED_TEXT_VARIANT || 'current').toLowerCase();
+  return [
+    'current', 'no_path', 'normalized_path', 'no_language',
+    'parent_only', 'enriched', 'code_breadcrumb',
+    'signature', 'signature_rbphp',
+  ].includes(v) ? v : 'current';
+}
+
+// Languages where the May-2026 ablation showed `signature` strictly
+// helped on R1 MRR@10 (PHP +0.79, Ruby +0.58 vs current). The
+// `signature_rbphp` variant only emits the # Signature: line for
+// these — every other language gets exactly the `current` output.
+const SIGNATURE_HELP_LANGUAGES = new Set(['ruby', 'php']);
+
+export function shouldRunEnrichment() {
+  return ENRICHMENT_VARIANTS.has(getEmbedTextVariant());
+}
+
+// `Foo::bar` for langs that conventionally use ::, `Foo.bar` otherwise.
+// Used by the experimental code_breadcrumb variant only.
+function breadcrumbSep(language) {
+  return ['ruby', 'php', 'cpp', 'c++', 'rust'].includes(language) ? '::' : '.';
+}
+
+/**
+ * Build `embedding_text` per the active R1 variant.
+ *
+ * Production callers always pass either no variant (env-driven, defaults
+ * to 'current') or `variant: 'current'` explicitly — both produce text
+ * byte-identical to shipped v6.2.
+ *
+ * `variant: 'current'` is also passed explicitly when populating
+ * `chunk.li_greedy_text` so the LI stage sees the shipped form even
+ * when the bi-encoder is consuming a different experimental variant.
+ */
+function buildEmbeddingText({ variant: variantOverride, content, relativePath, language, chunkType, symbol, hierarchyInfo }) {
+  const variant = variantOverride || getEmbedTextVariant();
+  const trimmed = content.trim();
+  const parts = [];
+
+  const pathLine = relativePath ? `# ${relativePath}` : null;
+  const parentLine = hierarchyInfo?.parentSymbol
+    ? `# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}` : null;
+  const symbolLine = (symbol && symbol !== 'unknown')
+    ? `# ${chunkType}: ${symbol}` : null;
+  const langLine = (language && language !== 'text')
+    ? `# Language: ${language}` : null;
+  const signatureLine = hierarchyInfo?.signature
+    ? `# Signature: ${hierarchyInfo.signature}` : null;
+
+  switch (variant) {
+    case 'no_path':
+      if (parentLine) parts.push(parentLine);
+      if (symbolLine) parts.push(symbolLine);
+      if (langLine) parts.push(langLine);
+      break;
+    case 'normalized_path':
+      if (relativePath) parts.push(`# ${normalizePathSlug(relativePath)}`);
+      if (parentLine) parts.push(parentLine);
+      if (symbolLine) parts.push(symbolLine);
+      if (langLine) parts.push(langLine);
+      break;
+    case 'no_language':
+      if (pathLine) parts.push(pathLine);
+      if (parentLine) parts.push(parentLine);
+      if (symbolLine) parts.push(symbolLine);
+      break;
+    case 'parent_only':
+      if (pathLine) parts.push(pathLine);
+      if (parentLine) parts.push(parentLine);
+      break;
+    case 'code_breadcrumb': {
+      if (pathLine) parts.push(pathLine);
+      const sep = breadcrumbSep(language);
+      if (hierarchyInfo?.parentSymbol && symbol && symbol !== 'unknown') {
+        parts.push(`# ${hierarchyInfo.parentSymbol}${sep}${symbol}`);
+      } else if (symbol && symbol !== 'unknown') {
+        parts.push(`# ${symbol}`);
+      } else if (hierarchyInfo?.parentSymbol) {
+        parts.push(`# ${hierarchyInfo.parentSymbol}`);
+      }
+      if (language && language !== 'text') parts.push(`# ${language}`);
+      break;
+    }
+    case 'signature':
+      // Same headers as current, plus a `# Signature:` line emitted
+      // between the symbol line and the language line. When no
+      // signature is available (regex fallback / merged sibling
+      // buffer / non-boundary chunk), this is byte-identical to
+      // current — the variant degrades gracefully.
+      if (pathLine) parts.push(pathLine);
+      if (parentLine) parts.push(parentLine);
+      if (symbolLine) parts.push(symbolLine);
+      if (signatureLine) parts.push(signatureLine);
+      if (langLine) parts.push(langLine);
+      break;
+    case 'signature_rbphp':
+      // Lang-conditioned: emit signature only for Ruby and PHP (the
+      // two languages where unconditional `signature` strictly helped
+      // on the May-2026 R1 ablation). Every other language gets the
+      // exact `current` output. Mirrors the v6.2 LI routing pattern.
+      if (pathLine) parts.push(pathLine);
+      if (parentLine) parts.push(parentLine);
+      if (symbolLine) parts.push(symbolLine);
+      if (signatureLine && SIGNATURE_HELP_LANGUAGES.has(language)) {
+        parts.push(signatureLine);
+      }
+      if (langLine) parts.push(langLine);
+      break;
+    case 'enriched':
+    case 'current':
+    default:
+      if (pathLine) parts.push(pathLine);
+      if (parentLine) parts.push(parentLine);
+      if (symbolLine) parts.push(symbolLine);
+      if (langLine) parts.push(langLine);
+      break;
+  }
+
+  parts.push(trimmed);
+  return parts.join('\n').slice(0, getEmbedTextCap());
+}
+
+/**
+ * Per-language path-line strategy for late-interaction MaxSim input.
+ *
+ *   - Python: no path line (at 97% MRR ceiling; path duplicated funcname).
+ *   - Java family (Java, PHP, C#, Kotlin, Scala): slug-stripped path.
+ *   - Everything else: full path. (For JS family, Ruby, Go, C/C++/Rust
+ *     and unknown languages, the LI input router in indexer-ann.js
+ *     bypasses li_text entirely and reads chunk.embedding_text directly,
+ *     so this branch only matters as a fallback if those routes ever
+ *     fail to find embedding_text.)
+ */
+function pathLineForLi(relativePath, language) {
+  if (!relativePath) return null;
+  if (language === 'python') return null;
+  if (JAVA_FAMILY.has(language)) {
+    return `# ${normalizePathSlug(relativePath)}`;
+  }
+  return `# ${relativePath}`;
+}
+
+/**
+ * Build `li_text` — the text fed to lateon-code MaxSim reranking.
+ *
+ * v6.2 (2026-05): language-conditioned path strategy. All languages
+ * share the same `# Parent / # symbol / # Language` label-shaped
+ * headers and differ only in how (or whether) the path comment is
+ * included. See pathLineForLi() docstring for the per-language map.
+ *
+ * Why language-conditioned vs uniform: per-language ablation showed
+ * uniform policies always regress at least one language (greedy
+ * regresses Python; v4 no-path regresses Ruby below baseline; v5
+ * slug-strip-uniform hurts JS). The v6.2 split — Python skips the
+ * path, Java family slug-strips, JS family / Ruby / Go / fallback get
+ * the full enriched embedding_text — is the empirical pareto front.
+ *
+ * The actual routing of `li_text` vs `embedding_text` lives in
+ * core/indexing/indexer-ann.js (`pickLiInput`).
+ */
+function buildLiText({ content, relativePath, language, chunkType, symbol, hierarchyInfo }) {
+  const trimmed = content.trim();
+  const lines = [];
+
+  const pathLine = pathLineForLi(relativePath, language);
+  if (pathLine) lines.push(pathLine);
+
+  if (hierarchyInfo?.parentSymbol) {
+    lines.push(`# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}`);
+  }
+  if (symbol && symbol !== 'unknown') {
+    lines.push(`# ${chunkType}: ${symbol}`);
+  }
+  if (language && language !== 'text') {
+    lines.push(`# Language: ${language}`);
+  }
+  lines.push(trimmed);
+
+  return lines.join('\n').slice(0, getEmbedTextCap());
+}
+
 /**
  * AST-like semantic code chunker supporting 35+ languages.
  * Uses regex boundary patterns from core/language-patterns.js registry.
@@ -100,6 +370,7 @@ export class ASTChunker {
           parentChunkId: chunk.parentChunkId,
           parentSymbol: chunk.parentSymbol,
           parentType: chunk.parentType,
+          signature: chunk.signature || null,
         }
       )
     );
@@ -606,19 +877,43 @@ export class ASTChunker {
     const hash = createHash('sha256').update(content).digest('hex').slice(0, 16);
     const relativePath = this.projectRoot ? path.relative(this.projectRoot, filePath) : filePath;
 
-    // Build contextualized embedding text
-    const embeddingParts = [];
-    embeddingParts.push(`# ${relativePath}`);
-    if (hierarchyInfo?.parentSymbol) {
-      embeddingParts.push(`# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}`);
-    }
-    if (symbol && symbol !== 'unknown') {
-      embeddingParts.push(`# ${chunkType}: ${symbol}`);
-    }
-    if (language && language !== 'text') {
-      embeddingParts.push(`# Language: ${language}`);
-    }
-    embeddingParts.push(content.trim());
+    // Production embedding text. With the default variant (current) this
+    // is byte-identical to shipped v6.2; under a research-only variant
+    // (SWEET_SEARCH_EMBED_TEXT_VARIANT) it produces the experimental form.
+    const embedding_text = buildEmbeddingText({
+      content,
+      relativePath,
+      language,
+      chunkType,
+      symbol,
+      hierarchyInfo,
+    });
+
+    // Research isolation: always build the shipped (current-variant)
+    // form alongside, so the LI stage can read a canonical input even
+    // when an R1 experiment is active on embedding_text. In production
+    // (variant=current) this is the same content as embedding_text and
+    // pickLiInput's preference for li_greedy_text changes nothing.
+    const li_greedy_text = buildEmbeddingText({
+      variant: 'current',
+      content,
+      relativePath,
+      language,
+      chunkType,
+      symbol,
+      hierarchyInfo,
+    });
+
+    // Build LI text via the v6.2 builder (language-conditioned path
+    // policy). See buildLiText() docstring for rationale.
+    const li_text = buildLiText({
+      content,
+      relativePath,
+      language,
+      chunkType,
+      symbol,
+      hierarchyInfo,
+    });
 
     const metadata = {
       type: 'codebase',
@@ -647,15 +942,26 @@ export class ASTChunker {
     return {
       text: content.trim(),
       content: content.trim(),
-      embedding_text: embeddingParts.join('\n').slice(0, 2000),
+      embedding_text,
+      li_greedy_text,
+      li_text,
       metadata,
       tags: ['codebase', language, this.inferProjectTag(filePath)]
     };
   }
 
   /**
-   * Enrich a chunk's embedding_text with scope chain and import information.
+   * Enrich a chunk's embedding_text with scope chain and import info.
    * Called after initial chunking when scope info is available.
+   *
+   * Production path (variant=current): updates BOTH `embedding_text`
+   * and `li_greedy_text` to the same enriched string — byte-identical
+   * to shipped v6.2.
+   *
+   * Research path (variant != current/enriched): only updates
+   * `li_greedy_text`. The variant's `embedding_text` is preserved so
+   * an R1 ablation can compare its embedding form against shipped
+   * without losing the LI side's enriched text.
    */
   static enrichEmbeddingText(chunk, scopeChain, imports) {
     const parts = [];
@@ -681,7 +987,16 @@ export class ASTChunker {
     }
 
     parts.push(chunk.content);
-    chunk.embedding_text = parts.join('\n').slice(0, 2000);
+    const enriched = parts.join('\n').slice(0, getEmbedTextCap());
+
+    // Always update the byte-stable LI passthrough form.
+    chunk.li_greedy_text = enriched;
+
+    // Only update embedding_text when the active variant participates
+    // in post-build enrichment.
+    if (shouldRunEnrichment()) {
+      chunk.embedding_text = enriched;
+    }
     return chunk;
   }
 

package/core/indexing/chunking/chunk-builder.js

@@ -5,10 +5,19 @@
 import { createHash } from 'crypto';
 import path from 'path';
 import { detectProjectBoundary } from '../../infrastructure/project-detector.js';
+import { JAVA_FAMILY, normalizePathSlug } from '../ast-chunker.js';
 
 const MAX_CHUNK_SIZE = 2000;  // chars — matches ast-chunker.js
 const MIN_CHUNK_SIZE = 30;    // chars — matches ast-chunker.js threshold
 
+// Embedding-text cap mirrors ast-chunker.js:getEmbedTextCap. Default 2000,
+// byte-identical to shipped. Override via SWEET_SEARCH_EMBED_TEXT_CAP for
+// the May-2026 budget-alignment ablation.
+function getEmbedTextCap() {
+  const v = parseInt(process.env.SWEET_SEARCH_EMBED_TEXT_CAP || '', 10);
+  return Number.isFinite(v) && v >= 500 ? v : MAX_CHUNK_SIZE;
+}
+
 /** Default config for markdown chunker */
 const MD_DEFAULTS = {
   maxChunkSize: MAX_CHUNK_SIZE,
@@ -59,10 +68,34 @@ function buildDocChunk(content, filePath, language, chunkType, symbol, lineStart
   }
   embeddingParts.push(trimmed);
 
+  // Build LI text via v6.2 — language-conditioned path strategy.
+  // Document chunks are typically markdown/text and fall under the
+  // "other" branch (full raw path). Code-language chunks reuse the
+  // shared JAVA_FAMILY set from ast-chunker.js so the path policy
+  // here cannot drift from the AST chunker.
+  const liParts = [];
+  if (relativePath) {
+    if (language === 'python') {
+      // skip path line for Python
+    } else if (JAVA_FAMILY.has(language)) {
+      liParts.push(`# ${normalizePathSlug(relativePath)}`);
+    } else {
+      liParts.push(`# ${relativePath}`);
+    }
+  }
+  if (symbol && symbol !== 'unknown') {
+    liParts.push(`# ${chunkType}: ${symbol}`);
+  }
+  if (language && language !== 'text') {
+    liParts.push(`# Language: ${language}`);
+  }
+  liParts.push(trimmed);
+
   return {
     text: trimmed,
     content: trimmed,
-    embedding_text: embeddingParts.join('\n').slice(0, 2000),
+    embedding_text: embeddingParts.join('\n').slice(0, getEmbedTextCap()),
+    li_text: liParts.join('\n').slice(0, getEmbedTextCap()),
     metadata: {
       type: 'document',
       file: path.basename(filePath),

package/core/indexing/index-codebase-v21.js

@@ -40,6 +40,7 @@ if (process.env.SWEET_SEARCH_UV_THREADPOOL_SIZE && !process.env.UV_THREADPOOL_SI
 import { existsSync } from 'fs';
 
 import { DB_PATHS, LATE_INTERACTION_CONFIG } from '../infrastructure/config/index.js';
+import { applyPersistedLiModel } from '../infrastructure/init-config.js';
 import { resolveRelationshipTargets } from '../graph/relationship-resolver.js';
 import { requireNativeAnn as requireNativeAnnBackend } from '../vector-store/hnsw-index.js';
 import { getStats as getIncrementalStats } from './incremental-tracker.js';
@@ -124,11 +125,18 @@ async function main() {
     setVerboseMode(true);
   }
 
-  // Apply late interaction model overrides before any model code runs
+  // Apply late interaction model overrides before any model code runs.
+  // Precedence: --no-late-interaction > --late-interaction-model=… > env
+  // var (already honoured by LATE_INTERACTION_CONFIG.model at module load) >
+  // .sweet-search/config.json::runtime.li.model > built-in default. Only
+  // touch the persisted-config branch when neither CLI flag was used —
+  // applyPersistedLiModel internally re-checks the env var.
   if (noLateInteraction) {
     LATE_INTERACTION_CONFIG.model = false;
   } else if (lateInteractionModel) {
     LATE_INTERACTION_CONFIG.model = lateInteractionModel;
+  } else {
+    applyPersistedLiModel(process.env.SWEET_SEARCH_PROJECT_ROOT || process.cwd());
   }
 
   log(`${colors.bright}╔═══════════════════════════════════════════════════╗${colors.reset}`, 'bright');
@@ -441,7 +449,28 @@ Output:
   }
 }
 
-if (import.meta.url === `file://${process.argv[1]}`) {
+// Direct-run guard. The previous `import.meta.url === \`file://${process.argv[1]}\``
+// form silently no-op'd under three real-world conditions:
+//   1. `npm install ../sweet-search-private` (file install) symlinks
+//      `node_modules/sweet-search/` to the source — `process.argv[1]` is the
+//      symlink path while `import.meta.url` resolves to the realpath.
+//   2. Paths containing spaces or unicode — the URL form encodes them but
+//      `file://` + raw path doesn't.
+//   3. Windows backslash vs URL forward-slash mismatch.
+// Resolve both sides through `realpathSync(fileURLToPath(...))` so the
+// comparison survives every common install layout. Falls back to never-direct
+// (safe default) if either side errors.
+import { realpathSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+const _isDirectRun = (() => {
+  if (!process.argv[1]) return false;
+  try {
+    return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1]);
+  } catch {
+    return false;
+  }
+})();
+if (_isDirectRun) {
   main().catch(err => {
     console.error(err);
     process.exit(1);

package/core/indexing/index.js

@@ -79,12 +79,15 @@ export {
   getEmbeddingPool,
 } from './indexer-pool.js';
 
-// Late-interaction skip policy (consumed by tests; honors unified exclude globs)
+// Shared indexing file policy (embedding, sparse/BM25 artifacts, and LI)
 export {
-  applyLiSkipPolicy,
+  applyIndexingChunkPolicy,
   isExcludedByConfig,
   chunkLooksGenerated,
-} from './li-skip-policy.js';
+} from './indexing-file-policy.js';
+
+// Late-interaction compatibility export
+export { applyLiSkipPolicy } from './li-skip-policy.js';
 
 // Sparse-gram artifact builder (tier-1 grep acceleration)
 export { buildSparseGramArtifact } from './indexer-sparse-gram.js';

package/core/indexing/indexer-ann.js

@@ -12,6 +12,7 @@ import { LateInteractionIndex } from '../ranking/late-interaction-index.js';
 import { truncateForHNSW, getEmbeddings, getModelInfo, fisherYatesShuffle } from '../embedding/embedding-service.js';
 import { buildFromCodebaseDb as buildQuantizedArtifacts, shouldSkipArtifactRebuild, updateArtifactState, ARTIFACT_THRESHOLDS } from './artifact-builder.js';
 import { log, logProgress } from './indexer-utils.js';
+import { JAVA_FAMILY } from './ast-chunker.js';
 
 // =============================================================================
 // DURABLE WRITE HELPERS (Phase E — fsync ordering for checkpoint safety)
@@ -20,6 +21,45 @@ import { log, logProgress } from './indexer-utils.js';
 const CHECKPOINT_INTERVAL_SEC = 30;
 const MIN_VECTORS_BETWEEN_SAVES = 1000;
 
+/**
+ * v6.2: language-family-conditioned LI input routing.
+ *
+ * Per-language ablation on GenCodeSearchNet (May 2026, see
+ * docs/JS_CHUNK_BLEEDING_ANALYSIS.md) determined that different
+ * languages benefit from different metadata richness in the late-
+ * interaction MaxSim input. v6.2 picks empirically:
+ *
+ *   chunk.li_text  (simple, per-lang path policy in chunker):
+ *     - python (no path; at 97% ceiling, path duplicated funcname)
+ *     - JAVA_FAMILY: java, php, csharp/c#, kotlin, scala
+ *       (slug-stripped path; framework signal preserved, hashed-slug
+ *       noise removed)
+ *
+ *   chunk.embedding_text (greedy enriched; Scope + Defines + Uses):
+ *     - javascript, typescript, jsx, tsx (closures + imports help)
+ *     - ruby, go, c, cpp, c++, rust
+ *     - any unknown language as the safe fallback
+ *
+ * The Java-family set is sourced from ast-chunker.js so that the
+ * routing here and the path-policy logic inside the chunker can never
+ * drift out of sync.
+ */
+export function pickLiInput(chunk) {
+  const lang = chunk?.metadata?.language;
+  if (lang === 'python' || JAVA_FAMILY.has(lang)) {
+    return chunk.li_text || chunk.embedding_text || chunk.text || chunk.content || '';
+  }
+  // For JS family / Ruby / Go / C/C++/Rust / unknown, read from
+  // `li_greedy_text` first. In production (variant=current) this is
+  // identical to embedding_text — same shipped v6.2 behavior. The
+  // preference matters only when a research-only embedding variant
+  // (SWEET_SEARCH_EMBED_TEXT_VARIANT) is active: then li_greedy_text
+  // still carries the shipped form, so an R1 ablation cannot
+  // accidentally mutate the LI input. Older chunks without
+  // li_greedy_text fall through to embedding_text as before.
+  return chunk.li_greedy_text || chunk.embedding_text || chunk.li_text || chunk.text || chunk.content || '';
+}
+
 function fsyncFile(filePath) {
   const fd = openSync(filePath, 'r');
   try { fsyncSync(fd); } finally { closeSync(fd); }
@@ -239,7 +279,7 @@ function buildLateInteractionBatches(chunks, options = {}) {
     attentionBudget = Math.max(1, Math.floor(batchSizeCap / 2)) * maxLength * maxLength;
   }
   const indexed = chunks.map((chunk) => {
-    const text = chunk.text || chunk.content || '';
+    const text = pickLiInput(chunk);
     return {
       chunk,
       text,
@@ -603,17 +643,16 @@ export async function buildLateInteractionIndex(chunks, dryRun = false, filesToR
   // Apply LI skip policy: last-mile guard before the slow encoder runs.
   // The embed indexer has already dropped glob-excluded files at discovery
   // time using the same loadProjectConfig() excludes; this pass adds the
-  // LI-specific checks globs can't do — content-based @generated markers
-  // and a per-file token budget. Disable via SWEET_SEARCH_LI_SKIP_DISABLE=1.
+  // LI-specific check globs can't do: content-based @generated markers.
+  // Disable via SWEET_SEARCH_LI_SKIP_DISABLE=1.
   let skippedSummary = null;
   if (Array.isArray(chunks) && chunks.length > 0) {
-    const { applyLiSkipPolicy } = await import('./li-skip-policy.js');
-    const { kept, stats } = applyLiSkipPolicy(chunks, { projectRoot });
+    const { applyIndexingChunkPolicy } = await import('./indexing-file-policy.js');
+    const { kept, stats } = applyIndexingChunkPolicy(chunks, { projectRoot });
     if (stats.totalSkipped > 0) {
       const breakdown = [
         stats.excluded > 0 ? `${stats.excluded} excluded` : null,
         stats.generated > 0 ? `${stats.generated} generated` : null,
-        stats.huge > 0 ? `${stats.huge} huge-file` : null,
       ].filter(Boolean).join(', ');
       log(`LI skip policy: dropped ${stats.totalSkipped} chunks across ${stats.skippedFiles} files (${breakdown}); kept ${kept.length} chunks across ${stats.keptFiles} files`, 'dim');
       chunks = kept;

package/core/indexing/indexer-build.js

@@ -537,9 +537,17 @@ export async function chunkFiles(files) {
     }
   }
 
+  // Embedding-text cap: defaults to 2000 (byte-identical to shipped). The
+  // SWEET_SEARCH_EMBED_TEXT_CAP env var (see ast-chunker.js:getEmbedTextCap)
+  // is honored by the chunk builders themselves; this final re-slice mirrors
+  // the same cap so an ablation can raise the ceiling end-to-end.
+  const _embCap = (() => {
+    const v = parseInt(process.env.SWEET_SEARCH_EMBED_TEXT_CAP || '', 10);
+    return Number.isFinite(v) && v >= 500 ? v : 2000;
+  })();
   const texts = allChunks.map(chunk => {
     if (chunk.embedding_text) {
-      return chunk.embedding_text.slice(0, 2000);
+      return chunk.embedding_text.slice(0, _embCap);
     }
     return `${chunk.file} ${chunk.metadata?.symbol || ''}\n${(chunk.text || chunk.content || '').slice(0, 1500)}`;
   });

package/core/indexing/indexer-phases.js

@@ -7,7 +7,7 @@ import fs from 'fs/promises';
 import { existsSync } from 'fs';
 import path from 'path';
 
-import { DB_PATHS, PROJECT_ROOT, EMBEDDING_CONFIG } from '../infrastructure/config/index.js';
+import { DB_PATHS, PROJECT_ROOT, EMBEDDING_CONFIG, HCGS_CONFIG } from '../infrastructure/config/index.js';
 import { getChangedFiles, updateState, getStats as getIncrementalStats, updatePhaseProgress, markPhaseComplete, clearPhaseProgress } from './incremental-tracker.js';
 import { backupSummaries, restoreSummaries, markForRegeneration } from '../graph/summary-manager.js';
 import { colors, log, logProgress, logError, discoverFiles, readFilesFromStdin, atomicSwapDatabase } from './indexer-utils.js';
@@ -300,8 +300,10 @@ export async function buildCodeGraphWithHCGSPhase(options = {}) {
     skipSummaryRegen,
   } = options;
 
+  const hcgsEnabled = HCGS_CONFIG.enabled;
+
   let summaryBackup = { summaries: [], count: 0 };
-  if (!dryRun) {
+  if (!dryRun && hcgsEnabled) {
     summaryBackup = await backupSummaries(DB_PATHS.codeGraph);
     if (summaryBackup.count > 0) {
       log(`Backed up ${summaryBackup.count} existing summaries (with type validation)`, 'green');
@@ -318,14 +320,14 @@ export async function buildCodeGraphWithHCGSPhase(options = {}) {
 
   const graphStats = await buildCodeGraph(allFiles, dryRun);
 
-  if (!dryRun && summaryBackup.count > 0) {
+  if (!dryRun && hcgsEnabled && summaryBackup.count > 0) {
     const restoreResult = await restoreSummaries(DB_PATHS.codeGraph, summaryBackup);
     log(`Restored ${restoreResult.restored} summaries (${restoreResult.skipped.total} skipped - entity removed/type changed)`, 'green');
   }
 
   let hcgsPromise = null;
 
-  const shouldRunHCGS = !dryRun && (
+  const shouldRunHCGS = !dryRun && hcgsEnabled && (
     fullReindex ||
     filesFromStdin ||
     (incrementalInfo && filesToIndex.length > 0)