sweet-search 2.4.2 → 2.5.1

package/core/search/sweet-search.js

@@ -21,6 +21,8 @@ import { HNSWIndex } from '../vector-store/hnsw-index.js';
 import { BinaryHNSWIndex } from '../vector-store/binary-hnsw-index.js';
 import { Reranker } from '../ranking/flashrank.js';
 import { LateInteractionIndex } from '../ranking/late-interaction-index.js';
+import { resolveSearchRerankPolicy } from '../ranking/late-interaction-policy.js';
+import { readPersistedLiPolicy } from '../infrastructure/index.js';
 import { getEmbedding, getBinaryEmbedding, truncateForHNSW, int8CosineSimilarity, warmup as warmupEmbedding, isWarm, registerAutoPersistOnExit } from '../embedding/embedding-service.js';
 import { FloatVectorStore, getFloatStorePath } from '../vector-store/float-vector-store.js';
 import { recordQueryTelemetry } from '../embedding/embedding-cache.js';
@@ -41,6 +43,7 @@ import * as semantic from './search-semantic.js';
 import * as hybrid from './search-hybrid.js';
 import * as postprocess from './search-postprocess.js';
 import * as pattern from './search-pattern.js';
+import { packageForAgent } from './context-expander.js';
 
 export { ROUTE_ALPHAS } from './search-fusion.js';
 
@@ -93,10 +96,13 @@ export class SweetSearch {
     const envOrProject = (envKey, cascadeKey, configKey) =>
       process.env[envKey] != null ? CASCADE_CONFIG[configKey] : projectCascade[cascadeKey];
 
-    this.graphSearch = new GraphSearch(options.graphDbPath || DB_PATHS.codeGraph);
-    this.codeGraphRepo = new CodeGraphRepository(options.graphDbPath || DB_PATHS.codeGraph);
-    this.hnswIndex = new HNSWIndex({ indexPath: options.hnswPath || DB_PATHS.hnswIndex });
-    this.binaryHnswIndex = new BinaryHNSWIndex({ indexPath: options.binaryHnswPath || DB_PATHS.binaryHnswIndex });
+    this.graphDbPath = options.graphDbPath || DB_PATHS.codeGraph;
+    this.graphSearch = new GraphSearch(this.graphDbPath);
+    this.codeGraphRepo = new CodeGraphRepository(this.graphDbPath);
+    this.hnswPath = options.hnswPath || DB_PATHS.hnswIndex;
+    this.binaryHnswPath = options.binaryHnswPath || DB_PATHS.binaryHnswIndex;
+    this.hnswIndex = new HNSWIndex({ indexPath: this.hnswPath });
+    this.binaryHnswIndex = new BinaryHNSWIndex({ indexPath: this.binaryHnswPath });
     this.reranker = new Reranker(options);
     this.lateInteractionIndex = new LateInteractionIndex(options.lateInteractionOptions || {});
     this.router = new QueryRouter();
@@ -108,7 +114,25 @@ export class SweetSearch {
     this.stage1Candidates = options.stage1Candidates ?? BINARY_HNSW_CONFIG.retrieval.stage1Candidates;
     this.stage2Candidates = options.stage2Candidates ?? BINARY_HNSW_CONFIG.retrieval.stage2Candidates;
     this.stage3Candidates = options.stage3Candidates ?? BINARY_HNSW_CONFIG.retrieval.stage3Candidates;
-    this.useLateInteraction = options.useLateInteraction ?? LATE_INTERACTION_CONFIG.enabled;
+    // Late-interaction search-rerank gating — see core/ranking/late-interaction-policy.js
+    // for the full precedence ladder. The constructor records the inputs and
+    // computes a tentative value (so callers reading `useLateInteraction`
+    // before init() get sensible defaults); init() recomputes once the LI
+    // index header has been loaded so the manifest's modelId can drive the
+    // auto policy.
+    this._liPolicyOptionOverride = typeof options.useLateInteraction === 'boolean'
+      ? options.useLateInteraction
+      : undefined;
+    this._liPolicyPersisted = readPersistedLiPolicy(projectRoot);
+    const liInitial = resolveSearchRerankPolicy({
+      optionOverride: this._liPolicyOptionOverride,
+      env: process.env,
+      persisted: this._liPolicyPersisted,
+      indexManifest: null,
+      activeConfigModel: LATE_INTERACTION_CONFIG.model,
+    });
+    this.useLateInteraction = LATE_INTERACTION_CONFIG.enabled ? liInitial.effective : false;
+    this._liPolicyResolved = liInitial;
     this.lateInteractionBlendWeight = options.lateInteractionBlendWeight ?? LATE_INTERACTION_CONFIG.blendWeight ?? 0.3;
     this.returnSummaryFirst = options.returnSummaryFirst ?? HCGS_CONFIG.returnSummaryFirst;
     this.summaryTokenBudget = options.summaryTokenBudget ?? HCGS_CONFIG.summaryTokenBudget;
@@ -152,9 +176,9 @@ export class SweetSearch {
     if (this.initialized) return;
     const start = Date.now();
 
-    this.hasGraphIndex = existsSync(DB_PATHS.codeGraph);
-    this.hasHnswIndex = existsSync(DB_PATHS.hnswIndex.replace('.idx', '.meta.json'));
-    this.hasBinaryHnswIndex = existsSync(DB_PATHS.binaryHnswIndex.replace('.idx', '.meta.json'));
+    this.hasGraphIndex = existsSync(this.graphDbPath);
+    this.hasHnswIndex = existsSync(this.hnswPath.replace('.idx', '.meta.json'));
+    this.hasBinaryHnswIndex = existsSync(this.binaryHnswPath.replace('.idx', '.meta.json'));
     this.hasCodebaseIndex = existsSync(this.codebaseDbPath);
     this.hasLateInteractionIndex = existsSync(this.lateInteractionIndex.indexPath);
     this.hasSparseGramIndex = existsSync(this.sparseGramIndexPath);
@@ -182,7 +206,7 @@ export class SweetSearch {
       // disable the entire 3-stage pipeline. Stage 2.5 falls back to SQLite.
       if (this.hasBinaryHnswIndex) {
         try {
-          const floatStorePath = getFloatStorePath(DB_PATHS.binaryHnswIndex);
+          const floatStorePath = getFloatStorePath(this.binaryHnswPath);
           const floatLoaded = await this.floatVectorStore.load(floatStorePath);
           if (floatLoaded) {
             const fStats = this.floatVectorStore.getStats();
@@ -212,15 +236,65 @@ export class SweetSearch {
         const stats = this.lateInteractionIndex.getStats();
         this.log(`LateInteraction: Loaded ${stats.documents} documents (${stats.estimatedSizeMB} MB, ${stats.avgTokensPerDoc} avg tokens)`);
 
-        // Preheat LI ONNX inference model (~900ms cold start otherwise).
-        // The index loads token vectors; this loads the query encoder model.
-        const { encodeQuery } = await import('../ranking/late-interaction-model.js');
-        await encodeQuery('warmup');
-        this.log('LateInteraction: ONNX model preheated');
+        // Re-resolve the rerank policy now that the index header has been
+        // loaded — the manifest's modelId is the source of truth for the
+        // auto rule (edge index → off, standard index → on, mismatch → off).
+        // The constructor only had the active LATE_INTERACTION_CONFIG.model
+        // to work with; this call corrects the decision when index and
+        // config disagree (env var changed, model bumped, etc.).
+        const manifest = {
+          modelId: this.lateInteractionIndex.modelId ?? null,
+          tokenDim: this.lateInteractionIndex.tokenDim ?? null,
+          modelMismatch: this.lateInteractionIndex.modelMismatch === true,
+          exists: true,
+        };
+        const resolved = resolveSearchRerankPolicy({
+          optionOverride: this._liPolicyOptionOverride,
+          env: process.env,
+          persisted: this._liPolicyPersisted,
+          indexManifest: manifest,
+          activeConfigModel: LATE_INTERACTION_CONFIG.model,
+        });
+        this._liPolicyResolved = resolved;
+        const previouslyOn = this.useLateInteraction;
+        this.useLateInteraction = LATE_INTERACTION_CONFIG.enabled && resolved.effective;
+        this.log(
+          `LateInteraction: rerank policy → ${this.useLateInteraction ? 'on' : 'off'} `
+          + `(${resolved.reason}${manifest.modelId ? `, index=${manifest.modelId}` : ''})`,
+        );
+        if (resolved.warning) {
+          // One-line warning — keeps the log digestible, full guidance is
+          // in docs/BENCH_TODO.md.
+          console.warn(`[SweetSearch] ${resolved.warning}`);
+        }
+
+        if (this.useLateInteraction) {
+          // Preheat LI ONNX inference model (~900ms cold start otherwise).
+          // Only when we will actually rerank — saves cold-start cost when
+          // policy resolves to off post-manifest-inspection.
+          const { encodeQuery } = await import('../ranking/late-interaction-model.js');
+          await encodeQuery('warmup');
+          this.log('LateInteraction: ONNX model preheated');
+        } else if (previouslyOn) {
+          // We loaded the index because the constructor's tentative
+          // resolution said on, but the manifest just told us otherwise —
+          // log so the user understands the gap between "index present"
+          // and "rerank active".
+          this.log('LateInteraction: index loaded but search rerank disabled by policy (read-semantic + ColGrep still use the index)');
+        }
       } catch (err) {
         this.log(`LateInteraction: Failed to load: ${err.message}`);
         this.hasLateInteractionIndex = false;
+        this.useLateInteraction = false;
       }
+    } else if (this.hasLateInteractionIndex && !this.useLateInteraction) {
+      // Index present but constructor-time policy resolved to off.
+      // Skip the (expensive) load + encoder warmup — read-semantic and
+      // ColGrep both lazy-load their own LI handle when actually invoked.
+      this.log(
+        `LateInteraction: index present, search rerank disabled by policy `
+        + `(${this._liPolicyResolved?.reason ?? 'unknown'})`,
+      );
     }
 
     if (this.hasSparseGramIndex) {
@@ -419,9 +493,39 @@ export class SweetSearch {
     }
 
     // Step 3: Post-retrieval processing (delegated to extracted module)
-    return this._applyPostRetrieval(results, query, options, {
+    const postRetrievalResult = await this._applyPostRetrieval(results, query, options, {
       stats, semanticStats, searchMode, effectiveGraphExpand, intentPolicy, start,
     });
+
+    // Step 4: Agent packaging (lexical/semantic/hybrid/structural).
+    // The pattern (colgrep) branch already returns its own pre-packaged response
+    // and short-circuits earlier in this method. For non-pattern modes, apply the
+    // shared packager when the caller explicitly asked for an agent format.
+    // Default behavior (no agent format) is unchanged.
+    const agentFormats = new Set(['agent', 'agent_preview', 'agent_full', 'agent_full_xl']);
+    if (agentFormats.has(options.format)) {
+      const finalResults = postRetrievalResult.results || [];
+      const finalStats = postRetrievalResult.stats || {};
+      const agentResponse = packageForAgent(finalResults, {
+        ...finalStats,
+        candidatePoolSize: finalStats.results_count ?? finalResults.length,
+      }, {
+        query,
+        regex: regex || '',
+        mode: finalStats.path || searchMode,
+        format: options.format,
+        tokenBudget: options.tokenBudget,
+        codeGraphRepo: this.codeGraphRepo || null,
+        locationMap: null,
+        projectRoot: this.projectRoot,
+        ablations: options.ablations,
+      });
+      // Preserve the underlying retrieval stats so callers can inspect both layers
+      agentResponse.stats = finalStats;
+      return agentResponse;
+    }
+
+    return postRetrievalResult;
   }
 
   /** Structural search path (GraphRAG structural queries — opt-in via explicit flag) */

package/mcp/server.js

@@ -18,11 +18,15 @@ import {
   HealthOutputSchema,
   RepoMapOutputSchema,
   VocabPrewarmOutputSchema,
+  ReadOutputSchema,
+  ReadSemanticOutputSchema,
   handleSearch,
   handleIndex,
   checkHealth,
   handleRepoMap,
   handleVocabPrewarm,
+  handleRead,
+  handleReadSemantic,
 } from './tool-handlers.js';
 
 const __filename = fileURLToPath(import.meta.url);
@@ -224,6 +228,43 @@ server.registerTool('vocab-prewarm', {
   },
 }, async (args) => handleVocabPrewarm(args, vocabDeps));
 
+server.registerTool('read', {
+  description: 'Read one or more files for exact code understanding. Replaces the default Read tool for most code-reading workflows. Uses the filesystem as ground truth, supports line ranges and batching, and attaches symbol-aware chunk metadata when the file is indexed.',
+  inputSchema: {
+    files: z.array(z.object({
+      path: z.string().describe('File path relative to project root (or absolute)'),
+      startLine: z.number().int().min(1).optional().describe('Start line (1-based, inclusive)'),
+      endLine: z.number().int().min(1).optional().describe('End line (1-based, inclusive)'),
+    })).min(1).max(20).describe('Files to read (1-20)'),
+    includeMetadata: z.boolean().default(true).optional()
+      .describe('Attach symbol-aware chunk metadata when the file is indexed'),
+  },
+  outputSchema: ReadOutputSchema,
+  annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
+}, async (args) => handleRead(args, { PROJECT_ROOT }));
+
+server.registerTool('read-semantic', {
+  description: 'Read only the spans of a file relevant to a query. Selects spans via hybrid retrieval (lexical + symbol + ColBERT-style late-interaction MaxSim) with RRF fusion and LI re-rank, then re-reads exact lines from disk. Returns 1-N small spans instead of the full file. Falls back to a plain read if the file is not indexed.',
+  inputSchema: {
+    file: z.string().describe('File path (project-relative or absolute)'),
+    query: z.string().min(1).max(500).describe('What you want to understand about this file'),
+    topK: z.number().int().min(1).max(20).default(5).optional()
+      .describe('Maximum spans before merging (default: 5)'),
+    threshold: z.number().min(0).max(1).default(0.4).optional()
+      .describe('MaxSim score floor (default: 0.4)'),
+    contextLines: z.number().int().min(0).max(20).default(2).optional()
+      .describe('Pre/post context lines per span (default: 2)'),
+    maxChars: z.number().int().min(200).max(64000).default(8000).optional()
+      .describe('Hard cap on returned text (default: 8000 chars)'),
+    maxTokens: z.number().int().min(50).max(16000).optional()
+      .describe('Convenience cap (~chars/4)'),
+    verbose: z.boolean().default(false).optional()
+      .describe('Include timings + per-signal scores'),
+  },
+  outputSchema: ReadSemanticOutputSchema,
+  annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
+}, async (args) => handleReadSemantic(args, { PROJECT_ROOT }));
+
 // ---------------------------------------------------------------------------
 // Resources
 // ---------------------------------------------------------------------------

package/mcp/tool-handlers.js

@@ -88,6 +88,65 @@ export const VocabPrewarmOutputSchema = z.object({
   dryRun: z.boolean().optional(),
 });
 
+const ReadFileResultSchema = z.object({
+  file: z.string(),
+  absolutePath: z.string().optional(),
+  ok: z.boolean(),
+  exact: z.boolean().optional(),
+  indexed: z.boolean().optional(),
+  language: z.string().nullable().optional(),
+  totalLines: z.number().int().optional(),
+  bytes: z.number().int().optional(),
+  mtimeMs: z.number().optional(),
+  range: z.object({
+    startLine: z.number().int(),
+    endLine: z.number().int(),
+  }).nullable().optional(),
+  text: z.string().optional(),
+  chunks: z.array(z.object({
+    id: z.string(),
+    symbol: z.string().nullable().optional(),
+    type: z.string().nullable().optional(),
+    startLine: z.number().int().nullable().optional(),
+    endLine: z.number().int().nullable().optional(),
+    signature: z.string().nullable().optional(),
+  })).optional(),
+  error: z.string().optional(),
+  timings: z.object({ totalMs: z.number() }).optional(),
+});
+
+export const ReadOutputSchema = z.object({
+  files: z.array(ReadFileResultSchema),
+  totalMs: z.number(),
+});
+
+const ReadSemanticSpanSchema = z.object({
+  startLine: z.number().int(),
+  endLine: z.number().int(),
+  score: z.number(),
+  symbols: z.array(z.string()).optional(),
+  types: z.array(z.string()).optional(),
+  chunkIds: z.array(z.string()).optional(),
+  text: z.string(),
+  truncated: z.boolean().optional(),
+});
+
+export const ReadSemanticOutputSchema = z.object({
+  file: z.string(),
+  query: z.string(),
+  ok: z.boolean(),
+  indexed: z.boolean(),
+  fellBack: z.boolean(),
+  reason: z.string().optional(),
+  language: z.string().nullable().optional(),
+  totalLines: z.number().int().optional(),
+  spans: z.array(ReadSemanticSpanSchema),
+  charsReturned: z.number().int().optional(),
+  approxTokensReturned: z.number().int().optional(),
+  signals: z.record(z.string(), z.any()).optional(),
+  timings: z.record(z.string(), z.number()).optional(),
+});
+
 // ---------------------------------------------------------------------------
 // Internal state for health DB cache (module-scoped, not exported)
 // ---------------------------------------------------------------------------
@@ -104,11 +163,6 @@ let _healthDb = null;
  */
 export async function handleSearch({ query, k, mode, structural, regex, format, tokenBudget }, { getSearcher }) {
   try {
-    // Agent format requires a regex (pattern search). If no regex, ignore the format
-    // to avoid silent fallback to non-pattern search without agent packaging.
-    const isAgentFormat = format && format.startsWith('agent');
-    const effectiveFormat = (isAgentFormat && !regex) ? undefined : format;
-
     const searcher = await getSearcher();
     const searchMode = structural ? 'structural' : mode;
     const searchResult = await searcher.search(query, {
@@ -117,7 +171,7 @@ export async function handleSearch({ query, k, mode, structural, regex, format,
       expand: true,
       rerank: true,
       ...(regex && { regex }),
-      ...(effectiveFormat && { format: effectiveFormat }),
+      ...(format && { format }),
       ...(tokenBudget && { tokenBudget }),
     });
 
@@ -474,3 +528,60 @@ export async function handleVocabPrewarm({ depth, modes, top, incremental, dryRu
     };
   }
 }
+
+// ---------------------------------------------------------------------------
+// read — filesystem-grounded reader
+// ---------------------------------------------------------------------------
+
+/**
+ * @param {{ files: Array<{path: string, startLine?: number, endLine?: number}>, includeMetadata?: boolean }} args
+ * @param {{ PROJECT_ROOT: string }} deps
+ */
+export async function handleRead(args, deps) {
+  try {
+    const { readFiles, formatReadResults } = await import('../core/search/index.js');
+    const result = await readFiles(args.files || [], {
+      projectRoot: deps.PROJECT_ROOT,
+      includeMetadata: args.includeMetadata !== false,
+    });
+    return {
+      content: [{ type: 'text', text: formatReadResults(result, 'agent') }],
+      structuredContent: result,
+    };
+  } catch (err) {
+    const msg = (err.message || 'read failed').split('\n')[0];
+    return { content: [{ type: 'text', text: `read error: ${msg}` }], isError: true };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// read-semantic — hybrid span selection + filesystem-grounded re-read
+// ---------------------------------------------------------------------------
+
+/**
+ * @param {{ file: string, query: string, topK?: number, threshold?: number, contextLines?: number, maxChars?: number, maxTokens?: number, verbose?: boolean }} args
+ * @param {{ PROJECT_ROOT: string }} deps
+ */
+export async function handleReadSemantic(args, deps) {
+  try {
+    const { readSemantic, formatReadSemanticResult } = await import('../core/search/index.js');
+    const result = await readSemantic({
+      path: args.file,
+      query: args.query,
+      topK: args.topK,
+      threshold: args.threshold,
+      contextLines: args.contextLines,
+      maxChars: args.maxChars,
+      maxTokens: args.maxTokens,
+      projectRoot: deps.PROJECT_ROOT,
+      verbose: args.verbose,
+    });
+    return {
+      content: [{ type: 'text', text: formatReadSemanticResult(result, 'agent') }],
+      structuredContent: result,
+    };
+  } catch (err) {
+    const msg = (err.message || 'read-semantic failed').split('\n')[0];
+    return { content: [{ type: 'text', text: `read-semantic error: ${msg}` }], isError: true };
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sweet-search",
-  "version": "2.4.2",
+  "version": "2.5.1",
   "description": "Sweet Search - SOTA Hybrid Code Search Engine with WASM CatBoost Query Router, Semantic/Lexical/Structural Search, and Multilingual Support",
   "type": "module",
   "main": "core/search/sweet-search.js",
@@ -99,6 +99,8 @@
     "eval:latency": "node eval/scripts/latency-stress.js",
     "eval:multirepo": "node eval/scripts/multirepo-bench.js",
     "eval:multirepo:test": "node eval/scripts/multirepo-bench.js --split=test",
+    "bench:read-workflows": "node eval/read-workflows/run-bench.js",
+    "bench:agent-read-workflows": "node eval/agent-read-workflows/run-bench.js",
     "eval:fetch-repos": "node eval/scripts/fetch-benchmark-repos.js",
     "features": "node core/training/query-router/features/extractor.js",
     "features:benchmark": "node core/training/query-router/features/extractor.js --benchmark",
@@ -140,12 +142,12 @@
     "vitest": "^4.0.16"
   },
   "optionalDependencies": {
-    "@sweet-search/native-darwin-arm64": "2.4.2",
-    "@sweet-search/native-darwin-x64": "2.4.2",
-    "@sweet-search/native-linux-arm64-gnu": "2.4.2",
-    "@sweet-search/native-linux-arm64-gnu-cuda": "2.4.2",
-    "@sweet-search/native-linux-x64-gnu": "2.4.2",
-    "@sweet-search/native-linux-x64-gnu-cuda": "2.4.2"
+    "@sweet-search/native-darwin-arm64": "2.5.1",
+    "@sweet-search/native-darwin-x64": "2.5.1",
+    "@sweet-search/native-linux-arm64-gnu": "2.5.1",
+    "@sweet-search/native-linux-arm64-gnu-cuda": "2.5.1",
+    "@sweet-search/native-linux-x64-gnu": "2.5.1",
+    "@sweet-search/native-linux-x64-gnu-cuda": "2.5.1"
   },
   "engines": {
     "node": ">=18.0.0"