sweet-search 2.5.2 → 2.5.4

package/core/search/search-pattern.js

@@ -17,7 +17,9 @@ import { PROJECT_ROOT } from '../infrastructure/config/index.js';
 import { generateRegexMatches } from './search-pattern-planner.js';
 import { buildBareGrepResults, filterMatchesBySymbolType, resolveSearchSymbolFilter, mapMatchesToChunks, readFileRange } from './search-pattern-chunks.js';
 import { isRipgrepAvailable, runRipgrepJson } from './search-pattern-ripgrep.js';
+import { ensureSparseGramIndex } from './search-pattern-prefilter.js';
 import { packageForAgent } from './context-expander.js';
+import { applyFileKindRanking, applyResultDemotions } from '../ranking/file-kind-ranking.js';
 
 // =============================================================================
 // Ripgrep runner (thin wrapper for external callers)
@@ -74,11 +76,61 @@ export function mergeRegexIntoQuery(query, regex) {
   return `${query} ${novel.join(' ')}`;
 }
 
+// =============================================================================
+// Grep engine selection — native in-process grep vs ripgrep fallback
+// =============================================================================
+
+/**
+ * Whether native in-process grep can serve this request without ripgrep.
+ *
+ * Native unified/narrowed grep (searchFull/searchLines + nativeGrep*) covers
+ * plain-regex queries whenever a sparse-gram index is loaded. The loaded index
+ * object only exposes searchFull/searchLines when the native addon built it, so
+ * their presence is a reliable signal that native grep is available — and it
+ * keeps the check deterministic for tests that supply a mock index.
+ *
+ * Fixed-string and glob queries are gated off the native path in
+ * generateRegexMatches, so they still require ripgrep as the fallback engine.
+ */
+function nativeGrepCanServe(searcher, options = {}) {
+  const fixedString = options.fixedString ?? false;
+  const globs = options.globs ?? [];
+  if (fixedString || (Array.isArray(globs) && globs.length > 0)) return false;
+  const index = ensureSparseGramIndex(searcher, options);
+  return !!(index && typeof index.searchFull === 'function' && typeof index.searchLines === 'function');
+}
+
+/**
+ * Fail fast when no grep engine can serve the request. Native in-process grep
+ * is preferred and needs no external binary; ripgrep is an optional fallback
+ * (required only for fixed-string/glob queries, or when the native addon/index
+ * is absent). Throws a single actionable error naming both engines when neither
+ * is available — generateRegexMatches itself prefers native and only reaches a
+ * ripgrep call in the fallback branches this guard protects.
+ *
+ * @returns {Promise<boolean>} true when native grep will serve (ripgrep unused)
+ */
+async function ensureGrepEngineAvailable(searcher, options, label) {
+  if (nativeGrepCanServe(searcher, options)) return true;
+  if (await isRipgrepAvailable()) return false;
+
+  const fixedString = options.fixedString ?? false;
+  const globs = options.globs ?? [];
+  const reason = (fixedString || (Array.isArray(globs) && globs.length > 0))
+    ? 'fixed-string and glob queries use the ripgrep fallback, which is not installed'
+    : 'native grep is unavailable (no sparse-gram index built, or the native addon is missing) and ripgrep is not installed';
+  throw new Error(
+    `${label} needs an in-process grep engine, but none is available: ${reason}. ` +
+    'Re-index to build the native sparse-gram index, or install ripgrep (brew install ripgrep).'
+  );
+}
+
 // =============================================================================
 // Bare grep (wired onto SweetSearch.prototype)
 // =============================================================================
 
 export async function bareGrep(query, routing, options = {}) {
+  await this?._refreshManifestPins?.({ reloadScope: 'grep' });
   const regex = options.regex || query;
   const searchDir = this?.projectRoot || options.projectRoot || PROJECT_ROOT;
   const maxMatches = options.maxMatches ?? 0;
@@ -91,9 +143,10 @@ export async function bareGrep(query, routing, options = {}) {
     throw new Error('Bare grep requires a regex or fixed-string pattern.');
   }
 
-  if (!await isRipgrepAvailable()) {
-    throw new Error('Bare grep requires ripgrep (rg). Install: brew install ripgrep');
-  }
+  // Native in-process grep serves this when a sparse-gram index is loaded;
+  // ripgrep is only required for fixed-string/glob queries or when native is
+  // unavailable. Throws a clear error only when neither engine can run.
+  await ensureGrepEngineAvailable(this, options, 'Bare grep');
 
   // Disable chunk gram for bare grep — bare grep uses file:line matches, not chunk IDs.
   const candidateResult = await generateRegexMatches(this || {}, regex, searchDir, options);
@@ -161,6 +214,7 @@ export async function bareGrep(query, routing, options = {}) {
  * Uses `this` — must be wired onto SweetSearch.prototype.
  */
 export async function patternSearch(query, routing, options = {}) {
+  await this?._refreshManifestPins?.({ reloadScope: 'all' });
   const {
     regex,
     k = 10,
@@ -180,9 +234,10 @@ export async function patternSearch(query, routing, options = {}) {
     throw new Error('Pattern search requires a late interaction index. Re-index with late interaction enabled.');
   }
 
-  if (!await isRipgrepAvailable()) {
-    throw new Error('Pattern search requires ripgrep (rg). Install: brew install ripgrep');
-  }
+  // Native in-process grep serves candidate generation when a sparse-gram index
+  // is loaded; ripgrep is only required for fixed-string/glob queries or when
+  // native is unavailable. Throws a clear error only when neither engine runs.
+  await ensureGrepEngineAvailable(this, options, 'Pattern search');
 
   await this.lateInteractionIndex.init();
 
@@ -310,32 +365,15 @@ export async function patternSearch(query, routing, options = {}) {
       s.grepDensity = matchCount;
       s.lateInteractionScore = s.lateInteractionScore * (1 + GREP_DENSITY_ALPHA * Math.log(matchCount));
     }
-    const TEST_DEMOTION = options.testDemotion ?? 0.05;
-    if (TEST_DEMOTION > 0) {
-      const queryLower = query.toLowerCase();
-      const queryMentionsTest = /\btest|spec|describe|it\b/.test(queryLower);
-      if (!queryMentionsTest) {
-        for (const s of scored) {
-          const doc = this.lateInteractionIndex.documents.get(s.id);
-          const file = doc?.metadata?.file || '';
-          const name = doc?.metadata?.name || '';
-          if (/test|spec|__test__|\.test\.|\.spec\./.test(file) ||
-              /test|spec/i.test(name)) {
-            s.lateInteractionScore -= TEST_DEMOTION;
-          }
-        }
-      }
-    }
-
     scored.sort((a, b) => b.lateInteractionScore - a.lateInteractionScore);
     log(`MaxSim rerank: ${scored.length} candidates in ${rerankTime.toFixed(1)}ms`);
   }
 
   const fileCache = new Map();
 
-  const results = scored.slice(0, k).map((s, rank) => {
+  let rankedResults = scored.map((s, rank) => {
     const doc = this.lateInteractionIndex.documents.get(s.id);
-    const meta = doc?.metadata || {};
+    const meta = doc?.metadata || this.lateInteractionIndex.aliasPointers?.get(s.id)?.metadata || {};
     const text = readFileRange(fileCache, meta.file, meta.startLine, meta.endLine, this.projectRoot);
 
     return {
@@ -355,6 +393,56 @@ export async function patternSearch(query, routing, options = {}) {
       metadata: meta,
     };
   });
+  // Mode E (2026-05-13): for agent-format ss-find queries, apply file-kind
+  // demotion before result demotions. The regex+symbol shape of ss-find is
+  // implementation-intent by construction (agents using patternSearch are
+  // hunting for code, not docs), so we force intent='implementation' rather
+  // than inferring it from the NL query — `classifyFileKindIntent` requires
+  // explicit "show me the function/class" verbs that ss-find prompts don't
+  // always include.
+  //
+  // CRITICAL — format-gated to agent variants only. Same gate as the BM25F
+  // boosts in applyResultDemotions (round-1/2 lessons: -27.57pp GCSN if
+  // ungated structural signals fire on benchmark NL traffic). Probes use
+  // format='agent', so behaviour matches expectations; GCSN bench uses
+  // mode='auto' with no format, so this skip preserves the 86.92% baseline.
+  //
+  // Targets stage3-taxonomy.md Mode E failures:
+  //   - JS-005: index.d.ts: interface AxiosHeaders outranks lib/core/AxiosHeaders.js
+  //   - TSL-004/008: packages/docs/content/packages/core.mdx outranks schemas.ts
+  // The .d.ts (types kind) and .mdx (docs kind) factors mirror hybrid's
+  // tuned defaults (typeFactor 0.70, docFactor 0.35).
+  const ssFindIsAgentFormat = options?.format === 'agent'
+    || options?.format === 'agent_full'
+    || options?.format === 'agent_full_xl'
+    || options?.format === 'agent_preview';
+  const skipPatternFileKind = Array.isArray(ablations)
+    ? ablations.includes('no-pattern-file-kind-ranking')
+    : (ablations instanceof Set ? ablations.has('no-pattern-file-kind-ranking') : false);
+  if (ssFindIsAgentFormat && !skipPatternFileKind) {
+    rankedResults = applyFileKindRanking(rankedResults, {
+      intent: 'implementation',
+      window: options.fileKindWindow ?? 100,
+      docFactor: options.patternDocFactor ?? 0.35,
+      testFactor: options.patternTestFactor ?? 0.35,
+      typeFactor: options.patternTypeFactor ?? 0.70,
+      ancillaryFactor: options.patternAncillaryFactor ?? 0.15,
+      tinyAncillaryFactor: options.patternTinyAncillaryFactor ?? 0.05,
+    });
+  }
+  rankedResults = applyResultDemotions(rankedResults, {
+    query,
+    ablations,
+    format: options?.format,
+    projectRoot: this.projectRoot,
+    codeGraphRepo: this.codeGraphRepo,
+  }).map((result, rank) => ({
+    ...result,
+    rank: rank + 1,
+    lateInteractionScore: result.score,
+  }));
+
+  const results = rankedResults.slice(0, k);
 
   const remaining = Math.max(0, k - results.length);
   if (remaining > 0 && unindexedMatches.length > 0) {
@@ -365,7 +453,7 @@ export async function patternSearch(query, routing, options = {}) {
       const key = `${m.file}:${m.line}`;
       if (seen.has(key)) continue;
       seen.add(key);
-
+      const content = m.content || readFileRange(fileCache, m.file, m.line, m.line, this.projectRoot) || m.matchText || '';
       results.push({
         id: `unindexed:${m.file}:${m.line}`,
         file: m.file,
@@ -373,8 +461,8 @@ export async function patternSearch(query, routing, options = {}) {
         type: 'code',
         startLine: m.line,
         endLine: m.line,
-        text: m.content,
-        content: m.content,
+        text: content,
+        content,
         score: 0,
         lateInteractionScore: 0,
         rank: results.length + 1,
@@ -461,4 +549,4 @@ export { generateRegexMatches } from './search-pattern-planner.js';
 export { hasCaseInsensitiveRegexFlag, extractRequiredLiteralsHeuristic, extractLiteralClausesHeuristic, extractLiteralClauses, normalizeLiteralClauses, querySparseGramCandidates, ensureSparseGramIndex, nativeGrepFilesWithMatches, nativeGrepLines, getSparseGramAllFiles } from './search-pattern-prefilter.js';
 export { buildChunkLocationMap, findChunkForLine, findChunkIntervalForLine, mapMatchesToChunks, readFileRange, getChunkLocationMap, getCodebaseChunkTypeMap, normalizeSearchSymbolType, resolveSearchSymbolFilter, isRipgrepCodePath, buildBareGrepResults, filterMatchesBySymbolType } from './search-pattern-chunks.js';
 export { isRipgrepAvailable, _resetRgCache, normalizeSearchPath, chunkRipgrepFiles } from './search-pattern-ripgrep.js';
-export { packageForAgent, estimateTokens, computeConfidence, computeSufficiency, allocateBudget, expandToSymbol, expandBySyntax, extractHeaderContext, truncateToTokenCap, findEnclosingEntity, checkStaleness } from './context-expander.js';
+export { packageForAgent, estimateTokens, computeConfidence, computeSufficiency, allocateBudget, expandToSymbol, expandBySyntax, expandLeadingTrivia, extractHeaderContext, truncateToTokenCap, findEnclosingEntity, checkStaleness, renderGraphNeighbors } from './context-expander.js';