sweet-search 2.6.12 → 2.6.13

package/core/search/grep-output-shaping.js

@@ -0,0 +1,193 @@
+/**
+ * Agent-facing grep output shaping: k-budget file-level diversity.
+ *
+ * Root cause (task bench, rstudio-education__gradethis-161): bareGrep returns
+ * matches sorted alphabetically by file and the agent wrapper printed the
+ * first k grouped by file — so one flooded early-alphabet file could consume
+ * the entire k budget and structurally hide every other matching file, with
+ * no signal that elision happened.
+ *
+ * These helpers are pure and option-gated: only the ss-grep agent wrapper
+ * enables them. The human-facing grep product shape and all NL ranking paths
+ * are untouched (no scoring — file order stays the engine's deterministic
+ * sorted order; raw match count is deliberately never used as a sort key).
+ */
+
+/**
+ * True when a match's repo-relative path is the drill-in target. Accepts the
+ * exact relative path as printed in grep output, a path with a leading "./",
+ * or a suffix (basename or trailing path segments) so agents can paste any
+ * reasonable spelling of the file they saw.
+ *
+ * @param {string} file - repo-relative match path (as emitted by the engine)
+ * @param {string} filter - user-supplied --in value
+ */
+export function matchesGrepFileFilter(file, filter) {
+  if (!file || !filter) return false;
+  const f = String(filter).replace(/^\.\//, '').replace(/\\/g, '/');
+  const target = String(file).replace(/\\/g, '/');
+  return target === f || target.endsWith(`/${f}`);
+}
+
+/**
+ * Streaming per-file diversification of a (file,line)-sorted match list.
+ * Keeps at most `perFileCap` matches per file and at most `maxFiles` distinct
+ * files; everything beyond is COUNTED but never stored, so memory is bounded
+ * by perFileCap*maxFiles regardless of total match count.
+ *
+ * PRECONDITION: matches must be grouped by file (bareGrep sorts by
+ * (file, line, column) before calling) — the walk uses a single per-file
+ * cursor precisely so no map proportional to distinct-file count is built.
+ *
+ * @param {Array<{file: string}>} matches - sorted by (file, line)
+ * @param {{perFileCap: number, maxFiles?: number, hiddenSampleSize?: number}} opts
+ * @returns {{kept: Array, fileSummary: {
+ *   files: Array<{file: string, total: number, kept: number}>,
+ *   hiddenFileCount: number, hiddenMatchCount: number,
+ *   hiddenSample: Array<{file: string, total: number}>,
+ * }}}
+ */
+export function applyGrepFileDiversity(matches, opts = {}) {
+  const perFileCap = Math.max(1, opts.perFileCap | 0);
+  const maxFiles = opts.maxFiles > 0 ? (opts.maxFiles | 0) : Infinity;
+  const hiddenSampleSize = opts.hiddenSampleSize ?? 3;
+
+  const kept = [];
+  const files = [];            // [{file, total, kept}] in first-appearance (sorted) order
+  const hiddenSample = [];     // first few hidden files, name + total
+  let hiddenFileCount = 0;
+  let hiddenMatchCount = 0;
+
+  let current = null;
+  for (const m of matches) {
+    if (!current || current.file !== m.file) {
+      if (files.length < maxFiles) {
+        current = { file: m.file, total: 0, kept: 0 };
+        files.push(current);
+      } else {
+        current = { file: m.file, total: 0, kept: -1 }; // hidden file: count only
+        hiddenFileCount++;
+        if (hiddenSample.length < hiddenSampleSize) {
+          hiddenSample.push(current);
+        }
+      }
+    }
+    current.total++;
+    if (current.kept >= 0 && current.kept < perFileCap) {
+      kept.push(m);
+      current.kept++;
+    } else if (current.kept < 0) {
+      hiddenMatchCount++;
+    }
+  }
+
+  return {
+    kept,
+    fileSummary: {
+      files,
+      hiddenFileCount,
+      hiddenMatchCount,
+      hiddenSample: hiddenSample.map(h => ({ file: h.file, total: h.total })),
+    },
+  };
+}
+
+/**
+ * Breadth-first budget allocation: round-robin one line per file per round,
+ * in the given (deterministic) file order, until `budget` lines are allocated
+ * or every file's available matches are exhausted. Flooding is structurally
+ * impossible — file A gets its (r+1)-th line only after every other file got
+ * its r-th — while few-files queries keep full depth (deep rounds fill just
+ * like the old flat output).
+ *
+ * @param {number[]} counts - available (fetched) matches per file
+ * @param {number} budget - total lines to allocate (k)
+ * @returns {number[]} allocation per file, same order as counts
+ */
+export function allocateGrepBudget(counts, budget) {
+  const alloc = counts.map(() => 0);
+  let remaining = Math.max(0, budget | 0);
+  let progressed = true;
+  while (remaining > 0 && progressed) {
+    progressed = false;
+    for (let i = 0; i < counts.length && remaining > 0; i++) {
+      if (alloc[i] < counts[i]) {
+        alloc[i]++;
+        remaining--;
+        progressed = true;
+      }
+    }
+  }
+  return alloc;
+}
+
+/**
+ * Render the diversified agent grep body: matches grouped per file (research:
+ * contiguous per-file blocks read better than interleaving) with allocation
+ * decided breadth-first, and a visible inline `(+N more in this file)` marker
+ * on the last shown line of every truncated file — elision is never silent
+ * and costs no extra lines.
+ *
+ * @param {Array<{file: string, line: number, matchText?: string}>} kept -
+ *   diversified matches in engine order (grouped by file)
+ * @param {{files: Array<{file, total, kept}>, hiddenFileCount, hiddenMatchCount,
+ *          hiddenSample: Array<{file, total}>}} fileSummary
+ * @param {number} k - body line budget
+ * @returns {{lines: string[], shownMatches: number, matchedFileCount: number,
+ *            truncatedFileCount: number, hiddenLine: string|null}}
+ */
+export function renderGrepBody(kept, fileSummary, k) {
+  const groups = new Map();
+  for (const m of kept) {
+    if (!groups.has(m.file)) groups.set(m.file, []);
+    groups.get(m.file).push(m);
+  }
+  const totals = new Map(fileSummary.files.map(f => [f.file, f.total]));
+  const ordered = [...groups.entries()];
+  const alloc = allocateGrepBudget(ordered.map(([, ms]) => ms.length), k);
+
+  const lines = [];
+  let shownMatches = 0;
+  let truncatedFileCount = 0;
+  const unallocated = []; // fetched files that got zero budget (more files than k)
+  ordered.forEach(([file, ms], i) => {
+    if (alloc[i] === 0) {
+      unallocated.push({ file, total: totals.get(file) ?? ms.length });
+      return;
+    }
+    const total = totals.get(file) ?? ms.length;
+    for (let j = 0; j < alloc[i]; j++) {
+      const m = ms[j];
+      const text = (m.matchText || '').replace(/\s+/g, ' ').trim().slice(0, 140);
+      let line = `${file}:${m.line}: ${text}`;
+      if (j === alloc[i] - 1 && total > alloc[i]) {
+        line += ` (+${total - alloc[i]} more in this file)`;
+        truncatedFileCount++;
+      }
+      lines.push(line);
+      shownMatches++;
+    }
+  });
+
+  // Files that matched but got no body line at all (more matching files than
+  // budget, or clipped by the engine's maxFiles fetch bound): one honest tail
+  // line naming the first few so the agent can jump straight to them.
+  const hiddenFiles = unallocated.length + fileSummary.hiddenFileCount;
+  const hiddenMatches = unallocated.reduce((a, f) => a + f.total, 0) + fileSummary.hiddenMatchCount;
+  let hiddenLine = null;
+  if (hiddenFiles > 0) {
+    const sample = [...unallocated.map(f => f.file), ...fileSummary.hiddenSample.map(f => f.file)]
+      .slice(0, 3);
+    hiddenLine = `# +${hiddenFiles} more file(s) with ${hiddenMatches} match(es)` +
+      (sample.length ? ` — e.g. ${sample.join(', ')}` : '') +
+      `; narrow the regex, raise -k, or drill in with --in <file>`;
+  }
+
+  return {
+    lines,
+    shownMatches,
+    matchedFileCount: fileSummary.files.length + fileSummary.hiddenFileCount,
+    truncatedFileCount,
+    hiddenLine,
+  };
+}

package/core/search/search-pattern.js

@@ -16,6 +16,7 @@
 import { PROJECT_ROOT } from '../infrastructure/config/index.js';
 import { generateRegexMatches } from './search-pattern-planner.js';
 import { buildBareGrepResults, filterMatchesBySymbolType, resolveSearchSymbolFilter, mapMatchesToChunks, readFileRange } from './search-pattern-chunks.js';
+import { applyGrepFileDiversity, matchesGrepFileFilter } from './grep-output-shaping.js';
 import { isRipgrepAvailable, runRipgrepJson } from './search-pattern-ripgrep.js';
 import { ensureSparseGramIndex } from './search-pattern-prefilter.js';
 import { packageForAgent } from './context-expander.js';
@@ -152,6 +153,11 @@ export async function bareGrep(query, routing, options = {}) {
   const candidateResult = await generateRegexMatches(this || {}, regex, searchDir, options);
   let matches = [...candidateResult.indexedMatches, ...candidateResult.overlayMatches];
   matches = filterMatchesBySymbolType(matches, symbolType, this);
+  // Agent drill-in scope (--in <file>): applied BEFORE sort/cap so a
+  // late-alphabet file's matches can never be pre-clipped by maxMatches.
+  if (options.fileFilter) {
+    matches = matches.filter(m => matchesGrepFileFilter(m.file, options.fileFilter));
+  }
   matches.sort((a, b) =>
     a.file.localeCompare(b.file) ||
     a.line - b.line ||
@@ -159,6 +165,16 @@ export async function bareGrep(query, routing, options = {}) {
   );
 
   const totalMatches = matches.length;
+  // Agent-only k-budget file diversity (option-gated; absent → byte-identical
+  // output). Streaming per-file cap: matches beyond the cap are counted, not
+  // stored, so memory is bounded by perFileCap*maxFiles, never total matches.
+  let fileSummary = null;
+  if (options.perFileCap > 0) {
+    ({ kept: matches, fileSummary } = applyGrepFileDiversity(matches, {
+      perFileCap: options.perFileCap,
+      maxFiles: options.maxFiles,
+    }));
+  }
   if (maxMatches > 0) {
     matches = matches.slice(0, maxMatches);
   }
@@ -170,6 +186,7 @@ export async function bareGrep(query, routing, options = {}) {
 
   return {
     results,
+    ...(fileSummary ? { fileSummary } : {}),
     stats: {
       path: 'grep',
       regex,

package/eval/agent-read-workflows/bin/_ss-helpers.mjs

@@ -18,6 +18,7 @@ import {
   buildGrepPattern, stripInertFlags, normalizeArgs, extractPositional,
   parseLineRange, looksLikeOption,
 } from './_ss-argparse.mjs';
+import { renderGrepBody } from '../../../core/search/grep-output-shaping.js';
 
 // Diagnostic-log isolation (agent-facing tools). The Sweet Search engine emits
 // model/index load banners via console.log → stdout ("LateInteraction: Loaded…",
@@ -115,13 +116,16 @@ async function ensureWarmServerReady({ timeoutMs = 60000, intervalMs = 500 } = {
 
 // --- subcommands ----------------------------------------------------------
 
-const GREP_USAGE = 'Usage: ss-grep <regex> [-i|--ignore-case] [-w|--word-regexp] [-F|--fixed-strings] [-k N]';
+const GREP_USAGE = 'Usage: ss-grep <regex> [-i|--ignore-case] [-w|--word-regexp] [-F|--fixed-strings] [--in <file>] [-k N]';
 async function cmdGrep(rawArgs) {
   const args = normalizeArgs(rawArgs);
   const ignoreCase = parseBoolFlag(args, ['-i', '--ignore-case']);
   const wordBound = parseBoolFlag(args, ['-w', '--word-regexp']);
   const fixedString = parseBoolFlag(args, ['-F', '--fixed-strings']);
   const k = readPositiveIntFlag(args, ['-k', '--top'], 20, GREP_USAGE);
+  // Drill-in scope: show matches from ONE file (the recovery affordance the
+  // diversified output advertises when it truncates a flooded file).
+  const inFile = readValueFlag(args, '--in', null, GREP_USAGE);
   stripInertFlags(args);
   const regex = buildGrepPattern(resolvePositional(args, GREP_USAGE), { ignoreCase, wordBound, fixedString });
   if (!regex) {
@@ -129,25 +133,47 @@ async function cmdGrep(rawArgs) {
     process.exit(2);
   }
   const s = await getSweetSearch();
-  const result = await s.bareGrep(regex, null, { regex, maxMatches: k * 5, contextLines: 0 });
-  // Group by file, take first k matches across all files (ordered as bareGrep returns).
-  const grouped = new Map();
-  for (const r of result.results.slice(0, k * 5)) {
-    if (!grouped.has(r.file)) grouped.set(r.file, []);
-    grouped.get(r.file).push(r);
-  }
-  let printed = 0;
-  process.stdout.write(`# ss-grep: ${result.results.length} total match(es) for /${regex}/\n`);
-  for (const [file, lines] of grouped) {
-    for (const r of lines) {
+
+  if (inFile) {
+    // Single-file scope: flat output, depth up to k within that file.
+    const result = await s.bareGrep(regex, null, {
+      regex, maxMatches: k, contextLines: 0, fileFilter: inFile,
+    });
+    const total = result.stats?.totalMatches ?? result.results.length;
+    process.stdout.write(`# ss-grep: ${total} total match(es) for /${regex}/ (scope: --in ${inFile})\n`);
+    result.results.forEach((r, i) => {
       const text = (r.matchText || '').replace(/\s+/g, ' ').trim().slice(0, 140);
-      process.stdout.write(`${file}:${r.line}: ${text}\n`);
-      printed++;
-      if (printed >= k) break;
-    }
-    if (printed >= k) break;
+      const marker = (i === result.results.length - 1 && total > result.results.length)
+        ? ` (+${total - result.results.length} more — raise -k)` : '';
+      process.stdout.write(`${r.file}:${r.line}: ${text}${marker}\n`);
+    });
+    if (result.results.length === 0) process.stdout.write('(no matches)\n');
+    process.exit(0);
+  }
+
+  // k-budget file diversity: fetch at most min(k,100) matches per file across
+  // at most k files (a file can never show more than k lines, and more than k
+  // files can never fit), then allocate the k body lines breadth-first so one
+  // flooded file can never hide every other matching file (the gradethis-161
+  // failure). Rendering stays grouped per file; truncation is marked inline
+  // and drillable via --in.
+  const result = await s.bareGrep(regex, null, {
+    regex, maxMatches: 0, contextLines: 0,
+    perFileCap: Math.min(k, 100), maxFiles: k,
+  });
+  const total = result.stats?.totalMatches ?? result.results.length;
+  const fileSummary = result.fileSummary
+    || { files: [], hiddenFileCount: 0, hiddenMatchCount: 0, hiddenSample: [] };
+  const body = renderGrepBody(result.results, fileSummary, k);
+
+  process.stdout.write(`# ss-grep: ${total} total match(es) for /${regex}/\n`);
+  if (body.truncatedFileCount > 0 || body.hiddenLine) {
+    process.stdout.write(`# ${body.matchedFileCount} file(s) matched; (+N more in this file)=truncated — ` +
+      `see the rest: ss-grep "<regex>" --in <file>\n`);
   }
-  if (printed === 0) process.stdout.write('(no matches)\n');
+  for (const line of body.lines) process.stdout.write(line + '\n');
+  if (body.hiddenLine) process.stdout.write(body.hiddenLine + '\n');
+  if (body.shownMatches === 0) process.stdout.write('(no matches)\n');
   process.exit(0);
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sweet-search",
-  "version": "2.6.12",
+  "version": "2.6.13",
   "description": "Sweet Search - SOTA Hybrid Code Search Engine with WASM CatBoost Query Router, Semantic/Lexical/Structural Search, and Multilingual Support",
   "type": "module",
   "main": "core/search/sweet-search.js",
@@ -167,13 +167,13 @@
     "vitest": "^4.0.16"
   },
   "optionalDependencies": {
-    "@sweet-search/native-darwin-arm64": "2.6.12",
-    "@sweet-search/native-darwin-x64": "2.6.12",
-    "@sweet-search/native-linux-arm64-gnu": "2.6.12",
-    "@sweet-search/native-linux-arm64-gnu-cuda": "2.6.12",
-    "@sweet-search/native-linux-x64-gnu": "2.6.12",
-    "@sweet-search/native-linux-x64-gnu-cuda": "2.6.12",
-    "@sweet-search/bg-priority": "2.6.12"
+    "@sweet-search/native-darwin-arm64": "2.6.13",
+    "@sweet-search/native-darwin-x64": "2.6.13",
+    "@sweet-search/native-linux-arm64-gnu": "2.6.13",
+    "@sweet-search/native-linux-arm64-gnu-cuda": "2.6.13",
+    "@sweet-search/native-linux-x64-gnu": "2.6.13",
+    "@sweet-search/native-linux-x64-gnu-cuda": "2.6.13",
+    "@sweet-search/bg-priority": "2.6.13"
   },
   "engines": {
     "node": ">=18.0.0"