sweet-search 2.6.12 → 2.6.14

package/core/search/context-expander.js

@@ -19,6 +19,7 @@
  */
 
 import { readFileRange } from './search-pattern-chunks.js';
+import { computeSufficiencyVerdict } from './query-sufficiency.js';
 import { statSync } from 'fs';
 import path from 'path';
 
@@ -1130,15 +1131,34 @@ function unresolvedExternalRefs(code, ownSymbolName, headerContext, neighborsRen
  *                            or in the surfaced neighbours list
  *   - high_confidence     : score gap puts top-1 well ahead of top-2
  *
- * sufficient := complete_symbol
- *               AND (header_resolved OR neighbors_present OR self_contained_strict)
- *               AND (high_confidence OR header_resolved OR neighbors_present)
+ * Query-conditioned rule (July 2026 — full200 bench forensics showed the
+ * structural rule mislabels systemically: `confidence=low sufficient=YES`
+ * was the modal trailer, 156×, and reasons were structural in ~all cases):
+ * the verdict is now 3-valued and REQUIRES positive query-match evidence
+ * in top-1 (see core/search/query-sufficiency.js):
+ *
+ *   yes     := strong query evidence (exact anchor hit or high subtoken
+ *              overlap) AND (confidence=high OR (confidence=medium AND
+ *              structural resolution))
+ *   no      := no results, or a query with real anchors finding nothing
+ *              in top-1 (confirmed absence)
+ *   unknown := everything ambiguous (incl. the old structural-only YES,
+ *              now reported as reason=well_formed_only)
+ *
+ * Structural facts (complete_symbol / header_resolved / neighbors_present /
+ * self_contained_strict) are kept as diagnostic reasons and as the
+ * resolution gate for medium-confidence YES — they can no longer produce
+ * YES on their own. `sufficient` (boolean) stays = (verdict === 'yes') for
+ * every structured consumer (MCP zod schema, bench counters).
  *
  * @param {object} topResult
  * @param {{ confidence: string }} confidenceInfo
- * @returns {{ sufficient: boolean, reasons: string[], unresolvedExternalCount: number }}
+ * @param {{ query?: string, regex?: string, lowerResults?: object[] }} [queryContext]
+ * @returns {{ sufficient: boolean, verdict: 'yes'|'no'|'unknown',
+ *             sufficiencyReason: string, reasons: string[],
+ *             unresolvedExternalCount: number, evidence: object|null }}
  */
-export function computeSufficiency(topResult, confidenceInfo) {
+export function computeSufficiency(topResult, confidenceInfo, queryContext = {}) {
   const reasons = [];
 
   const isComplete = !!(topResult.symbol &&
@@ -1168,16 +1188,31 @@ export function computeSufficiency(topResult, confidenceInfo) {
 
   if (confidenceInfo?.confidence === 'high') reasons.push('high_confidence');
 
-  // Tightened rule: complete symbol is necessary but NOT sufficient.
-  // We require at least one resolution reason AND at least one specificity
-  // reason. This stops a bare "complete + high_confidence" from claiming
-  // sufficient when the body still references a dozen helpers we never
-  // surfaced (the validation-pipeline failure mode).
   const hasResolution = hasHeader || hasNeighbors || reasons.includes('self_contained_strict');
-  const hasSpecificity = confidenceInfo?.confidence === 'high' || hasHeader || hasNeighbors;
-  const sufficient = isComplete && hasResolution && hasSpecificity;
 
-  return { sufficient, reasons, unresolvedExternalCount: unresolvedCount };
+  // Query-conditioned verdict: structural facts are the resolution gate,
+  // never the whole answer. See query-sufficiency.js for the fusion rule.
+  const { verdict, reason: sufficiencyReason, evidence } = computeSufficiencyVerdict({
+    topResult,
+    confidenceInfo,
+    query: queryContext.query || '',
+    regex: queryContext.regex || '',
+    structural: { isComplete, hasResolution },
+    lowerResults: queryContext.lowerResults || [],
+  });
+
+  if (evidence?.exactHit) reasons.push('query_literal_matched');
+  else if (evidence?.strength === 'strong') reasons.push('query_token_overlap');
+  else if (evidence?.strength === 'none') reasons.push('no_query_evidence');
+
+  return {
+    sufficient: verdict === 'yes',
+    verdict,
+    sufficiencyReason,
+    reasons,
+    unresolvedExternalCount: unresolvedCount,
+    evidence,
+  };
 }
 
 // =============================================================================
@@ -2287,15 +2322,27 @@ export function packageForAgent(rankedResults, searchStats, opts) {
 
   const packagingMs = Math.round(performance.now() - start);
 
-  // Sufficiency signal for top-1. Tightened in 2026-05: requires resolution
-  // (header_resolved OR neighbors_present OR self_contained_strict) instead
-  // of the old "complete_symbol + high_confidence" rule.
+  // Sufficiency signal for top-1. Query-conditioned since 2026-07: the
+  // 3-valued verdict requires positive query-match evidence; structural
+  // packaging facts remain as reasons + the medium-confidence resolution
+  // gate. Runs for summary-only top-1 too (verdict can be 'no'/'unknown'
+  // there; never 'yes' without code).
   let sufficient = false;
+  let sufficiencyVerdict = 'no';
+  let sufficiencyReason = 'no_results';
   let sufficiencyReasons = [];
   let unresolvedExternalCount = 0;
-  if (agentResults.length > 0 && agentResults[0].code) {
-    const sufficiency = computeSufficiency(agentResults[0], confidenceInfo);
+  if (agentResults.length > 0) {
+    const sufficiency = computeSufficiency(agentResults[0], confidenceInfo, {
+      query,
+      regex,
+      // Code-bearing lower ranks (full/preview) — soften a false 'no' when
+      // the pack's answer sits below top-1; never used to grant YES.
+      lowerResults: agentResults.slice(1, 4).filter(r => r.code),
+    });
     sufficient = sufficiency.sufficient;
+    sufficiencyVerdict = sufficiency.verdict;
+    sufficiencyReason = sufficiency.sufficiencyReason;
     sufficiencyReasons = sufficiency.reasons;
     unresolvedExternalCount = sufficiency.unresolvedExternalCount || 0;
   }
@@ -2317,6 +2364,8 @@ export function packageForAgent(rankedResults, searchStats, opts) {
     confidence: confidenceInfo.confidence,
     confidenceReason: confidenceInfo.confidenceReason,
     sufficient,
+    sufficiencyVerdict,
+    sufficiencyReason,
     sufficiencyReasons,
     unresolvedExternalCount,
 

package/core/search/grep-output-shaping.js

@@ -0,0 +1,193 @@
+/**
+ * Agent-facing grep output shaping: k-budget file-level diversity.
+ *
+ * Root cause (task bench, rstudio-education__gradethis-161): bareGrep returns
+ * matches sorted alphabetically by file and the agent wrapper printed the
+ * first k grouped by file — so one flooded early-alphabet file could consume
+ * the entire k budget and structurally hide every other matching file, with
+ * no signal that elision happened.
+ *
+ * These helpers are pure and option-gated: only the ss-grep agent wrapper
+ * enables them. The human-facing grep product shape and all NL ranking paths
+ * are untouched (no scoring — file order stays the engine's deterministic
+ * sorted order; raw match count is deliberately never used as a sort key).
+ */
+
+/**
+ * True when a match's repo-relative path is the drill-in target. Accepts the
+ * exact relative path as printed in grep output, a path with a leading "./",
+ * or a suffix (basename or trailing path segments) so agents can paste any
+ * reasonable spelling of the file they saw.
+ *
+ * @param {string} file - repo-relative match path (as emitted by the engine)
+ * @param {string} filter - user-supplied --in value
+ */
+export function matchesGrepFileFilter(file, filter) {
+  if (!file || !filter) return false;
+  const f = String(filter).replace(/^\.\//, '').replace(/\\/g, '/');
+  const target = String(file).replace(/\\/g, '/');
+  return target === f || target.endsWith(`/${f}`);
+}
+
+/**
+ * Streaming per-file diversification of a (file,line)-sorted match list.
+ * Keeps at most `perFileCap` matches per file and at most `maxFiles` distinct
+ * files; everything beyond is COUNTED but never stored, so memory is bounded
+ * by perFileCap*maxFiles regardless of total match count.
+ *
+ * PRECONDITION: matches must be grouped by file (bareGrep sorts by
+ * (file, line, column) before calling) — the walk uses a single per-file
+ * cursor precisely so no map proportional to distinct-file count is built.
+ *
+ * @param {Array<{file: string}>} matches - sorted by (file, line)
+ * @param {{perFileCap: number, maxFiles?: number, hiddenSampleSize?: number}} opts
+ * @returns {{kept: Array, fileSummary: {
+ *   files: Array<{file: string, total: number, kept: number}>,
+ *   hiddenFileCount: number, hiddenMatchCount: number,
+ *   hiddenSample: Array<{file: string, total: number}>,
+ * }}}
+ */
+export function applyGrepFileDiversity(matches, opts = {}) {
+  const perFileCap = Math.max(1, opts.perFileCap | 0);
+  const maxFiles = opts.maxFiles > 0 ? (opts.maxFiles | 0) : Infinity;
+  const hiddenSampleSize = opts.hiddenSampleSize ?? 3;
+
+  const kept = [];
+  const files = [];            // [{file, total, kept}] in first-appearance (sorted) order
+  const hiddenSample = [];     // first few hidden files, name + total
+  let hiddenFileCount = 0;
+  let hiddenMatchCount = 0;
+
+  let current = null;
+  for (const m of matches) {
+    if (!current || current.file !== m.file) {
+      if (files.length < maxFiles) {
+        current = { file: m.file, total: 0, kept: 0 };
+        files.push(current);
+      } else {
+        current = { file: m.file, total: 0, kept: -1 }; // hidden file: count only
+        hiddenFileCount++;
+        if (hiddenSample.length < hiddenSampleSize) {
+          hiddenSample.push(current);
+        }
+      }
+    }
+    current.total++;
+    if (current.kept >= 0 && current.kept < perFileCap) {
+      kept.push(m);
+      current.kept++;
+    } else if (current.kept < 0) {
+      hiddenMatchCount++;
+    }
+  }
+
+  return {
+    kept,
+    fileSummary: {
+      files,
+      hiddenFileCount,
+      hiddenMatchCount,
+      hiddenSample: hiddenSample.map(h => ({ file: h.file, total: h.total })),
+    },
+  };
+}
+
+/**
+ * Breadth-first budget allocation: round-robin one line per file per round,
+ * in the given (deterministic) file order, until `budget` lines are allocated
+ * or every file's available matches are exhausted. Flooding is structurally
+ * impossible — file A gets its (r+1)-th line only after every other file got
+ * its r-th — while few-files queries keep full depth (deep rounds fill just
+ * like the old flat output).
+ *
+ * @param {number[]} counts - available (fetched) matches per file
+ * @param {number} budget - total lines to allocate (k)
+ * @returns {number[]} allocation per file, same order as counts
+ */
+export function allocateGrepBudget(counts, budget) {
+  const alloc = counts.map(() => 0);
+  let remaining = Math.max(0, budget | 0);
+  let progressed = true;
+  while (remaining > 0 && progressed) {
+    progressed = false;
+    for (let i = 0; i < counts.length && remaining > 0; i++) {
+      if (alloc[i] < counts[i]) {
+        alloc[i]++;
+        remaining--;
+        progressed = true;
+      }
+    }
+  }
+  return alloc;
+}
+
+/**
+ * Render the diversified agent grep body: matches grouped per file (research:
+ * contiguous per-file blocks read better than interleaving) with allocation
+ * decided breadth-first, and a visible inline `(+N more in this file)` marker
+ * on the last shown line of every truncated file — elision is never silent
+ * and costs no extra lines.
+ *
+ * @param {Array<{file: string, line: number, matchText?: string}>} kept -
+ *   diversified matches in engine order (grouped by file)
+ * @param {{files: Array<{file, total, kept}>, hiddenFileCount, hiddenMatchCount,
+ *          hiddenSample: Array<{file, total}>}} fileSummary
+ * @param {number} k - body line budget
+ * @returns {{lines: string[], shownMatches: number, matchedFileCount: number,
+ *            truncatedFileCount: number, hiddenLine: string|null}}
+ */
+export function renderGrepBody(kept, fileSummary, k) {
+  const groups = new Map();
+  for (const m of kept) {
+    if (!groups.has(m.file)) groups.set(m.file, []);
+    groups.get(m.file).push(m);
+  }
+  const totals = new Map(fileSummary.files.map(f => [f.file, f.total]));
+  const ordered = [...groups.entries()];
+  const alloc = allocateGrepBudget(ordered.map(([, ms]) => ms.length), k);
+
+  const lines = [];
+  let shownMatches = 0;
+  let truncatedFileCount = 0;
+  const unallocated = []; // fetched files that got zero budget (more files than k)
+  ordered.forEach(([file, ms], i) => {
+    if (alloc[i] === 0) {
+      unallocated.push({ file, total: totals.get(file) ?? ms.length });
+      return;
+    }
+    const total = totals.get(file) ?? ms.length;
+    for (let j = 0; j < alloc[i]; j++) {
+      const m = ms[j];
+      const text = (m.matchText || '').replace(/\s+/g, ' ').trim().slice(0, 140);
+      let line = `${file}:${m.line}: ${text}`;
+      if (j === alloc[i] - 1 && total > alloc[i]) {
+        line += ` (+${total - alloc[i]} more in this file)`;
+        truncatedFileCount++;
+      }
+      lines.push(line);
+      shownMatches++;
+    }
+  });
+
+  // Files that matched but got no body line at all (more matching files than
+  // budget, or clipped by the engine's maxFiles fetch bound): one honest tail
+  // line naming the first few so the agent can jump straight to them.
+  const hiddenFiles = unallocated.length + fileSummary.hiddenFileCount;
+  const hiddenMatches = unallocated.reduce((a, f) => a + f.total, 0) + fileSummary.hiddenMatchCount;
+  let hiddenLine = null;
+  if (hiddenFiles > 0) {
+    const sample = [...unallocated.map(f => f.file), ...fileSummary.hiddenSample.map(f => f.file)]
+      .slice(0, 3);
+    hiddenLine = `# +${hiddenFiles} more file(s) with ${hiddenMatches} match(es)` +
+      (sample.length ? ` — e.g. ${sample.join(', ')}` : '') +
+      `; narrow the regex, raise -k, or drill in with --in <file>`;
+  }
+
+  return {
+    lines,
+    shownMatches,
+    matchedFileCount: fileSummary.files.length + fileSummary.hiddenFileCount,
+    truncatedFileCount,
+    hiddenLine,
+  };
+}

package/core/search/query-sufficiency.js

@@ -0,0 +1,341 @@
+/**
+ * Query-conditioned sufficiency evidence (2026-07).
+ *
+ * Answers ONE question, cheaply: does the top-1 packaged hit contain
+ * positive evidence that it matches what the QUERY asked for? This is the
+ * signal the structural sufficiency rule (complete_symbol / header_resolved /
+ * neighbors_present) never looked at — a perfectly well-formed function that
+ * has nothing to do with the query must not earn `sufficient=YES`.
+ *
+ * Literature anchor: Sufficient Context (Joren et al., ICLR 2025) — a naive
+ * lexical answer-presence check alone reaches F1 0.81 / precision 0.87 on
+ * sufficiency; fusing it (AND, not weighted sum) with an independent
+ * confidence signal is the high-precision pattern. See
+ * eval/task-completion-bench/analysis/sufficiency-redesign-2026-07-06.md.
+ *
+ * Constraints honoured here:
+ *  - engine-local, query-time only: pure string work over the already-
+ *    retrieved top-1 (haystack capped), no I/O, no LLM, sub-millisecond;
+ *  - only ever invoked from packageForAgent (agent formats) — NL/GCSN
+ *    ranking paths never reach this module;
+ *  - identifier-SHAPE heuristics, not capture-filtering stopword lists
+ *    (CLAUDE.md rule): the only stopword set below is a stable ~30-entry
+ *    English query-tokenization list.
+ */
+
+// Query-tokenization stopwords (standard English IR practice — the "OK to
+// keep" category). Filters function words from the subtoken-overlap signal;
+// never used to filter identifier captures.
+const EVIDENCE_STOPWORDS = new Set([
+  'a', 'an', 'and', 'are', 'can', 'could', 'did', 'do', 'does', 'for',
+  'from', 'how', 'in', 'into', 'is', 'it', 'its', 'not', 'of', 'on', 'or',
+  'should', 'that', 'the', 'this', 'to', 'was', 'were', 'what', 'when',
+  'where', 'which', 'with',
+]);
+
+function envFloat(name, fallback) {
+  const raw = process.env[name];
+  if (raw == null || raw === '') return fallback;
+  const parsed = Number.parseFloat(raw);
+  return Number.isFinite(parsed) && parsed >= 0 && parsed <= 1 ? parsed : fallback;
+}
+
+// Thresholds are parameterized (env-overridable for offline calibration
+// sweeps) — tuned on the task-level dev split of the full200 replay, never
+// hand-picked against held-out. See the design note for the discipline.
+export function sufficiencyThresholds() {
+  return {
+    overlapStrong: envFloat('SS_SUFF_OVERLAP_STRONG', 0.6),
+    overlapNone: envFloat('SS_SUFF_OVERLAP_NONE', 0.2),
+    minInformativeSubtokens: 2,
+    haystackCap: 32768,
+  };
+}
+
+const WORD_CHAR = /[A-Za-z0-9_$]/;
+
+function isWordChar(ch) {
+  return ch != null && WORD_CHAR.test(ch);
+}
+
+/**
+ * Word-boundary substring check without compiling a regex per token.
+ * A match counts only when the char before and after the occurrence are
+ * not identifier chars ("Chunker" must not match inside "ChunkerParams"
+ * unless the query token IS ChunkerParams).
+ */
+export function containsToken(haystack, token, { caseSensitive = true } = {}) {
+  if (!haystack || !token) return false;
+  const hay = caseSensitive ? haystack : haystack.toLowerCase();
+  const needle = caseSensitive ? token : token.toLowerCase();
+  let from = 0;
+  while (from <= hay.length - needle.length) {
+    const idx = hay.indexOf(needle, from);
+    if (idx === -1) return false;
+    const before = idx > 0 ? hay[idx - 1] : null;
+    const after = idx + needle.length < hay.length ? hay[idx + needle.length] : null;
+    if (!isWordChar(before) && !isWordChar(after)) return true;
+    from = idx + 1;
+  }
+  return false;
+}
+
+/**
+ * Identifier-shape heuristic (shape, not a word list): a token reads as a
+ * code identifier when it carries structure natural language words don't —
+ * camelCase humps, underscores/$, digits mixed into letters, dotted/pathy
+ * compounds, or ALL_CAPS constants. Pure lowercase single words ("parse",
+ * "config") are NOT identifier-shaped; they flow into the softer
+ * subtoken-overlap signal instead.
+ */
+export function looksLikeIdentifierToken(token) {
+  if (!token || token.length < 3) return false;
+  if (/[_$]/.test(token)) return true;
+  if (/[a-z][A-Z]/.test(token)) return true;                      // camelCase hump
+  if (/[A-Za-z]\d|\d[A-Za-z]/.test(token)) return true;           // letters+digits
+  if (/[.:/\\-]/.test(token) && /[A-Za-z]{2,}/.test(token)) return true; // dotted/pathy
+  if (/^[A-Z][A-Z0-9]{2,}$/.test(token)) return true;             // SCREAMING caps
+  if (/^[A-Z][a-z]+[A-Z]/.test(token)) return true;               // PascalCase
+  return false;
+}
+
+/**
+ * Pull literal runs out of a user regex (ss-find / ss-grep patterns):
+ * unescape metachar escapes, drop anchors, split on regex syntax, keep
+ * word-ish runs of length ≥ 3. "\\bChunkerParams\\b" → ["ChunkerParams"].
+ */
+export function regexLiteralRuns(regexSource) {
+  if (!regexSource) return [];
+  const unescaped = String(regexSource)
+    .replace(/\\[bBdDsSwWAZz]/g, ' ')       // char-class/anchor escapes → separator
+    .replace(/\\([^A-Za-z0-9])/g, '$1');    // \. \( … → literal char
+  const runs = unescaped.split(/[\^$.|?*+()[\]{}<>=!,\s]+/);
+  const out = [];
+  for (const run of runs) {
+    const trimmed = run.replace(/^[-:/\\]+|[-:/\\]+$/g, '');
+    if (trimmed.length >= 3 && /[A-Za-z]/.test(trimmed)) out.push(trimmed);
+  }
+  return out;
+}
+
+/**
+ * Split text into normalized informative subtokens: identifier chunks are
+ * split on case humps / non-alnum, lowercased, stopword- and length-filtered.
+ * Single-pass character scanner — this runs over the top-1 haystack on every
+ * agent-format search, so it must stay well under a millisecond at 32KB.
+ */
+export function informativeSubtokens(text) {
+  const out = new Set();
+  if (!text) return out;
+  const s = String(text);
+  const n = s.length;
+  let start = -1;        // current subtoken start
+  let hasAlpha = false;  // subtoken contains a letter (pure digits dropped)
+  const flush = (end) => {
+    if (start === -1) return;
+    if (hasAlpha && end - start >= 3) {
+      const norm = s.slice(start, end).toLowerCase();
+      if (!EVIDENCE_STOPWORDS.has(norm)) out.add(norm);
+    }
+    start = -1;
+    hasAlpha = false;
+  };
+  let prevLower = false; // previous char was a lowercase letter or digit
+  let prevUpper = false; // previous char was uppercase
+  for (let i = 0; i < n; i++) {
+    const c = s.charCodeAt(i);
+    const isLower = c >= 97 && c <= 122;
+    const isUpper = c >= 65 && c <= 90;
+    const isDigit = c >= 48 && c <= 57;
+    if (isLower || isUpper || isDigit) {
+      // camelCase hump: aB starts a new subtoken; ABc splits before the 'Bc'.
+      if (start !== -1 && isUpper && prevLower) {
+        flush(i);
+      } else if (start !== -1 && isLower && prevUpper && i - start >= 2) {
+        const split = i - 1; // "HTTPServer" → "http" + "server"
+        if (split - start >= 3) {
+          const norm = s.slice(start, split).toLowerCase();
+          if (!EVIDENCE_STOPWORDS.has(norm)) out.add(norm);
+        }
+        start = split;
+        hasAlpha = true;
+      }
+      if (start === -1) { start = i; hasAlpha = false; }
+      if (isLower || isUpper) hasAlpha = true;
+      prevLower = isLower || isDigit;
+      prevUpper = isUpper;
+    } else {
+      flush(i);
+      prevLower = false;
+      prevUpper = false;
+    }
+  }
+  flush(n);
+  return out;
+}
+
+/**
+ * Extract the query's evidence anchors.
+ *
+ * @param {string} query   natural-language or symbol query
+ * @param {string} [regex] optional user regex (ss-find)
+ * @returns {{ anchors: string[], subtokens: Set<string> }}
+ *   anchors: exact-match candidates (quoted literals, identifier-shaped
+ *   tokens, regex literal runs) — checked verbatim with word boundaries;
+ *   subtokens: normalized informative subtokens for the overlap signal.
+ */
+export function extractQueryEvidence(query, regex) {
+  const q = String(query || '');
+  const anchors = [];
+  const seen = new Set();
+  const push = (tok) => {
+    if (tok && tok.length >= 3 && !seen.has(tok)) { seen.add(tok); anchors.push(tok); }
+  };
+
+  // Quoted literals (error strings, config keys) — strongest anchors.
+  const quoteRe = /"([^"\n]{3,120})"|'([^'\n]{3,120})'|`([^`\n]{3,120})`/g;
+  let m;
+  while ((m = quoteRe.exec(q)) !== null) push(m[1] || m[2] || m[3]);
+
+  // Identifier-shaped tokens from the query text.
+  for (const raw of q.split(/[\s,;()"'`]+/)) {
+    const tok = raw.replace(/^[^A-Za-z0-9_$]+|[^A-Za-z0-9_$)]+$/g, '');
+    if (looksLikeIdentifierToken(tok)) push(tok);
+  }
+
+  // Literal runs from the regex (the exact thing the caller greps for).
+  for (const run of regexLiteralRuns(regex)) push(run);
+
+  return { anchors, subtokens: informativeSubtokens(q) };
+}
+
+/**
+ * Assess query-match evidence in the top-1 packaged result.
+ *
+ * @param {string} query
+ * @param {string} regex
+ * @param {object} topResult  packaged agent result (symbol, file, code,
+ *                            headerContext, summary)
+ * @returns {{ strength: 'strong'|'partial'|'none', exactHit: boolean,
+ *             matchedAnchor: string|null, overlap: number,
+ *             informativeCount: number }}
+ */
+export function assessQueryEvidence(query, regex, topResult) {
+  const t = sufficiencyThresholds();
+  const { anchors, subtokens } = extractQueryEvidence(query, regex);
+
+  const haystackParts = [
+    topResult?.symbol || '',
+    topResult?.file || '',
+    topResult?.headerContext || '',
+    topResult?.summary || '',
+    topResult?.code || '',
+  ];
+  const haystack = haystackParts.join('\n').slice(0, t.haystackCap);
+
+  let exactHit = false;
+  let matchedAnchor = null;
+  for (const anchor of anchors) {
+    // Case-sensitive when the anchor carries case information; boundary-
+    // checked either way so "Chunker" can't match inside "ChunkerParams".
+    const caseSensitive = /[A-Z]/.test(anchor);
+    if (containsToken(haystack, anchor, { caseSensitive })) {
+      exactHit = true;
+      matchedAnchor = anchor;
+      break;
+    }
+  }
+
+  const informativeCount = subtokens.size;
+  let overlap = 0;
+  if (informativeCount > 0) {
+    const hayTokens = informativeSubtokens(haystack);
+    let matched = 0;
+    for (const tok of subtokens) if (hayTokens.has(tok)) matched++;
+    overlap = matched / informativeCount;
+  }
+
+  let strength = 'partial';
+  if (exactHit || (overlap >= t.overlapStrong && informativeCount >= t.minInformativeSubtokens)) {
+    strength = 'strong';
+  } else if (informativeCount >= t.minInformativeSubtokens && overlap <= t.overlapNone) {
+    // Only a query with enough informative tokens can CONFIRM absence —
+    // a 1-token NL query never earns a discouraging 'none'.
+    strength = 'none';
+  }
+
+  return { strength, exactHit, matchedAnchor, overlap, informativeCount };
+}
+
+/**
+ * Fuse query-match evidence with the engine's confidence bucket and the
+ * structural packaging facts into a 3-valued verdict.
+ *
+ * Rule (design note §2; stricter-YES direction):
+ *   - YES requires strong evidence AND (confidence=high, or confidence=medium
+ *     with resolved structural context). Packaging alone can never say YES;
+ *     confidence=low can never say YES.
+ *   - 'no' is reserved for confirmed absence (no results, or a query with
+ *     real anchors/subtokens finding nothing in top-1). The borg-style
+ *     false-no — literal present with a clear margin — becomes YES.
+ *   - everything ambiguous is 'unknown', never a false binary.
+ *
+ * @param {object} args
+ * @param {object|null} args.topResult
+ * @param {{confidence: string}|null} args.confidenceInfo
+ * @param {string} args.query
+ * @param {string} [args.regex]
+ * @param {{isComplete: boolean, hasResolution: boolean}} args.structural
+ * @param {object[]} [args.lowerResults] code-bearing results below top-1
+ *   (full/preview tiers) — consulted ONLY to soften a would-be 'no': when
+ *   the pack's answer sits at rank 2-3, a flat 'no' would falsely push the
+ *   agent away from a pack that contains it. YES stays top-1-strict.
+ * @returns {{ verdict: 'yes'|'no'|'unknown', reason: string,
+ *             evidence: object|null }}
+ */
+export function computeSufficiencyVerdict({ topResult, confidenceInfo, query, regex, structural, lowerResults = [] }) {
+  if (!topResult) {
+    return { verdict: 'no', reason: 'no_results', evidence: null };
+  }
+
+  const evidence = assessQueryEvidence(query, regex, topResult);
+  const conf = confidenceInfo?.confidence || 'low';
+  const structuralOk = !!(structural?.isComplete && structural?.hasResolution);
+
+  if (evidence.strength === 'none') {
+    for (const r of lowerResults) {
+      if (!r || (!r.code && !r.symbol)) continue;
+      const lower = assessQueryEvidence(query, regex, r);
+      if (lower.strength === 'strong') {
+        return { verdict: 'unknown', reason: 'evidence_below_top1', evidence };
+      }
+    }
+    return { verdict: 'no', reason: 'no_query_evidence', evidence };
+  }
+
+  if (!topResult.code) {
+    // Summary-only top-1: the agent holds no code to answer from.
+    return { verdict: 'unknown', reason: 'top_summary_only', evidence };
+  }
+
+  if (evidence.strength === 'strong') {
+    if (conf === 'high') {
+      return { verdict: 'yes', reason: evidence.exactHit ? 'query_evidence_clear_margin' : 'query_overlap_clear_margin', evidence };
+    }
+    if (conf === 'medium') {
+      // Dev-split calibration (full200 replay): gating medium on structural
+      // resolution cost recall (0.35 vs 0.41) for no precision gain (0.706
+      // vs 0.700) — strong evidence + a medium margin is enough. Low never
+      // reaches yes: re-admitting low+exactHit dropped precision to 0.61.
+      return { verdict: 'yes', reason: 'query_evidence_moderate_margin', evidence };
+    }
+    return { verdict: 'unknown', reason: 'evidence_without_margin', evidence };
+  }
+
+  // Partial evidence — the old structural-YES shape lands here.
+  return {
+    verdict: 'unknown',
+    reason: structuralOk ? 'well_formed_only' : 'partial_query_evidence',
+    evidence,
+  };
+}

package/core/search/search-pattern.js

@@ -16,6 +16,7 @@
 import { PROJECT_ROOT } from '../infrastructure/config/index.js';
 import { generateRegexMatches } from './search-pattern-planner.js';
 import { buildBareGrepResults, filterMatchesBySymbolType, resolveSearchSymbolFilter, mapMatchesToChunks, readFileRange } from './search-pattern-chunks.js';
+import { applyGrepFileDiversity, matchesGrepFileFilter } from './grep-output-shaping.js';
 import { isRipgrepAvailable, runRipgrepJson } from './search-pattern-ripgrep.js';
 import { ensureSparseGramIndex } from './search-pattern-prefilter.js';
 import { packageForAgent } from './context-expander.js';
@@ -152,6 +153,11 @@ export async function bareGrep(query, routing, options = {}) {
   const candidateResult = await generateRegexMatches(this || {}, regex, searchDir, options);
   let matches = [...candidateResult.indexedMatches, ...candidateResult.overlayMatches];
   matches = filterMatchesBySymbolType(matches, symbolType, this);
+  // Agent drill-in scope (--in <file>): applied BEFORE sort/cap so a
+  // late-alphabet file's matches can never be pre-clipped by maxMatches.
+  if (options.fileFilter) {
+    matches = matches.filter(m => matchesGrepFileFilter(m.file, options.fileFilter));
+  }
   matches.sort((a, b) =>
     a.file.localeCompare(b.file) ||
     a.line - b.line ||
@@ -159,6 +165,16 @@ export async function bareGrep(query, routing, options = {}) {
   );
 
   const totalMatches = matches.length;
+  // Agent-only k-budget file diversity (option-gated; absent → byte-identical
+  // output). Streaming per-file cap: matches beyond the cap are counted, not
+  // stored, so memory is bounded by perFileCap*maxFiles, never total matches.
+  let fileSummary = null;
+  if (options.perFileCap > 0) {
+    ({ kept: matches, fileSummary } = applyGrepFileDiversity(matches, {
+      perFileCap: options.perFileCap,
+      maxFiles: options.maxFiles,
+    }));
+  }
   if (maxMatches > 0) {
     matches = matches.slice(0, maxMatches);
   }
@@ -170,6 +186,7 @@ export async function bareGrep(query, routing, options = {}) {
 
   return {
     results,
+    ...(fileSummary ? { fileSummary } : {}),
     stats: {
       path: 'grep',
       regex,

package/eval/agent-read-workflows/bin/_ss-argparse.mjs

@@ -192,3 +192,18 @@ export function extractPositional(args) {
   if (bad) return { pattern: undefined, unknownFlag: bad };
   return { pattern: args[0], unknownFlag: null };
 }
+
+// --- trailer rendering --------------------------------------------------------
+
+// Sufficiency trailer segment: 3-valued verdict (YES / no / unknown) + a
+// compact why-token, e.g. ` sufficient=YES (query_evidence_clear_margin)` vs
+// ` sufficient=unknown (well_formed_only)`. Falls back to the legacy boolean
+// when the engine predates sufficiencyVerdict. The full line shape stays
+// `# confidence=<bucket> (<reason>) sufficient=<verdict> (<why>)`.
+export function renderSufficiency(response) {
+  const verdict = response.sufficiencyVerdict
+    ? (response.sufficiencyVerdict === 'yes' ? 'YES' : response.sufficiencyVerdict)
+    : (response.sufficient ? 'YES' : 'no');
+  const why = response.sufficiencyReason ? ` (${response.sufficiencyReason})` : '';
+  return ` sufficient=${verdict}${why}`;
+}

package/eval/agent-read-workflows/bin/_ss-helpers.mjs

@@ -16,8 +16,9 @@ import { fileURLToPath } from 'node:url';
 import {
   parseBoolFlag, parseValueFlag, parsePositiveIntFlag,
   buildGrepPattern, stripInertFlags, normalizeArgs, extractPositional,
-  parseLineRange, looksLikeOption,
+  parseLineRange, looksLikeOption, renderSufficiency,
 } from './_ss-argparse.mjs';
+import { renderGrepBody } from '../../../core/search/grep-output-shaping.js';
 
 // Diagnostic-log isolation (agent-facing tools). The Sweet Search engine emits
 // model/index load banners via console.log → stdout ("LateInteraction: Loaded…",
@@ -115,13 +116,16 @@ async function ensureWarmServerReady({ timeoutMs = 60000, intervalMs = 500 } = {
 
 // --- subcommands ----------------------------------------------------------
 
-const GREP_USAGE = 'Usage: ss-grep <regex> [-i|--ignore-case] [-w|--word-regexp] [-F|--fixed-strings] [-k N]';
+const GREP_USAGE = 'Usage: ss-grep <regex> [-i|--ignore-case] [-w|--word-regexp] [-F|--fixed-strings] [--in <file>] [-k N]';
 async function cmdGrep(rawArgs) {
   const args = normalizeArgs(rawArgs);
   const ignoreCase = parseBoolFlag(args, ['-i', '--ignore-case']);
   const wordBound = parseBoolFlag(args, ['-w', '--word-regexp']);
   const fixedString = parseBoolFlag(args, ['-F', '--fixed-strings']);
   const k = readPositiveIntFlag(args, ['-k', '--top'], 20, GREP_USAGE);
+  // Drill-in scope: show matches from ONE file (the recovery affordance the
+  // diversified output advertises when it truncates a flooded file).
+  const inFile = readValueFlag(args, '--in', null, GREP_USAGE);
   stripInertFlags(args);
   const regex = buildGrepPattern(resolvePositional(args, GREP_USAGE), { ignoreCase, wordBound, fixedString });
   if (!regex) {
@@ -129,25 +133,47 @@ async function cmdGrep(rawArgs) {
     process.exit(2);
   }
   const s = await getSweetSearch();
-  const result = await s.bareGrep(regex, null, { regex, maxMatches: k * 5, contextLines: 0 });
-  // Group by file, take first k matches across all files (ordered as bareGrep returns).
-  const grouped = new Map();
-  for (const r of result.results.slice(0, k * 5)) {
-    if (!grouped.has(r.file)) grouped.set(r.file, []);
-    grouped.get(r.file).push(r);
-  }
-  let printed = 0;
-  process.stdout.write(`# ss-grep: ${result.results.length} total match(es) for /${regex}/\n`);
-  for (const [file, lines] of grouped) {
-    for (const r of lines) {
+
+  if (inFile) {
+    // Single-file scope: flat output, depth up to k within that file.
+    const result = await s.bareGrep(regex, null, {
+      regex, maxMatches: k, contextLines: 0, fileFilter: inFile,
+    });
+    const total = result.stats?.totalMatches ?? result.results.length;
+    process.stdout.write(`# ss-grep: ${total} total match(es) for /${regex}/ (scope: --in ${inFile})\n`);
+    result.results.forEach((r, i) => {
       const text = (r.matchText || '').replace(/\s+/g, ' ').trim().slice(0, 140);
-      process.stdout.write(`${file}:${r.line}: ${text}\n`);
-      printed++;
-      if (printed >= k) break;
-    }
-    if (printed >= k) break;
+      const marker = (i === result.results.length - 1 && total > result.results.length)
+        ? ` (+${total - result.results.length} more — raise -k)` : '';
+      process.stdout.write(`${r.file}:${r.line}: ${text}${marker}\n`);
+    });
+    if (result.results.length === 0) process.stdout.write('(no matches)\n');
+    process.exit(0);
+  }
+
+  // k-budget file diversity: fetch at most min(k,100) matches per file across
+  // at most k files (a file can never show more than k lines, and more than k
+  // files can never fit), then allocate the k body lines breadth-first so one
+  // flooded file can never hide every other matching file (the gradethis-161
+  // failure). Rendering stays grouped per file; truncation is marked inline
+  // and drillable via --in.
+  const result = await s.bareGrep(regex, null, {
+    regex, maxMatches: 0, contextLines: 0,
+    perFileCap: Math.min(k, 100), maxFiles: k,
+  });
+  const total = result.stats?.totalMatches ?? result.results.length;
+  const fileSummary = result.fileSummary
+    || { files: [], hiddenFileCount: 0, hiddenMatchCount: 0, hiddenSample: [] };
+  const body = renderGrepBody(result.results, fileSummary, k);
+
+  process.stdout.write(`# ss-grep: ${total} total match(es) for /${regex}/\n`);
+  if (body.truncatedFileCount > 0 || body.hiddenLine) {
+    process.stdout.write(`# ${body.matchedFileCount} file(s) matched; (+N more in this file)=truncated — ` +
+      `see the rest: ss-grep "<regex>" --in <file>\n`);
   }
-  if (printed === 0) process.stdout.write('(no matches)\n');
+  for (const line of body.lines) process.stdout.write(line + '\n');
+  if (body.hiddenLine) process.stdout.write(body.hiddenLine + '\n');
+  if (body.shownMatches === 0) process.stdout.write('(no matches)\n');
   process.exit(0);
 }
 
@@ -196,7 +222,7 @@ async function cmdFind(rawArgs) {
     ` budget=${response.tokenBudget} used=${response.tokensUsed} subMode=${response.subMode ?? format}\n`);
   if (response.confidence) {
     process.stdout.write(`# confidence=${response.confidence}${response.confidenceReason ? ' (' + response.confidenceReason + ')' : ''}` +
-      `${response.sufficient ? ' sufficient=YES' : ' sufficient=no'}\n`);
+      `${renderSufficiency(response)}\n`);
   }
 
   // Per-result blocks — identical shape to ss-search's agent packaging.
@@ -391,7 +417,7 @@ async function cmdAgentSearch(rawArgs) {
     ` results=${response.results.length} subMode=${response.subMode}\n`);
   if (response.confidence) {
     process.stdout.write(`# confidence=${response.confidence}${response.confidenceReason ? ' (' + response.confidenceReason + ')' : ''}` +
-      `${response.sufficient ? ' sufficient=YES' : ' sufficient=no'}\n`);
+      `${renderSufficiency(response)}\n`);
   }
 
   // Per-result blocks
@@ -448,6 +474,8 @@ async function cmdAgentSearch(rawArgs) {
     headerCount,
     confidence: response.confidence || null,
     sufficient: response.sufficient ?? null,
+    sufficiencyVerdict: response.sufficiencyVerdict ?? null,
+    sufficiencyReason: response.sufficiencyReason ?? null,
     sufficiencyReasons: Array.isArray(response.sufficiencyReasons) ? response.sufficiencyReasons : null,
     unresolvedExternalCount: typeof response.unresolvedExternalCount === 'number'
       ? response.unresolvedExternalCount : null,

package/mcp/tool-handlers.js

@@ -42,6 +42,8 @@ export const SearchOutputSchema = z.object({
   confidence: z.enum(['high', 'medium', 'low']).optional(),
   confidenceReason: z.string().optional(),
   sufficient: z.boolean().optional(),
+  sufficiencyVerdict: z.enum(['yes', 'no', 'unknown']).optional(),
+  sufficiencyReason: z.string().optional(),
   sufficiencyReasons: z.array(z.string()).optional(),
   packagingMs: z.number().optional(),
   latencyMs: z.number().optional(),
@@ -166,6 +168,8 @@ export async function handleSearch({ query, k, mode, structural, regex, format,
         confidence: searchResult.confidence,
         confidenceReason: searchResult.confidenceReason,
         sufficient: searchResult.sufficient,
+        ...(searchResult.sufficiencyVerdict ? { sufficiencyVerdict: searchResult.sufficiencyVerdict } : {}),
+        ...(searchResult.sufficiencyReason ? { sufficiencyReason: searchResult.sufficiencyReason } : {}),
         sufficiencyReasons: searchResult.sufficiencyReasons || [],
         packagingMs: searchResult.packagingMs,
         latencyMs: searchResult.latencyMs,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sweet-search",
-  "version": "2.6.12",
+  "version": "2.6.14",
   "description": "Sweet Search - SOTA Hybrid Code Search Engine with WASM CatBoost Query Router, Semantic/Lexical/Structural Search, and Multilingual Support",
   "type": "module",
   "main": "core/search/sweet-search.js",
@@ -167,13 +167,13 @@
     "vitest": "^4.0.16"
   },
   "optionalDependencies": {
-    "@sweet-search/native-darwin-arm64": "2.6.12",
-    "@sweet-search/native-darwin-x64": "2.6.12",
-    "@sweet-search/native-linux-arm64-gnu": "2.6.12",
-    "@sweet-search/native-linux-arm64-gnu-cuda": "2.6.12",
-    "@sweet-search/native-linux-x64-gnu": "2.6.12",
-    "@sweet-search/native-linux-x64-gnu-cuda": "2.6.12",
-    "@sweet-search/bg-priority": "2.6.12"
+    "@sweet-search/native-darwin-arm64": "2.6.14",
+    "@sweet-search/native-darwin-x64": "2.6.14",
+    "@sweet-search/native-linux-arm64-gnu": "2.6.14",
+    "@sweet-search/native-linux-arm64-gnu-cuda": "2.6.14",
+    "@sweet-search/native-linux-x64-gnu": "2.6.14",
+    "@sweet-search/native-linux-x64-gnu-cuda": "2.6.14",
+    "@sweet-search/bg-priority": "2.6.14"
   },
   "engines": {
     "node": ">=18.0.0"