token-goat 2.8.2 → 2.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,8 +4,13 @@ import {
4
4
  IMPORT_RE,
5
5
  ImageDecodeError,
6
6
  MAX_OVER_FETCH,
7
+ MAX_ZIP_INPUT_BYTES,
8
+ MAX_ZIP_OUTPUT_BYTES,
7
9
  OVER_FETCH_FACTOR,
8
10
  SKIP_DIRS,
11
+ UNTRUSTED_GITHUB_TAG,
12
+ ZipInputTooLargeError,
13
+ ZipOutputTooLargeError,
9
14
  capJsonRows,
10
15
  countRefs,
11
16
  countSymbols,
@@ -18,6 +23,7 @@ import {
18
23
  extractPdfMeta,
19
24
  extractPdfOutline,
20
25
  extractPdfText,
26
+ fenceUntrustedContent,
21
27
  formatCsvProfile,
22
28
  formatCsvTable,
23
29
  getFileEntry,
@@ -33,6 +39,7 @@ import {
33
39
  locatePdfPages,
34
40
  mergeNearbyHits,
35
41
  ocrImage,
42
+ ocrIntegrityFailed,
36
43
  parseWhereSpecs,
37
44
  pathEqClause,
38
45
  probeImageMeta,
@@ -42,19 +49,22 @@ import {
42
49
  queryRefs,
43
50
  queryRefsByContext,
44
51
  querySymbols,
52
+ scanForInjectionPatterns,
45
53
  searchEvidenceSemantically,
46
54
  searchSemantic,
47
55
  searchSymbolsFts,
48
56
  shrinkImage,
49
57
  storeBlob,
50
58
  stripLeadingAttributes,
59
+ stripTomlComment,
51
60
  tomlBracketDelta,
52
61
  trimToBudget,
62
+ unzipBounded,
53
63
  urlPolicyDenialReason,
54
64
  walkProject,
55
65
  yamlLineClosesQuote,
56
66
  yamlOpenQuoteAfter
57
- } from "./token-goat-chunk-AM23GDIS.mjs";
67
+ } from "./token-goat-chunk-4OTIB7SB.mjs";
58
68
  import {
59
69
  Database,
60
70
  PER_FILE_COUNTERFACTUAL_CEILING,
@@ -74,6 +84,7 @@ import {
74
84
  excludeTestsHiddenNote,
75
85
  extractErrorMessage,
76
86
  fileIsAbsent,
87
+ filtersFilteredToEmptyNotice,
77
88
  findHtmlHeadingMatches,
78
89
  findProject,
79
90
  fingerprintContent,
@@ -99,6 +110,7 @@ import {
99
110
  resolveIndexPath,
100
111
  resolveProjectRoot,
101
112
  runGit,
113
+ savedTokensFromBytes,
102
114
  shortFingerprint,
103
115
  stripAnsi,
104
116
  stripBom,
@@ -106,7 +118,7 @@ import {
106
118
  unsupportedLanguageName,
107
119
  windowsCmdQuoteArg,
108
120
  withExtension
109
- } from "./token-goat-chunk-IVCTQPZD.mjs";
121
+ } from "./token-goat-chunk-E76UNTVK.mjs";
110
122
  import {
111
123
  registerReset
112
124
  } from "./token-goat-chunk-AO2QD2AG.mjs";
@@ -190,18 +202,19 @@ function findTableHeaders(lines, isToml) {
190
202
  const restStart = closeIdx + openDelim.length;
191
203
  const m2 = TABLE_HEADER_RE.exec(line.slice(restStart));
192
204
  if (m2 !== null && m2[1] !== void 0) headers.push({ heading: m2[1].trim(), level: 1, index: i });
193
- openDelim = lineOpenDelimiterAfter(line, restStart);
205
+ openDelim = lineOpenDelimiterAfter(stripTomlComment(line.slice(restStart)), 0);
194
206
  continue;
195
207
  }
196
208
  if (isToml && arrayDepth > 0) {
197
- arrayDepth = Math.max(0, arrayDepth + tomlBracketDelta(line));
209
+ arrayDepth = Math.max(0, arrayDepth + tomlBracketDelta(stripTomlComment(line)));
198
210
  continue;
199
211
  }
200
212
  const m = TABLE_HEADER_RE.exec(line);
201
213
  if (m !== null && m[1] !== void 0) headers.push({ heading: m[1].trim(), level: 1, index: i });
202
214
  if (isToml) {
203
- openDelim = lineOpenDelimiterAfter(line, 0);
204
- if (openDelim === null) arrayDepth = Math.max(0, tomlBracketDelta(line));
215
+ const code = stripTomlComment(line);
216
+ openDelim = lineOpenDelimiterAfter(code, 0);
217
+ if (openDelim === null) arrayDepth = Math.max(0, tomlBracketDelta(code));
205
218
  }
206
219
  }
207
220
  return headers;
@@ -247,10 +260,11 @@ function findKeyValueHeaders(lines, language) {
247
260
  if (closed) openQuote = null;
248
261
  continue;
249
262
  }
250
- const m = (isEnv ? ENV_KEYVALUE_HEADER_RE : KEYVALUE_HEADER_RE).exec(line);
263
+ const scanned = isEnv ? line.replace(/^[ \t]+/, "") : line;
264
+ const m = (isEnv ? ENV_KEYVALUE_HEADER_RE : KEYVALUE_HEADER_RE).exec(scanned);
251
265
  if (m === null || m[1] === void 0) continue;
252
266
  headers.push({ heading: m[1], level: 1, index: i });
253
- openQuote = isEnv ? _detectOpenQuote(line.slice(m[0].length)) : yamlOpenQuoteAfter(line, m[0].length);
267
+ openQuote = isEnv ? _detectOpenQuote(scanned.slice(m[0].length)) : yamlOpenQuoteAfter(line, m[0].length);
254
268
  }
255
269
  return headers;
256
270
  }
@@ -3612,7 +3626,7 @@ function formatZipList(entries) {
3612
3626
  }
3613
3627
  async function extractZipEntry(data, entryPath) {
3614
3628
  const fflate = await requireFflate();
3615
- const result = fflate.unzipSync(data, { filter: (file) => file.name === entryPath });
3629
+ const result = unzipBounded(fflate, data, { limitBytes: MAX_ZIP_OUTPUT_BYTES, shouldExtract: (name) => name === entryPath });
3616
3630
  return result[entryPath];
3617
3631
  }
3618
3632
 
@@ -5262,10 +5276,16 @@ function readFileBytes(p) {
5262
5276
  verifyStillAbsent(p);
5263
5277
  return null;
5264
5278
  }
5265
- if (pinned !== void 0) return readPinnedBytes(p, pinned);
5279
+ if (pinned !== void 0) {
5280
+ const bytes = readPinnedBytes(p, pinned);
5281
+ if (bytes.length > MAX_ZIP_INPUT_BYTES) throw new ZipInputTooLargeError(p, bytes.length, MAX_ZIP_INPUT_BYTES);
5282
+ return bytes;
5283
+ }
5284
+ const stat = fs5.statSync(p);
5285
+ if (stat.size > MAX_ZIP_INPUT_BYTES) throw new ZipInputTooLargeError(p, stat.size, MAX_ZIP_INPUT_BYTES);
5266
5286
  return fs5.readFileSync(p);
5267
5287
  } catch (err) {
5268
- if (err instanceof ConfinementIdentityError) throw err;
5288
+ if (err instanceof ConfinementIdentityError || err instanceof ZipInputTooLargeError) throw err;
5269
5289
  return null;
5270
5290
  }
5271
5291
  }
@@ -5333,6 +5353,17 @@ function guardText(text, command) {
5333
5353
  const cfg = loadConfig();
5334
5354
  return cfg.overflow_guard.enabled ? trimToBudget(text, cfg.overflow_guard.max_tokens, command) : text;
5335
5355
  }
5356
+ function fenceGithubTextIfMatched(text) {
5357
+ let matches = [];
5358
+ try {
5359
+ if (loadConfig().injection.enabled) matches = scanForInjectionPatterns(text);
5360
+ } catch {
5361
+ matches = [];
5362
+ }
5363
+ if (matches.length === 0) return text;
5364
+ recordStat("injection_detected", 0, 0, void 0, matches.join(","));
5365
+ return fenceUntrustedContent(text, matches, UNTRUSTED_GITHUB_TAG);
5366
+ }
5336
5367
  function guardJsonRows(items) {
5337
5368
  const cfg = loadConfig();
5338
5369
  if (!cfg.overflow_guard.enabled) return { items: [...items], truncated: false, totalCount: items.length };
@@ -5351,7 +5382,7 @@ function sumFileSizes(filePaths) {
5351
5382
  function recordReadStat(kind, fullSourceBytes, emittedText, detail) {
5352
5383
  const emittedBytes = Buffer.byteLength(emittedText, "utf8");
5353
5384
  const bytesSaved = Math.max(1, fullSourceBytes - emittedBytes);
5354
- recordStat(kind, bytesSaved, Math.round(bytesSaved / 4), void 0, detail);
5385
+ recordStat(kind, bytesSaved, savedTokensFromBytes(bytesSaved), void 0, detail);
5355
5386
  }
5356
5387
  function findSpecSeparator(spec) {
5357
5388
  return spec.lastIndexOf("::");
@@ -5634,10 +5665,21 @@ function runSymbol(opts) {
5634
5665
  if (opts.name !== void 0 && emptyIndexRoot !== null) {
5635
5666
  const rootDir = emptyIndexRoot;
5636
5667
  const rawSymbols = querySymbols({ limit: FIND_SCAN_LIMIT, rootDir });
5637
- const candidates = rankSimilarNames(rawSymbols.map((s) => s.name), opts.name);
5638
- text2 += candidates.length > 0 ? `
5668
+ const exactMatches = rawSymbols.filter((s) => s.name === opts.name);
5669
+ if (exactMatches.length > 0) {
5670
+ const shown = exactMatches.slice(0, DIDYOUMEAN_LIMIT);
5671
+ const where = shown.map((s) => `${s.kind} at ${toDisplayPath(rootDir, s.filePath)}:${s.lineStart}`).join("; ");
5672
+ const more = exactMatches.length > shown.length ? ` (+${exactMatches.length - shown.length} more)` : "";
5673
+ const flags = [opts.kind !== void 0 ? "--kind" : null, opts.file !== void 0 ? "--file" : null].filter((f) => f !== null);
5674
+ const widen = flags.length > 0 ? `drop ${flags.join("/")} to see it` : "widen the search scope to see it";
5675
+ text2 += `
5676
+ '${opts.name}' IS indexed (${where}${more}) -- ${widen}`;
5677
+ } else {
5678
+ const candidates = rankSimilarNames(rawSymbols.map((s) => s.name), opts.name);
5679
+ text2 += candidates.length > 0 ? `
5639
5680
  ${didYouMean(candidates)}` : indexEmpty ? "" : `
5640
5681
  Try: token-goat semantic "${opts.name}"`;
5682
+ }
5641
5683
  const structuredFiles = [...new Set(rawSymbols.map((s) => s.filePath))].sort();
5642
5684
  const hit = findStructuredKeyPath(opts.name, structuredFiles);
5643
5685
  if (hit !== null) {
@@ -5911,9 +5953,11 @@ function resolveSymbolSpec(spec, forceRefresh, projectRoot) {
5911
5953
  let candidates = querySymbols({ name: lookupName, filePath: resolved, limit: 10 });
5912
5954
  if (candidates.length === 0) {
5913
5955
  const foldedFile = foldPath(file);
5956
+ const baseName = file.slice(Math.max(file.lastIndexOf("/"), file.lastIndexOf("\\")) + 1);
5914
5957
  candidates = querySymbols({
5915
5958
  name: lookupName,
5916
5959
  limit: 50,
5960
+ ...baseName !== "" ? { fileBaseName: baseName } : {},
5917
5961
  ...projectRoot !== void 0 ? { rootDir: projectRoot } : {}
5918
5962
  }).filter((s) => {
5919
5963
  const foldedFilePath = foldPath(s.filePath);
@@ -6047,6 +6091,12 @@ ${sub.text}`);
6047
6091
  function resolveAgainstProjectRoot(file, projectRoot) {
6048
6092
  return projectRoot !== void 0 && !path5.isAbsolute(file) ? path5.resolve(projectRoot, file) : file;
6049
6093
  }
6094
+ function literalHeadingExists(filePath, heading) {
6095
+ const ordinalMatch = /^([^#\r\n]+)#(\d+)$/.exec(heading);
6096
+ const base = (ordinalMatch?.[1] ?? heading).trim().toLowerCase();
6097
+ if (base.length === 0) return false;
6098
+ return listSections(filePath, readFileText).some((h) => h.trim().toLowerCase() === base);
6099
+ }
6050
6100
  function runSection(opts) {
6051
6101
  const crossFilePairs = parseCrossFileMultiSpec(opts.spec);
6052
6102
  if (crossFilePairs !== null) return runSectionCrossFile(crossFilePairs, opts);
@@ -6057,7 +6107,7 @@ function runSection(opts) {
6057
6107
  const specFilePath = opts.spec.slice(0, colonIdx);
6058
6108
  const filePath = resolveAgainstProjectRoot(specFilePath, opts.projectRoot);
6059
6109
  const heading = opts.spec.slice(colonIdx + 2);
6060
- if (heading.includes(",")) {
6110
+ if (heading.includes(",") && !literalHeadingExists(filePath, heading)) {
6061
6111
  const multiHeadings = heading.split(",").map((h) => h.trim()).filter((h) => h.length > 0);
6062
6112
  if (multiHeadings.length > 1) return runSectionMulti(specFilePath, filePath, multiHeadings, opts);
6063
6113
  }
@@ -6491,14 +6541,10 @@ function formatStatsSuffix(refCounts, sym) {
6491
6541
  return refCounts !== void 0 ? ` [${countNoun(refCounts.get(sym.name) ?? 0, "ref")}, ${hasRealDocstring(sym.docstring) ? "documented" : "undocumented"}]` : "";
6492
6542
  }
6493
6543
  function filteredToEmptyNotice(preFilterCount, minLines, grep) {
6494
- const plural = preFilterCount === 1 ? "symbol" : "symbols";
6495
6544
  const parts = [];
6496
6545
  if (minLines !== void 0) parts.push(`--min-lines ${minLines}`);
6497
6546
  if (grep !== void 0) parts.push(`--grep ${grep}`);
6498
- const cause = parts.length === 0 ? "the active filter" : parts.join(" + ");
6499
- const knob = parts.length > 1 ? "filters" : "filter";
6500
- const verb = preFilterCount === 1 ? "was" : "were";
6501
- return ` (all ${preFilterCount} indexed ${plural} ${verb} filtered out by ${cause}; the file is indexed -- widen or drop the ${knob} to see them)`;
6547
+ return filtersFilteredToEmptyNotice(preFilterCount, parts, "indexed symbol", "indexed symbols", "the file is indexed");
6502
6548
  }
6503
6549
  function prepareSymbolListing(file, opts) {
6504
6550
  const resolved = resolveIndexPath(file, opts.projectRoot ?? process.cwd());
@@ -6607,6 +6653,12 @@ function runSkeleton(opts) {
6607
6653
  recordReadStat("stub_view", fullSourceBytes, text, opts.file);
6608
6654
  return { text, code: 0 };
6609
6655
  }
6656
+ var DOC_SUMMARY_MAX_CHARS = 140;
6657
+ function clipDocSummary(firstLine) {
6658
+ if (firstLine.length <= DOC_SUMMARY_MAX_CHARS) return firstLine;
6659
+ const cut = firstLine.lastIndexOf(" ", DOC_SUMMARY_MAX_CHARS);
6660
+ return `${firstLine.slice(0, cut > 40 ? cut : DOC_SUMMARY_MAX_CHARS).trimEnd()}\u2026`;
6661
+ }
6610
6662
  function runOutline(opts) {
6611
6663
  const multiFiles = parseMultiFileSpec(opts.file);
6612
6664
  if (multiFiles !== null) return runPerFileListing(multiFiles, (file) => runOutline({ ...opts, file, includeFilePath: true }), opts.json === true);
@@ -6644,7 +6696,7 @@ function runOutline(opts) {
6644
6696
  const rangeStr = `${sym.lineStart.toString().padStart(4)}-${sym.lineEnd.toString().padEnd(6)}`;
6645
6697
  const kindStr = sym.kind.padEnd(14);
6646
6698
  const bodyLen = sym.lineEnd - sym.lineStart + 1;
6647
- const docFirst = hasRealDocstring(sym.docstring) ? ` # ${sym.docstring.split("\n")[0] ?? ""}` : "";
6699
+ const docFirst = hasRealDocstring(sym.docstring) ? ` # ${clipDocSummary(sym.docstring.split("\n")[0] ?? "")}` : "";
6648
6700
  const statsStr = formatStatsSuffix(refCounts, sym);
6649
6701
  lines.push(` ${rangeStr} ${kindStr} ${sym.name} (${bodyLen}\u2113)${docFirst}${statsStr}`);
6650
6702
  }
@@ -6684,11 +6736,11 @@ function runCsvQuery(opts) {
6684
6736
  const rowsJson = result.rows.map((r) => Object.fromEntries(result.header.map((h, i) => [h, r[i]])));
6685
6737
  const headTruncated = result.rows.length < result.totalRows;
6686
6738
  const capped = guardJsonRows(rowsJson);
6687
- const jsonText = JSON.stringify({ items: capped.items, truncated: capped.truncated || headTruncated, totalCount: result.totalRows });
6739
+ const jsonText = JSON.stringify({ items: capped.items, truncated: capped.truncated || headTruncated, totalCount: result.totalRows, ...result.totalRows === 0 && result.preFilterRows > 0 ? { filteredFromRows: result.preFilterRows } : {} });
6688
6740
  emit(jsonText);
6689
6741
  recordReadStat("csv_query", fullSourceBytes, jsonText, opts.file);
6690
6742
  } else {
6691
- const tableText = formatCsvTable(result);
6743
+ const tableText = formatCsvTable(result, (opts.where ?? []).map((w) => `--where ${w}`));
6692
6744
  emit(tableText);
6693
6745
  recordReadStat("csv_query", fullSourceBytes, tableText, opts.file);
6694
6746
  }
@@ -6831,6 +6883,7 @@ function resolveYamlMergeKeys(node) {
6831
6883
  }
6832
6884
  function parseYamlDocument(text) {
6833
6885
  const docs = loadAll(text).map(resolveYamlMergeKeys);
6886
+ if (docs.length === 0) return null;
6834
6887
  return docs.length === 1 ? docs[0] : docs;
6835
6888
  }
6836
6889
  function runYamlOutline(opts) {
@@ -7021,11 +7074,20 @@ function runOpenApiOp(opts) {
7021
7074
  return 0;
7022
7075
  }
7023
7076
  function archiveReadFailure(err, file) {
7024
- if (err instanceof ArchiveDependencyMissingError) return err.message;
7077
+ if (err instanceof ArchiveDependencyMissingError || err instanceof ZipOutputTooLargeError) return err.message;
7025
7078
  return `Failed to read archive (not a valid zip-format file): ${file}`;
7026
7079
  }
7027
7080
  async function runZipList(opts) {
7028
- const data = readFileBytes(opts.file);
7081
+ let data;
7082
+ try {
7083
+ data = readFileBytes(opts.file);
7084
+ } catch (err) {
7085
+ if (err instanceof ZipInputTooLargeError) {
7086
+ emitErr(err.message);
7087
+ return 1;
7088
+ }
7089
+ throw err;
7090
+ }
7029
7091
  if (data === null) {
7030
7092
  emitErr(`Could not read: ${opts.file}`);
7031
7093
  return 1;
@@ -7050,7 +7112,16 @@ async function runZipList(opts) {
7050
7112
  return 0;
7051
7113
  }
7052
7114
  async function runZipRead(opts) {
7053
- const data = readFileBytes(opts.file);
7115
+ let data;
7116
+ try {
7117
+ data = readFileBytes(opts.file);
7118
+ } catch (err) {
7119
+ if (err instanceof ZipInputTooLargeError) {
7120
+ emitErr(err.message);
7121
+ return 1;
7122
+ }
7123
+ throw err;
7124
+ }
7054
7125
  if (data === null) {
7055
7126
  emitErr(`Could not read: ${opts.file}`);
7056
7127
  return 1;
@@ -7146,11 +7217,11 @@ function runPrSlice(opts) {
7146
7217
  }
7147
7218
  const fullSourceBytes = Buffer.byteLength(diffText, "utf8");
7148
7219
  if (opts.json === true) {
7149
- const jsonText = JSON.stringify({ path: parsed.path, diff: fileDiff });
7220
+ const jsonText = JSON.stringify({ path: parsed.path, diff: fenceGithubTextIfMatched(fileDiff) });
7150
7221
  emit(jsonText);
7151
7222
  recordReadStat("pr_slice", fullSourceBytes, jsonText, `${repo}#${opts.pr} diff:${parsed.path}`);
7152
7223
  } else {
7153
- emitGuarded(fileDiff, "pr-slice");
7224
+ emitGuarded(fenceGithubTextIfMatched(fileDiff), "pr-slice");
7154
7225
  recordReadStat("pr_slice", fullSourceBytes, fileDiff, `${repo}#${opts.pr} diff:${parsed.path}`);
7155
7226
  }
7156
7227
  return 0;
@@ -7159,13 +7230,14 @@ function runPrSlice(opts) {
7159
7230
  const comments = fetchPrComments(opts.pr, repo);
7160
7231
  const fullSourceBytes = Buffer.byteLength(JSON.stringify(comments), "utf8");
7161
7232
  if (opts.json === true) {
7162
- const capped = guardJsonRows(comments);
7233
+ const fencedComments = comments.map((c) => ({ ...c, body: fenceGithubTextIfMatched(c.body) }));
7234
+ const capped = guardJsonRows(fencedComments);
7163
7235
  const jsonText = JSON.stringify({ items: capped.items, truncated: capped.truncated, totalCount: capped.totalCount });
7164
7236
  emit(jsonText);
7165
7237
  recordReadStat("pr_slice", fullSourceBytes, jsonText, `${repo}#${opts.pr} comments`);
7166
7238
  } else {
7167
7239
  const text = formatCommentsSlice(comments);
7168
- emitGuarded(text, "pr-slice");
7240
+ emitGuarded(fenceGithubTextIfMatched(text), "pr-slice");
7169
7241
  recordReadStat("pr_slice", fullSourceBytes, text, `${repo}#${opts.pr} comments`);
7170
7242
  }
7171
7243
  return 0;
@@ -7174,12 +7246,17 @@ function runPrSlice(opts) {
7174
7246
  const desc = fetchPrDescription(opts.pr, repo);
7175
7247
  const fullSourceBytes = Buffer.byteLength(JSON.stringify(desc), "utf8");
7176
7248
  if (opts.json === true) {
7177
- const jsonText = JSON.stringify(desc);
7249
+ const fencedDesc = {
7250
+ ...desc,
7251
+ title: fenceGithubTextIfMatched(desc.title),
7252
+ body: desc.body !== null ? fenceGithubTextIfMatched(desc.body) : null
7253
+ };
7254
+ const jsonText = JSON.stringify(fencedDesc);
7178
7255
  emit(jsonText);
7179
7256
  recordReadStat("pr_slice", fullSourceBytes, jsonText, `${repo}#${opts.pr} description`);
7180
7257
  } else {
7181
7258
  const text = formatDescriptionSlice(desc);
7182
- emitGuarded(text, "pr-slice");
7259
+ emitGuarded(fenceGithubTextIfMatched(text), "pr-slice");
7183
7260
  recordReadStat("pr_slice", fullSourceBytes, text, `${repo}#${opts.pr} description`);
7184
7261
  }
7185
7262
  return 0;
@@ -7222,7 +7299,6 @@ function runSqliteQuery(opts) {
7222
7299
  const totalCount = result.rows.length;
7223
7300
  const headTruncated = head !== void 0 && result.rows.length > head;
7224
7301
  const rows = head !== void 0 ? result.rows.slice(0, head) : result.rows;
7225
- const fullSourceBytes = sumFileSizes([opts.file]);
7226
7302
  if (opts.json === true) {
7227
7303
  const capped = guardJsonRows(rows);
7228
7304
  const jsonText = JSON.stringify({
@@ -7233,11 +7309,20 @@ function runSqliteQuery(opts) {
7233
7309
  rowCapped: result.rowCapped
7234
7310
  });
7235
7311
  emit(jsonText);
7236
- recordReadStat("sqlite_query", fullSourceBytes, jsonText, opts.file);
7312
+ const uncappedFull = guardJsonRows(result.rows);
7313
+ const baselineJsonText = JSON.stringify({
7314
+ columns: result.columns,
7315
+ items: uncappedFull.items,
7316
+ truncated: uncappedFull.truncated || result.rowCapped,
7317
+ totalCount,
7318
+ rowCapped: result.rowCapped
7319
+ });
7320
+ recordReadStat("sqlite_query", Buffer.byteLength(baselineJsonText, "utf8"), jsonText, opts.file);
7237
7321
  } else {
7238
7322
  const text = formatSqliteQueryTable({ ...result, rows }, { headTruncated });
7239
7323
  emit(text);
7240
- recordReadStat("sqlite_query", fullSourceBytes, text, opts.file);
7324
+ const baselineText = formatSqliteQueryTable({ ...result, rows: result.rows }, { headTruncated: false });
7325
+ recordReadStat("sqlite_query", Buffer.byteLength(baselineText, "utf8"), text, opts.file);
7241
7326
  }
7242
7327
  return 0;
7243
7328
  } catch (e) {
@@ -7382,6 +7467,9 @@ async function runImageText(file) {
7382
7467
  const data = fs5.readFileSync(file);
7383
7468
  const ocr = await ocrImage(data);
7384
7469
  if (ocr === null) {
7470
+ if (ocrIntegrityFailed()) {
7471
+ throw new Error(`${file} was not OCRed: the cached language model failed its checksum and was discarded, so the next run re-downloads it`);
7472
+ }
7385
7473
  if (isOcrEngineAvailable()) {
7386
7474
  throw new Error(`${file} could not be processed by OCR (unreadable image, timeout, or offline model fetch)`);
7387
7475
  }
@@ -7432,15 +7520,14 @@ ${emptyIndexMessage(rootDir2)}`, code: 1 };
7432
7520
  const match = resolution.entry;
7433
7521
  const rootDir = resolveProjectRoot({ project: opts.projectRoot ?? process.cwd() });
7434
7522
  const excludeTests = opts.excludeTests === true;
7435
- const unboundedQuery = excludeTests || opts.grep !== void 0;
7436
- const allCallers = resolveCallers(match.name, void 0, match.filePath, rootDir, unboundedQuery);
7523
+ const allCallers = resolveCallers(match.name, void 0, match.filePath, rootDir, true);
7437
7524
  const testFiltered = excludeTests ? allCallers.filter((c) => !isTestFile(c.file)) : allCallers;
7438
7525
  const hiddenByExcludeTests = excludeTests ? allCallers.length - testFiltered.length : 0;
7439
7526
  const preGrepCount = testFiltered.length;
7440
7527
  const matchesGrep = opts.grep !== void 0 ? compileGrepMatcher(opts.grep) : void 0;
7441
7528
  const callers = matchesGrep !== void 0 ? testFiltered.filter((c) => matchesGrep(c.caller)) : testFiltered;
7442
7529
  const hiddenByGrep = matchesGrep !== void 0 ? preGrepCount - callers.length : 0;
7443
- const totalCallers = unboundedQuery ? callers.length : queryRefCounts([match.name], globalDbPath(), rootDir).get(match.name) ?? callers.length;
7530
+ const totalCallers = callers.length;
7444
7531
  const section = findContainingSection(match.filePath, match.lineStart, match.lineEnd, readFileText);
7445
7532
  const limit = opts.limit ?? 20;
7446
7533
  const shown = callers.slice(0, limit);
@@ -9089,10 +9176,14 @@ function bfsCallChains(start, callersOf, maxDepth) {
9089
9176
  const tip = chain[chain.length - 1];
9090
9177
  if (tip === void 0) continue;
9091
9178
  const callers = callersOf(tip);
9092
- if (callers.length === 0 || chain.length > maxDepth) {
9179
+ if (callers.length === 0) {
9093
9180
  complete.push(chain);
9094
9181
  continue;
9095
9182
  }
9183
+ if (chain.length > maxDepth) {
9184
+ complete.push([...chain, "(depth-limit)"]);
9185
+ continue;
9186
+ }
9096
9187
  let expanded = false;
9097
9188
  for (const caller of callers) {
9098
9189
  if (chain.includes(caller)) {
@@ -9178,8 +9269,11 @@ function runCallers(opts) {
9178
9269
  emitErr2(`Symbol '${name}' not found in '${file}'`);
9179
9270
  return 1;
9180
9271
  }
9181
- const unbounded = opts.excludeTests === true || opts.grep !== void 0;
9182
- const resolved = resolveCallers(name, opts.limit, fileHint, rootDir, unbounded);
9272
+ const unbounded = opts.excludeTests === true || opts.grep !== void 0 || fileHint !== void 0;
9273
+ const requestedLimit = opts.limit ?? DEFAULT_REF_QUERY_LIMIT;
9274
+ const probed = resolveCallers(name, unbounded ? opts.limit : requestedLimit + 1, fileHint, rootDir, unbounded);
9275
+ const sqlTruncated = !unbounded && probed.length > requestedLimit;
9276
+ const resolved = sqlTruncated ? probed.slice(0, requestedLimit) : probed;
9183
9277
  const suppressed = opts.excludeTests === true ? resolved.filter((e) => isTestFile(e.file)).length : 0;
9184
9278
  let filtered = opts.excludeTests === true ? resolved.filter((e) => !isTestFile(e.file)) : resolved;
9185
9279
  const preGrepCount = filtered.length;
@@ -9220,7 +9314,7 @@ function runCallers(opts) {
9220
9314
  return { ...e, file: displayPath, filePath: displayPath };
9221
9315
  });
9222
9316
  const capped = guardJsonRows(rows);
9223
- const limitTruncated = entries.length < filtered.length;
9317
+ const limitTruncated = entries.length < filtered.length || sqlTruncated;
9224
9318
  const hiddenByGrep = preGrepCount - filtered.length;
9225
9319
  emit2(JSON.stringify({ items: capped.items, truncated: capped.truncated || limitTruncated, totalCount: filtered.length, ...hiddenByGrep > 0 ? { hiddenByGrep } : {}, ...opts.excludeTests === true && suppressed > 0 ? { hiddenByExcludeTests: suppressed } : {} }, null, 2));
9226
9320
  return 0;
@@ -9259,7 +9353,7 @@ function runCallChain(opts) {
9259
9353
  const getSyms = buildFileSymCache();
9260
9354
  let suppressedCount = 0;
9261
9355
  const callersOf = (n) => {
9262
- const refs = queryRefs({ name: n, limit: DEFAULT_REF_QUERY_LIMIT, rootDir });
9356
+ const refs = queryRefs({ name: n, limit: UNBOUNDED_REF_LIMIT, rootDir });
9263
9357
  if (refs.length === 0) return [];
9264
9358
  const scoped = fileHint !== void 0 && n === name ? filterRefsForSymbol(refs, n, fileHint, getSyms) : refs;
9265
9359
  const names = /* @__PURE__ */ new Set();
@@ -9322,12 +9416,16 @@ function runImpact(opts) {
9322
9416
  const hops = /* @__PURE__ */ new Map([[rootName, 0]]);
9323
9417
  const queue = [[rootName, 0]];
9324
9418
  let suppressedCount = 0;
9419
+ let depthCapped = 0;
9325
9420
  while (queue.length > 0) {
9326
9421
  const item = queue.shift();
9327
9422
  if (item === void 0) break;
9328
9423
  const [name, depth] = item;
9329
- if (depth >= DEPTH_CAP) continue;
9330
- const refs = queryRefs({ name, limit: DEFAULT_REF_QUERY_LIMIT, rootDir });
9424
+ if (depth >= DEPTH_CAP) {
9425
+ depthCapped += 1;
9426
+ continue;
9427
+ }
9428
+ const refs = queryRefs({ name, limit: UNBOUNDED_REF_LIMIT, rootDir });
9331
9429
  const scoped = fileHint !== void 0 && name === rootName ? filterRefsForSymbol(refs, name, fileHint, getSyms) : refs;
9332
9430
  for (const ref of scoped) {
9333
9431
  if (opts.excludeTests === true && isTestFile(ref.filePath)) {
@@ -9355,6 +9453,12 @@ function runImpact(opts) {
9355
9453
  const matchesGrep = opts.grep !== void 0 ? compileGrepMatcher(opts.grep) : void 0;
9356
9454
  const grepped = matchesGrep !== void 0 ? allSorted.filter(([symbol]) => matchesGrep(symbol)) : allSorted;
9357
9455
  const sorted = grepped.slice(0, top);
9456
+ if (depthCapped > 0) {
9457
+ emitErr2(`Impact truncated at the ${DEPTH_CAP}-hop depth limit: ${depthCapped} symbol${depthCapped === 1 ? "" : "s"} at that depth ${depthCapped === 1 ? "was" : "were"} not expanded, so callers reachable only beyond ${DEPTH_CAP} hops are missing.`);
9458
+ }
9459
+ if (grepped.length > sorted.length) {
9460
+ emitErr2(`Showing top ${sorted.length} of ${grepped.length} impacted symbols (raise --top to see the rest).`);
9461
+ }
9358
9462
  if (matchesGrep !== void 0 && sorted.length === 0 && allSorted.length > 0) {
9359
9463
  if (opts.json === true) {
9360
9464
  emitErr2(grepFilteredToEmptyNotice(allSorted.length, opts.grep, "impacted symbol", "impacted symbols"));
@@ -9484,14 +9588,17 @@ function runDead(opts) {
9484
9588
  }
9485
9589
  return 1;
9486
9590
  }
9487
- const syms = kinds.flatMap((k) => querySymbols({ kind: k, limit: 5e3, rootDir }));
9591
+ const syms = kinds.flatMap((k) => querySymbols({ kind: k, limit: UNBOUNDED_REF_LIMIT, rootDir }));
9488
9592
  const getSyms = buildFileSymCache();
9489
9593
  const results = [];
9490
9594
  let suppressed = 0;
9491
9595
  for (const sym of syms) {
9492
9596
  if (opts.includePrivate !== true && sym.name.startsWith("_")) continue;
9493
9597
  const refs = queryRefs({ name: sym.name, limit: DEFAULT_REF_QUERY_LIMIT, rootDir });
9494
- const scoped = filterRefsForSymbol(refs, sym.name, sym.filePath, getSyms);
9598
+ let scoped = filterRefsForSymbol(refs, sym.name, sym.filePath, getSyms);
9599
+ if (scoped.length === 0 && refs.length >= DEFAULT_REF_QUERY_LIMIT) {
9600
+ scoped = filterRefsForSymbol(queryRefs({ name: sym.name, limit: UNBOUNDED_REF_LIMIT, rootDir }), sym.name, sym.filePath, getSyms);
9601
+ }
9495
9602
  if (!isDeadSymbol(sym.name, scoped.length)) continue;
9496
9603
  if (sym.kind === "method") {
9497
9604
  const ownScope = enclosingNamedScope(getSyms(sym.filePath), sym.lineStart);
@@ -9826,7 +9933,7 @@ function runScope(opts) {
9826
9933
  }
9827
9934
  const filePath = resolveIndexPath(file);
9828
9935
  const syms = querySymbols({ filePath, limit: ALL_SYMBOLS_IN_FILE_LIMIT });
9829
- const enclosing = syms.filter((s) => s.lineStart <= line && line <= s.lineEnd).sort((a, b) => b.lineStart - a.lineStart);
9936
+ const enclosing = syms.filter((s) => s.lineStart <= line && line <= s.lineEnd).sort((a, b) => b.lineStart - a.lineStart || a.lineEnd - b.lineEnd || a.name.localeCompare(b.name, "en"));
9830
9937
  if (syms.length === 0) {
9831
9938
  if (!fs6.existsSync(filePath)) {
9832
9939
  emitErr2(`Could not read: ${file}`);
@@ -9910,17 +10017,18 @@ function canonicalCycleKey(cyclePath) {
9910
10017
  }
9911
10018
  return [...nodes.slice(minIdx), ...nodes.slice(0, minIdx)].join(" ");
9912
10019
  }
9913
- function findCycles(graph) {
10020
+ function findCyclesCapped(graph) {
9914
10021
  const cycles = [];
9915
10022
  const seen = /* @__PURE__ */ new Set();
9916
10023
  const sccs = tarjanSCCs(graph);
10024
+ const probeLimit = MAX_CYCLES + 1;
9917
10025
  for (const component of sccs) {
9918
10026
  let dfs2 = function(start, node, pathSoFar) {
9919
- if (cycles.length >= MAX_CYCLES) return;
10027
+ if (cycles.length >= probeLimit) return;
9920
10028
  stack.add(node);
9921
10029
  for (const nb of graph.get(node) ?? []) {
9922
10030
  if (!sccSet.has(nb)) continue;
9923
- if (cycles.length >= MAX_CYCLES) break;
10031
+ if (cycles.length >= probeLimit) break;
9924
10032
  if (nb === start) {
9925
10033
  const cyclePath = [...pathSoFar, node, start];
9926
10034
  const key = canonicalCycleKey(cyclePath);
@@ -9935,6 +10043,7 @@ function findCycles(graph) {
9935
10043
  stack.delete(node);
9936
10044
  };
9937
10045
  var dfs = dfs2;
10046
+ if (cycles.length >= probeLimit) break;
9938
10047
  if (component.length === 1) {
9939
10048
  const [only] = component;
9940
10049
  if (only !== void 0 && (graph.get(only) ?? []).includes(only)) cycles.push([only, only]);
@@ -9943,11 +10052,12 @@ function findCycles(graph) {
9943
10052
  const sccSet = new Set(component);
9944
10053
  const stack = /* @__PURE__ */ new Set();
9945
10054
  for (const start of component) {
9946
- if (cycles.length >= MAX_CYCLES) break;
10055
+ if (cycles.length >= probeLimit) break;
9947
10056
  dfs2(start, start, []);
9948
10057
  }
9949
10058
  }
9950
- return cycles;
10059
+ if (cycles.length > MAX_CYCLES) return { cycles: cycles.slice(0, MAX_CYCLES), truncated: true };
10060
+ return { cycles, truncated: false };
9951
10061
  }
9952
10062
  function runSimilar(opts) {
9953
10063
  if (opts.top !== void 0 && opts.top <= 0) {
@@ -10051,8 +10161,8 @@ function runCoverageGaps(opts) {
10051
10161
  }
10052
10162
  const top = opts.top ?? 50;
10053
10163
  const rootDir = resolveProjectRoot({ project: process.cwd() });
10054
- const allFns = querySymbols({ kind: "function", limit: 2e3, rootDir });
10055
- const allMethods = querySymbols({ kind: "method", limit: 2e3, rootDir });
10164
+ const allFns = querySymbols({ kind: "function", limit: UNBOUNDED_REF_LIMIT, rootDir });
10165
+ const allMethods = querySymbols({ kind: "method", limit: UNBOUNDED_REF_LIMIT, rootDir });
10056
10166
  const candidates = [...allFns, ...allMethods];
10057
10167
  const gaps = [];
10058
10168
  for (const sym of candidates) {
@@ -10063,6 +10173,9 @@ function runCoverageGaps(opts) {
10063
10173
  if (!hasTestRef) gaps.push({ name: sym.name, kind: sym.kind, file: sym.filePath, line: sym.lineStart });
10064
10174
  }
10065
10175
  const sliced = gaps.slice(0, top);
10176
+ if (gaps.length > sliced.length) {
10177
+ emitErr2(`Showing top ${sliced.length} of ${gaps.length} coverage gaps (raise --top to see the rest).`);
10178
+ }
10066
10179
  if (opts.json === true) {
10067
10180
  emit2(JSON.stringify(sliced, null, 2));
10068
10181
  return 0;
@@ -10131,16 +10244,16 @@ function runArch(opts) {
10131
10244
  }
10132
10245
  const hubs = [...importedBy.entries()].sort((a, b) => b[1].size - a[1].size).slice(0, top).map(([f, importers]) => ({ file: f, importedBy: importers.size }));
10133
10246
  const entryPoints = files.filter((f) => !importedBy.has(f) && (graph.get(f) ?? []).length > 0).slice(0, top).map((f) => ({ file: f }));
10134
- const cycles = findCycles(graph);
10247
+ const { cycles, truncated: cyclesTruncated } = findCyclesCapped(graph);
10135
10248
  if (opts.json === true) {
10136
- emit2(JSON.stringify({ hubs, entryPoints, cycles }, null, 2));
10249
+ emit2(JSON.stringify({ hubs, entryPoints, cycles, ...cyclesTruncated ? { cyclesTruncated: true } : {} }, null, 2));
10137
10250
  return 0;
10138
10251
  }
10139
10252
  emit2(`hubs (top ${top} most-imported):`);
10140
10253
  for (const h of hubs) emit2(` ${h.importedBy} importers ${toDisplayPath(getDisplayRoot(opts.cwd), h.file)}`);
10141
10254
  emit2(`entry points (imported by nobody, top ${top}):`);
10142
10255
  for (const e of entryPoints) emit2(` ${toDisplayPath(getDisplayRoot(opts.cwd), e.file)}`);
10143
- emit2(`cycles (${cycles.length} found):`);
10256
+ emit2(cyclesTruncated ? `cycles (first ${cycles.length}, truncated at the ${MAX_CYCLES}-cycle enumeration limit: more cycles exist):` : `cycles (${cycles.length} found):`);
10144
10257
  for (const c of cycles) emit2(` ${c.map((f) => toDisplayPath(getDisplayRoot(opts.cwd), f)).join(" -> ")}`);
10145
10258
  return 0;
10146
10259
  }
@@ -10192,17 +10305,21 @@ function runAsk(opts) {
10192
10305
  const BACKEND_ENV = "TOKEN_GOAT_ASK_BACKEND";
10193
10306
  const backendLabel = process.env[BACKEND_ENV] ?? "";
10194
10307
  const entries = hits.map((h) => ({ file: h.filePath, symbol: h.name, kind: h.kind, line: h.lineStart, readCmd: `token-goat read "${h.filePath}::${h.name}@${h.lineStart}"` }));
10195
- const degrade = () => {
10308
+ const degrade = (reason, extraNote) => {
10196
10309
  if (opts.json === true) {
10197
- emit2(JSON.stringify({ degraded: true, note: `Set ${BACKEND_ENV}=claude|codex for LLM synthesis`, context: entries }, null, 2));
10310
+ emit2(JSON.stringify({ degraded: true, note: reason, ...extraNote !== void 0 ? { hint: extraNote } : {}, context: entries }, null, 2));
10198
10311
  return 0;
10199
10312
  }
10200
- emit2(`[degraded mode - set ${BACKEND_ENV}=claude|codex for LLM synthesis]`);
10313
+ emit2(`[degraded mode - ${reason}]`);
10314
+ if (extraNote !== void 0) emit2(extraNote);
10201
10315
  for (const e of entries) emit2(`token-goat read "${toDisplayPath(rootDir, e.file)}::${e.symbol}@${e.line}"`);
10202
10316
  return 0;
10203
10317
  };
10204
- if (!backendLabel) return degrade();
10205
- if (hits.length === 0) return degrade();
10318
+ if (!backendLabel) return degrade(`set ${BACKEND_ENV}=claude|codex for LLM synthesis`);
10319
+ if (hits.length === 0) {
10320
+ if (isIndexEmptyForProject(globalDbPath(), rootDir)) return degrade(`${BACKEND_ENV}=${backendLabel} is set, but nothing is indexed for this project yet, so there is no context to answer from`, emptyIndexMessage(rootDir));
10321
+ return degrade(`${BACKEND_ENV}=${backendLabel} is set, but no indexed symbol matched this question, so there is no context to answer from -- try different wording or token-goat semantic`);
10322
+ }
10206
10323
  const isWin = process.platform === "win32";
10207
10324
  let backendPath = null;
10208
10325
  try {
@@ -10211,7 +10328,7 @@ function runAsk(opts) {
10211
10328
  if (found) backendPath = found;
10212
10329
  } catch {
10213
10330
  }
10214
- if (!backendPath) return degrade();
10331
+ if (!backendPath) return degrade(`${BACKEND_ENV}=${backendLabel} is set, but '${backendLabel}' was not found on PATH`);
10215
10332
  const context = hits.map((h, i) => `[${i + 1}] ${h.filePath}
10216
10333
  ${h.body ?? ""}`).join("\n\n");
10217
10334
  const prompt = `Answer the QUESTION using only the CODE SNIPPETS below.
@@ -10262,7 +10379,7 @@ ANSWER:`;
10262
10379
  }
10263
10380
  }
10264
10381
  }
10265
- return degrade();
10382
+ return degrade(`${BACKEND_ENV}=${backendLabel} ran but returned no answer`);
10266
10383
  }
10267
10384
 
10268
10385
  // src/content_store.ts