token-goat 2.8.3 → 2.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,7 +8,7 @@ import {
8
8
  import { createRequire } from "node:module";
9
9
  function resolveVersion() {
10
10
  if (true) {
11
- return "2.8.3";
11
+ return "2.8.6";
12
12
  }
13
13
  const require2 = createRequire(import.meta.url);
14
14
  const pkg = require2("../package.json");
@@ -263,6 +263,16 @@ function lowercaseDriveLetter(s) {
263
263
  }
264
264
  var EXTENDED_UNC_PREFIX_RE = /^\\\\\?\\UNC\\/i;
265
265
  var EXTENDED_PREFIX_RE = /^\\\\\?\\/;
266
+ function shellMountToWindowsPath(p) {
267
+ const s = p.includes("\\") ? p.replace(/\\/g, "/") : p;
268
+ const m = WSL_PATH_RE.exec(s);
269
+ if (m) return `${m[1].toLowerCase()}:/${m[2].replace(/^\/+/, "")}`;
270
+ if (process.platform === "win32") {
271
+ const g = MSYS_PATH_RE.exec(s);
272
+ if (g) return `${g[1].toLowerCase()}:${g[2] ?? "/"}`;
273
+ }
274
+ return s;
275
+ }
266
276
  function normalizePath(p) {
267
277
  let s = p;
268
278
  if (EXTENDED_UNC_PREFIX_RE.test(s)) {
@@ -270,20 +280,7 @@ function normalizePath(p) {
270
280
  } else if (EXTENDED_PREFIX_RE.test(s)) {
271
281
  s = s.slice(4);
272
282
  }
273
- if (s.includes("\\")) {
274
- s = s.replace(/\\/g, "/");
275
- }
276
- const m = WSL_PATH_RE.exec(s);
277
- if (m) {
278
- const driveLetter = m[1].toLowerCase();
279
- const rest = m[2];
280
- const restStripped = rest.replace(/^\/+/, "");
281
- s = `${driveLetter}:/${restStripped}`;
282
- }
283
- if (process.platform === "win32") {
284
- const g = MSYS_PATH_RE.exec(s);
285
- if (g) s = `${g[1].toLowerCase()}:${g[2] ?? "/"}`;
286
- }
283
+ s = shellMountToWindowsPath(s);
287
284
  s = expandShortPath(s);
288
285
  s = lowercaseDriveLetter(s);
289
286
  s = normalizeDarwinSystemAlias(s);
@@ -297,8 +294,10 @@ function normalizeDarwinSystemAlias(p) {
297
294
  }
298
295
  function resolveIndexPath(file, base = process.cwd()) {
299
296
  const isWindowsAbsolute = (s) => /^[a-zA-Z]:[/\\]/.test(s);
300
- const resolve5 = isWindowsAbsolute(file) || isWindowsAbsolute(base) ? path2.win32.resolve : path2.resolve;
301
- return normalizePath(resolve5(base, file));
297
+ const f = shellMountToWindowsPath(file);
298
+ const b = shellMountToWindowsPath(base);
299
+ const resolve5 = isWindowsAbsolute(f) || isWindowsAbsolute(b) ? path2.win32.resolve : path2.resolve;
300
+ return normalizePath(resolve5(b, f));
302
301
  }
303
302
  function toDisplayPath(root, target) {
304
303
  if (root === void 0) return target;
@@ -759,6 +758,14 @@ function grepFilteredToEmptyNotice(preFilterCount, grep, nounSingular, nounPlura
759
758
  const pronoun = preFilterCount === 1 ? "it" : "them";
760
759
  return ` (all ${preFilterCount} ${noun} ${verb} filtered out by --grep ${grep} -- widen or drop the filter to see ${pronoun})`;
761
760
  }
761
+ function filtersFilteredToEmptyNotice(preFilterCount, activeFilters, nounSingular, nounPlural, reassurance) {
762
+ const noun = preFilterCount === 1 ? nounSingular : nounPlural;
763
+ const cause = activeFilters.length === 0 ? "the active filter" : activeFilters.join(" + ");
764
+ const knob = activeFilters.length > 1 ? "filters" : "filter";
765
+ const verb = preFilterCount === 1 ? "was" : "were";
766
+ const tail = reassurance === void 0 ? "" : `; ${reassurance}`;
767
+ return ` (all ${preFilterCount} ${noun} ${verb} filtered out by ${cause}${tail} -- widen or drop the ${knob} to see them)`;
768
+ }
762
769
  function countNoun(count, singular, plural = `${singular}s`) {
763
770
  return `${count} ${count === 1 ? singular : plural}`;
764
771
  }
@@ -2161,7 +2168,15 @@ var CONFIG_DEFAULTS = {
2161
2168
  // 85+, while a photo with an incidental sign or logo in frame scores much lower and
2162
2169
  // noisier -- padding the threshold below the terminal/code norm still comfortably
2163
2170
  // excludes photographic false positives without needing a second heuristic.
2164
- ocr_min_confidence: 65
2171
+ ocr_min_confidence: 65,
2172
+ // Which resolution tier the model being shown the image is on, which decides what its pixels
2173
+ // cost. 'standard' (1568px long edge, 1568 visual tokens) is every model before Claude 4.7;
2174
+ // 'high' (2576px, 4784 tokens) is 4.7 and later, and bills the same large image up to roughly
2175
+ // three times higher. Only the saving *reported* by `token-goat stats` depends on this -- no
2176
+ // image is encoded differently -- and 'standard' is the default because it is the floor: it
2177
+ // caps the counterfactual at the smaller of the two bills and so can never credit a saving
2178
+ // that was not there. Set it to 'high' on a Claude 4.7+ model to see the larger real figure.
2179
+ vision_tier: "standard"
2165
2180
  },
2166
2181
  screenshot: {
2167
2182
  chrome_path: "",
@@ -2236,7 +2251,18 @@ var CONFIG_DEFAULTS = {
2236
2251
  },
2237
2252
  worker: {
2238
2253
  blocked_roots: [],
2239
- max_pool_workers: 4
2254
+ max_pool_workers: 4,
2255
+ // 4, not 2. Measured on a 26-core Windows host with a foreground CPU probe: at `priority`
2256
+ // below_normal, 2, 4 and 6 threads are all indistinguishable from an idle machine, including
2257
+ // when the indexer and the probe are pinned to the same 4 cores, and including 4 threads pinned
2258
+ // to 2 cores, which is genuine oversubscription. The same probe reads -10% at 16 threads and
2259
+ // -69% with a 292 ms stall at 4 threads on 2 cores once the priority is normal instead. So the
2260
+ // priority below is what keeps the foreground responsive, not this number, and 4 buys a
2261
+ // measured 1.77x on indexing for no foreground cost. It stays a cap rather than tracking the
2262
+ // core count, because where the platform refuses the priority change (some hardened Linux
2263
+ // setups, sandboxes) this is the only thing left holding indexing back.
2264
+ embed_threads: 4,
2265
+ priority: "below_normal"
2240
2266
  },
2241
2267
  indexing: {
2242
2268
  large_file_symbol_only_kb: 500,
@@ -2321,6 +2347,9 @@ function validatedFloat(raw, def, min, max) {
2321
2347
  if (!Number.isFinite(n)) return def;
2322
2348
  return Math.max(min, Math.min(max, n));
2323
2349
  }
2350
+ function validatedVisionTier(raw, def) {
2351
+ return raw === "standard" || raw === "high" ? raw : def;
2352
+ }
2324
2353
  function validatedStr(raw, def) {
2325
2354
  return typeof raw === "string" ? raw : def;
2326
2355
  }
@@ -2396,6 +2425,7 @@ var NUMERIC_FIELD_BOUNDS = {
2396
2425
  "webfetch.max_bytes": { min: 0, max: 100 * 1024 * 1024 * 1024 },
2397
2426
  "webfetch.compress_min_bytes": { min: 1024, max: 10 * 1024 * 1024 },
2398
2427
  "worker.max_pool_workers": { min: 1, max: 8 },
2428
+ "worker.embed_threads": { min: 1, max: 16 },
2399
2429
  "indexing.large_file_symbol_only_kb": { min: 1, max: 1048576, clampTo: "indexing.large_file_skip_kb" },
2400
2430
  "indexing.large_file_skip_kb": { min: 1, max: 1048576 },
2401
2431
  "context.model_window_tokens": { min: 1e4, max: 1e7 },
@@ -2432,7 +2462,11 @@ function validateNumericField(fieldKey, value, cfg) {
2432
2462
  }
2433
2463
  var ENUM_FIELD_VALUES = {
2434
2464
  "compression.profile": ["auto", "aggressive", "balanced", "minimal"],
2435
- "compact_assist.harness": ["auto", ...KNOWN_HARNESS_NAMES]
2465
+ "compact_assist.harness": ["auto", ...KNOWN_HARNESS_NAMES],
2466
+ // Deliberately has no entry above `normal`. This table is what `config set` checks; a
2467
+ // hand-edited TOML bypasses it, which is why resolveWorkerPriority (process_priority.ts) maps an
2468
+ // unrecognized value back to the default rather than trusting whatever the file said.
2469
+ "worker.priority": ["below_normal", "low", "normal"]
2436
2470
  };
2437
2471
  function validateEnumField(fieldKey, value) {
2438
2472
  const allowed = ENUM_FIELD_VALUES[fieldKey];
@@ -2684,6 +2718,8 @@ function mergeRawConfig(base, override) {
2684
2718
  const baseSection = baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal) ? baseVal : {};
2685
2719
  merged[key] = { ...baseSection, ...overrideVal };
2686
2720
  } else {
2721
+ const baseVal = base[key];
2722
+ if (baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal)) continue;
2687
2723
  merged[key] = overrideVal;
2688
2724
  }
2689
2725
  }
@@ -2877,8 +2913,10 @@ function _buildConfig(raw, projectRaw = {}) {
2877
2913
  is_cfg.screenshot_redirect = validatedBool(is_raw["screenshot_redirect"], is_cfg.screenshot_redirect);
2878
2914
  is_cfg.ocr_enabled = validatedBool(is_raw["ocr_enabled"], is_cfg.ocr_enabled);
2879
2915
  is_cfg.ocr_min_confidence = validatedInt(is_raw["ocr_min_confidence"], is_cfg.ocr_min_confidence, ...boundsOf("image_shrink.ocr_min_confidence"));
2916
+ is_cfg.vision_tier = validatedVisionTier(is_raw["vision_tier"], is_cfg.vision_tier);
2880
2917
  is_cfg.max_image_pixels = envInt("TOKEN_GOAT_MAX_IMAGE_PIXELS", is_cfg.max_image_pixels, ...boundsOf("image_shrink.max_image_pixels"));
2881
2918
  is_cfg.ocr_enabled = envBool("TOKEN_GOAT_OCR_ENABLED", is_cfg.ocr_enabled);
2919
+ is_cfg.vision_tier = validatedVisionTier(process.env["TOKEN_GOAT_VISION_TIER"], is_cfg.vision_tier);
2882
2920
  const sc_raw = section(raw, "screenshot");
2883
2921
  const sc_cfg = getDefaultConfig("screenshot");
2884
2922
  sc_cfg.chrome_path = validatedStr(sc_raw["chrome_path"], sc_cfg.chrome_path);
@@ -2985,6 +3023,10 @@ function _buildConfig(raw, projectRaw = {}) {
2985
3023
  wk.blocked_roots = validatedStrList(wk_raw["blocked_roots"], wk.blocked_roots);
2986
3024
  wk.max_pool_workers = validatedInt(wk_raw["max_pool_workers"], wk.max_pool_workers, ...boundsOf("worker.max_pool_workers"));
2987
3025
  wk.max_pool_workers = envInt("TOKEN_GOAT_WORKER_MAX_POOL", wk.max_pool_workers, ...boundsOf("worker.max_pool_workers"));
3026
+ wk.embed_threads = validatedInt(wk_raw["embed_threads"], wk.embed_threads, ...boundsOf("worker.embed_threads"));
3027
+ wk.embed_threads = envInt("TOKEN_GOAT_EMBED_THREADS", wk.embed_threads, ...boundsOf("worker.embed_threads"));
3028
+ wk.priority = validatedStr(wk_raw["priority"], wk.priority);
3029
+ wk.priority = envStr("TOKEN_GOAT_WORKER_PRIORITY", wk.priority);
2988
3030
  const ix_raw = section(raw, "indexing");
2989
3031
  const ix = getDefaultConfig("indexing");
2990
3032
  ix.large_file_symbol_only_kb = validatedInt(ix_raw["large_file_symbol_only_kb"], ix.large_file_symbol_only_kb, ...boundsOf("indexing.large_file_symbol_only_kb"));
@@ -3077,6 +3119,7 @@ var CONFIG_KEY_ENV_OVERRIDES = {
3077
3119
  "skill_preservation.orphan_sweep_enabled": ["TOKEN_GOAT_ORPHAN_SWEEP"],
3078
3120
  "image_shrink.max_image_pixels": ["TOKEN_GOAT_MAX_IMAGE_PIXELS"],
3079
3121
  "image_shrink.ocr_enabled": ["TOKEN_GOAT_OCR_ENABLED"],
3122
+ "image_shrink.vision_tier": ["TOKEN_GOAT_VISION_TIER"],
3080
3123
  "screenshot.block_private_targets": ["TOKEN_GOAT_SCREENSHOT_BLOCK_PRIVATE_TARGETS"],
3081
3124
  "repomap.compact_file_threshold": ["TOKEN_GOAT_REPOMAP_COMPACT_THRESHOLD"],
3082
3125
  "repomap.exclude_tests": ["TOKEN_GOAT_REPOMAP_EXCLUDE_TESTS"],
@@ -3114,6 +3157,8 @@ var CONFIG_KEY_ENV_OVERRIDES = {
3114
3157
  "webfetch.max_bytes": ["TOKEN_GOAT_WEB_CACHE_MAX_BYTES"],
3115
3158
  "webfetch.compress_bodies": ["TOKEN_GOAT_WEB_COMPRESS"],
3116
3159
  "worker.max_pool_workers": ["TOKEN_GOAT_WORKER_MAX_POOL"],
3160
+ "worker.embed_threads": ["TOKEN_GOAT_EMBED_THREADS"],
3161
+ "worker.priority": ["TOKEN_GOAT_WORKER_PRIORITY"],
3117
3162
  "compression.profile": ["TOKEN_GOAT_COMPRESS_PROFILE"],
3118
3163
  "context.model_window_tokens": ["TOKEN_GOAT_MODEL_WINDOW_TOKENS"],
3119
3164
  "indexing.cross_project_symbols": ["TOKEN_GOAT_CROSS_PROJECT_SYMBOLS"],
@@ -3199,7 +3244,8 @@ function saveConfig(config) {
3199
3244
  max_image_pixels: is_cfg.max_image_pixels,
3200
3245
  screenshot_redirect: is_cfg.screenshot_redirect,
3201
3246
  ocr_enabled: is_cfg.ocr_enabled,
3202
- ocr_min_confidence: is_cfg.ocr_min_confidence
3247
+ ocr_min_confidence: is_cfg.ocr_min_confidence,
3248
+ vision_tier: is_cfg.vision_tier
3203
3249
  },
3204
3250
  screenshot: {
3205
3251
  chrome_path: config.screenshot.chrome_path,
@@ -3260,7 +3306,9 @@ function saveConfig(config) {
3260
3306
  },
3261
3307
  worker: {
3262
3308
  blocked_roots: config.worker.blocked_roots,
3263
- max_pool_workers: config.worker.max_pool_workers
3309
+ max_pool_workers: config.worker.max_pool_workers,
3310
+ embed_threads: config.worker.embed_threads,
3311
+ priority: config.worker.priority
3264
3312
  },
3265
3313
  indexing: {
3266
3314
  large_file_symbol_only_kb: config.indexing.large_file_symbol_only_kb,
@@ -4175,9 +4223,11 @@ function makeSymbolEmitter(symbols, sections, seen, filePath, maxSymbols = 500,
4175
4223
  }
4176
4224
  function assignFlatEndLines(sections, totalLines) {
4177
4225
  for (let i = 0; i < sections.length; i++) {
4178
- const next = sections[i + 1];
4179
4226
  const s = sections[i];
4180
4227
  if (s === void 0) continue;
4228
+ let j = i + 1;
4229
+ while (j < sections.length && (sections[j]?.line ?? 0) <= s.line) j++;
4230
+ const next = sections[j];
4181
4231
  const end = next !== void 0 ? next.line - 1 : totalLines;
4182
4232
  s.endLine = end < s.line ? s.line : end;
4183
4233
  }
@@ -4472,6 +4522,7 @@ function findBlockOpenBrace(content, lineIndex, startLine, lastSearchLine, lineC
4472
4522
  else if (ch === ")" || ch === "]") {
4473
4523
  if (parenDepth > 0) parenDepth--;
4474
4524
  } else if (ch === ";") return null;
4525
+ else if (ch === "}" && parenDepth === 0) return null;
4475
4526
  else if (ch === "{") return i;
4476
4527
  }
4477
4528
  return null;
@@ -4567,8 +4618,9 @@ function extractEnv(content, filePath) {
4567
4618
  if (_lineClosesQuote(line, openQuote)) openQuote = null;
4568
4619
  continue;
4569
4620
  }
4570
- if (!line || line[0] === "#" || line[0] === ";" || line[0] === " " || line[0] === " ") continue;
4571
- const m = ENV_KEY_RE.exec(line);
4621
+ const trimmed = line.replace(/^[ \t]+/, "");
4622
+ if (!trimmed || trimmed[0] === "#" || trimmed[0] === ";") continue;
4623
+ const m = ENV_KEY_RE.exec(trimmed);
4572
4624
  if (m === null) continue;
4573
4625
  const name = m[1]?.trim() ?? "";
4574
4626
  if (!name || name.length > MAX_HEADING_LEN) continue;
@@ -4576,7 +4628,7 @@ function extractEnv(content, filePath) {
4576
4628
  if (seen.has(key)) continue;
4577
4629
  seen.add(key);
4578
4630
  symbols.push(makeLineSymbol(filePath, name, "env_key", i + 1));
4579
- openQuote = _detectOpenQuote(line.slice(m[0].length));
4631
+ openQuote = _detectOpenQuote(trimmed.slice(m[0].length));
4580
4632
  }
4581
4633
  return symbols;
4582
4634
  }
@@ -4955,7 +5007,8 @@ CREATE TABLE IF NOT EXISTS files (
4955
5007
  language TEXT,
4956
5008
  indexed_at REAL,
4957
5009
  embed_sha TEXT,
4958
- retry_count INTEGER NOT NULL DEFAULT 0
5010
+ retry_count INTEGER NOT NULL DEFAULT 0,
5011
+ parser_sha TEXT
4959
5012
  );
4960
5013
  -- Expression index on TG_LOWER(path) -- see pathEqClause (sql_path.ts) and TG_LOWER's
4961
5014
  -- registration above. TG_LOWER is registered { deterministic: true }, which is required for
@@ -5215,7 +5268,7 @@ CREATE TRIGGER IF NOT EXISTS cache_recall_au AFTER UPDATE ON cache_recall BEGIN
5215
5268
  VALUES (new.row_id, new.label, new.content);
5216
5269
  END;
5217
5270
  `;
5218
- var SCHEMA_VERSION = 12;
5271
+ var SCHEMA_VERSION = 13;
5219
5272
  function alterTableIdempotent(conn, sql) {
5220
5273
  try {
5221
5274
  conn.exec(sql);
@@ -5259,7 +5312,9 @@ var MIGRATIONS = {
5259
5312
  // .env would have kept serving its pre-fix chunks indefinitely. Deleting the rows here both
5260
5313
  // removes the stored secrets and, by clearing embed_sha, makes the next drain re-embed the file
5261
5314
  // through the redacting path.
5262
- 10: purgeDotenvEmbeddings
5315
+ 10: purgeDotenvEmbeddings,
5316
+ // v12 -> v13: adds files.parser_sha, the digest of the extraction logic that produced this file's rows, tracked separately from files.sha for the same reason embed_sha is -- content freshness and parse freshness are different questions, and the content sha alone could only ever answer the first. A pre-existing v12 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 / v9 -> v10 above. Deliberately left NULL for every existing row rather than backfilled with the current fingerprint: NULL is the truthful answer (nobody recorded which parser wrote those rows), and it is also the answer that makes the freshness gates reparse them once, which is exactly what a database indexed by an older parser needs.
5317
+ 12: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN parser_sha TEXT")
5263
5318
  };
5264
5319
  function runMigrations(conn, fromVersion, toVersion) {
5265
5320
  for (let v = fromVersion; v < toVersion; v++) {
@@ -5745,7 +5800,51 @@ var _KIND_GROUPS = [
5745
5800
  "exports",
5746
5801
  "imports",
5747
5802
  "changed_lookup",
5748
- "dep_docs"
5803
+ "dep_docs",
5804
+ // Every other SOURCE_READ kind in stats.ts's KIND_TO_SOURCE: the surgical-read commands over documents, structured data and session/PR state. They were registered and produced but grouped nowhere, so `stats --full` printed the whole family under 'Other', away from the read-savings siblings they are measured against. image_meta/image_text sit here rather than under 'Images' because stats.ts files them as SOURCE_READ: they save read bytes, they do not shrink an image.
5805
+ "brief_view",
5806
+ "conflicts",
5807
+ "coverage_report_gaps",
5808
+ "csv_query",
5809
+ "csv_profile",
5810
+ "compact_doc",
5811
+ "docx_outline",
5812
+ "docx_text",
5813
+ "gdrive_sections",
5814
+ "image_meta",
5815
+ "image_text",
5816
+ "json_query",
5817
+ "json_outline",
5818
+ "note_read",
5819
+ "note_list",
5820
+ "openapi_op",
5821
+ "openapi_outline",
5822
+ "pdf_extract",
5823
+ "pdf_locate",
5824
+ "pdf_outline",
5825
+ "pdf_meta",
5826
+ "pptx_outline",
5827
+ "pptx_slide",
5828
+ "pptx_notes",
5829
+ "pptx_text",
5830
+ "pr_slice",
5831
+ "session_outline",
5832
+ "session_slice",
5833
+ "sqlite_query",
5834
+ "sqlite_schema",
5835
+ "transcript",
5836
+ "transcript_outline",
5837
+ "video_chapters",
5838
+ "xlsx_sheets",
5839
+ "xlsx_head",
5840
+ "xlsx_range",
5841
+ "xlsx_query",
5842
+ "xml_query",
5843
+ "xml_outline",
5844
+ "yaml_query",
5845
+ "yaml_outline",
5846
+ "zip_list",
5847
+ "zip_read"
5749
5848
  ])
5750
5849
  },
5751
5850
  { label: "Lookups", members: /* @__PURE__ */ new Set(["symbol_lookup", "semantic_search", "map_lookup"]) },
@@ -5767,74 +5866,72 @@ var _KIND_GROUPS = [
5767
5866
  "session_hint_overhead",
5768
5867
  "session_hint_suppressed",
5769
5868
  "read_count_deny",
5770
- "read_dedup_hint",
5771
5869
  "grep_dedup_hint",
5772
5870
  "glob_dedup_hint",
5773
5871
  "diff_hint",
5774
5872
  "predictive_prefetch_hit",
5775
- "read_partial_overlap_hint",
5776
5873
  "structured_file_hint",
5777
5874
  "write_rewrite_hint",
5778
5875
  "websearch_dedup_hint",
5779
5876
  "large_file_hint_followed",
5780
- "large_file_hint_ignored"
5781
- ])
5782
- },
5783
- {
5784
- label: "Bash",
5785
- members: /* @__PURE__ */ new Set([
5786
- "bash_dedup_hint",
5787
- "bash_output_cached",
5788
- "bash_output_recall",
5789
- "bash_output_recall_miss",
5790
- "bash_dedup_stale",
5791
- "bash_range_read_hint",
5792
- "bash_streak_hint",
5793
- "bash_poll_hint",
5794
- "env_probe_cache_hit",
5795
- "git_diff_scope_hint",
5796
- "dep_list_cache_hit",
5797
- "bash_read_equiv_already_read",
5798
- "bash_grep_result_cache_hit",
5799
- "git_diff_context_trimmed"
5877
+ "large_file_hint_ignored",
5878
+ "evidence_cache_hit"
5800
5879
  ])
5801
5880
  },
5881
+ // Empty for the same reason as MCP below: every live Bash kind arrives through _kindGroupLabel's `bash_compress:` prefix branch, not through a literal name. The fifteen literal names this set used to carry (bash_output_cached, bash_dedup_hint, env_probe_cache_hit and the rest) came over with the Python port and were never recorded or registered anywhere in this tree, so they grouped rows that could not exist.
5882
+ { label: "Bash", members: /* @__PURE__ */ new Set() },
5802
5883
  {
5803
5884
  label: "Web",
5804
5885
  members: /* @__PURE__ */ new Set([
5805
- "web_dedup_hint",
5806
- "web_output_cached",
5807
- "web_output_recall",
5808
- "web_output_recall_miss",
5809
- "web_dedup_stale",
5810
5886
  "web_fetch",
5811
5887
  "injection_detected"
5812
5888
  ])
5813
5889
  },
5890
+ // Membership comes from _kindGroupLabel's `mcp:` prefix branch, not from this set, which is why
5891
+ // it is empty. The entry still has to exist: _renderByKindSection iterates _KIND_GROUPS' labels
5892
+ // (plus 'Other') to decide what to print, so a label _kindGroupLabel returns but that is missing
5893
+ // here does not fall back to 'Other' -- its rows disappear from the table entirely.
5894
+ { label: "MCP", members: /* @__PURE__ */ new Set() },
5814
5895
  {
5815
5896
  label: "Compact / Skills",
5816
5897
  members: /* @__PURE__ */ new Set([
5817
- "compact_manifest",
5818
- "compact_assist",
5819
- "compact_recovery",
5820
- "skill_body_recall",
5821
- "skill_compact_served",
5822
- "skill_cached",
5823
5898
  "skill_load",
5824
5899
  "skill_oversized_first_load",
5825
- "skill_compact_inlined",
5826
- "resume_packet",
5827
- "decision_log"
5900
+ "skill_compact_inlined"
5901
+ ])
5902
+ },
5903
+ // SOURCE_CONTENT: real rewrites of tool output that remove real bytes (agent report compaction, Grep fold, browser tab dedup, bash/content compression and the handoff pair). The by-source table has shown a 'content' row since the source was added, but the by-kind table had no member set for it, so every one of these kinds printed under 'Other'. The taskoutput: prefix branch in _kindGroupLabel routes here too.
5904
+ {
5905
+ label: "Content",
5906
+ members: /* @__PURE__ */ new Set([
5907
+ "content_compress",
5908
+ "content_retrieve",
5909
+ "agent_report_compact",
5910
+ "agent_report_compact_declined",
5911
+ "browser_tab_dedup",
5912
+ "grep:fold",
5913
+ "handoff_create",
5914
+ "handoff_resolve",
5915
+ "plan_echo_collapse"
5828
5916
  ])
5829
5917
  }
5830
5918
  ];
5831
5919
  function _kindGroupLabel(kind) {
5832
- if (kind.startsWith("bash_compress:")) {
5920
+ if (kind.startsWith("bash_compress:") || kind.startsWith("bashoutput:")) {
5833
5921
  return "Bash";
5834
5922
  }
5835
- if (kind.startsWith("webfetch:")) {
5923
+ if (kind.startsWith("webfetch:") || kind.startsWith("gdrive:")) {
5836
5924
  return "Web";
5837
5925
  }
5926
+ if (kind.startsWith("mcp:")) {
5927
+ return "MCP";
5928
+ }
5929
+ if (kind.startsWith("skill_body:") || kind.startsWith("skill_compact:")) {
5930
+ return "Compact / Skills";
5931
+ }
5932
+ if (kind.startsWith("taskoutput:")) {
5933
+ return "Content";
5934
+ }
5838
5935
  for (const group of _KIND_GROUPS) {
5839
5936
  if (group.members.has(kind)) {
5840
5937
  return group.label;
@@ -6138,6 +6235,10 @@ var SOURCE_SKILL = "skill";
6138
6235
  var SOURCE_CONTENT = "content";
6139
6236
  var SOURCE_OTHER = "other";
6140
6237
  var _BYTES_MODE_ONLY_KINDS = /* @__PURE__ */ new Set(["webfetch_image", "gdrive_image"]);
6238
+ var COUNT_ONLY_KINDS = /* @__PURE__ */ new Set(["secret_redacted"]);
6239
+ function savedTokensFromBytes(bytes) {
6240
+ return Math.round(Math.max(0, bytes) / 4);
6241
+ }
6141
6242
  var KIND_TO_SOURCE = {
6142
6243
  image_shrink: SOURCE_IMAGE,
6143
6244
  image_shrink_cache_hit: SOURCE_IMAGE,
@@ -6238,6 +6339,10 @@ var KIND_TO_SOURCE = {
6238
6339
  // Decline counterpart to agent_report_compact: the fence-collapse net-benefit gate ran and found at least one over-long fence, but declined to rewrite because net savings did not clear the notice cost. Always recorded at (0, 0) -- see the recordStat call site -- so it never contributes to any savings total; it exists purely to make gate hit-rate and near-misses visible instead of the decline being invisible.
6239
6340
  agent_report_compact_declined: SOURCE_CONTENT,
6240
6341
  content_compress: SOURCE_CONTENT,
6342
+ // Verbatim-repeat collapse of a browser tool's "Tab Context:" text block (hooks_browser_image.ts postBrowserImageHandler). SOURCE_CONTENT for the same reason as agent_report_compact above: it is a real rewrite with real bytes removed, not an advisory nudge. Deliberately not SOURCE_IMAGE -- it shares a handler with image_shrink but collapses text, and folding text bytes into the image ledger is the two-units-under-one-label mistake this file's image_shrink entry was just fixed for.
6343
+ browser_tab_dedup: SOURCE_CONTENT,
6344
+ // Collapse of the plan echo in an approved ExitPlanMode result (hooks_exitplanmode.ts). SOURCE_CONTENT for the same reason as agent_report_compact: real bytes removed from a tool result, not an advisory nudge. The handler shipped for releases emitting this rewrite and recording nothing at all, so the mechanism was invisible in `stats` and its net benefit could not be checked against the gate that admits it.
6345
+ plan_echo_collapse: SOURCE_CONTENT,
6241
6346
  // Lossless re-layout of Grep content-mode output (hooks_grep.ts foldGrepContentHandler). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as agent_report_compact above: its sibling grep_dedup_hint is advisory and saves nothing directly, whereas this is a real rewrite with real bytes removed. Filing it under the advisory bucket would silently add non-hint savings to hint_stats.ts's savedBytes (which reads by_source[SOURCE_HINT] wholesale) and overstate the hint ledger's net benefit.
6242
6347
  "grep:fold": SOURCE_CONTENT,
6243
6348
  content_retrieve: SOURCE_CONTENT,
@@ -6366,7 +6471,8 @@ CREATE TABLE IF NOT EXISTS stats (
6366
6471
  tokens_saved INTEGER NOT NULL DEFAULT 0,
6367
6472
  bytes_saved INTEGER NOT NULL DEFAULT 0,
6368
6473
  detail TEXT,
6369
- harness TEXT
6474
+ harness TEXT,
6475
+ traceparent TEXT
6370
6476
  );
6371
6477
  CREATE INDEX IF NOT EXISTS idx_stats_ts ON stats(ts);
6372
6478
  CREATE INDEX IF NOT EXISTS idx_stats_kind ON stats(kind);
@@ -6389,6 +6495,11 @@ function migrateGlobalSchema(db) {
6389
6495
  } catch (err) {
6390
6496
  if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
6391
6497
  }
6498
+ try {
6499
+ db.exec("ALTER TABLE stats ADD COLUMN traceparent TEXT");
6500
+ } catch (err) {
6501
+ if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
6502
+ }
6392
6503
  }
6393
6504
  var _harnessColumnByDb = /* @__PURE__ */ new WeakMap();
6394
6505
  function statsHasHarnessColumn(db) {
@@ -6405,6 +6516,21 @@ function statsHasHarnessColumn(db) {
6405
6516
  _harnessColumnByDb.set(db, present);
6406
6517
  return present;
6407
6518
  }
6519
+ var _traceparentColumnByDb = /* @__PURE__ */ new WeakMap();
6520
+ function statsHasTraceparentColumn(db) {
6521
+ const cached = _traceparentColumnByDb.get(db);
6522
+ if (cached !== void 0) return cached;
6523
+ let present;
6524
+ try {
6525
+ present = db.prepare("PRAGMA table_info(stats)").all().some(
6526
+ (c) => c.name === "traceparent"
6527
+ );
6528
+ } catch {
6529
+ present = false;
6530
+ }
6531
+ _traceparentColumnByDb.set(db, present);
6532
+ return present;
6533
+ }
6408
6534
  function getGlobalDb(homeDir) {
6409
6535
  const basePath = homeDir ? dataDirForHome(homeDir) : dataDir();
6410
6536
  const dbPath = path8.join(basePath, "global.db");
@@ -6423,11 +6549,18 @@ function noStatsMessage(windowDays, homeDir) {
6423
6549
  if (total === 0) return "No stats recorded yet.";
6424
6550
  return `No stats in the last ${countNoun(windowDays, "day")} (${total} recorded outside this window; use --window-days 0 for all time).`;
6425
6551
  }
6426
- function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail) {
6552
+ function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail, traceparent) {
6427
6553
  try {
6428
6554
  const db = _testDb ?? getGlobalDb();
6429
6555
  const ts = Math.floor(Date.now() / 1e3);
6430
- if (statsHasHarnessColumn(db)) {
6556
+ const tp = traceparent ?? process.env["TRACEPARENT"] ?? process.env["traceparent"] ?? null;
6557
+ const hasHarness = statsHasHarnessColumn(db);
6558
+ const hasTraceparent = statsHasTraceparentColumn(db);
6559
+ if (hasHarness && hasTraceparent) {
6560
+ db.prepare(
6561
+ "INSERT INTO stats (ts, kind, bytes_saved, tokens_saved, detail, harness, traceparent) VALUES (?, ?, ?, ?, ?, ?, ?)"
6562
+ ).run(ts, kind, bytesSaved, tokensSaved, detail ?? null, getHarnessName(), tp);
6563
+ } else if (hasHarness) {
6431
6564
  db.prepare(
6432
6565
  "INSERT INTO stats (ts, kind, bytes_saved, tokens_saved, detail, harness) VALUES (?, ?, ?, ?, ?, ?)"
6433
6566
  ).run(ts, kind, bytesSaved, tokensSaved, detail ?? null, getHarnessName());
@@ -6482,10 +6615,14 @@ function summarize(windowDays = 30, testDb, homeDir) {
6482
6615
  const stmt = db.prepare(query);
6483
6616
  const rows = sinceTs !== null ? stmt.all(sinceTs) : stmt.all();
6484
6617
  const tsToDateCache = {};
6618
+ const counts = {};
6485
6619
  for (const row of rows) {
6486
6620
  const bytesSaved = row.bytes_saved ?? 0;
6487
- const tokensSaved = row.tokens_saved ?? 0;
6621
+ const recorded = row.tokens_saved ?? 0;
6488
6622
  const kind = row.kind;
6623
+ const isCount = COUNT_ONLY_KINDS.has(kind);
6624
+ if (isCount) counts[kind] = (counts[kind] ?? 0) + recorded;
6625
+ const tokensSaved = isCount ? 0 : recorded;
6489
6626
  const tsRaw = row.ts;
6490
6627
  if (tsRaw === void 0) continue;
6491
6628
  const ts = tsRaw;
@@ -6544,6 +6681,7 @@ function summarize(windowDays = 30, testDb, homeDir) {
6544
6681
  by_project: byProjectList,
6545
6682
  by_source: bySourceDict,
6546
6683
  by_harness: byHarness,
6684
+ counts,
6547
6685
  by_command: Object.entries(byCommandDict).map(([command, bucket]) => ({ ...bucket, command })).filter((r) => r.events > 0),
6548
6686
  window_days: windowDays
6549
6687
  };
@@ -6554,6 +6692,10 @@ function _totalsLines(summary) {
6554
6692
  `Total events: ${summary.total_events}`,
6555
6693
  `Bytes saved: ${fmtBytes(summary.total_bytes_saved)}`,
6556
6694
  `Tokens saved: ${summary.total_tokens_saved}`,
6695
+ // Printed on its own line, below the token total and never inside it, because it counts
6696
+ // placeholders rather than tokens. Omitted entirely when nothing was redacted, so the line is
6697
+ // information rather than a permanent zero. See COUNT_ONLY_KINDS.
6698
+ ...summary.counts["secret_redacted"] ? [`Secrets hidden: ${summary.counts["secret_redacted"]} (a count, not tokens)`] : [],
6557
6699
  `Window: ${summary.window_days} days`
6558
6700
  ];
6559
6701
  }
@@ -6872,6 +7014,7 @@ export {
6872
7014
  toKB,
6873
7015
  compileGrepMatcher,
6874
7016
  grepFilteredToEmptyNotice,
7017
+ filtersFilteredToEmptyNotice,
6875
7018
  countNoun,
6876
7019
  excludeTestsHiddenNote,
6877
7020
  countContentLines,
@@ -6959,6 +7102,7 @@ export {
6959
7102
  fg,
6960
7103
  C,
6961
7104
  SOURCE_HINT,
7105
+ savedTokensFromBytes,
6962
7106
  formatLocalTimestamp,
6963
7107
  recordStat,
6964
7108
  recordUnmappedTool,