token-goat 2.8.3 → 2.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -17
- package/dist/{token-goat-chunk-6ODZ6PZK.mjs → token-goat-chunk-2JZ66BBE.mjs} +216 -72
- package/dist/{token-goat-chunk-PWVXXPCC.mjs → token-goat-chunk-DK4VLLYB.mjs} +563 -155
- package/dist/{token-goat-chunk-EFF2XCLB.mjs → token-goat-chunk-GUNYAGOZ.mjs} +175 -56
- package/dist/{token-goat-chunk-VAGPOZHO.mjs → token-goat-chunk-L2XHDICZ.mjs} +2 -2
- package/dist/{token-goat-chunk-DP3NNGGV.mjs → token-goat-chunk-LJEETTER.mjs} +5 -5
- package/dist/{token-goat-chunk-NKNCHJ4H.mjs → token-goat-chunk-MA5237JN.mjs} +327 -169
- package/dist/{token-goat-chunk-4OM2Q2SX.mjs → token-goat-chunk-SHN4UTL4.mjs} +5 -5
- package/dist/{token-goat-chunk-4HIMCBYK.mjs → token-goat-chunk-TELKICYU.mjs} +55 -6
- package/dist/{token-goat-chunk-222VPFP2.mjs → token-goat-chunk-U7M2LTGP.mjs} +1100 -269
- package/dist/{token-goat-chunk-TX4JFJTD.mjs → token-goat-chunk-XAWIELXH.mjs} +216 -105
- package/dist/{token-goat-chunk-C5IL6MJH.mjs → token-goat-chunk-ZIPBLIUZ.mjs} +6 -5
- package/dist/token-goat-hook.mjs +5 -5
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +3 -1
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
import { createRequire } from "node:module";
|
|
9
9
|
function resolveVersion() {
|
|
10
10
|
if (true) {
|
|
11
|
-
return "2.8.
|
|
11
|
+
return "2.8.6";
|
|
12
12
|
}
|
|
13
13
|
const require2 = createRequire(import.meta.url);
|
|
14
14
|
const pkg = require2("../package.json");
|
|
@@ -263,6 +263,16 @@ function lowercaseDriveLetter(s) {
|
|
|
263
263
|
}
|
|
264
264
|
var EXTENDED_UNC_PREFIX_RE = /^\\\\\?\\UNC\\/i;
|
|
265
265
|
var EXTENDED_PREFIX_RE = /^\\\\\?\\/;
|
|
266
|
+
function shellMountToWindowsPath(p) {
|
|
267
|
+
const s = p.includes("\\") ? p.replace(/\\/g, "/") : p;
|
|
268
|
+
const m = WSL_PATH_RE.exec(s);
|
|
269
|
+
if (m) return `${m[1].toLowerCase()}:/${m[2].replace(/^\/+/, "")}`;
|
|
270
|
+
if (process.platform === "win32") {
|
|
271
|
+
const g = MSYS_PATH_RE.exec(s);
|
|
272
|
+
if (g) return `${g[1].toLowerCase()}:${g[2] ?? "/"}`;
|
|
273
|
+
}
|
|
274
|
+
return s;
|
|
275
|
+
}
|
|
266
276
|
function normalizePath(p) {
|
|
267
277
|
let s = p;
|
|
268
278
|
if (EXTENDED_UNC_PREFIX_RE.test(s)) {
|
|
@@ -270,20 +280,7 @@ function normalizePath(p) {
|
|
|
270
280
|
} else if (EXTENDED_PREFIX_RE.test(s)) {
|
|
271
281
|
s = s.slice(4);
|
|
272
282
|
}
|
|
273
|
-
|
|
274
|
-
s = s.replace(/\\/g, "/");
|
|
275
|
-
}
|
|
276
|
-
const m = WSL_PATH_RE.exec(s);
|
|
277
|
-
if (m) {
|
|
278
|
-
const driveLetter = m[1].toLowerCase();
|
|
279
|
-
const rest = m[2];
|
|
280
|
-
const restStripped = rest.replace(/^\/+/, "");
|
|
281
|
-
s = `${driveLetter}:/${restStripped}`;
|
|
282
|
-
}
|
|
283
|
-
if (process.platform === "win32") {
|
|
284
|
-
const g = MSYS_PATH_RE.exec(s);
|
|
285
|
-
if (g) s = `${g[1].toLowerCase()}:${g[2] ?? "/"}`;
|
|
286
|
-
}
|
|
283
|
+
s = shellMountToWindowsPath(s);
|
|
287
284
|
s = expandShortPath(s);
|
|
288
285
|
s = lowercaseDriveLetter(s);
|
|
289
286
|
s = normalizeDarwinSystemAlias(s);
|
|
@@ -297,8 +294,10 @@ function normalizeDarwinSystemAlias(p) {
|
|
|
297
294
|
}
|
|
298
295
|
function resolveIndexPath(file, base = process.cwd()) {
|
|
299
296
|
const isWindowsAbsolute = (s) => /^[a-zA-Z]:[/\\]/.test(s);
|
|
300
|
-
const
|
|
301
|
-
|
|
297
|
+
const f = shellMountToWindowsPath(file);
|
|
298
|
+
const b = shellMountToWindowsPath(base);
|
|
299
|
+
const resolve5 = isWindowsAbsolute(f) || isWindowsAbsolute(b) ? path2.win32.resolve : path2.resolve;
|
|
300
|
+
return normalizePath(resolve5(b, f));
|
|
302
301
|
}
|
|
303
302
|
function toDisplayPath(root, target) {
|
|
304
303
|
if (root === void 0) return target;
|
|
@@ -759,6 +758,14 @@ function grepFilteredToEmptyNotice(preFilterCount, grep, nounSingular, nounPlura
|
|
|
759
758
|
const pronoun = preFilterCount === 1 ? "it" : "them";
|
|
760
759
|
return ` (all ${preFilterCount} ${noun} ${verb} filtered out by --grep ${grep} -- widen or drop the filter to see ${pronoun})`;
|
|
761
760
|
}
|
|
761
|
+
function filtersFilteredToEmptyNotice(preFilterCount, activeFilters, nounSingular, nounPlural, reassurance) {
|
|
762
|
+
const noun = preFilterCount === 1 ? nounSingular : nounPlural;
|
|
763
|
+
const cause = activeFilters.length === 0 ? "the active filter" : activeFilters.join(" + ");
|
|
764
|
+
const knob = activeFilters.length > 1 ? "filters" : "filter";
|
|
765
|
+
const verb = preFilterCount === 1 ? "was" : "were";
|
|
766
|
+
const tail = reassurance === void 0 ? "" : `; ${reassurance}`;
|
|
767
|
+
return ` (all ${preFilterCount} ${noun} ${verb} filtered out by ${cause}${tail} -- widen or drop the ${knob} to see them)`;
|
|
768
|
+
}
|
|
762
769
|
function countNoun(count, singular, plural = `${singular}s`) {
|
|
763
770
|
return `${count} ${count === 1 ? singular : plural}`;
|
|
764
771
|
}
|
|
@@ -2161,7 +2168,15 @@ var CONFIG_DEFAULTS = {
|
|
|
2161
2168
|
// 85+, while a photo with an incidental sign or logo in frame scores much lower and
|
|
2162
2169
|
// noisier -- padding the threshold below the terminal/code norm still comfortably
|
|
2163
2170
|
// excludes photographic false positives without needing a second heuristic.
|
|
2164
|
-
ocr_min_confidence: 65
|
|
2171
|
+
ocr_min_confidence: 65,
|
|
2172
|
+
// Which resolution tier the model being shown the image is on, which decides what its pixels
|
|
2173
|
+
// cost. 'standard' (1568px long edge, 1568 visual tokens) is every model before Claude 4.7;
|
|
2174
|
+
// 'high' (2576px, 4784 tokens) is 4.7 and later, and bills the same large image up to roughly
|
|
2175
|
+
// three times higher. Only the saving *reported* by `token-goat stats` depends on this -- no
|
|
2176
|
+
// image is encoded differently -- and 'standard' is the default because it is the floor: it
|
|
2177
|
+
// caps the counterfactual at the smaller of the two bills and so can never credit a saving
|
|
2178
|
+
// that was not there. Set it to 'high' on a Claude 4.7+ model to see the larger real figure.
|
|
2179
|
+
vision_tier: "standard"
|
|
2165
2180
|
},
|
|
2166
2181
|
screenshot: {
|
|
2167
2182
|
chrome_path: "",
|
|
@@ -2236,7 +2251,18 @@ var CONFIG_DEFAULTS = {
|
|
|
2236
2251
|
},
|
|
2237
2252
|
worker: {
|
|
2238
2253
|
blocked_roots: [],
|
|
2239
|
-
max_pool_workers: 4
|
|
2254
|
+
max_pool_workers: 4,
|
|
2255
|
+
// 4, not 2. Measured on a 26-core Windows host with a foreground CPU probe: at `priority`
|
|
2256
|
+
// below_normal, 2, 4 and 6 threads are all indistinguishable from an idle machine, including
|
|
2257
|
+
// when the indexer and the probe are pinned to the same 4 cores, and including 4 threads pinned
|
|
2258
|
+
// to 2 cores, which is genuine oversubscription. The same probe reads -10% at 16 threads and
|
|
2259
|
+
// -69% with a 292 ms stall at 4 threads on 2 cores once the priority is normal instead. So the
|
|
2260
|
+
// priority below is what keeps the foreground responsive, not this number, and 4 buys a
|
|
2261
|
+
// measured 1.77x on indexing for no foreground cost. It stays a cap rather than tracking the
|
|
2262
|
+
// core count, because where the platform refuses the priority change (some hardened Linux
|
|
2263
|
+
// setups, sandboxes) this is the only thing left holding indexing back.
|
|
2264
|
+
embed_threads: 4,
|
|
2265
|
+
priority: "below_normal"
|
|
2240
2266
|
},
|
|
2241
2267
|
indexing: {
|
|
2242
2268
|
large_file_symbol_only_kb: 500,
|
|
@@ -2321,6 +2347,9 @@ function validatedFloat(raw, def, min, max) {
|
|
|
2321
2347
|
if (!Number.isFinite(n)) return def;
|
|
2322
2348
|
return Math.max(min, Math.min(max, n));
|
|
2323
2349
|
}
|
|
2350
|
+
function validatedVisionTier(raw, def) {
|
|
2351
|
+
return raw === "standard" || raw === "high" ? raw : def;
|
|
2352
|
+
}
|
|
2324
2353
|
function validatedStr(raw, def) {
|
|
2325
2354
|
return typeof raw === "string" ? raw : def;
|
|
2326
2355
|
}
|
|
@@ -2396,6 +2425,7 @@ var NUMERIC_FIELD_BOUNDS = {
|
|
|
2396
2425
|
"webfetch.max_bytes": { min: 0, max: 100 * 1024 * 1024 * 1024 },
|
|
2397
2426
|
"webfetch.compress_min_bytes": { min: 1024, max: 10 * 1024 * 1024 },
|
|
2398
2427
|
"worker.max_pool_workers": { min: 1, max: 8 },
|
|
2428
|
+
"worker.embed_threads": { min: 1, max: 16 },
|
|
2399
2429
|
"indexing.large_file_symbol_only_kb": { min: 1, max: 1048576, clampTo: "indexing.large_file_skip_kb" },
|
|
2400
2430
|
"indexing.large_file_skip_kb": { min: 1, max: 1048576 },
|
|
2401
2431
|
"context.model_window_tokens": { min: 1e4, max: 1e7 },
|
|
@@ -2432,7 +2462,11 @@ function validateNumericField(fieldKey, value, cfg) {
|
|
|
2432
2462
|
}
|
|
2433
2463
|
var ENUM_FIELD_VALUES = {
|
|
2434
2464
|
"compression.profile": ["auto", "aggressive", "balanced", "minimal"],
|
|
2435
|
-
"compact_assist.harness": ["auto", ...KNOWN_HARNESS_NAMES]
|
|
2465
|
+
"compact_assist.harness": ["auto", ...KNOWN_HARNESS_NAMES],
|
|
2466
|
+
// Deliberately has no entry above `normal`. This table is what `config set` checks; a
|
|
2467
|
+
// hand-edited TOML bypasses it, which is why resolveWorkerPriority (process_priority.ts) maps an
|
|
2468
|
+
// unrecognized value back to the default rather than trusting whatever the file said.
|
|
2469
|
+
"worker.priority": ["below_normal", "low", "normal"]
|
|
2436
2470
|
};
|
|
2437
2471
|
function validateEnumField(fieldKey, value) {
|
|
2438
2472
|
const allowed = ENUM_FIELD_VALUES[fieldKey];
|
|
@@ -2684,6 +2718,8 @@ function mergeRawConfig(base, override) {
|
|
|
2684
2718
|
const baseSection = baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal) ? baseVal : {};
|
|
2685
2719
|
merged[key] = { ...baseSection, ...overrideVal };
|
|
2686
2720
|
} else {
|
|
2721
|
+
const baseVal = base[key];
|
|
2722
|
+
if (baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal)) continue;
|
|
2687
2723
|
merged[key] = overrideVal;
|
|
2688
2724
|
}
|
|
2689
2725
|
}
|
|
@@ -2877,8 +2913,10 @@ function _buildConfig(raw, projectRaw = {}) {
|
|
|
2877
2913
|
is_cfg.screenshot_redirect = validatedBool(is_raw["screenshot_redirect"], is_cfg.screenshot_redirect);
|
|
2878
2914
|
is_cfg.ocr_enabled = validatedBool(is_raw["ocr_enabled"], is_cfg.ocr_enabled);
|
|
2879
2915
|
is_cfg.ocr_min_confidence = validatedInt(is_raw["ocr_min_confidence"], is_cfg.ocr_min_confidence, ...boundsOf("image_shrink.ocr_min_confidence"));
|
|
2916
|
+
is_cfg.vision_tier = validatedVisionTier(is_raw["vision_tier"], is_cfg.vision_tier);
|
|
2880
2917
|
is_cfg.max_image_pixels = envInt("TOKEN_GOAT_MAX_IMAGE_PIXELS", is_cfg.max_image_pixels, ...boundsOf("image_shrink.max_image_pixels"));
|
|
2881
2918
|
is_cfg.ocr_enabled = envBool("TOKEN_GOAT_OCR_ENABLED", is_cfg.ocr_enabled);
|
|
2919
|
+
is_cfg.vision_tier = validatedVisionTier(process.env["TOKEN_GOAT_VISION_TIER"], is_cfg.vision_tier);
|
|
2882
2920
|
const sc_raw = section(raw, "screenshot");
|
|
2883
2921
|
const sc_cfg = getDefaultConfig("screenshot");
|
|
2884
2922
|
sc_cfg.chrome_path = validatedStr(sc_raw["chrome_path"], sc_cfg.chrome_path);
|
|
@@ -2985,6 +3023,10 @@ function _buildConfig(raw, projectRaw = {}) {
|
|
|
2985
3023
|
wk.blocked_roots = validatedStrList(wk_raw["blocked_roots"], wk.blocked_roots);
|
|
2986
3024
|
wk.max_pool_workers = validatedInt(wk_raw["max_pool_workers"], wk.max_pool_workers, ...boundsOf("worker.max_pool_workers"));
|
|
2987
3025
|
wk.max_pool_workers = envInt("TOKEN_GOAT_WORKER_MAX_POOL", wk.max_pool_workers, ...boundsOf("worker.max_pool_workers"));
|
|
3026
|
+
wk.embed_threads = validatedInt(wk_raw["embed_threads"], wk.embed_threads, ...boundsOf("worker.embed_threads"));
|
|
3027
|
+
wk.embed_threads = envInt("TOKEN_GOAT_EMBED_THREADS", wk.embed_threads, ...boundsOf("worker.embed_threads"));
|
|
3028
|
+
wk.priority = validatedStr(wk_raw["priority"], wk.priority);
|
|
3029
|
+
wk.priority = envStr("TOKEN_GOAT_WORKER_PRIORITY", wk.priority);
|
|
2988
3030
|
const ix_raw = section(raw, "indexing");
|
|
2989
3031
|
const ix = getDefaultConfig("indexing");
|
|
2990
3032
|
ix.large_file_symbol_only_kb = validatedInt(ix_raw["large_file_symbol_only_kb"], ix.large_file_symbol_only_kb, ...boundsOf("indexing.large_file_symbol_only_kb"));
|
|
@@ -3077,6 +3119,7 @@ var CONFIG_KEY_ENV_OVERRIDES = {
|
|
|
3077
3119
|
"skill_preservation.orphan_sweep_enabled": ["TOKEN_GOAT_ORPHAN_SWEEP"],
|
|
3078
3120
|
"image_shrink.max_image_pixels": ["TOKEN_GOAT_MAX_IMAGE_PIXELS"],
|
|
3079
3121
|
"image_shrink.ocr_enabled": ["TOKEN_GOAT_OCR_ENABLED"],
|
|
3122
|
+
"image_shrink.vision_tier": ["TOKEN_GOAT_VISION_TIER"],
|
|
3080
3123
|
"screenshot.block_private_targets": ["TOKEN_GOAT_SCREENSHOT_BLOCK_PRIVATE_TARGETS"],
|
|
3081
3124
|
"repomap.compact_file_threshold": ["TOKEN_GOAT_REPOMAP_COMPACT_THRESHOLD"],
|
|
3082
3125
|
"repomap.exclude_tests": ["TOKEN_GOAT_REPOMAP_EXCLUDE_TESTS"],
|
|
@@ -3114,6 +3157,8 @@ var CONFIG_KEY_ENV_OVERRIDES = {
|
|
|
3114
3157
|
"webfetch.max_bytes": ["TOKEN_GOAT_WEB_CACHE_MAX_BYTES"],
|
|
3115
3158
|
"webfetch.compress_bodies": ["TOKEN_GOAT_WEB_COMPRESS"],
|
|
3116
3159
|
"worker.max_pool_workers": ["TOKEN_GOAT_WORKER_MAX_POOL"],
|
|
3160
|
+
"worker.embed_threads": ["TOKEN_GOAT_EMBED_THREADS"],
|
|
3161
|
+
"worker.priority": ["TOKEN_GOAT_WORKER_PRIORITY"],
|
|
3117
3162
|
"compression.profile": ["TOKEN_GOAT_COMPRESS_PROFILE"],
|
|
3118
3163
|
"context.model_window_tokens": ["TOKEN_GOAT_MODEL_WINDOW_TOKENS"],
|
|
3119
3164
|
"indexing.cross_project_symbols": ["TOKEN_GOAT_CROSS_PROJECT_SYMBOLS"],
|
|
@@ -3199,7 +3244,8 @@ function saveConfig(config) {
|
|
|
3199
3244
|
max_image_pixels: is_cfg.max_image_pixels,
|
|
3200
3245
|
screenshot_redirect: is_cfg.screenshot_redirect,
|
|
3201
3246
|
ocr_enabled: is_cfg.ocr_enabled,
|
|
3202
|
-
ocr_min_confidence: is_cfg.ocr_min_confidence
|
|
3247
|
+
ocr_min_confidence: is_cfg.ocr_min_confidence,
|
|
3248
|
+
vision_tier: is_cfg.vision_tier
|
|
3203
3249
|
},
|
|
3204
3250
|
screenshot: {
|
|
3205
3251
|
chrome_path: config.screenshot.chrome_path,
|
|
@@ -3260,7 +3306,9 @@ function saveConfig(config) {
|
|
|
3260
3306
|
},
|
|
3261
3307
|
worker: {
|
|
3262
3308
|
blocked_roots: config.worker.blocked_roots,
|
|
3263
|
-
max_pool_workers: config.worker.max_pool_workers
|
|
3309
|
+
max_pool_workers: config.worker.max_pool_workers,
|
|
3310
|
+
embed_threads: config.worker.embed_threads,
|
|
3311
|
+
priority: config.worker.priority
|
|
3264
3312
|
},
|
|
3265
3313
|
indexing: {
|
|
3266
3314
|
large_file_symbol_only_kb: config.indexing.large_file_symbol_only_kb,
|
|
@@ -4175,9 +4223,11 @@ function makeSymbolEmitter(symbols, sections, seen, filePath, maxSymbols = 500,
|
|
|
4175
4223
|
}
|
|
4176
4224
|
function assignFlatEndLines(sections, totalLines) {
|
|
4177
4225
|
for (let i = 0; i < sections.length; i++) {
|
|
4178
|
-
const next = sections[i + 1];
|
|
4179
4226
|
const s = sections[i];
|
|
4180
4227
|
if (s === void 0) continue;
|
|
4228
|
+
let j = i + 1;
|
|
4229
|
+
while (j < sections.length && (sections[j]?.line ?? 0) <= s.line) j++;
|
|
4230
|
+
const next = sections[j];
|
|
4181
4231
|
const end = next !== void 0 ? next.line - 1 : totalLines;
|
|
4182
4232
|
s.endLine = end < s.line ? s.line : end;
|
|
4183
4233
|
}
|
|
@@ -4472,6 +4522,7 @@ function findBlockOpenBrace(content, lineIndex, startLine, lastSearchLine, lineC
|
|
|
4472
4522
|
else if (ch === ")" || ch === "]") {
|
|
4473
4523
|
if (parenDepth > 0) parenDepth--;
|
|
4474
4524
|
} else if (ch === ";") return null;
|
|
4525
|
+
else if (ch === "}" && parenDepth === 0) return null;
|
|
4475
4526
|
else if (ch === "{") return i;
|
|
4476
4527
|
}
|
|
4477
4528
|
return null;
|
|
@@ -4567,8 +4618,9 @@ function extractEnv(content, filePath) {
|
|
|
4567
4618
|
if (_lineClosesQuote(line, openQuote)) openQuote = null;
|
|
4568
4619
|
continue;
|
|
4569
4620
|
}
|
|
4570
|
-
|
|
4571
|
-
|
|
4621
|
+
const trimmed = line.replace(/^[ \t]+/, "");
|
|
4622
|
+
if (!trimmed || trimmed[0] === "#" || trimmed[0] === ";") continue;
|
|
4623
|
+
const m = ENV_KEY_RE.exec(trimmed);
|
|
4572
4624
|
if (m === null) continue;
|
|
4573
4625
|
const name = m[1]?.trim() ?? "";
|
|
4574
4626
|
if (!name || name.length > MAX_HEADING_LEN) continue;
|
|
@@ -4576,7 +4628,7 @@ function extractEnv(content, filePath) {
|
|
|
4576
4628
|
if (seen.has(key)) continue;
|
|
4577
4629
|
seen.add(key);
|
|
4578
4630
|
symbols.push(makeLineSymbol(filePath, name, "env_key", i + 1));
|
|
4579
|
-
openQuote = _detectOpenQuote(
|
|
4631
|
+
openQuote = _detectOpenQuote(trimmed.slice(m[0].length));
|
|
4580
4632
|
}
|
|
4581
4633
|
return symbols;
|
|
4582
4634
|
}
|
|
@@ -4955,7 +5007,8 @@ CREATE TABLE IF NOT EXISTS files (
|
|
|
4955
5007
|
language TEXT,
|
|
4956
5008
|
indexed_at REAL,
|
|
4957
5009
|
embed_sha TEXT,
|
|
4958
|
-
retry_count INTEGER NOT NULL DEFAULT 0
|
|
5010
|
+
retry_count INTEGER NOT NULL DEFAULT 0,
|
|
5011
|
+
parser_sha TEXT
|
|
4959
5012
|
);
|
|
4960
5013
|
-- Expression index on TG_LOWER(path) -- see pathEqClause (sql_path.ts) and TG_LOWER's
|
|
4961
5014
|
-- registration above. TG_LOWER is registered { deterministic: true }, which is required for
|
|
@@ -5215,7 +5268,7 @@ CREATE TRIGGER IF NOT EXISTS cache_recall_au AFTER UPDATE ON cache_recall BEGIN
|
|
|
5215
5268
|
VALUES (new.row_id, new.label, new.content);
|
|
5216
5269
|
END;
|
|
5217
5270
|
`;
|
|
5218
|
-
var SCHEMA_VERSION =
|
|
5271
|
+
var SCHEMA_VERSION = 13;
|
|
5219
5272
|
function alterTableIdempotent(conn, sql) {
|
|
5220
5273
|
try {
|
|
5221
5274
|
conn.exec(sql);
|
|
@@ -5259,7 +5312,9 @@ var MIGRATIONS = {
|
|
|
5259
5312
|
// .env would have kept serving its pre-fix chunks indefinitely. Deleting the rows here both
|
|
5260
5313
|
// removes the stored secrets and, by clearing embed_sha, makes the next drain re-embed the file
|
|
5261
5314
|
// through the redacting path.
|
|
5262
|
-
10: purgeDotenvEmbeddings
|
|
5315
|
+
10: purgeDotenvEmbeddings,
|
|
5316
|
+
// v12 -> v13: adds files.parser_sha, the digest of the extraction logic that produced this file's rows, tracked separately from files.sha for the same reason embed_sha is -- content freshness and parse freshness are different questions, and the content sha alone could only ever answer the first. A pre-existing v12 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 / v9 -> v10 above. Deliberately left NULL for every existing row rather than backfilled with the current fingerprint: NULL is the truthful answer (nobody recorded which parser wrote those rows), and it is also the answer that makes the freshness gates reparse them once, which is exactly what a database indexed by an older parser needs.
|
|
5317
|
+
12: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN parser_sha TEXT")
|
|
5263
5318
|
};
|
|
5264
5319
|
function runMigrations(conn, fromVersion, toVersion) {
|
|
5265
5320
|
for (let v = fromVersion; v < toVersion; v++) {
|
|
@@ -5745,7 +5800,51 @@ var _KIND_GROUPS = [
|
|
|
5745
5800
|
"exports",
|
|
5746
5801
|
"imports",
|
|
5747
5802
|
"changed_lookup",
|
|
5748
|
-
"dep_docs"
|
|
5803
|
+
"dep_docs",
|
|
5804
|
+
// Every other SOURCE_READ kind in stats.ts's KIND_TO_SOURCE: the surgical-read commands over documents, structured data and session/PR state. They were registered and produced but grouped nowhere, so `stats --full` printed the whole family under 'Other', away from the read-savings siblings they are measured against. image_meta/image_text sit here rather than under 'Images' because stats.ts files them as SOURCE_READ: they save read bytes, they do not shrink an image.
|
|
5805
|
+
"brief_view",
|
|
5806
|
+
"conflicts",
|
|
5807
|
+
"coverage_report_gaps",
|
|
5808
|
+
"csv_query",
|
|
5809
|
+
"csv_profile",
|
|
5810
|
+
"compact_doc",
|
|
5811
|
+
"docx_outline",
|
|
5812
|
+
"docx_text",
|
|
5813
|
+
"gdrive_sections",
|
|
5814
|
+
"image_meta",
|
|
5815
|
+
"image_text",
|
|
5816
|
+
"json_query",
|
|
5817
|
+
"json_outline",
|
|
5818
|
+
"note_read",
|
|
5819
|
+
"note_list",
|
|
5820
|
+
"openapi_op",
|
|
5821
|
+
"openapi_outline",
|
|
5822
|
+
"pdf_extract",
|
|
5823
|
+
"pdf_locate",
|
|
5824
|
+
"pdf_outline",
|
|
5825
|
+
"pdf_meta",
|
|
5826
|
+
"pptx_outline",
|
|
5827
|
+
"pptx_slide",
|
|
5828
|
+
"pptx_notes",
|
|
5829
|
+
"pptx_text",
|
|
5830
|
+
"pr_slice",
|
|
5831
|
+
"session_outline",
|
|
5832
|
+
"session_slice",
|
|
5833
|
+
"sqlite_query",
|
|
5834
|
+
"sqlite_schema",
|
|
5835
|
+
"transcript",
|
|
5836
|
+
"transcript_outline",
|
|
5837
|
+
"video_chapters",
|
|
5838
|
+
"xlsx_sheets",
|
|
5839
|
+
"xlsx_head",
|
|
5840
|
+
"xlsx_range",
|
|
5841
|
+
"xlsx_query",
|
|
5842
|
+
"xml_query",
|
|
5843
|
+
"xml_outline",
|
|
5844
|
+
"yaml_query",
|
|
5845
|
+
"yaml_outline",
|
|
5846
|
+
"zip_list",
|
|
5847
|
+
"zip_read"
|
|
5749
5848
|
])
|
|
5750
5849
|
},
|
|
5751
5850
|
{ label: "Lookups", members: /* @__PURE__ */ new Set(["symbol_lookup", "semantic_search", "map_lookup"]) },
|
|
@@ -5767,74 +5866,72 @@ var _KIND_GROUPS = [
|
|
|
5767
5866
|
"session_hint_overhead",
|
|
5768
5867
|
"session_hint_suppressed",
|
|
5769
5868
|
"read_count_deny",
|
|
5770
|
-
"read_dedup_hint",
|
|
5771
5869
|
"grep_dedup_hint",
|
|
5772
5870
|
"glob_dedup_hint",
|
|
5773
5871
|
"diff_hint",
|
|
5774
5872
|
"predictive_prefetch_hit",
|
|
5775
|
-
"read_partial_overlap_hint",
|
|
5776
5873
|
"structured_file_hint",
|
|
5777
5874
|
"write_rewrite_hint",
|
|
5778
5875
|
"websearch_dedup_hint",
|
|
5779
5876
|
"large_file_hint_followed",
|
|
5780
|
-
"large_file_hint_ignored"
|
|
5781
|
-
|
|
5782
|
-
},
|
|
5783
|
-
{
|
|
5784
|
-
label: "Bash",
|
|
5785
|
-
members: /* @__PURE__ */ new Set([
|
|
5786
|
-
"bash_dedup_hint",
|
|
5787
|
-
"bash_output_cached",
|
|
5788
|
-
"bash_output_recall",
|
|
5789
|
-
"bash_output_recall_miss",
|
|
5790
|
-
"bash_dedup_stale",
|
|
5791
|
-
"bash_range_read_hint",
|
|
5792
|
-
"bash_streak_hint",
|
|
5793
|
-
"bash_poll_hint",
|
|
5794
|
-
"env_probe_cache_hit",
|
|
5795
|
-
"git_diff_scope_hint",
|
|
5796
|
-
"dep_list_cache_hit",
|
|
5797
|
-
"bash_read_equiv_already_read",
|
|
5798
|
-
"bash_grep_result_cache_hit",
|
|
5799
|
-
"git_diff_context_trimmed"
|
|
5877
|
+
"large_file_hint_ignored",
|
|
5878
|
+
"evidence_cache_hit"
|
|
5800
5879
|
])
|
|
5801
5880
|
},
|
|
5881
|
+
// Empty for the same reason as MCP below: every live Bash kind arrives through _kindGroupLabel's `bash_compress:` prefix branch, not through a literal name. The fifteen literal names this set used to carry (bash_output_cached, bash_dedup_hint, env_probe_cache_hit and the rest) came over with the Python port and were never recorded or registered anywhere in this tree, so they grouped rows that could not exist.
|
|
5882
|
+
{ label: "Bash", members: /* @__PURE__ */ new Set() },
|
|
5802
5883
|
{
|
|
5803
5884
|
label: "Web",
|
|
5804
5885
|
members: /* @__PURE__ */ new Set([
|
|
5805
|
-
"web_dedup_hint",
|
|
5806
|
-
"web_output_cached",
|
|
5807
|
-
"web_output_recall",
|
|
5808
|
-
"web_output_recall_miss",
|
|
5809
|
-
"web_dedup_stale",
|
|
5810
5886
|
"web_fetch",
|
|
5811
5887
|
"injection_detected"
|
|
5812
5888
|
])
|
|
5813
5889
|
},
|
|
5890
|
+
// Membership comes from _kindGroupLabel's `mcp:` prefix branch, not from this set, which is why
|
|
5891
|
+
// it is empty. The entry still has to exist: _renderByKindSection iterates _KIND_GROUPS' labels
|
|
5892
|
+
// (plus 'Other') to decide what to print, so a label _kindGroupLabel returns but that is missing
|
|
5893
|
+
// here does not fall back to 'Other' -- its rows disappear from the table entirely.
|
|
5894
|
+
{ label: "MCP", members: /* @__PURE__ */ new Set() },
|
|
5814
5895
|
{
|
|
5815
5896
|
label: "Compact / Skills",
|
|
5816
5897
|
members: /* @__PURE__ */ new Set([
|
|
5817
|
-
"compact_manifest",
|
|
5818
|
-
"compact_assist",
|
|
5819
|
-
"compact_recovery",
|
|
5820
|
-
"skill_body_recall",
|
|
5821
|
-
"skill_compact_served",
|
|
5822
|
-
"skill_cached",
|
|
5823
5898
|
"skill_load",
|
|
5824
5899
|
"skill_oversized_first_load",
|
|
5825
|
-
"skill_compact_inlined"
|
|
5826
|
-
|
|
5827
|
-
|
|
5900
|
+
"skill_compact_inlined"
|
|
5901
|
+
])
|
|
5902
|
+
},
|
|
5903
|
+
// SOURCE_CONTENT: real rewrites of tool output that remove real bytes (agent report compaction, Grep fold, browser tab dedup, bash/content compression and the handoff pair). The by-source table has shown a 'content' row since the source was added, but the by-kind table had no member set for it, so every one of these kinds printed under 'Other'. The taskoutput: prefix branch in _kindGroupLabel routes here too.
|
|
5904
|
+
{
|
|
5905
|
+
label: "Content",
|
|
5906
|
+
members: /* @__PURE__ */ new Set([
|
|
5907
|
+
"content_compress",
|
|
5908
|
+
"content_retrieve",
|
|
5909
|
+
"agent_report_compact",
|
|
5910
|
+
"agent_report_compact_declined",
|
|
5911
|
+
"browser_tab_dedup",
|
|
5912
|
+
"grep:fold",
|
|
5913
|
+
"handoff_create",
|
|
5914
|
+
"handoff_resolve",
|
|
5915
|
+
"plan_echo_collapse"
|
|
5828
5916
|
])
|
|
5829
5917
|
}
|
|
5830
5918
|
];
|
|
5831
5919
|
function _kindGroupLabel(kind) {
|
|
5832
|
-
if (kind.startsWith("bash_compress:")) {
|
|
5920
|
+
if (kind.startsWith("bash_compress:") || kind.startsWith("bashoutput:")) {
|
|
5833
5921
|
return "Bash";
|
|
5834
5922
|
}
|
|
5835
|
-
if (kind.startsWith("webfetch:")) {
|
|
5923
|
+
if (kind.startsWith("webfetch:") || kind.startsWith("gdrive:")) {
|
|
5836
5924
|
return "Web";
|
|
5837
5925
|
}
|
|
5926
|
+
if (kind.startsWith("mcp:")) {
|
|
5927
|
+
return "MCP";
|
|
5928
|
+
}
|
|
5929
|
+
if (kind.startsWith("skill_body:") || kind.startsWith("skill_compact:")) {
|
|
5930
|
+
return "Compact / Skills";
|
|
5931
|
+
}
|
|
5932
|
+
if (kind.startsWith("taskoutput:")) {
|
|
5933
|
+
return "Content";
|
|
5934
|
+
}
|
|
5838
5935
|
for (const group of _KIND_GROUPS) {
|
|
5839
5936
|
if (group.members.has(kind)) {
|
|
5840
5937
|
return group.label;
|
|
@@ -6138,6 +6235,10 @@ var SOURCE_SKILL = "skill";
|
|
|
6138
6235
|
var SOURCE_CONTENT = "content";
|
|
6139
6236
|
var SOURCE_OTHER = "other";
|
|
6140
6237
|
var _BYTES_MODE_ONLY_KINDS = /* @__PURE__ */ new Set(["webfetch_image", "gdrive_image"]);
|
|
6238
|
+
var COUNT_ONLY_KINDS = /* @__PURE__ */ new Set(["secret_redacted"]);
|
|
6239
|
+
function savedTokensFromBytes(bytes) {
|
|
6240
|
+
return Math.round(Math.max(0, bytes) / 4);
|
|
6241
|
+
}
|
|
6141
6242
|
var KIND_TO_SOURCE = {
|
|
6142
6243
|
image_shrink: SOURCE_IMAGE,
|
|
6143
6244
|
image_shrink_cache_hit: SOURCE_IMAGE,
|
|
@@ -6238,6 +6339,10 @@ var KIND_TO_SOURCE = {
|
|
|
6238
6339
|
// Decline counterpart to agent_report_compact: the fence-collapse net-benefit gate ran and found at least one over-long fence, but declined to rewrite because net savings did not clear the notice cost. Always recorded at (0, 0) -- see the recordStat call site -- so it never contributes to any savings total; it exists purely to make gate hit-rate and near-misses visible instead of the decline being invisible.
|
|
6239
6340
|
agent_report_compact_declined: SOURCE_CONTENT,
|
|
6240
6341
|
content_compress: SOURCE_CONTENT,
|
|
6342
|
+
// Verbatim-repeat collapse of a browser tool's "Tab Context:" text block (hooks_browser_image.ts postBrowserImageHandler). SOURCE_CONTENT for the same reason as agent_report_compact above: it is a real rewrite with real bytes removed, not an advisory nudge. Deliberately not SOURCE_IMAGE -- it shares a handler with image_shrink but collapses text, and folding text bytes into the image ledger is the two-units-under-one-label mistake this file's image_shrink entry was just fixed for.
|
|
6343
|
+
browser_tab_dedup: SOURCE_CONTENT,
|
|
6344
|
+
// Collapse of the plan echo in an approved ExitPlanMode result (hooks_exitplanmode.ts). SOURCE_CONTENT for the same reason as agent_report_compact: real bytes removed from a tool result, not an advisory nudge. The handler shipped for releases emitting this rewrite and recording nothing at all, so the mechanism was invisible in `stats` and its net benefit could not be checked against the gate that admits it.
|
|
6345
|
+
plan_echo_collapse: SOURCE_CONTENT,
|
|
6241
6346
|
// Lossless re-layout of Grep content-mode output (hooks_grep.ts foldGrepContentHandler). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as agent_report_compact above: its sibling grep_dedup_hint is advisory and saves nothing directly, whereas this is a real rewrite with real bytes removed. Filing it under the advisory bucket would silently add non-hint savings to hint_stats.ts's savedBytes (which reads by_source[SOURCE_HINT] wholesale) and overstate the hint ledger's net benefit.
|
|
6242
6347
|
"grep:fold": SOURCE_CONTENT,
|
|
6243
6348
|
content_retrieve: SOURCE_CONTENT,
|
|
@@ -6366,7 +6471,8 @@ CREATE TABLE IF NOT EXISTS stats (
|
|
|
6366
6471
|
tokens_saved INTEGER NOT NULL DEFAULT 0,
|
|
6367
6472
|
bytes_saved INTEGER NOT NULL DEFAULT 0,
|
|
6368
6473
|
detail TEXT,
|
|
6369
|
-
harness TEXT
|
|
6474
|
+
harness TEXT,
|
|
6475
|
+
traceparent TEXT
|
|
6370
6476
|
);
|
|
6371
6477
|
CREATE INDEX IF NOT EXISTS idx_stats_ts ON stats(ts);
|
|
6372
6478
|
CREATE INDEX IF NOT EXISTS idx_stats_kind ON stats(kind);
|
|
@@ -6389,6 +6495,11 @@ function migrateGlobalSchema(db) {
|
|
|
6389
6495
|
} catch (err) {
|
|
6390
6496
|
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
6391
6497
|
}
|
|
6498
|
+
try {
|
|
6499
|
+
db.exec("ALTER TABLE stats ADD COLUMN traceparent TEXT");
|
|
6500
|
+
} catch (err) {
|
|
6501
|
+
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
6502
|
+
}
|
|
6392
6503
|
}
|
|
6393
6504
|
var _harnessColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
6394
6505
|
function statsHasHarnessColumn(db) {
|
|
@@ -6405,6 +6516,21 @@ function statsHasHarnessColumn(db) {
|
|
|
6405
6516
|
_harnessColumnByDb.set(db, present);
|
|
6406
6517
|
return present;
|
|
6407
6518
|
}
|
|
6519
|
+
var _traceparentColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
6520
|
+
function statsHasTraceparentColumn(db) {
|
|
6521
|
+
const cached = _traceparentColumnByDb.get(db);
|
|
6522
|
+
if (cached !== void 0) return cached;
|
|
6523
|
+
let present;
|
|
6524
|
+
try {
|
|
6525
|
+
present = db.prepare("PRAGMA table_info(stats)").all().some(
|
|
6526
|
+
(c) => c.name === "traceparent"
|
|
6527
|
+
);
|
|
6528
|
+
} catch {
|
|
6529
|
+
present = false;
|
|
6530
|
+
}
|
|
6531
|
+
_traceparentColumnByDb.set(db, present);
|
|
6532
|
+
return present;
|
|
6533
|
+
}
|
|
6408
6534
|
function getGlobalDb(homeDir) {
|
|
6409
6535
|
const basePath = homeDir ? dataDirForHome(homeDir) : dataDir();
|
|
6410
6536
|
const dbPath = path8.join(basePath, "global.db");
|
|
@@ -6423,11 +6549,18 @@ function noStatsMessage(windowDays, homeDir) {
|
|
|
6423
6549
|
if (total === 0) return "No stats recorded yet.";
|
|
6424
6550
|
return `No stats in the last ${countNoun(windowDays, "day")} (${total} recorded outside this window; use --window-days 0 for all time).`;
|
|
6425
6551
|
}
|
|
6426
|
-
function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail) {
|
|
6552
|
+
function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail, traceparent) {
|
|
6427
6553
|
try {
|
|
6428
6554
|
const db = _testDb ?? getGlobalDb();
|
|
6429
6555
|
const ts = Math.floor(Date.now() / 1e3);
|
|
6430
|
-
|
|
6556
|
+
const tp = traceparent ?? process.env["TRACEPARENT"] ?? process.env["traceparent"] ?? null;
|
|
6557
|
+
const hasHarness = statsHasHarnessColumn(db);
|
|
6558
|
+
const hasTraceparent = statsHasTraceparentColumn(db);
|
|
6559
|
+
if (hasHarness && hasTraceparent) {
|
|
6560
|
+
db.prepare(
|
|
6561
|
+
"INSERT INTO stats (ts, kind, bytes_saved, tokens_saved, detail, harness, traceparent) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
6562
|
+
).run(ts, kind, bytesSaved, tokensSaved, detail ?? null, getHarnessName(), tp);
|
|
6563
|
+
} else if (hasHarness) {
|
|
6431
6564
|
db.prepare(
|
|
6432
6565
|
"INSERT INTO stats (ts, kind, bytes_saved, tokens_saved, detail, harness) VALUES (?, ?, ?, ?, ?, ?)"
|
|
6433
6566
|
).run(ts, kind, bytesSaved, tokensSaved, detail ?? null, getHarnessName());
|
|
@@ -6482,10 +6615,14 @@ function summarize(windowDays = 30, testDb, homeDir) {
|
|
|
6482
6615
|
const stmt = db.prepare(query);
|
|
6483
6616
|
const rows = sinceTs !== null ? stmt.all(sinceTs) : stmt.all();
|
|
6484
6617
|
const tsToDateCache = {};
|
|
6618
|
+
const counts = {};
|
|
6485
6619
|
for (const row of rows) {
|
|
6486
6620
|
const bytesSaved = row.bytes_saved ?? 0;
|
|
6487
|
-
const
|
|
6621
|
+
const recorded = row.tokens_saved ?? 0;
|
|
6488
6622
|
const kind = row.kind;
|
|
6623
|
+
const isCount = COUNT_ONLY_KINDS.has(kind);
|
|
6624
|
+
if (isCount) counts[kind] = (counts[kind] ?? 0) + recorded;
|
|
6625
|
+
const tokensSaved = isCount ? 0 : recorded;
|
|
6489
6626
|
const tsRaw = row.ts;
|
|
6490
6627
|
if (tsRaw === void 0) continue;
|
|
6491
6628
|
const ts = tsRaw;
|
|
@@ -6544,6 +6681,7 @@ function summarize(windowDays = 30, testDb, homeDir) {
|
|
|
6544
6681
|
by_project: byProjectList,
|
|
6545
6682
|
by_source: bySourceDict,
|
|
6546
6683
|
by_harness: byHarness,
|
|
6684
|
+
counts,
|
|
6547
6685
|
by_command: Object.entries(byCommandDict).map(([command, bucket]) => ({ ...bucket, command })).filter((r) => r.events > 0),
|
|
6548
6686
|
window_days: windowDays
|
|
6549
6687
|
};
|
|
@@ -6554,6 +6692,10 @@ function _totalsLines(summary) {
|
|
|
6554
6692
|
`Total events: ${summary.total_events}`,
|
|
6555
6693
|
`Bytes saved: ${fmtBytes(summary.total_bytes_saved)}`,
|
|
6556
6694
|
`Tokens saved: ${summary.total_tokens_saved}`,
|
|
6695
|
+
// Printed on its own line, below the token total and never inside it, because it counts
|
|
6696
|
+
// placeholders rather than tokens. Omitted entirely when nothing was redacted, so the line is
|
|
6697
|
+
// information rather than a permanent zero. See COUNT_ONLY_KINDS.
|
|
6698
|
+
...summary.counts["secret_redacted"] ? [`Secrets hidden: ${summary.counts["secret_redacted"]} (a count, not tokens)`] : [],
|
|
6557
6699
|
`Window: ${summary.window_days} days`
|
|
6558
6700
|
];
|
|
6559
6701
|
}
|
|
@@ -6872,6 +7014,7 @@ export {
|
|
|
6872
7014
|
toKB,
|
|
6873
7015
|
compileGrepMatcher,
|
|
6874
7016
|
grepFilteredToEmptyNotice,
|
|
7017
|
+
filtersFilteredToEmptyNotice,
|
|
6875
7018
|
countNoun,
|
|
6876
7019
|
excludeTestsHiddenNote,
|
|
6877
7020
|
countContentLines,
|
|
@@ -6959,6 +7102,7 @@ export {
|
|
|
6959
7102
|
fg,
|
|
6960
7103
|
C,
|
|
6961
7104
|
SOURCE_HINT,
|
|
7105
|
+
savedTokensFromBytes,
|
|
6962
7106
|
formatLocalTimestamp,
|
|
6963
7107
|
recordStat,
|
|
6964
7108
|
recordUnmappedTool,
|