token-goat 2.8.2 → 2.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -18
- package/dist/{token-goat-chunk-AM23GDIS.mjs → token-goat-chunk-4OTIB7SB.mjs} +579 -162
- package/dist/{token-goat-chunk-TF5NT3H5.mjs → token-goat-chunk-CZALRRGN.mjs} +192 -60
- package/dist/{token-goat-chunk-IVCTQPZD.mjs → token-goat-chunk-E76UNTVK.mjs} +407 -215
- package/dist/{token-goat-chunk-SE6V7LUZ.mjs → token-goat-chunk-HNODKNWO.mjs} +5 -5
- package/dist/{token-goat-chunk-5MLXFSRI.mjs → token-goat-chunk-IZRXU64B.mjs} +51 -4
- package/dist/{token-goat-chunk-JU7QESB3.mjs → token-goat-chunk-L6YP6FKQ.mjs} +5 -5
- package/dist/{token-goat-chunk-AL644JUV.mjs → token-goat-chunk-LZOAPGWR.mjs} +6 -5
- package/dist/{token-goat-chunk-BTZSUIYA.mjs → token-goat-chunk-NZFTWBGV.mjs} +2 -2
- package/dist/{token-goat-chunk-LILS6TIU.mjs → token-goat-chunk-RE7S7H26.mjs} +182 -65
- package/dist/{token-goat-chunk-2MWF3OGR.mjs → token-goat-chunk-U4UI5F3S.mjs} +1072 -218
- package/dist/{token-goat-chunk-KBVN4ELV.mjs → token-goat-chunk-ZQ3PUOP3.mjs} +379 -147
- package/dist/token-goat-hook.mjs +5 -5
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +3 -1
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
import { createRequire } from "node:module";
|
|
9
9
|
function resolveVersion() {
|
|
10
10
|
if (true) {
|
|
11
|
-
return "2.8.
|
|
11
|
+
return "2.8.4";
|
|
12
12
|
}
|
|
13
13
|
const require2 = createRequire(import.meta.url);
|
|
14
14
|
const pkg = require2("../package.json");
|
|
@@ -263,6 +263,16 @@ function lowercaseDriveLetter(s) {
|
|
|
263
263
|
}
|
|
264
264
|
var EXTENDED_UNC_PREFIX_RE = /^\\\\\?\\UNC\\/i;
|
|
265
265
|
var EXTENDED_PREFIX_RE = /^\\\\\?\\/;
|
|
266
|
+
function shellMountToWindowsPath(p) {
|
|
267
|
+
const s = p.includes("\\") ? p.replace(/\\/g, "/") : p;
|
|
268
|
+
const m = WSL_PATH_RE.exec(s);
|
|
269
|
+
if (m) return `${m[1].toLowerCase()}:/${m[2].replace(/^\/+/, "")}`;
|
|
270
|
+
if (process.platform === "win32") {
|
|
271
|
+
const g = MSYS_PATH_RE.exec(s);
|
|
272
|
+
if (g) return `${g[1].toLowerCase()}:${g[2] ?? "/"}`;
|
|
273
|
+
}
|
|
274
|
+
return s;
|
|
275
|
+
}
|
|
266
276
|
function normalizePath(p) {
|
|
267
277
|
let s = p;
|
|
268
278
|
if (EXTENDED_UNC_PREFIX_RE.test(s)) {
|
|
@@ -270,20 +280,7 @@ function normalizePath(p) {
|
|
|
270
280
|
} else if (EXTENDED_PREFIX_RE.test(s)) {
|
|
271
281
|
s = s.slice(4);
|
|
272
282
|
}
|
|
273
|
-
|
|
274
|
-
s = s.replace(/\\/g, "/");
|
|
275
|
-
}
|
|
276
|
-
const m = WSL_PATH_RE.exec(s);
|
|
277
|
-
if (m) {
|
|
278
|
-
const driveLetter = m[1].toLowerCase();
|
|
279
|
-
const rest = m[2];
|
|
280
|
-
const restStripped = rest.replace(/^\/+/, "");
|
|
281
|
-
s = `${driveLetter}:/${restStripped}`;
|
|
282
|
-
}
|
|
283
|
-
if (process.platform === "win32") {
|
|
284
|
-
const g = MSYS_PATH_RE.exec(s);
|
|
285
|
-
if (g) s = `${g[1].toLowerCase()}:${g[2] ?? "/"}`;
|
|
286
|
-
}
|
|
283
|
+
s = shellMountToWindowsPath(s);
|
|
287
284
|
s = expandShortPath(s);
|
|
288
285
|
s = lowercaseDriveLetter(s);
|
|
289
286
|
s = normalizeDarwinSystemAlias(s);
|
|
@@ -297,8 +294,10 @@ function normalizeDarwinSystemAlias(p) {
|
|
|
297
294
|
}
|
|
298
295
|
function resolveIndexPath(file, base = process.cwd()) {
|
|
299
296
|
const isWindowsAbsolute = (s) => /^[a-zA-Z]:[/\\]/.test(s);
|
|
300
|
-
const
|
|
301
|
-
|
|
297
|
+
const f = shellMountToWindowsPath(file);
|
|
298
|
+
const b = shellMountToWindowsPath(base);
|
|
299
|
+
const resolve5 = isWindowsAbsolute(f) || isWindowsAbsolute(b) ? path2.win32.resolve : path2.resolve;
|
|
300
|
+
return normalizePath(resolve5(b, f));
|
|
302
301
|
}
|
|
303
302
|
function toDisplayPath(root, target) {
|
|
304
303
|
if (root === void 0) return target;
|
|
@@ -759,6 +758,14 @@ function grepFilteredToEmptyNotice(preFilterCount, grep, nounSingular, nounPlura
|
|
|
759
758
|
const pronoun = preFilterCount === 1 ? "it" : "them";
|
|
760
759
|
return ` (all ${preFilterCount} ${noun} ${verb} filtered out by --grep ${grep} -- widen or drop the filter to see ${pronoun})`;
|
|
761
760
|
}
|
|
761
|
+
function filtersFilteredToEmptyNotice(preFilterCount, activeFilters, nounSingular, nounPlural, reassurance) {
|
|
762
|
+
const noun = preFilterCount === 1 ? nounSingular : nounPlural;
|
|
763
|
+
const cause = activeFilters.length === 0 ? "the active filter" : activeFilters.join(" + ");
|
|
764
|
+
const knob = activeFilters.length > 1 ? "filters" : "filter";
|
|
765
|
+
const verb = preFilterCount === 1 ? "was" : "were";
|
|
766
|
+
const tail = reassurance === void 0 ? "" : `; ${reassurance}`;
|
|
767
|
+
return ` (all ${preFilterCount} ${noun} ${verb} filtered out by ${cause}${tail} -- widen or drop the ${knob} to see them)`;
|
|
768
|
+
}
|
|
762
769
|
function countNoun(count, singular, plural = `${singular}s`) {
|
|
763
770
|
return `${count} ${count === 1 ? singular : plural}`;
|
|
764
771
|
}
|
|
@@ -2161,7 +2168,15 @@ var CONFIG_DEFAULTS = {
|
|
|
2161
2168
|
// 85+, while a photo with an incidental sign or logo in frame scores much lower and
|
|
2162
2169
|
// noisier -- padding the threshold below the terminal/code norm still comfortably
|
|
2163
2170
|
// excludes photographic false positives without needing a second heuristic.
|
|
2164
|
-
ocr_min_confidence: 65
|
|
2171
|
+
ocr_min_confidence: 65,
|
|
2172
|
+
// Which resolution tier the model being shown the image is on, which decides what its pixels
|
|
2173
|
+
// cost. 'standard' (1568px long edge, 1568 visual tokens) is every model before Claude 4.7;
|
|
2174
|
+
// 'high' (2576px, 4784 tokens) is 4.7 and later, and bills the same large image up to roughly
|
|
2175
|
+
// three times higher. Only the saving *reported* by `token-goat stats` depends on this -- no
|
|
2176
|
+
// image is encoded differently -- and 'standard' is the default because it is the floor: it
|
|
2177
|
+
// caps the counterfactual at the smaller of the two bills and so can never credit a saving
|
|
2178
|
+
// that was not there. Set it to 'high' on a Claude 4.7+ model to see the larger real figure.
|
|
2179
|
+
vision_tier: "standard"
|
|
2165
2180
|
},
|
|
2166
2181
|
screenshot: {
|
|
2167
2182
|
chrome_path: "",
|
|
@@ -2321,6 +2336,9 @@ function validatedFloat(raw, def, min, max) {
|
|
|
2321
2336
|
if (!Number.isFinite(n)) return def;
|
|
2322
2337
|
return Math.max(min, Math.min(max, n));
|
|
2323
2338
|
}
|
|
2339
|
+
function validatedVisionTier(raw, def) {
|
|
2340
|
+
return raw === "standard" || raw === "high" ? raw : def;
|
|
2341
|
+
}
|
|
2324
2342
|
function validatedStr(raw, def) {
|
|
2325
2343
|
return typeof raw === "string" ? raw : def;
|
|
2326
2344
|
}
|
|
@@ -2684,6 +2702,8 @@ function mergeRawConfig(base, override) {
|
|
|
2684
2702
|
const baseSection = baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal) ? baseVal : {};
|
|
2685
2703
|
merged[key] = { ...baseSection, ...overrideVal };
|
|
2686
2704
|
} else {
|
|
2705
|
+
const baseVal = base[key];
|
|
2706
|
+
if (baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal)) continue;
|
|
2687
2707
|
merged[key] = overrideVal;
|
|
2688
2708
|
}
|
|
2689
2709
|
}
|
|
@@ -2877,8 +2897,10 @@ function _buildConfig(raw, projectRaw = {}) {
|
|
|
2877
2897
|
is_cfg.screenshot_redirect = validatedBool(is_raw["screenshot_redirect"], is_cfg.screenshot_redirect);
|
|
2878
2898
|
is_cfg.ocr_enabled = validatedBool(is_raw["ocr_enabled"], is_cfg.ocr_enabled);
|
|
2879
2899
|
is_cfg.ocr_min_confidence = validatedInt(is_raw["ocr_min_confidence"], is_cfg.ocr_min_confidence, ...boundsOf("image_shrink.ocr_min_confidence"));
|
|
2900
|
+
is_cfg.vision_tier = validatedVisionTier(is_raw["vision_tier"], is_cfg.vision_tier);
|
|
2880
2901
|
is_cfg.max_image_pixels = envInt("TOKEN_GOAT_MAX_IMAGE_PIXELS", is_cfg.max_image_pixels, ...boundsOf("image_shrink.max_image_pixels"));
|
|
2881
2902
|
is_cfg.ocr_enabled = envBool("TOKEN_GOAT_OCR_ENABLED", is_cfg.ocr_enabled);
|
|
2903
|
+
is_cfg.vision_tier = validatedVisionTier(process.env["TOKEN_GOAT_VISION_TIER"], is_cfg.vision_tier);
|
|
2882
2904
|
const sc_raw = section(raw, "screenshot");
|
|
2883
2905
|
const sc_cfg = getDefaultConfig("screenshot");
|
|
2884
2906
|
sc_cfg.chrome_path = validatedStr(sc_raw["chrome_path"], sc_cfg.chrome_path);
|
|
@@ -3077,6 +3099,7 @@ var CONFIG_KEY_ENV_OVERRIDES = {
|
|
|
3077
3099
|
"skill_preservation.orphan_sweep_enabled": ["TOKEN_GOAT_ORPHAN_SWEEP"],
|
|
3078
3100
|
"image_shrink.max_image_pixels": ["TOKEN_GOAT_MAX_IMAGE_PIXELS"],
|
|
3079
3101
|
"image_shrink.ocr_enabled": ["TOKEN_GOAT_OCR_ENABLED"],
|
|
3102
|
+
"image_shrink.vision_tier": ["TOKEN_GOAT_VISION_TIER"],
|
|
3080
3103
|
"screenshot.block_private_targets": ["TOKEN_GOAT_SCREENSHOT_BLOCK_PRIVATE_TARGETS"],
|
|
3081
3104
|
"repomap.compact_file_threshold": ["TOKEN_GOAT_REPOMAP_COMPACT_THRESHOLD"],
|
|
3082
3105
|
"repomap.exclude_tests": ["TOKEN_GOAT_REPOMAP_EXCLUDE_TESTS"],
|
|
@@ -3199,7 +3222,8 @@ function saveConfig(config) {
|
|
|
3199
3222
|
max_image_pixels: is_cfg.max_image_pixels,
|
|
3200
3223
|
screenshot_redirect: is_cfg.screenshot_redirect,
|
|
3201
3224
|
ocr_enabled: is_cfg.ocr_enabled,
|
|
3202
|
-
ocr_min_confidence: is_cfg.ocr_min_confidence
|
|
3225
|
+
ocr_min_confidence: is_cfg.ocr_min_confidence,
|
|
3226
|
+
vision_tier: is_cfg.vision_tier
|
|
3203
3227
|
},
|
|
3204
3228
|
screenshot: {
|
|
3205
3229
|
chrome_path: config.screenshot.chrome_path,
|
|
@@ -3787,7 +3811,12 @@ function lineCommentStartIndex(line, markers, from = 0) {
|
|
|
3787
3811
|
}
|
|
3788
3812
|
return cutIdx;
|
|
3789
3813
|
}
|
|
3790
|
-
function stripStringLiterals(line) {
|
|
3814
|
+
function stripStringLiterals(line, opts = {}) {
|
|
3815
|
+
const openDelim = (idx) => {
|
|
3816
|
+
const q = line[idx];
|
|
3817
|
+
if (opts.tripleQuotes && line[idx + 1] === q && line[idx + 2] === q) return q + q + q;
|
|
3818
|
+
return q;
|
|
3819
|
+
};
|
|
3791
3820
|
let out = "";
|
|
3792
3821
|
let i = 0;
|
|
3793
3822
|
const stack = [];
|
|
@@ -3803,9 +3832,10 @@ function stripStringLiterals(line) {
|
|
|
3803
3832
|
if (top === void 0) {
|
|
3804
3833
|
if (ch === '"' || ch === "'") {
|
|
3805
3834
|
const bareBraceHole = ch === '"' && i > 0 && line[i - 1] === "$";
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3835
|
+
const delim = openDelim(i);
|
|
3836
|
+
stack.push({ kind: "string", quote: delim, bareBraceHole });
|
|
3837
|
+
out += delim;
|
|
3838
|
+
i += delim.length;
|
|
3809
3839
|
continue;
|
|
3810
3840
|
}
|
|
3811
3841
|
out += ch;
|
|
@@ -3815,9 +3845,10 @@ function stripStringLiterals(line) {
|
|
|
3815
3845
|
if (top.kind === "hole") {
|
|
3816
3846
|
if (ch === '"' || ch === "'") {
|
|
3817
3847
|
const bareBraceHole = ch === '"' && i > 0 && line[i - 1] === "$";
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3848
|
+
const delim = openDelim(i);
|
|
3849
|
+
stack.push({ kind: "string", quote: delim, bareBraceHole });
|
|
3850
|
+
out += delim;
|
|
3851
|
+
i += delim.length;
|
|
3821
3852
|
continue;
|
|
3822
3853
|
}
|
|
3823
3854
|
if (ch === "{") {
|
|
@@ -3845,10 +3876,10 @@ function stripStringLiterals(line) {
|
|
|
3845
3876
|
i += 2;
|
|
3846
3877
|
continue;
|
|
3847
3878
|
}
|
|
3848
|
-
if (ch === top.quote) {
|
|
3879
|
+
if (ch === top.quote[0] && (top.quote.length === 1 || line.startsWith(top.quote, i))) {
|
|
3849
3880
|
stack.pop();
|
|
3850
|
-
out +=
|
|
3851
|
-
i
|
|
3881
|
+
out += top.quote;
|
|
3882
|
+
i += top.quote.length;
|
|
3852
3883
|
continue;
|
|
3853
3884
|
}
|
|
3854
3885
|
if (top.quote === '"') {
|
|
@@ -4168,9 +4199,11 @@ function makeSymbolEmitter(symbols, sections, seen, filePath, maxSymbols = 500,
|
|
|
4168
4199
|
}
|
|
4169
4200
|
function assignFlatEndLines(sections, totalLines) {
|
|
4170
4201
|
for (let i = 0; i < sections.length; i++) {
|
|
4171
|
-
const next = sections[i + 1];
|
|
4172
4202
|
const s = sections[i];
|
|
4173
4203
|
if (s === void 0) continue;
|
|
4204
|
+
let j = i + 1;
|
|
4205
|
+
while (j < sections.length && (sections[j]?.line ?? 0) <= s.line) j++;
|
|
4206
|
+
const next = sections[j];
|
|
4174
4207
|
const end = next !== void 0 ? next.line - 1 : totalLines;
|
|
4175
4208
|
s.endLine = end < s.line ? s.line : end;
|
|
4176
4209
|
}
|
|
@@ -4192,14 +4225,95 @@ function propagateEndLinesToSymbols(symbols, sections) {
|
|
|
4192
4225
|
return sym;
|
|
4193
4226
|
});
|
|
4194
4227
|
}
|
|
4228
|
+
function tripleQuoteDelimiters(opts) {
|
|
4229
|
+
const delims = [];
|
|
4230
|
+
if (opts?.tripleQuote === true) delims.push('"""');
|
|
4231
|
+
if (opts?.tripleSingleQuote === true) delims.push("'''");
|
|
4232
|
+
return delims;
|
|
4233
|
+
}
|
|
4234
|
+
function quoteRunLength(content, i) {
|
|
4235
|
+
let n = 0;
|
|
4236
|
+
while (content[i + n] === '"') n++;
|
|
4237
|
+
return n;
|
|
4238
|
+
}
|
|
4239
|
+
function skipRawStringQuotes(content, from, min) {
|
|
4240
|
+
for (let i = from; i < content.length; i++) {
|
|
4241
|
+
if (content[i] !== '"') continue;
|
|
4242
|
+
const run = quoteRunLength(content, i);
|
|
4243
|
+
if (run >= min) return i + run;
|
|
4244
|
+
i += run - 1;
|
|
4245
|
+
}
|
|
4246
|
+
return -1;
|
|
4247
|
+
}
|
|
4248
|
+
function stepPowershellString(content, i, quote) {
|
|
4249
|
+
const ch = content[i];
|
|
4250
|
+
if (quote === '"' && ch === "`") return { next: i + 1, open: true };
|
|
4251
|
+
return { next: i, open: ch !== quote };
|
|
4252
|
+
}
|
|
4253
|
+
function opensCsharpVerbatimString(content, i) {
|
|
4254
|
+
const prev = content[i - 1];
|
|
4255
|
+
return prev === "@" || prev === "$" && content[i - 2] === "@";
|
|
4256
|
+
}
|
|
4257
|
+
function toLineCommentPrefixes(prefix) {
|
|
4258
|
+
if (prefix === void 0) return [];
|
|
4259
|
+
return typeof prefix === "string" ? [prefix] : prefix;
|
|
4260
|
+
}
|
|
4261
|
+
function atLineComment(content, i, prefixes, exceptions) {
|
|
4262
|
+
if (exceptions.some((e) => content.startsWith(e, i))) return false;
|
|
4263
|
+
return prefixes.some((prefix) => content.startsWith(prefix, i));
|
|
4264
|
+
}
|
|
4265
|
+
function skipBlockComment(content, start, block, nested) {
|
|
4266
|
+
let i = start + block[0].length;
|
|
4267
|
+
let depth = 1;
|
|
4268
|
+
while (i < content.length) {
|
|
4269
|
+
if (nested && content.startsWith(block[0], i)) {
|
|
4270
|
+
depth++;
|
|
4271
|
+
i += block[0].length;
|
|
4272
|
+
continue;
|
|
4273
|
+
}
|
|
4274
|
+
if (content.startsWith(block[1], i)) {
|
|
4275
|
+
depth--;
|
|
4276
|
+
i += block[1].length;
|
|
4277
|
+
if (depth === 0) return i;
|
|
4278
|
+
continue;
|
|
4279
|
+
}
|
|
4280
|
+
i++;
|
|
4281
|
+
}
|
|
4282
|
+
return -1;
|
|
4283
|
+
}
|
|
4195
4284
|
function findMatchingBraceEndLine(content, openBraceIndex, totalLines, lineIndex, lineCommentPrefix, opts) {
|
|
4285
|
+
const linePrefixes = toLineCommentPrefixes(lineCommentPrefix);
|
|
4286
|
+
const lineExceptions = opts?.lineCommentExceptions ?? [];
|
|
4196
4287
|
const block = opts?.blockComment;
|
|
4197
4288
|
const backtick = opts?.backtickQuote === true;
|
|
4289
|
+
const escapes = opts?.stringEscapes ?? "backslash";
|
|
4290
|
+
const nestedBlock = opts?.nestedBlockComments === true;
|
|
4291
|
+
const rawString = opts?.rawStringQuotes === true;
|
|
4292
|
+
const tripleDelims = tripleQuoteDelimiters(opts);
|
|
4293
|
+
const lineString = opts?.lineStringPrefix;
|
|
4198
4294
|
let depth = 0;
|
|
4199
4295
|
let quote = null;
|
|
4296
|
+
let verbatim = false;
|
|
4200
4297
|
for (let i = openBraceIndex; i < content.length; i++) {
|
|
4201
4298
|
const ch = content[i];
|
|
4202
4299
|
if (quote !== null) {
|
|
4300
|
+
if (verbatim) {
|
|
4301
|
+
if (ch === quote) {
|
|
4302
|
+
if (content[i + 1] === quote) {
|
|
4303
|
+
i++;
|
|
4304
|
+
continue;
|
|
4305
|
+
}
|
|
4306
|
+
quote = null;
|
|
4307
|
+
verbatim = false;
|
|
4308
|
+
}
|
|
4309
|
+
continue;
|
|
4310
|
+
}
|
|
4311
|
+
if (escapes === "powershell") {
|
|
4312
|
+
const step = stepPowershellString(content, i, quote);
|
|
4313
|
+
i = step.next;
|
|
4314
|
+
if (!step.open) quote = null;
|
|
4315
|
+
continue;
|
|
4316
|
+
}
|
|
4203
4317
|
if (ch === "\\") {
|
|
4204
4318
|
i++;
|
|
4205
4319
|
continue;
|
|
@@ -4208,16 +4322,35 @@ function findMatchingBraceEndLine(content, openBraceIndex, totalLines, lineIndex
|
|
|
4208
4322
|
continue;
|
|
4209
4323
|
}
|
|
4210
4324
|
if (block !== void 0 && content.startsWith(block[0], i)) {
|
|
4211
|
-
const
|
|
4212
|
-
i =
|
|
4325
|
+
const end = skipBlockComment(content, i, block, nestedBlock);
|
|
4326
|
+
i = end === -1 ? content.length : end - 1;
|
|
4327
|
+
continue;
|
|
4328
|
+
}
|
|
4329
|
+
if (atLineComment(content, i, linePrefixes, lineExceptions)) {
|
|
4330
|
+
while (i < content.length && content[i] !== "\n") i++;
|
|
4213
4331
|
continue;
|
|
4214
4332
|
}
|
|
4215
|
-
if (
|
|
4333
|
+
if (lineString !== void 0 && content.startsWith(lineString, i)) {
|
|
4216
4334
|
while (i < content.length && content[i] !== "\n") i++;
|
|
4217
4335
|
continue;
|
|
4218
4336
|
}
|
|
4337
|
+
if (rawString && ch === '"') {
|
|
4338
|
+
const run = quoteRunLength(content, i);
|
|
4339
|
+
if (run >= 3) {
|
|
4340
|
+
const end = skipRawStringQuotes(content, i + run, run);
|
|
4341
|
+
i = end === -1 ? content.length : end - 1;
|
|
4342
|
+
continue;
|
|
4343
|
+
}
|
|
4344
|
+
}
|
|
4345
|
+
const tripleAt = tripleDelims.find((t) => content.startsWith(t, i));
|
|
4346
|
+
if (tripleAt !== void 0) {
|
|
4347
|
+
const end = content.indexOf(tripleAt, i + 3);
|
|
4348
|
+
i = end === -1 ? content.length : end + 2;
|
|
4349
|
+
continue;
|
|
4350
|
+
}
|
|
4219
4351
|
if (ch === '"' || ch === "'" || backtick && ch === "`") {
|
|
4220
4352
|
quote = ch;
|
|
4353
|
+
verbatim = escapes === "csharp" && ch === '"' && opensCsharpVerbatimString(content, i);
|
|
4221
4354
|
continue;
|
|
4222
4355
|
}
|
|
4223
4356
|
if (ch === "{") depth++;
|
|
@@ -4230,18 +4363,34 @@ function findMatchingBraceEndLine(content, openBraceIndex, totalLines, lineIndex
|
|
|
4230
4363
|
}
|
|
4231
4364
|
return opts?.noMatchValue ?? totalLines;
|
|
4232
4365
|
}
|
|
4233
|
-
function assignBraceBlockSpans(symbols, content,
|
|
4366
|
+
function assignBraceBlockSpans(symbols, content, opts = {}) {
|
|
4367
|
+
const lineCommentPrefix = opts.lineComment;
|
|
4368
|
+
const stringEscapes = opts.stringEscapes ?? "backslash";
|
|
4369
|
+
const nestedBlockComments = opts.nestedBlockComments ?? false;
|
|
4370
|
+
const tripleQuote = opts.tripleQuote ?? false;
|
|
4371
|
+
const lineStringPrefix = opts.lineStringPrefix;
|
|
4372
|
+
const firstLinePrefix = toLineCommentPrefixes(lineCommentPrefix)[0];
|
|
4234
4373
|
if (symbols.length === 0) return [...symbols];
|
|
4235
4374
|
const lines = content.split("\n");
|
|
4236
4375
|
const totalLines = lines.length;
|
|
4237
4376
|
const lineIndex = buildLineIndex(content);
|
|
4238
4377
|
const starts = [...new Set(symbols.map((s) => s.lineStart))].sort((a, b) => a - b);
|
|
4239
|
-
const blockComment =
|
|
4378
|
+
const blockComment = opts.blockComment !== void 0 ? opts.blockComment ?? void 0 : firstLinePrefix === "//" ? ["/*", "*/"] : firstLinePrefix === "#" ? ["<#", "#>"] : void 0;
|
|
4379
|
+
const scanOpts = {
|
|
4380
|
+
noMatchValue: -1,
|
|
4381
|
+
stringEscapes,
|
|
4382
|
+
tripleQuote,
|
|
4383
|
+
rawStringQuotes: opts.rawStringQuotes ?? false,
|
|
4384
|
+
tripleSingleQuote: opts.tripleSingleQuote ?? false,
|
|
4385
|
+
...blockComment === void 0 ? {} : { blockComment, nestedBlockComments },
|
|
4386
|
+
...lineStringPrefix === void 0 ? {} : { lineStringPrefix },
|
|
4387
|
+
...opts.lineCommentExceptions === void 0 ? {} : { lineCommentExceptions: opts.lineCommentExceptions }
|
|
4388
|
+
};
|
|
4240
4389
|
return symbols.map((sym) => {
|
|
4241
4390
|
if (sym.lineEnd !== sym.lineStart) return sym;
|
|
4242
4391
|
const nextStart = starts.find((s) => s > sym.lineStart);
|
|
4243
4392
|
const lastSearchLine = Math.min(nextStart !== void 0 ? nextStart - 1 : totalLines, sym.lineStart + BRACE_SEARCH_MAX_LINES);
|
|
4244
|
-
const openIndex = findBlockOpenBrace(content, lineIndex, sym.lineStart, lastSearchLine, lineCommentPrefix,
|
|
4393
|
+
const openIndex = findBlockOpenBrace(content, lineIndex, sym.lineStart, lastSearchLine, lineCommentPrefix, scanOpts);
|
|
4245
4394
|
if (openIndex === null) return sym;
|
|
4246
4395
|
const endLine = findMatchingBraceEndLine(
|
|
4247
4396
|
content,
|
|
@@ -4249,14 +4398,22 @@ function assignBraceBlockSpans(symbols, content, lineCommentPrefix) {
|
|
|
4249
4398
|
totalLines,
|
|
4250
4399
|
lineIndex,
|
|
4251
4400
|
lineCommentPrefix,
|
|
4252
|
-
|
|
4401
|
+
scanOpts
|
|
4253
4402
|
);
|
|
4254
4403
|
if (endLine <= sym.lineStart) return sym;
|
|
4255
4404
|
return { ...sym, lineEnd: endLine, body: lines.slice(sym.lineStart - 1, endLine).join("\n") };
|
|
4256
4405
|
});
|
|
4257
4406
|
}
|
|
4258
4407
|
var BRACE_SEARCH_MAX_LINES = 10;
|
|
4259
|
-
function findBlockOpenBrace(content, lineIndex, startLine, lastSearchLine, lineCommentPrefix,
|
|
4408
|
+
function findBlockOpenBrace(content, lineIndex, startLine, lastSearchLine, lineCommentPrefix, opts) {
|
|
4409
|
+
const blockComment = opts?.blockComment;
|
|
4410
|
+
const stringEscapes = opts?.stringEscapes ?? "backslash";
|
|
4411
|
+
const nestedBlockComments = opts?.nestedBlockComments === true;
|
|
4412
|
+
const rawString = opts?.rawStringQuotes === true;
|
|
4413
|
+
const tripleDelims = tripleQuoteDelimiters(opts);
|
|
4414
|
+
const lineString = opts?.lineStringPrefix;
|
|
4415
|
+
const linePrefixes = toLineCommentPrefixes(lineCommentPrefix);
|
|
4416
|
+
const lineExceptions = opts?.lineCommentExceptions ?? [];
|
|
4260
4417
|
const from = lineIndex[startLine - 1];
|
|
4261
4418
|
if (from === void 0) return null;
|
|
4262
4419
|
const to = lineIndex[lastSearchLine] ?? content.length;
|
|
@@ -4264,10 +4421,28 @@ function findBlockOpenBrace(content, lineIndex, startLine, lastSearchLine, lineC
|
|
|
4264
4421
|
let parenDepth = 0;
|
|
4265
4422
|
let linesSeen = 0;
|
|
4266
4423
|
let atLineStart = false;
|
|
4424
|
+
let verbatim = false;
|
|
4267
4425
|
for (let i = from; i < to; i++) {
|
|
4268
4426
|
const ch = content[i];
|
|
4269
4427
|
if (ch === void 0) break;
|
|
4270
4428
|
if (quote !== null) {
|
|
4429
|
+
if (verbatim) {
|
|
4430
|
+
if (ch === quote) {
|
|
4431
|
+
if (content[i + 1] === quote) {
|
|
4432
|
+
i++;
|
|
4433
|
+
continue;
|
|
4434
|
+
}
|
|
4435
|
+
quote = null;
|
|
4436
|
+
verbatim = false;
|
|
4437
|
+
}
|
|
4438
|
+
continue;
|
|
4439
|
+
}
|
|
4440
|
+
if (stringEscapes === "powershell") {
|
|
4441
|
+
const step = stepPowershellString(content, i, quote);
|
|
4442
|
+
i = step.next;
|
|
4443
|
+
if (!step.open) quote = null;
|
|
4444
|
+
continue;
|
|
4445
|
+
}
|
|
4271
4446
|
if (ch === "\\") {
|
|
4272
4447
|
i++;
|
|
4273
4448
|
continue;
|
|
@@ -4281,12 +4456,12 @@ function findBlockOpenBrace(content, lineIndex, startLine, lastSearchLine, lineC
|
|
|
4281
4456
|
continue;
|
|
4282
4457
|
}
|
|
4283
4458
|
if (blockComment !== void 0 && content.startsWith(blockComment[0], i)) {
|
|
4284
|
-
const
|
|
4285
|
-
if (
|
|
4286
|
-
i =
|
|
4459
|
+
const end = skipBlockComment(content, i, blockComment, nestedBlockComments);
|
|
4460
|
+
if (end === -1) return null;
|
|
4461
|
+
i = end - 1;
|
|
4287
4462
|
continue;
|
|
4288
4463
|
}
|
|
4289
|
-
if (
|
|
4464
|
+
if (atLineComment(content, i, linePrefixes, lineExceptions)) {
|
|
4290
4465
|
while (i + 1 < to && content[i + 1] !== "\n") i++;
|
|
4291
4466
|
continue;
|
|
4292
4467
|
}
|
|
@@ -4294,14 +4469,36 @@ function findBlockOpenBrace(content, lineIndex, startLine, lastSearchLine, lineC
|
|
|
4294
4469
|
atLineStart = false;
|
|
4295
4470
|
if (parenDepth === 0 && linesSeen >= 1 && startsWithBlockKeyword(content, i, to)) return null;
|
|
4296
4471
|
}
|
|
4472
|
+
if (lineString !== void 0 && content.startsWith(lineString, i)) {
|
|
4473
|
+
while (i + 1 < to && content[i + 1] !== "\n") i++;
|
|
4474
|
+
continue;
|
|
4475
|
+
}
|
|
4476
|
+
if (rawString && ch === '"') {
|
|
4477
|
+
const run = quoteRunLength(content, i);
|
|
4478
|
+
if (run >= 3) {
|
|
4479
|
+
const end = skipRawStringQuotes(content, i + run, run);
|
|
4480
|
+
if (end === -1) return null;
|
|
4481
|
+
i = end - 1;
|
|
4482
|
+
continue;
|
|
4483
|
+
}
|
|
4484
|
+
}
|
|
4485
|
+
const tripleAt = tripleDelims.find((t) => content.startsWith(t, i));
|
|
4486
|
+
if (tripleAt !== void 0) {
|
|
4487
|
+
const end = content.indexOf(tripleAt, i + 3);
|
|
4488
|
+
if (end === -1) return null;
|
|
4489
|
+
i = end + 2;
|
|
4490
|
+
continue;
|
|
4491
|
+
}
|
|
4297
4492
|
if (ch === '"' || ch === "'") {
|
|
4298
4493
|
quote = ch;
|
|
4494
|
+
verbatim = stringEscapes === "csharp" && ch === '"' && opensCsharpVerbatimString(content, i);
|
|
4299
4495
|
continue;
|
|
4300
4496
|
}
|
|
4301
4497
|
if (ch === "(" || ch === "[") parenDepth++;
|
|
4302
4498
|
else if (ch === ")" || ch === "]") {
|
|
4303
4499
|
if (parenDepth > 0) parenDepth--;
|
|
4304
4500
|
} else if (ch === ";") return null;
|
|
4501
|
+
else if (ch === "}" && parenDepth === 0) return null;
|
|
4305
4502
|
else if (ch === "{") return i;
|
|
4306
4503
|
}
|
|
4307
4504
|
return null;
|
|
@@ -4397,8 +4594,9 @@ function extractEnv(content, filePath) {
|
|
|
4397
4594
|
if (_lineClosesQuote(line, openQuote)) openQuote = null;
|
|
4398
4595
|
continue;
|
|
4399
4596
|
}
|
|
4400
|
-
|
|
4401
|
-
|
|
4597
|
+
const trimmed = line.replace(/^[ \t]+/, "");
|
|
4598
|
+
if (!trimmed || trimmed[0] === "#" || trimmed[0] === ";") continue;
|
|
4599
|
+
const m = ENV_KEY_RE.exec(trimmed);
|
|
4402
4600
|
if (m === null) continue;
|
|
4403
4601
|
const name = m[1]?.trim() ?? "";
|
|
4404
4602
|
if (!name || name.length > MAX_HEADING_LEN) continue;
|
|
@@ -4406,7 +4604,7 @@ function extractEnv(content, filePath) {
|
|
|
4406
4604
|
if (seen.has(key)) continue;
|
|
4407
4605
|
seen.add(key);
|
|
4408
4606
|
symbols.push(makeLineSymbol(filePath, name, "env_key", i + 1));
|
|
4409
|
-
openQuote = _detectOpenQuote(
|
|
4607
|
+
openQuote = _detectOpenQuote(trimmed.slice(m[0].length));
|
|
4410
4608
|
}
|
|
4411
4609
|
return symbols;
|
|
4412
4610
|
}
|
|
@@ -4785,7 +4983,8 @@ CREATE TABLE IF NOT EXISTS files (
|
|
|
4785
4983
|
language TEXT,
|
|
4786
4984
|
indexed_at REAL,
|
|
4787
4985
|
embed_sha TEXT,
|
|
4788
|
-
retry_count INTEGER NOT NULL DEFAULT 0
|
|
4986
|
+
retry_count INTEGER NOT NULL DEFAULT 0,
|
|
4987
|
+
parser_sha TEXT
|
|
4789
4988
|
);
|
|
4790
4989
|
-- Expression index on TG_LOWER(path) -- see pathEqClause (sql_path.ts) and TG_LOWER's
|
|
4791
4990
|
-- registration above. TG_LOWER is registered { deterministic: true }, which is required for
|
|
@@ -5045,7 +5244,7 @@ CREATE TRIGGER IF NOT EXISTS cache_recall_au AFTER UPDATE ON cache_recall BEGIN
|
|
|
5045
5244
|
VALUES (new.row_id, new.label, new.content);
|
|
5046
5245
|
END;
|
|
5047
5246
|
`;
|
|
5048
|
-
var SCHEMA_VERSION =
|
|
5247
|
+
var SCHEMA_VERSION = 13;
|
|
5049
5248
|
function alterTableIdempotent(conn, sql) {
|
|
5050
5249
|
try {
|
|
5051
5250
|
conn.exec(sql);
|
|
@@ -5089,7 +5288,9 @@ var MIGRATIONS = {
|
|
|
5089
5288
|
// .env would have kept serving its pre-fix chunks indefinitely. Deleting the rows here both
|
|
5090
5289
|
// removes the stored secrets and, by clearing embed_sha, makes the next drain re-embed the file
|
|
5091
5290
|
// through the redacting path.
|
|
5092
|
-
10: purgeDotenvEmbeddings
|
|
5291
|
+
10: purgeDotenvEmbeddings,
|
|
5292
|
+
// v12 -> v13: adds files.parser_sha, the digest of the extraction logic that produced this file's rows, tracked separately from files.sha for the same reason embed_sha is -- content freshness and parse freshness are different questions, and the content sha alone could only ever answer the first. A pre-existing v12 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 / v9 -> v10 above. Deliberately left NULL for every existing row rather than backfilled with the current fingerprint: NULL is the truthful answer (nobody recorded which parser wrote those rows), and it is also the answer that makes the freshness gates reparse them once, which is exactly what a database indexed by an older parser needs.
|
|
5293
|
+
12: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN parser_sha TEXT")
|
|
5093
5294
|
};
|
|
5094
5295
|
function runMigrations(conn, fromVersion, toVersion) {
|
|
5095
5296
|
for (let v = fromVersion; v < toVersion; v++) {
|
|
@@ -5575,7 +5776,51 @@ var _KIND_GROUPS = [
|
|
|
5575
5776
|
"exports",
|
|
5576
5777
|
"imports",
|
|
5577
5778
|
"changed_lookup",
|
|
5578
|
-
"dep_docs"
|
|
5779
|
+
"dep_docs",
|
|
5780
|
+
// Every other SOURCE_READ kind in stats.ts's KIND_TO_SOURCE: the surgical-read commands over documents, structured data and session/PR state. They were registered and produced but grouped nowhere, so `stats --full` printed the whole family under 'Other', away from the read-savings siblings they are measured against. image_meta/image_text sit here rather than under 'Images' because stats.ts files them as SOURCE_READ: they save read bytes, they do not shrink an image.
|
|
5781
|
+
"brief_view",
|
|
5782
|
+
"conflicts",
|
|
5783
|
+
"coverage_report_gaps",
|
|
5784
|
+
"csv_query",
|
|
5785
|
+
"csv_profile",
|
|
5786
|
+
"compact_doc",
|
|
5787
|
+
"docx_outline",
|
|
5788
|
+
"docx_text",
|
|
5789
|
+
"gdrive_sections",
|
|
5790
|
+
"image_meta",
|
|
5791
|
+
"image_text",
|
|
5792
|
+
"json_query",
|
|
5793
|
+
"json_outline",
|
|
5794
|
+
"note_read",
|
|
5795
|
+
"note_list",
|
|
5796
|
+
"openapi_op",
|
|
5797
|
+
"openapi_outline",
|
|
5798
|
+
"pdf_extract",
|
|
5799
|
+
"pdf_locate",
|
|
5800
|
+
"pdf_outline",
|
|
5801
|
+
"pdf_meta",
|
|
5802
|
+
"pptx_outline",
|
|
5803
|
+
"pptx_slide",
|
|
5804
|
+
"pptx_notes",
|
|
5805
|
+
"pptx_text",
|
|
5806
|
+
"pr_slice",
|
|
5807
|
+
"session_outline",
|
|
5808
|
+
"session_slice",
|
|
5809
|
+
"sqlite_query",
|
|
5810
|
+
"sqlite_schema",
|
|
5811
|
+
"transcript",
|
|
5812
|
+
"transcript_outline",
|
|
5813
|
+
"video_chapters",
|
|
5814
|
+
"xlsx_sheets",
|
|
5815
|
+
"xlsx_head",
|
|
5816
|
+
"xlsx_range",
|
|
5817
|
+
"xlsx_query",
|
|
5818
|
+
"xml_query",
|
|
5819
|
+
"xml_outline",
|
|
5820
|
+
"yaml_query",
|
|
5821
|
+
"yaml_outline",
|
|
5822
|
+
"zip_list",
|
|
5823
|
+
"zip_read"
|
|
5579
5824
|
])
|
|
5580
5825
|
},
|
|
5581
5826
|
{ label: "Lookups", members: /* @__PURE__ */ new Set(["symbol_lookup", "semantic_search", "map_lookup"]) },
|
|
@@ -5597,74 +5842,72 @@ var _KIND_GROUPS = [
|
|
|
5597
5842
|
"session_hint_overhead",
|
|
5598
5843
|
"session_hint_suppressed",
|
|
5599
5844
|
"read_count_deny",
|
|
5600
|
-
"read_dedup_hint",
|
|
5601
5845
|
"grep_dedup_hint",
|
|
5602
5846
|
"glob_dedup_hint",
|
|
5603
5847
|
"diff_hint",
|
|
5604
5848
|
"predictive_prefetch_hit",
|
|
5605
|
-
"read_partial_overlap_hint",
|
|
5606
5849
|
"structured_file_hint",
|
|
5607
5850
|
"write_rewrite_hint",
|
|
5608
5851
|
"websearch_dedup_hint",
|
|
5609
5852
|
"large_file_hint_followed",
|
|
5610
|
-
"large_file_hint_ignored"
|
|
5611
|
-
|
|
5612
|
-
},
|
|
5613
|
-
{
|
|
5614
|
-
label: "Bash",
|
|
5615
|
-
members: /* @__PURE__ */ new Set([
|
|
5616
|
-
"bash_dedup_hint",
|
|
5617
|
-
"bash_output_cached",
|
|
5618
|
-
"bash_output_recall",
|
|
5619
|
-
"bash_output_recall_miss",
|
|
5620
|
-
"bash_dedup_stale",
|
|
5621
|
-
"bash_range_read_hint",
|
|
5622
|
-
"bash_streak_hint",
|
|
5623
|
-
"bash_poll_hint",
|
|
5624
|
-
"env_probe_cache_hit",
|
|
5625
|
-
"git_diff_scope_hint",
|
|
5626
|
-
"dep_list_cache_hit",
|
|
5627
|
-
"bash_read_equiv_already_read",
|
|
5628
|
-
"bash_grep_result_cache_hit",
|
|
5629
|
-
"git_diff_context_trimmed"
|
|
5853
|
+
"large_file_hint_ignored",
|
|
5854
|
+
"evidence_cache_hit"
|
|
5630
5855
|
])
|
|
5631
5856
|
},
|
|
5857
|
+
// Empty for the same reason as MCP below: every live Bash kind arrives through _kindGroupLabel's `bash_compress:` prefix branch, not through a literal name. The fifteen literal names this set used to carry (bash_output_cached, bash_dedup_hint, env_probe_cache_hit and the rest) came over with the Python port and were never recorded or registered anywhere in this tree, so they grouped rows that could not exist.
|
|
5858
|
+
{ label: "Bash", members: /* @__PURE__ */ new Set() },
|
|
5632
5859
|
{
|
|
5633
5860
|
label: "Web",
|
|
5634
5861
|
members: /* @__PURE__ */ new Set([
|
|
5635
|
-
"web_dedup_hint",
|
|
5636
|
-
"web_output_cached",
|
|
5637
|
-
"web_output_recall",
|
|
5638
|
-
"web_output_recall_miss",
|
|
5639
|
-
"web_dedup_stale",
|
|
5640
5862
|
"web_fetch",
|
|
5641
5863
|
"injection_detected"
|
|
5642
5864
|
])
|
|
5643
5865
|
},
|
|
5866
|
+
// Membership comes from _kindGroupLabel's `mcp:` prefix branch, not from this set, which is why
|
|
5867
|
+
// it is empty. The entry still has to exist: _renderByKindSection iterates _KIND_GROUPS' labels
|
|
5868
|
+
// (plus 'Other') to decide what to print, so a label _kindGroupLabel returns but that is missing
|
|
5869
|
+
// here does not fall back to 'Other' -- its rows disappear from the table entirely.
|
|
5870
|
+
{ label: "MCP", members: /* @__PURE__ */ new Set() },
|
|
5644
5871
|
{
|
|
5645
5872
|
label: "Compact / Skills",
|
|
5646
5873
|
members: /* @__PURE__ */ new Set([
|
|
5647
|
-
"compact_manifest",
|
|
5648
|
-
"compact_assist",
|
|
5649
|
-
"compact_recovery",
|
|
5650
|
-
"skill_body_recall",
|
|
5651
|
-
"skill_compact_served",
|
|
5652
|
-
"skill_cached",
|
|
5653
5874
|
"skill_load",
|
|
5654
5875
|
"skill_oversized_first_load",
|
|
5655
|
-
"skill_compact_inlined"
|
|
5656
|
-
|
|
5657
|
-
|
|
5876
|
+
"skill_compact_inlined"
|
|
5877
|
+
])
|
|
5878
|
+
},
|
|
5879
|
+
// SOURCE_CONTENT: real rewrites of tool output that remove real bytes (agent report compaction, Grep fold, browser tab dedup, bash/content compression and the handoff pair). The by-source table has shown a 'content' row since the source was added, but the by-kind table had no member set for it, so every one of these kinds printed under 'Other'. The taskoutput: prefix branch in _kindGroupLabel routes here too.
|
|
5880
|
+
{
|
|
5881
|
+
label: "Content",
|
|
5882
|
+
members: /* @__PURE__ */ new Set([
|
|
5883
|
+
"content_compress",
|
|
5884
|
+
"content_retrieve",
|
|
5885
|
+
"agent_report_compact",
|
|
5886
|
+
"agent_report_compact_declined",
|
|
5887
|
+
"browser_tab_dedup",
|
|
5888
|
+
"grep:fold",
|
|
5889
|
+
"handoff_create",
|
|
5890
|
+
"handoff_resolve",
|
|
5891
|
+
"plan_echo_collapse"
|
|
5658
5892
|
])
|
|
5659
5893
|
}
|
|
5660
5894
|
];
|
|
5661
5895
|
function _kindGroupLabel(kind) {
|
|
5662
|
-
if (kind.startsWith("bash_compress:")) {
|
|
5896
|
+
if (kind.startsWith("bash_compress:") || kind.startsWith("bashoutput:")) {
|
|
5663
5897
|
return "Bash";
|
|
5664
5898
|
}
|
|
5665
|
-
if (kind.startsWith("webfetch:")) {
|
|
5899
|
+
if (kind.startsWith("webfetch:") || kind.startsWith("gdrive:")) {
|
|
5666
5900
|
return "Web";
|
|
5667
5901
|
}
|
|
5902
|
+
if (kind.startsWith("mcp:")) {
|
|
5903
|
+
return "MCP";
|
|
5904
|
+
}
|
|
5905
|
+
if (kind.startsWith("skill_body:") || kind.startsWith("skill_compact:")) {
|
|
5906
|
+
return "Compact / Skills";
|
|
5907
|
+
}
|
|
5908
|
+
if (kind.startsWith("taskoutput:")) {
|
|
5909
|
+
return "Content";
|
|
5910
|
+
}
|
|
5668
5911
|
for (const group of _KIND_GROUPS) {
|
|
5669
5912
|
if (group.members.has(kind)) {
|
|
5670
5913
|
return group.label;
|
|
@@ -5968,6 +6211,10 @@ var SOURCE_SKILL = "skill";
|
|
|
5968
6211
|
var SOURCE_CONTENT = "content";
|
|
5969
6212
|
var SOURCE_OTHER = "other";
|
|
5970
6213
|
var _BYTES_MODE_ONLY_KINDS = /* @__PURE__ */ new Set(["webfetch_image", "gdrive_image"]);
|
|
6214
|
+
var COUNT_ONLY_KINDS = /* @__PURE__ */ new Set(["secret_redacted"]);
|
|
6215
|
+
function savedTokensFromBytes(bytes) {
|
|
6216
|
+
return Math.round(Math.max(0, bytes) / 4);
|
|
6217
|
+
}
|
|
5971
6218
|
var KIND_TO_SOURCE = {
|
|
5972
6219
|
image_shrink: SOURCE_IMAGE,
|
|
5973
6220
|
image_shrink_cache_hit: SOURCE_IMAGE,
|
|
@@ -6057,6 +6304,10 @@ var KIND_TO_SOURCE = {
|
|
|
6057
6304
|
// Cold first load of an oversized skill where preSkillHandler inlined the compact slice in its reply instead of pointing at `skill-body --compact`. Unlike its skill_oversized_first_load sibling (event-only, 0 bytes -- the pointer deny saves nothing by itself, the follow-up command does) this one records real savings: the full body never landed, the slice did, so bytesSaved is body minus slice.
|
|
6058
6305
|
skill_compact_inlined: SOURCE_SKILL,
|
|
6059
6306
|
secret_redacted: SOURCE_OTHER,
|
|
6307
|
+
// Fail-soft diagnostic counters from hooks_edit.ts: they record that a side task threw, never a byte saving, so "other" is the right home. Listed explicitly rather than left to kindToSource()'s fallback so the registration guard can tell a deliberate placement from an unregistered kind.
|
|
6308
|
+
dirty_queue_append_failed: SOURCE_OTHER,
|
|
6309
|
+
worker_healthcheck_failed: SOURCE_OTHER,
|
|
6310
|
+
known_root_record_failed: SOURCE_OTHER,
|
|
6060
6311
|
// Measurement of what a compaction produced (hooks_compact.ts postCompactHandler): summary size and how many manifest paths survived into it. SOURCE_OTHER and always recorded at (0, 0) -- the summary was written whether or not token-goat was watching, so there is no counterfactual in which those bytes were saved. Filing it anywhere with a savings total would credit token-goat for the whole summary, which is the accounting mistake this registry exists to prevent.
|
|
6061
6312
|
compact_summary: SOURCE_OTHER,
|
|
6062
6313
|
// Envelope compaction of an oversized subagent report (hooks_agent_spawn.ts). SOURCE_CONTENT, not SOURCE_HINT: the handler's sibling session_hint entry is advisory (it only appends a recall pointer and genuinely saves nothing), whereas this kind records a real rewrite with real bytes removed, so filing it under the advisory bucket would understate the compaction and repeat the zero-savings desync this registry keeps getting bitten by.
|
|
@@ -6064,6 +6315,10 @@ var KIND_TO_SOURCE = {
|
|
|
6064
6315
|
// Decline counterpart to agent_report_compact: the fence-collapse net-benefit gate ran and found at least one over-long fence, but declined to rewrite because net savings did not clear the notice cost. Always recorded at (0, 0) -- see the recordStat call site -- so it never contributes to any savings total; it exists purely to make gate hit-rate and near-misses visible instead of the decline being invisible.
|
|
6065
6316
|
agent_report_compact_declined: SOURCE_CONTENT,
|
|
6066
6317
|
content_compress: SOURCE_CONTENT,
|
|
6318
|
+
// Verbatim-repeat collapse of a browser tool's "Tab Context:" text block (hooks_browser_image.ts postBrowserImageHandler). SOURCE_CONTENT for the same reason as agent_report_compact above: it is a real rewrite with real bytes removed, not an advisory nudge. Deliberately not SOURCE_IMAGE -- it shares a handler with image_shrink but collapses text, and folding text bytes into the image ledger is the two-units-under-one-label mistake this file's image_shrink entry was just fixed for.
|
|
6319
|
+
browser_tab_dedup: SOURCE_CONTENT,
|
|
6320
|
+
// Collapse of the plan echo in an approved ExitPlanMode result (hooks_exitplanmode.ts). SOURCE_CONTENT for the same reason as agent_report_compact: real bytes removed from a tool result, not an advisory nudge. The handler shipped for releases emitting this rewrite and recording nothing at all, so the mechanism was invisible in `stats` and its net benefit could not be checked against the gate that admits it.
|
|
6321
|
+
plan_echo_collapse: SOURCE_CONTENT,
|
|
6067
6322
|
// Lossless re-layout of Grep content-mode output (hooks_grep.ts foldGrepContentHandler). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as agent_report_compact above: its sibling grep_dedup_hint is advisory and saves nothing directly, whereas this is a real rewrite with real bytes removed. Filing it under the advisory bucket would silently add non-hint savings to hint_stats.ts's savedBytes (which reads by_source[SOURCE_HINT] wholesale) and overstate the hint ledger's net benefit.
|
|
6068
6323
|
"grep:fold": SOURCE_CONTENT,
|
|
6069
6324
|
content_retrieve: SOURCE_CONTENT,
|
|
@@ -6076,7 +6331,9 @@ var KIND_PREFIX_TO_SOURCE = [
|
|
|
6076
6331
|
["gdrive:", SOURCE_WEB],
|
|
6077
6332
|
["mcp:", SOURCE_MCP],
|
|
6078
6333
|
["skill_body:", SOURCE_SKILL],
|
|
6079
|
-
["skill_compact:", SOURCE_SKILL]
|
|
6334
|
+
["skill_compact:", SOURCE_SKILL],
|
|
6335
|
+
["bashoutput:", SOURCE_BASH],
|
|
6336
|
+
["taskoutput:", SOURCE_CONTENT]
|
|
6080
6337
|
];
|
|
6081
6338
|
var COMMAND_KINDS = {
|
|
6082
6339
|
symbol: /* @__PURE__ */ new Set(["symbol_lookup"]),
|
|
@@ -6190,7 +6447,8 @@ CREATE TABLE IF NOT EXISTS stats (
|
|
|
6190
6447
|
tokens_saved INTEGER NOT NULL DEFAULT 0,
|
|
6191
6448
|
bytes_saved INTEGER NOT NULL DEFAULT 0,
|
|
6192
6449
|
detail TEXT,
|
|
6193
|
-
harness TEXT
|
|
6450
|
+
harness TEXT,
|
|
6451
|
+
traceparent TEXT
|
|
6194
6452
|
);
|
|
6195
6453
|
CREATE INDEX IF NOT EXISTS idx_stats_ts ON stats(ts);
|
|
6196
6454
|
CREATE INDEX IF NOT EXISTS idx_stats_kind ON stats(kind);
|
|
@@ -6213,6 +6471,11 @@ function migrateGlobalSchema(db) {
|
|
|
6213
6471
|
} catch (err) {
|
|
6214
6472
|
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
6215
6473
|
}
|
|
6474
|
+
try {
|
|
6475
|
+
db.exec("ALTER TABLE stats ADD COLUMN traceparent TEXT");
|
|
6476
|
+
} catch (err) {
|
|
6477
|
+
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
6478
|
+
}
|
|
6216
6479
|
}
|
|
6217
6480
|
var _harnessColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
6218
6481
|
function statsHasHarnessColumn(db) {
|
|
@@ -6229,6 +6492,21 @@ function statsHasHarnessColumn(db) {
|
|
|
6229
6492
|
_harnessColumnByDb.set(db, present);
|
|
6230
6493
|
return present;
|
|
6231
6494
|
}
|
|
6495
|
+
var _traceparentColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
6496
|
+
function statsHasTraceparentColumn(db) {
|
|
6497
|
+
const cached = _traceparentColumnByDb.get(db);
|
|
6498
|
+
if (cached !== void 0) return cached;
|
|
6499
|
+
let present;
|
|
6500
|
+
try {
|
|
6501
|
+
present = db.prepare("PRAGMA table_info(stats)").all().some(
|
|
6502
|
+
(c) => c.name === "traceparent"
|
|
6503
|
+
);
|
|
6504
|
+
} catch {
|
|
6505
|
+
present = false;
|
|
6506
|
+
}
|
|
6507
|
+
_traceparentColumnByDb.set(db, present);
|
|
6508
|
+
return present;
|
|
6509
|
+
}
|
|
6232
6510
|
function getGlobalDb(homeDir) {
|
|
6233
6511
|
const basePath = homeDir ? dataDirForHome(homeDir) : dataDir();
|
|
6234
6512
|
const dbPath = path8.join(basePath, "global.db");
|
|
@@ -6247,11 +6525,18 @@ function noStatsMessage(windowDays, homeDir) {
|
|
|
6247
6525
|
if (total === 0) return "No stats recorded yet.";
|
|
6248
6526
|
return `No stats in the last ${countNoun(windowDays, "day")} (${total} recorded outside this window; use --window-days 0 for all time).`;
|
|
6249
6527
|
}
|
|
6250
|
-
function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail) {
|
|
6528
|
+
function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail, traceparent) {
|
|
6251
6529
|
try {
|
|
6252
6530
|
const db = _testDb ?? getGlobalDb();
|
|
6253
6531
|
const ts = Math.floor(Date.now() / 1e3);
|
|
6254
|
-
|
|
6532
|
+
const tp = traceparent ?? process.env["TRACEPARENT"] ?? process.env["traceparent"] ?? null;
|
|
6533
|
+
const hasHarness = statsHasHarnessColumn(db);
|
|
6534
|
+
const hasTraceparent = statsHasTraceparentColumn(db);
|
|
6535
|
+
if (hasHarness && hasTraceparent) {
|
|
6536
|
+
db.prepare(
|
|
6537
|
+
"INSERT INTO stats (ts, kind, bytes_saved, tokens_saved, detail, harness, traceparent) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
6538
|
+
).run(ts, kind, bytesSaved, tokensSaved, detail ?? null, getHarnessName(), tp);
|
|
6539
|
+
} else if (hasHarness) {
|
|
6255
6540
|
db.prepare(
|
|
6256
6541
|
"INSERT INTO stats (ts, kind, bytes_saved, tokens_saved, detail, harness) VALUES (?, ?, ?, ?, ?, ?)"
|
|
6257
6542
|
).run(ts, kind, bytesSaved, tokensSaved, detail ?? null, getHarnessName());
|
|
@@ -6306,10 +6591,14 @@ function summarize(windowDays = 30, testDb, homeDir) {
|
|
|
6306
6591
|
const stmt = db.prepare(query);
|
|
6307
6592
|
const rows = sinceTs !== null ? stmt.all(sinceTs) : stmt.all();
|
|
6308
6593
|
const tsToDateCache = {};
|
|
6594
|
+
const counts = {};
|
|
6309
6595
|
for (const row of rows) {
|
|
6310
6596
|
const bytesSaved = row.bytes_saved ?? 0;
|
|
6311
|
-
const
|
|
6597
|
+
const recorded = row.tokens_saved ?? 0;
|
|
6312
6598
|
const kind = row.kind;
|
|
6599
|
+
const isCount = COUNT_ONLY_KINDS.has(kind);
|
|
6600
|
+
if (isCount) counts[kind] = (counts[kind] ?? 0) + recorded;
|
|
6601
|
+
const tokensSaved = isCount ? 0 : recorded;
|
|
6313
6602
|
const tsRaw = row.ts;
|
|
6314
6603
|
if (tsRaw === void 0) continue;
|
|
6315
6604
|
const ts = tsRaw;
|
|
@@ -6368,6 +6657,7 @@ function summarize(windowDays = 30, testDb, homeDir) {
|
|
|
6368
6657
|
by_project: byProjectList,
|
|
6369
6658
|
by_source: bySourceDict,
|
|
6370
6659
|
by_harness: byHarness,
|
|
6660
|
+
counts,
|
|
6371
6661
|
by_command: Object.entries(byCommandDict).map(([command, bucket]) => ({ ...bucket, command })).filter((r) => r.events > 0),
|
|
6372
6662
|
window_days: windowDays
|
|
6373
6663
|
};
|
|
@@ -6378,6 +6668,10 @@ function _totalsLines(summary) {
|
|
|
6378
6668
|
`Total events: ${summary.total_events}`,
|
|
6379
6669
|
`Bytes saved: ${fmtBytes(summary.total_bytes_saved)}`,
|
|
6380
6670
|
`Tokens saved: ${summary.total_tokens_saved}`,
|
|
6671
|
+
// Printed on its own line, below the token total and never inside it, because it counts
|
|
6672
|
+
// placeholders rather than tokens. Omitted entirely when nothing was redacted, so the line is
|
|
6673
|
+
// information rather than a permanent zero. See COUNT_ONLY_KINDS.
|
|
6674
|
+
...summary.counts["secret_redacted"] ? [`Secrets hidden: ${summary.counts["secret_redacted"]} (a count, not tokens)`] : [],
|
|
6381
6675
|
`Window: ${summary.window_days} days`
|
|
6382
6676
|
];
|
|
6383
6677
|
}
|
|
@@ -6529,150 +6823,45 @@ function renderStats2(opts) {
|
|
|
6529
6823
|
|
|
6530
6824
|
// src/secret_redact.ts
|
|
6531
6825
|
var SECRET_PATTERNS = [
|
|
6532
|
-
// Anthropic keys share OpenAI's "sk-" prefix but are more specific
|
|
6533
|
-
// ("sk-ant-"), so they're matched first — the generic OpenAI pattern's
|
|
6534
|
-
// negative lookahead below is defense in depth, not the sole guard.
|
|
6826
|
+
// Anthropic keys share OpenAI's "sk-" prefix but are more specific ("sk-ant-"), so they're matched first — the generic OpenAI pattern's negative lookahead below is defense in depth, not the sole guard.
|
|
6535
6827
|
["anthropic_api_key", /sk-ant-[A-Za-z0-9_-]{20,}/g],
|
|
6536
|
-
// Modern (post-2024, now the default) OpenAI project keys: "sk-proj-" followed by a long
|
|
6537
|
-
// base64url-ish body that legitimately contains '-' and '_' -- the generic pattern below
|
|
6538
|
-
// deliberately excludes those characters (hyphenated prose would false-fire), so this needs
|
|
6539
|
-
// its own entry with the more specific prefix matched first, like sk-ant- above.
|
|
6828
|
+
// Modern (post-2024, now the default) OpenAI project keys: "sk-proj-" followed by a long base64url-ish body that legitimately contains '-' and '_' -- the generic pattern below deliberately excludes those characters (hyphenated prose would false-fire), so this needs its own entry with the more specific prefix matched first, like sk-ant- above.
|
|
6540
6829
|
["openai_project_key", /sk-proj-[A-Za-z0-9_-]{20,}/g],
|
|
6541
6830
|
["openai_api_key", /sk-(?!ant-|proj-)[A-Za-z0-9]{20,}/g],
|
|
6542
6831
|
["aws_access_key", /AKIA[0-9A-Z]{16}/g],
|
|
6543
|
-
// Fine-grained PATs ("github_pat_...") are matched before the classic gh[oprsu]_ pattern so
|
|
6544
|
-
// the full token is always consumed as one match -- a fine-grained token's own body can
|
|
6545
|
-
// contain '_' and could otherwise partially match the classic pattern.
|
|
6832
|
+
// Fine-grained PATs ("github_pat_...") are matched before the classic gh[oprsu]_ pattern so the full token is always consumed as one match -- a fine-grained token's own body can contain '_' and could otherwise partially match the classic pattern.
|
|
6546
6833
|
["github_token", /github_pat_[A-Za-z0-9_]{22,}/g],
|
|
6547
6834
|
["github_token", /gh[oprsu]_[A-Za-z0-9]{36,}/g],
|
|
6548
|
-
// xapp- is Slack's app-level token (Socket Mode), a bearer credential in its own right that
|
|
6549
|
-
// the xox[baprs]- prefix does not cover. It gets its own entry rather than joining the
|
|
6550
|
-
// alternation above because it needs the full segmented shape (xapp-<ver>-<app id>-<digits>-
|
|
6551
|
-
// <hex>) to be safe: a bare /xapp-[A-Za-z0-9-]+/ redacted ordinary identifiers like
|
|
6552
|
-
// "xapp-config" and the css class "xapp-container", mangling normal source. The xox* prefixes
|
|
6553
|
-
// are distinctive enough on their own; "xapp-" is not.
|
|
6835
|
+
// xapp- is Slack's app-level token (Socket Mode), a bearer credential in its own right that the xox[baprs]- prefix does not cover. It gets its own entry rather than joining the alternation above because it needs the full segmented shape (xapp-<ver>-<app id>-<digits>- <hex>) to be safe: a bare /xapp-[A-Za-z0-9-]+/ redacted ordinary identifiers like "xapp-config" and the css class "xapp-container", mangling normal source. The xox* prefixes are distinctive enough on their own; "xapp-" is not.
|
|
6554
6836
|
["slack_token", /xox[baprs]-[A-Za-z0-9-]+/g],
|
|
6555
6837
|
["slack_token", /xapp-\d-[A-Za-z0-9]{6,}-\d{8,}-[A-Za-z0-9]{16,}/g],
|
|
6556
|
-
// Matches the full block (BEGIN marker through its matching END marker), not just the
|
|
6557
|
-
// header -- the actual secret material is the base64 body between them, so redacting only
|
|
6558
|
-
// the header line would leave the key bytes themselves fully readable in the cached blob.
|
|
6559
|
-
// The body is lazy and additionally refuses to cross a following BEGIN marker. Both halves
|
|
6560
|
-
// matter. The laziness bounds a successful match at the very next END marker; the negative
|
|
6561
|
-
// lookahead bounds a *failing* one, because a BEGIN with no END of its own would otherwise
|
|
6562
|
-
// scan to end-of-input, and a blob full of such markers made the pass quadratic -- 6 ms, 20 ms
|
|
6563
|
-
// and 78 ms for 2000, 4000 and 8000 of them, four times the work for twice the input, the same
|
|
6564
|
-
// shape as the lookbehind incident described above. Private key blocks do not nest, so
|
|
6565
|
-
// refusing to cross a BEGIN costs nothing in correctness.
|
|
6566
|
-
// The algorithm and ` BLOCK` groups are backreferenced in the END marker rather than repeated,
|
|
6567
|
-
// so a BEGIN only ever pairs with its own END spelling. Written as two independent optional
|
|
6568
|
-
// groups, `BEGIN PGP PRIVATE KEY` would happily close on a distant `END RSA PRIVATE KEY BLOCK`
|
|
6569
|
-
// and redact everything in between. A non-participating group backreferences as the empty
|
|
6570
|
-
// string in JS, which is exactly what the unprefixed `BEGIN PRIVATE KEY` form needs.
|
|
6571
|
-
// The algorithm list covers every armored private-key header openssl, ssh-keygen and gpg
|
|
6572
|
-
// actually emit, not just the ones a first draft happened to think of. ENCRYPTED is the
|
|
6573
|
-
// PKCS#8 passphrase-protected form (`openssl genpkey -aes256`, `ssh-keygen -m PKCS8`) and is
|
|
6574
|
-
// the most common shape of all; DSA is legacy but still written verbatim; PGP carries the
|
|
6575
|
-
// ` BLOCK` suffix, which is why that suffix is optional here. All three used to fall through
|
|
6576
|
-
// to disk in full. An encrypted key is still key material: the passphrase can be attacked
|
|
6577
|
-
// offline once the bytes are cached, so it is redacted like any other.
|
|
6578
|
-
// PuTTY `.ppk` files are deliberately not matched. They have no END marker -- the private
|
|
6579
|
-
// section is a `Private-Lines: N` count followed by exactly N base64 lines -- so bounding a
|
|
6580
|
-
// match would take a stateful parse rather than a regex, on a path that runs over every
|
|
6581
|
-
// command output. A regex guess at where the body ends is exactly the over-eager match this
|
|
6582
|
-
// module's header warns against.
|
|
6838
|
+
// Matches the full block (BEGIN marker through its matching END marker), not just the header -- the actual secret material is the base64 body between them, so redacting only the header line would leave the key bytes themselves fully readable in the cached blob. The body is lazy and additionally refuses to cross a following BEGIN marker. Both halves matter. The laziness bounds a successful match at the very next END marker; the negative lookahead bounds a *failing* one, because a BEGIN with no END of its own would otherwise scan to end-of-input, and a blob full of such markers made the pass quadratic -- 6 ms, 20 ms and 78 ms for 2000, 4000 and 8000 of them, four times the work for twice the input, the same shape as the lookbehind incident described above. Private key blocks do not nest, so refusing to cross a BEGIN costs nothing in correctness. The algorithm and ` BLOCK` groups are backreferenced in the END marker rather than repeated, so a BEGIN only ever pairs with its own END spelling. Written as two independent optional groups, `BEGIN PGP PRIVATE KEY` would happily close on a distant `END RSA PRIVATE KEY BLOCK` and redact everything in between. A non-participating group backreferences as the empty string in JS, which is exactly what the unprefixed `BEGIN PRIVATE KEY` form needs. The algorithm list covers every armored private-key header openssl, ssh-keygen and gpg actually emit, not just the ones a first draft happened to think of. ENCRYPTED is the PKCS#8 passphrase-protected form (`openssl genpkey -aes256`, `ssh-keygen -m PKCS8`) and is the most common shape of all; DSA is legacy but still written verbatim; PGP carries the ` BLOCK` suffix, which is why that suffix is optional here. All three used to fall through to disk in full. An encrypted key is still key material: the passphrase can be attacked offline once the bytes are cached, so it is redacted like any other. PuTTY `.ppk` files are deliberately not matched. They have no END marker -- the private section is a `Private-Lines: N` count followed by exactly N base64 lines -- so bounding a match would take a stateful parse rather than a regex, on a path that runs over every command output. A regex guess at where the body ends is exactly the over-eager match this module's header warns against.
|
|
6583
6839
|
["private_key_block", /-----BEGIN (RSA |DSA |EC |OPENSSH |ENCRYPTED |PGP )?PRIVATE KEY( BLOCK)?-----(?:(?!-----BEGIN )[\s\S])*?-----END \1PRIVATE KEY\2-----/g],
|
|
6584
|
-
// Redacts only the token itself, not the "Authorization: Bearer " prefix -- the lookbehind
|
|
6585
|
-
// anchors on the header name and scheme so the surrounding request-log line stays readable,
|
|
6586
|
-
// matching how AWS_ACCESS_KEY_ID=... above keeps its own prefix intact.
|
|
6587
|
-
// The optional quotes on either side of the colon are what let this see a header carried in
|
|
6588
|
-
// JSON rather than in raw wire format. Without them the lookbehind demanded the colon sit
|
|
6589
|
-
// directly against the header name and the scheme directly against the space, so a body like
|
|
6590
|
-
// {"Authorization": "Bearer <token>"} -- the shape any logged fetch or MCP result arrives in
|
|
6591
|
-
// -- matched nothing and the token was cached verbatim.
|
|
6592
|
-
// `token` is the second scheme spelling in wide use -- it is what curl and gh examples pass for
|
|
6593
|
-
// GitHub and many other APIs -- and an opaque value behind it carries exactly the same authority
|
|
6594
|
-
// as one behind `Bearer`. The trailing gap is `{1,8}` rather than a single space for the same
|
|
6595
|
-
// reason every other gap in this lookbehind already is: a hand-aligned or reformatted header
|
|
6596
|
-
// ("Authorization: Bearer <token>") is ordinary, and demanding exactly one space there made
|
|
6597
|
-
// this the one position in the pattern that a second space defeated.
|
|
6840
|
+
// Redacts only the token itself, not the "Authorization: Bearer " prefix -- the lookbehind anchors on the header name and scheme so the surrounding request-log line stays readable, matching how AWS_ACCESS_KEY_ID=... above keeps its own prefix intact. The optional quotes on either side of the colon are what let this see a header carried in JSON rather than in raw wire format. Without them the lookbehind demanded the colon sit directly against the header name and the scheme directly against the space, so a body like {"Authorization": "Bearer <token>"} -- the shape any logged fetch or MCP result arrives in -- matched nothing and the token was cached verbatim. `token` is the second scheme spelling in wide use -- it is what curl and gh examples pass for GitHub and many other APIs -- and an opaque value behind it carries exactly the same authority as one behind `Bearer`. The trailing gap is `{1,8}` rather than a single space for the same reason every other gap in this lookbehind already is: a hand-aligned or reformatted header ("Authorization: Bearer <token>") is ordinary, and demanding exactly one space there made this the one position in the pattern that a second space defeated.
|
|
6598
6841
|
["auth_bearer_token", /(?<=Authorization["']?[ \t]{0,8}:[ \t]{0,8}["']?[ \t]{0,8}(?:Bearer|token)[ \t]{1,8})[A-Za-z0-9\-._~+/]{10,}=*/gi],
|
|
6599
6842
|
["auth_basic_token", /(?<=Authorization["']?[ \t]{0,8}:[ \t]{0,8}["']?[ \t]{0,8}Basic[ \t])[A-Za-z0-9+/]{6,}=*/gi],
|
|
6600
|
-
// JWTs have no distinctive prefix of their own, but the base64url encoding of the smallest
|
|
6601
|
-
// realistic header ('{"alg":' or similar) always starts with "eyJ", so that's the practical
|
|
6602
|
-
// anchor here -- each of the three dot-separated segments requires a minimum length to avoid
|
|
6603
|
-
// matching a short, coincidentally dotted token.
|
|
6843
|
+
// JWTs have no distinctive prefix of their own, but the base64url encoding of the smallest realistic header ('{"alg":' or similar) always starts with "eyJ", so that's the practical anchor here -- each of the three dot-separated segments requires a minimum length to avoid matching a short, coincidentally dotted token.
|
|
6604
6844
|
//
|
|
6605
|
-
// The trailing `={0,2}` on each segment is what makes a padded token match. Base64url as the JWT
|
|
6606
|
-
// spec defines it drops the `=` padding, but producers that reach for a plain base64 encoder
|
|
6607
|
-
// emit it anyway, and a `=` in the header or payload segment used to defeat the match outright:
|
|
6608
|
-
// not a partial redaction, but none at all, so the entire token was printed. `=` cannot appear
|
|
6609
|
-
// anywhere except the end of a segment, since it is not one of the characters the segment body
|
|
6610
|
-
// allows, so accepting it here cannot widen the match onto anything else.
|
|
6845
|
+
// The trailing `={0,2}` on each segment is what makes a padded token match. Base64url as the JWT spec defines it drops the `=` padding, but producers that reach for a plain base64 encoder emit it anyway, and a `=` in the header or payload segment used to defeat the match outright: not a partial redaction, but none at all, so the entire token was printed. `=` cannot appear anywhere except the end of a segment, since it is not one of the characters the segment body allows, so accepting it here cannot widen the match onto anything else.
|
|
6611
6846
|
["jwt", /eyJ[A-Za-z0-9_-]{10,}={0,2}\.[A-Za-z0-9_-]{10,}={0,2}\.[A-Za-z0-9_-]{10,}={0,2}/g],
|
|
6612
6847
|
["npm_token", /npm_[A-Za-z0-9]{36}/g],
|
|
6613
|
-
// rk_live_ (restricted keys) share the sk_live_/sk_test_ secret-key shape and risk level, so
|
|
6614
|
-
// one pattern covers all three rather than adding a near-duplicate entry.
|
|
6848
|
+
// rk_live_ (restricted keys) share the sk_live_/sk_test_ secret-key shape and risk level, so one pattern covers all three rather than adding a near-duplicate entry.
|
|
6615
6849
|
["stripe_key", /(?:sk_live_|sk_test_|rk_live_)[A-Za-z0-9]{20,}/g],
|
|
6616
6850
|
["google_api_key", /AIza[A-Za-z0-9_-]{35}/g],
|
|
6617
|
-
// Presigned-url signatures: AWS SigV4 (X-Amz-Signature), Google Cloud Storage
|
|
6618
|
-
// (X-Goog-Signature), and Azure blob SAS (sig). These are bearer credentials in query-string
|
|
6619
|
-
// clothing -- anyone holding the whole url can read or write the object until it expires, and
|
|
6620
|
-
// none of the prefix-anchored patterns above match them because the signature is a bare hex or
|
|
6621
|
-
// base64 blob with no distinctive prefix of its own. The '[?&]' anchor and the 16-char floor
|
|
6622
|
-
// are what make the short, generic 'sig' name safe to key on: a prose or code mention of "sig"
|
|
6623
|
-
// never sits directly after a query separator followed by that much opaque token.
|
|
6851
|
+
// Presigned-url signatures: AWS SigV4 (X-Amz-Signature), Google Cloud Storage (X-Goog-Signature), and Azure blob SAS (sig). These are bearer credentials in query-string clothing -- anyone holding the whole url can read or write the object until it expires, and none of the prefix-anchored patterns above match them because the signature is a bare hex or base64 blob with no distinctive prefix of its own. The '[?&]' anchor and the 16-char floor are what make the short, generic 'sig' name safe to key on: a prose or code mention of "sig" never sits directly after a query separator followed by that much opaque token.
|
|
6624
6852
|
["presigned_signature", /(?<=[?&](?:X-Amz-Signature|X-Goog-Signature|sig)=)[A-Za-z0-9%+/=_-]{16,}/gi],
|
|
6625
|
-
// A password inside a connection url. `postgres://user:hunter2@db.internal` carries the
|
|
6626
|
-
// credential in the authority section, where there is no `key=value` separator for the
|
|
6627
|
-
// generic pattern below to anchor on, so a DATABASE_URL echoed by a failing migration or a
|
|
6628
|
-
// psql error went through untouched. The anchors are what keep this narrow: a scheme's `://`,
|
|
6629
|
-
// a userinfo segment, the colon, and a following `@`. `http://host:8080/path` has no `@` and
|
|
6630
|
-
// is left alone; so is any url without credentials.
|
|
6853
|
+
// A password inside a connection url. `postgres://user:hunter2@db.internal` carries the credential in the authority section, where there is no `key=value` separator for the generic pattern below to anchor on, so a DATABASE_URL echoed by a failing migration or a psql error went through untouched. The anchors are what keep this narrow: a scheme's `://`, a userinfo segment, the colon, and a following `@`. `http://host:8080/path` has no `@` and is left alone; so is any url without credentials.
|
|
6631
6854
|
["url_credentials", /(?<=:\/\/[^\s:@/]{1,64}:)[^\s:@/]{1,256}(?=@)/g],
|
|
6632
|
-
//
|
|
6633
|
-
|
|
6634
|
-
// excludes whitespace, '&', ';', '#', quote characters, and '[' ']' ':' -- that exclusion is
|
|
6635
|
-
// what stops this from swallowing the rest of the line (a trailing comment or the next
|
|
6636
|
-
// key=value pair) or the remainder of a query string past the matched parameter, which is
|
|
6637
|
-
// exactly the kind of over-eager match this module's own design note above warns broad
|
|
6638
|
-
// heuristics produce. The '[' ']' ':' exclusion also matters because this pattern runs last: an
|
|
6639
|
-
// earlier pattern's own "OPENAI_API_KEY=[REDACTED:openai_project_key]" replacement text
|
|
6640
|
-
// contains "API_KEY=" too, and without excluding those characters this pattern would re-match
|
|
6641
|
-
// and double-redact its own placeholder. The length has a lower bound only (no upper bound):
|
|
6642
|
-
// capping it at 64 used to leave the tail of any longer secret unredacted in plain text, which
|
|
6643
|
-
// is worse than no redaction because it looks handled. A single negated-class quantifier like
|
|
6644
|
-
// this cannot backtrack catastrophically -- there is no nested or overlapping quantifier for
|
|
6645
|
-
// the engine to explore multiple ways of matching, so removing the upper bound does not
|
|
6646
|
-
// introduce a ReDoS risk.
|
|
6647
|
-
// Quotes are permitted around the separator, but never inside the value class. A quoted value
|
|
6648
|
-
// is the ordinary way secrets are written -- .env files, JSON, YAML, TOML all quote by default
|
|
6649
|
-
// -- and the lookbehind used to stop dead at the opening quote, so `API_KEY="..."` passed
|
|
6650
|
-
// through in full while the bare `API_KEY=...` was caught. The closing quote of the key name
|
|
6651
|
-
// blocked it from the other side too, which is what kept every JSON body unredacted. Keeping
|
|
6652
|
-
// quotes out of the value class is still what stops the match running past the closing quote.
|
|
6653
|
-
// The keyword may be a prefix of a longer key name rather than the whole of it, so the
|
|
6654
|
-
// trailing identifier class below is load-bearing: without it the lookbehind required the
|
|
6655
|
-
// keyword to sit immediately before the separator, and AWS_SECRET_ACCESS_KEY=,
|
|
6656
|
-
// SECRET_KEY=, and DB_PASSWORD_HASH= all passed through in full. That class matches
|
|
6657
|
-
// identifier characters only, so prose that merely mentions a keyword still never reaches
|
|
6658
|
-
// a separator and stays unredacted. api[_-]?key covers the apikey and api-key spellings too.
|
|
6659
|
-
// `& ; # , :` play two incompatible roles. They separate one field from the next (a query
|
|
6660
|
-
// string, a cookie header, an inline env list), and they are also perfectly ordinary credential
|
|
6661
|
-
// characters. Rejecting them outright got the first role right and the second badly wrong: the
|
|
6662
|
-
// match stopped at the first one and left everything after it in plain text, so
|
|
6663
|
-
// `password=corr&horse&battery` redacted four characters and printed the rest, and
|
|
6664
|
-
// `DB_PASSWORD=Aa1:xyz` matched nothing at all because the run before the `:` was under the
|
|
6665
|
-
// four-character floor. A tail left sitting in the open is the outcome this module's header
|
|
6666
|
-
// calls worse than no redaction, because it reads as handled.
|
|
6855
|
+
// Azure storage account connection strings (`DefaultEndpointsProtocol=https;AccountName=...; AccountKey=<base64>==;EndpointSuffix=core.windows.net`) carry a full read/write key to the account in the AccountKey field, and none of the generic patterns above catch it: generic_secret_assignment's keyword list (password|passwd|secret|api[_-]?key| access[_-]?token|refresh[_-]?token|id[_-]?token) has nothing that matches "AccountKey", and an unanchored base64-shape pattern was deliberately rejected -- this module's header already warns that a bare high-entropy-blob heuristic false-fires on ordinary code, JSON and log output, and an 88-char base64 value is exactly the shape a hash, a compiled asset digest or a generated id can also take. Anchoring on the literal `AccountKey=` field name instead keeps the match specific to this one connection-string field. The value class is base64 proper (letters, digits, `+`, `/`, trailing `=` padding) and none of those characters include `;`, so the match terminates on its own at the `;` that starts the next `Name=` field -- unlike generic_secret_assignment's separator characters (`& ; # , :`), which double as ordinary credential characters and need a lookahead to tell the two roles apart, `;` is never valid base64 and needs no such lookahead here. The lookbehind keeps `AccountKey=` itself in the output, matching auth_bearer_token and presigned_signature above, so `;EndpointSuffix=core.windows.net` after it stays fully readable too. `SharedAccessKey` is the same credential one Azure service over: Service Bus, Event Hubs and Relay spell it that way (`Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKeyName=Root; SharedAccessKey=<base64>`) and it carries the same authority over that namespace that AccountKey does over a storage account. `SharedAccessKeyName` is a plain identifier rather than a secret, and never matches: the separator in the lookbehind sits directly against the key name, so the `Name` in between stops it dead. The separator is spelled the way auth_bearer_token above spells its own, for the same reasons that comment records having learned the hard way. Optional quotes on either side of it, so the JSON and YAML forms a logged MCP result or api response actually arrives in are matched rather than stopped dead at the opening quote. A bounded gap rather than exactly one space, so a hand-aligned or reformatted `AccountKey = ...` in an appsettings file is not the single variant that defeats the whole pattern. Case-insensitive for the same reason presigned_signature is. The leading word boundary is what keeps the widened name from reaching into the middle of a longer identifier such as `myaccountkey=`.
|
|
6856
|
+
["azure_storage_key", /(?<=\b(?:AccountKey|SharedAccessKey)["']?[ \t]{0,8}[:=][ \t]{0,8}["']?)[A-Za-z0-9+/]{40,}=*/gi],
|
|
6857
|
+
// Generic key=value assignments in .env-file and connection-string/query-string shape. The lookbehind again redacts only the value, and the value's character class deliberately excludes whitespace, '&', ';', '#', quote characters, and '[' ']' ':' -- that exclusion is what stops this from swallowing the rest of the line (a trailing comment or the next key=value pair) or the remainder of a query string past the matched parameter, which is exactly the kind of over-eager match this module's own design note above warns broad heuristics produce. The '[' ']' ':' exclusion also matters because this pattern runs last: an earlier pattern's own "OPENAI_API_KEY=[REDACTED:openai_project_key]" replacement text contains "API_KEY=" too, and without excluding those characters this pattern would re-match and double-redact its own placeholder. The length has a lower bound only (no upper bound): capping it at 64 used to leave the tail of any longer secret unredacted in plain text, which is worse than no redaction because it looks handled. A single negated-class quantifier like this cannot backtrack catastrophically -- there is no nested or overlapping quantifier for the engine to explore multiple ways of matching, so removing the upper bound does not introduce a ReDoS risk. Quotes are permitted around the separator, but never inside the value class. A quoted value is the ordinary way secrets are written -- .env files, JSON, YAML, TOML all quote by default -- and the lookbehind used to stop dead at the opening quote, so `API_KEY="..."` passed through in full while the bare `API_KEY=...` was caught. The closing quote of the key name blocked it from the other side too, which is what kept every JSON body unredacted. Keeping quotes out of the value class is still what stops the match running past the closing quote. The keyword may be a prefix of a longer key name rather than the whole of it, so the trailing identifier class below is load-bearing: without it the lookbehind required the keyword to sit immediately before the separator, and AWS_SECRET_ACCESS_KEY=, SECRET_KEY=, and DB_PASSWORD_HASH= all passed through in full. That class matches identifier characters only, so prose that merely mentions a keyword still never reaches a separator and stays unredacted. api[_-]?key covers the apikey and api-key spellings too. `& ; # , :` play two incompatible roles. They separate one field from the next (a query string, a cookie header, an inline env list), and they are also perfectly ordinary credential characters. Rejecting them outright got the first role right and the second badly wrong: the match stopped at the first one and left everything after it in plain text, so `password=corr&horse&battery` redacted four characters and printed the rest, and `DB_PASSWORD=Aa1:xyz` matched nothing at all because the run before the `:` was under the four-character floor. A tail left sitting in the open is the outcome this module's header calls worse than no redaction, because it reads as handled.
|
|
6667
6858
|
//
|
|
6668
|
-
// So the separator role is decided by what follows rather than assumed: one of these characters
|
|
6669
|
-
// ends the value only when the next thing along is another `name=` / `name:` pair, which is what
|
|
6670
|
-
// an actual field separator is always followed by. `,OTHER=public` and `; other=1` still end it;
|
|
6671
|
-
// the `&` in the middle of a passphrase does not. Whitespace, quotes and brackets are unchanged
|
|
6672
|
-
// -- they end a value unconditionally, which is also what keeps this pattern from re-matching
|
|
6673
|
-
// the `[REDACTED:...]` placeholder it just wrote.
|
|
6859
|
+
// So the separator role is decided by what follows rather than assumed: one of these characters ends the value only when the next thing along is another `name=` / `name:` pair, which is what an actual field separator is always followed by. `,OTHER=public` and `; other=1` still end it; the `&` in the middle of a passphrase does not. Whitespace, quotes and brackets are unchanged -- they end a value unconditionally, which is also what keeps this pattern from re-matching the `[REDACTED:...]` placeholder it just wrote.
|
|
6674
6860
|
["generic_secret_assignment", /(?<=(?:password|passwd|secret|api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token)[a-z0-9_-]{0,64}["']?[ \t]{0,8}[:=][ \t]{0,8}["']?)(?:\\[^\n]|[^\s\\&;#,:'"[\]{}]|[&;#,:](?![ \t]*[A-Za-z_][A-Za-z0-9_.-]*[ \t]*[:=])){4,}/gi]
|
|
6675
6861
|
];
|
|
6862
|
+
function countRedactionPlaceholders(text) {
|
|
6863
|
+
return text.match(/\[REDACTED:[a-z0-9_]+\]/g)?.length ?? 0;
|
|
6864
|
+
}
|
|
6676
6865
|
function redactSecrets(text) {
|
|
6677
6866
|
let count = 0;
|
|
6678
6867
|
let out = text;
|
|
@@ -6801,6 +6990,7 @@ export {
|
|
|
6801
6990
|
toKB,
|
|
6802
6991
|
compileGrepMatcher,
|
|
6803
6992
|
grepFilteredToEmptyNotice,
|
|
6993
|
+
filtersFilteredToEmptyNotice,
|
|
6804
6994
|
countNoun,
|
|
6805
6995
|
excludeTestsHiddenNote,
|
|
6806
6996
|
countContentLines,
|
|
@@ -6888,6 +7078,7 @@ export {
|
|
|
6888
7078
|
fg,
|
|
6889
7079
|
C,
|
|
6890
7080
|
SOURCE_HINT,
|
|
7081
|
+
savedTokensFromBytes,
|
|
6891
7082
|
formatLocalTimestamp,
|
|
6892
7083
|
recordStat,
|
|
6893
7084
|
recordUnmappedTool,
|
|
@@ -6896,6 +7087,7 @@ export {
|
|
|
6896
7087
|
_useRichStats,
|
|
6897
7088
|
renderShortStats,
|
|
6898
7089
|
renderStats2 as renderStats,
|
|
7090
|
+
countRedactionPlaceholders,
|
|
6899
7091
|
redactSecrets,
|
|
6900
7092
|
FILTERS,
|
|
6901
7093
|
stripAnsiCodes
|