token-goat 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -9
- package/dist/{token-goat-chunk-VBXBLGTO.mjs → token-goat-chunk-222VPFP2.mjs} +147 -53
- package/dist/{token-goat-chunk-XEDQH5DA.mjs → token-goat-chunk-4HIMCBYK.mjs} +6 -6
- package/dist/{token-goat-chunk-Y2SYNH3S.mjs → token-goat-chunk-4OM2Q2SX.mjs} +5 -5
- package/dist/{token-goat-chunk-ELDJRLHZ.mjs → token-goat-chunk-6ODZ6PZK.mjs} +219 -147
- package/dist/{token-goat-chunk-73MF6YWW.mjs → token-goat-chunk-C5IL6MJH.mjs} +3 -3
- package/dist/{token-goat-chunk-YUGNM3KL.mjs → token-goat-chunk-DP3NNGGV.mjs} +5 -5
- package/dist/{token-goat-chunk-7JDXDERZ.mjs → token-goat-chunk-EFF2XCLB.mjs} +31 -18
- package/dist/{token-goat-chunk-AOBZUFNJ.mjs → token-goat-chunk-NKNCHJ4H.mjs} +80 -49
- package/dist/{token-goat-chunk-2G6RAB4G.mjs → token-goat-chunk-PWVXXPCC.mjs} +136 -31
- package/dist/{token-goat-chunk-RMDQFTQD.mjs → token-goat-chunk-TX4JFJTD.mjs} +76 -19
- package/dist/{token-goat-chunk-I6TOUPLP.mjs → token-goat-chunk-VAGPOZHO.mjs} +2 -2
- package/dist/token-goat-hook.mjs +5 -5
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +1 -1
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
buildLineIndex,
|
|
14
14
|
countContentLines,
|
|
15
15
|
countNoun,
|
|
16
|
+
countRedactionPlaceholders,
|
|
16
17
|
dataDir,
|
|
17
18
|
decodeSource,
|
|
18
19
|
detectHarness,
|
|
@@ -91,7 +92,7 @@ import {
|
|
|
91
92
|
withFileLock,
|
|
92
93
|
writeIfDifferent,
|
|
93
94
|
writeJsonSettings
|
|
94
|
-
} from "./token-goat-chunk-
|
|
95
|
+
} from "./token-goat-chunk-6ODZ6PZK.mjs";
|
|
95
96
|
import {
|
|
96
97
|
registerReset
|
|
97
98
|
} from "./token-goat-chunk-AO2QD2AG.mjs";
|
|
@@ -325,12 +326,81 @@ function parseXml(xml) {
|
|
|
325
326
|
return Object.fromEntries(root.children);
|
|
326
327
|
}
|
|
327
328
|
|
|
329
|
+
// src/zip_bounds.ts
|
|
330
|
+
var MAX_ZIP_INPUT_BYTES = 50 * 1024 * 1024;
|
|
331
|
+
var MAX_ZIP_OUTPUT_BYTES = 500 * 1024 * 1024;
|
|
332
|
+
var STREAM_CHUNK_BYTES = 64 * 1024;
|
|
333
|
+
var ZipOutputTooLargeError = class extends Error {
|
|
334
|
+
constructor(entryName, limitBytes, decompressedSoFarBytes) {
|
|
335
|
+
super(
|
|
336
|
+
`zip entry '${entryName}' is over the ${Math.round(limitBytes / (1024 * 1024))}MB decompressed-size limit (over ${Math.round(decompressedSoFarBytes / (1024 * 1024))}MB decompressed so far)`
|
|
337
|
+
);
|
|
338
|
+
this.name = "ZipOutputTooLargeError";
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
var ZipInputTooLargeError = class extends Error {
|
|
342
|
+
constructor(filePath, sizeBytes, limitBytes) {
|
|
343
|
+
super(`${filePath} is ${Math.round(sizeBytes / (1024 * 1024))}MB, over the ${Math.round(limitBytes / (1024 * 1024))}MB limit for zip-format archives`);
|
|
344
|
+
this.name = "ZipInputTooLargeError";
|
|
345
|
+
}
|
|
346
|
+
};
|
|
347
|
+
function concatChunks(chunks, total) {
|
|
348
|
+
const out = new Uint8Array(total);
|
|
349
|
+
let offset = 0;
|
|
350
|
+
for (const chunk of chunks) {
|
|
351
|
+
out.set(chunk, offset);
|
|
352
|
+
offset += chunk.length;
|
|
353
|
+
}
|
|
354
|
+
return out;
|
|
355
|
+
}
|
|
356
|
+
function unzipBounded(mod, data, opts) {
|
|
357
|
+
mod.unzipSync(data, { filter: () => false });
|
|
358
|
+
const results = {};
|
|
359
|
+
let firstError;
|
|
360
|
+
let totalDecompressed = 0;
|
|
361
|
+
const unzip = new mod.Unzip((file) => {
|
|
362
|
+
if (firstError !== void 0 || !opts.shouldExtract(file.name)) return;
|
|
363
|
+
if (typeof file.originalSize === "number" && totalDecompressed + file.originalSize > opts.limitBytes) {
|
|
364
|
+
firstError = new ZipOutputTooLargeError(file.name, opts.limitBytes, totalDecompressed + file.originalSize);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
const chunks = [];
|
|
368
|
+
let entryTotal = 0;
|
|
369
|
+
file.ondata = (err, chunk, final) => {
|
|
370
|
+
if (firstError !== void 0) return;
|
|
371
|
+
if (err) {
|
|
372
|
+
firstError = err instanceof Error ? err : new Error(String(err));
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
entryTotal += chunk.length;
|
|
376
|
+
totalDecompressed += chunk.length;
|
|
377
|
+
if (totalDecompressed > opts.limitBytes) {
|
|
378
|
+
firstError = new ZipOutputTooLargeError(file.name, opts.limitBytes, totalDecompressed);
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
chunks.push(chunk);
|
|
382
|
+
if (final) results[file.name] = concatChunks(chunks, entryTotal);
|
|
383
|
+
};
|
|
384
|
+
file.start();
|
|
385
|
+
});
|
|
386
|
+
unzip.register(mod.UnzipInflate);
|
|
387
|
+
let offset = 0;
|
|
388
|
+
for (; ; ) {
|
|
389
|
+
const end = Math.min(offset + STREAM_CHUNK_BYTES, data.length);
|
|
390
|
+
const isFinal = end >= data.length;
|
|
391
|
+
unzip.push(data.subarray(offset, end), isFinal);
|
|
392
|
+
offset = end;
|
|
393
|
+
if (firstError !== void 0 || isFinal) break;
|
|
394
|
+
}
|
|
395
|
+
if (firstError !== void 0) throw firstError;
|
|
396
|
+
return results;
|
|
397
|
+
}
|
|
398
|
+
|
|
328
399
|
// src/ooxml_extract.ts
|
|
329
400
|
var loadFflate = createLazyModuleLoader(
|
|
330
401
|
async () => await import("fflate"),
|
|
331
402
|
"office-file reading disabled (fflate unavailable)"
|
|
332
403
|
);
|
|
333
|
-
var MAX_OOXML_INPUT_BYTES = 50 * 1024 * 1024;
|
|
334
404
|
function accessFailureMessage(err, filePath) {
|
|
335
405
|
const code = err?.code;
|
|
336
406
|
if (code === "ENOENT") return `File not found: ${filePath}`;
|
|
@@ -346,8 +416,8 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
346
416
|
throw new Error(accessFailureMessage(err, filePath), { cause: err });
|
|
347
417
|
}
|
|
348
418
|
if (!stat2.isFile()) throw new Error(`not a valid ${kind} file: ${filePath}`);
|
|
349
|
-
if (stat2.size >
|
|
350
|
-
throw new Error(`${filePath} is ${Math.round(stat2.size / (1024 * 1024))}MB, over the ${
|
|
419
|
+
if (stat2.size > MAX_ZIP_INPUT_BYTES) {
|
|
420
|
+
throw new Error(`${filePath} is ${Math.round(stat2.size / (1024 * 1024))}MB, over the ${MAX_ZIP_INPUT_BYTES / (1024 * 1024)}MB limit for OOXML files`);
|
|
351
421
|
}
|
|
352
422
|
let data;
|
|
353
423
|
try {
|
|
@@ -356,8 +426,9 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
356
426
|
throw new Error(accessFailureMessage(err, filePath), { cause: err });
|
|
357
427
|
}
|
|
358
428
|
try {
|
|
359
|
-
return fflate
|
|
429
|
+
return unzipBounded(fflate, new Uint8Array(data), { limitBytes: MAX_ZIP_OUTPUT_BYTES, shouldExtract: () => true });
|
|
360
430
|
} catch (err) {
|
|
431
|
+
if (err instanceof ZipOutputTooLargeError) throw err;
|
|
361
432
|
throw new Error(`not a valid ${kind} file: ${filePath}`, { cause: err });
|
|
362
433
|
}
|
|
363
434
|
}
|
|
@@ -3408,6 +3479,15 @@ function denyOutput(message) {
|
|
|
3408
3479
|
function contextOutput(context) {
|
|
3409
3480
|
return { hookType: "context", context };
|
|
3410
3481
|
}
|
|
3482
|
+
function emitRewrite(updatedOutput, detail, savings) {
|
|
3483
|
+
const count = countRedactionPlaceholders(updatedOutput);
|
|
3484
|
+
if (count > 0) recordStat("secret_redacted", 0, count, void 0, detail);
|
|
3485
|
+
if (savings !== void 0) {
|
|
3486
|
+
const bytesSaved = savings.originalBytes - Buffer.byteLength(updatedOutput, "utf-8");
|
|
3487
|
+
if (bytesSaved > 0) recordStat(savings.kind, bytesSaved, Math.round(bytesSaved / 4));
|
|
3488
|
+
}
|
|
3489
|
+
return { hookType: "rewriteOutput", updatedOutput };
|
|
3490
|
+
}
|
|
3411
3491
|
function countNonEmptyLines(text) {
|
|
3412
3492
|
return text.split(/\r\n|\r|\n/).filter((line) => line.length > 0).length;
|
|
3413
3493
|
}
|
|
@@ -3735,8 +3815,7 @@ function getHintStatsTotals() {
|
|
|
3735
3815
|
return {
|
|
3736
3816
|
savedBytes,
|
|
3737
3817
|
spentBytes,
|
|
3738
|
-
legacyEmissions
|
|
3739
|
-
netBytes: spentBytes === null ? null : savedBytes - spentBytes
|
|
3818
|
+
legacyEmissions
|
|
3740
3819
|
};
|
|
3741
3820
|
}
|
|
3742
3821
|
function resetHintStats() {
|
|
@@ -6855,6 +6934,7 @@ ${neutralizeFenceMarkers(text, UNTRUSTED_FILE_TAG)}
|
|
|
6855
6934
|
</${UNTRUSTED_FILE_TAG}>`;
|
|
6856
6935
|
}
|
|
6857
6936
|
var UNTRUSTED_TOOL_TAG = "untrusted-tool-output";
|
|
6937
|
+
var UNTRUSTED_GITHUB_TAG = "untrusted-github-content";
|
|
6858
6938
|
|
|
6859
6939
|
// src/skill_cache.ts
|
|
6860
6940
|
import * as fs14 from "fs/promises";
|
|
@@ -7045,6 +7125,21 @@ async function hasSessionOutput(sessionId, skillName) {
|
|
|
7045
7125
|
return false;
|
|
7046
7126
|
}
|
|
7047
7127
|
}
|
|
7128
|
+
async function sessionOutputBodyBytes(sessionId, skillName) {
|
|
7129
|
+
try {
|
|
7130
|
+
if (!sessionId) return null;
|
|
7131
|
+
const name = safeSkillName(skillName);
|
|
7132
|
+
if (!name) return null;
|
|
7133
|
+
const safeSession = safeSessionFragment(sessionId);
|
|
7134
|
+
const metas = await listOutputs();
|
|
7135
|
+
const matches = metas.filter((m) => m.skillName === name && m.outputId.startsWith(`${safeSession}-`));
|
|
7136
|
+
if (matches.length === 0) return null;
|
|
7137
|
+
matches.sort((a, b) => b.ts - a.ts);
|
|
7138
|
+
return matches[0].bodyBytes;
|
|
7139
|
+
} catch {
|
|
7140
|
+
return null;
|
|
7141
|
+
}
|
|
7142
|
+
}
|
|
7048
7143
|
async function findCrossSessionEntry(skillName, contentSha) {
|
|
7049
7144
|
const name = safeSkillName(skillName);
|
|
7050
7145
|
if (!name || !contentSha) return null;
|
|
@@ -7085,9 +7180,11 @@ async function storeOutput(sessionId, skillName, body, opts) {
|
|
|
7085
7180
|
const ts = Date.now();
|
|
7086
7181
|
const bodyBytes = Buffer.byteLength(body, "utf-8");
|
|
7087
7182
|
const truncated = bodyBytes > 256 * 1024;
|
|
7088
|
-
|
|
7183
|
+
const redactedBody = redactSecrets(body);
|
|
7184
|
+
if (redactedBody.count > 0) recordStat("secret_redacted", 0, redactedBody.count, void 0, SKILLS_OUTPUT_SUBDIR);
|
|
7185
|
+
let storedBody = redactedBody.text;
|
|
7089
7186
|
if (truncated) {
|
|
7090
|
-
const buf = Buffer.from(
|
|
7187
|
+
const buf = Buffer.from(redactedBody.text, "utf-8");
|
|
7091
7188
|
let truncStart = Math.max(0, buf.length - 262144);
|
|
7092
7189
|
if (truncStart < buf.length) {
|
|
7093
7190
|
const byte = buf[truncStart];
|
|
@@ -7135,7 +7232,9 @@ async function storeCompact(sessionId, skillName, compactText, sourceSha) {
|
|
|
7135
7232
|
const safeSession = safeSessionFragment(sessionId);
|
|
7136
7233
|
const fileId = `${safeSession}@${sanitizeSkillId(name)}@compact`;
|
|
7137
7234
|
const dir = skillOutputsDir();
|
|
7138
|
-
|
|
7235
|
+
const redacted = redactSecrets(compactText);
|
|
7236
|
+
if (redacted.count > 0) recordStat("secret_redacted", 0, redacted.count, void 0, SKILLS_OUTPUT_SUBDIR);
|
|
7237
|
+
let text = redacted.text;
|
|
7139
7238
|
if (sourceSha) {
|
|
7140
7239
|
text = `<!-- source_sha: ${sourceSha.slice(0, 12)} -->
|
|
7141
7240
|
${text}`;
|
|
@@ -7587,7 +7686,9 @@ async function probeImageMeta(input) {
|
|
|
7587
7686
|
const sharp = await loadSharp();
|
|
7588
7687
|
if (sharp === null) return null;
|
|
7589
7688
|
try {
|
|
7590
|
-
const
|
|
7689
|
+
const cfg = loadConfig().image_shrink;
|
|
7690
|
+
const limitInputPixels = cfg.max_image_pixels > 0 ? cfg.max_image_pixels : false;
|
|
7691
|
+
const meta = await sharp(input, { limitInputPixels }).metadata();
|
|
7591
7692
|
return { width: meta.width ?? 0, height: meta.height ?? 0, format: meta.format ?? null, pages: meta.pages ?? 1 };
|
|
7592
7693
|
} catch (e) {
|
|
7593
7694
|
throw new ImageDecodeError(e?.message ?? "image could not be decoded");
|
|
@@ -9922,7 +10023,7 @@ function extractMarkdownHeadings(content, limit = MAX_HEADINGS) {
|
|
|
9922
10023
|
const lines2 = content.split("\n");
|
|
9923
10024
|
for (const [i, line] of eachUnfencedLine(lines2)) {
|
|
9924
10025
|
if (!line) continue;
|
|
9925
|
-
const match = /^(#+)\s+(
|
|
10026
|
+
const match = /^(#+)\s+([^\r\n]+?)(?:\s+#+)?\s*$/.exec(line);
|
|
9926
10027
|
if (!match || match.length < 3) continue;
|
|
9927
10028
|
const hashes = match[1];
|
|
9928
10029
|
const headingText = match[2];
|
|
@@ -10052,7 +10153,7 @@ function handleHtml(filePath, content, contentLengthHint) {
|
|
|
10052
10153
|
message: `Large HTML file (${formatBytes(length)}) \u2014 too large to preview (exceeds the in-hook scan cap). Use token-goat section to extract a section by heading, or convert to text: pandoc "${filePath}" -t plain`
|
|
10053
10154
|
};
|
|
10054
10155
|
}
|
|
10055
|
-
const title = content.match(/<title[^>]*>([^<]
|
|
10156
|
+
const title = content.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.trim();
|
|
10056
10157
|
const headings = findHtmlHeadingMatches(content).slice(0, 20).map(({ level, heading }) => {
|
|
10057
10158
|
if (!heading) return "";
|
|
10058
10159
|
return `${" ".repeat(level - 1)}h${level}: ${heading}`;
|
|
@@ -11229,7 +11330,7 @@ function preReadHandlerInner(event) {
|
|
|
11229
11330
|
const isSourceExt = isSourceExtension(basename12);
|
|
11230
11331
|
if (isSourceExt && reads >= 2) {
|
|
11231
11332
|
recordStat("read_count_deny", rereadCredit, Math.round(rereadCredit / 4));
|
|
11232
|
-
recordStat("session_hint",
|
|
11333
|
+
recordStat("session_hint", 0, 0);
|
|
11233
11334
|
return denyOutput(
|
|
11234
11335
|
"Read this file " + reads + ' times already \u2014 use `token-goat read "' + shown + '::Symbol"`, `token-goat skeleton ' + shown + "`, or `token-goat outline " + shown + "` to pull just the part you need. " + editAnywayHint(normalized)
|
|
11235
11336
|
);
|
|
@@ -12106,13 +12207,8 @@ function mapLookupBytesSaved(map, emittedText) {
|
|
|
12106
12207
|
...map.recentFiles.map((f) => normalizePath(path22.resolve(map.rootDir, f))),
|
|
12107
12208
|
...map.topSymbols.map((s) => normalizePath(s.filePath))
|
|
12108
12209
|
]);
|
|
12109
|
-
|
|
12110
|
-
|
|
12111
|
-
try {
|
|
12112
|
-
fullSourceBytes += fs26.statSync(fp).size;
|
|
12113
|
-
} catch {
|
|
12114
|
-
}
|
|
12115
|
-
}
|
|
12210
|
+
const listingText = Array.from(referencedFiles).sort().join("\n");
|
|
12211
|
+
const fullSourceBytes = Buffer.byteLength(listingText, "utf8");
|
|
12116
12212
|
const emittedBytes = Buffer.byteLength(emittedText, "utf8");
|
|
12117
12213
|
return Math.max(1, fullSourceBytes - emittedBytes);
|
|
12118
12214
|
}
|
|
@@ -13581,7 +13677,7 @@ function extractDart(content, filePath) {
|
|
|
13581
13677
|
const line = stripLineComment(blockStripped).trimEnd();
|
|
13582
13678
|
const stripped = line.trim();
|
|
13583
13679
|
if (!stripped) {
|
|
13584
|
-
const braceLine2 = stripStringLiterals(line);
|
|
13680
|
+
const braceLine2 = stripStringLiterals(line, { tripleQuotes: true });
|
|
13585
13681
|
braceDepth += (braceLine2.match(/\{/g) ?? []).length - (braceLine2.match(/\}/g) ?? []).length;
|
|
13586
13682
|
continue;
|
|
13587
13683
|
}
|
|
@@ -13675,7 +13771,7 @@ function extractDart(content, filePath) {
|
|
|
13675
13771
|
symbols.push(makeLineSymbol(filePath, fname, "function", lineNum, stripped.slice(0, 200), void 0, lines2, "c"));
|
|
13676
13772
|
}
|
|
13677
13773
|
}
|
|
13678
|
-
const braceLine = stripStringLiterals(line);
|
|
13774
|
+
const braceLine = stripStringLiterals(line, { tripleQuotes: true });
|
|
13679
13775
|
for (const ch of braceLine) {
|
|
13680
13776
|
if (ch === "{") {
|
|
13681
13777
|
braceDepth++;
|
|
@@ -17572,8 +17668,8 @@ var NO_TREE_SITTER_EXTRACTORS = {
|
|
|
17572
17668
|
toml: extractTomlSymbols,
|
|
17573
17669
|
css: extractCssSymbols,
|
|
17574
17670
|
dockerfile: extractDockerfileSymbols,
|
|
17575
|
-
csharp: (content, filePath) => assignBraceBlockSpans(extractCsharp(content, filePath).symbols, content, "//"),
|
|
17576
|
-
php: (content, filePath) => assignBraceBlockSpans(extractPhp(content, filePath).symbols, content, "//"),
|
|
17671
|
+
csharp: (content, filePath) => assignBraceBlockSpans(extractCsharp(content, filePath).symbols, content, { lineComment: "//", stringEscapes: "csharp", rawStringQuotes: true }),
|
|
17672
|
+
php: (content, filePath) => assignBraceBlockSpans(extractPhp(content, filePath).symbols, content, { lineComment: ["//", "#"], lineCommentExceptions: ["#["] }),
|
|
17577
17673
|
html: (content, filePath) => {
|
|
17578
17674
|
const r = extractHtml(content, filePath);
|
|
17579
17675
|
return [...r.symbols, ...sectionsToHeadingSymbols(r.sections, filePath)];
|
|
@@ -17582,13 +17678,13 @@ var NO_TREE_SITTER_EXTRACTORS = {
|
|
|
17582
17678
|
const r = extractLiquid(content, filePath);
|
|
17583
17679
|
return [...r.symbols, ...sectionsToHeadingSymbols(r.sections, filePath)];
|
|
17584
17680
|
},
|
|
17585
|
-
kotlin: (content, filePath) => assignBraceBlockSpans(extractKotlin(content, filePath).symbols, content, "//"),
|
|
17586
|
-
swift: (content, filePath) => assignBraceBlockSpans(extractSwift(content, filePath).symbols, content, "//"),
|
|
17587
|
-
scala: (content, filePath) => assignBraceBlockSpans(extractScala(content, filePath).symbols, content, "//"),
|
|
17681
|
+
kotlin: (content, filePath) => assignBraceBlockSpans(extractKotlin(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true }),
|
|
17682
|
+
swift: (content, filePath) => assignBraceBlockSpans(extractSwift(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true }),
|
|
17683
|
+
scala: (content, filePath) => assignBraceBlockSpans(extractScala(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true }),
|
|
17588
17684
|
lua: (content, filePath) => extractLua(content, filePath).symbols,
|
|
17589
17685
|
elixir: (content, filePath) => extractElixir(content, filePath).symbols,
|
|
17590
|
-
dart: (content, filePath) => assignBraceBlockSpans(extractDart(content, filePath).symbols, content, "//"),
|
|
17591
|
-
zig: (content, filePath) => assignBraceBlockSpans(extractZig(content, filePath).symbols, content, "//"),
|
|
17686
|
+
dart: (content, filePath) => assignBraceBlockSpans(extractDart(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true, tripleSingleQuote: true }),
|
|
17687
|
+
zig: (content, filePath) => assignBraceBlockSpans(extractZig(content, filePath).symbols, content, { lineComment: "//", blockComment: null, lineStringPrefix: "\\\\" }),
|
|
17592
17688
|
r: (content, filePath) => extractR(content, filePath).symbols,
|
|
17593
17689
|
graphql: (content, filePath) => extractGraphql(content, filePath).symbols,
|
|
17594
17690
|
sql: extractSql,
|
|
@@ -17596,7 +17692,7 @@ var NO_TREE_SITTER_EXTRACTORS = {
|
|
|
17596
17692
|
makefile: extractMakefile,
|
|
17597
17693
|
proto: (content, filePath) => extractProto(content, filePath).symbols,
|
|
17598
17694
|
terraform: extractTerraform,
|
|
17599
|
-
powershell: (content, filePath) => assignBraceBlockSpans(extractPowershell(content, filePath).symbols, content, "#"),
|
|
17695
|
+
powershell: (content, filePath) => assignBraceBlockSpans(extractPowershell(content, filePath).symbols, content, { lineComment: "#", stringEscapes: "powershell" }),
|
|
17600
17696
|
apex: (content, filePath) => extractApex(content, filePath).symbols,
|
|
17601
17697
|
salesforce_metadata: (content, filePath) => extractSalesforceMetadata(content, filePath).symbols,
|
|
17602
17698
|
env_file: extractEnv,
|
|
@@ -18945,6 +19041,11 @@ export {
|
|
|
18945
19041
|
locatePdfPages,
|
|
18946
19042
|
extractPdfOutline,
|
|
18947
19043
|
extractPdfMeta,
|
|
19044
|
+
MAX_ZIP_INPUT_BYTES,
|
|
19045
|
+
MAX_ZIP_OUTPUT_BYTES,
|
|
19046
|
+
ZipOutputTooLargeError,
|
|
19047
|
+
ZipInputTooLargeError,
|
|
19048
|
+
unzipBounded,
|
|
18948
19049
|
docxOutline,
|
|
18949
19050
|
docxText,
|
|
18950
19051
|
pptxOutline,
|
|
@@ -18979,6 +19080,7 @@ export {
|
|
|
18979
19080
|
passOutput,
|
|
18980
19081
|
denyOutput,
|
|
18981
19082
|
contextOutput,
|
|
19083
|
+
emitRewrite,
|
|
18982
19084
|
makeDedupHintHandlers,
|
|
18983
19085
|
registerHook,
|
|
18984
19086
|
runHook,
|
|
@@ -19092,7 +19194,9 @@ export {
|
|
|
19092
19194
|
scanForInjectionPatterns,
|
|
19093
19195
|
UNTRUSTED_WEB_TAG,
|
|
19094
19196
|
fenceUntrustedContent,
|
|
19197
|
+
UNTRUSTED_FILE_TAG,
|
|
19095
19198
|
UNTRUSTED_TOOL_TAG,
|
|
19199
|
+
UNTRUSTED_GITHUB_TAG,
|
|
19096
19200
|
SKILLS_OUTPUT_SUBDIR,
|
|
19097
19201
|
skillOutputsDir,
|
|
19098
19202
|
contentHash,
|
|
@@ -19101,6 +19205,7 @@ export {
|
|
|
19101
19205
|
extractChecklistSection,
|
|
19102
19206
|
listOutputs,
|
|
19103
19207
|
hasSessionOutput,
|
|
19208
|
+
sessionOutputBodyBytes,
|
|
19104
19209
|
storeOutput,
|
|
19105
19210
|
storeCompact,
|
|
19106
19211
|
incrementSkillHit,
|
|
@@ -4,8 +4,13 @@ import {
|
|
|
4
4
|
IMPORT_RE,
|
|
5
5
|
ImageDecodeError,
|
|
6
6
|
MAX_OVER_FETCH,
|
|
7
|
+
MAX_ZIP_INPUT_BYTES,
|
|
8
|
+
MAX_ZIP_OUTPUT_BYTES,
|
|
7
9
|
OVER_FETCH_FACTOR,
|
|
8
10
|
SKIP_DIRS,
|
|
11
|
+
UNTRUSTED_GITHUB_TAG,
|
|
12
|
+
ZipInputTooLargeError,
|
|
13
|
+
ZipOutputTooLargeError,
|
|
9
14
|
capJsonRows,
|
|
10
15
|
countRefs,
|
|
11
16
|
countSymbols,
|
|
@@ -18,6 +23,7 @@ import {
|
|
|
18
23
|
extractPdfMeta,
|
|
19
24
|
extractPdfOutline,
|
|
20
25
|
extractPdfText,
|
|
26
|
+
fenceUntrustedContent,
|
|
21
27
|
formatCsvProfile,
|
|
22
28
|
formatCsvTable,
|
|
23
29
|
getFileEntry,
|
|
@@ -42,6 +48,7 @@ import {
|
|
|
42
48
|
queryRefs,
|
|
43
49
|
queryRefsByContext,
|
|
44
50
|
querySymbols,
|
|
51
|
+
scanForInjectionPatterns,
|
|
45
52
|
searchEvidenceSemantically,
|
|
46
53
|
searchSemantic,
|
|
47
54
|
searchSymbolsFts,
|
|
@@ -50,11 +57,12 @@ import {
|
|
|
50
57
|
stripLeadingAttributes,
|
|
51
58
|
tomlBracketDelta,
|
|
52
59
|
trimToBudget,
|
|
60
|
+
unzipBounded,
|
|
53
61
|
urlPolicyDenialReason,
|
|
54
62
|
walkProject,
|
|
55
63
|
yamlLineClosesQuote,
|
|
56
64
|
yamlOpenQuoteAfter
|
|
57
|
-
} from "./token-goat-chunk-
|
|
65
|
+
} from "./token-goat-chunk-PWVXXPCC.mjs";
|
|
58
66
|
import {
|
|
59
67
|
Database,
|
|
60
68
|
PER_FILE_COUNTERFACTUAL_CEILING,
|
|
@@ -106,7 +114,7 @@ import {
|
|
|
106
114
|
unsupportedLanguageName,
|
|
107
115
|
windowsCmdQuoteArg,
|
|
108
116
|
withExtension
|
|
109
|
-
} from "./token-goat-chunk-
|
|
117
|
+
} from "./token-goat-chunk-6ODZ6PZK.mjs";
|
|
110
118
|
import {
|
|
111
119
|
registerReset
|
|
112
120
|
} from "./token-goat-chunk-AO2QD2AG.mjs";
|
|
@@ -139,7 +147,7 @@ function getTrackedFiles(cwd = process.cwd()) {
|
|
|
139
147
|
// src/section_reader.ts
|
|
140
148
|
import { readFileSync } from "node:fs";
|
|
141
149
|
function parseHeadingSpec(spec, headers) {
|
|
142
|
-
const m = /^(
|
|
150
|
+
const m = /^([^#\r\n]+)#(\d+)$/.exec(spec);
|
|
143
151
|
if (m !== null && m[1] !== void 0 && m[2] !== void 0) {
|
|
144
152
|
const specLower = spec.trim().toLowerCase();
|
|
145
153
|
const isLiteralHeading = headers?.some((h) => h.heading.trim().toLowerCase() === specLower) ?? false;
|
|
@@ -163,7 +171,7 @@ function normalizeHeadingStrip(s) {
|
|
|
163
171
|
return n.replace(/\s+/g, " ").trim();
|
|
164
172
|
}
|
|
165
173
|
var MIN_WIDEN_WORD_LEN = 3;
|
|
166
|
-
var MARKDOWN_HEADER_RE = /^(#{1,6})\s+(
|
|
174
|
+
var MARKDOWN_HEADER_RE = /^(#{1,6})\s+([^\r\n]+?)(?:\s+#+)?\s*$/;
|
|
167
175
|
var TABLE_HEADER_RE = /^\s*\[+\s*([^\]]+?)\s*\]+\s*(?:[#;].*)?$/;
|
|
168
176
|
var PYTHON_HEADER_RE = /^(\s*)(?:async\s+)?(?:def|class)\s+([A-Za-z_]\w*)/;
|
|
169
177
|
var KEYVALUE_HEADER_RE = /^([A-Za-z_][\w.-]*)\s*(?:=|:(?!\/\/))/;
|
|
@@ -3612,7 +3620,7 @@ function formatZipList(entries) {
|
|
|
3612
3620
|
}
|
|
3613
3621
|
async function extractZipEntry(data, entryPath) {
|
|
3614
3622
|
const fflate = await requireFflate();
|
|
3615
|
-
const result = fflate
|
|
3623
|
+
const result = unzipBounded(fflate, data, { limitBytes: MAX_ZIP_OUTPUT_BYTES, shouldExtract: (name) => name === entryPath });
|
|
3616
3624
|
return result[entryPath];
|
|
3617
3625
|
}
|
|
3618
3626
|
|
|
@@ -5262,10 +5270,16 @@ function readFileBytes(p) {
|
|
|
5262
5270
|
verifyStillAbsent(p);
|
|
5263
5271
|
return null;
|
|
5264
5272
|
}
|
|
5265
|
-
if (pinned !== void 0)
|
|
5273
|
+
if (pinned !== void 0) {
|
|
5274
|
+
const bytes = readPinnedBytes(p, pinned);
|
|
5275
|
+
if (bytes.length > MAX_ZIP_INPUT_BYTES) throw new ZipInputTooLargeError(p, bytes.length, MAX_ZIP_INPUT_BYTES);
|
|
5276
|
+
return bytes;
|
|
5277
|
+
}
|
|
5278
|
+
const stat = fs5.statSync(p);
|
|
5279
|
+
if (stat.size > MAX_ZIP_INPUT_BYTES) throw new ZipInputTooLargeError(p, stat.size, MAX_ZIP_INPUT_BYTES);
|
|
5266
5280
|
return fs5.readFileSync(p);
|
|
5267
5281
|
} catch (err) {
|
|
5268
|
-
if (err instanceof ConfinementIdentityError) throw err;
|
|
5282
|
+
if (err instanceof ConfinementIdentityError || err instanceof ZipInputTooLargeError) throw err;
|
|
5269
5283
|
return null;
|
|
5270
5284
|
}
|
|
5271
5285
|
}
|
|
@@ -5333,6 +5347,17 @@ function guardText(text, command) {
|
|
|
5333
5347
|
const cfg = loadConfig();
|
|
5334
5348
|
return cfg.overflow_guard.enabled ? trimToBudget(text, cfg.overflow_guard.max_tokens, command) : text;
|
|
5335
5349
|
}
|
|
5350
|
+
function fenceGithubTextIfMatched(text) {
|
|
5351
|
+
let matches = [];
|
|
5352
|
+
try {
|
|
5353
|
+
if (loadConfig().injection.enabled) matches = scanForInjectionPatterns(text);
|
|
5354
|
+
} catch {
|
|
5355
|
+
matches = [];
|
|
5356
|
+
}
|
|
5357
|
+
if (matches.length === 0) return text;
|
|
5358
|
+
recordStat("injection_detected", 0, 0, void 0, matches.join(","));
|
|
5359
|
+
return fenceUntrustedContent(text, matches, UNTRUSTED_GITHUB_TAG);
|
|
5360
|
+
}
|
|
5336
5361
|
function guardJsonRows(items) {
|
|
5337
5362
|
const cfg = loadConfig();
|
|
5338
5363
|
if (!cfg.overflow_guard.enabled) return { items: [...items], truncated: false, totalCount: items.length };
|
|
@@ -7021,11 +7046,20 @@ function runOpenApiOp(opts) {
|
|
|
7021
7046
|
return 0;
|
|
7022
7047
|
}
|
|
7023
7048
|
function archiveReadFailure(err, file) {
|
|
7024
|
-
if (err instanceof ArchiveDependencyMissingError) return err.message;
|
|
7049
|
+
if (err instanceof ArchiveDependencyMissingError || err instanceof ZipOutputTooLargeError) return err.message;
|
|
7025
7050
|
return `Failed to read archive (not a valid zip-format file): ${file}`;
|
|
7026
7051
|
}
|
|
7027
7052
|
async function runZipList(opts) {
|
|
7028
|
-
|
|
7053
|
+
let data;
|
|
7054
|
+
try {
|
|
7055
|
+
data = readFileBytes(opts.file);
|
|
7056
|
+
} catch (err) {
|
|
7057
|
+
if (err instanceof ZipInputTooLargeError) {
|
|
7058
|
+
emitErr(err.message);
|
|
7059
|
+
return 1;
|
|
7060
|
+
}
|
|
7061
|
+
throw err;
|
|
7062
|
+
}
|
|
7029
7063
|
if (data === null) {
|
|
7030
7064
|
emitErr(`Could not read: ${opts.file}`);
|
|
7031
7065
|
return 1;
|
|
@@ -7050,7 +7084,16 @@ async function runZipList(opts) {
|
|
|
7050
7084
|
return 0;
|
|
7051
7085
|
}
|
|
7052
7086
|
async function runZipRead(opts) {
|
|
7053
|
-
|
|
7087
|
+
let data;
|
|
7088
|
+
try {
|
|
7089
|
+
data = readFileBytes(opts.file);
|
|
7090
|
+
} catch (err) {
|
|
7091
|
+
if (err instanceof ZipInputTooLargeError) {
|
|
7092
|
+
emitErr(err.message);
|
|
7093
|
+
return 1;
|
|
7094
|
+
}
|
|
7095
|
+
throw err;
|
|
7096
|
+
}
|
|
7054
7097
|
if (data === null) {
|
|
7055
7098
|
emitErr(`Could not read: ${opts.file}`);
|
|
7056
7099
|
return 1;
|
|
@@ -7146,11 +7189,11 @@ function runPrSlice(opts) {
|
|
|
7146
7189
|
}
|
|
7147
7190
|
const fullSourceBytes = Buffer.byteLength(diffText, "utf8");
|
|
7148
7191
|
if (opts.json === true) {
|
|
7149
|
-
const jsonText = JSON.stringify({ path: parsed.path, diff: fileDiff });
|
|
7192
|
+
const jsonText = JSON.stringify({ path: parsed.path, diff: fenceGithubTextIfMatched(fileDiff) });
|
|
7150
7193
|
emit(jsonText);
|
|
7151
7194
|
recordReadStat("pr_slice", fullSourceBytes, jsonText, `${repo}#${opts.pr} diff:${parsed.path}`);
|
|
7152
7195
|
} else {
|
|
7153
|
-
emitGuarded(fileDiff, "pr-slice");
|
|
7196
|
+
emitGuarded(fenceGithubTextIfMatched(fileDiff), "pr-slice");
|
|
7154
7197
|
recordReadStat("pr_slice", fullSourceBytes, fileDiff, `${repo}#${opts.pr} diff:${parsed.path}`);
|
|
7155
7198
|
}
|
|
7156
7199
|
return 0;
|
|
@@ -7159,13 +7202,14 @@ function runPrSlice(opts) {
|
|
|
7159
7202
|
const comments = fetchPrComments(opts.pr, repo);
|
|
7160
7203
|
const fullSourceBytes = Buffer.byteLength(JSON.stringify(comments), "utf8");
|
|
7161
7204
|
if (opts.json === true) {
|
|
7162
|
-
const
|
|
7205
|
+
const fencedComments = comments.map((c) => ({ ...c, body: fenceGithubTextIfMatched(c.body) }));
|
|
7206
|
+
const capped = guardJsonRows(fencedComments);
|
|
7163
7207
|
const jsonText = JSON.stringify({ items: capped.items, truncated: capped.truncated, totalCount: capped.totalCount });
|
|
7164
7208
|
emit(jsonText);
|
|
7165
7209
|
recordReadStat("pr_slice", fullSourceBytes, jsonText, `${repo}#${opts.pr} comments`);
|
|
7166
7210
|
} else {
|
|
7167
7211
|
const text = formatCommentsSlice(comments);
|
|
7168
|
-
emitGuarded(text, "pr-slice");
|
|
7212
|
+
emitGuarded(fenceGithubTextIfMatched(text), "pr-slice");
|
|
7169
7213
|
recordReadStat("pr_slice", fullSourceBytes, text, `${repo}#${opts.pr} comments`);
|
|
7170
7214
|
}
|
|
7171
7215
|
return 0;
|
|
@@ -7174,12 +7218,17 @@ function runPrSlice(opts) {
|
|
|
7174
7218
|
const desc = fetchPrDescription(opts.pr, repo);
|
|
7175
7219
|
const fullSourceBytes = Buffer.byteLength(JSON.stringify(desc), "utf8");
|
|
7176
7220
|
if (opts.json === true) {
|
|
7177
|
-
const
|
|
7221
|
+
const fencedDesc = {
|
|
7222
|
+
...desc,
|
|
7223
|
+
title: fenceGithubTextIfMatched(desc.title),
|
|
7224
|
+
body: desc.body !== null ? fenceGithubTextIfMatched(desc.body) : null
|
|
7225
|
+
};
|
|
7226
|
+
const jsonText = JSON.stringify(fencedDesc);
|
|
7178
7227
|
emit(jsonText);
|
|
7179
7228
|
recordReadStat("pr_slice", fullSourceBytes, jsonText, `${repo}#${opts.pr} description`);
|
|
7180
7229
|
} else {
|
|
7181
7230
|
const text = formatDescriptionSlice(desc);
|
|
7182
|
-
emitGuarded(text, "pr-slice");
|
|
7231
|
+
emitGuarded(fenceGithubTextIfMatched(text), "pr-slice");
|
|
7183
7232
|
recordReadStat("pr_slice", fullSourceBytes, text, `${repo}#${opts.pr} description`);
|
|
7184
7233
|
}
|
|
7185
7234
|
return 0;
|
|
@@ -7222,7 +7271,6 @@ function runSqliteQuery(opts) {
|
|
|
7222
7271
|
const totalCount = result.rows.length;
|
|
7223
7272
|
const headTruncated = head !== void 0 && result.rows.length > head;
|
|
7224
7273
|
const rows = head !== void 0 ? result.rows.slice(0, head) : result.rows;
|
|
7225
|
-
const fullSourceBytes = sumFileSizes([opts.file]);
|
|
7226
7274
|
if (opts.json === true) {
|
|
7227
7275
|
const capped = guardJsonRows(rows);
|
|
7228
7276
|
const jsonText = JSON.stringify({
|
|
@@ -7233,11 +7281,20 @@ function runSqliteQuery(opts) {
|
|
|
7233
7281
|
rowCapped: result.rowCapped
|
|
7234
7282
|
});
|
|
7235
7283
|
emit(jsonText);
|
|
7236
|
-
|
|
7284
|
+
const uncappedFull = guardJsonRows(result.rows);
|
|
7285
|
+
const baselineJsonText = JSON.stringify({
|
|
7286
|
+
columns: result.columns,
|
|
7287
|
+
items: uncappedFull.items,
|
|
7288
|
+
truncated: uncappedFull.truncated || result.rowCapped,
|
|
7289
|
+
totalCount,
|
|
7290
|
+
rowCapped: result.rowCapped
|
|
7291
|
+
});
|
|
7292
|
+
recordReadStat("sqlite_query", Buffer.byteLength(baselineJsonText, "utf8"), jsonText, opts.file);
|
|
7237
7293
|
} else {
|
|
7238
7294
|
const text = formatSqliteQueryTable({ ...result, rows }, { headTruncated });
|
|
7239
7295
|
emit(text);
|
|
7240
|
-
|
|
7296
|
+
const baselineText = formatSqliteQueryTable({ ...result, rows: result.rows }, { headTruncated: false });
|
|
7297
|
+
recordReadStat("sqlite_query", Buffer.byteLength(baselineText, "utf8"), text, opts.file);
|
|
7241
7298
|
}
|
|
7242
7299
|
return 0;
|
|
7243
7300
|
} catch (e) {
|
|
@@ -10,11 +10,11 @@ import {
|
|
|
10
10
|
selectFilter,
|
|
11
11
|
shlexSplit,
|
|
12
12
|
wrappedShell
|
|
13
|
-
} from "./token-goat-chunk-
|
|
13
|
+
} from "./token-goat-chunk-EFF2XCLB.mjs";
|
|
14
14
|
import {
|
|
15
15
|
loadConfig,
|
|
16
16
|
recordStat
|
|
17
|
-
} from "./token-goat-chunk-
|
|
17
|
+
} from "./token-goat-chunk-6ODZ6PZK.mjs";
|
|
18
18
|
import "./token-goat-chunk-AO2QD2AG.mjs";
|
|
19
19
|
import "./token-goat-chunk-AEX54RUZ.mjs";
|
|
20
20
|
|
package/dist/token-goat-hook.mjs
CHANGED
|
@@ -2,11 +2,11 @@ import { createRequire as __cjsRequire } from 'node:module';
|
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
4
|
relayInProcess
|
|
5
|
-
} from "./token-goat-chunk-
|
|
6
|
-
import "./token-goat-chunk-
|
|
7
|
-
import "./token-goat-chunk-
|
|
8
|
-
import "./token-goat-chunk-
|
|
9
|
-
import "./token-goat-chunk-
|
|
5
|
+
} from "./token-goat-chunk-NKNCHJ4H.mjs";
|
|
6
|
+
import "./token-goat-chunk-4HIMCBYK.mjs";
|
|
7
|
+
import "./token-goat-chunk-PWVXXPCC.mjs";
|
|
8
|
+
import "./token-goat-chunk-EFF2XCLB.mjs";
|
|
9
|
+
import "./token-goat-chunk-6ODZ6PZK.mjs";
|
|
10
10
|
import "./token-goat-chunk-AO2QD2AG.mjs";
|
|
11
11
|
import "./token-goat-chunk-AEX54RUZ.mjs";
|
|
12
12
|
export {
|
package/dist/token-goat.core.mjs
CHANGED
|
@@ -2,13 +2,13 @@ import { createRequire as __cjsRequire } from 'node:module';
|
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
3
|
import {
|
|
4
4
|
run
|
|
5
|
-
} from "./token-goat-chunk-
|
|
6
|
-
import "./token-goat-chunk-
|
|
7
|
-
import "./token-goat-chunk-
|
|
8
|
-
import "./token-goat-chunk-
|
|
5
|
+
} from "./token-goat-chunk-222VPFP2.mjs";
|
|
6
|
+
import "./token-goat-chunk-TX4JFJTD.mjs";
|
|
7
|
+
import "./token-goat-chunk-4HIMCBYK.mjs";
|
|
8
|
+
import "./token-goat-chunk-PWVXXPCC.mjs";
|
|
9
9
|
import {
|
|
10
10
|
installEpipeGuard
|
|
11
|
-
} from "./token-goat-chunk-
|
|
11
|
+
} from "./token-goat-chunk-6ODZ6PZK.mjs";
|
|
12
12
|
import "./token-goat-chunk-AO2QD2AG.mjs";
|
|
13
13
|
import "./token-goat-chunk-AEX54RUZ.mjs";
|
|
14
14
|
|