token-goat 2.8.2 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -6
- package/dist/{token-goat-chunk-2MWF3OGR.mjs → token-goat-chunk-222VPFP2.mjs} +138 -49
- package/dist/{token-goat-chunk-5MLXFSRI.mjs → token-goat-chunk-4HIMCBYK.mjs} +2 -2
- package/dist/{token-goat-chunk-SE6V7LUZ.mjs → token-goat-chunk-4OM2Q2SX.mjs} +5 -5
- package/dist/{token-goat-chunk-IVCTQPZD.mjs → token-goat-chunk-6ODZ6PZK.mjs} +219 -147
- package/dist/{token-goat-chunk-AL644JUV.mjs → token-goat-chunk-C5IL6MJH.mjs} +3 -3
- package/dist/{token-goat-chunk-JU7QESB3.mjs → token-goat-chunk-DP3NNGGV.mjs} +5 -5
- package/dist/{token-goat-chunk-TF5NT3H5.mjs → token-goat-chunk-EFF2XCLB.mjs} +18 -5
- package/dist/{token-goat-chunk-KBVN4ELV.mjs → token-goat-chunk-NKNCHJ4H.mjs} +80 -49
- package/dist/{token-goat-chunk-AM23GDIS.mjs → token-goat-chunk-PWVXXPCC.mjs} +134 -29
- package/dist/{token-goat-chunk-LILS6TIU.mjs → token-goat-chunk-TX4JFJTD.mjs} +74 -17
- package/dist/{token-goat-chunk-BTZSUIYA.mjs → token-goat-chunk-VAGPOZHO.mjs} +2 -2
- package/dist/token-goat-hook.mjs +5 -5
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +1 -1
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
summarizeOutputDelta,
|
|
24
24
|
summarizeResidentContext,
|
|
25
25
|
taskListPruneHint
|
|
26
|
-
} from "./token-goat-chunk-
|
|
26
|
+
} from "./token-goat-chunk-4HIMCBYK.mjs";
|
|
27
27
|
import {
|
|
28
28
|
BODY_FIRST_TOOL_RESPONSE_KEYS,
|
|
29
29
|
OUTPUT_FIRST_TOOL_RESPONSE_KEYS,
|
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
contextOutput,
|
|
42
42
|
countSymbols,
|
|
43
43
|
denyOutput,
|
|
44
|
+
emitRewrite,
|
|
44
45
|
enqueueDirtyPathSafe,
|
|
45
46
|
ensureWorkerAlive,
|
|
46
47
|
estimateTokens,
|
|
@@ -109,6 +110,7 @@ import {
|
|
|
109
110
|
saveSessionState,
|
|
110
111
|
scanForInjectionPatterns,
|
|
111
112
|
serializeOutput,
|
|
113
|
+
sessionOutputBodyBytes,
|
|
112
114
|
sessionSidecarPath,
|
|
113
115
|
setLastTabContext,
|
|
114
116
|
shrinkImage,
|
|
@@ -118,7 +120,7 @@ import {
|
|
|
118
120
|
wasCliReadThisSession,
|
|
119
121
|
wasFileReadThisSession,
|
|
120
122
|
wasHintShown
|
|
121
|
-
} from "./token-goat-chunk-
|
|
123
|
+
} from "./token-goat-chunk-PWVXXPCC.mjs";
|
|
122
124
|
import {
|
|
123
125
|
canRunWrappedShell,
|
|
124
126
|
compressOutput,
|
|
@@ -129,8 +131,9 @@ import {
|
|
|
129
131
|
isRewriteWorthwhile,
|
|
130
132
|
resolveMinNetSavingsBytes,
|
|
131
133
|
shlexSplit
|
|
132
|
-
} from "./token-goat-chunk-
|
|
134
|
+
} from "./token-goat-chunk-EFF2XCLB.mjs";
|
|
133
135
|
import {
|
|
136
|
+
PER_FILE_COUNTERFACTUAL_CEILING,
|
|
134
137
|
VERSION,
|
|
135
138
|
detectHarness,
|
|
136
139
|
detectLanguage,
|
|
@@ -149,7 +152,7 @@ import {
|
|
|
149
152
|
runGit,
|
|
150
153
|
shortFingerprint,
|
|
151
154
|
toKB
|
|
152
|
-
} from "./token-goat-chunk-
|
|
155
|
+
} from "./token-goat-chunk-6ODZ6PZK.mjs";
|
|
153
156
|
|
|
154
157
|
// src/hooks_grep.ts
|
|
155
158
|
function grepIntInput(toolInput, key) {
|
|
@@ -213,8 +216,10 @@ function foldGrepContentHandler(event) {
|
|
|
213
216
|
return passOutput();
|
|
214
217
|
}
|
|
215
218
|
if (toolInput["-n"] === false) return passOutput();
|
|
216
|
-
const
|
|
217
|
-
if (!
|
|
219
|
+
const rawText = extractToolResponseField(event.raw, OUTPUT_FIRST_TOOL_RESPONSE_KEYS);
|
|
220
|
+
if (!rawText) return passOutput();
|
|
221
|
+
const redacted = redactSecrets(rawText);
|
|
222
|
+
const text = redacted.text;
|
|
218
223
|
const rawLines = text.split(/\r\n|\r|\n/);
|
|
219
224
|
const parsed = [];
|
|
220
225
|
for (const line of rawLines) {
|
|
@@ -257,6 +262,7 @@ function foldGrepContentHandler(event) {
|
|
|
257
262
|
}
|
|
258
263
|
const bytesDelta = originalBytes - rewrittenBytes;
|
|
259
264
|
recordStat("grep:fold", bytesDelta, Math.round(bytesDelta / 4));
|
|
265
|
+
if (redacted.count > 0) recordStat("secret_redacted", 0, redacted.count, void 0, "grep");
|
|
260
266
|
return { hookType: "rewriteOutput", updatedOutput: rewritten };
|
|
261
267
|
} catch {
|
|
262
268
|
return passOutput();
|
|
@@ -620,7 +626,7 @@ import crypto from "node:crypto";
|
|
|
620
626
|
var TRACKED_SKILL = "token-goat";
|
|
621
627
|
var MAX_COMMANDS_SHOWN = 8;
|
|
622
628
|
async function currentCommandNames() {
|
|
623
|
-
const { buildProgram } = await import("./token-goat-chunk-
|
|
629
|
+
const { buildProgram } = await import("./token-goat-chunk-4OM2Q2SX.mjs");
|
|
624
630
|
return flattenCommandNames(buildCommandManifest(buildProgram()));
|
|
625
631
|
}
|
|
626
632
|
async function recordSkillVersionSnapshot(sessionId, skillName) {
|
|
@@ -7030,15 +7036,22 @@ function postFetchHandler(event) {
|
|
|
7030
7036
|
if (injectionMatches.length > 0) {
|
|
7031
7037
|
recordStat("injection_detected", 0, 0, void 0, injectionMatches.join(","));
|
|
7032
7038
|
}
|
|
7039
|
+
const bodyRedacted = body ? redactSecrets(body) : { text: body, count: 0 };
|
|
7033
7040
|
if (!event.sessionId) {
|
|
7034
7041
|
if (injectionMatches.length > 0) {
|
|
7035
|
-
return
|
|
7042
|
+
return emitRewrite(fenceUntrustedContent(bodyRedacted.text, injectionMatches), "fetch");
|
|
7043
|
+
}
|
|
7044
|
+
if (bodyRedacted.count > 0) {
|
|
7045
|
+
return emitRewrite(bodyRedacted.text, "fetch");
|
|
7036
7046
|
}
|
|
7037
7047
|
return passOutput();
|
|
7038
7048
|
}
|
|
7039
7049
|
if (!body || body.length < 1024) {
|
|
7040
7050
|
if (injectionMatches.length > 0) {
|
|
7041
|
-
return
|
|
7051
|
+
return emitRewrite(fenceUntrustedContent(bodyRedacted.text, injectionMatches), "fetch");
|
|
7052
|
+
}
|
|
7053
|
+
if (bodyRedacted.count > 0) {
|
|
7054
|
+
return emitRewrite(bodyRedacted.text, "fetch");
|
|
7042
7055
|
}
|
|
7043
7056
|
return passOutput();
|
|
7044
7057
|
}
|
|
@@ -7054,8 +7067,9 @@ function postFetchHandler(event) {
|
|
|
7054
7067
|
}
|
|
7055
7068
|
const cacheId = storeWebOutput(url, storedBody, `${url}\0${prompt}`, storedBody !== body ? body : void 0);
|
|
7056
7069
|
recordWebFetch(url, prompt, cacheId);
|
|
7070
|
+
const storedRedacted = storedBody === body ? bodyRedacted : redactSecrets(storedBody);
|
|
7057
7071
|
if (injectionMatches.length > 0) {
|
|
7058
|
-
return
|
|
7072
|
+
return emitRewrite(fenceUntrustedContent(storedRedacted.text, injectionMatches), "fetch");
|
|
7059
7073
|
}
|
|
7060
7074
|
if (storedBody !== body) {
|
|
7061
7075
|
const noticeFor = (id) => `
|
|
@@ -7072,9 +7086,12 @@ function postFetchHandler(event) {
|
|
|
7072
7086
|
})) {
|
|
7073
7087
|
const bytesDelta = originalBytes - rewrittenBytes;
|
|
7074
7088
|
recordStat("webfetch:compress", bytesDelta, Math.round(bytesDelta / 4));
|
|
7075
|
-
return
|
|
7089
|
+
return emitRewrite(storedRedacted.text + notice, "fetch");
|
|
7076
7090
|
}
|
|
7077
7091
|
}
|
|
7092
|
+
if (storedRedacted.count > 0) {
|
|
7093
|
+
return emitRewrite(storedRedacted.text, "fetch");
|
|
7094
|
+
}
|
|
7078
7095
|
return passOutput();
|
|
7079
7096
|
} catch {
|
|
7080
7097
|
return passOutput();
|
|
@@ -7127,7 +7144,9 @@ async function preSkillHandler(event) {
|
|
|
7127
7144
|
return passOutput();
|
|
7128
7145
|
}
|
|
7129
7146
|
if (await hasSessionOutput(event.sessionId, skillName)) {
|
|
7130
|
-
|
|
7147
|
+
const cachedBytes = await sessionOutputBodyBytes(event.sessionId, skillName);
|
|
7148
|
+
const denyCredit = cachedBytes !== null ? Math.min(cachedBytes, PER_FILE_COUNTERFACTUAL_CEILING) : 0;
|
|
7149
|
+
recordStat("session_hint", denyCredit, Math.round(denyCredit / 4));
|
|
7131
7150
|
return denyOutput(
|
|
7132
7151
|
"Skill `" + skillName + "` was already loaded this session and is cached. Use `token-goat skill-body " + skillName + " --compact` to recall the compact slice (or `token-goat skill-body " + skillName + "` for the full body) instead of re-loading it."
|
|
7133
7152
|
);
|
|
@@ -9121,10 +9140,7 @@ function postBashOutputHandler(event) {
|
|
|
9121
9140
|
})) {
|
|
9122
9141
|
return passOutput();
|
|
9123
9142
|
}
|
|
9124
|
-
return {
|
|
9125
|
-
hookType: "rewriteOutput",
|
|
9126
|
-
updatedOutput: unchangedNotice
|
|
9127
|
-
};
|
|
9143
|
+
return emitRewrite(unchangedNotice, "bashoutput", { kind: "bashoutput:unchanged", originalBytes: Buffer.byteLength(rawOutput, "utf-8") });
|
|
9128
9144
|
}
|
|
9129
9145
|
if (!output.startsWith(prior.output)) {
|
|
9130
9146
|
storePollSnapshot(event.sessionId, bashId, output);
|
|
@@ -9143,10 +9159,7 @@ function postBashOutputHandler(event) {
|
|
|
9143
9159
|
})) {
|
|
9144
9160
|
return passOutput();
|
|
9145
9161
|
}
|
|
9146
|
-
return {
|
|
9147
|
-
hookType: "rewriteOutput",
|
|
9148
|
-
updatedOutput: `${deltaNotice}${delta}`
|
|
9149
|
-
};
|
|
9162
|
+
return emitRewrite(`${deltaNotice}${delta}`, "bashoutput", { kind: "bashoutput:delta", originalBytes: Buffer.byteLength(rawOutput, "utf-8") });
|
|
9150
9163
|
} catch {
|
|
9151
9164
|
return passOutput();
|
|
9152
9165
|
}
|
|
@@ -9201,10 +9214,7 @@ function postTaskOutputHandler(event) {
|
|
|
9201
9214
|
})) {
|
|
9202
9215
|
return passOutput();
|
|
9203
9216
|
}
|
|
9204
|
-
return {
|
|
9205
|
-
hookType: "rewriteOutput",
|
|
9206
|
-
updatedOutput: collapsed
|
|
9207
|
-
};
|
|
9217
|
+
return emitRewrite(collapsed, "taskoutput", { kind: "taskoutput:collapse", originalBytes: Buffer.byteLength(rawOutput, "utf-8") });
|
|
9208
9218
|
}
|
|
9209
9219
|
if (output === prior.output) {
|
|
9210
9220
|
storePollSnapshot2(event.sessionId, taskId, output);
|
|
@@ -9218,10 +9228,7 @@ function postTaskOutputHandler(event) {
|
|
|
9218
9228
|
})) {
|
|
9219
9229
|
return passOutput();
|
|
9220
9230
|
}
|
|
9221
|
-
return {
|
|
9222
|
-
hookType: "rewriteOutput",
|
|
9223
|
-
updatedOutput: unchangedNotice
|
|
9224
|
-
};
|
|
9231
|
+
return emitRewrite(unchangedNotice, "taskoutput", { kind: "taskoutput:unchanged", originalBytes: Buffer.byteLength(rawOutput, "utf-8") });
|
|
9225
9232
|
}
|
|
9226
9233
|
if (!output.startsWith(prior.output)) {
|
|
9227
9234
|
storePollSnapshot2(event.sessionId, taskId, output);
|
|
@@ -9241,10 +9248,7 @@ function postTaskOutputHandler(event) {
|
|
|
9241
9248
|
})) {
|
|
9242
9249
|
return passOutput();
|
|
9243
9250
|
}
|
|
9244
|
-
return {
|
|
9245
|
-
hookType: "rewriteOutput",
|
|
9246
|
-
updatedOutput: `${deltaNotice}${collapsedDelta}`
|
|
9247
|
-
};
|
|
9251
|
+
return emitRewrite(`${deltaNotice}${collapsedDelta}`, "taskoutput", { kind: "taskoutput:delta", originalBytes: Buffer.byteLength(rawOutput, "utf-8") });
|
|
9248
9252
|
} catch {
|
|
9249
9253
|
return passOutput();
|
|
9250
9254
|
}
|
|
@@ -9340,8 +9344,10 @@ var APPROVED_PLAN_MARKER = "## Approved Plan:";
|
|
|
9340
9344
|
function postExitPlanModeHandler(event) {
|
|
9341
9345
|
try {
|
|
9342
9346
|
if (getToolName(event) !== "ExitPlanMode") return passOutput();
|
|
9343
|
-
const
|
|
9344
|
-
if (!
|
|
9347
|
+
const rawOutput = extractToolResultText(event.raw);
|
|
9348
|
+
if (!rawOutput) return passOutput();
|
|
9349
|
+
const redacted = redactSecrets(rawOutput);
|
|
9350
|
+
const output = redacted.text;
|
|
9345
9351
|
const markerIndex = output.indexOf(APPROVED_PLAN_MARKER);
|
|
9346
9352
|
if (markerIndex === -1) return passOutput();
|
|
9347
9353
|
const planValue = getToolInput(event)["plan"];
|
|
@@ -9770,11 +9776,16 @@ function postMcpHandler(event) {
|
|
|
9770
9776
|
injectionMatches = [];
|
|
9771
9777
|
}
|
|
9772
9778
|
if (injectionMatches.length > 0) recordStat("injection_detected", 0, 0, void 0, injectionMatches.join(","));
|
|
9773
|
-
const
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
9779
|
+
const redactedResult = redactSecrets(resultText);
|
|
9780
|
+
const fenced = () => emitRewrite(
|
|
9781
|
+
fenceUntrustedContent(redactedResult.text, injectionMatches, UNTRUSTED_TOOL_TAG),
|
|
9782
|
+
"mcp"
|
|
9783
|
+
);
|
|
9784
|
+
const passOrFence = () => {
|
|
9785
|
+
if (injectionMatches.length > 0) return fenced();
|
|
9786
|
+
if (redactedResult.count > 0) return emitRewrite(redactedResult.text, "mcp");
|
|
9787
|
+
return passOutput();
|
|
9788
|
+
};
|
|
9778
9789
|
if (!event.sessionId) return passOrFence();
|
|
9779
9790
|
if (isMcpErrorResponse(event.raw)) return passOrFence();
|
|
9780
9791
|
const readOnly = isMcpReadOnly(toolName, toolInput);
|
|
@@ -9799,10 +9810,10 @@ function postMcpHandler(event) {
|
|
|
9799
9810
|
minNetSavingsBytes: resolveMinNetSavingsBytes()
|
|
9800
9811
|
});
|
|
9801
9812
|
if (worthwhile) {
|
|
9802
|
-
return
|
|
9803
|
-
|
|
9804
|
-
|
|
9805
|
-
|
|
9813
|
+
return emitRewrite(
|
|
9814
|
+
injectionMatches.length > 0 ? `${notice}${fenceUntrustedContent(redactedBody, injectionMatches, UNTRUSTED_TOOL_TAG)}` : `${notice}${redactedBody}`,
|
|
9815
|
+
"mcp"
|
|
9816
|
+
);
|
|
9806
9817
|
}
|
|
9807
9818
|
}
|
|
9808
9819
|
}
|
|
@@ -9848,14 +9859,32 @@ function preWebSearchDedupHandler(event) {
|
|
|
9848
9859
|
}
|
|
9849
9860
|
function postWebSearchHandler(event) {
|
|
9850
9861
|
try {
|
|
9851
|
-
if (getToolName(event) !== "WebSearch"
|
|
9852
|
-
const signature = webSearchSignatureInput(getToolInput(event));
|
|
9853
|
-
if (signature === null) return passOutput();
|
|
9854
|
-
if (isMcpErrorResponse(event.raw)) return passOutput();
|
|
9862
|
+
if (getToolName(event) !== "WebSearch") return passOutput();
|
|
9855
9863
|
const resultText = extractToolResultText(event.raw);
|
|
9856
9864
|
if (!resultText) return passOutput();
|
|
9865
|
+
let injectionMatches = [];
|
|
9866
|
+
try {
|
|
9867
|
+
if (loadConfig().injection.enabled) injectionMatches = scanForInjectionPatterns(resultText);
|
|
9868
|
+
} catch {
|
|
9869
|
+
injectionMatches = [];
|
|
9870
|
+
}
|
|
9871
|
+
if (injectionMatches.length > 0) recordStat("injection_detected", 0, 0, void 0, injectionMatches.join(","));
|
|
9872
|
+
const redacted = redactSecrets(resultText);
|
|
9873
|
+
const passOrFence = () => {
|
|
9874
|
+
if (injectionMatches.length > 0) {
|
|
9875
|
+
return emitRewrite(fenceUntrustedContent(redacted.text, injectionMatches), "websearch");
|
|
9876
|
+
}
|
|
9877
|
+
if (redacted.count > 0) {
|
|
9878
|
+
return emitRewrite(redacted.text, "websearch");
|
|
9879
|
+
}
|
|
9880
|
+
return passOutput();
|
|
9881
|
+
};
|
|
9882
|
+
if (!event.sessionId) return passOrFence();
|
|
9883
|
+
const signature = webSearchSignatureInput(getToolInput(event));
|
|
9884
|
+
if (signature === null) return passOrFence();
|
|
9885
|
+
if (isMcpErrorResponse(event.raw)) return passOrFence();
|
|
9857
9886
|
storeMcpOutput(event.sessionId, "WebSearch", signature, resultText);
|
|
9858
|
-
return
|
|
9887
|
+
return passOrFence();
|
|
9859
9888
|
} catch {
|
|
9860
9889
|
return passOutput();
|
|
9861
9890
|
}
|
|
@@ -10192,11 +10221,13 @@ function postAgentHandler(event) {
|
|
|
10192
10221
|
if (typeof finishedPrompt === "string" && finishedPrompt !== "") {
|
|
10193
10222
|
removeOutstandingAgentSpawn(finishedPrompt);
|
|
10194
10223
|
}
|
|
10195
|
-
const
|
|
10224
|
+
const redactedReport = redactSecrets(extractToolResultText(event.raw));
|
|
10225
|
+
const resultText = redactedReport.text;
|
|
10196
10226
|
const agentReportCfg = loadConfig().agent_report;
|
|
10197
10227
|
if (!resultText || resultText.length < agentReportCfg.min_bytes) return passOutput();
|
|
10198
10228
|
const id = storeMcpOutput(event.sessionId, "Agent", event.toolInput, resultText);
|
|
10199
10229
|
if (id === null) return passOutput();
|
|
10230
|
+
if (redactedReport.count > 0) recordStat("secret_redacted", 0, redactedReport.count, void 0, "agent");
|
|
10200
10231
|
recordStat("session_hint", 0, 0);
|
|
10201
10232
|
const recallHint = `token-goat mcp-output ${id} --full`;
|
|
10202
10233
|
const notice = `[token-goat] This subagent report (${toKB(resultText.length)}KB) is cached for later recall: ${recallHint}`;
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
buildLineIndex,
|
|
14
14
|
countContentLines,
|
|
15
15
|
countNoun,
|
|
16
|
+
countRedactionPlaceholders,
|
|
16
17
|
dataDir,
|
|
17
18
|
decodeSource,
|
|
18
19
|
detectHarness,
|
|
@@ -91,7 +92,7 @@ import {
|
|
|
91
92
|
withFileLock,
|
|
92
93
|
writeIfDifferent,
|
|
93
94
|
writeJsonSettings
|
|
94
|
-
} from "./token-goat-chunk-
|
|
95
|
+
} from "./token-goat-chunk-6ODZ6PZK.mjs";
|
|
95
96
|
import {
|
|
96
97
|
registerReset
|
|
97
98
|
} from "./token-goat-chunk-AO2QD2AG.mjs";
|
|
@@ -325,12 +326,81 @@ function parseXml(xml) {
|
|
|
325
326
|
return Object.fromEntries(root.children);
|
|
326
327
|
}
|
|
327
328
|
|
|
329
|
+
// src/zip_bounds.ts
|
|
330
|
+
var MAX_ZIP_INPUT_BYTES = 50 * 1024 * 1024;
|
|
331
|
+
var MAX_ZIP_OUTPUT_BYTES = 500 * 1024 * 1024;
|
|
332
|
+
var STREAM_CHUNK_BYTES = 64 * 1024;
|
|
333
|
+
var ZipOutputTooLargeError = class extends Error {
|
|
334
|
+
constructor(entryName, limitBytes, decompressedSoFarBytes) {
|
|
335
|
+
super(
|
|
336
|
+
`zip entry '${entryName}' is over the ${Math.round(limitBytes / (1024 * 1024))}MB decompressed-size limit (over ${Math.round(decompressedSoFarBytes / (1024 * 1024))}MB decompressed so far)`
|
|
337
|
+
);
|
|
338
|
+
this.name = "ZipOutputTooLargeError";
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
var ZipInputTooLargeError = class extends Error {
|
|
342
|
+
constructor(filePath, sizeBytes, limitBytes) {
|
|
343
|
+
super(`${filePath} is ${Math.round(sizeBytes / (1024 * 1024))}MB, over the ${Math.round(limitBytes / (1024 * 1024))}MB limit for zip-format archives`);
|
|
344
|
+
this.name = "ZipInputTooLargeError";
|
|
345
|
+
}
|
|
346
|
+
};
|
|
347
|
+
function concatChunks(chunks, total) {
|
|
348
|
+
const out = new Uint8Array(total);
|
|
349
|
+
let offset = 0;
|
|
350
|
+
for (const chunk of chunks) {
|
|
351
|
+
out.set(chunk, offset);
|
|
352
|
+
offset += chunk.length;
|
|
353
|
+
}
|
|
354
|
+
return out;
|
|
355
|
+
}
|
|
356
|
+
function unzipBounded(mod, data, opts) {
|
|
357
|
+
mod.unzipSync(data, { filter: () => false });
|
|
358
|
+
const results = {};
|
|
359
|
+
let firstError;
|
|
360
|
+
let totalDecompressed = 0;
|
|
361
|
+
const unzip = new mod.Unzip((file) => {
|
|
362
|
+
if (firstError !== void 0 || !opts.shouldExtract(file.name)) return;
|
|
363
|
+
if (typeof file.originalSize === "number" && totalDecompressed + file.originalSize > opts.limitBytes) {
|
|
364
|
+
firstError = new ZipOutputTooLargeError(file.name, opts.limitBytes, totalDecompressed + file.originalSize);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
const chunks = [];
|
|
368
|
+
let entryTotal = 0;
|
|
369
|
+
file.ondata = (err, chunk, final) => {
|
|
370
|
+
if (firstError !== void 0) return;
|
|
371
|
+
if (err) {
|
|
372
|
+
firstError = err instanceof Error ? err : new Error(String(err));
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
entryTotal += chunk.length;
|
|
376
|
+
totalDecompressed += chunk.length;
|
|
377
|
+
if (totalDecompressed > opts.limitBytes) {
|
|
378
|
+
firstError = new ZipOutputTooLargeError(file.name, opts.limitBytes, totalDecompressed);
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
chunks.push(chunk);
|
|
382
|
+
if (final) results[file.name] = concatChunks(chunks, entryTotal);
|
|
383
|
+
};
|
|
384
|
+
file.start();
|
|
385
|
+
});
|
|
386
|
+
unzip.register(mod.UnzipInflate);
|
|
387
|
+
let offset = 0;
|
|
388
|
+
for (; ; ) {
|
|
389
|
+
const end = Math.min(offset + STREAM_CHUNK_BYTES, data.length);
|
|
390
|
+
const isFinal = end >= data.length;
|
|
391
|
+
unzip.push(data.subarray(offset, end), isFinal);
|
|
392
|
+
offset = end;
|
|
393
|
+
if (firstError !== void 0 || isFinal) break;
|
|
394
|
+
}
|
|
395
|
+
if (firstError !== void 0) throw firstError;
|
|
396
|
+
return results;
|
|
397
|
+
}
|
|
398
|
+
|
|
328
399
|
// src/ooxml_extract.ts
|
|
329
400
|
var loadFflate = createLazyModuleLoader(
|
|
330
401
|
async () => await import("fflate"),
|
|
331
402
|
"office-file reading disabled (fflate unavailable)"
|
|
332
403
|
);
|
|
333
|
-
var MAX_OOXML_INPUT_BYTES = 50 * 1024 * 1024;
|
|
334
404
|
function accessFailureMessage(err, filePath) {
|
|
335
405
|
const code = err?.code;
|
|
336
406
|
if (code === "ENOENT") return `File not found: ${filePath}`;
|
|
@@ -346,8 +416,8 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
346
416
|
throw new Error(accessFailureMessage(err, filePath), { cause: err });
|
|
347
417
|
}
|
|
348
418
|
if (!stat2.isFile()) throw new Error(`not a valid ${kind} file: ${filePath}`);
|
|
349
|
-
if (stat2.size >
|
|
350
|
-
throw new Error(`${filePath} is ${Math.round(stat2.size / (1024 * 1024))}MB, over the ${
|
|
419
|
+
if (stat2.size > MAX_ZIP_INPUT_BYTES) {
|
|
420
|
+
throw new Error(`${filePath} is ${Math.round(stat2.size / (1024 * 1024))}MB, over the ${MAX_ZIP_INPUT_BYTES / (1024 * 1024)}MB limit for OOXML files`);
|
|
351
421
|
}
|
|
352
422
|
let data;
|
|
353
423
|
try {
|
|
@@ -356,8 +426,9 @@ async function readOoxmlZip(filePath, kind) {
|
|
|
356
426
|
throw new Error(accessFailureMessage(err, filePath), { cause: err });
|
|
357
427
|
}
|
|
358
428
|
try {
|
|
359
|
-
return fflate
|
|
429
|
+
return unzipBounded(fflate, new Uint8Array(data), { limitBytes: MAX_ZIP_OUTPUT_BYTES, shouldExtract: () => true });
|
|
360
430
|
} catch (err) {
|
|
431
|
+
if (err instanceof ZipOutputTooLargeError) throw err;
|
|
361
432
|
throw new Error(`not a valid ${kind} file: ${filePath}`, { cause: err });
|
|
362
433
|
}
|
|
363
434
|
}
|
|
@@ -3408,6 +3479,15 @@ function denyOutput(message) {
|
|
|
3408
3479
|
function contextOutput(context) {
|
|
3409
3480
|
return { hookType: "context", context };
|
|
3410
3481
|
}
|
|
3482
|
+
function emitRewrite(updatedOutput, detail, savings) {
|
|
3483
|
+
const count = countRedactionPlaceholders(updatedOutput);
|
|
3484
|
+
if (count > 0) recordStat("secret_redacted", 0, count, void 0, detail);
|
|
3485
|
+
if (savings !== void 0) {
|
|
3486
|
+
const bytesSaved = savings.originalBytes - Buffer.byteLength(updatedOutput, "utf-8");
|
|
3487
|
+
if (bytesSaved > 0) recordStat(savings.kind, bytesSaved, Math.round(bytesSaved / 4));
|
|
3488
|
+
}
|
|
3489
|
+
return { hookType: "rewriteOutput", updatedOutput };
|
|
3490
|
+
}
|
|
3411
3491
|
function countNonEmptyLines(text) {
|
|
3412
3492
|
return text.split(/\r\n|\r|\n/).filter((line) => line.length > 0).length;
|
|
3413
3493
|
}
|
|
@@ -3735,8 +3815,7 @@ function getHintStatsTotals() {
|
|
|
3735
3815
|
return {
|
|
3736
3816
|
savedBytes,
|
|
3737
3817
|
spentBytes,
|
|
3738
|
-
legacyEmissions
|
|
3739
|
-
netBytes: spentBytes === null ? null : savedBytes - spentBytes
|
|
3818
|
+
legacyEmissions
|
|
3740
3819
|
};
|
|
3741
3820
|
}
|
|
3742
3821
|
function resetHintStats() {
|
|
@@ -6855,6 +6934,7 @@ ${neutralizeFenceMarkers(text, UNTRUSTED_FILE_TAG)}
|
|
|
6855
6934
|
</${UNTRUSTED_FILE_TAG}>`;
|
|
6856
6935
|
}
|
|
6857
6936
|
var UNTRUSTED_TOOL_TAG = "untrusted-tool-output";
|
|
6937
|
+
var UNTRUSTED_GITHUB_TAG = "untrusted-github-content";
|
|
6858
6938
|
|
|
6859
6939
|
// src/skill_cache.ts
|
|
6860
6940
|
import * as fs14 from "fs/promises";
|
|
@@ -7045,6 +7125,21 @@ async function hasSessionOutput(sessionId, skillName) {
|
|
|
7045
7125
|
return false;
|
|
7046
7126
|
}
|
|
7047
7127
|
}
|
|
7128
|
+
async function sessionOutputBodyBytes(sessionId, skillName) {
|
|
7129
|
+
try {
|
|
7130
|
+
if (!sessionId) return null;
|
|
7131
|
+
const name = safeSkillName(skillName);
|
|
7132
|
+
if (!name) return null;
|
|
7133
|
+
const safeSession = safeSessionFragment(sessionId);
|
|
7134
|
+
const metas = await listOutputs();
|
|
7135
|
+
const matches = metas.filter((m) => m.skillName === name && m.outputId.startsWith(`${safeSession}-`));
|
|
7136
|
+
if (matches.length === 0) return null;
|
|
7137
|
+
matches.sort((a, b) => b.ts - a.ts);
|
|
7138
|
+
return matches[0].bodyBytes;
|
|
7139
|
+
} catch {
|
|
7140
|
+
return null;
|
|
7141
|
+
}
|
|
7142
|
+
}
|
|
7048
7143
|
async function findCrossSessionEntry(skillName, contentSha) {
|
|
7049
7144
|
const name = safeSkillName(skillName);
|
|
7050
7145
|
if (!name || !contentSha) return null;
|
|
@@ -7085,9 +7180,11 @@ async function storeOutput(sessionId, skillName, body, opts) {
|
|
|
7085
7180
|
const ts = Date.now();
|
|
7086
7181
|
const bodyBytes = Buffer.byteLength(body, "utf-8");
|
|
7087
7182
|
const truncated = bodyBytes > 256 * 1024;
|
|
7088
|
-
|
|
7183
|
+
const redactedBody = redactSecrets(body);
|
|
7184
|
+
if (redactedBody.count > 0) recordStat("secret_redacted", 0, redactedBody.count, void 0, SKILLS_OUTPUT_SUBDIR);
|
|
7185
|
+
let storedBody = redactedBody.text;
|
|
7089
7186
|
if (truncated) {
|
|
7090
|
-
const buf = Buffer.from(
|
|
7187
|
+
const buf = Buffer.from(redactedBody.text, "utf-8");
|
|
7091
7188
|
let truncStart = Math.max(0, buf.length - 262144);
|
|
7092
7189
|
if (truncStart < buf.length) {
|
|
7093
7190
|
const byte = buf[truncStart];
|
|
@@ -7135,7 +7232,9 @@ async function storeCompact(sessionId, skillName, compactText, sourceSha) {
|
|
|
7135
7232
|
const safeSession = safeSessionFragment(sessionId);
|
|
7136
7233
|
const fileId = `${safeSession}@${sanitizeSkillId(name)}@compact`;
|
|
7137
7234
|
const dir = skillOutputsDir();
|
|
7138
|
-
|
|
7235
|
+
const redacted = redactSecrets(compactText);
|
|
7236
|
+
if (redacted.count > 0) recordStat("secret_redacted", 0, redacted.count, void 0, SKILLS_OUTPUT_SUBDIR);
|
|
7237
|
+
let text = redacted.text;
|
|
7139
7238
|
if (sourceSha) {
|
|
7140
7239
|
text = `<!-- source_sha: ${sourceSha.slice(0, 12)} -->
|
|
7141
7240
|
${text}`;
|
|
@@ -7587,7 +7686,9 @@ async function probeImageMeta(input) {
|
|
|
7587
7686
|
const sharp = await loadSharp();
|
|
7588
7687
|
if (sharp === null) return null;
|
|
7589
7688
|
try {
|
|
7590
|
-
const
|
|
7689
|
+
const cfg = loadConfig().image_shrink;
|
|
7690
|
+
const limitInputPixels = cfg.max_image_pixels > 0 ? cfg.max_image_pixels : false;
|
|
7691
|
+
const meta = await sharp(input, { limitInputPixels }).metadata();
|
|
7591
7692
|
return { width: meta.width ?? 0, height: meta.height ?? 0, format: meta.format ?? null, pages: meta.pages ?? 1 };
|
|
7592
7693
|
} catch (e) {
|
|
7593
7694
|
throw new ImageDecodeError(e?.message ?? "image could not be decoded");
|
|
@@ -11229,7 +11330,7 @@ function preReadHandlerInner(event) {
|
|
|
11229
11330
|
const isSourceExt = isSourceExtension(basename12);
|
|
11230
11331
|
if (isSourceExt && reads >= 2) {
|
|
11231
11332
|
recordStat("read_count_deny", rereadCredit, Math.round(rereadCredit / 4));
|
|
11232
|
-
recordStat("session_hint",
|
|
11333
|
+
recordStat("session_hint", 0, 0);
|
|
11233
11334
|
return denyOutput(
|
|
11234
11335
|
"Read this file " + reads + ' times already \u2014 use `token-goat read "' + shown + '::Symbol"`, `token-goat skeleton ' + shown + "`, or `token-goat outline " + shown + "` to pull just the part you need. " + editAnywayHint(normalized)
|
|
11235
11336
|
);
|
|
@@ -12106,13 +12207,8 @@ function mapLookupBytesSaved(map, emittedText) {
|
|
|
12106
12207
|
...map.recentFiles.map((f) => normalizePath(path22.resolve(map.rootDir, f))),
|
|
12107
12208
|
...map.topSymbols.map((s) => normalizePath(s.filePath))
|
|
12108
12209
|
]);
|
|
12109
|
-
|
|
12110
|
-
|
|
12111
|
-
try {
|
|
12112
|
-
fullSourceBytes += fs26.statSync(fp).size;
|
|
12113
|
-
} catch {
|
|
12114
|
-
}
|
|
12115
|
-
}
|
|
12210
|
+
const listingText = Array.from(referencedFiles).sort().join("\n");
|
|
12211
|
+
const fullSourceBytes = Buffer.byteLength(listingText, "utf8");
|
|
12116
12212
|
const emittedBytes = Buffer.byteLength(emittedText, "utf8");
|
|
12117
12213
|
return Math.max(1, fullSourceBytes - emittedBytes);
|
|
12118
12214
|
}
|
|
@@ -13581,7 +13677,7 @@ function extractDart(content, filePath) {
|
|
|
13581
13677
|
const line = stripLineComment(blockStripped).trimEnd();
|
|
13582
13678
|
const stripped = line.trim();
|
|
13583
13679
|
if (!stripped) {
|
|
13584
|
-
const braceLine2 = stripStringLiterals(line);
|
|
13680
|
+
const braceLine2 = stripStringLiterals(line, { tripleQuotes: true });
|
|
13585
13681
|
braceDepth += (braceLine2.match(/\{/g) ?? []).length - (braceLine2.match(/\}/g) ?? []).length;
|
|
13586
13682
|
continue;
|
|
13587
13683
|
}
|
|
@@ -13675,7 +13771,7 @@ function extractDart(content, filePath) {
|
|
|
13675
13771
|
symbols.push(makeLineSymbol(filePath, fname, "function", lineNum, stripped.slice(0, 200), void 0, lines2, "c"));
|
|
13676
13772
|
}
|
|
13677
13773
|
}
|
|
13678
|
-
const braceLine = stripStringLiterals(line);
|
|
13774
|
+
const braceLine = stripStringLiterals(line, { tripleQuotes: true });
|
|
13679
13775
|
for (const ch of braceLine) {
|
|
13680
13776
|
if (ch === "{") {
|
|
13681
13777
|
braceDepth++;
|
|
@@ -17572,8 +17668,8 @@ var NO_TREE_SITTER_EXTRACTORS = {
|
|
|
17572
17668
|
toml: extractTomlSymbols,
|
|
17573
17669
|
css: extractCssSymbols,
|
|
17574
17670
|
dockerfile: extractDockerfileSymbols,
|
|
17575
|
-
csharp: (content, filePath) => assignBraceBlockSpans(extractCsharp(content, filePath).symbols, content, "//"),
|
|
17576
|
-
php: (content, filePath) => assignBraceBlockSpans(extractPhp(content, filePath).symbols, content, "//"),
|
|
17671
|
+
csharp: (content, filePath) => assignBraceBlockSpans(extractCsharp(content, filePath).symbols, content, { lineComment: "//", stringEscapes: "csharp", rawStringQuotes: true }),
|
|
17672
|
+
php: (content, filePath) => assignBraceBlockSpans(extractPhp(content, filePath).symbols, content, { lineComment: ["//", "#"], lineCommentExceptions: ["#["] }),
|
|
17577
17673
|
html: (content, filePath) => {
|
|
17578
17674
|
const r = extractHtml(content, filePath);
|
|
17579
17675
|
return [...r.symbols, ...sectionsToHeadingSymbols(r.sections, filePath)];
|
|
@@ -17582,13 +17678,13 @@ var NO_TREE_SITTER_EXTRACTORS = {
|
|
|
17582
17678
|
const r = extractLiquid(content, filePath);
|
|
17583
17679
|
return [...r.symbols, ...sectionsToHeadingSymbols(r.sections, filePath)];
|
|
17584
17680
|
},
|
|
17585
|
-
kotlin: (content, filePath) => assignBraceBlockSpans(extractKotlin(content, filePath).symbols, content, "//"),
|
|
17586
|
-
swift: (content, filePath) => assignBraceBlockSpans(extractSwift(content, filePath).symbols, content, "//"),
|
|
17587
|
-
scala: (content, filePath) => assignBraceBlockSpans(extractScala(content, filePath).symbols, content, "//"),
|
|
17681
|
+
kotlin: (content, filePath) => assignBraceBlockSpans(extractKotlin(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true }),
|
|
17682
|
+
swift: (content, filePath) => assignBraceBlockSpans(extractSwift(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true }),
|
|
17683
|
+
scala: (content, filePath) => assignBraceBlockSpans(extractScala(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true }),
|
|
17588
17684
|
lua: (content, filePath) => extractLua(content, filePath).symbols,
|
|
17589
17685
|
elixir: (content, filePath) => extractElixir(content, filePath).symbols,
|
|
17590
|
-
dart: (content, filePath) => assignBraceBlockSpans(extractDart(content, filePath).symbols, content, "//"),
|
|
17591
|
-
zig: (content, filePath) => assignBraceBlockSpans(extractZig(content, filePath).symbols, content, "//"),
|
|
17686
|
+
dart: (content, filePath) => assignBraceBlockSpans(extractDart(content, filePath).symbols, content, { lineComment: "//", nestedBlockComments: true, tripleQuote: true, tripleSingleQuote: true }),
|
|
17687
|
+
zig: (content, filePath) => assignBraceBlockSpans(extractZig(content, filePath).symbols, content, { lineComment: "//", blockComment: null, lineStringPrefix: "\\\\" }),
|
|
17592
17688
|
r: (content, filePath) => extractR(content, filePath).symbols,
|
|
17593
17689
|
graphql: (content, filePath) => extractGraphql(content, filePath).symbols,
|
|
17594
17690
|
sql: extractSql,
|
|
@@ -17596,7 +17692,7 @@ var NO_TREE_SITTER_EXTRACTORS = {
|
|
|
17596
17692
|
makefile: extractMakefile,
|
|
17597
17693
|
proto: (content, filePath) => extractProto(content, filePath).symbols,
|
|
17598
17694
|
terraform: extractTerraform,
|
|
17599
|
-
powershell: (content, filePath) => assignBraceBlockSpans(extractPowershell(content, filePath).symbols, content, "#"),
|
|
17695
|
+
powershell: (content, filePath) => assignBraceBlockSpans(extractPowershell(content, filePath).symbols, content, { lineComment: "#", stringEscapes: "powershell" }),
|
|
17600
17696
|
apex: (content, filePath) => extractApex(content, filePath).symbols,
|
|
17601
17697
|
salesforce_metadata: (content, filePath) => extractSalesforceMetadata(content, filePath).symbols,
|
|
17602
17698
|
env_file: extractEnv,
|
|
@@ -18945,6 +19041,11 @@ export {
|
|
|
18945
19041
|
locatePdfPages,
|
|
18946
19042
|
extractPdfOutline,
|
|
18947
19043
|
extractPdfMeta,
|
|
19044
|
+
MAX_ZIP_INPUT_BYTES,
|
|
19045
|
+
MAX_ZIP_OUTPUT_BYTES,
|
|
19046
|
+
ZipOutputTooLargeError,
|
|
19047
|
+
ZipInputTooLargeError,
|
|
19048
|
+
unzipBounded,
|
|
18948
19049
|
docxOutline,
|
|
18949
19050
|
docxText,
|
|
18950
19051
|
pptxOutline,
|
|
@@ -18979,6 +19080,7 @@ export {
|
|
|
18979
19080
|
passOutput,
|
|
18980
19081
|
denyOutput,
|
|
18981
19082
|
contextOutput,
|
|
19083
|
+
emitRewrite,
|
|
18982
19084
|
makeDedupHintHandlers,
|
|
18983
19085
|
registerHook,
|
|
18984
19086
|
runHook,
|
|
@@ -19092,7 +19194,9 @@ export {
|
|
|
19092
19194
|
scanForInjectionPatterns,
|
|
19093
19195
|
UNTRUSTED_WEB_TAG,
|
|
19094
19196
|
fenceUntrustedContent,
|
|
19197
|
+
UNTRUSTED_FILE_TAG,
|
|
19095
19198
|
UNTRUSTED_TOOL_TAG,
|
|
19199
|
+
UNTRUSTED_GITHUB_TAG,
|
|
19096
19200
|
SKILLS_OUTPUT_SUBDIR,
|
|
19097
19201
|
skillOutputsDir,
|
|
19098
19202
|
contentHash,
|
|
@@ -19101,6 +19205,7 @@ export {
|
|
|
19101
19205
|
extractChecklistSection,
|
|
19102
19206
|
listOutputs,
|
|
19103
19207
|
hasSessionOutput,
|
|
19208
|
+
sessionOutputBodyBytes,
|
|
19104
19209
|
storeOutput,
|
|
19105
19210
|
storeCompact,
|
|
19106
19211
|
incrementSkillHit,
|