token-goat 2.8.2 → 2.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -18
- package/dist/{token-goat-chunk-AM23GDIS.mjs → token-goat-chunk-4OTIB7SB.mjs} +579 -162
- package/dist/{token-goat-chunk-TF5NT3H5.mjs → token-goat-chunk-CZALRRGN.mjs} +192 -60
- package/dist/{token-goat-chunk-IVCTQPZD.mjs → token-goat-chunk-E76UNTVK.mjs} +407 -215
- package/dist/{token-goat-chunk-SE6V7LUZ.mjs → token-goat-chunk-HNODKNWO.mjs} +5 -5
- package/dist/{token-goat-chunk-5MLXFSRI.mjs → token-goat-chunk-IZRXU64B.mjs} +51 -4
- package/dist/{token-goat-chunk-JU7QESB3.mjs → token-goat-chunk-L6YP6FKQ.mjs} +5 -5
- package/dist/{token-goat-chunk-AL644JUV.mjs → token-goat-chunk-LZOAPGWR.mjs} +6 -5
- package/dist/{token-goat-chunk-BTZSUIYA.mjs → token-goat-chunk-NZFTWBGV.mjs} +2 -2
- package/dist/{token-goat-chunk-LILS6TIU.mjs → token-goat-chunk-RE7S7H26.mjs} +182 -65
- package/dist/{token-goat-chunk-2MWF3OGR.mjs → token-goat-chunk-U4UI5F3S.mjs} +1072 -218
- package/dist/{token-goat-chunk-KBVN4ELV.mjs → token-goat-chunk-ZQ3PUOP3.mjs} +379 -147
- package/dist/token-goat-hook.mjs +5 -5
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +3 -1
|
@@ -86,7 +86,7 @@ import {
|
|
|
86
86
|
runZipRead,
|
|
87
87
|
symbolNamesInFile,
|
|
88
88
|
upsertNote
|
|
89
|
-
} from "./token-goat-chunk-
|
|
89
|
+
} from "./token-goat-chunk-RE7S7H26.mjs";
|
|
90
90
|
import {
|
|
91
91
|
BASH_OUTPUT_SUBDIR,
|
|
92
92
|
GEMINI_TOOL_NAME_MAP,
|
|
@@ -113,16 +113,19 @@ import {
|
|
|
113
113
|
searchRecall,
|
|
114
114
|
storeWebOutput,
|
|
115
115
|
summarizeResidentContext
|
|
116
|
-
} from "./token-goat-chunk-
|
|
116
|
+
} from "./token-goat-chunk-IZRXU64B.mjs";
|
|
117
117
|
import {
|
|
118
118
|
AGENT_SALT_MARKER,
|
|
119
119
|
CONTEXT_AUTOCOMPACT_TOKENS,
|
|
120
120
|
DEFAULT_MAX_AGE_MS,
|
|
121
121
|
DEFAULT_MAX_COUNT,
|
|
122
|
+
MATERIALIZE_SHRUNK_IMAGE_JS,
|
|
122
123
|
MAX_FILES_SCANNED,
|
|
124
|
+
PARSER_FINGERPRINT,
|
|
123
125
|
SESSIONS_SUBDIR,
|
|
124
126
|
SKILLS_OUTPUT_SUBDIR,
|
|
125
127
|
SKIP_DIRS,
|
|
128
|
+
UNTRUSTED_FILE_TAG,
|
|
126
129
|
UNTRUSTED_TOOL_TAG,
|
|
127
130
|
UNTRUSTED_WEB_TAG,
|
|
128
131
|
WORKER_HEARTBEAT_STALE_MS,
|
|
@@ -158,6 +161,7 @@ import {
|
|
|
158
161
|
extractCompactFromMarker,
|
|
159
162
|
extractNamedSection,
|
|
160
163
|
fenceUntrustedContent,
|
|
164
|
+
fenceUntrustedOcrText,
|
|
161
165
|
findClaudeMdFiles,
|
|
162
166
|
findContentDuplicates,
|
|
163
167
|
findLatestSessionId,
|
|
@@ -256,7 +260,7 @@ import {
|
|
|
256
260
|
vscodeDecoderConfigured,
|
|
257
261
|
walkProject,
|
|
258
262
|
writeCompact
|
|
259
|
-
} from "./token-goat-chunk-
|
|
263
|
+
} from "./token-goat-chunk-4OTIB7SB.mjs";
|
|
260
264
|
import {
|
|
261
265
|
C,
|
|
262
266
|
CONFIG_KEY_ENV_OVERRIDES,
|
|
@@ -323,6 +327,7 @@ import {
|
|
|
323
327
|
runGit,
|
|
324
328
|
safeSlice,
|
|
325
329
|
saveConfig,
|
|
330
|
+
savedTokensFromBytes,
|
|
326
331
|
sleepSync,
|
|
327
332
|
stripAnsi,
|
|
328
333
|
stripOwnHooksFromMap,
|
|
@@ -338,7 +343,7 @@ import {
|
|
|
338
343
|
withFileLock,
|
|
339
344
|
withRetryOnLock,
|
|
340
345
|
writeJsonSettings
|
|
341
|
-
} from "./token-goat-chunk-
|
|
346
|
+
} from "./token-goat-chunk-E76UNTVK.mjs";
|
|
342
347
|
import {
|
|
343
348
|
__export
|
|
344
349
|
} from "./token-goat-chunk-AEX54RUZ.mjs";
|
|
@@ -3457,9 +3462,9 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
3457
3462
|
* @param {string} [path]
|
|
3458
3463
|
* @return {(string|null|Command)}
|
|
3459
3464
|
*/
|
|
3460
|
-
executableDir(
|
|
3461
|
-
if (
|
|
3462
|
-
this._executableDir =
|
|
3465
|
+
executableDir(path29) {
|
|
3466
|
+
if (path29 === void 0) return this._executableDir;
|
|
3467
|
+
this._executableDir = path29;
|
|
3463
3468
|
return this;
|
|
3464
3469
|
}
|
|
3465
3470
|
/**
|
|
@@ -3758,9 +3763,9 @@ function attemptedCommandName(argv) {
|
|
|
3758
3763
|
}
|
|
3759
3764
|
|
|
3760
3765
|
// src/cli.ts
|
|
3761
|
-
import * as
|
|
3762
|
-
import * as
|
|
3763
|
-
import { homedir as
|
|
3766
|
+
import * as fs28 from "fs";
|
|
3767
|
+
import * as path28 from "path";
|
|
3768
|
+
import { homedir as homedir13 } from "os";
|
|
3764
3769
|
|
|
3765
3770
|
// src/walk_index.ts
|
|
3766
3771
|
import * as fs2 from "node:fs";
|
|
@@ -4094,10 +4099,11 @@ import * as os from "node:os";
|
|
|
4094
4099
|
import * as path from "node:path";
|
|
4095
4100
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
4096
4101
|
|
|
4097
|
-
// pi built-in tool names -> token-goat internal tool names
|
|
4102
|
+
// pi built-in tool names -> token-goat internal tool names. powershell is pi's Windows shell twin of bash -- same input schema (PowerShellToolInput = BashToolInput, i.e. command/timeout) and registered under its own name (badlogic/pi-mono packages/coding-agent/src/core/tools/powershell.ts + tools/index.ts's ToolName union). Unmapped, every shell command in a powershell-tool pi session bypassed the Bash hooks entirely, the same shape as the Copilot shim's bash/powershell pairing.
|
|
4098
4103
|
const TOOL_TO_TG: Record<string, string> = {
|
|
4099
4104
|
read: "Read",
|
|
4100
4105
|
bash: "Bash",
|
|
4106
|
+
powershell: "Bash",
|
|
4101
4107
|
edit: "Edit",
|
|
4102
4108
|
write: "Write",
|
|
4103
4109
|
grep: "Grep",
|
|
@@ -4108,15 +4114,14 @@ const TOOL_TO_TG: Record<string, string> = {
|
|
|
4108
4114
|
const ARGS_TO_TG: Record<string, Record<string, string>> = {
|
|
4109
4115
|
read: { path: "file_path", offset: "offset", limit: "limit" },
|
|
4110
4116
|
bash: { command: "command", timeout: "timeout" },
|
|
4117
|
+
powershell: { command: "command", timeout: "timeout" },
|
|
4111
4118
|
edit: { path: "file_path" },
|
|
4112
4119
|
write: { path: "file_path" },
|
|
4113
4120
|
grep: { pattern: "pattern", path: "path" },
|
|
4114
4121
|
find: { pattern: "pattern", path: "path" },
|
|
4115
4122
|
};
|
|
4116
4123
|
|
|
4117
|
-
// Tools
|
|
4118
|
-
// pre_tool_use handler in token-goat (only Read/Grep/Bash/WebFetch do), so
|
|
4119
|
-
// it's excluded here rather than spawning a hook call that always no-ops.
|
|
4124
|
+
// Tools with a pre-hook whose output shape this extension can act on: deny ({block}), updatedInput (in-place arg rewrite), or the Read image-shrink materialization. Glob's pre handler (preGlobDedupHandler in hooks_glob.ts -- the old claim here that Glob has no pre handler was stale) emits only an advisory contextOutput hint, which pi's tool_call contract has no channel for (see the module docblock), so calling it would cost a hook call per find only to drop the answer; it stays excluded for that reason.
|
|
4120
4125
|
const PRE_HOOK_TOOLS = new Set(["Read", "Grep", "Bash", "WebFetch"]);
|
|
4121
4126
|
|
|
4122
4127
|
// resolveEntryPath reads a sidecar JSON file (token-goat-entry.json, written by
|
|
@@ -4268,6 +4273,30 @@ function extractUpdatedInput(resp: Record<string, unknown>): Record<string, unkn
|
|
|
4268
4273
|
// since pi's tool_call handler has no context-injection channel -- only
|
|
4269
4274
|
// in-place arg mutation. Returns undefined (leaving the read path untouched)
|
|
4270
4275
|
// if the context isn't a shrink payload or anything goes wrong writing it.
|
|
4276
|
+
// Best-effort sweep of previously materialized shrunk copies in the OS temp dir: typed twin of pruneMaterializedShrinks in shrink_block.ts (MATERIALIZE_SHRUNK_IMAGE_JS), which documents why pi keeps its own copy. The temp file only needs to outlive the single tool call whose path was rewritten to it, so anything older than an hour is finished with; the "token-goat-shrink-" prefix check confines the sweep to this mechanism's own files.
|
|
4277
|
+
const MATERIALIZED_SHRINK_MAX_AGE_MS = 60 * 60 * 1000;
|
|
4278
|
+
let lastMaterializedShrinkSweepAtMs = 0;
|
|
4279
|
+
function pruneMaterializedShrinks(): void {
|
|
4280
|
+
const now = Date.now();
|
|
4281
|
+
if (now - lastMaterializedShrinkSweepAtMs < MATERIALIZED_SHRINK_MAX_AGE_MS) return;
|
|
4282
|
+
lastMaterializedShrinkSweepAtMs = now;
|
|
4283
|
+
try {
|
|
4284
|
+
const dir = os.tmpdir();
|
|
4285
|
+
for (const file of fs.readdirSync(dir)) {
|
|
4286
|
+
if (!file.startsWith("token-goat-shrink-")) continue;
|
|
4287
|
+
const full = path.join(dir, file);
|
|
4288
|
+
try {
|
|
4289
|
+
const st = fs.statSync(full);
|
|
4290
|
+
if (st.isFile() && now - st.mtimeMs > MATERIALIZED_SHRINK_MAX_AGE_MS) fs.unlinkSync(full);
|
|
4291
|
+
} catch {
|
|
4292
|
+
// Best-effort per-file cleanup; one bad stat/unlink must not abort the sweep.
|
|
4293
|
+
}
|
|
4294
|
+
}
|
|
4295
|
+
} catch {
|
|
4296
|
+
// Best-effort; a readdir failure must never break the materialization below.
|
|
4297
|
+
}
|
|
4298
|
+
}
|
|
4299
|
+
|
|
4271
4300
|
function materializeShrunkImage(context: string | undefined): string | undefined {
|
|
4272
4301
|
if (typeof context !== "string") return undefined;
|
|
4273
4302
|
const idx = context.indexOf("data:image/");
|
|
@@ -4275,6 +4304,7 @@ function materializeShrunkImage(context: string | undefined): string | undefined
|
|
|
4275
4304
|
const match = /^data:image\\/([a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=]+)$/.exec(context.slice(idx).trim());
|
|
4276
4305
|
if (!match) return undefined;
|
|
4277
4306
|
try {
|
|
4307
|
+
pruneMaterializedShrinks();
|
|
4278
4308
|
const buf = Buffer.from(match[2], "base64");
|
|
4279
4309
|
const name = \`token-goat-shrink-\${process.pid}-\${Date.now()}-\${Math.random().toString(36).slice(2)}.\${match[1]}\`;
|
|
4280
4310
|
const file = path.join(os.tmpdir(), name);
|
|
@@ -4360,13 +4390,35 @@ export default function (pi: ExtensionAPI) {
|
|
|
4360
4390
|
.filter((c: { type?: string }) => c && c.type === "text")
|
|
4361
4391
|
.map((c: { text?: string }) => c.text ?? "")
|
|
4362
4392
|
.join("\\n");
|
|
4363
|
-
await callHook("post_tool_use", {
|
|
4393
|
+
const resp = await callHook("post_tool_use", {
|
|
4364
4394
|
session_id: sessionId,
|
|
4365
4395
|
tool_name: tg,
|
|
4366
4396
|
tool_input: toToolInput(event.toolName, (event.input ?? {}) as Record<string, unknown>),
|
|
4367
4397
|
cwd,
|
|
4368
4398
|
tool_response: { output },
|
|
4369
4399
|
});
|
|
4400
|
+
if (!resp) return;
|
|
4401
|
+
|
|
4402
|
+
// rewriteOutput: replace the tool result the model sees. pi's tool_result handler CAN modify the result: its own types declare "Fired after a tool executes. Can modify result." over ToolResultEvent and a ToolResultEventResult of { content, details, isError, usage }, and the shipped runtime applies it (dist/core/extensions/runner.js emitToolResult assigns handlerResult.content onto the event, and dist/core/agent-session.js afterToolCall returns hookResult.content as the tool result content). Without this, injection fencing, secret redaction and output compression were computed and thrown away on every pi session, the same whitelist-shaped drop fixed for opencode.
|
|
4403
|
+
const hso = resp["hookSpecificOutput"] as Record<string, unknown> | undefined;
|
|
4404
|
+
const updatedToolOutput = hso && typeof hso["updatedToolOutput"] === "string" ? (hso["updatedToolOutput"] as string) : undefined;
|
|
4405
|
+
if (updatedToolOutput === undefined) return;
|
|
4406
|
+
|
|
4407
|
+
// Replace only the text blocks, in place, and keep every non-text block (images) untouched: emitToolResult overwrites content wholesale, so anything not carried here is dropped from what the model sees. details/isError/usage are deliberately not returned, since runner.js only overwrites the fields a handler actually provides.
|
|
4408
|
+
const rewritten: typeof contentBlocks = [];
|
|
4409
|
+
let replaced = false;
|
|
4410
|
+
for (const block of contentBlocks) {
|
|
4411
|
+
const isText = Boolean(block) && (block as { type?: string }).type === "text";
|
|
4412
|
+
if (!isText) {
|
|
4413
|
+
rewritten.push(block);
|
|
4414
|
+
continue;
|
|
4415
|
+
}
|
|
4416
|
+
if (replaced) continue;
|
|
4417
|
+
replaced = true;
|
|
4418
|
+
rewritten.push({ type: "text", text: updatedToolOutput });
|
|
4419
|
+
}
|
|
4420
|
+
if (!replaced) rewritten.push({ type: "text", text: updatedToolOutput });
|
|
4421
|
+
return { content: rewritten };
|
|
4370
4422
|
});
|
|
4371
4423
|
|
|
4372
4424
|
// Compaction: pi's session_before_compact REPLACES the summary rather than
|
|
@@ -4553,7 +4605,7 @@ import os from "node:os"
|
|
|
4553
4605
|
import path from "node:path"
|
|
4554
4606
|
import { fileURLToPath, pathToFileURL } from "node:url"
|
|
4555
4607
|
|
|
4556
|
-
// opencode built-in tool id -> token-goat canonical tool name.
|
|
4608
|
+
// opencode built-in tool id -> token-goat canonical tool name. websearch/skill/task were re-verified against opencode's own source at the tag matching the installed release (anomalyco/opencode v1.18.16 packages/opencode/src/tool/): WebSearchTool registers as "websearch" (websearch.ts), SkillTool as "skill" (skill.ts), TaskTool as "task" (task.ts, params prompt/subagent_type/description -- the exact keys token-goat's hooks_agent_spawn.ts reads, which registers a lowercase 'task' handler). Unmapped, all three were dead mechanisms here: no WebSearch repeat-search deny/compression, no repeat-skill-load deny, no agent-spawn briefing or report compaction. apply_patch (patchText only, no per-file path) and lsp/plan/question/todo have no token-goat equivalent and stay unmapped.
|
|
4557
4609
|
const TOOL_TO_TG = {
|
|
4558
4610
|
read: "Read",
|
|
4559
4611
|
bash: "Bash",
|
|
@@ -4562,15 +4614,15 @@ const TOOL_TO_TG = {
|
|
|
4562
4614
|
grep: "Grep",
|
|
4563
4615
|
glob: "Glob",
|
|
4564
4616
|
webfetch: "WebFetch",
|
|
4617
|
+
websearch: "WebSearch",
|
|
4618
|
+
skill: "Skill",
|
|
4619
|
+
task: "task",
|
|
4565
4620
|
}
|
|
4566
4621
|
|
|
4567
|
-
// Tools with a
|
|
4568
|
-
|
|
4569
|
-
// in token-goat at all, so skip the subprocess call for them entirely. Glob
|
|
4570
|
-
// has none either, so it's excluded too rather than spawning a no-op call.
|
|
4571
|
-
const PRE_HOOK_TOOLS = new Set(["read", "bash", "grep", "webfetch"])
|
|
4622
|
+
// Tools with a pre_tool_use handler server-side whose OUTPUT SHAPE this hook can act on: deny (throw), updatedInput (args rewrite), or the Read image-shrink materialization. websearch (repeat-search deny within the dedup TTL, hooks_websearch.ts), skill (repeat-load deny, hooks_skill.ts) and task (briefing updatedInput rewrite, hooks_agent_spawn.ts) all qualify. Edit/Write have no pre-hook at all. Glob DOES have one (preGlobDedupHandler, hooks_glob.ts) -- the old claim here that it has none was stale -- but its only output is an advisory contextOutput hint, and tool.execute.before has no context channel (see the module docblock), so calling it would spawn a subprocess per glob only to drop the answer; it stays excluded for that reason, not the old one.
|
|
4623
|
+
const PRE_HOOK_TOOLS = new Set(["read", "bash", "grep", "webfetch", "websearch", "skill", "task"])
|
|
4572
4624
|
|
|
4573
|
-
// opencode tool args (camelCase) -> token-goat snake_case tool_input keys.
|
|
4625
|
+
// opencode tool args (camelCase) -> token-goat snake_case tool_input keys. websearch's query (websearch.ts Parameters), skill's name (skill.ts Parameters; hooks_skill.ts reads tool_input['skill']) and task's prompt/subagent_type/description (task.ts BaseParameterFields) are all from opencode's own schemas at v1.18.16.
|
|
4574
4626
|
const ARGS_TO_TG = {
|
|
4575
4627
|
read: { filePath: "file_path", offset: "offset", limit: "limit" },
|
|
4576
4628
|
bash: { command: "command", timeout: "timeout" },
|
|
@@ -4579,6 +4631,9 @@ const ARGS_TO_TG = {
|
|
|
4579
4631
|
grep: { pattern: "pattern", path: "path" },
|
|
4580
4632
|
glob: { pattern: "pattern", path: "path" },
|
|
4581
4633
|
webfetch: { url: "url" },
|
|
4634
|
+
websearch: { query: "query" },
|
|
4635
|
+
skill: { name: "skill" },
|
|
4636
|
+
task: { prompt: "prompt", subagent_type: "subagent_type", description: "description" },
|
|
4582
4637
|
}
|
|
4583
4638
|
|
|
4584
4639
|
function reverseArgMap(tool) {
|
|
@@ -4614,28 +4669,7 @@ function extractUpdatedInput(resp) {
|
|
|
4614
4669
|
return hso && typeof hso === "object" ? hso.updatedInput : undefined
|
|
4615
4670
|
}
|
|
4616
4671
|
|
|
4617
|
-
|
|
4618
|
-
// ("<summary>\\ndata:image/<fmt>;base64,<data>") into a real file on disk,
|
|
4619
|
-
// since tool.execute.before has no context-injection channel to hand the
|
|
4620
|
-
// shrunk image to the model directly -- only output.args mutation. Returns
|
|
4621
|
-
// undefined (leaving output.args untouched) if the context isn't a shrink
|
|
4622
|
-
// payload or anything goes wrong writing it.
|
|
4623
|
-
function materializeShrunkImage(context) {
|
|
4624
|
-
if (typeof context !== "string") return undefined
|
|
4625
|
-
const idx = context.indexOf("data:image/")
|
|
4626
|
-
if (idx === -1) return undefined
|
|
4627
|
-
const match = /^data:image\\/([a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=]+)$/.exec(context.slice(idx).trim())
|
|
4628
|
-
if (!match) return undefined
|
|
4629
|
-
try {
|
|
4630
|
-
const buf = Buffer.from(match[2], "base64")
|
|
4631
|
-
const name = \`token-goat-shrink-\${process.pid}-\${Date.now()}-\${Math.random().toString(36).slice(2)}.\${match[1]}\`
|
|
4632
|
-
const file = path.join(os.tmpdir(), name)
|
|
4633
|
-
fs.writeFileSync(file, buf)
|
|
4634
|
-
return file
|
|
4635
|
-
} catch {
|
|
4636
|
-
return undefined
|
|
4637
|
-
}
|
|
4638
|
-
}
|
|
4672
|
+
${MATERIALIZE_SHRUNK_IMAGE_JS}
|
|
4639
4673
|
|
|
4640
4674
|
export const TokenGoatPlugin = async ({ directory }) => {
|
|
4641
4675
|
return {
|
|
@@ -4685,6 +4719,14 @@ export const TokenGoatPlugin = async ({ directory }) => {
|
|
|
4685
4719
|
})
|
|
4686
4720
|
if (!resp) return
|
|
4687
4721
|
|
|
4722
|
+
// rewriteOutput: replace the tool result wholesale. This is the shape every post_tool_use rewrite producer emits (WebFetch injection fencing and secret redaction, websearch redaction, bash/grep output compression, agent-report compaction), and it went unapplied here for the same whitelist reason the response-contract docblock above used to omit it -- the plugin only handled the shapes its author listed, so opencode sessions got the appended hint channel but never a fenced, redacted or compressed result. Mutating output.output is the exact mechanism the context append below already relies on, so this claims no capability the append did not.
|
|
4723
|
+
const hso = resp.hookSpecificOutput
|
|
4724
|
+
const updatedToolOutput = hso && typeof hso.updatedToolOutput === "string" ? hso.updatedToolOutput : undefined
|
|
4725
|
+
if (updatedToolOutput !== undefined) {
|
|
4726
|
+
output.output = updatedToolOutput
|
|
4727
|
+
return
|
|
4728
|
+
}
|
|
4729
|
+
|
|
4688
4730
|
const context = extractContext(resp)
|
|
4689
4731
|
if (context && typeof output.output === "string") {
|
|
4690
4732
|
output.output += \`\\n\\n[token-goat] \${context}\`
|
|
@@ -4739,15 +4781,11 @@ var OPENCLAW_PLUGIN_SCRIPT = `// token-goat bridge plugin for OpenClaw
|
|
|
4739
4781
|
// Bridges OpenClaw's tool-call and session hooks to token-goat's subprocess
|
|
4740
4782
|
// hook protocol. https://github.com/DFKHelper/token-goat
|
|
4741
4783
|
//
|
|
4742
|
-
// NOTE ON PARAMETER SHAPES: OpenClaw's tool-
|
|
4743
|
-
// token-goat's tool_input unchanged (no key remapping) -- verified against
|
|
4744
|
-
// OpenClaw's own event-type source that these keys are already snake_case
|
|
4745
|
-
// (file_path, command, etc.), matching token-goat's own convention. The
|
|
4746
|
-
// built-in tool NAME list below is broader than any single source confirms,
|
|
4747
|
-
// since an unmatched name is a harmless no-op here, not a wrong rewrite.
|
|
4784
|
+
// NOTE ON PARAMETER SHAPES: OpenClaw's bash tool really does use token-goat's own key names (command/timeout), but the path-carrying tools (read/edit/write) send the file path under "path", not "file_path" -- read out of OpenClaw's own tool input schemas (src/agents/sessions/tools/read-tool-contract.ts, edit.ts, bash.ts in openclaw/openclaw). toToolInput below ADDS the canonical file_path alongside the original path so token-goat's getFilePath()-based handlers can see it, without renaming anything. The built-in tool NAME list below is broader than any single source confirms, since an unmatched name is a harmless no-op here, not a wrong rewrite.
|
|
4748
4785
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
4749
4786
|
import { spawnSync } from "node:child_process";
|
|
4750
4787
|
import * as fs from "node:fs";
|
|
4788
|
+
import * as os from "node:os";
|
|
4751
4789
|
import * as path from "node:path";
|
|
4752
4790
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
4753
4791
|
|
|
@@ -4758,20 +4796,43 @@ const TOOL_TO_TG = {
|
|
|
4758
4796
|
edit: "Edit",
|
|
4759
4797
|
apply_patch: "Edit",
|
|
4760
4798
|
exec: "Bash",
|
|
4799
|
+
bash: "Bash",
|
|
4761
4800
|
grep: "Grep",
|
|
4762
4801
|
glob: "Glob",
|
|
4802
|
+
find: "Glob",
|
|
4763
4803
|
webfetch: "WebFetch",
|
|
4764
4804
|
web_search: "WebFetch",
|
|
4765
4805
|
web_fetch: "WebFetch",
|
|
4766
4806
|
};
|
|
4767
4807
|
|
|
4768
|
-
// Tools with a
|
|
4769
|
-
// none; Glob has none either -- see pi.ts/opencode.ts's identical
|
|
4770
|
-
// PRE_HOOK_TOOLS note -- so it's excluded to avoid a wasted no-op spawn).
|
|
4808
|
+
// Tools with a pre_tool_use handler whose output shape this hook can act on: deny (block/blockReason), updatedInput (params rewrite), or the Read image-shrink materialization. Edit/Write have no pre handler at all. Glob (OpenClaw's find) DOES have one -- preGlobDedupHandler in hooks_glob.ts; the old claim here that it has none was stale -- but its only output is an advisory contextOutput hint and before_tool_call has no context channel, so calling it would spawn a hook subprocess per find only to drop the answer; it stays excluded for that reason (matching pi.ts/opencode.ts's corrected note).
|
|
4771
4809
|
const PRE_HOOK_TOOLS = new Set(["Read", "Grep", "Bash", "WebFetch"]);
|
|
4772
4810
|
|
|
4811
|
+
// OpenClaw tools whose file path travels under "path" (see the parameter-shapes note above). apply_patch is deliberately absent: its input is a patch document, not a single path.
|
|
4812
|
+
const PATH_ARG_TOOLS = new Set(["read", "write", "edit"]);
|
|
4813
|
+
|
|
4814
|
+
// Build token-goat's tool_input from OpenClaw's params: same object, plus the canonical file_path added alongside the original path for the tools that carry one. Never renames or drops a key.
|
|
4815
|
+
function toToolInput(toolName, params) {
|
|
4816
|
+
const p = params || {};
|
|
4817
|
+
if (PATH_ARG_TOOLS.has(toolName) && typeof p.path === "string" && p.file_path === undefined) {
|
|
4818
|
+
return { ...p, file_path: p.path };
|
|
4819
|
+
}
|
|
4820
|
+
return p;
|
|
4821
|
+
}
|
|
4822
|
+
|
|
4823
|
+
// Pull the additionalContext / systemMessage string out of a hook response, whichever shape it came back as (see the response-contract note in this module's header comment).
|
|
4824
|
+
function extractContext(resp) {
|
|
4825
|
+
if (!resp) return undefined;
|
|
4826
|
+
const hso = resp["hookSpecificOutput"];
|
|
4827
|
+
if (hso && typeof hso["additionalContext"] === "string") return hso["additionalContext"];
|
|
4828
|
+
if (typeof resp["systemMessage"] === "string") return resp["systemMessage"];
|
|
4829
|
+
return undefined;
|
|
4830
|
+
}
|
|
4831
|
+
|
|
4773
4832
|
${BRIDGE_RELAY_JS}
|
|
4774
4833
|
|
|
4834
|
+
${MATERIALIZE_SHRUNK_IMAGE_JS}
|
|
4835
|
+
|
|
4775
4836
|
export default definePluginEntry({
|
|
4776
4837
|
id: "token-goat",
|
|
4777
4838
|
name: "token-goat",
|
|
@@ -4807,7 +4868,7 @@ export default definePluginEntry({
|
|
|
4807
4868
|
const resp = await callHook("pre_tool_use", {
|
|
4808
4869
|
session_id: sid,
|
|
4809
4870
|
tool_name: tg,
|
|
4810
|
-
tool_input: event.params
|
|
4871
|
+
tool_input: toToolInput(event.toolName, event.params),
|
|
4811
4872
|
cwd: process.cwd(),
|
|
4812
4873
|
});
|
|
4813
4874
|
if (!resp) return {};
|
|
@@ -4826,6 +4887,12 @@ export default definePluginEntry({
|
|
|
4826
4887
|
return { params: { ...(event.params || {}), ...updated } };
|
|
4827
4888
|
}
|
|
4828
4889
|
|
|
4890
|
+
// Image shrink has no context channel here -- translate it into a rewritten path pointing at a materialized shrunk copy instead, via the same params-rewrite channel the updatedInput merge above already relies on. OpenClaw's read tool takes the path under "path" (see the parameter-shapes note above), and the rest of the original params are preserved.
|
|
4891
|
+
if (tg === "Read") {
|
|
4892
|
+
const shrunkPath = materializeShrunkImage(extractContext(resp));
|
|
4893
|
+
if (shrunkPath) return { params: { ...(event.params || {}), path: shrunkPath } };
|
|
4894
|
+
}
|
|
4895
|
+
|
|
4829
4896
|
return {};
|
|
4830
4897
|
});
|
|
4831
4898
|
|
|
@@ -4845,7 +4912,7 @@ export default definePluginEntry({
|
|
|
4845
4912
|
await callHook("post_tool_use", {
|
|
4846
4913
|
session_id: sid,
|
|
4847
4914
|
tool_name: tg,
|
|
4848
|
-
tool_input: event.params
|
|
4915
|
+
tool_input: toToolInput(event.toolName, event.params),
|
|
4849
4916
|
cwd: process.cwd(),
|
|
4850
4917
|
tool_response: toolResponse,
|
|
4851
4918
|
});
|
|
@@ -5694,7 +5761,7 @@ function renderTopSessionFilesFromDisk(topN = 5, overrideSessionsDir) {
|
|
|
5694
5761
|
var METHODOLOGY = {
|
|
5695
5762
|
estimate_scope: "Local estimate of content avoided or reduced by token-goat.",
|
|
5696
5763
|
billing: "They are not GitHub Copilot usage, provider-reported token consumption, or billing data.",
|
|
5697
|
-
byte_derived_formula: "Most read, hook, and command entries
|
|
5764
|
+
byte_derived_formula: "Most read, hook, and command entries go through savedTokensFromBytes in src/stats.ts, which is Math.round(bytes_saved / 4).",
|
|
5698
5765
|
filter_estimates: "Output compressors record their filter-calculated delta; image entries use the byte-derived approximation unless the source provides a narrower estimate.",
|
|
5699
5766
|
advisory_events: "Zero-byte, zero-token advisory events show that guidance fired, not that an agent followed it.",
|
|
5700
5767
|
audit: "Use stats --full or stats --json for source and command breakdowns; reconcile billing with provider-exported usage data."
|
|
@@ -5717,6 +5784,21 @@ function renderMethodology(json = false) {
|
|
|
5717
5784
|
""
|
|
5718
5785
|
].join("\n"));
|
|
5719
5786
|
}
|
|
5787
|
+
function statsJsonPayload(summary) {
|
|
5788
|
+
return {
|
|
5789
|
+
total_events: summary.total_events,
|
|
5790
|
+
total_bytes_saved: summary.total_bytes_saved,
|
|
5791
|
+
total_tokens_saved: summary.total_tokens_saved,
|
|
5792
|
+
by_kind: summary.by_kind,
|
|
5793
|
+
by_day: summary.by_day,
|
|
5794
|
+
by_project: summary.by_project,
|
|
5795
|
+
by_command: summary.by_command,
|
|
5796
|
+
by_source: summary.by_source,
|
|
5797
|
+
by_harness: summary.by_harness,
|
|
5798
|
+
counts: summary.counts,
|
|
5799
|
+
window_days: summary.window_days
|
|
5800
|
+
};
|
|
5801
|
+
}
|
|
5720
5802
|
function runStats(opts = {}) {
|
|
5721
5803
|
if (opts.methodology === true) {
|
|
5722
5804
|
renderMethodology(opts.json === true);
|
|
@@ -5725,19 +5807,7 @@ function runStats(opts = {}) {
|
|
|
5725
5807
|
const window = opts.windowDays ?? 30;
|
|
5726
5808
|
const summary = summarize(window, void 0, opts.homeDir);
|
|
5727
5809
|
if (opts.json === true) {
|
|
5728
|
-
|
|
5729
|
-
total_events: summary.total_events,
|
|
5730
|
-
total_bytes_saved: summary.total_bytes_saved,
|
|
5731
|
-
total_tokens_saved: summary.total_tokens_saved,
|
|
5732
|
-
by_kind: summary.by_kind,
|
|
5733
|
-
by_day: summary.by_day,
|
|
5734
|
-
by_project: summary.by_project,
|
|
5735
|
-
by_command: summary.by_command,
|
|
5736
|
-
by_source: summary.by_source,
|
|
5737
|
-
by_harness: summary.by_harness,
|
|
5738
|
-
window_days: summary.window_days
|
|
5739
|
-
};
|
|
5740
|
-
process.stdout.write(JSON.stringify(out2) + "\n");
|
|
5810
|
+
process.stdout.write(JSON.stringify(statsJsonPayload(summary)) + "\n");
|
|
5741
5811
|
return;
|
|
5742
5812
|
}
|
|
5743
5813
|
const renderOpts = { windowDays: window };
|
|
@@ -6286,7 +6356,9 @@ function checkUnmappedTools(dbPath) {
|
|
|
6286
6356
|
if (rows.length === 0) {
|
|
6287
6357
|
return { name, status: "ok", message: "every tool name seen so far reached a handler that wanted it" };
|
|
6288
6358
|
}
|
|
6289
|
-
const nearMisses = rows.filter(
|
|
6359
|
+
const nearMisses = rows.filter(
|
|
6360
|
+
(r) => r.near_miss !== null && r.near_miss !== void 0 && r.near_miss !== r.tool_name
|
|
6361
|
+
);
|
|
6290
6362
|
if (nearMisses.length > 0) {
|
|
6291
6363
|
const shown2 = nearMisses.slice(0, UNMAPPED_TOOL_SAMPLE).map((r) => `${r.harness} sent "${r.tool_name}" where "${r.near_miss}" is handled (${r.event_name}, ${r.hits}x)`);
|
|
6292
6364
|
const more2 = nearMisses.length > UNMAPPED_TOOL_SAMPLE ? ` (+${nearMisses.length - UNMAPPED_TOOL_SAMPLE} more)` : "";
|
|
@@ -9188,12 +9260,12 @@ function formatBudgetText(result, contextK) {
|
|
|
9188
9260
|
lines.push(` ${e.rel_path.padEnd(colW, " ")} ${String(e.lines).padStart(6, " ")} ${String(e.tokens).padStart(8, " ")}`);
|
|
9189
9261
|
}
|
|
9190
9262
|
lines.push(` ${"-".repeat(colW)} ${"-".repeat(6)} ${"-".repeat(8)}`);
|
|
9191
|
-
let
|
|
9263
|
+
let pct2 = "";
|
|
9192
9264
|
if (contextK) {
|
|
9193
|
-
|
|
9265
|
+
pct2 = ` (${Math.round(result.total_tokens / (contextK * 1e3) * 100)}% of ${contextK}K)`;
|
|
9194
9266
|
}
|
|
9195
9267
|
lines.push(
|
|
9196
|
-
` ${"Total".padEnd(colW, " ")} ${String(result.total_lines).padStart(6, " ")} ${String(result.total_tokens).padStart(8, " ")}${
|
|
9268
|
+
` ${"Total".padEnd(colW, " ")} ${String(result.total_lines).padStart(6, " ")} ${String(result.total_tokens).padStart(8, " ")}${pct2}`
|
|
9197
9269
|
);
|
|
9198
9270
|
if (result.skipped.length > 0) {
|
|
9199
9271
|
const skipped = result.skipped.slice(0, 5).join(", ");
|
|
@@ -11086,7 +11158,7 @@ function guardDeclarationRows(rows, budgetTokens) {
|
|
|
11086
11158
|
function recordDepDocsStat(fullSourceBytes, emittedText, packageName) {
|
|
11087
11159
|
const emittedBytes = Buffer.byteLength(emittedText, "utf8");
|
|
11088
11160
|
const bytesSaved = Math.max(1, fullSourceBytes - emittedBytes);
|
|
11089
|
-
recordStat("dep_docs", bytesSaved,
|
|
11161
|
+
recordStat("dep_docs", bytesSaved, savedTokensFromBytes(bytesSaved), void 0, packageName);
|
|
11090
11162
|
}
|
|
11091
11163
|
function runDepDocs(opts) {
|
|
11092
11164
|
const root = resolveProjectRoot({ project: opts.projectRoot ?? process.cwd() });
|
|
@@ -11503,15 +11575,15 @@ function cmdCompactHint(opts) {
|
|
|
11503
11575
|
const pressure = getContextPressure(cache ?? void 0);
|
|
11504
11576
|
const [manifest, eventCount] = sessionId !== null ? buildManifestWithCount(sessionId) : ["", 0];
|
|
11505
11577
|
const manifestTokens = estimateTokens(manifest);
|
|
11506
|
-
const
|
|
11578
|
+
const pct2 = (pressure.fillFraction * 100).toFixed(1);
|
|
11507
11579
|
if (opts.json === true) {
|
|
11508
|
-
const out2 = { tier: pressure.tier, fillFraction: pressure.fillFraction, pct: Number(
|
|
11580
|
+
const out2 = { tier: pressure.tier, fillFraction: pressure.fillFraction, pct: Number(pct2), manifestTokens, eventCount };
|
|
11509
11581
|
if (sessionId !== null) out2["sessionId"] = sessionId;
|
|
11510
11582
|
if (opts.trigger !== void 0) out2["trigger"] = opts.trigger;
|
|
11511
11583
|
process.stdout.write(JSON.stringify(out2, null, 2) + "\n");
|
|
11512
11584
|
return;
|
|
11513
11585
|
}
|
|
11514
|
-
process.stdout.write(`Compact hint \u2014 context: ${pressure.tier} (${
|
|
11586
|
+
process.stdout.write(`Compact hint \u2014 context: ${pressure.tier} (${pct2}% full)
|
|
11515
11587
|
`);
|
|
11516
11588
|
if (sessionId !== null) process.stdout.write(`Session: ${sessionId}
|
|
11517
11589
|
`);
|
|
@@ -11611,6 +11683,19 @@ function indexSizeBytes(dbPath) {
|
|
|
11611
11683
|
}
|
|
11612
11684
|
return total;
|
|
11613
11685
|
}
|
|
11686
|
+
function comparableSizes(result) {
|
|
11687
|
+
const walInFlight = result.checkpointBusy && result.afterBytes > result.beforeBytes;
|
|
11688
|
+
const before = walInFlight ? result.beforeDbBytes : result.beforeBytes;
|
|
11689
|
+
const after = walInFlight ? result.afterDbBytes : result.afterBytes;
|
|
11690
|
+
return { before, after, freed: before - after };
|
|
11691
|
+
}
|
|
11692
|
+
function dbFileBytes(dbPath) {
|
|
11693
|
+
try {
|
|
11694
|
+
return fs18.statSync(dbPath).size;
|
|
11695
|
+
} catch {
|
|
11696
|
+
return 0;
|
|
11697
|
+
}
|
|
11698
|
+
}
|
|
11614
11699
|
function tableExists(db, table) {
|
|
11615
11700
|
const row = db.prepare(`SELECT 1 AS present FROM sqlite_master WHERE type IN ('table','view') AND name = ?`).get(table);
|
|
11616
11701
|
return row?.present === 1;
|
|
@@ -11618,6 +11703,7 @@ function tableExists(db, table) {
|
|
|
11618
11703
|
function reclaimIndex(dbPath, opts = {}) {
|
|
11619
11704
|
const rebuild = opts.rebuild === true;
|
|
11620
11705
|
const beforeBytes = indexSizeBytes(dbPath);
|
|
11706
|
+
const beforeDbBytes = dbFileBytes(dbPath);
|
|
11621
11707
|
const db = getDb(dbPath);
|
|
11622
11708
|
const dropped = {};
|
|
11623
11709
|
if (rebuild) {
|
|
@@ -11642,6 +11728,8 @@ function reclaimIndex(dbPath, opts = {}) {
|
|
|
11642
11728
|
return {
|
|
11643
11729
|
beforeBytes,
|
|
11644
11730
|
afterBytes: indexSizeBytes(dbPath),
|
|
11731
|
+
beforeDbBytes,
|
|
11732
|
+
afterDbBytes: dbFileBytes(dbPath),
|
|
11645
11733
|
dropped,
|
|
11646
11734
|
rebuilt: rebuild,
|
|
11647
11735
|
checkpointBusy: checkpointBusy || finalCheckpointBusy,
|
|
@@ -11677,13 +11765,12 @@ function cmdReclaimIndex(opts) {
|
|
|
11677
11765
|
process.stdout.write(JSON.stringify({ dbPath, ...result }, null, 2) + "\n");
|
|
11678
11766
|
return;
|
|
11679
11767
|
}
|
|
11680
|
-
const freed = result
|
|
11768
|
+
const { before, after, freed } = comparableSizes(result);
|
|
11769
|
+
const delta = freed >= 0 ? `freed ${mb(freed)}` : `grew ${mb(-freed)}`;
|
|
11681
11770
|
process.stdout.write(`reclaim-index: ${dbPath}
|
|
11682
11771
|
`);
|
|
11683
|
-
process.stdout.write(
|
|
11684
|
-
|
|
11685
|
-
`
|
|
11686
|
-
);
|
|
11772
|
+
process.stdout.write(` ${mb(before)} -> ${mb(after)} (${delta})
|
|
11773
|
+
`);
|
|
11687
11774
|
if (result.rebuilt) {
|
|
11688
11775
|
for (const [table, n] of Object.entries(result.dropped)) {
|
|
11689
11776
|
process.stdout.write(` dropped ${n} row(s) from ${table}
|
|
@@ -11702,7 +11789,7 @@ function cmdReclaimIndex(opts) {
|
|
|
11702
11789
|
}
|
|
11703
11790
|
if (result.checkpointBusy) {
|
|
11704
11791
|
process.stdout.write(
|
|
11705
|
-
` note: a concurrent reader blocked WAL truncation, so some space
|
|
11792
|
+
` note: a concurrent reader blocked WAL truncation, so some space is still held in ${path18.basename(dbPath)}-wal and the figures above count the main file only. The next checkpoint releases it
|
|
11706
11793
|
`
|
|
11707
11794
|
);
|
|
11708
11795
|
}
|
|
@@ -12182,7 +12269,7 @@ function cmdProject(opts) {
|
|
|
12182
12269
|
}
|
|
12183
12270
|
function recordCompactDocStat(fullSourceBytes, emittedText, detail) {
|
|
12184
12271
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(emittedText, "utf8"));
|
|
12185
|
-
recordStat("compact_doc", bytesSaved,
|
|
12272
|
+
recordStat("compact_doc", bytesSaved, savedTokensFromBytes(bytesSaved), void 0, detail);
|
|
12186
12273
|
}
|
|
12187
12274
|
function cmdCompactDoc(opts) {
|
|
12188
12275
|
const resolved = path19.resolve(opts.filePath);
|
|
@@ -13107,10 +13194,10 @@ function printCopilotReport(report) {
|
|
|
13107
13194
|
w(` Conversation: ${conversationTokens.toLocaleString()} tok
|
|
13108
13195
|
`);
|
|
13109
13196
|
if (total > 0) {
|
|
13110
|
-
const
|
|
13197
|
+
const pct2 = (fixed / total * 100).toFixed(1);
|
|
13111
13198
|
w(` ${fixed.toLocaleString()} tok of system prompt and tool definitions ships with every
|
|
13112
13199
|
`);
|
|
13113
|
-
w(` request; at shutdown that was ${
|
|
13200
|
+
w(` request; at shutdown that was ${pct2}% of the context. No hook can reach it: Copilot
|
|
13114
13201
|
`);
|
|
13115
13202
|
w(" assembles both natively, with nothing between assembly and send. The levers are all\n");
|
|
13116
13203
|
w(" config: fewer MCP servers and custom tools, and --excluded-tools / --available-tools /\n");
|
|
@@ -13170,9 +13257,9 @@ function printCopilotReport(report) {
|
|
|
13170
13257
|
`);
|
|
13171
13258
|
} else {
|
|
13172
13259
|
for (const block of report.blocks) {
|
|
13173
|
-
const
|
|
13260
|
+
const pct2 = block.bytes > 0 ? (block.repeatBytes / block.bytes * 100).toFixed(1) : "0.0";
|
|
13174
13261
|
w(` ${block.kind}: ${countNoun(block.count, "injection")}, ${formatBytes(block.bytes)}`);
|
|
13175
|
-
w(block.repeatBytes > 0 ? `, ${formatBytes(block.repeatBytes)} re-sent verbatim (${
|
|
13262
|
+
w(block.repeatBytes > 0 ? `, ${formatBytes(block.repeatBytes)} re-sent verbatim (${pct2}%)
|
|
13176
13263
|
` : "\n");
|
|
13177
13264
|
}
|
|
13178
13265
|
}
|
|
@@ -13502,22 +13589,693 @@ function formatSessionSlice(turns) {
|
|
|
13502
13589
|
return parts.join("\n").trimEnd();
|
|
13503
13590
|
}
|
|
13504
13591
|
|
|
13505
|
-
// src/
|
|
13592
|
+
// src/session_audit.ts
|
|
13506
13593
|
import * as fs26 from "node:fs";
|
|
13594
|
+
import * as os12 from "node:os";
|
|
13507
13595
|
import * as path26 from "node:path";
|
|
13508
|
-
|
|
13509
|
-
|
|
13596
|
+
import * as readline3 from "node:readline";
|
|
13597
|
+
function emptyMeasured() {
|
|
13598
|
+
return { apiCalls: 0, inputTokens: 0, cacheCreationTokens: 0, cacheReadTokens: 0, outputTokens: 0 };
|
|
13599
|
+
}
|
|
13600
|
+
function emptyCategory() {
|
|
13601
|
+
return { count: 0, bytes: 0, estTokens: 0 };
|
|
13602
|
+
}
|
|
13603
|
+
function addCategory(cat, bytes) {
|
|
13604
|
+
cat.count += 1;
|
|
13605
|
+
cat.bytes += bytes;
|
|
13606
|
+
cat.estTokens += estimateTokensFromLength(bytes);
|
|
13607
|
+
}
|
|
13608
|
+
var LOCAL_ONLY_TYPES = /* @__PURE__ */ new Set([
|
|
13609
|
+
"file-history-snapshot",
|
|
13610
|
+
"queue-operation",
|
|
13611
|
+
"mode",
|
|
13612
|
+
"permission-mode",
|
|
13613
|
+
"last-prompt",
|
|
13614
|
+
"bridge-session",
|
|
13615
|
+
"ai-title"
|
|
13616
|
+
]);
|
|
13617
|
+
var ATTACHMENT_VISIBLE_FIELDS = {
|
|
13618
|
+
hook_success: ["content"],
|
|
13619
|
+
hook_additional_context: ["content"],
|
|
13620
|
+
task_reminder: ["content"],
|
|
13621
|
+
skill_listing: ["content"],
|
|
13622
|
+
agent_listing_delta: ["addedLines"],
|
|
13623
|
+
file: ["content"],
|
|
13624
|
+
queued_command: ["prompt"],
|
|
13625
|
+
plan_file_reference: ["planContent"],
|
|
13626
|
+
edited_text_file: ["snippet"],
|
|
13627
|
+
read_truncation_notice: ["banner"],
|
|
13628
|
+
total_tokens_reminder: ["text"],
|
|
13629
|
+
nested_memory: ["content"],
|
|
13630
|
+
mcp_instructions_delta: ["addedBlocks"],
|
|
13631
|
+
invoked_skills: ["skills"],
|
|
13632
|
+
date_change: ["newDate"],
|
|
13633
|
+
goal_status: ["condition"],
|
|
13634
|
+
task_status: ["deltaSummary", "description"],
|
|
13635
|
+
command_permissions: ["allowedTools"]
|
|
13636
|
+
};
|
|
13637
|
+
var CACHE_WRITE_MULTIPLIER = 1.25;
|
|
13638
|
+
var CACHE_READ_MULTIPLIER = 0.1;
|
|
13639
|
+
var READ_DIVERT_MARKER_RE = /(?:was already read this session|Already read |You've already read|Use `token-goat (?:section|read|bash-output|config-get|skeleton)|token-goat bash-output --file)/;
|
|
13640
|
+
var READ_DIVERT_MAX_BYTES = 2500;
|
|
13641
|
+
var READ_FULL_SERVE_MIN_BYTES = 10240;
|
|
13642
|
+
var BASH_FILTER_MARKER_RE = /\[token-goat[:\]]/;
|
|
13643
|
+
var BASH_SMALL_RESULT_MAX_BYTES = 1024;
|
|
13644
|
+
function commandHead(raw) {
|
|
13645
|
+
let s = raw.trim();
|
|
13646
|
+
for (let guard = 0; guard < 20; guard++) {
|
|
13647
|
+
const stripped = s.replace(/^[(\s]+/, "");
|
|
13648
|
+
const env = /^[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|\S*)\s+/.exec(stripped);
|
|
13649
|
+
if (env !== null) {
|
|
13650
|
+
s = stripped.slice(env[0].length);
|
|
13651
|
+
continue;
|
|
13652
|
+
}
|
|
13653
|
+
const cd = /^cd\s+(?:"[^"]*"|'[^']*'|[^\s;&]+)[^\S\n]*(?:&&|;|\n)\s*/.exec(stripped);
|
|
13654
|
+
if (cd !== null) {
|
|
13655
|
+
s = stripped.slice(cd[0].length);
|
|
13656
|
+
continue;
|
|
13657
|
+
}
|
|
13658
|
+
if (stripped === s) break;
|
|
13659
|
+
s = stripped;
|
|
13660
|
+
}
|
|
13661
|
+
const m = /^"([^"]+)"|^'([^']+)'|^(\S+)/.exec(s);
|
|
13662
|
+
const tok = m === null ? "" : m[1] ?? m[2] ?? m[3] ?? "";
|
|
13663
|
+
const base = tok.split(/[\\/]/).pop() ?? "";
|
|
13664
|
+
const head = base.replace(/\.exe$/i, "").toLowerCase();
|
|
13665
|
+
return head === "" ? "(none)" : head;
|
|
13666
|
+
}
|
|
13667
|
+
function normalizeReadPath(p) {
|
|
13668
|
+
return p.replace(/\\/g, "/").toLowerCase();
|
|
13669
|
+
}
|
|
13670
|
+
function deepStringBytes(value) {
|
|
13671
|
+
if (typeof value === "string") return Buffer.byteLength(value, "utf8");
|
|
13672
|
+
if (Array.isArray(value)) return value.reduce((acc, v) => acc + deepStringBytes(v), 0);
|
|
13673
|
+
if (value !== null && typeof value === "object") {
|
|
13674
|
+
return Object.values(value).reduce((acc, v) => acc + deepStringBytes(v), 0);
|
|
13675
|
+
}
|
|
13676
|
+
return 0;
|
|
13677
|
+
}
|
|
13678
|
+
var OMISSION_MARKER_RE = /(?:--- (\d+) lines omitted ---|\[token-goat: \+?(\d+) more [a-z ]*lines omitted\]|--- patch: (\d+) lines omitted by token-goat ---|\.\.\. (\d+) lines omitted by token-goat \.\.\.)/g;
|
|
13679
|
+
function toolResultText(content) {
|
|
13680
|
+
if (typeof content === "string") return content;
|
|
13681
|
+
if (Array.isArray(content)) {
|
|
13682
|
+
let text = "";
|
|
13683
|
+
for (const block of content) {
|
|
13684
|
+
if (block !== null && typeof block === "object" && typeof block.text === "string") {
|
|
13685
|
+
text += block.text;
|
|
13686
|
+
}
|
|
13687
|
+
}
|
|
13688
|
+
return text;
|
|
13689
|
+
}
|
|
13690
|
+
return "";
|
|
13691
|
+
}
|
|
13692
|
+
function listCorpusTranscripts(corpusDir) {
|
|
13693
|
+
const found = [];
|
|
13694
|
+
const walk = (dir, entries) => {
|
|
13695
|
+
for (const entry of entries) {
|
|
13696
|
+
const full = path26.join(dir, entry.name);
|
|
13697
|
+
if (entry.name.endsWith(".jsonl")) {
|
|
13698
|
+
found.push(full);
|
|
13699
|
+
} else if (entry.isDirectory()) {
|
|
13700
|
+
let inner;
|
|
13701
|
+
try {
|
|
13702
|
+
inner = fs26.readdirSync(full, { withFileTypes: true });
|
|
13703
|
+
} catch {
|
|
13704
|
+
continue;
|
|
13705
|
+
}
|
|
13706
|
+
walk(full, inner);
|
|
13707
|
+
}
|
|
13708
|
+
}
|
|
13709
|
+
};
|
|
13710
|
+
walk(corpusDir, fs26.readdirSync(corpusDir, { withFileTypes: true }));
|
|
13711
|
+
return found.sort();
|
|
13712
|
+
}
|
|
13713
|
+
function defaultCorpusDir() {
|
|
13714
|
+
return path26.join(os12.homedir(), ".claude", "projects");
|
|
13715
|
+
}
|
|
13716
|
+
async function auditOneFile(filePath, s, toolMap, attachmentMap, hookMap, laneObservations, bashHeadMap) {
|
|
13717
|
+
const isLane = filePath.split(/[\\/]/).includes("subagents");
|
|
13718
|
+
let laneFirstPrefix = null;
|
|
13719
|
+
let laneBriefBytes = -1;
|
|
13720
|
+
const toolNameById = /* @__PURE__ */ new Map();
|
|
13721
|
+
const bashHeadById = /* @__PURE__ */ new Map();
|
|
13722
|
+
let pendingBashReread = [];
|
|
13723
|
+
const readCallById = /* @__PURE__ */ new Map();
|
|
13724
|
+
const readPathEpoch = /* @__PURE__ */ new Map();
|
|
13725
|
+
let compactEpoch = 0;
|
|
13726
|
+
let sawTokenGoatHook = false;
|
|
13727
|
+
let fileRepeats = 0;
|
|
13728
|
+
let fileRepeatsFullNoRange = 0;
|
|
13729
|
+
let fileRepeatsFullNoRangeAfterCompaction = 0;
|
|
13730
|
+
const usageSeenIds = /* @__PURE__ */ new Set();
|
|
13731
|
+
const perCall = [];
|
|
13732
|
+
const laneCalls = [0, 0];
|
|
13733
|
+
let pendingReread = [];
|
|
13734
|
+
const lastVisibleByKind = /* @__PURE__ */ new Map();
|
|
13735
|
+
const flushLane = (lane) => {
|
|
13736
|
+
const kept = [];
|
|
13737
|
+
for (const p of pendingReread) {
|
|
13738
|
+
if (p.lane !== lane) {
|
|
13739
|
+
kept.push(p);
|
|
13740
|
+
continue;
|
|
13741
|
+
}
|
|
13742
|
+
attachmentRollup(attachmentMap, p.kind).rereadTokens += p.tokens * Math.max(0, laneCalls[lane] - p.atCall);
|
|
13743
|
+
}
|
|
13744
|
+
pendingReread = kept;
|
|
13745
|
+
const keptBash = [];
|
|
13746
|
+
for (const p of pendingBashReread) {
|
|
13747
|
+
if (p.lane !== lane) {
|
|
13748
|
+
keptBash.push(p);
|
|
13749
|
+
continue;
|
|
13750
|
+
}
|
|
13751
|
+
s.bashInterception.untouchedRereadTokens += p.tokens * Math.max(0, laneCalls[lane] - p.atCall);
|
|
13752
|
+
}
|
|
13753
|
+
pendingBashReread = keptBash;
|
|
13754
|
+
};
|
|
13755
|
+
const stream = fs26.createReadStream(filePath);
|
|
13756
|
+
const rl = readline3.createInterface({ input: stream, crlfDelay: Infinity });
|
|
13757
|
+
const streamFailure = new Promise((_, reject) => stream.on("error", reject));
|
|
13758
|
+
const consume = (async () => {
|
|
13759
|
+
for await (const line of rl) {
|
|
13760
|
+
if (line.length === 0) continue;
|
|
13761
|
+
s.lines += 1;
|
|
13762
|
+
const lineBytes = Buffer.byteLength(line, "utf8");
|
|
13763
|
+
s.totalBytes += lineBytes;
|
|
13764
|
+
let obj;
|
|
13765
|
+
try {
|
|
13766
|
+
obj = JSON.parse(line);
|
|
13767
|
+
} catch {
|
|
13768
|
+
s.parseFailedLines += 1;
|
|
13769
|
+
addCategory(s.estimated.otherLocal, lineBytes);
|
|
13770
|
+
continue;
|
|
13771
|
+
}
|
|
13772
|
+
const type = typeof obj["type"] === "string" ? obj["type"] : "(untyped)";
|
|
13773
|
+
const census = s.lineTypes[type] ??= { lines: 0, bytes: 0 };
|
|
13774
|
+
census.lines += 1;
|
|
13775
|
+
census.bytes += lineBytes;
|
|
13776
|
+
if (LOCAL_ONLY_TYPES.has(type)) {
|
|
13777
|
+
addCategory(s.estimated.otherLocal, lineBytes);
|
|
13778
|
+
continue;
|
|
13779
|
+
}
|
|
13780
|
+
const message = obj["message"];
|
|
13781
|
+
if (type === "assistant" && message !== void 0) {
|
|
13782
|
+
const usage = message.usage;
|
|
13783
|
+
if (usage !== void 0 && typeof message.id === "string" && !usageSeenIds.has(message.id)) {
|
|
13784
|
+
usageSeenIds.add(message.id);
|
|
13785
|
+
const num = (k) => typeof usage[k] === "number" ? usage[k] : 0;
|
|
13786
|
+
const input = num("input_tokens");
|
|
13787
|
+
const cacheWrite = num("cache_creation_input_tokens");
|
|
13788
|
+
const cacheRead = num("cache_read_input_tokens");
|
|
13789
|
+
const output = num("output_tokens");
|
|
13790
|
+
s.measured.apiCalls += 1;
|
|
13791
|
+
s.measured.inputTokens += input;
|
|
13792
|
+
s.measured.cacheCreationTokens += cacheWrite;
|
|
13793
|
+
s.measured.cacheReadTokens += cacheRead;
|
|
13794
|
+
s.measured.outputTokens += output;
|
|
13795
|
+
if (obj["isSidechain"] === true) {
|
|
13796
|
+
s.measuredSidechain.apiCalls += 1;
|
|
13797
|
+
s.measuredSidechain.inputTokens += input;
|
|
13798
|
+
s.measuredSidechain.cacheCreationTokens += cacheWrite;
|
|
13799
|
+
s.measuredSidechain.cacheReadTokens += cacheRead;
|
|
13800
|
+
s.measuredSidechain.outputTokens += output;
|
|
13801
|
+
}
|
|
13802
|
+
if (isLane && laneFirstPrefix === null) laneFirstPrefix = input + cacheWrite + cacheRead;
|
|
13803
|
+
perCall.push({ inputTotal: input + cacheWrite + cacheRead, cacheRead, output });
|
|
13804
|
+
const callLane = obj["isSidechain"] === true ? 1 : 0;
|
|
13805
|
+
laneCalls[callLane] = laneCalls[callLane] + 1;
|
|
13806
|
+
}
|
|
13807
|
+
const blocks = Array.isArray(message.content) ? message.content : [];
|
|
13808
|
+
for (const block of blocks) {
|
|
13809
|
+
if (block === null || typeof block !== "object") continue;
|
|
13810
|
+
if (block["type"] === "text" && typeof block["text"] === "string") {
|
|
13811
|
+
addCategory(s.estimated.assistantText, Buffer.byteLength(block["text"], "utf8"));
|
|
13812
|
+
} else if (block["type"] === "thinking" && typeof block["thinking"] === "string") {
|
|
13813
|
+
addCategory(s.estimated.assistantThinking, Buffer.byteLength(block["thinking"], "utf8"));
|
|
13814
|
+
} else if (block["type"] === "tool_use" && typeof block["id"] === "string") {
|
|
13815
|
+
const name = typeof block["name"] === "string" ? block["name"] : "(unnamed)";
|
|
13816
|
+
if (!toolNameById.has(block["id"])) {
|
|
13817
|
+
toolNameById.set(block["id"], name);
|
|
13818
|
+
toolRollup(toolMap, name).calls += 1;
|
|
13819
|
+
const input = block["input"] !== null && typeof block["input"] === "object" ? block["input"] : {};
|
|
13820
|
+
if (name === "Bash" && typeof input["command"] === "string") {
|
|
13821
|
+
bashHeadById.set(block["id"], commandHead(input["command"]));
|
|
13822
|
+
} else if (name === "Read" && typeof input["file_path"] === "string") {
|
|
13823
|
+
const norm = normalizeReadPath(input["file_path"]);
|
|
13824
|
+
const priorEpoch = readPathEpoch.get(norm);
|
|
13825
|
+
readCallById.set(block["id"], { wasSeenBefore: priorEpoch !== void 0, hasRange: input["offset"] !== void 0 || input["limit"] !== void 0, afterCompaction: priorEpoch !== void 0 && compactEpoch > priorEpoch });
|
|
13826
|
+
readPathEpoch.set(norm, compactEpoch);
|
|
13827
|
+
}
|
|
13828
|
+
}
|
|
13829
|
+
let inputBytes;
|
|
13830
|
+
try {
|
|
13831
|
+
inputBytes = Buffer.byteLength(JSON.stringify(block["input"] ?? null), "utf8");
|
|
13832
|
+
} catch {
|
|
13833
|
+
inputBytes = 0;
|
|
13834
|
+
}
|
|
13835
|
+
addCategory(s.estimated.toolUseInputs, inputBytes);
|
|
13836
|
+
}
|
|
13837
|
+
}
|
|
13838
|
+
continue;
|
|
13839
|
+
}
|
|
13840
|
+
if (type === "user" && message !== void 0) {
|
|
13841
|
+
const blocks = Array.isArray(message.content) ? message.content : [];
|
|
13842
|
+
let sawToolResult = false;
|
|
13843
|
+
for (const block of blocks) {
|
|
13844
|
+
if (block === null || typeof block !== "object" || block["type"] !== "tool_result") continue;
|
|
13845
|
+
sawToolResult = true;
|
|
13846
|
+
const text = toolResultText(block["content"]);
|
|
13847
|
+
const bytes = Buffer.byteLength(text, "utf8");
|
|
13848
|
+
const id = typeof block["tool_use_id"] === "string" ? block["tool_use_id"] : "";
|
|
13849
|
+
const name = toolNameById.get(id) ?? "(unknown)";
|
|
13850
|
+
addCategory(s.estimated.toolResults, bytes);
|
|
13851
|
+
const roll = toolRollup(toolMap, name);
|
|
13852
|
+
if (!toolNameById.has(id)) roll.calls += 1;
|
|
13853
|
+
roll.resultBytes += bytes;
|
|
13854
|
+
roll.resultEstTokens += estimateTokensFromLength(bytes);
|
|
13855
|
+
if (name === "Read") {
|
|
13856
|
+
s.readInterception.readResults += 1;
|
|
13857
|
+
if (bytes < READ_DIVERT_MAX_BYTES && READ_DIVERT_MARKER_RE.test(text)) {
|
|
13858
|
+
s.readInterception.divertedByMarker += 1;
|
|
13859
|
+
s.readInterception.divertedBytes += bytes;
|
|
13860
|
+
} else if (bytes >= READ_FULL_SERVE_MIN_BYTES) {
|
|
13861
|
+
s.readInterception.fullServesOver10k += 1;
|
|
13862
|
+
s.readInterception.fullServeBytesOver10k += bytes;
|
|
13863
|
+
const call = readCallById.get(id);
|
|
13864
|
+
if (call === void 0) {
|
|
13865
|
+
s.readInterception.fullServesPathUnknown += 1;
|
|
13866
|
+
} else if (call.wasSeenBefore) {
|
|
13867
|
+
s.readInterception.fullServesRepeat += 1;
|
|
13868
|
+
s.readInterception.repeatBytes += bytes;
|
|
13869
|
+
fileRepeats += 1;
|
|
13870
|
+
if (call.hasRange) {
|
|
13871
|
+
s.readInterception.repeatWithRange += 1;
|
|
13872
|
+
} else {
|
|
13873
|
+
s.readInterception.repeatFullNoRange += 1;
|
|
13874
|
+
fileRepeatsFullNoRange += 1;
|
|
13875
|
+
if (call.afterCompaction) fileRepeatsFullNoRangeAfterCompaction += 1;
|
|
13876
|
+
}
|
|
13877
|
+
} else {
|
|
13878
|
+
s.readInterception.fullServesFirstRead += 1;
|
|
13879
|
+
}
|
|
13880
|
+
}
|
|
13881
|
+
} else if (name === "Bash") {
|
|
13882
|
+
const b = s.bashInterception;
|
|
13883
|
+
b.bashResults += 1;
|
|
13884
|
+
if (BASH_FILTER_MARKER_RE.test(text)) {
|
|
13885
|
+
b.markedByFilter += 1;
|
|
13886
|
+
b.markedBytes += bytes;
|
|
13887
|
+
} else if (bytes < BASH_SMALL_RESULT_MAX_BYTES) {
|
|
13888
|
+
b.smallUntouched += 1;
|
|
13889
|
+
b.smallUntouchedBytes += bytes;
|
|
13890
|
+
} else {
|
|
13891
|
+
b.untouched += 1;
|
|
13892
|
+
b.untouchedBytes += bytes;
|
|
13893
|
+
const tokens = estimateTokensFromLength(bytes);
|
|
13894
|
+
b.untouchedEstTokens += tokens;
|
|
13895
|
+
const lane = obj["isSidechain"] === true ? 1 : 0;
|
|
13896
|
+
pendingBashReread.push({ tokens, atCall: laneCalls[lane], lane });
|
|
13897
|
+
const head = bashHeadById.get(id) ?? "(unknown)";
|
|
13898
|
+
let headRoll = bashHeadMap.get(head);
|
|
13899
|
+
if (headRoll === void 0) {
|
|
13900
|
+
headRoll = { head, results: 0, bytes: 0 };
|
|
13901
|
+
bashHeadMap.set(head, headRoll);
|
|
13902
|
+
}
|
|
13903
|
+
headRoll.results += 1;
|
|
13904
|
+
headRoll.bytes += bytes;
|
|
13905
|
+
}
|
|
13906
|
+
}
|
|
13907
|
+
OMISSION_MARKER_RE.lastIndex = 0;
|
|
13908
|
+
for (const m of text.matchAll(OMISSION_MARKER_RE)) {
|
|
13909
|
+
s.omissionMarkers.fires += 1;
|
|
13910
|
+
s.omissionMarkers.linesOmitted += Number(m[1] ?? m[2] ?? m[3] ?? m[4] ?? 0);
|
|
13911
|
+
}
|
|
13912
|
+
}
|
|
13913
|
+
if (!sawToolResult) {
|
|
13914
|
+
const target = obj["isMeta"] === true ? s.estimated.harnessMeta : s.estimated.userTurns;
|
|
13915
|
+
let contentBytes = 0;
|
|
13916
|
+
if (typeof message.content === "string") {
|
|
13917
|
+
contentBytes = Buffer.byteLength(message.content, "utf8");
|
|
13918
|
+
} else {
|
|
13919
|
+
for (const block of blocks) {
|
|
13920
|
+
if (block !== null && typeof block === "object" && typeof block["text"] === "string") {
|
|
13921
|
+
contentBytes += Buffer.byteLength(block["text"], "utf8");
|
|
13922
|
+
}
|
|
13923
|
+
}
|
|
13924
|
+
}
|
|
13925
|
+
addCategory(target, contentBytes);
|
|
13926
|
+
if (isLane && laneBriefBytes < 0 && obj["isMeta"] !== true) laneBriefBytes = contentBytes;
|
|
13927
|
+
}
|
|
13928
|
+
continue;
|
|
13929
|
+
}
|
|
13930
|
+
if (type === "attachment") {
|
|
13931
|
+
addCategory(s.estimated.attachments, lineBytes);
|
|
13932
|
+
const att = obj["attachment"] !== null && typeof obj["attachment"] === "object" ? obj["attachment"] : {};
|
|
13933
|
+
const kind = typeof att["type"] === "string" ? att["type"] : "(untyped)";
|
|
13934
|
+
const lane = obj["isSidechain"] === true ? 1 : 0;
|
|
13935
|
+
const roll = attachmentRollup(attachmentMap, kind);
|
|
13936
|
+
roll.injections += 1;
|
|
13937
|
+
roll.lineBytes += lineBytes;
|
|
13938
|
+
const fields = ATTACHMENT_VISIBLE_FIELDS[kind];
|
|
13939
|
+
if (fields !== void 0) {
|
|
13940
|
+
const visibleBytes = fields.reduce((acc, f) => acc + deepStringBytes(att[f]), 0);
|
|
13941
|
+
roll.visibleBytes += visibleBytes;
|
|
13942
|
+
const tokens = visibleBytes > 0 ? estimateTokensFromLength(visibleBytes) : 0;
|
|
13943
|
+
roll.estTokens += tokens;
|
|
13944
|
+
if (tokens > 0) pendingReread.push({ kind, tokens, atCall: laneCalls[lane], lane });
|
|
13945
|
+
let serialized;
|
|
13946
|
+
try {
|
|
13947
|
+
serialized = JSON.stringify(fields.map((f) => att[f]));
|
|
13948
|
+
} catch {
|
|
13949
|
+
serialized = "";
|
|
13950
|
+
}
|
|
13951
|
+
const previous = lastVisibleByKind.get(kind);
|
|
13952
|
+
if (previous !== void 0) {
|
|
13953
|
+
if (previous === serialized) roll.repeatedIdentical += 1;
|
|
13954
|
+
else roll.repeatedChanged += 1;
|
|
13955
|
+
}
|
|
13956
|
+
lastVisibleByKind.set(kind, serialized);
|
|
13957
|
+
}
|
|
13958
|
+
if (kind === "hook_success") {
|
|
13959
|
+
const command = typeof att["command"] === "string" ? att["command"] : "";
|
|
13960
|
+
const origin = command.includes("token-goat") || command.includes("token_goat") ? "token-goat" : "other";
|
|
13961
|
+
if (origin === "token-goat") sawTokenGoatHook = true;
|
|
13962
|
+
const event = typeof att["hookEvent"] === "string" ? att["hookEvent"] : "(none)";
|
|
13963
|
+
const hook = hookRollup(hookMap, origin, event);
|
|
13964
|
+
hook.fires += 1;
|
|
13965
|
+
hook.stdoutBytes += typeof att["stdout"] === "string" ? Buffer.byteLength(att["stdout"], "utf8") : 0;
|
|
13966
|
+
hook.contextBytes += typeof att["content"] === "string" ? Buffer.byteLength(att["content"], "utf8") : 0;
|
|
13967
|
+
}
|
|
13968
|
+
continue;
|
|
13969
|
+
}
|
|
13970
|
+
if (type === "system" || type === "summary") {
|
|
13971
|
+
if (obj["subtype"] === "compact_boundary") {
|
|
13972
|
+
flushLane(obj["isSidechain"] === true ? 1 : 0);
|
|
13973
|
+
compactEpoch += 1;
|
|
13974
|
+
}
|
|
13975
|
+
addCategory(s.estimated.system, lineBytes);
|
|
13976
|
+
continue;
|
|
13977
|
+
}
|
|
13978
|
+
addCategory(s.estimated.otherLocal, lineBytes);
|
|
13979
|
+
}
|
|
13980
|
+
})();
|
|
13981
|
+
void consume.catch(() => {
|
|
13982
|
+
});
|
|
13983
|
+
void streamFailure.catch(() => {
|
|
13984
|
+
});
|
|
13510
13985
|
try {
|
|
13511
|
-
|
|
13512
|
-
|
|
13986
|
+
await Promise.race([consume, streamFailure]);
|
|
13987
|
+
} finally {
|
|
13988
|
+
rl.close();
|
|
13989
|
+
stream.destroy();
|
|
13990
|
+
}
|
|
13991
|
+
flushLane(0);
|
|
13992
|
+
flushLane(1);
|
|
13993
|
+
if (sawTokenGoatHook) {
|
|
13994
|
+
s.readInterception.repeatInHookedSessions += fileRepeats;
|
|
13995
|
+
s.readInterception.repeatFullNoRangeInHookedSessions += fileRepeatsFullNoRange;
|
|
13996
|
+
s.readInterception.repeatFullNoRangeHookedAfterCompaction += fileRepeatsFullNoRangeAfterCompaction;
|
|
13997
|
+
}
|
|
13998
|
+
if (isLane) {
|
|
13999
|
+
let agentType = "(none)";
|
|
14000
|
+
try {
|
|
14001
|
+
const meta = JSON.parse(fs26.readFileSync(filePath.replace(/\.jsonl$/i, ".meta.json"), "utf8"));
|
|
14002
|
+
if (typeof meta["agentType"] === "string" && meta["agentType"] !== "") agentType = meta["agentType"];
|
|
14003
|
+
} catch {
|
|
13513
14004
|
}
|
|
13514
|
-
|
|
14005
|
+
laneObservations.push({ firstPrefixTokens: laneFirstPrefix, calls: usageSeenIds.size, briefBytes: Math.max(0, laneBriefBytes), agentType });
|
|
14006
|
+
}
|
|
14007
|
+
const callCount = perCall.length;
|
|
14008
|
+
for (let i = 0; i < callCount; i++) {
|
|
14009
|
+
const call = perCall[i];
|
|
14010
|
+
const d = s.positionDeciles[Math.min(9, Math.floor(i * 10 / callCount))];
|
|
14011
|
+
d.apiCalls += 1;
|
|
14012
|
+
d.inputTokens += call.inputTotal;
|
|
14013
|
+
d.cacheReadTokens += call.cacheRead;
|
|
14014
|
+
d.outputTokens += call.output;
|
|
14015
|
+
}
|
|
14016
|
+
}
|
|
14017
|
+
function toolRollup(toolMap, name) {
|
|
14018
|
+
let roll = toolMap.get(name);
|
|
14019
|
+
if (roll === void 0) {
|
|
14020
|
+
roll = { name, calls: 0, resultBytes: 0, resultEstTokens: 0 };
|
|
14021
|
+
toolMap.set(name, roll);
|
|
14022
|
+
}
|
|
14023
|
+
return roll;
|
|
14024
|
+
}
|
|
14025
|
+
function attachmentRollup(attachmentMap, kind) {
|
|
14026
|
+
let roll = attachmentMap.get(kind);
|
|
14027
|
+
if (roll === void 0) {
|
|
14028
|
+
roll = { kind, injections: 0, lineBytes: 0, visibleBytes: 0, estTokens: 0, rereadTokens: 0, billedEquivTokens: 0, repeatedIdentical: 0, repeatedChanged: 0 };
|
|
14029
|
+
attachmentMap.set(kind, roll);
|
|
14030
|
+
}
|
|
14031
|
+
return roll;
|
|
14032
|
+
}
|
|
14033
|
+
function hookRollup(hookMap, origin, event) {
|
|
14034
|
+
const key = `${origin}|${event}`;
|
|
14035
|
+
let roll = hookMap.get(key);
|
|
14036
|
+
if (roll === void 0) {
|
|
14037
|
+
roll = { origin, event, fires: 0, stdoutBytes: 0, contextBytes: 0 };
|
|
14038
|
+
hookMap.set(key, roll);
|
|
14039
|
+
}
|
|
14040
|
+
return roll;
|
|
14041
|
+
}
|
|
14042
|
+
async function auditSessionCorpus(opts = {}) {
|
|
14043
|
+
const corpusDir = path26.resolve(opts.dir ?? defaultCorpusDir());
|
|
14044
|
+
if (!fs26.existsSync(corpusDir)) {
|
|
14045
|
+
throw new Error(`session corpus directory not found: ${corpusDir}`);
|
|
14046
|
+
}
|
|
14047
|
+
const files = listCorpusTranscripts(corpusDir);
|
|
14048
|
+
if (files.length === 0) {
|
|
14049
|
+
throw new Error(`no .jsonl session transcripts found under ${corpusDir}`);
|
|
14050
|
+
}
|
|
14051
|
+
const started = Date.now();
|
|
14052
|
+
const summary = {
|
|
14053
|
+
corpusDir,
|
|
14054
|
+
filesScanned: 0,
|
|
14055
|
+
filesFailed: 0,
|
|
14056
|
+
lines: 0,
|
|
14057
|
+
parseFailedLines: 0,
|
|
14058
|
+
totalBytes: 0,
|
|
14059
|
+
runtimeMs: 0,
|
|
14060
|
+
measured: emptyMeasured(),
|
|
14061
|
+
measuredSidechain: emptyMeasured(),
|
|
14062
|
+
estimated: {
|
|
14063
|
+
userTurns: emptyCategory(),
|
|
14064
|
+
toolResults: emptyCategory(),
|
|
14065
|
+
assistantText: emptyCategory(),
|
|
14066
|
+
assistantThinking: emptyCategory(),
|
|
14067
|
+
toolUseInputs: emptyCategory(),
|
|
14068
|
+
attachments: emptyCategory(),
|
|
14069
|
+
harnessMeta: emptyCategory(),
|
|
14070
|
+
system: emptyCategory(),
|
|
14071
|
+
otherLocal: emptyCategory()
|
|
14072
|
+
},
|
|
14073
|
+
tools: [],
|
|
14074
|
+
attachmentKinds: [],
|
|
14075
|
+
hookOutputs: [],
|
|
14076
|
+
positionDeciles: Array.from({ length: 10 }, (_, i) => ({ decile: i + 1, apiCalls: 0, inputTokens: 0, cacheReadTokens: 0, outputTokens: 0 })),
|
|
14077
|
+
sidechainLanes: { laneFiles: 0, lanesWithUsage: 0, meanFirstCallPrefixTokens: 0, medianFirstCallPrefixTokens: 0, p90FirstCallPrefixTokens: 0, meanCallsPerLane: 0, meanBriefBytes: 0, prefixBilledEquivTokens: 0 },
|
|
14078
|
+
laneAgentTypes: [],
|
|
14079
|
+
readInterception: { readResults: 0, divertedByMarker: 0, divertedBytes: 0, fullServesOver10k: 0, fullServeBytesOver10k: 0, fullServesFirstRead: 0, fullServesRepeat: 0, repeatBytes: 0, repeatWithRange: 0, repeatFullNoRange: 0, repeatInHookedSessions: 0, repeatFullNoRangeInHookedSessions: 0, repeatFullNoRangeHookedAfterCompaction: 0, fullServesPathUnknown: 0 },
|
|
14080
|
+
bashInterception: { bashResults: 0, markedByFilter: 0, markedBytes: 0, smallUntouched: 0, smallUntouchedBytes: 0, untouched: 0, untouchedBytes: 0, untouchedEstTokens: 0, untouchedRereadTokens: 0, untouchedBilledEquivTokens: 0, untouchedHeads: [] },
|
|
14081
|
+
lineTypes: {},
|
|
14082
|
+
omissionMarkers: { fires: 0, linesOmitted: 0 }
|
|
14083
|
+
};
|
|
14084
|
+
const toolMap = /* @__PURE__ */ new Map();
|
|
14085
|
+
const attachmentMap = /* @__PURE__ */ new Map();
|
|
14086
|
+
const hookMap = /* @__PURE__ */ new Map();
|
|
14087
|
+
const laneObservations = [];
|
|
14088
|
+
const bashHeadMap = /* @__PURE__ */ new Map();
|
|
14089
|
+
for (const file of files) {
|
|
14090
|
+
try {
|
|
14091
|
+
await auditOneFile(file, summary, toolMap, attachmentMap, hookMap, laneObservations, bashHeadMap);
|
|
14092
|
+
summary.filesScanned += 1;
|
|
14093
|
+
} catch {
|
|
14094
|
+
summary.filesFailed += 1;
|
|
14095
|
+
}
|
|
14096
|
+
}
|
|
14097
|
+
const withUsage = laneObservations.filter((l) => l.firstPrefixTokens !== null);
|
|
14098
|
+
const prefixes = withUsage.map((l) => l.firstPrefixTokens).sort((a, b) => a - b);
|
|
14099
|
+
const mean = (arr) => arr.length === 0 ? 0 : Math.round(arr.reduce((a, b) => a + b, 0) / arr.length);
|
|
14100
|
+
summary.sidechainLanes = {
|
|
14101
|
+
laneFiles: laneObservations.length,
|
|
14102
|
+
lanesWithUsage: withUsage.length,
|
|
14103
|
+
meanFirstCallPrefixTokens: mean(prefixes),
|
|
14104
|
+
medianFirstCallPrefixTokens: prefixes.length === 0 ? 0 : prefixes[Math.floor(prefixes.length / 2)],
|
|
14105
|
+
p90FirstCallPrefixTokens: prefixes.length === 0 ? 0 : prefixes[Math.min(prefixes.length - 1, Math.floor(prefixes.length * 0.9))],
|
|
14106
|
+
meanCallsPerLane: mean(withUsage.map((l) => l.calls)),
|
|
14107
|
+
meanBriefBytes: mean(withUsage.map((l) => l.briefBytes)),
|
|
14108
|
+
prefixBilledEquivTokens: Math.round(withUsage.reduce((acc, l) => acc + l.firstPrefixTokens * (CACHE_WRITE_MULTIPLIER + CACHE_READ_MULTIPLIER * Math.max(0, l.calls - 1)), 0))
|
|
14109
|
+
};
|
|
14110
|
+
const byType = /* @__PURE__ */ new Map();
|
|
14111
|
+
for (const l of laneObservations) {
|
|
14112
|
+
const bucket = byType.get(l.agentType);
|
|
14113
|
+
if (bucket === void 0) byType.set(l.agentType, [l]);
|
|
14114
|
+
else bucket.push(l);
|
|
14115
|
+
}
|
|
14116
|
+
summary.laneAgentTypes = [...byType.entries()].map(([agentType, obs]) => {
|
|
14117
|
+
const typed = obs.filter((l) => l.firstPrefixTokens !== null).map((l) => l.firstPrefixTokens).sort((a, b) => a - b);
|
|
14118
|
+
return { agentType, lanes: obs.length, lanesWithUsage: typed.length, meanFirstCallPrefixTokens: mean(typed), medianFirstCallPrefixTokens: typed.length === 0 ? 0 : typed[Math.floor(typed.length / 2)] };
|
|
14119
|
+
}).sort((a, b) => b.lanes - a.lanes || a.agentType.localeCompare(b.agentType));
|
|
14120
|
+
summary.bashInterception.untouchedBilledEquivTokens = Math.round(CACHE_WRITE_MULTIPLIER * summary.bashInterception.untouchedEstTokens + CACHE_READ_MULTIPLIER * summary.bashInterception.untouchedRereadTokens);
|
|
14121
|
+
summary.bashInterception.untouchedHeads = [...bashHeadMap.values()].sort((a, b) => b.bytes - a.bytes || b.results - a.results || a.head.localeCompare(b.head));
|
|
14122
|
+
summary.tools = [...toolMap.values()].sort((a, b) => b.resultBytes - a.resultBytes || a.name.localeCompare(b.name));
|
|
14123
|
+
for (const roll of attachmentMap.values()) {
|
|
14124
|
+
roll.billedEquivTokens = Math.round(CACHE_WRITE_MULTIPLIER * roll.estTokens + CACHE_READ_MULTIPLIER * roll.rereadTokens);
|
|
14125
|
+
}
|
|
14126
|
+
summary.attachmentKinds = [...attachmentMap.values()].sort((a, b) => b.billedEquivTokens - a.billedEquivTokens || b.injections - a.injections || a.kind.localeCompare(b.kind));
|
|
14127
|
+
summary.hookOutputs = [...hookMap.values()].sort((a, b) => b.contextBytes - a.contextBytes || b.fires - a.fires || a.origin.localeCompare(b.origin) || a.event.localeCompare(b.event));
|
|
14128
|
+
summary.runtimeMs = Date.now() - started;
|
|
14129
|
+
return summary;
|
|
14130
|
+
}
|
|
14131
|
+
function fmt(n) {
|
|
14132
|
+
return n.toLocaleString("en-US");
|
|
14133
|
+
}
|
|
14134
|
+
function pct(part, whole) {
|
|
14135
|
+
return whole === 0 ? "0.0%" : `${(part / whole * 100).toFixed(1)}%`;
|
|
14136
|
+
}
|
|
14137
|
+
function formatSessionAudit(s) {
|
|
14138
|
+
const lines = [];
|
|
14139
|
+
lines.push("# Session corpus audit");
|
|
14140
|
+
lines.push(`Corpus: ${s.corpusDir}`);
|
|
14141
|
+
lines.push(`Files: ${fmt(s.filesScanned)} scanned, ${fmt(s.filesFailed)} unreadable`);
|
|
14142
|
+
lines.push(`Lines: ${fmt(s.lines)} (${fmt(s.parseFailedLines)} unparseable), bytes: ${fmt(s.totalBytes)}`);
|
|
14143
|
+
lines.push(`Runtime: ${(s.runtimeMs / 1e3).toFixed(1)}s`);
|
|
14144
|
+
lines.push("");
|
|
14145
|
+
lines.push("## Measured billed tokens (assistant message.usage, one count per API response)");
|
|
14146
|
+
const m = s.measured;
|
|
14147
|
+
lines.push(`API calls: ${fmt(m.apiCalls)} (sidechain: ${fmt(s.measuredSidechain.apiCalls)})`);
|
|
14148
|
+
lines.push(`Output tokens: ${fmt(m.outputTokens)}`);
|
|
14149
|
+
lines.push(`Input, uncached: ${fmt(m.inputTokens)}`);
|
|
14150
|
+
lines.push(`Input, cache-write: ${fmt(m.cacheCreationTokens)}`);
|
|
14151
|
+
lines.push(`Input, cache-read: ${fmt(m.cacheReadTokens)}`);
|
|
14152
|
+
const totalInput = m.inputTokens + m.cacheCreationTokens + m.cacheReadTokens;
|
|
14153
|
+
lines.push(`Cache-read share of input: ${pct(m.cacheReadTokens, totalInput)}`);
|
|
14154
|
+
lines.push("");
|
|
14155
|
+
lines.push("## Estimated content attribution (chars/3 heuristic; NOT billed units)");
|
|
14156
|
+
const e = s.estimated;
|
|
14157
|
+
const rows = [
|
|
14158
|
+
["tool results", e.toolResults],
|
|
14159
|
+
["assistant text", e.assistantText],
|
|
14160
|
+
["assistant thinking", e.assistantThinking],
|
|
14161
|
+
["tool call inputs", e.toolUseInputs],
|
|
14162
|
+
["attachments (harness)", e.attachments],
|
|
14163
|
+
["user turns", e.userTurns],
|
|
14164
|
+
["meta user lines", e.harnessMeta],
|
|
14165
|
+
["system lines", e.system],
|
|
14166
|
+
["local bookkeeping (never sent)", e.otherLocal]
|
|
14167
|
+
];
|
|
14168
|
+
const modelVisibleBytes = rows.slice(0, 8).reduce((acc, [, c]) => acc + c.bytes, 0);
|
|
14169
|
+
for (const [label, cat] of rows.sort((a, b) => b[1].bytes - a[1].bytes)) {
|
|
14170
|
+
lines.push(`${label.padEnd(31)} count ${fmt(cat.count).padStart(11)} bytes ${fmt(cat.bytes).padStart(15)} est-tokens ${fmt(cat.estTokens).padStart(13)} ${label === "local bookkeeping (never sent)" ? "(excluded from share)" : pct(cat.bytes, modelVisibleBytes)}`);
|
|
14171
|
+
}
|
|
14172
|
+
lines.push("");
|
|
14173
|
+
lines.push("## Tool results by tool (estimated content size; calls = tool_use invocations)");
|
|
14174
|
+
for (const t of s.tools.slice(0, 25)) {
|
|
14175
|
+
lines.push(`${t.name.padEnd(42)} calls ${fmt(t.calls).padStart(9)} bytes ${fmt(t.resultBytes).padStart(15)} est-tokens ${fmt(t.resultEstTokens).padStart(12)}`);
|
|
14176
|
+
}
|
|
14177
|
+
if (s.tools.length > 25) lines.push(`(${s.tools.length - 25} smaller tools omitted from this table; --json has all)`);
|
|
14178
|
+
lines.push("");
|
|
14179
|
+
lines.push("## Attachment kinds by modeled billed cost (model-visible fields only; NOT billed units)");
|
|
14180
|
+
lines.push("Model: est-tokens x 1.25 cache-write + reread-tokens x 0.1 cache-read; an injection stays in context until the next compact boundary on its lane, or end of transcript.");
|
|
14181
|
+
for (const a of s.attachmentKinds.slice(0, 15)) {
|
|
14182
|
+
lines.push(`${a.kind.padEnd(28)} inj ${fmt(a.injections).padStart(9)} visible-bytes ${fmt(a.visibleBytes).padStart(13)} est-tok ${fmt(a.estTokens).padStart(12)} reread-tok ${fmt(a.rereadTokens).padStart(14)} billed-equiv ${fmt(a.billedEquivTokens).padStart(12)} identical-reinject ${fmt(a.repeatedIdentical).padStart(8)}`);
|
|
14183
|
+
}
|
|
14184
|
+
if (s.attachmentKinds.length > 15) lines.push(`(${s.attachmentKinds.length - 15} smaller kinds omitted from this table; --json has all)`);
|
|
14185
|
+
lines.push("");
|
|
14186
|
+
lines.push("## Hook stdout channel (hook_success attachments; context-bytes is the model-visible share)");
|
|
14187
|
+
for (const h of s.hookOutputs) {
|
|
14188
|
+
lines.push(`${h.origin.padEnd(11)} ${h.event.padEnd(18)} fires ${fmt(h.fires).padStart(9)} stdout-bytes ${fmt(h.stdoutBytes).padStart(13)} context-bytes ${fmt(h.contextBytes).padStart(13)}`);
|
|
14189
|
+
}
|
|
14190
|
+
lines.push("");
|
|
14191
|
+
lines.push("## Measured billed tokens by session position (deciles of each session's API calls)");
|
|
14192
|
+
for (const d of s.positionDeciles) {
|
|
14193
|
+
lines.push(`decile ${d.decile.toString().padStart(2)} calls ${fmt(d.apiCalls).padStart(9)} input ${fmt(d.inputTokens).padStart(15)} cache-read ${fmt(d.cacheReadTokens).padStart(15)} output ${fmt(d.outputTokens).padStart(11)}`);
|
|
14194
|
+
}
|
|
14195
|
+
lines.push("");
|
|
14196
|
+
lines.push("## Subagent lanes (spawn-prefix carriage; same residency model as the attachment census; NOT billed units)");
|
|
14197
|
+
const sl = s.sidechainLanes;
|
|
14198
|
+
lines.push(`Lane files: ${fmt(sl.laneFiles)} (${fmt(sl.lanesWithUsage)} with usage)`);
|
|
14199
|
+
lines.push(`First-call prefix tokens: mean ${fmt(sl.meanFirstCallPrefixTokens)}, median ${fmt(sl.medianFirstCallPrefixTokens)}, p90 ${fmt(sl.p90FirstCallPrefixTokens)}`);
|
|
14200
|
+
lines.push(`Calls per lane: mean ${fmt(sl.meanCallsPerLane)}; task-brief bytes: mean ${fmt(sl.meanBriefBytes)}`);
|
|
14201
|
+
lines.push(`Prefix billed-equiv tokens: ${fmt(sl.prefixBilledEquivTokens)} (write x 1.25, then x 0.1 per later call in the lane)`);
|
|
14202
|
+
for (const t of s.laneAgentTypes.slice(0, 12)) {
|
|
14203
|
+
lines.push(` type ${t.agentType.padEnd(24)} lanes ${fmt(t.lanes).padStart(7)} (${fmt(t.lanesWithUsage)} with usage) prefix mean ${fmt(t.meanFirstCallPrefixTokens).padStart(9)}, median ${fmt(t.medianFirstCallPrefixTokens).padStart(9)}`);
|
|
14204
|
+
}
|
|
14205
|
+
if (s.laneAgentTypes.length > 12) lines.push(` (${s.laneAgentTypes.length - 12} smaller agent types omitted from this table; --json has all)`);
|
|
14206
|
+
lines.push("");
|
|
14207
|
+
lines.push("## Read interception (token-goat divert markers inside Read tool results)");
|
|
14208
|
+
const ri = s.readInterception;
|
|
14209
|
+
lines.push(`Read results: ${fmt(ri.readResults)}; diverted by marker: ${fmt(ri.divertedByMarker)} (${fmt(ri.divertedBytes)} bytes); full serves >=10 KiB: ${fmt(ri.fullServesOver10k)} (${fmt(ri.fullServeBytesOver10k)} bytes)`);
|
|
14210
|
+
lines.push(`Full-serve split (same transcript file; a session's lanes are separate files, so repeats UNDER-count): first read ${fmt(ri.fullServesFirstRead)}, repeat ${fmt(ri.fullServesRepeat)} (${fmt(ri.repeatBytes)} bytes), path unknown ${fmt(ri.fullServesPathUnknown)}`);
|
|
14211
|
+
lines.push(`Repeats: with offset/limit ${fmt(ri.repeatWithRange)} (deliberate paging), whole-file ${fmt(ri.repeatFullNoRange)} (divert-miss candidates), in sessions with a token-goat hook fire ${fmt(ri.repeatInHookedSessions)} (whole-file among them: ${fmt(ri.repeatFullNoRangeInHookedSessions)}, of which post-compaction and so correct by design: ${fmt(ri.repeatFullNoRangeHookedAfterCompaction)})`);
|
|
14212
|
+
lines.push("");
|
|
14213
|
+
lines.push("## Bash filter fire-rate (token-goat in-band markers inside Bash tool results)");
|
|
14214
|
+
const bi = s.bashInterception;
|
|
14215
|
+
lines.push(`Bash results: ${fmt(bi.bashResults)}; marked by a filter: ${fmt(bi.markedByFilter)} (${fmt(bi.markedBytes)} bytes); small unmarked <${fmt(BASH_SMALL_RESULT_MAX_BYTES)} B: ${fmt(bi.smallUntouched)} (${fmt(bi.smallUntouchedBytes)} bytes)`);
|
|
14216
|
+
lines.push(`Untouched >=${fmt(BASH_SMALL_RESULT_MAX_BYTES)} B: ${fmt(bi.untouched)} (${fmt(bi.untouchedBytes)} bytes, est-tokens ${fmt(bi.untouchedEstTokens)}, billed-equiv ${fmt(bi.untouchedBilledEquivTokens)})`);
|
|
14217
|
+
lines.push("A filter that matched but fell under the 100-byte net-savings floor leaves no transcript trace and counts as untouched here.");
|
|
14218
|
+
for (const h of bi.untouchedHeads.slice(0, 15)) {
|
|
14219
|
+
lines.push(` ${h.head.padEnd(28)} results ${fmt(h.results).padStart(9)} bytes ${fmt(h.bytes).padStart(15)}`);
|
|
14220
|
+
}
|
|
14221
|
+
if (bi.untouchedHeads.length > 15) lines.push(` (${bi.untouchedHeads.length - 15} smaller command heads omitted from this table; --json has all)`);
|
|
14222
|
+
lines.push("");
|
|
14223
|
+
lines.push("## Mid-trim omission markers inside tool results");
|
|
14224
|
+
lines.push(`fires: ${fmt(s.omissionMarkers.fires)}, lines discarded: ${fmt(s.omissionMarkers.linesOmitted)}`);
|
|
14225
|
+
return lines.join("\n");
|
|
14226
|
+
}
|
|
14227
|
+
|
|
14228
|
+
// src/cli_mcp_audit.ts
|
|
14229
|
+
import * as fs27 from "node:fs";
|
|
14230
|
+
import * as os13 from "node:os";
|
|
14231
|
+
import * as path27 from "node:path";
|
|
14232
|
+
function readMcpJsonFile(configPath2) {
|
|
14233
|
+
try {
|
|
14234
|
+
if (!fs27.existsSync(configPath2)) return null;
|
|
14235
|
+
const content = fs27.readFileSync(configPath2, "utf-8");
|
|
13515
14236
|
const parsed = JSON.parse(content);
|
|
13516
|
-
|
|
14237
|
+
const servers = parsed && typeof parsed === "object" ? parsed.mcpServers ?? parsed : null;
|
|
14238
|
+
return servers && typeof servers === "object" ? servers : null;
|
|
14239
|
+
} catch {
|
|
14240
|
+
return null;
|
|
14241
|
+
}
|
|
14242
|
+
}
|
|
14243
|
+
function driveLetterCaseVariants(p) {
|
|
14244
|
+
const m = /^([a-zA-Z]:)(.*)$/s.exec(p);
|
|
14245
|
+
if (m === null) return [p];
|
|
14246
|
+
const [, drive, rest] = m;
|
|
14247
|
+
return [`${drive.toLowerCase()}${rest}`, `${drive.toUpperCase()}${rest}`];
|
|
14248
|
+
}
|
|
14249
|
+
function readClaudeJsonConfig(claudeJsonPath, projectRoot) {
|
|
14250
|
+
try {
|
|
14251
|
+
if (!fs27.existsSync(claudeJsonPath)) return null;
|
|
14252
|
+
const parsed = JSON.parse(fs27.readFileSync(claudeJsonPath, "utf-8"));
|
|
14253
|
+
const projects = parsed && typeof parsed === "object" ? parsed.projects : null;
|
|
14254
|
+
if (!projects || typeof projects !== "object") return null;
|
|
14255
|
+
const slashForms = [projectRoot, projectRoot.replace(/\\/g, "/"), projectRoot.replace(/\//g, "\\")];
|
|
14256
|
+
const candidates = [...new Set(slashForms.flatMap(driveLetterCaseVariants))];
|
|
14257
|
+
for (const key of candidates) {
|
|
14258
|
+
const entry = projects[key];
|
|
14259
|
+
if (entry && typeof entry === "object") {
|
|
14260
|
+
const servers = entry["mcpServers"];
|
|
14261
|
+
if (servers && typeof servers === "object") return servers;
|
|
14262
|
+
}
|
|
14263
|
+
}
|
|
14264
|
+
return null;
|
|
13517
14265
|
} catch {
|
|
13518
14266
|
return null;
|
|
13519
14267
|
}
|
|
13520
14268
|
}
|
|
14269
|
+
function discoverMcpConfig(projectRoot, home) {
|
|
14270
|
+
const mcpJsonPath = path27.join(projectRoot, ".mcp.json");
|
|
14271
|
+
const claudeJsonPath = path27.join(home, ".claude.json");
|
|
14272
|
+
const sourcesChecked = [mcpJsonPath, claudeJsonPath];
|
|
14273
|
+
const fromMcpJson = readMcpJsonFile(mcpJsonPath);
|
|
14274
|
+
if (fromMcpJson !== null) return { servers: fromMcpJson, sourcePath: mcpJsonPath, sourcesChecked };
|
|
14275
|
+
const fromClaudeJson = readClaudeJsonConfig(claudeJsonPath, projectRoot);
|
|
14276
|
+
if (fromClaudeJson !== null) return { servers: fromClaudeJson, sourcePath: claudeJsonPath, sourcesChecked };
|
|
14277
|
+
return { servers: null, sourcePath: null, sourcesChecked };
|
|
14278
|
+
}
|
|
13521
14279
|
function analyzeMcpCache() {
|
|
13522
14280
|
const serverMetrics = /* @__PURE__ */ new Map();
|
|
13523
14281
|
const blobs = listBlobs(BASH_OUTPUT_SUBDIR).filter((b) => b.id.startsWith("mcp_"));
|
|
@@ -13539,8 +14297,9 @@ function analyzeMcpCache() {
|
|
|
13539
14297
|
}
|
|
13540
14298
|
return serverMetrics;
|
|
13541
14299
|
}
|
|
13542
|
-
function buildMcpAuditReport(projectRoot) {
|
|
13543
|
-
const
|
|
14300
|
+
function buildMcpAuditReport(projectRoot, home = os13.homedir()) {
|
|
14301
|
+
const discovery = discoverMcpConfig(projectRoot, home);
|
|
14302
|
+
const config = discovery.servers;
|
|
13544
14303
|
const cacheMetrics = analyzeMcpCache();
|
|
13545
14304
|
const servers = [];
|
|
13546
14305
|
let totalCost = 0;
|
|
@@ -13574,7 +14333,10 @@ function buildMcpAuditReport(projectRoot) {
|
|
|
13574
14333
|
servers.sort((a, b) => b.totalTokens - a.totalTokens);
|
|
13575
14334
|
return {
|
|
13576
14335
|
projectRoot,
|
|
13577
|
-
configFound:
|
|
14336
|
+
configFound: discovery.sourcePath !== null,
|
|
14337
|
+
configSourcePath: discovery.sourcePath,
|
|
14338
|
+
configSourcesChecked: discovery.sourcesChecked,
|
|
14339
|
+
costKnown: discovery.sourcePath !== null || cacheMetrics.size > 0,
|
|
13578
14340
|
servers,
|
|
13579
14341
|
totalCost
|
|
13580
14342
|
};
|
|
@@ -13586,11 +14348,12 @@ function printReport3(report) {
|
|
|
13586
14348
|
w("\n# token-goat mcp-audit\n");
|
|
13587
14349
|
w(`Project: ${report.projectRoot}
|
|
13588
14350
|
`);
|
|
13589
|
-
w(`Config found: ${report.
|
|
14351
|
+
w(report.configSourcePath !== null ? `Config found: yes (${report.configSourcePath})
|
|
14352
|
+
` : `Config found: no (checked: ${report.configSourcesChecked.join(", ")})
|
|
13590
14353
|
`);
|
|
13591
14354
|
w("\n## MCP servers\n");
|
|
13592
14355
|
if (report.servers.length === 0) {
|
|
13593
|
-
w(" none\n");
|
|
14356
|
+
w(report.costKnown ? " none\n" : " none discovered from a readable config source\n");
|
|
13594
14357
|
} else {
|
|
13595
14358
|
w("| Server | Per-Call (tok) | Calls | Total (tok) |\n");
|
|
13596
14359
|
w("|--------|---|---|---|\n");
|
|
@@ -13599,9 +14362,10 @@ function printReport3(report) {
|
|
|
13599
14362
|
`);
|
|
13600
14363
|
}
|
|
13601
14364
|
}
|
|
13602
|
-
w(`
|
|
14365
|
+
w(report.costKnown ? `
|
|
13603
14366
|
Total cost: ${report.totalCost} tok
|
|
13604
|
-
`);
|
|
14367
|
+
` : "\nTotal cost: unknown -- no readable MCP config was found and no MCP calls have been recorded in this session's cache yet\n");
|
|
14368
|
+
w("\nNote: plugin-provided MCP servers have no on-disk config token-goat can read, so a live session may have MCP servers this audit cannot see or price.\n");
|
|
13605
14369
|
}
|
|
13606
14370
|
async function runMcpAuditCommand(opts = {}) {
|
|
13607
14371
|
const projectRoot = resolveProjectRoot(opts.project !== void 0 ? { project: opts.project } : {});
|
|
@@ -13620,6 +14384,20 @@ var RECALL_COMMAND = {
|
|
|
13620
14384
|
web: "web-output",
|
|
13621
14385
|
mcp: "mcp-output"
|
|
13622
14386
|
};
|
|
14387
|
+
function fenceTagForCacheType(cacheType) {
|
|
14388
|
+
return cacheType === "web" ? UNTRUSTED_WEB_TAG : UNTRUSTED_TOOL_TAG;
|
|
14389
|
+
}
|
|
14390
|
+
function fenceSnippetIfMatched(hit) {
|
|
14391
|
+
let matches2 = [];
|
|
14392
|
+
try {
|
|
14393
|
+
if (loadConfig().injection.enabled) matches2 = scanForInjectionPatterns(hit.snippet);
|
|
14394
|
+
} catch {
|
|
14395
|
+
matches2 = [];
|
|
14396
|
+
}
|
|
14397
|
+
if (matches2.length === 0) return hit.snippet;
|
|
14398
|
+
recordStat("injection_detected", 0, 0, void 0, matches2.join(","));
|
|
14399
|
+
return fenceUntrustedContent(hit.snippet, matches2, fenceTagForCacheType(hit.cacheType));
|
|
14400
|
+
}
|
|
13623
14401
|
function printHits(query, hits) {
|
|
13624
14402
|
const w = (text) => {
|
|
13625
14403
|
process.stdout.write(text);
|
|
@@ -13635,7 +14413,7 @@ function printHits(query, hits) {
|
|
|
13635
14413
|
`);
|
|
13636
14414
|
w(` ${label}
|
|
13637
14415
|
`);
|
|
13638
|
-
w(` ${hit
|
|
14416
|
+
w(` ${fenceSnippetIfMatched(hit)}
|
|
13639
14417
|
|
|
13640
14418
|
`);
|
|
13641
14419
|
}
|
|
@@ -13648,7 +14426,8 @@ function runRecallCommand(query, opts = {}) {
|
|
|
13648
14426
|
};
|
|
13649
14427
|
const hits = browse ? listRecentRecall(scope) : searchRecall(query, scope);
|
|
13650
14428
|
if (opts.json === true) {
|
|
13651
|
-
|
|
14429
|
+
const fenced = hits.map((hit) => ({ ...hit, snippet: fenceSnippetIfMatched(hit) }));
|
|
14430
|
+
process.stdout.write(`${JSON.stringify(fenced)}
|
|
13652
14431
|
`);
|
|
13653
14432
|
return;
|
|
13654
14433
|
}
|
|
@@ -13671,19 +14450,20 @@ function printSummary(rows) {
|
|
|
13671
14450
|
};
|
|
13672
14451
|
w(pad("category", 22) + pad("emitted", 9) + pad("acted-on", 10) + pad("efficacy", 10) + pad("suppressed", 17) + pad("manual+", 9) + pad("manual-", 9) + "spent\n");
|
|
13673
14452
|
for (const row of rows) {
|
|
13674
|
-
const
|
|
14453
|
+
const pct2 = row.efficacyPct === null ? "n/a" : `${row.efficacyPct}%`;
|
|
13675
14454
|
w(
|
|
13676
|
-
pad(row.category, 22) + pad(String(row.emitted), 9) + pad(String(row.actedOn), 10) + pad(
|
|
14455
|
+
pad(row.category, 22) + pad(String(row.emitted), 9) + pad(String(row.actedOn), 10) + pad(pct2, 10) + pad(suppressedCell(row), 17) + pad(String(row.manualEffective), 9) + pad(String(row.manualIneffective), 9) + formatSpentCell(row) + "\n"
|
|
13677
14456
|
);
|
|
13678
14457
|
}
|
|
13679
14458
|
}
|
|
13680
14459
|
function printTotals(totals) {
|
|
13681
14460
|
const spent = totals.spentBytes === null ? "n/a" : String(totals.spentBytes);
|
|
13682
|
-
const net = totals.netBytes === null ? "n/a" : String(totals.netBytes);
|
|
13683
14461
|
const legacyNote = totals.legacyEmissions > 0 ? ` (excludes ${totals.legacyEmissions} legacy emission(s) recorded before spend tracking)` : "";
|
|
13684
|
-
process.stdout.write(
|
|
13685
|
-
|
|
13686
|
-
|
|
14462
|
+
process.stdout.write(
|
|
14463
|
+
`
|
|
14464
|
+
TOTAL saved=${totals.savedBytes} (all-time, every hint kind) spent=${spent} (hint_emissions ledger only)${legacyNote}
|
|
14465
|
+
`
|
|
14466
|
+
);
|
|
13687
14467
|
}
|
|
13688
14468
|
function runHintStatsCommand(opts = {}) {
|
|
13689
14469
|
if (opts.reset === true) {
|
|
@@ -13817,10 +14597,10 @@ function readBoundedText(text, file) {
|
|
|
13817
14597
|
if (text === void 0 && file === void 0) throw new CliError("provide text or --file");
|
|
13818
14598
|
let value;
|
|
13819
14599
|
if (file !== void 0) {
|
|
13820
|
-
if (
|
|
14600
|
+
if (fs28.statSync(file).size > CONTENT_MAX_INPUT_CHARS) {
|
|
13821
14601
|
throw new CliError(`file exceeds the ${CONTENT_MAX_INPUT_CHARS}-byte safety limit`);
|
|
13822
14602
|
}
|
|
13823
|
-
value =
|
|
14603
|
+
value = fs28.readFileSync(file, "utf8");
|
|
13824
14604
|
} else {
|
|
13825
14605
|
if (text === void 0) throw new CliError("provide text or --file");
|
|
13826
14606
|
value = text;
|
|
@@ -13957,10 +14737,10 @@ async function cmdIndex(pathArg, opts = {}) {
|
|
|
13957
14737
|
continue;
|
|
13958
14738
|
}
|
|
13959
14739
|
const sha = fingerprintFile(key);
|
|
13960
|
-
if (sha === null && !
|
|
14740
|
+
if (sha === null && !fs28.existsSync(key)) continue;
|
|
13961
14741
|
const entry = sha !== null ? getFileEntry(key, dbPath) : null;
|
|
13962
14742
|
const spellingStale = entry !== null && indexedPathSpellingIsStale(entry.filePath, key);
|
|
13963
|
-
const parseUnchanged = !force && !spellingStale && sha !== null && entry?.sha === sha;
|
|
14743
|
+
const parseUnchanged = !force && !spellingStale && sha !== null && entry?.sha === sha && entry.parserSha === PARSER_FINGERPRINT;
|
|
13964
14744
|
const embeddingsEnabled = loadConfig().indexing?.embeddings_enabled ?? true;
|
|
13965
14745
|
const depsAvailable = embeddingsEnabled && embeddingsDepsAvailable(getDb(dbPath));
|
|
13966
14746
|
const embedUnchanged = !force && parseUnchanged && sha !== null && isEmbedFresh(entry?.embedSha, sha, embeddingsEnabled, depsAvailable);
|
|
@@ -13983,7 +14763,7 @@ async function cmdIndex(pathArg, opts = {}) {
|
|
|
13983
14763
|
}
|
|
13984
14764
|
continue;
|
|
13985
14765
|
}
|
|
13986
|
-
if (!
|
|
14766
|
+
if (!fs28.existsSync(key)) continue;
|
|
13987
14767
|
}
|
|
13988
14768
|
if (!embedUnchanged) {
|
|
13989
14769
|
paintProgress("embedding");
|
|
@@ -14016,7 +14796,7 @@ function cmdMap(opts) {
|
|
|
14016
14796
|
out(text);
|
|
14017
14797
|
}
|
|
14018
14798
|
const bytesSaved = mapLookupBytesSaved(map, text);
|
|
14019
|
-
recordStat("map_lookup", bytesSaved,
|
|
14799
|
+
recordStat("map_lookup", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14020
14800
|
}
|
|
14021
14801
|
function cmdBridgesStatus(opts) {
|
|
14022
14802
|
if (opts.json === true) {
|
|
@@ -14043,7 +14823,7 @@ async function cmdMcpServe() {
|
|
|
14043
14823
|
let StdioServerTransport;
|
|
14044
14824
|
try {
|
|
14045
14825
|
;
|
|
14046
|
-
({ createMcpServer } = await import("./token-goat-chunk-
|
|
14826
|
+
({ createMcpServer } = await import("./token-goat-chunk-LZOAPGWR.mjs"));
|
|
14047
14827
|
({ StdioServerTransport } = await import("./token-goat-chunk-324QOJYZ.mjs"));
|
|
14048
14828
|
} catch (err2) {
|
|
14049
14829
|
process.stderr.write(
|
|
@@ -14056,8 +14836,8 @@ async function cmdMcpServe() {
|
|
|
14056
14836
|
const server = await createMcpServer();
|
|
14057
14837
|
const transport = new StdioServerTransport();
|
|
14058
14838
|
await server.connect(transport);
|
|
14059
|
-
await new Promise((
|
|
14060
|
-
server.onclose =
|
|
14839
|
+
await new Promise((resolve13) => {
|
|
14840
|
+
server.onclose = resolve13;
|
|
14061
14841
|
});
|
|
14062
14842
|
}
|
|
14063
14843
|
function printBridgeVerificationNotice(harness) {
|
|
@@ -14068,7 +14848,7 @@ async function cmdHook(event, opts) {
|
|
|
14068
14848
|
if (typeof opts.harness === "string" && opts.harness.length > 0) {
|
|
14069
14849
|
process.env[ENV_KEYS.HARNESS_OVERRIDE] = opts.harness;
|
|
14070
14850
|
}
|
|
14071
|
-
const { relay } = await import("./token-goat-chunk-
|
|
14851
|
+
const { relay } = await import("./token-goat-chunk-L6YP6FKQ.mjs");
|
|
14072
14852
|
await relay(event);
|
|
14073
14853
|
}
|
|
14074
14854
|
async function cmdInstall(opts) {
|
|
@@ -14181,16 +14961,16 @@ async function cmdInstall(opts) {
|
|
|
14181
14961
|
);
|
|
14182
14962
|
}
|
|
14183
14963
|
try {
|
|
14184
|
-
const skillDir =
|
|
14185
|
-
if (
|
|
14186
|
-
const entries =
|
|
14964
|
+
const skillDir = path28.join(homedir13(), ".claude", "skills");
|
|
14965
|
+
if (fs28.existsSync(skillDir)) {
|
|
14966
|
+
const entries = fs28.readdirSync(skillDir, { withFileTypes: true });
|
|
14187
14967
|
const skillNames = [];
|
|
14188
14968
|
const sessionId = getSessionId();
|
|
14189
14969
|
for (const entry of entries) {
|
|
14190
14970
|
if (!entry.isDirectory()) continue;
|
|
14191
|
-
const skillFile =
|
|
14192
|
-
if (
|
|
14193
|
-
const body =
|
|
14971
|
+
const skillFile = path28.join(skillDir, entry.name, "SKILL.md");
|
|
14972
|
+
if (fs28.existsSync(skillFile)) {
|
|
14973
|
+
const body = fs28.readFileSync(skillFile, "utf-8");
|
|
14194
14974
|
const compact = extractCompactFromMarker(body);
|
|
14195
14975
|
if (compact === null) continue;
|
|
14196
14976
|
const sourceSha = contentHash(body);
|
|
@@ -14200,10 +14980,10 @@ async function cmdInstall(opts) {
|
|
|
14200
14980
|
}
|
|
14201
14981
|
if (skillNames.length > 0) {
|
|
14202
14982
|
const dir = skillOutputsDir();
|
|
14203
|
-
await
|
|
14204
|
-
const pregenPath =
|
|
14983
|
+
await fs28.promises.mkdir(dir, { recursive: true });
|
|
14984
|
+
const pregenPath = path28.join(dir, "pregen.json");
|
|
14205
14985
|
const pregenData = { ts: Date.now(), names: skillNames };
|
|
14206
|
-
await
|
|
14986
|
+
await fs28.promises.writeFile(pregenPath, JSON.stringify(pregenData, null, 2));
|
|
14207
14987
|
out(`Pre-generated ${skillNames.length} skill compacts.`);
|
|
14208
14988
|
}
|
|
14209
14989
|
}
|
|
@@ -14381,6 +15161,15 @@ function cmdWaste(opts = {}) {
|
|
|
14381
15161
|
...opts.copilot === true ? { copilot: true } : {}
|
|
14382
15162
|
});
|
|
14383
15163
|
}
|
|
15164
|
+
async function cmdSessionAudit(opts = {}) {
|
|
15165
|
+
let summary;
|
|
15166
|
+
try {
|
|
15167
|
+
summary = await auditSessionCorpus({ ...opts.dir !== void 0 ? { dir: opts.dir } : {} });
|
|
15168
|
+
} catch (err2) {
|
|
15169
|
+
throw new CliError(err2 instanceof Error ? err2.message : String(err2));
|
|
15170
|
+
}
|
|
15171
|
+
out(opts.json === true ? JSON.stringify(summary) : formatSessionAudit(summary));
|
|
15172
|
+
}
|
|
14384
15173
|
async function cmdSessionOutline(sessionIdOrPath, opts = {}) {
|
|
14385
15174
|
const transcriptPath = resolveSessionTranscript(sessionIdOrPath, opts.project !== void 0 ? { project: opts.project } : {});
|
|
14386
15175
|
if (transcriptPath === null) {
|
|
@@ -14394,11 +15183,11 @@ ${formatSessionOutline(turns)}`;
|
|
|
14394
15183
|
out(text);
|
|
14395
15184
|
const fullSourceBytes = sessionTranscriptSize(transcriptPath);
|
|
14396
15185
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14397
|
-
recordStat("session_outline", bytesSaved,
|
|
15186
|
+
recordStat("session_outline", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14398
15187
|
}
|
|
14399
15188
|
function sessionTranscriptSize(transcriptPath) {
|
|
14400
15189
|
try {
|
|
14401
|
-
return
|
|
15190
|
+
return fs28.statSync(transcriptPath).size;
|
|
14402
15191
|
} catch {
|
|
14403
15192
|
return 0;
|
|
14404
15193
|
}
|
|
@@ -14416,7 +15205,7 @@ async function cmdSessionSlice(sessionIdOrPath, opts) {
|
|
|
14416
15205
|
out(text);
|
|
14417
15206
|
const fullSourceBytes = sessionTranscriptSize(transcriptPath);
|
|
14418
15207
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14419
|
-
recordStat("session_slice", bytesSaved,
|
|
15208
|
+
recordStat("session_slice", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14420
15209
|
}
|
|
14421
15210
|
function cmdMcpAudit(opts = {}) {
|
|
14422
15211
|
return runMcpAuditCommand({
|
|
@@ -14522,9 +15311,20 @@ function _applyFiltersAndPrint(content, opts, fenceUntrusted = false, fenceTag =
|
|
|
14522
15311
|
}
|
|
14523
15312
|
return emit2(result.join("\n"));
|
|
14524
15313
|
}
|
|
15314
|
+
function fenceFileTextIfMatched(text) {
|
|
15315
|
+
let matches2 = [];
|
|
15316
|
+
try {
|
|
15317
|
+
if (loadConfig().injection.enabled) matches2 = scanForInjectionPatterns(text);
|
|
15318
|
+
} catch {
|
|
15319
|
+
matches2 = [];
|
|
15320
|
+
}
|
|
15321
|
+
if (matches2.length === 0) return text;
|
|
15322
|
+
recordStat("injection_detected", 0, 0, void 0, matches2.join(","));
|
|
15323
|
+
return fenceUntrustedContent(text, matches2, UNTRUSTED_FILE_TAG);
|
|
15324
|
+
}
|
|
14525
15325
|
function fileSizeOrZero(filePath) {
|
|
14526
15326
|
try {
|
|
14527
|
-
return
|
|
15327
|
+
return fs28.statSync(filePath).size;
|
|
14528
15328
|
} catch {
|
|
14529
15329
|
return 0;
|
|
14530
15330
|
}
|
|
@@ -14539,11 +15339,11 @@ function cmdBashOutput(id, opts) {
|
|
|
14539
15339
|
}
|
|
14540
15340
|
let content;
|
|
14541
15341
|
try {
|
|
14542
|
-
const st =
|
|
15342
|
+
const st = fs28.statSync(opts.file);
|
|
14543
15343
|
if (st.isFIFO() || st.isSocket()) {
|
|
14544
15344
|
throw new CliError(`--file '${opts.file}' is a special file (FIFO or socket) \u2014 only regular files are supported`);
|
|
14545
15345
|
}
|
|
14546
|
-
content = redactIfDotenv(opts.file, decodeSource(
|
|
15346
|
+
content = redactIfDotenv(opts.file, decodeSource(fs28.readFileSync(opts.file)));
|
|
14547
15347
|
} catch (e) {
|
|
14548
15348
|
if (e instanceof CliError) throw e;
|
|
14549
15349
|
throw new CliError(`cannot read file: ${opts.file}`);
|
|
@@ -14585,10 +15385,10 @@ function cmdMcpOutput(id, opts) {
|
|
|
14585
15385
|
}
|
|
14586
15386
|
async function cmdPdfExtract(file, opts) {
|
|
14587
15387
|
const text = await runPdfExtractText(file, opts.pages, opts.layout === true);
|
|
14588
|
-
const printed = _applyFiltersAndPrint(text, opts);
|
|
15388
|
+
const printed = _applyFiltersAndPrint(text, opts, true, UNTRUSTED_FILE_TAG);
|
|
14589
15389
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14590
15390
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(printed, "utf8"));
|
|
14591
|
-
recordStat("pdf_extract", bytesSaved,
|
|
15391
|
+
recordStat("pdf_extract", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14592
15392
|
}
|
|
14593
15393
|
async function cmdPdfLocate(file, pattern, opts) {
|
|
14594
15394
|
const locateOpts = {
|
|
@@ -14598,24 +15398,25 @@ async function cmdPdfLocate(file, pattern, opts) {
|
|
|
14598
15398
|
if (opts.context !== void 0) locateOpts.context = requirePositiveInt("--context", opts.context);
|
|
14599
15399
|
if (opts.pages !== void 0) locateOpts.pages = opts.pages;
|
|
14600
15400
|
const matches2 = await runPdfLocate(file, pattern, locateOpts);
|
|
14601
|
-
const
|
|
15401
|
+
const fencedMatches = matches2.map((m) => ({ ...m, snippet: fenceFileTextIfMatched(m.snippet) }));
|
|
15402
|
+
const pages = fencedMatches.map((m) => m.page);
|
|
14602
15403
|
let printed;
|
|
14603
15404
|
if (opts.json === true) {
|
|
14604
|
-
printed = JSON.stringify({ file, pattern, matchCount:
|
|
15405
|
+
printed = JSON.stringify({ file, pattern, matchCount: fencedMatches.length, pages, matches: fencedMatches }, null, 2);
|
|
14605
15406
|
out(printed);
|
|
14606
|
-
} else if (
|
|
15407
|
+
} else if (fencedMatches.length === 0) {
|
|
14607
15408
|
printed = "(no matches)";
|
|
14608
15409
|
out(printed);
|
|
14609
15410
|
} else {
|
|
14610
|
-
const lines =
|
|
15411
|
+
const lines = fencedMatches.map((m) => `p${m.page}: ${m.snippet}`);
|
|
14611
15412
|
printed = `${lines.join("\n")}
|
|
14612
15413
|
|
|
14613
|
-
${countNoun(
|
|
15414
|
+
${countNoun(fencedMatches.length, "match", "matches")} across ${countNoun(pages.length, "page")}`;
|
|
14614
15415
|
out(printed);
|
|
14615
15416
|
}
|
|
14616
15417
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14617
15418
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(printed, "utf8"));
|
|
14618
|
-
recordStat("pdf_locate", bytesSaved,
|
|
15419
|
+
recordStat("pdf_locate", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14619
15420
|
}
|
|
14620
15421
|
async function cmdPdfOutline(file, opts) {
|
|
14621
15422
|
const entries = await runPdfOutline(file);
|
|
@@ -14627,11 +15428,12 @@ async function cmdPdfOutline(file, opts) {
|
|
|
14627
15428
|
}
|
|
14628
15429
|
return;
|
|
14629
15430
|
}
|
|
14630
|
-
const
|
|
15431
|
+
const fencedEntries = entries.map((e) => ({ ...e, title: fenceFileTextIfMatched(e.title) }));
|
|
15432
|
+
const text = opts.json === true ? JSON.stringify(fencedEntries, null, 2) : fencedEntries.map((e) => `${" ".repeat(e.level)}${e.title}${e.page !== null ? ` (p.${e.page})` : ""}`).join("\n");
|
|
14631
15433
|
out(text);
|
|
14632
15434
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14633
15435
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14634
|
-
recordStat("pdf_outline", bytesSaved,
|
|
15436
|
+
recordStat("pdf_outline", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14635
15437
|
}
|
|
14636
15438
|
async function cmdPdfMeta(file, opts = {}) {
|
|
14637
15439
|
const meta = await runPdfMeta(file);
|
|
@@ -14645,7 +15447,7 @@ async function cmdPdfMeta(file, opts = {}) {
|
|
|
14645
15447
|
out(text);
|
|
14646
15448
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14647
15449
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14648
|
-
recordStat("pdf_meta", bytesSaved,
|
|
15450
|
+
recordStat("pdf_meta", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14649
15451
|
}
|
|
14650
15452
|
async function cmdImageMeta(file, opts = {}) {
|
|
14651
15453
|
const meta = await runImageMeta(file);
|
|
@@ -14667,7 +15469,17 @@ ${msg}`;
|
|
|
14667
15469
|
out(text);
|
|
14668
15470
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14669
15471
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14670
|
-
recordStat("image_meta", bytesSaved,
|
|
15472
|
+
recordStat("image_meta", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
15473
|
+
}
|
|
15474
|
+
function fenceOcrText(text) {
|
|
15475
|
+
try {
|
|
15476
|
+
if (loadConfig().injection.enabled) {
|
|
15477
|
+
const matches2 = scanForInjectionPatterns(text);
|
|
15478
|
+
if (matches2.length > 0) recordStat("injection_detected", 0, 0, void 0, matches2.join(","));
|
|
15479
|
+
}
|
|
15480
|
+
} catch {
|
|
15481
|
+
}
|
|
15482
|
+
return fenceUntrustedOcrText(text);
|
|
14671
15483
|
}
|
|
14672
15484
|
async function cmdImageText(file, opts = {}) {
|
|
14673
15485
|
const result = await runImageText(file);
|
|
@@ -14679,18 +15491,22 @@ async function cmdImageText(file, opts = {}) {
|
|
|
14679
15491
|
}
|
|
14680
15492
|
let text;
|
|
14681
15493
|
if (opts.json === true) {
|
|
14682
|
-
text = JSON.stringify(
|
|
15494
|
+
text = JSON.stringify(
|
|
15495
|
+
result.text === null ? result : { ...result, text: fenceOcrText(result.text) },
|
|
15496
|
+
null,
|
|
15497
|
+
2
|
|
15498
|
+
);
|
|
14683
15499
|
} else {
|
|
14684
15500
|
const lines = [`Confidence: ${Math.round(result.confidence)}%`, `Characters: ${result.chars}`];
|
|
14685
15501
|
text = result.textHeavy && result.text !== null ? `${lines.join("\n")}
|
|
14686
15502
|
|
|
14687
|
-
${result.text}` : `${lines.join("\n")}
|
|
15503
|
+
${fenceOcrText(result.text)}` : `${lines.join("\n")}
|
|
14688
15504
|
(below usefulness threshold; text likely noise, not shown)`;
|
|
14689
15505
|
}
|
|
14690
15506
|
out(text);
|
|
14691
15507
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14692
15508
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14693
|
-
recordStat("image_text", bytesSaved,
|
|
15509
|
+
recordStat("image_text", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14694
15510
|
}
|
|
14695
15511
|
function cmdVideoChapters(file) {
|
|
14696
15512
|
const { chapters, subtitleStreams } = extractVideoChapters(file);
|
|
@@ -14716,7 +15532,7 @@ function cmdVideoChapters(file) {
|
|
|
14716
15532
|
out(text);
|
|
14717
15533
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14718
15534
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14719
|
-
recordStat("video_chapters", bytesSaved,
|
|
15535
|
+
recordStat("video_chapters", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14720
15536
|
}
|
|
14721
15537
|
function formatVideoTimestamp(totalSeconds) {
|
|
14722
15538
|
const hours = Math.floor(totalSeconds / 3600);
|
|
@@ -14743,23 +15559,24 @@ function cmdSharepointResolve(url) {
|
|
|
14743
15559
|
function recordXlsxStat(kind, file, emitted) {
|
|
14744
15560
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14745
15561
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(emitted, "utf8"));
|
|
14746
|
-
recordStat(kind, bytesSaved,
|
|
15562
|
+
recordStat(kind, bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14747
15563
|
}
|
|
14748
15564
|
async function cmdXlsxSheets(file, opts = {}) {
|
|
14749
15565
|
const sheets = await listSheets(file);
|
|
14750
|
-
const
|
|
15566
|
+
const fencedSheets = sheets.map((s) => ({ ...s, name: fenceFileTextIfMatched(s.name) }));
|
|
15567
|
+
const text = opts.json === true ? JSON.stringify(fencedSheets.map((s) => ({ name: s.name, ref: s.ref, rows: s.rows, cols: s.cols })), null, 2) : fencedSheets.map((s) => `${s.name} ${s.ref} (${s.rows} rows x ${s.cols} cols)`).join("\n");
|
|
14751
15568
|
out(text);
|
|
14752
15569
|
recordXlsxStat("xlsx_sheets", file, text);
|
|
14753
15570
|
}
|
|
14754
15571
|
async function cmdXlsxHead(file, opts) {
|
|
14755
15572
|
const rows = opts.rows !== void 0 ? requireNonNegativeInt("--rows", opts.rows) : 20;
|
|
14756
|
-
const text = await headSheet(file, opts.sheet, rows);
|
|
15573
|
+
const text = fenceFileTextIfMatched(await headSheet(file, opts.sheet, rows));
|
|
14757
15574
|
out(text);
|
|
14758
15575
|
recordXlsxStat("xlsx_head", file, text);
|
|
14759
15576
|
}
|
|
14760
15577
|
async function cmdXlsxRange(file, opts) {
|
|
14761
15578
|
const result = await rangeSheet(file, opts.sheet, opts.range, opts.formulas === true);
|
|
14762
|
-
const text = formatXlsxRange(result);
|
|
15579
|
+
const text = fenceFileTextIfMatched(formatXlsxRange(result));
|
|
14763
15580
|
out(text);
|
|
14764
15581
|
recordXlsxStat("xlsx_range", file, text);
|
|
14765
15582
|
}
|
|
@@ -14771,31 +15588,32 @@ async function cmdXlsxQuery(file, opts) {
|
|
|
14771
15588
|
...wheres !== void 0 ? { wheres } : {},
|
|
14772
15589
|
...opts.head !== void 0 ? { head: requireNonNegativeInt("--head", opts.head) } : {}
|
|
14773
15590
|
});
|
|
14774
|
-
const text = formatCsvTable(result);
|
|
15591
|
+
const text = fenceFileTextIfMatched(formatCsvTable(result, (opts.where ?? []).map((w) => `--where ${w}`)));
|
|
14775
15592
|
out(text);
|
|
14776
15593
|
recordXlsxStat("xlsx_query", file, text);
|
|
14777
15594
|
}
|
|
14778
15595
|
function recordDocStat(kind, file, emitted) {
|
|
14779
15596
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14780
15597
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(emitted, "utf8"));
|
|
14781
|
-
recordStat(kind, bytesSaved,
|
|
15598
|
+
recordStat(kind, bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14782
15599
|
}
|
|
14783
15600
|
async function cmdPptxOutline(file, opts) {
|
|
14784
15601
|
const slides = await pptxOutline(file);
|
|
14785
|
-
const
|
|
15602
|
+
const fencedSlides = slides.map((s) => ({ ...s, title: fenceFileTextIfMatched(s.title) }));
|
|
15603
|
+
const text = opts.json === true ? JSON.stringify(fencedSlides, null, 2) : fencedSlides.map((s) => `${s.slide}. ${s.title || "(untitled)"} [${s.bodyChars} body chars${s.hasNotes ? ", has notes" : ""}]`).join("\n");
|
|
14786
15604
|
out(text);
|
|
14787
15605
|
recordDocStat("pptx_outline", file, text);
|
|
14788
15606
|
}
|
|
14789
15607
|
async function cmdPptxSlide(file, opts) {
|
|
14790
15608
|
const n = requireNonNegativeInt("--slide", opts.slide);
|
|
14791
|
-
const text = await pptxSlideText(file, n, opts.notes === true);
|
|
15609
|
+
const text = fenceFileTextIfMatched(await pptxSlideText(file, n, opts.notes === true));
|
|
14792
15610
|
out(text);
|
|
14793
15611
|
recordDocStat("pptx_slide", file, text);
|
|
14794
15612
|
}
|
|
14795
15613
|
async function cmdPptxNotes(file, opts) {
|
|
14796
15614
|
const n = opts.slide !== void 0 ? requireNonNegativeInt("--slide", opts.slide) : void 0;
|
|
14797
15615
|
const text = await pptxNotesText(file, n);
|
|
14798
|
-
const printed = text.length > 0 ? text : "no speaker notes found";
|
|
15616
|
+
const printed = text.length > 0 ? fenceFileTextIfMatched(text) : "no speaker notes found";
|
|
14799
15617
|
out(printed);
|
|
14800
15618
|
recordDocStat("pptx_notes", file, printed);
|
|
14801
15619
|
}
|
|
@@ -14805,7 +15623,7 @@ async function cmdPptxText(file, opts) {
|
|
|
14805
15623
|
out("no matches");
|
|
14806
15624
|
return;
|
|
14807
15625
|
}
|
|
14808
|
-
const text = matches2.map((m) => `Slide ${m.slide}: ...${m.snippet}...`).join("\n");
|
|
15626
|
+
const text = fenceFileTextIfMatched(matches2.map((m) => `Slide ${m.slide}: ...${m.snippet}...`).join("\n"));
|
|
14809
15627
|
out(text);
|
|
14810
15628
|
recordDocStat("pptx_text", file, text);
|
|
14811
15629
|
}
|
|
@@ -14819,13 +15637,14 @@ async function cmdDocxOutline(file, opts) {
|
|
|
14819
15637
|
}
|
|
14820
15638
|
return;
|
|
14821
15639
|
}
|
|
14822
|
-
const
|
|
15640
|
+
const fencedHeadings = headings.map((h) => ({ ...h, text: fenceFileTextIfMatched(h.text) }));
|
|
15641
|
+
const text = opts.json === true ? JSON.stringify(fencedHeadings, null, 2) : fencedHeadings.map((h) => `${" ".repeat(h.level - 1)}${h.text}`).join("\n");
|
|
14823
15642
|
out(text);
|
|
14824
15643
|
recordDocStat("docx_outline", file, text);
|
|
14825
15644
|
}
|
|
14826
15645
|
async function cmdDocxText(file, opts) {
|
|
14827
15646
|
const text = await docxText(file);
|
|
14828
|
-
const printed = _applyFiltersAndPrint(text, opts);
|
|
15647
|
+
const printed = _applyFiltersAndPrint(text, opts, true, UNTRUSTED_FILE_TAG);
|
|
14829
15648
|
recordDocStat("docx_text", file, printed);
|
|
14830
15649
|
}
|
|
14831
15650
|
function cmdTranscriptOutline(file, opts) {
|
|
@@ -14963,7 +15782,7 @@ function emitExtraFileArgsNote(command, first, extras, opts = {}) {
|
|
|
14963
15782
|
}
|
|
14964
15783
|
async function cmdCompress(opts) {
|
|
14965
15784
|
try {
|
|
14966
|
-
const bashRunner = await import("./token-goat-chunk-
|
|
15785
|
+
const bashRunner = await import("./token-goat-chunk-NZFTWBGV.mjs");
|
|
14967
15786
|
if (opts.compress === false) {
|
|
14968
15787
|
process.exitCode = bashRunner.runRaw(opts.cmd, parseTimeout(opts.timeout, bashRunner.DEFAULT_TIMEOUT_SECONDS));
|
|
14969
15788
|
return;
|
|
@@ -14989,9 +15808,12 @@ async function cmdSkillBody(name, opts) {
|
|
|
14989
15808
|
if (filePath === null) {
|
|
14990
15809
|
throw new CliError(`skill '${name}' not found`);
|
|
14991
15810
|
}
|
|
14992
|
-
const body = decodeSource(
|
|
15811
|
+
const body = decodeSource(fs28.readFileSync(filePath));
|
|
14993
15812
|
if (opts.compact === true) {
|
|
14994
|
-
|
|
15813
|
+
const emitted = extractCompactFromMarker(body) ?? body;
|
|
15814
|
+
out(emitted);
|
|
15815
|
+
const bytesSaved = Buffer.byteLength(body, "utf8") - Buffer.byteLength(emitted, "utf8");
|
|
15816
|
+
if (bytesSaved > 0) recordStat("skill_body:compact", bytesSaved, savedTokensFromBytes(bytesSaved), void 0, name);
|
|
14995
15817
|
} else {
|
|
14996
15818
|
out(body);
|
|
14997
15819
|
}
|
|
@@ -15003,12 +15825,26 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15003
15825
|
const skills = await listSkills(sessionId);
|
|
15004
15826
|
let regenerated = 0;
|
|
15005
15827
|
let skipped = 0;
|
|
15828
|
+
let noMarker = 0;
|
|
15829
|
+
let unresolvable = 0;
|
|
15006
15830
|
for (const skill of skills) {
|
|
15007
15831
|
const filePath = await getSkillFilePath(skill.name);
|
|
15008
|
-
if (!filePath)
|
|
15009
|
-
|
|
15832
|
+
if (!filePath) {
|
|
15833
|
+
unresolvable++;
|
|
15834
|
+
continue;
|
|
15835
|
+
}
|
|
15836
|
+
let body2;
|
|
15837
|
+
try {
|
|
15838
|
+
body2 = decodeSource(fs28.readFileSync(filePath));
|
|
15839
|
+
} catch {
|
|
15840
|
+
unresolvable++;
|
|
15841
|
+
continue;
|
|
15842
|
+
}
|
|
15010
15843
|
const compact2 = extractCompactFromMarker(body2);
|
|
15011
|
-
if (compact2 === null)
|
|
15844
|
+
if (compact2 === null) {
|
|
15845
|
+
noMarker++;
|
|
15846
|
+
continue;
|
|
15847
|
+
}
|
|
15012
15848
|
const sourceSha2 = contentHash(body2);
|
|
15013
15849
|
if (skill.compactStale === false) {
|
|
15014
15850
|
skipped++;
|
|
@@ -15017,7 +15853,13 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15017
15853
|
regenerated++;
|
|
15018
15854
|
}
|
|
15019
15855
|
}
|
|
15020
|
-
|
|
15856
|
+
const parts = [`Regenerated ${regenerated}, skipped ${skipped} (fresh)`];
|
|
15857
|
+
if (noMarker > 0) parts.push(`no marker ${noMarker}`);
|
|
15858
|
+
if (unresolvable > 0) parts.push(`unresolvable ${unresolvable}`);
|
|
15859
|
+
out(`${parts.join(", ")}, total ${skills.length}.`);
|
|
15860
|
+
if (noMarker > 0) {
|
|
15861
|
+
out(`${noMarker} cached skill${noMarker === 1 ? " has" : "s have"} no COMPACT_END marker and cannot be compacted; run \`token-goat skill-size\` for per-skill marker recommendations.`);
|
|
15862
|
+
}
|
|
15021
15863
|
return;
|
|
15022
15864
|
}
|
|
15023
15865
|
let body;
|
|
@@ -15027,19 +15869,19 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15027
15869
|
if (!opts.path.trim()) {
|
|
15028
15870
|
throw new CliError("--path cannot be empty");
|
|
15029
15871
|
}
|
|
15030
|
-
if (!
|
|
15872
|
+
if (!fs28.existsSync(opts.path)) {
|
|
15031
15873
|
throw new CliError(`skill file not found: ${opts.path}`);
|
|
15032
15874
|
}
|
|
15033
15875
|
try {
|
|
15034
|
-
body =
|
|
15876
|
+
body = fs28.readFileSync(opts.path, "utf-8");
|
|
15035
15877
|
} catch (e) {
|
|
15036
15878
|
if (e.code === "ENOENT") {
|
|
15037
15879
|
throw new CliError(`skill file not found: ${opts.path}`);
|
|
15038
15880
|
}
|
|
15039
15881
|
throw new CliError(`failed to read skill file '${opts.path}': ${extractErrorMessage(e)}`);
|
|
15040
15882
|
}
|
|
15041
|
-
cacheName = name ??
|
|
15042
|
-
sourcePath =
|
|
15883
|
+
cacheName = name ?? path28.basename(path28.dirname(path28.resolve(opts.path)));
|
|
15884
|
+
sourcePath = path28.resolve(opts.path);
|
|
15043
15885
|
} else {
|
|
15044
15886
|
if (name === void 0 || !name.trim()) {
|
|
15045
15887
|
throw new CliError("skill-compact requires a <name> or --path <file>");
|
|
@@ -15048,7 +15890,7 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15048
15890
|
if (filePath === null) {
|
|
15049
15891
|
throw new CliError(`skill '${name}' not found`);
|
|
15050
15892
|
}
|
|
15051
|
-
body =
|
|
15893
|
+
body = fs28.readFileSync(filePath, "utf-8");
|
|
15052
15894
|
cacheName = name;
|
|
15053
15895
|
sourcePath = filePath;
|
|
15054
15896
|
}
|
|
@@ -15180,8 +16022,8 @@ async function cmdSkillDiff(name) {
|
|
|
15180
16022
|
}
|
|
15181
16023
|
const newer = versions[0];
|
|
15182
16024
|
const older = versions[1];
|
|
15183
|
-
const newerBody = await
|
|
15184
|
-
const olderBody = await
|
|
16025
|
+
const newerBody = await fs28.promises.readFile(path28.resolve(dir, `${newer.outputId}.txt`), "utf-8").catch(() => null);
|
|
16026
|
+
const olderBody = await fs28.promises.readFile(path28.resolve(dir, `${older.outputId}.txt`), "utf-8").catch(() => null);
|
|
15185
16027
|
if (newerBody === null || olderBody === null) {
|
|
15186
16028
|
out(`a cached version of '${name}' was evicted while diffing -- try again`);
|
|
15187
16029
|
return;
|
|
@@ -15210,7 +16052,7 @@ async function cmdSkillSection(nameHeading, headingArg) {
|
|
|
15210
16052
|
if (!filePath) {
|
|
15211
16053
|
throw new CliError(`skill '${skillName}' not found`);
|
|
15212
16054
|
}
|
|
15213
|
-
const body = decodeSource(
|
|
16055
|
+
const body = decodeSource(fs28.readFileSync(filePath));
|
|
15214
16056
|
const extracted = extractNamedSection(body, heading);
|
|
15215
16057
|
if (!extracted) {
|
|
15216
16058
|
const messages = [`Section '${heading}' not found in skill '${skillName}'`];
|
|
@@ -15225,7 +16067,7 @@ async function cmdSkillSection(nameHeading, headingArg) {
|
|
|
15225
16067
|
}
|
|
15226
16068
|
function atomicWriteBuffer(dest, data) {
|
|
15227
16069
|
try {
|
|
15228
|
-
if (
|
|
16070
|
+
if (fs28.statSync(dest).isDirectory()) {
|
|
15229
16071
|
const e = Object.assign(new Error(`EISDIR: illegal operation on a directory, open '${dest}'`), { code: "EISDIR", path: dest });
|
|
15230
16072
|
throw e;
|
|
15231
16073
|
}
|
|
@@ -15233,23 +16075,23 @@ function atomicWriteBuffer(dest, data) {
|
|
|
15233
16075
|
if (e.code !== "ENOENT") throw e;
|
|
15234
16076
|
}
|
|
15235
16077
|
const rnd = randomBytes(4).toString("hex");
|
|
15236
|
-
const tmp =
|
|
16078
|
+
const tmp = path28.join(path28.dirname(path28.resolve(dest)), `.tmp.${process.pid}.${rnd}`);
|
|
15237
16079
|
try {
|
|
15238
|
-
|
|
16080
|
+
fs28.writeFileSync(tmp, data, { mode: 384 });
|
|
15239
16081
|
try {
|
|
15240
|
-
const destMode =
|
|
15241
|
-
|
|
16082
|
+
const destMode = fs28.statSync(dest).mode;
|
|
16083
|
+
fs28.chmodSync(tmp, destMode);
|
|
15242
16084
|
} catch (e) {
|
|
15243
16085
|
if (e.code !== "ENOENT") throw e;
|
|
15244
16086
|
}
|
|
15245
16087
|
withRetryOnLock(() => {
|
|
15246
16088
|
try {
|
|
15247
|
-
|
|
16089
|
+
fs28.renameSync(tmp, dest);
|
|
15248
16090
|
} catch (e) {
|
|
15249
16091
|
if (e.code === "EXDEV") {
|
|
15250
|
-
|
|
16092
|
+
fs28.copyFileSync(tmp, dest);
|
|
15251
16093
|
try {
|
|
15252
|
-
|
|
16094
|
+
fs28.unlinkSync(tmp);
|
|
15253
16095
|
} catch (ue) {
|
|
15254
16096
|
process.stderr.write(`token-goat write-file: warning: could not remove temp file ${tmp}: ${ue.message}
|
|
15255
16097
|
`);
|
|
@@ -15261,7 +16103,7 @@ function atomicWriteBuffer(dest, data) {
|
|
|
15261
16103
|
});
|
|
15262
16104
|
} catch (e) {
|
|
15263
16105
|
try {
|
|
15264
|
-
|
|
16106
|
+
fs28.unlinkSync(tmp);
|
|
15265
16107
|
} catch {
|
|
15266
16108
|
}
|
|
15267
16109
|
throw e;
|
|
@@ -15271,9 +16113,9 @@ function mapFsError(e, src, dest, srcLabel = "source") {
|
|
|
15271
16113
|
const fe = e;
|
|
15272
16114
|
if (fe.code === "ENOENT") {
|
|
15273
16115
|
const errPath = fe.path ?? "";
|
|
15274
|
-
const isSource = src !== void 0 &&
|
|
16116
|
+
const isSource = src !== void 0 && path28.resolve(errPath) === path28.resolve(src);
|
|
15275
16117
|
if (isSource) throw new CliError(`${/\bfile$/i.test(srcLabel) ? srcLabel : `${srcLabel} file`} not found: ${src}`);
|
|
15276
|
-
const destDir = dest ?
|
|
16118
|
+
const destDir = dest ? path28.dirname(path28.resolve(dest)) : path28.dirname(path28.resolve(errPath || "."));
|
|
15277
16119
|
throw new CliError(`destination directory does not exist: ${destDir}`);
|
|
15278
16120
|
}
|
|
15279
16121
|
if (fe.code === "ENOTDIR") {
|
|
@@ -15284,7 +16126,7 @@ function mapFsError(e, src, dest, srcLabel = "source") {
|
|
|
15284
16126
|
}
|
|
15285
16127
|
if (fe.code === "EISDIR") {
|
|
15286
16128
|
const errPath = fe.path ?? "";
|
|
15287
|
-
const isSource = src !== void 0 && (errPath === "" ||
|
|
16129
|
+
const isSource = src !== void 0 && (errPath === "" || path28.resolve(errPath) === path28.resolve(src));
|
|
15288
16130
|
if (isSource) throw new CliError(`source is a directory, not a file: ${src}`);
|
|
15289
16131
|
throw new CliError(`destination is a directory, not a file: ${dest ?? (errPath || "(unknown)")}`);
|
|
15290
16132
|
}
|
|
@@ -15353,7 +16195,7 @@ function validateWritablePath(dest, label) {
|
|
|
15353
16195
|
throw new CliError(`${label} path contains a null byte`);
|
|
15354
16196
|
}
|
|
15355
16197
|
if (isWindows()) {
|
|
15356
|
-
const base =
|
|
16198
|
+
const base = path28.basename(dest);
|
|
15357
16199
|
const stem = base.replace(/\.[^.]*$/, "").toUpperCase();
|
|
15358
16200
|
if (WIN_RESERVED.has(stem)) {
|
|
15359
16201
|
throw new CliError(`${label} '${base}' is a reserved Windows device name`);
|
|
@@ -15383,7 +16225,7 @@ function readFileBoundedRaw(filePath, label, allowStdIn = false) {
|
|
|
15383
16225
|
throw new CliError(`${label} ${filePath} requires piped input; use ${altLabel} for interactive use`);
|
|
15384
16226
|
}
|
|
15385
16227
|
try {
|
|
15386
|
-
const st =
|
|
16228
|
+
const st = fs28.statSync(filePath);
|
|
15387
16229
|
if (st.isFIFO() || st.isSocket()) {
|
|
15388
16230
|
throw new CliError(`${label} '${filePath}' is a special file (FIFO or socket) \u2014 only regular files are supported`);
|
|
15389
16231
|
}
|
|
@@ -15391,7 +16233,7 @@ function readFileBoundedRaw(filePath, label, allowStdIn = false) {
|
|
|
15391
16233
|
if (st.size > maxBytes) {
|
|
15392
16234
|
throw new CliError(`${label} '${filePath}' exceeds size limit (${Math.round(st.size / 1024 / 1024)} MB); set TOKEN_GOAT_MAX_STDIN_MB to override`);
|
|
15393
16235
|
}
|
|
15394
|
-
return
|
|
16236
|
+
return fs28.readFileSync(filePath);
|
|
15395
16237
|
} catch (e) {
|
|
15396
16238
|
if (e instanceof CliError) throw e;
|
|
15397
16239
|
mapFsError(e, filePath, void 0, label);
|
|
@@ -15435,7 +16277,7 @@ function cmdNoteAdd(file, opts) {
|
|
|
15435
16277
|
throw new CliError("note content must be valid UTF-8 text");
|
|
15436
16278
|
}
|
|
15437
16279
|
const resolvedPath = resolveIndexPath(file);
|
|
15438
|
-
if (!
|
|
16280
|
+
if (!fs28.existsSync(resolvedPath)) {
|
|
15439
16281
|
throw new CliError(`File not found: '${resolvedPath}'`);
|
|
15440
16282
|
}
|
|
15441
16283
|
healStaleIndex(resolvedPath);
|
|
@@ -15494,7 +16336,7 @@ function cmdWriteFile(dest, opts) {
|
|
|
15494
16336
|
throw new CliError(`TOKEN_GOAT_MAX_STDIN_MB must be a positive integer; got '${process.env["TOKEN_GOAT_MAX_STDIN_MB"] ?? ""}'`);
|
|
15495
16337
|
}
|
|
15496
16338
|
const maxBytes = maxMB * 1024 * 1024;
|
|
15497
|
-
return new Promise((
|
|
16339
|
+
return new Promise((resolve13, reject) => {
|
|
15498
16340
|
const chunks = [];
|
|
15499
16341
|
let totalBytes = 0;
|
|
15500
16342
|
let settled = false;
|
|
@@ -15518,7 +16360,7 @@ function cmdWriteFile(dest, opts) {
|
|
|
15518
16360
|
try {
|
|
15519
16361
|
atomicWriteBuffer(dest, Buffer.concat(chunks));
|
|
15520
16362
|
enqueueDirtyPathSafe(dest);
|
|
15521
|
-
|
|
16363
|
+
resolve13();
|
|
15522
16364
|
} catch (e) {
|
|
15523
16365
|
try {
|
|
15524
16366
|
mapFsError(e, void 0, dest);
|
|
@@ -15649,14 +16491,14 @@ function writeReplacedBuffer(file, replacedBuf, preWriteStat) {
|
|
|
15649
16491
|
const until = process.env["TOKEN_GOAT_TEST_REPLACE_DELAY_UNTIL"];
|
|
15650
16492
|
const deadline = Date.now() + testDelayMs;
|
|
15651
16493
|
if (until !== void 0 && until !== "") {
|
|
15652
|
-
while (Date.now() < deadline && !
|
|
16494
|
+
while (Date.now() < deadline && !fs28.existsSync(until)) sleepSync(5);
|
|
15653
16495
|
} else {
|
|
15654
16496
|
sleepSync(testDelayMs);
|
|
15655
16497
|
}
|
|
15656
16498
|
}
|
|
15657
16499
|
let preRenameStat;
|
|
15658
16500
|
try {
|
|
15659
|
-
preRenameStat =
|
|
16501
|
+
preRenameStat = fs28.statSync(file);
|
|
15660
16502
|
} catch {
|
|
15661
16503
|
}
|
|
15662
16504
|
if (preRenameStat !== void 0 && (preRenameStat.mtimeMs !== preWriteStat.mtimeMs || preRenameStat.size !== preWriteStat.size)) {
|
|
@@ -15697,7 +16539,7 @@ function cmdReplace(file, opts) {
|
|
|
15697
16539
|
const targetBuf = readFileBoundedRaw(file, "target file", true);
|
|
15698
16540
|
let preWriteStat;
|
|
15699
16541
|
try {
|
|
15700
|
-
preWriteStat =
|
|
16542
|
+
preWriteStat = fs28.statSync(file);
|
|
15701
16543
|
} catch {
|
|
15702
16544
|
}
|
|
15703
16545
|
const usingFrom = opts.oldFrom !== void 0 || opts.newFrom !== void 0;
|
|
@@ -15791,7 +16633,7 @@ function cmdInsertSection(file, opts) {
|
|
|
15791
16633
|
}
|
|
15792
16634
|
let preWriteStat;
|
|
15793
16635
|
try {
|
|
15794
|
-
preWriteStat =
|
|
16636
|
+
preWriteStat = fs28.statSync(file);
|
|
15795
16637
|
} catch {
|
|
15796
16638
|
}
|
|
15797
16639
|
const result = readSection(file, opts.after);
|
|
@@ -15817,7 +16659,7 @@ function cmdInsertSection(file, opts) {
|
|
|
15817
16659
|
}
|
|
15818
16660
|
let rawBytes;
|
|
15819
16661
|
try {
|
|
15820
|
-
rawBytes =
|
|
16662
|
+
rawBytes = fs28.readFileSync(file);
|
|
15821
16663
|
} catch (e) {
|
|
15822
16664
|
mapFsError(e, void 0, file);
|
|
15823
16665
|
}
|
|
@@ -15833,7 +16675,7 @@ function cmdInsertSection(file, opts) {
|
|
|
15833
16675
|
if (preWriteStat !== void 0) {
|
|
15834
16676
|
let preRenameStat;
|
|
15835
16677
|
try {
|
|
15836
|
-
preRenameStat =
|
|
16678
|
+
preRenameStat = fs28.statSync(file);
|
|
15837
16679
|
} catch {
|
|
15838
16680
|
}
|
|
15839
16681
|
if (preRenameStat !== void 0 && (preRenameStat.mtimeMs !== preWriteStat.mtimeMs || preRenameStat.size !== preWriteStat.size)) {
|
|
@@ -15868,31 +16710,42 @@ ${content}`;
|
|
|
15868
16710
|
const sections = await getDocSections(fileId, { fresh: false });
|
|
15869
16711
|
emitted = formatSections(sections);
|
|
15870
16712
|
}
|
|
15871
|
-
|
|
15872
|
-
|
|
15873
|
-
|
|
16713
|
+
let toEmit = emitted;
|
|
16714
|
+
try {
|
|
16715
|
+
if (loadConfig().injection.enabled) {
|
|
16716
|
+
const matches2 = scanForInjectionPatterns(emitted);
|
|
16717
|
+
if (matches2.length > 0) {
|
|
16718
|
+
recordStat("injection_detected", 0, 0, void 0, matches2.join(","));
|
|
16719
|
+
toEmit = fenceUntrustedContent(emitted, matches2, UNTRUSTED_WEB_TAG);
|
|
16720
|
+
}
|
|
16721
|
+
}
|
|
16722
|
+
} catch {
|
|
16723
|
+
}
|
|
16724
|
+
out(toEmit);
|
|
16725
|
+
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(toEmit, "utf8"));
|
|
16726
|
+
recordStat("gdrive_sections", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
15874
16727
|
}
|
|
15875
16728
|
function expandGlobs(root, patterns, globFnOverride) {
|
|
15876
16729
|
const out2 = [];
|
|
15877
|
-
const globFn = globFnOverride ??
|
|
16730
|
+
const globFn = globFnOverride ?? fs28["globSync"];
|
|
15878
16731
|
for (const p of patterns) {
|
|
15879
16732
|
if (globFn !== void 0 && (p.includes("*") || p.includes("?") || p.includes("{"))) {
|
|
15880
16733
|
try {
|
|
15881
16734
|
const hits = globFn(p, { cwd: root });
|
|
15882
|
-
for (const h of hits) out2.push(
|
|
16735
|
+
for (const h of hits) out2.push(path28.isAbsolute(h) ? h : path28.join(root, h));
|
|
15883
16736
|
continue;
|
|
15884
16737
|
} catch {
|
|
15885
16738
|
}
|
|
15886
16739
|
}
|
|
15887
|
-
out2.push(
|
|
16740
|
+
out2.push(path28.isAbsolute(p) ? p : path28.join(root, p));
|
|
15888
16741
|
}
|
|
15889
16742
|
return out2;
|
|
15890
16743
|
}
|
|
15891
16744
|
function readIgnoreFile(root) {
|
|
15892
|
-
const ignorePath =
|
|
16745
|
+
const ignorePath = path28.join(root, ".tokengoatignore");
|
|
15893
16746
|
let raw;
|
|
15894
16747
|
try {
|
|
15895
|
-
raw =
|
|
16748
|
+
raw = fs28.readFileSync(ignorePath, "utf8");
|
|
15896
16749
|
} catch {
|
|
15897
16750
|
return void 0;
|
|
15898
16751
|
}
|
|
@@ -15933,14 +16786,14 @@ function cmdPack(patterns, opts) {
|
|
|
15933
16786
|
}
|
|
15934
16787
|
let instruction;
|
|
15935
16788
|
if (opts.instructionFile !== void 0) {
|
|
15936
|
-
instruction =
|
|
16789
|
+
instruction = fs28.readFileSync(opts.instructionFile, "utf8");
|
|
15937
16790
|
}
|
|
15938
16791
|
const formatted = formatPack(result, style, {
|
|
15939
16792
|
...opts.lineNumbers === true ? { line_numbers: true } : {},
|
|
15940
16793
|
...instruction !== void 0 ? { instruction } : {}
|
|
15941
16794
|
});
|
|
15942
16795
|
if (opts.output !== void 0) {
|
|
15943
|
-
|
|
16796
|
+
fs28.writeFileSync(opts.output, formatted, "utf8");
|
|
15944
16797
|
} else {
|
|
15945
16798
|
out(formatted);
|
|
15946
16799
|
}
|
|
@@ -15958,17 +16811,17 @@ function cmdTokens(patterns, opts) {
|
|
|
15958
16811
|
if (opts.tree === true) {
|
|
15959
16812
|
const dirs = /* @__PURE__ */ new Map();
|
|
15960
16813
|
for (const e of entries) {
|
|
15961
|
-
const dir =
|
|
16814
|
+
const dir = path28.dirname(e.rel_path);
|
|
15962
16815
|
if (!dirs.has(dir)) dirs.set(dir, []);
|
|
15963
16816
|
dirs.get(dir).push(e);
|
|
15964
16817
|
}
|
|
15965
16818
|
const lines2 = [];
|
|
15966
16819
|
for (const [dir, dirEntries] of dirs) {
|
|
15967
16820
|
const dirTokens = dirEntries.reduce((s, e) => s + e.tokens, 0);
|
|
15968
|
-
const
|
|
15969
|
-
lines2.push(`${dir}/ (${dirTokens} tokens, ${
|
|
16821
|
+
const pct2 = result.total_tokens > 0 ? Math.round(dirTokens / result.total_tokens * 100) : 0;
|
|
16822
|
+
lines2.push(`${dir}/ (${dirTokens} tokens, ${pct2}%)`);
|
|
15970
16823
|
for (const e of dirEntries) {
|
|
15971
|
-
lines2.push(` ${
|
|
16824
|
+
lines2.push(` ${path28.basename(e.rel_path).padEnd(30)} ${String(e.tokens).padStart(8)} tokens`);
|
|
15972
16825
|
}
|
|
15973
16826
|
}
|
|
15974
16827
|
out(lines2.join("\n"));
|
|
@@ -15999,7 +16852,7 @@ function cmdBudget(patterns, opts) {
|
|
|
15999
16852
|
}
|
|
16000
16853
|
}
|
|
16001
16854
|
function cmdFailures(src, opts) {
|
|
16002
|
-
const text = src !== void 0 ?
|
|
16855
|
+
const text = src !== void 0 ? fs28.readFileSync(src, "utf8") : fs28.readFileSync(0, "utf8");
|
|
16003
16856
|
const result = extractFailures(text, opts.runner !== void 0 ? { runner: opts.runner } : {});
|
|
16004
16857
|
if (opts.delta !== true) {
|
|
16005
16858
|
out(opts.json === true ? formatFailuresJson(result) : formatFailuresText(result));
|
|
@@ -16201,6 +17054,7 @@ function buildProgram() {
|
|
|
16201
17054
|
program2.command("waste").description("session spend-ledger: token cost per tool/file from the current Claude Code session transcript, plus waste signals").option("--project <path>", "project root to analyze").option("--transcript <path>", "explicit transcript JSONL path (default: most-recently-modified transcript for this project)").option("--top <n>", "number of top expensive tool calls to show (default: 10)").option("--copilot", "analyze a Copilot CLI session event log instead, reporting Copilot's own token split").option("--json", "output JSON").action(guard(cmdWaste));
|
|
16202
17055
|
program2.command("session-outline [session-id-or-path]").description("turn-by-turn structure (role, preview, tool calls, approx size) of a Claude Code session JSONL transcript, instead of a raw Read; defaults to the current project's most recent session").option("--project <path>", "project root to resolve the session transcript against").option("--json", "output JSON").action(guard(cmdSessionOutline));
|
|
16203
17056
|
program2.command("session-slice [session-id-or-path]").description("full content of one turn range from a Claude Code session JSONL transcript (see session-outline for turn numbers), instead of a raw Read").requiredOption("--range <spec>", "turn range, e.g. 5-9 or 12 (see session-outline for turn numbers)").option("--project <path>", "project root to resolve the session transcript against").option("--json", "output JSON").action(guard(cmdSessionSlice));
|
|
17057
|
+
program2.command("session-audit").description("corpus-wide token attribution across every local Claude Code session transcript: measured billed usage, estimated content size by source and by tool, and billed cost by session position (aggregate counts only, never transcript content)").option("--dir <path>", "transcript corpus root to scan (default: ~/.claude/projects)").option("--json", "output JSON").action(guard(cmdSessionAudit));
|
|
16204
17058
|
program2.command("mcp-audit").description("MCP server schema cost-vs-usage report: estimate per-server token cost from cached tool calls").option("--project <path>", "project root to analyze").option("--json", "output JSON").action(guard(cmdMcpAudit));
|
|
16205
17059
|
program2.command("recall [query]").description("search across every cached bash-output, web-output, and mcp-output entry (full-text); with no query, list them newest-first").option("--type <type>", "filter to one cache type: bash, web, or mcp").option("--limit <n>", "max results to return (default: 10)").option("--json", "output JSON").action(guard(cmdRecall));
|
|
16206
17060
|
program2.command("hint-stats").description("per-category efficacy report for token-goat's discretionary hint hooks (emitted/acted-on/suppression)").option("--json", "output JSON").option("--reset", "clear all tracked emissions and manual marks").option("--mark-effective <category>", 'record a manual "effective" vote for a hint category').option("--mark-ineffective <category>", 'record a manual "ineffective" vote for a hint category').action(guard(cmdHintStats));
|
|
@@ -16565,7 +17419,7 @@ function buildProgram() {
|
|
|
16565
17419
|
)
|
|
16566
17420
|
);
|
|
16567
17421
|
program2.command("gdrive-sections <file-id>").description("fetch and list sections from a public Google Doc").option("--heading <name>", "get content of one named section").option("--fresh", "skip the on-disk cache and force a live fetch").action(guard(cmdGdriveSections));
|
|
16568
|
-
program2.command("compress").alias("bash").alias("run").description("run a shell command and emit a compressed view of its output").requiredOption("-c, --cmd <command>", "the shell command to run, as one string").option("-f, --filter <name>", "filter name (auto-detected from the command when omitted)").option("--timeout <seconds>", "wall-clock timeout in seconds (0 = built-in default)").option("--no-compress", "stream output raw without compression (debug the wrapper)").option("--profile <name>", "compression profile: aggressive | balanced | minimal").option("--max-tokens <n>", "post-compress token cap (0 = no cap)").action(cmdCompress);
|
|
17422
|
+
program2.command("compress").alias("bash").alias("run").description("run a shell command (under a POSIX shell / bash) and emit a compressed view of its output").requiredOption("-c, --cmd <command>", "the shell command to run, as one string (use / for paths across platforms)").option("-f, --filter <name>", "filter name (auto-detected from the command when omitted)").option("--timeout <seconds>", "wall-clock timeout in seconds (0 = built-in default)").option("--no-compress", "stream output raw without compression (debug the wrapper)").option("--profile <name>", "compression profile: aggressive | balanced | minimal").option("--max-tokens <n>", "post-compress token cap (0 = no cap)").action(cmdCompress);
|
|
16569
17423
|
program2.command("version").description("print the token-goat version").action(
|
|
16570
17424
|
guard(() => {
|
|
16571
17425
|
out(VERSION);
|