token-goat 2.8.3 → 2.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -17
- package/dist/{token-goat-chunk-6ODZ6PZK.mjs → token-goat-chunk-2JZ66BBE.mjs} +216 -72
- package/dist/{token-goat-chunk-PWVXXPCC.mjs → token-goat-chunk-DK4VLLYB.mjs} +563 -155
- package/dist/{token-goat-chunk-EFF2XCLB.mjs → token-goat-chunk-GUNYAGOZ.mjs} +175 -56
- package/dist/{token-goat-chunk-VAGPOZHO.mjs → token-goat-chunk-L2XHDICZ.mjs} +2 -2
- package/dist/{token-goat-chunk-DP3NNGGV.mjs → token-goat-chunk-LJEETTER.mjs} +5 -5
- package/dist/{token-goat-chunk-NKNCHJ4H.mjs → token-goat-chunk-MA5237JN.mjs} +327 -169
- package/dist/{token-goat-chunk-4OM2Q2SX.mjs → token-goat-chunk-SHN4UTL4.mjs} +5 -5
- package/dist/{token-goat-chunk-4HIMCBYK.mjs → token-goat-chunk-TELKICYU.mjs} +55 -6
- package/dist/{token-goat-chunk-222VPFP2.mjs → token-goat-chunk-U7M2LTGP.mjs} +1100 -269
- package/dist/{token-goat-chunk-TX4JFJTD.mjs → token-goat-chunk-XAWIELXH.mjs} +216 -105
- package/dist/{token-goat-chunk-C5IL6MJH.mjs → token-goat-chunk-ZIPBLIUZ.mjs} +6 -5
- package/dist/token-goat-hook.mjs +5 -5
- package/dist/token-goat.core.mjs +5 -5
- package/package.json +3 -1
|
@@ -86,11 +86,12 @@ import {
|
|
|
86
86
|
runZipRead,
|
|
87
87
|
symbolNamesInFile,
|
|
88
88
|
upsertNote
|
|
89
|
-
} from "./token-goat-chunk-
|
|
89
|
+
} from "./token-goat-chunk-XAWIELXH.mjs";
|
|
90
90
|
import {
|
|
91
91
|
BASH_OUTPUT_SUBDIR,
|
|
92
92
|
GEMINI_TOOL_NAME_MAP,
|
|
93
93
|
HOOK_EVENTS,
|
|
94
|
+
RECALL_DEFAULT_LIMIT,
|
|
94
95
|
WEB_OUTPUT_SUBDIR,
|
|
95
96
|
accumulateResidentLine,
|
|
96
97
|
buildCommandManifest,
|
|
@@ -113,13 +114,15 @@ import {
|
|
|
113
114
|
searchRecall,
|
|
114
115
|
storeWebOutput,
|
|
115
116
|
summarizeResidentContext
|
|
116
|
-
} from "./token-goat-chunk-
|
|
117
|
+
} from "./token-goat-chunk-TELKICYU.mjs";
|
|
117
118
|
import {
|
|
118
119
|
AGENT_SALT_MARKER,
|
|
119
120
|
CONTEXT_AUTOCOMPACT_TOKENS,
|
|
120
121
|
DEFAULT_MAX_AGE_MS,
|
|
121
122
|
DEFAULT_MAX_COUNT,
|
|
123
|
+
MATERIALIZE_SHRUNK_IMAGE_JS,
|
|
122
124
|
MAX_FILES_SCANNED,
|
|
125
|
+
PARSER_FINGERPRINT,
|
|
123
126
|
SESSIONS_SUBDIR,
|
|
124
127
|
SKILLS_OUTPUT_SUBDIR,
|
|
125
128
|
SKIP_DIRS,
|
|
@@ -129,6 +132,7 @@ import {
|
|
|
129
132
|
WORKER_HEARTBEAT_STALE_MS,
|
|
130
133
|
WorkerAlreadyRunningError,
|
|
131
134
|
anchoredMarkerPattern,
|
|
135
|
+
applyIndexingPriority,
|
|
132
136
|
auditClaudeMd,
|
|
133
137
|
buildExtractiveCompact,
|
|
134
138
|
buildLineDiff,
|
|
@@ -158,7 +162,9 @@ import {
|
|
|
158
162
|
extractChecklistSection,
|
|
159
163
|
extractCompactFromMarker,
|
|
160
164
|
extractNamedSection,
|
|
165
|
+
fenceUntrusted,
|
|
161
166
|
fenceUntrustedContent,
|
|
167
|
+
fenceUntrustedOcrText,
|
|
162
168
|
findClaudeMdFiles,
|
|
163
169
|
findContentDuplicates,
|
|
164
170
|
findLatestSessionId,
|
|
@@ -174,6 +180,7 @@ import {
|
|
|
174
180
|
formatXlsxRange,
|
|
175
181
|
getContextPressure,
|
|
176
182
|
getDirtyPathsFor,
|
|
183
|
+
getEmbeddingCoverage,
|
|
177
184
|
getFileEntry,
|
|
178
185
|
getHintStatsSummary,
|
|
179
186
|
getHintStatsTotals,
|
|
@@ -187,6 +194,7 @@ import {
|
|
|
187
194
|
indexFileEmbeddings,
|
|
188
195
|
indexFileSync,
|
|
189
196
|
indexedPathSpellingIsStale,
|
|
197
|
+
injectionFencingEnabled,
|
|
190
198
|
installClaudeMd,
|
|
191
199
|
installCodex,
|
|
192
200
|
installCopilotCli,
|
|
@@ -236,7 +244,7 @@ import {
|
|
|
236
244
|
resetHintStats,
|
|
237
245
|
runContextStats,
|
|
238
246
|
runDetachedWorkerDaemon,
|
|
239
|
-
|
|
247
|
+
scanAndRecord,
|
|
240
248
|
shrinkImage,
|
|
241
249
|
skillOutputsDir,
|
|
242
250
|
startDetachedWorker,
|
|
@@ -257,7 +265,7 @@ import {
|
|
|
257
265
|
vscodeDecoderConfigured,
|
|
258
266
|
walkProject,
|
|
259
267
|
writeCompact
|
|
260
|
-
} from "./token-goat-chunk-
|
|
268
|
+
} from "./token-goat-chunk-DK4VLLYB.mjs";
|
|
261
269
|
import {
|
|
262
270
|
C,
|
|
263
271
|
CONFIG_KEY_ENV_OVERRIDES,
|
|
@@ -324,6 +332,7 @@ import {
|
|
|
324
332
|
runGit,
|
|
325
333
|
safeSlice,
|
|
326
334
|
saveConfig,
|
|
335
|
+
savedTokensFromBytes,
|
|
327
336
|
sleepSync,
|
|
328
337
|
stripAnsi,
|
|
329
338
|
stripOwnHooksFromMap,
|
|
@@ -339,7 +348,7 @@ import {
|
|
|
339
348
|
withFileLock,
|
|
340
349
|
withRetryOnLock,
|
|
341
350
|
writeJsonSettings
|
|
342
|
-
} from "./token-goat-chunk-
|
|
351
|
+
} from "./token-goat-chunk-2JZ66BBE.mjs";
|
|
343
352
|
import {
|
|
344
353
|
__export
|
|
345
354
|
} from "./token-goat-chunk-AEX54RUZ.mjs";
|
|
@@ -3458,9 +3467,9 @@ Expecting one of '${allowedValues.join("', '")}'`);
|
|
|
3458
3467
|
* @param {string} [path]
|
|
3459
3468
|
* @return {(string|null|Command)}
|
|
3460
3469
|
*/
|
|
3461
|
-
executableDir(
|
|
3462
|
-
if (
|
|
3463
|
-
this._executableDir =
|
|
3470
|
+
executableDir(path29) {
|
|
3471
|
+
if (path29 === void 0) return this._executableDir;
|
|
3472
|
+
this._executableDir = path29;
|
|
3464
3473
|
return this;
|
|
3465
3474
|
}
|
|
3466
3475
|
/**
|
|
@@ -3759,9 +3768,9 @@ function attemptedCommandName(argv) {
|
|
|
3759
3768
|
}
|
|
3760
3769
|
|
|
3761
3770
|
// src/cli.ts
|
|
3762
|
-
import * as
|
|
3763
|
-
import * as
|
|
3764
|
-
import { homedir as
|
|
3771
|
+
import * as fs28 from "fs";
|
|
3772
|
+
import * as path28 from "path";
|
|
3773
|
+
import { homedir as homedir13 } from "os";
|
|
3765
3774
|
|
|
3766
3775
|
// src/walk_index.ts
|
|
3767
3776
|
import * as fs2 from "node:fs";
|
|
@@ -4095,10 +4104,11 @@ import * as os from "node:os";
|
|
|
4095
4104
|
import * as path from "node:path";
|
|
4096
4105
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
4097
4106
|
|
|
4098
|
-
// pi built-in tool names -> token-goat internal tool names
|
|
4107
|
+
// pi built-in tool names -> token-goat internal tool names. powershell is pi's Windows shell twin of bash -- same input schema (PowerShellToolInput = BashToolInput, i.e. command/timeout) and registered under its own name (badlogic/pi-mono packages/coding-agent/src/core/tools/powershell.ts + tools/index.ts's ToolName union). Unmapped, every shell command in a powershell-tool pi session bypassed the Bash hooks entirely, the same shape as the Copilot shim's bash/powershell pairing.
|
|
4099
4108
|
const TOOL_TO_TG: Record<string, string> = {
|
|
4100
4109
|
read: "Read",
|
|
4101
4110
|
bash: "Bash",
|
|
4111
|
+
powershell: "Bash",
|
|
4102
4112
|
edit: "Edit",
|
|
4103
4113
|
write: "Write",
|
|
4104
4114
|
grep: "Grep",
|
|
@@ -4109,15 +4119,14 @@ const TOOL_TO_TG: Record<string, string> = {
|
|
|
4109
4119
|
const ARGS_TO_TG: Record<string, Record<string, string>> = {
|
|
4110
4120
|
read: { path: "file_path", offset: "offset", limit: "limit" },
|
|
4111
4121
|
bash: { command: "command", timeout: "timeout" },
|
|
4122
|
+
powershell: { command: "command", timeout: "timeout" },
|
|
4112
4123
|
edit: { path: "file_path" },
|
|
4113
4124
|
write: { path: "file_path" },
|
|
4114
4125
|
grep: { pattern: "pattern", path: "path" },
|
|
4115
4126
|
find: { pattern: "pattern", path: "path" },
|
|
4116
4127
|
};
|
|
4117
4128
|
|
|
4118
|
-
// Tools
|
|
4119
|
-
// pre_tool_use handler in token-goat (only Read/Grep/Bash/WebFetch do), so
|
|
4120
|
-
// it's excluded here rather than spawning a hook call that always no-ops.
|
|
4129
|
+
// Tools with a pre-hook whose output shape this extension can act on: deny ({block}), updatedInput (in-place arg rewrite), or the Read image-shrink materialization. Glob's pre handler (preGlobDedupHandler in hooks_glob.ts -- the old claim here that Glob has no pre handler was stale) emits only an advisory contextOutput hint, which pi's tool_call contract has no channel for (see the module docblock), so calling it would cost a hook call per find only to drop the answer; it stays excluded for that reason.
|
|
4121
4130
|
const PRE_HOOK_TOOLS = new Set(["Read", "Grep", "Bash", "WebFetch"]);
|
|
4122
4131
|
|
|
4123
4132
|
// resolveEntryPath reads a sidecar JSON file (token-goat-entry.json, written by
|
|
@@ -4269,6 +4278,30 @@ function extractUpdatedInput(resp: Record<string, unknown>): Record<string, unkn
|
|
|
4269
4278
|
// since pi's tool_call handler has no context-injection channel -- only
|
|
4270
4279
|
// in-place arg mutation. Returns undefined (leaving the read path untouched)
|
|
4271
4280
|
// if the context isn't a shrink payload or anything goes wrong writing it.
|
|
4281
|
+
// Best-effort sweep of previously materialized shrunk copies in the OS temp dir: typed twin of pruneMaterializedShrinks in shrink_block.ts (MATERIALIZE_SHRUNK_IMAGE_JS), which documents why pi keeps its own copy. The temp file only needs to outlive the single tool call whose path was rewritten to it, so anything older than an hour is finished with; the "token-goat-shrink-" prefix check confines the sweep to this mechanism's own files.
|
|
4282
|
+
const MATERIALIZED_SHRINK_MAX_AGE_MS = 60 * 60 * 1000;
|
|
4283
|
+
let lastMaterializedShrinkSweepAtMs = 0;
|
|
4284
|
+
function pruneMaterializedShrinks(): void {
|
|
4285
|
+
const now = Date.now();
|
|
4286
|
+
if (now - lastMaterializedShrinkSweepAtMs < MATERIALIZED_SHRINK_MAX_AGE_MS) return;
|
|
4287
|
+
lastMaterializedShrinkSweepAtMs = now;
|
|
4288
|
+
try {
|
|
4289
|
+
const dir = os.tmpdir();
|
|
4290
|
+
for (const file of fs.readdirSync(dir)) {
|
|
4291
|
+
if (!file.startsWith("token-goat-shrink-")) continue;
|
|
4292
|
+
const full = path.join(dir, file);
|
|
4293
|
+
try {
|
|
4294
|
+
const st = fs.statSync(full);
|
|
4295
|
+
if (st.isFile() && now - st.mtimeMs > MATERIALIZED_SHRINK_MAX_AGE_MS) fs.unlinkSync(full);
|
|
4296
|
+
} catch {
|
|
4297
|
+
// Best-effort per-file cleanup; one bad stat/unlink must not abort the sweep.
|
|
4298
|
+
}
|
|
4299
|
+
}
|
|
4300
|
+
} catch {
|
|
4301
|
+
// Best-effort; a readdir failure must never break the materialization below.
|
|
4302
|
+
}
|
|
4303
|
+
}
|
|
4304
|
+
|
|
4272
4305
|
function materializeShrunkImage(context: string | undefined): string | undefined {
|
|
4273
4306
|
if (typeof context !== "string") return undefined;
|
|
4274
4307
|
const idx = context.indexOf("data:image/");
|
|
@@ -4276,6 +4309,7 @@ function materializeShrunkImage(context: string | undefined): string | undefined
|
|
|
4276
4309
|
const match = /^data:image\\/([a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=]+)$/.exec(context.slice(idx).trim());
|
|
4277
4310
|
if (!match) return undefined;
|
|
4278
4311
|
try {
|
|
4312
|
+
pruneMaterializedShrinks();
|
|
4279
4313
|
const buf = Buffer.from(match[2], "base64");
|
|
4280
4314
|
const name = \`token-goat-shrink-\${process.pid}-\${Date.now()}-\${Math.random().toString(36).slice(2)}.\${match[1]}\`;
|
|
4281
4315
|
const file = path.join(os.tmpdir(), name);
|
|
@@ -4361,13 +4395,35 @@ export default function (pi: ExtensionAPI) {
|
|
|
4361
4395
|
.filter((c: { type?: string }) => c && c.type === "text")
|
|
4362
4396
|
.map((c: { text?: string }) => c.text ?? "")
|
|
4363
4397
|
.join("\\n");
|
|
4364
|
-
await callHook("post_tool_use", {
|
|
4398
|
+
const resp = await callHook("post_tool_use", {
|
|
4365
4399
|
session_id: sessionId,
|
|
4366
4400
|
tool_name: tg,
|
|
4367
4401
|
tool_input: toToolInput(event.toolName, (event.input ?? {}) as Record<string, unknown>),
|
|
4368
4402
|
cwd,
|
|
4369
4403
|
tool_response: { output },
|
|
4370
4404
|
});
|
|
4405
|
+
if (!resp) return;
|
|
4406
|
+
|
|
4407
|
+
// rewriteOutput: replace the tool result the model sees. pi's tool_result handler CAN modify the result: its own types declare "Fired after a tool executes. Can modify result." over ToolResultEvent and a ToolResultEventResult of { content, details, isError, usage }, and the shipped runtime applies it (dist/core/extensions/runner.js emitToolResult assigns handlerResult.content onto the event, and dist/core/agent-session.js afterToolCall returns hookResult.content as the tool result content). Without this, injection fencing, secret redaction and output compression were computed and thrown away on every pi session, the same whitelist-shaped drop fixed for opencode.
|
|
4408
|
+
const hso = resp["hookSpecificOutput"] as Record<string, unknown> | undefined;
|
|
4409
|
+
const updatedToolOutput = hso && typeof hso["updatedToolOutput"] === "string" ? (hso["updatedToolOutput"] as string) : undefined;
|
|
4410
|
+
if (updatedToolOutput === undefined) return;
|
|
4411
|
+
|
|
4412
|
+
// Replace only the text blocks, in place, and keep every non-text block (images) untouched: emitToolResult overwrites content wholesale, so anything not carried here is dropped from what the model sees. details/isError/usage are deliberately not returned, since runner.js only overwrites the fields a handler actually provides.
|
|
4413
|
+
const rewritten: typeof contentBlocks = [];
|
|
4414
|
+
let replaced = false;
|
|
4415
|
+
for (const block of contentBlocks) {
|
|
4416
|
+
const isText = Boolean(block) && (block as { type?: string }).type === "text";
|
|
4417
|
+
if (!isText) {
|
|
4418
|
+
rewritten.push(block);
|
|
4419
|
+
continue;
|
|
4420
|
+
}
|
|
4421
|
+
if (replaced) continue;
|
|
4422
|
+
replaced = true;
|
|
4423
|
+
rewritten.push({ type: "text", text: updatedToolOutput });
|
|
4424
|
+
}
|
|
4425
|
+
if (!replaced) rewritten.push({ type: "text", text: updatedToolOutput });
|
|
4426
|
+
return { content: rewritten };
|
|
4371
4427
|
});
|
|
4372
4428
|
|
|
4373
4429
|
// Compaction: pi's session_before_compact REPLACES the summary rather than
|
|
@@ -4554,7 +4610,7 @@ import os from "node:os"
|
|
|
4554
4610
|
import path from "node:path"
|
|
4555
4611
|
import { fileURLToPath, pathToFileURL } from "node:url"
|
|
4556
4612
|
|
|
4557
|
-
// opencode built-in tool id -> token-goat canonical tool name.
|
|
4613
|
+
// opencode built-in tool id -> token-goat canonical tool name. websearch/skill/task were re-verified against opencode's own source at the tag matching the installed release (anomalyco/opencode v1.18.16 packages/opencode/src/tool/): WebSearchTool registers as "websearch" (websearch.ts), SkillTool as "skill" (skill.ts), TaskTool as "task" (task.ts, params prompt/subagent_type/description -- the exact keys token-goat's hooks_agent_spawn.ts reads, which registers a lowercase 'task' handler). Unmapped, all three were dead mechanisms here: no WebSearch repeat-search deny/compression, no repeat-skill-load deny, no agent-spawn briefing or report compaction. apply_patch (patchText only, no per-file path) and lsp/plan/question/todo have no token-goat equivalent and stay unmapped.
|
|
4558
4614
|
const TOOL_TO_TG = {
|
|
4559
4615
|
read: "Read",
|
|
4560
4616
|
bash: "Bash",
|
|
@@ -4563,15 +4619,15 @@ const TOOL_TO_TG = {
|
|
|
4563
4619
|
grep: "Grep",
|
|
4564
4620
|
glob: "Glob",
|
|
4565
4621
|
webfetch: "WebFetch",
|
|
4622
|
+
websearch: "WebSearch",
|
|
4623
|
+
skill: "Skill",
|
|
4624
|
+
task: "task",
|
|
4566
4625
|
}
|
|
4567
4626
|
|
|
4568
|
-
// Tools with a
|
|
4569
|
-
|
|
4570
|
-
// in token-goat at all, so skip the subprocess call for them entirely. Glob
|
|
4571
|
-
// has none either, so it's excluded too rather than spawning a no-op call.
|
|
4572
|
-
const PRE_HOOK_TOOLS = new Set(["read", "bash", "grep", "webfetch"])
|
|
4627
|
+
// Tools with a pre_tool_use handler server-side whose OUTPUT SHAPE this hook can act on: deny (throw), updatedInput (args rewrite), or the Read image-shrink materialization. websearch (repeat-search deny within the dedup TTL, hooks_websearch.ts), skill (repeat-load deny, hooks_skill.ts) and task (briefing updatedInput rewrite, hooks_agent_spawn.ts) all qualify. Edit/Write have no pre-hook at all. Glob DOES have one (preGlobDedupHandler, hooks_glob.ts) -- the old claim here that it has none was stale -- but its only output is an advisory contextOutput hint, and tool.execute.before has no context channel (see the module docblock), so calling it would spawn a subprocess per glob only to drop the answer; it stays excluded for that reason, not the old one.
|
|
4628
|
+
const PRE_HOOK_TOOLS = new Set(["read", "bash", "grep", "webfetch", "websearch", "skill", "task"])
|
|
4573
4629
|
|
|
4574
|
-
// opencode tool args (camelCase) -> token-goat snake_case tool_input keys.
|
|
4630
|
+
// opencode tool args (camelCase) -> token-goat snake_case tool_input keys. websearch's query (websearch.ts Parameters), skill's name (skill.ts Parameters; hooks_skill.ts reads tool_input['skill']) and task's prompt/subagent_type/description (task.ts BaseParameterFields) are all from opencode's own schemas at v1.18.16.
|
|
4575
4631
|
const ARGS_TO_TG = {
|
|
4576
4632
|
read: { filePath: "file_path", offset: "offset", limit: "limit" },
|
|
4577
4633
|
bash: { command: "command", timeout: "timeout" },
|
|
@@ -4580,6 +4636,9 @@ const ARGS_TO_TG = {
|
|
|
4580
4636
|
grep: { pattern: "pattern", path: "path" },
|
|
4581
4637
|
glob: { pattern: "pattern", path: "path" },
|
|
4582
4638
|
webfetch: { url: "url" },
|
|
4639
|
+
websearch: { query: "query" },
|
|
4640
|
+
skill: { name: "skill" },
|
|
4641
|
+
task: { prompt: "prompt", subagent_type: "subagent_type", description: "description" },
|
|
4583
4642
|
}
|
|
4584
4643
|
|
|
4585
4644
|
function reverseArgMap(tool) {
|
|
@@ -4615,28 +4674,7 @@ function extractUpdatedInput(resp) {
|
|
|
4615
4674
|
return hso && typeof hso === "object" ? hso.updatedInput : undefined
|
|
4616
4675
|
}
|
|
4617
4676
|
|
|
4618
|
-
|
|
4619
|
-
// ("<summary>\\ndata:image/<fmt>;base64,<data>") into a real file on disk,
|
|
4620
|
-
// since tool.execute.before has no context-injection channel to hand the
|
|
4621
|
-
// shrunk image to the model directly -- only output.args mutation. Returns
|
|
4622
|
-
// undefined (leaving output.args untouched) if the context isn't a shrink
|
|
4623
|
-
// payload or anything goes wrong writing it.
|
|
4624
|
-
function materializeShrunkImage(context) {
|
|
4625
|
-
if (typeof context !== "string") return undefined
|
|
4626
|
-
const idx = context.indexOf("data:image/")
|
|
4627
|
-
if (idx === -1) return undefined
|
|
4628
|
-
const match = /^data:image\\/([a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=]+)$/.exec(context.slice(idx).trim())
|
|
4629
|
-
if (!match) return undefined
|
|
4630
|
-
try {
|
|
4631
|
-
const buf = Buffer.from(match[2], "base64")
|
|
4632
|
-
const name = \`token-goat-shrink-\${process.pid}-\${Date.now()}-\${Math.random().toString(36).slice(2)}.\${match[1]}\`
|
|
4633
|
-
const file = path.join(os.tmpdir(), name)
|
|
4634
|
-
fs.writeFileSync(file, buf)
|
|
4635
|
-
return file
|
|
4636
|
-
} catch {
|
|
4637
|
-
return undefined
|
|
4638
|
-
}
|
|
4639
|
-
}
|
|
4677
|
+
${MATERIALIZE_SHRUNK_IMAGE_JS}
|
|
4640
4678
|
|
|
4641
4679
|
export const TokenGoatPlugin = async ({ directory }) => {
|
|
4642
4680
|
return {
|
|
@@ -4686,6 +4724,14 @@ export const TokenGoatPlugin = async ({ directory }) => {
|
|
|
4686
4724
|
})
|
|
4687
4725
|
if (!resp) return
|
|
4688
4726
|
|
|
4727
|
+
// rewriteOutput: replace the tool result wholesale. This is the shape every post_tool_use rewrite producer emits (WebFetch injection fencing and secret redaction, websearch redaction, bash/grep output compression, agent-report compaction), and it went unapplied here for the same whitelist reason the response-contract docblock above used to omit it -- the plugin only handled the shapes its author listed, so opencode sessions got the appended hint channel but never a fenced, redacted or compressed result. Mutating output.output is the exact mechanism the context append below already relies on, so this claims no capability the append did not.
|
|
4728
|
+
const hso = resp.hookSpecificOutput
|
|
4729
|
+
const updatedToolOutput = hso && typeof hso.updatedToolOutput === "string" ? hso.updatedToolOutput : undefined
|
|
4730
|
+
if (updatedToolOutput !== undefined) {
|
|
4731
|
+
output.output = updatedToolOutput
|
|
4732
|
+
return
|
|
4733
|
+
}
|
|
4734
|
+
|
|
4689
4735
|
const context = extractContext(resp)
|
|
4690
4736
|
if (context && typeof output.output === "string") {
|
|
4691
4737
|
output.output += \`\\n\\n[token-goat] \${context}\`
|
|
@@ -4740,15 +4786,11 @@ var OPENCLAW_PLUGIN_SCRIPT = `// token-goat bridge plugin for OpenClaw
|
|
|
4740
4786
|
// Bridges OpenClaw's tool-call and session hooks to token-goat's subprocess
|
|
4741
4787
|
// hook protocol. https://github.com/DFKHelper/token-goat
|
|
4742
4788
|
//
|
|
4743
|
-
// NOTE ON PARAMETER SHAPES: OpenClaw's tool-
|
|
4744
|
-
// token-goat's tool_input unchanged (no key remapping) -- verified against
|
|
4745
|
-
// OpenClaw's own event-type source that these keys are already snake_case
|
|
4746
|
-
// (file_path, command, etc.), matching token-goat's own convention. The
|
|
4747
|
-
// built-in tool NAME list below is broader than any single source confirms,
|
|
4748
|
-
// since an unmatched name is a harmless no-op here, not a wrong rewrite.
|
|
4789
|
+
// NOTE ON PARAMETER SHAPES: OpenClaw's bash tool really does use token-goat's own key names (command/timeout), but the path-carrying tools (read/edit/write) send the file path under "path", not "file_path" -- read out of OpenClaw's own tool input schemas (src/agents/sessions/tools/read-tool-contract.ts, edit.ts, bash.ts in openclaw/openclaw). toToolInput below ADDS the canonical file_path alongside the original path so token-goat's getFilePath()-based handlers can see it, without renaming anything. The built-in tool NAME list below is broader than any single source confirms, since an unmatched name is a harmless no-op here, not a wrong rewrite.
|
|
4749
4790
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
4750
4791
|
import { spawnSync } from "node:child_process";
|
|
4751
4792
|
import * as fs from "node:fs";
|
|
4793
|
+
import * as os from "node:os";
|
|
4752
4794
|
import * as path from "node:path";
|
|
4753
4795
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
4754
4796
|
|
|
@@ -4759,20 +4801,43 @@ const TOOL_TO_TG = {
|
|
|
4759
4801
|
edit: "Edit",
|
|
4760
4802
|
apply_patch: "Edit",
|
|
4761
4803
|
exec: "Bash",
|
|
4804
|
+
bash: "Bash",
|
|
4762
4805
|
grep: "Grep",
|
|
4763
4806
|
glob: "Glob",
|
|
4807
|
+
find: "Glob",
|
|
4764
4808
|
webfetch: "WebFetch",
|
|
4765
4809
|
web_search: "WebFetch",
|
|
4766
4810
|
web_fetch: "WebFetch",
|
|
4767
4811
|
};
|
|
4768
4812
|
|
|
4769
|
-
// Tools with a
|
|
4770
|
-
// none; Glob has none either -- see pi.ts/opencode.ts's identical
|
|
4771
|
-
// PRE_HOOK_TOOLS note -- so it's excluded to avoid a wasted no-op spawn).
|
|
4813
|
+
// Tools with a pre_tool_use handler whose output shape this hook can act on: deny (block/blockReason), updatedInput (params rewrite), or the Read image-shrink materialization. Edit/Write have no pre handler at all. Glob (OpenClaw's find) DOES have one -- preGlobDedupHandler in hooks_glob.ts; the old claim here that it has none was stale -- but its only output is an advisory contextOutput hint and before_tool_call has no context channel, so calling it would spawn a hook subprocess per find only to drop the answer; it stays excluded for that reason (matching pi.ts/opencode.ts's corrected note).
|
|
4772
4814
|
const PRE_HOOK_TOOLS = new Set(["Read", "Grep", "Bash", "WebFetch"]);
|
|
4773
4815
|
|
|
4816
|
+
// OpenClaw tools whose file path travels under "path" (see the parameter-shapes note above). apply_patch is deliberately absent: its input is a patch document, not a single path.
|
|
4817
|
+
const PATH_ARG_TOOLS = new Set(["read", "write", "edit"]);
|
|
4818
|
+
|
|
4819
|
+
// Build token-goat's tool_input from OpenClaw's params: same object, plus the canonical file_path added alongside the original path for the tools that carry one. Never renames or drops a key.
|
|
4820
|
+
function toToolInput(toolName, params) {
|
|
4821
|
+
const p = params || {};
|
|
4822
|
+
if (PATH_ARG_TOOLS.has(toolName) && typeof p.path === "string" && p.file_path === undefined) {
|
|
4823
|
+
return { ...p, file_path: p.path };
|
|
4824
|
+
}
|
|
4825
|
+
return p;
|
|
4826
|
+
}
|
|
4827
|
+
|
|
4828
|
+
// Pull the additionalContext / systemMessage string out of a hook response, whichever shape it came back as (see the response-contract note in this module's header comment).
|
|
4829
|
+
function extractContext(resp) {
|
|
4830
|
+
if (!resp) return undefined;
|
|
4831
|
+
const hso = resp["hookSpecificOutput"];
|
|
4832
|
+
if (hso && typeof hso["additionalContext"] === "string") return hso["additionalContext"];
|
|
4833
|
+
if (typeof resp["systemMessage"] === "string") return resp["systemMessage"];
|
|
4834
|
+
return undefined;
|
|
4835
|
+
}
|
|
4836
|
+
|
|
4774
4837
|
${BRIDGE_RELAY_JS}
|
|
4775
4838
|
|
|
4839
|
+
${MATERIALIZE_SHRUNK_IMAGE_JS}
|
|
4840
|
+
|
|
4776
4841
|
export default definePluginEntry({
|
|
4777
4842
|
id: "token-goat",
|
|
4778
4843
|
name: "token-goat",
|
|
@@ -4808,7 +4873,7 @@ export default definePluginEntry({
|
|
|
4808
4873
|
const resp = await callHook("pre_tool_use", {
|
|
4809
4874
|
session_id: sid,
|
|
4810
4875
|
tool_name: tg,
|
|
4811
|
-
tool_input: event.params
|
|
4876
|
+
tool_input: toToolInput(event.toolName, event.params),
|
|
4812
4877
|
cwd: process.cwd(),
|
|
4813
4878
|
});
|
|
4814
4879
|
if (!resp) return {};
|
|
@@ -4827,6 +4892,12 @@ export default definePluginEntry({
|
|
|
4827
4892
|
return { params: { ...(event.params || {}), ...updated } };
|
|
4828
4893
|
}
|
|
4829
4894
|
|
|
4895
|
+
// Image shrink has no context channel here -- translate it into a rewritten path pointing at a materialized shrunk copy instead, via the same params-rewrite channel the updatedInput merge above already relies on. OpenClaw's read tool takes the path under "path" (see the parameter-shapes note above), and the rest of the original params are preserved.
|
|
4896
|
+
if (tg === "Read") {
|
|
4897
|
+
const shrunkPath = materializeShrunkImage(extractContext(resp));
|
|
4898
|
+
if (shrunkPath) return { params: { ...(event.params || {}), path: shrunkPath } };
|
|
4899
|
+
}
|
|
4900
|
+
|
|
4830
4901
|
return {};
|
|
4831
4902
|
});
|
|
4832
4903
|
|
|
@@ -4846,7 +4917,7 @@ export default definePluginEntry({
|
|
|
4846
4917
|
await callHook("post_tool_use", {
|
|
4847
4918
|
session_id: sid,
|
|
4848
4919
|
tool_name: tg,
|
|
4849
|
-
tool_input: event.params
|
|
4920
|
+
tool_input: toToolInput(event.toolName, event.params),
|
|
4850
4921
|
cwd: process.cwd(),
|
|
4851
4922
|
tool_response: toolResponse,
|
|
4852
4923
|
});
|
|
@@ -5695,7 +5766,7 @@ function renderTopSessionFilesFromDisk(topN = 5, overrideSessionsDir) {
|
|
|
5695
5766
|
var METHODOLOGY = {
|
|
5696
5767
|
estimate_scope: "Local estimate of content avoided or reduced by token-goat.",
|
|
5697
5768
|
billing: "They are not GitHub Copilot usage, provider-reported token consumption, or billing data.",
|
|
5698
|
-
byte_derived_formula: "Most read, hook, and command entries
|
|
5769
|
+
byte_derived_formula: "Most read, hook, and command entries go through savedTokensFromBytes in src/stats.ts, which is Math.round(bytes_saved / 4).",
|
|
5699
5770
|
filter_estimates: "Output compressors record their filter-calculated delta; image entries use the byte-derived approximation unless the source provides a narrower estimate.",
|
|
5700
5771
|
advisory_events: "Zero-byte, zero-token advisory events show that guidance fired, not that an agent followed it.",
|
|
5701
5772
|
audit: "Use stats --full or stats --json for source and command breakdowns; reconcile billing with provider-exported usage data."
|
|
@@ -5718,6 +5789,21 @@ function renderMethodology(json = false) {
|
|
|
5718
5789
|
""
|
|
5719
5790
|
].join("\n"));
|
|
5720
5791
|
}
|
|
5792
|
+
function statsJsonPayload(summary) {
|
|
5793
|
+
return {
|
|
5794
|
+
total_events: summary.total_events,
|
|
5795
|
+
total_bytes_saved: summary.total_bytes_saved,
|
|
5796
|
+
total_tokens_saved: summary.total_tokens_saved,
|
|
5797
|
+
by_kind: summary.by_kind,
|
|
5798
|
+
by_day: summary.by_day,
|
|
5799
|
+
by_project: summary.by_project,
|
|
5800
|
+
by_command: summary.by_command,
|
|
5801
|
+
by_source: summary.by_source,
|
|
5802
|
+
by_harness: summary.by_harness,
|
|
5803
|
+
counts: summary.counts,
|
|
5804
|
+
window_days: summary.window_days
|
|
5805
|
+
};
|
|
5806
|
+
}
|
|
5721
5807
|
function runStats(opts = {}) {
|
|
5722
5808
|
if (opts.methodology === true) {
|
|
5723
5809
|
renderMethodology(opts.json === true);
|
|
@@ -5726,19 +5812,7 @@ function runStats(opts = {}) {
|
|
|
5726
5812
|
const window = opts.windowDays ?? 30;
|
|
5727
5813
|
const summary = summarize(window, void 0, opts.homeDir);
|
|
5728
5814
|
if (opts.json === true) {
|
|
5729
|
-
|
|
5730
|
-
total_events: summary.total_events,
|
|
5731
|
-
total_bytes_saved: summary.total_bytes_saved,
|
|
5732
|
-
total_tokens_saved: summary.total_tokens_saved,
|
|
5733
|
-
by_kind: summary.by_kind,
|
|
5734
|
-
by_day: summary.by_day,
|
|
5735
|
-
by_project: summary.by_project,
|
|
5736
|
-
by_command: summary.by_command,
|
|
5737
|
-
by_source: summary.by_source,
|
|
5738
|
-
by_harness: summary.by_harness,
|
|
5739
|
-
window_days: summary.window_days
|
|
5740
|
-
};
|
|
5741
|
-
process.stdout.write(JSON.stringify(out2) + "\n");
|
|
5815
|
+
process.stdout.write(JSON.stringify(statsJsonPayload(summary)) + "\n");
|
|
5742
5816
|
return;
|
|
5743
5817
|
}
|
|
5744
5818
|
const renderOpts = { windowDays: window };
|
|
@@ -5958,6 +6032,42 @@ function checkSymbolCount(dbPath, rootDir) {
|
|
|
5958
6032
|
};
|
|
5959
6033
|
}
|
|
5960
6034
|
}
|
|
6035
|
+
var EMBED_COVERAGE_WARN_FRACTION = 0.25;
|
|
6036
|
+
function checkEmbeddingCoverage(dbPath, rootDir) {
|
|
6037
|
+
if (!fs12.existsSync(dbPath)) {
|
|
6038
|
+
return { name: "Embedding coverage", status: "ok", message: "no database yet" };
|
|
6039
|
+
}
|
|
6040
|
+
const cfg = loadConfig();
|
|
6041
|
+
if (!cfg.indexing.embeddings_enabled) {
|
|
6042
|
+
return { name: "Embedding coverage", status: "ok", message: "disabled (indexing.embeddings_enabled = false)" };
|
|
6043
|
+
}
|
|
6044
|
+
try {
|
|
6045
|
+
const { indexedFiles, embeddedFiles } = getEmbeddingCoverage(dbPath, rootDir);
|
|
6046
|
+
if (indexedFiles === 0) {
|
|
6047
|
+
return { name: "Embedding coverage", status: "ok", message: "no indexed files yet" };
|
|
6048
|
+
}
|
|
6049
|
+
const pct2 = Math.round(embeddedFiles / indexedFiles * 100);
|
|
6050
|
+
const sizeKb = cfg.indexing.large_file_symbol_only_kb;
|
|
6051
|
+
if (embeddedFiles / indexedFiles < EMBED_COVERAGE_WARN_FRACTION) {
|
|
6052
|
+
return {
|
|
6053
|
+
name: "Embedding coverage",
|
|
6054
|
+
status: "warn",
|
|
6055
|
+
message: `only ${embeddedFiles} of ${indexedFiles} indexed file(s) (${pct2}%) have embeddings \u2014 'semantic' searches those files only, and reports finding nothing in the same words it uses after searching everything. Files over indexing.large_file_symbol_only_kb (currently ${sizeKb} KB) are indexed for symbols only and are the usual reason; raise it with 'token-goat config set indexing.large_file_symbol_only_kb <KB>' and re-embed with 'token-goat index --force' to widen coverage. Exact symbol lookups are unaffected`
|
|
6056
|
+
};
|
|
6057
|
+
}
|
|
6058
|
+
return {
|
|
6059
|
+
name: "Embedding coverage",
|
|
6060
|
+
status: "ok",
|
|
6061
|
+
message: `${embeddedFiles} of ${indexedFiles} indexed file(s) (${pct2}%) have embeddings`
|
|
6062
|
+
};
|
|
6063
|
+
} catch (err2) {
|
|
6064
|
+
return {
|
|
6065
|
+
name: "Embedding coverage",
|
|
6066
|
+
status: "warn",
|
|
6067
|
+
message: `could not query embedding coverage: ${extractErrorMessage(err2)}`
|
|
6068
|
+
};
|
|
6069
|
+
}
|
|
6070
|
+
}
|
|
5961
6071
|
var DIRTY_QUEUE_BACKLOG_WARN_THRESHOLD = 500;
|
|
5962
6072
|
function checkDirtyQueueHealth(dataDir2) {
|
|
5963
6073
|
let pendingCount = 0;
|
|
@@ -6287,7 +6397,9 @@ function checkUnmappedTools(dbPath) {
|
|
|
6287
6397
|
if (rows.length === 0) {
|
|
6288
6398
|
return { name, status: "ok", message: "every tool name seen so far reached a handler that wanted it" };
|
|
6289
6399
|
}
|
|
6290
|
-
const nearMisses = rows.filter(
|
|
6400
|
+
const nearMisses = rows.filter(
|
|
6401
|
+
(r) => r.near_miss !== null && r.near_miss !== void 0 && r.near_miss !== r.tool_name
|
|
6402
|
+
);
|
|
6291
6403
|
if (nearMisses.length > 0) {
|
|
6292
6404
|
const shown2 = nearMisses.slice(0, UNMAPPED_TOOL_SAMPLE).map((r) => `${r.harness} sent "${r.tool_name}" where "${r.near_miss}" is handled (${r.event_name}, ${r.hits}x)`);
|
|
6293
6405
|
const more2 = nearMisses.length > UNMAPPED_TOOL_SAMPLE ? ` (+${nearMisses.length - UNMAPPED_TOOL_SAMPLE} more)` : "";
|
|
@@ -6324,6 +6436,7 @@ function runDoctor(dataDir2, configPath2, rootDir, processes) {
|
|
|
6324
6436
|
const actualConfigPath = configPath2 || configPath();
|
|
6325
6437
|
results.push(checkConfigValid(actualConfigPath));
|
|
6326
6438
|
results.push(checkEmbeddings(loadConfig(rootDir)));
|
|
6439
|
+
results.push(checkEmbeddingCoverage(path11.join(actualDataDir, "global.db"), rootDir));
|
|
6327
6440
|
for (const result of checkSecurityPosture(loadConfig(rootDir), actualDataDir)) results.push(result);
|
|
6328
6441
|
results.push(checkDiskSpace(actualDataDir));
|
|
6329
6442
|
const copilotResult = checkCopilotCli(copilotCliConfigPath(), copilotCliScriptPath());
|
|
@@ -9189,12 +9302,12 @@ function formatBudgetText(result, contextK) {
|
|
|
9189
9302
|
lines.push(` ${e.rel_path.padEnd(colW, " ")} ${String(e.lines).padStart(6, " ")} ${String(e.tokens).padStart(8, " ")}`);
|
|
9190
9303
|
}
|
|
9191
9304
|
lines.push(` ${"-".repeat(colW)} ${"-".repeat(6)} ${"-".repeat(8)}`);
|
|
9192
|
-
let
|
|
9305
|
+
let pct2 = "";
|
|
9193
9306
|
if (contextK) {
|
|
9194
|
-
|
|
9307
|
+
pct2 = ` (${Math.round(result.total_tokens / (contextK * 1e3) * 100)}% of ${contextK}K)`;
|
|
9195
9308
|
}
|
|
9196
9309
|
lines.push(
|
|
9197
|
-
` ${"Total".padEnd(colW, " ")} ${String(result.total_lines).padStart(6, " ")} ${String(result.total_tokens).padStart(8, " ")}${
|
|
9310
|
+
` ${"Total".padEnd(colW, " ")} ${String(result.total_lines).padStart(6, " ")} ${String(result.total_tokens).padStart(8, " ")}${pct2}`
|
|
9198
9311
|
);
|
|
9199
9312
|
if (result.skipped.length > 0) {
|
|
9200
9313
|
const skipped = result.skipped.slice(0, 5).join(", ");
|
|
@@ -10295,14 +10408,23 @@ function applyFiltersAndFold(lines, noNormalize, foldRepeats) {
|
|
|
10295
10408
|
}
|
|
10296
10409
|
function cmdLogfold(src, opts) {
|
|
10297
10410
|
const text = readInput(src);
|
|
10298
|
-
|
|
10411
|
+
const rawLines = splitLines(text);
|
|
10412
|
+
const allLines = rawLines.length > 1 && rawLines[rawLines.length - 1] === "" ? rawLines.slice(0, -1) : rawLines;
|
|
10413
|
+
let lines = allLines;
|
|
10299
10414
|
if (opts.tail !== void 0) {
|
|
10300
10415
|
const n = requireNonNegativeStrictInt("--tail", opts.tail);
|
|
10301
10416
|
lines = lines.slice(Math.max(0, lines.length - n));
|
|
10302
10417
|
}
|
|
10418
|
+
const inputLines = allLines.length;
|
|
10419
|
+
const shownLines = lines.length;
|
|
10420
|
+
const truncated = shownLines < inputLines;
|
|
10303
10421
|
const folded = applyFiltersAndFold(lines, opts.noNormalize === true, opts.foldRepeats === true);
|
|
10422
|
+
if (truncated) {
|
|
10423
|
+
process.stderr.write(`Showing last ${shownLines} of ${inputLines} lines (raise --tail to see more).
|
|
10424
|
+
`);
|
|
10425
|
+
}
|
|
10304
10426
|
if (opts.json === true) {
|
|
10305
|
-
process.stdout.write(JSON.stringify({ lines: folded }, null, 2) + "\n");
|
|
10427
|
+
process.stdout.write(JSON.stringify({ lines: folded, truncated, inputLines, shownLines }, null, 2) + "\n");
|
|
10306
10428
|
return;
|
|
10307
10429
|
}
|
|
10308
10430
|
for (const item of folded) {
|
|
@@ -10788,9 +10910,11 @@ function cmdHot(opts) {
|
|
|
10788
10910
|
}
|
|
10789
10911
|
}
|
|
10790
10912
|
entries.sort((a, b) => b.readCount - a.readCount);
|
|
10913
|
+
const eligibleCount = entries.length;
|
|
10791
10914
|
entries = entries.slice(0, limit);
|
|
10915
|
+
const truncated = entries.length < eligibleCount;
|
|
10792
10916
|
if (opts.json === true) {
|
|
10793
|
-
process.stdout.write(JSON.stringify({ entries }, null, 2) + "\n");
|
|
10917
|
+
process.stdout.write(JSON.stringify({ entries, truncated, totalCount: eligibleCount }, null, 2) + "\n");
|
|
10794
10918
|
return;
|
|
10795
10919
|
}
|
|
10796
10920
|
if (entries.length === 0) {
|
|
@@ -10806,6 +10930,10 @@ function cmdHot(opts) {
|
|
|
10806
10930
|
const hotDisplayRoot = getDisplayRoot();
|
|
10807
10931
|
for (const e of entries) {
|
|
10808
10932
|
process.stdout.write(`${e.readCount} ${toDisplayPath(hotDisplayRoot, e.path)}
|
|
10933
|
+
`);
|
|
10934
|
+
}
|
|
10935
|
+
if (truncated) {
|
|
10936
|
+
process.stdout.write(`...and ${eligibleCount - entries.length} more (raise --limit to see them).
|
|
10809
10937
|
`);
|
|
10810
10938
|
}
|
|
10811
10939
|
}
|
|
@@ -11087,7 +11215,7 @@ function guardDeclarationRows(rows, budgetTokens) {
|
|
|
11087
11215
|
function recordDepDocsStat(fullSourceBytes, emittedText, packageName) {
|
|
11088
11216
|
const emittedBytes = Buffer.byteLength(emittedText, "utf8");
|
|
11089
11217
|
const bytesSaved = Math.max(1, fullSourceBytes - emittedBytes);
|
|
11090
|
-
recordStat("dep_docs", bytesSaved,
|
|
11218
|
+
recordStat("dep_docs", bytesSaved, savedTokensFromBytes(bytesSaved), void 0, packageName);
|
|
11091
11219
|
}
|
|
11092
11220
|
function runDepDocs(opts) {
|
|
11093
11221
|
const root = resolveProjectRoot({ project: opts.projectRoot ?? process.cwd() });
|
|
@@ -11259,6 +11387,13 @@ async function buildResumePacket(sessionId) {
|
|
|
11259
11387
|
function emitErr(text) {
|
|
11260
11388
|
process.stderr.write(ensureNewline(text));
|
|
11261
11389
|
}
|
|
11390
|
+
function capAndNote(rows, limit) {
|
|
11391
|
+
const shown = rows.slice(0, limit);
|
|
11392
|
+
if (shown.length < rows.length) {
|
|
11393
|
+
emitErr(`Showing ${shown.length} of ${rows.length} entries (raise --limit to see the rest).`);
|
|
11394
|
+
}
|
|
11395
|
+
return shown;
|
|
11396
|
+
}
|
|
11262
11397
|
function parseLimitOpt(cmdName, limitStr, dflt = 30) {
|
|
11263
11398
|
if (limitStr === void 0) return dflt;
|
|
11264
11399
|
let n;
|
|
@@ -11298,7 +11433,7 @@ function getNewestSessionFiles() {
|
|
|
11298
11433
|
function cmdBashHistory(opts) {
|
|
11299
11434
|
const limit = parseLimitOpt("bash-history", opts.limit);
|
|
11300
11435
|
const blobs = listBlobs(BASH_OUTPUT_SUBDIR);
|
|
11301
|
-
const
|
|
11436
|
+
const allItems = blobs.map(({ id, mtime, value }) => {
|
|
11302
11437
|
if (typeof value !== "object" || value === null) return null;
|
|
11303
11438
|
const v = value;
|
|
11304
11439
|
return {
|
|
@@ -11308,7 +11443,8 @@ function cmdBashHistory(opts) {
|
|
|
11308
11443
|
exitCode: typeof v["exitCode"] === "number" ? v["exitCode"] : -1,
|
|
11309
11444
|
sizeBytes: typeof v["sizeBytes"] === "number" ? v["sizeBytes"] : 0
|
|
11310
11445
|
};
|
|
11311
|
-
}).filter((x) => x !== null).sort((a, b) => b.storedAt - a.storedAt)
|
|
11446
|
+
}).filter((x) => x !== null).sort((a, b) => b.storedAt - a.storedAt);
|
|
11447
|
+
const items = capAndNote(allItems, limit);
|
|
11312
11448
|
if (opts.json === true) {
|
|
11313
11449
|
process.stdout.write(JSON.stringify(items, null, 2) + "\n");
|
|
11314
11450
|
return;
|
|
@@ -11329,7 +11465,7 @@ function cmdBashHistory(opts) {
|
|
|
11329
11465
|
function cmdWebHistory(opts) {
|
|
11330
11466
|
const limit = parseLimitOpt("web-history", opts.limit);
|
|
11331
11467
|
const blobs = listBlobs(WEB_OUTPUT_SUBDIR);
|
|
11332
|
-
const
|
|
11468
|
+
const allItems = blobs.map(({ id, mtime, value }) => {
|
|
11333
11469
|
if (typeof value !== "object" || value === null) return null;
|
|
11334
11470
|
const v = value;
|
|
11335
11471
|
return {
|
|
@@ -11338,7 +11474,8 @@ function cmdWebHistory(opts) {
|
|
|
11338
11474
|
bytes: typeof v["content"] === "string" ? Buffer.byteLength(v["content"], "utf8") : 0,
|
|
11339
11475
|
storedAt: mtime
|
|
11340
11476
|
};
|
|
11341
|
-
}).filter((x) => x !== null).sort((a, b) => b.storedAt - a.storedAt)
|
|
11477
|
+
}).filter((x) => x !== null).sort((a, b) => b.storedAt - a.storedAt);
|
|
11478
|
+
const items = capAndNote(allItems, limit);
|
|
11342
11479
|
if (opts.json === true) {
|
|
11343
11480
|
process.stdout.write(JSON.stringify(items, null, 2) + "\n");
|
|
11344
11481
|
return;
|
|
@@ -11357,7 +11494,7 @@ function cmdWebHistory(opts) {
|
|
|
11357
11494
|
function cmdMcpHistory(opts) {
|
|
11358
11495
|
const limit = parseLimitOpt("mcp-history", opts.limit);
|
|
11359
11496
|
const blobs = listBlobs(BASH_OUTPUT_SUBDIR).filter((b) => b.id.startsWith("mcp_"));
|
|
11360
|
-
const
|
|
11497
|
+
const allItems = blobs.map(({ id, mtime, value }) => {
|
|
11361
11498
|
if (typeof value !== "object" || value === null) return null;
|
|
11362
11499
|
const v = value;
|
|
11363
11500
|
const command = typeof v["command"] === "string" ? v["command"] : "";
|
|
@@ -11368,7 +11505,8 @@ function cmdMcpHistory(opts) {
|
|
|
11368
11505
|
storedAt: typeof v["storedAt"] === "number" ? v["storedAt"] : mtime,
|
|
11369
11506
|
sizeBytes: typeof v["sizeBytes"] === "number" ? v["sizeBytes"] : 0
|
|
11370
11507
|
};
|
|
11371
|
-
}).filter((x) => x !== null).sort((a, b) => b.storedAt - a.storedAt)
|
|
11508
|
+
}).filter((x) => x !== null).sort((a, b) => b.storedAt - a.storedAt);
|
|
11509
|
+
const items = capAndNote(allItems, limit);
|
|
11372
11510
|
if (opts.json === true) {
|
|
11373
11511
|
process.stdout.write(JSON.stringify(items, null, 2) + "\n");
|
|
11374
11512
|
return;
|
|
@@ -11504,15 +11642,15 @@ function cmdCompactHint(opts) {
|
|
|
11504
11642
|
const pressure = getContextPressure(cache ?? void 0);
|
|
11505
11643
|
const [manifest, eventCount] = sessionId !== null ? buildManifestWithCount(sessionId) : ["", 0];
|
|
11506
11644
|
const manifestTokens = estimateTokens(manifest);
|
|
11507
|
-
const
|
|
11645
|
+
const pct2 = (pressure.fillFraction * 100).toFixed(1);
|
|
11508
11646
|
if (opts.json === true) {
|
|
11509
|
-
const out2 = { tier: pressure.tier, fillFraction: pressure.fillFraction, pct: Number(
|
|
11647
|
+
const out2 = { tier: pressure.tier, fillFraction: pressure.fillFraction, pct: Number(pct2), manifestTokens, eventCount };
|
|
11510
11648
|
if (sessionId !== null) out2["sessionId"] = sessionId;
|
|
11511
11649
|
if (opts.trigger !== void 0) out2["trigger"] = opts.trigger;
|
|
11512
11650
|
process.stdout.write(JSON.stringify(out2, null, 2) + "\n");
|
|
11513
11651
|
return;
|
|
11514
11652
|
}
|
|
11515
|
-
process.stdout.write(`Compact hint \u2014 context: ${pressure.tier} (${
|
|
11653
|
+
process.stdout.write(`Compact hint \u2014 context: ${pressure.tier} (${pct2}% full)
|
|
11516
11654
|
`);
|
|
11517
11655
|
if (sessionId !== null) process.stdout.write(`Session: ${sessionId}
|
|
11518
11656
|
`);
|
|
@@ -11612,6 +11750,19 @@ function indexSizeBytes(dbPath) {
|
|
|
11612
11750
|
}
|
|
11613
11751
|
return total;
|
|
11614
11752
|
}
|
|
11753
|
+
function comparableSizes(result) {
|
|
11754
|
+
const walInFlight = result.checkpointBusy && result.afterBytes > result.beforeBytes;
|
|
11755
|
+
const before = walInFlight ? result.beforeDbBytes : result.beforeBytes;
|
|
11756
|
+
const after = walInFlight ? result.afterDbBytes : result.afterBytes;
|
|
11757
|
+
return { before, after, freed: before - after };
|
|
11758
|
+
}
|
|
11759
|
+
function dbFileBytes(dbPath) {
|
|
11760
|
+
try {
|
|
11761
|
+
return fs18.statSync(dbPath).size;
|
|
11762
|
+
} catch {
|
|
11763
|
+
return 0;
|
|
11764
|
+
}
|
|
11765
|
+
}
|
|
11615
11766
|
function tableExists(db, table) {
|
|
11616
11767
|
const row = db.prepare(`SELECT 1 AS present FROM sqlite_master WHERE type IN ('table','view') AND name = ?`).get(table);
|
|
11617
11768
|
return row?.present === 1;
|
|
@@ -11619,6 +11770,7 @@ function tableExists(db, table) {
|
|
|
11619
11770
|
function reclaimIndex(dbPath, opts = {}) {
|
|
11620
11771
|
const rebuild = opts.rebuild === true;
|
|
11621
11772
|
const beforeBytes = indexSizeBytes(dbPath);
|
|
11773
|
+
const beforeDbBytes = dbFileBytes(dbPath);
|
|
11622
11774
|
const db = getDb(dbPath);
|
|
11623
11775
|
const dropped = {};
|
|
11624
11776
|
if (rebuild) {
|
|
@@ -11643,6 +11795,8 @@ function reclaimIndex(dbPath, opts = {}) {
|
|
|
11643
11795
|
return {
|
|
11644
11796
|
beforeBytes,
|
|
11645
11797
|
afterBytes: indexSizeBytes(dbPath),
|
|
11798
|
+
beforeDbBytes,
|
|
11799
|
+
afterDbBytes: dbFileBytes(dbPath),
|
|
11646
11800
|
dropped,
|
|
11647
11801
|
rebuilt: rebuild,
|
|
11648
11802
|
checkpointBusy: checkpointBusy || finalCheckpointBusy,
|
|
@@ -11678,13 +11832,12 @@ function cmdReclaimIndex(opts) {
|
|
|
11678
11832
|
process.stdout.write(JSON.stringify({ dbPath, ...result }, null, 2) + "\n");
|
|
11679
11833
|
return;
|
|
11680
11834
|
}
|
|
11681
|
-
const freed = result
|
|
11835
|
+
const { before, after, freed } = comparableSizes(result);
|
|
11836
|
+
const delta = freed >= 0 ? `freed ${mb(freed)}` : `grew ${mb(-freed)}`;
|
|
11682
11837
|
process.stdout.write(`reclaim-index: ${dbPath}
|
|
11683
11838
|
`);
|
|
11684
|
-
process.stdout.write(
|
|
11685
|
-
|
|
11686
|
-
`
|
|
11687
|
-
);
|
|
11839
|
+
process.stdout.write(` ${mb(before)} -> ${mb(after)} (${delta})
|
|
11840
|
+
`);
|
|
11688
11841
|
if (result.rebuilt) {
|
|
11689
11842
|
for (const [table, n] of Object.entries(result.dropped)) {
|
|
11690
11843
|
process.stdout.write(` dropped ${n} row(s) from ${table}
|
|
@@ -11703,7 +11856,7 @@ function cmdReclaimIndex(opts) {
|
|
|
11703
11856
|
}
|
|
11704
11857
|
if (result.checkpointBusy) {
|
|
11705
11858
|
process.stdout.write(
|
|
11706
|
-
` note: a concurrent reader blocked WAL truncation, so some space
|
|
11859
|
+
` note: a concurrent reader blocked WAL truncation, so some space is still held in ${path18.basename(dbPath)}-wal and the figures above count the main file only. The next checkpoint releases it
|
|
11707
11860
|
`
|
|
11708
11861
|
);
|
|
11709
11862
|
}
|
|
@@ -12183,7 +12336,7 @@ function cmdProject(opts) {
|
|
|
12183
12336
|
}
|
|
12184
12337
|
function recordCompactDocStat(fullSourceBytes, emittedText, detail) {
|
|
12185
12338
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(emittedText, "utf8"));
|
|
12186
|
-
recordStat("compact_doc", bytesSaved,
|
|
12339
|
+
recordStat("compact_doc", bytesSaved, savedTokensFromBytes(bytesSaved), void 0, detail);
|
|
12187
12340
|
}
|
|
12188
12341
|
function cmdCompactDoc(opts) {
|
|
12189
12342
|
const resolved = path19.resolve(opts.filePath);
|
|
@@ -12356,7 +12509,12 @@ function cmdHistory(opts) {
|
|
|
12356
12509
|
summary: typeof v["url"] === "string" ? v["url"] : ""
|
|
12357
12510
|
};
|
|
12358
12511
|
}).filter((x) => x !== null);
|
|
12359
|
-
const
|
|
12512
|
+
const allItems = [...bashItems, ...webItems].sort((a, b) => b.storedAt - a.storedAt);
|
|
12513
|
+
const items = allItems.slice(0, limit);
|
|
12514
|
+
if (items.length < allItems.length) {
|
|
12515
|
+
process.stderr.write(`Showing ${items.length} of ${allItems.length} entries (raise --limit to see the rest).
|
|
12516
|
+
`);
|
|
12517
|
+
}
|
|
12360
12518
|
if (opts.json === true) {
|
|
12361
12519
|
process.stdout.write(JSON.stringify(items, null, 2) + "\n");
|
|
12362
12520
|
return;
|
|
@@ -12525,7 +12683,9 @@ async function buildBootstrapAudit(opts = {}) {
|
|
|
12525
12683
|
const seenFiles = /* @__PURE__ */ new Set();
|
|
12526
12684
|
const agents = await scanMetadataRoot(path20.join(home, ".claude", "agents"), "agent", diagnostics, visitedDirs, seenFiles, opts.followLinks === true);
|
|
12527
12685
|
const skills = await scanMetadataRoot(path20.join(home, ".claude", "skills"), "skill", diagnostics, visitedDirs, seenFiles, opts.followLinks === true);
|
|
12528
|
-
const
|
|
12686
|
+
const rankedEntries = [...agents, ...skills].sort((a, b) => b.metadata_bytes - a.metadata_bytes || a.path.localeCompare(b.path));
|
|
12687
|
+
const largestTotal = rankedEntries.length;
|
|
12688
|
+
const largest = rankedEntries.slice(0, top);
|
|
12529
12689
|
const metadataBytes = [...agents, ...skills].reduce((sum, entry) => sum + entry.metadata_bytes, 0);
|
|
12530
12690
|
const totalTokens = context.total_tokens + Math.floor(metadataBytes / 4);
|
|
12531
12691
|
const warnTokens = parseBudget("--warn-tokens", opts.warnTokens);
|
|
@@ -12549,6 +12709,8 @@ async function buildBootstrapAudit(opts = {}) {
|
|
|
12549
12709
|
total_estimated_tokens: totalTokens,
|
|
12550
12710
|
counts: { agents: agents.length, skills: skills.length, metadata_files: agents.length + skills.length },
|
|
12551
12711
|
largest,
|
|
12712
|
+
largestTotal,
|
|
12713
|
+
largestTruncated: largest.length < largestTotal,
|
|
12552
12714
|
diagnostics,
|
|
12553
12715
|
budgets: { warn_tokens: warnTokens, fail_tokens: failTokens, warn_bytes: warnBytes, fail_bytes: failBytes, warnings, failures }
|
|
12554
12716
|
};
|
|
@@ -12572,6 +12734,8 @@ async function runBootstrapAudit(opts = {}) {
|
|
|
12572
12734
|
`);
|
|
12573
12735
|
process.stdout.write("Largest metadata entries:\n");
|
|
12574
12736
|
for (const entry of result.largest) process.stdout.write(` ${entry.metadata_bytes.toString().padStart(7)} bytes ${entry.path}
|
|
12737
|
+
`);
|
|
12738
|
+
if (result.largestTruncated) process.stdout.write(` ...and ${result.largestTotal - result.largest.length} more (raise --top to see them).
|
|
12575
12739
|
`);
|
|
12576
12740
|
for (const diagnostic of result.diagnostics) process.stderr.write(`token-goat: bootstrap-audit: skipped ${diagnostic.path} (${diagnostic.reason})
|
|
12577
12741
|
`);
|
|
@@ -13108,10 +13272,10 @@ function printCopilotReport(report) {
|
|
|
13108
13272
|
w(` Conversation: ${conversationTokens.toLocaleString()} tok
|
|
13109
13273
|
`);
|
|
13110
13274
|
if (total > 0) {
|
|
13111
|
-
const
|
|
13275
|
+
const pct2 = (fixed / total * 100).toFixed(1);
|
|
13112
13276
|
w(` ${fixed.toLocaleString()} tok of system prompt and tool definitions ships with every
|
|
13113
13277
|
`);
|
|
13114
|
-
w(` request; at shutdown that was ${
|
|
13278
|
+
w(` request; at shutdown that was ${pct2}% of the context. No hook can reach it: Copilot
|
|
13115
13279
|
`);
|
|
13116
13280
|
w(" assembles both natively, with nothing between assembly and send. The levers are all\n");
|
|
13117
13281
|
w(" config: fewer MCP servers and custom tools, and --excluded-tools / --available-tools /\n");
|
|
@@ -13171,9 +13335,9 @@ function printCopilotReport(report) {
|
|
|
13171
13335
|
`);
|
|
13172
13336
|
} else {
|
|
13173
13337
|
for (const block of report.blocks) {
|
|
13174
|
-
const
|
|
13338
|
+
const pct2 = block.bytes > 0 ? (block.repeatBytes / block.bytes * 100).toFixed(1) : "0.0";
|
|
13175
13339
|
w(` ${block.kind}: ${countNoun(block.count, "injection")}, ${formatBytes(block.bytes)}`);
|
|
13176
|
-
w(block.repeatBytes > 0 ? `, ${formatBytes(block.repeatBytes)} re-sent verbatim (${
|
|
13340
|
+
w(block.repeatBytes > 0 ? `, ${formatBytes(block.repeatBytes)} re-sent verbatim (${pct2}%)
|
|
13177
13341
|
` : "\n");
|
|
13178
13342
|
}
|
|
13179
13343
|
}
|
|
@@ -13503,14 +13667,650 @@ function formatSessionSlice(turns) {
|
|
|
13503
13667
|
return parts.join("\n").trimEnd();
|
|
13504
13668
|
}
|
|
13505
13669
|
|
|
13506
|
-
// src/
|
|
13670
|
+
// src/session_audit.ts
|
|
13507
13671
|
import * as fs26 from "node:fs";
|
|
13508
13672
|
import * as os12 from "node:os";
|
|
13509
13673
|
import * as path26 from "node:path";
|
|
13674
|
+
import * as readline3 from "node:readline";
|
|
13675
|
+
function emptyMeasured() {
|
|
13676
|
+
return { apiCalls: 0, inputTokens: 0, cacheCreationTokens: 0, cacheReadTokens: 0, outputTokens: 0 };
|
|
13677
|
+
}
|
|
13678
|
+
function emptyCategory() {
|
|
13679
|
+
return { count: 0, bytes: 0, estTokens: 0 };
|
|
13680
|
+
}
|
|
13681
|
+
function addCategory(cat, bytes) {
|
|
13682
|
+
cat.count += 1;
|
|
13683
|
+
cat.bytes += bytes;
|
|
13684
|
+
cat.estTokens += estimateTokensFromLength(bytes);
|
|
13685
|
+
}
|
|
13686
|
+
var LOCAL_ONLY_TYPES = /* @__PURE__ */ new Set([
|
|
13687
|
+
"file-history-snapshot",
|
|
13688
|
+
"queue-operation",
|
|
13689
|
+
"mode",
|
|
13690
|
+
"permission-mode",
|
|
13691
|
+
"last-prompt",
|
|
13692
|
+
"bridge-session",
|
|
13693
|
+
"ai-title"
|
|
13694
|
+
]);
|
|
13695
|
+
var ATTACHMENT_VISIBLE_FIELDS = {
|
|
13696
|
+
hook_success: ["content"],
|
|
13697
|
+
hook_additional_context: ["content"],
|
|
13698
|
+
task_reminder: ["content"],
|
|
13699
|
+
skill_listing: ["content"],
|
|
13700
|
+
agent_listing_delta: ["addedLines"],
|
|
13701
|
+
file: ["content"],
|
|
13702
|
+
queued_command: ["prompt"],
|
|
13703
|
+
plan_file_reference: ["planContent"],
|
|
13704
|
+
edited_text_file: ["snippet"],
|
|
13705
|
+
read_truncation_notice: ["banner"],
|
|
13706
|
+
total_tokens_reminder: ["text"],
|
|
13707
|
+
nested_memory: ["content"],
|
|
13708
|
+
mcp_instructions_delta: ["addedBlocks"],
|
|
13709
|
+
invoked_skills: ["skills"],
|
|
13710
|
+
date_change: ["newDate"],
|
|
13711
|
+
goal_status: ["condition"],
|
|
13712
|
+
task_status: ["deltaSummary", "description"],
|
|
13713
|
+
command_permissions: ["allowedTools"]
|
|
13714
|
+
};
|
|
13715
|
+
var CACHE_WRITE_MULTIPLIER = 1.25;
|
|
13716
|
+
var CACHE_READ_MULTIPLIER = 0.1;
|
|
13717
|
+
var READ_DIVERT_MARKER_RE = /(?:was already read this session|Already read |You've already read|Use `token-goat (?:section|read|bash-output|config-get|skeleton)|token-goat bash-output --file)/;
|
|
13718
|
+
var READ_DIVERT_MAX_BYTES = 2500;
|
|
13719
|
+
var READ_FULL_SERVE_MIN_BYTES = 10240;
|
|
13720
|
+
var BASH_FILTER_MARKER_RE = /\[token-goat[:\]]/;
|
|
13721
|
+
var BASH_SMALL_RESULT_MAX_BYTES = 1024;
|
|
13722
|
+
function commandHead(raw) {
|
|
13723
|
+
let s = raw.trim();
|
|
13724
|
+
for (let guard = 0; guard < 20; guard++) {
|
|
13725
|
+
const stripped = s.replace(/^[(\s]+/, "");
|
|
13726
|
+
const env = /^[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|\S*)\s+/.exec(stripped);
|
|
13727
|
+
if (env !== null) {
|
|
13728
|
+
s = stripped.slice(env[0].length);
|
|
13729
|
+
continue;
|
|
13730
|
+
}
|
|
13731
|
+
const cd = /^cd\s+(?:"[^"]*"|'[^']*'|[^\s;&]+)[^\S\n]*(?:&&|;|\n)\s*/.exec(stripped);
|
|
13732
|
+
if (cd !== null) {
|
|
13733
|
+
s = stripped.slice(cd[0].length);
|
|
13734
|
+
continue;
|
|
13735
|
+
}
|
|
13736
|
+
if (stripped === s) break;
|
|
13737
|
+
s = stripped;
|
|
13738
|
+
}
|
|
13739
|
+
const m = /^"([^"]+)"|^'([^']+)'|^(\S+)/.exec(s);
|
|
13740
|
+
const tok = m === null ? "" : m[1] ?? m[2] ?? m[3] ?? "";
|
|
13741
|
+
const base = tok.split(/[\\/]/).pop() ?? "";
|
|
13742
|
+
const head = base.replace(/\.exe$/i, "").toLowerCase();
|
|
13743
|
+
return head === "" ? "(none)" : head;
|
|
13744
|
+
}
|
|
13745
|
+
function normalizeReadPath(p) {
|
|
13746
|
+
return p.replace(/\\/g, "/").toLowerCase();
|
|
13747
|
+
}
|
|
13748
|
+
function deepStringBytes(value) {
|
|
13749
|
+
if (typeof value === "string") return Buffer.byteLength(value, "utf8");
|
|
13750
|
+
if (Array.isArray(value)) return value.reduce((acc, v) => acc + deepStringBytes(v), 0);
|
|
13751
|
+
if (value !== null && typeof value === "object") {
|
|
13752
|
+
return Object.values(value).reduce((acc, v) => acc + deepStringBytes(v), 0);
|
|
13753
|
+
}
|
|
13754
|
+
return 0;
|
|
13755
|
+
}
|
|
13756
|
+
var OMISSION_MARKER_RE = /(?:--- (\d+) lines omitted ---|\[token-goat: \+?(\d+) more [a-z ]*lines omitted\]|--- patch: (\d+) lines omitted by token-goat ---|\.\.\. (\d+) lines omitted by token-goat \.\.\.)/g;
|
|
13757
|
+
function toolResultText(content) {
|
|
13758
|
+
if (typeof content === "string") return content;
|
|
13759
|
+
if (Array.isArray(content)) {
|
|
13760
|
+
let text = "";
|
|
13761
|
+
for (const block of content) {
|
|
13762
|
+
if (block !== null && typeof block === "object" && typeof block.text === "string") {
|
|
13763
|
+
text += block.text;
|
|
13764
|
+
}
|
|
13765
|
+
}
|
|
13766
|
+
return text;
|
|
13767
|
+
}
|
|
13768
|
+
return "";
|
|
13769
|
+
}
|
|
13770
|
+
function listCorpusTranscripts(corpusDir) {
|
|
13771
|
+
const found = [];
|
|
13772
|
+
const walk = (dir, entries) => {
|
|
13773
|
+
for (const entry of entries) {
|
|
13774
|
+
const full = path26.join(dir, entry.name);
|
|
13775
|
+
if (entry.name.endsWith(".jsonl")) {
|
|
13776
|
+
found.push(full);
|
|
13777
|
+
} else if (entry.isDirectory()) {
|
|
13778
|
+
let inner;
|
|
13779
|
+
try {
|
|
13780
|
+
inner = fs26.readdirSync(full, { withFileTypes: true });
|
|
13781
|
+
} catch {
|
|
13782
|
+
continue;
|
|
13783
|
+
}
|
|
13784
|
+
walk(full, inner);
|
|
13785
|
+
}
|
|
13786
|
+
}
|
|
13787
|
+
};
|
|
13788
|
+
walk(corpusDir, fs26.readdirSync(corpusDir, { withFileTypes: true }));
|
|
13789
|
+
return found.sort();
|
|
13790
|
+
}
|
|
13791
|
+
function defaultCorpusDir() {
|
|
13792
|
+
return path26.join(os12.homedir(), ".claude", "projects");
|
|
13793
|
+
}
|
|
13794
|
+
async function auditOneFile(filePath, s, toolMap, attachmentMap, hookMap, laneObservations, bashHeadMap) {
|
|
13795
|
+
const isLane = filePath.split(/[\\/]/).includes("subagents");
|
|
13796
|
+
let laneFirstPrefix = null;
|
|
13797
|
+
let laneBriefBytes = -1;
|
|
13798
|
+
const toolNameById = /* @__PURE__ */ new Map();
|
|
13799
|
+
const bashHeadById = /* @__PURE__ */ new Map();
|
|
13800
|
+
let pendingBashReread = [];
|
|
13801
|
+
const readCallById = /* @__PURE__ */ new Map();
|
|
13802
|
+
const readPathEpoch = /* @__PURE__ */ new Map();
|
|
13803
|
+
let compactEpoch = 0;
|
|
13804
|
+
let sawTokenGoatHook = false;
|
|
13805
|
+
let fileRepeats = 0;
|
|
13806
|
+
let fileRepeatsFullNoRange = 0;
|
|
13807
|
+
let fileRepeatsFullNoRangeAfterCompaction = 0;
|
|
13808
|
+
const usageSeenIds = /* @__PURE__ */ new Set();
|
|
13809
|
+
const perCall = [];
|
|
13810
|
+
const laneCalls = [0, 0];
|
|
13811
|
+
let pendingReread = [];
|
|
13812
|
+
const lastVisibleByKind = /* @__PURE__ */ new Map();
|
|
13813
|
+
const flushLane = (lane) => {
|
|
13814
|
+
const kept = [];
|
|
13815
|
+
for (const p of pendingReread) {
|
|
13816
|
+
if (p.lane !== lane) {
|
|
13817
|
+
kept.push(p);
|
|
13818
|
+
continue;
|
|
13819
|
+
}
|
|
13820
|
+
attachmentRollup(attachmentMap, p.kind).rereadTokens += p.tokens * Math.max(0, laneCalls[lane] - p.atCall);
|
|
13821
|
+
}
|
|
13822
|
+
pendingReread = kept;
|
|
13823
|
+
const keptBash = [];
|
|
13824
|
+
for (const p of pendingBashReread) {
|
|
13825
|
+
if (p.lane !== lane) {
|
|
13826
|
+
keptBash.push(p);
|
|
13827
|
+
continue;
|
|
13828
|
+
}
|
|
13829
|
+
s.bashInterception.untouchedRereadTokens += p.tokens * Math.max(0, laneCalls[lane] - p.atCall);
|
|
13830
|
+
}
|
|
13831
|
+
pendingBashReread = keptBash;
|
|
13832
|
+
};
|
|
13833
|
+
const stream = fs26.createReadStream(filePath);
|
|
13834
|
+
const rl = readline3.createInterface({ input: stream, crlfDelay: Infinity });
|
|
13835
|
+
const streamFailure = new Promise((_, reject) => stream.on("error", reject));
|
|
13836
|
+
const consume = (async () => {
|
|
13837
|
+
for await (const line of rl) {
|
|
13838
|
+
if (line.length === 0) continue;
|
|
13839
|
+
s.lines += 1;
|
|
13840
|
+
const lineBytes = Buffer.byteLength(line, "utf8");
|
|
13841
|
+
s.totalBytes += lineBytes;
|
|
13842
|
+
let obj;
|
|
13843
|
+
try {
|
|
13844
|
+
obj = JSON.parse(line);
|
|
13845
|
+
} catch {
|
|
13846
|
+
s.parseFailedLines += 1;
|
|
13847
|
+
addCategory(s.estimated.otherLocal, lineBytes);
|
|
13848
|
+
continue;
|
|
13849
|
+
}
|
|
13850
|
+
const type = typeof obj["type"] === "string" ? obj["type"] : "(untyped)";
|
|
13851
|
+
const census = s.lineTypes[type] ??= { lines: 0, bytes: 0 };
|
|
13852
|
+
census.lines += 1;
|
|
13853
|
+
census.bytes += lineBytes;
|
|
13854
|
+
if (LOCAL_ONLY_TYPES.has(type)) {
|
|
13855
|
+
addCategory(s.estimated.otherLocal, lineBytes);
|
|
13856
|
+
continue;
|
|
13857
|
+
}
|
|
13858
|
+
const message = obj["message"];
|
|
13859
|
+
if (type === "assistant" && message !== void 0) {
|
|
13860
|
+
const usage = message.usage;
|
|
13861
|
+
if (usage !== void 0 && typeof message.id === "string" && !usageSeenIds.has(message.id)) {
|
|
13862
|
+
usageSeenIds.add(message.id);
|
|
13863
|
+
const num = (k) => typeof usage[k] === "number" ? usage[k] : 0;
|
|
13864
|
+
const input = num("input_tokens");
|
|
13865
|
+
const cacheWrite = num("cache_creation_input_tokens");
|
|
13866
|
+
const cacheRead = num("cache_read_input_tokens");
|
|
13867
|
+
const output = num("output_tokens");
|
|
13868
|
+
s.measured.apiCalls += 1;
|
|
13869
|
+
s.measured.inputTokens += input;
|
|
13870
|
+
s.measured.cacheCreationTokens += cacheWrite;
|
|
13871
|
+
s.measured.cacheReadTokens += cacheRead;
|
|
13872
|
+
s.measured.outputTokens += output;
|
|
13873
|
+
if (obj["isSidechain"] === true) {
|
|
13874
|
+
s.measuredSidechain.apiCalls += 1;
|
|
13875
|
+
s.measuredSidechain.inputTokens += input;
|
|
13876
|
+
s.measuredSidechain.cacheCreationTokens += cacheWrite;
|
|
13877
|
+
s.measuredSidechain.cacheReadTokens += cacheRead;
|
|
13878
|
+
s.measuredSidechain.outputTokens += output;
|
|
13879
|
+
}
|
|
13880
|
+
if (isLane && laneFirstPrefix === null) laneFirstPrefix = input + cacheWrite + cacheRead;
|
|
13881
|
+
perCall.push({ inputTotal: input + cacheWrite + cacheRead, cacheRead, output });
|
|
13882
|
+
const callLane = obj["isSidechain"] === true ? 1 : 0;
|
|
13883
|
+
laneCalls[callLane] = laneCalls[callLane] + 1;
|
|
13884
|
+
}
|
|
13885
|
+
const blocks = Array.isArray(message.content) ? message.content : [];
|
|
13886
|
+
for (const block of blocks) {
|
|
13887
|
+
if (block === null || typeof block !== "object") continue;
|
|
13888
|
+
if (block["type"] === "text" && typeof block["text"] === "string") {
|
|
13889
|
+
addCategory(s.estimated.assistantText, Buffer.byteLength(block["text"], "utf8"));
|
|
13890
|
+
} else if (block["type"] === "thinking" && typeof block["thinking"] === "string") {
|
|
13891
|
+
addCategory(s.estimated.assistantThinking, Buffer.byteLength(block["thinking"], "utf8"));
|
|
13892
|
+
} else if (block["type"] === "tool_use" && typeof block["id"] === "string") {
|
|
13893
|
+
const name = typeof block["name"] === "string" ? block["name"] : "(unnamed)";
|
|
13894
|
+
if (!toolNameById.has(block["id"])) {
|
|
13895
|
+
toolNameById.set(block["id"], name);
|
|
13896
|
+
toolRollup(toolMap, name).calls += 1;
|
|
13897
|
+
const input = block["input"] !== null && typeof block["input"] === "object" ? block["input"] : {};
|
|
13898
|
+
if (name === "Bash" && typeof input["command"] === "string") {
|
|
13899
|
+
bashHeadById.set(block["id"], commandHead(input["command"]));
|
|
13900
|
+
} else if (name === "Read" && typeof input["file_path"] === "string") {
|
|
13901
|
+
const norm = normalizeReadPath(input["file_path"]);
|
|
13902
|
+
const priorEpoch = readPathEpoch.get(norm);
|
|
13903
|
+
readCallById.set(block["id"], { wasSeenBefore: priorEpoch !== void 0, hasRange: input["offset"] !== void 0 || input["limit"] !== void 0, afterCompaction: priorEpoch !== void 0 && compactEpoch > priorEpoch });
|
|
13904
|
+
readPathEpoch.set(norm, compactEpoch);
|
|
13905
|
+
}
|
|
13906
|
+
}
|
|
13907
|
+
let inputBytes;
|
|
13908
|
+
try {
|
|
13909
|
+
inputBytes = Buffer.byteLength(JSON.stringify(block["input"] ?? null), "utf8");
|
|
13910
|
+
} catch {
|
|
13911
|
+
inputBytes = 0;
|
|
13912
|
+
}
|
|
13913
|
+
addCategory(s.estimated.toolUseInputs, inputBytes);
|
|
13914
|
+
}
|
|
13915
|
+
}
|
|
13916
|
+
continue;
|
|
13917
|
+
}
|
|
13918
|
+
if (type === "user" && message !== void 0) {
|
|
13919
|
+
const blocks = Array.isArray(message.content) ? message.content : [];
|
|
13920
|
+
let sawToolResult = false;
|
|
13921
|
+
for (const block of blocks) {
|
|
13922
|
+
if (block === null || typeof block !== "object" || block["type"] !== "tool_result") continue;
|
|
13923
|
+
sawToolResult = true;
|
|
13924
|
+
const text = toolResultText(block["content"]);
|
|
13925
|
+
const bytes = Buffer.byteLength(text, "utf8");
|
|
13926
|
+
const id = typeof block["tool_use_id"] === "string" ? block["tool_use_id"] : "";
|
|
13927
|
+
const name = toolNameById.get(id) ?? "(unknown)";
|
|
13928
|
+
addCategory(s.estimated.toolResults, bytes);
|
|
13929
|
+
const roll = toolRollup(toolMap, name);
|
|
13930
|
+
if (!toolNameById.has(id)) roll.calls += 1;
|
|
13931
|
+
roll.resultBytes += bytes;
|
|
13932
|
+
roll.resultEstTokens += estimateTokensFromLength(bytes);
|
|
13933
|
+
if (name === "Read") {
|
|
13934
|
+
s.readInterception.readResults += 1;
|
|
13935
|
+
if (bytes < READ_DIVERT_MAX_BYTES && READ_DIVERT_MARKER_RE.test(text)) {
|
|
13936
|
+
s.readInterception.divertedByMarker += 1;
|
|
13937
|
+
s.readInterception.divertedBytes += bytes;
|
|
13938
|
+
} else if (bytes >= READ_FULL_SERVE_MIN_BYTES) {
|
|
13939
|
+
s.readInterception.fullServesOver10k += 1;
|
|
13940
|
+
s.readInterception.fullServeBytesOver10k += bytes;
|
|
13941
|
+
const call = readCallById.get(id);
|
|
13942
|
+
if (call === void 0) {
|
|
13943
|
+
s.readInterception.fullServesPathUnknown += 1;
|
|
13944
|
+
} else if (call.wasSeenBefore) {
|
|
13945
|
+
s.readInterception.fullServesRepeat += 1;
|
|
13946
|
+
s.readInterception.repeatBytes += bytes;
|
|
13947
|
+
fileRepeats += 1;
|
|
13948
|
+
if (call.hasRange) {
|
|
13949
|
+
s.readInterception.repeatWithRange += 1;
|
|
13950
|
+
} else {
|
|
13951
|
+
s.readInterception.repeatFullNoRange += 1;
|
|
13952
|
+
fileRepeatsFullNoRange += 1;
|
|
13953
|
+
if (call.afterCompaction) fileRepeatsFullNoRangeAfterCompaction += 1;
|
|
13954
|
+
}
|
|
13955
|
+
} else {
|
|
13956
|
+
s.readInterception.fullServesFirstRead += 1;
|
|
13957
|
+
}
|
|
13958
|
+
}
|
|
13959
|
+
} else if (name === "Bash") {
|
|
13960
|
+
const b = s.bashInterception;
|
|
13961
|
+
b.bashResults += 1;
|
|
13962
|
+
if (BASH_FILTER_MARKER_RE.test(text)) {
|
|
13963
|
+
b.markedByFilter += 1;
|
|
13964
|
+
b.markedBytes += bytes;
|
|
13965
|
+
} else if (bytes < BASH_SMALL_RESULT_MAX_BYTES) {
|
|
13966
|
+
b.smallUntouched += 1;
|
|
13967
|
+
b.smallUntouchedBytes += bytes;
|
|
13968
|
+
} else {
|
|
13969
|
+
b.untouched += 1;
|
|
13970
|
+
b.untouchedBytes += bytes;
|
|
13971
|
+
const tokens = estimateTokensFromLength(bytes);
|
|
13972
|
+
b.untouchedEstTokens += tokens;
|
|
13973
|
+
const lane = obj["isSidechain"] === true ? 1 : 0;
|
|
13974
|
+
pendingBashReread.push({ tokens, atCall: laneCalls[lane], lane });
|
|
13975
|
+
const head = bashHeadById.get(id) ?? "(unknown)";
|
|
13976
|
+
let headRoll = bashHeadMap.get(head);
|
|
13977
|
+
if (headRoll === void 0) {
|
|
13978
|
+
headRoll = { head, results: 0, bytes: 0 };
|
|
13979
|
+
bashHeadMap.set(head, headRoll);
|
|
13980
|
+
}
|
|
13981
|
+
headRoll.results += 1;
|
|
13982
|
+
headRoll.bytes += bytes;
|
|
13983
|
+
}
|
|
13984
|
+
}
|
|
13985
|
+
OMISSION_MARKER_RE.lastIndex = 0;
|
|
13986
|
+
for (const m of text.matchAll(OMISSION_MARKER_RE)) {
|
|
13987
|
+
s.omissionMarkers.fires += 1;
|
|
13988
|
+
s.omissionMarkers.linesOmitted += Number(m[1] ?? m[2] ?? m[3] ?? m[4] ?? 0);
|
|
13989
|
+
}
|
|
13990
|
+
}
|
|
13991
|
+
if (!sawToolResult) {
|
|
13992
|
+
const target = obj["isMeta"] === true ? s.estimated.harnessMeta : s.estimated.userTurns;
|
|
13993
|
+
let contentBytes = 0;
|
|
13994
|
+
if (typeof message.content === "string") {
|
|
13995
|
+
contentBytes = Buffer.byteLength(message.content, "utf8");
|
|
13996
|
+
} else {
|
|
13997
|
+
for (const block of blocks) {
|
|
13998
|
+
if (block !== null && typeof block === "object" && typeof block["text"] === "string") {
|
|
13999
|
+
contentBytes += Buffer.byteLength(block["text"], "utf8");
|
|
14000
|
+
}
|
|
14001
|
+
}
|
|
14002
|
+
}
|
|
14003
|
+
addCategory(target, contentBytes);
|
|
14004
|
+
if (isLane && laneBriefBytes < 0 && obj["isMeta"] !== true) laneBriefBytes = contentBytes;
|
|
14005
|
+
}
|
|
14006
|
+
continue;
|
|
14007
|
+
}
|
|
14008
|
+
if (type === "attachment") {
|
|
14009
|
+
addCategory(s.estimated.attachments, lineBytes);
|
|
14010
|
+
const att = obj["attachment"] !== null && typeof obj["attachment"] === "object" ? obj["attachment"] : {};
|
|
14011
|
+
const kind = typeof att["type"] === "string" ? att["type"] : "(untyped)";
|
|
14012
|
+
const lane = obj["isSidechain"] === true ? 1 : 0;
|
|
14013
|
+
const roll = attachmentRollup(attachmentMap, kind);
|
|
14014
|
+
roll.injections += 1;
|
|
14015
|
+
roll.lineBytes += lineBytes;
|
|
14016
|
+
const fields = ATTACHMENT_VISIBLE_FIELDS[kind];
|
|
14017
|
+
if (fields !== void 0) {
|
|
14018
|
+
const visibleBytes = fields.reduce((acc, f) => acc + deepStringBytes(att[f]), 0);
|
|
14019
|
+
roll.visibleBytes += visibleBytes;
|
|
14020
|
+
const tokens = visibleBytes > 0 ? estimateTokensFromLength(visibleBytes) : 0;
|
|
14021
|
+
roll.estTokens += tokens;
|
|
14022
|
+
if (tokens > 0) pendingReread.push({ kind, tokens, atCall: laneCalls[lane], lane });
|
|
14023
|
+
let serialized;
|
|
14024
|
+
try {
|
|
14025
|
+
serialized = JSON.stringify(fields.map((f) => att[f]));
|
|
14026
|
+
} catch {
|
|
14027
|
+
serialized = "";
|
|
14028
|
+
}
|
|
14029
|
+
const previous = lastVisibleByKind.get(kind);
|
|
14030
|
+
if (previous !== void 0) {
|
|
14031
|
+
if (previous === serialized) roll.repeatedIdentical += 1;
|
|
14032
|
+
else roll.repeatedChanged += 1;
|
|
14033
|
+
}
|
|
14034
|
+
lastVisibleByKind.set(kind, serialized);
|
|
14035
|
+
}
|
|
14036
|
+
if (kind === "hook_success") {
|
|
14037
|
+
const command = typeof att["command"] === "string" ? att["command"] : "";
|
|
14038
|
+
const origin = command.includes("token-goat") || command.includes("token_goat") ? "token-goat" : "other";
|
|
14039
|
+
if (origin === "token-goat") sawTokenGoatHook = true;
|
|
14040
|
+
const event = typeof att["hookEvent"] === "string" ? att["hookEvent"] : "(none)";
|
|
14041
|
+
const hook = hookRollup(hookMap, origin, event);
|
|
14042
|
+
hook.fires += 1;
|
|
14043
|
+
hook.stdoutBytes += typeof att["stdout"] === "string" ? Buffer.byteLength(att["stdout"], "utf8") : 0;
|
|
14044
|
+
hook.contextBytes += typeof att["content"] === "string" ? Buffer.byteLength(att["content"], "utf8") : 0;
|
|
14045
|
+
}
|
|
14046
|
+
continue;
|
|
14047
|
+
}
|
|
14048
|
+
if (type === "system" || type === "summary") {
|
|
14049
|
+
if (obj["subtype"] === "compact_boundary") {
|
|
14050
|
+
flushLane(obj["isSidechain"] === true ? 1 : 0);
|
|
14051
|
+
compactEpoch += 1;
|
|
14052
|
+
}
|
|
14053
|
+
addCategory(s.estimated.system, lineBytes);
|
|
14054
|
+
continue;
|
|
14055
|
+
}
|
|
14056
|
+
addCategory(s.estimated.otherLocal, lineBytes);
|
|
14057
|
+
}
|
|
14058
|
+
})();
|
|
14059
|
+
void consume.catch(() => {
|
|
14060
|
+
});
|
|
14061
|
+
void streamFailure.catch(() => {
|
|
14062
|
+
});
|
|
14063
|
+
try {
|
|
14064
|
+
await Promise.race([consume, streamFailure]);
|
|
14065
|
+
} finally {
|
|
14066
|
+
rl.close();
|
|
14067
|
+
stream.destroy();
|
|
14068
|
+
}
|
|
14069
|
+
flushLane(0);
|
|
14070
|
+
flushLane(1);
|
|
14071
|
+
if (sawTokenGoatHook) {
|
|
14072
|
+
s.readInterception.repeatInHookedSessions += fileRepeats;
|
|
14073
|
+
s.readInterception.repeatFullNoRangeInHookedSessions += fileRepeatsFullNoRange;
|
|
14074
|
+
s.readInterception.repeatFullNoRangeHookedAfterCompaction += fileRepeatsFullNoRangeAfterCompaction;
|
|
14075
|
+
}
|
|
14076
|
+
if (isLane) {
|
|
14077
|
+
let agentType = "(none)";
|
|
14078
|
+
try {
|
|
14079
|
+
const meta = JSON.parse(fs26.readFileSync(filePath.replace(/\.jsonl$/i, ".meta.json"), "utf8"));
|
|
14080
|
+
if (typeof meta["agentType"] === "string" && meta["agentType"] !== "") agentType = meta["agentType"];
|
|
14081
|
+
} catch {
|
|
14082
|
+
}
|
|
14083
|
+
laneObservations.push({ firstPrefixTokens: laneFirstPrefix, calls: usageSeenIds.size, briefBytes: Math.max(0, laneBriefBytes), agentType });
|
|
14084
|
+
}
|
|
14085
|
+
const callCount = perCall.length;
|
|
14086
|
+
for (let i = 0; i < callCount; i++) {
|
|
14087
|
+
const call = perCall[i];
|
|
14088
|
+
const d = s.positionDeciles[Math.min(9, Math.floor(i * 10 / callCount))];
|
|
14089
|
+
d.apiCalls += 1;
|
|
14090
|
+
d.inputTokens += call.inputTotal;
|
|
14091
|
+
d.cacheReadTokens += call.cacheRead;
|
|
14092
|
+
d.outputTokens += call.output;
|
|
14093
|
+
}
|
|
14094
|
+
}
|
|
14095
|
+
function toolRollup(toolMap, name) {
|
|
14096
|
+
let roll = toolMap.get(name);
|
|
14097
|
+
if (roll === void 0) {
|
|
14098
|
+
roll = { name, calls: 0, resultBytes: 0, resultEstTokens: 0 };
|
|
14099
|
+
toolMap.set(name, roll);
|
|
14100
|
+
}
|
|
14101
|
+
return roll;
|
|
14102
|
+
}
|
|
14103
|
+
function attachmentRollup(attachmentMap, kind) {
|
|
14104
|
+
let roll = attachmentMap.get(kind);
|
|
14105
|
+
if (roll === void 0) {
|
|
14106
|
+
roll = { kind, injections: 0, lineBytes: 0, visibleBytes: 0, estTokens: 0, rereadTokens: 0, billedEquivTokens: 0, repeatedIdentical: 0, repeatedChanged: 0 };
|
|
14107
|
+
attachmentMap.set(kind, roll);
|
|
14108
|
+
}
|
|
14109
|
+
return roll;
|
|
14110
|
+
}
|
|
14111
|
+
function hookRollup(hookMap, origin, event) {
|
|
14112
|
+
const key = `${origin}|${event}`;
|
|
14113
|
+
let roll = hookMap.get(key);
|
|
14114
|
+
if (roll === void 0) {
|
|
14115
|
+
roll = { origin, event, fires: 0, stdoutBytes: 0, contextBytes: 0 };
|
|
14116
|
+
hookMap.set(key, roll);
|
|
14117
|
+
}
|
|
14118
|
+
return roll;
|
|
14119
|
+
}
|
|
14120
|
+
async function auditSessionCorpus(opts = {}) {
|
|
14121
|
+
const corpusDir = path26.resolve(opts.dir ?? defaultCorpusDir());
|
|
14122
|
+
if (!fs26.existsSync(corpusDir)) {
|
|
14123
|
+
throw new Error(`session corpus directory not found: ${corpusDir}`);
|
|
14124
|
+
}
|
|
14125
|
+
const files = listCorpusTranscripts(corpusDir);
|
|
14126
|
+
if (files.length === 0) {
|
|
14127
|
+
throw new Error(`no .jsonl session transcripts found under ${corpusDir}`);
|
|
14128
|
+
}
|
|
14129
|
+
const started = Date.now();
|
|
14130
|
+
const summary = {
|
|
14131
|
+
corpusDir,
|
|
14132
|
+
filesScanned: 0,
|
|
14133
|
+
filesFailed: 0,
|
|
14134
|
+
lines: 0,
|
|
14135
|
+
parseFailedLines: 0,
|
|
14136
|
+
totalBytes: 0,
|
|
14137
|
+
runtimeMs: 0,
|
|
14138
|
+
measured: emptyMeasured(),
|
|
14139
|
+
measuredSidechain: emptyMeasured(),
|
|
14140
|
+
estimated: {
|
|
14141
|
+
userTurns: emptyCategory(),
|
|
14142
|
+
toolResults: emptyCategory(),
|
|
14143
|
+
assistantText: emptyCategory(),
|
|
14144
|
+
assistantThinking: emptyCategory(),
|
|
14145
|
+
toolUseInputs: emptyCategory(),
|
|
14146
|
+
attachments: emptyCategory(),
|
|
14147
|
+
harnessMeta: emptyCategory(),
|
|
14148
|
+
system: emptyCategory(),
|
|
14149
|
+
otherLocal: emptyCategory()
|
|
14150
|
+
},
|
|
14151
|
+
tools: [],
|
|
14152
|
+
attachmentKinds: [],
|
|
14153
|
+
hookOutputs: [],
|
|
14154
|
+
positionDeciles: Array.from({ length: 10 }, (_, i) => ({ decile: i + 1, apiCalls: 0, inputTokens: 0, cacheReadTokens: 0, outputTokens: 0 })),
|
|
14155
|
+
sidechainLanes: { laneFiles: 0, lanesWithUsage: 0, meanFirstCallPrefixTokens: 0, medianFirstCallPrefixTokens: 0, p90FirstCallPrefixTokens: 0, meanCallsPerLane: 0, meanBriefBytes: 0, prefixBilledEquivTokens: 0 },
|
|
14156
|
+
laneAgentTypes: [],
|
|
14157
|
+
readInterception: { readResults: 0, divertedByMarker: 0, divertedBytes: 0, fullServesOver10k: 0, fullServeBytesOver10k: 0, fullServesFirstRead: 0, fullServesRepeat: 0, repeatBytes: 0, repeatWithRange: 0, repeatFullNoRange: 0, repeatInHookedSessions: 0, repeatFullNoRangeInHookedSessions: 0, repeatFullNoRangeHookedAfterCompaction: 0, fullServesPathUnknown: 0 },
|
|
14158
|
+
bashInterception: { bashResults: 0, markedByFilter: 0, markedBytes: 0, smallUntouched: 0, smallUntouchedBytes: 0, untouched: 0, untouchedBytes: 0, untouchedEstTokens: 0, untouchedRereadTokens: 0, untouchedBilledEquivTokens: 0, untouchedHeads: [] },
|
|
14159
|
+
lineTypes: {},
|
|
14160
|
+
omissionMarkers: { fires: 0, linesOmitted: 0 }
|
|
14161
|
+
};
|
|
14162
|
+
const toolMap = /* @__PURE__ */ new Map();
|
|
14163
|
+
const attachmentMap = /* @__PURE__ */ new Map();
|
|
14164
|
+
const hookMap = /* @__PURE__ */ new Map();
|
|
14165
|
+
const laneObservations = [];
|
|
14166
|
+
const bashHeadMap = /* @__PURE__ */ new Map();
|
|
14167
|
+
for (const file of files) {
|
|
14168
|
+
try {
|
|
14169
|
+
await auditOneFile(file, summary, toolMap, attachmentMap, hookMap, laneObservations, bashHeadMap);
|
|
14170
|
+
summary.filesScanned += 1;
|
|
14171
|
+
} catch {
|
|
14172
|
+
summary.filesFailed += 1;
|
|
14173
|
+
}
|
|
14174
|
+
}
|
|
14175
|
+
const withUsage = laneObservations.filter((l) => l.firstPrefixTokens !== null);
|
|
14176
|
+
const prefixes = withUsage.map((l) => l.firstPrefixTokens).sort((a, b) => a - b);
|
|
14177
|
+
const mean = (arr) => arr.length === 0 ? 0 : Math.round(arr.reduce((a, b) => a + b, 0) / arr.length);
|
|
14178
|
+
summary.sidechainLanes = {
|
|
14179
|
+
laneFiles: laneObservations.length,
|
|
14180
|
+
lanesWithUsage: withUsage.length,
|
|
14181
|
+
meanFirstCallPrefixTokens: mean(prefixes),
|
|
14182
|
+
medianFirstCallPrefixTokens: prefixes.length === 0 ? 0 : prefixes[Math.floor(prefixes.length / 2)],
|
|
14183
|
+
p90FirstCallPrefixTokens: prefixes.length === 0 ? 0 : prefixes[Math.min(prefixes.length - 1, Math.floor(prefixes.length * 0.9))],
|
|
14184
|
+
meanCallsPerLane: mean(withUsage.map((l) => l.calls)),
|
|
14185
|
+
meanBriefBytes: mean(withUsage.map((l) => l.briefBytes)),
|
|
14186
|
+
prefixBilledEquivTokens: Math.round(withUsage.reduce((acc, l) => acc + l.firstPrefixTokens * (CACHE_WRITE_MULTIPLIER + CACHE_READ_MULTIPLIER * Math.max(0, l.calls - 1)), 0))
|
|
14187
|
+
};
|
|
14188
|
+
const byType = /* @__PURE__ */ new Map();
|
|
14189
|
+
for (const l of laneObservations) {
|
|
14190
|
+
const bucket = byType.get(l.agentType);
|
|
14191
|
+
if (bucket === void 0) byType.set(l.agentType, [l]);
|
|
14192
|
+
else bucket.push(l);
|
|
14193
|
+
}
|
|
14194
|
+
summary.laneAgentTypes = [...byType.entries()].map(([agentType, obs]) => {
|
|
14195
|
+
const typed = obs.filter((l) => l.firstPrefixTokens !== null).map((l) => l.firstPrefixTokens).sort((a, b) => a - b);
|
|
14196
|
+
return { agentType, lanes: obs.length, lanesWithUsage: typed.length, meanFirstCallPrefixTokens: mean(typed), medianFirstCallPrefixTokens: typed.length === 0 ? 0 : typed[Math.floor(typed.length / 2)] };
|
|
14197
|
+
}).sort((a, b) => b.lanes - a.lanes || a.agentType.localeCompare(b.agentType));
|
|
14198
|
+
summary.bashInterception.untouchedBilledEquivTokens = Math.round(CACHE_WRITE_MULTIPLIER * summary.bashInterception.untouchedEstTokens + CACHE_READ_MULTIPLIER * summary.bashInterception.untouchedRereadTokens);
|
|
14199
|
+
summary.bashInterception.untouchedHeads = [...bashHeadMap.values()].sort((a, b) => b.bytes - a.bytes || b.results - a.results || a.head.localeCompare(b.head));
|
|
14200
|
+
summary.tools = [...toolMap.values()].sort((a, b) => b.resultBytes - a.resultBytes || a.name.localeCompare(b.name));
|
|
14201
|
+
for (const roll of attachmentMap.values()) {
|
|
14202
|
+
roll.billedEquivTokens = Math.round(CACHE_WRITE_MULTIPLIER * roll.estTokens + CACHE_READ_MULTIPLIER * roll.rereadTokens);
|
|
14203
|
+
}
|
|
14204
|
+
summary.attachmentKinds = [...attachmentMap.values()].sort((a, b) => b.billedEquivTokens - a.billedEquivTokens || b.injections - a.injections || a.kind.localeCompare(b.kind));
|
|
14205
|
+
summary.hookOutputs = [...hookMap.values()].sort((a, b) => b.contextBytes - a.contextBytes || b.fires - a.fires || a.origin.localeCompare(b.origin) || a.event.localeCompare(b.event));
|
|
14206
|
+
summary.runtimeMs = Date.now() - started;
|
|
14207
|
+
return summary;
|
|
14208
|
+
}
|
|
14209
|
+
function fmt(n) {
|
|
14210
|
+
return n.toLocaleString("en-US");
|
|
14211
|
+
}
|
|
14212
|
+
function pct(part, whole) {
|
|
14213
|
+
return whole === 0 ? "0.0%" : `${(part / whole * 100).toFixed(1)}%`;
|
|
14214
|
+
}
|
|
14215
|
+
function formatSessionAudit(s) {
|
|
14216
|
+
const lines = [];
|
|
14217
|
+
lines.push("# Session corpus audit");
|
|
14218
|
+
lines.push(`Corpus: ${s.corpusDir}`);
|
|
14219
|
+
lines.push(`Files: ${fmt(s.filesScanned)} scanned, ${fmt(s.filesFailed)} unreadable`);
|
|
14220
|
+
lines.push(`Lines: ${fmt(s.lines)} (${fmt(s.parseFailedLines)} unparseable), bytes: ${fmt(s.totalBytes)}`);
|
|
14221
|
+
lines.push(`Runtime: ${(s.runtimeMs / 1e3).toFixed(1)}s`);
|
|
14222
|
+
lines.push("");
|
|
14223
|
+
lines.push("## Measured billed tokens (assistant message.usage, one count per API response)");
|
|
14224
|
+
const m = s.measured;
|
|
14225
|
+
lines.push(`API calls: ${fmt(m.apiCalls)} (sidechain: ${fmt(s.measuredSidechain.apiCalls)})`);
|
|
14226
|
+
lines.push(`Output tokens: ${fmt(m.outputTokens)}`);
|
|
14227
|
+
lines.push(`Input, uncached: ${fmt(m.inputTokens)}`);
|
|
14228
|
+
lines.push(`Input, cache-write: ${fmt(m.cacheCreationTokens)}`);
|
|
14229
|
+
lines.push(`Input, cache-read: ${fmt(m.cacheReadTokens)}`);
|
|
14230
|
+
const totalInput = m.inputTokens + m.cacheCreationTokens + m.cacheReadTokens;
|
|
14231
|
+
lines.push(`Cache-read share of input: ${pct(m.cacheReadTokens, totalInput)}`);
|
|
14232
|
+
lines.push("");
|
|
14233
|
+
lines.push("## Estimated content attribution (chars/3 heuristic; NOT billed units)");
|
|
14234
|
+
const e = s.estimated;
|
|
14235
|
+
const rows = [
|
|
14236
|
+
["tool results", e.toolResults],
|
|
14237
|
+
["assistant text", e.assistantText],
|
|
14238
|
+
["assistant thinking", e.assistantThinking],
|
|
14239
|
+
["tool call inputs", e.toolUseInputs],
|
|
14240
|
+
["attachments (harness)", e.attachments],
|
|
14241
|
+
["user turns", e.userTurns],
|
|
14242
|
+
["meta user lines", e.harnessMeta],
|
|
14243
|
+
["system lines", e.system],
|
|
14244
|
+
["local bookkeeping (never sent)", e.otherLocal]
|
|
14245
|
+
];
|
|
14246
|
+
const modelVisibleBytes = rows.slice(0, 8).reduce((acc, [, c]) => acc + c.bytes, 0);
|
|
14247
|
+
for (const [label, cat] of rows.sort((a, b) => b[1].bytes - a[1].bytes)) {
|
|
14248
|
+
lines.push(`${label.padEnd(31)} count ${fmt(cat.count).padStart(11)} bytes ${fmt(cat.bytes).padStart(15)} est-tokens ${fmt(cat.estTokens).padStart(13)} ${label === "local bookkeeping (never sent)" ? "(excluded from share)" : pct(cat.bytes, modelVisibleBytes)}`);
|
|
14249
|
+
}
|
|
14250
|
+
lines.push("");
|
|
14251
|
+
lines.push("## Tool results by tool (estimated content size; calls = tool_use invocations)");
|
|
14252
|
+
for (const t of s.tools.slice(0, 25)) {
|
|
14253
|
+
lines.push(`${t.name.padEnd(42)} calls ${fmt(t.calls).padStart(9)} bytes ${fmt(t.resultBytes).padStart(15)} est-tokens ${fmt(t.resultEstTokens).padStart(12)}`);
|
|
14254
|
+
}
|
|
14255
|
+
if (s.tools.length > 25) lines.push(`(${s.tools.length - 25} smaller tools omitted from this table; --json has all)`);
|
|
14256
|
+
lines.push("");
|
|
14257
|
+
lines.push("## Attachment kinds by modeled billed cost (model-visible fields only; NOT billed units)");
|
|
14258
|
+
lines.push("Model: est-tokens x 1.25 cache-write + reread-tokens x 0.1 cache-read; an injection stays in context until the next compact boundary on its lane, or end of transcript.");
|
|
14259
|
+
for (const a of s.attachmentKinds.slice(0, 15)) {
|
|
14260
|
+
lines.push(`${a.kind.padEnd(28)} inj ${fmt(a.injections).padStart(9)} visible-bytes ${fmt(a.visibleBytes).padStart(13)} est-tok ${fmt(a.estTokens).padStart(12)} reread-tok ${fmt(a.rereadTokens).padStart(14)} billed-equiv ${fmt(a.billedEquivTokens).padStart(12)} identical-reinject ${fmt(a.repeatedIdentical).padStart(8)}`);
|
|
14261
|
+
}
|
|
14262
|
+
if (s.attachmentKinds.length > 15) lines.push(`(${s.attachmentKinds.length - 15} smaller kinds omitted from this table; --json has all)`);
|
|
14263
|
+
lines.push("");
|
|
14264
|
+
lines.push("## Hook stdout channel (hook_success attachments; context-bytes is the model-visible share)");
|
|
14265
|
+
for (const h of s.hookOutputs) {
|
|
14266
|
+
lines.push(`${h.origin.padEnd(11)} ${h.event.padEnd(18)} fires ${fmt(h.fires).padStart(9)} stdout-bytes ${fmt(h.stdoutBytes).padStart(13)} context-bytes ${fmt(h.contextBytes).padStart(13)}`);
|
|
14267
|
+
}
|
|
14268
|
+
lines.push("");
|
|
14269
|
+
lines.push("## Measured billed tokens by session position (deciles of each session's API calls)");
|
|
14270
|
+
for (const d of s.positionDeciles) {
|
|
14271
|
+
lines.push(`decile ${d.decile.toString().padStart(2)} calls ${fmt(d.apiCalls).padStart(9)} input ${fmt(d.inputTokens).padStart(15)} cache-read ${fmt(d.cacheReadTokens).padStart(15)} output ${fmt(d.outputTokens).padStart(11)}`);
|
|
14272
|
+
}
|
|
14273
|
+
lines.push("");
|
|
14274
|
+
lines.push("## Subagent lanes (spawn-prefix carriage; same residency model as the attachment census; NOT billed units)");
|
|
14275
|
+
const sl = s.sidechainLanes;
|
|
14276
|
+
lines.push(`Lane files: ${fmt(sl.laneFiles)} (${fmt(sl.lanesWithUsage)} with usage)`);
|
|
14277
|
+
lines.push(`First-call prefix tokens: mean ${fmt(sl.meanFirstCallPrefixTokens)}, median ${fmt(sl.medianFirstCallPrefixTokens)}, p90 ${fmt(sl.p90FirstCallPrefixTokens)}`);
|
|
14278
|
+
lines.push(`Calls per lane: mean ${fmt(sl.meanCallsPerLane)}; task-brief bytes: mean ${fmt(sl.meanBriefBytes)}`);
|
|
14279
|
+
lines.push(`Prefix billed-equiv tokens: ${fmt(sl.prefixBilledEquivTokens)} (write x 1.25, then x 0.1 per later call in the lane)`);
|
|
14280
|
+
for (const t of s.laneAgentTypes.slice(0, 12)) {
|
|
14281
|
+
lines.push(` type ${t.agentType.padEnd(24)} lanes ${fmt(t.lanes).padStart(7)} (${fmt(t.lanesWithUsage)} with usage) prefix mean ${fmt(t.meanFirstCallPrefixTokens).padStart(9)}, median ${fmt(t.medianFirstCallPrefixTokens).padStart(9)}`);
|
|
14282
|
+
}
|
|
14283
|
+
if (s.laneAgentTypes.length > 12) lines.push(` (${s.laneAgentTypes.length - 12} smaller agent types omitted from this table; --json has all)`);
|
|
14284
|
+
lines.push("");
|
|
14285
|
+
lines.push("## Read interception (token-goat divert markers inside Read tool results)");
|
|
14286
|
+
const ri = s.readInterception;
|
|
14287
|
+
lines.push(`Read results: ${fmt(ri.readResults)}; diverted by marker: ${fmt(ri.divertedByMarker)} (${fmt(ri.divertedBytes)} bytes); full serves >=10 KiB: ${fmt(ri.fullServesOver10k)} (${fmt(ri.fullServeBytesOver10k)} bytes)`);
|
|
14288
|
+
lines.push(`Full-serve split (same transcript file; a session's lanes are separate files, so repeats UNDER-count): first read ${fmt(ri.fullServesFirstRead)}, repeat ${fmt(ri.fullServesRepeat)} (${fmt(ri.repeatBytes)} bytes), path unknown ${fmt(ri.fullServesPathUnknown)}`);
|
|
14289
|
+
lines.push(`Repeats: with offset/limit ${fmt(ri.repeatWithRange)} (deliberate paging), whole-file ${fmt(ri.repeatFullNoRange)} (divert-miss candidates), in sessions with a token-goat hook fire ${fmt(ri.repeatInHookedSessions)} (whole-file among them: ${fmt(ri.repeatFullNoRangeInHookedSessions)}, of which post-compaction and so correct by design: ${fmt(ri.repeatFullNoRangeHookedAfterCompaction)})`);
|
|
14290
|
+
lines.push("");
|
|
14291
|
+
lines.push("## Bash filter fire-rate (token-goat in-band markers inside Bash tool results)");
|
|
14292
|
+
const bi = s.bashInterception;
|
|
14293
|
+
lines.push(`Bash results: ${fmt(bi.bashResults)}; marked by a filter: ${fmt(bi.markedByFilter)} (${fmt(bi.markedBytes)} bytes); small unmarked <${fmt(BASH_SMALL_RESULT_MAX_BYTES)} B: ${fmt(bi.smallUntouched)} (${fmt(bi.smallUntouchedBytes)} bytes)`);
|
|
14294
|
+
lines.push(`Untouched >=${fmt(BASH_SMALL_RESULT_MAX_BYTES)} B: ${fmt(bi.untouched)} (${fmt(bi.untouchedBytes)} bytes, est-tokens ${fmt(bi.untouchedEstTokens)}, billed-equiv ${fmt(bi.untouchedBilledEquivTokens)})`);
|
|
14295
|
+
lines.push("A filter that matched but fell under the 100-byte net-savings floor leaves no transcript trace and counts as untouched here.");
|
|
14296
|
+
for (const h of bi.untouchedHeads.slice(0, 15)) {
|
|
14297
|
+
lines.push(` ${h.head.padEnd(28)} results ${fmt(h.results).padStart(9)} bytes ${fmt(h.bytes).padStart(15)}`);
|
|
14298
|
+
}
|
|
14299
|
+
if (bi.untouchedHeads.length > 15) lines.push(` (${bi.untouchedHeads.length - 15} smaller command heads omitted from this table; --json has all)`);
|
|
14300
|
+
lines.push("");
|
|
14301
|
+
lines.push("## Mid-trim omission markers inside tool results");
|
|
14302
|
+
lines.push(`fires: ${fmt(s.omissionMarkers.fires)}, lines discarded: ${fmt(s.omissionMarkers.linesOmitted)}`);
|
|
14303
|
+
return lines.join("\n");
|
|
14304
|
+
}
|
|
14305
|
+
|
|
14306
|
+
// src/cli_mcp_audit.ts
|
|
14307
|
+
import * as fs27 from "node:fs";
|
|
14308
|
+
import * as os13 from "node:os";
|
|
14309
|
+
import * as path27 from "node:path";
|
|
13510
14310
|
function readMcpJsonFile(configPath2) {
|
|
13511
14311
|
try {
|
|
13512
|
-
if (!
|
|
13513
|
-
const content =
|
|
14312
|
+
if (!fs27.existsSync(configPath2)) return null;
|
|
14313
|
+
const content = fs27.readFileSync(configPath2, "utf-8");
|
|
13514
14314
|
const parsed = JSON.parse(content);
|
|
13515
14315
|
const servers = parsed && typeof parsed === "object" ? parsed.mcpServers ?? parsed : null;
|
|
13516
14316
|
return servers && typeof servers === "object" ? servers : null;
|
|
@@ -13526,8 +14326,8 @@ function driveLetterCaseVariants(p) {
|
|
|
13526
14326
|
}
|
|
13527
14327
|
function readClaudeJsonConfig(claudeJsonPath, projectRoot) {
|
|
13528
14328
|
try {
|
|
13529
|
-
if (!
|
|
13530
|
-
const parsed = JSON.parse(
|
|
14329
|
+
if (!fs27.existsSync(claudeJsonPath)) return null;
|
|
14330
|
+
const parsed = JSON.parse(fs27.readFileSync(claudeJsonPath, "utf-8"));
|
|
13531
14331
|
const projects = parsed && typeof parsed === "object" ? parsed.projects : null;
|
|
13532
14332
|
if (!projects || typeof projects !== "object") return null;
|
|
13533
14333
|
const slashForms = [projectRoot, projectRoot.replace(/\\/g, "/"), projectRoot.replace(/\//g, "\\")];
|
|
@@ -13545,8 +14345,8 @@ function readClaudeJsonConfig(claudeJsonPath, projectRoot) {
|
|
|
13545
14345
|
}
|
|
13546
14346
|
}
|
|
13547
14347
|
function discoverMcpConfig(projectRoot, home) {
|
|
13548
|
-
const mcpJsonPath =
|
|
13549
|
-
const claudeJsonPath =
|
|
14348
|
+
const mcpJsonPath = path27.join(projectRoot, ".mcp.json");
|
|
14349
|
+
const claudeJsonPath = path27.join(home, ".claude.json");
|
|
13550
14350
|
const sourcesChecked = [mcpJsonPath, claudeJsonPath];
|
|
13551
14351
|
const fromMcpJson = readMcpJsonFile(mcpJsonPath);
|
|
13552
14352
|
if (fromMcpJson !== null) return { servers: fromMcpJson, sourcePath: mcpJsonPath, sourcesChecked };
|
|
@@ -13575,7 +14375,7 @@ function analyzeMcpCache() {
|
|
|
13575
14375
|
}
|
|
13576
14376
|
return serverMetrics;
|
|
13577
14377
|
}
|
|
13578
|
-
function buildMcpAuditReport(projectRoot, home =
|
|
14378
|
+
function buildMcpAuditReport(projectRoot, home = os13.homedir()) {
|
|
13579
14379
|
const discovery = discoverMcpConfig(projectRoot, home);
|
|
13580
14380
|
const config = discovery.servers;
|
|
13581
14381
|
const cacheMetrics = analyzeMcpCache();
|
|
@@ -13665,15 +14465,14 @@ var RECALL_COMMAND = {
|
|
|
13665
14465
|
function fenceTagForCacheType(cacheType) {
|
|
13666
14466
|
return cacheType === "web" ? UNTRUSTED_WEB_TAG : UNTRUSTED_TOOL_TAG;
|
|
13667
14467
|
}
|
|
14468
|
+
function fenceRecallListing(text, hits) {
|
|
14469
|
+
const tags = new Set(hits.map((hit) => fenceTagForCacheType(hit.cacheType)));
|
|
14470
|
+
const tag = tags.size === 1 ? [...tags][0] ?? UNTRUSTED_TOOL_TAG : UNTRUSTED_TOOL_TAG;
|
|
14471
|
+
return fenceUntrusted(text, tag);
|
|
14472
|
+
}
|
|
13668
14473
|
function fenceSnippetIfMatched(hit) {
|
|
13669
|
-
|
|
13670
|
-
try {
|
|
13671
|
-
if (loadConfig().injection.enabled) matches2 = scanForInjectionPatterns(hit.snippet);
|
|
13672
|
-
} catch {
|
|
13673
|
-
matches2 = [];
|
|
13674
|
-
}
|
|
14474
|
+
const matches2 = scanAndRecord(hit.snippet);
|
|
13675
14475
|
if (matches2.length === 0) return hit.snippet;
|
|
13676
|
-
recordStat("injection_detected", 0, 0, void 0, matches2.join(","));
|
|
13677
14476
|
return fenceUntrustedContent(hit.snippet, matches2, fenceTagForCacheType(hit.cacheType));
|
|
13678
14477
|
}
|
|
13679
14478
|
function printHits(query, hits) {
|
|
@@ -13685,16 +14484,15 @@ function printHits(query, hits) {
|
|
|
13685
14484
|
`);
|
|
13686
14485
|
return;
|
|
13687
14486
|
}
|
|
13688
|
-
|
|
14487
|
+
const body = hits.map((hit) => {
|
|
13689
14488
|
const label = hit.label.length > 80 ? hit.label.slice(0, 77) + "..." : hit.label;
|
|
13690
|
-
|
|
13691
|
-
|
|
13692
|
-
|
|
13693
|
-
|
|
13694
|
-
|
|
13695
|
-
|
|
14489
|
+
return `[${pad(hit.cacheType, 4)}] ${hit.id} (token-goat ${RECALL_COMMAND[hit.cacheType]} ${hit.id})
|
|
14490
|
+
${label}
|
|
14491
|
+
${hit.snippet}
|
|
14492
|
+
`;
|
|
14493
|
+
}).join("\n");
|
|
14494
|
+
w(`${fenceRecallListing(body, hits)}
|
|
13696
14495
|
`);
|
|
13697
|
-
}
|
|
13698
14496
|
}
|
|
13699
14497
|
function runRecallCommand(query, opts = {}) {
|
|
13700
14498
|
const browse = query === void 0 || query.trim() === "";
|
|
@@ -13702,7 +14500,13 @@ function runRecallCommand(query, opts = {}) {
|
|
|
13702
14500
|
...opts.type !== void 0 ? { type: opts.type } : {},
|
|
13703
14501
|
...opts.limit !== void 0 ? { limit: opts.limit } : {}
|
|
13704
14502
|
};
|
|
13705
|
-
const
|
|
14503
|
+
const effectiveLimit = opts.limit ?? RECALL_DEFAULT_LIMIT;
|
|
14504
|
+
const fetched = browse ? listRecentRecall({ ...scope, limit: effectiveLimit + 1 }) : searchRecall(query, { ...scope, limit: effectiveLimit + 1 });
|
|
14505
|
+
const hits = fetched.slice(0, effectiveLimit);
|
|
14506
|
+
if (fetched.length > hits.length) {
|
|
14507
|
+
process.stderr.write(`Showing ${hits.length} entries; more are available (raise --limit to see them).
|
|
14508
|
+
`);
|
|
14509
|
+
}
|
|
13706
14510
|
if (opts.json === true) {
|
|
13707
14511
|
const fenced = hits.map((hit) => ({ ...hit, snippet: fenceSnippetIfMatched(hit) }));
|
|
13708
14512
|
process.stdout.write(`${JSON.stringify(fenced)}
|
|
@@ -13728,9 +14532,9 @@ function printSummary(rows) {
|
|
|
13728
14532
|
};
|
|
13729
14533
|
w(pad("category", 22) + pad("emitted", 9) + pad("acted-on", 10) + pad("efficacy", 10) + pad("suppressed", 17) + pad("manual+", 9) + pad("manual-", 9) + "spent\n");
|
|
13730
14534
|
for (const row of rows) {
|
|
13731
|
-
const
|
|
14535
|
+
const pct2 = row.efficacyPct === null ? "n/a" : `${row.efficacyPct}%`;
|
|
13732
14536
|
w(
|
|
13733
|
-
pad(row.category, 22) + pad(String(row.emitted), 9) + pad(String(row.actedOn), 10) + pad(
|
|
14537
|
+
pad(row.category, 22) + pad(String(row.emitted), 9) + pad(String(row.actedOn), 10) + pad(pct2, 10) + pad(suppressedCell(row), 17) + pad(String(row.manualEffective), 9) + pad(String(row.manualIneffective), 9) + formatSpentCell(row) + "\n"
|
|
13734
14538
|
);
|
|
13735
14539
|
}
|
|
13736
14540
|
}
|
|
@@ -13875,10 +14679,10 @@ function readBoundedText(text, file) {
|
|
|
13875
14679
|
if (text === void 0 && file === void 0) throw new CliError("provide text or --file");
|
|
13876
14680
|
let value;
|
|
13877
14681
|
if (file !== void 0) {
|
|
13878
|
-
if (
|
|
14682
|
+
if (fs28.statSync(file).size > CONTENT_MAX_INPUT_CHARS) {
|
|
13879
14683
|
throw new CliError(`file exceeds the ${CONTENT_MAX_INPUT_CHARS}-byte safety limit`);
|
|
13880
14684
|
}
|
|
13881
|
-
value =
|
|
14685
|
+
value = fs28.readFileSync(file, "utf8");
|
|
13882
14686
|
} else {
|
|
13883
14687
|
if (text === void 0) throw new CliError("provide text or --file");
|
|
13884
14688
|
value = text;
|
|
@@ -13959,6 +14763,7 @@ async function cmdSemantic(query, opts) {
|
|
|
13959
14763
|
process.exitCode = code;
|
|
13960
14764
|
}
|
|
13961
14765
|
async function cmdIndex(pathArg, opts = {}) {
|
|
14766
|
+
applyIndexingPriority();
|
|
13962
14767
|
const root = pathArg ?? process.cwd();
|
|
13963
14768
|
const dbPath = opts.dbPath ?? globalDbPath();
|
|
13964
14769
|
const force = opts.force === true;
|
|
@@ -14015,10 +14820,10 @@ async function cmdIndex(pathArg, opts = {}) {
|
|
|
14015
14820
|
continue;
|
|
14016
14821
|
}
|
|
14017
14822
|
const sha = fingerprintFile(key);
|
|
14018
|
-
if (sha === null && !
|
|
14823
|
+
if (sha === null && !fs28.existsSync(key)) continue;
|
|
14019
14824
|
const entry = sha !== null ? getFileEntry(key, dbPath) : null;
|
|
14020
14825
|
const spellingStale = entry !== null && indexedPathSpellingIsStale(entry.filePath, key);
|
|
14021
|
-
const parseUnchanged = !force && !spellingStale && sha !== null && entry?.sha === sha;
|
|
14826
|
+
const parseUnchanged = !force && !spellingStale && sha !== null && entry?.sha === sha && entry.parserSha === PARSER_FINGERPRINT;
|
|
14022
14827
|
const embeddingsEnabled = loadConfig().indexing?.embeddings_enabled ?? true;
|
|
14023
14828
|
const depsAvailable = embeddingsEnabled && embeddingsDepsAvailable(getDb(dbPath));
|
|
14024
14829
|
const embedUnchanged = !force && parseUnchanged && sha !== null && isEmbedFresh(entry?.embedSha, sha, embeddingsEnabled, depsAvailable);
|
|
@@ -14041,7 +14846,7 @@ async function cmdIndex(pathArg, opts = {}) {
|
|
|
14041
14846
|
}
|
|
14042
14847
|
continue;
|
|
14043
14848
|
}
|
|
14044
|
-
if (!
|
|
14849
|
+
if (!fs28.existsSync(key)) continue;
|
|
14045
14850
|
}
|
|
14046
14851
|
if (!embedUnchanged) {
|
|
14047
14852
|
paintProgress("embedding");
|
|
@@ -14074,7 +14879,7 @@ function cmdMap(opts) {
|
|
|
14074
14879
|
out(text);
|
|
14075
14880
|
}
|
|
14076
14881
|
const bytesSaved = mapLookupBytesSaved(map, text);
|
|
14077
|
-
recordStat("map_lookup", bytesSaved,
|
|
14882
|
+
recordStat("map_lookup", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14078
14883
|
}
|
|
14079
14884
|
function cmdBridgesStatus(opts) {
|
|
14080
14885
|
if (opts.json === true) {
|
|
@@ -14101,7 +14906,7 @@ async function cmdMcpServe() {
|
|
|
14101
14906
|
let StdioServerTransport;
|
|
14102
14907
|
try {
|
|
14103
14908
|
;
|
|
14104
|
-
({ createMcpServer } = await import("./token-goat-chunk-
|
|
14909
|
+
({ createMcpServer } = await import("./token-goat-chunk-ZIPBLIUZ.mjs"));
|
|
14105
14910
|
({ StdioServerTransport } = await import("./token-goat-chunk-324QOJYZ.mjs"));
|
|
14106
14911
|
} catch (err2) {
|
|
14107
14912
|
process.stderr.write(
|
|
@@ -14114,8 +14919,8 @@ async function cmdMcpServe() {
|
|
|
14114
14919
|
const server = await createMcpServer();
|
|
14115
14920
|
const transport = new StdioServerTransport();
|
|
14116
14921
|
await server.connect(transport);
|
|
14117
|
-
await new Promise((
|
|
14118
|
-
server.onclose =
|
|
14922
|
+
await new Promise((resolve13) => {
|
|
14923
|
+
server.onclose = resolve13;
|
|
14119
14924
|
});
|
|
14120
14925
|
}
|
|
14121
14926
|
function printBridgeVerificationNotice(harness) {
|
|
@@ -14126,7 +14931,7 @@ async function cmdHook(event, opts) {
|
|
|
14126
14931
|
if (typeof opts.harness === "string" && opts.harness.length > 0) {
|
|
14127
14932
|
process.env[ENV_KEYS.HARNESS_OVERRIDE] = opts.harness;
|
|
14128
14933
|
}
|
|
14129
|
-
const { relay } = await import("./token-goat-chunk-
|
|
14934
|
+
const { relay } = await import("./token-goat-chunk-LJEETTER.mjs");
|
|
14130
14935
|
await relay(event);
|
|
14131
14936
|
}
|
|
14132
14937
|
async function cmdInstall(opts) {
|
|
@@ -14239,16 +15044,16 @@ async function cmdInstall(opts) {
|
|
|
14239
15044
|
);
|
|
14240
15045
|
}
|
|
14241
15046
|
try {
|
|
14242
|
-
const skillDir =
|
|
14243
|
-
if (
|
|
14244
|
-
const entries =
|
|
15047
|
+
const skillDir = path28.join(homedir13(), ".claude", "skills");
|
|
15048
|
+
if (fs28.existsSync(skillDir)) {
|
|
15049
|
+
const entries = fs28.readdirSync(skillDir, { withFileTypes: true });
|
|
14245
15050
|
const skillNames = [];
|
|
14246
15051
|
const sessionId = getSessionId();
|
|
14247
15052
|
for (const entry of entries) {
|
|
14248
15053
|
if (!entry.isDirectory()) continue;
|
|
14249
|
-
const skillFile =
|
|
14250
|
-
if (
|
|
14251
|
-
const body =
|
|
15054
|
+
const skillFile = path28.join(skillDir, entry.name, "SKILL.md");
|
|
15055
|
+
if (fs28.existsSync(skillFile)) {
|
|
15056
|
+
const body = fs28.readFileSync(skillFile, "utf-8");
|
|
14252
15057
|
const compact = extractCompactFromMarker(body);
|
|
14253
15058
|
if (compact === null) continue;
|
|
14254
15059
|
const sourceSha = contentHash(body);
|
|
@@ -14258,10 +15063,10 @@ async function cmdInstall(opts) {
|
|
|
14258
15063
|
}
|
|
14259
15064
|
if (skillNames.length > 0) {
|
|
14260
15065
|
const dir = skillOutputsDir();
|
|
14261
|
-
await
|
|
14262
|
-
const pregenPath =
|
|
15066
|
+
await fs28.promises.mkdir(dir, { recursive: true });
|
|
15067
|
+
const pregenPath = path28.join(dir, "pregen.json");
|
|
14263
15068
|
const pregenData = { ts: Date.now(), names: skillNames };
|
|
14264
|
-
await
|
|
15069
|
+
await fs28.promises.writeFile(pregenPath, JSON.stringify(pregenData, null, 2));
|
|
14265
15070
|
out(`Pre-generated ${skillNames.length} skill compacts.`);
|
|
14266
15071
|
}
|
|
14267
15072
|
}
|
|
@@ -14439,6 +15244,15 @@ function cmdWaste(opts = {}) {
|
|
|
14439
15244
|
...opts.copilot === true ? { copilot: true } : {}
|
|
14440
15245
|
});
|
|
14441
15246
|
}
|
|
15247
|
+
async function cmdSessionAudit(opts = {}) {
|
|
15248
|
+
let summary;
|
|
15249
|
+
try {
|
|
15250
|
+
summary = await auditSessionCorpus({ ...opts.dir !== void 0 ? { dir: opts.dir } : {} });
|
|
15251
|
+
} catch (err2) {
|
|
15252
|
+
throw new CliError(err2 instanceof Error ? err2.message : String(err2));
|
|
15253
|
+
}
|
|
15254
|
+
out(opts.json === true ? JSON.stringify(summary) : formatSessionAudit(summary));
|
|
15255
|
+
}
|
|
14442
15256
|
async function cmdSessionOutline(sessionIdOrPath, opts = {}) {
|
|
14443
15257
|
const transcriptPath = resolveSessionTranscript(sessionIdOrPath, opts.project !== void 0 ? { project: opts.project } : {});
|
|
14444
15258
|
if (transcriptPath === null) {
|
|
@@ -14452,11 +15266,11 @@ ${formatSessionOutline(turns)}`;
|
|
|
14452
15266
|
out(text);
|
|
14453
15267
|
const fullSourceBytes = sessionTranscriptSize(transcriptPath);
|
|
14454
15268
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14455
|
-
recordStat("session_outline", bytesSaved,
|
|
15269
|
+
recordStat("session_outline", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14456
15270
|
}
|
|
14457
15271
|
function sessionTranscriptSize(transcriptPath) {
|
|
14458
15272
|
try {
|
|
14459
|
-
return
|
|
15273
|
+
return fs28.statSync(transcriptPath).size;
|
|
14460
15274
|
} catch {
|
|
14461
15275
|
return 0;
|
|
14462
15276
|
}
|
|
@@ -14474,7 +15288,7 @@ async function cmdSessionSlice(sessionIdOrPath, opts) {
|
|
|
14474
15288
|
out(text);
|
|
14475
15289
|
const fullSourceBytes = sessionTranscriptSize(transcriptPath);
|
|
14476
15290
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14477
|
-
recordStat("session_slice", bytesSaved,
|
|
15291
|
+
recordStat("session_slice", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14478
15292
|
}
|
|
14479
15293
|
function cmdMcpAudit(opts = {}) {
|
|
14480
15294
|
return runMcpAuditCommand({
|
|
@@ -14513,24 +15327,13 @@ function cmdHintStats(opts = {}) {
|
|
|
14513
15327
|
...opts.markIneffective !== void 0 && isHintCategory(opts.markIneffective) ? { markIneffective: opts.markIneffective } : {}
|
|
14514
15328
|
});
|
|
14515
15329
|
}
|
|
14516
|
-
function _applyFiltersAndPrint(content, opts,
|
|
15330
|
+
function _applyFiltersAndPrint(content, opts, fenceByProvenance = false, fenceTag = UNTRUSTED_WEB_TAG) {
|
|
14517
15331
|
const emit2 = (text) => {
|
|
14518
|
-
if (!
|
|
14519
|
-
out(text);
|
|
14520
|
-
return text;
|
|
14521
|
-
}
|
|
14522
|
-
let matches2 = [];
|
|
14523
|
-
try {
|
|
14524
|
-
if (loadConfig().injection.enabled) matches2 = scanForInjectionPatterns(text);
|
|
14525
|
-
} catch {
|
|
14526
|
-
matches2 = [];
|
|
14527
|
-
}
|
|
14528
|
-
if (matches2.length === 0) {
|
|
15332
|
+
if (!fenceByProvenance || text === "") {
|
|
14529
15333
|
out(text);
|
|
14530
15334
|
return text;
|
|
14531
15335
|
}
|
|
14532
|
-
|
|
14533
|
-
const fenced = fenceUntrustedContent(text, matches2, fenceTag);
|
|
15336
|
+
const fenced = fenceUntrusted(text, fenceTag);
|
|
14534
15337
|
out(fenced);
|
|
14535
15338
|
return fenced;
|
|
14536
15339
|
};
|
|
@@ -14568,6 +15371,11 @@ function _applyFiltersAndPrint(content, opts, fenceUntrusted = false, fenceTag =
|
|
|
14568
15371
|
const headN = opts.head !== void 0 ? requireNonNegativeInt("--head", opts.head) : 30;
|
|
14569
15372
|
const tailN = opts.tail !== void 0 ? requireNonNegativeInt("--tail", opts.tail) : 80;
|
|
14570
15373
|
const applyElision = (lines2, headN2, tailN2) => lines2.length > headN2 + tailN2 + 1 ? [...lines2.slice(0, headN2), "...(elided)...", ...lines2.slice(lines2.length - tailN2)] : lines2;
|
|
15374
|
+
const noteLineCap = (which, flag, shown, total) => {
|
|
15375
|
+
if (shown >= total) return;
|
|
15376
|
+
process.stderr.write(`Showing ${which} ${shown} of ${total} lines (raise --${flag}, or --full for the whole body).
|
|
15377
|
+
`);
|
|
15378
|
+
};
|
|
14571
15379
|
let result = lines;
|
|
14572
15380
|
if (opts.head === void 0 && opts.tail === void 0) {
|
|
14573
15381
|
result = applyElision(lines, headN, tailN);
|
|
@@ -14575,25 +15383,24 @@ function _applyFiltersAndPrint(content, opts, fenceUntrusted = false, fenceTag =
|
|
|
14575
15383
|
result = applyElision(lines, headN, tailN);
|
|
14576
15384
|
} else if (opts.head !== void 0) {
|
|
14577
15385
|
result = lines.slice(0, headN);
|
|
15386
|
+
noteLineCap("first", "head", result.length, lines.length);
|
|
14578
15387
|
} else if (opts.tail !== void 0) {
|
|
14579
15388
|
result = lines.slice(Math.max(0, lines.length - tailN));
|
|
15389
|
+
noteLineCap("last", "tail", result.length, lines.length);
|
|
14580
15390
|
}
|
|
14581
15391
|
return emit2(result.join("\n"));
|
|
14582
15392
|
}
|
|
14583
|
-
function
|
|
14584
|
-
|
|
14585
|
-
|
|
14586
|
-
|
|
14587
|
-
|
|
14588
|
-
matches2 = [];
|
|
14589
|
-
}
|
|
15393
|
+
function fenceFileText(text) {
|
|
15394
|
+
return fenceUntrusted(text, UNTRUSTED_FILE_TAG);
|
|
15395
|
+
}
|
|
15396
|
+
function fenceFileFieldIfMatched(text) {
|
|
15397
|
+
const matches2 = scanAndRecord(text);
|
|
14590
15398
|
if (matches2.length === 0) return text;
|
|
14591
|
-
recordStat("injection_detected", 0, 0, void 0, matches2.join(","));
|
|
14592
15399
|
return fenceUntrustedContent(text, matches2, UNTRUSTED_FILE_TAG);
|
|
14593
15400
|
}
|
|
14594
15401
|
function fileSizeOrZero(filePath) {
|
|
14595
15402
|
try {
|
|
14596
|
-
return
|
|
15403
|
+
return fs28.statSync(filePath).size;
|
|
14597
15404
|
} catch {
|
|
14598
15405
|
return 0;
|
|
14599
15406
|
}
|
|
@@ -14608,11 +15415,11 @@ function cmdBashOutput(id, opts) {
|
|
|
14608
15415
|
}
|
|
14609
15416
|
let content;
|
|
14610
15417
|
try {
|
|
14611
|
-
const st =
|
|
15418
|
+
const st = fs28.statSync(opts.file);
|
|
14612
15419
|
if (st.isFIFO() || st.isSocket()) {
|
|
14613
15420
|
throw new CliError(`--file '${opts.file}' is a special file (FIFO or socket) \u2014 only regular files are supported`);
|
|
14614
15421
|
}
|
|
14615
|
-
content = redactIfDotenv(opts.file, decodeSource(
|
|
15422
|
+
content = redactIfDotenv(opts.file, decodeSource(fs28.readFileSync(opts.file)));
|
|
14616
15423
|
} catch (e) {
|
|
14617
15424
|
if (e instanceof CliError) throw e;
|
|
14618
15425
|
throw new CliError(`cannot read file: ${opts.file}`);
|
|
@@ -14657,7 +15464,7 @@ async function cmdPdfExtract(file, opts) {
|
|
|
14657
15464
|
const printed = _applyFiltersAndPrint(text, opts, true, UNTRUSTED_FILE_TAG);
|
|
14658
15465
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14659
15466
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(printed, "utf8"));
|
|
14660
|
-
recordStat("pdf_extract", bytesSaved,
|
|
15467
|
+
recordStat("pdf_extract", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14661
15468
|
}
|
|
14662
15469
|
async function cmdPdfLocate(file, pattern, opts) {
|
|
14663
15470
|
const locateOpts = {
|
|
@@ -14667,25 +15474,27 @@ async function cmdPdfLocate(file, pattern, opts) {
|
|
|
14667
15474
|
if (opts.context !== void 0) locateOpts.context = requirePositiveInt("--context", opts.context);
|
|
14668
15475
|
if (opts.pages !== void 0) locateOpts.pages = opts.pages;
|
|
14669
15476
|
const matches2 = await runPdfLocate(file, pattern, locateOpts);
|
|
14670
|
-
const
|
|
14671
|
-
const pages = fencedMatches.map((m) => m.page);
|
|
15477
|
+
const pages = matches2.map((m) => m.page);
|
|
14672
15478
|
let printed;
|
|
14673
15479
|
if (opts.json === true) {
|
|
15480
|
+
const fencedMatches = matches2.map((m) => ({ ...m, snippet: fenceFileFieldIfMatched(m.snippet) }));
|
|
14674
15481
|
printed = JSON.stringify({ file, pattern, matchCount: fencedMatches.length, pages, matches: fencedMatches }, null, 2);
|
|
14675
15482
|
out(printed);
|
|
14676
|
-
} else if (
|
|
15483
|
+
} else if (matches2.length === 0) {
|
|
14677
15484
|
printed = "(no matches)";
|
|
14678
15485
|
out(printed);
|
|
14679
15486
|
} else {
|
|
14680
|
-
const lines =
|
|
14681
|
-
printed =
|
|
15487
|
+
const lines = matches2.map((m) => `p${m.page}: ${m.snippet}`);
|
|
15488
|
+
printed = fenceFileText(
|
|
15489
|
+
`${lines.join("\n")}
|
|
14682
15490
|
|
|
14683
|
-
${countNoun(
|
|
15491
|
+
${countNoun(matches2.length, "match", "matches")} across ${countNoun(pages.length, "page")}`
|
|
15492
|
+
);
|
|
14684
15493
|
out(printed);
|
|
14685
15494
|
}
|
|
14686
15495
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14687
15496
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(printed, "utf8"));
|
|
14688
|
-
recordStat("pdf_locate", bytesSaved,
|
|
15497
|
+
recordStat("pdf_locate", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14689
15498
|
}
|
|
14690
15499
|
async function cmdPdfOutline(file, opts) {
|
|
14691
15500
|
const entries = await runPdfOutline(file);
|
|
@@ -14697,12 +15506,11 @@ async function cmdPdfOutline(file, opts) {
|
|
|
14697
15506
|
}
|
|
14698
15507
|
return;
|
|
14699
15508
|
}
|
|
14700
|
-
const
|
|
14701
|
-
const text = opts.json === true ? JSON.stringify(fencedEntries, null, 2) : fencedEntries.map((e) => `${" ".repeat(e.level)}${e.title}${e.page !== null ? ` (p.${e.page})` : ""}`).join("\n");
|
|
15509
|
+
const text = opts.json === true ? JSON.stringify(entries.map((e) => ({ ...e, title: fenceFileFieldIfMatched(e.title) })), null, 2) : fenceFileText(entries.map((e) => `${" ".repeat(e.level)}${e.title}${e.page !== null ? ` (p.${e.page})` : ""}`).join("\n"));
|
|
14702
15510
|
out(text);
|
|
14703
15511
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14704
15512
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14705
|
-
recordStat("pdf_outline", bytesSaved,
|
|
15513
|
+
recordStat("pdf_outline", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14706
15514
|
}
|
|
14707
15515
|
async function cmdPdfMeta(file, opts = {}) {
|
|
14708
15516
|
const meta = await runPdfMeta(file);
|
|
@@ -14716,7 +15524,7 @@ async function cmdPdfMeta(file, opts = {}) {
|
|
|
14716
15524
|
out(text);
|
|
14717
15525
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14718
15526
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14719
|
-
recordStat("pdf_meta", bytesSaved,
|
|
15527
|
+
recordStat("pdf_meta", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14720
15528
|
}
|
|
14721
15529
|
async function cmdImageMeta(file, opts = {}) {
|
|
14722
15530
|
const meta = await runImageMeta(file);
|
|
@@ -14738,7 +15546,12 @@ ${msg}`;
|
|
|
14738
15546
|
out(text);
|
|
14739
15547
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14740
15548
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14741
|
-
recordStat("image_meta", bytesSaved,
|
|
15549
|
+
recordStat("image_meta", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
15550
|
+
}
|
|
15551
|
+
function fenceOcrText(text) {
|
|
15552
|
+
if (!injectionFencingEnabled()) return text;
|
|
15553
|
+
scanAndRecord(text);
|
|
15554
|
+
return fenceUntrustedOcrText(text);
|
|
14742
15555
|
}
|
|
14743
15556
|
async function cmdImageText(file, opts = {}) {
|
|
14744
15557
|
const result = await runImageText(file);
|
|
@@ -14751,7 +15564,7 @@ async function cmdImageText(file, opts = {}) {
|
|
|
14751
15564
|
let text;
|
|
14752
15565
|
if (opts.json === true) {
|
|
14753
15566
|
text = JSON.stringify(
|
|
14754
|
-
result.text === null ? result : { ...result, text:
|
|
15567
|
+
result.text === null ? result : { ...result, text: fenceOcrText(result.text) },
|
|
14755
15568
|
null,
|
|
14756
15569
|
2
|
|
14757
15570
|
);
|
|
@@ -14759,13 +15572,13 @@ async function cmdImageText(file, opts = {}) {
|
|
|
14759
15572
|
const lines = [`Confidence: ${Math.round(result.confidence)}%`, `Characters: ${result.chars}`];
|
|
14760
15573
|
text = result.textHeavy && result.text !== null ? `${lines.join("\n")}
|
|
14761
15574
|
|
|
14762
|
-
${
|
|
15575
|
+
${fenceOcrText(result.text)}` : `${lines.join("\n")}
|
|
14763
15576
|
(below usefulness threshold; text likely noise, not shown)`;
|
|
14764
15577
|
}
|
|
14765
15578
|
out(text);
|
|
14766
15579
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14767
15580
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14768
|
-
recordStat("image_text", bytesSaved,
|
|
15581
|
+
recordStat("image_text", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14769
15582
|
}
|
|
14770
15583
|
function cmdVideoChapters(file) {
|
|
14771
15584
|
const { chapters, subtitleStreams } = extractVideoChapters(file);
|
|
@@ -14791,7 +15604,7 @@ function cmdVideoChapters(file) {
|
|
|
14791
15604
|
out(text);
|
|
14792
15605
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14793
15606
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(text, "utf8"));
|
|
14794
|
-
recordStat("video_chapters", bytesSaved,
|
|
15607
|
+
recordStat("video_chapters", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14795
15608
|
}
|
|
14796
15609
|
function formatVideoTimestamp(totalSeconds) {
|
|
14797
15610
|
const hours = Math.floor(totalSeconds / 3600);
|
|
@@ -14818,24 +15631,23 @@ function cmdSharepointResolve(url) {
|
|
|
14818
15631
|
function recordXlsxStat(kind, file, emitted) {
|
|
14819
15632
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14820
15633
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(emitted, "utf8"));
|
|
14821
|
-
recordStat(kind, bytesSaved,
|
|
15634
|
+
recordStat(kind, bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14822
15635
|
}
|
|
14823
15636
|
async function cmdXlsxSheets(file, opts = {}) {
|
|
14824
15637
|
const sheets = await listSheets(file);
|
|
14825
|
-
const
|
|
14826
|
-
const text = opts.json === true ? JSON.stringify(fencedSheets.map((s) => ({ name: s.name, ref: s.ref, rows: s.rows, cols: s.cols })), null, 2) : fencedSheets.map((s) => `${s.name} ${s.ref} (${s.rows} rows x ${s.cols} cols)`).join("\n");
|
|
15638
|
+
const text = opts.json === true ? JSON.stringify(sheets.map((s) => ({ name: fenceFileFieldIfMatched(s.name), ref: s.ref, rows: s.rows, cols: s.cols })), null, 2) : fenceFileText(sheets.map((s) => `${s.name} ${s.ref} (${s.rows} rows x ${s.cols} cols)`).join("\n"));
|
|
14827
15639
|
out(text);
|
|
14828
15640
|
recordXlsxStat("xlsx_sheets", file, text);
|
|
14829
15641
|
}
|
|
14830
15642
|
async function cmdXlsxHead(file, opts) {
|
|
14831
15643
|
const rows = opts.rows !== void 0 ? requireNonNegativeInt("--rows", opts.rows) : 20;
|
|
14832
|
-
const text =
|
|
15644
|
+
const text = fenceFileText(await headSheet(file, opts.sheet, rows));
|
|
14833
15645
|
out(text);
|
|
14834
15646
|
recordXlsxStat("xlsx_head", file, text);
|
|
14835
15647
|
}
|
|
14836
15648
|
async function cmdXlsxRange(file, opts) {
|
|
14837
15649
|
const result = await rangeSheet(file, opts.sheet, opts.range, opts.formulas === true);
|
|
14838
|
-
const text =
|
|
15650
|
+
const text = fenceFileText(formatXlsxRange(result));
|
|
14839
15651
|
out(text);
|
|
14840
15652
|
recordXlsxStat("xlsx_range", file, text);
|
|
14841
15653
|
}
|
|
@@ -14847,32 +15659,33 @@ async function cmdXlsxQuery(file, opts) {
|
|
|
14847
15659
|
...wheres !== void 0 ? { wheres } : {},
|
|
14848
15660
|
...opts.head !== void 0 ? { head: requireNonNegativeInt("--head", opts.head) } : {}
|
|
14849
15661
|
});
|
|
14850
|
-
const text =
|
|
15662
|
+
const text = fenceFileText(formatCsvTable(result, (opts.where ?? []).map((w) => `--where ${w}`)));
|
|
14851
15663
|
out(text);
|
|
14852
15664
|
recordXlsxStat("xlsx_query", file, text);
|
|
14853
15665
|
}
|
|
14854
15666
|
function recordDocStat(kind, file, emitted) {
|
|
14855
15667
|
const fullSourceBytes = fileSizeOrZero(file);
|
|
14856
15668
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(emitted, "utf8"));
|
|
14857
|
-
recordStat(kind, bytesSaved,
|
|
15669
|
+
recordStat(kind, bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
14858
15670
|
}
|
|
14859
15671
|
async function cmdPptxOutline(file, opts) {
|
|
14860
15672
|
const slides = await pptxOutline(file);
|
|
14861
|
-
const
|
|
14862
|
-
|
|
15673
|
+
const text = opts.json === true ? JSON.stringify(slides.map((s) => ({ ...s, title: fenceFileFieldIfMatched(s.title) })), null, 2) : fenceFileText(
|
|
15674
|
+
slides.map((s) => `${s.slide}. ${s.title || "(untitled)"} [${s.bodyChars} body chars${s.hasNotes ? ", has notes" : ""}]`).join("\n")
|
|
15675
|
+
);
|
|
14863
15676
|
out(text);
|
|
14864
15677
|
recordDocStat("pptx_outline", file, text);
|
|
14865
15678
|
}
|
|
14866
15679
|
async function cmdPptxSlide(file, opts) {
|
|
14867
15680
|
const n = requireNonNegativeInt("--slide", opts.slide);
|
|
14868
|
-
const text =
|
|
15681
|
+
const text = fenceFileText(await pptxSlideText(file, n, opts.notes === true));
|
|
14869
15682
|
out(text);
|
|
14870
15683
|
recordDocStat("pptx_slide", file, text);
|
|
14871
15684
|
}
|
|
14872
15685
|
async function cmdPptxNotes(file, opts) {
|
|
14873
15686
|
const n = opts.slide !== void 0 ? requireNonNegativeInt("--slide", opts.slide) : void 0;
|
|
14874
15687
|
const text = await pptxNotesText(file, n);
|
|
14875
|
-
const printed = text.length > 0 ?
|
|
15688
|
+
const printed = text.length > 0 ? fenceFileText(text) : "no speaker notes found";
|
|
14876
15689
|
out(printed);
|
|
14877
15690
|
recordDocStat("pptx_notes", file, printed);
|
|
14878
15691
|
}
|
|
@@ -14882,7 +15695,7 @@ async function cmdPptxText(file, opts) {
|
|
|
14882
15695
|
out("no matches");
|
|
14883
15696
|
return;
|
|
14884
15697
|
}
|
|
14885
|
-
const text =
|
|
15698
|
+
const text = fenceFileText(matches2.map((m) => `Slide ${m.slide}: ...${m.snippet}...`).join("\n"));
|
|
14886
15699
|
out(text);
|
|
14887
15700
|
recordDocStat("pptx_text", file, text);
|
|
14888
15701
|
}
|
|
@@ -14896,8 +15709,7 @@ async function cmdDocxOutline(file, opts) {
|
|
|
14896
15709
|
}
|
|
14897
15710
|
return;
|
|
14898
15711
|
}
|
|
14899
|
-
const
|
|
14900
|
-
const text = opts.json === true ? JSON.stringify(fencedHeadings, null, 2) : fencedHeadings.map((h) => `${" ".repeat(h.level - 1)}${h.text}`).join("\n");
|
|
15712
|
+
const text = opts.json === true ? JSON.stringify(headings.map((h) => ({ ...h, text: fenceFileFieldIfMatched(h.text) })), null, 2) : fenceFileText(headings.map((h) => `${" ".repeat(h.level - 1)}${h.text}`).join("\n"));
|
|
14901
15713
|
out(text);
|
|
14902
15714
|
recordDocStat("docx_outline", file, text);
|
|
14903
15715
|
}
|
|
@@ -15041,7 +15853,7 @@ function emitExtraFileArgsNote(command, first, extras, opts = {}) {
|
|
|
15041
15853
|
}
|
|
15042
15854
|
async function cmdCompress(opts) {
|
|
15043
15855
|
try {
|
|
15044
|
-
const bashRunner = await import("./token-goat-chunk-
|
|
15856
|
+
const bashRunner = await import("./token-goat-chunk-L2XHDICZ.mjs");
|
|
15045
15857
|
if (opts.compress === false) {
|
|
15046
15858
|
process.exitCode = bashRunner.runRaw(opts.cmd, parseTimeout(opts.timeout, bashRunner.DEFAULT_TIMEOUT_SECONDS));
|
|
15047
15859
|
return;
|
|
@@ -15067,9 +15879,12 @@ async function cmdSkillBody(name, opts) {
|
|
|
15067
15879
|
if (filePath === null) {
|
|
15068
15880
|
throw new CliError(`skill '${name}' not found`);
|
|
15069
15881
|
}
|
|
15070
|
-
const body = decodeSource(
|
|
15882
|
+
const body = decodeSource(fs28.readFileSync(filePath));
|
|
15071
15883
|
if (opts.compact === true) {
|
|
15072
|
-
|
|
15884
|
+
const emitted = extractCompactFromMarker(body) ?? body;
|
|
15885
|
+
out(emitted);
|
|
15886
|
+
const bytesSaved = Buffer.byteLength(body, "utf8") - Buffer.byteLength(emitted, "utf8");
|
|
15887
|
+
if (bytesSaved > 0) recordStat("skill_body:compact", bytesSaved, savedTokensFromBytes(bytesSaved), void 0, name);
|
|
15073
15888
|
} else {
|
|
15074
15889
|
out(body);
|
|
15075
15890
|
}
|
|
@@ -15081,12 +15896,26 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15081
15896
|
const skills = await listSkills(sessionId);
|
|
15082
15897
|
let regenerated = 0;
|
|
15083
15898
|
let skipped = 0;
|
|
15899
|
+
let noMarker = 0;
|
|
15900
|
+
let unresolvable = 0;
|
|
15084
15901
|
for (const skill of skills) {
|
|
15085
15902
|
const filePath = await getSkillFilePath(skill.name);
|
|
15086
|
-
if (!filePath)
|
|
15087
|
-
|
|
15903
|
+
if (!filePath) {
|
|
15904
|
+
unresolvable++;
|
|
15905
|
+
continue;
|
|
15906
|
+
}
|
|
15907
|
+
let body2;
|
|
15908
|
+
try {
|
|
15909
|
+
body2 = decodeSource(fs28.readFileSync(filePath));
|
|
15910
|
+
} catch {
|
|
15911
|
+
unresolvable++;
|
|
15912
|
+
continue;
|
|
15913
|
+
}
|
|
15088
15914
|
const compact2 = extractCompactFromMarker(body2);
|
|
15089
|
-
if (compact2 === null)
|
|
15915
|
+
if (compact2 === null) {
|
|
15916
|
+
noMarker++;
|
|
15917
|
+
continue;
|
|
15918
|
+
}
|
|
15090
15919
|
const sourceSha2 = contentHash(body2);
|
|
15091
15920
|
if (skill.compactStale === false) {
|
|
15092
15921
|
skipped++;
|
|
@@ -15095,7 +15924,13 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15095
15924
|
regenerated++;
|
|
15096
15925
|
}
|
|
15097
15926
|
}
|
|
15098
|
-
|
|
15927
|
+
const parts = [`Regenerated ${regenerated}, skipped ${skipped} (fresh)`];
|
|
15928
|
+
if (noMarker > 0) parts.push(`no marker ${noMarker}`);
|
|
15929
|
+
if (unresolvable > 0) parts.push(`unresolvable ${unresolvable}`);
|
|
15930
|
+
out(`${parts.join(", ")}, total ${skills.length}.`);
|
|
15931
|
+
if (noMarker > 0) {
|
|
15932
|
+
out(`${noMarker} cached skill${noMarker === 1 ? " has" : "s have"} no COMPACT_END marker and cannot be compacted; run \`token-goat skill-size\` for per-skill marker recommendations.`);
|
|
15933
|
+
}
|
|
15099
15934
|
return;
|
|
15100
15935
|
}
|
|
15101
15936
|
let body;
|
|
@@ -15105,19 +15940,19 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15105
15940
|
if (!opts.path.trim()) {
|
|
15106
15941
|
throw new CliError("--path cannot be empty");
|
|
15107
15942
|
}
|
|
15108
|
-
if (!
|
|
15943
|
+
if (!fs28.existsSync(opts.path)) {
|
|
15109
15944
|
throw new CliError(`skill file not found: ${opts.path}`);
|
|
15110
15945
|
}
|
|
15111
15946
|
try {
|
|
15112
|
-
body =
|
|
15947
|
+
body = fs28.readFileSync(opts.path, "utf-8");
|
|
15113
15948
|
} catch (e) {
|
|
15114
15949
|
if (e.code === "ENOENT") {
|
|
15115
15950
|
throw new CliError(`skill file not found: ${opts.path}`);
|
|
15116
15951
|
}
|
|
15117
15952
|
throw new CliError(`failed to read skill file '${opts.path}': ${extractErrorMessage(e)}`);
|
|
15118
15953
|
}
|
|
15119
|
-
cacheName = name ??
|
|
15120
|
-
sourcePath =
|
|
15954
|
+
cacheName = name ?? path28.basename(path28.dirname(path28.resolve(opts.path)));
|
|
15955
|
+
sourcePath = path28.resolve(opts.path);
|
|
15121
15956
|
} else {
|
|
15122
15957
|
if (name === void 0 || !name.trim()) {
|
|
15123
15958
|
throw new CliError("skill-compact requires a <name> or --path <file>");
|
|
@@ -15126,7 +15961,7 @@ async function cmdSkillCompact(name, opts) {
|
|
|
15126
15961
|
if (filePath === null) {
|
|
15127
15962
|
throw new CliError(`skill '${name}' not found`);
|
|
15128
15963
|
}
|
|
15129
|
-
body =
|
|
15964
|
+
body = fs28.readFileSync(filePath, "utf-8");
|
|
15130
15965
|
cacheName = name;
|
|
15131
15966
|
sourcePath = filePath;
|
|
15132
15967
|
}
|
|
@@ -15258,8 +16093,8 @@ async function cmdSkillDiff(name) {
|
|
|
15258
16093
|
}
|
|
15259
16094
|
const newer = versions[0];
|
|
15260
16095
|
const older = versions[1];
|
|
15261
|
-
const newerBody = await
|
|
15262
|
-
const olderBody = await
|
|
16096
|
+
const newerBody = await fs28.promises.readFile(path28.resolve(dir, `${newer.outputId}.txt`), "utf-8").catch(() => null);
|
|
16097
|
+
const olderBody = await fs28.promises.readFile(path28.resolve(dir, `${older.outputId}.txt`), "utf-8").catch(() => null);
|
|
15263
16098
|
if (newerBody === null || olderBody === null) {
|
|
15264
16099
|
out(`a cached version of '${name}' was evicted while diffing -- try again`);
|
|
15265
16100
|
return;
|
|
@@ -15288,7 +16123,7 @@ async function cmdSkillSection(nameHeading, headingArg) {
|
|
|
15288
16123
|
if (!filePath) {
|
|
15289
16124
|
throw new CliError(`skill '${skillName}' not found`);
|
|
15290
16125
|
}
|
|
15291
|
-
const body = decodeSource(
|
|
16126
|
+
const body = decodeSource(fs28.readFileSync(filePath));
|
|
15292
16127
|
const extracted = extractNamedSection(body, heading);
|
|
15293
16128
|
if (!extracted) {
|
|
15294
16129
|
const messages = [`Section '${heading}' not found in skill '${skillName}'`];
|
|
@@ -15303,7 +16138,7 @@ async function cmdSkillSection(nameHeading, headingArg) {
|
|
|
15303
16138
|
}
|
|
15304
16139
|
function atomicWriteBuffer(dest, data) {
|
|
15305
16140
|
try {
|
|
15306
|
-
if (
|
|
16141
|
+
if (fs28.statSync(dest).isDirectory()) {
|
|
15307
16142
|
const e = Object.assign(new Error(`EISDIR: illegal operation on a directory, open '${dest}'`), { code: "EISDIR", path: dest });
|
|
15308
16143
|
throw e;
|
|
15309
16144
|
}
|
|
@@ -15311,23 +16146,23 @@ function atomicWriteBuffer(dest, data) {
|
|
|
15311
16146
|
if (e.code !== "ENOENT") throw e;
|
|
15312
16147
|
}
|
|
15313
16148
|
const rnd = randomBytes(4).toString("hex");
|
|
15314
|
-
const tmp =
|
|
16149
|
+
const tmp = path28.join(path28.dirname(path28.resolve(dest)), `.tmp.${process.pid}.${rnd}`);
|
|
15315
16150
|
try {
|
|
15316
|
-
|
|
16151
|
+
fs28.writeFileSync(tmp, data, { mode: 384 });
|
|
15317
16152
|
try {
|
|
15318
|
-
const destMode =
|
|
15319
|
-
|
|
16153
|
+
const destMode = fs28.statSync(dest).mode;
|
|
16154
|
+
fs28.chmodSync(tmp, destMode);
|
|
15320
16155
|
} catch (e) {
|
|
15321
16156
|
if (e.code !== "ENOENT") throw e;
|
|
15322
16157
|
}
|
|
15323
16158
|
withRetryOnLock(() => {
|
|
15324
16159
|
try {
|
|
15325
|
-
|
|
16160
|
+
fs28.renameSync(tmp, dest);
|
|
15326
16161
|
} catch (e) {
|
|
15327
16162
|
if (e.code === "EXDEV") {
|
|
15328
|
-
|
|
16163
|
+
fs28.copyFileSync(tmp, dest);
|
|
15329
16164
|
try {
|
|
15330
|
-
|
|
16165
|
+
fs28.unlinkSync(tmp);
|
|
15331
16166
|
} catch (ue) {
|
|
15332
16167
|
process.stderr.write(`token-goat write-file: warning: could not remove temp file ${tmp}: ${ue.message}
|
|
15333
16168
|
`);
|
|
@@ -15339,7 +16174,7 @@ function atomicWriteBuffer(dest, data) {
|
|
|
15339
16174
|
});
|
|
15340
16175
|
} catch (e) {
|
|
15341
16176
|
try {
|
|
15342
|
-
|
|
16177
|
+
fs28.unlinkSync(tmp);
|
|
15343
16178
|
} catch {
|
|
15344
16179
|
}
|
|
15345
16180
|
throw e;
|
|
@@ -15349,9 +16184,9 @@ function mapFsError(e, src, dest, srcLabel = "source") {
|
|
|
15349
16184
|
const fe = e;
|
|
15350
16185
|
if (fe.code === "ENOENT") {
|
|
15351
16186
|
const errPath = fe.path ?? "";
|
|
15352
|
-
const isSource = src !== void 0 &&
|
|
16187
|
+
const isSource = src !== void 0 && path28.resolve(errPath) === path28.resolve(src);
|
|
15353
16188
|
if (isSource) throw new CliError(`${/\bfile$/i.test(srcLabel) ? srcLabel : `${srcLabel} file`} not found: ${src}`);
|
|
15354
|
-
const destDir = dest ?
|
|
16189
|
+
const destDir = dest ? path28.dirname(path28.resolve(dest)) : path28.dirname(path28.resolve(errPath || "."));
|
|
15355
16190
|
throw new CliError(`destination directory does not exist: ${destDir}`);
|
|
15356
16191
|
}
|
|
15357
16192
|
if (fe.code === "ENOTDIR") {
|
|
@@ -15362,7 +16197,7 @@ function mapFsError(e, src, dest, srcLabel = "source") {
|
|
|
15362
16197
|
}
|
|
15363
16198
|
if (fe.code === "EISDIR") {
|
|
15364
16199
|
const errPath = fe.path ?? "";
|
|
15365
|
-
const isSource = src !== void 0 && (errPath === "" ||
|
|
16200
|
+
const isSource = src !== void 0 && (errPath === "" || path28.resolve(errPath) === path28.resolve(src));
|
|
15366
16201
|
if (isSource) throw new CliError(`source is a directory, not a file: ${src}`);
|
|
15367
16202
|
throw new CliError(`destination is a directory, not a file: ${dest ?? (errPath || "(unknown)")}`);
|
|
15368
16203
|
}
|
|
@@ -15431,7 +16266,7 @@ function validateWritablePath(dest, label) {
|
|
|
15431
16266
|
throw new CliError(`${label} path contains a null byte`);
|
|
15432
16267
|
}
|
|
15433
16268
|
if (isWindows()) {
|
|
15434
|
-
const base =
|
|
16269
|
+
const base = path28.basename(dest);
|
|
15435
16270
|
const stem = base.replace(/\.[^.]*$/, "").toUpperCase();
|
|
15436
16271
|
if (WIN_RESERVED.has(stem)) {
|
|
15437
16272
|
throw new CliError(`${label} '${base}' is a reserved Windows device name`);
|
|
@@ -15461,7 +16296,7 @@ function readFileBoundedRaw(filePath, label, allowStdIn = false) {
|
|
|
15461
16296
|
throw new CliError(`${label} ${filePath} requires piped input; use ${altLabel} for interactive use`);
|
|
15462
16297
|
}
|
|
15463
16298
|
try {
|
|
15464
|
-
const st =
|
|
16299
|
+
const st = fs28.statSync(filePath);
|
|
15465
16300
|
if (st.isFIFO() || st.isSocket()) {
|
|
15466
16301
|
throw new CliError(`${label} '${filePath}' is a special file (FIFO or socket) \u2014 only regular files are supported`);
|
|
15467
16302
|
}
|
|
@@ -15469,7 +16304,7 @@ function readFileBoundedRaw(filePath, label, allowStdIn = false) {
|
|
|
15469
16304
|
if (st.size > maxBytes) {
|
|
15470
16305
|
throw new CliError(`${label} '${filePath}' exceeds size limit (${Math.round(st.size / 1024 / 1024)} MB); set TOKEN_GOAT_MAX_STDIN_MB to override`);
|
|
15471
16306
|
}
|
|
15472
|
-
return
|
|
16307
|
+
return fs28.readFileSync(filePath);
|
|
15473
16308
|
} catch (e) {
|
|
15474
16309
|
if (e instanceof CliError) throw e;
|
|
15475
16310
|
mapFsError(e, filePath, void 0, label);
|
|
@@ -15513,7 +16348,7 @@ function cmdNoteAdd(file, opts) {
|
|
|
15513
16348
|
throw new CliError("note content must be valid UTF-8 text");
|
|
15514
16349
|
}
|
|
15515
16350
|
const resolvedPath = resolveIndexPath(file);
|
|
15516
|
-
if (!
|
|
16351
|
+
if (!fs28.existsSync(resolvedPath)) {
|
|
15517
16352
|
throw new CliError(`File not found: '${resolvedPath}'`);
|
|
15518
16353
|
}
|
|
15519
16354
|
healStaleIndex(resolvedPath);
|
|
@@ -15572,7 +16407,7 @@ function cmdWriteFile(dest, opts) {
|
|
|
15572
16407
|
throw new CliError(`TOKEN_GOAT_MAX_STDIN_MB must be a positive integer; got '${process.env["TOKEN_GOAT_MAX_STDIN_MB"] ?? ""}'`);
|
|
15573
16408
|
}
|
|
15574
16409
|
const maxBytes = maxMB * 1024 * 1024;
|
|
15575
|
-
return new Promise((
|
|
16410
|
+
return new Promise((resolve13, reject) => {
|
|
15576
16411
|
const chunks = [];
|
|
15577
16412
|
let totalBytes = 0;
|
|
15578
16413
|
let settled = false;
|
|
@@ -15596,7 +16431,7 @@ function cmdWriteFile(dest, opts) {
|
|
|
15596
16431
|
try {
|
|
15597
16432
|
atomicWriteBuffer(dest, Buffer.concat(chunks));
|
|
15598
16433
|
enqueueDirtyPathSafe(dest);
|
|
15599
|
-
|
|
16434
|
+
resolve13();
|
|
15600
16435
|
} catch (e) {
|
|
15601
16436
|
try {
|
|
15602
16437
|
mapFsError(e, void 0, dest);
|
|
@@ -15727,14 +16562,14 @@ function writeReplacedBuffer(file, replacedBuf, preWriteStat) {
|
|
|
15727
16562
|
const until = process.env["TOKEN_GOAT_TEST_REPLACE_DELAY_UNTIL"];
|
|
15728
16563
|
const deadline = Date.now() + testDelayMs;
|
|
15729
16564
|
if (until !== void 0 && until !== "") {
|
|
15730
|
-
while (Date.now() < deadline && !
|
|
16565
|
+
while (Date.now() < deadline && !fs28.existsSync(until)) sleepSync(5);
|
|
15731
16566
|
} else {
|
|
15732
16567
|
sleepSync(testDelayMs);
|
|
15733
16568
|
}
|
|
15734
16569
|
}
|
|
15735
16570
|
let preRenameStat;
|
|
15736
16571
|
try {
|
|
15737
|
-
preRenameStat =
|
|
16572
|
+
preRenameStat = fs28.statSync(file);
|
|
15738
16573
|
} catch {
|
|
15739
16574
|
}
|
|
15740
16575
|
if (preRenameStat !== void 0 && (preRenameStat.mtimeMs !== preWriteStat.mtimeMs || preRenameStat.size !== preWriteStat.size)) {
|
|
@@ -15775,7 +16610,7 @@ function cmdReplace(file, opts) {
|
|
|
15775
16610
|
const targetBuf = readFileBoundedRaw(file, "target file", true);
|
|
15776
16611
|
let preWriteStat;
|
|
15777
16612
|
try {
|
|
15778
|
-
preWriteStat =
|
|
16613
|
+
preWriteStat = fs28.statSync(file);
|
|
15779
16614
|
} catch {
|
|
15780
16615
|
}
|
|
15781
16616
|
const usingFrom = opts.oldFrom !== void 0 || opts.newFrom !== void 0;
|
|
@@ -15869,7 +16704,7 @@ function cmdInsertSection(file, opts) {
|
|
|
15869
16704
|
}
|
|
15870
16705
|
let preWriteStat;
|
|
15871
16706
|
try {
|
|
15872
|
-
preWriteStat =
|
|
16707
|
+
preWriteStat = fs28.statSync(file);
|
|
15873
16708
|
} catch {
|
|
15874
16709
|
}
|
|
15875
16710
|
const result = readSection(file, opts.after);
|
|
@@ -15895,7 +16730,7 @@ function cmdInsertSection(file, opts) {
|
|
|
15895
16730
|
}
|
|
15896
16731
|
let rawBytes;
|
|
15897
16732
|
try {
|
|
15898
|
-
rawBytes =
|
|
16733
|
+
rawBytes = fs28.readFileSync(file);
|
|
15899
16734
|
} catch (e) {
|
|
15900
16735
|
mapFsError(e, void 0, file);
|
|
15901
16736
|
}
|
|
@@ -15911,7 +16746,7 @@ function cmdInsertSection(file, opts) {
|
|
|
15911
16746
|
if (preWriteStat !== void 0) {
|
|
15912
16747
|
let preRenameStat;
|
|
15913
16748
|
try {
|
|
15914
|
-
preRenameStat =
|
|
16749
|
+
preRenameStat = fs28.statSync(file);
|
|
15915
16750
|
} catch {
|
|
15916
16751
|
}
|
|
15917
16752
|
if (preRenameStat !== void 0 && (preRenameStat.mtimeMs !== preWriteStat.mtimeMs || preRenameStat.size !== preWriteStat.size)) {
|
|
@@ -15946,42 +16781,32 @@ ${content}`;
|
|
|
15946
16781
|
const sections = await getDocSections(fileId, { fresh: false });
|
|
15947
16782
|
emitted = formatSections(sections);
|
|
15948
16783
|
}
|
|
15949
|
-
|
|
15950
|
-
try {
|
|
15951
|
-
if (loadConfig().injection.enabled) {
|
|
15952
|
-
const matches2 = scanForInjectionPatterns(emitted);
|
|
15953
|
-
if (matches2.length > 0) {
|
|
15954
|
-
recordStat("injection_detected", 0, 0, void 0, matches2.join(","));
|
|
15955
|
-
toEmit = fenceUntrustedContent(emitted, matches2, UNTRUSTED_WEB_TAG);
|
|
15956
|
-
}
|
|
15957
|
-
}
|
|
15958
|
-
} catch {
|
|
15959
|
-
}
|
|
16784
|
+
const toEmit = fenceUntrusted(emitted, UNTRUSTED_WEB_TAG);
|
|
15960
16785
|
out(toEmit);
|
|
15961
16786
|
const bytesSaved = Math.max(1, fullSourceBytes - Buffer.byteLength(toEmit, "utf8"));
|
|
15962
|
-
recordStat("gdrive_sections", bytesSaved,
|
|
16787
|
+
recordStat("gdrive_sections", bytesSaved, savedTokensFromBytes(bytesSaved));
|
|
15963
16788
|
}
|
|
15964
16789
|
function expandGlobs(root, patterns, globFnOverride) {
|
|
15965
16790
|
const out2 = [];
|
|
15966
|
-
const globFn = globFnOverride ??
|
|
16791
|
+
const globFn = globFnOverride ?? fs28["globSync"];
|
|
15967
16792
|
for (const p of patterns) {
|
|
15968
16793
|
if (globFn !== void 0 && (p.includes("*") || p.includes("?") || p.includes("{"))) {
|
|
15969
16794
|
try {
|
|
15970
16795
|
const hits = globFn(p, { cwd: root });
|
|
15971
|
-
for (const h of hits) out2.push(
|
|
16796
|
+
for (const h of hits) out2.push(path28.isAbsolute(h) ? h : path28.join(root, h));
|
|
15972
16797
|
continue;
|
|
15973
16798
|
} catch {
|
|
15974
16799
|
}
|
|
15975
16800
|
}
|
|
15976
|
-
out2.push(
|
|
16801
|
+
out2.push(path28.isAbsolute(p) ? p : path28.join(root, p));
|
|
15977
16802
|
}
|
|
15978
16803
|
return out2;
|
|
15979
16804
|
}
|
|
15980
16805
|
function readIgnoreFile(root) {
|
|
15981
|
-
const ignorePath =
|
|
16806
|
+
const ignorePath = path28.join(root, ".tokengoatignore");
|
|
15982
16807
|
let raw;
|
|
15983
16808
|
try {
|
|
15984
|
-
raw =
|
|
16809
|
+
raw = fs28.readFileSync(ignorePath, "utf8");
|
|
15985
16810
|
} catch {
|
|
15986
16811
|
return void 0;
|
|
15987
16812
|
}
|
|
@@ -16022,14 +16847,14 @@ function cmdPack(patterns, opts) {
|
|
|
16022
16847
|
}
|
|
16023
16848
|
let instruction;
|
|
16024
16849
|
if (opts.instructionFile !== void 0) {
|
|
16025
|
-
instruction =
|
|
16850
|
+
instruction = fs28.readFileSync(opts.instructionFile, "utf8");
|
|
16026
16851
|
}
|
|
16027
16852
|
const formatted = formatPack(result, style, {
|
|
16028
16853
|
...opts.lineNumbers === true ? { line_numbers: true } : {},
|
|
16029
16854
|
...instruction !== void 0 ? { instruction } : {}
|
|
16030
16855
|
});
|
|
16031
16856
|
if (opts.output !== void 0) {
|
|
16032
|
-
|
|
16857
|
+
fs28.writeFileSync(opts.output, formatted, "utf8");
|
|
16033
16858
|
} else {
|
|
16034
16859
|
out(formatted);
|
|
16035
16860
|
}
|
|
@@ -16039,25 +16864,27 @@ function cmdTokens(patterns, opts) {
|
|
|
16039
16864
|
const result = estimateBudget(root, expandGlobs(root, patterns ?? []));
|
|
16040
16865
|
let entries = [...result.entries];
|
|
16041
16866
|
if (opts.asc === true) entries.reverse();
|
|
16867
|
+
const eligibleCount = entries.length;
|
|
16042
16868
|
if (opts.top !== void 0) entries = entries.slice(0, requireNonNegativeInt("--top", opts.top));
|
|
16869
|
+
const truncated = entries.length < eligibleCount;
|
|
16043
16870
|
if (opts.json === true) {
|
|
16044
|
-
out(JSON.stringify({ entries, total_tokens: result.total_tokens, total_lines: result.total_lines }, null, 2));
|
|
16871
|
+
out(JSON.stringify({ entries, truncated, totalCount: eligibleCount, total_tokens: result.total_tokens, total_lines: result.total_lines }, null, 2));
|
|
16045
16872
|
return;
|
|
16046
16873
|
}
|
|
16047
16874
|
if (opts.tree === true) {
|
|
16048
16875
|
const dirs = /* @__PURE__ */ new Map();
|
|
16049
16876
|
for (const e of entries) {
|
|
16050
|
-
const dir =
|
|
16877
|
+
const dir = path28.dirname(e.rel_path);
|
|
16051
16878
|
if (!dirs.has(dir)) dirs.set(dir, []);
|
|
16052
16879
|
dirs.get(dir).push(e);
|
|
16053
16880
|
}
|
|
16054
16881
|
const lines2 = [];
|
|
16055
16882
|
for (const [dir, dirEntries] of dirs) {
|
|
16056
16883
|
const dirTokens = dirEntries.reduce((s, e) => s + e.tokens, 0);
|
|
16057
|
-
const
|
|
16058
|
-
lines2.push(`${dir}/ (${dirTokens} tokens, ${
|
|
16884
|
+
const pct2 = result.total_tokens > 0 ? Math.round(dirTokens / result.total_tokens * 100) : 0;
|
|
16885
|
+
lines2.push(`${dir}/ (${dirTokens} tokens, ${pct2}%)`);
|
|
16059
16886
|
for (const e of dirEntries) {
|
|
16060
|
-
lines2.push(` ${
|
|
16887
|
+
lines2.push(` ${path28.basename(e.rel_path).padEnd(30)} ${String(e.tokens).padStart(8)} tokens`);
|
|
16061
16888
|
}
|
|
16062
16889
|
}
|
|
16063
16890
|
out(lines2.join("\n"));
|
|
@@ -16075,6 +16902,9 @@ function cmdTokens(patterns, opts) {
|
|
|
16075
16902
|
for (const e of entries) {
|
|
16076
16903
|
lines.push(`${e.rel_path.padEnd(colW)} ${String(e.tokens).padStart(8)} ${String(e.lines).padStart(6)}`);
|
|
16077
16904
|
}
|
|
16905
|
+
if (truncated) {
|
|
16906
|
+
lines.push(`...and ${eligibleCount - entries.length} more (raise --top to see them).`);
|
|
16907
|
+
}
|
|
16078
16908
|
out(lines.join("\n"));
|
|
16079
16909
|
}
|
|
16080
16910
|
function cmdBudget(patterns, opts) {
|
|
@@ -16088,7 +16918,7 @@ function cmdBudget(patterns, opts) {
|
|
|
16088
16918
|
}
|
|
16089
16919
|
}
|
|
16090
16920
|
function cmdFailures(src, opts) {
|
|
16091
|
-
const text = src !== void 0 ?
|
|
16921
|
+
const text = src !== void 0 ? fs28.readFileSync(src, "utf8") : fs28.readFileSync(0, "utf8");
|
|
16092
16922
|
const result = extractFailures(text, opts.runner !== void 0 ? { runner: opts.runner } : {});
|
|
16093
16923
|
if (opts.delta !== true) {
|
|
16094
16924
|
out(opts.json === true ? formatFailuresJson(result) : formatFailuresText(result));
|
|
@@ -16290,6 +17120,7 @@ function buildProgram() {
|
|
|
16290
17120
|
program2.command("waste").description("session spend-ledger: token cost per tool/file from the current Claude Code session transcript, plus waste signals").option("--project <path>", "project root to analyze").option("--transcript <path>", "explicit transcript JSONL path (default: most-recently-modified transcript for this project)").option("--top <n>", "number of top expensive tool calls to show (default: 10)").option("--copilot", "analyze a Copilot CLI session event log instead, reporting Copilot's own token split").option("--json", "output JSON").action(guard(cmdWaste));
|
|
16291
17121
|
program2.command("session-outline [session-id-or-path]").description("turn-by-turn structure (role, preview, tool calls, approx size) of a Claude Code session JSONL transcript, instead of a raw Read; defaults to the current project's most recent session").option("--project <path>", "project root to resolve the session transcript against").option("--json", "output JSON").action(guard(cmdSessionOutline));
|
|
16292
17122
|
program2.command("session-slice [session-id-or-path]").description("full content of one turn range from a Claude Code session JSONL transcript (see session-outline for turn numbers), instead of a raw Read").requiredOption("--range <spec>", "turn range, e.g. 5-9 or 12 (see session-outline for turn numbers)").option("--project <path>", "project root to resolve the session transcript against").option("--json", "output JSON").action(guard(cmdSessionSlice));
|
|
17123
|
+
program2.command("session-audit").description("corpus-wide token attribution across every local Claude Code session transcript: measured billed usage, estimated content size by source and by tool, and billed cost by session position (aggregate counts only, never transcript content)").option("--dir <path>", "transcript corpus root to scan (default: ~/.claude/projects)").option("--json", "output JSON").action(guard(cmdSessionAudit));
|
|
16293
17124
|
program2.command("mcp-audit").description("MCP server schema cost-vs-usage report: estimate per-server token cost from cached tool calls").option("--project <path>", "project root to analyze").option("--json", "output JSON").action(guard(cmdMcpAudit));
|
|
16294
17125
|
program2.command("recall [query]").description("search across every cached bash-output, web-output, and mcp-output entry (full-text); with no query, list them newest-first").option("--type <type>", "filter to one cache type: bash, web, or mcp").option("--limit <n>", "max results to return (default: 10)").option("--json", "output JSON").action(guard(cmdRecall));
|
|
16295
17126
|
program2.command("hint-stats").description("per-category efficacy report for token-goat's discretionary hint hooks (emitted/acted-on/suppression)").option("--json", "output JSON").option("--reset", "clear all tracked emissions and manual marks").option("--mark-effective <category>", 'record a manual "effective" vote for a hint category').option("--mark-ineffective <category>", 'record a manual "ineffective" vote for a hint category').action(guard(cmdHintStats));
|
|
@@ -16654,7 +17485,7 @@ function buildProgram() {
|
|
|
16654
17485
|
)
|
|
16655
17486
|
);
|
|
16656
17487
|
program2.command("gdrive-sections <file-id>").description("fetch and list sections from a public Google Doc").option("--heading <name>", "get content of one named section").option("--fresh", "skip the on-disk cache and force a live fetch").action(guard(cmdGdriveSections));
|
|
16657
|
-
program2.command("compress").alias("bash").alias("run").description("run a shell command and emit a compressed view of its output").requiredOption("-c, --cmd <command>", "the shell command to run, as one string").option("-f, --filter <name>", "filter name (auto-detected from the command when omitted)").option("--timeout <seconds>", "wall-clock timeout in seconds (0 = built-in default)").option("--no-compress", "stream output raw without compression (debug the wrapper)").option("--profile <name>", "compression profile: aggressive | balanced | minimal").option("--max-tokens <n>", "post-compress token cap (0 = no cap)").action(cmdCompress);
|
|
17488
|
+
program2.command("compress").alias("bash").alias("run").description("run a shell command (under a POSIX shell / bash) and emit a compressed view of its output").requiredOption("-c, --cmd <command>", "the shell command to run, as one string (use / for paths across platforms)").option("-f, --filter <name>", "filter name (auto-detected from the command when omitted)").option("--timeout <seconds>", "wall-clock timeout in seconds (0 = built-in default)").option("--no-compress", "stream output raw without compression (debug the wrapper)").option("--profile <name>", "compression profile: aggressive | balanced | minimal").option("--max-tokens <n>", "post-compress token cap (0 = no cap)").action(cmdCompress);
|
|
16658
17489
|
program2.command("version").description("print the token-goat version").action(
|
|
16659
17490
|
guard(() => {
|
|
16660
17491
|
out(VERSION);
|