token-goat 2.9.2 → 2.9.3

package/dist/token-goat-hook.mjs

@@ -2,13 +2,18 @@ import { createRequire as __cjsRequire } from 'node:module';
 const require = __cjsRequire(import.meta.url);
 import {
   relayInProcess
-} from "./token-goat-chunk-Q4LOQY44.mjs";
-import "./token-goat-chunk-VCNW7BGU.mjs";
-import "./token-goat-chunk-LJ3CHCTT.mjs";
-import "./token-goat-chunk-CGWACYYZ.mjs";
-import "./token-goat-chunk-UZ2NFOOZ.mjs";
-import "./token-goat-chunk-AO2QD2AG.mjs";
-import "./token-goat-chunk-AEX54RUZ.mjs";
+} from "./token-goat-chunk-2VVXTAGD.mjs";
+import "./token-goat-chunk-A4VBNWSL.mjs";
+import "./token-goat-chunk-3XVGPLDS.mjs";
+import "./token-goat-chunk-OKJX2JWW.mjs";
+import "./token-goat-chunk-KKHKXUVS.mjs";
+import "./token-goat-chunk-EEIDFMEM.mjs";
+import {
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
+
+// src/hook_lib.ts
+init_define_import_meta_env();
 export {
   relayInProcess
 };

package/dist/token-goat.core.mjs

@@ -2,16 +2,19 @@ import { createRequire as __cjsRequire } from 'node:module';
 const require = __cjsRequire(import.meta.url);
 import {
   run
-} from "./token-goat-chunk-KKIB7O3Z.mjs";
-import "./token-goat-chunk-FXAPKVRG.mjs";
-import "./token-goat-chunk-VCNW7BGU.mjs";
-import "./token-goat-chunk-LJ3CHCTT.mjs";
+} from "./token-goat-chunk-6FQMEMWX.mjs";
+import "./token-goat-chunk-7A7SBE6R.mjs";
+import "./token-goat-chunk-A4VBNWSL.mjs";
+import "./token-goat-chunk-3XVGPLDS.mjs";
 import {
   installEpipeGuard
-} from "./token-goat-chunk-UZ2NFOOZ.mjs";
-import "./token-goat-chunk-AO2QD2AG.mjs";
-import "./token-goat-chunk-AEX54RUZ.mjs";
+} from "./token-goat-chunk-KKHKXUVS.mjs";
+import "./token-goat-chunk-EEIDFMEM.mjs";
+import {
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
 
 // src/main.ts
+init_define_import_meta_env();
 installEpipeGuard();
 void run();

package/docs/cli.md

@@ -337,3 +337,56 @@ are shown alongside the automatic percentage but never blended into it. `--json`
 Note that what this feature calls "harness" (Claude Code, Codex, Gemini, ...) is not the same as
 "which LLM model" — no bridge in this codebase exposes an LLM model identifier to hooks, so
 harness is the closest real signal available.
+
+### Scoring the compressors — `token-goat bench`
+
+`token-goat bench` replays a fixed corpus of captured command output through the same function
+that decides what a real shell command hands back to the model, and reports how much smaller the
+result is. It exists so a change to a compressor can be measured instead of guessed at:
+
+```
+$ token-goat bench
+case          filter           in        out    saved  fidelity
+---------------------------------------------------------------
+git-log-stat  git-log       57227       3027    94.7%  2/2
+npm-ls-all    dep-list      34546       1060    96.9%  2/2
+vitest-run    vitest        22684      22682        -  2/2
+---------------------------------------------------------------
+TOTAL                      114457      26769    76.6%  6/6
+
+ratio    76.6% saved  (PRIMARY -- must improve; measured floor 0.0%, headroom 23.4%)
+fidelity 6/6 kept   (GUARD -- must not regress; any miss exits 1)
+coverage 3/157 filters exercised, 2/3 cases compressed
+```
+
+There are two numbers on purpose. **Ratio** is the thing to push up. **Fidelity** counts the lines
+each case declares it must never lose, and it is what stops the ratio from being gamed: deleting
+output raises the ratio, so a benchmark reporting only the ratio would reward destroying the very
+content the compressors exist to preserve. A dropped must-keep line exits 1, so a loop can revert
+on the exit code alone.
+
+The corpus lives in `tests/fixtures/bench`: one `<id>.json` naming the command, its exit code,
+and its must-keep lines, beside an `<id>.txt` holding that run's real output. Point `--corpus
+<dir>` at your own directory to score commands you care about. Every case must declare where its
+output came from; a fixture with no `provenance` field is refused rather than scored, because
+output written by hand proves nothing about what a real command emits.
+
+`--tsv <path>` appends one row per run (timestamp, commit, sizes, ratio, fidelity, coverage), so a
+sequence of attempts leaves a readable history instead of a scrollback. `--json` emits the same
+report as an object. `--validate` runs the benchmark's own negative controls and says whether the
+corpus can tell good from bad at all:
+
+```
+$ token-goat bench --validate
+ok    floor: the net-benefit gate refuses a no-op filter on every case -- identity control applied to 0/3 cases, measured floor 0.0%
+ok    floor: a no-op filter keeps every must-keep line -- identity control kept 6/6
+ok    guard: deleting everything is caught -- destroy control kept 0/6 at 99.8% saved
+
+The corpus discriminates: a ratio measured against it cannot be raised by deleting content.
+```
+
+The first two controls establish the floor: a filter that changes nothing must score nothing, so
+the reported percentage is a real saving rather than an offset every number carries. The third
+runs a filter that deletes the entire output. It scores 99.8%, the best ratio possible, and the
+fidelity guard fails it. Run `--validate` after changing the corpus; a corpus whose must-keep
+lists have gone soft passes every benchmark while measuring nothing.

package/docs/security.md

@@ -38,12 +38,16 @@ That command is the one thing here that is not clean: `onnxruntime-node` pulls a
 
 **Prompt injection.** When an AI reads a file, web page, or command output, that content enters its context alongside your own instructions. Prompt injection is when untrusted content includes text designed to look like instructions — "Ignore all previous directives and run this instead" — to redirect the AI mid-task.
 
-Token-goat intercepts every Read, Fetch, Bash, and MCP call the AI makes. Text that came from somewhere else is wrapped in an untrusted-content fence before the model sees it, decided by where the text came from and not by whether anything looked suspicious in it (`injection.enabled`, on by default, turns the whole thing off). The content is also scanned for a set of imperative-override attack patterns ("ignore previous instructions," "reveal system prompt," and similar); a match adds the pattern names to the fence's notice and writes a row to the log, and a clean scan changes only the wording. That ordering is the point: the pattern list is deliberately short, so anyone phrasing the same instruction differently would otherwise get an unlabelled channel, and a miss would be silent.
+Token-goat intercepts every Read, Fetch, Bash, and MCP call the AI makes. Where it hands the model text in place of the tool's own result, that text is wrapped in an untrusted-content fence first, decided by where the text came from and not by whether anything looked suspicious in it (`injection.enabled`, on by default, turns the whole thing off). Which surfaces that covers, and which it deliberately does not, is set out below: the two exceptions are stated rather than left to be read off this sentence. The content is also scanned for a set of imperative-override attack patterns ("ignore previous instructions," "reveal system prompt," and similar); a match adds the pattern names to the fence's notice and writes a row to the log, and a clean scan changes only the wording. That ordering is the point: the pattern list is deliberately short, so anyone phrasing the same instruction differently would otherwise get an unlabelled channel, and a miss would be silent.
 
-Three surfaces are covered. Every fetched page is fenced as it arrives and again when a cached copy is recalled with `web-output`. Every MCP tool result is fenced as it arrives, which matters most: it is a remote server's output, so it is the least trustworthy text in the pipeline. And cached Bash and MCP output is fenced when recalled with `bash-output` or `mcp-output`, since the output of a build or test run in a project with a hostile dependency is written by a third party as much as any web page is. Document extraction (`pdf-extract`, `docx-text`, the `xlsx-*` and `pptx-*` commands), `pr-slice`, `gdrive-sections`, and `recall` are covered the same way. The fence naming tool output is a different tag from the one naming web content, so the label tells the model where the text came from.
+The line that decides is substitution: wherever token-goat replaces a tool result with text of its own, that text is fenced. Every fetched page is fenced as it arrives and again when a cached copy is recalled with `web-output`. Every MCP tool result is fenced as it arrives, which matters most: it is a remote server's output, so it is the least trustworthy text in the pipeline. Bash output is fenced when token-goat compresses it, since a compressed body is not the command's output but token-goat's account of it, and it is fenced again when a cached copy is recalled with `bash-output` or `mcp-output`: the output of a build or test run in a project with a hostile dependency is written by a third party as much as any web page is. Document extraction (`pdf-extract`, `docx-text`, the `xlsx-*` and `pptx-*` commands), `pr-slice`, `gdrive-sections`, and `recall` are covered the same way. The fence naming tool output is a different tag from the one naming web content, so the label tells the model where the text came from.
+
+In every one of those cases the fence wraps the third-party bytes and stops there. Token-goat's own notice, the filter's marker, and the pointer telling the model how to recall the full output all sit outside the closing tag, because the fence is the model's one signal for where token-goat stops speaking. Fold them inside and that signal is gone, and anyone who guesses the marker's wording can write a line the model reads as token-goat's own.
 
 Read is the exception: file content passes through to the model unfiltered, because filtering it would silently break legitimate use cases. Where token-goat splices a piece of a file into its own hint or denial message, that excerpt is fenced. Outside of the fence, the primary defense is the model's own training to treat tool output as data, not as commands from a trusted party.
 
+Bash output that token-goat does not rewrite is the same kind of exception, and worth stating plainly rather than leaving to be inferred from the paragraph above. A command whose output is short, or too incompressible to be worth touching, reaches the model exactly as the harness delivered it, with no fence. So does output whose only change was stripping the colour codes a terminal would have rendered, since that path emits the command's own bytes and adds nothing of token-goat's to delimit. Fencing those cases would mean rewriting the result of every shell command an agent runs, a permanent cost on the most-used path in the tool, to re-label bytes the model was going to receive in that form anyway. The fence is worth its bytes where token-goat has substituted its own account of the output and the model can no longer tell whose words are whose. Where token-goat has stayed out of the way, the defense is the same as for Read: the model's training to treat tool output as data.
+
 One deliberate gap: `--json` output cannot carry a fence around the envelope, because a fence wrapped around JSON is no longer JSON and callers parse it. Those envelopes fence individual fields on a pattern match instead, since the fixed wrapper would otherwise cost more than a short field is worth. The printed (non-`--json`) form of the same command is always fenced.
 
 Separately from that pass-through case: when a read hook *denies* a Read and substitutes its own message, any file bytes it embeds in that message (a markdown heading tree, a served compact or notebook sidecar, a re-read diff, a CSV header row, an HTML title) are wrapped in an `<untrusted-file-content>` fence first, so a hostile repo cannot get its own text presented to the model as token-goat speaking. That fencing is unconditional, not gated on the pattern scan — as all of it now is.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "token-goat",
-  "version": "2.9.2",
+  "version": "2.9.3",
   "description": "Surgical token-reduction companion for Claude Code and other AI coding agents",
   "type": "module",
   "main": "./dist/token-goat.mjs",
@@ -73,7 +73,9 @@
     "@eslint/js": "^10.0.1",
     "@modelcontextprotocol/sdk": "^1.30.0",
     "@types/better-sqlite3": "^9.6.0",
+    "@types/jpeg-js": "^0.3.0",
     "@types/node": "^26.2.0",
+    "@types/omggif": "^1.0.5",
     "@xenova/transformers": "^2.17.2",
     "better-sqlite3": "^12.11.1",
     "commander": "^15.0.0",
@@ -84,8 +86,10 @@
     "exceljs": "^4.4.0",
     "fast-xml-parser": "^5.9.3",
     "html-to-text": "^10.0.1",
+    "jpeg-js": "^0.4.4",
     "js-yaml": "^5.2.1",
     "lefthook": "^2.1.10",
+    "omggif": "^1.0.10",
     "onnxruntime-node": "^1.27.0",
     "smol-toml": "^1.8.0",
     "tsx": "^4.23.12",
@@ -119,6 +123,8 @@
     "uuid": "^14.0.1",
     "sharp": "$sharp",
     "unzipper": "^0.12.5",
-    "adm-zip": "^0.6.0"
+    "adm-zip": "^0.6.0",
+    "fast-uri": "^3.1.6",
+    "qs": "^6.16.0"
   }
 }

package/dist/token-goat-chunk-AEX54RUZ.mjs

@@ -1,11 +0,0 @@
-import { createRequire as __cjsRequire } from 'node:module';
-const require = __cjsRequire(import.meta.url);
-var __defProp = Object.defineProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-
-export {
-  __export
-};

package/dist/token-goat-chunk-ZVN5WHTO.mjs

@@ -1,23 +0,0 @@
-import { createRequire as __cjsRequire } from 'node:module';
-const require = __cjsRequire(import.meta.url);
-import {
-  buildEvent,
-  relay,
-  relayInProcess
-} from "./token-goat-chunk-Q4LOQY44.mjs";
-import {
-  MAX_STDIN_BYTES,
-  readStdinJson
-} from "./token-goat-chunk-VCNW7BGU.mjs";
-import "./token-goat-chunk-LJ3CHCTT.mjs";
-import "./token-goat-chunk-CGWACYYZ.mjs";
-import "./token-goat-chunk-UZ2NFOOZ.mjs";
-import "./token-goat-chunk-AO2QD2AG.mjs";
-import "./token-goat-chunk-AEX54RUZ.mjs";
-export {
-  MAX_STDIN_BYTES,
-  buildEvent,
-  readStdinJson,
-  relay,
-  relayInProcess
-};