token-diet-cli 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/LICENSE +1 -1
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/PKG-INFO +77 -11
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/README.md +75 -9
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/pyproject.toml +2 -2
- token_diet_cli-0.2.0/src/token_diet/__init__.py +80 -0
- token_diet_cli-0.2.0/src/token_diet/baseline.py +120 -0
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet/cli.py +224 -43
- token_diet_cli-0.2.0/src/token_diet/git_hooks.py +149 -0
- token_diet_cli-0.2.0/src/token_diet/mcp_audit.py +181 -0
- token_diet_cli-0.2.0/src/token_diet/metrics.py +31 -0
- token_diet_cli-0.2.0/src/token_diet/sections.py +139 -0
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/PKG-INFO +77 -11
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/SOURCES.txt +9 -0
- token_diet_cli-0.2.0/tests/test_baseline.py +213 -0
- token_diet_cli-0.2.0/tests/test_git_hooks.py +298 -0
- token_diet_cli-0.2.0/tests/test_mcp_audit.py +352 -0
- token_diet_cli-0.2.0/tests/test_sections.py +257 -0
- token_diet_cli-0.2.0/tests/test_token_audit.py +369 -0
- token_diet_cli-0.1.0/src/token_diet/__init__.py +0 -38
- token_diet_cli-0.1.0/tests/test_token_audit.py +0 -192
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/setup.cfg +0 -0
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet/__main__.py +0 -0
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/dependency_links.txt +0 -0
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/entry_points.txt +0 -0
- {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/top_level.txt +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: token-diet-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows
|
|
5
|
-
Author-email:
|
|
5
|
+
Author-email: joseoper <joseoper@users.noreply.github.com>
|
|
6
6
|
License: MIT
|
|
7
7
|
Keywords: tokens,llm,claude,gemini,codex,opencode,developer-tools
|
|
8
8
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -37,8 +37,10 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
|
|
|
37
37
|
|
|
38
38
|
`token-diet` scans your repository or workstation in milliseconds to:
|
|
39
39
|
- **Audit prompt size:** Flags instruction files that exceed context budgets (`[OK]`, `[WARN]`, `[ALERT]`).
|
|
40
|
-
- **Catch rogue git hooks:** Detects and disables silent LLM review hooks in
|
|
41
|
-
- **
|
|
40
|
+
- **Catch rogue git hooks:** Detects and disables silent LLM review hooks in the active hooks directory (`--fix-hooks`), including linked worktrees, submodules, and repositories using `core.hooksPath`.
|
|
41
|
+
- **Locate the bloat:** Breaks a Markdown directive down per level-2 heading so you know which section to trim (`--sections`).
|
|
42
|
+
- **Audit MCP overhead:** Reports the context cost of every declared MCP server in `opencode.json`, `.mcp.json`, and `mcp.json` (`--mcp`).
|
|
43
|
+
- **Guard CI pipelines:** Fails PR builds if prompt expansions exceed token thresholds or regress past a saved baseline (`--strict`, `--max-tokens`, `--baseline`, `--max-delta`).
|
|
42
44
|
- **Zero dependencies:** Pure Python standard library (Python >= 3.8). Runs in < 10ms.
|
|
43
45
|
|
|
44
46
|
---
|
|
@@ -47,9 +49,9 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
|
|
|
47
49
|
|
|
48
50
|
### Via Homebrew (macOS & Linux)
|
|
49
51
|
```bash
|
|
50
|
-
brew install
|
|
52
|
+
brew install joseoper/tap/token-diet
|
|
51
53
|
# Or:
|
|
52
|
-
brew tap
|
|
54
|
+
brew tap joseoper/tap
|
|
53
55
|
brew install token-diet
|
|
54
56
|
```
|
|
55
57
|
|
|
@@ -69,7 +71,7 @@ uv tool install token-diet-cli
|
|
|
69
71
|
|
|
70
72
|
### Running Directly from Source (No Installation Needed)
|
|
71
73
|
```bash
|
|
72
|
-
git clone https://github.com/
|
|
74
|
+
git clone https://github.com/joseoper/token-diet.git
|
|
73
75
|
cd token-diet
|
|
74
76
|
PYTHONPATH=src python3 -m token_diet
|
|
75
77
|
```
|
|
@@ -179,6 +181,60 @@ Emit structured JSON for automated pipelines, threshold assertions, or telemetry
|
|
|
179
181
|
token-diet --json
|
|
180
182
|
```
|
|
181
183
|
|
|
184
|
+
### 8. Locate Prompt Bloat per Heading (`--sections`)
|
|
185
|
+
A monolithic directive rarely bloats evenly. This breakdown attributes bytes and estimated tokens to every level-2 (`##`) heading, so you know exactly which section to compress or move into an on-demand skill:
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
token-diet --sections AGENTS.md
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
```
|
|
192
|
+
SECTION BREAKDOWN: MARKDOWN HEADING FOOTPRINT
|
|
193
|
+
================================================================================
|
|
194
|
+
AGENTS.md (23,914 B / ~5,978 tokens)
|
|
195
|
+
Line | Heading | Bytes | Tokens | Status
|
|
196
|
+
--------------------------------------------------------------------------------
|
|
197
|
+
1 | (preamble) | 412 | 103 | [OK]
|
|
198
|
+
12 | Core Directives | 19,220 | 4,805 | [ALERT]
|
|
199
|
+
98 | Additional Protocols | 4,282 | 1,070 | [WARN]
|
|
200
|
+
--------------------------------------------------------------------------------
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Content before the first `##` heading is grouped as `(preamble)`, and deeper headings (`###`) stay attributed to their parent section. Combine with `--json` for machine-readable output.
|
|
204
|
+
|
|
205
|
+
### 9. Audit MCP Server Overhead (`--mcp`)
|
|
206
|
+
Every declared MCP server consumes context, and some configs inline large tool schemas. `token-diet` discovers local `opencode.json`, `.mcp.json`, `mcp.json`, `.cursor/mcp.json`, and `.vscode/mcp.json` files and reports the payload cost per server:
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
token-diet --mcp
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
```
|
|
213
|
+
MCP SERVER PAYLOAD AUDIT
|
|
214
|
+
================================================================================
|
|
215
|
+
opencode.json [opencode] (2 server(s))
|
|
216
|
+
Server | Transport | Tools | Bytes | Tokens | Status
|
|
217
|
+
--------------------------------------------------------------------------------
|
|
218
|
+
engram | local | 12 | 18,430 | 4,607 | [ALERT]
|
|
219
|
+
web-search | remote | 0 | 142 | 35 | [OK]
|
|
220
|
+
--------------------------------------------------------------------------------
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
Config-declared `tools` blocks are counted and included in each server's payload. Schemas fetched by a server at runtime are not visible to a local audit; the reported figure is the local injection cost floor.
|
|
224
|
+
|
|
225
|
+
### 10. Fail on Regression vs a Saved Baseline (`--baseline`, `--max-delta`)
|
|
226
|
+
Record a baseline once, then gate every pull request against it. This catches slow, incremental growth that a fixed threshold never notices:
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
# Save the baseline on your default branch
|
|
230
|
+
token-diet --json > token-baseline.json
|
|
231
|
+
|
|
232
|
+
# Fail if the total grows more than 5% in a pull request
|
|
233
|
+
token-diet --baseline token-baseline.json --max-delta 5
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
`--baseline` accepts any `--json` report (`{"summary": {"total_tokens": N}}`) or a minimal `{"total_tokens": N}` file. `--max-delta` exits with code `1` only when tokens **increase** beyond the given percentage; a decrease, an unchanged total, or growth exactly at the budget passes.
|
|
237
|
+
|
|
182
238
|
---
|
|
183
239
|
|
|
184
240
|
## Health Status Indicators
|
|
@@ -188,7 +244,7 @@ token-diet --json
|
|
|
188
244
|
| 🟢 `[OK]` | `< 4 KB` | `< 1,000 tokens` | **Optimal Context:** Fits cleanly within prompt cache. Preserves fast, cheap inference. |
|
|
189
245
|
| 🟡 `[WARN]` | `4 KB – 16 KB` | `1,000 – 4,000 tokens` | **Moderate Size:** Acceptable, but review for duplicate instructions or unneeded verbosity. |
|
|
190
246
|
| 🔴 `[ALERT]` | `> 16 KB` | `> 4,000 tokens` | **Severe Prompt Bloat:** Monolithic rules degrade reasoning and waste context budget every turn. Decouple into On-Demand Skills. |
|
|
191
|
-
| 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:**
|
|
247
|
+
| 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:** The active `pre-commit` hook sends code to cloud LLMs on every `git commit`. Replace with static linters. |
|
|
192
248
|
|
|
193
249
|
---
|
|
194
250
|
|
|
@@ -256,9 +312,18 @@ token-diet/
|
|
|
256
312
|
│ └── token_diet/ # Core Python package (Zero external dependencies)
|
|
257
313
|
│ ├── __init__.py
|
|
258
314
|
│ ├── __main__.py
|
|
259
|
-
│
|
|
315
|
+
│ ├── baseline.py # Baseline comparison & regression budget gate
|
|
316
|
+
│ ├── cli.py # CLI implementation and footprint engine
|
|
317
|
+
│ ├── git_hooks.py # Worktree, gitdir and core.hooksPath resolution
|
|
318
|
+
│ ├── mcp_audit.py # MCP config discovery & payload accounting
|
|
319
|
+
│ ├── metrics.py # Shared byte thresholds and token estimation
|
|
320
|
+
│ └── sections.py # Per-heading Markdown footprint breakdown
|
|
260
321
|
├── tests/
|
|
261
|
-
│
|
|
322
|
+
│ ├── test_baseline.py # Baseline budget unit tests
|
|
323
|
+
│ ├── test_git_hooks.py # Hook resolution unit tests
|
|
324
|
+
│ ├── test_mcp_audit.py # MCP payload unit tests
|
|
325
|
+
│ ├── test_sections.py # Section breakdown unit tests
|
|
326
|
+
│ └── test_token_audit.py # Core engine and CLI integration tests
|
|
262
327
|
├── .gitignore
|
|
263
328
|
├── AGENTS.md # Agent-friendly repository directives
|
|
264
329
|
├── CONTRIBUTING.md # Contribution guidelines & engineering principles
|
|
@@ -269,7 +334,7 @@ token-diet/
|
|
|
269
334
|
|
|
270
335
|
---
|
|
271
336
|
|
|
272
|
-
## Core Methodology: The
|
|
337
|
+
## Core Methodology: The 6 Vectors of Token Leakage
|
|
273
338
|
|
|
274
339
|
Token Diet is built on empirical analysis of how developer environments silently deplete API quotas:
|
|
275
340
|
|
|
@@ -278,6 +343,7 @@ Token Diet is built on empirical analysis of how developer environments silently
|
|
|
278
343
|
3. **Reasoning Effort Inflation:** Running frontier models at `high` reasoning effort for simple syntax or diagnostic queries generates thousands of hidden chain-of-thought tokens. *Solution: Default to `medium` or `low`.*
|
|
279
344
|
4. **Static MCP Schemas:** Registering dozens of MCP tools serializes massive JSON parameter schemas into system prompts. *Solution: Prune unused tool servers and enforce lazy-loading.*
|
|
280
345
|
5. **Prompt Cache Invalidation:** Injecting dynamic timestamps or changing session IDs at the start of instructions destroys provider prompt caching discounts (up to 90%). *Solution: Maintain immutable instruction prefixes.*
|
|
346
|
+
6. **In-Session Runtime Inefficiencies:** Agents loading entire 1,500-line files to check one function, overwriting whole files instead of focused diffs, or running uncompacted 100k+ token threads. *Solution: Enforce active runtime hygiene (grep-first, line-range reads, targeted diffs, and context compaction thresholds; see [Remediation Playbook Phase 4](docs/remediation-playbook.md#phase-4-in-session-runtime-hygiene-active-execution-guards)).*
|
|
281
347
|
|
|
282
348
|
---
|
|
283
349
|
|
|
@@ -14,8 +14,10 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
|
|
|
14
14
|
|
|
15
15
|
`token-diet` scans your repository or workstation in milliseconds to:
|
|
16
16
|
- **Audit prompt size:** Flags instruction files that exceed context budgets (`[OK]`, `[WARN]`, `[ALERT]`).
|
|
17
|
-
- **Catch rogue git hooks:** Detects and disables silent LLM review hooks in
|
|
18
|
-
- **
|
|
17
|
+
- **Catch rogue git hooks:** Detects and disables silent LLM review hooks in the active hooks directory (`--fix-hooks`), including linked worktrees, submodules, and repositories using `core.hooksPath`.
|
|
18
|
+
- **Locate the bloat:** Breaks a Markdown directive down per level-2 heading so you know which section to trim (`--sections`).
|
|
19
|
+
- **Audit MCP overhead:** Reports the context cost of every declared MCP server in `opencode.json`, `.mcp.json`, and `mcp.json` (`--mcp`).
|
|
20
|
+
- **Guard CI pipelines:** Fails PR builds if prompt expansions exceed token thresholds or regress past a saved baseline (`--strict`, `--max-tokens`, `--baseline`, `--max-delta`).
|
|
19
21
|
- **Zero dependencies:** Pure Python standard library (Python >= 3.8). Runs in < 10ms.
|
|
20
22
|
|
|
21
23
|
---
|
|
@@ -24,9 +26,9 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
|
|
|
24
26
|
|
|
25
27
|
### Via Homebrew (macOS & Linux)
|
|
26
28
|
```bash
|
|
27
|
-
brew install
|
|
29
|
+
brew install joseoper/tap/token-diet
|
|
28
30
|
# Or:
|
|
29
|
-
brew tap
|
|
31
|
+
brew tap joseoper/tap
|
|
30
32
|
brew install token-diet
|
|
31
33
|
```
|
|
32
34
|
|
|
@@ -46,7 +48,7 @@ uv tool install token-diet-cli
|
|
|
46
48
|
|
|
47
49
|
### Running Directly from Source (No Installation Needed)
|
|
48
50
|
```bash
|
|
49
|
-
git clone https://github.com/
|
|
51
|
+
git clone https://github.com/joseoper/token-diet.git
|
|
50
52
|
cd token-diet
|
|
51
53
|
PYTHONPATH=src python3 -m token_diet
|
|
52
54
|
```
|
|
@@ -156,6 +158,60 @@ Emit structured JSON for automated pipelines, threshold assertions, or telemetry
|
|
|
156
158
|
token-diet --json
|
|
157
159
|
```
|
|
158
160
|
|
|
161
|
+
### 8. Locate Prompt Bloat per Heading (`--sections`)
|
|
162
|
+
A monolithic directive rarely bloats evenly. This breakdown attributes bytes and estimated tokens to every level-2 (`##`) heading, so you know exactly which section to compress or move into an on-demand skill:
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
token-diet --sections AGENTS.md
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
SECTION BREAKDOWN: MARKDOWN HEADING FOOTPRINT
|
|
170
|
+
================================================================================
|
|
171
|
+
AGENTS.md (23,914 B / ~5,978 tokens)
|
|
172
|
+
Line | Heading | Bytes | Tokens | Status
|
|
173
|
+
--------------------------------------------------------------------------------
|
|
174
|
+
1 | (preamble) | 412 | 103 | [OK]
|
|
175
|
+
12 | Core Directives | 19,220 | 4,805 | [ALERT]
|
|
176
|
+
98 | Additional Protocols | 4,282 | 1,070 | [WARN]
|
|
177
|
+
--------------------------------------------------------------------------------
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Content before the first `##` heading is grouped as `(preamble)`, and deeper headings (`###`) stay attributed to their parent section. Combine with `--json` for machine-readable output.
|
|
181
|
+
|
|
182
|
+
### 9. Audit MCP Server Overhead (`--mcp`)
|
|
183
|
+
Every declared MCP server consumes context, and some configs inline large tool schemas. `token-diet` discovers local `opencode.json`, `.mcp.json`, `mcp.json`, `.cursor/mcp.json`, and `.vscode/mcp.json` files and reports the payload cost per server:
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
token-diet --mcp
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
MCP SERVER PAYLOAD AUDIT
|
|
191
|
+
================================================================================
|
|
192
|
+
opencode.json [opencode] (2 server(s))
|
|
193
|
+
Server | Transport | Tools | Bytes | Tokens | Status
|
|
194
|
+
--------------------------------------------------------------------------------
|
|
195
|
+
engram | local | 12 | 18,430 | 4,607 | [ALERT]
|
|
196
|
+
web-search | remote | 0 | 142 | 35 | [OK]
|
|
197
|
+
--------------------------------------------------------------------------------
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Config-declared `tools` blocks are counted and included in each server's payload. Schemas fetched by a server at runtime are not visible to a local audit; the reported figure is the local injection cost floor.
|
|
201
|
+
|
|
202
|
+
### 10. Fail on Regression vs a Saved Baseline (`--baseline`, `--max-delta`)
|
|
203
|
+
Record a baseline once, then gate every pull request against it. This catches slow, incremental growth that a fixed threshold never notices:
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
# Save the baseline on your default branch
|
|
207
|
+
token-diet --json > token-baseline.json
|
|
208
|
+
|
|
209
|
+
# Fail if the total grows more than 5% in a pull request
|
|
210
|
+
token-diet --baseline token-baseline.json --max-delta 5
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
`--baseline` accepts any `--json` report (`{"summary": {"total_tokens": N}}`) or a minimal `{"total_tokens": N}` file. `--max-delta` exits with code `1` only when tokens **increase** beyond the given percentage; a decrease, an unchanged total, or growth exactly at the budget passes.
|
|
214
|
+
|
|
159
215
|
---
|
|
160
216
|
|
|
161
217
|
## Health Status Indicators
|
|
@@ -165,7 +221,7 @@ token-diet --json
|
|
|
165
221
|
| 🟢 `[OK]` | `< 4 KB` | `< 1,000 tokens` | **Optimal Context:** Fits cleanly within prompt cache. Preserves fast, cheap inference. |
|
|
166
222
|
| 🟡 `[WARN]` | `4 KB – 16 KB` | `1,000 – 4,000 tokens` | **Moderate Size:** Acceptable, but review for duplicate instructions or unneeded verbosity. |
|
|
167
223
|
| 🔴 `[ALERT]` | `> 16 KB` | `> 4,000 tokens` | **Severe Prompt Bloat:** Monolithic rules degrade reasoning and waste context budget every turn. Decouple into On-Demand Skills. |
|
|
168
|
-
| 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:**
|
|
224
|
+
| 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:** The active `pre-commit` hook sends code to cloud LLMs on every `git commit`. Replace with static linters. |
|
|
169
225
|
|
|
170
226
|
---
|
|
171
227
|
|
|
@@ -233,9 +289,18 @@ token-diet/
|
|
|
233
289
|
│ └── token_diet/ # Core Python package (Zero external dependencies)
|
|
234
290
|
│ ├── __init__.py
|
|
235
291
|
│ ├── __main__.py
|
|
236
|
-
│
|
|
292
|
+
│ ├── baseline.py # Baseline comparison & regression budget gate
|
|
293
|
+
│ ├── cli.py # CLI implementation and footprint engine
|
|
294
|
+
│ ├── git_hooks.py # Worktree, gitdir and core.hooksPath resolution
|
|
295
|
+
│ ├── mcp_audit.py # MCP config discovery & payload accounting
|
|
296
|
+
│ ├── metrics.py # Shared byte thresholds and token estimation
|
|
297
|
+
│ └── sections.py # Per-heading Markdown footprint breakdown
|
|
237
298
|
├── tests/
|
|
238
|
-
│
|
|
299
|
+
│ ├── test_baseline.py # Baseline budget unit tests
|
|
300
|
+
│ ├── test_git_hooks.py # Hook resolution unit tests
|
|
301
|
+
│ ├── test_mcp_audit.py # MCP payload unit tests
|
|
302
|
+
│ ├── test_sections.py # Section breakdown unit tests
|
|
303
|
+
│ └── test_token_audit.py # Core engine and CLI integration tests
|
|
239
304
|
├── .gitignore
|
|
240
305
|
├── AGENTS.md # Agent-friendly repository directives
|
|
241
306
|
├── CONTRIBUTING.md # Contribution guidelines & engineering principles
|
|
@@ -246,7 +311,7 @@ token-diet/
|
|
|
246
311
|
|
|
247
312
|
---
|
|
248
313
|
|
|
249
|
-
## Core Methodology: The
|
|
314
|
+
## Core Methodology: The 6 Vectors of Token Leakage
|
|
250
315
|
|
|
251
316
|
Token Diet is built on empirical analysis of how developer environments silently deplete API quotas:
|
|
252
317
|
|
|
@@ -255,6 +320,7 @@ Token Diet is built on empirical analysis of how developer environments silently
|
|
|
255
320
|
3. **Reasoning Effort Inflation:** Running frontier models at `high` reasoning effort for simple syntax or diagnostic queries generates thousands of hidden chain-of-thought tokens. *Solution: Default to `medium` or `low`.*
|
|
256
321
|
4. **Static MCP Schemas:** Registering dozens of MCP tools serializes massive JSON parameter schemas into system prompts. *Solution: Prune unused tool servers and enforce lazy-loading.*
|
|
257
322
|
5. **Prompt Cache Invalidation:** Injecting dynamic timestamps or changing session IDs at the start of instructions destroys provider prompt caching discounts (up to 90%). *Solution: Maintain immutable instruction prefixes.*
|
|
323
|
+
6. **In-Session Runtime Inefficiencies:** Agents loading entire 1,500-line files to check one function, overwriting whole files instead of focused diffs, or running uncompacted 100k+ token threads. *Solution: Enforce active runtime hygiene (grep-first, line-range reads, targeted diffs, and context compaction thresholds; see [Remediation Playbook Phase 4](docs/remediation-playbook.md#phase-4-in-session-runtime-hygiene-active-execution-guards)).*
|
|
258
324
|
|
|
259
325
|
---
|
|
260
326
|
|
|
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "token-diet-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.8"
|
|
11
11
|
license = { text = "MIT" }
|
|
12
12
|
authors = [
|
|
13
|
-
{ name = "
|
|
13
|
+
{ name = "joseoper", email = "joseoper@users.noreply.github.com" }
|
|
14
14
|
]
|
|
15
15
|
keywords = ["tokens", "llm", "claude", "gemini", "codex", "opencode", "developer-tools"]
|
|
16
16
|
classifiers = [
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""
|
|
2
|
+
token-diet: Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from token_diet.cli import (
|
|
6
|
+
estimate_tokens,
|
|
7
|
+
audit_file,
|
|
8
|
+
detect_local_prompts,
|
|
9
|
+
audit_git_hooks,
|
|
10
|
+
fix_git_hooks,
|
|
11
|
+
classify_size,
|
|
12
|
+
resolve_target_paths,
|
|
13
|
+
audit_known_prompts,
|
|
14
|
+
summarize_footprint,
|
|
15
|
+
main,
|
|
16
|
+
DEFAULT_PROMPT_PATHS,
|
|
17
|
+
LOCAL_PROMPT_CANDIDATES,
|
|
18
|
+
THRESHOLD_WARN_BYTES,
|
|
19
|
+
THRESHOLD_ALERT_BYTES,
|
|
20
|
+
)
|
|
21
|
+
from token_diet.baseline import (
|
|
22
|
+
check_budget,
|
|
23
|
+
compare_tokens,
|
|
24
|
+
extract_total_tokens,
|
|
25
|
+
load_baseline,
|
|
26
|
+
)
|
|
27
|
+
from token_diet.git_hooks import (
|
|
28
|
+
AI_HOOK_KEYWORDS,
|
|
29
|
+
has_ai_invocation,
|
|
30
|
+
read_core_hooks_path,
|
|
31
|
+
resolve_common_dir,
|
|
32
|
+
resolve_git_dir,
|
|
33
|
+
resolve_hooks_dir,
|
|
34
|
+
)
|
|
35
|
+
from token_diet.mcp_audit import (
|
|
36
|
+
MCP_CONFIG_CANDIDATES,
|
|
37
|
+
audit_mcp_config,
|
|
38
|
+
audit_mcp_overhead,
|
|
39
|
+
discover_mcp_configs,
|
|
40
|
+
)
|
|
41
|
+
from token_diet.sections import (
|
|
42
|
+
audit_sections,
|
|
43
|
+
audit_sections_for_paths,
|
|
44
|
+
split_sections,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
__version__ = "0.2.0"
|
|
48
|
+
__all__ = [
|
|
49
|
+
"estimate_tokens",
|
|
50
|
+
"audit_file",
|
|
51
|
+
"detect_local_prompts",
|
|
52
|
+
"audit_git_hooks",
|
|
53
|
+
"fix_git_hooks",
|
|
54
|
+
"classify_size",
|
|
55
|
+
"resolve_target_paths",
|
|
56
|
+
"audit_known_prompts",
|
|
57
|
+
"summarize_footprint",
|
|
58
|
+
"main",
|
|
59
|
+
"DEFAULT_PROMPT_PATHS",
|
|
60
|
+
"LOCAL_PROMPT_CANDIDATES",
|
|
61
|
+
"THRESHOLD_WARN_BYTES",
|
|
62
|
+
"THRESHOLD_ALERT_BYTES",
|
|
63
|
+
"AI_HOOK_KEYWORDS",
|
|
64
|
+
"has_ai_invocation",
|
|
65
|
+
"read_core_hooks_path",
|
|
66
|
+
"resolve_common_dir",
|
|
67
|
+
"resolve_git_dir",
|
|
68
|
+
"resolve_hooks_dir",
|
|
69
|
+
"check_budget",
|
|
70
|
+
"compare_tokens",
|
|
71
|
+
"extract_total_tokens",
|
|
72
|
+
"load_baseline",
|
|
73
|
+
"MCP_CONFIG_CANDIDATES",
|
|
74
|
+
"audit_mcp_config",
|
|
75
|
+
"audit_mcp_overhead",
|
|
76
|
+
"discover_mcp_configs",
|
|
77
|
+
"audit_sections",
|
|
78
|
+
"audit_sections_for_paths",
|
|
79
|
+
"split_sections",
|
|
80
|
+
]
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""
|
|
2
|
+
baseline.py
|
|
3
|
+
|
|
4
|
+
Baseline comparison and regression budget gatekeeper logic for token-diet.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import os
|
|
10
|
+
from typing import Any, Dict, Optional
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def extract_total_tokens(payload: Any) -> Optional[int]:
|
|
14
|
+
"""
|
|
15
|
+
Extract total estimated tokens from a numeric value or report dictionary.
|
|
16
|
+
|
|
17
|
+
Returns an integer token count if valid and non-negative, else None.
|
|
18
|
+
Rejects booleans, negative numbers, and non-numeric values.
|
|
19
|
+
"""
|
|
20
|
+
if isinstance(payload, bool):
|
|
21
|
+
return None
|
|
22
|
+
if isinstance(payload, (int, float)):
|
|
23
|
+
if math.isnan(payload) or math.isinf(payload):
|
|
24
|
+
return None
|
|
25
|
+
if payload < 0:
|
|
26
|
+
return None
|
|
27
|
+
return int(payload)
|
|
28
|
+
if isinstance(payload, dict):
|
|
29
|
+
token_val = payload.get("total_tokens")
|
|
30
|
+
if token_val is None and isinstance(payload.get("summary"), dict):
|
|
31
|
+
token_val = payload["summary"].get("total_tokens")
|
|
32
|
+
return extract_total_tokens(token_val)
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load_baseline(filepath: str) -> Dict[str, Any]:
|
|
37
|
+
"""
|
|
38
|
+
Load baseline token count from a JSON report file.
|
|
39
|
+
|
|
40
|
+
Returns a dict with 'ok', 'tokens', 'path', and 'error'.
|
|
41
|
+
Never raises an exception.
|
|
42
|
+
"""
|
|
43
|
+
if not os.path.exists(filepath):
|
|
44
|
+
return {
|
|
45
|
+
"ok": False,
|
|
46
|
+
"tokens": None,
|
|
47
|
+
"path": filepath,
|
|
48
|
+
"error": f"Baseline file not found: {filepath}",
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
with open(filepath, "r", encoding="utf-8") as f:
|
|
53
|
+
data = json.load(f)
|
|
54
|
+
except Exception as e:
|
|
55
|
+
return {
|
|
56
|
+
"ok": False,
|
|
57
|
+
"tokens": None,
|
|
58
|
+
"path": filepath,
|
|
59
|
+
"error": f"Failed to parse baseline JSON: {e}",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
tokens = extract_total_tokens(data)
|
|
63
|
+
if tokens is None:
|
|
64
|
+
return {
|
|
65
|
+
"ok": False,
|
|
66
|
+
"tokens": None,
|
|
67
|
+
"path": filepath,
|
|
68
|
+
"error": "No valid token count found in baseline file",
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return {
|
|
72
|
+
"ok": True,
|
|
73
|
+
"tokens": tokens,
|
|
74
|
+
"path": filepath,
|
|
75
|
+
"error": None,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def compare_tokens(current_tokens: int, baseline_tokens: int) -> Dict[str, Any]:
|
|
80
|
+
"""
|
|
81
|
+
Compare current tokens against baseline tokens and compute delta metrics.
|
|
82
|
+
|
|
83
|
+
Returns a dict with baseline_tokens, current_tokens, delta_tokens,
|
|
84
|
+
delta_percent (rounded to 2 decimal places), and direction.
|
|
85
|
+
"""
|
|
86
|
+
delta_tokens = current_tokens - baseline_tokens
|
|
87
|
+
if baseline_tokens > 0:
|
|
88
|
+
delta_percent = round(delta_tokens / baseline_tokens * 100.0, 2)
|
|
89
|
+
else:
|
|
90
|
+
delta_percent = 0.0
|
|
91
|
+
|
|
92
|
+
if delta_tokens > 0:
|
|
93
|
+
direction = "increase"
|
|
94
|
+
elif delta_tokens < 0:
|
|
95
|
+
direction = "decrease"
|
|
96
|
+
else:
|
|
97
|
+
direction = "unchanged"
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
"baseline_tokens": baseline_tokens,
|
|
101
|
+
"current_tokens": current_tokens,
|
|
102
|
+
"delta_tokens": delta_tokens,
|
|
103
|
+
"delta_percent": delta_percent,
|
|
104
|
+
"direction": direction,
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def check_budget(current_tokens: int, baseline_tokens: int, max_delta_percent: float) -> Dict[str, Any]:
|
|
109
|
+
"""
|
|
110
|
+
Check if token growth exceeds the maximum allowed delta percentage budget.
|
|
111
|
+
|
|
112
|
+
Budget is exceeded only when direction is 'increase' and delta_percent
|
|
113
|
+
is strictly greater than max_delta_percent.
|
|
114
|
+
"""
|
|
115
|
+
comparison = compare_tokens(current_tokens, baseline_tokens)
|
|
116
|
+
max_delta = float(max_delta_percent)
|
|
117
|
+
exceeded = comparison["direction"] == "increase" and comparison["delta_percent"] > max_delta
|
|
118
|
+
comparison["max_delta_percent"] = max_delta
|
|
119
|
+
comparison["exceeded"] = exceeded
|
|
120
|
+
return comparison
|