token-diet-cli 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/LICENSE +1 -1
  2. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/PKG-INFO +77 -11
  3. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/README.md +75 -9
  4. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/pyproject.toml +2 -2
  5. token_diet_cli-0.2.0/src/token_diet/__init__.py +80 -0
  6. token_diet_cli-0.2.0/src/token_diet/baseline.py +120 -0
  7. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet/cli.py +224 -43
  8. token_diet_cli-0.2.0/src/token_diet/git_hooks.py +149 -0
  9. token_diet_cli-0.2.0/src/token_diet/mcp_audit.py +181 -0
  10. token_diet_cli-0.2.0/src/token_diet/metrics.py +31 -0
  11. token_diet_cli-0.2.0/src/token_diet/sections.py +139 -0
  12. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/PKG-INFO +77 -11
  13. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/SOURCES.txt +9 -0
  14. token_diet_cli-0.2.0/tests/test_baseline.py +213 -0
  15. token_diet_cli-0.2.0/tests/test_git_hooks.py +298 -0
  16. token_diet_cli-0.2.0/tests/test_mcp_audit.py +352 -0
  17. token_diet_cli-0.2.0/tests/test_sections.py +257 -0
  18. token_diet_cli-0.2.0/tests/test_token_audit.py +369 -0
  19. token_diet_cli-0.1.0/src/token_diet/__init__.py +0 -38
  20. token_diet_cli-0.1.0/tests/test_token_audit.py +0 -192
  21. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/setup.cfg +0 -0
  22. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet/__main__.py +0 -0
  23. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/dependency_links.txt +0 -0
  24. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/entry_points.txt +0 -0
  25. {token_diet_cli-0.1.0 → token_diet_cli-0.2.0}/src/token_diet_cli.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  MIT License
2
2
 
3
- Copyright (c) 2026 josephrotts
3
+ Copyright (c) 2026 joseoper
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -1,8 +1,8 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: token-diet-cli
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows
5
- Author-email: josephrotts <387283+josephrotts@users.noreply.github.com>
5
+ Author-email: joseoper <joseoper@users.noreply.github.com>
6
6
  License: MIT
7
7
  Keywords: tokens,llm,claude,gemini,codex,opencode,developer-tools
8
8
  Classifier: Development Status :: 4 - Beta
@@ -37,8 +37,10 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
37
37
 
38
38
  `token-diet` scans your repository or workstation in milliseconds to:
39
39
  - **Audit prompt size:** Flags instruction files that exceed context budgets (`[OK]`, `[WARN]`, `[ALERT]`).
40
- - **Catch rogue git hooks:** Detects and disables silent LLM review hooks in `.git/hooks/pre-commit` (`--fix-hooks`).
41
- - **Guard CI pipelines:** Fails PR builds if prompt expansions exceed token thresholds (`--strict`, `--max-tokens`).
40
+ - **Catch rogue git hooks:** Detects and disables silent LLM review hooks in the active hooks directory (`--fix-hooks`), including linked worktrees, submodules, and repositories using `core.hooksPath`.
41
+ - **Locate the bloat:** Breaks a Markdown directive down per level-2 heading so you know which section to trim (`--sections`).
42
+ - **Audit MCP overhead:** Reports the context cost of every declared MCP server in `opencode.json`, `.mcp.json`, and `mcp.json` (`--mcp`).
43
+ - **Guard CI pipelines:** Fails PR builds if prompt expansions exceed token thresholds or regress past a saved baseline (`--strict`, `--max-tokens`, `--baseline`, `--max-delta`).
42
44
  - **Zero dependencies:** Pure Python standard library (Python >= 3.8). Runs in < 10ms.
43
45
 
44
46
  ---
@@ -47,9 +49,9 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
47
49
 
48
50
  ### Via Homebrew (macOS & Linux)
49
51
  ```bash
50
- brew install josephrotts/tap/token-diet
52
+ brew install joseoper/tap/token-diet
51
53
  # Or:
52
- brew tap josephrotts/tap
54
+ brew tap joseoper/tap
53
55
  brew install token-diet
54
56
  ```
55
57
 
@@ -69,7 +71,7 @@ uv tool install token-diet-cli
69
71
 
70
72
  ### Running Directly from Source (No Installation Needed)
71
73
  ```bash
72
- git clone https://github.com/josephrotts/token-diet.git
74
+ git clone https://github.com/joseoper/token-diet.git
73
75
  cd token-diet
74
76
  PYTHONPATH=src python3 -m token_diet
75
77
  ```
@@ -179,6 +181,60 @@ Emit structured JSON for automated pipelines, threshold assertions, or telemetry
179
181
  token-diet --json
180
182
  ```
181
183
 
184
+ ### 8. Locate Prompt Bloat per Heading (`--sections`)
185
+ A monolithic directive rarely bloats evenly. This breakdown attributes bytes and estimated tokens to every level-2 (`##`) heading, so you know exactly which section to compress or move into an on-demand skill:
186
+
187
+ ```bash
188
+ token-diet --sections AGENTS.md
189
+ ```
190
+
191
+ ```
192
+ SECTION BREAKDOWN: MARKDOWN HEADING FOOTPRINT
193
+ ================================================================================
194
+ AGENTS.md (23,914 B / ~5,978 tokens)
195
+ Line | Heading | Bytes | Tokens | Status
196
+ --------------------------------------------------------------------------------
197
+ 1 | (preamble) | 412 | 103 | [OK]
198
+ 12 | Core Directives | 19,220 | 4,805 | [ALERT]
199
+ 98 | Additional Protocols | 4,282 | 1,070 | [WARN]
200
+ --------------------------------------------------------------------------------
201
+ ```
202
+
203
+ Content before the first `##` heading is grouped as `(preamble)`, and deeper headings (`###`) stay attributed to their parent section. Combine with `--json` for machine-readable output.
204
+
205
+ ### 9. Audit MCP Server Overhead (`--mcp`)
206
+ Every declared MCP server consumes context, and some configs inline large tool schemas. `token-diet` discovers local `opencode.json`, `.mcp.json`, `mcp.json`, `.cursor/mcp.json`, and `.vscode/mcp.json` files and reports the payload cost per server:
207
+
208
+ ```bash
209
+ token-diet --mcp
210
+ ```
211
+
212
+ ```
213
+ MCP SERVER PAYLOAD AUDIT
214
+ ================================================================================
215
+ opencode.json [opencode] (2 server(s))
216
+ Server | Transport | Tools | Bytes | Tokens | Status
217
+ --------------------------------------------------------------------------------
218
+ engram | local | 12 | 18,430 | 4,607 | [ALERT]
219
+ web-search | remote | 0 | 142 | 35 | [OK]
220
+ --------------------------------------------------------------------------------
221
+ ```
222
+
223
+ Config-declared `tools` blocks are counted and included in each server's payload. Schemas fetched by a server at runtime are not visible to a local audit; the reported figure is the local injection cost floor.
224
+
225
+ ### 10. Fail on Regression vs a Saved Baseline (`--baseline`, `--max-delta`)
226
+ Record a baseline once, then gate every pull request against it. This catches slow, incremental growth that a fixed threshold never notices:
227
+
228
+ ```bash
229
+ # Save the baseline on your default branch
230
+ token-diet --json > token-baseline.json
231
+
232
+ # Fail if the total grows more than 5% in a pull request
233
+ token-diet --baseline token-baseline.json --max-delta 5
234
+ ```
235
+
236
+ `--baseline` accepts any `--json` report (`{"summary": {"total_tokens": N}}`) or a minimal `{"total_tokens": N}` file. `--max-delta` exits with code `1` only when tokens **increase** beyond the given percentage; a decrease, an unchanged total, or growth exactly at the budget passes.
237
+
182
238
  ---
183
239
 
184
240
  ## Health Status Indicators
@@ -188,7 +244,7 @@ token-diet --json
188
244
  | 🟢 `[OK]` | `< 4 KB` | `< 1,000 tokens` | **Optimal Context:** Fits cleanly within prompt cache. Preserves fast, cheap inference. |
189
245
  | 🟡 `[WARN]` | `4 KB – 16 KB` | `1,000 – 4,000 tokens` | **Moderate Size:** Acceptable, but review for duplicate instructions or unneeded verbosity. |
190
246
  | 🔴 `[ALERT]` | `> 16 KB` | `> 4,000 tokens` | **Severe Prompt Bloat:** Monolithic rules degrade reasoning and waste context budget every turn. Decouple into On-Demand Skills. |
191
- | 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:** Local `.git/hooks/pre-commit` sends code to cloud LLMs on every `git commit`. Replace with static linters. |
247
+ | 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:** The active `pre-commit` hook sends code to cloud LLMs on every `git commit`. Replace with static linters. |
192
248
 
193
249
  ---
194
250
 
@@ -256,9 +312,18 @@ token-diet/
256
312
  │ └── token_diet/ # Core Python package (Zero external dependencies)
257
313
  │ ├── __init__.py
258
314
  │ ├── __main__.py
259
- │ └── cli.py # CLI implementation and footprint engine
315
+ │ ├── baseline.py # Baseline comparison & regression budget gate
316
+ │ ├── cli.py # CLI implementation and footprint engine
317
+ │ ├── git_hooks.py # Worktree, gitdir and core.hooksPath resolution
318
+ │ ├── mcp_audit.py # MCP config discovery & payload accounting
319
+ │ ├── metrics.py # Shared byte thresholds and token estimation
320
+ │ └── sections.py # Per-heading Markdown footprint breakdown
260
321
  ├── tests/
261
- │ └── test_token_audit.py # Automated unit test suite (TDD)
322
+ │ ├── test_baseline.py # Baseline budget unit tests
323
+ │ ├── test_git_hooks.py # Hook resolution unit tests
324
+ │ ├── test_mcp_audit.py # MCP payload unit tests
325
+ │ ├── test_sections.py # Section breakdown unit tests
326
+ │ └── test_token_audit.py # Core engine and CLI integration tests
262
327
  ├── .gitignore
263
328
  ├── AGENTS.md # Agent-friendly repository directives
264
329
  ├── CONTRIBUTING.md # Contribution guidelines & engineering principles
@@ -269,7 +334,7 @@ token-diet/
269
334
 
270
335
  ---
271
336
 
272
- ## Core Methodology: The 5 Vectors of Token Leakage
337
+ ## Core Methodology: The 6 Vectors of Token Leakage
273
338
 
274
339
  Token Diet is built on empirical analysis of how developer environments silently deplete API quotas:
275
340
 
@@ -278,6 +343,7 @@ Token Diet is built on empirical analysis of how developer environments silently
278
343
  3. **Reasoning Effort Inflation:** Running frontier models at `high` reasoning effort for simple syntax or diagnostic queries generates thousands of hidden chain-of-thought tokens. *Solution: Default to `medium` or `low`.*
279
344
  4. **Static MCP Schemas:** Registering dozens of MCP tools serializes massive JSON parameter schemas into system prompts. *Solution: Prune unused tool servers and enforce lazy-loading.*
280
345
  5. **Prompt Cache Invalidation:** Injecting dynamic timestamps or changing session IDs at the start of instructions destroys provider prompt caching discounts (up to 90%). *Solution: Maintain immutable instruction prefixes.*
346
+ 6. **In-Session Runtime Inefficiencies:** Agents loading entire 1,500-line files to check one function, overwriting whole files instead of focused diffs, or running uncompacted 100k+ token threads. *Solution: Enforce active runtime hygiene (grep-first, line-range reads, targeted diffs, and context compaction thresholds; see [Remediation Playbook Phase 4](docs/remediation-playbook.md#phase-4-in-session-runtime-hygiene-active-execution-guards)).*
281
347
 
282
348
  ---
283
349
 
@@ -14,8 +14,10 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
14
14
 
15
15
  `token-diet` scans your repository or workstation in milliseconds to:
16
16
  - **Audit prompt size:** Flags instruction files that exceed context budgets (`[OK]`, `[WARN]`, `[ALERT]`).
17
- - **Catch rogue git hooks:** Detects and disables silent LLM review hooks in `.git/hooks/pre-commit` (`--fix-hooks`).
18
- - **Guard CI pipelines:** Fails PR builds if prompt expansions exceed token thresholds (`--strict`, `--max-tokens`).
17
+ - **Catch rogue git hooks:** Detects and disables silent LLM review hooks in the active hooks directory (`--fix-hooks`), including linked worktrees, submodules, and repositories using `core.hooksPath`.
18
+ - **Locate the bloat:** Breaks a Markdown directive down per level-2 heading so you know which section to trim (`--sections`).
19
+ - **Audit MCP overhead:** Reports the context cost of every declared MCP server in `opencode.json`, `.mcp.json`, and `mcp.json` (`--mcp`).
20
+ - **Guard CI pipelines:** Fails PR builds if prompt expansions exceed token thresholds or regress past a saved baseline (`--strict`, `--max-tokens`, `--baseline`, `--max-delta`).
19
21
  - **Zero dependencies:** Pure Python standard library (Python >= 3.8). Runs in < 10ms.
20
22
 
21
23
  ---
@@ -24,9 +26,9 @@ Worse, some local tools install hidden `pre-commit` hooks that send your entire
24
26
 
25
27
  ### Via Homebrew (macOS & Linux)
26
28
  ```bash
27
- brew install josephrotts/tap/token-diet
29
+ brew install joseoper/tap/token-diet
28
30
  # Or:
29
- brew tap josephrotts/tap
31
+ brew tap joseoper/tap
30
32
  brew install token-diet
31
33
  ```
32
34
 
@@ -46,7 +48,7 @@ uv tool install token-diet-cli
46
48
 
47
49
  ### Running Directly from Source (No Installation Needed)
48
50
  ```bash
49
- git clone https://github.com/josephrotts/token-diet.git
51
+ git clone https://github.com/joseoper/token-diet.git
50
52
  cd token-diet
51
53
  PYTHONPATH=src python3 -m token_diet
52
54
  ```
@@ -156,6 +158,60 @@ Emit structured JSON for automated pipelines, threshold assertions, or telemetry
156
158
  token-diet --json
157
159
  ```
158
160
 
161
+ ### 8. Locate Prompt Bloat per Heading (`--sections`)
162
+ A monolithic directive rarely bloats evenly. This breakdown attributes bytes and estimated tokens to every level-2 (`##`) heading, so you know exactly which section to compress or move into an on-demand skill:
163
+
164
+ ```bash
165
+ token-diet --sections AGENTS.md
166
+ ```
167
+
168
+ ```
169
+ SECTION BREAKDOWN: MARKDOWN HEADING FOOTPRINT
170
+ ================================================================================
171
+ AGENTS.md (23,914 B / ~5,978 tokens)
172
+ Line | Heading | Bytes | Tokens | Status
173
+ --------------------------------------------------------------------------------
174
+ 1 | (preamble) | 412 | 103 | [OK]
175
+ 12 | Core Directives | 19,220 | 4,805 | [ALERT]
176
+ 98 | Additional Protocols | 4,282 | 1,070 | [WARN]
177
+ --------------------------------------------------------------------------------
178
+ ```
179
+
180
+ Content before the first `##` heading is grouped as `(preamble)`, and deeper headings (`###`) stay attributed to their parent section. Combine with `--json` for machine-readable output.
181
+
182
+ ### 9. Audit MCP Server Overhead (`--mcp`)
183
+ Every declared MCP server consumes context, and some configs inline large tool schemas. `token-diet` discovers local `opencode.json`, `.mcp.json`, `mcp.json`, `.cursor/mcp.json`, and `.vscode/mcp.json` files and reports the payload cost per server:
184
+
185
+ ```bash
186
+ token-diet --mcp
187
+ ```
188
+
189
+ ```
190
+ MCP SERVER PAYLOAD AUDIT
191
+ ================================================================================
192
+ opencode.json [opencode] (2 server(s))
193
+ Server | Transport | Tools | Bytes | Tokens | Status
194
+ --------------------------------------------------------------------------------
195
+ engram | local | 12 | 18,430 | 4,607 | [ALERT]
196
+ web-search | remote | 0 | 142 | 35 | [OK]
197
+ --------------------------------------------------------------------------------
198
+ ```
199
+
200
+ Config-declared `tools` blocks are counted and included in each server's payload. Schemas fetched by a server at runtime are not visible to a local audit; the reported figure is the local injection cost floor.
201
+
202
+ ### 10. Fail on Regression vs a Saved Baseline (`--baseline`, `--max-delta`)
203
+ Record a baseline once, then gate every pull request against it. This catches slow, incremental growth that a fixed threshold never notices:
204
+
205
+ ```bash
206
+ # Save the baseline on your default branch
207
+ token-diet --json > token-baseline.json
208
+
209
+ # Fail if the total grows more than 5% in a pull request
210
+ token-diet --baseline token-baseline.json --max-delta 5
211
+ ```
212
+
213
+ `--baseline` accepts any `--json` report (`{"summary": {"total_tokens": N}}`) or a minimal `{"total_tokens": N}` file. `--max-delta` exits with code `1` only when tokens **increase** beyond the given percentage; a decrease, an unchanged total, or growth exactly at the budget passes.
214
+
159
215
  ---
160
216
 
161
217
  ## Health Status Indicators
@@ -165,7 +221,7 @@ token-diet --json
165
221
  | 🟢 `[OK]` | `< 4 KB` | `< 1,000 tokens` | **Optimal Context:** Fits cleanly within prompt cache. Preserves fast, cheap inference. |
166
222
  | 🟡 `[WARN]` | `4 KB – 16 KB` | `1,000 – 4,000 tokens` | **Moderate Size:** Acceptable, but review for duplicate instructions or unneeded verbosity. |
167
223
  | 🔴 `[ALERT]` | `> 16 KB` | `> 4,000 tokens` | **Severe Prompt Bloat:** Monolithic rules degrade reasoning and waste context budget every turn. Decouple into On-Demand Skills. |
168
- | 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:** Local `.git/hooks/pre-commit` sends code to cloud LLMs on every `git commit`. Replace with static linters. |
224
+ | 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:** The active `pre-commit` hook sends code to cloud LLMs on every `git commit`. Replace with static linters. |
169
225
 
170
226
  ---
171
227
 
@@ -233,9 +289,18 @@ token-diet/
233
289
  │ └── token_diet/ # Core Python package (Zero external dependencies)
234
290
  │ ├── __init__.py
235
291
  │ ├── __main__.py
236
- │ └── cli.py # CLI implementation and footprint engine
292
+ │ ├── baseline.py # Baseline comparison & regression budget gate
293
+ │ ├── cli.py # CLI implementation and footprint engine
294
+ │ ├── git_hooks.py # Worktree, gitdir and core.hooksPath resolution
295
+ │ ├── mcp_audit.py # MCP config discovery & payload accounting
296
+ │ ├── metrics.py # Shared byte thresholds and token estimation
297
+ │ └── sections.py # Per-heading Markdown footprint breakdown
237
298
  ├── tests/
238
- │ └── test_token_audit.py # Automated unit test suite (TDD)
299
+ │ ├── test_baseline.py # Baseline budget unit tests
300
+ │ ├── test_git_hooks.py # Hook resolution unit tests
301
+ │ ├── test_mcp_audit.py # MCP payload unit tests
302
+ │ ├── test_sections.py # Section breakdown unit tests
303
+ │ └── test_token_audit.py # Core engine and CLI integration tests
239
304
  ├── .gitignore
240
305
  ├── AGENTS.md # Agent-friendly repository directives
241
306
  ├── CONTRIBUTING.md # Contribution guidelines & engineering principles
@@ -246,7 +311,7 @@ token-diet/
246
311
 
247
312
  ---
248
313
 
249
- ## Core Methodology: The 5 Vectors of Token Leakage
314
+ ## Core Methodology: The 6 Vectors of Token Leakage
250
315
 
251
316
  Token Diet is built on empirical analysis of how developer environments silently deplete API quotas:
252
317
 
@@ -255,6 +320,7 @@ Token Diet is built on empirical analysis of how developer environments silently
255
320
  3. **Reasoning Effort Inflation:** Running frontier models at `high` reasoning effort for simple syntax or diagnostic queries generates thousands of hidden chain-of-thought tokens. *Solution: Default to `medium` or `low`.*
256
321
  4. **Static MCP Schemas:** Registering dozens of MCP tools serializes massive JSON parameter schemas into system prompts. *Solution: Prune unused tool servers and enforce lazy-loading.*
257
322
  5. **Prompt Cache Invalidation:** Injecting dynamic timestamps or changing session IDs at the start of instructions destroys provider prompt caching discounts (up to 90%). *Solution: Maintain immutable instruction prefixes.*
323
+ 6. **In-Session Runtime Inefficiencies:** Agents loading entire 1,500-line files to check one function, overwriting whole files instead of focused diffs, or running uncompacted 100k+ token threads. *Solution: Enforce active runtime hygiene (grep-first, line-range reads, targeted diffs, and context compaction thresholds; see [Remediation Playbook Phase 4](docs/remediation-playbook.md#phase-4-in-session-runtime-hygiene-active-execution-guards)).*
258
324
 
259
325
  ---
260
326
 
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "token-diet-cli"
7
- version = "0.1.0"
7
+ version = "0.2.0"
8
8
  description = "Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.8"
11
11
  license = { text = "MIT" }
12
12
  authors = [
13
- { name = "josephrotts", email = "387283+josephrotts@users.noreply.github.com" }
13
+ { name = "joseoper", email = "joseoper@users.noreply.github.com" }
14
14
  ]
15
15
  keywords = ["tokens", "llm", "claude", "gemini", "codex", "opencode", "developer-tools"]
16
16
  classifiers = [
@@ -0,0 +1,80 @@
1
+ """
2
+ token-diet: Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows.
3
+ """
4
+
5
+ from token_diet.cli import (
6
+ estimate_tokens,
7
+ audit_file,
8
+ detect_local_prompts,
9
+ audit_git_hooks,
10
+ fix_git_hooks,
11
+ classify_size,
12
+ resolve_target_paths,
13
+ audit_known_prompts,
14
+ summarize_footprint,
15
+ main,
16
+ DEFAULT_PROMPT_PATHS,
17
+ LOCAL_PROMPT_CANDIDATES,
18
+ THRESHOLD_WARN_BYTES,
19
+ THRESHOLD_ALERT_BYTES,
20
+ )
21
+ from token_diet.baseline import (
22
+ check_budget,
23
+ compare_tokens,
24
+ extract_total_tokens,
25
+ load_baseline,
26
+ )
27
+ from token_diet.git_hooks import (
28
+ AI_HOOK_KEYWORDS,
29
+ has_ai_invocation,
30
+ read_core_hooks_path,
31
+ resolve_common_dir,
32
+ resolve_git_dir,
33
+ resolve_hooks_dir,
34
+ )
35
+ from token_diet.mcp_audit import (
36
+ MCP_CONFIG_CANDIDATES,
37
+ audit_mcp_config,
38
+ audit_mcp_overhead,
39
+ discover_mcp_configs,
40
+ )
41
+ from token_diet.sections import (
42
+ audit_sections,
43
+ audit_sections_for_paths,
44
+ split_sections,
45
+ )
46
+
47
+ __version__ = "0.2.0"
48
+ __all__ = [
49
+ "estimate_tokens",
50
+ "audit_file",
51
+ "detect_local_prompts",
52
+ "audit_git_hooks",
53
+ "fix_git_hooks",
54
+ "classify_size",
55
+ "resolve_target_paths",
56
+ "audit_known_prompts",
57
+ "summarize_footprint",
58
+ "main",
59
+ "DEFAULT_PROMPT_PATHS",
60
+ "LOCAL_PROMPT_CANDIDATES",
61
+ "THRESHOLD_WARN_BYTES",
62
+ "THRESHOLD_ALERT_BYTES",
63
+ "AI_HOOK_KEYWORDS",
64
+ "has_ai_invocation",
65
+ "read_core_hooks_path",
66
+ "resolve_common_dir",
67
+ "resolve_git_dir",
68
+ "resolve_hooks_dir",
69
+ "check_budget",
70
+ "compare_tokens",
71
+ "extract_total_tokens",
72
+ "load_baseline",
73
+ "MCP_CONFIG_CANDIDATES",
74
+ "audit_mcp_config",
75
+ "audit_mcp_overhead",
76
+ "discover_mcp_configs",
77
+ "audit_sections",
78
+ "audit_sections_for_paths",
79
+ "split_sections",
80
+ ]
@@ -0,0 +1,120 @@
1
+ """
2
+ baseline.py
3
+
4
+ Baseline comparison and regression budget gatekeeper logic for token-diet.
5
+ """
6
+
7
+ import json
8
+ import math
9
+ import os
10
+ from typing import Any, Dict, Optional
11
+
12
+
13
+ def extract_total_tokens(payload: Any) -> Optional[int]:
14
+ """
15
+ Extract total estimated tokens from a numeric value or report dictionary.
16
+
17
+ Returns an integer token count if valid and non-negative, else None.
18
+ Rejects booleans, negative numbers, and non-numeric values.
19
+ """
20
+ if isinstance(payload, bool):
21
+ return None
22
+ if isinstance(payload, (int, float)):
23
+ if math.isnan(payload) or math.isinf(payload):
24
+ return None
25
+ if payload < 0:
26
+ return None
27
+ return int(payload)
28
+ if isinstance(payload, dict):
29
+ token_val = payload.get("total_tokens")
30
+ if token_val is None and isinstance(payload.get("summary"), dict):
31
+ token_val = payload["summary"].get("total_tokens")
32
+ return extract_total_tokens(token_val)
33
+ return None
34
+
35
+
36
+ def load_baseline(filepath: str) -> Dict[str, Any]:
37
+ """
38
+ Load baseline token count from a JSON report file.
39
+
40
+ Returns a dict with 'ok', 'tokens', 'path', and 'error'.
41
+ Never raises an exception.
42
+ """
43
+ if not os.path.exists(filepath):
44
+ return {
45
+ "ok": False,
46
+ "tokens": None,
47
+ "path": filepath,
48
+ "error": f"Baseline file not found: {filepath}",
49
+ }
50
+
51
+ try:
52
+ with open(filepath, "r", encoding="utf-8") as f:
53
+ data = json.load(f)
54
+ except Exception as e:
55
+ return {
56
+ "ok": False,
57
+ "tokens": None,
58
+ "path": filepath,
59
+ "error": f"Failed to parse baseline JSON: {e}",
60
+ }
61
+
62
+ tokens = extract_total_tokens(data)
63
+ if tokens is None:
64
+ return {
65
+ "ok": False,
66
+ "tokens": None,
67
+ "path": filepath,
68
+ "error": "No valid token count found in baseline file",
69
+ }
70
+
71
+ return {
72
+ "ok": True,
73
+ "tokens": tokens,
74
+ "path": filepath,
75
+ "error": None,
76
+ }
77
+
78
+
79
+ def compare_tokens(current_tokens: int, baseline_tokens: int) -> Dict[str, Any]:
80
+ """
81
+ Compare current tokens against baseline tokens and compute delta metrics.
82
+
83
+ Returns a dict with baseline_tokens, current_tokens, delta_tokens,
84
+ delta_percent (rounded to 2 decimal places), and direction.
85
+ """
86
+ delta_tokens = current_tokens - baseline_tokens
87
+ if baseline_tokens > 0:
88
+ delta_percent = round(delta_tokens / baseline_tokens * 100.0, 2)
89
+ else:
90
+ delta_percent = 0.0
91
+
92
+ if delta_tokens > 0:
93
+ direction = "increase"
94
+ elif delta_tokens < 0:
95
+ direction = "decrease"
96
+ else:
97
+ direction = "unchanged"
98
+
99
+ return {
100
+ "baseline_tokens": baseline_tokens,
101
+ "current_tokens": current_tokens,
102
+ "delta_tokens": delta_tokens,
103
+ "delta_percent": delta_percent,
104
+ "direction": direction,
105
+ }
106
+
107
+
108
+ def check_budget(current_tokens: int, baseline_tokens: int, max_delta_percent: float) -> Dict[str, Any]:
109
+ """
110
+ Check if token growth exceeds the maximum allowed delta percentage budget.
111
+
112
+ Budget is exceeded only when direction is 'increase' and delta_percent
113
+ is strictly greater than max_delta_percent.
114
+ """
115
+ comparison = compare_tokens(current_tokens, baseline_tokens)
116
+ max_delta = float(max_delta_percent)
117
+ exceeded = comparison["direction"] == "increase" and comparison["delta_percent"] > max_delta
118
+ comparison["max_delta_percent"] = max_delta
119
+ comparison["exceeded"] = exceeded
120
+ return comparison