token-diet-cli 0.1.0__py3-none-any.whl

token_diet/__init__.py

@@ -0,0 +1,38 @@
+"""
+token-diet: Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows.
+"""
+
+from token_diet.cli import (
+    estimate_tokens,
+    audit_file,
+    detect_local_prompts,
+    audit_git_hooks,
+    fix_git_hooks,
+    classify_size,
+    resolve_target_paths,
+    audit_known_prompts,
+    summarize_footprint,
+    main,
+    DEFAULT_PROMPT_PATHS,
+    LOCAL_PROMPT_CANDIDATES,
+    THRESHOLD_WARN_BYTES,
+    THRESHOLD_ALERT_BYTES,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "estimate_tokens",
+    "audit_file",
+    "detect_local_prompts",
+    "audit_git_hooks",
+    "fix_git_hooks",
+    "classify_size",
+    "resolve_target_paths",
+    "audit_known_prompts",
+    "summarize_footprint",
+    "main",
+    "DEFAULT_PROMPT_PATHS",
+    "LOCAL_PROMPT_CANDIDATES",
+    "THRESHOLD_WARN_BYTES",
+    "THRESHOLD_ALERT_BYTES",
+]

token_diet/__main__.py

@@ -0,0 +1,4 @@
+from token_diet.cli import main
+
+if __name__ == "__main__":
+    main()

token_diet/cli.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python3
+"""
+cli.py
+
+Core diagnostic logic to inspect, quantify, and audit token and byte footprint
+of prompt directives, project files, and system configurations with automated alerts.
+"""
+
+import os
+import sys
+import json
+import argparse
+from typing import List, Dict, Any, Optional, Tuple
+
+DEFAULT_PROMPT_PATHS = [
+    os.path.expanduser("~/.codex/AGENTS.md"),
+    os.path.expanduser("~/.codex/engram-instructions.md"),
+    os.path.expanduser("~/.codex/engram-compact-prompt.md"),
+    os.path.expanduser("~/.claude/CLAUDE.md"),
+    os.path.expanduser("~/.gemini/GEMINI.md"),
+    os.path.expanduser("~/.gemini/system.md"),
+    os.path.expanduser("~/.config/opencode/AGENTS.md"),
+]
+
+LOCAL_PROMPT_CANDIDATES = [
+    "AGENTS.md",
+    "CLAUDE.md",
+    "GEMINI.md",
+    ".cursorrules",
+    ".windsurfrules",
+    ".clinerules",
+    ".github/copilot-instructions.md",
+]
+
+# Size thresholds in bytes
+THRESHOLD_WARN_BYTES = 4_000   # ~1,000 tokens
+THRESHOLD_ALERT_BYTES = 16_000 # ~4,000 tokens
+
+
+def supports_color() -> bool:
+    """Returns True if the running terminal supports ANSI color codes."""
+    if os.environ.get("NO_COLOR") or not sys.stdout.isatty():
+        return False
+    return True
+
+
+def colorize(text: str, color_code: str) -> str:
+    """Applies ANSI color code if terminal supports color."""
+    if not supports_color():
+        return text
+    return f"\033[{color_code}m{text}\033[0m"
+
+
+def green(text: str) -> str:
+    return colorize(text, "92")
+
+
+def yellow(text: str) -> str:
+    return colorize(text, "93")
+
+
+def red(text: str) -> str:
+    return colorize(text, "91")
+
+
+def bold(text: str) -> str:
+    return colorize(text, "1")
+
+
+def estimate_tokens(char_count: int) -> int:
+    """Estimates tokens based on a conservative heuristic of ~4 characters per token."""
+    if char_count <= 0:
+        return 0
+    return char_count // 4
+
+
+def classify_size(size_bytes: int) -> Tuple[str, str]:
+    """Returns (status_label, severity) based on byte size."""
+    if size_bytes > THRESHOLD_ALERT_BYTES:
+        return "ALERT", "critical"
+    elif size_bytes > THRESHOLD_WARN_BYTES:
+        return "WARN", "warning"
+    return "OK", "optimal"
+
+
+def audit_file(filepath: str) -> Dict[str, Any]:
+    """Inspects a file and returns its size in bytes, estimated tokens, and health status."""
+    if not os.path.exists(filepath):
+        return {
+            "path": filepath,
+            "exists": False,
+            "bytes": 0,
+            "estimated_tokens": 0,
+            "status": "MISSING",
+            "severity": "none",
+        }
+    try:
+        size = os.path.getsize(filepath)
+        tokens = estimate_tokens(size)
+        status, severity = classify_size(size)
+        return {
+            "path": filepath,
+            "exists": True,
+            "bytes": size,
+            "estimated_tokens": tokens,
+            "status": status,
+            "severity": severity,
+        }
+    except Exception as e:
+        return {
+            "path": filepath,
+            "exists": False,
+            "error": str(e),
+            "bytes": 0,
+            "estimated_tokens": 0,
+            "status": "ERROR",
+            "severity": "critical",
+        }
+
+
+def detect_local_prompts(base_dir: str = ".") -> List[str]:
+    """Detects standard AI agent directive files in the given directory."""
+    found: List[str] = []
+    abs_base = os.path.abspath(base_dir)
+
+    for candidate in LOCAL_PROMPT_CANDIDATES:
+        p = os.path.join(abs_base, candidate)
+        if os.path.isfile(p):
+            found.append(p)
+
+    # Check .cursor/rules directory if it exists
+    cursor_rules_dir = os.path.join(abs_base, ".cursor", "rules")
+    if os.path.isdir(cursor_rules_dir):
+        for root, _, files in os.walk(cursor_rules_dir):
+            for f in sorted(files):
+                if f.endswith((".md", ".mdc", ".txt")):
+                    found.append(os.path.join(root, f))
+
+    return found
+
+
+def audit_git_hooks(repo_dir: str = ".") -> List[Dict[str, Any]]:
+    """Inspects .git/hooks for automated scripts that may silently trigger LLM calls."""
+    alerts: List[Dict[str, Any]] = []
+    hook_path = os.path.join(os.path.abspath(repo_dir), ".git", "hooks", "pre-commit")
+
+    if not os.path.exists(hook_path):
+        return alerts
+
+    try:
+        keywords = ["gga", "opencode", "gentle-ai", "claude", "gemini", "ollama", "llm review"]
+        matched = []
+        with open(hook_path, "r", encoding="utf-8", errors="ignore") as f:
+            for line in f:
+                stripped = line.strip()
+                if stripped.startswith("#"):
+                    continue
+                lower_line = stripped.lower()
+                for k in keywords:
+                    if k in lower_line and k not in matched:
+                        matched.append(k)
+
+        if matched:
+            alerts.append({
+                "hook": "pre-commit",
+                "path": hook_path,
+                "severity": "critical",
+                "matched_patterns": matched,
+                "message": (
+                    f"Pre-commit hook invokes AI/LLM review tooling ({', '.join(matched)}). "
+                    "This causes invisible token consumption on every commit."
+                ),
+            })
+    except Exception as e:
+        alerts.append({
+            "hook": "pre-commit",
+            "path": hook_path,
+            "severity": "warning",
+            "error": str(e),
+            "message": f"Could not inspect pre-commit hook: {e}",
+        })
+
+    return alerts
+
+
+def fix_git_hooks(repo_dir: str = ".") -> Dict[str, Any]:
+    """
+    Safely neutralizes unmonitored AI/LLM invocations in .git/hooks/pre-commit:
+    1. Creates a backup copy at .git/hooks/pre-commit.bak.
+    2. Comments out lines that invoke matched AI review keywords.
+    3. Preserves original executable file permissions.
+    """
+    hook_path = os.path.join(os.path.abspath(repo_dir), ".git", "hooks", "pre-commit")
+    if not os.path.exists(hook_path):
+        return {
+            "fixed": False,
+            "hook_path": hook_path,
+            "backup_path": None,
+            "disabled_count": 0,
+            "message": "No pre-commit hook found in .git/hooks/",
+        }
+
+    keywords = ["gga", "opencode", "gentle-ai", "claude", "gemini", "ollama", "llm review"]
+    try:
+        with open(hook_path, "r", encoding="utf-8", errors="ignore") as f:
+            lines = f.readlines()
+
+        modified_lines = []
+        disabled_count = 0
+        for line in lines:
+            stripped = line.strip()
+            # Preserve already-commented lines
+            if stripped.startswith("#"):
+                modified_lines.append(line)
+                continue
+
+            lower_line = line.lower()
+            if any(k in lower_line for k in keywords):
+                modified_lines.append(f"# [token-diet disabled]: {line}")
+                disabled_count += 1
+            else:
+                modified_lines.append(line)
+
+        if disabled_count == 0:
+            return {
+                "fixed": False,
+                "hook_path": hook_path,
+                "backup_path": None,
+                "disabled_count": 0,
+                "message": "No irregular AI hooks detected to disable in .git/hooks/pre-commit",
+            }
+
+        backup_path = hook_path + ".bak"
+        with open(backup_path, "w", encoding="utf-8") as f:
+            f.writelines(lines)
+
+        mode = os.stat(hook_path).st_mode
+        with open(hook_path, "w", encoding="utf-8") as f:
+            f.writelines(modified_lines)
+        os.chmod(hook_path, mode)
+
+        return {
+            "fixed": True,
+            "hook_path": hook_path,
+            "backup_path": backup_path,
+            "disabled_count": disabled_count,
+            "message": f"Successfully neutralized {disabled_count} AI invocation(s). Backup saved to {backup_path}",
+        }
+    except Exception as e:
+        return {
+            "fixed": False,
+            "hook_path": hook_path,
+            "backup_path": None,
+            "disabled_count": 0,
+            "error": str(e),
+            "message": f"Failed to fix pre-commit hook: {e}",
+        }
+
+
+def resolve_target_paths(input_paths: Optional[List[str]] = None, check_local: bool = True) -> List[str]:
+    """Resolves file paths, expanding directories recursively or defaulting to local/global prompts."""
+    if input_paths:
+        resolved: List[str] = []
+        for p in input_paths:
+            expanded = os.path.abspath(os.path.expanduser(p))
+            if os.path.isdir(expanded):
+                for root, _, files in os.walk(expanded):
+                    for f in sorted(files):
+                        if not f.startswith("."):
+                            resolved.append(os.path.join(root, f))
+            else:
+                resolved.append(expanded)
+        return resolved
+
+    if check_local:
+        local_files = detect_local_prompts(".")
+        if local_files:
+            return local_files
+
+    return DEFAULT_PROMPT_PATHS
+
+
+def audit_known_prompts(custom_paths: Optional[List[str]] = None, check_local: bool = True) -> List[Dict[str, Any]]:
+    """Audits target paths and returns itemized results."""
+    paths = resolve_target_paths(custom_paths, check_local=check_local)
+    return [audit_file(p) for p in paths]
+
+
+def summarize_footprint(results: List[Dict[str, Any]], hook_alerts: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:
+    """Calculates aggregate footprint summary, status totals, and health indicators."""
+    total_bytes = sum(r.get("bytes", 0) for r in results)
+    total_tokens = sum(r.get("estimated_tokens", 0) for r in results)
+    files_found = sum(1 for r in results if r.get("exists"))
+    alert_count = sum(1 for r in results if r.get("status") == "ALERT")
+    warn_count = sum(1 for r in results if r.get("status") == "WARN")
+
+    if hook_alerts:
+        alert_count += len([h for h in hook_alerts if h.get("severity") == "critical"])
+
+    overall_status = "HEALTHY"
+    if alert_count > 0:
+        overall_status = "CRITICAL_BLOAT"
+    elif warn_count > 0:
+        overall_status = "WARNING_MODERATE"
+
+    return {
+        "total_bytes": total_bytes,
+        "total_tokens": total_tokens,
+        "files_found": files_found,
+        "total_checked": len(results),
+        "alerts": alert_count,
+        "warnings": warn_count,
+        "status": overall_status,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="token-diet",
+        description="Audit token footprint and detect silent context leaks in AI agent workflows.",
+    )
+    parser.add_argument(
+        "paths",
+        nargs="*",
+        help="Optional files or directories to inspect. Defaults to auto-detecting project files in CWD or global prompts.",
+    )
+    parser.add_argument(
+        "--global",
+        dest="global_only",
+        action="store_true",
+        help="Inspect global assistant prompts (~/.claude, ~/.codex, ~/.gemini) instead of local project files.",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output structured JSON payload.",
+    )
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        metavar="N",
+        help="Exit with code 1 if total estimated tokens exceed N (useful for CI/CD gates).",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Exit with code 1 if any alert or warning threshold is exceeded.",
+    )
+    parser.add_argument(
+        "--fix-hooks",
+        action="store_true",
+        help="Automatically neutralize unmonitored AI review invocations in .git/hooks/pre-commit (creates .bak backup).",
+    )
+    args = parser.parse_args()
+
+    if args.fix_hooks:
+        fix_result = fix_git_hooks(".")
+        if args.json:
+            print(json.dumps({"fix_hooks": fix_result}, indent=2))
+        else:
+            if fix_result["fixed"]:
+                print(green(bold("🛠️  AUTO-REMEDIATION: PRE-COMMIT HOOKS NEUTRALIZED")))
+                print(f"  • Backup saved: {fix_result['backup_path']}")
+                print(f"  • Neutralized {fix_result['disabled_count']} AI review invocation(s).")
+                print("  • Safe pre-commit hook is active without silent token bleed.")
+            else:
+                print(yellow(f"ℹ️  {fix_result['message']}"))
+        return
+
+    check_local = not args.global_only
+    targets = args.paths if args.paths else None
+
+    # Detect git hooks in current working directory
+    hook_alerts = audit_git_hooks(".") if check_local and not targets else []
+
+    results = audit_known_prompts(targets, check_local=check_local)
+    summary = summarize_footprint(results, hook_alerts)
+
+    if args.json:
+        payload = {
+            "summary": summary,
+            "hook_alerts": hook_alerts,
+            "details": results,
+        }
+        print(json.dumps(payload, indent=2))
+        _check_exit_conditions(summary, args.max_tokens, args.strict)
+        return
+
+    # Terminal Output
+    print(bold("=" * 80))
+    print(bold("TOKEN DIET: FOOTPRINT & CONTEXT AUDIT"))
+    print(bold("=" * 80))
+    print(f'{"File / Directive":<46} | {"Bytes":>9} | {"Tokens":>9} | {"Status":>8}')
+    print("-" * 80)
+
+    for r in results:
+        status_path = r["path"]
+        if len(status_path) > 44:
+            status_path = "..." + status_path[-41:]
+
+        if not r["exists"]:
+            status_str = colorize("(missing)", "90")
+            print(f'{status_path:<46} | {"-":>9} | {"-":>9} | {status_str:>8}')
+        else:
+            status = r["status"]
+            if status == "ALERT":
+                badge = red("[ALERT]")
+            elif status == "WARN":
+                badge = yellow("[WARN]")
+            else:
+                badge = green("[OK]")
+
+            print(f'{status_path:<46} | {r["bytes"]:>9,d} | {r["estimated_tokens"]:>9,d} | {badge:>17}')
+
+    print(bold("=" * 80))
+    print(f"Total Bytes:     {summary['total_bytes']:,d} B")
+    print(f"Total Tokens:    ~{summary['total_tokens']:,d} estimated base tokens per turn")
+    print(f"Files Detected:  {summary['files_found']} of {summary['total_checked']}")
+
+    # Print Hook Alerts if any
+    if hook_alerts:
+        print("\n" + red(bold("🚨 IRREGULAR HOOK DETECTED:")))
+        for h in hook_alerts:
+            print(red(f"  • [{h['hook']}] {h['message']}"))
+            print(f"    Path: {h['path']}")
+        print(colorize("  💡 Tip: Run 'token-diet --fix-hooks' to automatically neutralize silent AI invocations.", "96"))
+
+    # Print Recommendations if bloated
+    if summary["alerts"] > 0:
+        print("\n" + red(bold("⚠️  CONTEXT BLOAT DETECTED:")))
+        print("  Files marked [ALERT] exceed 16 KB (>4k tokens) and severely penalize context windows.")
+        print("  Recommendation: Decouple procedures into On-Demand Skills using the LUT Router pattern.")
+    elif summary["warnings"] > 0:
+        print("\n" + yellow("ℹ️  MODERATE CONTEXT SIZE:"))
+        print("  Files marked [WARN] exceed 4 KB. Review if non-essential sections can be pruned.")
+
+    _check_exit_conditions(summary, args.max_tokens, args.strict)
+
+
+def _check_exit_conditions(summary: Dict[str, Any], max_tokens: Optional[int], strict: bool):
+    """Handles exit codes for CI/CD gates."""
+    if max_tokens is not None and summary["total_tokens"] > max_tokens:
+        sys.exit(1)
+    if strict and (summary["alerts"] > 0 or summary["warnings"] > 0):
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

token_diet_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,304 @@
+Metadata-Version: 2.4
+Name: token-diet-cli
+Version: 0.1.0
+Summary: Diagnostic tools and methodology to audit and optimize token footprint in AI coding workflows
+Author-email: josephrotts <387283+josephrotts@users.noreply.github.com>
+License: MIT
+Keywords: tokens,llm,claude,gemini,codex,opencode,developer-tools
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# token-diet
+
+Audit token footprint and detect silent context leaks in AI agent workflows.
+
+Zero-dependency CLI for **Claude Code**, **Cursor**, **Codex**, **Gemini**, and **OpenCode**.
+
+---
+
+## Why?
+
+AI coding assistants inject instruction files (`AGENTS.md`, `CLAUDE.md`, `.cursorrules`) into the system prompt on **every single turn**. A bloated 25 KB prompt file forces the model to re-process ~6,000 tokens on every message, draining rate limits and breaking prompt cache discounts.
+
+Worse, some local tools install hidden `pre-commit` hooks that send your entire git diff to an LLM on every commit.
+
+`token-diet` scans your repository or workstation in milliseconds to:
+- **Audit prompt size:** Flags instruction files that exceed context budgets (`[OK]`, `[WARN]`, `[ALERT]`).
+- **Catch rogue git hooks:** Detects and disables silent LLM review hooks in `.git/hooks/pre-commit` (`--fix-hooks`).
+- **Guard CI pipelines:** Fails PR builds if prompt expansions exceed token thresholds (`--strict`, `--max-tokens`).
+- **Zero dependencies:** Pure Python standard library (Python >= 3.8). Runs in < 10ms.
+
+---
+
+## Installation
+
+### Via Homebrew (macOS & Linux)
+```bash
+brew install josephrotts/tap/token-diet
+# Or:
+brew tap josephrotts/tap
+brew install token-diet
+```
+
+### Via `pipx` (Recommended for Isolated CLI)
+```bash
+pipx install token-diet-cli
+```
+
+### Via `pip` / `uv`
+```bash
+# Using pip:
+pip install token-diet-cli
+
+# Using uv:
+uv tool install token-diet-cli
+```
+
+### Running Directly from Source (No Installation Needed)
+```bash
+git clone https://github.com/josephrotts/token-diet.git
+cd token-diet
+PYTHONPATH=src python3 -m token_diet
+```
+
+---
+
+## Usage
+
+### 1. Instant Health Check in Current Directory
+Simply run `token-diet` in any project repository. It automatically scans local directives and checks for dangerous pre-commit hooks:
+
+```bash
+token-diet
+```
+
+**Example Output:**
+```text
+================================================================================
+TOKEN DIET: FOOTPRINT & CONTEXT AUDIT
+================================================================================
+File / Directive                               |     Bytes |    Tokens |   Status
+--------------------------------------------------------------------------------
+/Users/developer/my-project/AGENTS.md          |       962 |       240 |     [OK]
+/Users/developer/my-project/.cursorrules       |     1,240 |       310 |     [OK]
+================================================================================
+Total Bytes:     2,202 B
+Total Tokens:    ~550 estimated base tokens per turn
+Files Detected:  2 of 2
+```
+
+### 2. Automated Alerts: Context Bloat & Invisible Pre-Commit Hooks
+When a prompt file is oversized or a git hook silently burns tokens on every commit, `token-diet` issues an instant visual alert:
+
+```bash
+token-diet
+```
+
+**Example Alert Output:**
+```text
+================================================================================
+TOKEN DIET: FOOTPRINT & CONTEXT AUDIT
+================================================================================
+File / Directive                               |     Bytes |    Tokens |   Status
+--------------------------------------------------------------------------------
+/Users/developer/my-project/AGENTS.md          |    24,500 |     6,125 |  [ALERT]
+================================================================================
+Total Bytes:     24,500 B
+Total Tokens:    ~6,125 estimated base tokens per turn
+Files Detected:  1 of 1
+
+🚨 IRREGULAR HOOK DETECTED:
+  • [pre-commit] Pre-commit hook invokes AI/LLM review tooling (gga, opencode). This causes invisible token consumption on every commit.
+    Path: /Users/developer/my-project/.git/hooks/pre-commit
+  💡 Tip: Run 'token-diet --fix-hooks' to automatically neutralize silent AI invocations.
+
+⚠️  CONTEXT BLOAT DETECTED:
+  Files marked [ALERT] exceed 16 KB (>4k tokens) and severely penalize context windows.
+  Recommendation: Decouple procedures into On-Demand Skills using the LUT Router pattern.
+```
+
+### 3. Auto-Fix Silent Pre-Commit Hooks (`--fix-hooks`)
+Neutralize unmonitored AI review tooling in `.git/hooks/pre-commit` in a single command with an automatic backup (`.git/hooks/pre-commit.bak`):
+
+```bash
+token-diet --fix-hooks
+```
+
+**Example Output:**
+```text
+🛠️  AUTO-REMEDIATION: PRE-COMMIT HOOKS NEUTRALIZED
+  • Backup saved: /Users/developer/my-project/.git/hooks/pre-commit.bak
+  • Neutralized 1 AI review invocation(s).
+  • Safe pre-commit hook is active without silent token bleed.
+```
+
+### 4. Audit System-Wide Global Prompts (`--global`)
+Scan default assistant instruction files across your entire workstation (`~/.claude/CLAUDE.md`, `~/.codex/AGENTS.md`, `~/.gemini/GEMINI.md`, etc.):
+
+```bash
+token-diet --global
+```
+
+### 5. Audit Specific Paths or Folders
+```bash
+# Audit a single file
+token-diet AGENTS.md
+
+# Audit entire directories recursively
+token-diet docs/
+```
+
+### 6. Automated CI / Pre-Commit Gate
+Integrate into your CI pipeline or Git pre-commit workflow to prevent prompt regressions:
+
+```bash
+# Fail CI if prompt footprint exceeds 2,500 tokens:
+token-diet --max-tokens 2500 AGENTS.md
+
+# Fail CI if any file triggers a [WARN] or [ALERT]:
+token-diet --strict docs/
+```
+
+### 7. Structured Telemetry Output (`--json`)
+Emit structured JSON for automated pipelines, threshold assertions, or telemetry dashboards:
+
+```bash
+token-diet --json
+```
+
+---
+
+## Health Status Indicators
+
+| Indicator | Byte Threshold | Estimated Tokens | What it Means & Recommended Action |
+|---|:---:|:---:|---|
+| 🟢 `[OK]` | `< 4 KB` | `< 1,000 tokens` | **Optimal Context:** Fits cleanly within prompt cache. Preserves fast, cheap inference. |
+| 🟡 `[WARN]` | `4 KB – 16 KB` | `1,000 – 4,000 tokens` | **Moderate Size:** Acceptable, but review for duplicate instructions or unneeded verbosity. |
+| 🔴 `[ALERT]` | `> 16 KB` | `> 4,000 tokens` | **Severe Prompt Bloat:** Monolithic rules degrade reasoning and waste context budget every turn. Decouple into On-Demand Skills. |
+| 🚨 `[HOOK ALERT]` | — | — | **Hidden Token Bleed:** Local `.git/hooks/pre-commit` sends code to cloud LLMs on every `git commit`. Replace with static linters. |
+
+---
+
+## What To Do When a Leak is Detected (Action Paths)
+
+When `token-diet` flags an irregularity, follow the corresponding action path (detailed runbooks available in the [Remediation Playbook](docs/remediation-playbook.md)):
+
+### Path A: When an Irregular Git Hook is Flagged (`🚨 IRREGULAR HOOK DETECTED`)
+* **Problem:** A local hook inside `.git/hooks/pre-commit` secretly sends git diffs to cloud AI tools on every single commit.
+* **Action Path:**
+  1. Inspect the script: `cat .git/hooks/pre-commit`
+  2. Remove or comment out the automated AI review invocation.
+  3. Replace it with fast, local deterministic linters (`ruff`, `eslint`, `biome`).
+  4. Relocate semantic AI reviews to **Pull Request gates in CI** (see [CI/CD Integration](docs/ci-integration.md)), where reviews are metered and visible.
+
+### Path B: When an Agent Directive is Bloated (`🔴 [ALERT]`)
+* **Problem:** A root instruction file (`AGENTS.md`, `CLAUDE.md`, `.cursorrules`) exceeds 16 KB (> 4,000 tokens), exhausting your token budget on every turn and destroying cache discounts.
+* **Action Path (The Look-Up Table / LUT Router Pattern):**
+  1. **Extract Specialized Protocols:** Move detailed procedure manuals, test specifications, and review arbitrations into standalone on-demand skills (e.g., `skills/<skill-name>/SKILL.md`).
+  2. **Compress Root Directive:** Keep only core behavioral invariants and a routing index in the root file (< 3 KB / < 750 tokens):
+     ```markdown
+     ## On-Demand Skills
+     - When running database migrations, read `skills/db-migrate/SKILL.md`.
+     - When preparing a release, read `skills/oss-publisher/SKILL.md`.
+     ```
+  3. Re-run `token-diet` to confirm the status turns 🟢 `[OK]`.
+
+### Path C: When CI Fails on Token Budget Exceeded (`exit code 1`)
+* **Problem:** A Pull Request introduced unbudgeted prompt expansion that violated `--max-tokens <N>` or `--strict`.
+* **Action Path:**
+  1. Run `token-diet --json` to get the machine-readable breakdown of offending files.
+  2. Prune duplicate rules or extract text into skills.
+  3. Re-run local check before pushing: `token-diet --strict`.
+
+### Agentic Remediation: Prompt for AI Assistants
+To have your AI coding assistant (Claude Code, Gemini CLI, Cursor, Windsurf, OpenCode) remediate prompt bloat autonomously without manual edits, copy and paste this prompt:
+
+```text
+Run `token-diet` in this repository. If you detect any files marked [ALERT] or [WARN], follow `docs/remediation-playbook.md` to decouple operational procedures into on-demand skills under `skills/<skill-name>/SKILL.md`. Compress the root instruction file to a concise Look-Up Table (LUT Router) under 3 KB (< 750 tokens). If an irregular hook is flagged, run `token-diet --fix-hooks`. Verify that all tests pass and `token-diet` reports [OK].
+```
+
+---
+
+## Running Tests
+
+Token Diet includes an automated test suite with deterministic coverage:
+
+```bash
+python3 -m unittest discover -s tests -p 'test_*.py'
+```
+
+---
+
+## Repository Structure
+
+```
+token-diet/
+├── .github/
+│   └── workflows/
+│       └── ci.yml             # Automated multi-version CI test runner
+├── docs/
+│   ├── remediation-playbook.md # Step-by-step token containment and LUT routing
+│   └── ci-integration.md       # CI workflows & pre-commit gatekeeper examples
+├── src/
+│   └── token_diet/            # Core Python package (Zero external dependencies)
+│       ├── __init__.py
+│       ├── __main__.py
+│       └── cli.py             # CLI implementation and footprint engine
+├── tests/
+│   └── test_token_audit.py    # Automated unit test suite (TDD)
+├── .gitignore
+├── AGENTS.md                  # Agent-friendly repository directives
+├── CONTRIBUTING.md            # Contribution guidelines & engineering principles
+├── LICENSE                    # MIT Open Source License
+├── pyproject.toml             # PEP 621 packaging and CLI entrypoint
+└── README.md                  # Project overview and documentation
+```
+
+---
+
+## Core Methodology: The 5 Vectors of Token Leakage
+
+Token Diet is built on empirical analysis of how developer environments silently deplete API quotas:
+
+1. **Global Prompt Bloat:** Monolithic instruction files (`CLAUDE.md`, `AGENTS.md`) exceeding 20 KB inject tens of thousands of static tokens on every turn. *Solution: Keep root directives under 3 KB (<750 tokens) and route to on-demand skills.*
+2. **Invisible Git Hooks:** Pre-commit hooks that intercept local commits to invoke cloud LLM reviews drain quotas without developer consent. *Solution: Keep local commits deterministic; reserve AI reviews for PR gates.*
+3. **Reasoning Effort Inflation:** Running frontier models at `high` reasoning effort for simple syntax or diagnostic queries generates thousands of hidden chain-of-thought tokens. *Solution: Default to `medium` or `low`.*
+4. **Static MCP Schemas:** Registering dozens of MCP tools serializes massive JSON parameter schemas into system prompts. *Solution: Prune unused tool servers and enforce lazy-loading.*
+5. **Prompt Cache Invalidation:** Injecting dynamic timestamps or changing session IDs at the start of instructions destroys provider prompt caching discounts (up to 90%). *Solution: Maintain immutable instruction prefixes.*
+
+---
+
+## Platform Compatibility
+
+Token Diet is OS-agnostic and cross-platform:
+- **CLI Diagnostic Tool:** Pure Python standard library with zero third-party dependencies. Runs seamlessly on **macOS**, **Linux**, and **Windows** (Python >= 3.8).
+- **Universal Methodology:** Core principles apply across all assistants (Claude Code, Gemini CLI, OpenAI Codex, OpenCode, Cursor, Windsurf).
+
+---
+
+## Documentation & Guides
+
+For in-depth operational procedures and automated pipeline setups:
+- **[Remediation Playbook](docs/remediation-playbook.md):** Step-by-step runbook to prune prompt files, configure reasoning effort, and contain background services.
+- **[CI/CD Integration](docs/ci-integration.md):** GitHub Actions workflows and pre-commit configurations for automated token budget enforcement.
+- **[Agent Directives (AGENTS.md)](AGENTS.md):** Operational guidelines for AI coding assistants working on this codebase.
+- **[Contributing Guidelines](CONTRIBUTING.md):** How to run tests, submit pull requests, and coding standards.
+
+---
+
+## License
+
+This project is licensed under the terms of the [MIT License](LICENSE).

token_diet_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+token_diet/__init__.py,sha256=s4rY5xZKu2vgKxyRMTMgBWg_otFHbb5O6e7jVsQ6t-U,844
+token_diet/__main__.py,sha256=RUdFtZ0V7DqVae2VM23UN9llXRAhdkDSI9usEEKxSsg,71
+token_diet/cli.py,sha256=S84-yfTe4UQKLo2KW3_ZMWrltmRfgSb6bydDkNZKXEo,15488
+token_diet_cli-0.1.0.dist-info/licenses/LICENSE,sha256=0B21Z9VEU2PIYo8f9Zwv6_w8xIsnG2Z-iz-wu68Z7j0,1068
+token_diet_cli-0.1.0.dist-info/METADATA,sha256=SAVv5tf5YrnPepu4CXrN5MFcOdRxrhMw1B_zEQZWyC0,13647
+token_diet_cli-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+token_diet_cli-0.1.0.dist-info/entry_points.txt,sha256=WosBe1HHGSUBGvrpQNCMrZ_oTOr63wjrqi4onmheNeQ,51
+token_diet_cli-0.1.0.dist-info/top_level.txt,sha256=-QNFHy0ZgQmQwuX39uVnNuNWN5ANdj4biiTKjwKeig4,11
+token_diet_cli-0.1.0.dist-info/RECORD,,

token_diet_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

token_diet_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+token-diet = token_diet.cli:main

token_diet_cli-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 josephrotts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

token_diet_cli-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+token_diet