tokenaudit 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. tokenaudit-0.2.0/.dockerignore +15 -0
  2. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/.github/workflows/ci.yml +16 -2
  3. tokenaudit-0.2.0/.github/workflows/release.yml +83 -0
  4. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/.gitignore +3 -0
  5. tokenaudit-0.2.0/CHANGELOG.md +58 -0
  6. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/PKG-INFO +33 -5
  7. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/README.md +31 -3
  8. tokenaudit-0.2.0/action.yml +60 -0
  9. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/generate.py +3 -1
  10. tokenaudit-0.2.0/demo/Dockerfile +17 -0
  11. tokenaudit-0.2.0/demo/README.md +44 -0
  12. tokenaudit-0.2.0/demo/__init__.py +1 -0
  13. tokenaudit-0.2.0/demo/app.py +113 -0
  14. tokenaudit-0.2.0/demo/requirements.txt +7 -0
  15. tokenaudit-0.2.0/demo/static/index.html +139 -0
  16. tokenaudit-0.2.0/docs/case-studies/tradingagents.md +136 -0
  17. tokenaudit-0.2.0/docs/ci.md +74 -0
  18. tokenaudit-0.2.0/docs/rulebook.md +72 -0
  19. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/pyproject.toml +7 -3
  20. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/__init__.py +1 -1
  21. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/__main__.py +65 -6
  22. tokenaudit-0.2.0/src/tokenaudit/ci/__init__.py +22 -0
  23. tokenaudit-0.2.0/src/tokenaudit/ci/baseline.py +119 -0
  24. tokenaudit-0.2.0/src/tokenaudit/ci/check.py +163 -0
  25. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/sub_minimum.py +10 -1
  26. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/__init__.py +16 -6
  27. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/containers.py +17 -13
  28. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/rulebook/__init__.py +23 -7
  29. tokenaudit-0.2.0/src/tokenaudit/rulebook/anthropic.json +43 -0
  30. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/rulebook/google.json +4 -4
  31. tokenaudit-0.2.0/src/tokenaudit/rulebook/openai.json +170 -0
  32. tokenaudit-0.2.0/tests/test_ci.py +171 -0
  33. tokenaudit-0.2.0/tests/test_demo.py +68 -0
  34. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_rulebook.py +45 -2
  35. tokenaudit-0.2.0/tokenaudit-baseline.json +11 -0
  36. tokenaudit-0.1.0/CHANGELOG.md +0 -27
  37. tokenaudit-0.1.0/docs/rulebook.md +0 -57
  38. tokenaudit-0.1.0/src/tokenaudit/ci/__init__.py +0 -1
  39. tokenaudit-0.1.0/src/tokenaudit/rulebook/anthropic.json +0 -36
  40. tokenaudit-0.1.0/src/tokenaudit/rulebook/openai.json +0 -45
  41. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/.pre-commit-config.yaml +0 -0
  42. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/DECISIONS.md +0 -0
  43. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/LICENSE +0 -0
  44. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/Makefile +0 -0
  45. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/README.md +0 -0
  46. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/README.md +0 -0
  47. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/capture_tradingagents.sh +0 -0
  48. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/redact.py +0 -0
  49. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runner.py +0 -0
  50. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-01-NVDA-2026-08-14/decision.txt +0 -0
  51. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-01-NVDA-2026-08-14/manifest.json +0 -0
  52. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-01-NVDA-2026-08-14/trace.jsonl +0 -0
  53. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-02-AAPL-2026-08-14/decision.txt +0 -0
  54. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-02-AAPL-2026-08-14/manifest.json +0 -0
  55. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-02-AAPL-2026-08-14/trace.jsonl +0 -0
  56. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-03-NVDA-2026-08-21/decision.txt +0 -0
  57. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-03-NVDA-2026-08-21/manifest.json +0 -0
  58. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-03-NVDA-2026-08-21/trace.jsonl +0 -0
  59. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/breakpoint_limit.label.json +0 -0
  60. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/breakpoint_limit.trace.json +0 -0
  61. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/context_bloat.label.json +0 -0
  62. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/context_bloat.trace.json +0 -0
  63. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_cached.label.json +0 -0
  64. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_cached.trace.json +0 -0
  65. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_incremental.label.json +0 -0
  66. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_incremental.trace.json +0 -0
  67. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_small.label.json +0 -0
  68. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_small.trace.json +0 -0
  69. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/lookback_overflow.label.json +0 -0
  70. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/lookback_overflow.trace.json +0 -0
  71. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/manifest.json +0 -0
  72. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/model_switch.label.json +0 -0
  73. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/model_switch.trace.json +0 -0
  74. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/openai_healthy_cached.label.json +0 -0
  75. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/openai_healthy_cached.trace.json +0 -0
  76. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift.label.json +0 -0
  77. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift.trace.json +0 -0
  78. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_legacy_flat.label.json +0 -0
  79. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_legacy_flat.trace.json +0 -0
  80. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_no_content.label.json +0 -0
  81. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_no_content.trace.json +0 -0
  82. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_openinference.label.json +0 -0
  83. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_openinference.trace.json +0 -0
  84. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/retry_storm.label.json +0 -0
  85. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/retry_storm.trace.json +0 -0
  86. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/sub_minimum.label.json +0 -0
  87. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/sub_minimum.trace.json +0 -0
  88. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/tool_schema_churn.label.json +0 -0
  89. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/tool_schema_churn.trace.json +0 -0
  90. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/ttl_expiry.label.json +0 -0
  91. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/ttl_expiry.trace.json +0 -0
  92. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/eval/results.json +0 -0
  93. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/eval/results.md +0 -0
  94. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/eval/run_eval.py +0 -0
  95. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/counterfactual/__init__.py +0 -0
  96. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/__init__.py +0 -0
  97. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/base.py +0 -0
  98. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/corpus_record.json +0 -0
  99. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/engine.py +0 -0
  100. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/findings.py +0 -0
  101. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/__init__.py +0 -0
  102. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/breakpoint_limit.py +0 -0
  103. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/context_bloat.py +0 -0
  104. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/lookback_overflow.py +0 -0
  105. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/model_switch.py +0 -0
  106. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/prefix_drift.py +0 -0
  107. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/retry_storm.py +0 -0
  108. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/tool_schema_churn.py +0 -0
  109. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/ttl_expiry.py +0 -0
  110. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/examples/sample.trace.json +0 -0
  111. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/dedup.py +0 -0
  112. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/dialects.py +0 -0
  113. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/quality.py +0 -0
  114. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/sessionize.py +0 -0
  115. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/model/__init__.py +0 -0
  116. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/model/records.py +0 -0
  117. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/py.typed +0 -0
  118. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/__init__.py +0 -0
  119. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/corpus_record.py +0 -0
  120. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/health.py +0 -0
  121. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/html.py +0 -0
  122. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_corpus.py +0 -0
  123. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_detect.py +0 -0
  124. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_eval.py +0 -0
  125. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_ingest.py +0 -0
  126. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_package.py +0 -0
  127. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_redact.py +0 -0
  128. {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_report.py +0 -0
@@ -0,0 +1,15 @@
1
+ # Keep the demo image lean: the build needs src/, demo/, corpus/, and the
2
+ # packaging files — not local envs, caches, or git history.
3
+ .git
4
+ .venv
5
+ venv
6
+ dist
7
+ build
8
+ *.egg-info
9
+ .mypy_cache
10
+ .pytest_cache
11
+ .ruff_cache
12
+ __pycache__
13
+ docs
14
+ eval
15
+ tests
@@ -19,10 +19,10 @@ jobs:
19
19
  python-version: ${{ matrix.python-version }}
20
20
  cache: pip
21
21
  cache-dependency-path: pyproject.toml
22
- - name: Install package and dev tools
22
+ - name: Install package, dev tools, and demo deps
23
23
  run: |
24
24
  python -m pip install --upgrade pip
25
- pip install -e ".[dev]"
25
+ pip install -e ".[dev]" -r demo/requirements.txt
26
26
  - name: Lint
27
27
  run: ruff check .
28
28
  - name: Format check
@@ -31,3 +31,17 @@ jobs:
31
31
  run: mypy
32
32
  - name: Tests
33
33
  run: pytest
34
+
35
+ # Run the gate on the captured TradingAgents runs through action.yml itself.
36
+ cache-regression:
37
+ runs-on: ubuntu-latest
38
+ steps:
39
+ - uses: actions/checkout@v4
40
+ - uses: actions/setup-python@v5
41
+ with:
42
+ python-version: "3.12"
43
+ - name: tokenaudit check
44
+ uses: ./
45
+ with:
46
+ traces: corpus/captured/runs
47
+ baseline: tokenaudit-baseline.json
@@ -0,0 +1,83 @@
1
+ name: Release
2
+
3
+ # Publishes to PyPI when a vX.Y.Z tag is pushed. Uses trusted publishing:
4
+ # the PyPI project needs a GitHub publisher for release.yml (environment pypi).
5
+
6
+ on:
7
+ push:
8
+ tags: ["v*"]
9
+
10
+ jobs:
11
+ build:
12
+ runs-on: ubuntu-latest
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+ - uses: actions/setup-python@v5
16
+ with:
17
+ python-version: "3.12"
18
+ - name: Check tag matches version
19
+ run: |
20
+ tag="${GITHUB_REF_NAME#v}"
21
+ pkg="$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml","rb"))["project"]["version"])')"
22
+ mod="$(sed -n 's/^__version__ = "\(.*\)"$/\1/p' src/tokenaudit/__init__.py)"
23
+ echo "tag=$tag pyproject=$pkg __version__=$mod"
24
+ test "$tag" = "$pkg" && test "$tag" = "$mod"
25
+ - name: Lint, type check, test
26
+ run: |
27
+ python -m pip install --upgrade pip
28
+ pip install -e ".[dev]" -r demo/requirements.txt
29
+ ruff check .
30
+ ruff format --check .
31
+ mypy
32
+ pytest
33
+ - name: Build
34
+ run: |
35
+ pip install build
36
+ python -m build
37
+ - name: Smoke test wheel
38
+ run: |
39
+ python -m venv /tmp/smoke
40
+ /tmp/smoke/bin/pip install dist/*.whl
41
+ /tmp/smoke/bin/tokenaudit --version
42
+ /tmp/smoke/bin/tokenaudit diagnose --sample > /dev/null
43
+ /tmp/smoke/bin/tokenaudit check corpus/captured/runs --baseline tokenaudit-baseline.json
44
+ - uses: actions/upload-artifact@v4
45
+ with:
46
+ name: dist
47
+ path: dist/
48
+
49
+ publish:
50
+ needs: build
51
+ runs-on: ubuntu-latest
52
+ environment: pypi
53
+ permissions:
54
+ id-token: write
55
+ steps:
56
+ - uses: actions/download-artifact@v4
57
+ with:
58
+ name: dist
59
+ path: dist/
60
+ - uses: pypa/gh-action-pypi-publish@release/v1
61
+
62
+ github-release:
63
+ needs: publish
64
+ runs-on: ubuntu-latest
65
+ permissions:
66
+ contents: write
67
+ steps:
68
+ - uses: actions/checkout@v4
69
+ - uses: actions/download-artifact@v4
70
+ with:
71
+ name: dist
72
+ path: dist/
73
+ - name: Create GitHub release
74
+ env:
75
+ GH_TOKEN: ${{ github.token }}
76
+ run: |
77
+ version="${GITHUB_REF_NAME#v}"
78
+ awk -v v="$version" '
79
+ $0 ~ "^## " v "( |$)" { on = 1; next }
80
+ on && /^## / { exit }
81
+ on { print }
82
+ ' CHANGELOG.md > notes.md
83
+ gh release create "$GITHUB_REF_NAME" dist/* --title "$GITHUB_REF_NAME" --notes-file notes.md
@@ -22,3 +22,6 @@ htmlcov/
22
22
 
23
23
  # Capture workdir (throwaway venv + clone, never committed)
24
24
  corpus/captured/.work/
25
+
26
+ # Local TradingAgents clone (separate repo)
27
+ /TradingAgents/
@@ -0,0 +1,58 @@
1
+ # Changelog
2
+
3
+ ## Unreleased
4
+
5
+ ## 0.2.0 — 2026-11-10
6
+
7
+ - Fix the GitHub Action for external users. It installed the latest PyPI
8
+ release, and 0.1.0 has no `check` command. It now installs the code at the
9
+ ref it is used from, so `@v0.2.0` runs 0.2.0. The `version` input still
10
+ pins a PyPI release.
11
+ - Rulebook re-verified against current provider docs (version 2026.11.10):
12
+ - Anthropic: Opus, Sonnet and Haiku 5.5, Fable and Mythos 5.1/5.2.
13
+ Per-model read multipliers (0.05x and 0.025x) and base input prices.
14
+ - OpenAI: GPT-5.6 sizes, GPT-6 and GPT-6.1, 12h TTL. Pre-5.6 models no
15
+ longer publish a fixed minimum, so it is recorded as unknown.
16
+ - Google: added Gemini 4 Argon and 3.8 Flash; removed 3.7 and 3.5 Flash,
17
+ which are no longer in Google's table.
18
+ - `sub_minimum` abstains when a model has no published minimum.
19
+ - Release workflow: pushing a `vX.Y.Z` tag publishes to PyPI.
20
+ - The sdist no longer includes the experiment traces.
21
+ - **`ingest_text(text, source)`**: ingest trace content already in memory
22
+ (an upload, a stream) — nothing touches disk. `ingest_path` now wraps it.
23
+ - **Hosted demo** (`demo/`, not part of the package): a one-page FastAPI
24
+ app — drop a trace, get the report; two prerendered samples (the
25
+ synthetic prefix-drift fixture and the real captured TradingAgents
26
+ runs). Uploads are parsed in memory and never stored or logged.
27
+ - **CI cost-regression gate**: `tokenaudit check <traces> --baseline
28
+ baseline.json` fails a build (exit 1) on a new HIGH finding or a drop in
29
+ cache read rate / health floor beyond `--max-regression` points; create
30
+ or refresh the baseline with `--update`. Ships as a composite GitHub
31
+ Action (`action.yml`) with a markdown summary written to the job page
32
+ (and to a file via `--markdown`, for PR comments). See `docs/ci.md`.
33
+
34
+ ## 0.1.0 — 2026-08-30
35
+
36
+ First usable release.
37
+
38
+ - **Ingest** four real trace dialects into one normalized accounting model:
39
+ OTel GenAI current semconv, legacy flat names (older openllmetry),
40
+ OpenInference, and the OTel Python SDK's `to_json()` JSONL — with
41
+ layered-span deduplication (framework + SDK instrumentors double-report
42
+ the same call), a data-quality gate that refuses to price malformed
43
+ telemetry, and prompt privacy by construction (block hashes and lengths,
44
+ never text).
45
+ - **Nine deterministic detectors** — prefix drift, sub-minimum cache
46
+ blocks, model switch, TTL expiry, breakpoint limit, lookback overflow,
47
+ tool-schema churn, context bloat, retry storm — each with evidence
48
+ pointers, bounded impact ranges with stated assumptions, and abstention
49
+ (with a reason) as a first-class outcome distinct from "checked, clean".
50
+ - **Provider caching rulebook** packaged as sourced, dated JSON (Anthropic,
51
+ OpenAI, Google), with a CI staleness alarm and a rendered comparison
52
+ table (`tokenaudit rulebook`).
53
+ - **Evaluation** against a byte-reproducible labeled corpus, published in
54
+ `eval/results.md`; per-detector corpus records ship inside the package
55
+ and appear on findings.
56
+ - **CLI**: `tokenaudit diagnose` (text, `--json`, self-contained `--html`
57
+ report, `--sample`), `tokenaudit ingest`, `tokenaudit rulebook`.
58
+ - Zero runtime dependencies. Python 3.11+.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: tokenaudit
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Root-cause diagnosis of wasted LLM tokens, from the traces your agents already emit
5
5
  Project-URL: Repository, https://github.com/yashmyeole/tokenaudit
6
6
  Project-URL: Issues, https://github.com/yashmyeole/tokenaudit/issues
@@ -8,7 +8,7 @@ Author-email: Yash Manohar Yeole <yashmyeole@gmail.com>
8
8
  License-Expression: Apache-2.0
9
9
  License-File: LICENSE
10
10
  Keywords: cost,diagnosis,llm,observability,opentelemetry,prompt-caching,tokens,waste
11
- Classifier: Development Status :: 2 - Pre-Alpha
11
+ Classifier: Development Status :: 3 - Alpha
12
12
  Classifier: Intended Audience :: Developers
13
13
  Classifier: Operating System :: OS Independent
14
14
  Classifier: Programming Language :: Python :: 3
@@ -29,12 +29,14 @@ Description-Content-Type: text/markdown
29
29
 
30
30
  **Root-cause diagnosis of wasted LLM tokens — from the traces your agents already emit.**
31
31
 
32
- > Status: early development, pre-0.1. Interfaces will change.
32
+ > Status: alpha (0.2.0). Interfaces may change before 1.0.
33
33
 
34
34
  Every LLM cost tool can tell you *how much* you spent and *who* spent it. Almost none can tell you *why a specific request wasted money*. The biggest lever on an LLM bill — prompt caching, up to a 10× discount on repeated input — fails silently: change one byte of a cached prefix, drop below a model's minimum cacheable size, let a timer expire, and the provider just bills you full price. No error, no warning. There are [documented](https://barndoor.ai/what-happens-to-your-llm-bill-when-prompt-caching-fails/) [cases](https://github.com/TauricResearch/TradingAgents/issues/750) of agents running at ~0% cache hit rate for months, at 30–40% unnecessary cost, because of a single line of prompt construction.
35
35
 
36
36
  tokenaudit reads the traces agents already produce and, for each expensive request, answers four questions: **what was wasted, why, what it cost, and what to change.**
37
37
 
38
+ It works: diagnosing one of those documented cases produced a 9-file patch that took TradingAgents' cache hit rate from **7.5% to 32.4% (4.3×)** in a controlled, pre-registered before/after experiment — the [first case study](docs/case-studies/tradingagents.md).
39
+
38
40
  ## Quickstart
39
41
 
40
42
  pip install tokenaudit
@@ -43,6 +45,8 @@ tokenaudit reads the traces agents already produce and, for each expensive reque
43
45
 
44
46
  Zero runtime dependencies; reads OTLP/JSON files and OTel SDK JSONL. Try it
45
47
  on this repo's real captured runs: `tokenaudit diagnose corpus/captured/runs`.
48
+ Prefer a browser? The [demo app](demo/) diagnoses a dropped trace; uploads
49
+ are parsed in memory and never stored.
46
50
 
47
51
  The plan, in one line each:
48
52
 
@@ -51,6 +55,22 @@ The plan, in one line each:
51
55
  - Price each finding against a counterfactual — what the bill would have been under the fix — as a bounded range, never a point guess.
52
56
  - Publish per-detector precision and recall against a labeled corpus, so you know how much to trust each finding.
53
57
 
58
+ ## Catch regressions in CI
59
+
60
+ Prompt-caching regressions are silent: no test fails when a reordered tool
61
+ list doubles your input bill. If CI already produces traces, fail the pull
62
+ request instead ([full docs](docs/ci.md)):
63
+
64
+ ```yaml
65
+ - uses: yashmyeole/tokenaudit@v0.2.0
66
+ with:
67
+ traces: traces/
68
+ baseline: tokenaudit-baseline.json # tokenaudit check traces/ --baseline ... --update
69
+ ```
70
+
71
+ This repo gates itself: every build re-diagnoses the captured
72
+ TradingAgents runs against [tokenaudit-baseline.json](tokenaudit-baseline.json).
73
+
54
74
  ## Prior art
55
75
 
56
76
  This space is crowded, and being precise about what already exists is the point of this section.
@@ -96,8 +116,13 @@ If you know a tool that already does this, please open an issue — I would genu
96
116
  - [x] Detectors, starting with prefix drift, sub-minimum blocks, and model/version switch
97
117
  - [x] Evaluation harness: per-detector results published in [eval/results.md](eval/results.md)
98
118
  - [x] `tokenaudit diagnose` CLI, HTML report, cache health score
99
- - [ ] CI cost-regression mode: fail a build that tanks the cache hit rate
100
- - [ ] Case studies on public agents, with reproduction commands
119
+ - [x] CI cost-regression mode: fail a build that tanks the cache hit rate ([docs/ci.md](docs/ci.md))
120
+ - [x] Demo app: drag a trace in, get the report ([demo/](demo/))
121
+ - [x] Case studies on public agents, with reproduction commands ([TradingAgents: 4.3× cache hit rate](docs/case-studies/tradingagents.md))
122
+ - [ ] Detectors for misordered shared context and cross-run prefix drift
123
+ - [ ] Counterfactual pricing with per-model read rates and dollar ranges
124
+ - [ ] Raw provider SDK log ingest, tool-schema bloat detector
125
+ - [ ] Public demo deployment, case study on an Anthropic-model agent
101
126
 
102
127
  ## Development
103
128
 
@@ -106,6 +131,9 @@ If you know a tool that already does this, please open an issue — I would genu
106
131
  pre-commit install
107
132
  pytest
108
133
 
134
+ To release, bump the version in `pyproject.toml` and `src/tokenaudit/__init__.py`,
135
+ update CHANGELOG.md, and push a `vX.Y.Z` tag.
136
+
109
137
  ## License
110
138
 
111
139
  Apache-2.0
@@ -2,12 +2,14 @@
2
2
 
3
3
  **Root-cause diagnosis of wasted LLM tokens — from the traces your agents already emit.**
4
4
 
5
- > Status: early development, pre-0.1. Interfaces will change.
5
+ > Status: alpha (0.2.0). Interfaces may change before 1.0.
6
6
 
7
7
  Every LLM cost tool can tell you *how much* you spent and *who* spent it. Almost none can tell you *why a specific request wasted money*. The biggest lever on an LLM bill — prompt caching, up to a 10× discount on repeated input — fails silently: change one byte of a cached prefix, drop below a model's minimum cacheable size, let a timer expire, and the provider just bills you full price. No error, no warning. There are [documented](https://barndoor.ai/what-happens-to-your-llm-bill-when-prompt-caching-fails/) [cases](https://github.com/TauricResearch/TradingAgents/issues/750) of agents running at ~0% cache hit rate for months, at 30–40% unnecessary cost, because of a single line of prompt construction.
8
8
 
9
9
  tokenaudit reads the traces agents already produce and, for each expensive request, answers four questions: **what was wasted, why, what it cost, and what to change.**
10
10
 
11
+ It works: diagnosing one of those documented cases produced a 9-file patch that took TradingAgents' cache hit rate from **7.5% to 32.4% (4.3×)** in a controlled, pre-registered before/after experiment — the [first case study](docs/case-studies/tradingagents.md).
12
+
11
13
  ## Quickstart
12
14
 
13
15
  pip install tokenaudit
@@ -16,6 +18,8 @@ tokenaudit reads the traces agents already produce and, for each expensive reque
16
18
 
17
19
  Zero runtime dependencies; reads OTLP/JSON files and OTel SDK JSONL. Try it
18
20
  on this repo's real captured runs: `tokenaudit diagnose corpus/captured/runs`.
21
+ Prefer a browser? The [demo app](demo/) diagnoses a dropped trace; uploads
22
+ are parsed in memory and never stored.
19
23
 
20
24
  The plan, in one line each:
21
25
 
@@ -24,6 +28,22 @@ The plan, in one line each:
24
28
  - Price each finding against a counterfactual — what the bill would have been under the fix — as a bounded range, never a point guess.
25
29
  - Publish per-detector precision and recall against a labeled corpus, so you know how much to trust each finding.
26
30
 
31
+ ## Catch regressions in CI
32
+
33
+ Prompt-caching regressions are silent: no test fails when a reordered tool
34
+ list doubles your input bill. If CI already produces traces, fail the pull
35
+ request instead ([full docs](docs/ci.md)):
36
+
37
+ ```yaml
38
+ - uses: yashmyeole/tokenaudit@v0.2.0
39
+ with:
40
+ traces: traces/
41
+ baseline: tokenaudit-baseline.json # tokenaudit check traces/ --baseline ... --update
42
+ ```
43
+
44
+ This repo gates itself: every build re-diagnoses the captured
45
+ TradingAgents runs against [tokenaudit-baseline.json](tokenaudit-baseline.json).
46
+
27
47
  ## Prior art
28
48
 
29
49
  This space is crowded, and being precise about what already exists is the point of this section.
@@ -69,8 +89,13 @@ If you know a tool that already does this, please open an issue — I would genu
69
89
  - [x] Detectors, starting with prefix drift, sub-minimum blocks, and model/version switch
70
90
  - [x] Evaluation harness: per-detector results published in [eval/results.md](eval/results.md)
71
91
  - [x] `tokenaudit diagnose` CLI, HTML report, cache health score
72
- - [ ] CI cost-regression mode: fail a build that tanks the cache hit rate
73
- - [ ] Case studies on public agents, with reproduction commands
92
+ - [x] CI cost-regression mode: fail a build that tanks the cache hit rate ([docs/ci.md](docs/ci.md))
93
+ - [x] Demo app: drag a trace in, get the report ([demo/](demo/))
94
+ - [x] Case studies on public agents, with reproduction commands ([TradingAgents: 4.3× cache hit rate](docs/case-studies/tradingagents.md))
95
+ - [ ] Detectors for misordered shared context and cross-run prefix drift
96
+ - [ ] Counterfactual pricing with per-model read rates and dollar ranges
97
+ - [ ] Raw provider SDK log ingest, tool-schema bloat detector
98
+ - [ ] Public demo deployment, case study on an Anthropic-model agent
74
99
 
75
100
  ## Development
76
101
 
@@ -79,6 +104,9 @@ If you know a tool that already does this, please open an issue — I would genu
79
104
  pre-commit install
80
105
  pytest
81
106
 
107
+ To release, bump the version in `pyproject.toml` and `src/tokenaudit/__init__.py`,
108
+ update CHANGELOG.md, and push a `vX.Y.Z` tag.
109
+
82
110
  ## License
83
111
 
84
112
  Apache-2.0
@@ -0,0 +1,60 @@
1
+ # Composite GitHub Action wrapping `tokenaudit check`.
2
+ #
3
+ # The CLI already appends its markdown summary to $GITHUB_STEP_SUMMARY when
4
+ # that variable is set (it is, inside Actions), so the job's summary page
5
+ # shows the baseline-vs-current table with no extra wiring here.
6
+ name: "tokenaudit check"
7
+ description: >-
8
+ Fail a build that silently made LLM prompt caching worse: diagnoses a
9
+ trace run and compares cache health against a baseline committed to
10
+ the repo.
11
+ branding:
12
+ icon: "trending-down"
13
+ color: "orange"
14
+ inputs:
15
+ traces:
16
+ description: "Trace file or directory to diagnose (OTLP/JSON or OTel SDK JSONL)"
17
+ required: true
18
+ baseline:
19
+ description: "Path to the committed baseline JSON"
20
+ required: false
21
+ default: "tokenaudit-baseline.json"
22
+ max-regression:
23
+ description: "Allowed drop in cache read rate / health floor, in percentage points"
24
+ required: false
25
+ default: "5.0"
26
+ strict:
27
+ description: "Also fail on new non-HIGH findings (true/false)"
28
+ required: false
29
+ default: "false"
30
+ version:
31
+ description: "Install this PyPI release instead of the code at the action's ref"
32
+ required: false
33
+ default: ""
34
+ runs:
35
+ using: "composite"
36
+ steps:
37
+ - name: Install tokenaudit
38
+ shell: bash
39
+ env:
40
+ TOKENAUDIT_VERSION: ${{ inputs.version }}
41
+ ACTION_PATH: ${{ github.action_path }}
42
+ run: |
43
+ if [ -n "$TOKENAUDIT_VERSION" ]; then
44
+ python -m pip install --quiet "tokenaudit==$TOKENAUDIT_VERSION"
45
+ else
46
+ python -m pip install --quiet "$ACTION_PATH"
47
+ fi
48
+ tokenaudit --version
49
+ - name: Check for cache cost regression
50
+ shell: bash
51
+ env:
52
+ TRACES: ${{ inputs.traces }}
53
+ BASELINE: ${{ inputs.baseline }}
54
+ MAX_REGRESSION: ${{ inputs.max-regression }}
55
+ STRICT: ${{ inputs.strict }}
56
+ run: |
57
+ extra=""
58
+ if [ "$STRICT" = "true" ]; then extra="--strict"; fi
59
+ tokenaudit check "$TRACES" --baseline "$BASELINE" \
60
+ --max-regression "$MAX_REGRESSION" $extra
@@ -79,7 +79,9 @@ ANTHROPIC_MAX_BREAKPOINTS: int = _ANTHROPIC["max_explicit_breakpoints"]
79
79
  ANTHROPIC_LOOKBACK_POSITIONS: int = _ANTHROPIC["lookback_positions"]
80
80
 
81
81
  OPENAI_MIN_CACHEABLE: dict[str, int] = {
82
- model: entry["min_cacheable_tokens"] for model, entry in _OPENAI["models"].items()
82
+ model: entry["min_cacheable_tokens"]
83
+ for model, entry in _OPENAI["models"].items()
84
+ if entry["min_cacheable_tokens"] is not None
83
85
  }
84
86
  OPENAI_TTL_S: int = _OPENAI["models"]["gpt-5.6"]["ttl"][0]["seconds"] # sliding from last use
85
87
 
@@ -0,0 +1,17 @@
1
+ # Build from the REPO ROOT (the app prerenders reports from the captured
2
+ # runs in corpus/): docker build -f demo/Dockerfile -t tokenaudit-demo .
3
+ FROM python:3.12-slim
4
+
5
+ WORKDIR /app
6
+
7
+ # Layer the dependency install before the source copy so code-only edits
8
+ # rebuild in seconds.
9
+ COPY demo/requirements.txt demo/requirements.txt
10
+ RUN pip install --no-cache-dir -r demo/requirements.txt
11
+
12
+ COPY . .
13
+ RUN pip install --no-cache-dir .
14
+
15
+ EXPOSE 8000
16
+ # Render/Railway inject $PORT; default to 8000 for local docker run.
17
+ CMD uvicorn demo.app:app --host 0.0.0.0 --port ${PORT:-8000}
@@ -0,0 +1,44 @@
1
+ # Hosted demo
2
+
3
+ A one-page FastAPI app: drop a trace, get the diagnosis report back.
4
+ Uploads are parsed in memory with `tokenaudit.ingest.ingest_text` and never
5
+ written to disk, stored, or logged; the two sample reports are prerendered
6
+ at startup from files already public in this repo. The demo is not part of
7
+ the pip package — the library keeps zero runtime dependencies.
8
+
9
+ ## Run locally
10
+
11
+ pip install -e . -r demo/requirements.txt
12
+ uvicorn demo.app:app --reload
13
+ # open http://127.0.0.1:8000
14
+
15
+ Or with Docker (from the repo root — the image needs `corpus/` for the
16
+ real-capture sample):
17
+
18
+ docker build -f demo/Dockerfile -t tokenaudit-demo .
19
+ docker run --rm -p 8000:8000 tokenaudit-demo
20
+
21
+ ## Deploy on Render (free tier)
22
+
23
+ 1. render.com → New → **Web Service** → connect `yashmyeole/tokenaudit`.
24
+ 2. Language: **Docker**; Dockerfile path: `demo/Dockerfile`;
25
+ root directory: leave at repo root.
26
+ 3. Instance type: **Free**. Create the service.
27
+ 4. First build takes a few minutes; the URL looks like
28
+ `https://tokenaudit-demo.onrender.com`.
29
+
30
+ Free instances sleep after ~15 minutes idle, so the first visit after a
31
+ quiet spell takes ~30–60 s to wake — fine for a portfolio demo, and worth
32
+ a one-line note next to the link. Railway and Fly.io both deploy the same
33
+ Dockerfile if you prefer them (both currently want a card on file).
34
+
35
+ ## Endpoints
36
+
37
+ - `GET /` — the page
38
+ - `GET /api/sample/drift` — prerendered report for the bundled synthetic
39
+ prefix-drift fixture (labeled as synthetic)
40
+ - `GET /api/sample/tradingagents` — prerendered report for the three real
41
+ captured TradingAgents runs
42
+ - `POST /api/diagnose` — multipart upload (`traces`, up to 20 files /
43
+ 15 MB total), returns the self-contained HTML report; 422 with the
44
+ loud refusal message for unrecognized formats
@@ -0,0 +1 @@
1
+ """The hosted demo app (not part of the tokenaudit package)."""
@@ -0,0 +1,113 @@
1
+ """The hosted demo: drag a trace in, get the diagnosis back. Nothing stored.
2
+
3
+ Privacy is a design property here, not a promise: uploads are handled with
4
+ :func:`tokenaudit.ingest.ingest_text`, which parses the bytes in memory —
5
+ there is no temp file, no persistence layer to configure away, and the app
6
+ never logs request bodies. The two sample reports are prerendered at
7
+ startup from files already public in this repo.
8
+
9
+ The demo is deliberately not part of the ``tokenaudit`` package (the
10
+ library keeps zero runtime dependencies); it is a separate FastAPI app
11
+ deployed from ``demo/Dockerfile``.
12
+
13
+ Run locally: pip install -e . -r demo/requirements.txt
14
+ uvicorn demo.app:app --reload
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from collections.abc import Awaitable, Callable
20
+ from importlib.resources import files
21
+ from pathlib import Path
22
+
23
+ from fastapi import FastAPI, HTTPException, Request, Response, UploadFile
24
+ from fastapi.responses import FileResponse, HTMLResponse
25
+
26
+ from tokenaudit.detect import diagnose
27
+ from tokenaudit.ingest import UnknownTraceFormatError, ingest_text
28
+ from tokenaudit.report import render_html
29
+ from tokenaudit.rulebook import load_rulebook
30
+
31
+ MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # per request, checked before parsing
32
+ MAX_FILES = 20
33
+
34
+ _DEMO_DIR = Path(__file__).resolve().parent
35
+ _REPO_ROOT = _DEMO_DIR.parent
36
+ _CAPTURED = _REPO_ROOT / "corpus" / "captured" / "runs" / "tradingagents"
37
+
38
+ app = FastAPI(title="tokenaudit demo", docs_url=None, redoc_url=None, openapi_url=None)
39
+
40
+
41
+ def _diagnose_to_html(named_texts: list[tuple[str, str]]) -> str:
42
+ """Ingest + diagnose in memory and render the self-contained report."""
43
+ rulebook = load_rulebook()
44
+ per_result = []
45
+ for name, text in named_texts:
46
+ result = ingest_text(text, name)
47
+ per_result.append((result, diagnose(result)))
48
+ return render_html(per_result, rulebook)
49
+
50
+
51
+ def _prerender_samples() -> dict[str, str]:
52
+ samples: dict[str, str] = {}
53
+ drift = files("tokenaudit.examples").joinpath("sample.trace.json")
54
+ samples["drift"] = _diagnose_to_html(
55
+ [("sample.trace.json (synthetic prefix-drift demo)", drift.read_text(encoding="utf-8"))]
56
+ )
57
+ if _CAPTURED.is_dir():
58
+ captured = [
59
+ (str(p.relative_to(_REPO_ROOT)), p.read_text(encoding="utf-8"))
60
+ for p in sorted(_CAPTURED.rglob("*.jsonl"))
61
+ ]
62
+ if captured:
63
+ samples["tradingagents"] = _diagnose_to_html(captured)
64
+ return samples
65
+
66
+
67
+ _SAMPLES = _prerender_samples()
68
+
69
+
70
+ @app.middleware("http")
71
+ async def security_headers(
72
+ request: Request, call_next: Callable[[Request], Awaitable[Response]]
73
+ ) -> Response:
74
+ response = await call_next(request)
75
+ response.headers["X-Content-Type-Options"] = "nosniff"
76
+ response.headers["Referrer-Policy"] = "no-referrer"
77
+ return response
78
+
79
+
80
+ @app.get("/", include_in_schema=False)
81
+ async def index() -> FileResponse:
82
+ return FileResponse(_DEMO_DIR / "static" / "index.html", media_type="text/html")
83
+
84
+
85
+ @app.get("/api/sample/{name}")
86
+ async def sample(name: str) -> HTMLResponse:
87
+ if name not in _SAMPLES:
88
+ raise HTTPException(404, f"no sample named {name!r}")
89
+ return HTMLResponse(_SAMPLES[name])
90
+
91
+
92
+ @app.post("/api/diagnose")
93
+ async def diagnose_upload(traces: list[UploadFile]) -> HTMLResponse:
94
+ if not traces:
95
+ raise HTTPException(400, "no files uploaded")
96
+ if len(traces) > MAX_FILES:
97
+ raise HTTPException(413, f"too many files (max {MAX_FILES})")
98
+ named_texts: list[tuple[str, str]] = []
99
+ total = 0
100
+ for upload in traces:
101
+ data = await upload.read()
102
+ total += len(data)
103
+ if total > MAX_UPLOAD_BYTES:
104
+ raise HTTPException(413, f"upload exceeds {MAX_UPLOAD_BYTES // (1024 * 1024)} MB")
105
+ try:
106
+ text = data.decode("utf-8")
107
+ except UnicodeDecodeError as exc:
108
+ raise HTTPException(400, f"{upload.filename}: not UTF-8 text ({exc})") from exc
109
+ named_texts.append((upload.filename or "upload", text))
110
+ try:
111
+ return HTMLResponse(_diagnose_to_html(named_texts))
112
+ except UnknownTraceFormatError as exc:
113
+ raise HTTPException(422, str(exc)) from exc
@@ -0,0 +1,7 @@
1
+ # Dependencies of the hosted demo app ONLY — the tokenaudit package itself
2
+ # stays zero-dependency. httpx is here for FastAPI's TestClient (used by
3
+ # tests/test_demo.py and handy for poking the app locally).
4
+ fastapi>=0.115
5
+ python-multipart>=0.0.9 # FastAPI needs it for multipart file uploads
6
+ uvicorn>=0.30
7
+ httpx2>=0.1 # FastAPI's TestClient backend (tests/test_demo.py, local poking)