tokenaudit 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokenaudit-0.2.0/.dockerignore +15 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/.github/workflows/ci.yml +16 -2
- tokenaudit-0.2.0/.github/workflows/release.yml +83 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/.gitignore +3 -0
- tokenaudit-0.2.0/CHANGELOG.md +58 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/PKG-INFO +33 -5
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/README.md +31 -3
- tokenaudit-0.2.0/action.yml +60 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/generate.py +3 -1
- tokenaudit-0.2.0/demo/Dockerfile +17 -0
- tokenaudit-0.2.0/demo/README.md +44 -0
- tokenaudit-0.2.0/demo/__init__.py +1 -0
- tokenaudit-0.2.0/demo/app.py +113 -0
- tokenaudit-0.2.0/demo/requirements.txt +7 -0
- tokenaudit-0.2.0/demo/static/index.html +139 -0
- tokenaudit-0.2.0/docs/case-studies/tradingagents.md +136 -0
- tokenaudit-0.2.0/docs/ci.md +74 -0
- tokenaudit-0.2.0/docs/rulebook.md +72 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/pyproject.toml +7 -3
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/__init__.py +1 -1
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/__main__.py +65 -6
- tokenaudit-0.2.0/src/tokenaudit/ci/__init__.py +22 -0
- tokenaudit-0.2.0/src/tokenaudit/ci/baseline.py +119 -0
- tokenaudit-0.2.0/src/tokenaudit/ci/check.py +163 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/sub_minimum.py +10 -1
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/__init__.py +16 -6
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/containers.py +17 -13
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/rulebook/__init__.py +23 -7
- tokenaudit-0.2.0/src/tokenaudit/rulebook/anthropic.json +43 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/rulebook/google.json +4 -4
- tokenaudit-0.2.0/src/tokenaudit/rulebook/openai.json +170 -0
- tokenaudit-0.2.0/tests/test_ci.py +171 -0
- tokenaudit-0.2.0/tests/test_demo.py +68 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_rulebook.py +45 -2
- tokenaudit-0.2.0/tokenaudit-baseline.json +11 -0
- tokenaudit-0.1.0/CHANGELOG.md +0 -27
- tokenaudit-0.1.0/docs/rulebook.md +0 -57
- tokenaudit-0.1.0/src/tokenaudit/ci/__init__.py +0 -1
- tokenaudit-0.1.0/src/tokenaudit/rulebook/anthropic.json +0 -36
- tokenaudit-0.1.0/src/tokenaudit/rulebook/openai.json +0 -45
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/.pre-commit-config.yaml +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/DECISIONS.md +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/LICENSE +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/Makefile +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/README.md +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/README.md +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/capture_tradingagents.sh +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/redact.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runner.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-01-NVDA-2026-08-14/decision.txt +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-01-NVDA-2026-08-14/manifest.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-01-NVDA-2026-08-14/trace.jsonl +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-02-AAPL-2026-08-14/decision.txt +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-02-AAPL-2026-08-14/manifest.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-02-AAPL-2026-08-14/trace.jsonl +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-03-NVDA-2026-08-21/decision.txt +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-03-NVDA-2026-08-21/manifest.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/captured/runs/tradingagents/run-03-NVDA-2026-08-21/trace.jsonl +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/breakpoint_limit.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/breakpoint_limit.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/context_bloat.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/context_bloat.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_cached.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_cached.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_incremental.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_incremental.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_small.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/healthy_small.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/lookback_overflow.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/lookback_overflow.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/manifest.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/model_switch.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/model_switch.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/openai_healthy_cached.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/openai_healthy_cached.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_legacy_flat.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_legacy_flat.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_no_content.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_no_content.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_openinference.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/prefix_drift_openinference.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/retry_storm.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/retry_storm.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/sub_minimum.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/sub_minimum.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/tool_schema_churn.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/tool_schema_churn.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/ttl_expiry.label.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/corpus/synthetic/fixtures/ttl_expiry.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/eval/results.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/eval/results.md +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/eval/run_eval.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/counterfactual/__init__.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/__init__.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/base.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/corpus_record.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/engine.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/findings.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/__init__.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/breakpoint_limit.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/context_bloat.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/lookback_overflow.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/model_switch.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/prefix_drift.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/retry_storm.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/tool_schema_churn.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/detect/rules/ttl_expiry.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/examples/sample.trace.json +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/dedup.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/dialects.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/quality.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/ingest/sessionize.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/model/__init__.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/model/records.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/py.typed +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/__init__.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/corpus_record.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/health.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/src/tokenaudit/report/html.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_corpus.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_detect.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_eval.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_ingest.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_package.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_redact.py +0 -0
- {tokenaudit-0.1.0 → tokenaudit-0.2.0}/tests/test_report.py +0 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Keep the demo image lean: the build needs src/, demo/, corpus/, and the
|
|
2
|
+
# packaging files — not local envs, caches, or git history.
|
|
3
|
+
.git
|
|
4
|
+
.venv
|
|
5
|
+
venv
|
|
6
|
+
dist
|
|
7
|
+
build
|
|
8
|
+
*.egg-info
|
|
9
|
+
.mypy_cache
|
|
10
|
+
.pytest_cache
|
|
11
|
+
.ruff_cache
|
|
12
|
+
__pycache__
|
|
13
|
+
docs
|
|
14
|
+
eval
|
|
15
|
+
tests
|
|
@@ -19,10 +19,10 @@ jobs:
|
|
|
19
19
|
python-version: ${{ matrix.python-version }}
|
|
20
20
|
cache: pip
|
|
21
21
|
cache-dependency-path: pyproject.toml
|
|
22
|
-
- name: Install package and
|
|
22
|
+
- name: Install package, dev tools, and demo deps
|
|
23
23
|
run: |
|
|
24
24
|
python -m pip install --upgrade pip
|
|
25
|
-
pip install -e ".[dev]"
|
|
25
|
+
pip install -e ".[dev]" -r demo/requirements.txt
|
|
26
26
|
- name: Lint
|
|
27
27
|
run: ruff check .
|
|
28
28
|
- name: Format check
|
|
@@ -31,3 +31,17 @@ jobs:
|
|
|
31
31
|
run: mypy
|
|
32
32
|
- name: Tests
|
|
33
33
|
run: pytest
|
|
34
|
+
|
|
35
|
+
# Run the gate on the captured TradingAgents runs through action.yml itself.
|
|
36
|
+
cache-regression:
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v4
|
|
40
|
+
- uses: actions/setup-python@v5
|
|
41
|
+
with:
|
|
42
|
+
python-version: "3.12"
|
|
43
|
+
- name: tokenaudit check
|
|
44
|
+
uses: ./
|
|
45
|
+
with:
|
|
46
|
+
traces: corpus/captured/runs
|
|
47
|
+
baseline: tokenaudit-baseline.json
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Publishes to PyPI when a vX.Y.Z tag is pushed. Uses trusted publishing:
|
|
4
|
+
# the PyPI project needs a GitHub publisher for release.yml (environment pypi).
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
tags: ["v*"]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.12"
|
|
18
|
+
- name: Check tag matches version
|
|
19
|
+
run: |
|
|
20
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
21
|
+
pkg="$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml","rb"))["project"]["version"])')"
|
|
22
|
+
mod="$(sed -n 's/^__version__ = "\(.*\)"$/\1/p' src/tokenaudit/__init__.py)"
|
|
23
|
+
echo "tag=$tag pyproject=$pkg __version__=$mod"
|
|
24
|
+
test "$tag" = "$pkg" && test "$tag" = "$mod"
|
|
25
|
+
- name: Lint, type check, test
|
|
26
|
+
run: |
|
|
27
|
+
python -m pip install --upgrade pip
|
|
28
|
+
pip install -e ".[dev]" -r demo/requirements.txt
|
|
29
|
+
ruff check .
|
|
30
|
+
ruff format --check .
|
|
31
|
+
mypy
|
|
32
|
+
pytest
|
|
33
|
+
- name: Build
|
|
34
|
+
run: |
|
|
35
|
+
pip install build
|
|
36
|
+
python -m build
|
|
37
|
+
- name: Smoke test wheel
|
|
38
|
+
run: |
|
|
39
|
+
python -m venv /tmp/smoke
|
|
40
|
+
/tmp/smoke/bin/pip install dist/*.whl
|
|
41
|
+
/tmp/smoke/bin/tokenaudit --version
|
|
42
|
+
/tmp/smoke/bin/tokenaudit diagnose --sample > /dev/null
|
|
43
|
+
/tmp/smoke/bin/tokenaudit check corpus/captured/runs --baseline tokenaudit-baseline.json
|
|
44
|
+
- uses: actions/upload-artifact@v4
|
|
45
|
+
with:
|
|
46
|
+
name: dist
|
|
47
|
+
path: dist/
|
|
48
|
+
|
|
49
|
+
publish:
|
|
50
|
+
needs: build
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
environment: pypi
|
|
53
|
+
permissions:
|
|
54
|
+
id-token: write
|
|
55
|
+
steps:
|
|
56
|
+
- uses: actions/download-artifact@v4
|
|
57
|
+
with:
|
|
58
|
+
name: dist
|
|
59
|
+
path: dist/
|
|
60
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
61
|
+
|
|
62
|
+
github-release:
|
|
63
|
+
needs: publish
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
permissions:
|
|
66
|
+
contents: write
|
|
67
|
+
steps:
|
|
68
|
+
- uses: actions/checkout@v4
|
|
69
|
+
- uses: actions/download-artifact@v4
|
|
70
|
+
with:
|
|
71
|
+
name: dist
|
|
72
|
+
path: dist/
|
|
73
|
+
- name: Create GitHub release
|
|
74
|
+
env:
|
|
75
|
+
GH_TOKEN: ${{ github.token }}
|
|
76
|
+
run: |
|
|
77
|
+
version="${GITHUB_REF_NAME#v}"
|
|
78
|
+
awk -v v="$version" '
|
|
79
|
+
$0 ~ "^## " v "( |$)" { on = 1; next }
|
|
80
|
+
on && /^## / { exit }
|
|
81
|
+
on { print }
|
|
82
|
+
' CHANGELOG.md > notes.md
|
|
83
|
+
gh release create "$GITHUB_REF_NAME" dist/* --title "$GITHUB_REF_NAME" --notes-file notes.md
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 0.2.0 — 2026-11-10
|
|
6
|
+
|
|
7
|
+
- Fix the GitHub Action for external users. It installed the latest PyPI
|
|
8
|
+
release, and 0.1.0 has no `check` command. It now installs the code at the
|
|
9
|
+
ref it is used from, so `@v0.2.0` runs 0.2.0. The `version` input still
|
|
10
|
+
pins a PyPI release.
|
|
11
|
+
- Rulebook re-verified against current provider docs (version 2026.11.10):
|
|
12
|
+
- Anthropic: Opus, Sonnet and Haiku 5.5, Fable and Mythos 5.1/5.2.
|
|
13
|
+
Per-model read multipliers (0.05x and 0.025x) and base input prices.
|
|
14
|
+
- OpenAI: GPT-5.6 sizes, GPT-6 and GPT-6.1, 12h TTL. Pre-5.6 models no
|
|
15
|
+
longer publish a fixed minimum, so it is recorded as unknown.
|
|
16
|
+
- Google: added Gemini 4 Argon and 3.8 Flash; removed 3.7 and 3.5 Flash,
|
|
17
|
+
which are no longer in Google's table.
|
|
18
|
+
- `sub_minimum` abstains when a model has no published minimum.
|
|
19
|
+
- Release workflow: pushing a `vX.Y.Z` tag publishes to PyPI.
|
|
20
|
+
- The sdist no longer includes the experiment traces.
|
|
21
|
+
- **`ingest_text(text, source)`**: ingest trace content already in memory
|
|
22
|
+
(an upload, a stream) — nothing touches disk. `ingest_path` now wraps it.
|
|
23
|
+
- **Hosted demo** (`demo/`, not part of the package): a one-page FastAPI
|
|
24
|
+
app — drop a trace, get the report; two prerendered samples (the
|
|
25
|
+
synthetic prefix-drift fixture and the real captured TradingAgents
|
|
26
|
+
runs). Uploads are parsed in memory and never stored or logged.
|
|
27
|
+
- **CI cost-regression gate**: `tokenaudit check <traces> --baseline
|
|
28
|
+
baseline.json` fails a build (exit 1) on a new HIGH finding or a drop in
|
|
29
|
+
cache read rate / health floor beyond `--max-regression` points; create
|
|
30
|
+
or refresh the baseline with `--update`. Ships as a composite GitHub
|
|
31
|
+
Action (`action.yml`) with a markdown summary written to the job page
|
|
32
|
+
(and to a file via `--markdown`, for PR comments). See `docs/ci.md`.
|
|
33
|
+
|
|
34
|
+
## 0.1.0 — 2026-08-30
|
|
35
|
+
|
|
36
|
+
First usable release.
|
|
37
|
+
|
|
38
|
+
- **Ingest** four real trace dialects into one normalized accounting model:
|
|
39
|
+
OTel GenAI current semconv, legacy flat names (older openllmetry),
|
|
40
|
+
OpenInference, and the OTel Python SDK's `to_json()` JSONL — with
|
|
41
|
+
layered-span deduplication (framework + SDK instrumentors double-report
|
|
42
|
+
the same call), a data-quality gate that refuses to price malformed
|
|
43
|
+
telemetry, and prompt privacy by construction (block hashes and lengths,
|
|
44
|
+
never text).
|
|
45
|
+
- **Nine deterministic detectors** — prefix drift, sub-minimum cache
|
|
46
|
+
blocks, model switch, TTL expiry, breakpoint limit, lookback overflow,
|
|
47
|
+
tool-schema churn, context bloat, retry storm — each with evidence
|
|
48
|
+
pointers, bounded impact ranges with stated assumptions, and abstention
|
|
49
|
+
(with a reason) as a first-class outcome distinct from "checked, clean".
|
|
50
|
+
- **Provider caching rulebook** packaged as sourced, dated JSON (Anthropic,
|
|
51
|
+
OpenAI, Google), with a CI staleness alarm and a rendered comparison
|
|
52
|
+
table (`tokenaudit rulebook`).
|
|
53
|
+
- **Evaluation** against a byte-reproducible labeled corpus, published in
|
|
54
|
+
`eval/results.md`; per-detector corpus records ship inside the package
|
|
55
|
+
and appear on findings.
|
|
56
|
+
- **CLI**: `tokenaudit diagnose` (text, `--json`, self-contained `--html`
|
|
57
|
+
report, `--sample`), `tokenaudit ingest`, `tokenaudit rulebook`.
|
|
58
|
+
- Zero runtime dependencies. Python 3.11+.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tokenaudit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Root-cause diagnosis of wasted LLM tokens, from the traces your agents already emit
|
|
5
5
|
Project-URL: Repository, https://github.com/yashmyeole/tokenaudit
|
|
6
6
|
Project-URL: Issues, https://github.com/yashmyeole/tokenaudit/issues
|
|
@@ -8,7 +8,7 @@ Author-email: Yash Manohar Yeole <yashmyeole@gmail.com>
|
|
|
8
8
|
License-Expression: Apache-2.0
|
|
9
9
|
License-File: LICENSE
|
|
10
10
|
Keywords: cost,diagnosis,llm,observability,opentelemetry,prompt-caching,tokens,waste
|
|
11
|
-
Classifier: Development Status ::
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
12
|
Classifier: Intended Audience :: Developers
|
|
13
13
|
Classifier: Operating System :: OS Independent
|
|
14
14
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -29,12 +29,14 @@ Description-Content-Type: text/markdown
|
|
|
29
29
|
|
|
30
30
|
**Root-cause diagnosis of wasted LLM tokens — from the traces your agents already emit.**
|
|
31
31
|
|
|
32
|
-
> Status:
|
|
32
|
+
> Status: alpha (0.2.0). Interfaces may change before 1.0.
|
|
33
33
|
|
|
34
34
|
Every LLM cost tool can tell you *how much* you spent and *who* spent it. Almost none can tell you *why a specific request wasted money*. The biggest lever on an LLM bill — prompt caching, up to a 10× discount on repeated input — fails silently: change one byte of a cached prefix, drop below a model's minimum cacheable size, let a timer expire, and the provider just bills you full price. No error, no warning. There are [documented](https://barndoor.ai/what-happens-to-your-llm-bill-when-prompt-caching-fails/) [cases](https://github.com/TauricResearch/TradingAgents/issues/750) of agents running at ~0% cache hit rate for months, at 30–40% unnecessary cost, because of a single line of prompt construction.
|
|
35
35
|
|
|
36
36
|
tokenaudit reads the traces agents already produce and, for each expensive request, answers four questions: **what was wasted, why, what it cost, and what to change.**
|
|
37
37
|
|
|
38
|
+
It works: diagnosing one of those documented cases produced a 9-file patch that took TradingAgents' cache hit rate from **7.5% to 32.4% (4.3×)** in a controlled, pre-registered before/after experiment — the [first case study](docs/case-studies/tradingagents.md).
|
|
39
|
+
|
|
38
40
|
## Quickstart
|
|
39
41
|
|
|
40
42
|
pip install tokenaudit
|
|
@@ -43,6 +45,8 @@ tokenaudit reads the traces agents already produce and, for each expensive reque
|
|
|
43
45
|
|
|
44
46
|
Zero runtime dependencies; reads OTLP/JSON files and OTel SDK JSONL. Try it
|
|
45
47
|
on this repo's real captured runs: `tokenaudit diagnose corpus/captured/runs`.
|
|
48
|
+
Prefer a browser? The [demo app](demo/) diagnoses a dropped trace; uploads
|
|
49
|
+
are parsed in memory and never stored.
|
|
46
50
|
|
|
47
51
|
The plan, in one line each:
|
|
48
52
|
|
|
@@ -51,6 +55,22 @@ The plan, in one line each:
|
|
|
51
55
|
- Price each finding against a counterfactual — what the bill would have been under the fix — as a bounded range, never a point guess.
|
|
52
56
|
- Publish per-detector precision and recall against a labeled corpus, so you know how much to trust each finding.
|
|
53
57
|
|
|
58
|
+
## Catch regressions in CI
|
|
59
|
+
|
|
60
|
+
Prompt-caching regressions are silent: no test fails when a reordered tool
|
|
61
|
+
list doubles your input bill. If CI already produces traces, fail the pull
|
|
62
|
+
request instead ([full docs](docs/ci.md)):
|
|
63
|
+
|
|
64
|
+
```yaml
|
|
65
|
+
- uses: yashmyeole/tokenaudit@v0.2.0
|
|
66
|
+
with:
|
|
67
|
+
traces: traces/
|
|
68
|
+
baseline: tokenaudit-baseline.json # tokenaudit check traces/ --baseline ... --update
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
This repo gates itself: every build re-diagnoses the captured
|
|
72
|
+
TradingAgents runs against [tokenaudit-baseline.json](tokenaudit-baseline.json).
|
|
73
|
+
|
|
54
74
|
## Prior art
|
|
55
75
|
|
|
56
76
|
This space is crowded, and being precise about what already exists is the point of this section.
|
|
@@ -96,8 +116,13 @@ If you know a tool that already does this, please open an issue — I would genu
|
|
|
96
116
|
- [x] Detectors, starting with prefix drift, sub-minimum blocks, and model/version switch
|
|
97
117
|
- [x] Evaluation harness: per-detector results published in [eval/results.md](eval/results.md)
|
|
98
118
|
- [x] `tokenaudit diagnose` CLI, HTML report, cache health score
|
|
99
|
-
- [
|
|
100
|
-
- [
|
|
119
|
+
- [x] CI cost-regression mode: fail a build that tanks the cache hit rate ([docs/ci.md](docs/ci.md))
|
|
120
|
+
- [x] Demo app: drag a trace in, get the report ([demo/](demo/))
|
|
121
|
+
- [x] Case studies on public agents, with reproduction commands ([TradingAgents: 4.3× cache hit rate](docs/case-studies/tradingagents.md))
|
|
122
|
+
- [ ] Detectors for misordered shared context and cross-run prefix drift
|
|
123
|
+
- [ ] Counterfactual pricing with per-model read rates and dollar ranges
|
|
124
|
+
- [ ] Raw provider SDK log ingest, tool-schema bloat detector
|
|
125
|
+
- [ ] Public demo deployment, case study on an Anthropic-model agent
|
|
101
126
|
|
|
102
127
|
## Development
|
|
103
128
|
|
|
@@ -106,6 +131,9 @@ If you know a tool that already does this, please open an issue — I would genu
|
|
|
106
131
|
pre-commit install
|
|
107
132
|
pytest
|
|
108
133
|
|
|
134
|
+
To release, bump the version in `pyproject.toml` and `src/tokenaudit/__init__.py`,
|
|
135
|
+
update CHANGELOG.md, and push a `vX.Y.Z` tag.
|
|
136
|
+
|
|
109
137
|
## License
|
|
110
138
|
|
|
111
139
|
Apache-2.0
|
|
@@ -2,12 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
**Root-cause diagnosis of wasted LLM tokens — from the traces your agents already emit.**
|
|
4
4
|
|
|
5
|
-
> Status:
|
|
5
|
+
> Status: alpha (0.2.0). Interfaces may change before 1.0.
|
|
6
6
|
|
|
7
7
|
Every LLM cost tool can tell you *how much* you spent and *who* spent it. Almost none can tell you *why a specific request wasted money*. The biggest lever on an LLM bill — prompt caching, up to a 10× discount on repeated input — fails silently: change one byte of a cached prefix, drop below a model's minimum cacheable size, let a timer expire, and the provider just bills you full price. No error, no warning. There are [documented](https://barndoor.ai/what-happens-to-your-llm-bill-when-prompt-caching-fails/) [cases](https://github.com/TauricResearch/TradingAgents/issues/750) of agents running at ~0% cache hit rate for months, at 30–40% unnecessary cost, because of a single line of prompt construction.
|
|
8
8
|
|
|
9
9
|
tokenaudit reads the traces agents already produce and, for each expensive request, answers four questions: **what was wasted, why, what it cost, and what to change.**
|
|
10
10
|
|
|
11
|
+
It works: diagnosing one of those documented cases produced a 9-file patch that took TradingAgents' cache hit rate from **7.5% to 32.4% (4.3×)** in a controlled, pre-registered before/after experiment — the [first case study](docs/case-studies/tradingagents.md).
|
|
12
|
+
|
|
11
13
|
## Quickstart
|
|
12
14
|
|
|
13
15
|
pip install tokenaudit
|
|
@@ -16,6 +18,8 @@ tokenaudit reads the traces agents already produce and, for each expensive reque
|
|
|
16
18
|
|
|
17
19
|
Zero runtime dependencies; reads OTLP/JSON files and OTel SDK JSONL. Try it
|
|
18
20
|
on this repo's real captured runs: `tokenaudit diagnose corpus/captured/runs`.
|
|
21
|
+
Prefer a browser? The [demo app](demo/) diagnoses a dropped trace; uploads
|
|
22
|
+
are parsed in memory and never stored.
|
|
19
23
|
|
|
20
24
|
The plan, in one line each:
|
|
21
25
|
|
|
@@ -24,6 +28,22 @@ The plan, in one line each:
|
|
|
24
28
|
- Price each finding against a counterfactual — what the bill would have been under the fix — as a bounded range, never a point guess.
|
|
25
29
|
- Publish per-detector precision and recall against a labeled corpus, so you know how much to trust each finding.
|
|
26
30
|
|
|
31
|
+
## Catch regressions in CI
|
|
32
|
+
|
|
33
|
+
Prompt-caching regressions are silent: no test fails when a reordered tool
|
|
34
|
+
list doubles your input bill. If CI already produces traces, fail the pull
|
|
35
|
+
request instead ([full docs](docs/ci.md)):
|
|
36
|
+
|
|
37
|
+
```yaml
|
|
38
|
+
- uses: yashmyeole/tokenaudit@v0.2.0
|
|
39
|
+
with:
|
|
40
|
+
traces: traces/
|
|
41
|
+
baseline: tokenaudit-baseline.json # tokenaudit check traces/ --baseline ... --update
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
This repo gates itself: every build re-diagnoses the captured
|
|
45
|
+
TradingAgents runs against [tokenaudit-baseline.json](tokenaudit-baseline.json).
|
|
46
|
+
|
|
27
47
|
## Prior art
|
|
28
48
|
|
|
29
49
|
This space is crowded, and being precise about what already exists is the point of this section.
|
|
@@ -69,8 +89,13 @@ If you know a tool that already does this, please open an issue — I would genu
|
|
|
69
89
|
- [x] Detectors, starting with prefix drift, sub-minimum blocks, and model/version switch
|
|
70
90
|
- [x] Evaluation harness: per-detector results published in [eval/results.md](eval/results.md)
|
|
71
91
|
- [x] `tokenaudit diagnose` CLI, HTML report, cache health score
|
|
72
|
-
- [
|
|
73
|
-
- [
|
|
92
|
+
- [x] CI cost-regression mode: fail a build that tanks the cache hit rate ([docs/ci.md](docs/ci.md))
|
|
93
|
+
- [x] Demo app: drag a trace in, get the report ([demo/](demo/))
|
|
94
|
+
- [x] Case studies on public agents, with reproduction commands ([TradingAgents: 4.3× cache hit rate](docs/case-studies/tradingagents.md))
|
|
95
|
+
- [ ] Detectors for misordered shared context and cross-run prefix drift
|
|
96
|
+
- [ ] Counterfactual pricing with per-model read rates and dollar ranges
|
|
97
|
+
- [ ] Raw provider SDK log ingest, tool-schema bloat detector
|
|
98
|
+
- [ ] Public demo deployment, case study on an Anthropic-model agent
|
|
74
99
|
|
|
75
100
|
## Development
|
|
76
101
|
|
|
@@ -79,6 +104,9 @@ If you know a tool that already does this, please open an issue — I would genu
|
|
|
79
104
|
pre-commit install
|
|
80
105
|
pytest
|
|
81
106
|
|
|
107
|
+
To release, bump the version in `pyproject.toml` and `src/tokenaudit/__init__.py`,
|
|
108
|
+
update CHANGELOG.md, and push a `vX.Y.Z` tag.
|
|
109
|
+
|
|
82
110
|
## License
|
|
83
111
|
|
|
84
112
|
Apache-2.0
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Composite GitHub Action wrapping `tokenaudit check`.
|
|
2
|
+
#
|
|
3
|
+
# The CLI already appends its markdown summary to $GITHUB_STEP_SUMMARY when
|
|
4
|
+
# that variable is set (it is, inside Actions), so the job's summary page
|
|
5
|
+
# shows the baseline-vs-current table with no extra wiring here.
|
|
6
|
+
name: "tokenaudit check"
|
|
7
|
+
description: >-
|
|
8
|
+
Fail a build that silently made LLM prompt caching worse: diagnoses a
|
|
9
|
+
trace run and compares cache health against a baseline committed to
|
|
10
|
+
the repo.
|
|
11
|
+
branding:
|
|
12
|
+
icon: "trending-down"
|
|
13
|
+
color: "orange"
|
|
14
|
+
inputs:
|
|
15
|
+
traces:
|
|
16
|
+
description: "Trace file or directory to diagnose (OTLP/JSON or OTel SDK JSONL)"
|
|
17
|
+
required: true
|
|
18
|
+
baseline:
|
|
19
|
+
description: "Path to the committed baseline JSON"
|
|
20
|
+
required: false
|
|
21
|
+
default: "tokenaudit-baseline.json"
|
|
22
|
+
max-regression:
|
|
23
|
+
description: "Allowed drop in cache read rate / health floor, in percentage points"
|
|
24
|
+
required: false
|
|
25
|
+
default: "5.0"
|
|
26
|
+
strict:
|
|
27
|
+
description: "Also fail on new non-HIGH findings (true/false)"
|
|
28
|
+
required: false
|
|
29
|
+
default: "false"
|
|
30
|
+
version:
|
|
31
|
+
description: "Install this PyPI release instead of the code at the action's ref"
|
|
32
|
+
required: false
|
|
33
|
+
default: ""
|
|
34
|
+
runs:
|
|
35
|
+
using: "composite"
|
|
36
|
+
steps:
|
|
37
|
+
- name: Install tokenaudit
|
|
38
|
+
shell: bash
|
|
39
|
+
env:
|
|
40
|
+
TOKENAUDIT_VERSION: ${{ inputs.version }}
|
|
41
|
+
ACTION_PATH: ${{ github.action_path }}
|
|
42
|
+
run: |
|
|
43
|
+
if [ -n "$TOKENAUDIT_VERSION" ]; then
|
|
44
|
+
python -m pip install --quiet "tokenaudit==$TOKENAUDIT_VERSION"
|
|
45
|
+
else
|
|
46
|
+
python -m pip install --quiet "$ACTION_PATH"
|
|
47
|
+
fi
|
|
48
|
+
tokenaudit --version
|
|
49
|
+
- name: Check for cache cost regression
|
|
50
|
+
shell: bash
|
|
51
|
+
env:
|
|
52
|
+
TRACES: ${{ inputs.traces }}
|
|
53
|
+
BASELINE: ${{ inputs.baseline }}
|
|
54
|
+
MAX_REGRESSION: ${{ inputs.max-regression }}
|
|
55
|
+
STRICT: ${{ inputs.strict }}
|
|
56
|
+
run: |
|
|
57
|
+
extra=""
|
|
58
|
+
if [ "$STRICT" = "true" ]; then extra="--strict"; fi
|
|
59
|
+
tokenaudit check "$TRACES" --baseline "$BASELINE" \
|
|
60
|
+
--max-regression "$MAX_REGRESSION" $extra
|
|
@@ -79,7 +79,9 @@ ANTHROPIC_MAX_BREAKPOINTS: int = _ANTHROPIC["max_explicit_breakpoints"]
|
|
|
79
79
|
ANTHROPIC_LOOKBACK_POSITIONS: int = _ANTHROPIC["lookback_positions"]
|
|
80
80
|
|
|
81
81
|
OPENAI_MIN_CACHEABLE: dict[str, int] = {
|
|
82
|
-
model: entry["min_cacheable_tokens"]
|
|
82
|
+
model: entry["min_cacheable_tokens"]
|
|
83
|
+
for model, entry in _OPENAI["models"].items()
|
|
84
|
+
if entry["min_cacheable_tokens"] is not None
|
|
83
85
|
}
|
|
84
86
|
OPENAI_TTL_S: int = _OPENAI["models"]["gpt-5.6"]["ttl"][0]["seconds"] # sliding from last use
|
|
85
87
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Build from the REPO ROOT (the app prerenders reports from the captured
|
|
2
|
+
# runs in corpus/): docker build -f demo/Dockerfile -t tokenaudit-demo .
|
|
3
|
+
FROM python:3.12-slim
|
|
4
|
+
|
|
5
|
+
WORKDIR /app
|
|
6
|
+
|
|
7
|
+
# Layer the dependency install before the source copy so code-only edits
|
|
8
|
+
# rebuild in seconds.
|
|
9
|
+
COPY demo/requirements.txt demo/requirements.txt
|
|
10
|
+
RUN pip install --no-cache-dir -r demo/requirements.txt
|
|
11
|
+
|
|
12
|
+
COPY . .
|
|
13
|
+
RUN pip install --no-cache-dir .
|
|
14
|
+
|
|
15
|
+
EXPOSE 8000
|
|
16
|
+
# Render/Railway inject $PORT; default to 8000 for local docker run.
|
|
17
|
+
CMD uvicorn demo.app:app --host 0.0.0.0 --port ${PORT:-8000}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Hosted demo
|
|
2
|
+
|
|
3
|
+
A one-page FastAPI app: drop a trace, get the diagnosis report back.
|
|
4
|
+
Uploads are parsed in memory with `tokenaudit.ingest.ingest_text` and never
|
|
5
|
+
written to disk, stored, or logged; the two sample reports are prerendered
|
|
6
|
+
at startup from files already public in this repo. The demo is not part of
|
|
7
|
+
the pip package — the library keeps zero runtime dependencies.
|
|
8
|
+
|
|
9
|
+
## Run locally
|
|
10
|
+
|
|
11
|
+
pip install -e . -r demo/requirements.txt
|
|
12
|
+
uvicorn demo.app:app --reload
|
|
13
|
+
# open http://127.0.0.1:8000
|
|
14
|
+
|
|
15
|
+
Or with Docker (from the repo root — the image needs `corpus/` for the
|
|
16
|
+
real-capture sample):
|
|
17
|
+
|
|
18
|
+
docker build -f demo/Dockerfile -t tokenaudit-demo .
|
|
19
|
+
docker run --rm -p 8000:8000 tokenaudit-demo
|
|
20
|
+
|
|
21
|
+
## Deploy on Render (free tier)
|
|
22
|
+
|
|
23
|
+
1. render.com → New → **Web Service** → connect `yashmyeole/tokenaudit`.
|
|
24
|
+
2. Language: **Docker**; Dockerfile path: `demo/Dockerfile`;
|
|
25
|
+
root directory: leave at repo root.
|
|
26
|
+
3. Instance type: **Free**. Create the service.
|
|
27
|
+
4. First build takes a few minutes; the URL looks like
|
|
28
|
+
`https://tokenaudit-demo.onrender.com`.
|
|
29
|
+
|
|
30
|
+
Free instances sleep after ~15 minutes idle, so the first visit after a
|
|
31
|
+
quiet spell takes ~30–60 s to wake — fine for a portfolio demo, and worth
|
|
32
|
+
a one-line note next to the link. Railway and Fly.io both deploy the same
|
|
33
|
+
Dockerfile if you prefer them (both currently want a card on file).
|
|
34
|
+
|
|
35
|
+
## Endpoints
|
|
36
|
+
|
|
37
|
+
- `GET /` — the page
|
|
38
|
+
- `GET /api/sample/drift` — prerendered report for the bundled synthetic
|
|
39
|
+
prefix-drift fixture (labeled as synthetic)
|
|
40
|
+
- `GET /api/sample/tradingagents` — prerendered report for the three real
|
|
41
|
+
captured TradingAgents runs
|
|
42
|
+
- `POST /api/diagnose` — multipart upload (`traces`, up to 20 files /
|
|
43
|
+
15 MB total), returns the self-contained HTML report; 422 with the
|
|
44
|
+
loud refusal message for unrecognized formats
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The hosted demo app (not part of the tokenaudit package)."""
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""The hosted demo: drag a trace in, get the diagnosis back. Nothing stored.
|
|
2
|
+
|
|
3
|
+
Privacy is a design property here, not a promise: uploads are handled with
|
|
4
|
+
:func:`tokenaudit.ingest.ingest_text`, which parses the bytes in memory —
|
|
5
|
+
there is no temp file, no persistence layer to configure away, and the app
|
|
6
|
+
never logs request bodies. The two sample reports are prerendered at
|
|
7
|
+
startup from files already public in this repo.
|
|
8
|
+
|
|
9
|
+
The demo is deliberately not part of the ``tokenaudit`` package (the
|
|
10
|
+
library keeps zero runtime dependencies); it is a separate FastAPI app
|
|
11
|
+
deployed from ``demo/Dockerfile``.
|
|
12
|
+
|
|
13
|
+
Run locally: pip install -e . -r demo/requirements.txt
|
|
14
|
+
uvicorn demo.app:app --reload
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from collections.abc import Awaitable, Callable
|
|
20
|
+
from importlib.resources import files
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from fastapi import FastAPI, HTTPException, Request, Response, UploadFile
|
|
24
|
+
from fastapi.responses import FileResponse, HTMLResponse
|
|
25
|
+
|
|
26
|
+
from tokenaudit.detect import diagnose
|
|
27
|
+
from tokenaudit.ingest import UnknownTraceFormatError, ingest_text
|
|
28
|
+
from tokenaudit.report import render_html
|
|
29
|
+
from tokenaudit.rulebook import load_rulebook
|
|
30
|
+
|
|
31
|
+
MAX_UPLOAD_BYTES = 15 * 1024 * 1024 # per request, checked before parsing
|
|
32
|
+
MAX_FILES = 20
|
|
33
|
+
|
|
34
|
+
_DEMO_DIR = Path(__file__).resolve().parent
|
|
35
|
+
_REPO_ROOT = _DEMO_DIR.parent
|
|
36
|
+
_CAPTURED = _REPO_ROOT / "corpus" / "captured" / "runs" / "tradingagents"
|
|
37
|
+
|
|
38
|
+
app = FastAPI(title="tokenaudit demo", docs_url=None, redoc_url=None, openapi_url=None)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _diagnose_to_html(named_texts: list[tuple[str, str]]) -> str:
|
|
42
|
+
"""Ingest + diagnose in memory and render the self-contained report."""
|
|
43
|
+
rulebook = load_rulebook()
|
|
44
|
+
per_result = []
|
|
45
|
+
for name, text in named_texts:
|
|
46
|
+
result = ingest_text(text, name)
|
|
47
|
+
per_result.append((result, diagnose(result)))
|
|
48
|
+
return render_html(per_result, rulebook)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _prerender_samples() -> dict[str, str]:
|
|
52
|
+
samples: dict[str, str] = {}
|
|
53
|
+
drift = files("tokenaudit.examples").joinpath("sample.trace.json")
|
|
54
|
+
samples["drift"] = _diagnose_to_html(
|
|
55
|
+
[("sample.trace.json (synthetic prefix-drift demo)", drift.read_text(encoding="utf-8"))]
|
|
56
|
+
)
|
|
57
|
+
if _CAPTURED.is_dir():
|
|
58
|
+
captured = [
|
|
59
|
+
(str(p.relative_to(_REPO_ROOT)), p.read_text(encoding="utf-8"))
|
|
60
|
+
for p in sorted(_CAPTURED.rglob("*.jsonl"))
|
|
61
|
+
]
|
|
62
|
+
if captured:
|
|
63
|
+
samples["tradingagents"] = _diagnose_to_html(captured)
|
|
64
|
+
return samples
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_SAMPLES = _prerender_samples()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@app.middleware("http")
|
|
71
|
+
async def security_headers(
|
|
72
|
+
request: Request, call_next: Callable[[Request], Awaitable[Response]]
|
|
73
|
+
) -> Response:
|
|
74
|
+
response = await call_next(request)
|
|
75
|
+
response.headers["X-Content-Type-Options"] = "nosniff"
|
|
76
|
+
response.headers["Referrer-Policy"] = "no-referrer"
|
|
77
|
+
return response
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@app.get("/", include_in_schema=False)
|
|
81
|
+
async def index() -> FileResponse:
|
|
82
|
+
return FileResponse(_DEMO_DIR / "static" / "index.html", media_type="text/html")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@app.get("/api/sample/{name}")
|
|
86
|
+
async def sample(name: str) -> HTMLResponse:
|
|
87
|
+
if name not in _SAMPLES:
|
|
88
|
+
raise HTTPException(404, f"no sample named {name!r}")
|
|
89
|
+
return HTMLResponse(_SAMPLES[name])
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@app.post("/api/diagnose")
|
|
93
|
+
async def diagnose_upload(traces: list[UploadFile]) -> HTMLResponse:
|
|
94
|
+
if not traces:
|
|
95
|
+
raise HTTPException(400, "no files uploaded")
|
|
96
|
+
if len(traces) > MAX_FILES:
|
|
97
|
+
raise HTTPException(413, f"too many files (max {MAX_FILES})")
|
|
98
|
+
named_texts: list[tuple[str, str]] = []
|
|
99
|
+
total = 0
|
|
100
|
+
for upload in traces:
|
|
101
|
+
data = await upload.read()
|
|
102
|
+
total += len(data)
|
|
103
|
+
if total > MAX_UPLOAD_BYTES:
|
|
104
|
+
raise HTTPException(413, f"upload exceeds {MAX_UPLOAD_BYTES // (1024 * 1024)} MB")
|
|
105
|
+
try:
|
|
106
|
+
text = data.decode("utf-8")
|
|
107
|
+
except UnicodeDecodeError as exc:
|
|
108
|
+
raise HTTPException(400, f"{upload.filename}: not UTF-8 text ({exc})") from exc
|
|
109
|
+
named_texts.append((upload.filename or "upload", text))
|
|
110
|
+
try:
|
|
111
|
+
return HTMLResponse(_diagnose_to_html(named_texts))
|
|
112
|
+
except UnknownTraceFormatError as exc:
|
|
113
|
+
raise HTTPException(422, str(exc)) from exc
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Dependencies of the hosted demo app ONLY — the tokenaudit package itself
|
|
2
|
+
# stays zero-dependency. httpx is here for FastAPI's TestClient (used by
|
|
3
|
+
# tests/test_demo.py and handy for poking the app locally).
|
|
4
|
+
fastapi>=0.115
|
|
5
|
+
python-multipart>=0.0.9 # FastAPI needs it for multipart file uploads
|
|
6
|
+
uvicorn>=0.30
|
|
7
|
+
httpx2>=0.1 # FastAPI's TestClient backend (tests/test_demo.py, local poking)
|