agent-cost-attribution 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_cost_attribution-0.1.0/LICENSE +21 -0
- agent_cost_attribution-0.1.0/PKG-INFO +194 -0
- agent_cost_attribution-0.1.0/README.md +169 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution/__init__.py +12 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution/__main__.py +3 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution/cli.py +59 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution/health.py +50 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution/ledger.py +107 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution/pricing.py +51 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution/routing.py +122 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution.egg-info/PKG-INFO +194 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution.egg-info/SOURCES.txt +19 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution.egg-info/dependency_links.txt +1 -0
- agent_cost_attribution-0.1.0/agent_cost_attribution.egg-info/top_level.txt +1 -0
- agent_cost_attribution-0.1.0/pyproject.toml +43 -0
- agent_cost_attribution-0.1.0/setup.cfg +4 -0
- agent_cost_attribution-0.1.0/tests/test_cli.py +33 -0
- agent_cost_attribution-0.1.0/tests/test_health.py +41 -0
- agent_cost_attribution-0.1.0/tests/test_ledger.py +63 -0
- agent_cost_attribution-0.1.0/tests/test_pricing.py +40 -0
- agent_cost_attribution-0.1.0/tests/test_routing.py +141 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jeff Otterson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agent-cost-attribution
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Per-stage token/cost attribution and silent-degradation detection for multi-agent workflow runs (stdlib-only).
|
|
5
|
+
Author: Jeff Otterson
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Jott2121/agent-cost-attribution
|
|
8
|
+
Project-URL: Repository, https://github.com/Jott2121/agent-cost-attribution
|
|
9
|
+
Project-URL: Issues, https://github.com/Jott2121/agent-cost-attribution/issues
|
|
10
|
+
Keywords: ai,agents,llm,cost,tokens,observability,telemetry
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
20
|
+
Classifier: Topic :: System :: Monitoring
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# agent-cost-attribution
|
|
27
|
+
|
|
28
|
+
[](https://github.com/Jott2121/agent-cost-attribution/actions/workflows/ci.yml)
|
|
29
|
+
[](https://github.com/Jott2121/agent-cost-attribution/actions/workflows/codeql.yml)
|
|
30
|
+
[](https://github.com/Jott2121/agent-cost-attribution/actions/workflows/ci.yml)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
**Get the most capability per token out of agentic coding, and prove it. Headline result, measured by the meter in this repo: routing four mechanical agents to a cheap model cut run cost 67% (~$0.60 → ~$0.20) with identical facts extracted ([receipt](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/routing-savings.md)).**
|
|
34
|
+
|
|
35
|
+
Agentic workflows (fan-out subagents, multi-step research, tool-heavy pipelines) burn tokens fast,
|
|
36
|
+
and most of the waste is invisible. You can't fix what you can't see, and the platform's own "success"
|
|
37
|
+
flag is wrong more often than you'd think.
|
|
38
|
+
|
|
39
|
+
> 🧩 One layer of a five-repo [**cost-governance stack**](https://github.com/Jott2121/bow#the-system-a-cost-governance-stack) for operating AI agents cost-efficiently; [bow](https://github.com/Jott2121/bow) is the flagship that runs every layer in production.
|
|
40
|
+
|
|
41
|
+
This repo is two things:
|
|
42
|
+
|
|
43
|
+
1. **`agent_cost_attribution`**: a tiny, dependency-free **meter**. Point it at a workflow run's
|
|
44
|
+
telemetry and get a per-stage token waterfall, plus a **silent-degradation check** that flags runs
|
|
45
|
+
that *reported success while quietly breaking*.
|
|
46
|
+
2. **[PLAYBOOK.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/PLAYBOOK.md)**: a playbook of transferable practices for cutting token burn in
|
|
47
|
+
agentic coding **without losing capability**, each rated by expected savings, axis (tokens vs cost
|
|
48
|
+
vs latency), capability risk, and effort.
|
|
49
|
+
|
|
50
|
+
Everything here is **measured, not asserted**. The numbers in this repo were produced by the meter
|
|
51
|
+
included here; point it at your own telemetry to do the same.
|
|
52
|
+
|
|
53
|
+
## Try it in 30 seconds (zero dependencies, sample included)
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
git clone https://github.com/Jott2121/agent-cost-attribution
|
|
57
|
+
cd agent-cost-attribution
|
|
58
|
+
python3 -m agent_cost_attribution examples/sample-run.json
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Output (from the synthetic sample shipped in [`examples/sample-run.json`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/sample-run.json)):
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
sample-research wf_sample-001 status=completed total=1,006,200 tok ~$3.92 invariant_ok=True
|
|
65
|
+
Verify 720,748 71.6% ~$ 3.46 n=12 ####################################
|
|
66
|
+
Search 254,248 25.3% ~$ 0.41 n=6 #############
|
|
67
|
+
Scope 31,204 3.1% ~$ 0.05 n=1 ##
|
|
68
|
+
($ = estimate: list prices, 85%-input blend; telemetry has no I/O split)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Then point it at your own runs:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python3 -m agent_cost_attribution path/to/run.json
|
|
75
|
+
python3 -m agent_cost_attribution path/to/runs-dir/ # every wf_*.json in a directory
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
**Where does `run.json` come from?** The meter reads Claude Code workflow telemetry: the `wf_*.json`
|
|
79
|
+
files that Claude Code writes automatically when you run agentic workflows, found under
|
|
80
|
+
`~/.claude/projects/*/tasks/`. If you run Claude Code workflows, you already have these files; no
|
|
81
|
+
instrumentation, SDK, or proxy is required. The shipped sample is synthetic but format-exact, so you
|
|
82
|
+
can see the output before you have telemetry of your own.
|
|
83
|
+
|
|
84
|
+
You get three things per run: **tokens**, an estimated **dollar cost**, and a **trust check**.
|
|
85
|
+
|
|
86
|
+
- **`invariant_ok`** means the per-agent token counts sum **exactly** to the run total. The parser is
|
|
87
|
+
checking itself, so you can believe the breakdown.
|
|
88
|
+
- The **`~$` figures are estimates**: each agent's tokens are priced at its model's list price using a
|
|
89
|
+
documented input/output blend (the telemetry exposes only a single token count, no I/O split), so read
|
|
90
|
+
them as a calibrated band, not a billing statement. Because they're priced **per agent**, a
|
|
91
|
+
model-routing win shows up directly (route a stage to a cheaper model and its `~$` drops).
|
|
92
|
+
- A **`DEGRADED`** banner appears when a stage errored or ran anomalously cheap. Cost numbers on a
|
|
93
|
+
degraded run aren't trustworthy and shouldn't be published.
|
|
94
|
+
|
|
95
|
+
## How this differs from Langfuse / Helicone
|
|
96
|
+
|
|
97
|
+
Langfuse, Helicone, and similar LLM observability platforms are excellent at what they do: they sit
|
|
98
|
+
in your request path (SDK or proxy), capture every call, and give you dashboards over time. This tool
|
|
99
|
+
is deliberately the opposite shape. It is a **post-hoc forensic meter**: a single stdlib-only Python
|
|
100
|
+
package, no server, no account, no instrumentation, that reads telemetry files the platform already
|
|
101
|
+
wrote and answers two narrow questions per run: *which stage of this multi-agent workflow ate the
|
|
102
|
+
tokens*, and *can I trust this run's numbers at all* (the invariant check plus silent-degradation
|
|
103
|
+
detection). If you want fleet-wide dashboards, use those platforms. If you want to audit one run's
|
|
104
|
+
cost attribution in 30 seconds with zero setup, use this.
|
|
105
|
+
|
|
106
|
+
## The headline finding (why you should trust the method)
|
|
107
|
+
|
|
108
|
+
I built this meter to support an optimization plan I'd already written. **The meter overturned my own
|
|
109
|
+
plan.** I had assumed the token whale was the page-**Fetch** stage; the telemetry showed it was the
|
|
110
|
+
**Verify** stage (50-74% of healthy runs vs Fetch's ~19-37%). The "expensive" run I'd anchored my
|
|
111
|
+
baseline on turned out to be a **silently broken outlier**: it reported `status=completed` while all
|
|
112
|
+
75 of its verifier agents had errored, which is the *only* reason Fetch looked dominant there.
|
|
113
|
+
|
|
114
|
+
Along the way the meter also showed the platform's own status flag was unreliable in **both**
|
|
115
|
+
directions: one run said `completed` but was broken; another said `failed` but was perfectly healthy.
|
|
116
|
+
The lesson, and the reason the silent-degradation check exists: **trust per-stage health, not the
|
|
117
|
+
run's self-report.** Full numbers in [`examples/self-correction-deep-research.md`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/self-correction-deep-research.md).
|
|
118
|
+
|
|
119
|
+
Measurement that kills your own hypothesis is the whole point. The rest of this repo is built on it.
|
|
120
|
+
|
|
121
|
+
## The playbook (TL;DR; full version in [PLAYBOOK.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/PLAYBOOK.md))
|
|
122
|
+
|
|
123
|
+
- **Right-size the model per task**: mechanical sub-tasks on a cheap model, judgment/synthesis on the
|
|
124
|
+
strong one. (This repo's own builder/reviewer agents ran on the cheaper tier.)
|
|
125
|
+
- **Fan out only for read-heavy parallel work; keep writes single-threaded.**
|
|
126
|
+
- **Scope context tightly**: read the slice you need, don't re-read, hand a sub-agent only what it needs.
|
|
127
|
+
- **Don't re-gather redundantly · kill junk cheaply before expensive stages · dedup inputs · make your caps actually cap.**
|
|
128
|
+
- **Prompt caching · structured outputs.**
|
|
129
|
+
- **Meter every run · gate on quality · refuse the unsafe shortcut and prove it.**
|
|
130
|
+
|
|
131
|
+
Each practice is reported on the right axis and **never double-counted** (routing changes cost-per-token,
|
|
132
|
+
not token *count*; caching is a cost-axis win; only genuine token-count reductions go on the headline).
|
|
133
|
+
|
|
134
|
+
**Measured proof:** [`examples/routing-savings.md`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/routing-savings.md), a live before/after
|
|
135
|
+
where routing four mechanical agents to a cheap model cut run cost **67% (~$0.60 to ~$0.20)** with
|
|
136
|
+
identical facts extracted, isolated by the meter to exactly the routed stage. Reproducible from
|
|
137
|
+
[`examples/routing-demo.js`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/routing-demo.js).
|
|
138
|
+
|
|
139
|
+
## Making A1 stick: the routing layer ([ROUTING.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/ROUTING.md))
|
|
140
|
+
|
|
141
|
+
Right-sizing the model per task fails in practice when the model is a string literal at N call
|
|
142
|
+
sites: one unpinned call silently inherits your most expensive default. [ROUTING.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/ROUTING.md)
|
|
143
|
+
is the case study of retrofitting a live agent fleet (11 call sites, zero routing, all premium-pinned)
|
|
144
|
+
with a central role table, per-message escalation (`!tag` > imperative-only heuristic > cheap
|
|
145
|
+
default), fail-soft live tuning, and an independent cross-model QC pass that caught the policy's own
|
|
146
|
+
over-escalation bug before ship. Audited, built, and live in one day. The reusable module is
|
|
147
|
+
[`agent_cost_attribution/routing.py`](https://github.com/Jott2121/agent-cost-attribution/blob/main/agent_cost_attribution/routing.py) (stdlib-only, like the meter):
|
|
148
|
+
`Router` (fail-closed roles, fail-soft overrides), `MessageRouter` (chat escalation), and
|
|
149
|
+
`savings_estimate()` (planning estimates from the meter's real volumes; labeled estimates, never
|
|
150
|
+
published as measurements).
|
|
151
|
+
|
|
152
|
+
## What's here
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
README.md - this file
|
|
156
|
+
PLAYBOOK.md - the practices, each with what / why / how / savings / risk
|
|
157
|
+
ROUTING.md - case study + design rules: retrofitting model routing onto a live fleet in a day
|
|
158
|
+
agent_cost_attribution/ - the meter (stdlib-only): ledger, health, cli, plus routing.py (A1's enforcement layer)
|
|
159
|
+
tests/ - the meter's tests (the sum==total invariant is golden-tested)
|
|
160
|
+
examples/ - measured worked examples + sample-run.json (synthetic quickstart telemetry)
|
|
161
|
+
pyproject.toml - package metadata + pytest config
|
|
162
|
+
.github/ - CI (pytest on Python 3.9-3.12 + a smoke run of the meter on the sample)
|
|
163
|
+
LICENSE - MIT
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Who it's for
|
|
167
|
+
|
|
168
|
+
Anyone running agentic workflows who wants maximum capability per token, and a way to *find their own
|
|
169
|
+
waste* instead of guessing. It's also a worked demonstration of rigorous, measured agentic-coding
|
|
170
|
+
practice: measure, attribute, gate on quality, publish what you kept *and* what you killed.
|
|
171
|
+
|
|
172
|
+
## Reliability & security
|
|
173
|
+
|
|
174
|
+
A meter people trust has to be measured itself, so the repo is gated:
|
|
175
|
+
|
|
176
|
+
- **Coverage-gated test matrix** — pytest on Python 3.9–3.12, build fails below the coverage floor (currently 96% covered), plus a smoke-test of the meter on the shipped sample run.
|
|
177
|
+
- **CodeQL** — `security-extended` static analysis on every push, PR, and weekly; findings surface in the Security tab.
|
|
178
|
+
- **Pinned supply chain** — GitHub Actions pinned to commit SHAs, kept current by **Dependabot**.
|
|
179
|
+
- **Branch protection** — `main` requires CI + CodeQL to pass before a merge.
|
|
180
|
+
- **Disclosure policy** — see [SECURITY.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/SECURITY.md); private reporting is enabled.
|
|
181
|
+
|
|
182
|
+
## About
|
|
183
|
+
|
|
184
|
+
Built by **Jeff Otterson** ([Jott2121](https://github.com/Jott2121)). Part of the Fleet Mode line:
|
|
185
|
+
[**bow**](https://github.com/Jott2121/bow) (the flagship agent case study) ·
|
|
186
|
+
[**fleet-mode**](https://github.com/Jott2121/fleet-mode) (the orchestration doctrine as a live skill) ·
|
|
187
|
+
[**agent-gate**](https://github.com/Jott2121/agent-gate) · [**rag-guard**](https://github.com/Jott2121/rag-guard) ·
|
|
188
|
+
[**sabot**](https://github.com/Jott2121/sabot) (own-checks fault detection — its silent
|
|
189
|
+
model-downgrade operator grew out of the degraded-run detector in this repo).
|
|
190
|
+
The same discipline throughout: measure it, gate it, keep the receipts.
|
|
191
|
+
|
|
192
|
+
## License
|
|
193
|
+
|
|
194
|
+
MIT, see [LICENSE](https://github.com/Jott2121/agent-cost-attribution/blob/main/LICENSE).
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# agent-cost-attribution
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Jott2121/agent-cost-attribution/actions/workflows/ci.yml)
|
|
4
|
+
[](https://github.com/Jott2121/agent-cost-attribution/actions/workflows/codeql.yml)
|
|
5
|
+
[](https://github.com/Jott2121/agent-cost-attribution/actions/workflows/ci.yml)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
**Get the most capability per token out of agentic coding, and prove it. Headline result, measured by the meter in this repo: routing four mechanical agents to a cheap model cut run cost 67% (~$0.60 → ~$0.20) with identical facts extracted ([receipt](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/routing-savings.md)).**
|
|
9
|
+
|
|
10
|
+
Agentic workflows (fan-out subagents, multi-step research, tool-heavy pipelines) burn tokens fast,
|
|
11
|
+
and most of the waste is invisible. You can't fix what you can't see, and the platform's own "success"
|
|
12
|
+
flag is wrong more often than you'd think.
|
|
13
|
+
|
|
14
|
+
> 🧩 One layer of a five-repo [**cost-governance stack**](https://github.com/Jott2121/bow#the-system-a-cost-governance-stack) for operating AI agents cost-efficiently; [bow](https://github.com/Jott2121/bow) is the flagship that runs every layer in production.
|
|
15
|
+
|
|
16
|
+
This repo is two things:
|
|
17
|
+
|
|
18
|
+
1. **`agent_cost_attribution`**: a tiny, dependency-free **meter**. Point it at a workflow run's
|
|
19
|
+
telemetry and get a per-stage token waterfall, plus a **silent-degradation check** that flags runs
|
|
20
|
+
that *reported success while quietly breaking*.
|
|
21
|
+
2. **[PLAYBOOK.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/PLAYBOOK.md)**: a playbook of transferable practices for cutting token burn in
|
|
22
|
+
agentic coding **without losing capability**, each rated by expected savings, axis (tokens vs cost
|
|
23
|
+
vs latency), capability risk, and effort.
|
|
24
|
+
|
|
25
|
+
Everything here is **measured, not asserted**. The numbers in this repo were produced by the meter
|
|
26
|
+
included here; point it at your own telemetry to do the same.
|
|
27
|
+
|
|
28
|
+
## Try it in 30 seconds (zero dependencies, sample included)
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
git clone https://github.com/Jott2121/agent-cost-attribution
|
|
32
|
+
cd agent-cost-attribution
|
|
33
|
+
python3 -m agent_cost_attribution examples/sample-run.json
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Output (from the synthetic sample shipped in [`examples/sample-run.json`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/sample-run.json)):
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
sample-research wf_sample-001 status=completed total=1,006,200 tok ~$3.92 invariant_ok=True
|
|
40
|
+
Verify 720,748 71.6% ~$ 3.46 n=12 ####################################
|
|
41
|
+
Search 254,248 25.3% ~$ 0.41 n=6 #############
|
|
42
|
+
Scope 31,204 3.1% ~$ 0.05 n=1 ##
|
|
43
|
+
($ = estimate: list prices, 85%-input blend; telemetry has no I/O split)
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Then point it at your own runs:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
python3 -m agent_cost_attribution path/to/run.json
|
|
50
|
+
python3 -m agent_cost_attribution path/to/runs-dir/ # every wf_*.json in a directory
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
**Where does `run.json` come from?** The meter reads Claude Code workflow telemetry: the `wf_*.json`
|
|
54
|
+
files that Claude Code writes automatically when you run agentic workflows, found under
|
|
55
|
+
`~/.claude/projects/*/tasks/`. If you run Claude Code workflows, you already have these files; no
|
|
56
|
+
instrumentation, SDK, or proxy is required. The shipped sample is synthetic but format-exact, so you
|
|
57
|
+
can see the output before you have telemetry of your own.
|
|
58
|
+
|
|
59
|
+
You get three things per run: **tokens**, an estimated **dollar cost**, and a **trust check**.
|
|
60
|
+
|
|
61
|
+
- **`invariant_ok`** means the per-agent token counts sum **exactly** to the run total. The parser is
|
|
62
|
+
checking itself, so you can believe the breakdown.
|
|
63
|
+
- The **`~$` figures are estimates**: each agent's tokens are priced at its model's list price using a
|
|
64
|
+
documented input/output blend (the telemetry exposes only a single token count, no I/O split), so read
|
|
65
|
+
them as a calibrated band, not a billing statement. Because they're priced **per agent**, a
|
|
66
|
+
model-routing win shows up directly (route a stage to a cheaper model and its `~$` drops).
|
|
67
|
+
- A **`DEGRADED`** banner appears when a stage errored or ran anomalously cheap. Cost numbers on a
|
|
68
|
+
degraded run aren't trustworthy and shouldn't be published.
|
|
69
|
+
|
|
70
|
+
## How this differs from Langfuse / Helicone
|
|
71
|
+
|
|
72
|
+
Langfuse, Helicone, and similar LLM observability platforms are excellent at what they do: they sit
|
|
73
|
+
in your request path (SDK or proxy), capture every call, and give you dashboards over time. This tool
|
|
74
|
+
is deliberately the opposite shape. It is a **post-hoc forensic meter**: a single stdlib-only Python
|
|
75
|
+
package, no server, no account, no instrumentation, that reads telemetry files the platform already
|
|
76
|
+
wrote and answers two narrow questions per run: *which stage of this multi-agent workflow ate the
|
|
77
|
+
tokens*, and *can I trust this run's numbers at all* (the invariant check plus silent-degradation
|
|
78
|
+
detection). If you want fleet-wide dashboards, use those platforms. If you want to audit one run's
|
|
79
|
+
cost attribution in 30 seconds with zero setup, use this.
|
|
80
|
+
|
|
81
|
+
## The headline finding (why you should trust the method)
|
|
82
|
+
|
|
83
|
+
I built this meter to support an optimization plan I'd already written. **The meter overturned my own
|
|
84
|
+
plan.** I had assumed the token whale was the page-**Fetch** stage; the telemetry showed it was the
|
|
85
|
+
**Verify** stage (50-74% of healthy runs vs Fetch's ~19-37%). The "expensive" run I'd anchored my
|
|
86
|
+
baseline on turned out to be a **silently broken outlier**: it reported `status=completed` while all
|
|
87
|
+
75 of its verifier agents had errored, which is the *only* reason Fetch looked dominant there.
|
|
88
|
+
|
|
89
|
+
Along the way the meter also showed the platform's own status flag was unreliable in **both**
|
|
90
|
+
directions: one run said `completed` but was broken; another said `failed` but was perfectly healthy.
|
|
91
|
+
The lesson, and the reason the silent-degradation check exists: **trust per-stage health, not the
|
|
92
|
+
run's self-report.** Full numbers in [`examples/self-correction-deep-research.md`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/self-correction-deep-research.md).
|
|
93
|
+
|
|
94
|
+
Measurement that kills your own hypothesis is the whole point. The rest of this repo is built on it.
|
|
95
|
+
|
|
96
|
+
## The playbook (TL;DR; full version in [PLAYBOOK.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/PLAYBOOK.md))
|
|
97
|
+
|
|
98
|
+
- **Right-size the model per task**: mechanical sub-tasks on a cheap model, judgment/synthesis on the
|
|
99
|
+
strong one. (This repo's own builder/reviewer agents ran on the cheaper tier.)
|
|
100
|
+
- **Fan out only for read-heavy parallel work; keep writes single-threaded.**
|
|
101
|
+
- **Scope context tightly**: read the slice you need, don't re-read, hand a sub-agent only what it needs.
|
|
102
|
+
- **Don't re-gather redundantly · kill junk cheaply before expensive stages · dedup inputs · make your caps actually cap.**
|
|
103
|
+
- **Prompt caching · structured outputs.**
|
|
104
|
+
- **Meter every run · gate on quality · refuse the unsafe shortcut and prove it.**
|
|
105
|
+
|
|
106
|
+
Each practice is reported on the right axis and **never double-counted** (routing changes cost-per-token,
|
|
107
|
+
not token *count*; caching is a cost-axis win; only genuine token-count reductions go on the headline).
|
|
108
|
+
|
|
109
|
+
**Measured proof:** [`examples/routing-savings.md`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/routing-savings.md), a live before/after
|
|
110
|
+
where routing four mechanical agents to a cheap model cut run cost **67% (~$0.60 to ~$0.20)** with
|
|
111
|
+
identical facts extracted, isolated by the meter to exactly the routed stage. Reproducible from
|
|
112
|
+
[`examples/routing-demo.js`](https://github.com/Jott2121/agent-cost-attribution/blob/main/examples/routing-demo.js).
|
|
113
|
+
|
|
114
|
+
## Making A1 stick: the routing layer ([ROUTING.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/ROUTING.md))
|
|
115
|
+
|
|
116
|
+
Right-sizing the model per task fails in practice when the model is a string literal at N call
|
|
117
|
+
sites: one unpinned call silently inherits your most expensive default. [ROUTING.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/ROUTING.md)
|
|
118
|
+
is the case study of retrofitting a live agent fleet (11 call sites, zero routing, all premium-pinned)
|
|
119
|
+
with a central role table, per-message escalation (`!tag` > imperative-only heuristic > cheap
|
|
120
|
+
default), fail-soft live tuning, and an independent cross-model QC pass that caught the policy's own
|
|
121
|
+
over-escalation bug before ship. Audited, built, and live in one day. The reusable module is
|
|
122
|
+
[`agent_cost_attribution/routing.py`](https://github.com/Jott2121/agent-cost-attribution/blob/main/agent_cost_attribution/routing.py) (stdlib-only, like the meter):
|
|
123
|
+
`Router` (fail-closed roles, fail-soft overrides), `MessageRouter` (chat escalation), and
|
|
124
|
+
`savings_estimate()` (planning estimates from the meter's real volumes; labeled estimates, never
|
|
125
|
+
published as measurements).
|
|
126
|
+
|
|
127
|
+
## What's here
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
README.md - this file
|
|
131
|
+
PLAYBOOK.md - the practices, each with what / why / how / savings / risk
|
|
132
|
+
ROUTING.md - case study + design rules: retrofitting model routing onto a live fleet in a day
|
|
133
|
+
agent_cost_attribution/ - the meter (stdlib-only): ledger, health, cli, plus routing.py (A1's enforcement layer)
|
|
134
|
+
tests/ - the meter's tests (the sum==total invariant is golden-tested)
|
|
135
|
+
examples/ - measured worked examples + sample-run.json (synthetic quickstart telemetry)
|
|
136
|
+
pyproject.toml - package metadata + pytest config
|
|
137
|
+
.github/ - CI (pytest on Python 3.9-3.12 + a smoke run of the meter on the sample)
|
|
138
|
+
LICENSE - MIT
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Who it's for
|
|
142
|
+
|
|
143
|
+
Anyone running agentic workflows who wants maximum capability per token, and a way to *find their own
|
|
144
|
+
waste* instead of guessing. It's also a worked demonstration of rigorous, measured agentic-coding
|
|
145
|
+
practice: measure, attribute, gate on quality, publish what you kept *and* what you killed.
|
|
146
|
+
|
|
147
|
+
## Reliability & security
|
|
148
|
+
|
|
149
|
+
A meter people trust has to be measured itself, so the repo is gated:
|
|
150
|
+
|
|
151
|
+
- **Coverage-gated test matrix** — pytest on Python 3.9–3.12, build fails below the coverage floor (currently 96% covered), plus a smoke-test of the meter on the shipped sample run.
|
|
152
|
+
- **CodeQL** — `security-extended` static analysis on every push, PR, and weekly; findings surface in the Security tab.
|
|
153
|
+
- **Pinned supply chain** — GitHub Actions pinned to commit SHAs, kept current by **Dependabot**.
|
|
154
|
+
- **Branch protection** — `main` requires CI + CodeQL to pass before a merge.
|
|
155
|
+
- **Disclosure policy** — see [SECURITY.md](https://github.com/Jott2121/agent-cost-attribution/blob/main/SECURITY.md); private reporting is enabled.
|
|
156
|
+
|
|
157
|
+
## About
|
|
158
|
+
|
|
159
|
+
Built by **Jeff Otterson** ([Jott2121](https://github.com/Jott2121)). Part of the Fleet Mode line:
|
|
160
|
+
[**bow**](https://github.com/Jott2121/bow) (the flagship agent case study) ·
|
|
161
|
+
[**fleet-mode**](https://github.com/Jott2121/fleet-mode) (the orchestration doctrine as a live skill) ·
|
|
162
|
+
[**agent-gate**](https://github.com/Jott2121/agent-gate) · [**rag-guard**](https://github.com/Jott2121/rag-guard) ·
|
|
163
|
+
[**sabot**](https://github.com/Jott2121/sabot) (own-checks fault detection — its silent
|
|
164
|
+
model-downgrade operator grew out of the degraded-run detector in this repo).
|
|
165
|
+
The same discipline throughout: measure it, gate it, keep the receipts.
|
|
166
|
+
|
|
167
|
+
## License
|
|
168
|
+
|
|
169
|
+
MIT, see [LICENSE](https://github.com/Jott2121/agent-cost-attribution/blob/main/LICENSE).
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""agent-cost-attribution — per-stage token attribution for multi-agent workflow runs."""
|
|
2
|
+
from agent_cost_attribution.ledger import (
|
|
3
|
+
AgentRecord, agents, decompose, load_run, summary, total_tokens, verify_invariant,
|
|
4
|
+
)
|
|
5
|
+
from agent_cost_attribution.health import detect_degradation, is_healthy
|
|
6
|
+
from agent_cost_attribution.pricing import estimate_cost, cost_by_stage, total_cost, blended_rate
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"AgentRecord", "agents", "decompose", "load_run", "summary", "total_tokens",
|
|
10
|
+
"verify_invariant", "detect_degradation", "is_healthy",
|
|
11
|
+
"estimate_cost", "cost_by_stage", "total_cost", "blended_rate",
|
|
12
|
+
]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""CLI: python -m agent_cost_attribution <run.json | run-dir> ...
|
|
2
|
+
|
|
3
|
+
Prints a per-stage token waterfall + a silent-degradation health report for each run. Workload-
|
|
4
|
+
agnostic: any wf_*.json (or a directory of them) works — this is the reusable surface.
|
|
5
|
+
"""
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from agent_cost_attribution.ledger import load_run, summary
|
|
10
|
+
from agent_cost_attribution.health import detect_degradation
|
|
11
|
+
from agent_cost_attribution.pricing import cost_by_stage, DEFAULT_INPUT_SHARE
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _iter_paths(args):
|
|
15
|
+
for a in args:
|
|
16
|
+
p = Path(a)
|
|
17
|
+
if p.is_dir():
|
|
18
|
+
yield from sorted(p.glob("wf_*.json"))
|
|
19
|
+
else:
|
|
20
|
+
yield p
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def render(run, *, input_share=DEFAULT_INPUT_SHARE):
|
|
24
|
+
s = summary(run)
|
|
25
|
+
costs = cost_by_stage(run, input_share=input_share)
|
|
26
|
+
total_usd = sum(costs.values())
|
|
27
|
+
head = (f"{s['workflow'] or '?'} {s['run_id']} status={s['status']} "
|
|
28
|
+
f"total={s['total_tokens']:,} tok ~${total_usd:,.2f} invariant_ok={s['invariant_ok']}")
|
|
29
|
+
lines = [head]
|
|
30
|
+
for stage, st in sorted(s["stages"].items(), key=lambda kv: -kv[1]["tokens"]):
|
|
31
|
+
bar = "#" * int(round(st["pct"] / 2))
|
|
32
|
+
usd = costs.get(stage, 0.0)
|
|
33
|
+
lines.append(f" {stage:14s} {st['tokens']:>9,} {st['pct']:5.1f}% ~${usd:>7,.2f} n={st['count']:<3d} {bar}")
|
|
34
|
+
findings = detect_degradation(run)
|
|
35
|
+
if findings:
|
|
36
|
+
lines.append(" ! DEGRADED — cost numbers on this run are NOT publishable:")
|
|
37
|
+
for f in findings:
|
|
38
|
+
lines.append(f" [{f['severity']}] {f['stage']}: {f['kind']} — {f['detail']}")
|
|
39
|
+
lines.append(f" ($ = estimate: list prices, {input_share:.0%}-input blend; telemetry has no I/O split)")
|
|
40
|
+
return "\n".join(lines)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def main(argv=None):
|
|
44
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
45
|
+
if not argv:
|
|
46
|
+
print("usage: python -m agent_cost_attribution <run.json | run-dir> ...", file=sys.stderr)
|
|
47
|
+
return 2
|
|
48
|
+
paths = list(_iter_paths(argv))
|
|
49
|
+
if not paths:
|
|
50
|
+
print("no run files found", file=sys.stderr)
|
|
51
|
+
return 1
|
|
52
|
+
for p in paths:
|
|
53
|
+
run = load_run(p)
|
|
54
|
+
if not run:
|
|
55
|
+
print(f"{p}: unreadable/empty", file=sys.stderr)
|
|
56
|
+
continue
|
|
57
|
+
print(render(run))
|
|
58
|
+
print()
|
|
59
|
+
return 0
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Silent-degradation detector for multi-agent workflow runs.
|
|
2
|
+
|
|
3
|
+
The failure that broke the cost baseline (wf_3d01df61): a stage's agents can all ERROR — or run
|
|
4
|
+
anomalously cheap — while the run still reports a status and a plausible totalTokens, so a naive
|
|
5
|
+
per-stage decomposition silently mis-attributes cost (Fetch looked like 66% only because every
|
|
6
|
+
verifier crashed at ~1588 tok). This flags such stages so a cost number is never published on a
|
|
7
|
+
degraded run.
|
|
8
|
+
"""
|
|
9
|
+
import statistics
|
|
10
|
+
|
|
11
|
+
from agent_cost_attribution.ledger import agents
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def detect_degradation(run, *, min_agents=5, cheap_ratio=0.25):
|
|
15
|
+
"""Findings (empty == healthy). Each: {stage, kind, severity, detail}.
|
|
16
|
+
|
|
17
|
+
kind 'errors' : a stage has agents in state 'error'.
|
|
18
|
+
kind 'cheap' : a stage of >= min_agents whose mean tokens/agent is < cheap_ratio of the run-wide
|
|
19
|
+
baseline (median of per-stage mean costs) — the signature of a mass-degraded stage.
|
|
20
|
+
"""
|
|
21
|
+
ags = agents(run)
|
|
22
|
+
findings = []
|
|
23
|
+
|
|
24
|
+
by_stage = {}
|
|
25
|
+
for a in ags:
|
|
26
|
+
by_stage.setdefault(a.stage, []).append(a)
|
|
27
|
+
|
|
28
|
+
# Run-wide baseline: median of per-stage mean token costs (one data point per stage).
|
|
29
|
+
# Using per-stage means rather than per-agent tokens prevents a degraded stage's own
|
|
30
|
+
# cheap agents from dragging the median down and hiding itself.
|
|
31
|
+
stage_means = [statistics.mean(r.tokens for r in rows)
|
|
32
|
+
for rows in by_stage.values() if rows]
|
|
33
|
+
median = statistics.median(stage_means) if stage_means else 0
|
|
34
|
+
|
|
35
|
+
for stage, rows in by_stage.items():
|
|
36
|
+
errs = [r for r in rows if r.state == "error"]
|
|
37
|
+
if errs:
|
|
38
|
+
findings.append({"stage": stage, "kind": "errors", "severity": "high",
|
|
39
|
+
"detail": f"{len(errs)}/{len(rows)} agents in state 'error'"})
|
|
40
|
+
if median and len(rows) >= min_agents:
|
|
41
|
+
mean_tok = statistics.mean(r.tokens for r in rows)
|
|
42
|
+
if mean_tok < cheap_ratio * median:
|
|
43
|
+
findings.append({"stage": stage, "kind": "cheap", "severity": "high",
|
|
44
|
+
"detail": f"mean {mean_tok:.0f} tok/agent over {len(rows)} agents "
|
|
45
|
+
f"< {cheap_ratio:.0%} of run median {median:.0f}"})
|
|
46
|
+
return findings
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def is_healthy(run, **kw):
|
|
50
|
+
return not detect_degradation(run, **kw)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Per-stage token attribution for multi-agent workflow runs.
|
|
2
|
+
|
|
3
|
+
Reusable + stdlib-only: reads platform workflow telemetry (wf_*.json) and decomposes totalTokens
|
|
4
|
+
across stages (phases). No dependency on any workflow's internals, so it works for ANY workflow run,
|
|
5
|
+
not just deep-research.
|
|
6
|
+
|
|
7
|
+
Verified schema: a run JSON has top-level `totalTokens` and `workflowProgress`, a list whose entries
|
|
8
|
+
are either type=="workflow_phase" (headers, no tokens) or type=="workflow_agent" (per-agent records
|
|
9
|
+
with `phaseTitle`, `state`, `tokens`, `model`). Invariant: agent tokens sum EXACTLY to totalTokens.
|
|
10
|
+
|
|
11
|
+
Fail-soft: a missing/corrupt run degrades to {}, never crashes a caller.
|
|
12
|
+
"""
|
|
13
|
+
import json
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
_AGENT = "workflow_agent"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _as_int(value):
|
|
21
|
+
"""Coerce to int, fail-soft to 0 (a corrupt token/count field must never crash a parse)."""
|
|
22
|
+
try:
|
|
23
|
+
return int(value or 0)
|
|
24
|
+
except (ValueError, TypeError):
|
|
25
|
+
return 0
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class AgentRecord:
|
|
30
|
+
agent_id: str
|
|
31
|
+
label: str
|
|
32
|
+
stage: str
|
|
33
|
+
model: str
|
|
34
|
+
state: str
|
|
35
|
+
tokens: int
|
|
36
|
+
tool_calls: int
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load_run(path):
|
|
40
|
+
"""Load a run JSON; fail-soft to {} on missing/corrupt/non-object."""
|
|
41
|
+
try:
|
|
42
|
+
data = json.loads(Path(path).read_text())
|
|
43
|
+
except (OSError, ValueError, TypeError):
|
|
44
|
+
return {}
|
|
45
|
+
return data if isinstance(data, dict) else {}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def agents(run):
|
|
49
|
+
"""Per-agent records (type==workflow_agent). Phase-header rows are skipped."""
|
|
50
|
+
out = []
|
|
51
|
+
for e in (run.get("workflowProgress") or []):
|
|
52
|
+
if not isinstance(e, dict) or e.get("type") != _AGENT:
|
|
53
|
+
continue
|
|
54
|
+
out.append(AgentRecord(
|
|
55
|
+
agent_id=e.get("agentId", ""),
|
|
56
|
+
label=e.get("label", ""),
|
|
57
|
+
stage=e.get("phaseTitle", "?"),
|
|
58
|
+
model=e.get("model", ""),
|
|
59
|
+
state=e.get("state", "?"),
|
|
60
|
+
tokens=_as_int(e.get("tokens")),
|
|
61
|
+
tool_calls=_as_int(e.get("toolCalls")),
|
|
62
|
+
))
|
|
63
|
+
return out
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def total_tokens(run):
|
|
67
|
+
"""The run's reported total token count, fail-soft to 0."""
|
|
68
|
+
return _as_int(run.get("totalTokens"))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def decompose(run):
|
|
72
|
+
"""{stage: summed tokens}, preserving first-seen (phase) order."""
|
|
73
|
+
out = {}
|
|
74
|
+
for a in agents(run):
|
|
75
|
+
out[a.stage] = out.get(a.stage, 0) + a.tokens
|
|
76
|
+
return out
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def verify_invariant(run):
|
|
80
|
+
"""True iff per-agent tokens sum to the run's totalTokens — the parser's correctness check.
|
|
81
|
+
|
|
82
|
+
A run with no agent rows returns False (no data is not a satisfied invariant)."""
|
|
83
|
+
ags = agents(run)
|
|
84
|
+
if not ags:
|
|
85
|
+
return False
|
|
86
|
+
return sum(a.tokens for a in ags) == total_tokens(run)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def summary(run):
|
|
90
|
+
"""Per-stage decomposition with pct/count/state-mix, plus the invariant flag."""
|
|
91
|
+
tot = total_tokens(run)
|
|
92
|
+
stages = {}
|
|
93
|
+
for a in agents(run):
|
|
94
|
+
s = stages.setdefault(a.stage, {"tokens": 0, "count": 0, "states": {}})
|
|
95
|
+
s["tokens"] += a.tokens
|
|
96
|
+
s["count"] += 1
|
|
97
|
+
s["states"][a.state] = s["states"].get(a.state, 0) + 1
|
|
98
|
+
for s in stages.values():
|
|
99
|
+
s["pct"] = round(100 * s["tokens"] / tot, 1) if tot else 0.0
|
|
100
|
+
return {
|
|
101
|
+
"run_id": run.get("runId", ""),
|
|
102
|
+
"workflow": run.get("workflowName", ""),
|
|
103
|
+
"status": run.get("status", ""),
|
|
104
|
+
"total_tokens": tot,
|
|
105
|
+
"invariant_ok": verify_invariant(run),
|
|
106
|
+
"stages": stages,
|
|
107
|
+
}
|