pytest-llm-cost 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_llm_cost-0.1.0/LICENSE +21 -0
- pytest_llm_cost-0.1.0/PKG-INFO +124 -0
- pytest_llm_cost-0.1.0/docs/usage.md +93 -0
- pytest_llm_cost-0.1.0/pyproject.toml +59 -0
- pytest_llm_cost-0.1.0/setup.cfg +4 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost/__init__.py +20 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost/_core.py +351 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost/_hooks.py +359 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost/plugin.py +415 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost.egg-info/PKG-INFO +124 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost.egg-info/SOURCES.txt +17 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost.egg-info/dependency_links.txt +1 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost.egg-info/entry_points.txt +2 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost.egg-info/requires.txt +7 -0
- pytest_llm_cost-0.1.0/src/pytest_llm_cost.egg-info/top_level.txt +1 -0
- pytest_llm_cost-0.1.0/tests/test_litellm.py +31 -0
- pytest_llm_cost-0.1.0/tests/test_plugin.py +138 -0
- pytest_llm_cost-0.1.0/tests/test_sdks.py +142 -0
- pytest_llm_cost-0.1.0/tests/test_usage.py +122 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Matthew Busel
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pytest-llm-cost
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: pytest plugin: what each test's OpenAI, Anthropic and LiteLLM calls cost in USD, with a budget that fails the run
|
|
5
|
+
Author: Matthew Busel
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Mattbusel/pytest-llm-cost
|
|
8
|
+
Project-URL: Documentation, https://github.com/Mattbusel/pytest-llm-cost/blob/main/docs/usage.md
|
|
9
|
+
Project-URL: Changelog, https://github.com/Mattbusel/pytest-llm-cost/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Issues, https://github.com/Mattbusel/pytest-llm-cost/issues
|
|
11
|
+
Project-URL: Prices, https://github.com/Mattbusel/llm-prices
|
|
12
|
+
Keywords: pytest,llm,cost,budget,tokens,openai,anthropic,litellm,evals
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Framework :: Pytest
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Topic :: Software Development :: Testing
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: pytest>=7
|
|
25
|
+
Requires-Dist: llm-prices>=1.3
|
|
26
|
+
Provides-Extra: test
|
|
27
|
+
Requires-Dist: openai; extra == "test"
|
|
28
|
+
Requires-Dist: anthropic; extra == "test"
|
|
29
|
+
Requires-Dist: pytest-xdist; extra == "test"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# pytest-llm-cost
|
|
33
|
+
|
|
34
|
+
A pytest plugin that tells you what the LLM calls in your test suite cost, test by test, in US dollars. It can also fail the run when the suite goes over a budget.
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
$ pip install pytest-llm-cost
|
|
38
|
+
$ pytest --llm-budget 0.50
|
|
39
|
+
...
|
|
40
|
+
================================== LLM cost ===================================
|
|
41
|
+
test calls in out USD
|
|
42
|
+
tests/test_agent.py::test_plans_a_trip 6 48,200 3,900 $0.203100
|
|
43
|
+
tests/test_rag.py::test_answers_with_sources 2 12,400 610 $0.002226
|
|
44
|
+
tests/test_rag.py::test_refuses_off_topic 1 1,900 40 $0.000309
|
|
45
|
+
by model: anthropic/claude-sonnet-4-6 6 x $0.203100; openai/gpt-4o-mini 3 x $0.002535
|
|
46
|
+
total $0.205635 over 9 calls in 3 tests (llm-prices 1.3.0, checked 2026-11-15)
|
|
47
|
+
budget $0.5, 41% used
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
You do not need to change any code. Once installed, the plugin sees every response from:
|
|
51
|
+
|
|
52
|
+
- the **OpenAI** Python SDK: chat completions, the Responses API, embeddings, `parse`, streaming, sync and async;
|
|
53
|
+
- the **Anthropic** Python SDK: messages, `messages.stream()`, `create(stream=True)`, sync and async;
|
|
54
|
+
- **LiteLLM**: `litellm.completion` and `litellm.acompletion`. Each call is counted once, even though LiteLLM calls the OpenAI SDK underneath.
|
|
55
|
+
|
|
56
|
+
It reads the token counts each response reports. That includes cache reads and writes, so prompt caching is priced at the cached rate. It then prices those counts with [llm-prices](https://github.com/Mattbusel/llm-prices), a dated table of list prices for OpenAI, Anthropic, Google, Bedrock, Vertex, Azure, DeepSeek, Mistral, Groq, Together, Fireworks, xAI, Cohere and OpenRouter. Run `pip install -U llm-prices` to get new prices without upgrading the plugin.
|
|
57
|
+
|
|
58
|
+
## Options
|
|
59
|
+
|
|
60
|
+
| Option | ini key | What it does |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| `--llm-budget USD` | `llm_budget` | Fails the run (exit code 1) when the LLM spend is over USD. |
|
|
63
|
+
| `--llm-cost-top N` | `llm_cost_top` | Sets how many of the most expensive tests the report lists (default 10, 0 for all). |
|
|
64
|
+
| `--llm-cost-json PATH` | | Writes every call, with its tokens, its price and the price row used, to a JSON file for CI dashboards. |
|
|
65
|
+
|
|
66
|
+
```ini
|
|
67
|
+
# pytest.ini
|
|
68
|
+
[pytest]
|
|
69
|
+
llm_budget = 2.00
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The report only appears when the run made LLM calls or a budget is set. Turn the plugin off for one run with `pytest -p no:llm_cost`.
|
|
73
|
+
|
|
74
|
+
## Budget per test
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import pytest
|
|
78
|
+
|
|
79
|
+
@pytest.mark.llm_budget(0.02)
|
|
80
|
+
def test_summary_stays_cheap(agent):
|
|
81
|
+
agent.summarize(long_document)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The test fails when its own calls cost more than $0.02.
|
|
85
|
+
|
|
86
|
+
## The `llm_cost` fixture
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
def test_cache_hits_make_the_second_call_cheaper(client, llm_cost):
|
|
90
|
+
with llm_cost.measure() as first:
|
|
91
|
+
client.ask(question)
|
|
92
|
+
with llm_cost.measure() as second:
|
|
93
|
+
client.ask(question)
|
|
94
|
+
assert second.total < first.total / 2
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
| Attribute | |
|
|
98
|
+
|---|---|
|
|
99
|
+
| `llm_cost.total` | USD spent by this test so far (a float; `total_decimal` is exact) |
|
|
100
|
+
| `llm_cost.calls` | every response seen: `model`, `provider`, `input_tokens`, `output_tokens`, `usd`, `priced_as` |
|
|
101
|
+
| `llm_cost.input_tokens`, `llm_cost.output_tokens` | token totals for the test |
|
|
102
|
+
| `llm_cost.unpriced` | calls with no price (unknown model, or a stream that reported no usage) |
|
|
103
|
+
| `llm_cost.measure()` | a context manager with the same attributes for only the calls made inside the block |
|
|
104
|
+
| `llm_cost.record(model, usage=..., provider=...)` | adds a call the plugin cannot see, such as another SDK, raw HTTP or a cassette |
|
|
105
|
+
| `llm_cost.session_total` | USD spent by the whole run so far |
|
|
106
|
+
|
|
107
|
+
## How a call is priced
|
|
108
|
+
|
|
109
|
+
- The provider comes from the client's base URL: `api.openai.com` is OpenAI, `api.groq.com` is Groq, `*.openai.azure.com` is Azure, and so on. For LiteLLM it comes from LiteLLM's own provider name. A request served by Groq is never priced at OpenAI's rate.
|
|
110
|
+
- Any other host, such as a local server, a proxy or a mock, is priced by model id alone. A test that talks to a mocked `gpt-4o` therefore shows what that call would cost against the real API.
|
|
111
|
+
- A model llm-prices does not list (`llama3` on Ollama, a fine-tune id) is counted and reported as unpriced, never guessed.
|
|
112
|
+
- OpenAI streams only report usage when you pass `stream_options={"include_usage": True}`. Streams without it are counted and listed under "no usage".
|
|
113
|
+
|
|
114
|
+
## pytest-xdist
|
|
115
|
+
|
|
116
|
+
Each worker records its own calls and sends them to the controller, which prints one report and applies the budget to the whole run. Inside a test, `llm_cost.session_total` covers only that worker.
|
|
117
|
+
|
|
118
|
+
## Notes
|
|
119
|
+
|
|
120
|
+
- The plugin hooks an SDK only once something imports it, so a run that never imports `litellm` does not pay for importing it.
|
|
121
|
+
- Prices are list prices. Batch discounts, regional surcharges and negotiated rates are not applied.
|
|
122
|
+
- A failure inside the plugin never breaks your API call. It emits one warning and leaves that call unrecorded.
|
|
123
|
+
|
|
124
|
+
MIT licensed. Maintained by Matthew Busel. Price corrections go to [llm-prices](https://github.com/Mattbusel/llm-prices/issues).
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# pytest-llm-cost
|
|
2
|
+
|
|
3
|
+
A pytest plugin that tells you what the LLM calls in your test suite cost, test by test, in US dollars. It can also fail the run when the suite goes over a budget.
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
$ pip install pytest-llm-cost
|
|
7
|
+
$ pytest --llm-budget 0.50
|
|
8
|
+
...
|
|
9
|
+
================================== LLM cost ===================================
|
|
10
|
+
test calls in out USD
|
|
11
|
+
tests/test_agent.py::test_plans_a_trip 6 48,200 3,900 $0.203100
|
|
12
|
+
tests/test_rag.py::test_answers_with_sources 2 12,400 610 $0.002226
|
|
13
|
+
tests/test_rag.py::test_refuses_off_topic 1 1,900 40 $0.000309
|
|
14
|
+
by model: anthropic/claude-sonnet-4-6 6 x $0.203100; openai/gpt-4o-mini 3 x $0.002535
|
|
15
|
+
total $0.205635 over 9 calls in 3 tests (llm-prices 1.3.0, checked 2026-11-15)
|
|
16
|
+
budget $0.5, 41% used
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
You do not need to change any code. Once installed, the plugin sees every response from:
|
|
20
|
+
|
|
21
|
+
- the **OpenAI** Python SDK: chat completions, the Responses API, embeddings, `parse`, streaming, sync and async;
|
|
22
|
+
- the **Anthropic** Python SDK: messages, `messages.stream()`, `create(stream=True)`, sync and async;
|
|
23
|
+
- **LiteLLM**: `litellm.completion` and `litellm.acompletion`. Each call is counted once, even though LiteLLM calls the OpenAI SDK underneath.
|
|
24
|
+
|
|
25
|
+
It reads the token counts each response reports. That includes cache reads and writes, so prompt caching is priced at the cached rate. It then prices those counts with [llm-prices](https://github.com/Mattbusel/llm-prices), a dated table of list prices for OpenAI, Anthropic, Google, Bedrock, Vertex, Azure, DeepSeek, Mistral, Groq, Together, Fireworks, xAI, Cohere and OpenRouter. Run `pip install -U llm-prices` to get new prices without upgrading the plugin.
|
|
26
|
+
|
|
27
|
+
## Options
|
|
28
|
+
|
|
29
|
+
| Option | ini key | What it does |
|
|
30
|
+
|---|---|---|
|
|
31
|
+
| `--llm-budget USD` | `llm_budget` | Fails the run (exit code 1) when the LLM spend is over USD. |
|
|
32
|
+
| `--llm-cost-top N` | `llm_cost_top` | Sets how many of the most expensive tests the report lists (default 10, 0 for all). |
|
|
33
|
+
| `--llm-cost-json PATH` | | Writes every call, with its tokens, its price and the price row used, to a JSON file for CI dashboards. |
|
|
34
|
+
|
|
35
|
+
```ini
|
|
36
|
+
# pytest.ini
|
|
37
|
+
[pytest]
|
|
38
|
+
llm_budget = 2.00
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The report only appears when the run made LLM calls or a budget is set. Turn the plugin off for one run with `pytest -p no:llm_cost`.
|
|
42
|
+
|
|
43
|
+
## Budget per test
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import pytest
|
|
47
|
+
|
|
48
|
+
@pytest.mark.llm_budget(0.02)
|
|
49
|
+
def test_summary_stays_cheap(agent):
|
|
50
|
+
agent.summarize(long_document)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The test fails when its own calls cost more than $0.02.
|
|
54
|
+
|
|
55
|
+
## The `llm_cost` fixture
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
def test_cache_hits_make_the_second_call_cheaper(client, llm_cost):
|
|
59
|
+
with llm_cost.measure() as first:
|
|
60
|
+
client.ask(question)
|
|
61
|
+
with llm_cost.measure() as second:
|
|
62
|
+
client.ask(question)
|
|
63
|
+
assert second.total < first.total / 2
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
| Attribute | |
|
|
67
|
+
|---|---|
|
|
68
|
+
| `llm_cost.total` | USD spent by this test so far (a float; `total_decimal` is exact) |
|
|
69
|
+
| `llm_cost.calls` | every response seen: `model`, `provider`, `input_tokens`, `output_tokens`, `usd`, `priced_as` |
|
|
70
|
+
| `llm_cost.input_tokens`, `llm_cost.output_tokens` | token totals for the test |
|
|
71
|
+
| `llm_cost.unpriced` | calls with no price (unknown model, or a stream that reported no usage) |
|
|
72
|
+
| `llm_cost.measure()` | a context manager with the same attributes for only the calls made inside the block |
|
|
73
|
+
| `llm_cost.record(model, usage=..., provider=...)` | adds a call the plugin cannot see, such as another SDK, raw HTTP or a cassette |
|
|
74
|
+
| `llm_cost.session_total` | USD spent by the whole run so far |
|
|
75
|
+
|
|
76
|
+
## How a call is priced
|
|
77
|
+
|
|
78
|
+
- The provider comes from the client's base URL: `api.openai.com` is OpenAI, `api.groq.com` is Groq, `*.openai.azure.com` is Azure, and so on. For LiteLLM it comes from LiteLLM's own provider name. A request served by Groq is never priced at OpenAI's rate.
|
|
79
|
+
- Any other host, such as a local server, a proxy or a mock, is priced by model id alone. A test that talks to a mocked `gpt-4o` therefore shows what that call would cost against the real API.
|
|
80
|
+
- A model llm-prices does not list (`llama3` on Ollama, a fine-tune id) is counted and reported as unpriced, never guessed.
|
|
81
|
+
- OpenAI streams only report usage when you pass `stream_options={"include_usage": True}`. Streams without it are counted and listed under "no usage".
|
|
82
|
+
|
|
83
|
+
## pytest-xdist
|
|
84
|
+
|
|
85
|
+
Each worker records its own calls and sends them to the controller, which prints one report and applies the budget to the whole run. Inside a test, `llm_cost.session_total` covers only that worker.
|
|
86
|
+
|
|
87
|
+
## Notes
|
|
88
|
+
|
|
89
|
+
- The plugin hooks an SDK only once something imports it, so a run that never imports `litellm` does not pay for importing it.
|
|
90
|
+
- Prices are list prices. Batch discounts, regional surcharges and negotiated rates are not applied.
|
|
91
|
+
- A failure inside the plugin never breaks your API call. It emits one warning and leaves that call unrecorded.
|
|
92
|
+
|
|
93
|
+
MIT licensed. Maintained by Matthew Busel. Price corrections go to [llm-prices](https://github.com/Mattbusel/llm-prices/issues).
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pytest-llm-cost"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "pytest plugin: what each test's OpenAI, Anthropic and LiteLLM calls cost in USD, with a budget that fails the run"
|
|
5
|
+
readme = "docs/usage.md"
|
|
6
|
+
authors = [{ name = "Matthew Busel" }]
|
|
7
|
+
license = "MIT"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
requires-python = ">=3.9"
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 4 - Beta",
|
|
12
|
+
"Framework :: Pytest",
|
|
13
|
+
"Intended Audience :: Developers",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
17
|
+
"Topic :: Software Development :: Testing",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
19
|
+
]
|
|
20
|
+
keywords = ["pytest", "llm", "cost", "budget", "tokens", "openai", "anthropic", "litellm", "evals"]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"pytest>=7",
|
|
23
|
+
"llm-prices>=1.3",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/Mattbusel/pytest-llm-cost"
|
|
28
|
+
Documentation = "https://github.com/Mattbusel/pytest-llm-cost/blob/main/docs/usage.md"
|
|
29
|
+
Changelog = "https://github.com/Mattbusel/pytest-llm-cost/blob/main/CHANGELOG.md"
|
|
30
|
+
Issues = "https://github.com/Mattbusel/pytest-llm-cost/issues"
|
|
31
|
+
Prices = "https://github.com/Mattbusel/llm-prices"
|
|
32
|
+
|
|
33
|
+
[project.entry-points.pytest11]
|
|
34
|
+
llm_cost = "pytest_llm_cost.plugin"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
test = ["openai", "anthropic", "pytest-xdist"]
|
|
38
|
+
|
|
39
|
+
[build-system]
|
|
40
|
+
requires = ["setuptools>=77"]
|
|
41
|
+
build-backend = "setuptools.build_meta"
|
|
42
|
+
|
|
43
|
+
[tool.setuptools.packages.find]
|
|
44
|
+
where = ["src"]
|
|
45
|
+
|
|
46
|
+
[tool.pytest.ini_options]
|
|
47
|
+
testpaths = ["tests"]
|
|
48
|
+
addopts = "-p pytester"
|
|
49
|
+
|
|
50
|
+
[tool.ruff]
|
|
51
|
+
target-version = "py39"
|
|
52
|
+
line-length = 110
|
|
53
|
+
|
|
54
|
+
[tool.ruff.lint]
|
|
55
|
+
select = ["E", "F", "W", "B", "I"]
|
|
56
|
+
ignore = ["E501"]
|
|
57
|
+
|
|
58
|
+
[tool.black]
|
|
59
|
+
line-length = 110
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""pytest-llm-cost: what the LLM calls in your test suite cost, priced with llm-prices.
|
|
2
|
+
|
|
3
|
+
The plugin loads itself once installed. See ``docs/usage.md`` for options.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from ._core import Call, Tokens, find_price, provider_from_url, tokens_from_usage
|
|
7
|
+
from .plugin import CostView, LLMCost
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Call",
|
|
13
|
+
"CostView",
|
|
14
|
+
"LLMCost",
|
|
15
|
+
"Tokens",
|
|
16
|
+
"__version__",
|
|
17
|
+
"find_price",
|
|
18
|
+
"provider_from_url",
|
|
19
|
+
"tokens_from_usage",
|
|
20
|
+
]
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
"""Token usage parsing, provider detection, pricing and the call log.
|
|
2
|
+
|
|
3
|
+
Nothing here imports an LLM SDK. Usage objects are read by attribute or key, so pydantic models
|
|
4
|
+
from the OpenAI and Anthropic SDKs, LiteLLM's ``Usage`` and plain dicts all work.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import itertools
|
|
10
|
+
import threading
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from decimal import Decimal
|
|
13
|
+
from typing import Any, Dict, Iterator, List, Mapping, Optional
|
|
14
|
+
from urllib.parse import urlsplit
|
|
15
|
+
|
|
16
|
+
import llm_prices
|
|
17
|
+
|
|
18
|
+
_MILLION = Decimal(1_000_000)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _get(obj: Any, name: str, default: Any = None) -> Any:
|
|
22
|
+
"""``obj.name`` or ``obj[name]``, with ``default`` for missing and ``None`` values."""
|
|
23
|
+
if obj is None:
|
|
24
|
+
return default
|
|
25
|
+
if isinstance(obj, Mapping):
|
|
26
|
+
value = obj.get(name, default)
|
|
27
|
+
else:
|
|
28
|
+
try:
|
|
29
|
+
value = getattr(obj, name, default)
|
|
30
|
+
except Exception:
|
|
31
|
+
value = default
|
|
32
|
+
return default if value is None else value
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _count(value: Any) -> int:
|
|
36
|
+
try:
|
|
37
|
+
return max(int(value), 0)
|
|
38
|
+
except (TypeError, ValueError):
|
|
39
|
+
return 0
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class Tokens:
|
|
44
|
+
"""Token counts of one response, split the way llm-prices bills them."""
|
|
45
|
+
|
|
46
|
+
input: int = 0
|
|
47
|
+
output: int = 0
|
|
48
|
+
cache_read: int = 0
|
|
49
|
+
cache_write_5m: int = 0
|
|
50
|
+
cache_write_1h: int = 0
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def cache_write(self) -> int:
|
|
54
|
+
return self.cache_write_5m + self.cache_write_1h
|
|
55
|
+
|
|
56
|
+
def __add__(self, other: "Tokens") -> "Tokens":
|
|
57
|
+
return Tokens(
|
|
58
|
+
self.input + other.input,
|
|
59
|
+
self.output + other.output,
|
|
60
|
+
self.cache_read + other.cache_read,
|
|
61
|
+
self.cache_write_5m + other.cache_write_5m,
|
|
62
|
+
self.cache_write_1h + other.cache_write_1h,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def tokens_from_usage(usage: Any) -> Optional[Tokens]:
|
|
67
|
+
"""Reads the ``usage`` of a response, or ``None`` when it holds no token counts.
|
|
68
|
+
|
|
69
|
+
Understands OpenAI Chat Completions and embeddings (``prompt_tokens``), the OpenAI Responses
|
|
70
|
+
API (``input_tokens`` with ``input_tokens_details``), Anthropic Messages (``input_tokens``
|
|
71
|
+
plus ``cache_read_input_tokens`` and ``cache_creation``), LiteLLM's ``Usage`` and the
|
|
72
|
+
google-genai ``usage_metadata`` fields. In the OpenAI and LiteLLM shapes the prompt count
|
|
73
|
+
includes cached tokens, so they are taken out of ``input``.
|
|
74
|
+
"""
|
|
75
|
+
if usage is None:
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
if _get(usage, "prompt_tokens") is not None or _get(usage, "completion_tokens") is not None:
|
|
79
|
+
prompt = _count(_get(usage, "prompt_tokens", 0))
|
|
80
|
+
cached = _count(_get(_get(usage, "prompt_tokens_details"), "cached_tokens", 0))
|
|
81
|
+
cached = cached or _count(_get(usage, "cache_read_input_tokens", 0))
|
|
82
|
+
written = _count(_get(usage, "cache_creation_input_tokens", 0))
|
|
83
|
+
return Tokens(
|
|
84
|
+
input=max(prompt - cached - written, 0),
|
|
85
|
+
output=_count(_get(usage, "completion_tokens", 0)),
|
|
86
|
+
cache_read=cached,
|
|
87
|
+
cache_write_5m=written,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
if _get(usage, "input_tokens") is not None or _get(usage, "output_tokens") is not None:
|
|
91
|
+
details = _get(usage, "input_tokens_details")
|
|
92
|
+
if details is not None:
|
|
93
|
+
# OpenAI Responses API: input_tokens includes the cached tokens.
|
|
94
|
+
cached = _count(_get(details, "cached_tokens", 0))
|
|
95
|
+
return Tokens(
|
|
96
|
+
input=max(_count(_get(usage, "input_tokens", 0)) - cached, 0),
|
|
97
|
+
output=_count(_get(usage, "output_tokens", 0)),
|
|
98
|
+
cache_read=cached,
|
|
99
|
+
)
|
|
100
|
+
# Anthropic Messages: input_tokens excludes cache reads and writes.
|
|
101
|
+
creation = _get(usage, "cache_creation")
|
|
102
|
+
write_5m = _count(_get(creation, "ephemeral_5m_input_tokens", 0))
|
|
103
|
+
write_1h = _count(_get(creation, "ephemeral_1h_input_tokens", 0))
|
|
104
|
+
if write_5m + write_1h == 0:
|
|
105
|
+
write_5m = _count(_get(usage, "cache_creation_input_tokens", 0))
|
|
106
|
+
return Tokens(
|
|
107
|
+
input=_count(_get(usage, "input_tokens", 0)),
|
|
108
|
+
output=_count(_get(usage, "output_tokens", 0)),
|
|
109
|
+
cache_read=_count(_get(usage, "cache_read_input_tokens", 0)),
|
|
110
|
+
cache_write_5m=write_5m,
|
|
111
|
+
cache_write_1h=write_1h,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
if _get(usage, "prompt_token_count") is not None:
|
|
115
|
+
# google-genai usage_metadata
|
|
116
|
+
cached = _count(_get(usage, "cached_content_token_count", 0))
|
|
117
|
+
return Tokens(
|
|
118
|
+
input=max(_count(_get(usage, "prompt_token_count", 0)) - cached, 0),
|
|
119
|
+
output=_count(_get(usage, "candidates_token_count", 0))
|
|
120
|
+
+ _count(_get(usage, "thoughts_token_count", 0)),
|
|
121
|
+
cache_read=cached,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# Hosts whose prices are known, matched against the client's base_url. Anything else (a local
|
|
128
|
+
# server, a proxy, a mock) is priced by model id alone.
|
|
129
|
+
_EXACT_HOSTS = {
|
|
130
|
+
"api.openai.com": "openai",
|
|
131
|
+
"api.anthropic.com": "anthropic",
|
|
132
|
+
"api.deepseek.com": "deepseek",
|
|
133
|
+
"api.groq.com": "groq",
|
|
134
|
+
"api.x.ai": "xai",
|
|
135
|
+
"api.together.xyz": "together",
|
|
136
|
+
"api.together.ai": "together",
|
|
137
|
+
"api.fireworks.ai": "fireworks",
|
|
138
|
+
"openrouter.ai": "openrouter",
|
|
139
|
+
"api.mistral.ai": "mistral",
|
|
140
|
+
"generativelanguage.googleapis.com": "google",
|
|
141
|
+
"api.cohere.ai": "cohere",
|
|
142
|
+
"api.cohere.com": "cohere",
|
|
143
|
+
}
|
|
144
|
+
_HOST_SUFFIXES = (
|
|
145
|
+
(".openai.azure.com", "azure"),
|
|
146
|
+
(".cognitiveservices.azure.com", "azure"),
|
|
147
|
+
(".services.ai.azure.com", "azure"),
|
|
148
|
+
("aiplatform.googleapis.com", "vertex"),
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def provider_from_url(url: Any) -> Optional[str]:
|
|
153
|
+
"""The llm-prices provider name for an API base URL, or ``None`` for an unknown host."""
|
|
154
|
+
try:
|
|
155
|
+
host = (urlsplit(str(url)).hostname or "").lower()
|
|
156
|
+
except ValueError:
|
|
157
|
+
return None
|
|
158
|
+
if not host:
|
|
159
|
+
return None
|
|
160
|
+
if host in _EXACT_HOSTS:
|
|
161
|
+
return _EXACT_HOSTS[host]
|
|
162
|
+
if host.startswith("bedrock-runtime.") and host.endswith(".amazonaws.com"):
|
|
163
|
+
return "bedrock"
|
|
164
|
+
for suffix, provider in _HOST_SUFFIXES:
|
|
165
|
+
if host.endswith(suffix):
|
|
166
|
+
return provider
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def find_price(provider: Optional[str], model: str) -> Optional[llm_prices.Price]:
|
|
171
|
+
"""The llm-prices row for ``model``.
|
|
172
|
+
|
|
173
|
+
With a known provider only that provider's rows (and, for Bedrock, Vertex and Azure, the
|
|
174
|
+
list price of the model maker they resell) are searched, so a request served by Groq is
|
|
175
|
+
never priced at OpenAI's rate. A provider name llm-prices does not know (``ollama``, a
|
|
176
|
+
custom LiteLLM provider) means no price. Without a provider the model id is looked up
|
|
177
|
+
across the whole table.
|
|
178
|
+
"""
|
|
179
|
+
if not model:
|
|
180
|
+
return None
|
|
181
|
+
if provider:
|
|
182
|
+
if llm_prices.Provider.parse(provider) is None:
|
|
183
|
+
return None
|
|
184
|
+
return llm_prices.lookup_provider(provider, model)
|
|
185
|
+
return llm_prices.lookup(model)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def cost_of(price: Optional[llm_prices.Price], tokens: Tokens) -> Optional[Decimal]:
|
|
189
|
+
"""Exact USD cost of one response, or ``None`` when a needed price is not listed."""
|
|
190
|
+
if price is None:
|
|
191
|
+
return None
|
|
192
|
+
usage = llm_prices.Usage(
|
|
193
|
+
input=tokens.input,
|
|
194
|
+
output=tokens.output,
|
|
195
|
+
cache_read=tokens.cache_read,
|
|
196
|
+
cache_write_5m=tokens.cache_write_5m,
|
|
197
|
+
cache_write_1h=tokens.cache_write_1h,
|
|
198
|
+
)
|
|
199
|
+
breakdown = price.cost_usage(usage, exact=True)
|
|
200
|
+
if breakdown is not None:
|
|
201
|
+
return breakdown.total
|
|
202
|
+
# Embedding rows list an input price only; a response with no output tokens is still priced.
|
|
203
|
+
if tokens.output == 0 and tokens.cache_read == 0 and tokens.cache_write == 0:
|
|
204
|
+
rate = price.rate_decimal("input")
|
|
205
|
+
if rate is not None:
|
|
206
|
+
return rate * tokens.input / _MILLION
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@dataclass(frozen=True)
|
|
211
|
+
class Call:
|
|
212
|
+
"""One LLM response seen during the run."""
|
|
213
|
+
|
|
214
|
+
seq: int
|
|
215
|
+
test: str
|
|
216
|
+
source: str
|
|
217
|
+
provider: Optional[str]
|
|
218
|
+
model: str
|
|
219
|
+
tokens: Optional[Tokens]
|
|
220
|
+
cost: Optional[Decimal]
|
|
221
|
+
priced_as: Optional[str] = None
|
|
222
|
+
|
|
223
|
+
@property
|
|
224
|
+
def usd(self) -> Optional[float]:
|
|
225
|
+
"""Cost in USD as a float, ``None`` when unpriced."""
|
|
226
|
+
return None if self.cost is None else float(self.cost)
|
|
227
|
+
|
|
228
|
+
@property
|
|
229
|
+
def priced(self) -> bool:
|
|
230
|
+
return self.cost is not None
|
|
231
|
+
|
|
232
|
+
@property
|
|
233
|
+
def input_tokens(self) -> int:
|
|
234
|
+
"""Prompt tokens including cache reads and writes."""
|
|
235
|
+
t = self.tokens
|
|
236
|
+
return 0 if t is None else t.input + t.cache_read + t.cache_write
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def output_tokens(self) -> int:
|
|
240
|
+
return 0 if self.tokens is None else self.tokens.output
|
|
241
|
+
|
|
242
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
243
|
+
t = self.tokens
|
|
244
|
+
return {
|
|
245
|
+
"test": self.test,
|
|
246
|
+
"source": self.source,
|
|
247
|
+
"provider": self.provider,
|
|
248
|
+
"model": self.model,
|
|
249
|
+
"input_tokens": None if t is None else t.input,
|
|
250
|
+
"output_tokens": None if t is None else t.output,
|
|
251
|
+
"cache_read_tokens": None if t is None else t.cache_read,
|
|
252
|
+
"cache_write_5m_tokens": None if t is None else t.cache_write_5m,
|
|
253
|
+
"cache_write_1h_tokens": None if t is None else t.cache_write_1h,
|
|
254
|
+
"usd": None if self.cost is None else str(self.cost),
|
|
255
|
+
"priced_as": self.priced_as,
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
@classmethod
|
|
259
|
+
def from_dict(cls, seq: int, data: Mapping[str, Any]) -> "Call":
|
|
260
|
+
tokens = None
|
|
261
|
+
if data.get("input_tokens") is not None:
|
|
262
|
+
tokens = Tokens(
|
|
263
|
+
input=int(data["input_tokens"]),
|
|
264
|
+
output=int(data["output_tokens"] or 0),
|
|
265
|
+
cache_read=int(data.get("cache_read_tokens") or 0),
|
|
266
|
+
cache_write_5m=int(data.get("cache_write_5m_tokens") or 0),
|
|
267
|
+
cache_write_1h=int(data.get("cache_write_1h_tokens") or 0),
|
|
268
|
+
)
|
|
269
|
+
usd = data.get("usd")
|
|
270
|
+
return cls(
|
|
271
|
+
seq=seq,
|
|
272
|
+
test=str(data.get("test") or ""),
|
|
273
|
+
source=str(data.get("source") or ""),
|
|
274
|
+
provider=data.get("provider"),
|
|
275
|
+
model=str(data.get("model") or ""),
|
|
276
|
+
tokens=tokens,
|
|
277
|
+
cost=None if usd is None else Decimal(usd),
|
|
278
|
+
priced_as=data.get("priced_as"),
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
class Recorder:
|
|
283
|
+
"""The call log of one pytest session."""
|
|
284
|
+
|
|
285
|
+
def __init__(self) -> None:
|
|
286
|
+
self._lock = threading.Lock()
|
|
287
|
+
self._seq = itertools.count()
|
|
288
|
+
self.calls: List[Call] = []
|
|
289
|
+
self.current_test = ""
|
|
290
|
+
|
|
291
|
+
def record(
|
|
292
|
+
self,
|
|
293
|
+
*,
|
|
294
|
+
source: str,
|
|
295
|
+
model: Optional[str],
|
|
296
|
+
usage: Any = None,
|
|
297
|
+
tokens: Optional[Tokens] = None,
|
|
298
|
+
provider: Optional[str] = None,
|
|
299
|
+
test: Optional[str] = None,
|
|
300
|
+
) -> Call:
|
|
301
|
+
"""Prices one response and adds it to the log."""
|
|
302
|
+
if tokens is None:
|
|
303
|
+
tokens = tokens_from_usage(usage)
|
|
304
|
+
model = str(model or "")
|
|
305
|
+
price = cost = None
|
|
306
|
+
if tokens is not None:
|
|
307
|
+
price = find_price(provider, model)
|
|
308
|
+
cost = cost_of(price, tokens)
|
|
309
|
+
call = Call(
|
|
310
|
+
seq=next(self._seq),
|
|
311
|
+
test=self.current_test if test is None else test,
|
|
312
|
+
source=source,
|
|
313
|
+
provider=provider,
|
|
314
|
+
model=model,
|
|
315
|
+
tokens=tokens,
|
|
316
|
+
cost=cost,
|
|
317
|
+
priced_as=None if cost is None or price is None else f"{price.provider}/{price.model}",
|
|
318
|
+
)
|
|
319
|
+
with self._lock:
|
|
320
|
+
self.calls.append(call)
|
|
321
|
+
return call
|
|
322
|
+
|
|
323
|
+
def extend_from_dicts(self, rows: List[Mapping[str, Any]]) -> None:
|
|
324
|
+
with self._lock:
|
|
325
|
+
for row in rows:
|
|
326
|
+
self.calls.append(Call.from_dict(next(self._seq), row))
|
|
327
|
+
|
|
328
|
+
def snapshot(self) -> List[Call]:
|
|
329
|
+
with self._lock:
|
|
330
|
+
return list(self.calls)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
# The recorders of the running sessions, innermost last (pytester runs sessions inside a test).
|
|
334
|
+
_ACTIVE: List[Recorder] = []
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def push(recorder: Recorder) -> None:
|
|
338
|
+
_ACTIVE.append(recorder)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def pop(recorder: Recorder) -> None:
|
|
342
|
+
if recorder in _ACTIVE:
|
|
343
|
+
_ACTIVE.remove(recorder)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def active() -> Optional[Recorder]:
|
|
347
|
+
return _ACTIVE[-1] if _ACTIVE else None
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def total(calls: Iterator[Call]) -> Decimal:
|
|
351
|
+
return sum((c.cost for c in calls if c.cost is not None), Decimal(0))
|