pytest-llm-cost 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Matthew Busel
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,124 @@
1
+ Metadata-Version: 2.4
2
+ Name: pytest-llm-cost
3
+ Version: 0.1.0
4
+ Summary: pytest plugin: what each test's OpenAI, Anthropic and LiteLLM calls cost in USD, with a budget that fails the run
5
+ Author: Matthew Busel
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/Mattbusel/pytest-llm-cost
8
+ Project-URL: Documentation, https://github.com/Mattbusel/pytest-llm-cost/blob/main/docs/usage.md
9
+ Project-URL: Changelog, https://github.com/Mattbusel/pytest-llm-cost/blob/main/CHANGELOG.md
10
+ Project-URL: Issues, https://github.com/Mattbusel/pytest-llm-cost/issues
11
+ Project-URL: Prices, https://github.com/Mattbusel/llm-prices
12
+ Keywords: pytest,llm,cost,budget,tokens,openai,anthropic,litellm,evals
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Framework :: Pytest
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Topic :: Software Development :: Testing
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Requires-Python: >=3.9
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Requires-Dist: pytest>=7
25
+ Requires-Dist: llm-prices>=1.3
26
+ Provides-Extra: test
27
+ Requires-Dist: openai; extra == "test"
28
+ Requires-Dist: anthropic; extra == "test"
29
+ Requires-Dist: pytest-xdist; extra == "test"
30
+ Dynamic: license-file
31
+
32
+ # pytest-llm-cost
33
+
34
+ A pytest plugin that tells you what the LLM calls in your test suite cost, test by test, in US dollars. It can also fail the run when the suite goes over a budget.
35
+
36
+ ```
37
+ $ pip install pytest-llm-cost
38
+ $ pytest --llm-budget 0.50
39
+ ...
40
+ ================================== LLM cost ===================================
41
+ test calls in out USD
42
+ tests/test_agent.py::test_plans_a_trip 6 48,200 3,900 $0.203100
43
+ tests/test_rag.py::test_answers_with_sources 2 12,400 610 $0.002226
44
+ tests/test_rag.py::test_refuses_off_topic 1 1,900 40 $0.000309
45
+ by model: anthropic/claude-sonnet-4-6 6 x $0.203100; openai/gpt-4o-mini 3 x $0.002535
46
+ total $0.205635 over 9 calls in 3 tests (llm-prices 1.3.0, checked 2026-11-15)
47
+ budget $0.5, 41% used
48
+ ```
49
+
50
+ You do not need to change any code. Once installed, the plugin sees every response from:
51
+
52
+ - the **OpenAI** Python SDK: chat completions, the Responses API, embeddings, `parse`, streaming, sync and async;
53
+ - the **Anthropic** Python SDK: messages, `messages.stream()`, `create(stream=True)`, sync and async;
54
+ - **LiteLLM**: `litellm.completion` and `litellm.acompletion`. Each call is counted once, even though LiteLLM calls the OpenAI SDK underneath.
55
+
56
+ It reads the token counts each response reports. That includes cache reads and writes, so prompt caching is priced at the cached rate. It then prices those counts with [llm-prices](https://github.com/Mattbusel/llm-prices), a dated table of list prices for OpenAI, Anthropic, Google, Bedrock, Vertex, Azure, DeepSeek, Mistral, Groq, Together, Fireworks, xAI, Cohere and OpenRouter. Run `pip install -U llm-prices` to get new prices without upgrading the plugin.
57
+
58
+ ## Options
59
+
60
+ | Option | ini key | What it does |
61
+ |---|---|---|
62
+ | `--llm-budget USD` | `llm_budget` | Fails the run (exit code 1) when the LLM spend is over USD. |
63
+ | `--llm-cost-top N` | `llm_cost_top` | Sets how many of the most expensive tests the report lists (default 10, 0 for all). |
64
+ | `--llm-cost-json PATH` | | Writes every call, with its tokens, its price and the price row used, to a JSON file for CI dashboards. |
65
+
66
+ ```ini
67
+ # pytest.ini
68
+ [pytest]
69
+ llm_budget = 2.00
70
+ ```
71
+
72
+ The report only appears when the run made LLM calls or a budget is set. Turn the plugin off for one run with `pytest -p no:llm_cost`.
73
+
74
+ ## Budget per test
75
+
76
+ ```python
77
+ import pytest
78
+
79
+ @pytest.mark.llm_budget(0.02)
80
+ def test_summary_stays_cheap(agent):
81
+ agent.summarize(long_document)
82
+ ```
83
+
84
+ The test fails when its own calls cost more than $0.02.
85
+
86
+ ## The `llm_cost` fixture
87
+
88
+ ```python
89
+ def test_cache_hits_make_the_second_call_cheaper(client, llm_cost):
90
+ with llm_cost.measure() as first:
91
+ client.ask(question)
92
+ with llm_cost.measure() as second:
93
+ client.ask(question)
94
+ assert second.total < first.total / 2
95
+ ```
96
+
97
+ | Attribute | |
98
+ |---|---|
99
+ | `llm_cost.total` | USD spent by this test so far (a float; `total_decimal` is exact) |
100
+ | `llm_cost.calls` | every response seen: `model`, `provider`, `input_tokens`, `output_tokens`, `usd`, `priced_as` |
101
+ | `llm_cost.input_tokens`, `llm_cost.output_tokens` | token totals for the test |
102
+ | `llm_cost.unpriced` | calls with no price (unknown model, or a stream that reported no usage) |
103
+ | `llm_cost.measure()` | a context manager with the same attributes for only the calls made inside the block |
104
+ | `llm_cost.record(model, usage=..., provider=...)` | adds a call the plugin cannot see, such as another SDK, raw HTTP or a cassette |
105
+ | `llm_cost.session_total` | USD spent by the whole run so far |
106
+
107
+ ## How a call is priced
108
+
109
+ - The provider comes from the client's base URL: `api.openai.com` is OpenAI, `api.groq.com` is Groq, `*.openai.azure.com` is Azure, and so on. For LiteLLM it comes from LiteLLM's own provider name. A request served by Groq is never priced at OpenAI's rate.
110
+ - Any other host, such as a local server, a proxy or a mock, is priced by model id alone. A test that talks to a mocked `gpt-4o` therefore shows what that call would cost against the real API.
111
+ - A model llm-prices does not list (`llama3` on Ollama, a fine-tune id) is counted and reported as unpriced, never guessed.
112
+ - OpenAI streams only report usage when you pass `stream_options={"include_usage": True}`. Streams without it are counted and listed under "no usage".
113
+
114
+ ## pytest-xdist
115
+
116
+ Each worker records its own calls and sends them to the controller, which prints one report and applies the budget to the whole run. Inside a test, `llm_cost.session_total` covers only that worker.
117
+
118
+ ## Notes
119
+
120
+ - The plugin hooks an SDK only once something imports it, so a run that never imports `litellm` does not pay for importing it.
121
+ - Prices are list prices. Batch discounts, regional surcharges and negotiated rates are not applied.
122
+ - A failure inside the plugin never breaks your API call. It emits one warning and leaves that call unrecorded.
123
+
124
+ MIT licensed. Maintained by Matthew Busel. Price corrections go to [llm-prices](https://github.com/Mattbusel/llm-prices/issues).
@@ -0,0 +1,93 @@
1
+ # pytest-llm-cost
2
+
3
+ A pytest plugin that tells you what the LLM calls in your test suite cost, test by test, in US dollars. It can also fail the run when the suite goes over a budget.
4
+
5
+ ```
6
+ $ pip install pytest-llm-cost
7
+ $ pytest --llm-budget 0.50
8
+ ...
9
+ ================================== LLM cost ===================================
10
+ test calls in out USD
11
+ tests/test_agent.py::test_plans_a_trip 6 48,200 3,900 $0.203100
12
+ tests/test_rag.py::test_answers_with_sources 2 12,400 610 $0.002226
13
+ tests/test_rag.py::test_refuses_off_topic 1 1,900 40 $0.000309
14
+ by model: anthropic/claude-sonnet-4-6 6 x $0.203100; openai/gpt-4o-mini 3 x $0.002535
15
+ total $0.205635 over 9 calls in 3 tests (llm-prices 1.3.0, checked 2026-11-15)
16
+ budget $0.5, 41% used
17
+ ```
18
+
19
+ You do not need to change any code. Once installed, the plugin sees every response from:
20
+
21
+ - the **OpenAI** Python SDK: chat completions, the Responses API, embeddings, `parse`, streaming, sync and async;
22
+ - the **Anthropic** Python SDK: messages, `messages.stream()`, `create(stream=True)`, sync and async;
23
+ - **LiteLLM**: `litellm.completion` and `litellm.acompletion`. Each call is counted once, even though LiteLLM calls the OpenAI SDK underneath.
24
+
25
+ It reads the token counts each response reports. That includes cache reads and writes, so prompt caching is priced at the cached rate. It then prices those counts with [llm-prices](https://github.com/Mattbusel/llm-prices), a dated table of list prices for OpenAI, Anthropic, Google, Bedrock, Vertex, Azure, DeepSeek, Mistral, Groq, Together, Fireworks, xAI, Cohere and OpenRouter. Run `pip install -U llm-prices` to get new prices without upgrading the plugin.
26
+
27
+ ## Options
28
+
29
+ | Option | ini key | What it does |
30
+ |---|---|---|
31
+ | `--llm-budget USD` | `llm_budget` | Fails the run (exit code 1) when the LLM spend is over USD. |
32
+ | `--llm-cost-top N` | `llm_cost_top` | Sets how many of the most expensive tests the report lists (default 10, 0 for all). |
33
+ | `--llm-cost-json PATH` | | Writes every call, with its tokens, its price and the price row used, to a JSON file for CI dashboards. |
34
+
35
+ ```ini
36
+ # pytest.ini
37
+ [pytest]
38
+ llm_budget = 2.00
39
+ ```
40
+
41
+ The report only appears when the run made LLM calls or a budget is set. Turn the plugin off for one run with `pytest -p no:llm_cost`.
42
+
43
+ ## Budget per test
44
+
45
+ ```python
46
+ import pytest
47
+
48
+ @pytest.mark.llm_budget(0.02)
49
+ def test_summary_stays_cheap(agent):
50
+ agent.summarize(long_document)
51
+ ```
52
+
53
+ The test fails when its own calls cost more than $0.02.
54
+
55
+ ## The `llm_cost` fixture
56
+
57
+ ```python
58
+ def test_cache_hits_make_the_second_call_cheaper(client, llm_cost):
59
+ with llm_cost.measure() as first:
60
+ client.ask(question)
61
+ with llm_cost.measure() as second:
62
+ client.ask(question)
63
+ assert second.total < first.total / 2
64
+ ```
65
+
66
+ | Attribute | |
67
+ |---|---|
68
+ | `llm_cost.total` | USD spent by this test so far (a float; `total_decimal` is exact) |
69
+ | `llm_cost.calls` | every response seen: `model`, `provider`, `input_tokens`, `output_tokens`, `usd`, `priced_as` |
70
+ | `llm_cost.input_tokens`, `llm_cost.output_tokens` | token totals for the test |
71
+ | `llm_cost.unpriced` | calls with no price (unknown model, or a stream that reported no usage) |
72
+ | `llm_cost.measure()` | a context manager with the same attributes for only the calls made inside the block |
73
+ | `llm_cost.record(model, usage=..., provider=...)` | adds a call the plugin cannot see, such as another SDK, raw HTTP or a cassette |
74
+ | `llm_cost.session_total` | USD spent by the whole run so far |
75
+
76
+ ## How a call is priced
77
+
78
+ - The provider comes from the client's base URL: `api.openai.com` is OpenAI, `api.groq.com` is Groq, `*.openai.azure.com` is Azure, and so on. For LiteLLM it comes from LiteLLM's own provider name. A request served by Groq is never priced at OpenAI's rate.
79
+ - Any other host, such as a local server, a proxy or a mock, is priced by model id alone. A test that talks to a mocked `gpt-4o` therefore shows what that call would cost against the real API.
80
+ - A model llm-prices does not list (`llama3` on Ollama, a fine-tune id) is counted and reported as unpriced, never guessed.
81
+ - OpenAI streams only report usage when you pass `stream_options={"include_usage": True}`. Streams without it are counted and listed under "no usage".
82
+
83
+ ## pytest-xdist
84
+
85
+ Each worker records its own calls and sends them to the controller, which prints one report and applies the budget to the whole run. Inside a test, `llm_cost.session_total` covers only that worker.
86
+
87
+ ## Notes
88
+
89
+ - The plugin hooks an SDK only once something imports it, so a run that never imports `litellm` does not pay for importing it.
90
+ - Prices are list prices. Batch discounts, regional surcharges and negotiated rates are not applied.
91
+ - A failure inside the plugin never breaks your API call. It emits one warning and leaves that call unrecorded.
92
+
93
+ MIT licensed. Maintained by Matthew Busel. Price corrections go to [llm-prices](https://github.com/Mattbusel/llm-prices/issues).
@@ -0,0 +1,59 @@
1
+ [project]
2
+ name = "pytest-llm-cost"
3
+ version = "0.1.0"
4
+ description = "pytest plugin: what each test's OpenAI, Anthropic and LiteLLM calls cost in USD, with a budget that fails the run"
5
+ readme = "docs/usage.md"
6
+ authors = [{ name = "Matthew Busel" }]
7
+ license = "MIT"
8
+ license-files = ["LICENSE"]
9
+ requires-python = ">=3.9"
10
+ classifiers = [
11
+ "Development Status :: 4 - Beta",
12
+ "Framework :: Pytest",
13
+ "Intended Audience :: Developers",
14
+ "Operating System :: OS Independent",
15
+ "Programming Language :: Python :: 3",
16
+ "Programming Language :: Python :: 3 :: Only",
17
+ "Topic :: Software Development :: Testing",
18
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
19
+ ]
20
+ keywords = ["pytest", "llm", "cost", "budget", "tokens", "openai", "anthropic", "litellm", "evals"]
21
+ dependencies = [
22
+ "pytest>=7",
23
+ "llm-prices>=1.3",
24
+ ]
25
+
26
+ [project.urls]
27
+ Homepage = "https://github.com/Mattbusel/pytest-llm-cost"
28
+ Documentation = "https://github.com/Mattbusel/pytest-llm-cost/blob/main/docs/usage.md"
29
+ Changelog = "https://github.com/Mattbusel/pytest-llm-cost/blob/main/CHANGELOG.md"
30
+ Issues = "https://github.com/Mattbusel/pytest-llm-cost/issues"
31
+ Prices = "https://github.com/Mattbusel/llm-prices"
32
+
33
+ [project.entry-points.pytest11]
34
+ llm_cost = "pytest_llm_cost.plugin"
35
+
36
+ [project.optional-dependencies]
37
+ test = ["openai", "anthropic", "pytest-xdist"]
38
+
39
+ [build-system]
40
+ requires = ["setuptools>=77"]
41
+ build-backend = "setuptools.build_meta"
42
+
43
+ [tool.setuptools.packages.find]
44
+ where = ["src"]
45
+
46
+ [tool.pytest.ini_options]
47
+ testpaths = ["tests"]
48
+ addopts = "-p pytester"
49
+
50
+ [tool.ruff]
51
+ target-version = "py39"
52
+ line-length = 110
53
+
54
+ [tool.ruff.lint]
55
+ select = ["E", "F", "W", "B", "I"]
56
+ ignore = ["E501"]
57
+
58
+ [tool.black]
59
+ line-length = 110
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,20 @@
1
+ """pytest-llm-cost: what the LLM calls in your test suite cost, priced with llm-prices.
2
+
3
+ The plugin loads itself once installed. See ``docs/usage.md`` for options.
4
+ """
5
+
6
+ from ._core import Call, Tokens, find_price, provider_from_url, tokens_from_usage
7
+ from .plugin import CostView, LLMCost
8
+
9
+ __version__ = "0.1.0"
10
+
11
+ __all__ = [
12
+ "Call",
13
+ "CostView",
14
+ "LLMCost",
15
+ "Tokens",
16
+ "__version__",
17
+ "find_price",
18
+ "provider_from_url",
19
+ "tokens_from_usage",
20
+ ]
@@ -0,0 +1,351 @@
1
+ """Token usage parsing, provider detection, pricing and the call log.
2
+
3
+ Nothing here imports an LLM SDK. Usage objects are read by attribute or key, so pydantic models
4
+ from the OpenAI and Anthropic SDKs, LiteLLM's ``Usage`` and plain dicts all work.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import itertools
10
+ import threading
11
+ from dataclasses import dataclass
12
+ from decimal import Decimal
13
+ from typing import Any, Dict, Iterator, List, Mapping, Optional
14
+ from urllib.parse import urlsplit
15
+
16
+ import llm_prices
17
+
18
+ _MILLION = Decimal(1_000_000)
19
+
20
+
21
+ def _get(obj: Any, name: str, default: Any = None) -> Any:
22
+ """``obj.name`` or ``obj[name]``, with ``default`` for missing and ``None`` values."""
23
+ if obj is None:
24
+ return default
25
+ if isinstance(obj, Mapping):
26
+ value = obj.get(name, default)
27
+ else:
28
+ try:
29
+ value = getattr(obj, name, default)
30
+ except Exception:
31
+ value = default
32
+ return default if value is None else value
33
+
34
+
35
+ def _count(value: Any) -> int:
36
+ try:
37
+ return max(int(value), 0)
38
+ except (TypeError, ValueError):
39
+ return 0
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class Tokens:
44
+ """Token counts of one response, split the way llm-prices bills them."""
45
+
46
+ input: int = 0
47
+ output: int = 0
48
+ cache_read: int = 0
49
+ cache_write_5m: int = 0
50
+ cache_write_1h: int = 0
51
+
52
+ @property
53
+ def cache_write(self) -> int:
54
+ return self.cache_write_5m + self.cache_write_1h
55
+
56
+ def __add__(self, other: "Tokens") -> "Tokens":
57
+ return Tokens(
58
+ self.input + other.input,
59
+ self.output + other.output,
60
+ self.cache_read + other.cache_read,
61
+ self.cache_write_5m + other.cache_write_5m,
62
+ self.cache_write_1h + other.cache_write_1h,
63
+ )
64
+
65
+
66
+ def tokens_from_usage(usage: Any) -> Optional[Tokens]:
67
+ """Reads the ``usage`` of a response, or ``None`` when it holds no token counts.
68
+
69
+ Understands OpenAI Chat Completions and embeddings (``prompt_tokens``), the OpenAI Responses
70
+ API (``input_tokens`` with ``input_tokens_details``), Anthropic Messages (``input_tokens``
71
+ plus ``cache_read_input_tokens`` and ``cache_creation``), LiteLLM's ``Usage`` and the
72
+ google-genai ``usage_metadata`` fields. In the OpenAI and LiteLLM shapes the prompt count
73
+ includes cached tokens, so they are taken out of ``input``.
74
+ """
75
+ if usage is None:
76
+ return None
77
+
78
+ if _get(usage, "prompt_tokens") is not None or _get(usage, "completion_tokens") is not None:
79
+ prompt = _count(_get(usage, "prompt_tokens", 0))
80
+ cached = _count(_get(_get(usage, "prompt_tokens_details"), "cached_tokens", 0))
81
+ cached = cached or _count(_get(usage, "cache_read_input_tokens", 0))
82
+ written = _count(_get(usage, "cache_creation_input_tokens", 0))
83
+ return Tokens(
84
+ input=max(prompt - cached - written, 0),
85
+ output=_count(_get(usage, "completion_tokens", 0)),
86
+ cache_read=cached,
87
+ cache_write_5m=written,
88
+ )
89
+
90
+ if _get(usage, "input_tokens") is not None or _get(usage, "output_tokens") is not None:
91
+ details = _get(usage, "input_tokens_details")
92
+ if details is not None:
93
+ # OpenAI Responses API: input_tokens includes the cached tokens.
94
+ cached = _count(_get(details, "cached_tokens", 0))
95
+ return Tokens(
96
+ input=max(_count(_get(usage, "input_tokens", 0)) - cached, 0),
97
+ output=_count(_get(usage, "output_tokens", 0)),
98
+ cache_read=cached,
99
+ )
100
+ # Anthropic Messages: input_tokens excludes cache reads and writes.
101
+ creation = _get(usage, "cache_creation")
102
+ write_5m = _count(_get(creation, "ephemeral_5m_input_tokens", 0))
103
+ write_1h = _count(_get(creation, "ephemeral_1h_input_tokens", 0))
104
+ if write_5m + write_1h == 0:
105
+ write_5m = _count(_get(usage, "cache_creation_input_tokens", 0))
106
+ return Tokens(
107
+ input=_count(_get(usage, "input_tokens", 0)),
108
+ output=_count(_get(usage, "output_tokens", 0)),
109
+ cache_read=_count(_get(usage, "cache_read_input_tokens", 0)),
110
+ cache_write_5m=write_5m,
111
+ cache_write_1h=write_1h,
112
+ )
113
+
114
+ if _get(usage, "prompt_token_count") is not None:
115
+ # google-genai usage_metadata
116
+ cached = _count(_get(usage, "cached_content_token_count", 0))
117
+ return Tokens(
118
+ input=max(_count(_get(usage, "prompt_token_count", 0)) - cached, 0),
119
+ output=_count(_get(usage, "candidates_token_count", 0))
120
+ + _count(_get(usage, "thoughts_token_count", 0)),
121
+ cache_read=cached,
122
+ )
123
+
124
+ return None
125
+
126
+
127
+ # Hosts whose prices are known, matched against the client's base_url. Anything else (a local
128
+ # server, a proxy, a mock) is priced by model id alone.
129
+ _EXACT_HOSTS = {
130
+ "api.openai.com": "openai",
131
+ "api.anthropic.com": "anthropic",
132
+ "api.deepseek.com": "deepseek",
133
+ "api.groq.com": "groq",
134
+ "api.x.ai": "xai",
135
+ "api.together.xyz": "together",
136
+ "api.together.ai": "together",
137
+ "api.fireworks.ai": "fireworks",
138
+ "openrouter.ai": "openrouter",
139
+ "api.mistral.ai": "mistral",
140
+ "generativelanguage.googleapis.com": "google",
141
+ "api.cohere.ai": "cohere",
142
+ "api.cohere.com": "cohere",
143
+ }
144
+ _HOST_SUFFIXES = (
145
+ (".openai.azure.com", "azure"),
146
+ (".cognitiveservices.azure.com", "azure"),
147
+ (".services.ai.azure.com", "azure"),
148
+ ("aiplatform.googleapis.com", "vertex"),
149
+ )
150
+
151
+
152
+ def provider_from_url(url: Any) -> Optional[str]:
153
+ """The llm-prices provider name for an API base URL, or ``None`` for an unknown host."""
154
+ try:
155
+ host = (urlsplit(str(url)).hostname or "").lower()
156
+ except ValueError:
157
+ return None
158
+ if not host:
159
+ return None
160
+ if host in _EXACT_HOSTS:
161
+ return _EXACT_HOSTS[host]
162
+ if host.startswith("bedrock-runtime.") and host.endswith(".amazonaws.com"):
163
+ return "bedrock"
164
+ for suffix, provider in _HOST_SUFFIXES:
165
+ if host.endswith(suffix):
166
+ return provider
167
+ return None
168
+
169
+
170
+ def find_price(provider: Optional[str], model: str) -> Optional[llm_prices.Price]:
171
+ """The llm-prices row for ``model``.
172
+
173
+ With a known provider only that provider's rows (and, for Bedrock, Vertex and Azure, the
174
+ list price of the model maker they resell) are searched, so a request served by Groq is
175
+ never priced at OpenAI's rate. A provider name llm-prices does not know (``ollama``, a
176
+ custom LiteLLM provider) means no price. Without a provider the model id is looked up
177
+ across the whole table.
178
+ """
179
+ if not model:
180
+ return None
181
+ if provider:
182
+ if llm_prices.Provider.parse(provider) is None:
183
+ return None
184
+ return llm_prices.lookup_provider(provider, model)
185
+ return llm_prices.lookup(model)
186
+
187
+
188
+ def cost_of(price: Optional[llm_prices.Price], tokens: Tokens) -> Optional[Decimal]:
189
+ """Exact USD cost of one response, or ``None`` when a needed price is not listed."""
190
+ if price is None:
191
+ return None
192
+ usage = llm_prices.Usage(
193
+ input=tokens.input,
194
+ output=tokens.output,
195
+ cache_read=tokens.cache_read,
196
+ cache_write_5m=tokens.cache_write_5m,
197
+ cache_write_1h=tokens.cache_write_1h,
198
+ )
199
+ breakdown = price.cost_usage(usage, exact=True)
200
+ if breakdown is not None:
201
+ return breakdown.total
202
+ # Embedding rows list an input price only; a response with no output tokens is still priced.
203
+ if tokens.output == 0 and tokens.cache_read == 0 and tokens.cache_write == 0:
204
+ rate = price.rate_decimal("input")
205
+ if rate is not None:
206
+ return rate * tokens.input / _MILLION
207
+ return None
208
+
209
+
210
+ @dataclass(frozen=True)
211
+ class Call:
212
+ """One LLM response seen during the run."""
213
+
214
+ seq: int
215
+ test: str
216
+ source: str
217
+ provider: Optional[str]
218
+ model: str
219
+ tokens: Optional[Tokens]
220
+ cost: Optional[Decimal]
221
+ priced_as: Optional[str] = None
222
+
223
+ @property
224
+ def usd(self) -> Optional[float]:
225
+ """Cost in USD as a float, ``None`` when unpriced."""
226
+ return None if self.cost is None else float(self.cost)
227
+
228
+ @property
229
+ def priced(self) -> bool:
230
+ return self.cost is not None
231
+
232
+ @property
233
+ def input_tokens(self) -> int:
234
+ """Prompt tokens including cache reads and writes."""
235
+ t = self.tokens
236
+ return 0 if t is None else t.input + t.cache_read + t.cache_write
237
+
238
+ @property
239
+ def output_tokens(self) -> int:
240
+ return 0 if self.tokens is None else self.tokens.output
241
+
242
+ def to_dict(self) -> Dict[str, Any]:
243
+ t = self.tokens
244
+ return {
245
+ "test": self.test,
246
+ "source": self.source,
247
+ "provider": self.provider,
248
+ "model": self.model,
249
+ "input_tokens": None if t is None else t.input,
250
+ "output_tokens": None if t is None else t.output,
251
+ "cache_read_tokens": None if t is None else t.cache_read,
252
+ "cache_write_5m_tokens": None if t is None else t.cache_write_5m,
253
+ "cache_write_1h_tokens": None if t is None else t.cache_write_1h,
254
+ "usd": None if self.cost is None else str(self.cost),
255
+ "priced_as": self.priced_as,
256
+ }
257
+
258
+ @classmethod
259
+ def from_dict(cls, seq: int, data: Mapping[str, Any]) -> "Call":
260
+ tokens = None
261
+ if data.get("input_tokens") is not None:
262
+ tokens = Tokens(
263
+ input=int(data["input_tokens"]),
264
+ output=int(data["output_tokens"] or 0),
265
+ cache_read=int(data.get("cache_read_tokens") or 0),
266
+ cache_write_5m=int(data.get("cache_write_5m_tokens") or 0),
267
+ cache_write_1h=int(data.get("cache_write_1h_tokens") or 0),
268
+ )
269
+ usd = data.get("usd")
270
+ return cls(
271
+ seq=seq,
272
+ test=str(data.get("test") or ""),
273
+ source=str(data.get("source") or ""),
274
+ provider=data.get("provider"),
275
+ model=str(data.get("model") or ""),
276
+ tokens=tokens,
277
+ cost=None if usd is None else Decimal(usd),
278
+ priced_as=data.get("priced_as"),
279
+ )
280
+
281
+
282
+ class Recorder:
283
+ """The call log of one pytest session."""
284
+
285
+ def __init__(self) -> None:
286
+ self._lock = threading.Lock()
287
+ self._seq = itertools.count()
288
+ self.calls: List[Call] = []
289
+ self.current_test = ""
290
+
291
+ def record(
292
+ self,
293
+ *,
294
+ source: str,
295
+ model: Optional[str],
296
+ usage: Any = None,
297
+ tokens: Optional[Tokens] = None,
298
+ provider: Optional[str] = None,
299
+ test: Optional[str] = None,
300
+ ) -> Call:
301
+ """Prices one response and adds it to the log."""
302
+ if tokens is None:
303
+ tokens = tokens_from_usage(usage)
304
+ model = str(model or "")
305
+ price = cost = None
306
+ if tokens is not None:
307
+ price = find_price(provider, model)
308
+ cost = cost_of(price, tokens)
309
+ call = Call(
310
+ seq=next(self._seq),
311
+ test=self.current_test if test is None else test,
312
+ source=source,
313
+ provider=provider,
314
+ model=model,
315
+ tokens=tokens,
316
+ cost=cost,
317
+ priced_as=None if cost is None or price is None else f"{price.provider}/{price.model}",
318
+ )
319
+ with self._lock:
320
+ self.calls.append(call)
321
+ return call
322
+
323
+ def extend_from_dicts(self, rows: List[Mapping[str, Any]]) -> None:
324
+ with self._lock:
325
+ for row in rows:
326
+ self.calls.append(Call.from_dict(next(self._seq), row))
327
+
328
+ def snapshot(self) -> List[Call]:
329
+ with self._lock:
330
+ return list(self.calls)
331
+
332
+
333
+ # The recorders of the running sessions, innermost last (pytester runs sessions inside a test).
334
+ _ACTIVE: List[Recorder] = []
335
+
336
+
337
+ def push(recorder: Recorder) -> None:
338
+ _ACTIVE.append(recorder)
339
+
340
+
341
+ def pop(recorder: Recorder) -> None:
342
+ if recorder in _ACTIVE:
343
+ _ACTIVE.remove(recorder)
344
+
345
+
346
+ def active() -> Optional[Recorder]:
347
+ return _ACTIVE[-1] if _ACTIVE else None
348
+
349
+
350
+ def total(calls: Iterator[Call]) -> Decimal:
351
+ return sum((c.cost for c in calls if c.cost is not None), Decimal(0))