llmcalc 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmcalc-0.1.0/.github/workflows/ci.yml +40 -0
- llmcalc-0.1.0/.gitignore +12 -0
- llmcalc-0.1.0/AGENTS.md +34 -0
- llmcalc-0.1.0/CHANGELOG.md +4 -0
- llmcalc-0.1.0/LICENSE +21 -0
- llmcalc-0.1.0/PKG-INFO +73 -0
- llmcalc-0.1.0/README.md +43 -0
- llmcalc-0.1.0/llmcalc/__init__.py +17 -0
- llmcalc-0.1.0/llmcalc/api.py +120 -0
- llmcalc-0.1.0/llmcalc/cache.py +57 -0
- llmcalc-0.1.0/llmcalc/cli.py +131 -0
- llmcalc-0.1.0/llmcalc/config.py +60 -0
- llmcalc-0.1.0/llmcalc/errors.py +13 -0
- llmcalc-0.1.0/llmcalc/models.py +99 -0
- llmcalc-0.1.0/llmcalc/normalize.py +63 -0
- llmcalc-0.1.0/llmcalc/pricing_client.py +92 -0
- llmcalc-0.1.0/llmcalc/py.typed +0 -0
- llmcalc-0.1.0/pyproject.toml +63 -0
- llmcalc-0.1.0/tests/test_api.py +125 -0
- llmcalc-0.1.0/tests/test_cache.py +44 -0
- llmcalc-0.1.0/tests/test_cli.py +18 -0
- llmcalc-0.1.0/tests/test_models.py +27 -0
- llmcalc-0.1.0/tests/test_normalize.py +20 -0
- llmcalc-0.1.0/tests/test_pricing_client.py +125 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: ["main"]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
|
|
12
|
+
steps:
|
|
13
|
+
- name: Checkout
|
|
14
|
+
uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- name: Setup Python
|
|
17
|
+
uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.11"
|
|
20
|
+
|
|
21
|
+
- name: Install dependencies
|
|
22
|
+
run: |
|
|
23
|
+
python -m pip install --upgrade pip
|
|
24
|
+
pip install -e '.[dev]'
|
|
25
|
+
pip install build
|
|
26
|
+
|
|
27
|
+
- name: Lint
|
|
28
|
+
run: python -m ruff check .
|
|
29
|
+
|
|
30
|
+
- name: Type check
|
|
31
|
+
run: python -m mypy llmcalc
|
|
32
|
+
|
|
33
|
+
- name: Test
|
|
34
|
+
run: python -m pytest
|
|
35
|
+
|
|
36
|
+
- name: Build
|
|
37
|
+
run: python -m build
|
|
38
|
+
|
|
39
|
+
- name: Check distribution metadata
|
|
40
|
+
run: python -m twine check dist/*
|
llmcalc-0.1.0/.gitignore
ADDED
llmcalc-0.1.0/AGENTS.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# AGENTS.md
|
|
2
|
+
|
|
3
|
+
## Project
|
|
4
|
+
- Name: `llmcalc`
|
|
5
|
+
- Purpose: Calculate LLM token costs from `llmlite` pricing data.
|
|
6
|
+
- Runtime: Python `>=3.11`.
|
|
7
|
+
|
|
8
|
+
## Code Style
|
|
9
|
+
- Keep modules small and composable.
|
|
10
|
+
- Centralize defaults/env parsing in `llmcalc/config.py`.
|
|
11
|
+
- Prefer pure helpers for deterministic logic (normalization, math, parsing).
|
|
12
|
+
- Keep public API typed and stable.
|
|
13
|
+
|
|
14
|
+
## Local Commands
|
|
15
|
+
- Install: `pip install -e '.[dev]'`
|
|
16
|
+
- Lint: `python -m ruff check .`
|
|
17
|
+
- Type check: `python -m mypy llmcalc`
|
|
18
|
+
- Test: `python -m pytest`
|
|
19
|
+
- Build: `python -m build --no-isolation`
|
|
20
|
+
- Dist check: `python -m twine check dist/*`
|
|
21
|
+
|
|
22
|
+
## CLI Expectations
|
|
23
|
+
- Root command: `llmcalc`
|
|
24
|
+
- Version flags: `llmcalc --version` and `llmcalc -v`
|
|
25
|
+
- Cache defaults:
|
|
26
|
+
- Default TTL: `43200` seconds
|
|
27
|
+
- Env override: `LLMCALC_CACHE_TIMEOUT`
|
|
28
|
+
|
|
29
|
+
## Release Checklist
|
|
30
|
+
1. Run lint, mypy, and tests.
|
|
31
|
+
2. Build distributions.
|
|
32
|
+
3. Run `twine check`.
|
|
33
|
+
4. Update `CHANGELOG.md`.
|
|
34
|
+
5. Publish to PyPI.
|
llmcalc-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 llmcalc contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
llmcalc-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: llmcalc
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Calculate LLM token costs from llmlite pricing data
|
|
5
|
+
Project-URL: Homepage, https://github.com/onlyoneaman/llmcalc
|
|
6
|
+
Project-URL: Repository, https://github.com/onlyoneaman/llmcalc
|
|
7
|
+
Project-URL: Issues, https://github.com/onlyoneaman/llmcalc/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/onlyoneaman/llmcalc/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Aman
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: cost,llm,pricing,tokens
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: httpx<1,>=0.27
|
|
20
|
+
Requires-Dist: platformdirs<5,>=4.2
|
|
21
|
+
Requires-Dist: pydantic<3,>=2.8
|
|
22
|
+
Requires-Dist: typer<1,>=0.12
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: mypy<2,>=1.10; extra == 'dev'
|
|
25
|
+
Requires-Dist: pytest-asyncio<1,>=0.23; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest<9,>=8.2; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff<1,>=0.6; extra == 'dev'
|
|
28
|
+
Requires-Dist: twine<7,>=5; extra == 'dev'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# llmcalc
|
|
32
|
+
|
|
33
|
+
`llmcalc` is a Python package to calculate LLM token costs from `llmlite` pricing data.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install llmcalc
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Python usage
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
import asyncio
|
|
45
|
+
from llmcalc import calculate_token_cost
|
|
46
|
+
|
|
47
|
+
result = asyncio.run(calculate_token_cost("gpt-4o-mini", 1200, 800))
|
|
48
|
+
if result:
|
|
49
|
+
print(result.total_cost)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## CLI usage
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
llmcalc quote --model gpt-4o-mini --input 1200 --output 800
|
|
56
|
+
llmcalc model --model gpt-4o-mini --json
|
|
57
|
+
llmcalc cache clear
|
|
58
|
+
llmcalc --version
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Defaults
|
|
62
|
+
|
|
63
|
+
- Default cache TTL is `43200` seconds (12 hours).
|
|
64
|
+
- Override cache TTL with `LLMCALC_CACHE_TIMEOUT`.
|
|
65
|
+
- Override pricing source with `LLMCALC_PRICING_URL`.
|
|
66
|
+
- Set fallback currency label with `LLMCALC_CURRENCY` (used only when upstream omits currency).
|
|
67
|
+
|
|
68
|
+
## Release Validation
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
python -m build --no-isolation
|
|
72
|
+
python -m twine check dist/*
|
|
73
|
+
```
|
llmcalc-0.1.0/README.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# llmcalc
|
|
2
|
+
|
|
3
|
+
`llmcalc` is a Python package to calculate LLM token costs from `llmlite` pricing data.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install llmcalc
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Python usage
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
import asyncio
|
|
15
|
+
from llmcalc import calculate_token_cost
|
|
16
|
+
|
|
17
|
+
result = asyncio.run(calculate_token_cost("gpt-4o-mini", 1200, 800))
|
|
18
|
+
if result:
|
|
19
|
+
print(result.total_cost)
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## CLI usage
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
llmcalc quote --model gpt-4o-mini --input 1200 --output 800
|
|
26
|
+
llmcalc model --model gpt-4o-mini --json
|
|
27
|
+
llmcalc cache clear
|
|
28
|
+
llmcalc --version
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Defaults
|
|
32
|
+
|
|
33
|
+
- Default cache TTL is `43200` seconds (12 hours).
|
|
34
|
+
- Override cache TTL with `LLMCALC_CACHE_TIMEOUT`.
|
|
35
|
+
- Override pricing source with `LLMCALC_PRICING_URL`.
|
|
36
|
+
- Set fallback currency label with `LLMCALC_CURRENCY` (used only when upstream omits currency).
|
|
37
|
+
|
|
38
|
+
## Release Validation
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
python -m build --no-isolation
|
|
42
|
+
python -m twine check dist/*
|
|
43
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""llmcalc public package exports."""
|
|
2
|
+
|
|
3
|
+
from llmcalc.api import calculate_token_cost, calculate_usage_cost, clear_cache, get_model_costs
|
|
4
|
+
from llmcalc.config import get_package_version
|
|
5
|
+
from llmcalc.models import CostBreakdown, ModelPricing
|
|
6
|
+
|
|
7
|
+
__version__ = get_package_version()
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"__version__",
|
|
11
|
+
"CostBreakdown",
|
|
12
|
+
"ModelPricing",
|
|
13
|
+
"calculate_token_cost",
|
|
14
|
+
"calculate_usage_cost",
|
|
15
|
+
"clear_cache",
|
|
16
|
+
"get_model_costs",
|
|
17
|
+
]
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Public API for llmcalc."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from decimal import ROUND_HALF_UP, Decimal
|
|
6
|
+
from typing import Any, Protocol, runtime_checkable
|
|
7
|
+
|
|
8
|
+
from llmcalc.cache import clear_cache as clear_cache_file
|
|
9
|
+
from llmcalc.config import DEFAULT_CACHE_TIMEOUT_SECONDS, resolve_cache_timeout
|
|
10
|
+
from llmcalc.models import CostBreakdown, ModelPricing
|
|
11
|
+
from llmcalc.normalize import resolve_model_key
|
|
12
|
+
from llmcalc.pricing_client import get_pricing_table
|
|
13
|
+
|
|
14
|
+
DEFAULT_ROUNDING_PLACES = 6
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@runtime_checkable
|
|
18
|
+
class UsageLike(Protocol):
|
|
19
|
+
"""Protocol for usage objects accepted by `calculate_usage_cost`."""
|
|
20
|
+
|
|
21
|
+
prompt_tokens: int
|
|
22
|
+
completion_tokens: int
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _round_money(amount: Decimal, places: int = DEFAULT_ROUNDING_PLACES) -> Decimal:
|
|
26
|
+
quant = Decimal("1").scaleb(-places)
|
|
27
|
+
return amount.quantize(quant, rounding=ROUND_HALF_UP)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _validate_non_negative(value: int, field_name: str) -> None:
|
|
31
|
+
if value < 0:
|
|
32
|
+
raise ValueError(f"{field_name} must be non-negative")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def get_model_costs(
|
|
36
|
+
model: str,
|
|
37
|
+
cache_timeout: int | None = None,
|
|
38
|
+
) -> ModelPricing | None:
|
|
39
|
+
"""Return per-token pricing for a model, or `None` if not found."""
|
|
40
|
+
table = await get_pricing_table(cache_timeout=resolve_cache_timeout(cache_timeout))
|
|
41
|
+
resolved = resolve_model_key(model, table.keys())
|
|
42
|
+
if resolved is None:
|
|
43
|
+
return None
|
|
44
|
+
return table[resolved]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def calculate_token_cost(
|
|
48
|
+
model: str,
|
|
49
|
+
input_tokens: int,
|
|
50
|
+
output_tokens: int,
|
|
51
|
+
cache_timeout: int | None = None,
|
|
52
|
+
) -> CostBreakdown | None:
|
|
53
|
+
"""Calculate model usage cost from token counts, or return `None` if model is unavailable."""
|
|
54
|
+
_validate_non_negative(input_tokens, "input_tokens")
|
|
55
|
+
_validate_non_negative(output_tokens, "output_tokens")
|
|
56
|
+
|
|
57
|
+
model_costs = await get_model_costs(model, cache_timeout=cache_timeout)
|
|
58
|
+
if model_costs is None:
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
input_cost = _round_money(Decimal(input_tokens) * model_costs.input_cost_per_token)
|
|
62
|
+
output_cost = _round_money(Decimal(output_tokens) * model_costs.output_cost_per_token)
|
|
63
|
+
total_cost = _round_money(input_cost + output_cost)
|
|
64
|
+
|
|
65
|
+
return CostBreakdown(
|
|
66
|
+
input_cost=input_cost,
|
|
67
|
+
output_cost=output_cost,
|
|
68
|
+
total_cost=total_cost,
|
|
69
|
+
currency=model_costs.currency,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _value_from_mapping(usage: dict[str, Any], key_options: tuple[str, ...]) -> int | None:
|
|
74
|
+
for key in key_options:
|
|
75
|
+
value = usage.get(key)
|
|
76
|
+
if isinstance(value, int):
|
|
77
|
+
return value
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _get_usage_tokens(usage: Any) -> tuple[int, int]:
|
|
82
|
+
if isinstance(usage, dict):
|
|
83
|
+
input_tokens = _value_from_mapping(usage, ("input_tokens", "prompt_tokens"))
|
|
84
|
+
output_tokens = _value_from_mapping(usage, ("output_tokens", "completion_tokens"))
|
|
85
|
+
else:
|
|
86
|
+
input_tokens = getattr(usage, "input_tokens", None)
|
|
87
|
+
if not isinstance(input_tokens, int):
|
|
88
|
+
input_tokens = getattr(usage, "prompt_tokens", None)
|
|
89
|
+
|
|
90
|
+
output_tokens = getattr(usage, "output_tokens", None)
|
|
91
|
+
if not isinstance(output_tokens, int):
|
|
92
|
+
output_tokens = getattr(usage, "completion_tokens", None)
|
|
93
|
+
|
|
94
|
+
if not isinstance(input_tokens, int) or not isinstance(output_tokens, int):
|
|
95
|
+
raise ValueError("usage must provide input/prompt tokens and output/completion tokens")
|
|
96
|
+
|
|
97
|
+
return input_tokens, output_tokens
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
async def calculate_usage_cost(
|
|
101
|
+
model: str,
|
|
102
|
+
usage: UsageLike | dict[str, Any],
|
|
103
|
+
cache_timeout: int | None = None,
|
|
104
|
+
) -> CostBreakdown | None:
|
|
105
|
+
"""Calculate cost from an object that includes usage token fields."""
|
|
106
|
+
input_tokens, output_tokens = _get_usage_tokens(usage)
|
|
107
|
+
return await calculate_token_cost(
|
|
108
|
+
model=model,
|
|
109
|
+
input_tokens=input_tokens,
|
|
110
|
+
output_tokens=output_tokens,
|
|
111
|
+
cache_timeout=cache_timeout,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def clear_cache() -> None:
|
|
116
|
+
"""Clear local pricing cache file."""
|
|
117
|
+
clear_cache_file()
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
DEFAULT_CACHE_TIMEOUT = DEFAULT_CACHE_TIMEOUT_SECONDS
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Filesystem cache helpers for pricing payloads."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from platformdirs import user_cache_dir
|
|
11
|
+
|
|
12
|
+
CACHE_DIR_NAME = "llmcalc"
|
|
13
|
+
CACHE_FILE_NAME = "pricing_cache.json"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def cache_file_path() -> Path:
|
|
17
|
+
base_dir = Path(user_cache_dir(appname=CACHE_DIR_NAME, appauthor=CACHE_DIR_NAME))
|
|
18
|
+
return base_dir / CACHE_FILE_NAME
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def load_cached_pricing(max_age_seconds: int) -> dict[str, Any] | None:
|
|
22
|
+
path = cache_file_path()
|
|
23
|
+
if not path.exists():
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
28
|
+
except (json.JSONDecodeError, OSError):
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
fetched_at = payload.get("fetched_at")
|
|
32
|
+
data = payload.get("data")
|
|
33
|
+
if not isinstance(fetched_at, (int, float)) or not isinstance(data, dict):
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
if time.time() - fetched_at > max_age_seconds:
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
return data
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def save_cached_pricing(data: dict[str, Any]) -> None:
|
|
43
|
+
path = cache_file_path()
|
|
44
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
|
|
46
|
+
payload = {
|
|
47
|
+
"fetched_at": time.time(),
|
|
48
|
+
"data": data,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
path.write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def clear_cache() -> None:
|
|
55
|
+
path = cache_file_path()
|
|
56
|
+
if path.exists():
|
|
57
|
+
path.unlink()
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""CLI for llmcalc."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
from typing import Annotated, Any
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
|
|
11
|
+
from llmcalc import __version__
|
|
12
|
+
from llmcalc.api import (
|
|
13
|
+
calculate_token_cost,
|
|
14
|
+
clear_cache,
|
|
15
|
+
get_model_costs,
|
|
16
|
+
)
|
|
17
|
+
from llmcalc.config import DEFAULT_CACHE_TIMEOUT_SECONDS
|
|
18
|
+
|
|
19
|
+
DEFAULT_CACHE_TIMEOUT_HELP = (
|
|
20
|
+
f"Cache TTL in seconds (default: {DEFAULT_CACHE_TIMEOUT_SECONDS} or LLMCALC_CACHE_TIMEOUT)"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
app = typer.Typer(help="Calculate LLM token pricing from llmlite data.", no_args_is_help=True)
|
|
24
|
+
cache_app = typer.Typer(help="Cache commands.")
|
|
25
|
+
app.add_typer(cache_app, name="cache")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _model_option() -> Any:
|
|
29
|
+
return typer.Option(..., "--model", help="Model id")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _cache_timeout_option() -> Any:
|
|
33
|
+
return typer.Option(None, "--cache-timeout", help=DEFAULT_CACHE_TIMEOUT_HELP)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _json_option() -> Any:
|
|
37
|
+
return typer.Option(False, "--json", help="Emit JSON output")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _emit(data: dict[str, object], as_json: bool) -> None:
|
|
41
|
+
if as_json:
|
|
42
|
+
typer.echo(json.dumps(data, default=str))
|
|
43
|
+
return
|
|
44
|
+
|
|
45
|
+
for key, value in data.items():
|
|
46
|
+
typer.echo(f"{key}: {value}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _version_callback(value: bool) -> None:
|
|
50
|
+
if value:
|
|
51
|
+
typer.echo(f"llmcalc {__version__}")
|
|
52
|
+
raise typer.Exit()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@app.callback()
|
|
56
|
+
def app_main(
|
|
57
|
+
version: Annotated[
|
|
58
|
+
bool,
|
|
59
|
+
typer.Option(
|
|
60
|
+
"--version",
|
|
61
|
+
"-v",
|
|
62
|
+
help="Show package version and exit.",
|
|
63
|
+
callback=_version_callback,
|
|
64
|
+
is_eager=True,
|
|
65
|
+
),
|
|
66
|
+
] = False,
|
|
67
|
+
) -> None:
|
|
68
|
+
_ = version
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@app.command()
|
|
72
|
+
def quote(
|
|
73
|
+
model: str = _model_option(),
|
|
74
|
+
input_tokens: int = typer.Option(..., "--input", help="Input token count", min=0),
|
|
75
|
+
output_tokens: int = typer.Option(..., "--output", help="Output token count", min=0),
|
|
76
|
+
cache_timeout: int | None = _cache_timeout_option(),
|
|
77
|
+
as_json: bool = _json_option(),
|
|
78
|
+
) -> None:
|
|
79
|
+
"""Quote input/output/total cost for a model."""
|
|
80
|
+
result = asyncio.run(
|
|
81
|
+
calculate_token_cost(
|
|
82
|
+
model=model,
|
|
83
|
+
input_tokens=input_tokens,
|
|
84
|
+
output_tokens=output_tokens,
|
|
85
|
+
cache_timeout=cache_timeout,
|
|
86
|
+
)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
if result is None:
|
|
90
|
+
typer.echo(f"Model not found: {model}", err=True)
|
|
91
|
+
raise typer.Exit(code=1)
|
|
92
|
+
|
|
93
|
+
payload = {
|
|
94
|
+
"model": model,
|
|
95
|
+
"input_cost": result.input_cost,
|
|
96
|
+
"output_cost": result.output_cost,
|
|
97
|
+
"total_cost": result.total_cost,
|
|
98
|
+
"currency": result.currency,
|
|
99
|
+
}
|
|
100
|
+
_emit(payload, as_json)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@app.command("model")
|
|
104
|
+
def model_cmd(
|
|
105
|
+
model: str = _model_option(),
|
|
106
|
+
cache_timeout: int | None = _cache_timeout_option(),
|
|
107
|
+
as_json: bool = _json_option(),
|
|
108
|
+
) -> None:
|
|
109
|
+
"""Show per-token pricing for a model."""
|
|
110
|
+
result = asyncio.run(get_model_costs(model=model, cache_timeout=cache_timeout))
|
|
111
|
+
|
|
112
|
+
if result is None:
|
|
113
|
+
typer.echo(f"Model not found: {model}", err=True)
|
|
114
|
+
raise typer.Exit(code=1)
|
|
115
|
+
|
|
116
|
+
payload = {
|
|
117
|
+
"model": result.model,
|
|
118
|
+
"input_cost_per_token": result.input_cost_per_token,
|
|
119
|
+
"output_cost_per_token": result.output_cost_per_token,
|
|
120
|
+
"currency": result.currency,
|
|
121
|
+
"provider": result.provider,
|
|
122
|
+
"last_updated": result.last_updated,
|
|
123
|
+
}
|
|
124
|
+
_emit(payload, as_json)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@cache_app.command("clear")
|
|
128
|
+
def cache_clear() -> None:
|
|
129
|
+
"""Clear local pricing cache."""
|
|
130
|
+
clear_cache()
|
|
131
|
+
typer.echo("Cache cleared")
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Configuration defaults and environment-driven settings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from importlib import metadata
|
|
7
|
+
|
|
8
|
+
APP_NAME = "llmcalc"
|
|
9
|
+
FALLBACK_VERSION = "0.1.0"
|
|
10
|
+
DEFAULT_CACHE_TIMEOUT_SECONDS = 43200
|
|
11
|
+
DEFAULT_CURRENCY = "USD"
|
|
12
|
+
DEFAULT_PRICING_URL = (
|
|
13
|
+
"https://raw.githubusercontent.com/llmlite/llmlite/main/model_prices_and_context_window.json"
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_package_version() -> str:
|
|
18
|
+
"""Return installed package version with a source fallback."""
|
|
19
|
+
try:
|
|
20
|
+
return metadata.version(APP_NAME)
|
|
21
|
+
except metadata.PackageNotFoundError:
|
|
22
|
+
return FALLBACK_VERSION
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_user_agent() -> str:
|
|
26
|
+
"""Return HTTP user agent for outbound pricing requests."""
|
|
27
|
+
return f"{APP_NAME}/{get_package_version()}"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_pricing_url(explicit_url: str | None = None) -> str:
|
|
31
|
+
"""Resolve pricing source URL from explicit arg, env var, then default."""
|
|
32
|
+
return explicit_url or os.getenv("LLMCALC_PRICING_URL") or DEFAULT_PRICING_URL
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_default_currency() -> str:
|
|
36
|
+
"""Resolve fallback currency label from env var or default."""
|
|
37
|
+
raw = os.getenv("LLMCALC_CURRENCY", "").strip().upper()
|
|
38
|
+
return raw or DEFAULT_CURRENCY
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def resolve_cache_timeout(cache_timeout: int | None = None) -> int:
|
|
42
|
+
"""Resolve cache timeout from explicit value, env var, or default."""
|
|
43
|
+
if cache_timeout is not None:
|
|
44
|
+
if cache_timeout <= 0:
|
|
45
|
+
raise ValueError("cache_timeout must be positive")
|
|
46
|
+
return cache_timeout
|
|
47
|
+
|
|
48
|
+
env_value = os.getenv("LLMCALC_CACHE_TIMEOUT")
|
|
49
|
+
if env_value is None:
|
|
50
|
+
return DEFAULT_CACHE_TIMEOUT_SECONDS
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
parsed = int(env_value)
|
|
54
|
+
except ValueError:
|
|
55
|
+
return DEFAULT_CACHE_TIMEOUT_SECONDS
|
|
56
|
+
|
|
57
|
+
if parsed <= 0:
|
|
58
|
+
return DEFAULT_CACHE_TIMEOUT_SECONDS
|
|
59
|
+
|
|
60
|
+
return parsed
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Typed exceptions for llmcalc."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class PricingError(Exception):
|
|
5
|
+
"""Base pricing error type."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class PricingFetchError(PricingError):
|
|
9
|
+
"""Raised when pricing data cannot be fetched and no cache is usable."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class PricingSchemaError(PricingError):
|
|
13
|
+
"""Raised when upstream pricing data is invalid."""
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Pydantic models for pricing and cost breakdown data."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from decimal import Decimal
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class CostBreakdown(BaseModel):
|
|
12
|
+
"""Cost result for a token usage calculation."""
|
|
13
|
+
|
|
14
|
+
input_cost: Decimal = Field(ge=Decimal("0"))
|
|
15
|
+
output_cost: Decimal = Field(ge=Decimal("0"))
|
|
16
|
+
total_cost: Decimal = Field(ge=Decimal("0"))
|
|
17
|
+
currency: str = "USD"
|
|
18
|
+
|
|
19
|
+
model_config = ConfigDict(frozen=True)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ModelPricing(BaseModel):
|
|
23
|
+
"""Normalized token pricing for one model."""
|
|
24
|
+
|
|
25
|
+
model: str
|
|
26
|
+
input_cost_per_token: Decimal = Field(ge=Decimal("0"))
|
|
27
|
+
output_cost_per_token: Decimal = Field(ge=Decimal("0"))
|
|
28
|
+
provider: str | None = None
|
|
29
|
+
currency: str = "USD"
|
|
30
|
+
last_updated: str | None = None
|
|
31
|
+
|
|
32
|
+
model_config = ConfigDict(frozen=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class RawModelPricing(BaseModel):
|
|
36
|
+
"""Flexible upstream schema model from llmlite pricing JSON."""
|
|
37
|
+
|
|
38
|
+
input_cost_per_token: Decimal | None = None
|
|
39
|
+
output_cost_per_token: Decimal | None = None
|
|
40
|
+
|
|
41
|
+
# Common aliases used by some pricing datasets.
|
|
42
|
+
prompt_cost_per_token: Decimal | None = None
|
|
43
|
+
completion_cost_per_token: Decimal | None = None
|
|
44
|
+
input_cost_per_million_tokens: Decimal | None = None
|
|
45
|
+
output_cost_per_million_tokens: Decimal | None = None
|
|
46
|
+
|
|
47
|
+
provider: str | None = None
|
|
48
|
+
currency: str | None = None
|
|
49
|
+
last_updated: str | None = None
|
|
50
|
+
|
|
51
|
+
@field_validator("input_cost_per_token", "output_cost_per_token", mode="before")
|
|
52
|
+
@classmethod
|
|
53
|
+
def normalize_numeric_str(cls, value: Any) -> Any:
|
|
54
|
+
if isinstance(value, str):
|
|
55
|
+
return value.strip()
|
|
56
|
+
return value
|
|
57
|
+
|
|
58
|
+
@field_validator("prompt_cost_per_token", "completion_cost_per_token", mode="before")
|
|
59
|
+
@classmethod
|
|
60
|
+
def normalize_alias_numeric_str(cls, value: Any) -> Any:
|
|
61
|
+
if isinstance(value, str):
|
|
62
|
+
return value.strip()
|
|
63
|
+
return value
|
|
64
|
+
|
|
65
|
+
@field_validator(
|
|
66
|
+
"input_cost_per_million_tokens",
|
|
67
|
+
"output_cost_per_million_tokens",
|
|
68
|
+
mode="before",
|
|
69
|
+
)
|
|
70
|
+
@classmethod
|
|
71
|
+
def normalize_million_numeric_str(cls, value: Any) -> Any:
|
|
72
|
+
if isinstance(value, str):
|
|
73
|
+
return value.strip()
|
|
74
|
+
return value
|
|
75
|
+
|
|
76
|
+
def to_model_pricing(self, model: str, default_currency: str = "USD") -> ModelPricing:
|
|
77
|
+
input_cost = self.input_cost_per_token
|
|
78
|
+
output_cost = self.output_cost_per_token
|
|
79
|
+
|
|
80
|
+
if input_cost is None:
|
|
81
|
+
input_cost = self.prompt_cost_per_token
|
|
82
|
+
if output_cost is None:
|
|
83
|
+
output_cost = self.completion_cost_per_token
|
|
84
|
+
if input_cost is None and self.input_cost_per_million_tokens is not None:
|
|
85
|
+
input_cost = self.input_cost_per_million_tokens / Decimal("1000000")
|
|
86
|
+
if output_cost is None and self.output_cost_per_million_tokens is not None:
|
|
87
|
+
output_cost = self.output_cost_per_million_tokens / Decimal("1000000")
|
|
88
|
+
|
|
89
|
+
if input_cost is None or output_cost is None:
|
|
90
|
+
raise ValueError("Missing input/output token pricing fields")
|
|
91
|
+
|
|
92
|
+
return ModelPricing(
|
|
93
|
+
model=model,
|
|
94
|
+
input_cost_per_token=input_cost,
|
|
95
|
+
output_cost_per_token=output_cost,
|
|
96
|
+
provider=self.provider,
|
|
97
|
+
currency=self.currency or default_currency,
|
|
98
|
+
last_updated=self.last_updated,
|
|
99
|
+
)
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Model name normalization and lookup helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
|
|
7
|
+
# Keep aliases small and explicit; extend as model IDs evolve.
|
|
8
|
+
ALIASES: dict[str, str] = {
|
|
9
|
+
"gpt-4o-mini-latest": "gpt-4o-mini",
|
|
10
|
+
"gpt-4.1-mini-latest": "gpt-4.1-mini",
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
PROVIDER_PREFIXES: set[str] = {
|
|
14
|
+
"openai",
|
|
15
|
+
"anthropic",
|
|
16
|
+
"google",
|
|
17
|
+
"xai",
|
|
18
|
+
"meta",
|
|
19
|
+
"mistral",
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def normalize_model_name(model: str) -> str:
|
|
24
|
+
"""Normalize an input model identifier to a canonical lookup key."""
|
|
25
|
+
normalized = model.strip().lower()
|
|
26
|
+
if not normalized:
|
|
27
|
+
raise ValueError("model must not be empty")
|
|
28
|
+
|
|
29
|
+
if ":" in normalized:
|
|
30
|
+
provider, suffix = normalized.split(":", 1)
|
|
31
|
+
if provider and suffix:
|
|
32
|
+
normalized = suffix
|
|
33
|
+
|
|
34
|
+
if "/" in normalized:
|
|
35
|
+
provider, suffix = normalized.split("/", 1)
|
|
36
|
+
if provider in PROVIDER_PREFIXES and suffix:
|
|
37
|
+
normalized = suffix
|
|
38
|
+
|
|
39
|
+
return ALIASES.get(normalized, normalized)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def resolve_model_key(model: str, available_keys: Iterable[str]) -> str | None:
|
|
43
|
+
"""Resolve a requested model id against available pricing table keys."""
|
|
44
|
+
key_map: dict[str, str] = {}
|
|
45
|
+
for key in available_keys:
|
|
46
|
+
key_map[key.lower()] = key
|
|
47
|
+
key_map.setdefault(normalize_model_name(key), key)
|
|
48
|
+
|
|
49
|
+
candidates: list[str] = []
|
|
50
|
+
raw = model.strip().lower()
|
|
51
|
+
if raw:
|
|
52
|
+
candidates.append(raw)
|
|
53
|
+
candidates.append(normalize_model_name(model))
|
|
54
|
+
|
|
55
|
+
for candidate in candidates:
|
|
56
|
+
direct = key_map.get(candidate)
|
|
57
|
+
if direct is not None:
|
|
58
|
+
return direct
|
|
59
|
+
alias = ALIASES.get(candidate)
|
|
60
|
+
if alias is not None and alias in key_map:
|
|
61
|
+
return key_map[alias]
|
|
62
|
+
|
|
63
|
+
return None
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Pricing data fetch and normalization against llmlite pricing source."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
|
|
10
|
+
from llmcalc.cache import load_cached_pricing, save_cached_pricing
|
|
11
|
+
from llmcalc.config import (
|
|
12
|
+
DEFAULT_CACHE_TIMEOUT_SECONDS,
|
|
13
|
+
get_default_currency,
|
|
14
|
+
get_pricing_url,
|
|
15
|
+
get_user_agent,
|
|
16
|
+
)
|
|
17
|
+
from llmcalc.errors import PricingFetchError, PricingSchemaError
|
|
18
|
+
from llmcalc.models import ModelPricing, RawModelPricing
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def fetch_pricing_payload(pricing_url: str | None = None) -> dict[str, Any]:
|
|
22
|
+
"""Fetch pricing payload from configured remote source."""
|
|
23
|
+
source = get_pricing_url(pricing_url)
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
async with httpx.AsyncClient(
|
|
27
|
+
timeout=10,
|
|
28
|
+
headers={"User-Agent": get_user_agent()},
|
|
29
|
+
) as client:
|
|
30
|
+
response = await client.get(source)
|
|
31
|
+
response.raise_for_status()
|
|
32
|
+
payload = response.json()
|
|
33
|
+
except (httpx.HTTPError, ValueError) as exc:
|
|
34
|
+
raise PricingFetchError(f"failed to fetch pricing data from {source}") from exc
|
|
35
|
+
|
|
36
|
+
if not isinstance(payload, dict):
|
|
37
|
+
raise PricingSchemaError("pricing payload must be a JSON object")
|
|
38
|
+
|
|
39
|
+
return payload
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_pricing_payload(payload: Mapping[str, Any]) -> dict[str, ModelPricing]:
|
|
43
|
+
"""Parse raw pricing JSON into normalized `ModelPricing` objects."""
|
|
44
|
+
raw_table: Mapping[str, Any]
|
|
45
|
+
default_currency = get_default_currency()
|
|
46
|
+
|
|
47
|
+
wrapped_data = payload.get("data")
|
|
48
|
+
raw_table = wrapped_data if isinstance(wrapped_data, Mapping) else payload
|
|
49
|
+
|
|
50
|
+
parsed: dict[str, ModelPricing] = {}
|
|
51
|
+
for model_name, model_data in raw_table.items():
|
|
52
|
+
if not isinstance(model_name, str) or not isinstance(model_data, Mapping):
|
|
53
|
+
continue
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
raw_model = RawModelPricing.model_validate(model_data)
|
|
57
|
+
parsed[model_name] = raw_model.to_model_pricing(
|
|
58
|
+
model_name,
|
|
59
|
+
default_currency=default_currency,
|
|
60
|
+
)
|
|
61
|
+
except Exception:
|
|
62
|
+
# Some payloads contain non-pricing metadata entries; skip them.
|
|
63
|
+
continue
|
|
64
|
+
|
|
65
|
+
if not parsed:
|
|
66
|
+
raise PricingSchemaError("no valid model pricing entries found")
|
|
67
|
+
|
|
68
|
+
return parsed
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
async def get_pricing_table(
|
|
72
|
+
cache_timeout: int = DEFAULT_CACHE_TIMEOUT_SECONDS,
|
|
73
|
+
pricing_url: str | None = None,
|
|
74
|
+
) -> dict[str, ModelPricing]:
|
|
75
|
+
"""Get model pricing table using cache-first strategy with remote refresh fallback."""
|
|
76
|
+
cached_data = load_cached_pricing(cache_timeout)
|
|
77
|
+
if cached_data is not None:
|
|
78
|
+
try:
|
|
79
|
+
return parse_pricing_payload(cached_data)
|
|
80
|
+
except PricingSchemaError:
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
try:
|
|
84
|
+
fetched_payload = await fetch_pricing_payload(pricing_url=pricing_url)
|
|
85
|
+
except PricingFetchError:
|
|
86
|
+
if cached_data is not None:
|
|
87
|
+
return parse_pricing_payload(cached_data)
|
|
88
|
+
raise
|
|
89
|
+
|
|
90
|
+
parsed = parse_pricing_payload(fetched_payload)
|
|
91
|
+
save_cached_pricing(fetched_payload)
|
|
92
|
+
return parsed
|
|
File without changes
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25.0"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "llmcalc"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Calculate LLM token costs from llmlite pricing data"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Aman", url = "https://github.com/onlyoneaman" }
|
|
14
|
+
]
|
|
15
|
+
keywords = ["llm", "pricing", "tokens", "cost"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Typing :: Typed",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"httpx>=0.27,<1",
|
|
25
|
+
"platformdirs>=4.2,<5",
|
|
26
|
+
"pydantic>=2.8,<3",
|
|
27
|
+
"typer>=0.12,<1",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/onlyoneaman/llmcalc"
|
|
32
|
+
Repository = "https://github.com/onlyoneaman/llmcalc"
|
|
33
|
+
Issues = "https://github.com/onlyoneaman/llmcalc/issues"
|
|
34
|
+
Changelog = "https://github.com/onlyoneaman/llmcalc/blob/main/CHANGELOG.md"
|
|
35
|
+
|
|
36
|
+
[project.scripts]
|
|
37
|
+
llmcalc = "llmcalc.cli:app"
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
dev = [
|
|
41
|
+
"pytest>=8.2,<9",
|
|
42
|
+
"pytest-asyncio>=0.23,<1",
|
|
43
|
+
"ruff>=0.6,<1",
|
|
44
|
+
"mypy>=1.10,<2",
|
|
45
|
+
"twine>=5,<7",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[tool.pytest.ini_options]
|
|
49
|
+
addopts = "-q"
|
|
50
|
+
asyncio_mode = "auto"
|
|
51
|
+
testpaths = ["tests"]
|
|
52
|
+
|
|
53
|
+
[tool.ruff]
|
|
54
|
+
line-length = 100
|
|
55
|
+
target-version = "py311"
|
|
56
|
+
|
|
57
|
+
[tool.ruff.lint]
|
|
58
|
+
select = ["E", "F", "I", "UP", "B", "SIM"]
|
|
59
|
+
|
|
60
|
+
[tool.mypy]
|
|
61
|
+
python_version = "3.11"
|
|
62
|
+
strict = true
|
|
63
|
+
warn_unused_configs = true
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from decimal import Decimal
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
import llmcalc.api as api
|
|
7
|
+
from llmcalc.models import ModelPricing
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def _fake_pricing_table(cache_timeout: int = 86400):
|
|
11
|
+
_ = cache_timeout
|
|
12
|
+
return {
|
|
13
|
+
"gpt-4o-mini": ModelPricing(
|
|
14
|
+
model="gpt-4o-mini",
|
|
15
|
+
input_cost_per_token=Decimal("0.000001"),
|
|
16
|
+
output_cost_per_token=Decimal("0.000002"),
|
|
17
|
+
currency="USD",
|
|
18
|
+
)
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.asyncio
|
|
23
|
+
async def test_get_model_costs_found(monkeypatch) -> None:
|
|
24
|
+
monkeypatch.setattr(api, "get_pricing_table", _fake_pricing_table)
|
|
25
|
+
result = await api.get_model_costs("openai:gpt-4o-mini")
|
|
26
|
+
assert result is not None
|
|
27
|
+
assert result.model == "gpt-4o-mini"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@pytest.mark.asyncio
|
|
31
|
+
async def test_calculate_token_cost(monkeypatch) -> None:
|
|
32
|
+
monkeypatch.setattr(api, "get_pricing_table", _fake_pricing_table)
|
|
33
|
+
result = await api.calculate_token_cost("gpt-4o-mini", input_tokens=1000, output_tokens=500)
|
|
34
|
+
|
|
35
|
+
assert result is not None
|
|
36
|
+
assert result.input_cost == Decimal("0.001000")
|
|
37
|
+
assert result.output_cost == Decimal("0.001000")
|
|
38
|
+
assert result.total_cost == Decimal("0.002000")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.asyncio
|
|
42
|
+
async def test_calculate_token_cost_unknown_model(monkeypatch) -> None:
|
|
43
|
+
monkeypatch.setattr(api, "get_pricing_table", _fake_pricing_table)
|
|
44
|
+
result = await api.calculate_token_cost("unknown", input_tokens=1, output_tokens=1)
|
|
45
|
+
assert result is None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.mark.asyncio
|
|
49
|
+
async def test_calculate_usage_cost_dict(monkeypatch) -> None:
|
|
50
|
+
monkeypatch.setattr(api, "get_pricing_table", _fake_pricing_table)
|
|
51
|
+
result = await api.calculate_usage_cost(
|
|
52
|
+
"gpt-4o-mini",
|
|
53
|
+
usage={"prompt_tokens": 1000, "completion_tokens": 1000},
|
|
54
|
+
)
|
|
55
|
+
assert result is not None
|
|
56
|
+
assert result.total_cost == Decimal("0.003000")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class UsageObj:
|
|
61
|
+
input_tokens: int
|
|
62
|
+
output_tokens: int
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@pytest.mark.asyncio
|
|
66
|
+
async def test_calculate_usage_cost_object(monkeypatch) -> None:
|
|
67
|
+
monkeypatch.setattr(api, "get_pricing_table", _fake_pricing_table)
|
|
68
|
+
result = await api.calculate_usage_cost("gpt-4o-mini", usage=UsageObj(100, 100))
|
|
69
|
+
assert result is not None
|
|
70
|
+
assert result.total_cost == Decimal("0.000300")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@pytest.mark.asyncio
|
|
74
|
+
async def test_calculate_token_cost_negative_tokens(monkeypatch) -> None:
|
|
75
|
+
monkeypatch.setattr(api, "get_pricing_table", _fake_pricing_table)
|
|
76
|
+
with pytest.raises(ValueError):
|
|
77
|
+
await api.calculate_token_cost("gpt-4o-mini", input_tokens=-1, output_tokens=1)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@pytest.mark.asyncio
|
|
81
|
+
async def test_get_model_costs_uses_env_cache_timeout(monkeypatch) -> None:
|
|
82
|
+
captured = {"cache_timeout": None}
|
|
83
|
+
|
|
84
|
+
async def _capture(cache_timeout: int = 0):
|
|
85
|
+
captured["cache_timeout"] = cache_timeout
|
|
86
|
+
return await _fake_pricing_table(cache_timeout=cache_timeout)
|
|
87
|
+
|
|
88
|
+
monkeypatch.setattr(api, "get_pricing_table", _capture)
|
|
89
|
+
monkeypatch.setenv("LLMCALC_CACHE_TIMEOUT", "1800")
|
|
90
|
+
|
|
91
|
+
await api.get_model_costs("gpt-4o-mini")
|
|
92
|
+
assert captured["cache_timeout"] == 1800
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@pytest.mark.asyncio
|
|
96
|
+
async def test_get_model_costs_invalid_env_cache_timeout_falls_back(monkeypatch) -> None:
|
|
97
|
+
captured = {"cache_timeout": None}
|
|
98
|
+
|
|
99
|
+
async def _capture(cache_timeout: int = 0):
|
|
100
|
+
captured["cache_timeout"] = cache_timeout
|
|
101
|
+
return await _fake_pricing_table(cache_timeout=cache_timeout)
|
|
102
|
+
|
|
103
|
+
monkeypatch.setattr(api, "get_pricing_table", _capture)
|
|
104
|
+
monkeypatch.setenv("LLMCALC_CACHE_TIMEOUT", "bad-value")
|
|
105
|
+
|
|
106
|
+
await api.get_model_costs("gpt-4o-mini")
|
|
107
|
+
assert captured["cache_timeout"] == api.DEFAULT_CACHE_TIMEOUT
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@pytest.mark.asyncio
|
|
111
|
+
async def test_get_model_costs_rejects_non_positive_cache_timeout(monkeypatch) -> None:
|
|
112
|
+
monkeypatch.setattr(api, "get_pricing_table", _fake_pricing_table)
|
|
113
|
+
with pytest.raises(ValueError):
|
|
114
|
+
await api.get_model_costs("gpt-4o-mini", cache_timeout=0)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_clear_cache_calls_cache(monkeypatch) -> None:
|
|
118
|
+
called = {"value": False}
|
|
119
|
+
|
|
120
|
+
def _clear() -> None:
|
|
121
|
+
called["value"] = True
|
|
122
|
+
|
|
123
|
+
monkeypatch.setattr(api, "clear_cache_file", _clear)
|
|
124
|
+
api.clear_cache()
|
|
125
|
+
assert called["value"] is True
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import time
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from llmcalc import cache
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _set_cache_path(monkeypatch, path: Path) -> None:
|
|
9
|
+
monkeypatch.setattr(cache, "cache_file_path", lambda: path)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_save_and_load_cache(tmp_path, monkeypatch) -> None:
|
|
13
|
+
file_path = tmp_path / "pricing_cache.json"
|
|
14
|
+
_set_cache_path(monkeypatch, file_path)
|
|
15
|
+
|
|
16
|
+
data = {"gpt-4o-mini": {"input_cost_per_token": 0.1, "output_cost_per_token": 0.2}}
|
|
17
|
+
cache.save_cached_pricing(data)
|
|
18
|
+
|
|
19
|
+
loaded = cache.load_cached_pricing(max_age_seconds=3600)
|
|
20
|
+
assert loaded == data
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_load_cache_expired(tmp_path, monkeypatch) -> None:
|
|
24
|
+
file_path = tmp_path / "pricing_cache.json"
|
|
25
|
+
_set_cache_path(monkeypatch, file_path)
|
|
26
|
+
|
|
27
|
+
payload = {
|
|
28
|
+
"fetched_at": time.time() - 10,
|
|
29
|
+
"data": {"gpt-4o-mini": {"input_cost_per_token": 1, "output_cost_per_token": 2}},
|
|
30
|
+
}
|
|
31
|
+
file_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
32
|
+
|
|
33
|
+
assert cache.load_cached_pricing(max_age_seconds=1) is None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_clear_cache(tmp_path, monkeypatch) -> None:
|
|
37
|
+
file_path = tmp_path / "pricing_cache.json"
|
|
38
|
+
_set_cache_path(monkeypatch, file_path)
|
|
39
|
+
|
|
40
|
+
file_path.write_text("{}", encoding="utf-8")
|
|
41
|
+
assert file_path.exists()
|
|
42
|
+
|
|
43
|
+
cache.clear_cache()
|
|
44
|
+
assert not file_path.exists()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from typer.testing import CliRunner
|
|
2
|
+
|
|
3
|
+
from llmcalc import __version__
|
|
4
|
+
from llmcalc.cli import app
|
|
5
|
+
|
|
6
|
+
runner = CliRunner()
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_cli_version_long_flag() -> None:
|
|
10
|
+
result = runner.invoke(app, ["--version"])
|
|
11
|
+
assert result.exit_code == 0
|
|
12
|
+
assert f"llmcalc {__version__}" in result.stdout
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_cli_version_short_flag() -> None:
|
|
16
|
+
result = runner.invoke(app, ["-v"])
|
|
17
|
+
assert result.exit_code == 0
|
|
18
|
+
assert f"llmcalc {__version__}" in result.stdout
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from decimal import Decimal
|
|
2
|
+
|
|
3
|
+
from llmcalc.models import RawModelPricing
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_raw_model_pricing_from_alias_fields() -> None:
|
|
7
|
+
raw = RawModelPricing(
|
|
8
|
+
prompt_cost_per_token="0.000001",
|
|
9
|
+
completion_cost_per_token="0.000002",
|
|
10
|
+
provider="openai",
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
model = raw.to_model_pricing("gpt-4o-mini")
|
|
14
|
+
assert model.input_cost_per_token == Decimal("0.000001")
|
|
15
|
+
assert model.output_cost_per_token == Decimal("0.000002")
|
|
16
|
+
assert model.provider == "openai"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_raw_model_pricing_from_per_million_fields() -> None:
|
|
20
|
+
raw = RawModelPricing(
|
|
21
|
+
input_cost_per_million_tokens="2",
|
|
22
|
+
output_cost_per_million_tokens="4",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
model = raw.to_model_pricing("gpt-4o-mini")
|
|
26
|
+
assert model.input_cost_per_token == Decimal("0.000002")
|
|
27
|
+
assert model.output_cost_per_token == Decimal("0.000004")
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from llmcalc.normalize import normalize_model_name, resolve_model_key
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_normalize_model_name_prefixes() -> None:
|
|
5
|
+
assert normalize_model_name(" openai:gpt-4o-mini ") == "gpt-4o-mini"
|
|
6
|
+
assert normalize_model_name("openai/gpt-4.1-mini") == "gpt-4.1-mini"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_normalize_model_name_alias() -> None:
|
|
10
|
+
assert normalize_model_name("gpt-4o-mini-latest") == "gpt-4o-mini"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_resolve_model_key_case_insensitive_and_prefixed() -> None:
|
|
14
|
+
keys = ["GPT-4O-MINI", "claude-3-5-sonnet"]
|
|
15
|
+
assert resolve_model_key("openai:gpt-4o-mini", keys) == "GPT-4O-MINI"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_resolve_model_key_missing() -> None:
|
|
19
|
+
keys = ["gpt-4o-mini"]
|
|
20
|
+
assert resolve_model_key("unknown-model", keys) is None
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
from decimal import Decimal
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from llmcalc.errors import PricingFetchError, PricingSchemaError
|
|
6
|
+
from llmcalc.pricing_client import get_pricing_table, parse_pricing_payload
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_parse_pricing_payload_direct_table() -> None:
|
|
10
|
+
payload = {
|
|
11
|
+
"gpt-4o-mini": {
|
|
12
|
+
"input_cost_per_token": "0.000001",
|
|
13
|
+
"output_cost_per_token": "0.000002",
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
table = parse_pricing_payload(payload)
|
|
18
|
+
assert table["gpt-4o-mini"].input_cost_per_token == Decimal("0.000001")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_parse_pricing_payload_wrapped_table() -> None:
|
|
22
|
+
payload = {
|
|
23
|
+
"data": {
|
|
24
|
+
"gpt-4o-mini": {
|
|
25
|
+
"input_cost_per_million_tokens": "2",
|
|
26
|
+
"output_cost_per_million_tokens": "4",
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
table = parse_pricing_payload(payload)
|
|
32
|
+
assert table["gpt-4o-mini"].output_cost_per_token == Decimal("0.000004")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_parse_pricing_payload_raises_for_invalid() -> None:
|
|
36
|
+
with pytest.raises(PricingSchemaError):
|
|
37
|
+
parse_pricing_payload({"meta": {"foo": "bar"}})
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_parse_pricing_payload_uses_env_currency_fallback(monkeypatch) -> None:
|
|
41
|
+
monkeypatch.setenv("LLMCALC_CURRENCY", "inr")
|
|
42
|
+
payload = {
|
|
43
|
+
"gpt-4o-mini": {
|
|
44
|
+
"input_cost_per_token": "0.000001",
|
|
45
|
+
"output_cost_per_token": "0.000002",
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
table = parse_pricing_payload(payload)
|
|
50
|
+
assert table["gpt-4o-mini"].currency == "INR"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_parse_pricing_payload_prefers_payload_currency_over_env(monkeypatch) -> None:
|
|
54
|
+
monkeypatch.setenv("LLMCALC_CURRENCY", "inr")
|
|
55
|
+
payload = {
|
|
56
|
+
"gpt-4o-mini": {
|
|
57
|
+
"input_cost_per_token": "0.000001",
|
|
58
|
+
"output_cost_per_token": "0.000002",
|
|
59
|
+
"currency": "USD",
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
table = parse_pricing_payload(payload)
|
|
64
|
+
assert table["gpt-4o-mini"].currency == "USD"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@pytest.mark.asyncio
|
|
68
|
+
async def test_get_pricing_table_uses_cache(monkeypatch) -> None:
|
|
69
|
+
cached = {
|
|
70
|
+
"gpt-4o-mini": {
|
|
71
|
+
"input_cost_per_token": "0.000001",
|
|
72
|
+
"output_cost_per_token": "0.000002",
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
monkeypatch.setattr("llmcalc.pricing_client.load_cached_pricing", lambda _: cached)
|
|
77
|
+
|
|
78
|
+
async def _never_fetch(pricing_url=None):
|
|
79
|
+
_ = pricing_url
|
|
80
|
+
raise AssertionError("fetch should not be called when cache is valid")
|
|
81
|
+
|
|
82
|
+
monkeypatch.setattr("llmcalc.pricing_client.fetch_pricing_payload", _never_fetch)
|
|
83
|
+
|
|
84
|
+
table = await get_pricing_table(cache_timeout=3600)
|
|
85
|
+
assert "gpt-4o-mini" in table
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@pytest.mark.asyncio
|
|
89
|
+
async def test_get_pricing_table_fetches_and_saves(monkeypatch) -> None:
|
|
90
|
+
saved = {"payload": None}
|
|
91
|
+
|
|
92
|
+
monkeypatch.setattr("llmcalc.pricing_client.load_cached_pricing", lambda _: None)
|
|
93
|
+
|
|
94
|
+
async def _fetch(pricing_url=None):
|
|
95
|
+
_ = pricing_url
|
|
96
|
+
return {
|
|
97
|
+
"gpt-4o-mini": {
|
|
98
|
+
"input_cost_per_token": "0.000001",
|
|
99
|
+
"output_cost_per_token": "0.000002",
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
monkeypatch.setattr("llmcalc.pricing_client.fetch_pricing_payload", _fetch)
|
|
104
|
+
monkeypatch.setattr(
|
|
105
|
+
"llmcalc.pricing_client.save_cached_pricing",
|
|
106
|
+
lambda payload: saved.update(payload=payload),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
table = await get_pricing_table(cache_timeout=3600)
|
|
110
|
+
assert "gpt-4o-mini" in table
|
|
111
|
+
assert saved["payload"] is not None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@pytest.mark.asyncio
|
|
115
|
+
async def test_get_pricing_table_raises_when_fetch_fails_without_cache(monkeypatch) -> None:
|
|
116
|
+
monkeypatch.setattr("llmcalc.pricing_client.load_cached_pricing", lambda _: None)
|
|
117
|
+
|
|
118
|
+
async def _fetch(pricing_url=None):
|
|
119
|
+
_ = pricing_url
|
|
120
|
+
raise PricingFetchError("failed")
|
|
121
|
+
|
|
122
|
+
monkeypatch.setattr("llmcalc.pricing_client.fetch_pricing_payload", _fetch)
|
|
123
|
+
|
|
124
|
+
with pytest.raises(PricingFetchError):
|
|
125
|
+
await get_pricing_table(cache_timeout=3600)
|