dot-inference 2.0.2__tar.gz → 2.0.3__tar.gz

{dot_inference-2.0.2 → dot_inference-2.0.3}/CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.0.3] - 2026-09-30
+
+### Fixed
+
+- `LLMResponse.cost` and `StreamUsage.cost` no longer overstate calls that hit
+  the prompt cache: they are now the cost the provider bills when it reports
+  one (OpenRouter's `usage.cost`), instead of an estimate from the registry's
+  per-token prices that charged cached tokens at the full rate. The estimate
+  remains the fallback for providers that do not report a cost.
+
 ## [2.0.2] - 2026-08-27
 
 ### Fixed
@@ -62,6 +72,7 @@ this release.
 - `LLMCallable` and `EmbeddingCallable` return a `Coroutine` rather than the
   looser `Awaitable`, matching what `asyncio.run_coroutine_threadsafe` accepts.
 
-[Unreleased]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-inference/-/compare/v2.0.2...main
+[Unreleased]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-inference/-/compare/v2.0.3...main
+[2.0.3]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-inference/-/tags/v2.0.3
 [2.0.2]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-inference/-/tags/v2.0.2
 [2.0.0]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-inference/-/tags/v2.0.0

{dot_inference-2.0.2 → dot_inference-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dot-inference
-Version: 2.0.2
+Version: 2.0.3
 Summary: Unified async Python library for LLM inference and embeddings across multiple providers
 Project-URL: Homepage, https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-inference
 Project-URL: Repository, https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-inference
@@ -313,7 +313,7 @@ package is covered, anything underscore-prefixed is internal and may change in
 any release. Public names are never removed without a deprecation period.
 
 ```toml
-dependencies = ["dot-inference>=1.0,<2"]
+dependencies = ["dot-inference>=2.0,<3"]
 ```
 
 See [docs/VERSIONING.md](docs/VERSIONING.md) for the full policy.

{dot_inference-2.0.2 → dot_inference-2.0.3}/README.md

@@ -286,7 +286,7 @@ package is covered, anything underscore-prefixed is internal and may change in
 any release. Public names are never removed without a deprecation period.
 
 ```toml
-dependencies = ["dot-inference>=1.0,<2"]
+dependencies = ["dot-inference>=2.0,<3"]
 ```
 
 See [docs/VERSIONING.md](docs/VERSIONING.md) for the full policy.

{dot_inference-2.0.2 → dot_inference-2.0.3}/docs/ARCHITECTURE.md

@@ -164,7 +164,7 @@ sequenceDiagram
         end
     end
 
-    Compat->>Compat: Parse response + compute cost from ResolvedModelConfig
+    Compat->>Compat: Parse response + cost (provider-reported, else from ResolvedModelConfig)
     Compat-->>App: LLMResponse (content, cost, tokens)
 ```
 

{dot_inference-2.0.2 → dot_inference-2.0.3}/docs/VERSIONING.md

@@ -1,7 +1,7 @@
 # Versioning and stability
 
 `dot-inference` follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-This page says what that actually covers, so `>=1.0,<2` means something concrete.
+This page says what that actually covers, so `>=2.0,<3` means something concrete.
 
 ## What is public
 
@@ -94,11 +94,11 @@ manual. Use them to validate a release without committing to it.
 ### From PyPI — the supported way
 
 ```toml
-dependencies = ["dot-inference>=1.0,<2"]
+dependencies = ["dot-inference>=2.0,<3"]
 ```
 
 The upper bound is what makes the guarantees above useful: you get every fix and
-feature of the 1.x line, and never an unannounced break.
+feature of the 2.x line, and never an unannounced break.
 
 ### From git — development only
 

{dot_inference-2.0.2 → dot_inference-2.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dot-inference"
-version = "2.0.2"
+version = "2.0.3"
 description = "Unified async Python library for LLM inference and embeddings across multiple providers"
 readme = "README.md"
 authors = [{ name = "Kannon For Deep Tech", email = "louis.letarnec@deepika.ai" }]

{dot_inference-2.0.2 → dot_inference-2.0.3}/src/dot_inference/_openai_compat.py

@@ -99,6 +99,19 @@ def _extract_cached_tokens(usage: Any) -> int | None:
     return getattr(details, "cached_tokens", None)
 
 
+def _reported_cost(usage: Any) -> float | None:
+    """The request's cost as the provider billed it, if it reports one.
+
+    OpenRouter returns it as ``usage.cost`` (USD, cached tokens included) on
+    every response, streamed or not. Returns None when the provider does not
+    report it.
+    """
+    cost = getattr(usage, "cost", None) if usage is not None else None
+    if isinstance(cost, bool) or not isinstance(cost, (int, float)):
+        return None
+    return float(cost)
+
+
 class OpenAICompatClient(LLMClient):
     """LLM client for any OpenAI-compatible endpoint.
 
@@ -218,6 +231,18 @@ class OpenAICompatClient(LLMClient):
 
         return create_kwargs, response_model
 
+    def _cost(self, usage: Any) -> float | None:
+        """What the call cost: as the provider billed it when it says, else
+        estimated from the registry's per-token prices (None if unknown)."""
+        reported = _reported_cost(usage)
+        if reported is not None or self._model_config is None:
+            return reported
+        return compute_cost(
+            usage,
+            self._model_config.input_token_cost,
+            self._model_config.output_token_cost,
+        )
+
     async def _create_with_retry(self, create_kwargs: dict[str, Any]) -> Any:
         """Call ``chat.completions.create`` with transient-error retry."""
         for attempt in range(1, _MAX_ATTEMPTS + 1):
@@ -267,13 +292,7 @@ class OpenAICompatClient(LLMClient):
         completion_tokens = response.usage.completion_tokens if response.usage else None
         cached_tokens = _extract_cached_tokens(response.usage)
 
-        cost = None
-        if self._model_config:
-            cost = compute_cost(
-                response.usage,
-                self._model_config.input_token_cost,
-                self._model_config.output_token_cost,
-            )
+        cost = self._cost(response.usage)
         if response.usage:
             LOGGER.debug(
                 "tokens: %s in (%s cached) / %s out, cost=$%s",
@@ -473,14 +492,7 @@ class OpenAICompatClient(LLMClient):
                         if inspect.isawaitable(result):
                             await result
 
-        # Cost
-        cost = None
-        if self._model_config:
-            cost = compute_cost(
-                last_usage,
-                self._model_config.input_token_cost,
-                self._model_config.output_token_cost,
-            )
+        cost = self._cost(last_usage)
         if prompt_tokens is not None:
             LOGGER.debug(
                 "tokens: %s in (%s cached) / %s out, cost=$%s",

{dot_inference-2.0.2 → dot_inference-2.0.3}/tests/test_openai_compat.py

@@ -26,6 +26,7 @@ from dot_inference._types import (
     ReasoningDelta,
     Role,
     StreamDone,
+    StreamUsage,
     ToolDef,
 )
 
@@ -751,3 +752,86 @@ async def test_cached_tokens_none_when_provider_omits_details():
     client = OpenAICompatClient(model="test", base_url="http://test", client=mock_client)
     response = await client.call([Message(role=Role.USER, content="Hi")])
     assert response.cached_tokens is None
+
+
+# --- cost: as the provider billed it, else estimated from the registry ---
+
+
+def _priced_config():
+    return ResolvedModelConfig(
+        model_id="test",
+        provider="test",
+        api_model_name="test",
+        input_token_cost=0.001,
+        output_token_cost=0.002,
+    )
+
+
+def _usage(**cost):
+    return SimpleNamespace(
+        prompt_tokens=100, completion_tokens=10, prompt_tokens_details=None, **cost
+    )
+
+
+def _mock_client_with_usage(usage):
+    mock_msg = MagicMock(content="ok", tool_calls=None)
+    mock_response = MagicMock(choices=[MagicMock(message=mock_msg)], usage=usage)
+    mock_client = MagicMock()
+    mock_client.chat.completions.create = AsyncMock(return_value=mock_response)
+    return mock_client
+
+
+@pytest.mark.asyncio
+async def test_call_cost_is_the_reported_one():
+    """OpenRouter bills `usage.cost` (cache included): it wins over the estimate."""
+    client = OpenAICompatClient(
+        model="test",
+        base_url="http://test",
+        client=_mock_client_with_usage(_usage(cost=0.0042)),
+        model_config=_priced_config(),
+    )
+    response = await client.call([Message(role=Role.USER, content="Hi")])
+    assert response.cost == 0.0042
+
+
+@pytest.mark.asyncio
+async def test_call_cost_falls_back_to_the_registry_prices():
+    client = OpenAICompatClient(
+        model="test",
+        base_url="http://test",
+        client=_mock_client_with_usage(_usage()),
+        model_config=_priced_config(),
+    )
+    response = await client.call([Message(role=Role.USER, content="Hi")])
+    assert response.cost == 100 * 0.001 + 10 * 0.002
+
+
+@pytest.mark.asyncio
+async def test_call_cost_is_reported_without_registry_prices():
+    client = OpenAICompatClient(
+        model="test",
+        base_url="http://test",
+        client=_mock_client_with_usage(_usage(cost=0.0042)),
+    )
+    response = await client.call([Message(role=Role.USER, content="Hi")])
+    assert response.cost == 0.0042
+
+
+@pytest.mark.asyncio
+async def test_stream_cost_is_the_reported_one():
+    """The reported cost rides the last chunk, with the usage."""
+    chunks = [
+        _stream_chunk(content="hi", finish_reason="stop"),
+        SimpleNamespace(usage=_usage(cost=0.0042), choices=[]),
+    ]
+    client = OpenAICompatClient(
+        model="test",
+        base_url="http://test",
+        client=_make_mock_stream_client(chunks),
+        model_config=_priced_config(),
+    )
+    events = await _collect(client)
+
+    [usage] = [e for e in events if isinstance(e, StreamUsage)]
+    assert usage.cost == 0.0042
+    assert events[-1].response.cost == 0.0042

{dot_inference-2.0.2 → dot_inference-2.0.3}/uv.lock

@@ -119,7 +119,7 @@ wheels = [
 
 [[package]]
 name = "dot-inference"
-version = "2.0.2"
+version = "2.0.3"
 source = { editable = "." }
 dependencies = [
     { name = "httpx" },