otto-cli-agent 0.1.2__tar.gz → 0.1.3__tar.gz

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: otto-cli-agent
-Version: 0.1.2
+Version: 0.1.3
 Summary: Otto: a terminal AI agent built around measured results -- one agent loop, a rubric-first evaluator, tiered memory, multi-vendor routing, and six benchmark harnesses.
 Project-URL: Repository, https://github.com/siddharth23P/otto_agent
 Project-URL: Issues, https://github.com/siddharth23P/otto_agent/issues

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/README.md

@@ -138,6 +138,7 @@ one interpreter per run), `OTTO_PYTHON_SESSION_MEMORY_MB` (that interpreter's
 address-space ceiling on Linux, default 8192, 0 for none),
 `OTTO_EMBEDDING_MODEL` (`provider:model`; local BGE is the floor),
 `OTTO_MODEL_PRICES` (a JSON file that overrides the price table),
+`OTTO_PROMPT_CACHE=0` (turn off input-token caching, on by default for all four vendors),
 `OTTO_IGNORE_ROUTES=1` (use the shipped routing table untouched; evals do),
 `OTTO_BROWSER_PYTHON` (an interpreter with Playwright and `pyte`, which
 enables the browser and terminal tools), `OTTO_HOME` (where state lives,

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/agent/pipeline/usage.py

@@ -139,7 +139,13 @@ class UsageLedger:
         details = usage.get("input_token_details")
         if isinstance(details, Mapping):
             entry.cached_input_tokens += _int(details.get("cache_read"))
-            entry.cache_write_tokens += _int(details.get("cache_creation"))
+            # langchain-anthropic reports a write by its TTL (ephemeral_5m/_1h_input_tokens) and then
+            # leaves cache_creation at 0; either shape is a write. Priced at the 5-minute write rate,
+            # the TTL otto asks for.
+            entry.cache_write_tokens += max(
+                _int(details.get("cache_creation")),
+                _int(details.get("ephemeral_5m_input_tokens")) + _int(details.get("ephemeral_1h_input_tokens")),
+            )
         entry.reported = entry.reported or got
 
     @property
@@ -192,6 +198,7 @@ class UsageLedger:
             "input_tokens": self.input_tokens,
             "output_tokens": self.output_tokens,
             "cached_input_tokens": self.cached_input_tokens,
+            "cache_write_tokens": sum(m.cache_write_tokens for m in self.by_model.values()),
             "total_tokens": self.total_tokens,
             "cost": self.cost,
             "fully_priced": self.fully_priced,
@@ -202,6 +209,7 @@ class UsageLedger:
                     "input_tokens": m.input_tokens,
                     "output_tokens": m.output_tokens,
                     "cached_input_tokens": m.cached_input_tokens,
+                    "cache_write_tokens": m.cache_write_tokens,
                     "total_tokens": m.total_tokens,
                     "reported": m.reported,
                     "cost": m.cost,

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/agent/router/llm_provider/anthropic_provider.py

@@ -33,8 +33,14 @@ from agent.router.llm_provider.base import (
     ProviderError,
     ProviderUnavailable,
     connection_detail,
+    prompt_caching,
+    with_model_kwargs,
 )
 
+#: The request-level breakpoint for automatic prompt caching (5-minute TTL: an agent loop
+#: calls again well inside it, and a 1-hour write costs 2x input instead of 1.25x).
+PROMPT_CACHE = {"type": "ephemeral"}
+
 __all__ = ["AnthropicProvider"]
 
 
@@ -126,4 +132,8 @@ class AnthropicProvider(BaseProvider):
     def chat_model(self, model_id: str, **kwargs: Any) -> BaseChatModel:
         if self._base_url:
             kwargs.setdefault("base_url", self._base_url)
+        if prompt_caching():
+            # Automatic caching (base.py PROMPT_CACHE_ENV): ChatAnthropic puts model_kwargs at the
+            # top level of the request, where the direct API reads `cache_control`.
+            kwargs = with_model_kwargs(kwargs, cache_control=dict(PROMPT_CACHE))
         return ChatAnthropic(model=model_id, api_key=self._api_key, **kwargs)

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/agent/router/llm_provider/base.py

@@ -184,6 +184,36 @@ class AuthError(ProviderError):
     """Key missing, malformed, or rejected by the vendor."""
 
 
+#: Prompt (input-token) caching, on unless OTTO_PROMPT_CACHE is 0/false/off.
+#: What "on" means differs by vendor, because their caches do:
+#:   anthropic  opt-in: top-level `cache_control` (automatic caching) -- the
+#:              breakpoint follows the last block, so an agent loop re-reads its
+#:              whole transcript at 0.1x input and writes only the new part.
+#:   openai     automatic for prefixes over 1024 tokens; `prompt_cache_key`
+#:              routes a model's calls to the same cache.
+#:   gemini     implicit for 2.5 and later -- nothing to send; the discount
+#:              arrives as `cached_content_token_count`, already counted.
+#:   inception  automatic on Mercury; the provider now reports its
+#:              `cached_input_tokens` so the ledger prices them.
+PROMPT_CACHE_ENV = "OTTO_PROMPT_CACHE"
+
+
+def prompt_caching() -> bool:
+    return os.environ.get(PROMPT_CACHE_ENV, "1").strip().lower() not in ("0", "false", "off", "no")
+
+
+def with_model_kwargs(kwargs: dict[str, Any], **defaults: Any) -> dict[str, Any]:
+    """`kwargs` with `defaults` merged under its `model_kwargs`; a route that set one of them wins,
+    and a route that set it to None removes it."""
+    extra = dict(kwargs.pop("model_kwargs", None) or {})
+    for key, value in defaults.items():
+        extra.setdefault(key, value)
+    extra = {k: v for k, v in extra.items() if v is not None}
+    if extra:
+        kwargs["model_kwargs"] = extra
+    return kwargs
+
+
 def connection_detail(exc: BaseException) -> str:
     """A vendor SDK's message for a failed connection, with what failed underneath it.
 

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/agent/router/llm_provider/gemini_provider.py

@@ -117,6 +117,10 @@ class GeminiProvider(BaseProvider):
         return found
 
     def chat_model(self, model_id: str, **kwargs: Any) -> BaseChatModel:
+        # Prompt caching is implicit on Gemini 2.5 and later: a repeated prefix is billed at the
+        # cached rate with nothing sent, and ChatGoogleGenerativeAI reports it as
+        # input_token_details.cache_read (base.py PROMPT_CACHE_ENV). Explicit `cached_content`
+        # caches are not used -- they are stored, and billed, per hour.
         return ChatGoogleGenerativeAI(
             model=model_id, google_api_key=self._api_key, **kwargs
         )

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/agent/router/llm_provider/inception_provider.py

@@ -98,6 +98,28 @@ def _field(obj: Any, key: str) -> Any:
     return getattr(obj, key, None)
 
 
+def _cached_tokens(u: Any) -> int | None:
+    """Mercury's cache hits: `prompt_tokens_details.cached_tokens` (what the API sends, as OpenAI
+    does) or the SDK's own `cached_input_tokens`."""
+    direct = _field(u, "cached_input_tokens")
+    if direct:
+        return direct
+    details = _field(u, "prompt_tokens_details")
+    if details is None and isinstance(getattr(u, "model_extra", None), Mapping):
+        details = u.model_extra.get("prompt_tokens_details")
+    return _field(details, "cached_tokens")
+
+
+def _usage_metadata(input_tokens: int, output_tokens: int, total_tokens: int, cached: Any) -> dict:
+    """LangChain's usage shape. Mercury caches repeated prefixes by itself and reports the hits as
+    `cached_input_tokens`; they go to input_token_details.cache_read, where the ledger prices them
+    (agent/pipeline/usage.py)."""
+    metadata: dict = {"input_tokens": input_tokens, "output_tokens": output_tokens, "total_tokens": total_tokens}
+    if isinstance(cached, int) and cached > 0:
+        metadata["input_token_details"] = {"cache_read": cached}
+    return metadata
+
+
 def _usage(u: Any) -> dict[str, int] | None:
     """Vendor usage payload -> Langfuse's usage_details key names."""
     if u is None:
@@ -110,11 +132,10 @@ def _usage(u: Any) -> dict[str, int] | None:
     details = {k: v for k, v in details.items() if v is not None}
     if not details:
         return None
-    for attr, key in (("cached_input_tokens", "cached_input"),
-                      ("reasoning_tokens", "reasoning")):
-        value = _field(u, attr)
-        if value:
-            details[key] = value
+    if cached := _cached_tokens(u):
+        details["cached_input"] = cached
+    if reasoning := _field(u, "reasoning_tokens"):
+        details["reasoning"] = reasoning
     return details
 
 
@@ -380,11 +401,10 @@ class ChatInception(BaseChatModel):
                 "finish_reason": choice.finish_reason,
                 "warning": completion.warning,
             },
-            usage_metadata={
-                "input_tokens": usage.prompt_tokens,
-                "output_tokens": usage.completion_tokens,
-                "total_tokens": usage.total_tokens,
-            },
+            usage_metadata=_usage_metadata(
+                usage.prompt_tokens, usage.completion_tokens, usage.total_tokens,
+                _cached_tokens(usage),
+            ),
         )
         return ChatResult(
             generations=[
@@ -449,12 +469,11 @@ class ChatInception(BaseChatModel):
                         yield ChatGenerationChunk(
                             message=AIMessageChunk(
                                 content="",
-                                usage_metadata={
-                                    "input_tokens": input_tokens,
-                                    "output_tokens": output_tokens,
-                                    "total_tokens": _field(usage, "total_tokens")
-                                    or input_tokens + output_tokens,
-                                },
+                                usage_metadata=_usage_metadata(
+                                    input_tokens, output_tokens,
+                                    _field(usage, "total_tokens") or input_tokens + output_tokens,
+                                    _cached_tokens(usage),
+                                ),
                             )
                         )
                     continue

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/agent/router/llm_provider/openai_provider.py

@@ -31,8 +31,13 @@ from agent.router.llm_provider.base import (
     ProviderError,
     ProviderUnavailable,
     connection_detail,
+    prompt_caching,
+    with_model_kwargs,
 )
 
+#: prompt_cache_key is this plus the model id: one cache per model, shared by its calls.
+PROMPT_CACHE_KEY_PREFIX = "otto:"
+
 __all__ = ["OpenAIProvider"]
 
 
@@ -132,6 +137,11 @@ class OpenAIProvider(BaseProvider):
         # No hand-rolled chat method. Returning a BaseChatModel is what makes
         # this provider interchangeable inside a LangGraph node and visible to
         # Langfuse without any extra wiring.
+        if prompt_caching() and self._base_url is None:
+            # OpenAI caches long prefixes by itself; the key sends a model's calls to the same cache
+            # (base.py PROMPT_CACHE_ENV). Only for OpenAI itself: an OpenAI-compatible server
+            # (custom.py) may reject a field it does not know.
+            kwargs = with_model_kwargs(kwargs, prompt_cache_key=f"{PROMPT_CACHE_KEY_PREFIX}{model_id}")
         return ChatOpenAI(
             model=model_id,
             api_key=self._api_key,

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "otto-cli-agent"
-version = "0.1.2"
+version = "0.1.3"
 description = "Otto: a terminal AI agent built around measured results -- one agent loop, a rubric-first evaluator, tiered memory, multi-vendor routing, and six benchmark harnesses."
 readme = "docs/pypi.md"
 license = { file = "LICENSE" }

otto_cli_agent-0.1.3/tests/test_prompt_caching.py

@@ -0,0 +1,108 @@
+"""Input-token caching is on for all four providers (agent/router/llm_provider/base.py
+PROMPT_CACHE_ENV), and what each vendor reports as cached is priced as cached."""
+from __future__ import annotations
+
+import pytest
+
+from agent.pipeline.usage import UsageLedger
+from agent.router.llm_provider import base
+from agent.router.llm_provider.anthropic_provider import PROMPT_CACHE, AnthropicProvider
+from agent.router.llm_provider.inception_provider import _usage_metadata
+from agent.router.llm_provider.openai_provider import OpenAIProvider
+
+
+def _anthropic(**kw):
+    provider = AnthropicProvider.__new__(AnthropicProvider)
+    provider._api_key, provider._base_url = "sk-ant-test", kw.get("base_url")
+    return provider
+
+
+def _openai(base_url=None):
+    provider = OpenAIProvider.__new__(OpenAIProvider)
+    provider._api_key, provider._base_url = "sk-test", base_url
+    return provider
+
+
+def test_anthropic_asks_for_automatic_caching_at_the_top_of_the_request(monkeypatch):
+    monkeypatch.delenv(base.PROMPT_CACHE_ENV, raising=False)
+    llm = _anthropic().chat_model("claude-haiku-4-5-20251001", max_tokens=64)
+    assert llm.model_kwargs["cache_control"] == PROMPT_CACHE
+    payload = llm._get_request_payload([("human", "hi")])
+    assert payload["cache_control"] == {"type": "ephemeral"}
+
+
+def test_a_route_keeps_its_own_model_kwargs_and_may_opt_out(monkeypatch):
+    monkeypatch.delenv(base.PROMPT_CACHE_ENV, raising=False)
+    llm = _anthropic().chat_model("claude-haiku-4-5", model_kwargs={"service_tier": "auto"})
+    assert llm.model_kwargs == {"service_tier": "auto", "cache_control": PROMPT_CACHE}
+    llm = _anthropic().chat_model("claude-haiku-4-5", model_kwargs={"cache_control": {"type": "ephemeral", "ttl": "1h"}})
+    assert llm.model_kwargs["cache_control"]["ttl"] == "1h"
+    llm = _anthropic().chat_model("claude-haiku-4-5", model_kwargs={"cache_control": None})
+    assert "cache_control" not in llm.model_kwargs
+
+
+def test_openai_routes_a_models_calls_to_one_cache(monkeypatch):
+    monkeypatch.delenv(base.PROMPT_CACHE_ENV, raising=False)
+    llm = _openai().chat_model("gpt-5-mini")
+    assert llm.model_kwargs["prompt_cache_key"] == "otto:gpt-5-mini"
+    assert "prompt_cache_key" in llm._get_request_payload([("human", "hi")])
+    # An OpenAI-compatible server is not sent a field it may not know.
+    assert "prompt_cache_key" not in _openai("http://localhost:11434/v1").chat_model("llama3").model_kwargs
+
+
+@pytest.mark.parametrize("value", ["0", "false", "OFF"])
+def test_the_switch_turns_it_off(monkeypatch, value):
+    monkeypatch.setenv(base.PROMPT_CACHE_ENV, value)
+    assert "cache_control" not in _anthropic().chat_model("claude-haiku-4-5").model_kwargs
+    assert "prompt_cache_key" not in _openai().chat_model("gpt-5-mini").model_kwargs
+
+
+def test_inception_reports_its_cached_tokens():
+    assert _usage_metadata(100, 5, 105, 80) == {
+        "input_tokens": 100, "output_tokens": 5, "total_tokens": 105,
+        "input_token_details": {"cache_read": 80}}
+    assert "input_token_details" not in _usage_metadata(100, 5, 105, 0)
+    assert "input_token_details" not in _usage_metadata(100, 5, 105, None)
+
+
+def test_inception_streaming_carries_the_cached_count():
+    from tests.test_inception_stream import accumulate, content_chunk, model, usage_chunk
+
+    llm = model([content_chunk("hi"), usage_chunk(
+        {"prompt_tokens": 2000, "completion_tokens": 3, "total_tokens": 2003, "cached_input_tokens": 1800})])
+    assert accumulate(llm).usage_metadata["input_token_details"] == {"cache_read": 1800}
+
+
+def test_cached_reads_and_writes_reach_the_snapshot():
+    ledger = UsageLedger()
+    ledger.record("claude-haiku-4-5", {
+        "input_tokens": 5000, "output_tokens": 10, "total_tokens": 5010,
+        "input_token_details": {"cache_read": 4000, "cache_creation": 900}})
+    snap = ledger.snapshot()
+    assert snap["cached_input_tokens"] == 4000 and snap["cache_write_tokens"] == 900
+    assert snap["models"][0]["cache_write_tokens"] == 900
+
+
+def test_inception_reads_the_cached_count_where_the_api_puts_it():
+    from agent.router.llm_provider.inception_provider import _cached_tokens
+
+    assert _cached_tokens({"prompt_tokens": 9, "prompt_tokens_details": {"cached_tokens": 7}}) == 7
+    assert _cached_tokens({"cached_input_tokens": 5, "prompt_tokens_details": {"cached_tokens": 0}}) == 5
+    assert not _cached_tokens({"cached_input_tokens": None, "prompt_tokens_details": {"cached_tokens": 0}})
+    assert not _cached_tokens({"prompt_tokens": 9})
+
+
+def test_an_anthropic_write_reported_by_its_ttl_is_priced_as_a_write():
+    # The shape langchain-anthropic 1.7 returns for a first cached call (seen live, 2026-09-17).
+    ledger = UsageLedger()
+    ledger.record("claude-haiku-4-5", {
+        "input_tokens": 12203, "output_tokens": 4, "total_tokens": 12207,
+        "input_token_details": {"cache_read": 0, "cache_creation": 0,
+                                "ephemeral_5m_input_tokens": 12200, "ephemeral_1h_input_tokens": 0}})
+    assert ledger.snapshot()["cache_write_tokens"] == 12200
+    ledger.record("claude-haiku-4-5", {
+        "input_tokens": 12203, "output_tokens": 4, "total_tokens": 12207,
+        "input_token_details": {"cache_read": 12200, "cache_creation": 0,
+                                "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}})
+    snap = ledger.snapshot()
+    assert snap["cached_input_tokens"] == 12200 and snap["cache_write_tokens"] == 12200

{otto_cli_agent-0.1.2 → otto_cli_agent-0.1.3}/uv.lock

@@ -1842,7 +1842,7 @@ wheels = [
 
 [[package]]
 name = "otto-cli-agent"
-version = "0.1.2"
+version = "0.1.3"
 source = { editable = "." }
 dependencies = [
     { name = "emoji" },