callm-toolkit 0.2.0__tar.gz → 0.3.0__tar.gz

{callm_toolkit-0.2.0 → callm_toolkit-0.3.0}/CHANGELOG.md

@@ -6,6 +6,20 @@ All notable changes to this project are documented here. The format follows
 
 ## [Unreleased]
 
+## [0.3.0] - 2026-10-02
+
+### Added
+
+- OpenAI structured outputs: `client.chat.completions.parse(...)`, sync and async, is now
+  intercepted. PII masking, budgets, caching, retries, cost and telemetry apply, and the
+  caller still gets a `ParsedChatCompletion` with `.parsed`, also on cache hits. Different
+  `response_format` models never share a cache entry.
+
+### Fixed
+
+- Storing a `ParsedChatCompletion` for the cache no longer prints pydantic serializer
+  warnings.
+
 ## [0.2.0] - 2026-09-29
 
 ### Added
@@ -100,7 +114,8 @@ Initial release.
   budgets) from enclosing decorators and shields apply to inner calls.
 - `callm` CLI: `stats`, `calls`, `cache`, `pricing`, `telemetry`, `info`.
 
-[Unreleased]: https://github.com/TanbirRamim/callm/compare/v0.2.0...HEAD
+[Unreleased]: https://github.com/TanbirRamim/callm/compare/v0.3.0...HEAD
+[0.3.0]: https://github.com/TanbirRamim/callm/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/TanbirRamim/callm/compare/v0.1.2...v0.2.0
 [0.1.2]: https://github.com/TanbirRamim/callm/compare/v0.1.1...v0.1.2
 [0.1.1]: https://github.com/TanbirRamim/callm/compare/v0.1.0...v0.1.1

{callm_toolkit-0.2.0 → callm_toolkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: callm-toolkit
-Version: 0.2.0
+Version: 0.3.0
 Summary: The production toolkit for LLM calls: caching, retries, fallback, cost tracking, security and structured output in one decorator.
 Project-URL: Homepage, https://github.com/TanbirRamim/callm
 Project-URL: Documentation, https://tanbirramim.github.io/callm
@@ -364,7 +364,7 @@ def summarize():   ─────────►  │ 1. Telemetry     cost, la
 ```
 
 1. `@callm` sets a scope (a `ContextVar`) while your function runs.
-2. The official SDK methods (`chat.completions.create`, OpenAI's `responses.create`,
+2. The official SDK methods (`chat.completions.create` and `.parse`, OpenAI's `responses.create`,
    `messages.create`, `models.generate_content`, sync and async) are instrumented on first use. Outside a callm
    scope they call straight through, so importing callm never changes other code.
 3. Inside a scope, the SDK call is converted into a provider-neutral request and sent through

{callm_toolkit-0.2.0 → callm_toolkit-0.3.0}/README.md

@@ -305,7 +305,7 @@ def summarize():   ─────────►  │ 1. Telemetry     cost, la
 ```
 
 1. `@callm` sets a scope (a `ContextVar`) while your function runs.
-2. The official SDK methods (`chat.completions.create`, OpenAI's `responses.create`,
+2. The official SDK methods (`chat.completions.create` and `.parse`, OpenAI's `responses.create`,
    `messages.create`, `models.generate_content`, sync and async) are instrumented on first use. Outside a callm
    scope they call straight through, so importing callm never changes other code.
 3. Inside a scope, the SDK call is converted into a provider-neutral request and sent through

callm_toolkit-0.3.0/src/callm/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{callm_toolkit-0.2.0 → callm_toolkit-0.3.0}/src/callm/interception.py

@@ -103,6 +103,8 @@ class PatchTarget:
 PATCH_TARGETS: tuple[PatchTarget, ...] = (
     PatchTarget("openai", "openai.resources.chat.completions", "Completions", "create", False),
     PatchTarget("openai", "openai.resources.chat.completions", "AsyncCompletions", "create", True),
+    PatchTarget("openai", "openai.resources.chat.completions", "Completions", "parse", False),
+    PatchTarget("openai", "openai.resources.chat.completions", "AsyncCompletions", "parse", True),
     PatchTarget("openai-responses", "openai.resources.responses", "Responses", "create", False),
     PatchTarget("openai-responses", "openai.resources.responses", "AsyncResponses", "create", True),
     PatchTarget("anthropic", "anthropic.resources.messages", "Messages", "create", False),
@@ -122,7 +124,7 @@ PATCH_TARGETS: tuple[PatchTarget, ...] = (
                 "openai",
                 "openai.resources.chat.completions",
                 ("Completions", "AsyncCompletions"),
-                ("parse", "stream"),
+                ("stream",),
             ),
             (
                 "openai",
@@ -280,7 +282,38 @@ def clean_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
 def _finish(target: PatchTarget, state: CallState, response: LLMResponse) -> Any:
     if state.request.stream:
         return response.raw
-    return get_provider(target.provider).native_for(response)
+    native = get_provider(target.provider).native_for(response)
+    if target.provider == "openai" and target.method == "parse":
+        native = _as_parsed_completion(native, state.request.native_extra)
+    return native
+
+
+def _as_parsed_completion(native: Any, extra: dict[str, Any]) -> Any:
+    """The ``ParsedChatCompletion`` that ``chat.completions.parse`` promises its caller.
+
+    A live call already returns one. A cache hit, or a response synthesized after a
+    fallback, is a plain ``ChatCompletion``, so run it through the SDK's own parser
+    with the caller's ``response_format`` and ``tools``.
+    """
+    if type(native).__name__.startswith("Parsed"):
+        return native
+    try:
+        import openai
+        from openai.lib._parsing import parse_chat_completion
+    except ImportError:
+        return native
+    not_given: Any = getattr(openai, "omit", None) or getattr(openai, "NOT_GIVEN", None)
+    response_format: Any = extra.get("response_format", not_given)
+    input_tools: Any = extra.get("tools", not_given)
+    try:
+        return parse_chat_completion(
+            response_format=response_format,
+            input_tools=input_tools,
+            chat_completion=native,
+        )
+    except (TypeError, AttributeError):  # an AttrDict without the SDK types
+        logger.debug("callm: could not rebuild a ParsedChatCompletion", exc_info=True)
+        return native
 
 
 _warned_methods: set[str] = set()

{callm_toolkit-0.2.0 → callm_toolkit-0.3.0}/src/callm/providers/base.py

@@ -61,7 +61,14 @@ def dump_json(obj: Any) -> dict[str, Any] | None:
     dump = getattr(obj, "model_dump", None)
     if callable(dump):
         try:
-            result = dump(mode="json", exclude_none=True)
+            # warnings=False: ParsedChatCompletion types `parsed` loosely, so pydantic
+            # would warn on every call even though the dump is correct.
+            result = dump(mode="json", exclude_none=True, warnings=False)
+        except TypeError:  # an object whose model_dump has no `warnings` option
+            try:
+                result = dump(mode="json", exclude_none=True)
+            except Exception:
+                return None
         except Exception:
             return None
         return result if isinstance(result, dict) else None

callm_toolkit-0.3.0/tests/test_parse.py

@@ -0,0 +1,123 @@
+"""OpenAI structured outputs: ``client.chat.completions.parse``."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+from pydantic import BaseModel
+
+import callm
+from callm.errors import BudgetExceededError
+from helpers import openai_body
+
+
+class Invoice(BaseModel):
+    vendor: str
+    total: float
+
+
+INVOICE_JSON = json.dumps({"vendor": "ACME", "total": 12.5})
+
+
+@pytest.fixture(autouse=True)
+def _needs_parse(openai_client):
+    if not hasattr(openai_client.chat.completions, "parse"):
+        pytest.skip("this openai version has no chat.completions.parse")
+
+
+def ask(client, text="Extract the invoice: ACME, 12.50 EUR"):
+    return client.chat.completions.parse(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": text}],
+        response_format=Invoice,
+    )
+
+
+def test_parse_is_tracked_and_returns_the_parsed_object(openai_client, openai_server, records):
+    openai_server.queue(openai_body(text=INVOICE_JSON))
+
+    result = callm.callm(lambda: ask(openai_client))()
+
+    assert type(result).__name__.startswith("ParsedChatCompletion")
+    assert result.choices[0].message.parsed == Invoice(vendor="ACME", total=12.5)
+    sent = openai_server.last.body
+    assert sent["response_format"]["type"] == "json_schema"
+    assert sent["response_format"]["json_schema"]["name"] == "Invoice"
+    (record,) = records()
+    assert record.provider == "openai"
+    assert record.cost_usd and record.cost_usd > 0
+
+
+def test_cache_hit_still_returns_a_parsed_completion(openai_client, openai_server, records):
+    openai_server.queue(openai_body(text=INVOICE_JSON))
+    cached = callm.callm(cache=True)(lambda: ask(openai_client))
+
+    first = cached()
+    second = cached()
+
+    assert openai_server.count == 1
+    assert type(second).__name__.startswith("ParsedChatCompletion")
+    assert second.choices[0].message.parsed == first.choices[0].message.parsed
+    assert records()[0].cache_hit is True
+
+
+def test_different_response_formats_do_not_share_a_cache_entry(openai_client, openai_server):
+    class Receipt(BaseModel):
+        vendor: str
+        total: float
+
+    openai_server.queue(openai_body(text=INVOICE_JSON), openai_body(text=INVOICE_JSON))
+
+    @callm.callm(cache=True)
+    def both():
+        a = ask(openai_client)
+        b = openai_client.chat.completions.parse(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "Extract the invoice: ACME, 12.50 EUR"}],
+            response_format=Receipt,
+        )
+        return a, b
+
+    a, b = both()
+    assert openai_server.count == 2
+    assert isinstance(b.choices[0].message.parsed, Receipt)
+
+
+def test_pii_is_masked_in_parse_requests(openai_client, openai_server):
+    openai_server.queue(openai_body(text=INVOICE_JSON))
+
+    callm.callm(block_pii=True)(lambda: ask(openai_client, "Invoice from ops@acme.example"))()
+
+    assert "ops@acme.example" not in json.dumps(openai_server.last.body)
+
+
+def test_budget_blocks_parse_before_sending(openai_client, openai_server):
+    @callm.callm(max_cost=0.0000001)
+    def expensive():
+        return openai_client.chat.completions.parse(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "x" * 4000}],
+            response_format=Invoice,
+            max_completion_tokens=4000,
+        )
+
+    with pytest.raises(BudgetExceededError):
+        expensive()
+    assert openai_server.count == 0
+
+
+async def test_async_parse(async_openai_client, openai_server, records):
+    openai_server.queue(openai_body(text=INVOICE_JSON))
+
+    @callm.callm
+    async def run():
+        return await async_openai_client.chat.completions.parse(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "Extract the invoice"}],
+            response_format=Invoice,
+        )
+
+    result = await run()
+    assert result.choices[0].message.parsed == Invoice(vendor="ACME", total=12.5)
+    assert records()[0].provider == "openai"

{callm_toolkit-0.2.0 → callm_toolkit-0.3.0}/tests/test_regressions.py

@@ -513,8 +513,8 @@ def test_last_call_is_per_context_for_concurrent_tasks(async_openai_client):
 def test_unintercepted_sdk_methods_warn_inside_callm(openai_client, openai_server, caplog):
     import logging
 
-    if not hasattr(openai_client.chat.completions, "parse"):
-        pytest.skip("this openai version has no chat.completions.parse")
+    if not hasattr(openai_client.chat.completions, "stream"):
+        pytest.skip("this openai version has no chat.completions.stream")
     from callm import interception
 
     interception._warned_methods.clear()
@@ -522,18 +522,19 @@ def test_unintercepted_sdk_methods_warn_inside_callm(openai_client, openai_serve
 
     @callm.callm(block_pii=True, max_cost=1.0)
     def ask():
-        return openai_client.chat.completions.parse(model="gpt-4o-mini", messages=messages)
+        # stream() returns a manager and sends nothing until it is entered
+        return openai_client.chat.completions.stream(model="gpt-4o-mini", messages=messages)
 
     with caplog.at_level(logging.WARNING, logger="callm"):
         ask()
         ask()
     warnings = [r.getMessage() for r in caplog.records if "not intercepted" in r.getMessage()]
     assert len(warnings) == 1  # once per method, not per call
-    assert "chat.completions.parse" in warnings[0]
+    assert "chat.completions.stream" in warnings[0]
     assert "PII masking" in warnings[0] and "budgets" in warnings[0]
 
     caplog.clear()
     with caplog.at_level(logging.WARNING, logger="callm"):
         interception._warned_methods.clear()
-        openai_client.chat.completions.parse(model="gpt-4o-mini", messages=messages)  # outside
+        openai_client.chat.completions.stream(model="gpt-4o-mini", messages=messages)  # outside
     assert not [r for r in caplog.records if "not intercepted" in r.getMessage()]

callm_toolkit-0.2.0/src/callm/__about__.py

@@ -1 +0,0 @@
-__version__ = "0.2.0"