cmpnd 0.33.2__tar.gz → 0.33.4__tar.gz

{cmpnd-0.33.2 → cmpnd-0.33.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cmpnd
-Version: 0.33.2
+Version: 0.33.4
 Summary: DSPy observability and deployment SDK for cmpnd
 Project-URL: Homepage, https://cmpnd.ai
 Author-email: cmpnd <hello@cmpnd.ai>

{cmpnd-0.33.2 → cmpnd-0.33.4}/cmpnd/callback.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import contextvars
 import logging
+from collections.abc import Mapping
 from datetime import datetime, timezone
 from typing import Any
 from uuid import UUID, uuid4
@@ -320,17 +321,49 @@ def _present_field(value: Any, key: str) -> tuple[bool, Any]:
     return False, None
 
 
+def _usage_reported(usage: Any) -> bool:
+    """Whether a usage value reports anything.
+
+    An empty mapping counts as nothing. DSPy >= 3.4 writes ``usage={}`` for a
+    call its own cache answered, and a reported zero is a mapping of zeros,
+    never an empty one.
+    """
+    return usage is not None and not (isinstance(usage, Mapping) and not usage)
+
+
 def _history_usage(entry: Any) -> tuple[bool, Any]:
     """Return whether history carried usage and the value it carried."""
     response = _value_field(entry, "response")
     response_declares_usage, response_usage = _present_field(response, "usage")
     if response_declares_usage:
-        return response_usage is not None, response_usage
+        return _usage_reported(response_usage), response_usage
     if isinstance(entry, dict):
         usage = entry.get("usage")
-        return "usage" in entry and usage is not None, usage
+        return _usage_reported(usage), usage
     usage = _value_field(entry, "usage")
-    return usage is not None, usage
+    return _usage_reported(usage), usage
+
+
+def _lm15_cache_hit(entry: Any, response: Any) -> bool | None:
+    """Recognize a DSPy cache hit on a DSPy >= 3.4 lm15-native call.
+
+    Nothing on the history entry or the ``Response`` says the cache answered.
+    DSPy's ``CallResult.load`` rebuilds the cached ``Response`` with its token
+    counts intact and gives the call an empty ``usage``, and the history entry
+    copies that empty ``usage``. A fresh native call derives the entry's
+    ``usage`` from those same counts, so it is empty only when the
+    ``Response`` has none. An empty entry ``usage`` beside a ``Response`` that
+    does carry counts is therefore a cache hit.
+
+    Returns ``None`` when the shapes do not establish one either way.
+    """
+    if not isinstance(entry, dict) or entry.get("usage") != {}:
+        return None
+    usage = getattr(response, "usage", None)
+    if usage is None or isinstance(usage, Mapping):
+        return None
+    counts = (getattr(usage, name, None) for name in ("input_tokens", "output_tokens", "total_tokens"))
+    return True if any(isinstance(count, int) for count in counts) else None
 
 
 def _usage_dialect(instance: Any, usage: Any) -> str | None:
@@ -388,15 +421,22 @@ def _response_text(response: Any, key: str) -> str | None:
     return None
 
 
+#: lm15 finish reasons whose chat spelling differs. lm15 names a tool-call stop
+#: in the singular (``FinishReason`` in DSPy's vendored ``lm15/types.py``); its
+#: ``stop``, ``length`` and ``content_filter`` are already the chat words.
+_LM15_FINISH_REASONS = {"tool_call": "tool_calls"}
+
+
 def _finish_reason(response: Any) -> str | None:
     """Read why the model stopped, from whichever response shape answered.
 
-    Chat puts it on a choice. DSPy's typed ``LMResponse`` puts it on an output.
-    The Responses API has neither: it reports a ``status`` plus, when the answer
-    was cut short, an ``incomplete_details.reason``. Those map onto the chat
-    vocabulary, which is what serve's truncation warning reads — a Responses
-    ``max_output_tokens`` and a chat ``length`` are the same fact and must land
-    on the same word.
+    Chat puts it on a choice. DSPy's typed ``LMResponse`` (3.3) puts it on an
+    output. The lm15 ``Response`` a DSPy >= 3.4 engine returns carries it at the
+    top level. The Responses API has none of these: it reports a ``status``
+    plus, when the answer was cut short, an ``incomplete_details.reason``. All of
+    them map onto the chat vocabulary, which is what serve's truncation warning
+    reads — a Responses ``max_output_tokens`` and a chat ``length`` are the same
+    fact and must land on the same word.
 
     Returns ``None`` when the response reports nothing about it.
     """
@@ -415,6 +455,10 @@ def _finish_reason(response: Any) -> str | None:
         if reason:
             return str(reason)
 
+    reason = getattr(response, "finish_reason", None)
+    if isinstance(reason, str) and reason:
+        return _LM15_FINISH_REASONS.get(reason, reason)
+
     incomplete = getattr(response, "incomplete_details", None)
     reason = incomplete.get("reason") if isinstance(incomplete, dict) else getattr(incomplete, "reason", None)
     if reason:
@@ -1381,6 +1425,8 @@ class _LMMixin(_CallbackBase):
             cache_hit = _bool_field(response, "cache_hit")
             if cache_hit is None:
                 cache_hit = _bool_field(_value_field(response, "_hidden_params"), "cache_hit")
+            if cache_hit is None:
+                cache_hit = _lm15_cache_hit(latest_entry, response)
             observation.dspy_client_cache_hit = cache_hit
             observation.usage_reported = usage_reported and cache_hit is not True
             serialized_usage = models._safe_serialize(usage)

{cmpnd-0.33.2 → cmpnd-0.33.4}/cmpnd/context.py

@@ -13,8 +13,12 @@ if TYPE_CHECKING:
 # Current trace context
 _current_trace: ContextVar[models.TraceBuilder | None] = ContextVar("cmpnd_current_trace", default=None)
 
-# Stack of active spans (for nested spans)
-_span_stack: ContextVar[list[models.SpanBuilder] | None] = ContextVar("cmpnd_span_stack", default=None)
+# Stack of active spans (for nested spans). A tuple, replaced on every push and
+# pop and never mutated: a context copy (``contextvars.copy_context()``, an
+# asyncio task) shares the objects its parent context holds, so a mutable stack
+# would let concurrent runs in sibling copies read each other's spans as their
+# parent while each keeps its own trace.
+_span_stack: ContextVar[tuple[models.SpanBuilder, ...]] = ContextVar("cmpnd_span_stack", default=())
 
 # Current eval run context
 _current_eval_run: ContextVar[models.EvalRunBuilder | None] = ContextVar("cmpnd_current_eval_run", default=None)
@@ -37,26 +41,21 @@ def set_current_trace(trace: models.TraceBuilder | None) -> None:
 def get_current_span() -> models.SpanBuilder | None:
     """Get the current (innermost) span."""
     stack = _span_stack.get()
-    if stack is None:
-        return None
     return stack[-1] if stack else None
 
 
 def push_span(span: models.SpanBuilder) -> None:
-    """Push a span onto the context stack."""
-    stack = _span_stack.get()
-    if stack is None:
-        stack = []
-        _span_stack.set(stack)
-    stack.append(span)
+    """Push a span onto this context's stack."""
+    _span_stack.set((*_span_stack.get(), span))
 
 
 def pop_span() -> models.SpanBuilder | None:
-    """Pop the innermost span from the context stack."""
+    """Pop the innermost span from this context's stack."""
     stack = _span_stack.get()
-    if stack:
-        return stack.pop()
-    return None
+    if not stack:
+        return None
+    _span_stack.set(stack[:-1])
+    return stack[-1]
 
 
 def get_current_eval_run() -> models.EvalRunBuilder | None:
@@ -119,6 +118,6 @@ def create_child_span(
 def clear_context() -> None:
     """Clear all trace/span context. Useful for testing."""
     _current_trace.set(None)
-    _span_stack.set(None)
+    _span_stack.set(())
     _current_eval_run.set(None)
     _current_optimization_run_id.set(None)

{cmpnd-0.33.2 → cmpnd-0.33.4}/cmpnd/helpers.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from typing import Any
 
-from . import models
+from . import identity, models
 
 
 def truncate_field(value: Any, max_size: int, depth: int = 0) -> Any:
@@ -21,8 +21,44 @@ def truncate_field(value: Any, max_size: int, depth: int = 0) -> Any:
     return models.truncate_field(value, max_size, depth)
 
 
-def clean_inputs(inputs: dict[str, Any], _dspy: Any, max_field_size: int) -> dict[str, Any]:
-    """Clean and prepare inputs for storage, with per-field truncation."""
+def signature_label(signature: Any) -> str:
+    """Name a DSPy signature in a few words: its class name, else its fields.
+
+    A signature built from a string is a ``StringSignature`` whatever it holds,
+    so it is named by its ``in1, in2 -> out1`` summary instead.
+    """
+    name = getattr(signature, "__name__", None)
+    if name and name != "StringSignature":
+        return name
+    return identity.build_signature_summary(signature) or str(name or "")
+
+
+def _is_signature(value: Any, dspy: Any) -> bool:
+    signature_type = getattr(dspy, "Signature", None)
+    return isinstance(value, type) and isinstance(signature_type, type) and issubclass(value, signature_type)
+
+
+def _input_value(value: Any, dspy: Any) -> Any:
+    """Replace a signature class, alone or among positional args, by its label.
+
+    DSPy hands every adapter ``format``/``parse`` hook the predictor's
+    signature, and a ``Predict`` called with ``signature=`` gets one too. A
+    signature class's repr embeds its whole instructions and every field's
+    metadata, which the span already records once as its own attributes.
+    """
+    if _is_signature(value, dspy):
+        return signature_label(value)
+    if isinstance(value, (list, tuple)) and any(_is_signature(item, dspy) for item in value):
+        return [signature_label(item) if _is_signature(item, dspy) else item for item in value]
+    return value
+
+
+def clean_inputs(inputs: dict[str, Any], dspy: Any, max_field_size: int) -> dict[str, Any]:
+    """Clean and prepare inputs for storage, with per-field truncation.
+
+    A signature class among the inputs is recorded by its label, not its repr
+    (``_input_value``).
+    """
     result = {}
 
     # Merge kwargs into top level
@@ -32,10 +68,10 @@ def clean_inputs(inputs: dict[str, Any], _dspy: Any, max_field_size: int) -> dic
     kwargs = inputs.get("kwargs", {})
     for k, v in inputs.items():
         if k != "kwargs":
-            serialized = models._safe_serialize(v)
+            serialized = models._safe_serialize(_input_value(v, dspy))
             result[k] = truncate_field(serialized, max_field_size, depth=1)
     for k, v in kwargs.items():
-        serialized = models._safe_serialize(v)
+        serialized = models._safe_serialize(_input_value(v, dspy))
         result[k] = truncate_field(serialized, max_field_size, depth=1)
 
     # Remove empty args

{cmpnd-0.33.2 → cmpnd-0.33.4}/cmpnd/models.py

@@ -713,8 +713,6 @@ class SpanBuilder:
     selected_demo_ids: list[str] = field(default_factory=list)
 
     # Inputs/Outputs
-    # Initialised to {} by default; thin() sets these to None when stripping
-    # ADAPTER_FORMAT / ADAPTER_PARSE spans to reduce export payload size.
     inputs: dict[str, Any] | None = field(default_factory=dict)
     outputs: dict[str, Any] | None = field(default_factory=dict)
 

{cmpnd-0.33.2 → cmpnd-0.33.4}/docs/sdk-instrumentation.md

@@ -81,7 +81,7 @@ The hook methods, in the order you'd expect them to fire for a typical `dspy.Cha
 
 - **Module lifecycle** — `on_module_start` (`callback.py:195`) creates a new `SpanBuilder`, and, if this is the root module call, also creates a new `TraceBuilder` via `_create_trace` (`callback.py:579`). `on_module_end` (`callback.py:307`) closes the span, attaches outputs, and, if it was the root, exports the trace. A call is root when no span is on the stack — or when the only span above it is the `Evaluate` span, which contains many independent program runs and parents none of them, so each run under a `dspy.Evaluate` roots its own trace. Such a root span is built with `create_child_span(..., root=True)` so it does not adopt the `Evaluate` span as its parent.
 - **LM calls.** `on_lm_start` and `on_lm_end` wrap the provider call. The callback records the model, the current span token projections, and a versioned `lm_usage` evidence object. `capture_prompts` controls whether the prompt payload is recorded.
-- **Adapter format/parse** — `on_adapter_format_start` / `_end` and `on_adapter_parse_start` / `_end`. Both route through the shared helpers `_start_adapter_span` and `_end_adapter_span`. The callback records one span per adapter call. DSPy's `Adapter.__init_subclass__` wraps `format` and `parse` on every subclass, so a `JSONAdapter.format` call fires the hooks twice and a `BAMLAdapter.format` call three times, each with a fresh `call_id` and the same instance. `_start_adapter_span` skips a call when the current span is the same method's adapter span on the same instance, and `_end_adapter_span` returns early for the skipped `call_id`. The outermost span is kept, since its output is what reaches the LM. `tests/test_adapter_reentrancy.py` runs the real adapters under DSPy's dispatch.
+- **Adapter format/parse** — `on_adapter_format_start` / `_end` and `on_adapter_parse_start` / `_end`. Both route through the shared helpers `_start_adapter_span` and `_end_adapter_span`. The callback records one span per adapter call. DSPy's `Adapter.__init_subclass__` wraps `format` and `parse` on every subclass, so a `JSONAdapter.format` call fires the hooks twice and a `BAMLAdapter.format` call three times, each with a fresh `call_id` and the same instance. `_start_adapter_span` skips a call when the current span is the same method's adapter span on the same instance, and `_end_adapter_span` returns early for the skipped `call_id`. The outermost span is kept, since its output is what reaches the LM. `tests/test_adapter_reentrancy.py` runs the real adapters under DSPy's dispatch. An adapter span's inputs are the method's arguments, so they include the predictor's `signature`. `helpers.clean_inputs` records any signature class among a span's inputs by its label, the class name or the `in1, in2 -> out1` fields of a string signature. It never records the repr, which embeds the whole instructions and every field's metadata. The same rule covers a `Predict` called with `signature=`, as DSPy's RLM does on every turn from DSPy `main` on. The span records the signature's fields and instructions once, as its own attributes.
 - **Tools** — `on_tool_start` (`callback.py:700`) / `on_tool_end` (`callback.py:733`). One span per `dspy.Tool` invocation, with the tool's args as inputs.
 - **Interpreter (code execution)** — `_InterpreterMixin` (`callback.py:1119`), for the DSPy `CodeInterpreter` hooks DSPy >= 3.3.1 fires around a program running code in the interpreter (`RLM`, `ProgramOfThought`, `CodeAct`, `Flex` — every module reaching it through `_create_interpreter`; `ReAct` / `ReActV2` use no interpreter and fire none). `on_interpreter_execute_start` / `_end` emit one `INTERPRETER_EXECUTE` span per REPL cell — the code in, its captured stdout out, and the `SUBMIT()` result (a DSPy `FinalOutput`) recorded distinctly from stdout. `on_interpreter_tool_call_start` / `_end` synthesize a `TOOL` span for a tool called from interpreter code **only when no real `dspy.Tool` span covered the call** (a user tool routes through `Tool.__call__` and fires `on_tool_*`, so its row is suppressed to avoid a duplicate; the `llm_query` builtins fire no tool hook, so they get the synthesized span). `on_interpreter_startup_*` / `on_interpreter_shutdown_*` emit no span — their duration folds into an `interpreter.startup_ms` / `interpreter.shutdown_ms` attribute on the enclosing span, and an event with no enclosing span (a pooled pre-warm) is dropped.
 - **Evaluate** — `on_evaluate_start` (`callback.py:773`) / `on_evaluate_end` (`callback.py:782`). These are thin forwarders that delegate to `EvalHandler` (see "Eval handling" below).
@@ -135,18 +135,18 @@ If you see `UNKNOWN` in the UI, `_get_span_type_for_module` couldn't match the D
 
 ## Context stack
 
-DSPy's execution is threaded: a single program call can fan out to parallel LM calls, evaluators often run workers in threads, and the callback hooks can fire on any of them. The SDK tracks "which trace/span am I currently inside" using Python's `contextvars`, which copy-on-fork across `threading.Thread` and `asyncio` tasks the way you want.
+DSPy's execution is threaded: a single program call can fan out to parallel LM calls, evaluators often run workers in threads, and the callback hooks can fire on any of them. The SDK tracks "which trace/span am I currently inside" using Python's `contextvars`. A fresh `threading.Thread` or pool thread starts with empty context; `contextvars.copy_context()` and every `asyncio` task start with a copy of the caller's.
 
 Everything lives in `python-sdk/cmpnd/context.py`:
 
 - `_current_trace` (`context.py:14`) — the active `TraceBuilder`.
-- `_span_stack` (`context.py:19`) — a list of `SpanBuilder`s. Nested spans push/pop via `push_span` (`context.py:52`) and `pop_span` (`context.py:61`). `get_current_span` returns the innermost (`context.py:44`).
+- `_span_stack` — an immutable tuple of `SpanBuilder`s. `push_span` and `pop_span` set a new tuple rather than mutating the old one, because a context copy shares the objects its parent context holds: a mutable stack would be one list shared by every sibling copy, so concurrent runs (GEPA's `DspyAdapter.batch_evaluate`, which runs each candidate in a `copy_context()`, or `asyncio.gather` over `acall`) would read each other's spans as their parent. `get_current_span` returns the innermost. `tests/test_trace_context_concurrency.py` pins this.
 - `_current_eval_run` (`context.py:24`) — set while an `EvalHandler` is active.
 - `_current_optimization_run_id` (`context.py:29`) — set while a patched `GEPA.compile` is running, so evals triggered inside it can be linked back.
 
 `create_child_span` at `context.py:83` is the standard constructor: it reads the current trace + span, assigns `trace_id` and `parent_span_id`, and returns an unpushed `SpanBuilder`. The callback uses it for every child span; so does the `start_span` context manager below. Pass `root=True` to have it ignore the stack and leave `parent_span_id` unset, for a span that starts a trace beneath one that does not parent it.
 
-The important consequence: if DSPy spawns a worker thread the _wrong_ way (e.g. via `os.fork` or by passing work to a pool that doesn't inherit context), spans from that thread will get orphaned `trace_id`s. Almost all of DSPy's internal threading uses `concurrent.futures`, which inherits context vars, so in practice this just works.
+The important consequence: a span's `trace_id` comes from `_current_trace` and its `parent_span_id` from `_span_stack`, so the two stay consistent only while both are read from the same context. A thread that starts with empty context (DSPy's `ParallelExecutor` workers) roots its own traces, which is what `Evaluate` wants. A copied context starts inside whatever trace and span the caller had open.
 
 The inverse also has to be designed for, and is easier to miss: work that stays on the *calling* thread inherits context the SDK may not want it to. `dspy.Evaluate(num_threads=1)` runs every example inline rather than on a worker, so the `Evaluate` span is still on the stack when each program run begins — which is why the root rule above names that span explicitly instead of relying on a worker thread's fresh contextvars to hide it. `tests/test_eval_trace_grouping.py` runs a real `Evaluate` over both paths to keep them from drifting.
 
@@ -709,7 +709,9 @@ Every LM span carries one `LMUsageObservationV1`. The observation preserves the
 
 The queryable counters use `None` for an absent measurement and `0` for a reported zero. The legacy span counters keep their established defaults for compatibility. The evidence object also records requested and reported models, requested and effective service tiers, one unambiguous requested cache TTL, DSPy client-cache status, provider response identity, and an optional hosted egress call ID.
 
-The callback mints `source_observation_id` when the LM call starts and records `observed_at` when it ends. Export retries serialize the same observation, including the same identity, timestamp, and SHA-256 payload digest. A DSPy client-cache hit remains an SDK observation, but `usage_reported` is false because no provider call occurred.
+The callback mints `source_observation_id` when the LM call starts and records `observed_at` when it ends. Export retries serialize the same observation, including the same identity, timestamp, and SHA-256 payload digest. A DSPy client-cache hit remains an SDK observation, but `usage_reported` is false because no provider call occurred. An empty usage object also leaves `usage_reported` false, since it reports no count. On the LiteLLM path DSPy stamps `cache_hit` on the cached response, and `dspy_client_cache_hit` reads it there. An lm15-native call on DSPy 3.4 carries no such flag, so the SDK infers the hit from its shape: DSPy empties the history entry's `usage` while the cached `Response` keeps its token counts, which a fresh call never does (`_lm15_cache_hit` in `python-sdk/cmpnd/callback.py`).
+
+The LM span's `lm.finish_reason` attribute records why the model stopped, in the chat vocabulary serve's truncation warning reads. Each response shape spells it differently. Chat puts it on the first choice and DSPy 3.3's typed `LMResponse` on the first output. The lm15 `Response` a DSPy 3.4 engine returns carries it at the top level and names a tool-call stop `tool_call`, which the SDK records as `tool_calls`. The Responses API reports a `status`, and its `max_output_tokens` cutoff lands as `length`. `_finish_reason` in `python-sdk/cmpnd/callback.py` holds the mapping.
 
 `provider_cost` stays `None`. DSPy and LiteLLM can attach a calculated cost to history, but that value is not provider evidence. The SDK will populate this field only when an adapter can prove that the provider supplied both the amount and its currency.
 

{cmpnd-0.33.2 → cmpnd-0.33.4}/pyproject.toml

@@ -5,7 +5,7 @@ name = "cmpnd"
 # it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
 # hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
 # this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
-version = "0.33.2"
+version = "0.33.4"
 description = "DSPy observability and deployment SDK for cmpnd"
 readme = "README.md"
 requires-python = ">=3.10"

{cmpnd-0.33.2 → cmpnd-0.33.4}/tests/_traced_run.py

@@ -10,6 +10,7 @@ callback hands to its exporter, whether it streams them or batches them.
 
 from __future__ import annotations
 
+import inspect
 import json
 from typing import Any
 from unittest import mock
@@ -51,7 +52,7 @@ class CaptureExporter:
 
 
 def make_callback(**kwargs: Any) -> Any:
-    with mock.patch("cmpnd.exporter.ensure_exporter_started"):
+    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
         from cmpnd.callback import CmpndCallback
 
         return CmpndCallback(**kwargs)
@@ -62,7 +63,7 @@ def run(program: Any, lm: Any, adapter: Any, callback: Any = None, **inputs: Any
     exp = CaptureExporter()
     exp.lm = lm
     cb = callback or make_callback()
-    with mock.patch("cmpnd.exporter.get_exporter", return_value=exp):
+    with mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=exp):
         with dspy.context(lm=lm, adapter=adapter, callbacks=[cb]):
             result = program(**inputs)
     return result, exp
@@ -81,8 +82,17 @@ def run_rlm(
     rlm = dspy.RLM("question -> answer", max_iters=turns)
     adapter = adapter or dspy.ChatAdapter()
     interpreter = FakeInterpreter(lambda t: interpreter_output(t, output_size))
+    # DSPy 3.4 renamed RLM.forward's caller-owned interpreter parameter from
+    # positional ``interpreter`` to keyword-only ``interpreter_factory``.
+    new_rlm_api = "interpreter_factory" in inspect.signature(getattr(rlm, "forward")).parameters
+
+    def _call_rlm(**kw: Any) -> Any:
+        if new_rlm_api:
+            return rlm(interpreter_factory=lambda: interpreter, **kw)
+        return rlm(interpreter, **kw)
+
     return run(
-        lambda **kw: rlm(interpreter, **kw),
+        _call_rlm,
         DummyLM(rlm_answers(turns), adapter=adapter),
         adapter,
         callback=callback,

cmpnd-0.33.4/tests/conftest.py

@@ -0,0 +1,8 @@
+"""Shared pytest configuration for the cmpnd SDK test suite."""
+
+from __future__ import annotations
+
+# litellm imports openai._models during test collection. Initialize openai
+# first to avoid the circular import in openai 1.109.1 when that module loads
+# before openai's own __init__ has finished.
+import openai  # noqa: F401  (imported for its import-order side effect)

{cmpnd-0.33.2 → cmpnd-0.33.4}/tests/load/rlm_size_report.py

@@ -24,6 +24,7 @@ span, is not counted.
 from __future__ import annotations
 
 import argparse
+import inspect
 import json
 import os
 import subprocess
@@ -130,7 +131,7 @@ def run_rlm(turns: int, output_size: int, adapter: str = "chat") -> list[dict[st
     from cmpnd.configuration import configure
 
     configure(api_key="size-report", endpoint="http://localhost:8000")
-    with mock.patch("cmpnd.exporter.ensure_exporter_started"):
+    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
         from cmpnd.callback import CmpndCallback
 
         callback = CmpndCallback()
@@ -144,9 +145,14 @@ def run_rlm(turns: int, output_size: int, adapter: str = "chat") -> list[dict[st
 
     interpreter = FakeInterpreter(output)
     capture = _Capture()
-    with mock.patch("cmpnd.exporter.get_exporter", return_value=capture):
+    # DSPy 3.4 takes a factory instead of the positional interpreter.
+    new_rlm_api = "interpreter_factory" in inspect.signature(getattr(rlm, "forward")).parameters
+    with mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=capture):
         with dspy.context(lm=lm, adapter=adapter_instance, callbacks=[callback]):
-            rlm(interpreter, question="Which rows changed?")
+            if new_rlm_api:
+                rlm(interpreter_factory=lambda: interpreter, question="Which rows changed?")
+            else:
+                rlm(interpreter, question="Which rows changed?")
     return [s.to_api_dict() for s in capture.spans]
 
 

cmpnd-0.33.4/tests/test_lm15_native_calls.py

@@ -0,0 +1,157 @@
+"""LM calls DSPy answers through its lm15 engines, driven end to end.
+
+DSPy 3.4 runs an LM call through an engine that returns an lm15 ``Response``,
+and the history entry the callback reads holds that ``Response`` rather than a
+provider's chat or Responses object. The ``Response`` is a slots dataclass with
+a top-level ``finish_reason`` in lm15's own vocabulary, and a DSPy cache hit on
+this path leaves the entry's ``usage`` empty with nothing on the entry or the
+``Response`` saying it was a hit.
+
+These drive a real ``dspy.Predict`` over DSPy's own ``DummyLM``, which on 3.4
+answers through ``DummyEngine`` and so takes the same native path a provider
+call does. No shape is hand-built: if DSPy changes what the entry holds, these
+see it.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest import mock
+
+import pytest
+
+from cmpnd.configuration import configure
+from cmpnd.context import clear_context
+
+dspy = pytest.importorskip("dspy")
+
+try:
+    from dspy.clients.engines.dummy_engine import DummyEngine
+    from dspy.lm15 import Response
+except ImportError:  # DSPy < 3.4 answers DummyLM without an lm15 engine.
+    pytest.skip("needs DSPy>=3.4 (lm15 engines)", allow_module_level=True)
+
+from dspy.utils.dummies import DummyLM  # noqa: E402
+
+from tests.test_eval_trace_grouping import RecordingExporter  # noqa: E402
+
+
+class QA(dspy.Signature):
+    """Answer the question."""
+
+    question: str = dspy.InputField()
+    answer: str = dspy.OutputField()
+
+
+@pytest.fixture(autouse=True)
+def _sdk_configured():
+    configure(api_key="test_key", endpoint="http://localhost:8000")
+    yield
+    clear_context()
+    import cmpnd.configuration
+    from cmpnd.exporter import shutdown_exporter
+
+    cmpnd.configuration._config = None
+    shutdown_exporter()
+
+
+@pytest.fixture
+def memory_cache():
+    """DSPy's cache in memory only, empty, and restored afterwards."""
+    from dspy.clients import configure_cache
+
+    saved = dspy.cache
+    configure_cache(enable_disk_cache=False, enable_memory_cache=True)
+    yield
+    dspy.cache = saved
+
+
+class _FinishingEngine(DummyEngine):
+    """``DummyEngine`` answering with a chosen lm15 finish reason."""
+
+    def __init__(self, owner: Any, finish_reason: str) -> None:
+        super().__init__(owner)
+        self._finish_reason = finish_reason
+
+    def _complete_messages(self, messages: list[dict[str, Any]]) -> Response:
+        response = super()._complete_messages(messages)
+        return Response(
+            id=response.id,
+            model=response.model,
+            message=response.message,
+            finish_reason=self._finish_reason,
+            usage=response.usage,
+        )
+
+
+def _lm_spans(lm: Any, calls: int = 1) -> list[Any]:
+    """Run ``Predict(QA)`` ``calls`` times on ``lm``; return every LM span, in order."""
+    from cmpnd.callback import CmpndCallback
+
+    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
+        callback = CmpndCallback()
+    recorder = RecordingExporter()
+
+    with (
+        mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=recorder),
+        dspy.context(lm=lm, callbacks=[callback]),
+    ):
+        for _ in range(calls):
+            dspy.Predict(QA)(question="What is the capital of France?")
+
+    spans = [s for t in recorder.traces for s in (getattr(t, "_batch_spans", None) or [])] + recorder.spans
+    lm_spans = [s for s in spans if s.span_type.value == "lm_call"]
+    assert len(lm_spans) == calls, f"expected {calls} LM spans, got {[s.name for s in spans]}"
+    return sorted(lm_spans, key=lambda s: s.start_time)
+
+
+def test_history_holds_an_lm15_response():
+    """The premise: DummyLM's history entry carries the native ``Response``."""
+    lm = DummyLM([{"answer": "Paris"}])
+    _lm_spans(lm)
+    assert isinstance(lm.history[-1]["response"], Response)
+
+
+@pytest.mark.parametrize(
+    ("native", "recorded"),
+    [
+        ("stop", "stop"),
+        ("length", "length"),
+        # lm15 names a tool-call stop in the singular; serve reads the chat word.
+        ("tool_call", "tool_calls"),
+        ("content_filter", "content_filter"),
+    ],
+)
+def test_finish_reason_on_a_native_call(native: str, recorded: str):
+    lm = DummyLM([{"answer": "Paris"}])
+    lm._engine_spec = _FinishingEngine(lm, native)
+
+    (span,) = _lm_spans(lm)
+    assert span.attributes.get("lm.finish_reason") == recorded
+
+
+def test_a_dspy_cache_hit_reports_no_usage(memory_cache: None):
+    """A repeated identical call is answered from DSPy's cache.
+
+    The provider was not called, so there is no usage to report. The hit's
+    usage is unreported rather than zero, and the observation says DSPy's
+    cache answered.
+    """
+    lm = DummyLM({"capital of France": {"answer": "Paris"}})
+    # DummyLM opts out of DSPy's cache so a scripted list is consumed in order.
+    # This test is about the cache, so it opts back in.
+    lm._cache_responses = True
+
+    first, second = _lm_spans(lm, calls=2)
+
+    assert lm.history[-1]["usage"] == {}, "premise: DSPy answered the second call from its cache"
+    fresh = first.to_api_dict()["lm_usage"]
+    hit = second.to_api_dict()["lm_usage"]
+    assert fresh["usage_reported"] is True
+    assert fresh["dspy_client_cache_hit"] is None
+    assert hit["usage_reported"] is False
+    assert hit["usage_raw"] is None
+    assert hit["prompt_tokens"] is None
+    assert hit["dspy_client_cache_hit"] is True
+    # The cached answer still says why the model stopped.
+    assert second.attributes.get("lm.finish_reason") == "stop"

{cmpnd-0.33.2 → cmpnd-0.33.4}/tests/test_lm_usage_shapes.py

@@ -39,7 +39,8 @@ def _typed_lm_types_available() -> bool:
 
     ``dspy.core.types`` (``LMUsage``, ``LMResponse``) arrived in DSPy 3.3.0.
     Measured across the versions ``.github/workflows/ci-sdk.yml`` pins: absent on
-    3.1.3, 3.2.0 and 3.2.1, present on 3.3.0 and 3.3.1. Probed by import rather
+    3.1.3, 3.2.0 and 3.2.1, present on 3.3.0 and 3.3.1, and removed in 3.4.0b1,
+    whose ``dspy.core.types`` raises ``ImportError``. Probed by import rather
     than by version number, so the answer stays right whatever the matrix pins
     next.
 
@@ -98,7 +99,7 @@ def _span_for(usage: Any, response: Any = None) -> Any:
     """Drive on_lm_start/on_lm_end over one history entry; return the span."""
     from cmpnd.callback import CmpndCallback
 
-    with mock.patch("cmpnd.exporter.ensure_exporter_started"):
+    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
         callback = CmpndCallback()
     recorder = _RecordingExporter()
 
@@ -106,7 +107,7 @@ def _span_for(usage: Any, response: Any = None) -> Any:
     if response is not None:
         entry["response"] = response
 
-    with mock.patch("cmpnd.exporter.get_exporter", return_value=recorder):
+    with mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=recorder):
         callback.on_lm_start(call_id="lm1", instance=_lm([entry]), inputs={})
         callback.on_lm_end(call_id="lm1", outputs=None, exception=None)
 
@@ -118,13 +119,13 @@ def _span_for_entry(entry: Any, *, lm_kwargs: dict[str, Any] | None = None) -> A
     """Drive one complete history entry through the callback."""
     from cmpnd.callback import CmpndCallback
 
-    with mock.patch("cmpnd.exporter.ensure_exporter_started"):
+    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
         callback = CmpndCallback()
     recorder = _RecordingExporter()
     lm = _lm([entry])
     lm.kwargs = lm_kwargs or {}
 
-    with mock.patch("cmpnd.exporter.get_exporter", return_value=recorder):
+    with mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=recorder):
         callback.on_lm_start(call_id="sdk-call-1", instance=lm, inputs={})
         callback.on_lm_end(call_id="sdk-call-1", outputs=None, exception=None)
 
@@ -471,6 +472,18 @@ def test_dspy_cache_hit_is_not_provider_reported_usage():
     assert observation["usage_raw"] is None
 
 
+def test_an_empty_usage_object_is_not_reported_usage():
+    """An empty mapping reports nothing, unlike a mapping of zeros.
+
+    DSPy >= 3.4 writes ``usage={}`` for a call its cache answered, with nothing
+    else on this entry to say so, so the empty object is all the callback sees.
+    """
+    observation = _span_for_entry({"usage": {}}).to_api_dict()["lm_usage"]
+    assert observation["usage_reported"] is False
+    assert observation["usage_raw"] is None
+    assert observation["dspy_client_cache_hit"] is None
+
+
 @_needs_typed_lm_types
 def test_typed_history_preserves_response_usage_and_request_extensions():
     from dspy.core.types import (
@@ -580,7 +593,7 @@ def test_prediction_usage_aggregates_on_every_shape(shape: str):
 
     from cmpnd.callback import CmpndCallback
 
-    with mock.patch("cmpnd.exporter.ensure_exporter_started"):
+    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
         callback = CmpndCallback()
 
     span = mock.MagicMock()
@@ -608,7 +621,7 @@ def test_prediction_usage_sums_across_models():
 
     from cmpnd.callback import CmpndCallback
 
-    with mock.patch("cmpnd.exporter.ensure_exporter_started"):
+    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
         callback = CmpndCallback()
 
     span = mock.MagicMock()