cmpnd 0.86.0__tar.gz → 0.86.1__tar.gz

{cmpnd-0.86.0 → cmpnd-0.86.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cmpnd
-Version: 0.86.0
+Version: 0.86.1
 Summary: DSPy observability and deployment SDK for cmpnd
 Project-URL: Homepage, https://cmpnd.ai
 Author-email: cmpnd <hello@cmpnd.ai>

{cmpnd-0.86.0 → cmpnd-0.86.1}/cmpnd/callback.py

@@ -1052,8 +1052,10 @@ class _ModuleMixin(_CallbackBase):
                 cleaned_outputs = helpers.clean_outputs(captured_outputs, instance, dspy, self._max_output_field_size)
                 span.set_outputs(cleaned_outputs)
 
-                # Extract token usage if available
-                self._extract_usage_from_prediction(span, captured_outputs, dspy)
+        # A module span records no token usage. Its LM call spans carry each
+        # call's usage, and a trace's totals sum every span, so usage copied
+        # from the Prediction here would count each call once more per
+        # enclosing module.
 
         # Pop from context stack
         context.pop_span()
@@ -1462,32 +1464,6 @@ class _ModuleMixin(_CallbackBase):
             trace.tags["optimization_phase"] = phase
         return trace
 
-    def _extract_usage_from_prediction(self, span: models.SpanBuilder, outputs: Any, dspy: Any) -> None:
-        """Sum the LM usage a DSPy Prediction carries onto its module span.
-
-        This is what fills the trace row's own token totals, so a shape read
-        wrongly here reports a trace as having spent no tokens even when the LM
-        spans below it carry the right counts.
-        """
-        if not isinstance(outputs, dspy.Prediction):
-            return
-        usage_by_model = outputs.get_lm_usage() if hasattr(outputs, "get_lm_usage") else None
-        if not usage_by_model:
-            return
-        for model_usage in usage_by_model.values():
-            counts = _normalize_usage(model_usage)
-            span.prompt_tokens += counts["prompt_tokens"] or 0
-            span.completion_tokens += counts["completion_tokens"] or 0
-            span.total_tokens += counts["total_tokens"] or 0
-            span.cached_prompt_tokens += counts["cached_prompt_tokens"] or 0
-            # Summed only over the models that reported one, so a span whose
-            # models all stayed silent keeps the field absent rather than 0.
-            for name in ("reasoning_tokens", "cache_write_tokens"):
-                reported = counts[name]
-                if reported is None:
-                    continue
-                setattr(span, name, (getattr(span, name) or 0) + reported)
-
 
 class _LMMixin(_CallbackBase):
     """DSPy LM callbacks (on_lm_start/on_lm_end)."""

{cmpnd-0.86.0 → cmpnd-0.86.1}/docs/sdk-instrumentation.md

@@ -115,7 +115,7 @@ bindings, the judge picks one through `lms[name]`. A `TypeSafe` object passed in
 The hook methods, in the order you'd expect them to fire for a typical `dspy.ChainOfThought` call:
 
 - **Module lifecycle** — `on_module_start` (`callback.py`) creates a new `SpanBuilder`, and, if this is the root module call, also creates a new `TraceBuilder` via `_create_trace`. `on_module_end` closes the span, attaches outputs, and, if it was the root, exports the trace. A call is root when no span is on the stack — or when the only span above it is the `Evaluate` span, which contains many independent program runs and parents none of them, so each run under a `dspy.Evaluate` roots its own trace. Such a root span is built with `create_child_span(..., root=True)` so it does not adopt the `Evaluate` span as its parent.
-- **LM calls.** `on_lm_start` and `on_lm_end` wrap the provider call. The callback records the model, the current span token projections, and a versioned `lm_usage` evidence object. `capture_prompts` controls whether the prompt payload is recorded.
+- **LM calls.** `on_lm_start` and `on_lm_end` wrap the provider call. The callback records the model, the LM call span's token counts, and a versioned `lm_usage` evidence object. `capture_prompts` controls whether the prompt payload is recorded. Only an LM call span carries token counts. A module span records none, even when DSPy's `track_usage` puts the usage on its `Prediction`, because the SDK's trace totals and serve's trace fold both add up every span, and a module span holding its calls' usage would count each call again for every module that encloses it (`tests/test_token_attribution.py`).
 - **Adapter format/parse** — `on_adapter_format_start` / `_end` and `on_adapter_parse_start` / `_end`. Both route through the shared helpers `_start_adapter_span` and `_end_adapter_span`. The callback records one span per adapter call. DSPy's `Adapter.__init_subclass__` wraps `format` and `parse` on every subclass, so a `JSONAdapter.format` call fires the hooks twice and a `BAMLAdapter.format` call three times, each with a fresh `call_id` and the same instance. `_start_adapter_span` skips a call when the current span is the same method's adapter span on the same instance, and `_end_adapter_span` returns early for the skipped `call_id`. The outermost span is kept, since its output is what reaches the LM. `tests/test_adapter_reentrancy.py` runs the real adapters under DSPy's dispatch. An adapter span's inputs are the method's arguments, so they include the predictor's `signature`. `helpers.clean_inputs` records any signature class among a span's inputs by its label, the class name or the `in1, in2 -> out1` fields of a string signature. It never records the repr, which embeds the whole instructions and every field's metadata. The same rule covers a `Predict` called with `signature=`, as DSPy's RLM does on every turn from DSPy `main` on. The span records the signature's fields and instructions once, as its own attributes.
 - **Tools** — `on_tool_start` / `on_tool_end` (`callback.py`). One span per `dspy.Tool` invocation, with the tool's args as inputs.
 - **Interpreter (code execution)** — `_InterpreterMixin` (`callback.py`), for the DSPy `CodeInterpreter` hooks DSPy >= 3.3.1 fires around a program running code in the interpreter (`RLM`, `ProgramOfThought`, `CodeAct`, `Flex` — every module reaching it through `_create_interpreter`; `ReAct` / `ReActV2` use no interpreter and fire none). `on_interpreter_execute_start` / `_end` emit one `INTERPRETER_EXECUTE` span per REPL cell — the code in, its captured stdout out, and the `SUBMIT()` result (a DSPy `FinalOutput`) recorded distinctly from stdout. On DSPy 3.4.1 and later, `RLM` runs the sandbox side of its `dspy` stand-in as each session's first cell, marked by the `__dspy_host_token` variable (`dspy.primitives.facade.HOST_TOKEN_VAR`). That cell is DSPy's machinery rather than a step the program took, so it emits no span. `on_interpreter_tool_call_start` / `_end` synthesize a `TOOL` span for a tool called from interpreter code **only when no real `dspy.Tool` span covered the call** (a user tool routes through `Tool.__call__` and fires `on_tool_*`, so its row is suppressed to avoid a duplicate; the `llm_query` builtins fire no tool hook, so they get the synthesized span). `on_interpreter_startup_*` / `on_interpreter_shutdown_*` emit no span — their duration folds into an `interpreter.startup_ms` / `interpreter.shutdown_ms` attribute on the enclosing span, and an event with no enclosing span (a pooled pre-warm) is dropped.
@@ -840,7 +840,7 @@ A span's dict looks roughly like this:
     "span_id": "...",
     "trace_id": "...",
     "parent_span_id": "...",
-    "span_type": "predict",
+    "span_type": "lm_call",
     "start_time": "2020-01-02T00:00:00+00:00",
     "status": "ok",
     "project_tags": [],

{cmpnd-0.86.0 → cmpnd-0.86.1}/pyproject.toml

@@ -5,7 +5,7 @@ name = "cmpnd"
 # it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
 # hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
 # this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
-version = "0.86.0"
+version = "0.86.1"
 description = "DSPy observability and deployment SDK for cmpnd"
 readme = "README.md"
 requires-python = ">=3.11"

{cmpnd-0.86.0 → cmpnd-0.86.1}/tests/test_lm_usage_shapes.py

@@ -575,73 +575,3 @@ def test_finish_reason_from_typed_response():
 def test_finish_reason_absent_when_nothing_reports_one():
     span = _span_for(_responses_usage(), response=_ResponsesResponse("in_progress"))
     assert span.attributes.get("lm.finish_reason") == "in_progress"
-
-
-# --- Module-level aggregation (the trace's own totals) ---------------------
-
-
-@pytest.mark.parametrize("shape", _SHAPE_PARAMS)
-def test_prediction_usage_aggregates_on_every_shape(shape: str):
-    """``Prediction.get_lm_usage()`` feeds the module span, and so the trace row.
-
-    A trace reporting no prompt or completion tokens at all, while its LM spans
-    below carry counts, is this path reading zeros one level above them.
-    """
-    import dspy
-
-    from cmpnd.callback import CmpndCallback
-
-    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
-        callback = CmpndCallback()
-
-    span = mock.MagicMock()
-    span.prompt_tokens = 0
-    span.completion_tokens = 0
-    span.total_tokens = 0
-    span.cached_prompt_tokens = 0
-    span.reasoning_tokens = None
-    span.cache_write_tokens = None
-
-    prediction = dspy.Prediction(answer="Paris")
-    prediction.set_lm_usage({"openai/gpt-5.6-luna": _SHAPES[shape]()})
-
-    callback._extract_usage_from_prediction(span, prediction, dspy)
-
-    assert span.prompt_tokens == 2900
-    assert span.completion_tokens == 17
-    assert span.total_tokens == 2917
-    assert span.cached_prompt_tokens == 1024
-    assert span.reasoning_tokens == 12
-
-
-def test_prediction_usage_sums_across_models():
-    import dspy
-
-    from cmpnd.callback import CmpndCallback
-
-    with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
-        callback = CmpndCallback()
-
-    span = mock.MagicMock()
-    span.prompt_tokens = 0
-    span.completion_tokens = 0
-    span.total_tokens = 0
-    span.cached_prompt_tokens = 0
-    span.reasoning_tokens = None
-    span.cache_write_tokens = None
-
-    prediction = dspy.Prediction(answer="Paris")
-    prediction.set_lm_usage(
-        {
-            "openai/gpt-5.6-luna": _chat_usage(),
-            "anthropic/claude-sonnet-4": _responses_usage(),
-        }
-    )
-
-    callback._extract_usage_from_prediction(span, prediction, dspy)
-
-    assert span.prompt_tokens == 5800
-    assert span.completion_tokens == 34
-    assert span.reasoning_tokens == 24
-    # Nothing reported a cache write, so the sum stays absent rather than 0.
-    assert span.cache_write_tokens is None

cmpnd-0.86.1/tests/test_token_attribution.py

@@ -0,0 +1,122 @@
+"""Each LM call's tokens land on exactly one span: its own LM call span.
+
+A trace's token total is the sum over its spans, both in the SDK's own trace
+roll-up and in serve's trace fold. A module span that also carried the usage of
+the LM calls beneath it would count every call once per enclosing module.
+
+These tests run real DSPy modules over the in-process fake provider, with DSPy's
+``track_usage`` on so each ``Prediction`` reports its LM usage, and check what
+the callback hands the exporter.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest import mock
+
+import pytest
+
+dspy = pytest.importorskip("dspy")
+litellm = pytest.importorskip("litellm")
+
+from cmpnd import configure, shutdown_exporter  # noqa: E402
+from cmpnd.callback import CmpndCallback  # noqa: E402
+from cmpnd.context import clear_context  # noqa: E402
+from cmpnd.models import SpanType  # noqa: E402
+from tests.e2e import fake_lm  # noqa: E402
+
+
+class _RecordingExporter:
+    def __init__(self) -> None:
+        self.traces: list[Any] = []
+
+    def export_trace_start(self, trace: Any) -> None:
+        pass
+
+    def export_trace_finalize(self, trace: Any) -> None:
+        self.traces.append(trace)
+
+    def export_span(self, span: Any) -> None:
+        pass
+
+    def export_trace(self, trace: Any) -> None:
+        self.traces.append(trace)
+
+
+@pytest.fixture
+def traced():
+    """Run a callable under the cmpnd callback and return the exported trace."""
+    saved_providers = list(litellm.custom_provider_map)
+    fake_lm.register()
+    configure(api_key="test_key")
+    exporter = _RecordingExporter()
+
+    def run(program: Any, **kwargs: Any) -> Any:
+        with (
+            mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=exporter),
+            mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True),
+        ):
+            callback = CmpndCallback()
+            lm = dspy.LM(fake_lm.FAKE_MODEL, cache=False)
+            with dspy.context(lm=lm, callbacks=[callback], track_usage=True):
+                prediction = program(**kwargs)
+        # The fixture only matters if DSPy really reported usage on the Prediction.
+        assert prediction.get_lm_usage(), "track_usage did not attach LM usage"
+        assert len(exporter.traces) == 1
+        return exporter.traces[0]
+
+    yield run
+
+    litellm.custom_provider_map = saved_providers
+    clear_context()
+    import cmpnd.configuration
+
+    cmpnd.configuration._config = None
+    shutdown_exporter()
+
+
+def _spans(trace: Any) -> list[Any]:
+    return list(trace._batch_spans)
+
+
+def _lm_spans(trace: Any) -> list[Any]:
+    return [s for s in _spans(trace) if s.span_type == SpanType.LM_CALL]
+
+
+def _assert_tokens_only_on_lm_calls(trace: Any) -> None:
+    for span in _spans(trace):
+        if span.span_type == SpanType.LM_CALL:
+            continue
+        assert (span.prompt_tokens, span.completion_tokens, span.total_tokens, span.cached_prompt_tokens) == (
+            0,
+            0,
+            0,
+            0,
+        ), f"{span.span_type.value} span carries tokens"
+        assert span.reasoning_tokens is None
+        assert span.cache_write_tokens is None
+
+
+def test_module_over_one_lm_call_counts_the_call_once(traced):
+    trace = traced(dspy.Predict("text -> sentiment"), text="I love it")
+
+    [lm_span] = _lm_spans(trace)
+    assert lm_span.prompt_tokens > 0
+    assert lm_span.completion_tokens > 0
+    _assert_tokens_only_on_lm_calls(trace)
+    assert trace.prompt_tokens == lm_span.prompt_tokens
+    assert trace.completion_tokens == lm_span.completion_tokens
+    assert trace.total_tokens == lm_span.total_tokens
+
+
+def test_nested_modules_count_each_call_once(traced):
+    """ChainOfThought wraps a Predict: two module spans over one LM call."""
+    trace = traced(dspy.ChainOfThought("question -> answer"), question="What is 2 + 2?")
+
+    lm_spans = _lm_spans(trace)
+    assert len(lm_spans) == 1
+    assert len([s for s in _spans(trace) if s.span_type != SpanType.LM_CALL]) >= 2
+    _assert_tokens_only_on_lm_calls(trace)
+    assert trace.prompt_tokens == sum(s.prompt_tokens for s in lm_spans)
+    assert trace.completion_tokens == sum(s.completion_tokens for s in lm_spans)
+    assert trace.total_tokens == sum(s.total_tokens for s in lm_spans)

{cmpnd-0.86.0 → cmpnd-0.86.1}/uv.lock

@@ -505,7 +505,7 @@ wheels = [
 
 [[package]]
 name = "cmpnd"
-version = "0.86.0"
+version = "0.86.1"
 source = { editable = "." }
 default-groups = ["dspy-3-4-1"]
 dependencies = [