cmpnd 0.4.1__tar.gz → 0.4.2__tar.gz

{cmpnd-0.4.1 → cmpnd-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cmpnd
-Version: 0.4.1
+Version: 0.4.2
 Summary: DSPy observability and deployment SDK for cmpnd
 Project-URL: Homepage, https://cmpnd.ai
 Author-email: cmpnd <hello@cmpnd.ai>

{cmpnd-0.4.1 → cmpnd-0.4.2}/cmpnd/__init__.py

@@ -136,4 +136,10 @@ def auto_instrument() -> CmpndCallback:
 
     _program_patch.patch_program_state()
 
+    # Emit tool spans for dspy.RLM tool calls, which otherwise bypass
+    # dspy.Tool.__call__'s callbacks and produce no spans (CMP-187).
+    from . import _rlm_patch
+
+    _rlm_patch.patch_rlm_tools()
+
     return callback

cmpnd-0.4.2/cmpnd/_rlm_patch.py

@@ -0,0 +1,134 @@
+"""Emit tool spans for ``dspy.RLM`` tool calls (CMP-187).
+
+cmpnd creates a ``TOOL`` span only from DSPy's ``on_tool_start``/``on_tool_end``
+callbacks, and those fire only through ``dspy.Tool.__call__`` (decorated
+``@with_callbacks``). ``dspy.RLM._prepare_execution_tools`` hands the code
+interpreter the **bare** ``tool.func`` for each user tool
+(``{name: tool.func for name, tool in self._user_tools.items()}`` in
+``dspy/predict/rlm.py``), so ``Tool.__call__`` is bypassed and no span is
+created — unlike ``dspy.ReAct``, which calls the ``Tool`` object and gets tool
+spans for free.
+
+``patch_rlm_tools()`` wraps ``_prepare_execution_tools`` so each user tool's
+func is replaced by a shim that dispatches the tool lifecycle to the active
+``CmpndCallback``\\ (s) around the original func call. Routing through the
+callback (rather than through ``Tool.__call__``) keeps execution identical to
+today's bare-func path — no re-validation / pydantic coercion of the
+interpreter's native kwargs — while still emitting the span.
+
+``PythonInterpreter.execute`` dispatches tool calls inline on the calling
+(forward) thread, so the shim runs with cmpnd's ``_span_stack`` contextvar
+intact: the tool span nests under ``RLM.forward`` and any ``dspy.Predict`` run
+inside the tool re-parents under it.
+
+Signature preservation is mandatory: ``PythonInterpreter._extract_parameters``
+builds its sandbox tool stub from ``inspect.signature(fn)``. ``functools.wraps``
+sets ``__wrapped__``, which ``inspect.signature`` follows, so the real params
+survive the wrap (guarded by a test).
+
+Scope is user tools only; the ``llm_query`` / ``llm_query_batched`` builtins
+share the gap but ``llm_query_batched`` fans out over its own thread pool (a
+separate context-propagation problem), so they're left out here.
+
+Installed lazily from ``cmpnd.auto_instrument()`` alongside ``patch_gepa()`` /
+``patch_program_state()``. Idempotent; a no-op when dspy isn't installed.
+"""
+
+from __future__ import annotations
+
+import functools
+import uuid
+from typing import Any, Callable
+
+_patched = False
+
+
+def _active_cmpnd_callback() -> Any | None:
+    """The single registered CmpndCallback (or None).
+
+    cmpnd is built around exactly one callback — auto_instrument() refuses to
+    add a second — and the callbacks share one span-stack contextvar. So we
+    dispatch the tool lifecycle to just the first CmpndCallback: pushing a span
+    per callback would nest one tool span under another for a single tool call
+    and desync on_tool_end's pop from the stack.
+    """
+    try:
+        import dspy
+    except ImportError:
+        return None
+    from . import callback
+
+    registered = dspy.settings.get("callbacks", []) or []
+    for cb in registered:
+        if isinstance(cb, callback.CmpndCallback):
+            return cb
+    return None
+
+
+def _wrap_tool_func(tool_obj: Any) -> Callable[..., Any]:
+    """Wrap a dspy.Tool's func so a call emits a TOOL span around it.
+
+    The wrapper carries the original func's signature (``functools.wraps`` sets
+    ``__wrapped__``) so the interpreter's sandbox stub sees the real params. It
+    dispatches to the active cmpnd callback only; with none registered it's a
+    transparent passthrough.
+    """
+    func = tool_obj.func
+
+    @functools.wraps(func)
+    def shim(**kwargs: Any) -> Any:
+        cb = _active_cmpnd_callback()
+        if cb is None:
+            return func(**kwargs)
+
+        # on_tool_start (which creates + pushes the span) must succeed before we
+        # enter the try; on_tool_end is a no-op for an unknown call_id, so a
+        # failed start can't leave the span stack unbalanced.
+        call_id = uuid.uuid4().hex
+        cb.on_tool_start(call_id, tool_obj, kwargs)
+        output = None
+        exception = None
+        try:
+            output = func(**kwargs)
+            return output
+        except Exception as exc:  # noqa: BLE001 - re-raised after closing the span
+            exception = exc
+            raise
+        finally:
+            cb.on_tool_end(call_id, output, exception)
+
+    return shim
+
+
+def patch_rlm_tools() -> None:
+    """Wrap ``RLM._prepare_execution_tools`` so user tools emit spans.
+
+    Idempotent (a module-level flag guards double-patching); a no-op when dspy
+    or ``dspy.RLM`` isn't available.
+    """
+    global _patched
+    if _patched:
+        return
+
+    try:
+        import dspy
+    except ImportError:
+        return
+
+    rlm_cls = getattr(dspy, "RLM", None)
+    if rlm_cls is None or not hasattr(rlm_cls, "_prepare_execution_tools"):
+        return
+
+    orig = getattr(rlm_cls, "_prepare_execution_tools")
+
+    @functools.wraps(orig)
+    def _prepare_execution_tools(self: Any) -> dict[str, Callable[..., Any]]:
+        tools = orig(self)
+        user_tools = getattr(self, "_user_tools", {}) or {}
+        for name, tool_obj in user_tools.items():
+            if name in tools:
+                tools[name] = _wrap_tool_func(tool_obj)
+        return tools
+
+    setattr(rlm_cls, "_prepare_execution_tools", _prepare_execution_tools)
+    _patched = True

{cmpnd-0.4.1 → cmpnd-0.4.2}/pyproject.toml

@@ -5,7 +5,7 @@ name = "cmpnd"
 # it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
 # hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
 # this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
-version = "0.4.1"
+version = "0.4.2"
 description = "DSPy observability and deployment SDK for cmpnd"
 readme = "README.md"
 requires-python = ">=3.10"

{cmpnd-0.4.1 → cmpnd-0.4.2}/scripts/seed_review_run.py

@@ -236,6 +236,19 @@ _RLM_FIXTURE = (
     pathlib.Path(__file__).resolve().parents[2] / "serve" / "scripts" / "seed_fixtures" / "rlm_district_counter.json"
 )
 
+# Two flat-replay fixtures (CMP-187), shared with the local seeder
+# (serve/scripts/seed_fixtures/traces.py make_source_categorizer_trace /
+# make_tool_kind_demo_trace). Unlike _RLM_FIXTURE (pre-digested turns), these
+# are a flat list of spans carrying lid/parent/start_ns + ingest fields, so
+# _replay_flat_fixture just remaps ids + offsets timings. Loaded lazily so a
+# missing file degrades to skipping that seed.
+_SOURCE_CATEGORIZER_FIXTURE = (
+    pathlib.Path(__file__).resolve().parents[2] / "serve" / "scripts" / "seed_fixtures" / "rlm_source_categorizer.json"
+)
+_TOOL_KIND_DEMO_FIXTURE = (
+    pathlib.Path(__file__).resolve().parents[2] / "serve" / "scripts" / "seed_fixtures" / "tool_kind_demo.json"
+)
+
 
 def _notebook_trace(now: datetime.datetime) -> dict:
     """The captured "DistrictCounter" dspy.RLM run as a review-app seed trace
@@ -378,6 +391,75 @@ def _notebook_trace(now: datetime.datetime) -> dict:
     }
 
 
+def _replay_flat_fixture(
+    fixture_path: pathlib.Path,
+    now: datetime.datetime,
+    program_name: str,
+    project_tags: list[str],
+    request_preview: str,
+    response_preview: str,
+) -> dict:
+    """Replay a flat-format fixture (list of spans with lid/parent/start_ns/
+    duration_ns + ingest fields) as a review-app seed trace. Mirrors
+    make_source_categorizer_trace / make_tool_kind_demo_trace in the local
+    seeder, parameterized so both CMP-187 fixtures share one path. Mints a
+    fresh trace_id, maps every local id to a fresh uuid (parent_span_id via
+    that map, null for the root), offsets each span's start_ns by `now` with
+    the same at() helper the other seeds use, and copies the remaining fields
+    verbatim into the ingest span shape. Returns the same trace-payload dict
+    shape as _notebook_trace.
+    """
+    fx = json.loads(fixture_path.read_text())
+    root = fx[0]
+    trace_id = str(uuid.uuid4())
+    total_ns = root["duration_ns"]
+
+    def at(ns: int) -> str:
+        return (now + datetime.timedelta(microseconds=ns / 1000)).isoformat()
+
+    id_map = {s["lid"]: str(uuid.uuid4()) for s in fx}
+    skip_keys = {"lid", "parent", "start_ns", "signature_summary"}
+
+    spans: list[dict] = []
+    total_tokens = 0
+    for s in fx:
+        parent_lid = s["parent"]
+        span: dict = {
+            "span_id": id_map[s["lid"]],
+            "trace_id": trace_id,
+            "parent_span_id": id_map[parent_lid] if parent_lid is not None else None,
+            "start_time": at(s["start_ns"]),
+            "end_time": at(s["start_ns"] + s["duration_ns"]),
+        }
+        for k, v in s.items():
+            if k not in skip_keys:
+                span[k] = v
+        if s["span_type"] == "lm_call":
+            total_tokens += s.get("total_tokens") or 0
+        spans.append(span)
+
+    return {
+        "trace_id": trace_id,
+        "start_time": at(0),
+        "end_time": at(total_ns),
+        "duration_ns": total_ns,
+        "status": "ok",
+        "program_name": program_name,  # DSPy module class → programs.module_class → sidebar badge
+        "program_version": "0.1.0",
+        "program_signature_hash": root["program_signature_hash"],
+        "signature_name": root["signature_name"],
+        "signature_input_fields": root["signature_input_fields"],
+        "signature_output_fields": root["signature_output_fields"],
+        "adapter_type": "ChatAdapter",
+        "request_preview": request_preview,
+        "response_preview": response_preview,
+        "project_tags": project_tags,
+        "total_tokens": total_tokens,
+        "span_count": len(spans),
+        "spans": spans,
+    }
+
+
 def _post(endpoint: str, path: str, payload, api_key: str):
     body = json.dumps(payload).encode("utf-8")
     req = urllib.request.Request(
@@ -494,6 +576,42 @@ def main() -> int:
         # optimization run has none. Unlinked to any optimization run.
         code, _ = _post(endpoint, "/api/v1/traces/batch", [_notebook_trace(now)], api_key)
         log.info("seeded RLM notebook trace (status %s)", code)
+
+        # 4b) Plant the two CMP-187 tool-span demo traces so every review app
+        # shows RLM tool spans (SourceCategorizer, with a vision sub-LLM nested
+        # under Tool.inspect_page) and the function-tool vs module-tool
+        # differentiation (ToolKindDemo). Each degrades to a skip if its shared
+        # fixture is absent, rather than taking the seed down.
+        tool_demos: list[dict] = []
+        if _SOURCE_CATEGORIZER_FIXTURE.exists():
+            tool_demos.append(
+                _replay_flat_fixture(
+                    _SOURCE_CATEGORIZER_FIXTURE,
+                    now,
+                    "RLM",
+                    _SEED_TAGS + ["tool-demo"],
+                    "vector_pdf source_table/glenn-pdf-p15.pdf (page 1)",
+                    "candidate_columns, precinct",
+                )
+            )
+        else:
+            log.warning("skipping SourceCategorizer seed: fixture missing at %s", _SOURCE_CATEGORIZER_FIXTURE)
+        if _TOOL_KIND_DEMO_FIXTURE.exists():
+            tool_demos.append(
+                _replay_flat_fixture(
+                    _TOOL_KIND_DEMO_FIXTURE,
+                    now,
+                    "ToolKindDemo",
+                    _SEED_TAGS + ["tool-demo"],
+                    "espresso place review (function-tool + module-tool)",
+                    "The speaker loved the new downtown espresso place and its incredible pastries.",
+                )
+            )
+        else:
+            log.warning("skipping ToolKindDemo seed: fixture missing at %s", _TOOL_KIND_DEMO_FIXTURE)
+        if tool_demos:
+            code, _ = _post(endpoint, "/api/v1/traces/batch", tool_demos, api_key)
+            log.info("seeded %d CMP-187 tool-demo trace(s) (status %s)", len(tool_demos), code)
     except Exception as exc:  # noqa: BLE001 - non-gating; review apps are ephemeral
         log.warning("review seed failed (non-gating): %s", exc)
 

{cmpnd-0.4.1 → cmpnd-0.4.2}/tests/e2e/conftest.py

@@ -32,7 +32,7 @@ from tests.e2e import fake_lm
 E2E_ENDPOINT = os.getenv("CMPND_E2E_ENDPOINT", "http://localhost:3000")
 CMPND_E2E_API_KEY = os.getenv("CMPND_E2E_API_KEY")
 CMPND_E2E_GROQ_KEY = os.getenv("CMPND_E2E_GROQ_KEY")
-E2E_MODEL = "groq/meta-llama/llama-4-scout-17b-16e-instruct"
+E2E_MODEL = "groq/llama-3.3-70b-versatile"
 
 # ---------------------------------------------------------------------------
 # Skip logic — skip entire module when env vars are missing

{cmpnd-0.4.1 → cmpnd-0.4.2}/tests/stress/test_deploy_programs.py

@@ -80,7 +80,7 @@ class TestDeployReActWikipedia:
     @pytest.fixture(scope="class")
     def deployment(self):
         lm = dspy.LM(
-            "groq/meta-llama/llama-4-scout-17b-16e-instruct",
+            "groq/llama-3.3-70b-versatile",
             api_key=PLATFORM_GROQ_KEY or "",
             max_tokens=300,
             cache=False,

{cmpnd-0.4.1 → cmpnd-0.4.2}/tests/stress/test_multi_client_stress.py

@@ -41,7 +41,7 @@ from tests._identity_helpers import program_hash_for_sig
 PLATFORM_API_KEY = os.environ.get("PLATFORM_API_KEY")
 PLATFORM_GROQ_KEY = os.environ.get("PLATFORM_GROQ_KEY")
 PLATFORM_ENDPOINT = os.environ.get("PLATFORM_ENDPOINT", "http://localhost:3000")
-STRESS_MODEL = "groq/meta-llama/llama-4-scout-17b-16e-instruct"
+STRESS_MODEL = "groq/llama-3.3-70b-versatile"
 
 NUM_THREADS = 5
 ITERATIONS_PER_RUN = 3

{cmpnd-0.4.1 → cmpnd-0.4.2}/tests/stress/test_real_provider_twins.py

@@ -34,7 +34,7 @@ from tests.stress._reliability import soft_content_check, tolerate_external_outa
 PLATFORM_API_KEY = os.environ.get("PLATFORM_API_KEY")
 PLATFORM_GROQ_KEY = os.environ.get("PLATFORM_GROQ_KEY")
 PLATFORM_ENDPOINT = os.environ.get("PLATFORM_ENDPOINT", "http://localhost:3000")
-STRESS_MODEL = "groq/meta-llama/llama-4-scout-17b-16e-instruct"
+STRESS_MODEL = "groq/llama-3.3-70b-versatile"
 
 pytestmark = pytest.mark.skipif(
     not PLATFORM_API_KEY or not PLATFORM_GROQ_KEY,

cmpnd-0.4.2/tests/test_rlm_patch.py

@@ -0,0 +1,180 @@
+"""Tests for RLM tool-span instrumentation (_rlm_patch, CMP-187).
+
+dspy.RLM hands its code interpreter the bare tool.func, bypassing
+dspy.Tool.__call__'s @with_callbacks, so no tool span is created. The patch
+wraps RLM._prepare_execution_tools so each user tool emits a TOOL span via the
+active cmpnd callback while preserving the func's signature (the interpreter
+builds its sandbox stub from inspect.signature).
+"""
+
+import inspect
+import uuid
+
+import dspy
+import pytest
+
+import cmpnd
+from cmpnd import _rlm_patch, callback, configuration, context, models
+
+
+@pytest.fixture(autouse=True)
+def _configured():
+    configuration.configure(api_key="test_key", endpoint="http://localhost:8000")
+    yield
+
+
+def _make_rlm():
+    def add_pages(first: int, last: int) -> int:
+        """Sum a page range."""
+        return sum(range(first, last + 1))
+
+    return dspy.RLM("context, query -> answer", tools=[add_pages]), add_pages
+
+
+def test_patch_wraps_prepare_execution_tools_and_is_idempotent():
+    orig = dspy.RLM._prepare_execution_tools
+    _rlm_patch._patched = False
+    _rlm_patch.patch_rlm_tools()
+    wrapped = dspy.RLM._prepare_execution_tools
+    assert wrapped is not orig
+    _rlm_patch.patch_rlm_tools()
+    assert dspy.RLM._prepare_execution_tools is wrapped
+
+
+def test_wrapped_tool_preserves_signature():
+    """The interpreter reads inspect.signature(fn) to build its sandbox stub."""
+    _rlm_patch._patched = False
+    _rlm_patch.patch_rlm_tools()
+
+    rlm, add_pages = _make_rlm()
+    tools = rlm._prepare_execution_tools()
+    shim = tools["add_pages"]
+    assert inspect.signature(shim) == inspect.signature(add_pages)
+
+
+def test_tool_span_created_and_nested_under_rlm(monkeypatch):
+    """With a callback and an RLM.forward span on the stack, invoking a wrapped
+    tool creates a TOOL span parented to the RLM span."""
+    _rlm_patch._patched = False
+    _rlm_patch.patch_rlm_tools()
+
+    tracked = []
+    cb = callback.CmpndCallback()
+    monkeypatch.setattr(cb, "_track_span", lambda span: tracked.append(span))
+    monkeypatch.setattr(dspy.settings, "callbacks", [cb], raising=False)
+
+    context.clear_context()
+    context.set_current_trace(models.TraceBuilder(trace_id=uuid.uuid4()))
+    rlm_span = context.create_child_span("RLM.forward", models.SpanType.MODULE)
+    context.push_span(rlm_span)
+    try:
+        rlm, _ = _make_rlm()
+        tools = rlm._prepare_execution_tools()
+        result = tools["add_pages"](first=1, last=3)
+        # the tool span is unwound after the tool returns, leaving the RLM span
+        span_after_tool = context.get_current_span()
+    finally:
+        context.pop_span()
+
+    assert result == 6
+    assert span_after_tool is rlm_span
+    tool_spans = [s for s in tracked if s.span_type == models.SpanType.TOOL]
+    assert len(tool_spans) == 1
+    span = tool_spans[0]
+    assert span.name == "Tool.add_pages"
+    assert span.parent_span_id == rlm_span.span_id
+    assert span.trace_id == rlm_span.trace_id
+
+
+def test_multiple_callbacks_emit_one_tool_span(monkeypatch):
+    """Two registered CmpndCallbacks share one span-stack contextvar, so the
+    shim dispatches to only the first — one tool span, stack balanced, no
+    tool-span-nested-under-tool-span corruption."""
+    _rlm_patch._patched = False
+    _rlm_patch.patch_rlm_tools()
+
+    tracked_a, tracked_b = [], []
+    cb_a = callback.CmpndCallback()
+    cb_b = callback.CmpndCallback()
+    monkeypatch.setattr(cb_a, "_track_span", lambda span: tracked_a.append(span))
+    monkeypatch.setattr(cb_b, "_track_span", lambda span: tracked_b.append(span))
+    monkeypatch.setattr(dspy.settings, "callbacks", [cb_a, cb_b], raising=False)
+
+    context.clear_context()
+    context.set_current_trace(models.TraceBuilder(trace_id=uuid.uuid4()))
+    rlm_span = context.create_child_span("RLM.forward", models.SpanType.MODULE)
+    context.push_span(rlm_span)
+    try:
+        rlm, _ = _make_rlm()
+        tools = rlm._prepare_execution_tools()
+        tools["add_pages"](first=1, last=3)
+        span_after_tool = context.get_current_span()
+    finally:
+        context.pop_span()
+
+    assert span_after_tool is rlm_span  # stack unwound to exactly the RLM span
+    tool_spans = [s for s in tracked_a + tracked_b if s.span_type == models.SpanType.TOOL]
+    assert len(tool_spans) == 1
+    assert tool_spans[0].parent_span_id == rlm_span.span_id
+    assert not tracked_b  # only the first callback was dispatched to
+
+
+def test_tracing_off_calls_func_without_span():
+    """No cmpnd callback registered -> func still runs, no span emitted."""
+    _rlm_patch._patched = False
+    _rlm_patch.patch_rlm_tools()
+
+    context.clear_context()
+    original = list(dspy.settings.get("callbacks", []))
+    dspy.settings.callbacks = []
+    try:
+        rlm, _ = _make_rlm()
+        tools = rlm._prepare_execution_tools()
+        assert tools["add_pages"](first=2, last=4) == 9
+        assert context.get_current_span() is None
+    finally:
+        dspy.settings.callbacks = original
+
+
+def test_raising_tool_closes_span_and_reraises(monkeypatch):
+    _rlm_patch._patched = False
+    _rlm_patch.patch_rlm_tools()
+
+    tracked = []
+    cb = callback.CmpndCallback()
+    monkeypatch.setattr(cb, "_track_span", lambda span: tracked.append(span))
+    monkeypatch.setattr(dspy.settings, "callbacks", [cb], raising=False)
+
+    def boom(x: int) -> int:
+        raise ValueError("nope")
+
+    rlm = dspy.RLM("context, query -> answer", tools=[boom])
+
+    context.clear_context()
+    context.set_current_trace(models.TraceBuilder(trace_id=uuid.uuid4()))
+    rlm_span = context.create_child_span("RLM.forward", models.SpanType.MODULE)
+    context.push_span(rlm_span)
+    try:
+        tools = rlm._prepare_execution_tools()
+        raised = False
+        try:
+            tools["boom"](x=1)
+        except ValueError:
+            raised = True
+        assert raised
+        # the tool span is closed even when the tool raises
+        span_after_tool = context.get_current_span()
+    finally:
+        context.pop_span()
+
+    assert span_after_tool is rlm_span
+    tool_spans = [s for s in tracked if s.span_type == models.SpanType.TOOL]
+    assert len(tool_spans) == 1
+    assert tool_spans[0].status == models.TraceStatus.ERROR
+
+
+def test_auto_instrument_installs_rlm_patch():
+    _rlm_patch._patched = False
+    orig = dspy.RLM._prepare_execution_tools
+    cmpnd.auto_instrument()
+    assert dspy.RLM._prepare_execution_tools is not orig

{cmpnd-0.4.1 → cmpnd-0.4.2}/uv.lock

@@ -497,7 +497,7 @@ wheels = [
 
 [[package]]
 name = "cmpnd"
-version = "0.4.1"
+version = "0.4.2"
 source = { editable = "." }
 dependencies = [
     { name = "dspy" },