cmpnd 0.69.0__tar.gz → 0.70.0__tar.gz

{cmpnd-0.69.0 → cmpnd-0.70.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cmpnd
-Version: 0.69.0
+Version: 0.70.0
 Summary: DSPy observability and deployment SDK for cmpnd
 Project-URL: Homepage, https://cmpnd.ai
 Author-email: cmpnd <hello@cmpnd.ai>

{cmpnd-0.69.0 → cmpnd-0.70.0}/cmpnd/cli/commands/evals.py

@@ -9,7 +9,18 @@ from .. import api_client, deprecation, pagination
 
 
 def _add_list_args(p: ArgumentParser) -> None:
-    p.add_argument("--program", dest="program_name", help="Filter by program name")
+    p.add_argument(
+        "--program",
+        dest="program_name",
+        metavar="NAME",
+        help="Only the runs of the program with this name, current or retired",
+    )
+    p.add_argument(
+        "--module",
+        dest="module_class",
+        metavar="CLASS",
+        help="Only the runs that evaluated this module class, matched exactly (e.g. ReAct)",
+    )
     p.add_argument("--metric", dest="metric_name", help="Filter by metric name")
     p.add_argument("--status", help="Filter by status (running, completed, failed, cancelled)")
     pagination.add_args(p, with_page_aliases=True, default_limit=50)
@@ -63,6 +74,7 @@ def handle_list(client: api_client.ApiClient, args: Any) -> Any:
     body: dict[str, Any] = pagination.wire(args, "page_size")
     for attr, key in [
         ("program_name", "program_name"),
+        ("module_class", "module_class"),
         ("metric_name", "metric"),
         ("status", "status"),
     ]:

{cmpnd-0.69.0 → cmpnd-0.70.0}/cmpnd/dataset_sync.py

@@ -15,7 +15,7 @@ from . import _http, configuration, identity, models
 logger = logging.getLogger(__name__)
 
 
-def check_dataset_exists(dataset_hash: str) -> bool:
+def check_dataset_exists(dataset_hash: str, config: configuration.Config | None = None) -> bool:
     """Check if a dataset with this hash already exists on the backend.
 
     Makes a GET request to /api/v1/datasets/check/{dataset_hash}.
@@ -23,14 +23,17 @@ def check_dataset_exists(dataset_hash: str) -> bool:
 
     Args:
         dataset_hash: The SHA256[:16] hash of the dataset.
+        config: The account to ask, which must be the account the run publishes
+            to. None asks the process-global account.
 
     Returns:
         True if the dataset exists, False otherwise (including on errors).
     """
-    try:
-        config = configuration.require_config()
-    except RuntimeError:
-        return False
+    if config is None:
+        try:
+            config = configuration.require_config()
+        except RuntimeError:
+            return False
 
     url = f"{config.endpoint}/api/v1/datasets/check/{dataset_hash}"
 

{cmpnd-0.69.0 → cmpnd-0.70.0}/cmpnd/eval_handler.py

@@ -10,7 +10,7 @@ import logging
 from datetime import datetime, timezone
 from typing import Any
 
-from . import context, exporter, helpers, identity, models
+from . import context, helpers, identity, models
 
 logger = logging.getLogger(__name__)
 
@@ -85,7 +85,7 @@ class EvalHandler:
         logger.debug(f"Extracted program: {program}, type: {type(program) if program else None}")
 
         if program:
-            eval_run.program_name = program.__class__.__name__
+            eval_run.module_class = program.__class__.__name__
 
             # Hash via the IR reflector on the live module so the class tag
             # discriminates correctly; signature_summary is still derived from
@@ -165,16 +165,23 @@ class EvalHandler:
             if not input_fields and devset and hasattr(devset[0], "_input_keys"):
                 input_fields = set(devset[0]._input_keys)
             eval_run._input_fields = input_fields
-            # Fire pre-flight check (synchronous, short timeout)
+            # Fire pre-flight check (synchronous, short timeout). It asks the
+            # account the run publishes to: a client's own, else the global one.
             from . import dataset_sync
 
             try:
-                eval_run._dataset_exists = dataset_sync.check_dataset_exists(eval_run.dataset_hash)
+                eval_run._dataset_exists = dataset_sync.check_dataset_exists(
+                    eval_run.dataset_hash, self._callback._own_config
+                )
             except Exception:
                 eval_run._dataset_exists = None  # Will include examples as fallback
 
+        exp = self._callback._current_exporter()
+        eval_run.program_id = self._program_id(exp)
+
         logger.info(
-            f"EvalRun created: program_name={eval_run.program_name}, "
+            f"EvalRun created: program_id={eval_run.program_id}, "
+            f"module_class={eval_run.module_class}, "
             f"metric_name={eval_run.metric_name}, "
             f"dataset_hash={eval_run.dataset_hash}, "
             f"dataset_size={eval_run.dataset_size}"
@@ -184,7 +191,6 @@ class EvalHandler:
         # the exporter (serve/observe, or deploy in-Lambda) assigns it. Obtain
         # it synchronously now, before any eval trace ships, since every
         # in-flight trace and the dict keys / span below are stamped with it.
-        exp = exporter.get_exporter()
         if exp:
             eval_run.eval_run_id = exp.export_eval_start(eval_run)
 
@@ -201,13 +207,33 @@ class EvalHandler:
 
         # Capture evaluation config in span attributes
         span.attributes["eval.run_id"] = str(eval_run.eval_run_id)
-        span.attributes["eval.program_name"] = eval_run.program_name or ""
+        span.attributes["eval.module_class"] = eval_run.module_class or ""
         span.attributes["eval.metric_name"] = eval_run.metric_name or ""
         span.attributes["eval.dataset_size"] = str(eval_run.dataset_size)
 
         self._callback._call_id_to_span[call_id] = span
         context.push_span(span)
 
+    def _program_id(self, exp: Any) -> str | None:
+        """The program id the eval run carries, resolved the way a trace's is.
+
+        A program name the call declares wins over the callback's program id. The
+        exporter resolves the name now, on the caller's thread, because the run's
+        start is sent synchronously and must carry the id. The exporter keeps the
+        answer for the process, so a name its traces already resolved costs no
+        request. A name the exporter cannot resolve leaves the run without an id.
+        """
+        from . import callback
+
+        name = callback._declared_program_name()
+        if name is None:
+            return self._callback._resolve_program_id()
+        # Read with getattr because the member is optional: an in-memory collector
+        # that resolves no names has none.
+        resolve = getattr(exp, "resolve_program_name", None)
+        program_id = resolve(name) if callable(resolve) else None
+        return program_id if isinstance(program_id, str) else None
+
     def on_evaluate_end(
         self,
         call_id: str,
@@ -228,7 +254,7 @@ class EvalHandler:
                 span.status = models.TraceStatus.OK
             context.pop_span()
             if self._callback._should_stream_span(span):
-                exp = exporter.get_exporter()
+                exp = self._callback._current_exporter()
                 if exp:
                     exp.export_span(span)
 
@@ -373,7 +399,7 @@ class EvalHandler:
                 span.attributes["eval.score"] = str(overall_score)
 
         # Export the eval run
-        exp = exporter.get_exporter()
+        exp = self._callback._current_exporter()
         if exp:
             exp.export_eval_run(eval_run)
 

{cmpnd-0.69.0 → cmpnd-0.70.0}/cmpnd/models.py

@@ -1274,7 +1274,11 @@ class EvalRunBuilder:
     start_time: datetime | None = None
 
     # Eval Identity (defines comparability)
-    program_name: str | None = None
+    # The program the run evaluates: the id serve gave the program name the call
+    # declared, or the client's program id. None when the run names no program.
+    program_id: str | None = None
+    # The evaluated module's class, such as "ReAct".
+    module_class: str | None = None
     metric_name: str | None = None
     dataset_hash: str | None = None
     dataset_size: int = 0
@@ -1393,7 +1397,10 @@ class EvalRunBuilder:
         return {
             "eval_run_id": "new",
             "status": EvalRunStatus.RUNNING.value,
-            "program_name": self.program_name or "",
+            "program_id": self.program_id,
+            "module_class": self.module_class or "",
+            # Also under the key a serve from before module_class reads.
+            "program_name": self.module_class or "",
             "metric_name": self.metric_name or "",
             "dataset_hash": self.dataset_hash or "",
             "dataset_size": self.dataset_size,
@@ -1411,7 +1418,10 @@ class EvalRunBuilder:
         """Convert to API-compatible dictionary."""
         d = {
             "eval_run_id": str(self.eval_run_id) if self.eval_run_id else None,
-            "program_name": self.program_name or "",
+            "program_id": self.program_id,
+            "module_class": self.module_class or "",
+            # Also under the key a serve from before module_class reads.
+            "program_name": self.module_class or "",
             "metric_name": self.metric_name or "",
             "dataset_hash": self.dataset_hash or "",
             "dataset_size": self.dataset_size,

{cmpnd-0.69.0 → cmpnd-0.70.0}/docs/cli.md

@@ -458,6 +458,18 @@ program signature hash. `cmpnd programs optimized` commands and
 optimization result, not a program. Each group's `--help` names the column its
 IDs come from.
 
+`cmpnd evals list [--program NAME] [--module CLASS] [--metric] [--status]`
+lists eval runs, newest first. `--program` takes the program's name, current
+or retired, and a name no program answers to is an error that names it.
+`--module` takes the evaluated module's class, such as `ReAct`, and matches it
+exactly. The CLI sends them as `program_name` and `module_class`. Each eval
+run in `evals list`, `evals show` and `programs evals` names what it evaluated
+in three fields, as a trace does. `program_name` is the program's name, empty
+when the program has none or the run names no program. `program_id` is the
+program's `p_` id, and `module_class` is the evaluated module's class. A server
+that predates `module_class` sends the class in `program_name` and filters
+`--program` by class.
+
 `cmpnd deployments openapi <id>` prints the OpenAPI document a deployment
 serves, which an agent reads before it calls the deployment.
 

{cmpnd-0.69.0 → cmpnd-0.70.0}/docs/sdk-instrumentation.md

@@ -727,6 +727,8 @@ What it captures beyond the normal span flow:
 - An `EvalRunBuilder` (constructed at `eval_handler.py:79`) that groups all per-example traces under one eval ID.
 - The metric name (via `get_metric_name` from `python-sdk/cmpnd/helpers.py`) and inferred provider.
 - Links to the enclosing optimization run when one exists (`eval_handler.py:85`), so eval runs triggered inside `GEPA.compile` roll up under the optimization in the UI.
+- The program the run evaluates, as `program_id`. A program name the call declares wins: the exporter resolves it with `resolve_program_name`, the same once-per-process answer its traces use, on the caller's thread because the eval start is sent synchronously. Without a declared name, the run takes the callback's program id. A name serve refuses leaves the run without an id. The evaluated module's class goes as `module_class`, and again under `program_name`, the key a serve from before `module_class` reads.
+- The exporter it publishes through: the callback's own (`_current_exporter`), so a client's eval runs go to the client's account.
 - The dataset — on `on_evaluate_start` it triggers the `dataset_sync` pre-flight described above.
 
 Per-example results (scores, statuses, example hashes) are collected in `on_evaluate_end` and shipped as part of the `EvalRunBuilder`'s `to_api_dict()`, which the exporter routes to `/api/v1/eval_runs`.

{cmpnd-0.69.0 → cmpnd-0.70.0}/pyproject.toml

@@ -5,7 +5,7 @@ name = "cmpnd"
 # it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
 # hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
 # this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
-version = "0.69.0"
+version = "0.70.0"
 description = "DSPy observability and deployment SDK for cmpnd"
 readme = "README.md"
 requires-python = ">=3.11"

{cmpnd-0.69.0 → cmpnd-0.70.0}/tests/test_cli/test_evals.py

@@ -86,20 +86,34 @@ class TestEvalsList:
     # own spelling passes while the filter does nothing.
     def test_minimal_list(self, mock_client):
         mock_client.post.return_value = {"eval_runs": []}
-        args = argparse.Namespace(program_name=None, metric_name=None, status=None, limit=50, offset=0)
+        args = argparse.Namespace(
+            program_name=None, module_class=None, metric_name=None, status=None, limit=50, offset=0
+        )
         handle_list(mock_client, args)
         body = mock_client.post.call_args[1]["json"]
         assert body == {"page": 1, "page_size": 50}
 
     def test_with_filters(self, mock_client):
         mock_client.post.return_value = {"eval_runs": []}
-        args = argparse.Namespace(program_name="RAG", metric_name="accuracy", status="completed", limit=50, offset=0)
+        args = argparse.Namespace(
+            program_name="RAG", module_class=None, metric_name="accuracy", status="completed", limit=50, offset=0
+        )
         handle_list(mock_client, args)
         body = mock_client.post.call_args[1]["json"]
         assert body["program_name"] == "RAG"
         assert body["metric"] == "accuracy"
         assert body["status"] == "completed"
 
+    def test_program_names_the_program_and_module_names_the_class(self, mock_client):
+        # --program is the program's name; --module is the evaluated module's
+        # class, which the server matches exactly under module_class.
+        mock_client.post.return_value = {"eval_runs": []}
+        args = _build_parser().parse_args(["evals", "list", "--program", "capital-qa", "--module", "ReAct"])
+        handle_list(mock_client, args)
+        body = mock_client.post.call_args[1]["json"]
+        assert body["program_name"] == "capital-qa"
+        assert body["module_class"] == "ReAct"
+
     def test_deprecated_search_verb_dispatches_and_is_marked(self, mock_client):
         # The warning is the dispatcher's job (see test_main.py); the wiring is this one's.
         parser = _build_parser()

{cmpnd-0.69.0 → cmpnd-0.70.0}/tests/test_eval.py

@@ -199,7 +199,7 @@ class TestEvalLifecycle:
         eval_run = exporter.eval_runs[0]
 
         # Program metadata
-        assert eval_run.program_name == "TestProgram"
+        assert eval_run.module_class == "TestProgram"
         assert eval_run.metric_name == "exact_match"
         assert eval_run.dataset_size == 3
 

cmpnd-0.70.0/tests/test_eval_program_id.py

@@ -0,0 +1,192 @@
+"""An eval run names its program by id, and its module by class.
+
+The SDK resolves the program name a call declares to the program's id, once per
+process through the exporter, and sends it as ``program_id`` on the eval run's
+start and completion. The evaluated module's class goes as ``module_class``, and
+again under ``program_name``, where a serve from before ``module_class`` reads it.
+"""
+
+from unittest import mock
+
+import httpx
+import pytest
+
+from tests import _traced_run
+from tests._exporter_helpers import decode_body
+
+pytest.importorskip("dspy")
+
+import dspy  # noqa: E402
+
+from cmpnd import _http, configuration  # noqa: E402
+from cmpnd.callback import DECLARED_PROGRAM_SETTING  # noqa: E402
+from cmpnd.configuration import Config, configure  # noqa: E402
+from cmpnd.context import clear_context  # noqa: E402
+from cmpnd.exporter import BatchExporter  # noqa: E402
+from cmpnd.models import EvalRunBuilder  # noqa: E402
+from tests.test_eval import MockExporter  # noqa: E402
+
+PROGRAM_ID = "p_3pQr7t8sK2mN"
+
+
+def test_the_start_and_the_completion_carry_the_program_id_and_the_class():
+    eval_run = EvalRunBuilder(module_class="ReAct", program_id=PROGRAM_ID, metric_name="m")
+    for body in (eval_run.to_start_dict(), eval_run.to_api_dict()):
+        assert body["program_id"] == PROGRAM_ID
+        assert body["module_class"] == "ReAct"
+        assert body["program_name"] == "ReAct"
+
+
+def test_a_run_with_no_program_sends_no_id():
+    eval_run = EvalRunBuilder(module_class="Predict")
+    for body in (eval_run.to_start_dict(), eval_run.to_api_dict()):
+        assert body["program_id"] is None
+        assert body["module_class"] == "Predict"
+
+
+def test_the_start_puts_the_program_id_on_the_wire():
+    captured = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        captured["body"] = decode_body(request)
+        return httpx.Response(202, json={"eval_run_id": "ev_SERVERMINTED"})
+
+    config = Config(api_key="test", endpoint="http://localhost:9999")
+    exporter = BatchExporter(config, transport=httpx.MockTransport(handler))
+    exporter.export_eval_start(EvalRunBuilder(module_class="ReAct", program_id=PROGRAM_ID))
+
+    assert captured["body"]["program_id"] == PROGRAM_ID
+    assert captured["body"]["module_class"] == "ReAct"
+    assert captured["body"]["program_name"] == "ReAct"
+
+
+class QuestionAnswer(dspy.Module):
+    def __init__(self):
+        super().__init__()
+        self.predict = dspy.Predict("question -> answer")
+
+    def forward(self, question):
+        return self.predict(question=question)
+
+
+def _exact(example, prediction, trace=None):
+    return example.answer == prediction.answer
+
+
+class ResolvingExporter(MockExporter):
+    """A MockExporter that resolves declared program names, as BatchExporter does."""
+
+    def __init__(self, ids: dict[str, str | None]):
+        super().__init__()
+        self.ids = ids
+        self.resolved: list[str] = []
+
+    def resolve_program_name(self, name: str) -> str | None:
+        self.resolved.append(name)
+        return self.ids.get(name)
+
+
+class TestTheEvalHandlerNamesTheProgram:
+    def setup_method(self):
+        configure(api_key="test_key", endpoint="http://localhost:8000")
+
+    def teardown_method(self):
+        clear_context()
+        import cmpnd.configuration
+
+        cmpnd.configuration._config = None
+
+    def _start(self, exporter, *, declared: str | None = None, callback=None):
+        callback = callback or _traced_run.make_callback()
+        evaluate = dspy.Evaluate(devset=[], metric=_exact)
+        settings = {DECLARED_PROGRAM_SETTING: declared} if declared is not None else {}
+        with mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=exporter), dspy.context(**settings):
+            callback.on_evaluate_start(call_id="c", instance=evaluate, inputs={"program": QuestionAnswer()})
+        return exporter.eval_starts[0]
+
+    def test_a_declared_name_resolves_to_its_program_id(self):
+        exporter = ResolvingExporter({"capital-qa": PROGRAM_ID})
+        eval_run = self._start(exporter, declared="capital-qa")
+        assert exporter.resolved == ["capital-qa"]
+        assert eval_run.program_id == PROGRAM_ID
+        assert eval_run.module_class == "QuestionAnswer"
+
+    def test_a_name_serve_refuses_sends_no_id(self):
+        exporter = ResolvingExporter({})
+        eval_run = self._start(exporter, declared="capital-qa")
+        assert eval_run.program_id is None
+
+    def test_the_callbacks_program_id_applies_when_no_name_is_declared(self):
+        callback = _traced_run.make_callback()
+        exporter = ResolvingExporter({})
+        with mock.patch.object(callback, "_resolve_program_id", autospec=True, return_value=PROGRAM_ID):
+            eval_run = self._start(exporter, callback=callback)
+        assert exporter.resolved == []
+        assert eval_run.program_id == PROGRAM_ID
+
+    def test_a_declared_name_wins_over_the_callbacks_program_id(self):
+        callback = _traced_run.make_callback()
+        other = "p_000000000000"
+        exporter = ResolvingExporter({"capital-qa": PROGRAM_ID})
+        with mock.patch.object(callback, "_resolve_program_id", autospec=True, return_value=other):
+            eval_run = self._start(exporter, declared="capital-qa", callback=callback)
+        assert eval_run.program_id == PROGRAM_ID
+
+    def test_a_run_that_names_no_program_carries_no_id(self):
+        eval_run = self._start(MockExporter())
+        assert eval_run.program_id is None
+        assert eval_run.module_class == "QuestionAnswer"
+
+
+GLOBAL_ENDPOINT = "http://global-account.invalid"
+CLIENT_ENDPOINT = "http://client-account.invalid"
+
+
+class TestTheDatasetPreflightAsksTheAccountTheRunPublishesTo:
+    """The eval start asks serve whether the dataset exists, so the completion can
+    leave its examples out. It must ask the account the run publishes to: a client
+    publishes to its own account, which may lack a dataset the global one holds."""
+
+    def teardown_method(self):
+        clear_context()
+        configuration._config = None
+
+    def _start(self, callback):
+        """Start an eval of a one-example devset, and return the eval run and the
+        (url, authorization) of every dataset check. Only the global account holds
+        the dataset."""
+        checks: list[tuple[str, str]] = []
+
+        def handler(request: httpx.Request) -> httpx.Response:
+            authorization = request.headers["Authorization"]
+            checks.append((str(request.url), authorization))
+            return httpx.Response(200, json={"exists": authorization == "Bearer ck_global"})
+
+        real_new_client = _http.new_client
+
+        def new_client(**kwargs):
+            return real_new_client(transport=httpx.MockTransport(handler), **kwargs)
+
+        devset = [dspy.Example(question="2+2?", answer="4").with_inputs("question")]
+        evaluate = dspy.Evaluate(devset=devset, metric=_exact)
+        exporter = MockExporter()
+        with (
+            mock.patch("cmpnd._http.new_client", autospec=True, side_effect=new_client),
+            mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=exporter),
+            mock.patch("cmpnd._exporter_pool.ensure_exporter_started", autospec=True, return_value=exporter),
+        ):
+            callback.on_evaluate_start(call_id="c", instance=evaluate, inputs={"program": QuestionAnswer()})
+        return exporter.eval_starts[0], checks
+
+    def test_a_clients_eval_checks_the_clients_account(self):
+        configure(api_key="ck_global", endpoint=GLOBAL_ENDPOINT)
+        own = Config(api_key="ck_client", endpoint=CLIENT_ENDPOINT)
+        eval_run, checks = self._start(_traced_run.make_callback(own_config=own))
+        assert checks == [(f"{CLIENT_ENDPOINT}/api/v1/datasets/check/{eval_run.dataset_hash}", "Bearer ck_client")]
+        assert eval_run._dataset_exists is False
+
+    def test_the_global_callbacks_eval_checks_the_global_account(self):
+        configure(api_key="ck_global", endpoint=GLOBAL_ENDPOINT)
+        eval_run, checks = self._start(_traced_run.make_callback())
+        assert checks == [(f"{GLOBAL_ENDPOINT}/api/v1/datasets/check/{eval_run.dataset_hash}", "Bearer ck_global")]
+        assert eval_run._dataset_exists is True

{cmpnd-0.69.0 → cmpnd-0.70.0}/tests/test_eval_start.py

@@ -32,7 +32,7 @@ def test_export_eval_start_is_synchronous_and_returns_server_id():
         return httpx.Response(200, json={"eval_run_id": "ev_SERVERMINTED"})
 
     exporter = BatchExporter(_config(), transport=httpx.MockTransport(handler))
-    eval_run = EvalRunBuilder(program_name="P", metric_name="m")
+    eval_run = EvalRunBuilder(module_class="P", metric_name="m")
 
     assigned = exporter.export_eval_start(eval_run)
 
@@ -76,7 +76,7 @@ def test_eval_start_fails_loud_after_retries():
         return httpx.Response(500, json={"error": "boom"})
 
     exporter = BatchExporter(_config(), transport=httpx.MockTransport(handler))
-    eval_run = EvalRunBuilder(program_name="P")
+    eval_run = EvalRunBuilder(module_class="P")
 
     with pytest.raises(RuntimeError):
         exporter.export_eval_start(eval_run)

{cmpnd-0.69.0 → cmpnd-0.70.0}/tests/test_ingest_conformance.py

@@ -249,7 +249,8 @@ def _eval_run() -> EvalRunBuilder:
     now = datetime.now(timezone.utc)
     run = EvalRunBuilder(
         start_time=now,
-        program_name="QAProgram",
+        program_id="p_3pQr7t8sK2mN",
+        module_class="QAProgram",
         metric_name="exact_match",
         dataset_hash="0123456789abcdef",
         dataset_size=2,

{cmpnd-0.69.0 → cmpnd-0.70.0}/uv.lock

@@ -505,7 +505,7 @@ wheels = [
 
 [[package]]
 name = "cmpnd"
-version = "0.69.0"
+version = "0.70.0"
 source = { editable = "." }
 default-groups = ["dspy-3-4-1"]
 dependencies = [