cmpnd 0.69.0__tar.gz → 0.71.0__tar.gz

{cmpnd-0.69.0 → cmpnd-0.71.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cmpnd
-Version: 0.69.0
+Version: 0.71.0
 Summary: DSPy observability and deployment SDK for cmpnd
 Project-URL: Homepage, https://cmpnd.ai
 Author-email: cmpnd <hello@cmpnd.ai>

{cmpnd-0.69.0 → cmpnd-0.71.0}/cmpnd/cli/__init__.py

@@ -171,15 +171,8 @@ def main() -> None:
     evals.register(subparsers)
     datasets.register(subparsers)
     deployments.register(subparsers)
-
-    # Assessments and valsets CLIs are still in development, hidden behind
-    # CMPND_CLI_DEV=1 (mirroring the CMPND_CLI_ADMIN gate above). The env var
-    # only hides the commands from customers; the server still gates each action
-    # on the assessments:* / valsets:* scope, so this is discoverability, not
-    # enforcement.
-    if os.environ.get("CMPND_CLI_DEV") == "1":
-        assessments.register(subparsers)
-        valsets.register(subparsers)
+    assessments.register(subparsers)
+    valsets.register(subparsers)
 
     # Top-level Docker-style shortcuts (human-readable by default; --json opts in)
     top.register(subparsers)

{cmpnd-0.69.0 → cmpnd-0.71.0}/cmpnd/cli/commands/admin.py

@@ -14,8 +14,7 @@ default limits and feature flags, and `cmpnd admin signup …` sets who may sign
 and the plan a new account's org lands on. `cmpnd admin feature-flags …` reads and
 toggles the server's runtime flags. These hit
 /api/v1/admin/* and require an API key carrying the admin:* scopes — the server
-is the real authority. The CMPND_CLI_ADMIN env gate (mirroring CMPND_CLI_DEV)
-only keeps these out of customers' --help; it grants nothing.
+is the real authority. The CMPND_CLI_ADMIN env gate only keeps these out of customers' --help; it grants nothing.
 """
 
 from __future__ import annotations

{cmpnd-0.69.0 → cmpnd-0.71.0}/cmpnd/cli/commands/evals.py

@@ -9,7 +9,18 @@ from .. import api_client, deprecation, pagination
 
 
 def _add_list_args(p: ArgumentParser) -> None:
-    p.add_argument("--program", dest="program_name", help="Filter by program name")
+    p.add_argument(
+        "--program",
+        dest="program_name",
+        metavar="NAME",
+        help="Only the runs of the program with this name, current or retired",
+    )
+    p.add_argument(
+        "--module",
+        dest="module_class",
+        metavar="CLASS",
+        help="Only the runs that evaluated this module class, matched exactly (e.g. ReAct)",
+    )
     p.add_argument("--metric", dest="metric_name", help="Filter by metric name")
     p.add_argument("--status", help="Filter by status (running, completed, failed, cancelled)")
     pagination.add_args(p, with_page_aliases=True, default_limit=50)
@@ -63,6 +74,7 @@ def handle_list(client: api_client.ApiClient, args: Any) -> Any:
     body: dict[str, Any] = pagination.wire(args, "page_size")
     for attr, key in [
         ("program_name", "program_name"),
+        ("module_class", "module_class"),
         ("metric_name", "metric"),
         ("status", "status"),
     ]:

{cmpnd-0.69.0 → cmpnd-0.71.0}/cmpnd/client.py

@@ -17,10 +17,6 @@ and propagates to its own eval / parallel threads. It registers nothing global
 and holds no OS resource: exporters are pooled by credentials at module scope
 (:mod:`cmpnd._exporter_pool`), so a client may fall out of scope with nothing to
 release. ``close()`` is an optional alias for end-of-scope flushing.
-
-The assessment and valset calls are in development. Each raises unless
-``CMPND_CLI_DEV=1`` is set, the switch that hides the in-development
-``cmpnd assessments`` and ``cmpnd valsets`` CLI groups. Tracing needs no flag.
 """
 
 from __future__ import annotations
@@ -111,15 +107,6 @@ class Client:
         if not self._active:
             raise RuntimeError("client is closed; create a new cmpnd.Client")
 
-    def _check_in_development(self, method: str) -> None:
-        """Refuse an assessment or valset call unless ``CMPND_CLI_DEV=1`` is set."""
-        if os.getenv("CMPND_CLI_DEV") != "1":
-            raise RuntimeError(
-                f"cmpnd.Client.{method}() is in development and not yet generally available. "
-                "Set CMPND_CLI_DEV=1 to enable it."
-            )
-        self._check_active()
-
     def _scoped_callbacks(self) -> list[Any]:
         """This client's callback, plus the caller's own non-cmpnd callbacks.
 
@@ -918,7 +905,7 @@ class Client:
         e.g. ``annotations=[("output", "/trends/2", "the third trend"),
         ("span", "a1b2c3", "wrong field")]``.
         """
-        self._check_in_development("create_assessment")
+        self._check_active()
         _exporter_pool.flush(self._config)  # the referenced trace must be durable first
         return _assessments.create(
             trace_id,
@@ -937,7 +924,7 @@ class Client:
         Pass exactly one of ``program`` (all of that program's assessments, named by
         its name or ``p_`` id) or ``trace_id`` (one trace's assessments).
         """
-        self._check_in_development("list_assessments")
+        self._check_active()
         if program is not None and trace_id is not None:
             raise ValueError("pass exactly one of program or trace_id")
         page = _assessments.list(
@@ -952,24 +939,24 @@ class Client:
 
     def get_assessment(self, assessment_id: str) -> dict[str, Any]:
         """Fetch one assessment with its annotations."""
-        self._check_in_development("get_assessment")
+        self._check_active()
         return _assessments.get(assessment_id, api_key=self._config.api_key, endpoint=self._config.endpoint)
 
     def delete_assessment(self, assessment_id: str) -> None:
         """Delete one assessment. Raises if it does not exist."""
-        self._check_in_development("delete_assessment")
+        self._check_active()
         _assessments.delete(assessment_id, api_key=self._config.api_key, endpoint=self._config.endpoint)
 
     def create_valset(self, program: str, *, name: str | None = None) -> dict[str, Any]:
         """Create a valset owned by ``program``, a program name or ``p_`` id; name auto-assigned when omitted."""
-        self._check_in_development("create_valset")
+        self._check_active()
         return _valsets.create(
             self._program_id(program), name=name, api_key=self._config.api_key, endpoint=self._config.endpoint
         )
 
     def list_valsets(self, program: str, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
         """The valsets of ``program``, a program name or ``p_`` id, newest first (paged by limit/offset)."""
-        self._check_in_development("list_valsets")
+        self._check_active()
         page = _valsets.list(
             self._program_id(program),
             limit=limit,
@@ -981,12 +968,12 @@ class Client:
 
     def get_valset(self, valset_id: str) -> dict[str, Any]:
         """Fetch one valset with its field shape and current version."""
-        self._check_in_development("get_valset")
+        self._check_active()
         return _valsets.get(valset_id, api_key=self._config.api_key, endpoint=self._config.endpoint)
 
     def list_valset_entries(self, valset_id: str, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
         """A valset's active entries (paged by limit/offset)."""
-        self._check_in_development("list_valset_entries")
+        self._check_active()
         page = _valsets.list_entries(
             valset_id, limit=limit, offset=offset, api_key=self._config.api_key, endpoint=self._config.endpoint
         )
@@ -1002,7 +989,7 @@ class Client:
     ) -> dict[str, Any]:
         """Add one entry. Pass exactly one of ``from_assessment`` (the server
         derives inputs from its trace) or ``inputs``."""
-        self._check_in_development("create_valset_entry")
+        self._check_active()
         return _valsets.create_entry(
             valset_id,
             expected_outputs=expected_outputs,
@@ -1014,7 +1001,7 @@ class Client:
 
     def delete_valset_entry(self, entry_id: str) -> dict[str, Any]:
         """Retire an entry (versioned, not a hard delete)."""
-        self._check_in_development("delete_valset_entry")
+        self._check_active()
         return _valsets.delete_entry(entry_id, api_key=self._config.api_key, endpoint=self._config.endpoint)
 
     def _run_gepa(

{cmpnd-0.69.0 → cmpnd-0.71.0}/cmpnd/dataset_sync.py

@@ -15,7 +15,7 @@ from . import _http, configuration, identity, models
 logger = logging.getLogger(__name__)
 
 
-def check_dataset_exists(dataset_hash: str) -> bool:
+def check_dataset_exists(dataset_hash: str, config: configuration.Config | None = None) -> bool:
     """Check if a dataset with this hash already exists on the backend.
 
     Makes a GET request to /api/v1/datasets/check/{dataset_hash}.
@@ -23,14 +23,17 @@ def check_dataset_exists(dataset_hash: str) -> bool:
 
     Args:
         dataset_hash: The SHA256[:16] hash of the dataset.
+        config: The account to ask, which must be the account the run publishes
+            to. None asks the process-global account.
 
     Returns:
         True if the dataset exists, False otherwise (including on errors).
     """
-    try:
-        config = configuration.require_config()
-    except RuntimeError:
-        return False
+    if config is None:
+        try:
+            config = configuration.require_config()
+        except RuntimeError:
+            return False
 
     url = f"{config.endpoint}/api/v1/datasets/check/{dataset_hash}"
 

{cmpnd-0.69.0 → cmpnd-0.71.0}/cmpnd/eval_handler.py

@@ -10,7 +10,7 @@ import logging
 from datetime import datetime, timezone
 from typing import Any
 
-from . import context, exporter, helpers, identity, models
+from . import context, helpers, identity, models
 
 logger = logging.getLogger(__name__)
 
@@ -85,7 +85,7 @@ class EvalHandler:
         logger.debug(f"Extracted program: {program}, type: {type(program) if program else None}")
 
         if program:
-            eval_run.program_name = program.__class__.__name__
+            eval_run.module_class = program.__class__.__name__
 
             # Hash via the IR reflector on the live module so the class tag
             # discriminates correctly; signature_summary is still derived from
@@ -165,16 +165,23 @@ class EvalHandler:
             if not input_fields and devset and hasattr(devset[0], "_input_keys"):
                 input_fields = set(devset[0]._input_keys)
             eval_run._input_fields = input_fields
-            # Fire pre-flight check (synchronous, short timeout)
+            # Fire pre-flight check (synchronous, short timeout). It asks the
+            # account the run publishes to: a client's own, else the global one.
             from . import dataset_sync
 
             try:
-                eval_run._dataset_exists = dataset_sync.check_dataset_exists(eval_run.dataset_hash)
+                eval_run._dataset_exists = dataset_sync.check_dataset_exists(
+                    eval_run.dataset_hash, self._callback._own_config
+                )
             except Exception:
                 eval_run._dataset_exists = None  # Will include examples as fallback
 
+        exp = self._callback._current_exporter()
+        eval_run.program_id = self._program_id(exp)
+
         logger.info(
-            f"EvalRun created: program_name={eval_run.program_name}, "
+            f"EvalRun created: program_id={eval_run.program_id}, "
+            f"module_class={eval_run.module_class}, "
             f"metric_name={eval_run.metric_name}, "
             f"dataset_hash={eval_run.dataset_hash}, "
             f"dataset_size={eval_run.dataset_size}"
@@ -184,7 +191,6 @@ class EvalHandler:
         # the exporter (serve/observe, or deploy in-Lambda) assigns it. Obtain
         # it synchronously now, before any eval trace ships, since every
         # in-flight trace and the dict keys / span below are stamped with it.
-        exp = exporter.get_exporter()
         if exp:
             eval_run.eval_run_id = exp.export_eval_start(eval_run)
 
@@ -201,13 +207,33 @@ class EvalHandler:
 
         # Capture evaluation config in span attributes
         span.attributes["eval.run_id"] = str(eval_run.eval_run_id)
-        span.attributes["eval.program_name"] = eval_run.program_name or ""
+        span.attributes["eval.module_class"] = eval_run.module_class or ""
         span.attributes["eval.metric_name"] = eval_run.metric_name or ""
         span.attributes["eval.dataset_size"] = str(eval_run.dataset_size)
 
         self._callback._call_id_to_span[call_id] = span
         context.push_span(span)
 
+    def _program_id(self, exp: Any) -> str | None:
+        """The program id the eval run carries, resolved the way a trace's is.
+
+        A program name the call declares wins over the callback's program id. The
+        exporter resolves the name now, on the caller's thread, because the run's
+        start is sent synchronously and must carry the id. The exporter keeps the
+        answer for the process, so a name its traces already resolved costs no
+        request. A name the exporter cannot resolve leaves the run without an id.
+        """
+        from . import callback
+
+        name = callback._declared_program_name()
+        if name is None:
+            return self._callback._resolve_program_id()
+        # Read with getattr because the member is optional: an in-memory collector
+        # that resolves no names has none.
+        resolve = getattr(exp, "resolve_program_name", None)
+        program_id = resolve(name) if callable(resolve) else None
+        return program_id if isinstance(program_id, str) else None
+
     def on_evaluate_end(
         self,
         call_id: str,
@@ -228,7 +254,7 @@ class EvalHandler:
                 span.status = models.TraceStatus.OK
             context.pop_span()
             if self._callback._should_stream_span(span):
-                exp = exporter.get_exporter()
+                exp = self._callback._current_exporter()
                 if exp:
                     exp.export_span(span)
 
@@ -373,7 +399,7 @@ class EvalHandler:
                 span.attributes["eval.score"] = str(overall_score)
 
         # Export the eval run
-        exp = exporter.get_exporter()
+        exp = self._callback._current_exporter()
         if exp:
             exp.export_eval_run(eval_run)
 

{cmpnd-0.69.0 → cmpnd-0.71.0}/cmpnd/models.py

@@ -1274,7 +1274,11 @@ class EvalRunBuilder:
     start_time: datetime | None = None
 
     # Eval Identity (defines comparability)
-    program_name: str | None = None
+    # The program the run evaluates: the id serve gave the program name the call
+    # declared, or the client's program id. None when the run names no program.
+    program_id: str | None = None
+    # The evaluated module's class, such as "ReAct".
+    module_class: str | None = None
     metric_name: str | None = None
     dataset_hash: str | None = None
     dataset_size: int = 0
@@ -1393,7 +1397,10 @@ class EvalRunBuilder:
         return {
             "eval_run_id": "new",
             "status": EvalRunStatus.RUNNING.value,
-            "program_name": self.program_name or "",
+            "program_id": self.program_id,
+            "module_class": self.module_class or "",
+            # Also under the key a serve from before module_class reads.
+            "program_name": self.module_class or "",
             "metric_name": self.metric_name or "",
             "dataset_hash": self.dataset_hash or "",
             "dataset_size": self.dataset_size,
@@ -1411,7 +1418,10 @@ class EvalRunBuilder:
         """Convert to API-compatible dictionary."""
         d = {
             "eval_run_id": str(self.eval_run_id) if self.eval_run_id else None,
-            "program_name": self.program_name or "",
+            "program_id": self.program_id,
+            "module_class": self.module_class or "",
+            # Also under the key a serve from before module_class reads.
+            "program_name": self.module_class or "",
             "metric_name": self.metric_name or "",
             "dataset_hash": self.dataset_hash or "",
             "dataset_size": self.dataset_size,

{cmpnd-0.69.0 → cmpnd-0.71.0}/docs/cli.md

@@ -418,10 +418,7 @@ down or neutral conclusion, an optional prose body and annotations. An
 annotation attaches a note to a span or to a piece of the output addressed by
 a JSON pointer.
 
-The group is in development. The CLI registers it only when
-`CMPND_CLI_DEV=1` is set, so it does not appear in `cmpnd --help`. The
-variable only hides the commands. Serve still checks each action against the
-`assessments:*` scopes.
+Serve checks each action against the `assessments:*` scopes.
 
 - `cmpnd assessments create <trace-id> [--up|--down|--neutral] [-m "prose"]
   [--annotate KIND REF NOTE …]` records one judgement.
@@ -458,6 +455,18 @@ program signature hash. `cmpnd programs optimized` commands and
 optimization result, not a program. Each group's `--help` names the column its
 IDs come from.
 
+`cmpnd evals list [--program NAME] [--module CLASS] [--metric] [--status]`
+lists eval runs, newest first. `--program` takes the program's name, current
+or retired, and a name no program answers to is an error that names it.
+`--module` takes the evaluated module's class, such as `ReAct`, and matches it
+exactly. The CLI sends them as `program_name` and `module_class`. Each eval
+run in `evals list`, `evals show` and `programs evals` names what it evaluated
+in three fields, as a trace does. `program_name` is the program's name, empty
+when the program has none or the run names no program. `program_id` is the
+program's `p_` id, and `module_class` is the evaluated module's class. A server
+that predates `module_class` sends the class in `program_name` and filters
+`--program` by class.
+
 `cmpnd deployments openapi <id>` prints the OpenAPI document a deployment
 serves, which an agent reads before it calls the deployment.
 
@@ -522,9 +531,7 @@ that lookup, so it creates a valset by the program's name. On a server from befo
 name fails with an error that says to upgrade the server. Entries live in an
 `entries` subgroup, the same shape as `cmpnd datasets examples`.
 
-The group is in development. The CLI registers it only when
-`CMPND_CLI_DEV=1` is set. Serve still checks each action against the
-`valsets:*` scopes.
+Serve checks each action against the `valsets:*` scopes.
 
 - `cmpnd valsets create <program> [--name <name>]` creates a validation
   set for a program. The valset ID is its identity, and the name is a label

{cmpnd-0.69.0 → cmpnd-0.71.0}/docs/client.md

@@ -10,11 +10,6 @@ client methods too, and have no form outside a client. A client holds its own ta
 the global config that `auto_instrument()` reads, so both can run in one
 process.
 
-The client's assessment and valset calls are in development. Each one raises
-`RuntimeError` unless `CMPND_CLI_DEV=1` is set. The same variable reveals the
-in-development `cmpnd assessments` and `cmpnd valsets` CLI groups. Tracing
-through a client needs no flag.
-
 ## Entry point
 
 ```python
@@ -218,12 +213,12 @@ methods:
   file the deployment stores, or the archive-relative file `path` names.
   `sdk.get_deployment_signature(deployment_id)` returns the selected
   artifact's typed metadata. It reads no worker status or executions.
-- Assessments, in development behind `CMPND_CLI_DEV=1`:
+- Assessments:
   `sdk.create_assessment(trace_id, conclusion=..., body=...,
   annotations=...)`, `sdk.list_assessments(program=... | trace_id=...,
   limit=..., offset=...)`, `sdk.get_assessment(assessment_id)` and
   `sdk.delete_assessment(assessment_id)`.
-- Valsets, in development behind `CMPND_CLI_DEV=1`:
+- Valsets:
   `sdk.create_valset(program, name=...)`, `sdk.list_valsets(program,
   limit=..., offset=...)`, `sdk.get_valset(valset_id)`,
   `sdk.list_valset_entries(valset_id, limit=..., offset=...)`,

{cmpnd-0.69.0 → cmpnd-0.71.0}/docs/sdk-instrumentation.md

@@ -727,6 +727,8 @@ What it captures beyond the normal span flow:
 - An `EvalRunBuilder` (constructed at `eval_handler.py:79`) that groups all per-example traces under one eval ID.
 - The metric name (via `get_metric_name` from `python-sdk/cmpnd/helpers.py`) and inferred provider.
 - Links to the enclosing optimization run when one exists (`eval_handler.py:85`), so eval runs triggered inside `GEPA.compile` roll up under the optimization in the UI.
+- The program the run evaluates, as `program_id`. A program name the call declares wins: the exporter resolves it with `resolve_program_name`, the same once-per-process answer its traces use, on the caller's thread because the eval start is sent synchronously. Without a declared name, the run takes the callback's program id. A name serve refuses leaves the run without an id. The evaluated module's class goes as `module_class`, and again under `program_name`, the key a serve from before `module_class` reads.
+- The exporter it publishes through: the callback's own (`_current_exporter`), so a client's eval runs go to the client's account.
 - The dataset — on `on_evaluate_start` it triggers the `dataset_sync` pre-flight described above.
 
 Per-example results (scores, statuses, example hashes) are collected in `on_evaluate_end` and shipped as part of the `EvalRunBuilder`'s `to_api_dict()`, which the exporter routes to `/api/v1/eval_runs`.

{cmpnd-0.69.0 → cmpnd-0.71.0}/pyproject.toml

@@ -5,7 +5,7 @@ name = "cmpnd"
 # it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
 # hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
 # this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
-version = "0.69.0"
+version = "0.71.0"
 description = "DSPy observability and deployment SDK for cmpnd"
 readme = "README.md"
 requires-python = ">=3.11"

{cmpnd-0.69.0 → cmpnd-0.71.0}/tests/test_cli/test_evals.py

@@ -86,20 +86,34 @@ class TestEvalsList:
     # own spelling passes while the filter does nothing.
     def test_minimal_list(self, mock_client):
         mock_client.post.return_value = {"eval_runs": []}
-        args = argparse.Namespace(program_name=None, metric_name=None, status=None, limit=50, offset=0)
+        args = argparse.Namespace(
+            program_name=None, module_class=None, metric_name=None, status=None, limit=50, offset=0
+        )
         handle_list(mock_client, args)
         body = mock_client.post.call_args[1]["json"]
         assert body == {"page": 1, "page_size": 50}
 
     def test_with_filters(self, mock_client):
         mock_client.post.return_value = {"eval_runs": []}
-        args = argparse.Namespace(program_name="RAG", metric_name="accuracy", status="completed", limit=50, offset=0)
+        args = argparse.Namespace(
+            program_name="RAG", module_class=None, metric_name="accuracy", status="completed", limit=50, offset=0
+        )
         handle_list(mock_client, args)
         body = mock_client.post.call_args[1]["json"]
         assert body["program_name"] == "RAG"
         assert body["metric"] == "accuracy"
         assert body["status"] == "completed"
 
+    def test_program_names_the_program_and_module_names_the_class(self, mock_client):
+        # --program is the program's name; --module is the evaluated module's
+        # class, which the server matches exactly under module_class.
+        mock_client.post.return_value = {"eval_runs": []}
+        args = _build_parser().parse_args(["evals", "list", "--program", "capital-qa", "--module", "ReAct"])
+        handle_list(mock_client, args)
+        body = mock_client.post.call_args[1]["json"]
+        assert body["program_name"] == "capital-qa"
+        assert body["module_class"] == "ReAct"
+
     def test_deprecated_search_verb_dispatches_and_is_marked(self, mock_client):
         # The warning is the dispatcher's job (see test_main.py); the wiring is this one's.
         parser = _build_parser()

{cmpnd-0.69.0 → cmpnd-0.71.0}/tests/test_cli/test_main.py

@@ -169,6 +169,28 @@ class TestDevGate:
         output = json.loads(captured.out)
         assert "families" in output
 
+    def test_assessments_works_without_flag(self, capsys):
+        env = {k: v for k, v in os.environ.items() if k != "CMPND_CLI_DEV"}
+        env["CMPND_API_KEY"] = "test-key"
+        mock_response = {"assessments": [], "total": 0}
+        with mock.patch("sys.argv", ["cmpnd", "assessments", "list", "--trace", "t_1"]):
+            with mock.patch.dict(os.environ, env, clear=True):
+                with mock.patch("cmpnd.cli.api_client.ApiClient.get", autospec=True, return_value=mock_response) as get:
+                    main()
+        assert get.call_args.args[1] == "/api/v1/assessments"
+        assert json.loads(capsys.readouterr().out) == mock_response
+
+    def test_valsets_works_without_flag(self, capsys):
+        env = {k: v for k, v in os.environ.items() if k != "CMPND_CLI_DEV"}
+        env["CMPND_API_KEY"] = "test-key"
+        mock_response = {"id": "v_1", "name": "valset-2026-11-02"}
+        with mock.patch("sys.argv", ["cmpnd", "valsets", "show", "v_1"]):
+            with mock.patch.dict(os.environ, env, clear=True):
+                with mock.patch("cmpnd.cli.api_client.ApiClient.get", autospec=True, return_value=mock_response) as get:
+                    main()
+        assert get.call_args.args[1] == "/api/v1/valsets/v_1"
+        assert json.loads(capsys.readouterr().out) == mock_response
+
 
 class TestAdminGate:
     """CMP-126: `admin` commands gated behind CMPND_CLI_ADMIN=1; the

{cmpnd-0.69.0 → cmpnd-0.71.0}/tests/test_cli/test_parity.py

@@ -16,7 +16,6 @@ from __future__ import annotations
 
 import argparse
 import itertools
-import os
 import re
 from pathlib import Path
 from unittest import mock
@@ -26,6 +25,9 @@ from cmpnd.cli.api_client import ApiClient
 from cmpnd.cli.commands import (
     admin as _admin,
 )
+from cmpnd.cli.commands import (
+    assessments as _assessments,
+)
 from cmpnd.cli.commands import (
     auth as _auth,
 )
@@ -59,6 +61,9 @@ from cmpnd.cli.commands import (
 from cmpnd.cli.commands import (
     traces as _traces,
 )
+from cmpnd.cli.commands import (
+    valsets as _valsets,
+)
 
 #: Every module registering a resource-tier noun group. `login` is excluded
 #: deliberately: `login` / `logout` are task-tier top-level commands with no
@@ -75,6 +80,8 @@ _RESOURCE_MODULES = (
     _evals,
     _datasets,
     _deployments,
+    _assessments,
+    _valsets,
 )
 
 
@@ -281,10 +288,8 @@ class TestOptimizeBinding:
     partial revival (CLI but not REPL) is the failure worth catching."""
 
     def test_optimize_is_bound_on_both_surfaces(self):
-        env = {k: v for k, v in os.environ.items() if k != "CMPND_CLI_DEV"}
-        with mock.patch.dict(os.environ, env, clear=True):
-            assert "optimize" in _argparse_subcommands()
-            assert "optimize" in _shell_namespace()
+        assert "optimize" in _argparse_subcommands()
+        assert "optimize" in _shell_namespace()
 
 
 class TestResourceVocabulary:

{cmpnd-0.69.0 → cmpnd-0.71.0}/tests/test_client.py

@@ -327,11 +327,11 @@ class TestMisuseGuard:
             sdk5.run_program(a_program(), question="What is the capital of France?")
 
 
-class TestDevelopmentGate:
-    """The client's tracing needs no flag. Its assessment and valset calls are in
-    development, gated behind CMPND_CLI_DEV=1 as their CLI groups are."""
+class TestAssessmentsAndValsetsNeedNoFlag:
+    """The client's assessment and valset calls need no environment flag. A closed
+    client still refuses each one, as it refuses every other call."""
 
-    GATED = {
+    CALLS = {
         "create_assessment": ("t_1",),
         "list_assessments": ("p_0123456789Ab",),
         "get_assessment": ("a_1",),
@@ -344,23 +344,19 @@ class TestDevelopmentGate:
         "delete_valset_entry": ("e_1",),
     }
 
-    def test_construction_needs_no_flag(self, monkeypatch: pytest.MonkeyPatch):
-        monkeypatch.delenv("CMPND_CLI_DEV", raising=False)
-        sdk = cmpnd.Client(project_tags=["context-Z"], api_key="k", endpoint="http://x")
-        assert sdk._active
-
-    @pytest.mark.parametrize("method", sorted(GATED))
-    def test_an_assessment_or_valset_call_refuses_without_the_flag(self, method, monkeypatch: pytest.MonkeyPatch):
+    @pytest.mark.parametrize("method", sorted(CALLS))
+    def test_a_closed_client_refuses_an_assessment_or_valset_call(self, method, monkeypatch: pytest.MonkeyPatch):
         monkeypatch.delenv("CMPND_CLI_DEV", raising=False)
         sdk = cmpnd.Client(api_key="k", endpoint="http://x")
+        sdk.close()
         kwargs = {"expected_outputs": {}} if method == "create_valset_entry" else {}
-        with pytest.raises(RuntimeError, match=r"CMPND_CLI_DEV=1"):
-            getattr(sdk, method)(*self.GATED[method], **kwargs)
+        with pytest.raises(RuntimeError, match=r"client is closed"):
+            getattr(sdk, method)(*self.CALLS[method], **kwargs)
 
-    def test_every_assessment_and_valset_method_is_gated(self):
+    def test_every_assessment_and_valset_method_is_listed(self):
         public = {name for name in dir(cmpnd.Client) if not name.startswith("_")}
         data = {name for name in public if "assessment" in name or "valset" in name}
-        assert data == set(self.GATED)
+        assert data == set(self.CALLS)
 
 
 # --- Case 7: exceptions and cleanup --------------------------------------
@@ -794,7 +790,7 @@ class TestClientDataSurface:
     client's own credentials, list_* unwrap to lists, and client.traces warns."""
 
     def _client(self, monkeypatch):
-        monkeypatch.setenv("CMPND_CLI_DEV", "1")
+        monkeypatch.delenv("CMPND_CLI_DEV", raising=False)
         monkeypatch.setattr(cmpnd.client._exporter_pool, "flush", lambda cfg: None)
         return cmpnd.Client(api_key="k", endpoint="http://x")
 
@@ -811,6 +807,21 @@ class TestClientDataSurface:
         assert seen["trace_id"] == "t-1" and seen["conclusion"] == "down"
         assert seen["api_key"] == "k" and seen["endpoint"] == "http://x"
 
+    @pytest.mark.parametrize(
+        ("method", "module", "function", "ref"),
+        [
+            ("get_assessment", "_assessments", "get", "as_1"),
+            ("get_valset", "_valsets", "get", "v_1"),
+            ("delete_valset_entry", "_valsets", "delete_entry", "e_1"),
+        ],
+    )
+    def test_a_one_id_call_delegates_with_client_creds(self, method, module, function, ref, monkeypatch):
+        fake = mock.create_autospec(getattr(getattr(cmpnd, module), function), return_value={"id": ref})
+        monkeypatch.setattr(getattr(cmpnd, module), function, fake)
+
+        assert getattr(self._client(monkeypatch), method)(ref) == {"id": ref}
+        fake.assert_called_once_with(ref, api_key="k", endpoint="http://x")
+
     def test_get_deployment_signature_delegates_with_client_creds(self, monkeypatch):
         seen: dict = {}
         answer = object()

{cmpnd-0.69.0 → cmpnd-0.71.0}/tests/test_client_program_names.py

@@ -126,7 +126,6 @@ def _provider():
 
 @pytest.fixture(autouse=True)
 def _clean_slate(monkeypatch: pytest.MonkeyPatch):
-    monkeypatch.delenv("CMPND_CLI_DEV", raising=False)  # a client traces with no dev flag
     monkeypatch.delenv("CMPND_PROGRAM_ID", raising=False)
     configuration._config = None
     saved_callbacks = list(dspy.settings.callbacks or [])
@@ -328,7 +327,6 @@ def minting_server(monkeypatch: pytest.MonkeyPatch):
     def _new_client(**kw: Any) -> httpx.Client:
         return real_new_client(transport=httpx.MockTransport(srv.handler), **kw)
 
-    monkeypatch.setenv("CMPND_CLI_DEV", "1")
     with (
         mock.patch("cmpnd._http.new_client", autospec=True, side_effect=_new_client),
         mock.patch("cmpnd.exporter_http.BatchExporter", autospec=True, side_effect=_factory),