trodo-python 2.16.0__tar.gz → 2.17.0__tar.gz

{trodo_python-2.16.0 → trodo_python-2.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.16.0
+Version: 2.17.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-2.16.0 → trodo_python-2.17.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trodo-python"
-version = "2.16.0"
+version = "2.17.0"
 description = "Trodo Analytics SDK for Python — server-side event tracking"
 readme = "README.md"
 license = { text = "ISC" }

{trodo_python-2.16.0 → trodo_python-2.17.0}/tests/test_experiments.py

@@ -118,3 +118,59 @@ def test_ingest_requires_item_position_and_output():
         ExperimentManager(FakeHttp()).ingest("ds", [{"output": "o"}])
     with pytest.raises(ValueError, match="requires an 'output'"):
         ExperimentManager(FakeHttp()).ingest("ds", [{"item_position": 0}])
+
+
+class FakeManagedHttp(FakeHttp):
+    def __init__(self):
+        super().__init__()
+        self.create_body = None
+        self.run_body = None
+        self.compare_args = None
+
+    def create_experiment(self, body):
+        self.create_body = body
+        return {"experiment": {"id": "exp_p", "status": "pending"}}
+
+    def run_experiment(self, body):
+        self.run_body = body
+        return {"experiment": {"id": "exp_done", "status": "completed"}}
+
+    def get_experiment(self, experiment_id):
+        return {"experiment": {"id": experiment_id, "status": "completed"}}
+
+    def compare_experiments(self, a_id, b_id, strict=False):
+        self.compare_args = (a_id, b_id, strict)
+        return {"a": {"id": a_id}, "b": {"id": b_id}, "join": "dataset_item_id"}
+
+
+def test_create_posts_managed_body():
+    http = FakeManagedHttp()
+    exp = ExperimentManager(http).create(
+        dataset="golden",
+        models=[{"credential_id": "1", "provider": "openai", "model": "gpt-4o"}],
+        prompt_ref={"name": "support", "version": "abc"},
+        scorer_ids=[1, 2],
+    )
+    assert http.create_body["dataset_id"] == "golden"
+    assert http.create_body["scorer_ids"] == [1, 2]
+    assert exp["status"] == "pending"
+
+
+def test_run_posts_and_returns_completed():
+    http = FakeManagedHttp()
+    exp = ExperimentManager(http).run(
+        dataset="golden",
+        models=[{"credential_id": "1", "provider": "openai", "model": "gpt-4o"}],
+        name="ci",
+    )
+    assert http.run_body["name"] == "ci"
+    assert exp["status"] == "completed"
+
+
+def test_get_and_compare():
+    http = FakeManagedHttp()
+    mgr = ExperimentManager(http)
+    assert mgr.get("exp_1")["id"] == "exp_1"
+    res = mgr.compare("a", "b", strict=True)
+    assert http.compare_args == ("a", "b", True)
+    assert res["join"] == "dataset_item_id"

{trodo_python-2.16.0 → trodo_python-2.17.0}/trodo/__init__.py

@@ -119,9 +119,14 @@ __all__ = [
     "TemplateError",
     "ManagedPrompt",
     "PromptSummary",
-    # Datasets & experiments ingest
+    # Datasets & experiments
     "append_dataset",
     "ingest_experiment",
+    "create_experiment",
+    "run_experiment",
+    "get_experiment",
+    "wait_experiment",
+    "compare_experiments",
 ]
 
 # ============================================================================
@@ -405,6 +410,54 @@ def ingest_experiment(
     )
 
 
+def create_experiment(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    models: List[Dict[str, Any]],
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Create an experiment (immutable refs, status pending)."""
+    return _get_client().experiments.create(
+        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
+    )
+
+
+def run_experiment(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    models: List[Dict[str, Any]],
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Create and run an experiment synchronously."""
+    return _get_client().experiments.run(
+        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
+    )
+
+
+def get_experiment(experiment_id: str) -> Dict[str, Any]:
+    """Fetch an experiment by id."""
+    return _get_client().experiments.get(experiment_id)
+
+
+def wait_experiment(
+    experiment_id: str,
+    *,
+    interval_ms: int = 2500,
+    timeout_ms: int = 600_000,
+) -> Dict[str, Any]:
+    """Poll until the experiment completes or fails."""
+    return _get_client().experiments.wait(
+        experiment_id, interval_ms=interval_ms, timeout_ms=timeout_ms
+    )
+
+
+def compare_experiments(a_id: str, b_id: str, *, strict: bool = False) -> Dict[str, Any]:
+    """Compare two experiments side by side."""
+    return _get_client().experiments.compare(a_id, b_id, strict=strict)
+
+
 def enable_auto_events() -> None:
     _get_client().enable_auto_events()
 

{trodo_python-2.16.0 → trodo_python-2.17.0}/trodo/api/endpoints.py

@@ -23,6 +23,10 @@ RUNS_INGEST = "/api/sdk/runs/ingest"
 RUNS_START = "/api/sdk/runs/start"
 RUNS_BASE = "/api/sdk/runs"  # /runs/{run_id}/end, /spans, /feedback
 OTLP_TRACES = "/api/sdk/otel/v1/traces"
-# Datasets & experiments ingest
+# Datasets & experiments
 DATASETS_BASE = "/api/sdk/datasets"  # /datasets/{ref}/items
+EXPERIMENTS = "/api/sdk/experiments"
 EXPERIMENTS_INGEST = "/api/sdk/experiments/ingest"
+EXPERIMENTS_CREATE = "/api/sdk/experiments/create"
+EXPERIMENTS_RUN = "/api/sdk/experiments/run"
+EXPERIMENTS_COMPARE = "/api/sdk/experiments/compare"

{trodo_python-2.16.0 → trodo_python-2.17.0}/trodo/api/http_client.py

@@ -143,6 +143,28 @@ class HttpClient:
     def ingest_experiment(self, body: Dict[str, Any]) -> ApiResult:
         return self._request("/api/sdk/experiments/ingest", body)
 
+    def create_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/create", body)
+
+    def run_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/run", body)
+
+    def run_experiment_by_id(self, experiment_id: str) -> ApiResult:
+        from urllib.parse import quote
+        return self._request(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}/run", {})
+
+    def get_experiment(self, experiment_id: str) -> ApiResult:
+        from urllib.parse import quote
+        return self._get(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}")
+
+    def compare_experiments(
+        self, a_id: str, b_id: str, strict: bool = False
+    ) -> ApiResult:
+        params: Dict[str, Any] = {"a": a_id, "b": b_id}
+        if strict:
+            params["strict"] = "true"
+        return self._get("/api/sdk/experiments/compare", params)
+
     def post_track(self, session_data: Dict[str, Any]) -> ApiResult:
         return self._request("/api/sdk/track", {"sessionData": session_data})
 

{trodo_python-2.16.0 → trodo_python-2.17.0}/trodo/client.py

@@ -137,7 +137,7 @@ class TrodoClient:
 
     @property
     def experiments(self):
-        """Ingest experiment result batches for server-side grading."""
+        """Managed experiment runs + external ingest for server-side grading."""
         if self._experiments is None:
             from .managers.experiment_manager import ExperimentManager
             self._experiments = ExperimentManager(self._http)

trodo_python-2.17.0/trodo/managers/experiment_manager.py

@@ -0,0 +1,286 @@
+"""Experiment manager — managed runs and external ingest.
+
+Managed experiments (prompt + model over a dataset snapshot) use
+``create``, ``run``, ``get``, ``wait``, and ``compare``.
+External agent outputs use ``ingest``.
+
+Mirrors ``sdks/trodo-node-sdk/src/managers/ExperimentManager.ts``.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List, Optional, Union
+
+__all__ = ["ExperimentManager"]
+
+_TERMINAL = frozenset({"completed", "failed"})
+
+# Optional per-output fields for ingest
+_OUTPUT_OPTIONAL = (
+    "item_position",
+    "dataset_item_id",
+    "expected_output",
+    "query",
+    "context",
+    "cost_usd",
+    "latency_ms",
+)
+
+
+def _clean_output(output: Dict[str, Any]) -> Dict[str, Any]:
+    if not isinstance(output, dict):
+        raise ValueError("trodo: each experiment output must be a dict")
+    if output.get("output") is None:
+        raise ValueError("trodo: each output requires an 'output' (str)")
+    if output.get("item_position") is None and output.get("dataset_item_id") is None:
+        raise ValueError(
+            "trodo: each output requires 'item_position' and/or 'dataset_item_id'"
+        )
+    out: Dict[str, Any] = {"output": output["output"]}
+    for key in _OUTPUT_OPTIONAL:
+        if output.get(key) is not None:
+            out[key] = output[key]
+    return out
+
+
+def _unwrap_experiment(res: Any) -> Dict[str, Any]:
+    if isinstance(res, dict) and "experiment" in res:
+        return res["experiment"]
+    return res if isinstance(res, dict) else {}
+
+
+def _build_managed_body(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    dataset_version_no: Optional[int] = None,
+    name: Optional[str] = None,
+    prompt_ref: Optional[Dict[str, Any]] = None,
+    prompt_version_id: Optional[str] = None,
+    prompt_id: Optional[str] = None,
+    use_prompt: Optional[bool] = None,
+    models: Optional[List[Dict[str, Any]]] = None,
+    scorer_ids: Optional[List[Union[str, int]]] = None,
+    evaluator_ids: Optional[List[Union[str, int]]] = None,
+    deep_eval: Optional[bool] = None,
+    stability_runs: Optional[int] = None,
+    row_limit: Optional[int] = None,
+    concurrency: Optional[int] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    body: Dict[str, Any] = {"models": models or []}
+    ds = dataset_id or dataset
+    if ds:
+        body["dataset_id"] = ds
+    if dataset_version_no is not None:
+        body["dataset_version_no"] = dataset_version_no
+    if name is not None:
+        body["name"] = name
+    if prompt_ref is not None:
+        body["prompt_ref"] = prompt_ref
+    if prompt_version_id is not None:
+        body["prompt_version_id"] = prompt_version_id
+    if prompt_id is not None:
+        body["prompt_id"] = prompt_id
+    if use_prompt is False:
+        body["use_prompt"] = False
+    ids = scorer_ids if scorer_ids is not None else evaluator_ids
+    if ids is not None:
+        body["scorer_ids"] = ids
+        body["evaluator_ids"] = ids
+    if deep_eval is not None:
+        body["deep_eval"] = deep_eval
+    if stability_runs is not None:
+        body["stability_runs"] = stability_runs
+    if row_limit is not None:
+        body["row_limit"] = row_limit
+    if concurrency is not None:
+        body["concurrency"] = concurrency
+    if metadata is not None:
+        body["metadata"] = metadata
+    return body
+
+
+class ExperimentManager:
+    """Run and inspect Trodo experiments programmatically."""
+
+    def __init__(self, http_client: Any) -> None:
+        self._http = http_client
+
+    def ingest(
+        self,
+        dataset: str,
+        outputs: List[Dict[str, Any]],
+        name: Optional[str] = None,
+        task_name: Optional[str] = None,
+        judge: Optional[Dict[str, Any]] = None,
+        evaluator_ids: Optional[List[str]] = None,
+        dataset_version_no: Optional[int] = None,
+    ) -> Dict[str, Any]:
+        """Ingest externally-produced outputs and grade them (external task path)."""
+        if not dataset:
+            raise ValueError("trodo: ingest_experiment(dataset) requires a dataset")
+        if not outputs:
+            raise ValueError("trodo: ingest_experiment requires outputs")
+
+        body: Dict[str, Any] = {
+            "dataset": dataset,
+            "outputs": [_clean_output(o) for o in outputs],
+        }
+        if name is not None:
+            body["name"] = name
+        if task_name is not None:
+            body["task_name"] = task_name
+        if judge is not None:
+            body["judge"] = judge
+        if evaluator_ids is not None:
+            body["evaluator_ids"] = evaluator_ids
+        if dataset_version_no is not None:
+            body["dataset_version_no"] = dataset_version_no
+
+        return _unwrap_experiment(self._http.ingest_experiment(body))
+
+    def create(
+        self,
+        *,
+        dataset: Optional[str] = None,
+        dataset_id: Optional[str] = None,
+        models: List[Dict[str, Any]],
+        dataset_version_no: Optional[int] = None,
+        name: Optional[str] = None,
+        prompt_ref: Optional[Dict[str, Any]] = None,
+        prompt_version_id: Optional[str] = None,
+        prompt_id: Optional[str] = None,
+        use_prompt: Optional[bool] = None,
+        scorer_ids: Optional[List[Union[str, int]]] = None,
+        evaluator_ids: Optional[List[Union[str, int]]] = None,
+        deep_eval: Optional[bool] = None,
+        stability_runs: Optional[int] = None,
+        row_limit: Optional[int] = None,
+        concurrency: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Create an experiment (immutable refs, status pending)."""
+        self._validate_run_params(dataset, dataset_id, models)
+        body = _build_managed_body(
+            dataset=dataset,
+            dataset_id=dataset_id,
+            dataset_version_no=dataset_version_no,
+            name=name,
+            prompt_ref=prompt_ref,
+            prompt_version_id=prompt_version_id,
+            prompt_id=prompt_id,
+            use_prompt=use_prompt,
+            models=models,
+            scorer_ids=scorer_ids,
+            evaluator_ids=evaluator_ids,
+            deep_eval=deep_eval,
+            stability_runs=stability_runs,
+            row_limit=row_limit,
+            concurrency=concurrency,
+            metadata=metadata,
+        )
+        return _unwrap_experiment(self._http.create_experiment(body))
+
+    def run(
+        self,
+        *,
+        dataset: Optional[str] = None,
+        dataset_id: Optional[str] = None,
+        models: List[Dict[str, Any]],
+        dataset_version_no: Optional[int] = None,
+        name: Optional[str] = None,
+        prompt_ref: Optional[Dict[str, Any]] = None,
+        prompt_version_id: Optional[str] = None,
+        prompt_id: Optional[str] = None,
+        use_prompt: Optional[bool] = None,
+        scorer_ids: Optional[List[Union[str, int]]] = None,
+        evaluator_ids: Optional[List[Union[str, int]]] = None,
+        deep_eval: Optional[bool] = None,
+        stability_runs: Optional[int] = None,
+        row_limit: Optional[int] = None,
+        concurrency: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Create and execute an experiment synchronously."""
+        self._validate_run_params(dataset, dataset_id, models)
+        body = _build_managed_body(
+            dataset=dataset,
+            dataset_id=dataset_id,
+            dataset_version_no=dataset_version_no,
+            name=name,
+            prompt_ref=prompt_ref,
+            prompt_version_id=prompt_version_id,
+            prompt_id=prompt_id,
+            use_prompt=use_prompt,
+            models=models,
+            scorer_ids=scorer_ids,
+            evaluator_ids=evaluator_ids,
+            deep_eval=deep_eval,
+            stability_runs=stability_runs,
+            row_limit=row_limit,
+            concurrency=concurrency,
+            metadata=metadata,
+        )
+        return _unwrap_experiment(self._http.run_experiment(body))
+
+    def run_by_id(self, experiment_id: str) -> Dict[str, Any]:
+        """Execute a pending experiment created via ``create``."""
+        if not experiment_id:
+            raise ValueError("trodo: run_by_id requires an experiment id")
+        return _unwrap_experiment(self._http.run_experiment_by_id(experiment_id))
+
+    def get(self, experiment_id: str) -> Dict[str, Any]:
+        """Fetch an experiment by id."""
+        if not experiment_id:
+            raise ValueError("trodo: get requires an experiment id")
+        res = self._http.get_experiment(experiment_id)
+        if isinstance(res, dict) and res.get("__error"):
+            raise RuntimeError(f"trodo: get failed (HTTP {res.get('status')})")
+        return _unwrap_experiment(res)
+
+    def wait(
+        self,
+        experiment_id: str,
+        *,
+        interval_ms: int = 2500,
+        timeout_ms: int = 600_000,
+    ) -> Dict[str, Any]:
+        """Poll until the experiment reaches a terminal status."""
+        deadline = time.monotonic() + (timeout_ms / 1000.0)
+        while True:
+            exp = self.get(experiment_id)
+            status = str(exp.get("status") or "").lower()
+            if status in _TERMINAL:
+                return exp
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    f"trodo: wait timed out after {timeout_ms}ms (status: {status or 'unknown'})"
+                )
+            time.sleep(interval_ms / 1000.0)
+
+    def compare(
+        self, a_id: str, b_id: str, *, strict: bool = False
+    ) -> Dict[str, Any]:
+        """Compare two experiments side by side."""
+        if not a_id or not b_id:
+            raise ValueError("trodo: compare requires two experiment ids")
+        res = self._http.compare_experiments(a_id, b_id, strict=strict)
+        if isinstance(res, dict) and res.get("__error"):
+            raise RuntimeError(f"trodo: compare failed (HTTP {res.get('status')})")
+        return res if isinstance(res, dict) else {}
+
+    @staticmethod
+    def _validate_run_params(
+        dataset: Optional[str],
+        dataset_id: Optional[str],
+        models: Optional[List[Dict[str, Any]]],
+    ) -> None:
+        if not models:
+            raise ValueError("trodo: run requires a non-empty models list")
+        if not dataset and not dataset_id:
+            raise ValueError("trodo: run requires dataset or dataset_id")
+        for model in models:
+            if not model.get("credential_id") or not model.get("model"):
+                raise ValueError("trodo: each model needs credential_id and model")

{trodo_python-2.16.0 → trodo_python-2.17.0}/trodo_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.16.0
+Version: 2.17.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

trodo_python-2.16.0/trodo/managers/experiment_manager.py

@@ -1,105 +0,0 @@
-"""Experiment ingest — record a batch of model outputs against a dataset.
-
-An experiment takes a dataset (the fixed inputs) and a set of *outputs* your
-model produced for each item, then grades them server-side using the judge
-and/or evaluators you name. You run the model yourself and hand Trodo the
-results; Trodo scores and stores them so runs are comparable over time.
-
-``outputs`` are matched to dataset items by ``item_position`` (0-based),
-carrying the produced ``output`` and optional per-row context (``query``,
-``context``), cost/latency telemetry, and a per-row ``expected_output`` override.
-
-Mirrors ``sdks/trodo-node-sdk/src/managers/ExperimentManager.ts`` in shape.
-"""
-
-from __future__ import annotations
-
-from typing import Any, Dict, List, Optional
-
-__all__ = ["ExperimentManager"]
-
-# Optional per-output fields, in wire order. ``item_position`` + ``output`` are
-# always required.
-_OUTPUT_OPTIONAL = (
-    "expected_output",
-    "query",
-    "context",
-    "cost_usd",
-    "latency_ms",
-)
-
-
-def _clean_output(output: Dict[str, Any]) -> Dict[str, Any]:
-    if not isinstance(output, dict):
-        raise ValueError("trodo: each experiment output must be a dict")
-    if output.get("item_position") is None:
-        raise ValueError("trodo: each output requires an 'item_position' (int)")
-    if output.get("output") is None:
-        raise ValueError("trodo: each output requires an 'output' (str)")
-    out: Dict[str, Any] = {
-        "item_position": output["item_position"],
-        "output": output["output"],
-    }
-    for key in _OUTPUT_OPTIONAL:
-        if output.get(key) is not None:
-            out[key] = output[key]
-    return out
-
-
-class ExperimentManager:
-    """Ingest experiment result batches for server-side grading."""
-
-    def __init__(self, http_client: Any) -> None:
-        self._http = http_client
-
-    def ingest(
-        self,
-        dataset: str,
-        outputs: List[Dict[str, Any]],
-        name: Optional[str] = None,
-        task_name: Optional[str] = None,
-        judge: Optional[Dict[str, Any]] = None,
-        evaluator_ids: Optional[List[str]] = None,
-        dataset_version_no: Optional[int] = None,
-    ) -> Dict[str, Any]:
-        """Ingest a batch of *outputs* for *dataset* and return the experiment.
-
-        *dataset* is a dataset name or UUID. *outputs* is a list of dicts shaped
-        ``{"item_position": int, "output": str, "expected_output"?, "query"?,
-        "context"?, "cost_usd"?, "latency_ms"?}``. ``None`` optional fields are
-        dropped from the wire payload.
-
-        Grading is configured server-side: pass ``judge`` (``{"credential_id",
-        "provider", "model"}``) for an LLM judge and/or ``evaluator_ids`` for
-        named evaluators. ``dataset_version_no`` pins the dataset version graded
-        against.
-
-        Returns the experiment dict from ``{"experiment": {...}}``.
-
-        :raises ValueError: if *dataset* is empty, *outputs* is empty, or an
-            output is missing ``item_position``/``output``.
-        """
-        if not dataset:
-            raise ValueError("trodo: ingest_experiment(dataset) requires a dataset")
-        if not outputs:
-            raise ValueError("trodo: ingest_experiment requires outputs")
-
-        body: Dict[str, Any] = {
-            "dataset": dataset,
-            "outputs": [_clean_output(o) for o in outputs],
-        }
-        if name is not None:
-            body["name"] = name
-        if task_name is not None:
-            body["task_name"] = task_name
-        if judge is not None:
-            body["judge"] = judge
-        if evaluator_ids is not None:
-            body["evaluator_ids"] = evaluator_ids
-        if dataset_version_no is not None:
-            body["dataset_version_no"] = dataset_version_no
-
-        res = self._http.ingest_experiment(body)
-        if isinstance(res, dict) and "experiment" in res:
-            return res["experiment"]
-        return res