trodo-python 2.16.0__py3-none-any.whl → 2.17.0__py3-none-any.whl

trodo/__init__.py

@@ -119,9 +119,14 @@ __all__ = [
     "TemplateError",
     "ManagedPrompt",
     "PromptSummary",
-    # Datasets & experiments ingest
+    # Datasets & experiments
     "append_dataset",
     "ingest_experiment",
+    "create_experiment",
+    "run_experiment",
+    "get_experiment",
+    "wait_experiment",
+    "compare_experiments",
 ]
 
 # ============================================================================
@@ -405,6 +410,54 @@ def ingest_experiment(
     )
 
 
+def create_experiment(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    models: List[Dict[str, Any]],
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Create an experiment (immutable refs, status pending)."""
+    return _get_client().experiments.create(
+        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
+    )
+
+
+def run_experiment(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    models: List[Dict[str, Any]],
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Create and run an experiment synchronously."""
+    return _get_client().experiments.run(
+        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
+    )
+
+
+def get_experiment(experiment_id: str) -> Dict[str, Any]:
+    """Fetch an experiment by id."""
+    return _get_client().experiments.get(experiment_id)
+
+
+def wait_experiment(
+    experiment_id: str,
+    *,
+    interval_ms: int = 2500,
+    timeout_ms: int = 600_000,
+) -> Dict[str, Any]:
+    """Poll until the experiment completes or fails."""
+    return _get_client().experiments.wait(
+        experiment_id, interval_ms=interval_ms, timeout_ms=timeout_ms
+    )
+
+
+def compare_experiments(a_id: str, b_id: str, *, strict: bool = False) -> Dict[str, Any]:
+    """Compare two experiments side by side."""
+    return _get_client().experiments.compare(a_id, b_id, strict=strict)
+
+
 def enable_auto_events() -> None:
     _get_client().enable_auto_events()
 

trodo/api/endpoints.py

@@ -23,6 +23,10 @@ RUNS_INGEST = "/api/sdk/runs/ingest"
 RUNS_START = "/api/sdk/runs/start"
 RUNS_BASE = "/api/sdk/runs"  # /runs/{run_id}/end, /spans, /feedback
 OTLP_TRACES = "/api/sdk/otel/v1/traces"
-# Datasets & experiments ingest
+# Datasets & experiments
 DATASETS_BASE = "/api/sdk/datasets"  # /datasets/{ref}/items
+EXPERIMENTS = "/api/sdk/experiments"
 EXPERIMENTS_INGEST = "/api/sdk/experiments/ingest"
+EXPERIMENTS_CREATE = "/api/sdk/experiments/create"
+EXPERIMENTS_RUN = "/api/sdk/experiments/run"
+EXPERIMENTS_COMPARE = "/api/sdk/experiments/compare"

trodo/api/http_client.py

@@ -143,6 +143,28 @@ class HttpClient:
     def ingest_experiment(self, body: Dict[str, Any]) -> ApiResult:
         return self._request("/api/sdk/experiments/ingest", body)
 
+    def create_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/create", body)
+
+    def run_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/run", body)
+
+    def run_experiment_by_id(self, experiment_id: str) -> ApiResult:
+        from urllib.parse import quote
+        return self._request(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}/run", {})
+
+    def get_experiment(self, experiment_id: str) -> ApiResult:
+        from urllib.parse import quote
+        return self._get(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}")
+
+    def compare_experiments(
+        self, a_id: str, b_id: str, strict: bool = False
+    ) -> ApiResult:
+        params: Dict[str, Any] = {"a": a_id, "b": b_id}
+        if strict:
+            params["strict"] = "true"
+        return self._get("/api/sdk/experiments/compare", params)
+
     def post_track(self, session_data: Dict[str, Any]) -> ApiResult:
         return self._request("/api/sdk/track", {"sessionData": session_data})
 

trodo/client.py

@@ -137,7 +137,7 @@ class TrodoClient:
 
     @property
     def experiments(self):
-        """Ingest experiment result batches for server-side grading."""
+        """Managed experiment runs + external ingest for server-side grading."""
         if self._experiments is None:
             from .managers.experiment_manager import ExperimentManager
             self._experiments = ExperimentManager(self._http)

trodo/managers/experiment_manager.py

@@ -1,26 +1,25 @@
-"""Experiment ingest — record a batch of model outputs against a dataset.
+"""Experiment manager — managed runs and external ingest.
 
-An experiment takes a dataset (the fixed inputs) and a set of *outputs* your
-model produced for each item, then grades them server-side using the judge
-and/or evaluators you name. You run the model yourself and hand Trodo the
-results; Trodo scores and stores them so runs are comparable over time.
+Managed experiments (prompt + model over a dataset snapshot) use
+``create``, ``run``, ``get``, ``wait``, and ``compare``.
+External agent outputs use ``ingest``.
 
-``outputs`` are matched to dataset items by ``item_position`` (0-based),
-carrying the produced ``output`` and optional per-row context (``query``,
-``context``), cost/latency telemetry, and a per-row ``expected_output`` override.
-
-Mirrors ``sdks/trodo-node-sdk/src/managers/ExperimentManager.ts`` in shape.
+Mirrors ``sdks/trodo-node-sdk/src/managers/ExperimentManager.ts``.
 """
 
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional
+import time
+from typing import Any, Dict, List, Optional, Union
 
 __all__ = ["ExperimentManager"]
 
-# Optional per-output fields, in wire order. ``item_position`` + ``output`` are
-# always required.
+_TERMINAL = frozenset({"completed", "failed"})
+
+# Optional per-output fields for ingest
 _OUTPUT_OPTIONAL = (
+    "item_position",
+    "dataset_item_id",
     "expected_output",
     "query",
     "context",
@@ -32,22 +31,79 @@ _OUTPUT_OPTIONAL = (
 def _clean_output(output: Dict[str, Any]) -> Dict[str, Any]:
     if not isinstance(output, dict):
         raise ValueError("trodo: each experiment output must be a dict")
-    if output.get("item_position") is None:
-        raise ValueError("trodo: each output requires an 'item_position' (int)")
     if output.get("output") is None:
         raise ValueError("trodo: each output requires an 'output' (str)")
-    out: Dict[str, Any] = {
-        "item_position": output["item_position"],
-        "output": output["output"],
-    }
+    if output.get("item_position") is None and output.get("dataset_item_id") is None:
+        raise ValueError(
+            "trodo: each output requires 'item_position' and/or 'dataset_item_id'"
+        )
+    out: Dict[str, Any] = {"output": output["output"]}
     for key in _OUTPUT_OPTIONAL:
         if output.get(key) is not None:
             out[key] = output[key]
     return out
 
 
+def _unwrap_experiment(res: Any) -> Dict[str, Any]:
+    if isinstance(res, dict) and "experiment" in res:
+        return res["experiment"]
+    return res if isinstance(res, dict) else {}
+
+
+def _build_managed_body(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    dataset_version_no: Optional[int] = None,
+    name: Optional[str] = None,
+    prompt_ref: Optional[Dict[str, Any]] = None,
+    prompt_version_id: Optional[str] = None,
+    prompt_id: Optional[str] = None,
+    use_prompt: Optional[bool] = None,
+    models: Optional[List[Dict[str, Any]]] = None,
+    scorer_ids: Optional[List[Union[str, int]]] = None,
+    evaluator_ids: Optional[List[Union[str, int]]] = None,
+    deep_eval: Optional[bool] = None,
+    stability_runs: Optional[int] = None,
+    row_limit: Optional[int] = None,
+    concurrency: Optional[int] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    body: Dict[str, Any] = {"models": models or []}
+    ds = dataset_id or dataset
+    if ds:
+        body["dataset_id"] = ds
+    if dataset_version_no is not None:
+        body["dataset_version_no"] = dataset_version_no
+    if name is not None:
+        body["name"] = name
+    if prompt_ref is not None:
+        body["prompt_ref"] = prompt_ref
+    if prompt_version_id is not None:
+        body["prompt_version_id"] = prompt_version_id
+    if prompt_id is not None:
+        body["prompt_id"] = prompt_id
+    if use_prompt is False:
+        body["use_prompt"] = False
+    ids = scorer_ids if scorer_ids is not None else evaluator_ids
+    if ids is not None:
+        body["scorer_ids"] = ids
+        body["evaluator_ids"] = ids
+    if deep_eval is not None:
+        body["deep_eval"] = deep_eval
+    if stability_runs is not None:
+        body["stability_runs"] = stability_runs
+    if row_limit is not None:
+        body["row_limit"] = row_limit
+    if concurrency is not None:
+        body["concurrency"] = concurrency
+    if metadata is not None:
+        body["metadata"] = metadata
+    return body
+
+
 class ExperimentManager:
-    """Ingest experiment result batches for server-side grading."""
+    """Run and inspect Trodo experiments programmatically."""
 
     def __init__(self, http_client: Any) -> None:
         self._http = http_client
@@ -62,23 +118,7 @@ class ExperimentManager:
         evaluator_ids: Optional[List[str]] = None,
         dataset_version_no: Optional[int] = None,
     ) -> Dict[str, Any]:
-        """Ingest a batch of *outputs* for *dataset* and return the experiment.
-
-        *dataset* is a dataset name or UUID. *outputs* is a list of dicts shaped
-        ``{"item_position": int, "output": str, "expected_output"?, "query"?,
-        "context"?, "cost_usd"?, "latency_ms"?}``. ``None`` optional fields are
-        dropped from the wire payload.
-
-        Grading is configured server-side: pass ``judge`` (``{"credential_id",
-        "provider", "model"}``) for an LLM judge and/or ``evaluator_ids`` for
-        named evaluators. ``dataset_version_no`` pins the dataset version graded
-        against.
-
-        Returns the experiment dict from ``{"experiment": {...}}``.
-
-        :raises ValueError: if *dataset* is empty, *outputs* is empty, or an
-            output is missing ``item_position``/``output``.
-        """
+        """Ingest externally-produced outputs and grade them (external task path)."""
         if not dataset:
             raise ValueError("trodo: ingest_experiment(dataset) requires a dataset")
         if not outputs:
@@ -99,7 +139,148 @@ class ExperimentManager:
         if dataset_version_no is not None:
             body["dataset_version_no"] = dataset_version_no
 
-        res = self._http.ingest_experiment(body)
-        if isinstance(res, dict) and "experiment" in res:
-            return res["experiment"]
-        return res
+        return _unwrap_experiment(self._http.ingest_experiment(body))
+
+    def create(
+        self,
+        *,
+        dataset: Optional[str] = None,
+        dataset_id: Optional[str] = None,
+        models: List[Dict[str, Any]],
+        dataset_version_no: Optional[int] = None,
+        name: Optional[str] = None,
+        prompt_ref: Optional[Dict[str, Any]] = None,
+        prompt_version_id: Optional[str] = None,
+        prompt_id: Optional[str] = None,
+        use_prompt: Optional[bool] = None,
+        scorer_ids: Optional[List[Union[str, int]]] = None,
+        evaluator_ids: Optional[List[Union[str, int]]] = None,
+        deep_eval: Optional[bool] = None,
+        stability_runs: Optional[int] = None,
+        row_limit: Optional[int] = None,
+        concurrency: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Create an experiment (immutable refs, status pending)."""
+        self._validate_run_params(dataset, dataset_id, models)
+        body = _build_managed_body(
+            dataset=dataset,
+            dataset_id=dataset_id,
+            dataset_version_no=dataset_version_no,
+            name=name,
+            prompt_ref=prompt_ref,
+            prompt_version_id=prompt_version_id,
+            prompt_id=prompt_id,
+            use_prompt=use_prompt,
+            models=models,
+            scorer_ids=scorer_ids,
+            evaluator_ids=evaluator_ids,
+            deep_eval=deep_eval,
+            stability_runs=stability_runs,
+            row_limit=row_limit,
+            concurrency=concurrency,
+            metadata=metadata,
+        )
+        return _unwrap_experiment(self._http.create_experiment(body))
+
+    def run(
+        self,
+        *,
+        dataset: Optional[str] = None,
+        dataset_id: Optional[str] = None,
+        models: List[Dict[str, Any]],
+        dataset_version_no: Optional[int] = None,
+        name: Optional[str] = None,
+        prompt_ref: Optional[Dict[str, Any]] = None,
+        prompt_version_id: Optional[str] = None,
+        prompt_id: Optional[str] = None,
+        use_prompt: Optional[bool] = None,
+        scorer_ids: Optional[List[Union[str, int]]] = None,
+        evaluator_ids: Optional[List[Union[str, int]]] = None,
+        deep_eval: Optional[bool] = None,
+        stability_runs: Optional[int] = None,
+        row_limit: Optional[int] = None,
+        concurrency: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Create and execute an experiment synchronously."""
+        self._validate_run_params(dataset, dataset_id, models)
+        body = _build_managed_body(
+            dataset=dataset,
+            dataset_id=dataset_id,
+            dataset_version_no=dataset_version_no,
+            name=name,
+            prompt_ref=prompt_ref,
+            prompt_version_id=prompt_version_id,
+            prompt_id=prompt_id,
+            use_prompt=use_prompt,
+            models=models,
+            scorer_ids=scorer_ids,
+            evaluator_ids=evaluator_ids,
+            deep_eval=deep_eval,
+            stability_runs=stability_runs,
+            row_limit=row_limit,
+            concurrency=concurrency,
+            metadata=metadata,
+        )
+        return _unwrap_experiment(self._http.run_experiment(body))
+
+    def run_by_id(self, experiment_id: str) -> Dict[str, Any]:
+        """Execute a pending experiment created via ``create``."""
+        if not experiment_id:
+            raise ValueError("trodo: run_by_id requires an experiment id")
+        return _unwrap_experiment(self._http.run_experiment_by_id(experiment_id))
+
+    def get(self, experiment_id: str) -> Dict[str, Any]:
+        """Fetch an experiment by id."""
+        if not experiment_id:
+            raise ValueError("trodo: get requires an experiment id")
+        res = self._http.get_experiment(experiment_id)
+        if isinstance(res, dict) and res.get("__error"):
+            raise RuntimeError(f"trodo: get failed (HTTP {res.get('status')})")
+        return _unwrap_experiment(res)
+
+    def wait(
+        self,
+        experiment_id: str,
+        *,
+        interval_ms: int = 2500,
+        timeout_ms: int = 600_000,
+    ) -> Dict[str, Any]:
+        """Poll until the experiment reaches a terminal status."""
+        deadline = time.monotonic() + (timeout_ms / 1000.0)
+        while True:
+            exp = self.get(experiment_id)
+            status = str(exp.get("status") or "").lower()
+            if status in _TERMINAL:
+                return exp
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    f"trodo: wait timed out after {timeout_ms}ms (status: {status or 'unknown'})"
+                )
+            time.sleep(interval_ms / 1000.0)
+
+    def compare(
+        self, a_id: str, b_id: str, *, strict: bool = False
+    ) -> Dict[str, Any]:
+        """Compare two experiments side by side."""
+        if not a_id or not b_id:
+            raise ValueError("trodo: compare requires two experiment ids")
+        res = self._http.compare_experiments(a_id, b_id, strict=strict)
+        if isinstance(res, dict) and res.get("__error"):
+            raise RuntimeError(f"trodo: compare failed (HTTP {res.get('status')})")
+        return res if isinstance(res, dict) else {}
+
+    @staticmethod
+    def _validate_run_params(
+        dataset: Optional[str],
+        dataset_id: Optional[str],
+        models: Optional[List[Dict[str, Any]]],
+    ) -> None:
+        if not models:
+            raise ValueError("trodo: run requires a non-empty models list")
+        if not dataset and not dataset_id:
+            raise ValueError("trodo: run requires dataset or dataset_id")
+        for model in models:
+            if not model.get("credential_id") or not model.get("model"):
+                raise ValueError("trodo: each model needs credential_id and model")

{trodo_python-2.16.0.dist-info → trodo_python-2.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.16.0
+Version: 2.17.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-2.16.0.dist-info → trodo_python-2.17.0.dist-info}/RECORD

@@ -1,16 +1,16 @@
-trodo/__init__.py,sha256=V2vh8SxMkgOaBdowPlHzx_2hpX4T1lUlS2JmE6usz3M,23362
-trodo/client.py,sha256=syz4zzWokdBaFNnO5QhYnUKVNHf8KaunofVFc1ODohI,20187
+trodo/__init__.py,sha256=AzLMYxJTUCWHMNOIzgAVcQpYovCA_885ut3JcKqdHOM,24866
+trodo/client.py,sha256=F5-YJH0Kf9XNQYVFXI0MTGGNEMVN5EskKazZEHszaJc,20196
 trodo/types.py,sha256=eySgUvCXROG2TxtxgiU0MNr5iH0DEcduK8bmYtTKG44,3138
 trodo/user_context.py,sha256=9la6azzwEanVmdP4ps_xMoufbeWVeIGU-M8ychmgajg,7859
 trodo/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/api/async_client.py,sha256=rZN4aJ2QiKyrHBK260bApCUB9JaMWU6BQtzoSJZh7xk,3408
-trodo/api/endpoints.py,sha256=JVTn5l1U-7U2ibQoR9M_AjRgK6GMRgSx_DkV4opWouo,1171
-trodo/api/http_client.py,sha256=rWeRLqVXQ-iXgAIzg6701WtK5eSlPZTTJgKZFerLZhs,8085
+trodo/api/endpoints.py,sha256=TgXhV9WxQ16G1ORukYT9_PdGmx0prBFFa6fl-0GJLIY,1350
+trodo/api/http_client.py,sha256=cF6mFLuTVZgho4TqvO05vM1H6JaxTqujxA6vGeDqaQ8,9032
 trodo/auto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/auto/auto_event_manager.py,sha256=cztuRsRkNoJE5R4NfSfTrTJTGl4jx2Yb-Ncy0aVAPo8,4247
 trodo/managers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/managers/dataset_manager.py,sha256=gx0S8ujG3cbw4IskBNMdceBjDbcEh_nebwKSzEvm7hE,2827
-trodo/managers/experiment_manager.py,sha256=9GO1JUuV7FykaPfZXI8NVvEBDS2CZuyw9I3ywCrZBTI,3916
+trodo/managers/experiment_manager.py,sha256=V-vemLtenfSI5VIcP0HPiXzEy8AyTrRrbJVCA9Z6wjo,10579
 trodo/managers/group_manager.py,sha256=ki3Se3qEoSZfREX63oeDeBmEfZF-ISHLE8azEtLg0tM,3542
 trodo/managers/people_manager.py,sha256=mMVnx40Mlifx6NGgChvohC9ViK6dQu2mkXNHbV8pK1E,2882
 trodo/managers/prompt_manager.py,sha256=jFHkdvDSxvvb9c53EQvBQygdDJ7X-ylNWEY_ag_SeC0,11227
@@ -33,7 +33,7 @@ trodo/queue/event_queue.py,sha256=EVFZrhlq_kwC3jJ2GK0wMhHISf9UzLCZNDnT_aZ2I2A,87
 trodo/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/session/server_session.py,sha256=McsudEiq33XDq3nqxgzBcUvIjQxCMscwEuAPnYXrTjs,2136
 trodo/session/session_manager.py,sha256=JrgH1VeicmtlxPR4dXEuJbxhi23OelkgwW3-9Slv80o,2525
-trodo_python-2.16.0.dist-info/METADATA,sha256=i4J0WgohVDAcQi8OCVa_x6WpDhH9mWeYCEgX8BQ2lLs,23115
-trodo_python-2.16.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-trodo_python-2.16.0.dist-info/top_level.txt,sha256=VCQu1CJWFmNsqTs1YxMcw4Mq35Tc3z3uI9RwHEXAayQ,6
-trodo_python-2.16.0.dist-info/RECORD,,
+trodo_python-2.17.0.dist-info/METADATA,sha256=5MXB-Qq17K35ZJ41p6g0_1JcOrCgLa3-vycGNB8O_vg,23115
+trodo_python-2.17.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+trodo_python-2.17.0.dist-info/top_level.txt,sha256=VCQu1CJWFmNsqTs1YxMcw4Mq35Tc3z3uI9RwHEXAayQ,6
+trodo_python-2.17.0.dist-info/RECORD,,

{trodo_python-2.16.0.dist-info → trodo_python-2.17.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (83.0.0)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any