trodo-python 2.15.0__tar.gz → 2.17.0__tar.gz

{trodo_python-2.15.0 → trodo_python-2.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.15.0
+Version: 2.17.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-2.15.0 → trodo_python-2.17.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trodo-python"
-version = "2.15.0"
+version = "2.17.0"
 description = "Trodo Analytics SDK for Python — server-side event tracking"
 readme = "README.md"
 license = { text = "ISC" }

trodo_python-2.17.0/tests/test_datasets.py

@@ -0,0 +1,102 @@
+"""Dataset ingest — DatasetManager.append.
+
+Uses a fake HTTP client so no network is touched; asserts on the exact
+ref + body sent to the backend and the response returned to the caller.
+"""
+
+import pytest
+
+from trodo.managers.dataset_manager import DatasetManager
+
+
+class FakeHttp:
+    """Records the last append call; mirrors the prompt-test fakeApi."""
+
+    def __init__(self, response=None):
+        self._response = response or {
+            "dataset_id": "ds_1",
+            "dataset_name": "support-eval",
+            "appended": 1,
+            "item_count": 1,
+        }
+        self.calls = 0
+        self.last_ref = None
+        self.last_body = None
+
+    def append_dataset_items(self, ref, body):
+        self.calls += 1
+        self.last_ref = ref
+        self.last_body = body
+        return self._response
+
+
+def test_append_posts_ref_and_body_and_returns_response():
+    http = FakeHttp()
+    mgr = DatasetManager(http)
+    res = mgr.append(
+        "support-eval",
+        [{"input": {"q": "where is my order"}, "expected_output": "tracking link"}],
+    )
+
+    assert http.calls == 1
+    assert http.last_ref == "support-eval"
+    assert http.last_body == {
+        "items": [
+            {"input": {"q": "where is my order"}, "expected_output": "tracking link"}
+        ],
+        "create": True,
+    }
+    assert res["dataset_id"] == "ds_1"
+    assert res["item_count"] == 1
+
+
+def test_append_wraps_a_single_dict_into_a_list():
+    http = FakeHttp()
+    DatasetManager(http).append("ds", {"input": {"x": 1}})
+    assert http.last_body["items"] == [{"input": {"x": 1}}]
+
+
+def test_append_drops_none_optional_fields():
+    http = FakeHttp()
+    DatasetManager(http).append(
+        "ds",
+        [
+            {
+                "input": {"x": 1},
+                "expected_output": None,
+                "metadata": {"k": "v"},
+                "source_trace_ref": None,
+            }
+        ],
+    )
+    # None fields dropped; present ones kept.
+    assert http.last_body["items"] == [{"input": {"x": 1}, "metadata": {"k": "v"}}]
+
+
+def test_append_passes_create_false_through():
+    http = FakeHttp()
+    DatasetManager(http).append("ds", [{"input": {"x": 1}}], create=False)
+    assert http.last_body["create"] is False
+
+
+def test_append_keeps_source_trace_ref():
+    http = FakeHttp()
+    DatasetManager(http).append(
+        "ds", [{"input": {"x": 1}, "source_trace_ref": {"run_id": "r1"}}]
+    )
+    assert http.last_body["items"][0]["source_trace_ref"] == {"run_id": "r1"}
+
+
+def test_append_requires_a_ref():
+    with pytest.raises(ValueError, match="requires a dataset ref"):
+        DatasetManager(FakeHttp()).append("", [{"input": {"x": 1}}])
+
+
+def test_append_requires_items():
+    with pytest.raises(ValueError, match="requires items"):
+        DatasetManager(FakeHttp()).append("ds", [])
+
+
+def test_append_requires_input_on_each_item():
+    with pytest.raises(ValueError, match="requires an 'input'"):
+        DatasetManager(FakeHttp()).append("ds", [{"expected_output": "x"}])

trodo_python-2.17.0/tests/test_experiments.py

@@ -0,0 +1,176 @@
+"""Experiment ingest — ExperimentManager.ingest.
+
+Uses a fake HTTP client so no network is touched; asserts on the exact body
+posted to experiments/ingest and the experiment dict returned to the caller.
+"""
+
+import pytest
+
+from trodo.managers.experiment_manager import ExperimentManager
+
+
+class FakeHttp:
+    """Records the last ingest call; mirrors the prompt-test fakeApi."""
+
+    def __init__(self, response=None):
+        self._response = response if response is not None else {
+            "experiment": {"id": "exp_1", "status": "grading"}
+        }
+        self.calls = 0
+        self.last_body = None
+
+    def ingest_experiment(self, body):
+        self.calls += 1
+        self.last_body = body
+        return self._response
+
+
+def test_ingest_posts_body_and_returns_experiment():
+    http = FakeHttp()
+    mgr = ExperimentManager(http)
+    exp = mgr.ingest(
+        "support-eval",
+        [{"item_position": 0, "output": "here is your tracking link"}],
+    )
+
+    assert http.calls == 1
+    assert http.last_body == {
+        "dataset": "support-eval",
+        "outputs": [{"item_position": 0, "output": "here is your tracking link"}],
+    }
+    assert exp == {"id": "exp_1", "status": "grading"}
+
+
+def test_ingest_includes_all_optional_fields_when_set():
+    http = FakeHttp()
+    ExperimentManager(http).ingest(
+        "ds",
+        [{"item_position": 1, "output": "o"}],
+        name="run-1",
+        task_name="qa",
+        judge={"credential_id": "cred_1", "provider": "openai", "model": "gpt-4o"},
+        evaluator_ids=["eval_a", "eval_b"],
+        dataset_version_no=3,
+    )
+    assert http.last_body == {
+        "dataset": "ds",
+        "outputs": [{"item_position": 1, "output": "o"}],
+        "name": "run-1",
+        "task_name": "qa",
+        "judge": {"credential_id": "cred_1", "provider": "openai", "model": "gpt-4o"},
+        "evaluator_ids": ["eval_a", "eval_b"],
+        "dataset_version_no": 3,
+    }
+
+
+def test_ingest_omits_unset_optional_fields():
+    http = FakeHttp()
+    ExperimentManager(http).ingest("ds", [{"item_position": 0, "output": "o"}])
+    assert set(http.last_body.keys()) == {"dataset", "outputs"}
+
+
+def test_ingest_drops_none_per_output_fields():
+    http = FakeHttp()
+    ExperimentManager(http).ingest(
+        "ds",
+        [
+            {
+                "item_position": 0,
+                "output": "o",
+                "expected_output": "gold",
+                "query": None,
+                "context": None,
+                "cost_usd": 0.0021,
+                "latency_ms": None,
+            }
+        ],
+    )
+    assert http.last_body["outputs"] == [
+        {"item_position": 0, "output": "o", "expected_output": "gold", "cost_usd": 0.0021}
+    ]
+
+
+def test_ingest_keeps_item_position_zero():
+    # item_position 0 is falsy but valid — must not be dropped.
+    http = FakeHttp()
+    ExperimentManager(http).ingest("ds", [{"item_position": 0, "output": "o"}])
+    assert http.last_body["outputs"][0]["item_position"] == 0
+
+
+def test_ingest_returns_raw_response_when_no_experiment_key():
+    http = FakeHttp(response={"error": "bad"})
+    out = ExperimentManager(http).ingest("ds", [{"item_position": 0, "output": "o"}])
+    assert out == {"error": "bad"}
+
+
+def test_ingest_requires_a_dataset():
+    with pytest.raises(ValueError, match="requires a dataset"):
+        ExperimentManager(FakeHttp()).ingest("", [{"item_position": 0, "output": "o"}])
+
+
+def test_ingest_requires_outputs():
+    with pytest.raises(ValueError, match="requires outputs"):
+        ExperimentManager(FakeHttp()).ingest("ds", [])
+
+
+def test_ingest_requires_item_position_and_output():
+    with pytest.raises(ValueError, match="item_position"):
+        ExperimentManager(FakeHttp()).ingest("ds", [{"output": "o"}])
+    with pytest.raises(ValueError, match="requires an 'output'"):
+        ExperimentManager(FakeHttp()).ingest("ds", [{"item_position": 0}])
+
+
+class FakeManagedHttp(FakeHttp):
+    def __init__(self):
+        super().__init__()
+        self.create_body = None
+        self.run_body = None
+        self.compare_args = None
+
+    def create_experiment(self, body):
+        self.create_body = body
+        return {"experiment": {"id": "exp_p", "status": "pending"}}
+
+    def run_experiment(self, body):
+        self.run_body = body
+        return {"experiment": {"id": "exp_done", "status": "completed"}}
+
+    def get_experiment(self, experiment_id):
+        return {"experiment": {"id": experiment_id, "status": "completed"}}
+
+    def compare_experiments(self, a_id, b_id, strict=False):
+        self.compare_args = (a_id, b_id, strict)
+        return {"a": {"id": a_id}, "b": {"id": b_id}, "join": "dataset_item_id"}
+
+
+def test_create_posts_managed_body():
+    http = FakeManagedHttp()
+    exp = ExperimentManager(http).create(
+        dataset="golden",
+        models=[{"credential_id": "1", "provider": "openai", "model": "gpt-4o"}],
+        prompt_ref={"name": "support", "version": "abc"},
+        scorer_ids=[1, 2],
+    )
+    assert http.create_body["dataset_id"] == "golden"
+    assert http.create_body["scorer_ids"] == [1, 2]
+    assert exp["status"] == "pending"
+
+
+def test_run_posts_and_returns_completed():
+    http = FakeManagedHttp()
+    exp = ExperimentManager(http).run(
+        dataset="golden",
+        models=[{"credential_id": "1", "provider": "openai", "model": "gpt-4o"}],
+        name="ci",
+    )
+    assert http.run_body["name"] == "ci"
+    assert exp["status"] == "completed"
+
+
+def test_get_and_compare():
+    http = FakeManagedHttp()
+    mgr = ExperimentManager(http)
+    assert mgr.get("exp_1")["id"] == "exp_1"
+    res = mgr.compare("a", "b", strict=True)
+    assert http.compare_args == ("a", "b", True)
+    assert res["join"] == "dataset_item_id"

{trodo_python-2.15.0 → trodo_python-2.17.0}/trodo/__init__.py

@@ -41,7 +41,7 @@ Downstream microservice (join the caller's run instead of making a new one):
 
 from __future__ import annotations
 
-__version__ = "2.15.0"
+__version__ = "2.16.0"
 
 from typing import Any, Callable, Dict, List, Optional, Union
 
@@ -119,6 +119,14 @@ __all__ = [
     "TemplateError",
     "ManagedPrompt",
     "PromptSummary",
+    # Datasets & experiments
+    "append_dataset",
+    "ingest_experiment",
+    "create_experiment",
+    "run_experiment",
+    "get_experiment",
+    "wait_experiment",
+    "compare_experiments",
 ]
 
 # ============================================================================
@@ -344,6 +352,112 @@ def render_template(
     return _render_template(template, merged, strict=strict)
 
 
+# ----------------------------------------------------------------------------
+# Datasets & experiments ingest
+# ----------------------------------------------------------------------------
+
+def append_dataset(
+    ref: str,
+    items: Union[Dict[str, Any], List[Dict[str, Any]]],
+    create: bool = True,
+) -> Dict[str, Any]:
+    """Append items to a dataset by UUID or name.
+
+    *items* is a list of dicts shaped ``{"input": {<var>: value, ...},
+    "expected_output"?: str, "metadata"?: dict, "source_trace_ref"?: dict}``;
+    a single dict is accepted and wrapped. When *ref* is a name that doesn't
+    exist, ``create=True`` (default) auto-creates the dataset::
+
+        trodo.append_dataset("support-eval", [
+            {"input": {"q": "where is my order"}, "expected_output": "tracking link"},
+        ])
+
+    Returns ``{"dataset_id", "dataset_name", "appended", "item_count"}``.
+    """
+    return _get_client().datasets.append(ref, items, create=create)
+
+
+def ingest_experiment(
+    dataset: str,
+    outputs: List[Dict[str, Any]],
+    name: Optional[str] = None,
+    task_name: Optional[str] = None,
+    judge: Optional[Dict[str, Any]] = None,
+    evaluator_ids: Optional[List[str]] = None,
+    dataset_version_no: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Ingest a batch of model *outputs* for *dataset* and grade them.
+
+    *outputs* is a list of dicts shaped ``{"item_position": int, "output": str,
+    "expected_output"?, "query"?, "context"?, "cost_usd"?, "latency_ms"?}``.
+    Configure grading with ``judge`` (``{"credential_id", "provider", "model"}``)
+    and/or ``evaluator_ids``::
+
+        exp = trodo.ingest_experiment("support-eval", [
+            {"item_position": 0, "output": "here is your tracking link"},
+        ], judge={"credential_id": "cred_1", "provider": "openai", "model": "gpt-4o"})
+
+    Returns the experiment dict.
+    """
+    return _get_client().experiments.ingest(
+        dataset,
+        outputs,
+        name=name,
+        task_name=task_name,
+        judge=judge,
+        evaluator_ids=evaluator_ids,
+        dataset_version_no=dataset_version_no,
+    )
+
+
+def create_experiment(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    models: List[Dict[str, Any]],
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Create an experiment (immutable refs, status pending)."""
+    return _get_client().experiments.create(
+        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
+    )
+
+
+def run_experiment(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    models: List[Dict[str, Any]],
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Create and run an experiment synchronously."""
+    return _get_client().experiments.run(
+        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
+    )
+
+
+def get_experiment(experiment_id: str) -> Dict[str, Any]:
+    """Fetch an experiment by id."""
+    return _get_client().experiments.get(experiment_id)
+
+
+def wait_experiment(
+    experiment_id: str,
+    *,
+    interval_ms: int = 2500,
+    timeout_ms: int = 600_000,
+) -> Dict[str, Any]:
+    """Poll until the experiment completes or fails."""
+    return _get_client().experiments.wait(
+        experiment_id, interval_ms=interval_ms, timeout_ms=timeout_ms
+    )
+
+
+def compare_experiments(a_id: str, b_id: str, *, strict: bool = False) -> Dict[str, Any]:
+    """Compare two experiments side by side."""
+    return _get_client().experiments.compare(a_id, b_id, strict=strict)
+
+
 def enable_auto_events() -> None:
     _get_client().enable_auto_events()
 

{trodo_python-2.15.0 → trodo_python-2.17.0}/trodo/api/endpoints.py

@@ -23,3 +23,10 @@ RUNS_INGEST = "/api/sdk/runs/ingest"
 RUNS_START = "/api/sdk/runs/start"
 RUNS_BASE = "/api/sdk/runs"  # /runs/{run_id}/end, /spans, /feedback
 OTLP_TRACES = "/api/sdk/otel/v1/traces"
+# Datasets & experiments
+DATASETS_BASE = "/api/sdk/datasets"  # /datasets/{ref}/items
+EXPERIMENTS = "/api/sdk/experiments"
+EXPERIMENTS_INGEST = "/api/sdk/experiments/ingest"
+EXPERIMENTS_CREATE = "/api/sdk/experiments/create"
+EXPERIMENTS_RUN = "/api/sdk/experiments/run"
+EXPERIMENTS_COMPARE = "/api/sdk/experiments/compare"

{trodo_python-2.15.0 → trodo_python-2.17.0}/trodo/api/http_client.py

@@ -133,6 +133,38 @@ class HttpClient:
     def list_prompts(self) -> ApiResult:
         return self._get("/api/sdk/prompts")
 
+    def append_dataset_items(self, ref: str, body: Dict[str, Any]) -> ApiResult:
+        """Append items to a dataset by UUID or name (name is URL-encoded)."""
+        from urllib.parse import quote
+        return self._request(
+            f"/api/sdk/datasets/{quote(str(ref), safe='')}/items", body
+        )
+
+    def ingest_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/ingest", body)
+
+    def create_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/create", body)
+
+    def run_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/run", body)
+
+    def run_experiment_by_id(self, experiment_id: str) -> ApiResult:
+        from urllib.parse import quote
+        return self._request(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}/run", {})
+
+    def get_experiment(self, experiment_id: str) -> ApiResult:
+        from urllib.parse import quote
+        return self._get(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}")
+
+    def compare_experiments(
+        self, a_id: str, b_id: str, strict: bool = False
+    ) -> ApiResult:
+        params: Dict[str, Any] = {"a": a_id, "b": b_id}
+        if strict:
+            params["strict"] = "true"
+        return self._get("/api/sdk/experiments/compare", params)
+
     def post_track(self, session_data: Dict[str, Any]) -> ApiResult:
         return self._request("/api/sdk/track", {"sessionData": session_data})
 

{trodo_python-2.15.0 → trodo_python-2.17.0}/trodo/client.py

@@ -77,6 +77,8 @@ class TrodoClient:
 
         self._session_manager = SessionManager()
         self._prompts = None  # lazily-built PromptManager
+        self._datasets = None  # lazily-built DatasetManager
+        self._experiments = None  # lazily-built ExperimentManager
 
         if batch_enabled:
             self._event_queue: Optional[EventQueue] = EventQueue(batch_size)
@@ -125,6 +127,22 @@ class TrodoClient:
             self._prompts = PromptManager(self._http)
         return self._prompts
 
+    @property
+    def datasets(self):
+        """Append items to the team's evaluation datasets."""
+        if self._datasets is None:
+            from .managers.dataset_manager import DatasetManager
+            self._datasets = DatasetManager(self._http)
+        return self._datasets
+
+    @property
+    def experiments(self):
+        """Managed experiment runs + external ingest for server-side grading."""
+        if self._experiments is None:
+            from .managers.experiment_manager import ExperimentManager
+            self._experiments = ExperimentManager(self._http)
+        return self._experiments
+
     # --------------------------------------------------------------------------
     # Primary pattern: for_user()
     # --------------------------------------------------------------------------

trodo_python-2.17.0/trodo/managers/dataset_manager.py

@@ -0,0 +1,74 @@
+"""Dataset ingest — append evaluation items to a Trodo dataset.
+
+Datasets are the fixed inputs an experiment runs against: each item is a set of
+variable values (``input``) plus an optional gold answer (``expected_output``)
+and free-form ``metadata``. This manager appends items to a dataset addressed by
+UUID or by name; when a name is used and no such dataset exists it is
+auto-created (``create=True``, the default), so a first append bootstraps the
+dataset without a separate create call.
+
+Mirrors ``sdks/trodo-node-sdk/src/managers/DatasetManager.ts`` in shape.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Union
+
+__all__ = ["DatasetManager"]
+
+# The optional per-item fields, in wire order. ``input`` is always required.
+_ITEM_OPTIONAL = ("expected_output", "metadata", "source_trace_ref")
+
+
+def _clean_item(item: Dict[str, Any]) -> Dict[str, Any]:
+    if not isinstance(item, dict):
+        raise ValueError("trodo: each dataset item must be a dict")
+    if "input" not in item or item["input"] is None:
+        raise ValueError("trodo: each dataset item requires an 'input' dict")
+    out: Dict[str, Any] = {"input": item["input"]}
+    for key in _ITEM_OPTIONAL:
+        if item.get(key) is not None:
+            out[key] = item[key]
+    return out
+
+
+class DatasetManager:
+    """Append items to the team's evaluation datasets."""
+
+    def __init__(self, http_client: Any) -> None:
+        self._http = http_client
+
+    def append(
+        self,
+        ref: str,
+        items: Union[Dict[str, Any], List[Dict[str, Any]]],
+        create: bool = True,
+    ) -> Dict[str, Any]:
+        """Append one or more items to the dataset *ref* (a UUID or a name).
+
+        *items* is a list of dicts shaped ``{"input": {<var>: value, ...},
+        "expected_output"?: str, "metadata"?: dict, "source_trace_ref"?: dict}``;
+        a single dict is accepted and wrapped into a one-item list. ``None``
+        optional fields are dropped from the wire payload.
+
+        When *ref* is a name that doesn't exist yet, ``create=True`` (default)
+        auto-creates the dataset; pass ``create=False`` to require it to exist.
+
+        Returns the parsed response dict:
+        ``{"dataset_id", "dataset_name", "appended", "item_count"}``.
+
+        :raises ValueError: if *ref* is empty, *items* is empty, or an item has
+            no ``input``.
+        """
+        if not ref:
+            raise ValueError("trodo: append_dataset(ref) requires a dataset ref")
+        if isinstance(items, dict):
+            items = [items]
+        if not items:
+            raise ValueError("trodo: append_dataset(ref, items) requires items")
+
+        body = {
+            "items": [_clean_item(i) for i in items],
+            "create": bool(create),
+        }
+        return self._http.append_dataset_items(ref, body)

trodo_python-2.17.0/trodo/managers/experiment_manager.py

@@ -0,0 +1,286 @@
+"""Experiment manager — managed runs and external ingest.
+
+Managed experiments (prompt + model over a dataset snapshot) use
+``create``, ``run``, ``get``, ``wait``, and ``compare``.
+External agent outputs use ``ingest``.
+
+Mirrors ``sdks/trodo-node-sdk/src/managers/ExperimentManager.ts``.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List, Optional, Union
+
+__all__ = ["ExperimentManager"]
+
+_TERMINAL = frozenset({"completed", "failed"})
+
+# Optional per-output fields for ingest
+_OUTPUT_OPTIONAL = (
+    "item_position",
+    "dataset_item_id",
+    "expected_output",
+    "query",
+    "context",
+    "cost_usd",
+    "latency_ms",
+)
+
+
+def _clean_output(output: Dict[str, Any]) -> Dict[str, Any]:
+    if not isinstance(output, dict):
+        raise ValueError("trodo: each experiment output must be a dict")
+    if output.get("output") is None:
+        raise ValueError("trodo: each output requires an 'output' (str)")
+    if output.get("item_position") is None and output.get("dataset_item_id") is None:
+        raise ValueError(
+            "trodo: each output requires 'item_position' and/or 'dataset_item_id'"
+        )
+    out: Dict[str, Any] = {"output": output["output"]}
+    for key in _OUTPUT_OPTIONAL:
+        if output.get(key) is not None:
+            out[key] = output[key]
+    return out
+
+
+def _unwrap_experiment(res: Any) -> Dict[str, Any]:
+    if isinstance(res, dict) and "experiment" in res:
+        return res["experiment"]
+    return res if isinstance(res, dict) else {}
+
+
+def _build_managed_body(
+    *,
+    dataset: Optional[str] = None,
+    dataset_id: Optional[str] = None,
+    dataset_version_no: Optional[int] = None,
+    name: Optional[str] = None,
+    prompt_ref: Optional[Dict[str, Any]] = None,
+    prompt_version_id: Optional[str] = None,
+    prompt_id: Optional[str] = None,
+    use_prompt: Optional[bool] = None,
+    models: Optional[List[Dict[str, Any]]] = None,
+    scorer_ids: Optional[List[Union[str, int]]] = None,
+    evaluator_ids: Optional[List[Union[str, int]]] = None,
+    deep_eval: Optional[bool] = None,
+    stability_runs: Optional[int] = None,
+    row_limit: Optional[int] = None,
+    concurrency: Optional[int] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    body: Dict[str, Any] = {"models": models or []}
+    ds = dataset_id or dataset
+    if ds:
+        body["dataset_id"] = ds
+    if dataset_version_no is not None:
+        body["dataset_version_no"] = dataset_version_no
+    if name is not None:
+        body["name"] = name
+    if prompt_ref is not None:
+        body["prompt_ref"] = prompt_ref
+    if prompt_version_id is not None:
+        body["prompt_version_id"] = prompt_version_id
+    if prompt_id is not None:
+        body["prompt_id"] = prompt_id
+    if use_prompt is False:
+        body["use_prompt"] = False
+    ids = scorer_ids if scorer_ids is not None else evaluator_ids
+    if ids is not None:
+        body["scorer_ids"] = ids
+        body["evaluator_ids"] = ids
+    if deep_eval is not None:
+        body["deep_eval"] = deep_eval
+    if stability_runs is not None:
+        body["stability_runs"] = stability_runs
+    if row_limit is not None:
+        body["row_limit"] = row_limit
+    if concurrency is not None:
+        body["concurrency"] = concurrency
+    if metadata is not None:
+        body["metadata"] = metadata
+    return body
+
+
+class ExperimentManager:
+    """Run and inspect Trodo experiments programmatically."""
+
+    def __init__(self, http_client: Any) -> None:
+        self._http = http_client
+
+    def ingest(
+        self,
+        dataset: str,
+        outputs: List[Dict[str, Any]],
+        name: Optional[str] = None,
+        task_name: Optional[str] = None,
+        judge: Optional[Dict[str, Any]] = None,
+        evaluator_ids: Optional[List[str]] = None,
+        dataset_version_no: Optional[int] = None,
+    ) -> Dict[str, Any]:
+        """Ingest externally-produced outputs and grade them (external task path)."""
+        if not dataset:
+            raise ValueError("trodo: ingest_experiment(dataset) requires a dataset")
+        if not outputs:
+            raise ValueError("trodo: ingest_experiment requires outputs")
+
+        body: Dict[str, Any] = {
+            "dataset": dataset,
+            "outputs": [_clean_output(o) for o in outputs],
+        }
+        if name is not None:
+            body["name"] = name
+        if task_name is not None:
+            body["task_name"] = task_name
+        if judge is not None:
+            body["judge"] = judge
+        if evaluator_ids is not None:
+            body["evaluator_ids"] = evaluator_ids
+        if dataset_version_no is not None:
+            body["dataset_version_no"] = dataset_version_no
+
+        return _unwrap_experiment(self._http.ingest_experiment(body))
+
+    def create(
+        self,
+        *,
+        dataset: Optional[str] = None,
+        dataset_id: Optional[str] = None,
+        models: List[Dict[str, Any]],
+        dataset_version_no: Optional[int] = None,
+        name: Optional[str] = None,
+        prompt_ref: Optional[Dict[str, Any]] = None,
+        prompt_version_id: Optional[str] = None,
+        prompt_id: Optional[str] = None,
+        use_prompt: Optional[bool] = None,
+        scorer_ids: Optional[List[Union[str, int]]] = None,
+        evaluator_ids: Optional[List[Union[str, int]]] = None,
+        deep_eval: Optional[bool] = None,
+        stability_runs: Optional[int] = None,
+        row_limit: Optional[int] = None,
+        concurrency: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Create an experiment (immutable refs, status pending)."""
+        self._validate_run_params(dataset, dataset_id, models)
+        body = _build_managed_body(
+            dataset=dataset,
+            dataset_id=dataset_id,
+            dataset_version_no=dataset_version_no,
+            name=name,
+            prompt_ref=prompt_ref,
+            prompt_version_id=prompt_version_id,
+            prompt_id=prompt_id,
+            use_prompt=use_prompt,
+            models=models,
+            scorer_ids=scorer_ids,
+            evaluator_ids=evaluator_ids,
+            deep_eval=deep_eval,
+            stability_runs=stability_runs,
+            row_limit=row_limit,
+            concurrency=concurrency,
+            metadata=metadata,
+        )
+        return _unwrap_experiment(self._http.create_experiment(body))
+
+    def run(
+        self,
+        *,
+        dataset: Optional[str] = None,
+        dataset_id: Optional[str] = None,
+        models: List[Dict[str, Any]],
+        dataset_version_no: Optional[int] = None,
+        name: Optional[str] = None,
+        prompt_ref: Optional[Dict[str, Any]] = None,
+        prompt_version_id: Optional[str] = None,
+        prompt_id: Optional[str] = None,
+        use_prompt: Optional[bool] = None,
+        scorer_ids: Optional[List[Union[str, int]]] = None,
+        evaluator_ids: Optional[List[Union[str, int]]] = None,
+        deep_eval: Optional[bool] = None,
+        stability_runs: Optional[int] = None,
+        row_limit: Optional[int] = None,
+        concurrency: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Create and execute an experiment synchronously."""
+        self._validate_run_params(dataset, dataset_id, models)
+        body = _build_managed_body(
+            dataset=dataset,
+            dataset_id=dataset_id,
+            dataset_version_no=dataset_version_no,
+            name=name,
+            prompt_ref=prompt_ref,
+            prompt_version_id=prompt_version_id,
+            prompt_id=prompt_id,
+            use_prompt=use_prompt,
+            models=models,
+            scorer_ids=scorer_ids,
+            evaluator_ids=evaluator_ids,
+            deep_eval=deep_eval,
+            stability_runs=stability_runs,
+            row_limit=row_limit,
+            concurrency=concurrency,
+            metadata=metadata,
+        )
+        return _unwrap_experiment(self._http.run_experiment(body))
+
+    def run_by_id(self, experiment_id: str) -> Dict[str, Any]:
+        """Execute a pending experiment created via ``create``."""
+        if not experiment_id:
+            raise ValueError("trodo: run_by_id requires an experiment id")
+        return _unwrap_experiment(self._http.run_experiment_by_id(experiment_id))
+
+    def get(self, experiment_id: str) -> Dict[str, Any]:
+        """Fetch an experiment by id."""
+        if not experiment_id:
+            raise ValueError("trodo: get requires an experiment id")
+        res = self._http.get_experiment(experiment_id)
+        if isinstance(res, dict) and res.get("__error"):
+            raise RuntimeError(f"trodo: get failed (HTTP {res.get('status')})")
+        return _unwrap_experiment(res)
+
+    def wait(
+        self,
+        experiment_id: str,
+        *,
+        interval_ms: int = 2500,
+        timeout_ms: int = 600_000,
+    ) -> Dict[str, Any]:
+        """Poll until the experiment reaches a terminal status."""
+        deadline = time.monotonic() + (timeout_ms / 1000.0)
+        while True:
+            exp = self.get(experiment_id)
+            status = str(exp.get("status") or "").lower()
+            if status in _TERMINAL:
+                return exp
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    f"trodo: wait timed out after {timeout_ms}ms (status: {status or 'unknown'})"
+                )
+            time.sleep(interval_ms / 1000.0)
+
+    def compare(
+        self, a_id: str, b_id: str, *, strict: bool = False
+    ) -> Dict[str, Any]:
+        """Compare two experiments side by side."""
+        if not a_id or not b_id:
+            raise ValueError("trodo: compare requires two experiment ids")
+        res = self._http.compare_experiments(a_id, b_id, strict=strict)
+        if isinstance(res, dict) and res.get("__error"):
+            raise RuntimeError(f"trodo: compare failed (HTTP {res.get('status')})")
+        return res if isinstance(res, dict) else {}
+
+    @staticmethod
+    def _validate_run_params(
+        dataset: Optional[str],
+        dataset_id: Optional[str],
+        models: Optional[List[Dict[str, Any]]],
+    ) -> None:
+        if not models:
+            raise ValueError("trodo: run requires a non-empty models list")
+        if not dataset and not dataset_id:
+            raise ValueError("trodo: run requires dataset or dataset_id")
+        for model in models:
+            if not model.get("credential_id") or not model.get("model"):
+                raise ValueError("trodo: each model needs credential_id and model")

{trodo_python-2.15.0 → trodo_python-2.17.0}/trodo_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.15.0
+Version: 2.17.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-2.15.0 → trodo_python-2.17.0}/trodo_python.egg-info/SOURCES.txt

@@ -3,8 +3,10 @@ pyproject.toml
 tests/test_anon_distinct_id.py
 tests/test_auto_instrument_fixes.py
 tests/test_cross_process_session.py
+tests/test_datasets.py
 tests/test_end_run.py
 tests/test_error_enrichment.py
+tests/test_experiments.py
 tests/test_llm_usage_cost.py
 tests/test_processor_methods.py
 tests/test_prompt_trace.py
@@ -23,6 +25,8 @@ trodo/api/http_client.py
 trodo/auto/__init__.py
 trodo/auto/auto_event_manager.py
 trodo/managers/__init__.py
+trodo/managers/dataset_manager.py
+trodo/managers/experiment_manager.py
 trodo/managers/group_manager.py
 trodo/managers/people_manager.py
 trodo/managers/prompt_manager.py