trodo-python 2.26.1__py3-none-any.whl → 3.0.0__py3-none-any.whl

trodo/__init__.py

@@ -41,7 +41,7 @@ Downstream microservice (join the caller's run instead of making a new one):
 
 from __future__ import annotations
 
-__version__ = "2.26.1"
+__version__ = "2.26.2"
 
 from typing import Any, Callable, Dict, List, Optional, Union
 
@@ -120,15 +120,10 @@ __all__ = [
     "TemplateError",
     "ManagedPrompt",
     "PromptSummary",
-    # Datasets & experiments
+    # Datasets
+    "datasets",
     "append_dataset",
     "export_dataset",
-    "ingest_experiment",
-    "create_experiment",
-    "run_experiment",
-    "get_experiment",
-    "wait_experiment",
-    "compare_experiments",
 ]
 
 # ============================================================================
@@ -412,9 +407,30 @@ def render_template(
 
 
 # ----------------------------------------------------------------------------
-# Datasets & experiments ingest
+# Datasets
 # ----------------------------------------------------------------------------
 
+def datasets(api_key: Optional[str] = None, api_url: Optional[str] = None) -> Any:
+    """The team's datasets, managed with a Trodo API key (``TRODO_API_KEY``), as the CLI does::
+
+        ds = trodo.datasets()
+        ds.create("refund regressions", items=[{"input": "Refund order 42?", "expected": {"refund": True}}])
+        for row in ds.rows("refund regressions", rows="failed"):
+            ...
+        ds.update("refund regressions", [{"id": row["id"], "expected": "…"}])
+        ds.archive("refund regressions", [dup_id])      # restore() brings it back
+        ds.add_columns("refund regressions", ["lang"])
+
+    Works without ``trodo.init``; after it, ``append`` (site id) works here too.
+    Nothing deletes.
+    """
+    from .managers.dataset_manager import DatasetManager
+    from .api.management import ManagementClient
+    if api_key is None and api_url is None and _client is not None:
+        return _client.datasets
+    return DatasetManager(_client._http if _client is not None else None, ManagementClient(api_key, api_url))
+
+
 def append_dataset(
     ref: str,
     items: Union[Dict[str, Any], List[Dict[str, Any]]],
@@ -460,87 +476,6 @@ def export_dataset(
     )
 
 
-def ingest_experiment(
-    dataset: str,
-    outputs: List[Dict[str, Any]],
-    name: Optional[str] = None,
-    task_name: Optional[str] = None,
-    judge: Optional[Dict[str, Any]] = None,
-    evaluator_ids: Optional[List[str]] = None,
-    dataset_version_no: Optional[int] = None,
-) -> Dict[str, Any]:
-    """Ingest a batch of model *outputs* for *dataset* and grade them.
-
-    *outputs* is a list of dicts shaped ``{"item_position": int, "output": str,
-    "expected_output"?, "query"?, "context"?, "cost_usd"?, "latency_ms"?}``.
-    Configure grading with ``judge`` (``{"credential_id", "provider", "model"}``)
-    and/or ``evaluator_ids``::
-
-        exp = trodo.ingest_experiment("support-eval", [
-            {"item_position": 0, "output": "here is your tracking link"},
-        ], judge={"credential_id": "cred_1", "provider": "openai", "model": "gpt-4o"})
-
-    Returns the experiment dict.
-    """
-    return _get_client().experiments.ingest(
-        dataset,
-        outputs,
-        name=name,
-        task_name=task_name,
-        judge=judge,
-        evaluator_ids=evaluator_ids,
-        dataset_version_no=dataset_version_no,
-    )
-
-
-def create_experiment(
-    *,
-    dataset: Optional[str] = None,
-    dataset_id: Optional[str] = None,
-    models: List[Dict[str, Any]],
-    **kwargs: Any,
-) -> Dict[str, Any]:
-    """Create an experiment (immutable refs, status pending)."""
-    return _get_client().experiments.create(
-        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
-    )
-
-
-def run_experiment(
-    *,
-    dataset: Optional[str] = None,
-    dataset_id: Optional[str] = None,
-    models: List[Dict[str, Any]],
-    **kwargs: Any,
-) -> Dict[str, Any]:
-    """Create and run an experiment synchronously."""
-    return _get_client().experiments.run(
-        dataset=dataset, dataset_id=dataset_id, models=models, **kwargs
-    )
-
-
-def get_experiment(experiment_id: str) -> Dict[str, Any]:
-    """Fetch an experiment by id."""
-    return _get_client().experiments.get(experiment_id)
-
-
-def wait_experiment(
-    experiment_id: str,
-    *,
-    interval_ms: int = 2500,
-    timeout_ms: int = 600_000,
-) -> Dict[str, Any]:
-    """Poll until the experiment completes or fails."""
-    return _get_client().experiments.wait(
-        experiment_id, interval_ms=interval_ms, timeout_ms=timeout_ms
-    )
-
-
-def compare_experiments(a_id: str, b_id: str, *, strict: bool = False) -> Dict[str, Any]:
-    """Compare two experiments side by side."""
-    return _get_client().experiments.compare(a_id, b_id, strict=strict)
-
-
 def enable_auto_events() -> None:
     _get_client().enable_auto_events()
 

trodo/_version.py

@@ -1 +1 @@
-SDK_VERSION = "2.26.0"
+SDK_VERSION = "3.0.0"

trodo/api/endpoints.py

@@ -28,10 +28,5 @@ RUNS_START = "/api/sdk/runs/start"
 RUNS_BASE = "/api/sdk/runs"  # /runs/{run_id}/end, /spans, /feedback
 SPANS_APPEND = "/api/sdk/spans/append"
 OTLP_TRACES = "/api/sdk/otel/v1/traces"
-# Datasets & experiments
+# Datasets
 DATASETS_BASE = "/api/sdk/datasets"  # /datasets/{ref}/items
-EXPERIMENTS = "/api/sdk/experiments"
-EXPERIMENTS_INGEST = "/api/sdk/experiments/ingest"
-EXPERIMENTS_CREATE = "/api/sdk/experiments/create"
-EXPERIMENTS_RUN = "/api/sdk/experiments/run"
-EXPERIMENTS_COMPARE = "/api/sdk/experiments/compare"

trodo/api/http_client.py

@@ -219,31 +219,6 @@ class HttpClient:
             "version": int(version) if version and version.isdigit() else None,
         }
 
-    def ingest_experiment(self, body: Dict[str, Any]) -> ApiResult:
-        return self._request("/api/sdk/experiments/ingest", body)
-
-    def create_experiment(self, body: Dict[str, Any]) -> ApiResult:
-        return self._request("/api/sdk/experiments/create", body)
-
-    def run_experiment(self, body: Dict[str, Any]) -> ApiResult:
-        return self._request("/api/sdk/experiments/run", body)
-
-    def run_experiment_by_id(self, experiment_id: str) -> ApiResult:
-        from urllib.parse import quote
-        return self._request(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}/run", {})
-
-    def get_experiment(self, experiment_id: str) -> ApiResult:
-        from urllib.parse import quote
-        return self._get(f"/api/sdk/experiments/{quote(str(experiment_id), safe='')}")
-
-    def compare_experiments(
-        self, a_id: str, b_id: str, strict: bool = False
-    ) -> ApiResult:
-        params: Dict[str, Any] = {"a": a_id, "b": b_id}
-        if strict:
-            params["strict"] = "true"
-        return self._get("/api/sdk/experiments/compare", params)
-
     def post_track(self, session_data: Dict[str, Any]) -> ApiResult:
         return self._request("/api/sdk/track", {"sessionData": session_data})
 

trodo/api/management.py

@@ -0,0 +1,90 @@
+"""The team API (/api/v1) with a Trodo API key — what the CLI uses.
+
+Dataset management (create, read rows, edit, archive, add columns) goes
+through here, never through the site id: a site id can sit in a browser page,
+and nobody who only has that should be able to change a dataset. The key is a
+Trodo API key (Settings → MCP, or ``trodo login``), passed in or read from
+``TRODO_API_KEY``; ``TRODO_API_URL`` overrides the address.
+
+Reads retry on network errors and 502/503/504; writes are sent once.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Any, Dict, Iterator, Optional
+
+import requests
+
+from .._version import SDK_VERSION
+
+DEFAULT_API_URL = "https://api.trodo.ai"
+
+
+class TrodoApiError(RuntimeError):
+    """The API refused or failed. ``status`` and ``body`` say why."""
+
+    def __init__(self, message: str, status: Optional[int] = None, body: Any = None) -> None:
+        super().__init__(message)
+        self.status = status
+        self.body = body
+
+
+class ManagementClient:
+    def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None, timeout: int = 60) -> None:
+        key = api_key or os.environ.get("TRODO_API_KEY")
+        if not key:
+            raise ValueError(
+                "trodo: dataset management needs a Trodo API key: pass api_key= or set TRODO_API_KEY "
+                "(Settings → MCP, or `trodo login`). The site id cannot change datasets."
+            )
+        self.base = (api_url or os.environ.get("TRODO_API_URL") or DEFAULT_API_URL).rstrip("/") + "/api/v1"
+        self.timeout = timeout
+        self._session = requests.Session()
+        self._session.headers.update({
+            "Authorization": f"Bearer {key}",
+            "Content-Type": "application/json",
+            "User-Agent": f"trodo-python/{SDK_VERSION}",
+        })
+
+    def _send(self, method: str, path: str, *, params: Optional[Dict[str, Any]] = None,
+              body: Any = None, stream: bool = False) -> requests.Response:
+        url = self.base + path
+        clean = {k: v for k, v in (params or {}).items() if v is not None and v != ""}
+        tries = 4 if method == "GET" else 1
+        for attempt in range(tries):
+            try:
+                resp = self._session.request(method, url, params=clean, data=None if body is None else json.dumps(body),
+                                             timeout=self.timeout, stream=stream)
+            except requests.RequestException as exc:
+                if attempt + 1 < tries:
+                    time.sleep(0.5 * (2 ** attempt))
+                    continue
+                raise TrodoApiError(f"trodo: {method} {path} failed: {exc}") from exc
+            if resp.status_code in (502, 503, 504) and attempt + 1 < tries:
+                time.sleep(0.5 * (2 ** attempt))
+                continue
+            if resp.status_code >= 400:
+                try:
+                    data = resp.json()
+                except ValueError:
+                    data = {"message": resp.text[:300]}
+                msg = (data.get("message") or data.get("error")) if isinstance(data, dict) else None
+                raise TrodoApiError(f"trodo: {method} {path} → HTTP {resp.status_code} {msg or ''}".strip(), resp.status_code, data)
+            return resp
+        raise TrodoApiError(f"trodo: {method} {path} failed")  # pragma: no cover
+
+    def get(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any:
+        return self._send("GET", path, params=params).json()
+
+    def post(self, path: str, body: Any = None) -> Any:
+        return self._send("POST", path, body=body if body is not None else {}).json()
+
+    def lines(self, path: str, params: Optional[Dict[str, Any]] = None) -> Iterator[Dict[str, Any]]:
+        """A JSON Lines response, one object at a time."""
+        resp = self._send("GET", path, params=params, stream=True)
+        for raw in resp.iter_lines(decode_unicode=True):
+            if raw and raw.strip():
+                yield json.loads(raw)

trodo/client.py

@@ -84,7 +84,6 @@ class TrodoClient:
         self._session_manager = SessionManager(max_cached_users)
         self._prompts = None  # lazily-built PromptManager
         self._datasets = None  # lazily-built DatasetManager
-        self._experiments = None  # lazily-built ExperimentManager
         self._users = None  # lazily-built UserManager
 
         if batch_enabled:
@@ -149,14 +148,6 @@ class TrodoClient:
             self._datasets = DatasetManager(self._http)
         return self._datasets
 
-    @property
-    def experiments(self):
-        """Managed experiment runs + external ingest for server-side grading."""
-        if self._experiments is None:
-            from .managers.experiment_manager import ExperimentManager
-            self._experiments = ExperimentManager(self._http)
-        return self._experiments
-
     @property
     def users(self):
         """Create-or-update a user and their traits — the stateless-backend

trodo/managers/dataset_manager.py

@@ -1,6 +1,13 @@
-"""Dataset ingest — append evaluation items to a Trodo dataset.
+"""Datasets — append rows with the site id; manage them with an API key.
 
-Datasets are the fixed inputs an experiment runs against: each item is a set of
+``append`` and ``export`` work as before (the site id, an export token).
+Everything else — create, list, get, rows, update, archive, restore,
+add_columns — goes through the team API with a Trodo API key, the one the
+CLI uses (``TRODO_API_KEY``), because those change rows a site id must not
+reach. Nothing here deletes: archive takes rows out of the working set,
+the version before keeps them, and restore brings them back.
+
+Datasets are fixed inputs to evaluate against: each item is a set of
 variable values (``input``) plus an optional gold answer (``expected_output``)
 and free-form ``metadata``. This manager appends items to a dataset addressed by
 UUID or by name; when a name is used and no such dataset exists it is
@@ -13,7 +20,7 @@ Mirrors ``sdks/trodo-node-sdk/src/managers/DatasetManager.ts`` in shape.
 from __future__ import annotations
 
 import json
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Iterator, List, Optional, Union
 
 __all__ = ["DatasetManager"]
 
@@ -45,11 +52,157 @@ def _clean_item(item: Dict[str, Any]) -> Dict[str, Any]:
     return out
 
 
+def _wire_row(r: Dict[str, Any]) -> Dict[str, Any]:
+    """A row for POST /datasets/:id/items: ``expected`` is sent as ``expected_output``."""
+    if not isinstance(r, dict):
+        raise ValueError("trodo: each dataset row must be a dict")
+    out = dict(r)
+    if "expected" in out and "expected_output" not in out:
+        out["expected_output"] = out.pop("expected")
+    return out
+
+
 class DatasetManager:
-    """Append items to the team's evaluation datasets."""
+    """The team's datasets: append with the site id, manage with an API key."""
 
-    def __init__(self, http_client: Any) -> None:
+    def __init__(self, http_client: Any, api: Any = None) -> None:
         self._http = http_client
+        self._api = api  # a ManagementClient, built on first use
+
+    def _mgmt(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> Any:
+        if api_key or api_url:
+            from ..api.management import ManagementClient
+            return ManagementClient(api_key, api_url)
+        if self._api is None:
+            from ..api.management import ManagementClient
+            self._api = ManagementClient()
+        return self._api
+
+    def _resolve(self, api: Any, ref: str) -> Dict[str, Any]:
+        if not ref:
+            raise ValueError("trodo: name the dataset (its name or id)")
+        listed = api.get("/datasets").get("datasets") or []
+        low = str(ref).lower()
+        hit = next((d for d in listed if d.get("id") == ref), None) \
+            or next((d for d in listed if d.get("name") == ref), None) \
+            or next((d for d in listed if str(d.get("name", "")).lower() == low), None)
+        if not hit:
+            raise ValueError(f"trodo: no dataset {ref!r}")
+        return hit
+
+    def list(self, *, api_key: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Every dataset of the key's team, with its row counts."""
+        return self._mgmt(api_key).get("/datasets").get("datasets") or []
+
+    def get(self, ref: str, *, api_key: Optional[str] = None) -> Dict[str, Any]:
+        """One dataset by name or id: its columns, derived columns and counts (not the rows)."""
+        api = self._mgmt(api_key)
+        d = self._resolve(api, ref)
+        return api.get(f"/datasets/{d['id']}", {"summary": 1}).get("dataset") or d
+
+    def create(self, name: str, *, description: Optional[str] = None,
+               items: Optional[List[Dict[str, Any]]] = None, api_key: Optional[str] = None) -> Dict[str, Any]:
+        """A new dataset, optionally with rows ({input, output?, expected?, metadata?} or {run_id} …).
+
+        Returns ``{"dataset": {...}, "written": int}``.
+        """
+        if not name or not str(name).strip():
+            raise ValueError("trodo: create(name) needs a name")
+        api = self._mgmt(api_key)
+        body: Dict[str, Any] = {"name": name}
+        if description:
+            body["description"] = description
+        dataset = api.post("/datasets", body).get("dataset") or {}
+        written = 0
+        if items:
+            for i in range(0, len(items), 500):
+                out = api.post(f"/datasets/{dataset['id']}/items", {"items": [_wire_row(r) for r in items[i:i + 500]]})
+                written += int(out.get("written", out.get("added", 0)) or 0)
+        return {"dataset": dataset, "written": written}
+
+    def rows(self, ref: str, *, rows: Optional[str] = None, eval: Optional[str] = None,
+             format: Optional[str] = None, target: Optional[str] = None, version: Optional[int] = None,
+             api_key: Optional[str] = None) -> Iterator[Dict[str, Any]]:
+        """The dataset's rows, one dict at a time — the same as ``trodo datasets export``.
+
+        *rows*: all, expected, no_expected, passed, failed, marked_wrong. *format*: trodo
+        (default), openai, dpo. *target* (openai): expected_else_output, output, expected.
+        Each Trodo row carries ``id``, so an edit can go straight back through ``update``.
+        """
+        api = self._mgmt(api_key)
+        d = self._resolve(api, ref)
+        params = {"rows": rows, "eval": eval, "format": format, "target": target, "version": version}
+        return api.lines(f"/datasets/{d['id']}/export", params)
+
+    def update(self, ref: str, updates: List[Dict[str, Any]], *, label: str = "person",
+               version_name: Optional[str] = None, api_key: Optional[str] = None) -> Dict[str, Any]:
+        """Edit rows by id: ``[{"id", "input"?, "expected"?, "output"?, "metadata"?, "tags"?}]``.
+
+        *input* is replaced the way rows are added (text, a list of messages, or an object
+        of variables); *expected* may be any JSON; *metadata* merges. One version per 5,000
+        rows. *label* ``"derived"`` marks a changed Expected as written by a rule or model.
+        Returns ``{"updated", "unchanged", "missing", "version"}``.
+        """
+        api = self._mgmt(api_key)
+        d = self._resolve(api, ref)
+        rows = [u for u in (updates or []) if isinstance(u, dict) and u.get("id")]
+        if not rows:
+            raise ValueError("trodo: update needs rows with an id")
+        total: Dict[str, Any] = {"updated": 0, "unchanged": 0, "missing": [], "version": None}
+        for i in range(0, len(rows), 5000):
+            body: Dict[str, Any] = {"updates": rows[i:i + 5000], "label": "derived" if label == "derived" else "person"}
+            if version_name:
+                body["version_name"] = version_name
+            out = api.post(f"/datasets/{d['id']}/items/update", body)
+            total["updated"] += int(out.get("updated") or 0)
+            total["unchanged"] += int(out.get("unchanged") or 0)
+            total["missing"] += out.get("missing") or []
+            total["version"] = out.get("version") or total["version"]
+        return total
+
+    def _move(self, kind: str, ref: str, ids: List[str], version_name: Optional[str], api_key: Optional[str]) -> Dict[str, Any]:
+        api = self._mgmt(api_key)
+        d = self._resolve(api, ref)
+        ids = [str(i) for i in (ids or []) if i]
+        if not ids:
+            raise ValueError(f"trodo: {kind} needs row ids")
+        key = "archived" if kind == "archive" else "restored"
+        total: Dict[str, Any] = {key: 0, "unchanged": 0, "version": None}
+        for i in range(0, len(ids), 5000):
+            body: Dict[str, Any] = {"ids": ids[i:i + 5000]}
+            if version_name:
+                body["version_name"] = version_name
+            out = api.post(f"/datasets/{d['id']}/items/{kind}", body)
+            total[key] += int(out.get(key) or 0)
+            total["unchanged"] += int(out.get("unchanged") or 0)
+            total["version"] = out.get("version") or total["version"]
+        return total
+
+    def archive(self, ref: str, ids: List[str], *, version_name: Optional[str] = None,
+                api_key: Optional[str] = None) -> Dict[str, Any]:
+        """Take rows out of the working set (exports, labeling and sources skip them).
+
+        Nothing is deleted: the version before holds them and ``restore`` brings them back.
+        """
+        return self._move("archive", ref, ids, version_name, api_key)
+
+    def restore(self, ref: str, ids: List[str], *, version_name: Optional[str] = None,
+                api_key: Optional[str] = None) -> Dict[str, Any]:
+        """Bring archived rows back into the working set."""
+        return self._move("restore", ref, ids, version_name, api_key)
+
+    def add_columns(self, ref: str, columns: Optional[List[str]] = None, *,
+                    derived: Optional[List[Dict[str, str]]] = None, api_key: Optional[str] = None) -> Dict[str, Any]:
+        """Add input columns by name, and derived columns ``{"name", "path"}`` or ``{"name", "expr"}``.
+
+        Columns are only ever added; a name already there is left as it is.
+        Returns ``{"added", "columns", "derived_columns"}``.
+        """
+        if not columns and not derived:
+            raise ValueError("trodo: add_columns needs columns or derived")
+        api = self._mgmt(api_key)
+        d = self._resolve(api, ref)
+        return api.post(f"/datasets/{d['id']}/columns/add", {"columns": list(columns or []), "derived": list(derived or [])})
 
     def append(
         self,

trodo/otel/processor.py

@@ -11,6 +11,9 @@ import threading
 from dataclasses import dataclass, asdict
 from typing import Any, Dict, List, Optional, Union
 
+# How long end_run waits for the run's queued spans to send before sending the end.
+END_RUN_FLUSH_S = 5.0
+
 
 @dataclass
 class TrodoRun:
@@ -189,7 +192,21 @@ class TrodoSpanProcessor:
             pass
 
     def end_run(self, run_id: str, payload: Dict[str, Any]) -> None:
-        """Finalise a Run opened by start_run."""
+        """Finalise a Run opened by start_run.
+
+        The run's spans go first: joined spans wait in the sender's queue, and
+        an end that overtakes them gets the run evaluated on a partial trace.
+        Bounded, and a failure here never holds up the end.
+        """
+        try:
+            with self._lock:
+                buffered = self._pending.pop(run_id, None) or []
+            if buffered:
+                self._send_spans(run_id, [s.to_dict() for s in buffered])
+            if self._sender is not None:
+                self._sender.flush(END_RUN_FLUSH_S)
+        except Exception:
+            pass
         try:
             self._http.post_run_end(run_id, payload)
         except Exception:

{trodo_python-2.26.1.dist-info → trodo_python-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.26.1
+Version: 3.0.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-2.26.1.dist-info → trodo_python-3.0.0.dist-info}/RECORD

@@ -1,20 +1,20 @@
-trodo/__init__.py,sha256=RCoNO_seZkIUmweHvyKb8t6J5ydktoAHOg2nt2sp448,28752
-trodo/_version.py,sha256=MX7IGM5dYbXPeFIXOWPhV7DPbCslhEe8Q7DCvxwKmhY,23
-trodo/client.py,sha256=6k6HeKrGV2720kK17--u6BjbhAUFZNUzpd9yEYiaQOs,24137
+trodo/__init__.py,sha256=x7zE63FK8CFJ6ATUvsXxeTkvZUaLKXqAJr99I6nnyZc,27099
+trodo/_version.py,sha256=AsSlr9XWZaS6NVC4bCgB5xuBv1s5UrEYJlMsyJQfpMw,22
+trodo/client.py,sha256=eaZtHMtOI-VLgfoqM_KVWMuP_I5Ivs7VFnR0oU96Yqk,23743
 trodo/types.py,sha256=eySgUvCXROG2TxtxgiU0MNr5iH0DEcduK8bmYtTKG44,3138
 trodo/user_context.py,sha256=uHCI2WYoOI3cNwdIUEuZdX4VYSu14pzm348ksn6hn_c,8195
 trodo/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/api/_retry.py,sha256=mHhuzfe3QuYa6j9BqRYVClvnYH0o3WkXXngmSwqWfgY,1214
 trodo/api/async_client.py,sha256=qF97NbvoYbNhem8uXtPh14EG4JVAhEAzUCYn7busaaA,3609
-trodo/api/endpoints.py,sha256=SGgVELcVfwuCrRwlVPW2j0KYkaMtwLFPpdMtGtlqPeA,1574
-trodo/api/http_client.py,sha256=iOG9D4ackVAz0nQD5ZjwfIvRiEmgxga9uXgxrRQX6L4,13185
+trodo/api/endpoints.py,sha256=jToYE2u5EWbkVLiRnEEGRAPtE-EbjNFMPvSFF-EfVt4,1323
+trodo/api/http_client.py,sha256=kNqa18LuDC0f3eQFN3PQJk4I2PCB1McfSmRP_U_FuQQ,12103
+trodo/api/management.py,sha256=uwhr0ZI1pvguz5-itW1KcgRj0TXM2n4KDa-hGIaBdTA,4023
 trodo/auto/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/auto/auto_event_manager.py,sha256=cztuRsRkNoJE5R4NfSfTrTJTGl4jx2Yb-Ncy0aVAPo8,4247
 trodo/delivery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trodo/delivery/sender.py,sha256=urO7m6mRMnSEW7_cNAmJ4A6jtDNxuihZHOxCx1jQScI,21348
 trodo/managers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trodo/managers/dataset_manager.py,sha256=mifHYkHLWo0olXCNfQOqvpLPL8DDgbub7YAWWlrjIHQ,5609
-trodo/managers/experiment_manager.py,sha256=V-vemLtenfSI5VIcP0HPiXzEy8AyTrRrbJVCA9Z6wjo,10579
+trodo/managers/dataset_manager.py,sha256=HTFszK-wKmwjW-aw7xJWr5EKvDpJ_om496MLDr3gelg,13920
 trodo/managers/group_manager.py,sha256=ki3Se3qEoSZfREX63oeDeBmEfZF-ISHLE8azEtLg0tM,3542
 trodo/managers/people_manager.py,sha256=mMVnx40Mlifx6NGgChvohC9ViK6dQu2mkXNHbV8pK1E,2882
 trodo/managers/prompt_manager.py,sha256=qFVHgh7PW-SlgHp62ybirdYsJBg-fcQVMTTU3jPqFvo,13435
@@ -23,7 +23,7 @@ trodo/otel/__init__.py,sha256=yiRFXWUU45bAM2CV37XeO7zf1hmnmjufdP4XO50yEyE,624
 trodo/otel/auto_instrument.py,sha256=M5YVpFoPqn9bQohDFM7yyADrVGJPqFBToJLnBO7nOSA,29029
 trodo/otel/context.py,sha256=Jd0aTc0Q-1dM5kXXinhZD8YtnpdKgvneNiODyboVGKY,1418
 trodo/otel/helpers.py,sha256=XOMWcgZHaq5SQbkFxDaXPE4CDFjn01xjJmJ1vIxvwpw,20730
-trodo/otel/processor.py,sha256=WgtLdXT9gpR4TZ_9iuoX-kBP4UN4nelCtoDnogtW_Qc,8289
+trodo/otel/processor.py,sha256=GiMQR3uKHq_sbqXU00MdyarStaBzrZLmPgU1YnlzZb4,8965
 trodo/otel/prompt_trace.py,sha256=-kywRBU0Q2gWntS9HU-P5i7Kc5U-IJU7UVLjkdC2jw4,13593
 trodo/otel/register.py,sha256=7X3AfFQ5YvHGOUmdq8J38LQGvOJFPD907yT3uAp90lI,9607
 trodo/otel/transport.py,sha256=hzZz8gwSMGJ8CxdijmLn1Ljt18owr9XTWy13DLbwYbw,2441
@@ -40,7 +40,7 @@ trodo/session/server_session.py,sha256=McsudEiq33XDq3nqxgzBcUvIjQxCMscwEuAPnYXrT
 trodo/session/session_manager.py,sha256=7ht5LeeZX1HLLfPeNV_a8NkXGbHl3up0CRtCGr3EzjQ,2995
 trodo/util/__init__.py,sha256=Z9c4rPPdKg06Kk3byKheDqksSgR4WNvS475oQ5sNljc,54
 trodo/util/lru.py,sha256=QIsM7s6J_E9ZjiXatSyReZIEeGypTAcgrPWWy8YsRa4,2465
-trodo_python-2.26.1.dist-info/METADATA,sha256=2I7mNSS6gC_w6I3Mc9iOT-E6VHLboXBB9o8i4F-8YN0,25308
-trodo_python-2.26.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-trodo_python-2.26.1.dist-info/top_level.txt,sha256=VCQu1CJWFmNsqTs1YxMcw4Mq35Tc3z3uI9RwHEXAayQ,6
-trodo_python-2.26.1.dist-info/RECORD,,
+trodo_python-3.0.0.dist-info/METADATA,sha256=Uq0WsyxJPpE-A8CBpSrafzEvniEf2a__kDIEE9055Ls,25307
+trodo_python-3.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+trodo_python-3.0.0.dist-info/top_level.txt,sha256=VCQu1CJWFmNsqTs1YxMcw4Mq35Tc3z3uI9RwHEXAayQ,6
+trodo_python-3.0.0.dist-info/RECORD,,

{trodo_python-2.26.1.dist-info → trodo_python-3.0.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (84.0.0)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trodo/managers/experiment_manager.py

@@ -1,286 +0,0 @@
-"""Experiment manager — managed runs and external ingest.
-
-Managed experiments (prompt + model over a dataset snapshot) use
-``create``, ``run``, ``get``, ``wait``, and ``compare``.
-External agent outputs use ``ingest``.
-
-Mirrors ``sdks/trodo-node-sdk/src/managers/ExperimentManager.ts``.
-"""
-
-from __future__ import annotations
-
-import time
-from typing import Any, Dict, List, Optional, Union
-
-__all__ = ["ExperimentManager"]
-
-_TERMINAL = frozenset({"completed", "failed"})
-
-# Optional per-output fields for ingest
-_OUTPUT_OPTIONAL = (
-    "item_position",
-    "dataset_item_id",
-    "expected_output",
-    "query",
-    "context",
-    "cost_usd",
-    "latency_ms",
-)
-
-
-def _clean_output(output: Dict[str, Any]) -> Dict[str, Any]:
-    if not isinstance(output, dict):
-        raise ValueError("trodo: each experiment output must be a dict")
-    if output.get("output") is None:
-        raise ValueError("trodo: each output requires an 'output' (str)")
-    if output.get("item_position") is None and output.get("dataset_item_id") is None:
-        raise ValueError(
-            "trodo: each output requires 'item_position' and/or 'dataset_item_id'"
-        )
-    out: Dict[str, Any] = {"output": output["output"]}
-    for key in _OUTPUT_OPTIONAL:
-        if output.get(key) is not None:
-            out[key] = output[key]
-    return out
-
-
-def _unwrap_experiment(res: Any) -> Dict[str, Any]:
-    if isinstance(res, dict) and "experiment" in res:
-        return res["experiment"]
-    return res if isinstance(res, dict) else {}
-
-
-def _build_managed_body(
-    *,
-    dataset: Optional[str] = None,
-    dataset_id: Optional[str] = None,
-    dataset_version_no: Optional[int] = None,
-    name: Optional[str] = None,
-    prompt_ref: Optional[Dict[str, Any]] = None,
-    prompt_version_id: Optional[str] = None,
-    prompt_id: Optional[str] = None,
-    use_prompt: Optional[bool] = None,
-    models: Optional[List[Dict[str, Any]]] = None,
-    scorer_ids: Optional[List[Union[str, int]]] = None,
-    evaluator_ids: Optional[List[Union[str, int]]] = None,
-    deep_eval: Optional[bool] = None,
-    stability_runs: Optional[int] = None,
-    row_limit: Optional[int] = None,
-    concurrency: Optional[int] = None,
-    metadata: Optional[Dict[str, Any]] = None,
-) -> Dict[str, Any]:
-    body: Dict[str, Any] = {"models": models or []}
-    ds = dataset_id or dataset
-    if ds:
-        body["dataset_id"] = ds
-    if dataset_version_no is not None:
-        body["dataset_version_no"] = dataset_version_no
-    if name is not None:
-        body["name"] = name
-    if prompt_ref is not None:
-        body["prompt_ref"] = prompt_ref
-    if prompt_version_id is not None:
-        body["prompt_version_id"] = prompt_version_id
-    if prompt_id is not None:
-        body["prompt_id"] = prompt_id
-    if use_prompt is False:
-        body["use_prompt"] = False
-    ids = scorer_ids if scorer_ids is not None else evaluator_ids
-    if ids is not None:
-        body["scorer_ids"] = ids
-        body["evaluator_ids"] = ids
-    if deep_eval is not None:
-        body["deep_eval"] = deep_eval
-    if stability_runs is not None:
-        body["stability_runs"] = stability_runs
-    if row_limit is not None:
-        body["row_limit"] = row_limit
-    if concurrency is not None:
-        body["concurrency"] = concurrency
-    if metadata is not None:
-        body["metadata"] = metadata
-    return body
-
-
-class ExperimentManager:
-    """Run and inspect Trodo experiments programmatically."""
-
-    def __init__(self, http_client: Any) -> None:
-        self._http = http_client
-
-    def ingest(
-        self,
-        dataset: str,
-        outputs: List[Dict[str, Any]],
-        name: Optional[str] = None,
-        task_name: Optional[str] = None,
-        judge: Optional[Dict[str, Any]] = None,
-        evaluator_ids: Optional[List[str]] = None,
-        dataset_version_no: Optional[int] = None,
-    ) -> Dict[str, Any]:
-        """Ingest externally-produced outputs and grade them (external task path)."""
-        if not dataset:
-            raise ValueError("trodo: ingest_experiment(dataset) requires a dataset")
-        if not outputs:
-            raise ValueError("trodo: ingest_experiment requires outputs")
-
-        body: Dict[str, Any] = {
-            "dataset": dataset,
-            "outputs": [_clean_output(o) for o in outputs],
-        }
-        if name is not None:
-            body["name"] = name
-        if task_name is not None:
-            body["task_name"] = task_name
-        if judge is not None:
-            body["judge"] = judge
-        if evaluator_ids is not None:
-            body["evaluator_ids"] = evaluator_ids
-        if dataset_version_no is not None:
-            body["dataset_version_no"] = dataset_version_no
-
-        return _unwrap_experiment(self._http.ingest_experiment(body))
-
-    def create(
-        self,
-        *,
-        dataset: Optional[str] = None,
-        dataset_id: Optional[str] = None,
-        models: List[Dict[str, Any]],
-        dataset_version_no: Optional[int] = None,
-        name: Optional[str] = None,
-        prompt_ref: Optional[Dict[str, Any]] = None,
-        prompt_version_id: Optional[str] = None,
-        prompt_id: Optional[str] = None,
-        use_prompt: Optional[bool] = None,
-        scorer_ids: Optional[List[Union[str, int]]] = None,
-        evaluator_ids: Optional[List[Union[str, int]]] = None,
-        deep_eval: Optional[bool] = None,
-        stability_runs: Optional[int] = None,
-        row_limit: Optional[int] = None,
-        concurrency: Optional[int] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-    ) -> Dict[str, Any]:
-        """Create an experiment (immutable refs, status pending)."""
-        self._validate_run_params(dataset, dataset_id, models)
-        body = _build_managed_body(
-            dataset=dataset,
-            dataset_id=dataset_id,
-            dataset_version_no=dataset_version_no,
-            name=name,
-            prompt_ref=prompt_ref,
-            prompt_version_id=prompt_version_id,
-            prompt_id=prompt_id,
-            use_prompt=use_prompt,
-            models=models,
-            scorer_ids=scorer_ids,
-            evaluator_ids=evaluator_ids,
-            deep_eval=deep_eval,
-            stability_runs=stability_runs,
-            row_limit=row_limit,
-            concurrency=concurrency,
-            metadata=metadata,
-        )
-        return _unwrap_experiment(self._http.create_experiment(body))
-
-    def run(
-        self,
-        *,
-        dataset: Optional[str] = None,
-        dataset_id: Optional[str] = None,
-        models: List[Dict[str, Any]],
-        dataset_version_no: Optional[int] = None,
-        name: Optional[str] = None,
-        prompt_ref: Optional[Dict[str, Any]] = None,
-        prompt_version_id: Optional[str] = None,
-        prompt_id: Optional[str] = None,
-        use_prompt: Optional[bool] = None,
-        scorer_ids: Optional[List[Union[str, int]]] = None,
-        evaluator_ids: Optional[List[Union[str, int]]] = None,
-        deep_eval: Optional[bool] = None,
-        stability_runs: Optional[int] = None,
-        row_limit: Optional[int] = None,
-        concurrency: Optional[int] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-    ) -> Dict[str, Any]:
-        """Create and execute an experiment synchronously."""
-        self._validate_run_params(dataset, dataset_id, models)
-        body = _build_managed_body(
-            dataset=dataset,
-            dataset_id=dataset_id,
-            dataset_version_no=dataset_version_no,
-            name=name,
-            prompt_ref=prompt_ref,
-            prompt_version_id=prompt_version_id,
-            prompt_id=prompt_id,
-            use_prompt=use_prompt,
-            models=models,
-            scorer_ids=scorer_ids,
-            evaluator_ids=evaluator_ids,
-            deep_eval=deep_eval,
-            stability_runs=stability_runs,
-            row_limit=row_limit,
-            concurrency=concurrency,
-            metadata=metadata,
-        )
-        return _unwrap_experiment(self._http.run_experiment(body))
-
-    def run_by_id(self, experiment_id: str) -> Dict[str, Any]:
-        """Execute a pending experiment created via ``create``."""
-        if not experiment_id:
-            raise ValueError("trodo: run_by_id requires an experiment id")
-        return _unwrap_experiment(self._http.run_experiment_by_id(experiment_id))
-
-    def get(self, experiment_id: str) -> Dict[str, Any]:
-        """Fetch an experiment by id."""
-        if not experiment_id:
-            raise ValueError("trodo: get requires an experiment id")
-        res = self._http.get_experiment(experiment_id)
-        if isinstance(res, dict) and res.get("__error"):
-            raise RuntimeError(f"trodo: get failed (HTTP {res.get('status')})")
-        return _unwrap_experiment(res)
-
-    def wait(
-        self,
-        experiment_id: str,
-        *,
-        interval_ms: int = 2500,
-        timeout_ms: int = 600_000,
-    ) -> Dict[str, Any]:
-        """Poll until the experiment reaches a terminal status."""
-        deadline = time.monotonic() + (timeout_ms / 1000.0)
-        while True:
-            exp = self.get(experiment_id)
-            status = str(exp.get("status") or "").lower()
-            if status in _TERMINAL:
-                return exp
-            if time.monotonic() >= deadline:
-                raise TimeoutError(
-                    f"trodo: wait timed out after {timeout_ms}ms (status: {status or 'unknown'})"
-                )
-            time.sleep(interval_ms / 1000.0)
-
-    def compare(
-        self, a_id: str, b_id: str, *, strict: bool = False
-    ) -> Dict[str, Any]:
-        """Compare two experiments side by side."""
-        if not a_id or not b_id:
-            raise ValueError("trodo: compare requires two experiment ids")
-        res = self._http.compare_experiments(a_id, b_id, strict=strict)
-        if isinstance(res, dict) and res.get("__error"):
-            raise RuntimeError(f"trodo: compare failed (HTTP {res.get('status')})")
-        return res if isinstance(res, dict) else {}
-
-    @staticmethod
-    def _validate_run_params(
-        dataset: Optional[str],
-        dataset_id: Optional[str],
-        models: Optional[List[Dict[str, Any]]],
-    ) -> None:
-        if not models:
-            raise ValueError("trodo: run requires a non-empty models list")
-        if not dataset and not dataset_id:
-            raise ValueError("trodo: run requires dataset or dataset_id")
-        for model in models:
-            if not model.get("credential_id") or not model.get("model"):
-                raise ValueError("trodo: each model needs credential_id and model")