trodo-python 2.15.0__tar.gz → 2.16.0__tar.gz

{trodo_python-2.15.0 → trodo_python-2.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.15.0
+Version: 2.16.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-2.15.0 → trodo_python-2.16.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trodo-python"
-version = "2.15.0"
+version = "2.16.0"
 description = "Trodo Analytics SDK for Python — server-side event tracking"
 readme = "README.md"
 license = { text = "ISC" }

trodo_python-2.16.0/tests/test_datasets.py

@@ -0,0 +1,102 @@
+"""Dataset ingest — DatasetManager.append.
+
+Uses a fake HTTP client so no network is touched; asserts on the exact
+ref + body sent to the backend and the response returned to the caller.
+"""
+
+import pytest
+
+from trodo.managers.dataset_manager import DatasetManager
+
+
+class FakeHttp:
+    """Records the last append call; mirrors the prompt-test fakeApi."""
+
+    def __init__(self, response=None):
+        self._response = response or {
+            "dataset_id": "ds_1",
+            "dataset_name": "support-eval",
+            "appended": 1,
+            "item_count": 1,
+        }
+        self.calls = 0
+        self.last_ref = None
+        self.last_body = None
+
+    def append_dataset_items(self, ref, body):
+        self.calls += 1
+        self.last_ref = ref
+        self.last_body = body
+        return self._response
+
+
+def test_append_posts_ref_and_body_and_returns_response():
+    http = FakeHttp()
+    mgr = DatasetManager(http)
+    res = mgr.append(
+        "support-eval",
+        [{"input": {"q": "where is my order"}, "expected_output": "tracking link"}],
+    )
+
+    assert http.calls == 1
+    assert http.last_ref == "support-eval"
+    assert http.last_body == {
+        "items": [
+            {"input": {"q": "where is my order"}, "expected_output": "tracking link"}
+        ],
+        "create": True,
+    }
+    assert res["dataset_id"] == "ds_1"
+    assert res["item_count"] == 1
+
+
+def test_append_wraps_a_single_dict_into_a_list():
+    http = FakeHttp()
+    DatasetManager(http).append("ds", {"input": {"x": 1}})
+    assert http.last_body["items"] == [{"input": {"x": 1}}]
+
+
+def test_append_drops_none_optional_fields():
+    http = FakeHttp()
+    DatasetManager(http).append(
+        "ds",
+        [
+            {
+                "input": {"x": 1},
+                "expected_output": None,
+                "metadata": {"k": "v"},
+                "source_trace_ref": None,
+            }
+        ],
+    )
+    # None fields dropped; present ones kept.
+    assert http.last_body["items"] == [{"input": {"x": 1}, "metadata": {"k": "v"}}]
+
+
+def test_append_passes_create_false_through():
+    http = FakeHttp()
+    DatasetManager(http).append("ds", [{"input": {"x": 1}}], create=False)
+    assert http.last_body["create"] is False
+
+
+def test_append_keeps_source_trace_ref():
+    http = FakeHttp()
+    DatasetManager(http).append(
+        "ds", [{"input": {"x": 1}, "source_trace_ref": {"run_id": "r1"}}]
+    )
+    assert http.last_body["items"][0]["source_trace_ref"] == {"run_id": "r1"}
+
+
+def test_append_requires_a_ref():
+    with pytest.raises(ValueError, match="requires a dataset ref"):
+        DatasetManager(FakeHttp()).append("", [{"input": {"x": 1}}])
+
+
+def test_append_requires_items():
+    with pytest.raises(ValueError, match="requires items"):
+        DatasetManager(FakeHttp()).append("ds", [])
+
+
+def test_append_requires_input_on_each_item():
+    with pytest.raises(ValueError, match="requires an 'input'"):
+        DatasetManager(FakeHttp()).append("ds", [{"expected_output": "x"}])

trodo_python-2.16.0/tests/test_experiments.py

@@ -0,0 +1,120 @@
+"""Experiment ingest — ExperimentManager.ingest.
+
+Uses a fake HTTP client so no network is touched; asserts on the exact body
+posted to experiments/ingest and the experiment dict returned to the caller.
+"""
+
+import pytest
+
+from trodo.managers.experiment_manager import ExperimentManager
+
+
+class FakeHttp:
+    """Records the last ingest call; mirrors the prompt-test fakeApi."""
+
+    def __init__(self, response=None):
+        self._response = response if response is not None else {
+            "experiment": {"id": "exp_1", "status": "grading"}
+        }
+        self.calls = 0
+        self.last_body = None
+
+    def ingest_experiment(self, body):
+        self.calls += 1
+        self.last_body = body
+        return self._response
+
+
+def test_ingest_posts_body_and_returns_experiment():
+    http = FakeHttp()
+    mgr = ExperimentManager(http)
+    exp = mgr.ingest(
+        "support-eval",
+        [{"item_position": 0, "output": "here is your tracking link"}],
+    )
+
+    assert http.calls == 1
+    assert http.last_body == {
+        "dataset": "support-eval",
+        "outputs": [{"item_position": 0, "output": "here is your tracking link"}],
+    }
+    assert exp == {"id": "exp_1", "status": "grading"}
+
+
+def test_ingest_includes_all_optional_fields_when_set():
+    http = FakeHttp()
+    ExperimentManager(http).ingest(
+        "ds",
+        [{"item_position": 1, "output": "o"}],
+        name="run-1",
+        task_name="qa",
+        judge={"credential_id": "cred_1", "provider": "openai", "model": "gpt-4o"},
+        evaluator_ids=["eval_a", "eval_b"],
+        dataset_version_no=3,
+    )
+    assert http.last_body == {
+        "dataset": "ds",
+        "outputs": [{"item_position": 1, "output": "o"}],
+        "name": "run-1",
+        "task_name": "qa",
+        "judge": {"credential_id": "cred_1", "provider": "openai", "model": "gpt-4o"},
+        "evaluator_ids": ["eval_a", "eval_b"],
+        "dataset_version_no": 3,
+    }
+
+
+def test_ingest_omits_unset_optional_fields():
+    http = FakeHttp()
+    ExperimentManager(http).ingest("ds", [{"item_position": 0, "output": "o"}])
+    assert set(http.last_body.keys()) == {"dataset", "outputs"}
+
+
+def test_ingest_drops_none_per_output_fields():
+    http = FakeHttp()
+    ExperimentManager(http).ingest(
+        "ds",
+        [
+            {
+                "item_position": 0,
+                "output": "o",
+                "expected_output": "gold",
+                "query": None,
+                "context": None,
+                "cost_usd": 0.0021,
+                "latency_ms": None,
+            }
+        ],
+    )
+    assert http.last_body["outputs"] == [
+        {"item_position": 0, "output": "o", "expected_output": "gold", "cost_usd": 0.0021}
+    ]
+
+
+def test_ingest_keeps_item_position_zero():
+    # item_position 0 is falsy but valid — must not be dropped.
+    http = FakeHttp()
+    ExperimentManager(http).ingest("ds", [{"item_position": 0, "output": "o"}])
+    assert http.last_body["outputs"][0]["item_position"] == 0
+
+
+def test_ingest_returns_raw_response_when_no_experiment_key():
+    http = FakeHttp(response={"error": "bad"})
+    out = ExperimentManager(http).ingest("ds", [{"item_position": 0, "output": "o"}])
+    assert out == {"error": "bad"}
+
+
+def test_ingest_requires_a_dataset():
+    with pytest.raises(ValueError, match="requires a dataset"):
+        ExperimentManager(FakeHttp()).ingest("", [{"item_position": 0, "output": "o"}])
+
+
+def test_ingest_requires_outputs():
+    with pytest.raises(ValueError, match="requires outputs"):
+        ExperimentManager(FakeHttp()).ingest("ds", [])
+
+
+def test_ingest_requires_item_position_and_output():
+    with pytest.raises(ValueError, match="item_position"):
+        ExperimentManager(FakeHttp()).ingest("ds", [{"output": "o"}])
+    with pytest.raises(ValueError, match="requires an 'output'"):
+        ExperimentManager(FakeHttp()).ingest("ds", [{"item_position": 0}])

{trodo_python-2.15.0 → trodo_python-2.16.0}/trodo/__init__.py

@@ -41,7 +41,7 @@ Downstream microservice (join the caller's run instead of making a new one):
 
 from __future__ import annotations
 
-__version__ = "2.15.0"
+__version__ = "2.16.0"
 
 from typing import Any, Callable, Dict, List, Optional, Union
 
@@ -119,6 +119,9 @@ __all__ = [
     "TemplateError",
     "ManagedPrompt",
     "PromptSummary",
+    # Datasets & experiments ingest
+    "append_dataset",
+    "ingest_experiment",
 ]
 
 # ============================================================================
@@ -344,6 +347,64 @@ def render_template(
     return _render_template(template, merged, strict=strict)
 
 
+# ----------------------------------------------------------------------------
+# Datasets & experiments ingest
+# ----------------------------------------------------------------------------
+
+def append_dataset(
+    ref: str,
+    items: Union[Dict[str, Any], List[Dict[str, Any]]],
+    create: bool = True,
+) -> Dict[str, Any]:
+    """Append items to a dataset by UUID or name.
+
+    *items* is a list of dicts shaped ``{"input": {<var>: value, ...},
+    "expected_output"?: str, "metadata"?: dict, "source_trace_ref"?: dict}``;
+    a single dict is accepted and wrapped. When *ref* is a name that doesn't
+    exist, ``create=True`` (default) auto-creates the dataset::
+
+        trodo.append_dataset("support-eval", [
+            {"input": {"q": "where is my order"}, "expected_output": "tracking link"},
+        ])
+
+    Returns ``{"dataset_id", "dataset_name", "appended", "item_count"}``.
+    """
+    return _get_client().datasets.append(ref, items, create=create)
+
+
+def ingest_experiment(
+    dataset: str,
+    outputs: List[Dict[str, Any]],
+    name: Optional[str] = None,
+    task_name: Optional[str] = None,
+    judge: Optional[Dict[str, Any]] = None,
+    evaluator_ids: Optional[List[str]] = None,
+    dataset_version_no: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Ingest a batch of model *outputs* for *dataset* and grade them.
+
+    *outputs* is a list of dicts shaped ``{"item_position": int, "output": str,
+    "expected_output"?, "query"?, "context"?, "cost_usd"?, "latency_ms"?}``.
+    Configure grading with ``judge`` (``{"credential_id", "provider", "model"}``)
+    and/or ``evaluator_ids``::
+
+        exp = trodo.ingest_experiment("support-eval", [
+            {"item_position": 0, "output": "here is your tracking link"},
+        ], judge={"credential_id": "cred_1", "provider": "openai", "model": "gpt-4o"})
+
+    Returns the experiment dict.
+    """
+    return _get_client().experiments.ingest(
+        dataset,
+        outputs,
+        name=name,
+        task_name=task_name,
+        judge=judge,
+        evaluator_ids=evaluator_ids,
+        dataset_version_no=dataset_version_no,
+    )
+
+
 def enable_auto_events() -> None:
     _get_client().enable_auto_events()
 

{trodo_python-2.15.0 → trodo_python-2.16.0}/trodo/api/endpoints.py

@@ -23,3 +23,6 @@ RUNS_INGEST = "/api/sdk/runs/ingest"
 RUNS_START = "/api/sdk/runs/start"
 RUNS_BASE = "/api/sdk/runs"  # /runs/{run_id}/end, /spans, /feedback
 OTLP_TRACES = "/api/sdk/otel/v1/traces"
+# Datasets & experiments ingest
+DATASETS_BASE = "/api/sdk/datasets"  # /datasets/{ref}/items
+EXPERIMENTS_INGEST = "/api/sdk/experiments/ingest"

{trodo_python-2.15.0 → trodo_python-2.16.0}/trodo/api/http_client.py

@@ -133,6 +133,16 @@ class HttpClient:
     def list_prompts(self) -> ApiResult:
         return self._get("/api/sdk/prompts")
 
+    def append_dataset_items(self, ref: str, body: Dict[str, Any]) -> ApiResult:
+        """Append items to a dataset by UUID or name (name is URL-encoded)."""
+        from urllib.parse import quote
+        return self._request(
+            f"/api/sdk/datasets/{quote(str(ref), safe='')}/items", body
+        )
+
+    def ingest_experiment(self, body: Dict[str, Any]) -> ApiResult:
+        return self._request("/api/sdk/experiments/ingest", body)
+
     def post_track(self, session_data: Dict[str, Any]) -> ApiResult:
         return self._request("/api/sdk/track", {"sessionData": session_data})
 

{trodo_python-2.15.0 → trodo_python-2.16.0}/trodo/client.py

@@ -77,6 +77,8 @@ class TrodoClient:
 
         self._session_manager = SessionManager()
         self._prompts = None  # lazily-built PromptManager
+        self._datasets = None  # lazily-built DatasetManager
+        self._experiments = None  # lazily-built ExperimentManager
 
         if batch_enabled:
             self._event_queue: Optional[EventQueue] = EventQueue(batch_size)
@@ -125,6 +127,22 @@ class TrodoClient:
             self._prompts = PromptManager(self._http)
         return self._prompts
 
+    @property
+    def datasets(self):
+        """Append items to the team's evaluation datasets."""
+        if self._datasets is None:
+            from .managers.dataset_manager import DatasetManager
+            self._datasets = DatasetManager(self._http)
+        return self._datasets
+
+    @property
+    def experiments(self):
+        """Ingest experiment result batches for server-side grading."""
+        if self._experiments is None:
+            from .managers.experiment_manager import ExperimentManager
+            self._experiments = ExperimentManager(self._http)
+        return self._experiments
+
     # --------------------------------------------------------------------------
     # Primary pattern: for_user()
     # --------------------------------------------------------------------------

trodo_python-2.16.0/trodo/managers/dataset_manager.py

@@ -0,0 +1,74 @@
+"""Dataset ingest — append evaluation items to a Trodo dataset.
+
+Datasets are the fixed inputs an experiment runs against: each item is a set of
+variable values (``input``) plus an optional gold answer (``expected_output``)
+and free-form ``metadata``. This manager appends items to a dataset addressed by
+UUID or by name; when a name is used and no such dataset exists it is
+auto-created (``create=True``, the default), so a first append bootstraps the
+dataset without a separate create call.
+
+Mirrors ``sdks/trodo-node-sdk/src/managers/DatasetManager.ts`` in shape.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Union
+
+__all__ = ["DatasetManager"]
+
+# The optional per-item fields, in wire order. ``input`` is always required.
+_ITEM_OPTIONAL = ("expected_output", "metadata", "source_trace_ref")
+
+
+def _clean_item(item: Dict[str, Any]) -> Dict[str, Any]:
+    if not isinstance(item, dict):
+        raise ValueError("trodo: each dataset item must be a dict")
+    if "input" not in item or item["input"] is None:
+        raise ValueError("trodo: each dataset item requires an 'input' dict")
+    out: Dict[str, Any] = {"input": item["input"]}
+    for key in _ITEM_OPTIONAL:
+        if item.get(key) is not None:
+            out[key] = item[key]
+    return out
+
+
+class DatasetManager:
+    """Append items to the team's evaluation datasets."""
+
+    def __init__(self, http_client: Any) -> None:
+        self._http = http_client
+
+    def append(
+        self,
+        ref: str,
+        items: Union[Dict[str, Any], List[Dict[str, Any]]],
+        create: bool = True,
+    ) -> Dict[str, Any]:
+        """Append one or more items to the dataset *ref* (a UUID or a name).
+
+        *items* is a list of dicts shaped ``{"input": {<var>: value, ...},
+        "expected_output"?: str, "metadata"?: dict, "source_trace_ref"?: dict}``;
+        a single dict is accepted and wrapped into a one-item list. ``None``
+        optional fields are dropped from the wire payload.
+
+        When *ref* is a name that doesn't exist yet, ``create=True`` (default)
+        auto-creates the dataset; pass ``create=False`` to require it to exist.
+
+        Returns the parsed response dict:
+        ``{"dataset_id", "dataset_name", "appended", "item_count"}``.
+
+        :raises ValueError: if *ref* is empty, *items* is empty, or an item has
+            no ``input``.
+        """
+        if not ref:
+            raise ValueError("trodo: append_dataset(ref) requires a dataset ref")
+        if isinstance(items, dict):
+            items = [items]
+        if not items:
+            raise ValueError("trodo: append_dataset(ref, items) requires items")
+
+        body = {
+            "items": [_clean_item(i) for i in items],
+            "create": bool(create),
+        }
+        return self._http.append_dataset_items(ref, body)

trodo_python-2.16.0/trodo/managers/experiment_manager.py

@@ -0,0 +1,105 @@
+"""Experiment ingest — record a batch of model outputs against a dataset.
+
+An experiment takes a dataset (the fixed inputs) and a set of *outputs* your
+model produced for each item, then grades them server-side using the judge
+and/or evaluators you name. You run the model yourself and hand Trodo the
+results; Trodo scores and stores them so runs are comparable over time.
+
+``outputs`` are matched to dataset items by ``item_position`` (0-based),
+carrying the produced ``output`` and optional per-row context (``query``,
+``context``), cost/latency telemetry, and a per-row ``expected_output`` override.
+
+Mirrors ``sdks/trodo-node-sdk/src/managers/ExperimentManager.ts`` in shape.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+__all__ = ["ExperimentManager"]
+
+# Optional per-output fields, in wire order. ``item_position`` + ``output`` are
+# always required.
+_OUTPUT_OPTIONAL = (
+    "expected_output",
+    "query",
+    "context",
+    "cost_usd",
+    "latency_ms",
+)
+
+
+def _clean_output(output: Dict[str, Any]) -> Dict[str, Any]:
+    if not isinstance(output, dict):
+        raise ValueError("trodo: each experiment output must be a dict")
+    if output.get("item_position") is None:
+        raise ValueError("trodo: each output requires an 'item_position' (int)")
+    if output.get("output") is None:
+        raise ValueError("trodo: each output requires an 'output' (str)")
+    out: Dict[str, Any] = {
+        "item_position": output["item_position"],
+        "output": output["output"],
+    }
+    for key in _OUTPUT_OPTIONAL:
+        if output.get(key) is not None:
+            out[key] = output[key]
+    return out
+
+
+class ExperimentManager:
+    """Ingest experiment result batches for server-side grading."""
+
+    def __init__(self, http_client: Any) -> None:
+        self._http = http_client
+
+    def ingest(
+        self,
+        dataset: str,
+        outputs: List[Dict[str, Any]],
+        name: Optional[str] = None,
+        task_name: Optional[str] = None,
+        judge: Optional[Dict[str, Any]] = None,
+        evaluator_ids: Optional[List[str]] = None,
+        dataset_version_no: Optional[int] = None,
+    ) -> Dict[str, Any]:
+        """Ingest a batch of *outputs* for *dataset* and return the experiment.
+
+        *dataset* is a dataset name or UUID. *outputs* is a list of dicts shaped
+        ``{"item_position": int, "output": str, "expected_output"?, "query"?,
+        "context"?, "cost_usd"?, "latency_ms"?}``. ``None`` optional fields are
+        dropped from the wire payload.
+
+        Grading is configured server-side: pass ``judge`` (``{"credential_id",
+        "provider", "model"}``) for an LLM judge and/or ``evaluator_ids`` for
+        named evaluators. ``dataset_version_no`` pins the dataset version graded
+        against.
+
+        Returns the experiment dict from ``{"experiment": {...}}``.
+
+        :raises ValueError: if *dataset* is empty, *outputs* is empty, or an
+            output is missing ``item_position``/``output``.
+        """
+        if not dataset:
+            raise ValueError("trodo: ingest_experiment(dataset) requires a dataset")
+        if not outputs:
+            raise ValueError("trodo: ingest_experiment requires outputs")
+
+        body: Dict[str, Any] = {
+            "dataset": dataset,
+            "outputs": [_clean_output(o) for o in outputs],
+        }
+        if name is not None:
+            body["name"] = name
+        if task_name is not None:
+            body["task_name"] = task_name
+        if judge is not None:
+            body["judge"] = judge
+        if evaluator_ids is not None:
+            body["evaluator_ids"] = evaluator_ids
+        if dataset_version_no is not None:
+            body["dataset_version_no"] = dataset_version_no
+
+        res = self._http.ingest_experiment(body)
+        if isinstance(res, dict) and "experiment" in res:
+            return res["experiment"]
+        return res

{trodo_python-2.15.0 → trodo_python-2.16.0}/trodo_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trodo-python
-Version: 2.15.0
+Version: 2.16.0
 Summary: Trodo Analytics SDK for Python — server-side event tracking
 License: ISC
 Keywords: analytics,tracking,trodo,server-side

{trodo_python-2.15.0 → trodo_python-2.16.0}/trodo_python.egg-info/SOURCES.txt

@@ -3,8 +3,10 @@ pyproject.toml
 tests/test_anon_distinct_id.py
 tests/test_auto_instrument_fixes.py
 tests/test_cross_process_session.py
+tests/test_datasets.py
 tests/test_end_run.py
 tests/test_error_enrichment.py
+tests/test_experiments.py
 tests/test_llm_usage_cost.py
 tests/test_processor_methods.py
 tests/test_prompt_trace.py
@@ -23,6 +25,8 @@ trodo/api/http_client.py
 trodo/auto/__init__.py
 trodo/auto/auto_event_manager.py
 trodo/managers/__init__.py
+trodo/managers/dataset_manager.py
+trodo/managers/experiment_manager.py
 trodo/managers/group_manager.py
 trodo/managers/people_manager.py
 trodo/managers/prompt_manager.py