render-lab-tasks-braintrust 0.1.0__tar.gz

render_lab_tasks_braintrust-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+dist/
+*.egg-info/
+.env
+.env.*
+!.env.example

render_lab_tasks_braintrust-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Render Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

render_lab_tasks_braintrust-0.1.0/PKG-INFO

@@ -0,0 +1,84 @@
+Metadata-Version: 2.5
+Name: render-lab-tasks-braintrust
+Version: 0.1.0
+Summary: braintrust tasks for Render Workflows
+Project-URL: Repository, https://github.com/render-lab/render-tasks-python
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: httpx<0.29,>=0.28
+Requires-Dist: render-lab-tasks-core<0.2,>=0.1.1
+Requires-Dist: render==1.0.1
+Description-Content-Type: text/markdown
+
+# render-lab-tasks-braintrust
+
+Unreleased Python port: **12 registered tasks** from
+[render-lab/render-tasks](https://github.com/render-lab/render-tasks/tree/45f9c2d44bd28e01ae9813e0d2ff56ee6c533816/packages/tasks-braintrust).
+Python 3.12+, Render SDK 1.0.1. Requires core 0.1.1 or later.
+
+From the repository root:
+
+```sh
+uv sync --all-packages --locked
+```
+
+Import `render_lab_tasks_braintrust.tasks` to register tasks. The package root is inert.
+Compose the exported `app` with `Workflows.from_workflows`; call tasks through
+`ctx.run`. Every operation also exports `*_impl(ctx, input, *, deps=None)` for
+injection. `Client(http, env)` accepts a caller-owned HTTPX client, and
+`Deps(braintrust=client)` injects it. Default dependencies open and close the HTTP
+client per invocation; credentials are read lazily at use. No HTTP retries or
+background vendor polling occur; `retry.py` owns the durable retry policies.
+
+## Task surface
+
+| Registered task | Python export |
+| --- | --- |
+| `braintrust.addFeedback` | `add_feedback` |
+| `braintrust.awaitReview` | `await_review` |
+| `braintrust.deleteDatasetRows` | `delete_dataset_rows` |
+| `braintrust.ensureDataset` | `ensure_dataset` |
+| `braintrust.ensureExperiment` | `ensure_experiment` |
+| `braintrust.gateExperiment` | `gate_experiment` |
+| `braintrust.getReviewProgress` | `get_review_progress` |
+| `braintrust.listDatasetRows` | `list_dataset_rows` |
+| `braintrust.logExperimentResults` | `log_experiment_results` |
+| `braintrust.runEval` | `run_eval` |
+| `braintrust.summarizeExperiment` | `summarize_experiment` |
+| `braintrust.upsertDatasetRows` | `upsert_dataset_rows` |
+
+Typed JSON inputs and results are in `types.py`. Task names and JSON field names
+match the pinned source; Python function names use snake_case.
+
+## Environment
+
+| Variable | Requirement |
+| --- | --- |
+| `BRAINTRUST_API_KEY` | Required at first API call. |
+| `BRAINTRUST_API_URL` | Optional; https://api.braintrust.dev by default. |
+
+## Behavior and limits
+
+Stable row IDs are required for upserts and experiment logs. Eval starts and release gates have no retries. Review polling counts rows carrying a score against root-row totals, matching the source; confirm review semantics with nested spans. Dataset pages and experiment summaries reject serialized results at or above 4 MiB. Gate errors contain the violated thresholds. Review polling and result counts remain unverified against a real project.
+
+## Verification
+
+All registered tasks have hermetic contract fixtures executed independently against
+the pinned TS implementation. SDK-backed tasks also have explicit HTTP request
+fixtures. See `tests/test_batch3_contracts.py` and `tests/test_batch3_edges.py`.
+These are not live vendor results.
+
+Vendor webhooks subpaths and other registration-free TS exports are outside this
+batch unless explicitly listed above. Full registered-task coverage does not
+imply all supporting exports are ported.
+
+Pending scopes, test resources, replay, and hosted checks are in the
+[live-testing backlog](https://github.com/render-lab/render-tasks-python/issues/1)
+and [verification tracker](../../docs/verification-tracker.md).
+
+## Installation
+
+```sh
+pip install render-lab-tasks-braintrust==0.1.0
+```

render_lab_tasks_braintrust-0.1.0/README.md

@@ -0,0 +1,71 @@
+# render-lab-tasks-braintrust
+
+Unreleased Python port: **12 registered tasks** from
+[render-lab/render-tasks](https://github.com/render-lab/render-tasks/tree/45f9c2d44bd28e01ae9813e0d2ff56ee6c533816/packages/tasks-braintrust).
+Python 3.12+, Render SDK 1.0.1. Requires core 0.1.1 or later.
+
+From the repository root:
+
+```sh
+uv sync --all-packages --locked
+```
+
+Import `render_lab_tasks_braintrust.tasks` to register tasks. The package root is inert.
+Compose the exported `app` with `Workflows.from_workflows`; call tasks through
+`ctx.run`. Every operation also exports `*_impl(ctx, input, *, deps=None)` for
+injection. `Client(http, env)` accepts a caller-owned HTTPX client, and
+`Deps(braintrust=client)` injects it. Default dependencies open and close the HTTP
+client per invocation; credentials are read lazily at use. No HTTP retries or
+background vendor polling occur; `retry.py` owns the durable retry policies.
+
+## Task surface
+
+| Registered task | Python export |
+| --- | --- |
+| `braintrust.addFeedback` | `add_feedback` |
+| `braintrust.awaitReview` | `await_review` |
+| `braintrust.deleteDatasetRows` | `delete_dataset_rows` |
+| `braintrust.ensureDataset` | `ensure_dataset` |
+| `braintrust.ensureExperiment` | `ensure_experiment` |
+| `braintrust.gateExperiment` | `gate_experiment` |
+| `braintrust.getReviewProgress` | `get_review_progress` |
+| `braintrust.listDatasetRows` | `list_dataset_rows` |
+| `braintrust.logExperimentResults` | `log_experiment_results` |
+| `braintrust.runEval` | `run_eval` |
+| `braintrust.summarizeExperiment` | `summarize_experiment` |
+| `braintrust.upsertDatasetRows` | `upsert_dataset_rows` |
+
+Typed JSON inputs and results are in `types.py`. Task names and JSON field names
+match the pinned source; Python function names use snake_case.
+
+## Environment
+
+| Variable | Requirement |
+| --- | --- |
+| `BRAINTRUST_API_KEY` | Required at first API call. |
+| `BRAINTRUST_API_URL` | Optional; https://api.braintrust.dev by default. |
+
+## Behavior and limits
+
+Stable row IDs are required for upserts and experiment logs. Eval starts and release gates have no retries. Review polling counts rows carrying a score against root-row totals, matching the source; confirm review semantics with nested spans. Dataset pages and experiment summaries reject serialized results at or above 4 MiB. Gate errors contain the violated thresholds. Review polling and result counts remain unverified against a real project.
+
+## Verification
+
+All registered tasks have hermetic contract fixtures executed independently against
+the pinned TS implementation. SDK-backed tasks also have explicit HTTP request
+fixtures. See `tests/test_batch3_contracts.py` and `tests/test_batch3_edges.py`.
+These are not live vendor results.
+
+Vendor webhooks subpaths and other registration-free TS exports are outside this
+batch unless explicitly listed above. Full registered-task coverage does not
+imply all supporting exports are ported.
+
+Pending scopes, test resources, replay, and hosted checks are in the
+[live-testing backlog](https://github.com/render-lab/render-tasks-python/issues/1)
+and [verification tracker](../../docs/verification-tracker.md).
+
+## Installation
+
+```sh
+pip install render-lab-tasks-braintrust==0.1.0
+```

render_lab_tasks_braintrust-0.1.0/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling>=1.27,<2"]
+build-backend = "hatchling.build"
+
+[project]
+name = "render-lab-tasks-braintrust"
+version = "0.1.0"
+description = "braintrust tasks for Render Workflows"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.12"
+dependencies = ["render==1.0.1", "httpx>=0.28,<0.29", "render-lab-tasks-core>=0.1.1,<0.2"]
+
+[project.urls]
+Repository = "https://github.com/render-lab/render-tasks-python"
+
+[tool.uv.sources]
+render-lab-tasks-core = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/render_lab_tasks_braintrust"]

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/__init__.py

@@ -0,0 +1 @@
+"""Import .tasks explicitly to register durable tasks."""

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/_app.py

@@ -0,0 +1,3 @@
+from render import Workflows
+
+app = Workflows()

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/add_feedback.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import AddFeedbackInput, AddFeedbackResult
+
+
+async def add_feedback_impl(
+    ctx: TaskContext, input: AddFeedbackInput, *, deps: Deps | None = None
+) -> AddFeedbackResult:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.add_feedback(input)
+
+
+@app.task(name="braintrust.addFeedback", retry=BRAINTRUST_RETRY)
+async def add_feedback(ctx: TaskContext, input: AddFeedbackInput) -> AddFeedbackResult:
+    return await add_feedback_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/await_review.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_REVIEW_RETRY
+from .types import AwaitReviewInput, ReviewProgressDTO
+
+
+async def await_review_impl(
+    ctx: TaskContext, input: AwaitReviewInput, *, deps: Deps | None = None
+) -> ReviewProgressDTO:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.await_review(input)
+
+
+@app.task(name="braintrust.awaitReview", retry=BRAINTRUST_REVIEW_RETRY)
+async def await_review(ctx: TaskContext, input: AwaitReviewInput) -> ReviewProgressDTO:
+    return await await_review_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/client.py

@@ -0,0 +1,453 @@
+"""Injected vendor interface and one-attempt async HTTP adapter."""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import AsyncIterator, Mapping, Sequence
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import Any, Protocol, cast
+
+import httpx
+from render_lab_tasks_core.http import (
+    HttpClient,
+    enc,
+    js_string,
+    nullish,
+    pick,
+    required,
+)
+
+from . import types as t
+
+
+def obj(value: Any) -> dict[str, Any]:
+    return value if isinstance(value, dict) else {}
+
+
+def array(value: Any) -> list[Any]:
+    return value if isinstance(value, list) else []
+
+
+def number(value: Any, default: Any = None) -> Any:
+    return value if isinstance(value, (int, float)) and not isinstance(value, bool) else default
+
+
+def json_text(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
+
+
+def budget(label: str, value: Any) -> None:
+    size = len(json_text(value).encode())
+    if size >= 4 * 1024 * 1024:
+        raise ValueError(
+            f"Braintrust {label} result is {size} bytes, at or above "
+            f"the 4194304 byte limit. Lower the limit or narrow the "
+            f"request."
+        )
+
+
+def validate_names(input: Mapping[str, Any], task: str, *keys: str) -> None:
+    for key in keys:
+        if not input[key].strip():
+            raise ValueError(f"braintrust.{task} requires a non-blank {key}.")
+
+
+def validate_rows(rows: Sequence[Mapping[str, Any]], task: str) -> None:
+    if not 1 <= len(rows) <= 1000:
+        raise ValueError(f"braintrust.{task} accepts 1 to 1000 rows per call.")
+    seen: set[str] = set()
+    for row in rows:
+        id = row["id"]
+        if not isinstance(id, str) or not id.strip():
+            raise ValueError(f"braintrust.{task} requires a stable non-blank id on every row.")
+        if id in seen:
+            raise ValueError(f'braintrust.{task} received a duplicate row id "{id}".')
+        seen.add(id)
+
+
+def dataset(value: Any) -> t.DatasetDTO:
+    row = obj(value)
+    return {
+        "id": nullish(row.get("id"), ""),
+        "projectId": nullish(row.get("project_id"), ""),
+        "name": nullish(row.get("name"), ""),
+        "description": row.get("description"),
+    }
+
+
+def dataset_row(value: Any) -> t.DatasetRowDTO:
+    row = obj(value)
+    result = {
+        "id": nullish(row.get("id"), ""),
+        "input": row.get("input"),
+        **pick(row, "expected", "metadata"),
+    }
+    if isinstance(row.get("tags"), list):
+        result["tags"] = row["tags"]
+    return cast(t.DatasetRowDTO, result)
+
+
+def function_ref(ref: Mapping[str, Any]) -> dict[str, Any]:
+    return {"function_id": ref["functionId"], **pick(ref, "version")}
+
+
+def summary(value: Any) -> t.ExperimentSummaryDTO:
+    row = obj(value)
+    return {
+        "projectName": nullish(row.get("project_name"), ""),
+        "experimentName": nullish(row.get("experiment_name"), ""),
+        "experimentId": nullish(row.get("experiment_id"), ""),
+        "experimentUrl": nullish(row.get("experiment_url"), ""),
+        "comparisonExperimentName": row.get("comparison_experiment_name"),
+        "scores": {
+            k: {
+                "score": number(obj(v).get("score")),
+                "diff": number(obj(v).get("diff")),
+                "improvements": number(obj(v).get("improvements"), 0),
+                "regressions": number(obj(v).get("regressions"), 0),
+            }
+            for k, v in obj(row.get("scores")).items()
+        },
+        "metrics": {
+            k: {
+                "metric": number(obj(v).get("metric")),
+                "unit": obj(v).get("unit") if isinstance(obj(v).get("unit"), str) else None,
+            }
+            for k, v in obj(row.get("metrics")).items()
+        },
+    }
+
+
+def build_review_query(input: Mapping[str, Any]) -> str:
+    key = '"' + input["scoreKey"].replace('"', '""') + '"'
+    experiment = "'" + input["experimentId"].replace("'", "''") + "'"
+    return (
+        f"\nSELECT\n  sum(CASE WHEN scores.{key} IS NOT NULL THEN "
+        f"1 ELSE 0 END) AS reviewed,\n  sum(CASE WHEN is_root "
+        f"THEN 1 ELSE 0 END) AS total\nFROM "
+        f"experiment({experiment})\n"
+    )
+
+
+class Port(Protocol):
+    async def add_feedback(self, input: t.AddFeedbackInput) -> t.AddFeedbackResult: ...
+    async def await_review(self, input: t.AwaitReviewInput) -> t.ReviewProgressDTO: ...
+    async def delete_dataset_rows(
+        self, input: t.DeleteDatasetRowsInput
+    ) -> t.DeleteDatasetRowsResult: ...
+    async def ensure_dataset(self, input: t.EnsureDatasetInput) -> t.DatasetDTO: ...
+    async def ensure_experiment(self, input: t.EnsureExperimentInput) -> t.ExperimentDTO: ...
+    async def gate_experiment(self, input: t.GateExperimentInput) -> t.GateExperimentResult: ...
+    async def get_review_progress(self, input: t.GetReviewProgressInput) -> t.ReviewProgressDTO: ...
+    async def list_dataset_rows(self, input: t.ListDatasetRowsInput) -> t.ListDatasetRowsResult: ...
+    async def log_experiment_results(
+        self, input: t.LogExperimentResultsInput
+    ) -> t.LogExperimentResultsResult: ...
+    async def run_eval(self, input: t.RunEvalInput) -> t.ExperimentSummaryDTO: ...
+    async def summarize_experiment(
+        self, input: t.SummarizeExperimentInput
+    ) -> t.ExperimentSummaryDTO: ...
+    async def upsert_dataset_rows(
+        self, input: t.UpsertDatasetRowsInput
+    ) -> t.UpsertDatasetRowsResult: ...
+
+
+@dataclass(frozen=True)
+class Deps:
+    braintrust: Port
+
+
+class Client:
+    def __init__(self, http: httpx.AsyncClient, env: Mapping[str, str] | None = None) -> None:
+        self.env = os.environ if env is None else env
+        self.api = HttpClient(
+            http,
+            lambda: self.env.get("BRAINTRUST_API_URL", "https://api.braintrust.dev"),
+            auth=lambda: {"authorization": "Bearer " + required(self.env, "BRAINTRUST_API_KEY")},
+            label="Braintrust API",
+        )
+
+    async def ensure_dataset(self, input: t.EnsureDatasetInput) -> t.DatasetDTO:
+        validate_names(input, "ensureDataset", "projectId", "name")
+        rows = array(
+            obj(
+                await self.api.call(
+                    "/v1/dataset?project_id="
+                    + enc(input["projectId"])
+                    + "&dataset_name="
+                    + enc(input["name"])
+                )
+            ).get("objects")
+        )
+        row = (
+            rows[0]
+            if rows
+            else await self.api.call(
+                "/v1/dataset",
+                method="POST",
+                body={"project_id": input["projectId"], **pick(input, "name", "description")},
+            )
+        )
+        return dataset(row)
+
+    async def ensure_experiment(self, input: t.EnsureExperimentInput) -> t.ExperimentDTO:
+        validate_names(input, "ensureExperiment", "projectId", "name")
+        rows = array(
+            obj(
+                await self.api.call(
+                    "/v1/experiment?project_id="
+                    + enc(input["projectId"])
+                    + "&experiment_name="
+                    + enc(input["name"])
+                )
+            ).get("objects")
+        )
+        body = {"project_id": input["projectId"], **pick(input, "name", "description")}
+        if "baseExperimentId" in input:
+            body["base_exp_id"] = input["baseExperimentId"]
+        row = obj(
+            rows[0] if rows else await self.api.call("/v1/experiment", method="POST", body=body)
+        )
+        result: t.ExperimentDTO = {**dataset(row), "baseExperimentId": row.get("base_exp_id")}
+        if "baseExperimentId" in input and result["baseExperimentId"] != input["baseExperimentId"]:
+            raise ValueError(
+                f"braintrust.ensureExperiment: experiment "
+                f'"{input["name"]}" already exists with base '
+                f"{json_text(result['baseExperimentId'])}, which "
+                f"conflicts with the requested base "
+                f"{json_text(input['baseExperimentId'])}."
+            )
+        return result
+
+    async def upsert_dataset_rows(
+        self, input: t.UpsertDatasetRowsInput
+    ) -> t.UpsertDatasetRowsResult:
+        validate_rows(input["rows"], "upsertDatasetRows")
+        events = [pick(row, "id", "input", "expected", "metadata", "tags") for row in input["rows"]]
+        data = obj(
+            await self.api.call(
+                "/v1/dataset/" + enc(input["datasetId"]) + "/insert",
+                method="POST",
+                body={"events": events},
+            )
+        )
+        return {
+            "rowIds": data["row_ids"]
+            if isinstance(data.get("row_ids"), list)
+            else [r["id"] for r in input["rows"]]
+        }
+
+    async def list_dataset_rows(self, input: t.ListDatasetRowsInput) -> t.ListDatasetRowsResult:
+        if "limit" in input and not 1 <= input["limit"] <= 1000:
+            raise ValueError("braintrust.listDatasetRows limit must be between 1 and 1000.")
+        data = obj(
+            await self.api.call(
+                "/v1/dataset/" + enc(input["datasetId"]) + "/fetch",
+                method="POST",
+                body=pick(input, "limit", "cursor"),
+            )
+        )
+        result: t.ListDatasetRowsResult = {
+            "rows": [dataset_row(r) for r in array(data.get("events"))],
+            "nextCursor": data.get("cursor"),
+        }
+        budget("listDatasetRows", result)
+        return result
+
+    async def delete_dataset_rows(
+        self, input: t.DeleteDatasetRowsInput
+    ) -> t.DeleteDatasetRowsResult:
+        ids = input["rowIds"]
+        if not ids:
+            return {"deletedRowIds": []}
+        if len(ids) > 1000:
+            raise ValueError("braintrust.deleteDatasetRows accepts at most 1000 ids per call.")
+        if any(not isinstance(id, str) or not id.strip() for id in ids):
+            raise ValueError("braintrust.deleteDatasetRows requires non-blank row ids.")
+        await self.api.call(
+            "/v1/dataset/" + enc(input["datasetId"]) + "/insert",
+            method="POST",
+            body={"events": [{"id": id, "_object_delete": True} for id in ids]},
+        )
+        return {"deletedRowIds": ids}
+
+    async def run_eval(self, input: t.RunEvalInput) -> t.ExperimentSummaryDTO:
+        if not input["scores"]:
+            raise ValueError("braintrust.runEval requires at least one scoring function reference.")
+        body: dict[str, Any] = {
+            "project_id": input["projectId"],
+            "data": {"dataset_id": input["datasetId"]},
+            "task": function_ref(input["task"]),
+            "scores": [function_ref(r) for r in input["scores"]],
+            "stream": False,
+            **pick(input, "metadata"),
+        }
+        if "experimentName" in input:
+            body["experiment_name"] = input["experimentName"]
+        if "repoCommit" in input:
+            body["repo_info"] = {"commit": input["repoCommit"]}
+        data = obj(await self.api.call("/v1/eval", method="POST", body=body))
+        result = summary(nullish(data.get("summary"), data))
+        budget("runEval", result)
+        return result
+
+    async def log_experiment_results(
+        self, input: t.LogExperimentResultsInput
+    ) -> t.LogExperimentResultsResult:
+        validate_rows(input["rows"], "logExperimentResults")
+        for row in input["rows"]:
+            for key, value in row.get("scores", {}).items():
+                if value < 0 or value > 1:
+                    raise ValueError(
+                        f'braintrust.logExperimentResults score "{key}" on row '
+                        f'"{row["id"]}" is {js_string(value)}; scores must be '
+                        f"within [0, 1]."
+                    )
+        data = obj(
+            await self.api.call(
+                "/v1/experiment/" + enc(input["experimentId"]) + "/insert",
+                method="POST",
+                body={
+                    "events": [
+                        pick(
+                            r,
+                            "id",
+                            "input",
+                            "output",
+                            "expected",
+                            "scores",
+                            "metrics",
+                            "metadata",
+                            "tags",
+                        )
+                        for r in input["rows"]
+                    ]
+                },
+            )
+        )
+        return {
+            "rowIds": data["row_ids"]
+            if isinstance(data.get("row_ids"), list)
+            else [r["id"] for r in input["rows"]]
+        }
+
+    async def _summary(self, input: Mapping[str, Any]) -> t.ExperimentSummaryDTO:
+        path = "/v1/experiment/" + enc(input["experimentId"]) + "/summarize?summarize_scores=true"
+        if "comparisonExperimentId" in input:
+            path += "&comparison_experiment_id=" + enc(input["comparisonExperimentId"])
+        result = summary(await self.api.call(path))
+        budget("summarizeExperiment", result)
+        return result
+
+    async def summarize_experiment(
+        self, input: t.SummarizeExperimentInput
+    ) -> t.ExperimentSummaryDTO:
+        validate_names(input, "summarizeExperiment", "experimentId")
+        return await self._summary(input)
+
+    async def add_feedback(self, input: t.AddFeedbackInput) -> t.AddFeedbackResult:
+        if not input.get("scores") and not input.get("comment"):
+            raise ValueError("braintrust.addFeedback requires at least one score or a comment.")
+        await self.api.call(
+            "/v1/experiment/" + enc(input["experimentId"]) + "/feedback",
+            method="POST",
+            body={
+                "feedback": [{"id": input["rowId"], **pick(input, "scores", "comment", "source")}]
+            },
+        )
+        return {"experimentId": input["experimentId"], "rowId": input["rowId"], "applied": True}
+
+    async def _review(self, input: Mapping[str, Any]) -> t.ReviewProgressDTO:
+        rows = obj(
+            await self.api.call(
+                "/btql", method="POST", body={"query": build_review_query(input), "fmt": "json"}
+            )
+        ).get("data")
+        row = obj(rows[0]) if isinstance(rows, list) and rows else {}
+        reviewed = number(row.get("reviewed"), 0)
+        total = number(row.get("total"), 0)
+        return {
+            "experimentId": input["experimentId"],
+            "scoreKey": input["scoreKey"],
+            "reviewed": reviewed,
+            "total": total,
+            "pending": max(0, total - reviewed),
+            "complete": total > 0 and reviewed >= total,
+        }
+
+    async def get_review_progress(self, input: t.GetReviewProgressInput) -> t.ReviewProgressDTO:
+        validate_names(input, "getReviewProgress", "experimentId", "scoreKey")
+        return await self._review(input)
+
+    async def await_review(self, input: t.AwaitReviewInput) -> t.ReviewProgressDTO:
+        result = await self._review(input)
+        if result["complete"] and (
+            "minimumReviewed" not in input or result["reviewed"] >= input["minimumReviewed"]
+        ):
+            return result
+        raise RuntimeError(
+            f"Braintrust review of experiment "
+            f'"{input["experimentId"]}" for score '
+            f'"{input["scoreKey"]}" is still pending.'
+        )
+
+    async def gate_experiment(self, input: t.GateExperimentInput) -> t.GateExperimentResult:
+        thresholds = input["thresholds"]
+        if "minimumReviewed" in thresholds and "reviewScoreKey" not in input:
+            raise ValueError(
+                "braintrust.gateExperiment requires reviewScoreKey when "
+                "thresholds.minimumReviewed is set."
+            )
+        result = await self._summary(input)
+        violations: list[t.GateViolation] = []
+        for key, minimum in thresholds.get("minimumScores", {}).items():
+            actual = obj(result["scores"].get(key)).get("score")
+            if actual is None or actual < minimum:
+                violations.append(
+                    {"metric": key, "actual": actual, "operator": ">=", "threshold": minimum}
+                )
+        if "maximumErrorRate" in thresholds:
+            actual = obj(result["metrics"].get("error_rate")).get("metric")
+            if actual is None or actual > thresholds["maximumErrorRate"]:
+                violations.append(
+                    {
+                        "metric": "error_rate",
+                        "actual": actual,
+                        "operator": "<=",
+                        "threshold": thresholds["maximumErrorRate"],
+                    }
+                )
+        if "minimumReviewed" in thresholds:
+            progress = await self._review(
+                {"experimentId": input["experimentId"], "scoreKey": input["reviewScoreKey"]}
+            )
+            if progress["reviewed"] < thresholds["minimumReviewed"]:
+                violations.append(
+                    {
+                        "metric": "reviewed",
+                        "actual": progress["reviewed"],
+                        "operator": ">=",
+                        "threshold": thresholds["minimumReviewed"],
+                    }
+                )
+        if violations:
+            raise ValueError("Release gate failed: " + json_text(violations))
+        return {"passed": True, "summary": result}
+
+
+@asynccontextmanager
+async def dependencies(deps: Deps | None) -> AsyncIterator[Deps]:
+    if deps is not None:
+        yield deps
+    else:
+        async with httpx.AsyncClient(
+            timeout=30, transport=httpx.AsyncHTTPTransport(retries=0)
+        ) as http:
+            yield Deps(braintrust=Client(http))
+
+
+BraintrustPort = Port
+BraintrustClient = Client
+BraintrustDeps = Deps

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/delete_dataset_rows.py

@@ -0,0 +1,20 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import DeleteDatasetRowsInput, DeleteDatasetRowsResult
+
+
+async def delete_dataset_rows_impl(
+    ctx: TaskContext, input: DeleteDatasetRowsInput, *, deps: Deps | None = None
+) -> DeleteDatasetRowsResult:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.delete_dataset_rows(input)
+
+
+@app.task(name="braintrust.deleteDatasetRows", retry=BRAINTRUST_RETRY)
+async def delete_dataset_rows(
+    ctx: TaskContext, input: DeleteDatasetRowsInput
+) -> DeleteDatasetRowsResult:
+    return await delete_dataset_rows_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/ensure_dataset.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import DatasetDTO, EnsureDatasetInput
+
+
+async def ensure_dataset_impl(
+    ctx: TaskContext, input: EnsureDatasetInput, *, deps: Deps | None = None
+) -> DatasetDTO:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.ensure_dataset(input)
+
+
+@app.task(name="braintrust.ensureDataset", retry=BRAINTRUST_RETRY)
+async def ensure_dataset(ctx: TaskContext, input: EnsureDatasetInput) -> DatasetDTO:
+    return await ensure_dataset_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/ensure_experiment.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import EnsureExperimentInput, ExperimentDTO
+
+
+async def ensure_experiment_impl(
+    ctx: TaskContext, input: EnsureExperimentInput, *, deps: Deps | None = None
+) -> ExperimentDTO:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.ensure_experiment(input)
+
+
+@app.task(name="braintrust.ensureExperiment", retry=BRAINTRUST_RETRY)
+async def ensure_experiment(ctx: TaskContext, input: EnsureExperimentInput) -> ExperimentDTO:
+    return await ensure_experiment_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/gate_experiment.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_NO_RETRY
+from .types import GateExperimentInput, GateExperimentResult
+
+
+async def gate_experiment_impl(
+    ctx: TaskContext, input: GateExperimentInput, *, deps: Deps | None = None
+) -> GateExperimentResult:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.gate_experiment(input)
+
+
+@app.task(name="braintrust.gateExperiment", retry=BRAINTRUST_NO_RETRY)
+async def gate_experiment(ctx: TaskContext, input: GateExperimentInput) -> GateExperimentResult:
+    return await gate_experiment_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/get_review_progress.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import GetReviewProgressInput, ReviewProgressDTO
+
+
+async def get_review_progress_impl(
+    ctx: TaskContext, input: GetReviewProgressInput, *, deps: Deps | None = None
+) -> ReviewProgressDTO:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.get_review_progress(input)
+
+
+@app.task(name="braintrust.getReviewProgress", retry=BRAINTRUST_RETRY)
+async def get_review_progress(ctx: TaskContext, input: GetReviewProgressInput) -> ReviewProgressDTO:
+    return await get_review_progress_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/list_dataset_rows.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import ListDatasetRowsInput, ListDatasetRowsResult
+
+
+async def list_dataset_rows_impl(
+    ctx: TaskContext, input: ListDatasetRowsInput, *, deps: Deps | None = None
+) -> ListDatasetRowsResult:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.list_dataset_rows(input)
+
+
+@app.task(name="braintrust.listDatasetRows", retry=BRAINTRUST_RETRY)
+async def list_dataset_rows(ctx: TaskContext, input: ListDatasetRowsInput) -> ListDatasetRowsResult:
+    return await list_dataset_rows_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/log_experiment_results.py

@@ -0,0 +1,20 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import LogExperimentResultsInput, LogExperimentResultsResult
+
+
+async def log_experiment_results_impl(
+    ctx: TaskContext, input: LogExperimentResultsInput, *, deps: Deps | None = None
+) -> LogExperimentResultsResult:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.log_experiment_results(input)
+
+
+@app.task(name="braintrust.logExperimentResults", retry=BRAINTRUST_RETRY)
+async def log_experiment_results(
+    ctx: TaskContext, input: LogExperimentResultsInput
+) -> LogExperimentResultsResult:
+    return await log_experiment_results_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/retry.py

@@ -0,0 +1,5 @@
+from render import Retry
+
+BRAINTRUST_RETRY = Retry(max_retries=3, wait_duration_ms=1000, backoff_scaling=2)
+BRAINTRUST_REVIEW_RETRY = Retry(max_retries=20160, wait_duration_ms=30000, backoff_scaling=1)
+BRAINTRUST_NO_RETRY = Retry(max_retries=0, wait_duration_ms=0, backoff_scaling=1)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/run_eval.py

@@ -0,0 +1,18 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_NO_RETRY
+from .types import ExperimentSummaryDTO, RunEvalInput
+
+
+async def run_eval_impl(
+    ctx: TaskContext, input: RunEvalInput, *, deps: Deps | None = None
+) -> ExperimentSummaryDTO:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.run_eval(input)
+
+
+@app.task(name="braintrust.runEval", retry=BRAINTRUST_NO_RETRY)
+async def run_eval(ctx: TaskContext, input: RunEvalInput) -> ExperimentSummaryDTO:
+    return await run_eval_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/summarize_experiment.py

@@ -0,0 +1,20 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import ExperimentSummaryDTO, SummarizeExperimentInput
+
+
+async def summarize_experiment_impl(
+    ctx: TaskContext, input: SummarizeExperimentInput, *, deps: Deps | None = None
+) -> ExperimentSummaryDTO:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.summarize_experiment(input)
+
+
+@app.task(name="braintrust.summarizeExperiment", retry=BRAINTRUST_RETRY)
+async def summarize_experiment(
+    ctx: TaskContext, input: SummarizeExperimentInput
+) -> ExperimentSummaryDTO:
+    return await summarize_experiment_impl(ctx, input)

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/tasks.py

@@ -0,0 +1,41 @@
+from ._app import app
+from .add_feedback import add_feedback, add_feedback_impl
+from .await_review import await_review, await_review_impl
+from .delete_dataset_rows import delete_dataset_rows, delete_dataset_rows_impl
+from .ensure_dataset import ensure_dataset, ensure_dataset_impl
+from .ensure_experiment import ensure_experiment, ensure_experiment_impl
+from .gate_experiment import gate_experiment, gate_experiment_impl
+from .get_review_progress import get_review_progress, get_review_progress_impl
+from .list_dataset_rows import list_dataset_rows, list_dataset_rows_impl
+from .log_experiment_results import log_experiment_results, log_experiment_results_impl
+from .run_eval import run_eval, run_eval_impl
+from .summarize_experiment import summarize_experiment, summarize_experiment_impl
+from .upsert_dataset_rows import upsert_dataset_rows, upsert_dataset_rows_impl
+
+__all__ = [
+    "app",
+    "add_feedback",
+    "add_feedback_impl",
+    "await_review",
+    "await_review_impl",
+    "delete_dataset_rows",
+    "delete_dataset_rows_impl",
+    "ensure_dataset",
+    "ensure_dataset_impl",
+    "ensure_experiment",
+    "ensure_experiment_impl",
+    "gate_experiment",
+    "gate_experiment_impl",
+    "get_review_progress",
+    "get_review_progress_impl",
+    "list_dataset_rows",
+    "list_dataset_rows_impl",
+    "log_experiment_results",
+    "log_experiment_results_impl",
+    "run_eval",
+    "run_eval_impl",
+    "summarize_experiment",
+    "summarize_experiment_impl",
+    "upsert_dataset_rows",
+    "upsert_dataset_rows_impl",
+]

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/types.py

@@ -0,0 +1,193 @@
+"""JSON contracts ported from the pinned TypeScript pack."""
+
+from __future__ import annotations
+
+from typing import Literal, NotRequired, TypedDict
+
+type JsonPrimitive = str | float | bool | None
+
+type JsonValue = JsonPrimitive | list[JsonValue] | dict[str, JsonValue]
+
+type JsonObject = dict[str, JsonValue]
+
+
+class GateThresholds(TypedDict):
+    minimumScores: NotRequired[dict[str, float]]
+    maximumErrorRate: NotRequired[float]
+    minimumReviewed: NotRequired[float]
+
+
+class GateViolation(TypedDict):
+    metric: str
+    actual: float | None
+    operator: Literal[">="] | Literal["<="]
+    threshold: float
+
+
+class DatasetDTO(TypedDict):
+    id: str
+    projectId: str
+    name: str
+    description: str | None
+
+
+class EnsureDatasetInput(TypedDict):
+    projectId: str
+    name: str
+    description: NotRequired[str]
+
+
+class DatasetRowDTO(TypedDict):
+    id: str
+    input: JsonValue
+    expected: NotRequired[JsonValue]
+    metadata: NotRequired[JsonObject]
+    tags: NotRequired[list[str]]
+
+
+class UpsertDatasetRowsInput(TypedDict):
+    datasetId: str
+    rows: list[DatasetRowDTO]
+
+
+class UpsertDatasetRowsResult(TypedDict):
+    rowIds: list[str]
+
+
+class ListDatasetRowsInput(TypedDict):
+    datasetId: str
+    limit: NotRequired[float]
+    cursor: NotRequired[str]
+
+
+class ListDatasetRowsResult(TypedDict):
+    rows: list[DatasetRowDTO]
+    nextCursor: str | None
+
+
+class DeleteDatasetRowsInput(TypedDict):
+    datasetId: str
+    rowIds: list[str]
+
+
+class DeleteDatasetRowsResult(TypedDict):
+    deletedRowIds: list[str]
+
+
+class SavedFunctionRef(TypedDict):
+    functionId: str
+    version: NotRequired[str]
+
+
+class RunEvalInput(TypedDict):
+    projectId: str
+    datasetId: str
+    task: SavedFunctionRef
+    scores: list[SavedFunctionRef]
+    experimentName: NotRequired[str]
+    metadata: NotRequired[JsonObject]
+    repoCommit: NotRequired[str]
+
+
+class ScoreSummaryDTO(TypedDict):
+    score: float | None
+    diff: float | None
+    improvements: float
+    regressions: float
+
+
+class MetricSummaryDTO(TypedDict):
+    metric: float | None
+    unit: str | None
+
+
+class ExperimentSummaryDTO(TypedDict):
+    projectName: str
+    experimentName: str
+    experimentId: str
+    experimentUrl: str
+    comparisonExperimentName: str | None
+    scores: dict[str, ScoreSummaryDTO]
+    metrics: dict[str, MetricSummaryDTO]
+
+
+class ExperimentDTO(TypedDict):
+    id: str
+    projectId: str
+    name: str
+    description: str | None
+    baseExperimentId: str | None
+
+
+class EnsureExperimentInput(TypedDict):
+    projectId: str
+    name: str
+    description: NotRequired[str]
+    baseExperimentId: NotRequired[str]
+
+
+class ExperimentResultRowDTO(TypedDict):
+    id: str
+    input: JsonValue
+    output: JsonValue
+    expected: NotRequired[JsonValue]
+    scores: NotRequired[dict[str, float]]
+    metrics: NotRequired[dict[str, float]]
+    metadata: NotRequired[JsonObject]
+    tags: NotRequired[list[str]]
+
+
+class LogExperimentResultsInput(TypedDict):
+    experimentId: str
+    rows: list[ExperimentResultRowDTO]
+
+
+class LogExperimentResultsResult(TypedDict):
+    rowIds: list[str]
+
+
+class SummarizeExperimentInput(TypedDict):
+    experimentId: str
+    comparisonExperimentId: NotRequired[str]
+
+
+class GateExperimentInput(SummarizeExperimentInput):
+    thresholds: GateThresholds
+    reviewScoreKey: NotRequired[str]
+
+
+class GateExperimentResult(TypedDict):
+    passed: Literal[True]
+    summary: ExperimentSummaryDTO
+
+
+class AddFeedbackInput(TypedDict):
+    experimentId: str
+    rowId: str
+    scores: NotRequired[dict[str, float | None]]
+    comment: NotRequired[str]
+    source: NotRequired[Literal["app"] | Literal["api"]]
+
+
+class AddFeedbackResult(TypedDict):
+    experimentId: str
+    rowId: str
+    applied: Literal[True]
+
+
+class GetReviewProgressInput(TypedDict):
+    experimentId: str
+    scoreKey: str
+
+
+class ReviewProgressDTO(TypedDict):
+    experimentId: str
+    scoreKey: str
+    reviewed: float
+    total: float
+    pending: float
+    complete: bool
+
+
+class AwaitReviewInput(GetReviewProgressInput):
+    minimumReviewed: NotRequired[float]

render_lab_tasks_braintrust-0.1.0/src/render_lab_tasks_braintrust/upsert_dataset_rows.py

@@ -0,0 +1,20 @@
+from render import TaskContext
+
+from ._app import app
+from .client import Deps, dependencies
+from .retry import BRAINTRUST_RETRY
+from .types import UpsertDatasetRowsInput, UpsertDatasetRowsResult
+
+
+async def upsert_dataset_rows_impl(
+    ctx: TaskContext, input: UpsertDatasetRowsInput, *, deps: Deps | None = None
+) -> UpsertDatasetRowsResult:
+    async with dependencies(deps) as resolved:
+        return await resolved.braintrust.upsert_dataset_rows(input)
+
+
+@app.task(name="braintrust.upsertDatasetRows", retry=BRAINTRUST_RETRY)
+async def upsert_dataset_rows(
+    ctx: TaskContext, input: UpsertDatasetRowsInput
+) -> UpsertDatasetRowsResult:
+    return await upsert_dataset_rows_impl(ctx, input)