veval-sdk 1.1.0__tar.gz

veval_sdk-1.1.0/.gitignore

@@ -0,0 +1,9 @@
+bin/
+obj/
+node_modules/
+dist/
+__pycache__/
+*.egg-info/
+.venv/
+build/
+*.pyc

veval_sdk-1.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jonathan Dinelle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

veval_sdk-1.1.0/PKG-INFO

@@ -0,0 +1,208 @@
+Metadata-Version: 2.5
+Name: veval-sdk
+Version: 1.1.0
+Summary: Trace, evaluate, and test AI agents with a few lines of code.
+Project-URL: Homepage, https://veval.dev
+Project-URL: Documentation, https://docs.veval.dev
+Project-URL: Repository, https://github.com/JonathanDinelle/veval-sdks
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,evaluation,llm,testing,tracing
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# veval-sdk
+
+Python SDK for [Veval](https://veval.dev) — trace, evaluate, and test AI agents.
+
+## Install
+
+```bash
+pip install veval-sdk
+```
+
+## Quick start
+
+Wrap your agent with `run_async` to send a trace to the Veval dashboard:
+
+```python
+import asyncio
+from veval import VevalSdk, VevalOptions, VevalExecutionContext
+
+sdk = VevalSdk(VevalOptions(api_key="veval-..."))
+
+async def my_agent(ctx: VevalExecutionContext) -> str:
+    result = await ctx.track_step_async(
+        name="call-llm",
+        input=ctx.input,
+        step=lambda: call_my_llm(ctx.input),
+    )
+    return result
+
+output = asyncio.run(sdk.run_async("my-agent", my_agent, input="Hello"))
+```
+
+Every call to `track_step_async` records the step name, input, output, timing, and any metadata you attach.
+
+## Tracking steps
+
+Use the `StepHandle` to attach token counts, cost, and model name:
+
+```python
+async def agent(ctx: VevalExecutionContext) -> str:
+    async def llm_call(handle):
+        response = await call_my_llm(ctx.input)
+        handle.set_meta("tokens_in", response.usage.input_tokens)
+        handle.set_meta("tokens_out", response.usage.output_tokens)
+        handle.set_meta("cost_usd", response.usage.cost)
+        handle.set_meta("model", "claude-opus-4-6")
+        return response.text
+
+    return await ctx.track_step_async("call-llm", ctx.input, llm_call)
+```
+
+## Assertions
+
+Use `TraceAssert` to validate behaviour at the trace level:
+
+```python
+from veval import TraceAssert
+
+assertions = [
+    TraceAssert.no_errors(),
+    TraceAssert.max_steps(10),
+    TraceAssert.step_exists("call-llm"),
+    TraceAssert.max_cost(0.05),
+    TraceAssert.max_duration(30_000),
+    TraceAssert.output_contains("success"),
+    TraceAssert.tool_called("web-search"),
+]
+```
+
+## Scenarios
+
+Run a named scenario against a list of test items and assertions:
+
+```python
+from veval import ScenarioItem
+
+items = [
+    ScenarioItem(name="basic greeting", input="Hello"),
+    ScenarioItem(name="edge case", input=""),
+]
+
+result = asyncio.run(
+    sdk.run_scenario_async("smoke-test", my_agent, assertions, items=items)
+)
+
+print(f"Passed: {result.pass_count}/{len(result.results)}")
+for r in result.results:
+    if not r.passed:
+        print(f"  FAIL {r.item.name}: {r.failures}")
+```
+
+You can also pull items from the Veval dashboard by omitting `items`:
+
+```python
+result = asyncio.run(sdk.run_scenario_async("smoke-test", my_agent, assertions))
+```
+
+## Replay and snapshot testing
+
+Load a previously recorded trace and replay it with mocked LLM responses — no real calls, deterministic results:
+
+```python
+trace = asyncio.run(sdk.get_trace_async("tr_abc123"))
+
+replay = asyncio.run(sdk.replay_async(
+    trace,
+    my_agent,
+    ReplayOptions(mock_llm_responses=True, assertions=assertions),
+))
+
+print("Passed" if replay.passed else replay.failures)
+```
+
+Store a known-good run — every step with its input and output — and detect when a new run drifts from it: a changed prompt or tool argument, a step added, dropped, repeated, or reordered:
+
+```python
+# Save a baseline from a recorded trace. The trace is pinned, so retention never deletes it.
+asyncio.run(sdk.save_snapshot_async("my-baseline", "tr_abc123"))
+
+# In tests: an assertion like any other. A missing baseline fails; it never passes silently.
+replay = asyncio.run(sdk.replay_async(trace, my_agent, ReplayOptions(
+    mock_llm_responses=True,
+    assertions=[TraceAssert.matches_snapshot(sdk, "my-baseline")],
+    compare_with_recording=SnapshotOptions(),  # also fail if steps/inputs drift from the trace
+)))
+
+# Or compare directly, and record the result in the dashboard.
+baseline = asyncio.run(sdk.get_snapshot_async("my-baseline"))
+diff = asyncio.run(sdk.compare_snapshot_async("my-baseline", baseline, ctx))
+if diff.has_changes:
+    print(diff.summary())  # every change, with a line diff of changed inputs
+```
+
+## Test SDK
+
+`VevalTestSdk` is a drop-in replacement that blocks real LLM calls, making it safe to use in unit tests. It still reports to the dashboard.
+
+```python
+from veval import VevalTestSdk
+
+sdk = VevalTestSdk(options).with_replay(trace)
+output = asyncio.run(sdk.run_async("my-agent", my_agent, input="test input"))
+```
+
+## API reference
+
+### `VevalOptions`
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `api_key` | `str` | `""` | Your Veval API key — it also determines the workspace |
+| `project_id` | `str` | `""` | Deprecated and ignored; will be removed |
+| `flush_interval_ms` | `int` | `5000` | Batch flush interval |
+| `flush_batch_size` | `int` | `50` | Max traces per flush |
+
+### `VevalSdk`
+
+| Method | Description |
+|---|---|
+| `run_async(name, callback, input)` | Run agent and send trace |
+| `get_trace_async(trace_id)` | Fetch a recorded trace |
+| `save_snapshot_async(name, trace_id_or_ctx)` | Store a named baseline (pins the trace) |
+| `get_snapshot_async(name)` | Load the latest stored baseline |
+| `load_snapshot_async(trace_id)` | Build a snapshot from a trace (not pinned) |
+| `compare_snapshot_async(name, snapshot, ctx, options)` | Diff a run against a baseline and record it |
+| `replay_async(trace, callback, options)` | Replay trace with mocked outputs |
+| `run_scenario_async(name, agent, assertions, items)` | Run a test scenario |
+
+### `TraceAssert` built-ins
+
+| Assertion | Description |
+|---|---|
+| `no_errors()` | All steps must succeed |
+| `max_steps(n)` | Total step count must not exceed `n` |
+| `step_exists(name)` | A step with this name must appear |
+| `max_cost(usd)` | Total cost must not exceed `usd` |
+| `max_duration(ms)` | Total step duration must not exceed `ms` |
+| `output_contains(text)` | At least one step output must contain `text` |
+| `tool_called(name)` | A tool step with this name must appear |
+
+Custom assertions implement `async ITraceAssertion.evaluate_async(ctx) -> Optional[str]` — return `None` to pass or an error string to fail.
+
+## Requirements
+
+- Python 3.10+
+- No third-party dependencies

veval_sdk-1.1.0/README.md

@@ -0,0 +1,185 @@
+# veval-sdk
+
+Python SDK for [Veval](https://veval.dev) — trace, evaluate, and test AI agents.
+
+## Install
+
+```bash
+pip install veval-sdk
+```
+
+## Quick start
+
+Wrap your agent with `run_async` to send a trace to the Veval dashboard:
+
+```python
+import asyncio
+from veval import VevalSdk, VevalOptions, VevalExecutionContext
+
+sdk = VevalSdk(VevalOptions(api_key="veval-..."))
+
+async def my_agent(ctx: VevalExecutionContext) -> str:
+    result = await ctx.track_step_async(
+        name="call-llm",
+        input=ctx.input,
+        step=lambda: call_my_llm(ctx.input),
+    )
+    return result
+
+output = asyncio.run(sdk.run_async("my-agent", my_agent, input="Hello"))
+```
+
+Every call to `track_step_async` records the step name, input, output, timing, and any metadata you attach.
+
+## Tracking steps
+
+Use the `StepHandle` to attach token counts, cost, and model name:
+
+```python
+async def agent(ctx: VevalExecutionContext) -> str:
+    async def llm_call(handle):
+        response = await call_my_llm(ctx.input)
+        handle.set_meta("tokens_in", response.usage.input_tokens)
+        handle.set_meta("tokens_out", response.usage.output_tokens)
+        handle.set_meta("cost_usd", response.usage.cost)
+        handle.set_meta("model", "claude-opus-4-6")
+        return response.text
+
+    return await ctx.track_step_async("call-llm", ctx.input, llm_call)
+```
+
+## Assertions
+
+Use `TraceAssert` to validate behaviour at the trace level:
+
+```python
+from veval import TraceAssert
+
+assertions = [
+    TraceAssert.no_errors(),
+    TraceAssert.max_steps(10),
+    TraceAssert.step_exists("call-llm"),
+    TraceAssert.max_cost(0.05),
+    TraceAssert.max_duration(30_000),
+    TraceAssert.output_contains("success"),
+    TraceAssert.tool_called("web-search"),
+]
+```
+
+## Scenarios
+
+Run a named scenario against a list of test items and assertions:
+
+```python
+from veval import ScenarioItem
+
+items = [
+    ScenarioItem(name="basic greeting", input="Hello"),
+    ScenarioItem(name="edge case", input=""),
+]
+
+result = asyncio.run(
+    sdk.run_scenario_async("smoke-test", my_agent, assertions, items=items)
+)
+
+print(f"Passed: {result.pass_count}/{len(result.results)}")
+for r in result.results:
+    if not r.passed:
+        print(f"  FAIL {r.item.name}: {r.failures}")
+```
+
+You can also pull items from the Veval dashboard by omitting `items`:
+
+```python
+result = asyncio.run(sdk.run_scenario_async("smoke-test", my_agent, assertions))
+```
+
+## Replay and snapshot testing
+
+Load a previously recorded trace and replay it with mocked LLM responses — no real calls, deterministic results:
+
+```python
+trace = asyncio.run(sdk.get_trace_async("tr_abc123"))
+
+replay = asyncio.run(sdk.replay_async(
+    trace,
+    my_agent,
+    ReplayOptions(mock_llm_responses=True, assertions=assertions),
+))
+
+print("Passed" if replay.passed else replay.failures)
+```
+
+Store a known-good run — every step with its input and output — and detect when a new run drifts from it: a changed prompt or tool argument, a step added, dropped, repeated, or reordered:
+
+```python
+# Save a baseline from a recorded trace. The trace is pinned, so retention never deletes it.
+asyncio.run(sdk.save_snapshot_async("my-baseline", "tr_abc123"))
+
+# In tests: an assertion like any other. A missing baseline fails; it never passes silently.
+replay = asyncio.run(sdk.replay_async(trace, my_agent, ReplayOptions(
+    mock_llm_responses=True,
+    assertions=[TraceAssert.matches_snapshot(sdk, "my-baseline")],
+    compare_with_recording=SnapshotOptions(),  # also fail if steps/inputs drift from the trace
+)))
+
+# Or compare directly, and record the result in the dashboard.
+baseline = asyncio.run(sdk.get_snapshot_async("my-baseline"))
+diff = asyncio.run(sdk.compare_snapshot_async("my-baseline", baseline, ctx))
+if diff.has_changes:
+    print(diff.summary())  # every change, with a line diff of changed inputs
+```
+
+## Test SDK
+
+`VevalTestSdk` is a drop-in replacement that blocks real LLM calls, making it safe to use in unit tests. It still reports to the dashboard.
+
+```python
+from veval import VevalTestSdk
+
+sdk = VevalTestSdk(options).with_replay(trace)
+output = asyncio.run(sdk.run_async("my-agent", my_agent, input="test input"))
+```
+
+## API reference
+
+### `VevalOptions`
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `api_key` | `str` | `""` | Your Veval API key — it also determines the workspace |
+| `project_id` | `str` | `""` | Deprecated and ignored; will be removed |
+| `flush_interval_ms` | `int` | `5000` | Batch flush interval |
+| `flush_batch_size` | `int` | `50` | Max traces per flush |
+
+### `VevalSdk`
+
+| Method | Description |
+|---|---|
+| `run_async(name, callback, input)` | Run agent and send trace |
+| `get_trace_async(trace_id)` | Fetch a recorded trace |
+| `save_snapshot_async(name, trace_id_or_ctx)` | Store a named baseline (pins the trace) |
+| `get_snapshot_async(name)` | Load the latest stored baseline |
+| `load_snapshot_async(trace_id)` | Build a snapshot from a trace (not pinned) |
+| `compare_snapshot_async(name, snapshot, ctx, options)` | Diff a run against a baseline and record it |
+| `replay_async(trace, callback, options)` | Replay trace with mocked outputs |
+| `run_scenario_async(name, agent, assertions, items)` | Run a test scenario |
+
+### `TraceAssert` built-ins
+
+| Assertion | Description |
+|---|---|
+| `no_errors()` | All steps must succeed |
+| `max_steps(n)` | Total step count must not exceed `n` |
+| `step_exists(name)` | A step with this name must appear |
+| `max_cost(usd)` | Total cost must not exceed `usd` |
+| `max_duration(ms)` | Total step duration must not exceed `ms` |
+| `output_contains(text)` | At least one step output must contain `text` |
+| `tool_called(name)` | A tool step with this name must appear |
+
+Custom assertions implement `async ITraceAssertion.evaluate_async(ctx) -> Optional[str]` — return `None` to pass or an error string to fail.
+
+## Requirements
+
+- Python 3.10+
+- No third-party dependencies

veval_sdk-1.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "veval-sdk"
+version = "1.1.0"
+description = "Trace, evaluate, and test AI agents with a few lines of code."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+keywords = ["ai", "agents", "llm", "evaluation", "testing", "tracing"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = []
+
+[tool.hatch.build.targets.wheel]
+packages = ["veval"]
+
+[project.urls]
+Homepage = "https://veval.dev"
+Documentation = "https://docs.veval.dev"
+Repository = "https://github.com/JonathanDinelle/veval-sdks"

veval_sdk-1.1.0/tests/test_sdk.py

@@ -0,0 +1,218 @@
+"""Core SDK tests. Run with `pytest sdk/python/tests` or `python sdk/python/tests/test_sdk.py`."""
+import asyncio
+import os
+import sys
+import warnings
+from datetime import datetime, timezone
+
+# Point every SDK at a closed local port: reporting calls fail fast and are swallowed, and nothing
+# reaches the real API. Set before any SDK is constructed (the endpoint is read at construction).
+os.environ["VEVAL_INTERNAL_ENDPOINT"] = "http://127.0.0.1:1"
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from veval import (  # noqa: E402
+    ReplayOptions, ScenarioItem, StepData, TraceAssert, TraceData, VevalExecutionContext,
+    VevalOptions, VevalSdk, VevalTestSdk,
+)
+
+
+def sdk():
+    return VevalSdk(VevalOptions(api_key="test"))
+
+
+def make_test_sdk():
+    return VevalTestSdk(VevalOptions(api_key="test"))
+
+
+def returning(value):
+    # A zero-argument callback: track_step_async passes a StepHandle to callbacks that take parameters.
+    async def produce():
+        return value
+    return produce
+
+
+def run(*steps):
+    async def go():
+        ctx = VevalExecutionContext("tr_run")
+        for name, inp, out in steps:
+            await ctx.track_step_async(name, inp, returning(out))
+        return ctx
+    return asyncio.run(go())
+
+
+def recording(*steps):
+    return TraceData(trace_id="tr_recorded", input="hello", status="success", steps=list(steps))
+
+
+def recorded(name, inp, out, type="llm"):
+    return StepData(step_id=name, name=name, type=type, input=inp, output=out, status="success")
+
+
+# --- options -----------------------------------------------------------------------------------------
+
+def test_endpoint_comes_from_env_not_a_public_option():
+    options = VevalOptions(api_key="k")
+    assert options._endpoint == "http://127.0.0.1:1"
+    assert not hasattr(options, "endpoint")
+
+
+def test_project_id_is_deprecated_and_not_sent():
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        VevalOptions(api_key="k", project_id="p")
+    assert any(issubclass(w.category, DeprecationWarning) for w in caught)
+
+    now = datetime.now(timezone.utc)
+    payload = sdk()._build_payload("tr", "agent", run(("s", "in", "out")), None, None, "success", None, now, now)
+    assert "project_id" not in payload
+
+
+# --- running and steps -------------------------------------------------------------------------------
+
+def test_run_async_returns_result_and_records_steps_with_metadata():
+    seen = {}
+
+    async def agent(ctx):
+        seen["ctx"] = ctx
+        ctx.set_metadata("user_id", "u_1")
+
+        async def call(step):
+            step.set_meta("type", "llm")
+            step.set_meta("cost_usd", 0.01)
+            step.set_meta("provider", "anthropic")
+            return "answer"
+        return await ctx.track_step_async("call-llm", "prompt", call)
+
+    assert asyncio.run(sdk().run_async("agent", agent)) == "answer"
+    step = seen["ctx"].steps[0]
+    assert (step.name, step.type, step.cost_usd, step.output, step.status) == ("call-llm", "llm", 0.01, "answer", "success")
+    assert step.metadata["provider"] == "anthropic"
+    assert seen["ctx"].trace_meta["user_id"] == "u_1"
+
+
+def test_run_async_reraises_agent_errors_and_marks_step_failed():
+    seen = {}
+
+    async def boom():
+        raise RuntimeError("kaboom")
+
+    async def agent(ctx):
+        seen["ctx"] = ctx
+        return await ctx.track_step_async("boom", None, boom)
+
+    try:
+        asyncio.run(sdk().run_async("agent", agent))
+        raise AssertionError("expected the agent error to propagate")
+    except RuntimeError as ex:
+        assert str(ex) == "kaboom"
+    assert (seen["ctx"].steps[0].status, seen["ctx"].steps[0].error) == ("error", "kaboom")
+
+
+# --- replay ------------------------------------------------------------------------------------------
+
+def test_replay_serves_recorded_outputs_and_never_runs_the_step():
+    ran = []
+
+    async def live():
+        ran.append(True)
+        return "live"
+
+    async def agent(ctx):
+        return await ctx.track_step_async("classify", "hello", live)
+
+    result = asyncio.run(sdk().replay_async(recording(recorded("classify", "hello", "greeting")), agent, ReplayOptions(mock_llm_responses=True)))
+    assert result.output == "greeting"
+    assert ran == []
+    assert result.passed
+
+
+def test_replay_fails_when_a_step_has_no_recording_instead_of_calling_live():
+    async def agent(ctx):
+        return await ctx.track_step_async("unrecorded", None, returning("live"))
+
+    result = asyncio.run(sdk().replay_async(recording(recorded("classify", "hello", "greeting")), agent, ReplayOptions(mock_llm_responses=True)))
+    assert not result.passed
+    assert "no mock output for step 'unrecorded'" in result.failures[0]
+
+
+def test_with_replay_makes_the_production_entry_point_replay_unchanged():
+    replaying = make_test_sdk().with_replay(recording(recorded("classify", "hello", "greeting")))
+
+    async def agent(ctx):
+        return await ctx.track_step_async("classify", "hello", returning("live"))
+
+    assert asyncio.run(replaying.run_async("agent", agent)) == "greeting"
+    assert replaying.last_status == "success"
+
+
+# --- assertions --------------------------------------------------------------------------------------
+
+def test_built_in_assertions_pass_and_fail_as_documented():
+    async def build():
+        ctx = VevalExecutionContext("tr")
+
+        async def fetch(step):
+            step.set_meta("type", "tool")
+            step.set_meta("cost_usd", 0.2)
+            return "Montreal weather"
+        await ctx.track_step_async("fetch", "q", fetch)
+        await ctx.track_step_async("write", "d", returning("briefing"))
+        return ctx
+    ctx = asyncio.run(build())
+    check = lambda a: asyncio.run(a.evaluate_async(ctx))  # noqa: E731
+
+    assert check(TraceAssert.no_errors()) is None
+    assert check(TraceAssert.max_steps(2)) is None
+    assert check(TraceAssert.max_steps(1)).startswith("MaxSteps")
+    assert check(TraceAssert.step_exists("write")) is None
+    assert check(TraceAssert.step_exists("nope")).startswith("StepExists")
+    assert check(TraceAssert.tool_called("fetch")) is None
+    assert check(TraceAssert.tool_called("write")).startswith("ToolCalled")
+    assert check(TraceAssert.max_cost(0.1)).startswith("MaxCost")
+    assert check(TraceAssert.output_contains("Montreal")) is None
+    assert check(TraceAssert.output_contains("Paris")).startswith("OutputContains")
+
+
+# --- judge -------------------------------------------------------------------------------------------
+
+def test_judge_mock_passes_records_verdict_and_reports_failures():
+    judged = make_test_sdk().with_judge_mock("polite", True, 0.9, "Friendly.").with_judge_mock("concise", False, 0.4, "Rambles.")
+    ctx = run(("reply", "hi", "Hello there!"))
+
+    assert asyncio.run(TraceAssert.judge(judged, "polite").evaluate_async(ctx)) is None
+    assert asyncio.run(TraceAssert.judge(judged, "concise").evaluate_async(ctx)) == "Judge: concise (score 0.40) — Rambles."
+    assert [(j["criteria"], j["passed"]) for j in ctx.judgments] == [("polite", True), ("concise", False)]
+
+
+def test_judge_without_a_mock_fails_instead_of_making_a_billed_call():
+    failure = asyncio.run(TraceAssert.judge(make_test_sdk(), "unmocked").evaluate_async(run(("reply", "hi", "yo"))))
+    assert failure.startswith("Judge: evaluation failed")
+    assert "no judge mock for criteria 'unmocked'" in failure
+
+
+# --- scenarios ---------------------------------------------------------------------------------------
+
+def test_scenario_runs_each_item_with_shared_and_per_item_assertions():
+    async def agent(ctx):
+        async def reply():
+            if ctx.input == "crash":
+                raise RuntimeError("agent crashed")
+            return f"echo: {ctx.input}"
+        return await ctx.track_step_async("reply", ctx.input, reply)
+
+    result = asyncio.run(make_test_sdk().run_scenario_async("echo", agent, [TraceAssert.step_exists("reply")], [
+        ScenarioItem(name="ok", input="hi", assertions=[TraceAssert.output_contains("echo: hi")]),
+        ScenarioItem(name="wrong output", input="bye", assertions=[TraceAssert.output_contains("echo: hi")]),
+        ScenarioItem(name="crash", input="crash"),
+    ]))
+
+    assert [r.passed for r in result.results] == [True, False, False]
+    assert result.pass_count == 1
+    assert any("agent crashed" in f for f in result.results[2].failures)
+
+
+if __name__ == "__main__":
+    tests = [f for name, f in sorted(globals().items()) if name.startswith("test_")]
+    for t in tests:
+        t()
+    print(f"python sdk tests: {len(tests)} passed")