veval-sdk 1.1.0__py3-none-any.whl

veval/__init__.py

@@ -0,0 +1,47 @@
+from ._options import VevalOptions
+from ._step import Step, StepHandle
+from ._context import VevalExecutionContext
+from ._tracing import (
+    TraceData,
+    StepData,
+    ReplayOptions,
+    ReplayResult,
+)
+from ._snapshots import (
+    SnapshotStep,
+    SnapshotData,
+    SnapshotOptions,
+    SnapshotChange,
+    SnapshotAlignmentRow,
+    SnapshotDiff,
+    compare_snapshots,
+)
+from ._assertions import ITraceAssertion, TraceAssert
+from ._scenarios import ScenarioItem, ItemRunResult, ScenarioRunResult
+from ._sdk import VevalSdk
+from ._test_sdk import VevalTestSdk
+
+__all__ = [
+    "VevalOptions",
+    "Step",
+    "StepHandle",
+    "VevalExecutionContext",
+    "TraceData",
+    "StepData",
+    "SnapshotStep",
+    "SnapshotData",
+    "SnapshotOptions",
+    "SnapshotChange",
+    "SnapshotAlignmentRow",
+    "SnapshotDiff",
+    "compare_snapshots",
+    "ReplayOptions",
+    "ReplayResult",
+    "ITraceAssertion",
+    "TraceAssert",
+    "ScenarioItem",
+    "ItemRunResult",
+    "ScenarioRunResult",
+    "VevalSdk",
+    "VevalTestSdk",
+]

veval/_assertions.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+from typing import Any, Optional, TYPE_CHECKING
+from ._step import Step
+from ._snapshots import SnapshotOptions, compare_snapshots
+
+if TYPE_CHECKING:
+    from ._context import VevalExecutionContext
+
+
+class ITraceAssertion:
+    async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+        raise NotImplementedError
+
+
+def _flatten_steps(steps: list[Step]) -> list[Step]:
+    result: list[Step] = []
+    for s in steps:
+        result.append(s)
+        result.extend(_flatten_steps(s.children))
+    return result
+
+
+class TraceAssert:
+    @staticmethod
+    def max_steps(max: int) -> ITraceAssertion:
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                count = len(_flatten_steps(ctx.steps))
+                return f"MaxSteps: expected at most {max} steps, got {count}" if count > max else None
+        return _Assertion()
+
+    @staticmethod
+    def no_errors() -> ITraceAssertion:
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                errors = [s.name for s in _flatten_steps(ctx.steps) if s.status == "error"]
+                return f"NoErrors: found {len(errors)} step(s) with error status: {', '.join(errors)}" if errors else None
+        return _Assertion()
+
+    @staticmethod
+    def step_exists(step_name: str) -> ITraceAssertion:
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                found = any(s.name == step_name for s in _flatten_steps(ctx.steps))
+                return None if found else f"StepExists: step '{step_name}' not found"
+        return _Assertion()
+
+    @staticmethod
+    def max_cost(max_cost: float) -> ITraceAssertion:
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                total = sum(s.cost_usd or 0 for s in _flatten_steps(ctx.steps))
+                return f"MaxCost: expected at most {max_cost}, got {total}" if total > max_cost else None
+        return _Assertion()
+
+    @staticmethod
+    def max_duration(max_ms: int) -> ITraceAssertion:
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                total = sum(s.duration_ms or 0 for s in _flatten_steps(ctx.steps))
+                return f"MaxDuration: expected at most {max_ms}ms, got {total}ms" if total > max_ms else None
+        return _Assertion()
+
+    @staticmethod
+    def output_contains(expected: str) -> ITraceAssertion:
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                found = any(
+                    expected in str(s.output)
+                    for s in _flatten_steps(ctx.steps)
+                    if s.output is not None
+                )
+                return None if found else f"OutputContains: no step output contains '{expected}'"
+        return _Assertion()
+
+    @staticmethod
+    def tool_called(tool_name: str) -> ITraceAssertion:
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                found = any(
+                    s.type == "tool" and s.name == tool_name
+                    for s in _flatten_steps(ctx.steps)
+                )
+                return None if found else f"ToolCalled: no tool step named '{tool_name}' was found"
+        return _Assertion()
+
+    @staticmethod
+    def judge(
+        veval: Any,
+        criteria: str,
+        model: Optional[str] = None,
+        threshold: Optional[float] = None,
+        reference_output: Optional[Any] = None,
+        samples: Optional[int] = None,
+    ) -> ITraceAssertion:
+        """Scores the trace's output against a rubric using an LLM judge, evaluated server-side."""
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                try:
+                    result = await veval.judge_async(
+                        criteria, ctx, model=model, threshold=threshold,
+                        reference_output=reference_output, samples=samples,
+                    )
+                except Exception as ex:
+                    # Never let a network/API failure during judging silently pass a test.
+                    return f"Judge: evaluation failed — {ex}"
+
+                ctx.record_judgment(criteria, result["score"], result["passed"], result["reasoning"])
+
+                if result["passed"]:
+                    return None
+                return f"Judge: {criteria} (score {result['score']:.2f}) — {result['reasoning']}"
+        return _Assertion()
+
+    @staticmethod
+    def matches_snapshot(
+        snapshot_or_sdk: Any,
+        snapshot_name: Optional[str] = None,
+        options: Optional[SnapshotOptions] = None,
+    ) -> ITraceAssertion:
+        """
+        Fails when the run's step sequence or step inputs differ from the snapshot — e.g. a changed
+        prompt, a dropped or repeated call. Pass a SnapshotData, or (sdk, name) to load a stored
+        baseline; a missing baseline fails the assertion.
+        """
+        by_name = snapshot_name is not None
+
+        class _Assertion(ITraceAssertion):
+            async def evaluate_async(self, ctx: VevalExecutionContext) -> Optional[str]:
+                name = snapshot_name if by_name else snapshot_or_sdk.name
+                try:
+                    snapshot = await snapshot_or_sdk.get_snapshot_async(snapshot_name) if by_name else snapshot_or_sdk
+                except Exception as ex:
+                    # A baseline we couldn't load must fail loudly, never pass vacuously.
+                    return f"MatchesSnapshot: could not load snapshot '{name}' — {ex}"
+                if snapshot is None:
+                    return f"MatchesSnapshot: no snapshot named '{name}'. Save one with save_snapshot_async first."
+
+                diff = compare_snapshots(snapshot, ctx, options)
+                return "MatchesSnapshot: " + diff.summary(name or snapshot.name) if diff.has_changes else None
+        return _Assertion()

veval/_context.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+import inspect
+from collections import deque
+from typing import Any, Callable, Awaitable, Optional, TypeVar, TYPE_CHECKING
+from ._step import Step, StepHandle
+
+if TYPE_CHECKING:
+    from ._tracing import TraceData
+
+T = TypeVar("T")
+
+
+class VevalExecutionContext:
+    def __init__(self, trace_id: str, input: Any = None):
+        self.trace_id = trace_id
+        self.input = input
+        self._steps: list[Step] = []
+        self._metadata: dict[str, Any] = {}
+        self._judgments: list[dict] = []
+        self._mock_outputs: Optional[dict[str, deque]] = None
+        self._strict_mock_mode = False
+
+    @property
+    def steps(self) -> list[Step]:
+        return self._steps
+
+    @property
+    def trace_meta(self) -> dict[str, Any]:
+        return self._metadata
+
+    @property
+    def judgments(self) -> list[dict]:
+        return self._judgments
+
+    def record_judgment(self, criteria: str, score: float, passed: bool, reasoning: str) -> None:
+        self._judgments.append({
+            "criteria": criteria,
+            "score": score,
+            "passed": passed,
+            "reasoning": reasoning,
+        })
+
+    async def track_step_async(
+        self,
+        name: str,
+        input: Any,
+        step: Callable[..., Awaitable[T]],
+    ) -> T:
+        if self._mock_outputs is not None and self._strict_mock_mode:
+            q = self._mock_outputs.get(name)
+            if not q:
+                available = ", ".join(self._mock_outputs.keys())
+                raise RuntimeError(
+                    f"Replay mode: no mock output for step '{name}'. "
+                    f"Available steps: {available}. "
+                    "This would have made a real LLM call."
+                )
+
+        if self._mock_outputs is not None:
+            q = self._mock_outputs.get(name)
+            if q:
+                recorded = q.popleft()
+                mock = recorded.output
+                s = Step(name)
+                s.input = input
+                # Keep the recorded type so tool_called and snapshot types behave the same as in the original run.
+                s.type = recorded.type or "custom"
+                s.metadata["_source"] = "replay"
+                s.complete(mock)
+                self._steps.append(s)
+                return mock  # type: ignore[return-value]
+
+        s = Step(name)
+        s.input = input
+        self._steps.append(s)
+        handle = StepHandle(s)
+
+        try:
+            sig = inspect.signature(step)
+            if len(sig.parameters) > 0:
+                result = await step(handle)
+            else:
+                result = await step()  # type: ignore[call-arg]
+            s.complete(result)
+            return result
+        except Exception as ex:
+            s.fail(str(ex))
+            raise
+
+    def set_metadata(self, key: str, value: Any) -> None:
+        self._metadata[key] = value
+
+    def load_mock_outputs(self, trace: TraceData, strict: bool = True) -> None:
+        if not trace.steps:
+            raise RuntimeError(
+                f"Replay trace '{trace.trace_id}' has no steps. Cannot mock LLM calls. "
+                "Ensure the trace was recorded with steps before using it for replay."
+            )
+        self._mock_outputs = {}
+        self._strict_mock_mode = strict
+        for step in trace.steps:
+            if step.name not in self._mock_outputs:
+                self._mock_outputs[step.name] = deque()
+            self._mock_outputs[step.name].append(step)

veval/_http_client.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+import json
+from typing import Any, Optional
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError, URLError
+from ._tracing import TraceData
+from ._snapshots import SnapshotData
+
+
+class VevalHttpClient:
+    def __init__(self, api_key: str, endpoint: str):
+        self._api_key = api_key
+        self._endpoint = endpoint.rstrip("/")
+
+    def _headers(self) -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+        }
+
+    async def send_trace_async(self, payload: Any) -> None:
+        try:
+            import asyncio
+            await asyncio.get_event_loop().run_in_executor(
+                None, self._post, f"{self._endpoint}/v1/traces", payload
+            )
+        except Exception:
+            pass
+
+    async def get_trace_async(self, trace_id: str) -> Optional[TraceData]:
+        try:
+            import asyncio
+            data = await asyncio.get_event_loop().run_in_executor(
+                None, self._get, f"{self._endpoint}/v1/traces/{trace_id}"
+            )
+            return TraceData.from_dict(data) if data else None
+        except Exception:
+            return None
+
+    async def get_scenario_items_async(self, scenario_name: str) -> list[dict]:
+        try:
+            import asyncio
+            encoded = quote(scenario_name, safe="")
+            data = await asyncio.get_event_loop().run_in_executor(
+                None, self._get, f"{self._endpoint}/v1/scenarios/{encoded}/items"
+            )
+            return data.get("items", []) if data else []
+        except Exception:
+            return []
+
+    async def post_scenario_run_async(self, scenario_name: str, payload: Any) -> None:
+        try:
+            import asyncio
+            encoded = quote(scenario_name, safe="")
+            await asyncio.get_event_loop().run_in_executor(
+                None, self._post, f"{self._endpoint}/v1/scenarios/{encoded}/runs", payload
+            )
+        except Exception:
+            pass
+
+    async def judge_async(self, payload: Any) -> dict:
+        # Unlike the other calls on this client, judge_async deliberately does not swallow
+        # failures — a network/API error here must surface as an assertion failure, not a
+        # silent pass. Callers are expected to catch.
+        import asyncio
+        return await asyncio.get_event_loop().run_in_executor(
+            None, self._post_json, f"{self._endpoint}/v1/judge", payload
+        )
+
+    async def create_snapshot_async(self, payload: Any) -> SnapshotData:
+        # Raises on failure — a baseline that silently didn't save would make every later
+        # comparison fail, or compare against a stale one.
+        import asyncio
+        data = await asyncio.get_event_loop().run_in_executor(
+            None, self._post_json, f"{self._endpoint}/v1/snapshots", payload
+        )
+        return SnapshotData.from_dict(data)
+
+    async def get_snapshot_async(self, name: str) -> Optional[SnapshotData]:
+        # Returns None only when no baseline exists (404); any other failure raises, so a network
+        # error can never look like "nothing to compare against".
+        import asyncio
+        encoded = quote(name, safe="")
+        data = await asyncio.get_event_loop().run_in_executor(
+            None, self._get_or_none_on_404, f"{self._endpoint}/v1/snapshots/{encoded}"
+        )
+        return SnapshotData.from_dict(data) if data is not None else None
+
+    def _get_or_none_on_404(self, url: str) -> Optional[dict]:
+        req = Request(url, headers=self._headers(), method="GET")
+        try:
+            with urlopen(req, timeout=10) as resp:
+                return json.loads(resp.read().decode("utf-8"))
+        except HTTPError as ex:
+            if ex.code == 404:
+                return None
+            raise
+
+    def _post_json(self, url: str, payload: Any) -> dict:
+        body = json.dumps(payload, default=str).encode("utf-8")
+        req = Request(url, data=body, headers=self._headers(), method="POST")
+        with urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+
+    def _post(self, url: str, payload: Any) -> None:
+        body = json.dumps(payload, default=str).encode("utf-8")
+        req = Request(url, data=body, headers=self._headers(), method="POST")
+        try:
+            with urlopen(req, timeout=10):
+                pass
+        except URLError:
+            pass
+
+    def _get(self, url: str) -> Optional[dict]:
+        req = Request(url, headers=self._headers(), method="GET")
+        try:
+            with urlopen(req, timeout=10) as resp:
+                return json.loads(resp.read().decode("utf-8"))
+        except URLError:
+            return None

veval/_options.py

@@ -0,0 +1,29 @@
+import os
+import warnings
+
+
+class VevalOptions:
+    def __init__(
+        self,
+        api_key: str = "",
+        project_id: str = "",
+        flush_interval_ms: int = 5000,
+        flush_batch_size: int = 50,
+    ):
+        # The API key also determines the workspace.
+        self.api_key = api_key
+        if project_id:
+            warnings.warn(
+                "VevalOptions.project_id is ignored: the API key determines the workspace. "
+                "It will be removed in a future version.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        self.project_id = project_id
+        # Veval's own hosted API, which enforces billing/quota (test-run limits, judge credits, etc.) —
+        # not a self-hosted override for SDK consumers, so it isn't a public option.
+        # VEVAL_INTERNAL_ENDPOINT is an undocumented escape hatch for Veval's own local development
+        # against a non-production API instance.
+        self._endpoint = os.environ.get("VEVAL_INTERNAL_ENDPOINT", "https://api.veval.dev")
+        self.flush_interval_ms = flush_interval_ms
+        self.flush_batch_size = flush_batch_size

veval/_scenarios.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any, Optional, TYPE_CHECKING
+from ._assertions import ITraceAssertion
+
+if TYPE_CHECKING:
+    from ._context import VevalExecutionContext
+
+
+@dataclass
+class ScenarioItem:
+    name: Optional[str] = None
+    trace_id: Optional[str] = None
+    input: Any = None
+    assertions: list[ITraceAssertion] = field(default_factory=list)
+
+
+@dataclass
+class ItemRunResult:
+    item: ScenarioItem = field(default_factory=ScenarioItem)
+    failures: list[str] = field(default_factory=list)
+    context: Optional[VevalExecutionContext] = None
+
+    @property
+    def passed(self) -> bool:
+        return len(self.failures) == 0
+
+
+@dataclass
+class ScenarioRunResult:
+    results: list[ItemRunResult] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return all(r.passed for r in self.results)
+
+    @property
+    def pass_count(self) -> int:
+        return sum(1 for r in self.results if r.passed)
+
+    @property
+    def fail_count(self) -> int:
+        return sum(1 for r in self.results if not r.passed)