trybench-sdk 0.1.0__py3-none-any.whl

bench_sdk/__init__.py

@@ -0,0 +1,6 @@
+"""Bench server tracing. Capturing events never starts a paid evaluation."""
+from .client import Bench, Span
+
+__all__ = ["Bench", "Span", "EvaluationContext"]
+
+from .evaluation import EvaluationContext

bench_sdk/client.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import contextvars
+import ipaddress
+import json
+import math
+import random
+import re
+import secrets
+import threading
+import time
+import urllib.error
+import urllib.request
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Callable, Iterator
+from urllib.parse import urlsplit
+
+_SENSITIVE = re.compile(r"authorization|cookie|password|secret|token|api.?key|email|phone|address|user.?id|(?:first|last|full).?name|card.?number", re.I)
+_METADATA = re.compile(r"^(code\.(filepath|lineno)|gen_ai\.(system|provider\.name|operation\.name|request\.model|response\.model|tool\.(name|type|call\.id)|usage\.(input_tokens|output_tokens))|bench\.(component_id|environment|prompt_version|duration_ms|cost\.(usd|source|pricing_version)))$")
+_SECRETS = re.compile(r"(?:bench_sk_|apikey_|sk-)[a-zA-Z0-9_-]{8,}|Bearer\s+[a-zA-Z0-9._~+/-]+", re.I)
+_EMAIL = re.compile(r"(?<![a-z0-9.!#$%&'*+/=?^_`{|}~-])[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9-]+(?:\.[a-z0-9-]+)+", re.I)
+_PHONE = re.compile(r"\+\d[\d ()-]{8,}\d")
+_IP = re.compile(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b")
+_CARD = re.compile(r"\b(?:[0-9]{4}(?:[ -][0-9]{4}){3}[ -][0-9]{3}|[0-9]{4}(?:[ -][0-9]{4}){3}|[0-9]{4}[ -][0-9]{6}[ -][0-9]{5}|[0-9]{13,19})\b")
+_KINDS = {"LLM", "TOOL", "CHAIN", "RETRIEVER", "AGENT", "EMBEDDING"}
+
+
+def _ip(match: re.Match) -> str:
+    try:
+        ipaddress.IPv4Address(match[0])
+        return "[REDACTED_IP]"
+    except ValueError:
+        return match[0]
+
+
+def _card(match: re.Match) -> str:
+    digits = "".join(c for c in match[0] if c.isdigit())
+    candidates = [digits]
+    if len(digits) == 19 and len(match[0]) > 19:
+        candidates.append(digits[:16])
+    for candidate in candidates:
+        total = sum(n if i % 2 == 0 else n * 2 - (9 if n > 4 else 0) for i, n in enumerate(map(int, reversed(candidate))))
+        if total > 0 and total % 10 == 0:
+            return "[REDACTED_PAYMENT_NUMBER]"
+    return match[0]
+
+
+def _redact(value: Any, depth: int = 0, max_items: int = 100) -> Any:
+    if depth > 12:
+        return "[DEPTH_LIMIT]"
+    if isinstance(value, str):
+        if len(value) > 16000:
+            return "[CONTENT_LIMIT]"
+        # Encoded JSON must receive the same field filtering as an object.
+        if value.lstrip().startswith(("{", "[")):
+            try:
+                return json.dumps(_redact(json.loads(value), depth + 1, max_items), ensure_ascii=False)[:16000]
+            except (ValueError, RecursionError):
+                pass
+        value = _SECRETS.sub("[REDACTED_SECRET]", value)
+        value = _EMAIL.sub("[REDACTED_EMAIL]", value)
+        value = _CARD.sub(_card, _IP.sub(_ip, value))
+        return _PHONE.sub("[REDACTED_PHONE]", value)[:16000]
+    if isinstance(value, dict):
+        return {_redact(str(k), depth + 1, max_items): "[REDACTED]" if _SENSITIVE.search(str(k)) and not (_METADATA.fullmatch(str(k)) and str(k).startswith("gen_ai.usage.") and isinstance(v, (int, float))) else _redact(v, depth + 1, max_items) for k, v in list(value.items())[:max_items]}
+    if isinstance(value, (list, tuple)):
+        return [_redact(v, depth + 1, max_items) for v in value[:max_items]]
+    if not isinstance(value, bool) and (isinstance(value, int) or (isinstance(value, float) and math.isfinite(value) and value.is_integer())):
+        normalized = str(int(value))
+        if 13 <= len(normalized) <= 19:
+            filtered = _CARD.sub(_card, normalized)
+            return filtered if filtered != normalized else value
+    if value is None or isinstance(value, (bool, int)):
+        return value
+    if isinstance(value, float) and math.isfinite(value):
+        return value
+    return None
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+
+class Span:
+    """Use set_output for a request's final value; model errors are not captured."""
+    def __init__(self, name: str, kind: str, input: Any, attributes: dict | None, model: str | None, component_id: int | None):
+        self.name, self.kind, self.input = name, kind, input
+        self.attributes, self.model, self.component_id = attributes or {}, model, component_id
+        self.output: Any = None
+
+    def set_output(self, value: Any) -> None:
+        self.output = value
+
+
+class Bench:
+    """Thread-safe, bounded telemetry with explicit flush at lifecycle boundaries.
+
+    trace() is a context manager usable around sync code and across await points.
+    Async tasks inherit their parent's context without sharing sibling span IDs.
+    """
+    def __init__(self, *, api_key: str, repository: str, branch: str, system_name: str | None = None,
+                 environment: str | None = None, endpoint: str = "https://api.trybench.ai",
+                 capture_content: bool = False, sample_rate: float = 1, max_queue_size: int = 200,
+                 timeout: float = 5, redact: Callable[[Any], Any] | None = None,
+                 on_error: Callable[[str], None] | None = None,
+                 transport: Callable[[str, dict[str, str], bytes, float], int] | None = None):
+        if not api_key.startswith("bench_sk_") or not repository or not branch:
+            raise ValueError("A Bench key, repository and branch are required.")
+        if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) or any(len(v) > 200 or re.search(r"[\x00-\x1f\x7f]", v) or _redact(v) != v for v in (repository, branch)):
+            raise ValueError("Use static repository and branch names without personal data or secrets.")
+        parsed = urlsplit(endpoint)
+        if not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment or not (parsed.scheme == "https" or (parsed.scheme == "http" and parsed.hostname in ("localhost", "127.0.0.1", "::1"))):
+            raise ValueError("Use HTTPS or a loopback HTTP endpoint.")
+        if environment is not None and not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", environment):
+            raise ValueError("Use a short environment name.")
+        if not 0 <= sample_rate <= 1 or type(max_queue_size) is not int or not 1 <= max_queue_size <= 2000 or not 0.1 <= timeout <= 30:
+            raise ValueError("Sampling, queue size or timeout is out of range.")
+        self._key, self._repository, self._branch = api_key, repository, branch
+        self._system, self._environment = _redact(system_name or repository.rsplit("/", 1)[-1]), environment
+        self._endpoint, self._capture, self._sample = endpoint.rstrip("/") + "/api/traces", capture_content, sample_rate
+        self._max, self._timeout, self._redactor, self._on_error = max_queue_size, timeout, redact, on_error
+        self._transport = transport or self._http
+        self._context: contextvars.ContextVar = contextvars.ContextVar("bench_span", default=None)
+        self._evaluation: contextvars.ContextVar = contextvars.ContextVar("bench_evaluation", default=None)
+        self._lock, self._flush_lock = threading.Lock(), threading.Lock()
+        self._queue: list[dict] = []
+        self._closed, self._dropped = False, 0
+
+    def _safe(self, value: Any) -> Any:
+        return _redact(self._redactor(value) if self._redactor else value)
+
+    def _report(self, message: str) -> None:
+        if self._on_error:
+            try:
+                self._on_error(message)
+            except Exception:
+                pass
+
+    @contextmanager
+    def trace(self, name: str, *, kind: str = "LLM", input: Any = None, model: str | None = None,
+              attributes: dict | None = None, component_id: int | None = None) -> Iterator[Span]:
+        if kind not in _KINDS:
+            raise ValueError("Invalid span kind.")
+        evaluation = self._evaluation.get()
+        if evaluation is not None:
+            evaluation.start()
+        parent = self._context.get()
+        trace_id = parent[0] if parent else secrets.token_hex(16)
+        sampled = evaluation is not None or (parent[2] if parent else random.random() < self._sample)
+        span_id, started, duration_start = secrets.token_hex(8), _now(), time.monotonic()
+        token = self._context.set((trace_id, span_id, sampled))
+        span, status = Span(name, kind, input, attributes, model, component_id), "error"
+        try:
+            yield span
+            status = "ok"
+        finally:
+            self._context.reset(token)
+            if sampled:
+                self._record(span, trace_id, span_id, parent[1] if parent else None, started, status, evaluation, (time.monotonic() - duration_start) * 1000)
+
+    def _record(self, span: Span, trace_id: str, span_id: str, parent: str | None, started: str, status: str, evaluation=None, duration_ms: float = 0) -> None:
+        row = None
+        try:
+            attrs = {k: v for k, v in span.attributes.items() if self._capture or _METADATA.fullmatch(k)}
+            attrs["bench.duration_ms"] = duration_ms
+            if self._environment:
+                attrs["bench.environment"] = self._environment
+            if span.component_id is not None:
+                attrs["bench.component_id"] = span.component_id
+            row = {"span_id": span_id, "name": str(self._safe(span.name))[:200], "kind": span.kind,
+                   "started_at": started, "ended_at": _now(), "status": status, "attributes": self._safe(attrs)}
+            if parent:
+                row["parent_span_id"] = parent
+            if span.model:
+                row["model_name"] = str(self._safe(span.model))[:200]
+            if self._capture or evaluation is not None:
+                row["input_value"] = json.dumps(self._safe(span.input), ensure_ascii=False, allow_nan=False)
+                row["output_value"] = json.dumps(self._safe(span.output), ensure_ascii=False, allow_nan=False)
+            trace = {"trace_id": trace_id, "source": "bench_sdk", "spans": [row]}
+            if len(json.dumps(trace).encode()) > 200000:
+                raise ValueError("Trace too large")
+            if evaluation is not None:
+                evaluation.finish(row)
+                return
+            with self._lock:
+                if self._closed or len(self._queue) >= self._max:
+                    self._dropped += 1
+                else:
+                    self._queue.append(trace)
+        except Exception:
+            if evaluation is not None:
+                evaluation.finish(None)
+                return
+            with self._lock:
+                self._dropped += 1
+            self._report("Trace could not be captured; dropped.")
+
+    async def evaluate_system(self, **options) -> dict:
+        """Run application cases locally. Returns a redacted report; no implicit upload."""
+        from .evaluation import evaluate
+        return await evaluate(self, **options)
+
+    async def simulate_system(self, **options) -> dict:
+        """Replay customer turns against a fresh app session and independently observed state."""
+        from .evaluation import simulate
+        return await simulate(self, **options)
+
+    async def publish_system_evaluation(self, system_id: int, report: dict) -> None:
+        """Explicitly save a redacted report. Raises on failure; spends no evaluations."""
+        import asyncio
+        if type(system_id) is not int or not 1 <= system_id <= 9007199254740991:
+            raise ValueError("Select a system.")
+        raw = self._redactor(report) if self._redactor else report
+        body = json.dumps(_redact(raw, max_items=300), allow_nan=False).encode()
+        if len(body) > 500000:
+            raise ValueError("Runtime report exceeds 500 KB. Retain it locally or split the suite.")
+        url = self._endpoint.removesuffix("/api/traces") + f"/api/ai-systems/{system_id}/runtime-evaluations"
+        status = await asyncio.to_thread(self._transport, url, {"Content-Type": "application/json", "Authorization": "Bearer " + self._key}, body, self._timeout)
+        if not 200 <= status < 300:
+            raise RuntimeError(f"Could not save application results (HTTP {status}). Local results remain available.")
+
+    @staticmethod
+    def _http(url: str, headers: dict[str, str], body: bytes, timeout: float) -> int:
+        request = urllib.request.Request(url, body, headers, method="POST")
+        try:
+            with urllib.request.build_opener(_NoRedirect()).open(request, timeout=timeout) as response:
+                return response.status
+        except urllib.error.HTTPError as error:
+            status = error.code
+            error.close()
+            return status
+
+    @property
+    def stats(self) -> dict[str, int]:
+        with self._lock:
+            return {"queued": len(self._queue), "dropped": self._dropped}
+
+    def flush(self) -> None:
+        """Bounded network work. Delivery failures never raise into application code."""
+        with self._flush_lock:
+            with self._lock:
+                pending, self._queue = self._queue, []
+            while pending:
+                batch, size = [], 0
+                while pending and len(batch) < 20:
+                    next_size = len(json.dumps(pending[0]).encode())
+                    if size + next_size > 800000:
+                        break
+                    size += next_size
+                    batch.append(pending.pop(0))
+                body = json.dumps({"repo_full_name": self._repository, "branch": self._branch, "system_name": self._system, "capture_content": self._capture, "traces": batch}).encode()
+                delivered = False
+                for attempt in range(2):
+                    try:
+                        status = self._transport(self._endpoint, {"Content-Type": "application/json", "Authorization": "Bearer " + self._key}, body, self._timeout)
+                        if 200 <= status < 300:
+                            delivered = True
+                            break
+                        if status != 429 and status < 500:
+                            break
+                    except Exception:
+                        pass
+                    if attempt == 0:
+                        time.sleep(0.25)
+                if not delivered:
+                    with self._lock:
+                        self._dropped += len(batch)
+                    self._report("Bench trace delivery failed; batch dropped.")
+
+    async def aflush(self) -> None:
+        """Flush without blocking an asyncio event loop."""
+        import asyncio
+        await asyncio.to_thread(self.flush)
+
+    def shutdown(self) -> None:
+        with self._lock:
+            self._closed = True
+        self.flush()
+
+    def __enter__(self) -> Bench:
+        return self
+
+    def __exit__(self, *_: Any) -> None:
+        self.shutdown()

bench_sdk/evaluation.py

@@ -0,0 +1,258 @@
+"""Application checks share the versioned Bench report contract, without network work."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import inspect
+import json
+import re
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+_ALIASES = {"expected_output": "expectedOutput", "expected_state": "expectedState", "business_outcome": "businessOutcome",
+            "required_tools": "requiredTools", "forbidden_tools": "forbiddenTools", "max_tool_calls": "maxToolCalls", "max_model_calls": "maxModelCalls"}
+
+
+def snapshot(value: Any) -> Any:
+    return json.loads(json.dumps(value, allow_nan=False))
+
+
+def equal(a: Any, b: Any) -> bool:
+    if isinstance(a, bool) or isinstance(b, bool):
+        return type(a) is type(b) and a == b
+    if isinstance(a, dict) and isinstance(b, dict):
+        return a.keys() == b.keys() and all(equal(a[k], b[k]) for k in a)
+    if isinstance(a, list) and isinstance(b, list):
+        return len(a) == len(b) and all(equal(x, y) for x, y in zip(a, b))
+    return a == b
+
+
+@dataclass(frozen=True)
+class EvaluationContext:
+    case_id: str
+    signal: threading.Event
+    deadline: float
+    turn_index: int | None = None
+
+    @property
+    def cancelled(self) -> bool:
+        return self.signal.is_set() or time.monotonic() >= self.deadline
+
+    def raise_if_cancelled(self) -> None:
+        if self.cancelled:
+            raise asyncio.CancelledError()
+
+
+class Capture:
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.active, self.limited, self.pending = True, False, 0
+        self.spans: list[dict] = []
+
+    def start(self):
+        with self.lock:
+            self.pending += 1
+
+    def finish(self, row):
+        with self.lock:
+            self.pending -= 1
+            if self.active:
+                if row is None or len(self.spans) >= 100:
+                    self.limited = True
+                else:
+                    self.spans.append(row)
+
+    def freeze(self):
+        with self.lock:
+            self.active = False
+            return list(self.spans), self.limited, self.pending
+
+
+def validate(source_revision, context_revision, cases, timeout):
+    if not isinstance(source_revision, str) or not re.fullmatch(r"[a-f0-9]{40}", source_revision) or not isinstance(context_revision, str) or not 1 <= len(context_revision) <= 200:
+        raise ValueError("Pin a full source revision and a context revision.")
+    if not isinstance(cases, list) or not 1 <= len(cases) <= 100 or isinstance(timeout, bool) or not 0 < timeout <= 300:
+        raise ValueError("Use 1 to 100 cases and a timeout up to 300 seconds.")
+    pinned = snapshot(cases)
+    if len(json.dumps(pinned).encode()) > 500000:
+        raise ValueError("Case suite exceeds 500 KB.")
+    ids = set()
+    for case in pinned:
+        if not isinstance(case, dict):
+            raise ValueError("A case must be an object.")
+        for key, alias in _ALIASES.items():
+            if key in case:
+                if alias in case:
+                    raise ValueError("Use one spelling per case field.")
+                case[alias] = case.pop(key)
+        name = case.get("id")
+        if not isinstance(name, str) or not 1 <= len(name) <= 100 or name in ids:
+            raise ValueError("Case IDs must be unique and nonempty.")
+        ids.add(name)
+        if "input" not in case or case.get("split", "regression") not in ("capability", "incident", "regression", "holdout"):
+            raise ValueError("Each case needs an input and valid split.")
+        if "businessOutcome" in case and (not isinstance(case["businessOutcome"], str) or not 1 <= len(case["businessOutcome"]) <= 2000):
+            raise ValueError("Provide a bounded business outcome.")
+        count = 0
+        for key in ("requiredTools", "forbiddenTools"):
+            names = case.get(key, [])
+            if not isinstance(names, list) or any(not isinstance(n, str) or not 1 <= len(n) <= 200 for n in names):
+                raise ValueError("Invalid tool assertions.")
+            count += len(names)
+        if count > 90:
+            raise ValueError("Use at most 90 tool assertions per case.")
+        for key in ("maxToolCalls", "maxModelCalls"):
+            if key in case and (type(case[key]) is not int or not 0 <= case[key] <= 10000):
+                raise ValueError("Call limits must be nonnegative integers up to 10000.")
+    return pinned
+
+
+async def invoke(fn: Callable, *args):
+    # Sync callbacks run off-loop. Cancellation cannot kill a Python thread.
+    result = fn(*args) if inspect.iscoroutinefunction(fn) else await asyncio.to_thread(fn, *args)
+    return await result if inspect.isawaitable(result) else result
+
+
+def score(case, output, state, observed, spans, error):
+    if not any(not s.get("parent_span_id") and s["kind"] == "AGENT" and s["status"] == "ok" for s in spans):
+        error = error or "Application root was not recorded. No complete score."
+    if "expectedState" in case and not observed:
+        error = error or "Application state was not observed. No complete score."
+    checks, findings = [], []
+    def check(id, passed, reason):
+        checks.append(dict(id=id, passed=passed, reason=reason))
+    if "expectedOutput" in case:
+        check("expected-output", equal(output, case["expectedOutput"]), "Compare actual output with the expected outcome.")
+    if "expectedState" in case and observed:
+        check("expected-state", equal(state, case["expectedState"]), "Compare observed tool effects with the expected business state.")
+    tools = [s for s in spans if s["kind"] == "TOOL"]
+    for name in case.get("requiredTools", []):
+        check("required-tool:" + name, any(s["name"] == name and s["status"] == "ok" for s in tools), "A successful recorded tool call is required.")
+    for name in case.get("forbiddenTools", []):
+        check("forbidden-tool:" + name, not any(s["name"] == name for s in tools), "This tool must not be invoked.")
+    for field, kind, label in (("maxToolCalls", "TOOL", "tool"), ("maxModelCalls", "LLM", "model")):
+        if field in case:
+            count = sum(s["kind"] == kind for s in spans)
+            check(label + "-call-limit", count <= case[field], f"{count} recorded calls; limit {case[field]}.")
+    for tool in tools:
+        if tool["status"] == "error":
+            findings.append(dict(category="tool", title=tool["name"] + " failed", evidence_span_ids=[tool["span_id"]], confidence="observed_failure", fix_brief="Inspect this tool's contract and dependencies. Reproduce the failure, fix it, and rerun incident and regression cases."))
+    for item in checks:
+        if not item["passed"]:
+            findings.append(dict(category="quality" if item["id"] in ("expected-output", "expected-state") else "harness", title=item["id"], evidence_span_ids=[s["span_id"] for s in spans], confidence="hypothesis", fix_brief="Inspect the real application path, tools, state, routing and retries. Change one suspected cause and rerun unchanged incident, regression and holdout cases."))
+    status = "error" if error else "unscored" if not checks else "passed" if all(c["passed"] for c in checks) else "failed"
+    return dict(id=case["id"], split=case.get("split", "regression"), status=status, checks=checks, findings=findings, spans=spans, **({"error": error} if error else {}))
+
+
+async def evaluate(bench, *, source_revision: str, context_revision: str, cases: list[dict], run: Callable,
+                   observe: Callable | None = None, timeout: float = 30, cancel_event: threading.Event | None = None) -> dict:
+    pinned = validate(source_revision, context_revision, cases, timeout)
+    if not callable(run) or (observe is not None and not callable(observe)):
+        raise ValueError("A real application entry point is required.")
+    if bench._closed or bench._evaluation.get() is not None:
+        raise ValueError("Use an open client outside another application evaluation.")
+    results = []
+    for case in pinned:
+        if cancel_event and cancel_event.is_set():
+            break
+        capture = Capture()
+        context = EvaluationContext(case["id"], threading.Event(), time.monotonic() + timeout)
+        async def execute():
+            own = bench._evaluation.set(capture)
+            parent = bench._context.set(None)
+            try:
+                with bench.trace("system-entrypoint", kind="AGENT", input=snapshot(case["input"])) as root:
+                    output = snapshot(await invoke(run, snapshot(case["input"]), context))
+                    state = snapshot(await invoke(observe, context)) if observe else None
+                    root.set_output(output)
+                return output, state
+            finally:
+                bench._context.reset(parent)
+                bench._evaluation.reset(own)
+        async def cancellation():
+            while not (cancel_event and cancel_event.is_set()):
+                await asyncio.sleep(0.01)
+        task, cancel = asyncio.create_task(execute()), asyncio.create_task(cancellation())
+        error, output, state, observed, stopped = None, None, None, False, False
+        try:
+            done, _ = await asyncio.wait([task, cancel], timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
+            if task not in done or context.cancelled or (cancel_event and cancel_event.is_set()):
+                stopped = True
+                error = "Application timed out or was stopped. No complete score."
+            else:
+                try:
+                    output, state = task.result()
+                    observed = observe is not None
+                except (Exception, asyncio.CancelledError):
+                    error = "Application execution failed. Inspect recorded spans."
+        finally:
+            context.signal.set()
+            cancel.cancel()
+            if not task.done():
+                task.cancel()
+            # Consume eventual exceptions without waiting forever for uncooperative code.
+            task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
+            spans, limited, pending = capture.freeze()
+        if limited or pending:
+            error = "Application evidence is incomplete. Await all tools and stay within capture limits."
+            stopped = stopped or bool(pending)
+        result = score(case, output, state, observed, spans, error)
+        try:
+            result["case_definition"] = bench._safe(case)
+            result["output"] = bench._safe(output)
+            if observed:
+                result["observedState"] = bench._safe(state)
+        except Exception:
+            result.update(status="error", error="Application evidence could not be redacted.")
+        results.append(result)
+        if stopped:
+            break
+    counts = {"passed": 0, "failed": 0, "errors": 0, "unscored": 0}
+    for case in results:
+        counts["errors" if case["status"] == "error" else case["status"]] += 1
+    complete = len(results) == len(pinned) and not counts["errors"] and not counts["unscored"]
+    digest = hashlib.sha256(json.dumps(dict(context=context_revision, cases=pinned), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest()
+    return dict(schema_version=1, execution_mode="application_runtime", evidence_origin="sdk_client_reported",
+                environment=bench._environment or "unspecified", source_revision=source_revision, context_revision=context_revision,
+                suite_hash=digest, planned_case_count=len(pinned), coverage=dict(instrumentation="explicit_spans", tool_dependencies="application_configured", hosted_validation=False),
+                cases=results, summary=dict(status="completed" if complete else "incomplete", score=100 * counts["passed"] / len(pinned) if complete else None, **counts))
+
+
+async def simulate(bench, *, create_session: Callable, **options) -> dict:
+    if not callable(create_session):
+        raise ValueError("An application session factory is required.")
+    cases = snapshot(options["cases"])
+    for case in cases:
+        data = case.get("input", {})
+        if not isinstance(data, dict) or not isinstance(data.get("turns"), list) or not 1 <= len(data["turns"]) <= 20:
+            raise ValueError("A simulation requires 1 to 20 scripted user turns.")
+        if "expected_state" not in case and "expectedState" not in case:
+            raise ValueError("A simulation requires the expected business state.")
+        if "initial_state" in data:
+            if "initialState" in data:
+                raise ValueError("Use one spelling for initial state.")
+            data["initialState"] = data.pop("initial_state")
+        if "initialState" not in data:
+            raise ValueError("Provide initial_state for a fresh simulation session.")
+    observations = {}
+    async def run(data, context):
+        session = await invoke(create_session, data["initialState"], context)
+        try:
+            if not all(callable(getattr(session, method, None)) for method in ("turn", "observe", "close")):
+                raise ValueError("A session must provide turn, observe and close.")
+            reply = None
+            for index, message in enumerate(data["turns"]):
+                context.raise_if_cancelled()
+                turn = EvaluationContext(context.case_id, context.signal, context.deadline, index)
+                reply = await invoke(session.turn, message, turn)
+            context.raise_if_cancelled()
+            observations[context.case_id] = snapshot(await invoke(session.observe))
+            return reply
+        finally:
+            if callable(getattr(session, "close", None)):
+                await invoke(session.close)
+    def observe(context):
+        return observations.pop(context.case_id)
+    return await evaluate(bench, **{**options, "cases": cases}, run=run, observe=observe)

trybench_sdk-0.1.0.dist-info/METADATA

@@ -0,0 +1,155 @@
+Metadata-Version: 2.4
+Name: trybench-sdk
+Version: 0.1.0
+Summary: Trace AI applications and test their behavior with Bench
+Author: Bench
+License-Expression: Apache-2.0
+Project-URL: Documentation, https://docs.usebench.ai/sdk/python
+Project-URL: Repository, https://github.com/trybench/bench-sdk
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Bench Python SDK · Beta
+
+**Beta, version 0.1.0.** Pin versions and test upgrades in staging.
+
+Trace Python applications, agents and tools without changing their behavior.
+Python 3.10+. No third-party runtime dependencies. Apache-2.0.
+
+Install with `python -m pip install trybench-sdk`. The Python import is `bench_sdk`.
+
+```python
+import os
+from bench_sdk import Bench
+
+bench = Bench(
+    api_key=os.environ["BENCH_API_KEY"],
+    repository="your-team/your-app",
+    branch="main",
+    environment="staging",
+    system_name="Support agent",
+    endpoint=os.environ.get("BENCH_API_BASE_URL", "https://api.trybench.ai"),
+)
+
+try:
+    with bench.trace("support-request", kind="AGENT") as span:
+        # Call your existing application here.
+        result = handle_request()
+        span.set_output(result)
+finally:
+    # Deliver success and failure events after this application's work finishes.
+    bench.shutdown()
+```
+
+Use the same `with bench.trace(...)` around `await agent.ainvoke(...)` in an
+async application, then `await bench.aflush()`. Context is preserved across
+await points and inherited by nested tasks. Wrap tools in nested spans with
+`kind="TOOL"`. Separate requests started outside a parent get separate trace IDs.
+
+Works with explicit wrappers around Deep Agents, LangGraph, LangChain, model
+clients and custom Python code. It does not automatically instrument framework
+internals or consume a streaming result; keep the span open while reading the stream.
+
+Inputs, outputs and custom attributes are omitted by default. With permission,
+set `capture_content=True`, pass `input=...` and call `span.set_output(...)`.
+Built-in matching removes common secrets, emails, supported phone numbers, IPv4
+addresses, card patterns and sensitive structured fields before transmission.
+Add a `redact(value)` callback for application-specific data. These rules do not
+recognize every personal detail in free text.
+
+Set `component_id` to a real prompt component from Bench to link the event to
+its criteria. Use operational attributes such as `gen_ai.usage.input_tokens`
+and `gen_ai.usage.output_tokens` for usage counts. Never invent component IDs.
+
+The queue defaults to 200 spans. Flush explicitly at request or process lifecycle
+boundaries. Transient delivery failures retry once with unchanged IDs, then drop
+the batch. `bench.stats` reports queued and dropped spans. `on_error(message)`
+receives a fixed message, without content or keys. A telemetry failure does not
+replace an application exception. This is a bounded queue, not durable storage.
+
+## Test your application
+
+`await bench.evaluate_system(...)` calls your application's request handler with
+pinned cases. It captures the real nested tool/model traces, compares the final
+output and independently observed state, and returns a redacted report locally.
+Use a test database and test service credentials.
+
+```python
+report = await bench.evaluate_system(
+    source_revision=os.environ["GIT_COMMIT_SHA"],  # Full 40-character commit SHA
+    context_revision="refund-policy-v1",
+    cases=[{
+        "id": "outside-refund-policy", "split": "regression",
+        "input": {"days": 45}, "expected_output": {"refunded": False},
+        "forbidden_tools": ["issue-refund"],
+    }],
+    run=lambda request, context: handle_request(request),
+)
+assert report["summary"]["status"] == "completed"
+assert all(case["status"] == "passed" for case in report["cases"])
+# Explicit upload, only when you want this report saved in Bench:
+await bench.publish_system_evaluation(int(os.environ["BENCH_SYSTEM_ID"]), report)
+```
+
+`run(input, context)` and `observe(context)` may be synchronous or asynchronous.
+When a case has `expected_state`, supply `observe` to read the authoritative test
+state. Context provides `case_id`, `cancelled`, `signal`, `deadline` and
+`raise_if_cancelled()`. Inputs are JSON snapshots; changes inside the application
+do not change the case's assertions.
+
+`await bench.simulate_system(...)` accepts the same revisions and cases, plus
+`create_session(initial_state, context)`. Cases use
+`input={"initial_state": {...}, "turns": [...]}` and require `expected_state`.
+Return an object with `turn(message, context)`, `observe()` and `close()` methods.
+A fresh session receives 1 to 20 scripted customer turns. Bench snapshots observed
+state before closing the session. `context.turn_index` identifies the turn.
+
+Both helpers use a 30-second timeout per case, configurable with `timeout` up to
+300 seconds. Pass a `threading.Event` as `cancel_event` to stop the suite. Missing
+assertions, missing state, unfinished traces and timeouts remain incomplete.
+Callbacks must honor cancellation; Python cannot forcibly stop a synchronous
+thread. Await all child work and isolate external side effects. These helpers
+are local execution, not a process sandbox or a hosted verification claim.
+
+Tests record redacted content even when production capture is metadata-only, so
+use synthetic inputs. Reports are not uploaded and paid checks are not started
+unless you take a separate explicit action. Automatic framework adapters are
+coming soon.
+
+Run checks from this directory:
+
+```sh
+PYTHONPATH=src python -m unittest discover -s tests -v
+python -m build
+python -m twine check dist/*
+```
+
+See [Bench documentation](https://docs.usebench.ai/sdk/python) and the repository's
+[publishing guide](../PUBLISHING.md).
+
+## Latency, tool calls and cost
+
+Every recorded call carries start/end timestamps, status, parent span ID and an
+automatically measured `bench.duration_ms` from a monotonic clock. Wrap each tool
+execution, including retries, with a TOOL span to retain its individual timing.
+Use `gen_ai.operation.name=execute_tool` and `gen_ai.tool.name` for tool identity.
+Production sampling can omit traces; a rate of 1 records each instrumented call.
+The bounded delivery queue is not a guarantee against network or process loss.
+
+Add `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model`,
+`gen_ai.usage.input_tokens` and `gen_ai.usage.output_tokens` when your provider
+returns them. Add `bench.cost.usd` for the cost of that individual call and
+`bench.cost.source` as `reported` or `estimated`. For estimates, also include
+`bench.cost.pricing_version`. These fields survive metadata-only capture, so you
+can measure usage without recording prompts or responses. Missing cost is unknown,
+not zero. Do not repeat a child cost on its parent or count overlapping token
+categories twice. The SDK does not guess provider prices or a tool's own charges.
+
+The `gen_ai.*` names follow selected OpenTelemetry conventions. `bench.cost.*` and
+`bench.duration_ms` are Bench extensions. Events currently use Bench JSON over
+HTTPS; this release is not an OTLP exporter or collector.

trybench_sdk-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+bench_sdk/__init__.py,sha256=2gfNuc8rQrrMiUBUNm7HY9XhACWOtC_c9qicn36n1DI,202
+bench_sdk/client.py,sha256=XyZTqlgjvOWqFEzmhtn4cI72L031u5T45El8-evCAb4,14778
+bench_sdk/evaluation.py,sha256=RkmHdtQLejxNomykzrX1i4e1okYuwhABfNVOt7IWHmM,13843
+bench_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trybench_sdk-0.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+trybench_sdk-0.1.0.dist-info/METADATA,sha256=83ik3zqA12bzVC_cWBWcig6yUwhUkO8molQceqVuZe8,7290
+trybench_sdk-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+trybench_sdk-0.1.0.dist-info/top_level.txt,sha256=H5RBp5ZBWSuTh8LoxKB2m1SDDsZs96MPy0eimVW0BZ8,10
+trybench_sdk-0.1.0.dist-info/RECORD,,

trybench_sdk-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

trybench_sdk-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

trybench_sdk-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+bench_sdk