overturn 0.1.0__tar.gz

overturn-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+node_modules/
+dist/
+.next/
+.turbo/
+coverage/
+*.log
+.env
+.env.*
+!.env.example
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.DS_Store
+overturn-report.*

overturn-0.1.0/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.5
+Name: overturn
+Version: 0.1.0
+Summary: Overturn SDK: record agent runs, turn human corrections into regression cases, replay safely
+Project-URL: Homepage, https://github.com/overrule-ai/overturn
+Project-URL: Repository, https://github.com/overrule-ai/overturn
+Project-URL: Documentation, https://github.com/overrule-ai/overturn/blob/main/docs/sdk.md
+Author: Overturn contributors
+License-Expression: Apache-2.0
+Keywords: agents,evals,llm,regression-testing,replay
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.5
+Requires-Dist: uuid-utils>=0.9
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.30; extra == 'anthropic'
+Provides-Extra: openai
+Requires-Dist: openai>=1.40; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# overturn
+
+Python SDK for Overturn. See the repository README.

overturn-0.1.0/README.md

@@ -0,0 +1,3 @@
+# overturn
+
+Python SDK for Overturn. See the repository README.

overturn-0.1.0/overturn/__init__.py

@@ -0,0 +1,67 @@
+"""Overturn Python SDK.
+
+from overturn import Overturn, tool
+
+ov = Overturn(api_key=os.environ["OVERTURN_API_KEY"])
+client = ov.wrap_openai(OpenAI())
+
+@tool(side_effect=False, match_on=["email"])
+def lookup_customer(email: str) -> dict: ...
+
+with ov.run(agent="ticket-triage", version="1.4.0", input=ticket) as run:
+    ...
+    run.end(output=decision)
+"""
+
+from .api import OverturnApi, OverturnApiError
+from .client import Overturn
+from .context import current_run
+from .fingerprint import FINGERPRINT_VERSION, canonical_json, fingerprint, hash_json, tool_fingerprint
+from .pricing import estimate_cost_usd
+from .replay import (
+    MaxStepsExceededError,
+    ReplayModelCall,
+    ReplayModelOutput,
+    ReplayState,
+    SideEffectRefusedError,
+    ToolRecording,
+    ToolRecordings,
+    UnreplayableError,
+)
+from .run import Run
+from .tools import Tool, tool
+from .transport import Transport, TransportError
+from .types import IngestPayload, RunUpsert, StepUpsert, Usage
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "FINGERPRINT_VERSION",
+    "IngestPayload",
+    "MaxStepsExceededError",
+    "ReplayModelCall",
+    "ReplayModelOutput",
+    "Overturn",
+    "OverturnApi",
+    "OverturnApiError",
+    "ReplayState",
+    "Run",
+    "RunUpsert",
+    "SideEffectRefusedError",
+    "StepUpsert",
+    "Tool",
+    "ToolRecording",
+    "ToolRecordings",
+    "Transport",
+    "TransportError",
+    "UnreplayableError",
+    "Usage",
+    "__version__",
+    "canonical_json",
+    "current_run",
+    "estimate_cost_usd",
+    "fingerprint",
+    "hash_json",
+    "tool",
+    "tool_fingerprint",
+]

overturn-0.1.0/overturn/_json.py

@@ -0,0 +1,35 @@
+"""Small typed accessors for loosely-typed JSON-like data (SDK response dumps)."""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+
+def as_dict(v: Any) -> dict[str, Any]:
+    return cast(dict[str, Any], v) if isinstance(v, dict) else {}
+
+
+def as_list(v: Any) -> list[Any]:
+    return cast(list[Any], v) if isinstance(v, list) else []
+
+
+def get_dict(d: dict[str, Any], key: str) -> dict[str, Any]:
+    return as_dict(d.get(key))
+
+
+def get_list(d: dict[str, Any], key: str) -> list[Any]:
+    return as_list(d.get(key))
+
+
+def get_str(d: dict[str, Any], key: str) -> str | None:
+    v = d.get(key)
+    return v if isinstance(v, str) else None
+
+
+def get_int(d: dict[str, Any], key: str) -> int | None:
+    v = d.get(key)
+    return v if isinstance(v, int) and not isinstance(v, bool) else None
+
+
+def is_jsonable(v: Any) -> bool:
+    return isinstance(v, (str, int, float, bool, list, dict))

overturn-0.1.0/overturn/anthropic_wrap.py

@@ -0,0 +1,229 @@
+"""wrap_anthropic: record messages.create (sync/async, streaming or not) as llm_call steps."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import AsyncIterator, Iterator
+from datetime import datetime, timezone
+from typing import Any, TypeVar, cast
+
+from . import context
+from ._json import get_dict, get_int, get_str, is_jsonable
+from .pricing import estimate_cost_usd
+from .types import Usage
+
+T = TypeVar("T")
+
+_INPUT_KEYS = {"messages", "system", "tools", "tool_choice"}
+_SKIP_KEYS = {"model", "stream", *_INPUT_KEYS}
+
+
+def split_message_params(params: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any], str | None]:
+    inp = {k: v for k, v in params.items() if k in _INPUT_KEYS}
+    rest: dict[str, Any] = {k: v for k, v in params.items() if k not in _SKIP_KEYS and is_jsonable(v)}
+    model = params.get("model")
+    return inp, rest, model if isinstance(model, str) else None
+
+
+def _dump(obj: Any) -> dict[str, Any]:
+    dump = getattr(obj, "model_dump", None)
+    if callable(dump):
+        return cast(dict[str, Any], dump(mode="json", exclude_none=True))
+    if isinstance(obj, dict):
+        return cast(dict[str, Any], obj)
+    return {}
+
+
+def _usage_from(d: dict[str, Any]) -> Usage | None:
+    if not isinstance(d.get("usage"), dict):
+        return None
+    ud = get_dict(d, "usage")
+    inp = get_int(ud, "input_tokens")
+    out = get_int(ud, "output_tokens")
+    return Usage(
+        input_tokens=inp,
+        output_tokens=out,
+        total_tokens=(inp or 0) + (out or 0) if inp is not None or out is not None else None,
+        cached_input_tokens=get_int(ud, "cache_read_input_tokens"),
+    )
+
+
+def _record(
+    run: Any,
+    model: str | None,
+    inp: Any,
+    params: Any,
+    message: Any,
+    started: datetime,
+    first: float | None,
+    error: str | None = None,
+) -> None:
+    if run is None:
+        return
+    d = _dump(message) if message is not None else {}
+    usage = _usage_from(d) if message is not None else None
+    if usage is not None and first is not None:
+        usage.first_token_ms = first
+    resolved = get_str(d, "model") or model
+    output: dict[str, Any] | None = None
+    if message is not None:
+        output = {
+            "message": {"role": get_str(d, "role") or "assistant", "content": d.get("content")},
+            "finish_reason": get_str(d, "stop_reason"),
+        }
+    run.step(
+        kind="llm_call",
+        name="messages",
+        provider="anthropic",
+        model=resolved,
+        input=inp,
+        params=params,
+        output=output,
+        usage=usage,
+        cost_usd=estimate_cost_usd(resolved, usage),
+        latency_ms=(datetime.now(timezone.utc) - started).total_seconds() * 1000,
+        error=error,
+        started_at=started,
+    )
+
+
+def assemble_message_stream(events: list[Any]) -> dict[str, Any]:
+    """Rebuilds a Message from raw stream events (message_start, content_block_*, message_delta)."""
+    message: dict[str, Any] = {
+        "role": "assistant",
+        "content": [],
+        "model": None,
+        "stop_reason": None,
+        "usage": {},
+    }
+    blocks: dict[int, dict[str, Any]] = {}
+    json_parts: dict[int, list[str]] = {}
+    usage: dict[str, Any] = {}
+    for ev in events:
+        e = _dump(ev)
+        t = get_str(e, "type")
+        if t == "message_start":
+            m = get_dict(e, "message")
+            message["model"] = get_str(m, "model")
+            usage.update(get_dict(m, "usage"))
+        elif t == "content_block_start":
+            idx = get_int(e, "index") or 0
+            block = dict(get_dict(e, "content_block"))
+            blocks[idx] = block
+            if get_str(block, "type") == "tool_use":
+                json_parts[idx] = []
+                block["input"] = {}
+            elif get_str(block, "type") == "text":
+                block["text"] = get_str(block, "text") or ""
+        elif t == "content_block_delta":
+            idx = get_int(e, "index") or 0
+            delta = get_dict(e, "delta")
+            block = blocks.setdefault(idx, {})
+            if get_str(delta, "type") == "text_delta":
+                block["text"] = (get_str(block, "text") or "") + (get_str(delta, "text") or "")
+            elif get_str(delta, "type") == "input_json_delta":
+                json_parts.setdefault(idx, []).append(get_str(delta, "partial_json") or "")
+        elif t == "message_delta":
+            delta = get_dict(e, "delta")
+            if get_str(delta, "stop_reason"):
+                message["stop_reason"] = get_str(delta, "stop_reason")
+            usage.update({k: v for k, v in get_dict(e, "usage").items() if v is not None})
+    message["usage"] = usage
+    import json as _json
+
+    for idx, parts in json_parts.items():
+        raw = "".join(parts)
+        try:
+            blocks[idx]["input"] = _json.loads(raw) if raw else {}
+        except ValueError:
+            blocks[idx]["input"] = {"_raw": raw}
+    message["content"] = [blocks[i] for i in sorted(blocks)]
+    return message
+
+
+class _TappedStream:
+    def __init__(self, stream: Any, on_done: Any) -> None:
+        self._stream = stream
+        self._on_done = on_done
+        self._events: list[Any] = []
+        self._t0 = time.perf_counter()
+        self._first: float | None = None
+
+    def __iter__(self) -> Iterator[Any]:
+        try:
+            for ev in self._stream:
+                if self._first is None:
+                    self._first = (time.perf_counter() - self._t0) * 1000
+                self._events.append(ev)
+                yield ev
+        finally:
+            self._on_done(self._events, self._first)
+
+    def __aiter__(self) -> AsyncIterator[Any]:
+        return self._aiter()
+
+    async def _aiter(self) -> AsyncIterator[Any]:
+        try:
+            async for ev in self._stream:
+                if self._first is None:
+                    self._first = (time.perf_counter() - self._t0) * 1000
+                self._events.append(ev)
+                yield ev
+        finally:
+            self._on_done(self._events, self._first)
+
+    def __getattr__(self, item: str) -> Any:
+        return getattr(self._stream, item)
+
+
+def wrap_anthropic(client: T) -> T:
+    """Patches client.messages.create in place (Anthropic and AsyncAnthropic)."""
+    messages = cast(Any, client).messages
+    original = messages.create
+    is_async = type(client).__name__.startswith("Async")
+
+    def on_done_for(run: Any, model: str | None, inp: Any, params: Any, started: datetime) -> Any:
+        def on_done(events: list[Any], first: float | None) -> None:
+            _record(run, model, inp, params, assemble_message_stream(events), started, first)
+
+        return on_done
+
+    if is_async:
+
+        async def create_async(*args: Any, **kwargs: Any) -> Any:
+            run = context.current_run()
+            if run is None:
+                return await original(*args, **kwargs)
+            started = datetime.now(timezone.utc)
+            inp, params, model = split_message_params(kwargs)
+            try:
+                result = await original(*args, **kwargs)
+            except Exception as err:
+                _record(run, model, inp, params, None, started, None, error=f"{type(err).__name__}: {err}")
+                raise
+            if kwargs.get("stream"):
+                return _TappedStream(result, on_done_for(run, model, inp, params, started))
+            _record(run, model, inp, params, result, started, None)
+            return result
+
+        messages.create = create_async
+    else:
+
+        def create_sync(*args: Any, **kwargs: Any) -> Any:
+            run = context.current_run()
+            if run is None:
+                return original(*args, **kwargs)
+            started = datetime.now(timezone.utc)
+            inp, params, model = split_message_params(kwargs)
+            try:
+                result = original(*args, **kwargs)
+            except Exception as err:
+                _record(run, model, inp, params, None, started, None, error=f"{type(err).__name__}: {err}")
+                raise
+            if kwargs.get("stream"):
+                return _TappedStream(result, on_done_for(run, model, inp, params, started))
+            _record(run, model, inp, params, result, started, None)
+            return result
+
+        messages.create = create_sync
+    return client

overturn-0.1.0/overturn/api.py

@@ -0,0 +1,60 @@
+"""Thin typed-ish client for the Overturn REST API (used by correct())."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+import httpx
+
+from ._json import as_dict
+
+CorrectionKind = Literal["approve", "edit", "reject", "escalate"]
+
+
+class OverturnApiError(Exception):
+    def __init__(self, status: int, body: Any, message: str) -> None:
+        super().__init__(message)
+        self.status = status
+        self.body = body
+
+
+class OverturnApi:
+    def __init__(self, api_key: str, base_url: str, http_client: httpx.Client | None = None) -> None:
+        self.api_key = api_key
+        self.base_url = base_url.rstrip("/")
+        self._client = http_client or httpx.Client(timeout=30.0)
+
+    def request(self, method: str, path: str, body: Any = None, params: dict[str, Any] | None = None) -> Any:
+        res = self._client.request(
+            method,
+            f"{self.base_url}{path}",
+            json=body,
+            params={k: v for k, v in (params or {}).items() if v is not None},
+            headers={"authorization": f"Bearer {self.api_key}", "user-agent": "overturn-sdk-python/0.1.0"},
+        )
+        parsed: Any = res.json() if res.content else None
+        if not res.is_success:
+            msg = as_dict(parsed).get("message", res.text[:200])
+            raise OverturnApiError(
+                res.status_code, parsed, f"{method} {path} failed: {res.status_code} {msg}"
+            )
+        return parsed
+
+    # ---- endpoints ----
+    def me(self) -> dict[str, Any]:
+        return self.request("GET", "/v1/me")
+
+    def list_runs(self, **params: Any) -> dict[str, Any]:
+        return self.request("GET", "/v1/runs", params=params)
+
+    def get_run(self, run_id: str) -> dict[str, Any]:
+        return self.request("GET", f"/v1/runs/{run_id}")
+
+    def create_correction(self, body: dict[str, Any]) -> dict[str, Any]:
+        return self.request("POST", "/v1/corrections", body=body)
+
+    def list_cases(self, **params: Any) -> dict[str, Any]:
+        return self.request("GET", "/v1/cases", params=params)
+
+    def get_case(self, case_id: str) -> dict[str, Any]:
+        return self.request("GET", f"/v1/cases/{case_id}")

overturn-0.1.0/overturn/client.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Callable, Sequence
+from typing import Any, ParamSpec, TypeVar
+
+import httpx
+
+from . import context
+from . import tools as _tools
+from .anthropic_wrap import wrap_anthropic
+from .api import CorrectionKind, OverturnApi
+from .openai_wrap import wrap_openai
+from .replay import ReplayModelCall, ReplayState, ToolRecordings
+from .run import Run
+from .tools import Tool
+from .transport import OnError, Transport
+from .types import RunUpsert, StepUpsert
+
+P = ParamSpec("P")
+R = TypeVar("R")
+T = TypeVar("T")
+
+Redact = Callable[[StepUpsert], StepUpsert | None]
+
+
+class Overturn:
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        *,
+        redact: Redact | None = None,
+        enabled: bool = True,
+        debug: bool | None = None,
+        on_error: OnError | None = None,
+        flush_interval_s: float = 1.0,
+        max_batch_size: int = 100,
+        http_client: httpx.Client | None = None,
+        background: bool = True,
+    ) -> None:
+        self.api_key = api_key or os.environ.get("OVERTURN_API_KEY", "")
+        self.base_url = base_url or os.environ.get("OVERTURN_BASE_URL", "https://api.overturn.run")
+        self.debug = debug if debug is not None else os.environ.get("OVERTURN_DEBUG") == "1"
+        self.redact = redact
+        self.replay: ReplayState | None = None
+        self.transport: Transport | None = None
+        self.api = OverturnApi(self.api_key, self.base_url, http_client=http_client)
+        if enabled and self.api_key:
+            self.transport = Transport(
+                self.api_key,
+                self.base_url,
+                flush_interval_s=flush_interval_s,
+                max_batch_size=max_batch_size,
+                on_error=on_error,
+                debug=self.debug,
+                http_client=http_client,
+                background=background,
+            )
+        elif enabled and self.debug:
+            print("[overturn] no API key set; nothing will be recorded")
+
+    # ---- sink ----
+    def emit_run(self, run: RunUpsert) -> None:
+        if _tools.get_replay_state() is not None:
+            return  # replay runs are never recorded as production runs
+        if self.transport:
+            self.transport.enqueue_run(run)
+
+    def emit_step(self, step: StepUpsert) -> None:
+        replay = _tools.get_replay_state()
+        if replay is not None:
+            replay.record(step)
+            return
+        if not self.transport:
+            return
+        out = self.redact(step) if self.redact else step
+        if out is not None:
+            self.transport.enqueue_step(out)
+
+    # ---- API ----
+    def run(
+        self,
+        agent: str,
+        *,
+        version: str | None = None,
+        input: Any = None,
+        tags: list[str] | None = None,
+        metadata: dict[str, Any] | None = None,
+        external_id: str | None = None,
+        id: str | None = None,
+    ) -> Run:
+        """Start a run. Use as a context manager: `with ov.run(agent="x") as run: ...`"""
+        return Run(
+            self,
+            agent=agent,
+            version=version,
+            input=input,
+            tags=tags,
+            metadata=metadata,
+            external_id=external_id,
+            id=id,
+        )
+
+    start_run = run
+
+    def current_run(self) -> Run | None:
+        return context.current_run()
+
+    def tool(
+        self,
+        name: str,
+        fn: Callable[P, R],
+        *,
+        side_effect: bool = False,
+        match_on: Sequence[str] | None = None,
+        ignore_args: Sequence[str] | None = None,
+        description: str | None = None,
+    ) -> Tool[P, R]:
+        return Tool(
+            fn,
+            name=name,
+            side_effect=side_effect,
+            match_on=match_on,
+            ignore_args=ignore_args,
+            description=description,
+        )
+
+    def wrap_openai(self, client: T) -> T:
+        return wrap_openai(client)
+
+    def wrap_anthropic(self, client: T) -> T:
+        return wrap_anthropic(client)
+
+    def correct(
+        self,
+        *,
+        run_id: str,
+        kind: CorrectionKind,
+        step_id: str | None = None,
+        corrected_output: Any = None,
+        note: str | None = None,
+        actor: str | None = None,
+    ) -> dict[str, Any]:
+        """Record a human correction and freeze a regression case. Sent immediately; returns
+        {"correction": ..., "case": ...}."""
+        body: dict[str, Any] = {"runId": run_id, "kind": kind, "source": "sdk"}
+        if step_id is not None:
+            body["stepId"] = step_id
+        if corrected_output is not None:
+            body["correctedOutput"] = corrected_output
+        if note is not None:
+            body["note"] = note
+        if actor is not None:
+            body["actor"] = actor
+        return self.api.create_correction(body)
+
+    def replay_mode(
+        self,
+        recordings: ToolRecordings,
+        *,
+        allow_side_effects: bool = False,
+        model_call: ReplayModelCall | None = None,
+        max_steps: int = 50,
+    ) -> ReplayState:
+        """Every @tool is answered from recordings and never executed, until exit_replay_mode().
+        With model_call, wrapped model clients answer from the target model instead of the provider."""
+        self.replay = ReplayState(
+            recordings, allow_side_effects=allow_side_effects, model_call=model_call, max_steps=max_steps
+        )
+        _tools.set_replay_state(self.replay)
+        return self.replay
+
+    def exit_replay_mode(self) -> None:
+        self.replay = None
+        _tools.set_replay_state(None)
+
+    def flush(self) -> None:
+        if self.transport:
+            self.transport.flush()
+
+    def shutdown(self) -> None:
+        if self.transport:
+            self.transport.close()

overturn-0.1.0/overturn/context.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .run import Run
+
+_current_run: ContextVar[Run | None] = ContextVar("overturn_current_run", default=None)
+_parent_step: ContextVar[str | None] = ContextVar("overturn_parent_step", default=None)
+
+
+def current_run() -> Run | None:
+    return _current_run.get()
+
+
+def parent_step_id() -> str | None:
+    return _parent_step.get()