agentsproof 0.1.0__py3-none-any.whl

agentsproof/__init__.py

@@ -0,0 +1,17 @@
+from .client import AgentsProof
+from .run import AgentRun
+from .tracer import atrace_llm, atrace_tool, trace_llm, trace_tool
+from .types import GoldenCase, ProofSuiteResult, StepPayload, StepType
+
+__all__ = [
+    "AgentsProof",
+    "AgentRun",
+    "trace_llm",
+    "trace_tool",
+    "atrace_llm",
+    "atrace_tool",
+    "StepPayload",
+    "StepType",
+    "GoldenCase",
+    "ProofSuiteResult",
+]

agentsproof/client.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Optional
+
+from .proof_suite import arun_proof_suite, run_proof_suite
+from .run import AgentRun
+from .types import ProofSuiteResult
+
+
+class AgentsProof:
+    def __init__(self, *, api_key: str, base_url: str = "https://agentsproof.dev/api") -> None:
+        self._api_key = api_key
+        self._base_url = base_url.rstrip("/")
+
+    def start_run(
+        self,
+        *,
+        project_slug: str,
+        input: Any,
+        label: Optional[str] = None,
+        goal: Optional[str] = None,
+        expected_output: Any = None,
+        metadata: Optional[dict] = None,
+    ) -> AgentRun:
+        return AgentRun(
+            project_slug=project_slug,
+            input=input,
+            label=label,
+            goal=goal,
+            expected_output=expected_output,
+            metadata=metadata,
+            api_key=self._api_key,
+            base_url=self._base_url,
+        )
+
+    def run_proof_suite(
+        self,
+        *,
+        project_slug: str,
+        suite_slug: str,
+        handler: Callable[[Any, Any], Any],
+    ) -> ProofSuiteResult:
+        """Run approved Goldens locally against your agent (sync). Handler must be a regular function."""
+        return run_proof_suite(
+            project_slug=project_slug,
+            suite_slug=suite_slug,
+            handler=handler,
+            api_key=self._api_key,
+            base_url=self._base_url,
+        )
+
+    async def arun_proof_suite(
+        self,
+        *,
+        project_slug: str,
+        suite_slug: str,
+        handler: Callable[[Any, Any], Any],
+    ) -> ProofSuiteResult:
+        """Run approved Goldens locally against your agent (async). Handler can be sync or async."""
+        return await arun_proof_suite(
+            project_slug=project_slug,
+            suite_slug=suite_slug,
+            handler=handler,
+            api_key=self._api_key,
+            base_url=self._base_url,
+        )

agentsproof/proof_suite.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import inspect
+from typing import Any, Callable, Optional
+
+import httpx
+
+from .run import AgentRun
+from .types import GoldenCase, ProofSuiteResult
+
+
+def run_proof_suite(
+    *,
+    project_slug: str,
+    suite_slug: str,
+    handler: Callable[[Any, Any], Any],
+    api_key: str,
+    base_url: str,
+) -> ProofSuiteResult:
+    """Sync version. handler(input, ctx) must be a regular (non-async) function."""
+    with httpx.Client() as client:
+        res = client.get(
+            f"{base_url}/proof-suites/{suite_slug}/cases",
+            params={"projectSlug": project_slug},
+            headers={"x-api-key": api_key},
+            timeout=15,
+        )
+        if not res.is_success:
+            raise RuntimeError(f"AgentsProof: failed to load proof suite — {res.status_code}")
+        data = res.json()
+
+    proof_run_id: str = data["proofRunId"]
+    cases: list[GoldenCase] = data["cases"]
+
+    for golden in cases:
+        run_id: Optional[str] = None
+        passed = False
+        failure_summary: Optional[str] = None
+        handler_run: Optional[AgentRun] = None
+
+        def start_run(**overrides: Any) -> AgentRun:
+            nonlocal handler_run
+            r = AgentRun(
+                project_slug=project_slug,
+                label=overrides.get("label", f"Proof case: {golden.get('name', '')}"),
+                input=overrides.get("input", golden.get("input")),
+                goal=overrides.get("goal", golden.get("goal")),
+                expected_output=overrides.get("expected_output", golden.get("expected_output")),
+                metadata={**overrides.get("metadata", {}), "goldenId": golden["id"], "proofRunId": proof_run_id},
+                api_key=api_key,
+                base_url=base_url,
+            )
+            handler_run = r
+            return r
+
+        class Ctx:
+            golden_case = golden
+
+            @staticmethod
+            def start_run(**overrides: Any) -> AgentRun:
+                return start_run(**overrides)
+
+        try:
+            handler(golden.get("input"), Ctx)
+            if handler_run is not None:
+                run_id = handler_run.remote_run_id
+            passed = True
+        except Exception as err:
+            failure_summary = str(err)
+
+        try:
+            with httpx.Client() as client:
+                client.post(
+                    f"{base_url}/proof-runs/{proof_run_id}/case-results",
+                    headers={"x-api-key": api_key},
+                    json={
+                        "goldenId": golden["id"],
+                        "runId": run_id,
+                        "score": None,
+                        "passed": passed,
+                        "failureSummary": failure_summary,
+                    },
+                    timeout=10,
+                )
+        except Exception:
+            pass
+
+    with httpx.Client() as client:
+        res = client.post(
+            f"{base_url}/proof-runs/{proof_run_id}/complete",
+            headers={"x-api-key": api_key},
+            timeout=30,
+        )
+        if not res.is_success:
+            raise RuntimeError(f"AgentsProof: failed to complete proof suite — {res.status_code}")
+        return res.json()
+
+
+async def arun_proof_suite(
+    *,
+    project_slug: str,
+    suite_slug: str,
+    handler: Callable[[Any, Any], Any],
+    api_key: str,
+    base_url: str,
+) -> ProofSuiteResult:
+    """Async version. handler(input, ctx) can be sync or async."""
+    async with httpx.AsyncClient() as client:
+        res = await client.get(
+            f"{base_url}/proof-suites/{suite_slug}/cases",
+            params={"projectSlug": project_slug},
+            headers={"x-api-key": api_key},
+            timeout=15,
+        )
+        if not res.is_success:
+            raise RuntimeError(f"AgentsProof: failed to load proof suite — {res.status_code}")
+        data = res.json()
+
+    proof_run_id: str = data["proofRunId"]
+    cases: list[GoldenCase] = data["cases"]
+
+    for golden in cases:
+        run_id: Optional[str] = None
+        passed = False
+        failure_summary: Optional[str] = None
+        handler_run: Optional[AgentRun] = None
+
+        def start_run(**overrides: Any) -> AgentRun:
+            nonlocal handler_run
+            r = AgentRun(
+                project_slug=project_slug,
+                label=overrides.get("label", f"Proof case: {golden.get('name', '')}"),
+                input=overrides.get("input", golden.get("input")),
+                goal=overrides.get("goal", golden.get("goal")),
+                expected_output=overrides.get("expected_output", golden.get("expected_output")),
+                metadata={**overrides.get("metadata", {}), "goldenId": golden["id"], "proofRunId": proof_run_id},
+                api_key=api_key,
+                base_url=base_url,
+            )
+            handler_run = r
+            return r
+
+        class Ctx:
+            golden_case = golden
+
+            @staticmethod
+            def start_run(**overrides: Any) -> AgentRun:
+                return start_run(**overrides)
+
+        try:
+            result = handler(golden.get("input"), Ctx)
+            if inspect.iscoroutine(result):
+                await result
+            if handler_run is not None:
+                run_id = handler_run.remote_run_id
+            passed = True
+        except Exception as err:
+            failure_summary = str(err)
+
+        try:
+            async with httpx.AsyncClient() as client:
+                await client.post(
+                    f"{base_url}/proof-runs/{proof_run_id}/case-results",
+                    headers={"x-api-key": api_key},
+                    json={
+                        "goldenId": golden["id"],
+                        "runId": run_id,
+                        "score": None,
+                        "passed": passed,
+                        "failureSummary": failure_summary,
+                    },
+                    timeout=10,
+                )
+        except Exception:
+            pass
+
+    async with httpx.AsyncClient() as client:
+        res = await client.post(
+            f"{base_url}/proof-runs/{proof_run_id}/complete",
+            headers={"x-api-key": api_key},
+            timeout=30,
+        )
+        if not res.is_success:
+            raise RuntimeError(f"AgentsProof: failed to complete proof suite — {res.status_code}")
+        return res.json()

agentsproof/run.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import inspect
+import threading
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Callable, Optional, TypeVar
+
+import httpx
+
+from .types import StepPayload, StepType
+
+T = TypeVar("T")
+
+
+class AgentRun:
+    def __init__(
+        self,
+        *,
+        project_slug: str,
+        input: Any,
+        api_key: str,
+        base_url: str,
+        label: Optional[str] = None,
+        goal: Optional[str] = None,
+        expected_output: Any = None,
+        metadata: Optional[dict] = None,
+    ) -> None:
+        self.run_id = uuid.uuid4().hex[:12]
+        self._api_key = api_key
+        self._base_url = base_url
+        self._steps: list = []
+        self._started_at = time.monotonic()
+        self._remote_run_id: Optional[str] = None
+
+        self._init_remote(
+            project_slug=project_slug,
+            input=input,
+            label=label,
+            goal=goal,
+            expected_output=expected_output,
+            metadata=metadata or {},
+        )
+
+    def _init_remote(
+        self,
+        *,
+        project_slug: str,
+        input: Any,
+        label: Optional[str],
+        goal: Optional[str],
+        expected_output: Any,
+        metadata: dict,
+    ) -> None:
+        with httpx.Client() as client:
+            res = client.post(
+                f"{self._base_url}/runs",
+                headers={"x-api-key": self._api_key},
+                json={
+                    "label": label,
+                    "input": input,
+                    "projectSlug": project_slug,
+                    "clientRunId": self.run_id,
+                    "goal": goal,
+                    "expectedOutput": expected_output,
+                    "metadata": metadata,
+                },
+                timeout=10,
+            )
+            res.raise_for_status()
+            self._remote_run_id = res.json()["runId"]
+
+    def log_step(self, payload: StepPayload) -> None:
+        step = {
+            **payload,
+            "step_index": len(self._steps),
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+        self._steps.append(step)
+
+        def _send() -> None:
+            try:
+                with httpx.Client() as client:
+                    client.post(
+                        f"{self._base_url}/runs/{self._remote_run_id}/steps",
+                        headers={"x-api-key": self._api_key},
+                        json=step,
+                        timeout=10,
+                    )
+            except Exception:
+                pass  # SDK must never crash the agent
+
+        threading.Thread(target=_send, daemon=True).start()
+
+    def trace(self, type: StepType, name: str, fn: Callable[[], T], input: Any = None) -> T:
+        """Wrap a sync callable and auto-log it as a step with latency captured."""
+        t0 = time.monotonic()
+        try:
+            result = fn()
+        except Exception as err:
+            self.log_step({
+                "type": type,
+                "name": name,
+                "input": input,
+                "output": {"error": str(err)},
+                "latency_ms": (time.monotonic() - t0) * 1000,
+            })
+            raise
+        self.log_step({
+            "type": type,
+            "name": name,
+            "input": input,
+            "output": result,
+            "latency_ms": (time.monotonic() - t0) * 1000,
+        })
+        return result
+
+    async def atrace(self, type: StepType, name: str, fn: Callable[[], Any], input: Any = None) -> Any:
+        """Wrap a sync or async callable and auto-log it as a step. Use in async contexts."""
+        t0 = time.monotonic()
+        try:
+            result = await fn() if inspect.iscoroutinefunction(fn) else fn()
+        except Exception as err:
+            self.log_step({
+                "type": type,
+                "name": name,
+                "input": input,
+                "output": {"error": str(err)},
+                "latency_ms": (time.monotonic() - t0) * 1000,
+            })
+            raise
+        self.log_step({
+            "type": type,
+            "name": name,
+            "input": input,
+            "output": result,
+            "latency_ms": (time.monotonic() - t0) * 1000,
+        })
+        return result
+
+    def complete(self, output: Any) -> dict:
+        """Finish the run and trigger grading. Returns {"publicUrl": "..."}."""
+        with httpx.Client() as client:
+            res = client.post(
+                f"{self._base_url}/runs/{self._remote_run_id}/complete",
+                headers={"x-api-key": self._api_key},
+                json={"output": output},
+                timeout=30,
+            )
+            res.raise_for_status()
+            return res.json()
+
+    async def acomplete(self, output: Any) -> dict:
+        """Async version of complete()."""
+        async with httpx.AsyncClient() as client:
+            res = await client.post(
+                f"{self._base_url}/runs/{self._remote_run_id}/complete",
+                headers={"x-api-key": self._api_key},
+                json={"output": output},
+                timeout=30,
+            )
+            res.raise_for_status()
+            return res.json()
+
+    @property
+    def elapsed_ms(self) -> float:
+        return (time.monotonic() - self._started_at) * 1000
+
+    @property
+    def remote_run_id(self) -> Optional[str]:
+        return self._remote_run_id

agentsproof/tracer.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Any, Callable, TypeVar
+
+from .run import AgentRun
+
+T = TypeVar("T")
+
+
+def trace_llm(run: AgentRun, name: str, fn: Callable[[], T], input: Any = None) -> T:
+    """Convenience wrapper for tracing a sync LLM call."""
+    return run.trace("llm_call", name, fn, input)
+
+
+def trace_tool(run: AgentRun, name: str, fn: Callable[[], T], input: Any = None) -> T:
+    """Convenience wrapper for tracing a sync tool call."""
+    return run.trace("tool_call", name, fn, input)
+
+
+async def atrace_llm(run: AgentRun, name: str, fn: Callable[[], Any], input: Any = None) -> Any:
+    """Convenience wrapper for tracing a sync or async LLM call."""
+    return await run.atrace("llm_call", name, fn, input)
+
+
+async def atrace_tool(run: AgentRun, name: str, fn: Callable[[], Any], input: Any = None) -> Any:
+    """Convenience wrapper for tracing a sync or async tool call."""
+    return await run.atrace("tool_call", name, fn, input)

agentsproof/types.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Coroutine, Dict, List, Literal, Optional, Union
+
+StepType = Literal["llm_call", "tool_call", "tool_result", "memory_read", "memory_write"]
+
+# Using plain dicts with TypedDict for IDE support without runtime overhead
+try:
+    from typing import TypedDict
+except ImportError:
+    from typing_extensions import TypedDict
+
+
+class StepPayload(TypedDict, total=False):
+    type: StepType
+    name: str
+    input: Any
+    output: Any
+    latency_ms: float
+    token_count: int
+    cost_usd: float
+
+
+class GoldenCase(TypedDict, total=False):
+    id: str
+    name: str
+    input: Any
+    goal: str
+    expected_output: Any
+    expected_behavior: str
+    success_criteria: List[str]
+    trace_assertions: List[str]
+    custom_grader_ids: List[str]
+
+
+class ProofSuiteResult(TypedDict, total=False):
+    passed_cases: int
+    failed_cases: int
+    overall_score: float
+    public_url: str

agentsproof-0.1.0.dist-info/METADATA

@@ -0,0 +1,150 @@
+Metadata-Version: 2.4
+Name: agentsproof
+Version: 0.1.0
+Summary: Observability and proof reporting for AI agents
+Project-URL: Homepage, https://agentsproof.dev
+License: MIT
+Keywords: agents,ai,llm,observability,tracing
+Requires-Python: >=3.9
+Requires-Dist: httpx>=0.27.0
+Description-Content-Type: text/markdown
+
+# agentsproof
+
+Drop the SDK into your Python agent, define what "good" means, and get a shareable proof report.
+
+## Install
+
+```bash
+pip install agentsproof
+```
+
+## Quick start — single run (sync)
+
+Works with any Python agent — OpenAI, Anthropic, LangChain, LlamaIndex, or plain functions.
+
+```python
+import os
+from agentsproof import AgentsProof
+
+ap = AgentsProof(api_key=os.environ["AGENTSPROOF_API_KEY"])
+
+def run_my_agent(user_query: str):
+    run = ap.start_run(
+        project_slug="my-coding-agent",
+        label="Answer coding question",
+        input={"query": user_query},
+        goal="Search the web for relevant docs and return a working code solution",
+    )
+
+    # Wrap any callable — the SDK captures latency and output automatically
+    plan = run.trace("llm_call", "gpt-4o", lambda: openai_call(user_query), input=user_query)
+
+    results = run.trace("tool_call", "web_search", lambda: web_search(plan))
+
+    final_answer = run.trace("llm_call", "gpt-4o", lambda: openai_call(results))
+
+    result = run.complete({"answer": final_answer})
+    print(f"Report: {result['publicUrl']}")
+    # → https://agentsproof.dev/r/abc123
+```
+
+## Quick start — async agent
+
+```python
+import asyncio
+import os
+from agentsproof import AgentsProof
+
+ap = AgentsProof(api_key=os.environ["AGENTSPROOF_API_KEY"])
+
+async def run_my_agent(user_query: str):
+    run = ap.start_run(
+        project_slug="my-coding-agent",
+        input={"query": user_query},
+        goal="Return a working code solution",
+    )
+
+    # Use atrace() for async callables
+    plan = await run.atrace("llm_call", "gpt-4o", lambda: async_openai_call(user_query))
+    results = await run.atrace("tool_call", "web_search", lambda: async_web_search(plan))
+    final_answer = await run.atrace("llm_call", "gpt-4o", lambda: async_openai_call(results))
+
+    result = await run.acomplete({"answer": final_answer})
+    print(f"Report: {result['publicUrl']}")
+
+asyncio.run(run_my_agent("How do I reverse a list in Python?"))
+```
+
+## Proof Suites — regression testing
+
+```python
+import os
+from agentsproof import AgentsProof
+
+ap = AgentsProof(api_key=os.environ["AGENTSPROOF_API_KEY"])
+
+def handler(input, ctx):
+    run = ctx.start_run()
+    result = my_agent(input)
+    run.complete({"answer": result})
+
+result = ap.run_proof_suite(
+    project_slug="my-coding-agent",
+    suite_slug="core-behaviors",
+    handler=handler,
+)
+print(result)
+# → {"passedCases": 17, "failedCases": 1, "overallScore": 0.91, "publicUrl": "..."}
+```
+
+### Async proof suite
+
+```python
+async def async_handler(input, ctx):
+    run = ctx.start_run()
+    result = await my_async_agent(input)
+    await run.acomplete({"answer": result})
+
+result = await ap.arun_proof_suite(
+    project_slug="my-coding-agent",
+    suite_slug="core-behaviors",
+    handler=async_handler,
+)
+```
+
+## API
+
+### `AgentsProof(api_key, base_url?)`
+Create a client. Get your API key from [agentsproof.dev](https://agentsproof.dev).
+
+### `client.start_run(...)` → `AgentRun`
+
+| Param | Type | Required | Description |
+|---|---|---|---|
+| `project_slug` | `str` | yes | Your project identifier |
+| `input` | `Any` | yes | The initial input or prompt to the agent |
+| `label` | `str` | no | Human-readable label for this run |
+| `goal` | `str` | no | What this run should accomplish |
+| `expected_output` | `Any` | no | Expected output for grading comparison |
+| `metadata` | `dict` | no | Optional key/value metadata |
+
+### `run.trace(type, name, fn, input?)` → `T`
+Wrap a **sync** callable and auto-log it as a step with latency captured.
+
+### `run.atrace(type, name, fn, input?)` → `Awaitable[T]`
+Wrap a **sync or async** callable. Use in `async` agent code.
+
+### `run.log_step(payload)`
+Manually log a step. Step types: `llm_call` | `tool_call` | `tool_result` | `memory_read` | `memory_write`.
+
+### `run.complete(output)` → `{"publicUrl": str}`
+Finish the run, trigger grading, and get back the public report URL.
+
+### `run.acomplete(output)` → `Awaitable[{"publicUrl": str}]`
+Async version of `complete()`.
+
+### `client.run_proof_suite(...)` / `client.arun_proof_suite(...)`
+Run approved Goldens locally against your agent. AgentsProof never executes user code remotely.
+
+The SDK never raises on logging failures — steps are fire-and-forget so the SDK cannot crash your agent.

agentsproof-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+agentsproof/__init__.py,sha256=3uSqAQCT-h9cqZkJBF1xIZ4nV5tdSQ6BsLqDDm0MkLY,395
+agentsproof/client.py,sha256=yqZBU-bdksXzQphord9-OuFDy92G7dzqVb3UcfpEQ_I,1956
+agentsproof/proof_suite.py,sha256=Eg1kQ65JePcP7pui_8pzAMVkjGrXNLWBNEnqNaRWLEc,6350
+agentsproof/run.py,sha256=ZrZuNZstF4GBdatBMFnrYrA7dAiiyjUPYTWEZ2WtFFQ,5401
+agentsproof/tracer.py,sha256=xFjOkeaSQz6swML-kY9fMaiTXsnzcJfKmH9PNLhJ68Y,971
+agentsproof/types.py,sha256=EaLiw6By8enmCL0u7-oDoCVAL7stAdG3-HiUlhBVy48,947
+agentsproof-0.1.0.dist-info/METADATA,sha256=QPbBFUR134vZQ-kAYWerVOnExPw3nCZUvwIR3wYeAm4,4627
+agentsproof-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+agentsproof-0.1.0.dist-info/RECORD,,

agentsproof-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any