agentproof-sim 0.1.1__py3-none-any.whl

agentproof/mutations/state.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from agentproof.mutations.base import Mutation, register_mutation
+
+
+@register_mutation
+class StaleState(Mutation):
+    mutation_type = "stale_state"
+    description = "Return configured stale data for a read tool without changing canonical state."
+
+
+@register_mutation
+class StateChangedAfterRead(Mutation):
+    mutation_type = "state_changed_after_read"
+    description = "Mutate configured world state after a read response and before later actions."

agentproof/mutations/timing.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from agentproof.mutations.base import Mutation, register_mutation
+
+
+@register_mutation
+class DelayedEvent(Mutation):
+    mutation_type = "delayed_event"
+    description = "Push event delivery later on the virtual clock."

agentproof/mutations/tool_faults.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from agentproof.mutations.base import Mutation, register_mutation
+
+
+@register_mutation
+class ToolTimeout(Mutation):
+    mutation_type = "tool_timeout"
+    description = "Raise a timeout before the virtual tool handler commits."
+
+
+@register_mutation
+class TimeoutAfterCommit(Mutation):
+    mutation_type = "timeout_after_commit"
+    description = "Commit the virtual tool outcome, then surface a timeout to the agent."
+
+
+@register_mutation
+class ToolError(Mutation):
+    mutation_type = "tool_error"
+    description = "Raise a configured tool error before the handler runs."
+
+
+@register_mutation
+class ToolLatency(Mutation):
+    mutation_type = "tool_latency"
+    description = "Advance the virtual clock without sleeping wall-clock time."
+
+
+@register_mutation
+class RateLimited(Mutation):
+    mutation_type = "rate_limited"
+    description = "Raise a recognizable rate-limit error before the handler runs."
+
+
+@register_mutation
+class MalformedResponse(Mutation):
+    mutation_type = "malformed_response"
+    description = "Replace a successful tool response with malformed deterministic data."
+
+
+@register_mutation
+class MissingField(Mutation):
+    mutation_type = "missing_field"
+    description = "Remove one configured field from a dictionary tool response."
+
+
+@register_mutation
+class DuplicateToolResult(Mutation):
+    mutation_type = "duplicate_tool_result"
+    description = "Return a deterministic envelope containing the same tool result twice."
+    stable = False
+
+
+@register_mutation
+class PermissionDenied(Mutation):
+    mutation_type = "permission_denied"
+    description = "Raise a permission-denied error before the handler runs."

agentproof/py.typed

@@ -0,0 +1 @@
+

agentproof/pytest_plugin.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+
+def pytest_addoption(parser: Any) -> None:
+    group = parser.getgroup("agentproof")
+    group.addoption(
+        "--agentproof-artifacts",
+        action="store",
+        default=None,
+        help="Directory for AgentProof artifacts during pytest runs.",
+    )
+
+
+def pytest_configure(config: Any) -> None:
+    config.addinivalue_line("markers", "agentproof: AgentProof framework tests")
+    config.addinivalue_line("markers", "live: opt-in live model/provider tests")
+    config.addinivalue_line("markers", "openai: OpenAI Agents SDK integration tests")
+    config.addinivalue_line("markers", "langchain: LangChain/LangGraph integration tests")
+
+
+@pytest.fixture
+def agentproof_artifacts(tmp_path: Path, request: Any) -> Path:
+    configured = request.config.getoption("--agentproof-artifacts")
+    path = Path(configured) if configured else tmp_path / ".agentproof" / "runs"
+    path.mkdir(mode=0o700, parents=True, exist_ok=True)
+    return path
+
+
+@pytest.fixture
+def run_agentproof_suite(agentproof_artifacts: Path) -> Any:
+    async def run(suite: Any, **kwargs: Any) -> Any:
+        kwargs.setdefault("artifacts_dir", agentproof_artifacts)
+        return await suite.run(**kwargs)
+
+    return run

agentproof/replay/player.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import sys
+from contextlib import suppress
+from pathlib import Path
+from types import ModuleType
+
+from agentproof.api import AgentTest
+from agentproof.core.result import SuiteResult
+from agentproof.mutations import mutation_from_spec
+from agentproof.replay.schema import RUN_ARTIFACT_SCHEMA_VERSION, RunArtifact
+
+
+def load_artifact(path: str | Path) -> RunArtifact:
+    artifact = RunArtifact.model_validate_json(Path(path).read_text(encoding="utf-8"))
+    if artifact.schema_version != RUN_ARTIFACT_SCHEMA_VERSION:
+        raise ValueError(
+            f"unsupported artifact schema {artifact.schema_version}; "
+            f"expected {RUN_ARTIFACT_SCHEMA_VERSION}"
+        )
+    return artifact
+
+
+async def replay_artifact(path: str | Path) -> SuiteResult:
+    artifact = load_artifact(path)
+    source_path = artifact.metadata.get("source_path")
+    suite_name = artifact.metadata.get("suite_name")
+    if not isinstance(source_path, str):
+        raise ValueError("artifact does not contain metadata.source_path for deterministic replay")
+    suite = _load_suite(Path(source_path), suite_name if isinstance(suite_name, str) else None)
+    mutation = mutation_from_spec(artifact.mutation) if artifact.mutation else None
+    return await suite.run(
+        scenario=artifact.scenario,
+        mutations=[] if mutation is None else [mutation],
+        seed=artifact.seed,
+        artifacts_dir=Path(path).parent / "replay",
+        store_artifacts=False,
+        source_path=source_path,
+        suite_name=suite_name if isinstance(suite_name, str) else None,
+    )
+
+
+def replay_artifact_sync(path: str | Path) -> SuiteResult:
+    return asyncio.run(replay_artifact(path))
+
+
+def _load_suite(source_path: Path, suite_name: str | None) -> AgentTest:
+    module = _load_module_from_path(source_path)
+    suites = [
+        (name, value)
+        for name, value in vars(module).items()
+        if isinstance(value, AgentTest) and (suite_name is None or name == suite_name)
+    ]
+    if not suites:
+        raise ValueError(f"no AgentTest suite found in {source_path}")
+    return suites[0][1]
+
+
+def _load_module_from_path(source_path: Path) -> ModuleType:
+    module_name = f"agentproof_replay_{abs(hash(source_path))}"
+    spec = importlib.util.spec_from_file_location(module_name, source_path)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"cannot import {source_path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    sys.path.insert(0, str(source_path.parent))
+    try:
+        spec.loader.exec_module(module)
+    finally:
+        with suppress(ValueError):
+            sys.path.remove(str(source_path.parent))
+    return module

agentproof/replay/recorder.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import os
+from contextlib import suppress
+from pathlib import Path
+
+from agentproof.core.redaction import redact
+from agentproof.core.result import RunResult
+from agentproof.replay.schema import RunArtifact
+
+
+def write_repro_artifact(result: RunResult, artifacts_dir: Path) -> Path:
+    run_dir = artifacts_dir / result.run_id
+    run_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
+    artifact = RunArtifact(
+        run_id=result.run_id,
+        scenario=result.scenario,
+        seed=result.seed,
+        mutation=result.mutation,
+        initial_world_snapshot=redact(result.initial_world_snapshot),
+        trace=result.trace,
+        violated_invariants=result.violated_invariants,
+        metadata=redact(result.metadata),
+    )
+    path = run_dir / "repro.json"
+    path.write_text(artifact.model_dump_json(indent=2), encoding="utf-8")
+    with suppress(OSError):
+        os.chmod(path, 0o600)
+    return path

agentproof/replay/schema.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from agentproof.core.faults import MutationSpec
+from agentproof.core.trace import TraceEvent
+
+RUN_ARTIFACT_SCHEMA_VERSION = 1
+
+
+class RunArtifact(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    schema_version: int = RUN_ARTIFACT_SCHEMA_VERSION
+    run_id: str
+    scenario: str
+    seed: int
+    mutation: MutationSpec | None
+    initial_world_snapshot: dict[str, Any]
+    trace: list[TraceEvent]
+    violated_invariants: list[str] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)

agentproof/reporting/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from agentproof.reporting.console import render_suite_result
+from agentproof.reporting.json_report import suite_to_json, write_json_report
+from agentproof.reporting.junit import write_junit_report
+
+__all__ = ["render_suite_result", "suite_to_json", "write_json_report", "write_junit_report"]

agentproof/reporting/console.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from rich.console import Console
+from rich.table import Table
+
+from agentproof.core.result import SuiteResult
+
+
+def render_suite_result(result: SuiteResult, *, console: Console | None = None) -> None:
+    output = console or Console()
+    output.print("[bold]AgentProof[/bold]")
+    output.print(
+        f"{result.total_count} runs, {result.passed_count} passed, {result.failed_count} failed",
+    )
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Run")
+    table.add_column("Status")
+    table.add_column("Severity")
+    table.add_column("Failure")
+    for item in result.results:
+        failure_text = item.error_message or ", ".join(item.violated_invariants)
+        table.add_row(item.name, item.status, item.severity, failure_text)
+    output.print(table)
+    for item in result.failures:
+        output.print(f"\n[bold red]{item.status}[/bold red] {item.name}")
+        if item.invariant_failures:
+            for failure in item.invariant_failures:
+                output.print(f"Invariant violated: {failure.name}")
+                output.print(failure.message)
+        if item.effects:
+            output.print(f"Committed effects: {len(item.effects)}")
+        if item.artifact_path:
+            output.print(f"Reproduce: agentproof replay {item.artifact_path}")
+        output.print(f"Seed: {item.seed}")

agentproof/reporting/json_report.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from agentproof.core.redaction import redact
+from agentproof.core.result import SuiteResult
+
+
+def suite_to_json(result: SuiteResult) -> dict[str, Any]:
+    return redact(
+        {
+            "schema_version": result.schema_version,
+            "run_id": result.run_id,
+            "summary": {
+                "total": result.total_count,
+                "passed": result.passed_count,
+                "failed": result.failed_count,
+            },
+            "results": [
+                {
+                    "name": item.name,
+                    "scenario": item.scenario,
+                    "adapter": item.adapter,
+                    "seed": item.seed,
+                    "mutation": item.mutation.model_dump(mode="json") if item.mutation else None,
+                    "severity": item.severity,
+                    "status": item.status,
+                    "violated_invariants": item.violated_invariants,
+                    "invariant_failures": [
+                        failure.model_dump(mode="json") for failure in item.invariant_failures
+                    ],
+                    "error_message": item.error_message,
+                    "effects": [effect.model_dump(mode="json") for effect in item.effects],
+                    "trace": [event.model_dump(mode="json") for event in item.trace],
+                    "artifact_path": str(item.artifact_path) if item.artifact_path else None,
+                }
+                for item in result.results
+            ],
+            "metadata": result.metadata,
+        }
+    )
+
+
+def write_json_report(result: SuiteResult, path: str | Path) -> None:
+    Path(path).write_text(json.dumps(suite_to_json(result), indent=2), encoding="utf-8")

agentproof/reporting/junit.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+from agentproof.core.result import SuiteResult
+
+
+def write_junit_report(result: SuiteResult, path: str | Path) -> None:
+    testsuite = ET.Element(
+        "testsuite",
+        {
+            "name": "agentproof",
+            "tests": str(result.total_count),
+            "failures": str(
+                len(
+                    [
+                        item
+                        for item in result.results
+                        if item.status in {"INVARIANT_FAILURE", "BASELINE_FAILURE"}
+                    ]
+                )
+            ),
+            "errors": str(
+                len(
+                    [
+                        item
+                        for item in result.results
+                        if item.status
+                        in {"AGENT_ERROR", "TEST_ERROR", "ADAPTER_ERROR", "UNSUPPORTED"}
+                    ]
+                )
+            ),
+            "skipped": str(len([item for item in result.results if item.status == "SKIPPED"])),
+        },
+    )
+    for item in result.results:
+        testcase = ET.SubElement(
+            testsuite,
+            "testcase",
+            {
+                "classname": f"agentproof.{item.scenario}",
+                "name": item.name,
+            },
+        )
+        message = item.error_message or ",".join(item.violated_invariants)
+        body = _trace_body(item)
+        if item.status in {"INVARIANT_FAILURE", "BASELINE_FAILURE"}:
+            failure = ET.SubElement(testcase, "failure", {"message": message or item.status})
+            failure.text = body
+        elif item.status in {"AGENT_ERROR", "TEST_ERROR", "ADAPTER_ERROR", "UNSUPPORTED"}:
+            error = ET.SubElement(testcase, "error", {"message": message or item.status})
+            error.text = body
+        elif item.status == "SKIPPED":
+            ET.SubElement(testcase, "skipped", {"message": message or "skipped"})
+    tree = ET.ElementTree(testsuite)
+    ET.indent(tree, space="  ")
+    tree.write(path, encoding="utf-8", xml_declaration=True)
+
+
+def _trace_body(item: object) -> str:
+    trace = getattr(item, "trace", [])
+    lines = []
+    for event in trace:
+        lines.append(f"{event.seq} {event.kind} {event.name} {event.data}")
+    return "\n".join(lines)

agentproof/tools/definition.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from agentproof.core.effects import EffectDraft
+
+ToolEffect = Literal["read", "write", "delete", "external", "financial", "privileged"]
+ToolHandler = Callable[..., Awaitable[Any] | Any]
+
+
+@dataclass(frozen=True)
+class ToolOutcome:
+    value: Any
+    effects: list[EffectDraft | dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class ToolDefinition:
+    name: str
+    description: str
+    input_schema: dict[str, Any]
+    handler: ToolHandler
+    effect: ToolEffect | None = None
+    idempotent: bool | None = None

agentproof/tools/invocation.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+class ToolClient:
+    def __init__(self, world: Any) -> None:
+        self._world = world
+
+    async def call(self, name: str, **kwargs: Any) -> Any:
+        return await self._world.tools.invoke(name, kwargs)

agentproof/tools/registry.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import inspect
+from typing import Any
+
+from agentproof.core.faults import AgentProofToolError, ToolValidationError
+from agentproof.tools.definition import ToolDefinition, ToolOutcome
+
+
+class ToolRegistry:
+    def __init__(self, world: Any) -> None:
+        self._world = world
+        self._tools: dict[str, ToolDefinition] = {}
+
+    def register(
+        self,
+        *,
+        name: str,
+        description: str,
+        input_schema: dict[str, Any],
+        handler: Any,
+        effect: str | None = None,
+        idempotent: bool | None = None,
+    ) -> ToolDefinition:
+        definition = ToolDefinition(
+            name=name,
+            description=description,
+            input_schema=input_schema,
+            handler=handler,
+            effect=effect,  # type: ignore[arg-type]
+            idempotent=idempotent,
+        )
+        self._tools[name] = definition
+        return definition
+
+    def get(self, name: str) -> ToolDefinition:
+        return self._tools[name]
+
+    def all(self) -> list[ToolDefinition]:
+        return list(self._tools.values())
+
+    async def invoke(self, name: str, arguments: dict[str, Any] | None = None) -> Any:
+        if name not in self._tools:
+            raise KeyError(f"unknown tool: {name}")
+        args = dict(arguments or {})
+        definition = self._tools[name]
+        self._validate(definition, args)
+        invocation_id = self._world.next_id("invocation")
+        self._world.trace.record(
+            "tool_call",
+            name,
+            {"arguments": args, "invocation_id": invocation_id},
+        )
+
+        action = self._world.faults.before_tool_call(
+            tool_name=name,
+            args=args,
+            invocation_id=invocation_id,
+        )
+        if action is not None and action.handled:
+            if action.exception is not None:
+                self._world.trace.record(
+                    "tool_error",
+                    name,
+                    {
+                        "error_type": type(action.exception).__name__,
+                        "message": str(action.exception),
+                        "invocation_id": invocation_id,
+                    },
+                )
+                raise action.exception
+            self._world.trace.record(
+                "tool_result",
+                name,
+                {"result": action.return_value, "invocation_id": invocation_id},
+            )
+            return action.return_value
+
+        try:
+            raw = await self._call_handler(definition, args)
+        except AgentProofToolError:
+            raise
+        except Exception as exc:
+            self._world.trace.record(
+                "tool_error",
+                name,
+                {
+                    "error_type": type(exc).__name__,
+                    "message": str(exc),
+                    "invocation_id": invocation_id,
+                },
+            )
+            raise
+
+        outcome = raw if isinstance(raw, ToolOutcome) else ToolOutcome(value=raw)
+        for draft in outcome.effects:
+            effect = self._world.effects.commit(
+                draft,
+                tool_name=name,
+                invocation_id=invocation_id,
+                committed_at=self._world.clock.now(),
+            )
+            self._world.trace.record(
+                "effect",
+                effect.type,
+                {"effect": effect.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+
+        try:
+            self._world.faults.after_commit_before_response(
+                tool_name=name,
+                invocation_id=invocation_id,
+            )
+        except AgentProofToolError as exc:
+            self._world.trace.record(
+                "tool_error",
+                name,
+                {
+                    "error_type": type(exc).__name__,
+                    "message": str(exc),
+                    "invocation_id": invocation_id,
+                },
+            )
+            raise
+
+        response = self._world.faults.after_tool_response(
+            tool_name=name,
+            invocation_id=invocation_id,
+            response=outcome.value,
+        )
+        self._world.trace.record(
+            "tool_result",
+            name,
+            {"result": response, "invocation_id": invocation_id},
+        )
+        return response
+
+    async def _call_handler(self, definition: ToolDefinition, args: dict[str, Any]) -> Any:
+        signature = inspect.signature(definition.handler)
+        kwargs = dict(args)
+        if "world" in signature.parameters:
+            kwargs["world"] = self._world
+        result = definition.handler(**kwargs)
+        if inspect.isawaitable(result):
+            return await result
+        return result
+
+    @staticmethod
+    def _validate(definition: ToolDefinition, args: dict[str, Any]) -> None:
+        schema = definition.input_schema or {}
+        required = schema.get("required", [])
+        for field in required:
+            if field not in args:
+                raise ToolValidationError(definition.name, f"missing required field: {field}")