agentproof-sim 0.1.1__py3-none-any.whl

agentproof/core/runner.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import inspect
+import uuid
+from pathlib import Path
+from typing import Any
+
+from agentproof.adapters.base import AgentRunResult
+from agentproof.core.faults import AgentProofError, MutationSpec, UnsupportedMutationError
+from agentproof.core.invariant import Invariant, evaluate_invariants
+from agentproof.core.result import InvariantFailure, RunResult, SuiteResult
+from agentproof.core.scenario import Scenario
+from agentproof.core.world import World
+from agentproof.mutations.base import Mutation
+from agentproof.replay.recorder import write_repro_artifact
+
+
+class ScenarioRunner:
+    def __init__(
+        self,
+        *,
+        adapter: Any,
+        scenarios: list[Scenario],
+        invariants: list[Invariant],
+        mutations: list[Mutation],
+    ) -> None:
+        self.adapter = adapter
+        self.scenarios = scenarios
+        self.invariants = invariants
+        self.mutations = mutations
+
+    async def run(
+        self,
+        *,
+        scenario: str | None = None,
+        mutations: list[Mutation] | None = None,
+        mutation_name: str | None = None,
+        seed: int = 42,
+        artifacts_dir: Path = Path(".agentproof/runs"),
+        store_artifacts: bool = True,
+        source_path: str | None = None,
+        suite_name: str | None = None,
+    ) -> SuiteResult:
+        selected_scenarios = [item for item in self.scenarios if scenario in (None, item.name)]
+        if not selected_scenarios:
+            raise ValueError(f"unknown scenario: {scenario}")
+
+        suite_run_id = f"ap_{uuid.uuid4().hex[:12]}"
+        results: list[RunResult] = []
+        for scenario_item in selected_scenarios:
+            baseline = await self._run_one(
+                scenario=scenario_item,
+                mutation=None,
+                seed=seed,
+                suite_run_id=suite_run_id,
+                artifacts_dir=artifacts_dir,
+                store_artifacts=store_artifacts,
+                source_path=source_path,
+                suite_name=suite_name,
+            )
+            results.append(baseline)
+            if baseline.status != "PASS":
+                continue
+
+            candidate_mutations = mutations if mutations is not None else self.mutations
+            for mutation in candidate_mutations:
+                if mutation_name is not None and mutation.spec.type != mutation_name:
+                    continue
+                expanded = await self._expand_mutation(scenario_item, mutation, seed)
+                for item in expanded:
+                    results.append(
+                        await self._run_one(
+                            scenario=scenario_item,
+                            mutation=item,
+                            seed=seed,
+                            suite_run_id=suite_run_id,
+                            artifacts_dir=artifacts_dir,
+                            store_artifacts=store_artifacts,
+                            source_path=source_path,
+                            suite_name=suite_name,
+                        )
+                    )
+        return SuiteResult(
+            run_id=suite_run_id,
+            results=results,
+            metadata={"source_path": source_path, "suite_name": suite_name},
+        )
+
+    async def _expand_mutation(
+        self,
+        scenario: Scenario,
+        mutation: Mutation,
+        seed: int,
+    ) -> list[Mutation]:
+        if mutation.target is not None or mutation.spec.type in {
+            "duplicate_user_request",
+            "delayed_event",
+            "duplicate_event",
+            "reorder_tool_results",
+        }:
+            return [mutation]
+        world = World(seed=seed)
+        await _maybe_await(scenario.func(world))
+        tools = world.tools.all()
+        if not tools:
+            return [mutation]
+        result: list[Mutation] = []
+        for tool in tools:
+            if mutation.spec.type == "timeout_after_commit" and tool.effect not in {
+                "write",
+                "delete",
+                "external",
+                "financial",
+                "privileged",
+            }:
+                continue
+            result.append(
+                type(mutation)(
+                    target=tool.name,
+                    occurrence=mutation.occurrence,
+                    params=mutation.params,
+                    severity=mutation.severity,
+                )
+            )
+        return result or [mutation]
+
+    async def _run_one(
+        self,
+        *,
+        scenario: Scenario,
+        mutation: Mutation | None,
+        seed: int,
+        suite_run_id: str,
+        artifacts_dir: Path,
+        store_artifacts: bool,
+        source_path: str | None,
+        suite_name: str | None,
+    ) -> RunResult:
+        world = World(seed=seed, metadata={"source_path": source_path, "suite_name": suite_name})
+        run_id = world.next_id("run")
+        run_id = f"{suite_run_id}_{run_id}"
+        status = "PASS"
+        output: str | None = None
+        error_message: str | None = None
+        invariant_failures: list[InvariantFailure] = []
+        invariant_errors: list[InvariantFailure] = []
+        initial_snapshot: dict[str, Any] = {}
+
+        try:
+            await _maybe_await(scenario.func(world))
+            initial_snapshot = world.snapshot()
+            if mutation is not None:
+                mutation.install(world)
+            inputs = world.faults.prepare_user_inputs(world.user_inputs)
+            for user_input in inputs:
+                adapter_result = await self.adapter.run(world=world, user_input=user_input)
+                output = _stringify_output(adapter_result)
+                world.trace.record(
+                    "agent_output",
+                    getattr(self.adapter, "name", type(self.adapter).__name__),
+                    {"output": output, "metadata": getattr(adapter_result, "metadata", {})},
+                )
+        except UnsupportedMutationError as exc:
+            status = "UNSUPPORTED"
+            error_message = str(exc)
+        except AgentProofError as exc:
+            status = "AGENT_ERROR"
+            error_message = str(exc)
+        except Exception as exc:
+            status = "AGENT_ERROR"
+            error_message = f"{type(exc).__name__}: {exc}"
+
+        if status == "PASS":
+            invariant_failures, invariant_errors = await evaluate_invariants(self.invariants, world)
+            if invariant_errors:
+                status = "TEST_ERROR"
+                error_message = invariant_errors[0].message
+            elif invariant_failures:
+                status = "BASELINE_FAILURE" if mutation is None else "INVARIANT_FAILURE"
+                error_message = invariant_failures[0].message
+
+        mutation_spec: MutationSpec | None = mutation.spec if mutation is not None else None
+        result = RunResult(
+            run_id=run_id,
+            scenario=scenario.name,
+            adapter=getattr(self.adapter, "name", type(self.adapter).__name__),
+            seed=seed,
+            mutation=mutation_spec,
+            severity=mutation_spec.severity if mutation_spec else "high",
+            status=status,  # type: ignore[arg-type]
+            final_output=output,
+            trace=world.trace.all(),
+            effects=world.effects.all(),
+            violated_invariants=[failure.name for failure in invariant_failures],
+            invariant_failures=invariant_failures + invariant_errors,
+            error_message=error_message,
+            initial_world_snapshot=initial_snapshot,
+            metadata={"source_path": source_path, "suite_name": suite_name},
+        )
+        if store_artifacts and result.failed:
+            result.artifact_path = write_repro_artifact(result, artifacts_dir)
+        return result
+
+
+async def _maybe_await(value: Any) -> Any:
+    if inspect.isawaitable(value):
+        return await value
+    return value
+
+
+def _stringify_output(adapter_result: AgentRunResult | Any) -> str | None:
+    if isinstance(adapter_result, AgentRunResult):
+        return adapter_result.final_output
+    if adapter_result is None:
+        return None
+    return str(adapter_result)

agentproof/core/scenario.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+ScenarioFunc = Callable[[Any], Awaitable[None] | None]
+
+
+@dataclass(frozen=True)
+class Scenario:
+    name: str
+    func: ScenarioFunc

agentproof/core/trace.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from agentproof.core.redaction import redact
+
+TraceKind = Literal[
+    "user_input",
+    "tool_call",
+    "tool_result",
+    "tool_error",
+    "effect",
+    "fault",
+    "state_change",
+    "event_scheduled",
+    "event_delivered",
+    "agent_output",
+    "invariant_pass",
+    "invariant_fail",
+]
+
+
+class TraceEvent(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    seq: int
+    timestamp: float
+    kind: TraceKind
+    name: str
+    data: dict[str, Any] = Field(default_factory=dict)
+
+
+class TraceRecorder:
+    def __init__(self, *, clock: Any) -> None:
+        self._clock = clock
+        self._events: list[TraceEvent] = []
+
+    def record(self, kind: TraceKind, name: str, data: dict[str, Any] | None = None) -> TraceEvent:
+        event = TraceEvent(
+            seq=len(self._events) + 1,
+            timestamp=float(self._clock.now()),
+            kind=kind,
+            name=name,
+            data=redact(data or {}),
+        )
+        self._events.append(event)
+        return event
+
+    def all(self) -> list[TraceEvent]:
+        return list(self._events)
+
+    def clear(self) -> None:
+        self._events.clear()
+
+    def snapshot(self) -> list[dict[str, Any]]:
+        return [event.model_dump(mode="json") for event in self._events]

agentproof/core/world.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import copy
+import random
+from collections.abc import Iterator, MutableMapping
+from typing import Any
+
+from agentproof.core.clock import VirtualClock
+from agentproof.core.effects import EffectLedger
+from agentproof.core.events import EventQueue
+from agentproof.core.faults import FaultController
+from agentproof.core.redaction import redact
+from agentproof.core.trace import TraceRecorder
+
+
+class StateStore(MutableMapping[str, Any]):
+    def __init__(self, world: World, data: dict[str, Any] | None = None) -> None:
+        object.__setattr__(self, "_world", world)
+        object.__setattr__(self, "_data", copy.deepcopy(data or {}))
+        object.__setattr__(self, "_trace_enabled", True)
+
+    def __getitem__(self, key: str) -> Any:
+        return self._data[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self._data[key] = value
+        if self._trace_enabled:
+            self._world.trace.record(
+                "state_change", key, {"path": [key], "value": copy.deepcopy(value)}
+            )
+
+    def __delitem__(self, key: str) -> None:
+        del self._data[key]
+        if self._trace_enabled:
+            self._world.trace.record("state_change", key, {"path": [key], "deleted": True})
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._data)
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    def __getattr__(self, key: str) -> Any:
+        try:
+            return self._data[key]
+        except KeyError as exc:
+            raise AttributeError(key) from exc
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        if key.startswith("_"):
+            object.__setattr__(self, key, value)
+        else:
+            self[key] = value
+
+    def snapshot(self) -> dict[str, Any]:
+        return copy.deepcopy(self._data)
+
+    def replace(self, data: dict[str, Any]) -> None:
+        self._trace_enabled = False
+        try:
+            self._data.clear()
+            self._data.update(copy.deepcopy(data))
+        finally:
+            self._trace_enabled = True
+
+
+class World:
+    """Stateful simulated environment exposed to an agent through virtual tools."""
+
+    def __init__(self, *, seed: int = 0, metadata: dict[str, Any] | None = None) -> None:
+        self.seed = seed
+        self.random = random.Random(seed)
+        self.clock = VirtualClock()
+        self.trace = TraceRecorder(clock=self.clock)
+        self.state = StateStore(self)
+        self.effects = EffectLedger()
+        self.faults = FaultController(self)
+        self.events = EventQueue(self)
+        self.metadata: dict[str, Any] = dict(metadata or {})
+        self.user_inputs: list[str] = []
+        self._id_counters: dict[str, int] = {}
+
+        from agentproof.tools.registry import ToolRegistry
+
+        self.tools = ToolRegistry(self)
+
+    def input(self, text: str) -> None:
+        self.user_inputs.append(text)
+        self.trace.record("user_input", "user", {"text": text})
+
+    def next_id(self, prefix: str) -> str:
+        self._id_counters[prefix] = self._id_counters.get(prefix, 0) + 1
+        return f"{prefix}_{self._id_counters[prefix]:03d}"
+
+    def snapshot(self) -> dict[str, Any]:
+        return {
+            "seed": self.seed,
+            "clock": self.clock.now(),
+            "state": self.state.snapshot(),
+            "inputs": copy.deepcopy(self.user_inputs),
+            "effects": self.effects.snapshot(),
+            "metadata": redact(copy.deepcopy(self.metadata)),
+        }
+
+    def clone_from_initial(self) -> World:
+        cloned = World(seed=self.seed, metadata=copy.deepcopy(self.metadata))
+        cloned.state.replace(self.state.snapshot())
+        cloned.user_inputs = copy.deepcopy(self.user_inputs)
+        return cloned
+
+    def set_path(self, path: list[Any], value: Any) -> None:
+        if not path:
+            raise ValueError("path must not be empty")
+        cursor: Any = self.state._data
+        for key in path[:-1]:
+            cursor = cursor.setdefault(key, {}) if isinstance(cursor, dict) else cursor[key]
+        final_key = path[-1]
+        if isinstance(cursor, dict):
+            cursor[final_key] = value
+        else:
+            cursor[final_key] = value
+        self.trace.record(
+            "state_change",
+            ".".join(str(item) for item in path),
+            {"path": path, "value": copy.deepcopy(value)},
+        )

agentproof/invariants/__init__.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from agentproof.invariants.builtin import (
+    at_most_once,
+    forbid_resource,
+    max_effect_sum,
+    max_tool_calls,
+    never_call,
+    no_duplicate_effects,
+    requires_before,
+)
+
+__all__ = [
+    "at_most_once",
+    "forbid_resource",
+    "max_effect_sum",
+    "max_tool_calls",
+    "never_call",
+    "no_duplicate_effects",
+    "requires_before",
+]

agentproof/invariants/builtin.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import fnmatch
+from typing import Any
+
+from agentproof.core.invariant import Invariant, invariant
+
+
+def at_most_once(tool_name: str, *, severity: str = "medium") -> Invariant:
+    return max_tool_calls(tool_name, 1, severity=severity)
+
+
+def max_tool_calls(tool_name: str, count: int, *, severity: str = "medium") -> Invariant:
+    @invariant(name=f"max_tool_calls_{tool_name}_{count}", severity=severity)
+    def check(world: Any) -> None:
+        observed = len(
+            [
+                event
+                for event in world.trace.all()
+                if event.kind == "tool_call" and event.name == tool_name
+            ]
+        )
+        assert observed <= count, f"{tool_name} called {observed} times, expected <= {count}"
+
+    return check
+
+
+def never_call(tool_name: str, *, severity: str = "high") -> Invariant:
+    @invariant(name=f"never_call_{tool_name}", severity=severity)
+    def check(world: Any) -> None:
+        observed = len(
+            [
+                event
+                for event in world.trace.all()
+                if event.kind == "tool_call" and event.name == tool_name
+            ]
+        )
+        assert observed == 0, f"{tool_name} was called {observed} times"
+
+    return check
+
+
+def requires_before(required_tool: str, action_tool: str, *, severity: str = "high") -> Invariant:
+    @invariant(name=f"requires_{required_tool}_before_{action_tool}", severity=severity)
+    def check(world: Any) -> None:
+        required_seen = False
+        for event in world.trace.all():
+            if event.kind == "tool_call" and event.name == required_tool:
+                required_seen = True
+            if event.kind == "tool_call" and event.name == action_tool:
+                assert required_seen, f"{action_tool} called before {required_tool}"
+
+    return check
+
+
+def max_effect_sum(
+    effect_type: str,
+    field: str,
+    maximum: float,
+    *,
+    where: dict[str, Any] | None = None,
+    severity: str = "high",
+) -> Invariant:
+    @invariant(name=f"max_effect_sum_{effect_type}_{field}", severity=severity)
+    def check(world: Any) -> None:
+        observed = world.effects.sum(type=effect_type, field=field, where=where)
+        assert observed <= maximum, f"{field} sum {observed:.2f}, expected <= {maximum:.2f}"
+
+    return check
+
+
+def no_duplicate_effects(
+    effect_type: str,
+    *,
+    key_fields: list[str],
+    severity: str = "high",
+) -> Invariant:
+    @invariant(name=f"no_duplicate_effects_{effect_type}", severity=severity)
+    def check(world: Any) -> None:
+        duplicates = world.effects.duplicate_keys(type=effect_type, key_fields=key_fields)
+        assert not duplicates, f"duplicate {effect_type} effects for keys {duplicates}"
+
+    return check
+
+
+def forbid_resource(resource_pattern: str, *, severity: str = "critical") -> Invariant:
+    @invariant(name=f"forbid_resource_{resource_pattern}", severity=severity)
+    def check(world: Any) -> None:
+        forbidden = [
+            effect.resource
+            for effect in world.effects.all()
+            if effect.resource and fnmatch.fnmatch(effect.resource, resource_pattern)
+        ]
+        assert not forbidden, f"forbidden resources affected: {forbidden}"
+
+    return check

agentproof/invariants/temporal.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from agentproof.invariants.builtin import requires_before
+
+__all__ = ["requires_before"]

agentproof/mutations/__init__.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from agentproof.mutations.base import MUTATION_TYPES, Mutation, mutation_from_spec
+from agentproof.mutations.duplication import DuplicateEvent, DuplicateUserRequest
+from agentproof.mutations.ordering import ReorderToolResults
+from agentproof.mutations.state import StaleState, StateChangedAfterRead
+from agentproof.mutations.timing import DelayedEvent
+from agentproof.mutations.tool_faults import (
+    DuplicateToolResult,
+    MalformedResponse,
+    MissingField,
+    PermissionDenied,
+    RateLimited,
+    TimeoutAfterCommit,
+    ToolError,
+    ToolLatency,
+    ToolTimeout,
+)
+
+
+def standard_reliability_pack() -> list[Mutation]:
+    """Return stable single-fault mutations used by the default deterministic pack."""
+
+    return [
+        ToolTimeout(),
+        TimeoutAfterCommit(severity="high"),
+        ToolError(),
+        ToolLatency(params={"seconds": 1.0}),
+        RateLimited(),
+        MalformedResponse(),
+        MissingField(params={"field": "status"}),
+        DuplicateUserRequest(severity="high"),
+        StaleState(params={"value": {}}),
+        StateChangedAfterRead(params={"path": ["agentproof", "state_changed"], "value": True}),
+        PermissionDenied(),
+        DelayedEvent(),
+        DuplicateEvent(),
+    ]
+
+
+__all__ = [
+    "MUTATION_TYPES",
+    "DelayedEvent",
+    "DuplicateEvent",
+    "DuplicateToolResult",
+    "DuplicateUserRequest",
+    "MalformedResponse",
+    "MissingField",
+    "Mutation",
+    "PermissionDenied",
+    "RateLimited",
+    "ReorderToolResults",
+    "StaleState",
+    "StateChangedAfterRead",
+    "TimeoutAfterCommit",
+    "ToolError",
+    "ToolLatency",
+    "ToolTimeout",
+    "mutation_from_spec",
+    "standard_reliability_pack",
+]

agentproof/mutations/base.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any, ClassVar
+
+from agentproof.core.faults import MutationSpec, Severity
+
+
+@dataclass(frozen=True)
+class Mutation:
+    target: str | None = None
+    occurrence: int | None = 1
+    params: Mapping[str, Any] = field(default_factory=dict)
+    severity: Severity = "medium"
+
+    mutation_type: ClassVar[str] = "mutation"
+    description: ClassVar[str] = "Generic mutation."
+    stable: ClassVar[bool] = True
+
+    @property
+    def spec(self) -> MutationSpec:
+        return MutationSpec(
+            type=self.mutation_type,
+            target=self.target,
+            occurrence=self.occurrence,
+            params=dict(self.params),
+            severity=self.severity,
+        )
+
+    def install(self, world: Any) -> None:
+        world.faults.install(self.spec)
+
+
+def mutation_from_spec(spec: MutationSpec) -> Mutation:
+    cls = MUTATION_TYPES.get(spec.type)
+    if cls is None:
+        return Mutation(
+            target=spec.target,
+            occurrence=spec.occurrence,
+            params=spec.params,
+            severity=spec.severity,
+        )
+    return cls(
+        target=spec.target,
+        occurrence=spec.occurrence,
+        params=spec.params,
+        severity=spec.severity,
+    )
+
+
+MUTATION_TYPES: dict[str, type[Mutation]] = {}
+
+
+def register_mutation(cls: type[Mutation]) -> type[Mutation]:
+    MUTATION_TYPES[cls.mutation_type] = cls
+    return cls

agentproof/mutations/duplication.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from agentproof.mutations.base import Mutation, register_mutation
+
+
+@register_mutation
+class DuplicateUserRequest(Mutation):
+    mutation_type = "duplicate_user_request"
+    description = "Deliver the same user input to the adapter twice in one run."
+
+
+@register_mutation
+class DuplicateEvent(Mutation):
+    mutation_type = "duplicate_event"
+    description = "Deliver a scheduled event more than once."

agentproof/mutations/ordering.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from agentproof.mutations.base import Mutation, register_mutation
+
+
+@register_mutation
+class ReorderToolResults(Mutation):
+    mutation_type = "reorder_tool_results"
+    description = "Experimental placeholder for controlled parallel result reordering."
+    stable = False