agentproof-sim 0.1.1__py3-none-any.whl

agentproof/__init__.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from agentproof.api import AgentTest
+from agentproof.core.effects import Effect, EffectDraft
+from agentproof.core.faults import MutationSpec
+from agentproof.core.invariant import Invariant, invariant
+from agentproof.core.result import RunResult, SuiteResult
+from agentproof.core.scenario import Scenario
+from agentproof.core.trace import TraceEvent
+from agentproof.core.world import World
+from agentproof.mutations.base import Mutation
+
+
+def _distribution_version() -> str:
+    for distribution_name in ("agentproof-sim", "agentproof"):
+        try:
+            return version(distribution_name)
+        except PackageNotFoundError:
+            continue
+    return "0.1.1"
+
+
+__version__ = _distribution_version()
+
+__all__ = [
+    "AgentTest",
+    "Effect",
+    "EffectDraft",
+    "Invariant",
+    "Mutation",
+    "MutationSpec",
+    "RunResult",
+    "Scenario",
+    "SuiteResult",
+    "TraceEvent",
+    "World",
+    "__version__",
+    "invariant",
+]

agentproof/adapters/__init__.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from agentproof.adapters.base import AgentAdapter, AgentRunResult
+from agentproof.adapters.langchain import LangChainAdapter
+from agentproof.adapters.native import NativeAdapter
+from agentproof.adapters.openai_agents import OpenAIAgentsAdapter
+
+__all__ = [
+    "AgentAdapter",
+    "AgentRunResult",
+    "LangChainAdapter",
+    "NativeAdapter",
+    "OpenAIAgentsAdapter",
+]

agentproof/adapters/base.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AgentRunResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    final_output: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    raw_result: Any | None = None
+
+
+class AgentAdapter(Protocol):
+    name: str
+
+    async def run(self, *, world: Any, user_input: str) -> AgentRunResult: ...

agentproof/adapters/langchain.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from pydantic import BaseModel, create_model
+
+from agentproof.adapters.base import AgentRunResult
+
+
+class LangChainAdapter:
+    """Adapter for LangChain v1 / LangGraph tools backed by AgentProof."""
+
+    name = "langchain-langgraph"
+
+    def __init__(self, agent: Any | None = None) -> None:
+        self.agent = agent
+
+    def build_tools(self, world: Any) -> list[Any]:
+        try:
+            from langchain_core.tools import StructuredTool
+        except ImportError as exc:
+            raise RuntimeError("Install agentproof-sim[langchain] to use LangChainAdapter") from exc
+
+        tools = []
+        for definition in world.tools.all():
+            args_schema = _pydantic_model_from_schema(definition.name, definition.input_schema)
+
+            tools.append(
+                StructuredTool.from_function(
+                    coroutine=_make_coroutine(world, definition.name),
+                    name=definition.name,
+                    description=definition.description,
+                    args_schema=args_schema,
+                )
+            )
+        return tools
+
+    def create_agent(self, *, model: Any, world: Any, system_prompt: str | None = None) -> Any:
+        try:
+            from langchain.agents import create_agent
+        except ImportError as exc:
+            raise RuntimeError("Install agentproof-sim[langchain] to use LangChainAdapter") from exc
+        return create_agent(model=model, tools=self.build_tools(world), system_prompt=system_prompt)
+
+    async def run(self, *, world: Any, user_input: str) -> AgentRunResult:
+        if self.agent is None:
+            raise ValueError("LangChainAdapter requires an agent/graph for run()")
+        result = await self.agent.ainvoke({"messages": [{"role": "user", "content": user_input}]})
+        return AgentRunResult(
+            final_output=json.dumps(_json_safe(result), sort_keys=True),
+            metadata={"framework": "langchain-langgraph"},
+            raw_result=result,
+        )
+
+
+def _pydantic_model_from_schema(name: str, schema: dict[str, Any]) -> type[BaseModel]:
+    properties = schema.get("properties", {})
+    required = set(schema.get("required", []))
+    fields: dict[str, tuple[Any, Any]] = {}
+    for field_name, field_schema in properties.items():
+        field_type = _python_type(field_schema)
+        default = ... if field_name in required else None
+        fields[field_name] = (field_type, default)
+    return create_model(f"AgentProof{name.title().replace('_', '')}Input", **fields)  # type: ignore[call-overload]
+
+
+def _make_coroutine(world: Any, tool_name: str) -> Any:
+    async def call(**kwargs: Any) -> Any:
+        return await world.tools.invoke(tool_name, kwargs)
+
+    return call
+
+
+def _python_type(schema: dict[str, Any]) -> Any:
+    schema_type = schema.get("type")
+    if schema_type == "integer":
+        return int
+    if schema_type == "number":
+        return float
+    if schema_type == "boolean":
+        return bool
+    if schema_type == "array":
+        return list
+    if schema_type == "object":
+        return dict
+    return str
+
+
+def _json_safe(value: Any) -> Any:
+    try:
+        json.dumps(value)
+    except TypeError:
+        return str(value)
+    return value

agentproof/adapters/native.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import inspect
+from typing import Any
+
+from agentproof.adapters.base import AgentRunResult
+from agentproof.tools.invocation import ToolClient
+
+
+class NativeAdapter:
+    name = "native"
+
+    def __init__(self, agent: Any | None = None) -> None:
+        self.agent = agent
+
+    async def run(self, *, world: Any, user_input: str) -> AgentRunResult:
+        if self.agent is None:
+            raise ValueError("NativeAdapter requires an agent callable")
+        tools = ToolClient(world)
+        result = self.agent(user_input, tools)
+        if inspect.isawaitable(result):
+            result = await result
+        return AgentRunResult(final_output=None if result is None else str(result))

agentproof/adapters/openai_agents.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from agentproof.adapters.base import AgentRunResult
+
+
+class OpenAIAgentsAdapter:
+    """Adapter for the real OpenAI Agents SDK boundary."""
+
+    name = "openai-agents"
+
+    def __init__(self, agent: Any | None = None) -> None:
+        self.agent = agent
+
+    def build_function_tools(self, world: Any) -> list[Any]:
+        try:
+            from agents import FunctionTool
+        except ImportError as exc:
+            raise RuntimeError("Install agentproof-sim[openai] to use OpenAIAgentsAdapter") from exc
+
+        wrapped = []
+        for definition in world.tools.all():
+
+            async def on_invoke_tool(
+                ctx: Any, input: str, *, tool_name: str = definition.name
+            ) -> Any:
+                del ctx
+                payload = json.loads(input or "{}")
+                return await world.tools.invoke(tool_name, payload)
+
+            wrapped.append(
+                FunctionTool(
+                    name=definition.name,
+                    description=definition.description,
+                    params_json_schema=definition.input_schema,
+                    on_invoke_tool=on_invoke_tool,
+                    strict_json_schema=False,
+                )
+            )
+        return wrapped
+
+    async def run(self, *, world: Any, user_input: str) -> AgentRunResult:
+        if self.agent is None:
+            raise ValueError("OpenAIAgentsAdapter requires an Agents SDK Agent for run()")
+        try:
+            from agents import Runner
+        except ImportError as exc:
+            raise RuntimeError("Install agentproof-sim[openai] to use OpenAIAgentsAdapter") from exc
+
+        original_tools = list(getattr(self.agent, "tools", []))
+        self.agent.tools = self.build_function_tools(world)
+        try:
+            result = await Runner.run(self.agent, user_input)
+        finally:
+            self.agent.tools = original_tools
+        return AgentRunResult(
+            final_output=str(getattr(result, "final_output", "")),
+            metadata={"framework": "openai-agents"},
+            raw_result=result,
+        )

agentproof/api.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+from pathlib import Path
+from typing import Any
+
+from agentproof.adapters.native import NativeAdapter
+from agentproof.core.invariant import Invariant
+from agentproof.core.result import SuiteResult
+from agentproof.core.runner import ScenarioRunner
+from agentproof.core.scenario import Scenario, ScenarioFunc
+from agentproof.mutations.base import Mutation
+
+
+class AgentTest:
+    def __init__(
+        self,
+        *,
+        agent: Any,
+        adapter: str | Any = "native",
+        mutations: list[Mutation] | None = None,
+        invariants: list[Invariant] | None = None,
+        name: str | None = None,
+    ) -> None:
+        self.name = name or "agentproof_suite"
+        self.agent = agent
+        self.adapter = self._coerce_adapter(adapter, agent)
+        self.mutations = list(mutations or [])
+        self.invariants = list(invariants or [])
+        self.scenarios: list[Scenario] = []
+
+    def scenario(self, func: ScenarioFunc | None = None, *, name: str | None = None) -> Any:
+        def wrap(inner: ScenarioFunc) -> ScenarioFunc:
+            self.scenarios.append(Scenario(name=name or inner.__name__, func=inner))
+            return inner
+
+        if func is None:
+            return wrap
+        return wrap(func)
+
+    def add_invariant(self, item: Invariant) -> Invariant:
+        self.invariants.append(item)
+        return item
+
+    async def run(
+        self,
+        *,
+        scenario: str | None = None,
+        mutations: list[Mutation] | None = None,
+        mutation_name: str | None = None,
+        seed: int = 42,
+        artifacts_dir: str | Path = ".agentproof/runs",
+        store_artifacts: bool = True,
+        source_path: str | None = None,
+        suite_name: str | None = None,
+    ) -> SuiteResult:
+        runner = ScenarioRunner(
+            adapter=self.adapter,
+            scenarios=self.scenarios,
+            invariants=self._all_invariants(),
+            mutations=self.mutations,
+        )
+        return await runner.run(
+            scenario=scenario,
+            mutations=mutations,
+            mutation_name=mutation_name,
+            seed=seed,
+            artifacts_dir=Path(artifacts_dir),
+            store_artifacts=store_artifacts,
+            source_path=source_path,
+            suite_name=suite_name or self.name,
+        )
+
+    def run_sync(self, **kwargs: Any) -> SuiteResult:
+        return asyncio.run(self.run(**kwargs))
+
+    def _all_invariants(self) -> list[Invariant]:
+        discovered: list[Invariant] = []
+        for scenario in self.scenarios:
+            module = inspect.getmodule(scenario.func)
+            if module is None:
+                continue
+            for value in vars(module).values():
+                if isinstance(value, Invariant) and value not in discovered:
+                    discovered.append(value)
+        result = list(self.invariants)
+        for item in discovered:
+            if item not in result:
+                result.append(item)
+        return result
+
+    @staticmethod
+    def _coerce_adapter(adapter: str | Any, agent: Any) -> Any:
+        if adapter == "native":
+            return NativeAdapter(agent)
+        if isinstance(adapter, NativeAdapter) and adapter.agent is None:
+            adapter.agent = agent
+        return adapter

agentproof/cli.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import importlib.util
+import sys
+from contextlib import suppress
+from pathlib import Path
+from types import ModuleType
+
+import click
+from rich.console import Console
+
+from agentproof.api import AgentTest
+from agentproof.core.faults import MutationSpec, Severity
+from agentproof.mutations import MUTATION_TYPES, mutation_from_spec
+from agentproof.replay.player import replay_artifact_sync
+from agentproof.reporting.console import render_suite_result
+from agentproof.reporting.json_report import write_json_report
+from agentproof.reporting.junit import write_junit_report
+
+
+@click.group()
+def main() -> None:
+    """AgentProof command line interface."""
+
+
+@main.command()
+@click.option("--force", is_flag=True, help="Overwrite existing scaffold files.")
+def init(force: bool) -> None:
+    files = {
+        "agentproof.toml": '[agentproof]\nseed = 42\nrepetitions = 1\nfail_on = "high"\n',
+        "tests/agentproof/.gitkeep": "",
+    }
+    for raw_path, content in files.items():
+        path = Path(raw_path)
+        if path.exists() and not force:
+            raise click.ClickException(f"{path} already exists; use --force to overwrite")
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content, encoding="utf-8")
+    Path(".agentproof").mkdir(mode=0o700, exist_ok=True)
+    click.echo("Initialized AgentProof files.")
+
+
+@main.command(name="mutations")
+def list_mutations() -> None:
+    for name, cls in sorted(MUTATION_TYPES.items()):
+        stability = "stable" if cls.stable else "experimental"
+        click.echo(f"{name}\t{stability}\t{cls.description}")
+
+
+@main.command()
+@click.argument("path", required=False, default=".")
+@click.option("--scenario", "scenario_name", default=None, help="Run only one scenario.")
+@click.option(
+    "--mutation",
+    "mutation_value",
+    default=None,
+    help="Run only one mutation, optionally type:target.",
+)
+@click.option("--seed", default=42, type=int, show_default=True)
+@click.option(
+    "--fail-on",
+    type=click.Choice(["low", "medium", "high", "critical"]),
+    default="high",
+    show_default=True,
+)
+@click.option("--json", "json_path", type=click.Path(dir_okay=False, path_type=Path))
+@click.option("--junit", "junit_path", type=click.Path(dir_okay=False, path_type=Path))
+@click.option("--no-color", is_flag=True)
+def run(
+    path: str,
+    scenario_name: str | None,
+    mutation_value: str | None,
+    seed: int,
+    fail_on: Severity,
+    json_path: Path | None,
+    junit_path: Path | None,
+    no_color: bool,
+) -> None:
+    suites = _load_suites(Path(path))
+    if not suites:
+        raise click.ClickException(f"no AgentTest suites found under {path}")
+    mutation_override = [_parse_mutation(mutation_value)] if mutation_value else None
+    aggregate = None
+    exit_code = 0
+    for source_path, suite_name, suite in suites:
+        result = suite.run_sync(
+            scenario=scenario_name,
+            mutations=mutation_override,
+            mutation_name=None,
+            seed=seed,
+            source_path=str(source_path),
+            suite_name=suite_name,
+        )
+        aggregate = result
+        render_suite_result(result, console=Console(no_color=no_color))
+        exit_code = max(exit_code, result.exit_code(fail_on))
+    if aggregate is not None and json_path is not None:
+        write_json_report(aggregate, json_path)
+    if aggregate is not None and junit_path is not None:
+        write_junit_report(aggregate, junit_path)
+    raise SystemExit(exit_code)
+
+
+@main.command()
+@click.argument("artifact", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option(
+    "--fail-on",
+    type=click.Choice(["low", "medium", "high", "critical"]),
+    default="high",
+    show_default=True,
+)
+def replay(artifact: Path, fail_on: Severity) -> None:
+    result = replay_artifact_sync(artifact)
+    render_suite_result(result)
+    raise SystemExit(result.exit_code(fail_on))
+
+
+@main.command()
+def doctor() -> None:
+    click.echo(f"Python: {sys.version.split()[0]}")
+    for module in ("agents", "langchain", "langgraph"):
+        try:
+            imported = __import__(module)
+        except Exception as exc:
+            click.echo(f"{module}: unavailable ({exc})")
+        else:
+            version = getattr(imported, "__version__", "installed")
+            click.echo(f"{module}: {version}")
+    Path(".agentproof").mkdir(mode=0o700, exist_ok=True)
+    click.echo(".agentproof: writable")
+
+
+def _parse_mutation(value: str) -> object:
+    mutation_type, _, target = value.partition(":")
+    severity: Severity = (
+        "high" if mutation_type in {"timeout_after_commit", "duplicate_user_request"} else "medium"
+    )
+    spec = MutationSpec(type=mutation_type, target=target or None, severity=severity)
+    return mutation_from_spec(spec)
+
+
+def _load_suites(path: Path) -> list[tuple[Path, str, AgentTest]]:
+    source_files = [path] if path.is_file() else sorted(path.rglob("*.py"))
+    suites: list[tuple[Path, str, AgentTest]] = []
+    for source in source_files:
+        if any(part.startswith(".") for part in source.parts):
+            continue
+        module = _load_module(source)
+        for name, value in vars(module).items():
+            if isinstance(value, AgentTest):
+                suites.append((source, name, value))
+    return suites
+
+
+def _load_module(path: Path) -> ModuleType:
+    module_name = f"agentproof_cli_{abs(hash(path))}"
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if spec is None or spec.loader is None:
+        raise click.ClickException(f"cannot import {path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    sys.path.insert(0, str(path.parent))
+    try:
+        spec.loader.exec_module(module)
+    finally:
+        with suppress(ValueError):
+            sys.path.remove(str(path.parent))
+    return module
+
+
+if __name__ == "__main__":
+    main()

agentproof/core/clock.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+
+class VirtualClock:
+    def __init__(self, start: float = 0.0) -> None:
+        self._now = float(start)
+
+    def now(self) -> float:
+        return self._now
+
+    def advance(self, seconds: float) -> float:
+        if seconds < 0:
+            raise ValueError("seconds must be non-negative")
+        self._now += float(seconds)
+        return self._now
+
+    def set(self, timestamp: float) -> None:
+        self._now = float(timestamp)

agentproof/core/effects.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class EffectDraft(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    type: str
+    data: dict[str, Any] = Field(default_factory=dict)
+    operation: str | None = None
+    resource: str | None = None
+    idempotency_key: str | None = None
+
+
+class Effect(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    type: str
+    tool_name: str
+    operation: str | None = None
+    resource: str | None = None
+    data: dict[str, Any]
+    committed_at: float
+    idempotency_key: str | None = None
+    invocation_id: str
+
+
+class EffectLedger:
+    """Records simulated side effects explicitly committed by virtual tool handlers."""
+
+    def __init__(self) -> None:
+        self._effects: list[Effect] = []
+
+    def commit(
+        self,
+        draft: EffectDraft | dict[str, Any],
+        *,
+        tool_name: str,
+        invocation_id: str,
+        committed_at: float,
+    ) -> Effect:
+        draft_model = draft if isinstance(draft, EffectDraft) else EffectDraft.model_validate(draft)
+        effect = Effect(
+            id=f"eff_{len(self._effects) + 1:03d}",
+            type=draft_model.type,
+            tool_name=tool_name,
+            operation=draft_model.operation,
+            resource=draft_model.resource,
+            data=draft_model.data,
+            committed_at=committed_at,
+            idempotency_key=draft_model.idempotency_key,
+            invocation_id=invocation_id,
+        )
+        self._effects.append(effect)
+        return effect
+
+    def append(self, effect: Effect) -> None:
+        self._effects.append(effect)
+
+    def all(self) -> list[Effect]:
+        return list(self._effects)
+
+    def filter(
+        self,
+        *,
+        type: str | None = None,
+        where: dict[str, Any] | None = None,
+        tool_name: str | None = None,
+    ) -> list[Effect]:
+        effects = self._effects
+        if type is not None:
+            effects = [effect for effect in effects if effect.type == type]
+        if tool_name is not None:
+            effects = [effect for effect in effects if effect.tool_name == tool_name]
+        if where:
+            effects = [
+                effect
+                for effect in effects
+                if all(effect.data.get(key) == value for key, value in where.items())
+            ]
+        return list(effects)
+
+    def sum(
+        self,
+        *,
+        type: str,
+        field: str,
+        where: dict[str, Any] | None = None,
+    ) -> float:
+        total = 0.0
+        for effect in self.filter(type=type, where=where):
+            value = effect.data.get(field, 0)
+            if isinstance(value, int | float):
+                total += float(value)
+        return total
+
+    def duplicate_keys(self, *, type: str, key_fields: list[str]) -> dict[tuple[Any, ...], int]:
+        counts: dict[tuple[Any, ...], int] = {}
+        for effect in self.filter(type=type):
+            key = tuple(effect.data.get(field) for field in key_fields)
+            counts[key] = counts.get(key, 0) + 1
+        return {key: count for key, count in counts.items() if count > 1}
+
+    def snapshot(self) -> list[dict[str, Any]]:
+        return [effect.model_dump(mode="json") for effect in self._effects]