agentproof-sim 0.1.1__py3-none-any.whl

agentproof/core/events.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import copy
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ScheduledEvent(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    name: str
+    payload: dict[str, Any] = Field(default_factory=dict)
+    deliver_at: float
+    delivered: bool = False
+
+
+class EventQueue:
+    def __init__(self, world: Any) -> None:
+        self._world = world
+        self._events: list[ScheduledEvent] = []
+
+    def schedule(self, *, name: str, payload: dict[str, Any], delay: float = 0.0) -> ScheduledEvent:
+        event = ScheduledEvent(
+            id=self._world.next_id("event"),
+            name=name,
+            payload=copy.deepcopy(payload),
+            deliver_at=float(self._world.clock.now()) + float(delay),
+        )
+        self._events.append(event)
+        self._world.trace.record(
+            "event_scheduled",
+            name,
+            {"event_id": event.id, "payload": payload, "deliver_at": event.deliver_at},
+        )
+        return event
+
+    def deliver_due(self) -> list[ScheduledEvent]:
+        delivered: list[ScheduledEvent] = []
+        for event in self._events:
+            if event.delivered or event.deliver_at > self._world.clock.now():
+                continue
+            if self._world.faults.before_event_delivery(event):
+                continue
+            event.delivered = True
+            for index in range(self._world.faults.duplicate_event_count(event)):
+                payload = copy.deepcopy(event.payload)
+                delivered_event = event.model_copy(deep=True)
+                delivered_event.id = event.id if index == 0 else f"{event.id}_dup_{index}"
+                delivered.append(delivered_event)
+                self._world.trace.record(
+                    "event_delivered",
+                    event.name,
+                    {"event_id": delivered_event.id, "payload": payload},
+                )
+        return delivered
+
+    def pending(self) -> list[ScheduledEvent]:
+        return [event.model_copy(deep=True) for event in self._events if not event.delivered]
+
+    def snapshot(self) -> list[dict[str, Any]]:
+        return [event.model_dump(mode="json") for event in self._events]

agentproof/core/faults.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+Severity = Literal["low", "medium", "high", "critical"]
+
+
+class AgentProofError(Exception):
+    """Base exception for AgentProof-controlled errors."""
+
+
+class AgentProofToolError(AgentProofError):
+    code = "tool_error"
+
+    def __init__(self, tool_name: str, message: str | None = None) -> None:
+        self.tool_name = tool_name
+        super().__init__(message or f"{self.code}: {tool_name}")
+
+
+class ToolTimeoutError(AgentProofToolError, TimeoutError):
+    code = "timeout"
+
+
+class ToolRateLimitError(AgentProofToolError):
+    code = "rate_limited"
+
+
+class ToolPermissionDeniedError(AgentProofToolError):
+    code = "permission_denied"
+
+
+class ToolInjectedError(AgentProofToolError):
+    code = "tool_error"
+
+
+class ToolValidationError(AgentProofToolError):
+    code = "validation_error"
+
+
+class UnsupportedMutationError(AgentProofError):
+    pass
+
+
+class MutationSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    type: str
+    target: str | None = None
+    occurrence: int | None = 1
+    params: dict[str, Any] = Field(default_factory=dict)
+    severity: Severity = "medium"
+
+
+@dataclass(frozen=True)
+class PreCallAction:
+    return_value: Any = None
+    exception: AgentProofToolError | None = None
+    handled: bool = False
+
+
+class FaultController:
+    def __init__(self, world: Any) -> None:
+        self._world = world
+        self._mutations: list[MutationSpec] = []
+        self._counts: dict[tuple[str, str, str | None], int] = {}
+
+    @property
+    def mutations(self) -> list[MutationSpec]:
+        return list(self._mutations)
+
+    def install(self, spec: MutationSpec) -> None:
+        if spec.type == "reorder_tool_results":
+            raise UnsupportedMutationError(
+                "reorder_tool_results requires a controlled parallel tool-result scheduler and "
+                "is not stable in this MVP"
+            )
+        self._mutations.append(spec)
+
+    def prepare_user_inputs(self, inputs: list[str]) -> list[str]:
+        spec = self._match("before_agent_input", None, "duplicate_user_request")
+        if spec is None:
+            return list(inputs)
+        if not inputs:
+            return []
+        index = int(spec.params.get("index", 0))
+        index = min(max(index, 0), len(inputs) - 1)
+        duplicated = list(inputs)
+        duplicated.insert(index + 1, inputs[index])
+        self._world.trace.record(
+            "fault",
+            "duplicate_user_request",
+            {"mutation": spec.model_dump(mode="json"), "input": inputs[index]},
+        )
+        self._world.trace.record("user_input", "user", {"text": inputs[index], "duplicate": True})
+        return duplicated
+
+    def before_tool_call(
+        self,
+        *,
+        tool_name: str,
+        args: dict[str, Any],
+        invocation_id: str,
+    ) -> PreCallAction | None:
+        del args
+        latency = self._match("before_tool_call", tool_name, "tool_latency")
+        if latency is not None:
+            seconds = float(latency.params.get("seconds", 1.0))
+            self._world.clock.advance(seconds)
+            self._world.trace.record(
+                "fault",
+                "tool_latency",
+                {
+                    "mutation": latency.model_dump(mode="json"),
+                    "invocation_id": invocation_id,
+                    "advanced_seconds": seconds,
+                },
+            )
+            return None
+
+        timeout = self._match("before_tool_call", tool_name, "tool_timeout")
+        if timeout is not None:
+            self._world.trace.record(
+                "fault",
+                "tool_timeout",
+                {"mutation": timeout.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+            return PreCallAction(
+                exception=ToolTimeoutError(tool_name, "tool timed out before commit"),
+                handled=True,
+            )
+
+        injected = self._match("before_tool_call", tool_name, "tool_error")
+        if injected is not None:
+            self._world.trace.record(
+                "fault",
+                "tool_error",
+                {"mutation": injected.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+            message = str(injected.params.get("message", "injected tool error"))
+            return PreCallAction(exception=ToolInjectedError(tool_name, message), handled=True)
+
+        rate_limited = self._match("before_tool_call", tool_name, "rate_limited")
+        if rate_limited is not None:
+            self._world.trace.record(
+                "fault",
+                "rate_limited",
+                {"mutation": rate_limited.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+            return PreCallAction(
+                exception=ToolRateLimitError(tool_name, "tool call was rate limited"),
+                handled=True,
+            )
+
+        denied = self._match("before_tool_call", tool_name, "permission_denied")
+        if denied is not None:
+            self._world.trace.record(
+                "fault",
+                "permission_denied",
+                {"mutation": denied.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+            return PreCallAction(
+                exception=ToolPermissionDeniedError(tool_name, "permission denied"),
+                handled=True,
+            )
+
+        stale = self._match("before_tool_call", tool_name, "stale_state")
+        if stale is not None:
+            if "value" not in stale.params:
+                raise UnsupportedMutationError("stale_state requires params['value']")
+            self._world.trace.record(
+                "fault",
+                "stale_state",
+                {"mutation": stale.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+            return PreCallAction(return_value=stale.params["value"], handled=True)
+
+        return None
+
+    def after_commit_before_response(self, *, tool_name: str, invocation_id: str) -> None:
+        spec = self._match("after_commit_before_response", tool_name, "timeout_after_commit")
+        if spec is None:
+            return
+        self._world.trace.record(
+            "fault",
+            "timeout_after_commit",
+            {"mutation": spec.model_dump(mode="json"), "invocation_id": invocation_id},
+        )
+        raise ToolTimeoutError(tool_name, "tool timed out after commit")
+
+    def after_tool_response(
+        self,
+        *,
+        tool_name: str,
+        invocation_id: str,
+        response: Any,
+    ) -> Any:
+        malformed = self._match("after_tool_response", tool_name, "malformed_response")
+        if malformed is not None:
+            self._world.trace.record(
+                "fault",
+                "malformed_response",
+                {"mutation": malformed.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+            return malformed.params.get("value", "<<agentproof:malformed-response>>")
+
+        missing = self._match("after_tool_response", tool_name, "missing_field")
+        if missing is not None:
+            field = missing.params.get("field")
+            if not isinstance(field, str):
+                raise UnsupportedMutationError("missing_field requires params['field']")
+            self._world.trace.record(
+                "fault",
+                "missing_field",
+                {
+                    "mutation": missing.model_dump(mode="json"),
+                    "invocation_id": invocation_id,
+                    "field": field,
+                },
+            )
+            if isinstance(response, dict):
+                cloned = dict(response)
+                cloned.pop(field, None)
+                return cloned
+            return response
+
+        duplicate = self._match("after_tool_response", tool_name, "duplicate_tool_result")
+        if duplicate is not None:
+            self._world.trace.record(
+                "fault",
+                "duplicate_tool_result",
+                {"mutation": duplicate.model_dump(mode="json"), "invocation_id": invocation_id},
+            )
+            return {"agentproof_duplicate_tool_result": [response, response]}
+
+        changed = self._match("after_tool_response", tool_name, "state_changed_after_read")
+        if changed is not None:
+            path = changed.params.get("path")
+            if not isinstance(path, list) or not path:
+                raise UnsupportedMutationError("state_changed_after_read requires params['path']")
+            value = changed.params.get("value")
+            self._world.set_path(path, value)
+            self._world.trace.record(
+                "fault",
+                "state_changed_after_read",
+                {
+                    "mutation": changed.model_dump(mode="json"),
+                    "invocation_id": invocation_id,
+                    "path": path,
+                    "value": value,
+                },
+            )
+            return response
+
+        return response
+
+    def before_event_delivery(self, event: Any) -> bool:
+        delayed = self._match("before_event_delivery", event.name, "delayed_event")
+        if delayed is None:
+            return False
+        delay = float(delayed.params.get("seconds", 1.0))
+        event.deliver_at += delay
+        self._world.trace.record(
+            "fault",
+            "delayed_event",
+            {"mutation": delayed.model_dump(mode="json"), "event_id": event.id, "seconds": delay},
+        )
+        return True
+
+    def duplicate_event_count(self, event: Any) -> int:
+        duplicate = self._match("on_event_delivery", event.name, "duplicate_event")
+        if duplicate is None:
+            return 1
+        count = int(duplicate.params.get("count", 2))
+        self._world.trace.record(
+            "fault",
+            "duplicate_event",
+            {"mutation": duplicate.model_dump(mode="json"), "event_id": event.id, "count": count},
+        )
+        return max(count, 1)
+
+    def _match(self, stage: str, target: str | None, mutation_type: str) -> MutationSpec | None:
+        for spec in self._mutations:
+            if spec.type != mutation_type:
+                continue
+            if spec.target is not None and spec.target != target:
+                continue
+            key = (stage, mutation_type, spec.target)
+            self._counts[key] = self._counts.get(key, 0) + 1
+            if spec.occurrence is None or self._counts[key] == spec.occurrence:
+                return spec
+        return None

agentproof/core/invariant.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import inspect
+import traceback
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+from agentproof.core.result import InvariantFailure
+
+InvariantFunc = Callable[[Any], Awaitable[None] | None]
+
+
+@dataclass(frozen=True)
+class Invariant:
+    name: str
+    func: InvariantFunc
+    severity: str = "high"
+
+    async def check(self, world: Any) -> InvariantFailure | None:
+        try:
+            result = self.func(world)
+            if inspect.isawaitable(result):
+                await result
+        except AssertionError as exc:
+            return InvariantFailure(
+                name=self.name,
+                message=str(exc) or "assertion failed",
+                exception_type=type(exc).__name__,
+                traceback="".join(
+                    traceback.format_exception(type(exc), exc, exc.__traceback__, limit=6)
+                ),
+            )
+        return None
+
+    async def check_programming_error(self, world: Any) -> InvariantFailure | None:
+        try:
+            result = self.func(world)
+            if inspect.isawaitable(result):
+                await result
+        except AssertionError:
+            raise
+        except Exception as exc:
+            return InvariantFailure(
+                name=self.name,
+                message=str(exc),
+                exception_type=type(exc).__name__,
+                traceback="".join(
+                    traceback.format_exception(type(exc), exc, exc.__traceback__, limit=6)
+                ),
+            )
+        return None
+
+
+def invariant(
+    func: InvariantFunc | None = None, *, name: str | None = None, severity: str = "high"
+) -> Any:
+    def wrap(inner: InvariantFunc) -> Invariant:
+        return Invariant(name=name or inner.__name__, func=inner, severity=severity)
+
+    if func is None:
+        return wrap
+    return wrap(func)
+
+
+async def evaluate_invariants(
+    invariants: list[Invariant],
+    world: Any,
+) -> tuple[list[InvariantFailure], list[InvariantFailure]]:
+    assertion_failures: list[InvariantFailure] = []
+    programming_errors: list[InvariantFailure] = []
+    for item in invariants:
+        try:
+            result = item.func(world)
+            if inspect.isawaitable(result):
+                await result
+        except AssertionError as exc:
+            failure = InvariantFailure(
+                name=item.name,
+                message=str(exc) or "assertion failed",
+                exception_type=type(exc).__name__,
+                traceback="".join(
+                    traceback.format_exception(type(exc), exc, exc.__traceback__, limit=6)
+                ),
+            )
+            assertion_failures.append(failure)
+            world.trace.record(
+                "invariant_fail",
+                item.name,
+                {"message": failure.message, "exception_type": failure.exception_type},
+            )
+        except Exception as exc:
+            failure = InvariantFailure(
+                name=item.name,
+                message=str(exc),
+                exception_type=type(exc).__name__,
+                traceback="".join(
+                    traceback.format_exception(type(exc), exc, exc.__traceback__, limit=6)
+                ),
+            )
+            programming_errors.append(failure)
+            world.trace.record(
+                "invariant_fail",
+                item.name,
+                {"message": failure.message, "exception_type": failure.exception_type},
+            )
+        else:
+            world.trace.record("invariant_pass", item.name, {})
+    return assertion_failures, programming_errors

agentproof/core/redaction.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+SECRET_KEY_FRAGMENTS = ("api_key", "authorization", "token", "password", "secret")
+SECRET_STRING_PREFIXES = ("sk-", "Bearer ")
+REDACTED = "[REDACTED]"
+
+
+def redact(value: Any) -> Any:
+    """Return a JSON-safe copy with obvious secrets removed."""
+
+    if isinstance(value, Mapping):
+        redacted: dict[str, Any] = {}
+        for key, item in value.items():
+            key_text = str(key)
+            if _is_secret_key(key_text):
+                redacted[key_text] = REDACTED
+            else:
+                redacted[key_text] = redact(item)
+        return redacted
+    if isinstance(value, str):
+        return REDACTED if value.startswith(SECRET_STRING_PREFIXES) else value
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [redact(item) for item in value]
+    return value
+
+
+def _is_secret_key(key: str) -> bool:
+    lowered = key.lower().replace("-", "_")
+    return any(fragment in lowered for fragment in SECRET_KEY_FRAGMENTS)

agentproof/core/result.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from agentproof.core.effects import Effect
+from agentproof.core.faults import MutationSpec, Severity
+from agentproof.core.trace import TraceEvent
+
+RunStatus = Literal[
+    "PASS",
+    "INVARIANT_FAILURE",
+    "BASELINE_FAILURE",
+    "AGENT_ERROR",
+    "TOOL_ERROR_EXPECTED",
+    "TEST_ERROR",
+    "ADAPTER_ERROR",
+    "UNSUPPORTED",
+    "SKIPPED",
+]
+
+SEVERITY_ORDER: dict[Severity, int] = {
+    "low": 0,
+    "medium": 1,
+    "high": 2,
+    "critical": 3,
+}
+
+
+class InvariantFailure(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    message: str
+    exception_type: str
+    traceback: str
+
+
+class RunResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    run_id: str
+    scenario: str
+    adapter: str
+    seed: int
+    mutation: MutationSpec | None = None
+    severity: Severity = "medium"
+    status: RunStatus = "PASS"
+    final_output: str | None = None
+    trace: list[TraceEvent] = Field(default_factory=list)
+    effects: list[Effect] = Field(default_factory=list)
+    violated_invariants: list[str] = Field(default_factory=list)
+    invariant_failures: list[InvariantFailure] = Field(default_factory=list)
+    error_message: str | None = None
+    artifact_path: Path | None = None
+    initial_world_snapshot: dict[str, Any] = Field(default_factory=dict)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def failed(self) -> bool:
+        return self.status in {
+            "INVARIANT_FAILURE",
+            "BASELINE_FAILURE",
+            "AGENT_ERROR",
+            "TEST_ERROR",
+            "ADAPTER_ERROR",
+            "UNSUPPORTED",
+        }
+
+    @property
+    def name(self) -> str:
+        if self.mutation is None:
+            return f"{self.scenario}:baseline"
+        target = f":{self.mutation.target}" if self.mutation.target else ""
+        return f"{self.scenario}:{self.mutation.type}{target}"
+
+
+class SuiteResult(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    schema_version: int = 1
+    run_id: str
+    results: list[RunResult]
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def failures(self) -> list[RunResult]:
+        return [result for result in self.results if result.failed]
+
+    @property
+    def failed_count(self) -> int:
+        return len(self.failures)
+
+    @property
+    def passed_count(self) -> int:
+        return len([result for result in self.results if result.status == "PASS"])
+
+    @property
+    def total_count(self) -> int:
+        return len(self.results)
+
+    def exit_code(self, fail_on: Severity = "high") -> int:
+        threshold = SEVERITY_ORDER[fail_on]
+        for result in self.failures:
+            if SEVERITY_ORDER[result.severity] >= threshold:
+                return 1
+        return 0