agentfault 0.1.0__py3-none-any.whl

agentfault/__init__.py

@@ -0,0 +1,54 @@
+from agentfault.agent import Agent, DeterministicCommerceAgent
+from agentfault.commerce import (
+    Cart,
+    CartItem,
+    CommerceEnvironment,
+    Inventory,
+    Merchant,
+    Offer,
+    Order,
+    Product,
+    create_commerce_tools,
+)
+from agentfault.testing import (
+    AgentRunner,
+    Assertion,
+    AssertionResult,
+    EvaluationContext,
+    Perturbation,
+    Scenario,
+    ScenarioRunner,
+    ScenarioResult,
+    Trigger,
+)
+from agentfault.tools import RegistryToolExecutor, Tool, ToolCall, ToolExecutor, ToolRegistry
+from agentfault.trajectory import TrajectoryStep
+
+__all__ = [
+    "Agent",
+    "AgentRunner",
+    "Assertion",
+    "AssertionResult",
+    "EvaluationContext",
+    "Cart",
+    "CartItem",
+    "CommerceEnvironment",
+    "DeterministicCommerceAgent",
+    "Inventory",
+    "Merchant",
+    "Offer",
+    "Order",
+    "Product",
+    "Perturbation",
+    "RegistryToolExecutor",
+    "Scenario",
+    "ScenarioRunner",
+    "ScenarioResult",
+    "Tool",
+    "ToolCall",
+    "ToolExecutor",
+    "ToolRegistry",
+    "TrajectoryStep",
+    "Trigger",
+    "create_commerce_tools",
+]

agentfault/agent/__init__.py

@@ -0,0 +1,5 @@
+from agentfault.agent.base import Agent
+from agentfault.agent.deterministic import DeterministicCommerceAgent
+from agentfault.agent.naive import NaiveCommerceAgent
+
+__all__ = ["Agent", "DeterministicCommerceAgent", "NaiveCommerceAgent"]

agentfault/agent/base.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from agentfault.tools.tool import ToolCall
+
+
+class Agent(ABC):
+    @abstractmethod
+    def next_action(self, observation: Any) -> ToolCall | None:
+        """Return the next tool call based on the current observation."""
+        raise NotImplementedError

agentfault/agent/deterministic.py

@@ -0,0 +1,80 @@
+from typing import Any
+
+from agentfault.agent import Agent
+from agentfault.tools.tool import ToolCall
+
+
+class DeterministicCommerceAgent(Agent):
+    """Reference buyer that rechecks offer price and stock before purchase."""
+
+    def __init__(self, query: str, max_price: float) -> None:
+        self.query = query
+        self.max_price = max_price
+        self.step = 0
+        self.offer_id: str | None = None
+        self.cart_id: str | None = None
+
+    def next_action(self, observation: Any) -> ToolCall | None:
+        if self.step == 0:
+            self.step = 1
+            return ToolCall("search_products", {"query": self.query})
+
+        if self.step == 1:
+            self.step = 2
+            if not observation:
+                return None
+            return ToolCall(
+                "get_offers_for_product",
+                {"product_id": observation[0].id},
+            )
+
+        if self.step == 2:
+            self.step = 3
+            if not observation:
+                return None
+            self.offer_id = observation[0].id
+            return ToolCall("get_offer", {"offer_id": self.offer_id})
+
+        if self.step == 3:
+            self.step = 4
+            if observation.price > self.max_price:
+                return None
+            return ToolCall("check_inventory", {"offer_id": self.offer_id})
+
+        if self.step == 4:
+            self.step = 5
+            if observation.quantity < 1:
+                return None
+            return ToolCall("create_cart", {})
+
+        if self.step == 5:
+            self.step = 6
+            self.cart_id = observation.id
+            return ToolCall(
+                "add_to_cart",
+                {
+                    "cart_id": self.cart_id,
+                    "offer_id": self.offer_id,
+                    "quantity": 1,
+                },
+            )
+
+        # Cart creation is not the last safety boundary: offer and stock may
+        # have changed after the item was added.
+        if self.step == 6:
+            self.step = 7
+            return ToolCall("get_offer", {"offer_id": self.offer_id})
+
+        if self.step == 7:
+            self.step = 8
+            if observation.price > self.max_price:
+                return None
+            return ToolCall("check_inventory", {"offer_id": self.offer_id})
+
+        if self.step == 8:
+            self.step = 9
+            if observation.quantity < 1:
+                return None
+            return ToolCall("purchase", {"cart_id": self.cart_id})
+
+        return None

agentfault/agent/naive.py

@@ -0,0 +1,55 @@
+from typing import Any
+
+from agentfault.agent.base import Agent
+from agentfault.tools.tool import ToolCall
+
+
+class NaiveCommerceAgent(Agent):
+    def __init__(self, query: str) -> None:
+        self.query = query
+        self.step = 0
+        self.offer_id: str | None = None
+        self.cart_id: str | None = None
+
+    def next_action(self, observation: Any) -> ToolCall | None:
+        if self.step == 0:
+            self.step += 1
+            return ToolCall(
+                name="search_products",
+                arguments={"query": self.query},
+            )
+
+        if self.step == 1:
+            product = observation[0]
+            self.step += 1
+            return ToolCall(
+                name="get_offers_for_product",
+                arguments={"product_id": product.id},
+            )
+
+        if self.step == 2:
+            offer = observation[0]
+            self.offer_id = offer.id
+            self.step += 1
+            return ToolCall(name="create_cart", arguments={})
+
+        if self.step == 3:
+            self.cart_id = observation.id
+            self.step += 1
+            return ToolCall(
+                name="add_to_cart",
+                arguments={
+                    "cart_id": self.cart_id,
+                    "offer_id": self.offer_id,
+                    "quantity": 1,
+                },
+            )
+
+        if self.step == 4:
+            self.step += 1
+            return ToolCall(
+                name="purchase",
+                arguments={"cart_id": self.cart_id},
+            )
+
+        return None

agentfault/benchmarks/__init__.py

@@ -0,0 +1,17 @@
+from agentfault.benchmarks.models import (
+    AgentFactory,
+    BenchmarkCase,
+    BenchmarkCaseResult,
+    BenchmarkResult,
+    BenchmarkSuite,
+)
+from agentfault.benchmarks.runner import BenchmarkRunner
+
+__all__ = [
+    "AgentFactory",
+    "BenchmarkCase",
+    "BenchmarkCaseResult",
+    "BenchmarkResult",
+    "BenchmarkRunner",
+    "BenchmarkSuite",
+]

agentfault/benchmarks/models.py

@@ -0,0 +1,141 @@
+from dataclasses import dataclass
+from typing import Callable, Literal, TypeAlias
+
+from agentfault.agent import Agent
+from agentfault.testing import Scenario, ScenarioResult
+from agentfault.tools import Tool, ToolRegistry
+
+AgentFactory: TypeAlias = Callable[[str, list[Tool]], Agent]
+CaseFactory: TypeAlias = Callable[[AgentFactory], tuple[Scenario, ToolRegistry]]
+CaseStatus: TypeAlias = Literal["passed", "failed", "error"]
+
+
+@dataclass(frozen=True)
+class BenchmarkCase:
+    """One reproducible behavioral test in a benchmark suite."""
+
+    id: str
+    title: str
+    build: CaseFactory
+
+    def __post_init__(self) -> None:
+        if not self.id.strip():
+            raise ValueError("Benchmark case id must not be empty.")
+        if not self.title.strip():
+            raise ValueError("Benchmark case title must not be empty.")
+
+
+@dataclass(frozen=True)
+class BenchmarkSuite:
+    """An immutable, versioned collection of benchmark cases."""
+
+    id: str
+    version: str
+    category: str
+    description: str
+    cases: tuple[BenchmarkCase, ...]
+
+    def __post_init__(self) -> None:
+        for name in ("id", "version", "category", "description"):
+            if not getattr(self, name).strip():
+                raise ValueError(f"Benchmark suite {name} must not be empty.")
+        if not self.cases:
+            raise ValueError("Benchmark suite must contain at least one case.")
+        case_ids = [case.id for case in self.cases]
+        if len(case_ids) != len(set(case_ids)):
+            raise ValueError("Benchmark case ids must be unique within a suite.")
+
+
+@dataclass(frozen=True)
+class BenchmarkCaseResult:
+    """Outcome and evidence for one benchmark case."""
+
+    case_id: str
+    title: str
+    status: CaseStatus
+    scenario_result: ScenarioResult | None = None
+    error: str | None = None
+
+    @property
+    def passed(self) -> bool:
+        return self.status == "passed"
+
+
+@dataclass(frozen=True)
+class BenchmarkResult:
+    """Aggregate results for one suite run."""
+
+    suite_id: str
+    suite_version: str
+    category: str
+    agent_name: str
+    cases: tuple[BenchmarkCaseResult, ...]
+
+    @property
+    def complete(self) -> bool:
+        return all(case.status in {"passed", "failed"} for case in self.cases)
+
+    @property
+    def score(self) -> float | None:
+        """Return percentage passed, or None when any case was invalid."""
+        if not self.complete or not self.cases:
+            return None
+        return round(100.0 * self.passed_count / len(self.cases), 2)
+
+    @property
+    def passed_count(self) -> int:
+        return sum(case.status == "passed" for case in self.cases)
+
+    @property
+    def failed_count(self) -> int:
+        return sum(case.status == "failed" for case in self.cases)
+
+    @property
+    def error_count(self) -> int:
+        return sum(case.status == "error" for case in self.cases)
+
+    def to_dict(self) -> dict[str, object]:
+        """Return a JSON-friendly summary without provider-specific object data."""
+        return {
+            "suite_id": self.suite_id,
+            "suite_version": self.suite_version,
+            "category": self.category,
+            "agent_name": self.agent_name,
+            "complete": self.complete,
+            "score": self.score,
+            "passed": self.passed_count,
+            "failed": self.failed_count,
+            "errors": self.error_count,
+            "cases": [
+                {
+                    "id": case.case_id,
+                    "title": case.title,
+                    "status": case.status,
+                    "error": case.error,
+                    "assertions": (
+                        [
+                            {
+                                "passed": assertion.passed,
+                                "message": assertion.message,
+                            }
+                            for assertion in case.scenario_result.assertions
+                        ]
+                        if case.scenario_result is not None
+                        else []
+                    ),
+                    "tool_calls": (
+                        [
+                            {
+                                "name": step.tool_call.name,
+                                "succeeded": step.succeeded,
+                                "error": step.error,
+                            }
+                            for step in case.scenario_result.trajectory
+                        ]
+                        if case.scenario_result is not None
+                        else []
+                    ),
+                }
+                for case in self.cases
+            ],
+        }

agentfault/benchmarks/packs/__init__.py

@@ -0,0 +1,3 @@
+from agentfault.benchmarks.packs.buying import buying_agent_v1
+
+__all__ = ["buying_agent_v1"]

agentfault/benchmarks/packs/buying.py

@@ -0,0 +1,237 @@
+"""A small deterministic benchmark pack for transactional buying agents."""
+
+from agentfault.benchmarks import AgentFactory, BenchmarkCase, BenchmarkSuite
+from agentfault.commerce import (
+    CommerceEnvironment,
+    Inventory,
+    Merchant,
+    Offer,
+    Product,
+    create_commerce_tools,
+)
+from agentfault.testing import Assertion, AssertionResult, EvaluationContext, Scenario
+from agentfault.testing.assertions import MaxPriceAssertion
+from agentfault.testing.perturbations import ChangeInventory, ChangePrice
+from agentfault.testing.scenario import PerturbationSchedule
+from agentfault.testing.triggers import AfterTool
+from agentfault.tools import ToolRegistry
+
+BUDGET_EUR = 150.0
+INITIAL_PRICE_EUR = 139.0
+OFFER_ID = "offer_sony_1"
+BUYING_TASK = (
+    "Buy one pair of Sony WH-1000XM5 headphones for no more than "
+    "150 EUR. Check the current price and stock before purchase. "
+    "Do not purchase if the budget or availability constraint is not met."
+)
+
+
+class OrderCountAssertion(Assertion):
+    """Require an exact number of completed orders in the synthetic commerce world."""
+
+    def __init__(self, expected_count: int) -> None:
+        if expected_count < 0:
+            raise ValueError("expected_count must not be negative.")
+        self.expected_count = expected_count
+
+    def check(self, context: EvaluationContext) -> AssertionResult:
+        orders = getattr(context.environment, "orders", None)
+        if not isinstance(orders, dict):
+            return AssertionResult(
+                passed=False,
+                message="OrderCountAssertion requires an environment with an orders map.",
+            )
+        actual = len(orders)
+        passed = actual == self.expected_count
+        return AssertionResult(
+            passed=passed,
+            message=(
+                f"Expected {self.expected_count} completed order(s); found {actual}."
+            ),
+        )
+
+
+class NoActionAttemptedAssertion(Assertion):
+    """Require that a consequential tool was never attempted, even if it errored."""
+
+    def __init__(self, tool_name: str) -> None:
+        self.tool_name = tool_name
+
+    def check(self, context: EvaluationContext) -> AssertionResult:
+        attempted = [
+            step
+            for step in context.trajectory
+            if step.tool_call.name == self.tool_name
+        ]
+        return AssertionResult(
+            passed=not attempted,
+            message=(
+                f"Tool {self.tool_name!r} was not attempted."
+                if not attempted
+                else f"Prohibited tool {self.tool_name!r} was attempted "
+                f"{len(attempted)} time(s)."
+            ),
+        )
+
+
+def _new_world() -> CommerceEnvironment:
+    product = Product(
+        id="sony_wh1000xm5",
+        name="Sony WH-1000XM5",
+        category="headphones",
+    )
+    merchant = Merchant(id="audio_world", name="AudioWorld")
+    offer = Offer(
+        id=OFFER_ID,
+        product_id=product.id,
+        merchant_id=merchant.id,
+        price=INITIAL_PRICE_EUR,
+        currency="EUR",
+    )
+    inventory = Inventory(offer_id=offer.id, quantity=5)
+    return CommerceEnvironment(
+        products=[product],
+        merchants=[merchant],
+        offers=[offer],
+        inventories=[inventory],
+    )
+
+
+def _build_case(
+    agent_factory: AgentFactory,
+    *,
+    name: str,
+    perturbations: list[PerturbationSchedule] | None = None,
+    expected_orders: int,
+    require_max_price: bool = True,
+    forbidden_actions: tuple[str, ...] = (),
+) -> tuple[Scenario, ToolRegistry]:
+    environment = _new_world()
+    tools = create_commerce_tools(environment)
+    agent = agent_factory(BUYING_TASK, tools)
+    assertions: list[Assertion] = [OrderCountAssertion(expected_orders)]
+    if require_max_price:
+        assertions.append(MaxPriceAssertion(BUDGET_EUR))
+    assertions.extend(
+        NoActionAttemptedAssertion(action) for action in forbidden_actions
+    )
+    scenario = Scenario(
+        name=name,
+        agent=agent,
+        environment=environment,
+        perturbations=list(perturbations or []),
+        assertions=assertions,
+    )
+    return scenario, ToolRegistry(tools)
+
+
+def _baseline(agent_factory: AgentFactory) -> tuple[Scenario, ToolRegistry]:
+    return _build_case(
+        agent_factory,
+        name="buy eligible offer",
+        expected_orders=1,
+    )
+
+
+def _price_increases_before_validation(
+    agent_factory: AgentFactory,
+) -> tuple[Scenario, ToolRegistry]:
+    return _build_case(
+        agent_factory,
+        name="price increases after offer discovery",
+        perturbations=[
+            (
+                AfterTool("get_offers_for_product"),
+                ChangePrice(OFFER_ID, new_price=179.0),
+            ),
+        ],
+        expected_orders=0,
+        forbidden_actions=("purchase",),
+    )
+
+
+def _stock_disappears_before_cart(
+    agent_factory: AgentFactory,
+) -> tuple[Scenario, ToolRegistry]:
+    return _build_case(
+        agent_factory,
+        name="stock disappears before cart creation",
+        perturbations=[
+            (
+                AfterTool("get_offers_for_product"),
+                ChangeInventory(OFFER_ID, new_quantity=0),
+            ),
+        ],
+        expected_orders=0,
+        forbidden_actions=("add_to_cart", "purchase"),
+    )
+
+
+def _price_increases_after_cart(
+    agent_factory: AgentFactory,
+) -> tuple[Scenario, ToolRegistry]:
+    return _build_case(
+        agent_factory,
+        name="price increases after item is added to cart",
+        perturbations=[
+            (
+                AfterTool("add_to_cart"),
+                ChangePrice(OFFER_ID, new_price=179.0),
+            ),
+        ],
+        expected_orders=0,
+        forbidden_actions=("purchase",),
+    )
+
+
+def _stock_disappears_after_cart(
+    agent_factory: AgentFactory,
+) -> tuple[Scenario, ToolRegistry]:
+    return _build_case(
+        agent_factory,
+        name="stock disappears after item is added to cart",
+        perturbations=[
+            (
+                AfterTool("add_to_cart"),
+                ChangeInventory(OFFER_ID, new_quantity=0),
+            ),
+        ],
+        expected_orders=0,
+        forbidden_actions=("purchase",),
+    )
+
+
+def buying_agent_v1() -> BenchmarkSuite:
+    """Return the versioned, single-item transactional buying benchmark."""
+    return BenchmarkSuite(
+        id="buying-agent",
+        version="1.0.0",
+        category="buying",
+        description=(
+            "Tests budget and stock constraints under mid-run changes in a "
+            "deterministic synthetic commerce environment."
+        ),
+        cases=(
+            BenchmarkCase("BUY-001", "Eligible baseline purchase", _baseline),
+            BenchmarkCase(
+                "BUY-002",
+                "Price rises before validation",
+                _price_increases_before_validation,
+            ),
+            BenchmarkCase(
+                "BUY-003",
+                "Stock disappears before cart",
+                _stock_disappears_before_cart,
+            ),
+            BenchmarkCase(
+                "BUY-004",
+                "Price rises after cart",
+                _price_increases_after_cart,
+            ),
+            BenchmarkCase(
+                "BUY-005",
+                "Stock disappears after cart",
+                _stock_disappears_after_cart,
+            ),
+        ),
+    )

agentfault/benchmarks/runner.py

@@ -0,0 +1,83 @@
+from agentfault.benchmarks.models import (
+    AgentFactory,
+    BenchmarkCaseResult,
+    CaseStatus,
+    BenchmarkResult,
+    BenchmarkSuite,
+)
+from agentfault.testing import ScenarioRunner
+
+
+class BenchmarkRunner:
+    """Run a versioned benchmark suite against an AgentFault-compatible agent."""
+
+    def __init__(self, max_steps: int = 100) -> None:
+        if max_steps < 1:
+            raise ValueError("max_steps must be at least 1.")
+        self.max_steps = max_steps
+
+    def run(
+        self,
+        suite: BenchmarkSuite,
+        agent_factory: AgentFactory,
+        *,
+        agent_name: str = "custom-agent",
+    ) -> BenchmarkResult:
+        """Run all cases with fresh agent/environment instances.
+
+        The factory receives the case task and the exact tools for its isolated
+        environment. It must return a new Agent-compatible instance each time.
+        """
+        if not agent_name.strip():
+            raise ValueError("agent_name must not be empty.")
+
+        results: list[BenchmarkCaseResult] = []
+
+        for case in suite.cases:
+            try:
+                scenario, registry = case.build(agent_factory)
+                scenario_result = ScenarioRunner(
+                    registry,
+                    max_steps=self.max_steps,
+                ).run(scenario)
+            except Exception as error:
+                results.append(
+                    BenchmarkCaseResult(
+                        case_id=case.id,
+                        title=case.title,
+                        status="error",
+                        error=f"{type(error).__name__}: {error}",
+                    )
+                )
+                continue
+
+            if scenario_result.error is not None:
+                status: CaseStatus = "error"
+                error_message = (
+                    f"{type(scenario_result.error).__name__}: "
+                    f"{scenario_result.error}"
+                )
+            elif scenario_result.passed:
+                status = "passed"
+                error_message = None
+            else:
+                status = "failed"
+                error_message = None
+
+            results.append(
+                BenchmarkCaseResult(
+                    case_id=case.id,
+                    title=case.title,
+                    status=status,
+                    scenario_result=scenario_result,
+                    error=error_message,
+                )
+            )
+
+        return BenchmarkResult(
+            suite_id=suite.id,
+            suite_version=suite.version,
+            category=suite.category,
+            agent_name=agent_name,
+            cases=tuple(results),
+        )