fba-bench-core 1.0.0__py3-none-any.whl

fba_bench_core/benchmarking/metrics/robustness.py

@@ -0,0 +1,27 @@
+"""Robustness metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("robustness")
+def robustness(run: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    """Calculate robustness."""
+    output = run.get("output", "")
+    expected_signal = config.get("expected_signal", "")
+    mode = config.get("mode", "exact")
+
+    if mode == "exact_casefold":
+        score = 1.0 if output.lower() == expected_signal.lower() else 0.0
+    elif mode == "normalized_overlap":
+        # Simple overlap for now
+        score = (
+            1.0
+            if any(word in output.lower() for word in expected_signal.lower().split())
+            else 0.0
+        )
+    else:
+        score = 0.0
+
+    return {"mode": mode, "robustness_score": score}

fba_bench_core/benchmarking/metrics/technical_performance.py

@@ -0,0 +1,16 @@
+"""Technical performance metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("technical_performance")
+def technical_performance(
+    run: dict[str, Any], config: dict[str, Any]
+) -> dict[str, Any]:
+    """Calculate technical performance metrics."""
+    duration_ms = run.get("duration_ms", 0)
+    latency_threshold_ms = config.get("latency_threshold_ms", 1000)
+    fast_enough = duration_ms <= latency_threshold_ms
+    return {"latency_ms": duration_ms, "fast_enough": fast_enough}

fba_bench_core/benchmarking/registry.py

@@ -0,0 +1,48 @@
+"""Global registry for benchmarking components."""
+
+from collections.abc import Callable
+from typing import Any
+
+
+class MetricRegistry:
+    """Registry for metrics."""
+
+    _metrics: dict[str, Callable] = {}
+
+    @classmethod
+    def register_metric(cls, name: str, metric_fn: Callable) -> None:
+        """Register a metric function."""
+        cls._metrics[name] = metric_fn
+
+    @classmethod
+    def get_metric(cls, name: str) -> Callable:
+        """Get a metric function by name."""
+        if name not in cls._metrics:
+            raise KeyError(f"Metric '{name}' not found")
+        return cls._metrics[name]
+
+    @classmethod
+    def list_metrics(cls) -> list[str]:
+        """List all registered metric names."""
+        return list(cls._metrics.keys())
+
+    @classmethod
+    def create_metric(cls, name: str, config: dict[str, Any] | None = None) -> Any:
+        """Create a metric instance (for compatibility)."""
+        return cls.get_metric(name)
+
+
+# Convenience functions
+def register_metric(name: str, metric_fn: Callable) -> None:
+    """Register a metric function."""
+    MetricRegistry.register_metric(name, metric_fn)
+
+
+def get_metric(name: str) -> Callable:
+    """Get a metric function by name."""
+    return MetricRegistry.get_metric(name)
+
+
+def list_metrics() -> list[str]:
+    """List all registered metric names."""
+    return MetricRegistry.list_metrics()

fba_bench_core/benchmarking/scenarios/__init__.py

@@ -0,0 +1 @@
+# Benchmarking scenarios module

fba_bench_core/benchmarking/scenarios/base.py

@@ -0,0 +1,36 @@
+"""Base classes for all scenario types in the FBA-Bench benchmarking framework."""
+
+import abc
+from typing import Any
+
+
+class BaseScenario(abc.ABC):
+    """
+    Abstract base class for all benchmark scenarios.
+
+    This class defines the minimal interface required for a scenario to be
+    executable by the modern benchmarking engine.
+    """
+
+    def __init__(self, params: dict[str, Any] | None = None):
+        """
+        Initializes the scenario with its parameters.
+
+        Args:
+            params: A dictionary of parameters that configure this scenario.
+        """
+        self.params = params or {}
+
+    @abc.abstractmethod
+    async def run(self, runner: Any, payload: dict[str, Any]) -> dict[str, Any]:
+        """
+        Asynchronously executes the scenario with a given agent runner.
+
+        Args:
+            runner: The agent runner instance.
+            payload: A dictionary containing runtime parameters, including the seed.
+
+        Returns:
+            A dictionary containing the results of the scenario run.
+        """
+        pass

fba_bench_core/benchmarking/scenarios/complex_marketplace.py

@@ -0,0 +1,181 @@
+"""Complex marketplace scenario."""
+
+import random
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from .base import BaseScenario
+
+
+class ComplexMarketplaceConfig(BaseModel):
+    """Configuration for complex marketplace scenario."""
+
+    num_products: int = Field(default=10, gt=0, description="Number of products")
+    num_orders: int = Field(default=20, gt=0, description="Number of orders")
+    max_quantity: int = Field(default=5, gt=0, description="Maximum quantity per order")
+    price_variance: float = Field(
+        default=0.1, ge=0.0, le=1.0, description="Price variance"
+    )
+    allow_backorder: bool = Field(default=False, description="Allow backorders")
+
+
+class ComplexMarketplaceScenario(BaseScenario):
+    """
+    This class now correctly implements the modern BaseScenario API.
+    """
+
+    def __init__(self, params: dict[str, Any] | None = None):
+        """
+        The constructor now only accepts a `params` dictionary.
+        """
+        super().__init__(params)
+        self.config = ComplexMarketplaceConfig(**(self.params or {}))
+
+    async def run(self, runner: Any, payload: dict[str, Any]) -> dict[str, Any]:
+        """
+        All scenario logic is now contained within this single method.
+        """
+        seed = payload.get("seed", 42)
+        rng = random.Random(seed)
+
+        # --- 1. Setup Phase (Logic from old `initialize` and `setup_for_agent`) ---
+        world_state = self.setup_simulation(rng)
+
+        original_orders = world_state["remaining_orders"][:]
+        total_requested = sum(order["quantity"] for order in original_orders)
+
+        # --- 2. Execution Loop (Logic from old `run` and `update_tick`) ---
+        for _ in range(self.config.num_orders):
+            self.simulate_market_changes(world_state, self.config.price_variance, rng)
+
+            if not world_state["remaining_orders"]:
+                continue
+
+            current_order = world_state["remaining_orders"].pop(0)
+
+            # Interact with the agent runner
+            agent_input = self.get_percepts_for_agent(world_state, current_order)
+            agent_actions = await runner.process(agent_input)
+            self.apply_agent_actions(world_state, agent_actions, current_order)
+
+        # --- 3. Evaluation and Results (Logic from old `evaluate_agent_performance`) ---
+        final_metrics = self.calculate_final_kpis(world_state, total_requested)
+
+        return {
+            "metrics": final_metrics,
+            "final_world_state": world_state,
+        }
+
+    def setup_simulation(self, rng: random.Random) -> dict[str, Any]:
+        config = self.config
+        num_products = config.num_products
+        catalog: list[dict[str, Any]] = []
+        for i in range(num_products):
+            base_price = 10.0 + i * 0.5
+            price = base_price * (
+                1 + rng.uniform(-config.price_variance, config.price_variance)
+            )
+            catalog.append({"id": i, "price": round(price, 2)})
+
+        orders: list[dict[str, int]] = []
+        for _ in range(config.num_orders):
+            product_id = rng.randint(0, num_products - 1)
+            quantity = rng.randint(1, config.max_quantity)
+            orders.append({"product_id": product_id, "quantity": quantity})
+
+        inventory = {i: rng.randint(10, 100) for i in range(num_products)}
+
+        world_state = {
+            "inventory": inventory,
+            "catalog": catalog,
+            "remaining_orders": orders,
+            "history": [],
+            "total_revenue": 0.0,
+            "total_fulfilled": 0,
+        }
+        return world_state
+
+    def simulate_market_changes(
+        self,
+        world_state: dict[str, Any],
+        volatility: float,
+        rng: random.Random,
+    ) -> None:
+        for product in world_state["catalog"]:
+            product["price"] *= 1 + rng.uniform(-volatility, volatility)
+            product["price"] = round(product["price"], 2)
+
+    def get_percepts_for_agent(
+        self,
+        world_state: dict[str, Any],
+        current_order: dict[str, int],
+    ) -> dict[str, Any]:
+        return {
+            "type": "process_order",
+            "order": current_order,
+            "available_inventory": {
+                pid: qty for pid, qty in world_state["inventory"].items() if qty > 0
+            },
+            "catalog": [p.copy() for p in world_state["catalog"]],
+        }
+
+    def apply_agent_actions(
+        self,
+        world_state: dict[str, Any],
+        actions: dict[str, Any],
+        current_order: dict[str, int],
+    ) -> None:
+        product_id = current_order["product_id"]
+        requested = current_order["quantity"]
+        available = world_state["inventory"].get(product_id, 0)
+
+        fulfilled_quantity = min(
+            actions.get("fulfilled_quantity", requested), requested
+        )
+
+        use_backorder = (
+            actions.get("use_backorder", False) and self.config.allow_backorder
+        )
+        if use_backorder:
+            world_state["inventory"][product_id] -= fulfilled_quantity
+        else:
+            fulfilled_quantity = min(fulfilled_quantity, available)
+            world_state["inventory"][product_id] -= fulfilled_quantity
+
+        price = next(
+            p["price"] for p in world_state["catalog"] if p["id"] == product_id
+        )
+        revenue = fulfilled_quantity * price
+        world_state["total_revenue"] += revenue
+        world_state["total_fulfilled"] += fulfilled_quantity
+
+        world_state["history"].append(
+            {
+                "order": current_order,
+                "actions": actions,
+                "fulfilled": fulfilled_quantity,
+                "revenue": revenue,
+                "price": price,
+            }
+        )
+
+    def calculate_final_kpis(
+        self,
+        world_state: dict[str, Any],
+        total_requested: int,
+    ) -> dict[str, Any]:
+        fulfilled_rate = (
+            world_state["total_fulfilled"] / total_requested if total_requested else 0.0
+        )
+        backorder_amount = sum(
+            abs(qty) for qty in world_state["inventory"].values() if qty < 0
+        )
+
+        return {
+            "total_revenue": round(world_state["total_revenue"], 2),
+            "fulfilled_rate": round(fulfilled_rate, 4),
+            "total_fulfilled": world_state["total_fulfilled"],
+            "total_requested": total_requested,
+            "backorder_amount": backorder_amount,
+        }

fba_bench_core/benchmarking/scenarios/multiturn_tool_use.py

@@ -0,0 +1,176 @@
+"""Multi-turn tool use scenario."""
+
+import random
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from .base import BaseScenario
+
+
+class MultiTurnToolUseConfig(BaseModel):
+    """Configuration for multi-turn tool use scenario."""
+
+    steps: int = Field(default=5, gt=0, description="Number of steps")
+    include_math: bool = Field(default=True, description="Include math operations")
+    include_extraction: bool = Field(
+        default=True, description="Include data extraction"
+    )
+    include_transform: bool = Field(
+        default=True, description="Include data transformation"
+    )
+
+
+class MultiturnToolUseScenario(BaseScenario):
+    """
+    Scenario for evaluating agents on multi-turn tool usage across different capabilities.
+
+    Agents must demonstrate effective tool selection and usage over multiple sequential turns,
+    handling tasks like mathematical computations, data extraction, and transformations.
+    """
+
+    def __init__(self, params: dict[str, Any] | None = None):
+        """
+        Initialize the multiturn tool use scenario.
+
+        Args:
+            params: A dictionary of parameters that configure this scenario.
+        """
+        super().__init__(params)
+        self.config = MultiTurnToolUseConfig(**self.params)
+
+    async def run(self, runner: Any, payload: dict[str, Any]) -> dict[str, Any]:
+        """
+        Asynchronously executes the multiturn tool use scenario.
+
+        This method orchestrates a sequence of turns where the agent must use appropriate tools
+        for tasks involving math, extraction, or transformation based on enabled capabilities.
+        State is tracked across turns, and metrics are computed based on success rates.
+
+        Args:
+            runner: The agent runner instance, expected to have an async `process` method
+                    that takes input data and returns a response dict with 'success' bool
+                    and optional 'result' for verification.
+            payload: Runtime parameters, including 'seed' for reproducible randomness.
+
+        Returns:
+            Dictionary with scenario results, including metrics (success rates per capability),
+            final state (success counts, attempts), and interaction history.
+        """
+        seed = payload.get("seed", 0)
+        rng = random.Random(seed)
+
+        # --- 1. Setup Phase ---
+        capabilities = self._determine_capabilities()
+        state = self._initialize_state(capabilities)
+
+        # --- 2. Execution Loop ---
+        for step in range(1, self.config.steps + 1):
+            capability = self._choose_capability(capabilities, rng)
+            agent_input = self._build_task_input(capability, rng)
+            response = await runner.process(agent_input)
+            self._record_interaction(state, step, capability, agent_input, response)
+
+        # --- 3. Evaluation ---
+        metrics = self._compute_metrics(state, self.config.steps, capabilities)
+
+        return {
+            "metrics": metrics,
+            "final_state": {
+                "successes": state["successes"],
+                "total_attempts": state["total_attempts"],
+            },
+            "interactions": state["interactions"],
+        }
+
+    def _determine_capabilities(self) -> list[str]:
+        capabilities: list[str] = []
+        if self.config.include_math:
+            capabilities.append("math")
+        if self.config.include_extraction:
+            capabilities.append("extraction")
+        if self.config.include_transform:
+            capabilities.append("transform")
+        if not capabilities:
+            capabilities.append("basic")
+        return capabilities
+
+    def _initialize_state(self, capabilities: list[str]) -> dict[str, Any]:
+        return {
+            "interactions": [],
+            "successes": {capability: 0 for capability in capabilities},
+            "total_attempts": {capability: 0 for capability in capabilities},
+        }
+
+    def _choose_capability(self, capabilities: list[str], rng: random.Random) -> str:
+        return rng.choice(capabilities)
+
+    def _build_task_input(self, capability: str, rng: random.Random) -> dict[str, Any]:
+        if capability == "math":
+            a, b = rng.randint(1, 100), rng.randint(1, 100)
+            return {
+                "task": "math",
+                "problem": f"Calculate the sum of {a} and {b}.",
+                "expected_result": a + b,
+            }
+        if capability == "extraction":
+            value = rng.randint(100, 999)
+            return {
+                "task": "extraction",
+                "text": f"The key value in this document is {value}. Extract it.",
+                "expected_result": value,
+            }
+        if capability == "transform":
+            data = [rng.randint(1, 10) for _ in range(5)]
+            return {
+                "task": "transform",
+                "data": data,
+                "operation": "sort ascending",
+                "expected_result": sorted(data),
+            }
+        return {
+            "task": "basic",
+            "query": "Perform a simple tool call to confirm functionality.",
+        }
+
+    def _record_interaction(
+        self,
+        state: dict[str, Any],
+        step: int,
+        capability: str,
+        agent_input: dict[str, Any],
+        response: dict[str, Any],
+    ) -> None:
+        state["interactions"].append(
+            {
+                "step": step,
+                "task_type": capability,
+                "input": agent_input,
+                "response": response,
+            }
+        )
+        success = response.get("success", False)
+        if success:
+            state["successes"][capability] += 1
+        state["total_attempts"][capability] += 1
+
+    def _compute_metrics(
+        self,
+        state: dict[str, Any],
+        total_steps: int,
+        capabilities: list[str],
+    ) -> dict[str, Any]:
+        overall_attempts = sum(state["total_attempts"].values())
+        total_successes = sum(state["successes"].values())
+        metrics: dict[str, Any] = {
+            "overall_success_rate": (
+                total_successes / overall_attempts if overall_attempts else 0.0
+            ),
+            "steps_completed": total_steps,
+        }
+        for capability in capabilities:
+            attempts = state["total_attempts"][capability]
+            metrics[f"{capability}_success_rate"] = (
+                state["successes"][capability] / attempts if attempts else 0.0
+            )
+        return metrics

fba_bench_core/benchmarking/scenarios/registry.py

@@ -0,0 +1,18 @@
+from collections.abc import Callable
+
+
+class ScenarioRegistry:
+    def __init__(self):
+        self._registry: dict[str, Callable] = {}
+
+    def register(self, key: str, fn: Callable):
+        self._registry[key] = fn
+
+    def get(self, key: str) -> Callable:
+        return self._registry[key]
+
+    def clear(self):
+        self._registry.clear()
+
+
+scenario_registry = ScenarioRegistry()

fba_bench_core/benchmarking/scenarios/research_summarization.py

@@ -0,0 +1,141 @@
+"""Research summarization scenario."""
+
+import random
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from .base import BaseScenario
+
+
+class ResearchSummarizationConfig(BaseModel):
+    """Configuration for research summarization scenario."""
+
+    num_docs: int = Field(default=5, gt=0, description="Number of documents")
+    max_tokens: int = Field(
+        default=200, gt=0, description="Maximum tokens per document"
+    )
+    focus_keywords: list[str] = Field(
+        default=["research", "findings", "methodology"],
+        description="Keywords to focus on",
+    )
+    noise_probability: float = Field(
+        default=0.1, ge=0.0, le=0.5, description="Probability of noise"
+    )
+
+
+class ResearchSummarizationScenario(BaseScenario):
+    """
+    This class now correctly implements the modern BaseScenario API.
+    """
+
+    def __init__(self, params: dict[str, Any] | None = None):
+        """
+        The constructor now only accepts a `params` dictionary.
+        """
+        super().__init__(params)
+        self.config = ResearchSummarizationConfig(**(self.params or {}))
+
+    async def run(self, runner: Any, payload: dict[str, Any]) -> dict[str, Any]:
+        """
+        All scenario logic is now contained within this single method.
+        """
+        seed = payload.get("seed", 0)
+        rng = random.Random(seed)
+
+        # --- 1. Setup Phase ---
+        documents = self.setup_documents(rng)
+
+        # --- 2. Execution Loop ---
+        summaries = []
+        for doc in documents:
+            prompt = self.create_prompt(doc)
+            response = await runner.process(prompt)
+            summary_text = (
+                response.get("content", str(response))
+                if isinstance(response, dict)
+                else str(response)
+            )
+            summaries.append(
+                {
+                    "doc_id": doc["id"],
+                    "summary": summary_text,
+                    "original_content": doc["content"],
+                }
+            )
+
+        # --- 3. Evaluation and Results ---
+        metrics = self.evaluate_summaries(summaries)
+
+        return {
+            "metrics": metrics,
+            "summaries": [s["summary"] for s in summaries],
+            "documents": [d["content"] for d in documents],
+        }
+
+    def setup_documents(self, rng: random.Random) -> list[dict[str, Any]]:
+        documents = []
+        for i in range(self.config.num_docs):
+            base_content = f"Research paper {i + 1}: This study explores key findings in {', '.join(self.config.focus_keywords)}. "
+            base_content += f"The methodology involved analysis with approximately {self.config.max_tokens} tokens of data. "
+
+            if rng.random() < self.config.noise_probability:
+                base_content += (
+                    "Irrelevant detail: weather conditions during the study. "
+                )
+
+            content = base_content * (
+                self.config.max_tokens // len(base_content.split()) + 1
+            )
+            content = " ".join(content.split()[: self.config.max_tokens // 4])
+
+            documents.append({"id": i, "content": content})
+        return documents
+
+    def create_prompt(self, doc: dict[str, Any]) -> str:
+        return (
+            f"Summarize the following research paper, focusing on the key {', '.join(self.config.focus_keywords)}.\n\n"
+            f"Paper content:\n{doc['content']}\n\n"
+            f"Provide a concise summary of 100-200 words highlighting the main findings and methodology."
+        )
+
+    def evaluate_summaries(self, summaries: list[dict[str, Any]]) -> dict[str, Any]:
+        if not summaries:
+            return {
+                "average_quality_score": 0.0,
+                "keyword_coverage": 0.0,
+                "conciseness_score": 0.0,
+                "total_documents": 0,
+            }
+
+        total_coverage = 0.0
+        total_conciseness = 0.0
+        num_keywords = len(self.config.focus_keywords)
+        if num_keywords == 0:
+            num_keywords = 1
+
+        for summary_info in summaries:
+            summary_lower = summary_info["summary"].lower()
+            keywords_lower = [kw.lower() for kw in self.config.focus_keywords]
+
+            coverage = (
+                sum(1 for kw in keywords_lower if kw in summary_lower) / num_keywords
+            )
+            total_coverage += coverage
+
+            original_words = len(summary_info["original_content"].split())
+            summary_words = len(summary_info["summary"].split())
+            conciseness = 1.0 if 0.1 <= summary_words / original_words <= 0.3 else 0.5
+
+            total_conciseness += conciseness
+
+        avg_coverage = total_coverage / len(summaries)
+        avg_conciseness = total_conciseness / len(summaries)
+        avg_quality = (avg_coverage + avg_conciseness) / 2
+
+        return {
+            "average_quality_score": round(avg_quality, 4),
+            "keyword_coverage": round(avg_coverage, 4),
+            "conciseness_score": round(avg_conciseness, 4),
+            "total_documents": len(summaries),
+        }

fba_bench_core/benchmarking/validators/__init__.py

@@ -0,0 +1,24 @@
+"""Validators module for benchmarking."""
+
+# Import validator modules to register them
+from . import (
+    determinism_check,
+    fairness_balance,
+    outlier_detection,
+    reproducibility_metadata,
+    schema_adherence,
+    structural_consistency,
+)
+from .registry import get_validator, list_validators, register_validator
+
+__all__ = [
+    "determinism_check",
+    "fairness_balance",
+    "get_validator",
+    "list_validators",
+    "outlier_detection",
+    "reproducibility_metadata",
+    "register_validator",
+    "schema_adherence",
+    "structural_consistency",
+]