fba-bench-core 1.0.0__py3-none-any.whl

fba_bench_core/__init__.py

@@ -0,0 +1,11 @@
+"""Phase 2 scaffold for the fba_bench_core package.
+
+This module is a minimal placeholder created during Phase 2 of the
+core rebuild. It intentionally contains no runtime logic; only the
+package-level metadata required to allow safe imports and satisfy
+linters during the rescue process.
+"""
+
+__all__ = []
+
+__version__ = "1.0.0"

fba_bench_core/agents/__init__.py

@@ -0,0 +1,15 @@
+"""Package exports for fba_bench_core.agents.
+
+Exports the BaseAgent abstract class and exposes the registry module to allow
+external code to discover and register agent implementations. Also export the
+typed base configuration model to make it easy for downstream users to extend.
+"""
+
+from __future__ import annotations
+
+from fba_bench_core.config import BaseAgentConfig
+
+from . import registry
+from .base import BaseAgent
+
+__all__ = ["BaseAgent", "registry", "BaseAgentConfig"]

fba_bench_core/agents/base.py

@@ -0,0 +1,83 @@
+"""Typed BaseAgent for fba_bench_core.
+
+Phase D change:
+- Replace legacy **kwargs configuration with a typed Pydantic configuration
+  object. Downstream implementations should subclass the provided config model
+  for specialized parameters.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from fba_bench_core.config import BaseAgentConfig
+from fba_bench_core.domain.events import BaseEvent, Command
+
+
+class BaseAgent(ABC):
+    """Abstract base class for agents that receive a validated configuration.
+
+    Rationale:
+        Using a typed configuration object prevents downstream implementations
+        from hiding untyped parameters behind `Any` and enables validation at
+        construction time. Implementations that require additional fields may
+        subclass `BaseAgentConfig` (see examples in the config module).
+
+    Initialization:
+        The agent receives a single `config: BaseAgentConfig` argument. The
+        `agent_id` is expected to be present on that config and becomes the
+        agent's immutable identifier.
+
+    Immutability:
+        The provided config model is frozen (Pydantic frozen model). The agent
+        stores the config object directly and exposes it via a read-only
+        property to avoid accidental mutation.
+    """
+
+    def __init__(self, config: BaseAgentConfig) -> None:
+        """Initialize the base agent with a validated, typed configuration.
+
+        Parameters:
+            config: An instance of BaseAgentConfig or a subclass thereof. The
+                    model is validated by Pydantic prior to construction.
+
+        Notes:
+            - Do not accept `**kwargs` here: typed configs are required.
+            - The agent keeps a reference to the provided config (which is
+              immutable/frozen). Use `agent.config.model_copy()` to obtain a
+              mutable copy if necessary.
+        """
+        self._config = config
+        self._agent_id = config.agent_id
+
+    @property
+    def agent_id(self) -> str:
+        """Return the agent's unique identifier (from config.agent_id)."""
+        return self._agent_id
+
+    @property
+    def config(self) -> BaseAgentConfig:
+        """Return the typed configuration object for this agent.
+
+        The returned object is immutable (Pydantic frozen model). Downstream
+        code that needs to modify configuration should create a new instance
+        (e.g., via `model_copy(update={...})`).
+        """
+        return self._config
+
+    def get_config(self) -> dict:
+        """Return a serializable shallow mapping of the configuration.
+
+        Returns:
+            A dict produced by Pydantic's model_dump() representing the config.
+        """
+        return self._config.model_dump()
+
+    @abstractmethod
+    async def decide(self, events: list[BaseEvent]) -> list[Command]:
+        """Decide on a list of Commands given observed domain events.
+
+        Implementations must be async coroutines and must not mutate the
+        provided `events` list.
+        """
+        raise NotImplementedError

fba_bench_core/agents/registry.py

@@ -0,0 +1,16 @@
+"""Phase 5 placeholder for the agent registry.
+
+This module will hold mappings of agent names to agent classes and will be
+populated during Phase 5. It intentionally contains no runtime logic now.
+"""
+
+AGENT_REGISTRY: dict[str, type] = {}  # Populated in a later phase (Phase 5)
+
+
+def create_runner(key: str, config: dict):
+    """Stub function for creating runners."""
+
+    class DummyRunner:
+        agent_id = config.get("agent_id", "dummy")
+
+    return DummyRunner()

fba_bench_core/benchmarking/__init__.py

@@ -0,0 +1,6 @@
+"""Benchmarking module for FBA Bench Core.
+
+This module provides benchmarking functionality including engine, scenarios, metrics, and validators.
+"""
+
+__all__ = []

fba_bench_core/benchmarking/core/__init__.py

@@ -0,0 +1 @@
+# Benchmarking core module

fba_bench_core/benchmarking/engine/__init__.py

@@ -0,0 +1,12 @@
+"""Public API for the FBA-Bench Benchmarking Engine."""
+
+from .core import Engine
+from .models import EngineConfig, EngineReport, RunnerSpec, ScenarioSpec
+
+__all__ = [
+    "Engine",
+    "EngineConfig",
+    "EngineReport",
+    "RunnerSpec",
+    "ScenarioSpec",
+]

fba_bench_core/benchmarking/engine/core.py

@@ -0,0 +1,135 @@
+import asyncio
+
+from agent_runners.registry import create_runner
+from fba_bench_core.benchmarking.metrics.registry import MetricRegistry
+from fba_bench_core.benchmarking.scenarios.registry import scenario_registry
+from fba_bench_core.benchmarking.validators.registry import ValidatorRegistry
+
+from .models import EngineConfig, EngineReport, RunReport, ScenarioReport
+
+
+class Engine:
+    def __init__(self, config: EngineConfig):
+        self.config = config
+
+    async def run(self) -> EngineReport:
+        scenario_reports = []
+        for scenario_spec in self.config.scenarios:
+            scenario_fn = scenario_registry.get(scenario_spec.key)
+            if scenario_fn is None:
+                raise ValueError(
+                    f"Scenario '{scenario_spec.key}' not found in registry"
+                )
+
+            # Assume first runner for minimal implementation
+            if not self.config.runners:
+                raise ValueError("No runners configured")
+            runner_spec = self.config.runners[0]
+            runner = create_runner(runner_spec.key, runner_spec.config)
+
+            repetitions = scenario_spec.repetitions or 1
+            seeds = scenario_spec.seeds or [42] * repetitions
+            scenario_runs = []
+            timeout = scenario_spec.timeout_seconds
+            retries = self.config.retries
+
+            for i in range(repetitions):
+                seed = seeds[i] if i < len(seeds) else 42
+                payload = {"seed": seed}
+                status = None
+                output = None
+                for attempt in range(retries + 1):
+                    try:
+                        if timeout is not None:
+                            output = await asyncio.wait_for(
+                                scenario_fn(runner, payload), timeout=timeout
+                            )
+                        else:
+                            output = await scenario_fn(runner, payload)
+                        status = "success"
+                        break
+                    except TimeoutError:
+                        status = "timeout"
+                        output = None
+                        break  # Do not retry timeouts
+                    except Exception:
+                        if attempt < retries:
+                            continue
+                        status = "error"
+                        output = None
+                        break
+
+                # Apply metrics
+                metrics = {}
+                if self.config.metrics:
+                    metric_reg = MetricRegistry()
+                    for metric_name in self.config.metrics:
+                        try:
+                            metric = metric_reg.create_metric(metric_name)
+                            if metric:
+                                metrics[metric_name] = (
+                                    metric.calculate(output or {})
+                                    if output is not None
+                                    else 0.0
+                                )
+                        except Exception:
+                            metrics[metric_name] = 0.0
+
+                run_report = RunReport(
+                    status=status,
+                    output=output,
+                    seed=seed,
+                    metrics=metrics,
+                )
+                scenario_runs.append(run_report)
+
+            # Compute aggregates for metrics
+            aggregates = {}
+            if scenario_runs and self.config.metrics:
+                all_metrics = set()
+                for r in scenario_runs:
+                    all_metrics.update(r.metrics.keys())
+                mean_metrics = {}
+                for m in all_metrics:
+                    values = [r.metrics.get(m, 0.0) for r in scenario_runs]
+                    mean_metrics[m] = sum(values) / len(values)
+                aggregates["metrics"] = {"mean": mean_metrics}
+
+            # Apply validators
+            if self.config.validators and scenario_runs:
+                validations = []
+                val_reg = ValidatorRegistry()
+                run_data = [r.model_dump() for r in scenario_runs]
+                for val_name in self.config.validators:
+                    try:
+                        validator = val_reg.create_validator(val_name)
+                        if validator:
+                            result = validator.validate(run_data)
+                            result_dict = (
+                                result.to_dict()
+                                if hasattr(result, "to_dict")
+                                else result
+                            )
+                            validations.append(
+                                {"name": val_name, "result": result_dict}
+                            )
+                    except Exception:
+                        validations.append(
+                            {
+                                "name": val_name,
+                                "result": {
+                                    "is_valid": False,
+                                    "error": "Validation failed",
+                                },
+                            }
+                        )
+                aggregates["validations"] = validations
+
+            scenario_report = ScenarioReport(
+                key=scenario_spec.key,
+                runs=scenario_runs,
+                aggregates=aggregates,
+            )
+            scenario_reports.append(scenario_report)
+
+        return EngineReport(scenario_reports=scenario_reports)

fba_bench_core/benchmarking/engine/models.py

@@ -0,0 +1,62 @@
+from typing import Any
+
+from pydantic import BaseModel, field_validator
+
+
+class RunnerSpec(BaseModel):
+    key: str
+    config: dict[str, Any]
+
+
+class ScenarioSpec(BaseModel):
+    key: str
+    timeout_seconds: float | None = None
+    repetitions: int | None = None
+    seeds: list[int] | None = None
+
+
+class EngineConfig(BaseModel):
+    scenarios: list[ScenarioSpec]
+    runners: list[RunnerSpec]
+    metrics: list[str] | None = None
+    validators: list[str] | None = None
+    parallelism: int = 1
+    retries: int = 0
+
+    @field_validator("parallelism")
+    @classmethod
+    def validate_parallelism(cls, v):
+        if v <= 0:
+            raise ValueError("parallelism must be greater than 0")
+        return v
+
+    @field_validator("scenarios")
+    @classmethod
+    def validate_scenarios(cls, v):
+        if not v:
+            raise ValueError("scenarios cannot be empty")
+        return v
+
+    @field_validator("runners")
+    @classmethod
+    def validate_runners(cls, v):
+        if not v:
+            raise ValueError("runners cannot be empty")
+        return v
+
+
+class RunReport(BaseModel):
+    status: str
+    output: Any = None
+    seed: int | None = None
+    metrics: dict[str, Any] = {}
+
+
+class ScenarioReport(BaseModel):
+    key: str
+    runs: list[RunReport] = []
+    aggregates: dict[str, Any] = {}
+
+
+class EngineReport(BaseModel):
+    scenario_reports: list[ScenarioReport]

fba_bench_core/benchmarking/metrics/__init__.py

@@ -0,0 +1,30 @@
+"""Metrics module for benchmarking."""
+
+# Import metric modules to register them
+from . import (
+    accuracy_score,
+    aggregate,
+    completeness,
+    cost_efficiency,
+    custom_scriptable,
+    keyword_coverage,
+    policy_compliance,
+    robustness,
+    technical_performance,
+)
+from .registry import get_metric, list_metrics, register_metric
+
+__all__ = [
+    "accuracy_score",
+    "aggregate",
+    "completeness",
+    "cost_efficiency",
+    "custom_scriptable",
+    "keyword_coverage",
+    "policy_compliance",
+    "robustness",
+    "technical_performance",
+    "get_metric",
+    "list_metrics",
+    "register_metric",
+]

fba_bench_core/benchmarking/metrics/accuracy_score.py

@@ -0,0 +1,27 @@
+"""Accuracy score metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("accuracy_score")
+def accuracy_score(run: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    """Calculate accuracy score."""
+    output = run.get("output", "")
+    expected = config.get("expected_output", "")
+    mode = config.get("mode", "exact")
+    case_insensitive = config.get("case_insensitive", False)
+
+    if case_insensitive:
+        output = output.lower()
+        expected = expected.lower()
+
+    if mode == "exact":
+        score = 1.0 if output == expected else 0.0
+    elif mode == "contains":
+        score = 1.0 if expected in output else 0.0
+    else:
+        score = 0.0  # default
+
+    return {"mode": mode, "accuracy": score}

fba_bench_core/benchmarking/metrics/aggregate.py

@@ -0,0 +1,39 @@
+"""Aggregation utilities for metrics."""
+
+from typing import Any
+
+
+def aggregate_all(data: list[dict[str, Any]]) -> dict[str, Any]:
+    """Aggregate all metrics from a list of data points."""
+    if not data:
+        return {}
+    # Simple implementation: return the first item
+    return data[0] if data else {}
+
+
+def aggregate_metric_values(data: list[dict[str, Any]], field: str) -> dict[str, Any]:
+    """Aggregate metric values for a specific field."""
+    values = [item.get(field) for item in data if field in item]
+    if not values:
+        return {}
+    # Filter out None values for type safety
+    clean_values = [v for v in values if v is not None]
+    if not clean_values:
+        return {}
+    numeric_values = [v for v in clean_values if isinstance(v, (int, float))]
+    boolean_values = [v for v in clean_values if isinstance(v, bool)]
+    by_field = {}
+    if numeric_values:
+        by_field["numeric"] = {
+            "mean": sum(numeric_values) / len(numeric_values),
+            "min": min(numeric_values),
+            "max": max(numeric_values),
+            "count": len(numeric_values),
+        }
+    if boolean_values:
+        by_field["boolean"] = {
+            "success_rate": sum(boolean_values) / len(boolean_values),
+            "true_count": sum(1 for b in boolean_values if b),
+            "false_count": len(boolean_values) - sum(1 for b in boolean_values if b),
+        }
+    return by_field

fba_bench_core/benchmarking/metrics/completeness.py

@@ -0,0 +1,38 @@
+"""Completeness metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("completeness")
+def completeness(run: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    """Calculate completeness."""
+    output = run.get("output", {})
+    required_fields = config.get("required_fields", [])
+    allow_nested = config.get("allow_nested", False)
+    if not required_fields:
+        return {"required": 0, "present": 0, "completeness": 1.0}
+
+    present = 0
+    for field in required_fields:
+        if allow_nested and "." in field:
+            keys = field.split(".")
+            current = output
+            found = True
+            for key in keys:
+                if isinstance(current, dict) and key in current:
+                    current = current[key]
+                else:
+                    found = False
+                    break
+            if found:
+                present += 1
+        elif field in output:
+            present += 1
+
+    return {
+        "required": len(required_fields),
+        "present": present,
+        "completeness": present / len(required_fields),
+    }

fba_bench_core/benchmarking/metrics/cost_efficiency.py

@@ -0,0 +1,32 @@
+"""Cost efficiency metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("cost_efficiency")
+def cost_efficiency(run: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    """Calculate cost efficiency."""
+    output = run.get("output", {})
+    cost = output.get("cost", 0)
+    token_usage = output.get("token_usage", {})
+    total_tokens = token_usage.get("total_tokens", 0)
+    token_to_cost_rate = config.get("token_to_cost_rate", 0)
+    score_value = config.get("score_value", 1.0)
+
+    if cost > 0:
+        efficiency = score_value / cost
+        supported = True
+        reason = None
+    elif total_tokens > 0 and token_to_cost_rate > 0:
+        cost = total_tokens * token_to_cost_rate
+        efficiency = score_value / cost
+        supported = True
+        reason = None
+    else:
+        efficiency = 0.0
+        supported = False
+        reason = "missing_usage"
+
+    return {"supported": supported, "reason": reason, "efficiency": efficiency}

fba_bench_core/benchmarking/metrics/custom_scriptable.py

@@ -0,0 +1,17 @@
+"""Custom scriptable metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("custom_scriptable")
+def custom_scriptable(run: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    """Calculate custom scriptable metric."""
+    expression = config.get("expression", "0")
+    try:
+        # Simple eval with run and config in scope, restricted globals
+        result = eval(expression, {"__builtins__": {}}, {"run": run, "config": config})
+        return {"result": result, "expression": expression}
+    except Exception as e:
+        return {"result": False, "error": str(e), "expression": expression}

fba_bench_core/benchmarking/metrics/keyword_coverage.py

@@ -0,0 +1,41 @@
+"""Keyword coverage metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("keyword_coverage")
+def keyword_coverage(run: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    """Calculate keyword coverage."""
+    field_path = config.get("field_path", "")
+    keywords = config.get("keywords", [])
+    unique_match = config.get("unique_match", True)
+
+    text = ""
+    data = run.get("output", {})
+    if field_path:
+        keys = field_path.split(".")
+        for key in keys:
+            if isinstance(data, dict):
+                data = data.get(key, "")
+            else:
+                break
+        text = str(data) if data else ""
+    else:
+        text = str(data)
+
+    if not keywords:
+        return {"keyword_total": 0, "keyword_hits": 0, "coverage": 0.0}
+
+    if unique_match:
+        found = set(kw for kw in keywords if kw in text)
+        hits = len(found)
+    else:
+        hits = sum(text.count(kw) for kw in keywords)
+
+    return {
+        "keyword_total": len(keywords),
+        "keyword_hits": hits,
+        "coverage": hits / len(keywords),
+    }

fba_bench_core/benchmarking/metrics/policy_compliance.py

@@ -0,0 +1,18 @@
+"""Policy compliance metric."""
+
+from typing import Any
+
+from .registry import register_metric
+
+
+@register_metric("policy_compliance")
+def policy_compliance(run: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    """Calculate policy compliance."""
+    output = run.get("output", {})
+    violations = output.get("policy_violations", 0)
+    if isinstance(violations, (list, tuple)):
+        violations = len(violations)
+    elif isinstance(violations, dict):
+        violations = violations.get("count", 0)
+    compliant = violations == 0
+    return {"policy_violations": violations, "compliant": compliant}

fba_bench_core/benchmarking/metrics/registry.py

@@ -0,0 +1,57 @@
+"""Registry for metrics."""
+
+from collections.abc import Callable
+
+
+class MetricRegistry:
+    _metrics: dict[str, Callable] = {}
+
+    @classmethod
+    def register(cls, name: str, metric_class: Callable) -> None:
+        """Register a metric class."""
+        cls._metrics[name] = metric_class
+
+    @classmethod
+    def create_metric(cls, name: str, config=None) -> Callable | None:
+        """Create a metric instance."""
+        fn = cls._metrics.get(name)
+        if fn:
+            return fn
+        return None
+
+    @classmethod
+    def get_metric(cls, name: str) -> Callable | None:
+        """Get a metric class by name."""
+        return cls._metrics.get(name)
+
+    @classmethod
+    def list_metrics(cls) -> list[str]:
+        """List all registered metric names."""
+        return list(cls._metrics.keys())
+
+
+# Global instance for function-based API
+registry = MetricRegistry()
+
+
+def get_metric(name: str) -> Callable:
+    """Get a metric by name, raising KeyError if not found."""
+    metric = registry.get_metric(name)
+    if metric is None:
+        raise KeyError(f"Metric '{name}' not found")
+    return metric
+
+
+def list_metrics() -> list[str]:
+    """List all registered metric names."""
+    return registry.list_metrics()
+
+
+def register_metric(name: str):
+    """Decorator to register a metric function with the given name."""
+
+    def decorator(func: Callable) -> Callable:
+        registry.register(name, func)
+        return func
+
+    return decorator