floeval 0.1.0b1__py3-none-any.whl

floeval/__init__.py

@@ -0,0 +1,5 @@
+"""Floeval - Multi-backend evaluation framework for LLM and RAG systems."""
+
+from floeval.api import Evaluation, Dataset, DatasetLoader, MetricRegistry
+
+__all__ = ["Evaluation", "Dataset", "DatasetLoader", "MetricRegistry"]

floeval/api/__init__.py

@@ -0,0 +1,8 @@
+"""Public API for Floeval - Evaluation Framework."""
+
+from floeval.api.dataset import DatasetLoader
+from floeval.api.evaluation import Evaluation
+from floeval.api.metrics.registry import MetricRegistry
+from floeval.config.schemas.io.dataset import Dataset
+
+__all__ = ["Evaluation", "Dataset", "DatasetLoader", "MetricRegistry"]

floeval/api/agent_evaluation.py

@@ -0,0 +1,318 @@
+"""Agent evaluation orchestrator."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import json
+import logging
+from typing import Any, Awaitable, Callable, Mapping
+
+from pydantic import BaseModel, Field
+
+import floeval.metric_providers  # noqa: F401 - trigger metric registration
+from floeval.api.metrics.base import BaseMetric, MetricResult
+from floeval.api.metrics.registry import MetricRegistry
+from floeval.config.schemas.io.agent_dataset import (
+    AgentDataset,
+    AgentSample,
+    _to_display_str,
+)
+from floeval.config.schemas.io.llm import OpenAIProviderConfig
+from floeval.core.execution.llm_executor import OpenAIProvider
+
+logger = logging.getLogger(__name__)
+
+MetricSpec = BaseMetric | str | dict[str, Any]
+
+
+class AgentEvaluationResult(BaseModel):
+    """Agent evaluation results."""
+
+    sample_results: list[dict[str, Any]] = Field(default_factory=list)
+    summary: dict[str, Any] = Field(default_factory=dict)
+
+
+class AgentEvaluation:
+    """Agent evaluation orchestrator.
+
+    Supports Mode 1 (pre-captured traces), Mode 2 (partial + agent callable),
+    Mode 4 (partial + agent_runner e.g. FloTorchRunner).
+    """
+
+    def __init__(
+        self,
+        dataset: AgentDataset,
+        metrics: list[MetricSpec],
+        llm_config: OpenAIProviderConfig | None = None,
+        agent: Callable[[str], str | Any] | Callable[[str], Awaitable[str | Any]] | None = None,
+        agent_runner: Any | None = None,
+        default_provider: str | None = "builtin",
+        metric_params: Mapping[str, dict[str, Any]] | None = None,
+    ):
+        self.dataset = dataset
+        self.metrics = metrics
+        self.llm_config = llm_config
+        self.agent = agent
+        self.agent_runner = agent_runner
+        self.default_provider = default_provider or "builtin"
+        self.metric_params = dict(metric_params or {})
+        self._registry = MetricRegistry()
+        self._resolved_metrics = self._resolve_metrics(metrics)
+
+    def _resolve_metrics(self, specs: list[MetricSpec]) -> list[BaseMetric]:
+        """Resolve metric specs to instances."""
+        resolved: list[BaseMetric] = []
+
+        for spec in specs:
+            if isinstance(spec, BaseMetric):
+                if (
+                    self.llm_config is not None
+                    and hasattr(spec, "llm_config")
+                    and spec.llm_config is None
+                ):
+                    spec.llm_config = self.llm_config
+                resolved.append(spec)
+                continue
+
+            if isinstance(spec, dict):
+                metric_id = spec.get("id")
+                if not metric_id:
+                    raise ValueError("Metric dict spec must include 'id'")
+                provider = (
+                    spec.get("provider")
+                    or self.default_provider
+                    or self._registry.resolve_best(metric_id, self.default_provider)
+                )
+                params = spec.get("params", {}) or {}
+                metric = self._create_metric(provider, metric_id, params)
+                resolved.append(metric)
+                continue
+
+            if isinstance(spec, str):
+                if ":" in spec:
+                    provider, metric_id = spec.split(":", 1)
+                else:
+                    metric_id = spec
+                    provider = self._registry.resolve_best(metric_id, self.default_provider)
+                metric = self._create_metric(provider, metric_id, params={})
+                resolved.append(metric)
+                continue
+
+            raise TypeError(f"Invalid metric spec: {spec!r}")
+
+        return resolved
+
+    def _merge_params(
+        self, provider: str, metric_id: str, params: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Merge user-provided params with evaluation-level defaults.
+
+        Precedence (highest to lowest):
+        1) Explicit params passed in the metric spec dict
+        2) metric_params mapping (keyed by "provider:metric" or "metric")
+        3) llm_config when metrics expect it
+        """
+        merged: dict[str, Any] = {}
+        merged.update(self.metric_params.get(metric_id, {}))
+        merged.update(self.metric_params.get(f"{provider}:{metric_id}", {}))
+        merged.update(params)
+        return merged
+
+    def _inject_context_params(
+        self, provider: str, metric_id: str, merged: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Inject context-derived params based on metric constructor signature.
+
+        Uses introspection to avoid hardcoding metric names. Injects:
+        - llm_config: when metric accepts it and not in merged
+        - llm_provider: when metric accepts it, not in merged, and we have llm_config
+        """
+        metric_factory = self._registry.get_class(provider, metric_id)
+        if metric_factory is None:
+            return merged
+
+        try:
+            if callable(metric_factory) and not isinstance(metric_factory, type):
+                sig = inspect.signature(metric_factory)
+            else:
+                sig = inspect.signature(metric_factory.__init__)
+        except (TypeError, ValueError, AttributeError):
+            return merged
+
+        if self.llm_config is None:
+            return merged
+
+        if "llm_config" in sig.parameters and "llm_config" not in merged:
+            merged["llm_config"] = self.llm_config
+
+        if "llm_provider" in sig.parameters and "llm_provider" not in merged:
+            merged["llm_provider"] = OpenAIProvider(
+                config_name=f"{provider}:{metric_id}",
+                **self.llm_config.model_dump(),
+            )
+
+        return merged
+
+    def _create_metric(
+        self, provider: str, metric_id: str, params: dict[str, Any]
+    ) -> BaseMetric:
+        """Create metric instance with dynamic dependency injection."""
+        merged = self._merge_params(provider, metric_id, params)
+        merged = self._inject_context_params(provider, metric_id, merged)
+
+        try:
+            return self._registry.create(provider, metric_id, **merged)
+        except TypeError as e:
+            if "llm_config" in merged or "llm_provider" in merged or "adapter" in merged:
+                fallback = {k: v for k, v in merged.items() if k not in ("llm_config", "llm_provider", "adapter")
+                }
+                try:
+                    return self._registry.create(provider, metric_id, **fallback)
+                except TypeError:
+                    pass
+            raise e
+
+    def _ensure_full_samples(self) -> list[AgentSample]:
+        """Ensure all samples have traces. Run agent/runner if partial."""
+        if not self.dataset.is_partial:
+            return self.dataset.all_full
+
+        partial = self.dataset.all_partial
+        if not partial:
+            return []
+
+        if self.agent_runner is not None:
+            if hasattr(self.agent_runner, "run_on_dataset"):
+                return self.agent_runner.run_on_dataset(partial)
+            full = []
+            for p in partial:
+                text = _to_display_str(p.user_input)
+                trace = self.agent_runner.run(text)
+                full.append(AgentSample.from_partial(p, trace))
+            return full
+
+        if self.agent is not None:
+            from floeval.utils.agent_trace import TraceCollector
+
+            collector = TraceCollector(self.agent)
+            return collector.collect(partial)
+
+        raise ValueError(
+            "Dataset has partial samples but no agent or agent_runner provided. "
+            "Pass agent (callable) for Mode 2 or agent_runner for Mode 4."
+        )
+
+    async def _ensure_full_samples_async(self) -> list[AgentSample]:
+        """Async variant: run agent/runner without nesting asyncio.run()."""
+        if not self.dataset.is_partial:
+            return self.dataset.all_full
+
+        partial = self.dataset.all_partial
+        if not partial:
+            return []
+
+        if self.agent_runner is not None:
+            runner = self.agent_runner
+            if hasattr(runner, "run_on_dataset_async"):
+                return await runner.run_on_dataset_async(partial)
+            if hasattr(runner, "run_on_dataset"):
+                # Sync run_on_dataset uses asyncio.run() internally; we're already in a loop.
+                # Run it in a thread so asyncio.run() gets a fresh loop in that thread.
+                loop = asyncio.get_running_loop()
+                return await loop.run_in_executor(
+                    None, lambda: runner.run_on_dataset(partial)
+                )
+            full = []
+            for p in partial:
+                text = _to_display_str(p.user_input)
+                if hasattr(runner, "arun") and asyncio.iscoroutinefunction(runner.arun):
+                    trace = await runner.arun(text)
+                else:
+                    loop = asyncio.get_running_loop()
+                    trace = await loop.run_in_executor(None, lambda t=text: runner.run(t))
+                full.append(AgentSample.from_partial(p, trace))
+            return full
+
+        if self.agent is not None:
+            from floeval.utils.agent_trace import TraceCollector
+
+            collector = TraceCollector(self.agent)
+            loop = asyncio.get_running_loop()
+            return await loop.run_in_executor(None, lambda: collector.collect(partial))
+
+        raise ValueError(
+            "Dataset has partial samples but no agent or agent_runner provided. "
+            "Pass agent (callable) for Mode 2 or agent_runner for Mode 4."
+        )
+
+    def run(self) -> AgentEvaluationResult:
+        """Run evaluation synchronously."""
+        return asyncio.run(self.arun())
+
+    async def arun(self) -> AgentEvaluationResult:
+        """Run evaluation asynchronously."""
+        full_samples = await self._ensure_full_samples_async()
+
+        if full_samples and self.dataset.is_partial:
+            captured = [
+                {
+                    "user_input": s.user_input,
+                    "trace": {
+                        "messages": [m.model_dump() for m in s.trace.messages],
+                        "final_response": s.trace.final_response,
+                    },
+                }
+                for s in full_samples
+            ]
+            logger.debug(
+                "Captured traces (full dataset for evaluation): %s",
+                json.dumps(captured, indent=2, default=str),
+            )
+
+        if not full_samples:
+            return AgentEvaluationResult(
+                sample_results=[],
+                summary={"error": "No full samples to evaluate"},
+            )
+
+        sample_results: list[dict[str, Any]] = []
+        metric_scores: dict[str, list[float]] = {}
+
+        for sample in full_samples:
+            row: dict[str, Any] = {
+                "user_input": sample.user_input,
+                "final_response": sample.trace.final_response,
+                "reference_outcome": sample.reference_outcome,
+                "metrics": {},
+            }
+
+            for metric in self._resolved_metrics:
+                name = getattr(metric, "name", metric.__class__.__name__)
+                try:
+                    if asyncio.iscoroutinefunction(getattr(metric, "aevaluate", None)):
+                        result = await metric.aevaluate(sample)
+                    else:
+                        result = metric.evaluate(sample)
+                except Exception as e:
+                    logger.exception("Metric %s failed for sample: %s", name, e)
+                    result = MetricResult(score=None, metadata={"error": str(e)})
+
+                row["metrics"][name] = {
+                    "score": result.score,
+                    "metadata": result.metadata,
+                }
+                if result.score is not None:
+                    metric_scores.setdefault(name, []).append(result.score)
+
+            sample_results.append(row)
+
+        summary = {}
+        for name, scores in metric_scores.items():
+            if scores:
+                summary[name] = sum(scores) / len(scores)
+
+        return AgentEvaluationResult(
+            sample_results=sample_results,
+            summary=summary,
+        )

floeval/api/dataset.py

@@ -0,0 +1,149 @@
+"""Dataset and Sample classes.
+
+The RAGAS adapter supports Pydantic models via `model_dump()`.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Sequence, Union
+
+from floeval.api.dataset_loaders.base import BaseDatasetLoader
+from floeval.api.dataset_loaders.local_file_loader import get_loader_for_file
+from floeval.config.schemas.io.dataset import (
+    Dataset,
+    PartialDataset,
+    PartialSample,
+    Sample,
+)
+
+
+def dataset_from_dict(
+    data: Dict[str, Any], partial_dataset: bool
+) -> Dataset | PartialDataset:
+    """Create dataset from a dict object.
+
+    Dataset should be of the form:
+    {
+        "samples": [
+            {
+                "user_input": "What is the capital of France?",
+                "llm_response": "The capital of France is Paris."
+            },
+            ...
+        ]
+    }
+    Note: In case of partial dataset, the llm_response field can be omitted or set to empty string.
+    The loader will handle it accordingly.
+
+    Args:
+        data: A dict containing the dataset information, typically loaded from a JSON file.
+        partial_dataset: flag to return PartialDataset instead of Dataset.
+
+    Returns:
+        A Dataset or PartialDataset instance.
+    """
+    raw_samples = data.get("samples", [])
+    if partial_dataset:
+        samples = [PartialSample(**s) for s in raw_samples]
+        return PartialDataset(samples=samples)
+    samples = [Sample(**s) for s in raw_samples]
+    return Dataset(samples=samples)
+
+
+def dataset_from_json(
+    path: Union[str, Path], partial_dataset: bool
+) -> Dataset | PartialDataset:
+    """Load dataset from JSON file."""
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    return dataset_from_dict(data, partial_dataset=partial_dataset)
+
+
+def dataset_from_file(
+    ds_path: Union[str, Path], partial_dataset: bool
+) -> Dataset | PartialDataset:
+    """Convenience method to load dataset from file with type detection."""
+    ds_path = Path(ds_path)
+    if not ds_path.is_file():
+        raise FileNotFoundError(f"Dataset file not found: {ds_path}")
+
+    loader_cls: type[BaseDatasetLoader] = get_loader_for_file(str(ds_path))
+    if partial_dataset:
+        samples = loader_cls.to_partial_samples(ds_path)
+        return PartialDataset(samples=samples)
+    else:
+        samples = loader_cls.to_samples(ds_path)
+        return Dataset(samples=samples)
+
+
+def dataset_from_samples(
+    samples: Sequence[Union[Sample, Dict[str, Any]]], partial_dataset: bool
+) -> Dataset | PartialDataset:
+    """Convenience helper: build dataset from Sample objects or flat dicts.
+
+    Args:
+        samples: Sequence of Sample/PartialSample objects or dicts convertible to Sample.
+        partial_dataset: If True, convert to PartialSample and return PartialDataset.
+
+    Returns:
+        Dataset or PartialDataset containing the provided samples.
+    """
+    if partial_dataset:
+        normalized_samples = []
+        for s in samples:
+            if isinstance(s, Sample):
+                normalized_samples.append(PartialSample(**s.model_dump()))
+            elif isinstance(s, PartialSample):
+                normalized_samples.append(s)
+            elif isinstance(s, dict):
+                normalized_samples.append(PartialSample(**s))
+            else:
+                raise ValueError(
+                    f"Invalid sample type: {type(s)}; expected PartialSample, or dict"
+                )
+        return PartialDataset(samples=normalized_samples)
+
+    # --- complete dataset case (containing llm_response field) --
+    normalized_samples = []
+    for s in samples:
+        if isinstance(s, Sample):
+            normalized_samples.append(s)
+        elif isinstance(s, dict):
+            normalized_samples.append(Sample(**s))
+        else:
+            raise ValueError(f"Invalid sample type: {type(s)}; expected Sample or dict")
+    return Dataset(samples=normalized_samples)
+
+
+class DatasetLoader:
+    """Unified interface for loading datasets from various sources."""
+
+    @staticmethod
+    def from_dict(
+        data: Dict[str, Any], partial_dataset=False
+    ) -> Dataset | PartialDataset:
+        """Create dataset from a dict shaped like PRD examples."""
+        return dataset_from_dict(data, partial_dataset=partial_dataset)
+
+    @staticmethod
+    def from_json(
+        path: Union[str, Path], partial_dataset=False
+    ) -> Dataset | PartialDataset:
+        """Load dataset from JSON file."""
+        return dataset_from_json(path, partial_dataset=partial_dataset)
+
+    @staticmethod
+    def from_file(
+        ds_path: Union[str, Path], partial_dataset=False
+    ) -> Dataset | PartialDataset:
+        """Convenience method to load dataset from file with type detection."""
+        return dataset_from_file(ds_path, partial_dataset=partial_dataset)
+
+    @staticmethod
+    def from_samples(
+        samples: Sequence[Union[Sample, Dict[str, Any]]], partial_dataset=False
+    ) -> Dataset | PartialDataset:
+        """Build dataset from Sample objects or flat dicts."""
+        return dataset_from_samples(samples, partial_dataset=partial_dataset)

floeval/api/dataset_loaders/agent_file_loader.py

@@ -0,0 +1,169 @@
+"""Dataset loader for agent evaluation with robust error handling."""
+
+import json
+import logging
+from pathlib import Path
+
+from floeval.config.schemas.io.agent_dataset import (
+    AgentDataset,
+    AgentSample,
+    AgentTrace,
+    AIMessage,
+    HumanMessage,
+    PartialAgentSample,
+    ToolCall,
+    ToolMessage,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetLoadError(Exception):
+    """Raised when dataset loading fails."""
+
+
+class AgentDatasetLoader:
+    """Load agent datasets from files."""
+
+    @staticmethod
+    def from_file(path: str | Path) -> AgentDataset:
+        """Load dataset from JSON or JSONL file.
+
+        Raises:
+            FileNotFoundError: File does not exist.
+            DatasetLoadError: File format invalid or parsing failed.
+        """
+        path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(f"Dataset file not found: {path}")
+
+        try:
+            if path.suffix == ".jsonl":
+                return AgentDatasetLoader._load_jsonl(path)
+            if path.suffix == ".json":
+                return AgentDatasetLoader._load_json(path)
+            raise DatasetLoadError(
+                f"Unsupported format: {path.suffix}. Use .json or .jsonl"
+            )
+        except DatasetLoadError:
+            raise
+        except Exception as e:
+            raise DatasetLoadError(f"Failed to load dataset: {e}") from e
+
+    @staticmethod
+    def _parse_reference_tool_calls(data: dict) -> list[ToolCall] | None:
+        """Parse reference tool calls from data dict."""
+        raw = data.get("reference_tool_calls")
+        if not raw:
+            return None
+        return [ToolCall(**tc) for tc in raw]
+
+    @staticmethod
+    def _load_jsonl(path: Path) -> AgentDataset:
+        """Load JSONL file."""
+        samples = []
+
+        with open(path, encoding="utf-8") as f:
+            for line_num, line in enumerate(f, start=1):
+                line = line.strip()
+                if not line:
+                    continue
+
+                try:
+                    data = json.loads(line)
+                    sample = AgentDatasetLoader._parse_sample(data)
+                    samples.append(sample)
+                except json.JSONDecodeError as e:
+                    raise DatasetLoadError(
+                        f"Invalid JSON on line {line_num}: {e}"
+                    ) from e
+                except Exception as e:
+                    raise DatasetLoadError(
+                        f"Error parsing sample on line {line_num}: {e}"
+                    ) from e
+
+        if not samples:
+            raise DatasetLoadError(f"No valid samples found in {path}")
+
+        return AgentDataset(samples=samples)
+
+    @staticmethod
+    def _load_json(path: Path) -> AgentDataset:
+        """Load JSON file."""
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+
+        if not isinstance(data, dict):
+            raise DatasetLoadError("JSON must be an object with 'samples' array")
+
+        if "samples" not in data:
+            raise DatasetLoadError("JSON must have 'samples' key")
+
+        if not isinstance(data["samples"], list):
+            raise DatasetLoadError("'samples' must be an array")
+
+        samples = [
+            AgentDatasetLoader._parse_sample(s) for s in data["samples"]
+        ]
+
+        if not samples:
+            raise DatasetLoadError("'samples' array is empty")
+
+        return AgentDataset(samples=samples)
+
+    @staticmethod
+    def _parse_sample(data: dict) -> AgentSample | PartialAgentSample:
+        """Parse dict to sample (auto-detect partial vs full)."""
+        if "trace" not in data:
+            return PartialAgentSample(
+                user_input=data["user_input"],
+                reference_outcome=data.get("reference_outcome"),
+                reference_tool_calls=AgentDatasetLoader._parse_reference_tool_calls(data),
+                metadata=data.get("metadata", {}),
+            )
+
+        trace_data = data["trace"]
+        messages = []
+
+        for msg_data in trace_data["messages"]:
+            role = msg_data.get("role", "")
+
+            if role == "human":
+                messages.append(
+                    HumanMessage(content=msg_data.get("content", ""))
+                )
+            elif role == "ai":
+                tool_calls = [
+                    ToolCall(**tc) for tc in msg_data.get("tool_calls", [])
+                ]
+                messages.append(
+                    AIMessage(
+                        content=msg_data.get("content", ""),
+                        tool_calls=tool_calls,
+                    )
+                )
+            elif role == "tool":
+                messages.append(
+                    ToolMessage(
+                        content=msg_data.get("content", ""),
+                        tool_name=msg_data.get("tool_name", ""),
+                        tool_call_id=msg_data.get("tool_call_id"),
+                    )
+                )
+            else:
+                raise ValueError(f"Unknown role: {role}")
+
+        trace = AgentTrace(
+            messages=messages,
+            final_response=trace_data.get("final_response", ""),
+            metadata=trace_data.get("metadata", {}),
+        )
+
+        return AgentSample(
+            user_input=data["user_input"],
+            trace=trace,
+            reference_outcome=data.get("reference_outcome"),
+            reference_tool_calls=AgentDatasetLoader._parse_reference_tool_calls(data),
+            metadata=data.get("metadata", {}),
+        )

floeval/api/dataset_loaders/base.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+
+from floeval.config.schemas.io.dataset import PartialSample, Sample
+
+
+class BaseDatasetLoader(ABC):
+    """Abstract base class for dataset loaders."""
+
+    @classmethod
+    @abstractmethod
+    def to_samples(cls, *args, **kwargs) -> list[Sample]:
+        """Abstractmethod to convert raw data into a list of Sample objects.
+
+        Returns:
+            list[Sample]: A list of Sample objects representing the dataset.
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def to_partial_samples(cls, *args, **kwargs) -> list[PartialSample]:
+        """Abstractmethod to convert raw data into a list of PartialSample objects.
+
+        PartialSample allows empty llm_response field (for PartialDataset creation)
+
+        Returns:
+            list[PartialSample]: A list of PartialSample objects with llm_response field empty.
+        """
+        ...