ragbits-evaluate 0.0.30.dev29302392__py3-none-any.whl

ragbits/evaluate/evaluator.py

@@ -0,0 +1,244 @@
+import asyncio
+import random
+import time
+from collections.abc import Awaitable, Callable, Iterable, Sized
+from dataclasses import dataclass
+from typing import Generic, ParamSpec, TypeVar
+
+from pydantic import BaseModel
+from tqdm import tqdm
+
+from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
+from ragbits.core.utils.helpers import batched
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.metrics.base import MetricSet
+from ragbits.evaluate.pipelines.base import EvaluationDataT, EvaluationPipeline, EvaluationResultT, EvaluationTargetT
+
+_CallP = ParamSpec("_CallP")
+_CallReturnT = TypeVar("_CallReturnT")
+
+
+@dataclass
+class EvaluationTimePerf:
+    """
+    Container for evaluation time performance metrics.
+    """
+
+    total_time_in_seconds: float
+    samples_per_second: float
+    latency_in_seconds: float
+
+
+@dataclass
+class EvaluatorResult(Generic[EvaluationResultT]):
+    """
+    Container for evaluation results.
+    """
+
+    metrics: dict[str, int | float]
+    results: list[EvaluationResultT]
+    errors: list[Exception]
+    time_perf: EvaluationTimePerf
+
+
+class EvaluationConfig(BaseModel):
+    """
+    Schema for the evaluation run config.
+    """
+
+    pipeline: ObjectConstructionConfig
+    dataloader: ObjectConstructionConfig
+    metrics: dict[str, ObjectConstructionConfig]
+
+
+class EvaluatorConfig(BaseModel):
+    """
+    Schema for the evaluator config.
+    """
+
+    evaluation: EvaluationConfig
+    evaluator: dict | None = None
+
+
+class Evaluator(WithConstructionConfig):
+    """
+    Evaluator class.
+    """
+
+    def __init__(
+        self,
+        batch_size: int = 10,
+        num_retries: int = 3,
+        backoff_multiplier: int = 1,
+        backoff_max: int = 60,
+        parallelize_batches: bool = False,
+    ) -> None:
+        """
+        Initialize the Evaluator instance.
+
+        Args:
+            batch_size: batch size for the evaluation pipeline inference.
+            num_retries: The number of retries per evaluation pipeline inference error.
+            backoff_multiplier: The base delay multiplier for exponential backoff (in seconds).
+            backoff_max: The maximum allowed delay (in seconds) between retries.
+            parallelize_batches: Whether to process samples within each batch in parallel (asyncio.gather).
+        """
+        self.batch_size = batch_size
+        self.num_retries = num_retries
+        self.backoff_multiplier = backoff_multiplier
+        self.backoff_max = backoff_max
+        self.parallelize_batches = parallelize_batches
+
+    @classmethod
+    async def run_from_config(cls, config: dict) -> EvaluatorResult:
+        """
+        Run the evaluation based on configuration.
+
+        Args:
+            config: Evaluation config.
+
+        Returns:
+            The evaluation results.
+        """
+        evaluator_config = EvaluatorConfig.model_validate(config)
+        evaluation_config = EvaluationConfig.model_validate(evaluator_config.evaluation)
+        pipeline: EvaluationPipeline = EvaluationPipeline.subclass_from_config(evaluation_config.pipeline)
+        dataloader: DataLoader = DataLoader.subclass_from_config(evaluation_config.dataloader)
+        metricset: MetricSet = MetricSet.from_config(evaluation_config.metrics)
+
+        evaluator = cls.from_config(evaluator_config.evaluator or {})
+        return await evaluator.compute(
+            pipeline=pipeline,
+            dataloader=dataloader,
+            metricset=metricset,
+        )
+
+    async def compute(
+        self,
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataloader: DataLoader[EvaluationDataT],
+        metricset: MetricSet[EvaluationResultT],
+    ) -> EvaluatorResult[EvaluationResultT]:
+        """
+        Compute the evaluation results for the given pipeline and data.
+
+        Args:
+            pipeline: The pipeline to be evaluated.
+            dataloader: The dataloader to load the data.
+            metricset: The metrics to be computed.
+
+        Returns:
+            The evaluation results.
+        """
+        await pipeline.prepare()
+
+        dataset = await dataloader.load()
+        results, errors, time_perf = await self._call_pipeline(pipeline, dataset)
+        metrics = await metricset.compute(results)
+
+        return EvaluatorResult(
+            metrics=metrics,
+            results=results,
+            errors=errors,
+            time_perf=time_perf,
+        )
+
+    async def _call_pipeline(
+        self,
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataset: Iterable[EvaluationDataT],
+    ) -> tuple[list[EvaluationResultT], list[Exception], EvaluationTimePerf]:
+        """
+        Call the pipeline with the given data.
+
+        Args:
+            pipeline: The pipeline to be called.
+            dataset: The dataset to be processed.
+
+        Returns:
+            The evaluation results and performance metrics.
+        """
+        start_time = time.perf_counter()
+
+        total_samples = len(dataset) if isinstance(dataset, Sized) else None
+        batches = batched(dataset, self.batch_size)
+        outputs: list[Iterable[EvaluationResultT] | Exception] = []
+
+        with tqdm(total=total_samples, desc="Evaluation", unit="sample") as progress_bar:
+            for batch in batches:
+                batch_list = list(batch)
+
+                if self.parallelize_batches:
+                    tasks = [self._call_with_error_handling(pipeline, [sample]) for sample in batch_list]
+                    batch_results = await asyncio.gather(*tasks)
+
+                    for result in batch_results:
+                        outputs.append(result)
+                        progress_bar.update(1)
+                else:
+                    result = await self._call_with_error_handling(pipeline, batch_list)
+                    outputs.append(result)
+                    progress_bar.update(len(batch_list))
+
+        end_time = time.perf_counter()
+
+        errors = [output for output in outputs if isinstance(output, Exception)]
+        results = [item for output in outputs if not isinstance(output, Exception) for item in output]
+
+        return results, errors, self._compute_time_perf(start_time, end_time, len(results))
+
+    async def _call_with_error_handling(
+        self,
+        executable: Callable[_CallP, Awaitable[_CallReturnT]],
+        *executable_args: _CallP.args,
+        **executable_kwargs: _CallP.kwargs,
+    ) -> _CallReturnT | Exception:
+        """
+        Call executable with a standarized error handling.
+        If an error occurs, the executable is retried `num_retries` times using randomized exponential backoff.
+
+        Args:
+            executable: The callable function to execute.
+            executable_args: Positional arguments to pass to the executable.
+            executable_kwargs: Keyword arguments to pass to the executable.
+
+        Returns:
+            The result of the executable if successful.
+
+        Raises:
+            Exception: The last encountered exception after all retries are exhausted.
+        """
+        for i in range(max(0, self.num_retries) + 1):
+            try:
+                return await executable(*executable_args, **executable_kwargs)
+            except Exception as exc:
+                if i == self.num_retries:
+                    return exc
+
+                delay = random.uniform(0, min(2**i * self.backoff_multiplier, self.backoff_max))  # noqa: S311
+                await asyncio.sleep(delay)
+
+        raise RuntimeError("Unreachable code reached")  # mypy quirk
+
+    @staticmethod
+    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> EvaluationTimePerf:
+        """
+        Compute the performance metrics.
+
+        Args:
+            start_time: The start time.
+            end_time: The end time.
+            num_samples: The number of samples.
+
+        Returns:
+            The performance metrics.
+        """
+        latency = end_time - start_time
+        throughput = num_samples / latency
+        latency_sample = 1.0 / throughput if throughput > 0 else 0.0
+
+        return EvaluationTimePerf(
+            total_time_in_seconds=latency,
+            samples_per_second=throughput,
+            latency_in_seconds=latency_sample,
+        )

ragbits/evaluate/factories/__init__.py

@@ -0,0 +1,42 @@
+import asyncio
+
+from continuous_eval.metrics.retrieval.matching_strategy import RougeChunkMatch
+from datasets import load_dataset
+
+from ragbits.core.embeddings.dense import LiteLLMEmbedder
+from ragbits.core.sources.hf import HuggingFaceSource
+from ragbits.core.vector_stores.in_memory import InMemoryVectorStore
+from ragbits.document_search import DocumentSearch
+from ragbits.document_search.documents.document import DocumentMeta
+from ragbits.evaluate.dataloaders.document_search import DocumentSearchDataLoader
+from ragbits.evaluate.metrics import MetricSet
+from ragbits.evaluate.metrics.document_search import DocumentSearchPrecisionRecallF1
+
+
+async def _add_example_documents(document_search: DocumentSearch) -> None:
+    dataset = load_dataset(path="deepsense-ai/synthetic-rag-dataset_v1.0", split="train")
+    documents = [DocumentMeta.from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
+    await document_search.ingest(documents)
+
+
+def basic_document_search_factory() -> DocumentSearch:
+    """
+    Factory for basic example document search instance.
+    """
+    document_search: DocumentSearch = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
+    asyncio.run(_add_example_documents(document_search))
+    return document_search
+
+
+def synthetic_rag_dataset() -> DocumentSearchDataLoader:
+    """
+    Factory for synthetic RAG dataset.
+    """
+    return DocumentSearchDataLoader(source=HuggingFaceSource(path="deepsense-ai/synthetic-rag-dataset_v1.0"))
+
+
+def precision_recall_f1() -> MetricSet:
+    """
+    Factory of precision recall f1 metric set for retrival evaluation.
+    """
+    return MetricSet(DocumentSearchPrecisionRecallF1(matching_strategy=RougeChunkMatch()))

ragbits/evaluate/metrics/__init__.py

@@ -0,0 +1,3 @@
+from ragbits.evaluate.metrics.base import Metric, MetricSet
+
+__all__ = ["Metric", "MetricSet"]

ragbits/evaluate/metrics/base.py

@@ -0,0 +1,89 @@
+import asyncio
+from abc import ABC, abstractmethod
+from types import ModuleType
+from typing import ClassVar, Generic
+
+from typing_extensions import Self
+
+from ragbits.core.utils.config_handling import WithConstructionConfig
+from ragbits.evaluate import metrics
+from ragbits.evaluate.pipelines.base import EvaluationResultT
+
+
+class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
+    """
+    Base class for metrics.
+    """
+
+    default_module: ClassVar[ModuleType | None] = metrics
+    configuration_key: ClassVar[str] = "metric"
+
+    def __init__(self, weight: float = 1.0) -> None:
+        """
+        Initialize the metric.
+
+        Args:
+            weight: Metric value weight in the final score, used during optimization.
+        """
+        super().__init__()
+        self.weight = weight
+
+    @abstractmethod
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
+        """
+        Compute the metric.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+
+
+class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
+    """
+    Represents a set of metrics.
+    """
+
+    configuration_key: ClassVar[str] = "metrics"
+    default_module: ClassVar[ModuleType | None] = metrics
+
+    def __init__(self, *metrics: Metric[EvaluationResultT]) -> None:
+        """
+        Initialize the metric set.
+
+        Args:
+            metrics: The metrics.
+        """
+        self.metrics = metrics
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `MetricSet` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the metric set.
+
+        Returns:
+            An instance of the metric set class initialized with the provided configuration.
+        """
+        return cls(*[Metric.subclass_from_config(metric_config) for metric_config in config.values()])
+
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
+        """
+        Compute the metrics.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metrics.
+        """
+        metric_results = await asyncio.gather(*[metric.compute(results) for metric in self.metrics])
+        return {
+            name: metric.weight * value
+            for metric, result in zip(self.metrics, metric_results, strict=False)
+            for name, value in result.items()
+        }

ragbits/evaluate/metrics/document_search.py

@@ -0,0 +1,90 @@
+import importlib
+from abc import ABC
+
+from continuous_eval.metrics.retrieval import PrecisionRecallF1, RankedRetrievalMetrics
+from continuous_eval.metrics.retrieval.matching_strategy import MatchingStrategy
+from typing_extensions import Self
+
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.document_search import DocumentSearchResult
+
+
+class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
+    """
+    Metric for document search evaluation based on Relari backend.
+    More details can be found [here](https://docs.relari.ai/category/retrieval-rag).
+    """
+
+    metric_cls: type[PrecisionRecallF1 | RankedRetrievalMetrics]
+
+    def __init__(self, matching_strategy: MatchingStrategy, weight: float = 1.0) -> None:
+        """
+        Initialize the document search metric.
+
+        Args:
+            matching_strategy: Matching strategys that determine relevance.
+            weight: Metric value weight in the final score, used during optimization.
+        """
+        super().__init__(weight=weight)
+        self.metric = self.metric_cls(matching_strategy)
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `DocumentSearchMetric` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the metric.
+
+        Returns:
+            An instance of the metric class initialized with the provided configuration.
+        """
+        matching_strategy_cls = getattr(
+            importlib.import_module("continuous_eval.metrics.retrieval.matching_strategy"),
+            config["matching_strategy"]["type"],
+        )
+        matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
+        return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))
+
+    async def compute(self, results: list[DocumentSearchResult]) -> dict:
+        """
+        Compute the metric.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+        return self.metric.aggregate(
+            [
+                self.metric(
+                    [
+                        element.text_representation
+                        for element in result.predicted_elements
+                        if element.text_representation
+                    ],
+                    result.reference_passages,
+                )
+                for result in results
+                if result.reference_passages is not None
+            ]
+        )
+
+
+class DocumentSearchPrecisionRecallF1(DocumentSearchMetric):
+    """
+    Precision, recall, and F1 score for context retrieval.
+    More details can be found [here](https://docs.relari.ai/metrics/Retrieval/Deterministic/precision_recall).
+    """
+
+    metric_cls = PrecisionRecallF1
+
+
+class DocumentSearchRankedRetrievalMetrics(DocumentSearchMetric):
+    """
+    Rank-aware metrics takes into account the order in which the contexts are retrieved.
+    More details can be found [here](https://docs.relari.ai/metrics/Retrieval/Deterministic/rank_aware_metrics).
+    """
+
+    metric_cls = RankedRetrievalMetrics

ragbits/evaluate/metrics/gaia.py

@@ -0,0 +1,84 @@
+from statistics import mean
+
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.gaia import GaiaResult
+
+
+class GaiaOutcome(Metric[GaiaResult]):
+    """
+    Computes task success rate over GAIA tasks.
+    Measures the fraction of tasks that were successfully solved.
+    """
+
+    @staticmethod
+    async def compute(results: list[GaiaResult]) -> dict:
+        """Compute task success rate.
+
+        Returns:
+            Dictionary with gaia_task_success_rate: fraction of successfully solved tasks.
+        """
+        success_count = sum(1 for r in results if r.task_success)
+        success_rate = (success_count / len(results)) if results else 0.0
+
+        return {"gaia_task_success_rate": float(success_rate)}
+
+
+class GaiaTooling(Metric[GaiaResult]):
+    """
+    Tool utilization and performance metrics:
+    - gaia_tool_trigger_rate: fraction of tasks where tools were used
+    - gaia_avg_num_tool_calls: average number of tool calls per task
+    - gaia_avg_tool_error_count: average number of tool errors per task
+    - averaged_freq: average tool usage/calls per task
+    """
+
+    @staticmethod
+    async def compute(results: list[GaiaResult]) -> dict:
+        """Compute tool utilization and performance metrics.
+
+        Returns:
+            Dictionary with tool trigger rate, average tool calls, average errors,
+            and flattened tool frequency usage as numeric metrics.
+        """
+        tool_triggered_count = sum(1 for r in results if r.tool_triggered)
+        tool_trigger_rate = (tool_triggered_count / len(results)) if results else 0.0
+        avg_tool_calls = float(mean(r.num_tool_calls for r in results)) if results else 0.0
+        avg_tool_errors = float(mean(r.tool_error_count for r in results)) if results else 0.0
+
+        # tool frequency as average per task (mean calls per task per tool)
+        total_tasks = len(results) if results else 1
+        aggregated_counts: dict[str, int] = {}
+        for r in results:
+            if r.tool_names:
+                for name in r.tool_names:
+                    aggregated_counts[name] = aggregated_counts.get(name, 0) + 1
+        averaged_freq: dict[str, float] = {
+            f"gaia_tool_frequency_usage.{name}": (count / total_tasks) for name, count in aggregated_counts.items()
+        }
+
+        return {
+            "gaia_tool_trigger_rate": float(tool_trigger_rate),
+            "gaia_avg_num_tool_calls": avg_tool_calls,
+            "gaia_avg_tool_error_count": avg_tool_errors,
+            **averaged_freq,
+        }
+
+
+class GaiaEfficiency(Metric[GaiaResult]):
+    """
+    Efficiency and resource usage metrics:
+    - gaia_avg_latency_ms: average response latency in milliseconds
+    """
+
+    @staticmethod
+    async def compute(results: list[GaiaResult]) -> dict:
+        """Compute efficiency and resource usage metrics.
+
+        Returns:
+            Dictionary with average latency.
+        """
+        avg_latency = float(mean(r.total_latency_ms for r in results)) if results else 0.0
+
+        return {
+            "gaia_avg_latency_ms": avg_latency,
+        }

ragbits/evaluate/metrics/hotpot_qa.py

@@ -0,0 +1,51 @@
+from collections import defaultdict
+from collections.abc import Iterable
+
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.hotpot_qa import HotpotQAResult
+
+
+class HotpotQAExactMatch(Metric[HotpotQAResult]):
+    """Computes EM over HotpotQA by type and overall."""
+
+    @staticmethod
+    async def compute(results: list[HotpotQAResult]) -> dict:
+        """Compute EM. Returns hotpotqa_<type>_em and hotpotqa_overall_em."""
+        buckets: dict[str, list[float]] = defaultdict(list)
+        for r in results:
+            em = r.em_value
+            t = r.qtype or "unknown"
+            buckets[t].append(em)
+            buckets["overall"].append(em)
+
+        def avg(vals: Iterable[float]) -> float:
+            lst = list(vals)
+            return float(sum(lst) / len(lst)) if lst else 0.0
+
+        metrics: dict[str, float] = {}
+        for t, vals in buckets.items():
+            metrics[f"hotpotqa_{t}_em"] = avg(vals)
+        return metrics
+
+
+class HotpotQAF1(Metric[HotpotQAResult]):
+    """Computes token-level F1 over HotpotQA by type and overall."""
+
+    @staticmethod
+    async def compute(results: list[HotpotQAResult]) -> dict:
+        """Compute F1. Returns hotpotqa_<type>_f1 and hotpotqa_overall_f1."""
+        buckets: dict[str, list[float]] = defaultdict(list)
+        for r in results:
+            f1v = r.f1_value
+            t = r.qtype or "unknown"
+            buckets[t].append(f1v)
+            buckets["overall"].append(f1v)
+
+        def avg(vals: Iterable[float]) -> float:
+            lst = list(vals)
+            return float(sum(lst) / len(lst)) if lst else 0.0
+
+        metrics: dict[str, float] = {}
+        for t, vals in buckets.items():
+            metrics[f"hotpotqa_{t}_f1"] = avg(vals)
+        return metrics