ragbits-evaluate 0.17.1__tar.gz → 0.19.0__tar.gz

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/CHANGELOG.md

@@ -2,6 +2,28 @@
 
 ## Unreleased
 
+## 0.19.0 (2025-05-27)
+
+### Changed
+
+- ragbits-core updated to version v0.19.0
+
+- Add evals for question answering (#577)
+- Add support for slicing dataset (#576)
+- Separate load and map ops in data loaders (#576)
+
+## 0.18.0 (2025-05-22)
+
+### Changed
+
+- ragbits-core updated to version v0.18.0
+
+- Add support for custom column names in evaluation dataset (#566)
+- Add support for reference document ids and page numbers in evaluation dataset (#566)
+- BREAKING CHANGE: Adjust eval pipline interface to batch processing (#555)
+- Rename DocumentMeta create_text_document_from_literal to from_literal (#561)
+- Adjust typing for DocumentSearch (#554)
+
 ## 0.17.1 (2025-05-09)
 
 ### Changed

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragbits-evaluate
-Version: 0.17.1
+Version: 0.19.0
 Summary: Evaluation module for Ragbits components
 Project-URL: Homepage, https://github.com/deepsense-ai/ragbits
 Project-URL: Bug Reports, https://github.com/deepsense-ai/ragbits/issues
@@ -27,7 +27,7 @@ Requires-Dist: distilabel<2.0.0,>=1.4.1
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: neptune[optuna]<2.0.0,>=1.12.0
 Requires-Dist: optuna<5.0.0,>=4.0.0
-Requires-Dist: ragbits-core==0.17.1
+Requires-Dist: ragbits-core==0.19.0
 Provides-Extra: relari
 Requires-Dist: continuous-eval<1.0.0,>=0.3.12; extra == 'relari'
 Description-Content-Type: text/markdown

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ragbits-evaluate"
-version = "0.17.1"
+version = "0.19.0"
 description = "Evaluation module for Ragbits components"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -32,7 +32,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
-dependencies = ["hydra-core>=1.3.2,<2.0.0", "neptune[optuna]>=1.12.0,<2.0.0", "optuna>=4.0.0,<5.0.0", "distilabel>=1.4.1,<2.0.0", "datasets>=3.0.1,<4.0.0", "ragbits-core==0.17.1"]
+dependencies = ["hydra-core>=1.3.2,<2.0.0", "neptune[optuna]>=1.12.0,<2.0.0", "optuna>=4.0.0,<5.0.0", "distilabel>=1.4.1,<2.0.0", "datasets>=3.0.1,<4.0.0", "ragbits-core==0.19.0"]
 
 [project.urls]
 "Homepage" = "https://github.com/deepsense-ai/ragbits"

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/src/ragbits/evaluate/cli.py

@@ -140,10 +140,10 @@ def run() -> None:
         metric_results = await evaluator.compute(
             pipeline=state.pipeline,
             dataloader=state.dataloader,
-            metrics=state.metrics,
+            metricset=state.metrics,
         )
         evaluation_results = EvaluationResult(
-            metrics={"metrics": metric_results["metrics"], "time_perf": metric_results["time_perf"]}
+            metrics={"metrics": metric_results.metrics, "time_perf": metric_results.time_perf}
         )
         print_output(evaluation_results)
 

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/src/ragbits/evaluate/dataloaders/base.py

@@ -3,12 +3,14 @@ from collections.abc import Iterable
 from types import ModuleType
 from typing import ClassVar, Generic
 
+from datasets import load_dataset
 from pydantic import BaseModel
 from typing_extensions import Self
 
 from ragbits.core.sources.base import Source
 from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
 from ragbits.evaluate import dataloaders
+from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
 from ragbits.evaluate.pipelines.base import EvaluationDataT
 
 
@@ -28,14 +30,19 @@ class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
     default_module: ClassVar[ModuleType | None] = dataloaders
     configuration_key: ClassVar[str] = "dataloader"
 
-    def __init__(self, source: Source) -> None:
+    def __init__(self, source: Source, *, split: str = "data", required_keys: set[str] | None = None) -> None:
         """
         Initialize the data loader.
 
         Args:
             source: The source to load the evaluation data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
+            required_keys: The required columns for the evaluation data.
         """
         self.source = source
+        self.split = split
+        self.required_keys = required_keys or set()
 
     @classmethod
     def from_config(cls, config: dict) -> Self:
@@ -52,11 +59,37 @@ class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
         config["source"] = Source.subclass_from_config(dataloader_config.source)
         return super().from_config(config)
 
-    @abstractmethod
     async def load(self) -> Iterable[EvaluationDataT]:
         """
         Load the data.
 
         Returns:
-            The loaded data.
+            The loaded evaluation data.
+
+        Raises:
+            DataLoaderIncorrectFormatDataError: If evaluation dataset is incorrectly formatted.
+        """
+        data_path = await self.source.fetch()
+        dataset = load_dataset(
+            path=str(data_path.parent),
+            data_files={"data": str(data_path.name)},
+            split=self.split,
+        )
+        if not self.required_keys.issubset(dataset.features):
+            raise DataLoaderIncorrectFormatDataError(
+                required_features=list(self.required_keys),
+                data_path=data_path,
+            )
+        return await self.map(dataset.to_list())
+
+    @abstractmethod
+    async def map(self, dataset: Iterable[dict]) -> Iterable[EvaluationDataT]:
+        """
+        Map the dataset to the evaluation data.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The evaluation data.
         """

ragbits_evaluate-0.19.0/src/ragbits/evaluate/dataloaders/document_search.py

@@ -0,0 +1,61 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.document_search import DocumentSearchData
+
+
+class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
+    """
+    Document search evaluation data loader.
+
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        document_ids_key: str = "document_ids",
+        passages_key: str = "passages",
+        page_numbers_key: str = "page_numbers",
+    ) -> None:
+        """
+        Initialize the document search data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
+            question_key: The dataset column name that contains the question.
+            document_ids_key: The dataset column name that contains the document ids. Document ids are optional.
+            passages_key: The dataset column name that contains the passages. Passages are optional.
+            page_numbers_key: The dataset column name that contains the page numbers. Page numbers are optional.
+        """
+        super().__init__(source=source, split=split, required_keys={question_key})
+        self.question_key = question_key
+        self.document_ids_key = document_ids_key
+        self.passages_key = passages_key
+        self.page_numbers_key = page_numbers_key
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[DocumentSearchData]:
+        """
+        Map the dataset to the document search data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The document search data.
+        """
+        return [
+            DocumentSearchData(
+                question=data.get(self.question_key, ""),
+                reference_document_ids=data.get(self.document_ids_key),
+                reference_passages=data.get(self.passages_key),
+                reference_page_numbers=data.get(self.page_numbers_key),
+            )
+            for data in dataset
+        ]

ragbits_evaluate-0.19.0/src/ragbits/evaluate/dataloaders/question_answer.py

@@ -0,0 +1,57 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.question_answer import QuestionAnswerData
+
+
+class QuestionAnswerDataLoader(DataLoader[QuestionAnswerData]):
+    """
+    Question answer evaluation data loader.
+
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        answer_key: str = "answer",
+        context_key: str = "context",
+    ) -> None:
+        """
+        Initialize the question answer data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from.
+            required_keys: The required keys to load the data from.
+            question_key: The dataset column name that contains the question.
+            answer_key: The dataset column name that contains the answer.
+            context_key: The dataset column name that contains the context. Context is optional.
+        """
+        super().__init__(source=source, split=split, required_keys={question_key, answer_key})
+        self.question_key = question_key
+        self.answer_key = answer_key
+        self.context_key = context_key
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[QuestionAnswerData]:
+        """
+        Map the dataset to the question answer data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The question answer data.
+        """
+        return [
+            QuestionAnswerData(
+                question=data.get(self.question_key, ""),
+                reference_answer=data.get(self.answer_key, ""),
+                reference_context=data.get(self.context_key),
+            )
+            for data in dataset
+        ]

ragbits_evaluate-0.19.0/src/ragbits/evaluate/evaluator.py

@@ -0,0 +1,224 @@
+import asyncio
+import random
+import time
+from collections.abc import Awaitable, Callable, Iterable
+from dataclasses import dataclass
+from typing import Generic, ParamSpec, TypeVar
+
+from pydantic import BaseModel
+from tqdm import tqdm
+
+from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
+from ragbits.core.utils.helpers import batched
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.metrics.base import MetricSet
+from ragbits.evaluate.pipelines.base import EvaluationDataT, EvaluationPipeline, EvaluationResultT, EvaluationTargetT
+
+_CallP = ParamSpec("_CallP")
+_CallReturnT = TypeVar("_CallReturnT")
+
+
+@dataclass
+class EvaluationTimePerf:
+    """
+    Container for evaluation time performance metrics.
+    """
+
+    total_time_in_seconds: float
+    samples_per_second: float
+    latency_in_seconds: float
+
+
+@dataclass
+class EvaluatorResult(Generic[EvaluationResultT]):
+    """
+    Container for evaluation results.
+    """
+
+    metrics: dict[str, int | float]
+    results: list[EvaluationResultT]
+    errors: list[Exception]
+    time_perf: EvaluationTimePerf
+
+
+class EvaluationConfig(BaseModel):
+    """
+    Schema for the evaluation run config.
+    """
+
+    pipeline: ObjectConstructionConfig
+    dataloader: ObjectConstructionConfig
+    metrics: dict[str, ObjectConstructionConfig]
+
+
+class EvaluatorConfig(BaseModel):
+    """
+    Schema for the evaluator config.
+    """
+
+    evaluation: EvaluationConfig
+    evaluator: dict | None = None
+
+
+class Evaluator(WithConstructionConfig):
+    """
+    Evaluator class.
+    """
+
+    def __init__(
+        self,
+        batch_size: int = 10,
+        num_retries: int = 3,
+        backoff_multiplier: int = 1,
+        backoff_max: int = 60,
+    ) -> None:
+        """
+        Initialize the Evaluator instance.
+
+        Args:
+            batch_size: batch size for the evaluation pipeline inference.
+            num_retries: The number of retries per evaluation pipeline inference error.
+            backoff_multiplier: The base delay multiplier for exponential backoff (in seconds).
+            backoff_max: The maximum allowed delay (in seconds) between retries.
+        """
+        self.batch_size = batch_size
+        self.num_retries = num_retries
+        self.backoff_multiplier = backoff_multiplier
+        self.backoff_max = backoff_max
+
+    @classmethod
+    async def run_from_config(cls, config: dict) -> EvaluatorResult:
+        """
+        Run the evaluation based on configuration.
+
+        Args:
+            config: Evaluation config.
+
+        Returns:
+            The evaluation results.
+        """
+        evaluator_config = EvaluatorConfig.model_validate(config)
+        evaluation_config = EvaluationConfig.model_validate(evaluator_config.evaluation)
+        pipeline: EvaluationPipeline = EvaluationPipeline.subclass_from_config(evaluation_config.pipeline)
+        dataloader: DataLoader = DataLoader.subclass_from_config(evaluation_config.dataloader)
+        metricset: MetricSet = MetricSet.from_config(evaluation_config.metrics)
+
+        evaluator = cls.from_config(evaluator_config.evaluator or {})
+        return await evaluator.compute(
+            pipeline=pipeline,
+            dataloader=dataloader,
+            metricset=metricset,
+        )
+
+    async def compute(
+        self,
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataloader: DataLoader[EvaluationDataT],
+        metricset: MetricSet[EvaluationResultT],
+    ) -> EvaluatorResult[EvaluationResultT]:
+        """
+        Compute the evaluation results for the given pipeline and data.
+
+        Args:
+            pipeline: The pipeline to be evaluated.
+            dataloader: The dataloader to load the data.
+            metricset: The metrics to be computed.
+
+        Returns:
+            The evaluation results.
+        """
+        await pipeline.prepare()
+
+        dataset = await dataloader.load()
+        results, errors, time_perf = await self._call_pipeline(pipeline, dataset)
+        metrics = await metricset.compute(results)
+
+        return EvaluatorResult(
+            metrics=metrics,
+            results=results,
+            errors=errors,
+            time_perf=time_perf,
+        )
+
+    async def _call_pipeline(
+        self,
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataset: Iterable[EvaluationDataT],
+    ) -> tuple[list[EvaluationResultT], list[Exception], EvaluationTimePerf]:
+        """
+        Call the pipeline with the given data.
+
+        Args:
+            pipeline: The pipeline to be called.
+            dataset: The dataset to be processed.
+
+        Returns:
+            The evaluation results and performance metrics.
+        """
+        start_time = time.perf_counter()
+        outputs = [
+            await self._call_with_error_handling(pipeline, data)
+            for data in tqdm(batched(dataset, self.batch_size), desc="Evaluation")
+        ]
+        end_time = time.perf_counter()
+
+        errors = [output for output in outputs if isinstance(output, Exception)]
+        results = [item for output in outputs if not isinstance(output, Exception) for item in output]
+
+        return results, errors, self._compute_time_perf(start_time, end_time, len(outputs))
+
+    async def _call_with_error_handling(
+        self,
+        executable: Callable[_CallP, Awaitable[_CallReturnT]],
+        *executable_args: _CallP.args,
+        **executable_kwargs: _CallP.kwargs,
+    ) -> _CallReturnT | Exception:
+        """
+        Call executable with a standarized error handling.
+        If an error occurs, the executable is retried `num_retries` times using randomized exponential backoff.
+
+        Args:
+            executable: The callable function to execute.
+            executable_args: Positional arguments to pass to the executable.
+            executable_kwargs: Keyword arguments to pass to the executable.
+
+        Returns:
+            The result of the executable if successful.
+
+        Raises:
+            Exception: The last encountered exception after all retries are exhausted.
+        """
+        for i in range(max(0, self.num_retries) + 1):
+            try:
+                return await executable(*executable_args, **executable_kwargs)
+            except Exception as exc:
+                if i == self.num_retries:
+                    return exc
+
+                delay = random.uniform(0, min(2**i * self.backoff_multiplier, self.backoff_max))  # noqa: S311
+                await asyncio.sleep(delay)
+
+        raise RuntimeError("Unreachable code reached")  # mypy quirk
+
+    @staticmethod
+    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> EvaluationTimePerf:
+        """
+        Compute the performance metrics.
+
+        Args:
+            start_time: The start time.
+            end_time: The end time.
+            num_samples: The number of samples.
+
+        Returns:
+            The performance metrics.
+        """
+        latency = end_time - start_time
+        throughput = num_samples / latency
+        latency_sample = 1.0 / throughput if throughput > 0 else 0.0
+
+        return EvaluationTimePerf(
+            total_time_in_seconds=latency,
+            samples_per_second=throughput,
+            latency_in_seconds=latency_sample,
+        )

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/src/ragbits/evaluate/factories/__init__.py

@@ -1,43 +1,21 @@
 import asyncio
 
+from continuous_eval.metrics.retrieval.matching_strategy import RougeChunkMatch
 from datasets import load_dataset
 
 from ragbits.core.embeddings.dense import LiteLLMEmbedder
 from ragbits.core.sources.hf import HuggingFaceSource
-from ragbits.core.utils.config_handling import ObjectConstructionConfig
 from ragbits.core.vector_stores.in_memory import InMemoryVectorStore
 from ragbits.document_search import DocumentSearch
 from ragbits.document_search.documents.document import DocumentMeta
 from ragbits.evaluate.dataloaders.document_search import DocumentSearchDataLoader
 from ragbits.evaluate.metrics import MetricSet
-
-DS_PRECISION_RECALL_F1 = {
-    "precision_recall_f1": ObjectConstructionConfig.model_validate(
-        {
-            "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1",
-            "config": {
-                "matching_strategy": {
-                    "type": "RougeChunkMatch",
-                    "config": {
-                        "threshold": 0.5,
-                    },
-                },
-            },
-        }
-    ),
-}
-
-
-def precision_recall_f1() -> MetricSet:
-    """
-    Factory of precision recall f1 metric set for retrival evaluation.
-    """
-    return MetricSet.from_config(config=DS_PRECISION_RECALL_F1)
+from ragbits.evaluate.metrics.document_search import DocumentSearchPrecisionRecallF1
 
 
 async def _add_example_documents(document_search: DocumentSearch) -> None:
     dataset = load_dataset(path="deepsense-ai/synthetic-rag-dataset_v1.0", split="train")
-    documents = [DocumentMeta.create_text_document_from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
+    documents = [DocumentMeta.from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
     await document_search.ingest(documents)
 
 
@@ -45,7 +23,7 @@ def basic_document_search_factory() -> DocumentSearch:
     """
     Factory for basic example document search instance.
     """
-    document_search = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
+    document_search: DocumentSearch = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
     asyncio.run(_add_example_documents(document_search))
     return document_search
 
@@ -55,3 +33,10 @@ def synthetic_rag_dataset() -> DocumentSearchDataLoader:
     Factory for synthetic RAG dataset.
     """
     return DocumentSearchDataLoader(source=HuggingFaceSource(path="deepsense-ai/synthetic-rag-dataset_v1.0"))
+
+
+def precision_recall_f1() -> MetricSet:
+    """
+    Factory of precision recall f1 metric set for retrival evaluation.
+    """
+    return MetricSet(DocumentSearchPrecisionRecallF1(matching_strategy=RougeChunkMatch()))

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/src/ragbits/evaluate/metrics/base.py

@@ -1,3 +1,4 @@
+import asyncio
 from abc import ABC, abstractmethod
 from types import ModuleType
 from typing import ClassVar, Generic
@@ -19,7 +20,7 @@ class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
 
     def __init__(self, weight: float = 1.0) -> None:
         """
-        Initializes the metric.
+        Initialize the metric.
 
         Args:
             weight: Metric value weight in the final score, used during optimization.
@@ -28,7 +29,7 @@ class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
         self.weight = weight
 
     @abstractmethod
-    def compute(self, results: list[EvaluationResultT]) -> dict:
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metric.
 
@@ -70,7 +71,7 @@ class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
         """
         return cls(*[Metric.subclass_from_config(metric_config) for metric_config in config.values()])
 
-    def compute(self, results: list[EvaluationResultT]) -> dict:
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metrics.
 
@@ -80,6 +81,9 @@ class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
         Returns:
             The computed metrics.
         """
+        metric_results = await asyncio.gather(*[metric.compute(results) for metric in self.metrics])
         return {
-            name: metric.weight * value for metric in self.metrics for name, value in metric.compute(results).items()
+            name: metric.weight * value
+            for metric, result in zip(self.metrics, metric_results, strict=False)
+            for name, value in result.items()
         }

{ragbits_evaluate-0.17.1 → ragbits_evaluate-0.19.0}/src/ragbits/evaluate/metrics/document_search.py

@@ -46,7 +46,7 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
         matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
         return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))
 
-    def compute(self, results: list[DocumentSearchResult]) -> dict:
+    async def compute(self, results: list[DocumentSearchResult]) -> dict:
         """
         Compute the metric.
 
@@ -57,7 +57,18 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
             The computed metric.
         """
         return self.metric.aggregate(
-            [self.metric(result.predicted_passages, result.reference_passages) for result in results]
+            [
+                self.metric(
+                    [
+                        element.text_representation
+                        for element in result.predicted_elements
+                        if element.text_representation
+                    ],
+                    result.reference_passages,
+                )
+                for result in results
+                if result.reference_passages is not None
+            ]
         )