ragbits-evaluate 0.16.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

ragbits/evaluate/cli.py

@@ -8,12 +8,13 @@ from pydantic import BaseModel
 
 from ragbits.cli._utils import get_instance_or_exit
 from ragbits.cli.state import print_output
-from ragbits.core.utils.config_handling import WithConstructionConfig, import_by_path
+from ragbits.core.utils.config_handling import WithConstructionConfig
 from ragbits.evaluate.config import eval_config
-from ragbits.evaluate.dataloaders import DataLoader, get_dataloader_instance
+from ragbits.evaluate.dataloaders import DataLoader
 from ragbits.evaluate.evaluator import Evaluator
 from ragbits.evaluate.metrics.base import MetricSet
 from ragbits.evaluate.pipelines import get_evaluation_pipeline_for_target
+from ragbits.evaluate.pipelines.base import EvaluationPipeline
 
 eval_app = typer.Typer(no_args_is_help=True)
 
@@ -30,9 +31,9 @@ def register(app: typer.Typer) -> None:
 
 @dataclass
 class _CLIState:
-    evaluation_target: WithConstructionConfig | None = None
-    metrics: MetricSet | None = None
     dataloader: DataLoader | None = None
+    pipeline: EvaluationPipeline | None = None
+    metrics: MetricSet | None = None
 
 
 class EvaluationResult(BaseModel):
@@ -46,26 +47,18 @@ state: _CLIState = _CLIState()
 
 @eval_app.callback()
 def common_args(
-    target_cls: Annotated[
-        str,
-        typer.Option(
-            help="A path to target class to be evaluated in a format python.path:ModuleName",
-            exists=True,
-            resolve_path=True,
-        ),
-    ],
-    dataloader_args: Annotated[
-        str,
+    dataloader_factory_path: Annotated[
+        str | None,
         typer.Option(
-            help="Comma separated arguments of dataloader",
+            help="A path to evaluation data loader factory in format python.path:function_name",
             exists=True,
             resolve_path=True,
         ),
-    ],
-    dataloader_cls: Annotated[
-        str | None,
+    ] = None,
+    dataloader_yaml_path: Annotated[
+        Path | None,
         typer.Option(
-            help="Dataloader class path in a format python.path:ModuleName to override the default",
+            help="A path to evaluation data loader configuration",
             exists=True,
             resolve_path=True,
         ),
@@ -73,7 +66,7 @@ def common_args(
     target_factory_path: Annotated[
         str | None,
         typer.Option(
-            help="A path to a factory of the target class in format: python.path:function_name",
+            help="A path to a factory of the evaluation target class in format: python.path:function_name",
             exists=True,
             resolve_path=True,
         ),
@@ -81,7 +74,7 @@ def common_args(
     target_yaml_path: Annotated[
         Path | None,
         typer.Option(
-            help="A path to a YAML configuration file of the target class",
+            help="A path to a YAML configuration file of the evaluation target class",
             exists=True,
             resolve_path=True,
         ),
@@ -106,40 +99,48 @@ def common_args(
     """
     Common arguments for the evaluate commands.
     """
-    state.evaluation_target = get_instance_or_exit(
-        import_by_path(target_cls),
+    evaluation_target = get_instance_or_exit(
+        cls=WithConstructionConfig,
         factory_path=target_factory_path,
         yaml_path=target_yaml_path,
+        config_override=eval_config,
     )
-    # TODO validate if given metric set is suitable for evaluation target
-    state.metrics = get_instance_or_exit(
-        MetricSet, factory_path=metrics_factory_path, yaml_path=metrics_yaml_path, config_override=eval_config
+    state.pipeline = get_evaluation_pipeline_for_target(evaluation_target)
+    # TODO: validate if given dataloader is suitable for evaluation pipeline
+    state.dataloader = get_instance_or_exit(
+        cls=DataLoader,
+        factory_path=dataloader_factory_path,
+        yaml_path=dataloader_yaml_path,
+        config_override=eval_config,
     )
-    # TODO validate if given dataloader is suitable for evaluation target
-    state.dataloader = get_dataloader_instance(
-        config=eval_config, dataloader_args=dataloader_args, dataloader_cls_override=dataloader_cls
+    # TODO: validate if given metric set is suitable for evaluation pipeline
+    state.metrics = get_instance_or_exit(
+        cls=MetricSet,
+        factory_path=metrics_factory_path,
+        yaml_path=metrics_yaml_path,
+        config_override=eval_config,
     )
 
 
 @eval_app.command()
-def run_evaluation() -> None:
+def run() -> None:
     """
-    Evaluate the set-up pipeline.
+    Evaluate the pipeline.
     """
 
     async def run() -> None:
-        if state.evaluation_target is None:
-            raise ValueError("Evaluation target not initialized")
+        if state.dataloader is None:
+            raise ValueError("Evaluation dataloader not initialized")
+        if state.pipeline is None:
+            raise ValueError("Evaluation pipeline not initialized")
         if state.metrics is None:
             raise ValueError("Evaluation metrics not initialized")
-        if state.dataloader is None:
-            raise ValueError("Dataloader not initialized")
-        evaluation_pipeline = get_evaluation_pipeline_for_target(evaluation_target=state.evaluation_target)
+
         evaluator = Evaluator()
         metric_results = await evaluator.compute(
-            pipeline=evaluation_pipeline,
-            metrics=state.metrics,
+            pipeline=state.pipeline,
             dataloader=state.dataloader,
+            metrics=state.metrics,
         )
         evaluation_results = EvaluationResult(
             metrics={"metrics": metric_results["metrics"], "time_perf": metric_results["time_perf"]}

ragbits/evaluate/config.py

@@ -3,11 +3,9 @@ from ragbits.core.utils._pyproject import get_config_instance
 
 
 class EvaluateConfig(CoreConfig):
-    """Configuration of ragbits evaluate module"""
-
-    component_preference_factories: dict[str, str] = {"metrics": "ragbits.evaluate.factories:precision_recall_f1"}
-
-    dataloader_default_class: str = "ragbits.evaluate.dataloaders.hf:HFDataLoader"
+    """
+    Configuration for the ragbits-evaluate package, loaded from downstream projects' pyproject.toml files.
+    """
 
 
 eval_config = get_config_instance(EvaluateConfig, subproject="evaluate")

ragbits/evaluate/dataloaders/__init__.py

@@ -1,21 +1,3 @@
-from ragbits.core.utils.config_handling import import_by_path
-from ragbits.evaluate.config import EvaluateConfig
 from ragbits.evaluate.dataloaders.base import DataLoader
 
 __all__ = ["DataLoader"]
-
-
-def get_dataloader_instance(
-    config: EvaluateConfig, dataloader_args: str, dataloader_cls_override: str | None = None
-) -> DataLoader:
-    """
-    A function for instantiation of dataloader
-    Args:
-        config: configuration of ragbits.evaluate module
-        dataloader_args: comma separated arguments of dataloader
-        dataloader_cls_override: optional path to override of default dataloader class
-    Returns:
-        DataLoader
-    """
-    dataloader_cls = dataloader_cls_override or config.dataloader_default_class
-    return import_by_path(dataloader_cls)(*dataloader_args.split(","))

ragbits/evaluate/dataloaders/base.py

@@ -1,18 +1,59 @@
 from abc import ABC, abstractmethod
-from typing import Generic, TypeVar
+from collections.abc import Iterable
+from types import ModuleType
+from typing import ClassVar, Generic
 
-from ragbits.core.utils.config_handling import WithConstructionConfig
+from pydantic import BaseModel
+from typing_extensions import Self
 
-DataT = TypeVar("DataT")
+from ragbits.core.sources.base import Source
+from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
+from ragbits.evaluate import dataloaders
+from ragbits.evaluate.pipelines.base import EvaluationDataT
 
 
-class DataLoader(WithConstructionConfig, Generic[DataT], ABC):
+class DataLoaderConfig(BaseModel):
     """
-    Data loader.
+    Schema for the data loader config.
     """
 
+    source: ObjectConstructionConfig
+
+
+class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
+    """
+    Evaluation data loader.
+    """
+
+    default_module: ClassVar[ModuleType | None] = dataloaders
+    configuration_key: ClassVar[str] = "dataloader"
+
+    def __init__(self, source: Source) -> None:
+        """
+        Initialize the data loader.
+
+        Args:
+            source: The source to load the evaluation data from.
+        """
+        self.source = source
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `DataLoader` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the data loader.
+
+        Returns:
+            An instance of the data loader class initialized with the provided configuration.
+        """
+        dataloader_config = DataLoaderConfig.model_validate(config)
+        config["source"] = Source.subclass_from_config(dataloader_config.source)
+        return super().from_config(config)
+
     @abstractmethod
-    async def load(self) -> DataT:
+    async def load(self) -> Iterable[EvaluationDataT]:
         """
         Load the data.
 

ragbits/evaluate/dataloaders/document_search.py

@@ -0,0 +1,45 @@
+from collections.abc import Iterable
+
+from datasets import load_dataset
+
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
+from ragbits.evaluate.pipelines.document_search import DocumentSearchData
+
+
+class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
+    """
+    Document search evaluation data loader.
+
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files)
+    and contain the following features: "question, "passages".
+    """
+
+    async def load(self) -> Iterable[DocumentSearchData]:
+        """
+        Load the data from source and format them.
+
+        Returns:
+            The document search evaluation data.
+
+        Raises:
+            DataLoaderIncorrectFormatDataError: If evaluation dataset is incorrectly formatted.
+        """
+        data_path = await self.source.fetch()
+        dataset = load_dataset(
+            path=str(data_path.parent),
+            split=data_path.stem,
+        )
+        if "question" not in dataset.features or "passages" not in dataset.features:
+            raise DataLoaderIncorrectFormatDataError(
+                required_features=["question", "passages"],
+                data_path=data_path,
+            )
+
+        return [
+            DocumentSearchData(
+                question=data["question"],
+                reference_passages=data["passages"],
+            )
+            for data in dataset
+        ]

ragbits/evaluate/dataloaders/exceptions.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+
+class DataLoaderError(Exception):
+    """
+    Class for all exceptions raised by the data loader.
+    """
+
+    def __init__(self, message: str, data_path: Path) -> None:
+        super().__init__(message)
+        self.message = message
+        self.data_path = data_path
+
+
+class DataLoaderIncorrectFormatDataError(DataLoaderError):
+    """
+    Raised when the data are incorrectly formatted.
+    """
+
+    def __init__(self, required_features: list[str], data_path: Path) -> None:
+        super().__init__(
+            message=f"Dataset {data_path} is incorrectly formatted. Required features: {required_features}",
+            data_path=data_path,
+        )
+        self.required_features = required_features

ragbits/evaluate/evaluator.py

@@ -9,30 +9,46 @@ from tqdm.asyncio import tqdm
 from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
 from ragbits.evaluate.dataloaders.base import DataLoader
 from ragbits.evaluate.metrics.base import MetricSet
-from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult
+from ragbits.evaluate.pipelines.base import EvaluationDataT, EvaluationPipeline, EvaluationResultT, EvaluationTargetT
 
 
-class EvaluatorConfig(BaseModel):
+class EvaluationConfig(BaseModel):
     """
-    Schema for for the dict taken by `Evaluator.run_from_config` method.
+    Schema for the evaluation run config.
     """
 
-    dataloader: ObjectConstructionConfig
     pipeline: ObjectConstructionConfig
+    dataloader: ObjectConstructionConfig
     metrics: dict[str, ObjectConstructionConfig]
 
 
+class EvaluatorConfig(BaseModel):
+    """
+    Schema for the evaluator config.
+    """
+
+    evaluation: EvaluationConfig
+    evaluator: dict | None = None
+
+
 class Evaluator(WithConstructionConfig):
     """
     Evaluator class.
     """
 
-    CONCURRENCY: int = 10
+    def __init__(self, batch_size: int = 10) -> None:
+        """
+        Initialize the evaluator.
+
+        Args:
+            batch_size: batch size for the evaluation pipeline inference.
+        """
+        self.batch_size = batch_size
 
     @classmethod
     async def run_from_config(cls, config: dict) -> dict:
         """
-        Runs the evaluation based on configuration.
+        Run the evaluation based on configuration.
 
         Args:
             config: Evaluation config.
@@ -40,12 +56,14 @@ class Evaluator(WithConstructionConfig):
         Returns:
             The evaluation results.
         """
-        model = EvaluatorConfig.model_validate(config)
-        dataloader: DataLoader = DataLoader.subclass_from_config(model.dataloader)
-        pipeline: EvaluationPipeline = EvaluationPipeline.subclass_from_config(model.pipeline)
-        metrics: MetricSet = MetricSet.from_config(model.metrics)
-
-        return await cls().compute(
+        evaluator_config = EvaluatorConfig.model_validate(config)
+        evaluation_config = EvaluationConfig.model_validate(evaluator_config.evaluation)
+        pipeline: EvaluationPipeline = EvaluationPipeline.subclass_from_config(evaluation_config.pipeline)
+        dataloader: DataLoader = DataLoader.subclass_from_config(evaluation_config.dataloader)
+        metrics: MetricSet = MetricSet.from_config(evaluation_config.metrics)
+
+        evaluator = cls.from_config(evaluator_config.evaluator or {})
+        return await evaluator.compute(
             pipeline=pipeline,
             dataloader=dataloader,
             metrics=metrics,
@@ -53,9 +71,9 @@ class Evaluator(WithConstructionConfig):
 
     async def compute(
         self,
-        pipeline: EvaluationPipeline,
-        dataloader: DataLoader,
-        metrics: MetricSet,
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataloader: DataLoader[EvaluationDataT],
+        metrics: MetricSet[EvaluationResultT],
     ) -> dict:
         """
         Compute the evaluation results for the given pipeline and data.
@@ -83,9 +101,9 @@ class Evaluator(WithConstructionConfig):
 
     async def _call_pipeline(
         self,
-        pipeline: EvaluationPipeline,
-        dataset: Iterable[dict],
-    ) -> tuple[list[EvaluationResult], dict]:
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataset: Iterable[EvaluationDataT],
+    ) -> tuple[list[EvaluationResultT], dict]:
         """
         Call the pipeline with the given data.
 
@@ -96,9 +114,9 @@ class Evaluator(WithConstructionConfig):
         Returns:
             The evaluation results and performance metrics.
         """
-        semaphore = asyncio.Semaphore(self.CONCURRENCY)
+        semaphore = asyncio.Semaphore(self.batch_size)
 
-        async def _call_pipeline_with_semaphore(data: dict) -> EvaluationResult:
+        async def _call_pipeline_with_semaphore(data: EvaluationDataT) -> EvaluationResultT:
             async with semaphore:
                 return await pipeline(data)
 
@@ -109,7 +127,7 @@ class Evaluator(WithConstructionConfig):
         return pipe_outputs, self._compute_time_perf(start_time, end_time, len(pipe_outputs))
 
     @staticmethod
-    def _results_processor(results: list[EvaluationResult]) -> dict:
+    def _results_processor(results: list[EvaluationResultT]) -> dict:
         """
         Process the results.
 
@@ -122,7 +140,7 @@ class Evaluator(WithConstructionConfig):
         return {"results": [asdict(result) for result in results]}
 
     @staticmethod
-    def _compute_metrics(metrics: MetricSet, results: list[EvaluationResult]) -> dict:
+    def _compute_metrics(metrics: MetricSet[EvaluationResultT], results: list[EvaluationResultT]) -> dict:
         """
         Compute a metric using the given inputs.
 

ragbits/evaluate/factories/__init__.py

@@ -3,10 +3,12 @@ import asyncio
 from datasets import load_dataset
 
 from ragbits.core.embeddings.dense import LiteLLMEmbedder
+from ragbits.core.sources.hf import HuggingFaceSource
 from ragbits.core.utils.config_handling import ObjectConstructionConfig
 from ragbits.core.vector_stores.in_memory import InMemoryVectorStore
 from ragbits.document_search import DocumentSearch
 from ragbits.document_search.documents.document import DocumentMeta
+from ragbits.evaluate.dataloaders.document_search import DocumentSearchDataLoader
 from ragbits.evaluate.metrics import MetricSet
 
 DS_PRECISION_RECALL_F1 = {
@@ -27,7 +29,9 @@ DS_PRECISION_RECALL_F1 = {
 
 
 def precision_recall_f1() -> MetricSet:
-    """A factory of precision recall f1 metric set for retrival evaluation"""
+    """
+    Factory of precision recall f1 metric set for retrival evaluation.
+    """
     return MetricSet.from_config(config=DS_PRECISION_RECALL_F1)
 
 
@@ -38,7 +42,16 @@ async def _add_example_documents(document_search: DocumentSearch) -> None:
 
 
 def basic_document_search_factory() -> DocumentSearch:
-    """A factory for basic example document search instance"""
+    """
+    Factory for basic example document search instance.
+    """
     document_search = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
     asyncio.run(_add_example_documents(document_search))
     return document_search
+
+
+def synthetic_rag_dataset() -> DocumentSearchDataLoader:
+    """
+    Factory for synthetic RAG dataset.
+    """
+    return DocumentSearchDataLoader(source=HuggingFaceSource(path="deepsense-ai/synthetic-rag-dataset_v1.0"))

ragbits/evaluate/metrics/base.py

@@ -1,19 +1,22 @@
 from abc import ABC, abstractmethod
-from typing import Generic, TypeVar
+from types import ModuleType
+from typing import ClassVar, Generic
 
 from typing_extensions import Self
 
 from ragbits.core.utils.config_handling import WithConstructionConfig
-from ragbits.evaluate.pipelines.base import EvaluationResult
+from ragbits.evaluate import metrics
+from ragbits.evaluate.pipelines.base import EvaluationResultT
 
-ResultT = TypeVar("ResultT", bound=EvaluationResult)
 
-
-class Metric(WithConstructionConfig, Generic[ResultT], ABC):
+class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
     """
     Base class for metrics.
     """
 
+    default_module: ClassVar[ModuleType | None] = metrics
+    configuration_key: ClassVar[str] = "metric"
+
     def __init__(self, weight: float = 1.0) -> None:
         """
         Initializes the metric.
@@ -25,7 +28,7 @@ class Metric(WithConstructionConfig, Generic[ResultT], ABC):
         self.weight = weight
 
     @abstractmethod
-    def compute(self, results: list[ResultT]) -> dict:
+    def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metric.
 
@@ -37,16 +40,17 @@ class Metric(WithConstructionConfig, Generic[ResultT], ABC):
         """
 
 
-class MetricSet(WithConstructionConfig, Generic[ResultT]):
+class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
     """
     Represents a set of metrics.
     """
 
-    configuration_key = "metrics"
+    configuration_key: ClassVar[str] = "metrics"
+    default_module: ClassVar[ModuleType | None] = metrics
 
-    def __init__(self, *metrics: Metric[ResultT]) -> None:
+    def __init__(self, *metrics: Metric[EvaluationResultT]) -> None:
         """
-        Initializes the metric set.
+        Initialize the metric set.
 
         Args:
             metrics: The metrics.
@@ -66,7 +70,7 @@ class MetricSet(WithConstructionConfig, Generic[ResultT]):
         """
         return cls(*[Metric.subclass_from_config(metric_config) for metric_config in config.values()])
 
-    def compute(self, results: list[ResultT]) -> dict:
+    def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metrics.
 

ragbits/evaluate/metrics/document_search.py

@@ -19,7 +19,7 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
 
     def __init__(self, matching_strategy: MatchingStrategy, weight: float = 1.0) -> None:
         """
-        Initializes the document search metric.
+        Initialize the document search metric.
 
         Args:
             matching_strategy: Matching strategys that determine relevance.

ragbits/evaluate/optimizer.py

@@ -17,10 +17,10 @@ from ragbits.evaluate.utils import setup_optuna_neptune_callback
 
 class OptimizerConfig(BaseModel):
     """
-    Schema for the dict taken by `Optimizer.run_from_config` method.
+    Schema for the optimizer config.
     """
 
-    experiment: EvaluatorConfig
+    evaluator: EvaluatorConfig
     optimizer: dict | None = None
     neptune_callback: bool = False
 
@@ -32,7 +32,7 @@ class Optimizer(WithConstructionConfig):
 
     def __init__(self, direction: str = "maximize", n_trials: int = 10, max_retries_for_trial: int = 1) -> None:
         """
-        Initializes the pipeline optimizer.
+        Initialize the pipeline optimizer.
 
         Args:
             direction: Direction of optimization.
@@ -49,7 +49,7 @@ class Optimizer(WithConstructionConfig):
     @classmethod
     def run_from_config(cls, config: dict) -> list[tuple[dict, float, dict[str, float]]]:
         """
-        Runs the optimization process configured with a config object.
+        Run the optimization process configured with a config object.
 
         Args:
             config: Optimizer config.
@@ -58,16 +58,16 @@ class Optimizer(WithConstructionConfig):
             List of tested configs with associated scores and metrics.
         """
         optimizer_config = OptimizerConfig.model_validate(config)
-        evaluator_config = EvaluatorConfig.model_validate(optimizer_config.experiment)
+        evaluator_config = EvaluatorConfig.model_validate(optimizer_config.evaluator)
 
-        dataloader: DataLoader = DataLoader.subclass_from_config(evaluator_config.dataloader)
-        metrics: MetricSet = MetricSet.from_config(evaluator_config.metrics)
+        dataloader: DataLoader = DataLoader.subclass_from_config(evaluator_config.evaluation.dataloader)
+        metrics: MetricSet = MetricSet.from_config(evaluator_config.evaluation.metrics)
 
-        pipeline_class = import_by_path(evaluator_config.pipeline.type)
-        pipeline_config = dict(optimizer_config.experiment.pipeline.config)
+        pipeline_class = import_by_path(evaluator_config.evaluation.pipeline.type)
+        pipeline_config = dict(evaluator_config.evaluation.pipeline.config)
         callbacks = [setup_optuna_neptune_callback()] if optimizer_config.neptune_callback else []
 
-        optimizer = cls.from_config(config.get("optimizer", {}))
+        optimizer = cls.from_config(optimizer_config.optimizer or {})
         return optimizer.optimize(
             pipeline_class=pipeline_class,
             pipeline_config=pipeline_config,
@@ -85,7 +85,7 @@ class Optimizer(WithConstructionConfig):
         callbacks: list[Callable] | None = None,
     ) -> list[tuple[dict, float, dict[str, float]]]:
         """
-        Runs the optimization process for given parameters.
+        Run the optimization process for given parameters.
 
         Args:
             pipeline_class: Pipeline to be optimized.
@@ -134,7 +134,7 @@ class Optimizer(WithConstructionConfig):
         metrics: MetricSet,
     ) -> float:
         """
-        Runs a single experiment.
+        Run a single experiment.
         """
         evaluator = Evaluator()
         event_loop = asyncio.get_event_loop()

ragbits/evaluate/pipelines/__init__.py

@@ -1,14 +1,14 @@
 from ragbits.core.utils.config_handling import WithConstructionConfig
 from ragbits.document_search import DocumentSearch
-from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult
+from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult
 from ragbits.evaluate.pipelines.document_search import DocumentSearchPipeline
 
+__all__ = ["DocumentSearchPipeline", "EvaluationData", "EvaluationPipeline", "EvaluationResult"]
+
 _target_to_evaluation_pipeline: dict[type[WithConstructionConfig], type[EvaluationPipeline]] = {
-    DocumentSearch: DocumentSearchPipeline
+    DocumentSearch: DocumentSearchPipeline,
 }
 
-__all__ = ["DocumentSearchPipeline", "EvaluationPipeline", "EvaluationResult"]
-
 
 def get_evaluation_pipeline_for_target(evaluation_target: WithConstructionConfig) -> EvaluationPipeline:
     """

ragbits/evaluate/pipelines/base.py

@@ -1,12 +1,24 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Generic, TypeVar
+from types import ModuleType
+from typing import ClassVar, Generic, TypeVar
+
+from pydantic import BaseModel
 
 from ragbits.core.utils.config_handling import WithConstructionConfig
+from ragbits.evaluate import pipelines
 
+EvaluationDataT = TypeVar("EvaluationDataT", bound="EvaluationData")
+EvaluationResultT = TypeVar("EvaluationResultT", bound="EvaluationResult")
 EvaluationTargetT = TypeVar("EvaluationTargetT", bound=WithConstructionConfig)
 
 
+class EvaluationData(BaseModel, ABC):
+    """
+    Represents the data for a single evaluation.
+    """
+
+
 @dataclass
 class EvaluationResult(ABC):
     """
@@ -14,18 +26,34 @@ class EvaluationResult(ABC):
     """
 
 
-class EvaluationPipeline(Generic[EvaluationTargetT], WithConstructionConfig, ABC):
+class EvaluationPipeline(WithConstructionConfig, Generic[EvaluationTargetT, EvaluationDataT, EvaluationResultT], ABC):
     """
-    Collection evaluation pipeline.
+    Evaluation pipeline.
     """
 
-    def __init__(self, evaluation_target: EvaluationTargetT):
+    default_module: ClassVar[ModuleType | None] = pipelines
+    configuration_key: ClassVar[str] = "pipeline"
+
+    def __init__(self, evaluation_target: EvaluationTargetT) -> None:
+        """
+        Initialize the evaluation pipeline.
+
+        Args:
+            evaluation_target: Evaluation target instance.
+        """
+        super().__init__()
         self.evaluation_target = evaluation_target
 
+    async def prepare(self) -> None:
+        """
+        Prepare pipeline for evaluation. Optional step.
+        """
+        pass
+
     @abstractmethod
-    async def __call__(self, data: dict) -> EvaluationResult:
+    async def __call__(self, data: EvaluationDataT) -> EvaluationResultT:
         """
-        Runs the evaluation pipeline.
+        Run the evaluation pipeline.
 
         Args:
             data: The evaluation data.
@@ -33,9 +61,3 @@ class EvaluationPipeline(Generic[EvaluationTargetT], WithConstructionConfig, ABC
         Returns:
             The evaluation result.
         """
-
-    async def prepare(self) -> None:
-        """
-        Prepares pipeline for evaluation.
-        """
-        pass

ragbits/evaluate/pipelines/document_search.py

@@ -5,7 +5,16 @@ from typing_extensions import Self
 
 from ragbits.core.sources.hf import HuggingFaceSource
 from ragbits.document_search import DocumentSearch
-from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult
+from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult
+
+
+class DocumentSearchData(EvaluationData):
+    """
+    Represents the evaluation data for document search.
+    """
+
+    question: str
+    reference_passages: list[str]
 
 
 @dataclass
@@ -19,14 +28,14 @@ class DocumentSearchResult(EvaluationResult):
     predicted_passages: list[str]
 
 
-class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch]):
+class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch, DocumentSearchData, DocumentSearchResult]):
     """
     Document search evaluation pipeline.
     """
 
     def __init__(self, evaluation_target: DocumentSearch, source: dict | None = None) -> None:
         """
-        Initializes the document search pipeline.
+        Initialize the document search evaluation pipeline.
 
         Args:
             evaluation_target: Document Search instance.
@@ -51,12 +60,12 @@ class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch]):
         # TODO: optimize this for cases with duplicated document search configs between runs
         if config.get("source"):
             config["vector_store"]["config"]["index_name"] = str(uuid4())
-        document_search = DocumentSearch.from_config(config)
-        return cls(evaluation_target=document_search, source=config.get("source"))
+        evaluation_target = DocumentSearch.from_config(config)
+        return cls(evaluation_target=evaluation_target, source=config.get("source"))
 
     async def prepare(self) -> None:
         """
-        Ingests corpus data for evaluation.
+        Ingest corpus data for evaluation.
         """
         if self.source:
             # For now we only support HF sources for pre-evaluation ingest
@@ -67,9 +76,9 @@ class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch]):
             )
             await self.evaluation_target.ingest(sources)
 
-    async def __call__(self, data: dict) -> DocumentSearchResult:
+    async def __call__(self, data: DocumentSearchData) -> DocumentSearchResult:
         """
-        Runs the document search evaluation pipeline.
+        Run the document search evaluation pipeline.
 
         Args:
             data: The evaluation data.
@@ -77,10 +86,11 @@ class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch]):
         Returns:
             The evaluation result.
         """
-        elements = await self.evaluation_target.search(data["question"])
+        elements = await self.evaluation_target.search(data.question)
         predicted_passages = [element.text_representation for element in elements if element.text_representation]
+
         return DocumentSearchResult(
-            question=data["question"],
-            reference_passages=data["passage"],
+            question=data.question,
+            reference_passages=data.reference_passages,
             predicted_passages=predicted_passages,
         )

{ragbits_evaluate-0.16.0.dist-info → ragbits_evaluate-0.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragbits-evaluate
-Version: 0.16.0
+Version: 0.17.0
 Summary: Evaluation module for Ragbits components
 Project-URL: Homepage, https://github.com/deepsense-ai/ragbits
 Project-URL: Bug Reports, https://github.com/deepsense-ai/ragbits/issues
@@ -22,11 +22,12 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
-Requires-Dist: distilabel==1.4.1
-Requires-Dist: hydra-core~=1.3.2
-Requires-Dist: neptune[optuna]~=1.12.0
-Requires-Dist: optuna==4.0.0
-Requires-Dist: ragbits-core==0.16.0
+Requires-Dist: datasets<4.0.0,>=3.0.1
+Requires-Dist: distilabel<2.0.0,>=1.4.1
+Requires-Dist: hydra-core<2.0.0,>=1.3.2
+Requires-Dist: neptune[optuna]<2.0.0,>=1.12.0
+Requires-Dist: optuna<5.0.0,>=4.0.0
+Requires-Dist: ragbits-core==0.17.0
 Provides-Extra: relari
 Requires-Dist: continuous-eval<1.0.0,>=0.3.12; extra == 'relari'
 Description-Content-Type: text/markdown

{ragbits_evaluate-0.16.0.dist-info → ragbits_evaluate-0.17.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
 ragbits/evaluate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragbits/evaluate/cli.py,sha256=zOPnd9LhPVNN-SRIt0R3y_qmDDqSVnbaLPMiFUJo1Jo,4617
-ragbits/evaluate/config.py,sha256=FfeC04f95hygTans3GduiSFuzhInUGWhmxLK4ylL95c,467
-ragbits/evaluate/evaluator.py,sha256=iMyxMXJhKNKOWMs9xM4uIxpe_bYNXcerfldZ2z32ghA,4777
-ragbits/evaluate/optimizer.py,sha256=rxIK8UMlSyBpP5HL7_TvYPceNoyqmsi7Gvs7SpPnWhA,8471
+ragbits/evaluate/cli.py,sha256=MEDo8ubk81TCNx-fq-liF0P5hjn2-kPpIfq54fReKIY,4509
+ragbits/evaluate/config.py,sha256=2WSmbVxyQi893L2FSjRFQoXkWZp1GetcNmR2GCDe0tA,339
+ragbits/evaluate/evaluator.py,sha256=Cif-QX2n5awOGm-AfFy2nRXkb_m4vGY_JZ_o4K4PhZI,5552
+ragbits/evaluate/optimizer.py,sha256=egcU54aADqKrN31NPqj7cNIQO4UISfG7VtkOAQyQUOY,8471
 ragbits/evaluate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/utils.py,sha256=rTTmrP4nv3D7174cMEfohxrDN5thPScH0BsXaptMHqQ,3757
-ragbits/evaluate/dataloaders/__init__.py,sha256=ezGvdCOGBUrOh8M_ceJnOO9hR-jWTdjKr0voHg4Vsvc,802
-ragbits/evaluate/dataloaders/base.py,sha256=yzsGpCfQ3u1z3XpqTqG8VnKdv3o5CTpToDw8hh3wvv0,417
-ragbits/evaluate/dataloaders/hf.py,sha256=FwAGeeqZ0ZxieJyeBcdWo__adjoG3Db34tgcT1RRMng,707
-ragbits/evaluate/dataloaders/local.py,sha256=FrDjeAguKfimM8l35LLthr-kH7slxJG9zRnu1YZuo0A,1155
+ragbits/evaluate/dataloaders/__init__.py,sha256=UFJFjmvi3GUQFsx6A5sYD01HH2f7TXcHRW2VNM1pmIA,83
+ragbits/evaluate/dataloaders/base.py,sha256=ovL38_tH12q9wd3yeflIlovGuSD8S1X9HUUtwv17QrM,1774
+ragbits/evaluate/dataloaders/document_search.py,sha256=sqNPQf1ZYAqM_xMjuwh63ET00zEmKtAzqXX04cazuB8,1579
+ragbits/evaluate/dataloaders/exceptions.py,sha256=xUOBLj1JuCkcqzRVnu0A0I_i1THxbDt2MEDVdDGjDyY,735
 ragbits/evaluate/dataset_generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/dataset_generator/pipeline.py,sha256=dgnV-Qm0Z7S1Y6ga9-9RscXxxr3krOKsIj7E9WS4ANk,4940
 ragbits/evaluate/dataset_generator/utils.py,sha256=zD-ksXlX62kkIgzBefE4ILsP7He9bHimnZ63LLsMKCA,1325
@@ -23,13 +23,13 @@ ragbits/evaluate/dataset_generator/tasks/filter/dont_know.py,sha256=ydMHyI0JrWZf
 ragbits/evaluate/dataset_generator/tasks/text_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/dataset_generator/tasks/text_generation/base.py,sha256=2h-Y14H3fRHKbTNvXWKRus8t0hdTITd9LMoIFVwfKfA,2138
 ragbits/evaluate/dataset_generator/tasks/text_generation/qa.py,sha256=QAClPbTVNCe4QzVOGuepRnsmkt9ZF6bXBAuJI2elRuE,3851
-ragbits/evaluate/factories/__init__.py,sha256=FZulUBra8TD37YMk6vDZXc9uq15o-eq9knZ42w_NGTg,1696
+ragbits/evaluate/factories/__init__.py,sha256=De2ZgQ4YXgvpMOvm81fSDPSMvKpIBjS-aqeE0dxEU1s,2074
 ragbits/evaluate/metrics/__init__.py,sha256=Mr83ytGyvdXtBlr7Bbo0-5auE0530xsd3wffKSIf8cE,95
-ragbits/evaluate/metrics/base.py,sha256=iJ_ise9XRS6IL6zwJZmN18rLNwI-drLsb2BJOzqkEGI,2099
-ragbits/evaluate/metrics/document_search.py,sha256=UrIIPhDdgXkozHFVac87jBURjVvgLS5gaaN8uYJ1jJY,2852
-ragbits/evaluate/pipelines/__init__.py,sha256=2ESwj6TybDxxCHrndTANpbek3TmpYSIeH1L4jFCKa48,1259
-ragbits/evaluate/pipelines/base.py,sha256=Qo3a3il_6edAuk_1vGMp9O_FeNQDJFqJn1WQOKEK_fs,985
-ragbits/evaluate/pipelines/document_search.py,sha256=B0mNFxgSCfhCSjpmQhZRqc1InKKjP1kWDAekQQG9Fj4,3010
-ragbits_evaluate-0.16.0.dist-info/METADATA,sha256=KtxocDx-eexYcmfmeKKr4GGfnutJdGicww6rl4_KXdY,2234
-ragbits_evaluate-0.16.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ragbits_evaluate-0.16.0.dist-info/RECORD,,
+ragbits/evaluate/metrics/base.py,sha256=axkGuKJU5u94SnRjpWsdG4jFWjy8rmkSHVRcgz1JLTo,2342
+ragbits/evaluate/metrics/document_search.py,sha256=WeC0xuLYci_Vbdw-E4OjawTqmLkcFKjDWSJGITC9-AQ,2851
+ragbits/evaluate/pipelines/__init__.py,sha256=Bqp_L7aRq12Ua19ELZDsdYvra6-GlLrQ9cIG2IWArko,1294
+ragbits/evaluate/pipelines/base.py,sha256=1GPu3MV-2o0PdUuFM4IcLeg1baYv9acqCcGrQykmRSs,1682
+ragbits/evaluate/pipelines/document_search.py,sha256=xMcSnahy7fifk2bJoolX9OWCXz4FjSJQfBDHIB1d2mQ,3266
+ragbits_evaluate-0.17.0.dist-info/METADATA,sha256=fdHH9MszU2DO5pp18ikVVnOEPkTnQ_TQwddvcvEwWj4,2300
+ragbits_evaluate-0.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ragbits_evaluate-0.17.0.dist-info/RECORD,,

ragbits/evaluate/dataloaders/hf.py

@@ -1,29 +0,0 @@
-from typing import TypeAlias
-
-from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict, load_dataset
-
-from ragbits.evaluate.dataloaders.base import DataLoader
-
-HFData: TypeAlias = DatasetDict | Dataset | IterableDatasetDict | IterableDataset
-
-
-class HFDataLoader(DataLoader[HFData]):
-    """
-    Hugging Face data loader.
-    """
-
-    def __init__(self, path: str, split: str) -> None:
-        self.path = path
-        self.split = split
-
-    async def load(self) -> HFData:
-        """
-        Load the data from Hugging Face.
-
-        Returns:
-            The loaded data.
-        """
-        return load_dataset(
-            path=self.path,
-            split=self.split,
-        )

ragbits/evaluate/dataloaders/local.py

@@ -1,45 +0,0 @@
-from typing import TypeAlias
-
-from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict, load_dataset
-
-from .base import DataLoader
-
-HFData: TypeAlias = DatasetDict | Dataset | IterableDatasetDict | IterableDataset
-
-
-class LocalDataLoader(DataLoader[DatasetDict]):
-    """
-    Local data loader.
-    """
-
-    AVAILABLE_BUILDERS = {
-        "json",
-        "csv",
-        "parquet",
-        "arrow",
-        "text",
-        "xml",
-        "webdataset",
-        "imagefolder",
-        "audiofolder",
-        "videofolder",
-    }
-
-    def __init__(self, path: str, split: str, builder: str) -> None:
-        self.path = path
-        self.split = split
-        self.builder = builder
-
-        if self.builder not in self.AVAILABLE_BUILDERS:
-            raise ValueError(
-                f"Unsupported builder '{self.builder}'. Available builders: {', '.join(self.AVAILABLE_BUILDERS)}"
-            )
-
-    async def load(self) -> DatasetDict:
-        """
-        Load the data from the local file.
-
-        Returns:
-            The loaded data.
-        """
-        return load_dataset(self.builder, data_files=self.path, split=self.split)