ragbits-evaluate 0.5.0__py3-none-any.whl → 1.4.0.dev202602030301__py3-none-any.whl

ragbits/evaluate/cli.py

@@ -0,0 +1,150 @@
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from pydantic import BaseModel
+
+from ragbits.cli._utils import get_instance_or_exit
+from ragbits.cli.state import print_output
+from ragbits.core.utils.config_handling import WithConstructionConfig
+from ragbits.evaluate.config import eval_config
+from ragbits.evaluate.dataloaders import DataLoader
+from ragbits.evaluate.evaluator import Evaluator
+from ragbits.evaluate.metrics.base import MetricSet
+from ragbits.evaluate.pipelines import get_evaluation_pipeline_for_target
+from ragbits.evaluate.pipelines.base import EvaluationPipeline
+
+eval_app = typer.Typer(no_args_is_help=True)
+
+
+def register(app: typer.Typer) -> None:
+    """
+    Register the CLI commands for the package.
+
+    Args:
+        app: The Typer object to register the commands with.
+    """
+    app.add_typer(eval_app, name="evaluate", help="Commands for interacting with ragbits evaluate module")
+
+
+@dataclass
+class _CLIState:
+    dataloader: DataLoader | None = None
+    pipeline: EvaluationPipeline | None = None
+    metrics: MetricSet | None = None
+
+
+class EvaluationResult(BaseModel):
+    """A container for evaluation results"""
+
+    metrics: dict
+
+
+state: _CLIState = _CLIState()
+
+
+@eval_app.callback()
+def common_args(
+    dataloader_factory_path: Annotated[
+        str | None,
+        typer.Option(
+            help="A path to evaluation data loader factory in format python.path:function_name",
+            exists=True,
+            resolve_path=True,
+        ),
+    ] = None,
+    dataloader_yaml_path: Annotated[
+        Path | None,
+        typer.Option(
+            help="A path to evaluation data loader configuration",
+            exists=True,
+            resolve_path=True,
+        ),
+    ] = None,
+    target_factory_path: Annotated[
+        str | None,
+        typer.Option(
+            help="A path to a factory of the evaluation target class in format: python.path:function_name",
+            exists=True,
+            resolve_path=True,
+        ),
+    ] = None,
+    target_yaml_path: Annotated[
+        Path | None,
+        typer.Option(
+            help="A path to a YAML configuration file of the evaluation target class",
+            exists=True,
+            resolve_path=True,
+        ),
+    ] = None,
+    metrics_factory_path: Annotated[
+        str | None,
+        typer.Option(
+            help="A path to metrics factory in format python.path:function_name",
+            exists=True,
+            resolve_path=True,
+        ),
+    ] = None,
+    metrics_yaml_path: Annotated[
+        Path | None,
+        typer.Option(
+            help="A path to metrics configuration",
+            exists=True,
+            resolve_path=True,
+        ),
+    ] = None,
+) -> None:
+    """
+    Common arguments for the evaluate commands.
+    """
+    evaluation_target = get_instance_or_exit(
+        cls=WithConstructionConfig,
+        factory_path=target_factory_path,
+        yaml_path=target_yaml_path,
+        config_override=eval_config,
+    )
+    state.pipeline = get_evaluation_pipeline_for_target(evaluation_target)
+    # TODO: validate if given dataloader is suitable for evaluation pipeline
+    state.dataloader = get_instance_or_exit(
+        cls=DataLoader,
+        factory_path=dataloader_factory_path,
+        yaml_path=dataloader_yaml_path,
+        config_override=eval_config,
+    )
+    # TODO: validate if given metric set is suitable for evaluation pipeline
+    state.metrics = get_instance_or_exit(
+        cls=MetricSet,
+        factory_path=metrics_factory_path,
+        yaml_path=metrics_yaml_path,
+        config_override=eval_config,
+    )
+
+
+@eval_app.command()
+def run() -> None:
+    """
+    Evaluate the pipeline.
+    """
+
+    async def run() -> None:
+        if state.dataloader is None:
+            raise ValueError("Evaluation dataloader not initialized")
+        if state.pipeline is None:
+            raise ValueError("Evaluation pipeline not initialized")
+        if state.metrics is None:
+            raise ValueError("Evaluation metrics not initialized")
+
+        evaluator = Evaluator()
+        metric_results = await evaluator.compute(
+            pipeline=state.pipeline,
+            dataloader=state.dataloader,
+            metricset=state.metrics,
+        )
+        evaluation_results = EvaluationResult(
+            metrics={"metrics": metric_results.metrics, "time_perf": metric_results.time_perf}
+        )
+        print_output(evaluation_results)
+
+    asyncio.run(run())

ragbits/evaluate/config.py

@@ -0,0 +1,11 @@
+from ragbits.core.config import CoreConfig
+from ragbits.core.utils._pyproject import get_config_instance
+
+
+class EvaluateConfig(CoreConfig):
+    """
+    Configuration for the ragbits-evaluate package, loaded from downstream projects' pyproject.toml files.
+    """
+
+
+eval_config = get_config_instance(EvaluateConfig, subproject="evaluate")

ragbits/evaluate/dataloaders/__init__.py

@@ -0,0 +1,3 @@
+from ragbits.evaluate.dataloaders.base import DataLoader
+
+__all__ = ["DataLoader"]

ragbits/evaluate/dataloaders/base.py

@@ -0,0 +1,95 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from types import ModuleType
+from typing import ClassVar, Generic
+
+from datasets import load_dataset
+from pydantic import BaseModel
+from typing_extensions import Self
+
+from ragbits.core.sources.base import Source
+from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
+from ragbits.evaluate import dataloaders
+from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
+from ragbits.evaluate.pipelines.base import EvaluationDataT
+
+
+class DataLoaderConfig(BaseModel):
+    """
+    Schema for the data loader config.
+    """
+
+    source: ObjectConstructionConfig
+
+
+class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
+    """
+    Evaluation data loader.
+    """
+
+    default_module: ClassVar[ModuleType | None] = dataloaders
+    configuration_key: ClassVar[str] = "dataloader"
+
+    def __init__(self, source: Source, *, split: str = "data", required_keys: set[str] | None = None) -> None:
+        """
+        Initialize the data loader.
+
+        Args:
+            source: The source to load the evaluation data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
+            required_keys: The required columns for the evaluation data.
+        """
+        self.source = source
+        self.split = split
+        self.required_keys = required_keys or set()
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `DataLoader` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the data loader.
+
+        Returns:
+            An instance of the data loader class initialized with the provided configuration.
+        """
+        dataloader_config = DataLoaderConfig.model_validate(config)
+        config["source"] = Source.subclass_from_config(dataloader_config.source)
+        return super().from_config(config)
+
+    async def load(self) -> Iterable[EvaluationDataT]:
+        """
+        Load the data.
+
+        Returns:
+            The loaded evaluation data.
+
+        Raises:
+            DataLoaderIncorrectFormatDataError: If evaluation dataset is incorrectly formatted.
+        """
+        data_path = await self.source.fetch()
+        dataset = load_dataset(
+            path=str(data_path.parent),
+            data_files={"data": str(data_path.name)},
+            split=self.split,
+        )
+        if not self.required_keys.issubset(dataset.features):
+            raise DataLoaderIncorrectFormatDataError(
+                required_features=list(self.required_keys),
+                data_path=data_path,
+            )
+        return await self.map(dataset.to_list())
+
+    @abstractmethod
+    async def map(self, dataset: Iterable[dict]) -> Iterable[EvaluationDataT]:
+        """
+        Map the dataset to the evaluation data.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The evaluation data.
+        """

ragbits/evaluate/dataloaders/document_search.py

@@ -0,0 +1,61 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.document_search import DocumentSearchData
+
+
+class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
+    """
+    Document search evaluation data loader.
+
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        document_ids_key: str = "document_ids",
+        passages_key: str = "passages",
+        page_numbers_key: str = "page_numbers",
+    ) -> None:
+        """
+        Initialize the document search data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
+            question_key: The dataset column name that contains the question.
+            document_ids_key: The dataset column name that contains the document ids. Document ids are optional.
+            passages_key: The dataset column name that contains the passages. Passages are optional.
+            page_numbers_key: The dataset column name that contains the page numbers. Page numbers are optional.
+        """
+        super().__init__(source=source, split=split, required_keys={question_key})
+        self.question_key = question_key
+        self.document_ids_key = document_ids_key
+        self.passages_key = passages_key
+        self.page_numbers_key = page_numbers_key
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[DocumentSearchData]:
+        """
+        Map the dataset to the document search data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The document search data.
+        """
+        return [
+            DocumentSearchData(
+                question=data.get(self.question_key, ""),
+                reference_document_ids=data.get(self.document_ids_key),
+                reference_passages=data.get(self.passages_key),
+                reference_page_numbers=data.get(self.page_numbers_key),
+            )
+            for data in dataset
+        ]

ragbits/evaluate/dataloaders/exceptions.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+
+class DataLoaderError(Exception):
+    """
+    Class for all exceptions raised by the data loader.
+    """
+
+    def __init__(self, message: str, data_path: Path) -> None:
+        super().__init__(message)
+        self.message = message
+        self.data_path = data_path
+
+
+class DataLoaderIncorrectFormatDataError(DataLoaderError):
+    """
+    Raised when the data are incorrectly formatted.
+    """
+
+    def __init__(self, required_features: list[str], data_path: Path) -> None:
+        super().__init__(
+            message=f"Dataset {data_path} is incorrectly formatted. Required features: {required_features}",
+            data_path=data_path,
+        )
+        self.required_features = required_features

ragbits/evaluate/dataloaders/gaia.py

@@ -0,0 +1,78 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.gaia import GaiaData
+
+
+class GaiaDataLoader(DataLoader[GaiaData]):
+    """
+    GAIA benchmark evaluation data loader.
+
+    The source should point to a local/remote JSON or JSONL file exported from the
+    Hugging Face dataset `gaia-benchmark/GAIA`. Rows are expected to contain at least:
+    - "task_id" (str)
+    - "Question" (str)
+    - "Level" (int)
+    - "Final answer" (str)
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        task_id_key: str = "task_id",
+        question_key: str = "Question",
+        level_key: str = "Level",
+        final_answer_key: str = "Final answer",
+        file_name_key: str = "file_name",
+        skip_file_attachments: bool = False,
+    ) -> None:
+        """
+        Initialize the GAIA data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from (file name generated by the source helper).
+            task_id_key: Column name for GAIA task identifier.
+            question_key: Column name for the natural language question.
+            level_key: Column name for numeric difficulty level (1, 2, 3).
+            final_answer_key: Column name for the final ground-truth answer.
+            file_name_key: Column name with optional associated file name (may be empty).
+            skip_file_attachments: If True, skip rows that have a non-empty file attachment.
+        """
+        required = {task_id_key, question_key, level_key, final_answer_key}
+        super().__init__(source=source, split=split, required_keys=required)
+        self.task_id_key = task_id_key
+        self.question_key = question_key
+        self.level_key = level_key
+        self.final_answer_key = final_answer_key
+        self.file_name_key = file_name_key
+        self.skip_file_attachments = skip_file_attachments
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[GaiaData]:
+        """
+        Map the dataset to the GAIA evaluation data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The GAIA evaluation data rows.
+        """
+        return [
+            GaiaData(
+                task_id=str(row.get(self.task_id_key, "")),
+                question=str(row.get(self.question_key, "")),
+                level=int(row.get(self.level_key, 1)),
+                reference_answer=str(row.get(self.final_answer_key, "")),
+                file_name=(row.get(self.file_name_key) or None),
+            )
+            for row in dataset
+            if (
+                not self.skip_file_attachments
+                or not row.get(self.file_name_key)
+                or str(row.get(self.file_name_key)).strip() == ""
+            )
+        ]

ragbits/evaluate/dataloaders/hotpot_qa.py

@@ -0,0 +1,95 @@
+from collections.abc import Iterable
+from typing import Any
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.hotpot_qa import HotpotQAData
+
+
+class HotpotQADataLoader(DataLoader[HotpotQAData]):
+    """
+    HotpotQA evaluation data loader.
+
+    The source should point to a local/remote JSON file exported from Hugging Face, where each example includes at
+    least the following keys:
+    - "id" (str)
+    - "question" (str)
+    - "answer" (str)
+    - "type" ("bridge" | "comparison")
+    - "level" ("easy" | "medium" | "hard")
+    - "context" (object with keys: "title": list[str], "sentences": list[list[str]])
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        id_key: str = "id",
+        question_key: str = "question",
+        answer_key: str = "answer",
+        type_key: str = "type",
+        level_key: str = "level",
+        context_key: str = "context",
+        # filter
+        level_filter: str | None = None,  # one of: easy|medium|hard
+    ) -> None:
+        """
+        Initialize the HotpotQA data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from.
+            id_key: Column with unique id.
+            question_key: Column with question text.
+            answer_key: Column with ground truth answer.
+            type_key: Column with question type ("bridge" | "comparison").
+            level_key: Column with difficulty ("easy" | "medium" | "hard").
+            context_key: Column with context object containing titles and sentences.
+            level_filter: If provided, return only examples with this level.
+        """
+        required = {id_key, question_key, answer_key, type_key, level_key, context_key}
+        super().__init__(source=source, split=split, required_keys=required)
+        self.id_key = id_key
+        self.question_key = question_key
+        self.answer_key = answer_key
+        self.type_key = type_key
+        self.level_key = level_key
+        self.context_key = context_key
+        self.level_filter = level_filter
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[HotpotQAData]:
+        """
+        Map the dataset to the HotpotQA evaluation data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The HotpotQA evaluation data rows.
+        """
+
+        def to_context_rows(context: dict[str, Any]) -> list[str]:
+            titles = context.get("title", []) or []
+            sentences = context.get("sentences", []) or []
+            rows: list[str] = []
+            for title, sent_list in zip(titles, sentences, strict=False):
+                doc_text = "\n".join(sent_list) if isinstance(sent_list, list) else str(sent_list)
+                rows.append(f"{title}\n{doc_text}")
+            if not rows and isinstance(sentences, list):
+                flat = "\n".join([" ".join(s) if isinstance(s, list) else str(s) for s in sentences])
+                rows = [flat]
+            return rows
+
+        return [
+            HotpotQAData(
+                id=row.get(self.id_key, ""),
+                question=row.get(self.question_key, ""),
+                reference_answer=str(row.get(self.answer_key, "")),
+                qtype=str(row.get(self.type_key, "")),
+                level=(row.get(self.level_key) or "").lower(),
+                reference_context=to_context_rows(row.get(self.context_key, {}) or {}),
+            )
+            for row in dataset
+            if not self.level_filter or (row.get(self.level_key, "").lower() == self.level_filter)
+        ]

ragbits/evaluate/dataloaders/human_eval.py

@@ -0,0 +1,70 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.human_eval import HumanEvalData
+
+
+class HumanEvalDataLoader(DataLoader[HumanEvalData]):
+    """
+    HumanEval evaluation data loader.
+
+    The source should point to a local/remote JSONL file in HumanEval format, where each line is a JSON object
+    with at least the following keys: "
+    - task_id" (str)
+    - "prompt" (str)
+    - "entry_point" (str)
+    - "test" (str)
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        task_id_key: str = "task_id",
+        prompt_key: str = "prompt",
+        entry_point_key: str = "entry_point",
+        test_key: str = "test",
+        canonical_solution_key: str | None = "canonical_solution",
+    ) -> None:
+        """
+        Initialize the HumanEval data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from.
+            task_id_key: Dataset column with the HumanEval task identifier.
+            prompt_key: Dataset column with the Python prompt (function signature and docstring).
+            entry_point_key: Dataset column with the function name to evaluate.
+            test_key: Dataset column with the Python test harness defining `check(candidate)`.
+            canonical_solution_key: Optional dataset column with the reference solution (not used for scoring).
+        """
+        required = {task_id_key, prompt_key, entry_point_key, test_key}
+        super().__init__(source=source, split=split, required_keys=required)
+        self.task_id_key = task_id_key
+        self.prompt_key = prompt_key
+        self.entry_point_key = entry_point_key
+        self.test_key = test_key
+        self.canonical_solution_key = canonical_solution_key
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[HumanEvalData]:
+        """
+        Map the dataset to the HumanEval evaluation data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The HumanEval evaluation data rows.
+        """
+        return [
+            HumanEvalData(
+                task_id=row.get(self.task_id_key, ""),
+                prompt=row.get(self.prompt_key, ""),
+                entry_point=row.get(self.entry_point_key, ""),
+                test=row.get(self.test_key, ""),
+                canonical_solution=(row.get(self.canonical_solution_key) if self.canonical_solution_key else None),
+            )
+            for row in dataset
+        ]

ragbits/evaluate/dataloaders/question_answer.py

@@ -0,0 +1,56 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.question_answer import QuestionAnswerData
+
+
+class QuestionAnswerDataLoader(DataLoader[QuestionAnswerData]):
+    """
+    Question answer evaluation data loader.
+
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        answer_key: str = "answer",
+        context_key: str = "context",
+    ) -> None:
+        """
+        Initialize the question answer data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from.
+            question_key: The dataset column name that contains the question.
+            answer_key: The dataset column name that contains the answer.
+            context_key: The dataset column name that contains the context. Context is optional.
+        """
+        super().__init__(source=source, split=split, required_keys={question_key, answer_key})
+        self.question_key = question_key
+        self.answer_key = answer_key
+        self.context_key = context_key
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[QuestionAnswerData]:
+        """
+        Map the dataset to the question answer data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The question answer data.
+        """
+        return [
+            QuestionAnswerData(
+                question=data.get(self.question_key, ""),
+                reference_answer=data.get(self.answer_key, ""),
+                reference_context=data.get(self.context_key),
+            )
+            for data in dataset
+        ]

ragbits/evaluate/dataset_generator/pipeline.py

@@ -7,7 +7,7 @@ from distilabel.steps.base import Step
 from omegaconf import DictConfig, OmegaConf
 from pydantic import BaseModel
 
-from ragbits.core.utils.config_handling import get_cls_from_config
+from ragbits.core.utils.config_handling import import_by_path
 
 module = sys.modules[__name__]
 
@@ -120,14 +120,14 @@ class DatasetGenerationPipeline:
         tasks = []
         for task_config in self.config.tasks:
             llm_config = task_config.llm
-            llm = get_cls_from_config(llm_config.provider_type, module)(**llm_config.kwargs)
+            llm = import_by_path(llm_config.provider_type, module)(**llm_config.kwargs)
             task_kwargs: dict[Any, Any] = {"llm": llm}
             task_kwargs.update(task_config.kwargs or {})  # type: ignore
-            task = get_cls_from_config(task_config.type, module)(**task_kwargs)
+            task = import_by_path(task_config.type, module)(**task_kwargs)
             tasks.append(task)
             filter_types = getattr(task_config, "filters", None) or []
             for filter_type in filter_types:
-                filter = get_cls_from_config(filter_type, module)(tasks[-1])
+                filter = import_by_path(filter_type, module)(tasks[-1])
                 tasks.append(filter)
         return tasks