ragbits-evaluate 0.18.0__py3-none-any.whl → 0.19.1__py3-none-any.whl

ragbits/evaluate/dataloaders/base.py

@@ -3,12 +3,14 @@ from collections.abc import Iterable
 from types import ModuleType
 from typing import ClassVar, Generic
 
+from datasets import load_dataset
 from pydantic import BaseModel
 from typing_extensions import Self
 
 from ragbits.core.sources.base import Source
 from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
 from ragbits.evaluate import dataloaders
+from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
 from ragbits.evaluate.pipelines.base import EvaluationDataT
 
 
@@ -28,14 +30,19 @@ class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
     default_module: ClassVar[ModuleType | None] = dataloaders
     configuration_key: ClassVar[str] = "dataloader"
 
-    def __init__(self, source: Source) -> None:
+    def __init__(self, source: Source, *, split: str = "data", required_keys: set[str] | None = None) -> None:
         """
         Initialize the data loader.
 
         Args:
             source: The source to load the evaluation data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
+            required_keys: The required columns for the evaluation data.
         """
         self.source = source
+        self.split = split
+        self.required_keys = required_keys or set()
 
     @classmethod
     def from_config(cls, config: dict) -> Self:
@@ -52,11 +59,37 @@ class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
         config["source"] = Source.subclass_from_config(dataloader_config.source)
         return super().from_config(config)
 
-    @abstractmethod
     async def load(self) -> Iterable[EvaluationDataT]:
         """
         Load the data.
 
         Returns:
-            The loaded data.
+            The loaded evaluation data.
+
+        Raises:
+            DataLoaderIncorrectFormatDataError: If evaluation dataset is incorrectly formatted.
+        """
+        data_path = await self.source.fetch()
+        dataset = load_dataset(
+            path=str(data_path.parent),
+            data_files={"data": str(data_path.name)},
+            split=self.split,
+        )
+        if not self.required_keys.issubset(dataset.features):
+            raise DataLoaderIncorrectFormatDataError(
+                required_features=list(self.required_keys),
+                data_path=data_path,
+            )
+        return await self.map(dataset.to_list())
+
+    @abstractmethod
+    async def map(self, dataset: Iterable[dict]) -> Iterable[EvaluationDataT]:
+        """
+        Map the dataset to the evaluation data.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The evaluation data.
         """

ragbits/evaluate/dataloaders/document_search.py

@@ -1,10 +1,7 @@
 from collections.abc import Iterable
 
-from datasets import load_dataset
-
 from ragbits.core.sources.base import Source
 from ragbits.evaluate.dataloaders.base import DataLoader
-from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
 from ragbits.evaluate.pipelines.document_search import DocumentSearchData
 
 
@@ -12,13 +9,14 @@ class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
     """
     Document search evaluation data loader.
 
-    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files)
-    and contain the following features: "question, "passages".
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
     """
 
     def __init__(
         self,
         source: Source,
+        *,
+        split: str = "data",
         question_key: str = "question",
         document_ids_key: str = "document_ids",
         passages_key: str = "passages",
@@ -29,42 +27,32 @@ class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
 
         Args:
             source: The source to load the data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
             question_key: The dataset column name that contains the question.
             document_ids_key: The dataset column name that contains the document ids. Document ids are optional.
             passages_key: The dataset column name that contains the passages. Passages are optional.
             page_numbers_key: The dataset column name that contains the page numbers. Page numbers are optional.
         """
-        super().__init__(source)
+        super().__init__(source=source, split=split, required_keys={question_key})
         self.question_key = question_key
         self.document_ids_key = document_ids_key
         self.passages_key = passages_key
         self.page_numbers_key = page_numbers_key
 
-    async def load(self) -> Iterable[DocumentSearchData]:
+    async def map(self, dataset: Iterable[dict]) -> Iterable[DocumentSearchData]:
         """
-        Load the data from source and format them.
+        Map the dataset to the document search data schema.
 
-        Returns:
-            The document search evaluation data.
+        Args:
+            dataset: The dataset to map.
 
-        Raises:
-            DataLoaderIncorrectFormatDataError: If evaluation dataset is incorrectly formatted.
+        Returns:
+            The document search data.
         """
-        data_path = await self.source.fetch()
-        dataset = load_dataset(
-            path=str(data_path.parent),
-            split="train",
-            data_files={"train": str(data_path.name)},
-        )
-        if self.question_key not in dataset.features:
-            raise DataLoaderIncorrectFormatDataError(
-                required_features=[self.question_key],
-                data_path=data_path,
-            )
-
         return [
             DocumentSearchData(
-                question=data.get(self.question_key),
+                question=data.get(self.question_key, ""),
                 reference_document_ids=data.get(self.document_ids_key),
                 reference_passages=data.get(self.passages_key),
                 reference_page_numbers=data.get(self.page_numbers_key),

ragbits/evaluate/dataloaders/question_answer.py

@@ -0,0 +1,57 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.question_answer import QuestionAnswerData
+
+
+class QuestionAnswerDataLoader(DataLoader[QuestionAnswerData]):
+    """
+    Question answer evaluation data loader.
+
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        answer_key: str = "answer",
+        context_key: str = "context",
+    ) -> None:
+        """
+        Initialize the question answer data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from.
+            required_keys: The required keys to load the data from.
+            question_key: The dataset column name that contains the question.
+            answer_key: The dataset column name that contains the answer.
+            context_key: The dataset column name that contains the context. Context is optional.
+        """
+        super().__init__(source=source, split=split, required_keys={question_key, answer_key})
+        self.question_key = question_key
+        self.answer_key = answer_key
+        self.context_key = context_key
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[QuestionAnswerData]:
+        """
+        Map the dataset to the question answer data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The question answer data.
+        """
+        return [
+            QuestionAnswerData(
+                question=data.get(self.question_key, ""),
+                reference_answer=data.get(self.answer_key, ""),
+                reference_context=data.get(self.context_key),
+            )
+            for data in dataset
+        ]

ragbits/evaluate/metrics/question_answer.py

@@ -0,0 +1,182 @@
+import asyncio
+from abc import ABC, abstractmethod
+from itertools import chain
+from typing import Generic, TypeVar
+
+from continuous_eval.llm_factory import LLMInterface
+from continuous_eval.metrics.base import LLMBasedMetric
+from continuous_eval.metrics.generation.text import (
+    LLMBasedAnswerCorrectness,
+    LLMBasedAnswerRelevance,
+    LLMBasedFaithfulness,
+    LLMBasedStyleConsistency,
+)
+from typing_extensions import Self
+
+from ragbits.agents.types import QuestionAnswerPromptOutputT
+from ragbits.core.llms.base import LLM
+from ragbits.core.utils.helpers import batched
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.question_answer import QuestionAnswerResult
+
+MetricT = TypeVar("MetricT", bound=LLMBasedMetric)
+
+
+class _MetricLMM(LLMInterface):
+    """
+    Implementation of required interface of Relari generative metrics based on LiteLMM.
+    """
+
+    def __init__(self, llm: LLM) -> None:
+        self._llm = llm
+
+    def run(self, prompt: dict[str, str], temperature: float = 0, max_tokens: int = 1024) -> str:
+        formatted_prompt = [
+            {"role": "system", "content": prompt["system_prompt"]},
+            {"role": "user", "content": prompt["user_prompt"]},
+        ]
+        options = self._llm.options_cls(
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        return asyncio.run(self._llm.generate(formatted_prompt, options=options))
+
+
+class QuestionAnswerMetric(Generic[MetricT], Metric[QuestionAnswerResult], ABC):
+    """
+    Metric for question answer evaluation based on Relari backend.
+    More details can be found [here](https://docs.relari.ai/category/text-generation).
+    """
+
+    metric_cls: type[MetricT]
+
+    def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
+        """
+        Initialize the agent metric.
+
+        Args:
+            llm: Judge LLM instance.
+            batch_size: Batch size for metric computation.
+            weight: Metric value weight in the final score, used during optimization.
+        """
+        super().__init__(weight=weight)
+        self.metric = self.metric_cls(_MetricLMM(llm))
+        self.batch_size = batch_size
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `QuestionAnswerMetric` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the metric.
+
+        Returns:
+            An instance of the metric class initialized with the provided configuration.
+        """
+        config["llm"] = LLM.from_config(config["llm"])
+        config["batch_size"] = config.get("batch_size", 15)
+        config["weight"] = config.get("weight", 1.0)
+        return super().from_config(config)
+
+    async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
+        """
+        Compute the metric.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+        metric_results = chain.from_iterable(
+            [
+                await asyncio.gather(*[asyncio.to_thread(self._call_metric, result) for result in batch])
+                for batch in batched(results, self.batch_size)
+            ]
+        )
+        return self.metric.aggregate(list(metric_results))
+
+    @abstractmethod
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        """
+        Call the metric with the proper arguments.
+        """
+
+
+class QuestionAnswerAnswerCorrectness(QuestionAnswerMetric[LLMBasedAnswerCorrectness]):
+    """
+    Metric checking answer correctness based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_correctness).
+    """
+
+    metric_cls: type[LLMBasedAnswerCorrectness] = LLMBasedAnswerCorrectness
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            ground_truth_answers=result.reference_answer,
+        )
+
+
+class QuestionAnswerAnswerFaithfulness(QuestionAnswerMetric[LLMBasedFaithfulness]):
+    """
+    Metric checking answer faithfulness based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_faithfulness).
+    """
+
+    metric_cls: type[LLMBasedFaithfulness] = LLMBasedFaithfulness
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            retrieved_context=result.reference_context,
+        )
+
+
+class QuestionAnswerAnswerRelevance(QuestionAnswerMetric[LLMBasedAnswerRelevance]):
+    """
+    Metric checking answer relevance based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_relevance).
+    """
+
+    metric_cls: type[LLMBasedAnswerRelevance] = LLMBasedAnswerRelevance
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+        )
+
+
+class QuestionAnswerAnswerConsistency(QuestionAnswerMetric[LLMBasedStyleConsistency]):
+    """
+    Metric checking answer relevance based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_style).
+    """
+
+    metric_cls: type[LLMBasedStyleConsistency] = LLMBasedStyleConsistency
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            ground_truth_answers=result.reference_answer,
+        )

ragbits/evaluate/pipelines/question_answer.py

@@ -0,0 +1,96 @@
+import asyncio
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any, Generic
+
+from typing_extensions import Self
+
+from ragbits.agents._main import AgentResult
+from ragbits.agents.types import (
+    QuestionAnswerAgent,
+    QuestionAnswerPromptInput,
+    QuestionAnswerPromptOutputT,
+)
+from ragbits.core.llms.base import LLMClientOptionsT
+from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult
+
+
+class QuestionAnswerData(EvaluationData):
+    """
+    Represents the evaluation data for question answer.
+    """
+
+    question: str
+    reference_answer: str
+    reference_context: Any | None = None
+
+
+@dataclass
+class QuestionAnswerResult(EvaluationResult, Generic[QuestionAnswerPromptOutputT]):
+    """
+    Represents the result of a single evaluation.
+    """
+
+    question: str
+    predicted_result: AgentResult[QuestionAnswerPromptOutputT]
+    reference_answer: str
+    reference_context: Any | None = None
+
+
+class QuestionAnswerPipeline(
+    EvaluationPipeline[
+        QuestionAnswerAgent[LLMClientOptionsT, QuestionAnswerPromptInput, QuestionAnswerPromptOutputT],
+        QuestionAnswerData,
+        QuestionAnswerResult,
+    ]
+):
+    """
+    Question answer evaluation pipeline.
+    """
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `QuestionAnswerPipeline` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the pipeline.
+
+        Returns:
+            An instance of the pipeline class initialized with the provided configuration.
+        """
+        config["evaluation_target"] = QuestionAnswerAgent.from_config(config)
+        return super().from_config(config)
+
+    async def __call__(
+        self, data: Iterable[QuestionAnswerData]
+    ) -> Iterable[QuestionAnswerResult[QuestionAnswerPromptOutputT]]:
+        """
+        Run the question answer evaluation pipeline.
+
+        Args:
+            data: The evaluation data batch.
+
+        Returns:
+            The evaluation result batch.
+        """
+        results = await asyncio.gather(
+            *[
+                self.evaluation_target.run(
+                    QuestionAnswerPromptInput(
+                        question=row.question,
+                        context=row.reference_context,
+                    )
+                )
+                for row in data
+            ]
+        )
+        return [
+            QuestionAnswerResult(
+                question=row.question,
+                predicted_result=result,
+                reference_answer=row.reference_answer,
+                reference_context=row.reference_context,
+            )
+            for row, result in zip(data, results, strict=False)
+        ]

{ragbits_evaluate-0.18.0.dist-info → ragbits_evaluate-0.19.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragbits-evaluate
-Version: 0.18.0
+Version: 0.19.1
 Summary: Evaluation module for Ragbits components
 Project-URL: Homepage, https://github.com/deepsense-ai/ragbits
 Project-URL: Bug Reports, https://github.com/deepsense-ai/ragbits/issues
@@ -27,7 +27,7 @@ Requires-Dist: distilabel<2.0.0,>=1.4.1
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: neptune[optuna]<2.0.0,>=1.12.0
 Requires-Dist: optuna<5.0.0,>=4.0.0
-Requires-Dist: ragbits-core==0.18.0
+Requires-Dist: ragbits-core==0.19.1
 Provides-Extra: relari
 Requires-Dist: continuous-eval<1.0.0,>=0.3.12; extra == 'relari'
 Description-Content-Type: text/markdown

{ragbits_evaluate-0.18.0.dist-info → ragbits_evaluate-0.19.1.dist-info}/RECORD

@@ -6,9 +6,10 @@ ragbits/evaluate/optimizer.py,sha256=RqYgoiCIEhjXO0HEN6uwldblHyoPuT3qUdncuHPZgCg
 ragbits/evaluate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/utils.py,sha256=w-hbvKRHI9tEva9wKDTVla0Wm2eCHT2MxVkof27Sqfw,4831
 ragbits/evaluate/dataloaders/__init__.py,sha256=UFJFjmvi3GUQFsx6A5sYD01HH2f7TXcHRW2VNM1pmIA,83
-ragbits/evaluate/dataloaders/base.py,sha256=ovL38_tH12q9wd3yeflIlovGuSD8S1X9HUUtwv17QrM,1774
-ragbits/evaluate/dataloaders/document_search.py,sha256=BLOaXP6TVtSsV2xScY4X_th285hWI4b9lcRuUXNxZ3U,2813
+ragbits/evaluate/dataloaders/base.py,sha256=x8rEl5utNOziF_9urL0grkqoXwMgaDWYSM5akw3Kt9Y,3213
+ragbits/evaluate/dataloaders/document_search.py,sha256=c9Bc4ZtFEKAiG9B70JFiBZlZDkBSGNWFRKabF7PMTU0,2495
 ragbits/evaluate/dataloaders/exceptions.py,sha256=xUOBLj1JuCkcqzRVnu0A0I_i1THxbDt2MEDVdDGjDyY,735
+ragbits/evaluate/dataloaders/question_answer.py,sha256=naXFDtla0otOTWSyHVvWvgDYEq-Wry4irnAJR2tHMNg,2032
 ragbits/evaluate/dataset_generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/dataset_generator/pipeline.py,sha256=dgnV-Qm0Z7S1Y6ga9-9RscXxxr3krOKsIj7E9WS4ANk,4940
 ragbits/evaluate/dataset_generator/utils.py,sha256=zD-ksXlX62kkIgzBefE4ILsP7He9bHimnZ63LLsMKCA,1325
@@ -27,9 +28,11 @@ ragbits/evaluate/factories/__init__.py,sha256=7nh0J80EfqMWRGtHx4hkfHNMztfC6FMhH8
 ragbits/evaluate/metrics/__init__.py,sha256=Mr83ytGyvdXtBlr7Bbo0-5auE0530xsd3wffKSIf8cE,95
 ragbits/evaluate/metrics/base.py,sha256=bOscQ_nJXLGWmP2ls9jncrUoeghNBnKDJsab71pFEjo,2519
 ragbits/evaluate/metrics/document_search.py,sha256=MfvMwEPenqiJdKYuW6WLvmtMch9ZVYb0T6ibpOF3vGI,3189
+ragbits/evaluate/metrics/question_answer.py,sha256=_XMFjkJcG-xdOO2fCfoKIhJb5VVM_GK_yKhFGXO8FRM,6566
 ragbits/evaluate/pipelines/__init__.py,sha256=Bqp_L7aRq12Ua19ELZDsdYvra6-GlLrQ9cIG2IWArko,1294
 ragbits/evaluate/pipelines/base.py,sha256=QV3fjPnbJjeCgcbt8yV1Ho3BamEUc3wSca3MAzaBlV0,1739
 ragbits/evaluate/pipelines/document_search.py,sha256=tgk-I21eshdBbWVsuNa1zWK_fWuDNXhhMCn1_Fdu_Ko,3840
-ragbits_evaluate-0.18.0.dist-info/METADATA,sha256=in_9YOI8UrGJR_tXGPKJYoinR6Ju4Taa0PaK9DQVCnw,2300
-ragbits_evaluate-0.18.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ragbits_evaluate-0.18.0.dist-info/RECORD,,
+ragbits/evaluate/pipelines/question_answer.py,sha256=3CYVHDLnOy4z7kgYPMluiJ8POulHo-w3PEiqvqsF4Dc,2797
+ragbits_evaluate-0.19.1.dist-info/METADATA,sha256=xO13dMZHaA1yUZ-kppa6R8KzVSsn_G4VjHKqXMew3ak,2300
+ragbits_evaluate-0.19.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ragbits_evaluate-0.19.1.dist-info/RECORD,,