ragbits-evaluate 0.17.1__py3-none-any.whl → 0.19.0__py3-none-any.whl

ragbits/evaluate/cli.py

@@ -140,10 +140,10 @@ def run() -> None:
         metric_results = await evaluator.compute(
             pipeline=state.pipeline,
             dataloader=state.dataloader,
-            metrics=state.metrics,
+            metricset=state.metrics,
         )
         evaluation_results = EvaluationResult(
-            metrics={"metrics": metric_results["metrics"], "time_perf": metric_results["time_perf"]}
+            metrics={"metrics": metric_results.metrics, "time_perf": metric_results.time_perf}
         )
         print_output(evaluation_results)
 

ragbits/evaluate/dataloaders/base.py

@@ -3,12 +3,14 @@ from collections.abc import Iterable
 from types import ModuleType
 from typing import ClassVar, Generic
 
+from datasets import load_dataset
 from pydantic import BaseModel
 from typing_extensions import Self
 
 from ragbits.core.sources.base import Source
 from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
 from ragbits.evaluate import dataloaders
+from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
 from ragbits.evaluate.pipelines.base import EvaluationDataT
 
 
@@ -28,14 +30,19 @@ class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
     default_module: ClassVar[ModuleType | None] = dataloaders
     configuration_key: ClassVar[str] = "dataloader"
 
-    def __init__(self, source: Source) -> None:
+    def __init__(self, source: Source, *, split: str = "data", required_keys: set[str] | None = None) -> None:
         """
         Initialize the data loader.
 
         Args:
             source: The source to load the evaluation data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
+            required_keys: The required columns for the evaluation data.
         """
         self.source = source
+        self.split = split
+        self.required_keys = required_keys or set()
 
     @classmethod
     def from_config(cls, config: dict) -> Self:
@@ -52,11 +59,37 @@ class DataLoader(WithConstructionConfig, Generic[EvaluationDataT], ABC):
         config["source"] = Source.subclass_from_config(dataloader_config.source)
         return super().from_config(config)
 
-    @abstractmethod
     async def load(self) -> Iterable[EvaluationDataT]:
         """
         Load the data.
 
         Returns:
-            The loaded data.
+            The loaded evaluation data.
+
+        Raises:
+            DataLoaderIncorrectFormatDataError: If evaluation dataset is incorrectly formatted.
+        """
+        data_path = await self.source.fetch()
+        dataset = load_dataset(
+            path=str(data_path.parent),
+            data_files={"data": str(data_path.name)},
+            split=self.split,
+        )
+        if not self.required_keys.issubset(dataset.features):
+            raise DataLoaderIncorrectFormatDataError(
+                required_features=list(self.required_keys),
+                data_path=data_path,
+            )
+        return await self.map(dataset.to_list())
+
+    @abstractmethod
+    async def map(self, dataset: Iterable[dict]) -> Iterable[EvaluationDataT]:
+        """
+        Map the dataset to the evaluation data.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The evaluation data.
         """

ragbits/evaluate/dataloaders/document_search.py

@@ -1,9 +1,7 @@
 from collections.abc import Iterable
 
-from datasets import load_dataset
-
+from ragbits.core.sources.base import Source
 from ragbits.evaluate.dataloaders.base import DataLoader
-from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
 from ragbits.evaluate.pipelines.document_search import DocumentSearchData
 
 
@@ -11,35 +9,53 @@ class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
     """
     Document search evaluation data loader.
 
-    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files)
-    and contain the following features: "question, "passages".
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
     """
 
-    async def load(self) -> Iterable[DocumentSearchData]:
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        document_ids_key: str = "document_ids",
+        passages_key: str = "passages",
+        page_numbers_key: str = "page_numbers",
+    ) -> None:
         """
-        Load the data from source and format them.
-
-        Returns:
-            The document search evaluation data.
+        Initialize the document search data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from. Split is fixed for data loaders to "data",
+                but you can slice it using the [Hugging Face API](https://huggingface.co/docs/datasets/v1.11.0/splits.html#slicing-api).
+            question_key: The dataset column name that contains the question.
+            document_ids_key: The dataset column name that contains the document ids. Document ids are optional.
+            passages_key: The dataset column name that contains the passages. Passages are optional.
+            page_numbers_key: The dataset column name that contains the page numbers. Page numbers are optional.
+        """
+        super().__init__(source=source, split=split, required_keys={question_key})
+        self.question_key = question_key
+        self.document_ids_key = document_ids_key
+        self.passages_key = passages_key
+        self.page_numbers_key = page_numbers_key
 
-        Raises:
-            DataLoaderIncorrectFormatDataError: If evaluation dataset is incorrectly formatted.
+    async def map(self, dataset: Iterable[dict]) -> Iterable[DocumentSearchData]:
         """
-        data_path = await self.source.fetch()
-        dataset = load_dataset(
-            path=str(data_path.parent),
-            split=data_path.stem,
-        )
-        if "question" not in dataset.features or "passages" not in dataset.features:
-            raise DataLoaderIncorrectFormatDataError(
-                required_features=["question", "passages"],
-                data_path=data_path,
-            )
+        Map the dataset to the document search data schema.
+
+        Args:
+            dataset: The dataset to map.
 
+        Returns:
+            The document search data.
+        """
         return [
             DocumentSearchData(
-                question=data["question"],
-                reference_passages=data["passages"],
+                question=data.get(self.question_key, ""),
+                reference_document_ids=data.get(self.document_ids_key),
+                reference_passages=data.get(self.passages_key),
+                reference_page_numbers=data.get(self.page_numbers_key),
             )
             for data in dataset
         ]

ragbits/evaluate/dataloaders/question_answer.py

@@ -0,0 +1,57 @@
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.question_answer import QuestionAnswerData
+
+
+class QuestionAnswerDataLoader(DataLoader[QuestionAnswerData]):
+    """
+    Question answer evaluation data loader.
+
+    The source used for this data loader should point to a file that can be loaded by [Hugging Face](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        answer_key: str = "answer",
+        context_key: str = "context",
+    ) -> None:
+        """
+        Initialize the question answer data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from.
+            required_keys: The required keys to load the data from.
+            question_key: The dataset column name that contains the question.
+            answer_key: The dataset column name that contains the answer.
+            context_key: The dataset column name that contains the context. Context is optional.
+        """
+        super().__init__(source=source, split=split, required_keys={question_key, answer_key})
+        self.question_key = question_key
+        self.answer_key = answer_key
+        self.context_key = context_key
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[QuestionAnswerData]:
+        """
+        Map the dataset to the question answer data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The question answer data.
+        """
+        return [
+            QuestionAnswerData(
+                question=data.get(self.question_key, ""),
+                reference_answer=data.get(self.answer_key, ""),
+                reference_context=data.get(self.context_key),
+            )
+            for data in dataset
+        ]

ragbits/evaluate/evaluator.py

@@ -1,16 +1,45 @@
 import asyncio
+import random
 import time
-from collections.abc import Iterable
-from dataclasses import asdict
+from collections.abc import Awaitable, Callable, Iterable
+from dataclasses import dataclass
+from typing import Generic, ParamSpec, TypeVar
 
 from pydantic import BaseModel
-from tqdm.asyncio import tqdm
+from tqdm import tqdm
 
 from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
+from ragbits.core.utils.helpers import batched
 from ragbits.evaluate.dataloaders.base import DataLoader
 from ragbits.evaluate.metrics.base import MetricSet
 from ragbits.evaluate.pipelines.base import EvaluationDataT, EvaluationPipeline, EvaluationResultT, EvaluationTargetT
 
+_CallP = ParamSpec("_CallP")
+_CallReturnT = TypeVar("_CallReturnT")
+
+
+@dataclass
+class EvaluationTimePerf:
+    """
+    Container for evaluation time performance metrics.
+    """
+
+    total_time_in_seconds: float
+    samples_per_second: float
+    latency_in_seconds: float
+
+
+@dataclass
+class EvaluatorResult(Generic[EvaluationResultT]):
+    """
+    Container for evaluation results.
+    """
+
+    metrics: dict[str, int | float]
+    results: list[EvaluationResultT]
+    errors: list[Exception]
+    time_perf: EvaluationTimePerf
+
 
 class EvaluationConfig(BaseModel):
     """
@@ -36,17 +65,29 @@ class Evaluator(WithConstructionConfig):
     Evaluator class.
     """
 
-    def __init__(self, batch_size: int = 10) -> None:
+    def __init__(
+        self,
+        batch_size: int = 10,
+        num_retries: int = 3,
+        backoff_multiplier: int = 1,
+        backoff_max: int = 60,
+    ) -> None:
         """
-        Initialize the evaluator.
+        Initialize the Evaluator instance.
 
         Args:
             batch_size: batch size for the evaluation pipeline inference.
+            num_retries: The number of retries per evaluation pipeline inference error.
+            backoff_multiplier: The base delay multiplier for exponential backoff (in seconds).
+            backoff_max: The maximum allowed delay (in seconds) between retries.
         """
         self.batch_size = batch_size
+        self.num_retries = num_retries
+        self.backoff_multiplier = backoff_multiplier
+        self.backoff_max = backoff_max
 
     @classmethod
-    async def run_from_config(cls, config: dict) -> dict:
+    async def run_from_config(cls, config: dict) -> EvaluatorResult:
         """
         Run the evaluation based on configuration.
 
@@ -60,50 +101,50 @@ class Evaluator(WithConstructionConfig):
         evaluation_config = EvaluationConfig.model_validate(evaluator_config.evaluation)
         pipeline: EvaluationPipeline = EvaluationPipeline.subclass_from_config(evaluation_config.pipeline)
         dataloader: DataLoader = DataLoader.subclass_from_config(evaluation_config.dataloader)
-        metrics: MetricSet = MetricSet.from_config(evaluation_config.metrics)
+        metricset: MetricSet = MetricSet.from_config(evaluation_config.metrics)
 
         evaluator = cls.from_config(evaluator_config.evaluator or {})
         return await evaluator.compute(
             pipeline=pipeline,
             dataloader=dataloader,
-            metrics=metrics,
+            metricset=metricset,
         )
 
     async def compute(
         self,
         pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
         dataloader: DataLoader[EvaluationDataT],
-        metrics: MetricSet[EvaluationResultT],
-    ) -> dict:
+        metricset: MetricSet[EvaluationResultT],
+    ) -> EvaluatorResult[EvaluationResultT]:
         """
         Compute the evaluation results for the given pipeline and data.
 
         Args:
             pipeline: The pipeline to be evaluated.
             dataloader: The dataloader to load the data.
-            metrics: The metrics to be computed.
+            metricset: The metrics to be computed.
 
         Returns:
             The evaluation results.
         """
-        dataset = await dataloader.load()
         await pipeline.prepare()
 
-        results, perf_results = await self._call_pipeline(pipeline, dataset)
-        computed_metrics = self._compute_metrics(metrics, results)
-        processed_results = self._results_processor(results)
+        dataset = await dataloader.load()
+        results, errors, time_perf = await self._call_pipeline(pipeline, dataset)
+        metrics = await metricset.compute(results)
 
-        return {
-            **perf_results,
-            **computed_metrics,
-            **processed_results,
-        }
+        return EvaluatorResult(
+            metrics=metrics,
+            results=results,
+            errors=errors,
+            time_perf=time_perf,
+        )
 
     async def _call_pipeline(
         self,
         pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
         dataset: Iterable[EvaluationDataT],
-    ) -> tuple[list[EvaluationResultT], dict]:
+    ) -> tuple[list[EvaluationResultT], list[Exception], EvaluationTimePerf]:
         """
         Call the pipeline with the given data.
 
@@ -114,47 +155,53 @@ class Evaluator(WithConstructionConfig):
         Returns:
             The evaluation results and performance metrics.
         """
-        semaphore = asyncio.Semaphore(self.batch_size)
-
-        async def _call_pipeline_with_semaphore(data: EvaluationDataT) -> EvaluationResultT:
-            async with semaphore:
-                return await pipeline(data)
-
         start_time = time.perf_counter()
-        pipe_outputs = await tqdm.gather(*[_call_pipeline_with_semaphore(data) for data in dataset], desc="Evaluation")
+        outputs = [
+            await self._call_with_error_handling(pipeline, data)
+            for data in tqdm(batched(dataset, self.batch_size), desc="Evaluation")
+        ]
         end_time = time.perf_counter()
 
-        return pipe_outputs, self._compute_time_perf(start_time, end_time, len(pipe_outputs))
+        errors = [output for output in outputs if isinstance(output, Exception)]
+        results = [item for output in outputs if not isinstance(output, Exception) for item in output]
 
-    @staticmethod
-    def _results_processor(results: list[EvaluationResultT]) -> dict:
+        return results, errors, self._compute_time_perf(start_time, end_time, len(outputs))
+
+    async def _call_with_error_handling(
+        self,
+        executable: Callable[_CallP, Awaitable[_CallReturnT]],
+        *executable_args: _CallP.args,
+        **executable_kwargs: _CallP.kwargs,
+    ) -> _CallReturnT | Exception:
         """
-        Process the results.
+        Call executable with a standarized error handling.
+        If an error occurs, the executable is retried `num_retries` times using randomized exponential backoff.
 
         Args:
-            results: The evaluation results.
+            executable: The callable function to execute.
+            executable_args: Positional arguments to pass to the executable.
+            executable_kwargs: Keyword arguments to pass to the executable.
 
         Returns:
-            The processed results.
-        """
-        return {"results": [asdict(result) for result in results]}
+            The result of the executable if successful.
 
-    @staticmethod
-    def _compute_metrics(metrics: MetricSet[EvaluationResultT], results: list[EvaluationResultT]) -> dict:
+        Raises:
+            Exception: The last encountered exception after all retries are exhausted.
         """
-        Compute a metric using the given inputs.
+        for i in range(max(0, self.num_retries) + 1):
+            try:
+                return await executable(*executable_args, **executable_kwargs)
+            except Exception as exc:
+                if i == self.num_retries:
+                    return exc
 
-        Args:
-            metrics: The metrics to be computed.
-            results: The evaluation results.
+                delay = random.uniform(0, min(2**i * self.backoff_multiplier, self.backoff_max))  # noqa: S311
+                await asyncio.sleep(delay)
 
-        Returns:
-            The computed metric.
-        """
-        return {"metrics": metrics.compute(results)}
+        raise RuntimeError("Unreachable code reached")  # mypy quirk
 
     @staticmethod
-    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> dict:
+    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> EvaluationTimePerf:
         """
         Compute the performance metrics.
 
@@ -170,10 +217,8 @@ class Evaluator(WithConstructionConfig):
         throughput = num_samples / latency
         latency_sample = 1.0 / throughput if throughput > 0 else 0.0
 
-        return {
-            "time_perf": {
-                "total_time_in_seconds": latency,
-                "samples_per_second": throughput,
-                "latency_in_seconds": latency_sample,
-            },
-        }
+        return EvaluationTimePerf(
+            total_time_in_seconds=latency,
+            samples_per_second=throughput,
+            latency_in_seconds=latency_sample,
+        )

ragbits/evaluate/factories/__init__.py

@@ -1,43 +1,21 @@
 import asyncio
 
+from continuous_eval.metrics.retrieval.matching_strategy import RougeChunkMatch
 from datasets import load_dataset
 
 from ragbits.core.embeddings.dense import LiteLLMEmbedder
 from ragbits.core.sources.hf import HuggingFaceSource
-from ragbits.core.utils.config_handling import ObjectConstructionConfig
 from ragbits.core.vector_stores.in_memory import InMemoryVectorStore
 from ragbits.document_search import DocumentSearch
 from ragbits.document_search.documents.document import DocumentMeta
 from ragbits.evaluate.dataloaders.document_search import DocumentSearchDataLoader
 from ragbits.evaluate.metrics import MetricSet
-
-DS_PRECISION_RECALL_F1 = {
-    "precision_recall_f1": ObjectConstructionConfig.model_validate(
-        {
-            "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1",
-            "config": {
-                "matching_strategy": {
-                    "type": "RougeChunkMatch",
-                    "config": {
-                        "threshold": 0.5,
-                    },
-                },
-            },
-        }
-    ),
-}
-
-
-def precision_recall_f1() -> MetricSet:
-    """
-    Factory of precision recall f1 metric set for retrival evaluation.
-    """
-    return MetricSet.from_config(config=DS_PRECISION_RECALL_F1)
+from ragbits.evaluate.metrics.document_search import DocumentSearchPrecisionRecallF1
 
 
 async def _add_example_documents(document_search: DocumentSearch) -> None:
     dataset = load_dataset(path="deepsense-ai/synthetic-rag-dataset_v1.0", split="train")
-    documents = [DocumentMeta.create_text_document_from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
+    documents = [DocumentMeta.from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
     await document_search.ingest(documents)
 
 
@@ -45,7 +23,7 @@ def basic_document_search_factory() -> DocumentSearch:
     """
     Factory for basic example document search instance.
     """
-    document_search = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
+    document_search: DocumentSearch = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
     asyncio.run(_add_example_documents(document_search))
     return document_search
 
@@ -55,3 +33,10 @@ def synthetic_rag_dataset() -> DocumentSearchDataLoader:
     Factory for synthetic RAG dataset.
     """
     return DocumentSearchDataLoader(source=HuggingFaceSource(path="deepsense-ai/synthetic-rag-dataset_v1.0"))
+
+
+def precision_recall_f1() -> MetricSet:
+    """
+    Factory of precision recall f1 metric set for retrival evaluation.
+    """
+    return MetricSet(DocumentSearchPrecisionRecallF1(matching_strategy=RougeChunkMatch()))

ragbits/evaluate/metrics/base.py

@@ -1,3 +1,4 @@
+import asyncio
 from abc import ABC, abstractmethod
 from types import ModuleType
 from typing import ClassVar, Generic
@@ -19,7 +20,7 @@ class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
 
     def __init__(self, weight: float = 1.0) -> None:
         """
-        Initializes the metric.
+        Initialize the metric.
 
         Args:
             weight: Metric value weight in the final score, used during optimization.
@@ -28,7 +29,7 @@ class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
         self.weight = weight
 
     @abstractmethod
-    def compute(self, results: list[EvaluationResultT]) -> dict:
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metric.
 
@@ -70,7 +71,7 @@ class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
         """
         return cls(*[Metric.subclass_from_config(metric_config) for metric_config in config.values()])
 
-    def compute(self, results: list[EvaluationResultT]) -> dict:
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metrics.
 
@@ -80,6 +81,9 @@ class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
         Returns:
             The computed metrics.
         """
+        metric_results = await asyncio.gather(*[metric.compute(results) for metric in self.metrics])
         return {
-            name: metric.weight * value for metric in self.metrics for name, value in metric.compute(results).items()
+            name: metric.weight * value
+            for metric, result in zip(self.metrics, metric_results, strict=False)
+            for name, value in result.items()
         }

ragbits/evaluate/metrics/document_search.py

@@ -46,7 +46,7 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
         matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
         return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))
 
-    def compute(self, results: list[DocumentSearchResult]) -> dict:
+    async def compute(self, results: list[DocumentSearchResult]) -> dict:
         """
         Compute the metric.
 
@@ -57,7 +57,18 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
             The computed metric.
         """
         return self.metric.aggregate(
-            [self.metric(result.predicted_passages, result.reference_passages) for result in results]
+            [
+                self.metric(
+                    [
+                        element.text_representation
+                        for element in result.predicted_elements
+                        if element.text_representation
+                    ],
+                    result.reference_passages,
+                )
+                for result in results
+                if result.reference_passages is not None
+            ]
         )
 
 

ragbits/evaluate/metrics/question_answer.py

@@ -0,0 +1,182 @@
+import asyncio
+from abc import ABC, abstractmethod
+from itertools import chain
+from typing import Generic, TypeVar
+
+from continuous_eval.llm_factory import LLMInterface
+from continuous_eval.metrics.base import LLMBasedMetric
+from continuous_eval.metrics.generation.text import (
+    LLMBasedAnswerCorrectness,
+    LLMBasedAnswerRelevance,
+    LLMBasedFaithfulness,
+    LLMBasedStyleConsistency,
+)
+from typing_extensions import Self
+
+from ragbits.agents.types import QuestionAnswerPromptOutputT
+from ragbits.core.llms.base import LLM
+from ragbits.core.utils.helpers import batched
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.question_answer import QuestionAnswerResult
+
+MetricT = TypeVar("MetricT", bound=LLMBasedMetric)
+
+
+class _MetricLMM(LLMInterface):
+    """
+    Implementation of required interface of Relari generative metrics based on LiteLMM.
+    """
+
+    def __init__(self, llm: LLM) -> None:
+        self._llm = llm
+
+    def run(self, prompt: dict[str, str], temperature: float = 0, max_tokens: int = 1024) -> str:
+        formatted_prompt = [
+            {"role": "system", "content": prompt["system_prompt"]},
+            {"role": "user", "content": prompt["user_prompt"]},
+        ]
+        options = self._llm.options_cls(
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        return asyncio.run(self._llm.generate(formatted_prompt, options=options))
+
+
+class QuestionAnswerMetric(Generic[MetricT], Metric[QuestionAnswerResult], ABC):
+    """
+    Metric for question answer evaluation based on Relari backend.
+    More details can be found [here](https://docs.relari.ai/category/text-generation).
+    """
+
+    metric_cls: type[MetricT]
+
+    def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
+        """
+        Initialize the agent metric.
+
+        Args:
+            llm: Judge LLM instance.
+            batch_size: Batch size for metric computation.
+            weight: Metric value weight in the final score, used during optimization.
+        """
+        super().__init__(weight=weight)
+        self.metric = self.metric_cls(_MetricLMM(llm))
+        self.batch_size = batch_size
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `QuestionAnswerMetric` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the metric.
+
+        Returns:
+            An instance of the metric class initialized with the provided configuration.
+        """
+        config["llm"] = LLM.from_config(config["llm"])
+        config["batch_size"] = config.get("batch_size", 15)
+        config["weight"] = config.get("weight", 1.0)
+        return super().from_config(config)
+
+    async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
+        """
+        Compute the metric.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+        metric_results = chain.from_iterable(
+            [
+                await asyncio.gather(*[asyncio.to_thread(self._call_metric, result) for result in batch])
+                for batch in batched(results, self.batch_size)
+            ]
+        )
+        return self.metric.aggregate(list(metric_results))
+
+    @abstractmethod
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        """
+        Call the metric with the proper arguments.
+        """
+
+
+class QuestionAnswerAnswerCorrectness(QuestionAnswerMetric[LLMBasedAnswerCorrectness]):
+    """
+    Metric checking answer correctness based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_correctness).
+    """
+
+    metric_cls: type[LLMBasedAnswerCorrectness] = LLMBasedAnswerCorrectness
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            ground_truth_answers=result.reference_answer,
+        )
+
+
+class QuestionAnswerAnswerFaithfulness(QuestionAnswerMetric[LLMBasedFaithfulness]):
+    """
+    Metric checking answer faithfulness based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_faithfulness).
+    """
+
+    metric_cls: type[LLMBasedFaithfulness] = LLMBasedFaithfulness
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            retrieved_context=result.reference_context,
+        )
+
+
+class QuestionAnswerAnswerRelevance(QuestionAnswerMetric[LLMBasedAnswerRelevance]):
+    """
+    Metric checking answer relevance based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_relevance).
+    """
+
+    metric_cls: type[LLMBasedAnswerRelevance] = LLMBasedAnswerRelevance
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+        )
+
+
+class QuestionAnswerAnswerConsistency(QuestionAnswerMetric[LLMBasedStyleConsistency]):
+    """
+    Metric checking answer relevance based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_style).
+    """
+
+    metric_cls: type[LLMBasedStyleConsistency] = LLMBasedStyleConsistency
+
+    def _call_metric(self, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        return self.metric(
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            ground_truth_answers=result.reference_answer,
+        )

ragbits/evaluate/optimizer.py

@@ -61,7 +61,7 @@ class Optimizer(WithConstructionConfig):
         evaluator_config = EvaluatorConfig.model_validate(optimizer_config.evaluator)
 
         dataloader: DataLoader = DataLoader.subclass_from_config(evaluator_config.evaluation.dataloader)
-        metrics: MetricSet = MetricSet.from_config(evaluator_config.evaluation.metrics)
+        metricset: MetricSet = MetricSet.from_config(evaluator_config.evaluation.metrics)
 
         pipeline_class = import_by_path(evaluator_config.evaluation.pipeline.type)
         pipeline_config = dict(evaluator_config.evaluation.pipeline.config)
@@ -71,7 +71,7 @@ class Optimizer(WithConstructionConfig):
         return optimizer.optimize(
             pipeline_class=pipeline_class,
             pipeline_config=pipeline_config,
-            metrics=metrics,
+            metricset=metricset,
             dataloader=dataloader,
             callbacks=callbacks,
         )
@@ -81,7 +81,7 @@ class Optimizer(WithConstructionConfig):
         pipeline_class: type[EvaluationPipeline],
         pipeline_config: dict,
         dataloader: DataLoader,
-        metrics: MetricSet,
+        metricset: MetricSet,
         callbacks: list[Callable] | None = None,
     ) -> list[tuple[dict, float, dict[str, float]]]:
         """
@@ -91,7 +91,7 @@ class Optimizer(WithConstructionConfig):
             pipeline_class: Pipeline to be optimized.
             pipeline_config: Configuration defining the optimization process.
             dataloader: Data loader.
-            metrics: Metrics to be optimized.
+            metricset: Metrics to be optimized.
             callbacks: Experiment callbacks.
 
         Returns:
@@ -104,7 +104,7 @@ class Optimizer(WithConstructionConfig):
                 pipeline_class=pipeline_class,
                 pipeline_config=pipeline_config,
                 dataloader=dataloader,
-                metrics=metrics,
+                metricset=metricset,
             )
 
         study = optuna.create_study(direction=self.direction)
@@ -131,7 +131,7 @@ class Optimizer(WithConstructionConfig):
         pipeline_class: type[EvaluationPipeline],
         pipeline_config: dict,
         dataloader: DataLoader,
-        metrics: MetricSet,
+        metricset: MetricSet,
     ) -> float:
         """
         Run a single experiment.
@@ -153,11 +153,11 @@ class Optimizer(WithConstructionConfig):
                     evaluator.compute(
                         pipeline=pipeline,
                         dataloader=dataloader,
-                        metrics=metrics,
+                        metricset=metricset,
                     )
                 )
-                score = sum(results["metrics"].values())
-                metrics_values = results["metrics"]
+                score = sum(results.metrics.values())
+                metrics_values = results.metrics
                 break
             except Exception as exc:
                 message = (

ragbits/evaluate/pipelines/base.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from collections.abc import Iterable
 from dataclasses import dataclass
 from types import ModuleType
 from typing import ClassVar, Generic, TypeVar
@@ -51,7 +52,7 @@ class EvaluationPipeline(WithConstructionConfig, Generic[EvaluationTargetT, Eval
         pass
 
     @abstractmethod
-    async def __call__(self, data: EvaluationDataT) -> EvaluationResultT:
+    async def __call__(self, data: Iterable[EvaluationDataT]) -> Iterable[EvaluationResultT]:
         """
         Run the evaluation pipeline.
 

ragbits/evaluate/pipelines/document_search.py

@@ -1,3 +1,5 @@
+import asyncio
+from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from uuid import uuid4
 
@@ -5,6 +7,7 @@ from typing_extensions import Self
 
 from ragbits.core.sources.hf import HuggingFaceSource
 from ragbits.document_search import DocumentSearch
+from ragbits.document_search.documents.element import Element
 from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult
 
 
@@ -14,7 +17,9 @@ class DocumentSearchData(EvaluationData):
     """
 
     question: str
-    reference_passages: list[str]
+    reference_document_ids: list[str | int] | None = None
+    reference_passages: list[str] | None = None
+    reference_page_numbers: list[int] | None = None
 
 
 @dataclass
@@ -24,8 +29,10 @@ class DocumentSearchResult(EvaluationResult):
     """
 
     question: str
-    reference_passages: list[str]
-    predicted_passages: list[str]
+    predicted_elements: Sequence[Element]
+    reference_document_ids: list[str | int] | None = None
+    reference_passages: list[str] | None = None
+    reference_page_numbers: list[int] | None = None
 
 
 class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch, DocumentSearchData, DocumentSearchResult]):
@@ -60,7 +67,7 @@ class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch, DocumentSearchDa
         # TODO: optimize this for cases with duplicated document search configs between runs
         if config.get("source"):
             config["vector_store"]["config"]["index_name"] = str(uuid4())
-        evaluation_target = DocumentSearch.from_config(config)
+        evaluation_target: DocumentSearch = DocumentSearch.from_config(config)
         return cls(evaluation_target=evaluation_target, source=config.get("source"))
 
     async def prepare(self) -> None:
@@ -76,21 +83,24 @@ class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch, DocumentSearchDa
             )
             await self.evaluation_target.ingest(sources)
 
-    async def __call__(self, data: DocumentSearchData) -> DocumentSearchResult:
+    async def __call__(self, data: Iterable[DocumentSearchData]) -> Iterable[DocumentSearchResult]:
         """
         Run the document search evaluation pipeline.
 
         Args:
-            data: The evaluation data.
+            data: The evaluation data batch.
 
         Returns:
-            The evaluation result.
+            The evaluation result batch.
         """
-        elements = await self.evaluation_target.search(data.question)
-        predicted_passages = [element.text_representation for element in elements if element.text_representation]
-
-        return DocumentSearchResult(
-            question=data.question,
-            reference_passages=data.reference_passages,
-            predicted_passages=predicted_passages,
-        )
+        results = await asyncio.gather(*[self.evaluation_target.search(row.question) for row in data])
+        return [
+            DocumentSearchResult(
+                question=row.question,
+                predicted_elements=elements,
+                reference_document_ids=row.reference_document_ids,
+                reference_passages=row.reference_passages,
+                reference_page_numbers=row.reference_page_numbers,
+            )
+            for row, elements in zip(data, results, strict=False)
+        ]

ragbits/evaluate/pipelines/question_answer.py

@@ -0,0 +1,96 @@
+import asyncio
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any, Generic
+
+from typing_extensions import Self
+
+from ragbits.agents._main import AgentResult
+from ragbits.agents.types import (
+    QuestionAnswerAgent,
+    QuestionAnswerPromptInput,
+    QuestionAnswerPromptOutputT,
+)
+from ragbits.core.llms.base import LLMClientOptionsT
+from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult
+
+
+class QuestionAnswerData(EvaluationData):
+    """
+    Represents the evaluation data for question answer.
+    """
+
+    question: str
+    reference_answer: str
+    reference_context: Any | None = None
+
+
+@dataclass
+class QuestionAnswerResult(EvaluationResult, Generic[QuestionAnswerPromptOutputT]):
+    """
+    Represents the result of a single evaluation.
+    """
+
+    question: str
+    predicted_result: AgentResult[QuestionAnswerPromptOutputT]
+    reference_answer: str
+    reference_context: Any | None = None
+
+
+class QuestionAnswerPipeline(
+    EvaluationPipeline[
+        QuestionAnswerAgent[LLMClientOptionsT, QuestionAnswerPromptInput, QuestionAnswerPromptOutputT],
+        QuestionAnswerData,
+        QuestionAnswerResult,
+    ]
+):
+    """
+    Question answer evaluation pipeline.
+    """
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `QuestionAnswerPipeline` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the pipeline.
+
+        Returns:
+            An instance of the pipeline class initialized with the provided configuration.
+        """
+        config["evaluation_target"] = QuestionAnswerAgent.from_config(config)
+        return super().from_config(config)
+
+    async def __call__(
+        self, data: Iterable[QuestionAnswerData]
+    ) -> Iterable[QuestionAnswerResult[QuestionAnswerPromptOutputT]]:
+        """
+        Run the question answer evaluation pipeline.
+
+        Args:
+            data: The evaluation data batch.
+
+        Returns:
+            The evaluation result batch.
+        """
+        results = await asyncio.gather(
+            *[
+                self.evaluation_target.run(
+                    QuestionAnswerPromptInput(
+                        question=row.question,
+                        context=row.reference_context,
+                    )
+                )
+                for row in data
+            ]
+        )
+        return [
+            QuestionAnswerResult(
+                question=row.question,
+                predicted_result=result,
+                reference_answer=row.reference_answer,
+                reference_context=row.reference_context,
+            )
+            for row, result in zip(data, results, strict=False)
+        ]

ragbits/evaluate/utils.py

@@ -1,5 +1,7 @@
 import json
 import sys
+import traceback
+from dataclasses import asdict
 from datetime import datetime
 from pathlib import Path
 from typing import Any
@@ -12,13 +14,15 @@ from neptune.utils import stringify_unsupported
 from neptune_optuna import NeptuneCallback
 from omegaconf import DictConfig
 
+from ragbits.evaluate.evaluator import EvaluatorResult
 
-def log_evaluation_to_file(results: dict, output_dir: Path | None = None) -> Path:
+
+def log_evaluation_to_file(result: EvaluatorResult, output_dir: Path | None = None) -> Path:
     """
-    Log the evaluation results locally.
+    Log the evaluation result locally.
 
     Args:
-        results: The evaluation results.
+        result: The evaluation result.
         output_dir: The output directory.
 
     Returns:
@@ -27,27 +31,57 @@ def log_evaluation_to_file(results: dict, output_dir: Path | None = None) -> Pat
     output_dir = output_dir or Path(HydraConfig.get().runtime.output_dir)
     metrics_file = output_dir / "metrics.json"
     results_file = output_dir / "results.json"
-
-    _save_json(metrics_file, metrics=results["metrics"], time_perf=results["time_perf"])
-    _save_json(results_file, results=results["results"])
+    errors_file = output_dir / "errors.json"
+
+    _save_json(metrics_file, metrics=result.metrics, time_perf=asdict(result.time_perf))
+    _save_json(results_file, results=[asdict(entry) for entry in result.results])
+    _save_json(
+        errors_file,
+        errors=[
+            {
+                "type": exc.__class__.__name__,
+                "message": str(exc),
+                "stacktrace": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+            }
+            for exc in result.errors
+        ],
+    )
 
     return output_dir
 
 
-def log_evaluation_to_neptune(results: dict, config: DictConfig, tags: str | list[str] | None = None) -> None:
+def log_evaluation_to_neptune(result: EvaluatorResult, config: DictConfig, tags: str | list[str] | None = None) -> None:
     """
-    Log the evaluation results to Neptune.
+    Log the evaluation result to Neptune.
 
     Args:
-        results: Evaluation results.
-        config: Evaluation configuration.
-        tags: Experiment tags.
+        result: The evaluation result.
+        config: The evaluation configuration.
+        tags: The experiment tags.
     """
     run = Run(tags=tags)
     run["config"] = stringify_unsupported(config)
-    run["evaluation/metrics"] = stringify_unsupported(results["metrics"])
-    run["evaluation/time_perf"] = stringify_unsupported(results["time_perf"])
-    run["evaluation/results"].upload(File.from_content(json.dumps(results["results"], indent=4), extension="json"))
+    run["evaluation/metrics"] = stringify_unsupported(result.metrics)
+    run["evaluation/time_perf"] = stringify_unsupported(asdict(result.time_perf))
+    run["evaluation/results"].upload(
+        File.from_content(json.dumps([asdict(entry) for entry in result.results], indent=4), extension="json")
+    )
+    run["evaluation/errors"].upload(
+        File.from_content(
+            json.dumps(
+                [
+                    {
+                        "type": exc.__class__.__name__,
+                        "message": str(exc),
+                        "stacktrace": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+                    }
+                    for exc in result.errors
+                ],
+                indent=4,
+            ),
+            extension="json",
+        )
+    )
 
 
 def log_dataset_to_file(dataset: Dataset, output_dir: Path | None = None) -> Path:

{ragbits_evaluate-0.17.1.dist-info → ragbits_evaluate-0.19.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragbits-evaluate
-Version: 0.17.1
+Version: 0.19.0
 Summary: Evaluation module for Ragbits components
 Project-URL: Homepage, https://github.com/deepsense-ai/ragbits
 Project-URL: Bug Reports, https://github.com/deepsense-ai/ragbits/issues
@@ -27,7 +27,7 @@ Requires-Dist: distilabel<2.0.0,>=1.4.1
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: neptune[optuna]<2.0.0,>=1.12.0
 Requires-Dist: optuna<5.0.0,>=4.0.0
-Requires-Dist: ragbits-core==0.17.1
+Requires-Dist: ragbits-core==0.19.0
 Provides-Extra: relari
 Requires-Dist: continuous-eval<1.0.0,>=0.3.12; extra == 'relari'
 Description-Content-Type: text/markdown

{ragbits_evaluate-0.17.1.dist-info → ragbits_evaluate-0.19.0.dist-info}/RECORD

@@ -1,14 +1,15 @@
 ragbits/evaluate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragbits/evaluate/cli.py,sha256=MEDo8ubk81TCNx-fq-liF0P5hjn2-kPpIfq54fReKIY,4509
+ragbits/evaluate/cli.py,sha256=vP8l2DyNXpR6jQP83wXKP_RRnGjEXjKnTVBg9RPbDKo,4505
 ragbits/evaluate/config.py,sha256=2WSmbVxyQi893L2FSjRFQoXkWZp1GetcNmR2GCDe0tA,339
-ragbits/evaluate/evaluator.py,sha256=Cif-QX2n5awOGm-AfFy2nRXkb_m4vGY_JZ_o4K4PhZI,5552
-ragbits/evaluate/optimizer.py,sha256=egcU54aADqKrN31NPqj7cNIQO4UISfG7VtkOAQyQUOY,8471
+ragbits/evaluate/evaluator.py,sha256=awRDaDTubHtM_1SANIeE5GhQ0v9fawF0q1Tj6FWZDvQ,7348
+ragbits/evaluate/optimizer.py,sha256=RqYgoiCIEhjXO0HEN6uwldblHyoPuT3qUdncuHPZgCg,8485
 ragbits/evaluate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragbits/evaluate/utils.py,sha256=rTTmrP4nv3D7174cMEfohxrDN5thPScH0BsXaptMHqQ,3757
+ragbits/evaluate/utils.py,sha256=w-hbvKRHI9tEva9wKDTVla0Wm2eCHT2MxVkof27Sqfw,4831
 ragbits/evaluate/dataloaders/__init__.py,sha256=UFJFjmvi3GUQFsx6A5sYD01HH2f7TXcHRW2VNM1pmIA,83
-ragbits/evaluate/dataloaders/base.py,sha256=ovL38_tH12q9wd3yeflIlovGuSD8S1X9HUUtwv17QrM,1774
-ragbits/evaluate/dataloaders/document_search.py,sha256=sqNPQf1ZYAqM_xMjuwh63ET00zEmKtAzqXX04cazuB8,1579
+ragbits/evaluate/dataloaders/base.py,sha256=x8rEl5utNOziF_9urL0grkqoXwMgaDWYSM5akw3Kt9Y,3213
+ragbits/evaluate/dataloaders/document_search.py,sha256=c9Bc4ZtFEKAiG9B70JFiBZlZDkBSGNWFRKabF7PMTU0,2495
 ragbits/evaluate/dataloaders/exceptions.py,sha256=xUOBLj1JuCkcqzRVnu0A0I_i1THxbDt2MEDVdDGjDyY,735
+ragbits/evaluate/dataloaders/question_answer.py,sha256=naXFDtla0otOTWSyHVvWvgDYEq-Wry4irnAJR2tHMNg,2032
 ragbits/evaluate/dataset_generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/dataset_generator/pipeline.py,sha256=dgnV-Qm0Z7S1Y6ga9-9RscXxxr3krOKsIj7E9WS4ANk,4940
 ragbits/evaluate/dataset_generator/utils.py,sha256=zD-ksXlX62kkIgzBefE4ILsP7He9bHimnZ63LLsMKCA,1325
@@ -23,13 +24,15 @@ ragbits/evaluate/dataset_generator/tasks/filter/dont_know.py,sha256=ydMHyI0JrWZf
 ragbits/evaluate/dataset_generator/tasks/text_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/dataset_generator/tasks/text_generation/base.py,sha256=2h-Y14H3fRHKbTNvXWKRus8t0hdTITd9LMoIFVwfKfA,2138
 ragbits/evaluate/dataset_generator/tasks/text_generation/qa.py,sha256=QAClPbTVNCe4QzVOGuepRnsmkt9ZF6bXBAuJI2elRuE,3851
-ragbits/evaluate/factories/__init__.py,sha256=De2ZgQ4YXgvpMOvm81fSDPSMvKpIBjS-aqeE0dxEU1s,2074
+ragbits/evaluate/factories/__init__.py,sha256=7nh0J80EfqMWRGtHx4hkfHNMztfC6FMhH8gHumwcH9w,1727
 ragbits/evaluate/metrics/__init__.py,sha256=Mr83ytGyvdXtBlr7Bbo0-5auE0530xsd3wffKSIf8cE,95
-ragbits/evaluate/metrics/base.py,sha256=axkGuKJU5u94SnRjpWsdG4jFWjy8rmkSHVRcgz1JLTo,2342
-ragbits/evaluate/metrics/document_search.py,sha256=WeC0xuLYci_Vbdw-E4OjawTqmLkcFKjDWSJGITC9-AQ,2851
+ragbits/evaluate/metrics/base.py,sha256=bOscQ_nJXLGWmP2ls9jncrUoeghNBnKDJsab71pFEjo,2519
+ragbits/evaluate/metrics/document_search.py,sha256=MfvMwEPenqiJdKYuW6WLvmtMch9ZVYb0T6ibpOF3vGI,3189
+ragbits/evaluate/metrics/question_answer.py,sha256=_XMFjkJcG-xdOO2fCfoKIhJb5VVM_GK_yKhFGXO8FRM,6566
 ragbits/evaluate/pipelines/__init__.py,sha256=Bqp_L7aRq12Ua19ELZDsdYvra6-GlLrQ9cIG2IWArko,1294
-ragbits/evaluate/pipelines/base.py,sha256=1GPu3MV-2o0PdUuFM4IcLeg1baYv9acqCcGrQykmRSs,1682
-ragbits/evaluate/pipelines/document_search.py,sha256=xMcSnahy7fifk2bJoolX9OWCXz4FjSJQfBDHIB1d2mQ,3266
-ragbits_evaluate-0.17.1.dist-info/METADATA,sha256=Kcrrm9-4VOIfvHk4q0J9jTxqTZYGCG4nIQdFUY7NJGU,2300
-ragbits_evaluate-0.17.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ragbits_evaluate-0.17.1.dist-info/RECORD,,
+ragbits/evaluate/pipelines/base.py,sha256=QV3fjPnbJjeCgcbt8yV1Ho3BamEUc3wSca3MAzaBlV0,1739
+ragbits/evaluate/pipelines/document_search.py,sha256=tgk-I21eshdBbWVsuNa1zWK_fWuDNXhhMCn1_Fdu_Ko,3840
+ragbits/evaluate/pipelines/question_answer.py,sha256=3CYVHDLnOy4z7kgYPMluiJ8POulHo-w3PEiqvqsF4Dc,2797
+ragbits_evaluate-0.19.0.dist-info/METADATA,sha256=h-R6_pE37yh9C7RgpMyb2NS6F_l-wrNEdou2MrhZoOk,2300
+ragbits_evaluate-0.19.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ragbits_evaluate-0.19.0.dist-info/RECORD,,