ragbits-evaluate 0.17.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

ragbits/evaluate/cli.py

@@ -140,10 +140,10 @@ def run() -> None:
         metric_results = await evaluator.compute(
             pipeline=state.pipeline,
             dataloader=state.dataloader,
-            metrics=state.metrics,
+            metricset=state.metrics,
         )
         evaluation_results = EvaluationResult(
-            metrics={"metrics": metric_results["metrics"], "time_perf": metric_results["time_perf"]}
+            metrics={"metrics": metric_results.metrics, "time_perf": metric_results.time_perf}
         )
         print_output(evaluation_results)
 

ragbits/evaluate/dataloaders/document_search.py

@@ -2,6 +2,7 @@ from collections.abc import Iterable
 
 from datasets import load_dataset
 
+from ragbits.core.sources.base import Source
 from ragbits.evaluate.dataloaders.base import DataLoader
 from ragbits.evaluate.dataloaders.exceptions import DataLoaderIncorrectFormatDataError
 from ragbits.evaluate.pipelines.document_search import DocumentSearchData
@@ -15,6 +16,30 @@ class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
     and contain the following features: "question, "passages".
     """
 
+    def __init__(
+        self,
+        source: Source,
+        question_key: str = "question",
+        document_ids_key: str = "document_ids",
+        passages_key: str = "passages",
+        page_numbers_key: str = "page_numbers",
+    ) -> None:
+        """
+        Initialize the document search data loader.
+
+        Args:
+            source: The source to load the data from.
+            question_key: The dataset column name that contains the question.
+            document_ids_key: The dataset column name that contains the document ids. Document ids are optional.
+            passages_key: The dataset column name that contains the passages. Passages are optional.
+            page_numbers_key: The dataset column name that contains the page numbers. Page numbers are optional.
+        """
+        super().__init__(source)
+        self.question_key = question_key
+        self.document_ids_key = document_ids_key
+        self.passages_key = passages_key
+        self.page_numbers_key = page_numbers_key
+
     async def load(self) -> Iterable[DocumentSearchData]:
         """
         Load the data from source and format them.
@@ -28,18 +53,21 @@ class DocumentSearchDataLoader(DataLoader[DocumentSearchData]):
         data_path = await self.source.fetch()
         dataset = load_dataset(
             path=str(data_path.parent),
-            split=data_path.stem,
+            split="train",
+            data_files={"train": str(data_path.name)},
         )
-        if "question" not in dataset.features or "passages" not in dataset.features:
+        if self.question_key not in dataset.features:
             raise DataLoaderIncorrectFormatDataError(
-                required_features=["question", "passages"],
+                required_features=[self.question_key],
                 data_path=data_path,
             )
 
         return [
             DocumentSearchData(
-                question=data["question"],
-                reference_passages=data["passages"],
+                question=data.get(self.question_key),
+                reference_document_ids=data.get(self.document_ids_key),
+                reference_passages=data.get(self.passages_key),
+                reference_page_numbers=data.get(self.page_numbers_key),
             )
             for data in dataset
         ]

ragbits/evaluate/evaluator.py

@@ -1,16 +1,45 @@
 import asyncio
+import random
 import time
-from collections.abc import Iterable
-from dataclasses import asdict
+from collections.abc import Awaitable, Callable, Iterable
+from dataclasses import dataclass
+from typing import Generic, ParamSpec, TypeVar
 
 from pydantic import BaseModel
-from tqdm.asyncio import tqdm
+from tqdm import tqdm
 
 from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
+from ragbits.core.utils.helpers import batched
 from ragbits.evaluate.dataloaders.base import DataLoader
 from ragbits.evaluate.metrics.base import MetricSet
 from ragbits.evaluate.pipelines.base import EvaluationDataT, EvaluationPipeline, EvaluationResultT, EvaluationTargetT
 
+_CallP = ParamSpec("_CallP")
+_CallReturnT = TypeVar("_CallReturnT")
+
+
+@dataclass
+class EvaluationTimePerf:
+    """
+    Container for evaluation time performance metrics.
+    """
+
+    total_time_in_seconds: float
+    samples_per_second: float
+    latency_in_seconds: float
+
+
+@dataclass
+class EvaluatorResult(Generic[EvaluationResultT]):
+    """
+    Container for evaluation results.
+    """
+
+    metrics: dict[str, int | float]
+    results: list[EvaluationResultT]
+    errors: list[Exception]
+    time_perf: EvaluationTimePerf
+
 
 class EvaluationConfig(BaseModel):
     """
@@ -36,17 +65,29 @@ class Evaluator(WithConstructionConfig):
     Evaluator class.
     """
 
-    def __init__(self, batch_size: int = 10) -> None:
+    def __init__(
+        self,
+        batch_size: int = 10,
+        num_retries: int = 3,
+        backoff_multiplier: int = 1,
+        backoff_max: int = 60,
+    ) -> None:
         """
-        Initialize the evaluator.
+        Initialize the Evaluator instance.
 
         Args:
             batch_size: batch size for the evaluation pipeline inference.
+            num_retries: The number of retries per evaluation pipeline inference error.
+            backoff_multiplier: The base delay multiplier for exponential backoff (in seconds).
+            backoff_max: The maximum allowed delay (in seconds) between retries.
         """
         self.batch_size = batch_size
+        self.num_retries = num_retries
+        self.backoff_multiplier = backoff_multiplier
+        self.backoff_max = backoff_max
 
     @classmethod
-    async def run_from_config(cls, config: dict) -> dict:
+    async def run_from_config(cls, config: dict) -> EvaluatorResult:
         """
         Run the evaluation based on configuration.
 
@@ -60,50 +101,50 @@ class Evaluator(WithConstructionConfig):
         evaluation_config = EvaluationConfig.model_validate(evaluator_config.evaluation)
         pipeline: EvaluationPipeline = EvaluationPipeline.subclass_from_config(evaluation_config.pipeline)
         dataloader: DataLoader = DataLoader.subclass_from_config(evaluation_config.dataloader)
-        metrics: MetricSet = MetricSet.from_config(evaluation_config.metrics)
+        metricset: MetricSet = MetricSet.from_config(evaluation_config.metrics)
 
         evaluator = cls.from_config(evaluator_config.evaluator or {})
         return await evaluator.compute(
             pipeline=pipeline,
             dataloader=dataloader,
-            metrics=metrics,
+            metricset=metricset,
         )
 
     async def compute(
         self,
         pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
         dataloader: DataLoader[EvaluationDataT],
-        metrics: MetricSet[EvaluationResultT],
-    ) -> dict:
+        metricset: MetricSet[EvaluationResultT],
+    ) -> EvaluatorResult[EvaluationResultT]:
         """
         Compute the evaluation results for the given pipeline and data.
 
         Args:
             pipeline: The pipeline to be evaluated.
             dataloader: The dataloader to load the data.
-            metrics: The metrics to be computed.
+            metricset: The metrics to be computed.
 
         Returns:
             The evaluation results.
         """
-        dataset = await dataloader.load()
         await pipeline.prepare()
 
-        results, perf_results = await self._call_pipeline(pipeline, dataset)
-        computed_metrics = self._compute_metrics(metrics, results)
-        processed_results = self._results_processor(results)
+        dataset = await dataloader.load()
+        results, errors, time_perf = await self._call_pipeline(pipeline, dataset)
+        metrics = await metricset.compute(results)
 
-        return {
-            **perf_results,
-            **computed_metrics,
-            **processed_results,
-        }
+        return EvaluatorResult(
+            metrics=metrics,
+            results=results,
+            errors=errors,
+            time_perf=time_perf,
+        )
 
     async def _call_pipeline(
         self,
         pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
         dataset: Iterable[EvaluationDataT],
-    ) -> tuple[list[EvaluationResultT], dict]:
+    ) -> tuple[list[EvaluationResultT], list[Exception], EvaluationTimePerf]:
         """
         Call the pipeline with the given data.
 
@@ -114,47 +155,53 @@ class Evaluator(WithConstructionConfig):
         Returns:
             The evaluation results and performance metrics.
         """
-        semaphore = asyncio.Semaphore(self.batch_size)
-
-        async def _call_pipeline_with_semaphore(data: EvaluationDataT) -> EvaluationResultT:
-            async with semaphore:
-                return await pipeline(data)
-
         start_time = time.perf_counter()
-        pipe_outputs = await tqdm.gather(*[_call_pipeline_with_semaphore(data) for data in dataset], desc="Evaluation")
+        outputs = [
+            await self._call_with_error_handling(pipeline, data)
+            for data in tqdm(batched(dataset, self.batch_size), desc="Evaluation")
+        ]
         end_time = time.perf_counter()
 
-        return pipe_outputs, self._compute_time_perf(start_time, end_time, len(pipe_outputs))
+        errors = [output for output in outputs if isinstance(output, Exception)]
+        results = [item for output in outputs if not isinstance(output, Exception) for item in output]
 
-    @staticmethod
-    def _results_processor(results: list[EvaluationResultT]) -> dict:
+        return results, errors, self._compute_time_perf(start_time, end_time, len(outputs))
+
+    async def _call_with_error_handling(
+        self,
+        executable: Callable[_CallP, Awaitable[_CallReturnT]],
+        *executable_args: _CallP.args,
+        **executable_kwargs: _CallP.kwargs,
+    ) -> _CallReturnT | Exception:
         """
-        Process the results.
+        Call executable with a standarized error handling.
+        If an error occurs, the executable is retried `num_retries` times using randomized exponential backoff.
 
         Args:
-            results: The evaluation results.
+            executable: The callable function to execute.
+            executable_args: Positional arguments to pass to the executable.
+            executable_kwargs: Keyword arguments to pass to the executable.
 
         Returns:
-            The processed results.
-        """
-        return {"results": [asdict(result) for result in results]}
+            The result of the executable if successful.
 
-    @staticmethod
-    def _compute_metrics(metrics: MetricSet[EvaluationResultT], results: list[EvaluationResultT]) -> dict:
+        Raises:
+            Exception: The last encountered exception after all retries are exhausted.
         """
-        Compute a metric using the given inputs.
+        for i in range(max(0, self.num_retries) + 1):
+            try:
+                return await executable(*executable_args, **executable_kwargs)
+            except Exception as exc:
+                if i == self.num_retries:
+                    return exc
 
-        Args:
-            metrics: The metrics to be computed.
-            results: The evaluation results.
+                delay = random.uniform(0, min(2**i * self.backoff_multiplier, self.backoff_max))  # noqa: S311
+                await asyncio.sleep(delay)
 
-        Returns:
-            The computed metric.
-        """
-        return {"metrics": metrics.compute(results)}
+        raise RuntimeError("Unreachable code reached")  # mypy quirk
 
     @staticmethod
-    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> dict:
+    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> EvaluationTimePerf:
         """
         Compute the performance metrics.
 
@@ -170,10 +217,8 @@ class Evaluator(WithConstructionConfig):
         throughput = num_samples / latency
         latency_sample = 1.0 / throughput if throughput > 0 else 0.0
 
-        return {
-            "time_perf": {
-                "total_time_in_seconds": latency,
-                "samples_per_second": throughput,
-                "latency_in_seconds": latency_sample,
-            },
-        }
+        return EvaluationTimePerf(
+            total_time_in_seconds=latency,
+            samples_per_second=throughput,
+            latency_in_seconds=latency_sample,
+        )

ragbits/evaluate/factories/__init__.py

@@ -1,43 +1,21 @@
 import asyncio
 
+from continuous_eval.metrics.retrieval.matching_strategy import RougeChunkMatch
 from datasets import load_dataset
 
 from ragbits.core.embeddings.dense import LiteLLMEmbedder
 from ragbits.core.sources.hf import HuggingFaceSource
-from ragbits.core.utils.config_handling import ObjectConstructionConfig
 from ragbits.core.vector_stores.in_memory import InMemoryVectorStore
 from ragbits.document_search import DocumentSearch
 from ragbits.document_search.documents.document import DocumentMeta
 from ragbits.evaluate.dataloaders.document_search import DocumentSearchDataLoader
 from ragbits.evaluate.metrics import MetricSet
-
-DS_PRECISION_RECALL_F1 = {
-    "precision_recall_f1": ObjectConstructionConfig.model_validate(
-        {
-            "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1",
-            "config": {
-                "matching_strategy": {
-                    "type": "RougeChunkMatch",
-                    "config": {
-                        "threshold": 0.5,
-                    },
-                },
-            },
-        }
-    ),
-}
-
-
-def precision_recall_f1() -> MetricSet:
-    """
-    Factory of precision recall f1 metric set for retrival evaluation.
-    """
-    return MetricSet.from_config(config=DS_PRECISION_RECALL_F1)
+from ragbits.evaluate.metrics.document_search import DocumentSearchPrecisionRecallF1
 
 
 async def _add_example_documents(document_search: DocumentSearch) -> None:
     dataset = load_dataset(path="deepsense-ai/synthetic-rag-dataset_v1.0", split="train")
-    documents = [DocumentMeta.create_text_document_from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
+    documents = [DocumentMeta.from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
     await document_search.ingest(documents)
 
 
@@ -45,7 +23,7 @@ def basic_document_search_factory() -> DocumentSearch:
     """
     Factory for basic example document search instance.
     """
-    document_search = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
+    document_search: DocumentSearch = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
     asyncio.run(_add_example_documents(document_search))
     return document_search
 
@@ -55,3 +33,10 @@ def synthetic_rag_dataset() -> DocumentSearchDataLoader:
     Factory for synthetic RAG dataset.
     """
     return DocumentSearchDataLoader(source=HuggingFaceSource(path="deepsense-ai/synthetic-rag-dataset_v1.0"))
+
+
+def precision_recall_f1() -> MetricSet:
+    """
+    Factory of precision recall f1 metric set for retrival evaluation.
+    """
+    return MetricSet(DocumentSearchPrecisionRecallF1(matching_strategy=RougeChunkMatch()))

ragbits/evaluate/metrics/base.py

@@ -1,3 +1,4 @@
+import asyncio
 from abc import ABC, abstractmethod
 from types import ModuleType
 from typing import ClassVar, Generic
@@ -19,7 +20,7 @@ class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
 
     def __init__(self, weight: float = 1.0) -> None:
         """
-        Initializes the metric.
+        Initialize the metric.
 
         Args:
             weight: Metric value weight in the final score, used during optimization.
@@ -28,7 +29,7 @@ class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
         self.weight = weight
 
     @abstractmethod
-    def compute(self, results: list[EvaluationResultT]) -> dict:
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metric.
 
@@ -70,7 +71,7 @@ class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
         """
         return cls(*[Metric.subclass_from_config(metric_config) for metric_config in config.values()])
 
-    def compute(self, results: list[EvaluationResultT]) -> dict:
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metrics.
 
@@ -80,6 +81,9 @@ class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
         Returns:
             The computed metrics.
         """
+        metric_results = await asyncio.gather(*[metric.compute(results) for metric in self.metrics])
         return {
-            name: metric.weight * value for metric in self.metrics for name, value in metric.compute(results).items()
+            name: metric.weight * value
+            for metric, result in zip(self.metrics, metric_results, strict=False)
+            for name, value in result.items()
         }

ragbits/evaluate/metrics/document_search.py

@@ -46,7 +46,7 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
         matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
         return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))
 
-    def compute(self, results: list[DocumentSearchResult]) -> dict:
+    async def compute(self, results: list[DocumentSearchResult]) -> dict:
         """
         Compute the metric.
 
@@ -57,7 +57,18 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
             The computed metric.
         """
         return self.metric.aggregate(
-            [self.metric(result.predicted_passages, result.reference_passages) for result in results]
+            [
+                self.metric(
+                    [
+                        element.text_representation
+                        for element in result.predicted_elements
+                        if element.text_representation
+                    ],
+                    result.reference_passages,
+                )
+                for result in results
+                if result.reference_passages is not None
+            ]
         )
 
 

ragbits/evaluate/optimizer.py

@@ -61,7 +61,7 @@ class Optimizer(WithConstructionConfig):
         evaluator_config = EvaluatorConfig.model_validate(optimizer_config.evaluator)
 
         dataloader: DataLoader = DataLoader.subclass_from_config(evaluator_config.evaluation.dataloader)
-        metrics: MetricSet = MetricSet.from_config(evaluator_config.evaluation.metrics)
+        metricset: MetricSet = MetricSet.from_config(evaluator_config.evaluation.metrics)
 
         pipeline_class = import_by_path(evaluator_config.evaluation.pipeline.type)
         pipeline_config = dict(evaluator_config.evaluation.pipeline.config)
@@ -71,7 +71,7 @@ class Optimizer(WithConstructionConfig):
         return optimizer.optimize(
             pipeline_class=pipeline_class,
             pipeline_config=pipeline_config,
-            metrics=metrics,
+            metricset=metricset,
             dataloader=dataloader,
             callbacks=callbacks,
         )
@@ -81,7 +81,7 @@ class Optimizer(WithConstructionConfig):
         pipeline_class: type[EvaluationPipeline],
         pipeline_config: dict,
         dataloader: DataLoader,
-        metrics: MetricSet,
+        metricset: MetricSet,
         callbacks: list[Callable] | None = None,
     ) -> list[tuple[dict, float, dict[str, float]]]:
         """
@@ -91,7 +91,7 @@ class Optimizer(WithConstructionConfig):
             pipeline_class: Pipeline to be optimized.
             pipeline_config: Configuration defining the optimization process.
             dataloader: Data loader.
-            metrics: Metrics to be optimized.
+            metricset: Metrics to be optimized.
             callbacks: Experiment callbacks.
 
         Returns:
@@ -104,7 +104,7 @@ class Optimizer(WithConstructionConfig):
                 pipeline_class=pipeline_class,
                 pipeline_config=pipeline_config,
                 dataloader=dataloader,
-                metrics=metrics,
+                metricset=metricset,
             )
 
         study = optuna.create_study(direction=self.direction)
@@ -131,7 +131,7 @@ class Optimizer(WithConstructionConfig):
         pipeline_class: type[EvaluationPipeline],
         pipeline_config: dict,
         dataloader: DataLoader,
-        metrics: MetricSet,
+        metricset: MetricSet,
     ) -> float:
         """
         Run a single experiment.
@@ -153,11 +153,11 @@ class Optimizer(WithConstructionConfig):
                     evaluator.compute(
                         pipeline=pipeline,
                         dataloader=dataloader,
-                        metrics=metrics,
+                        metricset=metricset,
                     )
                 )
-                score = sum(results["metrics"].values())
-                metrics_values = results["metrics"]
+                score = sum(results.metrics.values())
+                metrics_values = results.metrics
                 break
             except Exception as exc:
                 message = (

ragbits/evaluate/pipelines/base.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from collections.abc import Iterable
 from dataclasses import dataclass
 from types import ModuleType
 from typing import ClassVar, Generic, TypeVar
@@ -51,7 +52,7 @@ class EvaluationPipeline(WithConstructionConfig, Generic[EvaluationTargetT, Eval
         pass
 
     @abstractmethod
-    async def __call__(self, data: EvaluationDataT) -> EvaluationResultT:
+    async def __call__(self, data: Iterable[EvaluationDataT]) -> Iterable[EvaluationResultT]:
         """
         Run the evaluation pipeline.
 

ragbits/evaluate/pipelines/document_search.py

@@ -1,3 +1,5 @@
+import asyncio
+from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from uuid import uuid4
 
@@ -5,6 +7,7 @@ from typing_extensions import Self
 
 from ragbits.core.sources.hf import HuggingFaceSource
 from ragbits.document_search import DocumentSearch
+from ragbits.document_search.documents.element import Element
 from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult
 
 
@@ -14,7 +17,9 @@ class DocumentSearchData(EvaluationData):
     """
 
     question: str
-    reference_passages: list[str]
+    reference_document_ids: list[str | int] | None = None
+    reference_passages: list[str] | None = None
+    reference_page_numbers: list[int] | None = None
 
 
 @dataclass
@@ -24,8 +29,10 @@ class DocumentSearchResult(EvaluationResult):
     """
 
     question: str
-    reference_passages: list[str]
-    predicted_passages: list[str]
+    predicted_elements: Sequence[Element]
+    reference_document_ids: list[str | int] | None = None
+    reference_passages: list[str] | None = None
+    reference_page_numbers: list[int] | None = None
 
 
 class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch, DocumentSearchData, DocumentSearchResult]):
@@ -60,7 +67,7 @@ class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch, DocumentSearchDa
         # TODO: optimize this for cases with duplicated document search configs between runs
         if config.get("source"):
             config["vector_store"]["config"]["index_name"] = str(uuid4())
-        evaluation_target = DocumentSearch.from_config(config)
+        evaluation_target: DocumentSearch = DocumentSearch.from_config(config)
         return cls(evaluation_target=evaluation_target, source=config.get("source"))
 
     async def prepare(self) -> None:
@@ -76,21 +83,24 @@ class DocumentSearchPipeline(EvaluationPipeline[DocumentSearch, DocumentSearchDa
             )
             await self.evaluation_target.ingest(sources)
 
-    async def __call__(self, data: DocumentSearchData) -> DocumentSearchResult:
+    async def __call__(self, data: Iterable[DocumentSearchData]) -> Iterable[DocumentSearchResult]:
         """
         Run the document search evaluation pipeline.
 
         Args:
-            data: The evaluation data.
+            data: The evaluation data batch.
 
         Returns:
-            The evaluation result.
+            The evaluation result batch.
         """
-        elements = await self.evaluation_target.search(data.question)
-        predicted_passages = [element.text_representation for element in elements if element.text_representation]
-
-        return DocumentSearchResult(
-            question=data.question,
-            reference_passages=data.reference_passages,
-            predicted_passages=predicted_passages,
-        )
+        results = await asyncio.gather(*[self.evaluation_target.search(row.question) for row in data])
+        return [
+            DocumentSearchResult(
+                question=row.question,
+                predicted_elements=elements,
+                reference_document_ids=row.reference_document_ids,
+                reference_passages=row.reference_passages,
+                reference_page_numbers=row.reference_page_numbers,
+            )
+            for row, elements in zip(data, results, strict=False)
+        ]

ragbits/evaluate/utils.py

@@ -1,5 +1,7 @@
 import json
 import sys
+import traceback
+from dataclasses import asdict
 from datetime import datetime
 from pathlib import Path
 from typing import Any
@@ -12,13 +14,15 @@ from neptune.utils import stringify_unsupported
 from neptune_optuna import NeptuneCallback
 from omegaconf import DictConfig
 
+from ragbits.evaluate.evaluator import EvaluatorResult
 
-def log_evaluation_to_file(results: dict, output_dir: Path | None = None) -> Path:
+
+def log_evaluation_to_file(result: EvaluatorResult, output_dir: Path | None = None) -> Path:
     """
-    Log the evaluation results locally.
+    Log the evaluation result locally.
 
     Args:
-        results: The evaluation results.
+        result: The evaluation result.
         output_dir: The output directory.
 
     Returns:
@@ -27,27 +31,57 @@ def log_evaluation_to_file(results: dict, output_dir: Path | None = None) -> Pat
     output_dir = output_dir or Path(HydraConfig.get().runtime.output_dir)
     metrics_file = output_dir / "metrics.json"
     results_file = output_dir / "results.json"
-
-    _save_json(metrics_file, metrics=results["metrics"], time_perf=results["time_perf"])
-    _save_json(results_file, results=results["results"])
+    errors_file = output_dir / "errors.json"
+
+    _save_json(metrics_file, metrics=result.metrics, time_perf=asdict(result.time_perf))
+    _save_json(results_file, results=[asdict(entry) for entry in result.results])
+    _save_json(
+        errors_file,
+        errors=[
+            {
+                "type": exc.__class__.__name__,
+                "message": str(exc),
+                "stacktrace": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+            }
+            for exc in result.errors
+        ],
+    )
 
     return output_dir
 
 
-def log_evaluation_to_neptune(results: dict, config: DictConfig, tags: str | list[str] | None = None) -> None:
+def log_evaluation_to_neptune(result: EvaluatorResult, config: DictConfig, tags: str | list[str] | None = None) -> None:
     """
-    Log the evaluation results to Neptune.
+    Log the evaluation result to Neptune.
 
     Args:
-        results: Evaluation results.
-        config: Evaluation configuration.
-        tags: Experiment tags.
+        result: The evaluation result.
+        config: The evaluation configuration.
+        tags: The experiment tags.
     """
     run = Run(tags=tags)
     run["config"] = stringify_unsupported(config)
-    run["evaluation/metrics"] = stringify_unsupported(results["metrics"])
-    run["evaluation/time_perf"] = stringify_unsupported(results["time_perf"])
-    run["evaluation/results"].upload(File.from_content(json.dumps(results["results"], indent=4), extension="json"))
+    run["evaluation/metrics"] = stringify_unsupported(result.metrics)
+    run["evaluation/time_perf"] = stringify_unsupported(asdict(result.time_perf))
+    run["evaluation/results"].upload(
+        File.from_content(json.dumps([asdict(entry) for entry in result.results], indent=4), extension="json")
+    )
+    run["evaluation/errors"].upload(
+        File.from_content(
+            json.dumps(
+                [
+                    {
+                        "type": exc.__class__.__name__,
+                        "message": str(exc),
+                        "stacktrace": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+                    }
+                    for exc in result.errors
+                ],
+                indent=4,
+            ),
+            extension="json",
+        )
+    )
 
 
 def log_dataset_to_file(dataset: Dataset, output_dir: Path | None = None) -> Path:

{ragbits_evaluate-0.17.0.dist-info → ragbits_evaluate-0.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragbits-evaluate
-Version: 0.17.0
+Version: 0.18.0
 Summary: Evaluation module for Ragbits components
 Project-URL: Homepage, https://github.com/deepsense-ai/ragbits
 Project-URL: Bug Reports, https://github.com/deepsense-ai/ragbits/issues
@@ -27,7 +27,7 @@ Requires-Dist: distilabel<2.0.0,>=1.4.1
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: neptune[optuna]<2.0.0,>=1.12.0
 Requires-Dist: optuna<5.0.0,>=4.0.0
-Requires-Dist: ragbits-core==0.17.0
+Requires-Dist: ragbits-core==0.18.0
 Provides-Extra: relari
 Requires-Dist: continuous-eval<1.0.0,>=0.3.12; extra == 'relari'
 Description-Content-Type: text/markdown

{ragbits_evaluate-0.17.0.dist-info → ragbits_evaluate-0.18.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 ragbits/evaluate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragbits/evaluate/cli.py,sha256=MEDo8ubk81TCNx-fq-liF0P5hjn2-kPpIfq54fReKIY,4509
+ragbits/evaluate/cli.py,sha256=vP8l2DyNXpR6jQP83wXKP_RRnGjEXjKnTVBg9RPbDKo,4505
 ragbits/evaluate/config.py,sha256=2WSmbVxyQi893L2FSjRFQoXkWZp1GetcNmR2GCDe0tA,339
-ragbits/evaluate/evaluator.py,sha256=Cif-QX2n5awOGm-AfFy2nRXkb_m4vGY_JZ_o4K4PhZI,5552
-ragbits/evaluate/optimizer.py,sha256=egcU54aADqKrN31NPqj7cNIQO4UISfG7VtkOAQyQUOY,8471
+ragbits/evaluate/evaluator.py,sha256=awRDaDTubHtM_1SANIeE5GhQ0v9fawF0q1Tj6FWZDvQ,7348
+ragbits/evaluate/optimizer.py,sha256=RqYgoiCIEhjXO0HEN6uwldblHyoPuT3qUdncuHPZgCg,8485
 ragbits/evaluate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragbits/evaluate/utils.py,sha256=rTTmrP4nv3D7174cMEfohxrDN5thPScH0BsXaptMHqQ,3757
+ragbits/evaluate/utils.py,sha256=w-hbvKRHI9tEva9wKDTVla0Wm2eCHT2MxVkof27Sqfw,4831
 ragbits/evaluate/dataloaders/__init__.py,sha256=UFJFjmvi3GUQFsx6A5sYD01HH2f7TXcHRW2VNM1pmIA,83
 ragbits/evaluate/dataloaders/base.py,sha256=ovL38_tH12q9wd3yeflIlovGuSD8S1X9HUUtwv17QrM,1774
-ragbits/evaluate/dataloaders/document_search.py,sha256=sqNPQf1ZYAqM_xMjuwh63ET00zEmKtAzqXX04cazuB8,1579
+ragbits/evaluate/dataloaders/document_search.py,sha256=BLOaXP6TVtSsV2xScY4X_th285hWI4b9lcRuUXNxZ3U,2813
 ragbits/evaluate/dataloaders/exceptions.py,sha256=xUOBLj1JuCkcqzRVnu0A0I_i1THxbDt2MEDVdDGjDyY,735
 ragbits/evaluate/dataset_generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/dataset_generator/pipeline.py,sha256=dgnV-Qm0Z7S1Y6ga9-9RscXxxr3krOKsIj7E9WS4ANk,4940
@@ -23,13 +23,13 @@ ragbits/evaluate/dataset_generator/tasks/filter/dont_know.py,sha256=ydMHyI0JrWZf
 ragbits/evaluate/dataset_generator/tasks/text_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/dataset_generator/tasks/text_generation/base.py,sha256=2h-Y14H3fRHKbTNvXWKRus8t0hdTITd9LMoIFVwfKfA,2138
 ragbits/evaluate/dataset_generator/tasks/text_generation/qa.py,sha256=QAClPbTVNCe4QzVOGuepRnsmkt9ZF6bXBAuJI2elRuE,3851
-ragbits/evaluate/factories/__init__.py,sha256=De2ZgQ4YXgvpMOvm81fSDPSMvKpIBjS-aqeE0dxEU1s,2074
+ragbits/evaluate/factories/__init__.py,sha256=7nh0J80EfqMWRGtHx4hkfHNMztfC6FMhH8gHumwcH9w,1727
 ragbits/evaluate/metrics/__init__.py,sha256=Mr83ytGyvdXtBlr7Bbo0-5auE0530xsd3wffKSIf8cE,95
-ragbits/evaluate/metrics/base.py,sha256=axkGuKJU5u94SnRjpWsdG4jFWjy8rmkSHVRcgz1JLTo,2342
-ragbits/evaluate/metrics/document_search.py,sha256=WeC0xuLYci_Vbdw-E4OjawTqmLkcFKjDWSJGITC9-AQ,2851
+ragbits/evaluate/metrics/base.py,sha256=bOscQ_nJXLGWmP2ls9jncrUoeghNBnKDJsab71pFEjo,2519
+ragbits/evaluate/metrics/document_search.py,sha256=MfvMwEPenqiJdKYuW6WLvmtMch9ZVYb0T6ibpOF3vGI,3189
 ragbits/evaluate/pipelines/__init__.py,sha256=Bqp_L7aRq12Ua19ELZDsdYvra6-GlLrQ9cIG2IWArko,1294
-ragbits/evaluate/pipelines/base.py,sha256=1GPu3MV-2o0PdUuFM4IcLeg1baYv9acqCcGrQykmRSs,1682
-ragbits/evaluate/pipelines/document_search.py,sha256=xMcSnahy7fifk2bJoolX9OWCXz4FjSJQfBDHIB1d2mQ,3266
-ragbits_evaluate-0.17.0.dist-info/METADATA,sha256=fdHH9MszU2DO5pp18ikVVnOEPkTnQ_TQwddvcvEwWj4,2300
-ragbits_evaluate-0.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ragbits_evaluate-0.17.0.dist-info/RECORD,,
+ragbits/evaluate/pipelines/base.py,sha256=QV3fjPnbJjeCgcbt8yV1Ho3BamEUc3wSca3MAzaBlV0,1739
+ragbits/evaluate/pipelines/document_search.py,sha256=tgk-I21eshdBbWVsuNa1zWK_fWuDNXhhMCn1_Fdu_Ko,3840
+ragbits_evaluate-0.18.0.dist-info/METADATA,sha256=in_9YOI8UrGJR_tXGPKJYoinR6Ju4Taa0PaK9DQVCnw,2300
+ragbits_evaluate-0.18.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ragbits_evaluate-0.18.0.dist-info/RECORD,,