ragbits-evaluate 0.5.0__py3-none-any.whl → 1.4.0.dev202602030301__py3-none-any.whl

ragbits/evaluate/dataset_generator/prompts/qa.py

@@ -23,7 +23,7 @@ class BasicAnswerGenPrompt(Prompt[BasicAnswerGenInput, str]):
         "If you don't know the answer just say: I don't know."
     )
 
-    user_prompt: str = "Text:\n<|text_start|>\n {{ chunk }} \n<|text_end|>\n\nQuestion:\n " "{{ question }} \n\nAnswer:"
+    user_prompt: str = "Text:\n<|text_start|>\n {{ chunk }} \n<|text_end|>\n\nQuestion:\n {{ question }} \n\nAnswer:"
 
 
 class PassagesGenInput(BaseModel):
@@ -49,9 +49,7 @@ class PassagesGenPrompt(Prompt[PassagesGenInput, str]):
         "FULL SENTENCES"
     )
 
-    user_prompt: str = (
-        "Question:\n {{ question }} \nAnswer:\n {{ basic_answer }} \nChunk:\n " "{{ chunk }}\n\nPassages:"
-    )
+    user_prompt: str = "Question:\n {{ question }} \nAnswer:\n {{ basic_answer }} \nChunk:\n {{ chunk }}\n\nPassages:"
 
 
 class QueryGenInput(BaseModel):

ragbits/evaluate/dataset_generator/tasks/corpus_generation.py

@@ -7,7 +7,7 @@ from distilabel.steps.base import Step
 
 from ragbits.core.llms.base import LLM
 from ragbits.core.prompt import Prompt
-from ragbits.core.utils.config_handling import get_cls_from_config
+from ragbits.core.utils.config_handling import import_by_path
 
 module = sys.modules[__name__]
 
@@ -23,9 +23,7 @@ class CorpusGenerationStep(Step):
     ):
         super().__init__()
         self._llm = llm
-        self._prompt_class = (
-            get_cls_from_config(prompt_class, module) if isinstance(prompt_class, str) else prompt_class
-        )
+        self._prompt_class = import_by_path(prompt_class, module) if isinstance(prompt_class, str) else prompt_class
         self._num_per_topic = num_per_topic
 
     @property

ragbits/evaluate/dataset_generator/tasks/text_generation/base.py

@@ -2,11 +2,11 @@ import sys
 from abc import ABC, abstractmethod
 from typing import Any
 
-from distilabel.llms.base import LLM
+from distilabel.models import LLM
 from distilabel.steps.tasks import TextGeneration
 
 from ragbits.core.prompt import ChatFormat, Prompt
-from ragbits.core.utils.config_handling import get_cls_from_config
+from ragbits.core.utils.config_handling import import_by_path
 
 module = sys.modules[__name__]
 
@@ -18,9 +18,7 @@ class BaseDistilabelTask(TextGeneration, ABC):
         super().__init__(llm=llm)
         self._inputs = inputs
         self._outputs = outputs
-        self._prompt_class = (
-            get_cls_from_config(prompt_class, module) if isinstance(prompt_class, str) else prompt_class
-        )
+        self._prompt_class = import_by_path(prompt_class, module) if isinstance(prompt_class, str) else prompt_class
 
     @property
     def inputs(self) -> list[str]:

ragbits/evaluate/dataset_generator/tasks/text_generation/qa.py

@@ -1,9 +1,9 @@
 from typing import Any
 
-from distilabel.llms.base import LLM
+from distilabel.models import LLM
 
-from ...utils import get_closest_substring, get_passages_list
-from .base import BaseDistilabelTask
+from ragbits.evaluate.dataset_generator.tasks.text_generation.base import BaseDistilabelTask
+from ragbits.evaluate.dataset_generator.utils import get_closest_substring, get_passages_list
 
 
 class QueryGenTask(BaseDistilabelTask):

ragbits/evaluate/evaluator.py

@@ -1,53 +1,153 @@
+import asyncio
+import random
 import time
-from collections.abc import Iterable
-from dataclasses import asdict
-from typing import Any
+from collections.abc import Awaitable, Callable, Iterable, Sized
+from dataclasses import dataclass
+from typing import Generic, ParamSpec, TypeVar
 
-from tqdm.asyncio import tqdm
+from pydantic import BaseModel
+from tqdm import tqdm
 
-from ragbits.evaluate.loaders.base import DataLoader
+from ragbits.core.utils.config_handling import ObjectConstructionConfig, WithConstructionConfig
+from ragbits.core.utils.helpers import batched
+from ragbits.evaluate.dataloaders.base import DataLoader
 from ragbits.evaluate.metrics.base import MetricSet
-from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult
+from ragbits.evaluate.pipelines.base import EvaluationDataT, EvaluationPipeline, EvaluationResultT, EvaluationTargetT
 
+_CallP = ParamSpec("_CallP")
+_CallReturnT = TypeVar("_CallReturnT")
 
-class Evaluator:
+
+@dataclass
+class EvaluationTimePerf:
+    """
+    Container for evaluation time performance metrics.
+    """
+
+    total_time_in_seconds: float
+    samples_per_second: float
+    latency_in_seconds: float
+
+
+@dataclass
+class EvaluatorResult(Generic[EvaluationResultT]):
+    """
+    Container for evaluation results.
+    """
+
+    metrics: dict[str, int | float]
+    results: list[EvaluationResultT]
+    errors: list[Exception]
+    time_perf: EvaluationTimePerf
+
+
+class EvaluationConfig(BaseModel):
+    """
+    Schema for the evaluation run config.
+    """
+
+    pipeline: ObjectConstructionConfig
+    dataloader: ObjectConstructionConfig
+    metrics: dict[str, ObjectConstructionConfig]
+
+
+class EvaluatorConfig(BaseModel):
+    """
+    Schema for the evaluator config.
+    """
+
+    evaluation: EvaluationConfig
+    evaluator: dict | None = None
+
+
+class Evaluator(WithConstructionConfig):
     """
     Evaluator class.
     """
 
+    def __init__(
+        self,
+        batch_size: int = 10,
+        num_retries: int = 3,
+        backoff_multiplier: int = 1,
+        backoff_max: int = 60,
+        parallelize_batches: bool = False,
+    ) -> None:
+        """
+        Initialize the Evaluator instance.
+
+        Args:
+            batch_size: batch size for the evaluation pipeline inference.
+            num_retries: The number of retries per evaluation pipeline inference error.
+            backoff_multiplier: The base delay multiplier for exponential backoff (in seconds).
+            backoff_max: The maximum allowed delay (in seconds) between retries.
+            parallelize_batches: Whether to process samples within each batch in parallel (asyncio.gather).
+        """
+        self.batch_size = batch_size
+        self.num_retries = num_retries
+        self.backoff_multiplier = backoff_multiplier
+        self.backoff_max = backoff_max
+        self.parallelize_batches = parallelize_batches
+
+    @classmethod
+    async def run_from_config(cls, config: dict) -> EvaluatorResult:
+        """
+        Run the evaluation based on configuration.
+
+        Args:
+            config: Evaluation config.
+
+        Returns:
+            The evaluation results.
+        """
+        evaluator_config = EvaluatorConfig.model_validate(config)
+        evaluation_config = EvaluationConfig.model_validate(evaluator_config.evaluation)
+        pipeline: EvaluationPipeline = EvaluationPipeline.subclass_from_config(evaluation_config.pipeline)
+        dataloader: DataLoader = DataLoader.subclass_from_config(evaluation_config.dataloader)
+        metricset: MetricSet = MetricSet.from_config(evaluation_config.metrics)
+
+        evaluator = cls.from_config(evaluator_config.evaluator or {})
+        return await evaluator.compute(
+            pipeline=pipeline,
+            dataloader=dataloader,
+            metricset=metricset,
+        )
+
     async def compute(
         self,
-        pipeline: EvaluationPipeline,
-        dataloader: DataLoader,
-        metrics: MetricSet,
-    ) -> dict[str, Any]:
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataloader: DataLoader[EvaluationDataT],
+        metricset: MetricSet[EvaluationResultT],
+    ) -> EvaluatorResult[EvaluationResultT]:
         """
         Compute the evaluation results for the given pipeline and data.
 
         Args:
             pipeline: The pipeline to be evaluated.
             dataloader: The dataloader to load the data.
-            metrics: The metrics to be computed.
+            metricset: The metrics to be computed.
 
         Returns:
             The evaluation results.
         """
+        await pipeline.prepare()
+
         dataset = await dataloader.load()
-        results, perf_results = await self._call_pipeline(pipeline, dataset)
-        computed_metrics = self._compute_metrics(metrics, results)
-        processed_results = self._results_processor(results)
+        results, errors, time_perf = await self._call_pipeline(pipeline, dataset)
+        metrics = await metricset.compute(results)
 
-        return {
-            **perf_results,
-            **computed_metrics,
-            **processed_results,
-        }
+        return EvaluatorResult(
+            metrics=metrics,
+            results=results,
+            errors=errors,
+            time_perf=time_perf,
+        )
 
     async def _call_pipeline(
         self,
-        pipeline: EvaluationPipeline,
-        dataset: Iterable,
-    ) -> tuple[list[EvaluationResult], dict[str, Any]]:
+        pipeline: EvaluationPipeline[EvaluationTargetT, EvaluationDataT, EvaluationResultT],
+        dataset: Iterable[EvaluationDataT],
+    ) -> tuple[list[EvaluationResultT], list[Exception], EvaluationTimePerf]:
         """
         Call the pipeline with the given data.
 
@@ -59,39 +159,69 @@ class Evaluator:
             The evaluation results and performance metrics.
         """
         start_time = time.perf_counter()
-        pipe_outputs = await tqdm.gather(*[pipeline(data) for data in dataset], desc="Evaluation")
+
+        total_samples = len(dataset) if isinstance(dataset, Sized) else None
+        batches = batched(dataset, self.batch_size)
+        outputs: list[Iterable[EvaluationResultT] | Exception] = []
+
+        with tqdm(total=total_samples, desc="Evaluation", unit="sample") as progress_bar:
+            for batch in batches:
+                batch_list = list(batch)
+
+                if self.parallelize_batches:
+                    tasks = [self._call_with_error_handling(pipeline, [sample]) for sample in batch_list]
+                    batch_results = await asyncio.gather(*tasks)
+
+                    for result in batch_results:
+                        outputs.append(result)
+                        progress_bar.update(1)
+                else:
+                    result = await self._call_with_error_handling(pipeline, batch_list)
+                    outputs.append(result)
+                    progress_bar.update(len(batch_list))
+
         end_time = time.perf_counter()
-        return pipe_outputs, self._compute_time_perf(start_time, end_time, len(pipe_outputs))
 
-    @staticmethod
-    def _results_processor(results: list[EvaluationResult]) -> dict[str, Any]:
+        errors = [output for output in outputs if isinstance(output, Exception)]
+        results = [item for output in outputs if not isinstance(output, Exception) for item in output]
+
+        return results, errors, self._compute_time_perf(start_time, end_time, len(results))
+
+    async def _call_with_error_handling(
+        self,
+        executable: Callable[_CallP, Awaitable[_CallReturnT]],
+        *executable_args: _CallP.args,
+        **executable_kwargs: _CallP.kwargs,
+    ) -> _CallReturnT | Exception:
         """
-        Process the results.
+        Call executable with a standarized error handling.
+        If an error occurs, the executable is retried `num_retries` times using randomized exponential backoff.
 
         Args:
-            results: The evaluation results.
+            executable: The callable function to execute.
+            executable_args: Positional arguments to pass to the executable.
+            executable_kwargs: Keyword arguments to pass to the executable.
 
         Returns:
-            The processed results.
-        """
-        return {"results": [asdict(result) for result in results]}
+            The result of the executable if successful.
 
-    @staticmethod
-    def _compute_metrics(metrics: MetricSet, results: list[EvaluationResult]) -> dict[str, Any]:
+        Raises:
+            Exception: The last encountered exception after all retries are exhausted.
         """
-        Compute a metric using the given inputs.
+        for i in range(max(0, self.num_retries) + 1):
+            try:
+                return await executable(*executable_args, **executable_kwargs)
+            except Exception as exc:
+                if i == self.num_retries:
+                    return exc
 
-        Args:
-            metrics: The metrics to be computed.
-            results: The evaluation results.
+                delay = random.uniform(0, min(2**i * self.backoff_multiplier, self.backoff_max))  # noqa: S311
+                await asyncio.sleep(delay)
 
-        Returns:
-            The computed metric.
-        """
-        return {"metrics": metrics.compute(results)}
+        raise RuntimeError("Unreachable code reached")  # mypy quirk
 
     @staticmethod
-    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> dict[str, Any]:
+    def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> EvaluationTimePerf:
         """
         Compute the performance metrics.
 
@@ -107,10 +237,8 @@ class Evaluator:
         throughput = num_samples / latency
         latency_sample = 1.0 / throughput if throughput > 0 else 0.0
 
-        return {
-            "time_perf": {
-                "total_time_in_seconds": latency,
-                "samples_per_second": throughput,
-                "latency_in_seconds": latency_sample,
-            },
-        }
+        return EvaluationTimePerf(
+            total_time_in_seconds=latency,
+            samples_per_second=throughput,
+            latency_in_seconds=latency_sample,
+        )

ragbits/evaluate/factories/__init__.py

@@ -0,0 +1,42 @@
+import asyncio
+
+from continuous_eval.metrics.retrieval.matching_strategy import RougeChunkMatch
+from datasets import load_dataset
+
+from ragbits.core.embeddings.dense import LiteLLMEmbedder
+from ragbits.core.sources.hf import HuggingFaceSource
+from ragbits.core.vector_stores.in_memory import InMemoryVectorStore
+from ragbits.document_search import DocumentSearch
+from ragbits.document_search.documents.document import DocumentMeta
+from ragbits.evaluate.dataloaders.document_search import DocumentSearchDataLoader
+from ragbits.evaluate.metrics import MetricSet
+from ragbits.evaluate.metrics.document_search import DocumentSearchPrecisionRecallF1
+
+
+async def _add_example_documents(document_search: DocumentSearch) -> None:
+    dataset = load_dataset(path="deepsense-ai/synthetic-rag-dataset_v1.0", split="train")
+    documents = [DocumentMeta.from_literal(doc) for chunks in dataset["chunks"] for doc in chunks]
+    await document_search.ingest(documents)
+
+
+def basic_document_search_factory() -> DocumentSearch:
+    """
+    Factory for basic example document search instance.
+    """
+    document_search: DocumentSearch = DocumentSearch(vector_store=InMemoryVectorStore(embedder=LiteLLMEmbedder()))
+    asyncio.run(_add_example_documents(document_search))
+    return document_search
+
+
+def synthetic_rag_dataset() -> DocumentSearchDataLoader:
+    """
+    Factory for synthetic RAG dataset.
+    """
+    return DocumentSearchDataLoader(source=HuggingFaceSource(path="deepsense-ai/synthetic-rag-dataset_v1.0"))
+
+
+def precision_recall_f1() -> MetricSet:
+    """
+    Factory of precision recall f1 metric set for retrival evaluation.
+    """
+    return MetricSet(DocumentSearchPrecisionRecallF1(matching_strategy=RougeChunkMatch()))

ragbits/evaluate/metrics/__init__.py

@@ -1,24 +1,3 @@
-import sys
+from ragbits.evaluate.metrics.base import Metric, MetricSet
 
-from omegaconf import ListConfig
-
-from ragbits.core.utils.config_handling import get_cls_from_config
-
-from .base import MetricSet
-
-module = sys.modules[__name__]
-
-
-def metric_set_factory(cfg: ListConfig) -> MetricSet:
-    """
-    A function creating MetricSet instance from the configuration
-    Args:
-        cfg - metric cnfiguration
-    Returns:
-        MetricSet
-    """
-    metrics = []
-    for metric_cfg in cfg:
-        metric_module = get_cls_from_config(metric_cfg.type, module)
-        metrics.append(metric_module(metric_cfg))
-    return MetricSet(*metrics)
+__all__ = ["Metric", "MetricSet"]

ragbits/evaluate/metrics/base.py

@@ -1,31 +1,35 @@
+import asyncio
 from abc import ABC, abstractmethod
-from typing import Any, Generic, TypeVar
+from types import ModuleType
+from typing import ClassVar, Generic
 
-from omegaconf import DictConfig
+from typing_extensions import Self
 
-from ragbits.evaluate.pipelines.base import EvaluationResult
+from ragbits.core.utils.config_handling import WithConstructionConfig
+from ragbits.evaluate import metrics
+from ragbits.evaluate.pipelines.base import EvaluationResultT
 
-ResultT = TypeVar("ResultT", bound=EvaluationResult)
 
-
-class Metric(Generic[ResultT], ABC):
+class Metric(WithConstructionConfig, Generic[EvaluationResultT], ABC):
     """
     Base class for metrics.
     """
 
-    def __init__(self, config: DictConfig | None = None) -> None:
+    default_module: ClassVar[ModuleType | None] = metrics
+    configuration_key: ClassVar[str] = "metric"
+
+    def __init__(self, weight: float = 1.0) -> None:
         """
-        Initializes the metric.
+        Initialize the metric.
 
         Args:
-            config: The metric configuration.
+            weight: Metric value weight in the final score, used during optimization.
         """
         super().__init__()
-        self.config = config
-        self.weight: float = getattr(self.config, "weight", 1.0)
+        self.weight = weight
 
     @abstractmethod
-    def compute(self, results: list[ResultT]) -> dict[str, Any]:
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metric.
 
@@ -37,21 +41,37 @@ class Metric(Generic[ResultT], ABC):
         """
 
 
-class MetricSet(Generic[ResultT]):
+class MetricSet(WithConstructionConfig, Generic[EvaluationResultT]):
     """
     Represents a set of metrics.
     """
 
-    def __init__(self, *metrics: Metric[ResultT]) -> None:
+    configuration_key: ClassVar[str] = "metrics"
+    default_module: ClassVar[ModuleType | None] = metrics
+
+    def __init__(self, *metrics: Metric[EvaluationResultT]) -> None:
         """
-        Initializes the metric set.
+        Initialize the metric set.
 
         Args:
             metrics: The metrics.
         """
         self.metrics = metrics
 
-    def compute(self, results: list[ResultT]) -> dict[str, Any]:
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `MetricSet` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the metric set.
+
+        Returns:
+            An instance of the metric set class initialized with the provided configuration.
+        """
+        return cls(*[Metric.subclass_from_config(metric_config) for metric_config in config.values()])
+
+    async def compute(self, results: list[EvaluationResultT]) -> dict:
         """
         Compute the metrics.
 
@@ -61,6 +81,9 @@ class MetricSet(Generic[ResultT]):
         Returns:
             The computed metrics.
         """
+        metric_results = await asyncio.gather(*[metric.compute(results) for metric in self.metrics])
         return {
-            name: metric.weight * value for metric in self.metrics for name, value in metric.compute(results).items()
+            name: metric.weight * value
+            for metric, result in zip(self.metrics, metric_results, strict=False)
+            for name, value in result.items()
         }

ragbits/evaluate/metrics/document_search.py

@@ -1,10 +1,9 @@
 import importlib
 from abc import ABC
-from typing import Any
 
 from continuous_eval.metrics.retrieval import PrecisionRecallF1, RankedRetrievalMetrics
-from continuous_eval.metrics.retrieval.matching_strategy import RougeChunkMatch
-from omegaconf import DictConfig, OmegaConf
+from continuous_eval.metrics.retrieval.matching_strategy import MatchingStrategy
+from typing_extensions import Self
 
 from ragbits.evaluate.metrics.base import Metric
 from ragbits.evaluate.pipelines.document_search import DocumentSearchResult
@@ -17,30 +16,37 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
     """
 
     metric_cls: type[PrecisionRecallF1 | RankedRetrievalMetrics]
-    default_matching_strategy: type[RougeChunkMatch] = RougeChunkMatch
-    default_matching_options: DictConfig = OmegaConf.create({"threshold": 0.5})
 
-    def __init__(self, config: DictConfig | None = None) -> None:
+    def __init__(self, matching_strategy: MatchingStrategy, weight: float = 1.0) -> None:
         """
-        Initializes the metric.
+        Initialize the document search metric.
 
         Args:
-            config: The metric configuration.
+            matching_strategy: Matching strategys that determine relevance.
+            weight: Metric value weight in the final score, used during optimization.
         """
-        super().__init__(config)
-        if not self.config:
-            matching_strategy = self.default_matching_strategy
-            options = self.default_matching_options
-
-        else:
-            matching_strategy = getattr(
-                importlib.import_module("continuous_eval.metrics.retrieval.matching_strategy"),
-                self.config.matching_strategy,
-            )
-            options = self.config.options
-        self.metric = self.metric_cls(matching_strategy(**options))
-
-    def compute(self, results: list[DocumentSearchResult]) -> dict[str, Any]:
+        super().__init__(weight=weight)
+        self.metric = self.metric_cls(matching_strategy)
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `DocumentSearchMetric` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the metric.
+
+        Returns:
+            An instance of the metric class initialized with the provided configuration.
+        """
+        matching_strategy_cls = getattr(
+            importlib.import_module("continuous_eval.metrics.retrieval.matching_strategy"),
+            config["matching_strategy"]["type"],
+        )
+        matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
+        return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))
+
+    async def compute(self, results: list[DocumentSearchResult]) -> dict:
         """
         Compute the metric.
 
@@ -51,7 +57,18 @@ class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
             The computed metric.
         """
         return self.metric.aggregate(
-            [self.metric(result.predicted_passages, result.reference_passages) for result in results]
+            [
+                self.metric(
+                    [
+                        element.text_representation
+                        for element in result.predicted_elements
+                        if element.text_representation
+                    ],
+                    result.reference_passages,
+                )
+                for result in results
+                if result.reference_passages is not None
+            ]
         )