ragbits-evaluate 0.2.0__tar.gz

ragbits_evaluate-0.2.0/.gitignore

@@ -0,0 +1,97 @@
+# Directories
+.vscode/
+.idea/
+.neptune/
+.pytest_cache/
+.mypy_cache/
+venv/
+__pycache__/
+**.egg-info/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Sphinx documentation
+docs/_build/
+public/
+# autogenerated package license table
+docs/licenses_table.rst
+
+# license dump file
+licenses.txt
+
+# File formats
+*.onnx
+*.pyc
+*.pt
+*.pth
+*.pkl
+*.mar
+*.torchscript
+**/.ipynb_checkpoints
+**/dist/
+**/checkpoints/
+**/outputs/
+**/multirun/
+
+# Other env files
+.python-version
+pyvenv.cfg
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# dotenv
+.env
+
+# coverage and pytest reports
+coverage.xml
+report.xml
+
+# CMake
+cmake-build-*/
+
+# Terraform
+**/.terraform.lock.hcl
+**/.terraform
+
+# benchmarks
+benchmarks/sql/data/
+
+# mkdocs generated files
+site/
+
+# build artifacts
+dist/

ragbits_evaluate-0.2.0/CHANGELOG.md

@@ -0,0 +1,13 @@
+# CHANGELOG
+
+## Unreleased
+
+## 0.2.0 (2024-10-23)
+
+- Initial release of the package.
+- Evaluation pipeline framework with capability to define evaluators & metrics.
+- Evaluation pipeline for `ragbits-document-search`.
+
+### Changed
+
+- ragbits-core updated to version v0.2.0

ragbits_evaluate-0.2.0/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.3
+Name: ragbits-evaluate
+Version: 0.2.0
+Summary: Evaluation module for Ragbits components
+Author-email: "deepsense.ai" <ragbits@deepsense.ai>
+License-Expression: MIT
+Keywords: Evaluation,GenAI,Generative AI,LLMs,Large Language Models,RAG,Retrieval Augmented Generation
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: hydra-core~=1.3.2
+Requires-Dist: neptune~=1.12.0
+Requires-Dist: ragbits-core==0.2.0
+Provides-Extra: relari
+Requires-Dist: continuous-eval~=0.3.12; extra == 'relari'
+Description-Content-Type: text/markdown
+
+# Ragbits Evaluate

ragbits_evaluate-0.2.0/README.md

@@ -0,0 +1 @@
+# Ragbits Evaluate

ragbits_evaluate-0.2.0/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "ragbits-evaluate"
+version = "0.2.0"
+description = "Evaluation module for Ragbits components"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [
+    { name = "deepsense.ai", email = "ragbits@deepsense.ai"}
+]
+keywords = [
+    "Retrieval Augmented Generation",
+    "RAG",
+    "Large Language Models",
+    "LLMs",
+    "Generative AI",
+    "GenAI",
+    "Evaluation"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = ["hydra-core~=1.3.2", "neptune~=1.12.0", "ragbits-core==0.2.0"]
+
+[project.optional-dependencies]
+relari = [
+    "continuous-eval~=0.3.12",
+]
+
+[tool.uv]
+dev-dependencies = [
+    "pre-commit~=3.8.0",
+    "pytest~=8.3.3",
+    "pytest-cov~=5.0.0",
+    "pytest-asyncio~=0.24.0",
+    "pip-licenses>=4.0.0,<5.0.0"
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ragbits"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

ragbits_evaluate-0.2.0/src/ragbits/evaluate/evaluator.py

@@ -0,0 +1,112 @@
+import time
+from dataclasses import asdict
+from typing import Any, Iterable
+
+from tqdm.asyncio import tqdm
+
+from ragbits.evaluate.loaders.base import DataLoader
+from ragbits.evaluate.metrics.base import MetricSet
+from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult
+
+
+class Evaluator:
+    """
+    Evaluator class.
+    """
+
+    async def compute(
+        self,
+        pipeline: EvaluationPipeline,
+        dataloader: DataLoader,
+        metrics: MetricSet,
+    ) -> dict[str, Any]:
+        """
+        Compute the evaluation results for the given pipeline and data.
+
+        Args:
+            pipeline: The pipeline to be evaluated.
+            dataloader: The dataloader to load the data.
+            metrics: The metrics to be computed.
+
+        Returns:
+            The evaluation results.
+        """
+        dataset = await dataloader.load()
+        results, perf_results = await self._call_pipeline(pipeline, dataset)
+        computed_metrics = self._compute_metrics(metrics, results)
+        processed_results = self._results_processor(results)
+
+        return {
+            **perf_results,
+            **computed_metrics,
+            **processed_results,
+        }
+
+    async def _call_pipeline(
+        self,
+        pipeline: EvaluationPipeline,
+        dataset: Iterable,
+    ) -> tuple[list[EvaluationResult], dict[str, Any]]:
+        """
+        Call the pipeline with the given data.
+
+        Args:
+            pipeline: The pipeline to be called.
+            data: The evaluation data.
+
+        Returns:
+            The evaluation results and performance metrics.
+        """
+        start_time = time.perf_counter()
+        pipe_outputs = await tqdm.gather(*[pipeline(data) for data in dataset], desc="Evaluation")
+        end_time = time.perf_counter()
+        return pipe_outputs, self._compute_time_perf(start_time, end_time, len(pipe_outputs))
+
+    def _results_processor(self, results: list[EvaluationResult]) -> dict[str, Any]:
+        """
+        Process the results.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The processed results.
+        """
+        return {"results": [asdict(result) for result in results]}
+
+    def _compute_metrics(self, metrics: MetricSet, results: list[EvaluationResult]) -> dict[str, Any]:
+        """
+        Compute a metric using the given inputs.
+
+        Args:
+            metrics: The metrics to be computed.
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+        return {"metrics": metrics.compute(results)}
+
+    def _compute_time_perf(self, start_time: float, end_time: float, num_samples: int) -> dict[str, Any]:
+        """
+        Compute the performance metrics.
+
+        Args:
+            start_time: The start time.
+            end_time: The end time.
+            num_samples: The number of samples.
+
+        Returns:
+            The performance metrics.
+        """
+        latency = end_time - start_time
+        throughput = num_samples / latency
+        latency_sample = 1.0 / throughput if throughput > 0 else 0.0
+
+        return {
+            "time_perf": {
+                "total_time_in_seconds": latency,
+                "samples_per_second": throughput,
+                "latency_in_seconds": latency_sample,
+            },
+        }

ragbits_evaluate-0.2.0/src/ragbits/evaluate/loaders/base.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from omegaconf import DictConfig
+
+DataT = TypeVar("DataT")
+
+
+class DataLoader(Generic[DataT], ABC):
+    """
+    Data loader.
+    """
+
+    def __init__(self, config: DictConfig) -> None:
+        self.config = config
+
+    @abstractmethod
+    async def load(self) -> DataT:
+        """
+        Load the data.
+
+        Returns:
+            The loaded data.
+        """

ragbits_evaluate-0.2.0/src/ragbits/evaluate/loaders/hf.py

@@ -0,0 +1,25 @@
+from typing import TypeAlias
+
+from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict, load_dataset
+
+from ragbits.evaluate.loaders.base import DataLoader
+
+HFData: TypeAlias = DatasetDict | Dataset | IterableDatasetDict | IterableDataset
+
+
+class HFDataLoader(DataLoader[HFData]):
+    """
+    Hugging Face data loader.
+    """
+
+    async def load(self) -> HFData:
+        """
+        Load the data from Hugging Face.
+
+        Returns:
+            The loaded data.
+        """
+        return load_dataset(
+            path=self.config.path,
+            split=self.config.split,
+        )

ragbits_evaluate-0.2.0/src/ragbits/evaluate/metrics/base.py

@@ -0,0 +1,78 @@
+from abc import ABC, abstractmethod
+from typing import Any, Generic, Optional, TypeVar
+
+from omegaconf import DictConfig
+from typing_extensions import Self
+
+from ragbits.evaluate.pipelines.base import EvaluationResult
+
+ResultT = TypeVar("ResultT", bound=EvaluationResult)
+
+
+class Metric(Generic[ResultT], ABC):
+    """
+    Base class for metrics.
+    """
+
+    def __init__(self, config: Optional[DictConfig] = None) -> None:
+        """
+        Initializes the metric.
+
+        Args:
+            config: The metric configuration.
+        """
+        super().__init__()
+        self.config = getattr(config, self.__class__.__name__, DictConfig({}))
+
+    @abstractmethod
+    def compute(self, results: list[ResultT]) -> dict[str, Any]:
+        """
+        Compute the metric.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+
+
+class MetricSet(Generic[ResultT]):
+    """
+    Represents a set of metrics.
+    """
+
+    def __init__(self, *metrics: type[Metric[ResultT]]) -> None:
+        """
+        Initializes the metric set.
+
+        Args:
+            metrics: The metrics.
+        """
+        self._metrics = metrics
+        self.metrics: list[Metric[ResultT]] = []
+
+    def __call__(self, config: Optional[DictConfig] = None) -> Self:
+        """
+        Initializes the metrics.
+
+        Args:
+            config: The configuration for the metrics.
+
+        Returns:
+            The initialized metric set.
+        """
+        self.metrics = [metric(config) for metric in self._metrics]
+        return self
+
+    def compute(self, results: list[ResultT]) -> dict[str, Any]:
+        """
+        Compute the metrics.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metrics.
+        """
+        return {name: value for metric in self.metrics for name, value in metric.compute(results).items()}

ragbits_evaluate-0.2.0/src/ragbits/evaluate/metrics/document_search.py

@@ -0,0 +1,68 @@
+import importlib
+from abc import ABC
+from typing import Any, Optional
+
+from continuous_eval.metrics.retrieval import PrecisionRecallF1, RankedRetrievalMetrics
+from omegaconf import DictConfig
+
+from ragbits.evaluate.metrics.base import Metric, MetricSet
+from ragbits.evaluate.pipelines.document_search import DocumentSearchResult
+
+
+class DocumentSearchMetric(Metric[DocumentSearchResult], ABC):
+    """
+    Metric for document search evaluation based on Relari backend.
+    More details can be found [here](https://docs.relari.ai/category/retrieval-rag).
+    """
+
+    metric_cls: type[PrecisionRecallF1 | RankedRetrievalMetrics]
+
+    def __init__(self, config: Optional[DictConfig] = None) -> None:
+        """
+        Initializes the metric.
+
+        Args:
+            config: The metric configuration.
+        """
+        super().__init__(config)
+
+        matching_strategy = getattr(
+            importlib.import_module("continuous_eval.metrics.retrieval.matching_strategy"),
+            self.config.matching_strategy,
+        )
+        self.metric = self.metric_cls(matching_strategy(**self.config.options))
+
+    def compute(self, results: list[DocumentSearchResult]) -> dict[str, Any]:
+        """
+        Compute the metric.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+        return self.metric.aggregate(
+            [self.metric(result.predicted_passages, result.reference_passages) for result in results]
+        )
+
+
+class DocumentSearchPrecisionRecallF1(DocumentSearchMetric):
+    """
+    Precision, recall, and F1 score for context retrieval.
+    More details can be found [here](https://docs.relari.ai/metrics/Retrieval/Deterministic/precision_recall).
+    """
+
+    metric_cls = PrecisionRecallF1
+
+
+class DocumentSearchRankedRetrievalMetrics(DocumentSearchMetric):
+    """
+    Rank-aware metrics takes into account the order in which the contexts are retrieved.
+    More details can be found [here](https://docs.relari.ai/metrics/Retrieval/Deterministic/rank_aware_metrics).
+    """
+
+    metric_cls = RankedRetrievalMetrics
+
+
+document_search_metrics = MetricSet(DocumentSearchPrecisionRecallF1, DocumentSearchRankedRetrievalMetrics)

ragbits_evaluate-0.2.0/src/ragbits/evaluate/pipelines/base.py

@@ -0,0 +1,40 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from omegaconf import DictConfig
+
+
+@dataclass
+class EvaluationResult(ABC):
+    """
+    Represents the result of a single evaluation.
+    """
+
+
+class EvaluationPipeline(ABC):
+    """
+    Collection evaluation pipeline.
+    """
+
+    def __init__(self, config: Optional[DictConfig] = None) -> None:
+        """
+        Initializes the evaluation pipeline.
+
+        Args:
+            config: The evaluation pipeline configuration.
+        """
+        super().__init__()
+        self.config = config or DictConfig({})
+
+    @abstractmethod
+    async def __call__(self, data: dict[str, Any]) -> EvaluationResult:
+        """
+        Runs the evaluation pipeline.
+
+        Args:
+            data: The evaluation data.
+
+        Returns:
+            The evaluation result.
+        """

ragbits_evaluate-0.2.0/src/ragbits/evaluate/pipelines/document_search.py

@@ -0,0 +1,51 @@
+from dataclasses import dataclass
+from functools import cached_property
+
+from ragbits.document_search import DocumentSearch
+from ragbits.document_search.documents.element import TextElement
+from ragbits.evaluate.pipelines.base import EvaluationPipeline, EvaluationResult
+
+
+@dataclass
+class DocumentSearchResult(EvaluationResult):
+    """
+    Represents the result of a single evaluation.
+    """
+
+    question: str
+    reference_passages: list[str]
+    predicted_passages: list[str]
+
+
+class DocumentSearchPipeline(EvaluationPipeline):
+    """
+    Document search evaluation pipeline.
+    """
+
+    @cached_property
+    def document_search(self) -> "DocumentSearch":
+        """
+        Returns the document search instance.
+
+        Returns:
+            The document search instance.
+        """
+        return DocumentSearch.from_config(self.config)  # type: ignore
+
+    async def __call__(self, data: dict) -> DocumentSearchResult:
+        """
+        Runs the document search evaluation pipeline.
+
+        Args:
+            data: The evaluation data.
+
+        Returns:
+            The evaluation result.
+        """
+        elements = await self.document_search.search(data["question"])
+        predicted_passages = [element.content for element in elements if isinstance(element, TextElement)]
+        return DocumentSearchResult(
+            question=data["question"],
+            reference_passages=data["passages"],
+            predicted_passages=predicted_passages,
+        )

ragbits_evaluate-0.2.0/src/ragbits/evaluate/utils.py

@@ -0,0 +1,91 @@
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from hydra.core.hydra_config import HydraConfig
+from neptune import Run
+from neptune.utils import stringify_unsupported
+from omegaconf import DictConfig
+
+
+def _save(file_path: Path, **data: Any) -> None:
+    """
+    Save the data to a file. Add the current timestamp and Python version to the data.
+
+    Args:
+        file_path: The path to the file.
+        data: The data to be saved.
+    """
+    current_time = datetime.now()
+
+    data["_timestamp"] = current_time.isoformat()
+    data["_python_version"] = sys.version
+    data["_interpreter_path"] = sys.executable
+
+    with open(file_path, "w", encoding="utf-8") as file:
+        json.dump(data, file, indent=4)
+
+
+def log_to_file(results: dict[str, Any], output_dir: Path | None = None) -> Path:
+    """
+    Log the evaluation results locally.
+
+    Args:
+        results: The evaluation results.
+        output_dir: The output directory.
+
+    Returns:
+        The output directory.
+    """
+    output_dir = output_dir or Path(HydraConfig.get().runtime.output_dir)
+    metrics_file = output_dir / "metrics.json"
+    results_file = output_dir / "results.json"
+
+    _save(metrics_file, metrics=results["metrics"], time_perf=results["time_perf"])
+    _save(results_file, results=results["results"])
+
+    return output_dir
+
+
+def setup_neptune(config: DictConfig) -> Run | None:
+    """
+    Setup the Neptune run.
+
+    Args:
+        config: The Hydra configuration.
+
+    Returns:
+        The Neptune run.
+    """
+    if config.neptune.run:
+        run = Run(
+            project=config.neptune.project,
+            tags=[
+                config.task.type,
+                config.task.name,
+                config.data.name,
+            ],
+        )
+        run["config"] = stringify_unsupported(config)
+        return run
+    return None
+
+
+def log_to_neptune(run: Run, results: dict[str, Any], output_dir: Path | None = None) -> None:
+    """
+    Log the evaluation results to Neptune.
+
+    Args:
+        run: The Neptune run.
+        results: The evaluation results.
+        output_dir: The output directory.
+    """
+    output_dir = output_dir or Path(HydraConfig.get().runtime.output_dir)
+
+    run["evaluation/metrics"] = stringify_unsupported(results["metrics"])
+    run["evaluation/time_perf"] = stringify_unsupported(results["time_perf"])
+    run["evaluation/results"] = stringify_unsupported(results["results"])
+    run["evaluation/metrics.json"].upload((output_dir / "metrics.json").as_posix())
+    run["evaluation/results.json"].upload((output_dir / "results.json").as_posix())