ragbits-evaluate 0.0.30.dev29302392__py3-none-any.whl

ragbits/evaluate/metrics/human_eval.py

@@ -0,0 +1,105 @@
+import math
+from statistics import mean
+
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.human_eval import HumanEvalResult
+
+
+class HumanEvalPassAtK(Metric[HumanEvalResult]):
+    """
+    Computes pass@k over HumanEval tasks.
+    Measures the fraction of tasks with at least one passing sample out of k attempts.
+    """
+
+    def __init__(self, k: int = 1) -> None:
+        super().__init__()
+        self.k = k
+
+    async def compute(self, results: list[HumanEvalResult]) -> dict:
+        """Compute pass@k averaged over tasks.
+
+        Returns:
+            Dictionary with humaneval_pass@k: fraction of tasks with at least one passing sample.
+        """
+        values = []
+        for r in results:
+            n = len(r.passed_mask)
+            m = sum(1 for x in r.passed_mask if x)
+            k = min(self.k, n)
+            if n == 0 or k == 0:
+                values.append(0.0)
+                continue
+            if m == 0:
+                values.append(0.0)
+                continue
+            if m == n:
+                values.append(1.0)
+                continue
+            # 1 - C(n-m, k) / C(n, k)
+            denom = math.comb(n, k)
+            numer = math.comb(n - m, k) if n - m >= k else 0
+            values.append(1.0 - (numer / denom))
+        return {f"humaneval_pass@{self.k}": float(mean(values)) if values else 0.0}
+
+
+class HumanEvalQualityPerf(Metric[HumanEvalResult]):
+    """
+    Code quality and execution performance metrics:
+    - humaneval_compile_rate: fraction of samples that compiled
+    - humaneval_syntax_error_rate: fraction of samples with syntax error (compile failed)
+    - humaneval_assert_fail_rate: fraction of samples that ran but failed assertions
+    - humaneval_runtime_error_rate: fraction of samples with other runtime errors
+    - humaneval_timeout_rate: fraction of samples that timed out
+    - humaneval_tasks_solved: fraction of tasks with any passing sample
+    - humaneval_avg_exec_time_sec: average exec time over compilable runs
+    """
+
+    @staticmethod
+    async def compute(results: list[HumanEvalResult]) -> dict:
+        """Compute code quality and execution performance metrics.
+
+        Returns:
+            Dictionary with compile rates, error rates, tasks solved rate, and average execution time.
+        """
+        total_samples = sum(len(r.passed_mask) for r in results)
+        compiled = 0
+        syntax_errors = 0
+        assert_fails = 0
+        runtime_errors = 0
+        timeouts = 0
+        any_pass = sum(1 for r in results if any(r.passed_mask))
+        durations: list[float] = []
+
+        for r in results:
+            for ok, err, dur in zip(r.compile_ok_mask, r.errors, r.exec_durations_sec, strict=False):
+                if ok:
+                    compiled += 1
+                    durations.append(dur)
+                    if err:
+                        if err.startswith("AssertionError"):
+                            assert_fails += 1
+                        elif err.startswith("TimeoutError"):
+                            timeouts += 1
+                        else:
+                            runtime_errors += 1
+                else:
+                    # Compile failed: count as syntax error
+                    syntax_errors += 1
+
+        compile_rate = (compiled / total_samples) if total_samples else 0.0
+        syntax_error_rate = (syntax_errors / total_samples) if total_samples else 0.0
+        assert_fail_rate = (assert_fails / total_samples) if total_samples else 0.0
+        runtime_error_rate = (runtime_errors / total_samples) if total_samples else 0.0
+        timeout_rate = (timeouts / total_samples) if total_samples else 0.0
+        tasks_solved = (any_pass / len(results)) if results else 0.0
+        avg_exec_time = float(mean(durations)) if durations else 0.0
+
+        return {
+            "humaneval_compile_rate": float(compile_rate),
+            "humaneval_syntax_error_rate": float(syntax_error_rate),
+            "humaneval_assert_fail_rate": float(assert_fail_rate),
+            "humaneval_runtime_error_rate": float(runtime_error_rate),
+            "humaneval_timeout_rate": float(timeout_rate),
+            "humaneval_tasks_solved": float(tasks_solved),
+            "humaneval_avg_exec_time_sec": avg_exec_time,
+        }

ragbits/evaluate/metrics/question_answer.py

@@ -0,0 +1,205 @@
+import asyncio
+from abc import ABC, abstractmethod
+from asyncio import AbstractEventLoop
+from itertools import chain
+from typing import Generic, TypeVar
+
+from continuous_eval.llm_factory import LLMInterface
+from continuous_eval.metrics.base import LLMBasedMetric
+from continuous_eval.metrics.generation.text import (
+    LLMBasedAnswerCorrectness,
+    LLMBasedAnswerRelevance,
+    LLMBasedFaithfulness,
+    LLMBasedStyleConsistency,
+)
+from typing_extensions import Self
+
+from ragbits.agents.types import QuestionAnswerPromptOutputT
+from ragbits.core.llms.base import LLM
+from ragbits.core.utils.helpers import batched
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.question_answer import QuestionAnswerResult
+
+MetricT = TypeVar("MetricT", bound=LLMBasedMetric)
+
+
+class _MetricLMM(LLMInterface):
+    """
+    Implementation of required interface of Relari generative metrics based on LiteLMM.
+    """
+
+    def __init__(self, llm: LLM, loop: AbstractEventLoop) -> None:
+        self._llm = llm
+        self._loop = loop
+
+    def run(self, prompt: dict[str, str], temperature: float = 0, max_tokens: int = 1024) -> str:
+        formatted_prompt = [
+            {"role": "system", "content": prompt["system_prompt"]},
+            {"role": "user", "content": prompt["user_prompt"]},
+        ]
+        options = self._llm.options_cls(
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        return asyncio.run_coroutine_threadsafe(
+            self._llm.generate(formatted_prompt, options=options),
+            self._loop,
+        ).result()
+
+
+class QuestionAnswerMetric(Generic[MetricT], Metric[QuestionAnswerResult], ABC):
+    """
+    Metric for question answer evaluation based on Relari backend.
+    More details can be found [here](https://docs.relari.ai/category/text-generation).
+    """
+
+    metric_cls: type[MetricT]
+
+    def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
+        """
+        Initialize the agent metric.
+
+        Args:
+            llm: Judge LLM instance.
+            batch_size: Batch size for metric computation.
+            weight: Metric value weight in the final score, used during optimization.
+        """
+        super().__init__(weight=weight)
+        self.llm = llm
+        self.batch_size = batch_size
+
+    @classmethod
+    def from_config(cls, config: dict) -> Self:
+        """
+        Create an instance of `QuestionAnswerMetric` from a configuration dictionary.
+
+        Args:
+            config: A dictionary containing configuration settings for the metric.
+
+        Returns:
+            An instance of the metric class initialized with the provided configuration.
+        """
+        config["llm"] = LLM.from_config(config["llm"])
+        config["batch_size"] = config.get("batch_size", 15)
+        config["weight"] = config.get("weight", 1.0)
+        return super().from_config(config)
+
+    async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
+        """
+        Compute the metric.
+
+        Args:
+            results: The evaluation results.
+
+        Returns:
+            The computed metric.
+        """
+        metric = self.metric_cls(_MetricLMM(self.llm, loop=asyncio.get_running_loop()))
+        metric_results = chain.from_iterable(
+            [
+                await asyncio.gather(*[asyncio.to_thread(self._call_metric, metric, result) for result in batch])
+                for batch in batched(results, self.batch_size)
+            ]
+        )
+        return metric.aggregate(list(metric_results))
+
+    @staticmethod
+    @abstractmethod
+    def _call_metric(metric: MetricT, result: QuestionAnswerResult[QuestionAnswerPromptOutputT]) -> dict:
+        """
+        Call the metric with the proper arguments.
+        """
+
+
+class QuestionAnswerAnswerCorrectness(QuestionAnswerMetric[LLMBasedAnswerCorrectness]):
+    """
+    Metric checking answer correctness based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_correctness).
+    """
+
+    metric_cls: type[LLMBasedAnswerCorrectness] = LLMBasedAnswerCorrectness
+
+    @staticmethod
+    def _call_metric(
+        metric: LLMBasedAnswerCorrectness,
+        result: QuestionAnswerResult[QuestionAnswerPromptOutputT],
+    ) -> dict:
+        return metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            ground_truth_answers=result.reference_answer,
+        )
+
+
+class QuestionAnswerAnswerFaithfulness(QuestionAnswerMetric[LLMBasedFaithfulness]):
+    """
+    Metric checking answer faithfulness based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_faithfulness).
+    """
+
+    metric_cls: type[LLMBasedFaithfulness] = LLMBasedFaithfulness
+
+    @staticmethod
+    def _call_metric(
+        metric: LLMBasedFaithfulness,
+        result: QuestionAnswerResult[QuestionAnswerPromptOutputT],
+    ) -> dict:
+        return metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            retrieved_context=result.reference_context,
+        )
+
+
+class QuestionAnswerAnswerRelevance(QuestionAnswerMetric[LLMBasedAnswerRelevance]):
+    """
+    Metric checking answer relevance based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_relevance).
+    """
+
+    metric_cls: type[LLMBasedAnswerRelevance] = LLMBasedAnswerRelevance
+
+    @staticmethod
+    def _call_metric(
+        metric: LLMBasedAnswerRelevance,
+        result: QuestionAnswerResult[QuestionAnswerPromptOutputT],
+    ) -> dict:
+        return metric(
+            question=result.question,
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+        )
+
+
+class QuestionAnswerAnswerConsistency(QuestionAnswerMetric[LLMBasedStyleConsistency]):
+    """
+    Metric checking answer relevance based on LLM.
+    More details can be found [here](https://docs.relari.ai/metrics/Generation/LLM-Based/llm_style).
+    """
+
+    metric_cls: type[LLMBasedStyleConsistency] = LLMBasedStyleConsistency
+
+    @staticmethod
+    def _call_metric(
+        metric: LLMBasedStyleConsistency,
+        result: QuestionAnswerResult[QuestionAnswerPromptOutputT],
+    ) -> dict:
+        return metric(
+            answer=(
+                result.predicted_result.content
+                if isinstance(result.predicted_result.content, str)
+                else result.predicted_result.content.answer
+            ),
+            ground_truth_answers=result.reference_answer,
+        )

ragbits/evaluate/optimizer.py

@@ -0,0 +1,210 @@
+import asyncio
+import warnings
+from collections.abc import Callable
+from copy import deepcopy
+
+import optuna
+from optuna import Trial
+from pydantic import BaseModel
+
+from ragbits.core.utils.config_handling import WithConstructionConfig, import_by_path
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.evaluator import Evaluator, EvaluatorConfig
+from ragbits.evaluate.metrics.base import MetricSet
+from ragbits.evaluate.pipelines.base import EvaluationPipeline
+from ragbits.evaluate.utils import setup_optuna_neptune_callback
+
+
+class OptimizerConfig(BaseModel):
+    """
+    Schema for the optimizer config.
+    """
+
+    evaluator: EvaluatorConfig
+    optimizer: dict | None = None
+    neptune_callback: bool = False
+
+
+class Optimizer(WithConstructionConfig):
+    """
+    Optimizer class.
+    """
+
+    def __init__(self, direction: str = "maximize", n_trials: int = 10, max_retries_for_trial: int = 1) -> None:
+        """
+        Initialize the pipeline optimizer.
+
+        Args:
+            direction: Direction of optimization.
+            n_trials: The number of trials for each process.
+            max_retries_for_trial: The number of retires for single process.
+        """
+        self.direction = direction
+        self.n_trials = n_trials
+        self.max_retries_for_trial = max_retries_for_trial
+        # workaround for optuna not allowing different choices for different trials
+        # TODO check how optuna handles parallelism. discuss if we want to have parallel studies
+        self._choices_cache: dict[str, list] = {}
+
+    @classmethod
+    def run_from_config(cls, config: dict) -> list[tuple[dict, float, dict[str, float]]]:
+        """
+        Run the optimization process configured with a config object.
+
+        Args:
+            config: Optimizer config.
+
+        Returns:
+            List of tested configs with associated scores and metrics.
+        """
+        optimizer_config = OptimizerConfig.model_validate(config)
+        evaluator_config = EvaluatorConfig.model_validate(optimizer_config.evaluator)
+
+        dataloader: DataLoader = DataLoader.subclass_from_config(evaluator_config.evaluation.dataloader)
+        metricset: MetricSet = MetricSet.from_config(evaluator_config.evaluation.metrics)
+
+        pipeline_class = import_by_path(evaluator_config.evaluation.pipeline.type)
+        pipeline_config = dict(evaluator_config.evaluation.pipeline.config)
+        callbacks = [setup_optuna_neptune_callback()] if optimizer_config.neptune_callback else []
+
+        optimizer = cls.from_config(optimizer_config.optimizer or {})
+        return optimizer.optimize(
+            pipeline_class=pipeline_class,
+            pipeline_config=pipeline_config,
+            metricset=metricset,
+            dataloader=dataloader,
+            callbacks=callbacks,
+        )
+
+    def optimize(
+        self,
+        pipeline_class: type[EvaluationPipeline],
+        pipeline_config: dict,
+        dataloader: DataLoader,
+        metricset: MetricSet,
+        callbacks: list[Callable] | None = None,
+    ) -> list[tuple[dict, float, dict[str, float]]]:
+        """
+        Run the optimization process for given parameters.
+
+        Args:
+            pipeline_class: Pipeline to be optimized.
+            pipeline_config: Configuration defining the optimization process.
+            dataloader: Data loader.
+            metricset: Metrics to be optimized.
+            callbacks: Experiment callbacks.
+
+        Returns:
+            List of tested configs with associated scores and metrics.
+        """
+
+        def objective(trial: Trial) -> float:
+            return self._objective(
+                trial=trial,
+                pipeline_class=pipeline_class,
+                pipeline_config=pipeline_config,
+                dataloader=dataloader,
+                metricset=metricset,
+            )
+
+        study = optuna.create_study(direction=self.direction)
+        study.optimize(
+            func=objective,
+            n_trials=self.n_trials,
+            callbacks=callbacks,
+        )
+        return sorted(
+            [
+                (
+                    trial.user_attrs["config"],
+                    trial.user_attrs["score"],
+                    trial.user_attrs["metrics"],
+                )
+                for trial in study.get_trials()
+            ],
+            key=lambda x: -x[1] if self.direction == "maximize" else x[1],
+        )
+
+    def _objective(
+        self,
+        trial: Trial,
+        pipeline_class: type[EvaluationPipeline],
+        pipeline_config: dict,
+        dataloader: DataLoader,
+        metricset: MetricSet,
+    ) -> float:
+        """
+        Run a single experiment.
+        """
+        evaluator = Evaluator()
+        event_loop = asyncio.get_event_loop()
+
+        score = 1e16 if self.direction == "maximize" else -1e16
+        metrics_values = None
+        config_for_trial = None
+
+        for attempt in range(1, self.max_retries_for_trial + 1):
+            try:
+                config_for_trial = deepcopy(pipeline_config)
+                self._set_values_for_optimized_params(cfg=config_for_trial, trial=trial, ancestors=[])
+                pipeline = pipeline_class.from_config(config_for_trial)
+
+                results = event_loop.run_until_complete(
+                    evaluator.compute(
+                        pipeline=pipeline,
+                        dataloader=dataloader,
+                        metricset=metricset,
+                    )
+                )
+                score = sum(results.metrics.values())
+                metrics_values = results.metrics
+                break
+            except Exception as exc:
+                message = (
+                    f"Execution of the trial failed: {exc}. A retry will be initiated"
+                    if attempt < self.max_retries_for_trial
+                    else f"Execution of the trial failed: {exc}. Setting the score to {score}"
+                )
+                warnings.warn(message=message, category=UserWarning)
+
+        trial.set_user_attr("score", score)
+        trial.set_user_attr("metrics", metrics_values)
+        trial.set_user_attr("config", config_for_trial)
+
+        return score
+
+    def _set_values_for_optimized_params(self, cfg: dict, trial: Trial, ancestors: list[str]) -> None:  # noqa: PLR0912
+        """
+        Recursive method for sampling parameter values for optuna trial.
+        """
+        for key, value in cfg.items():
+            if isinstance(value, dict):
+                if value.get("optimize"):
+                    param_id = f"{'.'.join(ancestors)}.{key}"  # type: ignore
+                    choices = value.get("choices")
+                    values_range = value.get("range")
+                    if choices and values_range:
+                        raise ValueError("Choices and range cannot be defined in couple")
+                    choices_index = self._choices_cache.get(param_id)
+                    if choices and not choices_index:
+                        choices_index = list(range(len(choices)))
+                        self._choices_cache[param_id] = choices_index
+                    if values_range:
+                        if isinstance(values_range[0], float) and isinstance(values_range[1], float):
+                            cfg[key] = trial.suggest_float(name=param_id, low=values_range[0], high=values_range[1])
+                        elif isinstance(values_range[0], int) and isinstance(values_range[1], int):
+                            cfg[key] = trial.suggest_int(name=param_id, low=values_range[0], high=values_range[1])
+                    else:
+                        if not choices:
+                            raise ValueError("Either choices or range must be specified")
+                        choice_idx = trial.suggest_categorical(name=param_id, choices=choices_index)  # type: ignore
+                        choice = choices[choice_idx]
+                        if isinstance(choice, dict):
+                            self._set_values_for_optimized_params(choice, trial, ancestors + [key, str(choice_idx)])  # type: ignore
+                        cfg[key] = choice
+                else:
+                    self._set_values_for_optimized_params(value, trial, ancestors + [key])  # type: ignore
+            elif isinstance(value, list):
+                for param in value:
+                    if isinstance(param, dict):
+                        self._set_values_for_optimized_params(param, trial, ancestors + [key])  # type: ignore

ragbits/evaluate/pipelines/__init__.py

@@ -0,0 +1,37 @@
+from ragbits.core.utils.config_handling import WithConstructionConfig
+from ragbits.document_search import DocumentSearch
+from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult
+from ragbits.evaluate.pipelines.document_search import DocumentSearchPipeline
+from ragbits.evaluate.pipelines.gaia import GaiaPipeline
+from ragbits.evaluate.pipelines.hotpot_qa import HotpotQAPipeline
+from ragbits.evaluate.pipelines.human_eval import HumanEvalPipeline
+
+__all__ = [
+    "DocumentSearchPipeline",
+    "EvaluationData",
+    "EvaluationPipeline",
+    "EvaluationResult",
+    "GaiaPipeline",
+    "HotpotQAPipeline",
+    "HumanEvalPipeline",
+]
+
+_target_to_evaluation_pipeline: dict[type[WithConstructionConfig], type[EvaluationPipeline]] = {
+    DocumentSearch: DocumentSearchPipeline,
+}
+
+
+def get_evaluation_pipeline_for_target(evaluation_target: WithConstructionConfig) -> EvaluationPipeline:
+    """
+    A function instantiating evaluation pipeline for given WithConstructionConfig object
+    Args:
+        evaluation_target: WithConstructionConfig object to be evaluated
+    Returns:
+        instance of evaluation pipeline
+    Raises:
+        ValueError for classes with no registered evaluation pipeline
+    """
+    for supported_type, evaluation_pipeline_type in _target_to_evaluation_pipeline.items():
+        if isinstance(evaluation_target, supported_type):
+            return evaluation_pipeline_type(evaluation_target=evaluation_target)
+    raise ValueError(f"Evaluation pipeline not implemented for {evaluation_target.__class__}")

ragbits/evaluate/pipelines/base.py

@@ -0,0 +1,64 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from dataclasses import dataclass
+from types import ModuleType
+from typing import ClassVar, Generic, TypeVar
+
+from pydantic import BaseModel
+
+from ragbits.core.utils.config_handling import WithConstructionConfig
+from ragbits.evaluate import pipelines
+
+EvaluationDataT = TypeVar("EvaluationDataT", bound="EvaluationData")
+EvaluationResultT = TypeVar("EvaluationResultT", bound="EvaluationResult")
+EvaluationTargetT = TypeVar("EvaluationTargetT", bound=WithConstructionConfig)
+
+
+class EvaluationData(BaseModel, ABC):
+    """
+    Represents the data for a single evaluation.
+    """
+
+
+@dataclass
+class EvaluationResult(ABC):
+    """
+    Represents the result of a single evaluation.
+    """
+
+
+class EvaluationPipeline(WithConstructionConfig, Generic[EvaluationTargetT, EvaluationDataT, EvaluationResultT], ABC):
+    """
+    Evaluation pipeline.
+    """
+
+    default_module: ClassVar[ModuleType | None] = pipelines
+    configuration_key: ClassVar[str] = "pipeline"
+
+    def __init__(self, evaluation_target: EvaluationTargetT) -> None:
+        """
+        Initialize the evaluation pipeline.
+
+        Args:
+            evaluation_target: Evaluation target instance.
+        """
+        super().__init__()
+        self.evaluation_target = evaluation_target
+
+    async def prepare(self) -> None:
+        """
+        Prepare pipeline for evaluation. Optional step.
+        """
+        pass
+
+    @abstractmethod
+    async def __call__(self, data: Iterable[EvaluationDataT]) -> Iterable[EvaluationResultT]:
+        """
+        Run the evaluation pipeline.
+
+        Args:
+            data: The evaluation data.
+
+        Returns:
+            The evaluation result.
+        """