EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl

euroeval/metrics/llm_as_a_judge.py

@@ -0,0 +1,260 @@
+"""Metrics based on LLM-as-a-judge."""
+
+import collections.abc as c
+import logging
+import typing as t
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from ..exceptions import InvalidBenchmark
+from ..logging_utils import log
+from ..utils import extract_json_dict_from_string
+from .base import Metric
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+
+class LLMAsAJudgeMetric(Metric):
+    """Use an LLM to judge the quality of the predictions."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        judge_id: str,
+        judge_kwargs: dict[str, t.Any],
+        user_prompt: str,
+        response_format: t.Type[BaseModel],
+        scoring_fn: t.Callable[[BaseModel | None], float],
+        condition_formatting_fn: t.Callable[[str], str] = lambda x: x,
+        system_prompt: str | None = None,
+    ) -> None:
+        """Initialise the LLM as a judge metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            judge_id:
+                The model ID of the LLM to use as a judge.
+            judge_kwargs:
+                Generation parameters for the judge model, such as temperature.
+            user_prompt:
+                The user prompt to use for the judge model. The prompt should be
+                formatted with the variables `prediction` and `condition`, to
+                include the model predictions and a description of what the prediction
+                should be judged on, respectively. If the condition is not needed,
+                it can be omitted from the prompt, but the `prediction` variable must
+                still be present.
+            response_format:
+                The response format to use for the judge model. This should be a
+                Pydantic model that defines the expected structure of the judge's
+                response.
+            scoring_fn:
+                A function that takes the judge's response and returns a score.
+            condition_formatting_fn (optional):
+                A function to format the condition string before it is included in the
+                user prompt. Defaults to a no-op function that returns the input
+                unchanged.
+            system_prompt (optional):
+                The system prompt to use for the judge model. If not provided, no system
+                prompt will be used.
+        """
+        super().__init__(name=name, pretty_name=pretty_name)
+        self.judge_id = judge_id
+        self.judge_kwargs = judge_kwargs
+        self.user_prompt = user_prompt
+        self.response_format = response_format
+        self.scoring_fn = scoring_fn
+        self.condition_formatting_fn = condition_formatting_fn
+        self.system_prompt = system_prompt
+
+        # Add response format to the generation kwargs
+        self.judge_kwargs["response_format"] = self.response_format
+
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Calculate the metric score using the judge model.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                The ground truth references.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+            dataset_config:
+                The dataset configuration.
+            benchmark_config:
+                The benchmark configuration.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+
+        Raises:
+            InvalidBenchmark:
+                If the number of predictions does not match the number of references,
+                or if the user prompt requires a condition but none is provided.
+        """
+        # Importing here to avoid circular imports
+        from ..benchmark_modules import LiteLLMModel
+        from ..model_cache import ModelCache
+
+        if not predictions or not references:
+            return None
+        elif len(predictions) != len(references):
+            raise InvalidBenchmark(
+                f"The number of predictions ({len(predictions):,}) does not match the "
+                f"number of references ({len(references):,})."
+            )
+
+        # Load the judge model
+        judge_model_config = LiteLLMModel.get_model_config(
+            model_id=self.judge_id, benchmark_config=benchmark_config
+        )
+        self.judge = LiteLLMModel(
+            model_config=judge_model_config,
+            dataset_config=dataset_config,
+            benchmark_config=benchmark_config,
+            log_metadata=False,
+            **self.judge_kwargs,
+        )
+
+        # Create a cache for the judge model
+        judge_cache = ModelCache(
+            model_cache_dir=Path(judge_model_config.model_cache_dir),
+            cache_name=f"{dataset_config.name}-model-outputs.json",
+            max_generated_tokens=dataset_config.max_generated_tokens,
+            progress_bar=benchmark_config.progress_bar,
+        )
+        judge_cache.load()
+
+        # Prepare the messages for the LLM
+        conversations = [
+            [
+                dict(
+                    role="user",
+                    content=self._apply_user_prompt(
+                        prediction=prediction, condition=condition
+                    ),
+                )
+            ]
+            for prediction, condition in zip(predictions, references)
+        ]
+        if self.system_prompt:
+            conversations = [
+                [dict(role="system", content=self.system_prompt), *conversation]
+                for conversation in conversations
+            ]
+
+        # Get the non-cached conversations and generate the completions for them
+        non_cached_conversations = [
+            (idx, conversation)
+            for idx, conversation in enumerate(conversations)
+            if conversation not in judge_cache
+        ]
+        if non_cached_conversations:
+            model_inputs = dict(messages=[c for _, c in non_cached_conversations])
+            non_cached_outputs = self.judge.generate(inputs=model_inputs)
+
+            # Store the non-cached outputs in the cache
+            judge_cache.add_to_cache(
+                model_inputs=model_inputs, model_output=non_cached_outputs
+            )
+            judge_cache.save()
+
+        # Load all the outputs from the cache, in the original order, and parse them
+        raw_outputs = [judge_cache[conversation] for conversation in conversations]
+        json_dicts = [
+            extract_json_dict_from_string(s=output.sequence) for output in raw_outputs
+        ]
+        outputs = [
+            self.response_format.model_validate(obj=json_dict)
+            if json_dict is not None
+            else None
+            for json_dict in json_dicts
+        ]
+
+        # Calculate the scores using the scoring function
+        scores = [self.scoring_fn(output) for output in outputs]
+        if not scores:
+            log(
+                f"No scores were calculated for {self.pretty_name}.",
+                level=logging.WARNING,
+            )
+            return None
+        return sum(scores) / len(scores)
+
+    def _apply_user_prompt(self, prediction: str, condition: str | None = None) -> str:
+        """Apply the user prompt to the prediction and condition.
+
+        Args:
+            prediction:
+                The model prediction.
+            condition (optional):
+                A description of what the prediction should be judged on. If not
+                provided, it will be omitted from the prompt.
+
+        Returns:
+            The formatted user prompt with the prediction and reference.
+
+        Raises:
+            InvalidBenchmark:
+                If the user prompt requires a reference but none is provided.
+        """
+        condition_required = "{condition}" in self.user_prompt
+        if condition_required and condition is None:
+            raise InvalidBenchmark(
+                f"The user prompt for the {self.pretty_name!r} metric requires a "
+                "condition, but none was provided."
+            )
+        if condition is not None:
+            return self.user_prompt.format(
+                prediction=prediction, condition=self.condition_formatting_fn(condition)
+            )
+        return self.user_prompt.format(prediction=prediction)
+
+
+### Fluency metric ###
+
+
+class Fluency(BaseModel):
+    """Response format for the fluency metric.
+
+    Attributes:
+        fluency:
+            The fluency rating, an integer between 1 and 5.
+    """
+
+    fluency: t.Annotated[int, Field(ge=1, le=5)]
+
+
+fluency_metric = LLMAsAJudgeMetric(
+    name="fluency",
+    pretty_name="Fluency",
+    judge_id="gpt-5-2025-08-07",
+    judge_kwargs=dict(temperature=1.0),
+    user_prompt="Please rate the fluency of the following text on a scale from 1 to 5, "
+    "with the following definitions:\n"
+    "- 1: Very poor fluency, many grammatical errors\n"
+    "- 2: Poor fluency, several grammatical errors\n"
+    "- 3: Average fluency, a few grammatical errors\n"
+    "- 4: Good fluency, no grammatical errors but sounds a bit off\n"
+    "- 5: Excellent fluency, no grammatical errors and sounds natural\n\n"
+    "Text: {prediction!r}\n\n"
+    "Output your rating as a JSON object with a single key 'fluency'.",
+    response_format=Fluency,
+    scoring_fn=lambda output: (output.fluency - 1) / 4.0 if output is not None else 0.0,
+)

euroeval/metrics/pipeline.py

@@ -0,0 +1,289 @@
+"""Metrics based on a scikit-learn Pipeline."""
+
+import collections.abc as c
+import logging
+import typing as t
+from pathlib import Path
+
+import cloudpickle
+import huggingface_hub as hf_hub
+import numpy as np
+from scipy.special import expit as sigmoid
+
+from ..exceptions import InvalidBenchmark
+from ..logging_utils import log, no_terminal_output
+from ..utils import unscramble
+from .base import Metric
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+    from sklearn.pipeline import Pipeline
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+
+T = t.TypeVar("T", bound=int | float | str | bool)
+
+
+class PreprocessingFunction(t.Protocol):
+    """A protocol for a preprocessing function."""
+
+    def __call__(
+        self, predictions: c.Sequence[int], dataset: "Dataset"
+    ) -> c.Sequence[int]:
+        """Preprocess the model predictions before they are passed to the pipeline.
+
+        Args:
+            predictions:
+                The model predictions.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+
+        Returns:
+            The preprocessed model predictions.
+        """
+        ...
+
+
+class PipelineMetric(Metric):
+    """Load a scikit-learn pipeline and use it to get scores from the predictions."""
+
+    def __init__(
+        self,
+        name: str,
+        pretty_name: str,
+        pipeline_repo: str,
+        pipeline_scoring_function: c.Callable[["Pipeline", c.Sequence], float],
+        pipeline_file_name: str = "pipeline.pkl",
+        preprocessing_fn: PreprocessingFunction | None = None,
+        postprocessing_fn: c.Callable[[float], tuple[float, str]] | None = None,
+    ) -> None:
+        """Initialise the pipeline transform metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+            pipeline_repo:
+                The Hugging Face repository ID of the scikit-learn pipeline to load.
+            pipeline_scoring_method:
+                The method to use for scoring the predictions with the pipeline. Takes
+                a 1D sequence of predictions and returns a float score.
+            pipeline_file_name (optional):
+                The name of the file to download from the Hugging Face repository.
+                Defaults to "pipeline.joblib".
+            preprocessing_fn (optional):
+                A function to apply to the predictions before they are passed to the
+                pipeline. This is useful for preprocessing the predictions to match
+                the expected input format of the pipeline. Defaults to a no-op function
+                that returns the input unchanged.
+            postprocessing_fn (optional):
+                A function to apply to the metric scores after they are computed,
+                taking the score to the postprocessed score along with its string
+                representation. Defaults to x -> (100 * x, f"{x:.2%}").
+        """
+        super().__init__(
+            name=name, pretty_name=pretty_name, postprocessing_fn=postprocessing_fn
+        )
+        self.pipeline_repo = pipeline_repo
+        self.pipeline_file_name = pipeline_file_name
+        self.pipeline_scoring_function = pipeline_scoring_function
+        self.pipeline: "Pipeline | None" = None
+        self.preprocessing_fn = preprocessing_fn
+
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Calculate the metric score using the scikit-learn pipeline.
+
+        Args:
+            predictions:
+                The model predictions.
+            references:
+                Not used, but required for consistency with the Metric interface.
+            dataset:
+                The dataset used for evaluation. This is only used in case any
+                additional metadata is used to compute the metrics.
+            dataset_config:
+                The dataset configuration.
+            benchmark_config:
+                The benchmark configuration.
+
+        Returns:
+            The calculated metric score, or None if the score should be ignored.
+        """
+        if self.pipeline is None:
+            self.pipeline = self._download_pipeline(
+                cache_dir=benchmark_config.cache_dir
+            )
+        if self.preprocessing_fn is not None:
+            predictions = self.preprocessing_fn(
+                predictions=predictions, dataset=dataset
+            )
+        return self.pipeline_scoring_function(self.pipeline, predictions)
+
+    def _download_pipeline(self, cache_dir: str) -> "Pipeline":
+        """Download the scikit-learn pipeline from the given URL.
+
+        Args:
+            cache_dir:
+                The directory to use for caching the downloaded pipeline.
+
+        Returns:
+            The downloaded scikit-learn pipeline.
+
+        Raises:
+            InvalidBenchmark:
+                If the loading of the pipeline fails for any reason.
+        """
+        log(f"Loading pipeline from {self.pipeline_repo}...", level=logging.DEBUG)
+        with no_terminal_output():
+            folder_path = hf_hub.HfApi(
+                token=unscramble("XbjeOLhwebEaSaDUMqqaPaPIhgOcyOfDpGnX_")
+            ).snapshot_download(
+                repo_id=self.pipeline_repo, repo_type="model", cache_dir=cache_dir
+            )
+        model_path = Path(folder_path, self.pipeline_file_name)
+        try:
+            with model_path.open(mode="rb") as f:
+                pipeline = cloudpickle.load(f)
+        except Exception as e:
+            raise InvalidBenchmark(
+                f"Failed to load pipeline from {self.pipeline_repo!r}: {e}"
+            ) from e
+        log(f"Successfully loaded pipeline: {pipeline}", level=logging.DEBUG)
+        return pipeline
+
+
+### European Values Metric ###
+
+
+def european_values_preprocessing_fn(
+    predictions: c.Sequence[int], dataset: "Dataset"
+) -> c.Sequence[int]:
+    """Preprocess the model predictions for the European Values metric.
+
+    Args:
+        predictions:
+            The model predictions, a sequence of integers representing the predicted
+            choices for each question.
+        dataset:
+            The dataset used for evaluation. This is only used in case any additional
+            metadata is used to compute the metrics.
+
+    Returns:
+        The preprocessed model predictions, a sequence of integers representing the
+        final predicted choices for each question after any necessary aggregation and
+        mapping.
+
+    Raises:
+        AssertionError:
+            If the number of predictions is not a multiple of 53, which is required
+            for the European Values metric.
+    """
+    num_questions = 53
+    num_phrasings_per_question = 5
+
+    # Convert the predictions to integers
+    integer_predictions = []
+    for prediction, idx_to_choice in zip(predictions, dataset["idx_to_choice"]):
+        idx_to_choice = {
+            int(idx): int(choice)
+            for idx, choice in idx_to_choice.items()
+            if choice is not None
+        }
+        if prediction not in idx_to_choice:
+            raise InvalidBenchmark(
+                f"The prediction {prediction} is not a valid index for the "
+                f"question with choices {idx_to_choice}."
+            )
+        integer_prediction = idx_to_choice[prediction]
+        integer_predictions.append(integer_prediction)
+
+    assert len(predictions) % num_questions == 0, (
+        f"The number of predictions ({len(predictions)}) is not a multiple of "
+        f"{num_questions}, which is required for the European Values metric."
+    )
+
+    # When we are using the situational version of the dataset, there are 5 phrasings
+    # for each question, so we need to aggregate the predictions by question, which we
+    # do using majority voting.
+    using_situational = len(predictions) == num_questions * num_phrasings_per_question
+    if using_situational:
+        # Reshape the predictions to a 2D array with `num_phrasings_per_question` rows
+        # (one for each phrasing) and `num_questions` columns (one for each question).
+        # The five phrasings for each question appear right after each other, e.g.,
+        # (0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, ...)
+        # Shape: (num_questions, num_phrasings_per_question)
+        arr = np.array(
+            [
+                integer_predictions[i : i + num_phrasings_per_question]
+                for i in range(0, len(predictions), num_phrasings_per_question)
+            ]
+        )
+
+        # Double check that we reshaped the predictions correctly
+        for idx, pred in enumerate(integer_predictions):
+            assert arr[idx // 5, idx % 5] == pred, (
+                f"Reshaped predictions do not match the original predictions at index "
+                f"{idx}: {arr[idx // 5, idx % 5]} != {pred}."
+            )
+
+        # Use majority voting to get the final prediction for each question
+        # Shape: (53,)
+        arr = np.apply_along_axis(lambda x: np.bincount(x).argmax(), axis=1, arr=arr)
+
+        # Convert the array to a list
+        integer_predictions = arr.tolist()
+
+    # Some of the questions are categorical and we're only interested in whether the
+    # model chooses a specific choice or not. This mapping takes the question index
+    # to the choice value that we're interested in.
+    question_choices = {
+        0: 1,
+        1: 5,
+        3: 3,
+        6: 1,
+        15: 4,
+        20: 2,
+        47: 8,
+        48: 7,
+        49: 4,
+        51: 4,
+        52: 4,
+    }
+
+    # Map the predictions to the choices we're interested in
+    integer_predictions = list(integer_predictions)
+    for question_idx, choice in question_choices.items():
+        integer_predictions[question_idx] = (
+            1 if integer_predictions[question_idx] == choice else 0
+        )
+
+    return integer_predictions
+
+
+def european_values_scoring_function(
+    pipeline: "Pipeline", predictions: c.Sequence[int]
+) -> float:
+    """Scoring function for the European Values metric."""
+    normalised_predictions = pipeline[0].transform([predictions])
+    log_likelihoods = pipeline[1].transform(normalised_predictions)[0]
+    score = sigmoid(pipeline[2].alpha_ * (log_likelihoods - pipeline[2].center_))
+    return score.item()
+
+
+european_values_metric = PipelineMetric(
+    name="european_values",
+    pretty_name="European Values",
+    pipeline_repo="EuroEval/european-values-pipeline",
+    pipeline_scoring_function=european_values_scoring_function,
+    preprocessing_fn=european_values_preprocessing_fn,
+)

euroeval/metrics/speed.py

@@ -0,0 +1,48 @@
+"""Inference speed metric."""
+
+import collections.abc as c
+import typing as t
+
+from .base import Metric
+
+if t.TYPE_CHECKING:
+    from datasets.arrow_dataset import Dataset
+
+    from ..data_models import BenchmarkConfig, DatasetConfig
+
+
+class SpeedMetric(Metric):
+    """Speed metric."""
+
+    def __init__(self, name: str, pretty_name: str) -> None:
+        """Initialise the speed metric.
+
+        Args:
+            name:
+                The name of the metric in snake_case.
+            pretty_name:
+                The pretty name of the metric, used for display purposes.
+        """
+        super().__init__(
+            name=name,
+            pretty_name=pretty_name,
+            postprocessing_fn=lambda raw_score: (raw_score, f"{raw_score:,.0f}"),
+        )
+
+    def __call__(
+        self,
+        predictions: c.Sequence,
+        references: c.Sequence,
+        dataset: "Dataset",
+        dataset_config: "DatasetConfig",
+        benchmark_config: "BenchmarkConfig",
+    ) -> float | None:
+        """Not used with the speed metric, but required for consistency."""
+        raise NotImplementedError
+
+
+speed_metric = SpeedMetric(name="speed", pretty_name="Tokens per second")
+
+speed_short_metric = SpeedMetric(
+    name="speed_short", pretty_name="Tokens per second on short documents"
+)