EuroEval 15.2.0__py3-none-any.whl

euroeval/scores.py

@@ -0,0 +1,90 @@
+"""Aggregation of raw scores into the mean and a confidence interval."""
+
+import logging
+import typing as t
+import warnings
+
+import numpy as np
+
+if t.TYPE_CHECKING:
+    from .data_models import MetricConfig
+    from .types import ScoreDict
+
+logger = logging.getLogger("euroeval")
+
+
+def log_scores(
+    dataset_name: str,
+    metric_configs: list["MetricConfig"],
+    scores: list[dict[str, float]],
+    model_id: str,
+) -> "ScoreDict":
+    """Log the scores.
+
+    Args:
+        dataset_name:
+            Name of the dataset.
+        metric_configs:
+            List of metrics to log.
+        scores:
+            The scores that are to be logged. This is a list of dictionaries full of
+            scores.
+        model_id:
+            The full Hugging Face Hub path to the pretrained transformer model.
+
+    Returns:
+        A dictionary with keys 'raw_scores' and 'total', with 'raw_scores' being
+        identical to `scores` and 'total' being a dictionary with the aggregated scores
+        (means and standard errors).
+    """
+    logger.info(f"Finished evaluation of {model_id} on {dataset_name}.")
+
+    total_dict: dict[str, float] = dict()
+    for metric_cfg in metric_configs:
+        test_score, test_se = aggregate_scores(scores=scores, metric_config=metric_cfg)
+        test_score, test_score_str = metric_cfg.postprocessing_fn(test_score)
+        test_se, test_se_str = metric_cfg.postprocessing_fn(test_se)
+        total_dict[f"test_{metric_cfg.name}"] = test_score
+        total_dict[f"test_{metric_cfg.name}_se"] = test_se
+        logger.info(f"{metric_cfg.pretty_name}: {test_score_str} ± {test_se_str}")
+
+    return dict(raw=scores, total=total_dict)
+
+
+def aggregate_scores(
+    scores: list[dict[str, float]], metric_config: "MetricConfig"
+) -> tuple[float, float]:
+    """Helper function to compute the mean with confidence intervals.
+
+    Args:
+        scores:
+            Dictionary with the names of the metrics as keys, of the form
+            "<split>_<metric_name>", such as "val_f1", and values the metric values.
+        metric_config:
+            The configuration of the metric, which is used to collect the correct
+            metric from `scores`.
+
+    Returns:
+        A pair of floats, containing the score and the radius of its 95% confidence
+        interval.
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+
+        test_scores = [
+            (
+                dct[metric_config.name]
+                if metric_config.name in dct
+                else dct[f"test_{metric_config.name}"]
+            )
+            for dct in scores
+        ]
+        test_score = np.mean(test_scores).item()
+
+        if len(test_scores) > 1:
+            sample_std = np.std(test_scores, ddof=1)
+            test_se = sample_std / np.sqrt(len(test_scores))
+        else:
+            test_se = np.nan
+
+        return (test_score, 1.96 * test_se)

euroeval/speed_benchmark.py

@@ -0,0 +1,124 @@
+"""Benchmarking model inference speed."""
+
+import logging
+
+import pyinfer
+from tqdm.auto import tqdm
+from transformers import AutoTokenizer
+
+from .benchmark_modules import (
+    BenchmarkModule,
+    HuggingFaceEncoderModel,
+    LiteLLMModel,
+    VLLMModel,
+)
+from .data_models import BenchmarkConfig
+from .exceptions import InvalidBenchmark
+from .utils import clear_memory
+
+logger = logging.getLogger("euroeval")
+
+
+def benchmark_speed(
+    model: "BenchmarkModule", benchmark_config: "BenchmarkConfig"
+) -> list[dict[str, float]]:
+    """Benchmark model inference speed.
+
+    Args:
+        model:
+            Model to use.
+        benchmark_config:
+            Configuration for the benchmark.
+
+    Returns:
+        Dictionary of scores.
+    """
+    scores: list[dict[str, float]] = list()
+    for idx in tqdm(
+        iterable=range(benchmark_config.num_iterations),
+        desc="Benchmarking",
+        disable=not benchmark_config.progress_bar,
+    ):
+        itr_scores = benchmark_speed_single_iteration(model=model, itr_idx=idx)
+        clear_memory()
+        scores.append(itr_scores)
+        logger.debug(f"Scores for iteration {idx}: {itr_scores}")
+    return scores
+
+
+def benchmark_speed_single_iteration(
+    model: "BenchmarkModule", itr_idx: int
+) -> dict[str, float]:
+    """Run a single iteration of the speed benchmark.
+
+    Args:
+        model:
+            The model to use in the benchmark.
+        itr_idx:
+            The index of the iteration.
+
+    Returns:
+        A dictionary containing the scores for the current iteration.
+    """
+    gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2", trust_remote_code=True)
+
+    base_doc = "Document which contains roughly 10 tokens. "
+    multiplier = 10 * (1 + itr_idx)
+    doc = base_doc * multiplier
+    short_multiplier = 1.25 * (1 + itr_idx)
+    short_doc = base_doc * round(short_multiplier)
+
+    def generate_messages_predict(doc: str) -> None:
+        model.generate(inputs=dict(messages=[[dict(role="user", content=doc)]]))
+
+    def generate_prompt_predict(doc: str) -> None:
+        model.generate(inputs=dict(text=[doc]))
+
+    def encoder_predict(doc: str) -> None:
+        tokenizer = model.get_tokenizer()
+        pytorch_model = model.get_pytorch_module()
+        inputs = {
+            key: tensor.to(pytorch_model.device)
+            for key, tensor in tokenizer(
+                text=[doc], truncation=True, return_tensors="pt"
+            ).items()
+        }
+        pytorch_model(**inputs)
+
+    if isinstance(model, VLLMModel):
+        predict = generate_prompt_predict
+    elif isinstance(model, LiteLLMModel):
+        predict = generate_messages_predict
+    elif isinstance(model, HuggingFaceEncoderModel):
+        predict = encoder_predict
+    else:
+        raise ValueError(f"Model type {model} not supported for speed benchmark")
+
+    try:
+        # Do a warmup run, as the first run is always slower
+        pyinfer.InferenceReport(model=predict, inputs=base_doc, n_seconds=1).run(
+            print_report=False
+        )
+
+        speed_scores = pyinfer.InferenceReport(
+            model=predict, inputs=doc, n_seconds=3
+        ).run(print_report=False)
+        num_gpt2_tokens = len(gpt2_tokenizer([doc], truncation=True)["input_ids"][0])
+        gpt2_tokens_per_second = speed_scores["Infer(p/sec)"] * num_gpt2_tokens
+
+        speed_scores_short = pyinfer.InferenceReport(
+            model=predict, inputs=short_doc, n_seconds=3
+        ).run(print_report=False)
+        num_gpt2_tokens_short = len(
+            gpt2_tokenizer([short_doc], truncation=True)["input_ids"][0]
+        )
+        gpt2_tokens_per_second_short = (
+            speed_scores_short["Infer(p/sec)"] * num_gpt2_tokens_short
+        )
+
+    except (RuntimeError, ValueError, IndexError) as e:
+        raise InvalidBenchmark(f"Speed benchmark failed with error: {e!r}")
+
+    return dict(
+        test_speed=gpt2_tokens_per_second, test_speed_short=gpt2_tokens_per_second_short
+    )

euroeval/task_utils/__init__.py

@@ -0,0 +1 @@
+"""Utility functions related to the different tasks and task groups."""

euroeval/task_utils/multiple_choice_classification.py

@@ -0,0 +1,176 @@
+"""Utility functions related to the multiple-choice classification task group."""
+
+import hashlib
+import logging
+import re
+import typing as t
+from collections import defaultdict
+
+import numpy as np
+from datasets import Dataset
+from transformers import BatchEncoding, PreTrainedTokenizer, Trainer
+
+if t.TYPE_CHECKING:
+    from ..types import Labels, Predictions
+
+logger = logging.getLogger("euroeval")
+
+
+class MultipleChoiceClassificationTrainer(Trainer):
+    """Trainer subclass for question answering tasks."""
+
+    def evaluate(
+        self,
+        eval_dataset: "Dataset | None" = None,
+        ignore_keys: list[str] | None = None,
+        metric_key_prefix: str = "eval",
+    ) -> dict[str, float] | None:
+        """Evaluate the model on the given dataset.
+
+        Args:
+            eval_dataset:
+                The dataset to evaluate on. If None, then use the stored evaluation
+                dataset.
+            ignore_keys:
+                The keys to ignore when computing the metrics.
+            metric_key_prefix:
+                The prefix to use for the metric keys.
+
+        Returns:
+            The metrics computed on the evaluation dataset.
+        """
+        eval_dataloader = self.get_eval_dataloader(eval_dataset)
+
+        eval_loop = (
+            self.prediction_loop
+            if self.args.use_legacy_prediction_loop
+            else self.evaluation_loop
+        )
+        output = eval_loop(
+            eval_dataloader,
+            description="Evaluation",
+            prediction_loss_only=None,
+            ignore_keys=ignore_keys,
+            metric_key_prefix=metric_key_prefix,
+        )
+
+        if metric_key_prefix == "test":
+            preds_and_labels = postprocess_predictions_and_labels(
+                predictions=output.predictions, dataset=eval_dataset
+            )
+            output.metrics.update(self.compute_metrics(preds_and_labels))
+
+            # Prefix all keys with metric_key_prefix + '_'
+            for key in list(output.metrics.keys()):
+                if not key.startswith(f"{metric_key_prefix}_"):
+                    output.metrics[f"{metric_key_prefix}_{key}"] = output.metrics.pop(
+                        key
+                    )
+
+        # Only the main node log the results by default
+        if self.args.should_log:
+            self.log(output.metrics)
+
+        self.control = self.callback_handler.on_evaluate(
+            self.args,
+            self.state,
+            self.control,  # type: ignore[has-type]
+            output.metrics,
+        )
+        return output.metrics
+
+
+def prepare_examples(
+    examples: "BatchEncoding", tokenizer: "PreTrainedTokenizer"
+) -> "BatchEncoding":
+    """Prepare the features.
+
+    Args:
+        examples:
+            The examples to prepare.
+        tokenizer:
+            The tokenizer to use to prepare the examples.
+
+    Returns:
+        The prepared examples.
+    """
+    doc: str = examples["text"][0]
+    sections = doc.split("\n")
+
+    choice_idxs = [
+        idx
+        for idx, section in enumerate(sections)
+        if re.match(pattern=r"^[a-e]\. ", string=section) is not None
+    ]
+    choices = [sections[idx] for idx in choice_idxs]
+
+    # Check that the choices are present, and that all of them are at the end
+    assert len(choices) > 0, "No choices found in the document."
+    assert all(
+        choice_idx == len(sections) - i
+        for i, choice_idx in enumerate(sorted(choice_idxs, reverse=True), start=1)
+    ), "Choices are not at the end of the document."
+
+    question_idx = min(choice_idxs) - 2  # -2 to remove the 'Choices:' line
+    context_and_question = "\n".join(sections[: question_idx + 1]).strip()
+
+    new_examples = tokenizer(
+        text=[context_and_question] * len(choices),
+        text_pair=[choice[3:] for choice in choices],
+        padding=True,
+        truncation=True,
+    )
+    new_examples["label"] = [
+        int(choice.startswith(f"{letter}. ") and letter == examples["label"][0])
+        for letter, choice in zip("abcde", choices)
+    ]
+    new_examples["id"] = [hashlib.md5(string=doc.encode()).hexdigest()] * len(choices)
+    return new_examples
+
+
+def postprocess_predictions_and_labels(
+    predictions: np.ndarray, dataset: "Dataset"
+) -> tuple["Predictions", "Labels"]:
+    """Postprocess the predictions and labels.
+
+    Args:
+        predictions:
+            The model predictions, of shape (num_examples, 2).
+        dataset:
+            The dataset containing the examples.
+
+    Returns:
+        The postprocessed predictions and labels.
+    """
+    mapping = {0: "a", 1: "b", 2: "c", 3: "d", 4: "e"}
+
+    all_predictions: list[str] = list()
+    all_labels: list[str] = list()
+
+    pred_label_dict = defaultdict(list)
+    for pred_arr, example in zip(predictions, dataset):
+        pred_label_dict[example["id"]].append((pred_arr[1], example["label"]))
+
+    # Compute the final predictions and labels
+    for id_ in set(dataset["id"]):
+        preds, labels = zip(*pred_label_dict[id_])
+
+        # Some IDs appear multiple times in the dataset, since we are bootstrapping.
+        # Here we separate them into their respective groups.
+        assert len(labels) % sum(labels) == 0, (
+            "The number of labels is not divisible by the sum of the labels."
+        )
+        group_size = len(labels) // sum(labels)
+        preds_groups = [
+            preds[i : i + group_size] for i in range(0, len(preds), group_size)
+        ]
+        labels_groups = [
+            labels[i : i + group_size] for i in range(0, len(labels), group_size)
+        ]
+        for preds_group, labels_group in zip(preds_groups, labels_groups):
+            prediction: str = mapping[np.argmax(preds_group).item()]
+            label: str = mapping[np.argmax(labels_group).item()]
+            all_predictions.append(prediction)
+            all_labels.append(label)
+
+    return all_predictions, all_labels