EuroEval 16.4.0__py3-none-any.whl → 16.5.0__py3-none-any.whl

euroeval/generation.py

@@ -1,5 +1,6 @@
 """Functions related to text generation of models."""
 
+import collections.abc as c
 import logging
 import sys
 import typing as t
@@ -33,11 +34,11 @@ if t.TYPE_CHECKING:
 
 def generate(
     model: "BenchmarkModule",
-    datasets: list["DatasetDict"],
+    datasets: c.Sequence["DatasetDict"],
     model_config: "ModelConfig",
     dataset_config: "DatasetConfig",
     benchmark_config: "BenchmarkConfig",
-) -> list[dict[str, float]]:
+) -> c.Sequence[dict[str, float]]:
     """Evaluate a model on a dataset through generation.
 
     Args:
@@ -253,7 +254,7 @@ def generate_single_iteration(
 def debug_log(
     batch: dict[str, t.Any],
     model_output: "GenerativeModelOutput",
-    extracted_labels: list[dict | str | list[str]],
+    extracted_labels: c.Sequence[dict | str | c.Sequence[str]],
     dataset_config: "DatasetConfig",
 ) -> None:
     """Log inputs and outputs for debugging purposes.
@@ -331,7 +332,7 @@ def debug_log(
     else:
         input_texts = batch["text"]
 
-    metadata_keys: list[str] = [
+    metadata_keys: c.Sequence[str] = [
         key
         for key in batch.keys()
         if key not in ["text", "messages", "label", "labels", "target_text"]

euroeval/generation_utils.py

@@ -1,5 +1,6 @@
 """Utility functions related to generative models."""
 
+import collections.abc as c
 import itertools as it
 import json
 import logging
@@ -25,7 +26,7 @@ def extract_few_shot_examples(
     dataset_config: "DatasetConfig",
     benchmark_config: "BenchmarkConfig",
     itr_idx: int,
-) -> list[dict[str, t.Any]]:
+) -> c.Sequence[dict[str, t.Any]]:
     """Extract few-shot examples from a dataset.
 
     This will always extract the examples from the training split.
@@ -78,7 +79,7 @@ def extract_few_shot_examples(
                     lambda example: len(example["text"]) < max_num_tokens
                 )
                 num_short_examples = len(train_with_short_examples)
-                if num_short_examples >= dataset_config.num_few_shot_examples:
+                if num_short_examples >= num_few_shots:
                     break
             else:
                 raise InvalidBenchmark(
@@ -143,7 +144,7 @@ def extract_few_shot_examples(
                     lambda example: len(example["context"]) < max_num_tokens
                 )
                 num_short_examples = len(train_with_short_examples)
-                if num_short_examples >= dataset_config.num_few_shot_examples:
+                if num_short_examples >= num_few_shots:
                     break
             else:
                 raise InvalidBenchmark(
@@ -170,7 +171,7 @@ def extract_few_shot_examples(
 
 def apply_prompt(
     examples: dict[str, t.Any],
-    few_shot_examples: list[dict[str, t.Any]],
+    few_shot_examples: c.Sequence[dict[str, t.Any]],
     model_config: "ModelConfig",
     dataset_config: "DatasetConfig",
     generative_type: GenerativeType | None,
@@ -431,7 +432,7 @@ def apply_prompt(
 
 
 def raise_if_wrong_params(
-    model_config: "ModelConfig", allowed_params: dict[re.Pattern, list[str]]
+    model_config: "ModelConfig", allowed_params: dict[re.Pattern, c.Sequence[str]]
 ) -> None:
     """Raise an error if the model configuration has invalid parameters.