EuroEval 15.8.0__py3-none-any.whl → 15.8.1__py3-none-any.whl

euroeval/data_models.py

@@ -529,12 +529,16 @@ class DatasetConfig:
         else:
             sep_word = main_language.or_separator
 
+        local_labels: list[str] = []
+        for label in self.labels:
+            if label not in self.prompt_label_mapping:
+                continue
+            local_label = self.prompt_label_mapping[label]
+            if local_label not in local_labels:
+                local_labels.append(local_label)
+
         # Convert labels to single-quoted labels - and remove duplicates
-        quoted_labels = [
-            f"'{self.prompt_label_mapping[label]}'"
-            for label in set(self.labels)
-            if label in self.prompt_label_mapping
-        ]
+        quoted_labels = [f"'{label}'" for label in local_labels]
 
         if not quoted_labels:
             return ""

euroeval/task_group_utils/sequence_classification.py

@@ -144,9 +144,27 @@ def extract_labels_from_generation(
         )
         if labels is not None:
             return labels
-    return get_closest_word_edit_labels(
-        generated_sequences=model_output.sequences, dataset_config=dataset_config
-    )
+
+    candidate_labels = [
+        dataset_config.prompt_label_mapping[lbl]
+        for lbl in dataset_config.id2label.values()
+    ]
+    new_predicted_labels: list[str] = list()
+    for predicted_label in model_output.sequences:
+        # If the prediction includes a boxed answer, use that instead of the full
+        # generation
+        if (m := re.search(r"boxed\{(.*?)\}", predicted_label)) is not None:
+            predicted_label = m.group(1)
+
+        # Pick the label with the smallest word edit distance to the predicted label
+        edit_distances = [
+            Levenshtein.distance(s1=predicted_label.lower(), s2=candidate_label.lower())
+            for candidate_label in candidate_labels
+        ]
+        predicted_label = candidate_labels[np.argmin(edit_distances).item()]
+        new_predicted_labels.append(predicted_label)
+
+    return new_predicted_labels
 
 
 def get_closest_logprobs_labels(
@@ -305,32 +323,3 @@ def get_closest_logprobs_labels(
 
     assert len(output_labels) == len(generation_logprobs)
     return output_labels
-
-
-def get_closest_word_edit_labels(
-    generated_sequences: list[str], dataset_config: "DatasetConfig"
-) -> list[str]:
-    """Get the labels with the smallest edit distance to the predicted labels.
-
-    Args:
-        generated_sequences:
-            The generated sequences from the model.
-        dataset_config:
-            The configuration of the dataset.
-
-    Returns:
-        The candidate labels with the smallest edit distance to the predicted labels.
-    """
-    candidate_labels = [
-        dataset_config.prompt_label_mapping[lbl]
-        for lbl in dataset_config.id2label.values()
-    ]
-    new_predicted_labels: list[str] = list()
-    for predicted_label in generated_sequences:
-        edit_distances = [
-            Levenshtein.distance(s1=predicted_label.lower(), s2=candidate_label.lower())
-            for candidate_label in candidate_labels
-        ]
-        closest_label = candidate_labels[np.argmin(edit_distances).item()]
-        new_predicted_labels.append(closest_label)
-    return new_predicted_labels

{euroeval-15.8.0.dist-info → euroeval-15.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.8.0
+Version: 15.8.1
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues

{euroeval-15.8.0.dist-info → euroeval-15.8.1.dist-info}/RECORD

@@ -5,7 +5,7 @@ euroeval/callbacks.py,sha256=F1AJCLB8FJpxqYprwLi_PsH4Bc0x4lyR8UiTG-GlFLY,2452
 euroeval/cli.py,sha256=EMB6g6kRvxIqlfYLSoMzwLAtEd-fqXipo4A_HTkhjkA,8575
 euroeval/constants.py,sha256=p6kp_R6-Tq5LBvyXyT6Sa6N3SkjEElGS2LSZRBoQaYs,1985
 euroeval/data_loading.py,sha256=L_REtxefte5Ke4xE_Cz01zkfCyKlOYhSqT5ZXXulHPc,3992
-euroeval/data_models.py,sha256=t5FwpGxiSIMe7iKae-tT7usUWki-ILzAFFm7dPJoFsk,22973
+euroeval/data_models.py,sha256=59ca6gxmHwMdeIIU6f-gGXOVIXXDQPOt7m5nCXHK86E,23166
 euroeval/enums.py,sha256=L9LcNeruuhHvze9vKRogXY9vonRzoBqDzWSP6hxKQ7A,3195
 euroeval/exceptions.py,sha256=LRd7HoudupRp5-AX3L0X4hIAWCa6JVx-LViHPg7u7dg,5821
 euroeval/finetuning.py,sha256=uuaUxNQJb7TivPQuI1OYQ_MIKbD-6-7mpkobLKsDefQ,10667
@@ -51,11 +51,11 @@ euroeval/prompt_templates/summarization.py,sha256=mcWeKNhGWmp7IG_iY64T-VOSabQg5w
 euroeval/task_group_utils/__init__.py,sha256=CorGVkixkoEDOQuDsrOGlTmF1zmM0wnGHs8psWTfD28,72
 euroeval/task_group_utils/multiple_choice_classification.py,sha256=nB78TzOgd0HBvTclmjOYJid9ZVAgu8IHZsqB_n1SAZU,6178
 euroeval/task_group_utils/question_answering.py,sha256=kZBABJ_WYNTH4Xgo2jIvfx7iYvfoGt0EUObSaXRCGmk,27700
-euroeval/task_group_utils/sequence_classification.py,sha256=MCdO5h3v_LWTkrvKAeefPq7rl1H5mFed50nAL4uZq0E,13837
+euroeval/task_group_utils/sequence_classification.py,sha256=Yqx0pUhuHYmSkv1ZUfOndSLTvpr0lWCk19oYITfSjV4,13555
 euroeval/task_group_utils/text_to_text.py,sha256=Nu1_qRPLbboCd9Q5rxqY4fQFJ_aGXu80aWQqoTG1cYc,5047
 euroeval/task_group_utils/token_classification.py,sha256=3idWB81Fcx9UhTuk-gxMfXENrCBmiWBDUWdULXoIhpw,17863
-euroeval-15.8.0.dist-info/METADATA,sha256=-GcGBuEnlAPmpT9ItDAmS0psT__jwbVoNkTYOiSeRzA,13669
-euroeval-15.8.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-euroeval-15.8.0.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
-euroeval-15.8.0.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
-euroeval-15.8.0.dist-info/RECORD,,
+euroeval-15.8.1.dist-info/METADATA,sha256=Fdzj20PR6wWZUx_7f_bhPh8S4DF6ghZwMIDrJ4ozxFE,13669
+euroeval-15.8.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+euroeval-15.8.1.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
+euroeval-15.8.1.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
+euroeval-15.8.1.dist-info/RECORD,,