EuroEval 15.6.0__py3-none-any.whl → 15.6.1__py3-none-any.whl

euroeval/data_models.py

@@ -388,8 +388,10 @@ class DatasetConfig:
             language.
         _prompt_label_mapping (optional):
             A mapping from the labels to another phrase which is used as a substitute
-            for the label in few-shot evaluation. Defaults to the template for the task
-            and language.
+            for the label in few-shot evaluation. If "auto" then the mapping will be set
+            to a 1:1 mapping between the labels and themselves. If None then the mapping
+            will be set to the default mapping for the task and language. Defaults to
+            None.
         unofficial (optional):
             Whether the dataset is unofficial. Defaults to False.
     """
@@ -405,7 +407,7 @@ class DatasetConfig:
     _num_few_shot_examples: int | None = None
     _max_generated_tokens: int | None = None
     _labels: list[str] | None = None
-    _prompt_label_mapping: dict[str, str] | None = None
+    _prompt_label_mapping: dict[str, str] | t.Literal["auto"] | None = None
     unofficial: bool = False
 
     @property
@@ -475,7 +477,9 @@ class DatasetConfig:
     @property
     def prompt_label_mapping(self) -> dict[str, str]:
         """Mapping from English labels to localised labels."""
-        if self._prompt_label_mapping is not None:
+        if self._prompt_label_mapping == "auto":
+            return {label: label for label in self.labels}
+        elif self._prompt_label_mapping is not None:
             return self._prompt_label_mapping
 
         main_language = self.languages[0]

euroeval/dataset_configs/faroese.py

@@ -12,6 +12,7 @@ FOSENT_CONFIG = DatasetConfig(
     huggingface_id="EuroEval/fosent",
     task=SENT,
     languages=[FO],
+    _num_few_shot_examples=5,
 )
 
 SCALA_FO_CONFIG = DatasetConfig(

euroeval/dataset_configs/norwegian.py

@@ -83,6 +83,7 @@ NOR_COMMON_SENSE_QA_CONFIG = DatasetConfig(
     huggingface_id="EuroEval/nor-common-sense-qa",
     task=COMMON_SENSE,
     languages=[NB, NN, NO],
+    _labels=["a", "b", "c", "d", "e"],
 )
 
 
@@ -105,7 +106,6 @@ NORGLM_MULTI_QA = DatasetConfig(
     huggingface_id="EuroEval/norglm-multi-qa",
     task=RC,
     languages=[NB, NN, NO],
-    _num_few_shot_examples=2,
     unofficial=True,
 )
 

{euroeval-15.6.0.dist-info → euroeval-15.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.6.0
+Version: 15.6.1
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
@@ -237,6 +237,7 @@ A huge thank you to all the contributors who have helped make this project a suc
 <a href="https://github.com/ThomasKluiters"><img src="https://avatars.githubusercontent.com/u/8137941" width=50 alt="Contributor avatar for ThomasKluiters"/></a>
 <a href="https://github.com/BramVanroy"><img src="https://avatars.githubusercontent.com/u/2779410" width=50 alt="Contributor avatar for BramVanroy"/></a>
 <a href="https://github.com/peregilk"><img src="https://avatars.githubusercontent.com/u/9079808" width=50 alt="Contributor avatar for peregilk"/></a>
+<a href="https://github.com/Rijgersberg"><img src="https://avatars.githubusercontent.com/u/8604946" width=50 alt="Contributor avatar for Rijgersberg"/></a>
 
 ### Special Thanks
 - Thanks to [Google](https://google.com/) for sponsoring Gemini credits as part of their

{euroeval-15.6.0.dist-info → euroeval-15.6.1.dist-info}/RECORD

@@ -5,7 +5,7 @@ euroeval/callbacks.py,sha256=F1AJCLB8FJpxqYprwLi_PsH4Bc0x4lyR8UiTG-GlFLY,2452
 euroeval/cli.py,sha256=EMB6g6kRvxIqlfYLSoMzwLAtEd-fqXipo4A_HTkhjkA,8575
 euroeval/constants.py,sha256=t2mAT8tE3Dn2lXWHTnaFoaOIaUcdiBjJTASCt7nSdkg,1984
 euroeval/data_loading.py,sha256=7xXdoFSvEDzpw1FNR8E8YV4c9Vy86hlU5-qLm9RUejE,3318
-euroeval/data_models.py,sha256=RjU7REmUMxSMeZfTeUNYb6XRlHMUri7Tk_zwexUOupU,22840
+euroeval/data_models.py,sha256=oZLrGg1dhIIwbgtEzq4U_fu_ZbBsz35mrqsyizuZNPw,23138
 euroeval/enums.py,sha256=L9LcNeruuhHvze9vKRogXY9vonRzoBqDzWSP6hxKQ7A,3195
 euroeval/exceptions.py,sha256=LRd7HoudupRp5-AX3L0X4hIAWCa6JVx-LViHPg7u7dg,5821
 euroeval/finetuning.py,sha256=IieAhgvxjeLHAHBief1Ay-STcCosQmrDHFTRTXFZX0Q,10743
@@ -31,12 +31,12 @@ euroeval/dataset_configs/__init__.py,sha256=fkD1hzW7szJLc1MdK-AY4EBFWBUX5Z8t4f9u
 euroeval/dataset_configs/danish.py,sha256=MTt9EcriSer0QaFQ7_6evYxh-g9OPjroWegYdFpiKag,3395
 euroeval/dataset_configs/dutch.py,sha256=N3zL0vGe4OyPgVU_AiYNNfk96jSc_JDtKrVIHbaEYCU,3536
 euroeval/dataset_configs/english.py,sha256=yHw7D0zSNVbiSBAjR1mWX4V5FSkhqy4y-o-pnyWCLxE,2323
-euroeval/dataset_configs/faroese.py,sha256=9QYFtd3GqaFcyQjsmru_yvJuTjParyz8Ra_ekw_3xbA,1320
+euroeval/dataset_configs/faroese.py,sha256=QQgLe5gv0f3AtXe5rV65xZ98gFgyITQPDr3UwO4Bnv4,1350
 euroeval/dataset_configs/french.py,sha256=ATsj8_9_GxFTQgmfrniPQFZ1R9hoQCI1_ieWTnscFHU,2382
 euroeval/dataset_configs/german.py,sha256=QO6PrBQY6kyZeQMU1vg6KrC_sKyj9U2ukS9nbKO19is,2560
 euroeval/dataset_configs/icelandic.py,sha256=mncl7X4yO9gBmYqXMBfm7FKU1jcKryerSgd0dqlIA_4,4198
 euroeval/dataset_configs/italian.py,sha256=5yYMMBbxkfSDpLgJ9IH_pgkpzEp-74vMMvx-dT8x4WY,2345
-euroeval/dataset_configs/norwegian.py,sha256=3kKhri5qWIiFwNSzNFHjNbRpkW1NSK_PUltAGQpxmAY,5172
+euroeval/dataset_configs/norwegian.py,sha256=2SD5681gZFa1Ig-AEpnyStbivan_bq_Pada4qwE7tw0,5181
 euroeval/dataset_configs/spanish.py,sha256=fc0dHWU7-g_p6kaSGA8nD1vLVQF_yqR2PkixrYyWywc,2212
 euroeval/dataset_configs/swedish.py,sha256=SOD2nKQTVwTpTvr362mDPHon42kr9vWs5C0mK02Fh-o,2811
 euroeval/prompt_templates/__init__.py,sha256=HWMZpybxs2xHPnVeJ43893conARahIVLWNXeRhXEGZw,357
@@ -52,8 +52,8 @@ euroeval/task_group_utils/question_answering.py,sha256=kZBABJ_WYNTH4Xgo2jIvfx7iY
 euroeval/task_group_utils/sequence_classification.py,sha256=gqd0-l5o7vAY5QIpGSkSqwJwez3Y0r5SqOiywfPNW8A,12239
 euroeval/task_group_utils/text_to_text.py,sha256=QECnGdZ0YLjsbMc6LwXqVi4KMuITdiOjmJUNQtAAOW0,5712
 euroeval/task_group_utils/token_classification.py,sha256=3idWB81Fcx9UhTuk-gxMfXENrCBmiWBDUWdULXoIhpw,17863
-euroeval-15.6.0.dist-info/METADATA,sha256=m1NE2zaj_hbP-3kW-2_oC9Ug-POilMU1fVWQTt-SNIU,13027
-euroeval-15.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-euroeval-15.6.0.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
-euroeval-15.6.0.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
-euroeval-15.6.0.dist-info/RECORD,,
+euroeval-15.6.1.dist-info/METADATA,sha256=4i98IBxn6yWh4ugBW-SnljmDfKEXBSfRGjZyf_dlOUs,13183
+euroeval-15.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+euroeval-15.6.1.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
+euroeval-15.6.1.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
+euroeval-15.6.1.dist-info/RECORD,,