EuroEval 15.12.0__py3-none-any.whl → 15.13.0__py3-none-any.whl

euroeval/dataset_configs/danish.py

@@ -118,3 +118,13 @@ BELEBELE_DA_CONFIG = DatasetConfig(
     languages=[DA],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_DA_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-da",
+    pretty_name="the truncated version of the Danish part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-da-mini",
+    task=RC,
+    languages=[DA],
+    unofficial=True,
+)

euroeval/dataset_configs/dutch.py

@@ -110,3 +110,13 @@ BELEBELE_NL_CONFIG = DatasetConfig(
     languages=[NL],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_NL_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-nl",
+    pretty_name="the truncated version of the Dutch part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-nl-mini",
+    task=RC,
+    languages=[NL],
+    unofficial=True,
+)

euroeval/dataset_configs/english.py

@@ -95,3 +95,13 @@ MMLU_CONFIG = DatasetConfig(
     languages=[EN],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_EN_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-en",
+    pretty_name="the truncated version of the English part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-en-mini",
+    task=RC,
+    languages=[EN],
+    unofficial=True,
+)

euroeval/dataset_configs/faroese.py

@@ -52,3 +52,13 @@ WIKIANN_FO_CONFIG = DatasetConfig(
     languages=[FO],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_FO_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-fo",
+    pretty_name="the truncated version of the Faroese part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-fo-mini",
+    task=RC,
+    languages=[FO],
+    unofficial=True,
+)

euroeval/dataset_configs/finnish.py

@@ -68,3 +68,13 @@ BELEBELE_FI_CONFIG = DatasetConfig(
     languages=[FI],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_FI_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-fi",
+    pretty_name="the truncated version of the Finnish part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-fi-mini",
+    task=RC,
+    languages=[FI],
+    unofficial=True,
+)

euroeval/dataset_configs/french.py

@@ -81,3 +81,13 @@ BELEBELE_FR_CONFIG = DatasetConfig(
     languages=[FR],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_FR_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-fr",
+    pretty_name="the truncated version of the French part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-fr-mini",
+    task=RC,
+    languages=[FR],
+    unofficial=True,
+)

euroeval/dataset_configs/german.py

@@ -89,3 +89,13 @@ BELEBELE_DE_CONFIG = DatasetConfig(
     languages=[DE],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_DE_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-de",
+    pretty_name="the truncated version of the German part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-de-mini",
+    task=RC,
+    languages=[DE],
+    unofficial=True,
+)

euroeval/dataset_configs/icelandic.py

@@ -146,3 +146,13 @@ BELEBELE_IS_CONFIG = DatasetConfig(
     languages=[IS],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_IS_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-is",
+    pretty_name="the truncated version of the Icelandic part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-is-mini",
+    task=RC,
+    languages=[IS],
+    unofficial=True,
+)

euroeval/dataset_configs/italian.py

@@ -89,3 +89,13 @@ BELEBELE_IT_CONFIG = DatasetConfig(
     languages=[IT],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_IT_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-it",
+    pretty_name="the truncated version of the Italian part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-it-mini",
+    task=RC,
+    languages=[IT],
+    unofficial=True,
+)

euroeval/dataset_configs/norwegian.py

@@ -184,3 +184,23 @@ BELEBELE_NO_CONFIG = DatasetConfig(
     languages=[NB, NN, NO],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_NB_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-nb",
+    pretty_name="the truncated version of the Norwegian Bokmål part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-no-mini",
+    task=RC,
+    languages=[NB, NO],
+    unofficial=True,
+)
+
+MULTI_WIKI_QA_NN_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-nn",
+    pretty_name="the truncated version of the Norwegian Nynorsk part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-nn-mini",
+    task=RC,
+    languages=[NN],
+    unofficial=True,
+)

euroeval/dataset_configs/portuguese.py

@@ -2,7 +2,7 @@
 
 from ..data_models import DatasetConfig
 from ..languages import PT
-from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, SENT, SUMM
+from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
 
 ### Official datasets ###
 
@@ -16,27 +16,6 @@ SST2_PT_CONFIG = DatasetConfig(
     _labels=["positive", "negative"],
 )
 
-
-MMLU_PT_CONFIG = DatasetConfig(
-    name="mmlu-pt",
-    pretty_name="the truncated version of the Portuguese knowledge dataset MMLU-pt, "
-    "translated from the English MMLU dataset",
-    huggingface_id="EuroEval/mmlu-pt-mini",
-    task=KNOW,
-    languages=[PT],
-)
-
-
-GOLDENSWAG_PT_CONFIG = DatasetConfig(
-    name="goldenswag-pt",
-    pretty_name="the truncated version of the Portuguese common-sense reasoning "
-    "dataset GoldenSwag-pt, translated from the English GoldenSwag dataset",
-    huggingface_id="EuroEval/goldenswag-pt-mini",
-    task=COMMON_SENSE,
-    languages=[PT],
-)
-
-
 SCALA_PT = DatasetConfig(
     name="scala-pt",
     pretty_name="the Portuguese part of the linguistic acceptability dataset ScaLA",
@@ -53,6 +32,15 @@ HAREM_CONFIG = DatasetConfig(
     languages=[PT],
 )
 
+MULTI_WIKI_QA_PT_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-pt",
+    pretty_name="the truncated version of the Portuguese part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-pt-pt-mini",
+    task=RC,
+    languages=[PT],
+)
+
 PUBLICO_CONFIG = DatasetConfig(
     name="publico",
     pretty_name="the truncated version of the Portuguese summarisation dataset Público",
@@ -61,6 +49,24 @@ PUBLICO_CONFIG = DatasetConfig(
     languages=[PT],
 )
 
+MMLU_PT_CONFIG = DatasetConfig(
+    name="mmlu-pt",
+    pretty_name="the truncated version of the Portuguese knowledge dataset MMLU-pt, "
+    "translated from the English MMLU dataset",
+    huggingface_id="EuroEval/mmlu-pt-mini",
+    task=KNOW,
+    languages=[PT],
+)
+
+GOLDENSWAG_PT_CONFIG = DatasetConfig(
+    name="goldenswag-pt",
+    pretty_name="the truncated version of the Portuguese common-sense reasoning "
+    "dataset GoldenSwag-pt, translated from the English GoldenSwag dataset",
+    huggingface_id="EuroEval/goldenswag-pt-mini",
+    task=COMMON_SENSE,
+    languages=[PT],
+)
+
 
 ### Unofficial datasets ###
 
@@ -71,4 +77,5 @@ BOOLQ_PT_CONFIG = DatasetConfig(
     huggingface_id="EuroEval/boolq-pt",
     task=MCRC,
     languages=[PT],
+    unofficial=True,
 )

euroeval/dataset_configs/spanish.py

@@ -87,3 +87,13 @@ BELEBELE_ES_CONFIG = DatasetConfig(
     languages=[ES],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_ES_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-es",
+    pretty_name="the truncated version of the Spanish part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-es-mini",
+    task=RC,
+    languages=[ES],
+    unofficial=True,
+)

euroeval/dataset_configs/swedish.py

@@ -98,3 +98,13 @@ BELEBELE_SV_CONFIG = DatasetConfig(
     languages=[SV],
     unofficial=True,
 )
+
+MULTI_WIKI_QA_SV_CONFIG = DatasetConfig(
+    name="multi-wiki-qa-sv",
+    pretty_name="the truncated version of the Swedish part of the reading "
+    "comprehension dataset MultiWikiQA",
+    huggingface_id="EuroEval/multi-wiki-qa-sv-mini",
+    task=RC,
+    languages=[SV],
+    unofficial=True,
+)

euroeval/prompt_templates/reading_comprehension.py

@@ -1,7 +1,7 @@
 """Templates for the Reading Comprehension task."""
 
 from ..data_models import PromptConfig
-from ..languages import DA, DE, EN, ES, FI, FO, FR, IS, IT, NB, NL, NN, NO, SV
+from ..languages import DA, DE, EN, ES, FI, FO, FR, IS, IT, NB, NL, NN, NO, PT, SV
 
 RC_TEMPLATES = {
     DA: PromptConfig(
@@ -117,6 +117,15 @@ RC_TEMPLATES = {
         "teksten ovenfor med maks 3 ord.\n\nSpørsmål: {question}",
         default_prompt_label_mapping=dict(),
     ),
+    PT: PromptConfig(
+        default_prompt_prefix="Os textos que se seguem são acompanhados de perguntas "
+        "e respostas.",
+        default_prompt_template="Texto: {text}\nPergunta: {question}\nResposta com "
+        "um máximo de 3 palavras: {label}",
+        default_instruction_prompt="Texto: {text}\n\nResponde à seguinte pergunta "
+        "sobre o texto acima num máximo de 3 palavras.\n\nPergunta: {question}",
+        default_prompt_label_mapping=dict(),
+    ),
     SV: PromptConfig(
         default_prompt_prefix="Nedan följer texter med tillhörande frågor och svar.",
         default_prompt_template="Text: {text}\nFråga: {question}\nSvar på max 3 ord: "

{euroeval-15.12.0.dist-info → euroeval-15.13.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.12.0
+Version: 15.13.0
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
@@ -29,7 +29,7 @@ License: MIT License
         SOFTWARE.
 License-File: LICENSE
 Requires-Python: <4.0,>=3.10
-Requires-Dist: accelerate>=0.34.2
+Requires-Dist: accelerate>=1.9.0
 Requires-Dist: bert-score>=0.3.13
 Requires-Dist: click>=8.1.3
 Requires-Dist: datasets>=3.5.0

{euroeval-15.12.0.dist-info → euroeval-15.13.0.dist-info}/RECORD

@@ -30,24 +30,24 @@ euroeval/benchmark_modules/hf.py,sha256=-W_bWEdm0zePkn4nDz4l0T4hhJJnlfwHrtIO3m5B
 euroeval/benchmark_modules/litellm.py,sha256=_gKBbJsXzo_cHJVaeuQpHRBENEZUGS_vcC-uGIhhmHA,52111
 euroeval/benchmark_modules/vllm.py,sha256=kq3PMUuRT0NOky6XSHl1JeHTDGehwcub0HcGC5S_Wv4,38834
 euroeval/dataset_configs/__init__.py,sha256=EbjEyHwBtSztASl8_xblD8hessruDdV4Eg1vXrmGOuY,1935
-euroeval/dataset_configs/danish.py,sha256=MTt9EcriSer0QaFQ7_6evYxh-g9OPjroWegYdFpiKag,3395
-euroeval/dataset_configs/dutch.py,sha256=r21nxEvMmBkKqPXVW082batPsxJ9d0RB4DzngOTMJSk,3185
-euroeval/dataset_configs/english.py,sha256=1q8XJqIVWBBNkldL7t-cVnU2O9EUb9_xoVRSN8arN90,2561
-euroeval/dataset_configs/faroese.py,sha256=QQgLe5gv0f3AtXe5rV65xZ98gFgyITQPDr3UwO4Bnv4,1350
-euroeval/dataset_configs/finnish.py,sha256=_8YWIlZNpO8Qi233bH7cKwm3tq3WETLfC_6mzg7LLog,2045
-euroeval/dataset_configs/french.py,sha256=ATsj8_9_GxFTQgmfrniPQFZ1R9hoQCI1_ieWTnscFHU,2382
-euroeval/dataset_configs/german.py,sha256=QO6PrBQY6kyZeQMU1vg6KrC_sKyj9U2ukS9nbKO19is,2560
-euroeval/dataset_configs/icelandic.py,sha256=mncl7X4yO9gBmYqXMBfm7FKU1jcKryerSgd0dqlIA_4,4198
-euroeval/dataset_configs/italian.py,sha256=KNjCvTzsEqH_EEk3At8slKqNwWWiIdbv_t5ke7n9nZI,2660
-euroeval/dataset_configs/norwegian.py,sha256=30YGdDPtDszG10BNDVHb-XXTGgGIIgDUNGoeM9q0K_E,5385
-euroeval/dataset_configs/portuguese.py,sha256=-HSDsujWfK__nV2SCu-z0ne0AXLDszOT05oYphQUDTw,2063
-euroeval/dataset_configs/spanish.py,sha256=Yzm1kiilEKoHyd3xD2wrw596Ac9UcaWhlE93GlOFjlc,2558
-euroeval/dataset_configs/swedish.py,sha256=SOD2nKQTVwTpTvr362mDPHon42kr9vWs5C0mK02Fh-o,2811
+euroeval/dataset_configs/danish.py,sha256=-y-n08hTApwTdSVdjRlZYa3gOX92cTGhg8xsuG-Lhww,3691
+euroeval/dataset_configs/dutch.py,sha256=siyFeEKYx2gBpyqQPtOZ0cD8FTsIMUqzRX5xrQfrNXI,3480
+euroeval/dataset_configs/english.py,sha256=uQAaGWpHk8xqFCeIhmmPXYTb1cZomeEdRaRe9qIZQrg,2858
+euroeval/dataset_configs/faroese.py,sha256=gkgxQTWGFbfg9Eo1z-NSLROgKDcaij9tAN2mfgtrt0M,1647
+euroeval/dataset_configs/finnish.py,sha256=OyveLgyii0hOlo6HZsqAq4rwDrj8tl2qstRfQKugURo,2342
+euroeval/dataset_configs/french.py,sha256=DKKZEtohWkw_ouBaxWcPzp-K6NhQNtvCKxj8NLbIpUc,2678
+euroeval/dataset_configs/german.py,sha256=3bfRgkqIGkAhcw4kwcJN9PKuJSmi1r6AFTJY-IWKgWM,2856
+euroeval/dataset_configs/icelandic.py,sha256=g21IHjcwEZvf_yJ9PobeuBOqRiLOk0oCdEjY34g-UMk,4497
+euroeval/dataset_configs/italian.py,sha256=rHLMkSXT0kFoQlkwHODxO50WBRIfGtkAnW_C-sfIu74,2957
+euroeval/dataset_configs/norwegian.py,sha256=-WvQM44xCwjrqBzlAy4rjf6v87fGera2JmZV_069TeQ,6003
+euroeval/dataset_configs/portuguese.py,sha256=3SqbwD0PNTILGALzh50pVoEwC-spRD75ZeE2NEj151E,2367
+euroeval/dataset_configs/spanish.py,sha256=VKfBIpBRR38ckuULw7Ftmc-0smsm6GshUAik2-Y1Npw,2855
+euroeval/dataset_configs/swedish.py,sha256=WpExi4TJqy_Ruwy4Kvde94jM605vT_88el_KKUzLV4E,3108
 euroeval/prompt_templates/__init__.py,sha256=HWMZpybxs2xHPnVeJ43893conARahIVLWNXeRhXEGZw,357
 euroeval/prompt_templates/linguistic_acceptability.py,sha256=ZN71BEt4HAhSYY-GWjh-S-iVvq5AODQJThkrjDhy4oM,7138
 euroeval/prompt_templates/multiple_choice.py,sha256=F9ItGQtnaaez15A8MQ1UCpKRDsLM-AZyRdYetGAofa0,5494
 euroeval/prompt_templates/named_entity_recognition.py,sha256=ga21s9T4_Hhbf88boWm7gnL7OgD7txuS_EeDgXaxEoE,13602
-euroeval/prompt_templates/reading_comprehension.py,sha256=yLqryWQAW04GULz_EyNDLOS7ZrDUeasuLFt-dtqCnYk,6585
+euroeval/prompt_templates/reading_comprehension.py,sha256=3Nch-9zHfUDIwy-k5mP-TRhHQRQ9nad8HdhpJ1S8nGc,7072
 euroeval/prompt_templates/sentiment_classification.py,sha256=2Xsmj8lbaAXACHhwbbR4dWhoKyKB87TqpMO-ssQ-Djo,7649
 euroeval/prompt_templates/summarization.py,sha256=I98LlUOBVa_xo02npq7BWKKZOXGqm-_15i64QzbEsb0,5334
 euroeval/task_group_utils/__init__.py,sha256=CorGVkixkoEDOQuDsrOGlTmF1zmM0wnGHs8psWTfD28,72
@@ -56,8 +56,8 @@ euroeval/task_group_utils/question_answering.py,sha256=agwtWOmctgat98yqgFiMSPY6z
 euroeval/task_group_utils/sequence_classification.py,sha256=igmD24aMNN7QBJ8NDzgEnGwM-jq_zhC37QxazNm7GZ4,12711
 euroeval/task_group_utils/text_to_text.py,sha256=xOpja-W4E-1peMjZX8G-3G5iRgmFHHygrQ5WN1hB3FI,4550
 euroeval/task_group_utils/token_classification.py,sha256=wCy3aI-Sn9f-87tHzAnYDA6EbY3ah3xao1SnfnoRNz4,17490
-euroeval-15.12.0.dist-info/METADATA,sha256=8cY6HWgAZgrCkIA20lVKuf42y-e7U1MZQZSTdF3e7ig,13479
-euroeval-15.12.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-euroeval-15.12.0.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
-euroeval-15.12.0.dist-info/licenses/LICENSE,sha256=guvz_zBHgkQSY_QiUU0Bkc1k-L_PFZuLjIPfuKne2OY,1080
-euroeval-15.12.0.dist-info/RECORD,,
+euroeval-15.13.0.dist-info/METADATA,sha256=HnDtAE2-sYFmSl4yM9PQhgUrfklR_OB5C5aXPOgz5U8,13478
+euroeval-15.13.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+euroeval-15.13.0.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
+euroeval-15.13.0.dist-info/licenses/LICENSE,sha256=guvz_zBHgkQSY_QiUU0Bkc1k-L_PFZuLjIPfuKne2OY,1080
+euroeval-15.13.0.dist-info/RECORD,,