EuroEval 15.4.1__tar.gz → 15.5.0__tar.gz

{euroeval-15.4.1 → euroeval-15.5.0}/.github/ISSUE_TEMPLATE/benchmark_dataset_request.yaml

@@ -2,6 +2,7 @@ name: 📚 Benchmark Dataset Request
 description: Do you think a particular benchmark dataset is missing in EuroEval?
 title: "[BENCHMARK DATASET REQUEST] <dataset-name>"
 labels: "benchmark dataset request"
+type: task
 
 body:
 - type: input
@@ -30,6 +31,7 @@ body:
       - label: Icelandic
       - label: Italian
       - label: Norwegian (Bokmål or Nynorsk)
+      - label: Spanish
       - label: Swedish
   validations:
     required: true

{euroeval-15.4.1 → euroeval-15.5.0}/.github/ISSUE_TEMPLATE/bug.yaml

@@ -1,7 +1,7 @@
 name: 🐛 Bug Report
 description: Have you experienced a bug using the `euroeval` package?
 title: "[BUG] <name-of-bug>"
-labels: bug
+type: bug
 
 body:
 - type: markdown
@@ -46,8 +46,9 @@ body:
       - 3.10.x
       - 3.11.x
       - 3.12.x
+      - 3.13.x
       - Older than 3.10.x
-      - Newer than 3.12.x
+      - Newer than 3.13.x
   validations:
     required: true
 - type: input
@@ -57,6 +58,20 @@ body:
     placeholder: Output of `pip list | grep EuroEval`
   validations:
     required: true
+- type: input
+  attributes:
+    label: Transformers version
+    description: What version of 🤗 transformers are you using?
+    placeholder: Output of `pip list | grep transformers`
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: vLLM version
+    description: What version of vLLM are you using?
+    placeholder: Output of `pip list | grep vllm`
+  validations:
+    required: true
 - type: markdown
   attributes:
     value: >

{euroeval-15.4.1 → euroeval-15.5.0}/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,7 +1,7 @@
 name: 🚀 Feature Request
 description: Is the EuroEval benchmark missing a feature?
 title: "[FEATURE REQUEST] <name-of-feature>"
-labels: enhancement
+type: feature
 
 body:
 - type: textarea
@@ -11,16 +11,6 @@ body:
       A clear and concise description of the feature proposal. Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., *"I'm working on X and would like Y to be possible"*.
   validations:
     required: true
-- type: textarea
-  attributes:
-    label: Alternatives
-    description: >
-      A description of any alternative solutions or features you've considered, if any.
-- type: textarea
-  attributes:
-    label: Additional context
-    description: >
-      Add any other context or screenshots about the feature request.
 - type: markdown
   attributes:
     value: >

{euroeval-15.4.1 → euroeval-15.5.0}/.github/ISSUE_TEMPLATE/model_evaluation_request.yaml

@@ -2,12 +2,25 @@ name: 📊 Model Evaluation Request
 description: Would you like to have a particular model included in the leaderboards?
 title: "[MODEL EVALUATION REQUEST] <model-name>"
 labels: "model evaluation request"
+type: task
 
 body:
 - type: input
   attributes:
     label: Model ID
-    description: What is the Hugging Face model ID?
+    description: What is the model ID, either on the Hugging Face Hub or on LiteLLM?
+  validations:
+    required: true
+- type: checkboxes
+  attributes:
+    label: Evaluation languages
+    description: >
+      What languages should this model be evaluated on? Tick all that apply. If the
+      model is multilingual (e.g., Mistral, Llama), then tick all the languages.
+    options:
+      - label: Romance languages (French, Italian, Spanish)
+      - label: Scandinavian languages (Danish, Faroese, Icelandic, Norwegian, Swedish)
+      - label: West Germanic languages (Dutch, English, German)
   validations:
     required: true
 - type: dropdown
@@ -20,23 +33,14 @@ body:
       - Sequence-to-sequence model (e.g., T5)
   validations:
     required: true
-- type: checkboxes
+- type: dropdown
   attributes:
-    label: Evaluation languages
-    description: >
-      What languages should this model be evaluated on? Tick all that apply. If the
-      model is multilingual (e.g., Mistral, Llama), then tick all the languages.
+    label: Model size
+    description: What is the size of the model?
     options:
-      - label: Danish
-      - label: Dutch
-      - label: English
-      - label: Faroese
-      - label: French
-      - label: German
-      - label: Icelandic
-      - label: Italian
-      - label: Norwegian (Bokmål or Nynorsk)
-      - label: Swedish
+      - Small (<=8B parameters)
+      - Large (>8B parameters)
+      - N/A
   validations:
     required: true
 - type: dropdown
@@ -46,6 +50,7 @@ body:
     options:
       - Not a merged model
       - Merged model
+      - N/A
   validations:
     required: true
 - type: markdown

{euroeval-15.4.1 → euroeval-15.5.0}/.github/workflows/ci.yaml

@@ -43,7 +43,6 @@ jobs:
       - name: Install uv and set up Python
         uses: astral-sh/setup-uv@v4
         with:
-          enable-cache: true
           python-version: ${{ matrix.python-version }}
 
       - name: Install Dependencies
@@ -75,7 +74,6 @@ jobs:
       - name: Install uv and set up Python
         uses: astral-sh/setup-uv@v4
         with:
-          enable-cache: true
           python-version: ${{ matrix.python-version }}
 
       - name: Install Dependencies
@@ -91,6 +89,8 @@ jobs:
           HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
 
       - name: Delete EuroEval cache
         run: rm -rf .euroeval_cache

{euroeval-15.4.1 → euroeval-15.5.0}/.gitignore

@@ -115,3 +115,7 @@ site/
 
 # Helper files for docs
 docs/datasets/dataset_example_commands.txt
+
+# Various graphics
+gfx/euroeval-italian.png
+gfx/euroeval-italian.xcf

{euroeval-15.4.1 → euroeval-15.5.0}/.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
       - id: trailing-whitespace
       - id: debug-statements
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.2
+    rev: v0.11.4
     hooks:
       - id: ruff
         args:

{euroeval-15.4.1 → euroeval-15.5.0}/CHANGELOG.md

@@ -10,6 +10,91 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 
 
+## [v15.5.0] - 2025-04-07
+### Added
+- Now allows supplying a parameter to API models, which is done by using
+  `<model-id>@<parameter>` as the model ID (only a single parameter is supported). The
+  parameters allowed are "low" and "high" for OpenAI models (which is the reasoning
+  effort of the model, supported by the o1- and o3-series, default is "medium"), and
+  "thinking" for Anthropic models, to enable thinking mode (supported for
+  Claude-Sonnet-3.7+). These will appear in the leaderboards as
+  `<model-id>@<parameter>`.
+- Added metadata for Google Gemini and xAI Grok models.
+- Allows all vLLM versions from v0.8.0 again, as the issue with the generation output
+  has been resolved.
+- Added overall progress indicator during evaluation. This was contributed by
+  [@mathiasesn](https://github.com/mathiasesn) ✨
+
+### Changed
+- Now does not use logprobs in text classification tasks with Google VertexAI models, as
+  they heavily rate limit logprobs usage. This shouldn't affect the scores significantly
+  in any case, as the models are very confident in their predictions.
+- Updated `litellm` to `>=1.63.0`, allowing better support for reasoning models.
+
+### Fixed
+- The Gemini-2.5-pro model uses different error messages than the other Gemini models,
+  which caused an error when evaluating it. This has been fixed now.
+- Now registers the Gemini-2.5-pro model series as reasoning models, as otherwise they
+  did not generate any text as they were just generating reasoning tokens.
+- Previously, if there were multiple labels whose first tokens were identical and that
+  the (generative) model did not output the label as the first output token, we would
+  randomly choose one of the labels, resulting in an evaluation error. This is very
+  rare, but *does* happen for very particular (model, dataset) pairs. If we are in this
+  case, we now resort to choosing the label with closest word edit distance instead of
+  relying on logprobs of the first token.
+- Now defaults to BF16 if the model is registered as using FP32, assuming that BF16 is
+  supported by the GPU.
+- Improved model existence pipeline for Ollama model IDs with multiple forward slashes
+  in the name, which caused some models to not be detected as existing.
+
+
+## [v15.4.2] - 2025-03-31
+### Added
+- Now added version metadata to results, to easier track which versions of the various
+  dependencies were used when evaluating a model. This currently includes
+  `transformers`, `torch`, `vllm` and `outlines`.
+
+### Changed
+- Changed the name of the German 'mlsum' summarisation dataset to 'mlsum-de', to reflect
+  that it is the German version of the dataset, and to avoid confusion with the Spanish
+  'mlsum-es' dataset.
+
+### Fixed
+- Now uses `fp16` instead of `bf16` when evaluating decoder models on GPUs with CUDA
+  compatibility < 8.0. This was contributed by
+  [@marksverdhei](https://github.com/marksverdhei) ✨
+- Corrected the name of the French sentiment dataset AlloCiné. This was contributed by
+  [@Alkarex](https://github.com/Alkarex) ✨
+- Evaluating a specific model revision did not work for adapter models, as there was a
+  confusion between the revision of the adapter and the revision of the base model. We
+  now use the revision for the adapter and use the latest revision for the base model.
+- In the (very unlikely) scenario that the model's tokeniser has the same first token
+  for two different labels in a text classification task, we now also use the second
+  token to ensure that we determine the correct label. If this is not possible, then we
+  warn the user.
+- Now catches `TypeError` when trying to generate with vLLM, and retries 3 times before
+  giving up on evaluating the dataset.
+- A bug in `transformers` caused models with the `image-text-to-text` pipeline tag to
+  not be detected as generative models. This has been patched now, and will be fixed
+  properly when [this transformers
+  PR](https://github.com/huggingface/transformers/pull/37107) has been merged.
+- Force `vllm` v0.8.0 for now, as the severe degradation in generation output of some
+  models has not been resolved in versions v0.8.2 and v0.8.3.
+- Only accepts the local labels for text classification tasks when evaluating decoder
+  models now, where we before accepted both the local and English labels. The reason is
+  that this caused a confusion mat times when there was a unique local label starting
+  with a particular letter, but a different English label starting with the same letter,
+  causing some models to be evaluated on the wrong label.
+- When fetching the model information from the Hugging Face API we now attempt 3 times,
+  as the API sometimes fails. If it still fails after 3 attempts, we raise the
+  `HuggingFaceHubDown` exception.
+- Now uses `fp16` instead of `bf16` when evaluating decoder models on GPUs with CUDA
+  compatibility < 8.0. This was contributed by
+  [@marksverdhei](https://github.com/marksverdhei) ✨
+- Fixed docs for ScandiQA-da and ScandiQA-sv, where it was incorrectly stated that
+  the splits were made by considering the original train/validation/test splits.
+
+
 ## [v15.4.1] - 2025-03-25
 ### Fixed
 - Disallow `vllm` v0.8.1, as it causes severe degradation in generation output of
@@ -73,18 +158,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [v15.3.0] - 2025-03-12
 ### Added
 - Added support for evaluating Italian 🇮🇹! This includes the reading comprehension
-  dataset [SQuAD-it](https://hf.co/datasets/crux82/squad_it), the summarization
-  dataset [IlPost](https://hf.co/datasets/ARTeLab/ilpost), the sentiment
-  classification
-  [Sentipolc-16](https://hf.co/datasets/cardiffnlp/tweet_sentiment_multilingual),
-  the common-sense reasoning dataset
-  [HellaSwag-it](https://hf.co/datasets/alexandrainst/m_hellaswag), the linguistic acceptability
-  dataset ScaLA with the [Italian Universal Dependencies
+  dataset [SQuAD-it](https://hf.co/datasets/crux82/squad_it), the summarization dataset
+  [IlPost](https://hf.co/datasets/ARTeLab/ilpost), the sentiment classification
+  [Sentipolc-16](https://hf.co/datasets/cardiffnlp/tweet_sentiment_multilingual), the
+  common-sense reasoning dataset
+  [HellaSwag-it](https://hf.co/datasets/alexandrainst/m_hellaswag), the linguistic
+  acceptability dataset ScaLA with the [Italian Universal Dependencies
   treebank](https://github.com/UniversalDependencies/UD_Italian-ISDT), the knowledge
   dataset [MMLU-it](https://hf.co/datasets/alexandrainst/m_mmlu), and the named entity
-  recognition dataset [MultiNERD
-  IT](https://hf.co/datasets/Babelscape/multinerd) (and unofficially
-  [WikiNEuRal IT](https://hf.co/datasets/Babelscape/wikineural)). This was contributed by [@viggo-gascou](https://github.com/viggo-gascou) ✨
+  recognition dataset [MultiNERD IT](https://hf.co/datasets/Babelscape/multinerd) (and
+  unofficially [WikiNEuRal IT](https://hf.co/datasets/Babelscape/wikineural)). This was
+  contributed by [@viggo-gascou](https://github.com/viggo-gascou) ✨
 - Added the new Norwegian knowledge dataset NRK-Quiz-QA, consisting of quizzes on the
   Norwegian language and culture, in both Bokmål and Nynorsk. The dataset has been split
   into 635 / 256 / 2,048 samples for train, val, and test, respectively. This replaces
@@ -211,7 +295,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 - Added support for French! 🇫🇷This includes the sentiment classification dataset
-  [Allocine](https://hf.co/datasets/tblard/allocine), the linguistic acceptability
+  [AlloCiné](https://hf.co/datasets/tblard/allocine), the linguistic acceptability
   dataset ScaLA with the [French Universal
   Dependencies](https://github.com/UniversalDependencies/UD_French-GSD), the reading
   comprehension dataset [FQuAD](https://hf.co/datasets/illuin/fquad) (and unofficially

{euroeval-15.4.1 → euroeval-15.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.4.1
+Version: 15.5.0
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
@@ -37,11 +37,12 @@ Requires-Dist: demjson3>=3.0.6
 Requires-Dist: evaluate>=0.4.1
 Requires-Dist: huggingface-hub>=0.24.0
 Requires-Dist: levenshtein>=0.24.0
-Requires-Dist: litellm>=1.61.13
+Requires-Dist: litellm>=1.63.0
 Requires-Dist: more-itertools>=10.5.0
 Requires-Dist: numpy<2.0.0,>=1.23.0
 Requires-Dist: ollama>=0.4.7
 Requires-Dist: pandas>=2.2.0
+Requires-Dist: peft>=0.15.0
 Requires-Dist: protobuf~=3.20.0
 Requires-Dist: pydantic>=2.6.0
 Requires-Dist: pyinfer>=0.0.3
@@ -61,12 +62,12 @@ Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == '
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'all'
 Requires-Dist: gradio>=4.26.0; extra == 'all'
 Requires-Dist: outlines>=0.1.11; extra == 'all'
-Requires-Dist: vllm!=0.8.1,>=0.8.0; (platform_system == 'Linux') and extra == 'all'
+Requires-Dist: vllm>=0.8.0; (platform_system == 'Linux') and extra == 'all'
 Provides-Extra: generative
 Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'generative'
 Requires-Dist: outlines>=0.1.11; extra == 'generative'
-Requires-Dist: vllm!=0.8.1,>=0.8.0; (platform_system == 'Linux') and extra == 'generative'
+Requires-Dist: vllm>=0.8.0; (platform_system == 'Linux') and extra == 'generative'
 Provides-Extra: human-evaluation
 Requires-Dist: gradio>=4.26.0; extra == 'human-evaluation'
 Provides-Extra: test
@@ -217,6 +218,7 @@ Replace <name-of-script> with the specific script you wish to execute, e.g.,
 $ uv run src/scripts/create_allocine.py
 ```
 
+
 ## Special Thanks :pray:
 - Thanks [@Mikeriess](https://github.com/Mikeriess) for evaluating many of the larger
   models on the leaderboards.

{euroeval-15.4.1 → euroeval-15.5.0}/README.md

@@ -142,6 +142,7 @@ Replace <name-of-script> with the specific script you wish to execute, e.g.,
 $ uv run src/scripts/create_allocine.py
 ```
 
+
 ## Special Thanks :pray:
 - Thanks [@Mikeriess](https://github.com/Mikeriess) for evaluating many of the larger
   models on the leaderboards.

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/danish.md

@@ -285,11 +285,10 @@ the translated contexts still contained the answer to the question, potentially
 changing the answers slightly.
 
 The original full dataset consists of 6,810 / 500 / 500 samples for training,
-validation and testing, respectively. We use a 1,024 / 256 / 2,048 split for training,
-validation and testing, respectively (so 3,328 samples used in total). All validation
-samples in our version also belong to the original validation set, and all original test
-samples are included in our test set. The remaining 1,548 test samples in our version
-was sampled from the original training set.
+validation and testing, respectively (so 3,328 samples used in total).
+We use a 1,024 / 256 / 2,048 split for training, validation and testing, respectively,
+where the splits are made by randomly sampling from the full dataset without considering
+the original train/validation/test splits.
 
 Here are a few examples from the training split:
 
@@ -451,12 +450,14 @@ Here are a few examples from the training split:
 {
   "text": "Hvilket af følgende områder har kommunerne ansvaret for driften af?\nSvarmuligheder:\na. Domstole\nb. Vuggestuer\nc. Sygehuse",
   "label": "b"
-}```
+}
+```
 ```json
 {
   "text": "Hvilken organisation blev Danmark medlem af i 1945?\nSvarmuligheder:\na. Verdenshandelsorganisationen (WTO)\nb. Den Europæiske Union (EU)\nc. De Forenede Nationer (FN)",
   "label": "c"
-}```
+}
+```
 
 When evaluating generative models, we use the following setup (see the
 [methodology](/methodology) for more information on how these are used):

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/dutch.md

@@ -133,7 +133,7 @@ $ euroeval --model <model-id> --dataset dbrd
 
 ## Named Entity Recognition
 
-### CoNLL-2002-nl
+### CoNLL-nl
 
 This dataset was published in [this paper](https://aclanthology.org/W02-2024/) and
 consists of named entity recognition annotations of the Belgian newspaper "De Morgen" of

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/english.md

@@ -81,7 +81,7 @@ $ euroeval --model <model-id> --dataset sst5
 
 ## Named Entity Recognition
 
-### CoNLL-2003-En
+### CoNLL-en
 
 This dataset was published in [this paper](https://aclanthology.org/W03-0419/) and was
 part of the CoNNL-2003 shared task. The data comes from the [Reuters

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/faroese.md

@@ -282,10 +282,10 @@ $ euroeval --model <model-id> --dataset scala-fo
 
 ### FoQA
 
-This dataset will be published in an upcoming paper and is based on the Faroese
-Wikipedia. The questions and answers were automatically generated using GPT-4-turbo,
-which were verified by a native speaker, and some of them were also corrected by the
-same native speaker.
+This dataset was published in [this paper](https://doi.org/10.48550/arXiv.2502.07642)
+and is based on the Faroese Wikipedia. The questions and answers were automatically
+generated using GPT-4-turbo, which were verified by a native speaker, and some of them
+were also corrected by the same native speaker.
 
 The original full dataset consists of 2,000 samples, and we split these into 848 / 128 /
 1,024 samples for training, validation and testing, respectively.

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/french.md

@@ -7,11 +7,11 @@ information about what these constitute.
 
 ## Sentiment Classification
 
-### Allocine
+### AlloCiné
 
 This dataset was published in [this Github
 repository](https://github.com/TheophileBlard/french-sentiment-analysis-with-bert) and
-features reviews from the French movie review website Allocine. The reviews range from
+features reviews from the French movie review website [AlloCiné](https://www.allocine.fr/). The reviews range from
 0.5 to 5 (inclusive), with steps of 0.5. The negative samples are reviews with a rating
 of at most 2, and the positive ones are reviews with a rating of at least 4. The reviews
 in between were discarded.

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/icelandic.md

@@ -9,9 +9,9 @@ information about what these constitute.
 
 ### Hotter and Colder Sentiment
 
-This dataset is being published in an upcoming paper, and consists of texts from
-Icelandic blog post, annotated with sentiment labels (and many others) via a
-crowdsourcing platform.
+This dataset was published in [this paper](https://doi.org/10.48550/arXiv.2502.16987),
+and consists of texts from Icelandic blog post, annotated with sentiment labels (and
+many others) via a crowdsourcing platform.
 
 The original full dataset consists of 2,901 samples, and we use a 1,021 / 255 / 1,607
 split for training, validation and testing, respectively (so all samples are used in
@@ -73,13 +73,14 @@ $ euroeval --model <model-id> --dataset hotter-and-colder-sentiment
 
 ### MIM-GOLD-NER
 
-This dataset was published in [this paper]() and is based on the [Tagged Icelandic
-Corpus (MIM)](https://clarin.is/en/resources/mim/), which consists of Icelandic books,
-news articles, periodicals, parliament speeches, legal texts, adjudications and
-government websites. It has been annotated with named entities in a semi-automated
-fashion, where each labels has been manually verified. The entity types in the dataset
-is a superset of the CoNLL-2003 tags, with the following additional labels: `DATE`,
-`TIME`, `MONEY`, `PERCENT`. These labels have been removed.
+This dataset was published in [this
+paper](https://repository.clarin.is/repository/xmlui/handle/20.500.12537/230) and is
+based on the [Tagged Icelandic Corpus (MIM)](https://clarin.is/en/resources/mim/), which
+consists of Icelandic books, news articles, periodicals, parliament speeches, legal
+texts, adjudications and government websites. It has been annotated with named entities
+in a semi-automated fashion, where each labels has been manually verified. The entity
+types in the dataset is a superset of the CoNLL-2003 tags, with the following additional
+labels: `DATE`, `TIME`, `MONEY`, `PERCENT`. These labels have been removed.
 
 The original full dataset consists of 1,000,000 tokens. We use a 1,024 / 256 / 2,048
 split for training, validation and testing, respectively.
@@ -526,17 +527,20 @@ Here are a few examples from the training split:
 {
   "text": "Hver var talinn heilagur maður eftir dauða sinn, er tákngervingur alþýðuhreyfingar vestanlands og talinn góður til áheita?\nSvarmöguleikar:\na. Þórður Jónsson helgi\nb. Guðmundur Arason\nc. Snorri Þorgrímsson\nd. Jón Hreggviðsson",
   "label": "a"
-}```
+}
+```
 ```json
 {
   "text": "Í kringum hvaða ár hófst verslun á Arngerðareyri?\nSvarmöguleikar:\na. 1895\nb. 1884\nc. 1870\nd. 1902",
   "label": "b"
-}```
+}
+```
 ```json
 {
   "text": "Hvenær var ákveðið að uppstigningardagur skyldi vera kirkjudagur aldraðra á Íslandi?\nSvarmöguleikar:\na. Árið 1975\nb. Árið 1985\nc. Árið 1982\nd. Árið 1990",
   "label": "c"
-}```
+}
+```
 
 When evaluating generative models, we use the following setup (see the
 [methodology](/methodology) for more information on how these are used):

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/italian.md

@@ -71,11 +71,10 @@ $ euroeval --model <model-id> --dataset sentipolc16
 ### MultiNERD IT
 
 This dataset was published in [this
-paper](https://aclanthology.org/2022.findings-naacl.60/) and
-consists of sentences from Wikipedia and Wikinews in 10 different languages. It is an
-extension of the combination of
-(WikiNEuRal)[https://www.github.com/Babelscape/wikineural] and
-(NER4EL)[https://www.github.com/Babelscape/ner4el]. The original test set was created
+paper](https://aclanthology.org/2022.findings-naacl.60/) and consists of sentences from
+Wikipedia and Wikinews in 10 different languages. It is an extension of the combination
+of [WikiNEuRal](https://www.github.com/Babelscape/wikineural) and
+[NER4EL](https://www.github.com/Babelscape/ner4el). The original test set was created
 from manual annotations, while the training set is based on an automatic annotation
 pipeline.
 
@@ -519,7 +518,7 @@ $ euroeval --model <model-id> --dataset hellaswag-it
 
 ## Summarization
 
-### IlPost-sum
+### IlPost-Sum
 
 This dataset was published in [this paper](https://www.mdpi.com/2078-2489/13/5/228) and
 consists of news articles from [Il Post](https://www.ilpost.it/). The summaries were

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/norwegian.md

@@ -388,17 +388,20 @@ Here are a few examples from the training split:
 {
   "text": "Vi har hatt krig i nesten ti år. Jeg føler meg noen ganger trist fordi jeg har mistet flere venner og min far på grunn av krigen.",
   "label": "correct"
-}```
+}
+```
 ```json
 {
   "text": "Hvis jeg ikke sier in n genting, kan han spille hele dagen.",
   "label": "incorrect"
-}```
+}
+```
 ```json
 {
   "text": "De føler at samfunnet trenger ikke dem.",
   "label": "incorrect"
-}```
+}
+```
 
 When evaluating generative models, we use the following setup (see the
 [methodology](/methodology) for more information on how these are used):
@@ -660,17 +663,20 @@ Here are a few examples from the training split:
 {
   "text": "Gunnar har hatt plutselige og sterke smerteanfall siden han var liten gutt. Det var vondt å tisse og det gjorde vondt i ryggen og magen. Det hjalp litt å drikke vann. Reseptbelagte medisiner kan være nødvendig under anfall.\nSvaralternativer:\na. Nyrestein, kronisk\nb. Irritabel tarmsyndrom\nc. Angst\nd. Urinveisinfeksjon",
   "label": "a"
-}```
+}
+```
 ```json
 {
   "text": "80 år gamle Harrison Ford er nok ein gong aktuell i rolla som Indiana Jones. Kva heiter filmen?\nSvaralternativer:\na. Indiana Jones and the Nasty Nazis\nb. Indiana Jones and the Dial of Destiny\nc. Indiana Jones and the Hunt for Power\nd. Indiana Jones Forever",
   "label": "b"
-}```
+}
+```
 ```json
 {
   "text": "I 1980 måtte denne bassisten overnatte ni netter i fengsel i Japan fordi han prøvde å få med seg ca. 200 gram marihuana inn i landet. Hvem var det?\nSvaralternativer:\na. Sting\nb. Lemmy Kilmister\nc. Paul McCartney\nd. Bootsy Collins",
   "label": "c"
-}```
+}
+```
 
 When evaluating generative models, we use the following setup (see the
 [methodology](/methodology) for more information on how these are used):
@@ -868,17 +874,20 @@ Here are a few examples from the training split:
 {
   "text": "Hvor er det sannsynlig at en fugl lager hjemmet sitt?\nSvaralternativer:\na. I skogen\nb. I et rede\nc. På taket\nd. På blader\ne. I himmelen",
   "label": "a"
-}```
+}
+```
 ```json
 {
   "text": "Hvis et hjem har et abonnoment, hva får de sannsyneligvis hver dag i posten?\nSvaralternativer:\na. Delestykker\nb. En avis\nc. En gate\nd. En vaskemaskin\ne. Jordas overflate",
   "label": "b"
-}```
+}
+```
 ```json
 {
   "text": "Når du ikke klarer å gjøre noe ferdig, hva feilet du i da?\nSvaralternativer:\na. Å vinne\nb. Å bestå\nc. Å fullfør\nd. Å gjøre det bra\ne. Å lykkes",
   "label": "c"
-}```
+}
+```
 
 When evaluating generative models, we use the following setup (see the
 [methodology](/methodology) for more information on how these are used):

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/spanish.md

@@ -475,7 +475,7 @@ $ euroeval --model <model-id> --dataset hellaswag-es
 
 ## Summarization
 
-### MLSum-es-mini
+### MLSum-es
 
 The dataset was published in [this paper](https://aclanthology.org/2020.emnlp-main.647/) and is obtained from online newspapers.
 

{euroeval-15.4.1 → euroeval-15.5.0}/docs/datasets/swedish.md

@@ -231,11 +231,10 @@ the translated contexts still contained the answer to the question, potentially
 changing the answers slightly.
 
 The original full dataset consists of 6,810 / 500 / 500 samples for training,
-validation and testing, respectively. We use a 1,024 / 256 / 2,048 split for training,
-validation and testing, respectively (so 3,328 samples used in total). All validation
-samples in our version also belong to the original validation set, and all original test
-samples are included in our test set. The remaining 1,548 test samples in our version
-was sampled from the original training set.
+validation and testing, respectively (so 3,328 samples used in total).
+We use a 1,024 / 256 / 2,048 split for training, validation and testing, respectively,
+where the splits are made by randomly sampling from the full dataset without considering
+the original train/validation/test splits.
 
 Here are a few examples from the training split: