ScandEval 16.11.0__tar.gz → 16.13.0__tar.gz

{scandeval-16.11.0 → scandeval-16.13.0}/.github/ISSUE_TEMPLATE/benchmark_dataset_request.yaml

@@ -25,6 +25,7 @@ body:
     description: What languages is the dataset in?
     options:
       - label: Albanian
+      - label: Belarusian
       - label: Bulgarian
       - label: Bosnian
       - label: Catalan

{scandeval-16.11.0 → scandeval-16.13.0}/.github/ISSUE_TEMPLATE/model_evaluation_request.yaml

@@ -22,7 +22,7 @@ body:
       - label: Finnic languages (Estonian, Finnish)
       - label: Romance languages (Catalan, French, Italian, Portuguese, Romanian, Spanish)
       - label: Scandinavian languages (Danish, Faroese, Icelandic, Norwegian, Swedish)
-      - label: Slavic languages (Bulgarian, Bosnian, Croatian, Czech, Polish, Serbian, Slovak, Slovenian, Ukrainian)
+      - label: Slavic languages (Belarusian, Bulgarian, Bosnian, Croatian, Czech, Polish, Serbian, Slovak, Slovenian, Ukrainian)
       - label: West Germanic languages (Dutch, English, German)
       - label: Albanian
       - label: Greek

scandeval-16.13.0/.github/auto_assign.yaml

@@ -0,0 +1,9 @@
+addAssignees: author
+numberOfAssignees: 1
+
+addReviewers: true
+numberOfReviewers: 1
+reviewers:
+  - saattrupdan
+
+runOnDraft: true

scandeval-16.13.0/.github/workflows/auto_assign_reviewers.yaml

@@ -0,0 +1,15 @@
+name: 'Auto Assign'
+on:
+  pull_request:
+    types: [opened, ready_for_review]
+
+jobs:
+  add-reviews:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: kentaro-m/auto-assign-action@v2.0.1
+        with:
+          configuration-path: .github/auto_assign.yaml

{scandeval-16.11.0 → scandeval-16.13.0}/.github/workflows/ci.yaml

@@ -31,7 +31,7 @@ jobs:
         uses: astral-sh/setup-uv@v6
         with:
           enable-cache: false
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit hooks
         uses: pre-commit/action@v3.0.1
@@ -43,7 +43,7 @@ jobs:
       pull-requests: write
     strategy:
         matrix:
-            python-version: ["3.11", "3.12", "3.13"]
+            python-version: ["3.12", "3.13"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
@@ -58,7 +58,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Dependencies
-        run: uv sync --no-dev
+        run: uv sync --no-dev --all-extras
 
       - name: Start Ollama server
         run: curl -fsSL https://ollama.com/install.sh | sh && ollama serve &
@@ -95,7 +95,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Dependencies
-        run: uv sync --no-dev
+        run: uv sync --no-dev --all-extras
 
       - name: Start Ollama server
         run: curl -fsSL https://ollama.com/install.sh | sh && ollama serve &

{scandeval-16.11.0 → scandeval-16.13.0}/.gitignore

@@ -125,3 +125,6 @@ gfx/different-poses/*
 
 # Contracts
 generated_contracts/
+
+# Test config
+euroeval_config.py

{scandeval-16.11.0 → scandeval-16.13.0}/.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
       - id: trailing-whitespace
       - id: debug-statements
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.13
+    rev: v0.15.0
     hooks:
       - id: ruff
         args:
@@ -34,11 +34,11 @@ repos:
     hooks:
     -   id: nbstripout
 -   repo: https://github.com/facebook/pyrefly-pre-commit
-    rev: 0.49.0
+    rev: 0.51.1
     hooks:
     -   id: pyrefly-check
         name: Pyrefly (type checking)
-        pass_filenames: true
+        pass_filenames: false
 -   repo: https://github.com/DavidAnson/markdownlint-cli2
     rev: v0.20.0
     hooks:

{scandeval-16.11.0 → scandeval-16.13.0}/CHANGELOG.md

@@ -7,6 +7,77 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [v16.13.0] - 2026-02-06
+
+### Added
+
+- Added support for Belarusian 🇧🇾! This includes the sentiment classification dataset,
+  the linguistic acceptability dataset ScaLA-be, the named entity recognition dataset
+  WikiANN-be, the reading comprehension dataset MultiWikiQA-be, and the common-sense
+  reasoning dataset BE-WSC. This was added by @oliverkinch ✨
+- Added support for evaluating Hugging Face dataset repos directly, if they have a
+  `euroeval_config.py` file. We plan to allow support for a JSON/YAML config file in the
+  future, for simpler datasets.
+
+### Changed
+
+- Replaced all `DatasetConfig` arguments starting with underscores with their
+  non-underscored version (e.g., `_labels` -> `labels`), as this caused some confusion
+  when defining custom datasets. We still maintain the underscored versions for
+  backwards compatibility, but raise a warning when using them.
+- Now logs when the model inference service is temporarily unavailable, even when the
+  verbose flag is not set.
+- When evaluating local models, we now automatically add the "/v1" suffix to the API
+  base URL if not present and required by the API.
+
+### Fixed
+
+- Now allows all attention backends compatible with vLLM to be used, through the
+  `--attention-backend` CLI option. This was already possible through the Python API,
+  but was artificially restricted in the CLI.
+- When intialising a custom `Task` object, we now default the `default_labels` argument
+  to an empty list.
+
+### Deprecated
+
+- All underscored versions of `DatasetConfig` arguments are deprecated. Please use their
+  non-underscored version instead.
+
+## [v16.12.0] - 2026-02-02
+
+### Added
+
+- Added the bias detection task (`multiple-choice-stereotype-bias`) along with the Dutch
+  dataset MBBQ-NL. This was added by @caldaibis ✨
+- Added support for vLLM Metal, so that generative models can now be evaluated on Apple
+  Silicon. Note that this currently does not support structured generation, which means
+  that classification and named entity recognitions tasks unfortunately won't work yet.
+  This is due to [this xgrammar
+  issue](https://github.com/vllm-project/vllm/issues/31901).
+
+### Changed
+
+- Replaced deprecated `VLLM_ATTENTION_BACKEND` environment variable with vLLM's
+  `AttentionConfig` API. Added `--attention-backend` CLI option to configure the
+  attention backend. Defaults to FLASHINFER. This was added by @SwekeR-463 ✨
+- Now requires Python >=3.12, as Python 3.11 does not support some dependencies.
+- We now up the vLLM maximum context length for reasoning models, from 8,192 to
+  16,384, to accommodate for reasoning tokens for some datasets that have long documents.
+- We opened up the pinned vLLM version now, now set to version `>=0.14.1`.
+- Made changes to the codebase that makes it compatible with Transformers 5.0, for when
+  vLLM starts supporting it.
+
+### Fixed
+
+- Fixed an issue where a model was incorrectly classified as an encoder model if it had
+  no pipeline tag on the Hugging Face Hub and it relied on a custom implementation that
+  isn't integrated into the `transformers` library.
+- Fixed an issue when a model config had no `pad_token_id` and/or `eos_token_id`.
+- There was an error when evaluating local adapter models, which has been fixed now.
+- Now ensures that the vLLM argument `max_num_batched_tokens` is at least as large as the
+  maximum context length of the model, which gave errors with models that had a maximum
+  context length of less than 8,192.
+
 ## [v16.11.0] - 2026-01-21
 
 ### Added

{scandeval-16.11.0 → scandeval-16.13.0}/Dockerfile.cuda

@@ -3,7 +3,7 @@ FROM nvidia/cuda:12.2.0-base-ubuntu22.04
 # Install dependencies
 RUN apt-get -y update && \
     apt-get -y upgrade && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc python3.11 python3-pip python3-dev git-all && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc python3.12 python3-pip python3-dev git-all && \
     python3 -m pip install --upgrade pip wheel && \
     python3 -m pip install euroeval[all]
 

scandeval-16.13.0/PKG-INFO

@@ -0,0 +1,334 @@
+Metadata-Version: 2.4
+Name: ScandEval
+Version: 16.13.0
+Summary: The robust European language model benchmark.
+Project-URL: Repository, https://github.com/EuroEval/EuroEval
+Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues
+Author-email: Dan Saattrup Smart <dan.smart@alexandra.dk>
+Maintainer-email: Dan Saattrup Smart <dan.smart@alexandra.dk>
+License: MIT License
+        
+        Copyright (c) 2022-2026 Dan Saattrup Smart
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Requires-Python: <4.0,>=3.12
+Requires-Dist: accelerate>=1.9.0
+Requires-Dist: bert-score>=0.3.13
+Requires-Dist: click>=8.1.3
+Requires-Dist: cloudpickle>=3.1.1
+Requires-Dist: datasets>=3.5.0
+Requires-Dist: demjson3>=3.0.6
+Requires-Dist: evaluate>=0.4.1
+Requires-Dist: huggingface-hub>=0.30.1
+Requires-Dist: levenshtein>=0.24.0
+Requires-Dist: litellm>=1.75.6
+Requires-Dist: mistral-common[soundfile]
+Requires-Dist: more-itertools>=10.5.0
+Requires-Dist: numpy>=2.0.0
+Requires-Dist: ollama>=0.5.1
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: peft>=0.15.0
+Requires-Dist: protobuf>=2.0.0
+Requires-Dist: pydantic>=2.6.0
+Requires-Dist: pyinfer>=0.0.3
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: rouge-score>=0.1.2
+Requires-Dist: sacrebleu>=2.5.1
+Requires-Dist: sacremoses>=0.1.1
+Requires-Dist: scikit-learn==1.6.1
+Requires-Dist: sentencepiece>=0.1.96
+Requires-Dist: seqeval>=1.2.2
+Requires-Dist: setuptools>=75.8.2
+Requires-Dist: tenacity>=9.0.0
+Requires-Dist: termcolor>=2.0.0
+Requires-Dist: torch>=2.6.0
+Requires-Dist: transformers[mistral-common]<5.0.0,>=4.56.0
+Provides-Extra: all
+Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == 'all'
+Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'all'
+Requires-Dist: ray>=2.53.0; (platform_system == 'Linux') and extra == 'all'
+Requires-Dist: timm>=1.0.19; extra == 'all'
+Requires-Dist: vllm-metal>=0.1.0; (platform_system == 'Darwin') and extra == 'all'
+Requires-Dist: vllm==0.11.0; (platform_system == 'Darwin') and extra == 'all'
+Requires-Dist: vllm[flashinfer]>=0.14.1; (platform_system == 'Linux') and extra == 'all'
+Provides-Extra: generative
+Requires-Dist: bitsandbytes>=0.43.1; (platform_system == 'Linux') and extra == 'generative'
+Requires-Dist: fbgemm-gpu>=1.0.0; (platform_system == 'Linux') and extra == 'generative'
+Requires-Dist: ray>=2.53.0; (platform_system == 'Linux') and extra == 'generative'
+Requires-Dist: timm>=1.0.19; extra == 'generative'
+Requires-Dist: vllm-metal>=0.1.0; (platform_system == 'Darwin') and extra == 'generative'
+Requires-Dist: vllm==0.11.0; (platform_system == 'Darwin') and extra == 'generative'
+Requires-Dist: vllm[flashinfer]>=0.14.1; (platform_system == 'Linux') and extra == 'generative'
+Description-Content-Type: text/markdown
+
+<!-- This disables the requirement that the first line is a top-level heading -->
+<!-- markdownlint-configure-file { "MD041": false } -->
+
+<div align='center'>
+<img
+    src="https://raw.githubusercontent.com/EuroEval/EuroEval/main/gfx/euroeval.png"
+    height="500"
+    width="372"
+>
+</div>
+
+### The robust European language model benchmark
+
+(formerly known as ScandEval)
+
+______________________________________________________________________
+[![Documentation](https://img.shields.io/badge/docs-passing-green)](https://euroeval.com)
+[![PyPI Status](https://badge.fury.io/py/euroeval.svg)](https://pypi.org/project/euroeval/)
+[![First paper](https://img.shields.io/badge/arXiv-2304.00906-b31b1b.svg)](https://arxiv.org/abs/2304.00906)
+[![Second paper](https://img.shields.io/badge/arXiv-2406.13469-b31b1b.svg)](https://arxiv.org/abs/2406.13469)
+[![License](https://img.shields.io/github/license/EuroEval/EuroEval)](https://github.com/EuroEval/EuroEval/blob/main/LICENSE)
+[![LastCommit](https://img.shields.io/github/last-commit/EuroEval/EuroEval)](https://github.com/EuroEval/EuroEval/commits/main)
+[![Code Coverage](https://img.shields.io/badge/Coverage-73%25-yellow.svg)](https://github.com/EuroEval/EuroEval/tree/main/tests)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.0-4baaaa.svg)](https://github.com/EuroEval/EuroEval/blob/main/CODE_OF_CONDUCT.md)
+
+## Maintainer
+
+- Dan Saattrup Smart ([@saattrupdan](https://github.com/saattrupdan), <dan.smart@alexandra.dk>)
+
+## Installation and usage
+
+See the [documentation](https://euroeval.com/python-package/) for more information.
+
+## Reproducing the evaluation datasets
+
+All datasets used in this project are generated using the scripts located in the
+[src/scripts](src/scripts) folder. To reproduce a dataset, run the corresponding script
+with the following command
+
+```bash
+uv run src/scripts/<name-of-script>.py
+```
+
+Replace <name-of-script> with the specific script you wish to execute, e.g.,
+
+```bash
+uv run src/scripts/create_allocine.py
+```
+
+## Contributors :pray:
+
+A huge thank you to all the contributors who have helped make this project a success!
+
+<a href="https://github.com/peter-sk">
+    <img
+        src="https://avatars.githubusercontent.com/u/6168908"
+        width=50
+        alt="Contributor avatar for peter-sk"
+    />
+</a>
+<a href="https://github.com/AJDERS">
+    <img
+        src="https://avatars.githubusercontent.com/u/38854604"
+        width=50
+        alt="Contributor avatar for AJDERS"
+    />
+</a>
+<a href="https://github.com/oliverkinch">
+    <img
+        src="https://avatars.githubusercontent.com/u/71556498"
+        width=50
+        alt="Contributor avatar for oliverkinch"
+    />
+</a>
+<a href="https://github.com/versae">
+    <img
+        src="https://avatars.githubusercontent.com/u/173537"
+        width=50
+        alt="Contributor avatar for versae"
+    />
+</a>
+<a href="https://github.com/KennethEnevoldsen">
+    <img
+        src="https://avatars.githubusercontent.com/u/23721977"
+        width=50
+        alt="Contributor avatar for KennethEnevoldsen"
+    />
+</a>
+<a href="https://github.com/viggo-gascou">
+    <img
+        src="https://avatars.githubusercontent.com/u/94069687"
+        width=50
+        alt="Contributor avatar for viggo-gascou"
+    />
+</a>
+<a href="https://github.com/mathiasesn">
+    <img
+        src="https://avatars.githubusercontent.com/u/27091759"
+        width=50
+        alt="Contributor avatar for mathiasesn"
+    />
+</a>
+<a href="https://github.com/Alkarex">
+    <img
+        src="https://avatars.githubusercontent.com/u/1008324"
+        width=50
+        alt="Contributor avatar for Alkarex"
+    />
+</a>
+<a href="https://github.com/marksverdhei">
+    <img
+        src="https://avatars.githubusercontent.com/u/46672778"
+        width=50
+        alt="Contributor avatar for marksverdhei"
+    />
+</a>
+<a href="https://github.com/Mikeriess">
+    <img
+        src="https://avatars.githubusercontent.com/u/19728563"
+        width=50
+        alt="Contributor avatar for Mikeriess"
+    />
+</a>
+<a href="https://github.com/ThomasKluiters">
+    <img
+        src="https://avatars.githubusercontent.com/u/8137941"
+        width=50
+        alt="Contributor avatar for ThomasKluiters"
+    />
+</a>
+<a href="https://github.com/BramVanroy">
+    <img
+        src="https://avatars.githubusercontent.com/u/2779410"
+        width=50
+        alt="Contributor avatar for BramVanroy"
+    />
+</a>
+<a href="https://github.com/peregilk">
+    <img
+        src="https://avatars.githubusercontent.com/u/9079808"
+        width=50
+        alt="Contributor avatar for peregilk"
+    />
+</a>
+<a href="https://github.com/Rijgersberg">
+    <img
+        src="https://avatars.githubusercontent.com/u/8604946"
+        width=50
+        alt="Contributor avatar for Rijgersberg"
+    />
+</a>
+<a href="https://github.com/duarteocarmo">
+    <img
+        src="https://avatars.githubusercontent.com/u/26342344"
+        width=50
+        alt="Contributor avatar for duarteocarmo"
+    />
+</a>
+<a href="https://github.com/slowwavesleep">
+    <img
+        src="https://avatars.githubusercontent.com/u/44175589"
+        width=50
+        alt="Contributor avatar for slowwavesleep"
+    />
+</a>
+<a href="https://github.com/mrkowalski">
+    <img
+        src="https://avatars.githubusercontent.com/u/6357044"
+        width=50
+        alt="Contributor avatar for mrkowalski"
+    />
+</a>
+<a href="https://github.com/simonevanbruggen">
+    <img
+        src="https://avatars.githubusercontent.com/u/24842609"
+        width=50
+        alt="Contributor avatar for simonevanbruggen"
+    />
+</a>
+<a href="https://github.com/tvosch">
+    <img
+        src="https://avatars.githubusercontent.com/u/110661769"
+        width=50
+        alt="Contributor avatar for tvosch"
+    />
+</a>
+<a href="https://github.com/Touzen">
+    <img
+        src="https://avatars.githubusercontent.com/u/1416265"
+        width=50
+        alt="Contributor avatar for Touzen"
+    />
+</a>
+<a href="https://github.com/caldaibis">
+    <img
+        src="https://avatars.githubusercontent.com/u/16032437"
+        width=50
+        alt="Contributor avatar for caldaibis"
+    />
+</a>
+<a href="https://github.com/SwekeR-463">
+    <img
+        src="https://avatars.githubusercontent.com/u/114919896?v=4"
+        width=50
+        alt="Contributor avatar for SwekeR-463"
+    />
+</a>
+
+### Contribute to EuroEval
+
+We welcome contributions to EuroEval! Whether you're fixing bugs, adding features, or
+contributing new datasets, your help makes this project better for everyone.
+
+- **General contributions**: Check out our [contribution guidelines](CONTRIBUTING.md)
+  for information on how to get started.
+- **Adding datasets**: If you're interested in adding a new dataset to EuroEval, we have
+  a [dedicated guide](NEW_DATASET_GUIDE.md) with step-by-step instructions.
+
+### Special thanks
+
+- Thanks to [Google](https://google.com/) for sponsoring Gemini credits as part of their
+  [Google Cloud for Researchers Program](https://cloud.google.com/edu/researchers).
+- Thanks [@Mikeriess](https://github.com/Mikeriess) for evaluating many of the larger
+  models on the leaderboards.
+- Thanks to [OpenAI](https://openai.com/) for sponsoring OpenAI credits as part of their
+  [Researcher Access Program](https://openai.com/form/researcher-access-program/).
+- Thanks to [UWV](https://www.uwv.nl/) and [KU
+  Leuven](https://www.arts.kuleuven.be/ling/ccl) for sponsoring the Azure OpenAI
+  credits used to evaluate GPT-4-turbo in Dutch.
+- Thanks to [Miðeind](https://mideind.is/en) for sponsoring the OpenAI
+  credits used to evaluate GPT-4-turbo in Icelandic and Faroese.
+- Thanks to [CHC](https://chc.au.dk/) for sponsoring the OpenAI credits used to
+  evaluate GPT-4-turbo in German.
+
+## Citing EuroEval
+
+If you want to cite the framework then feel free to use this:
+
+```bibtex
+@article{smart2024encoder,
+  title={Encoder vs Decoder: Comparative Analysis of Encoder and Decoder Language Models on Multilingual NLU Tasks},
+  author={Smart, Dan Saattrup and Enevoldsen, Kenneth and Schneider-Kamp, Peter},
+  journal={arXiv preprint arXiv:2406.13469},
+  year={2024}
+}
+@inproceedings{smart2023scandeval,
+  author = {Smart, Dan Saattrup},
+  booktitle = {Proceedings of the 24th Nordic Conference on Computational Linguistics (NoDaLiDa)},
+  month = may,
+  pages = {185--201},
+  title = {{ScandEval: A Benchmark for Scandinavian Natural Language Processing}},
+  year = {2023}
+}
+```