EuroEval 15.2.0__tar.gz

euroeval-15.2.0/.github/ISSUE_TEMPLATE/benchmark_dataset_request.yaml

@@ -0,0 +1,44 @@
+name: 📚 Benchmark Dataset Request
+description: Do you think a particular benchmark dataset is missing in ScandEval?
+title: "[BENCHMARK DATASET REQUEST] <dataset-name>"
+labels: "benchmark dataset request"
+
+body:
+- type: input
+  attributes:
+    label: Dataset name
+    description: What is the name of the dataset?
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: Dataset link
+    description: Please give a link to where the dataset is hosted (doesn't have to be on the Hugging Face Hub)
+  validations:
+    required: true
+- type: checkboxes
+  attributes:
+    label: Dataset languages
+    description: What languages is the dataset in?
+    options:
+      - label: Danish
+      - label: Dutch
+      - label: English
+      - label: Faroese
+      - label: French
+      - label: German
+      - label: Icelandic
+      - label: Norwegian (Bokmål or Nynorsk)
+      - label: Swedish
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Describe the dataset
+    description: Describe what the dataset is measuring, and why you think it is important to include it as a benchmark dataset in ScandEval.
+  validations:
+    required: true
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

euroeval-15.2.0/.github/ISSUE_TEMPLATE/bug.yaml

@@ -0,0 +1,63 @@
+name: 🐛 Bug Report
+description: Have you experienced a bug using the `scandeval` package?
+title: "[BUG] <name-of-bug>"
+labels: bug
+
+body:
+- type: markdown
+  attributes:
+    value: >
+      #### Before submitting a bug, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/Scandeval/ScandEval/issues?q=is%3Aissue).
+- type: textarea
+  attributes:
+    label: 🐛 Describe the bug
+    description: |
+      Please provide a clear and concise description of what the bug is. If relevant, add a minimal example so that we can reproduce the error by running the code.
+  validations:
+    required: true
+- type: dropdown
+  attributes:
+    label: Operating System
+    description: What operating system are you on?
+    options:
+      - Linux
+      - MacOS
+      - Windows
+      - Other
+  validations:
+    required: true
+- type: dropdown
+  attributes:
+    label: Device
+    description: What hardware device do you use?
+    options:
+      - CUDA GPU
+      - AMD GPU
+      - Mac GPU
+      - CPU
+      - Other
+  validations:
+    required: true
+- type: dropdown
+  attributes:
+    label: Python version
+    description: What Python version are you using?
+    options:
+      - 3.10.x
+      - 3.11.x
+      - 3.12.x
+      - Older than 3.10.x
+      - Newer than 3.12.x
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: ScandEval version
+    description: What version of ScandEval are you using?
+    placeholder: Output of `pip list | grep ScandEval`
+  validations:
+    required: true
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

euroeval-15.2.0/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,27 @@
+name: 🚀 Feature Request
+description: Is the ScandEval benchmark missing a feature?
+title: "[FEATURE REQUEST] <name-of-feature>"
+labels: enhancement
+
+body:
+- type: textarea
+  attributes:
+    label: 🚀 The feature, motivation and pitch
+    description: >
+      A clear and concise description of the feature proposal. Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., *"I'm working on X and would like Y to be possible"*.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Alternatives
+    description: >
+      A description of any alternative solutions or features you've considered, if any.
+- type: textarea
+  attributes:
+    label: Additional context
+    description: >
+      Add any other context or screenshots about the feature request.
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

euroeval-15.2.0/.github/ISSUE_TEMPLATE/model_evaluation_request.yaml

@@ -0,0 +1,53 @@
+name: 📊 Model Evaluation Request
+description: Would you like to have a particular model included in the leaderboards?
+title: "[MODEL EVALUATION REQUEST] <model-name>"
+labels: "model evaluation request"
+
+body:
+- type: input
+  attributes:
+    label: Model ID
+    description: What is the Hugging Face model ID?
+  validations:
+    required: true
+- type: dropdown
+  attributes:
+    label: Model type
+    description: What is the architecture of the model?
+    options:
+      - Decoder model (e.g., GPT)
+      - Encoder model (e.g., BERT)
+      - Sequence-to-sequence model (e.g., T5)
+  validations:
+    required: true
+- type: checkboxes
+  attributes:
+    label: Evaluation languages
+    description: >
+      What languages should this model be evaluated on? Tick all that apply. If the
+      model is multilingual (e.g., Mistral, Llama), then tick all the languages.
+    options:
+      - label: Danish
+      - label: Dutch
+      - label: English
+      - label: Faroese
+      - label: French
+      - label: German
+      - label: Icelandic
+      - label: Norwegian (Bokmål or Nynorsk)
+      - label: Swedish
+  validations:
+    required: true
+- type: dropdown
+  attributes:
+    label: Merged model
+    description: Is the model a merge of other models, or built on top of a merged model?
+    options:
+      - Not a merged model
+      - Merged model
+  validations:
+    required: true
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

euroeval-15.2.0/.github/workflows/ci.yaml

@@ -0,0 +1,90 @@
+name: CI
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+    branches:
+      - main
+
+jobs:
+  code-check:
+    if: github.event.pull_request.draft == false
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: pre-commit/action@v3.0.1
+
+  pytest-linux:
+    if: github.event.pull_request.draft == false
+    permissions:
+      contents: read
+      pull-requests: write
+    strategy:
+        matrix:
+            python-version: ["3.10", "3.11", "3.12"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Install uv and set up Python
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Dependencies
+        run: uv sync --no-dev --extra test
+
+      - name: Test with pytest
+        run: uv run pytest
+        env:
+          HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }}
+          HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+
+      - name: Delete ScandEval cache
+        run: rm -rf .scandeval_cache
+
+  pytest-macos:
+    if: github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'macos')
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv and set up Python
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Dependencies
+        run: uv sync --no-dev --extra test
+
+      - name: Test with pytest
+        run: uv run pytest
+        env:
+          HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }}
+          HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+
+      - name: Delete ScandEval cache
+        run: rm -rf .scandeval_cache

euroeval-15.2.0/.gitignore

@@ -0,0 +1,117 @@
+# Byte-compiled / optimized / DLL files
+**/__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+.venv
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+
+# Translations
+*.mo
+*.pot
+
+# Log files
+*.log
+
+# PyBuilder
+target/
+
+# DotEnv configuration
+.env
+
+# Database
+*.db
+*.rdb
+
+# Pycharm
+.idea
+
+# VS Code
+.vscode/
+
+# Spyder
+.spyproject/
+
+# Jupyter NB Checkpoints
+.ipynb_checkpoints/
+
+# Mac OS-specific storage files
+.DS_Store
+
+# vim
+*.swp
+*.swo
+
+# Mypy cache
+.mypy_cache/
+
+# pytest cache
+.pytest_cache/
+
+# Ruff cache
+.ruff_cache/
+
+# Checkpoints
+checkpoint-*
+
+# Cache
+.scandeval_cache/
+.euroeval_cache/
+
+# Result log
+scandeval_benchmark_results.jsonl
+euroeval_benchmark_results.jsonl
+
+# Experiments
+gai-experiments.xlsx
+~$gai-experiments.xlsx
+
+# Upload of leaderboard
+src/upload/
+
+# Model outputs in debugging mode
+*.json
+
+# Mkdocs
+site/
+
+# Helper files for docs
+docs/datasets/dataset_example_commands.txt

euroeval-15.2.0/.pre-commit-config.yaml

@@ -0,0 +1,41 @@
+repos:
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+      - id: python-use-type-annotations
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: debug-statements
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.8
+    hooks:
+      - id: ruff
+        args:
+          - --fix
+          - --exit-non-zero-on-fix
+        types_or:
+          - python
+          - pyi
+          - jupyter
+      - id: ruff-format
+        types_or:
+          - python
+          - pyi
+          - jupyter
+-   repo: https://github.com/kynan/nbstripout
+    rev: 0.8.1
+    hooks:
+    -   id: nbstripout
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.15.0
+    hooks:
+    -   id: mypy
+        args:
+          - --install-types
+          - --non-interactive
+          - --ignore-missing-imports
+          - --show-error-codes
+          - --check-untyped-defs