mteb 2.0.4__py3-none-any.whl → 2.1.0__py3-none-any.whl

mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py

@@ -0,0 +1,88 @@
+from mteb.abstasks.multilabel_classification import (
+    AbsTaskMultilabelClassification,
+)
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class CovidDisinformationNLMultiLabelClassification(AbsTaskMultilabelClassification):
+    metadata = TaskMetadata(
+        name="CovidDisinformationNLMultiLabelClassification",
+        dataset={
+            "path": "clips/mteb-nl-COVID-19-disinformation",
+            "revision": "7ad922bdef875db1f530847c6ffff05fc154f2e8",
+        },
+        description="The dataset is curated to address questions of interest to journalists, fact-checkers, "
+        "social media platforms, policymakers, and the general public.",
+        reference="https://aclanthology.org/2021.findings-emnlp.56.pdf",
+        type="MultilabelClassification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2020-01-01", "2021-04-01"),
+        domains=["Web", "Social", "Written"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{alam-etal-2021-fighting-covid,
+  address = {Punta Cana, Dominican Republic},
+  author = {Alam, Firoj  and
+Shaar, Shaden  and
+Dalvi, Fahim  and
+Sajjad, Hassan  and
+Nikolov, Alex  and
+Mubarak, Hamdy  and
+Da San Martino, Giovanni  and
+Abdelali, Ahmed  and
+Durrani, Nadir  and
+Darwish, Kareem  and
+Al-Homaid, Abdulaziz  and
+Zaghouani, Wajdi  and
+Caselli, Tommaso  and
+Danoe, Gijs  and
+Stolk, Friso  and
+Bruntink, Britt  and
+Nakov, Preslav},
+  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2021},
+  doi = {10.18653/v1/2021.findings-emnlp.56},
+  editor = {Moens, Marie-Francine  and
+Huang, Xuanjing  and
+Specia, Lucia  and
+Yih, Scott Wen-tau},
+  month = nov,
+  pages = {611--649},
+  publisher = {Association for Computational Linguistics},
+  title = {Fighting the {COVID}-19 Infodemic: Modeling the Perspective of Journalists, Fact-Checkers, Social Media Platforms, Policy Makers, and the Society},
+  url = {https://aclanthology.org/2021.findings-emnlp.56/},
+  year = {2021},
+}
+""",
+    )
+
+    def dataset_transform(self) -> None:
+        labels = [
+            "q2_label",
+            "q3_label",
+            "q4_label",
+            "q5_label",
+            "q6_label",
+            "q7_label",
+        ]
+        _dataset = {}
+
+        def map_labels(example):
+            ml_labels = []
+            for i, label in enumerate(labels):
+                if example[label] == "yes":
+                    ml_labels.append(i)
+            return {"label": ml_labels}
+
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].filter(
+                lambda ex: ex["q1_label"] == "yes"
+            )
+            self.dataset[split] = self.dataset[split].map(map_labels)

mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py

@@ -0,0 +1,44 @@
+from mteb.abstasks.multilabel_classification import (
+    AbsTaskMultilabelClassification,
+)
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VABBMultiLabelClassification(AbsTaskMultilabelClassification):
+    samples_per_label = 128
+    metadata = TaskMetadata(
+        name="VABBMultiLabelClassification",
+        dataset={
+            "path": "clips/mteb-nl-vabb-mlcls-pr",
+            "revision": "584c70f5104671772119f21e9f8a3c912ac07d4a",
+        },
+        description="This dataset contains the fourteenth edition of the Flemish Academic Bibliography for the Social "
+        "Sciences and Humanities (VABB-SHW), a database of academic publications from the social sciences "
+        "and humanities authored by researchers affiliated to Flemish universities (more information). "
+        "Publications in the database are used as one of the parameters of the Flemish performance-based "
+        "research funding system",
+        reference="https://zenodo.org/records/14214806",
+        type="MultilabelClassification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2020-01-01", "2021-04-01"),
+        domains=["Academic", "Written"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@dataset{aspeslagh2024vabb,
+  author = {Aspeslagh, Pieter and Guns, Raf and Engels, Tim C. E.},
+  doi = {10.5281/zenodo.14214806},
+  publisher = {Zenodo},
+  title = {VABB-SHW: Dataset of Flemish Academic Bibliography for the Social Sciences and Humanities (edition 14)},
+  url = {https://doi.org/10.5281/zenodo.14214806},
+  year = {2024},
+}
+""",
+    )

mteb/tasks/pair_classification/__init__.py

@@ -9,6 +9,7 @@ from .ind import *
 from .ita import *
 from .kor import *
 from .multilingual import *
+from .nld import *
 from .pol import *
 from .por import *
 from .rus import *

mteb/tasks/pair_classification/nld/__init__.py

@@ -0,0 +1,7 @@
+from .sick_nl_pair_classification import SICKNLPairClassification
+from .xlwic_nl_pair_classification import XLWICNLPairClassification
+
+__all__ = [
+    "SICKNLPairClassification",
+    "XLWICNLPairClassification",
+]

mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py

@@ -0,0 +1,36 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class SICKNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="SICKNLPairClassification",
+        dataset={
+            "path": "clips/mteb-nl-sick-pcls-pr",
+            "revision": "a13a1892bcb4c077dc416d390389223eea5f20f0",
+        },
+        description="SICK-NL is a Dutch translation of SICK ",
+        reference="https://aclanthology.org/2021.eacl-main.126/",
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        date=("2020-09-01", "2021-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=[],
+        license="mit",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation=r"""
+@inproceedings{wijnholds2021sick,
+  author = {Wijnholds, Gijs and Moortgat, Michael},
+  booktitle = {Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume},
+  pages = {1474--1479},
+  title = {SICK-NL: A Dataset for Dutch Natural Language Inference},
+  year = {2021},
+}
+""",
+    )

mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class XLWICNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="XLWICNLPairClassification",
+        description="The Word-in-Context dataset (WiC) addresses the dependence on sense inventories by reformulating "
+        "the standard disambiguation task as a binary classification problem; but, it is limited to the "
+        "English language. We put forward a large multilingual benchmark, XL-WiC, featuring gold standards "
+        "in 12 new languages from varied language families and with different degrees of resource "
+        "availability, opening room for evaluation scenarios such as zero-shot cross-lingual transfer. ",
+        reference="https://aclanthology.org/2020.emnlp-main.584.pdf",
+        dataset={
+            "path": "clips/mteb-nl-xlwic",
+            "revision": "0b33ce358b1b5d500ff3715ba3d777b4d2c21cb0",
+        },
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        date=("2019-10-04", "2019-10-04"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        domains=["Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="created",
+        bibtex_citation=r"""
+@inproceedings{raganato2020xl,
+  author = {Raganato, A and Pasini, T and Camacho-Collados, J and Pilehvar, M and others},
+  booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
+  organization = {Association for Computational Linguistics (ACL)},
+  pages = {7193--7206},
+  title = {XL-WiC: A multilingual benchmark for evaluating semantic contextualization},
+  year = {2020},
+}
+""",
+    )

mteb/tasks/retrieval/multilingual/vdr_multilingual_retrieval.py

@@ -101,7 +101,7 @@ class VDRMultilingualRetrieval(AbsTaskRetrieval):
             "revision": "9e26ae152f5950ab1a5ff1c58edade3acc894793",
         },
         type="Retrieval",
-        category="it2it",
+        category="t2i",
         modalities=["text", "image"],
         eval_splits=[_EVAL_SPLIT],
         eval_langs=_LANGS,

mteb/tasks/retrieval/nld/__init__.py

@@ -1,4 +1,5 @@
 from .argu_ana_nl_retrieval import ArguAnaNL
+from .bbsard_nl_retrieval import BBSARDNLRetrieval
 from .climate_fevernl_retrieval import ClimateFEVERNL
 from .cqa_dupstack_android_nl_retrieval import CQADupstackAndroidNLRetrieval
 from .cqa_dupstack_english_nl_retrieval import CQADupstackEnglishNLRetrieval
@@ -13,17 +14,21 @@ from .cqa_dupstack_unix_nl_retrieval import CQADupstackUnixNLRetrieval
 from .cqa_dupstack_webmasters_nl_retrieval import CQADupstackWebmastersNLRetrieval
 from .cqa_dupstack_wordpress_nl_retrieval import CQADupstackWordpressNLRetrieval
 from .db_pedia_nl_retrieval import DBPediaNL
+from .dutch_news_articles_retrieval import DutchNewsArticlesRetrieval
 from .fevernl_retrieval import FEVERNL
 from .fi_qa2018_nl_retrieval import FiQA2018NL
 from .hotpot_qanl_retrieval import HotpotQANL
+from .legal_qa_nl_retrieval import LegalQANLRetrieval
 from .mmarconl_retrieval import MMMARCONL
 from .nf_corpus_nl_retrieval import NFCorpusNL
 from .nqnl_retrieval import NQNL
+from .open_tender_retrieval import OpenTenderRetrieval
 from .quora_nl_retrieval import QuoraNLRetrieval
 from .sci_fact_nl_retrieval import SciFactNL
 from .scidocsnl_retrieval import SCIDOCSNL
 from .touche2020_nl_retrieval import Touche2020NL
 from .treccovidnl_retrieval import TRECCOVIDNL
+from .vabb_retrieval import VABBRetrieval
 
 __all__ = [
     "FEVERNL",
@@ -32,6 +37,7 @@ __all__ = [
     "SCIDOCSNL",
     "TRECCOVIDNL",
     "ArguAnaNL",
+    "BBSARDNLRetrieval",
     "CQADupstackAndroidNLRetrieval",
     "CQADupstackEnglishNLRetrieval",
     "CQADupstackGamingNLRetrieval",
@@ -46,10 +52,14 @@ __all__ = [
     "CQADupstackWordpressNLRetrieval",
     "ClimateFEVERNL",
     "DBPediaNL",
+    "DutchNewsArticlesRetrieval",
     "FiQA2018NL",
     "HotpotQANL",
+    "LegalQANLRetrieval",
     "NFCorpusNL",
+    "OpenTenderRetrieval",
     "QuoraNLRetrieval",
     "SciFactNL",
     "Touche2020NL",
+    "VABBRetrieval",
 ]

mteb/tasks/retrieval/nld/bbsard_nl_retrieval.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class BBSARDNLRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="bBSARDNLRetrieval",
+        description="Building on the Belgian Statutory Article Retrieval Dataset (BSARD) in French, we introduce the "
+        "bilingual version of this dataset, bBSARD. The dataset contains parallel Belgian statutory "
+        "articles in both French and Dutch, along with legal questions from BSARD and their Dutch "
+        "translation.",
+        reference="https://aclanthology.org/2025.regnlp-1.3.pdf",
+        dataset={
+            "path": "clips/mteb-nl-bbsard",
+            "revision": "52027c212ba9765a3e9737c9cbf9a06ae83cbb93",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2021-05-01", "2021-08-26"),
+        domains=["Legal", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@article{lotfi2025bilingual,
+  author = {Lotfi, Ehsan and Banar, Nikolay and Yuzbashyan, Nerses and Daelemans, Walter},
+  journal = {COLING 2025},
+  pages = {10},
+  title = {Bilingual BSARD: Extending Statutory Article Retrieval to Dutch},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/nld/dutch_news_articles_retrieval.py

@@ -0,0 +1,30 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchNewsArticlesRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="DutchNewsArticlesRetrieval",
+        description="This dataset contains all the articles published by the NOS as of the 1st of January 2010. The "
+        "data is obtained by scraping the NOS website. The NOS is one of the biggest (online) news "
+        "organizations in the Netherlands.",
+        reference="https://www.kaggle.com/datasets/maxscheijen/dutch-news-articles",
+        dataset={
+            "path": "clips/mteb-nl-news-articles-ret",
+            "revision": "c8042a86f3eb0d1fcec79a4a44ebf1eafe635462",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Written", "News"],
+        task_subtypes=["Article retrieval"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="",
+    )

mteb/tasks/retrieval/nld/legal_qa_nl_retrieval.py

@@ -0,0 +1,39 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class LegalQANLRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="LegalQANLRetrieval",
+        description="To this end, we create and publish a Dutch legal QA dataset, consisting of question-answer pairs "
+        "with attributions to Dutch law articles.",
+        reference="https://aclanthology.org/2024.nllp-1.12/",
+        dataset={
+            "path": "clips/mteb-nl-legalqa-pr",
+            "revision": "8f593522dfbe7ec07055ca9d38a700e7643d3882",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2021-05-01", "2021-08-26"),
+        domains=["Legal", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{redelaar2024attributed,
+  author = {Redelaar, Felicia and Van Drie, Romy and Verberne, Suzan and De Boer, Maaike},
+  booktitle = {Proceedings of the natural legal language processing workshop 2024},
+  pages = {154--165},
+  title = {Attributed Question Answering for Preconditions in the Dutch Law},
+  year = {2024},
+}
+""",
+    )

mteb/tasks/retrieval/nld/open_tender_retrieval.py

@@ -0,0 +1,38 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class OpenTenderRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="OpenTenderRetrieval",
+        description="This dataset contains Belgian and Dutch tender calls from OpenTender in Dutch",
+        reference="https://arxiv.org/abs/2509.12340",
+        dataset={
+            "path": "clips/mteb-nl-opentender-ret",
+            "revision": "83eec1aa9c58f1dc8acfac015f653a9c25bda3f4",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Government", "Written"],
+        task_subtypes=["Article retrieval"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@misc{banar2025mtebnle5nlembeddingbenchmark,
+  archiveprefix = {arXiv},
+  author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans},
+  eprint = {2509.12340},
+  primaryclass = {cs.CL},
+  title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch},
+  url = {https://arxiv.org/abs/2509.12340},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/nld/vabb_retrieval.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VABBRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="VABBRetrieval",
+        description="This dataset contains the fourteenth edition of the Flemish Academic Bibliography for the Social "
+        "Sciences and Humanities (VABB-SHW), a database of academic publications from the social sciences "
+        "and humanities authored by researchers affiliated to Flemish universities (more information). "
+        "Publications in the database are used as one of the parameters of the Flemish performance-based "
+        "research funding system",
+        reference="https://zenodo.org/records/14214806",
+        dataset={
+            "path": "clips/mteb-nl-vabb-ret",
+            "revision": "af4a1e5b3ed451103894f86ff6b3ce85085d7b48",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Academic", "Written"],
+        task_subtypes=["Article retrieval"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@dataset{aspeslagh2024vabb,
+  author = {Aspeslagh, Pieter and Guns, Raf and Engels, Tim C. E.},
+  doi = {10.5281/zenodo.14214806},
+  publisher = {Zenodo},
+  title = {VABB-SHW: Dataset of Flemish Academic Bibliography for the Social Sciences and Humanities (edition 14)},
+  url = {https://doi.org/10.5281/zenodo.14214806},
+  year = {2024},
+}
+""",
+    )

mteb/tasks/sts/__init__.py

@@ -7,6 +7,7 @@ from .fra import *
 from .jpn import *
 from .kor import *
 from .multilingual import *
+from .nld import *
 from .pol import *
 from .por import *
 from .ron import *

mteb/tasks/sts/nld/__init__.py

@@ -0,0 +1,5 @@
+from .sick_nl_sts import SICKNLSTS
+
+__all__ = [
+    "SICKNLSTS",
+]

mteb/tasks/sts/nld/sick_nl_sts.py

@@ -0,0 +1,41 @@
+from mteb.abstasks import AbsTaskSTS
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class SICKNLSTS(AbsTaskSTS):
+    fast_loading = True
+    metadata = TaskMetadata(
+        name="SICK-NL-STS",
+        dataset={
+            "path": "clips/mteb-nl-sick-sts-pr",
+            "revision": "7f88f003fc4e37ed8cd9ade84e390d871b032fef",
+        },
+        description="SICK-NL (read: signal), a dataset targeting Natural Language Inference in Dutch. SICK-NL is "
+        "obtained by translating the SICK dataset of (Marelli et al., 2014) from English into Dutch.",
+        reference="https://aclanthology.org/2021.eacl-main.126/",
+        type="STS",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="cosine_spearman",
+        date=("2012-01-01", "2017-12-31"),
+        domains=["News", "Social", "Web", "Spoken", "Written"],
+        task_subtypes=[],
+        license="mit",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated",
+        bibtex_citation=r"""
+@inproceedings{wijnholds2021sick,
+  author = {Wijnholds, Gijs and Moortgat, Michael},
+  booktitle = {Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume},
+  pages = {1474--1479},
+  title = {SICK-NL: A Dataset for Dutch Natural Language Inference},
+  year = {2021},
+}
+""",
+    )
+
+    min_score = 0
+    max_score = 5