mteb 2.0.5__py3-none-any.whl → 2.1.0__py3-none-any.whl

mteb/tasks/classification/nld/vaccin_chat_nl_classification.py

@@ -0,0 +1,46 @@
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VaccinChatNLClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="VaccinChatNLClassification",
+        description="VaccinChatNL is a Flemish Dutch FAQ dataset on the topic of COVID-19 vaccinations in Flanders.",
+        reference="https://huggingface.co/datasets/clips/VaccinChatNL",
+        dataset={
+            "path": "clips/VaccinChatNL",
+            "revision": "bd27d0058bea2ad52470d9072a3b5da6b97c1ac3",
+        },
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        date=("2022-01-01", "2022-09-01"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        domains=["Spoken", "Web"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="created",
+        bibtex_citation=r"""
+@inproceedings{buhmann-etal-2022-domain,
+  address = {Gyeongju, Republic of Korea},
+  author = {Buhmann, Jeska and De Bruyn, Maxime and Lotfi, Ehsan and Daelemans, Walter},
+  booktitle = {Proceedings of the 29th International Conference on Computational Linguistics},
+  month = oct,
+  pages = {3539--3549},
+  publisher = {International Committee on Computational Linguistics},
+  title = {Domain- and Task-Adaptation for {V}accin{C}hat{NL}, a {D}utch {COVID}-19 {FAQ} Answering Corpus and Classification Model},
+  url = {https://aclanthology.org/2022.coling-1.312},
+  year = {2022},
+}
+""",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].rename_columns(
+                {"sentence1": "text"}
+            )

mteb/tasks/clustering/__init__.py

@@ -5,6 +5,7 @@ from .fra import *
 from .jpn import *
 from .kor import *
 from .multilingual import *
+from .nld import *
 from .nob import *
 from .pol import *
 from .rom import *

mteb/tasks/clustering/nld/__init__.py

@@ -0,0 +1,17 @@
+from .dutch_news_articles_clustering_p2p import DutchNewsArticlesClusteringP2P
+from .dutch_news_articles_clustering_s2s import DutchNewsArticlesClusteringS2S
+from .iconclass_clustering_s2s import IconclassClusteringS2S
+from .open_tender_clustering_p2p import OpenTenderClusteringP2P
+from .open_tender_clustering_s2s import OpenTenderClusteringS2S
+from .vabb_clustering_p2p import VABBClusteringP2P
+from .vabb_clustering_s2s import VABBClusteringS2S
+
+__all__ = [
+    "DutchNewsArticlesClusteringP2P",
+    "DutchNewsArticlesClusteringS2S",
+    "IconclassClusteringS2S",
+    "OpenTenderClusteringP2P",
+    "OpenTenderClusteringS2S",
+    "VABBClusteringP2P",
+    "VABBClusteringS2S",
+]

mteb/tasks/clustering/nld/dutch_news_articles_clustering_p2p.py

@@ -0,0 +1,37 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchNewsArticlesClusteringP2P(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="DutchNewsArticlesClusteringP2P",
+        dataset={
+            "path": "clips/mteb-nl-news-articles-cls",
+            "revision": "0a7227d31f85c5676be92767f8df5405ea93de54",
+        },
+        description="This dataset contains all the articles published by the NOS as of the 1st of January 2010. The "
+        "data is obtained by scraping the NOS website. The NOS is one of the biggest (online) news "
+        "organizations in the Netherlands.",
+        reference="https://www.kaggle.com/datasets/maxscheijen/dutch-news-articles",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Written", "News"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].rename_columns(
+                {"label": "labels", "text": "sentences"}
+            )

mteb/tasks/clustering/nld/dutch_news_articles_clustering_s2s.py

@@ -0,0 +1,37 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchNewsArticlesClusteringS2S(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="DutchNewsArticlesClusteringS2S",
+        dataset={
+            "path": "clips/mteb-nl-news-articles-cls",
+            "revision": "0a7227d31f85c5676be92767f8df5405ea93de54",
+        },
+        description="This dataset contains all the articles published by the NOS as of the 1st of January 2010. The "
+        "data is obtained by scraping the NOS website. The NOS is one of the biggest (online) news "
+        "organizations in the Netherlands.",
+        reference="https://www.kaggle.com/datasets/maxscheijen/dutch-news-articles",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Written", "News"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].rename_columns(
+                {"label": "labels", "title": "sentences"}
+            )

mteb/tasks/clustering/nld/iconclass_clustering_s2s.py

@@ -0,0 +1,47 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class IconclassClusteringS2S(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="IconclassClusteringS2S",
+        dataset={
+            "path": "clips/mteb-nl-iconclass-cls",
+            "revision": "1cd02f1579dab39fedc95de8cc15fd620557a9f2",
+        },
+        description="Iconclass is an iconographic thesaurus, which is widely used in the digital heritage domain to "
+        "describe subjects depicted in artworks. The task is to classify the first layer of Iconclass",
+        reference="https://dl.acm.org/doi/pdf/10.1145/3575865",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Written", "Fiction"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@article{banar2023transfer,
+  author = {Banar, Nikolay and Daelemans, Walter and Kestemont, Mike},
+  journal = {ACM Journal on Computing and Cultural Heritage},
+  number = {2},
+  pages = {1--16},
+  publisher = {ACM New York, NY},
+  title = {Transfer learning for the visual arts: The multi-modal retrieval of iconclass codes},
+  volume = {16},
+  year = {2023},
+}
+""",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].map(
+                lambda ex: {"labels": ex["label"], "sentences": ex["text"]}
+            )

mteb/tasks/clustering/nld/open_tender_clustering_p2p.py

@@ -0,0 +1,51 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class OpenTenderClusteringP2P(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="OpenTenderClusteringP2P",
+        dataset={
+            "path": "clips/mteb-nl-opentender-cls-pr",
+            "revision": "9af5657575a669dc18c7f897a67287ff7d1a0c65",
+        },
+        description="This dataset contains all the articles published by the NOS as of the 1st of January 2010. The "
+        "data is obtained by scraping the NOS website. The NOS is one of the biggest (online) news "
+        "organizations in the Netherlands.",
+        reference="https://arxiv.org/abs/2509.12340",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2025-08-01", "2025-08-10"),
+        domains=["Government", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@misc{banar2025mtebnle5nlembeddingbenchmark,
+  archiveprefix = {arXiv},
+  author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans},
+  eprint = {2509.12340},
+  primaryclass = {cs.CL},
+  title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch},
+  url = {https://arxiv.org/abs/2509.12340},
+  year = {2025},
+}
+""",
+    )
+
+    def dataset_transform(self):
+        # reuse the dataset for classification
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].map(
+                lambda ex: {
+                    "labels": ex["label"],
+                    "sentences": ex["text"],
+                }
+            )

mteb/tasks/clustering/nld/open_tender_clustering_s2s.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class OpenTenderClusteringS2S(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="OpenTenderClusteringS2S",
+        dataset={
+            "path": "clips/mteb-nl-opentender-clst-s2s-pr",
+            "revision": "ad86cf1813d130e17dda0092d1c4f2c664e68d0c",
+        },
+        description="This dataset contains all the articles published by the NOS as of the 1st of January 2010. The "
+        "data is obtained by scraping the NOS website. The NOS is one of the biggest (online) news "
+        "organizations in the Netherlands.",
+        reference="https://arxiv.org/abs/2509.12340",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2025-08-01", "2025-08-10"),
+        domains=["Government", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@misc{banar2025mtebnle5nlembeddingbenchmark,
+  archiveprefix = {arXiv},
+  author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans},
+  eprint = {2509.12340},
+  primaryclass = {cs.CL},
+  title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch},
+  url = {https://arxiv.org/abs/2509.12340},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/clustering/nld/vabb_clustering_p2p.py

@@ -0,0 +1,51 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VABBClusteringP2P(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="VABBClusteringP2P",
+        dataset={
+            "path": "clips/mteb-nl-vabb-cls",
+            "revision": "544acc2e46909eab2b49962b043a18b9c9772770",
+        },
+        description="This dataset contains the fourteenth edition of the Flemish Academic Bibliography for the Social "
+        "Sciences and Humanities (VABB-SHW), a database of academic publications from the social sciences "
+        "and humanities authored by researchers affiliated to Flemish universities (more information). "
+        "Publications in the database are used as one of the parameters of the Flemish performance-based "
+        "research funding system",
+        reference="https://zenodo.org/records/14214806",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Academic", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@dataset{aspeslagh2024vabb,
+  author = {Aspeslagh, Pieter and Guns, Raf and Engels, Tim C. E.},
+  doi = {10.5281/zenodo.14214806},
+  publisher = {Zenodo},
+  title = {VABB-SHW: Dataset of Flemish Academic Bibliography for the Social Sciences and Humanities (edition 14)},
+  url = {https://doi.org/10.5281/zenodo.14214806},
+  year = {2024},
+}
+""",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].map(
+                lambda ex: {
+                    "labels": ex["org_discipline"],
+                    "sentences": f"{ex['title']}\n{ex['abstract']}",
+                }
+            )

mteb/tasks/clustering/nld/vabb_clustering_s2s.py

@@ -0,0 +1,51 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VABBClusteringS2S(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="VABBClusteringS2S",
+        dataset={
+            "path": "clips/mteb-nl-vabb-cls",
+            "revision": "544acc2e46909eab2b49962b043a18b9c9772770",
+        },
+        description="This dataset contains the fourteenth edition of the Flemish Academic Bibliography for the Social "
+        "Sciences and Humanities (VABB-SHW), a database of academic publications from the social sciences "
+        "and humanities authored by researchers affiliated to Flemish universities (more information). "
+        "Publications in the database are used as one of the parameters of the Flemish performance-based "
+        "research funding system",
+        reference="https://zenodo.org/records/14214806",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Academic", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@dataset{aspeslagh2024vabb,
+  author = {Aspeslagh, Pieter and Guns, Raf and Engels, Tim C. E.},
+  doi = {10.5281/zenodo.14214806},
+  publisher = {Zenodo},
+  title = {VABB-SHW: Dataset of Flemish Academic Bibliography for the Social Sciences and Humanities (edition 14)},
+  url = {https://doi.org/10.5281/zenodo.14214806},
+  year = {2024},
+}
+""",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].rename_columns(
+                {"title": "sentences"}
+            )
+            self.dataset[split] = self.dataset[split].map(
+                lambda ex: {"labels": ex["org_discipline"]}
+            )

mteb/tasks/multilabel_classification/__init__.py

@@ -3,6 +3,7 @@ from .ita import *
 from .kor import *
 from .mlt import *
 from .multilingual import *
+from .nld import *
 from .por import *
 from .rus import *
 from .swe import *

mteb/tasks/multilabel_classification/nld/__init__.py

@@ -0,0 +1,9 @@
+from .covid_disinformation_nl_multi_label_classification import (
+    CovidDisinformationNLMultiLabelClassification,
+)
+from .vabb_multi_label_classification import VABBMultiLabelClassification
+
+__all__ = [
+    "CovidDisinformationNLMultiLabelClassification",
+    "VABBMultiLabelClassification",
+]

mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py

@@ -0,0 +1,88 @@
+from mteb.abstasks.multilabel_classification import (
+    AbsTaskMultilabelClassification,
+)
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class CovidDisinformationNLMultiLabelClassification(AbsTaskMultilabelClassification):
+    metadata = TaskMetadata(
+        name="CovidDisinformationNLMultiLabelClassification",
+        dataset={
+            "path": "clips/mteb-nl-COVID-19-disinformation",
+            "revision": "7ad922bdef875db1f530847c6ffff05fc154f2e8",
+        },
+        description="The dataset is curated to address questions of interest to journalists, fact-checkers, "
+        "social media platforms, policymakers, and the general public.",
+        reference="https://aclanthology.org/2021.findings-emnlp.56.pdf",
+        type="MultilabelClassification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2020-01-01", "2021-04-01"),
+        domains=["Web", "Social", "Written"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{alam-etal-2021-fighting-covid,
+  address = {Punta Cana, Dominican Republic},
+  author = {Alam, Firoj  and
+Shaar, Shaden  and
+Dalvi, Fahim  and
+Sajjad, Hassan  and
+Nikolov, Alex  and
+Mubarak, Hamdy  and
+Da San Martino, Giovanni  and
+Abdelali, Ahmed  and
+Durrani, Nadir  and
+Darwish, Kareem  and
+Al-Homaid, Abdulaziz  and
+Zaghouani, Wajdi  and
+Caselli, Tommaso  and
+Danoe, Gijs  and
+Stolk, Friso  and
+Bruntink, Britt  and
+Nakov, Preslav},
+  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2021},
+  doi = {10.18653/v1/2021.findings-emnlp.56},
+  editor = {Moens, Marie-Francine  and
+Huang, Xuanjing  and
+Specia, Lucia  and
+Yih, Scott Wen-tau},
+  month = nov,
+  pages = {611--649},
+  publisher = {Association for Computational Linguistics},
+  title = {Fighting the {COVID}-19 Infodemic: Modeling the Perspective of Journalists, Fact-Checkers, Social Media Platforms, Policy Makers, and the Society},
+  url = {https://aclanthology.org/2021.findings-emnlp.56/},
+  year = {2021},
+}
+""",
+    )
+
+    def dataset_transform(self) -> None:
+        labels = [
+            "q2_label",
+            "q3_label",
+            "q4_label",
+            "q5_label",
+            "q6_label",
+            "q7_label",
+        ]
+        _dataset = {}
+
+        def map_labels(example):
+            ml_labels = []
+            for i, label in enumerate(labels):
+                if example[label] == "yes":
+                    ml_labels.append(i)
+            return {"label": ml_labels}
+
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].filter(
+                lambda ex: ex["q1_label"] == "yes"
+            )
+            self.dataset[split] = self.dataset[split].map(map_labels)

mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py

@@ -0,0 +1,44 @@
+from mteb.abstasks.multilabel_classification import (
+    AbsTaskMultilabelClassification,
+)
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VABBMultiLabelClassification(AbsTaskMultilabelClassification):
+    samples_per_label = 128
+    metadata = TaskMetadata(
+        name="VABBMultiLabelClassification",
+        dataset={
+            "path": "clips/mteb-nl-vabb-mlcls-pr",
+            "revision": "584c70f5104671772119f21e9f8a3c912ac07d4a",
+        },
+        description="This dataset contains the fourteenth edition of the Flemish Academic Bibliography for the Social "
+        "Sciences and Humanities (VABB-SHW), a database of academic publications from the social sciences "
+        "and humanities authored by researchers affiliated to Flemish universities (more information). "
+        "Publications in the database are used as one of the parameters of the Flemish performance-based "
+        "research funding system",
+        reference="https://zenodo.org/records/14214806",
+        type="MultilabelClassification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2020-01-01", "2021-04-01"),
+        domains=["Academic", "Written"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@dataset{aspeslagh2024vabb,
+  author = {Aspeslagh, Pieter and Guns, Raf and Engels, Tim C. E.},
+  doi = {10.5281/zenodo.14214806},
+  publisher = {Zenodo},
+  title = {VABB-SHW: Dataset of Flemish Academic Bibliography for the Social Sciences and Humanities (edition 14)},
+  url = {https://doi.org/10.5281/zenodo.14214806},
+  year = {2024},
+}
+""",
+    )

mteb/tasks/pair_classification/__init__.py

@@ -9,6 +9,7 @@ from .ind import *
 from .ita import *
 from .kor import *
 from .multilingual import *
+from .nld import *
 from .pol import *
 from .por import *
 from .rus import *

mteb/tasks/pair_classification/nld/__init__.py

@@ -0,0 +1,7 @@
+from .sick_nl_pair_classification import SICKNLPairClassification
+from .xlwic_nl_pair_classification import XLWICNLPairClassification
+
+__all__ = [
+    "SICKNLPairClassification",
+    "XLWICNLPairClassification",
+]

mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py

@@ -0,0 +1,36 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class SICKNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="SICKNLPairClassification",
+        dataset={
+            "path": "clips/mteb-nl-sick-pcls-pr",
+            "revision": "a13a1892bcb4c077dc416d390389223eea5f20f0",
+        },
+        description="SICK-NL is a Dutch translation of SICK ",
+        reference="https://aclanthology.org/2021.eacl-main.126/",
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        date=("2020-09-01", "2021-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=[],
+        license="mit",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation=r"""
+@inproceedings{wijnholds2021sick,
+  author = {Wijnholds, Gijs and Moortgat, Michael},
+  booktitle = {Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume},
+  pages = {1474--1479},
+  title = {SICK-NL: A Dataset for Dutch Natural Language Inference},
+  year = {2021},
+}
+""",
+    )

mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class XLWICNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="XLWICNLPairClassification",
+        description="The Word-in-Context dataset (WiC) addresses the dependence on sense inventories by reformulating "
+        "the standard disambiguation task as a binary classification problem; but, it is limited to the "
+        "English language. We put forward a large multilingual benchmark, XL-WiC, featuring gold standards "
+        "in 12 new languages from varied language families and with different degrees of resource "
+        "availability, opening room for evaluation scenarios such as zero-shot cross-lingual transfer. ",
+        reference="https://aclanthology.org/2020.emnlp-main.584.pdf",
+        dataset={
+            "path": "clips/mteb-nl-xlwic",
+            "revision": "0b33ce358b1b5d500ff3715ba3d777b4d2c21cb0",
+        },
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        date=("2019-10-04", "2019-10-04"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        domains=["Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="created",
+        bibtex_citation=r"""
+@inproceedings{raganato2020xl,
+  author = {Raganato, A and Pasini, T and Camacho-Collados, J and Pilehvar, M and others},
+  booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
+  organization = {Association for Computational Linguistics (ACL)},
+  pages = {7193--7206},
+  title = {XL-WiC: A multilingual benchmark for evaluating semantic contextualization},
+  year = {2020},
+}
+""",
+    )