mteb 2.0.5__py3-none-any.whl → 2.1.0__py3-none-any.whl

mteb/descriptive_stats/MultilabelClassification/VABBMultiLabelClassification.json

@@ -0,0 +1,156 @@
+{
+    "test": {
+        "num_samples": 3235,
+        "number_texts_intersect_with_train": 2265,
+        "text_statistics": {
+            "total_text_length": 2927178,
+            "min_text_length": 47,
+            "average_text_length": 904.8463678516229,
+            "max_text_length": 35167,
+            "unique_texts": 3235
+        },
+        "image_statistics": null,
+        "label_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.8593508500772797,
+            "max_labels_per_text": 8,
+            "unique_labels": 19,
+            "labels": {
+                "Social Sciences (General)": {
+                    "count": 472
+                },
+                "Sociology": {
+                    "count": 366
+                },
+                "Other Disciplines": {
+                    "count": 557
+                },
+                "Educational Sciences": {
+                    "count": 302
+                },
+                "Economics & Business": {
+                    "count": 393
+                },
+                "Social Health Sciences": {
+                    "count": 255
+                },
+                "Political Sciences": {
+                    "count": 201
+                },
+                "Communication Studies": {
+                    "count": 155
+                },
+                "History": {
+                    "count": 287
+                },
+                "Humanities (General)": {
+                    "count": 398
+                },
+                "Law": {
+                    "count": 999
+                },
+                "Philosophy": {
+                    "count": 233
+                },
+                "Art History": {
+                    "count": 349
+                },
+                "Theology": {
+                    "count": 58
+                },
+                "Psychology": {
+                    "count": 174
+                },
+                "Criminology": {
+                    "count": 284
+                },
+                "Linguistics": {
+                    "count": 254
+                },
+                "Literature": {
+                    "count": 140
+                },
+                "Archaeology": {
+                    "count": 138
+                }
+            }
+        }
+    },
+    "train": {
+        "num_samples": 6471,
+        "number_texts_intersect_with_train": null,
+        "text_statistics": {
+            "total_text_length": 5847151,
+            "min_text_length": 38,
+            "average_text_length": 903.5931077113274,
+            "max_text_length": 35167,
+            "unique_texts": 6471
+        },
+        "image_statistics": null,
+        "label_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.8822438572090867,
+            "max_labels_per_text": 11,
+            "unique_labels": 19,
+            "labels": {
+                "Social Sciences (General)": {
+                    "count": 1021
+                },
+                "Sociology": {
+                    "count": 660
+                },
+                "Other Disciplines": {
+                    "count": 1124
+                },
+                "Educational Sciences": {
+                    "count": 541
+                },
+                "Economics & Business": {
+                    "count": 756
+                },
+                "Linguistics": {
+                    "count": 501
+                },
+                "Humanities (General)": {
+                    "count": 995
+                },
+                "Political Sciences": {
+                    "count": 380
+                },
+                "Communication Studies": {
+                    "count": 327
+                },
+                "Psychology": {
+                    "count": 375
+                },
+                "Philosophy": {
+                    "count": 434
+                },
+                "Art History": {
+                    "count": 703
+                },
+                "History": {
+                    "count": 522
+                },
+                "Law": {
+                    "count": 2150
+                },
+                "Theology": {
+                    "count": 116
+                },
+                "Social Health Sciences": {
+                    "count": 489
+                },
+                "Criminology": {
+                    "count": 576
+                },
+                "Literature": {
+                    "count": 275
+                },
+                "Archaeology": {
+                    "count": 235
+                }
+            }
+        }
+    }
+}

mteb/descriptive_stats/PairClassification/SICKNLPairClassification.json

@@ -0,0 +1,35 @@
+{
+    "test": {
+        "num_samples": 2115,
+        "number_of_characters": 194569,
+        "unique_pairs": 2115,
+        "text1_statistics": {
+            "total_text_length": 100505,
+            "min_text_length": 10,
+            "average_text_length": 47.520094562647756,
+            "max_text_length": 158,
+            "unique_texts": 1769
+        },
+        "text2_statistics": {
+            "total_text_length": 94064,
+            "min_text_length": 10,
+            "average_text_length": 44.47470449172577,
+            "max_text_length": 158,
+            "unique_texts": 1738
+        },
+        "labels_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.0,
+            "max_labels_per_text": 1,
+            "unique_labels": 2,
+            "labels": {
+                "1": {
+                    "count": 1404
+                },
+                "0": {
+                    "count": 711
+                }
+            }
+        }
+    }
+}

mteb/descriptive_stats/PairClassification/XLWICNLPairClassification.json

@@ -0,0 +1,35 @@
+{
+    "test": {
+        "num_samples": 1004,
+        "number_of_characters": 103768,
+        "unique_pairs": 1004,
+        "text1_statistics": {
+            "total_text_length": 52390,
+            "min_text_length": 15,
+            "average_text_length": 52.18127490039841,
+            "max_text_length": 139,
+            "unique_texts": 851
+        },
+        "text2_statistics": {
+            "total_text_length": 51378,
+            "min_text_length": 9,
+            "average_text_length": 51.17330677290837,
+            "max_text_length": 131,
+            "unique_texts": 863
+        },
+        "labels_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.0,
+            "max_labels_per_text": 1,
+            "unique_labels": 2,
+            "labels": {
+                "1": {
+                    "count": 502
+                },
+                "0": {
+                    "count": 502
+                }
+            }
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/DutchNewsArticlesRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 256524,
+        "number_of_characters": 415016602,
+        "documents_text_statistics": {
+            "total_text_length": 414968472,
+            "min_text_length": 0,
+            "average_text_length": 1623.9902005291087,
+            "max_text_length": 29368,
+            "unique_texts": 255216
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 48130,
+            "min_text_length": 11,
+            "average_text_length": 48.13,
+            "max_text_length": 110,
+            "unique_texts": 1000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/LegalQANLRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 30905,
+        "number_of_characters": 20629665,
+        "documents_text_statistics": {
+            "total_text_length": 20619027,
+            "min_text_length": 41,
+            "average_text_length": 669.3837288575788,
+            "max_text_length": 1716,
+            "unique_texts": 30172
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 10638,
+            "min_text_length": 27,
+            "average_text_length": 104.29411764705883,
+            "max_text_length": 369,
+            "unique_texts": 102
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 157,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.5392156862745099,
+            "max_relevant_docs_per_query": 8,
+            "unique_relevant_docs": 148
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/OpenTenderRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 138633,
+        "number_of_characters": 59639635,
+        "documents_text_statistics": {
+            "total_text_length": 59576581,
+            "min_text_length": 2,
+            "average_text_length": 432.86552643624714,
+            "max_text_length": 16782,
+            "unique_texts": 122413
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 63054,
+            "min_text_length": 9,
+            "average_text_length": 63.054,
+            "max_text_length": 286,
+            "unique_texts": 992
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/VABBRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 10318,
+        "number_of_characters": 7839416,
+        "documents_text_statistics": {
+            "total_text_length": 7765564,
+            "min_text_length": 9,
+            "average_text_length": 833.393861343636,
+            "max_text_length": 35146,
+            "unique_texts": 9123
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 73852,
+            "min_text_length": 7,
+            "average_text_length": 73.852,
+            "max_text_length": 258,
+            "unique_texts": 999
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/bBSARDNLRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 22637,
+        "number_of_characters": 21218611,
+        "documents_text_statistics": {
+            "total_text_length": 21197901,
+            "min_text_length": 7,
+            "average_text_length": 945.7015837608744,
+            "max_text_length": 37834,
+            "unique_texts": 22415
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 20710,
+            "min_text_length": 22,
+            "average_text_length": 93.28828828828829,
+            "max_text_length": 250,
+            "unique_texts": 222
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1059,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 4.77027027027027,
+            "max_relevant_docs_per_query": 57,
+            "unique_relevant_docs": 491
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/STS/SICK-NL-STS.json

@@ -0,0 +1,28 @@
+{
+    "test": {
+        "num_samples": 4902,
+        "number_of_characters": 463327,
+        "unique_pairs": 4902,
+        "text1_statistics": {
+            "total_text_length": 233941,
+            "min_text_length": 10,
+            "average_text_length": 47.72358221134231,
+            "max_text_length": 158,
+            "unique_texts": 3378
+        },
+        "text2_statistics": {
+            "total_text_length": 229386,
+            "min_text_length": 10,
+            "average_text_length": 46.79436964504284,
+            "max_text_length": 158,
+            "unique_texts": 3327
+        },
+        "image1_statistics": null,
+        "image2_statistics": null,
+        "label_statistics": {
+            "min_score": 1.0,
+            "avg_score": 3.528012039368932,
+            "max_score": 5.0
+        }
+    }
+}

mteb/tasks/classification/nld/__init__.py

@@ -2,8 +2,24 @@ from .dutch_book_review_sentiment_classification import (
     DutchBookReviewSentimentClassification,
     DutchBookReviewSentimentClassificationV2,
 )
+from .dutch_cola_classification import DutchColaClassification
+from .dutch_government_bias_classification import DutchGovernmentBiasClassification
+from .dutch_news_articles_classification import DutchNewsArticlesClassification
+from .dutch_sarcastic_headlines_classification import (
+    DutchSarcasticHeadlinesClassification,
+)
+from .iconclass_classification import IconclassClassification
+from .open_tender_classification import OpenTenderClassification
+from .vaccin_chat_nl_classification import VaccinChatNLClassification
 
 __all__ = [
     "DutchBookReviewSentimentClassification",
     "DutchBookReviewSentimentClassificationV2",
+    "DutchColaClassification",
+    "DutchGovernmentBiasClassification",
+    "DutchNewsArticlesClassification",
+    "DutchSarcasticHeadlinesClassification",
+    "IconclassClassification",
+    "OpenTenderClassification",
+    "VaccinChatNLClassification",
 ]

mteb/tasks/classification/nld/dutch_cola_classification.py

@@ -0,0 +1,38 @@
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchColaClassification(AbsTaskClassification):
+    samples_per_label = 128
+    metadata = TaskMetadata(
+        name="DutchColaClassification",
+        description="Dutch CoLA is a corpus of linguistic acceptability for Dutch.",
+        reference="https://huggingface.co/datasets/GroNLP/dutch-cola",
+        dataset={
+            "path": "clips/mteb-nl-dutch-cola",
+            "revision": "2269ed7d95d8abaab829f1592b4b2047372e9f81",
+        },
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        date=("2024-03-01", "2024-05-01"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        domains=["Written"],
+        task_subtypes=["Linguistic acceptability"],
+        license="not specified",  # specified as unknown
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@misc{gronlp_2024,
+  author = {Bylinina, Lisa and Abdi, Silvana and Brouwer, Hylke and Elzinga, Martine and Gunput, Shenza and Huisman, Sem and Krooneman, Collin and Poot, David and Top, Jelmer and Weideman, Cain},
+  doi = { 10.57967/hf/3825 },
+  publisher = { Hugging Face },
+  title = { {Dutch-CoLA (Revision 5a4196c)} },
+  url = { https://huggingface.co/datasets/GroNLP/dutch-cola },
+  year = {2024},
+}
+""",
+    )

mteb/tasks/classification/nld/dutch_government_bias_classification.py

@@ -0,0 +1,37 @@
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchGovernmentBiasClassification(AbsTaskClassification):
+    samples_per_label = 32
+    metadata = TaskMetadata(
+        name="DutchGovernmentBiasClassification",
+        description="The Dutch Government Data for Bias Detection (DGDB) is a dataset sourced from the Dutch House of Representatives and annotated for bias by experts",
+        reference="https://dl.acm.org/doi/pdf/10.1145/3696410.3714526",
+        dataset={
+            "path": "clips/mteb-nl-dutch-government-bias-detection",
+            "revision": "bf5e20ee2d3ce2e24e4de50f5dd8573e0e0e2fec",
+        },
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        date=("2019-10-04", "2019-10-04"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        domains=["Written", "Government"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{de2025detecting,
+  author = {de Swart, Milena and Den Hengst, Floris and Chen, Jieying},
+  booktitle = {Proceedings of the ACM on Web Conference 2025},
+  pages = {5034--5044},
+  title = {Detecting Linguistic Bias in Government Documents Using Large language Models},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/classification/nld/dutch_news_articles_classification.py

@@ -0,0 +1,30 @@
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchNewsArticlesClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="DutchNewsArticlesClassification",
+        dataset={
+            "path": "clips/mteb-nl-news-articles-cls",
+            "revision": "0a7227d31f85c5676be92767f8df5405ea93de54",
+        },
+        description="This dataset contains all the articles published by the NOS as of the 1st of January 2010. The "
+        "data is obtained by scraping the NOS website. The NOS is one of the biggest (online) news "
+        "organizations in the Netherlands.",
+        reference="https://www.kaggle.com/datasets/maxscheijen/dutch-news-articles",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Written", "News"],
+        task_subtypes=["Topic classification"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="",
+    )

mteb/tasks/classification/nld/dutch_sarcastic_headlines_classification.py

@@ -0,0 +1,36 @@
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchSarcasticHeadlinesClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="DutchSarcasticHeadlinesClassification",
+        description="This dataset contains news headlines of two Dutch news websites. All sarcastic headlines were "
+        "collected from the Speld.nl (the Dutch equivalent of The Onion) whereas all 'normal' headlines "
+        "were collected from the news website Nu.nl.",
+        reference="https://www.kaggle.com/datasets/harrotuin/dutch-news-headlines",
+        dataset={
+            "path": "clips/mteb-nl-sarcastic-headlines",
+            "revision": "7e520e36394795859583f84f81fcb97de915d05a",
+        },
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        date=("2019-01-01", "2020-01-01"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        domains=["News", "Written", "Fiction"],
+        task_subtypes=[],
+        license="cc0-1.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""""",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].rename_columns(
+                {"headline": "text", "is_sarcastic": "label"}
+            )

mteb/tasks/classification/nld/iconclass_classification.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class IconclassClassification(AbsTaskClassification):
+    samples_per_label = 32
+    metadata = TaskMetadata(
+        name="IconclassClassification",
+        description="Iconclass is an iconographic thesaurus, which is widely used in the digital heritage domain to "
+        "describe subjects depicted in artworks. The task is to classify the first layer of Iconclass",
+        reference="https://dl.acm.org/doi/pdf/10.1145/3575865",
+        dataset={
+            "path": "clips/mteb-nl-iconclass-cls",
+            "revision": "1cd02f1579dab39fedc95de8cc15fd620557a9f2",
+        },
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        date=("2020-01-01", "2020-05-01"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        domains=["Written", "Fiction"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@article{banar2023transfer,
+  author = {Banar, Nikolay and Daelemans, Walter and Kestemont, Mike},
+  journal = {ACM Journal on Computing and Cultural Heritage},
+  number = {2},
+  pages = {1--16},
+  publisher = {ACM New York, NY},
+  title = {Transfer learning for the visual arts: The multi-modal retrieval of iconclass codes},
+  volume = {16},
+  year = {2023},
+}
+""",
+    )

mteb/tasks/classification/nld/open_tender_classification.py

@@ -0,0 +1,38 @@
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class OpenTenderClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="OpenTenderClassification",
+        dataset={
+            "path": "clips/mteb-nl-opentender-cls-pr",
+            "revision": "9af5657575a669dc18c7f897a67287ff7d1a0c65",
+        },
+        description="This dataset contains Belgian and Dutch tender calls from OpenTender in Dutch",
+        reference="https://arxiv.org/abs/2509.12340",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2025-08-01", "2025-08-10"),
+        domains=["Government", "Written"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@misc{banar2025mtebnle5nlembeddingbenchmark,
+  archiveprefix = {arXiv},
+  author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans},
+  eprint = {2509.12340},
+  primaryclass = {cs.CL},
+  title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch},
+  url = {https://arxiv.org/abs/2509.12340},
+  year = {2025},
+}
+""",
+    )