mteb 2.0.5__py3-none-any.whl → 2.1.1__py3-none-any.whl

mteb/tasks/clustering/nld/vabb_clustering_s2s.py

@@ -0,0 +1,51 @@
+from mteb.abstasks.clustering import AbsTaskClustering
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VABBClusteringS2S(AbsTaskClustering):
+    max_fraction_of_documents_to_embed = 1.0
+    metadata = TaskMetadata(
+        name="VABBClusteringS2S",
+        dataset={
+            "path": "clips/mteb-nl-vabb-cls",
+            "revision": "544acc2e46909eab2b49962b043a18b9c9772770",
+        },
+        description="This dataset contains the fourteenth edition of the Flemish Academic Bibliography for the Social "
+        "Sciences and Humanities (VABB-SHW), a database of academic publications from the social sciences "
+        "and humanities authored by researchers affiliated to Flemish universities (more information). "
+        "Publications in the database are used as one of the parameters of the Flemish performance-based "
+        "research funding system",
+        reference="https://zenodo.org/records/14214806",
+        type="Clustering",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="v_measure",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Academic", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@dataset{aspeslagh2024vabb,
+  author = {Aspeslagh, Pieter and Guns, Raf and Engels, Tim C. E.},
+  doi = {10.5281/zenodo.14214806},
+  publisher = {Zenodo},
+  title = {VABB-SHW: Dataset of Flemish Academic Bibliography for the Social Sciences and Humanities (edition 14)},
+  url = {https://doi.org/10.5281/zenodo.14214806},
+  year = {2024},
+}
+""",
+    )
+
+    def dataset_transform(self):
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].rename_columns(
+                {"title": "sentences"}
+            )
+            self.dataset[split] = self.dataset[split].map(
+                lambda ex: {"labels": ex["org_discipline"]}
+            )

mteb/tasks/multilabel_classification/__init__.py

@@ -3,6 +3,7 @@ from .ita import *
 from .kor import *
 from .mlt import *
 from .multilingual import *
+from .nld import *
 from .por import *
 from .rus import *
 from .swe import *

mteb/tasks/multilabel_classification/nld/__init__.py

@@ -0,0 +1,9 @@
+from .covid_disinformation_nl_multi_label_classification import (
+    CovidDisinformationNLMultiLabelClassification,
+)
+from .vabb_multi_label_classification import VABBMultiLabelClassification
+
+__all__ = [
+    "CovidDisinformationNLMultiLabelClassification",
+    "VABBMultiLabelClassification",
+]

mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py

@@ -0,0 +1,88 @@
+from mteb.abstasks.multilabel_classification import (
+    AbsTaskMultilabelClassification,
+)
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class CovidDisinformationNLMultiLabelClassification(AbsTaskMultilabelClassification):
+    metadata = TaskMetadata(
+        name="CovidDisinformationNLMultiLabelClassification",
+        dataset={
+            "path": "clips/mteb-nl-COVID-19-disinformation",
+            "revision": "7ad922bdef875db1f530847c6ffff05fc154f2e8",
+        },
+        description="The dataset is curated to address questions of interest to journalists, fact-checkers, "
+        "social media platforms, policymakers, and the general public.",
+        reference="https://aclanthology.org/2021.findings-emnlp.56.pdf",
+        type="MultilabelClassification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2020-01-01", "2021-04-01"),
+        domains=["Web", "Social", "Written"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{alam-etal-2021-fighting-covid,
+  address = {Punta Cana, Dominican Republic},
+  author = {Alam, Firoj  and
+Shaar, Shaden  and
+Dalvi, Fahim  and
+Sajjad, Hassan  and
+Nikolov, Alex  and
+Mubarak, Hamdy  and
+Da San Martino, Giovanni  and
+Abdelali, Ahmed  and
+Durrani, Nadir  and
+Darwish, Kareem  and
+Al-Homaid, Abdulaziz  and
+Zaghouani, Wajdi  and
+Caselli, Tommaso  and
+Danoe, Gijs  and
+Stolk, Friso  and
+Bruntink, Britt  and
+Nakov, Preslav},
+  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2021},
+  doi = {10.18653/v1/2021.findings-emnlp.56},
+  editor = {Moens, Marie-Francine  and
+Huang, Xuanjing  and
+Specia, Lucia  and
+Yih, Scott Wen-tau},
+  month = nov,
+  pages = {611--649},
+  publisher = {Association for Computational Linguistics},
+  title = {Fighting the {COVID}-19 Infodemic: Modeling the Perspective of Journalists, Fact-Checkers, Social Media Platforms, Policy Makers, and the Society},
+  url = {https://aclanthology.org/2021.findings-emnlp.56/},
+  year = {2021},
+}
+""",
+    )
+
+    def dataset_transform(self) -> None:
+        labels = [
+            "q2_label",
+            "q3_label",
+            "q4_label",
+            "q5_label",
+            "q6_label",
+            "q7_label",
+        ]
+        _dataset = {}
+
+        def map_labels(example):
+            ml_labels = []
+            for i, label in enumerate(labels):
+                if example[label] == "yes":
+                    ml_labels.append(i)
+            return {"label": ml_labels}
+
+        for split in self.dataset:
+            self.dataset[split] = self.dataset[split].filter(
+                lambda ex: ex["q1_label"] == "yes"
+            )
+            self.dataset[split] = self.dataset[split].map(map_labels)

mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py

@@ -0,0 +1,44 @@
+from mteb.abstasks.multilabel_classification import (
+    AbsTaskMultilabelClassification,
+)
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class VABBMultiLabelClassification(AbsTaskMultilabelClassification):
+    samples_per_label = 128
+    metadata = TaskMetadata(
+        name="VABBMultiLabelClassification",
+        dataset={
+            "path": "clips/mteb-nl-vabb-mlcls-pr",
+            "revision": "584c70f5104671772119f21e9f8a3c912ac07d4a",
+        },
+        description="This dataset contains the fourteenth edition of the Flemish Academic Bibliography for the Social "
+        "Sciences and Humanities (VABB-SHW), a database of academic publications from the social sciences "
+        "and humanities authored by researchers affiliated to Flemish universities (more information). "
+        "Publications in the database are used as one of the parameters of the Flemish performance-based "
+        "research funding system",
+        reference="https://zenodo.org/records/14214806",
+        type="MultilabelClassification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="f1",
+        date=("2020-01-01", "2021-04-01"),
+        domains=["Academic", "Written"],
+        task_subtypes=[],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@dataset{aspeslagh2024vabb,
+  author = {Aspeslagh, Pieter and Guns, Raf and Engels, Tim C. E.},
+  doi = {10.5281/zenodo.14214806},
+  publisher = {Zenodo},
+  title = {VABB-SHW: Dataset of Flemish Academic Bibliography for the Social Sciences and Humanities (edition 14)},
+  url = {https://doi.org/10.5281/zenodo.14214806},
+  year = {2024},
+}
+""",
+    )

mteb/tasks/pair_classification/__init__.py

@@ -9,6 +9,7 @@ from .ind import *
 from .ita import *
 from .kor import *
 from .multilingual import *
+from .nld import *
 from .pol import *
 from .por import *
 from .rus import *

mteb/tasks/pair_classification/multilingual/indic_xnli_pair_classification.py

@@ -23,14 +23,15 @@ class IndicXnliPairClassification(AbsTaskPairClassification):
             "path": "mteb/IndicXnliPairClassification",
             "revision": "027e97b9afe84ea3447b57b7705b8864bb2b3a83",
         },
-        description="""INDICXNLI is similar to existing XNLI dataset in shape/form, but
-        focusses on Indic language family.
-        The train (392,702), validation (2,490), and evaluation sets (5,010) of English
-        XNLI were translated from English into each of the eleven Indic languages. IndicTrans
-        is a large Transformer-based sequence to sequence model. It is trained on Samanantar
-        dataset (Ramesh et al., 2021), which is the largest parallel multi- lingual corpus
-        over eleven Indic languages.
-        """,
+        description=(
+            "INDICXNLI is similar to existing XNLI dataset in shape/form, but "
+            "focuses on Indic language family. "
+            "The train (392,702), validation (2,490), and evaluation sets (5,010) of English "
+            "XNLI were translated from English into each of the eleven Indic languages. IndicTrans "
+            "is a large Transformer-based sequence to sequence model. It is trained on Samanantar "
+            "dataset (Ramesh et al., 2021), which is the largest parallel multi- lingual corpus "
+            "over eleven Indic languages."
+        ),
         reference="https://gem-benchmark.com/data_cards/opusparcus",
         category="t2t",
         modalities=["text"],

mteb/tasks/pair_classification/nld/__init__.py

@@ -0,0 +1,7 @@
+from .sick_nl_pair_classification import SICKNLPairClassification
+from .xlwic_nl_pair_classification import XLWICNLPairClassification
+
+__all__ = [
+    "SICKNLPairClassification",
+    "XLWICNLPairClassification",
+]

mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py

@@ -0,0 +1,36 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class SICKNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="SICKNLPairClassification",
+        dataset={
+            "path": "clips/mteb-nl-sick-pcls-pr",
+            "revision": "a13a1892bcb4c077dc416d390389223eea5f20f0",
+        },
+        description="SICK-NL is a Dutch translation of SICK ",
+        reference="https://aclanthology.org/2021.eacl-main.126/",
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        date=("2020-09-01", "2021-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=[],
+        license="mit",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation=r"""
+@inproceedings{wijnholds2021sick,
+  author = {Wijnholds, Gijs and Moortgat, Michael},
+  booktitle = {Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume},
+  pages = {1474--1479},
+  title = {SICK-NL: A Dataset for Dutch Natural Language Inference},
+  year = {2021},
+}
+""",
+    )

mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class XLWICNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="XLWICNLPairClassification",
+        description="The Word-in-Context dataset (WiC) addresses the dependence on sense inventories by reformulating "
+        "the standard disambiguation task as a binary classification problem; but, it is limited to the "
+        "English language. We put forward a large multilingual benchmark, XL-WiC, featuring gold standards "
+        "in 12 new languages from varied language families and with different degrees of resource "
+        "availability, opening room for evaluation scenarios such as zero-shot cross-lingual transfer. ",
+        reference="https://aclanthology.org/2020.emnlp-main.584.pdf",
+        dataset={
+            "path": "clips/mteb-nl-xlwic",
+            "revision": "0b33ce358b1b5d500ff3715ba3d777b4d2c21cb0",
+        },
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        date=("2019-10-04", "2019-10-04"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        domains=["Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="created",
+        bibtex_citation=r"""
+@inproceedings{raganato2020xl,
+  author = {Raganato, A and Pasini, T and Camacho-Collados, J and Pilehvar, M and others},
+  booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
+  organization = {Association for Computational Linguistics (ACL)},
+  pages = {7193--7206},
+  title = {XL-WiC: A multilingual benchmark for evaluating semantic contextualization},
+  year = {2020},
+}
+""",
+    )

mteb/tasks/retrieval/code/code_rag.py

@@ -60,9 +60,9 @@ class CodeRAGProgrammingSolutionsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """
@@ -117,9 +117,9 @@ class CodeRAGOnlineTutorialsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """
@@ -177,9 +177,9 @@ class CodeRAGLibraryDocumentationSolutionsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """
@@ -234,9 +234,9 @@ class CodeRAGStackoverflowPostsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """

mteb/tasks/retrieval/dan/dan_fever_retrieval.py

@@ -56,7 +56,7 @@ Derczynski, Leon},
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
         self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text
         self.queries = dict[query_id, str] #id => query

mteb/tasks/retrieval/dan/tv2_nordretrieval.py

@@ -69,9 +69,9 @@ Piperidis, Stelios},
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text
         self.queries = dict[query_id, str] #id => query
         self.relevant_docs = dict[query_id, dict[[doc_id, score]]
         """

mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py

@@ -45,9 +45,9 @@ class TwitterHjerneRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text
         self.queries = dict[query_id, str] #id => query
         self.relevant_docs = dict[query_id, dict[[doc_id, score]]
         """

mteb/tasks/retrieval/eng/__init__.py

@@ -22,6 +22,7 @@ from .cirr_it2i_retrieval import CIRRIT2IRetrieval
 from .climate_fever_retrieval import (
     ClimateFEVER,
     ClimateFEVERHardNegatives,
+    ClimateFEVERHardNegativesV2,
     ClimateFEVERRetrievalv2,
 )
 from .cqa_dupstack_android_retrieval import CQADupstackAndroidRetrieval
@@ -57,7 +58,7 @@ from .dapfam_patent_retrieval import (
     DAPFAMOutTitlAbsToTitlAbsClmRetrieval,
     DAPFAMOutTitlAbsToTitlAbsRetrieval,
 )
-from .dbpedia_retrieval import DBPedia, DBPediaHardNegatives
+from .dbpedia_retrieval import DBPedia, DBPediaHardNegatives, DBPediaHardNegativesV2
 from .edis_t2it_retrieval import EDIST2ITRetrieval
 from .encyclopedia_vqa_it2it_retrieval import EncyclopediaVQAIT2ITRetrieval
 from .english_finance1_retrieval import EnglishFinance1Retrieval
@@ -70,7 +71,7 @@ from .fashion200k_i2t_retrieval import Fashion200kI2TRetrieval
 from .fashion200k_t2i_retrieval import Fashion200kT2IRetrieval
 from .fashion_iq_it2i_retrieval import FashionIQIT2IRetrieval
 from .feedback_qa_retrieval import FeedbackQARetrieval
-from .fever_retrieval import FEVER, FEVERHardNegatives
+from .fever_retrieval import FEVER, FEVERHardNegatives, FEVERHardNegativesV2
 from .fi_qa2018_retrieval import FiQA2018
 from .fin_qa_retrieval import FinQARetrieval
 from .finance_bench_retrieval import FinanceBenchRetrieval
@@ -85,7 +86,11 @@ from .hateful_memes_i2t_retrieval import HatefulMemesI2TRetrieval
 from .hateful_memes_t2i_retrieval import HatefulMemesT2IRetrieval
 from .hc3_finance_retrieval import HC3FinanceRetrieval
 from .hella_swag_retrieval import HellaSwag
-from .hotpot_qa_retrieval import HotpotQA, HotpotQAHardNegatives
+from .hotpot_qa_retrieval import (
+    HotpotQA,
+    HotpotQAHardNegatives,
+    HotpotQAHardNegativesV2,
+)
 from .image_co_de_t2i_retrieval import ImageCoDeT2IRetrieval
 from .info_seek_it2it_retrieval import InfoSeekIT2ITRetrieval
 from .info_seek_it2t_retrieval import InfoSeekIT2TRetrieval
@@ -133,7 +138,11 @@ from .oven_it2it_retrieval import OVENIT2ITRetrieval
 from .oven_it2t_retrieval import OVENIT2TRetrieval
 from .piqa_retrieval import PIQA
 from .quail_retrieval import Quail
-from .quora_retrieval import QuoraRetrieval, QuoraRetrievalHardNegatives
+from .quora_retrieval import (
+    QuoraRetrieval,
+    QuoraRetrievalHardNegatives,
+    QuoraRetrievalHardNegativesV2,
+)
 from .r2_med_retrieval import (
     R2MEDBioinformaticsRetrieval,
     R2MEDBiologyRetrieval,
@@ -247,6 +256,7 @@ __all__ = [
     "ChemNQRetrieval",
     "ClimateFEVER",
     "ClimateFEVERHardNegatives",
+    "ClimateFEVERHardNegativesV2",
     "ClimateFEVERRetrievalv2",
     "DAPFAMAllTitlAbsClmToFullTextRetrieval",
     "DAPFAMAllTitlAbsClmToTitlAbsClmRetrieval",
@@ -268,6 +278,7 @@ __all__ = [
     "DAPFAMOutTitlAbsToTitlAbsRetrieval",
     "DBPedia",
     "DBPediaHardNegatives",
+    "DBPediaHardNegativesV2",
     "EDIST2ITRetrieval",
     "EncyclopediaVQAIT2ITRetrieval",
     "EnglishFinance1Retrieval",
@@ -276,6 +287,7 @@ __all__ = [
     "EnglishFinance4Retrieval",
     "EnglishHealthcare1Retrieval",
     "FEVERHardNegatives",
+    "FEVERHardNegativesV2",
     "FaithDialRetrieval",
     "Fashion200kI2TRetrieval",
     "Fashion200kT2IRetrieval",
@@ -296,6 +308,7 @@ __all__ = [
     "HellaSwag",
     "HotpotQA",
     "HotpotQAHardNegatives",
+    "HotpotQAHardNegativesV2",
     "ImageCoDeT2IRetrieval",
     "InfoSeekIT2ITRetrieval",
     "InfoSeekIT2TRetrieval",
@@ -345,6 +358,7 @@ __all__ = [
     "Quail",
     "QuoraRetrieval",
     "QuoraRetrievalHardNegatives",
+    "QuoraRetrievalHardNegativesV2",
     "R2MEDBioinformaticsRetrieval",
     "R2MEDBiologyRetrieval",
     "R2MEDIIYiClinicalRetrieval",