mteb 2.0.5__py3-none-any.whl → 2.1.19__py3-none-any.whl

mteb/tasks/pair_classification/multilingual/indic_xnli_pair_classification.py

@@ -23,14 +23,15 @@ class IndicXnliPairClassification(AbsTaskPairClassification):
             "path": "mteb/IndicXnliPairClassification",
             "revision": "027e97b9afe84ea3447b57b7705b8864bb2b3a83",
         },
-        description="""INDICXNLI is similar to existing XNLI dataset in shape/form, but
-        focusses on Indic language family.
-        The train (392,702), validation (2,490), and evaluation sets (5,010) of English
-        XNLI were translated from English into each of the eleven Indic languages. IndicTrans
-        is a large Transformer-based sequence to sequence model. It is trained on Samanantar
-        dataset (Ramesh et al., 2021), which is the largest parallel multi- lingual corpus
-        over eleven Indic languages.
-        """,
+        description=(
+            "INDICXNLI is similar to existing XNLI dataset in shape/form, but "
+            "focuses on Indic language family. "
+            "The train (392,702), validation (2,490), and evaluation sets (5,010) of English "
+            "XNLI were translated from English into each of the eleven Indic languages. IndicTrans "
+            "is a large Transformer-based sequence to sequence model. It is trained on Samanantar "
+            "dataset (Ramesh et al., 2021), which is the largest parallel multi- lingual corpus "
+            "over eleven Indic languages."
+        ),
         reference="https://gem-benchmark.com/data_cards/opusparcus",
         category="t2t",
         modalities=["text"],

mteb/tasks/pair_classification/nld/__init__.py

@@ -0,0 +1,7 @@
+from .sick_nl_pair_classification import SICKNLPairClassification
+from .xlwic_nl_pair_classification import XLWICNLPairClassification
+
+__all__ = [
+    "SICKNLPairClassification",
+    "XLWICNLPairClassification",
+]

mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py

@@ -0,0 +1,39 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class SICKNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="SICKNLPairClassification",
+        dataset={
+            "path": "clips/mteb-nl-sick-pcls-pr",
+            "revision": "a13a1892bcb4c077dc416d390389223eea5f20f0",
+        },
+        description="SICK-NL is a Dutch translation of SICK ",
+        reference="https://aclanthology.org/2021.eacl-main.126/",
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        date=("2020-09-01", "2021-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=[],
+        license="mit",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation=r"""
+@inproceedings{wijnholds2021sick,
+  author = {Wijnholds, Gijs and Moortgat, Michael},
+  booktitle = {Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume},
+  pages = {1474--1479},
+  title = {SICK-NL: A Dataset for Dutch Natural Language Inference},
+  year = {2021},
+}
+""",
+        prompt={
+            "query": "Zoek tekst die semantisch vergelijkbaar is met de gegeven tekst."
+        },
+    )

mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py

@@ -0,0 +1,44 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class XLWICNLPairClassification(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="XLWICNLPairClassification",
+        description="The Word-in-Context dataset (WiC) addresses the dependence on sense inventories by reformulating "
+        "the standard disambiguation task as a binary classification problem; but, it is limited to the "
+        "English language. We put forward a large multilingual benchmark, XL-WiC, featuring gold standards "
+        "in 12 new languages from varied language families and with different degrees of resource "
+        "availability, opening room for evaluation scenarios such as zero-shot cross-lingual transfer. ",
+        reference="https://aclanthology.org/2020.emnlp-main.584.pdf",
+        dataset={
+            "path": "clips/mteb-nl-xlwic",
+            "revision": "0b33ce358b1b5d500ff3715ba3d777b4d2c21cb0",
+        },
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        date=("2019-10-04", "2019-10-04"),
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="max_ap",
+        domains=["Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="created",
+        bibtex_citation=r"""
+@inproceedings{raganato2020xl,
+  author = {Raganato, A and Pasini, T and Camacho-Collados, J and Pilehvar, M and others},
+  booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
+  organization = {Association for Computational Linguistics (ACL)},
+  pages = {7193--7206},
+  title = {XL-WiC: A multilingual benchmark for evaluating semantic contextualization},
+  year = {2020},
+}
+""",
+        prompt={
+            "query": "Zoek tekst die semantisch vergelijkbaar is met de gegeven tekst."
+        },
+    )

mteb/tasks/pair_classification/vie/sprint_duplicate_questions_pcvn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SprintDuplicateQuestionsPCVN(AbsTaskPairClassification):
     metadata = TaskMetadata(
         name="SprintDuplicateQuestions-VN",
-        description="""A translated dataset from Duplicate questions from the Sprint community.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Duplicate questions from the Sprint community. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.aclweb.org/anthology/D18-1131/",
         dataset={
             "path": "GreenNode/sprintduplicatequestions-pairclassification-vn",

mteb/tasks/pair_classification/vie/twitter_sem_eval2015_pcvn.py

@@ -9,11 +9,7 @@ class TwitterSemEval2015PCVN(AbsTaskPairClassification):
             "path": "GreenNode/twittersemeval2015-pairclassification-vn",
             "revision": "9215a3c954078fd15c2bbecca914477d53944de1",
         },
-        description="""A translated dataset from Paraphrase-Pairs of Tweets from the SemEval 2015 workshop.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Paraphrase-Pairs of Tweets from the SemEval 2015 workshop. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://alt.qcri.org/semeval2015/task1/",
         category="t2c",
         type="PairClassification",

mteb/tasks/pair_classification/vie/twitter_url_corpus_pcvn.py

@@ -9,11 +9,7 @@ class TwitterURLCorpusPC(AbsTaskPairClassification):
             "path": "GreenNode/twitterurlcorpus-pairclassification-vn",
             "revision": "6e6a40aaade2129f70432f2156a6d24b63d72be3",
         },
-        description="""A translated dataset from Paraphrase-Pairs of Tweets.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Paraphrase-Pairs of Tweets. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://languagenet.github.io/",
         category="t2c",
         type="PairClassification",

mteb/tasks/regression/multilingual/ru_sci_bench_regression.py

@@ -5,9 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class RuSciBenchCitedCountRegression(AbsTaskRegression):
     metadata = TaskMetadata(
         name="RuSciBenchCitedCountRegression",
-        description="""Predicts the number of times a scientific article has been cited by other papers.
-        The prediction is based on the article's title and abstract. The data is sourced from the Russian electronic
-        library of scientific publications (eLibrary.ru) and includes papers with both Russian and English abstracts.""",
+        description="Predicts the number of times a scientific article has been cited by other papers. The prediction is based on the article's title and abstract. The data is sourced from the Russian electronic library of scientific publications (eLibrary.ru) and includes papers with both Russian and English abstracts.",
         reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
         dataset={
             "path": "mlsa-iai-msu-lab/ru_sci_bench_mteb",
@@ -51,9 +49,7 @@ class RuSciBenchCitedCountRegression(AbsTaskRegression):
 class RuSciBenchYearPublRegression(AbsTaskRegression):
     metadata = TaskMetadata(
         name="RuSciBenchYearPublRegression",
-        description="""Predicts the publication year of a scientific article. The prediction is based on the
-        article's title and abstract. The data is sourced from the Russian electronic library of scientific
-        publications (eLibrary.ru) and includes papers with both Russian and English abstracts.""",
+        description="Predicts the publication year of a scientific article. The prediction is based on the article's title and abstract. The data is sourced from the Russian electronic library of scientific publications (eLibrary.ru) and includes papers with both Russian and English abstracts.",
         reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
         dataset={
             "path": "mlsa-iai-msu-lab/ru_sci_bench_mteb",

mteb/tasks/reranking/multilingual/x_glue_wpr_reranking.py

@@ -78,8 +78,7 @@ _CITATION = r"""
 class XGlueWPRReranking(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="XGlueWPRReranking",
-        description="""XGLUE is a new benchmark dataset to evaluate the performance of cross-lingual pre-trained models
-        with respect to cross-lingual natural language understanding and generation. XGLUE is composed of 11 tasks spans 19 languages.""",
+        description="XGLUE is a new benchmark dataset to evaluate the performance of cross-lingual pre-trained models with respect to cross-lingual natural language understanding and generation. XGLUE is composed of 11 tasks spans 19 languages.",
         reference="https://github.com/microsoft/XGLUE",
         dataset={
             "path": "mteb/XGlueWPRReranking",

mteb/tasks/reranking/vie/ask_ubuntu_dup_questions_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class AskUbuntuDupQuestionsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="AskUbuntuDupQuestions-VN",
-        description="""A translated dataset from AskUbuntu Question Dataset - Questions from AskUbuntu with manual annotations marking pairs of questions as similar or non-similar
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from AskUbuntu Question Dataset - Questions from AskUbuntu with manual annotations marking pairs of questions as similar or non-similar The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://github.com/taolei87/askubuntu",
         dataset={
             "path": "mteb/AskUbuntuDupQuestions-VN",

mteb/tasks/reranking/vie/sci_docs_reranking_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SciDocsRerankingVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="SciDocsRR-VN",
-        description="""A translated dataset from Ranking of related scientific papers based on their title.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Ranking of related scientific papers based on their title. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://allenai.org/data/scidocs",
         dataset={
             "path": "mteb/SciDocsRR-VN",

mteb/tasks/reranking/vie/stack_overflow_dup_questions_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class StackOverflowDupQuestionsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="StackOverflowDupQuestions-VN",
-        description="""A translated dataset from Stack Overflow Duplicate Questions Task for questions with the tags Java, JavaScript and Python
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Stack Overflow Duplicate Questions Task for questions with the tags Java, JavaScript and Python The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.microsoft.com/en-us/research/uploads/prod/2019/03/nl4se18LinkSO.pdf",
         dataset={
             "path": "mteb/StackOverflowDupQuestions-VN",

mteb/tasks/retrieval/code/code_rag.py

@@ -60,9 +60,9 @@ class CodeRAGProgrammingSolutionsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """
@@ -117,9 +117,9 @@ class CodeRAGOnlineTutorialsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """
@@ -177,9 +177,9 @@ class CodeRAGLibraryDocumentationSolutionsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """
@@ -234,9 +234,9 @@ class CodeRAGStackoverflowPostsRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = Dict[doc_id, Dict[str, str]] #id => dict with document data like title and text
         self.queries = Dict[query_id, str] #id => query
         self.relevant_docs = Dict[query_id, Dict[[doc_id, score]]
         """

mteb/tasks/retrieval/dan/dan_fever_retrieval.py

@@ -56,7 +56,7 @@ Derczynski, Leon},
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
         self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text
         self.queries = dict[query_id, str] #id => query

mteb/tasks/retrieval/dan/tv2_nordretrieval.py

@@ -69,9 +69,9 @@ Piperidis, Stelios},
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text
         self.queries = dict[query_id, str] #id => query
         self.relevant_docs = dict[query_id, dict[[doc_id, score]]
         """

mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py

@@ -45,9 +45,9 @@ class TwitterHjerneRetrieval(AbsTaskRetrieval):
         self.data_loaded = True
 
     def dataset_transform(self) -> None:
-        """And transform to a retrieval datset, which have the following attributes
+        """And transform to a retrieval dataset, which have the following attributes
 
-        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document datas like title and text
+        self.corpus = dict[doc_id, dict[str, str]] #id => dict with document data like title and text
         self.queries = dict[query_id, str] #id => query
         self.relevant_docs = dict[query_id, dict[[doc_id, score]]
         """

mteb/tasks/retrieval/eng/__init__.py

@@ -22,6 +22,7 @@ from .cirr_it2i_retrieval import CIRRIT2IRetrieval
 from .climate_fever_retrieval import (
     ClimateFEVER,
     ClimateFEVERHardNegatives,
+    ClimateFEVERHardNegativesV2,
     ClimateFEVERRetrievalv2,
 )
 from .cqa_dupstack_android_retrieval import CQADupstackAndroidRetrieval
@@ -57,7 +58,7 @@ from .dapfam_patent_retrieval import (
     DAPFAMOutTitlAbsToTitlAbsClmRetrieval,
     DAPFAMOutTitlAbsToTitlAbsRetrieval,
 )
-from .dbpedia_retrieval import DBPedia, DBPediaHardNegatives
+from .dbpedia_retrieval import DBPedia, DBPediaHardNegatives, DBPediaHardNegativesV2
 from .edis_t2it_retrieval import EDIST2ITRetrieval
 from .encyclopedia_vqa_it2it_retrieval import EncyclopediaVQAIT2ITRetrieval
 from .english_finance1_retrieval import EnglishFinance1Retrieval
@@ -70,7 +71,7 @@ from .fashion200k_i2t_retrieval import Fashion200kI2TRetrieval
 from .fashion200k_t2i_retrieval import Fashion200kT2IRetrieval
 from .fashion_iq_it2i_retrieval import FashionIQIT2IRetrieval
 from .feedback_qa_retrieval import FeedbackQARetrieval
-from .fever_retrieval import FEVER, FEVERHardNegatives
+from .fever_retrieval import FEVER, FEVERHardNegatives, FEVERHardNegativesV2
 from .fi_qa2018_retrieval import FiQA2018
 from .fin_qa_retrieval import FinQARetrieval
 from .finance_bench_retrieval import FinanceBenchRetrieval
@@ -85,7 +86,11 @@ from .hateful_memes_i2t_retrieval import HatefulMemesI2TRetrieval
 from .hateful_memes_t2i_retrieval import HatefulMemesT2IRetrieval
 from .hc3_finance_retrieval import HC3FinanceRetrieval
 from .hella_swag_retrieval import HellaSwag
-from .hotpot_qa_retrieval import HotpotQA, HotpotQAHardNegatives
+from .hotpot_qa_retrieval import (
+    HotpotQA,
+    HotpotQAHardNegatives,
+    HotpotQAHardNegativesV2,
+)
 from .image_co_de_t2i_retrieval import ImageCoDeT2IRetrieval
 from .info_seek_it2it_retrieval import InfoSeekIT2ITRetrieval
 from .info_seek_it2t_retrieval import InfoSeekIT2TRetrieval
@@ -133,7 +138,11 @@ from .oven_it2it_retrieval import OVENIT2ITRetrieval
 from .oven_it2t_retrieval import OVENIT2TRetrieval
 from .piqa_retrieval import PIQA
 from .quail_retrieval import Quail
-from .quora_retrieval import QuoraRetrieval, QuoraRetrievalHardNegatives
+from .quora_retrieval import (
+    QuoraRetrieval,
+    QuoraRetrievalHardNegatives,
+    QuoraRetrievalHardNegativesV2,
+)
 from .r2_med_retrieval import (
     R2MEDBioinformaticsRetrieval,
     R2MEDBiologyRetrieval,
@@ -247,6 +256,7 @@ __all__ = [
     "ChemNQRetrieval",
     "ClimateFEVER",
     "ClimateFEVERHardNegatives",
+    "ClimateFEVERHardNegativesV2",
     "ClimateFEVERRetrievalv2",
     "DAPFAMAllTitlAbsClmToFullTextRetrieval",
     "DAPFAMAllTitlAbsClmToTitlAbsClmRetrieval",
@@ -268,6 +278,7 @@ __all__ = [
     "DAPFAMOutTitlAbsToTitlAbsRetrieval",
     "DBPedia",
     "DBPediaHardNegatives",
+    "DBPediaHardNegativesV2",
     "EDIST2ITRetrieval",
     "EncyclopediaVQAIT2ITRetrieval",
     "EnglishFinance1Retrieval",
@@ -276,6 +287,7 @@ __all__ = [
     "EnglishFinance4Retrieval",
     "EnglishHealthcare1Retrieval",
     "FEVERHardNegatives",
+    "FEVERHardNegativesV2",
     "FaithDialRetrieval",
     "Fashion200kI2TRetrieval",
     "Fashion200kT2IRetrieval",
@@ -296,6 +308,7 @@ __all__ = [
     "HellaSwag",
     "HotpotQA",
     "HotpotQAHardNegatives",
+    "HotpotQAHardNegativesV2",
     "ImageCoDeT2IRetrieval",
     "InfoSeekIT2ITRetrieval",
     "InfoSeekIT2TRetrieval",
@@ -345,6 +358,7 @@ __all__ = [
     "Quail",
     "QuoraRetrieval",
     "QuoraRetrievalHardNegatives",
+    "QuoraRetrievalHardNegativesV2",
     "R2MEDBioinformaticsRetrieval",
     "R2MEDBiologyRetrieval",
     "R2MEDIIYiClinicalRetrieval",

mteb/tasks/retrieval/eng/climate_fever_retrieval.py

@@ -1,30 +1,21 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class ClimateFEVER(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="ClimateFEVER",
-        description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims (queries) regarding climate-change. The underlying corpus is the same as FVER.",
-        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
-        dataset={
-            "path": "mteb/climate-fever",
-            "revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
-        },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
+_climate_fever_metadata = dict(
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["eng-Latn"],
+    main_score="ndcg_at_10",
+    date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
+    domains=["Encyclopaedic", "Written"],
+    task_subtypes=["Claim verification"],
+    license="cc-by-sa-4.0",
+    annotations_creators="human-annotated",
+    dialect=[],
+    sample_creation="found",
+    bibtex_citation=r"""
 @misc{diggelmann2021climatefever,
   archiveprefix = {arXiv},
   author = {Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
@@ -34,82 +25,82 @@ class ClimateFEVER(AbsTaskRetrieval):
   year = {2021},
 }
 """,
+)
+
+
+class ClimateFEVER(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="ClimateFEVER",
+        description=(
+            "CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims "
+            "(queries) regarding climate-change. The underlying corpus is the same as FEVER."
+        ),
+        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
+        dataset={
+            "path": "mteb/climate-fever",
+            "revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
+        },
+        prompt={
+            "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
+        },
+        **_climate_fever_metadata,
+    )
+
+
+class ClimateFEVERRetrievalv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="ClimateFEVER.v2",
+        description=(
+            "CLIMATE-FEVER is a dataset following the FEVER methodology, containing 1,535 real-world climate change claims. "
+            "This updated version addresses corpus mismatches and qrel inconsistencies in MTEB, restoring labels while refining corpus-query alignment for better accuracy."
+        ),
+        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
+        dataset={
+            "path": "mteb/climate-fever-v2",
+            "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4",
+        },
         prompt={
             "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
         },
+        adapted_from=["ClimateFEVER"],
+        **_climate_fever_metadata,
     )
 
 
 class ClimateFEVERHardNegatives(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ClimateFEVERHardNegatives",
-        description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.",
+        description=(
+            "CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+        ),
         reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
         dataset={
             "path": "mteb/ClimateFEVER_test_top_250_only_w_correct-v2",
             "revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21",
         },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
-@misc{diggelmann2021climatefever,
-  archiveprefix = {arXiv},
-  author = {Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
-  eprint = {2012.00614},
-  primaryclass = {cs.CL},
-  title = {CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims},
-  year = {2021},
-}
-""",
         adapted_from=["ClimateFEVER"],
+        superseded_by="ClimateFEVERHardNegatives.v2",
+        **_climate_fever_metadata,
     )
 
 
-class ClimateFEVERRetrievalv2(AbsTaskRetrieval):
+class ClimateFEVERHardNegativesV2(AbsTaskRetrieval):
     metadata = TaskMetadata(
-        name="ClimateFEVER.v2",
-        description="CLIMATE-FEVER is a dataset following the FEVER methodology, containing 1,535 real-world climate change claims. This updated version addresses corpus mismatches and qrel inconsistencies in MTEB, restoring labels while refining corpus-query alignment for better accuracy. ",
+        name="ClimateFEVERHardNegatives.v2",
+        description=(
+            "CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct. "
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval. You can get more information on the effect of different prompt in the [PR](https://github.com/embeddings-benchmark/mteb/pull/3469#issuecomment-3436467106)"
+        ),
         reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
         dataset={
-            "path": "mteb/climate-fever-v2",
-            "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4",
+            "path": "mteb/ClimateFEVER_test_top_250_only_w_correct-v2",
+            "revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21",
         },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
-        domains=["Academic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
-@misc{diggelmann2021climatefever,
-  archiveprefix = {arXiv},
-  author = {Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
-  eprint = {2012.00614},
-  primaryclass = {cs.CL},
-  title = {CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims},
-  year = {2021},
-}
-""",
+        adapted_from=["ClimateFEVER"],
         prompt={
             "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
         },
-        adapted_from=["ClimateFEVER"],
+        **_climate_fever_metadata,
     )