mteb 2.0.5__py3-none-any.whl → 2.1.1__py3-none-any.whl

mteb/tasks/retrieval/eng/climate_fever_retrieval.py

@@ -1,30 +1,21 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class ClimateFEVER(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="ClimateFEVER",
-        description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims (queries) regarding climate-change. The underlying corpus is the same as FVER.",
-        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
-        dataset={
-            "path": "mteb/climate-fever",
-            "revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
-        },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
+_climate_fever_metadata = dict(
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["eng-Latn"],
+    main_score="ndcg_at_10",
+    date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
+    domains=["Encyclopaedic", "Written"],
+    task_subtypes=["Claim verification"],
+    license="cc-by-sa-4.0",
+    annotations_creators="human-annotated",
+    dialect=[],
+    sample_creation="found",
+    bibtex_citation=r"""
 @misc{diggelmann2021climatefever,
   archiveprefix = {arXiv},
   author = {Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
@@ -34,82 +25,82 @@ class ClimateFEVER(AbsTaskRetrieval):
   year = {2021},
 }
 """,
+)
+
+
+class ClimateFEVER(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="ClimateFEVER",
+        description=(
+            "CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims "
+            "(queries) regarding climate-change. The underlying corpus is the same as FEVER."
+        ),
+        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
+        dataset={
+            "path": "mteb/climate-fever",
+            "revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
+        },
+        prompt={
+            "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
+        },
+        **_climate_fever_metadata,
+    )
+
+
+class ClimateFEVERRetrievalv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="ClimateFEVER.v2",
+        description=(
+            "CLIMATE-FEVER is a dataset following the FEVER methodology, containing 1,535 real-world climate change claims. "
+            "This updated version addresses corpus mismatches and qrel inconsistencies in MTEB, restoring labels while refining corpus-query alignment for better accuracy."
+        ),
+        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
+        dataset={
+            "path": "mteb/climate-fever-v2",
+            "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4",
+        },
         prompt={
             "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
         },
+        adapted_from=["ClimateFEVER"],
+        **_climate_fever_metadata,
     )
 
 
 class ClimateFEVERHardNegatives(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ClimateFEVERHardNegatives",
-        description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.",
+        description=(
+            "CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+        ),
         reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
         dataset={
             "path": "mteb/ClimateFEVER_test_top_250_only_w_correct-v2",
             "revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21",
         },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
-@misc{diggelmann2021climatefever,
-  archiveprefix = {arXiv},
-  author = {Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
-  eprint = {2012.00614},
-  primaryclass = {cs.CL},
-  title = {CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims},
-  year = {2021},
-}
-""",
         adapted_from=["ClimateFEVER"],
+        superseded_by="ClimateFEVERHardNegatives.v2",
+        **_climate_fever_metadata,
     )
 
 
-class ClimateFEVERRetrievalv2(AbsTaskRetrieval):
+class ClimateFEVERHardNegativesV2(AbsTaskRetrieval):
     metadata = TaskMetadata(
-        name="ClimateFEVER.v2",
-        description="CLIMATE-FEVER is a dataset following the FEVER methodology, containing 1,535 real-world climate change claims. This updated version addresses corpus mismatches and qrel inconsistencies in MTEB, restoring labels while refining corpus-query alignment for better accuracy. ",
+        name="ClimateFEVERHardNegatives.v2",
+        description=(
+            "CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct. "
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval. You can get more information on the effect of different prompt in the [PR](https://github.com/embeddings-benchmark/mteb/pull/3469#issuecomment-3436467106)"
+        ),
         reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
         dataset={
-            "path": "mteb/climate-fever-v2",
-            "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4",
+            "path": "mteb/ClimateFEVER_test_top_250_only_w_correct-v2",
+            "revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21",
         },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2001-01-01", "2020-12-31"),  # launch of wiki -> paper publication
-        domains=["Academic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
-@misc{diggelmann2021climatefever,
-  archiveprefix = {arXiv},
-  author = {Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
-  eprint = {2012.00614},
-  primaryclass = {cs.CL},
-  title = {CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims},
-  year = {2021},
-}
-""",
+        adapted_from=["ClimateFEVER"],
         prompt={
             "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
         },
-        adapted_from=["ClimateFEVER"],
+        **_climate_fever_metadata,
     )

mteb/tasks/retrieval/eng/dbpedia_retrieval.py

@@ -1,30 +1,21 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class DBPedia(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="DBPedia",
-        description="DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base",
-        reference="https://github.com/iai-group/DBpedia-Entity/",
-        dataset={
-            "path": "mteb/dbpedia",
-            "revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
-        },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2017-01-01", "2017-01-01"),  # best guess: based on publication date
-        domains=["Written", "Encyclopaedic"],
-        task_subtypes=[],
-        license="mit",
-        annotations_creators="derived",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
+_dbpedia_metadata = dict(
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["eng-Latn"],
+    main_score="ndcg_at_10",
+    date=("2017-01-01", "2017-01-01"),  # best guess: based on publication date
+    domains=["Written", "Encyclopaedic"],
+    task_subtypes=[],
+    license="mit",
+    annotations_creators="derived",
+    dialect=[],
+    sample_creation="found",
+    bibtex_citation=r"""
 @inproceedings{Hasibi:2017:DVT,
   author = {Hasibi, Faegheh and Nikolaev, Fedor and Xiong, Chenyan and Balog, Krisztian and Bratsberg, Svein Erik and Kotov, Alexander and Callan, Jamie},
   booktitle = {Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval},
@@ -36,45 +27,59 @@ class DBPedia(AbsTaskRetrieval):
   year = {2017},
 }
 """,
+)
+
+
+class DBPedia(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="DBPedia",
+        description="DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base",
+        reference="https://github.com/iai-group/DBpedia-Entity/",
+        dataset={
+            "path": "mteb/dbpedia",
+            "revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
+        },
         prompt={
             "query": "Given a query, retrieve relevant entity descriptions from DBPedia"
         },
+        **_dbpedia_metadata,
     )
 
 
 class DBPediaHardNegatives(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="DBPediaHardNegatives",
-        description="DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.",
+        description=(
+            "DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+        ),
+        reference="https://github.com/iai-group/DBpedia-Entity/",
+        dataset={
+            "path": "mteb/DBPedia_test_top_250_only_w_correct-v2",
+            "revision": "943ec7fdfef3728b2ad1966c5b6479ff9ffd26c9",
+        },
+        superseded_by="DBPediaHardNegatives.v2",
+        adapted_from=["DBPedia"],
+        **_dbpedia_metadata,
+    )
+
+
+class DBPediaHardNegativesV2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="DBPediaHardNegatives.v2",
+        description=(
+            "DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct. "
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval. You can get more information on the effect of different prompt in the [PR](https://github.com/embeddings-benchmark/mteb/pull/3469#issuecomment-3436467106)"
+        ),
         reference="https://github.com/iai-group/DBpedia-Entity/",
         dataset={
             "path": "mteb/DBPedia_test_top_250_only_w_correct-v2",
             "revision": "943ec7fdfef3728b2ad1966c5b6479ff9ffd26c9",
         },
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2017-01-01", "2017-01-01"),  # best guess: based on publication date
-        domains=["Written", "Encyclopaedic"],
-        task_subtypes=[],
-        license="mit",
-        annotations_creators="derived",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
-@inproceedings{Hasibi:2017:DVT,
-  author = {Hasibi, Faegheh and Nikolaev, Fedor and Xiong, Chenyan and Balog, Krisztian and Bratsberg, Svein Erik and Kotov, Alexander and Callan, Jamie},
-  booktitle = {Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval},
-  doi = {10.1145/3077136.3080751},
-  pages = {1265--1268},
-  publisher = {ACM},
-  series = {SIGIR '17},
-  title = {DBpedia-Entity V2: A Test Collection for Entity Search},
-  year = {2017},
-}
-""",
         adapted_from=["DBPedia"],
+        prompt={
+            "query": "Given a query, retrieve relevant entity descriptions from DBPedia"
+        },
+        **_dbpedia_metadata,
     )

mteb/tasks/retrieval/eng/fever_retrieval.py

@@ -1,36 +1,22 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class FEVER(AbsTaskRetrieval):
-    ignore_identical_ids = True
-
-    metadata = TaskMetadata(
-        name="FEVER",
-        dataset={
-            "path": "mteb/fever",
-            "revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
-        },
-        description=(
-            "FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences"
-            + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were"
-            + " derived from."
-        ),
-        reference="https://fever.ai/",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=None,
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-nc-sa-3.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
+_fever_metadata = dict(
+    reference="https://fever.ai/",
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["eng-Latn"],
+    main_score="ndcg_at_10",
+    date=None,
+    domains=["Encyclopaedic", "Written"],
+    task_subtypes=["Claim verification"],
+    license="cc-by-nc-sa-3.0",
+    annotations_creators="human-annotated",
+    dialect=[],
+    sample_creation="found",
+    bibtex_citation=r"""
 @inproceedings{thorne-etal-2018-fever,
   address = {New Orleans, Louisiana},
   author = {Thorne, James  and
@@ -50,9 +36,27 @@ Stent, Amanda},
   year = {2018},
 }
 """,
+)
+
+
+class FEVER(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="FEVER",
+        dataset={
+            "path": "mteb/fever",
+            "revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
+        },
+        description=(
+            "FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences "
+            "extracted from Wikipedia and subsequently verified without knowledge of the sentence they were "
+            "derived from."
+        ),
         prompt={
             "query": "Given a claim, retrieve documents that support or refute the claim"
         },
+        **_fever_metadata,
     )
 
 
@@ -66,43 +70,34 @@ class FEVERHardNegatives(AbsTaskRetrieval):
             "revision": "080c9ed6267b65029207906e815d44a9240bafca",
         },
         description=(
-            "FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences"
-            + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were"
-            + " derived from. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+            "FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences "
+            "extracted from Wikipedia and subsequently verified without knowledge of the sentence they were "
+            "derived from. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+        ),
+        adapted_from=["FEVER"],
+        superseded_by="FEVERHardNegatives.v2",
+        **_fever_metadata,
+    )
+
+
+class FEVERHardNegativesV2(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="FEVERHardNegatives.v2",
+        dataset={
+            "path": "mteb/FEVER_test_top_250_only_w_correct-v2",
+            "revision": "080c9ed6267b65029207906e815d44a9240bafca",
+        },
+        description=(
+            "FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences "
+            "extracted from Wikipedia and subsequently verified without knowledge of the sentence they were "
+            "derived from. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct. "
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval. You can get more information on the effect of different prompt in the [PR](https://github.com/embeddings-benchmark/mteb/pull/3469#issuecomment-3436467106)"
         ),
-        reference="https://fever.ai/",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=None,
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=["Claim verification"],
-        license="cc-by-nc-sa-3.0",
-        annotations_creators="human-annotated",
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation=r"""
-@inproceedings{thorne-etal-2018-fever,
-  address = {New Orleans, Louisiana},
-  author = {Thorne, James  and
-Vlachos, Andreas  and
-Christodoulopoulos, Christos  and
-Mittal, Arpit},
-  booktitle = {Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},
-  doi = {10.18653/v1/N18-1074},
-  editor = {Walker, Marilyn  and
-Ji, Heng  and
-Stent, Amanda},
-  month = jun,
-  pages = {809--819},
-  publisher = {Association for Computational Linguistics},
-  title = {{FEVER}: a Large-scale Dataset for Fact Extraction and {VER}ification},
-  url = {https://aclanthology.org/N18-1074},
-  year = {2018},
-}
-""",
         adapted_from=["FEVER"],
+        prompt={
+            "query": "Given a claim, retrieve documents that support or refute the claim"
+        },
+        **_fever_metadata,
     )

mteb/tasks/retrieval/eng/hateful_memes_i2t_retrieval.py

@@ -24,9 +24,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
     shared_corpus = shared_corpus.map(
         lambda x: {
             "id": "corpus-" + str(x["id"]),
-            # "text": x["text"],
             "modality": "text",
-            "image": None,
         },
         remove_columns=[
             "split",
@@ -40,9 +38,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
         queries[split] = split_dataset.map(
             lambda x: {
                 "id": "query-" + str(x["id"]),
-                "text": None,
                 "modality": "image",
-                # "image": x["image"],
             },
             remove_columns=[
                 "split",

mteb/tasks/retrieval/eng/hateful_memes_t2i_retrieval.py

@@ -24,9 +24,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
     shared_corpus = shared_corpus.map(
         lambda x: {
             "id": "corpus-" + str(x["id"]),
-            "text": None,
             "modality": "image",
-            # "image": None,
         },
         remove_columns=[
             "split",
@@ -40,9 +38,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
         queries[split] = split_dataset.map(
             lambda x: {
                 "id": "query-" + str(x["id"]),
-                # "text": None,
                 "modality": "text",
-                "image": None,
             },
             remove_columns=[
                 "split",

mteb/tasks/retrieval/eng/hotpot_qa_retrieval.py

@@ -1,33 +1,22 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class HotpotQA(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="HotpotQA",
-        dataset={
-            "path": "mteb/hotpotqa",
-            "revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
-        },
-        description=(
-            "HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong"
-            + " supervision for supporting facts to enable more explainable question answering systems."
-        ),
-        reference="https://hotpotqa.github.io/",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2018-01-01", "2018-12-31"),  # best guess: based on publication date
-        domains=["Web", "Written"],
-        task_subtypes=["Question answering"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
+_hotpot_qa_metadata = dict(
+    reference="https://hotpotqa.github.io/",
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["eng-Latn"],
+    main_score="ndcg_at_10",
+    date=("2018-01-01", "2018-12-31"),  # best guess: based on publication date
+    domains=["Web", "Written"],
+    task_subtypes=["Question answering"],
+    license="cc-by-sa-4.0",
+    annotations_creators="human-annotated",
+    dialect=[],
+    sample_creation="found",
+    bibtex_citation=r"""
 @inproceedings{yang-etal-2018-hotpotqa,
   address = {Brussels, Belgium},
   author = {Yang, Zhilin  and
@@ -51,9 +40,24 @@ Tsujii, Jun{'}ichi},
   year = {2018},
 }
 """,
+)
+
+
+class HotpotQA(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="HotpotQA",
+        dataset={
+            "path": "mteb/hotpotqa",
+            "revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
+        },
+        description=(
+            "HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong "
+            "supervision for supporting facts to enable more explainable question answering systems."
+        ),
         prompt={
             "query": "Given a multi-hop question, retrieve documents that can help answer the question"
         },
+        **_hotpot_qa_metadata,
     )
 
 
@@ -65,46 +69,32 @@ class HotpotQAHardNegatives(AbsTaskRetrieval):
             "revision": "617612fa63afcb60e3b134bed8b7216a99707c37",
         },
         description=(
-            "HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong"
-            + " supervision for supporting facts to enable more explainable question answering systems.  The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+            "HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong "
+            "supervision for supporting facts to enable more explainable question answering systems. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
         ),
-        reference="https://hotpotqa.github.io/",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=("2018-01-01", "2018-12-31"),  # best guess: based on publication date
-        domains=["Web", "Written"],
-        task_subtypes=["Question answering"],
-        license="cc-by-sa-4.0",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
-@inproceedings{yang-etal-2018-hotpotqa,
-  address = {Brussels, Belgium},
-  author = {Yang, Zhilin  and
-Qi, Peng  and
-Zhang, Saizheng  and
-Bengio, Yoshua  and
-Cohen, William  and
-Salakhutdinov, Ruslan  and
-Manning, Christopher D.},
-  booktitle = {Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
-  doi = {10.18653/v1/D18-1259},
-  editor = {Riloff, Ellen  and
-Chiang, David  and
-Hockenmaier, Julia  and
-Tsujii, Jun{'}ichi},
-  month = oct # {-} # nov,
-  pages = {2369--2380},
-  publisher = {Association for Computational Linguistics},
-  title = {{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering},
-  url = {https://aclanthology.org/D18-1259},
-  year = {2018},
-}
-""",
         adapted_from=["HotpotQA"],
+        superseded_by="HotpotQAHardNegatives.v2",
+        **_hotpot_qa_metadata,
+    )
+
+
+class HotpotQAHardNegativesV2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="HotpotQAHardNegatives.v2",
+        dataset={
+            "path": "mteb/HotpotQA_test_top_250_only_w_correct-v2",
+            "revision": "617612fa63afcb60e3b134bed8b7216a99707c37",
+        },
+        description=(
+            "HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong "
+            "supervision for supporting facts to enable more explainable question answering systems. "
+            "The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval. You can get more information on the effect of different prompt in the [PR](https://github.com/embeddings-benchmark/mteb/pull/3469#issuecomment-3436467106)"
+        ),
+        adapted_from=["HotpotQA"],
+        prompt={
+            "query": "Given a multi-hop question, retrieve documents that can help answer the question"
+        },
+        **_hotpot_qa_metadata,
     )

mteb/tasks/retrieval/eng/legal_summarization_retrieval.py

@@ -5,7 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class LegalSummarization(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="LegalSummarization",
-        description="The dataset consistes of 439 pairs of contracts and their summarizations from https://tldrlegal.com and https://tosdr.org/.",
+        description="The dataset consists of 439 pairs of contracts and their summarizations from https://tldrlegal.com and https://tosdr.org/.",
         reference="https://github.com/lauramanor/legal_summarization",
         dataset={
             "path": "mteb/legal_summarization",