mteb 2.1.7__py3-none-any.whl → 2.1.8__py3-none-any.whl

mteb/tasks/reranking/vie/sci_docs_reranking_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SciDocsRerankingVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="SciDocsRR-VN",
-        description="""A translated dataset from Ranking of related scientific papers based on their title.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Ranking of related scientific papers based on their title. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://allenai.org/data/scidocs",
         dataset={
             "path": "mteb/SciDocsRR-VN",

mteb/tasks/reranking/vie/stack_overflow_dup_questions_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class StackOverflowDupQuestionsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="StackOverflowDupQuestions-VN",
-        description="""A translated dataset from Stack Overflow Duplicate Questions Task for questions with the tags Java, JavaScript and Python
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Stack Overflow Duplicate Questions Task for questions with the tags Java, JavaScript and Python The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.microsoft.com/en-us/research/uploads/prod/2019/03/nl4se18LinkSO.pdf",
         dataset={
             "path": "mteb/StackOverflowDupQuestions-VN",

mteb/tasks/retrieval/eng/lit_search_retrieval.py

@@ -7,14 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class LitSearchRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="LitSearchRetrieval",
-        description="""
-        The dataset contains the query set and retrieval corpus for the paper LitSearch: A Retrieval Benchmark for
-        Scientific Literature Search. It introduces LitSearch, a retrieval benchmark comprising 597 realistic literature
-        search queries about recent ML and NLP papers. LitSearch is constructed using a combination of (1) questions
-        generated by GPT-4 based on paragraphs containing inline citations from research papers and (2) questions about
-        recently published papers, manually written by their authors. All LitSearch questions were manually examined or
-        edited by experts to ensure high quality.
-        """,
+        description="The dataset contains the query set and retrieval corpus for the paper LitSearch: A Retrieval Benchmark for Scientific Literature Search. It introduces LitSearch, a retrieval benchmark comprising 597 realistic literature search queries about recent ML and NLP papers. LitSearch is constructed using a combination of (1) questions generated by GPT-4 based on paragraphs containing inline citations from research papers and (2) questions about recently published papers, manually written by their authors. All LitSearch questions were manually examined or edited by experts to ensure high quality.",
         reference="https://github.com/princeton-nlp/LitSearch",
         dataset={
             "path": "princeton-nlp/LitSearch",

mteb/tasks/retrieval/jpn/ja_cwir_retrieval.py

@@ -9,10 +9,7 @@ class JaCWIRRetrieval(AbsTaskRetrieval):
 
     metadata = TaskMetadata(
         name="JaCWIRRetrieval",
-        description="""JaCWIR is a small-scale Japanese information retrieval evaluation dataset consisting of
-5000 question texts and approximately 500k web page titles and web page introductions or summaries
-(meta descriptions, etc.). The question texts are created based on one of the 500k web pages,
-and that data is used as a positive example for the question text.""",
+        description="JaCWIR is a small-scale Japanese information retrieval evaluation dataset consisting of 5000 question texts and approximately 500k web page titles and web page introductions or summaries (meta descriptions, etc.). The question texts are created based on one of the 500k web pages, and that data is used as a positive example for the question text.",
         reference="https://huggingface.co/datasets/hotchpotch/JaCWIR",
         dataset={
             "path": "mteb/JaCWIRRetrieval",

mteb/tasks/retrieval/multilingual/__init__.py

@@ -81,6 +81,18 @@ from .vidore2_bench_retrieval import (
     Vidore2ESGReportsHLRetrieval,
     Vidore2ESGReportsRetrieval,
 )
+from .vidore3_bench_retrieval import (
+    Vidore3ComputerScienceRetrieval,
+    Vidore3EnergyRetrieval,
+    Vidore3FinanceEnRetrieval,
+    Vidore3FinanceFrRetrieval,
+    Vidore3HrRetrieval,
+    Vidore3IndustrialRetrieval,
+    Vidore3NuclearRetrieval,
+    Vidore3PharmaceuticalsRetrieval,
+    Vidore3PhysicsRetrieval,
+    Vidore3TelecomRetrieval,
+)
 from .web_faq_retrieval import WebFAQRetrieval
 from .wikipedia_retrieval_multilingual import WikipediaRetrievalMultilingual
 from .wit_t2i_retrieval import WITT2IRetrieval
@@ -161,6 +173,16 @@ __all__ = [
     "Vidore2ESGReportsHLRetrieval",
     "Vidore2ESGReportsRetrieval",
     "Vidore2EconomicsReportsRetrieval",
+    "Vidore3ComputerScienceRetrieval",
+    "Vidore3EnergyRetrieval",
+    "Vidore3FinanceEnRetrieval",
+    "Vidore3FinanceFrRetrieval",
+    "Vidore3HrRetrieval",
+    "Vidore3IndustrialRetrieval",
+    "Vidore3NuclearRetrieval",
+    "Vidore3PharmaceuticalsRetrieval",
+    "Vidore3PhysicsRetrieval",
+    "Vidore3TelecomRetrieval",
     "WITT2IRetrieval",
     "WebFAQRetrieval",
     "WikipediaRetrievalMultilingual",

mteb/tasks/retrieval/multilingual/mkqa_retrieval.py

@@ -34,8 +34,7 @@ _EVAL_LANGS = {
 class MKQARetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="MKQARetrieval",
-        description="""Multilingual Knowledge Questions & Answers (MKQA)contains 10,000 queries sampled from the Google Natural Questions dataset.
-        For each query we collect new passage-independent answers. These queries and answers are then human translated into 25 Non-English languages.""",
+        description="Multilingual Knowledge Questions & Answers (MKQA)contains 10,000 queries sampled from the Google Natural Questions dataset. For each query we collect new passage-independent answers. These queries and answers are then human translated into 25 Non-English languages.",
         reference="https://github.com/apple/ml-mkqa",
         dataset={
             "path": "mteb/MKQARetrieval",

mteb/tasks/retrieval/multilingual/mlqa_retrieval.py

@@ -75,10 +75,7 @@ _EVAL_LANGS = extend_lang_pairs()
 class MLQARetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="MLQARetrieval",
-        description="""MLQA (MultiLingual Question Answering) is a benchmark dataset for evaluating cross-lingual question answering performance.
-        MLQA consists of over 5K extractive QA instances (12K in English) in SQuAD format in seven languages - English, Arabic,
-        German, Spanish, Hindi, Vietnamese and Simplified Chinese. MLQA is highly parallel, with QA instances parallel between
-        4 different languages on average.""",
+        description="MLQA (MultiLingual Question Answering) is a benchmark dataset for evaluating cross-lingual question answering performance. MLQA consists of over 5K extractive QA instances (12K in English) in SQuAD format in seven languages - English, Arabic, German, Spanish, Hindi, Vietnamese and Simplified Chinese. MLQA is highly parallel, with QA instances parallel between 4 different languages on average.",
         reference="https://huggingface.co/datasets/mlqa",
         dataset={
             "path": "mteb/MLQARetrieval",

mteb/tasks/retrieval/multilingual/multi_long_doc_retrieval.py

@@ -21,8 +21,7 @@ _LANGUAGES = {
 class MultiLongDocRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="MultiLongDocRetrieval",
-        description="""Multi Long Doc Retrieval (MLDR) 'is curated by the multilingual articles from Wikipedia, Wudao and mC4 (see Table 7), and NarrativeQA (Kocˇisky ́ et al., 2018; Gu ̈nther et al., 2023), which is only for English.' (Chen et al., 2024).
-        It is constructed by sampling lengthy articles from Wikipedia, Wudao and mC4 datasets and randomly choose paragraphs from them. Then we use GPT-3.5 to generate questions based on these paragraphs. The generated question and the sampled article constitute a new text pair to the dataset.""",
+        description="Multi Long Doc Retrieval (MLDR) 'is curated by the multilingual articles from Wikipedia, Wudao and mC4 (see Table 7), and NarrativeQA (Kocˇisky ́ et al., 2018; Gu ̈nther et al., 2023), which is only for English.' (Chen et al., 2024). It is constructed by sampling lengthy articles from Wikipedia, Wudao and mC4 datasets and randomly choose paragraphs from them. Then we use GPT-3.5 to generate questions based on these paragraphs. The generated question and the sampled article constitute a new text pair to the dataset.",
         reference="https://arxiv.org/abs/2402.03216",  # also: https://huggingface.co/datasets/Shitao/MLDR
         dataset={
             "path": "mteb/MultiLongDocRetrieval",

mteb/tasks/retrieval/multilingual/ru_sci_bench_retrieval.py

@@ -68,11 +68,7 @@ class RuSciBenchCiteRetrieval(AbsTaskRetrieval):
             "path": "mlsa-iai-msu-lab/ru_sci_bench_cite_retrieval",
             "revision": "6cb447d02f41b8b775d5d9df7faf472f44d2f1db",
         },
-        description="""This task is focused on Direct Citation Prediction for scientific papers from eLibrary,
-        Russia's largest electronic library of scientific publications. Given a query paper (title and abstract),
-        the goal is to retrieve papers that are directly cited by it from a larger corpus of papers.
-        The dataset for this task consists of 3,000 query papers, 15,000 relevant (cited) papers,
-        and 75,000 irrelevant papers. The task is available for both Russian and English scientific texts.""",
+        description="This task is focused on Direct Citation Prediction for scientific papers from eLibrary, Russia's largest electronic library of scientific publications. Given a query paper (title and abstract), the goal is to retrieve papers that are directly cited by it from a larger corpus of papers. The dataset for this task consists of 3,000 query papers, 15,000 relevant (cited) papers, and 75,000 irrelevant papers. The task is available for both Russian and English scientific texts.",
         reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
         type="Retrieval",
         category="t2t",
@@ -130,13 +126,7 @@ class RuSciBenchCociteRetrieval(AbsTaskRetrieval):
             "path": "mlsa-iai-msu-lab/ru_sci_bench_cocite_retrieval",
             "revision": "a5da47a245275669d2b6ddf8f96c5338dd2428b4",
         },
-        description="""This task focuses on Co-citation Prediction for scientific papers from eLibrary,
-        Russia's largest electronic library of scientific publications. Given a query paper (title and abstract),
-        the goal is to retrieve other papers that are co-cited with it. Two papers are considered co-cited
-        if they are both cited by at least 5 of the same other papers. Similar to the Direct Citation task,
-        this task employs a retrieval setup: for a given query paper, all other papers in the corpus that
-        are not co-cited with it are considered negative examples. The task is available for both Russian
-        and English scientific texts.""",
+        description="This task focuses on Co-citation Prediction for scientific papers from eLibrary, Russia's largest electronic library of scientific publications. Given a query paper (title and abstract), the goal is to retrieve other papers that are co-cited with it. Two papers are considered co-cited if they are both cited by at least 5 of the same other papers. Similar to the Direct Citation task, this task employs a retrieval setup: for a given query paper, all other papers in the corpus that are not co-cited with it are considered negative examples. The task is available for both Russian and English scientific texts.",
         reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py

@@ -0,0 +1,399 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+_LANGS = {
+    "french": ["fra-Latn"],
+    "spanish": ["spa-Latn"],
+    "english": ["eng-Latn"],
+    "german": ["deu-Latn"],
+    "italian": ["ita-Latn"],
+    "portuguese": ["por-Latn"],
+}
+
+
+class Vidore3FinanceEnRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3FinanceEnRetrieval",
+        description="Retrieve associated pages according to questions. This task, Finance - EN, is a corpus of reports from american banking companies, intended for long-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_finance_en_mteb_format",
+            "revision": "fa78cb14152b3dde8c5defdc4e3ddf50de69dfeb",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Financial"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class Vidore3FinanceFrRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3FinanceFrRetrieval",
+        description="Retrieve associated pages according to questions. This task, Finance - FR, is a corpus of reports from french companies in the luxury domain, intended for long-document understanding tasks. Original queries were created in french, then translated to english, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_finance_fr_mteb_format",
+            "revision": "8a2adfda85a7967c7252129703d9b3c7c9f038a9",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Financial"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=True,
+    )
+
+
+class Vidore3IndustrialRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3IndustrialRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Industrial reports, is a corpus of technical documents on military aircraft (fueling, mechanics...), intended for complex-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_industrial_mteb_format",
+            "revision": "f732b725cf4a70803210edfe265a04f8bd5328f6",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Engineering"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=True,
+    )
+
+
+class Vidore3PharmaceuticalsRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3PharmaceuticalsRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Pharmaceutical, is a corpus of slides from the FDA, intended for long-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_pharmaceuticals_mteb_format",
+            "revision": "237ed4f43c7fb3c4df07ec4e9dd0a4366be555b0",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Medical"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=True,
+    )
+
+
+class Vidore3ComputerScienceRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3ComputerScienceRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Computer Science, is a corpus of textbooks from the openstacks website, intended for long-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_computer_science_mteb_format",
+            "revision": "fb7fb69f81f7db62790f40494124b8ad22b424ab",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Engineering", "Programming"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=True,
+    )
+
+
+class Vidore3HrRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3HrRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, HR, is a corpus of reports released by the european union, intended for complex-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_hr_mteb_format",
+            "revision": "bc7d43d64815ed30f664168c8052106484aba7fd",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=True,
+    )
+
+
+class Vidore3EnergyRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3EnergyRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Energy Fr, is a corpus of reports on energy supply in europe, intended for complex-document understanding tasks. Original queries were created in french, then translated to english, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_energy_mteb_format",
+            "revision": "84fca99e5978604bae30f2436eacb6dbaa0532e9",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Engineering", "Chemistry", "Academic"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=True,
+    )
+
+
+class Vidore3PhysicsRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3PhysicsRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Physics, is a corpus of course slides on french bachelor level physics lectures, intended for complex visual understanding tasks. Original queries were created in french, then translated to english, german, italian, portuguese and spanish.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "vidore/vidore_v3_physics_mteb_format",
+            "revision": "2c18ef90ab3ef93a9d86ecc6521cdae2a29f8300",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Engineering", "Academic"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=True,
+    )
+
+
+class Vidore3NuclearRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3NuclearRetrieval",
+        description="Retrieve associated pages according to questions.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "mteb-private/Vidore3NuclearRetrieval",
+            "revision": "a463fc67fefc01152153101e88a32d5f9515e3e3",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Engineering", "Chemistry"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=False,
+    )
+
+
+class Vidore3TelecomRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="Vidore3TelecomRetrieval",
+        description="Retrieve associated pages according to questions.",
+        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        dataset={
+            "path": "mteb-private/Vidore3TelecomRetrieval",
+            "revision": "a54635a274ef2835721b7cbe3eb27483b9ec964b",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=_LANGS,
+        main_score="ndcg_at_10",
+        date=("2025-10-01", "2025-11-01"),
+        domains=["Engineering", "Programming"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created and machine-translated",
+        bibtex_citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+        is_public=False,
+    )

mteb/tasks/retrieval/slk/slovak_sum_retrieval.py

@@ -7,13 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SlovakSumRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="SlovakSumRetrieval",
-        description="""
-            SlovakSum, a Slovak news summarization dataset consisting of over 200 thousand
-            news articles with titles and short abstracts obtained from multiple Slovak newspapers.
-
-            Originally intended as a summarization task, but since no human annotations were provided
-            here reformulated to a retrieval task.
-        """,
+        description="SlovakSum, a Slovak news summarization dataset consisting of over 200 thousand news articles with titles and short abstracts obtained from multiple Slovak newspapers. Originally intended as a summarization task, but since no human annotations were provided here reformulated to a retrieval task.",
         reference="https://huggingface.co/datasets/NaiveNeuron/slovaksum",
         dataset={
             "path": "NaiveNeuron/slovaksum",

mteb/tasks/retrieval/vie/argu_ana_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class ArguAnaVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ArguAna-VN",
-        description="""A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://argumentation.bplaced.net/arguana/data",
         dataset={
             "path": "GreenNode/arguana-vn",

mteb/tasks/retrieval/vie/climate_fevervn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class ClimateFEVERVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ClimateFEVER-VN",
-        description="""A translated dataset from CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
         dataset={
             "path": "GreenNode/climate-fever-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_android_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackAndroidVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackAndroid-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-android-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_gis_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackGisVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackGis-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-gis-vn",