mteb 2.5.2__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/tasks/retrieval/eng/cub200_i2i_retrieval.py

@@ -25,7 +25,7 @@ class CUB200I2I(AbsTaskRetrieval):
         modalities=["image"],
         sample_creation="created",
         bibtex_citation=r"""
-@article{article,
+@article{welinder2010caltech,
   author = {Welinder, Peter and Branson, Steve and Mita, Takeshi and Wah, Catherine and Schroff, Florian and Belongie, Serge and Perona, Pietro},
   month = {09},
   pages = {},

mteb/tasks/retrieval/kor/__init__.py

@@ -1,5 +1,19 @@
 from .auto_rag_retrieval import AutoRAGRetrieval
 from .ko_strategy_qa import KoStrategyQA
+from .kovidore2_bench_retrieval import (
+    KoVidore2CybersecurityRetrieval,
+    KoVidore2EconomicRetrieval,
+    KoVidore2EnergyRetrieval,
+    KoVidore2HrRetrieval,
+)
 from .squad_kor_v1_retrieval import SQuADKorV1Retrieval
 
-__all__ = ["AutoRAGRetrieval", "KoStrategyQA", "SQuADKorV1Retrieval"]
+__all__ = [
+    "AutoRAGRetrieval",
+    "KoStrategyQA",
+    "KoVidore2CybersecurityRetrieval",
+    "KoVidore2EconomicRetrieval",
+    "KoVidore2EnergyRetrieval",
+    "KoVidore2HrRetrieval",
+    "SQuADKorV1Retrieval",
+]

mteb/tasks/retrieval/kor/kovidore2_bench_retrieval.py

@@ -0,0 +1,142 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class KoVidore2CybersecurityRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2CybersecurityRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Cybersecurity, is a corpus of technical reports on cyber threat trends and security incident responses in Korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-cybersecurity-mteb",
+            "revision": "577d7c45f79d8eb4e7584db3990f91daa7e47956",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2EconomicRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2EconomicRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Economic trends, is a corpus of periodic reports on major economic indicators in Korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-economic-mteb",
+            "revision": "0189c26211290a902cd9d41a0db932808a54c0a8",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2EnergyRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2EnergyRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Energy, is a corpus of reports on energy market trends, policy planning, and industry statistics, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-energy-mteb",
+            "revision": "8c09a3d22b1fa3a7f5e815e9521da9b048754211",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2HrRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2HrRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, HR, is a corpus of reports on workforce outlook and employment policy in korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-hr-mteb",
+            "revision": "d9432c782a9a3e2eed064f6fac08b4c967d92b99",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )

mteb/tasks/retrieval/multilingual/__init__.py

@@ -6,6 +6,7 @@ from .cross_lingual_semantic_discrimination_wmt21 import (
     CrossLingualSemanticDiscriminationWMT21,
 )
 from .cur_ev1_retrieval import CUREv1Retrieval
+from .euro_pirq_retrieval import EuroPIRQRetrieval
 from .indic_qa_retrieval import IndicQARetrieval
 from .jina_vdr_bench_retrieval import (
     JinaVDRAirbnbSyntheticRetrieval,
@@ -107,6 +108,7 @@ __all__ = [
     "CUREv1Retrieval",
     "CrossLingualSemanticDiscriminationWMT19",
     "CrossLingualSemanticDiscriminationWMT21",
+    "EuroPIRQRetrieval",
     "IndicQARetrieval",
     "JinaVDRAirbnbSyntheticRetrieval",
     "JinaVDRArabicChartQARetrieval",

mteb/tasks/retrieval/multilingual/euro_pirq_retrieval.py

@@ -0,0 +1,43 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+_LANGUAGES = {
+    "en": ["eng-Latn"],
+    "fi": ["fin-Latn"],
+    "pt": ["por-Latn"],
+}
+
+
+class EuroPIRQRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="EuroPIRQRetrieval",
+        description="The EuroPIRQ retrieval dataset is a multilingual collection designed for evaluating retrieval and cross-lingual retrieval tasks. Dataset contains 10,000 parallel passages & 100 parallel queries (synthetic) in three languages: English, Portuguese, and Finnish, constructed from the European Union's DGT-Acquis corpus.",
+        reference="https://huggingface.co/datasets/eherra/EuroPIRQ-retrieval",
+        dataset={
+            "path": "eherra/EuroPIRQ-retrieval",
+            "revision": "59225ed25fbcea2185e1acbc8c3c80f1a8cd8341",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=_LANGUAGES,
+        main_score="ndcg_at_10",
+        date=("2025-12-01", "2025-12-31"),
+        domains=["Legal"],
+        task_subtypes=[],
+        license="not specified",
+        annotations_creators="LM-generated and reviewed",
+        dialect=[],
+        sample_creation="found",
+        is_public=True,
+        bibtex_citation=r"""
+@misc{eherra_2025_europirq,
+  author = { {Elias Herranen} },
+  publisher = { Hugging Face },
+  title = { EuroPIRQ: European Parallel Information Retrieval Queries },
+  url = { https://huggingface.co/datasets/eherra/EuroPIRQ-retrieval },
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py

@@ -15,7 +15,7 @@ class Vidore3FinanceEnRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3FinanceEnRetrieval",
         description="Retrieve associated pages according to questions. This task, Finance - EN, is a corpus of reports from american banking companies, intended for long-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_finance_en_mteb_format",
             "revision": "fa78cb14152b3dde8c5defdc4e3ddf50de69dfeb",
@@ -34,15 +34,14 @@ class Vidore3FinanceEnRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -53,7 +52,7 @@ class Vidore3FinanceFrRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3FinanceFrRetrieval",
         description="Retrieve associated pages according to questions. This task, Finance - FR, is a corpus of reports from french companies in the luxury domain, intended for long-document understanding tasks. Original queries were created in french, then translated to english, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_finance_fr_mteb_format",
             "revision": "8a2adfda85a7967c7252129703d9b3c7c9f038a9",
@@ -71,15 +70,14 @@ class Vidore3FinanceFrRetrieval(AbsTaskRetrieval):
         dialect=[],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -91,7 +89,7 @@ class Vidore3IndustrialRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3IndustrialRetrieval",
         description="Retrieve associated pages according to questions. This dataset, Industrial reports, is a corpus of technical documents on military aircraft (fueling, mechanics...), intended for complex-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_industrial_mteb_format",
             "revision": "f732b725cf4a70803210edfe265a04f8bd5328f6",
@@ -110,15 +108,14 @@ class Vidore3IndustrialRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -130,7 +127,7 @@ class Vidore3PharmaceuticalsRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3PharmaceuticalsRetrieval",
         description="Retrieve associated pages according to questions. This dataset, Pharmaceutical, is a corpus of slides from the FDA, intended for long-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_pharmaceuticals_mteb_format",
             "revision": "237ed4f43c7fb3c4df07ec4e9dd0a4366be555b0",
@@ -149,15 +146,14 @@ class Vidore3PharmaceuticalsRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -169,7 +165,7 @@ class Vidore3ComputerScienceRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3ComputerScienceRetrieval",
         description="Retrieve associated pages according to questions. This dataset, Computer Science, is a corpus of textbooks from the openstacks website, intended for long-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_computer_science_mteb_format",
             "revision": "fb7fb69f81f7db62790f40494124b8ad22b424ab",
@@ -188,15 +184,14 @@ class Vidore3ComputerScienceRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -208,7 +203,7 @@ class Vidore3HrRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3HrRetrieval",
         description="Retrieve associated pages according to questions. This dataset, HR, is a corpus of reports released by the european union, intended for complex-document understanding tasks. Original queries were created in english, then translated to french, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_hr_mteb_format",
             "revision": "bc7d43d64815ed30f664168c8052106484aba7fd",
@@ -227,15 +222,14 @@ class Vidore3HrRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -247,7 +241,7 @@ class Vidore3EnergyRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3EnergyRetrieval",
         description="Retrieve associated pages according to questions. This dataset, Energy Fr, is a corpus of reports on energy supply in europe, intended for complex-document understanding tasks. Original queries were created in french, then translated to english, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_energy_mteb_format",
             "revision": "84fca99e5978604bae30f2436eacb6dbaa0532e9",
@@ -266,15 +260,14 @@ class Vidore3EnergyRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -286,7 +279,7 @@ class Vidore3PhysicsRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3PhysicsRetrieval",
         description="Retrieve associated pages according to questions. This dataset, Physics, is a corpus of course slides on french bachelor level physics lectures, intended for complex visual understanding tasks. Original queries were created in french, then translated to english, german, italian, portuguese and spanish.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "vidore/vidore_v3_physics_mteb_format",
             "revision": "2c18ef90ab3ef93a9d86ecc6521cdae2a29f8300",
@@ -305,15 +298,14 @@ class Vidore3PhysicsRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -325,7 +317,7 @@ class Vidore3NuclearRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3NuclearRetrieval",
         description="Retrieve associated pages according to questions.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "mteb-private/Vidore3NuclearRetrieval",
             "revision": "a463fc67fefc01152153101e88a32d5f9515e3e3",
@@ -344,15 +336,14 @@ class Vidore3NuclearRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},
@@ -364,7 +355,7 @@ class Vidore3TelecomRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Vidore3TelecomRetrieval",
         description="Retrieve associated pages according to questions.",
-        reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+        reference="https://arxiv.org/abs/2601.08620",
         dataset={
             "path": "mteb-private/Vidore3TelecomRetrieval",
             "revision": "a54635a274ef2835721b7cbe3eb27483b9ec964b",
@@ -383,15 +374,14 @@ class Vidore3TelecomRetrieval(AbsTaskRetrieval):
         modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
-@misc{mace2025vidorev3,
-  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
-  day = {5},
-  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
-  journal = {Hugging Face Blog},
-  month = {November},
-  publisher = {Hugging Face},
-  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
-  year = {2025},
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
 }
 """,
         prompt={"query": "Find a screenshot that is relevant to the user's question."},

mteb/tasks/retrieval/nob/norquad.py

@@ -54,7 +54,7 @@ Fishel, Mark},
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -71,7 +71,7 @@ Fishel, Mark},
         text2id = {}
 
         for split in self.dataset:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.shuffle(seed=42)
             max_samples = min(1024, len(ds))
             ds = ds.select(

mteb/tasks/retrieval/nob/snl_retrieval.py

@@ -41,7 +41,7 @@ class SNLRetrieval(AbsTaskRetrieval):
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -58,7 +58,7 @@ class SNLRetrieval(AbsTaskRetrieval):
         text2id = {}
 
         for split in self.dataset:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.shuffle(seed=42)
 
             self.queries[split] = {}

mteb/tasks/retrieval/tur/tur_hist_quad.py

@@ -59,7 +59,7 @@ class TurHistQuadRetrieval(AbsTaskRetrieval):
         text2id = {}
 
         for split in self.metadata.eval_splits:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.shuffle(seed=42)
             max_samples = min(1024, len(ds))
             ds = ds.select(