mteb 2.5.2__py3-none-any.whl → 2.7.9__py3-none-any.whl

mteb/tasks/retrieval/kor/kovidore2_bench_retrieval.py

@@ -0,0 +1,142 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class KoVidore2CybersecurityRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2CybersecurityRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Cybersecurity, is a corpus of technical reports on cyber threat trends and security incident responses in Korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-cybersecurity-mteb",
+            "revision": "577d7c45f79d8eb4e7584db3990f91daa7e47956",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2EconomicRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2EconomicRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Economic trends, is a corpus of periodic reports on major economic indicators in Korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-economic-mteb",
+            "revision": "0189c26211290a902cd9d41a0db932808a54c0a8",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2EnergyRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2EnergyRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Energy, is a corpus of reports on energy market trends, policy planning, and industry statistics, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-energy-mteb",
+            "revision": "8c09a3d22b1fa3a7f5e815e9521da9b048754211",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2HrRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2HrRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, HR, is a corpus of reports on workforce outlook and employment policy in korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-hr-mteb",
+            "revision": "d9432c782a9a3e2eed064f6fac08b4c967d92b99",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )

mteb/tasks/retrieval/multilingual/__init__.py

@@ -6,6 +6,7 @@ from .cross_lingual_semantic_discrimination_wmt21 import (
     CrossLingualSemanticDiscriminationWMT21,
 )
 from .cur_ev1_retrieval import CUREv1Retrieval
+from .euro_pirq_retrieval import EuroPIRQRetrieval
 from .indic_qa_retrieval import IndicQARetrieval
 from .jina_vdr_bench_retrieval import (
     JinaVDRAirbnbSyntheticRetrieval,
@@ -107,6 +108,7 @@ __all__ = [
     "CUREv1Retrieval",
     "CrossLingualSemanticDiscriminationWMT19",
     "CrossLingualSemanticDiscriminationWMT21",
+    "EuroPIRQRetrieval",
     "IndicQARetrieval",
     "JinaVDRAirbnbSyntheticRetrieval",
     "JinaVDRArabicChartQARetrieval",

mteb/tasks/retrieval/multilingual/cross_lingual_semantic_discrimination_wmt19.py

@@ -53,7 +53,7 @@ class CrossLingualSemanticDiscriminationWMT19(AbsTaskRetrieval):
     )
     num_of_distractors = 4
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         """Generic data loader function for original clsd datasets with the format shown in "hf_dataset_link".
         Loading the hf dataset, it populates the following three variables to be used for retrieval evaluation.
 

mteb/tasks/retrieval/multilingual/cross_lingual_semantic_discrimination_wmt21.py

@@ -54,7 +54,7 @@ class CrossLingualSemanticDiscriminationWMT21(AbsTaskRetrieval):
 
     num_of_distractors = 4
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         """Generic data loader function for original clsd datasets with the format shown in "hf_dataset_link".
         Loading the hf dataset, it populates the following three variables to be used for retrieval evaluation.
 

mteb/tasks/retrieval/multilingual/cur_ev1_retrieval.py

@@ -111,7 +111,7 @@ class CUREv1Retrieval(AbsTaskRetrieval):
 
         return queries
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 

mteb/tasks/retrieval/multilingual/euro_pirq_retrieval.py

@@ -0,0 +1,43 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+_LANGUAGES = {
+    "en": ["eng-Latn"],
+    "fi": ["fin-Latn"],
+    "pt": ["por-Latn"],
+}
+
+
+class EuroPIRQRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="EuroPIRQRetrieval",
+        description="The EuroPIRQ retrieval dataset is a multilingual collection designed for evaluating retrieval and cross-lingual retrieval tasks. Dataset contains 10,000 parallel passages & 100 parallel queries (synthetic) in three languages: English, Portuguese, and Finnish, constructed from the European Union's DGT-Acquis corpus.",
+        reference="https://huggingface.co/datasets/eherra/EuroPIRQ-retrieval",
+        dataset={
+            "path": "eherra/EuroPIRQ-retrieval",
+            "revision": "59225ed25fbcea2185e1acbc8c3c80f1a8cd8341",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=_LANGUAGES,
+        main_score="ndcg_at_10",
+        date=("2025-12-01", "2025-12-31"),
+        domains=["Legal"],
+        task_subtypes=[],
+        license="not specified",
+        annotations_creators="LM-generated and reviewed",
+        dialect=[],
+        sample_creation="found",
+        is_public=True,
+        bibtex_citation=r"""
+@misc{eherra_2025_europirq,
+  author = { {Elias Herranen} },
+  publisher = { Hugging Face },
+  title = { EuroPIRQ: European Parallel Information Retrieval Queries },
+  url = { https://huggingface.co/datasets/eherra/EuroPIRQ-retrieval },
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/multilingual/jina_vdr_bench_retrieval.py

@@ -148,7 +148,7 @@ def _load_data(
     return corpus, queries, relevant_docs
 
 
-def load_data(self) -> None:
+def load_data(self, num_proc: int = 1, **kwargs) -> None:
     if self.data_loaded:
         return
 

mteb/tasks/retrieval/multilingual/miracl_vision_retrieval.py

@@ -143,7 +143,7 @@ class MIRACLVisionRetrieval(AbsTaskRetrieval):
         prompt={"query": "Find a screenshot that is relevant to the user's query."},
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 

mteb/tasks/retrieval/multilingual/mr_tidy_retrieval.py

@@ -108,7 +108,7 @@ class MrTidyRetrieval(AbsTaskRetrieval):
 """,
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 

mteb/tasks/retrieval/multilingual/public_health_qa_retrieval.py

@@ -97,7 +97,7 @@ class PublicHealthQARetrieval(AbsTaskRetrieval):
 """,
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 

mteb/tasks/retrieval/multilingual/ru_sci_bench_retrieval.py

@@ -30,15 +30,15 @@ def load_ruscibench_data(
 
     for lang in langs:
         lang_corpus = cast(
-            datasets.Dataset,
+            "datasets.Dataset",
             datasets.load_dataset(path, f"corpus-{lang}", revision=revision),
         )["corpus"]
         lang_queries = cast(
-            datasets.Dataset,
+            "datasets.Dataset",
             datasets.load_dataset(path, f"queries-{lang}", revision=revision),
         )["queries"]
         lang_qrels = cast(
-            datasets.Dataset,
+            "datasets.Dataset",
             datasets.load_dataset(path, f"{lang}", revision=revision),
         )["test"]
         corpus[lang] = {
@@ -103,7 +103,7 @@ class RuSciBenchCiteRetrieval(AbsTaskRetrieval):
         },
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 
@@ -161,7 +161,7 @@ class RuSciBenchCociteRetrieval(AbsTaskRetrieval):
         },
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 

mteb/tasks/retrieval/multilingual/statcan_dialogue_dataset_retrieval.py

@@ -96,7 +96,7 @@ de Vries, Harm},
 """,
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 

mteb/tasks/retrieval/multilingual/vdr_multilingual_retrieval.py

@@ -126,7 +126,7 @@ class VDRMultilingualRetrieval(AbsTaskRetrieval):
 """,
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 

mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py

@@ -16,6 +16,7 @@ def _load_data(
     splits: list[str],
     langs: list | None = None,
     revision: str | None = None,
+    num_proc: int = 1,
 ):
     if langs is None:
         corpus = {}
@@ -32,6 +33,7 @@ def _load_data(
             "queries",
             split=split,
             revision=revision,
+            num_proc=num_proc,
         )
         query_ds = query_ds.map(
             lambda x: {
@@ -40,6 +42,7 @@ def _load_data(
                 "modality": "text",
             },
             remove_columns=["query-id", "query"],
+            num_proc=num_proc,
         )
 
         corpus_ds = load_dataset(
@@ -47,6 +50,7 @@ def _load_data(
             "corpus",
             split=split,
             revision=revision,
+            num_proc=num_proc,
         )
         corpus_ds = corpus_ds.map(
             lambda x: {
@@ -54,6 +58,7 @@ def _load_data(
                 "modality": "image",
             },
             remove_columns=["corpus-id"],
+            num_proc=num_proc,
         )
         corpus_ds = corpus_ds.select_columns(["id", "image"])
 
@@ -62,6 +67,7 @@ def _load_data(
             "qrels",
             split=split,
             revision=revision,
+            num_proc=num_proc,
         )
 
         if langs is None:
@@ -125,7 +131,7 @@ class Vidore2ESGReportsRetrieval(AbsTaskRetrieval):
         prompt={"query": "Find a screenshot that relevant to the user's question."},
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 
@@ -134,6 +140,7 @@ class Vidore2ESGReportsRetrieval(AbsTaskRetrieval):
             splits=self.metadata.eval_splits,
             langs=_LANGS.keys(),
             revision=self.metadata.dataset["revision"],
+            num_proc=num_proc,
         )
 
         self.data_loaded = True
@@ -172,7 +179,7 @@ class Vidore2EconomicsReportsRetrieval(AbsTaskRetrieval):
         prompt={"query": "Find a screenshot that relevant to the user's question."},
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 
@@ -181,6 +188,7 @@ class Vidore2EconomicsReportsRetrieval(AbsTaskRetrieval):
             splits=self.metadata.eval_splits,
             langs=_LANGS.keys(),
             revision=self.metadata.dataset["revision"],
+            num_proc=num_proc,
         )
 
         self.data_loaded = True
@@ -219,7 +227,7 @@ class Vidore2BioMedicalLecturesRetrieval(AbsTaskRetrieval):
         prompt={"query": "Find a screenshot that relevant to the user's question."},
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 
@@ -228,6 +236,7 @@ class Vidore2BioMedicalLecturesRetrieval(AbsTaskRetrieval):
             splits=self.metadata.eval_splits,
             langs=_LANGS.keys(),
             revision=self.metadata.dataset["revision"],
+            num_proc=num_proc,
         )
 
         self.data_loaded = True
@@ -266,7 +275,7 @@ class Vidore2ESGReportsHLRetrieval(AbsTaskRetrieval):
         prompt={"query": "Find a screenshot that relevant to the user's question."},
     )
 
-    def load_data(self) -> None:
+    def load_data(self, num_proc: int = 1, **kwargs) -> None:
         if self.data_loaded:
             return
 
@@ -274,6 +283,7 @@ class Vidore2ESGReportsHLRetrieval(AbsTaskRetrieval):
             path=self.metadata.dataset["path"],
             splits=self.metadata.eval_splits,
             revision=self.metadata.dataset["revision"],
+            num_proc=num_proc,
         )
 
         self.data_loaded = True