mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/tasks/retrieval/kor/kovidore2_bench_retrieval.py

@@ -0,0 +1,142 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class KoVidore2CybersecurityRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2CybersecurityRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Cybersecurity, is a corpus of technical reports on cyber threat trends and security incident responses in Korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-cybersecurity-mteb",
+            "revision": "577d7c45f79d8eb4e7584db3990f91daa7e47956",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2EconomicRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2EconomicRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Economic trends, is a corpus of periodic reports on major economic indicators in Korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-economic-mteb",
+            "revision": "0189c26211290a902cd9d41a0db932808a54c0a8",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2EnergyRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2EnergyRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, Energy, is a corpus of reports on energy market trends, policy planning, and industry statistics, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-energy-mteb",
+            "revision": "8c09a3d22b1fa3a7f5e815e9521da9b048754211",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )
+
+
+class KoVidore2HrRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="KoVidore2HrRetrieval",
+        description="Retrieve associated pages according to questions. This dataset, HR, is a corpus of reports on workforce outlook and employment policy in korea, intended for complex-document understanding tasks.",
+        reference="https://github.com/whybe-choi/kovidore-data-generator",
+        dataset={
+            "path": "whybe-choi/kovidore-v2-hr-mteb",
+            "revision": "d9432c782a9a3e2eed064f6fac08b4c967d92b99",
+        },
+        type="DocumentUnderstanding",
+        category="t2i",
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2025-12-21", "2026-01-06"),
+        domains=["Social"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["text", "image"],
+        sample_creation="created",
+        bibtex_citation="""
+@misc{choi2026kovidorev2,
+  author = {Yongbin Choi},
+  note = {A benchmark for evaluating Korean vision document retrieval with multi-page reasoning queries in practical domains},
+  title = {KoViDoRe v2: a comprehensive evaluation of vision document retrieval for enterprise use-cases},
+  url = {https://github.com/whybe-choi/kovidore-data-generator},
+  year = {2026},
+}
+""",
+        prompt={"query": "Find a screenshot that is relevant to the user's question."},
+    )

mteb/tasks/retrieval/kor/squad_kor_v1_retrieval.py

@@ -0,0 +1,47 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class SQuADKorV1Retrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SQuADKorV1Retrieval",
+        description="Korean translation of SQuAD v1.0 dataset for retrieval task, based on Korean Wikipedia articles.",
+        reference="https://huggingface.co/datasets/yjoonjang/squad_kor_v1",
+        dataset={
+            "path": "yjoonjang/squad_kor_v1",
+            "revision": "2b4ee1f3b143a04792da93a3df21933c5fe9eed3",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2018-01-01", "2019-12-31"),
+        domains=["Encyclopaedic", "Written"],
+        task_subtypes=["Question answering"],
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{rajpurkar-etal-2016-squad,
+  address = {Austin, Texas},
+  author = {Rajpurkar, Pranav  and
+Zhang, Jian  and
+Lopyrev, Konstantin  and
+Liang, Percy},
+  booktitle = {Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
+  doi = {10.18653/v1/D16-1264},
+  editor = {Su, Jian  and
+Duh, Kevin  and
+Carreras, Xavier},
+  month = nov,
+  pages = {2383--2392},
+  publisher = {Association for Computational Linguistics},
+  title = {{SQ}u{AD}: 100,000+ Questions for Machine Comprehension of Text},
+  url = {https://aclanthology.org/D16-1264},
+  year = {2016},
+}
+""",
+    )

mteb/tasks/retrieval/multilingual/__init__.py

@@ -6,6 +6,7 @@ from .cross_lingual_semantic_discrimination_wmt21 import (
     CrossLingualSemanticDiscriminationWMT21,
 )
 from .cur_ev1_retrieval import CUREv1Retrieval
+from .euro_pirq_retrieval import EuroPIRQRetrieval
 from .indic_qa_retrieval import IndicQARetrieval
 from .jina_vdr_bench_retrieval import (
     JinaVDRAirbnbSyntheticRetrieval,
@@ -81,6 +82,18 @@ from .vidore2_bench_retrieval import (
     Vidore2ESGReportsHLRetrieval,
     Vidore2ESGReportsRetrieval,
 )
+from .vidore3_bench_retrieval import (
+    Vidore3ComputerScienceRetrieval,
+    Vidore3EnergyRetrieval,
+    Vidore3FinanceEnRetrieval,
+    Vidore3FinanceFrRetrieval,
+    Vidore3HrRetrieval,
+    Vidore3IndustrialRetrieval,
+    Vidore3NuclearRetrieval,
+    Vidore3PharmaceuticalsRetrieval,
+    Vidore3PhysicsRetrieval,
+    Vidore3TelecomRetrieval,
+)
 from .web_faq_retrieval import WebFAQRetrieval
 from .wikipedia_retrieval_multilingual import WikipediaRetrievalMultilingual
 from .wit_t2i_retrieval import WITT2IRetrieval
@@ -95,6 +108,7 @@ __all__ = [
     "CUREv1Retrieval",
     "CrossLingualSemanticDiscriminationWMT19",
     "CrossLingualSemanticDiscriminationWMT21",
+    "EuroPIRQRetrieval",
     "IndicQARetrieval",
     "JinaVDRAirbnbSyntheticRetrieval",
     "JinaVDRArabicChartQARetrieval",
@@ -161,6 +175,16 @@ __all__ = [
     "Vidore2ESGReportsHLRetrieval",
     "Vidore2ESGReportsRetrieval",
     "Vidore2EconomicsReportsRetrieval",
+    "Vidore3ComputerScienceRetrieval",
+    "Vidore3EnergyRetrieval",
+    "Vidore3FinanceEnRetrieval",
+    "Vidore3FinanceFrRetrieval",
+    "Vidore3HrRetrieval",
+    "Vidore3IndustrialRetrieval",
+    "Vidore3NuclearRetrieval",
+    "Vidore3PharmaceuticalsRetrieval",
+    "Vidore3PhysicsRetrieval",
+    "Vidore3TelecomRetrieval",
     "WITT2IRetrieval",
     "WebFAQRetrieval",
     "WikipediaRetrievalMultilingual",

mteb/tasks/retrieval/multilingual/belebele_retrieval.py

@@ -230,10 +230,11 @@ class BelebeleRetrieval(AbsTaskRetrieval):
             ds_corpus = self.dataset[lang_corpus]
             ds_question = self.dataset[lang_question]
 
-            question_ids = {
-                question: _id
-                for _id, question in enumerate(set(ds_question["question"]))
-            }
+            question_ids = {}
+            for row in ds_question:
+                question = row["question"]
+                if question not in question_ids:
+                    question_ids[question] = len(question_ids)
 
             link_to_context_id = {}
             context_idx = 0

mteb/tasks/retrieval/multilingual/euro_pirq_retrieval.py

@@ -0,0 +1,43 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+_LANGUAGES = {
+    "en": ["eng-Latn"],
+    "fi": ["fin-Latn"],
+    "pt": ["por-Latn"],
+}
+
+
+class EuroPIRQRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="EuroPIRQRetrieval",
+        description="The EuroPIRQ retrieval dataset is a multilingual collection designed for evaluating retrieval and cross-lingual retrieval tasks. Dataset contains 10,000 parallel passages & 100 parallel queries (synthetic) in three languages: English, Portuguese, and Finnish, constructed from the European Union's DGT-Acquis corpus.",
+        reference="https://huggingface.co/datasets/eherra/EuroPIRQ-retrieval",
+        dataset={
+            "path": "eherra/EuroPIRQ-retrieval",
+            "revision": "59225ed25fbcea2185e1acbc8c3c80f1a8cd8341",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=_LANGUAGES,
+        main_score="ndcg_at_10",
+        date=("2025-12-01", "2025-12-31"),
+        domains=["Legal"],
+        task_subtypes=[],
+        license="not specified",
+        annotations_creators="LM-generated and reviewed",
+        dialect=[],
+        sample_creation="found",
+        is_public=True,
+        bibtex_citation=r"""
+@misc{eherra_2025_europirq,
+  author = { {Elias Herranen} },
+  publisher = { Hugging Face },
+  title = { EuroPIRQ: European Parallel Information Retrieval Queries },
+  url = { https://huggingface.co/datasets/eherra/EuroPIRQ-retrieval },
+  year = {2025},
+}
+""",
+    )