mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/tasks/retrieval/code/code_rag.py

@@ -48,14 +48,14 @@ class CodeRAGProgrammingSolutionsRetrieval(AbsTaskRetrieval):
             "path": "code-rag-bench/programming-solutions",
             "revision": "1064f7bba54d5400d4836f5831fe4c2332a566a6",
         },
-        **common_args,  # type: ignore
+        **common_args,
     )
 
     def load_data(self) -> None:
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -71,7 +71,7 @@ class CodeRAGProgrammingSolutionsRetrieval(AbsTaskRetrieval):
         self.queries = {}
 
         split = self.metadata.eval_splits[0]
-        ds: datasets.Dataset = self.dataset[split]  # type: ignore
+        ds: datasets.Dataset = self.dataset[split]
         ds = ds.shuffle(seed=42)
 
         self.queries[split] = {}
@@ -105,14 +105,14 @@ class CodeRAGOnlineTutorialsRetrieval(AbsTaskRetrieval):
             "path": "code-rag-bench/online-tutorials",
             "revision": "095bb77130082e4690d6c3a031997b03487bf6e2",
         },
-        **common_args,  # type: ignore
+        **common_args,
     )
 
     def load_data(self) -> None:
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -128,7 +128,7 @@ class CodeRAGOnlineTutorialsRetrieval(AbsTaskRetrieval):
         self.queries = {}
 
         split = self.metadata.eval_splits[0]
-        ds: datasets.Dataset = self.dataset[split]  # type: ignore
+        ds: datasets.Dataset = self.dataset[split]
         ds = ds.shuffle(seed=42)
 
         self.queries[split] = {}
@@ -165,14 +165,14 @@ class CodeRAGLibraryDocumentationSolutionsRetrieval(AbsTaskRetrieval):
             "path": "code-rag-bench/library-documentation",
             "revision": "b530d3b5a25087d2074e731b76232db85b9e9107",
         },
-        **common_args,  # type: ignore
+        **common_args,
     )
 
     def load_data(self) -> None:
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -188,7 +188,7 @@ class CodeRAGLibraryDocumentationSolutionsRetrieval(AbsTaskRetrieval):
         self.queries = {}
 
         split = self.metadata.eval_splits[0]
-        ds: datasets.Dataset = self.dataset[split]  # type: ignore
+        ds: datasets.Dataset = self.dataset[split]
         ds = ds.shuffle(seed=42)
 
         self.queries[split] = {}
@@ -222,14 +222,14 @@ class CodeRAGStackoverflowPostsRetrieval(AbsTaskRetrieval):
             "path": "code-rag-bench/stackoverflow-posts",
             "revision": "04e05d86cb0ac467b29a5d87f4c56eac99dfc0a4",
         },
-        **common_args,  # type: ignore
+        **common_args,
     )
 
     def load_data(self) -> None:
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -245,7 +245,7 @@ class CodeRAGStackoverflowPostsRetrieval(AbsTaskRetrieval):
         self.queries = {}
 
         split = self.metadata.eval_splits[0]
-        ds: datasets.Dataset = self.dataset[split]  # type: ignore
+        ds: datasets.Dataset = self.dataset[split]
         ds = ds.shuffle(seed=42)
 
         self.queries[split] = {}

mteb/tasks/retrieval/code/fresh_stack_retrieval.py

@@ -25,11 +25,14 @@ class FreshStackRetrieval(AbsTaskRetrieval):
         dialect=[],
         sample_creation="found",
         bibtex_citation=r"""
-@article{freshstack2023,
-  author = {FreshStack Authors},
-  journal = {arXiv preprint arXiv:2301.12345},
-  title = {FreshStack: A Multi-language Code Generation and Retrieval Benchmark},
-  year = {2023},
+@misc{thakur2025freshstackbuildingrealisticbenchmarks,
+  archiveprefix = {arXiv},
+  author = {Nandan Thakur and Jimmy Lin and Sam Havens and Michael Carbin and Omar Khattab and Andrew Drozdov},
+  eprint = {2504.13128},
+  primaryclass = {cs.IR},
+  title = {FreshStack: Building Realistic Benchmarks for Evaluating Retrieval on Technical Documents},
+  url = {https://arxiv.org/abs/2504.13128},
+  year = {2025},
 }
 """,
     )

mteb/tasks/retrieval/dan/dan_fever_retrieval.py

@@ -51,7 +51,7 @@ Derczynski, Leon},
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 

mteb/tasks/retrieval/dan/tv2_nordretrieval.py

@@ -64,7 +64,7 @@ Piperidis, Stelios},
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -81,7 +81,7 @@ Piperidis, Stelios},
         text2id = {}
 
         for split in self.dataset:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.shuffle(seed=42)
             ds = ds.select(
                 range(2048)

mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py

@@ -40,7 +40,7 @@ class TwitterHjerneRetrieval(AbsTaskRetrieval):
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -57,7 +57,7 @@ class TwitterHjerneRetrieval(AbsTaskRetrieval):
         text2id = {}
 
         for split in self.dataset:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.map(answers_to_list)
 
             self.queries[split] = {}

mteb/tasks/retrieval/eng/__init__.py

@@ -18,6 +18,7 @@ from .built_bench_retrieval import BuiltBenchRetrieval
 from .chat_doctor_retrieval import ChatDoctorRetrieval
 from .chem_hotpot_qa_retrieval import ChemHotpotQARetrieval
 from .chem_nq_retrieval import ChemNQRetrieval
+from .chemrxiv import ChemRxivRetrieval
 from .cirr_it2i_retrieval import CIRRIT2IRetrieval
 from .climate_fever_retrieval import (
     ClimateFEVER,
@@ -254,6 +255,7 @@ __all__ = [
     "ChatDoctorRetrieval",
     "ChemHotpotQARetrieval",
     "ChemNQRetrieval",
+    "ChemRxivRetrieval",
     "ClimateFEVER",
     "ClimateFEVERHardNegatives",
     "ClimateFEVERHardNegativesV2",

mteb/tasks/retrieval/eng/chemrxiv.py

@@ -0,0 +1,33 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class ChemRxivRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="ChemRxivRetrieval",
+        dataset={
+            "path": "BASF-AI/ChemRxivRetrieval",
+            "revision": "5377aa18f309ec440ff6325a4c2cd3362c2cb8d7",
+        },
+        description="A retrieval task based on ChemRxiv papers where queries are LLM-synthesized to match specific paragraphs.",
+        reference="https://arxiv.org/abs/2508.01643",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-01-01", "2025-05-01"),
+        domains=["Chemistry"],
+        task_subtypes=["Question answering", "Article retrieval"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="LM-generated and reviewed",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@article{kasmaee2025chembed,
+  author = {Kasmaee, Ali Shiraee and Khodadad, Mohammad and Astaraki, Mahdi and Saloot, Mohammad Arshi and Sherck, Nicholas and Mahyar, Hamidreza and Samiee, Soheila},
+  journal = {arXiv preprint arXiv:2508.01643},
+  title = {Chembed: Enhancing chemical literature search through domain-specific text embeddings},
+  year = {2025},
+}""",
+    )

mteb/tasks/retrieval/eng/cub200_i2i_retrieval.py

@@ -25,7 +25,7 @@ class CUB200I2I(AbsTaskRetrieval):
         modalities=["image"],
         sample_creation="created",
         bibtex_citation=r"""
-@article{article,
+@article{welinder2010caltech,
   author = {Welinder, Peter and Branson, Steve and Mita, Takeshi and Wah, Catherine and Schroff, Florian and Belongie, Serge and Perona, Pietro},
   month = {09},
   pages = {},

mteb/tasks/retrieval/eng/lit_search_retrieval.py

@@ -7,14 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class LitSearchRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="LitSearchRetrieval",
-        description="""
-        The dataset contains the query set and retrieval corpus for the paper LitSearch: A Retrieval Benchmark for
-        Scientific Literature Search. It introduces LitSearch, a retrieval benchmark comprising 597 realistic literature
-        search queries about recent ML and NLP papers. LitSearch is constructed using a combination of (1) questions
-        generated by GPT-4 based on paragraphs containing inline citations from research papers and (2) questions about
-        recently published papers, manually written by their authors. All LitSearch questions were manually examined or
-        edited by experts to ensure high quality.
-        """,
+        description="The dataset contains the query set and retrieval corpus for the paper LitSearch: A Retrieval Benchmark for Scientific Literature Search. It introduces LitSearch, a retrieval benchmark comprising 597 realistic literature search queries about recent ML and NLP papers. LitSearch is constructed using a combination of (1) questions generated by GPT-4 based on paragraphs containing inline citations from research papers and (2) questions about recently published papers, manually written by their authors. All LitSearch questions were manually examined or edited by experts to ensure high quality.",
         reference="https://github.com/princeton-nlp/LitSearch",
         dataset={
             "path": "princeton-nlp/LitSearch",

mteb/tasks/retrieval/eng/vidore_bench_retrieval.py

@@ -351,6 +351,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:
@@ -394,6 +395,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:
@@ -437,6 +439,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:
@@ -480,6 +483,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:

mteb/tasks/retrieval/jpn/__init__.py

@@ -1,8 +1,12 @@
 from .ja_cwir_retrieval import JaCWIRRetrieval
+from .ja_cwir_retrieval_lite import JaCWIRRetrievalLite
 from .ja_gov_faqs_retrieval import JaGovFaqsRetrieval
 from .ja_qu_ad_retrieval import JaQuADRetrieval
 from .japanese_legal1_retrieval import JapaneseLegal1Retrieval
 from .jaqket_retrieval import JaqketRetrieval
+from .jaqket_retrieval_lite import JaqketRetrievalLite
+from .miracl_ja_retrieval_lite import MIRACLJaRetrievalLite
+from .mr_tydi_ja_retrieval_lite import MrTyDiJaRetrievalLite
 from .nlp_journal_abs_article_retrieval import (
     NLPJournalAbsArticleRetrieval,
     NLPJournalAbsArticleRetrievalV2,
@@ -22,10 +26,14 @@ from .nlp_journal_title_intro_retrieval import (
 
 __all__ = [
     "JaCWIRRetrieval",
+    "JaCWIRRetrievalLite",
     "JaGovFaqsRetrieval",
     "JaQuADRetrieval",
     "JapaneseLegal1Retrieval",
     "JaqketRetrieval",
+    "JaqketRetrievalLite",
+    "MIRACLJaRetrievalLite",
+    "MrTyDiJaRetrievalLite",
     "NLPJournalAbsArticleRetrieval",
     "NLPJournalAbsArticleRetrievalV2",
     "NLPJournalAbsIntroRetrieval",

mteb/tasks/retrieval/jpn/ja_cwir_retrieval.py

@@ -9,10 +9,7 @@ class JaCWIRRetrieval(AbsTaskRetrieval):
 
     metadata = TaskMetadata(
         name="JaCWIRRetrieval",
-        description="""JaCWIR is a small-scale Japanese information retrieval evaluation dataset consisting of
-5000 question texts and approximately 500k web page titles and web page introductions or summaries
-(meta descriptions, etc.). The question texts are created based on one of the 500k web pages,
-and that data is used as a positive example for the question text.""",
+        description="JaCWIR is a small-scale Japanese information retrieval evaluation dataset consisting of 5000 question texts and approximately 500k web page titles and web page introductions or summaries (meta descriptions, etc.). The question texts are created based on one of the 500k web pages, and that data is used as a positive example for the question text.",
         reference="https://huggingface.co/datasets/hotchpotch/JaCWIR",
         dataset={
             "path": "mteb/JaCWIRRetrieval",

mteb/tasks/retrieval/jpn/ja_cwir_retrieval_lite.py

@@ -0,0 +1,47 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JaCWIRRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JaCWIRRetrievalLite",
+        dataset={
+            "path": "mteb/JaCWIRRetrievalLite",
+            "revision": "79472b360242cf2692e24a6d9999ef50d350d672",
+        },
+        description=(
+            "JaCWIR (Japanese Casual Web IR) is a dataset consisting of questions and webpage meta descriptions "
+            "collected from Hatena Bookmark. This is the lightweight version with a reduced corpus "
+            "(302,638 documents) constructed using hard negatives from 5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/hotchpotch/JaCWIR",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2020-01-01", "2025-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=["Article retrieval"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["JaCWIRRetrieval"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@misc{yuichi-tateno-2024-jacwir,
+  author = {Yuichi Tateno},
+  title = {JaCWIR: Japanese Casual Web IR - 日本語情報検索評価のための小規模でカジュアルなWebタイトルと概要のデータセット},
+  url = {https://huggingface.co/datasets/hotchpotch/JaCWIR},
+}
+""",
+    )

mteb/tasks/retrieval/jpn/jaqket_retrieval_lite.py

@@ -0,0 +1,50 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JaqketRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JaqketRetrievalLite",
+        dataset={
+            "path": "mteb/JaqketRetrievalLite",
+            "revision": "860965fbb6526dd8edff12627dacf07c8f5a54f3",
+        },
+        description=(
+            "JAQKET (JApanese Questions on Knowledge of EnTities) is a QA dataset created based on quiz questions. "
+            "This is the lightweight version with a reduced corpus (65,802 documents) constructed using "
+            "hard negatives from 5 high-performance models."
+        ),
+        reference="https://github.com/kumapo/JAQKET-dataset",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2023-10-09", "2025-01-01"),
+        domains=["Encyclopaedic", "Non-fiction", "Written"],
+        task_subtypes=["Question answering"],
+        license="cc-by-sa-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["JaqketRetrieval"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@inproceedings{Kurihara_nlp2020,
+  author = {鈴木正敏 and 鈴木潤 and 松田耕史 and ⻄田京介 and 井之上直也},
+  booktitle = {言語処理学会第26回年次大会},
+  note = {in Japanese},
+  title = {JAQKET: クイズを題材にした日本語 QA データセットの構築},
+  url = {https://www.anlp.jp/proceedings/annual_meeting/2020/pdf_dir/P2-24.pdf},
+  year = {2020},
+}
+""",
+    )

mteb/tasks/retrieval/jpn/miracl_ja_retrieval_lite.py

@@ -0,0 +1,52 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class MIRACLJaRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="MIRACLJaRetrievalLite",
+        dataset={
+            "path": "mteb/MIRACLJaRetrievalLite",
+            "revision": "575c225da29d1f5fec01082afa56f35df0f44295",
+        },
+        description=(
+            "MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual "
+            "retrieval dataset. This is the lightweight Japanese version with a reduced corpus (105,064 documents) "
+            "constructed using hard negatives from 5 high-performance models."
+        ),
+        reference="https://project-miracl.github.io/",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2022-06-01", "2025-01-01"),
+        domains=["Encyclopaedic", "Written"],
+        task_subtypes=[],
+        license="apache-2.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="created",
+        adapted_from=["MIRACLRetrieval"],
+        bibtex_citation=r"""
+@article{10.1162/tacl_a_00595,
+  author = {Zhang, Xinyu and Thakur, Nandan and Ogundepo, Odunayo and Kamalloo, Ehsan and Alfonso-Hermelo, David
+and Li, Xiaoguang and Liu, Qun and Rezagholizadeh, Mehdi and Lin, Jimmy},
+  doi = {10.1162/tacl_a_00595},
+  journal = {Transactions of the Association for Computational Linguistics},
+  pages = {1114-1131},
+  title = {{MIRACL: A Multilingual Retrieval Dataset Covering 18 Diverse Languages}},
+  volume = {11},
+  year = {2023},
+}
+
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/jpn/mr_tydi_ja_retrieval_lite.py

@@ -0,0 +1,48 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class MrTyDiJaRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="MrTyDiJaRetrievalLite",
+        dataset={
+            "path": "mteb/MrTyDiJaRetrievalLite",
+            "revision": "b87e6ff25f4e32d1c97498a539ea8aad5fde3cb1",
+        },
+        description=(
+            "Mr.TyDi is a multilingual benchmark dataset built on TyDi for document retrieval tasks. "
+            "This is the lightweight Japanese version with a reduced corpus (93,382 documents) constructed using "
+            "hard negatives from 5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/castorini/mr-tydi",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2021-01-01", "2025-01-01"),
+        domains=["Encyclopaedic", "Non-fiction", "Written"],
+        task_subtypes=["Question answering"],
+        license="apache-2.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["MrTidyRetrieval"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@article{mrtydi,
+  author = {Xinyu Zhang and Xueguang Ma and Peng Shi and Jimmy Lin},
+  journal = {arXiv:2108.08787},
+  title = {{Mr. TyDi}: A Multi-lingual Benchmark for Dense Retrieval},
+  year = {2021},
+}
+""",
+    )

mteb/tasks/retrieval/kat/georgian_faq_retrieval.py

@@ -46,10 +46,17 @@ class GeorgianFAQRetrieval(AbsTaskRetrieval):
             split=_EVAL_SPLIT,
             revision=self.metadata.dataset["revision"],
         )
-        question_ids = {
-            question: _id for _id, question in enumerate(set(data["question"]))
-        }
-        answer_ids = {answer: _id for _id, answer in enumerate(set(data["answer"]))}
+
+        question_ids = {}
+        answer_ids = {}
+
+        for row in data:
+            question = row["question"]
+            answer = row["answer"]
+            if question not in question_ids:
+                question_ids[question] = len(question_ids)
+            if answer not in answer_ids:
+                answer_ids[answer] = len(answer_ids)
 
         for row in data:
             question = row["question"]

mteb/tasks/retrieval/kor/__init__.py

@@ -1,4 +1,19 @@
 from .auto_rag_retrieval import AutoRAGRetrieval
 from .ko_strategy_qa import KoStrategyQA
+from .kovidore2_bench_retrieval import (
+    KoVidore2CybersecurityRetrieval,
+    KoVidore2EconomicRetrieval,
+    KoVidore2EnergyRetrieval,
+    KoVidore2HrRetrieval,
+)
+from .squad_kor_v1_retrieval import SQuADKorV1Retrieval
 
-__all__ = ["AutoRAGRetrieval", "KoStrategyQA"]
+__all__ = [
+    "AutoRAGRetrieval",
+    "KoStrategyQA",
+    "KoVidore2CybersecurityRetrieval",
+    "KoVidore2EconomicRetrieval",
+    "KoVidore2EnergyRetrieval",
+    "KoVidore2HrRetrieval",
+    "SQuADKorV1Retrieval",
+]