mteb 2.1.4__py3-none-any.whl → 2.5.2__py3-none-any.whl

mteb/tasks/reranking/multilingual/multi_long_doc_reranking.py

@@ -0,0 +1,70 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class MultiLongDocReranking(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="MultiLongDocReranking",
+        description=(
+            "Reranking version of MultiLongDocRetrieval (MLDR). MLDR is a Multilingual Long-Document "
+            "Retrieval dataset built on Wikipedia, Wudao and mC4, covering 13 typologically diverse languages. "
+            "Specifically, we sample lengthy articles from Wikipedia, Wudao and mC4 datasets and randomly choose "
+            "paragraphs from them. Then we use GPT-3.5 to generate questions based on these paragraphs. "
+            "The generated question and the sampled article constitute a new text pair to the dataset."
+        ),
+        reference="https://huggingface.co/datasets/Shitao/MLDR",
+        dataset={
+            "path": "mteb/MultiLongDocReranking",
+            "revision": "ad09ce14c17bce6edae151b7f6ef12e15d91dbf3",
+        },
+        type="Reranking",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs={
+            "ar": ["ara-Arab"],
+            "de": ["deu-Latn"],
+            "en": ["eng-Latn"],
+            "es": ["spa-Latn"],
+            "fr": ["fra-Latn"],
+            "hi": ["hin-Deva"],
+            "it": ["ita-Latn"],
+            "ja": ["jpn-Jpan"],
+            "ko": ["kor-Kore"],
+            "pt": ["por-Latn"],
+            "ru": ["rus-Cyrl"],
+            "th": ["tha-Thai"],
+            "zh": ["zho-Hans"],
+        },
+        main_score="ndcg_at_10",
+        date=(
+            "2000-01-01",
+            "2024-12-31",
+        ),  # Not found in the paper, guessed using the paper's publication date and constituent datasets
+        domains=[
+            "Encyclopaedic",
+            "Written",
+            "Web",
+            "Non-fiction",
+            "Fiction",
+        ],  # narrativeqa, wikipedia, wudao, mC4
+        task_subtypes=[],
+        license="mit",
+        annotations_creators="LM-generated",  # gpt-3.5
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@misc{bge-m3,
+  archiveprefix = {arXiv},
+  author = {Jianlv Chen and Shitao Xiao and Peitian Zhang and Kun Luo and Defu Lian and Zheng Liu},
+  eprint = {2402.03216},
+  primaryclass = {cs.CL},
+  title = {BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation},
+  year = {2024},
+}
+""",
+        prompt={
+            "query": "Given a question, rerank long documents based on their relevance to answer the question"
+        },
+        adapted_from=["MultiLongDocRetrieval"],
+    )

mteb/tasks/reranking/multilingual/x_glue_wpr_reranking.py

@@ -78,8 +78,7 @@ _CITATION = r"""
 class XGlueWPRReranking(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="XGlueWPRReranking",
-        description="""XGLUE is a new benchmark dataset to evaluate the performance of cross-lingual pre-trained models
-        with respect to cross-lingual natural language understanding and generation. XGLUE is composed of 11 tasks spans 19 languages.""",
+        description="XGLUE is a new benchmark dataset to evaluate the performance of cross-lingual pre-trained models with respect to cross-lingual natural language understanding and generation. XGLUE is composed of 11 tasks spans 19 languages.",
         reference="https://github.com/microsoft/XGLUE",
         dataset={
             "path": "mteb/XGlueWPRReranking",

mteb/tasks/reranking/vie/ask_ubuntu_dup_questions_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class AskUbuntuDupQuestionsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="AskUbuntuDupQuestions-VN",
-        description="""A translated dataset from AskUbuntu Question Dataset - Questions from AskUbuntu with manual annotations marking pairs of questions as similar or non-similar
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from AskUbuntu Question Dataset - Questions from AskUbuntu with manual annotations marking pairs of questions as similar or non-similar The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://github.com/taolei87/askubuntu",
         dataset={
             "path": "mteb/AskUbuntuDupQuestions-VN",

mteb/tasks/reranking/vie/sci_docs_reranking_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SciDocsRerankingVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="SciDocsRR-VN",
-        description="""A translated dataset from Ranking of related scientific papers based on their title.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Ranking of related scientific papers based on their title. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://allenai.org/data/scidocs",
         dataset={
             "path": "mteb/SciDocsRR-VN",

mteb/tasks/reranking/vie/stack_overflow_dup_questions_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class StackOverflowDupQuestionsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="StackOverflowDupQuestions-VN",
-        description="""A translated dataset from Stack Overflow Duplicate Questions Task for questions with the tags Java, JavaScript and Python
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Stack Overflow Duplicate Questions Task for questions with the tags Java, JavaScript and Python The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.microsoft.com/en-us/research/uploads/prod/2019/03/nl4se18LinkSO.pdf",
         dataset={
             "path": "mteb/StackOverflowDupQuestions-VN",

mteb/tasks/retrieval/code/fresh_stack_retrieval.py

@@ -25,11 +25,14 @@ class FreshStackRetrieval(AbsTaskRetrieval):
         dialect=[],
         sample_creation="found",
         bibtex_citation=r"""
-@article{freshstack2023,
-  author = {FreshStack Authors},
-  journal = {arXiv preprint arXiv:2301.12345},
-  title = {FreshStack: A Multi-language Code Generation and Retrieval Benchmark},
-  year = {2023},
+@misc{thakur2025freshstackbuildingrealisticbenchmarks,
+  archiveprefix = {arXiv},
+  author = {Nandan Thakur and Jimmy Lin and Sam Havens and Michael Carbin and Omar Khattab and Andrew Drozdov},
+  eprint = {2504.13128},
+  primaryclass = {cs.IR},
+  title = {FreshStack: Building Realistic Benchmarks for Evaluating Retrieval on Technical Documents},
+  url = {https://arxiv.org/abs/2504.13128},
+  year = {2025},
 }
 """,
     )

mteb/tasks/retrieval/eng/lit_search_retrieval.py

@@ -7,14 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class LitSearchRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="LitSearchRetrieval",
-        description="""
-        The dataset contains the query set and retrieval corpus for the paper LitSearch: A Retrieval Benchmark for
-        Scientific Literature Search. It introduces LitSearch, a retrieval benchmark comprising 597 realistic literature
-        search queries about recent ML and NLP papers. LitSearch is constructed using a combination of (1) questions
-        generated by GPT-4 based on paragraphs containing inline citations from research papers and (2) questions about
-        recently published papers, manually written by their authors. All LitSearch questions were manually examined or
-        edited by experts to ensure high quality.
-        """,
+        description="The dataset contains the query set and retrieval corpus for the paper LitSearch: A Retrieval Benchmark for Scientific Literature Search. It introduces LitSearch, a retrieval benchmark comprising 597 realistic literature search queries about recent ML and NLP papers. LitSearch is constructed using a combination of (1) questions generated by GPT-4 based on paragraphs containing inline citations from research papers and (2) questions about recently published papers, manually written by their authors. All LitSearch questions were manually examined or edited by experts to ensure high quality.",
         reference="https://github.com/princeton-nlp/LitSearch",
         dataset={
             "path": "princeton-nlp/LitSearch",

mteb/tasks/retrieval/eng/vidore_bench_retrieval.py

@@ -351,6 +351,7 @@ class VidoreSyntheticDocQAAIRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:
@@ -394,6 +395,7 @@ class VidoreSyntheticDocQAEnergyRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:
@@ -437,6 +439,7 @@ class VidoreSyntheticDocQAGovernmentReportsRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:
@@ -480,6 +483,7 @@ class VidoreSyntheticDocQAHealthcareIndustryRetrieval(AbsTaskRetrieval):
 }
 """,
         prompt={"query": "Find a screenshot that relevant to the user's question."},
+        adapted_from=["VidoreDocVQARetrieval"],
     )
 
     def load_data(self) -> None:

mteb/tasks/retrieval/jpn/__init__.py

@@ -1,8 +1,12 @@
 from .ja_cwir_retrieval import JaCWIRRetrieval
+from .ja_cwir_retrieval_lite import JaCWIRRetrievalLite
 from .ja_gov_faqs_retrieval import JaGovFaqsRetrieval
 from .ja_qu_ad_retrieval import JaQuADRetrieval
 from .japanese_legal1_retrieval import JapaneseLegal1Retrieval
 from .jaqket_retrieval import JaqketRetrieval
+from .jaqket_retrieval_lite import JaqketRetrievalLite
+from .miracl_ja_retrieval_lite import MIRACLJaRetrievalLite
+from .mr_tydi_ja_retrieval_lite import MrTyDiJaRetrievalLite
 from .nlp_journal_abs_article_retrieval import (
     NLPJournalAbsArticleRetrieval,
     NLPJournalAbsArticleRetrievalV2,
@@ -22,10 +26,14 @@ from .nlp_journal_title_intro_retrieval import (
 
 __all__ = [
     "JaCWIRRetrieval",
+    "JaCWIRRetrievalLite",
     "JaGovFaqsRetrieval",
     "JaQuADRetrieval",
     "JapaneseLegal1Retrieval",
     "JaqketRetrieval",
+    "JaqketRetrievalLite",
+    "MIRACLJaRetrievalLite",
+    "MrTyDiJaRetrievalLite",
     "NLPJournalAbsArticleRetrieval",
     "NLPJournalAbsArticleRetrievalV2",
     "NLPJournalAbsIntroRetrieval",

mteb/tasks/retrieval/jpn/ja_cwir_retrieval.py

@@ -9,10 +9,7 @@ class JaCWIRRetrieval(AbsTaskRetrieval):
 
     metadata = TaskMetadata(
         name="JaCWIRRetrieval",
-        description="""JaCWIR is a small-scale Japanese information retrieval evaluation dataset consisting of
-5000 question texts and approximately 500k web page titles and web page introductions or summaries
-(meta descriptions, etc.). The question texts are created based on one of the 500k web pages,
-and that data is used as a positive example for the question text.""",
+        description="JaCWIR is a small-scale Japanese information retrieval evaluation dataset consisting of 5000 question texts and approximately 500k web page titles and web page introductions or summaries (meta descriptions, etc.). The question texts are created based on one of the 500k web pages, and that data is used as a positive example for the question text.",
         reference="https://huggingface.co/datasets/hotchpotch/JaCWIR",
         dataset={
             "path": "mteb/JaCWIRRetrieval",

mteb/tasks/retrieval/jpn/ja_cwir_retrieval_lite.py

@@ -0,0 +1,47 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JaCWIRRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JaCWIRRetrievalLite",
+        dataset={
+            "path": "mteb/JaCWIRRetrievalLite",
+            "revision": "79472b360242cf2692e24a6d9999ef50d350d672",
+        },
+        description=(
+            "JaCWIR (Japanese Casual Web IR) is a dataset consisting of questions and webpage meta descriptions "
+            "collected from Hatena Bookmark. This is the lightweight version with a reduced corpus "
+            "(302,638 documents) constructed using hard negatives from 5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/hotchpotch/JaCWIR",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2020-01-01", "2025-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=["Article retrieval"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["JaCWIRRetrieval"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@misc{yuichi-tateno-2024-jacwir,
+  author = {Yuichi Tateno},
+  title = {JaCWIR: Japanese Casual Web IR - 日本語情報検索評価のための小規模でカジュアルなWebタイトルと概要のデータセット},
+  url = {https://huggingface.co/datasets/hotchpotch/JaCWIR},
+}
+""",
+    )

mteb/tasks/retrieval/jpn/jaqket_retrieval_lite.py

@@ -0,0 +1,50 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JaqketRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JaqketRetrievalLite",
+        dataset={
+            "path": "mteb/JaqketRetrievalLite",
+            "revision": "860965fbb6526dd8edff12627dacf07c8f5a54f3",
+        },
+        description=(
+            "JAQKET (JApanese Questions on Knowledge of EnTities) is a QA dataset created based on quiz questions. "
+            "This is the lightweight version with a reduced corpus (65,802 documents) constructed using "
+            "hard negatives from 5 high-performance models."
+        ),
+        reference="https://github.com/kumapo/JAQKET-dataset",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2023-10-09", "2025-01-01"),
+        domains=["Encyclopaedic", "Non-fiction", "Written"],
+        task_subtypes=["Question answering"],
+        license="cc-by-sa-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["JaqketRetrieval"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@inproceedings{Kurihara_nlp2020,
+  author = {鈴木正敏 and 鈴木潤 and 松田耕史 and ⻄田京介 and 井之上直也},
+  booktitle = {言語処理学会第26回年次大会},
+  note = {in Japanese},
+  title = {JAQKET: クイズを題材にした日本語 QA データセットの構築},
+  url = {https://www.anlp.jp/proceedings/annual_meeting/2020/pdf_dir/P2-24.pdf},
+  year = {2020},
+}
+""",
+    )

mteb/tasks/retrieval/jpn/miracl_ja_retrieval_lite.py

@@ -0,0 +1,52 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class MIRACLJaRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="MIRACLJaRetrievalLite",
+        dataset={
+            "path": "mteb/MIRACLJaRetrievalLite",
+            "revision": "575c225da29d1f5fec01082afa56f35df0f44295",
+        },
+        description=(
+            "MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual "
+            "retrieval dataset. This is the lightweight Japanese version with a reduced corpus (105,064 documents) "
+            "constructed using hard negatives from 5 high-performance models."
+        ),
+        reference="https://project-miracl.github.io/",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2022-06-01", "2025-01-01"),
+        domains=["Encyclopaedic", "Written"],
+        task_subtypes=[],
+        license="apache-2.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="created",
+        adapted_from=["MIRACLRetrieval"],
+        bibtex_citation=r"""
+@article{10.1162/tacl_a_00595,
+  author = {Zhang, Xinyu and Thakur, Nandan and Ogundepo, Odunayo and Kamalloo, Ehsan and Alfonso-Hermelo, David
+and Li, Xiaoguang and Liu, Qun and Rezagholizadeh, Mehdi and Lin, Jimmy},
+  doi = {10.1162/tacl_a_00595},
+  journal = {Transactions of the Association for Computational Linguistics},
+  pages = {1114-1131},
+  title = {{MIRACL: A Multilingual Retrieval Dataset Covering 18 Diverse Languages}},
+  volume = {11},
+  year = {2023},
+}
+
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/jpn/mr_tydi_ja_retrieval_lite.py

@@ -0,0 +1,48 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class MrTyDiJaRetrievalLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="MrTyDiJaRetrievalLite",
+        dataset={
+            "path": "mteb/MrTyDiJaRetrievalLite",
+            "revision": "b87e6ff25f4e32d1c97498a539ea8aad5fde3cb1",
+        },
+        description=(
+            "Mr.TyDi is a multilingual benchmark dataset built on TyDi for document retrieval tasks. "
+            "This is the lightweight Japanese version with a reduced corpus (93,382 documents) constructed using "
+            "hard negatives from 5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/castorini/mr-tydi",
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2021-01-01", "2025-01-01"),
+        domains=["Encyclopaedic", "Non-fiction", "Written"],
+        task_subtypes=["Question answering"],
+        license="apache-2.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["MrTidyRetrieval"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@article{mrtydi,
+  author = {Xinyu Zhang and Xueguang Ma and Peng Shi and Jimmy Lin},
+  journal = {arXiv:2108.08787},
+  title = {{Mr. TyDi}: A Multi-lingual Benchmark for Dense Retrieval},
+  year = {2021},
+}
+""",
+    )

mteb/tasks/retrieval/kat/georgian_faq_retrieval.py

@@ -46,10 +46,17 @@ class GeorgianFAQRetrieval(AbsTaskRetrieval):
             split=_EVAL_SPLIT,
             revision=self.metadata.dataset["revision"],
         )
-        question_ids = {
-            question: _id for _id, question in enumerate(set(data["question"]))
-        }
-        answer_ids = {answer: _id for _id, answer in enumerate(set(data["answer"]))}
+
+        question_ids = {}
+        answer_ids = {}
+
+        for row in data:
+            question = row["question"]
+            answer = row["answer"]
+            if question not in question_ids:
+                question_ids[question] = len(question_ids)
+            if answer not in answer_ids:
+                answer_ids[answer] = len(answer_ids)
 
         for row in data:
             question = row["question"]

mteb/tasks/retrieval/kor/__init__.py

@@ -1,4 +1,5 @@
 from .auto_rag_retrieval import AutoRAGRetrieval
 from .ko_strategy_qa import KoStrategyQA
+from .squad_kor_v1_retrieval import SQuADKorV1Retrieval
 
-__all__ = ["AutoRAGRetrieval", "KoStrategyQA"]
+__all__ = ["AutoRAGRetrieval", "KoStrategyQA", "SQuADKorV1Retrieval"]

mteb/tasks/retrieval/kor/squad_kor_v1_retrieval.py

@@ -0,0 +1,47 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class SQuADKorV1Retrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SQuADKorV1Retrieval",
+        description="Korean translation of SQuAD v1.0 dataset for retrieval task, based on Korean Wikipedia articles.",
+        reference="https://huggingface.co/datasets/yjoonjang/squad_kor_v1",
+        dataset={
+            "path": "yjoonjang/squad_kor_v1",
+            "revision": "2b4ee1f3b143a04792da93a3df21933c5fe9eed3",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["kor-Hang"],
+        main_score="ndcg_at_10",
+        date=("2018-01-01", "2019-12-31"),
+        domains=["Encyclopaedic", "Written"],
+        task_subtypes=["Question answering"],
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{rajpurkar-etal-2016-squad,
+  address = {Austin, Texas},
+  author = {Rajpurkar, Pranav  and
+Zhang, Jian  and
+Lopyrev, Konstantin  and
+Liang, Percy},
+  booktitle = {Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing},
+  doi = {10.18653/v1/D16-1264},
+  editor = {Su, Jian  and
+Duh, Kevin  and
+Carreras, Xavier},
+  month = nov,
+  pages = {2383--2392},
+  publisher = {Association for Computational Linguistics},
+  title = {{SQ}u{AD}: 100,000+ Questions for Machine Comprehension of Text},
+  url = {https://aclanthology.org/D16-1264},
+  year = {2016},
+}
+""",
+    )

mteb/tasks/retrieval/multilingual/__init__.py

@@ -81,6 +81,18 @@ from .vidore2_bench_retrieval import (
     Vidore2ESGReportsHLRetrieval,
     Vidore2ESGReportsRetrieval,
 )
+from .vidore3_bench_retrieval import (
+    Vidore3ComputerScienceRetrieval,
+    Vidore3EnergyRetrieval,
+    Vidore3FinanceEnRetrieval,
+    Vidore3FinanceFrRetrieval,
+    Vidore3HrRetrieval,
+    Vidore3IndustrialRetrieval,
+    Vidore3NuclearRetrieval,
+    Vidore3PharmaceuticalsRetrieval,
+    Vidore3PhysicsRetrieval,
+    Vidore3TelecomRetrieval,
+)
 from .web_faq_retrieval import WebFAQRetrieval
 from .wikipedia_retrieval_multilingual import WikipediaRetrievalMultilingual
 from .wit_t2i_retrieval import WITT2IRetrieval
@@ -161,6 +173,16 @@ __all__ = [
     "Vidore2ESGReportsHLRetrieval",
     "Vidore2ESGReportsRetrieval",
     "Vidore2EconomicsReportsRetrieval",
+    "Vidore3ComputerScienceRetrieval",
+    "Vidore3EnergyRetrieval",
+    "Vidore3FinanceEnRetrieval",
+    "Vidore3FinanceFrRetrieval",
+    "Vidore3HrRetrieval",
+    "Vidore3IndustrialRetrieval",
+    "Vidore3NuclearRetrieval",
+    "Vidore3PharmaceuticalsRetrieval",
+    "Vidore3PhysicsRetrieval",
+    "Vidore3TelecomRetrieval",
     "WITT2IRetrieval",
     "WebFAQRetrieval",
     "WikipediaRetrievalMultilingual",

mteb/tasks/retrieval/multilingual/belebele_retrieval.py

@@ -230,10 +230,11 @@ class BelebeleRetrieval(AbsTaskRetrieval):
             ds_corpus = self.dataset[lang_corpus]
             ds_question = self.dataset[lang_question]
 
-            question_ids = {
-                question: _id
-                for _id, question in enumerate(set(ds_question["question"]))
-            }
+            question_ids = {}
+            for row in ds_question:
+                question = row["question"]
+                if question not in question_ids:
+                    question_ids[question] = len(question_ids)
 
             link_to_context_id = {}
             context_idx = 0