mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/tasks/retrieval/vie/fi_qa2018_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class FiQA2018VN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="FiQA2018-VN",
-        description="""A translated dataset from Financial Opinion Mining and Question Answering
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Financial Opinion Mining and Question Answering The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://sites.google.com/view/fiqa/",
         dataset={
             "path": "GreenNode/fiqa-vn",

mteb/tasks/retrieval/vie/green_node_table_markdown_retrieval.py

@@ -26,5 +26,20 @@ class GreenNodeTableMarkdownRetrieval(AbsTaskRetrieval):
         annotations_creators="human-annotated",
         dialect=[],
         sample_creation="found",
-        bibtex_citation="",  # TODO: Add bibtex citation when the paper is published
+        bibtex_citation=r"""
+@inproceedings{10.1007/978-981-95-1746-6_17,
+  abstract = {Information retrieval often comes in plain text, lacking semi-structured text such as HTML and markdown, retrieving data that contains rich format such as table became non-trivial. In this paper, we tackle this challenge by introducing a new dataset, GreenNode Table Retrieval VN (GN-TRVN), which is collected from a massive corpus, a wide range of topics, and a longer context compared to ViQuAD2.0. To evaluate the effectiveness of our proposed dataset, we introduce two versions, M3-GN-VN and M3-GN-VN-Mixed, by fine-tuning the M3-Embedding model on this dataset. Experimental results show that our models consistently outperform the baselines, including the base model, across most evaluation criteria on various datasets such as VieQuADRetrieval, ZacLegalTextRetrieval, and GN-TRVN. In general, we release a more comprehensive dataset and two model versions that improve response performance for Vietnamese Markdown Table Retrieval.},
+  address = {Singapore},
+  author = {Pham, Bao Loc
+and Hoang, Quoc Viet
+and Luu, Quy Tung
+and Vo, Trong Thu},
+  booktitle = {Proceedings of the Fifth International Conference on Intelligent Systems and Networks},
+  isbn = {978-981-95-1746-6},
+  pages = {153--163},
+  publisher = {Springer Nature Singapore},
+  title = {GN-TRVN: A Benchmark for Vietnamese Table Markdown Retrieval Task},
+  year = {2026},
+}
+""",
     )

mteb/tasks/retrieval/vie/hotpot_qavn_retrieval.py

@@ -9,12 +9,7 @@ class HotpotQAVN(AbsTaskRetrieval):
             "path": "GreenNode/hotpotqa-vn",
             "revision": "8a5220c7af5084f0d5d2afeb74f9c2b41b759ff0",
         },
-        description="""A translated dataset from HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong
-            supervision for supporting facts to enable more explainable question answering systems.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong supervision for supporting facts to enable more explainable question answering systems. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://hotpotqa.github.io/",
         type="Retrieval",
         category="t2t",
@@ -41,3 +36,42 @@ class HotpotQAVN(AbsTaskRetrieval):
 """,
         adapted_from=["HotpotQA"],
     )
+
+
+class NanoHotpotQAVN(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoHotpotQA-VN",
+        dataset={
+            "path": "GreenNode/nano-hotpotqa-vn",
+            "revision": "f4de19a2fae1a582de114e5bcd178bb262183113",
+        },
+        description="NanoHotpotQAVN is a small version of A translated dataset from HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong supervision for supporting facts to enable more explainable question answering systems. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
+        reference="https://hotpotqa.github.io/",
+        type="Retrieval",
+        category="t2t",
+        eval_splits=["test"],
+        eval_langs=["vie-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-07-29", "2025-07-30"),
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="machine-translated and LM verified",
+        domains=["Web", "Written"],
+        task_subtypes=["Question answering"],
+        bibtex_citation=r"""
+@misc{pham2025vnmtebvietnamesemassivetext,
+  archiveprefix = {arXiv},
+  author = {Loc Pham and Tung Luu and Thu Vo and Minh Nguyen and Viet Hoang},
+  eprint = {2507.21500},
+  primaryclass = {cs.CL},
+  title = {VN-MTEB: Vietnamese Massive Text Embedding Benchmark},
+  url = {https://arxiv.org/abs/2507.21500},
+  year = {2025},
+}
+""",
+        prompt={
+            "query": "Given a multi-hop question, retrieve documents that can help answer the question"
+        },
+        adapted_from=["HotpotQA-VN"],
+    )

mteb/tasks/retrieval/vie/msmarcovn_retrieval.py

@@ -9,11 +9,7 @@ class MSMARCOVN(AbsTaskRetrieval):
             "path": "GreenNode/msmarco-vn",
             "revision": "85d1ad4cc9070b8d019d65f5af1631a2ab91e294",
         },
-        description="""A translated dataset from MS MARCO is a collection of datasets focused on deep learning in search
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from MS MARCO is a collection of datasets focused on deep learning in search The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://microsoft.github.io/msmarco/",
         type="Retrieval",
         category="t2t",
@@ -51,3 +47,51 @@ class MSMARCOVN(AbsTaskRetrieval):
 """,
         adapted_from=["MSMARCO"],
     )
+
+
+class NanoMSMARCOVN(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoMSMARCO-VN",
+        dataset={
+            "path": "GreenNode/nano-msmarco-vn",
+            "revision": "f149369c82ec228b05b0f6677699ab4bfbab73f6",
+        },
+        description="NanoMSMARCOVN is a small version of A translated dataset from MS MARCO is a collection of datasets focused on deep learning in search The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
+        reference="https://microsoft.github.io/msmarco/",
+        type="Retrieval",
+        category="t2t",
+        eval_splits=["dev"],
+        eval_langs=["vie-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-07-29", "2025-07-30"),
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="machine-translated and LM verified",
+        domains=[
+            "Encyclopaedic",
+            "Academic",
+            "Blog",
+            "News",
+            "Medical",
+            "Government",
+            "Reviews",
+            "Non-fiction",
+            "Social",
+            "Web",
+        ],
+        task_subtypes=["Question answering"],
+        bibtex_citation=r"""
+@misc{pham2025vnmtebvietnamesemassivetext,
+  archiveprefix = {arXiv},
+  author = {Loc Pham and Tung Luu and Thu Vo and Minh Nguyen and Viet Hoang},
+  eprint = {2507.21500},
+  primaryclass = {cs.CL},
+  title = {VN-MTEB: Vietnamese Massive Text Embedding Benchmark},
+  url = {https://arxiv.org/abs/2507.21500},
+  year = {2025},
+}
+""",
+        prompt={"query": "Given a query, retrieve relevant documents from MS MARCO-VN"},
+        adapted_from=["MSMARCO-VN"],
+    )

mteb/tasks/retrieval/vie/nf_corpus_vn_retrieval.py

@@ -9,11 +9,7 @@ class NFCorpusVN(AbsTaskRetrieval):
             "path": "GreenNode/nfcorpus-vn",
             "revision": "a13d72fbb859be3dc19ab669d1ec9510407d2dcd",
         },
-        description="""A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/nqvn_retrieval.py

@@ -9,11 +9,7 @@ class NQVN(AbsTaskRetrieval):
             "path": "GreenNode/nq-vn",
             "revision": "40a6d7f343b9c9f4855a426d8c431ad5f8aaf56b",
         },
-        description="""A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://ai.google.com/research/NaturalQuestions/",
         type="Retrieval",
         category="t2t",
@@ -40,3 +36,42 @@ class NQVN(AbsTaskRetrieval):
 """,
         adapted_from=["NQ"],
     )
+
+
+class NanoNQVN(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoNQ-VN",
+        dataset={
+            "path": "GreenNode/nano-nq-vn",
+            "revision": "1ad4d6556fe0e5314994839089ce070fb0db8b19",
+        },
+        description="NanoNQVN is a small version of A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
+        reference="https://ai.google.com/research/NaturalQuestions/",
+        type="Retrieval",
+        category="t2t",
+        eval_splits=["test"],
+        eval_langs=["vie-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-07-29", "2025-07-30"),
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="machine-translated and LM verified",
+        domains=["Written", "Encyclopaedic"],
+        task_subtypes=["Question answering"],
+        bibtex_citation=r"""
+@misc{pham2025vnmtebvietnamesemassivetext,
+  archiveprefix = {arXiv},
+  author = {Loc Pham and Tung Luu and Thu Vo and Minh Nguyen and Viet Hoang},
+  eprint = {2507.21500},
+  primaryclass = {cs.CL},
+  title = {VN-MTEB: Vietnamese Massive Text Embedding Benchmark},
+  url = {https://arxiv.org/abs/2507.21500},
+  year = {2025},
+}
+""",
+        prompt={
+            "query": "Given a question, retrieve Wikipedia passages that answer the question"
+        },
+        adapted_from=["NQ-VN"],
+    )

mteb/tasks/retrieval/vie/quora_vn_retrieval.py

@@ -9,12 +9,7 @@ class QuoraVN(AbsTaskRetrieval):
             "path": "GreenNode/quora-vn",
             "revision": "3363d81e41b67c1032bf3b234882a03d271e2289",
         },
-        description="""A translated dataset from QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a
-            question, find other (duplicate) questions.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a question, find other (duplicate) questions. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/sci_fact_vn_retrieval.py

@@ -9,11 +9,7 @@ class SciFactVN(AbsTaskRetrieval):
             "path": "GreenNode/scifact-vn",
             "revision": "483a7cf890c523c954e7751d328c5bb65061dcff",
         },
-        description="""A translated dataset from SciFact verifies scientific claims using evidence from the research literature containing scientific paper abstracts.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from SciFact verifies scientific claims using evidence from the research literature containing scientific paper abstracts. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://github.com/allenai/scifact",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/scidocsvn_retrieval.py

@@ -9,12 +9,7 @@ class SCIDOCSVN(AbsTaskRetrieval):
             "path": "GreenNode/scidocs-vn",
             "revision": "724cddfa9d328a193f303a0a9b7789468ac79f26",
         },
-        description="""A translated dataset from SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation
-            prediction, to document classification and recommendation.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation prediction, to document classification and recommendation. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://allenai.org/data/scidocs",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/touche2020_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class Touche2020VN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Touche2020-VN",
-        description="""A translated dataset from Touché Task 1: Argument Retrieval for Controversial Questions
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Touché Task 1: Argument Retrieval for Controversial Questions The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://webis.de/events/touche-20/shared-task-1.html",
         dataset={
             "path": "GreenNode/webis-touche2020-vn",

mteb/tasks/retrieval/vie/treccovidvn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class TRECCOVIDVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="TRECCOVID-VN",
-        description="""A translated dataset from TRECCOVID is an ad-hoc search challenge based on the COVID-19 dataset containing scientific articles related to the COVID-19 pandemic.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from TRECCOVID is an ad-hoc search challenge based on the COVID-19 dataset containing scientific articles related to the COVID-19 pandemic. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://ir.nist.gov/covidSubmit/index.html",
         dataset={
             "path": "GreenNode/trec-covid-vn",

mteb/tasks/retrieval/vie/tvpl_retrieval.py

@@ -0,0 +1,42 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+TEST_SAMPLES = 2048
+
+
+class TVPLRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="TVPLRetrieval",
+        description="A Vietnamese dataset for evaluating legal text retrieval. From Thu vien phap luat (TVPL) dataset: Optimizing Answer Generator in Vietnamese Legal Question Answering Systems Using Language Models.",
+        reference="https://aclanthology.org/2020.coling-main.233.pdf",
+        dataset={
+            "path": "GreenNode/TVPL-Retrieval-VN",
+            "revision": "6661dba4dfedff606537732d9f35f2c3738b081a",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["vie-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-07-29", "2025-07-30"),
+        license="cc-by-sa-4.0",
+        dialect=[],
+        annotations_creators="human-annotated",
+        domains=["Legal"],
+        task_subtypes=["Question answering"],
+        sample_creation="found",
+        bibtex_citation=r"""
+@article{10.1145/3732938,
+  address = {New York, NY, USA},
+  author = {Le, Huong and Luu, Ngoc and Nguyen, Thanh and Dao, Tuan and Dinh, Sang},
+  doi = {10.1145/3732938},
+  issn = {2375-4699},
+  journal = {ACM Trans. Asian Low-Resour. Lang. Inf. Process.},
+  publisher = {Association for Computing Machinery},
+  title = {Optimizing Answer Generator in Vietnamese Legal Question Answering Systems Using Language Models},
+  url = {https://doi.org/10.1145/3732938},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/vie/zac_legal_text_retrieval.py

@@ -24,5 +24,19 @@ class ZacLegalTextRetrieval(AbsTaskRetrieval):
         annotations_creators="human-annotated",
         dialect=[],
         sample_creation="found",
-        bibtex_citation="",  # TODO: Add bibtex citation when the paper is published
+        bibtex_citation=r"""
+@inproceedings{10.1007/978-981-95-1746-6_17,
+  address = {Singapore},
+  author = {Pham, Bao Loc
+and Hoang, Quoc Viet
+and Luu, Quy Tung
+and Vo, Trong Thu},
+  booktitle = {Proceedings of the Fifth International Conference on Intelligent Systems and Networks},
+  isbn = {978-981-95-1746-6},
+  pages = {153--163},
+  publisher = {Springer Nature Singapore},
+  title = {GN-TRVN: A Benchmark for Vietnamese Table Markdown Retrieval Task},
+  year = {2026},
+}
+""",
     )

mteb/tasks/sts/nld/sick_nl_sts.py

@@ -35,6 +35,7 @@ class SICKNLSTS(AbsTaskSTS):
   year = {2021},
 }
 """,
+        prompt={"query": "Haal semantisch vergelijkbare tekst op"},
     )
 
     min_score = 0

mteb/tasks/sts/vie/biosses_stsvn.py

@@ -9,11 +9,7 @@ class BiossesSTSVN(AbsTaskSTS):
             "path": "GreenNode/biosses-sts-vn",
             "revision": "1dae4a6df91c0852680cd4ab48c8c1d8a9ed49b2",
         },
-        description="""A translated dataset from Biomedical Semantic Similarity Estimation.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Biomedical Semantic Similarity Estimation. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://tabilab.cmpe.boun.edu.tr/BIOSSES/DataSet.html",
         type="STS",
         category="t2c",

mteb/tasks/sts/vie/sickr_stsvn.py

@@ -9,11 +9,7 @@ class SickrSTSVN(AbsTaskSTS):
             "path": "GreenNode/sickr-sts-vn",
             "revision": "bc89f0401983c456b609f7fb324278f346b2cccf",
         },
-        description="""A translated dataset from Semantic Textual Similarity SICK-R dataset as described here:
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Semantic Textual Similarity SICK-R dataset as described here: The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://aclanthology.org/2020.lrec-1.207",
         type="STS",
         category="t2c",

mteb/tasks/sts/vie/sts_benchmark_stsvn.py

@@ -9,11 +9,7 @@ class STSBenchmarkSTSVN(AbsTaskSTS):
             "path": "GreenNode/stsbenchmark-sts-vn",
             "revision": "f24d66738cda4a02138ada5af7689a92ce1fcad6",
         },
-        description="""A translated dataset from Semantic Textual Similarity Benchmark (STSbenchmark) dataset.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Semantic Textual Similarity Benchmark (STSbenchmark) dataset. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://github.com/PhilipMay/stsb-multi-mt/",
         type="STS",
         category="t2c",

mteb/tasks/zeroshot_classification/eng/gtsrb.py

@@ -9,7 +9,7 @@ from mteb.abstasks.zeroshot_classification import (
 class GTSRBZeroShotClassification(AbsTaskZeroShotClassification):
     metadata = TaskMetadata(
         name="GTSRBZeroShot",
-        description="""The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.""",
+        description="The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.",
         reference="https://benchmark.ini.rub.de/",
         dataset={
             "path": "clip-benchmark/wds_gtsrb",

mteb/tasks/zeroshot_classification/eng/patch_camelyon.py

@@ -9,7 +9,7 @@ from mteb.abstasks.zeroshot_classification import (
 class PatchCamelyonZeroShotClassification(AbsTaskZeroShotClassification):
     metadata = TaskMetadata(
         name="PatchCamelyonZeroShot",
-        description="""Histopathology diagnosis classification dataset.""",
+        description="Histopathology diagnosis classification dataset.",
         reference="https://link.springer.com/chapter/10.1007/978-3-030-00934-2_24",
         dataset={
             "path": "clip-benchmark/wds_vtab-pcam",

mteb/tasks/zeroshot_classification/eng/ucf101.py

@@ -7,11 +7,7 @@ from mteb.abstasks.zeroshot_classification import (
 class UCF101ZeroShotClassification(AbsTaskZeroShotClassification):
     metadata = TaskMetadata(
         name="UCF101ZeroShot",
-        description="""UCF101 is an action recognition data set of realistic
-action videos collected from YouTube, having 101 action categories. This
-version of the dataset does not contain images but images saved frame by
-frame. Train and test splits are generated based on the authors' first
-version train/test list.""",
+        description="UCF101 is an action recognition data set of realistic action videos collected from YouTube, having 101 action categories. This version of the dataset does not contain images but images saved frame by frame. Train and test splits are generated based on the authors' first version train/test list.",
         reference="https://huggingface.co/datasets/flwrlabs/ucf101",
         dataset={
             "path": "flwrlabs/ucf101",

mteb/types/__init__.py

@@ -4,6 +4,7 @@ from ._encoder_io import (
     Conversation,
     ConversationTurn,
     CorpusDatasetType,
+    EncodeKwargs,
     InstructionDatasetType,
     PromptType,
     QueryDatasetType,
@@ -30,6 +31,7 @@ __all__ = [
     "Conversation",
     "ConversationTurn",
     "CorpusDatasetType",
+    "EncodeKwargs",
     "HFSubset",
     "ISOLanguage",
     "ISOLanguageScript",

mteb/types/_encoder_io.py

@@ -1,13 +1,30 @@
+from __future__ import annotations
+
 from collections.abc import Mapping
 from enum import Enum
-from typing import TypedDict
+from typing import TYPE_CHECKING, TypedDict
 
 import numpy as np
 import torch
 from datasets import Dataset
-from PIL import Image
 from typing_extensions import NotRequired
 
+if TYPE_CHECKING:
+    from PIL import Image
+
+
+class EncodeKwargs(TypedDict):
+    """Keyword arguments for encoding methods.
+
+    Attributes:
+        batch_size: The batch size to use for encoding.
+        show_progress_bar: Whether to show a progress bar during encoding.
+    """
+
+    batch_size: NotRequired[int]
+    show_progress_bar: NotRequired[bool]
+
+
 # --- Output types ---
 Array = np.ndarray | torch.Tensor
 """General array type, can be a numpy array or a torch tensor."""

mteb/types/_result.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, NamedTuple
 
 HFSubset = str
@@ -8,7 +9,7 @@ SplitName = str
 Score = Any
 """A score value, could e.g. be accuracy. Normally it is a float or int, but it can take on any value. Should be json serializable."""
 
-ScoresDict = dict[str, Score]
+ScoresDict = Mapping[str, Score]
 """A dictionary of scores, typically also include metadata, e.g {'main_score': 0.5, 'accuracy': 0.5, 'f1': 0.6, 'hf_subset': 'en-de', 'languages': ['eng-Latn', 'deu-Latn']}"""
 
 

mteb/types/statistics.py

@@ -10,8 +10,14 @@ class SplitDescriptiveStatistics(TypedDict):
 
 
 class DescriptiveStatistics(TypedDict, SplitDescriptiveStatistics):
-    """Class for descriptive statistics for the full task."""
+    """Class for descriptive statistics for the full task.
 
+    Attributes:
+        num_samples: Total number of samples
+        hf_subset_descriptive_stats: HFSubset descriptive statistics (only for multilingual datasets)
+    """
+
+    num_samples: int
     hf_subset_descriptive_stats: NotRequired[dict[HFSubset, SplitDescriptiveStatistics]]
 
 
@@ -88,9 +94,9 @@ class ScoreStatistics(TypedDict):
         max_score: Maximum score
     """
 
-    min_score: int
+    min_score: int | float
     avg_score: float
-    max_score: int
+    max_score: int | float
 
 
 class TopRankedStatistics(TypedDict):