mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/tasks/retrieval/nld/sci_fact_nl_retrieval.py

@@ -1,30 +1,26 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class SciFactNL(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="SciFact-NL",
-        dataset={
-            "path": "clips/beir-nl-scifact",
-            "revision": "856d8dfc294b138856bbf3042450e3782321e44e",
-        },
-        description="SciFactNL verifies scientific claims in Dutch using evidence from the research literature containing scientific paper abstracts.",
-        reference="https://huggingface.co/datasets/clips/beir-nl-scifact",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["nld-Latn"],
-        main_score="ndcg_at_10",
-        date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
-        domains=["Academic", "Medical", "Written"],
-        task_subtypes=[],
-        license="cc-by-4.0",
-        annotations_creators="derived",
-        dialect=[],
-        sample_creation="machine-translated and verified",  # manually checked a small subset
-        bibtex_citation=r"""
+_sci_fact_nl_metadata = dict(
+    dataset={
+        "path": "clips/beir-nl-scifact",
+        "revision": "856d8dfc294b138856bbf3042450e3782321e44e",
+    },
+    reference="https://huggingface.co/datasets/clips/beir-nl-scifact",
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["nld-Latn"],
+    main_score="ndcg_at_10",
+    date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
+    domains=["Academic", "Medical", "Written"],
+    task_subtypes=[],
+    license="cc-by-4.0",
+    annotations_creators="derived",
+    dialect=[],
+    sample_creation="machine-translated and verified",  # manually checked a small subset
+    bibtex_citation=r"""
 @misc{banar2024beirnlzeroshotinformationretrieval,
   archiveprefix = {arXiv},
   author = {Nikolay Banar and Ehsan Lotfi and Walter Daelemans},
@@ -35,5 +31,27 @@ class SciFactNL(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+)
+
+
+class SciFactNL(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SciFact-NL",
+        description="SciFactNL verifies scientific claims in Dutch using evidence from the research literature "
+        "containing scientific paper abstracts.",
         adapted_from=["SciFact"],
+        **_sci_fact_nl_metadata,
+    )
+
+
+class SciFactNLv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SciFact-NL.v2",
+        description="SciFactNL verifies scientific claims in Dutch using evidence from the research literature "
+        "containing scientific paper abstracts. This version adds a Dutch prompt to the dataset.",
+        adapted_from=["SciFact-NL"],
+        prompt={
+            "query": "Given a scientific claim, retrieve documents that support or refute the claim"
+        },
+        **_sci_fact_nl_metadata,
     )

mteb/tasks/retrieval/nld/scidocsnl_retrieval.py

@@ -1,33 +1,26 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class SCIDOCSNL(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="SCIDOCS-NL",
-        dataset={
-            "path": "clips/beir-nl-scidocs",
-            "revision": "4e018aa220029f9d1bd5a31de3650e322e32ea38",
-        },
-        description=(
-            "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation"
-            + " prediction, to document classification and recommendation. SciDocs-NL is a Dutch translation."
-        ),
-        reference="https://huggingface.co/datasets/clips/beir-nl-scidocs",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["nld-Latn"],
-        main_score="ndcg_at_10",
-        date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
-        domains=["Academic", "Written", "Non-fiction"],
-        task_subtypes=[],
-        license="cc-by-sa-4.0",
-        annotations_creators="derived",
-        dialect=[],
-        sample_creation="machine-translated and verified",  # manually checked a small subset
-        bibtex_citation=r"""
+_scidocsnl_metadata = dict(
+    dataset={
+        "path": "clips/beir-nl-scidocs",
+        "revision": "4e018aa220029f9d1bd5a31de3650e322e32ea38",
+    },
+    reference="https://huggingface.co/datasets/clips/beir-nl-scidocs",
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["nld-Latn"],
+    main_score="ndcg_at_10",
+    date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
+    domains=["Academic", "Written", "Non-fiction"],
+    task_subtypes=[],
+    license="cc-by-sa-4.0",
+    annotations_creators="derived",
+    dialect=[],
+    sample_creation="machine-translated and verified",  # manually checked a small subset
+    bibtex_citation=r"""
 @misc{banar2024beirnlzeroshotinformationretrieval,
   archiveprefix = {arXiv},
   author = {Nikolay Banar and Ehsan Lotfi and Walter Daelemans},
@@ -38,5 +31,29 @@ class SCIDOCSNL(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+)
+
+
+class SCIDOCSNL(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SCIDOCS-NL",
+        description="SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from "
+        "citation prediction, to document classification and recommendation. SciDocs-NL is a Dutch "
+        "translation.",
         adapted_from=["SCIDOCS"],
+        **_scidocsnl_metadata,
+    )
+
+
+class SCIDOCSNLv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SCIDOCS-NL.v2",
+        description="SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from "
+        "citation prediction, to document classification and recommendation. SciDocs-NL is a Dutch "
+        "translation. This version adds a Dutch prompt to the dataset.",
+        adapted_from=["SCIDOCS-NL"],
+        **_scidocsnl_metadata,
+        prompt={
+            "query": "Gegeven de titel van een wetenschappelijk artikel, haal de abstracts op van artikelen die door het gegeven artikel worden geciteerd"
+        },
     )

mteb/tasks/retrieval/nld/vabb_retrieval.py

@@ -38,4 +38,7 @@ class VABBRetrieval(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+        prompt={
+            "query": "Gegeven een titel, haal de wetenschappelijke abstract op die het beste bij de titel past"
+        },
     )

mteb/tasks/retrieval/nob/norquad.py

@@ -54,7 +54,7 @@ Fishel, Mark},
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -71,7 +71,7 @@ Fishel, Mark},
         text2id = {}
 
         for split in self.dataset:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.shuffle(seed=42)
             max_samples = min(1024, len(ds))
             ds = ds.select(

mteb/tasks/retrieval/nob/snl_retrieval.py

@@ -41,7 +41,7 @@ class SNLRetrieval(AbsTaskRetrieval):
         """Load dataset from HuggingFace hub"""
         if self.data_loaded:
             return
-        self.dataset = datasets.load_dataset(**self.metadata.dataset)  # type: ignore
+        self.dataset = datasets.load_dataset(**self.metadata.dataset)
         self.dataset_transform()
         self.data_loaded = True
 
@@ -58,7 +58,7 @@ class SNLRetrieval(AbsTaskRetrieval):
         text2id = {}
 
         for split in self.dataset:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.shuffle(seed=42)
 
             self.queries[split] = {}

mteb/tasks/retrieval/slk/slovak_sum_retrieval.py

@@ -7,13 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SlovakSumRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="SlovakSumRetrieval",
-        description="""
-            SlovakSum, a Slovak news summarization dataset consisting of over 200 thousand
-            news articles with titles and short abstracts obtained from multiple Slovak newspapers.
-
-            Originally intended as a summarization task, but since no human annotations were provided
-            here reformulated to a retrieval task.
-        """,
+        description="SlovakSum, a Slovak news summarization dataset consisting of over 200 thousand news articles with titles and short abstracts obtained from multiple Slovak newspapers. Originally intended as a summarization task, but since no human annotations were provided here reformulated to a retrieval task.",
         reference="https://huggingface.co/datasets/NaiveNeuron/slovaksum",
         dataset={
             "path": "NaiveNeuron/slovaksum",

mteb/tasks/retrieval/tur/tur_hist_quad.py

@@ -59,7 +59,7 @@ class TurHistQuadRetrieval(AbsTaskRetrieval):
         text2id = {}
 
         for split in self.metadata.eval_splits:
-            ds: datasets.Dataset = self.dataset[split]  # type: ignore
+            ds: datasets.Dataset = self.dataset[split]
             ds = ds.shuffle(seed=42)
             max_samples = min(1024, len(ds))
             ds = ds.select(

mteb/tasks/retrieval/vie/__init__.py

@@ -1,5 +1,5 @@
 from .argu_ana_vn_retrieval import ArguAnaVN
-from .climate_fevervn_retrieval import ClimateFEVERVN
+from .climate_fevervn_retrieval import ClimateFEVERVN, NanoClimateFEVERVN
 from .cqa_dupstack_android_vn_retrieval import CQADupstackAndroidVN
 from .cqa_dupstack_gis_vn_retrieval import CQADupstackGisVN
 from .cqa_dupstack_mathematica_vn_retrieval import CQADupstackMathematicaVN
@@ -10,19 +10,20 @@ from .cqa_dupstack_tex_vn_retrieval import CQADupstackTexVN
 from .cqa_dupstack_unix_vn_retrieval import CQADupstackUnixVN
 from .cqa_dupstack_webmasters_vn_retrieval import CQADupstackWebmastersVN
 from .cqa_dupstack_wordpress_vn_retrieval import CQADupstackWordpressVN
-from .db_pedia_vn_retrieval import DBPediaVN
-from .fevervn_retrieval import FEVERVN
+from .db_pedia_vn_retrieval import DBPediaVN, NanoDBPediaVN
+from .fevervn_retrieval import FEVERVN, NanoFEVERVN
 from .fi_qa2018_vn_retrieval import FiQA2018VN
 from .green_node_table_markdown_retrieval import GreenNodeTableMarkdownRetrieval
-from .hotpot_qavn_retrieval import HotpotQAVN
-from .msmarcovn_retrieval import MSMARCOVN
+from .hotpot_qavn_retrieval import HotpotQAVN, NanoHotpotQAVN
+from .msmarcovn_retrieval import MSMARCOVN, NanoMSMARCOVN
 from .nf_corpus_vn_retrieval import NFCorpusVN
-from .nqvn_retrieval import NQVN
+from .nqvn_retrieval import NQVN, NanoNQVN
 from .quora_vn_retrieval import QuoraVN
 from .sci_fact_vn_retrieval import SciFactVN
 from .scidocsvn_retrieval import SCIDOCSVN
 from .touche2020_vn_retrieval import Touche2020VN
 from .treccovidvn_retrieval import TRECCOVIDVN
+from .tvpl_retrieval import TVPLRetrieval
 from .vie_qu_ad_retrieval import VieQuADRetrieval
 from .zac_legal_text_retrieval import ZacLegalTextRetrieval
 
@@ -49,8 +50,15 @@ __all__ = [
     "GreenNodeTableMarkdownRetrieval",
     "HotpotQAVN",
     "NFCorpusVN",
+    "NanoClimateFEVERVN",
+    "NanoDBPediaVN",
+    "NanoFEVERVN",
+    "NanoHotpotQAVN",
+    "NanoMSMARCOVN",
+    "NanoNQVN",
     "QuoraVN",
     "SciFactVN",
+    "TVPLRetrieval",
     "Touche2020VN",
     "VieQuADRetrieval",
     "ZacLegalTextRetrieval",

mteb/tasks/retrieval/vie/argu_ana_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class ArguAnaVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ArguAna-VN",
-        description="""A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://argumentation.bplaced.net/arguana/data",
         dataset={
             "path": "GreenNode/arguana-vn",

mteb/tasks/retrieval/vie/climate_fevervn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class ClimateFEVERVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ClimateFEVER-VN",
-        description="""A translated dataset from CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
         dataset={
             "path": "GreenNode/climate-fever-vn",
@@ -40,3 +36,42 @@ class ClimateFEVERVN(AbsTaskRetrieval):
 """,
         adapted_from=["ClimateFEVER"],
     )
+
+
+class NanoClimateFEVERVN(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoClimateFEVER-VN",
+        description="NanoClimateFEVERVN is a small version of A translated dataset from CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
+        reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
+        dataset={
+            "path": "GreenNode/nano-climate-fever-vn",
+            "revision": "1852e852f07403d4529a8520d52b91ff6d57869b",
+        },
+        type="Retrieval",
+        category="t2t",
+        eval_splits=["test"],
+        eval_langs=["vie-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-07-29", "2025-07-30"),
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="machine-translated and LM verified",
+        domains=["Encyclopaedic", "Written"],
+        task_subtypes=["Claim verification"],
+        bibtex_citation=r"""
+@misc{pham2025vnmtebvietnamesemassivetext,
+  archiveprefix = {arXiv},
+  author = {Loc Pham and Tung Luu and Thu Vo and Minh Nguyen and Viet Hoang},
+  eprint = {2507.21500},
+  primaryclass = {cs.CL},
+  title = {VN-MTEB: Vietnamese Massive Text Embedding Benchmark},
+  url = {https://arxiv.org/abs/2507.21500},
+  year = {2025},
+}
+""",
+        prompt={
+            "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
+        },
+        adapted_from=["ClimateFEVER-VN"],
+    )

mteb/tasks/retrieval/vie/cqa_dupstack_android_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackAndroidVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackAndroid-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-android-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_gis_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackGisVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackGis-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-gis-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_mathematica_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackMathematicaVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackMathematica-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-mathematica-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_physics_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackPhysicsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackPhysics-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-physics-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_programmers_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackProgrammersRetrievalVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackProgrammers-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-programmers-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_stats_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackStatsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackStats-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-stats-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_tex_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackTexVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackTex-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-tex-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_unix_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackUnixVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackUnix-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-unix-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_webmasters_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackWebmastersVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackWebmasters-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-webmasters-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_wordpress_vn_retrieval.py

@@ -9,11 +9,7 @@ class CQADupstackWordpressVN(AbsTaskRetrieval):
             "path": "GreenNode/cqadupstack-wordpress-vn",
             "revision": "2230f80e1baf42aa005731ca86577621c566fcd7",
         },
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/db_pedia_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class DBPediaVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="DBPedia-VN",
-        description="""A translated dataset from DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://github.com/iai-group/DBpedia-Entity/",
         dataset={
             "path": "GreenNode/dbpedia-vn",
@@ -40,3 +36,42 @@ class DBPediaVN(AbsTaskRetrieval):
 """,
         adapted_from=["DBPedia"],
     )
+
+
+class NanoDBPediaVN(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoDBPedia-VN",
+        description="NanoDBPediaVN is a small version of A translated dataset from DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
+        reference="https://github.com/iai-group/DBpedia-Entity/",
+        dataset={
+            "path": "GreenNode/nano-dbpedia-vn",
+            "revision": "bbc3259bc63bf1e250d7034024092cc3230d5850",
+        },
+        type="Retrieval",
+        category="t2t",
+        eval_splits=["test"],
+        eval_langs=["vie-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-07-29", "2025-07-30"),
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="machine-translated and LM verified",
+        domains=["Written", "Encyclopaedic"],
+        task_subtypes=[],
+        bibtex_citation=r"""
+@misc{pham2025vnmtebvietnamesemassivetext,
+  archiveprefix = {arXiv},
+  author = {Loc Pham and Tung Luu and Thu Vo and Minh Nguyen and Viet Hoang},
+  eprint = {2507.21500},
+  primaryclass = {cs.CL},
+  title = {VN-MTEB: Vietnamese Massive Text Embedding Benchmark},
+  url = {https://arxiv.org/abs/2507.21500},
+  year = {2025},
+}
+""",
+        prompt={
+            "query": "Given a query, retrieve relevant entity descriptions from DBPedia"
+        },
+        adapted_from=["DBPedia-VN"],
+    )

mteb/tasks/retrieval/vie/fevervn_retrieval.py

@@ -9,13 +9,7 @@ class FEVERVN(AbsTaskRetrieval):
             "path": "GreenNode/fever-vn",
             "revision": "a543dd8b98aed3603110c01d26db05ba39b87d49",
         },
-        description="""A translated dataset from FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences
-            extracted from Wikipedia and subsequently verified without knowledge of the sentence they were
-            derived from.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://fever.ai/",
         type="Retrieval",
         category="t2t",
@@ -42,3 +36,42 @@ class FEVERVN(AbsTaskRetrieval):
 """,
         adapted_from=["FEVER"],
     )
+
+
+class NanoFEVERVN(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoFEVER-VN",
+        dataset={
+            "path": "GreenNode/nano-fever-vn",
+            "revision": "457ca6b058ed19b28f2359e2d816d7527af6bef8",
+        },
+        description="NanoFEVERVN is a small version of A translated dataset from FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
+        reference="https://fever.ai/",
+        type="Retrieval",
+        category="t2t",
+        eval_splits=["test"],
+        eval_langs=["vie-Latn"],
+        main_score="ndcg_at_10",
+        date=("2025-07-29", "2025-07-30"),
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="machine-translated and LM verified",
+        domains=["Encyclopaedic", "Written"],
+        task_subtypes=["Claim verification"],
+        bibtex_citation=r"""
+@misc{pham2025vnmtebvietnamesemassivetext,
+  archiveprefix = {arXiv},
+  author = {Loc Pham and Tung Luu and Thu Vo and Minh Nguyen and Viet Hoang},
+  eprint = {2507.21500},
+  primaryclass = {cs.CL},
+  title = {VN-MTEB: Vietnamese Massive Text Embedding Benchmark},
+  url = {https://arxiv.org/abs/2507.21500},
+  year = {2025},
+}
+""",
+        prompt={
+            "query": "Given a claim, retrieve documents that support or refute the claim"
+        },
+        adapted_from=["FEVER-VN"],
+    )