mteb 2.1.4__py3-none-any.whl → 2.5.2__py3-none-any.whl

mteb/tasks/retrieval/nld/sci_fact_nl_retrieval.py

@@ -1,30 +1,26 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class SciFactNL(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="SciFact-NL",
-        dataset={
-            "path": "clips/beir-nl-scifact",
-            "revision": "856d8dfc294b138856bbf3042450e3782321e44e",
-        },
-        description="SciFactNL verifies scientific claims in Dutch using evidence from the research literature containing scientific paper abstracts.",
-        reference="https://huggingface.co/datasets/clips/beir-nl-scifact",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["nld-Latn"],
-        main_score="ndcg_at_10",
-        date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
-        domains=["Academic", "Medical", "Written"],
-        task_subtypes=[],
-        license="cc-by-4.0",
-        annotations_creators="derived",
-        dialect=[],
-        sample_creation="machine-translated and verified",  # manually checked a small subset
-        bibtex_citation=r"""
+_sci_fact_nl_metadata = dict(
+    dataset={
+        "path": "clips/beir-nl-scifact",
+        "revision": "856d8dfc294b138856bbf3042450e3782321e44e",
+    },
+    reference="https://huggingface.co/datasets/clips/beir-nl-scifact",
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["nld-Latn"],
+    main_score="ndcg_at_10",
+    date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
+    domains=["Academic", "Medical", "Written"],
+    task_subtypes=[],
+    license="cc-by-4.0",
+    annotations_creators="derived",
+    dialect=[],
+    sample_creation="machine-translated and verified",  # manually checked a small subset
+    bibtex_citation=r"""
 @misc{banar2024beirnlzeroshotinformationretrieval,
   archiveprefix = {arXiv},
   author = {Nikolay Banar and Ehsan Lotfi and Walter Daelemans},
@@ -35,5 +31,27 @@ class SciFactNL(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+)
+
+
+class SciFactNL(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SciFact-NL",
+        description="SciFactNL verifies scientific claims in Dutch using evidence from the research literature "
+        "containing scientific paper abstracts.",
         adapted_from=["SciFact"],
+        **_sci_fact_nl_metadata,
+    )
+
+
+class SciFactNLv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SciFact-NL.v2",
+        description="SciFactNL verifies scientific claims in Dutch using evidence from the research literature "
+        "containing scientific paper abstracts. This version adds a Dutch prompt to the dataset.",
+        adapted_from=["SciFact-NL"],
+        prompt={
+            "query": "Given a scientific claim, retrieve documents that support or refute the claim"
+        },
+        **_sci_fact_nl_metadata,
     )

mteb/tasks/retrieval/nld/scidocsnl_retrieval.py

@@ -1,33 +1,26 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
-
-class SCIDOCSNL(AbsTaskRetrieval):
-    metadata = TaskMetadata(
-        name="SCIDOCS-NL",
-        dataset={
-            "path": "clips/beir-nl-scidocs",
-            "revision": "4e018aa220029f9d1bd5a31de3650e322e32ea38",
-        },
-        description=(
-            "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation"
-            + " prediction, to document classification and recommendation. SciDocs-NL is a Dutch translation."
-        ),
-        reference="https://huggingface.co/datasets/clips/beir-nl-scidocs",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["nld-Latn"],
-        main_score="ndcg_at_10",
-        date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
-        domains=["Academic", "Written", "Non-fiction"],
-        task_subtypes=[],
-        license="cc-by-sa-4.0",
-        annotations_creators="derived",
-        dialect=[],
-        sample_creation="machine-translated and verified",  # manually checked a small subset
-        bibtex_citation=r"""
+_scidocsnl_metadata = dict(
+    dataset={
+        "path": "clips/beir-nl-scidocs",
+        "revision": "4e018aa220029f9d1bd5a31de3650e322e32ea38",
+    },
+    reference="https://huggingface.co/datasets/clips/beir-nl-scidocs",
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["nld-Latn"],
+    main_score="ndcg_at_10",
+    date=("2020-05-01", "2020-05-01"),  # best guess: based on submission date
+    domains=["Academic", "Written", "Non-fiction"],
+    task_subtypes=[],
+    license="cc-by-sa-4.0",
+    annotations_creators="derived",
+    dialect=[],
+    sample_creation="machine-translated and verified",  # manually checked a small subset
+    bibtex_citation=r"""
 @misc{banar2024beirnlzeroshotinformationretrieval,
   archiveprefix = {arXiv},
   author = {Nikolay Banar and Ehsan Lotfi and Walter Daelemans},
@@ -38,5 +31,29 @@ class SCIDOCSNL(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+)
+
+
+class SCIDOCSNL(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SCIDOCS-NL",
+        description="SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from "
+        "citation prediction, to document classification and recommendation. SciDocs-NL is a Dutch "
+        "translation.",
         adapted_from=["SCIDOCS"],
+        **_scidocsnl_metadata,
+    )
+
+
+class SCIDOCSNLv2(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="SCIDOCS-NL.v2",
+        description="SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from "
+        "citation prediction, to document classification and recommendation. SciDocs-NL is a Dutch "
+        "translation. This version adds a Dutch prompt to the dataset.",
+        adapted_from=["SCIDOCS-NL"],
+        **_scidocsnl_metadata,
+        prompt={
+            "query": "Gegeven de titel van een wetenschappelijk artikel, haal de abstracts op van artikelen die door het gegeven artikel worden geciteerd"
+        },
     )

mteb/tasks/retrieval/nld/vabb_retrieval.py

@@ -38,4 +38,7 @@ class VABBRetrieval(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+        prompt={
+            "query": "Gegeven een titel, haal de wetenschappelijke abstract op die het beste bij de titel past"
+        },
     )

mteb/tasks/retrieval/slk/slovak_sum_retrieval.py

@@ -7,13 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SlovakSumRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="SlovakSumRetrieval",
-        description="""
-            SlovakSum, a Slovak news summarization dataset consisting of over 200 thousand
-            news articles with titles and short abstracts obtained from multiple Slovak newspapers.
-
-            Originally intended as a summarization task, but since no human annotations were provided
-            here reformulated to a retrieval task.
-        """,
+        description="SlovakSum, a Slovak news summarization dataset consisting of over 200 thousand news articles with titles and short abstracts obtained from multiple Slovak newspapers. Originally intended as a summarization task, but since no human annotations were provided here reformulated to a retrieval task.",
         reference="https://huggingface.co/datasets/NaiveNeuron/slovaksum",
         dataset={
             "path": "NaiveNeuron/slovaksum",

mteb/tasks/retrieval/vie/argu_ana_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class ArguAnaVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ArguAna-VN",
-        description="""A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://argumentation.bplaced.net/arguana/data",
         dataset={
             "path": "GreenNode/arguana-vn",

mteb/tasks/retrieval/vie/climate_fevervn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class ClimateFEVERVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="ClimateFEVER-VN",
-        description="""A translated dataset from CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html",
         dataset={
             "path": "GreenNode/climate-fever-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_android_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackAndroidVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackAndroid-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-android-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_gis_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackGisVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackGis-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-gis-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_mathematica_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackMathematicaVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackMathematica-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-mathematica-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_physics_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackPhysicsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackPhysics-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-physics-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_programmers_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackProgrammersRetrievalVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackProgrammers-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-programmers-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_stats_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackStatsVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackStats-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-stats-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_tex_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackTexVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackTex-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-tex-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_unix_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackUnixVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackUnix-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-unix-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_webmasters_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class CQADupstackWebmastersVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="CQADupstackWebmasters-VN",
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         dataset={
             "path": "GreenNode/cqadupstack-webmasters-vn",

mteb/tasks/retrieval/vie/cqa_dupstack_wordpress_vn_retrieval.py

@@ -9,11 +9,7 @@ class CQADupstackWordpressVN(AbsTaskRetrieval):
             "path": "GreenNode/cqadupstack-wordpress-vn",
             "revision": "2230f80e1baf42aa005731ca86577621c566fcd7",
         },
-        description="""A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from CQADupStack: A Benchmark Data Set for Community Question-Answering Research The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="http://nlp.cis.unimelb.edu.au/resources/cqadupstack/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/db_pedia_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class DBPediaVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="DBPedia-VN",
-        description="""A translated dataset from DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from DBpedia-Entity is a standard test collection for entity search over the DBpedia knowledge base The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://github.com/iai-group/DBpedia-Entity/",
         dataset={
             "path": "GreenNode/dbpedia-vn",

mteb/tasks/retrieval/vie/fevervn_retrieval.py

@@ -9,13 +9,7 @@ class FEVERVN(AbsTaskRetrieval):
             "path": "GreenNode/fever-vn",
             "revision": "a543dd8b98aed3603110c01d26db05ba39b87d49",
         },
-        description="""A translated dataset from FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences
-            extracted from Wikipedia and subsequently verified without knowledge of the sentence they were
-            derived from.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from FEVER (Fact Extraction and VERification) consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://fever.ai/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/fi_qa2018_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class FiQA2018VN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="FiQA2018-VN",
-        description="""A translated dataset from Financial Opinion Mining and Question Answering
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Financial Opinion Mining and Question Answering The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://sites.google.com/view/fiqa/",
         dataset={
             "path": "GreenNode/fiqa-vn",

mteb/tasks/retrieval/vie/green_node_table_markdown_retrieval.py

@@ -26,5 +26,20 @@ class GreenNodeTableMarkdownRetrieval(AbsTaskRetrieval):
         annotations_creators="human-annotated",
         dialect=[],
         sample_creation="found",
-        bibtex_citation="",  # TODO: Add bibtex citation when the paper is published
+        bibtex_citation=r"""
+@inproceedings{10.1007/978-981-95-1746-6_17,
+  abstract = {Information retrieval often comes in plain text, lacking semi-structured text such as HTML and markdown, retrieving data that contains rich format such as table became non-trivial. In this paper, we tackle this challenge by introducing a new dataset, GreenNode Table Retrieval VN (GN-TRVN), which is collected from a massive corpus, a wide range of topics, and a longer context compared to ViQuAD2.0. To evaluate the effectiveness of our proposed dataset, we introduce two versions, M3-GN-VN and M3-GN-VN-Mixed, by fine-tuning the M3-Embedding model on this dataset. Experimental results show that our models consistently outperform the baselines, including the base model, across most evaluation criteria on various datasets such as VieQuADRetrieval, ZacLegalTextRetrieval, and GN-TRVN. In general, we release a more comprehensive dataset and two model versions that improve response performance for Vietnamese Markdown Table Retrieval.},
+  address = {Singapore},
+  author = {Pham, Bao Loc
+and Hoang, Quoc Viet
+and Luu, Quy Tung
+and Vo, Trong Thu},
+  booktitle = {Proceedings of the Fifth International Conference on Intelligent Systems and Networks},
+  isbn = {978-981-95-1746-6},
+  pages = {153--163},
+  publisher = {Springer Nature Singapore},
+  title = {GN-TRVN: A Benchmark for Vietnamese Table Markdown Retrieval Task},
+  year = {2026},
+}
+""",
     )

mteb/tasks/retrieval/vie/hotpot_qavn_retrieval.py

@@ -9,12 +9,7 @@ class HotpotQAVN(AbsTaskRetrieval):
             "path": "GreenNode/hotpotqa-vn",
             "revision": "8a5220c7af5084f0d5d2afeb74f9c2b41b759ff0",
         },
-        description="""A translated dataset from HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong
-            supervision for supporting facts to enable more explainable question answering systems.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong supervision for supporting facts to enable more explainable question answering systems. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://hotpotqa.github.io/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/msmarcovn_retrieval.py

@@ -9,11 +9,7 @@ class MSMARCOVN(AbsTaskRetrieval):
             "path": "GreenNode/msmarco-vn",
             "revision": "85d1ad4cc9070b8d019d65f5af1631a2ab91e294",
         },
-        description="""A translated dataset from MS MARCO is a collection of datasets focused on deep learning in search
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from MS MARCO is a collection of datasets focused on deep learning in search The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://microsoft.github.io/msmarco/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/nf_corpus_vn_retrieval.py

@@ -9,11 +9,7 @@ class NFCorpusVN(AbsTaskRetrieval):
             "path": "GreenNode/nfcorpus-vn",
             "revision": "a13d72fbb859be3dc19ab669d1ec9510407d2dcd",
         },
-        description="""A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/nqvn_retrieval.py

@@ -9,11 +9,7 @@ class NQVN(AbsTaskRetrieval):
             "path": "GreenNode/nq-vn",
             "revision": "40a6d7f343b9c9f4855a426d8c431ad5f8aaf56b",
         },
-        description="""A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://ai.google.com/research/NaturalQuestions/",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/quora_vn_retrieval.py

@@ -9,12 +9,7 @@ class QuoraVN(AbsTaskRetrieval):
             "path": "GreenNode/quora-vn",
             "revision": "3363d81e41b67c1032bf3b234882a03d271e2289",
         },
-        description="""A translated dataset from QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a
-            question, find other (duplicate) questions.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a question, find other (duplicate) questions. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/sci_fact_vn_retrieval.py

@@ -9,11 +9,7 @@ class SciFactVN(AbsTaskRetrieval):
             "path": "GreenNode/scifact-vn",
             "revision": "483a7cf890c523c954e7751d328c5bb65061dcff",
         },
-        description="""A translated dataset from SciFact verifies scientific claims using evidence from the research literature containing scientific paper abstracts.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from SciFact verifies scientific claims using evidence from the research literature containing scientific paper abstracts. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://github.com/allenai/scifact",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/scidocsvn_retrieval.py

@@ -9,12 +9,7 @@ class SCIDOCSVN(AbsTaskRetrieval):
             "path": "GreenNode/scidocs-vn",
             "revision": "724cddfa9d328a193f303a0a9b7789468ac79f26",
         },
-        description="""A translated dataset from SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation
-            prediction, to document classification and recommendation.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation prediction, to document classification and recommendation. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://allenai.org/data/scidocs",
         type="Retrieval",
         category="t2t",

mteb/tasks/retrieval/vie/touche2020_vn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class Touche2020VN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="Touche2020-VN",
-        description="""A translated dataset from Touché Task 1: Argument Retrieval for Controversial Questions
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Touché Task 1: Argument Retrieval for Controversial Questions The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://webis.de/events/touche-20/shared-task-1.html",
         dataset={
             "path": "GreenNode/webis-touche2020-vn",

mteb/tasks/retrieval/vie/treccovidvn_retrieval.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class TRECCOVIDVN(AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="TRECCOVID-VN",
-        description="""A translated dataset from TRECCOVID is an ad-hoc search challenge based on the COVID-19 dataset containing scientific articles related to the COVID-19 pandemic.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from TRECCOVID is an ad-hoc search challenge based on the COVID-19 dataset containing scientific articles related to the COVID-19 pandemic. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://ir.nist.gov/covidSubmit/index.html",
         dataset={
             "path": "GreenNode/trec-covid-vn",

mteb/tasks/sts/nld/sick_nl_sts.py

@@ -35,6 +35,7 @@ class SICKNLSTS(AbsTaskSTS):
   year = {2021},
 }
 """,
+        prompt={"query": "Haal semantisch vergelijkbare tekst op"},
     )
 
     min_score = 0

mteb/tasks/sts/vie/biosses_stsvn.py

@@ -9,11 +9,7 @@ class BiossesSTSVN(AbsTaskSTS):
             "path": "GreenNode/biosses-sts-vn",
             "revision": "1dae4a6df91c0852680cd4ab48c8c1d8a9ed49b2",
         },
-        description="""A translated dataset from Biomedical Semantic Similarity Estimation.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Biomedical Semantic Similarity Estimation. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://tabilab.cmpe.boun.edu.tr/BIOSSES/DataSet.html",
         type="STS",
         category="t2c",