mteb 2.0.5__py3-none-any.whl → 2.1.1__py3-none-any.whl

mteb/tasks/retrieval/eng/memotion_i2t_retrieval.py

@@ -20,7 +20,6 @@ def _load_data(path: str, splits: str, revision: str | None = None):
             "id": f"corpus-{split_name}-{idx}",
             "text": x["text_corrected"],
             "modality": "text",
-            "image": None,
         }
 
     split_datasets = {}
@@ -56,9 +55,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
         queries[split] = split_dataset.map(
             lambda x, idx: {
                 "id": f"query-{split}-{idx}",
-                "text": None,
                 "modality": "image",
-                # "image": None,
             },
             with_indices=True,
             remove_columns=[

mteb/tasks/retrieval/eng/memotion_t2i_retrieval.py

@@ -18,7 +18,6 @@ def _load_data(path: str, splits: str, revision: str | None = None):
     def map_function(split_name):
         return lambda x, idx: {
             "id": f"corpus-{split_name}-{idx}",
-            "text": None,
             "modality": "image",
         }
 
@@ -56,7 +55,6 @@ def _load_data(path: str, splits: str, revision: str | None = None):
                 "id": f"query-{split}-{idx}",
                 "text": x["text_corrected"],
                 "modality": "text",
-                "image": None,
             },
             with_indices=True,
             remove_columns=[

mteb/tasks/retrieval/eng/oven_it2t_retrieval.py

@@ -12,7 +12,7 @@ class OVENIT2TRetrieval(AbsTaskRetrieval):
             "revision": "2192074af29422bc1dc41cf07936f198b8c69bd0",
         },
         type="Any2AnyRetrieval",
-        category="it2i",
+        category="it2t",
         eval_splits=["test"],
         eval_langs=["eng-Latn"],
         main_score="ndcg_at_10",

mteb/tasks/retrieval/eng/quora_retrieval.py

@@ -1,6 +1,32 @@
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
 
+_quora_metadata = dict(
+    reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs",
+    type="Retrieval",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["test"],
+    eval_langs=["eng-Latn"],
+    main_score="ndcg_at_10",
+    date=None,
+    domains=["Written", "Web", "Blog"],
+    task_subtypes=["Question answering"],
+    license="not specified",
+    annotations_creators="human-annotated",
+    dialect=[],
+    sample_creation="found",
+    bibtex_citation=r"""
+@misc{quora-question-pairs,
+  author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung},
+  publisher = {Kaggle},
+  title = {Quora Question Pairs},
+  url = {https://kaggle.com/competitions/quora-question-pairs},
+  year = {2017},
+}
+""",
+)
+
 
 class QuoraRetrieval(AbsTaskRetrieval):
     ignore_identical_ids = True
@@ -15,32 +41,10 @@ class QuoraRetrieval(AbsTaskRetrieval):
             "QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a"
             + " question, find other (duplicate) questions."
         ),
-        reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=None,
-        domains=["Written", "Web", "Blog"],
-        task_subtypes=["Question answering"],
-        license="not specified",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
-@misc{quora-question-pairs,
-  author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung},
-  publisher = {Kaggle},
-  title = {Quora Question Pairs},
-  url = {https://kaggle.com/competitions/quora-question-pairs},
-  year = {2017},
-}
-""",
         prompt={
             "query": "Given a question, retrieve questions that are semantically equivalent to the given question"
         },
+        **_quora_metadata,
     )
 
 
@@ -57,28 +61,29 @@ class QuoraRetrievalHardNegatives(AbsTaskRetrieval):
             "QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a"
             + " question, find other (duplicate) questions. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
         ),
-        reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs",
-        type="Retrieval",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["eng-Latn"],
-        main_score="ndcg_at_10",
-        date=None,
-        domains=None,
-        task_subtypes=None,
-        license=None,
-        annotations_creators=None,
-        dialect=None,
-        sample_creation=None,
-        bibtex_citation=r"""
-@misc{quora-question-pairs,
-  author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung},
-  publisher = {Kaggle},
-  title = {Quora Question Pairs},
-  url = {https://kaggle.com/competitions/quora-question-pairs},
-  year = {2017},
-}
-""",
         adapted_from=["QuoraRetrieval"],
+        superseded_by="QuoraRetrievalHardNegatives.v2",
+        **_quora_metadata,
+    )
+
+
+class QuoraRetrievalHardNegativesV2(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="QuoraRetrievalHardNegatives.v2",
+        dataset={
+            "path": "mteb/QuoraRetrieval_test_top_250_only_w_correct-v2",
+            "revision": "907a33577e9506221d3ba20f5a851b7c3f8dc6d3",
+        },
+        description=(
+            "QuoraRetrieval is based on questions that are marked as duplicates on the Quora platform. Given a "
+            "question, find other (duplicate) questions. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval. You can get more information on the effect of different prompt in the [PR](https://github.com/embeddings-benchmark/mteb/pull/3469#issuecomment-3436467106)"
+        ),
+        adapted_from=["QuoraRetrieval"],
+        prompt={
+            "query": "Given a question, retrieve questions that are semantically equivalent to the given question"
+        },
+        **_quora_metadata,
     )

mteb/tasks/retrieval/eng/sci_mmir_i2t_retrieval.py

@@ -20,9 +20,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
         corpus[split] = split_dataset.map(
             lambda x, idx: {
                 "id": f"corpus-{split}-{idx}",
-                # "text": None,
                 "modality": "text",
-                "image": None,
             },
             with_indices=True,
             remove_columns=[
@@ -37,9 +35,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
         queries[split] = split_dataset.map(
             lambda x, idx: {
                 "id": f"query-{split}-{idx}",
-                "text": None,
                 "modality": "image",
-                # "image": None,
             },
             with_indices=True,
             remove_columns=[

mteb/tasks/retrieval/eng/sci_mmir_t2i_retrieval.py

@@ -20,9 +20,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
         corpus[split] = split_dataset.map(
             lambda x, idx: {
                 "id": f"corpus-{split}-{idx}",
-                "text": None,
                 "modality": "image",
-                # "image": None,
             },
             with_indices=True,
             remove_columns=[
@@ -37,9 +35,7 @@ def _load_data(path: str, splits: str, revision: str | None = None):
         queries[split] = split_dataset.map(
             lambda x, idx: {
                 "id": f"query-{split}-{idx}",
-                # "text": None,
                 "modality": "text",
-                "image": None,
             },
             with_indices=True,
             remove_columns=[

mteb/tasks/retrieval/eng/vidore_bench_retrieval.py

@@ -24,7 +24,6 @@ def _load_data(
             lambda x: {
                 "id": f"query-{split}-{x['query-id']}",
                 "text": x["query"],
-                "image": None,
                 "modality": "text",
             },
             remove_columns=["query-id", "query"],
@@ -40,7 +39,6 @@ def _load_data(
         corpus_ds = corpus_ds.map(
             lambda x: {
                 "id": f"corpus-{split}-{x['corpus-id']}",
-                "text": None,
                 "modality": "image",
             },
             remove_columns=["corpus-id"],

mteb/tasks/retrieval/jpn/ja_gov_faqs_retrieval.py

@@ -7,7 +7,7 @@ class JaGovFaqsRetrieval(AbsTaskRetrieval):
 
     metadata = TaskMetadata(
         name="JaGovFaqsRetrieval",
-        description="JaGovFaqs is a dataset consisting of FAQs manully extracted from the website of Japanese bureaus. The dataset consists of 22k FAQs, where the queries (questions) and corpus (answers) have been shuffled, and the goal is to match the answer with the question.",
+        description="JaGovFaqs is a dataset consisting of FAQs manually extracted from the website of Japanese bureaus. The dataset consists of 22k FAQs, where the queries (questions) and corpus (answers) have been shuffled, and the goal is to match the answer with the question.",
         reference="https://github.com/sbintuitions/JMTEB",
         dataset={
             "path": "mteb/JaGovFaqsRetrieval",

mteb/tasks/retrieval/multilingual/belebele_retrieval.py

@@ -132,7 +132,7 @@ _LANGUAGES = [
 
 
 def get_lang_pairs() -> dict[str, list[str]]:
-    # add pairs with same langauge as the source and target
+    # add pairs with same language as the source and target
     # add pairs with english as source or target
     lang_pairs = {}
     for x in _LANGUAGES:

mteb/tasks/retrieval/multilingual/jina_vdr_bench_retrieval.py

@@ -72,7 +72,6 @@ def _load_single_language(
         lambda x: {
             "id": f"query-{split}-{x['query-id']}",
             "text": x["query"],
-            "image": None,
             "modality": "text",
         },
         remove_columns=["query-id", "query"],
@@ -87,7 +86,6 @@ def _load_single_language(
     corpus_ds = corpus_ds.map(
         lambda x: {
             "id": f"corpus-{split}-{x['corpus-id']}",
-            "text": None,
             "modality": "image",
         },
         remove_columns=["corpus-id"],

mteb/tasks/retrieval/multilingual/miracl_retrieval.py

@@ -92,7 +92,7 @@ class MIRACLRetrievalHardNegativesV2(AbsTaskRetrieval):
             "MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval "
             "dataset that focuses on search across 18 different languages. The hard negative version has been "
             "created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
-            "V2 uses a more appropriate prompt rather than the default prompt for retrieval."
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval. You can get more information on the effect of different prompt in the [PR](https://github.com/embeddings-benchmark/mteb/pull/3469#issuecomment-3436467106)"
         ),
         dataset={
             "path": "mteb/MIRACLRetrievalHardNegatives",

mteb/tasks/retrieval/multilingual/miracl_vision_retrieval.py

@@ -30,7 +30,7 @@ _LANGUAGES = {
 def _load_miracl_data(
     path: str,
     langs: list,
-    splits: str,
+    splits: list[str],
     revision: str | None = None,
 ):
     corpus = {lang: dict.fromkeys(splits) for lang in langs}
@@ -65,9 +65,7 @@ def _load_miracl_data(
         images_data = images_data.map(
             lambda x: {
                 "id": imgid2docid[str(x["file_name"])],
-                # "modality": "text",
                 "modality": "image",
-                "text": None,
             },
             remove_columns=["file_name"],
         )
@@ -86,7 +84,6 @@ def _load_miracl_data(
                 "id": str(x["_id"]),
                 "text": x["text"],
                 "modality": "text",
-                "image": None,
             },
             remove_columns=["_id"],
         )
@@ -108,10 +105,6 @@ def _load_miracl_data(
                 relevant_docs[lang][split][query_id] = {}
             relevant_docs[lang][split][query_id][doc_id] = score
 
-    corpus = datasets.DatasetDict(corpus)
-    queries = datasets.DatasetDict(queries)
-    relevant_docs = datasets.DatasetDict(relevant_docs)
-
     return corpus, queries, relevant_docs
 
 
@@ -156,7 +149,7 @@ class MIRACLVisionRetrieval(AbsTaskRetrieval):
 
         self.corpus, self.queries, self.relevant_docs = _load_miracl_data(
             path=self.metadata.dataset["path"],
-            splits=self.metadata.eval_splits[0],
+            splits=self.metadata.eval_splits,
             langs=self.hf_subsets,
             revision=self.metadata.dataset["revision"],
         )

mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py

@@ -37,7 +37,6 @@ def _load_data(
             lambda x: {
                 "id": f"query-{split}-{x['query-id']}",
                 "text": x["query"],
-                "image": None,
                 "modality": "text",
             },
             remove_columns=["query-id", "query"],
@@ -52,7 +51,6 @@ def _load_data(
         corpus_ds = corpus_ds.map(
             lambda x: {
                 "id": f"corpus-{split}-{x['corpus-id']}",
-                "text": None,
                 "modality": "image",
             },
             remove_columns=["corpus-id"],

mteb/tasks/retrieval/multilingual/wit_t2i_retrieval.py

@@ -34,7 +34,6 @@ def _load_wit_data(path: str, langs: list, splits: str, revision: str | None = N
         lang_corpus = lang_data.map(
             lambda x: {
                 "id": "corpus-" + x["image_id"],
-                "text": None,
                 "modality": "image",
                 "image": x["image"],
             },
@@ -60,7 +59,6 @@ def _load_wit_data(path: str, langs: list, splits: str, revision: str | None = N
                         "id": query_id,
                         "text": caption,
                         "modality": "text",
-                        "image": None,
                     }
                 )
                 if query_id not in relevant_docs[lang][split]:

mteb/tasks/retrieval/multilingual/x_flickr30k_co_t2i_retrieval.py

@@ -1,4 +1,4 @@
-from datasets import DatasetDict, load_dataset
+from datasets import DatasetDict, Image, load_dataset
 
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
@@ -16,7 +16,7 @@ _LANGUAGES = {
 
 
 def _load_xflickrco_data(
-    path: str, langs: list, splits: str, revision: str | None = None
+    path: str, langs: list, splits: list[str], revision: str | None = None
 ):
     corpus = {lang: dict.fromkeys(splits) for lang in langs}
     queries = {lang: dict.fromkeys(splits) for lang in langs}
@@ -32,22 +32,23 @@ def _load_xflickrco_data(
         lang_corpus = lang_data.map(
             lambda x: {
                 "id": "corpus-" + x["id"],
-                "text": None,
                 "modality": "image",
-                "image": x["image"]["bytes"],
+                "image": x["image"],
             },
             remove_columns=["sentences"],
         )
+        lang_corpus = lang_corpus.cast_column("image", Image())
 
         lang_queries = lang_data.map(
             lambda x: {
                 "id": "query-" + x["id"],
                 "text": x["sentences"],
                 "modality": "text",
-                "image": None,
             },
             remove_columns=["sentences"],
         )
+        # None values
+        lang_queries = lang_queries.remove_columns(["image"])
 
         relevant_docs[lang][split] = {}
         for row in lang_data:

mteb/tasks/retrieval/multilingual/xm3600_t2i_retrieval.py

@@ -1,4 +1,4 @@
-from datasets import Dataset, DatasetDict, load_dataset
+from datasets import Dataset, DatasetDict, Image, load_dataset
 
 from mteb.abstasks.retrieval import AbsTaskRetrieval
 from mteb.abstasks.task_metadata import TaskMetadata
@@ -61,9 +61,8 @@ def _load_xm3600_data(
         lang_corpus = lang_data.map(
             lambda x: {
                 "id": "corpus-" + x["image_id"],
-                "text": None,
                 "modality": "image",
-                "image": x["image"]["bytes"],
+                "image": x["image"],
             },
             remove_columns=[
                 "captions",
@@ -73,6 +72,7 @@ def _load_xm3600_data(
                 "image_id",
             ],
         )
+        lang_corpus = lang_corpus.cast_column("image", Image())
 
         corpus[lang][split] = lang_corpus
 
@@ -90,7 +90,6 @@ def _load_xm3600_data(
                         "id": query_id,
                         "text": caption,
                         "modality": "text",
-                        "image": None,
                     }
                 )
                 if query_id not in relevant_docs[lang][split]:

mteb/tasks/retrieval/nld/__init__.py

@@ -1,4 +1,5 @@
 from .argu_ana_nl_retrieval import ArguAnaNL
+from .bbsard_nl_retrieval import BBSARDNLRetrieval
 from .climate_fevernl_retrieval import ClimateFEVERNL
 from .cqa_dupstack_android_nl_retrieval import CQADupstackAndroidNLRetrieval
 from .cqa_dupstack_english_nl_retrieval import CQADupstackEnglishNLRetrieval
@@ -13,17 +14,21 @@ from .cqa_dupstack_unix_nl_retrieval import CQADupstackUnixNLRetrieval
 from .cqa_dupstack_webmasters_nl_retrieval import CQADupstackWebmastersNLRetrieval
 from .cqa_dupstack_wordpress_nl_retrieval import CQADupstackWordpressNLRetrieval
 from .db_pedia_nl_retrieval import DBPediaNL
+from .dutch_news_articles_retrieval import DutchNewsArticlesRetrieval
 from .fevernl_retrieval import FEVERNL
 from .fi_qa2018_nl_retrieval import FiQA2018NL
 from .hotpot_qanl_retrieval import HotpotQANL
+from .legal_qa_nl_retrieval import LegalQANLRetrieval
 from .mmarconl_retrieval import MMMARCONL
 from .nf_corpus_nl_retrieval import NFCorpusNL
 from .nqnl_retrieval import NQNL
+from .open_tender_retrieval import OpenTenderRetrieval
 from .quora_nl_retrieval import QuoraNLRetrieval
 from .sci_fact_nl_retrieval import SciFactNL
 from .scidocsnl_retrieval import SCIDOCSNL
 from .touche2020_nl_retrieval import Touche2020NL
 from .treccovidnl_retrieval import TRECCOVIDNL
+from .vabb_retrieval import VABBRetrieval
 
 __all__ = [
     "FEVERNL",
@@ -32,6 +37,7 @@ __all__ = [
     "SCIDOCSNL",
     "TRECCOVIDNL",
     "ArguAnaNL",
+    "BBSARDNLRetrieval",
     "CQADupstackAndroidNLRetrieval",
     "CQADupstackEnglishNLRetrieval",
     "CQADupstackGamingNLRetrieval",
@@ -46,10 +52,14 @@ __all__ = [
     "CQADupstackWordpressNLRetrieval",
     "ClimateFEVERNL",
     "DBPediaNL",
+    "DutchNewsArticlesRetrieval",
     "FiQA2018NL",
     "HotpotQANL",
+    "LegalQANLRetrieval",
     "NFCorpusNL",
+    "OpenTenderRetrieval",
     "QuoraNLRetrieval",
     "SciFactNL",
     "Touche2020NL",
+    "VABBRetrieval",
 ]

mteb/tasks/retrieval/nld/bbsard_nl_retrieval.py

@@ -0,0 +1,41 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class BBSARDNLRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="bBSARDNLRetrieval",
+        description="Building on the Belgian Statutory Article Retrieval Dataset (BSARD) in French, we introduce the "
+        "bilingual version of this dataset, bBSARD. The dataset contains parallel Belgian statutory "
+        "articles in both French and Dutch, along with legal questions from BSARD and their Dutch "
+        "translation.",
+        reference="https://aclanthology.org/2025.regnlp-1.3.pdf",
+        dataset={
+            "path": "clips/mteb-nl-bbsard",
+            "revision": "52027c212ba9765a3e9737c9cbf9a06ae83cbb93",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2021-05-01", "2021-08-26"),
+        domains=["Legal", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@article{lotfi2025bilingual,
+  author = {Lotfi, Ehsan and Banar, Nikolay and Yuzbashyan, Nerses and Daelemans, Walter},
+  journal = {COLING 2025},
+  pages = {10},
+  title = {Bilingual BSARD: Extending Statutory Article Retrieval to Dutch},
+  year = {2025},
+}
+""",
+    )

mteb/tasks/retrieval/nld/dutch_news_articles_retrieval.py

@@ -0,0 +1,30 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class DutchNewsArticlesRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="DutchNewsArticlesRetrieval",
+        description="This dataset contains all the articles published by the NOS as of the 1st of January 2010. The "
+        "data is obtained by scraping the NOS website. The NOS is one of the biggest (online) news "
+        "organizations in the Netherlands.",
+        reference="https://www.kaggle.com/datasets/maxscheijen/dutch-news-articles",
+        dataset={
+            "path": "clips/mteb-nl-news-articles-ret",
+            "revision": "c8042a86f3eb0d1fcec79a4a44ebf1eafe635462",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Written", "News"],
+        task_subtypes=["Article retrieval"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="",
+    )

mteb/tasks/retrieval/nld/legal_qa_nl_retrieval.py

@@ -0,0 +1,39 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class LegalQANLRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="LegalQANLRetrieval",
+        description="To this end, we create and publish a Dutch legal QA dataset, consisting of question-answer pairs "
+        "with attributions to Dutch law articles.",
+        reference="https://aclanthology.org/2024.nllp-1.12/",
+        dataset={
+            "path": "clips/mteb-nl-legalqa-pr",
+            "revision": "8f593522dfbe7ec07055ca9d38a700e7643d3882",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2021-05-01", "2021-08-26"),
+        domains=["Legal", "Written"],
+        task_subtypes=[],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@inproceedings{redelaar2024attributed,
+  author = {Redelaar, Felicia and Van Drie, Romy and Verberne, Suzan and De Boer, Maaike},
+  booktitle = {Proceedings of the natural legal language processing workshop 2024},
+  pages = {154--165},
+  title = {Attributed Question Answering for Preconditions in the Dutch Law},
+  year = {2024},
+}
+""",
+    )

mteb/tasks/retrieval/nld/open_tender_retrieval.py

@@ -0,0 +1,38 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class OpenTenderRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="OpenTenderRetrieval",
+        description="This dataset contains Belgian and Dutch tender calls from OpenTender in Dutch",
+        reference="https://arxiv.org/abs/2509.12340",
+        dataset={
+            "path": "clips/mteb-nl-opentender-ret",
+            "revision": "83eec1aa9c58f1dc8acfac015f653a9c25bda3f4",
+        },
+        type="Retrieval",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["nld-Latn"],
+        main_score="ndcg_at_10",
+        date=("2009-11-01", "2010-01-01"),
+        domains=["Government", "Written"],
+        task_subtypes=["Article retrieval"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation=r"""
+@misc{banar2025mtebnle5nlembeddingbenchmark,
+  archiveprefix = {arXiv},
+  author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans},
+  eprint = {2509.12340},
+  primaryclass = {cs.CL},
+  title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch},
+  url = {https://arxiv.org/abs/2509.12340},
+  year = {2025},
+}
+""",
+    )