mteb 2.1.5__tar.gz → 2.1.7__tar.gz

{mteb-2.1.5 → mteb-2.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.1.5
+Version: 2.1.7
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>
@@ -16,7 +16,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
-Requires-Python: <3.14,>=3.10
+Requires-Python: <3.15,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: datasets>=2.19.0

mteb-2.1.7/mteb/descriptive_stats/Retrieval/ArguAna-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 10080,
+        "number_of_characters": 11742019,
+        "documents_text_statistics": {
+            "total_text_length": 9897099,
+            "min_text_length": 1,
+            "average_text_length": 1141.0074936592114,
+            "max_text_length": 7337,
+            "unique_texts": 8624
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 1844920,
+            "min_text_length": 252,
+            "average_text_length": 1312.176386913229,
+            "max_text_length": 6050,
+            "unique_texts": 1298
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1406,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1406
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb-2.1.7/mteb/descriptive_stats/Retrieval/NFCorpus-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 3956,
+        "number_of_characters": 6345348,
+        "documents_text_statistics": {
+            "total_text_length": 6337710,
+            "min_text_length": 144,
+            "average_text_length": 1744.483897605285,
+            "max_text_length": 8480,
+            "unique_texts": 3593
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 7638,
+            "min_text_length": 3,
+            "average_text_length": 23.647058823529413,
+            "max_text_length": 89,
+            "unique_texts": 323
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 12334,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 38.18575851393189,
+            "max_relevant_docs_per_query": 475,
+            "unique_relevant_docs": 3128
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb-2.1.7/mteb/descriptive_stats/Retrieval/SCIDOCS-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 26657,
+        "number_of_characters": 34261482,
+        "documents_text_statistics": {
+            "total_text_length": 34181168,
+            "min_text_length": 10,
+            "average_text_length": 1332.2355692403632,
+            "max_text_length": 9275,
+            "unique_texts": 25656
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 80314,
+            "min_text_length": 17,
+            "average_text_length": 80.314,
+            "max_text_length": 227,
+            "unique_texts": 1000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 4928,
+            "min_relevant_docs_per_query": 27,
+            "average_relevant_docs_per_query": 4.928,
+            "max_relevant_docs_per_query": 30,
+            "unique_relevant_docs": 25657
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb-2.1.7/mteb/descriptive_stats/Retrieval/SciFact-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 5483,
+        "number_of_characters": 8526662,
+        "documents_text_statistics": {
+            "total_text_length": 8496576,
+            "min_text_length": 228,
+            "average_text_length": 1639.3162261238665,
+            "max_text_length": 9187,
+            "unique_texts": 5183
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 30086,
+            "min_text_length": 24,
+            "average_text_length": 100.28666666666666,
+            "max_text_length": 228,
+            "unique_texts": 300
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 339,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.13,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 283
+        },
+        "top_ranked_statistics": null
+    }
+}

{mteb-2.1.5 → mteb-2.1.7}/mteb/models/model_implementations/colqwen_models.py

@@ -220,3 +220,60 @@ colnomic_7b = ModelMeta(
     training_datasets=COLNOMIC_TRAINING_DATA,
     citation=COLNOMIC_CITATION,
 )
+
+
+EVOQWEN_TRAINING_DATA = {
+    "colpali_train_set",
+    "VisRAG-Ret-Train-Synthetic-data",
+    "VisRAG-Ret-Train-In-domain-data",
+}
+
+evoqwen25_vl_retriever_3b_v1 = ModelMeta(
+    loader=ColQwen2_5Wrapper,
+    loader_kwargs=dict(
+        torch_dtype=torch.float16, attn_implementation="flash_attention_2"
+    ),
+    name="ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-3B-v1",
+    languages=["eng-Latn"],
+    revision="aeacaa2775f2758d82721eb1cf2f5daf1a392da9",
+    release_date="2025-11-04",
+    modalities=["image", "text"],
+    n_parameters=3_000_000_000,
+    memory_usage_mb=7200,
+    max_tokens=128000,
+    embed_dim=128,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code="https://github.com/illuin-tech/colpali",
+    public_training_data="https://huggingface.co/datasets/vidore/colpali_train_set",
+    framework=["ColPali"],
+    reference="https://huggingface.co/ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-3B-v1",
+    similarity_fn_name="MaxSim",
+    use_instructions=True,
+    training_datasets=EVOQWEN_TRAINING_DATA,
+)
+
+evoqwen25_vl_retriever_7b_v1 = ModelMeta(
+    loader=ColQwen2_5Wrapper,
+    loader_kwargs=dict(
+        torch_dtype=torch.float16, attn_implementation="flash_attention_2"
+    ),
+    name="ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-7B-v1",
+    languages=["eng-Latn"],
+    revision="8952ac6ee0e7de2e9211b165921518caf9202110",
+    release_date="2025-11-04",
+    modalities=["image", "text"],
+    n_parameters=7_000_000_000,
+    memory_usage_mb=14400,
+    max_tokens=128000,
+    embed_dim=128,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code="https://github.com/illuin-tech/colpali",
+    public_training_data="https://huggingface.co/datasets/vidore/colpali_train_set",
+    framework=["ColPali"],
+    reference="https://huggingface.co/ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-7B-v1",
+    similarity_fn_name="MaxSim",
+    use_instructions=True,
+    training_datasets=EVOQWEN_TRAINING_DATA,
+)

{mteb-2.1.5 → mteb-2.1.7}/mteb/models/model_implementations/kalm_models.py

@@ -247,6 +247,40 @@ kalm_v2_training_data = {
 }
 
 
+KaLM_Embedding_gemma_3_12b_training_data = [
+    "BornholmBitextMining",
+    "BibleNLPBitextMining",  # train[256:] to avoid the test set in MTEB
+    "FinParaSTS",
+    "SemRel24STS",
+    "STSB",
+    "GermanSTSBenchmark",
+    "TERRa",
+    "StatcanDialogueDatasetRetrieval",
+    "SwissJudgementClassification",
+    "GreekLegalCodeClassification",
+    "CataloniaTweetClassification",
+    "NordicLangClassification",
+    "CyrillicTurkicLangClassification",
+    "PoemSentimentClassification",
+    "CzechProductReviewSentimentClassification",
+    "DBpediaClassification",
+    "IndicLangClassification",
+    "NusaParagraphEmotionClassification",
+    "OdiaNewsClassification",
+    "ScalaClassification",
+    "ItaCaseholdClassification",
+    "CSFDSKMovieReviewSentimentClassification",
+    "ToxicConversationsClassification",
+    "PAC",
+    "SpartQA",
+    "WinoGrande",
+    "Quail",
+    "HellaSwag",
+    "AlphaNLI",
+]
+KaLM_Embedding_gemma_3_12b_training_data += kalm_training_data
+
+
 KaLM_task_prompts = {
     "AmazonCounterfactualClassification": "Given an Amazon review, judge whether it is counterfactual.",
     "AmazonPolarityClassification": "Classifying Amazon reviews into positive or negative sentiment",
@@ -450,13 +484,7 @@ KaLM_v2_task_prompts = {
     "SummEval-document": "Retrieve semantically similar summaries",
 }
 
-KaLM_X_task_prompts = {
-    "Classification": "classify the query into different classes.",
-    "MultilabelClassification": "Instruct: classify the query into different classes.",
-    "Clustering": "classify the query into different classes.",
-    "Reranking-query": "Given a query, retrieve documents that answer the query.",
-    "Retrieval-query": "Given a query, retrieve documents that answer the query.",
-    "InstructionRetrieval-query": "Given a query, retrieve documents that answer the query.",
+KaLM_Embedding_gemma_3_12b_task_prompts = {
     "AmazonCounterfactualClassification": "Classify a given Amazon customer review text as either counterfactual or not-counterfactual",
     "AmazonPolarityClassification": "Classify Amazon reviews into positive or negative sentiment",
     "AmazonReviewsClassification": "Classify the given Amazon review into its appropriate rating category",
@@ -478,26 +506,38 @@ KaLM_X_task_prompts = {
     "ArxivClusteringP2P": "Identify the main and secondary category of Arxiv papers based on the titles and abstracts",
     "ArxivClusteringS2S": "Identify the main and secondary category of Arxiv papers based on the titles",
     "BiorxivClusteringP2P": "Identify the main category of Biorxiv papers based on the titles and abstracts",
+    "BiorxivClusteringP2P.v2": "Identify the main category of Biorxiv papers based on the titles and abstracts",
     "BiorxivClusteringS2S": "Identify the main category of Biorxiv papers based on the titles",
+    "BiorxivClusteringS2S.v2": "Identify the main category of Biorxiv papers based on the titles",
     "MedrxivClusteringP2P": "Identify the main category of Medrxiv papers based on the titles and abstracts",
+    "MedrxivClusteringP2P.v2": "Identify the main category of Medrxiv papers based on the titles and abstracts",
     "MedrxivClusteringS2S": "Identify the main category of Medrxiv papers based on the titles",
+    "MedrxivClusteringS2S.v2": "Identify the main category of Medrxiv papers based on the titles",
     "RedditClustering": "Identify the topic or theme of Reddit posts based on the titles",
     "RedditClusteringP2P": "Identify the topic or theme of Reddit posts based on the titles and posts",
     "StackExchangeClustering": "Identify the topic or theme of StackExchange posts based on the titles",
+    "StackExchangeClustering.v2": "Identify the topic or theme of StackExchange posts based on the titles",
     "StackExchangeClusteringP2P": "Identify the topic or theme of StackExchange posts based on the given paragraphs",
+    "StackExchangeClusteringP2P.v2": "Identify the topic or theme of StackExchange posts based on the given paragraphs",
     "TwentyNewsgroupsClustering": "Identify the topic or theme of the given news articles",
     "CLSClusteringS2S": "Identify the main category of scholar papers based on the titles",
     "CLSClusteringP2P": "Identify the main category of scholar papers based on the titles and abstracts",
+    "CLSClusteringP2P.v2": "Identify the main category of scholar papers based on the titles and abstracts",
     "ThuNewsClusteringS2S": "Identify the topic or theme of the given news articles based on the titles",
     "ThuNewsClusteringP2P": "Identify the topic or theme of the given news articles based on the titles and contents",
     "AskUbuntuDupQuestions-query": "Retrieve duplicate questions from AskUbuntu forum",
     "MindSmallReranking-query": "Retrieve relevant news articles based on user browsing history",
     "SciDocsRR-query": "Given a title of a scientific paper, retrieve the titles of other relevant papers",
     "StackOverflowDupQuestions-query": "Retrieve duplicate questions from StackOverflow forum",
+    "SprintDuplicateQuestions": "Retrieve semantically duplicate questions",
+    "TwitterSemEval2015": "Retrieve tweets that are semantically similar to the given tweet",
+    "TwitterURLCorpus": "Retrieve tweets that are semantically similar to the given tweet",
     "T2Reranking-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "MMarcoReranking-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "CMedQAv1-reranking-query": "Given a Chinese community medical question, retrieve replies that best answer the question",
     "CMedQAv2-reranking-query": "Given a Chinese community medical question, retrieve replies that best answer the question",
+    "Ocnli": "Retrieve semantically similar text.",
+    "Cmnli": "Retrieve semantically similar text.",
     "ArguAna-query": "Given a claim, find documents that refute the claim",
     "ArguAna-document": "Given a claim, find documents that refute the claim",
     "ClimateFEVER-query": "Given a claim about climate change, retrieve documents that support or refute the claim",
@@ -512,22 +552,26 @@ KaLM_X_task_prompts = {
     "NFCorpus-query": "Given a question, retrieve relevant documents that best answer the question",
     "NQ-query": "Given a question, retrieve Wikipedia passages that answer the question",
     "QuoraRetrieval-query": "Given a question, retrieve questions that are semantically equivalent to the given question",
-    "SCIDOCS-query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper",
+    "SCIDOCS-query": "Given a title of a scientific paper, retrieve the titles of other relevant papers",
     "SciFact-query": "Given a scientific claim, retrieve documents that support or refute the claim",
     "Touche2020-query": "Given a question, retrieve detailed and persuasive arguments that answer the question",
     "Touche2020Retrieval.v3-query": "Given a question, retrieve detailed and persuasive arguments that answer the question",
-    "TRECCOVID-query": "Given a query on COVID-19, retrieve documents that answer the query",
+    "TRECCOVID-query": "Given a medical query, retrieve documents that answer the query",
     "T2Retrieval-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "MMarcoRetrieval-query": "Given a web search query, retrieve relevant passages that answer the query",
+    "VoyageMMarcoReranking-query": "Given a Japanese search query, retrieve web passages that answer the question",
     "DuRetrieval-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "CovidRetrieval-query": "Given a question on COVID-19, retrieve news articles that answer the question",
     "CmedqaRetrieval-query": "Given a Chinese community medical question, retrieve replies that best answer the question",
     "EcomRetrieval-query": "Given a user query from an e-commerce website, retrieve description sentences of relevant products",
     "MedicalRetrieval-query": "Given a medical question, retrieve user replies that best answer the question",
     "VideoRetrieval-query": "Given a video search query, retrieve the titles of relevant videos",
+    "STSBenchmarkMultilingualSTS": "Retrieve semantically similar text",
+    "SICKFr": "Retrieve semantically similar text",
+    "SummEvalFr": "Given a news summary, retrieve other semantically similar summaries",
     "MasakhaNEWSClassification": "Classify the News in the given texts into one of the seven category: politics,sports,health,business,entertainment,technology,religion ",
-    "AlloProfClusteringP2P": "Identify the main category of Allo Prof document based on the titles and descriptions",
-    "AlloProfClusteringS2S": "Identify the topic of document titles from Allo Prof dataset",
+    "OpusparcusPC": "Retrieve semantically similar text",
+    "PAWSX": "Retrieve semantically similar text",
     "HALClusteringS2S": "Identify the main category of academic passage based on the titles and contents",
     "MasakhaNEWSClusteringP2P": "Identify the topic or theme of the given news articles based on the titles and contents",
     "MasakhaNEWSClusteringS2S": "Identify the topic or theme of the given news articles based on the titles",
@@ -541,10 +585,23 @@ KaLM_X_task_prompts = {
     "XPQARetrieval-query": "Given a question, retrieve passages that answer the question",
     "MintakaRetrieval-query": "Given a question, retrieve passages that answer the question",
     "CBD": "Classify the sentiment of polish tweet reviews",
-    "PolEmo2.0-IN": "Classify the sentiment of in-domain (medicine and hotels) online reviews",
-    "PolEmo2.0-OUT": "Classify the sentiment of out-of-domain (products and school) online reviews",
+    "PolEmo2.0-IN": "Classify the sentiment of medicine and hotels online reviews",
+    "PolEmo2.0-OUT": "Classify the sentiment of products and school online reviews",
     "AllegroReviews": "Classify the sentiment of reviews from e-commerce marketplace Allegro",
-    "PAC": 'Classify the sentence into one of the two types: "BEZPIECZNE_POSTANOWIENIE_UMOWNE" and "KLAUZULA_ABUZYWNA"',
+    "PAC": 'Classify Polish contract clauses into one of the following two types: "Safe Contract Clauses" and "Unfair Contract Clauses".',
+    "SICK-E-PL": "Retrieve semantically similar text",
+    "SICK-R-PL": "Retrieve semantically similar text",
+    "STS22": "Retrieve semantically similar text",
+    "AFQMC": "Retrieve semantically similar text",
+    "BQ": "Retrieve semantically similar text",
+    "LCQMC": "Retrieve semantically similar text",
+    "QBQTC": "Retrieve semantically similar text",
+    "STS12": "Retrieve semantically similar text",
+    "PpcPC": "Retrieve semantically similar text",
+    "CDSC-E": "Retrieve semantically similar text",
+    "BornholmBitextMining": "Retrieve parallel sentences",
+    "NorwegianCourtsBitextMining": "Retrieve parallel sentences",
+    "PSC": "Retrieve semantically similar text",
     "EightTagsClustering": "Identify of headlines from social media posts in Polish  into 8 categories: film, history, food, medicine, motorization, work, sport and technology",
     "ArguAna-PL-query": "Given a claim, find documents that refute the claim",
     "DBPedia-PL-query": "Given a query, retrieve relevant entity descriptions from DBPedia",
@@ -554,9 +611,9 @@ KaLM_X_task_prompts = {
     "NFCorpus-PL-query": "Given a question, retrieve relevant documents that best answer the question",
     "NQ-PL-query": "Given a question, retrieve Wikipedia passages that answer the question",
     "Quora-PL-query": "Given a question, retrieve questions that are semantically equivalent to the given question",
-    "SCIDOCS-PL-query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper",
+    "SCIDOCS-PL-query": "Given a title of a scientific paper, retrieve the titles of other relevant papers",
     "SciFact-PL-query": "Given a scientific claim, retrieve documents that support or refute the claim",
-    "TRECCOVID-PL-query": "Given a query on COVID-19, retrieve documents that answer the query",
+    "TRECCOVID-PL-query": "Given a medical query, retrieve documents that answer the query",
     "GeoreviewClassification": "Classify the organization rating based on the reviews",
     "HeadlineClassification": "Classify the topic or theme of the given news headline",
     "InappropriatenessClassification": "Classify the given message as either sensitive topic or not",
@@ -567,9 +624,12 @@ KaLM_X_task_prompts = {
     "GeoreviewClusteringP2P": "Identify the organization category based on the reviews",
     "RuSciBenchGRNTIClusteringP2P": "Identify the category of scientific papers based on the titles and abstracts",
     "RuSciBenchOECDClusteringP2P": "Identify the category of scientific papers based on the titles and abstracts",
+    "TERRa": "Given a premise, retrieve a hypothesis that is entailed by the premise",
     "RuBQReranking-query": "Given a question, retrieve Wikipedia passages that answer the question",
     "RiaNewsRetrieval-query": "Given a headline, retrieval relevant articles",
     "RuBQRetrieval-query": "Given a question, retrieve Wikipedia passages that answer the question",
+    "RUParaPhraserSTS": "Retrieve semantically similar text",
+    "RuSTSBenchmarkSTS": "Retrieve semantically similar text",
     "AppsRetrieval-query": "Given a question about code problem, retrieval code that can solve user's problem",
     "COIRCodeSearchNetRetrieval-query": "Given a code snippet, retrieve the comment corresponding to that code.",
     "CodeEditSearchRetrieval-query": "Given a piece of code, retrieval code that in the ",
@@ -582,8 +642,19 @@ KaLM_X_task_prompts = {
     "CosQA-query": "Given a question about coding, retrieval code or passage that can solve user's question",
     "StackOverflowQA-query": "Given a question about coding, retrieval code or passage that can solve user's question",
     "SyntheticText2SQL-query": "Given a user's question, retrieve SQL queries that are appropriate responses to the question",
-    "BulgarianStoreReviewSentimentClassfication": "Classify user reviews into positive or negative sentiment",
-    "CzechProductReviewSentimentClassification": "Classify product reviews into positive or negative sentiment",
+    "BibleNLPBitextMining": "Retrieve parallel sentences",
+    "BUCC.v2": "Retrieve parallel sentences",
+    "DiaBlaBitextMining": "Retrieve parallel sentences",
+    "FloresBitextMining": "Retrieve parallel sentences",
+    "IN22GenBitextMining": "Retrieve parallel sentences",
+    "IndicGenBenchFloresBitextMining": "Retrieve parallel sentences",
+    "NollySentiBitextMining": "Retrieve parallel sentences",
+    "NTREXBitextMining": "Retrieve parallel sentences",
+    "NusaTranslationBitextMining": "Retrieve parallel sentences",
+    "NusaXBitextMining": "Retrieve parallel sentences",
+    "Tatoeba": "Retrieve parallel sentences",
+    "BulgarianStoreReviewSentimentClassfication": "Classify user reviews into positive, negative or mixed sentiment",
+    "CzechProductReviewSentimentClassification": "Classify product reviews into positive, neutral, or negative sentiment",
     "GreekLegalCodeClassification": "Given a greek legal text, classify its topic",
     "DBpediaClassification": "Given a Wikipedia articles, categorized it into classes based on its DBpedia ontology",
     "FinancialPhrasebankClassification": "Given financial news, categorized by sentiment into positive, negative, or neutral",
@@ -611,7 +682,7 @@ KaLM_X_task_prompts = {
     "PunjabiNewsClassification": "Given a news article, categorized it into two-classes",
     "SinhalaNewsClassification": "Given a news article, categorized it into political, business, technology, sports and Entertainment",
     "CSFDSKMovieReviewSentimentClassification": "Given a movie review, classify its rating on a scale from 0 to 5",
-    "SiswatiNewsClassification": "Given a news article, classify its topic",
+    "SiswatiNewsClassification": "Given a news article in Siswati, classify its topic",
     "SlovakMovieReviewSentimentClassification": "Given a movie review, categorized it into positive or negative",
     "SwahiliNewsClassification": "Given a news article, classify its domain",
     "TswanaNewsClassification": "Given a news article, classify its topic",
@@ -621,7 +692,9 @@ KaLM_X_task_prompts = {
     "ArXivHierarchicalClusteringP2P": "Identify the main and secondary category of Arxiv papers based on the titles and abstracts",
     "ArXivHierarchicalClusteringS2S": "Identify the main and secondary category of Arxiv papers based on the titles",
     "BigPatentClustering.v2": "Identify the category of documents from the Big Patent dataset",
+    "AlloProfClusteringS2S": "Identify the topic of document titles from Allo Prof dataset",
     "AlloProfClusteringS2S.v2": "Identify the topic of document titles from Allo Prof dataset",
+    "AlloProfClusteringP2P": "Identify the topic of document titles and descriptions from Allo Prof dataset",
     "HALClusteringS2S.v2": "Identify the topic of titles from HAL",
     "SIB200ClusteringS2S": "Identify the category of documents",
     "WikiClusteringP2P.v2": "Identify the category of wiki passages",
@@ -629,26 +702,60 @@ KaLM_X_task_prompts = {
     "KorHateSpeechMLClassification": "Given a Korean online news comments, classify its fine-grained hate speech classes",
     "MalteseNewsClassification": "Given a maltese new, classify its topic",
     "MultiEURLEXMultilabelClassification": "Given a text, classify its topic",
-    "BrazilianToxicTweetsClassification": "Given a tweet, classify its topic",
+    "BrazilianToxicTweetsClassification": "Classify the toxic tweets in Brazilian Portuguese into one of the six categories: LGBTQ+phobia, Xenophobia, Obscene, Insult, Misogyny and Racism.",
+    "CTKFactsNLI": "Retrieve semantically similar text",
+    "indonli": "Retrieve semantically similar text",
+    "ArmenianParaphrasePC": "Retrieve semantically similar text",
+    "PawsXPairClassification": "Retrieve semantically similar text",
+    "RTE3": "Retrieve semantically similar text",
+    "XNLI": "Retrieve semantically similar text",
+    "GermanSTSBenchmark": "Retrieve semantically similar text",
+    "SICK-R": "Retrieve semantically similar text",
+    "STS13": "Retrieve semantically similar text",
+    "STS14": "Retrieve semantically similar text",
+    "STSBenchmark": "Retrieve semantically similar text",
+    "FaroeseSTS": "Retrieve semantically similar text",
+    "FinParaSTS": "Retrieve semantically similar text",
+    "JSICK": "Retrieve semantically similar text",
+    "IndicCrosslingualSTS": "Retrieve parallel sentences",
+    "SemRel24STS": "Retrieve semantically similar text",
+    "STS17": "Retrieve semantically similar text",
+    "STS22.v2": "Retrieve semantically similar text",
+    "STSES": "Retrieve semantically similar text",
+    "STSB": "Retrieve semantically similar text",
     "AILAStatutes-query": "Identifying the most relevant statutes for a given situation",
-    "HagridRetrieval-query": "Retrieval the relevant passage for the given query",
-    "LegalBenchCorporateLobbying-query": "Retrieval the relevant passage for the given query",
+    "HagridRetrieval-query": "Given an information-seeking question, retrieve the best replies to answer the question",
+    "LegalBenchCorporateLobbying-query": "Given a query, retrieve relevant legal bill summaries",
     "LEMBPasskeyRetrieval-query": "Retrieval the relevant passage for the given query",
     "BelebeleRetrieval-query": "Retrieval the relevant passage for the given query",
     "MLQARetrieval-query": "Retrieval the relevant passage for the given query",
     "StatcanDialogueDatasetRetrieval-query": "Retrieval the relevant passage for the given query",
     "WikipediaRetrievalMultilingual-query": "Retrieval the relevant passage for the given query",
-    "Core17InstructionRetrieval-query": "Retrieval the relevant passage for the given query",
-    "News21InstructionRetrieval-query": "Retrieval the relevant passage for the given query",
-    "Robust04InstructionRetrieval-query": "Retrieval the relevant passage for the given query",
+    "Core17InstructionRetrieval-query": "Retrieval the relevant passage for the given query with conditions",
+    "News21InstructionRetrieval-query": "Retrieval the relevant passage for the given query with conditions",
+    "Robust04InstructionRetrieval-query": "Retrieval the relevant passage for the given query with conditions",
     "WebLINXCandidatesReranking-query": "Retrieval the relevant passage for the given query",
     "WikipediaRerankingMultilingual-query": "Retrieval the relevant passage for the given query",
+    "STS15": "Retrieve semantically similar text",
     "MIRACLRetrievalHardNegatives-query": "Retrieval relevant passage for the given query",
+    "BIOSSES": "Retrieve semantically similar text",
     "CQADupstackRetrieval-query": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackGamingRetrieval-query": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackGamingRetrieval-document": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackUnixRetrieval-query": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackUnixRetrieval-document": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
+    "STS16": "Retrieve semantically similar text",
+    "SummEval": "Retrieve semantically similar text",
+    "ATEC": "Retrieve semantically similar text",
+    "ScalaClassification": "Classify passages into correct or correct in Scandinavian Languages based on linguistic acceptability",
+    "SpartQA-query": "Given the following spatial reasoning question, retrieve the right answer.",
+    "CEDRClassification": "Given a comment as query, classify expressed emotions into joy, sadness, surprise, fear, and anger",
+    "DalajClassification": "Classify texts based on linguistic acceptability in Swedish",
+    "TempReasonL1-query": "Given the following question about time, retrieve the correct answer.",
+    "WinoGrande-query": "Given the following sentence, retrieve an appropriate answer to fill in the missing underscored part.",
+    "NordicLangClassification": "Classify texts based on language",
+    "TwitterHjerneRetrieval-query": "Retrieve answers to questions asked in Danish tweets",
+    "SwednClusteringP2P": "Identify news categories in Swedish passages",
 }
 
 KaLM_INSTRUCTION = "Instruct: {instruction} \n Query: "
@@ -795,3 +902,30 @@ KaLM_Embedding_KaLM_embedding_multilingual_mini_instruct_v2_5 = ModelMeta(
     adapted_from="HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v2",
     superseded_by=None,
 )
+
+KaLM_Embedding_gemma_3_12b_2511 = ModelMeta(
+    loader=InstructSentenceTransformerModel,
+    loader_kwargs=dict(
+        instruction_template=KaLM_INSTRUCTION,
+        max_seq_length=512,
+        apply_instruction_to_passages=True,
+        prompts_dict=KaLM_Embedding_gemma_3_12b_task_prompts,
+    ),
+    name="tencent/KaLM-Embedding-Gemma3-12B-2511",
+    revision="edf22f4753f58b05e3f5495818d31f12db63056d",
+    languages=None,
+    open_weights=True,
+    release_date="2025-11-06",
+    n_parameters=11.76 * 1e9,
+    memory_usage_mb=44884,
+    max_tokens=32768,
+    embed_dim=3840,
+    license=None,
+    reference="https://kalm-embedding.github.io/",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    public_training_code="https://github.com/HITsz-TMG/KaLM-Embedding",
+    public_training_data=None,
+    training_datasets=KaLM_Embedding_gemma_3_12b_training_data,
+)

mteb-2.1.7/mteb/models/model_implementations/rasgaard_models.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+from mteb.models.model_implementations.model2vec_models import Model2VecModel
+from mteb.models.model_meta import ModelMeta, ScoringFunction
+
+potion_base_8m = ModelMeta(
+    loader=Model2VecModel,  # type: ignore
+    name="rasgaard/m2v-dfm-large",
+    languages=["dan-Latn"],
+    open_weights=True,
+    revision="387897cfb09992e6d45ea9cd7b28b9fcf119e23a",
+    release_date="2025-10-08",
+    n_parameters=22893312,
+    memory_usage_mb=87,
+    max_tokens=np.inf,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=["NumPy", "Sentence Transformers"],
+    reference="https://huggingface.co/minishlab/rasgaard/m2v-dfm-large",
+    use_instructions=False,
+    adapted_from="KennethEnevoldsen/dfm-sentence-encoder-large",
+    superseded_by=None,
+    training_datasets=set(),  # distilled
+    public_training_code="https://github.com/MinishLab/model2vec",
+    public_training_data="https://huggingface.co/datasets/HuggingFaceFW/fineweb-2",  # distilled on this
+)

{mteb-2.1.5 → mteb-2.1.7}/mteb/models/model_implementations/tarka_models.py

@@ -40,7 +40,7 @@ tarka_embedding_150m_v1 = ModelMeta(
     name="Tarka-AIR/Tarka-Embedding-150M-V1",
     languages=MULTILINGUAL_EVALUATED_LANGUAGES,
     open_weights=True,
-    revision="c5f4f43",
+    revision="b0ffecc4ef0d873e517507ed080e43b88b2704b9",
     release_date="2025-11-04",
     n_parameters=155_714_304,
     embed_dim=768,

{mteb-2.1.5 → mteb-2.1.7}/mteb/tasks/classification/nld/dutch_book_review_sentiment_classification.py

@@ -49,7 +49,8 @@ class DutchBookReviewSentimentClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="DutchBookReviewSentimentClassification.v2",
         description="""A Dutch book review for sentiment classification.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900).
+        Additionally, a Dutch prompt was included.""",
         reference="https://github.com/benjaminvdb/DBRD",
         dataset={
             "path": "mteb/dutch_book_review_sentiment",
@@ -86,4 +87,7 @@ Suzan, Verberne},
 }
 """,
         adapted_from=["DutchBookReviewSentimentClassification"],
+        prompt={
+            "query": "Classificeer de gegeven boekrecensie als positieve of negatieve sentiment"
+        },
     )

{mteb-2.1.5 → mteb-2.1.7}/mteb/tasks/classification/nld/dutch_cola_classification.py

@@ -35,4 +35,7 @@ class DutchColaClassification(AbsTaskClassification):
   year = {2024},
 }
 """,
+        prompt={
+            "query": "Classificeer de gegeven zin als grammaticaal aanvaardbaar of niet aanvaardbaar"
+        },
     )

{mteb-2.1.5 → mteb-2.1.7}/mteb/tasks/classification/nld/dutch_government_bias_classification.py

@@ -34,4 +34,7 @@ class DutchGovernmentBiasClassification(AbsTaskClassification):
   year = {2025},
 }
 """,
+        prompt={
+            "query": "Classificeer het gegeven overheidsdocument als bevooroordeeld of niet bevooroordeeld"
+        },
     )

{mteb-2.1.5 → mteb-2.1.7}/mteb/tasks/classification/nld/dutch_news_articles_classification.py

@@ -27,4 +27,7 @@ class DutchNewsArticlesClassification(AbsTaskClassification):
         dialect=[],
         sample_creation="found",
         bibtex_citation="",
+        prompt={
+            "query": "Classificeer het gegeven nieuwsartikel in het juiste onderwerp of thema"
+        },
     )

{mteb-2.1.5 → mteb-2.1.7}/mteb/tasks/classification/nld/dutch_sarcastic_headlines_classification.py

@@ -27,6 +27,9 @@ class DutchSarcasticHeadlinesClassification(AbsTaskClassification):
         dialect=[],
         sample_creation="found",
         bibtex_citation="""""",
+        prompt={
+            "query": "Classificeer de gegeven krantenkop als sarcastisch of niet sarcastisch"
+        },
     )
 
     def dataset_transform(self):

{mteb-2.1.5 → mteb-2.1.7}/mteb/tasks/classification/nld/iconclass_classification.py

@@ -38,4 +38,7 @@ class IconclassClassification(AbsTaskClassification):
   year = {2023},
 }
 """,
+        prompt={
+            "query": "Classificeer de gegeven titel van het kunstwerk in het juiste onderwerp of thema"
+        },
     )

{mteb-2.1.5 → mteb-2.1.7}/mteb/tasks/classification/nld/open_tender_classification.py

@@ -35,4 +35,7 @@ class OpenTenderClassification(AbsTaskClassification):
   year = {2025},
 }
 """,
+        prompt={
+            "query": "Classificeer de gegeven aanbestedingsbeschrijving in het juiste onderwerp of thema"
+        },
     )