mteb 2.1.4__py3-none-any.whl → 2.1.6__py3-none-any.whl

mteb/models/model_implementations/colqwen_models.py

@@ -220,3 +220,60 @@ colnomic_7b = ModelMeta(
     training_datasets=COLNOMIC_TRAINING_DATA,
     citation=COLNOMIC_CITATION,
 )
+
+
+EVOQWEN_TRAINING_DATA = {
+    "colpali_train_set",
+    "VisRAG-Ret-Train-Synthetic-data",
+    "VisRAG-Ret-Train-In-domain-data",
+}
+
+evoqwen25_vl_retriever_3b_v1 = ModelMeta(
+    loader=ColQwen2_5Wrapper,
+    loader_kwargs=dict(
+        torch_dtype=torch.float16, attn_implementation="flash_attention_2"
+    ),
+    name="ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-3B-v1",
+    languages=["eng-Latn"],
+    revision="aeacaa2775f2758d82721eb1cf2f5daf1a392da9",
+    release_date="2025-11-04",
+    modalities=["image", "text"],
+    n_parameters=3_000_000_000,
+    memory_usage_mb=7200,
+    max_tokens=128000,
+    embed_dim=128,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code="https://github.com/illuin-tech/colpali",
+    public_training_data="https://huggingface.co/datasets/vidore/colpali_train_set",
+    framework=["ColPali"],
+    reference="https://huggingface.co/ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-3B-v1",
+    similarity_fn_name="MaxSim",
+    use_instructions=True,
+    training_datasets=EVOQWEN_TRAINING_DATA,
+)
+
+evoqwen25_vl_retriever_7b_v1 = ModelMeta(
+    loader=ColQwen2_5Wrapper,
+    loader_kwargs=dict(
+        torch_dtype=torch.float16, attn_implementation="flash_attention_2"
+    ),
+    name="ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-7B-v1",
+    languages=["eng-Latn"],
+    revision="8952ac6ee0e7de2e9211b165921518caf9202110",
+    release_date="2025-11-04",
+    modalities=["image", "text"],
+    n_parameters=7_000_000_000,
+    memory_usage_mb=14400,
+    max_tokens=128000,
+    embed_dim=128,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code="https://github.com/illuin-tech/colpali",
+    public_training_data="https://huggingface.co/datasets/vidore/colpali_train_set",
+    framework=["ColPali"],
+    reference="https://huggingface.co/ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-7B-v1",
+    similarity_fn_name="MaxSim",
+    use_instructions=True,
+    training_datasets=EVOQWEN_TRAINING_DATA,
+)

mteb/models/model_implementations/kalm_models.py

@@ -247,6 +247,40 @@ kalm_v2_training_data = {
 }
 
 
+KaLM_Embedding_gemma_3_12b_training_data = [
+    "BornholmBitextMining",
+    "BibleNLPBitextMining",  # train[256:] to avoid the test set in MTEB
+    "FinParaSTS",
+    "SemRel24STS",
+    "STSB",
+    "GermanSTSBenchmark",
+    "TERRa",
+    "StatcanDialogueDatasetRetrieval",
+    "SwissJudgementClassification",
+    "GreekLegalCodeClassification",
+    "CataloniaTweetClassification",
+    "NordicLangClassification",
+    "CyrillicTurkicLangClassification",
+    "PoemSentimentClassification",
+    "CzechProductReviewSentimentClassification",
+    "DBpediaClassification",
+    "IndicLangClassification",
+    "NusaParagraphEmotionClassification",
+    "OdiaNewsClassification",
+    "ScalaClassification",
+    "ItaCaseholdClassification",
+    "CSFDSKMovieReviewSentimentClassification",
+    "ToxicConversationsClassification",
+    "PAC",
+    "SpartQA",
+    "WinoGrande",
+    "Quail",
+    "HellaSwag",
+    "AlphaNLI",
+]
+KaLM_Embedding_gemma_3_12b_training_data += kalm_training_data
+
+
 KaLM_task_prompts = {
     "AmazonCounterfactualClassification": "Given an Amazon review, judge whether it is counterfactual.",
     "AmazonPolarityClassification": "Classifying Amazon reviews into positive or negative sentiment",
@@ -450,13 +484,7 @@ KaLM_v2_task_prompts = {
     "SummEval-document": "Retrieve semantically similar summaries",
 }
 
-KaLM_X_task_prompts = {
-    "Classification": "classify the query into different classes.",
-    "MultilabelClassification": "Instruct: classify the query into different classes.",
-    "Clustering": "classify the query into different classes.",
-    "Reranking-query": "Given a query, retrieve documents that answer the query.",
-    "Retrieval-query": "Given a query, retrieve documents that answer the query.",
-    "InstructionRetrieval-query": "Given a query, retrieve documents that answer the query.",
+KaLM_Embedding_gemma_3_12b_task_prompts = {
     "AmazonCounterfactualClassification": "Classify a given Amazon customer review text as either counterfactual or not-counterfactual",
     "AmazonPolarityClassification": "Classify Amazon reviews into positive or negative sentiment",
     "AmazonReviewsClassification": "Classify the given Amazon review into its appropriate rating category",
@@ -478,26 +506,38 @@ KaLM_X_task_prompts = {
     "ArxivClusteringP2P": "Identify the main and secondary category of Arxiv papers based on the titles and abstracts",
     "ArxivClusteringS2S": "Identify the main and secondary category of Arxiv papers based on the titles",
     "BiorxivClusteringP2P": "Identify the main category of Biorxiv papers based on the titles and abstracts",
+    "BiorxivClusteringP2P.v2": "Identify the main category of Biorxiv papers based on the titles and abstracts",
     "BiorxivClusteringS2S": "Identify the main category of Biorxiv papers based on the titles",
+    "BiorxivClusteringS2S.v2": "Identify the main category of Biorxiv papers based on the titles",
     "MedrxivClusteringP2P": "Identify the main category of Medrxiv papers based on the titles and abstracts",
+    "MedrxivClusteringP2P.v2": "Identify the main category of Medrxiv papers based on the titles and abstracts",
     "MedrxivClusteringS2S": "Identify the main category of Medrxiv papers based on the titles",
+    "MedrxivClusteringS2S.v2": "Identify the main category of Medrxiv papers based on the titles",
     "RedditClustering": "Identify the topic or theme of Reddit posts based on the titles",
     "RedditClusteringP2P": "Identify the topic or theme of Reddit posts based on the titles and posts",
     "StackExchangeClustering": "Identify the topic or theme of StackExchange posts based on the titles",
+    "StackExchangeClustering.v2": "Identify the topic or theme of StackExchange posts based on the titles",
     "StackExchangeClusteringP2P": "Identify the topic or theme of StackExchange posts based on the given paragraphs",
+    "StackExchangeClusteringP2P.v2": "Identify the topic or theme of StackExchange posts based on the given paragraphs",
     "TwentyNewsgroupsClustering": "Identify the topic or theme of the given news articles",
     "CLSClusteringS2S": "Identify the main category of scholar papers based on the titles",
     "CLSClusteringP2P": "Identify the main category of scholar papers based on the titles and abstracts",
+    "CLSClusteringP2P.v2": "Identify the main category of scholar papers based on the titles and abstracts",
     "ThuNewsClusteringS2S": "Identify the topic or theme of the given news articles based on the titles",
     "ThuNewsClusteringP2P": "Identify the topic or theme of the given news articles based on the titles and contents",
     "AskUbuntuDupQuestions-query": "Retrieve duplicate questions from AskUbuntu forum",
     "MindSmallReranking-query": "Retrieve relevant news articles based on user browsing history",
     "SciDocsRR-query": "Given a title of a scientific paper, retrieve the titles of other relevant papers",
     "StackOverflowDupQuestions-query": "Retrieve duplicate questions from StackOverflow forum",
+    "SprintDuplicateQuestions": "Retrieve semantically duplicate questions",
+    "TwitterSemEval2015": "Retrieve tweets that are semantically similar to the given tweet",
+    "TwitterURLCorpus": "Retrieve tweets that are semantically similar to the given tweet",
     "T2Reranking-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "MMarcoReranking-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "CMedQAv1-reranking-query": "Given a Chinese community medical question, retrieve replies that best answer the question",
     "CMedQAv2-reranking-query": "Given a Chinese community medical question, retrieve replies that best answer the question",
+    "Ocnli": "Retrieve semantically similar text.",
+    "Cmnli": "Retrieve semantically similar text.",
     "ArguAna-query": "Given a claim, find documents that refute the claim",
     "ArguAna-document": "Given a claim, find documents that refute the claim",
     "ClimateFEVER-query": "Given a claim about climate change, retrieve documents that support or refute the claim",
@@ -512,22 +552,26 @@ KaLM_X_task_prompts = {
     "NFCorpus-query": "Given a question, retrieve relevant documents that best answer the question",
     "NQ-query": "Given a question, retrieve Wikipedia passages that answer the question",
     "QuoraRetrieval-query": "Given a question, retrieve questions that are semantically equivalent to the given question",
-    "SCIDOCS-query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper",
+    "SCIDOCS-query": "Given a title of a scientific paper, retrieve the titles of other relevant papers",
     "SciFact-query": "Given a scientific claim, retrieve documents that support or refute the claim",
     "Touche2020-query": "Given a question, retrieve detailed and persuasive arguments that answer the question",
     "Touche2020Retrieval.v3-query": "Given a question, retrieve detailed and persuasive arguments that answer the question",
-    "TRECCOVID-query": "Given a query on COVID-19, retrieve documents that answer the query",
+    "TRECCOVID-query": "Given a medical query, retrieve documents that answer the query",
     "T2Retrieval-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "MMarcoRetrieval-query": "Given a web search query, retrieve relevant passages that answer the query",
+    "VoyageMMarcoReranking-query": "Given a Japanese search query, retrieve web passages that answer the question",
     "DuRetrieval-query": "Given a Chinese search query, retrieve web passages that answer the question",
     "CovidRetrieval-query": "Given a question on COVID-19, retrieve news articles that answer the question",
     "CmedqaRetrieval-query": "Given a Chinese community medical question, retrieve replies that best answer the question",
     "EcomRetrieval-query": "Given a user query from an e-commerce website, retrieve description sentences of relevant products",
     "MedicalRetrieval-query": "Given a medical question, retrieve user replies that best answer the question",
     "VideoRetrieval-query": "Given a video search query, retrieve the titles of relevant videos",
+    "STSBenchmarkMultilingualSTS": "Retrieve semantically similar text",
+    "SICKFr": "Retrieve semantically similar text",
+    "SummEvalFr": "Given a news summary, retrieve other semantically similar summaries",
     "MasakhaNEWSClassification": "Classify the News in the given texts into one of the seven category: politics,sports,health,business,entertainment,technology,religion ",
-    "AlloProfClusteringP2P": "Identify the main category of Allo Prof document based on the titles and descriptions",
-    "AlloProfClusteringS2S": "Identify the topic of document titles from Allo Prof dataset",
+    "OpusparcusPC": "Retrieve semantically similar text",
+    "PAWSX": "Retrieve semantically similar text",
     "HALClusteringS2S": "Identify the main category of academic passage based on the titles and contents",
     "MasakhaNEWSClusteringP2P": "Identify the topic or theme of the given news articles based on the titles and contents",
     "MasakhaNEWSClusteringS2S": "Identify the topic or theme of the given news articles based on the titles",
@@ -541,10 +585,23 @@ KaLM_X_task_prompts = {
     "XPQARetrieval-query": "Given a question, retrieve passages that answer the question",
     "MintakaRetrieval-query": "Given a question, retrieve passages that answer the question",
     "CBD": "Classify the sentiment of polish tweet reviews",
-    "PolEmo2.0-IN": "Classify the sentiment of in-domain (medicine and hotels) online reviews",
-    "PolEmo2.0-OUT": "Classify the sentiment of out-of-domain (products and school) online reviews",
+    "PolEmo2.0-IN": "Classify the sentiment of medicine and hotels online reviews",
+    "PolEmo2.0-OUT": "Classify the sentiment of products and school online reviews",
     "AllegroReviews": "Classify the sentiment of reviews from e-commerce marketplace Allegro",
-    "PAC": 'Classify the sentence into one of the two types: "BEZPIECZNE_POSTANOWIENIE_UMOWNE" and "KLAUZULA_ABUZYWNA"',
+    "PAC": 'Classify Polish contract clauses into one of the following two types: "Safe Contract Clauses" and "Unfair Contract Clauses".',
+    "SICK-E-PL": "Retrieve semantically similar text",
+    "SICK-R-PL": "Retrieve semantically similar text",
+    "STS22": "Retrieve semantically similar text",
+    "AFQMC": "Retrieve semantically similar text",
+    "BQ": "Retrieve semantically similar text",
+    "LCQMC": "Retrieve semantically similar text",
+    "QBQTC": "Retrieve semantically similar text",
+    "STS12": "Retrieve semantically similar text",
+    "PpcPC": "Retrieve semantically similar text",
+    "CDSC-E": "Retrieve semantically similar text",
+    "BornholmBitextMining": "Retrieve parallel sentences",
+    "NorwegianCourtsBitextMining": "Retrieve parallel sentences",
+    "PSC": "Retrieve semantically similar text",
     "EightTagsClustering": "Identify of headlines from social media posts in Polish  into 8 categories: film, history, food, medicine, motorization, work, sport and technology",
     "ArguAna-PL-query": "Given a claim, find documents that refute the claim",
     "DBPedia-PL-query": "Given a query, retrieve relevant entity descriptions from DBPedia",
@@ -554,9 +611,9 @@ KaLM_X_task_prompts = {
     "NFCorpus-PL-query": "Given a question, retrieve relevant documents that best answer the question",
     "NQ-PL-query": "Given a question, retrieve Wikipedia passages that answer the question",
     "Quora-PL-query": "Given a question, retrieve questions that are semantically equivalent to the given question",
-    "SCIDOCS-PL-query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper",
+    "SCIDOCS-PL-query": "Given a title of a scientific paper, retrieve the titles of other relevant papers",
     "SciFact-PL-query": "Given a scientific claim, retrieve documents that support or refute the claim",
-    "TRECCOVID-PL-query": "Given a query on COVID-19, retrieve documents that answer the query",
+    "TRECCOVID-PL-query": "Given a medical query, retrieve documents that answer the query",
     "GeoreviewClassification": "Classify the organization rating based on the reviews",
     "HeadlineClassification": "Classify the topic or theme of the given news headline",
     "InappropriatenessClassification": "Classify the given message as either sensitive topic or not",
@@ -567,9 +624,12 @@ KaLM_X_task_prompts = {
     "GeoreviewClusteringP2P": "Identify the organization category based on the reviews",
     "RuSciBenchGRNTIClusteringP2P": "Identify the category of scientific papers based on the titles and abstracts",
     "RuSciBenchOECDClusteringP2P": "Identify the category of scientific papers based on the titles and abstracts",
+    "TERRa": "Given a premise, retrieve a hypothesis that is entailed by the premise",
     "RuBQReranking-query": "Given a question, retrieve Wikipedia passages that answer the question",
     "RiaNewsRetrieval-query": "Given a headline, retrieval relevant articles",
     "RuBQRetrieval-query": "Given a question, retrieve Wikipedia passages that answer the question",
+    "RUParaPhraserSTS": "Retrieve semantically similar text",
+    "RuSTSBenchmarkSTS": "Retrieve semantically similar text",
     "AppsRetrieval-query": "Given a question about code problem, retrieval code that can solve user's problem",
     "COIRCodeSearchNetRetrieval-query": "Given a code snippet, retrieve the comment corresponding to that code.",
     "CodeEditSearchRetrieval-query": "Given a piece of code, retrieval code that in the ",
@@ -582,8 +642,19 @@ KaLM_X_task_prompts = {
     "CosQA-query": "Given a question about coding, retrieval code or passage that can solve user's question",
     "StackOverflowQA-query": "Given a question about coding, retrieval code or passage that can solve user's question",
     "SyntheticText2SQL-query": "Given a user's question, retrieve SQL queries that are appropriate responses to the question",
-    "BulgarianStoreReviewSentimentClassfication": "Classify user reviews into positive or negative sentiment",
-    "CzechProductReviewSentimentClassification": "Classify product reviews into positive or negative sentiment",
+    "BibleNLPBitextMining": "Retrieve parallel sentences",
+    "BUCC.v2": "Retrieve parallel sentences",
+    "DiaBlaBitextMining": "Retrieve parallel sentences",
+    "FloresBitextMining": "Retrieve parallel sentences",
+    "IN22GenBitextMining": "Retrieve parallel sentences",
+    "IndicGenBenchFloresBitextMining": "Retrieve parallel sentences",
+    "NollySentiBitextMining": "Retrieve parallel sentences",
+    "NTREXBitextMining": "Retrieve parallel sentences",
+    "NusaTranslationBitextMining": "Retrieve parallel sentences",
+    "NusaXBitextMining": "Retrieve parallel sentences",
+    "Tatoeba": "Retrieve parallel sentences",
+    "BulgarianStoreReviewSentimentClassfication": "Classify user reviews into positive, negative or mixed sentiment",
+    "CzechProductReviewSentimentClassification": "Classify product reviews into positive, neutral, or negative sentiment",
     "GreekLegalCodeClassification": "Given a greek legal text, classify its topic",
     "DBpediaClassification": "Given a Wikipedia articles, categorized it into classes based on its DBpedia ontology",
     "FinancialPhrasebankClassification": "Given financial news, categorized by sentiment into positive, negative, or neutral",
@@ -611,7 +682,7 @@ KaLM_X_task_prompts = {
     "PunjabiNewsClassification": "Given a news article, categorized it into two-classes",
     "SinhalaNewsClassification": "Given a news article, categorized it into political, business, technology, sports and Entertainment",
     "CSFDSKMovieReviewSentimentClassification": "Given a movie review, classify its rating on a scale from 0 to 5",
-    "SiswatiNewsClassification": "Given a news article, classify its topic",
+    "SiswatiNewsClassification": "Given a news article in Siswati, classify its topic",
     "SlovakMovieReviewSentimentClassification": "Given a movie review, categorized it into positive or negative",
     "SwahiliNewsClassification": "Given a news article, classify its domain",
     "TswanaNewsClassification": "Given a news article, classify its topic",
@@ -621,7 +692,9 @@ KaLM_X_task_prompts = {
     "ArXivHierarchicalClusteringP2P": "Identify the main and secondary category of Arxiv papers based on the titles and abstracts",
     "ArXivHierarchicalClusteringS2S": "Identify the main and secondary category of Arxiv papers based on the titles",
     "BigPatentClustering.v2": "Identify the category of documents from the Big Patent dataset",
+    "AlloProfClusteringS2S": "Identify the topic of document titles from Allo Prof dataset",
     "AlloProfClusteringS2S.v2": "Identify the topic of document titles from Allo Prof dataset",
+    "AlloProfClusteringP2P": "Identify the topic of document titles and descriptions from Allo Prof dataset",
     "HALClusteringS2S.v2": "Identify the topic of titles from HAL",
     "SIB200ClusteringS2S": "Identify the category of documents",
     "WikiClusteringP2P.v2": "Identify the category of wiki passages",
@@ -629,26 +702,60 @@ KaLM_X_task_prompts = {
     "KorHateSpeechMLClassification": "Given a Korean online news comments, classify its fine-grained hate speech classes",
     "MalteseNewsClassification": "Given a maltese new, classify its topic",
     "MultiEURLEXMultilabelClassification": "Given a text, classify its topic",
-    "BrazilianToxicTweetsClassification": "Given a tweet, classify its topic",
+    "BrazilianToxicTweetsClassification": "Classify the toxic tweets in Brazilian Portuguese into one of the six categories: LGBTQ+phobia, Xenophobia, Obscene, Insult, Misogyny and Racism.",
+    "CTKFactsNLI": "Retrieve semantically similar text",
+    "indonli": "Retrieve semantically similar text",
+    "ArmenianParaphrasePC": "Retrieve semantically similar text",
+    "PawsXPairClassification": "Retrieve semantically similar text",
+    "RTE3": "Retrieve semantically similar text",
+    "XNLI": "Retrieve semantically similar text",
+    "GermanSTSBenchmark": "Retrieve semantically similar text",
+    "SICK-R": "Retrieve semantically similar text",
+    "STS13": "Retrieve semantically similar text",
+    "STS14": "Retrieve semantically similar text",
+    "STSBenchmark": "Retrieve semantically similar text",
+    "FaroeseSTS": "Retrieve semantically similar text",
+    "FinParaSTS": "Retrieve semantically similar text",
+    "JSICK": "Retrieve semantically similar text",
+    "IndicCrosslingualSTS": "Retrieve parallel sentences",
+    "SemRel24STS": "Retrieve semantically similar text",
+    "STS17": "Retrieve semantically similar text",
+    "STS22.v2": "Retrieve semantically similar text",
+    "STSES": "Retrieve semantically similar text",
+    "STSB": "Retrieve semantically similar text",
     "AILAStatutes-query": "Identifying the most relevant statutes for a given situation",
-    "HagridRetrieval-query": "Retrieval the relevant passage for the given query",
-    "LegalBenchCorporateLobbying-query": "Retrieval the relevant passage for the given query",
+    "HagridRetrieval-query": "Given an information-seeking question, retrieve the best replies to answer the question",
+    "LegalBenchCorporateLobbying-query": "Given a query, retrieve relevant legal bill summaries",
     "LEMBPasskeyRetrieval-query": "Retrieval the relevant passage for the given query",
     "BelebeleRetrieval-query": "Retrieval the relevant passage for the given query",
     "MLQARetrieval-query": "Retrieval the relevant passage for the given query",
     "StatcanDialogueDatasetRetrieval-query": "Retrieval the relevant passage for the given query",
     "WikipediaRetrievalMultilingual-query": "Retrieval the relevant passage for the given query",
-    "Core17InstructionRetrieval-query": "Retrieval the relevant passage for the given query",
-    "News21InstructionRetrieval-query": "Retrieval the relevant passage for the given query",
-    "Robust04InstructionRetrieval-query": "Retrieval the relevant passage for the given query",
+    "Core17InstructionRetrieval-query": "Retrieval the relevant passage for the given query with conditions",
+    "News21InstructionRetrieval-query": "Retrieval the relevant passage for the given query with conditions",
+    "Robust04InstructionRetrieval-query": "Retrieval the relevant passage for the given query with conditions",
     "WebLINXCandidatesReranking-query": "Retrieval the relevant passage for the given query",
     "WikipediaRerankingMultilingual-query": "Retrieval the relevant passage for the given query",
+    "STS15": "Retrieve semantically similar text",
     "MIRACLRetrievalHardNegatives-query": "Retrieval relevant passage for the given query",
+    "BIOSSES": "Retrieve semantically similar text",
     "CQADupstackRetrieval-query": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackGamingRetrieval-query": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackGamingRetrieval-document": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackUnixRetrieval-query": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
     "CQADupstackUnixRetrieval-document": "Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question",
+    "STS16": "Retrieve semantically similar text",
+    "SummEval": "Retrieve semantically similar text",
+    "ATEC": "Retrieve semantically similar text",
+    "ScalaClassification": "Classify passages into correct or correct in Scandinavian Languages based on linguistic acceptability",
+    "SpartQA-query": "Given the following spatial reasoning question, retrieve the right answer.",
+    "CEDRClassification": "Given a comment as query, classify expressed emotions into joy, sadness, surprise, fear, and anger",
+    "DalajClassification": "Classify texts based on linguistic acceptability in Swedish",
+    "TempReasonL1-query": "Given the following question about time, retrieve the correct answer.",
+    "WinoGrande-query": "Given the following sentence, retrieve an appropriate answer to fill in the missing underscored part.",
+    "NordicLangClassification": "Classify texts based on language",
+    "TwitterHjerneRetrieval-query": "Retrieve answers to questions asked in Danish tweets",
+    "SwednClusteringP2P": "Identify news categories in Swedish passages",
 }
 
 KaLM_INSTRUCTION = "Instruct: {instruction} \n Query: "
@@ -795,3 +902,30 @@ KaLM_Embedding_KaLM_embedding_multilingual_mini_instruct_v2_5 = ModelMeta(
     adapted_from="HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v2",
     superseded_by=None,
 )
+
+KaLM_Embedding_gemma_3_12b_2511 = ModelMeta(
+    loader=InstructSentenceTransformerModel,
+    loader_kwargs=dict(
+        instruction_template=KaLM_INSTRUCTION,
+        max_seq_length=512,
+        apply_instruction_to_passages=True,
+        prompts_dict=KaLM_Embedding_gemma_3_12b_task_prompts,
+    ),
+    name="tencent/KaLM-Embedding-Gemma3-12B-2511",
+    revision="edf22f4753f58b05e3f5495818d31f12db63056d",
+    languages=None,
+    open_weights=True,
+    release_date="2025-11-06",
+    n_parameters=11.76 * 1e9,
+    memory_usage_mb=44884,
+    max_tokens=32768,
+    embed_dim=3840,
+    license=None,
+    reference="https://kalm-embedding.github.io/",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    public_training_code="https://github.com/HITsz-TMG/KaLM-Embedding",
+    public_training_data=None,
+    training_datasets=KaLM_Embedding_gemma_3_12b_training_data,
+)

mteb/models/model_implementations/rasgaard_models.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+from mteb.models.model_implementations.model2vec_models import Model2VecModel
+from mteb.models.model_meta import ModelMeta, ScoringFunction
+
+potion_base_8m = ModelMeta(
+    loader=Model2VecModel,  # type: ignore
+    name="rasgaard/m2v-dfm-large",
+    languages=["dan-Latn"],
+    open_weights=True,
+    revision="387897cfb09992e6d45ea9cd7b28b9fcf119e23a",
+    release_date="2025-10-08",
+    n_parameters=22893312,
+    memory_usage_mb=87,
+    max_tokens=np.inf,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=["NumPy", "Sentence Transformers"],
+    reference="https://huggingface.co/minishlab/rasgaard/m2v-dfm-large",
+    use_instructions=False,
+    adapted_from="KennethEnevoldsen/dfm-sentence-encoder-large",
+    superseded_by=None,
+    training_datasets=set(),  # distilled
+    public_training_code="https://github.com/MinishLab/model2vec",
+    public_training_data="https://huggingface.co/datasets/HuggingFaceFW/fineweb-2",  # distilled on this
+)

mteb/models/model_implementations/tarka_models.py

@@ -0,0 +1,58 @@
+from mteb.models.model_implementations.google_models import gemma_embedding_loader
+from mteb.models.model_meta import ModelMeta
+
+Tarka_Embedding_150M_V1_CITATION = """@misc{tarka_ai_research_2025,
+	author       = { Tarka AI Research },
+	title        = { Tarka-Embedding-150M-V1 (Revision c5f4f43) },
+	year         = 2025,
+	url          = { https://huggingface.co/Tarka-AIR/Tarka-Embedding-150M-V1 },
+	doi          = { 10.57967/hf/6875 },
+	publisher    = { Hugging Face }
+}"""
+
+MULTILINGUAL_EVALUATED_LANGUAGES = [
+    "arb-Arab",
+    "eng-Latn",
+    "spa-Latn",
+    "deu-Latn",
+    "fra-Latn",
+    "jpn-Jpan",
+    "kor-Hang",
+    "zho-Hans",
+]
+
+training_data = {
+    "T2Retrieval",
+    "DuRetrieval",
+    "MMarcoReranking",
+    "CMedQAv2-reranking",
+    "NQ",
+    "MSMARCO",
+    "HotpotQA",
+    "FEVER",
+    "MrTidyRetrieval",
+    "MIRACLRetrieval",
+    "CodeSearchNet",
+}
+
+tarka_embedding_150m_v1 = ModelMeta(
+    loader=gemma_embedding_loader,
+    name="Tarka-AIR/Tarka-Embedding-150M-V1",
+    languages=MULTILINGUAL_EVALUATED_LANGUAGES,
+    open_weights=True,
+    revision="b0ffecc4ef0d873e517507ed080e43b88b2704b9",
+    release_date="2025-11-04",
+    n_parameters=155_714_304,
+    embed_dim=768,
+    max_tokens=2048,
+    license="gemma",
+    reference="https://huggingface.co/Tarka-AIR/Tarka-Embedding-150M-V1",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=training_data,
+    similarity_fn_name="cosine",
+    memory_usage_mb=576,
+    citation=Tarka_Embedding_150M_V1_CITATION,
+)

mteb/models/search_wrappers.py

@@ -191,6 +191,7 @@ class SearchEncoderWrapper:
             cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist()
             cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist()
 
+            sub_corpus_ids = list(sub_corpus_ids)
             for query_itr in range(len(query_embeddings)):
                 query_id = query_idx_to_id[query_itr]
                 for sub_corpus_id, score in zip(

{mteb-2.1.4.dist-info → mteb-2.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.1.4
+Version: 2.1.6
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>
@@ -16,7 +16,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
-Requires-Python: <3.14,>=3.10
+Requires-Python: <3.15,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: datasets>=2.19.0

{mteb-2.1.4.dist-info → mteb-2.1.6.dist-info}/RECORD

@@ -1419,7 +1419,7 @@ mteb/models/get_model_meta.py,sha256=VpZZNINk-QrNeVpPZnlqzlLhtBs8G84eRwTzAb_gRD4
 mteb/models/instruct_wrapper.py,sha256=HxHmnlxkjtZhfgTZRYJBT3Nma7Dhx6a9e2Bg-cO_IYs,8844
 mteb/models/model_meta.py,sha256=b-Nel9nX5bJk4cgJnqkBzEKyMY7uXvxlCBSxmmH1Ios,14769
 mteb/models/models_protocols.py,sha256=D2hYWn_UBGMaKtRwBx3u0B0ni6lHJjSzTxX21XFNwIc,8917
-mteb/models/search_wrappers.py,sha256=0McxwGnqyiYKPHjHsxWZp1pP9qGuHemZjeX1z5ZgNAI,15588
+mteb/models/search_wrappers.py,sha256=W99EeMDQ58N0auVp2-7T39orG7GZLekSsVCdxEZLxw0,15638
 mteb/models/sentence_transformer_wrapper.py,sha256=n5CMsM6Lpg_CFHH0NkpJusMsaLUTt-L9vRmFINQ961k,12338
 mteb/models/cache_wrappers/__init__.py,sha256=j3JBHN73Tr7uMUO92FEvKXstnybxrPpGWmKXU2lAoIE,88
 mteb/models/cache_wrappers/cache_backend_protocol.py,sha256=TR7kD7KbN1J4piszIecpegtLZYGy7sRHZt3SDWlImKk,1665
@@ -1448,7 +1448,7 @@ mteb/models/model_implementations/codesage_models.py,sha256=fy0_WYcU4DnYH9DQ1C5F
 mteb/models/model_implementations/cohere_models.py,sha256=H7Mjn57kmeBhIaJx6riaCEEgZS01YBWBBGPagEU87ZQ,13690
 mteb/models/model_implementations/cohere_v.py,sha256=K6VEw1NkyM2PuMd18kHE6aqPrcByYSwEmAKjvLods_w,15760
 mteb/models/model_implementations/colpali_models.py,sha256=uVmK3jXO-GDn-7i6cJFWdc0u0-MU3INHHL1rXUPhBec,8944
-mteb/models/model_implementations/colqwen_models.py,sha256=G_yNAQRI9Evf1x2qsLHOZX687iYy4WIsnlR0U-jHJzo,6804
+mteb/models/model_implementations/colqwen_models.py,sha256=Y6IBhYKbxjkC3AePa0l37-F50xcX1mtPCXpgW597HyA,8754
 mteb/models/model_implementations/colsmol_models.py,sha256=O2M7Ksydh94M_Iax4KytHb-wOL18N0BIYLKSsLF8BFs,2967
 mteb/models/model_implementations/conan_models.py,sha256=G-s7xo9VtNX-f7lWKtYVGHHiMMN0Xp44PlNIp7E0LAo,6502
 mteb/models/model_implementations/dino_models.py,sha256=QFgaFHR5YKrylqJGSljXCBn2W7qHhmF6KdXkvHrQNEI,16380
@@ -1471,7 +1471,7 @@ mteb/models/model_implementations/inf_models.py,sha256=lvXUFhAYDltq2_Xa9MHcwfhh1
 mteb/models/model_implementations/jasper_models.py,sha256=KzjVnQ1HwaVO9Z7kk1ZkjFrhvlKupeWCmkSljnZv-IM,4071
 mteb/models/model_implementations/jina_clip.py,sha256=CfiIxbhKspjQajNtObCfGPHOWPk6uLn4cuwydQHFTMo,5118
 mteb/models/model_implementations/jina_models.py,sha256=QWoesiTygdFTLcdGpdx26wOUI1AXRz3jLmxGHJ0WMNE,29919
-mteb/models/model_implementations/kalm_models.py,sha256=ruNzMk3MRLW0up91aXwPIMqorRyujDF291B78D_m7bM,52827
+mteb/models/model_implementations/kalm_models.py,sha256=FmW7Z5Qs6WYBLuKvql3u4IJW36kj4k-Ypah8qTBEBkg,59837
 mteb/models/model_implementations/lens_models.py,sha256=fC7_NB1F8vBAlXD0p0-hALf6eZTPFJwpz57dy71OlwI,1696
 mteb/models/model_implementations/lgai_embedding_models.py,sha256=S83pbfkMH3YUNl4skusgbK-Rn-uLuScQVxgXwegR_N4,2333
 mteb/models/model_implementations/linq_models.py,sha256=rnW27MybLMQ2Y3OxDyBTMSIsx_hXC0DlMD4kFv7NJV0,1918
@@ -1505,6 +1505,7 @@ mteb/models/model_implementations/qtack_models.py,sha256=biZLH5E3UWIcMZXIZNGgBZF
 mteb/models/model_implementations/qwen3_models.py,sha256=IpPJC_y-Kt_1Bq6nT2lX3-lMl-hl0BV6VhAL8Czfyfg,5133
 mteb/models/model_implementations/qzhou_models.py,sha256=90tsGftdCX90Gvu63bxtMyzczGi_u9bsUlg4WAVeH9Q,3536
 mteb/models/model_implementations/random_baseline.py,sha256=NH-epZ73BYvV6qYgu09BsoiFxBWypUNLdJIXRp5QPhM,8030
+mteb/models/model_implementations/rasgaard_models.py,sha256=fHMXP4S17fZQ93n2KkUsAoxyGmVyCNexz6ek6c4zO0E,1021
 mteb/models/model_implementations/reasonir_model.py,sha256=wSCcJpUgZ0pG2g3vTEzYNmPlPG_CVn_rR0ENVCines0,2218
 mteb/models/model_implementations/repllama_models.py,sha256=89HoqEpzkNysHeuf_-YhU8WETamHTogSRztGIRo6G1s,7321
 mteb/models/model_implementations/rerankers_custom.py,sha256=ro73A9-hHudy3_qIMrhP-ja-3Xqu78r_aORm856zHQc,10651
@@ -1521,6 +1522,7 @@ mteb/models/model_implementations/shuu_model.py,sha256=KkcuVYjIzoha3Fvxh8ppqHQ9B
 mteb/models/model_implementations/siglip_models.py,sha256=tvi8QB2ayBoeXsxwHrl5RFlkknvE6FM9N06zSBWGQD0,12602
 mteb/models/model_implementations/sonar_models.py,sha256=Nc6kAJRWSrxA57DPRrgOPHqS1dNhz2vsE_1ZA2JtigQ,4784
 mteb/models/model_implementations/stella_models.py,sha256=NL3tk-rnuBdznsQ-nmelqun4tFO2xKoNPPOOVKqnPGU,8062
+mteb/models/model_implementations/tarka_models.py,sha256=D3b8lPMuSZTKBCDqL5ozI_5lf5og0qKJQD4HBhXyi7o,1601
 mteb/models/model_implementations/text2vec_models.py,sha256=zaHWRc2W0RYZAOetinqRzug9UGW0HmY5U-jYsLXA8wo,4160
 mteb/models/model_implementations/ua_sentence_models.py,sha256=fcvXR4-Rrt-UDTlDkh2ZAO1gO_ufCOHiT6EhoeKiHx8,1224
 mteb/models/model_implementations/uae_models.py,sha256=KZxH5a3t-sfh33xUBkLizEuyFAyPlGfnRsn-S7mjq74,3112
@@ -2536,9 +2538,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=CRAUc5IvqI3_9SyXDwv-PWLCXwXdZem9RePeYESRtuw,996
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=YwJsxTf1eaCI_RE-J37a-gK5wDeGAsmkeZKoZCFihSo,3755
-mteb-2.1.4.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.1.4.dist-info/METADATA,sha256=3HJ0LwqN5B9zLnqA4j5PsmVknk75iAHG-qwtZqJdKW0,13573
-mteb-2.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.1.4.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.1.4.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.1.4.dist-info/RECORD,,
+mteb-2.1.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.1.6.dist-info/METADATA,sha256=hB6t2OTnS5La-oqlUBIi67CI21acqqhYkdVSnXLriGU,13573
+mteb-2.1.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.1.6.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.1.6.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.1.6.dist-info/RECORD,,