mteb 2.0.4__py3-none-any.whl → 2.1.0__py3-none-any.whl

mteb/descriptive_stats/Retrieval/MSMARCO-Fa.json

@@ -1,31 +1,30 @@
 {
     "dev": {
         "num_samples": 8848803,
-        "number_of_characters": 2707180637,
-        "num_documents": 8841823,
-        "min_document_length": 0,
-        "average_document_length": 306.1560844409575,
-        "max_document_length": 1617,
-        "unique_documents": 8841823,
-        "num_queries": 6980,
-        "min_query_length": 7,
-        "average_query_length": 29.044126074498568,
-        "max_query_length": 158,
-        "unique_queries": 6980,
-        "none_queries": 0,
-        "num_relevant_docs": 7437,
-        "min_relevant_docs_per_query": 1,
-        "average_relevant_docs_per_query": 1.0654727793696275,
-        "max_relevant_docs_per_query": 4,
-        "unique_relevant_docs": 7433,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
+        "number_of_characters": 2707180622,
+        "documents_text_statistics": {
+            "total_text_length": 2706977894,
+            "min_text_length": 0,
+            "average_text_length": 306.15608274447476,
+            "max_text_length": 1617,
+            "unique_texts": 8827413
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 202728,
+            "min_text_length": 7,
+            "average_text_length": 29.044126074498568,
+            "max_text_length": 158,
+            "unique_texts": 6978
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 7437,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0654727793696275,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 7433
+        },
+        "top_ranked_statistics": null
     }
 }

mteb/descriptive_stats/Retrieval/MSMARCO.json

@@ -1,89 +1,30 @@
 {
-    "train": {
-        "num_samples": 9344762,
-        "number_of_characters": 2994608051,
-        "num_documents": 8841823,
-        "min_document_length": 4,
-        "average_document_length": 336.79716603691344,
-        "max_document_length": 1670,
-        "unique_documents": 8841823,
-        "num_queries": 502939,
-        "min_query_length": 5,
-        "average_query_length": 33.21898281898998,
-        "max_query_length": 215,
-        "unique_queries": 502939,
-        "none_queries": 0,
-        "num_relevant_docs": 532751,
-        "min_relevant_docs_per_query": 1,
-        "average_relevant_docs_per_query": 1.0592755781516248,
-        "max_relevant_docs_per_query": 7,
-        "unique_relevant_docs": 516472,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
-    },
     "dev": {
         "num_samples": 8848803,
-        "number_of_characters": 2978133099,
-        "num_documents": 8841823,
-        "min_document_length": 4,
-        "average_document_length": 336.79716603691344,
-        "max_document_length": 1670,
-        "unique_documents": 8841823,
-        "num_queries": 6980,
-        "min_query_length": 9,
-        "average_query_length": 33.2621776504298,
-        "max_query_length": 186,
-        "unique_queries": 6980,
-        "none_queries": 0,
-        "num_relevant_docs": 7437,
-        "min_relevant_docs_per_query": 1,
-        "average_relevant_docs_per_query": 1.0654727793696275,
-        "max_relevant_docs_per_query": 4,
-        "unique_relevant_docs": 7433,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
-    },
-    "test": {
-        "num_samples": 8841866,
-        "number_of_characters": 2977902337,
-        "num_documents": 8841823,
-        "min_document_length": 4,
-        "average_document_length": 336.79716603691344,
-        "max_document_length": 1670,
-        "unique_documents": 8841823,
-        "num_queries": 43,
-        "min_query_length": 16,
-        "average_query_length": 32.74418604651163,
-        "max_query_length": 55,
-        "unique_queries": 43,
-        "none_queries": 0,
-        "num_relevant_docs": 9260,
-        "min_relevant_docs_per_query": 132,
-        "average_relevant_docs_per_query": 95.3953488372093,
-        "max_relevant_docs_per_query": 582,
-        "unique_relevant_docs": 9139,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
+        "number_of_characters": 2969291276,
+        "documents_text_statistics": {
+            "total_text_length": 2969059106,
+            "min_text_length": 3,
+            "average_text_length": 335.79716603691344,
+            "max_text_length": 1669,
+            "unique_texts": 8841661
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 232170,
+            "min_text_length": 9,
+            "average_text_length": 33.2621776504298,
+            "max_text_length": 186,
+            "unique_texts": 6980
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 7437,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0654727793696275,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 7433
+        },
+        "top_ranked_statistics": null
     }
 }

mteb/descriptive_stats/Retrieval/OpenTenderRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 138633,
+        "number_of_characters": 59639635,
+        "documents_text_statistics": {
+            "total_text_length": 59576581,
+            "min_text_length": 2,
+            "average_text_length": 432.86552643624714,
+            "max_text_length": 16782,
+            "unique_texts": 122413
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 63054,
+            "min_text_length": 9,
+            "average_text_length": 63.054,
+            "max_text_length": 286,
+            "unique_texts": 992
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/Touche2020.json

@@ -1,31 +1,30 @@
 {
     "test": {
         "num_samples": 382594,
-        "number_of_characters": 658107591,
-        "num_documents": 382545,
-        "min_document_length": 3,
-        "average_document_length": 1720.3347658445412,
-        "max_document_length": 106072,
-        "unique_documents": 382545,
-        "num_queries": 49,
-        "min_query_length": 16,
-        "average_query_length": 43.42857142857143,
-        "max_query_length": 83,
-        "unique_queries": 49,
-        "none_queries": 0,
-        "num_relevant_docs": 2214,
-        "min_relevant_docs_per_query": 40,
-        "average_relevant_docs_per_query": 19.020408163265305,
-        "max_relevant_docs_per_query": 52,
-        "unique_relevant_docs": 2099,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
+        "number_of_characters": 658104319,
+        "documents_text_statistics": {
+            "total_text_length": 658102191,
+            "min_text_length": 3,
+            "average_text_length": 1720.326212602439,
+            "max_text_length": 106072,
+            "unique_texts": 379559
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 2128,
+            "min_text_length": 16,
+            "average_text_length": 43.42857142857143,
+            "max_text_length": 83,
+            "unique_texts": 49
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 932,
+            "min_relevant_docs_per_query": 40,
+            "average_relevant_docs_per_query": 19.020408163265305,
+            "max_relevant_docs_per_query": 52,
+            "unique_relevant_docs": 2099
+        },
+        "top_ranked_statistics": null
     }
 }

mteb/descriptive_stats/Retrieval/VABBRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 10318,
+        "number_of_characters": 7839416,
+        "documents_text_statistics": {
+            "total_text_length": 7765564,
+            "min_text_length": 9,
+            "average_text_length": 833.393861343636,
+            "max_text_length": 35146,
+            "unique_texts": 9123
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 73852,
+            "min_text_length": 7,
+            "average_text_length": 73.852,
+            "max_text_length": 258,
+            "unique_texts": 999
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/bBSARDNLRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 22637,
+        "number_of_characters": 21218611,
+        "documents_text_statistics": {
+            "total_text_length": 21197901,
+            "min_text_length": 7,
+            "average_text_length": 945.7015837608744,
+            "max_text_length": 37834,
+            "unique_texts": 22415
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 20710,
+            "min_text_length": 22,
+            "average_text_length": 93.28828828828829,
+            "max_text_length": 250,
+            "unique_texts": 222
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1059,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 4.77027027027027,
+            "max_relevant_docs_per_query": 57,
+            "unique_relevant_docs": 491
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/STS/SICK-NL-STS.json

@@ -0,0 +1,28 @@
+{
+    "test": {
+        "num_samples": 4902,
+        "number_of_characters": 463327,
+        "unique_pairs": 4902,
+        "text1_statistics": {
+            "total_text_length": 233941,
+            "min_text_length": 10,
+            "average_text_length": 47.72358221134231,
+            "max_text_length": 158,
+            "unique_texts": 3378
+        },
+        "text2_statistics": {
+            "total_text_length": 229386,
+            "min_text_length": 10,
+            "average_text_length": 46.79436964504284,
+            "max_text_length": 158,
+            "unique_texts": 3327
+        },
+        "image1_statistics": null,
+        "image2_statistics": null,
+        "label_statistics": {
+            "min_score": 1.0,
+            "avg_score": 3.528012039368932,
+            "max_score": 5.0
+        }
+    }
+}

mteb/descriptive_stats/Summarization/SummEval.json

@@ -1,55 +1,32 @@
 {
     "test": {
         "num_samples": 100,
-        "number_of_characters": 212735,
-        "min_text_length": 626,
-        "avg_text_length": 2100.35,
-        "max_text_length": 3153,
-        "unique_texts": 100,
-        "min_human_summaries_length": 11,
-        "avg_human_summaries_length": 11.0,
-        "max_human_summaries_length": 11,
-        "unique_human_summaries": 1100,
-        "min_machine_summaries_length": 16,
-        "avg_machine_summaries_length": 16.0,
-        "max_machine_summaries_length": 16,
-        "unique_machine_summaries": 1548,
-        "min_relevance": [
-            1.0,
-            1.3333333333333333,
-            3.6666666666666665,
-            2.3333333333333335,
-            3.6666666666666665,
-            3.0,
-            4.333333333333333,
-            4.0,
-            2.6666666666666665,
-            4.0,
-            2.0,
-            4.666666666666667,
-            4.333333333333333,
-            1.0,
-            2.0,
-            1.0
-        ],
-        "avg_relevance": 3.7770833333333336,
-        "max_relevance": [
-            5.0,
-            4.666666666666667,
-            4.333333333333333,
-            2.6666666666666665,
-            4.666666666666667,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            4.0,
-            4.333333333333333,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            2.3333333333333335,
-            4.666666666666667,
-            4.666666666666667
-        ]
+        "number_of_characters": 1007527,
+        "text_statistics": {
+            "total_text_length": 210035,
+            "min_text_length": 626,
+            "average_text_length": 2100.35,
+            "max_text_length": 3153,
+            "unique_texts": 100
+        },
+        "human_summaries_statistics": {
+            "total_text_length": 248982,
+            "min_text_length": 86,
+            "average_text_length": 226.34727272727272,
+            "max_text_length": 717,
+            "unique_texts": 1100
+        },
+        "machine_summaries_statistics": {
+            "total_text_length": 548510,
+            "min_text_length": 35,
+            "average_text_length": 342.81875,
+            "max_text_length": 718,
+            "unique_texts": 1548
+        },
+        "score_statistics": {
+            "min_score": 1.0,
+            "avg_score": 3.777083333333336,
+            "max_score": 5.0
+        }
     }
 }

mteb/descriptive_stats/Summarization/SummEvalFr.json

@@ -1,55 +1,32 @@
 {
     "test": {
         "num_samples": 100,
-        "number_of_characters": 242873,
-        "min_text_length": 668,
-        "avg_text_length": 2401.73,
-        "max_text_length": 3699,
-        "unique_texts": 100,
-        "min_human_summaries_length": 11,
-        "avg_human_summaries_length": 11.0,
-        "max_human_summaries_length": 11,
-        "unique_human_summaries": 1100,
-        "min_machine_summaries_length": 16,
-        "avg_machine_summaries_length": 16.0,
-        "max_machine_summaries_length": 16,
-        "unique_machine_summaries": 1540,
-        "min_relevance": [
-            1.0,
-            1.333333333333333,
-            3.666666666666666,
-            2.333333333333333,
-            3.666666666666666,
-            3.0,
-            4.333333333333333,
-            4.0,
-            2.666666666666666,
-            4.0,
-            2.0,
-            4.666666666666667,
-            4.333333333333333,
-            1.0,
-            2.0,
-            1.0
-        ],
-        "avg_relevance": 3.7770833333333336,
-        "max_relevance": [
-            5.0,
-            4.666666666666667,
-            4.333333333333333,
-            2.666666666666666,
-            4.666666666666667,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            4.0,
-            4.333333333333333,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            2.333333333333333,
-            4.666666666666667,
-            4.666666666666667
-        ]
+        "number_of_characters": 1139767,
+        "text_statistics": {
+            "total_text_length": 240173,
+            "min_text_length": 668,
+            "average_text_length": 2401.73,
+            "max_text_length": 3699,
+            "unique_texts": 100
+        },
+        "human_summaries_statistics": {
+            "total_text_length": 284479,
+            "min_text_length": 76,
+            "average_text_length": 258.61727272727273,
+            "max_text_length": 815,
+            "unique_texts": 1100
+        },
+        "machine_summaries_statistics": {
+            "total_text_length": 615115,
+            "min_text_length": 0,
+            "average_text_length": 384.446875,
+            "max_text_length": 1079,
+            "unique_texts": 1540
+        },
+        "score_statistics": {
+            "min_score": 1.0,
+            "avg_score": 3.777083333333336,
+            "max_score": 5.0
+        }
     }
 }

mteb/models/model_implementations/kalm_models.py

@@ -766,3 +766,32 @@ HIT_TMG__KaLM_embedding_multilingual_mini_instruct_v2 = ModelMeta(
     superseded_by=None,
     citation=KALM_EMBEDDING_CITATION,
 )
+
+KaLM_Embedding_KaLM_embedding_multilingual_mini_instruct_v2_5 = ModelMeta(
+    loader=InstructSentenceTransformerModel,
+    loader_kwargs=dict(
+        instruction_template=KaLM_INSTRUCTION,
+        max_seq_length=512,
+        apply_instruction_to_passages=False,
+        prompts_dict=KaLM_v2_task_prompts,
+    ),
+    name="KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5",
+    revision="6a4cfc1084cb459ebd4729b53a8656a61448c720",
+    release_date="2025-09-30",
+    languages=["eng-Latn", "zho-Hans"],
+    n_parameters=494032768,
+    memory_usage_mb=1885,
+    max_tokens=512,
+    embed_dim=896,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/KaLM-Embedding/KaLM-embedding-finetuning-data",
+    framework=["PyTorch", "Sentence Transformers"],
+    reference="https://huggingface.co/KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5",
+    similarity_fn_name="cosine",
+    use_instructions=True,
+    training_datasets=kalm_v2_training_data,
+    adapted_from="HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v2",
+    superseded_by=None,
+)

mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py

@@ -21,7 +21,7 @@ class BUCCBitextMining(AbsTaskBitextMining):
         name="BUCC",
         dataset={
             "path": "mteb/BUCC",
-            "revision": "39f20d5ac4a82e59dbcecaabdd599b82cbefa666",
+            "revision": "414572247440f0ccacf7eb0bb70a31533a0e5443",
         },
         description="BUCC bitext mining dataset",
         reference="https://comparable.limsi.fr/bucc2018/bucc2018-task.html",

mteb/tasks/classification/eng/financial_phrasebank_classification.py

@@ -36,9 +36,6 @@ class FinancialPhrasebankClassification(AbsTaskClassification):
         superseded_by="FinancialPhrasebankClassification.v2",
     )
 
-    def dataset_transform(self):
-        self.dataset = self.dataset.rename_column("sentence", "text")
-
 
 class FinancialPhrasebankClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(

mteb/tasks/classification/kor/kor_hate_classification.py

@@ -44,18 +44,6 @@ class KorHateClassification(AbsTaskClassification):
         superseded_by="KorHateClassification.v2",
     )
 
-    def dataset_transform(self):
-        keep_cols = ["comments", "hate"]
-        rename_dict = dict(zip(keep_cols, ["text", "label"]))
-        remove_cols = [
-            col for col in self.dataset["test"].column_names if col not in keep_cols
-        ]
-        self.dataset = self.dataset.rename_columns(rename_dict)
-        self.dataset = self.dataset.remove_columns(remove_cols)
-        self.dataset = self.stratified_subsampling(
-            self.dataset, seed=self.seed, splits=["train"]
-        )
-
 
 class KorHateClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(

mteb/tasks/classification/nld/__init__.py

@@ -2,8 +2,24 @@ from .dutch_book_review_sentiment_classification import (
     DutchBookReviewSentimentClassification,
     DutchBookReviewSentimentClassificationV2,
 )
+from .dutch_cola_classification import DutchColaClassification
+from .dutch_government_bias_classification import DutchGovernmentBiasClassification
+from .dutch_news_articles_classification import DutchNewsArticlesClassification
+from .dutch_sarcastic_headlines_classification import (
+    DutchSarcasticHeadlinesClassification,
+)
+from .iconclass_classification import IconclassClassification
+from .open_tender_classification import OpenTenderClassification
+from .vaccin_chat_nl_classification import VaccinChatNLClassification
 
 __all__ = [
     "DutchBookReviewSentimentClassification",
     "DutchBookReviewSentimentClassificationV2",
+    "DutchColaClassification",
+    "DutchGovernmentBiasClassification",
+    "DutchNewsArticlesClassification",
+    "DutchSarcasticHeadlinesClassification",
+    "IconclassClassification",
+    "OpenTenderClassification",
+    "VaccinChatNLClassification",
 ]