mteb 2.0.4__py3-none-any.whl → 2.0.5__py3-none-any.whl

mteb/descriptive_stats/Retrieval/MSMARCO-Fa.json

@@ -1,31 +1,30 @@
 {
     "dev": {
         "num_samples": 8848803,
-        "number_of_characters": 2707180637,
-        "num_documents": 8841823,
-        "min_document_length": 0,
-        "average_document_length": 306.1560844409575,
-        "max_document_length": 1617,
-        "unique_documents": 8841823,
-        "num_queries": 6980,
-        "min_query_length": 7,
-        "average_query_length": 29.044126074498568,
-        "max_query_length": 158,
-        "unique_queries": 6980,
-        "none_queries": 0,
-        "num_relevant_docs": 7437,
-        "min_relevant_docs_per_query": 1,
-        "average_relevant_docs_per_query": 1.0654727793696275,
-        "max_relevant_docs_per_query": 4,
-        "unique_relevant_docs": 7433,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
+        "number_of_characters": 2707180622,
+        "documents_text_statistics": {
+            "total_text_length": 2706977894,
+            "min_text_length": 0,
+            "average_text_length": 306.15608274447476,
+            "max_text_length": 1617,
+            "unique_texts": 8827413
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 202728,
+            "min_text_length": 7,
+            "average_text_length": 29.044126074498568,
+            "max_text_length": 158,
+            "unique_texts": 6978
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 7437,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0654727793696275,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 7433
+        },
+        "top_ranked_statistics": null
     }
 }

mteb/descriptive_stats/Retrieval/MSMARCO.json

@@ -1,89 +1,30 @@
 {
-    "train": {
-        "num_samples": 9344762,
-        "number_of_characters": 2994608051,
-        "num_documents": 8841823,
-        "min_document_length": 4,
-        "average_document_length": 336.79716603691344,
-        "max_document_length": 1670,
-        "unique_documents": 8841823,
-        "num_queries": 502939,
-        "min_query_length": 5,
-        "average_query_length": 33.21898281898998,
-        "max_query_length": 215,
-        "unique_queries": 502939,
-        "none_queries": 0,
-        "num_relevant_docs": 532751,
-        "min_relevant_docs_per_query": 1,
-        "average_relevant_docs_per_query": 1.0592755781516248,
-        "max_relevant_docs_per_query": 7,
-        "unique_relevant_docs": 516472,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
-    },
     "dev": {
         "num_samples": 8848803,
-        "number_of_characters": 2978133099,
-        "num_documents": 8841823,
-        "min_document_length": 4,
-        "average_document_length": 336.79716603691344,
-        "max_document_length": 1670,
-        "unique_documents": 8841823,
-        "num_queries": 6980,
-        "min_query_length": 9,
-        "average_query_length": 33.2621776504298,
-        "max_query_length": 186,
-        "unique_queries": 6980,
-        "none_queries": 0,
-        "num_relevant_docs": 7437,
-        "min_relevant_docs_per_query": 1,
-        "average_relevant_docs_per_query": 1.0654727793696275,
-        "max_relevant_docs_per_query": 4,
-        "unique_relevant_docs": 7433,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
-    },
-    "test": {
-        "num_samples": 8841866,
-        "number_of_characters": 2977902337,
-        "num_documents": 8841823,
-        "min_document_length": 4,
-        "average_document_length": 336.79716603691344,
-        "max_document_length": 1670,
-        "unique_documents": 8841823,
-        "num_queries": 43,
-        "min_query_length": 16,
-        "average_query_length": 32.74418604651163,
-        "max_query_length": 55,
-        "unique_queries": 43,
-        "none_queries": 0,
-        "num_relevant_docs": 9260,
-        "min_relevant_docs_per_query": 132,
-        "average_relevant_docs_per_query": 95.3953488372093,
-        "max_relevant_docs_per_query": 582,
-        "unique_relevant_docs": 9139,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
+        "number_of_characters": 2969291276,
+        "documents_text_statistics": {
+            "total_text_length": 2969059106,
+            "min_text_length": 3,
+            "average_text_length": 335.79716603691344,
+            "max_text_length": 1669,
+            "unique_texts": 8841661
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 232170,
+            "min_text_length": 9,
+            "average_text_length": 33.2621776504298,
+            "max_text_length": 186,
+            "unique_texts": 6980
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 7437,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0654727793696275,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 7433
+        },
+        "top_ranked_statistics": null
     }
 }

mteb/descriptive_stats/Retrieval/Touche2020.json

@@ -1,31 +1,30 @@
 {
     "test": {
         "num_samples": 382594,
-        "number_of_characters": 658107591,
-        "num_documents": 382545,
-        "min_document_length": 3,
-        "average_document_length": 1720.3347658445412,
-        "max_document_length": 106072,
-        "unique_documents": 382545,
-        "num_queries": 49,
-        "min_query_length": 16,
-        "average_query_length": 43.42857142857143,
-        "max_query_length": 83,
-        "unique_queries": 49,
-        "none_queries": 0,
-        "num_relevant_docs": 2214,
-        "min_relevant_docs_per_query": 40,
-        "average_relevant_docs_per_query": 19.020408163265305,
-        "max_relevant_docs_per_query": 52,
-        "unique_relevant_docs": 2099,
-        "num_instructions": null,
-        "min_instruction_length": null,
-        "average_instruction_length": null,
-        "max_instruction_length": null,
-        "unique_instructions": null,
-        "num_top_ranked": null,
-        "min_top_ranked_per_query": null,
-        "average_top_ranked_per_query": null,
-        "max_top_ranked_per_query": null
+        "number_of_characters": 658104319,
+        "documents_text_statistics": {
+            "total_text_length": 658102191,
+            "min_text_length": 3,
+            "average_text_length": 1720.326212602439,
+            "max_text_length": 106072,
+            "unique_texts": 379559
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 2128,
+            "min_text_length": 16,
+            "average_text_length": 43.42857142857143,
+            "max_text_length": 83,
+            "unique_texts": 49
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 932,
+            "min_relevant_docs_per_query": 40,
+            "average_relevant_docs_per_query": 19.020408163265305,
+            "max_relevant_docs_per_query": 52,
+            "unique_relevant_docs": 2099
+        },
+        "top_ranked_statistics": null
     }
 }

mteb/descriptive_stats/Summarization/SummEval.json

@@ -1,55 +1,32 @@
 {
     "test": {
         "num_samples": 100,
-        "number_of_characters": 212735,
-        "min_text_length": 626,
-        "avg_text_length": 2100.35,
-        "max_text_length": 3153,
-        "unique_texts": 100,
-        "min_human_summaries_length": 11,
-        "avg_human_summaries_length": 11.0,
-        "max_human_summaries_length": 11,
-        "unique_human_summaries": 1100,
-        "min_machine_summaries_length": 16,
-        "avg_machine_summaries_length": 16.0,
-        "max_machine_summaries_length": 16,
-        "unique_machine_summaries": 1548,
-        "min_relevance": [
-            1.0,
-            1.3333333333333333,
-            3.6666666666666665,
-            2.3333333333333335,
-            3.6666666666666665,
-            3.0,
-            4.333333333333333,
-            4.0,
-            2.6666666666666665,
-            4.0,
-            2.0,
-            4.666666666666667,
-            4.333333333333333,
-            1.0,
-            2.0,
-            1.0
-        ],
-        "avg_relevance": 3.7770833333333336,
-        "max_relevance": [
-            5.0,
-            4.666666666666667,
-            4.333333333333333,
-            2.6666666666666665,
-            4.666666666666667,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            4.0,
-            4.333333333333333,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            2.3333333333333335,
-            4.666666666666667,
-            4.666666666666667
-        ]
+        "number_of_characters": 1007527,
+        "text_statistics": {
+            "total_text_length": 210035,
+            "min_text_length": 626,
+            "average_text_length": 2100.35,
+            "max_text_length": 3153,
+            "unique_texts": 100
+        },
+        "human_summaries_statistics": {
+            "total_text_length": 248982,
+            "min_text_length": 86,
+            "average_text_length": 226.34727272727272,
+            "max_text_length": 717,
+            "unique_texts": 1100
+        },
+        "machine_summaries_statistics": {
+            "total_text_length": 548510,
+            "min_text_length": 35,
+            "average_text_length": 342.81875,
+            "max_text_length": 718,
+            "unique_texts": 1548
+        },
+        "score_statistics": {
+            "min_score": 1.0,
+            "avg_score": 3.777083333333336,
+            "max_score": 5.0
+        }
     }
 }

mteb/descriptive_stats/Summarization/SummEvalFr.json

@@ -1,55 +1,32 @@
 {
     "test": {
         "num_samples": 100,
-        "number_of_characters": 242873,
-        "min_text_length": 668,
-        "avg_text_length": 2401.73,
-        "max_text_length": 3699,
-        "unique_texts": 100,
-        "min_human_summaries_length": 11,
-        "avg_human_summaries_length": 11.0,
-        "max_human_summaries_length": 11,
-        "unique_human_summaries": 1100,
-        "min_machine_summaries_length": 16,
-        "avg_machine_summaries_length": 16.0,
-        "max_machine_summaries_length": 16,
-        "unique_machine_summaries": 1540,
-        "min_relevance": [
-            1.0,
-            1.333333333333333,
-            3.666666666666666,
-            2.333333333333333,
-            3.666666666666666,
-            3.0,
-            4.333333333333333,
-            4.0,
-            2.666666666666666,
-            4.0,
-            2.0,
-            4.666666666666667,
-            4.333333333333333,
-            1.0,
-            2.0,
-            1.0
-        ],
-        "avg_relevance": 3.7770833333333336,
-        "max_relevance": [
-            5.0,
-            4.666666666666667,
-            4.333333333333333,
-            2.666666666666666,
-            4.666666666666667,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            4.0,
-            4.333333333333333,
-            4.666666666666667,
-            4.666666666666667,
-            4.333333333333333,
-            2.333333333333333,
-            4.666666666666667,
-            4.666666666666667
-        ]
+        "number_of_characters": 1139767,
+        "text_statistics": {
+            "total_text_length": 240173,
+            "min_text_length": 668,
+            "average_text_length": 2401.73,
+            "max_text_length": 3699,
+            "unique_texts": 100
+        },
+        "human_summaries_statistics": {
+            "total_text_length": 284479,
+            "min_text_length": 76,
+            "average_text_length": 258.61727272727273,
+            "max_text_length": 815,
+            "unique_texts": 1100
+        },
+        "machine_summaries_statistics": {
+            "total_text_length": 615115,
+            "min_text_length": 0,
+            "average_text_length": 384.446875,
+            "max_text_length": 1079,
+            "unique_texts": 1540
+        },
+        "score_statistics": {
+            "min_score": 1.0,
+            "avg_score": 3.777083333333336,
+            "max_score": 5.0
+        }
     }
 }

mteb/models/model_implementations/kalm_models.py

@@ -766,3 +766,32 @@ HIT_TMG__KaLM_embedding_multilingual_mini_instruct_v2 = ModelMeta(
     superseded_by=None,
     citation=KALM_EMBEDDING_CITATION,
 )
+
+KaLM_Embedding_KaLM_embedding_multilingual_mini_instruct_v2_5 = ModelMeta(
+    loader=InstructSentenceTransformerModel,
+    loader_kwargs=dict(
+        instruction_template=KaLM_INSTRUCTION,
+        max_seq_length=512,
+        apply_instruction_to_passages=False,
+        prompts_dict=KaLM_v2_task_prompts,
+    ),
+    name="KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5",
+    revision="6a4cfc1084cb459ebd4729b53a8656a61448c720",
+    release_date="2025-09-30",
+    languages=["eng-Latn", "zho-Hans"],
+    n_parameters=494032768,
+    memory_usage_mb=1885,
+    max_tokens=512,
+    embed_dim=896,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/KaLM-Embedding/KaLM-embedding-finetuning-data",
+    framework=["PyTorch", "Sentence Transformers"],
+    reference="https://huggingface.co/KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5",
+    similarity_fn_name="cosine",
+    use_instructions=True,
+    training_datasets=kalm_v2_training_data,
+    adapted_from="HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v2",
+    superseded_by=None,
+)

mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py

@@ -21,7 +21,7 @@ class BUCCBitextMining(AbsTaskBitextMining):
         name="BUCC",
         dataset={
             "path": "mteb/BUCC",
-            "revision": "39f20d5ac4a82e59dbcecaabdd599b82cbefa666",
+            "revision": "414572247440f0ccacf7eb0bb70a31533a0e5443",
         },
         description="BUCC bitext mining dataset",
         reference="https://comparable.limsi.fr/bucc2018/bucc2018-task.html",

mteb/tasks/classification/eng/financial_phrasebank_classification.py

@@ -36,9 +36,6 @@ class FinancialPhrasebankClassification(AbsTaskClassification):
         superseded_by="FinancialPhrasebankClassification.v2",
     )
 
-    def dataset_transform(self):
-        self.dataset = self.dataset.rename_column("sentence", "text")
-
 
 class FinancialPhrasebankClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(

mteb/tasks/classification/kor/kor_hate_classification.py

@@ -44,18 +44,6 @@ class KorHateClassification(AbsTaskClassification):
         superseded_by="KorHateClassification.v2",
     )
 
-    def dataset_transform(self):
-        keep_cols = ["comments", "hate"]
-        rename_dict = dict(zip(keep_cols, ["text", "label"]))
-        remove_cols = [
-            col for col in self.dataset["test"].column_names if col not in keep_cols
-        ]
-        self.dataset = self.dataset.rename_columns(rename_dict)
-        self.dataset = self.dataset.remove_columns(remove_cols)
-        self.dataset = self.stratified_subsampling(
-            self.dataset, seed=self.seed, splits=["train"]
-        )
-
 
 class KorHateClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(

mteb/tasks/clustering/swe/swedn_clustering.py

@@ -8,14 +8,14 @@ class SwednClustering(AbsTaskClusteringLegacy):
         name="SwednClustering",
         dataset={
             "path": "mteb/SwednClustering",
-            "revision": "7125017ead5797297f46e17b31bf78b56d12c2b2",
+            "revision": "45d2a99c3f1b6ee6189a6bb762ed74b7ef45dd9d",
         },
         description="The SWE-DN corpus is based on 1,963,576 news articles from the Swedish newspaper Dagens Nyheter (DN) during the years 2000--2020. The articles are filtered to resemble the CNN/DailyMail dataset both regarding textual structure. This dataset uses the category labels as clusters.",
         reference="https://spraakbanken.gu.se/en/resources/swedn",
         type="Clustering",
         category="t2c",
         modalities=["text"],
-        eval_splits=["all"],
+        eval_splits=["test"],
         eval_langs=["swe-Latn"],
         main_score="v_measure",
         date=("2000-01-01", "2020-12-31"),  # best guess

mteb/tasks/retrieval/multilingual/vdr_multilingual_retrieval.py

@@ -101,7 +101,7 @@ class VDRMultilingualRetrieval(AbsTaskRetrieval):
             "revision": "9e26ae152f5950ab1a5ff1c58edade3acc894793",
         },
         type="Retrieval",
-        category="it2it",
+        category="t2i",
         modalities=["text", "image"],
         eval_splits=[_EVAL_SPLIT],
         eval_langs=_LANGS,

{mteb-2.0.4.dist-info → mteb-2.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.0.4
+Version: 2.0.5
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>