mteb 2.1.0__py3-none-any.whl → 2.1.1__py3-none-any.whl

mteb/_create_dataloaders.py

@@ -277,6 +277,8 @@ def _custom_collate_fn(batch: list[dict[str, Any]]) -> dict[str, Any]:
             # Leave the images as a list to avoid stacking errors.
             collated[key] = [item[key] for item in batch]
         else:
+            if any(item[key] is None for item in batch):
+                raise ValueError(f"Found None in batch for key '{key}'")
             collated[key] = default_collate([item[key] for item in batch])
     return collated
 

mteb/abstasks/_stratification.py

@@ -134,7 +134,7 @@ def _get_most_desired_combination(samples_with_combination: dict):
 class IterativeStratification(_BaseKFold):
     """Iteratively stratify a multi-label data set into folds
 
-    Construct an interative stratifier that splits the data set into folds trying to maintain balanced representation
+    Construct an iterative stratifier that splits the data set into folds trying to maintain balanced representation
     with respect to order-th label combinations.
     """
 

mteb/abstasks/abstask.py

@@ -459,7 +459,7 @@ class AbsTask(ABC):
         """Filter the languages of the task.
 
         Args:
-            languages: list of languages to filter the task by can be either a 3-letter langauge code (e.g. "eng") or also include the script
+            languages: list of languages to filter the task by can be either a 3-letter language code (e.g. "eng") or also include the script
                 (e.g. "eng-Latn")
             script: A list of scripts to filter the task by. Will be ignored if language code specified the script. If None, all scripts are included.
                 If the language code does not specify the script the intersection of the language and script will be used.
@@ -491,6 +491,11 @@ class AbsTask(ABC):
                 if lang_scripts.contains_languages(langs):
                     subsets_to_keep.append(hf_subset)
 
+        if len(subsets_to_keep) == 0:
+            raise ValueError(
+                f"No subsets were found for {self.metadata.name} with filters: language code {languages}, script {script}, hf subsets {hf_subsets}."
+            )
+
         self.hf_subsets = subsets_to_keep
         return self
 

mteb/abstasks/dataset_card_template.md

@@ -85,7 +85,7 @@ desc_stats = task.metadata.descriptive_stats
 ```
 
 ```json
-{{ descritptive_stats | default("{}", true) }}
+{{ descriptive_stats | default("{}", true) }}
 ```
 
 </details>

mteb/abstasks/retrieval.py

@@ -653,6 +653,8 @@ class AbsTaskRetrieval(AbsTask):
             FileNotFoundError: If the specified path does not exist.
             ValueError: If the loaded top ranked results are not in the expected format.
         """
+        self._top_k = top_k
+
         top_ranked_path = Path(top_ranked_path)
         if top_ranked_path.is_dir():
             top_ranked_path = self._predictions_path(top_ranked_path)
@@ -682,7 +684,6 @@ class AbsTaskRetrieval(AbsTask):
                     top_k_sorted[query_id] = sorted_keys[: self._top_k]
 
                 self.dataset[subset][split]["top_ranked"] = top_k_sorted
-        self._top_k = top_k
         return self
 
 

mteb/abstasks/retrieval_dataset_loaders.py

@@ -176,7 +176,7 @@ class RetrievalDatasetLoader:
                 {
                     "query-id": Value("string"),
                     "corpus-id": Value("string"),
-                    "score": Value("uint16"),
+                    "score": Value("int32"),
                 }
             )
         )

mteb/abstasks/task_metadata.py

@@ -532,7 +532,7 @@ class TaskMetadata(BaseModel):
                 citation=self.bibtex_citation,
                 dataset_description=self.description,
                 dataset_reference=self.reference,
-                descritptive_stats=descriptive_stats,
+                descriptive_stats=descriptive_stats,
                 dataset_task_name=self.name,
                 category=self.category,
                 domains=", ".join(self.domains) if self.domains else None,

mteb/benchmarks/benchmarks/benchmarks.py

@@ -641,7 +641,7 @@ MTEB_KOR = Benchmark(
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/kr.svg",
     tasks=get_tasks(
         languages=["kor"],
-        tasks=[  # @KennethEnevoldsen: We could probably expand this to a more solid benchamrk, but for now I have left it as is.
+        tasks=[  # @KennethEnevoldsen: We could probably expand this to a more solid benchmark, but for now I have left it as is.
             # Classification
             "KLUE-TC",
             # Reranking
@@ -975,8 +975,6 @@ MTEB_INDIC = Benchmark(
                 # Bitext
                 "IN22ConvBitextMining",
                 "IN22GenBitextMining",
-                "IndicGenBenchFloresBitextMining",
-                "LinceMTBitextMining",
                 # clustering
                 "SIB200ClusteringS2S",
                 # classification
@@ -985,7 +983,6 @@ MTEB_INDIC = Benchmark(
                 "HindiDiscourseClassification",
                 "SentimentAnalysisHindi",
                 "MalayalamNewsClassification",
-                "IndicLangClassification",
                 "MTOPIntentClassification",
                 "MultiHateClassification",
                 "TweetSentimentClassification",
@@ -1008,7 +1005,7 @@ MTEB_INDIC = Benchmark(
         # STS
         (get_task("IndicCrosslingualSTS"),)
     ),
-    description="A regional geopolitical text embedding benchmark targetting embedding performance on Indic languages.",
+    description="A regional geopolitical text embedding benchmark targeting embedding performance on Indic languages.",
     reference=None,
     citation=MMTEB_CITATION,
     contacts=["KennethEnevoldsen", "isaac-chung"],
@@ -1016,7 +1013,7 @@ MTEB_INDIC = Benchmark(
 
 
 eu_languages = [
-    # official EU languages (56) - we could include the whole economic area e.g. Norway - additioanlly we could include minority languages (probably a good idea?)
+    # official EU languages (56) - we could include the whole economic area e.g. Norway - additionally we could include minority languages (probably a good idea?)
     # germanic
     "dan",
     "eng",
@@ -1084,7 +1081,6 @@ MTEB_EU = Benchmark(
             "AmazonCounterfactualClassification",
             "MassiveScenarioClassification",
             "MultiHateClassification",
-            "NordicLangClassification",
             "ScalaClassification",
             "SwissJudgementClassification",
             "TweetSentimentClassification",
@@ -1142,7 +1138,7 @@ MTEB_EU = Benchmark(
         languages=eu_languages,
         exclusive_language_filter=True,
     ),
-    description="A regional geopolitical text embedding benchmark targetting embedding performance on European languages.",
+    description="A regional geopolitical text embedding benchmark targeting embedding performance on European languages.",
     reference=None,
     citation=MMTEB_CITATION,
     contacts=["KennethEnevoldsen", "isaac-chung"],
@@ -1858,7 +1854,7 @@ MIEB_ENG = MIEBBenchmark(
     ),
     description="""MIEB(eng) is a comprehensive image embeddings benchmark, spanning 8 task types, covering 125 tasks.
     In addition to image classification (zero shot and linear probing), clustering, retrieval, MIEB includes tasks in compositionality evaluation,
-    document undestanding, visual STS, and CV-centric tasks.""",
+    document understanding, visual STS, and CV-centric tasks.""",
     reference="https://arxiv.org/abs/2504.10471",
     contacts=["gowitheflow-1998", "isaac-chung"],
     citation=r"""
@@ -1892,7 +1888,7 @@ MIEB_MULTILINGUAL = MIEBBenchmark(
     ),
     description="""MIEB(Multilingual) is a comprehensive image embeddings benchmark, spanning 10 task types, covering 130 tasks and a total of 39 languages.
     In addition to image classification (zero shot and linear probing), clustering, retrieval, MIEB includes tasks in compositionality evaluation,
-    document undestanding, visual STS, and CV-centric tasks. This benchmark consists of MIEB(eng) + 3 multilingual retrieval
+    document understanding, visual STS, and CV-centric tasks. This benchmark consists of MIEB(eng) + 3 multilingual retrieval
     datasets + the multilingual parts of VisualSTS-b and VisualSTS-16.""",
     reference="https://arxiv.org/abs/2504.10471",
     contacts=["gowitheflow-1998", "isaac-chung"],
@@ -2113,7 +2109,7 @@ BUILT_MTEB = Benchmark(
             "BuiltBenchReranking",
         ],
     ),
-    description='"Built-Bench" is an ongoing effort aimed at evaluating text embedding models in the context of built asset management, spanning over various dicsiplines such as architeture, engineering, constrcution, and operations management of the built environment.',
+    description='"Built-Bench" is an ongoing effort aimed at evaluating text embedding models in the context of built asset management, spanning over various disciplines such as architecture, engineering, construction, and operations management of the built environment.',
     reference="https://arxiv.org/abs/2411.12056",
     citation=r"""
 @article{shahinmoghadam2024benchmarking,

mteb/benchmarks/get_benchmark.py

@@ -14,7 +14,7 @@ def _build_registry() -> dict[str, Benchmark]:
 
     benchmark_registry = {
         inst.name: inst
-        for nam, inst in benchmark_module.__dict__.items()
+        for _, inst in benchmark_module.__dict__.items()
         if isinstance(inst, Benchmark)
     }
     return benchmark_registry

mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/XFlickr30kCoT2IRetrieval.json

@@ -1,183 +1,273 @@
 {
     "test": {
-        "number_of_characters": 1149877,
         "num_samples": 32000,
-        "num_queries": 16000,
-        "num_documents": 16000,
-        "min_document_length": 0,
-        "average_document_length": 0,
-        "max_document_length": 0,
-        "unique_documents": 0,
-        "num_document_images": 16000,
-        "min_query_length": 12,
-        "average_query_length": 71.8673125,
-        "max_query_length": 385,
-        "unique_queries": 15987,
-        "num_query_images": 0,
-        "min_relevant_docs_per_query": 1,
-        "average_relevant_docs_per_query": 1.0,
-        "max_relevant_docs_per_query": 1,
-        "unique_relevant_docs": 16000,
+        "number_of_characters": 1021877,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 176,
+            "average_image_width": 514.5045,
+            "max_image_width": 640,
+            "min_image_height": 144,
+            "average_image_height": 444.223,
+            "max_image_height": 640,
+            "unique_images": 2000
+        },
+        "queries_text_statistics": {
+            "total_text_length": 1021877,
+            "min_text_length": 4,
+            "average_text_length": 63.8673125,
+            "max_text_length": 377,
+            "unique_texts": 15986
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 16000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 16000
+        },
+        "top_ranked_statistics": null,
         "hf_subset_descriptive_stats": {
             "de": {
-                "number_of_characters": 132154,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 4,
-                "average_query_length": 66.077,
-                "max_query_length": 220,
-                "unique_queries": 1994,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 132154,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 132154,
+                    "min_text_length": 4,
+                    "average_text_length": 66.077,
+                    "max_text_length": 220,
+                    "unique_texts": 1994
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             },
             "en": {
-                "number_of_characters": 153801,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 34,
-                "average_query_length": 76.9005,
-                "max_query_length": 377,
-                "unique_queries": 2000,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 153801,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 153801,
+                    "min_text_length": 34,
+                    "average_text_length": 76.9005,
+                    "max_text_length": 377,
+                    "unique_texts": 2000
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             },
             "es": {
-                "number_of_characters": 160049,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 23,
-                "average_query_length": 80.0245,
-                "max_query_length": 342,
-                "unique_queries": 2000,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 160049,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 160049,
+                    "min_text_length": 23,
+                    "average_text_length": 80.0245,
+                    "max_text_length": 342,
+                    "unique_texts": 2000
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             },
             "id": {
-                "number_of_characters": 167858,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 4,
-                "average_query_length": 83.929,
-                "max_query_length": 211,
-                "unique_queries": 2000,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 167858,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 167858,
+                    "min_text_length": 4,
+                    "average_text_length": 83.929,
+                    "max_text_length": 211,
+                    "unique_texts": 2000
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             },
             "ja": {
-                "number_of_characters": 75480,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 9,
-                "average_query_length": 37.74,
-                "max_query_length": 179,
-                "unique_queries": 2000,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 75480,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 75480,
+                    "min_text_length": 9,
+                    "average_text_length": 37.74,
+                    "max_text_length": 179,
+                    "unique_texts": 2000
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             },
             "ru": {
-                "number_of_characters": 149947,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 10,
-                "average_query_length": 74.9735,
-                "max_query_length": 294,
-                "unique_queries": 1997,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 149947,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 149947,
+                    "min_text_length": 10,
+                    "average_text_length": 74.9735,
+                    "max_text_length": 294,
+                    "unique_texts": 1997
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             },
             "tr": {
-                "number_of_characters": 136134,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 19,
-                "average_query_length": 68.067,
-                "max_query_length": 199,
-                "unique_queries": 1997,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 136134,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 136134,
+                    "min_text_length": 19,
+                    "average_text_length": 68.067,
+                    "max_text_length": 199,
+                    "unique_texts": 1997
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             },
             "zh": {
-                "number_of_characters": 46454,
                 "num_samples": 4000,
-                "num_queries": 2000,
-                "num_documents": 2000,
-                "min_document_length": 0,
-                "average_document_length": 0,
-                "max_document_length": 0,
-                "unique_documents": 0,
-                "num_document_images": 2000,
-                "min_query_length": 10,
-                "average_query_length": 23.227,
-                "max_query_length": 66,
-                "unique_queries": 1999,
-                "num_query_images": 0,
-                "min_relevant_docs_per_query": 1,
-                "average_relevant_docs_per_query": 1.0,
-                "max_relevant_docs_per_query": 1,
-                "unique_relevant_docs": 2000
+                "number_of_characters": 46454,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 176,
+                    "average_image_width": 514.5045,
+                    "max_image_width": 640,
+                    "min_image_height": 144,
+                    "average_image_height": 444.223,
+                    "max_image_height": 640,
+                    "unique_images": 2000
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 46454,
+                    "min_text_length": 10,
+                    "average_text_length": 23.227,
+                    "max_text_length": 66,
+                    "unique_texts": 1999
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 2000,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 2000
+                },
+                "top_ranked_statistics": null
             }
         }
     }