mteb 2.6.4__py3-none-any.whl → 2.6.6__py3-none-any.whl

mteb/abstasks/classification.py

@@ -98,9 +98,8 @@ class AbsTaskClassification(AbsTask):
             text: str (for text) or PIL.Image (for image). Column name can be changed via `input_column_name` attribute.
             label: int. Column name can be changed via `label_column_name` attribute.
         evaluator_model: The model to use for evaluation. Can be any sklearn compatible model. Default is `LogisticRegression`.
-            Full details of api in [`SklearnModelProtocol`][mteb._evaluators.sklearn_evaluator.SklearnModelProtocol].
-        samples_per_label: Number of samples per label to use for training the evaluator model. Default is 8.
-        n_experiments: Number of experiments to run. Default is 10.
+       samples_per_label: Number of samples per label to use for training the evaluator model. Default is 8.
+       n_experiments: Number of experiments to run. Default is 10.
         train_split: Name of the split to use for training the evaluator model. Default is "train".
         label_column_name: Name of the column containing the labels. Default is "label".
         input_column_name: Name of the column containing the input data. Default is "text".

mteb/abstasks/multilabel_classification.py

@@ -70,10 +70,10 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
         input_column_name: Name of the column containing the input text.
         label_column_name: Name of the column containing the labels.
         samples_per_label: Number of samples to use pr. label. These samples are embedded and a classifier is fit using the labels and samples.
-        evaluator: Classifier to use for evaluation. Must implement the SklearnModelProtocol.
+        evaluator_model: Classifier to use for evaluation. Must implement the SklearnModelProtocol.
     """
 
-    evaluator: SklearnModelProtocol = KNeighborsClassifier(n_neighbors=5)  # type: ignore[assignment]
+    evaluator_model: SklearnModelProtocol = KNeighborsClassifier(n_neighbors=5)
     input_column_name: str = "text"
     label_column_name: str = "label"
 
@@ -169,7 +169,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
             y_train = train_split.select(sample_indices)[self.label_column_name]
             y_train = binarizer.transform(y_train)
             y_pred, current_classifier = _evaluate_classifier(
-                X_train, y_train, X_test, self.evaluator
+                X_train, y_train, X_test, self.evaluator_model
             )
             if prediction_folder:
                 all_predictions.append(y_pred.tolist())

mteb/abstasks/regression.py

@@ -84,7 +84,7 @@ class AbsTaskRegression(AbsTaskClassification):
         n_samples: Number of samples to use for training the regression model. If the dataset has fewer samples than n_samples, all samples are used.
         abstask_prompt: Prompt to use for the task for instruction model if not prompt is provided in TaskMetadata.prompt.
         evaluator_model: The model to use for evaluation. Can be any sklearn compatible model. Default is `LinearRegression`.
-            Full details of api in [`SklearnModelProtocol`][mteb._evaluators.sklearn_evaluator.SklearnModelProtocol].
+
     """
 
     evaluator: type[SklearnEvaluator] = SklearnEvaluator

mteb/abstasks/retrieval.py

@@ -285,7 +285,7 @@ class AbsTaskRetrieval(AbsTask):
         *,
         encode_kwargs: dict[str, Any],
         prediction_folder: Path | None = None,
-        **kwargs,
+        **kwargs: Any,
     ) -> Mapping[HFSubset, ScoresDict]:
         """Evaluate the model on the retrieval task.
 

mteb/abstasks/task_metadata.py

@@ -485,7 +485,6 @@ class TaskMetadata(BaseModel):
         dataset_type = [
             *self._hf_task_type(),
             *self._hf_task_category(),
-            *self._hf_subtypes(),
         ]
         languages = self._hf_languages()
 
@@ -587,10 +586,8 @@ class TaskMetadata(BaseModel):
 
     def _hf_subtypes(self) -> list[str]:
         # to get full list of available task_ids execute
-        # requests.post("https://huggingface.co/api/validate-yaml", json={
-        #   "content": "---\ntask_ids: 'test'\n---",
-        #   "repoType": "dataset"
-        # })
+        # https://huggingface.co/api/datasets-tags-by-type?type=task_ids
+        # ref https://huggingface-openapi.hf.space/#tag/datasets/GET/api/datasets-tags-by-type
         mteb_to_hf_subtype = {
             "Article retrieval": ["document-retrieval"],
             "Conversational retrieval": ["conversational", "utterance-retrieval"],
@@ -612,7 +609,7 @@ class TaskMetadata(BaseModel):
                 "hate-speech-detection",
             ],
             "Thematic clustering": [],
-            "Scientific Reranking": [],
+            "Scientific Reranking": ["text-scoring"],
             "Claim verification": ["fact-checking", "fact-checking-retrieval"],
             "Topic classification": ["topic-classification"],
             "Code retrieval": [],
@@ -620,21 +617,21 @@ class TaskMetadata(BaseModel):
             "Cross-Lingual Semantic Discrimination": [],
             "Textual Entailment": ["natural-language-inference"],
             "Counterfactual Detection": [],
-            "Emotion classification": [],
+            "Emotion classification": ["sentiment-classification"],
             "Reasoning as Retrieval": [],
             "Rendered Texts Understanding": [],
             "Image Text Retrieval": [],
             "Object recognition": [],
             "Scene recognition": [],
             "Caption Pairing": ["image-captioning"],
-            "Emotion recognition": [],
+            "Emotion recognition": ["sentiment-scoring"],
             "Textures recognition": [],
             "Activity recognition": [],
             "Tumor detection": [],
             "Duplicate Detection": [],
             "Rendered semantic textual similarity": [
                 "semantic-similarity-scoring",
-                "rendered semantic textual similarity",
+                "semantic-similarity-classification",
             ],
             "Intent classification": [
                 "intent-classification",
@@ -648,10 +645,8 @@ class TaskMetadata(BaseModel):
 
     def _hf_task_type(self) -> list[str]:
         # to get full list of task_types execute:
-        # requests.post("https://huggingface.co/api/validate-yaml", json={
-        #     "content": "---\ntask_categories: ['test']\n---", "repoType": "dataset"
-        # }).json()
-        # or look at https://huggingface.co/tasks
+        # https://huggingface.co/api/datasets-tags-by-type?type=task_categories
+        # ref https://huggingface-openapi.hf.space/#tag/datasets/GET/api/datasets-tags-by-type
         mteb_task_type_to_datasets = {
             # Text
             "BitextMining": ["translation"],
@@ -670,7 +665,7 @@ class TaskMetadata(BaseModel):
             "Any2AnyRetrieval": ["visual-document-retrieval"],
             "Any2AnyMultilingualRetrieval": ["visual-document-retrieval"],
             "VisionCentricQA": ["visual-question-answering"],
-            "ImageClustering": ["image-clustering"],
+            "ImageClustering": ["image-feature-extraction"],
             "ImageClassification": ["image-classification"],
             "ImageMultilabelClassification": ["image-classification"],
             "DocumentUnderstanding": ["visual-document-retrieval"],

mteb/descriptive_stats/Retrieval/NanoClimateFEVER-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 102198,
+        "number_of_characters": 47870352,
+        "documents_text_statistics": {
+            "total_text_length": 47719757,
+            "min_text_length": 9,
+            "average_text_length": 472.01951591046225,
+            "max_text_length": 8686,
+            "unique_texts": 101097
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 150595,
+            "min_text_length": 30,
+            "average_text_length": 136.78019981834694,
+            "max_text_length": 404,
+            "unique_texts": 1099
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 3401,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.089009990917348,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 1123
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoDBPedia-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 132137,
+        "number_of_characters": 43323279,
+        "documents_text_statistics": {
+            "total_text_length": 43311486,
+            "min_text_length": 11,
+            "average_text_length": 328.5778249819823,
+            "max_text_length": 8576,
+            "unique_texts": 131814
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 11793,
+            "min_text_length": 6,
+            "average_text_length": 36.62422360248447,
+            "max_text_length": 100,
+            "unique_texts": 321
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 11620,
+            "min_relevant_docs_per_query": 31,
+            "average_relevant_docs_per_query": 36.08695652173913,
+            "max_relevant_docs_per_query": 1288,
+            "unique_relevant_docs": 32537
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoFEVER-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 106558,
+        "number_of_characters": 48164581,
+        "documents_text_statistics": {
+            "total_text_length": 47886101,
+            "min_text_length": 9,
+            "average_text_length": 472.6783768310499,
+            "max_text_length": 8689,
+            "unique_texts": 101308
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 278480,
+            "min_text_length": 11,
+            "average_text_length": 53.04380952380952,
+            "max_text_length": 196,
+            "unique_texts": 5124
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 6254,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1912380952380952,
+            "max_relevant_docs_per_query": 15,
+            "unique_relevant_docs": 1324
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoHotpotQA-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 117974,
+        "number_of_characters": 35927363,
+        "documents_text_statistics": {
+            "total_text_length": 35335613,
+            "min_text_length": 22,
+            "average_text_length": 316.47705838625023,
+            "max_text_length": 4105,
+            "unique_texts": 111651
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 591750,
+            "min_text_length": 21,
+            "average_text_length": 93.61651637399146,
+            "max_text_length": 280,
+            "unique_texts": 6321
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 12642,
+            "min_relevant_docs_per_query": 2,
+            "average_relevant_docs_per_query": 2.0,
+            "max_relevant_docs_per_query": 2,
+            "unique_relevant_docs": 11874
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoMSMARCO-VN.json

@@ -0,0 +1,30 @@
+{
+    "dev": {
+        "num_samples": 107153,
+        "number_of_characters": 33316879,
+        "documents_text_statistics": {
+            "total_text_length": 33200903,
+            "min_text_length": 2,
+            "average_text_length": 320.30199218561575,
+            "max_text_length": 1712,
+            "unique_texts": 103641
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 115976,
+            "min_text_length": 8,
+            "average_text_length": 33.15494568324757,
+            "max_text_length": 190,
+            "unique_texts": 3498
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 3700,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0577472841623785,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 3698
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoNQ-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 104095,
+        "number_of_characters": 52312680,
+        "documents_text_statistics": {
+            "total_text_length": 52220289,
+            "min_text_length": 10,
+            "average_text_length": 510.98673124908265,
+            "max_text_length": 10245,
+            "unique_texts": 102181
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 92391,
+            "min_text_length": 22,
+            "average_text_length": 48.62684210526316,
+            "max_text_length": 113,
+            "unique_texts": 1900
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 2283,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.201578947368421,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 2283
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/TVPLRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 20561,
+        "number_of_characters": 10832770,
+        "documents_text_statistics": {
+            "total_text_length": 9929303,
+            "min_text_length": 9,
+            "average_text_length": 938.8524016641452,
+            "max_text_length": 6319,
+            "unique_texts": 10573
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 903467,
+            "min_text_length": 13,
+            "average_text_length": 90.48242363545317,
+            "max_text_length": 228,
+            "unique_texts": 9985
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 11158,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1174762143214823,
+            "max_relevant_docs_per_query": 8,
+            "unique_relevant_docs": 10576
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/models/model_implementations/align_models.py

@@ -118,7 +118,7 @@ align_base = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/kakaobrain/coyo-align",
     public_training_data=True,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/kakaobrain/align-base",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,

mteb/models/model_implementations/andersborges.py

@@ -17,7 +17,7 @@ model2vecdk = ModelMeta(
     embed_dim=256,
     license="mit",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["NumPy", "Sentence Transformers"],
+    framework=["NumPy", "Sentence Transformers", "safetensors"],
     reference="https://huggingface.co/andersborges/model2vecdk",
     use_instructions=False,
     adapted_from="https://huggingface.co/jealk/TTC-L2V-supervised-2",
@@ -48,7 +48,7 @@ model2vecdk_stem = ModelMeta(
     embed_dim=256,
     license="mit",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["NumPy", "Sentence Transformers"],
+    framework=["NumPy", "Sentence Transformers", "safetensors"],
     reference="https://huggingface.co/andersborges/model2vecdk",
     use_instructions=False,
     adapted_from="https://huggingface.co/jealk/TTC-L2V-supervised-2",

mteb/models/model_implementations/ara_models.py

@@ -16,7 +16,7 @@ arabic_triplet_matryoshka = ModelMeta(
     max_tokens=768,
     reference="https://huggingface.co/Omartificial-Intelligence-Space/Arabic-Triplet-Matryoshka-V2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=False,
     public_training_code=None,
     adapted_from="aubmindlab/bert-base-arabertv02",

mteb/models/model_implementations/arctic_models.py

@@ -145,7 +145,7 @@ arctic_embed_xs = ModelMeta(
     release_date="2024-07-08",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=22_600_000,
     memory_usage_mb=86,
     max_tokens=512,
@@ -171,7 +171,7 @@ arctic_embed_s = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=32_200_000,
     memory_usage_mb=127,
     max_tokens=512,
@@ -197,7 +197,7 @@ arctic_embed_m = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=109_000_000,
     memory_usage_mb=415,
     max_tokens=512,
@@ -223,7 +223,7 @@ arctic_embed_m_long = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=137_000_000,
     memory_usage_mb=522,
     max_tokens=2048,
@@ -248,7 +248,7 @@ arctic_embed_l = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=335_000_000,
     memory_usage_mb=1274,
     max_tokens=512,
@@ -278,7 +278,7 @@ arctic_embed_m_v1_5 = ModelMeta(
     release_date="2024-07-08",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors", "GGUF"],
     n_parameters=109_000_000,
     memory_usage_mb=415,
     max_tokens=512,
@@ -304,7 +304,7 @@ arctic_embed_m_v2_0 = ModelMeta(
     release_date="2024-12-04",  # initial commit of hf model.
     languages=LANGUAGES_V2_0,
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=305_000_000,
     memory_usage_mb=1165,
     max_tokens=8192,
@@ -329,7 +329,7 @@ arctic_embed_l_v2_0 = ModelMeta(
     release_date="2024-12-04",  # initial commit of hf model.
     languages=LANGUAGES_V2_0,
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=568_000_000,
     memory_usage_mb=2166,
     max_tokens=8192,

mteb/models/model_implementations/b1ade_models.py

@@ -22,7 +22,7 @@ b1ade_embed = ModelMeta(
     max_tokens=4096,
     reference="https://huggingface.co/w601sxs/b1ade-embed",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,