mteb 2.6.3__py3-none-any.whl → 2.6.5__py3-none-any.whl

mteb/abstasks/classification.py

@@ -98,9 +98,8 @@ class AbsTaskClassification(AbsTask):
             text: str (for text) or PIL.Image (for image). Column name can be changed via `input_column_name` attribute.
             label: int. Column name can be changed via `label_column_name` attribute.
         evaluator_model: The model to use for evaluation. Can be any sklearn compatible model. Default is `LogisticRegression`.
-            Full details of api in [`SklearnModelProtocol`][mteb._evaluators.sklearn_evaluator.SklearnModelProtocol].
-        samples_per_label: Number of samples per label to use for training the evaluator model. Default is 8.
-        n_experiments: Number of experiments to run. Default is 10.
+       samples_per_label: Number of samples per label to use for training the evaluator model. Default is 8.
+       n_experiments: Number of experiments to run. Default is 10.
         train_split: Name of the split to use for training the evaluator model. Default is "train".
         label_column_name: Name of the column containing the labels. Default is "label".
         input_column_name: Name of the column containing the input data. Default is "text".

mteb/abstasks/multilabel_classification.py

@@ -70,10 +70,10 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
         input_column_name: Name of the column containing the input text.
         label_column_name: Name of the column containing the labels.
         samples_per_label: Number of samples to use pr. label. These samples are embedded and a classifier is fit using the labels and samples.
-        evaluator: Classifier to use for evaluation. Must implement the SklearnModelProtocol.
+        evaluator_model: Classifier to use for evaluation. Must implement the SklearnModelProtocol.
     """
 
-    evaluator: SklearnModelProtocol = KNeighborsClassifier(n_neighbors=5)  # type: ignore[assignment]
+    evaluator_model: SklearnModelProtocol = KNeighborsClassifier(n_neighbors=5)
     input_column_name: str = "text"
     label_column_name: str = "label"
 
@@ -169,7 +169,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
             y_train = train_split.select(sample_indices)[self.label_column_name]
             y_train = binarizer.transform(y_train)
             y_pred, current_classifier = _evaluate_classifier(
-                X_train, y_train, X_test, self.evaluator
+                X_train, y_train, X_test, self.evaluator_model
             )
             if prediction_folder:
                 all_predictions.append(y_pred.tolist())

mteb/abstasks/regression.py

@@ -84,7 +84,7 @@ class AbsTaskRegression(AbsTaskClassification):
         n_samples: Number of samples to use for training the regression model. If the dataset has fewer samples than n_samples, all samples are used.
         abstask_prompt: Prompt to use for the task for instruction model if not prompt is provided in TaskMetadata.prompt.
         evaluator_model: The model to use for evaluation. Can be any sklearn compatible model. Default is `LinearRegression`.
-            Full details of api in [`SklearnModelProtocol`][mteb._evaluators.sklearn_evaluator.SklearnModelProtocol].
+
     """
 
     evaluator: type[SklearnEvaluator] = SklearnEvaluator

mteb/abstasks/retrieval.py

@@ -285,7 +285,7 @@ class AbsTaskRetrieval(AbsTask):
         *,
         encode_kwargs: dict[str, Any],
         prediction_folder: Path | None = None,
-        **kwargs,
+        **kwargs: Any,
     ) -> Mapping[HFSubset, ScoresDict]:
         """Evaluate the model on the retrieval task.
 

mteb/abstasks/task_metadata.py

@@ -485,7 +485,6 @@ class TaskMetadata(BaseModel):
         dataset_type = [
             *self._hf_task_type(),
             *self._hf_task_category(),
-            *self._hf_subtypes(),
         ]
         languages = self._hf_languages()
 
@@ -587,10 +586,8 @@ class TaskMetadata(BaseModel):
 
     def _hf_subtypes(self) -> list[str]:
         # to get full list of available task_ids execute
-        # requests.post("https://huggingface.co/api/validate-yaml", json={
-        #   "content": "---\ntask_ids: 'test'\n---",
-        #   "repoType": "dataset"
-        # })
+        # https://huggingface.co/api/datasets-tags-by-type?type=task_ids
+        # ref https://huggingface-openapi.hf.space/#tag/datasets/GET/api/datasets-tags-by-type
         mteb_to_hf_subtype = {
             "Article retrieval": ["document-retrieval"],
             "Conversational retrieval": ["conversational", "utterance-retrieval"],
@@ -612,7 +609,7 @@ class TaskMetadata(BaseModel):
                 "hate-speech-detection",
             ],
             "Thematic clustering": [],
-            "Scientific Reranking": [],
+            "Scientific Reranking": ["text-scoring"],
             "Claim verification": ["fact-checking", "fact-checking-retrieval"],
             "Topic classification": ["topic-classification"],
             "Code retrieval": [],
@@ -620,21 +617,21 @@ class TaskMetadata(BaseModel):
             "Cross-Lingual Semantic Discrimination": [],
             "Textual Entailment": ["natural-language-inference"],
             "Counterfactual Detection": [],
-            "Emotion classification": [],
+            "Emotion classification": ["sentiment-classification"],
             "Reasoning as Retrieval": [],
             "Rendered Texts Understanding": [],
             "Image Text Retrieval": [],
             "Object recognition": [],
             "Scene recognition": [],
             "Caption Pairing": ["image-captioning"],
-            "Emotion recognition": [],
+            "Emotion recognition": ["sentiment-scoring"],
             "Textures recognition": [],
             "Activity recognition": [],
             "Tumor detection": [],
             "Duplicate Detection": [],
             "Rendered semantic textual similarity": [
                 "semantic-similarity-scoring",
-                "rendered semantic textual similarity",
+                "semantic-similarity-classification",
             ],
             "Intent classification": [
                 "intent-classification",
@@ -648,10 +645,8 @@ class TaskMetadata(BaseModel):
 
     def _hf_task_type(self) -> list[str]:
         # to get full list of task_types execute:
-        # requests.post("https://huggingface.co/api/validate-yaml", json={
-        #     "content": "---\ntask_categories: ['test']\n---", "repoType": "dataset"
-        # }).json()
-        # or look at https://huggingface.co/tasks
+        # https://huggingface.co/api/datasets-tags-by-type?type=task_categories
+        # ref https://huggingface-openapi.hf.space/#tag/datasets/GET/api/datasets-tags-by-type
         mteb_task_type_to_datasets = {
             # Text
             "BitextMining": ["translation"],
@@ -670,7 +665,7 @@ class TaskMetadata(BaseModel):
             "Any2AnyRetrieval": ["visual-document-retrieval"],
             "Any2AnyMultilingualRetrieval": ["visual-document-retrieval"],
             "VisionCentricQA": ["visual-question-answering"],
-            "ImageClustering": ["image-clustering"],
+            "ImageClustering": ["image-feature-extraction"],
             "ImageClassification": ["image-classification"],
             "ImageMultilabelClassification": ["image-classification"],
             "DocumentUnderstanding": ["visual-document-retrieval"],

mteb/models/model_implementations/align_models.py

@@ -118,7 +118,7 @@ align_base = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/kakaobrain/coyo-align",
     public_training_data=True,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/kakaobrain/align-base",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,

mteb/models/model_implementations/andersborges.py

@@ -17,7 +17,7 @@ model2vecdk = ModelMeta(
     embed_dim=256,
     license="mit",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["NumPy", "Sentence Transformers"],
+    framework=["NumPy", "Sentence Transformers", "safetensors"],
     reference="https://huggingface.co/andersborges/model2vecdk",
     use_instructions=False,
     adapted_from="https://huggingface.co/jealk/TTC-L2V-supervised-2",
@@ -48,7 +48,7 @@ model2vecdk_stem = ModelMeta(
     embed_dim=256,
     license="mit",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["NumPy", "Sentence Transformers"],
+    framework=["NumPy", "Sentence Transformers", "safetensors"],
     reference="https://huggingface.co/andersborges/model2vecdk",
     use_instructions=False,
     adapted_from="https://huggingface.co/jealk/TTC-L2V-supervised-2",

mteb/models/model_implementations/ara_models.py

@@ -16,7 +16,7 @@ arabic_triplet_matryoshka = ModelMeta(
     max_tokens=768,
     reference="https://huggingface.co/Omartificial-Intelligence-Space/Arabic-Triplet-Matryoshka-V2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=False,
     public_training_code=None,
     adapted_from="aubmindlab/bert-base-arabertv02",

mteb/models/model_implementations/arctic_models.py

@@ -145,7 +145,7 @@ arctic_embed_xs = ModelMeta(
     release_date="2024-07-08",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=22_600_000,
     memory_usage_mb=86,
     max_tokens=512,
@@ -171,7 +171,7 @@ arctic_embed_s = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=32_200_000,
     memory_usage_mb=127,
     max_tokens=512,
@@ -197,7 +197,7 @@ arctic_embed_m = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=109_000_000,
     memory_usage_mb=415,
     max_tokens=512,
@@ -223,7 +223,7 @@ arctic_embed_m_long = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=137_000_000,
     memory_usage_mb=522,
     max_tokens=2048,
@@ -248,7 +248,7 @@ arctic_embed_l = ModelMeta(
     release_date="2024-04-12",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=335_000_000,
     memory_usage_mb=1274,
     max_tokens=512,
@@ -278,7 +278,7 @@ arctic_embed_m_v1_5 = ModelMeta(
     release_date="2024-07-08",  # initial commit of hf model.
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors", "GGUF"],
     n_parameters=109_000_000,
     memory_usage_mb=415,
     max_tokens=512,
@@ -304,7 +304,7 @@ arctic_embed_m_v2_0 = ModelMeta(
     release_date="2024-12-04",  # initial commit of hf model.
     languages=LANGUAGES_V2_0,
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=305_000_000,
     memory_usage_mb=1165,
     max_tokens=8192,
@@ -329,7 +329,7 @@ arctic_embed_l_v2_0 = ModelMeta(
     release_date="2024-12-04",  # initial commit of hf model.
     languages=LANGUAGES_V2_0,
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=568_000_000,
     memory_usage_mb=2166,
     max_tokens=8192,

mteb/models/model_implementations/b1ade_models.py

@@ -22,7 +22,7 @@ b1ade_embed = ModelMeta(
     max_tokens=4096,
     reference="https://huggingface.co/w601sxs/b1ade-embed",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,

mteb/models/model_implementations/bge_models.py

@@ -331,7 +331,13 @@ bge_small_en_v1_5 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-small-en-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=True,
     public_training_code=None,
     public_training_data="https://data.baai.ac.cn/details/BAAI-MTP",
@@ -357,7 +363,13 @@ bge_base_en_v1_5 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-base-en-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=True,
     public_training_code=None,  # seemingly released (at least for some models, but the link is broken
     public_training_data="https://data.baai.ac.cn/details/BAAI-MTP",
@@ -383,7 +395,13 @@ bge_large_en_v1_5 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-large-en-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=True,
     citation=BGE_15_CITATION,
     public_training_code=None,  # seemingly released (at least for some models, but the link is broken
@@ -409,7 +427,7 @@ bge_small_zh = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-small-zh",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -436,7 +454,7 @@ bge_base_zh = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-base-zh",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -463,7 +481,7 @@ bge_large_zh = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-large-zh",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -490,7 +508,7 @@ bge_small_en = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-small-en",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data="https://data.baai.ac.cn/details/BAAI-MTP",
@@ -517,7 +535,13 @@ bge_base_en = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-base-en",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "Transformers",
+        "ONNX",
+        "safetensors",
+    ],
     use_instructions=True,
     public_training_code=None,  # seemingly released (at least for some models, but the link is broken
     public_training_data="https://data.baai.ac.cn/details/BAAI-MTP",
@@ -544,7 +568,7 @@ bge_large_en = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-large-en",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,  # seemingly released (at least for some models, but the link is broken
     public_training_data="https://data.baai.ac.cn/details/BAAI-MTP",
@@ -572,7 +596,7 @@ bge_small_zh_v1_5 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-small-zh-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -598,7 +622,7 @@ bge_base_zh_v1_5 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-base-zh-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -624,7 +648,7 @@ bge_large_zh_v1_5 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/BAAI/bge-large-zh-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -647,13 +671,13 @@ bge_m3 = ModelMeta(
     max_tokens=8194,
     reference="https://huggingface.co/BAAI/bge-m3",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX"],
     use_instructions=False,
     public_training_code=None,
     public_training_data="https://huggingface.co/datasets/cfli/bge-full-data",
     training_datasets=bge_m3_training_data,
     citation="""@misc{bge-m3,
-      title={BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation}, 
+      title={BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation},
       author={Jianlv Chen and Shitao Xiao and Peitian Zhang and Kun Luo and Defu Lian and Zheng Liu},
       year={2024},
       eprint={2402.03216},
@@ -743,7 +767,7 @@ bge_multilingual_gemma2 = ModelMeta(
     max_tokens=8192,  # from old C-MTEB leaderboard
     reference="https://huggingface.co/BAAI/bge-multilingual-gemma2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
@@ -754,7 +778,7 @@ bge_multilingual_gemma2 = ModelMeta(
     | bge_full_data
     | bge_m3_training_data,
     citation="""@misc{bge-m3,
-      title={BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation}, 
+      title={BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation},
       author={Jianlv Chen and Shitao Xiao and Peitian Zhang and Kun Luo and Defu Lian and Zheng Liu},
       year={2024},
       eprint={2402.03216},
@@ -764,7 +788,7 @@ bge_multilingual_gemma2 = ModelMeta(
 
 
 @misc{bge_embedding,
-      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
+      title={C-Pack: Packaged Resources To Advance General Chinese Embedding},
       author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff},
       year={2023},
       eprint={2309.07597},
@@ -790,7 +814,7 @@ bge_en_icl = ModelMeta(
     max_tokens=32768,
     reference="https://huggingface.co/BAAI/bge-en-icl",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=False,
     public_training_code="https://github.com/FlagOpen/FlagEmbedding",
     public_training_data="https://huggingface.co/datasets/cfli/bge-full-data",
@@ -824,13 +848,13 @@ bge_m3_unsupervised = ModelMeta(
     max_tokens=8192,
     reference="https://huggingface.co/BAAI/bge-m3-unsupervised",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code="https://github.com/FlagOpen/FlagEmbedding",
     public_training_data="https://huggingface.co/datasets/cfli/bge-full-data",
     training_datasets=bge_m3_training_data,
     citation="""@misc{bge-m3,
-      title={BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation}, 
+      title={BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation},
       author={Jianlv Chen and Shitao Xiao and Peitian Zhang and Kun Luo and Defu Lian and Zheng Liu},
       year={2024},
       eprint={2402.03216},
@@ -854,7 +878,7 @@ manu__bge_m3_custom_fr = ModelMeta(
     open_weights=True,
     public_training_code=None,
     public_training_data=None,
-    framework=["PyTorch", "Sentence Transformers"],
+    framework=["PyTorch", "Sentence Transformers", "safetensors"],
     reference="https://huggingface.co/manu/bge-m3-custom-fr",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=None,

mteb/models/model_implementations/bica_model.py

@@ -15,20 +15,20 @@ bica_base = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/bisectgroup/BiCA-base",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code="https://github.com/NiravBhattLab/BiCA",
     public_training_data="https://huggingface.co/datasets/bisectgroup/hard-negatives-traversal",
     adapted_from="thenlper/gte-base",
     citation="""
 @misc{sinha2025bicaeffectivebiomedicaldense,
-      title={BiCA: Effective Biomedical Dense Retrieval with Citation-Aware Hard Negatives}, 
+      title={BiCA: Effective Biomedical Dense Retrieval with Citation-Aware Hard Negatives},
       author={Aarush Sinha and Pavan Kumar S and Roshan Balaji and Nirav Pravinbhai Bhatt},
       year={2025},
       eprint={2511.08029},
       archivePrefix={arXiv},
       primaryClass={cs.IR},
-      url={https://arxiv.org/abs/2511.08029}, 
+      url={https://arxiv.org/abs/2511.08029},
 }
 """,
     training_datasets=set(),

mteb/models/model_implementations/blip2_models.py

@@ -179,7 +179,7 @@ blip2_opt_2_7b = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/LAVIS/tree/main/projects/blip2",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/Salesforce/blip2-opt-2.7b",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -203,7 +203,7 @@ blip2_opt_6_7b_coco = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/LAVIS/tree/main/projects/blip2",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/Salesforce/blip2-opt-6.7b-coco",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,

mteb/models/model_implementations/blip_models.py

@@ -143,7 +143,7 @@ blip_image_captioning_large = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/Salesforce/blip-image-captioning-large",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -171,7 +171,7 @@ blip_image_captioning_base = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/Salesforce/blip-image-captioning-base",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -200,7 +200,7 @@ blip_vqa_base = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/Salesforce/blip-vqa-base",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -227,7 +227,7 @@ blip_vqa_capfilt_large = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/Salesforce/blip-vqa-capfilt-large",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -254,7 +254,7 @@ blip_itm_base_coco = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/Salesforce/blip-itm-base-coco",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -281,7 +281,7 @@ blip_itm_large_coco = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/Salesforce/blip-itm-large-coco",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -309,7 +309,7 @@ blip_itm_base_flickr = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/Salesforce/blip-itm-base-flickr",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
@@ -337,7 +337,7 @@ blip_itm_large_flickr = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/salesforce/BLIP",
     public_training_data="https://github.com/salesforce/BLIP",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers"],
     reference="https://huggingface.co/Salesforce/blip-itm-large-flickr",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,

mteb/models/model_implementations/bmretriever_models.py

@@ -104,7 +104,7 @@ BMRetriever_410M = ModelMeta(
     license="mit",
     reference="https://huggingface.co/BMRetriever/BMRetriever-410M",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -134,7 +134,7 @@ BMRetriever_1B = ModelMeta(
     license="mit",
     reference="https://huggingface.co/BMRetriever/BMRetriever-1B",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -164,7 +164,7 @@ BMRetriever_2B = ModelMeta(
     license="mit",
     reference="https://huggingface.co/BMRetriever/BMRetriever-2B",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,
@@ -194,7 +194,7 @@ BMRetriever_7B = ModelMeta(
     license="mit",
     reference="https://huggingface.co/BMRetriever/BMRetriever-7B",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,

mteb/models/model_implementations/cadet_models.py

@@ -47,7 +47,7 @@ cadet_embed = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/manveertamber/cadet-embed-base-v1",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=True,
     public_training_code="https://github.com/manveertamber/cadet-dense-retrieval",
     # we provide the code to generate the training data

mteb/models/model_implementations/cde_models.py

@@ -227,7 +227,7 @@ cde_small_v1 = ModelMeta(
     embed_dim=768,
     license="mit",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers"],
+    framework=["Sentence Transformers", "safetensors", "Transformers"],
     reference="https://huggingface.co/jxm/cde-small-v1",
     use_instructions=True,
     adapted_from="nomic-ai/nomic-bert-2048",
@@ -256,7 +256,7 @@ cde_small_v2 = ModelMeta(
     embed_dim=768,
     license="mit",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers"],
+    framework=["Sentence Transformers", "safetensors", "Transformers"],
     reference="https://huggingface.co/jxm/cde-small-v1",
     use_instructions=True,
     adapted_from="answerdotai/ModernBERT-base",