mteb 2.5.2__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/models/model_implementations/jasper_models.py

@@ -298,7 +298,7 @@ jasper_en_v1 = ModelMeta(
     license="apache-2.0",
     reference="https://huggingface.co/infgrad/jasper_en_vision_language_v1",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=True,
     adapted_from=None,
     superseded_by=None,
@@ -345,7 +345,7 @@ Jasper_Token_Compression_600M = ModelMeta(
     max_tokens=32768,
     reference="https://huggingface.co/infgrad/Jasper-Token-Compression-600M",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=True,
     public_training_code="https://github.com/DunZhang/Jasper-Token-Compression-Training",
     # public_training_data: unsupervised data for distillation

mteb/models/model_implementations/jina_clip.py

@@ -7,6 +7,7 @@ from tqdm.auto import tqdm
 from mteb._requires_package import requires_image_dependencies
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
+from mteb.models.model_implementations.colpali_models import COLPALI_TRAINING_DATA
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.types import Array, BatchedInput, PromptType
 
@@ -120,8 +121,17 @@ class JinaCLIPModel(AbsEncoder):
         raise ValueError
 
 
+_JINA_CLIP_TRAIN_DATASETS_V1 = {
+    # LAION400M
+    # ShareGPT4V
+    "MSMARCO",
+    "NQ",
+    "HotpotQA",
+    # Natural Language Inference (NLI) dataset (Bowman et al., 2015)
+}
+
 jina_clip_v1 = ModelMeta(
-    loader=JinaCLIPModel,  # type: ignore
+    loader=JinaCLIPModel,
     name="jinaai/jina-clip-v1",
     model_type=["dense"],
     languages=["eng-Latn"],
@@ -136,17 +146,45 @@ jina_clip_v1 = ModelMeta(
     open_weights=True,
     public_training_code=None,
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "ONNX", "safetensors"],
     reference="https://huggingface.co/jinaai/jina-clip-v1",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,
-    training_datasets={
-        # LAION400M
-        # ShareGPT4V
-        "MSMARCO",
-        # NQ
-        # HotpotQA
-        # Natural Language Inference (NLI) dataset (Bowman et al., 2015)
-    },
+    training_datasets=_JINA_CLIP_TRAIN_DATASETS_V1,
     citation=JINA_CLIP_CITATION,
+    superseded_by="jinaai/jina-clip-v2",
+)
+
+jina_clip_v2 = ModelMeta(
+    loader=JinaCLIPModel,
+    name="jinaai/jina-clip-v2",
+    revision="344d954da76eb8ad47a7aaff42d012e30c15b8fe",
+    release_date="2024-10-09",
+    languages=["eng-Latn"],
+    n_parameters=865278477,
+    memory_usage_mb=1650.0,
+    max_tokens=8192,
+    embed_dim=1024,
+    license="cc-by-nc-4.0",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data=None,
+    framework=["PyTorch", "Sentence Transformers"],
+    reference="https://huggingface.co/jinaai/jina-clip-v2",
+    similarity_fn_name=ScoringFunction.COSINE,
+    use_instructions=False,
+    training_datasets=_JINA_CLIP_TRAIN_DATASETS_V1 | COLPALI_TRAINING_DATA,
+    modalities=["text", "image"],
+    model_type=["dense"],
+    citation="""
+@misc{koukounas2024jinaclipv2multilingualmultimodalembeddings,
+      title={jina-clip-v2: Multilingual Multimodal Embeddings for Text and Images},
+      author={Andreas Koukounas and Georgios Mastrapas and Bo Wang and Mohammad Kalim Akram and Sedigheh Eslami and Michael Günther and Isabelle Mohr and Saba Sturua and Scott Martens and Nan Wang and Han Xiao},
+      year={2024},
+      eprint={2412.08802},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2412.08802},
+}
+""",
 )

mteb/models/model_implementations/jina_models.py

@@ -257,6 +257,7 @@ class JinaRerankerV3Wrapper(CrossEncoderWrapper):
         self,
         model: CrossEncoder | str,
         revision: str | None = None,
+        device: str | None = None,
         trust_remote_code: bool = True,
         **kwargs: Any,
     ) -> None:
@@ -267,10 +268,7 @@ class JinaRerankerV3Wrapper(CrossEncoderWrapper):
             model, trust_remote_code=trust_remote_code, dtype="auto"
         )
 
-        device = kwargs.get("device", None)
-        if device is None:
-            device = get_device_name()
-            logger.info(f"Use pytorch device: {device}")
+        device = device or get_device_name()
 
         self.model.to(device)
         self.model.eval()
@@ -320,6 +318,7 @@ class JinaWrapper(SentenceTransformerEncoderWrapper):
         self,
         model: str,
         revision: str,
+        device: str | None = None,
         model_prompts: dict[str, str] | None = None,
         **kwargs,
     ) -> None:
@@ -339,7 +338,9 @@ class JinaWrapper(SentenceTransformerEncoderWrapper):
         )
         import flash_attn  # noqa: F401
 
-        super().__init__(model, revision, model_prompts, **kwargs)
+        super().__init__(
+            model, revision, device=device, model_prompts=model_prompts, **kwargs
+        )
 
     def encode(
         self,
@@ -732,7 +733,7 @@ jina_reranker_v3 = ModelMeta(
     embed_dim=None,
     license="cc-by-nc-4.0",
     similarity_fn_name=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     use_instructions=None,
     reference="https://huggingface.co/jinaai/jina-reranker-v3",
     public_training_code=None,
@@ -775,7 +776,7 @@ jina_embeddings_v4 = ModelMeta(
     embed_dim=2048,
     license="cc-by-nc-4.0",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=True,
     reference="https://huggingface.co/jinaai/jina-embeddings-v4",
     public_training_code=None,
@@ -795,7 +796,7 @@ jina_embeddings_v4 = ModelMeta(
 
 
 jina_embeddings_v3 = ModelMeta(
-    loader=JinaWrapper,  # type: ignore
+    loader=JinaWrapper,
     loader_kwargs=dict(
         trust_remote_code=True,
         model_prompts={
@@ -823,7 +824,13 @@ jina_embeddings_v3 = ModelMeta(
     embed_dim=1024,
     license="cc-by-nc-4.0",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "Transformers",
+        "ONNX",
+        "safetensors",
+    ],
     use_instructions=True,
     reference="https://huggingface.co/jinaai/jina-embeddings-v3",
     public_training_code=None,
@@ -878,7 +885,7 @@ jina_embeddings_v2_base_en = ModelMeta(
     max_tokens=8192,
     reference="https://huggingface.co/jinaai/jina-embeddings-v2-base-en",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     use_instructions=False,
     superseded_by=None,
     adapted_from="jina-bert-base-en-v1",  # pretrained on C4 with Alibi to support longer context.
@@ -942,7 +949,7 @@ jina_embeddings_v2_small_en = ModelMeta(
     max_tokens=8192,
     reference="https://huggingface.co/jinaai/jina-embeddings-v2-small-en",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     use_instructions=False,
     superseded_by=None,
     adapted_from="jina-bert-smalll-en-v1",  # pretrained on C4 with Alibi to support longer context

mteb/models/model_implementations/kblab.py

@@ -16,16 +16,22 @@ sbert_swedish = ModelMeta(
     max_tokens=384,
     reference="https://huggingface.co/KBLab/sentence-bert-swedish-cased",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "safetensors",
+        "GGUF",
+        "Transformers",
+    ],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
     training_datasets=None,
     adapted_from="sentence-transformers/all-mpnet-base-v2",
-    citation="""@misc{rekathati2021introducing,  
-  author = {Rekathati, Faton},  
-  title = {The KBLab Blog: Introducing a Swedish Sentence Transformer},  
-  url = {https://kb-labb.github.io/posts/2021-08-23-a-swedish-sentence-transformer/},  
-  year = {2021}  
+    citation="""@misc{rekathati2021introducing,
+  author = {Rekathati, Faton},
+  title = {The KBLab Blog: Introducing a Swedish Sentence Transformer},
+  url = {https://kb-labb.github.io/posts/2021-08-23-a-swedish-sentence-transformer/},
+  year = {2021}
 }""",
 )

mteb/models/model_implementations/kennethenevoldsen_models.py

@@ -4,7 +4,7 @@ from mteb.models.sentence_transformer_wrapper import (
 )
 
 dfm_enc_large = ModelMeta(
-    loader=sentence_transformers_loader,  # type: ignore
+    loader=sentence_transformers_loader,
     name="KennethEnevoldsen/dfm-sentence-encoder-large",
     model_type=["dense"],
     languages=["dan-Latn"],
@@ -18,7 +18,7 @@ dfm_enc_large = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/KennethEnevoldsen/dfm-sentence-encoder-large",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=False,
     superseded_by=None,
     adapted_from="chcaa/dfm-encoder-large-v1",
@@ -39,7 +39,7 @@ dfm_enc_large = ModelMeta(
 )
 
 dfm_enc_med = ModelMeta(
-    loader=sentence_transformers_loader,  # type: ignore
+    loader=sentence_transformers_loader,
     name="KennethEnevoldsen/dfm-sentence-encoder-medium",
     model_type=["dense"],
     languages=["dan-Latn"],
@@ -53,7 +53,7 @@ dfm_enc_med = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/KennethEnevoldsen/dfm-sentence-encoder-medium",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers"],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,

mteb/models/model_implementations/kfst.py

@@ -16,7 +16,7 @@ xlmr_scandi = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/KFST/XLMRoberta-en-da-sv-nb",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,

mteb/models/model_implementations/kowshik24_models.py

@@ -15,7 +15,7 @@ kowshik24_bangla_embedding_model = ModelMeta(
     max_tokens=128,
     reference="https://huggingface.co/Kowshik24/bangla-sentence-transformer-ft-matryoshka-paraphrase-multilingual-mpnet-base-v2",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code="https://github.com/kowshik24/Bangla-Embedding",
     public_training_data="https://huggingface.co/datasets/sartajekram/BanglaRQA",

mteb/models/model_implementations/lgai_embedding_models.py

@@ -58,7 +58,7 @@ lgai_embedding_en = ModelMeta(
     max_tokens=32768,
     reference="https://huggingface.co/annamodels/LGAI-Embedding-Preview",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,

mteb/models/model_implementations/linq_models.py

@@ -44,7 +44,7 @@ Linq_Embed_Mistral = ModelMeta(
     max_tokens=32768,
     reference="https://huggingface.co/Linq-AI-Research/Linq-Embed-Mistral",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=True,
     public_training_code=None,
     public_training_data=None,

mteb/models/model_implementations/listconranker.py

@@ -125,7 +125,7 @@ listconranker = ModelMeta(
     license="mit",
     max_tokens=512,
     reference="https://huggingface.co/ByteDance/ListConRanker",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Sentence Transformers", "safetensors", "Transformers"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,

mteb/models/model_implementations/llm2clip_models.py

@@ -181,7 +181,7 @@ llm2clip_training_sets = set(
 )
 
 llm2clip_openai_l_14_336 = ModelMeta(
-    loader=llm2clip_loader,  # type: ignore
+    loader=llm2clip_loader,
     name="microsoft/LLM2CLIP-Openai-L-14-336",
     model_type=["dense"],
     languages=["eng-Latn"],
@@ -196,7 +196,7 @@ llm2clip_openai_l_14_336 = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/microsoft/LLM2CLIP",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "safetensors"],
     reference="https://huggingface.co/microsoft/LLM2CLIP-Openai-L-14-336",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,
@@ -206,7 +206,7 @@ llm2clip_openai_l_14_336 = ModelMeta(
 
 # NOTE: https://huggingface.co/microsoft/LLM2CLIP-Openai-L-14-224/discussions/1
 llm2clip_openai_l_14_224 = ModelMeta(
-    loader=llm2clip_loader,  # type: ignore
+    loader=llm2clip_loader,
     name="microsoft/LLM2CLIP-Openai-L-14-224",
     model_type=["dense"],
     languages=["eng-Latn"],
@@ -221,7 +221,7 @@ llm2clip_openai_l_14_224 = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/microsoft/LLM2CLIP",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "safetensors"],
     reference="https://huggingface.co/microsoft/LLM2CLIP-Openai-L-14-224",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,
@@ -230,7 +230,7 @@ llm2clip_openai_l_14_224 = ModelMeta(
 )
 
 llm2clip_openai_b_16 = ModelMeta(
-    loader=llm2clip_loader,  # type: ignore
+    loader=llm2clip_loader,
     name="microsoft/LLM2CLIP-Openai-B-16",
     model_type=["dense"],
     languages=["eng-Latn"],
@@ -245,7 +245,7 @@ llm2clip_openai_b_16 = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/microsoft/LLM2CLIP",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "safetensors"],
     reference="https://huggingface.co/microsoft/LLM2CLIP-Openai-B-16",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,

mteb/models/model_implementations/llm2vec_models.py

@@ -145,7 +145,7 @@ llm2vec_llama3_8b_supervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
     training_datasets=llm2vec_supervised_training_data,
@@ -174,7 +174,7 @@ llm2vec_llama3_8b_unsupervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -203,7 +203,7 @@ llm2vec_mistral7b_supervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -232,7 +232,7 @@ llm2vec_mistral7b_unsupervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -261,7 +261,7 @@ llm2vec_llama2_7b_supervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -290,7 +290,7 @@ llm2vec_llama2_7b_unsupervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
     training_datasets=set(),
@@ -319,7 +319,7 @@ llm2vec_sheared_llama_supervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -348,7 +348,7 @@ llm2vec_sheared_llama_unsupervised = ModelMeta(
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",

mteb/models/model_implementations/mcinext_models.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import time
+import warnings
 from typing import Any
 
 import numpy as np
@@ -246,7 +247,9 @@ class HakimModelWrapper(AbsEncoder):
         task_prompt, task_id = DATASET_TASKS.get(task_name, (None, None))
 
         if not task_prompt:
-            logger.warning(f"Unknown dataset: {task_name}, no preprocessing applied.")
+            msg = f"Unknown dataset: {task_name}, no preprocessing applied."
+            logger.warning(msg)
+            warnings.warn(msg)
             return sample
 
         task_prompt = f"مسئله : {task_prompt}"

mteb/models/model_implementations/mdbr_models.py

@@ -1,5 +1,7 @@
 from mteb.models.model_implementations.arctic_models import arctic_v1_training_datasets
-from mteb.models.model_implementations.mxbai_models import mixedbread_training_data
+from mteb.models.model_implementations.mixedbread_ai_models import (
+    mixedbread_training_data,
+)
 from mteb.models.model_meta import ModelMeta
 from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
 
@@ -35,7 +37,13 @@ mdbr_leaf_ir = ModelMeta(
     release_date="2025-08-27",
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     n_parameters=22_861_056,
     memory_usage_mb=86,
     max_tokens=512,
@@ -63,7 +71,13 @@ mdbr_leaf_mt = ModelMeta(
     release_date="2025-08-27",
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     n_parameters=22_958_592,
     memory_usage_mb=86,
     max_tokens=512,