mteb 2.5.2__py3-none-any.whl → 2.7.9__py3-none-any.whl

mteb/models/model_implementations/llm2clip_models.py

@@ -1,15 +1,20 @@
+from __future__ import annotations
+
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
 from mteb._requires_package import requires_image_dependencies, requires_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 LLM2CLIP_CITATION = """@misc{huang2024llm2clippowerfullanguagemodel,
   title={LLM2CLIP: Powerful Language Model Unlock Richer Visual Representation},
@@ -181,7 +186,7 @@ llm2clip_training_sets = set(
 )
 
 llm2clip_openai_l_14_336 = ModelMeta(
-    loader=llm2clip_loader,  # type: ignore
+    loader=llm2clip_loader,
     name="microsoft/LLM2CLIP-Openai-L-14-336",
     model_type=["dense"],
     languages=["eng-Latn"],
@@ -189,6 +194,7 @@ llm2clip_openai_l_14_336 = ModelMeta(
     release_date="2024-11-07",
     modalities=["image", "text"],
     n_parameters=579_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=None,
     embed_dim=1280,
@@ -196,7 +202,7 @@ llm2clip_openai_l_14_336 = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/microsoft/LLM2CLIP",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "safetensors"],
     reference="https://huggingface.co/microsoft/LLM2CLIP-Openai-L-14-336",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,
@@ -206,7 +212,7 @@ llm2clip_openai_l_14_336 = ModelMeta(
 
 # NOTE: https://huggingface.co/microsoft/LLM2CLIP-Openai-L-14-224/discussions/1
 llm2clip_openai_l_14_224 = ModelMeta(
-    loader=llm2clip_loader,  # type: ignore
+    loader=llm2clip_loader,
     name="microsoft/LLM2CLIP-Openai-L-14-224",
     model_type=["dense"],
     languages=["eng-Latn"],
@@ -214,6 +220,7 @@ llm2clip_openai_l_14_224 = ModelMeta(
     release_date="2024-11-07",
     modalities=["image", "text"],
     n_parameters=578_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=None,
     embed_dim=1280,
@@ -221,7 +228,7 @@ llm2clip_openai_l_14_224 = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/microsoft/LLM2CLIP",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "safetensors"],
     reference="https://huggingface.co/microsoft/LLM2CLIP-Openai-L-14-224",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,
@@ -230,7 +237,7 @@ llm2clip_openai_l_14_224 = ModelMeta(
 )
 
 llm2clip_openai_b_16 = ModelMeta(
-    loader=llm2clip_loader,  # type: ignore
+    loader=llm2clip_loader,
     name="microsoft/LLM2CLIP-Openai-B-16",
     model_type=["dense"],
     languages=["eng-Latn"],
@@ -238,6 +245,7 @@ llm2clip_openai_b_16 = ModelMeta(
     release_date="2024-11-07",
     modalities=["image", "text"],
     n_parameters=361_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=None,
     embed_dim=1280,
@@ -245,7 +253,7 @@ llm2clip_openai_b_16 = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/microsoft/LLM2CLIP",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "safetensors"],
     reference="https://huggingface.co/microsoft/LLM2CLIP-Openai-B-16",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,

mteb/models/model_implementations/llm2vec_models.py

@@ -1,16 +1,22 @@
+from __future__ import annotations
+
 import logging
-from collections.abc import Callable
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
-from torch.utils.data import DataLoader
 
 from mteb._requires_package import requires_package, suggest_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.models.models_protocols import EncoderProtocol
-from mteb.types import Array, BatchedInput, PromptType
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.models.models_protocols import EncoderProtocol
+    from mteb.types import Array, BatchedInput, PromptType
 
 logger = logging.getLogger(__name__)
 
@@ -139,13 +145,14 @@ llm2vec_llama3_8b_supervised = ModelMeta(
     # TODO: Not sure what to put here as a model is made of two peft repos, each with a different revision
     release_date="2024-04-09",
     n_parameters=7_505_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=28629,
     max_tokens=8192,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
     training_datasets=llm2vec_supervised_training_data,
@@ -168,13 +175,14 @@ llm2vec_llama3_8b_unsupervised = ModelMeta(
     revision="1cb7b735326d13a8541db8f57f35da5373f5e9c6",
     release_date="2024-04-09",
     n_parameters=7_505_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=28629,
     max_tokens=8192,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -197,13 +205,14 @@ llm2vec_mistral7b_supervised = ModelMeta(
     revision="0ae69bdd5816105778b971c3138e8f8a18eaa3ae",
     release_date="2024-04-09",
     n_parameters=7_111_000_000,
+    n_embedding_parameters=131_072_000,
     memory_usage_mb=27126,
     max_tokens=32768,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -226,13 +235,14 @@ llm2vec_mistral7b_unsupervised = ModelMeta(
     revision="2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8",
     release_date="2024-04-09",
     n_parameters=7_111_000_000,
+    n_embedding_parameters=131_072_000,
     memory_usage_mb=27126,
     max_tokens=32768,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -255,13 +265,14 @@ llm2vec_llama2_7b_supervised = ModelMeta(
     revision="2c055a5d77126c0d3dc6cd8ffa30e2908f4f45f8",
     release_date="2024-04-09",
     n_parameters=7_111_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=27126,
     max_tokens=32768,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -284,13 +295,14 @@ llm2vec_llama2_7b_unsupervised = ModelMeta(
     revision="a76944871d169ebe7c97eb921764cd063afed785",
     release_date="2024-04-09",
     n_parameters=7_111_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=27126,
     max_tokens=32768,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
     training_datasets=set(),
@@ -313,13 +325,14 @@ llm2vec_sheared_llama_supervised = ModelMeta(
     revision="a5943d406c6b016fef3f07906aac183cf1a0b47d",
     release_date="2024-04-09",
     n_parameters=7_111_000_000,
+    n_embedding_parameters=65_536_000,
     memory_usage_mb=27126,
     max_tokens=32768,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",
@@ -342,13 +355,14 @@ llm2vec_sheared_llama_unsupervised = ModelMeta(
     revision="a5943d406c6b016fef3f07906aac183cf1a0b47d",
     release_date="2024-04-09",
     n_parameters=7_111_000_000,
+    n_embedding_parameters=65_536_000,
     memory_usage_mb=27126,
     max_tokens=32768,
     embed_dim=4096,
     license="mit",
     reference="https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["LLM2Vec", "PyTorch"],
+    framework=["LLM2Vec", "PyTorch", "safetensors"],
     use_instructions=True,
     citation=LLM2VEC_CITATION,
     public_training_code="https://github.com/McGill-NLP/llm2vec/tree/250292a307428240d801fadd85825464e71c3277/train_configs",

mteb/models/model_implementations/mcinext_models.py

@@ -1,15 +1,19 @@
+from __future__ import annotations
+
 import logging
 import os
 import time
-from typing import Any
+import warnings
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import requests
 
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta
-from mteb.types import PromptType
 
+if TYPE_CHECKING:
+    from mteb.types import PromptType
 logger = logging.getLogger(__name__)
 
 HAKIM_CITATION = """@article{sarmadi2025hakim,
@@ -246,7 +250,9 @@ class HakimModelWrapper(AbsEncoder):
         task_prompt, task_id = DATASET_TASKS.get(task_name, (None, None))
 
         if not task_prompt:
-            logger.warning(f"Unknown dataset: {task_name}, no preprocessing applied.")
+            msg = f"Unknown dataset: {task_name}, no preprocessing applied."
+            logger.warning(msg)
+            warnings.warn(msg)
             return sample
 
         task_prompt = f"مسئله : {task_prompt}"
@@ -352,6 +358,7 @@ hakim = ModelMeta(
     revision="1",
     release_date="2025-05-10",
     n_parameters=124_441_344,
+    n_embedding_parameters=None,
     memory_usage_mb=475,
     embed_dim=768,
     license="not specified",
@@ -420,6 +427,7 @@ hakim_small = ModelMeta(
     revision="1",
     release_date="2025-05-10",
     n_parameters=38_736_384,
+    n_embedding_parameters=None,
     memory_usage_mb=148,
     embed_dim=512,
     license="not specified",
@@ -487,6 +495,7 @@ hakim_unsup = ModelMeta(
     revision="1",
     release_date="2025-05-10",
     n_parameters=124_441_344,
+    n_embedding_parameters=None,
     memory_usage_mb=475,
     embed_dim=768,
     license="not specified",

mteb/models/model_implementations/mdbr_models.py

@@ -1,5 +1,7 @@
 from mteb.models.model_implementations.arctic_models import arctic_v1_training_datasets
-from mteb.models.model_implementations.mxbai_models import mixedbread_training_data
+from mteb.models.model_implementations.mixedbread_ai_models import (
+    mixedbread_training_data,
+)
 from mteb.models.model_meta import ModelMeta
 from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
 
@@ -35,8 +37,15 @@ mdbr_leaf_ir = ModelMeta(
     release_date="2025-08-27",
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     n_parameters=22_861_056,
+    n_embedding_parameters=11_720_448,
     memory_usage_mb=86,
     max_tokens=512,
     embed_dim=768,
@@ -63,8 +72,15 @@ mdbr_leaf_mt = ModelMeta(
     release_date="2025-08-27",
     languages=["eng-Latn"],
     open_weights=True,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     n_parameters=22_958_592,
+    n_embedding_parameters=11_720_448,
     memory_usage_mb=86,
     max_tokens=512,
     embed_dim=1024,