mteb 2.5.2__py3-none-any.whl → 2.7.9__py3-none-any.whl

mteb/models/model_implementations/moka_models.py

@@ -97,6 +97,7 @@ m3e_base = ModelMeta(
     revision="764b537a0e50e5c7d64db883f2d2e051cbe3c64c",
     release_date="2023-06-06",  # first commit
     n_parameters=int(102 * 1e6),
+    n_embedding_parameters=16_226_304,
     memory_usage_mb=390,
     embed_dim=768,
     # They don't give a specific license but commercial use is not allowed
@@ -104,7 +105,7 @@ m3e_base = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/moka-ai/m3e-base",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -123,6 +124,7 @@ m3e_small = ModelMeta(
     revision="44c696631b2a8c200220aaaad5f987f096e986df",
     release_date="2023-06-02",  # first commit
     n_parameters=None,
+    n_embedding_parameters=10_817_536,
     memory_usage_mb=None,  # Can't be seen on HF page
     embed_dim=512,
     # They don't give a specific license but commercial use is not allowed
@@ -149,6 +151,7 @@ m3e_large = ModelMeta(
     revision="12900375086c37ba5d83d1e417b21dc7d1d1f388",
     release_date="2023-06-21",  # first commit
     n_parameters=None,
+    n_embedding_parameters=21_635_072,
     memory_usage_mb=None,  # Can't be seen on HF page
     embed_dim=768,
     # They don't give a specific license but commercial use is not allowed

mteb/models/model_implementations/nbailab.py

@@ -12,13 +12,14 @@ nb_sbert = ModelMeta(
     revision="b95656350a076aeafd2d23763660f80655408cc6",
     release_date="2022-11-23",
     n_parameters=1_780_000_000,
+    n_embedding_parameters=91_812_096,
     memory_usage_mb=678,
     embed_dim=4096,
     license="apache-2.0",
     max_tokens=75,
     reference="https://huggingface.co/NbAiLab/nb-sbert-base",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
     use_instructions=False,
     public_training_code=None,
     public_training_data="https://huggingface.co/datasets/NbAiLab/mnli-norwegian",
@@ -34,13 +35,14 @@ nb_bert_large = ModelMeta(
     revision="f9d0fc184adab4dc354d85e1854b7634540d7550",
     release_date="2021-04-29",
     n_parameters=355087360,
+    n_embedding_parameters=51_200_000,
     memory_usage_mb=1359,
     embed_dim=1024,
     license="cc-by-4.0",
     max_tokens=512,
     reference="https://huggingface.co/NbAiLab/nb-bert-large",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data="https://huggingface.co/NbAiLab/nb-bert-large#training-data",
@@ -56,13 +58,14 @@ nb_bert_base = ModelMeta(
     revision="9417c3f62a3adc99f17ff92bff446f35d011f994",
     release_date="2021-01-13",
     n_parameters=177853440,
+    n_embedding_parameters=91_812_096,
     memory_usage_mb=681,
     embed_dim=768,
     license="cc-by-4.0",
     max_tokens=512,
     reference="https://huggingface.co/NbAiLab/nb-bert-base",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data="https://huggingface.co/NbAiLab/nb-bert-base#training-data",

mteb/models/model_implementations/no_instruct_sentence_models.py

@@ -1,15 +1,22 @@
-from collections.abc import Generator
+from __future__ import annotations
+
 from itertools import islice
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import torch
-from torch.utils.data import DataLoader
 
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
+from mteb.types import PromptType
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput
 
 
 # https://docs.python.org/3/library/itertools.html#itertools.batched
@@ -30,13 +37,13 @@ class NoInstructModel(AbsEncoder):
         self,
         model_name: str,
         revision: str,
+        device: str | None = None,
         model_prompts: dict[str, str] | None = None,
         **kwargs: Any,
     ):
         from transformers import AutoModel, AutoTokenizer
 
         self.model_name = model_name
-        device = kwargs.pop("device", None)
         self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
         self.model = AutoModel.from_pretrained(
             model_name, revision=revision, **kwargs
@@ -103,13 +110,14 @@ no_instruct_small_v0 = ModelMeta(
     revision="b38747000553d8268915c95a55fc87e707c9aadd",
     release_date="2024-05-01",  # first commit
     n_parameters=33_400_000,
+    n_embedding_parameters=11_720_448,
     memory_usage_mb=127,
     max_tokens=512,
     embed_dim=384,
     license="mit",
     reference="https://huggingface.co/avsolatorio/NoInstruct-small-Embedding-v0",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Sentence Transformers", "safetensors", "Transformers"],
     use_instructions=False,
     adapted_from=None,
     superseded_by=None,

mteb/models/model_implementations/nomic_models.py

@@ -1,15 +1,21 @@
+from __future__ import annotations
+
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
 import torch.nn.functional as F
 from packaging.version import Version
-from torch.utils.data import DataLoader
 
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.models.sentence_transformer_wrapper import SentenceTransformerEncoderWrapper
-from mteb.types import Array, BatchedInput, PromptType
+from mteb.types import PromptType
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +29,7 @@ class NomicWrapper(SentenceTransformerEncoderWrapper):
         self,
         model_name: str,
         revision: str,
+        device: str | None = None,
         model_prompts: dict[str, str] | None = None,
         **kwargs: Any,
     ):
@@ -37,7 +44,9 @@ class NomicWrapper(SentenceTransformerEncoderWrapper):
                 f"Current transformers version is {transformers.__version__} is lower than the required version"
                 f" {MODERN_BERT_TRANSFORMERS_MIN_VERSION}"
             )
-        super().__init__(model_name, revision, model_prompts, **kwargs)
+        super().__init__(
+            model_name, revision, device=device, model_prompts=model_prompts, **kwargs
+        )
 
     def to(self, device: torch.device) -> None:
         self.model.to(device)
@@ -193,7 +202,7 @@ NOMIC_CITATION = """
 """
 
 nomic_embed_v1_5 = ModelMeta(
-    loader=NomicWrapper,  # type: ignore
+    loader=NomicWrapper,
     loader_kwargs=dict(
         trust_remote_code=True,
         model_prompts=model_prompts,
@@ -206,13 +215,20 @@ nomic_embed_v1_5 = ModelMeta(
     release_date="2024-02-10",  # first commit
     citation=NOMIC_CITATION,
     n_parameters=137_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=522,
     max_tokens=8192,
     embed_dim=768,
     license="apache-2.0",
     reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=True,
     adapted_from=None,
     superseded_by=None,
@@ -222,7 +238,7 @@ nomic_embed_v1_5 = ModelMeta(
 )
 
 nomic_embed_v1 = ModelMeta(
-    loader=NomicWrapper,  # type: ignore
+    loader=NomicWrapper,
     loader_kwargs=dict(
         trust_remote_code=True,
         model_prompts=model_prompts,
@@ -234,13 +250,20 @@ nomic_embed_v1 = ModelMeta(
     revision="0759316f275aa0cb93a5b830973843ca66babcf5",
     release_date="2024-01-31",  # first commit
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=522,
     max_tokens=8192,
     embed_dim=768,
     license="apache-2.0",
     reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=True,
     citation=NOMIC_CITATION,
     adapted_from=None,
@@ -251,7 +274,7 @@ nomic_embed_v1 = ModelMeta(
 )
 
 nomic_embed_v1_ablated = ModelMeta(
-    loader=NomicWrapper,  # type: ignore
+    loader=NomicWrapper,
     loader_kwargs=dict(
         trust_remote_code=True,
         model_prompts=model_prompts,
@@ -263,13 +286,14 @@ nomic_embed_v1_ablated = ModelMeta(
     revision="7d948905c5d5d3874fa55a925d68e49dbf411e5f",
     release_date="2024-01-15",  # first commit
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=8192,
     embed_dim=768,
     license="apache-2.0",
     reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1-ablated",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX"],
     use_instructions=True,
     adapted_from=None,
     superseded_by=None,
@@ -279,7 +303,7 @@ nomic_embed_v1_ablated = ModelMeta(
 )
 
 nomic_embed_v1_unsupervised = ModelMeta(
-    loader=NomicWrapper,  # type: ignore
+    loader=NomicWrapper,
     loader_kwargs=dict(
         trust_remote_code=True,
         model_prompts=model_prompts,
@@ -291,13 +315,14 @@ nomic_embed_v1_unsupervised = ModelMeta(
     revision="b53d557b15ae63852847c222d336c1609eced93c",
     release_date="2024-01-15",  # first commit
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=8192,
     embed_dim=768,
     license="apache-2.0",
     reference="https://huggingface.co/nomic-ai/nomic-embed-text-v1-unsupervised",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "Transformers"],
     use_instructions=True,
     adapted_from=None,
     superseded_by=None,
@@ -319,13 +344,14 @@ nomic_modern_bert_embed = ModelMeta(
     revision="5960f1566fb7cb1adf1eb6e816639cf4646d9b12",
     release_date="2024-12-29",
     n_parameters=149_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=568,
     max_tokens=8192,
     embed_dim=768,
     license="apache-2.0",
     reference="https://huggingface.co/nomic-ai/modernbert-embed-base",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     use_instructions=True,
     adapted_from="answerdotai/ModernBERT-base",
     public_training_code="https://github.com/nomic-ai/contrastors/blob/5f7b461e5a13b5636692d1c9f1141b27232fe966/src/contrastors/configs/train/contrastive_pretrain_modernbert.yaml",
@@ -334,7 +360,7 @@ nomic_modern_bert_embed = ModelMeta(
     training_datasets=nomic_training_data,
     public_training_data=None,
     citation="""@misc{nussbaum2024nomic,
-      title={Nomic Embed: Training a Reproducible Long Context Text Embedder}, 
+      title={Nomic Embed: Training a Reproducible Long Context Text Embedder},
       author={Zach Nussbaum and John X. Morris and Brandon Duderstadt and Andriy Mulyar},
       year={2024},
       eprint={2402.01613},
@@ -446,7 +472,7 @@ m_languages = [
 ]
 
 nomic_embed_text_v2_moe = ModelMeta(
-    loader=NomicWrapper,  # type: ignore
+    loader=NomicWrapper,
     loader_kwargs=dict(
         trust_remote_code=True,
         model_prompts=model_prompts,
@@ -458,13 +484,15 @@ nomic_embed_text_v2_moe = ModelMeta(
     revision="1066b6599d099fbb93dfcb64f9c37a7c9e503e85",
     release_date="2025-02-07",
     n_parameters=475292928,
+    n_embedding_parameters=192036864,
+    n_active_parameters_override=141628032,
     memory_usage_mb=1813,
     max_tokens=512,
     embed_dim=768,
     license="apache-2.0",
     reference="https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=True,
     adapted_from="nomic-ai/nomic-xlm-2048",
     public_training_data="https://github.com/nomic-ai/contrastors?tab=readme-ov-file#data-access",
@@ -472,12 +500,12 @@ nomic_embed_text_v2_moe = ModelMeta(
     training_datasets=None,  # did not look into this further
     superseded_by=None,
     citation="""@misc{nussbaum2025trainingsparsemixtureexperts,
-      title={Training Sparse Mixture Of Experts Text Embedding Models}, 
+      title={Training Sparse Mixture Of Experts Text Embedding Models},
       author={Zach Nussbaum and Brandon Duderstadt},
       year={2025},
       eprint={2502.07972},
       archivePrefix={arXiv},
       primaryClass={cs.CL},
-      url={https://arxiv.org/abs/2502.07972}, 
+      url={https://arxiv.org/abs/2502.07972},
 }""",
 )

mteb/models/model_implementations/nomic_models_vision.py

@@ -4,17 +4,18 @@ from typing import TYPE_CHECKING, Any
 
 import torch
 import torch.nn.functional as F
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
 from mteb._requires_package import requires_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
 
 if TYPE_CHECKING:
     from PIL import Image
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 NOMIC_EMBED_VISION_CITATION = """@article{nussbaum2024nomicembedvision,
       title={Nomic Embed Vision: Expanding the Latent Space},
@@ -174,6 +175,7 @@ nomic_embed_vision_v1_5 = ModelMeta(
     release_date="2024-06-08",
     modalities=["image", "text"],
     n_parameters=92_900_000,
+    n_embedding_parameters=None,
     memory_usage_mb=355,
     max_tokens=2048,
     embed_dim=768,
@@ -181,7 +183,7 @@ nomic_embed_vision_v1_5 = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/nomic-ai/contrastors",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "ONNX", "safetensors"],
     reference="https://huggingface.co/nomic-ai/nomic-embed-vision-v1.5",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,

mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py

@@ -1,16 +1,18 @@
+from __future__ import annotations
+
 from typing import TYPE_CHECKING, Any
 
 import torch
+from packaging.version import Version
 from torch.utils.data import DataLoader
+from transformers import __version__ as transformers_version
 
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta
-from mteb.types import Array, BatchedInput, PromptType
 
 if TYPE_CHECKING:
-    pass
-
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 LLAMA_NEMORETRIEVER_CITATION = """@misc{xu2025llamanemoretrievercolembedtopperforming,
       title={Llama Nemoretriever Colembed: Top-Performing Text-Image Retrieval Model},
@@ -34,6 +36,14 @@ class LlamaNemoretrieverColembed(AbsEncoder):
         attn_implementation="flash_attention_2",
         **kwargs,
     ):
+        required_transformers_version = "4.49.0"
+
+        if Version(transformers_version) != Version(required_transformers_version):
+            raise RuntimeError(
+                f"transformers version {transformers_version} is not match with required "
+                f"install version {required_transformers_version} to run `nvidia/llama-nemoretriever-colembed`"
+            )
+
         from transformers import AutoModel
 
         self.model = AutoModel.from_pretrained(
@@ -148,10 +158,11 @@ llama_nemoretriever_colembed_1b_v1 = ModelMeta(
     name="nvidia/llama-nemoretriever-colembed-1b-v1",
     model_type=["late-interaction"],
     languages=["eng-Latn"],
-    revision="1f0fdea7f5b19532a750be109b19072d719b8177",
+    revision="6eade800103413033f260bb55b49fe039fd28a6e",
     release_date="2025-06-27",
     modalities=["image", "text"],
     n_parameters=2_418_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=4610,
     max_tokens=8192,
     embed_dim=2048,
@@ -159,7 +170,7 @@ llama_nemoretriever_colembed_1b_v1 = ModelMeta(
     open_weights=True,
     public_training_code="Proprietary Code",
     public_training_data="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1",
     similarity_fn_name="MaxSim",
     use_instructions=True,
@@ -175,10 +186,11 @@ llama_nemoretriever_colembed_3b_v1 = ModelMeta(
     name="nvidia/llama-nemoretriever-colembed-3b-v1",
     model_type=["late-interaction"],
     languages=["eng-Latn"],
-    revision="50c36f4d5271c6851aa08bd26d69f6e7ca8b870c",
+    revision="4194bdd2cd2871f220ddba6273ce173ef1217a1e",
     release_date="2025-06-27",
     modalities=["image", "text"],
     n_parameters=4_407_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=8403,
     max_tokens=8192,
     embed_dim=3072,
@@ -186,7 +198,7 @@ llama_nemoretriever_colembed_3b_v1 = ModelMeta(
     open_weights=True,
     public_training_code="Proprietary Code",
     public_training_data="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset",
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/nvidia/llama-nemoretriever-colembed-3b-v1",
     similarity_fn_name="MaxSim",
     use_instructions=True,