mteb 2.7.3__py3-none-any.whl → 2.7.4__py3-none-any.whl

mteb/models/model_implementations/vdr_models.py

@@ -38,6 +38,7 @@ vdr_2b_multi_v1 = ModelMeta(
     release_date="2024-01-08",
     modalities=["text"],  # TODO: integrate with image
     n_parameters=2_000_000_000,
+    n_embedding_parameters=233_373_696,
     memory_usage_mb=4213,
     max_tokens=32768,
     embed_dim=1536,

mteb/models/model_implementations/vi_vn_models.py

@@ -16,6 +16,7 @@ greennode_embedding_large_vn_v1 = ModelMeta(
     loader=sentence_transformers_loader,
     open_weights=True,
     n_parameters=568_000_000,
+    n_embedding_parameters=256_002_048,
     memory_usage_mb=2167,
     embed_dim=1024,
     license="cc-by-4.0",
@@ -41,6 +42,7 @@ greennode_embedding_large_vn_mixed_v1 = ModelMeta(
     loader=sentence_transformers_loader,
     open_weights=True,
     n_parameters=568_000_000,
+    n_embedding_parameters=256_002_048,
     memory_usage_mb=2167,
     embed_dim=1024,
     license="cc-by-4.0",
@@ -66,6 +68,7 @@ aiteamvn_vietnamese_embeddings = ModelMeta(
     loader=sentence_transformers_loader,
     open_weights=True,
     n_parameters=568_000_000,
+    n_embedding_parameters=256_002_048,
     memory_usage_mb=2166,
     embed_dim=1024,
     license="cc-by-4.0",
@@ -98,6 +101,7 @@ hiieu_halong_embedding = ModelMeta(
     use_instructions=False,
     open_weights=True,
     n_parameters=278_000_000,
+    n_embedding_parameters=192_001_536,
     memory_usage_mb=1061,
     embed_dim=768,
     license="apache-2.0",
@@ -129,6 +133,7 @@ sup_simcse_vietnamese_phobert_base_ = ModelMeta(
     use_instructions=False,
     open_weights=True,
     n_parameters=135_000_000,
+    n_embedding_parameters=49_152_768,
     memory_usage_mb=517,
     max_tokens=256,
     embed_dim=768,
@@ -167,6 +172,7 @@ bkai_foundation_models_vietnamese_bi_encoder = ModelMeta(
     use_instructions=False,
     open_weights=True,
     n_parameters=135_000_000,
+    n_embedding_parameters=49_152_768,
     memory_usage_mb=515,
     max_tokens=256,
     embed_dim=768,

mteb/models/model_implementations/vista_models.py

@@ -258,6 +258,7 @@ visualized_bge_base = ModelMeta(
     release_date="2024-06-06",
     modalities=["image", "text"],
     n_parameters=196_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=1631,
     max_tokens=512,
     embed_dim=768,
@@ -286,6 +287,7 @@ visualized_bge_m3 = ModelMeta(
     release_date="2024-06-06",
     modalities=["image", "text"],
     n_parameters=872_909_505,
+    n_embedding_parameters=None,
     memory_usage_mb=4263,
     max_tokens=8192,
     embed_dim=1024,

mteb/models/model_implementations/vlm2vec_models.py

@@ -280,6 +280,7 @@ vlm2vec_lora = ModelMeta(
     release_date="2024-10-08",
     modalities=["image", "text"],
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=131072,
     embed_dim=3072,
@@ -304,6 +305,7 @@ vlm2vec_full = ModelMeta(
     release_date="2024-10-08",
     modalities=["image", "text"],
     n_parameters=4_150_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=7909,
     max_tokens=131072,
     embed_dim=3072,

mteb/models/model_implementations/voyage_models.py

@@ -308,6 +308,7 @@ voyage_3_large = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2025/01/07/voyage-3-large/",
@@ -336,6 +337,7 @@ voyage_3_5 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2025/05/20/voyage-3-5/",
@@ -363,6 +365,7 @@ voyage_3_5_int8 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2025/05/20/voyage-3-5/",
@@ -390,6 +393,7 @@ voyage_3_5_binary = ModelMeta(
     embed_dim=1024,  # Same as original after unpacking from bits
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2025/05/20/voyage-3-5/",
@@ -417,6 +421,7 @@ voyage_large_2_instruct = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/05/05/voyage-large-2-instruct-instruction-tuned-and-rank-1-on-mteb/",
@@ -443,6 +448,7 @@ voyage_finance_2 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/06/03/domain-specific-embeddings-finance-edition-voyage-finance-2/",
@@ -469,6 +475,7 @@ voyage_law_2 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/04/15/domain-specific-embeddings-and-retrieval-legal-edition-voyage-law-2/",
@@ -495,6 +502,7 @@ voyage_code_2 = ModelMeta(
     embed_dim=1536,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/01/23/voyage-code-2-elevate-your-code-retrieval/",
@@ -521,6 +529,7 @@ voyage_code_3 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/12/04/voyage-code-3/",
@@ -548,6 +557,7 @@ voyage_large_2 = ModelMeta(
     embed_dim=1536,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2023/10/29/voyage-embeddings/",
@@ -574,6 +584,7 @@ voyage_2 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2023/10/29/voyage-embeddings/",
@@ -599,6 +610,7 @@ voyage_multilingual_2 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/06/10/voyage-multilingual-2-multilingual-embedding-model/",
@@ -625,6 +637,7 @@ voyage_3 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/09/18/voyage-3/",
@@ -651,6 +664,7 @@ voyage_3_lite = ModelMeta(
     embed_dim=512,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://blog.voyageai.com/2024/09/18/voyage-3/",
@@ -679,6 +693,7 @@ voyage_3_exp = ModelMeta(
     open_weights=False,
     # from their card https://huggingface.co/voyageai/voyage-3-m-exp#model-information
     n_parameters=int(6918 * 1e6),
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     license=None,
     reference="https://huggingface.co/voyageai/voyage-3-m-exp",

mteb/models/model_implementations/voyage_v.py

@@ -215,6 +215,7 @@ voyage_v = ModelMeta(
     revision="1",
     release_date="2024-11-10",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=32768,
     embed_dim=1024,

mteb/models/model_implementations/xyz_models.py

@@ -31,6 +31,7 @@ xyz_embedding = ModelMeta(
     revision="4004120220b99baea764a1d3508427248ac3bccf",
     release_date="2024-09-13",
     n_parameters=326000000,
+    n_embedding_parameters=21_635_072,
     memory_usage_mb=1242,
     max_tokens=512,
     embed_dim=768,

mteb/models/model_implementations/youtu_models.py

@@ -121,6 +121,7 @@ Youtu_Embedding_V1 = ModelMeta(
     release_date="2025-09-28",
     open_weights=True,
     n_parameters=2672957440,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     embed_dim=2048,
     license="apache-2.0",

mteb/models/model_implementations/yuan_models.py

@@ -20,6 +20,7 @@ yuan_embedding_2_zh = ModelMeta(
     revision="b5ebcace6f4fc6e5a4d1852557eb2dc2d1040cee",
     release_date="2025-11-24",
     n_parameters=326000000,
+    n_embedding_parameters=21_635_072,
     memory_usage_mb=1242,
     embed_dim=1792,
     license="apache-2.0",

mteb/models/model_implementations/yuan_models_en.py

@@ -43,6 +43,7 @@ yuan_embedding_2_en = ModelMeta(
     revision="b2fd15da3bcae3473c8529593825c15068f09fce",
     release_date="2025-11-27",
     n_parameters=595776512,
+    n_embedding_parameters=None,
     memory_usage_mb=2272,
     embed_dim=1024,
     max_tokens=2048,

mteb/models/model_meta.py

@@ -10,6 +10,7 @@ from functools import partial
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal, cast
 
+import numpy as np
 from huggingface_hub import (
     ModelCard,
     get_safetensors_metadata,
@@ -27,6 +28,8 @@ from huggingface_hub.errors import (
     SafetensorsParsingError,
 )
 from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+from sentence_transformers.models import Transformer
+from torch import nn
 from transformers import AutoConfig
 
 from mteb._helpful_enum import HelpfulStrEnum
@@ -99,8 +102,9 @@ class ModelMeta(BaseModel):
         loader: The function that loads the model. If None it assumes that the model is not implemented.
         loader_kwargs: The keyword arguments to pass to the loader function.
         name: The name of the model, ideally the name on huggingface. It should be in the format "organization/model_name".
-        n_parameters: The number of parameters in the model, e.g. 7_000_000 for a 7M parameter model. Can be None if the number of parameters is not known (e.g. for proprietary models) or
-            if the loader returns a SentenceTransformer model from which it can be derived.
+        n_parameters: The total number of parameters in the model, e.g. `7_000_000` for a 7M parameter model. Can be none in case the number of parameters is unknown.
+        n_embedding_parameters: The number of parameters used for the embedding layer. Can be None if the number of embedding parameters is not known (e.g. for proprietary models).
+        n_active_parameters_override: The number of active parameters used bu model. Should be used **only** for Mixture of Experts models.
         memory_usage_mb: The memory usage of the model in MB. Can be None if the memory usage is not known (e.g. for proprietary models). To calculate it use the `calculate_memory_usage_mb` method.
         max_tokens: The maximum number of tokens the model can handle. Can be None if the maximum number of tokens is not known (e.g. for proprietary
             models).
@@ -139,6 +143,8 @@ class ModelMeta(BaseModel):
     release_date: StrDate | None
     languages: list[ISOLanguageScript] | None
     n_parameters: int | None
+    n_active_parameters_override: int | None = None
+    n_embedding_parameters: int | None = None
     memory_usage_mb: float | None
     max_tokens: float | None
     embed_dim: int | None
@@ -197,6 +203,16 @@ class ModelMeta(BaseModel):
         """
         return "cross-encoder" in self.model_type
 
+    @property
+    def n_active_parameters(self):
+        """Number of active parameters. Assumed to be `n_parameters - n_embedding_parameters`. Can be overwritten using `n_active_parameters_override` e.g. for MoE models."""
+        if self.n_active_parameters_override is not None:
+            return self.n_active_parameters_override
+
+        if self.n_parameters is not None and self.n_embedding_parameters is not None:
+            return self.n_parameters - self.n_embedding_parameters
+        return None
+
     @field_validator("similarity_fn_name", mode="before")
     @classmethod
     def _validate_similarity_fn_name(cls, value: str) -> ScoringFunction | None:
@@ -389,6 +405,14 @@ class ModelMeta(BaseModel):
             else model.model_card_data.base_model
         )
         meta = cls._from_hub(name, revision, compute_metadata)
+        try:
+            first = model[0]
+
+            if isinstance(first, Transformer):
+                emb = first.auto_model.get_input_embeddings()
+                meta.n_embedding_parameters = int(np.prod(emb.weight.shape))
+        except Exception as e:
+            logger.warning(f"Could not calculate embedding parameters for {name}: {e}")
         meta.revision = model.model_card_data.base_model_revision or meta.revision
         meta.max_tokens = model.max_seq_length
         meta.embed_dim = model.get_sentence_embedding_dimension()
@@ -460,6 +484,15 @@ class ModelMeta(BaseModel):
         from mteb.models import CrossEncoderWrapper
 
         meta = cls._from_hub(model.model.name_or_path, revision, compute_metadata)
+        try:
+            emb = model.model.get_input_embeddings()
+
+            if isinstance(emb, nn.Embedding):
+                meta.n_embedding_parameters = int(np.prod(emb.weight.shape))
+        except Exception as e:
+            logger.warning(
+                f"Could not calculate embedding parameters for {model.model.name_or_path}: {e}"
+            )
         meta.revision = model.config._commit_hash or meta.revision
         meta.loader = CrossEncoderWrapper
         meta.embed_dim = None

mteb/tasks/retrieval/eng/__init__.py

@@ -14,6 +14,28 @@ from .birco_whats_that_book_reranking import BIRCOWhatsThatBookReranking
 from .blink_it2i_retrieval import BLINKIT2IRetrieval
 from .blink_it2t_retrieval import BLINKIT2TRetrieval
 from .bright_retrieval import BrightLongRetrieval, BrightRetrieval
+from .bright_v1_1_retrieval import (
+    BrightAopsRetrieval,
+    BrightBiologyLongRetrieval,
+    BrightBiologyRetrieval,
+    BrightEarthScienceLongRetrieval,
+    BrightEarthScienceRetrieval,
+    BrightEconomicsLongRetrieval,
+    BrightEconomicsRetrieval,
+    BrightLeetcodeRetrieval,
+    BrightPonyLongRetrieval,
+    BrightPonyRetrieval,
+    BrightPsychologyLongRetrieval,
+    BrightPsychologyRetrieval,
+    BrightRoboticsLongRetrieval,
+    BrightRoboticsRetrieval,
+    BrightStackoverflowLongRetrieval,
+    BrightStackoverflowRetrieval,
+    BrightSustainableLivingLongRetrieval,
+    BrightSustainableLivingRetrieval,
+    BrightTheoremQAQuestionsRetrieval,
+    BrightTheoremQATheoremsRetrieval,
+)
 from .built_bench_retrieval import BuiltBenchRetrieval
 from .chat_doctor_retrieval import ChatDoctorRetrieval
 from .chem_hotpot_qa_retrieval import ChemHotpotQARetrieval
@@ -236,8 +258,28 @@ __all__ = [
     "BarExamQARetrieval",
     "BillSumCARetrieval",
     "BillSumUSRetrieval",
+    "BrightAopsRetrieval",
+    "BrightBiologyLongRetrieval",
+    "BrightBiologyRetrieval",
+    "BrightEarthScienceLongRetrieval",
+    "BrightEarthScienceRetrieval",
+    "BrightEconomicsLongRetrieval",
+    "BrightEconomicsRetrieval",
+    "BrightLeetcodeRetrieval",
     "BrightLongRetrieval",
+    "BrightPonyLongRetrieval",
+    "BrightPonyRetrieval",
+    "BrightPsychologyLongRetrieval",
+    "BrightPsychologyRetrieval",
     "BrightRetrieval",
+    "BrightRoboticsLongRetrieval",
+    "BrightRoboticsRetrieval",
+    "BrightStackoverflowLongRetrieval",
+    "BrightStackoverflowRetrieval",
+    "BrightSustainableLivingLongRetrieval",
+    "BrightSustainableLivingRetrieval",
+    "BrightTheoremQAQuestionsRetrieval",
+    "BrightTheoremQATheoremsRetrieval",
     "BuiltBenchRetrieval",
     "CIRRIT2IRetrieval",
     "CQADupstackAndroidRetrieval",

mteb/tasks/retrieval/eng/bright_retrieval.py

@@ -1,3 +1,4 @@
+import warnings
 from collections import defaultdict
 
 import datasets
@@ -86,6 +87,12 @@ def load_data(self) -> None:
     if self.data_loaded:
         return
 
+    warnings.warn(
+        "This task contains wrong prompts in the metadata. "
+        "Please use BRIGHT(v1.1) benchmark instead.",
+        category=DeprecationWarning,
+    )
+
     self.corpus, self.queries, self.relevant_docs = self.load_bright_data(
         path=self.metadata.dataset["path"],
         domains=list(self.metadata.eval_langs.keys()),
@@ -104,7 +111,7 @@ class BrightRetrieval(AbsTaskRetrieval):
             "revision": "a75a0eb483f6a5233a6efc2d63d71540a4443dfb",
         },
         reference="https://huggingface.co/datasets/xlangai/BRIGHT",
-        description="Bright retrieval dataset.",
+        description="BRIGHT: A Realistic and Challenging Benchmark for Reasoning-Intensive Retrieval",
         type="Retrieval",
         category="t2t",
         eval_splits=["standard"],
@@ -129,6 +136,7 @@ class BrightRetrieval(AbsTaskRetrieval):
   year = {2024},
 }
 """,
+        superseded_by="BrightBiologyRetrieval",
     )
     load_bright_data = load_bright_data
     load_data = load_data