mteb 2.7.2__py3-none-any.whl → 2.7.4__py3-none-any.whl

mteb/models/model_implementations/bica_model.py

@@ -9,6 +9,7 @@ bica_base = ModelMeta(
     revision="31237a836e5ae908c308a256573e5f0986498574",
     release_date="2025-11-14",
     n_parameters=110_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=418,
     embed_dim=768,
     license="mit",

mteb/models/model_implementations/blip2_models.py

@@ -1,14 +1,19 @@
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
 
 import torch
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
 from mteb._requires_package import requires_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 BLIP2_CITATION = """@inproceedings{li2023blip2,
     title={{BLIP-2:} Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models},
@@ -172,6 +177,7 @@ blip2_opt_2_7b = ModelMeta(
     release_date="2024-03-22",
     modalities=["image", "text"],
     n_parameters=3_740_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=14285,
     max_tokens=None,
     embed_dim=768,
@@ -196,6 +202,7 @@ blip2_opt_6_7b_coco = ModelMeta(
     release_date="2024-03-31",
     modalities=["image", "text"],
     n_parameters=7_750_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=29577,
     max_tokens=None,
     embed_dim=768,

mteb/models/model_implementations/blip_models.py

@@ -1,14 +1,19 @@
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
 
 import torch
 from torch.nn.functional import normalize
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 BLIP_CITATION = """@misc{https://doi.org/10.48550/arxiv.2201.12086,
     doi = {10.48550/ARXIV.2201.12086},
@@ -136,6 +141,7 @@ blip_image_captioning_large = ModelMeta(
     release_date="2023-12-07",
     modalities=["image", "text"],
     n_parameters=470_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1792,
     max_tokens=512,
     embed_dim=768,
@@ -164,6 +170,7 @@ blip_image_captioning_base = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -193,6 +200,7 @@ blip_vqa_base = ModelMeta(
     release_date="2023-12-07",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1467,
     max_tokens=512,
     embed_dim=768,
@@ -220,6 +228,7 @@ blip_vqa_capfilt_large = ModelMeta(
     release_date="2023-01-22",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -247,6 +256,7 @@ blip_itm_base_coco = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -274,6 +284,7 @@ blip_itm_large_coco = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=470_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1793,
     max_tokens=512,
     embed_dim=768,
@@ -302,6 +313,7 @@ blip_itm_base_flickr = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -330,6 +342,7 @@ blip_itm_large_flickr = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=470_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1793,
     max_tokens=512,
     embed_dim=768,

mteb/models/model_implementations/bm25.py

@@ -1,18 +1,22 @@
+from __future__ import annotations
+
 import logging
+from typing import TYPE_CHECKING
 
 from mteb._create_dataloaders import _create_text_queries_dataloader
 from mteb._requires_package import requires_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.model_meta import ModelMeta
-from mteb.models.models_protocols import SearchProtocol
-from mteb.types import (
-    CorpusDatasetType,
-    EncodeKwargs,
-    InstructionDatasetType,
-    QueryDatasetType,
-    RetrievalOutputType,
-    TopRankedDocumentsType,
-)
+
+if TYPE_CHECKING:
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.models.models_protocols import SearchProtocol
+    from mteb.types import (
+        CorpusDatasetType,
+        EncodeKwargs,
+        QueryDatasetType,
+        RetrievalOutputType,
+        TopRankedDocumentsType,
+    )
 
 logger = logging.getLogger(__name__)
 
@@ -75,7 +79,6 @@ def bm25_loader(model_name, **kwargs) -> SearchProtocol:
             hf_subset: str,
             top_k: int,
             encode_kwargs: EncodeKwargs,
-            instructions: InstructionDatasetType | None = None,
             top_ranked: TopRankedDocumentsType | None = None,
         ) -> RetrievalOutputType:
             logger.info("Encoding Queries...")
@@ -98,13 +101,17 @@ def bm25_loader(model_name, **kwargs) -> SearchProtocol:
                 query_results = queries_results[qi]
                 scores = queries_scores[qi]
                 doc_id_to_score = {}
+                query_documents = (
+                    top_ranked[qid] if top_ranked and qid in top_ranked else None
+                )
 
                 # Iterate over results
-                for ri in range(len(query_results)):
-                    doc_idx = query_results[ri]
-                    score = scores[ri]
+                for doc_idx, score in zip(query_results, scores):
                     doc_id = self.corpus_idx_to_id[doc_idx]
 
+                    # handle reranking with a filtered set of documents
+                    if query_documents is not None and doc_id not in query_documents:
+                        continue
                     doc_id_to_score[doc_id] = float(score)
 
                 results[qid] = doc_id_to_score
@@ -127,6 +134,7 @@ bm25_s = ModelMeta(
     revision="0_1_10",
     release_date="2024-07-10",  # release of version 0.1.10
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     embed_dim=None,
     license=None,

mteb/models/model_implementations/bmretriever_models.py

@@ -1,5 +1,6 @@
-from collections.abc import Callable
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
 
 import torch
 from sentence_transformers import SentenceTransformer
@@ -9,6 +10,9 @@ from mteb.models import ModelMeta
 from mteb.models.instruct_wrapper import InstructSentenceTransformerModel
 from mteb.types import PromptType
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 def instruction_template(
     instruction: str, prompt_type: PromptType | None = None
@@ -99,6 +103,7 @@ BMRetriever_410M = ModelMeta(
     release_date="2024-04-29",
     embed_dim=1024,
     n_parameters=353_822_720,
+    n_embedding_parameters=51_511_296,
     memory_usage_mb=1349,
     max_tokens=2048,
     license="mit",
@@ -129,6 +134,7 @@ BMRetriever_1B = ModelMeta(
     release_date="2024-04-29",
     embed_dim=2048,
     n_parameters=908_759_040,
+    n_embedding_parameters=103_022_592,
     memory_usage_mb=3466,
     max_tokens=2048,
     license="mit",
@@ -159,6 +165,7 @@ BMRetriever_2B = ModelMeta(
     release_date="2024-04-29",
     embed_dim=2048,
     n_parameters=2_506_172_416,
+    n_embedding_parameters=524_288_000,
     memory_usage_mb=9560,
     max_tokens=8192,
     license="mit",
@@ -189,6 +196,7 @@ BMRetriever_7B = ModelMeta(
     release_date="2024-04-29",
     embed_dim=4096,
     n_parameters=7_110_660_096,
+    n_embedding_parameters=131_072_000,
     memory_usage_mb=27124,
     max_tokens=32768,
     license="mit",

mteb/models/model_implementations/cadet_models.py

@@ -41,6 +41,7 @@ cadet_embed = ModelMeta(
     open_weights=True,
     release_date="2025-05-11",
     n_parameters=109_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=418,
     embed_dim=768,
     license="apache-2.0",

mteb/models/model_implementations/cde_models.py

@@ -1,27 +1,31 @@
+from __future__ import annotations
+
 import logging
-from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import torch
-from torch.utils.data import DataLoader
 
 import mteb
 from mteb._create_dataloaders import _corpus_to_dict
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.models.models_protocols import PromptType
 from mteb.models.sentence_transformer_wrapper import SentenceTransformerEncoderWrapper
-from mteb.types import Array, BatchedInput
+from mteb.types import PromptType
 
 from .bge_models import bge_full_data
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from torch.utils.data import DataLoader
+
     from mteb.abstasks import (
         AbsTaskClassification,
         AbsTaskRetrieval,
         AbsTaskSummarization,
     )
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput
 logger = logging.getLogger(__name__)
 
 CDE_CITATION = """@misc{morris2024contextualdocumentembeddings,
@@ -222,6 +226,7 @@ cde_small_v1 = ModelMeta(
     revision="e151df18af0d7f1d1c37b074fee58406ececf19f",
     release_date="2024-09-24",
     n_parameters=int(281 * 1e6),
+    n_embedding_parameters=None,
     memory_usage_mb=1072,  # Though the second-stage model is only 140M
     max_tokens=512,
     embed_dim=768,
@@ -251,6 +256,7 @@ cde_small_v2 = ModelMeta(
     revision="4e1d021a6c3fd7ce8aa0a7204057eee5ae61d390",
     release_date="2025-01-13",
     n_parameters=int(306 * 1e6),
+    n_embedding_parameters=None,
     memory_usage_mb=1166,  # Though the second-stage model is only 140M
     max_tokens=512,
     embed_dim=768,

mteb/models/model_implementations/clip_models.py

@@ -1,13 +1,18 @@
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
 
 import torch
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 
 class CLIPModel(AbsEncoder):
@@ -123,6 +128,7 @@ clip_vit_large_patch14 = ModelMeta(
     release_date="2021-02-26",
     modalities=["image", "text"],
     n_parameters=428_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=1631,
     max_tokens=77,
     embed_dim=768,
@@ -147,6 +153,7 @@ clip_vit_base_patch32 = ModelMeta(
     release_date="2021-02-26",
     modalities=["image", "text"],
     n_parameters=151_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=576,
     max_tokens=77,
     embed_dim=512,
@@ -171,6 +178,7 @@ clip_vit_base_patch16 = ModelMeta(
     release_date="2021-02-26",
     modalities=["image", "text"],
     n_parameters=151_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=576,
     max_tokens=77,
     embed_dim=512,

mteb/models/model_implementations/clips_models.py

@@ -30,6 +30,7 @@ e5_nl_small = ModelMeta(
     revision="0243664a6c5e12eef854b091eb283e51833c3e9f",
     release_date="2025-09-23",
     n_parameters=40_800_000,
+    n_embedding_parameters=19_200_768,
     memory_usage_mb=78,
     embed_dim=384,
     license="mit",
@@ -57,6 +58,7 @@ e5_nl_base = ModelMeta(
     revision="6bd5722f236da48b4b8bcb28cc1fc478f7089956",
     release_date="2025-09-23",
     n_parameters=124_400_000,
+    n_embedding_parameters=38_401_536,
     memory_usage_mb=237,
     embed_dim=768,
     license="mit",
@@ -84,6 +86,7 @@ e5_nl_large = ModelMeta(
     revision="683333f86ed9eb3699b5567f0fdabeb958d412b0",
     release_date="2025-09-23",
     n_parameters=355_000_000,
+    n_embedding_parameters=51_202_048,
     memory_usage_mb=1355,
     embed_dim=1024,
     license="mit",

mteb/models/model_implementations/codefuse_models.py

@@ -236,6 +236,7 @@ F2LLM_0B6 = ModelMeta(
     revision="36416618b83d4bd84a8ca30c2ee01ed518f9f2e7",
     release_date="2025-09-18",
     n_parameters=595_776_512,
+    n_embedding_parameters=None,
     memory_usage_mb=1137,
     embed_dim=1024,
     license="apache-2.0",
@@ -266,6 +267,7 @@ F2LLM_1B7 = ModelMeta(
     revision="fdce0e09655f42cea26f7f66f5a70cd4507ea45c",
     release_date="2025-09-18",
     n_parameters=1_720_574_976,
+    n_embedding_parameters=None,
     memory_usage_mb=3282,
     embed_dim=2560,
     license="apache-2.0",
@@ -296,6 +298,7 @@ F2LLM_4B = ModelMeta(
     revision="9fe95901ed2b6b59dd7673d6e93c9d76766a1e25",
     release_date="2025-09-18",
     n_parameters=4_021_774_336,
+    n_embedding_parameters=None,
     memory_usage_mb=7672,
     embed_dim=2560,
     license="apache-2.0",
@@ -318,6 +321,7 @@ C2LLM_0B5 = ModelMeta(
     release_date="2025-12-22",
     languages=c2llm_languages,
     n_parameters=497252096,
+    n_embedding_parameters=None,
     memory_usage_mb=948.0,
     max_tokens=32768,
     embed_dim=896,
@@ -346,6 +350,7 @@ C2LLM_7B = ModelMeta(
     release_date="2025-12-22",
     languages=c2llm_languages,
     n_parameters=7667028992,
+    n_embedding_parameters=None,
     memory_usage_mb=14624.0,
     max_tokens=32768,
     embed_dim=3584,

mteb/models/model_implementations/codesage_models.py

@@ -28,6 +28,7 @@ codesage_large = ModelMeta(
     release_date="2024-02-03",
     modalities=["text"],
     n_parameters=1_300_000_000,
+    n_embedding_parameters=100_667_392,
     memory_usage_mb=4959,
     max_tokens=2048,
     embed_dim=2048,
@@ -55,6 +56,7 @@ codesage_base = ModelMeta(
     release_date="2024-02-03",
     modalities=["text"],
     n_parameters=356_000_000,
+    n_embedding_parameters=50_333_696,
     memory_usage_mb=1358,
     max_tokens=2048,
     embed_dim=1024,
@@ -82,6 +84,7 @@ codesage_small = ModelMeta(
     release_date="2024-02-03",
     modalities=["text"],
     n_parameters=130_000_000,
+    n_embedding_parameters=50_333_696,
     memory_usage_mb=496,
     max_tokens=2048,
     embed_dim=1024,

mteb/models/model_implementations/cohere_models.py

@@ -1,18 +1,24 @@
+from __future__ import annotations
+
 import logging
 import time
 from functools import wraps
-from typing import Any, Literal, get_args
+from typing import TYPE_CHECKING, Any, Literal, get_args
 
 import numpy as np
 import torch
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
 from mteb._requires_package import requires_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
+from mteb.types import PromptType
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput
 
 logger = logging.getLogger(__name__)
 
@@ -386,6 +392,7 @@ cohere_mult_3 = ModelMeta(
     revision="1",
     release_date="2023-11-02",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=None,
     embed_dim=512,
@@ -412,6 +419,7 @@ cohere_eng_3 = ModelMeta(
     revision="1",
     release_date="2023-11-02",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=512,
     embed_dim=1024,
@@ -437,6 +445,7 @@ cohere_mult_light_3 = ModelMeta(
     reference="https://cohere.com/blog/introducing-embed-v3",
     release_date="2023-11-02",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=512,
     embed_dim=384,
@@ -462,6 +471,7 @@ cohere_eng_light_3 = ModelMeta(
     revision="1",
     release_date="2023-11-02",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=512,
     embed_dim=384,

mteb/models/model_implementations/cohere_v.py

@@ -1,15 +1,15 @@
+from __future__ import annotations
+
 import base64
 import io
 import os
 import time
-from typing import Any, Literal, get_args
+from typing import TYPE_CHECKING, Any, Literal, get_args
 
 import torch
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
 from mteb._requires_package import requires_image_dependencies, requires_package
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models import ModelMeta
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_implementations.cohere_models import (
@@ -18,7 +18,12 @@ from mteb.models.model_implementations.cohere_models import (
     retry_with_rate_limit,
 )
 from mteb.models.model_meta import ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 
 def _post_process_embeddings(
@@ -386,6 +391,7 @@ cohere_mult_3 = ModelMeta(
     revision="1",
     release_date="2024-10-24",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=None,
     embed_dim=1024,
@@ -410,6 +416,7 @@ cohere_eng_3 = ModelMeta(
     revision="1",
     release_date="2024-10-24",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=None,
     embed_dim=1024,
@@ -434,6 +441,7 @@ cohere_embed_v4_multimodal = ModelMeta(
     revision="1",
     release_date="2024-12-01",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=128000,
     embed_dim=1536,
@@ -458,6 +466,7 @@ cohere_embed_v4_multimodal_binary = ModelMeta(
     revision="1",
     release_date="2024-12-01",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=128000,
     embed_dim=1536,
@@ -483,6 +492,7 @@ cohere_embed_v4_multimodal_int8 = ModelMeta(
     revision="1",
     release_date="2024-12-01",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=128000,
     embed_dim=1536,

mteb/models/model_implementations/colpali_models.py

@@ -4,20 +4,21 @@ import logging
 from typing import TYPE_CHECKING, Any
 
 import torch
-from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
 from mteb._requires_package import (
     requires_image_dependencies,
     requires_package,
 )
-from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
-from mteb.types import Array, BatchedInput, PromptType
 
 if TYPE_CHECKING:
     from PIL import Image
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
 
 logger = logging.getLogger(__name__)
 
@@ -219,6 +220,7 @@ colpali_v1_1 = ModelMeta(
     release_date="2024-08-21",
     modalities=["image", "text"],
     n_parameters=2_920_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=4700,
     max_tokens=16384,
     embed_dim=128,
@@ -246,6 +248,7 @@ colpali_v1_2 = ModelMeta(
     release_date="2024-08-26",
     modalities=["image", "text"],
     n_parameters=2_920_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=4700,
     max_tokens=16384,
     embed_dim=128,
@@ -273,6 +276,7 @@ colpali_v1_3 = ModelMeta(
     release_date="2024-11-01",
     modalities=["image", "text"],
     n_parameters=2_920_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=4700,
     max_tokens=16384,
     embed_dim=128,