mteb 2.7.3__py3-none-any.whl → 2.7.4__py3-none-any.whl

mteb/descriptive_stats/Retrieval/BrightSustainableLivingLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 662,
+        "number_of_characters": 21154322,
+        "documents_text_statistics": {
+            "total_text_length": 21080575,
+            "min_text_length": 30,
+            "average_text_length": 38051.579422382674,
+            "max_text_length": 5732344,
+            "unique_texts": 551
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 73747,
+            "min_text_length": 158,
+            "average_text_length": 682.8425925925926,
+            "max_text_length": 2843,
+            "unique_texts": 108
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 129,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1944444444444444,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 129
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 59832,
+            "min_top_ranked_per_query": 554,
+            "average_top_ranked_per_query": 554.0,
+            "max_top_ranked_per_query": 554
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightSustainableLivingRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 60900,
+        "number_of_characters": 20971763,
+        "documents_text_statistics": {
+            "total_text_length": 20898016,
+            "min_text_length": 1,
+            "average_text_length": 343.7626003421503,
+            "max_text_length": 158296,
+            "unique_texts": 50142
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 73747,
+            "min_text_length": 158,
+            "average_text_length": 682.8425925925926,
+            "max_text_length": 2843,
+            "unique_texts": 108
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 604,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 5.592592592592593,
+            "max_relevant_docs_per_query": 59,
+            "unique_relevant_docs": 604
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 6565536,
+            "min_top_ranked_per_query": 60792,
+            "average_top_ranked_per_query": 60792.0,
+            "max_top_ranked_per_query": 60792
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightTheoremQAQuestionsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 188207,
+        "number_of_characters": 141817604,
+        "documents_text_statistics": {
+            "total_text_length": 141734227,
+            "min_text_length": 58,
+            "average_text_length": 753.8974425803981,
+            "max_text_length": 7334,
+            "unique_texts": 176508
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 83377,
+            "min_text_length": 12,
+            "average_text_length": 406.7170731707317,
+            "max_text_length": 1255,
+            "unique_texts": 201
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 469,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 2.299019607843137,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 234
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 37946536,
+            "min_top_ranked_per_query": 176970,
+            "average_top_ranked_per_query": 185105.05365853658,
+            "max_top_ranked_per_query": 188176
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightTheoremQATheoremsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 23904,
+        "number_of_characters": 20825122,
+        "documents_text_statistics": {
+            "total_text_length": 20797224,
+            "min_text_length": 74,
+            "average_text_length": 872.4033726246906,
+            "max_text_length": 19104,
+            "unique_texts": 23839
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 27898,
+            "min_text_length": 13,
+            "average_text_length": 429.2,
+            "max_text_length": 1255,
+            "unique_texts": 65
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 126,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.9384615384615385,
+            "max_relevant_docs_per_query": 6,
+            "unique_relevant_docs": 95
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 1549535,
+            "min_top_ranked_per_query": 23839,
+            "average_top_ranked_per_query": 23839.0,
+            "max_top_ranked_per_query": 23839
+        }
+    }
+}

mteb/models/model_implementations/align_models.py

@@ -116,6 +116,7 @@ align_base = ModelMeta(
     release_date="2023-02-24",
     modalities=["image", "text"],
     n_parameters=176_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=671,
     max_tokens=64,
     embed_dim=768,

mteb/models/model_implementations/amazon_models.py

@@ -8,6 +8,7 @@ amazon_titan_text_embeddings_v2 = ModelMeta(
     release_date="2024-04-30",
     languages=["eng-Latn"],
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     max_tokens=None,
     embed_dim=None,

mteb/models/model_implementations/andersborges.py

@@ -12,6 +12,7 @@ model2vecdk = ModelMeta(
     revision="cb576c78dcc1b729e4612645f61db59929d69e61",
     release_date="2025-11-21",
     n_parameters=48042496,
+    n_embedding_parameters=None,
     memory_usage_mb=183,
     max_tokens=np.inf,
     embed_dim=256,
@@ -43,6 +44,7 @@ model2vecdk_stem = ModelMeta(
     revision="cb576c78dcc1b729e4612645f61db59929d69e61",
     release_date="2025-11-21",
     n_parameters=48578560,
+    n_embedding_parameters=None,
     memory_usage_mb=185,
     max_tokens=np.inf,
     embed_dim=256,

mteb/models/model_implementations/ara_models.py

@@ -10,6 +10,7 @@ arabic_triplet_matryoshka = ModelMeta(
     revision="ed357f222f0b6ea6670d2c9b5a1cb93950d34200",
     release_date="2024-07-28",
     n_parameters=135_000_000,
+    n_embedding_parameters=49_152_000,
     memory_usage_mb=516,
     embed_dim=768,
     license="apache-2.0",

mteb/models/model_implementations/arctic_models.py

@@ -147,6 +147,7 @@ arctic_embed_xs = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=22_600_000,
+    n_embedding_parameters=11_720_448,
     memory_usage_mb=86,
     max_tokens=512,
     embed_dim=384,
@@ -173,6 +174,7 @@ arctic_embed_s = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=32_200_000,
+    n_embedding_parameters=11_720_448,
     memory_usage_mb=127,
     max_tokens=512,
     embed_dim=384,
@@ -199,6 +201,7 @@ arctic_embed_m = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=109_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=415,
     max_tokens=512,
     embed_dim=768,
@@ -225,6 +228,7 @@ arctic_embed_m_long = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=137_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=522,
     max_tokens=2048,
     embed_dim=768,
@@ -250,6 +254,7 @@ arctic_embed_l = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=335_000_000,
+    n_embedding_parameters=31_254_528,
     memory_usage_mb=1274,
     max_tokens=512,
     embed_dim=1024,
@@ -280,6 +285,7 @@ arctic_embed_m_v1_5 = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors", "GGUF"],
     n_parameters=109_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=415,
     max_tokens=512,
     embed_dim=768,
@@ -306,6 +312,7 @@ arctic_embed_m_v2_0 = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=305_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=1165,
     max_tokens=8192,
     embed_dim=768,
@@ -331,6 +338,7 @@ arctic_embed_l_v2_0 = ModelMeta(
     open_weights=True,
     framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     n_parameters=568_000_000,
+    n_embedding_parameters=256_002_048,
     memory_usage_mb=2166,
     max_tokens=8192,
     embed_dim=1024,

mteb/models/model_implementations/b1ade_models.py

@@ -16,6 +16,7 @@ b1ade_embed = ModelMeta(
     open_weights=True,
     release_date="2025-03-10",
     n_parameters=335_000_000,
+    n_embedding_parameters=31_254_528,
     memory_usage_mb=1278,
     embed_dim=1024,
     license="mit",

mteb/models/model_implementations/bedrock_models.py

@@ -179,6 +179,7 @@ amazon_titan_embed_text_v1 = ModelMeta(
     embed_dim=1536,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     public_training_code=None,
     public_training_data=None,  # assumed
@@ -206,6 +207,7 @@ amazon_titan_embed_text_v2 = ModelMeta(
     embed_dim=1024,
     open_weights=False,
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     public_training_code=None,
     public_training_data=None,  # assumed
@@ -235,6 +237,7 @@ cohere_embed_english_v3 = ModelMeta(
     revision="1",
     release_date="2023-11-02",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     public_training_code=None,
     public_training_data=None,  # assumed
@@ -263,6 +266,7 @@ cohere_embed_multilingual_v3 = ModelMeta(
     revision="1",
     release_date="2023-11-02",
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     public_training_code=None,
     public_training_data=None,  # assumed

mteb/models/model_implementations/bge_models.py

@@ -6,7 +6,29 @@ from mteb.models.sentence_transformer_wrapper import sentence_transformers_loade
 
 from .e5_instruct import E5_MISTRAL_TRAINING_DATA
 
-model_prompts = {"query": "Represent this sentence for searching relevant passages: "}
+model_prompts = {
+    "query": "Represent this sentence for searching relevant passages: ",
+    "BrightBiologyRetrieval-query": "Represent this biology post for searching relevant passages: ",
+    "BrightEarthScienceRetrieval-query": "Represent this earth_science post for searching relevant passages: ",
+    "BrightEconomicsRetrieval-query": "Represent this economics post for searching relevant passages: ",
+    "BrightPsychologyRetrieval-query": "Represent this psychology post for searching relevant passages: ",
+    "BrightRoboticsRetrieval-query": "Represent this robotics post for searching relevant passages: ",
+    "BrightStackoverflowRetrieval-query": "Represent this stackoverflow post for searching relevant passages: ",
+    "BrightSustainableLivingRetrieval-query": "Represent this sustainable_living post for searching relevant passages: ",
+    "BrightPonyRetrieval-query": "Represent this Pony question for searching relevant passages: ",
+    "BrightLeetcodeRetrieval-query": "Represent this Coding problem for searching relevant examples: ",
+    "BrightAopsRetrieval-query": "Represent this Math problem for searching relevant examples: ",
+    "BrightTheoremQATheoremsRetrieval-query": "Represent this Math problem for searching relevant theorems: ",
+    "BrightTheoremQAQuestionsRetrieval-query": "Represent this Math problem for searching relevant examples: ",
+    "BrightBiologyLongRetrieval-query": "Represent this biology post for searching relevant documents: ",
+    "BrightEarthScienceLongRetrieval-query": "Represent this earth_science post for searching relevant documents: ",
+    "BrightEconomicsLongRetrieval-query": "Represent this economics post for searching relevant documents: ",
+    "BrightPsychologyLongRetrieval-query": "Represent this psychology post for searching relevant documents: ",
+    "BrightRoboticsLongRetrieval-query": "Represent this robotics post for searching relevant document: ",
+    "BrightStackoverflowLongRetrieval-query": "Represent this stackoverflow post for searching relevant document: ",
+    "BrightSustainableLivingLongRetrieval-query": "Represent this sustainable_living post for searching relevant documents: ",
+    "BrightPonyLongRetrieval-query": "Represent this Pony question for searching relevant documents: ",
+}
 BGE_15_CITATION = """@misc{bge_embedding,
       title={C-Pack: Packaged Resources To Advance General Chinese Embedding},
       author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff},
@@ -325,6 +347,7 @@ bge_small_en_v1_5 = ModelMeta(
     revision="5c38ec7c405ec4b44b94cc5a9bb96e735b38267a",
     release_date="2023-09-12",  # initial commit of hf model.
     n_parameters=33_400_000,
+    n_embedding_parameters=11_720_448,
     memory_usage_mb=127,
     embed_dim=512,
     license="mit",
@@ -357,6 +380,7 @@ bge_base_en_v1_5 = ModelMeta(
     revision="a5beb1e3e68b9ab74eb54cfd186867f64f240e1a",
     release_date="2023-09-11",  # initial commit of hf model.
     n_parameters=109_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=390,
     embed_dim=768,
     license="mit",
@@ -389,6 +413,7 @@ bge_large_en_v1_5 = ModelMeta(
     revision="d4aa6901d3a41ba39fb536a557fa166f842b0e09",
     release_date="2023-09-12",  # initial commit of hf model.
     n_parameters=335_000_000,
+    n_embedding_parameters=31_254_528,
     memory_usage_mb=1242,
     embed_dim=1024,
     license="mit",
@@ -421,6 +446,7 @@ bge_small_zh = ModelMeta(
     revision="1d2363c5de6ce9ba9c890c8e23a4c72dce540ca8",
     release_date="2023-08-05",  # initial commit of hf model.
     n_parameters=33_400_000,
+    n_embedding_parameters=10_817_536,
     memory_usage_mb=127,
     embed_dim=512,
     license="mit",
@@ -448,6 +474,7 @@ bge_base_zh = ModelMeta(
     revision="0e5f83d4895db7955e4cb9ed37ab73f7ded339b6",
     release_date="2023-08-05",  # initial commit of hf model.
     n_parameters=109_000_000,
+    n_embedding_parameters=16_226_304,
     memory_usage_mb=390,
     embed_dim=768,
     license="mit",
@@ -475,6 +502,7 @@ bge_large_zh = ModelMeta(
     revision="b5d9f5c027e87b6f0b6fa4b614f8f9cdc45ce0e8",
     release_date="2023-08-02",  # initial commit of hf model.
     n_parameters=335_000_000,
+    n_embedding_parameters=21_635_072,
     memory_usage_mb=1242,
     embed_dim=1024,
     license="mit",
@@ -502,6 +530,7 @@ bge_small_en = ModelMeta(
     revision="4778d71a06863076696b03fd2777eb118712cad8",
     release_date="2023-08-05",  # initial commit of hf model.
     n_parameters=33_400_000,
+    n_embedding_parameters=11_720_448,
     memory_usage_mb=127,
     embed_dim=512,
     license="mit",
@@ -529,6 +558,7 @@ bge_base_en = ModelMeta(
     revision="b737bf5dcc6ee8bdc530531266b4804a5d77b5d8",
     release_date="2023-08-05",  # initial commit of hf model.
     n_parameters=109_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=390,
     embed_dim=768,
     license="mit",
@@ -562,6 +592,7 @@ bge_large_en = ModelMeta(
     revision="abe7d9d814b775ca171121fb03f394dc42974275",
     release_date="2023-08-05",  # initial commit of hf model.
     n_parameters=335_000_000,
+    n_embedding_parameters=31_254_528,
     memory_usage_mb=1242,
     embed_dim=1024,
     license="mit",
@@ -590,6 +621,7 @@ bge_small_zh_v1_5 = ModelMeta(
     revision="7999e1d3359715c523056ef9478215996d62a620",
     release_date="2023-09-12",  # initial commit of hf model.
     n_parameters=33_400_000,
+    n_embedding_parameters=10_817_536,
     memory_usage_mb=91,
     embed_dim=512,
     license="mit",
@@ -616,6 +648,7 @@ bge_base_zh_v1_5 = ModelMeta(
     revision="f03589ceff5aac7111bd60cfc7d497ca17ecac65",
     release_date="2023-09-11",  # initial commit of hf model.
     n_parameters=109_000_000,
+    n_embedding_parameters=16_226_304,
     memory_usage_mb=416,
     embed_dim=768,
     license="mit",
@@ -642,6 +675,7 @@ bge_large_zh_v1_5 = ModelMeta(
     revision="79e7739b6ab944e86d6171e44d24c997fc1e0116",
     release_date="2023-09-12",  # initial commit of hf model.
     n_parameters=335_000_000,
+    n_embedding_parameters=21_635_072,
     memory_usage_mb=1278,
     embed_dim=1024,
     license="mit",
@@ -665,6 +699,7 @@ bge_m3 = ModelMeta(
     revision="5617a9f61b028005a4858fdac845db406aefb181",
     release_date="2024-06-28",
     n_parameters=568_000_000,
+    n_embedding_parameters=256_002_048,
     memory_usage_mb=2167,
     embed_dim=1024,
     license="mit",
@@ -761,6 +796,7 @@ bge_multilingual_gemma2 = ModelMeta(
     revision="992e13d8984fde2c31ef8a3cb2c038aeec513b8a",
     release_date="2024-07-25",  # initial commit of hf model.
     n_parameters=int(9.24 * 1e9),
+    n_embedding_parameters=917_511_168,
     memory_usage_mb=35254,
     embed_dim=3584,  # from old C-MTEB leaderboard
     license="https://ai.google.dev/gemma/terms",
@@ -808,6 +844,7 @@ bge_en_icl = ModelMeta(
     revision="971c7e1445cc86656ca0bd85ed770b8675a40bb5",
     release_date="2024-07-25",  # initial commit of hf model.
     n_parameters=int(7.11 * 1e9),
+    n_embedding_parameters=131_084_288,
     memory_usage_mb=27125,
     embed_dim=4096,
     license="apache-2.0",
@@ -842,6 +879,7 @@ bge_m3_unsupervised = ModelMeta(
     revision="46f03bc86361cf88102b0b517b36c8259f2946b1",
     release_date="2024-01-30",  # January 30, 2024 - BGE-M3 release date
     n_parameters=568_000_000,
+    n_embedding_parameters=256_002_048,
     memory_usage_mb=2167,
     embed_dim=1024,
     license="mit",
@@ -871,6 +909,7 @@ manu__bge_m3_custom_fr = ModelMeta(
     languages=None,
     loader=sentence_transformers_loader,
     n_parameters=567754752,
+    n_embedding_parameters=256_002_048,
     memory_usage_mb=2166,
     max_tokens=8194.0,
     embed_dim=1024,

mteb/models/model_implementations/bica_model.py

@@ -9,6 +9,7 @@ bica_base = ModelMeta(
     revision="31237a836e5ae908c308a256573e5f0986498574",
     release_date="2025-11-14",
     n_parameters=110_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=418,
     embed_dim=768,
     license="mit",

mteb/models/model_implementations/blip2_models.py

@@ -177,6 +177,7 @@ blip2_opt_2_7b = ModelMeta(
     release_date="2024-03-22",
     modalities=["image", "text"],
     n_parameters=3_740_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=14285,
     max_tokens=None,
     embed_dim=768,
@@ -201,6 +202,7 @@ blip2_opt_6_7b_coco = ModelMeta(
     release_date="2024-03-31",
     modalities=["image", "text"],
     n_parameters=7_750_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=29577,
     max_tokens=None,
     embed_dim=768,

mteb/models/model_implementations/blip_models.py

@@ -141,6 +141,7 @@ blip_image_captioning_large = ModelMeta(
     release_date="2023-12-07",
     modalities=["image", "text"],
     n_parameters=470_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1792,
     max_tokens=512,
     embed_dim=768,
@@ -169,6 +170,7 @@ blip_image_captioning_base = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -198,6 +200,7 @@ blip_vqa_base = ModelMeta(
     release_date="2023-12-07",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1467,
     max_tokens=512,
     embed_dim=768,
@@ -225,6 +228,7 @@ blip_vqa_capfilt_large = ModelMeta(
     release_date="2023-01-22",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -252,6 +256,7 @@ blip_itm_base_coco = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -279,6 +284,7 @@ blip_itm_large_coco = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=470_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1793,
     max_tokens=512,
     embed_dim=768,
@@ -307,6 +313,7 @@ blip_itm_base_flickr = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=247_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=942,
     max_tokens=512,
     embed_dim=768,
@@ -335,6 +342,7 @@ blip_itm_large_flickr = ModelMeta(
     release_date="2023-08-01",
     modalities=["image", "text"],
     n_parameters=470_000_000,
+    n_embedding_parameters=23_442_432,
     memory_usage_mb=1793,
     max_tokens=512,
     embed_dim=768,

mteb/models/model_implementations/bm25.py

@@ -13,7 +13,6 @@ if TYPE_CHECKING:
     from mteb.types import (
         CorpusDatasetType,
         EncodeKwargs,
-        InstructionDatasetType,
         QueryDatasetType,
         RetrievalOutputType,
         TopRankedDocumentsType,
@@ -80,7 +79,6 @@ def bm25_loader(model_name, **kwargs) -> SearchProtocol:
             hf_subset: str,
             top_k: int,
             encode_kwargs: EncodeKwargs,
-            instructions: InstructionDatasetType | None = None,
             top_ranked: TopRankedDocumentsType | None = None,
         ) -> RetrievalOutputType:
             logger.info("Encoding Queries...")
@@ -103,13 +101,17 @@ def bm25_loader(model_name, **kwargs) -> SearchProtocol:
                 query_results = queries_results[qi]
                 scores = queries_scores[qi]
                 doc_id_to_score = {}
+                query_documents = (
+                    top_ranked[qid] if top_ranked and qid in top_ranked else None
+                )
 
                 # Iterate over results
-                for ri in range(len(query_results)):
-                    doc_idx = query_results[ri]
-                    score = scores[ri]
+                for doc_idx, score in zip(query_results, scores):
                     doc_id = self.corpus_idx_to_id[doc_idx]
 
+                    # handle reranking with a filtered set of documents
+                    if query_documents is not None and doc_id not in query_documents:
+                        continue
                     doc_id_to_score[doc_id] = float(score)
 
                 results[qid] = doc_id_to_score
@@ -132,6 +134,7 @@ bm25_s = ModelMeta(
     revision="0_1_10",
     release_date="2024-07-10",  # release of version 0.1.10
     n_parameters=None,
+    n_embedding_parameters=None,
     memory_usage_mb=None,
     embed_dim=None,
     license=None,

mteb/models/model_implementations/bmretriever_models.py

@@ -103,6 +103,7 @@ BMRetriever_410M = ModelMeta(
     release_date="2024-04-29",
     embed_dim=1024,
     n_parameters=353_822_720,
+    n_embedding_parameters=51_511_296,
     memory_usage_mb=1349,
     max_tokens=2048,
     license="mit",
@@ -133,6 +134,7 @@ BMRetriever_1B = ModelMeta(
     release_date="2024-04-29",
     embed_dim=2048,
     n_parameters=908_759_040,
+    n_embedding_parameters=103_022_592,
     memory_usage_mb=3466,
     max_tokens=2048,
     license="mit",
@@ -163,6 +165,7 @@ BMRetriever_2B = ModelMeta(
     release_date="2024-04-29",
     embed_dim=2048,
     n_parameters=2_506_172_416,
+    n_embedding_parameters=524_288_000,
     memory_usage_mb=9560,
     max_tokens=8192,
     license="mit",
@@ -193,6 +196,7 @@ BMRetriever_7B = ModelMeta(
     release_date="2024-04-29",
     embed_dim=4096,
     n_parameters=7_110_660_096,
+    n_embedding_parameters=131_072_000,
     memory_usage_mb=27124,
     max_tokens=32768,
     license="mit",

mteb/models/model_implementations/cadet_models.py

@@ -41,6 +41,7 @@ cadet_embed = ModelMeta(
     open_weights=True,
     release_date="2025-05-11",
     n_parameters=109_000_000,
+    n_embedding_parameters=23_440_896,
     memory_usage_mb=418,
     embed_dim=768,
     license="apache-2.0",

mteb/models/model_implementations/cde_models.py

@@ -226,6 +226,7 @@ cde_small_v1 = ModelMeta(
     revision="e151df18af0d7f1d1c37b074fee58406ececf19f",
     release_date="2024-09-24",
     n_parameters=int(281 * 1e6),
+    n_embedding_parameters=None,
     memory_usage_mb=1072,  # Though the second-stage model is only 140M
     max_tokens=512,
     embed_dim=768,
@@ -255,6 +256,7 @@ cde_small_v2 = ModelMeta(
     revision="4e1d021a6c3fd7ce8aa0a7204057eee5ae61d390",
     release_date="2025-01-13",
     n_parameters=int(306 * 1e6),
+    n_embedding_parameters=None,
     memory_usage_mb=1166,  # Though the second-stage model is only 140M
     max_tokens=512,
     embed_dim=768,

mteb/models/model_implementations/clip_models.py

@@ -128,6 +128,7 @@ clip_vit_large_patch14 = ModelMeta(
     release_date="2021-02-26",
     modalities=["image", "text"],
     n_parameters=428_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=1631,
     max_tokens=77,
     embed_dim=768,
@@ -152,6 +153,7 @@ clip_vit_base_patch32 = ModelMeta(
     release_date="2021-02-26",
     modalities=["image", "text"],
     n_parameters=151_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=576,
     max_tokens=77,
     embed_dim=512,
@@ -176,6 +178,7 @@ clip_vit_base_patch16 = ModelMeta(
     release_date="2021-02-26",
     modalities=["image", "text"],
     n_parameters=151_000_000,
+    n_embedding_parameters=None,
     memory_usage_mb=576,
     max_tokens=77,
     embed_dim=512,