mteb 2.7.2__py3-none-any.whl → 2.7.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. mteb/_create_dataloaders.py +16 -9
  2. mteb/_evaluators/any_sts_evaluator.py +10 -5
  3. mteb/_evaluators/clustering_evaluator.py +10 -4
  4. mteb/_evaluators/evaluator.py +9 -4
  5. mteb/_evaluators/image/imagetext_pairclassification_evaluator.py +6 -4
  6. mteb/_evaluators/pair_classification_evaluator.py +10 -5
  7. mteb/_evaluators/retrieval_evaluator.py +19 -13
  8. mteb/_evaluators/retrieval_metrics.py +9 -3
  9. mteb/_evaluators/sklearn_evaluator.py +14 -10
  10. mteb/_evaluators/text/bitext_mining_evaluator.py +8 -3
  11. mteb/_evaluators/text/summarization_evaluator.py +8 -4
  12. mteb/_evaluators/zeroshot_classification_evaluator.py +10 -3
  13. mteb/_helpful_enum.py +5 -1
  14. mteb/abstasks/_data_filter/filters.py +8 -2
  15. mteb/abstasks/_data_filter/task_pipelines.py +7 -2
  16. mteb/abstasks/_statistics_calculation.py +6 -4
  17. mteb/abstasks/abstask.py +17 -9
  18. mteb/abstasks/aggregate_task_metadata.py +20 -9
  19. mteb/abstasks/aggregated_task.py +15 -8
  20. mteb/abstasks/classification.py +15 -6
  21. mteb/abstasks/clustering.py +17 -8
  22. mteb/abstasks/clustering_legacy.py +14 -6
  23. mteb/abstasks/image/image_text_pair_classification.py +17 -7
  24. mteb/abstasks/multilabel_classification.py +11 -5
  25. mteb/abstasks/pair_classification.py +19 -9
  26. mteb/abstasks/regression.py +14 -6
  27. mteb/abstasks/retrieval.py +28 -17
  28. mteb/abstasks/retrieval_dataset_loaders.py +11 -8
  29. mteb/abstasks/sts.py +19 -10
  30. mteb/abstasks/task_metadata.py +17 -8
  31. mteb/abstasks/text/bitext_mining.py +14 -7
  32. mteb/abstasks/text/summarization.py +17 -7
  33. mteb/abstasks/zeroshot_classification.py +15 -7
  34. mteb/benchmarks/_create_table.py +13 -3
  35. mteb/benchmarks/benchmark.py +11 -1
  36. mteb/benchmarks/benchmarks/__init__.py +2 -0
  37. mteb/benchmarks/benchmarks/benchmarks.py +41 -2
  38. mteb/benchmarks/benchmarks/rteb_benchmarks.py +20 -9
  39. mteb/cache.py +10 -5
  40. mteb/cli/_display_tasks.py +9 -3
  41. mteb/cli/build_cli.py +5 -2
  42. mteb/cli/generate_model_card.py +9 -2
  43. mteb/deprecated_evaluator.py +16 -12
  44. mteb/descriptive_stats/Retrieval/BrightAopsRetrieval.json +35 -0
  45. mteb/descriptive_stats/Retrieval/BrightBiologyLongRetrieval.json +35 -0
  46. mteb/descriptive_stats/Retrieval/BrightBiologyRetrieval.json +35 -0
  47. mteb/descriptive_stats/Retrieval/BrightEarthScienceLongRetrieval.json +35 -0
  48. mteb/descriptive_stats/Retrieval/BrightEarthScienceRetrieval.json +35 -0
  49. mteb/descriptive_stats/Retrieval/BrightEconomicsLongRetrieval.json +35 -0
  50. mteb/descriptive_stats/Retrieval/BrightEconomicsRetrieval.json +35 -0
  51. mteb/descriptive_stats/Retrieval/BrightLeetcodeRetrieval.json +35 -0
  52. mteb/descriptive_stats/Retrieval/BrightPonyLongRetrieval.json +35 -0
  53. mteb/descriptive_stats/Retrieval/BrightPonyRetrieval.json +35 -0
  54. mteb/descriptive_stats/Retrieval/BrightPsychologyLongRetrieval.json +35 -0
  55. mteb/descriptive_stats/Retrieval/BrightPsychologyRetrieval.json +35 -0
  56. mteb/descriptive_stats/Retrieval/BrightRoboticsLongRetrieval.json +35 -0
  57. mteb/descriptive_stats/Retrieval/BrightRoboticsRetrieval.json +35 -0
  58. mteb/descriptive_stats/Retrieval/BrightStackoverflowLongRetrieval.json +35 -0
  59. mteb/descriptive_stats/Retrieval/BrightStackoverflowRetrieval.json +35 -0
  60. mteb/descriptive_stats/Retrieval/BrightSustainableLivingLongRetrieval.json +35 -0
  61. mteb/descriptive_stats/Retrieval/BrightSustainableLivingRetrieval.json +35 -0
  62. mteb/descriptive_stats/Retrieval/BrightTheoremQAQuestionsRetrieval.json +35 -0
  63. mteb/descriptive_stats/Retrieval/BrightTheoremQATheoremsRetrieval.json +35 -0
  64. mteb/evaluate.py +20 -18
  65. mteb/filter_tasks.py +12 -7
  66. mteb/get_tasks.py +9 -4
  67. mteb/languages/language_scripts.py +8 -3
  68. mteb/leaderboard/app.py +7 -3
  69. mteb/leaderboard/table.py +7 -2
  70. mteb/load_results.py +9 -3
  71. mteb/models/abs_encoder.py +22 -12
  72. mteb/models/cache_wrappers/cache_backend_protocol.py +5 -3
  73. mteb/models/cache_wrappers/cache_backends/_hash_utils.py +8 -4
  74. mteb/models/cache_wrappers/cache_backends/faiss_cache.py +8 -3
  75. mteb/models/cache_wrappers/cache_wrapper.py +14 -9
  76. mteb/models/get_model_meta.py +11 -4
  77. mteb/models/instruct_wrapper.py +13 -5
  78. mteb/models/model_implementations/align_models.py +10 -4
  79. mteb/models/model_implementations/amazon_models.py +1 -0
  80. mteb/models/model_implementations/andersborges.py +2 -0
  81. mteb/models/model_implementations/ara_models.py +1 -0
  82. mteb/models/model_implementations/arctic_models.py +8 -0
  83. mteb/models/model_implementations/b1ade_models.py +1 -0
  84. mteb/models/model_implementations/bedrock_models.py +20 -6
  85. mteb/models/model_implementations/bge_models.py +40 -1
  86. mteb/models/model_implementations/bica_model.py +1 -0
  87. mteb/models/model_implementations/blip2_models.py +11 -4
  88. mteb/models/model_implementations/blip_models.py +17 -4
  89. mteb/models/model_implementations/bm25.py +22 -14
  90. mteb/models/model_implementations/bmretriever_models.py +10 -2
  91. mteb/models/model_implementations/cadet_models.py +1 -0
  92. mteb/models/model_implementations/cde_models.py +11 -5
  93. mteb/models/model_implementations/clip_models.py +12 -4
  94. mteb/models/model_implementations/clips_models.py +3 -0
  95. mteb/models/model_implementations/codefuse_models.py +5 -0
  96. mteb/models/model_implementations/codesage_models.py +3 -0
  97. mteb/models/model_implementations/cohere_models.py +14 -4
  98. mteb/models/model_implementations/cohere_v.py +14 -4
  99. mteb/models/model_implementations/colpali_models.py +7 -3
  100. mteb/models/model_implementations/colqwen_models.py +17 -31
  101. mteb/models/model_implementations/colsmol_models.py +3 -1
  102. mteb/models/model_implementations/conan_models.py +11 -4
  103. mteb/models/model_implementations/dino_models.py +28 -4
  104. mteb/models/model_implementations/e5_instruct.py +4 -0
  105. mteb/models/model_implementations/e5_models.py +9 -0
  106. mteb/models/model_implementations/e5_v.py +10 -4
  107. mteb/models/model_implementations/eagerworks_models.py +11 -4
  108. mteb/models/model_implementations/emillykkejensen_models.py +3 -0
  109. mteb/models/model_implementations/en_code_retriever.py +1 -0
  110. mteb/models/model_implementations/euler_models.py +1 -0
  111. mteb/models/model_implementations/evaclip_models.py +13 -4
  112. mteb/models/model_implementations/fa_models.py +9 -0
  113. mteb/models/model_implementations/facebookai.py +2 -0
  114. mteb/models/model_implementations/geogpt_models.py +1 -0
  115. mteb/models/model_implementations/gme_v_models.py +7 -3
  116. mteb/models/model_implementations/google_models.py +15 -4
  117. mteb/models/model_implementations/granite_vision_embedding_models.py +7 -5
  118. mteb/models/model_implementations/gritlm_models.py +2 -0
  119. mteb/models/model_implementations/gte_models.py +9 -0
  120. mteb/models/model_implementations/hinvec_models.py +6 -1
  121. mteb/models/model_implementations/human.py +1 -0
  122. mteb/models/model_implementations/ibm_granite_models.py +6 -0
  123. mteb/models/model_implementations/inf_models.py +2 -0
  124. mteb/models/model_implementations/jasper_models.py +14 -5
  125. mteb/models/model_implementations/jina_clip.py +10 -4
  126. mteb/models/model_implementations/jina_models.py +17 -5
  127. mteb/models/model_implementations/kalm_models.py +24 -12
  128. mteb/models/model_implementations/kblab.py +1 -0
  129. mteb/models/model_implementations/kennethenevoldsen_models.py +2 -0
  130. mteb/models/model_implementations/kfst.py +1 -0
  131. mteb/models/model_implementations/kowshik24_models.py +1 -0
  132. mteb/models/model_implementations/lens_models.py +2 -0
  133. mteb/models/model_implementations/lgai_embedding_models.py +1 -0
  134. mteb/models/model_implementations/linq_models.py +7 -1
  135. mteb/models/model_implementations/listconranker.py +10 -4
  136. mteb/models/model_implementations/llm2clip_models.py +12 -4
  137. mteb/models/model_implementations/llm2vec_models.py +20 -6
  138. mteb/models/model_implementations/mcinext_models.py +8 -2
  139. mteb/models/model_implementations/mdbr_models.py +2 -0
  140. mteb/models/model_implementations/misc_models.py +63 -0
  141. mteb/models/model_implementations/mixedbread_ai_models.py +3 -0
  142. mteb/models/model_implementations/mme5_models.py +2 -1
  143. mteb/models/model_implementations/moco_models.py +11 -4
  144. mteb/models/model_implementations/mod_models.py +2 -1
  145. mteb/models/model_implementations/model2vec_models.py +23 -4
  146. mteb/models/model_implementations/moka_models.py +3 -0
  147. mteb/models/model_implementations/nbailab.py +3 -0
  148. mteb/models/model_implementations/no_instruct_sentence_models.py +13 -5
  149. mteb/models/model_implementations/nomic_models.py +16 -4
  150. mteb/models/model_implementations/nomic_models_vision.py +5 -3
  151. mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py +9 -3
  152. mteb/models/model_implementations/nvidia_models.py +15 -4
  153. mteb/models/model_implementations/octen_models.py +3 -1
  154. mteb/models/model_implementations/openai_models.py +14 -4
  155. mteb/models/model_implementations/openclip_models.py +17 -4
  156. mteb/models/model_implementations/opensearch_neural_sparse_models.py +15 -4
  157. mteb/models/model_implementations/ops_moa_models.py +9 -2
  158. mteb/models/model_implementations/ordalietech_solon_embeddings_mini_beta_1_1.py +1 -0
  159. mteb/models/model_implementations/pawan_models.py +1 -0
  160. mteb/models/model_implementations/piccolo_models.py +2 -0
  161. mteb/models/model_implementations/promptriever_models.py +16 -6
  162. mteb/models/model_implementations/pylate_models.py +22 -13
  163. mteb/models/model_implementations/qodo_models.py +2 -0
  164. mteb/models/model_implementations/qtack_models.py +1 -0
  165. mteb/models/model_implementations/qwen3_models.py +11 -1
  166. mteb/models/model_implementations/qzhou_models.py +2 -0
  167. mteb/models/model_implementations/random_baseline.py +4 -3
  168. mteb/models/model_implementations/rasgaard_models.py +1 -0
  169. mteb/models/model_implementations/reasonir_model.py +65 -0
  170. mteb/models/model_implementations/repllama_models.py +15 -6
  171. mteb/models/model_implementations/rerankers_custom.py +13 -4
  172. mteb/models/model_implementations/rerankers_monot5_based.py +24 -4
  173. mteb/models/model_implementations/richinfoai_models.py +1 -0
  174. mteb/models/model_implementations/ru_sentence_models.py +20 -0
  175. mteb/models/model_implementations/ruri_models.py +10 -0
  176. mteb/models/model_implementations/salesforce_models.py +10 -1
  177. mteb/models/model_implementations/samilpwc_models.py +1 -0
  178. mteb/models/model_implementations/sarashina_embedding_models.py +2 -0
  179. mteb/models/model_implementations/searchmap_models.py +1 -0
  180. mteb/models/model_implementations/seed_1_6_embedding_models.py +5 -2
  181. mteb/models/model_implementations/seed_1_6_embedding_models_1215.py +6 -2
  182. mteb/models/model_implementations/seed_models.py +2 -1
  183. mteb/models/model_implementations/sentence_transformers_models.py +18 -0
  184. mteb/models/model_implementations/shuu_model.py +1 -0
  185. mteb/models/model_implementations/siglip_models.py +19 -4
  186. mteb/models/model_implementations/slm_models.py +7 -4
  187. mteb/models/model_implementations/sonar_models.py +2 -1
  188. mteb/models/model_implementations/spartan8806_atles_champion.py +1 -0
  189. mteb/models/model_implementations/stella_models.py +6 -0
  190. mteb/models/model_implementations/tarka_models.py +2 -0
  191. mteb/models/model_implementations/text2vec_models.py +3 -0
  192. mteb/models/model_implementations/ua_sentence_models.py +1 -0
  193. mteb/models/model_implementations/uae_models.py +10 -4
  194. mteb/models/model_implementations/vdr_models.py +8 -1
  195. mteb/models/model_implementations/vi_vn_models.py +6 -0
  196. mteb/models/model_implementations/vista_models.py +11 -4
  197. mteb/models/model_implementations/vlm2vec_models.py +11 -4
  198. mteb/models/model_implementations/voyage_models.py +25 -4
  199. mteb/models/model_implementations/voyage_v.py +11 -6
  200. mteb/models/model_implementations/xyz_models.py +1 -0
  201. mteb/models/model_implementations/youtu_models.py +1 -0
  202. mteb/models/model_implementations/yuan_models.py +1 -0
  203. mteb/models/model_implementations/yuan_models_en.py +2 -1
  204. mteb/models/model_meta.py +47 -9
  205. mteb/models/models_protocols.py +19 -18
  206. mteb/models/search_encoder_index/search_backend_protocol.py +7 -3
  207. mteb/models/search_encoder_index/search_indexes/faiss_search_index.py +12 -4
  208. mteb/models/search_wrappers.py +19 -12
  209. mteb/models/sentence_transformer_wrapper.py +4 -3
  210. mteb/models/vllm_wrapper.py +8 -6
  211. mteb/results/benchmark_results.py +22 -17
  212. mteb/results/model_result.py +21 -15
  213. mteb/results/task_result.py +15 -9
  214. mteb/similarity_functions.py +8 -2
  215. mteb/tasks/aggregated_tasks/eng/cqadupstack_retrieval.py +3 -3
  216. mteb/tasks/aggregated_tasks/eng/sts17_multilingual_visual_sts_eng.py +3 -3
  217. mteb/tasks/aggregated_tasks/eng/sts_benchmark_multilingual_visual_sts_eng.py +3 -3
  218. mteb/tasks/aggregated_tasks/fas/cqadupstack_retrieval_fa.py +3 -3
  219. mteb/tasks/aggregated_tasks/fas/syn_per_chatbot_conv_sa_classification.py +3 -3
  220. mteb/tasks/aggregated_tasks/multilingual/sts17_multilingual_vision_sts.py +3 -3
  221. mteb/tasks/aggregated_tasks/multilingual/sts_benchmark_multilingual_visual_sts.py +3 -3
  222. mteb/tasks/aggregated_tasks/nld/cqadupstack_nl_retrieval.py +3 -3
  223. mteb/tasks/aggregated_tasks/pol/cqadupstack_retrieval_pl.py +3 -3
  224. mteb/tasks/clustering/nob/snl_clustering.py +7 -2
  225. mteb/tasks/clustering/nob/vg_clustering.py +7 -2
  226. mteb/tasks/retrieval/eng/__init__.py +42 -0
  227. mteb/tasks/retrieval/eng/bright_retrieval.py +9 -1
  228. mteb/tasks/retrieval/eng/bright_v1_1_retrieval.py +968 -0
  229. mteb/tasks/retrieval/eng/limit_retrieval.py +6 -1
  230. mteb/tasks/retrieval/multilingual/ru_sci_bench_retrieval.py +3 -3
  231. mteb/types/_encoder_io.py +1 -1
  232. mteb/types/statistics.py +9 -2
  233. {mteb-2.7.2.dist-info → mteb-2.7.4.dist-info}/METADATA +1 -1
  234. {mteb-2.7.2.dist-info → mteb-2.7.4.dist-info}/RECORD +238 -217
  235. {mteb-2.7.2.dist-info → mteb-2.7.4.dist-info}/WHEEL +0 -0
  236. {mteb-2.7.2.dist-info → mteb-2.7.4.dist-info}/entry_points.txt +0 -0
  237. {mteb-2.7.2.dist-info → mteb-2.7.4.dist-info}/licenses/LICENSE +0 -0
  238. {mteb-2.7.2.dist-info → mteb-2.7.4.dist-info}/top_level.txt +0 -0
@@ -1,30 +1,36 @@
1
+ from __future__ import annotations
2
+
1
3
  import heapq
2
4
  import logging
3
5
  import shutil
4
6
  import tempfile
5
7
  from pathlib import Path
6
- from typing import Any
8
+ from typing import TYPE_CHECKING, Any
7
9
 
8
10
  import torch
9
- from torch.utils.data import DataLoader
10
11
 
11
12
  from mteb._create_dataloaders import (
12
13
  create_dataloader,
13
14
  )
14
15
  from mteb._requires_package import requires_package
15
- from mteb.abstasks.task_metadata import TaskMetadata
16
16
  from mteb.models.abs_encoder import AbsEncoder
17
17
  from mteb.models.model_meta import ModelMeta, ScoringFunction
18
- from mteb.types import (
19
- Array,
20
- BatchedInput,
21
- CorpusDatasetType,
22
- EncodeKwargs,
23
- PromptType,
24
- QueryDatasetType,
25
- RetrievalOutputType,
26
- TopRankedDocumentsType,
27
- )
18
+ from mteb.types import PromptType
19
+
20
+ if TYPE_CHECKING:
21
+ from torch.utils.data import DataLoader
22
+
23
+ from mteb.abstasks.task_metadata import TaskMetadata
24
+ from mteb.types import (
25
+ Array,
26
+ BatchedInput,
27
+ CorpusDatasetType,
28
+ EncodeKwargs,
29
+ QueryDatasetType,
30
+ RetrievalOutputType,
31
+ TopRankedDocumentsType,
32
+ )
33
+
28
34
 
29
35
  logger = logging.getLogger(__name__)
30
36
 
@@ -346,6 +352,7 @@ colbert_v2 = ModelMeta(
346
352
  public_training_data=None,
347
353
  release_date="2024-09-21",
348
354
  n_parameters=int(110 * 1e6),
355
+ n_embedding_parameters=23_440_896,
349
356
  memory_usage_mb=418,
350
357
  max_tokens=180,
351
358
  embed_dim=None,
@@ -402,6 +409,7 @@ jina_colbert_v2 = ModelMeta(
402
409
  public_training_data=None,
403
410
  release_date="2024-08-16",
404
411
  n_parameters=int(559 * 1e6),
412
+ n_embedding_parameters=None,
405
413
  memory_usage_mb=1067,
406
414
  max_tokens=8192,
407
415
  embed_dim=None,
@@ -458,6 +466,7 @@ lightonai__gte_moderncolbert_v1 = ModelMeta(
458
466
  public_training_data="https://huggingface.co/datasets/lightonai/ms-marco-en-bge-gemma",
459
467
  release_date="2025-04-30",
460
468
  n_parameters=int(149 * 1e6),
469
+ n_embedding_parameters=None,
461
470
  memory_usage_mb=None,
462
471
  max_tokens=8192,
463
472
  embed_dim=None,
@@ -36,6 +36,7 @@ Qodo_Embed_1_1_5B = ModelMeta(
36
36
  revision="84bbef079b32e8823ec226d4e9e92902706b0eb6",
37
37
  release_date="2025-02-19",
38
38
  n_parameters=1_780_000_000,
39
+ n_embedding_parameters=232_928_256,
39
40
  memory_usage_mb=6776,
40
41
  embed_dim=1536,
41
42
  license="https://huggingface.co/Qodo/Qodo-Embed-1-1.5B/blob/main/LICENSE",
@@ -59,6 +60,7 @@ Qodo_Embed_1_7B = ModelMeta(
59
60
  revision="f9edd9bf7f687c0e832424058e265120f603cd81",
60
61
  release_date="2025-02-24",
61
62
  n_parameters=7_613_000_000,
63
+ n_embedding_parameters=None,
62
64
  memory_usage_mb=29040,
63
65
  embed_dim=3584,
64
66
  license="https://huggingface.co/Qodo/Qodo-Embed-1-1.5B/blob/main/LICENSE",
@@ -31,6 +31,7 @@ mini_gte = ModelMeta(
31
31
  revision="7fbe6f9b4cc42615e0747299f837ad7769025492",
32
32
  release_date="2025-01-28",
33
33
  n_parameters=int(66.3 * 1e6),
34
+ n_embedding_parameters=23_440_896,
34
35
  memory_usage_mb=253,
35
36
  embed_dim=768,
36
37
  license="apache-2.0",
@@ -1,6 +1,13 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING
4
+
1
5
  from mteb.models.instruct_wrapper import InstructSentenceTransformerModel
2
6
  from mteb.models.model_meta import ModelMeta
3
- from mteb.models.models_protocols import EncoderProtocol, PromptType
7
+ from mteb.types import PromptType
8
+
9
+ if TYPE_CHECKING:
10
+ from mteb.models.models_protocols import EncoderProtocol
4
11
 
5
12
 
6
13
  def instruction_template(
@@ -140,6 +147,7 @@ Qwen3_Embedding_0B6 = ModelMeta(
140
147
  revision="b22da495047858cce924d27d76261e96be6febc0", # Commit of @tomaarsen
141
148
  release_date="2025-06-05",
142
149
  n_parameters=595776512,
150
+ n_embedding_parameters=None,
143
151
  memory_usage_mb=1136,
144
152
  embed_dim=1024,
145
153
  max_tokens=32768,
@@ -163,6 +171,7 @@ Qwen3_Embedding_4B = ModelMeta(
163
171
  revision="636cd9bf47d976946cdbb2b0c3ca0cb2f8eea5ff", # Commit of @tomaarsen
164
172
  release_date="2025-06-05",
165
173
  n_parameters=4021774336,
174
+ n_embedding_parameters=None,
166
175
  memory_usage_mb=7671,
167
176
  embed_dim=2560,
168
177
  max_tokens=32768,
@@ -186,6 +195,7 @@ Qwen3_Embedding_8B = ModelMeta(
186
195
  revision="4e423935c619ae4df87b646a3ce949610c66241c", # Commit of @tomaarsen
187
196
  release_date="2025-06-05",
188
197
  n_parameters=7567295488,
198
+ n_embedding_parameters=None,
189
199
  memory_usage_mb=14433,
190
200
  embed_dim=4096,
191
201
  max_tokens=32768,
@@ -64,6 +64,7 @@ QZhou_Embedding = ModelMeta(
64
64
  revision="f1e6c03ee3882e7b9fa5cec91217715272e433b8",
65
65
  release_date="2025-08-24",
66
66
  n_parameters=7_070_619_136,
67
+ n_embedding_parameters=None,
67
68
  memory_usage_mb=14436,
68
69
  embed_dim=3584,
69
70
  license="apache-2.0",
@@ -98,6 +99,7 @@ QZhou_Embedding_Zh = ModelMeta(
98
99
  revision="0321ccb126413d1e49c5ce908e802b63d35f18e2",
99
100
  release_date="2025-09-28",
100
101
  n_parameters=7_575_747_328,
102
+ n_embedding_parameters=None,
101
103
  memory_usage_mb=29431,
102
104
  embed_dim=1792,
103
105
  license="apache-2.0",
@@ -5,18 +5,19 @@ from typing import TYPE_CHECKING, Any, Literal
5
5
 
6
6
  import numpy as np
7
7
  import torch
8
- from torch.utils.data import DataLoader
9
8
 
10
- from mteb.abstasks.task_metadata import TaskMetadata
11
9
  from mteb.models.model_meta import ModelMeta
12
10
  from mteb.similarity_functions import (
13
11
  select_pairwise_similarity,
14
12
  select_similarity,
15
13
  )
16
- from mteb.types._encoder_io import Array, BatchedInput, PromptType
17
14
 
18
15
  if TYPE_CHECKING:
19
16
  from PIL import Image
17
+ from torch.utils.data import DataLoader
18
+
19
+ from mteb.abstasks.task_metadata import TaskMetadata
20
+ from mteb.types._encoder_io import Array, BatchedInput, PromptType
20
21
 
21
22
 
22
23
  def _string_to_vector(text: str | None, size: int) -> np.ndarray:
@@ -12,6 +12,7 @@ potion_base_8m = ModelMeta(
12
12
  revision="387897cfb09992e6d45ea9cd7b28b9fcf119e23a",
13
13
  release_date="2025-10-08",
14
14
  n_parameters=22893312,
15
+ n_embedding_parameters=22893312,
15
16
  memory_usage_mb=87,
16
17
  max_tokens=np.inf,
17
18
  embed_dim=256,
@@ -36,12 +36,76 @@ REASONIR_TRAINING_DATA = {
36
36
  "DuRetrieval",
37
37
  "QuoraRetrieval",
38
38
  }
39
+ _prompts_dict = {
40
+ "BrightBiologyRetrieval": {
41
+ "query": "Given a Biology post, retrieve relevant passages that help answer the post"
42
+ },
43
+ "BrightEarthScienceRetrieval": {
44
+ "query": "Given a Earth Science post, retrieve relevant passages that help answer the post"
45
+ },
46
+ "BrightEconomicsRetrieval": {
47
+ "query": "Given a Economics post, retrieve relevant passages that help answer the post"
48
+ },
49
+ "BrightPsychologyRetrieval": {
50
+ "query": "Given a Psychology post, retrieve relevant passages that help answer the post"
51
+ },
52
+ "BrightRoboticsRetrieval": {
53
+ "query": "Given a Robotics post, retrieve relevant passages that help answer the post"
54
+ },
55
+ "BrightStackoverflowRetrieval": {
56
+ "query": "Given a Stackoverflow post, retrieve relevant passages that help answer the post"
57
+ },
58
+ "BrightSustainableLivingRetrieval": {
59
+ "query": "Given a Sustainable Living post, retrieve relevant passages that help answer the post"
60
+ },
61
+ "BrightPonyRetrieval": {
62
+ "query": "Given a Pony question, retrieve relevant passages that help answer the question"
63
+ },
64
+ "BrightLeetcodeRetrieval": {
65
+ "query": "Given a coding problem, retrieve relevant examples that help answer the problem",
66
+ },
67
+ "BrightAopsRetrieval": {
68
+ "query": "Given a Math problem, retrieve relevant examples that help answer the problem"
69
+ },
70
+ "BrightTheoremQATheoremsRetrieval": {
71
+ "query": "Given a Math problem, retrieve relevant theorems that help answer the problem",
72
+ },
73
+ "BrightTheoremQAQuestionsRetrieval": {
74
+ "query": "Given a Math problem, retrieve relevant examples that help answer the problem",
75
+ },
76
+ "BrightBiologyLongRetrieval": {
77
+ "query": "Given a Biology post, retrieve relevant documents that help answer the post"
78
+ },
79
+ "BrightEarthScienceLongRetrieval": {
80
+ "query": "Given a Earth Science post, retrieve relevant documents that help answer the post"
81
+ },
82
+ "BrightEconomicsLongRetrieval": {
83
+ "query": "Given a Economics post, retrieve relevant documents that help answer the post"
84
+ },
85
+ "BrightPsychologyLongRetrieval": {
86
+ "query": "Given a Psychology post, retrieve relevant documents that help answer the post"
87
+ },
88
+ "BrightRoboticsLongRetrieval": {
89
+ "query": "Given a Robotics post, retrieve relevant documents that help answer the post"
90
+ },
91
+ "BrightStackoverflowLongRetrieval": {
92
+ "query": "Given a Stackoverflow post, retrieve relevant documents that help answer the post"
93
+ },
94
+ "BrightSustainableLivingLongRetrieval": {
95
+ "query": "Given a Sustainable Living post, retrieve relevant documents that help answer the post"
96
+ },
97
+ "BrightPonyLongRetrieval": {
98
+ "query": "Given a Pony question, retrieve relevant documents that help answer the question"
99
+ },
100
+ }
101
+
39
102
 
40
103
  ReasonIR_8B = ModelMeta(
41
104
  loader=InstructSentenceTransformerModel,
42
105
  loader_kwargs=dict(
43
106
  instruction_template=instruction_template,
44
107
  trust_remote_code=True,
108
+ prompts_dict=_prompts_dict,
45
109
  ),
46
110
  name="ReasonIR/ReasonIR-8B",
47
111
  model_type=["dense"],
@@ -50,6 +114,7 @@ ReasonIR_8B = ModelMeta(
50
114
  revision="c3d0690370ff4a8c3d3882d8dfa85c43650034fa",
51
115
  release_date="2025-04-29",
52
116
  n_parameters=7_500_000_000,
117
+ n_embedding_parameters=None,
53
118
  memory_usage_mb=None,
54
119
  embed_dim=4096,
55
120
  license="cc-by-nc-4.0",
@@ -1,22 +1,29 @@
1
+ from __future__ import annotations
2
+
1
3
  import logging
2
- from collections.abc import Callable
3
- from typing import Any
4
+ from typing import TYPE_CHECKING, Any
4
5
 
5
6
  import numpy as np
6
7
  import torch
7
8
  import torch.nn.functional as F
8
- from torch.utils.data import DataLoader
9
9
  from tqdm.auto import tqdm
10
10
 
11
11
  from mteb._requires_package import requires_package
12
- from mteb.abstasks.task_metadata import TaskMetadata
13
12
  from mteb.models.abs_encoder import AbsEncoder
14
13
  from mteb.models.model_meta import (
15
14
  ModelMeta,
16
15
  ScoringFunction,
17
16
  )
18
- from mteb.models.models_protocols import EncoderProtocol
19
- from mteb.types import Array, BatchedInput, PromptType
17
+ from mteb.types import PromptType
18
+
19
+ if TYPE_CHECKING:
20
+ from collections.abc import Callable
21
+
22
+ from torch.utils.data import DataLoader
23
+
24
+ from mteb.abstasks.task_metadata import TaskMetadata
25
+ from mteb.models.models_protocols import EncoderProtocol
26
+ from mteb.types import Array, BatchedInput
20
27
 
21
28
  logger = logging.getLogger(__name__)
22
29
 
@@ -172,6 +179,7 @@ repllama_llama2_original = ModelMeta(
172
179
  "mMARCO-NL", # translation not trained on
173
180
  },
174
181
  n_parameters=7_000_000,
182
+ n_embedding_parameters=131_072_000,
175
183
  memory_usage_mb=27,
176
184
  max_tokens=4096,
177
185
  embed_dim=4096,
@@ -201,6 +209,7 @@ repllama_llama2_reproduced = ModelMeta(
201
209
  revision="01c7f73d771dfac7d292323805ebc428287df4f9-ad5c1d0938a1e02954bcafb4d811ba2f34052e71", # base-peft revision
202
210
  release_date="2024-09-15",
203
211
  n_parameters=7_000_000,
212
+ n_embedding_parameters=None,
204
213
  memory_usage_mb=27,
205
214
  max_tokens=4096,
206
215
  embed_dim=4096,
@@ -1,16 +1,22 @@
1
+ from __future__ import annotations
2
+
1
3
  import logging
2
- from typing import Any
4
+ from typing import TYPE_CHECKING, Any
3
5
 
4
6
  import torch
5
- from torch.utils.data import DataLoader
6
7
 
7
8
  from mteb._requires_package import requires_package
8
- from mteb.abstasks.task_metadata import TaskMetadata
9
9
  from mteb.models.model_meta import ModelMeta
10
- from mteb.types import Array, BatchedInput, PromptType
11
10
 
12
11
  from .bge_models import bge_m3_training_data
13
12
 
13
+ if TYPE_CHECKING:
14
+ from torch.utils.data import DataLoader
15
+
16
+ from mteb.abstasks.task_metadata import TaskMetadata
17
+ from mteb.types import Array, BatchedInput, PromptType
18
+
19
+
14
20
  logger = logging.getLogger(__name__)
15
21
 
16
22
 
@@ -225,6 +231,7 @@ monobert_large = ModelMeta(
225
231
  revision="0a97706f3827389da43b83348d5d18c9d53876fa",
226
232
  release_date="2020-05-28",
227
233
  n_parameters=None,
234
+ n_embedding_parameters=31_254_528,
228
235
  memory_usage_mb=None,
229
236
  max_tokens=None,
230
237
  embed_dim=None,
@@ -250,6 +257,7 @@ jina_reranker_multilingual = ModelMeta(
250
257
  revision="126747772a932960028d9f4dc93bd5d9c4869be4",
251
258
  release_date="2024-09-26",
252
259
  n_parameters=None,
260
+ n_embedding_parameters=None,
253
261
  memory_usage_mb=531,
254
262
  max_tokens=None,
255
263
  embed_dim=None,
@@ -313,6 +321,7 @@ bge_reranker_v2_m3 = ModelMeta(
313
321
  revision="953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e",
314
322
  release_date="2024-06-24",
315
323
  n_parameters=None,
324
+ n_embedding_parameters=256_002_048,
316
325
  memory_usage_mb=2166,
317
326
  max_tokens=None,
318
327
  embed_dim=None,
@@ -1,15 +1,21 @@
1
+ from __future__ import annotations
2
+
1
3
  import logging
2
- from typing import Any
4
+ from typing import TYPE_CHECKING, Any
3
5
 
4
6
  import torch
5
- from torch.utils.data import DataLoader
6
7
 
7
- from mteb.abstasks.task_metadata import TaskMetadata
8
8
  from mteb.models.model_meta import ModelMeta
9
- from mteb.types import Array, BatchedInput, PromptType
10
9
 
11
10
  from .rerankers_custom import RerankerWrapper
12
11
 
12
+ if TYPE_CHECKING:
13
+ from torch.utils.data import DataLoader
14
+
15
+ from mteb.abstasks.task_metadata import TaskMetadata
16
+ from mteb.types import Array, BatchedInput, PromptType
17
+
18
+
13
19
  logger = logging.getLogger(__name__)
14
20
 
15
21
 
@@ -321,6 +327,7 @@ monot5_small = ModelMeta(
321
327
  revision="77f8e3f7b1eb1afe353aa21a7c3a2fc8feca702e",
322
328
  release_date="2022-03-28",
323
329
  n_parameters=None,
330
+ n_embedding_parameters=16_449_536,
324
331
  memory_usage_mb=None,
325
332
  max_tokens=None,
326
333
  embed_dim=None,
@@ -363,6 +370,7 @@ monot5_base = ModelMeta(
363
370
  url={https://arxiv.org/abs/2206.02873},
364
371
  }""",
365
372
  n_parameters=None,
373
+ n_embedding_parameters=24_674_304,
366
374
  memory_usage_mb=None,
367
375
  max_tokens=None,
368
376
  embed_dim=None,
@@ -387,6 +395,7 @@ monot5_large = ModelMeta(
387
395
  revision="48cfad1d8dd587670393f27ee8ec41fde63e3d98",
388
396
  release_date="2022-03-28",
389
397
  n_parameters=None,
398
+ n_embedding_parameters=32_899_072,
390
399
  memory_usage_mb=None,
391
400
  max_tokens=None,
392
401
  embed_dim=None,
@@ -420,6 +429,7 @@ monot5_3b = ModelMeta(
420
429
  revision="bc0c419a438c81f592f878ce32430a1823f5db6c",
421
430
  release_date="2022-03-28",
422
431
  n_parameters=None,
432
+ n_embedding_parameters=32_899_072,
423
433
  memory_usage_mb=None,
424
434
  max_tokens=None,
425
435
  embed_dim=None,
@@ -476,6 +486,7 @@ flant5_base = ModelMeta(
476
486
  # "qed": ["train"],
477
487
  ),
478
488
  n_parameters=None,
489
+ n_embedding_parameters=24_674_304,
479
490
  memory_usage_mb=944,
480
491
  max_tokens=None,
481
492
  embed_dim=None,
@@ -522,6 +533,7 @@ flant5_large = ModelMeta(
522
533
  # "qed": ["train"],
523
534
  ),
524
535
  n_parameters=None,
536
+ n_embedding_parameters=32_899_072,
525
537
  memory_usage_mb=2987,
526
538
  max_tokens=None,
527
539
  embed_dim=None,
@@ -568,6 +580,7 @@ flant5_xl = ModelMeta(
568
580
  # "qed": ["train"],
569
581
  ),
570
582
  n_parameters=None,
583
+ n_embedding_parameters=65_798_144,
571
584
  memory_usage_mb=10871,
572
585
  max_tokens=None,
573
586
  embed_dim=None,
@@ -614,6 +627,7 @@ flant5_xxl = ModelMeta(
614
627
  # "qed": ["train"],
615
628
  ),
616
629
  n_parameters=None,
630
+ n_embedding_parameters=131_596_288,
617
631
  memory_usage_mb=42980,
618
632
  max_tokens=None,
619
633
  embed_dim=None,
@@ -638,6 +652,7 @@ llama2_7b = ModelMeta(
638
652
  revision="01c7f73d771dfac7d292323805ebc428287df4f9",
639
653
  release_date="2023-07-18",
640
654
  n_parameters=None,
655
+ n_embedding_parameters=131_072_000,
641
656
  memory_usage_mb=None,
642
657
  max_tokens=None,
643
658
  embed_dim=None,
@@ -680,6 +695,7 @@ llama2_7b_chat = ModelMeta(
680
695
  url={https://arxiv.org/abs/2307.09288},
681
696
  }""",
682
697
  n_parameters=None,
698
+ n_embedding_parameters=131_072_000,
683
699
  memory_usage_mb=None,
684
700
  max_tokens=None,
685
701
  embed_dim=None,
@@ -704,6 +720,7 @@ mistral_7b = ModelMeta(
704
720
  revision="3ad372fc79158a2148299e3318516c786aeded6c",
705
721
  release_date="2023-12-11",
706
722
  n_parameters=None,
723
+ n_embedding_parameters=None,
707
724
  memory_usage_mb=None,
708
725
  max_tokens=None,
709
726
  embed_dim=None,
@@ -740,6 +757,7 @@ followir_7b = ModelMeta(
740
757
  # "jhu-clsp/FollowIR-train"
741
758
  ),
742
759
  n_parameters=None,
760
+ n_embedding_parameters=None,
743
761
  memory_usage_mb=13813,
744
762
  max_tokens=None,
745
763
  embed_dim=None,
@@ -890,6 +908,7 @@ mt5_base_mmarco_v2 = ModelMeta(
890
908
  """,
891
909
  training_datasets={"MSMARCO"},
892
910
  n_parameters=None,
911
+ n_embedding_parameters=192_086_016,
893
912
  memory_usage_mb=None,
894
913
  max_tokens=None,
895
914
  embed_dim=None,
@@ -913,6 +932,7 @@ mt5_13b_mmarco_100k = ModelMeta(
913
932
  revision="e1a4317e102a525ea9e16745ad21394a4f1bffbc",
914
933
  release_date="2022-11-04",
915
934
  n_parameters=None,
935
+ n_embedding_parameters=1_024_458_752,
916
936
  memory_usage_mb=None,
917
937
  max_tokens=None,
918
938
  embed_dim=None,
@@ -15,6 +15,7 @@ ritrieve_zh_v1 = ModelMeta(
15
15
  revision="f8d5a707656c55705027678e311f9202c8ced12c",
16
16
  release_date="2025-03-25",
17
17
  n_parameters=int(326 * 1e6),
18
+ n_embedding_parameters=21_635_072,
18
19
  memory_usage_mb=1242,
19
20
  embed_dim=1792,
20
21
  license="mit",
@@ -244,6 +244,7 @@ rubert_tiny = ModelMeta(
244
244
  revision="5441c5ea8026d4f6d7505ec004845409f1259fb1",
245
245
  release_date="2021-05-24",
246
246
  n_parameters=11_900_000,
247
+ n_embedding_parameters=9_223_968,
247
248
  memory_usage_mb=45,
248
249
  embed_dim=312,
249
250
  license="mit",
@@ -270,6 +271,7 @@ rubert_tiny2 = ModelMeta(
270
271
  revision="dad72b8f77c5eef6995dd3e4691b758ba56b90c3",
271
272
  release_date="2021-10-28",
272
273
  n_parameters=29_400_000,
274
+ n_embedding_parameters=26_154_336,
273
275
  memory_usage_mb=112,
274
276
  embed_dim=312,
275
277
  license="mit",
@@ -297,6 +299,7 @@ sbert_large_nlu_ru = ModelMeta(
297
299
  revision="af977d5dfa46a3635e29bf0ef383f2df2a08d47a",
298
300
  release_date="2020-11-20",
299
301
  n_parameters=427_000_000,
302
+ n_embedding_parameters=123_021_312,
300
303
  memory_usage_mb=1629,
301
304
  embed_dim=1024,
302
305
  license="mit",
@@ -323,6 +326,7 @@ sbert_large_mt_nlu_ru = ModelMeta(
323
326
  revision="05300876c2b83f46d3ddd422a7f17e45cf633bb0",
324
327
  release_date="2021-05-18",
325
328
  n_parameters=427_000_000,
329
+ n_embedding_parameters=123_021_312,
326
330
  memory_usage_mb=1629,
327
331
  embed_dim=1024,
328
332
  license="not specified",
@@ -351,6 +355,7 @@ user_base_ru = ModelMeta(
351
355
  revision="436a489a2087d61aa670b3496a9915f84e46c861",
352
356
  release_date="2024-06-10",
353
357
  n_parameters=427_000_000,
358
+ n_embedding_parameters=38_603_520,
354
359
  memory_usage_mb=473,
355
360
  embed_dim=768,
356
361
  license="apache-2.0",
@@ -412,6 +417,7 @@ user_bge_m3 = ModelMeta(
412
417
  revision="0cc6cfe48e260fb0474c753087a69369e88709ae",
413
418
  release_date="2024-07-05",
414
419
  n_parameters=359_026_688,
420
+ n_embedding_parameters=47_273_984,
415
421
  memory_usage_mb=1370,
416
422
  embed_dim=1024,
417
423
  license="apache-2.0",
@@ -463,6 +469,7 @@ deberta_v1_ru = ModelMeta(
463
469
  revision="bdd30b0e19757e6940c92c7aff19e8fc0a60dff4",
464
470
  release_date="2023-02-07",
465
471
  n_parameters=124_000_000,
472
+ n_embedding_parameters=38_603_520,
466
473
  memory_usage_mb=473,
467
474
  embed_dim=768,
468
475
  license="apache-2.0",
@@ -494,6 +501,7 @@ rubert_base_cased = ModelMeta(
494
501
  revision="4036cab694767a299f2b9e6492909664d9414229",
495
502
  release_date="2020-03-04",
496
503
  n_parameters=1280_000_000,
504
+ n_embedding_parameters=91_812_096,
497
505
  memory_usage_mb=4883,
498
506
  embed_dim=768,
499
507
  license="not specified",
@@ -530,6 +538,7 @@ distilrubert_small_cased_conversational = ModelMeta(
530
538
  revision="e348066b4a7279b97138038299bddc6580a9169a",
531
539
  release_date="2022-06-28",
532
540
  n_parameters=107_000_000,
541
+ n_embedding_parameters=91_812_096,
533
542
  memory_usage_mb=408,
534
543
  embed_dim=768,
535
544
  license="not specified",
@@ -565,6 +574,7 @@ rubert_base_cased_sentence = ModelMeta(
565
574
  revision="78b5122d6365337dd4114281b0d08cd1edbb3bc8",
566
575
  release_date="2020-03-04",
567
576
  n_parameters=107_000_000,
577
+ n_embedding_parameters=91_812_096,
568
578
  memory_usage_mb=408,
569
579
  embed_dim=768,
570
580
  license="not specified",
@@ -590,6 +600,7 @@ labse_en_ru = ModelMeta(
590
600
  revision="cf0714e606d4af551e14ad69a7929cd6b0da7f7e",
591
601
  release_date="2021-06-10",
592
602
  n_parameters=129_000_000,
603
+ n_embedding_parameters=42_303_744,
593
604
  memory_usage_mb=492,
594
605
  embed_dim=768,
595
606
  license="not specified",
@@ -618,6 +629,7 @@ rubert_tiny_turbo = ModelMeta(
618
629
  revision="8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054",
619
630
  release_date="2024-06-21",
620
631
  n_parameters=29_200_000,
632
+ n_embedding_parameters=26_154_336,
621
633
  memory_usage_mb=111,
622
634
  embed_dim=312,
623
635
  license="mit",
@@ -641,6 +653,7 @@ rubert_mini_frida = ModelMeta(
641
653
  revision="19b279b78afd945b5ccae78f63e284909814adc2",
642
654
  release_date="2025-03-02",
643
655
  n_parameters=32_300_000,
656
+ n_embedding_parameters=26_154_336,
644
657
  memory_usage_mb=123,
645
658
  embed_dim=312,
646
659
  license="mit",
@@ -669,6 +682,7 @@ labse_ru_turbo = ModelMeta(
669
682
  revision="1940b046c6b5e125df11722b899130329d0a46da",
670
683
  release_date="2024-06-27",
671
684
  n_parameters=129_000_000,
685
+ n_embedding_parameters=42_303_744,
672
686
  memory_usage_mb=490,
673
687
  embed_dim=768,
674
688
  license="mit",
@@ -720,6 +734,7 @@ rosberta_ru_en = ModelMeta(
720
734
  use_instructions=True,
721
735
  reference="https://huggingface.co/ai-forever/ru-en-RoSBERTa",
722
736
  n_parameters=404_000_000,
737
+ n_embedding_parameters=100_869_120,
723
738
  memory_usage_mb=1540,
724
739
  max_tokens=512,
725
740
  embed_dim=1024,
@@ -886,6 +901,7 @@ frida = ModelMeta(
886
901
  use_instructions=True,
887
902
  reference="https://huggingface.co/ai-forever/FRIDA",
888
903
  n_parameters=823_000_000,
904
+ n_embedding_parameters=143_847_936,
889
905
  memory_usage_mb=3141,
890
906
  max_tokens=512,
891
907
  embed_dim=1536,
@@ -918,6 +934,7 @@ giga_embeddings = ModelMeta(
918
934
  revision="0ad5b29bfecd806cecc9d66b927d828a736594dc",
919
935
  release_date="2025-09-23",
920
936
  n_parameters=3_227_176_961,
937
+ n_embedding_parameters=None,
921
938
  memory_usage_mb=12865,
922
939
  embed_dim=2048,
923
940
  license="mit",
@@ -950,6 +967,7 @@ berta = ModelMeta(
950
967
  revision="914c8c8aed14042ed890fc2c662d5e9e66b2faa7",
951
968
  release_date="2025-03-10",
952
969
  n_parameters=128_000_000,
970
+ n_embedding_parameters=42_303_744,
953
971
  memory_usage_mb=489,
954
972
  embed_dim=768,
955
973
  license="mit",
@@ -1025,6 +1043,7 @@ user2_small = ModelMeta(
1025
1043
  use_instructions=True,
1026
1044
  reference="https://huggingface.co/collections/deepvk/user2-6802650d7210f222ec60e05f",
1027
1045
  n_parameters=34_400_000,
1046
+ n_embedding_parameters=None,
1028
1047
  memory_usage_mb=131,
1029
1048
  max_tokens=8192,
1030
1049
  embed_dim=384,
@@ -1058,6 +1077,7 @@ user2_base = ModelMeta(
1058
1077
  use_instructions=True,
1059
1078
  reference="https://huggingface.co/collections/deepvk/user2-6802650d7210f222ec60e05f",
1060
1079
  n_parameters=149_000_000,
1080
+ n_embedding_parameters=None,
1061
1081
  memory_usage_mb=568,
1062
1082
  max_tokens=8192,
1063
1083
  embed_dim=768,