mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (527) hide show
  1. mteb/__init__.py +6 -0
  2. mteb/_create_dataloaders.py +22 -20
  3. mteb/_evaluators/any_sts_evaluator.py +23 -14
  4. mteb/_evaluators/classification_metrics.py +54 -0
  5. mteb/_evaluators/clustering_evaluator.py +3 -3
  6. mteb/_evaluators/evaluator.py +4 -2
  7. mteb/_evaluators/image/imagetext_pairclassification_evaluator.py +18 -11
  8. mteb/_evaluators/pair_classification_evaluator.py +34 -40
  9. mteb/_evaluators/retrieval_evaluator.py +2 -2
  10. mteb/_evaluators/retrieval_metrics.py +18 -17
  11. mteb/_evaluators/sklearn_evaluator.py +25 -37
  12. mteb/_evaluators/text/bitext_mining_evaluator.py +31 -19
  13. mteb/_evaluators/text/summarization_evaluator.py +27 -20
  14. mteb/_evaluators/zeroshot_classification_evaluator.py +7 -5
  15. mteb/abstasks/_data_filter/__init__.py +0 -0
  16. mteb/abstasks/_data_filter/filters.py +125 -0
  17. mteb/abstasks/_data_filter/task_pipelines.py +105 -0
  18. mteb/abstasks/_statistics_calculation.py +23 -11
  19. mteb/abstasks/_stratification.py +18 -18
  20. mteb/abstasks/abstask.py +35 -28
  21. mteb/abstasks/aggregate_task_metadata.py +1 -9
  22. mteb/abstasks/aggregated_task.py +10 -29
  23. mteb/abstasks/classification.py +15 -12
  24. mteb/abstasks/clustering.py +20 -16
  25. mteb/abstasks/clustering_legacy.py +13 -10
  26. mteb/abstasks/image/image_text_pair_classification.py +7 -4
  27. mteb/abstasks/multilabel_classification.py +33 -22
  28. mteb/abstasks/pair_classification.py +27 -11
  29. mteb/abstasks/regression.py +4 -4
  30. mteb/abstasks/retrieval.py +28 -24
  31. mteb/abstasks/retrieval_dataset_loaders.py +2 -2
  32. mteb/abstasks/sts.py +14 -4
  33. mteb/abstasks/task_metadata.py +32 -33
  34. mteb/abstasks/text/bitext_mining.py +39 -28
  35. mteb/abstasks/text/reranking.py +8 -6
  36. mteb/abstasks/text/summarization.py +10 -5
  37. mteb/abstasks/zeroshot_classification.py +8 -4
  38. mteb/benchmarks/_create_table.py +84 -37
  39. mteb/benchmarks/benchmark.py +77 -16
  40. mteb/benchmarks/benchmarks/__init__.py +12 -0
  41. mteb/benchmarks/benchmarks/benchmarks.py +361 -16
  42. mteb/benchmarks/get_benchmark.py +14 -53
  43. mteb/cache.py +227 -37
  44. mteb/cli/_display_tasks.py +2 -2
  45. mteb/cli/build_cli.py +110 -14
  46. mteb/cli/generate_model_card.py +43 -23
  47. mteb/deprecated_evaluator.py +71 -62
  48. mteb/descriptive_stats/BitextMining/RuSciBenchBitextMining.v2.json +61 -0
  49. mteb/descriptive_stats/Classification/HebrewSentimentAnalysis.v3.json +60 -0
  50. mteb/descriptive_stats/Classification/TurkishConstitutionalCourtViolation.json +54 -0
  51. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2CybersecurityRetrieval.json +32 -0
  52. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EconomicRetrieval.json +32 -0
  53. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EnergyRetrieval.json +32 -0
  54. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2HrRetrieval.json +32 -0
  55. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3ComputerScienceRetrieval.json +214 -0
  56. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3EnergyRetrieval.json +214 -0
  57. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3FinanceEnRetrieval.json +214 -0
  58. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3FinanceFrRetrieval.json +214 -0
  59. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3HrRetrieval.json +214 -0
  60. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3IndustrialRetrieval.json +214 -0
  61. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3NuclearRetrieval.json +214 -0
  62. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PharmaceuticalsRetrieval.json +214 -0
  63. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PhysicsRetrieval.json +214 -0
  64. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3TelecomRetrieval.json +214 -0
  65. mteb/descriptive_stats/PairClassification/TERRa.V2.json +35 -0
  66. mteb/descriptive_stats/Reranking/JQaRARerankingLite.json +35 -0
  67. mteb/descriptive_stats/Reranking/JaCWIRRerankingLite.json +35 -0
  68. mteb/descriptive_stats/Reranking/MultiLongDocReranking.json +466 -0
  69. mteb/descriptive_stats/Retrieval/ArguAna-NL.v2.json +30 -0
  70. mteb/descriptive_stats/Retrieval/ChemRxivRetrieval.json +30 -0
  71. mteb/descriptive_stats/Retrieval/EuroPIRQRetrieval.json +116 -0
  72. mteb/descriptive_stats/Retrieval/JaCWIRRetrievalLite.json +30 -0
  73. mteb/descriptive_stats/Retrieval/JaqketRetrievalLite.json +30 -0
  74. mteb/descriptive_stats/Retrieval/MIRACLJaRetrievalLite.json +30 -0
  75. mteb/descriptive_stats/Retrieval/MrTyDiJaRetrievalLite.json +30 -0
  76. mteb/descriptive_stats/Retrieval/NFCorpus-NL.v2.json +30 -0
  77. mteb/descriptive_stats/Retrieval/NanoClimateFEVER-VN.json +30 -0
  78. mteb/descriptive_stats/Retrieval/NanoDBPedia-VN.json +30 -0
  79. mteb/descriptive_stats/Retrieval/NanoFEVER-VN.json +30 -0
  80. mteb/descriptive_stats/Retrieval/NanoHotpotQA-VN.json +30 -0
  81. mteb/descriptive_stats/Retrieval/NanoMSMARCO-VN.json +30 -0
  82. mteb/descriptive_stats/Retrieval/NanoNQ-VN.json +30 -0
  83. mteb/descriptive_stats/Retrieval/SCIDOCS-NL.v2.json +30 -0
  84. mteb/descriptive_stats/Retrieval/SQuADKorV1Retrieval.json +30 -0
  85. mteb/descriptive_stats/Retrieval/SciFact-NL.v2.json +30 -0
  86. mteb/descriptive_stats/Retrieval/TVPLRetrieval.json +30 -0
  87. mteb/evaluate.py +106 -75
  88. mteb/filter_tasks.py +25 -26
  89. mteb/get_tasks.py +29 -30
  90. mteb/languages/language_scripts.py +5 -3
  91. mteb/leaderboard/app.py +414 -151
  92. mteb/leaderboard/benchmark_selector.py +14 -5
  93. mteb/leaderboard/figures.py +13 -15
  94. mteb/leaderboard/table.py +82 -17
  95. mteb/load_results.py +12 -12
  96. mteb/models/__init__.py +4 -1
  97. mteb/models/abs_encoder.py +31 -23
  98. mteb/models/cache_wrappers/__init__.py +2 -1
  99. mteb/models/cache_wrappers/cache_backend_protocol.py +3 -5
  100. mteb/models/cache_wrappers/cache_backends/_hash_utils.py +7 -6
  101. mteb/models/cache_wrappers/cache_backends/faiss_cache.py +6 -2
  102. mteb/models/cache_wrappers/cache_backends/numpy_cache.py +43 -25
  103. mteb/models/cache_wrappers/cache_wrapper.py +3 -3
  104. mteb/models/get_model_meta.py +25 -118
  105. mteb/models/instruct_wrapper.py +33 -9
  106. mteb/models/model_implementations/align_models.py +8 -1
  107. mteb/models/model_implementations/amazon_models.py +1 -0
  108. mteb/models/model_implementations/andersborges.py +65 -0
  109. mteb/models/model_implementations/ara_models.py +9 -1
  110. mteb/models/model_implementations/arctic_models.py +16 -8
  111. mteb/models/model_implementations/b1ade_models.py +2 -1
  112. mteb/models/model_implementations/bedrock_models.py +4 -0
  113. mteb/models/model_implementations/bge_models.py +101 -17
  114. mteb/models/model_implementations/bica_model.py +35 -0
  115. mteb/models/model_implementations/blip2_models.py +13 -2
  116. mteb/models/model_implementations/blip_models.py +43 -16
  117. mteb/models/model_implementations/bm25.py +5 -4
  118. mteb/models/model_implementations/bmretriever_models.py +10 -4
  119. mteb/models/model_implementations/cadet_models.py +10 -1
  120. mteb/models/model_implementations/cde_models.py +25 -4
  121. mteb/models/model_implementations/clip_models.py +9 -6
  122. mteb/models/model_implementations/clips_models.py +100 -0
  123. mteb/models/model_implementations/codefuse_models.py +165 -3
  124. mteb/models/model_implementations/codesage_models.py +18 -3
  125. mteb/models/model_implementations/cohere_models.py +13 -6
  126. mteb/models/model_implementations/cohere_v.py +7 -2
  127. mteb/models/model_implementations/colpali_models.py +17 -9
  128. mteb/models/model_implementations/colqwen_models.py +275 -5
  129. mteb/models/model_implementations/colsmol_models.py +4 -2
  130. mteb/models/model_implementations/conan_models.py +2 -1
  131. mteb/models/model_implementations/dino_models.py +194 -23
  132. mteb/models/model_implementations/e5_instruct.py +27 -4
  133. mteb/models/model_implementations/e5_models.py +21 -110
  134. mteb/models/model_implementations/e5_v.py +7 -6
  135. mteb/models/model_implementations/eagerworks_models.py +164 -0
  136. mteb/models/model_implementations/emillykkejensen_models.py +91 -0
  137. mteb/models/model_implementations/en_code_retriever.py +2 -1
  138. mteb/models/model_implementations/euler_models.py +32 -0
  139. mteb/models/model_implementations/evaclip_models.py +4 -0
  140. mteb/models/model_implementations/fa_models.py +67 -9
  141. mteb/models/model_implementations/facebookai.py +205 -0
  142. mteb/models/model_implementations/geogpt_models.py +2 -1
  143. mteb/models/model_implementations/gme_v_models.py +17 -10
  144. mteb/models/model_implementations/google_models.py +17 -6
  145. mteb/models/model_implementations/granite_vision_embedding_models.py +8 -3
  146. mteb/models/model_implementations/gritlm_models.py +4 -2
  147. mteb/models/model_implementations/gte_models.py +99 -9
  148. mteb/models/model_implementations/hinvec_models.py +2 -1
  149. mteb/models/model_implementations/human.py +1 -0
  150. mteb/models/model_implementations/ibm_granite_models.py +36 -6
  151. mteb/models/model_implementations/inf_models.py +4 -2
  152. mteb/models/model_implementations/jasper_models.py +256 -3
  153. mteb/models/model_implementations/jina_clip.py +49 -10
  154. mteb/models/model_implementations/jina_models.py +222 -11
  155. mteb/models/model_implementations/kalm_models.py +203 -25
  156. mteb/models/model_implementations/kblab.py +37 -0
  157. mteb/models/model_implementations/kennethenevoldsen_models.py +74 -0
  158. mteb/models/model_implementations/kfst.py +25 -0
  159. mteb/models/model_implementations/kowshik24_models.py +32 -0
  160. mteb/models/model_implementations/lens_models.py +2 -0
  161. mteb/models/model_implementations/lgai_embedding_models.py +2 -1
  162. mteb/models/model_implementations/linq_models.py +4 -3
  163. mteb/models/model_implementations/listconranker.py +2 -2
  164. mteb/models/model_implementations/llm2clip_models.py +9 -6
  165. mteb/models/model_implementations/llm2vec_models.py +16 -8
  166. mteb/models/model_implementations/mcinext_models.py +7 -1
  167. mteb/models/model_implementations/mdbr_models.py +19 -3
  168. mteb/models/model_implementations/misc_models.py +422 -60
  169. mteb/models/model_implementations/mixedbread_ai_models.py +332 -0
  170. mteb/models/model_implementations/mme5_models.py +2 -1
  171. mteb/models/model_implementations/moco_models.py +15 -4
  172. mteb/models/model_implementations/mod_models.py +191 -0
  173. mteb/models/model_implementations/model2vec_models.py +27 -14
  174. mteb/models/model_implementations/moka_models.py +4 -1
  175. mteb/models/model_implementations/nbailab.py +70 -0
  176. mteb/models/model_implementations/no_instruct_sentence_models.py +3 -2
  177. mteb/models/model_implementations/nomic_models.py +173 -6
  178. mteb/models/model_implementations/nomic_models_vision.py +8 -3
  179. mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py +32 -19
  180. mteb/models/model_implementations/nvidia_models.py +155 -20
  181. mteb/models/model_implementations/octen_models.py +254 -0
  182. mteb/models/model_implementations/openai_models.py +20 -16
  183. mteb/models/model_implementations/openclip_models.py +37 -13
  184. mteb/models/model_implementations/opensearch_neural_sparse_models.py +10 -5
  185. mteb/models/model_implementations/ops_moa_models.py +5 -3
  186. mteb/models/model_implementations/ordalietech_solon_embeddings_mini_beta_1_1.py +1 -1
  187. mteb/models/model_implementations/pawan_models.py +39 -0
  188. mteb/models/model_implementations/piccolo_models.py +9 -1
  189. mteb/models/model_implementations/pixie_models.py +56 -0
  190. mteb/models/model_implementations/promptriever_models.py +12 -8
  191. mteb/models/model_implementations/pylate_models.py +46 -12
  192. mteb/models/model_implementations/qodo_models.py +4 -2
  193. mteb/models/model_implementations/qtack_models.py +2 -1
  194. mteb/models/model_implementations/qwen3_models.py +9 -6
  195. mteb/models/model_implementations/qzhou_models.py +5 -3
  196. mteb/models/model_implementations/random_baseline.py +19 -24
  197. mteb/models/model_implementations/rasgaard_models.py +34 -0
  198. mteb/models/model_implementations/reasonir_model.py +2 -1
  199. mteb/models/model_implementations/repllama_models.py +5 -3
  200. mteb/models/model_implementations/rerankers_custom.py +15 -9
  201. mteb/models/model_implementations/rerankers_monot5_based.py +31 -31
  202. mteb/models/model_implementations/richinfoai_models.py +2 -1
  203. mteb/models/model_implementations/ru_sentence_models.py +71 -20
  204. mteb/models/model_implementations/ruri_models.py +322 -0
  205. mteb/models/model_implementations/salesforce_models.py +6 -3
  206. mteb/models/model_implementations/samilpwc_models.py +2 -1
  207. mteb/models/model_implementations/sarashina_embedding_models.py +168 -0
  208. mteb/models/model_implementations/searchmap_models.py +2 -1
  209. mteb/models/model_implementations/seed_1_6_embedding_models.py +8 -2
  210. mteb/models/model_implementations/seed_1_6_embedding_models_1215.py +625 -0
  211. mteb/models/model_implementations/seed_models.py +1 -0
  212. mteb/models/model_implementations/sentence_transformers_models.py +177 -18
  213. mteb/models/model_implementations/shuu_model.py +32 -31
  214. mteb/models/model_implementations/siglip_models.py +30 -20
  215. mteb/models/model_implementations/slm_models.py +416 -0
  216. mteb/models/model_implementations/sonar_models.py +1 -0
  217. mteb/models/model_implementations/spartan8806_atles_champion.py +34 -0
  218. mteb/models/model_implementations/stella_models.py +23 -4
  219. mteb/models/model_implementations/tarka_models.py +376 -0
  220. mteb/models/model_implementations/text2vec_models.py +9 -3
  221. mteb/models/model_implementations/ua_sentence_models.py +11 -1
  222. mteb/models/model_implementations/uae_models.py +8 -1
  223. mteb/models/model_implementations/vdr_models.py +3 -1
  224. mteb/models/model_implementations/vi_vn_models.py +45 -6
  225. mteb/models/model_implementations/vista_models.py +2 -0
  226. mteb/models/model_implementations/vlm2vec_models.py +5 -3
  227. mteb/models/model_implementations/voyage_models.py +99 -0
  228. mteb/models/model_implementations/voyage_v.py +17 -9
  229. mteb/models/model_implementations/xyz_models.py +1 -0
  230. mteb/models/model_implementations/youtu_models.py +2 -1
  231. mteb/models/model_implementations/yuan_models.py +34 -0
  232. mteb/models/model_implementations/yuan_models_en.py +58 -0
  233. mteb/models/model_meta.py +498 -29
  234. mteb/models/models_protocols.py +22 -6
  235. mteb/models/search_encoder_index/__init__.py +7 -0
  236. mteb/models/search_encoder_index/search_backend_protocol.py +50 -0
  237. mteb/models/search_encoder_index/search_indexes/__init__.py +5 -0
  238. mteb/models/search_encoder_index/search_indexes/faiss_search_index.py +160 -0
  239. mteb/models/search_wrappers.py +197 -65
  240. mteb/models/sentence_transformer_wrapper.py +52 -32
  241. mteb/models/vllm_wrapper.py +327 -0
  242. mteb/py.typed +0 -0
  243. mteb/results/benchmark_results.py +114 -65
  244. mteb/results/model_result.py +63 -26
  245. mteb/results/task_result.py +117 -77
  246. mteb/similarity_functions.py +60 -7
  247. mteb/tasks/bitext_mining/multilingual/__init__.py +2 -1
  248. mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py +4 -2
  249. mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining_fast.py +1 -1
  250. mteb/tasks/bitext_mining/multilingual/ru_sci_bench_bitext_mining.py +47 -5
  251. mteb/tasks/bitext_mining/multilingual/web_faq_bitext_mining.py +2 -6
  252. mteb/tasks/classification/ara/ajgt.py +1 -2
  253. mteb/tasks/classification/ara/hotel_review_sentiment_classification.py +1 -2
  254. mteb/tasks/classification/ara/online_store_review_sentiment_classification.py +1 -2
  255. mteb/tasks/classification/ara/restaurant_review_sentiment_classification.py +1 -2
  256. mteb/tasks/classification/ara/tweet_emotion_classification.py +1 -2
  257. mteb/tasks/classification/ara/tweet_sarcasm_classification.py +1 -2
  258. mteb/tasks/classification/ben/bengali_document_classification.py +1 -2
  259. mteb/tasks/classification/ben/bengali_hate_speech_classification.py +1 -2
  260. mteb/tasks/classification/ben/bengali_sentiment_analysis.py +1 -2
  261. mteb/tasks/classification/ces/csfdcz_movie_review_sentiment_classification.py +1 -2
  262. mteb/tasks/classification/ces/czech_product_review_sentiment_classification.py +1 -2
  263. mteb/tasks/classification/ces/czech_so_me_sentiment_classification.py +1 -2
  264. mteb/tasks/classification/dan/angry_tweets_classification.py +1 -2
  265. mteb/tasks/classification/dan/danish_political_comments_classification.py +1 -2
  266. mteb/tasks/classification/dan/ddisco_cohesion_classification.py +1 -2
  267. mteb/tasks/classification/dan/dk_hate_classification.py +2 -3
  268. mteb/tasks/classification/deu/german_politicians_twitter_sentiment_classification.py +1 -2
  269. mteb/tasks/classification/deu/ten_k_gnad_classification.py +1 -2
  270. mteb/tasks/classification/eng/amazon_polarity_classification.py +1 -2
  271. mteb/tasks/classification/eng/arxiv_classification.py +1 -2
  272. mteb/tasks/classification/eng/banking77_classification.py +1 -2
  273. mteb/tasks/classification/eng/dbpedia_classification.py +1 -2
  274. mteb/tasks/classification/eng/emotion_classification.py +1 -2
  275. mteb/tasks/classification/eng/financial_phrasebank_classification.py +1 -2
  276. mteb/tasks/classification/eng/frenk_en_classification.py +1 -2
  277. mteb/tasks/classification/eng/gtsrb_classification.py +1 -1
  278. mteb/tasks/classification/eng/imdb_classification.py +1 -2
  279. mteb/tasks/classification/eng/legal_bench_classification.py +14 -120
  280. mteb/tasks/classification/eng/news_classification.py +1 -2
  281. mteb/tasks/classification/eng/patch_camelyon_classification.py +1 -1
  282. mteb/tasks/classification/eng/patent_classification.py +1 -2
  283. mteb/tasks/classification/eng/poem_sentiment_classification.py +1 -2
  284. mteb/tasks/classification/eng/sds_eye_protection_classification.py +1 -2
  285. mteb/tasks/classification/eng/sds_gloves_classification.py +1 -2
  286. mteb/tasks/classification/eng/toxic_chat_classification.py +2 -19
  287. mteb/tasks/classification/eng/toxic_conversations_classification.py +1 -2
  288. mteb/tasks/classification/eng/tweet_sentiment_extraction_classification.py +1 -2
  289. mteb/tasks/classification/eng/tweet_topic_single_classification.py +2 -13
  290. mteb/tasks/classification/eng/ucf101_classification.py +1 -5
  291. mteb/tasks/classification/eng/wikipedia_bio_met_chem_classification.py +1 -2
  292. mteb/tasks/classification/eng/wikipedia_chem_fields_classification.py +1 -2
  293. mteb/tasks/classification/eng/wikipedia_comp_chem_spectroscopy_classification.py +1 -2
  294. mteb/tasks/classification/eng/wikipedia_crystallography_analytical_classification.py +1 -2
  295. mteb/tasks/classification/eng/wikipedia_theoretical_applied_classification.py +1 -2
  296. mteb/tasks/classification/eng/yahoo_answers_topics_classification.py +1 -2
  297. mteb/tasks/classification/eng/yelp_review_full_classification.py +1 -2
  298. mteb/tasks/classification/est/estonian_valence.py +2 -3
  299. mteb/tasks/classification/fas/fa_mteb_classification.py +7 -14
  300. mteb/tasks/classification/fil/filipino_hate_speech_classification.py +1 -2
  301. mteb/tasks/classification/fin/fin_toxicity_classification.py +2 -11
  302. mteb/tasks/classification/fra/french_book_reviews.py +1 -2
  303. mteb/tasks/classification/fra/movie_review_sentiment_classification.py +1 -2
  304. mteb/tasks/classification/guj/gujarati_news_classification.py +1 -2
  305. mteb/tasks/classification/heb/__init__.py +6 -1
  306. mteb/tasks/classification/heb/hebrew_sentiment_analysis.py +62 -4
  307. mteb/tasks/classification/hin/hindi_discourse_classification.py +1 -2
  308. mteb/tasks/classification/hin/sentiment_analysis_hindi.py +1 -2
  309. mteb/tasks/classification/hrv/frenk_hr_classification.py +1 -2
  310. mteb/tasks/classification/ind/indonesian_id_clickbait_classification.py +1 -2
  311. mteb/tasks/classification/ind/indonesian_mongabay_conservation_classification.py +1 -2
  312. mteb/tasks/classification/ita/italian_linguist_acceptability_classification.py +1 -2
  313. mteb/tasks/classification/jav/javanese_imdb_classification.py +1 -2
  314. mteb/tasks/classification/jpn/wrime_classification.py +1 -2
  315. mteb/tasks/classification/kan/kannada_news_classification.py +1 -2
  316. mteb/tasks/classification/kor/klue_tc.py +1 -2
  317. mteb/tasks/classification/kor/kor_hate_classification.py +2 -17
  318. mteb/tasks/classification/kor/kor_sarcasm_classification.py +2 -19
  319. mteb/tasks/classification/kur/kurdish_sentiment_classification.py +3 -4
  320. mteb/tasks/classification/mal/malayalam_news_classification.py +1 -2
  321. mteb/tasks/classification/mar/marathi_news_classification.py +1 -2
  322. mteb/tasks/classification/mkd/macedonian_tweet_sentiment_classification.py +1 -2
  323. mteb/tasks/classification/multilingual/catalonia_tweet_classification.py +1 -6
  324. mteb/tasks/classification/multilingual/multi_hate_classification.py +1 -4
  325. mteb/tasks/classification/multilingual/ru_sci_bench_classification.py +4 -23
  326. mteb/tasks/classification/multilingual/scala_classification.py +2 -3
  327. mteb/tasks/classification/multilingual/sib200_classification.py +1 -6
  328. mteb/tasks/classification/mya/myanmar_news.py +1 -2
  329. mteb/tasks/classification/nep/nepali_news_classification.py +1 -2
  330. mteb/tasks/classification/nld/dutch_book_review_sentiment_classification.py +4 -2
  331. mteb/tasks/classification/nld/dutch_cola_classification.py +3 -0
  332. mteb/tasks/classification/nld/dutch_government_bias_classification.py +3 -0
  333. mteb/tasks/classification/nld/dutch_news_articles_classification.py +3 -0
  334. mteb/tasks/classification/nld/dutch_sarcastic_headlines_classification.py +3 -0
  335. mteb/tasks/classification/nld/iconclass_classification.py +3 -0
  336. mteb/tasks/classification/nld/open_tender_classification.py +3 -0
  337. mteb/tasks/classification/nld/vaccin_chat_nl_classification.py +3 -0
  338. mteb/tasks/classification/nob/no_rec_classification.py +1 -2
  339. mteb/tasks/classification/nob/norwegian_parliament_classification.py +1 -2
  340. mteb/tasks/classification/ory/odia_news_classification.py +1 -2
  341. mteb/tasks/classification/pol/polish_classification.py +3 -6
  342. mteb/tasks/classification/ron/moroco.py +1 -2
  343. mteb/tasks/classification/ron/romanian_reviews_sentiment.py +1 -2
  344. mteb/tasks/classification/ron/romanian_sentiment_classification.py +1 -2
  345. mteb/tasks/classification/rus/georeview_classification.py +1 -2
  346. mteb/tasks/classification/rus/headline_classification.py +1 -2
  347. mteb/tasks/classification/rus/inappropriateness_classification.py +1 -2
  348. mteb/tasks/classification/rus/ru_reviews_classification.py +1 -2
  349. mteb/tasks/classification/rus/ru_toixic_classification_okmlcup.py +1 -2
  350. mteb/tasks/classification/rus/senti_ru_eval.py +1 -2
  351. mteb/tasks/classification/sin/sinhala_news_classification.py +1 -2
  352. mteb/tasks/classification/sin/sinhala_news_source_classification.py +1 -2
  353. mteb/tasks/classification/slk/csfdsk_movie_review_sentiment_classification.py +1 -2
  354. mteb/tasks/classification/slk/slovak_hate_speech_classification.py +1 -2
  355. mteb/tasks/classification/slk/slovak_movie_review_sentiment_classification.py +1 -2
  356. mteb/tasks/classification/slv/frenk_sl_classification.py +1 -2
  357. mteb/tasks/classification/spa/spanish_news_classification.py +1 -2
  358. mteb/tasks/classification/spa/spanish_sentiment_classification.py +1 -2
  359. mteb/tasks/classification/ssw/siswati_news_classification.py +1 -2
  360. mteb/tasks/classification/swa/swahili_news_classification.py +1 -2
  361. mteb/tasks/classification/swe/dalaj_classification.py +1 -2
  362. mteb/tasks/classification/swe/swe_rec_classification.py +1 -2
  363. mteb/tasks/classification/swe/swedish_sentiment_classification.py +1 -2
  364. mteb/tasks/classification/tam/tamil_news_classification.py +1 -2
  365. mteb/tasks/classification/tel/telugu_andhra_jyoti_news_classification.py +1 -2
  366. mteb/tasks/classification/tha/wisesight_sentiment_classification.py +1 -2
  367. mteb/tasks/classification/tsn/tswana_news_classification.py +1 -2
  368. mteb/tasks/classification/tur/__init__.py +4 -0
  369. mteb/tasks/classification/tur/turkish_constitutional_court.py +41 -0
  370. mteb/tasks/classification/tur/turkish_movie_sentiment_classification.py +1 -2
  371. mteb/tasks/classification/tur/turkish_product_sentiment_classification.py +1 -2
  372. mteb/tasks/classification/ukr/ukr_formality_classification.py +2 -15
  373. mteb/tasks/classification/urd/urdu_roman_sentiment_classification.py +1 -2
  374. mteb/tasks/classification/vie/amazon_counterfactual_vn_classification.py +1 -6
  375. mteb/tasks/classification/vie/amazon_polarity_vn_classification.py +1 -6
  376. mteb/tasks/classification/vie/amazon_reviews_vn_classification.py +1 -5
  377. mteb/tasks/classification/vie/banking77_vn_classification.py +1 -5
  378. mteb/tasks/classification/vie/emotion_vn_classification.py +1 -5
  379. mteb/tasks/classification/vie/imdb_vn_classification.py +1 -5
  380. mteb/tasks/classification/vie/massive_intent_vn_classification.py +1 -5
  381. mteb/tasks/classification/vie/massive_scenario_vn_classification.py +1 -5
  382. mteb/tasks/classification/vie/mtop_domain_vn_classification.py +1 -5
  383. mteb/tasks/classification/vie/mtop_intent_vn_classification.py +1 -5
  384. mteb/tasks/classification/vie/toxic_conversations_vn_classification.py +1 -5
  385. mteb/tasks/classification/vie/tweet_sentiment_extraction_vn_classification.py +1 -5
  386. mteb/tasks/classification/vie/vie_student_feedback_classification.py +1 -2
  387. mteb/tasks/classification/zho/cmteb_classification.py +5 -10
  388. mteb/tasks/classification/zho/yue_openrice_review_classification.py +1 -2
  389. mteb/tasks/classification/zul/isi_zulu_news_classification.py +1 -2
  390. mteb/tasks/clustering/eng/hume_wiki_cities_clustering.py +1 -1
  391. mteb/tasks/clustering/eng/wiki_cities_clustering.py +1 -1
  392. mteb/tasks/clustering/jpn/mews_c16_ja_clustering.py +1 -3
  393. mteb/tasks/clustering/multilingual/sib200_clustering_s2s.py +1 -6
  394. mteb/tasks/clustering/nld/dutch_news_articles_clustering_p2p.py +3 -0
  395. mteb/tasks/clustering/nld/dutch_news_articles_clustering_s2s.py +3 -0
  396. mteb/tasks/clustering/nld/iconclass_clustering_s2s.py +3 -0
  397. mteb/tasks/clustering/nld/open_tender_clustering_p2p.py +3 -0
  398. mteb/tasks/clustering/nld/open_tender_clustering_s2s.py +3 -0
  399. mteb/tasks/clustering/nld/vabb_clustering_p2p.py +3 -0
  400. mteb/tasks/clustering/nld/vabb_clustering_s2s.py +3 -0
  401. mteb/tasks/clustering/vie/reddit_clustering_p2p_vn.py +1 -5
  402. mteb/tasks/clustering/vie/reddit_clustering_vn.py +1 -5
  403. mteb/tasks/clustering/vie/stack_exchange_clustering_p2p_vn.py +1 -5
  404. mteb/tasks/clustering/vie/stack_exchange_clustering_vn.py +1 -5
  405. mteb/tasks/clustering/vie/twenty_newsgroups_clustering_vn.py +1 -5
  406. mteb/tasks/clustering/zho/cmteb_clustering.py +2 -2
  407. mteb/tasks/image_text_pair_classification/eng/sugar_crepe.py +1 -1
  408. mteb/tasks/multilabel_classification/ita/emit_classification.py +1 -5
  409. mteb/tasks/multilabel_classification/kor/kor_hate_speech_ml_classification.py +1 -9
  410. mteb/tasks/multilabel_classification/mlt/maltese_news_classification.py +1 -6
  411. mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py +3 -0
  412. mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py +3 -0
  413. mteb/tasks/multilabel_classification/por/brazilian_toxic_tweets_classification.py +1 -6
  414. mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_group_classification.py +1 -1
  415. mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_subclass_classification.py +1 -2
  416. mteb/tasks/pair_classification/dan/talemaader_pc.py +1 -6
  417. mteb/tasks/pair_classification/eng/legal_bench_pc.py +1 -9
  418. mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py +3 -0
  419. mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py +3 -0
  420. mteb/tasks/pair_classification/rus/__init__.py +2 -2
  421. mteb/tasks/pair_classification/rus/terra.py +51 -25
  422. mteb/tasks/pair_classification/vie/sprint_duplicate_questions_pcvn.py +1 -5
  423. mteb/tasks/pair_classification/vie/twitter_sem_eval2015_pcvn.py +1 -5
  424. mteb/tasks/pair_classification/vie/twitter_url_corpus_pcvn.py +1 -5
  425. mteb/tasks/regression/multilingual/ru_sci_bench_regression.py +2 -6
  426. mteb/tasks/reranking/jpn/__init__.py +9 -1
  427. mteb/tasks/reranking/jpn/j_qa_ra_reranking_lite.py +49 -0
  428. mteb/tasks/reranking/jpn/ja_cwir_reranking_lite.py +47 -0
  429. mteb/tasks/reranking/multilingual/__init__.py +2 -0
  430. mteb/tasks/reranking/multilingual/multi_long_doc_reranking.py +70 -0
  431. mteb/tasks/reranking/multilingual/wikipedia_reranking_multilingual.py +1 -1
  432. mteb/tasks/reranking/multilingual/x_glue_wpr_reranking.py +1 -2
  433. mteb/tasks/reranking/vie/ask_ubuntu_dup_questions_vn.py +1 -5
  434. mteb/tasks/reranking/vie/sci_docs_reranking_vn.py +1 -5
  435. mteb/tasks/reranking/vie/stack_overflow_dup_questions_vn.py +1 -5
  436. mteb/tasks/retrieval/code/code_rag.py +12 -12
  437. mteb/tasks/retrieval/code/fresh_stack_retrieval.py +8 -5
  438. mteb/tasks/retrieval/dan/dan_fever_retrieval.py +1 -1
  439. mteb/tasks/retrieval/dan/tv2_nordretrieval.py +2 -2
  440. mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py +2 -2
  441. mteb/tasks/retrieval/eng/__init__.py +2 -0
  442. mteb/tasks/retrieval/eng/chemrxiv.py +33 -0
  443. mteb/tasks/retrieval/eng/cub200_i2i_retrieval.py +1 -1
  444. mteb/tasks/retrieval/eng/lit_search_retrieval.py +1 -8
  445. mteb/tasks/retrieval/eng/vidore_bench_retrieval.py +4 -0
  446. mteb/tasks/retrieval/jpn/__init__.py +8 -0
  447. mteb/tasks/retrieval/jpn/ja_cwir_retrieval.py +1 -4
  448. mteb/tasks/retrieval/jpn/ja_cwir_retrieval_lite.py +47 -0
  449. mteb/tasks/retrieval/jpn/jaqket_retrieval_lite.py +50 -0
  450. mteb/tasks/retrieval/jpn/miracl_ja_retrieval_lite.py +52 -0
  451. mteb/tasks/retrieval/jpn/mr_tydi_ja_retrieval_lite.py +48 -0
  452. mteb/tasks/retrieval/kat/georgian_faq_retrieval.py +11 -4
  453. mteb/tasks/retrieval/kor/__init__.py +16 -1
  454. mteb/tasks/retrieval/kor/kovidore2_bench_retrieval.py +142 -0
  455. mteb/tasks/retrieval/kor/squad_kor_v1_retrieval.py +47 -0
  456. mteb/tasks/retrieval/multilingual/__init__.py +24 -0
  457. mteb/tasks/retrieval/multilingual/belebele_retrieval.py +5 -4
  458. mteb/tasks/retrieval/multilingual/euro_pirq_retrieval.py +43 -0
  459. mteb/tasks/retrieval/multilingual/jina_vdr_bench_retrieval.py +56 -42
  460. mteb/tasks/retrieval/multilingual/mkqa_retrieval.py +1 -2
  461. mteb/tasks/retrieval/multilingual/mlqa_retrieval.py +1 -4
  462. mteb/tasks/retrieval/multilingual/multi_long_doc_retrieval.py +1 -2
  463. mteb/tasks/retrieval/multilingual/public_health_qa_retrieval.py +9 -4
  464. mteb/tasks/retrieval/multilingual/ru_sci_bench_retrieval.py +2 -12
  465. mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py +4 -2
  466. mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py +389 -0
  467. mteb/tasks/retrieval/nld/__init__.py +8 -4
  468. mteb/tasks/retrieval/nld/argu_ana_nl_retrieval.py +46 -27
  469. mteb/tasks/retrieval/nld/bbsard_nl_retrieval.py +3 -0
  470. mteb/tasks/retrieval/nld/dutch_news_articles_retrieval.py +3 -0
  471. mteb/tasks/retrieval/nld/legal_qa_nl_retrieval.py +3 -0
  472. mteb/tasks/retrieval/nld/nf_corpus_nl_retrieval.py +42 -25
  473. mteb/tasks/retrieval/nld/open_tender_retrieval.py +3 -0
  474. mteb/tasks/retrieval/nld/sci_fact_nl_retrieval.py +42 -24
  475. mteb/tasks/retrieval/nld/scidocsnl_retrieval.py +44 -27
  476. mteb/tasks/retrieval/nld/vabb_retrieval.py +3 -0
  477. mteb/tasks/retrieval/nob/norquad.py +2 -2
  478. mteb/tasks/retrieval/nob/snl_retrieval.py +2 -2
  479. mteb/tasks/retrieval/slk/slovak_sum_retrieval.py +1 -7
  480. mteb/tasks/retrieval/tur/tur_hist_quad.py +1 -1
  481. mteb/tasks/retrieval/vie/__init__.py +14 -6
  482. mteb/tasks/retrieval/vie/argu_ana_vn_retrieval.py +1 -5
  483. mteb/tasks/retrieval/vie/climate_fevervn_retrieval.py +40 -5
  484. mteb/tasks/retrieval/vie/cqa_dupstack_android_vn_retrieval.py +1 -5
  485. mteb/tasks/retrieval/vie/cqa_dupstack_gis_vn_retrieval.py +1 -5
  486. mteb/tasks/retrieval/vie/cqa_dupstack_mathematica_vn_retrieval.py +1 -5
  487. mteb/tasks/retrieval/vie/cqa_dupstack_physics_vn_retrieval.py +1 -5
  488. mteb/tasks/retrieval/vie/cqa_dupstack_programmers_vn_retrieval.py +1 -5
  489. mteb/tasks/retrieval/vie/cqa_dupstack_stats_vn_retrieval.py +1 -5
  490. mteb/tasks/retrieval/vie/cqa_dupstack_tex_vn_retrieval.py +1 -5
  491. mteb/tasks/retrieval/vie/cqa_dupstack_unix_vn_retrieval.py +1 -5
  492. mteb/tasks/retrieval/vie/cqa_dupstack_webmasters_vn_retrieval.py +1 -5
  493. mteb/tasks/retrieval/vie/cqa_dupstack_wordpress_vn_retrieval.py +1 -5
  494. mteb/tasks/retrieval/vie/db_pedia_vn_retrieval.py +40 -5
  495. mteb/tasks/retrieval/vie/fevervn_retrieval.py +40 -7
  496. mteb/tasks/retrieval/vie/fi_qa2018_vn_retrieval.py +1 -5
  497. mteb/tasks/retrieval/vie/green_node_table_markdown_retrieval.py +16 -1
  498. mteb/tasks/retrieval/vie/hotpot_qavn_retrieval.py +40 -6
  499. mteb/tasks/retrieval/vie/msmarcovn_retrieval.py +49 -5
  500. mteb/tasks/retrieval/vie/nf_corpus_vn_retrieval.py +1 -5
  501. mteb/tasks/retrieval/vie/nqvn_retrieval.py +40 -5
  502. mteb/tasks/retrieval/vie/quora_vn_retrieval.py +1 -6
  503. mteb/tasks/retrieval/vie/sci_fact_vn_retrieval.py +1 -5
  504. mteb/tasks/retrieval/vie/scidocsvn_retrieval.py +1 -6
  505. mteb/tasks/retrieval/vie/touche2020_vn_retrieval.py +1 -5
  506. mteb/tasks/retrieval/vie/treccovidvn_retrieval.py +1 -5
  507. mteb/tasks/retrieval/vie/tvpl_retrieval.py +42 -0
  508. mteb/tasks/retrieval/vie/zac_legal_text_retrieval.py +15 -1
  509. mteb/tasks/sts/nld/sick_nl_sts.py +1 -0
  510. mteb/tasks/sts/vie/biosses_stsvn.py +1 -5
  511. mteb/tasks/sts/vie/sickr_stsvn.py +1 -5
  512. mteb/tasks/sts/vie/sts_benchmark_stsvn.py +1 -5
  513. mteb/tasks/zeroshot_classification/eng/gtsrb.py +1 -1
  514. mteb/tasks/zeroshot_classification/eng/patch_camelyon.py +1 -1
  515. mteb/tasks/zeroshot_classification/eng/ucf101.py +1 -5
  516. mteb/types/__init__.py +2 -0
  517. mteb/types/_encoder_io.py +19 -2
  518. mteb/types/_result.py +2 -1
  519. mteb/types/statistics.py +9 -3
  520. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/METADATA +25 -8
  521. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/RECORD +525 -438
  522. mteb/models/model_implementations/mxbai_models.py +0 -102
  523. mteb/models/model_implementations/nb_sbert.py +0 -25
  524. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/WHEEL +0 -0
  525. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/entry_points.txt +0 -0
  526. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/licenses/LICENSE +0 -0
  527. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/top_level.txt +0 -0
@@ -19,6 +19,7 @@ from mteb.types import (
19
19
  Array,
20
20
  BatchedInput,
21
21
  CorpusDatasetType,
22
+ EncodeKwargs,
22
23
  PromptType,
23
24
  QueryDatasetType,
24
25
  RetrievalOutputType,
@@ -29,7 +30,7 @@ logger = logging.getLogger(__name__)
29
30
 
30
31
 
31
32
  class PylateSearchEncoder:
32
- """Mixin class to add PyLate-based indexing and search to an encoder. Implements :class:`SearchProtocol`"""
33
+ """Mixin class to add PyLate-based indexing and search to an encoder. Implements [SearchProtocol][mteb.models.SearchProtocol]"""
33
34
 
34
35
  base_index_dir: Path | None = None
35
36
  _index_dir: Path | None = None
@@ -45,7 +46,7 @@ class PylateSearchEncoder:
45
46
  task_metadata: TaskMetadata,
46
47
  hf_split: str,
47
48
  hf_subset: str,
48
- encode_kwargs: dict[str, Any],
49
+ encode_kwargs: EncodeKwargs,
49
50
  ) -> None:
50
51
  """Index the corpus for retrieval.
51
52
 
@@ -78,7 +79,7 @@ class PylateSearchEncoder:
78
79
  hf_split: str,
79
80
  hf_subset: str,
80
81
  top_k: int,
81
- encode_kwargs: dict[str, Any],
82
+ encode_kwargs: EncodeKwargs,
82
83
  top_ranked: TopRankedDocumentsType | None = None,
83
84
  ) -> RetrievalOutputType:
84
85
  queries_dataloader = create_dataloader(
@@ -136,7 +137,7 @@ class PylateSearchEncoder:
136
137
  hf_subset: str,
137
138
  hf_split: str,
138
139
  top_k: int,
139
- encode_kwargs: dict[str, Any],
140
+ encode_kwargs: EncodeKwargs,
140
141
  ) -> dict[str, list[tuple[float, str]]]:
141
142
  from pylate import indexes, retrieve
142
143
 
@@ -200,7 +201,7 @@ class PylateSearchEncoder:
200
201
  task_metadata: TaskMetadata,
201
202
  hf_subset: str,
202
203
  hf_split: str,
203
- encode_kwargs: dict[str, Any],
204
+ encode_kwargs: EncodeKwargs,
204
205
  ) -> dict[str, list[tuple[float, str]]]:
205
206
  """Rerank with PyLate's rank.rerank using per-query candidates.
206
207
 
@@ -328,18 +329,16 @@ class MultiVectorModel(AbsEncoder, PylateSearchEncoder):
328
329
  inputs,
329
330
  prompt_name=prompt_name,
330
331
  is_query=prompt_type == PromptType.query,
331
- convert_to_tensor=True,
332
332
  **kwargs,
333
333
  )
334
334
 
335
- # encode returns a list of tensors shaped (x, token_dim), pad to uniform length
336
- pred = torch.nn.utils.rnn.pad_sequence(pred, batch_first=True, padding_value=0)
337
- return pred.cpu().numpy()
335
+ return pred
338
336
 
339
337
 
340
338
  colbert_v2 = ModelMeta(
341
339
  loader=MultiVectorModel,
342
340
  name="colbert-ir/colbertv2.0",
341
+ model_type=["late-interaction"],
343
342
  languages=["eng-Latn"],
344
343
  open_weights=True,
345
344
  revision="c1e84128e85ef755c096a95bdb06b47793b13acf",
@@ -352,7 +351,7 @@ colbert_v2 = ModelMeta(
352
351
  embed_dim=None,
353
352
  license="mit",
354
353
  similarity_fn_name=ScoringFunction.MAX_SIM,
355
- framework=["PyLate", "ColBERT"],
354
+ framework=["PyLate", "ColBERT", "Transformers", "ONNX", "safetensors"],
356
355
  reference="https://huggingface.co/colbert-ir/colbertv2.0",
357
356
  use_instructions=False,
358
357
  adapted_from=None,
@@ -372,6 +371,7 @@ jina_colbert_v2 = ModelMeta(
372
371
  trust_remote_code=True,
373
372
  ),
374
373
  name="jinaai/jina-colbert-v2",
374
+ model_type=["late-interaction"],
375
375
  languages=[
376
376
  "ara-Arab",
377
377
  "ben-Beng",
@@ -407,7 +407,7 @@ jina_colbert_v2 = ModelMeta(
407
407
  embed_dim=None,
408
408
  license="cc-by-nc-4.0",
409
409
  similarity_fn_name=ScoringFunction.MAX_SIM,
410
- framework=["PyLate", "ColBERT"],
410
+ framework=["PyLate", "ColBERT", "ONNX", "safetensors"],
411
411
  reference="https://huggingface.co/jinaai/jina-colbert-v2",
412
412
  use_instructions=False,
413
413
  adapted_from=None,
@@ -418,12 +418,37 @@ jina_colbert_v2 = ModelMeta(
418
418
  "DuRetrieval",
419
419
  "MIRACL",
420
420
  },
421
+ citation="""@inproceedings{xiao-etal-2024-jina,
422
+ title = "{J}ina-{C}ol{BERT}-v2: A General-Purpose Multilingual Late Interaction Retriever",
423
+ author = {Jha, Rohan and
424
+ Wang, Bo and
425
+ G{\"u}nther, Michael and
426
+ Mastrapas, Georgios and
427
+ Sturua, Saba and
428
+ Mohr, Isabelle and
429
+ Koukounas, Andreas and
430
+ Wang, Mohammad Kalim and
431
+ Wang, Nan and
432
+ Xiao, Han},
433
+ editor = {S{\"a}lev{\"a}, Jonne and
434
+ Owodunni, Abraham},
435
+ booktitle = "Proceedings of the Fourth Workshop on Multilingual Representation Learning (MRL 2024)",
436
+ month = nov,
437
+ year = "2024",
438
+ address = "Miami, Florida, USA",
439
+ publisher = "Association for Computational Linguistics",
440
+ url = "https://aclanthology.org/2024.mrl-1.11/",
441
+ doi = "10.18653/v1/2024.mrl-1.11",
442
+ pages = "159--166",
443
+ abstract = "Multi-vector dense models, such as ColBERT, have proven highly effective in information retrieval. ColBERT`s late interaction scoring approximates the joint query-document attention seen in cross-encoders while maintaining inference efficiency closer to traditional dense retrieval models, thanks to its bi-encoder architecture and recent optimizations in indexing and search. In this paper, we introduce a novel architecture and a training framework to support long context window and multilingual retrieval. Leveraging Matryoshka Representation Loss, we further demonstrate that the reducing the embedding dimensionality from 128 to 64 has insignificant impact on the model`s retrieval performance and cut storage requirements by up to 50{\\%}. Our new model, Jina-ColBERT-v2, demonstrates strong performance across a range of English and multilingual retrieval tasks,"
444
+ }""",
421
445
  )
422
446
 
423
447
 
424
448
  lightonai__gte_moderncolbert_v1 = ModelMeta(
425
449
  loader=MultiVectorModel,
426
450
  name="lightonai/GTE-ModernColBERT-v1",
451
+ model_type=["late-interaction"],
427
452
  languages=[
428
453
  "eng-Latn",
429
454
  ],
@@ -438,7 +463,7 @@ lightonai__gte_moderncolbert_v1 = ModelMeta(
438
463
  embed_dim=None,
439
464
  license="apache-2.0",
440
465
  similarity_fn_name="MaxSim",
441
- framework=["PyLate", "ColBERT"],
466
+ framework=["PyLate", "ColBERT", "safetensors", "Sentence Transformers"],
442
467
  reference="https://huggingface.co/lightonai/GTE-ModernColBERT-v1",
443
468
  use_instructions=False,
444
469
  adapted_from="Alibaba-NLP/gte-modernbert-base",
@@ -447,4 +472,13 @@ lightonai__gte_moderncolbert_v1 = ModelMeta(
447
472
  "MSMARCO",
448
473
  "mMARCO-NL",
449
474
  },
475
+ citation="""@inproceedings{reimers-2019-sentence-bert,
476
+ title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
477
+ author = "Reimers, Nils and Gurevych, Iryna",
478
+ booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
479
+ month = "11",
480
+ year = "2019",
481
+ publisher = "Association for Computational Linguistics",
482
+ url = "https://arxiv.org/abs/1908.10084"
483
+ }""",
450
484
  )
@@ -30,6 +30,7 @@ qodo_languages = [
30
30
  Qodo_Embed_1_1_5B = ModelMeta(
31
31
  loader=sentence_transformers_loader,
32
32
  name="Qodo/Qodo-Embed-1-1.5B",
33
+ model_type=["dense"],
33
34
  languages=qodo_languages,
34
35
  open_weights=True,
35
36
  revision="84bbef079b32e8823ec226d4e9e92902706b0eb6",
@@ -41,7 +42,7 @@ Qodo_Embed_1_1_5B = ModelMeta(
41
42
  max_tokens=32768,
42
43
  reference="https://huggingface.co/Qodo/Qodo-Embed-1-1.5B",
43
44
  similarity_fn_name=ScoringFunction.COSINE,
44
- framework=["Sentence Transformers", "PyTorch"],
45
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
45
46
  use_instructions=False,
46
47
  public_training_code=None,
47
48
  public_training_data=None,
@@ -52,6 +53,7 @@ Qodo_Embed_1_1_5B = ModelMeta(
52
53
  Qodo_Embed_1_7B = ModelMeta(
53
54
  loader=sentence_transformers_loader,
54
55
  name="Qodo/Qodo-Embed-1-7B",
56
+ model_type=["dense"],
55
57
  languages=qodo_languages,
56
58
  open_weights=True,
57
59
  revision="f9edd9bf7f687c0e832424058e265120f603cd81",
@@ -63,7 +65,7 @@ Qodo_Embed_1_7B = ModelMeta(
63
65
  max_tokens=32768,
64
66
  reference="https://huggingface.co/Qodo/Qodo-Embed-1-7B",
65
67
  similarity_fn_name=ScoringFunction.COSINE,
66
- framework=["Sentence Transformers", "PyTorch"],
68
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
67
69
  use_instructions=False,
68
70
  public_training_code=None,
69
71
  public_training_data=None,
@@ -25,6 +25,7 @@ mini_gte_datasets = {
25
25
  mini_gte = ModelMeta(
26
26
  loader=sentence_transformers_loader,
27
27
  name="prdev/mini-gte",
28
+ model_type=["dense"],
28
29
  languages=["eng-Latn"],
29
30
  open_weights=True,
30
31
  revision="7fbe6f9b4cc42615e0747299f837ad7769025492",
@@ -36,7 +37,7 @@ mini_gte = ModelMeta(
36
37
  max_tokens=512,
37
38
  reference="https://huggingface.co/prdev/mini-gte",
38
39
  similarity_fn_name=ScoringFunction.COSINE,
39
- framework=["Sentence Transformers", "PyTorch"],
40
+ framework=["Sentence Transformers", "PyTorch", "safetensors"],
40
41
  use_instructions=False,
41
42
  public_training_code=None,
42
43
  public_training_data=None,
@@ -134,18 +134,19 @@ def q3e_instruct_loader(
134
134
  Qwen3_Embedding_0B6 = ModelMeta(
135
135
  loader=q3e_instruct_loader,
136
136
  name="Qwen/Qwen3-Embedding-0.6B",
137
+ model_type=["dense"],
137
138
  languages=multilingual_langs,
138
139
  open_weights=True,
139
140
  revision="b22da495047858cce924d27d76261e96be6febc0", # Commit of @tomaarsen
140
141
  release_date="2025-06-05",
141
142
  n_parameters=595776512,
142
- memory_usage_mb=2272,
143
+ memory_usage_mb=1136,
143
144
  embed_dim=1024,
144
145
  max_tokens=32768,
145
146
  license="apache-2.0",
146
147
  reference="https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
147
148
  similarity_fn_name="cosine",
148
- framework=["Sentence Transformers", "PyTorch"],
149
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
149
150
  use_instructions=True,
150
151
  public_training_code=None,
151
152
  public_training_data=None,
@@ -156,18 +157,19 @@ Qwen3_Embedding_0B6 = ModelMeta(
156
157
  Qwen3_Embedding_4B = ModelMeta(
157
158
  loader=q3e_instruct_loader,
158
159
  name="Qwen/Qwen3-Embedding-4B",
160
+ model_type=["dense"],
159
161
  languages=multilingual_langs,
160
162
  open_weights=True,
161
163
  revision="636cd9bf47d976946cdbb2b0c3ca0cb2f8eea5ff", # Commit of @tomaarsen
162
164
  release_date="2025-06-05",
163
165
  n_parameters=4021774336,
164
- memory_usage_mb=15341,
166
+ memory_usage_mb=7671,
165
167
  embed_dim=2560,
166
168
  max_tokens=32768,
167
169
  license="apache-2.0",
168
170
  reference="https://huggingface.co/Qwen/Qwen3-Embedding-4B",
169
171
  similarity_fn_name="cosine",
170
- framework=["Sentence Transformers", "PyTorch"],
172
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
171
173
  use_instructions=True,
172
174
  public_training_code=None,
173
175
  public_training_data=None,
@@ -178,18 +180,19 @@ Qwen3_Embedding_4B = ModelMeta(
178
180
  Qwen3_Embedding_8B = ModelMeta(
179
181
  loader=q3e_instruct_loader,
180
182
  name="Qwen/Qwen3-Embedding-8B",
183
+ model_type=["dense"],
181
184
  languages=multilingual_langs,
182
185
  open_weights=True,
183
186
  revision="4e423935c619ae4df87b646a3ce949610c66241c", # Commit of @tomaarsen
184
187
  release_date="2025-06-05",
185
188
  n_parameters=7567295488,
186
- memory_usage_mb=28866,
189
+ memory_usage_mb=14433,
187
190
  embed_dim=4096,
188
191
  max_tokens=32768,
189
192
  license="apache-2.0",
190
193
  reference="https://huggingface.co/Qwen/Qwen3-Embedding-8B",
191
194
  similarity_fn_name="cosine",
192
- framework=["Sentence Transformers", "PyTorch"],
195
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
193
196
  use_instructions=True,
194
197
  public_training_code=None,
195
198
  public_training_data=None,
@@ -58,18 +58,19 @@ QZhou_Embedding = ModelMeta(
58
58
  apply_instruction_to_passages=False,
59
59
  ),
60
60
  name="Kingsoft-LLM/QZhou-Embedding",
61
+ model_type=["dense"],
61
62
  languages=["eng-Latn", "zho-Hans"],
62
63
  open_weights=True,
63
64
  revision="f1e6c03ee3882e7b9fa5cec91217715272e433b8",
64
65
  release_date="2025-08-24",
65
66
  n_parameters=7_070_619_136,
66
- memory_usage_mb=29070,
67
+ memory_usage_mb=14436,
67
68
  embed_dim=3584,
68
69
  license="apache-2.0",
69
70
  max_tokens=8192,
70
71
  reference="https://huggingface.co/Kingsoft-LLM/QZhou-Embedding",
71
72
  similarity_fn_name="cosine",
72
- framework=["Sentence Transformers", "PyTorch"],
73
+ framework=["Sentence Transformers", "PyTorch", "safetensors"],
73
74
  use_instructions=True,
74
75
  public_training_code=None,
75
76
  public_training_data="https://huggingface.co/datasets/cfli/bge-full-data",
@@ -91,6 +92,7 @@ QZhou_Embedding_Zh = ModelMeta(
91
92
  apply_instruction_to_passages=False,
92
93
  ),
93
94
  name="Kingsoft-LLM/QZhou-Embedding-Zh",
95
+ model_type=["dense"],
94
96
  languages=["zho-Hans"],
95
97
  open_weights=True,
96
98
  revision="0321ccb126413d1e49c5ce908e802b63d35f18e2",
@@ -102,7 +104,7 @@ QZhou_Embedding_Zh = ModelMeta(
102
104
  max_tokens=8192,
103
105
  reference="http://huggingface.co/Kingsoft-LLM/QZhou-Embedding-Zh",
104
106
  similarity_fn_name="cosine",
105
- framework=["Sentence Transformers", "PyTorch"],
107
+ framework=["Sentence Transformers", "PyTorch", "safetensors"],
106
108
  use_instructions=True,
107
109
  public_training_code=None,
108
110
  public_training_data=None,
@@ -1,15 +1,23 @@
1
+ from __future__ import annotations
2
+
1
3
  import hashlib
2
- from typing import Any, Literal
4
+ from typing import TYPE_CHECKING, Any, Literal
3
5
 
4
6
  import numpy as np
5
7
  import torch
6
- from PIL import Image
7
8
  from torch.utils.data import DataLoader
8
9
 
9
10
  from mteb.abstasks.task_metadata import TaskMetadata
10
11
  from mteb.models.model_meta import ModelMeta
12
+ from mteb.similarity_functions import (
13
+ select_pairwise_similarity,
14
+ select_similarity,
15
+ )
11
16
  from mteb.types._encoder_io import Array, BatchedInput, PromptType
12
17
 
18
+ if TYPE_CHECKING:
19
+ from PIL import Image
20
+
13
21
 
14
22
  def _string_to_vector(text: str | None, size: int) -> np.ndarray:
15
23
  """Generate a deterministic random vector based on a string.
@@ -60,7 +68,7 @@ _common_mock_metadata = dict(
60
68
  license="mit",
61
69
  max_tokens=np.inf,
62
70
  reference=None,
63
- similarity_fn_name="cosine", # type: ignore
71
+ similarity_fn_name="cosine",
64
72
  framework=[],
65
73
  use_instructions=False,
66
74
  public_training_code=None, # No training code, as this is a random baseline
@@ -155,15 +163,9 @@ class RandomEncoderBaseline:
155
163
  Returns:
156
164
  Cosine similarity matrix between the two sets of embeddings
157
165
  """
158
- norm1 = np.linalg.norm(
159
- embeddings1.reshape(-1, self.embedding_dim), axis=1, keepdims=True
166
+ return select_similarity(
167
+ embeddings1, embeddings2, self.mteb_model_meta.similarity_fn_name
160
168
  )
161
- norm2 = np.linalg.norm(
162
- embeddings2.reshape(-1, self.embedding_dim), axis=1, keepdims=True
163
- )
164
- normalized1 = embeddings1 / (norm1 + 1e-10)
165
- normalized2 = embeddings2 / (norm2 + 1e-10)
166
- return np.dot(normalized1, normalized2.T)
167
169
 
168
170
  def similarity_pairwise(
169
171
  self,
@@ -179,22 +181,15 @@ class RandomEncoderBaseline:
179
181
  Returns:
180
182
  Cosine similarity for each pair of embeddings
181
183
  """
182
- norm1 = np.linalg.norm(
183
- embeddings1.reshape(-1, self.embedding_dim), axis=1, keepdims=True
184
- )
185
- norm2 = np.linalg.norm(
186
- embeddings2.reshape(-1, self.embedding_dim), axis=1, keepdims=True
184
+ return select_pairwise_similarity(
185
+ embeddings1, embeddings2, self.mteb_model_meta.similarity_fn_name
187
186
  )
188
- normalized1 = embeddings1 / (norm1 + 1e-10)
189
- normalized2 = embeddings2 / (norm2 + 1e-10)
190
- normalized1 = np.asarray(normalized1)
191
- normalized2 = np.asarray(normalized2)
192
- return np.sum(normalized1 * normalized2, axis=1)
193
187
 
194
188
 
195
189
  random_encoder_baseline = ModelMeta(
196
- loader=RandomEncoderBaseline, # type: ignore
190
+ loader=RandomEncoderBaseline,
197
191
  name="baseline/random-encoder-baseline",
192
+ model_type=["dense"],
198
193
  modalities=["text", "image"],
199
194
  **_common_mock_metadata,
200
195
  )
@@ -237,9 +232,9 @@ class RandomCrossEncoderBaseline:
237
232
 
238
233
 
239
234
  random_cross_encoder_baseline = ModelMeta(
240
- loader=RandomCrossEncoderBaseline, # type: ignore
235
+ loader=RandomCrossEncoderBaseline,
241
236
  name="baseline/random-cross-encoder-baseline",
237
+ model_type=["cross-encoder"],
242
238
  modalities=["text", "image"],
243
- is_cross_encoder=True,
244
239
  **_common_mock_metadata,
245
240
  )
@@ -0,0 +1,34 @@
1
+ import numpy as np
2
+
3
+ from mteb.models.model_implementations.model2vec_models import Model2VecModel
4
+ from mteb.models.model_meta import ModelMeta, ScoringFunction
5
+
6
+ potion_base_8m = ModelMeta(
7
+ loader=Model2VecModel,
8
+ name="rasgaard/m2v-dfm-large",
9
+ model_type=["dense"],
10
+ languages=["dan-Latn"],
11
+ open_weights=True,
12
+ revision="387897cfb09992e6d45ea9cd7b28b9fcf119e23a",
13
+ release_date="2025-10-08",
14
+ n_parameters=22893312,
15
+ memory_usage_mb=87,
16
+ max_tokens=np.inf,
17
+ embed_dim=256,
18
+ license="mit",
19
+ similarity_fn_name=ScoringFunction.COSINE,
20
+ framework=["NumPy", "Sentence Transformers", "safetensors"],
21
+ reference="https://huggingface.co/rasgaard/m2v-dfm-large",
22
+ use_instructions=False,
23
+ adapted_from="KennethEnevoldsen/dfm-sentence-encoder-large",
24
+ superseded_by=None,
25
+ training_datasets=set(), # distilled
26
+ public_training_code="https://github.com/MinishLab/model2vec",
27
+ public_training_data="https://huggingface.co/datasets/HuggingFaceFW/fineweb-2", # distilled on this
28
+ citation="""@article{minishlab2024model2vec,
29
+ author = {Tulkens, Stephan and {van Dongen}, Thomas},
30
+ title = {Model2Vec: Fast State-of-the-Art Static Embeddings},
31
+ year = {2024},
32
+ url = {https://github.com/MinishLab/model2vec}
33
+ }""",
34
+ )
@@ -44,6 +44,7 @@ ReasonIR_8B = ModelMeta(
44
44
  trust_remote_code=True,
45
45
  ),
46
46
  name="ReasonIR/ReasonIR-8B",
47
+ model_type=["dense"],
47
48
  languages=["eng-Latn"],
48
49
  open_weights=True,
49
50
  revision="c3d0690370ff4a8c3d3882d8dfa85c43650034fa",
@@ -55,7 +56,7 @@ ReasonIR_8B = ModelMeta(
55
56
  max_tokens=131072,
56
57
  reference="https://huggingface.co/ReasonIR/ReasonIR-8B",
57
58
  similarity_fn_name="cosine",
58
- framework=["Sentence Transformers", "PyTorch"],
59
+ framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
59
60
  use_instructions=True,
60
61
  training_datasets=REASONIR_TRAINING_DATA,
61
62
  public_training_code="https://github.com/facebookresearch/ReasonIR/tree/main/training",
@@ -154,7 +154,7 @@ REPLLAMA_CITATION = """
154
154
  """
155
155
 
156
156
  repllama_llama2_original = ModelMeta(
157
- loader=RepLLaMAModel, # type: ignore
157
+ loader=RepLLaMAModel,
158
158
  loader_kwargs=dict(
159
159
  base_model_name_or_path="meta-llama/Llama-2-7b-hf",
160
160
  device_map="auto",
@@ -162,6 +162,7 @@ repllama_llama2_original = ModelMeta(
162
162
  model_prompts=model_prompts,
163
163
  ),
164
164
  name="castorini/repllama-v1-7b-lora-passage",
165
+ model_type=["dense"],
165
166
  languages=["eng-Latn"],
166
167
  open_weights=True,
167
168
  revision="01c7f73d771dfac7d292323805ebc428287df4f9-6097554dfe6e7d93e92f55010b678bcca1e233a8", # base-peft revision
@@ -186,7 +187,7 @@ repllama_llama2_original = ModelMeta(
186
187
 
187
188
 
188
189
  repllama_llama2_reproduced = ModelMeta(
189
- loader=RepLLaMAModel, # type: ignore
190
+ loader=RepLLaMAModel,
190
191
  loader_kwargs=dict(
191
192
  base_model_name_or_path="meta-llama/Llama-2-7b-hf",
192
193
  device_map="auto",
@@ -194,6 +195,7 @@ repllama_llama2_reproduced = ModelMeta(
194
195
  model_prompts=model_prompts,
195
196
  ),
196
197
  name="samaya-ai/RepLLaMA-reproduced",
198
+ model_type=["dense"],
197
199
  languages=["eng-Latn"],
198
200
  open_weights=True,
199
201
  revision="01c7f73d771dfac7d292323805ebc428287df4f9-ad5c1d0938a1e02954bcafb4d811ba2f34052e71", # base-peft revision
@@ -205,7 +207,7 @@ repllama_llama2_reproduced = ModelMeta(
205
207
  license="apache-2.0",
206
208
  reference="https://huggingface.co/samaya-ai/RepLLaMA-reproduced",
207
209
  similarity_fn_name=ScoringFunction.COSINE,
208
- framework=["PyTorch", "Tevatron"],
210
+ framework=["PyTorch", "Tevatron", "safetensors"],
209
211
  use_instructions=True,
210
212
  citation=REPLLAMA_CITATION,
211
213
  public_training_code=None,
@@ -214,11 +214,12 @@ class JinaReranker(RerankerWrapper):
214
214
 
215
215
 
216
216
  monobert_large = ModelMeta(
217
- loader=MonoBERTReranker, # type: ignore
217
+ loader=MonoBERTReranker,
218
218
  loader_kwargs=dict(
219
219
  fp_options="float16",
220
220
  ),
221
221
  name="castorini/monobert-large-msmarco",
222
+ model_type=["cross-encoder"],
222
223
  languages=["eng-Latn"],
223
224
  open_weights=True,
224
225
  revision="0a97706f3827389da43b83348d5d18c9d53876fa",
@@ -233,17 +234,17 @@ monobert_large = ModelMeta(
233
234
  similarity_fn_name=None,
234
235
  use_instructions=None,
235
236
  training_datasets=None,
236
- framework=["Sentence Transformers", "PyTorch"],
237
- is_cross_encoder=True,
237
+ framework=["Sentence Transformers", "PyTorch", "Transformers"],
238
238
  )
239
239
 
240
240
  # languages unclear: https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual/discussions/28
241
241
  jina_reranker_multilingual = ModelMeta(
242
- loader=JinaReranker, # type: ignore
242
+ loader=JinaReranker,
243
243
  loader_kwargs=dict(
244
244
  fp_options="float16",
245
245
  ),
246
246
  name="jinaai/jina-reranker-v2-base-multilingual",
247
+ model_type=["cross-encoder"],
247
248
  languages=["eng-Latn"],
248
249
  open_weights=True,
249
250
  revision="126747772a932960028d9f4dc93bd5d9c4869be4",
@@ -258,16 +259,22 @@ jina_reranker_multilingual = ModelMeta(
258
259
  similarity_fn_name=None,
259
260
  use_instructions=None,
260
261
  training_datasets=None,
261
- framework=["Sentence Transformers", "PyTorch"],
262
- is_cross_encoder=True,
262
+ framework=[
263
+ "Sentence Transformers",
264
+ "PyTorch",
265
+ "Transformers",
266
+ "ONNX",
267
+ "safetensors",
268
+ ],
263
269
  )
264
270
 
265
271
  bge_reranker_v2_m3 = ModelMeta(
266
- loader=BGEReranker, # type: ignore
272
+ loader=BGEReranker,
267
273
  loader_kwargs=dict(
268
274
  fp_options="float16",
269
275
  ),
270
276
  name="BAAI/bge-reranker-v2-m3",
277
+ model_type=["cross-encoder"],
271
278
  languages=[
272
279
  "eng-Latn",
273
280
  "ara-Arab",
@@ -315,8 +322,7 @@ bge_reranker_v2_m3 = ModelMeta(
315
322
  similarity_fn_name=None,
316
323
  use_instructions=None,
317
324
  training_datasets=bge_m3_training_data,
318
- framework=["Sentence Transformers", "PyTorch"],
319
- is_cross_encoder=True,
325
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
320
326
  citation="""
321
327
  @misc{li2023making,
322
328
  title={Making Large Language Models A Better Foundation For Dense Retrieval},