mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (527) hide show
  1. mteb/__init__.py +6 -0
  2. mteb/_create_dataloaders.py +22 -20
  3. mteb/_evaluators/any_sts_evaluator.py +23 -14
  4. mteb/_evaluators/classification_metrics.py +54 -0
  5. mteb/_evaluators/clustering_evaluator.py +3 -3
  6. mteb/_evaluators/evaluator.py +4 -2
  7. mteb/_evaluators/image/imagetext_pairclassification_evaluator.py +18 -11
  8. mteb/_evaluators/pair_classification_evaluator.py +34 -40
  9. mteb/_evaluators/retrieval_evaluator.py +2 -2
  10. mteb/_evaluators/retrieval_metrics.py +18 -17
  11. mteb/_evaluators/sklearn_evaluator.py +25 -37
  12. mteb/_evaluators/text/bitext_mining_evaluator.py +31 -19
  13. mteb/_evaluators/text/summarization_evaluator.py +27 -20
  14. mteb/_evaluators/zeroshot_classification_evaluator.py +7 -5
  15. mteb/abstasks/_data_filter/__init__.py +0 -0
  16. mteb/abstasks/_data_filter/filters.py +125 -0
  17. mteb/abstasks/_data_filter/task_pipelines.py +105 -0
  18. mteb/abstasks/_statistics_calculation.py +23 -11
  19. mteb/abstasks/_stratification.py +18 -18
  20. mteb/abstasks/abstask.py +35 -28
  21. mteb/abstasks/aggregate_task_metadata.py +1 -9
  22. mteb/abstasks/aggregated_task.py +10 -29
  23. mteb/abstasks/classification.py +15 -12
  24. mteb/abstasks/clustering.py +20 -16
  25. mteb/abstasks/clustering_legacy.py +13 -10
  26. mteb/abstasks/image/image_text_pair_classification.py +7 -4
  27. mteb/abstasks/multilabel_classification.py +33 -22
  28. mteb/abstasks/pair_classification.py +27 -11
  29. mteb/abstasks/regression.py +4 -4
  30. mteb/abstasks/retrieval.py +28 -24
  31. mteb/abstasks/retrieval_dataset_loaders.py +2 -2
  32. mteb/abstasks/sts.py +14 -4
  33. mteb/abstasks/task_metadata.py +32 -33
  34. mteb/abstasks/text/bitext_mining.py +39 -28
  35. mteb/abstasks/text/reranking.py +8 -6
  36. mteb/abstasks/text/summarization.py +10 -5
  37. mteb/abstasks/zeroshot_classification.py +8 -4
  38. mteb/benchmarks/_create_table.py +84 -37
  39. mteb/benchmarks/benchmark.py +77 -16
  40. mteb/benchmarks/benchmarks/__init__.py +12 -0
  41. mteb/benchmarks/benchmarks/benchmarks.py +361 -16
  42. mteb/benchmarks/get_benchmark.py +14 -53
  43. mteb/cache.py +227 -37
  44. mteb/cli/_display_tasks.py +2 -2
  45. mteb/cli/build_cli.py +110 -14
  46. mteb/cli/generate_model_card.py +43 -23
  47. mteb/deprecated_evaluator.py +71 -62
  48. mteb/descriptive_stats/BitextMining/RuSciBenchBitextMining.v2.json +61 -0
  49. mteb/descriptive_stats/Classification/HebrewSentimentAnalysis.v3.json +60 -0
  50. mteb/descriptive_stats/Classification/TurkishConstitutionalCourtViolation.json +54 -0
  51. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2CybersecurityRetrieval.json +32 -0
  52. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EconomicRetrieval.json +32 -0
  53. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EnergyRetrieval.json +32 -0
  54. mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2HrRetrieval.json +32 -0
  55. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3ComputerScienceRetrieval.json +214 -0
  56. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3EnergyRetrieval.json +214 -0
  57. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3FinanceEnRetrieval.json +214 -0
  58. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3FinanceFrRetrieval.json +214 -0
  59. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3HrRetrieval.json +214 -0
  60. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3IndustrialRetrieval.json +214 -0
  61. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3NuclearRetrieval.json +214 -0
  62. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PharmaceuticalsRetrieval.json +214 -0
  63. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PhysicsRetrieval.json +214 -0
  64. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3TelecomRetrieval.json +214 -0
  65. mteb/descriptive_stats/PairClassification/TERRa.V2.json +35 -0
  66. mteb/descriptive_stats/Reranking/JQaRARerankingLite.json +35 -0
  67. mteb/descriptive_stats/Reranking/JaCWIRRerankingLite.json +35 -0
  68. mteb/descriptive_stats/Reranking/MultiLongDocReranking.json +466 -0
  69. mteb/descriptive_stats/Retrieval/ArguAna-NL.v2.json +30 -0
  70. mteb/descriptive_stats/Retrieval/ChemRxivRetrieval.json +30 -0
  71. mteb/descriptive_stats/Retrieval/EuroPIRQRetrieval.json +116 -0
  72. mteb/descriptive_stats/Retrieval/JaCWIRRetrievalLite.json +30 -0
  73. mteb/descriptive_stats/Retrieval/JaqketRetrievalLite.json +30 -0
  74. mteb/descriptive_stats/Retrieval/MIRACLJaRetrievalLite.json +30 -0
  75. mteb/descriptive_stats/Retrieval/MrTyDiJaRetrievalLite.json +30 -0
  76. mteb/descriptive_stats/Retrieval/NFCorpus-NL.v2.json +30 -0
  77. mteb/descriptive_stats/Retrieval/NanoClimateFEVER-VN.json +30 -0
  78. mteb/descriptive_stats/Retrieval/NanoDBPedia-VN.json +30 -0
  79. mteb/descriptive_stats/Retrieval/NanoFEVER-VN.json +30 -0
  80. mteb/descriptive_stats/Retrieval/NanoHotpotQA-VN.json +30 -0
  81. mteb/descriptive_stats/Retrieval/NanoMSMARCO-VN.json +30 -0
  82. mteb/descriptive_stats/Retrieval/NanoNQ-VN.json +30 -0
  83. mteb/descriptive_stats/Retrieval/SCIDOCS-NL.v2.json +30 -0
  84. mteb/descriptive_stats/Retrieval/SQuADKorV1Retrieval.json +30 -0
  85. mteb/descriptive_stats/Retrieval/SciFact-NL.v2.json +30 -0
  86. mteb/descriptive_stats/Retrieval/TVPLRetrieval.json +30 -0
  87. mteb/evaluate.py +106 -75
  88. mteb/filter_tasks.py +25 -26
  89. mteb/get_tasks.py +29 -30
  90. mteb/languages/language_scripts.py +5 -3
  91. mteb/leaderboard/app.py +414 -151
  92. mteb/leaderboard/benchmark_selector.py +14 -5
  93. mteb/leaderboard/figures.py +13 -15
  94. mteb/leaderboard/table.py +82 -17
  95. mteb/load_results.py +12 -12
  96. mteb/models/__init__.py +4 -1
  97. mteb/models/abs_encoder.py +31 -23
  98. mteb/models/cache_wrappers/__init__.py +2 -1
  99. mteb/models/cache_wrappers/cache_backend_protocol.py +3 -5
  100. mteb/models/cache_wrappers/cache_backends/_hash_utils.py +7 -6
  101. mteb/models/cache_wrappers/cache_backends/faiss_cache.py +6 -2
  102. mteb/models/cache_wrappers/cache_backends/numpy_cache.py +43 -25
  103. mteb/models/cache_wrappers/cache_wrapper.py +3 -3
  104. mteb/models/get_model_meta.py +25 -118
  105. mteb/models/instruct_wrapper.py +33 -9
  106. mteb/models/model_implementations/align_models.py +8 -1
  107. mteb/models/model_implementations/amazon_models.py +1 -0
  108. mteb/models/model_implementations/andersborges.py +65 -0
  109. mteb/models/model_implementations/ara_models.py +9 -1
  110. mteb/models/model_implementations/arctic_models.py +16 -8
  111. mteb/models/model_implementations/b1ade_models.py +2 -1
  112. mteb/models/model_implementations/bedrock_models.py +4 -0
  113. mteb/models/model_implementations/bge_models.py +101 -17
  114. mteb/models/model_implementations/bica_model.py +35 -0
  115. mteb/models/model_implementations/blip2_models.py +13 -2
  116. mteb/models/model_implementations/blip_models.py +43 -16
  117. mteb/models/model_implementations/bm25.py +5 -4
  118. mteb/models/model_implementations/bmretriever_models.py +10 -4
  119. mteb/models/model_implementations/cadet_models.py +10 -1
  120. mteb/models/model_implementations/cde_models.py +25 -4
  121. mteb/models/model_implementations/clip_models.py +9 -6
  122. mteb/models/model_implementations/clips_models.py +100 -0
  123. mteb/models/model_implementations/codefuse_models.py +165 -3
  124. mteb/models/model_implementations/codesage_models.py +18 -3
  125. mteb/models/model_implementations/cohere_models.py +13 -6
  126. mteb/models/model_implementations/cohere_v.py +7 -2
  127. mteb/models/model_implementations/colpali_models.py +17 -9
  128. mteb/models/model_implementations/colqwen_models.py +275 -5
  129. mteb/models/model_implementations/colsmol_models.py +4 -2
  130. mteb/models/model_implementations/conan_models.py +2 -1
  131. mteb/models/model_implementations/dino_models.py +194 -23
  132. mteb/models/model_implementations/e5_instruct.py +27 -4
  133. mteb/models/model_implementations/e5_models.py +21 -110
  134. mteb/models/model_implementations/e5_v.py +7 -6
  135. mteb/models/model_implementations/eagerworks_models.py +164 -0
  136. mteb/models/model_implementations/emillykkejensen_models.py +91 -0
  137. mteb/models/model_implementations/en_code_retriever.py +2 -1
  138. mteb/models/model_implementations/euler_models.py +32 -0
  139. mteb/models/model_implementations/evaclip_models.py +4 -0
  140. mteb/models/model_implementations/fa_models.py +67 -9
  141. mteb/models/model_implementations/facebookai.py +205 -0
  142. mteb/models/model_implementations/geogpt_models.py +2 -1
  143. mteb/models/model_implementations/gme_v_models.py +17 -10
  144. mteb/models/model_implementations/google_models.py +17 -6
  145. mteb/models/model_implementations/granite_vision_embedding_models.py +8 -3
  146. mteb/models/model_implementations/gritlm_models.py +4 -2
  147. mteb/models/model_implementations/gte_models.py +99 -9
  148. mteb/models/model_implementations/hinvec_models.py +2 -1
  149. mteb/models/model_implementations/human.py +1 -0
  150. mteb/models/model_implementations/ibm_granite_models.py +36 -6
  151. mteb/models/model_implementations/inf_models.py +4 -2
  152. mteb/models/model_implementations/jasper_models.py +256 -3
  153. mteb/models/model_implementations/jina_clip.py +49 -10
  154. mteb/models/model_implementations/jina_models.py +222 -11
  155. mteb/models/model_implementations/kalm_models.py +203 -25
  156. mteb/models/model_implementations/kblab.py +37 -0
  157. mteb/models/model_implementations/kennethenevoldsen_models.py +74 -0
  158. mteb/models/model_implementations/kfst.py +25 -0
  159. mteb/models/model_implementations/kowshik24_models.py +32 -0
  160. mteb/models/model_implementations/lens_models.py +2 -0
  161. mteb/models/model_implementations/lgai_embedding_models.py +2 -1
  162. mteb/models/model_implementations/linq_models.py +4 -3
  163. mteb/models/model_implementations/listconranker.py +2 -2
  164. mteb/models/model_implementations/llm2clip_models.py +9 -6
  165. mteb/models/model_implementations/llm2vec_models.py +16 -8
  166. mteb/models/model_implementations/mcinext_models.py +7 -1
  167. mteb/models/model_implementations/mdbr_models.py +19 -3
  168. mteb/models/model_implementations/misc_models.py +422 -60
  169. mteb/models/model_implementations/mixedbread_ai_models.py +332 -0
  170. mteb/models/model_implementations/mme5_models.py +2 -1
  171. mteb/models/model_implementations/moco_models.py +15 -4
  172. mteb/models/model_implementations/mod_models.py +191 -0
  173. mteb/models/model_implementations/model2vec_models.py +27 -14
  174. mteb/models/model_implementations/moka_models.py +4 -1
  175. mteb/models/model_implementations/nbailab.py +70 -0
  176. mteb/models/model_implementations/no_instruct_sentence_models.py +3 -2
  177. mteb/models/model_implementations/nomic_models.py +173 -6
  178. mteb/models/model_implementations/nomic_models_vision.py +8 -3
  179. mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py +32 -19
  180. mteb/models/model_implementations/nvidia_models.py +155 -20
  181. mteb/models/model_implementations/octen_models.py +254 -0
  182. mteb/models/model_implementations/openai_models.py +20 -16
  183. mteb/models/model_implementations/openclip_models.py +37 -13
  184. mteb/models/model_implementations/opensearch_neural_sparse_models.py +10 -5
  185. mteb/models/model_implementations/ops_moa_models.py +5 -3
  186. mteb/models/model_implementations/ordalietech_solon_embeddings_mini_beta_1_1.py +1 -1
  187. mteb/models/model_implementations/pawan_models.py +39 -0
  188. mteb/models/model_implementations/piccolo_models.py +9 -1
  189. mteb/models/model_implementations/pixie_models.py +56 -0
  190. mteb/models/model_implementations/promptriever_models.py +12 -8
  191. mteb/models/model_implementations/pylate_models.py +46 -12
  192. mteb/models/model_implementations/qodo_models.py +4 -2
  193. mteb/models/model_implementations/qtack_models.py +2 -1
  194. mteb/models/model_implementations/qwen3_models.py +9 -6
  195. mteb/models/model_implementations/qzhou_models.py +5 -3
  196. mteb/models/model_implementations/random_baseline.py +19 -24
  197. mteb/models/model_implementations/rasgaard_models.py +34 -0
  198. mteb/models/model_implementations/reasonir_model.py +2 -1
  199. mteb/models/model_implementations/repllama_models.py +5 -3
  200. mteb/models/model_implementations/rerankers_custom.py +15 -9
  201. mteb/models/model_implementations/rerankers_monot5_based.py +31 -31
  202. mteb/models/model_implementations/richinfoai_models.py +2 -1
  203. mteb/models/model_implementations/ru_sentence_models.py +71 -20
  204. mteb/models/model_implementations/ruri_models.py +322 -0
  205. mteb/models/model_implementations/salesforce_models.py +6 -3
  206. mteb/models/model_implementations/samilpwc_models.py +2 -1
  207. mteb/models/model_implementations/sarashina_embedding_models.py +168 -0
  208. mteb/models/model_implementations/searchmap_models.py +2 -1
  209. mteb/models/model_implementations/seed_1_6_embedding_models.py +8 -2
  210. mteb/models/model_implementations/seed_1_6_embedding_models_1215.py +625 -0
  211. mteb/models/model_implementations/seed_models.py +1 -0
  212. mteb/models/model_implementations/sentence_transformers_models.py +177 -18
  213. mteb/models/model_implementations/shuu_model.py +32 -31
  214. mteb/models/model_implementations/siglip_models.py +30 -20
  215. mteb/models/model_implementations/slm_models.py +416 -0
  216. mteb/models/model_implementations/sonar_models.py +1 -0
  217. mteb/models/model_implementations/spartan8806_atles_champion.py +34 -0
  218. mteb/models/model_implementations/stella_models.py +23 -4
  219. mteb/models/model_implementations/tarka_models.py +376 -0
  220. mteb/models/model_implementations/text2vec_models.py +9 -3
  221. mteb/models/model_implementations/ua_sentence_models.py +11 -1
  222. mteb/models/model_implementations/uae_models.py +8 -1
  223. mteb/models/model_implementations/vdr_models.py +3 -1
  224. mteb/models/model_implementations/vi_vn_models.py +45 -6
  225. mteb/models/model_implementations/vista_models.py +2 -0
  226. mteb/models/model_implementations/vlm2vec_models.py +5 -3
  227. mteb/models/model_implementations/voyage_models.py +99 -0
  228. mteb/models/model_implementations/voyage_v.py +17 -9
  229. mteb/models/model_implementations/xyz_models.py +1 -0
  230. mteb/models/model_implementations/youtu_models.py +2 -1
  231. mteb/models/model_implementations/yuan_models.py +34 -0
  232. mteb/models/model_implementations/yuan_models_en.py +58 -0
  233. mteb/models/model_meta.py +498 -29
  234. mteb/models/models_protocols.py +22 -6
  235. mteb/models/search_encoder_index/__init__.py +7 -0
  236. mteb/models/search_encoder_index/search_backend_protocol.py +50 -0
  237. mteb/models/search_encoder_index/search_indexes/__init__.py +5 -0
  238. mteb/models/search_encoder_index/search_indexes/faiss_search_index.py +160 -0
  239. mteb/models/search_wrappers.py +197 -65
  240. mteb/models/sentence_transformer_wrapper.py +52 -32
  241. mteb/models/vllm_wrapper.py +327 -0
  242. mteb/py.typed +0 -0
  243. mteb/results/benchmark_results.py +114 -65
  244. mteb/results/model_result.py +63 -26
  245. mteb/results/task_result.py +117 -77
  246. mteb/similarity_functions.py +60 -7
  247. mteb/tasks/bitext_mining/multilingual/__init__.py +2 -1
  248. mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py +4 -2
  249. mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining_fast.py +1 -1
  250. mteb/tasks/bitext_mining/multilingual/ru_sci_bench_bitext_mining.py +47 -5
  251. mteb/tasks/bitext_mining/multilingual/web_faq_bitext_mining.py +2 -6
  252. mteb/tasks/classification/ara/ajgt.py +1 -2
  253. mteb/tasks/classification/ara/hotel_review_sentiment_classification.py +1 -2
  254. mteb/tasks/classification/ara/online_store_review_sentiment_classification.py +1 -2
  255. mteb/tasks/classification/ara/restaurant_review_sentiment_classification.py +1 -2
  256. mteb/tasks/classification/ara/tweet_emotion_classification.py +1 -2
  257. mteb/tasks/classification/ara/tweet_sarcasm_classification.py +1 -2
  258. mteb/tasks/classification/ben/bengali_document_classification.py +1 -2
  259. mteb/tasks/classification/ben/bengali_hate_speech_classification.py +1 -2
  260. mteb/tasks/classification/ben/bengali_sentiment_analysis.py +1 -2
  261. mteb/tasks/classification/ces/csfdcz_movie_review_sentiment_classification.py +1 -2
  262. mteb/tasks/classification/ces/czech_product_review_sentiment_classification.py +1 -2
  263. mteb/tasks/classification/ces/czech_so_me_sentiment_classification.py +1 -2
  264. mteb/tasks/classification/dan/angry_tweets_classification.py +1 -2
  265. mteb/tasks/classification/dan/danish_political_comments_classification.py +1 -2
  266. mteb/tasks/classification/dan/ddisco_cohesion_classification.py +1 -2
  267. mteb/tasks/classification/dan/dk_hate_classification.py +2 -3
  268. mteb/tasks/classification/deu/german_politicians_twitter_sentiment_classification.py +1 -2
  269. mteb/tasks/classification/deu/ten_k_gnad_classification.py +1 -2
  270. mteb/tasks/classification/eng/amazon_polarity_classification.py +1 -2
  271. mteb/tasks/classification/eng/arxiv_classification.py +1 -2
  272. mteb/tasks/classification/eng/banking77_classification.py +1 -2
  273. mteb/tasks/classification/eng/dbpedia_classification.py +1 -2
  274. mteb/tasks/classification/eng/emotion_classification.py +1 -2
  275. mteb/tasks/classification/eng/financial_phrasebank_classification.py +1 -2
  276. mteb/tasks/classification/eng/frenk_en_classification.py +1 -2
  277. mteb/tasks/classification/eng/gtsrb_classification.py +1 -1
  278. mteb/tasks/classification/eng/imdb_classification.py +1 -2
  279. mteb/tasks/classification/eng/legal_bench_classification.py +14 -120
  280. mteb/tasks/classification/eng/news_classification.py +1 -2
  281. mteb/tasks/classification/eng/patch_camelyon_classification.py +1 -1
  282. mteb/tasks/classification/eng/patent_classification.py +1 -2
  283. mteb/tasks/classification/eng/poem_sentiment_classification.py +1 -2
  284. mteb/tasks/classification/eng/sds_eye_protection_classification.py +1 -2
  285. mteb/tasks/classification/eng/sds_gloves_classification.py +1 -2
  286. mteb/tasks/classification/eng/toxic_chat_classification.py +2 -19
  287. mteb/tasks/classification/eng/toxic_conversations_classification.py +1 -2
  288. mteb/tasks/classification/eng/tweet_sentiment_extraction_classification.py +1 -2
  289. mteb/tasks/classification/eng/tweet_topic_single_classification.py +2 -13
  290. mteb/tasks/classification/eng/ucf101_classification.py +1 -5
  291. mteb/tasks/classification/eng/wikipedia_bio_met_chem_classification.py +1 -2
  292. mteb/tasks/classification/eng/wikipedia_chem_fields_classification.py +1 -2
  293. mteb/tasks/classification/eng/wikipedia_comp_chem_spectroscopy_classification.py +1 -2
  294. mteb/tasks/classification/eng/wikipedia_crystallography_analytical_classification.py +1 -2
  295. mteb/tasks/classification/eng/wikipedia_theoretical_applied_classification.py +1 -2
  296. mteb/tasks/classification/eng/yahoo_answers_topics_classification.py +1 -2
  297. mteb/tasks/classification/eng/yelp_review_full_classification.py +1 -2
  298. mteb/tasks/classification/est/estonian_valence.py +2 -3
  299. mteb/tasks/classification/fas/fa_mteb_classification.py +7 -14
  300. mteb/tasks/classification/fil/filipino_hate_speech_classification.py +1 -2
  301. mteb/tasks/classification/fin/fin_toxicity_classification.py +2 -11
  302. mteb/tasks/classification/fra/french_book_reviews.py +1 -2
  303. mteb/tasks/classification/fra/movie_review_sentiment_classification.py +1 -2
  304. mteb/tasks/classification/guj/gujarati_news_classification.py +1 -2
  305. mteb/tasks/classification/heb/__init__.py +6 -1
  306. mteb/tasks/classification/heb/hebrew_sentiment_analysis.py +62 -4
  307. mteb/tasks/classification/hin/hindi_discourse_classification.py +1 -2
  308. mteb/tasks/classification/hin/sentiment_analysis_hindi.py +1 -2
  309. mteb/tasks/classification/hrv/frenk_hr_classification.py +1 -2
  310. mteb/tasks/classification/ind/indonesian_id_clickbait_classification.py +1 -2
  311. mteb/tasks/classification/ind/indonesian_mongabay_conservation_classification.py +1 -2
  312. mteb/tasks/classification/ita/italian_linguist_acceptability_classification.py +1 -2
  313. mteb/tasks/classification/jav/javanese_imdb_classification.py +1 -2
  314. mteb/tasks/classification/jpn/wrime_classification.py +1 -2
  315. mteb/tasks/classification/kan/kannada_news_classification.py +1 -2
  316. mteb/tasks/classification/kor/klue_tc.py +1 -2
  317. mteb/tasks/classification/kor/kor_hate_classification.py +2 -17
  318. mteb/tasks/classification/kor/kor_sarcasm_classification.py +2 -19
  319. mteb/tasks/classification/kur/kurdish_sentiment_classification.py +3 -4
  320. mteb/tasks/classification/mal/malayalam_news_classification.py +1 -2
  321. mteb/tasks/classification/mar/marathi_news_classification.py +1 -2
  322. mteb/tasks/classification/mkd/macedonian_tweet_sentiment_classification.py +1 -2
  323. mteb/tasks/classification/multilingual/catalonia_tweet_classification.py +1 -6
  324. mteb/tasks/classification/multilingual/multi_hate_classification.py +1 -4
  325. mteb/tasks/classification/multilingual/ru_sci_bench_classification.py +4 -23
  326. mteb/tasks/classification/multilingual/scala_classification.py +2 -3
  327. mteb/tasks/classification/multilingual/sib200_classification.py +1 -6
  328. mteb/tasks/classification/mya/myanmar_news.py +1 -2
  329. mteb/tasks/classification/nep/nepali_news_classification.py +1 -2
  330. mteb/tasks/classification/nld/dutch_book_review_sentiment_classification.py +4 -2
  331. mteb/tasks/classification/nld/dutch_cola_classification.py +3 -0
  332. mteb/tasks/classification/nld/dutch_government_bias_classification.py +3 -0
  333. mteb/tasks/classification/nld/dutch_news_articles_classification.py +3 -0
  334. mteb/tasks/classification/nld/dutch_sarcastic_headlines_classification.py +3 -0
  335. mteb/tasks/classification/nld/iconclass_classification.py +3 -0
  336. mteb/tasks/classification/nld/open_tender_classification.py +3 -0
  337. mteb/tasks/classification/nld/vaccin_chat_nl_classification.py +3 -0
  338. mteb/tasks/classification/nob/no_rec_classification.py +1 -2
  339. mteb/tasks/classification/nob/norwegian_parliament_classification.py +1 -2
  340. mteb/tasks/classification/ory/odia_news_classification.py +1 -2
  341. mteb/tasks/classification/pol/polish_classification.py +3 -6
  342. mteb/tasks/classification/ron/moroco.py +1 -2
  343. mteb/tasks/classification/ron/romanian_reviews_sentiment.py +1 -2
  344. mteb/tasks/classification/ron/romanian_sentiment_classification.py +1 -2
  345. mteb/tasks/classification/rus/georeview_classification.py +1 -2
  346. mteb/tasks/classification/rus/headline_classification.py +1 -2
  347. mteb/tasks/classification/rus/inappropriateness_classification.py +1 -2
  348. mteb/tasks/classification/rus/ru_reviews_classification.py +1 -2
  349. mteb/tasks/classification/rus/ru_toixic_classification_okmlcup.py +1 -2
  350. mteb/tasks/classification/rus/senti_ru_eval.py +1 -2
  351. mteb/tasks/classification/sin/sinhala_news_classification.py +1 -2
  352. mteb/tasks/classification/sin/sinhala_news_source_classification.py +1 -2
  353. mteb/tasks/classification/slk/csfdsk_movie_review_sentiment_classification.py +1 -2
  354. mteb/tasks/classification/slk/slovak_hate_speech_classification.py +1 -2
  355. mteb/tasks/classification/slk/slovak_movie_review_sentiment_classification.py +1 -2
  356. mteb/tasks/classification/slv/frenk_sl_classification.py +1 -2
  357. mteb/tasks/classification/spa/spanish_news_classification.py +1 -2
  358. mteb/tasks/classification/spa/spanish_sentiment_classification.py +1 -2
  359. mteb/tasks/classification/ssw/siswati_news_classification.py +1 -2
  360. mteb/tasks/classification/swa/swahili_news_classification.py +1 -2
  361. mteb/tasks/classification/swe/dalaj_classification.py +1 -2
  362. mteb/tasks/classification/swe/swe_rec_classification.py +1 -2
  363. mteb/tasks/classification/swe/swedish_sentiment_classification.py +1 -2
  364. mteb/tasks/classification/tam/tamil_news_classification.py +1 -2
  365. mteb/tasks/classification/tel/telugu_andhra_jyoti_news_classification.py +1 -2
  366. mteb/tasks/classification/tha/wisesight_sentiment_classification.py +1 -2
  367. mteb/tasks/classification/tsn/tswana_news_classification.py +1 -2
  368. mteb/tasks/classification/tur/__init__.py +4 -0
  369. mteb/tasks/classification/tur/turkish_constitutional_court.py +41 -0
  370. mteb/tasks/classification/tur/turkish_movie_sentiment_classification.py +1 -2
  371. mteb/tasks/classification/tur/turkish_product_sentiment_classification.py +1 -2
  372. mteb/tasks/classification/ukr/ukr_formality_classification.py +2 -15
  373. mteb/tasks/classification/urd/urdu_roman_sentiment_classification.py +1 -2
  374. mteb/tasks/classification/vie/amazon_counterfactual_vn_classification.py +1 -6
  375. mteb/tasks/classification/vie/amazon_polarity_vn_classification.py +1 -6
  376. mteb/tasks/classification/vie/amazon_reviews_vn_classification.py +1 -5
  377. mteb/tasks/classification/vie/banking77_vn_classification.py +1 -5
  378. mteb/tasks/classification/vie/emotion_vn_classification.py +1 -5
  379. mteb/tasks/classification/vie/imdb_vn_classification.py +1 -5
  380. mteb/tasks/classification/vie/massive_intent_vn_classification.py +1 -5
  381. mteb/tasks/classification/vie/massive_scenario_vn_classification.py +1 -5
  382. mteb/tasks/classification/vie/mtop_domain_vn_classification.py +1 -5
  383. mteb/tasks/classification/vie/mtop_intent_vn_classification.py +1 -5
  384. mteb/tasks/classification/vie/toxic_conversations_vn_classification.py +1 -5
  385. mteb/tasks/classification/vie/tweet_sentiment_extraction_vn_classification.py +1 -5
  386. mteb/tasks/classification/vie/vie_student_feedback_classification.py +1 -2
  387. mteb/tasks/classification/zho/cmteb_classification.py +5 -10
  388. mteb/tasks/classification/zho/yue_openrice_review_classification.py +1 -2
  389. mteb/tasks/classification/zul/isi_zulu_news_classification.py +1 -2
  390. mteb/tasks/clustering/eng/hume_wiki_cities_clustering.py +1 -1
  391. mteb/tasks/clustering/eng/wiki_cities_clustering.py +1 -1
  392. mteb/tasks/clustering/jpn/mews_c16_ja_clustering.py +1 -3
  393. mteb/tasks/clustering/multilingual/sib200_clustering_s2s.py +1 -6
  394. mteb/tasks/clustering/nld/dutch_news_articles_clustering_p2p.py +3 -0
  395. mteb/tasks/clustering/nld/dutch_news_articles_clustering_s2s.py +3 -0
  396. mteb/tasks/clustering/nld/iconclass_clustering_s2s.py +3 -0
  397. mteb/tasks/clustering/nld/open_tender_clustering_p2p.py +3 -0
  398. mteb/tasks/clustering/nld/open_tender_clustering_s2s.py +3 -0
  399. mteb/tasks/clustering/nld/vabb_clustering_p2p.py +3 -0
  400. mteb/tasks/clustering/nld/vabb_clustering_s2s.py +3 -0
  401. mteb/tasks/clustering/vie/reddit_clustering_p2p_vn.py +1 -5
  402. mteb/tasks/clustering/vie/reddit_clustering_vn.py +1 -5
  403. mteb/tasks/clustering/vie/stack_exchange_clustering_p2p_vn.py +1 -5
  404. mteb/tasks/clustering/vie/stack_exchange_clustering_vn.py +1 -5
  405. mteb/tasks/clustering/vie/twenty_newsgroups_clustering_vn.py +1 -5
  406. mteb/tasks/clustering/zho/cmteb_clustering.py +2 -2
  407. mteb/tasks/image_text_pair_classification/eng/sugar_crepe.py +1 -1
  408. mteb/tasks/multilabel_classification/ita/emit_classification.py +1 -5
  409. mteb/tasks/multilabel_classification/kor/kor_hate_speech_ml_classification.py +1 -9
  410. mteb/tasks/multilabel_classification/mlt/maltese_news_classification.py +1 -6
  411. mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py +3 -0
  412. mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py +3 -0
  413. mteb/tasks/multilabel_classification/por/brazilian_toxic_tweets_classification.py +1 -6
  414. mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_group_classification.py +1 -1
  415. mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_subclass_classification.py +1 -2
  416. mteb/tasks/pair_classification/dan/talemaader_pc.py +1 -6
  417. mteb/tasks/pair_classification/eng/legal_bench_pc.py +1 -9
  418. mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py +3 -0
  419. mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py +3 -0
  420. mteb/tasks/pair_classification/rus/__init__.py +2 -2
  421. mteb/tasks/pair_classification/rus/terra.py +51 -25
  422. mteb/tasks/pair_classification/vie/sprint_duplicate_questions_pcvn.py +1 -5
  423. mteb/tasks/pair_classification/vie/twitter_sem_eval2015_pcvn.py +1 -5
  424. mteb/tasks/pair_classification/vie/twitter_url_corpus_pcvn.py +1 -5
  425. mteb/tasks/regression/multilingual/ru_sci_bench_regression.py +2 -6
  426. mteb/tasks/reranking/jpn/__init__.py +9 -1
  427. mteb/tasks/reranking/jpn/j_qa_ra_reranking_lite.py +49 -0
  428. mteb/tasks/reranking/jpn/ja_cwir_reranking_lite.py +47 -0
  429. mteb/tasks/reranking/multilingual/__init__.py +2 -0
  430. mteb/tasks/reranking/multilingual/multi_long_doc_reranking.py +70 -0
  431. mteb/tasks/reranking/multilingual/wikipedia_reranking_multilingual.py +1 -1
  432. mteb/tasks/reranking/multilingual/x_glue_wpr_reranking.py +1 -2
  433. mteb/tasks/reranking/vie/ask_ubuntu_dup_questions_vn.py +1 -5
  434. mteb/tasks/reranking/vie/sci_docs_reranking_vn.py +1 -5
  435. mteb/tasks/reranking/vie/stack_overflow_dup_questions_vn.py +1 -5
  436. mteb/tasks/retrieval/code/code_rag.py +12 -12
  437. mteb/tasks/retrieval/code/fresh_stack_retrieval.py +8 -5
  438. mteb/tasks/retrieval/dan/dan_fever_retrieval.py +1 -1
  439. mteb/tasks/retrieval/dan/tv2_nordretrieval.py +2 -2
  440. mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py +2 -2
  441. mteb/tasks/retrieval/eng/__init__.py +2 -0
  442. mteb/tasks/retrieval/eng/chemrxiv.py +33 -0
  443. mteb/tasks/retrieval/eng/cub200_i2i_retrieval.py +1 -1
  444. mteb/tasks/retrieval/eng/lit_search_retrieval.py +1 -8
  445. mteb/tasks/retrieval/eng/vidore_bench_retrieval.py +4 -0
  446. mteb/tasks/retrieval/jpn/__init__.py +8 -0
  447. mteb/tasks/retrieval/jpn/ja_cwir_retrieval.py +1 -4
  448. mteb/tasks/retrieval/jpn/ja_cwir_retrieval_lite.py +47 -0
  449. mteb/tasks/retrieval/jpn/jaqket_retrieval_lite.py +50 -0
  450. mteb/tasks/retrieval/jpn/miracl_ja_retrieval_lite.py +52 -0
  451. mteb/tasks/retrieval/jpn/mr_tydi_ja_retrieval_lite.py +48 -0
  452. mteb/tasks/retrieval/kat/georgian_faq_retrieval.py +11 -4
  453. mteb/tasks/retrieval/kor/__init__.py +16 -1
  454. mteb/tasks/retrieval/kor/kovidore2_bench_retrieval.py +142 -0
  455. mteb/tasks/retrieval/kor/squad_kor_v1_retrieval.py +47 -0
  456. mteb/tasks/retrieval/multilingual/__init__.py +24 -0
  457. mteb/tasks/retrieval/multilingual/belebele_retrieval.py +5 -4
  458. mteb/tasks/retrieval/multilingual/euro_pirq_retrieval.py +43 -0
  459. mteb/tasks/retrieval/multilingual/jina_vdr_bench_retrieval.py +56 -42
  460. mteb/tasks/retrieval/multilingual/mkqa_retrieval.py +1 -2
  461. mteb/tasks/retrieval/multilingual/mlqa_retrieval.py +1 -4
  462. mteb/tasks/retrieval/multilingual/multi_long_doc_retrieval.py +1 -2
  463. mteb/tasks/retrieval/multilingual/public_health_qa_retrieval.py +9 -4
  464. mteb/tasks/retrieval/multilingual/ru_sci_bench_retrieval.py +2 -12
  465. mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py +4 -2
  466. mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py +389 -0
  467. mteb/tasks/retrieval/nld/__init__.py +8 -4
  468. mteb/tasks/retrieval/nld/argu_ana_nl_retrieval.py +46 -27
  469. mteb/tasks/retrieval/nld/bbsard_nl_retrieval.py +3 -0
  470. mteb/tasks/retrieval/nld/dutch_news_articles_retrieval.py +3 -0
  471. mteb/tasks/retrieval/nld/legal_qa_nl_retrieval.py +3 -0
  472. mteb/tasks/retrieval/nld/nf_corpus_nl_retrieval.py +42 -25
  473. mteb/tasks/retrieval/nld/open_tender_retrieval.py +3 -0
  474. mteb/tasks/retrieval/nld/sci_fact_nl_retrieval.py +42 -24
  475. mteb/tasks/retrieval/nld/scidocsnl_retrieval.py +44 -27
  476. mteb/tasks/retrieval/nld/vabb_retrieval.py +3 -0
  477. mteb/tasks/retrieval/nob/norquad.py +2 -2
  478. mteb/tasks/retrieval/nob/snl_retrieval.py +2 -2
  479. mteb/tasks/retrieval/slk/slovak_sum_retrieval.py +1 -7
  480. mteb/tasks/retrieval/tur/tur_hist_quad.py +1 -1
  481. mteb/tasks/retrieval/vie/__init__.py +14 -6
  482. mteb/tasks/retrieval/vie/argu_ana_vn_retrieval.py +1 -5
  483. mteb/tasks/retrieval/vie/climate_fevervn_retrieval.py +40 -5
  484. mteb/tasks/retrieval/vie/cqa_dupstack_android_vn_retrieval.py +1 -5
  485. mteb/tasks/retrieval/vie/cqa_dupstack_gis_vn_retrieval.py +1 -5
  486. mteb/tasks/retrieval/vie/cqa_dupstack_mathematica_vn_retrieval.py +1 -5
  487. mteb/tasks/retrieval/vie/cqa_dupstack_physics_vn_retrieval.py +1 -5
  488. mteb/tasks/retrieval/vie/cqa_dupstack_programmers_vn_retrieval.py +1 -5
  489. mteb/tasks/retrieval/vie/cqa_dupstack_stats_vn_retrieval.py +1 -5
  490. mteb/tasks/retrieval/vie/cqa_dupstack_tex_vn_retrieval.py +1 -5
  491. mteb/tasks/retrieval/vie/cqa_dupstack_unix_vn_retrieval.py +1 -5
  492. mteb/tasks/retrieval/vie/cqa_dupstack_webmasters_vn_retrieval.py +1 -5
  493. mteb/tasks/retrieval/vie/cqa_dupstack_wordpress_vn_retrieval.py +1 -5
  494. mteb/tasks/retrieval/vie/db_pedia_vn_retrieval.py +40 -5
  495. mteb/tasks/retrieval/vie/fevervn_retrieval.py +40 -7
  496. mteb/tasks/retrieval/vie/fi_qa2018_vn_retrieval.py +1 -5
  497. mteb/tasks/retrieval/vie/green_node_table_markdown_retrieval.py +16 -1
  498. mteb/tasks/retrieval/vie/hotpot_qavn_retrieval.py +40 -6
  499. mteb/tasks/retrieval/vie/msmarcovn_retrieval.py +49 -5
  500. mteb/tasks/retrieval/vie/nf_corpus_vn_retrieval.py +1 -5
  501. mteb/tasks/retrieval/vie/nqvn_retrieval.py +40 -5
  502. mteb/tasks/retrieval/vie/quora_vn_retrieval.py +1 -6
  503. mteb/tasks/retrieval/vie/sci_fact_vn_retrieval.py +1 -5
  504. mteb/tasks/retrieval/vie/scidocsvn_retrieval.py +1 -6
  505. mteb/tasks/retrieval/vie/touche2020_vn_retrieval.py +1 -5
  506. mteb/tasks/retrieval/vie/treccovidvn_retrieval.py +1 -5
  507. mteb/tasks/retrieval/vie/tvpl_retrieval.py +42 -0
  508. mteb/tasks/retrieval/vie/zac_legal_text_retrieval.py +15 -1
  509. mteb/tasks/sts/nld/sick_nl_sts.py +1 -0
  510. mteb/tasks/sts/vie/biosses_stsvn.py +1 -5
  511. mteb/tasks/sts/vie/sickr_stsvn.py +1 -5
  512. mteb/tasks/sts/vie/sts_benchmark_stsvn.py +1 -5
  513. mteb/tasks/zeroshot_classification/eng/gtsrb.py +1 -1
  514. mteb/tasks/zeroshot_classification/eng/patch_camelyon.py +1 -1
  515. mteb/tasks/zeroshot_classification/eng/ucf101.py +1 -5
  516. mteb/types/__init__.py +2 -0
  517. mteb/types/_encoder_io.py +19 -2
  518. mteb/types/_result.py +2 -1
  519. mteb/types/statistics.py +9 -3
  520. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/METADATA +25 -8
  521. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/RECORD +525 -438
  522. mteb/models/model_implementations/mxbai_models.py +0 -102
  523. mteb/models/model_implementations/nb_sbert.py +0 -25
  524. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/WHEEL +0 -0
  525. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/entry_points.txt +0 -0
  526. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/licenses/LICENSE +0 -0
  527. {mteb-2.1.4.dist-info → mteb-2.7.2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,322 @@
1
+ from mteb.models.model_meta import ModelMeta
2
+ from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
3
+
4
+ RURI_V3_PROMPTS = {
5
+ "Retrieval-query": "検索クエリ: ",
6
+ "Retrieval-document": "検索文書: ",
7
+ "Reranking-query": "検索クエリ: ",
8
+ "Reranking-document": "検索文書: ",
9
+ "Classification": "トピック: ",
10
+ "Clustering": "トピック: ",
11
+ }
12
+
13
+ RURI_V1_V2_PROMPTS = {
14
+ "query": "クエリ: ",
15
+ "document": "文章: ",
16
+ }
17
+
18
+
19
+ RURI_CITATION = r"""@misc{Ruri,
20
+ title={{Ruri: Japanese General Text Embeddings}},
21
+ author={Hayato Tsukagoshi and Ryohei Sasano},
22
+ year={2024},
23
+ eprint={2409.07737},
24
+ archivePrefix={arXiv},
25
+ primaryClass={cs.CL},
26
+ url={https://arxiv.org/abs/2409.07737},
27
+ }"""
28
+
29
+ cl_nagoya_ruri_v3_30m = ModelMeta(
30
+ loader=sentence_transformers_loader,
31
+ loader_kwargs=dict(
32
+ model_prompts=RURI_V3_PROMPTS,
33
+ ),
34
+ name="cl-nagoya/ruri-v3-30m",
35
+ model_type=["dense"],
36
+ languages=["jpn-Jpan"],
37
+ open_weights=True,
38
+ revision="24899e5de370b56d179604a007c0d727bf144504",
39
+ release_date="2025-04-07",
40
+ n_parameters=36_705_536,
41
+ memory_usage_mb=140,
42
+ embed_dim=256,
43
+ license="apache-2.0",
44
+ max_tokens=8192,
45
+ reference="https://huggingface.co/cl-nagoya/ruri-v3-30m",
46
+ similarity_fn_name="cosine",
47
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
48
+ use_instructions=True,
49
+ superseded_by=None,
50
+ training_datasets={
51
+ "cl-nagoya/ruri-v3-dataset-ft",
52
+ },
53
+ adapted_from="sbintuitions/modernbert-ja-30m",
54
+ public_training_code=None,
55
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
56
+ citation=RURI_CITATION,
57
+ contacts=["hpprc"],
58
+ )
59
+
60
+ cl_nagoya_ruri_v3_70m = ModelMeta(
61
+ loader=sentence_transformers_loader,
62
+ loader_kwargs=dict(
63
+ model_prompts=RURI_V3_PROMPTS,
64
+ ),
65
+ name="cl-nagoya/ruri-v3-70m",
66
+ model_type=["dense"],
67
+ languages=["jpn-Jpan"],
68
+ open_weights=True,
69
+ revision="07a8b0aba47d29d2ca21f89b915c1efe2c23d1cc",
70
+ release_date="2025-04-09",
71
+ n_parameters=36_705_536,
72
+ memory_usage_mb=140,
73
+ embed_dim=256,
74
+ license="apache-2.0",
75
+ max_tokens=8192,
76
+ reference="https://huggingface.co/cl-nagoya/ruri-v3-70m",
77
+ similarity_fn_name="cosine",
78
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
79
+ use_instructions=True,
80
+ superseded_by=None,
81
+ training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
82
+ adapted_from="sbintuitions/modernbert-ja-70m",
83
+ public_training_code=None,
84
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
85
+ citation=RURI_CITATION,
86
+ contacts=["hpprc"],
87
+ )
88
+
89
+ cl_nagoya_ruri_v3_130m = ModelMeta(
90
+ loader=sentence_transformers_loader,
91
+ loader_kwargs=dict(
92
+ model_prompts=RURI_V3_PROMPTS,
93
+ ),
94
+ name="cl-nagoya/ruri-v3-130m",
95
+ model_type=["dense"],
96
+ languages=["jpn-Jpan"],
97
+ open_weights=True,
98
+ revision="e3114c6ee10dbab8b4b235fbc6dcf9dd4d5ac1a6",
99
+ release_date="2025-04-09",
100
+ n_parameters=132_140_544,
101
+ memory_usage_mb=504,
102
+ embed_dim=512,
103
+ license="apache-2.0",
104
+ max_tokens=8192,
105
+ reference="https://huggingface.co/cl-nagoya/ruri-v3-130m",
106
+ similarity_fn_name="cosine",
107
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
108
+ use_instructions=True,
109
+ superseded_by=None,
110
+ training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
111
+ adapted_from="sbintuitions/modernbert-ja-130m",
112
+ public_training_code=None,
113
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
114
+ citation=RURI_CITATION,
115
+ contacts=["hpprc"],
116
+ )
117
+
118
+ cl_nagoya_ruri_v3_310m = ModelMeta(
119
+ loader=sentence_transformers_loader,
120
+ loader_kwargs=dict(
121
+ model_prompts=RURI_V3_PROMPTS,
122
+ ),
123
+ name="cl-nagoya/ruri-v3-310m",
124
+ model_type=["dense"],
125
+ languages=["jpn-Jpan"],
126
+ open_weights=True,
127
+ revision="18b60fb8c2b9df296fb4212bb7d23ef94e579cd3",
128
+ release_date="2025-04-09",
129
+ n_parameters=314_611_968,
130
+ memory_usage_mb=1200,
131
+ embed_dim=768,
132
+ license="apache-2.0",
133
+ max_tokens=8192,
134
+ reference="https://huggingface.co/cl-nagoya/ruri-v3-310m",
135
+ similarity_fn_name="cosine",
136
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
137
+ use_instructions=True,
138
+ superseded_by=None,
139
+ training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
140
+ adapted_from="sbintuitions/modernbert-ja-310m",
141
+ public_training_code=None,
142
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
143
+ citation=RURI_CITATION,
144
+ contacts=["hpprc"],
145
+ )
146
+
147
+ cl_nagoya_ruri_small_v2 = ModelMeta(
148
+ loader=sentence_transformers_loader,
149
+ loader_kwargs=dict(
150
+ model_prompts=RURI_V1_V2_PROMPTS,
151
+ trust_remote_code=True,
152
+ ),
153
+ name="cl-nagoya/ruri-small-v2",
154
+ model_type=["dense"],
155
+ languages=["jpn-Jpan"],
156
+ open_weights=True,
157
+ revision="db18646e673b713cd0518a5bb0fefdce21e77cd9",
158
+ release_date="2024-12-05",
159
+ n_parameters=68_087_808,
160
+ memory_usage_mb=260,
161
+ embed_dim=768,
162
+ license="apache-2.0",
163
+ max_tokens=512,
164
+ reference="https://huggingface.co/cl-nagoya/ruri-small-v2",
165
+ similarity_fn_name="cosine",
166
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
167
+ use_instructions=True,
168
+ adapted_from="line-corporation/line-distilbert-base-japanese",
169
+ superseded_by=None,
170
+ training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
171
+ public_training_code=None,
172
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-v2-ft",
173
+ citation=RURI_CITATION,
174
+ contacts=["hpprc"],
175
+ )
176
+
177
+ cl_nagoya_ruri_base_v2 = ModelMeta(
178
+ loader=sentence_transformers_loader,
179
+ loader_kwargs=dict(
180
+ model_prompts=RURI_V1_V2_PROMPTS,
181
+ ),
182
+ name="cl-nagoya/ruri-base-v2",
183
+ model_type=["dense"],
184
+ languages=["jpn-Jpan"],
185
+ open_weights=True,
186
+ revision="8ce03882903668a01c83ca3b8111ac025a3bc734",
187
+ release_date="2024-12-05",
188
+ n_parameters=111_207_168,
189
+ memory_usage_mb=424,
190
+ embed_dim=768,
191
+ license="apache-2.0",
192
+ max_tokens=512,
193
+ reference="https://huggingface.co/cl-nagoya/ruri-base-v2",
194
+ similarity_fn_name="cosine",
195
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
196
+ use_instructions=True,
197
+ adapted_from="tohoku-nlp/bert-base-japanese-v3",
198
+ superseded_by=None,
199
+ training_datasets=None,
200
+ public_training_code=None,
201
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-v2-ft",
202
+ citation=RURI_CITATION,
203
+ contacts=["hpprc"],
204
+ )
205
+
206
+ cl_nagoya_ruri_large_v2 = ModelMeta(
207
+ loader=sentence_transformers_loader,
208
+ loader_kwargs=dict(
209
+ model_prompts=RURI_V1_V2_PROMPTS,
210
+ ),
211
+ name="cl-nagoya/ruri-large-v2",
212
+ model_type=["dense"],
213
+ languages=["jpn-Jpan"],
214
+ open_weights=True,
215
+ revision="42898ef34a5574977380ebf0dfd28cbfbd36438b",
216
+ release_date="2024-12-06",
217
+ n_parameters=337_441_792,
218
+ memory_usage_mb=1287,
219
+ embed_dim=1024,
220
+ license="apache-2.0",
221
+ max_tokens=512,
222
+ reference="https://huggingface.co/cl-nagoya/ruri-large-v2",
223
+ similarity_fn_name="cosine",
224
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
225
+ use_instructions=True,
226
+ adapted_from="tohoku-nlp/bert-large-japanese-v2",
227
+ superseded_by=None,
228
+ training_datasets=None,
229
+ public_training_code=None,
230
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-v2-ft",
231
+ citation=RURI_CITATION,
232
+ contacts=["hpprc"],
233
+ )
234
+
235
+ cl_nagoya_ruri_small_v1 = ModelMeta(
236
+ loader=sentence_transformers_loader,
237
+ loader_kwargs=dict(
238
+ model_prompts=RURI_V1_V2_PROMPTS,
239
+ trust_remote_code=True,
240
+ ),
241
+ name="cl-nagoya/ruri-small",
242
+ model_type=["dense"],
243
+ languages=["jpn-Jpan"],
244
+ open_weights=True,
245
+ revision="bc56ce90cd7a979f6eb199fc52dfe700bfd94bc3",
246
+ release_date="2024-08-28",
247
+ n_parameters=68_087_808,
248
+ memory_usage_mb=130,
249
+ embed_dim=768,
250
+ license="apache-2.0",
251
+ max_tokens=512,
252
+ reference="https://huggingface.co/cl-nagoya/ruri-small",
253
+ similarity_fn_name="cosine",
254
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
255
+ use_instructions=True,
256
+ adapted_from="line-corporation/line-distilbert-base-japanese",
257
+ superseded_by="cl-nagoya/ruri-small-v2",
258
+ training_datasets=None,
259
+ public_training_code=None,
260
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-ft",
261
+ citation=RURI_CITATION,
262
+ contacts=["hpprc"],
263
+ )
264
+
265
+ cl_nagoya_ruri_base_v1 = ModelMeta(
266
+ loader=sentence_transformers_loader,
267
+ loader_kwargs=dict(
268
+ model_prompts=RURI_V1_V2_PROMPTS,
269
+ ),
270
+ name="cl-nagoya/ruri-base",
271
+ model_type=["dense"],
272
+ languages=["jpn-Jpan"],
273
+ open_weights=True,
274
+ revision="1ae40b8b6c78518a499425086bab8fc16c2e4b0e",
275
+ release_date="2024-08-28",
276
+ n_parameters=111_207_168,
277
+ memory_usage_mb=212,
278
+ embed_dim=768,
279
+ license="apache-2.0",
280
+ max_tokens=512,
281
+ reference="https://huggingface.co/cl-nagoya/ruri-base",
282
+ similarity_fn_name="cosine",
283
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
284
+ use_instructions=True,
285
+ adapted_from="tohoku-nlp/bert-base-japanese-v3",
286
+ superseded_by="cl-nagoya/ruri-base-v2",
287
+ training_datasets=None,
288
+ public_training_code=None,
289
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-ft",
290
+ citation=RURI_CITATION,
291
+ contacts=["hpprc"],
292
+ )
293
+
294
+
295
+ cl_nagoya_ruri_large_v1 = ModelMeta(
296
+ loader=sentence_transformers_loader,
297
+ loader_kwargs=dict(
298
+ model_prompts=RURI_V1_V2_PROMPTS,
299
+ ),
300
+ name="cl-nagoya/ruri-large",
301
+ model_type=["dense"],
302
+ languages=["jpn-Jpan"],
303
+ open_weights=True,
304
+ revision="a011c39b13e8bc137ee13c6bc82191ece46c414c",
305
+ release_date="2024-08-28",
306
+ n_parameters=337_441_792,
307
+ memory_usage_mb=644,
308
+ embed_dim=1024,
309
+ license="apache-2.0",
310
+ max_tokens=512,
311
+ reference="https://huggingface.co/cl-nagoya/ruri-large",
312
+ similarity_fn_name="cosine",
313
+ framework=["PyTorch", "Sentence Transformers", "safetensors"],
314
+ use_instructions=True,
315
+ adapted_from="tohoku-nlp/bert-large-japanese-v2",
316
+ superseded_by="cl-nagoya/ruri-large-v2",
317
+ training_datasets=None,
318
+ public_training_code=None,
319
+ public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-ft",
320
+ citation=RURI_CITATION,
321
+ contacts=["hpprc"],
322
+ )
@@ -46,6 +46,7 @@ SFR_Embedding_2_R = ModelMeta(
46
46
  normalized=True,
47
47
  ),
48
48
  name="Salesforce/SFR-Embedding-2_R",
49
+ model_type=["dense"],
49
50
  languages=["eng-Latn"],
50
51
  open_weights=True,
51
52
  revision="91762139d94ed4371a9fa31db5551272e0b83818",
@@ -57,7 +58,7 @@ SFR_Embedding_2_R = ModelMeta(
57
58
  max_tokens=32768,
58
59
  reference="https://huggingface.co/Salesforce/SFR-Embedding-2_R",
59
60
  similarity_fn_name=ScoringFunction.COSINE,
60
- framework=["Sentence Transformers", "PyTorch"],
61
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
61
62
  use_instructions=True,
62
63
  adapted_from="intfloat/e5-mistral-7b-instruct",
63
64
  public_training_code=None,
@@ -83,6 +84,7 @@ SFR_Embedding_Code_2B_R = ModelMeta(
83
84
  normalized=True,
84
85
  ),
85
86
  name="Salesforce/SFR-Embedding-Code-2B_R",
87
+ model_type=["dense"],
86
88
  languages=["eng-Latn"],
87
89
  open_weights=True,
88
90
  revision="c73d8631a005876ed5abde34db514b1fb6566973",
@@ -94,7 +96,7 @@ SFR_Embedding_Code_2B_R = ModelMeta(
94
96
  max_tokens=8192,
95
97
  reference="https://huggingface.co/Salesforce/SFR-Embedding-Code-2B_R",
96
98
  similarity_fn_name=ScoringFunction.COSINE,
97
- framework=["Sentence Transformers", "PyTorch"],
99
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
98
100
  use_instructions=True,
99
101
  adapted_from="google/gemma-2-2b-it",
100
102
  public_training_code=None,
@@ -120,6 +122,7 @@ SFR_Embedding_Mistral = ModelMeta(
120
122
  normalized=True,
121
123
  ),
122
124
  name="Salesforce/SFR-Embedding-Mistral",
125
+ model_type=["dense"],
123
126
  languages=["eng-Latn"],
124
127
  open_weights=True,
125
128
  revision="938c560d1c236aa563b2dbdf084f28ab28bccb11",
@@ -131,7 +134,7 @@ SFR_Embedding_Mistral = ModelMeta(
131
134
  max_tokens=32768,
132
135
  reference="https://huggingface.co/Salesforce/SFR-Embedding-Mistral",
133
136
  similarity_fn_name=ScoringFunction.COSINE,
134
- framework=["Sentence Transformers", "PyTorch"],
137
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
135
138
  use_instructions=True,
136
139
  public_training_code=None,
137
140
  public_training_data=None,
@@ -43,6 +43,7 @@ samilpwc_expr = ModelMeta(
43
43
  apply_instruction_to_passages=False,
44
44
  ),
45
45
  name="SamilPwC-AXNode-GenAI/PwC-Embedding_expr",
46
+ model_type=["dense"],
46
47
  languages=[
47
48
  "kor-Hang",
48
49
  ],
@@ -56,7 +57,7 @@ samilpwc_expr = ModelMeta(
56
57
  max_tokens=514,
57
58
  reference="https://huggingface.co/SamilPwC-AXNode-GenAI/PwC-Embedding_expr",
58
59
  similarity_fn_name="cosine",
59
- framework=["Sentence Transformers", "PyTorch"],
60
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
60
61
  use_instructions=True,
61
62
  public_training_code=None,
62
63
  public_training_data=None,
@@ -0,0 +1,168 @@
1
+ from mteb.models.instruct_wrapper import InstructSentenceTransformerModel
2
+ from mteb.models.model_meta import ModelMeta
3
+ from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
4
+ from mteb.types import PromptType
5
+
6
+ SARASHINA_V2_INSTRUCTIONS = {
7
+ "Retrieval": {
8
+ "query": "クエリを与えるので、もっともクエリに意味が似ている一節を探してください。",
9
+ "document": "text: ",
10
+ },
11
+ "Reranking": {
12
+ "query": "クエリを与えるので、もっともクエリに意味が似ている一節を探してください。",
13
+ "document": "text: ",
14
+ },
15
+ "Classification": "与えられたドキュメントを適切なカテゴリに分類してください。",
16
+ "Clustering": "与えられたドキュメントのトピックまたはテーマを特定してください。",
17
+ # optimization regarding JMTEB
18
+ "LivedoorNewsClustering.v2": "与えられたニュース記事のトピックを特定してください。",
19
+ "MewsC16JaClustering": "与えられたニュース記事のトピックを特定してください。",
20
+ "SIB200ClusteringS2S": "与えられたテキストのトピックを特定してください。",
21
+ "AmazonReviewsClassification": "与えられたAmazonレビューを適切な評価カテゴリに分類してください。",
22
+ "AmazonCounterfactualClassification": "与えられたAmazonのカスタマーレビューのテキストを反事実か反事実でないかに分類してください。",
23
+ "MassiveIntentClassification": "ユーザーの発話をクエリとして与えるので、ユーザーの意図を見つけてください。",
24
+ "MassiveScenarioClassification": "ユーザーの発話をクエリとして与えるので、ユーザーシナリオを見つけてください。",
25
+ "JapaneseSentimentClassification": "与えられたテキストの感情極性をポジティブ(1)かネガティブか(0)に分類してください。",
26
+ "SIB200Classification": "与えられたテキストのトピックを特定してください。",
27
+ "WRIMEClassification": "与えられたテキストの感情極性(-2:強いネガティブ、-1:ネガティブ、0:ニュートラル、1:ポジティブ、2:強いポジティブ)を分類してください。",
28
+ "JSTS": "クエリを与えるので,もっともクエリに意味が似ている一節を探してください。",
29
+ "JSICK": "クエリを与えるので,もっともクエリに意味が似ている一節を探してください。",
30
+ "JaqketRetrieval": {
31
+ "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
32
+ "document": "text: ",
33
+ },
34
+ "MrTidyRetrieval": {
35
+ "query": "質問を与えるので、その質問に答えるWikipediaの文章を検索するしてください。",
36
+ "document": "text: ",
37
+ },
38
+ "JaGovFaqsRetrieval": {
39
+ "query": "質問を与えるので、その質問に答えるのに役立つ関連文書を検索してください。",
40
+ "document": "text: ",
41
+ },
42
+ "NLPJournalTitleAbsRetrieval.V2": {
43
+ "query": "論文のタイトルを与えるので、タイトルに対応する要約を検索してください。",
44
+ "document": "text: ",
45
+ },
46
+ "NLPJournalTitleIntroRetrieval.V2": {
47
+ "query": "論文のタイトルを与えるので、タイトルに対応する要約を検索してください。",
48
+ "document": "text: ",
49
+ },
50
+ "NLPJournalAbsIntroRetrieval.V2": {
51
+ "query": "論文の序論を与えるので、序論に対応する全文を検索してください。",
52
+ "document": "text: ",
53
+ },
54
+ "NLPJournalAbsArticleRetrieval.V2": {
55
+ "query": "論文の序論を与えるので、序論に対応する全文を検索してください。",
56
+ "document": "text: ",
57
+ },
58
+ "JaCWIRRetrieval": {
59
+ "query": "記事のタイトルを与えるので、そのタイトルと合っている記事の中身を検索してください。",
60
+ "document": "text: ",
61
+ },
62
+ "MIRACLRetrieval": {
63
+ "query": "質問を与えるので、その質問に答えるのに役立つ関連文書を検索してください。",
64
+ "document": "text: ",
65
+ },
66
+ "MintakaRetrieval": {
67
+ "query": "質問を与えるので、その質問に答えられるテキストを検索してください。",
68
+ "document": "text: ",
69
+ },
70
+ "MultiLongDocRetrieval": {
71
+ "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
72
+ "document": "text: ",
73
+ },
74
+ "ESCIReranking": {
75
+ "query": "クエリを与えるので、与えられたWeb検索クエリに答える関連文章を検索してください。",
76
+ "document": "text: ",
77
+ },
78
+ "JQaRAReranking": {
79
+ "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
80
+ "document": "text: ",
81
+ },
82
+ "JaCWIRReranking": {
83
+ "query": "記事のタイトルを与えるので、そのタイトルと合っている記事の中身を検索してください。",
84
+ "document": "text: ",
85
+ },
86
+ "MIRACLReranking": {
87
+ "query": "質問を与えるので、その質問に答えるのに役立つ関連文書を検索してください。",
88
+ "document": "text: ",
89
+ },
90
+ "MultiLongDocReranking": {
91
+ "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
92
+ "document": "text: ",
93
+ },
94
+ }
95
+
96
+
97
+ def sarashina_instruction_template(
98
+ instruction: str, prompt_type: PromptType | None = None
99
+ ) -> str:
100
+ """Instruction template for Sarashina v2 model.
101
+
102
+ Returns the instruction as-is since the prompts already contain the full format.
103
+ For document prompts, returns the instruction directly (e.g., "text: ").
104
+ """
105
+ if not instruction:
106
+ return ""
107
+ if prompt_type == PromptType.document:
108
+ return "text: "
109
+ return f"task: {instruction}\nquery: "
110
+
111
+
112
+ sbintuitions_sarashina_embedding_v2_1b = ModelMeta(
113
+ loader=InstructSentenceTransformerModel,
114
+ loader_kwargs=dict(
115
+ instruction_template=sarashina_instruction_template,
116
+ apply_instruction_to_passages=True,
117
+ prompts_dict=SARASHINA_V2_INSTRUCTIONS,
118
+ max_seq_length=8192,
119
+ ),
120
+ name="sbintuitions/sarashina-embedding-v2-1b",
121
+ model_type=["dense"],
122
+ languages=["jpn-Jpan"],
123
+ open_weights=True,
124
+ revision="1f3408afaa7b617e3445d891310a9c26dd0c68a5",
125
+ release_date="2025-07-30",
126
+ n_parameters=1_224_038_144,
127
+ memory_usage_mb=4669,
128
+ embed_dim=1792,
129
+ license="https://huggingface.co/sbintuitions/sarashina-embedding-v2-1b/blob/main/LICENSE",
130
+ max_tokens=8192,
131
+ reference="https://huggingface.co/sbintuitions/sarashina-embedding-v2-1b",
132
+ similarity_fn_name="cosine",
133
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
134
+ use_instructions=True,
135
+ adapted_from="sbintuitions/sarashina2.2-1b",
136
+ superseded_by=None,
137
+ training_datasets={"NQ", "MrTidyRetrieval"},
138
+ public_training_code=None,
139
+ public_training_data="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b",
140
+ citation=None,
141
+ contacts=["Sraym1217", "akiFQC", "lsz05"],
142
+ )
143
+
144
+ sbintuitions_sarashina_embedding_v1_1b = ModelMeta(
145
+ loader=sentence_transformers_loader,
146
+ name="sbintuitions/sarashina-embedding-v1-1b",
147
+ model_type=["dense"],
148
+ languages=["jpn-Jpan"],
149
+ open_weights=True,
150
+ revision="d060fcd8984075071e7fad81baff035cbb3b6c7e",
151
+ release_date="2024-11-22",
152
+ n_parameters=1_224_038_144,
153
+ memory_usage_mb=4669,
154
+ embed_dim=1792,
155
+ license="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b/blob/main/LICENSE",
156
+ max_tokens=8192,
157
+ reference="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b",
158
+ similarity_fn_name="cosine",
159
+ framework=["Sentence Transformers", "PyTorch", "safetensors", "Transformers"],
160
+ use_instructions=False,
161
+ adapted_from="sbintuitions/sarashina2.1-1b",
162
+ superseded_by="sbintuitions/sarashina-embedding-v2-1b",
163
+ training_datasets={"NQ", "MrTidyRetrieval"},
164
+ public_training_code=None,
165
+ public_training_data="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b",
166
+ citation=None,
167
+ contacts=["akiFQC", "lsz05"],
168
+ )
@@ -20,6 +20,7 @@ searchmap_preview = ModelMeta(
20
20
  "model_prompts": task_instructions,
21
21
  },
22
22
  name="VPLabs/SearchMap_Preview",
23
+ model_type=["dense"],
23
24
  revision="69de17ef48278ed08ba1a4e65ead8179912b696e",
24
25
  languages=["eng-Latn"],
25
26
  open_weights=True,
@@ -32,7 +33,7 @@ searchmap_preview = ModelMeta(
32
33
  max_tokens=8192,
33
34
  reference="https://huggingface.co/VPLabs/SearchMap_Preview",
34
35
  similarity_fn_name="cosine",
35
- framework=["Sentence Transformers", "PyTorch"],
36
+ framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
36
37
  public_training_code=None,
37
38
  public_training_data=None,
38
39
  training_datasets=None,
@@ -1,14 +1,15 @@
1
+ from __future__ import annotations
2
+
1
3
  import base64
2
4
  import logging
3
5
  import os
4
6
  import time
5
7
  from concurrent.futures import ThreadPoolExecutor, as_completed
6
8
  from io import BytesIO
7
- from typing import Any
9
+ from typing import TYPE_CHECKING, Any
8
10
 
9
11
  import requests
10
12
  import torch
11
- from PIL import Image
12
13
  from torch.utils.data import DataLoader
13
14
 
14
15
  from mteb._requires_package import requires_package
@@ -19,6 +20,10 @@ from mteb.models.model_implementations.nvidia_models import nvidia_training_data
19
20
  from mteb.models.model_meta import ModelMeta
20
21
  from mteb.types import Array, BatchedInput, PromptType
21
22
 
23
+ if TYPE_CHECKING:
24
+ from PIL import Image
25
+
26
+
22
27
  logger = logging.getLogger(__name__)
23
28
 
24
29
 
@@ -408,6 +413,7 @@ TASK_NAME_TO_INSTRUCTION = {
408
413
 
409
414
  seed_embedding = ModelMeta(
410
415
  name="Bytedance/Seed1.6-embedding",
416
+ model_type=["dense"],
411
417
  revision="1",
412
418
  release_date="2025-06-18",
413
419
  languages=[