mteb 2.0.5__py3-none-any.whl → 2.1.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (412) hide show
  1. mteb/__init__.py +10 -1
  2. mteb/_create_dataloaders.py +8 -3
  3. mteb/_evaluators/any_sts_evaluator.py +14 -12
  4. mteb/_evaluators/clustering_evaluator.py +1 -1
  5. mteb/_evaluators/image/imagetext_pairclassification_evaluator.py +2 -2
  6. mteb/_evaluators/pair_classification_evaluator.py +3 -1
  7. mteb/_evaluators/retrieval_metrics.py +0 -9
  8. mteb/_evaluators/sklearn_evaluator.py +15 -28
  9. mteb/_evaluators/text/bitext_mining_evaluator.py +4 -1
  10. mteb/_evaluators/text/summarization_evaluator.py +4 -2
  11. mteb/_evaluators/zeroshot_classification_evaluator.py +2 -2
  12. mteb/abstasks/_stratification.py +1 -1
  13. mteb/abstasks/abstask.py +6 -1
  14. mteb/abstasks/clustering.py +1 -1
  15. mteb/abstasks/dataset_card_template.md +1 -1
  16. mteb/abstasks/multilabel_classification.py +2 -2
  17. mteb/abstasks/retrieval.py +2 -1
  18. mteb/abstasks/retrieval_dataset_loaders.py +1 -1
  19. mteb/abstasks/task_metadata.py +2 -1
  20. mteb/benchmarks/_create_table.py +1 -3
  21. mteb/benchmarks/benchmark.py +18 -1
  22. mteb/benchmarks/benchmarks/__init__.py +4 -0
  23. mteb/benchmarks/benchmarks/benchmarks.py +125 -16
  24. mteb/benchmarks/get_benchmark.py +3 -1
  25. mteb/cache.py +7 -3
  26. mteb/descriptive_stats/Classification/DutchColaClassification.json +54 -0
  27. mteb/descriptive_stats/Classification/DutchGovernmentBiasClassification.json +54 -0
  28. mteb/descriptive_stats/Classification/DutchNewsArticlesClassification.json +90 -0
  29. mteb/descriptive_stats/Classification/DutchSarcasticHeadlinesClassification.json +54 -0
  30. mteb/descriptive_stats/Classification/IconclassClassification.json +96 -0
  31. mteb/descriptive_stats/Classification/OpenTenderClassification.json +222 -0
  32. mteb/descriptive_stats/Classification/VaccinChatNLClassification.json +1068 -0
  33. mteb/descriptive_stats/Clustering/DutchNewsArticlesClusteringP2P.json +45 -0
  34. mteb/descriptive_stats/Clustering/DutchNewsArticlesClusteringS2S.json +45 -0
  35. mteb/descriptive_stats/Clustering/IconclassClusteringS2S.json +48 -0
  36. mteb/descriptive_stats/Clustering/OpenTenderClusteringP2P.json +111 -0
  37. mteb/descriptive_stats/Clustering/OpenTenderClusteringS2S.json +111 -0
  38. mteb/descriptive_stats/Clustering/VABBClusteringP2P.json +60 -0
  39. mteb/descriptive_stats/Clustering/VABBClusteringS2S.json +60 -0
  40. mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/XFlickr30kCoT2IRetrieval.json +243 -153
  41. mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/XM3600T2IRetrieval.json +999 -629
  42. mteb/descriptive_stats/Image/Any2AnyRetrieval/OVENIT2TRetrieval.json +33 -17
  43. mteb/descriptive_stats/Image/DocumentUnderstanding/MIRACLVisionRetrieval.json +574 -0
  44. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3ComputerScienceRetrieval.json +214 -0
  45. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3EnergyRetrieval.json +214 -0
  46. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3FinanceEnRetrieval.json +214 -0
  47. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3FinanceFrRetrieval.json +214 -0
  48. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3HrRetrieval.json +214 -0
  49. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3IndustrialRetrieval.json +214 -0
  50. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3NuclearRetrieval.json +214 -0
  51. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PharmaceuticalsRetrieval.json +214 -0
  52. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PhysicsRetrieval.json +214 -0
  53. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3TelecomRetrieval.json +214 -0
  54. mteb/descriptive_stats/MultilabelClassification/CovidDisinformationNLMultiLabelClassification.json +84 -0
  55. mteb/descriptive_stats/MultilabelClassification/VABBMultiLabelClassification.json +156 -0
  56. mteb/descriptive_stats/PairClassification/SICKNLPairClassification.json +35 -0
  57. mteb/descriptive_stats/PairClassification/XLWICNLPairClassification.json +35 -0
  58. mteb/descriptive_stats/Retrieval/ArguAna-NL.v2.json +30 -0
  59. mteb/descriptive_stats/Retrieval/ClimateFEVERHardNegatives.v2.json +30 -0
  60. mteb/descriptive_stats/Retrieval/DBPediaHardNegatives.v2.json +30 -0
  61. mteb/descriptive_stats/Retrieval/DutchNewsArticlesRetrieval.json +30 -0
  62. mteb/descriptive_stats/Retrieval/FEVERHardNegatives.v2.json +30 -0
  63. mteb/descriptive_stats/Retrieval/HotpotQAHardNegatives.v2.json +30 -0
  64. mteb/descriptive_stats/Retrieval/LegalQANLRetrieval.json +30 -0
  65. mteb/descriptive_stats/Retrieval/NFCorpus-NL.v2.json +30 -0
  66. mteb/descriptive_stats/Retrieval/OpenTenderRetrieval.json +30 -0
  67. mteb/descriptive_stats/Retrieval/QuoraRetrievalHardNegatives.v2.json +30 -0
  68. mteb/descriptive_stats/Retrieval/RiaNewsRetrievalHardNegatives.v2.json +30 -0
  69. mteb/descriptive_stats/Retrieval/SCIDOCS-NL.v2.json +30 -0
  70. mteb/descriptive_stats/Retrieval/SciFact-NL.v2.json +30 -0
  71. mteb/descriptive_stats/Retrieval/VABBRetrieval.json +30 -0
  72. mteb/descriptive_stats/Retrieval/VDRMultilingualRetrieval.json +184 -0
  73. mteb/descriptive_stats/Retrieval/WinoGrande.json +14 -14
  74. mteb/descriptive_stats/Retrieval/bBSARDNLRetrieval.json +30 -0
  75. mteb/descriptive_stats/STS/SICK-NL-STS.json +28 -0
  76. mteb/evaluate.py +26 -6
  77. mteb/languages/check_language_code.py +11 -3
  78. mteb/languages/language_scripts.py +4 -0
  79. mteb/leaderboard/app.py +5 -3
  80. mteb/leaderboard/benchmark_selector.py +4 -2
  81. mteb/leaderboard/text_segments.py +1 -1
  82. mteb/models/cache_wrappers/cache_wrapper.py +1 -1
  83. mteb/models/instruct_wrapper.py +3 -0
  84. mteb/models/model_implementations/align_models.py +6 -0
  85. mteb/models/model_implementations/andersborges.py +51 -0
  86. mteb/models/model_implementations/ara_models.py +7 -0
  87. mteb/models/model_implementations/b1ade_models.py +1 -1
  88. mteb/models/model_implementations/bge_models.py +1 -3
  89. mteb/models/model_implementations/blip2_models.py +9 -0
  90. mteb/models/model_implementations/blip_models.py +19 -0
  91. mteb/models/model_implementations/bmretriever_models.py +1 -1
  92. mteb/models/model_implementations/cadet_models.py +8 -0
  93. mteb/models/model_implementations/cde_models.py +12 -0
  94. mteb/models/model_implementations/codefuse_models.py +15 -0
  95. mteb/models/model_implementations/codesage_models.py +12 -0
  96. mteb/models/model_implementations/cohere_models.py +1 -1
  97. mteb/models/model_implementations/colqwen_models.py +57 -0
  98. mteb/models/model_implementations/emillykkejensen_models.py +70 -0
  99. mteb/models/model_implementations/gme_v_models.py +2 -2
  100. mteb/models/model_implementations/ibm_granite_models.py +1 -1
  101. mteb/models/model_implementations/inf_models.py +3 -3
  102. mteb/models/model_implementations/jasper_models.py +253 -2
  103. mteb/models/model_implementations/jina_models.py +12 -2
  104. mteb/models/model_implementations/kalm_models.py +159 -25
  105. mteb/models/model_implementations/llm2vec_models.py +1 -1
  106. mteb/models/model_implementations/misc_models.py +8 -2
  107. mteb/models/model_implementations/moco_models.py +9 -0
  108. mteb/models/model_implementations/mxbai_models.py +1 -1
  109. mteb/models/model_implementations/openclip_models.py +16 -0
  110. mteb/models/model_implementations/piccolo_models.py +6 -0
  111. mteb/models/model_implementations/rasgaard_models.py +33 -0
  112. mteb/models/model_implementations/reasonir_model.py +1 -1
  113. mteb/models/model_implementations/salesforce_models.py +1 -1
  114. mteb/models/model_implementations/seed_1_6_embedding_models.py +1 -1
  115. mteb/models/model_implementations/spartan8806_atles_champion.py +26 -0
  116. mteb/models/model_implementations/tarka_models.py +374 -0
  117. mteb/models/model_implementations/voyage_models.py +6 -7
  118. mteb/models/model_implementations/voyage_v.py +10 -9
  119. mteb/models/model_implementations/yuan_models.py +33 -0
  120. mteb/models/search_wrappers.py +6 -5
  121. mteb/results/task_result.py +19 -17
  122. mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py +4 -2
  123. mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining_fast.py +1 -1
  124. mteb/tasks/bitext_mining/multilingual/ru_sci_bench_bitext_mining.py +1 -5
  125. mteb/tasks/bitext_mining/multilingual/web_faq_bitext_mining.py +2 -6
  126. mteb/tasks/classification/ara/ajgt.py +1 -2
  127. mteb/tasks/classification/ara/hotel_review_sentiment_classification.py +1 -2
  128. mteb/tasks/classification/ara/online_store_review_sentiment_classification.py +1 -2
  129. mteb/tasks/classification/ara/restaurant_review_sentiment_classification.py +1 -2
  130. mteb/tasks/classification/ara/tweet_emotion_classification.py +1 -2
  131. mteb/tasks/classification/ara/tweet_sarcasm_classification.py +1 -2
  132. mteb/tasks/classification/ben/bengali_document_classification.py +1 -2
  133. mteb/tasks/classification/ben/bengali_hate_speech_classification.py +1 -2
  134. mteb/tasks/classification/ben/bengali_sentiment_analysis.py +1 -2
  135. mteb/tasks/classification/ces/csfdcz_movie_review_sentiment_classification.py +1 -2
  136. mteb/tasks/classification/ces/czech_product_review_sentiment_classification.py +1 -2
  137. mteb/tasks/classification/ces/czech_so_me_sentiment_classification.py +1 -2
  138. mteb/tasks/classification/dan/angry_tweets_classification.py +2 -3
  139. mteb/tasks/classification/dan/danish_political_comments_classification.py +1 -2
  140. mteb/tasks/classification/dan/ddisco_cohesion_classification.py +1 -2
  141. mteb/tasks/classification/dan/dk_hate_classification.py +1 -2
  142. mteb/tasks/classification/deu/german_politicians_twitter_sentiment_classification.py +1 -2
  143. mteb/tasks/classification/deu/ten_k_gnad_classification.py +1 -2
  144. mteb/tasks/classification/eng/amazon_polarity_classification.py +1 -2
  145. mteb/tasks/classification/eng/arxiv_classification.py +1 -2
  146. mteb/tasks/classification/eng/banking77_classification.py +1 -2
  147. mteb/tasks/classification/eng/dbpedia_classification.py +1 -2
  148. mteb/tasks/classification/eng/emotion_classification.py +1 -2
  149. mteb/tasks/classification/eng/financial_phrasebank_classification.py +1 -2
  150. mteb/tasks/classification/eng/frenk_en_classification.py +1 -2
  151. mteb/tasks/classification/eng/gtsrb_classification.py +1 -1
  152. mteb/tasks/classification/eng/imdb_classification.py +1 -2
  153. mteb/tasks/classification/eng/legal_bench_classification.py +15 -121
  154. mteb/tasks/classification/eng/news_classification.py +1 -2
  155. mteb/tasks/classification/eng/patch_camelyon_classification.py +1 -1
  156. mteb/tasks/classification/eng/patent_classification.py +1 -2
  157. mteb/tasks/classification/eng/poem_sentiment_classification.py +1 -2
  158. mteb/tasks/classification/eng/sds_eye_protection_classification.py +1 -2
  159. mteb/tasks/classification/eng/sds_gloves_classification.py +1 -2
  160. mteb/tasks/classification/eng/toxic_chat_classification.py +2 -19
  161. mteb/tasks/classification/eng/toxic_conversations_classification.py +1 -2
  162. mteb/tasks/classification/eng/tweet_sentiment_extraction_classification.py +1 -2
  163. mteb/tasks/classification/eng/tweet_topic_single_classification.py +2 -13
  164. mteb/tasks/classification/eng/ucf101_classification.py +1 -5
  165. mteb/tasks/classification/eng/wikipedia_bio_met_chem_classification.py +1 -2
  166. mteb/tasks/classification/eng/wikipedia_chem_fields_classification.py +1 -2
  167. mteb/tasks/classification/eng/wikipedia_comp_chem_spectroscopy_classification.py +1 -2
  168. mteb/tasks/classification/eng/wikipedia_crystallography_analytical_classification.py +1 -2
  169. mteb/tasks/classification/eng/wikipedia_theoretical_applied_classification.py +1 -2
  170. mteb/tasks/classification/eng/yahoo_answers_topics_classification.py +1 -2
  171. mteb/tasks/classification/eng/yelp_review_full_classification.py +1 -2
  172. mteb/tasks/classification/est/estonian_valence.py +1 -2
  173. mteb/tasks/classification/fas/fa_mteb_classification.py +7 -14
  174. mteb/tasks/classification/fil/filipino_hate_speech_classification.py +1 -2
  175. mteb/tasks/classification/fin/fin_toxicity_classification.py +2 -11
  176. mteb/tasks/classification/fra/french_book_reviews.py +1 -2
  177. mteb/tasks/classification/fra/movie_review_sentiment_classification.py +1 -2
  178. mteb/tasks/classification/guj/gujarati_news_classification.py +1 -2
  179. mteb/tasks/classification/heb/hebrew_sentiment_analysis.py +1 -2
  180. mteb/tasks/classification/hin/hindi_discourse_classification.py +1 -2
  181. mteb/tasks/classification/hin/sentiment_analysis_hindi.py +1 -2
  182. mteb/tasks/classification/hrv/frenk_hr_classification.py +1 -2
  183. mteb/tasks/classification/ind/indonesian_id_clickbait_classification.py +1 -2
  184. mteb/tasks/classification/ind/indonesian_mongabay_conservation_classification.py +1 -2
  185. mteb/tasks/classification/ita/italian_linguist_acceptability_classification.py +1 -2
  186. mteb/tasks/classification/jav/javanese_imdb_classification.py +1 -2
  187. mteb/tasks/classification/jpn/wrime_classification.py +1 -2
  188. mteb/tasks/classification/kan/kannada_news_classification.py +1 -2
  189. mteb/tasks/classification/kor/klue_tc.py +1 -2
  190. mteb/tasks/classification/kor/kor_hate_classification.py +2 -17
  191. mteb/tasks/classification/kor/kor_sarcasm_classification.py +2 -19
  192. mteb/tasks/classification/kur/kurdish_sentiment_classification.py +1 -2
  193. mteb/tasks/classification/mal/malayalam_news_classification.py +1 -2
  194. mteb/tasks/classification/mar/marathi_news_classification.py +1 -2
  195. mteb/tasks/classification/mkd/macedonian_tweet_sentiment_classification.py +1 -2
  196. mteb/tasks/classification/multilingual/catalonia_tweet_classification.py +1 -6
  197. mteb/tasks/classification/multilingual/multi_hate_classification.py +1 -4
  198. mteb/tasks/classification/multilingual/ru_sci_bench_classification.py +4 -23
  199. mteb/tasks/classification/multilingual/scala_classification.py +1 -2
  200. mteb/tasks/classification/multilingual/sib200_classification.py +1 -6
  201. mteb/tasks/classification/mya/myanmar_news.py +2 -3
  202. mteb/tasks/classification/nep/nepali_news_classification.py +1 -2
  203. mteb/tasks/classification/nld/__init__.py +16 -0
  204. mteb/tasks/classification/nld/dutch_book_review_sentiment_classification.py +4 -2
  205. mteb/tasks/classification/nld/dutch_cola_classification.py +41 -0
  206. mteb/tasks/classification/nld/dutch_government_bias_classification.py +40 -0
  207. mteb/tasks/classification/nld/dutch_news_articles_classification.py +33 -0
  208. mteb/tasks/classification/nld/dutch_sarcastic_headlines_classification.py +39 -0
  209. mteb/tasks/classification/nld/iconclass_classification.py +44 -0
  210. mteb/tasks/classification/nld/open_tender_classification.py +41 -0
  211. mteb/tasks/classification/nld/vaccin_chat_nl_classification.py +49 -0
  212. mteb/tasks/classification/nob/no_rec_classification.py +1 -2
  213. mteb/tasks/classification/nob/norwegian_parliament_classification.py +1 -2
  214. mteb/tasks/classification/ory/odia_news_classification.py +1 -2
  215. mteb/tasks/classification/pol/polish_classification.py +3 -6
  216. mteb/tasks/classification/ron/moroco.py +1 -2
  217. mteb/tasks/classification/ron/romanian_reviews_sentiment.py +1 -2
  218. mteb/tasks/classification/ron/romanian_sentiment_classification.py +1 -2
  219. mteb/tasks/classification/rus/georeview_classification.py +1 -2
  220. mteb/tasks/classification/rus/headline_classification.py +1 -2
  221. mteb/tasks/classification/rus/inappropriateness_classification.py +1 -2
  222. mteb/tasks/classification/rus/ru_reviews_classification.py +1 -2
  223. mteb/tasks/classification/rus/ru_toixic_classification_okmlcup.py +1 -2
  224. mteb/tasks/classification/rus/senti_ru_eval.py +1 -2
  225. mteb/tasks/classification/sin/sinhala_news_classification.py +1 -2
  226. mteb/tasks/classification/sin/sinhala_news_source_classification.py +1 -2
  227. mteb/tasks/classification/slk/csfdsk_movie_review_sentiment_classification.py +1 -2
  228. mteb/tasks/classification/slk/slovak_hate_speech_classification.py +1 -2
  229. mteb/tasks/classification/slk/slovak_movie_review_sentiment_classification.py +1 -2
  230. mteb/tasks/classification/slv/frenk_sl_classification.py +1 -2
  231. mteb/tasks/classification/spa/spanish_news_classification.py +1 -2
  232. mteb/tasks/classification/spa/spanish_sentiment_classification.py +1 -2
  233. mteb/tasks/classification/ssw/siswati_news_classification.py +1 -2
  234. mteb/tasks/classification/swa/swahili_news_classification.py +1 -2
  235. mteb/tasks/classification/swe/dalaj_classification.py +1 -2
  236. mteb/tasks/classification/swe/swe_rec_classification.py +1 -2
  237. mteb/tasks/classification/swe/swedish_sentiment_classification.py +1 -2
  238. mteb/tasks/classification/tam/tamil_news_classification.py +1 -2
  239. mteb/tasks/classification/tel/telugu_andhra_jyoti_news_classification.py +1 -2
  240. mteb/tasks/classification/tha/wisesight_sentiment_classification.py +1 -2
  241. mteb/tasks/classification/tha/wongnai_reviews_classification.py +1 -1
  242. mteb/tasks/classification/tsn/tswana_news_classification.py +1 -2
  243. mteb/tasks/classification/tur/turkish_movie_sentiment_classification.py +1 -2
  244. mteb/tasks/classification/tur/turkish_product_sentiment_classification.py +1 -2
  245. mteb/tasks/classification/ukr/ukr_formality_classification.py +2 -15
  246. mteb/tasks/classification/urd/urdu_roman_sentiment_classification.py +1 -2
  247. mteb/tasks/classification/vie/amazon_counterfactual_vn_classification.py +1 -6
  248. mteb/tasks/classification/vie/amazon_polarity_vn_classification.py +1 -6
  249. mteb/tasks/classification/vie/amazon_reviews_vn_classification.py +1 -5
  250. mteb/tasks/classification/vie/banking77_vn_classification.py +1 -5
  251. mteb/tasks/classification/vie/emotion_vn_classification.py +1 -5
  252. mteb/tasks/classification/vie/imdb_vn_classification.py +1 -5
  253. mteb/tasks/classification/vie/massive_intent_vn_classification.py +1 -5
  254. mteb/tasks/classification/vie/massive_scenario_vn_classification.py +1 -5
  255. mteb/tasks/classification/vie/mtop_domain_vn_classification.py +1 -5
  256. mteb/tasks/classification/vie/mtop_intent_vn_classification.py +1 -5
  257. mteb/tasks/classification/vie/toxic_conversations_vn_classification.py +1 -5
  258. mteb/tasks/classification/vie/tweet_sentiment_extraction_vn_classification.py +1 -5
  259. mteb/tasks/classification/vie/vie_student_feedback_classification.py +1 -2
  260. mteb/tasks/classification/zho/cmteb_classification.py +5 -10
  261. mteb/tasks/classification/zho/yue_openrice_review_classification.py +1 -2
  262. mteb/tasks/classification/zul/isi_zulu_news_classification.py +1 -2
  263. mteb/tasks/clustering/__init__.py +1 -0
  264. mteb/tasks/clustering/jpn/mews_c16_ja_clustering.py +1 -3
  265. mteb/tasks/clustering/multilingual/sib200_clustering_s2s.py +1 -6
  266. mteb/tasks/clustering/nld/__init__.py +17 -0
  267. mteb/tasks/clustering/nld/dutch_news_articles_clustering_p2p.py +40 -0
  268. mteb/tasks/clustering/nld/dutch_news_articles_clustering_s2s.py +40 -0
  269. mteb/tasks/clustering/nld/iconclass_clustering_s2s.py +50 -0
  270. mteb/tasks/clustering/nld/open_tender_clustering_p2p.py +54 -0
  271. mteb/tasks/clustering/nld/open_tender_clustering_s2s.py +44 -0
  272. mteb/tasks/clustering/nld/vabb_clustering_p2p.py +54 -0
  273. mteb/tasks/clustering/nld/vabb_clustering_s2s.py +54 -0
  274. mteb/tasks/clustering/vie/reddit_clustering_p2p_vn.py +1 -5
  275. mteb/tasks/clustering/vie/reddit_clustering_vn.py +1 -5
  276. mteb/tasks/clustering/vie/stack_exchange_clustering_p2p_vn.py +1 -5
  277. mteb/tasks/clustering/vie/stack_exchange_clustering_vn.py +1 -5
  278. mteb/tasks/clustering/vie/twenty_newsgroups_clustering_vn.py +1 -5
  279. mteb/tasks/multilabel_classification/__init__.py +1 -0
  280. mteb/tasks/multilabel_classification/ita/emit_classification.py +1 -5
  281. mteb/tasks/multilabel_classification/kor/kor_hate_speech_ml_classification.py +1 -9
  282. mteb/tasks/multilabel_classification/mlt/maltese_news_classification.py +1 -6
  283. mteb/tasks/multilabel_classification/nld/__init__.py +9 -0
  284. mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py +91 -0
  285. mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py +47 -0
  286. mteb/tasks/multilabel_classification/por/brazilian_toxic_tweets_classification.py +1 -6
  287. mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_group_classification.py +1 -1
  288. mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_subclass_classification.py +1 -2
  289. mteb/tasks/pair_classification/__init__.py +1 -0
  290. mteb/tasks/pair_classification/dan/talemaader_pc.py +1 -6
  291. mteb/tasks/pair_classification/eng/legal_bench_pc.py +1 -9
  292. mteb/tasks/pair_classification/multilingual/indic_xnli_pair_classification.py +9 -8
  293. mteb/tasks/pair_classification/nld/__init__.py +7 -0
  294. mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py +39 -0
  295. mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py +44 -0
  296. mteb/tasks/pair_classification/vie/sprint_duplicate_questions_pcvn.py +1 -5
  297. mteb/tasks/pair_classification/vie/twitter_sem_eval2015_pcvn.py +1 -5
  298. mteb/tasks/pair_classification/vie/twitter_url_corpus_pcvn.py +1 -5
  299. mteb/tasks/regression/multilingual/ru_sci_bench_regression.py +2 -6
  300. mteb/tasks/reranking/multilingual/x_glue_wpr_reranking.py +1 -2
  301. mteb/tasks/reranking/vie/ask_ubuntu_dup_questions_vn.py +1 -5
  302. mteb/tasks/reranking/vie/sci_docs_reranking_vn.py +1 -5
  303. mteb/tasks/reranking/vie/stack_overflow_dup_questions_vn.py +1 -5
  304. mteb/tasks/retrieval/code/code_rag.py +8 -8
  305. mteb/tasks/retrieval/dan/dan_fever_retrieval.py +1 -1
  306. mteb/tasks/retrieval/dan/tv2_nordretrieval.py +2 -2
  307. mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py +2 -2
  308. mteb/tasks/retrieval/eng/__init__.py +18 -4
  309. mteb/tasks/retrieval/eng/climate_fever_retrieval.py +68 -77
  310. mteb/tasks/retrieval/eng/dbpedia_retrieval.py +55 -50
  311. mteb/tasks/retrieval/eng/fever_retrieval.py +62 -67
  312. mteb/tasks/retrieval/eng/hateful_memes_i2t_retrieval.py +0 -4
  313. mteb/tasks/retrieval/eng/hateful_memes_t2i_retrieval.py +0 -4
  314. mteb/tasks/retrieval/eng/hotpot_qa_retrieval.py +57 -67
  315. mteb/tasks/retrieval/eng/legal_summarization_retrieval.py +1 -1
  316. mteb/tasks/retrieval/eng/lit_search_retrieval.py +1 -8
  317. mteb/tasks/retrieval/eng/memotion_i2t_retrieval.py +0 -3
  318. mteb/tasks/retrieval/eng/memotion_t2i_retrieval.py +0 -2
  319. mteb/tasks/retrieval/eng/oven_it2t_retrieval.py +1 -1
  320. mteb/tasks/retrieval/eng/quora_retrieval.py +51 -46
  321. mteb/tasks/retrieval/eng/sci_mmir_i2t_retrieval.py +0 -4
  322. mteb/tasks/retrieval/eng/sci_mmir_t2i_retrieval.py +0 -4
  323. mteb/tasks/retrieval/eng/vidore_bench_retrieval.py +0 -2
  324. mteb/tasks/retrieval/eng/wino_grande_retrieval.py +1 -1
  325. mteb/tasks/retrieval/jpn/ja_cwir_retrieval.py +1 -4
  326. mteb/tasks/retrieval/jpn/ja_gov_faqs_retrieval.py +1 -1
  327. mteb/tasks/retrieval/kat/georgian_faq_retrieval.py +11 -4
  328. mteb/tasks/retrieval/multilingual/__init__.py +22 -0
  329. mteb/tasks/retrieval/multilingual/belebele_retrieval.py +6 -5
  330. mteb/tasks/retrieval/multilingual/jina_vdr_bench_retrieval.py +0 -2
  331. mteb/tasks/retrieval/multilingual/miracl_retrieval.py +1 -1
  332. mteb/tasks/retrieval/multilingual/miracl_vision_retrieval.py +2 -9
  333. mteb/tasks/retrieval/multilingual/mkqa_retrieval.py +1 -2
  334. mteb/tasks/retrieval/multilingual/mlqa_retrieval.py +1 -4
  335. mteb/tasks/retrieval/multilingual/multi_long_doc_retrieval.py +1 -2
  336. mteb/tasks/retrieval/multilingual/public_health_qa_retrieval.py +9 -4
  337. mteb/tasks/retrieval/multilingual/ru_sci_bench_retrieval.py +2 -12
  338. mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py +0 -2
  339. mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py +399 -0
  340. mteb/tasks/retrieval/multilingual/wit_t2i_retrieval.py +0 -2
  341. mteb/tasks/retrieval/multilingual/x_flickr30k_co_t2i_retrieval.py +6 -5
  342. mteb/tasks/retrieval/multilingual/xm3600_t2i_retrieval.py +3 -4
  343. mteb/tasks/retrieval/nld/__init__.py +18 -4
  344. mteb/tasks/retrieval/nld/argu_ana_nl_retrieval.py +46 -27
  345. mteb/tasks/retrieval/nld/bbsard_nl_retrieval.py +44 -0
  346. mteb/tasks/retrieval/nld/dutch_news_articles_retrieval.py +33 -0
  347. mteb/tasks/retrieval/nld/legal_qa_nl_retrieval.py +42 -0
  348. mteb/tasks/retrieval/nld/nf_corpus_nl_retrieval.py +42 -25
  349. mteb/tasks/retrieval/nld/open_tender_retrieval.py +41 -0
  350. mteb/tasks/retrieval/nld/sci_fact_nl_retrieval.py +42 -24
  351. mteb/tasks/retrieval/nld/scidocsnl_retrieval.py +44 -27
  352. mteb/tasks/retrieval/nld/vabb_retrieval.py +44 -0
  353. mteb/tasks/retrieval/nob/norquad.py +2 -2
  354. mteb/tasks/retrieval/nob/snl_retrieval.py +2 -2
  355. mteb/tasks/retrieval/rus/__init__.py +11 -2
  356. mteb/tasks/retrieval/rus/ria_news_retrieval.py +48 -44
  357. mteb/tasks/retrieval/slk/slovak_sum_retrieval.py +1 -7
  358. mteb/tasks/retrieval/tur/tur_hist_quad.py +2 -2
  359. mteb/tasks/retrieval/vie/argu_ana_vn_retrieval.py +1 -5
  360. mteb/tasks/retrieval/vie/climate_fevervn_retrieval.py +1 -5
  361. mteb/tasks/retrieval/vie/cqa_dupstack_android_vn_retrieval.py +1 -5
  362. mteb/tasks/retrieval/vie/cqa_dupstack_gis_vn_retrieval.py +1 -5
  363. mteb/tasks/retrieval/vie/cqa_dupstack_mathematica_vn_retrieval.py +1 -5
  364. mteb/tasks/retrieval/vie/cqa_dupstack_physics_vn_retrieval.py +1 -5
  365. mteb/tasks/retrieval/vie/cqa_dupstack_programmers_vn_retrieval.py +1 -5
  366. mteb/tasks/retrieval/vie/cqa_dupstack_stats_vn_retrieval.py +1 -5
  367. mteb/tasks/retrieval/vie/cqa_dupstack_tex_vn_retrieval.py +1 -5
  368. mteb/tasks/retrieval/vie/cqa_dupstack_unix_vn_retrieval.py +1 -5
  369. mteb/tasks/retrieval/vie/cqa_dupstack_webmasters_vn_retrieval.py +1 -5
  370. mteb/tasks/retrieval/vie/cqa_dupstack_wordpress_vn_retrieval.py +1 -5
  371. mteb/tasks/retrieval/vie/db_pedia_vn_retrieval.py +1 -5
  372. mteb/tasks/retrieval/vie/fevervn_retrieval.py +1 -7
  373. mteb/tasks/retrieval/vie/fi_qa2018_vn_retrieval.py +1 -5
  374. mteb/tasks/retrieval/vie/green_node_table_markdown_retrieval.py +16 -1
  375. mteb/tasks/retrieval/vie/hotpot_qavn_retrieval.py +1 -6
  376. mteb/tasks/retrieval/vie/msmarcovn_retrieval.py +1 -5
  377. mteb/tasks/retrieval/vie/nf_corpus_vn_retrieval.py +1 -5
  378. mteb/tasks/retrieval/vie/nqvn_retrieval.py +1 -5
  379. mteb/tasks/retrieval/vie/quora_vn_retrieval.py +1 -6
  380. mteb/tasks/retrieval/vie/sci_fact_vn_retrieval.py +1 -5
  381. mteb/tasks/retrieval/vie/scidocsvn_retrieval.py +1 -6
  382. mteb/tasks/retrieval/vie/touche2020_vn_retrieval.py +1 -5
  383. mteb/tasks/retrieval/vie/treccovidvn_retrieval.py +1 -5
  384. mteb/tasks/sts/__init__.py +1 -0
  385. mteb/tasks/sts/nld/__init__.py +5 -0
  386. mteb/tasks/sts/nld/sick_nl_sts.py +42 -0
  387. mteb/tasks/sts/vie/biosses_stsvn.py +1 -5
  388. mteb/tasks/sts/vie/sickr_stsvn.py +1 -5
  389. mteb/tasks/sts/vie/sts_benchmark_stsvn.py +1 -5
  390. mteb/tasks/zeroshot_classification/eng/gtsrb.py +1 -1
  391. mteb/tasks/zeroshot_classification/eng/patch_camelyon.py +1 -1
  392. mteb/tasks/zeroshot_classification/eng/ucf101.py +1 -5
  393. mteb-2.1.19.dist-info/METADATA +253 -0
  394. {mteb-2.0.5.dist-info → mteb-2.1.19.dist-info}/RECORD +398 -330
  395. mteb/descriptive_stats/Classification/PersianTextTone.json +0 -56
  396. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchCount.json +0 -37
  397. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchDepth.json +0 -25
  398. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchDistance.json +0 -25
  399. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchRelation.json +0 -25
  400. mteb/descriptive_stats/Image/VisualSTS/STS12VisualSTS.json +0 -20
  401. mteb/descriptive_stats/Image/VisualSTS/STS13VisualSTS.json +0 -20
  402. mteb/descriptive_stats/Image/VisualSTS/STS14VisualSTS.json +0 -20
  403. mteb/descriptive_stats/Image/VisualSTS/STS15VisualSTS.json +0 -20
  404. mteb/descriptive_stats/Image/VisualSTS/STS16VisualSTS.json +0 -20
  405. mteb/descriptive_stats/Image/VisualSTS/STS17MultilingualVisualSTS.json +0 -220
  406. mteb/descriptive_stats/Image/VisualSTS/STSBenchmarkMultilingualVisualSTS.json +0 -402
  407. mteb/descriptive_stats/Reranking/InstructIR.json +0 -31
  408. mteb-2.0.5.dist-info/METADATA +0 -455
  409. {mteb-2.0.5.dist-info → mteb-2.1.19.dist-info}/WHEEL +0 -0
  410. {mteb-2.0.5.dist-info → mteb-2.1.19.dist-info}/entry_points.txt +0 -0
  411. {mteb-2.0.5.dist-info → mteb-2.1.19.dist-info}/licenses/LICENSE +0 -0
  412. {mteb-2.0.5.dist-info → mteb-2.1.19.dist-info}/top_level.txt +0 -0
mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3NuclearRetrieval.json ADDED
@@ -0,0 +1,214 @@
1
+ {
2
+ "test": {
3
+ "num_samples": 18876,
4
+ "number_of_characters": 286851,
5
+ "documents_text_statistics": null,
6
+ "documents_image_statistics": {
7
+ "min_image_width": 1686,
8
+ "average_image_width": 1805.5227434257285,
9
+ "max_image_width": 2432,
10
+ "min_image_height": 1696,
11
+ "average_image_height": 2117.6122956645345,
12
+ "max_image_height": 2289,
13
+ "unique_images": 2814
14
+ },
15
+ "queries_text_statistics": {
16
+ "total_text_length": 286851,
17
+ "min_text_length": 38,
18
+ "average_text_length": 144.0015060240964,
19
+ "max_text_length": 357,
20
+ "unique_texts": 1992
21
+ },
22
+ "queries_image_statistics": null,
23
+ "relevant_docs_statistics": {
24
+ "num_relevant_docs": 56196,
25
+ "min_relevant_docs_per_query": 1,
26
+ "average_relevant_docs_per_query": 4.701807228915663,
27
+ "max_relevant_docs_per_query": 29,
28
+ "unique_relevant_docs": 5742
29
+ },
30
+ "top_ranked_statistics": null,
31
+ "hf_subset_descriptive_stats": {
32
+ "french": {
33
+ "num_samples": 3146,
34
+ "number_of_characters": 52386,
35
+ "documents_text_statistics": null,
36
+ "documents_image_statistics": {
37
+ "min_image_width": 1686,
38
+ "average_image_width": 1805.5227434257285,
39
+ "max_image_width": 2432,
40
+ "min_image_height": 1696,
41
+ "average_image_height": 2117.6122956645345,
42
+ "max_image_height": 2289,
43
+ "unique_images": 2814
44
+ },
45
+ "queries_text_statistics": {
46
+ "total_text_length": 52386,
47
+ "min_text_length": 53,
48
+ "average_text_length": 157.789156626506,
49
+ "max_text_length": 357,
50
+ "unique_texts": 332
51
+ },
52
+ "queries_image_statistics": null,
53
+ "relevant_docs_statistics": {
54
+ "num_relevant_docs": 9366,
55
+ "min_relevant_docs_per_query": 1,
56
+ "average_relevant_docs_per_query": 4.701807228915663,
57
+ "max_relevant_docs_per_query": 29,
58
+ "unique_relevant_docs": 957
59
+ },
60
+ "top_ranked_statistics": null
61
+ },
62
+ "spanish": {
63
+ "num_samples": 3146,
64
+ "number_of_characters": 49073,
65
+ "documents_text_statistics": null,
66
+ "documents_image_statistics": {
67
+ "min_image_width": 1686,
68
+ "average_image_width": 1805.5227434257285,
69
+ "max_image_width": 2432,
70
+ "min_image_height": 1696,
71
+ "average_image_height": 2117.6122956645345,
72
+ "max_image_height": 2289,
73
+ "unique_images": 2814
74
+ },
75
+ "queries_text_statistics": {
76
+ "total_text_length": 49073,
77
+ "min_text_length": 47,
78
+ "average_text_length": 147.81024096385542,
79
+ "max_text_length": 308,
80
+ "unique_texts": 332
81
+ },
82
+ "queries_image_statistics": null,
83
+ "relevant_docs_statistics": {
84
+ "num_relevant_docs": 9366,
85
+ "min_relevant_docs_per_query": 1,
86
+ "average_relevant_docs_per_query": 4.701807228915663,
87
+ "max_relevant_docs_per_query": 29,
88
+ "unique_relevant_docs": 957
89
+ },
90
+ "top_ranked_statistics": null
91
+ },
92
+ "english": {
93
+ "num_samples": 3146,
94
+ "number_of_characters": 41172,
95
+ "documents_text_statistics": null,
96
+ "documents_image_statistics": {
97
+ "min_image_width": 1686,
98
+ "average_image_width": 1805.5227434257285,
99
+ "max_image_width": 2432,
100
+ "min_image_height": 1696,
101
+ "average_image_height": 2117.6122956645345,
102
+ "max_image_height": 2289,
103
+ "unique_images": 2814
104
+ },
105
+ "queries_text_statistics": {
106
+ "total_text_length": 41172,
107
+ "min_text_length": 38,
108
+ "average_text_length": 124.01204819277109,
109
+ "max_text_length": 245,
110
+ "unique_texts": 332
111
+ },
112
+ "queries_image_statistics": null,
113
+ "relevant_docs_statistics": {
114
+ "num_relevant_docs": 9366,
115
+ "min_relevant_docs_per_query": 1,
116
+ "average_relevant_docs_per_query": 4.701807228915663,
117
+ "max_relevant_docs_per_query": 29,
118
+ "unique_relevant_docs": 957
119
+ },
120
+ "top_ranked_statistics": null
121
+ },
122
+ "german": {
123
+ "num_samples": 3146,
124
+ "number_of_characters": 48467,
125
+ "documents_text_statistics": null,
126
+ "documents_image_statistics": {
127
+ "min_image_width": 1686,
128
+ "average_image_width": 1805.5227434257285,
129
+ "max_image_width": 2432,
130
+ "min_image_height": 1696,
131
+ "average_image_height": 2117.6122956645345,
132
+ "max_image_height": 2289,
133
+ "unique_images": 2814
134
+ },
135
+ "queries_text_statistics": {
136
+ "total_text_length": 48467,
137
+ "min_text_length": 45,
138
+ "average_text_length": 145.98493975903614,
139
+ "max_text_length": 300,
140
+ "unique_texts": 332
141
+ },
142
+ "queries_image_statistics": null,
143
+ "relevant_docs_statistics": {
144
+ "num_relevant_docs": 9366,
145
+ "min_relevant_docs_per_query": 1,
146
+ "average_relevant_docs_per_query": 4.701807228915663,
147
+ "max_relevant_docs_per_query": 29,
148
+ "unique_relevant_docs": 957
149
+ },
150
+ "top_ranked_statistics": null
151
+ },
152
+ "italian": {
153
+ "num_samples": 3146,
154
+ "number_of_characters": 49711,
155
+ "documents_text_statistics": null,
156
+ "documents_image_statistics": {
157
+ "min_image_width": 1686,
158
+ "average_image_width": 1805.5227434257285,
159
+ "max_image_width": 2432,
160
+ "min_image_height": 1696,
161
+ "average_image_height": 2117.6122956645345,
162
+ "max_image_height": 2289,
163
+ "unique_images": 2814
164
+ },
165
+ "queries_text_statistics": {
166
+ "total_text_length": 49711,
167
+ "min_text_length": 46,
168
+ "average_text_length": 149.73192771084337,
169
+ "max_text_length": 335,
170
+ "unique_texts": 332
171
+ },
172
+ "queries_image_statistics": null,
173
+ "relevant_docs_statistics": {
174
+ "num_relevant_docs": 9366,
175
+ "min_relevant_docs_per_query": 1,
176
+ "average_relevant_docs_per_query": 4.701807228915663,
177
+ "max_relevant_docs_per_query": 29,
178
+ "unique_relevant_docs": 957
179
+ },
180
+ "top_ranked_statistics": null
181
+ },
182
+ "portuguese": {
183
+ "num_samples": 3146,
184
+ "number_of_characters": 46042,
185
+ "documents_text_statistics": null,
186
+ "documents_image_statistics": {
187
+ "min_image_width": 1686,
188
+ "average_image_width": 1805.5227434257285,
189
+ "max_image_width": 2432,
190
+ "min_image_height": 1696,
191
+ "average_image_height": 2117.6122956645345,
192
+ "max_image_height": 2289,
193
+ "unique_images": 2814
194
+ },
195
+ "queries_text_statistics": {
196
+ "total_text_length": 46042,
197
+ "min_text_length": 47,
198
+ "average_text_length": 138.68072289156626,
199
+ "max_text_length": 301,
200
+ "unique_texts": 332
201
+ },
202
+ "queries_image_statistics": null,
203
+ "relevant_docs_statistics": {
204
+ "num_relevant_docs": 9366,
205
+ "min_relevant_docs_per_query": 1,
206
+ "average_relevant_docs_per_query": 4.701807228915663,
207
+ "max_relevant_docs_per_query": 29,
208
+ "unique_relevant_docs": 957
209
+ },
210
+ "top_ranked_statistics": null
211
+ }
212
+ }
213
+ }
214
+ }
mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PharmaceuticalsRetrieval.json ADDED
@@ -0,0 +1,214 @@
1
+ {
2
+ "test": {
3
+ "num_samples": 16062,
4
+ "number_of_characters": 273516,
5
+ "documents_text_statistics": null,
6
+ "documents_image_statistics": {
7
+ "min_image_width": 1221,
8
+ "average_image_width": 1857.1033290099438,
9
+ "max_image_width": 4000,
10
+ "min_image_height": 1125,
11
+ "average_image_height": 1578.8672719412018,
12
+ "max_image_height": 2250,
13
+ "unique_images": 2298
14
+ },
15
+ "queries_text_statistics": {
16
+ "total_text_length": 273516,
17
+ "min_text_length": 25,
18
+ "average_text_length": 125.23626373626374,
19
+ "max_text_length": 295,
20
+ "unique_texts": 2184
21
+ },
22
+ "queries_image_statistics": null,
23
+ "relevant_docs_statistics": {
24
+ "num_relevant_docs": 62352,
25
+ "min_relevant_docs_per_query": 1,
26
+ "average_relevant_docs_per_query": 4.758241758241758,
27
+ "max_relevant_docs_per_query": 26,
28
+ "unique_relevant_docs": 5676
29
+ },
30
+ "top_ranked_statistics": null,
31
+ "hf_subset_descriptive_stats": {
32
+ "french": {
33
+ "num_samples": 2677,
34
+ "number_of_characters": 49632,
35
+ "documents_text_statistics": null,
36
+ "documents_image_statistics": {
37
+ "min_image_width": 1221,
38
+ "average_image_width": 1857.1033290099438,
39
+ "max_image_width": 4000,
40
+ "min_image_height": 1125,
41
+ "average_image_height": 1578.8672719412018,
42
+ "max_image_height": 2250,
43
+ "unique_images": 2298
44
+ },
45
+ "queries_text_statistics": {
46
+ "total_text_length": 49632,
47
+ "min_text_length": 37,
48
+ "average_text_length": 136.35164835164835,
49
+ "max_text_length": 273,
50
+ "unique_texts": 364
51
+ },
52
+ "queries_image_statistics": null,
53
+ "relevant_docs_statistics": {
54
+ "num_relevant_docs": 10392,
55
+ "min_relevant_docs_per_query": 1,
56
+ "average_relevant_docs_per_query": 4.758241758241758,
57
+ "max_relevant_docs_per_query": 26,
58
+ "unique_relevant_docs": 946
59
+ },
60
+ "top_ranked_statistics": null
61
+ },
62
+ "spanish": {
63
+ "num_samples": 2677,
64
+ "number_of_characters": 46902,
65
+ "documents_text_statistics": null,
66
+ "documents_image_statistics": {
67
+ "min_image_width": 1221,
68
+ "average_image_width": 1857.1033290099438,
69
+ "max_image_width": 4000,
70
+ "min_image_height": 1125,
71
+ "average_image_height": 1578.8672719412018,
72
+ "max_image_height": 2250,
73
+ "unique_images": 2298
74
+ },
75
+ "queries_text_statistics": {
76
+ "total_text_length": 46902,
77
+ "min_text_length": 34,
78
+ "average_text_length": 128.85164835164835,
79
+ "max_text_length": 256,
80
+ "unique_texts": 364
81
+ },
82
+ "queries_image_statistics": null,
83
+ "relevant_docs_statistics": {
84
+ "num_relevant_docs": 10392,
85
+ "min_relevant_docs_per_query": 1,
86
+ "average_relevant_docs_per_query": 4.758241758241758,
87
+ "max_relevant_docs_per_query": 26,
88
+ "unique_relevant_docs": 946
89
+ },
90
+ "top_ranked_statistics": null
91
+ },
92
+ "english": {
93
+ "num_samples": 2677,
94
+ "number_of_characters": 39067,
95
+ "documents_text_statistics": null,
96
+ "documents_image_statistics": {
97
+ "min_image_width": 1221,
98
+ "average_image_width": 1857.1033290099438,
99
+ "max_image_width": 4000,
100
+ "min_image_height": 1125,
101
+ "average_image_height": 1578.8672719412018,
102
+ "max_image_height": 2250,
103
+ "unique_images": 2298
104
+ },
105
+ "queries_text_statistics": {
106
+ "total_text_length": 39067,
107
+ "min_text_length": 25,
108
+ "average_text_length": 107.32692307692308,
109
+ "max_text_length": 213,
110
+ "unique_texts": 364
111
+ },
112
+ "queries_image_statistics": null,
113
+ "relevant_docs_statistics": {
114
+ "num_relevant_docs": 10392,
115
+ "min_relevant_docs_per_query": 1,
116
+ "average_relevant_docs_per_query": 4.758241758241758,
117
+ "max_relevant_docs_per_query": 26,
118
+ "unique_relevant_docs": 946
119
+ },
120
+ "top_ranked_statistics": null
121
+ },
122
+ "german": {
123
+ "num_samples": 2677,
124
+ "number_of_characters": 47757,
125
+ "documents_text_statistics": null,
126
+ "documents_image_statistics": {
127
+ "min_image_width": 1221,
128
+ "average_image_width": 1857.1033290099438,
129
+ "max_image_width": 4000,
130
+ "min_image_height": 1125,
131
+ "average_image_height": 1578.8672719412018,
132
+ "max_image_height": 2250,
133
+ "unique_images": 2298
134
+ },
135
+ "queries_text_statistics": {
136
+ "total_text_length": 47757,
137
+ "min_text_length": 36,
138
+ "average_text_length": 131.20054945054946,
139
+ "max_text_length": 295,
140
+ "unique_texts": 364
141
+ },
142
+ "queries_image_statistics": null,
143
+ "relevant_docs_statistics": {
144
+ "num_relevant_docs": 10392,
145
+ "min_relevant_docs_per_query": 1,
146
+ "average_relevant_docs_per_query": 4.758241758241758,
147
+ "max_relevant_docs_per_query": 26,
148
+ "unique_relevant_docs": 946
149
+ },
150
+ "top_ranked_statistics": null
151
+ },
152
+ "italian": {
153
+ "num_samples": 2677,
154
+ "number_of_characters": 46246,
155
+ "documents_text_statistics": null,
156
+ "documents_image_statistics": {
157
+ "min_image_width": 1221,
158
+ "average_image_width": 1857.1033290099438,
159
+ "max_image_width": 4000,
160
+ "min_image_height": 1125,
161
+ "average_image_height": 1578.8672719412018,
162
+ "max_image_height": 2250,
163
+ "unique_images": 2298
164
+ },
165
+ "queries_text_statistics": {
166
+ "total_text_length": 46246,
167
+ "min_text_length": 31,
168
+ "average_text_length": 127.04945054945055,
169
+ "max_text_length": 280,
170
+ "unique_texts": 364
171
+ },
172
+ "queries_image_statistics": null,
173
+ "relevant_docs_statistics": {
174
+ "num_relevant_docs": 10392,
175
+ "min_relevant_docs_per_query": 1,
176
+ "average_relevant_docs_per_query": 4.758241758241758,
177
+ "max_relevant_docs_per_query": 26,
178
+ "unique_relevant_docs": 946
179
+ },
180
+ "top_ranked_statistics": null
181
+ },
182
+ "portuguese": {
183
+ "num_samples": 2677,
184
+ "number_of_characters": 43912,
185
+ "documents_text_statistics": null,
186
+ "documents_image_statistics": {
187
+ "min_image_width": 1221,
188
+ "average_image_width": 1857.1033290099438,
189
+ "max_image_width": 4000,
190
+ "min_image_height": 1125,
191
+ "average_image_height": 1578.8672719412018,
192
+ "max_image_height": 2250,
193
+ "unique_images": 2298
194
+ },
195
+ "queries_text_statistics": {
196
+ "total_text_length": 43912,
197
+ "min_text_length": 36,
198
+ "average_text_length": 120.63736263736264,
199
+ "max_text_length": 252,
200
+ "unique_texts": 364
201
+ },
202
+ "queries_image_statistics": null,
203
+ "relevant_docs_statistics": {
204
+ "num_relevant_docs": 10392,
205
+ "min_relevant_docs_per_query": 1,
206
+ "average_relevant_docs_per_query": 4.758241758241758,
207
+ "max_relevant_docs_per_query": 26,
208
+ "unique_relevant_docs": 946
209
+ },
210
+ "top_ranked_statistics": null
211
+ }
212
+ }
213
+ }
214
+ }
mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3PhysicsRetrieval.json ADDED
@@ -0,0 +1,214 @@
1
+ {
2
+ "test": {
3
+ "num_samples": 11856,
4
+ "number_of_characters": 220508,
5
+ "documents_text_statistics": null,
6
+ "documents_image_statistics": {
7
+ "min_image_width": 2667,
8
+ "average_image_width": 2667.0,
9
+ "max_image_width": 2667,
10
+ "min_image_height": 1500,
11
+ "average_image_height": 1500.0,
12
+ "max_image_height": 1500,
13
+ "unique_images": 1674
14
+ },
15
+ "queries_text_statistics": {
16
+ "total_text_length": 220508,
17
+ "min_text_length": 37,
18
+ "average_text_length": 121.69315673289184,
19
+ "max_text_length": 286,
20
+ "unique_texts": 1812
21
+ },
22
+ "queries_image_statistics": null,
23
+ "relevant_docs_statistics": {
24
+ "num_relevant_docs": 78408,
25
+ "min_relevant_docs_per_query": 1,
26
+ "average_relevant_docs_per_query": 7.211920529801325,
27
+ "max_relevant_docs_per_query": 28,
28
+ "unique_relevant_docs": 5772
29
+ },
30
+ "top_ranked_statistics": null,
31
+ "hf_subset_descriptive_stats": {
32
+ "french": {
33
+ "num_samples": 1976,
34
+ "number_of_characters": 38216,
35
+ "documents_text_statistics": null,
36
+ "documents_image_statistics": {
37
+ "min_image_width": 2667,
38
+ "average_image_width": 2667.0,
39
+ "max_image_width": 2667,
40
+ "min_image_height": 1500,
41
+ "average_image_height": 1500.0,
42
+ "max_image_height": 1500,
43
+ "unique_images": 1674
44
+ },
45
+ "queries_text_statistics": {
46
+ "total_text_length": 38216,
47
+ "min_text_length": 40,
48
+ "average_text_length": 126.54304635761589,
49
+ "max_text_length": 267,
50
+ "unique_texts": 302
51
+ },
52
+ "queries_image_statistics": null,
53
+ "relevant_docs_statistics": {
54
+ "num_relevant_docs": 13068,
55
+ "min_relevant_docs_per_query": 1,
56
+ "average_relevant_docs_per_query": 7.211920529801325,
57
+ "max_relevant_docs_per_query": 28,
58
+ "unique_relevant_docs": 962
59
+ },
60
+ "top_ranked_statistics": null
61
+ },
62
+ "spanish": {
63
+ "num_samples": 1976,
64
+ "number_of_characters": 37169,
65
+ "documents_text_statistics": null,
66
+ "documents_image_statistics": {
67
+ "min_image_width": 2667,
68
+ "average_image_width": 2667.0,
69
+ "max_image_width": 2667,
70
+ "min_image_height": 1500,
71
+ "average_image_height": 1500.0,
72
+ "max_image_height": 1500,
73
+ "unique_images": 1674
74
+ },
75
+ "queries_text_statistics": {
76
+ "total_text_length": 37169,
77
+ "min_text_length": 39,
78
+ "average_text_length": 123.07615894039735,
79
+ "max_text_length": 263,
80
+ "unique_texts": 302
81
+ },
82
+ "queries_image_statistics": null,
83
+ "relevant_docs_statistics": {
84
+ "num_relevant_docs": 13068,
85
+ "min_relevant_docs_per_query": 1,
86
+ "average_relevant_docs_per_query": 7.211920529801325,
87
+ "max_relevant_docs_per_query": 28,
88
+ "unique_relevant_docs": 962
89
+ },
90
+ "top_ranked_statistics": null
91
+ },
92
+ "english": {
93
+ "num_samples": 1976,
94
+ "number_of_characters": 34092,
95
+ "documents_text_statistics": null,
96
+ "documents_image_statistics": {
97
+ "min_image_width": 2667,
98
+ "average_image_width": 2667.0,
99
+ "max_image_width": 2667,
100
+ "min_image_height": 1500,
101
+ "average_image_height": 1500.0,
102
+ "max_image_height": 1500,
103
+ "unique_images": 1674
104
+ },
105
+ "queries_text_statistics": {
106
+ "total_text_length": 34092,
107
+ "min_text_length": 38,
108
+ "average_text_length": 112.88741721854305,
109
+ "max_text_length": 244,
110
+ "unique_texts": 302
111
+ },
112
+ "queries_image_statistics": null,
113
+ "relevant_docs_statistics": {
114
+ "num_relevant_docs": 13068,
115
+ "min_relevant_docs_per_query": 1,
116
+ "average_relevant_docs_per_query": 7.211920529801325,
117
+ "max_relevant_docs_per_query": 28,
118
+ "unique_relevant_docs": 962
119
+ },
120
+ "top_ranked_statistics": null
121
+ },
122
+ "german": {
123
+ "num_samples": 1976,
124
+ "number_of_characters": 39603,
125
+ "documents_text_statistics": null,
126
+ "documents_image_statistics": {
127
+ "min_image_width": 2667,
128
+ "average_image_width": 2667.0,
129
+ "max_image_width": 2667,
130
+ "min_image_height": 1500,
131
+ "average_image_height": 1500.0,
132
+ "max_image_height": 1500,
133
+ "unique_images": 1674
134
+ },
135
+ "queries_text_statistics": {
136
+ "total_text_length": 39603,
137
+ "min_text_length": 37,
138
+ "average_text_length": 131.13576158940398,
139
+ "max_text_length": 286,
140
+ "unique_texts": 302
141
+ },
142
+ "queries_image_statistics": null,
143
+ "relevant_docs_statistics": {
144
+ "num_relevant_docs": 13068,
145
+ "min_relevant_docs_per_query": 1,
146
+ "average_relevant_docs_per_query": 7.211920529801325,
147
+ "max_relevant_docs_per_query": 28,
148
+ "unique_relevant_docs": 962
149
+ },
150
+ "top_ranked_statistics": null
151
+ },
152
+ "italian": {
153
+ "num_samples": 1976,
154
+ "number_of_characters": 36485,
155
+ "documents_text_statistics": null,
156
+ "documents_image_statistics": {
157
+ "min_image_width": 2667,
158
+ "average_image_width": 2667.0,
159
+ "max_image_width": 2667,
160
+ "min_image_height": 1500,
161
+ "average_image_height": 1500.0,
162
+ "max_image_height": 1500,
163
+ "unique_images": 1674
164
+ },
165
+ "queries_text_statistics": {
166
+ "total_text_length": 36485,
167
+ "min_text_length": 39,
168
+ "average_text_length": 120.8112582781457,
169
+ "max_text_length": 253,
170
+ "unique_texts": 302
171
+ },
172
+ "queries_image_statistics": null,
173
+ "relevant_docs_statistics": {
174
+ "num_relevant_docs": 13068,
175
+ "min_relevant_docs_per_query": 1,
176
+ "average_relevant_docs_per_query": 7.211920529801325,
177
+ "max_relevant_docs_per_query": 28,
178
+ "unique_relevant_docs": 962
179
+ },
180
+ "top_ranked_statistics": null
181
+ },
182
+ "portuguese": {
183
+ "num_samples": 1976,
184
+ "number_of_characters": 34943,
185
+ "documents_text_statistics": null,
186
+ "documents_image_statistics": {
187
+ "min_image_width": 2667,
188
+ "average_image_width": 2667.0,
189
+ "max_image_width": 2667,
190
+ "min_image_height": 1500,
191
+ "average_image_height": 1500.0,
192
+ "max_image_height": 1500,
193
+ "unique_images": 1674
194
+ },
195
+ "queries_text_statistics": {
196
+ "total_text_length": 34943,
197
+ "min_text_length": 38,
198
+ "average_text_length": 115.70529801324503,
199
+ "max_text_length": 240,
200
+ "unique_texts": 302
201
+ },
202
+ "queries_image_statistics": null,
203
+ "relevant_docs_statistics": {
204
+ "num_relevant_docs": 13068,
205
+ "min_relevant_docs_per_query": 1,
206
+ "average_relevant_docs_per_query": 7.211920529801325,
207
+ "max_relevant_docs_per_query": 28,
208
+ "unique_relevant_docs": 962
209
+ },
210
+ "top_ranked_statistics": null
211
+ }
212
+ }
213
+ }
214
+ }