mteb 2.1.0__py3-none-any.whl → 2.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. mteb/_create_dataloaders.py +2 -0
  2. mteb/_evaluators/retrieval_metrics.py +0 -9
  3. mteb/abstasks/_stratification.py +1 -1
  4. mteb/abstasks/abstask.py +6 -1
  5. mteb/abstasks/dataset_card_template.md +1 -1
  6. mteb/abstasks/retrieval.py +2 -1
  7. mteb/abstasks/retrieval_dataset_loaders.py +1 -1
  8. mteb/abstasks/task_metadata.py +1 -1
  9. mteb/benchmarks/benchmarks/benchmarks.py +9 -13
  10. mteb/benchmarks/get_benchmark.py +1 -1
  11. mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/XFlickr30kCoT2IRetrieval.json +243 -153
  12. mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/XM3600T2IRetrieval.json +999 -629
  13. mteb/descriptive_stats/Image/Any2AnyRetrieval/OVENIT2TRetrieval.json +33 -17
  14. mteb/descriptive_stats/Image/DocumentUnderstanding/MIRACLVisionRetrieval.json +574 -0
  15. mteb/descriptive_stats/Retrieval/ClimateFEVERHardNegatives.v2.json +30 -0
  16. mteb/descriptive_stats/Retrieval/DBPediaHardNegatives.v2.json +30 -0
  17. mteb/descriptive_stats/Retrieval/FEVERHardNegatives.v2.json +30 -0
  18. mteb/descriptive_stats/Retrieval/HotpotQAHardNegatives.v2.json +30 -0
  19. mteb/descriptive_stats/Retrieval/QuoraRetrievalHardNegatives.v2.json +30 -0
  20. mteb/descriptive_stats/Retrieval/RiaNewsRetrievalHardNegatives.v2.json +30 -0
  21. mteb/descriptive_stats/Retrieval/VDRMultilingualRetrieval.json +184 -0
  22. mteb/languages/check_language_code.py +11 -3
  23. mteb/languages/language_scripts.py +4 -0
  24. mteb/leaderboard/app.py +1 -1
  25. mteb/leaderboard/benchmark_selector.py +1 -0
  26. mteb/leaderboard/text_segments.py +1 -1
  27. mteb/models/model_implementations/b1ade_models.py +1 -1
  28. mteb/models/model_implementations/bge_models.py +1 -3
  29. mteb/models/model_implementations/bmretriever_models.py +1 -1
  30. mteb/models/model_implementations/gme_v_models.py +2 -2
  31. mteb/models/model_implementations/ibm_granite_models.py +1 -1
  32. mteb/models/model_implementations/inf_models.py +3 -3
  33. mteb/models/model_implementations/jina_models.py +12 -2
  34. mteb/models/model_implementations/llm2vec_models.py +1 -1
  35. mteb/models/model_implementations/misc_models.py +2 -2
  36. mteb/models/model_implementations/mxbai_models.py +1 -1
  37. mteb/models/model_implementations/reasonir_model.py +1 -1
  38. mteb/models/model_implementations/salesforce_models.py +1 -1
  39. mteb/models/model_implementations/seed_1_6_embedding_models.py +1 -1
  40. mteb/models/model_implementations/voyage_v.py +9 -9
  41. mteb/results/task_result.py +6 -8
  42. mteb/tasks/classification/dan/angry_tweets_classification.py +2 -2
  43. mteb/tasks/classification/eng/legal_bench_classification.py +3 -3
  44. mteb/tasks/classification/mya/myanmar_news.py +2 -2
  45. mteb/tasks/classification/tha/wongnai_reviews_classification.py +1 -1
  46. mteb/tasks/classification/ukr/ukr_formality_classification.py +2 -2
  47. mteb/tasks/pair_classification/multilingual/indic_xnli_pair_classification.py +9 -8
  48. mteb/tasks/retrieval/code/code_rag.py +8 -8
  49. mteb/tasks/retrieval/dan/dan_fever_retrieval.py +1 -1
  50. mteb/tasks/retrieval/dan/tv2_nordretrieval.py +2 -2
  51. mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py +2 -2
  52. mteb/tasks/retrieval/eng/__init__.py +18 -4
  53. mteb/tasks/retrieval/eng/climate_fever_retrieval.py +68 -77
  54. mteb/tasks/retrieval/eng/dbpedia_retrieval.py +55 -50
  55. mteb/tasks/retrieval/eng/fever_retrieval.py +62 -67
  56. mteb/tasks/retrieval/eng/hateful_memes_i2t_retrieval.py +0 -4
  57. mteb/tasks/retrieval/eng/hateful_memes_t2i_retrieval.py +0 -4
  58. mteb/tasks/retrieval/eng/hotpot_qa_retrieval.py +57 -67
  59. mteb/tasks/retrieval/eng/legal_summarization_retrieval.py +1 -1
  60. mteb/tasks/retrieval/eng/memotion_i2t_retrieval.py +0 -3
  61. mteb/tasks/retrieval/eng/memotion_t2i_retrieval.py +0 -2
  62. mteb/tasks/retrieval/eng/oven_it2t_retrieval.py +1 -1
  63. mteb/tasks/retrieval/eng/quora_retrieval.py +51 -46
  64. mteb/tasks/retrieval/eng/sci_mmir_i2t_retrieval.py +0 -4
  65. mteb/tasks/retrieval/eng/sci_mmir_t2i_retrieval.py +0 -4
  66. mteb/tasks/retrieval/eng/vidore_bench_retrieval.py +0 -2
  67. mteb/tasks/retrieval/jpn/ja_gov_faqs_retrieval.py +1 -1
  68. mteb/tasks/retrieval/multilingual/belebele_retrieval.py +1 -1
  69. mteb/tasks/retrieval/multilingual/jina_vdr_bench_retrieval.py +0 -2
  70. mteb/tasks/retrieval/multilingual/miracl_retrieval.py +1 -1
  71. mteb/tasks/retrieval/multilingual/miracl_vision_retrieval.py +2 -9
  72. mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py +0 -2
  73. mteb/tasks/retrieval/multilingual/wit_t2i_retrieval.py +0 -2
  74. mteb/tasks/retrieval/multilingual/x_flickr30k_co_t2i_retrieval.py +6 -5
  75. mteb/tasks/retrieval/multilingual/xm3600_t2i_retrieval.py +3 -4
  76. mteb/tasks/retrieval/nob/norquad.py +2 -2
  77. mteb/tasks/retrieval/nob/snl_retrieval.py +2 -2
  78. mteb/tasks/retrieval/rus/__init__.py +11 -2
  79. mteb/tasks/retrieval/rus/ria_news_retrieval.py +48 -44
  80. mteb/tasks/retrieval/tur/tur_hist_quad.py +2 -2
  81. {mteb-2.1.0.dist-info → mteb-2.1.2.dist-info}/METADATA +5 -5
  82. {mteb-2.1.0.dist-info → mteb-2.1.2.dist-info}/RECORD +86 -91
  83. mteb/descriptive_stats/Classification/PersianTextTone.json +0 -56
  84. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchCount.json +0 -37
  85. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchDepth.json +0 -25
  86. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchDistance.json +0 -25
  87. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchRelation.json +0 -25
  88. mteb/descriptive_stats/Image/VisualSTS/STS12VisualSTS.json +0 -20
  89. mteb/descriptive_stats/Image/VisualSTS/STS13VisualSTS.json +0 -20
  90. mteb/descriptive_stats/Image/VisualSTS/STS14VisualSTS.json +0 -20
  91. mteb/descriptive_stats/Image/VisualSTS/STS15VisualSTS.json +0 -20
  92. mteb/descriptive_stats/Image/VisualSTS/STS16VisualSTS.json +0 -20
  93. mteb/descriptive_stats/Image/VisualSTS/STS17MultilingualVisualSTS.json +0 -220
  94. mteb/descriptive_stats/Image/VisualSTS/STSBenchmarkMultilingualVisualSTS.json +0 -402
  95. mteb/descriptive_stats/Reranking/InstructIR.json +0 -31
  96. {mteb-2.1.0.dist-info → mteb-2.1.2.dist-info}/WHEEL +0 -0
  97. {mteb-2.1.0.dist-info → mteb-2.1.2.dist-info}/entry_points.txt +0 -0
  98. {mteb-2.1.0.dist-info → mteb-2.1.2.dist-info}/licenses/LICENSE +0 -0
  99. {mteb-2.1.0.dist-info → mteb-2.1.2.dist-info}/top_level.txt +0 -0
mteb/descriptive_stats/Image/VisualSTS/STSBenchmarkMultilingualVisualSTS.json DELETED
@@ -1,402 +0,0 @@
1
- {
2
- "dev": {
3
- "num_samples": 15000,
4
- "min_image1_width": 448,
5
- "average_image1_width": 448.0,
6
- "max_image1_width": 448,
7
- "min_image1_height": 448,
8
- "average_image1_height": 448.0,
9
- "max_image1_height": 448,
10
- "min_image2_width": 448,
11
- "average_image2_width": 448.0,
12
- "max_image2_width": 448,
13
- "min_image2_height": 448,
14
- "average_image2_height": 448.0,
15
- "max_image2_height": 448,
16
- "min_score": 0.0,
17
- "avg_score": 2.3639075540602206,
18
- "max_score": 5.0,
19
- "hf_subset_descriptive_stats": {
20
- "en": {
21
- "num_samples": 1500,
22
- "min_image1_width": 448,
23
- "average_image1_width": 448.0,
24
- "max_image1_width": 448,
25
- "min_image1_height": 448,
26
- "average_image1_height": 448.0,
27
- "max_image1_height": 448,
28
- "min_image2_width": 448,
29
- "average_image2_width": 448.0,
30
- "max_image2_width": 448,
31
- "min_image2_height": 448,
32
- "average_image2_height": 448.0,
33
- "max_image2_height": 448,
34
- "min_score": 0.0,
35
- "avg_score": 2.3639075540602206,
36
- "max_score": 5.0
37
- },
38
- "de": {
39
- "num_samples": 1500,
40
- "min_image1_width": 448,
41
- "average_image1_width": 448.0,
42
- "max_image1_width": 448,
43
- "min_image1_height": 448,
44
- "average_image1_height": 448.0,
45
- "max_image1_height": 448,
46
- "min_image2_width": 448,
47
- "average_image2_width": 448.0,
48
- "max_image2_width": 448,
49
- "min_image2_height": 448,
50
- "average_image2_height": 448.0,
51
- "max_image2_height": 448,
52
- "min_score": 0.0,
53
- "avg_score": 2.3639075540602206,
54
- "max_score": 5.0
55
- },
56
- "es": {
57
- "num_samples": 1500,
58
- "min_image1_width": 448,
59
- "average_image1_width": 448.0,
60
- "max_image1_width": 448,
61
- "min_image1_height": 448,
62
- "average_image1_height": 448.0,
63
- "max_image1_height": 448,
64
- "min_image2_width": 448,
65
- "average_image2_width": 448.0,
66
- "max_image2_width": 448,
67
- "min_image2_height": 448,
68
- "average_image2_height": 448.0,
69
- "max_image2_height": 448,
70
- "min_score": 0.0,
71
- "avg_score": 2.3639075540602206,
72
- "max_score": 5.0
73
- },
74
- "fr": {
75
- "num_samples": 1500,
76
- "min_image1_width": 448,
77
- "average_image1_width": 448.0,
78
- "max_image1_width": 448,
79
- "min_image1_height": 448,
80
- "average_image1_height": 448.0,
81
- "max_image1_height": 448,
82
- "min_image2_width": 448,
83
- "average_image2_width": 448.0,
84
- "max_image2_width": 448,
85
- "min_image2_height": 448,
86
- "average_image2_height": 448.0,
87
- "max_image2_height": 448,
88
- "min_score": 0.0,
89
- "avg_score": 2.3639075540602206,
90
- "max_score": 5.0
91
- },
92
- "it": {
93
- "num_samples": 1500,
94
- "min_image1_width": 448,
95
- "average_image1_width": 448.0,
96
- "max_image1_width": 448,
97
- "min_image1_height": 448,
98
- "average_image1_height": 448.0,
99
- "max_image1_height": 448,
100
- "min_image2_width": 448,
101
- "average_image2_width": 448.0,
102
- "max_image2_width": 448,
103
- "min_image2_height": 448,
104
- "average_image2_height": 448.0,
105
- "max_image2_height": 448,
106
- "min_score": 0.0,
107
- "avg_score": 2.3639075540602206,
108
- "max_score": 5.0
109
- },
110
- "nl": {
111
- "num_samples": 1500,
112
- "min_image1_width": 448,
113
- "average_image1_width": 448.0,
114
- "max_image1_width": 448,
115
- "min_image1_height": 448,
116
- "average_image1_height": 448.0,
117
- "max_image1_height": 448,
118
- "min_image2_width": 448,
119
- "average_image2_width": 448.0,
120
- "max_image2_width": 448,
121
- "min_image2_height": 448,
122
- "average_image2_height": 448.0,
123
- "max_image2_height": 448,
124
- "min_score": 0.0,
125
- "avg_score": 2.3639075540602206,
126
- "max_score": 5.0
127
- },
128
- "pl": {
129
- "num_samples": 1500,
130
- "min_image1_width": 448,
131
- "average_image1_width": 448.0,
132
- "max_image1_width": 448,
133
- "min_image1_height": 448,
134
- "average_image1_height": 448.0,
135
- "max_image1_height": 448,
136
- "min_image2_width": 448,
137
- "average_image2_width": 448.0,
138
- "max_image2_width": 448,
139
- "min_image2_height": 448,
140
- "average_image2_height": 448.0,
141
- "max_image2_height": 448,
142
- "min_score": 0.0,
143
- "avg_score": 2.3639075540602206,
144
- "max_score": 5.0
145
- },
146
- "pt": {
147
- "num_samples": 1500,
148
- "min_image1_width": 448,
149
- "average_image1_width": 448.0,
150
- "max_image1_width": 448,
151
- "min_image1_height": 448,
152
- "average_image1_height": 448.0,
153
- "max_image1_height": 448,
154
- "min_image2_width": 448,
155
- "average_image2_width": 448.0,
156
- "max_image2_width": 448,
157
- "min_image2_height": 448,
158
- "average_image2_height": 448.0,
159
- "max_image2_height": 448,
160
- "min_score": 0.0,
161
- "avg_score": 2.3639075540602206,
162
- "max_score": 5.0
163
- },
164
- "ru": {
165
- "num_samples": 1500,
166
- "min_image1_width": 448,
167
- "average_image1_width": 448.0,
168
- "max_image1_width": 448,
169
- "min_image1_height": 448,
170
- "average_image1_height": 448.0,
171
- "max_image1_height": 448,
172
- "min_image2_width": 448,
173
- "average_image2_width": 448.0,
174
- "max_image2_width": 448,
175
- "min_image2_height": 448,
176
- "average_image2_height": 448.0,
177
- "max_image2_height": 448,
178
- "min_score": 0.0,
179
- "avg_score": 2.3639075540602206,
180
- "max_score": 5.0
181
- },
182
- "zh": {
183
- "num_samples": 1500,
184
- "min_image1_width": 448,
185
- "average_image1_width": 448.0,
186
- "max_image1_width": 448,
187
- "min_image1_height": 448,
188
- "average_image1_height": 448.0,
189
- "max_image1_height": 448,
190
- "min_image2_width": 448,
191
- "average_image2_width": 448.0,
192
- "max_image2_width": 448,
193
- "min_image2_height": 448,
194
- "average_image2_height": 448.0,
195
- "max_image2_height": 448,
196
- "min_score": 0.0,
197
- "avg_score": 2.3639075540602206,
198
- "max_score": 5.0
199
- }
200
- }
201
- },
202
- "test": {
203
- "num_samples": 13790,
204
- "min_image1_width": 448,
205
- "average_image1_width": 448.0,
206
- "max_image1_width": 448,
207
- "min_image1_height": 448,
208
- "average_image1_height": 448.0,
209
- "max_image1_height": 448,
210
- "min_image2_width": 448,
211
- "average_image2_width": 448.0,
212
- "max_image2_width": 448,
213
- "min_image2_height": 448,
214
- "average_image2_height": 448.0,
215
- "max_image2_height": 448,
216
- "min_score": 0.0,
217
- "avg_score": 2.6079166059890806,
218
- "max_score": 5.0,
219
- "hf_subset_descriptive_stats": {
220
- "en": {
221
- "num_samples": 1379,
222
- "min_image1_width": 448,
223
- "average_image1_width": 448.0,
224
- "max_image1_width": 448,
225
- "min_image1_height": 448,
226
- "average_image1_height": 448.0,
227
- "max_image1_height": 448,
228
- "min_image2_width": 448,
229
- "average_image2_width": 448.0,
230
- "max_image2_width": 448,
231
- "min_image2_height": 448,
232
- "average_image2_height": 448.0,
233
- "max_image2_height": 448,
234
- "min_score": 0.0,
235
- "avg_score": 2.6079166059890806,
236
- "max_score": 5.0
237
- },
238
- "de": {
239
- "num_samples": 1379,
240
- "min_image1_width": 448,
241
- "average_image1_width": 448.0,
242
- "max_image1_width": 448,
243
- "min_image1_height": 448,
244
- "average_image1_height": 448.0,
245
- "max_image1_height": 448,
246
- "min_image2_width": 448,
247
- "average_image2_width": 448.0,
248
- "max_image2_width": 448,
249
- "min_image2_height": 448,
250
- "average_image2_height": 448.0,
251
- "max_image2_height": 448,
252
- "min_score": 0.0,
253
- "avg_score": 2.6079166059890806,
254
- "max_score": 5.0
255
- },
256
- "es": {
257
- "num_samples": 1379,
258
- "min_image1_width": 448,
259
- "average_image1_width": 448.0,
260
- "max_image1_width": 448,
261
- "min_image1_height": 448,
262
- "average_image1_height": 448.0,
263
- "max_image1_height": 448,
264
- "min_image2_width": 448,
265
- "average_image2_width": 448.0,
266
- "max_image2_width": 448,
267
- "min_image2_height": 448,
268
- "average_image2_height": 448.0,
269
- "max_image2_height": 448,
270
- "min_score": 0.0,
271
- "avg_score": 2.6079166059890806,
272
- "max_score": 5.0
273
- },
274
- "fr": {
275
- "num_samples": 1379,
276
- "min_image1_width": 448,
277
- "average_image1_width": 448.0,
278
- "max_image1_width": 448,
279
- "min_image1_height": 448,
280
- "average_image1_height": 448.0,
281
- "max_image1_height": 448,
282
- "min_image2_width": 448,
283
- "average_image2_width": 448.0,
284
- "max_image2_width": 448,
285
- "min_image2_height": 448,
286
- "average_image2_height": 448.0,
287
- "max_image2_height": 448,
288
- "min_score": 0.0,
289
- "avg_score": 2.6079166059890806,
290
- "max_score": 5.0
291
- },
292
- "it": {
293
- "num_samples": 1379,
294
- "min_image1_width": 448,
295
- "average_image1_width": 448.0,
296
- "max_image1_width": 448,
297
- "min_image1_height": 448,
298
- "average_image1_height": 448.0,
299
- "max_image1_height": 448,
300
- "min_image2_width": 448,
301
- "average_image2_width": 448.0,
302
- "max_image2_width": 448,
303
- "min_image2_height": 448,
304
- "average_image2_height": 448.0,
305
- "max_image2_height": 448,
306
- "min_score": 0.0,
307
- "avg_score": 2.6079166059890806,
308
- "max_score": 5.0
309
- },
310
- "nl": {
311
- "num_samples": 1379,
312
- "min_image1_width": 448,
313
- "average_image1_width": 448.0,
314
- "max_image1_width": 448,
315
- "min_image1_height": 448,
316
- "average_image1_height": 448.0,
317
- "max_image1_height": 448,
318
- "min_image2_width": 448,
319
- "average_image2_width": 448.0,
320
- "max_image2_width": 448,
321
- "min_image2_height": 448,
322
- "average_image2_height": 448.0,
323
- "max_image2_height": 448,
324
- "min_score": 0.0,
325
- "avg_score": 2.6079166059890806,
326
- "max_score": 5.0
327
- },
328
- "pl": {
329
- "num_samples": 1379,
330
- "min_image1_width": 448,
331
- "average_image1_width": 448.0,
332
- "max_image1_width": 448,
333
- "min_image1_height": 448,
334
- "average_image1_height": 448.0,
335
- "max_image1_height": 448,
336
- "min_image2_width": 448,
337
- "average_image2_width": 448.0,
338
- "max_image2_width": 448,
339
- "min_image2_height": 448,
340
- "average_image2_height": 448.0,
341
- "max_image2_height": 448,
342
- "min_score": 0.0,
343
- "avg_score": 2.6079166059890806,
344
- "max_score": 5.0
345
- },
346
- "pt": {
347
- "num_samples": 1379,
348
- "min_image1_width": 448,
349
- "average_image1_width": 448.0,
350
- "max_image1_width": 448,
351
- "min_image1_height": 448,
352
- "average_image1_height": 448.0,
353
- "max_image1_height": 448,
354
- "min_image2_width": 448,
355
- "average_image2_width": 448.0,
356
- "max_image2_width": 448,
357
- "min_image2_height": 448,
358
- "average_image2_height": 448.0,
359
- "max_image2_height": 448,
360
- "min_score": 0.0,
361
- "avg_score": 2.6079166059890806,
362
- "max_score": 5.0
363
- },
364
- "ru": {
365
- "num_samples": 1379,
366
- "min_image1_width": 448,
367
- "average_image1_width": 448.0,
368
- "max_image1_width": 448,
369
- "min_image1_height": 448,
370
- "average_image1_height": 448.0,
371
- "max_image1_height": 448,
372
- "min_image2_width": 448,
373
- "average_image2_width": 448.0,
374
- "max_image2_width": 448,
375
- "min_image2_height": 448,
376
- "average_image2_height": 448.0,
377
- "max_image2_height": 448,
378
- "min_score": 0.0,
379
- "avg_score": 2.6079166059890806,
380
- "max_score": 5.0
381
- },
382
- "zh": {
383
- "num_samples": 1379,
384
- "min_image1_width": 448,
385
- "average_image1_width": 448.0,
386
- "max_image1_width": 448,
387
- "min_image1_height": 448,
388
- "average_image1_height": 448.0,
389
- "max_image1_height": 448,
390
- "min_image2_width": 448,
391
- "average_image2_width": 448.0,
392
- "max_image2_width": 448,
393
- "min_image2_height": 448,
394
- "average_image2_height": 448.0,
395
- "max_image2_height": 448,
396
- "min_score": 0.0,
397
- "avg_score": 2.6079166059890806,
398
- "max_score": 5.0
399
- }
400
- }
401
- }
402
- }
mteb/descriptive_stats/Reranking/InstructIR.json DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 25978,
4
- "number_of_characters": 10114013,
5
- "num_documents": 16072,
6
- "min_document_length": 36,
7
- "average_document_length": 610.031981085117,
8
- "max_document_length": 1700,
9
- "unique_documents": 16072,
10
- "num_queries": 9906,
11
- "min_query_length": 24,
12
- "average_query_length": 31.25166565717747,
13
- "max_query_length": 40,
14
- "unique_queries": 9906,
15
- "none_queries": 0,
16
- "num_relevant_docs": 9906,
17
- "min_relevant_docs_per_query": 1,
18
- "average_relevant_docs_per_query": 1.0,
19
- "max_relevant_docs_per_query": 1,
20
- "unique_relevant_docs": 9906,
21
- "num_instructions": 9906,
22
- "min_instruction_length": 146,
23
- "average_instruction_length": 3063330,
24
- "max_instruction_length": 616,
25
- "unique_instructions": 9906,
26
- "num_top_ranked": null,
27
- "min_top_ranked_per_query": null,
28
- "average_top_ranked_per_query": null,
29
- "max_top_ranked_per_query": null
30
- }
31
- }
{mteb-2.1.0.dist-info → mteb-2.1.2.dist-info}/WHEEL RENAMED
File without changes