mteb 2.1.0__py3-none-any.whl → 2.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. mteb/_create_dataloaders.py +2 -0
  2. mteb/abstasks/_stratification.py +1 -1
  3. mteb/abstasks/abstask.py +6 -1
  4. mteb/abstasks/dataset_card_template.md +1 -1
  5. mteb/abstasks/retrieval.py +2 -1
  6. mteb/abstasks/retrieval_dataset_loaders.py +1 -1
  7. mteb/abstasks/task_metadata.py +1 -1
  8. mteb/benchmarks/benchmarks/benchmarks.py +7 -11
  9. mteb/benchmarks/get_benchmark.py +1 -1
  10. mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/XFlickr30kCoT2IRetrieval.json +243 -153
  11. mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/XM3600T2IRetrieval.json +999 -629
  12. mteb/descriptive_stats/Image/Any2AnyRetrieval/OVENIT2TRetrieval.json +33 -17
  13. mteb/descriptive_stats/Image/DocumentUnderstanding/MIRACLVisionRetrieval.json +574 -0
  14. mteb/descriptive_stats/Retrieval/ClimateFEVERHardNegatives.v2.json +30 -0
  15. mteb/descriptive_stats/Retrieval/DBPediaHardNegatives.v2.json +30 -0
  16. mteb/descriptive_stats/Retrieval/FEVERHardNegatives.v2.json +30 -0
  17. mteb/descriptive_stats/Retrieval/HotpotQAHardNegatives.v2.json +30 -0
  18. mteb/descriptive_stats/Retrieval/QuoraRetrievalHardNegatives.v2.json +30 -0
  19. mteb/descriptive_stats/Retrieval/RiaNewsRetrievalHardNegatives.v2.json +30 -0
  20. mteb/descriptive_stats/Retrieval/VDRMultilingualRetrieval.json +184 -0
  21. mteb/languages/check_language_code.py +11 -3
  22. mteb/languages/language_scripts.py +4 -0
  23. mteb/leaderboard/text_segments.py +1 -1
  24. mteb/models/model_implementations/b1ade_models.py +1 -1
  25. mteb/models/model_implementations/bge_models.py +1 -3
  26. mteb/models/model_implementations/bmretriever_models.py +1 -1
  27. mteb/models/model_implementations/gme_v_models.py +2 -2
  28. mteb/models/model_implementations/ibm_granite_models.py +1 -1
  29. mteb/models/model_implementations/inf_models.py +3 -3
  30. mteb/models/model_implementations/jina_models.py +12 -2
  31. mteb/models/model_implementations/llm2vec_models.py +1 -1
  32. mteb/models/model_implementations/misc_models.py +2 -2
  33. mteb/models/model_implementations/mxbai_models.py +1 -1
  34. mteb/models/model_implementations/salesforce_models.py +1 -1
  35. mteb/models/model_implementations/seed_1_6_embedding_models.py +1 -1
  36. mteb/models/model_implementations/voyage_v.py +9 -9
  37. mteb/results/task_result.py +6 -8
  38. mteb/tasks/classification/dan/angry_tweets_classification.py +2 -2
  39. mteb/tasks/classification/eng/legal_bench_classification.py +3 -3
  40. mteb/tasks/classification/mya/myanmar_news.py +2 -2
  41. mteb/tasks/classification/tha/wongnai_reviews_classification.py +1 -1
  42. mteb/tasks/classification/ukr/ukr_formality_classification.py +2 -2
  43. mteb/tasks/pair_classification/multilingual/indic_xnli_pair_classification.py +9 -8
  44. mteb/tasks/retrieval/code/code_rag.py +8 -8
  45. mteb/tasks/retrieval/dan/dan_fever_retrieval.py +1 -1
  46. mteb/tasks/retrieval/dan/tv2_nordretrieval.py +2 -2
  47. mteb/tasks/retrieval/dan/twitter_hjerne_retrieval.py +2 -2
  48. mteb/tasks/retrieval/eng/__init__.py +18 -4
  49. mteb/tasks/retrieval/eng/climate_fever_retrieval.py +68 -77
  50. mteb/tasks/retrieval/eng/dbpedia_retrieval.py +55 -50
  51. mteb/tasks/retrieval/eng/fever_retrieval.py +62 -67
  52. mteb/tasks/retrieval/eng/hateful_memes_i2t_retrieval.py +0 -4
  53. mteb/tasks/retrieval/eng/hateful_memes_t2i_retrieval.py +0 -4
  54. mteb/tasks/retrieval/eng/hotpot_qa_retrieval.py +57 -67
  55. mteb/tasks/retrieval/eng/legal_summarization_retrieval.py +1 -1
  56. mteb/tasks/retrieval/eng/memotion_i2t_retrieval.py +0 -3
  57. mteb/tasks/retrieval/eng/memotion_t2i_retrieval.py +0 -2
  58. mteb/tasks/retrieval/eng/oven_it2t_retrieval.py +1 -1
  59. mteb/tasks/retrieval/eng/quora_retrieval.py +51 -46
  60. mteb/tasks/retrieval/eng/sci_mmir_i2t_retrieval.py +0 -4
  61. mteb/tasks/retrieval/eng/sci_mmir_t2i_retrieval.py +0 -4
  62. mteb/tasks/retrieval/eng/vidore_bench_retrieval.py +0 -2
  63. mteb/tasks/retrieval/jpn/ja_gov_faqs_retrieval.py +1 -1
  64. mteb/tasks/retrieval/multilingual/belebele_retrieval.py +1 -1
  65. mteb/tasks/retrieval/multilingual/jina_vdr_bench_retrieval.py +0 -2
  66. mteb/tasks/retrieval/multilingual/miracl_retrieval.py +1 -1
  67. mteb/tasks/retrieval/multilingual/miracl_vision_retrieval.py +2 -9
  68. mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py +0 -2
  69. mteb/tasks/retrieval/multilingual/wit_t2i_retrieval.py +0 -2
  70. mteb/tasks/retrieval/multilingual/x_flickr30k_co_t2i_retrieval.py +6 -5
  71. mteb/tasks/retrieval/multilingual/xm3600_t2i_retrieval.py +3 -4
  72. mteb/tasks/retrieval/nob/norquad.py +2 -2
  73. mteb/tasks/retrieval/nob/snl_retrieval.py +2 -2
  74. mteb/tasks/retrieval/rus/__init__.py +11 -2
  75. mteb/tasks/retrieval/rus/ria_news_retrieval.py +48 -44
  76. mteb/tasks/retrieval/tur/tur_hist_quad.py +2 -2
  77. {mteb-2.1.0.dist-info → mteb-2.1.1.dist-info}/METADATA +5 -5
  78. {mteb-2.1.0.dist-info → mteb-2.1.1.dist-info}/RECORD +82 -87
  79. mteb/descriptive_stats/Classification/PersianTextTone.json +0 -56
  80. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchCount.json +0 -37
  81. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchDepth.json +0 -25
  82. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchDistance.json +0 -25
  83. mteb/descriptive_stats/Image/Any2TextMutipleChoice/CVBenchRelation.json +0 -25
  84. mteb/descriptive_stats/Image/VisualSTS/STS12VisualSTS.json +0 -20
  85. mteb/descriptive_stats/Image/VisualSTS/STS13VisualSTS.json +0 -20
  86. mteb/descriptive_stats/Image/VisualSTS/STS14VisualSTS.json +0 -20
  87. mteb/descriptive_stats/Image/VisualSTS/STS15VisualSTS.json +0 -20
  88. mteb/descriptive_stats/Image/VisualSTS/STS16VisualSTS.json +0 -20
  89. mteb/descriptive_stats/Image/VisualSTS/STS17MultilingualVisualSTS.json +0 -220
  90. mteb/descriptive_stats/Image/VisualSTS/STSBenchmarkMultilingualVisualSTS.json +0 -402
  91. mteb/descriptive_stats/Reranking/InstructIR.json +0 -31
  92. {mteb-2.1.0.dist-info → mteb-2.1.1.dist-info}/WHEEL +0 -0
  93. {mteb-2.1.0.dist-info → mteb-2.1.1.dist-info}/entry_points.txt +0 -0
  94. {mteb-2.1.0.dist-info → mteb-2.1.1.dist-info}/licenses/LICENSE +0 -0
  95. {mteb-2.1.0.dist-info → mteb-2.1.1.dist-info}/top_level.txt +0 -0
@@ -1,56 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 2928,
4
- "number_of_characters": 648514,
5
- "number_texts_intersect_with_train": 6,
6
- "min_text_length": 41,
7
- "average_text_length": 221.4870218579235,
8
- "max_text_length": 949,
9
- "unique_texts": 2925,
10
- "min_labels_per_text": 1,
11
- "average_label_per_text": 1.0,
12
- "max_labels_per_text": 1,
13
- "unique_labels": 4,
14
- "labels": {
15
- "3": {
16
- "count": 1680
17
- },
18
- "2": {
19
- "count": 374
20
- },
21
- "0": {
22
- "count": 459
23
- },
24
- "1": {
25
- "count": 415
26
- }
27
- }
28
- },
29
- "train": {
30
- "num_samples": 16587,
31
- "number_of_characters": 3664508,
32
- "number_texts_intersect_with_train": null,
33
- "min_text_length": 39,
34
- "average_text_length": 220.92650871164165,
35
- "max_text_length": 1079,
36
- "unique_texts": 16576,
37
- "min_labels_per_text": 1,
38
- "average_label_per_text": 1.0,
39
- "max_labels_per_text": 1,
40
- "unique_labels": 4,
41
- "labels": {
42
- "1": {
43
- "count": 2434
44
- },
45
- "3": {
46
- "count": 9712
47
- },
48
- "0": {
49
- "count": 2476
50
- },
51
- "2": {
52
- "count": 1965
53
- }
54
- }
55
- }
56
- }
@@ -1,37 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 788,
4
- "min_image_width": 200,
5
- "average_image_width": 757.6789340101523,
6
- "max_image_width": 2200,
7
- "min_image_height": 181,
8
- "average_image_height": 631.3147208121827,
9
- "max_image_height": 2200,
10
- "min_num_choices": 4,
11
- "average_num_choices": 4.550761421319797,
12
- "max_num_choices": 6,
13
- "min_question_length": 30,
14
- "average_question_length": 34.35406091370558,
15
- "max_question_length": 45,
16
- "answers": {
17
- "2": {
18
- "count": 169
19
- },
20
- "4": {
21
- "count": 63
22
- },
23
- "3": {
24
- "count": 167
25
- },
26
- "1": {
27
- "count": 184
28
- },
29
- "0": {
30
- "count": 182
31
- },
32
- "5": {
33
- "count": 23
34
- }
35
- }
36
- }
37
- }
@@ -1,25 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 600,
4
- "min_image_width": 561,
5
- "average_image_width": 1090.9616666666666,
6
- "max_image_width": 1600,
7
- "min_image_height": 427,
8
- "average_image_height": 715.985,
9
- "max_image_height": 900,
10
- "min_num_choices": 2,
11
- "average_num_choices": 2.0,
12
- "max_num_choices": 2,
13
- "min_question_length": 130,
14
- "average_question_length": 136.04333333333332,
15
- "max_question_length": 147,
16
- "answers": {
17
- "0": {
18
- "count": 300
19
- },
20
- "1": {
21
- "count": 300
22
- }
23
- }
24
- }
25
- }
@@ -1,25 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 600,
4
- "min_image_width": 561,
5
- "average_image_width": 1099.2883333333334,
6
- "max_image_width": 1600,
7
- "min_image_height": 427,
8
- "average_image_height": 720.9983333333333,
9
- "max_image_height": 900,
10
- "min_num_choices": 2,
11
- "average_num_choices": 2.0,
12
- "max_num_choices": 2,
13
- "min_question_length": 204,
14
- "average_question_length": 212.40333333333334,
15
- "max_question_length": 223,
16
- "answers": {
17
- "0": {
18
- "count": 303
19
- },
20
- "1": {
21
- "count": 297
22
- }
23
- }
24
- }
25
- }
@@ -1,25 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 650,
4
- "min_image_width": 189,
5
- "average_image_width": 546.3169230769231,
6
- "max_image_width": 2200,
7
- "min_image_height": 190,
8
- "average_image_height": 448.4492307692308,
9
- "max_image_height": 2200,
10
- "min_num_choices": 2,
11
- "average_num_choices": 2.0,
12
- "max_num_choices": 2,
13
- "min_question_length": 132,
14
- "average_question_length": 181.45846153846153,
15
- "max_question_length": 224,
16
- "answers": {
17
- "0": {
18
- "count": 327
19
- },
20
- "1": {
21
- "count": 323
22
- }
23
- }
24
- }
25
- }
@@ -1,20 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 3108,
4
- "min_image1_width": 448,
5
- "average_image1_width": 448.0,
6
- "max_image1_width": 448,
7
- "min_image1_height": 448,
8
- "average_image1_height": 448.0,
9
- "max_image1_height": 448,
10
- "min_image2_width": 448,
11
- "average_image2_width": 448.0,
12
- "max_image2_width": 448,
13
- "min_image2_height": 448,
14
- "average_image2_height": 448.0,
15
- "max_image2_height": 448,
16
- "min_score": 0.0,
17
- "avg_score": 3.5060643500643507,
18
- "max_score": 5.0
19
- }
20
- }
@@ -1,20 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 1500,
4
- "min_image1_width": 448,
5
- "average_image1_width": 448.0,
6
- "max_image1_width": 448,
7
- "min_image1_height": 448,
8
- "average_image1_height": 448.0,
9
- "max_image1_height": 448,
10
- "min_image2_width": 448,
11
- "average_image2_width": 448.0,
12
- "max_image2_width": 448,
13
- "min_image2_height": 448,
14
- "average_image2_height": 448.0,
15
- "max_image2_height": 448,
16
- "min_score": 0.0,
17
- "avg_score": 2.3361888888888864,
18
- "max_score": 5.0
19
- }
20
- }
@@ -1,20 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 3750,
4
- "min_image1_width": 448,
5
- "average_image1_width": 448.0,
6
- "max_image1_width": 448,
7
- "min_image1_height": 448,
8
- "average_image1_height": 448.0,
9
- "max_image1_height": 448,
10
- "min_image2_width": 448,
11
- "average_image2_width": 448.0,
12
- "max_image2_width": 448,
13
- "min_image2_height": 448,
14
- "average_image2_height": 448.0,
15
- "max_image2_height": 448,
16
- "min_score": 0.0,
17
- "avg_score": 2.8114334391534355,
18
- "max_score": 5.0
19
- }
20
- }
@@ -1,20 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 3000,
4
- "min_image1_width": 448,
5
- "average_image1_width": 448.0,
6
- "max_image1_width": 448,
7
- "min_image1_height": 448,
8
- "average_image1_height": 448.0,
9
- "max_image1_height": 448,
10
- "min_image2_width": 448,
11
- "average_image2_width": 448.0,
12
- "max_image2_width": 448,
13
- "min_image2_height": 448,
14
- "average_image2_height": 448.0,
15
- "max_image2_height": 448,
16
- "min_score": 0.0,
17
- "avg_score": 2.40591333333333,
18
- "max_score": 5.0
19
- }
20
- }
@@ -1,20 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 1186,
4
- "min_image1_width": 448,
5
- "average_image1_width": 448.0,
6
- "max_image1_width": 448,
7
- "min_image1_height": 448,
8
- "average_image1_height": 448.0,
9
- "max_image1_height": 448,
10
- "min_image2_width": 448,
11
- "average_image2_width": 448.0,
12
- "max_image2_width": 448,
13
- "min_image2_height": 448,
14
- "average_image2_height": 448.0,
15
- "max_image2_height": 448,
16
- "min_score": 0.0,
17
- "avg_score": 2.4131534569983137,
18
- "max_score": 5.0
19
- }
20
- }
@@ -1,220 +0,0 @@
1
- {
2
- "test": {
3
- "num_samples": 5346,
4
- "min_image1_width": 448,
5
- "average_image1_width": 448.0,
6
- "max_image1_width": 448,
7
- "min_image1_height": 448,
8
- "average_image1_height": 448.0,
9
- "max_image1_height": 448,
10
- "min_image2_width": 448,
11
- "average_image2_width": 448.0,
12
- "max_image2_width": 448,
13
- "min_image2_height": 448,
14
- "average_image2_height": 448.0,
15
- "max_image2_height": 448,
16
- "min_score": 0.0,
17
- "avg_score": 2.3554804214989464,
18
- "max_score": 5.0,
19
- "hf_subset_descriptive_stats": {
20
- "ko-ko": {
21
- "num_samples": 2846,
22
- "min_image1_width": 448,
23
- "average_image1_width": 448.0,
24
- "max_image1_width": 448,
25
- "min_image1_height": 448,
26
- "average_image1_height": 448.0,
27
- "max_image1_height": 448,
28
- "min_image2_width": 448,
29
- "average_image2_width": 448.0,
30
- "max_image2_width": 448,
31
- "min_image2_height": 448,
32
- "average_image2_height": 448.0,
33
- "max_image2_height": 448,
34
- "min_score": 0.0,
35
- "avg_score": 2.469359920356055,
36
- "max_score": 5.0
37
- },
38
- "ar-ar": {
39
- "num_samples": 250,
40
- "min_image1_width": 448,
41
- "average_image1_width": 448.0,
42
- "max_image1_width": 448,
43
- "min_image1_height": 448,
44
- "average_image1_height": 448.0,
45
- "max_image1_height": 448,
46
- "min_image2_width": 448,
47
- "average_image2_width": 448.0,
48
- "max_image2_width": 448,
49
- "min_image2_height": 448,
50
- "average_image2_height": 448.0,
51
- "max_image2_height": 448,
52
- "min_score": 0.0,
53
- "avg_score": 2.216800000000001,
54
- "max_score": 5.0
55
- },
56
- "en-ar": {
57
- "num_samples": 250,
58
- "min_image1_width": 448,
59
- "average_image1_width": 448.0,
60
- "max_image1_width": 448,
61
- "min_image1_height": 448,
62
- "average_image1_height": 448.0,
63
- "max_image1_height": 448,
64
- "min_image2_width": 448,
65
- "average_image2_width": 448.0,
66
- "max_image2_width": 448,
67
- "min_image2_height": 448,
68
- "average_image2_height": 448.0,
69
- "max_image2_height": 448,
70
- "min_score": 0.0,
71
- "avg_score": 2.1423999999999994,
72
- "max_score": 5.0
73
- },
74
- "en-de": {
75
- "num_samples": 250,
76
- "min_image1_width": 448,
77
- "average_image1_width": 448.0,
78
- "max_image1_width": 448,
79
- "min_image1_height": 448,
80
- "average_image1_height": 448.0,
81
- "max_image1_height": 448,
82
- "min_image2_width": 448,
83
- "average_image2_width": 448.0,
84
- "max_image2_width": 448,
85
- "min_image2_height": 448,
86
- "average_image2_height": 448.0,
87
- "max_image2_height": 448,
88
- "min_score": 0.0,
89
- "avg_score": 2.2776000000000014,
90
- "max_score": 5.0
91
- },
92
- "en-en": {
93
- "num_samples": 250,
94
- "min_image1_width": 448,
95
- "average_image1_width": 448.0,
96
- "max_image1_width": 448,
97
- "min_image1_height": 448,
98
- "average_image1_height": 448.0,
99
- "max_image1_height": 448,
100
- "min_image2_width": 448,
101
- "average_image2_width": 448.0,
102
- "max_image2_width": 448,
103
- "min_image2_height": 448,
104
- "average_image2_height": 448.0,
105
- "max_image2_height": 448,
106
- "min_score": 0.0,
107
- "avg_score": 2.2776000000000014,
108
- "max_score": 5.0
109
- },
110
- "en-tr": {
111
- "num_samples": 250,
112
- "min_image1_width": 448,
113
- "average_image1_width": 448.0,
114
- "max_image1_width": 448,
115
- "min_image1_height": 448,
116
- "average_image1_height": 448.0,
117
- "max_image1_height": 448,
118
- "min_image2_width": 448,
119
- "average_image2_width": 448.0,
120
- "max_image2_width": 448,
121
- "min_image2_height": 448,
122
- "average_image2_height": 448.0,
123
- "max_image2_height": 448,
124
- "min_score": 0.0,
125
- "avg_score": 2.1335999999999986,
126
- "max_score": 5.0
127
- },
128
- "es-en": {
129
- "num_samples": 250,
130
- "min_image1_width": 448,
131
- "average_image1_width": 448.0,
132
- "max_image1_width": 448,
133
- "min_image1_height": 448,
134
- "average_image1_height": 448.0,
135
- "max_image1_height": 448,
136
- "min_image2_width": 448,
137
- "average_image2_width": 448.0,
138
- "max_image2_width": 448,
139
- "min_image2_height": 448,
140
- "average_image2_height": 448.0,
141
- "max_image2_height": 448,
142
- "min_score": 0.0,
143
- "avg_score": 2.1464000000000003,
144
- "max_score": 5.0
145
- },
146
- "es-es": {
147
- "num_samples": 250,
148
- "min_image1_width": 448,
149
- "average_image1_width": 448.0,
150
- "max_image1_width": 448,
151
- "min_image1_height": 448,
152
- "average_image1_height": 448.0,
153
- "max_image1_height": 448,
154
- "min_image2_width": 448,
155
- "average_image2_width": 448.0,
156
- "max_image2_width": 448,
157
- "min_image2_height": 448,
158
- "average_image2_height": 448.0,
159
- "max_image2_height": 448,
160
- "min_score": 0.0,
161
- "avg_score": 2.2312000000000007,
162
- "max_score": 5.0
163
- },
164
- "fr-en": {
165
- "num_samples": 250,
166
- "min_image1_width": 448,
167
- "average_image1_width": 448.0,
168
- "max_image1_width": 448,
169
- "min_image1_height": 448,
170
- "average_image1_height": 448.0,
171
- "max_image1_height": 448,
172
- "min_image2_width": 448,
173
- "average_image2_width": 448.0,
174
- "max_image2_width": 448,
175
- "min_image2_height": 448,
176
- "average_image2_height": 448.0,
177
- "max_image2_height": 448,
178
- "min_score": 0.0,
179
- "avg_score": 2.2776000000000014,
180
- "max_score": 5.0
181
- },
182
- "it-en": {
183
- "num_samples": 250,
184
- "min_image1_width": 448,
185
- "average_image1_width": 448.0,
186
- "max_image1_width": 448,
187
- "min_image1_height": 448,
188
- "average_image1_height": 448.0,
189
- "max_image1_height": 448,
190
- "min_image2_width": 448,
191
- "average_image2_width": 448.0,
192
- "max_image2_width": 448,
193
- "min_image2_height": 448,
194
- "average_image2_height": 448.0,
195
- "max_image2_height": 448,
196
- "min_score": 0.0,
197
- "avg_score": 2.2776000000000014,
198
- "max_score": 5.0
199
- },
200
- "nl-en": {
201
- "num_samples": 250,
202
- "min_image1_width": 448,
203
- "average_image1_width": 448.0,
204
- "max_image1_width": 448,
205
- "min_image1_height": 448,
206
- "average_image1_height": 448.0,
207
- "max_image1_height": 448,
208
- "min_image2_width": 448,
209
- "average_image2_width": 448.0,
210
- "max_image2_width": 448,
211
- "min_image2_height": 448,
212
- "average_image2_height": 448.0,
213
- "max_image2_height": 448,
214
- "min_score": 0.0,
215
- "avg_score": 2.2776000000000014,
216
- "max_score": 5.0
217
- }
218
- }
219
- }
220
- }