mteb 2.0.4__py3-none-any.whl → 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. mteb/__init__.py +10 -1
  2. mteb/benchmarks/benchmarks/__init__.py +2 -0
  3. mteb/benchmarks/benchmarks/benchmarks.py +75 -0
  4. mteb/descriptive_stats/BitextMining/BUCC.json +70 -40
  5. mteb/descriptive_stats/Classification/DKHateClassification.json +40 -24
  6. mteb/descriptive_stats/Classification/DutchColaClassification.json +54 -0
  7. mteb/descriptive_stats/Classification/DutchGovernmentBiasClassification.json +54 -0
  8. mteb/descriptive_stats/Classification/DutchNewsArticlesClassification.json +90 -0
  9. mteb/descriptive_stats/Classification/DutchSarcasticHeadlinesClassification.json +54 -0
  10. mteb/descriptive_stats/Classification/FinancialPhrasebankClassification.json +23 -15
  11. mteb/descriptive_stats/Classification/IconclassClassification.json +96 -0
  12. mteb/descriptive_stats/Classification/ImdbClassification.json +40 -24
  13. mteb/descriptive_stats/Classification/KorHateClassification.json +23 -15
  14. mteb/descriptive_stats/Classification/OpenTenderClassification.json +222 -0
  15. mteb/descriptive_stats/Classification/VaccinChatNLClassification.json +1068 -0
  16. mteb/descriptive_stats/Clustering/ArxivClusteringP2P.json +555 -550
  17. mteb/descriptive_stats/Clustering/ArxivClusteringP2P.v2.json +546 -541
  18. mteb/descriptive_stats/Clustering/ArxivClusteringS2S.json +555 -550
  19. mteb/descriptive_stats/Clustering/DutchNewsArticlesClusteringP2P.json +45 -0
  20. mteb/descriptive_stats/Clustering/DutchNewsArticlesClusteringS2S.json +45 -0
  21. mteb/descriptive_stats/Clustering/IconclassClusteringS2S.json +48 -0
  22. mteb/descriptive_stats/Clustering/MLSUMClusteringP2P.json +2466 -2416
  23. mteb/descriptive_stats/Clustering/OpenTenderClusteringP2P.json +111 -0
  24. mteb/descriptive_stats/Clustering/OpenTenderClusteringS2S.json +111 -0
  25. mteb/descriptive_stats/Clustering/RedditClusteringP2P.json +1365 -1360
  26. mteb/descriptive_stats/Clustering/SNLClustering.json +378 -373
  27. mteb/descriptive_stats/Clustering/SwednClustering.json +28 -23
  28. mteb/descriptive_stats/Clustering/VABBClusteringP2P.json +60 -0
  29. mteb/descriptive_stats/Clustering/VABBClusteringS2S.json +60 -0
  30. mteb/descriptive_stats/Clustering/VGClustering.json +54 -49
  31. mteb/descriptive_stats/Image/Any2AnyMultilingualRetrieval/WITT2IRetrieval.json +324 -204
  32. mteb/descriptive_stats/Image/Any2AnyRetrieval/MemotionI2TRetrieval.json +28 -18
  33. mteb/descriptive_stats/Image/DocumentUnderstanding/JinaVDRAirbnbSyntheticRetrieval.json +334 -0
  34. mteb/descriptive_stats/Image/DocumentUnderstanding/JinaVDRGitHubReadmeRetrieval.json +544 -0
  35. mteb/descriptive_stats/Image/DocumentUnderstanding/JinaVDRTweetStockSyntheticsRetrieval.json +334 -0
  36. mteb/descriptive_stats/Image/DocumentUnderstanding/JinaVDRWikimediaCommonsDocumentsRetrieval.json +634 -0
  37. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore2ESGReportsRetrieval.json +154 -0
  38. mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore2EconomicsReportsRetrieval.json +154 -0
  39. mteb/descriptive_stats/Image/ImageClassification/Imagenet1k.json +6039 -3007
  40. mteb/descriptive_stats/Image/ZeroShotClassification/Imagenet1kZeroShot.json +3024 -3010
  41. mteb/descriptive_stats/Image/ZeroShotClassification/PatchCamelyonZeroShot.json +30 -16
  42. mteb/descriptive_stats/MultilabelClassification/CovidDisinformationNLMultiLabelClassification.json +84 -0
  43. mteb/descriptive_stats/MultilabelClassification/VABBMultiLabelClassification.json +156 -0
  44. mteb/descriptive_stats/PairClassification/SICKNLPairClassification.json +35 -0
  45. mteb/descriptive_stats/PairClassification/XLWICNLPairClassification.json +35 -0
  46. mteb/descriptive_stats/Reranking/MIRACLReranking.json +555 -479
  47. mteb/descriptive_stats/Reranking/MindSmallReranking.json +29 -25
  48. mteb/descriptive_stats/Retrieval/AlloprofRetrieval.json +25 -26
  49. mteb/descriptive_stats/Retrieval/Code1Retrieval.json +30 -0
  50. mteb/descriptive_stats/Retrieval/DanFEVER.json +25 -26
  51. mteb/descriptive_stats/Retrieval/DutchNewsArticlesRetrieval.json +30 -0
  52. mteb/descriptive_stats/Retrieval/EnglishFinance1Retrieval.json +30 -0
  53. mteb/descriptive_stats/Retrieval/EnglishFinance2Retrieval.json +30 -0
  54. mteb/descriptive_stats/Retrieval/EnglishFinance3Retrieval.json +30 -0
  55. mteb/descriptive_stats/Retrieval/EnglishFinance4Retrieval.json +30 -0
  56. mteb/descriptive_stats/Retrieval/EnglishHealthcare1Retrieval.json +30 -0
  57. mteb/descriptive_stats/Retrieval/French1Retrieval.json +30 -0
  58. mteb/descriptive_stats/Retrieval/FrenchLegal1Retrieval.json +30 -0
  59. mteb/descriptive_stats/Retrieval/German1Retrieval.json +30 -0
  60. mteb/descriptive_stats/Retrieval/GermanHealthcare1Retrieval.json +30 -0
  61. mteb/descriptive_stats/Retrieval/GermanLegal1Retrieval.json +30 -0
  62. mteb/descriptive_stats/Retrieval/JapaneseCode1Retrieval.json +30 -0
  63. mteb/descriptive_stats/Retrieval/JapaneseLegal1Retrieval.json +30 -0
  64. mteb/descriptive_stats/Retrieval/LegalQANLRetrieval.json +30 -0
  65. mteb/descriptive_stats/Retrieval/MIRACLRetrieval.json +475 -494
  66. mteb/descriptive_stats/Retrieval/MSMARCO-Fa.json +25 -26
  67. mteb/descriptive_stats/Retrieval/MSMARCO.json +25 -84
  68. mteb/descriptive_stats/Retrieval/OpenTenderRetrieval.json +30 -0
  69. mteb/descriptive_stats/Retrieval/Touche2020.json +25 -26
  70. mteb/descriptive_stats/Retrieval/VABBRetrieval.json +30 -0
  71. mteb/descriptive_stats/Retrieval/bBSARDNLRetrieval.json +30 -0
  72. mteb/descriptive_stats/STS/SICK-NL-STS.json +28 -0
  73. mteb/descriptive_stats/Summarization/SummEval.json +27 -50
  74. mteb/descriptive_stats/Summarization/SummEvalFr.json +27 -50
  75. mteb/models/model_implementations/kalm_models.py +29 -0
  76. mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py +1 -1
  77. mteb/tasks/classification/eng/financial_phrasebank_classification.py +0 -3
  78. mteb/tasks/classification/kor/kor_hate_classification.py +0 -12
  79. mteb/tasks/classification/nld/__init__.py +16 -0
  80. mteb/tasks/classification/nld/dutch_cola_classification.py +38 -0
  81. mteb/tasks/classification/nld/dutch_government_bias_classification.py +37 -0
  82. mteb/tasks/classification/nld/dutch_news_articles_classification.py +30 -0
  83. mteb/tasks/classification/nld/dutch_sarcastic_headlines_classification.py +36 -0
  84. mteb/tasks/classification/nld/iconclass_classification.py +41 -0
  85. mteb/tasks/classification/nld/open_tender_classification.py +38 -0
  86. mteb/tasks/classification/nld/vaccin_chat_nl_classification.py +46 -0
  87. mteb/tasks/clustering/__init__.py +1 -0
  88. mteb/tasks/clustering/nld/__init__.py +17 -0
  89. mteb/tasks/clustering/nld/dutch_news_articles_clustering_p2p.py +37 -0
  90. mteb/tasks/clustering/nld/dutch_news_articles_clustering_s2s.py +37 -0
  91. mteb/tasks/clustering/nld/iconclass_clustering_s2s.py +47 -0
  92. mteb/tasks/clustering/nld/open_tender_clustering_p2p.py +51 -0
  93. mteb/tasks/clustering/nld/open_tender_clustering_s2s.py +41 -0
  94. mteb/tasks/clustering/nld/vabb_clustering_p2p.py +51 -0
  95. mteb/tasks/clustering/nld/vabb_clustering_s2s.py +51 -0
  96. mteb/tasks/clustering/swe/swedn_clustering.py +2 -2
  97. mteb/tasks/multilabel_classification/__init__.py +1 -0
  98. mteb/tasks/multilabel_classification/nld/__init__.py +9 -0
  99. mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py +88 -0
  100. mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py +44 -0
  101. mteb/tasks/pair_classification/__init__.py +1 -0
  102. mteb/tasks/pair_classification/nld/__init__.py +7 -0
  103. mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py +36 -0
  104. mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py +41 -0
  105. mteb/tasks/retrieval/multilingual/vdr_multilingual_retrieval.py +1 -1
  106. mteb/tasks/retrieval/nld/__init__.py +10 -0
  107. mteb/tasks/retrieval/nld/bbsard_nl_retrieval.py +41 -0
  108. mteb/tasks/retrieval/nld/dutch_news_articles_retrieval.py +30 -0
  109. mteb/tasks/retrieval/nld/legal_qa_nl_retrieval.py +39 -0
  110. mteb/tasks/retrieval/nld/open_tender_retrieval.py +38 -0
  111. mteb/tasks/retrieval/nld/vabb_retrieval.py +41 -0
  112. mteb/tasks/sts/__init__.py +1 -0
  113. mteb/tasks/sts/nld/__init__.py +5 -0
  114. mteb/tasks/sts/nld/sick_nl_sts.py +41 -0
  115. {mteb-2.0.4.dist-info → mteb-2.1.0.dist-info}/METADATA +2 -204
  116. {mteb-2.0.4.dist-info → mteb-2.1.0.dist-info}/RECORD +120 -49
  117. {mteb-2.0.4.dist-info → mteb-2.1.0.dist-info}/WHEEL +0 -0
  118. {mteb-2.0.4.dist-info → mteb-2.1.0.dist-info}/entry_points.txt +0 -0
  119. {mteb-2.0.4.dist-info → mteb-2.1.0.dist-info}/licenses/LICENSE +0 -0
  120. {mteb-2.0.4.dist-info → mteb-2.1.0.dist-info}/top_level.txt +0 -0
mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore2ESGReportsRetrieval.json ADDED
@@ -0,0 +1,154 @@
1
+ {
2
+ "test": {
3
+ "num_samples": 6380,
4
+ "number_of_characters": 23416,
5
+ "documents_text_statistics": null,
6
+ "documents_image_statistics": {
7
+ "min_image_width": 1654,
8
+ "average_image_width": 2866.7932379713916,
9
+ "max_image_width": 5334,
10
+ "min_image_height": 1334,
11
+ "average_image_height": 2160.2958387516255,
12
+ "max_image_height": 14167,
13
+ "unique_images": 1538
14
+ },
15
+ "queries_text_statistics": {
16
+ "total_text_length": 23416,
17
+ "min_text_length": 42,
18
+ "average_text_length": 102.70175438596492,
19
+ "max_text_length": 177,
20
+ "unique_texts": 228
21
+ },
22
+ "queries_image_statistics": null,
23
+ "relevant_docs_statistics": {
24
+ "num_relevant_docs": 3552,
25
+ "min_relevant_docs_per_query": 1,
26
+ "average_relevant_docs_per_query": 3.8947368421052633,
27
+ "max_relevant_docs_per_query": 28,
28
+ "unique_relevant_docs": 796
29
+ },
30
+ "top_ranked_statistics": null,
31
+ "hf_subset_descriptive_stats": {
32
+ "french": {
33
+ "num_samples": 1595,
34
+ "number_of_characters": 6255,
35
+ "documents_text_statistics": null,
36
+ "documents_image_statistics": {
37
+ "min_image_width": 1654,
38
+ "average_image_width": 2866.7932379713916,
39
+ "max_image_width": 5334,
40
+ "min_image_height": 1334,
41
+ "average_image_height": 2160.2958387516255,
42
+ "max_image_height": 14167,
43
+ "unique_images": 1538
44
+ },
45
+ "queries_text_statistics": {
46
+ "total_text_length": 6255,
47
+ "min_text_length": 61,
48
+ "average_text_length": 109.73684210526316,
49
+ "max_text_length": 160,
50
+ "unique_texts": 57
51
+ },
52
+ "queries_image_statistics": null,
53
+ "relevant_docs_statistics": {
54
+ "num_relevant_docs": 888,
55
+ "min_relevant_docs_per_query": 1,
56
+ "average_relevant_docs_per_query": 3.8947368421052633,
57
+ "max_relevant_docs_per_query": 28,
58
+ "unique_relevant_docs": 199
59
+ },
60
+ "top_ranked_statistics": null
61
+ },
62
+ "spanish": {
63
+ "num_samples": 1595,
64
+ "number_of_characters": 5915,
65
+ "documents_text_statistics": null,
66
+ "documents_image_statistics": {
67
+ "min_image_width": 1654,
68
+ "average_image_width": 2866.7932379713916,
69
+ "max_image_width": 5334,
70
+ "min_image_height": 1334,
71
+ "average_image_height": 2160.2958387516255,
72
+ "max_image_height": 14167,
73
+ "unique_images": 1538
74
+ },
75
+ "queries_text_statistics": {
76
+ "total_text_length": 5915,
77
+ "min_text_length": 52,
78
+ "average_text_length": 103.7719298245614,
79
+ "max_text_length": 149,
80
+ "unique_texts": 57
81
+ },
82
+ "queries_image_statistics": null,
83
+ "relevant_docs_statistics": {
84
+ "num_relevant_docs": 888,
85
+ "min_relevant_docs_per_query": 1,
86
+ "average_relevant_docs_per_query": 3.8947368421052633,
87
+ "max_relevant_docs_per_query": 28,
88
+ "unique_relevant_docs": 199
89
+ },
90
+ "top_ranked_statistics": null
91
+ },
92
+ "english": {
93
+ "num_samples": 1595,
94
+ "number_of_characters": 5051,
95
+ "documents_text_statistics": null,
96
+ "documents_image_statistics": {
97
+ "min_image_width": 1654,
98
+ "average_image_width": 2866.7932379713916,
99
+ "max_image_width": 5334,
100
+ "min_image_height": 1334,
101
+ "average_image_height": 2160.2958387516255,
102
+ "max_image_height": 14167,
103
+ "unique_images": 1538
104
+ },
105
+ "queries_text_statistics": {
106
+ "total_text_length": 5051,
107
+ "min_text_length": 47,
108
+ "average_text_length": 88.6140350877193,
109
+ "max_text_length": 131,
110
+ "unique_texts": 57
111
+ },
112
+ "queries_image_statistics": null,
113
+ "relevant_docs_statistics": {
114
+ "num_relevant_docs": 888,
115
+ "min_relevant_docs_per_query": 1,
116
+ "average_relevant_docs_per_query": 3.8947368421052633,
117
+ "max_relevant_docs_per_query": 28,
118
+ "unique_relevant_docs": 199
119
+ },
120
+ "top_ranked_statistics": null
121
+ },
122
+ "german": {
123
+ "num_samples": 1595,
124
+ "number_of_characters": 6195,
125
+ "documents_text_statistics": null,
126
+ "documents_image_statistics": {
127
+ "min_image_width": 1654,
128
+ "average_image_width": 2866.7932379713916,
129
+ "max_image_width": 5334,
130
+ "min_image_height": 1334,
131
+ "average_image_height": 2160.2958387516255,
132
+ "max_image_height": 14167,
133
+ "unique_images": 1538
134
+ },
135
+ "queries_text_statistics": {
136
+ "total_text_length": 6195,
137
+ "min_text_length": 42,
138
+ "average_text_length": 108.6842105263158,
139
+ "max_text_length": 177,
140
+ "unique_texts": 57
141
+ },
142
+ "queries_image_statistics": null,
143
+ "relevant_docs_statistics": {
144
+ "num_relevant_docs": 888,
145
+ "min_relevant_docs_per_query": 1,
146
+ "average_relevant_docs_per_query": 3.8947368421052633,
147
+ "max_relevant_docs_per_query": 28,
148
+ "unique_relevant_docs": 199
149
+ },
150
+ "top_ranked_statistics": null
151
+ }
152
+ }
153
+ }
154
+ }
mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore2EconomicsReportsRetrieval.json ADDED
@@ -0,0 +1,154 @@
1
+ {
2
+ "test": {
3
+ "num_samples": 2040,
4
+ "number_of_characters": 22843,
5
+ "documents_text_statistics": null,
6
+ "documents_image_statistics": {
7
+ "min_image_width": 700,
8
+ "average_image_width": 809.070796460177,
9
+ "max_image_width": 1050,
10
+ "min_image_height": 800,
11
+ "average_image_height": 1063.495575221239,
12
+ "max_image_height": 1100,
13
+ "unique_images": 426
14
+ },
15
+ "queries_text_statistics": {
16
+ "total_text_length": 22843,
17
+ "min_text_length": 57,
18
+ "average_text_length": 98.46120689655173,
19
+ "max_text_length": 165,
20
+ "unique_texts": 232
21
+ },
22
+ "queries_image_statistics": null,
23
+ "relevant_docs_statistics": {
24
+ "num_relevant_docs": 14512,
25
+ "min_relevant_docs_per_query": 1,
26
+ "average_relevant_docs_per_query": 15.637931034482758,
27
+ "max_relevant_docs_per_query": 113,
28
+ "unique_relevant_docs": 1020
29
+ },
30
+ "top_ranked_statistics": null,
31
+ "hf_subset_descriptive_stats": {
32
+ "french": {
33
+ "num_samples": 510,
34
+ "number_of_characters": 6080,
35
+ "documents_text_statistics": null,
36
+ "documents_image_statistics": {
37
+ "min_image_width": 700,
38
+ "average_image_width": 809.070796460177,
39
+ "max_image_width": 1050,
40
+ "min_image_height": 800,
41
+ "average_image_height": 1063.495575221239,
42
+ "max_image_height": 1100,
43
+ "unique_images": 426
44
+ },
45
+ "queries_text_statistics": {
46
+ "total_text_length": 6080,
47
+ "min_text_length": 63,
48
+ "average_text_length": 104.82758620689656,
49
+ "max_text_length": 165,
50
+ "unique_texts": 58
51
+ },
52
+ "queries_image_statistics": null,
53
+ "relevant_docs_statistics": {
54
+ "num_relevant_docs": 3628,
55
+ "min_relevant_docs_per_query": 1,
56
+ "average_relevant_docs_per_query": 15.637931034482758,
57
+ "max_relevant_docs_per_query": 113,
58
+ "unique_relevant_docs": 255
59
+ },
60
+ "top_ranked_statistics": null
61
+ },
62
+ "spanish": {
63
+ "num_samples": 510,
64
+ "number_of_characters": 6049,
65
+ "documents_text_statistics": null,
66
+ "documents_image_statistics": {
67
+ "min_image_width": 700,
68
+ "average_image_width": 809.070796460177,
69
+ "max_image_width": 1050,
70
+ "min_image_height": 800,
71
+ "average_image_height": 1063.495575221239,
72
+ "max_image_height": 1100,
73
+ "unique_images": 426
74
+ },
75
+ "queries_text_statistics": {
76
+ "total_text_length": 6049,
77
+ "min_text_length": 65,
78
+ "average_text_length": 104.29310344827586,
79
+ "max_text_length": 163,
80
+ "unique_texts": 58
81
+ },
82
+ "queries_image_statistics": null,
83
+ "relevant_docs_statistics": {
84
+ "num_relevant_docs": 3628,
85
+ "min_relevant_docs_per_query": 1,
86
+ "average_relevant_docs_per_query": 15.637931034482758,
87
+ "max_relevant_docs_per_query": 113,
88
+ "unique_relevant_docs": 255
89
+ },
90
+ "top_ranked_statistics": null
91
+ },
92
+ "english": {
93
+ "num_samples": 510,
94
+ "number_of_characters": 4799,
95
+ "documents_text_statistics": null,
96
+ "documents_image_statistics": {
97
+ "min_image_width": 700,
98
+ "average_image_width": 809.070796460177,
99
+ "max_image_width": 1050,
100
+ "min_image_height": 800,
101
+ "average_image_height": 1063.495575221239,
102
+ "max_image_height": 1100,
103
+ "unique_images": 426
104
+ },
105
+ "queries_text_statistics": {
106
+ "total_text_length": 4799,
107
+ "min_text_length": 57,
108
+ "average_text_length": 82.74137931034483,
109
+ "max_text_length": 129,
110
+ "unique_texts": 58
111
+ },
112
+ "queries_image_statistics": null,
113
+ "relevant_docs_statistics": {
114
+ "num_relevant_docs": 3628,
115
+ "min_relevant_docs_per_query": 1,
116
+ "average_relevant_docs_per_query": 15.637931034482758,
117
+ "max_relevant_docs_per_query": 113,
118
+ "unique_relevant_docs": 255
119
+ },
120
+ "top_ranked_statistics": null
121
+ },
122
+ "german": {
123
+ "num_samples": 510,
124
+ "number_of_characters": 5915,
125
+ "documents_text_statistics": null,
126
+ "documents_image_statistics": {
127
+ "min_image_width": 700,
128
+ "average_image_width": 809.070796460177,
129
+ "max_image_width": 1050,
130
+ "min_image_height": 800,
131
+ "average_image_height": 1063.495575221239,
132
+ "max_image_height": 1100,
133
+ "unique_images": 426
134
+ },
135
+ "queries_text_statistics": {
136
+ "total_text_length": 5915,
137
+ "min_text_length": 59,
138
+ "average_text_length": 101.98275862068965,
139
+ "max_text_length": 146,
140
+ "unique_texts": 58
141
+ },
142
+ "queries_image_statistics": null,
143
+ "relevant_docs_statistics": {
144
+ "num_relevant_docs": 3628,
145
+ "min_relevant_docs_per_query": 1,
146
+ "average_relevant_docs_per_query": 15.637931034482758,
147
+ "max_relevant_docs_per_query": 113,
148
+ "unique_relevant_docs": 255
149
+ },
150
+ "top_ranked_statistics": null
151
+ }
152
+ }
153
+ }
154
+ }