EuroEval 16.4.0__py3-none-any.whl → 16.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of EuroEval might be problematic. Click here for more details.
- euroeval/__init__.py +6 -0
- euroeval/benchmark_config_factory.py +51 -46
- euroeval/benchmark_modules/base.py +6 -5
- euroeval/benchmark_modules/hf.py +2 -9
- euroeval/benchmark_modules/litellm.py +14 -12
- euroeval/benchmark_modules/vllm.py +17 -10
- euroeval/benchmarker.py +61 -44
- euroeval/caching_utils.py +1 -1
- euroeval/cli.py +86 -8
- euroeval/constants.py +3 -0
- euroeval/data_loading.py +78 -30
- euroeval/data_models.py +326 -326
- euroeval/dataset_configs/__init__.py +10 -3
- euroeval/dataset_configs/bulgarian.py +56 -0
- euroeval/dataset_configs/czech.py +25 -29
- euroeval/dataset_configs/danish.py +51 -88
- euroeval/dataset_configs/dutch.py +48 -86
- euroeval/dataset_configs/english.py +45 -76
- euroeval/dataset_configs/estonian.py +36 -38
- euroeval/dataset_configs/faroese.py +19 -60
- euroeval/dataset_configs/finnish.py +36 -68
- euroeval/dataset_configs/french.py +39 -74
- euroeval/dataset_configs/german.py +45 -81
- euroeval/dataset_configs/greek.py +64 -0
- euroeval/dataset_configs/icelandic.py +54 -91
- euroeval/dataset_configs/italian.py +42 -78
- euroeval/dataset_configs/latvian.py +28 -34
- euroeval/dataset_configs/lithuanian.py +22 -26
- euroeval/dataset_configs/norwegian.py +72 -114
- euroeval/dataset_configs/polish.py +33 -60
- euroeval/dataset_configs/portuguese.py +33 -65
- euroeval/dataset_configs/serbian.py +64 -0
- euroeval/dataset_configs/slovak.py +19 -24
- euroeval/dataset_configs/spanish.py +42 -76
- euroeval/dataset_configs/swedish.py +48 -84
- euroeval/dataset_configs/ukrainian.py +64 -0
- euroeval/exceptions.py +1 -1
- euroeval/finetuning.py +3 -2
- euroeval/generation.py +5 -4
- euroeval/generation_utils.py +6 -5
- euroeval/languages.py +395 -323
- euroeval/metrics/huggingface.py +14 -3
- euroeval/metrics/llm_as_a_judge.py +1 -1
- euroeval/model_cache.py +6 -5
- euroeval/model_loading.py +1 -1
- euroeval/prompt_templates/__init__.py +2 -0
- euroeval/prompt_templates/classification.py +206 -0
- euroeval/prompt_templates/linguistic_acceptability.py +82 -43
- euroeval/prompt_templates/multiple_choice.py +81 -41
- euroeval/prompt_templates/named_entity_recognition.py +125 -44
- euroeval/prompt_templates/reading_comprehension.py +92 -43
- euroeval/prompt_templates/sentiment_classification.py +91 -43
- euroeval/prompt_templates/summarization.py +64 -39
- euroeval/prompt_templates/token_classification.py +279 -0
- euroeval/scores.py +4 -3
- euroeval/speed_benchmark.py +2 -1
- euroeval/task_group_utils/multiple_choice_classification.py +2 -1
- euroeval/task_group_utils/question_answering.py +24 -13
- euroeval/task_group_utils/sequence_classification.py +5 -4
- euroeval/task_group_utils/text_to_text.py +2 -1
- euroeval/task_group_utils/token_classification.py +11 -8
- euroeval/tasks.py +44 -1
- euroeval/tokenisation_utils.py +19 -10
- euroeval/types.py +10 -9
- euroeval/utils.py +6 -3
- {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/METADATA +194 -37
- euroeval-16.5.0.dist-info/RECORD +81 -0
- euroeval-16.4.0.dist-info/RECORD +0 -75
- {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/WHEEL +0 -0
- {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/entry_points.txt +0 -0
- {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,80 +1,74 @@
|
|
|
1
1
|
"""All Icelandic dataset configurations used in EuroEval."""
|
|
2
2
|
|
|
3
3
|
from ..data_models import DatasetConfig
|
|
4
|
-
from ..languages import
|
|
4
|
+
from ..languages import ICELANDIC
|
|
5
5
|
from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
|
|
6
6
|
|
|
7
7
|
### Official datasets ###
|
|
8
8
|
|
|
9
9
|
HOTTER_AND_COLDER_SENTIMENT_CONFIG = DatasetConfig(
|
|
10
10
|
name="hotter-and-colder-sentiment",
|
|
11
|
-
pretty_name="
|
|
12
|
-
"and
|
|
13
|
-
huggingface_id="EuroEval/hotter-and-colder-sentiment",
|
|
11
|
+
pretty_name="Hotter and Colder Sentiment",
|
|
12
|
+
source="EuroEval/hotter-and-colder-sentiment",
|
|
14
13
|
task=SENT,
|
|
15
|
-
languages=[
|
|
14
|
+
languages=[ICELANDIC],
|
|
16
15
|
)
|
|
17
16
|
|
|
18
17
|
SCALA_IS_CONFIG = DatasetConfig(
|
|
19
18
|
name="scala-is",
|
|
20
|
-
pretty_name="
|
|
21
|
-
|
|
19
|
+
pretty_name="ScaLA-is",
|
|
20
|
+
source="EuroEval/scala-is",
|
|
22
21
|
task=LA,
|
|
23
|
-
languages=[
|
|
22
|
+
languages=[ICELANDIC],
|
|
24
23
|
)
|
|
25
24
|
|
|
26
25
|
MIM_GOLD_NER_CONFIG = DatasetConfig(
|
|
27
26
|
name="mim-gold-ner",
|
|
28
|
-
pretty_name="
|
|
29
|
-
"
|
|
30
|
-
huggingface_id="EuroEval/mim-gold-ner-mini",
|
|
27
|
+
pretty_name="MIM-GOLD-NER",
|
|
28
|
+
source="EuroEval/mim-gold-ner-mini",
|
|
31
29
|
task=NER,
|
|
32
|
-
languages=[
|
|
30
|
+
languages=[ICELANDIC],
|
|
33
31
|
)
|
|
34
32
|
|
|
35
33
|
NQII_CONFIG = DatasetConfig(
|
|
36
34
|
name="nqii",
|
|
37
|
-
pretty_name="
|
|
38
|
-
"
|
|
39
|
-
huggingface_id="EuroEval/nqii-mini",
|
|
35
|
+
pretty_name="NQiI",
|
|
36
|
+
source="EuroEval/nqii-mini",
|
|
40
37
|
task=RC,
|
|
41
|
-
languages=[
|
|
38
|
+
languages=[ICELANDIC],
|
|
42
39
|
)
|
|
43
40
|
|
|
44
41
|
RRN_CONFIG = DatasetConfig(
|
|
45
42
|
name="rrn",
|
|
46
|
-
pretty_name="
|
|
47
|
-
"
|
|
48
|
-
huggingface_id="EuroEval/rrn-mini",
|
|
43
|
+
pretty_name="RRN",
|
|
44
|
+
source="EuroEval/rrn-mini",
|
|
49
45
|
task=SUMM,
|
|
50
|
-
languages=[
|
|
46
|
+
languages=[ICELANDIC],
|
|
51
47
|
)
|
|
52
48
|
|
|
53
49
|
ICELANDIC_KNOWLEDGE_CONFIG = DatasetConfig(
|
|
54
50
|
name="icelandic-knowledge",
|
|
55
|
-
pretty_name="
|
|
56
|
-
"
|
|
57
|
-
huggingface_id="EuroEval/icelandic-knowledge",
|
|
51
|
+
pretty_name="Icelandic Knowledge",
|
|
52
|
+
source="EuroEval/icelandic-knowledge",
|
|
58
53
|
task=KNOW,
|
|
59
|
-
languages=[
|
|
54
|
+
languages=[ICELANDIC],
|
|
60
55
|
)
|
|
61
56
|
|
|
62
57
|
WINOGRANDE_IS_CONFIG = DatasetConfig(
|
|
63
58
|
name="winogrande-is",
|
|
64
|
-
pretty_name="
|
|
65
|
-
"
|
|
66
|
-
huggingface_id="EuroEval/winogrande-is",
|
|
59
|
+
pretty_name="Winogrande-is",
|
|
60
|
+
source="EuroEval/winogrande-is",
|
|
67
61
|
task=COMMON_SENSE,
|
|
68
|
-
languages=[
|
|
62
|
+
languages=[ICELANDIC],
|
|
69
63
|
_labels=["a", "b"],
|
|
70
64
|
)
|
|
71
65
|
|
|
72
|
-
|
|
73
|
-
name="
|
|
74
|
-
pretty_name="
|
|
75
|
-
|
|
66
|
+
VALEU_IS_CONFIG = DatasetConfig(
|
|
67
|
+
name="valeu-is",
|
|
68
|
+
pretty_name="VaLEU-is",
|
|
69
|
+
source="EuroEval/european-values-is",
|
|
76
70
|
task=EUROPEAN_VALUES,
|
|
77
|
-
languages=[
|
|
71
|
+
languages=[ICELANDIC],
|
|
78
72
|
splits=["test"],
|
|
79
73
|
bootstrap_samples=False,
|
|
80
74
|
_instruction_prompt="{text}",
|
|
@@ -85,112 +79,81 @@ EUROPEAN_VALUES_IS_CONFIG = DatasetConfig(
|
|
|
85
79
|
|
|
86
80
|
ICE_EC_CONFIG = DatasetConfig(
|
|
87
81
|
name="ice-ec",
|
|
88
|
-
pretty_name="
|
|
89
|
-
|
|
82
|
+
pretty_name="ICE-EC",
|
|
83
|
+
source="EuroEval/ice-ec",
|
|
90
84
|
task=LA,
|
|
91
|
-
languages=[
|
|
85
|
+
languages=[ICELANDIC],
|
|
92
86
|
unofficial=True,
|
|
93
87
|
)
|
|
94
88
|
|
|
95
89
|
ICE_EC_FULL_CONFIG = DatasetConfig(
|
|
96
90
|
name="ice-ec-full",
|
|
97
|
-
pretty_name="
|
|
98
|
-
|
|
91
|
+
pretty_name="ICE-EC Full",
|
|
92
|
+
source="EuroEval/ice-ec-full",
|
|
99
93
|
task=LA,
|
|
100
|
-
languages=[
|
|
94
|
+
languages=[ICELANDIC],
|
|
101
95
|
unofficial=True,
|
|
102
96
|
)
|
|
103
97
|
|
|
104
98
|
ICE_LINGUISTIC_CONFIG = DatasetConfig(
|
|
105
99
|
name="ice-linguistic",
|
|
106
|
-
pretty_name="
|
|
107
|
-
|
|
100
|
+
pretty_name="IceLinguistic",
|
|
101
|
+
source="EuroEval/ice-linguistic",
|
|
108
102
|
task=LA,
|
|
109
|
-
languages=[
|
|
103
|
+
languages=[ICELANDIC],
|
|
110
104
|
unofficial=True,
|
|
111
105
|
)
|
|
112
106
|
|
|
113
107
|
ICELANDIC_QA_CONFIG = DatasetConfig(
|
|
114
108
|
name="icelandic-qa",
|
|
115
|
-
pretty_name="
|
|
116
|
-
|
|
109
|
+
pretty_name="Icelandic QA",
|
|
110
|
+
source="EuroEval/icelandic-qa",
|
|
117
111
|
task=RC,
|
|
118
|
-
languages=[
|
|
112
|
+
languages=[ICELANDIC],
|
|
119
113
|
unofficial=True,
|
|
120
114
|
)
|
|
121
115
|
|
|
122
116
|
MMLU_IS_CONFIG = DatasetConfig(
|
|
123
117
|
name="mmlu-is",
|
|
124
|
-
pretty_name="
|
|
125
|
-
"
|
|
126
|
-
huggingface_id="EuroEval/mmlu-is-mini",
|
|
118
|
+
pretty_name="MMLU-is",
|
|
119
|
+
source="EuroEval/mmlu-is-mini",
|
|
127
120
|
task=KNOW,
|
|
128
|
-
languages=[
|
|
121
|
+
languages=[ICELANDIC],
|
|
129
122
|
unofficial=True,
|
|
130
123
|
)
|
|
131
124
|
|
|
132
125
|
ARC_IS_CONFIG = DatasetConfig(
|
|
133
126
|
name="arc-is",
|
|
134
|
-
pretty_name="
|
|
135
|
-
"
|
|
136
|
-
huggingface_id="EuroEval/arc-is-mini",
|
|
127
|
+
pretty_name="ARC-is",
|
|
128
|
+
source="EuroEval/arc-is-mini",
|
|
137
129
|
task=KNOW,
|
|
138
|
-
languages=[
|
|
130
|
+
languages=[ICELANDIC],
|
|
139
131
|
unofficial=True,
|
|
140
132
|
)
|
|
141
133
|
|
|
142
134
|
HELLASWAG_IS_CONFIG = DatasetConfig(
|
|
143
135
|
name="hellaswag-is",
|
|
144
|
-
pretty_name="
|
|
145
|
-
"
|
|
146
|
-
huggingface_id="EuroEval/hellaswag-is-mini",
|
|
136
|
+
pretty_name="HellaSwag-is",
|
|
137
|
+
source="EuroEval/hellaswag-is-mini",
|
|
147
138
|
task=COMMON_SENSE,
|
|
148
|
-
languages=[
|
|
139
|
+
languages=[ICELANDIC],
|
|
149
140
|
unofficial=True,
|
|
150
141
|
)
|
|
151
142
|
|
|
152
143
|
BELEBELE_IS_CONFIG = DatasetConfig(
|
|
153
144
|
name="belebele-is",
|
|
154
|
-
pretty_name="
|
|
155
|
-
"
|
|
156
|
-
huggingface_id="EuroEval/belebele-is-mini",
|
|
145
|
+
pretty_name="Belebele-is",
|
|
146
|
+
source="EuroEval/belebele-is-mini",
|
|
157
147
|
task=MCRC,
|
|
158
|
-
languages=[
|
|
148
|
+
languages=[ICELANDIC],
|
|
159
149
|
unofficial=True,
|
|
160
150
|
)
|
|
161
151
|
|
|
162
152
|
MULTI_WIKI_QA_IS_CONFIG = DatasetConfig(
|
|
163
153
|
name="multi-wiki-qa-is",
|
|
164
|
-
pretty_name="
|
|
165
|
-
"
|
|
166
|
-
huggingface_id="EuroEval/multi-wiki-qa-is-mini",
|
|
154
|
+
pretty_name="MultiWikiQA-is",
|
|
155
|
+
source="EuroEval/multi-wiki-qa-is-mini",
|
|
167
156
|
task=RC,
|
|
168
|
-
languages=[
|
|
169
|
-
unofficial=True,
|
|
170
|
-
)
|
|
171
|
-
|
|
172
|
-
EUROPEAN_VALUES_SITUATIONAL_IS_CONFIG = DatasetConfig(
|
|
173
|
-
name="european-values-situational-is",
|
|
174
|
-
pretty_name="the Icelandic version of the European values evaluation dataset, "
|
|
175
|
-
"where the questions are phrased in a situational way",
|
|
176
|
-
huggingface_id="EuroEval/european-values-situational-is",
|
|
177
|
-
task=EUROPEAN_VALUES,
|
|
178
|
-
languages=[IS],
|
|
179
|
-
splits=["test"],
|
|
180
|
-
bootstrap_samples=False,
|
|
181
|
-
_instruction_prompt="{text}",
|
|
182
|
-
unofficial=True,
|
|
183
|
-
)
|
|
184
|
-
|
|
185
|
-
EUROPEAN_VALUES_COMPLETIONS_IS_CONFIG = DatasetConfig(
|
|
186
|
-
name="european-values-completions-is",
|
|
187
|
-
pretty_name="the Icelandic version of the European values evaluation dataset, "
|
|
188
|
-
"where the questions are phrased as sentence completions",
|
|
189
|
-
huggingface_id="EuroEval/european-values-completions-is",
|
|
190
|
-
task=EUROPEAN_VALUES,
|
|
191
|
-
languages=[IS],
|
|
192
|
-
splits=["test"],
|
|
193
|
-
bootstrap_samples=False,
|
|
194
|
-
_instruction_prompt="{text}",
|
|
157
|
+
languages=[ICELANDIC],
|
|
195
158
|
unofficial=True,
|
|
196
159
|
)
|
|
@@ -1,78 +1,73 @@
|
|
|
1
1
|
"""All Italian dataset configurations used in EuroEval."""
|
|
2
2
|
|
|
3
3
|
from ..data_models import DatasetConfig
|
|
4
|
-
from ..languages import
|
|
4
|
+
from ..languages import ITALIAN
|
|
5
5
|
from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
|
|
6
6
|
|
|
7
7
|
### Official datasets ###
|
|
8
8
|
|
|
9
9
|
SENTIPOLC_CONFIG = DatasetConfig(
|
|
10
10
|
name="sentipolc16",
|
|
11
|
-
pretty_name="
|
|
12
|
-
"
|
|
13
|
-
huggingface_id="EuroEval/sentipolc16-mini",
|
|
11
|
+
pretty_name="Sentipolc16",
|
|
12
|
+
source="EuroEval/sentipolc16-mini",
|
|
14
13
|
task=SENT,
|
|
15
|
-
languages=[
|
|
14
|
+
languages=[ITALIAN],
|
|
16
15
|
)
|
|
17
16
|
|
|
18
17
|
SCALA_IT_CONFIG = DatasetConfig(
|
|
19
18
|
name="scala-it",
|
|
20
|
-
pretty_name="
|
|
21
|
-
|
|
19
|
+
pretty_name="ScaLA-it",
|
|
20
|
+
source="EuroEval/scala-it",
|
|
22
21
|
task=LA,
|
|
23
|
-
languages=[
|
|
22
|
+
languages=[ITALIAN],
|
|
24
23
|
)
|
|
25
24
|
|
|
26
25
|
MULTINERD_IT_CONFIG = DatasetConfig(
|
|
27
26
|
name="multinerd-it",
|
|
28
|
-
pretty_name="
|
|
29
|
-
"
|
|
30
|
-
huggingface_id="EuroEval/multinerd-mini-it",
|
|
27
|
+
pretty_name="MultiNERD-it",
|
|
28
|
+
source="EuroEval/multinerd-mini-it",
|
|
31
29
|
task=NER,
|
|
32
|
-
languages=[
|
|
30
|
+
languages=[ITALIAN],
|
|
33
31
|
)
|
|
34
32
|
|
|
35
33
|
SQUAD_IT_CONFIG = DatasetConfig(
|
|
36
34
|
name="squad-it",
|
|
37
|
-
pretty_name="
|
|
38
|
-
"
|
|
39
|
-
huggingface_id="EuroEval/squad-it-mini",
|
|
35
|
+
pretty_name="SQuAD-it",
|
|
36
|
+
source="EuroEval/squad-it-mini",
|
|
40
37
|
task=RC,
|
|
41
|
-
languages=[
|
|
38
|
+
languages=[ITALIAN],
|
|
42
39
|
)
|
|
43
40
|
|
|
44
41
|
ILPOST_SUM_CONFIG = DatasetConfig(
|
|
45
42
|
name="ilpost-sum",
|
|
46
|
-
pretty_name="
|
|
47
|
-
|
|
43
|
+
pretty_name="IlPost-Sum",
|
|
44
|
+
source="EuroEval/ilpost-sum",
|
|
48
45
|
task=SUMM,
|
|
49
|
-
languages=[
|
|
46
|
+
languages=[ITALIAN],
|
|
50
47
|
)
|
|
51
48
|
|
|
52
49
|
MMLU_IT_CONFIG = DatasetConfig(
|
|
53
50
|
name="mmlu-it",
|
|
54
|
-
pretty_name="
|
|
55
|
-
"
|
|
56
|
-
huggingface_id="EuroEval/mmlu-it-mini",
|
|
51
|
+
pretty_name="MMLU-it",
|
|
52
|
+
source="EuroEval/mmlu-it-mini",
|
|
57
53
|
task=KNOW,
|
|
58
|
-
languages=[
|
|
54
|
+
languages=[ITALIAN],
|
|
59
55
|
)
|
|
60
56
|
|
|
61
57
|
HELLASWAG_IT_CONFIG = DatasetConfig(
|
|
62
58
|
name="hellaswag-it",
|
|
63
|
-
pretty_name="
|
|
64
|
-
"
|
|
65
|
-
huggingface_id="EuroEval/hellaswag-it-mini",
|
|
59
|
+
pretty_name="HellaSwag-it",
|
|
60
|
+
source="EuroEval/hellaswag-it-mini",
|
|
66
61
|
task=COMMON_SENSE,
|
|
67
|
-
languages=[
|
|
62
|
+
languages=[ITALIAN],
|
|
68
63
|
)
|
|
69
64
|
|
|
70
|
-
|
|
71
|
-
name="
|
|
72
|
-
pretty_name="
|
|
73
|
-
|
|
65
|
+
VALEU_IT_CONFIG = DatasetConfig(
|
|
66
|
+
name="valeu-it",
|
|
67
|
+
pretty_name="VaLEU-it",
|
|
68
|
+
source="EuroEval/european-values-it",
|
|
74
69
|
task=EUROPEAN_VALUES,
|
|
75
|
-
languages=[
|
|
70
|
+
languages=[ITALIAN],
|
|
76
71
|
splits=["test"],
|
|
77
72
|
bootstrap_samples=False,
|
|
78
73
|
_instruction_prompt="{text}",
|
|
@@ -83,77 +78,46 @@ EUROPEAN_VALUES_IT_CONFIG = DatasetConfig(
|
|
|
83
78
|
|
|
84
79
|
WIKINEURAL_IT_CONFIG = DatasetConfig(
|
|
85
80
|
name="wikineural-it",
|
|
86
|
-
pretty_name="
|
|
87
|
-
"
|
|
88
|
-
huggingface_id="EuroEval/wikineural-mini-it",
|
|
81
|
+
pretty_name="WikiNeural-it",
|
|
82
|
+
source="EuroEval/wikineural-mini-it",
|
|
89
83
|
task=NER,
|
|
90
|
-
languages=[
|
|
84
|
+
languages=[ITALIAN],
|
|
91
85
|
unofficial=True,
|
|
92
86
|
)
|
|
93
87
|
|
|
94
88
|
BELEBELE_IT_CONFIG = DatasetConfig(
|
|
95
89
|
name="belebele-it",
|
|
96
|
-
pretty_name="
|
|
97
|
-
"
|
|
98
|
-
huggingface_id="EuroEval/belebele-it-mini",
|
|
90
|
+
pretty_name="Belebele-it",
|
|
91
|
+
source="EuroEval/belebele-it-mini",
|
|
99
92
|
task=MCRC,
|
|
100
|
-
languages=[
|
|
93
|
+
languages=[ITALIAN],
|
|
101
94
|
unofficial=True,
|
|
102
95
|
)
|
|
103
96
|
|
|
104
97
|
MULTI_WIKI_QA_IT_CONFIG = DatasetConfig(
|
|
105
98
|
name="multi-wiki-qa-it",
|
|
106
|
-
pretty_name="
|
|
107
|
-
"
|
|
108
|
-
huggingface_id="EuroEval/multi-wiki-qa-it-mini",
|
|
99
|
+
pretty_name="MultiWikiQA-it",
|
|
100
|
+
source="EuroEval/multi-wiki-qa-it-mini",
|
|
109
101
|
task=RC,
|
|
110
|
-
languages=[
|
|
102
|
+
languages=[ITALIAN],
|
|
111
103
|
unofficial=True,
|
|
112
104
|
)
|
|
113
105
|
|
|
114
106
|
GOLDENSWAG_IT_CONFIG = DatasetConfig(
|
|
115
107
|
name="goldenswag-it",
|
|
116
|
-
pretty_name="
|
|
117
|
-
"
|
|
118
|
-
huggingface_id="EuroEval/goldenswag-it-mini",
|
|
108
|
+
pretty_name="GoldenSwag-it",
|
|
109
|
+
source="EuroEval/goldenswag-it-mini",
|
|
119
110
|
task=COMMON_SENSE,
|
|
120
|
-
languages=[
|
|
111
|
+
languages=[ITALIAN],
|
|
121
112
|
unofficial=True,
|
|
122
113
|
)
|
|
123
114
|
|
|
124
115
|
WINOGRANDE_IT_CONFIG = DatasetConfig(
|
|
125
116
|
name="winogrande-it",
|
|
126
|
-
pretty_name="
|
|
127
|
-
"
|
|
128
|
-
huggingface_id="EuroEval/winogrande-it",
|
|
117
|
+
pretty_name="Winogrande-it",
|
|
118
|
+
source="EuroEval/winogrande-it",
|
|
129
119
|
task=COMMON_SENSE,
|
|
130
|
-
languages=[
|
|
120
|
+
languages=[ITALIAN],
|
|
131
121
|
_labels=["a", "b"],
|
|
132
122
|
unofficial=True,
|
|
133
123
|
)
|
|
134
|
-
|
|
135
|
-
EUROPEAN_VALUES_SITUATIONAL_IT_CONFIG = DatasetConfig(
|
|
136
|
-
name="european-values-situational-it",
|
|
137
|
-
pretty_name="the Italian version of the European values evaluation dataset, "
|
|
138
|
-
"where the questions are phrased in a situational way",
|
|
139
|
-
huggingface_id="EuroEval/european-values-situational-it",
|
|
140
|
-
task=EUROPEAN_VALUES,
|
|
141
|
-
languages=[IT],
|
|
142
|
-
splits=["test"],
|
|
143
|
-
bootstrap_samples=False,
|
|
144
|
-
_instruction_prompt="{text}",
|
|
145
|
-
unofficial=True,
|
|
146
|
-
)
|
|
147
|
-
|
|
148
|
-
EUROPEAN_VALUES_COMPLETIONS_IT_CONFIG = DatasetConfig(
|
|
149
|
-
name="european-values-completions-it",
|
|
150
|
-
pretty_name="the Italian version of the European values evaluation dataset, where "
|
|
151
|
-
"the questions are phrased as sentence completions",
|
|
152
|
-
huggingface_id="EuroEval/european-values-completions-it",
|
|
153
|
-
task=EUROPEAN_VALUES,
|
|
154
|
-
languages=[IT],
|
|
155
|
-
splits=["test"],
|
|
156
|
-
bootstrap_samples=False,
|
|
157
|
-
_instruction_prompt="{text}",
|
|
158
|
-
unofficial=True,
|
|
159
|
-
)
|
|
@@ -1,70 +1,66 @@
|
|
|
1
1
|
"""All Latvian dataset configurations used in EuroEval."""
|
|
2
2
|
|
|
3
3
|
from ..data_models import DatasetConfig
|
|
4
|
-
from ..languages import
|
|
4
|
+
from ..languages import LATVIAN
|
|
5
5
|
from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
|
|
6
6
|
|
|
7
7
|
### Official datasets ###
|
|
8
8
|
|
|
9
9
|
LATVIAN_TWITTER_SENTIMENT_CONFIG = DatasetConfig(
|
|
10
10
|
name="latvian-twitter-sentiment",
|
|
11
|
-
pretty_name="
|
|
12
|
-
|
|
11
|
+
pretty_name="Latvian Twitter Sentiment",
|
|
12
|
+
source="EuroEval/latvian-twitter-sentiment-mini",
|
|
13
13
|
task=SENT,
|
|
14
|
-
languages=[
|
|
14
|
+
languages=[LATVIAN],
|
|
15
15
|
)
|
|
16
16
|
|
|
17
17
|
SCALA_LV_CONFIG = DatasetConfig(
|
|
18
18
|
name="scala-lv",
|
|
19
|
-
pretty_name="
|
|
20
|
-
|
|
19
|
+
pretty_name="ScaLA-lv",
|
|
20
|
+
source="EuroEval/scala-lv",
|
|
21
21
|
task=LA,
|
|
22
|
-
languages=[
|
|
22
|
+
languages=[LATVIAN],
|
|
23
23
|
)
|
|
24
24
|
|
|
25
25
|
FULLSTACK_NER_LV_CONFIG = DatasetConfig(
|
|
26
26
|
name="fullstack-ner-lv",
|
|
27
|
-
pretty_name="
|
|
28
|
-
"
|
|
29
|
-
huggingface_id="EuroEval/fullstack-ner-lv-mini",
|
|
27
|
+
pretty_name="FullStack NER-lv",
|
|
28
|
+
source="EuroEval/fullstack-ner-lv-mini",
|
|
30
29
|
task=NER,
|
|
31
|
-
languages=[
|
|
30
|
+
languages=[LATVIAN],
|
|
32
31
|
)
|
|
33
32
|
|
|
34
33
|
MULTI_WIKI_QA_LV_CONFIG = DatasetConfig(
|
|
35
34
|
name="multi-wiki-qa-lv",
|
|
36
|
-
pretty_name="
|
|
37
|
-
"
|
|
38
|
-
huggingface_id="EuroEval/multi-wiki-qa-lv-mini",
|
|
35
|
+
pretty_name="MultiWikiQA-lv",
|
|
36
|
+
source="EuroEval/multi-wiki-qa-lv-mini",
|
|
39
37
|
task=RC,
|
|
40
|
-
languages=[
|
|
38
|
+
languages=[LATVIAN],
|
|
41
39
|
)
|
|
42
40
|
|
|
43
41
|
LSM_CONFIG = DatasetConfig(
|
|
44
42
|
name="lsm",
|
|
45
|
-
pretty_name="
|
|
46
|
-
|
|
43
|
+
pretty_name="LSM",
|
|
44
|
+
source="EuroEval/lsm-mini",
|
|
47
45
|
task=SUMM,
|
|
48
|
-
languages=[
|
|
46
|
+
languages=[LATVIAN],
|
|
49
47
|
)
|
|
50
48
|
|
|
51
49
|
|
|
52
50
|
MMLU_LV_CONFIG = DatasetConfig(
|
|
53
51
|
name="mmlu-lv",
|
|
54
|
-
pretty_name="
|
|
55
|
-
"
|
|
56
|
-
huggingface_id="EuroEval/mmlu-lv-mini",
|
|
52
|
+
pretty_name="MMLU-lv",
|
|
53
|
+
source="EuroEval/mmlu-lv-mini",
|
|
57
54
|
task=KNOW,
|
|
58
|
-
languages=[
|
|
55
|
+
languages=[LATVIAN],
|
|
59
56
|
)
|
|
60
57
|
|
|
61
58
|
COPA_LV_CONFIG = DatasetConfig(
|
|
62
59
|
name="copa-lv",
|
|
63
|
-
pretty_name="
|
|
64
|
-
"
|
|
65
|
-
huggingface_id="EuroEval/copa-lv",
|
|
60
|
+
pretty_name="COPA-lv",
|
|
61
|
+
source="EuroEval/copa-lv",
|
|
66
62
|
task=COMMON_SENSE,
|
|
67
|
-
languages=[
|
|
63
|
+
languages=[LATVIAN],
|
|
68
64
|
_labels=["a", "b"],
|
|
69
65
|
)
|
|
70
66
|
|
|
@@ -73,21 +69,19 @@ COPA_LV_CONFIG = DatasetConfig(
|
|
|
73
69
|
|
|
74
70
|
WIKIANN_LV_CONFIG = DatasetConfig(
|
|
75
71
|
name="wikiann-lv",
|
|
76
|
-
pretty_name="
|
|
77
|
-
"
|
|
78
|
-
huggingface_id="EuroEval/wikiann-lv-mini",
|
|
72
|
+
pretty_name="WikiANN-lv",
|
|
73
|
+
source="EuroEval/wikiann-lv-mini",
|
|
79
74
|
task=NER,
|
|
80
|
-
languages=[
|
|
75
|
+
languages=[LATVIAN],
|
|
81
76
|
unofficial=True,
|
|
82
77
|
)
|
|
83
78
|
|
|
84
79
|
WINOGRANDE_LV_CONFIG = DatasetConfig(
|
|
85
80
|
name="winogrande-lv",
|
|
86
|
-
pretty_name="
|
|
87
|
-
"
|
|
88
|
-
huggingface_id="EuroEval/winogrande-lv",
|
|
81
|
+
pretty_name="Winogrande-lv",
|
|
82
|
+
source="EuroEval/winogrande-lv",
|
|
89
83
|
task=COMMON_SENSE,
|
|
90
|
-
languages=[
|
|
84
|
+
languages=[LATVIAN],
|
|
91
85
|
_labels=["a", "b"],
|
|
92
86
|
unofficial=True,
|
|
93
87
|
)
|
|
@@ -1,68 +1,64 @@
|
|
|
1
1
|
"""All Lithuanian dataset configurations used in EuroEval."""
|
|
2
2
|
|
|
3
3
|
from ..data_models import DatasetConfig
|
|
4
|
-
from ..languages import
|
|
4
|
+
from ..languages import LITHUANIAN
|
|
5
5
|
from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
|
|
6
6
|
|
|
7
7
|
### Official datasets ###
|
|
8
8
|
|
|
9
9
|
LITHUANIAN_EMOTIONS_CONFIG = DatasetConfig(
|
|
10
10
|
name="lithuanian-emotions",
|
|
11
|
-
pretty_name="
|
|
12
|
-
"
|
|
13
|
-
huggingface_id="EuroEval/lithuanian-emotions-mini",
|
|
11
|
+
pretty_name="Lithuanian Emotions",
|
|
12
|
+
source="EuroEval/lithuanian-emotions-mini",
|
|
14
13
|
task=SENT,
|
|
15
|
-
languages=[
|
|
14
|
+
languages=[LITHUANIAN],
|
|
16
15
|
)
|
|
17
16
|
|
|
18
17
|
SCALA_LT_CONFIG = DatasetConfig(
|
|
19
18
|
name="scala-lt",
|
|
20
|
-
pretty_name="
|
|
21
|
-
|
|
19
|
+
pretty_name="ScaLA-lt",
|
|
20
|
+
source="EuroEval/scala-lt",
|
|
22
21
|
task=LA,
|
|
23
|
-
languages=[
|
|
22
|
+
languages=[LITHUANIAN],
|
|
24
23
|
)
|
|
25
24
|
|
|
26
25
|
WIKIANN_LT_CONFIG = DatasetConfig(
|
|
27
26
|
name="wikiann-lt",
|
|
28
|
-
pretty_name="
|
|
29
|
-
"
|
|
30
|
-
huggingface_id="EuroEval/wikiann-lt-mini",
|
|
27
|
+
pretty_name="WikiANN-lt",
|
|
28
|
+
source="EuroEval/wikiann-lt-mini",
|
|
31
29
|
task=NER,
|
|
32
|
-
languages=[
|
|
30
|
+
languages=[LITHUANIAN],
|
|
33
31
|
)
|
|
34
32
|
|
|
35
33
|
MULTI_WIKI_QA_LT_CONFIG = DatasetConfig(
|
|
36
34
|
name="multi-wiki-qa-lt",
|
|
37
|
-
pretty_name="
|
|
38
|
-
"
|
|
39
|
-
huggingface_id="EuroEval/multi-wiki-qa-lt-mini",
|
|
35
|
+
pretty_name="MultiWikiQA-lt",
|
|
36
|
+
source="EuroEval/multi-wiki-qa-lt-mini",
|
|
40
37
|
task=RC,
|
|
41
|
-
languages=[
|
|
38
|
+
languages=[LITHUANIAN],
|
|
42
39
|
)
|
|
43
40
|
|
|
44
41
|
LRYTAS_CONFIG = DatasetConfig(
|
|
45
42
|
name="lrytas",
|
|
46
|
-
pretty_name="
|
|
47
|
-
|
|
43
|
+
pretty_name="Lrytas",
|
|
44
|
+
source="EuroEval/lrytas-mini",
|
|
48
45
|
task=SUMM,
|
|
49
|
-
languages=[
|
|
46
|
+
languages=[LITHUANIAN],
|
|
50
47
|
)
|
|
51
48
|
|
|
52
49
|
LT_HISTORY_CONFIG = DatasetConfig(
|
|
53
50
|
name="lt-history",
|
|
54
|
-
pretty_name="
|
|
55
|
-
|
|
51
|
+
pretty_name="LT-History",
|
|
52
|
+
source="EuroEval/lt-history",
|
|
56
53
|
task=KNOW,
|
|
57
|
-
languages=[
|
|
54
|
+
languages=[LITHUANIAN],
|
|
58
55
|
)
|
|
59
56
|
|
|
60
57
|
WINOGRANDE_LT_CONFIG = DatasetConfig(
|
|
61
58
|
name="winogrande-lt",
|
|
62
|
-
pretty_name="
|
|
63
|
-
"
|
|
64
|
-
huggingface_id="EuroEval/winogrande-lt",
|
|
59
|
+
pretty_name="Winogrande-lt",
|
|
60
|
+
source="EuroEval/winogrande-lt",
|
|
65
61
|
task=COMMON_SENSE,
|
|
66
|
-
languages=[
|
|
62
|
+
languages=[LITHUANIAN],
|
|
67
63
|
_labels=["a", "b"],
|
|
68
64
|
)
|