EuroEval 16.4.0__py3-none-any.whl → 16.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of EuroEval might be problematic. Click here for more details.

Files changed (71) hide show
  1. euroeval/__init__.py +6 -0
  2. euroeval/benchmark_config_factory.py +51 -46
  3. euroeval/benchmark_modules/base.py +6 -5
  4. euroeval/benchmark_modules/hf.py +2 -9
  5. euroeval/benchmark_modules/litellm.py +14 -12
  6. euroeval/benchmark_modules/vllm.py +17 -10
  7. euroeval/benchmarker.py +61 -44
  8. euroeval/caching_utils.py +1 -1
  9. euroeval/cli.py +86 -8
  10. euroeval/constants.py +3 -0
  11. euroeval/data_loading.py +78 -30
  12. euroeval/data_models.py +326 -326
  13. euroeval/dataset_configs/__init__.py +10 -3
  14. euroeval/dataset_configs/bulgarian.py +56 -0
  15. euroeval/dataset_configs/czech.py +25 -29
  16. euroeval/dataset_configs/danish.py +51 -88
  17. euroeval/dataset_configs/dutch.py +48 -86
  18. euroeval/dataset_configs/english.py +45 -76
  19. euroeval/dataset_configs/estonian.py +36 -38
  20. euroeval/dataset_configs/faroese.py +19 -60
  21. euroeval/dataset_configs/finnish.py +36 -68
  22. euroeval/dataset_configs/french.py +39 -74
  23. euroeval/dataset_configs/german.py +45 -81
  24. euroeval/dataset_configs/greek.py +64 -0
  25. euroeval/dataset_configs/icelandic.py +54 -91
  26. euroeval/dataset_configs/italian.py +42 -78
  27. euroeval/dataset_configs/latvian.py +28 -34
  28. euroeval/dataset_configs/lithuanian.py +22 -26
  29. euroeval/dataset_configs/norwegian.py +72 -114
  30. euroeval/dataset_configs/polish.py +33 -60
  31. euroeval/dataset_configs/portuguese.py +33 -65
  32. euroeval/dataset_configs/serbian.py +64 -0
  33. euroeval/dataset_configs/slovak.py +19 -24
  34. euroeval/dataset_configs/spanish.py +42 -76
  35. euroeval/dataset_configs/swedish.py +48 -84
  36. euroeval/dataset_configs/ukrainian.py +64 -0
  37. euroeval/exceptions.py +1 -1
  38. euroeval/finetuning.py +3 -2
  39. euroeval/generation.py +5 -4
  40. euroeval/generation_utils.py +6 -5
  41. euroeval/languages.py +395 -323
  42. euroeval/metrics/huggingface.py +14 -3
  43. euroeval/metrics/llm_as_a_judge.py +1 -1
  44. euroeval/model_cache.py +6 -5
  45. euroeval/model_loading.py +1 -1
  46. euroeval/prompt_templates/__init__.py +2 -0
  47. euroeval/prompt_templates/classification.py +206 -0
  48. euroeval/prompt_templates/linguistic_acceptability.py +82 -43
  49. euroeval/prompt_templates/multiple_choice.py +81 -41
  50. euroeval/prompt_templates/named_entity_recognition.py +125 -44
  51. euroeval/prompt_templates/reading_comprehension.py +92 -43
  52. euroeval/prompt_templates/sentiment_classification.py +91 -43
  53. euroeval/prompt_templates/summarization.py +64 -39
  54. euroeval/prompt_templates/token_classification.py +279 -0
  55. euroeval/scores.py +4 -3
  56. euroeval/speed_benchmark.py +2 -1
  57. euroeval/task_group_utils/multiple_choice_classification.py +2 -1
  58. euroeval/task_group_utils/question_answering.py +24 -13
  59. euroeval/task_group_utils/sequence_classification.py +5 -4
  60. euroeval/task_group_utils/text_to_text.py +2 -1
  61. euroeval/task_group_utils/token_classification.py +11 -8
  62. euroeval/tasks.py +44 -1
  63. euroeval/tokenisation_utils.py +19 -10
  64. euroeval/types.py +10 -9
  65. euroeval/utils.py +6 -3
  66. {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/METADATA +194 -37
  67. euroeval-16.5.0.dist-info/RECORD +81 -0
  68. euroeval-16.4.0.dist-info/RECORD +0 -75
  69. {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/WHEEL +0 -0
  70. {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/entry_points.txt +0 -0
  71. {euroeval-16.4.0.dist-info → euroeval-16.5.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,80 +1,74 @@
1
1
  """All Icelandic dataset configurations used in EuroEval."""
2
2
 
3
3
  from ..data_models import DatasetConfig
4
- from ..languages import IS
4
+ from ..languages import ICELANDIC
5
5
  from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
6
6
 
7
7
  ### Official datasets ###
8
8
 
9
9
  HOTTER_AND_COLDER_SENTIMENT_CONFIG = DatasetConfig(
10
10
  name="hotter-and-colder-sentiment",
11
- pretty_name="the sentiment classification part of the Icelandic dataset Hotter "
12
- "and Colder",
13
- huggingface_id="EuroEval/hotter-and-colder-sentiment",
11
+ pretty_name="Hotter and Colder Sentiment",
12
+ source="EuroEval/hotter-and-colder-sentiment",
14
13
  task=SENT,
15
- languages=[IS],
14
+ languages=[ICELANDIC],
16
15
  )
17
16
 
18
17
  SCALA_IS_CONFIG = DatasetConfig(
19
18
  name="scala-is",
20
- pretty_name="the Icelandic part of the linguistic acceptability dataset ScaLA",
21
- huggingface_id="EuroEval/scala-is",
19
+ pretty_name="ScaLA-is",
20
+ source="EuroEval/scala-is",
22
21
  task=LA,
23
- languages=[IS],
22
+ languages=[ICELANDIC],
24
23
  )
25
24
 
26
25
  MIM_GOLD_NER_CONFIG = DatasetConfig(
27
26
  name="mim-gold-ner",
28
- pretty_name="the truncated version of the Icelandic named entity recognition "
29
- "dataset MIM-GOLD-NER",
30
- huggingface_id="EuroEval/mim-gold-ner-mini",
27
+ pretty_name="MIM-GOLD-NER",
28
+ source="EuroEval/mim-gold-ner-mini",
31
29
  task=NER,
32
- languages=[IS],
30
+ languages=[ICELANDIC],
33
31
  )
34
32
 
35
33
  NQII_CONFIG = DatasetConfig(
36
34
  name="nqii",
37
- pretty_name="the truncated version of the Icelandic reading comprehension dataset "
38
- "Natural Questions in Icelandic",
39
- huggingface_id="EuroEval/nqii-mini",
35
+ pretty_name="NQiI",
36
+ source="EuroEval/nqii-mini",
40
37
  task=RC,
41
- languages=[IS],
38
+ languages=[ICELANDIC],
42
39
  )
43
40
 
44
41
  RRN_CONFIG = DatasetConfig(
45
42
  name="rrn",
46
- pretty_name="the truncated version of the Icelandic summarisation dataset "
47
- "RÚV Radio News",
48
- huggingface_id="EuroEval/rrn-mini",
43
+ pretty_name="RRN",
44
+ source="EuroEval/rrn-mini",
49
45
  task=SUMM,
50
- languages=[IS],
46
+ languages=[ICELANDIC],
51
47
  )
52
48
 
53
49
  ICELANDIC_KNOWLEDGE_CONFIG = DatasetConfig(
54
50
  name="icelandic-knowledge",
55
- pretty_name="the Icelandic knowledge dataset IcelandicKnowledge, derived from the "
56
- "IcelandicQA dataset",
57
- huggingface_id="EuroEval/icelandic-knowledge",
51
+ pretty_name="Icelandic Knowledge",
52
+ source="EuroEval/icelandic-knowledge",
58
53
  task=KNOW,
59
- languages=[IS],
54
+ languages=[ICELANDIC],
60
55
  )
61
56
 
62
57
  WINOGRANDE_IS_CONFIG = DatasetConfig(
63
58
  name="winogrande-is",
64
- pretty_name="the Icelandic common-sense reasoning dataset "
65
- "Winogrande-is, manually translated from the English Winogrande dataset",
66
- huggingface_id="EuroEval/winogrande-is",
59
+ pretty_name="Winogrande-is",
60
+ source="EuroEval/winogrande-is",
67
61
  task=COMMON_SENSE,
68
- languages=[IS],
62
+ languages=[ICELANDIC],
69
63
  _labels=["a", "b"],
70
64
  )
71
65
 
72
- EUROPEAN_VALUES_IS_CONFIG = DatasetConfig(
73
- name="european-values-is",
74
- pretty_name="the Icelandic version of the European values evaluation dataset",
75
- huggingface_id="EuroEval/european-values-is",
66
+ VALEU_IS_CONFIG = DatasetConfig(
67
+ name="valeu-is",
68
+ pretty_name="VaLEU-is",
69
+ source="EuroEval/european-values-is",
76
70
  task=EUROPEAN_VALUES,
77
- languages=[IS],
71
+ languages=[ICELANDIC],
78
72
  splits=["test"],
79
73
  bootstrap_samples=False,
80
74
  _instruction_prompt="{text}",
@@ -85,112 +79,81 @@ EUROPEAN_VALUES_IS_CONFIG = DatasetConfig(
85
79
 
86
80
  ICE_EC_CONFIG = DatasetConfig(
87
81
  name="ice-ec",
88
- pretty_name="the truncated version of the Icelandic Error Corpus",
89
- huggingface_id="EuroEval/ice-ec",
82
+ pretty_name="ICE-EC",
83
+ source="EuroEval/ice-ec",
90
84
  task=LA,
91
- languages=[IS],
85
+ languages=[ICELANDIC],
92
86
  unofficial=True,
93
87
  )
94
88
 
95
89
  ICE_EC_FULL_CONFIG = DatasetConfig(
96
90
  name="ice-ec-full",
97
- pretty_name="the Icelandic Error Corpus",
98
- huggingface_id="EuroEval/ice-ec-full",
91
+ pretty_name="ICE-EC Full",
92
+ source="EuroEval/ice-ec-full",
99
93
  task=LA,
100
- languages=[IS],
94
+ languages=[ICELANDIC],
101
95
  unofficial=True,
102
96
  )
103
97
 
104
98
  ICE_LINGUISTIC_CONFIG = DatasetConfig(
105
99
  name="ice-linguistic",
106
- pretty_name="the Icelandic linguistic acceptability dataset IceLinguistic",
107
- huggingface_id="EuroEval/ice-linguistic",
100
+ pretty_name="IceLinguistic",
101
+ source="EuroEval/ice-linguistic",
108
102
  task=LA,
109
- languages=[IS],
103
+ languages=[ICELANDIC],
110
104
  unofficial=True,
111
105
  )
112
106
 
113
107
  ICELANDIC_QA_CONFIG = DatasetConfig(
114
108
  name="icelandic-qa",
115
- pretty_name="the Icelandic reading comprehension dataset IcelandicQA",
116
- huggingface_id="EuroEval/icelandic-qa",
109
+ pretty_name="Icelandic QA",
110
+ source="EuroEval/icelandic-qa",
117
111
  task=RC,
118
- languages=[IS],
112
+ languages=[ICELANDIC],
119
113
  unofficial=True,
120
114
  )
121
115
 
122
116
  MMLU_IS_CONFIG = DatasetConfig(
123
117
  name="mmlu-is",
124
- pretty_name="the truncated version of the Icelandic knowledge dataset MMLU-is, "
125
- "translated from the English MMLU dataset",
126
- huggingface_id="EuroEval/mmlu-is-mini",
118
+ pretty_name="MMLU-is",
119
+ source="EuroEval/mmlu-is-mini",
127
120
  task=KNOW,
128
- languages=[IS],
121
+ languages=[ICELANDIC],
129
122
  unofficial=True,
130
123
  )
131
124
 
132
125
  ARC_IS_CONFIG = DatasetConfig(
133
126
  name="arc-is",
134
- pretty_name="the truncated version of the Icelandic knowledge dataset ARC-is, "
135
- "translated from the English ARC dataset",
136
- huggingface_id="EuroEval/arc-is-mini",
127
+ pretty_name="ARC-is",
128
+ source="EuroEval/arc-is-mini",
137
129
  task=KNOW,
138
- languages=[IS],
130
+ languages=[ICELANDIC],
139
131
  unofficial=True,
140
132
  )
141
133
 
142
134
  HELLASWAG_IS_CONFIG = DatasetConfig(
143
135
  name="hellaswag-is",
144
- pretty_name="the truncated version of the Icelandic common-sense reasoning dataset "
145
- "HellaSwag-is, translated from the English HellaSwag dataset",
146
- huggingface_id="EuroEval/hellaswag-is-mini",
136
+ pretty_name="HellaSwag-is",
137
+ source="EuroEval/hellaswag-is-mini",
147
138
  task=COMMON_SENSE,
148
- languages=[IS],
139
+ languages=[ICELANDIC],
149
140
  unofficial=True,
150
141
  )
151
142
 
152
143
  BELEBELE_IS_CONFIG = DatasetConfig(
153
144
  name="belebele-is",
154
- pretty_name="the Icelandic multiple choice reading comprehension dataset "
155
- "BeleBele-is, translated from the English BeleBele dataset",
156
- huggingface_id="EuroEval/belebele-is-mini",
145
+ pretty_name="Belebele-is",
146
+ source="EuroEval/belebele-is-mini",
157
147
  task=MCRC,
158
- languages=[IS],
148
+ languages=[ICELANDIC],
159
149
  unofficial=True,
160
150
  )
161
151
 
162
152
  MULTI_WIKI_QA_IS_CONFIG = DatasetConfig(
163
153
  name="multi-wiki-qa-is",
164
- pretty_name="the truncated version of the Icelandic part of the reading "
165
- "comprehension dataset MultiWikiQA",
166
- huggingface_id="EuroEval/multi-wiki-qa-is-mini",
154
+ pretty_name="MultiWikiQA-is",
155
+ source="EuroEval/multi-wiki-qa-is-mini",
167
156
  task=RC,
168
- languages=[IS],
169
- unofficial=True,
170
- )
171
-
172
- EUROPEAN_VALUES_SITUATIONAL_IS_CONFIG = DatasetConfig(
173
- name="european-values-situational-is",
174
- pretty_name="the Icelandic version of the European values evaluation dataset, "
175
- "where the questions are phrased in a situational way",
176
- huggingface_id="EuroEval/european-values-situational-is",
177
- task=EUROPEAN_VALUES,
178
- languages=[IS],
179
- splits=["test"],
180
- bootstrap_samples=False,
181
- _instruction_prompt="{text}",
182
- unofficial=True,
183
- )
184
-
185
- EUROPEAN_VALUES_COMPLETIONS_IS_CONFIG = DatasetConfig(
186
- name="european-values-completions-is",
187
- pretty_name="the Icelandic version of the European values evaluation dataset, "
188
- "where the questions are phrased as sentence completions",
189
- huggingface_id="EuroEval/european-values-completions-is",
190
- task=EUROPEAN_VALUES,
191
- languages=[IS],
192
- splits=["test"],
193
- bootstrap_samples=False,
194
- _instruction_prompt="{text}",
157
+ languages=[ICELANDIC],
195
158
  unofficial=True,
196
159
  )
@@ -1,78 +1,73 @@
1
1
  """All Italian dataset configurations used in EuroEval."""
2
2
 
3
3
  from ..data_models import DatasetConfig
4
- from ..languages import IT
4
+ from ..languages import ITALIAN
5
5
  from ..tasks import COMMON_SENSE, EUROPEAN_VALUES, KNOW, LA, MCRC, NER, RC, SENT, SUMM
6
6
 
7
7
  ### Official datasets ###
8
8
 
9
9
  SENTIPOLC_CONFIG = DatasetConfig(
10
10
  name="sentipolc16",
11
- pretty_name="the truncated version of the Italian sentiment classification "
12
- "dataset Sentipolc-16",
13
- huggingface_id="EuroEval/sentipolc16-mini",
11
+ pretty_name="Sentipolc16",
12
+ source="EuroEval/sentipolc16-mini",
14
13
  task=SENT,
15
- languages=[IT],
14
+ languages=[ITALIAN],
16
15
  )
17
16
 
18
17
  SCALA_IT_CONFIG = DatasetConfig(
19
18
  name="scala-it",
20
- pretty_name="the Italian part of the linguistic acceptability dataset ScaLA",
21
- huggingface_id="EuroEval/scala-it",
19
+ pretty_name="ScaLA-it",
20
+ source="EuroEval/scala-it",
22
21
  task=LA,
23
- languages=[IT],
22
+ languages=[ITALIAN],
24
23
  )
25
24
 
26
25
  MULTINERD_IT_CONFIG = DatasetConfig(
27
26
  name="multinerd-it",
28
- pretty_name="the truncated version of the Italian part of the named "
29
- "entity recognition dataset MultiNERD",
30
- huggingface_id="EuroEval/multinerd-mini-it",
27
+ pretty_name="MultiNERD-it",
28
+ source="EuroEval/multinerd-mini-it",
31
29
  task=NER,
32
- languages=[IT],
30
+ languages=[ITALIAN],
33
31
  )
34
32
 
35
33
  SQUAD_IT_CONFIG = DatasetConfig(
36
34
  name="squad-it",
37
- pretty_name="the truncated version of the Italian reading comprehension dataset "
38
- "SQuAD-it, translated from the English SQuAD dataset",
39
- huggingface_id="EuroEval/squad-it-mini",
35
+ pretty_name="SQuAD-it",
36
+ source="EuroEval/squad-it-mini",
40
37
  task=RC,
41
- languages=[IT],
38
+ languages=[ITALIAN],
42
39
  )
43
40
 
44
41
  ILPOST_SUM_CONFIG = DatasetConfig(
45
42
  name="ilpost-sum",
46
- pretty_name="the truncated version of the Italian summarisation dataset IlPost-Sum",
47
- huggingface_id="EuroEval/ilpost-sum",
43
+ pretty_name="IlPost-Sum",
44
+ source="EuroEval/ilpost-sum",
48
45
  task=SUMM,
49
- languages=[IT],
46
+ languages=[ITALIAN],
50
47
  )
51
48
 
52
49
  MMLU_IT_CONFIG = DatasetConfig(
53
50
  name="mmlu-it",
54
- pretty_name="the truncated version of the Italian knowledge dataset MMLU-it, "
55
- "translated from the English MMLU dataset",
56
- huggingface_id="EuroEval/mmlu-it-mini",
51
+ pretty_name="MMLU-it",
52
+ source="EuroEval/mmlu-it-mini",
57
53
  task=KNOW,
58
- languages=[IT],
54
+ languages=[ITALIAN],
59
55
  )
60
56
 
61
57
  HELLASWAG_IT_CONFIG = DatasetConfig(
62
58
  name="hellaswag-it",
63
- pretty_name="the truncated version of the Italian common-sense reasoning dataset "
64
- "HellaSwag-it, translated from the English HellaSwag dataset",
65
- huggingface_id="EuroEval/hellaswag-it-mini",
59
+ pretty_name="HellaSwag-it",
60
+ source="EuroEval/hellaswag-it-mini",
66
61
  task=COMMON_SENSE,
67
- languages=[IT],
62
+ languages=[ITALIAN],
68
63
  )
69
64
 
70
- EUROPEAN_VALUES_IT_CONFIG = DatasetConfig(
71
- name="european-values-it",
72
- pretty_name="the Italian version of the European values evaluation dataset",
73
- huggingface_id="EuroEval/european-values-it",
65
+ VALEU_IT_CONFIG = DatasetConfig(
66
+ name="valeu-it",
67
+ pretty_name="VaLEU-it",
68
+ source="EuroEval/european-values-it",
74
69
  task=EUROPEAN_VALUES,
75
- languages=[IT],
70
+ languages=[ITALIAN],
76
71
  splits=["test"],
77
72
  bootstrap_samples=False,
78
73
  _instruction_prompt="{text}",
@@ -83,77 +78,46 @@ EUROPEAN_VALUES_IT_CONFIG = DatasetConfig(
83
78
 
84
79
  WIKINEURAL_IT_CONFIG = DatasetConfig(
85
80
  name="wikineural-it",
86
- pretty_name="the truncated version of the Italian named "
87
- "entity recognition dataset WikiNEuRal IT",
88
- huggingface_id="EuroEval/wikineural-mini-it",
81
+ pretty_name="WikiNeural-it",
82
+ source="EuroEval/wikineural-mini-it",
89
83
  task=NER,
90
- languages=[IT],
84
+ languages=[ITALIAN],
91
85
  unofficial=True,
92
86
  )
93
87
 
94
88
  BELEBELE_IT_CONFIG = DatasetConfig(
95
89
  name="belebele-it",
96
- pretty_name="the Italian multiple choice reading comprehension dataset "
97
- "BeleBele-it, translated from the English BeleBele dataset",
98
- huggingface_id="EuroEval/belebele-it-mini",
90
+ pretty_name="Belebele-it",
91
+ source="EuroEval/belebele-it-mini",
99
92
  task=MCRC,
100
- languages=[IT],
93
+ languages=[ITALIAN],
101
94
  unofficial=True,
102
95
  )
103
96
 
104
97
  MULTI_WIKI_QA_IT_CONFIG = DatasetConfig(
105
98
  name="multi-wiki-qa-it",
106
- pretty_name="the truncated version of the Italian part of the reading "
107
- "comprehension dataset MultiWikiQA",
108
- huggingface_id="EuroEval/multi-wiki-qa-it-mini",
99
+ pretty_name="MultiWikiQA-it",
100
+ source="EuroEval/multi-wiki-qa-it-mini",
109
101
  task=RC,
110
- languages=[IT],
102
+ languages=[ITALIAN],
111
103
  unofficial=True,
112
104
  )
113
105
 
114
106
  GOLDENSWAG_IT_CONFIG = DatasetConfig(
115
107
  name="goldenswag-it",
116
- pretty_name="the truncated version of the Italian common-sense reasoning "
117
- "dataset GoldenSwag-it, translated from the English GoldenSwag dataset",
118
- huggingface_id="EuroEval/goldenswag-it-mini",
108
+ pretty_name="GoldenSwag-it",
109
+ source="EuroEval/goldenswag-it-mini",
119
110
  task=COMMON_SENSE,
120
- languages=[IT],
111
+ languages=[ITALIAN],
121
112
  unofficial=True,
122
113
  )
123
114
 
124
115
  WINOGRANDE_IT_CONFIG = DatasetConfig(
125
116
  name="winogrande-it",
126
- pretty_name="the Italian common-sense reasoning dataset Winogrande-it, translated "
127
- "from the English Winogrande dataset",
128
- huggingface_id="EuroEval/winogrande-it",
117
+ pretty_name="Winogrande-it",
118
+ source="EuroEval/winogrande-it",
129
119
  task=COMMON_SENSE,
130
- languages=[IT],
120
+ languages=[ITALIAN],
131
121
  _labels=["a", "b"],
132
122
  unofficial=True,
133
123
  )
134
-
135
- EUROPEAN_VALUES_SITUATIONAL_IT_CONFIG = DatasetConfig(
136
- name="european-values-situational-it",
137
- pretty_name="the Italian version of the European values evaluation dataset, "
138
- "where the questions are phrased in a situational way",
139
- huggingface_id="EuroEval/european-values-situational-it",
140
- task=EUROPEAN_VALUES,
141
- languages=[IT],
142
- splits=["test"],
143
- bootstrap_samples=False,
144
- _instruction_prompt="{text}",
145
- unofficial=True,
146
- )
147
-
148
- EUROPEAN_VALUES_COMPLETIONS_IT_CONFIG = DatasetConfig(
149
- name="european-values-completions-it",
150
- pretty_name="the Italian version of the European values evaluation dataset, where "
151
- "the questions are phrased as sentence completions",
152
- huggingface_id="EuroEval/european-values-completions-it",
153
- task=EUROPEAN_VALUES,
154
- languages=[IT],
155
- splits=["test"],
156
- bootstrap_samples=False,
157
- _instruction_prompt="{text}",
158
- unofficial=True,
159
- )
@@ -1,70 +1,66 @@
1
1
  """All Latvian dataset configurations used in EuroEval."""
2
2
 
3
3
  from ..data_models import DatasetConfig
4
- from ..languages import LV
4
+ from ..languages import LATVIAN
5
5
  from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
6
6
 
7
7
  ### Official datasets ###
8
8
 
9
9
  LATVIAN_TWITTER_SENTIMENT_CONFIG = DatasetConfig(
10
10
  name="latvian-twitter-sentiment",
11
- pretty_name="the truncated version of the Latvian sentiment classification dataset",
12
- huggingface_id="EuroEval/latvian-twitter-sentiment-mini",
11
+ pretty_name="Latvian Twitter Sentiment",
12
+ source="EuroEval/latvian-twitter-sentiment-mini",
13
13
  task=SENT,
14
- languages=[LV],
14
+ languages=[LATVIAN],
15
15
  )
16
16
 
17
17
  SCALA_LV_CONFIG = DatasetConfig(
18
18
  name="scala-lv",
19
- pretty_name="the Latvian part of the linguistic acceptability dataset ScaLA",
20
- huggingface_id="EuroEval/scala-lv",
19
+ pretty_name="ScaLA-lv",
20
+ source="EuroEval/scala-lv",
21
21
  task=LA,
22
- languages=[LV],
22
+ languages=[LATVIAN],
23
23
  )
24
24
 
25
25
  FULLSTACK_NER_LV_CONFIG = DatasetConfig(
26
26
  name="fullstack-ner-lv",
27
- pretty_name="the truncated version of the Latvian named entity recognition "
28
- "dataset FullStack-NER-lv",
29
- huggingface_id="EuroEval/fullstack-ner-lv-mini",
27
+ pretty_name="FullStack NER-lv",
28
+ source="EuroEval/fullstack-ner-lv-mini",
30
29
  task=NER,
31
- languages=[LV],
30
+ languages=[LATVIAN],
32
31
  )
33
32
 
34
33
  MULTI_WIKI_QA_LV_CONFIG = DatasetConfig(
35
34
  name="multi-wiki-qa-lv",
36
- pretty_name="the truncated version of the Latvian part of the reading "
37
- "comprehension dataset MultiWikiQA",
38
- huggingface_id="EuroEval/multi-wiki-qa-lv-mini",
35
+ pretty_name="MultiWikiQA-lv",
36
+ source="EuroEval/multi-wiki-qa-lv-mini",
39
37
  task=RC,
40
- languages=[LV],
38
+ languages=[LATVIAN],
41
39
  )
42
40
 
43
41
  LSM_CONFIG = DatasetConfig(
44
42
  name="lsm",
45
- pretty_name="the truncated version of the Latvian summarisation dataset LSM",
46
- huggingface_id="EuroEval/lsm-mini",
43
+ pretty_name="LSM",
44
+ source="EuroEval/lsm-mini",
47
45
  task=SUMM,
48
- languages=[LV],
46
+ languages=[LATVIAN],
49
47
  )
50
48
 
51
49
 
52
50
  MMLU_LV_CONFIG = DatasetConfig(
53
51
  name="mmlu-lv",
54
- pretty_name="the truncated version of the Latvian knowledge dataset MMLU-lv, "
55
- "translated from the English MMLU dataset",
56
- huggingface_id="EuroEval/mmlu-lv-mini",
52
+ pretty_name="MMLU-lv",
53
+ source="EuroEval/mmlu-lv-mini",
57
54
  task=KNOW,
58
- languages=[LV],
55
+ languages=[LATVIAN],
59
56
  )
60
57
 
61
58
  COPA_LV_CONFIG = DatasetConfig(
62
59
  name="copa-lv",
63
- pretty_name="the Latvian common-sense reasoning dataset COPA-lv, translated from "
64
- "the English COPA dataset",
65
- huggingface_id="EuroEval/copa-lv",
60
+ pretty_name="COPA-lv",
61
+ source="EuroEval/copa-lv",
66
62
  task=COMMON_SENSE,
67
- languages=[LV],
63
+ languages=[LATVIAN],
68
64
  _labels=["a", "b"],
69
65
  )
70
66
 
@@ -73,21 +69,19 @@ COPA_LV_CONFIG = DatasetConfig(
73
69
 
74
70
  WIKIANN_LV_CONFIG = DatasetConfig(
75
71
  name="wikiann-lv",
76
- pretty_name="the truncated version of the Latvian part of the named entity "
77
- "recognition dataset WikiANN",
78
- huggingface_id="EuroEval/wikiann-lv-mini",
72
+ pretty_name="WikiANN-lv",
73
+ source="EuroEval/wikiann-lv-mini",
79
74
  task=NER,
80
- languages=[LV],
75
+ languages=[LATVIAN],
81
76
  unofficial=True,
82
77
  )
83
78
 
84
79
  WINOGRANDE_LV_CONFIG = DatasetConfig(
85
80
  name="winogrande-lv",
86
- pretty_name="the Latvian common-sense reasoning dataset Winogrande-lv, translated "
87
- "from the English Winogrande dataset",
88
- huggingface_id="EuroEval/winogrande-lv",
81
+ pretty_name="Winogrande-lv",
82
+ source="EuroEval/winogrande-lv",
89
83
  task=COMMON_SENSE,
90
- languages=[LV],
84
+ languages=[LATVIAN],
91
85
  _labels=["a", "b"],
92
86
  unofficial=True,
93
87
  )
@@ -1,68 +1,64 @@
1
1
  """All Lithuanian dataset configurations used in EuroEval."""
2
2
 
3
3
  from ..data_models import DatasetConfig
4
- from ..languages import LT
4
+ from ..languages import LITHUANIAN
5
5
  from ..tasks import COMMON_SENSE, KNOW, LA, NER, RC, SENT, SUMM
6
6
 
7
7
  ### Official datasets ###
8
8
 
9
9
  LITHUANIAN_EMOTIONS_CONFIG = DatasetConfig(
10
10
  name="lithuanian-emotions",
11
- pretty_name="the truncated version of the Lithuanian sentiment "
12
- "classification dataset Lithuanian Emotions",
13
- huggingface_id="EuroEval/lithuanian-emotions-mini",
11
+ pretty_name="Lithuanian Emotions",
12
+ source="EuroEval/lithuanian-emotions-mini",
14
13
  task=SENT,
15
- languages=[LT],
14
+ languages=[LITHUANIAN],
16
15
  )
17
16
 
18
17
  SCALA_LT_CONFIG = DatasetConfig(
19
18
  name="scala-lt",
20
- pretty_name="the Lithuanian part of the linguistic acceptability dataset ScaLA",
21
- huggingface_id="EuroEval/scala-lt",
19
+ pretty_name="ScaLA-lt",
20
+ source="EuroEval/scala-lt",
22
21
  task=LA,
23
- languages=[LT],
22
+ languages=[LITHUANIAN],
24
23
  )
25
24
 
26
25
  WIKIANN_LT_CONFIG = DatasetConfig(
27
26
  name="wikiann-lt",
28
- pretty_name="the truncated version of the Lithuanian part of the named entity "
29
- "recognition dataset WikiANN",
30
- huggingface_id="EuroEval/wikiann-lt-mini",
27
+ pretty_name="WikiANN-lt",
28
+ source="EuroEval/wikiann-lt-mini",
31
29
  task=NER,
32
- languages=[LT],
30
+ languages=[LITHUANIAN],
33
31
  )
34
32
 
35
33
  MULTI_WIKI_QA_LT_CONFIG = DatasetConfig(
36
34
  name="multi-wiki-qa-lt",
37
- pretty_name="the truncated version of the Lithuanian part of the reading "
38
- "comprehension dataset MultiWikiQA",
39
- huggingface_id="EuroEval/multi-wiki-qa-lt-mini",
35
+ pretty_name="MultiWikiQA-lt",
36
+ source="EuroEval/multi-wiki-qa-lt-mini",
40
37
  task=RC,
41
- languages=[LT],
38
+ languages=[LITHUANIAN],
42
39
  )
43
40
 
44
41
  LRYTAS_CONFIG = DatasetConfig(
45
42
  name="lrytas",
46
- pretty_name="the truncated version of the Lithuanian summarisation dataset Lrytas",
47
- huggingface_id="EuroEval/lrytas-mini",
43
+ pretty_name="Lrytas",
44
+ source="EuroEval/lrytas-mini",
48
45
  task=SUMM,
49
- languages=[LT],
46
+ languages=[LITHUANIAN],
50
47
  )
51
48
 
52
49
  LT_HISTORY_CONFIG = DatasetConfig(
53
50
  name="lt-history",
54
- pretty_name="the Lithuanian knowledge dataset LT-History",
55
- huggingface_id="EuroEval/lt-history",
51
+ pretty_name="LT-History",
52
+ source="EuroEval/lt-history",
56
53
  task=KNOW,
57
- languages=[LT],
54
+ languages=[LITHUANIAN],
58
55
  )
59
56
 
60
57
  WINOGRANDE_LT_CONFIG = DatasetConfig(
61
58
  name="winogrande-lt",
62
- pretty_name="the Lithuanian common-sense reasoning dataset Winogrande-lt, "
63
- "translated from the English Winogrande dataset",
64
- huggingface_id="EuroEval/winogrande-lt",
59
+ pretty_name="Winogrande-lt",
60
+ source="EuroEval/winogrande-lt",
65
61
  task=COMMON_SENSE,
66
- languages=[LT],
62
+ languages=[LITHUANIAN],
67
63
  _labels=["a", "b"],
68
64
  )