EuroEval 15.12.0__py3-none-any.whl → 16.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- euroeval/__init__.py +32 -14
- euroeval/benchmark_config_factory.py +92 -180
- euroeval/benchmark_modules/base.py +49 -39
- euroeval/benchmark_modules/fresh.py +35 -21
- euroeval/benchmark_modules/hf.py +280 -244
- euroeval/benchmark_modules/litellm.py +752 -312
- euroeval/benchmark_modules/vllm.py +570 -268
- euroeval/benchmarker.py +651 -528
- euroeval/caching_utils.py +79 -0
- euroeval/callbacks.py +5 -7
- euroeval/cli.py +49 -38
- euroeval/constants.py +44 -25
- euroeval/data_loading.py +111 -55
- euroeval/data_models.py +490 -323
- euroeval/dataset_configs/__init__.py +26 -4
- euroeval/dataset_configs/bosnian.py +39 -0
- euroeval/dataset_configs/bulgarian.py +56 -0
- euroeval/dataset_configs/croatian.py +56 -0
- euroeval/dataset_configs/czech.py +75 -0
- euroeval/dataset_configs/danish.py +78 -50
- euroeval/dataset_configs/dutch.py +74 -44
- euroeval/dataset_configs/english.py +71 -36
- euroeval/dataset_configs/estonian.py +111 -0
- euroeval/dataset_configs/faroese.py +25 -18
- euroeval/dataset_configs/finnish.py +63 -26
- euroeval/dataset_configs/french.py +65 -32
- euroeval/dataset_configs/german.py +77 -36
- euroeval/dataset_configs/greek.py +64 -0
- euroeval/dataset_configs/icelandic.py +68 -57
- euroeval/dataset_configs/italian.py +68 -36
- euroeval/dataset_configs/latvian.py +87 -0
- euroeval/dataset_configs/lithuanian.py +64 -0
- euroeval/dataset_configs/norwegian.py +98 -72
- euroeval/dataset_configs/polish.py +96 -0
- euroeval/dataset_configs/portuguese.py +63 -40
- euroeval/dataset_configs/serbian.py +64 -0
- euroeval/dataset_configs/slovak.py +55 -0
- euroeval/dataset_configs/slovene.py +56 -0
- euroeval/dataset_configs/spanish.py +68 -34
- euroeval/dataset_configs/swedish.py +82 -41
- euroeval/dataset_configs/ukrainian.py +64 -0
- euroeval/enums.py +12 -6
- euroeval/exceptions.py +21 -1
- euroeval/finetuning.py +34 -26
- euroeval/generation.py +76 -41
- euroeval/generation_utils.py +169 -34
- euroeval/languages.py +1020 -188
- euroeval/logging_utils.py +268 -0
- euroeval/metrics/__init__.py +6 -0
- euroeval/metrics/base.py +85 -0
- euroeval/metrics/huggingface.py +216 -0
- euroeval/metrics/llm_as_a_judge.py +260 -0
- euroeval/metrics/pipeline.py +289 -0
- euroeval/metrics/speed.py +48 -0
- euroeval/model_cache.py +40 -21
- euroeval/model_config.py +4 -5
- euroeval/model_loading.py +3 -0
- euroeval/prompt_templates/__init__.py +2 -0
- euroeval/prompt_templates/classification.py +206 -0
- euroeval/prompt_templates/linguistic_acceptability.py +157 -22
- euroeval/prompt_templates/multiple_choice.py +159 -17
- euroeval/prompt_templates/named_entity_recognition.py +318 -21
- euroeval/prompt_templates/reading_comprehension.py +207 -16
- euroeval/prompt_templates/sentiment_classification.py +205 -22
- euroeval/prompt_templates/summarization.py +122 -22
- euroeval/prompt_templates/token_classification.py +279 -0
- euroeval/scores.py +20 -9
- euroeval/speed_benchmark.py +11 -12
- euroeval/task_group_utils/multiple_choice_classification.py +21 -12
- euroeval/task_group_utils/question_answering.py +101 -73
- euroeval/task_group_utils/sequence_classification.py +144 -61
- euroeval/task_group_utils/text_to_text.py +33 -12
- euroeval/task_group_utils/token_classification.py +86 -89
- euroeval/tasks.py +75 -16
- euroeval/tokenisation_utils.py +603 -0
- euroeval/types.py +17 -11
- euroeval/utils.py +332 -137
- euroeval-16.7.1.dist-info/METADATA +623 -0
- euroeval-16.7.1.dist-info/RECORD +84 -0
- {euroeval-15.12.0.dist-info → euroeval-16.7.1.dist-info}/entry_points.txt +0 -1
- euroeval/human_evaluation.py +0 -737
- euroeval/metrics.py +0 -452
- euroeval/tokenization_utils.py +0 -498
- euroeval-15.12.0.dist-info/METADATA +0 -285
- euroeval-15.12.0.dist-info/RECORD +0 -63
- {euroeval-15.12.0.dist-info → euroeval-16.7.1.dist-info}/WHEEL +0 -0
- {euroeval-15.12.0.dist-info → euroeval-16.7.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,17 +1,74 @@
|
|
|
1
1
|
"""Templates for the Linguistic Acceptability task."""
|
|
2
2
|
|
|
3
|
+
import typing as t
|
|
4
|
+
|
|
3
5
|
from ..data_models import PromptConfig
|
|
4
|
-
from ..languages import
|
|
6
|
+
from ..languages import (
|
|
7
|
+
BULGARIAN,
|
|
8
|
+
CROATIAN,
|
|
9
|
+
CZECH,
|
|
10
|
+
DANISH,
|
|
11
|
+
DUTCH,
|
|
12
|
+
ENGLISH,
|
|
13
|
+
ESTONIAN,
|
|
14
|
+
FAROESE,
|
|
15
|
+
FINNISH,
|
|
16
|
+
FRENCH,
|
|
17
|
+
GERMAN,
|
|
18
|
+
GREEK,
|
|
19
|
+
ICELANDIC,
|
|
20
|
+
ITALIAN,
|
|
21
|
+
LATVIAN,
|
|
22
|
+
LITHUANIAN,
|
|
23
|
+
NORWEGIAN,
|
|
24
|
+
NORWEGIAN_BOKMÅL,
|
|
25
|
+
NORWEGIAN_NYNORSK,
|
|
26
|
+
POLISH,
|
|
27
|
+
PORTUGUESE,
|
|
28
|
+
SERBIAN,
|
|
29
|
+
SLOVAK,
|
|
30
|
+
SLOVENE,
|
|
31
|
+
SPANISH,
|
|
32
|
+
SWEDISH,
|
|
33
|
+
UKRAINIAN,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
if t.TYPE_CHECKING:
|
|
37
|
+
from ..languages import Language
|
|
5
38
|
|
|
6
|
-
LA_TEMPLATES = {
|
|
7
|
-
|
|
39
|
+
LA_TEMPLATES: dict["Language", PromptConfig] = {
|
|
40
|
+
BULGARIAN: PromptConfig(
|
|
41
|
+
default_prompt_label_mapping=dict(correct="да", incorrect="не"),
|
|
42
|
+
default_prompt_prefix="Следват изречения и дали са граматически правилни.",
|
|
43
|
+
default_prompt_template="Изречение: {text}\nГраматически правилно: {label}",
|
|
44
|
+
default_instruction_prompt="Изречение: {text}\n\nОпределете дали изречението е "
|
|
45
|
+
"граматически правилно или не. Отговорете с {labels_str}, и нищо друго.",
|
|
46
|
+
),
|
|
47
|
+
CROATIAN: PromptConfig(
|
|
48
|
+
default_prompt_label_mapping=dict(correct="da", incorrect="ne"),
|
|
49
|
+
default_prompt_prefix="Sljedeće su rečenice i jesu li gramatički ispravne.",
|
|
50
|
+
default_prompt_template="Rečenica: {text}\nGramatički ispravna: {label}",
|
|
51
|
+
default_instruction_prompt=(
|
|
52
|
+
"Rečenica: {text}\n\n"
|
|
53
|
+
"Odredite je li rečenica gramatički ispravna ili ne. "
|
|
54
|
+
"Odgovorite s {labels_str}, i ništa drugo."
|
|
55
|
+
),
|
|
56
|
+
),
|
|
57
|
+
CZECH: PromptConfig(
|
|
58
|
+
default_prompt_label_mapping=dict(correct="ano", incorrect="ne"),
|
|
59
|
+
default_prompt_prefix="Následující jsou věty a zda jsou gramaticky správné.",
|
|
60
|
+
default_prompt_template="Věta: {text}\nGramaticky správná: {label}",
|
|
61
|
+
default_instruction_prompt="Věta: {text}\n\nUrčete, zda je věta gramaticky "
|
|
62
|
+
"správná nebo ne. Odpovězte {labels_str}, a nic jiné.",
|
|
63
|
+
),
|
|
64
|
+
DANISH: PromptConfig(
|
|
8
65
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nej"),
|
|
9
66
|
default_prompt_prefix="Følgende er sætninger og om de er grammatisk korrekte.",
|
|
10
67
|
default_prompt_template="Sætning: {text}\nGrammatisk korrekt: {label}",
|
|
11
68
|
default_instruction_prompt="Sætning: {text}\n\nBestem om sætningen er "
|
|
12
69
|
"grammatisk korrekt eller ej. Svar kun med {labels_str}, og intet andet.",
|
|
13
70
|
),
|
|
14
|
-
|
|
71
|
+
GERMAN: PromptConfig(
|
|
15
72
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nein"),
|
|
16
73
|
default_prompt_prefix="Die folgenden Sätze und ob sie grammatikalisch korrekt "
|
|
17
74
|
"sind.",
|
|
@@ -20,7 +77,15 @@ LA_TEMPLATES = {
|
|
|
20
77
|
"grammatikalisch korrekt ist oder nicht. Antworten Sie mit {labels_str}, und "
|
|
21
78
|
"nichts anderes.",
|
|
22
79
|
),
|
|
23
|
-
|
|
80
|
+
GREEK: PromptConfig(
|
|
81
|
+
default_prompt_label_mapping=dict(correct="ναι", incorrect="όχι"),
|
|
82
|
+
default_prompt_prefix="Οι ακόλουθες είναι προτάσεις και εάν είναι "
|
|
83
|
+
"γραμματικά σωστές.",
|
|
84
|
+
default_prompt_template="Πρόταση: {text}\nΓραμματικά σωστή: {label}",
|
|
85
|
+
default_instruction_prompt="Πρόταση: {text}\n\nΠροσδιορίστε εάν η πρόταση "
|
|
86
|
+
"είναι γραμματικά σωστή ή όχι. Απαντήστε με {labels_str}, και τίποτα άλλο.",
|
|
87
|
+
),
|
|
88
|
+
ENGLISH: PromptConfig(
|
|
24
89
|
default_prompt_label_mapping=dict(correct="yes", incorrect="no"),
|
|
25
90
|
default_prompt_prefix="The following are sentences and whether they are "
|
|
26
91
|
"grammatically correct.",
|
|
@@ -28,7 +93,7 @@ LA_TEMPLATES = {
|
|
|
28
93
|
default_instruction_prompt="Sentence: {text}\n\nDetermine whether the sentence "
|
|
29
94
|
"is grammatically correct or not. Answer with {labels_str}, and nothing else.",
|
|
30
95
|
),
|
|
31
|
-
|
|
96
|
+
SPANISH: PromptConfig(
|
|
32
97
|
default_prompt_label_mapping=dict(correct="sí", incorrect="no"),
|
|
33
98
|
default_prompt_prefix="Lo siguiente son textos y si son gramaticalmente "
|
|
34
99
|
"correctos.",
|
|
@@ -36,7 +101,23 @@ LA_TEMPLATES = {
|
|
|
36
101
|
default_instruction_prompt="Texto: {text}\n\nDetermina si el texto es "
|
|
37
102
|
"gramaticalmente correcto o no. Responde con {labels_str}, y nada más.",
|
|
38
103
|
),
|
|
39
|
-
|
|
104
|
+
ESTONIAN: PromptConfig(
|
|
105
|
+
default_prompt_label_mapping=dict(correct="jah", incorrect="ei"),
|
|
106
|
+
default_prompt_prefix="Järgnevad on laused ja kas need on grammatiliselt "
|
|
107
|
+
"õiged.",
|
|
108
|
+
default_prompt_template="Lause: {text}\nGrammatikaliselt õige: {label}",
|
|
109
|
+
default_instruction_prompt="Lause: {text}\n\nOtsusta, kas lause on "
|
|
110
|
+
"grammatiliselt õige või mitte. Vasta {labels_str}, ja mitte midagi muud.",
|
|
111
|
+
),
|
|
112
|
+
POLISH: PromptConfig(
|
|
113
|
+
default_prompt_label_mapping=dict(correct="tak", incorrect="nie"),
|
|
114
|
+
default_prompt_prefix="Poniżej znajdują się teksty i informacja, czy są "
|
|
115
|
+
"gramatycznie poprawne.",
|
|
116
|
+
default_prompt_template="Tekst: {text}\nGramatycznie poprawny: {label}",
|
|
117
|
+
default_instruction_prompt="Tekst: {text}\n\nOkreśl, czy tekst jest "
|
|
118
|
+
"gramatycznie poprawny. Odpowiedz używając wyłącznie {labels_str}.",
|
|
119
|
+
),
|
|
120
|
+
PORTUGUESE: PromptConfig(
|
|
40
121
|
default_prompt_label_mapping=dict(correct="sim", incorrect="não"),
|
|
41
122
|
default_prompt_prefix="Seguem-se abaixo textos e se são "
|
|
42
123
|
"gramaticalmente correctos",
|
|
@@ -44,7 +125,7 @@ LA_TEMPLATES = {
|
|
|
44
125
|
default_instruction_prompt="Texto: {text}\n\nDetermina se o texto é "
|
|
45
126
|
"gramaticalmente correcto ou não. Responde com {labels_str}, e nada mais.",
|
|
46
127
|
),
|
|
47
|
-
|
|
128
|
+
FINNISH: PromptConfig(
|
|
48
129
|
default_prompt_label_mapping=dict(correct="kyllä", incorrect="ei"),
|
|
49
130
|
default_prompt_prefix="Seuraavat ovat lauseita ja ovatko ne "
|
|
50
131
|
"kieliopillisesti oikein.",
|
|
@@ -52,15 +133,15 @@ LA_TEMPLATES = {
|
|
|
52
133
|
default_instruction_prompt="Lause: {text}\n\nMääritä onko lause "
|
|
53
134
|
"oikein vai ei. Vastaa {labels_str}, ja ei mitään muuta.",
|
|
54
135
|
),
|
|
55
|
-
|
|
136
|
+
FAROESE: PromptConfig(
|
|
56
137
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nei"),
|
|
57
138
|
default_prompt_prefix="Hetta eru nakrir setningar og um teir eru mállæruliga "
|
|
58
139
|
"rættir.",
|
|
59
140
|
default_prompt_template="Setningur: {text}\nMállæruliga rættur: {label}",
|
|
60
|
-
default_instruction_prompt="Setningur: {text}\n\
|
|
141
|
+
default_instruction_prompt="Setningur: {text}\n\nGreindu hvort setningurin er "
|
|
61
142
|
"mállæruliga rættur ella ikki. Svara við {labels_str}, og einki annað.",
|
|
62
143
|
),
|
|
63
|
-
|
|
144
|
+
FRENCH: PromptConfig(
|
|
64
145
|
default_prompt_label_mapping=dict(correct="oui", incorrect="non"),
|
|
65
146
|
default_prompt_prefix="Les phrases suivantes indiquent si elles sont "
|
|
66
147
|
"grammaticalement correctes.",
|
|
@@ -69,15 +150,16 @@ LA_TEMPLATES = {
|
|
|
69
150
|
default_instruction_prompt="Phrase: {text}\n\nDéterminez si la phrase est "
|
|
70
151
|
"grammaticalement correcte ou non. Répondez par {labels_str}, et rien d'autre.",
|
|
71
152
|
),
|
|
72
|
-
|
|
153
|
+
ICELANDIC: PromptConfig(
|
|
73
154
|
default_prompt_label_mapping=dict(correct="já", incorrect="nei"),
|
|
74
|
-
default_prompt_prefix="
|
|
75
|
-
"málfræðilega réttar.",
|
|
155
|
+
default_prompt_prefix="Hér fyrir neðan eru setningar ásamt mati á því hvort "
|
|
156
|
+
"þær eru málfræðilega réttar.",
|
|
76
157
|
default_prompt_template="Setning: {text}\nMálfræðilega rétt: {label}",
|
|
77
|
-
default_instruction_prompt="Setning: {text}\n\
|
|
78
|
-
"málfræðilega rétt
|
|
158
|
+
default_instruction_prompt="Setning: {text}\n\nGreindu hvort setningin er "
|
|
159
|
+
"málfræðilega rétt. Svaraðu með 'já' ef setningin er rétt og 'nei' ef hún "
|
|
160
|
+
"er það ekki.",
|
|
79
161
|
),
|
|
80
|
-
|
|
162
|
+
ITALIAN: PromptConfig(
|
|
81
163
|
default_prompt_label_mapping=dict(correct="si", incorrect="no"),
|
|
82
164
|
default_prompt_prefix="Di seguito sono riportate le frasi e la loro "
|
|
83
165
|
"correttezza grammaticale.",
|
|
@@ -85,7 +167,22 @@ LA_TEMPLATES = {
|
|
|
85
167
|
default_instruction_prompt="Frase: {text}\n\nStabilite se la frase è "
|
|
86
168
|
"grammaticalmente corretta o meno. Rispondere con {labels_str}, e nient'altro.",
|
|
87
169
|
),
|
|
88
|
-
|
|
170
|
+
LITHUANIAN: PromptConfig(
|
|
171
|
+
default_prompt_label_mapping=dict(correct="taip", incorrect="ne"),
|
|
172
|
+
default_prompt_prefix="Toliau pateikti sakiniai ir ar jie yra gramatiškai "
|
|
173
|
+
"teisingi.",
|
|
174
|
+
default_prompt_template="Sakinys: {text}\nGramatiškai teisingas: {label}",
|
|
175
|
+
default_instruction_prompt="Sakinys: {text}\n\nNustatykite, ar sakinys yra "
|
|
176
|
+
"gramatiškai teisingas, ar ne. Atsakykite su {labels_str}, ir nieko kito.",
|
|
177
|
+
),
|
|
178
|
+
LATVIAN: PromptConfig(
|
|
179
|
+
default_prompt_label_mapping=dict(correct="jā", incorrect="nē"),
|
|
180
|
+
default_prompt_prefix="Šie ir teikumi un to gramatiskie pareizumi.",
|
|
181
|
+
default_prompt_template="Teikums: {text}\nGramatiski pareizs: {label}",
|
|
182
|
+
default_instruction_prompt="Teikums: {text}\n\nNoteiciet, vai teikums ir "
|
|
183
|
+
"gramatiski pareizs vai nē. Atbildiet ar {labels_str}, un neko citu.",
|
|
184
|
+
),
|
|
185
|
+
NORWEGIAN_BOKMÅL: PromptConfig(
|
|
89
186
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nei"),
|
|
90
187
|
default_prompt_prefix="Følgende er setninger og hvorvidt de er grammatisk "
|
|
91
188
|
"korrekte.",
|
|
@@ -93,7 +190,7 @@ LA_TEMPLATES = {
|
|
|
93
190
|
default_instruction_prompt="Setning: {text}\n\nBestem om setningen er "
|
|
94
191
|
"grammatisk korrekt eller ikke. Svar med {labels_str}, og ikke noe annet.",
|
|
95
192
|
),
|
|
96
|
-
|
|
193
|
+
DUTCH: PromptConfig(
|
|
97
194
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nee"),
|
|
98
195
|
default_prompt_prefix="Hieronder staan zinnen en of ze grammaticaal correct "
|
|
99
196
|
"zijn.",
|
|
@@ -101,7 +198,7 @@ LA_TEMPLATES = {
|
|
|
101
198
|
default_instruction_prompt="Zin: {text}\n\nBepaal of de zin grammaticaal "
|
|
102
199
|
"correct is of niet. Antwoord met {labels_str}, en verder niets.",
|
|
103
200
|
),
|
|
104
|
-
|
|
201
|
+
NORWEGIAN_NYNORSK: PromptConfig(
|
|
105
202
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nei"),
|
|
106
203
|
default_prompt_prefix="Følgende er setninger og hvorvidt de er grammatisk "
|
|
107
204
|
"korrekte.",
|
|
@@ -109,7 +206,7 @@ LA_TEMPLATES = {
|
|
|
109
206
|
default_instruction_prompt="Setning: {text}\n\nBestem om setningen er "
|
|
110
207
|
"grammatisk korrekt eller ikke. Svar med {labels_str}, og ikke noe annet.",
|
|
111
208
|
),
|
|
112
|
-
|
|
209
|
+
NORWEGIAN: PromptConfig(
|
|
113
210
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nei"),
|
|
114
211
|
default_prompt_prefix="Følgende er setninger og hvorvidt de er grammatisk "
|
|
115
212
|
"korrekte.",
|
|
@@ -117,7 +214,34 @@ LA_TEMPLATES = {
|
|
|
117
214
|
default_instruction_prompt="Setning: {text}\n\nBestem om setningen er "
|
|
118
215
|
"grammatisk korrekt eller ikke. Svar med {labels_str}, og ikke noe annet.",
|
|
119
216
|
),
|
|
120
|
-
|
|
217
|
+
SLOVAK: PromptConfig(
|
|
218
|
+
default_prompt_label_mapping=dict(correct="áno", incorrect="nie"),
|
|
219
|
+
default_prompt_prefix="Nasledujú vety a či sú gramaticky správne.",
|
|
220
|
+
default_prompt_template="Veta: {text}\nGramaticky správna: {label}",
|
|
221
|
+
default_instruction_prompt=(
|
|
222
|
+
"Veta: {text}\n\nUrčite, či je veta gramaticky správna alebo nie. "
|
|
223
|
+
"Odpovedzte so {labels_str}, a nič iné."
|
|
224
|
+
),
|
|
225
|
+
),
|
|
226
|
+
SLOVENE: PromptConfig(
|
|
227
|
+
default_prompt_label_mapping=dict(correct="da", incorrect="ne"),
|
|
228
|
+
default_prompt_prefix="Sledeče so stavki in ali so slovnično pravilni.",
|
|
229
|
+
default_prompt_template="Stavek: {text}\nSlovnično pravilno: {label}",
|
|
230
|
+
default_instruction_prompt=(
|
|
231
|
+
"Stavek: {text}\n\nUgotovite, ali je stavek slovnično pravilen ali ne. "
|
|
232
|
+
"Odgovorite z {labels_str}, in nič drugega."
|
|
233
|
+
),
|
|
234
|
+
),
|
|
235
|
+
SERBIAN: PromptConfig(
|
|
236
|
+
default_prompt_label_mapping=dict(correct="da", incorrect="ne"),
|
|
237
|
+
default_prompt_prefix="U nastavku su rečenice i da li su gramatički ispravne.",
|
|
238
|
+
default_prompt_template="Rečenica: {text}\nGramatički ispravna: {label}",
|
|
239
|
+
default_instruction_prompt=(
|
|
240
|
+
"Rečenica: {text}\n\nOdredite da li je rečenica gramatički ispravna ili "
|
|
241
|
+
"ne. Odgovorite sa {labels_str}, i ništa drugo."
|
|
242
|
+
),
|
|
243
|
+
),
|
|
244
|
+
SWEDISH: PromptConfig(
|
|
121
245
|
default_prompt_label_mapping=dict(correct="ja", incorrect="nej"),
|
|
122
246
|
default_prompt_prefix="Följande är meningar och huruvida de är grammatiskt "
|
|
123
247
|
"korrekta.",
|
|
@@ -125,4 +249,15 @@ LA_TEMPLATES = {
|
|
|
125
249
|
default_instruction_prompt="Mening: {text}\n\nBestäm om meningen är "
|
|
126
250
|
"grammatiskt korrekt eller inte. Svara med {labels_str}, och inget annat.",
|
|
127
251
|
),
|
|
252
|
+
UKRAINIAN: PromptConfig(
|
|
253
|
+
default_prompt_label_mapping=dict(correct="так", incorrect="ні"),
|
|
254
|
+
default_prompt_prefix=(
|
|
255
|
+
"Нижче наведені речення і їхня граматична правильність."
|
|
256
|
+
),
|
|
257
|
+
default_prompt_template=("Речення: {text}\nГраматично правильно: {label}"),
|
|
258
|
+
default_instruction_prompt=(
|
|
259
|
+
"Речення: {text}\n\nВизначте, чи речення граматично правильне чи ні. "
|
|
260
|
+
"Відповідайте {labels_str}, і нічого більше."
|
|
261
|
+
),
|
|
262
|
+
),
|
|
128
263
|
}
|
|
@@ -1,18 +1,79 @@
|
|
|
1
1
|
"""Templates for all multiple choice tasks."""
|
|
2
2
|
|
|
3
|
+
import typing as t
|
|
4
|
+
|
|
3
5
|
from ..data_models import PromptConfig
|
|
4
|
-
from ..languages import
|
|
6
|
+
from ..languages import (
|
|
7
|
+
BULGARIAN,
|
|
8
|
+
CROATIAN,
|
|
9
|
+
CZECH,
|
|
10
|
+
DANISH,
|
|
11
|
+
DUTCH,
|
|
12
|
+
ENGLISH,
|
|
13
|
+
ESTONIAN,
|
|
14
|
+
FINNISH,
|
|
15
|
+
FRENCH,
|
|
16
|
+
GERMAN,
|
|
17
|
+
GREEK,
|
|
18
|
+
ICELANDIC,
|
|
19
|
+
ITALIAN,
|
|
20
|
+
LATVIAN,
|
|
21
|
+
LITHUANIAN,
|
|
22
|
+
NORWEGIAN,
|
|
23
|
+
NORWEGIAN_BOKMÅL,
|
|
24
|
+
NORWEGIAN_NYNORSK,
|
|
25
|
+
POLISH,
|
|
26
|
+
PORTUGUESE,
|
|
27
|
+
SERBIAN,
|
|
28
|
+
SLOVAK,
|
|
29
|
+
SLOVENE,
|
|
30
|
+
SPANISH,
|
|
31
|
+
SWEDISH,
|
|
32
|
+
UKRAINIAN,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
if t.TYPE_CHECKING:
|
|
36
|
+
from ..languages import Language
|
|
5
37
|
|
|
6
38
|
# TODO: Missing Faroese
|
|
7
|
-
MULTIPLE_CHOICE_TEMPLATES = {
|
|
8
|
-
|
|
39
|
+
MULTIPLE_CHOICE_TEMPLATES: dict["Language", PromptConfig] = {
|
|
40
|
+
BULGARIAN: PromptConfig(
|
|
41
|
+
default_prompt_prefix="Следват въпроси с множествен избор (с отговори).",
|
|
42
|
+
default_prompt_template="Въпрос: {text}\nОтговор: {label}",
|
|
43
|
+
default_instruction_prompt="Въпрос: {text}\n\nОтговорете на горния въпрос "
|
|
44
|
+
"като отговорите с {labels_str}, и нищо друго.",
|
|
45
|
+
default_prompt_label_mapping="auto",
|
|
46
|
+
),
|
|
47
|
+
CROATIAN: PromptConfig(
|
|
48
|
+
default_prompt_prefix=(
|
|
49
|
+
"Sljedeća su pitanja s višestrukim izborom (s odgovorima)."
|
|
50
|
+
),
|
|
51
|
+
default_prompt_template=("Pitanje: {text}\nOdgovor: {label}"),
|
|
52
|
+
default_instruction_prompt=(
|
|
53
|
+
"Pitanje: {text}\n\n"
|
|
54
|
+
"Odgovorite na gornje pitanje koristeći {labels_str}, i ništa drugo."
|
|
55
|
+
),
|
|
56
|
+
default_prompt_label_mapping="auto",
|
|
57
|
+
),
|
|
58
|
+
CZECH: PromptConfig(
|
|
59
|
+
default_prompt_prefix=(
|
|
60
|
+
"Následující jsou otázky s výběrem z více možností (s odpověďmi)."
|
|
61
|
+
),
|
|
62
|
+
default_prompt_template="Otázka: {text}\nOdpověď: {label}",
|
|
63
|
+
default_instruction_prompt=(
|
|
64
|
+
"Otázka: {text}\n\nOdpovězte na výše uvedenou otázku "
|
|
65
|
+
"pomocí {labels_str}, a nic jiného."
|
|
66
|
+
),
|
|
67
|
+
default_prompt_label_mapping="auto",
|
|
68
|
+
),
|
|
69
|
+
DANISH: PromptConfig(
|
|
9
70
|
default_prompt_prefix="Følgende er multiple choice spørgsmål (med svar).",
|
|
10
71
|
default_prompt_template="Spørgsmål: {text}\nSvar: {label}",
|
|
11
72
|
default_instruction_prompt="Spørgsmål: {text}\n\nBesvar ovenstående spørgsmål "
|
|
12
73
|
"ved at svare med {labels_str}, og intet andet.",
|
|
13
74
|
default_prompt_label_mapping="auto",
|
|
14
75
|
),
|
|
15
|
-
|
|
76
|
+
GERMAN: PromptConfig(
|
|
16
77
|
default_prompt_prefix="Die folgenden Fragen sind Multiple-Choice-Fragen "
|
|
17
78
|
"(mit Antworten).",
|
|
18
79
|
default_prompt_template="Frage: {text}\nAntwort: {label}",
|
|
@@ -20,7 +81,15 @@ MULTIPLE_CHOICE_TEMPLATES = {
|
|
|
20
81
|
"mit {labels_str}, und nichts anderes.",
|
|
21
82
|
default_prompt_label_mapping="auto",
|
|
22
83
|
),
|
|
23
|
-
|
|
84
|
+
GREEK: PromptConfig(
|
|
85
|
+
default_prompt_prefix="Ακολουθούν ερωτήσεις πολλαπλών επιλογών "
|
|
86
|
+
"(με απαντήσεις).",
|
|
87
|
+
default_prompt_template="Ερώτηση: {text}\nΑπάντηση: {label}",
|
|
88
|
+
default_instruction_prompt="Ερώτηση: {text}\n\nΑπαντήστε στην παραπάνω ερώτηση "
|
|
89
|
+
"χρησιμοποιώντας {labels_str}, και τίποτα άλλο.",
|
|
90
|
+
default_prompt_label_mapping="auto",
|
|
91
|
+
),
|
|
92
|
+
ENGLISH: PromptConfig(
|
|
24
93
|
default_prompt_prefix="The following are multiple choice questions (with "
|
|
25
94
|
"answers).",
|
|
26
95
|
default_prompt_template="Question: {text}\nAnswer: {label}",
|
|
@@ -28,7 +97,7 @@ MULTIPLE_CHOICE_TEMPLATES = {
|
|
|
28
97
|
"replying with {labels_str}, and nothing else.",
|
|
29
98
|
default_prompt_label_mapping="auto",
|
|
30
99
|
),
|
|
31
|
-
|
|
100
|
+
SPANISH: PromptConfig(
|
|
32
101
|
default_prompt_prefix="Las siguientes son preguntas de opción múltiple "
|
|
33
102
|
"(con respuestas).",
|
|
34
103
|
default_prompt_template="Pregunta: {text}\nRespuesta: {label}",
|
|
@@ -36,7 +105,15 @@ MULTIPLE_CHOICE_TEMPLATES = {
|
|
|
36
105
|
"usando solo {labels_str}, y nada más.",
|
|
37
106
|
default_prompt_label_mapping="auto",
|
|
38
107
|
),
|
|
39
|
-
|
|
108
|
+
ESTONIAN: PromptConfig(
|
|
109
|
+
default_prompt_prefix="Järgnevad on vastusevariantidega küsimused (koos "
|
|
110
|
+
"vastustega).",
|
|
111
|
+
default_prompt_template="Küsimus: {text}\nVastus: {label}",
|
|
112
|
+
default_instruction_prompt="Küsimus: {text}\n\nVasta ülaltoodud küsimusele "
|
|
113
|
+
"ainult {labels_str}, ja mitte millegi muuga.",
|
|
114
|
+
default_prompt_label_mapping="auto",
|
|
115
|
+
),
|
|
116
|
+
PORTUGUESE: PromptConfig(
|
|
40
117
|
default_prompt_prefix="As seguintes são perguntas de escolha múltipla "
|
|
41
118
|
"(com respostas).",
|
|
42
119
|
default_prompt_template="Pergunta: {text}\nResposta: {label}",
|
|
@@ -44,14 +121,14 @@ MULTIPLE_CHOICE_TEMPLATES = {
|
|
|
44
121
|
"acima usando só {labels_str}, e nada mais.",
|
|
45
122
|
default_prompt_label_mapping="auto",
|
|
46
123
|
),
|
|
47
|
-
|
|
124
|
+
FINNISH: PromptConfig(
|
|
48
125
|
default_prompt_prefix="Seuraavat ovat monivalintakysymyksiä (vastauksineen).",
|
|
49
126
|
default_prompt_template="Kysymys: {text}\nVastaus: {label}",
|
|
50
127
|
default_instruction_prompt="Kysymys: {text}\n\nVastaa yllä olevaan kysymykseen "
|
|
51
128
|
"käyttämällä {labels_str}, äläkä mitään muuta.",
|
|
52
129
|
default_prompt_label_mapping="auto",
|
|
53
130
|
),
|
|
54
|
-
|
|
131
|
+
FRENCH: PromptConfig(
|
|
55
132
|
default_prompt_prefix="Les questions suivantes sont des questions à choix "
|
|
56
133
|
"multiples (avec réponses).",
|
|
57
134
|
default_prompt_template="Question: {text}\nRéponse: {label}",
|
|
@@ -59,54 +136,119 @@ MULTIPLE_CHOICE_TEMPLATES = {
|
|
|
59
136
|
"ci-dessus par {labels_str}, et rien d'autre.",
|
|
60
137
|
default_prompt_label_mapping="auto",
|
|
61
138
|
),
|
|
62
|
-
|
|
139
|
+
ICELANDIC: PromptConfig(
|
|
63
140
|
default_prompt_prefix="Eftirfarandi eru fjölvalsspurningar (með svörum).",
|
|
64
141
|
default_prompt_template="Spurningar: {text}\nSvara: {label}",
|
|
65
142
|
default_instruction_prompt="Spurningar: {text}\n\nSvaraðu eftirfarandi "
|
|
66
143
|
"spurningum með {labels_str}, og engu öðru.",
|
|
67
144
|
default_prompt_label_mapping="auto",
|
|
68
145
|
),
|
|
69
|
-
|
|
146
|
+
ITALIAN: PromptConfig(
|
|
70
147
|
default_prompt_prefix="Le seguenti sono domande a scelta multipla "
|
|
71
148
|
"(con relative risposte).",
|
|
72
|
-
default_prompt_template="Domanda: {text}\
|
|
149
|
+
default_prompt_template="Domanda: {text}\nRisposta: {label}",
|
|
73
150
|
default_instruction_prompt="Domanda: {text}\n\nRispondete alla domanda "
|
|
74
151
|
"precedente con {labels_str}, e nient'altro.",
|
|
75
152
|
default_prompt_label_mapping="auto",
|
|
76
153
|
),
|
|
77
|
-
|
|
154
|
+
LITHUANIAN: PromptConfig(
|
|
155
|
+
default_prompt_prefix="Toliau pateikti daugiavariančiai klausimai "
|
|
156
|
+
"(su atsakymais).",
|
|
157
|
+
default_prompt_template="Klausimas: {text}\nAtsakymas: {label}",
|
|
158
|
+
default_instruction_prompt="Klausimas: {text}\n\nAtsakykite į aukščiau "
|
|
159
|
+
"pateiktą klausimą atsakydami {labels_str}, ir nieko daugiau.",
|
|
160
|
+
default_prompt_label_mapping="auto",
|
|
161
|
+
),
|
|
162
|
+
LATVIAN: PromptConfig(
|
|
163
|
+
default_prompt_prefix="Tālāk seko jautājumi ar vairākām atbilžu izvēlēm "
|
|
164
|
+
"(ar atbildēm).",
|
|
165
|
+
default_prompt_template="Jautājums: {text}\nAtbilde: {label}",
|
|
166
|
+
default_instruction_prompt="Jautājums: {text}\n\nAtbildiet uz iepriekšējo "
|
|
167
|
+
"jautājumu, atbildot ar {labels_str}, un nekas cits.",
|
|
168
|
+
default_prompt_label_mapping="auto",
|
|
169
|
+
),
|
|
170
|
+
NORWEGIAN_BOKMÅL: PromptConfig(
|
|
78
171
|
default_prompt_prefix="Følgende er flervalgsspørsmål (med svar).",
|
|
79
172
|
default_prompt_template="Spørsmål: {text}\nSvar: {label}",
|
|
80
173
|
default_instruction_prompt="Spørsmål: {text}\n\nBesvar følgende spørsmål med "
|
|
81
174
|
"{labels_str}, og ikke noe annet.",
|
|
82
175
|
default_prompt_label_mapping="auto",
|
|
83
176
|
),
|
|
84
|
-
|
|
177
|
+
DUTCH: PromptConfig(
|
|
85
178
|
default_prompt_prefix="Hieronder staan meerkeuzevragen (met antwoorden).",
|
|
86
179
|
default_prompt_template="Vraag: {text}\nAntwoord: {label}",
|
|
87
180
|
default_instruction_prompt="Vraag: {text}\n\nBeantwoord de bovenstaande vraag "
|
|
88
181
|
"met {labels_str}, en niets anders.",
|
|
89
182
|
default_prompt_label_mapping="auto",
|
|
90
183
|
),
|
|
91
|
-
|
|
184
|
+
NORWEGIAN_NYNORSK: PromptConfig(
|
|
92
185
|
default_prompt_prefix="Følgende er flervalgsspørsmål (med svar).",
|
|
93
186
|
default_prompt_template="Spørsmål: {text}\nSvar: {label}",
|
|
94
187
|
default_instruction_prompt="Spørsmål: {text}\n\nBesvar følgende spørsmål med "
|
|
95
188
|
"{labels_str}, og ikke noe annet.",
|
|
96
189
|
default_prompt_label_mapping="auto",
|
|
97
190
|
),
|
|
98
|
-
|
|
191
|
+
NORWEGIAN: PromptConfig(
|
|
99
192
|
default_prompt_prefix="Følgende er flervalgsspørsmål (med svar).",
|
|
100
193
|
default_prompt_template="Spørsmål: {text}\nSvar: {label}",
|
|
101
194
|
default_instruction_prompt="Spørsmål: {text}\n\nBesvar følgende spørsmål med "
|
|
102
195
|
"{labels_str}, og ikke noe annet.",
|
|
103
196
|
default_prompt_label_mapping="auto",
|
|
104
197
|
),
|
|
105
|
-
|
|
198
|
+
POLISH: PromptConfig(
|
|
199
|
+
default_prompt_prefix="Poniżej znajdują się pytania wielokrotnego wyboru "
|
|
200
|
+
"(z odpowiedziami).",
|
|
201
|
+
default_prompt_template="Pytanie: {text}\nOdpowiedź: {label}",
|
|
202
|
+
default_instruction_prompt="Pytanie: {text}\n\nOdpowiedz na powyższe pytanie, "
|
|
203
|
+
"używając {labels_str} i niczego więcej.",
|
|
204
|
+
default_prompt_label_mapping="auto",
|
|
205
|
+
),
|
|
206
|
+
SLOVAK: PromptConfig(
|
|
207
|
+
default_prompt_prefix=(
|
|
208
|
+
"Nasledujú otázky s viacerými možnosťami (s odpoveďami)."
|
|
209
|
+
),
|
|
210
|
+
default_prompt_template="Otázka: {text}\nOdpoveď: {label}",
|
|
211
|
+
default_instruction_prompt=(
|
|
212
|
+
"Otázka: {text}\n\n"
|
|
213
|
+
"Odpovedzte na nasledujúcu otázku použitím {labels_str}, a nič iné."
|
|
214
|
+
),
|
|
215
|
+
default_prompt_label_mapping="auto",
|
|
216
|
+
),
|
|
217
|
+
SLOVENE: PromptConfig(
|
|
218
|
+
default_prompt_prefix="Naslednja so vprašanja z več možnostmi (z odgovori).",
|
|
219
|
+
default_prompt_template="Vprašanje: {text}\nOdgovor: {label}",
|
|
220
|
+
default_instruction_prompt=(
|
|
221
|
+
"Vprašanje: {text}\n\n"
|
|
222
|
+
"Odgovorite na navedeno vprašanje z uporabo {labels_str}, in nič drugega."
|
|
223
|
+
),
|
|
224
|
+
default_prompt_label_mapping="auto",
|
|
225
|
+
),
|
|
226
|
+
SERBIAN: PromptConfig(
|
|
227
|
+
default_prompt_prefix=("Slede pitanja višestrukog izbora (sa odgovorima)."),
|
|
228
|
+
default_prompt_template="Pitanje: {text}\nOdgovor: {label}",
|
|
229
|
+
default_instruction_prompt=(
|
|
230
|
+
"Pitanje: {text}\n\n"
|
|
231
|
+
"Odgovorite na navedeno pitanje koristeći {labels_str}, i ništa drugo."
|
|
232
|
+
),
|
|
233
|
+
default_prompt_label_mapping="auto",
|
|
234
|
+
),
|
|
235
|
+
SWEDISH: PromptConfig(
|
|
106
236
|
default_prompt_prefix="Följande är flervalsfrågor (med svar).",
|
|
107
237
|
default_prompt_template="Fråga: {text}\nSvar: {label}",
|
|
108
238
|
default_instruction_prompt="Fråga: {text}\n\nBesvara följande fråga med "
|
|
109
239
|
"{labels_str}, och inget annat.",
|
|
110
240
|
default_prompt_label_mapping="auto",
|
|
111
241
|
),
|
|
242
|
+
UKRAINIAN: PromptConfig(
|
|
243
|
+
default_prompt_prefix=(
|
|
244
|
+
"Нижче наведено питання з кількома варіантами відповідей (з відповідями)."
|
|
245
|
+
),
|
|
246
|
+
default_prompt_template=("Питання: {text}\nВідповідь: {label}"),
|
|
247
|
+
default_instruction_prompt=(
|
|
248
|
+
"Питання: {text}\n\n"
|
|
249
|
+
"Дайте відповідь на наведене вище питання, використовуючи "
|
|
250
|
+
"{labels_str}, і нічого іншого."
|
|
251
|
+
),
|
|
252
|
+
default_prompt_label_mapping="auto",
|
|
253
|
+
),
|
|
112
254
|
}
|