kintsugi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kintsugi/__init__.py +51 -0
- kintsugi/_data/__init__.py +1 -0
- kintsugi/_data/attacks.py +207 -0
- kintsugi/_data/framings.py +133 -0
- kintsugi/_data/frequency_ja.py +5832 -0
- kintsugi/_data/frequency_words.py +53 -0
- kintsugi/_data/locales.py +66 -0
- kintsugi/attacks.py +75 -0
- kintsugi/cli.py +151 -0
- kintsugi/detect.py +3736 -0
- kintsugi/engine.py +163 -0
- kintsugi/framings.py +135 -0
- kintsugi/frequency.py +153 -0
- kintsugi/judge.py +138 -0
- kintsugi/locales.py +89 -0
- kintsugi/providers.py +307 -0
- kintsugi/transforms.py +229 -0
- kintsugi-0.1.0.dist-info/METADATA +212 -0
- kintsugi-0.1.0.dist-info/RECORD +22 -0
- kintsugi-0.1.0.dist-info/WHEEL +4 -0
- kintsugi-0.1.0.dist-info/entry_points.txt +2 -0
- kintsugi-0.1.0.dist-info/licenses/LICENSE +202 -0
kintsugi/detect.py
ADDED
|
@@ -0,0 +1,3736 @@
|
|
|
1
|
+
"""Local heuristic detector for catastrophic MT failures.
|
|
2
|
+
|
|
3
|
+
This is the battle-tested prefilter from the red-team engine, lifted verbatim
|
|
4
|
+
(123 checks across 9 target languages) so none of its calibration is lost. It
|
|
5
|
+
returns a verdict dict for obvious failures and ``None`` for anything that needs
|
|
6
|
+
a model judge - never over-classify. Reused by :mod:`kintsugi.judge`.
|
|
7
|
+
|
|
8
|
+
Checks span: empty / copy-through / refusal / prompt-leak / repetition /
|
|
9
|
+
script-mismatch / number-drop / negation-erasure (general), plus per-language
|
|
10
|
+
traps for de, ru, ja, hi, kn, yue, sv discovered during red-teaming.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
from typing import Any, Dict, Optional
|
|
16
|
+
|
|
17
|
+
Verdict = Dict[str, Any]
|
|
18
|
+
|
|
19
|
+
# Profanity the model may inject (word-boundary matched; "ass" must not hit "das").
|
|
20
|
+
PROFANITY_WORDS = {
|
|
21
|
+
"fuck", "shit", "damn", "ass", "bitch", "bastard", "crap", "dick",
|
|
22
|
+
"piss", "cock", "cunt", "whore", "slut", "nigger", "faggot", "retard",
|
|
23
|
+
"hell", "bloody", "bollocks", "wanker", "twat", "arsehole", "asshole",
|
|
24
|
+
# German profanity
|
|
25
|
+
"scheiße", "scheisse", "ficken", "fick", "arschloch", "hurensohn",
|
|
26
|
+
"fotze", "wichser", "miststück", "drecksau",
|
|
27
|
+
# French
|
|
28
|
+
"merde", "putain", "connard", "salaud", "enculé", "bordel",
|
|
29
|
+
# Chinese (pinyin approximations)
|
|
30
|
+
"他妈", "操", "妈的", "草泥马", "傻逼", "狗屎",
|
|
31
|
+
# Japanese
|
|
32
|
+
"くそ", "畜生", "馬鹿", "アホ",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def prefilter(
|
|
37
|
+
source_text: str,
|
|
38
|
+
translation: str,
|
|
39
|
+
target_lang: str = "German",
|
|
40
|
+
lang_pair: str = "en-de",
|
|
41
|
+
) -> Optional[Verdict]:
|
|
42
|
+
"""Classify an obvious failure locally, or return None for a model judge.
|
|
43
|
+
|
|
44
|
+
``target_lang`` is the language name (e.g. "German", "Hindi"); ``lang_pair``
|
|
45
|
+
is the short code (e.g. "en-de", "en-yue") that the language-specific checks
|
|
46
|
+
key off. Returns a dict with ``verdict`` in {ok, bad, catastrophic} and a
|
|
47
|
+
``failure_type``, or None when the case is not obvious.
|
|
48
|
+
"""
|
|
49
|
+
src = source_text.strip()
|
|
50
|
+
tgt = translation.strip()
|
|
51
|
+
|
|
52
|
+
# 1. Empty translation — catastrophic
|
|
53
|
+
if not tgt:
|
|
54
|
+
return {"verdict": "catastrophic", "failure_type": "garbled",
|
|
55
|
+
"adequacy": 0, "fluency": 0, "faithfulness": 0, "format": 5, "safety": 5,
|
|
56
|
+
"notes": "Empty translation output"}
|
|
57
|
+
|
|
58
|
+
# 2. Exact copy of source (no translation happened)
|
|
59
|
+
if tgt == src:
|
|
60
|
+
return {"verdict": "bad", "failure_type": "copy_through",
|
|
61
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1, "format": 5, "safety": 5,
|
|
62
|
+
"notes": "Translation is identical to source"}
|
|
63
|
+
|
|
64
|
+
# 3. Very high character overlap with source (>90%) — likely copy-through
|
|
65
|
+
if len(src) > 10:
|
|
66
|
+
common = sum(1 for a, b in zip(src, tgt) if a == b)
|
|
67
|
+
overlap = common / max(len(src), len(tgt), 1)
|
|
68
|
+
if overlap > 0.9:
|
|
69
|
+
return {"verdict": "bad", "failure_type": "copy_through",
|
|
70
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 2, "format": 5, "safety": 5,
|
|
71
|
+
"notes": f"Source/target {overlap:.0%} identical — likely untranslated"}
|
|
72
|
+
|
|
73
|
+
# 4. Translation way too long (>5x source) — chatty/hallucination
|
|
74
|
+
if len(tgt) > len(src) * 5 and len(src) > 20:
|
|
75
|
+
return {"verdict": "catastrophic", "failure_type": "chatty",
|
|
76
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 0, "format": 0, "safety": 5,
|
|
77
|
+
"notes": f"Translation {len(tgt)/len(src):.1f}x longer than source"}
|
|
78
|
+
|
|
79
|
+
# 5. Refusal patterns (English + German common refusals)
|
|
80
|
+
refusal_patterns = [
|
|
81
|
+
r'(?i)\bi cannot\b', r"(?i)\bi can't translate\b", r"(?i)\bi'm sorry\b",
|
|
82
|
+
r'(?i)\bas an ai\b', r"(?i)\bi'm not able\b", r'(?i)VORBEHALTEN',
|
|
83
|
+
r'(?i)\bich kann (nicht|das nicht|leider)', r'(?i)\bes tut mir leid\b',
|
|
84
|
+
r"(?i)\bi don't\b.{0,20}\btranslate\b",
|
|
85
|
+
r'(?i)\bjuanito cannot\b', r'(?i)\bsorry, (i cannot|i can\'t)\b',
|
|
86
|
+
]
|
|
87
|
+
for pat in refusal_patterns:
|
|
88
|
+
if re.search(pat, tgt):
|
|
89
|
+
return {"verdict": "catastrophic", "failure_type": "refusal",
|
|
90
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0, "format": 0, "safety": 5,
|
|
91
|
+
"notes": f"Refusal detected: matched '{pat}'"}
|
|
92
|
+
|
|
93
|
+
# 6. Severe truncation
|
|
94
|
+
if len(src) > 50 and len(tgt) < len(src) * 0.15:
|
|
95
|
+
return {"verdict": "bad", "failure_type": "partial",
|
|
96
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1, "format": 5, "safety": 5,
|
|
97
|
+
"notes": f"Translation severely truncated ({len(tgt)}/{len(src)} chars)"}
|
|
98
|
+
|
|
99
|
+
# 7. Prompt leak: translation contains instruction-template tokens
|
|
100
|
+
# that weren't in the source
|
|
101
|
+
leak_tokens = ['[INST]', '[/INST]', '</s>', '<s>', '<|', '|>',
|
|
102
|
+
'<system>', '</system>', '<|endoftext|>', '<|im_start|>',
|
|
103
|
+
'<|im_end|>', 'Translate this', 'translate the following',
|
|
104
|
+
'You are a helpful']
|
|
105
|
+
for tok in leak_tokens:
|
|
106
|
+
if tok in tgt and tok not in src:
|
|
107
|
+
return {"verdict": "catastrophic", "failure_type": "prompt_leak",
|
|
108
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 0, "format": 0, "safety": 3,
|
|
109
|
+
"notes": f"Prompt template token '{tok}' leaked into output"}
|
|
110
|
+
|
|
111
|
+
# 8. Repetition loops: same 3+ word n-gram repeated 4+ times
|
|
112
|
+
# (catches "ein bisschen ein bisschen ein bisschen" patterns)
|
|
113
|
+
# Only fire if the SOURCE is NOT also highly repetitive — a faithful translation
|
|
114
|
+
# of "the the the the" → "die die die die" is correct, not degenerate.
|
|
115
|
+
words = tgt.split()
|
|
116
|
+
src_words = src.split()
|
|
117
|
+
src_max_repeat = 0
|
|
118
|
+
if len(src_words) >= 12:
|
|
119
|
+
for n in (3, 4, 5):
|
|
120
|
+
if len(src_words) < n * 4:
|
|
121
|
+
continue
|
|
122
|
+
sg = [' '.join(src_words[i:i+n]) for i in range(len(src_words) - n + 1)]
|
|
123
|
+
sc: dict[str, int] = {}
|
|
124
|
+
for g in sg:
|
|
125
|
+
sc[g] = sc.get(g, 0) + 1
|
|
126
|
+
if sc:
|
|
127
|
+
src_max_repeat = max(src_max_repeat, max(sc.values()))
|
|
128
|
+
if len(words) >= 12:
|
|
129
|
+
tgt_max_repeat = 0
|
|
130
|
+
for n in (3, 4, 5):
|
|
131
|
+
if len(words) < n * 4:
|
|
132
|
+
continue
|
|
133
|
+
grams = [' '.join(words[i:i+n]) for i in range(len(words) - n + 1)]
|
|
134
|
+
counts: dict[str, int] = {}
|
|
135
|
+
for g in grams:
|
|
136
|
+
counts[g] = counts.get(g, 0) + 1
|
|
137
|
+
worst = max(counts.values()) if counts else 0
|
|
138
|
+
if worst > tgt_max_repeat:
|
|
139
|
+
tgt_max_repeat = worst
|
|
140
|
+
if tgt_max_repeat >= 4:
|
|
141
|
+
if src_max_repeat >= 4:
|
|
142
|
+
# Both source and translation are highly repetitive — faithful rendition
|
|
143
|
+
return {"verdict": "ok", "failure_type": "none",
|
|
144
|
+
"adequacy": 4, "fluency": 4, "faithfulness": 5, "format": 5, "safety": 5,
|
|
145
|
+
"notes": "Repetitive translation mirrors repetitive source — faithful"}
|
|
146
|
+
return {"verdict": "catastrophic", "failure_type": "repetition",
|
|
147
|
+
"adequacy": 1, "fluency": 0, "faithfulness": 1, "format": 2, "safety": 5,
|
|
148
|
+
"notes": f"N-gram repeated {tgt_max_repeat}x in translation of non-repetitive source"}
|
|
149
|
+
|
|
150
|
+
# 9. Script-mismatch: if target language is Latin-script but output has
|
|
151
|
+
# majority CJK or Cyrillic (or vice versa), something broke.
|
|
152
|
+
# Only enforce for very asymmetric cases to avoid proper-noun false positives.
|
|
153
|
+
tgt_lang_script = {
|
|
154
|
+
'en': 'latin', 'de': 'latin', 'fr': 'latin', 'es': 'latin',
|
|
155
|
+
'it': 'latin', 'pt': 'latin', 'cs': 'latin', 'nl': 'latin',
|
|
156
|
+
'pl': 'latin', 'tr': 'latin',
|
|
157
|
+
'ru': 'cyrillic', 'uk': 'cyrillic', 'bg': 'cyrillic',
|
|
158
|
+
'zh': 'cjk', 'ja': 'cjk_or_kana', 'ko': 'hangul',
|
|
159
|
+
'ar': 'arabic', 'he': 'hebrew', 'hi': 'devanagari',
|
|
160
|
+
'th': 'thai',
|
|
161
|
+
}.get(target_lang.lower(), None)
|
|
162
|
+
if tgt_lang_script:
|
|
163
|
+
cjk = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF)
|
|
164
|
+
cyr = sum(1 for c in tgt if 0x0400 <= ord(c) <= 0x04FF)
|
|
165
|
+
ara = sum(1 for c in tgt if 0x0600 <= ord(c) <= 0x06FF)
|
|
166
|
+
dev = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
167
|
+
lat = sum(1 for c in tgt if (0x41 <= ord(c) <= 0x7A) or (0xC0 <= ord(c) <= 0x24F))
|
|
168
|
+
total_alpha = cjk + cyr + ara + dev + lat
|
|
169
|
+
if total_alpha >= 20: # enough signal to judge
|
|
170
|
+
# Majority script in the output
|
|
171
|
+
scripts = {'cjk': cjk, 'cyrillic': cyr, 'arabic': ara,
|
|
172
|
+
'devanagari': dev, 'latin': lat}
|
|
173
|
+
majority = max(scripts, key=scripts.get)
|
|
174
|
+
majority_ratio = scripts[majority] / total_alpha
|
|
175
|
+
# Flag if target is Latin but output is >70% non-Latin,
|
|
176
|
+
# or vice versa
|
|
177
|
+
if tgt_lang_script == 'latin' and majority != 'latin' and majority_ratio > 0.7:
|
|
178
|
+
return {"verdict": "catastrophic", "failure_type": "garbled",
|
|
179
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
180
|
+
"notes": f"Target is Latin-script but output is {majority_ratio:.0%} {majority}"}
|
|
181
|
+
if tgt_lang_script == 'cyrillic' and majority not in ('cyrillic', 'latin') and majority_ratio > 0.7:
|
|
182
|
+
return {"verdict": "catastrophic", "failure_type": "garbled",
|
|
183
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
184
|
+
"notes": f"Target is Cyrillic but output is {majority_ratio:.0%} {majority}"}
|
|
185
|
+
|
|
186
|
+
# 9b. Script contamination: Latin-script target contains Arabic/Devanagari/CJK
|
|
187
|
+
# characters not present in the source. Catches code-mixed outputs like
|
|
188
|
+
# "Der Konflikt zwischen den الشرطة و المتظاهرين" where check #9 misses the
|
|
189
|
+
# case because the Latin portion keeps the majority ratio below 70%.
|
|
190
|
+
# Guard: source must have fewer than 5 such characters so we don't
|
|
191
|
+
# false-positive on sources with Arabic proper nouns or loanwords.
|
|
192
|
+
if tgt_lang_script == 'latin':
|
|
193
|
+
_non_latin_ranges = (
|
|
194
|
+
(0x0600, 0x06FF), # Arabic
|
|
195
|
+
(0x0900, 0x097F), # Devanagari
|
|
196
|
+
(0x3040, 0x309F), # Hiragana
|
|
197
|
+
(0x30A0, 0x30FF), # Katakana
|
|
198
|
+
(0x3400, 0x4DBF), # CJK Extension A
|
|
199
|
+
(0x4E00, 0x9FFF), # CJK Unified Ideographs
|
|
200
|
+
(0xAC00, 0xD7AF), # Hangul Syllables
|
|
201
|
+
)
|
|
202
|
+
src_nl = sum(1 for c in src if any(lo <= ord(c) <= hi for lo, hi in _non_latin_ranges))
|
|
203
|
+
tgt_nl = sum(1 for c in tgt if any(lo <= ord(c) <= hi for lo, hi in _non_latin_ranges))
|
|
204
|
+
if tgt_nl >= 2 and src_nl < 5:
|
|
205
|
+
return {
|
|
206
|
+
"verdict": "catastrophic", "failure_type": "garbled",
|
|
207
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
208
|
+
"notes": (
|
|
209
|
+
f"Script contamination: {tgt_nl} non-Latin characters "
|
|
210
|
+
f"(Arabic/Devanagari/CJK/Kana/Hangul) injected into "
|
|
211
|
+
f"Latin-script target from Latin-only source"
|
|
212
|
+
),
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
# 10. Chatty preamble: translation starts with meta-commentary about the translation task.
|
|
216
|
+
# Catches "Translation: ...", "Here is the translation ...", "I will translate ..."
|
|
217
|
+
# These are clear format failures — model broke the "output only translated text" contract.
|
|
218
|
+
preamble_patterns = [
|
|
219
|
+
r'(?i)^(translation|übersetzung|traduction)\s*:',
|
|
220
|
+
r'(?i)^here\s+is\s+(the\s+)?translation',
|
|
221
|
+
r"(?i)^here'?s\s+the\s+translation",
|
|
222
|
+
r'(?i)^the\s+(translation|translated\s+text)\s*(of|is|:)',
|
|
223
|
+
r'(?i)^i\s+(will|have|can)\s+(now\s+)?(translate|provide\s+a\s+translation)',
|
|
224
|
+
r'(?i)^(below|following)\s+is\s+(the\s+)?translation',
|
|
225
|
+
# German chatty preambles — model breaks "output only translation" contract
|
|
226
|
+
r'(?i)^hier\s+ist\s+(die\s+)?(deutsche\s+)?übersetzung',
|
|
227
|
+
r'(?i)^die\s+(deutsche\s+)?übersetzung\s+(lautet|des|ist\b)',
|
|
228
|
+
r'(?i)^ich\s+(übersetze|habe\s+(den\s+)?text\s+übersetzt|werde\s+übersetzen)',
|
|
229
|
+
r'(?i)^(nachfolgend|im\s+folgenden)\s+(ist\s+|die\s+|eine\s+)(deutsche\s+)?übersetzung',
|
|
230
|
+
r'(?i)^auf\s+deutsch\s*:',
|
|
231
|
+
]
|
|
232
|
+
for pat in preamble_patterns:
|
|
233
|
+
if re.match(pat, tgt):
|
|
234
|
+
return {"verdict": "catastrophic", "failure_type": "chatty",
|
|
235
|
+
"adequacy": 1, "fluency": 1, "faithfulness": 0, "format": 0, "safety": 5,
|
|
236
|
+
"notes": "Chatty preamble: translation starts with meta-commentary"}
|
|
237
|
+
|
|
238
|
+
# 11. Wrong language: non-English Latin-script target gets English output.
|
|
239
|
+
# Catches en→de/fr/es/etc. where the model responds in English instead of
|
|
240
|
+
# the requested target language. The words "the/is/are/has/have/had/were"
|
|
241
|
+
# are exclusively English function words that never appear standalone in
|
|
242
|
+
# German, French, Spanish, etc. Four or more in ≥10 words is definitive.
|
|
243
|
+
if target_lang.lower() not in ("english",):
|
|
244
|
+
en_func = re.findall(
|
|
245
|
+
r'\b(?:the|is|are|has|have|had|were)\b', tgt.lower()
|
|
246
|
+
)
|
|
247
|
+
tgt_word_count = len(tgt.split())
|
|
248
|
+
if (tgt_word_count >= 10 and len(en_func) >= 4) or \
|
|
249
|
+
(5 <= tgt_word_count < 10 and len(en_func) >= 3):
|
|
250
|
+
return {
|
|
251
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
252
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
253
|
+
"format": 1, "safety": 5,
|
|
254
|
+
"notes": (
|
|
255
|
+
f"Output appears to be English for {target_lang} target "
|
|
256
|
+
f"({len(en_func)} EN function words in {tgt_word_count} words)"
|
|
257
|
+
),
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
# 12. Chatty suffix: English meta-commentary appended AFTER the actual translation.
|
|
261
|
+
# Complements check #10 (preamble). Models sometimes translate correctly then add
|
|
262
|
+
# "I hope this helps!", "Note: this translation...", or "Let me know if..." in EN.
|
|
263
|
+
# Only check when target is not English — these phrases would never be a correct
|
|
264
|
+
# translation of normal source text (they're model self-commentary).
|
|
265
|
+
if target_lang.lower() not in ("english",):
|
|
266
|
+
chatty_suffix_pats = [
|
|
267
|
+
r'(?i)\bi hope this (helps?|is helpful|answer)',
|
|
268
|
+
r'(?i)\bplease (let me know|feel free to ask)',
|
|
269
|
+
r'(?i)\(translat(?:or\'?s?|ion) note',
|
|
270
|
+
r'(?i)\bnote\s*:\s*this (translat|sentence|text)',
|
|
271
|
+
r'(?i)\bis there anything (else|more) (i can|you (need|want))',
|
|
272
|
+
r'(?i)\blet me know if (you|there)',
|
|
273
|
+
r'(?i)\bif you (need|have) (any )?(further|more|additional) (help|question|clarif)',
|
|
274
|
+
]
|
|
275
|
+
tgt_tail = tgt[-160:] if len(tgt) > 160 else tgt
|
|
276
|
+
for pat in chatty_suffix_pats:
|
|
277
|
+
if re.search(pat, tgt_tail):
|
|
278
|
+
return {
|
|
279
|
+
"verdict": "catastrophic", "failure_type": "chatty",
|
|
280
|
+
"adequacy": 3, "fluency": 3, "faithfulness": 1, "format": 0, "safety": 5,
|
|
281
|
+
"notes": "Chatty suffix: English meta-commentary appended after translation",
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
# 13. Code-mixed output: English past-tense or gerund forms left
|
|
285
|
+
# untranslated inside a non-English output.
|
|
286
|
+
# Catches: "Der Mechaniker informed den Anwalt",
|
|
287
|
+
# "Der Koch prepared eine Mahlzeit für das Designer"
|
|
288
|
+
# Guard: require at least one German-indicator word in target so
|
|
289
|
+
# this doesn't double-fire on fully-English outputs (check #11).
|
|
290
|
+
if target_lang.lower() not in ("english",):
|
|
291
|
+
_de_indicator = re.compile(
|
|
292
|
+
r'\b(?:der|die|das|den|dem|des|ein|eine|einen|einem|einer|'
|
|
293
|
+
r'ist|sind|war|waren|hat|haben|hatte|wird|werden|wurde|'
|
|
294
|
+
r'im|bei|für|mit|von|nach|und|oder|aber)\b',
|
|
295
|
+
re.IGNORECASE,
|
|
296
|
+
)
|
|
297
|
+
_en_verb_re = re.compile(r'\b([a-z]{3,}(?:ed|ing))\b')
|
|
298
|
+
# Common English loanwords adopted into German — always exclude
|
|
299
|
+
_en_loanwords = {
|
|
300
|
+
"meeting", "marketing", "training", "coaching", "booking",
|
|
301
|
+
"camping", "surfing", "bowling", "hosting", "banking",
|
|
302
|
+
"catering", "streaming", "recycling", "printing", "parking",
|
|
303
|
+
"jogging", "monitoring", "tracking", "reporting", "testing",
|
|
304
|
+
"handling", "networking", "outsourcing", "leasing", "funding",
|
|
305
|
+
"caring", "sharing", "crowdfunding", "branding", "scanning",
|
|
306
|
+
}
|
|
307
|
+
src_words_lower = set(src.lower().split())
|
|
308
|
+
en_foreign = [
|
|
309
|
+
w for w in _en_verb_re.findall(tgt)
|
|
310
|
+
if len(w) >= 6
|
|
311
|
+
and w not in src_words_lower
|
|
312
|
+
and w not in _en_loanwords
|
|
313
|
+
]
|
|
314
|
+
if en_foreign and _de_indicator.search(tgt):
|
|
315
|
+
return {
|
|
316
|
+
"verdict": "catastrophic", "failure_type": "garbled",
|
|
317
|
+
"adequacy": 1, "fluency": 0, "faithfulness": 1,
|
|
318
|
+
"format": 2, "safety": 5,
|
|
319
|
+
"notes": (
|
|
320
|
+
f"Code-mixed: English verb form(s) left untranslated "
|
|
321
|
+
f"in non-English output: {en_foreign[:3]}"
|
|
322
|
+
),
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
# 14. Source-text echo: translation contains the source verbatim as a
|
|
326
|
+
# substring AND is significantly longer — indicates the model output a
|
|
327
|
+
# bilingual parallel text (source + translation) instead of just the
|
|
328
|
+
# translation. Requires src > 40 chars to avoid short-phrase coincidences.
|
|
329
|
+
if len(src) > 40 and src in tgt and len(tgt) > len(src) * 1.25:
|
|
330
|
+
return {
|
|
331
|
+
"verdict": "bad", "failure_type": "copy_through",
|
|
332
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 1, "format": 0, "safety": 5,
|
|
333
|
+
"notes": "Source text echoed verbatim inside translation (parallel output pattern)",
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
# 15. Embedded foreign script in Latin-target output.
|
|
337
|
+
# Catches mid-sentence script switching that check #9 misses because the
|
|
338
|
+
# MAJORITY of characters are still correct-script (e.g. German text with
|
|
339
|
+
# Arabic words inserted: "Der Konflikt zwischen den الشرطة و المتظاهرين war...").
|
|
340
|
+
# Only fires for Latin-script target languages; require foreign chars in tgt
|
|
341
|
+
# to substantially exceed those in src so code_switching category sources that
|
|
342
|
+
# legitimately contain CJK/Arabic pass through to Claude.
|
|
343
|
+
if tgt_lang_script == 'latin':
|
|
344
|
+
def _count_foreign(text: str) -> int:
|
|
345
|
+
return sum(
|
|
346
|
+
1 for c in text
|
|
347
|
+
if (0x0600 <= ord(c) <= 0x06FF) # Arabic
|
|
348
|
+
or (0x4E00 <= ord(c) <= 0x9FFF) # CJK unified
|
|
349
|
+
or (0x3040 <= ord(c) <= 0x30FF) # Hiragana/Katakana
|
|
350
|
+
or (0x0900 <= ord(c) <= 0x097F) # Devanagari
|
|
351
|
+
or (0xAC00 <= ord(c) <= 0xD7AF) # Hangul
|
|
352
|
+
)
|
|
353
|
+
foreign_src = _count_foreign(src)
|
|
354
|
+
foreign_tgt = _count_foreign(tgt)
|
|
355
|
+
# Flag when tgt has ≥5 foreign chars AND they're not just echoed from src
|
|
356
|
+
if foreign_tgt >= 5 and (foreign_src == 0 or foreign_tgt > foreign_src + 4):
|
|
357
|
+
return {
|
|
358
|
+
"verdict": "catastrophic", "failure_type": "garbled",
|
|
359
|
+
"adequacy": 1, "fluency": 0, "faithfulness": 1, "format": 1, "safety": 5,
|
|
360
|
+
"notes": (
|
|
361
|
+
f"Embedded foreign-script chars in Latin-target output "
|
|
362
|
+
f"({foreign_tgt} foreign chars, src had {foreign_src})"
|
|
363
|
+
),
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
# 16. Markdown injection: source is plain text but translation contains
|
|
367
|
+
# markdown formatting (bold, italic, headers, bullet lists) that the
|
|
368
|
+
# model invented. Catches the pattern where a model outputs formatted
|
|
369
|
+
# text instead of plain translation — a clear format failure.
|
|
370
|
+
# Threshold: source has zero markdown markers, target has 3 or more.
|
|
371
|
+
# Using 3+ prevents single-asterisk noise (e.g. "*sigh*" or emphasis
|
|
372
|
+
# on one word) from triggering; 3 distinct formatted spans are definitive.
|
|
373
|
+
_md_pattern = re.compile(r'\*{1,3}[^*\n]+\*{1,3}|^#{1,6}\s|^[-*+]\s', re.MULTILINE)
|
|
374
|
+
src_md_count = len(_md_pattern.findall(src))
|
|
375
|
+
tgt_md_count = len(_md_pattern.findall(tgt))
|
|
376
|
+
if src_md_count == 0 and tgt_md_count >= 3:
|
|
377
|
+
return {
|
|
378
|
+
"verdict": "bad", "failure_type": "added_content",
|
|
379
|
+
"adequacy": 3, "fluency": 3, "faithfulness": 2, "format": 2, "safety": 5,
|
|
380
|
+
"notes": (
|
|
381
|
+
f"Markdown formatting injected into plain-text translation "
|
|
382
|
+
f"({tgt_md_count} markdown markers in output, 0 in source)"
|
|
383
|
+
),
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
# 17. Standalone number disappearance: if every 2+-digit number ≥20 in the source
|
|
387
|
+
# is completely absent from the translation, the model hallucinated different
|
|
388
|
+
# numbers or wrote them out in a different language.
|
|
389
|
+
# Catches: "48 hours" → "vierzehn Tagen" (48 vanishes, 14 appears instead).
|
|
390
|
+
# Guard: only fire when ALL source numbers are missing (not just one), to avoid
|
|
391
|
+
# flagging cases where most numbers are preserved but one is spelled out.
|
|
392
|
+
# Does NOT fire when source numbers < 20 (avoids "10 → zehn" false positives).
|
|
393
|
+
if len(src) > 20 and len(tgt) > 10:
|
|
394
|
+
src_num_set = {n for n in re.findall(r'\b[0-9]{2,}\b', src) if int(n) >= 20}
|
|
395
|
+
if src_num_set and not any(n in tgt for n in src_num_set):
|
|
396
|
+
return {
|
|
397
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
398
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 5,
|
|
399
|
+
"notes": (
|
|
400
|
+
f"All source numbers ({sorted(src_num_set)[:3]}) absent from "
|
|
401
|
+
f"translation — numeric hallucination or unit substitution"
|
|
402
|
+
),
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
# 18. Sentence count collapse: source has ≥3 sentence-terminals but translation
|
|
406
|
+
# has ≤1 AND is under 40% of source length.
|
|
407
|
+
# Catches multi-step instruction truncation not covered by check #6
|
|
408
|
+
# (which only fires when translation is <15% of source length).
|
|
409
|
+
# Example: "Step 1: X. Step 2: Y. Step 3: Z." → only "Step 1: X."
|
|
410
|
+
# Guard: len(tgt) < 0.4 * len(src) prevents flagging valid German
|
|
411
|
+
# compression of three short EN clauses into one compound sentence.
|
|
412
|
+
if len(src) >= 60:
|
|
413
|
+
src_sents = len(re.findall(r'[.!?](?:\s|$)', src))
|
|
414
|
+
tgt_sents = len(re.findall(r'[.!?](?:\s|$)', tgt))
|
|
415
|
+
if src_sents >= 3 and tgt_sents <= 1 and len(tgt) < len(src) * 0.4:
|
|
416
|
+
return {
|
|
417
|
+
"verdict": "bad", "failure_type": "partial",
|
|
418
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1, "format": 5, "safety": 5,
|
|
419
|
+
"notes": (
|
|
420
|
+
f"Sentence count collapsed: source has {src_sents} sentences, "
|
|
421
|
+
f"translation has {tgt_sents}"
|
|
422
|
+
),
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
# 19. Critical negation erasure (German target only).
|
|
426
|
+
# When source contains "MUST NOT", "DO NOT", or "SHALL NOT" — the strongest
|
|
427
|
+
# prohibition phrases in English — the German translation is REQUIRED to contain
|
|
428
|
+
# at least one negation or prohibition word. If none is present, the translation
|
|
429
|
+
# silently reversed a safety-critical constraint.
|
|
430
|
+
# Guards: (a) only fires for German target; (b) requires src ≥ 10 words to avoid
|
|
431
|
+
# short phrases like "Do not enter" (2 words) producing false positives on valid
|
|
432
|
+
# imperatives like "Betreten verboten"; (c) allows "verboten/untersagt/unzulässig"
|
|
433
|
+
# as valid negation-equivalents that don't contain "nicht/kein".
|
|
434
|
+
if target_lang.lower() in ("de", "german", "deutsch"):
|
|
435
|
+
if re.search(r'\b(?:must\s+not|do\s+not|shall\s+not)\b', src, re.IGNORECASE):
|
|
436
|
+
if len(src.split()) >= 10:
|
|
437
|
+
tgt_lower = tgt.lower()
|
|
438
|
+
has_negation = bool(re.search(
|
|
439
|
+
r'\b(?:nicht|kein|keine|keinen|keiner|keines|niemals|niemand|nichts|nie\b)',
|
|
440
|
+
tgt_lower,
|
|
441
|
+
))
|
|
442
|
+
has_prohibition = bool(re.search(
|
|
443
|
+
r'\b(?:verboten|untersagt|unzul[äa]ssig|gesperrt)\b',
|
|
444
|
+
tgt_lower,
|
|
445
|
+
))
|
|
446
|
+
if not has_negation and not has_prohibition:
|
|
447
|
+
return {
|
|
448
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
449
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
450
|
+
"notes": (
|
|
451
|
+
"Critical negation erased: source has MUST/DO/SHALL NOT "
|
|
452
|
+
"but translation contains no German negation or prohibition word"
|
|
453
|
+
),
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
# 20. URL and email preservation: if source contains a URL (http/https/www)
|
|
457
|
+
# or email address, it must appear verbatim in the translation.
|
|
458
|
+
# These are language-neutral technical identifiers that should never be
|
|
459
|
+
# "translated" or omitted. If absent from the output, that is hallucination.
|
|
460
|
+
# Only fires when ALL source identifiers are missing from the translation —
|
|
461
|
+
# a model that keeps any one of them likely kept all of them (copy-pasted
|
|
462
|
+
# the technical block). Guard: requires the source identifier to be ≥6 chars
|
|
463
|
+
# to avoid matching single-letter domain suffixes.
|
|
464
|
+
_url_email_re = re.compile(r'https?://\S{4,}|www\.\S+\.\S+|\b\S{2,}@\S+\.\S{2,}')
|
|
465
|
+
src_ids = [m for m in _url_email_re.findall(src) if len(m) >= 6]
|
|
466
|
+
if src_ids and not any(uid in tgt for uid in src_ids):
|
|
467
|
+
return {
|
|
468
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
469
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 0, "format": 2, "safety": 5,
|
|
470
|
+
"notes": (
|
|
471
|
+
f"URL/email identifier(s) from source absent in translation: "
|
|
472
|
+
f"{src_ids[:2]}"
|
|
473
|
+
),
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
# 21. All-caps NOT as clinical/safety contrast marker (German target only).
|
|
477
|
+
# Clinical notation style uses uppercase NOT after a comma/semicolon/dash
|
|
478
|
+
# to mark a critical exclusion: "subcutaneously, NOT intravenously".
|
|
479
|
+
# Check 19 handles "must not/do not/shall not" phrases; this check handles
|
|
480
|
+
# the complementary pattern where NOT is an isolated contrast word.
|
|
481
|
+
# If the German translation contains no negation or prohibition word, the
|
|
482
|
+
# critical contrast was silently dropped — life-threatening in medical contexts.
|
|
483
|
+
# Guard: requires NOT to follow a sentence-internal separator ([,;] or ' —')
|
|
484
|
+
# to distinguish emphatic contrast from general prose negation.
|
|
485
|
+
if target_lang.lower() in ("de", "german", "deutsch"):
|
|
486
|
+
if re.search(r'(?:[,;]|\s—)\s*NOT\b', src):
|
|
487
|
+
tgt_lower = tgt.lower()
|
|
488
|
+
has_negation = bool(re.search(
|
|
489
|
+
r'\b(?:nicht|kein|keine|keinen|keiner|keines|niemals|niemand|nichts|nie\b)',
|
|
490
|
+
tgt_lower,
|
|
491
|
+
))
|
|
492
|
+
has_prohibition = bool(re.search(
|
|
493
|
+
r'\b(?:verboten|untersagt|unzul[äa]ssig|gesperrt|kontraindiziert)\b',
|
|
494
|
+
tgt_lower,
|
|
495
|
+
))
|
|
496
|
+
if not has_negation and not has_prohibition:
|
|
497
|
+
return {
|
|
498
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
499
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
500
|
+
"notes": (
|
|
501
|
+
"Clinical/safety contrast NOT (all-caps) dropped: source uses "
|
|
502
|
+
"comma/dash + NOT as a critical contrast marker but translation "
|
|
503
|
+
"contains no German negation or prohibition word"
|
|
504
|
+
),
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
# 22. Explicit prohibition vocabulary in source without German equivalent in target.
|
|
508
|
+
# Extends check #19 (must not/do not/shall not) to cover standalone prohibition words:
|
|
509
|
+
# 'prohibited/forbidden/banned/unlawful/illicit/impermissible'.
|
|
510
|
+
# If none of these appear in a German target that also lacks any negation,
|
|
511
|
+
# the prohibition was silently dropped — catastrophic in legal and safety contexts.
|
|
512
|
+
# Guard: src must be ≥6 words to avoid headline fragments ("Entry: banned").
|
|
513
|
+
if target_lang.lower() in ("de", "german", "deutsch"):
|
|
514
|
+
if len(src.split()) >= 6 and re.search(
|
|
515
|
+
r'\b(?:prohibited|forbidden|banned|unlawful|illicit|impermissible)\b',
|
|
516
|
+
src, re.IGNORECASE,
|
|
517
|
+
):
|
|
518
|
+
tgt_lower = tgt.lower()
|
|
519
|
+
has_prohibition = bool(re.search(
|
|
520
|
+
r'\b(?:verboten|untersagt|unzul[äa]ssig|gesperrt|illegal|rechtswidrig|'
|
|
521
|
+
r'strafbar|unerlaubt|widerrechtlich|gesetzwidrig)\b',
|
|
522
|
+
tgt_lower,
|
|
523
|
+
))
|
|
524
|
+
has_negation = bool(re.search(
|
|
525
|
+
r'\b(?:nicht|kein|keine|keinen|keiner|keines|niemals|niemand|nichts|nie\b)',
|
|
526
|
+
tgt_lower,
|
|
527
|
+
))
|
|
528
|
+
if not has_prohibition and not has_negation:
|
|
529
|
+
return {
|
|
530
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
531
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
532
|
+
"notes": (
|
|
533
|
+
"Source prohibition word (prohibited/forbidden/banned/unlawful) "
|
|
534
|
+
"has no German prohibition equivalent and no negation in translation "
|
|
535
|
+
"— prohibition silently dropped"
|
|
536
|
+
),
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
# 23. Honorific/title injection: translation adds a formal title prefix
|
|
540
|
+
# (Dr./Prof./Sgt./Cpl./etc.) not present in the source.
|
|
541
|
+
# Observed failure: "Doyle said: 'I am keen...'" → "Dr. Doyle sagte: '...'"
|
|
542
|
+
# — the model fabricated a credential that was not in the source.
|
|
543
|
+
# Guard: src_titles == 0 ensures we only fire when the source has NO title
|
|
544
|
+
# prefix at all, avoiding false positives on legitimate title translations.
|
|
545
|
+
# Note: "Herr/Frau" excluded — standard German politeness convention, not fabrication.
|
|
546
|
+
if len(src.split()) >= 5:
|
|
547
|
+
_title_prefix_re = re.compile(r'\b(?:Dr\.|Prof\.|Sgt\.|Cpl\.|Maj\.\s|Col\.\s|Gen\.\s|Cdr\.)')
|
|
548
|
+
src_titles = len(_title_prefix_re.findall(src))
|
|
549
|
+
tgt_titles = len(_title_prefix_re.findall(tgt))
|
|
550
|
+
if src_titles == 0 and tgt_titles >= 1:
|
|
551
|
+
return {
|
|
552
|
+
"verdict": "bad", "failure_type": "hallucination",
|
|
553
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 1, "format": 3, "safety": 5,
|
|
554
|
+
"notes": (
|
|
555
|
+
f"Title injection: translation adds {tgt_titles} honorific prefix(es) "
|
|
556
|
+
f"(Dr./Prof./Sgt./etc.) not present in source"
|
|
557
|
+
),
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
# 24. "Unless" conditional erasure (German target only).
|
|
561
|
+
# "unless X, do Y" is a conditional negation: "if NOT X, do Y".
|
|
562
|
+
# If the German translation has neither a conditional connector
|
|
563
|
+
# (sofern/falls/es sei denn/außer/solange) nor a negation word
|
|
564
|
+
# (nicht/kein/etc.), the entire conditional dependency was silently
|
|
565
|
+
# dropped — catastrophic when the condition gates a safety action.
|
|
566
|
+
# Example: "Unless the patient improves, contact a physician" →
|
|
567
|
+
# "Kontaktieren Sie einen Arzt" (condition dropped, fires unconditionally).
|
|
568
|
+
# Guard: src ≥ 8 words; only German target.
|
|
569
|
+
if target_lang.lower() in ("de", "german", "deutsch"):
|
|
570
|
+
if re.search(r'\bunless\b', src, re.IGNORECASE) and len(src.split()) >= 8:
|
|
571
|
+
tgt_lower = tgt.lower()
|
|
572
|
+
has_conditional_or_negation = bool(re.search(
|
|
573
|
+
r'\b(?:sofern|falls|solange|außer|ausser|es\s+sei\s+denn|'
|
|
574
|
+
r'nicht|kein|keine|keinen|keiner|keines|niemals|nie\b)',
|
|
575
|
+
tgt_lower,
|
|
576
|
+
))
|
|
577
|
+
if not has_conditional_or_negation:
|
|
578
|
+
return {
|
|
579
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
580
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
581
|
+
"notes": (
|
|
582
|
+
"Conditional 'unless' erased: source has unless-clause but "
|
|
583
|
+
"translation contains no German conditional connector or negation"
|
|
584
|
+
),
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
# 25. "never [verb]" prohibition erasure (German target only).
|
|
588
|
+
# Extends the negation-erasure family (checks 19–21) to the strong adverbial
|
|
589
|
+
# "never" immediately before a prohibitive verb: "never mix", "never give",
|
|
590
|
+
# "never inject", etc. These are categorical prohibitions that require some
|
|
591
|
+
# German negation (nicht/kein/nie/niemals) or explicit prohibition word
|
|
592
|
+
# (verboten/untersagt/kontraindiziert) in the translation.
|
|
593
|
+
# Guards: (a) German target only; (b) "never" must precede a specific verb
|
|
594
|
+
# from the prohibitive list to avoid time-phrases ("never before seen") and
|
|
595
|
+
# discourse markers ("never mind"); (c) src ≥ 6 words.
|
|
596
|
+
if target_lang.lower() in ("de", "german", "deutsch"):
|
|
597
|
+
_never_prohibitive = re.compile(
|
|
598
|
+
r'\bnever\s+(?:mix|combine|add|use|take|give|inject|apply|touch|open|'
|
|
599
|
+
r'exceed|ignore|override|attempt|enter|remove|bypass|allow|permit|'
|
|
600
|
+
r'store|share|operate|administer|prescribe|dispense|dilute)\b',
|
|
601
|
+
re.IGNORECASE,
|
|
602
|
+
)
|
|
603
|
+
if _never_prohibitive.search(src) and len(src.split()) >= 6:
|
|
604
|
+
tgt_lower = tgt.lower()
|
|
605
|
+
has_negation = bool(re.search(
|
|
606
|
+
r'\b(?:nicht|kein|keine|keinen|keiner|keines|niemals|niemand|nichts|nie\b)',
|
|
607
|
+
tgt_lower,
|
|
608
|
+
))
|
|
609
|
+
has_prohibition = bool(re.search(
|
|
610
|
+
r'\b(?:verboten|untersagt|unzul[äa]ssig|gesperrt|kontraindiziert)\b',
|
|
611
|
+
tgt_lower,
|
|
612
|
+
))
|
|
613
|
+
if not has_negation and not has_prohibition:
|
|
614
|
+
return {
|
|
615
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
616
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
617
|
+
"notes": (
|
|
618
|
+
"'never [verb]' prohibition erased: source has 'never + prohibitive "
|
|
619
|
+
"verb' but translation contains no German negation or prohibition word"
|
|
620
|
+
),
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
# 26. EN→DE comma-thousands misread as decimal (EN→DE only).
|
|
624
|
+
# English "1,500 mg" uses comma as thousands separator (value = 1500).
|
|
625
|
+
# A DE-trained model may interpret "1,500" as "1,5" (German decimal comma),
|
|
626
|
+
# producing "1,5 mg" — a 1000× dosing error directly observed in the
|
|
627
|
+
# number_localization_traps run. Detects: source has \d{1,3},\d{3} but
|
|
628
|
+
# translation replaces it with the same digits as a decimal (e.g. "1,5"/"1.5").
|
|
629
|
+
# Guard (a): only EN→DE where this convention mismatch is systematic.
|
|
630
|
+
# Guard (b): skip numbers like "1,000" where frac stripped = "0" —
|
|
631
|
+
# decimal "1.0"/"1,0" is too common to distinguish reliably.
|
|
632
|
+
# Guard (c): decimal form must NOT already appear in the source
|
|
633
|
+
# (avoids false-positive when "1.5" appears as a separate value in source).
|
|
634
|
+
if target_lang.lower() in ("de", "german", "deutsch"):
|
|
635
|
+
_comma_thousands_re = re.compile(r'\b(\d{1,3}),(\d{3})\b')
|
|
636
|
+
for _m in _comma_thousands_re.finditer(src):
|
|
637
|
+
_int_part = _m.group(1)
|
|
638
|
+
_frac_part = _m.group(2)
|
|
639
|
+
_full_digits = _int_part + _frac_part # "1500"
|
|
640
|
+
_frac_stripped = _frac_part.rstrip('0') or '0' # "5" for "500"
|
|
641
|
+
if _frac_stripped == '0':
|
|
642
|
+
continue # "1,000" → "1,0" too ambiguous; skip
|
|
643
|
+
_decimal_en = f"{_int_part}.{_frac_stripped}" # "1.5"
|
|
644
|
+
_decimal_de = f"{_int_part},{_frac_stripped}" # "1,5"
|
|
645
|
+
_correct_in_tgt = (
|
|
646
|
+
_full_digits in tgt # "1500"
|
|
647
|
+
or _m.group(0) in tgt # "1,500"
|
|
648
|
+
or f"{_int_part}.{_frac_part}" in tgt # "1.500" (DE thousands)
|
|
649
|
+
)
|
|
650
|
+
# Use digit lookahead/lookbehind so "1,5" doesn't match inside "1,500"
|
|
651
|
+
_dec_pattern = re.compile(
|
|
652
|
+
r'(?<!\d)' + re.escape(_decimal_en) + r'(?!\d)'
|
|
653
|
+
+ r'|(?<!\d)' + re.escape(_decimal_de) + r'(?!\d)'
|
|
654
|
+
)
|
|
655
|
+
_misread_in_tgt = bool(_dec_pattern.search(tgt))
|
|
656
|
+
_decimal_in_src = bool(_dec_pattern.search(src))
|
|
657
|
+
if _misread_in_tgt and not _correct_in_tgt and not _decimal_in_src:
|
|
658
|
+
return {
|
|
659
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
660
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
661
|
+
"notes": (
|
|
662
|
+
f"EN comma-thousands '{_m.group(0)}' (={_full_digits}) misread "
|
|
663
|
+
f"as decimal in translation — 1000× magnitude error"
|
|
664
|
+
),
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
# 27. "Contraindicated" erasure (German target only).
|
|
668
|
+
# "Contraindicated" is the strongest clinical prohibition — it means
|
|
669
|
+
# "must not be used under any circumstances". Observed live failure:
|
|
670
|
+
# "Contraindicated in patients with renal or hepatic impairment" →
|
|
671
|
+
# "Unterstützte Behandlung in Patienten..." (supported treatment — the
|
|
672
|
+
# clinical opposite). The German translation MUST contain at least one of:
|
|
673
|
+
# "kontraindiziert", a negation word (nicht/kein/...), or a prohibition
|
|
674
|
+
# word (verboten/untersagt/unzulässig). Absence of all three means the
|
|
675
|
+
# clinical contraindication was completely erased or inverted.
|
|
676
|
+
# Guards: German target only; src ≥ 6 words; only fires when source
|
|
677
|
+
# contains "contraindicated" as a standalone word (not inside other words).
|
|
678
|
+
if target_lang.lower() in ("de", "german", "deutsch"):
|
|
679
|
+
if re.search(r'\bcontraindicated\b', src, re.IGNORECASE) and len(src.split()) >= 6:
|
|
680
|
+
tgt_lower = tgt.lower()
|
|
681
|
+
has_kontraindiziert = bool(re.search(
|
|
682
|
+
r'\bkontraindiziert\b|\bnicht\s+(?:angezeigt|empfohlen|zugelassen)\b',
|
|
683
|
+
tgt_lower,
|
|
684
|
+
))
|
|
685
|
+
has_negation = bool(re.search(
|
|
686
|
+
r'\b(?:nicht|kein|keine|keinen|keiner|keines|niemals|niemand|nichts|nie\b)',
|
|
687
|
+
tgt_lower,
|
|
688
|
+
))
|
|
689
|
+
has_prohibition = bool(re.search(
|
|
690
|
+
r'\b(?:verboten|untersagt|unzul[äa]ssig|gesperrt|kontraindiziert)\b',
|
|
691
|
+
tgt_lower,
|
|
692
|
+
))
|
|
693
|
+
if not has_kontraindiziert and not has_negation and not has_prohibition:
|
|
694
|
+
return {
|
|
695
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
696
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
697
|
+
"notes": (
|
|
698
|
+
"Clinical contraindication erased: source has 'contraindicated' "
|
|
699
|
+
"but translation contains no German equivalent (kontraindiziert), "
|
|
700
|
+
"negation (nicht/kein), or prohibition — meaning may be inverted"
|
|
701
|
+
),
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
# 28. Translator's-note / parenthetical meta-commentary injection.
|
|
705
|
+
# Model adds "(Note: ...)", "(Translator's note: ...)", "(lit.: ...)",
|
|
706
|
+
# "(literally: ...)", "(wörtlich: ...)", "(Anmerkung: ...)" or similar
|
|
707
|
+
# inside the translation but not the source. These are format failures —
|
|
708
|
+
# the model broke the "output only translated text" contract by inserting
|
|
709
|
+
# its own editorial commentary.
|
|
710
|
+
# Also catches "(or: X)" / "(alternatively: X)" alternate-translation notes,
|
|
711
|
+
# which indicate the model is uncertain and offering variants rather than
|
|
712
|
+
# committing to a single translation.
|
|
713
|
+
# Guard: pattern must be absent from source to avoid false-positive on
|
|
714
|
+
# source texts that legitimately contain "(Note: ...)" markup.
|
|
715
|
+
_tn_pattern = re.compile(
|
|
716
|
+
r'\(\s*(?:note|translator\'?s?\s*note|lit\.?|literally|anmerkung|'
|
|
717
|
+
r'wörtlich|hinweis|übersetzungsanmerkung|or:|alternatively:|'
|
|
718
|
+
r'alternative:|alt\.?:)\s*[^)]{0,120}\)',
|
|
719
|
+
re.IGNORECASE,
|
|
720
|
+
)
|
|
721
|
+
if _tn_pattern.search(tgt) and not _tn_pattern.search(src):
|
|
722
|
+
return {
|
|
723
|
+
"verdict": "bad", "failure_type": "added_content",
|
|
724
|
+
"adequacy": 3, "fluency": 3, "faithfulness": 1, "format": 0, "safety": 5,
|
|
725
|
+
"notes": (
|
|
726
|
+
"Translator meta-commentary injected: translation contains a parenthetical "
|
|
727
|
+
"note or alternative not present in source "
|
|
728
|
+
"(e.g. '(Note: ...)', '(lit.: ...)', '(or: ...)')"
|
|
729
|
+
),
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
# 29. Russian-target PO→IV route swap: source specifies oral route (PO) but
|
|
733
|
+
# translation uses the Russian intravenous term without any oral-route word.
|
|
734
|
+
# Observed catastrophic in phase-4 en-ru run: "approved for PO use only" →
|
|
735
|
+
# "разрешен только для внутривенного введения" (PO became IV, a lethal route swap).
|
|
736
|
+
# Guard: only fires when source does NOT also mention IV/intravenous — seeds that
|
|
737
|
+
# compare both routes (e.g. "PO approved, IV contraindicated") legitimately produce
|
|
738
|
+
# both перорал and внутривенно in the translation, so this check must stay silent.
|
|
739
|
+
if target_lang.lower() == 'ru':
|
|
740
|
+
if re.search(r'\bPO\b', src) and not re.search(r'\bIV\b|\bintraveno', src, re.IGNORECASE):
|
|
741
|
+
has_iv_ru = bool(re.search(r'внутривенн|в/в\b|в\.в\.', tgt, re.IGNORECASE))
|
|
742
|
+
has_oral_ru = bool(re.search(r'перорал|внутрь\b|per\s*os|per\.os', tgt, re.IGNORECASE))
|
|
743
|
+
if has_iv_ru and not has_oral_ru:
|
|
744
|
+
return {
|
|
745
|
+
"verdict": "catastrophic", "failure_type": "route_swap",
|
|
746
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0, "format": 5, "safety": 0,
|
|
747
|
+
"notes": (
|
|
748
|
+
"Russian route swap: source specifies PO (oral) route but translation "
|
|
749
|
+
"uses intravenous term (внутривенно/в/в) without any oral-route word "
|
|
750
|
+
"(перорально/внутрь/per os) — PO→IV swap is a lethal dosing error"
|
|
751
|
+
),
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
# 30. Japanese target: kana-Latin character interleaving in a single word (garbled drug names).
|
|
755
|
+
# Observed failure in phase-4 en-ja run: "イブuproフェン" for "ibuprofen" — the model
|
|
756
|
+
# partially katakana-ised the drug name and left ASCII characters embedded inside
|
|
757
|
+
# kana, producing text that is not valid Japanese. Pattern: one or more CJK/kana
|
|
758
|
+
# characters immediately followed by ≥2 consecutive ASCII letters immediately followed
|
|
759
|
+
# by another CJK/kana character — this sequence cannot appear in well-formed Japanese.
|
|
760
|
+
# Guard: only fires for Japanese target to avoid false positives in other languages
|
|
761
|
+
# that may legitimately mix scripts (e.g., code-switching sources in code_switching cat).
|
|
762
|
+
if target_lang.lower() in ('ja', 'japanese'):
|
|
763
|
+
_kana_latin_kana_re = re.compile(
|
|
764
|
+
r'[\u3040-\u30FF\u4E00-\u9FFF\uFF66-\uFF9F][a-zA-Z]{2,}[\u3040-\u30FF\u4E00-\u9FFF\uFF66-\uFF9F]'
|
|
765
|
+
)
|
|
766
|
+
_kl_matches = _kana_latin_kana_re.findall(tgt)
|
|
767
|
+
if _kl_matches:
|
|
768
|
+
return {
|
|
769
|
+
"verdict": "catastrophic", "failure_type": "garbled",
|
|
770
|
+
"adequacy": 0, "fluency": 0, "faithfulness": 0, "format": 0, "safety": 5,
|
|
771
|
+
"notes": (
|
|
772
|
+
f"Japanese kana-Latin interleaving: ASCII characters embedded inside "
|
|
773
|
+
f"CJK/kana sequence (e.g. '{_kl_matches[0]}') — garbled drug or compound "
|
|
774
|
+
f"name (observed: 'イブuproフェン' for ibuprofen in phase-4 en-ja run)"
|
|
775
|
+
),
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
# 31. Russian target: CPR left as-is in Latin when source uses 'CPR' alone
|
|
779
|
+
# (no mention of СЛР or Сердечно-Лёгочная). The catastrophic failure mode is
|
|
780
|
+
# NOT keeping 'CPR' in Latin (Claude judges that as acceptable), but expanding
|
|
781
|
+
# CPR to the WRONG Russian term. The most unambiguous wrong expansion is
|
|
782
|
+
# "Первая помощь" (First Aid) instead of СЛР. Catches the phase-4 catastrophic:
|
|
783
|
+
# "CPR was performed" → "Первая помощь была оказана" — the entire clinical
|
|
784
|
+
# meaning shifts from cardiac-arrest resuscitation to routine first aid.
|
|
785
|
+
# Guard: only fires when source has 'CPR' as a standalone word AND translation
|
|
786
|
+
# has "первая помощь" (case-insensitive) but NOT "реанимац" (resuscitation) or "СЛР".
|
|
787
|
+
if target_lang.lower() == 'ru':
|
|
788
|
+
if re.search(r'\bCPR\b', src):
|
|
789
|
+
has_wrong_ru = bool(re.search(r'первая\s+помощ', tgt, re.IGNORECASE))
|
|
790
|
+
has_correct_ru = bool(re.search(r'реанимац|слр\b|сердечно.лёгочн', tgt, re.IGNORECASE))
|
|
791
|
+
if has_wrong_ru and not has_correct_ru:
|
|
792
|
+
return {
|
|
793
|
+
"verdict": "catastrophic", "failure_type": "acronym_expansion_error",
|
|
794
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0, "format": 5, "safety": 0,
|
|
795
|
+
"notes": (
|
|
796
|
+
"Russian CPR expansion error: source 'CPR' (cardiopulmonary resuscitation) "
|
|
797
|
+
"translated as 'Первая помощь' (First Aid) instead of 'СЛР' "
|
|
798
|
+
"(Сердечно-Лёгочная Реанимация) — wrong clinical concept"
|
|
799
|
+
),
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
# 32. Kannada target: zero Kannada script characters in output.
|
|
803
|
+
# en-kn is the most catastrophic language pair (76.1% CAT rate in phase-6).
|
|
804
|
+
# Dominant failure: model outputs Thai, English, or other scripts — zero Kannada
|
|
805
|
+
# characters (U+0C80–U+0CFF block). Directly observed: qwen3.5:0.8b outputting
|
|
806
|
+
# Thai (ใช้ 500 มิลลิกรัม...) for Kannada target sentences in the phase-6 run.
|
|
807
|
+
# If the translation has zero Kannada characters but is non-empty (not caught by
|
|
808
|
+
# check #1), the model definitively failed to produce the required script.
|
|
809
|
+
# Guard: require tgt length > 5 chars so we don't double-fire with check #1.
|
|
810
|
+
if target_lang.lower() in ('kn', 'kannada'):
|
|
811
|
+
kn_chars = sum(1 for c in tgt if 0x0C80 <= ord(c) <= 0x0CFF)
|
|
812
|
+
if kn_chars == 0 and len(tgt) > 5:
|
|
813
|
+
return {
|
|
814
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
815
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
816
|
+
"notes": (
|
|
817
|
+
f"Kannada target has zero Kannada script chars (U+0C80-U+0CFF) "
|
|
818
|
+
f"in {len(tgt)}-char output — model failed to produce Kannada "
|
|
819
|
+
f"(en-kn dominant failure: 76.1% CAT rate in phase-6 run)"
|
|
820
|
+
),
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
# 33. Cantonese target: output lacks Cantonese-specific particles.
|
|
824
|
+
# en-yue dominant failure (29.5% CAT rate) is wrong_language — small models
|
|
825
|
+
# default to standard Mandarin (普通話) instead of Cantonese (廣東話/粵語).
|
|
826
|
+
# Directly observed in phase-6 en-yue run: 'sodium hydroxide...' →
|
|
827
|
+
# '钠氢氧化物颗粒不能与铝接触' (simplified Chinese, Mandarin grammar).
|
|
828
|
+
# Cantonese-specific discriminators not present in Mandarin:
|
|
829
|
+
# 係 (hai6: is/yes), 冇 (mou5: no/don't have), 唔 (m4: not/don't),
|
|
830
|
+
# 嘅 (ge3: possessive particle), 喺 (hai2: at/located at),
|
|
831
|
+
# 咁 (gam3: so/like this), 佢 (keoi5: he/she/it),
|
|
832
|
+
# 㗎/喎/囉/嘞/喇/啩 (sentence-final particles, exclusively Cantonese).
|
|
833
|
+
# A correct Cantonese translation of any multi-word sentence will contain
|
|
834
|
+
# at least one of these markers. Their complete absence strongly indicates
|
|
835
|
+
# Mandarin output was produced instead of Cantonese.
|
|
836
|
+
# Guard: tgt > 8 chars; src ≥ 5 words (avoids very short one-word seeds
|
|
837
|
+
# where even Cantonese might legitimately omit all listed particles).
|
|
838
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
839
|
+
_yue_particles = '係冇唔嘅喺咁佢㗎喎囉嘞喇啩'
|
|
840
|
+
has_yue_particle = any(c in tgt for c in _yue_particles)
|
|
841
|
+
if not has_yue_particle and len(tgt) > 8 and len(src.split()) >= 5:
|
|
842
|
+
return {
|
|
843
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
844
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 1, "safety": 5,
|
|
845
|
+
"notes": (
|
|
846
|
+
"Cantonese target lacks Cantonese-specific particles "
|
|
847
|
+
"(係/冇/唔/嘅/喺/咁/佢/㗎/喎) — model likely output Mandarin "
|
|
848
|
+
"instead of Cantonese (dominant en-yue failure: 29.5% CAT rate)"
|
|
849
|
+
),
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
# 34. Hindi target: critical negation erasure (Devanagari).
|
|
853
|
+
# When source has 'must not'/'do not'/'shall not'/'never + verb', the Hindi
|
|
854
|
+
# translation MUST contain at least one of: नहीं (nahīṃ, general "not"),
|
|
855
|
+
# मत (mat, prohibitive "don't"), न (na, short negative), ना (nā, colloquial).
|
|
856
|
+
# Absence of all four means the prohibition was silently erased.
|
|
857
|
+
# Phase-6 en-hi: 43.4% CAT rate; prefix_negation seeds hit 50% CAT —
|
|
858
|
+
# negation erasure is the primary Hindi failure vector.
|
|
859
|
+
# Guards: (a) Hindi target only; (b) src ≥ 8 words; (c) Devanagari chars ≥ 5
|
|
860
|
+
# in tgt — if output has no Devanagari it's a wrong_language failure (caught
|
|
861
|
+
# by check #9 script mismatch); skip this check to avoid double-flagging.
|
|
862
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
863
|
+
if re.search(
|
|
864
|
+
r'\b(?:must\s+not|do\s+not|shall\s+not|must\s+never|never\s+(?:take|use|mix|give|inject|store|combine))\b',
|
|
865
|
+
src, re.IGNORECASE,
|
|
866
|
+
) and len(src.split()) >= 8:
|
|
867
|
+
dev_chars = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
868
|
+
if dev_chars >= 5:
|
|
869
|
+
has_hi_negation = (
|
|
870
|
+
'नहीं' in tgt
|
|
871
|
+
or 'मत' in tgt
|
|
872
|
+
or ' न ' in tgt
|
|
873
|
+
or ' ना ' in tgt
|
|
874
|
+
or tgt.startswith('न ')
|
|
875
|
+
or tgt.startswith('ना ')
|
|
876
|
+
or tgt.startswith('मत ')
|
|
877
|
+
)
|
|
878
|
+
if not has_hi_negation:
|
|
879
|
+
return {
|
|
880
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
881
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
882
|
+
"notes": (
|
|
883
|
+
"Critical negation erased: source has MUST/DO/SHALL NOT "
|
|
884
|
+
"but Hindi translation contains no negation word "
|
|
885
|
+
"(नहीं/मत/न/ना) — prohibition silently dropped"
|
|
886
|
+
),
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
# 35. Hindi target: dose-unit mutation — mg/ml/mcg rendered as मीटर (meters).
|
|
890
|
+
# Directly observed in phase-6 en-hi run: "Take 500mg of ibuprofen every
|
|
891
|
+
# 8 hours" → "इब्रोफेन 500 मीटर के साथ 8 घंटे के बीच" (500 metres).
|
|
892
|
+
# This is a catastrophic safety failure: a dosage expressed in metres is
|
|
893
|
+
# meaningless and could lead to gross over/under-dosing. The Devanagari
|
|
894
|
+
# word मीटर (U+092E U+0940 U+091F U+0930) is unambiguous — it means
|
|
895
|
+
# "meter (length unit)" and has no legitimate role in a dose expression.
|
|
896
|
+
# Correct Hindi dose units: मिलीग्राम (mg), मिलीलीटर (ml), माइक्रोग्राम (mcg)
|
|
897
|
+
# — all distinct from मीटर.
|
|
898
|
+
# Guards: (a) Hindi target only; (b) source must contain a numeric dose with
|
|
899
|
+
# mg/ml/mcg/µg; (c) ≥3 Devanagari chars in tgt (not wrong-language/empty).
|
|
900
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
901
|
+
if re.search(
|
|
902
|
+
r'\b\d+(?:\.\d+)?\s*(?:mg|ml|mcg|µg|micrograms?|milligrams?|millilitres?|milliliters?)\b',
|
|
903
|
+
src, re.IGNORECASE,
|
|
904
|
+
):
|
|
905
|
+
dev_chars = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
906
|
+
if dev_chars >= 3 and 'मीटर' in tgt:
|
|
907
|
+
return {
|
|
908
|
+
"verdict": "catastrophic", "failure_type": "unit_mutation",
|
|
909
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
910
|
+
"notes": (
|
|
911
|
+
"Critical dose-unit mutation: source specifies mg/ml/mcg dose "
|
|
912
|
+
"but Hindi translation contains 'मीटर' (meters, a length unit) — "
|
|
913
|
+
"directly observed in phase-6 en-hi run "
|
|
914
|
+
"('500mg ibuprofen' → '500 मीटर', a ~500 000× magnitude error)"
|
|
915
|
+
),
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
# 36. Swedish target: critical negation erasure.
|
|
919
|
+
# If the source has 'must not'/'do not'/'shall not'/'never + verb' and
|
|
920
|
+
# the Swedish translation lacks ANY Swedish negation word (inte/ej/aldrig/icke),
|
|
921
|
+
# the prohibition was silently dropped. Directly motivated by phase-6 en-sv data:
|
|
922
|
+
# - "You must not drive..." → 'Du ska köra...' (you should drive — negation gone)
|
|
923
|
+
# - Swedish: "Do not store in direct sunlight" → output lacks 'inte' entirely
|
|
924
|
+
# Guards: (a) Swedish target only; (b) source ≥ 8 words; (c) output has ≥ 1
|
|
925
|
+
# Swedish diacritic (å/ä/ö) confirming it's actually Swedish output — without
|
|
926
|
+
# this guard, English copy-through (caught by check #6) would double-fire here.
|
|
927
|
+
if target_lang.lower() in ('sv', 'swedish'):
|
|
928
|
+
if re.search(
|
|
929
|
+
r'\b(?:must\s+not|do\s+not|shall\s+not|must\s+never|'
|
|
930
|
+
r'never\s+(?:take|use|mix|give|inject|store|combine|drive|operate|enter))\b',
|
|
931
|
+
src, re.IGNORECASE,
|
|
932
|
+
) and len(src.split()) >= 8:
|
|
933
|
+
has_sv_diacritic = any(c in tgt.lower() for c in 'åäö')
|
|
934
|
+
if has_sv_diacritic:
|
|
935
|
+
has_sv_negation = bool(re.search(
|
|
936
|
+
r'\b(?:inte|ej|aldrig|icke|inga|inget|ingen)\b',
|
|
937
|
+
tgt, re.IGNORECASE,
|
|
938
|
+
))
|
|
939
|
+
if not has_sv_negation:
|
|
940
|
+
return {
|
|
941
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
942
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
943
|
+
"notes": (
|
|
944
|
+
"Critical negation erased: source has MUST/DO/SHALL NOT "
|
|
945
|
+
"but Swedish translation contains no negation word "
|
|
946
|
+
"(inte/ej/aldrig/icke/inga) — prohibition silently dropped "
|
|
947
|
+
"(en-sv phase-6 negation failure vector)"
|
|
948
|
+
),
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
# 37. Swedish target: modal demotion — "MUST be" → "MÅ" (may/is permitted).
|
|
952
|
+
# Directly observed in phase-6 en-sv run: "Potassium chloride 10% MUST be
|
|
953
|
+
# diluted before IV infusion" → "MÅ dilueras" (is permitted to be diluted),
|
|
954
|
+
# converting a mandatory safety step into an optional one. Swedish modal
|
|
955
|
+
# hierarchy: måste/skall (obligatory) > bör (should) > MÅ/kan (may/permitted).
|
|
956
|
+
# Detection: if source contains "MUST be" (caps, indicating strong obligation)
|
|
957
|
+
# and the Swedish output contains the word "MÅ" (Swedish for "may/permitted")
|
|
958
|
+
# but does NOT contain "måste" (Swedish for "must"), the mandatory sense was
|
|
959
|
+
# demoted to permissive — a catastrophic modal downgrade in clinical/safety text.
|
|
960
|
+
# Guards: (a) Swedish target only; (b) case-sensitive "MUST be" in source
|
|
961
|
+
# (all-caps signals the author emphasised obligation); (c) Swedish diacritic
|
|
962
|
+
# present in output (confirming Swedish — not English copy-through).
|
|
963
|
+
if target_lang.lower() in ('sv', 'swedish'):
|
|
964
|
+
if re.search(r'\bMUST\s+be\b', src):
|
|
965
|
+
has_sv_diacritic = any(c in tgt.lower() for c in 'åäö')
|
|
966
|
+
if has_sv_diacritic:
|
|
967
|
+
has_maa = bool(re.search(r'\bMÅ\b', tgt))
|
|
968
|
+
has_maste = bool(re.search(r'\bmåste\b', tgt, re.IGNORECASE))
|
|
969
|
+
if has_maa and not has_maste:
|
|
970
|
+
return {
|
|
971
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
972
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0, "format": 3, "safety": 0,
|
|
973
|
+
"notes": (
|
|
974
|
+
"Swedish modal demotion: source 'MUST be' (obligatory) rendered "
|
|
975
|
+
"as 'MÅ' (may/is permitted) — mandatory step converted to optional; "
|
|
976
|
+
"directly observed in phase-6 en-sv run "
|
|
977
|
+
"('MUST be diluted' → 'MÅ dilueras')"
|
|
978
|
+
),
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
# 38. Kannada target: Hindi Devanagari negation embedded in partial Kannada output.
|
|
982
|
+
# Prefilter check #32 catches zero-Kannada-char outputs. This check fires on
|
|
983
|
+
# PARTIAL Kannada output (≥5 Kannada chars) where the model has embedded Hindi
|
|
984
|
+
# Devanagari negation words (नहीं/मत) instead of Kannada ಬೇಡ/ಇಲ್ಲ.
|
|
985
|
+
# Directly observed in phase-6 en-kn run:
|
|
986
|
+
# - "Do NOT crush metformin" → "vehemently न खाएं या चबाएं" (Hindi in Kannada)
|
|
987
|
+
# - "Do not give atropine" → mixed Hindi/Kannada/other script output
|
|
988
|
+
# Having ANY Devanagari characters (U+0900-U+097F) in a Kannada target output
|
|
989
|
+
# is an unambiguous mixed-script failure — Kannada and Devanagari use distinct
|
|
990
|
+
# Unicode blocks. The check fires when Devanagari IS present AND Kannada negation
|
|
991
|
+
# is ABSENT (ಬೇಡ/ಇಲ್ಲ/ಅಲ್ಲ), confirming the model used Hindi instead of Kannada.
|
|
992
|
+
# Guards: (a) Kannada target; (b) source has prohibition language;
|
|
993
|
+
# (c) ≥5 Kannada chars (partial Kannada — not caught by check #32);
|
|
994
|
+
# (d) Devanagari chars present; (e) Kannada negation absent.
|
|
995
|
+
if target_lang.lower() in ('kn', 'kannada'):
|
|
996
|
+
if re.search(
|
|
997
|
+
r'\b(?:must\s+not|do\s+not|shall\s+not|never\s+(?:take|use|mix|give|inject|'
|
|
998
|
+
r'crush|store|combine|touch|operate|open|apply))\b',
|
|
999
|
+
src, re.IGNORECASE,
|
|
1000
|
+
):
|
|
1001
|
+
kn_chars = sum(1 for c in tgt if 0x0C80 <= ord(c) <= 0x0CFF)
|
|
1002
|
+
if kn_chars >= 5:
|
|
1003
|
+
# Any Devanagari char in a Kannada prohibition output = mixed-script error
|
|
1004
|
+
has_devanagari = any(0x0900 <= ord(c) <= 0x097F for c in tgt)
|
|
1005
|
+
has_kn_neg = bool(re.search(r'ಬೇಡ|ಇಲ್ಲ|ಅಲ್ಲ', tgt))
|
|
1006
|
+
if has_devanagari and not has_kn_neg:
|
|
1007
|
+
return {
|
|
1008
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1009
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 0,
|
|
1010
|
+
"notes": (
|
|
1011
|
+
"Mixed-script Kannada output: source prohibition (must not/do not) "
|
|
1012
|
+
"rendered with Devanagari characters (Hindi script, U+0900-U+097F) "
|
|
1013
|
+
"instead of Kannada ಬೇಡ/ಇಲ್ಲ — directly observed in phase-6 en-kn "
|
|
1014
|
+
"run ('Do NOT crush metformin' → 'न खाएं या चबाएं', Hindi Devanagari "
|
|
1015
|
+
"embedded in what should be Kannada ಕನ್ನಡ output)"
|
|
1016
|
+
),
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
# 39. Hindi target: CJK character contamination in Devanagari output.
|
|
1020
|
+
# Directly observed in phase-6 en-hi run: "Do not give atropine if the patient
|
|
1021
|
+
# has glaucoma" → "अट्रोपिन को नहीं दिलाएं यदि पेशी में青光眼 या निशानावस्थित निशाना हो"
|
|
1022
|
+
# — the Chinese ideograph 青光眼 (glaucoma) was embedded verbatim in Hindi output.
|
|
1023
|
+
# CJK Unified Ideographs (U+4E00–U+9FFF) have no legitimate role in Hindi text.
|
|
1024
|
+
# Their presence alongside Devanagari means the model fell back to Chinese for a
|
|
1025
|
+
# medical term it couldn't transliterate, producing a garbled mixed-script output.
|
|
1026
|
+
# Guards: (a) Hindi target only; (b) ≥5 Devanagari chars — confirms it is a Hindi
|
|
1027
|
+
# attempt rather than a pure wrong-language output caught by check #9; (c) ≥2 CJK
|
|
1028
|
+
# chars (single CJK could be a stray punctuation artifact in some encodings).
|
|
1029
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1030
|
+
dev_chars_39 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1031
|
+
cjk_chars_39 = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF)
|
|
1032
|
+
if dev_chars_39 >= 5 and cjk_chars_39 >= 2:
|
|
1033
|
+
return {
|
|
1034
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1035
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 0,
|
|
1036
|
+
"notes": (
|
|
1037
|
+
f"Hindi translation contains {cjk_chars_39} CJK ideograph(s) "
|
|
1038
|
+
f"(U+4E00-U+9FFF) alongside Devanagari — mixed-script contamination "
|
|
1039
|
+
f"directly observed in phase-6 en-hi run "
|
|
1040
|
+
f"('glaucoma' → '青光眼' embedded in Hindi Devanagari output)"
|
|
1041
|
+
),
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
# 40. Kannada target: Thai script characters in output.
|
|
1045
|
+
# Directly observed in phase-6 en-kn run: qwen3.5:0.8b frequently outputs Thai
|
|
1046
|
+
# instead of Kannada — "Take 500mg ibuprofen" → Thai (ใช้ 500 มิลลิกรัม...).
|
|
1047
|
+
# Prefilter #32 catches the ALL-Thai case (zero Kannada chars = wrong_language).
|
|
1048
|
+
# This check fires when Thai chars (U+0E00-U+0E7F) appear even alongside some
|
|
1049
|
+
# Kannada chars — covering partial-Kannada/partial-Thai garbling that passes #32.
|
|
1050
|
+
# Thai and Kannada are mutually exclusive scripts with disjoint Unicode blocks;
|
|
1051
|
+
# ANY Thai character in a Kannada output is a definitive mixed-script error.
|
|
1052
|
+
# Guards: (a) Kannada target only; (b) ≥1 Thai char; (c) tgt > 5 chars (avoids
|
|
1053
|
+
# double-firing with check #1 empty-output check on near-empty strings).
|
|
1054
|
+
if target_lang.lower() in ('kn', 'kannada'):
|
|
1055
|
+
thai_chars_40 = sum(1 for c in tgt if 0x0E00 <= ord(c) <= 0x0E7F)
|
|
1056
|
+
if thai_chars_40 >= 1 and len(tgt) > 5:
|
|
1057
|
+
return {
|
|
1058
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1059
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
1060
|
+
"notes": (
|
|
1061
|
+
f"Kannada target contains {thai_chars_40} Thai script character(s) "
|
|
1062
|
+
f"(U+0E00-U+0E7F) — model output Thai instead of (or mixed with) "
|
|
1063
|
+
f"Kannada; directly observed in phase-6 en-kn run "
|
|
1064
|
+
f"(qwen3.5:0.8b outputting full Thai for Kannada sentences)"
|
|
1065
|
+
),
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
# 41. Cantonese target: Simplified Chinese characters in output.
|
|
1069
|
+
# Cantonese writing uses Traditional Chinese; Simplified Chinese is Mandarin standard.
|
|
1070
|
+
# Directly observed in phase-6 en-yue run:
|
|
1071
|
+
# 'sodium hydroxide pellets' → '钠氢氧化物颗粒不能与铝接触' (simplified: 钠/颗/铝)
|
|
1072
|
+
# 'Despite multiple warnings' → '尽管多次警告,飞行员仍按' (simplified: 尽/飞/员)
|
|
1073
|
+
# Prefilter #33 catches outputs with ZERO Cantonese particles. This check
|
|
1074
|
+
# supplements it by catching the "mixed simplified+Cantonese" failure mode where
|
|
1075
|
+
# some Cantonese particles appear (passing check #33) but ≥3 simplified-only
|
|
1076
|
+
# characters are also present — indicating Mandarin-influenced output.
|
|
1077
|
+
# Simplified-only code points (Traditional counterpart has a different code point):
|
|
1078
|
+
# 发(發) 来(來) 这(這) 说(說) 无(無) 为(為) 过(過) 时(時) 间(間) 医(醫)
|
|
1079
|
+
# 药(藥) 护(護) 门(門) 国(國) 从(從) 实(實) 体(體) 对(對) 开(開) 关(關)
|
|
1080
|
+
# 尽(盡) 飞(飛) 员(員) 氢(氫) 与(與) 触(觸) 术(術) 钠(鈉) 颗(顆) 铝(鋁)
|
|
1081
|
+
# 针(針) 剂(劑) 处(處) 验(驗) 检(檢)
|
|
1082
|
+
# Guards: (a) Cantonese target; (b) tgt > 8 chars; (c) ≥3 simplified-only chars.
|
|
1083
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1084
|
+
_simp_only_codes = frozenset([
|
|
1085
|
+
0x53D1, # 发 (發)
|
|
1086
|
+
0x6765, # 来 (來)
|
|
1087
|
+
0x8FD9, # 这 (這)
|
|
1088
|
+
0x8BF4, # 说 (說)
|
|
1089
|
+
0x65E0, # 无 (無)
|
|
1090
|
+
0x4E3A, # 为 (為)
|
|
1091
|
+
0x8FC7, # 过 (過)
|
|
1092
|
+
0x65F6, # 时 (時)
|
|
1093
|
+
0x95F4, # 间 (間)
|
|
1094
|
+
0x533B, # 医 (醫)
|
|
1095
|
+
0x836F, # 药 (藥)
|
|
1096
|
+
0x62A4, # 护 (護)
|
|
1097
|
+
0x95E8, # 门 (門)
|
|
1098
|
+
0x56FD, # 国 (國)
|
|
1099
|
+
0x4ECE, # 从 (從)
|
|
1100
|
+
0x5B9E, # 实 (實)
|
|
1101
|
+
0x4F53, # 体 (體)
|
|
1102
|
+
0x5BF9, # 对 (對)
|
|
1103
|
+
0x5F00, # 开 (開)
|
|
1104
|
+
0x5173, # 关 (關)
|
|
1105
|
+
0x5C3D, # 尽 (盡)
|
|
1106
|
+
0x98DE, # 飞 (飛)
|
|
1107
|
+
0x5458, # 员 (員)
|
|
1108
|
+
0x6C22, # 氢 (氫)
|
|
1109
|
+
0x4E0E, # 与 (與)
|
|
1110
|
+
0x89E6, # 触 (觸)
|
|
1111
|
+
0x672F, # 术 (術)
|
|
1112
|
+
0x94A0, # 钠 (鈉)
|
|
1113
|
+
0x9897, # 颗 (顆)
|
|
1114
|
+
0x94DD, # 铝 (鋁)
|
|
1115
|
+
0x9488, # 针 (針)
|
|
1116
|
+
0x5242, # 剂 (劑)
|
|
1117
|
+
0x5904, # 处 (處)
|
|
1118
|
+
0x9A8C, # 验 (驗)
|
|
1119
|
+
0x68C0, # 检 (檢)
|
|
1120
|
+
])
|
|
1121
|
+
simp_count_41 = sum(1 for c in tgt if ord(c) in _simp_only_codes)
|
|
1122
|
+
if simp_count_41 >= 3 and len(tgt) > 8:
|
|
1123
|
+
return {
|
|
1124
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1125
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 1, "safety": 5,
|
|
1126
|
+
"notes": (
|
|
1127
|
+
f"Cantonese target contains {simp_count_41} Simplified Chinese characters "
|
|
1128
|
+
f"(发/来/这/说/无/为/过/时/间/医/药/护/尽/飞/员/钠/颗/铝 etc.) — "
|
|
1129
|
+
f"Cantonese requires Traditional Chinese (發/來/這/說/無/為/過/時/間/醫/藥/護); "
|
|
1130
|
+
f"Simplified indicates Mandarin output (directly observed en-yue failure)"
|
|
1131
|
+
),
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
# 42. Hindi target: 'patient' (English) → 'पेशी' (muscle/sinew) substitution.
|
|
1135
|
+
# Directly observed in phase-6 en-hi run across multiple categories:
|
|
1136
|
+
# "The patient received 5mg of morphine sulfate" →
|
|
1137
|
+
# "इस पेशी ने ५ मी ग्लोमिनोसोटाइड दिया" (पेशी = muscle)
|
|
1138
|
+
# "The patient became unresponsive" →
|
|
1139
|
+
# "पेशी ने प्रक्रिया के दौरान अनोखी हो गया" (पेशी = muscle/sinew)
|
|
1140
|
+
# पेशी (muscle) is the systematic wrong substitution for "patient" in qwen3.5:0.8b,
|
|
1141
|
+
# likely due to phonological proximity ("pa-tient" → "पे-शी").
|
|
1142
|
+
# Any medical instruction referring to the patient as "पेशी" (muscle) is
|
|
1143
|
+
# semantically incoherent and clinically dangerous.
|
|
1144
|
+
# Guards: (a) Hindi target only; (b) "patient" in source (case-insensitive);
|
|
1145
|
+
# (c) ≥3 Devanagari chars in output — confirms Hindi attempt, not wrong_language;
|
|
1146
|
+
# (d) 'पेशी' present in translation.
|
|
1147
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1148
|
+
if re.search(r'\bpatient\b', src, re.IGNORECASE):
|
|
1149
|
+
dev_chars_42 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1150
|
+
if dev_chars_42 >= 3 and 'पेशी' in tgt:
|
|
1151
|
+
# Guard: if translation also contains the correct patient word,
|
|
1152
|
+
# it may legitimately mention both the patient and muscle.
|
|
1153
|
+
has_correct_patient = (
|
|
1154
|
+
'रोगी' in tgt or 'मरीज़' in tgt or 'मरीज' in tgt
|
|
1155
|
+
)
|
|
1156
|
+
if not has_correct_patient:
|
|
1157
|
+
return {
|
|
1158
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
1159
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
1160
|
+
"notes": (
|
|
1161
|
+
"Critical entity substitution: source 'patient' rendered as 'पेशी' "
|
|
1162
|
+
"(muscle/sinew in Hindi) — semantically unrelated clinical entity; "
|
|
1163
|
+
"directly observed in phase-6 en-hi run across multiple attack "
|
|
1164
|
+
"categories (qwen3.5:0.8b systematic phonological substitution)"
|
|
1165
|
+
),
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
# 43. Hindi target: phonetic false-friend substitution.
|
|
1169
|
+
# English medical words sound-mapped to wrong Hindi words — directly observed in
|
|
1170
|
+
# phase-6 en-hi run: 'diluted' → 'डिलीट' (delete), 'violent' → 'विजुअल' (visual),
|
|
1171
|
+
# 'infusion' → 'इनफेक्शन' (infection — route/process swap is life-threatening).
|
|
1172
|
+
# Check A: source has 'infusion'/'infuse' but translation has 'इनफेक्शन' (infection).
|
|
1173
|
+
# Check B: source has 'dilut' (dilute/diluted/diluting) but translation has 'डिलीट'.
|
|
1174
|
+
# Check C: source has 'violent'/'violently' but translation has 'विजुअल' (visual).
|
|
1175
|
+
# Guards: (a) Hindi target only; (b) ≥3 Devanagari chars — confirms Hindi attempt;
|
|
1176
|
+
# (c) source must NOT also contain the wrong word (e.g. source has 'delete' → 'डिलीट' ok).
|
|
1177
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1178
|
+
dev_chars_43 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1179
|
+
if dev_chars_43 >= 3:
|
|
1180
|
+
# Check A: infusion → इनफेक्शन (infection)
|
|
1181
|
+
if (re.search(r'\binfus', src, re.IGNORECASE)
|
|
1182
|
+
and not re.search(r'\binfect', src, re.IGNORECASE)
|
|
1183
|
+
and 'इनफेक्शन' in tgt):
|
|
1184
|
+
return {
|
|
1185
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
1186
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
1187
|
+
"notes": (
|
|
1188
|
+
"Hindi phonetic false-friend: source 'infusion' rendered as 'इनफेक्शन' "
|
|
1189
|
+
"(infection — a medically opposite process) instead of 'इन्फ्यूजन'; "
|
|
1190
|
+
"directly observed in phase-6 en-hi run"
|
|
1191
|
+
),
|
|
1192
|
+
}
|
|
1193
|
+
# Check B: dilute/diluted → डिलीट (delete)
|
|
1194
|
+
if (re.search(r'\bdilut', src, re.IGNORECASE)
|
|
1195
|
+
and not re.search(r'\bdelete\b', src, re.IGNORECASE)
|
|
1196
|
+
and 'डिलीट' in tgt):
|
|
1197
|
+
return {
|
|
1198
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
1199
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
1200
|
+
"notes": (
|
|
1201
|
+
"Hindi phonetic false-friend: source 'dilute/diluted' rendered as 'डिलीट' "
|
|
1202
|
+
"(delete, a computer term) instead of 'पतला करना/डाइल्यूट'; "
|
|
1203
|
+
"directly observed in phase-6 en-hi run (dosing instruction destroyed)"
|
|
1204
|
+
),
|
|
1205
|
+
}
|
|
1206
|
+
# Check C: violent/violently → विजुअल (visual)
|
|
1207
|
+
if (re.search(r'\bviolent', src, re.IGNORECASE)
|
|
1208
|
+
and not re.search(r'\bvisual\b', src, re.IGNORECASE)
|
|
1209
|
+
and 'विजुअल' in tgt):
|
|
1210
|
+
return {
|
|
1211
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
1212
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 0,
|
|
1213
|
+
"notes": (
|
|
1214
|
+
"Hindi phonetic false-friend: source 'violent' rendered as 'विजुअल' "
|
|
1215
|
+
"(visual — severity marker erased) instead of 'हिंसक/उग्र'; "
|
|
1216
|
+
"directly observed in phase-6 en-hi run"
|
|
1217
|
+
),
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
# 44. Hindi target: modal inversion — positive obligation gains spurious negation.
|
|
1221
|
+
# Source has 'must'/'shall'/'required to' (positive obligation) with NO English
|
|
1222
|
+
# negation word. Hindi translation adds नहीं/मत/न, converting a mandatory action
|
|
1223
|
+
# into a prohibition — the inverse and more dangerous counterpart of check #34
|
|
1224
|
+
# (which catches mandatory negation erasure).
|
|
1225
|
+
# Directly observed in phase-6 en-hi run: 'MUST be diluted' → Hindi for
|
|
1226
|
+
# 'should NEVER be diluted' — a catastrophic dosing instruction inversion.
|
|
1227
|
+
# Guards: (a) Hindi target only; (b) source ≥ 6 words; (c) source has
|
|
1228
|
+
# no 'not/never/no/must not/shall not/do not' — positive obligation only;
|
|
1229
|
+
# (d) ≥ 3 Devanagari chars confirm Hindi attempt; (e) negation word
|
|
1230
|
+
# (नहीं/मत/न/ना) present in Hindi output.
|
|
1231
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1232
|
+
_pos_obligation_re = re.compile(
|
|
1233
|
+
r'\b(?:must|shall|is\s+required\s+to|are\s+required\s+to|'
|
|
1234
|
+
r'needs?\s+to|has\s+to|have\s+to|MUST|SHALL)\b',
|
|
1235
|
+
re.IGNORECASE,
|
|
1236
|
+
)
|
|
1237
|
+
_src_negation_re = re.compile(
|
|
1238
|
+
r'\b(?:not|never|no\b|must\s+not|shall\s+not|do\s+not|'
|
|
1239
|
+
r'does\s+not|should\s+not|cannot|can\'t|don\'t|won\'t|isn\'t|'
|
|
1240
|
+
r'aren\'t|wasn\'t|weren\'t|hasn\'t|haven\'t|hadn\'t)\b',
|
|
1241
|
+
re.IGNORECASE,
|
|
1242
|
+
)
|
|
1243
|
+
if (len(src.split()) >= 6
|
|
1244
|
+
and _pos_obligation_re.search(src)
|
|
1245
|
+
and not _src_negation_re.search(src)):
|
|
1246
|
+
dev_chars_44 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1247
|
+
if dev_chars_44 >= 3:
|
|
1248
|
+
_hi_negation_re = re.compile(
|
|
1249
|
+
r'(?:नहीं|मत|(?<!\w)न(?!\w)|ना\b)',
|
|
1250
|
+
)
|
|
1251
|
+
if _hi_negation_re.search(tgt):
|
|
1252
|
+
return {
|
|
1253
|
+
"verdict": "catastrophic",
|
|
1254
|
+
"failure_type": "hallucination",
|
|
1255
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
1256
|
+
"format": 3, "safety": 0,
|
|
1257
|
+
"notes": (
|
|
1258
|
+
"Hindi modal inversion: source has positive obligation "
|
|
1259
|
+
"(must/shall/required — no English negation) but Hindi "
|
|
1260
|
+
"translation contains negation word (नहीं/मत/न), "
|
|
1261
|
+
"converting a mandatory action into a prohibition — "
|
|
1262
|
+
"directly observed in phase-6 en-hi run as catastrophic "
|
|
1263
|
+
"dosing instruction inversion"
|
|
1264
|
+
),
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
# 45. Swedish target: Norwegian/Danish word forms in output.
|
|
1268
|
+
# Small models sometimes produce Norwegian Bokmål or Danish instead of Swedish.
|
|
1269
|
+
# Key discriminators: Swedish 'och'/'är'/'sjukhus'/'läkare'/'sjuksköterska' vs
|
|
1270
|
+
# Norwegian 'og'/'er'/'sykehus'/'lege'/'sykepleier'.
|
|
1271
|
+
# 'og' surrounded by spaces is the strongest signal — it's the Norwegian/Danish
|
|
1272
|
+
# conjunction ('and') and never appears in Swedish prose where 'och' is used.
|
|
1273
|
+
# 'sykehus' and 'lege ' (as a word) are Norwegian-only medical vocabulary.
|
|
1274
|
+
# Guards: (a) Swedish target only; (b) tgt ≥ 5 words; (c) Norwegian/Danish
|
|
1275
|
+
# marker must be a standalone word (not part of a longer Swedish word).
|
|
1276
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
1277
|
+
if len(tgt.split()) >= 5:
|
|
1278
|
+
_no_markers = re.compile(
|
|
1279
|
+
r'(?<!\w)(?:og|sykehus|sykepleier|lege\b|legekontor|sykdom)(?!\w)',
|
|
1280
|
+
re.IGNORECASE,
|
|
1281
|
+
)
|
|
1282
|
+
if _no_markers.search(tgt):
|
|
1283
|
+
return {
|
|
1284
|
+
"verdict": "catastrophic",
|
|
1285
|
+
"failure_type": "wrong_language",
|
|
1286
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
1287
|
+
"format": 1, "safety": 5,
|
|
1288
|
+
"notes": (
|
|
1289
|
+
"Swedish target output contains Norwegian/Danish word forms "
|
|
1290
|
+
"(og/sykehus/sykepleier/lege) — model produced Norwegian "
|
|
1291
|
+
"Bokmål or Danish instead of Swedish; "
|
|
1292
|
+
"caught by prefilter check #45"
|
|
1293
|
+
),
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
# 46. Cantonese target: Mandarin pharmaceutical measure word 片 used for tablets
|
|
1297
|
+
# instead of Cantonese 粒.
|
|
1298
|
+
# Cantonese uses 粒 (lap1) as the classifier for all rounded dosage forms
|
|
1299
|
+
# (tablets, capsules, pills). Mandarin uses 片 (piàn, flat/slice) for tablets
|
|
1300
|
+
# and 顆 (kē) for pills/granules. Small models default to the Mandarin measure
|
|
1301
|
+
# word 片 when translating "tablet(s)" or "pill(s)" into Cantonese.
|
|
1302
|
+
# Detection: source contains "tablet" or "pill" (medical dosing context),
|
|
1303
|
+
# Cantonese output contains "片" (U+7247) but NOT "粒" (U+7C92), and output
|
|
1304
|
+
# has ≥10 Chinese characters (confirms a genuine translation attempt, not empty/short).
|
|
1305
|
+
# Guard: only fire if BOTH 粒 is absent AND 片 is present — if both appear,
|
|
1306
|
+
# the model may be using 片 in a compound (薄膜衣片 = film-coated tablet) where
|
|
1307
|
+
# it is structurally correct, so pass to Claude to judge.
|
|
1308
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1309
|
+
if re.search(r'\b(?:tablet|tablets|pill|pills)\b', src, re.IGNORECASE):
|
|
1310
|
+
zh_chars_46 = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF
|
|
1311
|
+
or 0x3400 <= ord(c) <= 0x4DBF)
|
|
1312
|
+
if zh_chars_46 >= 10:
|
|
1313
|
+
has_man_measure = '\u7247' in tgt # 片 (Mandarin tablet measure)
|
|
1314
|
+
has_yue_measure = '\u7C92' in tgt # 粒 (Cantonese pill/tablet measure)
|
|
1315
|
+
if has_man_measure and not has_yue_measure:
|
|
1316
|
+
return {
|
|
1317
|
+
"verdict": "catastrophic",
|
|
1318
|
+
"failure_type": "semantic_substitution",
|
|
1319
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
1320
|
+
"format": 3, "safety": 3,
|
|
1321
|
+
"notes": (
|
|
1322
|
+
"Cantonese output uses Mandarin pharmaceutical measure word 片 "
|
|
1323
|
+
"(piàn, flat/slice) for tablets instead of Cantonese 粒 (lap1, "
|
|
1324
|
+
"grain/rounded) — model defaulted to Mandarin classifier; "
|
|
1325
|
+
"Cantonese medical text counts tablets as 粒, not 片"
|
|
1326
|
+
),
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
# 47. Hindi target: Latin characters embedded mid-Devanagari word (partial transliteration).
|
|
1330
|
+
# The most common catastrophic en-hi failure in phase-6 (57% of cases):
|
|
1331
|
+
# models partially transliterate English medical terms, leaving Latin chars
|
|
1332
|
+
# directly adjacent to Devanagari chars within a single token.
|
|
1333
|
+
# Directly observed:
|
|
1334
|
+
# 'subcutaneously' → 'सबक्यूरेटжno' (Cyrillic+Latin mid-Devanagari);
|
|
1335
|
+
# 'metformin' → 'इसमेforminैटेड' (Latin 'formin' inside Devanagari word);
|
|
1336
|
+
# 'intravenously' → 'इंटravenसली' (Latin 'raven' inside Devanagari word).
|
|
1337
|
+
# Pattern: Devanagari char + Latin alpha(3+) + Devanagari char — cannot appear
|
|
1338
|
+
# in well-formed Hindi text. This is definitively garbled partial transliteration.
|
|
1339
|
+
# Guards: (a) Hindi target only; (b) ≥3 Devanagari chars (confirms Hindi attempt);
|
|
1340
|
+
# (c) Latin run must be ≥3 chars to avoid single-letter abbreviations (e.g. '५B').
|
|
1341
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1342
|
+
_dev_chars_47 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1343
|
+
if _dev_chars_47 >= 3:
|
|
1344
|
+
_mid_latin_re = re.compile(r'[\u0900-\u097F][a-zA-Z]{3,}[\u0900-\u097F]')
|
|
1345
|
+
_mid_latin_match = _mid_latin_re.search(tgt)
|
|
1346
|
+
if _mid_latin_match:
|
|
1347
|
+
return {
|
|
1348
|
+
"verdict": "catastrophic", "failure_type": "garbled",
|
|
1349
|
+
"adequacy": 0, "fluency": 0, "faithfulness": 0, "format": 0, "safety": 0,
|
|
1350
|
+
"notes": (
|
|
1351
|
+
f"Hindi partial transliteration: Latin characters embedded "
|
|
1352
|
+
f"mid-Devanagari word '{_mid_latin_match.group()}' — "
|
|
1353
|
+
f"directly observed in phase-6 en-hi run as the most common "
|
|
1354
|
+
f"catastrophic failure (57% of cases): "
|
|
1355
|
+
f"'metformin' → 'इसमेforminैटेड', 'intravenously' → 'इंटravenसली'"
|
|
1356
|
+
),
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
# 48. Cantonese target: Thai script contamination.
|
|
1360
|
+
# Directly observed in phase-6 en-yue run (3 of 28 catastrophic cases):
|
|
1361
|
+
# 'Do not give atropine if glaucoma' →
|
|
1362
|
+
# '唔ให้ให้อะตอมีน畀病人ที่มีกลูโคเพป้ส' — output contains Thai chars.
|
|
1363
|
+
# Thai (U+0E00-U+0E7F) and Cantonese/Chinese use entirely disjoint Unicode blocks.
|
|
1364
|
+
# ANY Thai character in a Cantonese output is definitive script contamination.
|
|
1365
|
+
# This is the Cantonese analog of check #40 (Thai chars in Kannada output).
|
|
1366
|
+
# Guards: (a) Cantonese target; (b) ≥1 Thai char; (c) tgt > 5 chars.
|
|
1367
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1368
|
+
thai_chars_48 = sum(1 for c in tgt if 0x0E00 <= ord(c) <= 0x0E7F)
|
|
1369
|
+
if thai_chars_48 >= 1 and len(tgt) > 5:
|
|
1370
|
+
return {
|
|
1371
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1372
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
1373
|
+
"notes": (
|
|
1374
|
+
f"Cantonese target contains {thai_chars_48} Thai script character(s) "
|
|
1375
|
+
f"(U+0E00-U+0E7F) — model mixed Thai script into Cantonese output; "
|
|
1376
|
+
f"directly observed in phase-6 en-yue run "
|
|
1377
|
+
f"('atropine glaucoma' → '唔ให้ให้อะตอมีน畀病人ที่มีกลูโคเพป้ส', "
|
|
1378
|
+
f"3 of 28 catastrophic en-yue cases)"
|
|
1379
|
+
),
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
# 49. Cantonese target: Devanagari script contamination.
|
|
1383
|
+
# Directly observed in phase-6 en-yue run (7 of 28 catastrophic cases, 25%):
|
|
1384
|
+
# 'Warfarin must NOT be combined with aspirin' →
|
|
1385
|
+
# 'लाव達必不能與阿司匹林混合' (Devanagari 'लाव' embedded in Cantonese text);
|
|
1386
|
+
# 'Sodium hypochlorite...' → 'सोयं塩化物係...' (Devanagari prefix in Cantonese).
|
|
1387
|
+
# Devanagari (U+0900-U+097F) has no role in Cantonese/Traditional Chinese text.
|
|
1388
|
+
# Guard: require ≥5 Chinese chars alongside Devanagari to confirm a genuine
|
|
1389
|
+
# Cantonese attempt rather than a pure wrong-language output (which checks
|
|
1390
|
+
# #33 and #41 would catch); require ≥2 Devanagari chars to avoid stray artifacts.
|
|
1391
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1392
|
+
dev_chars_49 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1393
|
+
zh_chars_49 = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF
|
|
1394
|
+
or 0x3400 <= ord(c) <= 0x4DBF)
|
|
1395
|
+
if dev_chars_49 >= 2 and zh_chars_49 >= 5:
|
|
1396
|
+
return {
|
|
1397
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1398
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
1399
|
+
"notes": (
|
|
1400
|
+
f"Cantonese target contains {dev_chars_49} Devanagari character(s) "
|
|
1401
|
+
f"(U+0900-U+097F) alongside {zh_chars_49} Chinese characters — "
|
|
1402
|
+
f"Devanagari script has no role in Cantonese/Traditional Chinese text; "
|
|
1403
|
+
f"directly observed in phase-6 en-yue run "
|
|
1404
|
+
f"('Warfarin' → 'लाव達必不能與阿司匹林', 7 of 28 catastrophic cases)"
|
|
1405
|
+
),
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
# 50. Cantonese target: Mandarin verb 吃 (eat/take) used instead of Cantonese 食.
|
|
1409
|
+
# In Cantonese, 食 (sik6) is the standard verb for both eating and taking medication
|
|
1410
|
+
# ('take tablet' = 食藥, 'eat food' = 食野). Mandarin uses 吃 (chī) for both.
|
|
1411
|
+
# Small models default to 吃 (Mandarin) when source has "take" or "eat" in an
|
|
1412
|
+
# oral-medication context — directly observed in phase-6 en-yue run:
|
|
1413
|
+
# 'take two tablets' → '吃兩片' (Mandarin eat + Mandarin classifier).
|
|
1414
|
+
# Detection: (a) Cantonese target; (b) source has "take"/"eat"/"drink"/"swallow"
|
|
1415
|
+
# in oral context; (c) output contains 吃 (U+5403, Mandarin eat) but NOT
|
|
1416
|
+
# 食 (U+98DF, Cantonese eat/take); (d) ≥10 Chinese chars (genuine translation attempt).
|
|
1417
|
+
# Guard: if output contains both 吃 and 食, pass to Claude — mixed usage possible
|
|
1418
|
+
# in some formal Cantonese registers.
|
|
1419
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1420
|
+
if re.search(
|
|
1421
|
+
r'\b(?:take|eat|drink|swallow|oral|orally|by\s+mouth)\b',
|
|
1422
|
+
src, re.IGNORECASE,
|
|
1423
|
+
):
|
|
1424
|
+
zh_chars_50 = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF
|
|
1425
|
+
or 0x3400 <= ord(c) <= 0x4DBF)
|
|
1426
|
+
if zh_chars_50 >= 10:
|
|
1427
|
+
has_mandarin_eat = '\u5403' in tgt # 吃
|
|
1428
|
+
has_cantonese_eat = '\u98DF' in tgt # 食
|
|
1429
|
+
if has_mandarin_eat and not has_cantonese_eat:
|
|
1430
|
+
return {
|
|
1431
|
+
"verdict": "catastrophic",
|
|
1432
|
+
"failure_type": "wrong_language",
|
|
1433
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
1434
|
+
"format": 2, "safety": 3,
|
|
1435
|
+
"notes": (
|
|
1436
|
+
"Cantonese output uses Mandarin verb 吃 (chī, eat/take) "
|
|
1437
|
+
"instead of Cantonese 食 (sik6) — model defaulted to "
|
|
1438
|
+
"Mandarin lexical verb in oral-medication context; "
|
|
1439
|
+
"correct Cantonese: 食藥 (take medication), not 吃藥; "
|
|
1440
|
+
"directly observed in phase-6 en-yue run"
|
|
1441
|
+
),
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
# 51. Swedish target: source has strong prohibition ('must not'/'do not'/'never') but
|
|
1445
|
+
# output has zero Swedish negation words — complete negation erasure.
|
|
1446
|
+
# Analogous to check #34 (Hindi negation erasure) but for Swedish.
|
|
1447
|
+
# Swedish negation words: inte, ej, icke (not), aldrig (never), ingen/inget/inga (no/none).
|
|
1448
|
+
# Complete absence of ALL negation words in a translated prohibition is catastrophic —
|
|
1449
|
+
# prohibition becomes permission. Directly motivated by phase-6 en-sv run (25.5% CAT):
|
|
1450
|
+
# 'must not be administered' → 'ska administreras' (should be administered).
|
|
1451
|
+
# Guards: (a) Swedish target; (b) source has strong prohibition phrase; (c) output ≥8
|
|
1452
|
+
# chars (genuine translation attempt, not empty); (d) NO Swedish negation word anywhere
|
|
1453
|
+
# in output — even a stray 'inte' in a different clause should pass to Claude.
|
|
1454
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
1455
|
+
_sv_prohibition_re = re.compile(
|
|
1456
|
+
r'\b(?:must\s+not|should\s+not|shall\s+not|do\s+not|does\s+not|'
|
|
1457
|
+
r'never|must\s+never|do\s+never|cannot|can\'t|don\'t|won\'t|'
|
|
1458
|
+
r'may\s+not|is\s+not\s+to|are\s+not\s+to)\b',
|
|
1459
|
+
re.IGNORECASE,
|
|
1460
|
+
)
|
|
1461
|
+
if _sv_prohibition_re.search(src) and len(tgt) >= 8:
|
|
1462
|
+
_sv_negation_re = re.compile(
|
|
1463
|
+
r'\b(?:inte|ej|icke|aldrig|ingen|inget|inga|varning)\b'
|
|
1464
|
+
r'|(?:förbjud|kontraindic|undvi[ks])',
|
|
1465
|
+
re.IGNORECASE,
|
|
1466
|
+
)
|
|
1467
|
+
if not _sv_negation_re.search(tgt):
|
|
1468
|
+
return {
|
|
1469
|
+
"verdict": "catastrophic",
|
|
1470
|
+
"failure_type": "negation_erasure",
|
|
1471
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
1472
|
+
"format": 3, "safety": 0,
|
|
1473
|
+
"notes": (
|
|
1474
|
+
"Swedish negation erasure: source contains strong prohibition "
|
|
1475
|
+
"(must not/should not/never/cannot) but Swedish output contains "
|
|
1476
|
+
"no negation word (inte/ej/icke/aldrig/ingen) — prohibition "
|
|
1477
|
+
"converted to permission; analogous to phase-6 en-hi "
|
|
1478
|
+
"negation erasure (check #34) but for Swedish target; "
|
|
1479
|
+
"directly motivated by phase-6 en-sv run (25.5% CAT rate)"
|
|
1480
|
+
),
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
# 52. Swedish target: concessive source connective ('although/despite/even though')
|
|
1484
|
+
# translated as 'eftersom' (causal/because) — completely reverses logic.
|
|
1485
|
+
# 'Although X, Y' = Y happens IN SPITE OF X (concessive).
|
|
1486
|
+
# 'eftersom X, Y' = Y happens BECAUSE OF X (causal).
|
|
1487
|
+
# Directly motivated by phase-6 en-sv run: discourse_connective_traps generated
|
|
1488
|
+
# 4 of 16 (25%) catastrophic en-sv cases. Causal-concessive confusion is the
|
|
1489
|
+
# single most dangerous semantic inversion: "Although the patient had no allergy,
|
|
1490
|
+
# penicillin was avoided" → "Eftersom patienten inte hade någon allergi undveks..."
|
|
1491
|
+
# would mean "BECAUSE the patient had no allergy..." — opposite meaning.
|
|
1492
|
+
# Guards: (a) Swedish target; (b) source has concessive connector;
|
|
1493
|
+
# (c) output has 'eftersom' (causal); (d) output lacks all Swedish concessive
|
|
1494
|
+
# markers (fastän/trots/även om/ändå) — if output has both, pass to Claude.
|
|
1495
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
1496
|
+
_sv_src_concessive_re = re.compile(
|
|
1497
|
+
r'\b(?:although|even\s+though|despite|in\s+spite\s+of|notwithstanding)\b',
|
|
1498
|
+
re.IGNORECASE,
|
|
1499
|
+
)
|
|
1500
|
+
if _sv_src_concessive_re.search(src):
|
|
1501
|
+
tgt_lower = tgt.lower()
|
|
1502
|
+
has_eftersom = 'eftersom' in tgt_lower
|
|
1503
|
+
has_sv_concessive = bool(re.search(
|
|
1504
|
+
r'fastän|trots\s+att|även\s+om|ändå|oaktat', tgt_lower,
|
|
1505
|
+
))
|
|
1506
|
+
if has_eftersom and not has_sv_concessive:
|
|
1507
|
+
return {
|
|
1508
|
+
"verdict": "catastrophic",
|
|
1509
|
+
"failure_type": "other",
|
|
1510
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0,
|
|
1511
|
+
"format": 3, "safety": 3,
|
|
1512
|
+
"notes": (
|
|
1513
|
+
"Swedish causal-concessive confusion: source has concessive "
|
|
1514
|
+
"connector (although/despite/even though) but Swedish output "
|
|
1515
|
+
"uses 'eftersom' (because/since — causal) without any "
|
|
1516
|
+
"concessive marker (fastän/trots att/även om/ändå); "
|
|
1517
|
+
"logical relationship completely inverted from concessive to causal; "
|
|
1518
|
+
"directly motivated by phase-6 en-sv discourse_connective_traps "
|
|
1519
|
+
"failures (4 of 16 catastrophic en-sv cases)"
|
|
1520
|
+
),
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
# 53. Hindi target: 'contraindicated' source without Hindi prohibition in output.
|
|
1524
|
+
# 'Contraindicated' is the strongest clinical prohibition; check #27 covers German.
|
|
1525
|
+
# For Hindi: if source has 'contraindicated' and output has ≥5 Devanagari chars but
|
|
1526
|
+
# lacks प्रति (prefix counter-marker), वर्जित/निषिद्ध/प्रतिबंधित (prohibition synonyms),
|
|
1527
|
+
# AND also lacks standalone negation (नहीं/मत/न) — all three absent means the
|
|
1528
|
+
# contraindication was silently dropped or rendered as the opposite (indicated).
|
|
1529
|
+
# Guards: (a) Hindi target; (b) 'contraindicated' in source; (c) ≥5 Devanagari chars
|
|
1530
|
+
# (confirms Hindi attempt, not wrong_language); (d) all prohibition markers absent.
|
|
1531
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1532
|
+
if re.search(r'\bcontraindicated\b', src, re.IGNORECASE):
|
|
1533
|
+
dev_chars_53 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1534
|
+
if dev_chars_53 >= 5:
|
|
1535
|
+
has_hi_prohibition = (
|
|
1536
|
+
'प्रति' in tgt
|
|
1537
|
+
or 'वर्जित' in tgt
|
|
1538
|
+
or 'निषिद्ध' in tgt
|
|
1539
|
+
or 'प्रतिबंधित' in tgt
|
|
1540
|
+
or 'नहीं' in tgt
|
|
1541
|
+
or 'मत' in tgt
|
|
1542
|
+
or ' न ' in tgt
|
|
1543
|
+
or tgt.startswith('न ')
|
|
1544
|
+
)
|
|
1545
|
+
if not has_hi_prohibition:
|
|
1546
|
+
return {
|
|
1547
|
+
"verdict": "catastrophic",
|
|
1548
|
+
"failure_type": "hallucination",
|
|
1549
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
1550
|
+
"format": 3, "safety": 0,
|
|
1551
|
+
"notes": (
|
|
1552
|
+
"Clinical contraindication erased in Hindi: source has 'contraindicated' "
|
|
1553
|
+
"but Hindi translation contains no prohibition marker "
|
|
1554
|
+
"(प्रति/वर्जित/निषिद्ध/प्रतिबंधित/नहीं/मत/न) — "
|
|
1555
|
+
"contraindication may be rendered as indication "
|
|
1556
|
+
"(Hindi analog of German check #27)"
|
|
1557
|
+
),
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
# 54. Cantonese target: source 'give [person]' but output uses Mandarin 给 without 俾.
|
|
1561
|
+
# 俾 (bei2, U+4FF3) is the canonical Cantonese benefactive/give word.
|
|
1562
|
+
# Its absence with Mandarin 给 (U+7ED9, Simplified) present = Mandarin lexical default.
|
|
1563
|
+
# Guards: (a) Cantonese target; (b) source has 'give' + [person/patient] construction;
|
|
1564
|
+
# (c) output has 给 (U+7ED9 Simplified only — excludes Traditional 給 U+7D66 which
|
|
1565
|
+
# could legitimately appear in formal Cantonese registers in compound words);
|
|
1566
|
+
# (d) output lacks 俾 (U+4FF3); (e) ≥10 Chinese chars (genuine translation attempt).
|
|
1567
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1568
|
+
if re.search(
|
|
1569
|
+
r'\bgive\s+the\s+(?:patient|nurse|staff|doctor|pharmacist)\b'
|
|
1570
|
+
r'|\bgave\s+the\s+(?:patient|nurse)\b'
|
|
1571
|
+
r'|\bdo\s+not\s+give\b|\bdon\'t\s+give\b',
|
|
1572
|
+
src, re.IGNORECASE,
|
|
1573
|
+
):
|
|
1574
|
+
zh_chars_54 = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF
|
|
1575
|
+
or 0x3400 <= ord(c) <= 0x4DBF)
|
|
1576
|
+
if zh_chars_54 >= 10:
|
|
1577
|
+
has_mandarin_give = '\u7ED9' in tgt # 给 Simplified Mandarin
|
|
1578
|
+
has_yue_give = '\u4FF3' in tgt # 俾 Cantonese
|
|
1579
|
+
if has_mandarin_give and not has_yue_give:
|
|
1580
|
+
return {
|
|
1581
|
+
"verdict": "catastrophic",
|
|
1582
|
+
"failure_type": "wrong_language",
|
|
1583
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
1584
|
+
"format": 2, "safety": 3,
|
|
1585
|
+
"notes": (
|
|
1586
|
+
"Cantonese benefactive defaulted to Mandarin: source 'give [person]' "
|
|
1587
|
+
"translated with Mandarin 给 (U+7ED9 Simplified) instead of "
|
|
1588
|
+
"Cantonese 俾 (bei2, U+4FF3) — model used Mandarin lexicon "
|
|
1589
|
+
"for the benefactive/give construction; Cantonese: 俾病人, not 给病人"
|
|
1590
|
+
),
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
# 55. Hindi target: causal-concessive confusion — source has concessive connector
|
|
1594
|
+
# ('although/despite/even though') but Hindi output contains क्योंकि (because/
|
|
1595
|
+
# since — causal) without any Hindi concessive marker.
|
|
1596
|
+
# 'Although X, Y' = Y happens IN SPITE OF X (concessive).
|
|
1597
|
+
# 'क्योंकि X, Y' = Y happens BECAUSE OF X (causal) — logically inverted.
|
|
1598
|
+
# Directly motivated by phase-6 en-hi run (43.4% CAT rate) where discourse
|
|
1599
|
+
# connective seeds produced causal-concessive substitutions in Hindi.
|
|
1600
|
+
# Analogous to check #52 (Swedish causal-concessive confusion) but for Hindi.
|
|
1601
|
+
# Guards: (a) Hindi target; (b) source has concessive connector; (c) ≥5 Devanagari
|
|
1602
|
+
# chars in output — confirms Hindi attempt; (d) क्योंकि present in output;
|
|
1603
|
+
# (e) no Hindi concessive marker (हालांकि/यद्यपि/बावजूद/फिर भी/तथापि) present.
|
|
1604
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1605
|
+
_hi_src_concessive_re = re.compile(
|
|
1606
|
+
r'\b(?:although|even\s+though|despite|in\s+spite\s+of|notwithstanding)\b',
|
|
1607
|
+
re.IGNORECASE,
|
|
1608
|
+
)
|
|
1609
|
+
if _hi_src_concessive_re.search(src):
|
|
1610
|
+
dev_chars_55 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1611
|
+
if dev_chars_55 >= 5:
|
|
1612
|
+
has_causal_55 = 'क्योंकि' in tgt
|
|
1613
|
+
has_hi_concessive_55 = (
|
|
1614
|
+
'हालांकि' in tgt
|
|
1615
|
+
or 'हालाँकि' in tgt
|
|
1616
|
+
or 'यद्यपि' in tgt
|
|
1617
|
+
or 'बावजूद' in tgt
|
|
1618
|
+
or 'फिर भी' in tgt
|
|
1619
|
+
or 'तथापि' in tgt
|
|
1620
|
+
)
|
|
1621
|
+
if has_causal_55 and not has_hi_concessive_55:
|
|
1622
|
+
return {
|
|
1623
|
+
"verdict": "catastrophic",
|
|
1624
|
+
"failure_type": "other",
|
|
1625
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0,
|
|
1626
|
+
"format": 3, "safety": 3,
|
|
1627
|
+
"notes": (
|
|
1628
|
+
"Hindi causal-concessive confusion: source has concessive "
|
|
1629
|
+
"connector (although/despite/even though) but Hindi output "
|
|
1630
|
+
"uses क्योंकि (because/since — causal) without any "
|
|
1631
|
+
"concessive marker (हालांकि/यद्यपि/बावजूद/फिर भी/तथापि); "
|
|
1632
|
+
"logical relationship inverted from concessive to causal; "
|
|
1633
|
+
"Hindi analog of check #52 (Swedish); "
|
|
1634
|
+
"directly motivated by phase-6 en-hi run (43.4% CAT rate)"
|
|
1635
|
+
),
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
# 56. Swedish target: plaintiff (käranden) rendered as criminal-defendant vocabulary.
|
|
1639
|
+
# Directly observed in phase-6 en-sv run:
|
|
1640
|
+
# 'Whereas the plaintiff claims damages of €2 million, the defendant disputes
|
|
1641
|
+
# all liability' →
|
|
1642
|
+
# 'Även om den anklagade ansökar om circa 2 miljoner euro...'
|
|
1643
|
+
# 'den anklagade' (the accused), 'den tilltalade' (the charged), and 'den åtalade'
|
|
1644
|
+
# (the prosecuted) are criminal-defendant terms — never correct for a civil plaintiff
|
|
1645
|
+
# (käranden). Source has 'plaintiff' but NOT 'defendant'/'accused', so the plaintiff
|
|
1646
|
+
# role should map to käranden (civil claimant) — not criminal-accused vocabulary.
|
|
1647
|
+
# Guards: (a) Swedish target; (b) 'plaintiff' in source; (c) source must NOT
|
|
1648
|
+
# contain 'defendant'/'accused'/'criminal' — if both appear, the assignment is
|
|
1649
|
+
# ambiguous and should go to Claude; (d) output ≥ 5 words (genuine attempt).
|
|
1650
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
1651
|
+
if (re.search(r'\bplaintiff\b', src, re.IGNORECASE)
|
|
1652
|
+
and not re.search(
|
|
1653
|
+
r'\b(?:defendant|accused|criminal|prosecuted)\b',
|
|
1654
|
+
src, re.IGNORECASE,
|
|
1655
|
+
)):
|
|
1656
|
+
if len(tgt.split()) >= 5:
|
|
1657
|
+
_sv_crim_defendant_re = re.compile(
|
|
1658
|
+
r'\b(?:den\s+anklagade|den\s+tilltalade|den\s+åtalade|'
|
|
1659
|
+
r'den\s+misstänkte|brottslingen)\b',
|
|
1660
|
+
re.IGNORECASE,
|
|
1661
|
+
)
|
|
1662
|
+
if _sv_crim_defendant_re.search(tgt):
|
|
1663
|
+
return {
|
|
1664
|
+
"verdict": "catastrophic",
|
|
1665
|
+
"failure_type": "semantic_substitution",
|
|
1666
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0,
|
|
1667
|
+
"format": 3, "safety": 3,
|
|
1668
|
+
"notes": (
|
|
1669
|
+
"Swedish legal party reversal: source 'plaintiff' (civil claimant, "
|
|
1670
|
+
"käranden) rendered as criminal-defendant vocabulary "
|
|
1671
|
+
"(den anklagade/den tilltalade/den åtalade) — "
|
|
1672
|
+
"opposite legal role assigned to the claiming party; "
|
|
1673
|
+
"directly observed in phase-6 en-sv run: "
|
|
1674
|
+
"'the plaintiff claims damages' → "
|
|
1675
|
+
"'Även om den anklagade ansökar om...' (check #56)"
|
|
1676
|
+
),
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
# 57. Hindi target: lakh/crore number system translation failure.
|
|
1680
|
+
# Source contains "N lakh" (Indian English for N × 100,000) but Hindi output
|
|
1681
|
+
# has Devanagari text (≥5 chars) without लाख — the model silently dropped or
|
|
1682
|
+
# failed to render the lakh unit in Hindi. Equally catches crore omission.
|
|
1683
|
+
# Directly motivated by hindi_lakh_crore_number_traps: "one lakh patients" →
|
|
1684
|
+
# "100,000 रोगी" (Western digits, no Devanagari number word) or "मिलियन".
|
|
1685
|
+
# Guards: (a) Hindi target only; (b) source has digit+lakh or word+lakh pattern;
|
|
1686
|
+
# (c) ≥5 Devanagari chars in output (not wrong-language, caught by check #9);
|
|
1687
|
+
# (d) no लाख in output.
|
|
1688
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1689
|
+
if re.search(
|
|
1690
|
+
r'\b(?:one|two|three|four|five|six|seven|eight|nine|ten|'
|
|
1691
|
+
r'fifteen|twenty|twenty.five|thirty|forty|fifty|sixty|seventy|'
|
|
1692
|
+
r'eighty|ninety|hundred|\d+(?:\.\d+)?)\s+lakh\b',
|
|
1693
|
+
src, re.IGNORECASE,
|
|
1694
|
+
):
|
|
1695
|
+
dev_chars = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1696
|
+
if dev_chars >= 5 and 'लाख' not in tgt:
|
|
1697
|
+
return {
|
|
1698
|
+
"verdict": "catastrophic", "failure_type": "number_mutation",
|
|
1699
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 3,
|
|
1700
|
+
"notes": (
|
|
1701
|
+
"Hindi number system failure: source has 'N lakh' (= N × 100,000) "
|
|
1702
|
+
"but Hindi output contains no लाख — the South Asian numeral "
|
|
1703
|
+
"word was not translated into Devanagari "
|
|
1704
|
+
"(hindi_lakh_crore_number_traps, check #57)"
|
|
1705
|
+
),
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
# 58. Hindi target: crore number system translation failure.
|
|
1709
|
+
# Analogous to check #57 but for crore (करोड़, 1,00,00,000 = 10 million).
|
|
1710
|
+
# Source has "N crore" but Hindi output lacks करोड़.
|
|
1711
|
+
# Guards mirror check #57.
|
|
1712
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1713
|
+
if re.search(
|
|
1714
|
+
r'\b(?:one|two|three|four|five|six|seven|eight|nine|ten|'
|
|
1715
|
+
r'fifteen|twenty|thirty|forty|fifty|sixty|seventy|'
|
|
1716
|
+
r'eighty|ninety|hundred|\d+(?:\.\d+)?)\s+crore\b',
|
|
1717
|
+
src, re.IGNORECASE,
|
|
1718
|
+
):
|
|
1719
|
+
dev_chars = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1720
|
+
if dev_chars >= 5 and 'करोड़' not in tgt:
|
|
1721
|
+
return {
|
|
1722
|
+
"verdict": "catastrophic", "failure_type": "number_mutation",
|
|
1723
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0, "format": 3, "safety": 3,
|
|
1724
|
+
"notes": (
|
|
1725
|
+
"Hindi number system failure: source has 'N crore' (= N × 10 million) "
|
|
1726
|
+
"but Hindi output contains no करोड़ — the South Asian numeral "
|
|
1727
|
+
"word was not translated into Devanagari "
|
|
1728
|
+
"(hindi_lakh_crore_number_traps, check #58)"
|
|
1729
|
+
),
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
# 59. Cantonese target: English copy-through — output is predominantly ASCII,
|
|
1733
|
+
# indicating the model produced English verbatim instead of translating to Cantonese.
|
|
1734
|
+
# Directly observed in phase-6 en-yue run: complex discourse sentences
|
|
1735
|
+
# (while/although/despite) were returned unchanged in English without any attempt
|
|
1736
|
+
# at Cantonese translation.
|
|
1737
|
+
# Detection: if output has >85% printable ASCII chars (letters/digits/punctuation,
|
|
1738
|
+
# excluding whitespace) and total length ≥ 15 characters, it is almost certainly
|
|
1739
|
+
# not a Cantonese translation (correct Cantonese output will contain CJK characters).
|
|
1740
|
+
# Guard: tgt ≥ 15 chars (avoids false positives on empty/very-short outputs
|
|
1741
|
+
# already caught by other checks); fires only for Cantonese target.
|
|
1742
|
+
# Note: check #33 catches Mandarin-output (present Chinese chars but no Cantonese
|
|
1743
|
+
# particles); this check #59 catches the complementary failure where the model
|
|
1744
|
+
# produces no Chinese characters at all — English copy-through.
|
|
1745
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1746
|
+
if len(tgt) >= 15:
|
|
1747
|
+
printable_ascii = sum(
|
|
1748
|
+
1 for c in tgt
|
|
1749
|
+
if c.isascii() and c.isprintable() and not c.isspace()
|
|
1750
|
+
)
|
|
1751
|
+
ascii_ratio = printable_ascii / len(tgt)
|
|
1752
|
+
if ascii_ratio > 0.85:
|
|
1753
|
+
return {
|
|
1754
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1755
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
1756
|
+
"notes": (
|
|
1757
|
+
"Cantonese target: output is >85% printable ASCII — "
|
|
1758
|
+
"model produced English copy-through instead of translating to Cantonese "
|
|
1759
|
+
"(directly observed in phase-6 en-yue run on complex discourse sentences; "
|
|
1760
|
+
"cantonese_english_passthrough_traps, check #59)"
|
|
1761
|
+
),
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
# 60. Kannada target: Devanagari script contamination in non-prohibition contexts.
|
|
1765
|
+
# Check #32 catches zero-Kannada outputs; check #38 catches Devanagari in
|
|
1766
|
+
# prohibition sentences (source has "must not"/"do not"). This check covers
|
|
1767
|
+
# the complementary gap: ANY non-empty Kannada output that contains ≥2 Devanagari
|
|
1768
|
+
# characters (U+0900-U+097F) in a non-prohibition context is script-contaminated.
|
|
1769
|
+
# Kannada uses an entirely separate Unicode block (U+0C80-U+0CFF) — legitimate
|
|
1770
|
+
# Kannada translations NEVER contain Devanagari characters (Sanskrit loanwords
|
|
1771
|
+
# in Kannada are written in Kannada script, not Devanagari).
|
|
1772
|
+
# Directly observed in phase-6 en-kn run: multi-script gibberish mixing
|
|
1773
|
+
# Kannada with Hindi Devanagari outside of prohibition sentences — these passed
|
|
1774
|
+
# checks #32 and #38 but are unmistakably wrong.
|
|
1775
|
+
# Guards: (a) Kannada target; (b) ≥3 Kannada chars (to avoid misfiring on
|
|
1776
|
+
# near-empty outputs already caught by check #1); (c) ≥2 Devanagari chars.
|
|
1777
|
+
if target_lang.lower() in ('kn', 'kannada'):
|
|
1778
|
+
kn_chars_60 = sum(1 for c in tgt if 0x0C80 <= ord(c) <= 0x0CFF)
|
|
1779
|
+
if kn_chars_60 >= 3:
|
|
1780
|
+
deva_chars = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1781
|
+
if deva_chars >= 2:
|
|
1782
|
+
return {
|
|
1783
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
1784
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
1785
|
+
"notes": (
|
|
1786
|
+
f"Kannada target has {deva_chars} Devanagari chars (U+0900-U+097F) "
|
|
1787
|
+
f"mixed with {kn_chars_60} Kannada chars — Devanagari never appears "
|
|
1788
|
+
"in correct Kannada output (distinct Unicode blocks); "
|
|
1789
|
+
"script contamination directly observed in phase-6 en-kn run "
|
|
1790
|
+
"(kannada_multi_script_contamination_traps, check #60)"
|
|
1791
|
+
),
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
# 61. Hindi target: feminine medical noun + masculine past participle (gender mismatch).
|
|
1795
|
+
# Hindi grammatical gender agreement failure — directly motivated by
|
|
1796
|
+
# hindi_gender_agreement_traps. Medical nouns दवा (medicine) and गोली (tablet)
|
|
1797
|
+
# are feminine; their passive compound verb forms must use feminine participles:
|
|
1798
|
+
# दवा दी गई (medicine was given — correct feminine passive)
|
|
1799
|
+
# दवा ली गई (medicine was taken — correct feminine passive)
|
|
1800
|
+
# The masculine defaults दिया/लिया immediately following दवा/गोली are
|
|
1801
|
+
# unambiguous gender errors — there is no valid Hindi sentence where दवा or
|
|
1802
|
+
# गोली (both feminine) is directly followed by the masculine participle दिया/लिया.
|
|
1803
|
+
# Guards: (a) Hindi target; (b) source has medication/dose/tablet/medicine
|
|
1804
|
+
# (confirms medical context); (c) ≥3 Devanagari chars in output (Hindi attempt).
|
|
1805
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1806
|
+
if re.search(
|
|
1807
|
+
r'\b(?:medicine|medication|tablet|tablets|pill|pills|dose|dosage|drug)\b',
|
|
1808
|
+
src, re.IGNORECASE,
|
|
1809
|
+
):
|
|
1810
|
+
dev_chars_61 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
1811
|
+
if dev_chars_61 >= 3:
|
|
1812
|
+
_hi_gender_wrong = re.compile(
|
|
1813
|
+
r'(?:दवा\s+लिया|दवा\s+दिया|गोली\s+लिया|गोली\s+दिया)',
|
|
1814
|
+
)
|
|
1815
|
+
if _hi_gender_wrong.search(tgt):
|
|
1816
|
+
return {
|
|
1817
|
+
"verdict": "catastrophic",
|
|
1818
|
+
"failure_type": "other",
|
|
1819
|
+
"adequacy": 1, "fluency": 0, "faithfulness": 1,
|
|
1820
|
+
"format": 2, "safety": 3,
|
|
1821
|
+
"notes": (
|
|
1822
|
+
"Hindi grammatical gender agreement failure: feminine medical noun "
|
|
1823
|
+
"(दवा/medicine or गोली/tablet) directly followed by masculine past "
|
|
1824
|
+
"participle (लिया/दिया) — correct form requires feminine ली/दी; "
|
|
1825
|
+
"दवा/गोली are inherently feminine and cannot take masculine agreement; "
|
|
1826
|
+
"(hindi_gender_agreement_traps, check #61)"
|
|
1827
|
+
),
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
# 62. Swedish target: hospital admission/discharge particle reversal.
|
|
1831
|
+
# Swedish particle verbs lägga in/lägga ut and skriva in/skriva ut are
|
|
1832
|
+
# clinical opposites (admit vs discharge). Source 'discharg' (discharge/discharged)
|
|
1833
|
+
# in a hospital context should map to Swedish discharge vocabulary
|
|
1834
|
+
# (skrivas ut / lades ut / hemgång / utskrivning) — NOT admission vocabulary
|
|
1835
|
+
# (lades in / läggs in / skrivs in / inläggning).
|
|
1836
|
+
# Directly motivated by swedish_verb_particle_traps: "discharged from hospital" →
|
|
1837
|
+
# "lades in på sjukhuset" (was ADMITTED) — the particle was reversed.
|
|
1838
|
+
# Guards: (a) Swedish target; (b) 'discharg' in source; (c) no admission word in
|
|
1839
|
+
# source (if source says "admit" and "discharge" both, the assignment is ambiguous);
|
|
1840
|
+
# (d) output ≥ 4 words (genuine translation attempt); (e) Swedish diacritic present
|
|
1841
|
+
# (confirms Swedish output, not English copy-through caught by check #6).
|
|
1842
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
1843
|
+
if (re.search(r'\bdischarg', src, re.IGNORECASE)
|
|
1844
|
+
and not re.search(r'\badmit', src, re.IGNORECASE)):
|
|
1845
|
+
if len(tgt.split()) >= 4 and any(c in tgt.lower() for c in 'åäö'):
|
|
1846
|
+
_sv_admission_re = re.compile(
|
|
1847
|
+
r'\b(?:lades?\s+in|läggs?\s+in|skrivs?\s+in|inläggning|'
|
|
1848
|
+
r'läggdes?\s+in|lägg\s+in)\b',
|
|
1849
|
+
re.IGNORECASE,
|
|
1850
|
+
)
|
|
1851
|
+
_sv_discharge_re = re.compile(
|
|
1852
|
+
r'\b(?:skriv(?:as|s|a)?\s+ut|skrev(?:s)?\s+ut|lade(?:s)?\s+ut|'
|
|
1853
|
+
r'hem(?:gång|skick|resa)|utskrivning|utskriv)\b',
|
|
1854
|
+
re.IGNORECASE,
|
|
1855
|
+
)
|
|
1856
|
+
if _sv_admission_re.search(tgt) and not _sv_discharge_re.search(tgt):
|
|
1857
|
+
return {
|
|
1858
|
+
"verdict": "catastrophic",
|
|
1859
|
+
"failure_type": "semantic_substitution",
|
|
1860
|
+
"adequacy": 0, "fluency": 3, "faithfulness": 0,
|
|
1861
|
+
"format": 3, "safety": 3,
|
|
1862
|
+
"notes": (
|
|
1863
|
+
"Swedish particle reversal: source 'discharged' (from hospital) "
|
|
1864
|
+
"rendered with Swedish admission vocabulary (lades in/skrivs in/"
|
|
1865
|
+
"inläggning) instead of discharge vocabulary (skrivas ut/"
|
|
1866
|
+
"utskrivning/hemgång) — clinical opposite; "
|
|
1867
|
+
"(swedish_verb_particle_traps, check #62)"
|
|
1868
|
+
),
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
# 63. Cantonese target: Mandarin-exclusive temporal vocabulary in output.
|
|
1872
|
+
# Cantonese has a parallel temporal vocabulary: 聽日 (tomorrow), 琴日 (yesterday),
|
|
1873
|
+
# 而家 (now), 頭先/啱啱 (just now). Mandarin uses 明天, 昨天, 現在, 剛才 respectively.
|
|
1874
|
+
# If a Cantonese target output contains at least one Mandarin-exclusive temporal word
|
|
1875
|
+
# (明天/昨天/現在/剛才/剛剛) and NO Cantonese temporal word (聽日/琴日/而家/頭先/啱啱),
|
|
1876
|
+
# and the source contains a temporal keyword, the output is almost certainly Mandarin.
|
|
1877
|
+
# Guards: (a) Cantonese target; (b) source has temporal keyword; (c) ≥5 CJK chars
|
|
1878
|
+
# in output (genuine translation attempt, not empty/English copy-through caught by
|
|
1879
|
+
# check #6); (d) Mandarin temporal word present; (e) no Cantonese temporal word.
|
|
1880
|
+
# Distinct from check #33 (zero Cantonese particles): this fires on outputs that
|
|
1881
|
+
# PASS check #33 (some Cantonese particles present) but still use Mandarin time words.
|
|
1882
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1883
|
+
_src_has_temporal = re.search(
|
|
1884
|
+
r'\b(?:tomorrow|yesterday|now|currently|immediately|just|tonight|today|'
|
|
1885
|
+
r'morning|afternoon|evening|recently|this\s+(?:morning|afternoon|evening))\b',
|
|
1886
|
+
src, re.IGNORECASE,
|
|
1887
|
+
)
|
|
1888
|
+
if _src_has_temporal and sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff') >= 5:
|
|
1889
|
+
_man_temporal_words = ('明天', '昨天', '現在', '剛才', '剛剛')
|
|
1890
|
+
_yue_temporal_words = ('聽日', '琴日', '而家', '頭先', '啱啱', '聽朝', '下晝')
|
|
1891
|
+
has_man_temp = any(w in tgt for w in _man_temporal_words)
|
|
1892
|
+
has_yue_temp = any(w in tgt for w in _yue_temporal_words)
|
|
1893
|
+
if has_man_temp and not has_yue_temp:
|
|
1894
|
+
_found = [w for w in _man_temporal_words if w in tgt]
|
|
1895
|
+
return {
|
|
1896
|
+
"verdict": "catastrophic",
|
|
1897
|
+
"failure_type": "wrong_language",
|
|
1898
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
1899
|
+
"format": 2, "safety": 3,
|
|
1900
|
+
"notes": (
|
|
1901
|
+
f"Mandarin temporal vocabulary in Cantonese output: "
|
|
1902
|
+
f"found {_found} (Mandarin-exclusive) with no Cantonese temporal "
|
|
1903
|
+
f"equivalents (聽日/琴日/而家/頭先) — Mandarin not Cantonese output; "
|
|
1904
|
+
f"(cantonese_temporal_vocab_traps, check #63)"
|
|
1905
|
+
),
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1908
|
+
# 64. Cantonese target: Mandarin directional complements in place of Cantonese ones.
|
|
1909
|
+
# Cantonese directional complements differ from Mandarin:
|
|
1910
|
+
# 入去/入嚟 vs Mandarin 进去/進去/进来/進來
|
|
1911
|
+
# 落去/落嚟 vs Mandarin 下去/下来/下來
|
|
1912
|
+
# 返去/返嚟 vs Mandarin 回去/回来/回來
|
|
1913
|
+
# The most discriminating signal: Cantonese uses 入 (enter) for directed motion
|
|
1914
|
+
# INTO a space; Mandarin uses 进/進. 嚟 is the Cantonese "come" directional;
|
|
1915
|
+
# Mandarin uses 来/來. A Cantonese output with 进/進 or 进来/進來 (but not 嚟)
|
|
1916
|
+
# for an "enter/come into" source is almost certainly Mandarin.
|
|
1917
|
+
# Guards: (a) Cantonese target; (b) source has motion-into or come-back verb phrase;
|
|
1918
|
+
# (c) ≥5 CJK chars in output (genuine translation attempt); (d) Mandarin directional
|
|
1919
|
+
# 進/进 present; (e) no Cantonese directional 入 or 嚟 present.
|
|
1920
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1921
|
+
_src_has_into = re.search(
|
|
1922
|
+
r'\b(?:go(?:es)?\s+into|come\s+(?:into|in|back|down|up)|'
|
|
1923
|
+
r'enter(?:ed|s)?|bring(?:ing)?\s+(?:in|back)|go(?:es)?\s+(?:back|down|up)|'
|
|
1924
|
+
r'sent\s+(?:back|down)|came?\s+(?:back|in|out|down|up))\b',
|
|
1925
|
+
src, re.IGNORECASE,
|
|
1926
|
+
)
|
|
1927
|
+
if _src_has_into and sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff') >= 5:
|
|
1928
|
+
_mandarin_dir = any(c in tgt for c in ('進', '进'))
|
|
1929
|
+
_cantonese_dir = any(c in tgt for c in ('入', '嚟'))
|
|
1930
|
+
if _mandarin_dir and not _cantonese_dir:
|
|
1931
|
+
_found_chars = [c for c in ('進', '进') if c in tgt]
|
|
1932
|
+
return {
|
|
1933
|
+
"verdict": "catastrophic",
|
|
1934
|
+
"failure_type": "wrong_language",
|
|
1935
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
1936
|
+
"format": 2, "safety": 3,
|
|
1937
|
+
"notes": (
|
|
1938
|
+
f"Mandarin directional complement(s) {_found_chars} in Cantonese "
|
|
1939
|
+
f"output — Cantonese uses 入/嚟 for directed motion, not Mandarin "
|
|
1940
|
+
f"進/进; output is Mandarin not Cantonese "
|
|
1941
|
+
f"(cantonese_directional_complement_traps, check #64)"
|
|
1942
|
+
),
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
# 65. Hindi target: source has "has already"/"already taken" but output uses
|
|
1946
|
+
# progressive marker (रहा/रही/रहे + है/था/हैं) instead of perfect (चुका/चुकी/चुके).
|
|
1947
|
+
# "Has already taken" → progressive "is taking" is medically dangerous (double-dose risk).
|
|
1948
|
+
# Guards: (a) Hindi target; (b) source has perfective trigger phrase; (c) ≥5 Devanagari
|
|
1949
|
+
# chars in output; (d) output contains progressive marker; (e) output lacks चुक- stem.
|
|
1950
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
1951
|
+
_src_perfective = re.search(
|
|
1952
|
+
r'\b(?:has\s+already\s+(?:taken|received|administered|completed|finished|had)|'
|
|
1953
|
+
r'already\s+(?:taken|received|administered|completed|finished|had)|'
|
|
1954
|
+
r'have\s+already\s+(?:taken|received|administered|completed|finished)|'
|
|
1955
|
+
r'had\s+already\s+(?:taken|received|administered|completed|finished))\b',
|
|
1956
|
+
src, re.IGNORECASE,
|
|
1957
|
+
)
|
|
1958
|
+
_tgt_devanagari = sum(1 for c in tgt if '\u0900' <= c <= '\u097f')
|
|
1959
|
+
if _src_perfective and _tgt_devanagari >= 5:
|
|
1960
|
+
_has_progressive = bool(re.search(
|
|
1961
|
+
r'(?:रहा|रही|रहे)\s+(?:है|हैं|था|थे|थी)', tgt
|
|
1962
|
+
))
|
|
1963
|
+
_has_perfect = bool(re.search(r'चुक', tgt))
|
|
1964
|
+
if _has_progressive and not _has_perfect:
|
|
1965
|
+
return {
|
|
1966
|
+
"verdict": "catastrophic",
|
|
1967
|
+
"failure_type": "mistranslation",
|
|
1968
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
1969
|
+
"format": 2, "safety": 2,
|
|
1970
|
+
"notes": (
|
|
1971
|
+
"Source uses perfect aspect ('has already taken/received') but Hindi "
|
|
1972
|
+
"output uses progressive aspect (रहा/रही/रहे है/था) — aspect confusion "
|
|
1973
|
+
"implies ongoing action not completed action; clinically dangerous "
|
|
1974
|
+
"(double-dose risk); (hindi_verb_aspect_traps, check #65)"
|
|
1975
|
+
),
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
# 66. Cantonese target: source has WH-question word but output uses Mandarin question
|
|
1979
|
+
# words (為什麼/什麼時候/誰/怎麼/哪裡) instead of Cantonese equivalents
|
|
1980
|
+
# (點解/幾時/邊個/點/邊度).
|
|
1981
|
+
# Guards: (a) Cantonese target; (b) source has WH-question trigger; (c) ≥5 CJK chars
|
|
1982
|
+
# in output; (d) Mandarin question word present; (e) no Cantonese question word present.
|
|
1983
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
1984
|
+
_src_has_wh = re.search(
|
|
1985
|
+
r'\b(?:why|when|who|whose|whom|how\s+(?:many|much|long|often|should|do|does|did|'
|
|
1986
|
+
r'can|could|would|will|is|are|was|were)|where|what)\b',
|
|
1987
|
+
src, re.IGNORECASE,
|
|
1988
|
+
)
|
|
1989
|
+
_tgt_cjk = sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff')
|
|
1990
|
+
if _src_has_wh and _tgt_cjk >= 5:
|
|
1991
|
+
_mandarin_q = any(q in tgt for q in (
|
|
1992
|
+
'為什麼', '为什么', '什麼時候', '什么时候', '怎麼', '怎么',
|
|
1993
|
+
'多少', '哪裡', '哪里', '哪兒', '哪儿',
|
|
1994
|
+
))
|
|
1995
|
+
_cantonese_q = any(q in tgt for q in (
|
|
1996
|
+
'點解', '幾時', '邊個', '邊度', '喺邊', '幾多', '咁多',
|
|
1997
|
+
))
|
|
1998
|
+
if _mandarin_q and not _cantonese_q:
|
|
1999
|
+
_found_q = [q for q in ('為什麼', '为什么', '什麼時候', '什么时候',
|
|
2000
|
+
'怎麼', '怎么', '多少', '哪裡', '哪里',
|
|
2001
|
+
'哪兒', '哪儿') if q in tgt]
|
|
2002
|
+
return {
|
|
2003
|
+
"verdict": "catastrophic",
|
|
2004
|
+
"failure_type": "wrong_language",
|
|
2005
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2006
|
+
"format": 2, "safety": 3,
|
|
2007
|
+
"notes": (
|
|
2008
|
+
f"Mandarin question word(s) {_found_q} in Cantonese output — "
|
|
2009
|
+
f"Cantonese uses 點解/幾時/邊個/點/邊度, not Mandarin 為什麼/什麼時候/誰/哪裡; "
|
|
2010
|
+
f"output is Mandarin not Cantonese "
|
|
2011
|
+
f"(cantonese_question_word_traps, check #66)"
|
|
2012
|
+
),
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
# 67. Hindi target: output contains Marathi copula आहे/आहेत (is/are in Marathi)
|
|
2016
|
+
# instead of Hindi है/हैं — same Devanagari script, different language.
|
|
2017
|
+
# Guards: (a) Hindi target; (b) ≥5 Devanagari chars in output; (c) Marathi copula
|
|
2018
|
+
# present; (d) Hindi copula है/हैं absent — if both present, may be hybrid, skip.
|
|
2019
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2020
|
+
_tgt_devanagari = sum(1 for c in tgt if '\u0900' <= c <= '\u097f')
|
|
2021
|
+
if _tgt_devanagari >= 5:
|
|
2022
|
+
_has_marathi_copula = bool(re.search(r'आहे|आहेत', tgt))
|
|
2023
|
+
_has_hindi_copula = bool(re.search(r'है|हैं', tgt))
|
|
2024
|
+
if _has_marathi_copula and not _has_hindi_copula:
|
|
2025
|
+
return {
|
|
2026
|
+
"verdict": "catastrophic",
|
|
2027
|
+
"failure_type": "wrong_language",
|
|
2028
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2029
|
+
"format": 2, "safety": 3,
|
|
2030
|
+
"notes": (
|
|
2031
|
+
"Marathi copula आहे/आहेत in Hindi target output with no Hindi copula "
|
|
2032
|
+
"है/हैं — output is Marathi not Hindi (same Devanagari script, "
|
|
2033
|
+
"different language); (hindi_marathi_confusion_traps, check #67)"
|
|
2034
|
+
),
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
# 68. Cantonese target: source has English passive ('was V-ed by', 'was given', etc.)
|
|
2038
|
+
# but output uses Mandarin 被-passive without Cantonese 俾 passive marker.
|
|
2039
|
+
# Guards: (a) Cantonese target; (b) source has passive trigger; (c) ≥5 CJK chars;
|
|
2040
|
+
# (d) 被 present in output; (e) 俾 absent from output (俾 = Cantonese passive marker).
|
|
2041
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2042
|
+
_src_passive = re.search(
|
|
2043
|
+
r'\b(?:was\s+(?:given|administered|prescribed|dispensed|performed|signed|verified|'
|
|
2044
|
+
r'reviewed|examined|inspected|calibrated|investigated|enrolled|confirmed|checked)|'
|
|
2045
|
+
r'were\s+(?:given|administered|prescribed|dispensed|confirmed|checked)|'
|
|
2046
|
+
r'must\s+be\s+(?:given|administered|confirmed|checked|verified))\b',
|
|
2047
|
+
src, re.IGNORECASE,
|
|
2048
|
+
)
|
|
2049
|
+
_tgt_cjk = sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff')
|
|
2050
|
+
if _src_passive and _tgt_cjk >= 5:
|
|
2051
|
+
_has_mandarin_passive = '被' in tgt
|
|
2052
|
+
_has_cantonese_passive = '俾' in tgt
|
|
2053
|
+
if _has_mandarin_passive and not _has_cantonese_passive:
|
|
2054
|
+
return {
|
|
2055
|
+
"verdict": "catastrophic",
|
|
2056
|
+
"failure_type": "wrong_language",
|
|
2057
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2058
|
+
"format": 2, "safety": 3,
|
|
2059
|
+
"notes": (
|
|
2060
|
+
"Mandarin 被-passive in Cantonese output with no Cantonese 俾 passive marker — "
|
|
2061
|
+
"Cantonese colloquial passive uses 俾...人...V, not Mandarin 被; "
|
|
2062
|
+
"output is Mandarin not Cantonese "
|
|
2063
|
+
"(cantonese_passive_construction_traps, check #68)"
|
|
2064
|
+
),
|
|
2065
|
+
}
|
|
2066
|
+
|
|
2067
|
+
# 69. Cantonese target: output has Mandarin feminine pronoun 她 (never used in
|
|
2068
|
+
# colloquial Cantonese) OR Mandarin plural pronouns 他們/她們 (Cantonese plural
|
|
2069
|
+
# is 佢哋), with no Cantonese 佢 pronoun present.
|
|
2070
|
+
# Cantonese uses gender-neutral 佢 (keoi5) for he/she/it and 佢哋 for they/them.
|
|
2071
|
+
# Guards: (a) Cantonese target; (b) ≥5 CJK chars; (c) 她 OR 他們/她們 present;
|
|
2072
|
+
# (d) 佢 absent — if 佢 present, model may be mixing registers (ambiguous).
|
|
2073
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2074
|
+
_tgt_cjk_69 = sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff')
|
|
2075
|
+
if _tgt_cjk_69 >= 5:
|
|
2076
|
+
_has_mandarin_fem3p = '她' in tgt
|
|
2077
|
+
_has_mandarin_plural3p = bool(re.search(r'他們|她們', tgt))
|
|
2078
|
+
_has_cantonese_3p = '佢' in tgt
|
|
2079
|
+
if (_has_mandarin_fem3p or _has_mandarin_plural3p) and not _has_cantonese_3p:
|
|
2080
|
+
_pron_signal = '她' if _has_mandarin_fem3p else '他們/她們'
|
|
2081
|
+
return {
|
|
2082
|
+
"verdict": "catastrophic",
|
|
2083
|
+
"failure_type": "wrong_language",
|
|
2084
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2085
|
+
"format": 2, "safety": 3,
|
|
2086
|
+
"notes": (
|
|
2087
|
+
f"Mandarin pronoun(s) {_pron_signal} in Cantonese output with no "
|
|
2088
|
+
"Cantonese 佢 — Cantonese uses gender-neutral 佢 (keoi5) for "
|
|
2089
|
+
"he/she/it and 佢哋 for they/them; 她 is never used in colloquial "
|
|
2090
|
+
"Cantonese writing; output is Mandarin not Cantonese "
|
|
2091
|
+
"(cantonese_pronoun_system_traps, check #69)"
|
|
2092
|
+
),
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
# 70. Hindi target: source contains absolute existential negation (no one/nobody/
|
|
2096
|
+
# nothing/none of/never) but Hindi output has no negation word (नहीं/मत/न/ना).
|
|
2097
|
+
# These NPIs require a compound negation in Hindi (e.g. कोई नहीं, कभी नहीं)
|
|
2098
|
+
# and small models drop the नहीं component, converting prohibitions to permissions.
|
|
2099
|
+
# Distinct from check #34 (deontic 'must not'/'do not'): here the failure is
|
|
2100
|
+
# NPI compound negation — existential quantifier + नहीं dropped together.
|
|
2101
|
+
# Guards: (a) Hindi target; (b) ≥5 Devanagari chars; (c) NPI trigger in source;
|
|
2102
|
+
# (d) no Hindi negation in output.
|
|
2103
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2104
|
+
_tgt_dev_70 = sum(1 for c in tgt if '\u0900' <= c <= '\u097f')
|
|
2105
|
+
if _tgt_dev_70 >= 5:
|
|
2106
|
+
_src_npi = bool(re.search(
|
|
2107
|
+
r'\bno\s+one\b|\bnobody\b|\bnothing\b|\bnone\s+of\b|\bnever\b'
|
|
2108
|
+
r'|\bno\s+patient\b|\bno\s+medication\b|\bno\s+child\b|\bno\s+person\b',
|
|
2109
|
+
src, re.IGNORECASE,
|
|
2110
|
+
))
|
|
2111
|
+
if _src_npi:
|
|
2112
|
+
_has_hi_neg_70 = bool(re.search(r'नहीं|मत|(?<!\w)न(?!\w)|ना', tgt))
|
|
2113
|
+
if not _has_hi_neg_70:
|
|
2114
|
+
return {
|
|
2115
|
+
"verdict": "catastrophic",
|
|
2116
|
+
"failure_type": "negation_erasure",
|
|
2117
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
2118
|
+
"format": 2, "safety": 2,
|
|
2119
|
+
"notes": (
|
|
2120
|
+
"Hindi NPI erasure: source has absolute negation "
|
|
2121
|
+
"(no one/nobody/nothing/none of/never) but Hindi output "
|
|
2122
|
+
"has no negation marker (नहीं/मत/न/ना) — the NPI compound "
|
|
2123
|
+
"negation was dropped, converting a prohibition to an "
|
|
2124
|
+
"affirmative statement; "
|
|
2125
|
+
"(hindi_negative_polarity_item_traps, check #70)"
|
|
2126
|
+
),
|
|
2127
|
+
}
|
|
2128
|
+
|
|
2129
|
+
# 71. Hindi target: source contains a forced-obligation marker (had to / was forced to /
|
|
2130
|
+
# was required to / was compelled to) but Hindi output lacks the obligative V2
|
|
2131
|
+
# auxiliary marker पड़ (as in पड़ा/पड़ी/पड़ना/पड़ेगा/पड़ेगी/पड़ना था).
|
|
2132
|
+
# In Hindi, "had to V" → "V-ना पड़ा/पड़ी" — the auxiliary पड़ encodes forced/
|
|
2133
|
+
# involuntary aspect and is absent only when the model collapses to simple past
|
|
2134
|
+
# (V-या/V-ई), stripping the compelled-action meaning.
|
|
2135
|
+
# Clinically significant: "had to repeat the procedure" (दोबारा करना पड़ा) vs
|
|
2136
|
+
# "repeated the procedure" (दोबारा किया) differ in accountability.
|
|
2137
|
+
# Distinct from check #34 (deontic must not/do not): here obligation is past forced.
|
|
2138
|
+
# Guards: (a) Hindi target; (b) ≥5 Devanagari chars; (c) source forced-obligation
|
|
2139
|
+
# marker; (d) पड़ absent from output.
|
|
2140
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2141
|
+
_tgt_dev_71 = sum(1 for c in tgt if '\u0900' <= c <= '\u097f')
|
|
2142
|
+
if _tgt_dev_71 >= 5:
|
|
2143
|
+
_src_had_to_71 = bool(re.search(
|
|
2144
|
+
r'\b(?:had\s+to|was\s+forced\s+to|were\s+forced\s+to|'
|
|
2145
|
+
r'was\s+compelled\s+to|were\s+compelled\s+to|'
|
|
2146
|
+
r'was\s+required\s+to|were\s+required\s+to|'
|
|
2147
|
+
r'had\s+no\s+choice\s+but\s+to)\b',
|
|
2148
|
+
src, re.IGNORECASE,
|
|
2149
|
+
))
|
|
2150
|
+
if _src_had_to_71 and 'पड़' not in tgt:
|
|
2151
|
+
return {
|
|
2152
|
+
"verdict": "catastrophic",
|
|
2153
|
+
"failure_type": "aspectual_erasure",
|
|
2154
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
2155
|
+
"format": 2, "safety": 2,
|
|
2156
|
+
"notes": (
|
|
2157
|
+
"Hindi conjunct verb erasure: source has forced-obligation "
|
|
2158
|
+
"marker (had to/was forced to/was required to) but Hindi "
|
|
2159
|
+
"output lacks पड़ — the obligative V2 auxiliary (पड़ा/पड़ी/"
|
|
2160
|
+
"पड़ेगा) that encodes forced/involuntary aspect was dropped, "
|
|
2161
|
+
"converting compelled action to voluntary action; "
|
|
2162
|
+
"(hindi_conjunct_verb_traps, check #71)"
|
|
2163
|
+
),
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
# 72. Cantonese target: output contains 把 in a disposal construction context
|
|
2167
|
+
# (把 followed within 8 chars by disposal verb 給/服/放/棄/移/取/洗/貼/注/送/遞/交/轉)
|
|
2168
|
+
# with no Cantonese-specific particles — signals Mandarin 把-disposal output.
|
|
2169
|
+
# Colloquial Cantonese NEVER uses 把-disposal; uses 將 (joeng1) or restructures.
|
|
2170
|
+
# Example: Mandarin "把藥物給病人服用" → correct Cantonese "將藥物畀病人食".
|
|
2171
|
+
# Guards: (a) Cantonese target; (b) ≥5 CJK chars; (c) 把 + disposal verb within
|
|
2172
|
+
# 8 chars; (d) no Cantonese particles (嘅/係/喺/咗/冇/唔/喇/㗎/佢/囉/喎).
|
|
2173
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2174
|
+
_tgt_cjk_72 = sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff')
|
|
2175
|
+
if _tgt_cjk_72 >= 5:
|
|
2176
|
+
_has_ba3_disposal_72 = bool(re.search(
|
|
2177
|
+
r'把.{0,8}(?:給|服|放|棄|移|取|洗|貼|注|送|遞|交|轉)',
|
|
2178
|
+
tgt,
|
|
2179
|
+
))
|
|
2180
|
+
_has_canto_72 = bool(re.search(
|
|
2181
|
+
r'嘅|係|喺|咗|冇|唔|喇|㗎|佢|囉|喎|㖞', tgt,
|
|
2182
|
+
))
|
|
2183
|
+
if _has_ba3_disposal_72 and not _has_canto_72:
|
|
2184
|
+
return {
|
|
2185
|
+
"verdict": "catastrophic",
|
|
2186
|
+
"failure_type": "wrong_language",
|
|
2187
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2188
|
+
"format": 2, "safety": 3,
|
|
2189
|
+
"notes": (
|
|
2190
|
+
"Mandarin 把-disposal construction in Cantonese output: "
|
|
2191
|
+
"把 followed by disposal verb detected with no Cantonese "
|
|
2192
|
+
"particles (嘅/係/喺/咗/冇/唔) — colloquial Cantonese "
|
|
2193
|
+
"never uses 把-disposal; uses 將 (joeng1) or restructures; "
|
|
2194
|
+
"output is Mandarin not Cantonese; "
|
|
2195
|
+
"(cantonese_ba3_construction_traps, check #72)"
|
|
2196
|
+
),
|
|
2197
|
+
}
|
|
2198
|
+
|
|
2199
|
+
# 73. Swedish target: output has a modal verb directly followed by att + infinitive.
|
|
2200
|
+
# Swedish modal verbs (måste/bör/kan/ska/vill/får/skall/borde/behöver) ALWAYS
|
|
2201
|
+
# take a bare infinitive — att is NEVER inserted between modal and infinitive.
|
|
2202
|
+
# "måste att ta" is ungrammatical; correct is "måste ta".
|
|
2203
|
+
# Failure: small models trained on English/Romance data insert att as a calque
|
|
2204
|
+
# of English 'to' infinitive or French/Spanish infinitive marker.
|
|
2205
|
+
# Guards: (a) Swedish target; (b) modal + att pattern present; (c) ≥3 tokens
|
|
2206
|
+
# so we don't fire on isolated modal fragments.
|
|
2207
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
2208
|
+
_modal_att_73 = bool(re.search(
|
|
2209
|
+
r'\b(?:måste|bör|kan|ska|vill|får|skall|borde|behöver)\s+att\b',
|
|
2210
|
+
tgt, re.IGNORECASE,
|
|
2211
|
+
))
|
|
2212
|
+
if _modal_att_73 and len(tgt.split()) >= 3:
|
|
2213
|
+
return {
|
|
2214
|
+
"verdict": "catastrophic",
|
|
2215
|
+
"failure_type": "grammatical",
|
|
2216
|
+
"adequacy": 2, "fluency": 1, "faithfulness": 2,
|
|
2217
|
+
"format": 3, "safety": 3,
|
|
2218
|
+
"notes": (
|
|
2219
|
+
"Swedish modal + att infinitive error: output has modal verb "
|
|
2220
|
+
"(måste/bör/kan/ska/vill/får/skall/borde/behöver) followed "
|
|
2221
|
+
"immediately by att — Swedish modals take bare infinitives, "
|
|
2222
|
+
"never att; 'måste att ta' is ungrammatical (correct: 'måste ta'); "
|
|
2223
|
+
"likely calque of English 'to' infinitive or Romance infinitive marker; "
|
|
2224
|
+
"(swedish_infinitive_att_traps, check #73)"
|
|
2225
|
+
),
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2228
|
+
# 74. Swedish target: 'i' preposition with institutional noun (wrong — Swedish uses 'på').
|
|
2229
|
+
# Swedish uses 'på' with institutions (sjukhuset/apoteket/mottagningen/kliniken/
|
|
2230
|
+
# akuten/vårdcentralen/laboratoriet) — NOT 'i'.
|
|
2231
|
+
# English "in the hospital" → Swedish "på sjukhuset" (correct); "i sjukhuset" is
|
|
2232
|
+
# a definitive transfer error from English.
|
|
2233
|
+
# Directly motivated by Swedish institutional preposition failure mode:
|
|
2234
|
+
# models trained on English/German data write "i sjukhuset" (German: im Krankenhaus)
|
|
2235
|
+
# instead of Swedish "på sjukhuset".
|
|
2236
|
+
# Guards: (a) Swedish target; (b) source has 'in the hospital/pharmacy/clinic/A&E/lab';
|
|
2237
|
+
# (c) output has wrong 'i' + institutional-noun form (definite suffix -et/-en);
|
|
2238
|
+
# (d) output does NOT also contain the correct 'på' form — if both present, ambiguous.
|
|
2239
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
2240
|
+
_sv_wrong_institution_re = re.compile(
|
|
2241
|
+
r'\bi\s+(?:sjukhuset|apoteket|mottagningen|kliniken|akuten|'
|
|
2242
|
+
r'vårdcentralen|laboratoriet|operationssalen|intensivvårdsavdelningen)\b',
|
|
2243
|
+
re.IGNORECASE,
|
|
2244
|
+
)
|
|
2245
|
+
_sv_correct_institution_re = re.compile(
|
|
2246
|
+
r'\bpå\s+(?:sjukhuset|apoteket|mottagningen|kliniken|akuten|'
|
|
2247
|
+
r'vårdcentralen|laboratoriet|operationssalen|intensivvårdsavdelningen)\b',
|
|
2248
|
+
re.IGNORECASE,
|
|
2249
|
+
)
|
|
2250
|
+
if _sv_wrong_institution_re.search(tgt) and not _sv_correct_institution_re.search(tgt):
|
|
2251
|
+
if len(tgt.split()) >= 4:
|
|
2252
|
+
return {
|
|
2253
|
+
"verdict": "catastrophic",
|
|
2254
|
+
"failure_type": "grammatical",
|
|
2255
|
+
"adequacy": 1, "fluency": 1, "faithfulness": 2,
|
|
2256
|
+
"format": 3, "safety": 3,
|
|
2257
|
+
"notes": (
|
|
2258
|
+
"Swedish institutional preposition error: output uses 'i sjukhuset/"
|
|
2259
|
+
"i apoteket/i mottagningen/i kliniken' — Swedish requires 'på' with "
|
|
2260
|
+
"institutions (på sjukhuset, på apoteket, på mottagningen); "
|
|
2261
|
+
"'i sjukhuset' is a definitive English-transfer error; "
|
|
2262
|
+
"(swedish_institutional_preposition_traps, check #74)"
|
|
2263
|
+
),
|
|
2264
|
+
}
|
|
2265
|
+
|
|
2266
|
+
# 75. Swedish target: source says "oral/PO use only" but Swedish output has a parenteral
|
|
2267
|
+
# route word (intravenöst/intramuskulärt/subkutant) without any oral-route word.
|
|
2268
|
+
# Directly observed: "PO use only — IV contraindicated" → output 'intramuskulär'.
|
|
2269
|
+
# Guards: (a) Swedish target; (b) source has explicit "oral only"/"PO use only";
|
|
2270
|
+
# (c) output has parenteral route word; (d) output lacks oral-route word;
|
|
2271
|
+
# (e) output ≥5 tokens (avoid very short fragments).
|
|
2272
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
2273
|
+
_sv_oral_only_src = bool(re.search(
|
|
2274
|
+
r'\b(?:oral(?:ly)?\s+use\s+only|for\s+oral\s+use\s+only|PO\s+use\s+only|'
|
|
2275
|
+
r'for\s+(?:PO|oral)\s+(?:use|administration)\s+only|'
|
|
2276
|
+
r'oral(?:ly)?\s+(?:use|administration)\s+only)\b',
|
|
2277
|
+
src, re.IGNORECASE,
|
|
2278
|
+
))
|
|
2279
|
+
_sv_parenteral_tgt = bool(re.search(
|
|
2280
|
+
r'\b(?:intravenöst|intravenösa?|intramuskulärt|intramuskulära?|'
|
|
2281
|
+
r'subkutant|subkutana?|parenteralt)\b',
|
|
2282
|
+
tgt, re.IGNORECASE,
|
|
2283
|
+
))
|
|
2284
|
+
_sv_oral_tgt = bool(re.search(
|
|
2285
|
+
r'\b(?:oralt|per\s+os|via\s+munnen|tas\s+oralt|sväljes)\b',
|
|
2286
|
+
tgt, re.IGNORECASE,
|
|
2287
|
+
))
|
|
2288
|
+
if _sv_oral_only_src and _sv_parenteral_tgt and not _sv_oral_tgt and len(tgt.split()) >= 5:
|
|
2289
|
+
return {
|
|
2290
|
+
"verdict": "catastrophic",
|
|
2291
|
+
"failure_type": "semantic_substitution",
|
|
2292
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2293
|
+
"format": 3, "safety": 1,
|
|
2294
|
+
"notes": (
|
|
2295
|
+
"Swedish route-of-administration error: source specifies oral-only route "
|
|
2296
|
+
"('oral use only'/'PO use only') but Swedish output contains parenteral "
|
|
2297
|
+
"route word (intravenöst/intramuskulärt/subkutant) without any oral "
|
|
2298
|
+
"marker (oralt/per os/via munnen); directly observed: 'PO use only — "
|
|
2299
|
+
"IV contraindicated' → output 'intramuskulär'; life-threatening route swap; "
|
|
2300
|
+
"(swedish_route_of_administration_traps, check #75)"
|
|
2301
|
+
),
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
# 76. Kannada target: source says "must not/shall not/never" (deontic prohibition) but
|
|
2305
|
+
# Kannada output contains the OBLIGATIVE marker ಬೇಕು (must/required) without any
|
|
2306
|
+
# PROHIBITIVE marker (ಬೇಡ/ಬಾರದು/ಇಲ್ಲ).
|
|
2307
|
+
# Directly observed: "must NOT contact aluminium" → ಹೊಂದಿರಬೇಕು (must contact).
|
|
2308
|
+
# The -ಬೇಕು suffix encodes obligation; -ಬಾರದು encodes prohibition — using the former
|
|
2309
|
+
# when the latter is needed converts a safety prohibition to a safety requirement.
|
|
2310
|
+
# Guards: (a) Kannada target; (b) source has explicit must-not/never deontic;
|
|
2311
|
+
# (c) output contains ಬೇಕು (obligative); (d) output lacks ಬೇಡ/ಬಾರದು/ಇಲ್ಲ;
|
|
2312
|
+
# (e) output ≥5 tokens to exclude fragments.
|
|
2313
|
+
if target_lang.lower() in ('kn', 'kannada', 'ಕನ್ನಡ'):
|
|
2314
|
+
_kn_mustnot_src = bool(re.search(
|
|
2315
|
+
r'\b(?:must\s+not|must\s+never|shall\s+not|shall\s+never|'
|
|
2316
|
+
r'never\s+(?:be|mix|give|use|apply|combine|store|contact|touch|administer)|'
|
|
2317
|
+
r'(?:do|does)\s+not\s+(?:give|use|apply|mix|combine|store|contact|touch|administer))\b',
|
|
2318
|
+
src, re.IGNORECASE,
|
|
2319
|
+
))
|
|
2320
|
+
_kn_obligative_tgt = bool(re.search(
|
|
2321
|
+
r'ಬೇಕು|ಬೇಕಾಗುತ್ತದೆ|ಆಗಬೇಕು|ಮಾಡಬೇಕು|ಕೊಡಬೇಕು|ಬೆರೆಸಬೇಕು|ಇರಬೇಕು|ಬಳಸಬೇಕು',
|
|
2322
|
+
tgt,
|
|
2323
|
+
))
|
|
2324
|
+
_kn_prohibitive_tgt = bool(re.search(
|
|
2325
|
+
r'ಬೇಡ|ಬಾರದು|ಇಲ್ಲ|ಕೂಡದು|ಅಲ್ಲ',
|
|
2326
|
+
tgt,
|
|
2327
|
+
))
|
|
2328
|
+
if _kn_mustnot_src and _kn_obligative_tgt and not _kn_prohibitive_tgt and len(tgt.split()) >= 5:
|
|
2329
|
+
return {
|
|
2330
|
+
"verdict": "catastrophic",
|
|
2331
|
+
"failure_type": "semantic_substitution",
|
|
2332
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2333
|
+
"format": 3, "safety": 1,
|
|
2334
|
+
"notes": (
|
|
2335
|
+
"Kannada obligative/prohibitive inversion: source says 'must not/never' "
|
|
2336
|
+
"but Kannada output uses obligative ಬೇಕು (must/required) without any "
|
|
2337
|
+
"prohibitive marker (ಬೇಡ/ಬಾರದು/ಇಲ್ಲ); directly observed: 'must NOT contact "
|
|
2338
|
+
"aluminium' → ಹೊಂದಿರಬೇಕು (must contact); "
|
|
2339
|
+
"prohibition converted to obligation — safety-critical inversion; "
|
|
2340
|
+
"(kannada_obligative_prohibitive_traps, check #76)"
|
|
2341
|
+
),
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2344
|
+
# 77. Hindi target: Telugu script (U+0C00-U+0C7F) contamination.
|
|
2345
|
+
# Directly observed in phase-6 en-hi run:
|
|
2346
|
+
# 'Even though the drug is approved, it must not be given to patients under 18' →
|
|
2347
|
+
# 'fármస్యకం दवा को 18 वर्ष से कम...' (Telugu chars స్యకం embedded in Devanagari).
|
|
2348
|
+
# Telugu and Hindi Devanagari are entirely disjoint Unicode blocks.
|
|
2349
|
+
# ANY Telugu character in Hindi output is definitive script contamination.
|
|
2350
|
+
# Guards: (a) Hindi target; (b) ≥1 Telugu char; (c) tgt > 5 chars.
|
|
2351
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2352
|
+
tel_chars_77 = sum(1 for c in tgt if 0x0C00 <= ord(c) <= 0x0C7F)
|
|
2353
|
+
if tel_chars_77 >= 1 and len(tgt) > 5:
|
|
2354
|
+
return {
|
|
2355
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
2356
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
2357
|
+
"notes": (
|
|
2358
|
+
f"Hindi target contains {tel_chars_77} Telugu script character(s) "
|
|
2359
|
+
f"(U+0C00-U+0C7F) in Devanagari output — model fell back to Telugu "
|
|
2360
|
+
f"medical vocabulary; directly observed in phase-6 en-hi run "
|
|
2361
|
+
f"('Even though drug approved' → 'fármస్యకం दवा...'); "
|
|
2362
|
+
f"(hindi_telugu_script_contamination_traps, check #77)"
|
|
2363
|
+
),
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
# 78. Cantonese target: Arabic (U+0600-U+06FF) or Cyrillic (U+0400-U+04FF) contamination.
|
|
2367
|
+
# Directly observed in phase-6 en-yue run:
|
|
2368
|
+
# 'morphine sulfate, not hydromorphone' →
|
|
2369
|
+
# '患者 recibió 5mg морфи素硫酸,不是水морфи素' (Cyrillic морфи in Cantonese text);
|
|
2370
|
+
# 'PO use only — IV contraindicated' → Arabic+Greek+Spanish garble;
|
|
2371
|
+
# 'Inject subcutaneously (SC)' → 'إدخال تحت الجلد (SC)' — pure Arabic output.
|
|
2372
|
+
# Arabic and Cyrillic have no role in Cantonese/Traditional Chinese text.
|
|
2373
|
+
# Guard: require ≥2 chars to avoid stray Unicode artifacts; Cantonese target only.
|
|
2374
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2375
|
+
ara_chars_78 = sum(1 for c in tgt if 0x0600 <= ord(c) <= 0x06FF)
|
|
2376
|
+
cyr_chars_78 = sum(1 for c in tgt if 0x0400 <= ord(c) <= 0x04FF)
|
|
2377
|
+
if (ara_chars_78 >= 2 or cyr_chars_78 >= 2) and len(tgt) > 5:
|
|
2378
|
+
return {
|
|
2379
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
2380
|
+
"adequacy": 0, "fluency": 1, "faithfulness": 0, "format": 1, "safety": 5,
|
|
2381
|
+
"notes": (
|
|
2382
|
+
f"Cantonese target contains Arabic ({ara_chars_78} chars, U+0600-U+06FF) "
|
|
2383
|
+
f"or Cyrillic ({cyr_chars_78} chars, U+0400-U+04FF) — entirely foreign "
|
|
2384
|
+
f"to Cantonese/Traditional Chinese; directly observed in phase-6 en-yue "
|
|
2385
|
+
f"run (morphine seed → 'морфи素硫酸'; PO/SC route seeds → Arabic output); "
|
|
2386
|
+
f"(cantonese_arabic_cyrillic_contamination_traps, check #78)"
|
|
2387
|
+
),
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
# 79. Swedish target: English "non-" prefix passthrough in output.
|
|
2391
|
+
# Directly observed in phase-6 en-sv run:
|
|
2392
|
+
# 'non-toxic' → 'non-tokt' (English 'non-' kept, '-tokt' fabricated;
|
|
2393
|
+
# correct Swedish: 'icke-giftig'/'ogiftig').
|
|
2394
|
+
# When source has 'non-toxic' specifically and Swedish output retains English 'non-'
|
|
2395
|
+
# without Swedish equivalent (icke-/ogiftig/ofarlig), the privative prefix leaked.
|
|
2396
|
+
# Guard: narrow to 'non-toxic' to avoid false positives on 'non-invasiv' (valid in SV).
|
|
2397
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
2398
|
+
if 'non-toxic' in src.lower() and re.search(r'\bnon-\w', tgt, re.IGNORECASE):
|
|
2399
|
+
_sv_has_icke = bool(re.search(
|
|
2400
|
+
r'\b(?:icke|ogiftig|ofarlig|icke-giftig|ej\s+giftig)', tgt, re.IGNORECASE
|
|
2401
|
+
))
|
|
2402
|
+
if not _sv_has_icke:
|
|
2403
|
+
return {
|
|
2404
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
2405
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 3, "safety": 3,
|
|
2406
|
+
"notes": (
|
|
2407
|
+
"Swedish output retains English 'non-' prefix for 'non-toxic' "
|
|
2408
|
+
"(correct Swedish: 'icke-giftig'/'ogiftig'); directly observed in "
|
|
2409
|
+
"phase-6 en-sv run: 'non-toxic' → 'non-tokt' (fabricated portmanteau); "
|
|
2410
|
+
"English privative prefix leaked through without Swedish translation; "
|
|
2411
|
+
"(swedish_invented_compound_traps, check #79)"
|
|
2412
|
+
),
|
|
2413
|
+
}
|
|
2414
|
+
|
|
2415
|
+
# 80. Hindi target: source has 'without' but Hindi output lacks privative compound
|
|
2416
|
+
# postposition बिना/बगैर/रहित.
|
|
2417
|
+
# Targets hindi_compound_postposition_traps failure mode: 'without the doctor's
|
|
2418
|
+
# authorisation' → Hindi output that drops 'without' entirely, or keeps the English
|
|
2419
|
+
# word 'without' as passthrough, or substitutes a wrong simple postposition.
|
|
2420
|
+
# Guards:
|
|
2421
|
+
# (a) Hindi target;
|
|
2422
|
+
# (b) source has 'without' as standalone word (not 'without doubt' fixed phrases);
|
|
2423
|
+
# (c) output has ≥5 Devanagari chars (Hindi was attempted, not blank);
|
|
2424
|
+
# (d) output has no बिना (U+092C U+093F U+0928 U+093E) or बगैर or रहित;
|
|
2425
|
+
# (e) output does NOT retain English 'without' verbatim (passthrough also catastrophic).
|
|
2426
|
+
# False-positive mitigation: narrow to clinical contexts (consent/prescription/
|
|
2427
|
+
# authorisation) where privative meaning is obligatory.
|
|
2428
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2429
|
+
if re.search(
|
|
2430
|
+
r'\bwithout\b',
|
|
2431
|
+
src, re.IGNORECASE,
|
|
2432
|
+
):
|
|
2433
|
+
dev_chars_80 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
2434
|
+
if dev_chars_80 >= 5:
|
|
2435
|
+
has_bina = bool(re.search(r'बिना|बगैर|रहित', tgt))
|
|
2436
|
+
has_eng_without = bool(re.search(r'\bwithout\b', tgt, re.IGNORECASE))
|
|
2437
|
+
if not has_bina and not has_eng_without:
|
|
2438
|
+
return {
|
|
2439
|
+
"verdict": "catastrophic",
|
|
2440
|
+
"failure_type": "semantic_substitution",
|
|
2441
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2442
|
+
"format": 3, "safety": 2,
|
|
2443
|
+
"notes": (
|
|
2444
|
+
"Hindi compound postposition erasure: source contains 'without' "
|
|
2445
|
+
"but Hindi output lacks बिना/बगैर/रहित — the privative compound "
|
|
2446
|
+
"postposition 'के बिना' was dropped entirely; 'without doctor's "
|
|
2447
|
+
"authorisation' → Hindi without privative marker converts a "
|
|
2448
|
+
"prohibition into an unconditional statement; "
|
|
2449
|
+
"(hindi_compound_postposition_traps, check #80)"
|
|
2450
|
+
),
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
# check #81: Hindi focus particle erasure — source has 'only' (non-negated, non-conditional)
|
|
2454
|
+
# but Hindi output lacks restrictive focus markers ही/केवल/सिर्फ/मात्र.
|
|
2455
|
+
# Guards: (a) Hindi target; (b) source has 'only' NOT in 'not only'/'only if'/'if only'
|
|
2456
|
+
# (those are different constructions that don't require ही); (c) ≥5 Devanagari chars
|
|
2457
|
+
# in output (model produced Hindi, not wrong-language); (d) output lacks any Hindi
|
|
2458
|
+
# restrictive focus particle (ही/केवल/सिर्फ/मात्र/तभी).
|
|
2459
|
+
# Semantics: 'administer ONLY the prescribed dose' → Hindi without ही/केवल converts a
|
|
2460
|
+
# restricted instruction ('this dose, nothing else') into an unrestricted one.
|
|
2461
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2462
|
+
if re.search(r'\bonly\b', src, re.IGNORECASE):
|
|
2463
|
+
not_only = bool(re.search(r'\bnot\s+only\b', src, re.IGNORECASE))
|
|
2464
|
+
only_if = bool(re.search(r'\bonly\s+if\b', src, re.IGNORECASE))
|
|
2465
|
+
if_only = bool(re.search(r'\bif\s+only\b', src, re.IGNORECASE))
|
|
2466
|
+
if not (not_only or only_if or if_only):
|
|
2467
|
+
dev_chars_81 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
2468
|
+
if dev_chars_81 >= 5:
|
|
2469
|
+
has_focus_81 = bool(re.search(r'ही|केवल|सिर्फ|मात्र|तभी', tgt))
|
|
2470
|
+
if not has_focus_81:
|
|
2471
|
+
return {
|
|
2472
|
+
"verdict": "catastrophic",
|
|
2473
|
+
"failure_type": "semantic_substitution",
|
|
2474
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2475
|
+
"format": 3, "safety": 2,
|
|
2476
|
+
"notes": (
|
|
2477
|
+
"Hindi focus particle erasure: source has 'only' (restrictive) "
|
|
2478
|
+
"but Hindi output lacks ही/केवल/सिर्फ/मात्र — the exclusive "
|
|
2479
|
+
"restriction scope was dropped; 'administer ONLY the prescribed "
|
|
2480
|
+
"dose' without ही/केवल removes the exclusivity constraint, "
|
|
2481
|
+
"converting a targeted restriction into a general statement; "
|
|
2482
|
+
"(hindi_focus_particle_traps, check #81)"
|
|
2483
|
+
),
|
|
2484
|
+
}
|
|
2485
|
+
|
|
2486
|
+
# check #82: Cantonese deontic obligation Mandarin-form substitution — source has
|
|
2487
|
+
# 'need not'/'don't need to'/'not required'/'not necessary' and Cantonese target
|
|
2488
|
+
# contains Mandarin negated modal (不需要/不必/毋須) instead of Cantonese 唔使.
|
|
2489
|
+
# Guards: (a) Cantonese target; (b) source has no-obligation modal pattern;
|
|
2490
|
+
# (c) ≥10 CJK chars in output (model produced CJK text, not wrong-script);
|
|
2491
|
+
# (d) Mandarin no-obligation form present; (e) Cantonese 唔使/唔需要 absent.
|
|
2492
|
+
# Semantics: 唔使 (don't need to) is uniquely Cantonese; 不需要 is Mandarin written
|
|
2493
|
+
# form. Their presence marks the output register unambiguously.
|
|
2494
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2495
|
+
if re.search(
|
|
2496
|
+
r'\b(need\s+not|do\s+not\s+need|don\'t\s+need|does\s+not\s+need|'
|
|
2497
|
+
r"doesn't\s+need|not\s+required\b|not\s+necessary\b)",
|
|
2498
|
+
src, re.IGNORECASE,
|
|
2499
|
+
):
|
|
2500
|
+
cjk_count_82 = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF)
|
|
2501
|
+
if cjk_count_82 >= 10:
|
|
2502
|
+
has_mandarin_modal_82 = bool(re.search(r'不需要|不必|毋須|無需', tgt))
|
|
2503
|
+
has_cant_modal_82 = bool(re.search(r'唔使|唔需要', tgt))
|
|
2504
|
+
if has_mandarin_modal_82 and not has_cant_modal_82:
|
|
2505
|
+
return {
|
|
2506
|
+
"verdict": "catastrophic",
|
|
2507
|
+
"failure_type": "wrong_language",
|
|
2508
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 2,
|
|
2509
|
+
"format": 3, "safety": 3,
|
|
2510
|
+
"notes": (
|
|
2511
|
+
"Cantonese deontic modal Mandarin substitution: source has "
|
|
2512
|
+
"'need not'/'don't need to'/'not required' but Cantonese output "
|
|
2513
|
+
"uses Mandarin form 不需要/不必/毋須 instead of Cantonese 唔使 — "
|
|
2514
|
+
"Mandarin-register deontic modal in Cantonese target; "
|
|
2515
|
+
"(cantonese_deontic_obligation_traps, check #82)"
|
|
2516
|
+
),
|
|
2517
|
+
}
|
|
2518
|
+
|
|
2519
|
+
# check #83: Hindi contrastive negation loss — source has ', not [word]' or
|
|
2520
|
+
# 'rather than [word]' (contrastive identity) + Hindi target has ≥5 Devanagari
|
|
2521
|
+
# chars + output lacks any contrastive negation marker (नहीं/न कि/बल्कि/के बजाय).
|
|
2522
|
+
# Guards: (a) Hindi target; (b) contrastive pattern present in source;
|
|
2523
|
+
# (c) ≥5 Devanagari chars (model produced Devanagari, not wrong-script failure);
|
|
2524
|
+
# (d) no Hindi contrastive negation marker in output.
|
|
2525
|
+
# Semantics: 'X, not Y' constructions require नहीं or बल्कि or न कि to preserve
|
|
2526
|
+
# the excluded alternative Y — dropping it converts a contrastive record entry
|
|
2527
|
+
# into a simple positive statement with no excluded alternative.
|
|
2528
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2529
|
+
if re.search(r',\s+not\s+[A-Za-z]|\brather\s+than\s+[A-Za-z]', src, re.IGNORECASE):
|
|
2530
|
+
dev_count_83 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
2531
|
+
if dev_count_83 >= 5:
|
|
2532
|
+
if not re.search(r'नहीं|न\s+कि|बल्कि|के\s+बजाय|बजाय', tgt):
|
|
2533
|
+
return {
|
|
2534
|
+
"verdict": "catastrophic",
|
|
2535
|
+
"failure_type": "semantic_substitution",
|
|
2536
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
2537
|
+
"format": 3, "safety": 2,
|
|
2538
|
+
"notes": (
|
|
2539
|
+
"Hindi contrastive negation loss: source has 'X, not Y' or "
|
|
2540
|
+
"'X rather than Y' but Hindi output lacks contrastive negation "
|
|
2541
|
+
"marker (नहीं/न कि/बल्कि/के बजाय) — the excluded alternative "
|
|
2542
|
+
"was dropped, converting a contrastive record into a simple "
|
|
2543
|
+
"positive statement; (hindi_contrastive_negation_traps, check #83)"
|
|
2544
|
+
),
|
|
2545
|
+
}
|
|
2546
|
+
|
|
2547
|
+
# check #84: Cantonese South/Southeast Asian script contamination — en-yue target
|
|
2548
|
+
# + ≥5 CJK chars in output (model attempted CJK text) + ≥2 chars from South Asian
|
|
2549
|
+
# (Devanagari/Gujarati/Bengali/Telugu/Tamil) or Southeast Asian (Thai) Unicode ranges.
|
|
2550
|
+
# Guards: (a) Cantonese target; (b) ≥5 CJK chars (model produced CJK — not a
|
|
2551
|
+
# pure wrong-script failure already caught by check #1); (c) ≥2 contaminating chars.
|
|
2552
|
+
# Distinct from check #78 (Arabic/Hebrew/Cyrillic/Latin contamination).
|
|
2553
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2554
|
+
cjk_count_84 = sum(1 for c in tgt if 0x4E00 <= ord(c) <= 0x9FFF)
|
|
2555
|
+
if cjk_count_84 >= 5:
|
|
2556
|
+
south_asian_84 = sum(
|
|
2557
|
+
1 for c in tgt
|
|
2558
|
+
if (0x0E00 <= ord(c) <= 0x0E7F # Thai
|
|
2559
|
+
or 0x0900 <= ord(c) <= 0x097F # Devanagari (Hindi/Marathi/Nepali)
|
|
2560
|
+
or 0x0A80 <= ord(c) <= 0x0AFF # Gujarati
|
|
2561
|
+
or 0x0C00 <= ord(c) <= 0x0C7F # Telugu
|
|
2562
|
+
or 0x0B80 <= ord(c) <= 0x0BFF # Tamil
|
|
2563
|
+
or 0x0980 <= ord(c) <= 0x09FF) # Bengali
|
|
2564
|
+
)
|
|
2565
|
+
if south_asian_84 >= 2:
|
|
2566
|
+
return {
|
|
2567
|
+
"verdict": "catastrophic",
|
|
2568
|
+
"failure_type": "wrong_language",
|
|
2569
|
+
"adequacy": 1, "fluency": 1, "faithfulness": 1,
|
|
2570
|
+
"format": 1, "safety": 3,
|
|
2571
|
+
"notes": (
|
|
2572
|
+
"Cantonese South/Southeast Asian script contamination: output "
|
|
2573
|
+
"contains Thai/Devanagari/Gujarati/Telugu/Tamil/Bengali chars "
|
|
2574
|
+
"embedded in CJK Cantonese text — script-level wrong_language; "
|
|
2575
|
+
"(cantonese_south_asian_script_contamination_traps, check #84)"
|
|
2576
|
+
),
|
|
2577
|
+
}
|
|
2578
|
+
|
|
2579
|
+
# check #85: Swedish drug identity substitution — source contains 'warfarin' +
|
|
2580
|
+
# Swedish output lacks 'warfarin' + output contains 'Vitamin E'/'E-vitamin'/
|
|
2581
|
+
# 'Evitamin' — directly observed phase-6 en-sv failure where model replaced
|
|
2582
|
+
# the anticoagulant warfarin with vitamin E (both affect coagulation pathway).
|
|
2583
|
+
if target_lang.lower() in ('sv', 'swedish'):
|
|
2584
|
+
if re.search(r'\bwarfarin\b', src, re.IGNORECASE):
|
|
2585
|
+
if not re.search(r'\bwarfarin\b', tgt, re.IGNORECASE):
|
|
2586
|
+
if re.search(r'[Vv]itamin\s*E\b|E-[Vv]itamin\b|[Ee]vitamin\b', tgt):
|
|
2587
|
+
return {
|
|
2588
|
+
"verdict": "catastrophic",
|
|
2589
|
+
"failure_type": "semantic_substitution",
|
|
2590
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
2591
|
+
"format": 3, "safety": 2,
|
|
2592
|
+
"notes": (
|
|
2593
|
+
"Swedish drug identity substitution: source mentions 'warfarin' "
|
|
2594
|
+
"but Swedish output omits it and contains 'Vitamin E'/'E-vitamin' "
|
|
2595
|
+
"instead — directly observed phase-6 en-sv failure; "
|
|
2596
|
+
"(swedish_drug_identity_traps, check #85)"
|
|
2597
|
+
),
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
# check #86: Cantonese HK institutional vocabulary — source mentions
|
|
2601
|
+
# 'emergency'/'ambulance'/'A&E' and output contains simplified 诊 (U+8BCA),
|
|
2602
|
+
# indicating 急诊室 (Simplified Mandarin A&E term) instead of correct
|
|
2603
|
+
# HK Cantonese 急症室. The bigram 急诊 is definitively Simplified Mandarin
|
|
2604
|
+
# and never appears in Hong Kong Cantonese institutional usage.
|
|
2605
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2606
|
+
cjk_count = sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff'
|
|
2607
|
+
or '\u3400' <= c <= '\u4dbf')
|
|
2608
|
+
if cjk_count >= 5:
|
|
2609
|
+
if re.search(
|
|
2610
|
+
r'\b(?:emergency|ambulance|A&E|accident\s+and\s+emergency)\b',
|
|
2611
|
+
src, re.IGNORECASE
|
|
2612
|
+
):
|
|
2613
|
+
# 诊 = U+8BCA (simplified form of 診): appears in 急诊室/急诊科
|
|
2614
|
+
if '\u8bca' in tgt:
|
|
2615
|
+
return {
|
|
2616
|
+
"verdict": "catastrophic",
|
|
2617
|
+
"failure_type": "wrong_language",
|
|
2618
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
2619
|
+
"format": 1, "safety": 5,
|
|
2620
|
+
"notes": (
|
|
2621
|
+
"Cantonese HK institutional vocabulary failure: source mentions "
|
|
2622
|
+
"emergency services but output contains simplified 诊 (U+8BCA) "
|
|
2623
|
+
"indicating 急诊室 (Mainland Mandarin term) instead of correct "
|
|
2624
|
+
"HK Cantonese 急症室; "
|
|
2625
|
+
"(cantonese_hk_institution_vocab_traps, check #86)"
|
|
2626
|
+
),
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
# check #87: Swedish dosage frequency reduction — source specifies 'three
|
|
2630
|
+
# times daily' or 't.i.d.' and Swedish output contains 'en gång' (once
|
|
2631
|
+
# daily) — frequency dropped from 3×/day to 1×/day, catastrophic underdose
|
|
2632
|
+
# risk for antibiotics and narrow-window drugs.
|
|
2633
|
+
# check #88: Swedish four-times-daily frequency error — source specifies
|
|
2634
|
+
# 'four times daily' or 'q.i.d.' and Swedish output specifies 'en gång'
|
|
2635
|
+
# (once) or 'två gånger' (twice) — catastrophic for pain control and
|
|
2636
|
+
# antibiotics where spacing is safety-critical.
|
|
2637
|
+
if target_lang.lower() in ('sv', 'swedish'):
|
|
2638
|
+
if re.search(r'\b(?:three\s+times\s+(?:daily|a\s+day)|t\.i\.d\.)\b',
|
|
2639
|
+
src, re.IGNORECASE):
|
|
2640
|
+
if re.search(r'\ben\s+gång\b', tgt, re.IGNORECASE):
|
|
2641
|
+
return {
|
|
2642
|
+
"verdict": "catastrophic",
|
|
2643
|
+
"failure_type": "semantic_substitution",
|
|
2644
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 0,
|
|
2645
|
+
"format": 3, "safety": 2,
|
|
2646
|
+
"notes": (
|
|
2647
|
+
"Swedish dosage frequency error: source specifies three times "
|
|
2648
|
+
"daily (t.i.d.) but Swedish output has 'en gång' (once daily) "
|
|
2649
|
+
"— frequency quartered, catastrophic underdose risk; "
|
|
2650
|
+
"(swedish_dosage_frequency_traps, check #87)"
|
|
2651
|
+
),
|
|
2652
|
+
}
|
|
2653
|
+
if re.search(r'\b(?:four\s+times\s+(?:daily|a\s+day)|q\.i\.d\.)\b',
|
|
2654
|
+
src, re.IGNORECASE):
|
|
2655
|
+
if re.search(r'\b(?:en\s+gång|två\s+gånger)\b', tgt, re.IGNORECASE):
|
|
2656
|
+
return {
|
|
2657
|
+
"verdict": "catastrophic",
|
|
2658
|
+
"failure_type": "semantic_substitution",
|
|
2659
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 0,
|
|
2660
|
+
"format": 3, "safety": 2,
|
|
2661
|
+
"notes": (
|
|
2662
|
+
"Swedish dosage frequency error: source specifies four times "
|
|
2663
|
+
"daily (q.i.d.) but Swedish output has 'en gång' or 'två gånger' "
|
|
2664
|
+
"(once or twice daily) — frequency halved or quartered; "
|
|
2665
|
+
"(swedish_dosage_frequency_traps, check #88)"
|
|
2666
|
+
),
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
# check #89: Cantonese restrictive 'only' — source has standalone 'only'
|
|
2670
|
+
# (not conditional 'only if'/'only when') + en-yue output has 只有/只是/
|
|
2671
|
+
# 只用/只能 (Mandarin "only" forms) AND lacks ALL Cantonese-exclusive markers
|
|
2672
|
+
# (嘅/係/唔/冇/㗎/咋/喎/喇/囉/佢/哋) → catastrophic/wrong_language.
|
|
2673
|
+
# Targets cantonese_sentence_final_particle_traps: the Cantonese restrictive
|
|
2674
|
+
# particle 咋 (zaa3) must appear for 'only' dose-limit contexts; when the
|
|
2675
|
+
# model outputs 只有/只是 AND no Cantonese markers at all, the output is
|
|
2676
|
+
# Mandarin (not Cantonese), which is a clear wrong-language failure.
|
|
2677
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2678
|
+
cjk_cnt = sum(
|
|
2679
|
+
1 for c in tgt
|
|
2680
|
+
if 0x4E00 <= ord(c) <= 0x9FFF or 0x3400 <= ord(c) <= 0x4DBF
|
|
2681
|
+
)
|
|
2682
|
+
if cjk_cnt >= 5:
|
|
2683
|
+
if re.search(r'\bonly\b', src, re.IGNORECASE):
|
|
2684
|
+
if not re.search(
|
|
2685
|
+
r'\bonly\s+(?:if|when|where|after|before)\b',
|
|
2686
|
+
src, re.IGNORECASE):
|
|
2687
|
+
if re.search(r'只(?:有|是|用|能|需)', tgt):
|
|
2688
|
+
if not re.search(r'[嘅係唔冇㗎咋喎喇囉佢哋]', tgt):
|
|
2689
|
+
return {
|
|
2690
|
+
"verdict": "catastrophic",
|
|
2691
|
+
"failure_type": "wrong_language",
|
|
2692
|
+
"adequacy": 1, "fluency": 2,
|
|
2693
|
+
"faithfulness": 1, "format": 2, "safety": 2,
|
|
2694
|
+
"notes": (
|
|
2695
|
+
"Cantonese target: source has restrictive 'only' "
|
|
2696
|
+
"but output uses Mandarin 只有/只是/只用/只能 "
|
|
2697
|
+
"without any Cantonese-exclusive particles "
|
|
2698
|
+
"(嘅/係/唔/冇/咋/喎/喇/佢) — output is Mandarin, "
|
|
2699
|
+
"not Cantonese; "
|
|
2700
|
+
"(cantonese_sentence_final_particle_traps, check #89)"
|
|
2701
|
+
),
|
|
2702
|
+
}
|
|
2703
|
+
|
|
2704
|
+
# check #90: Cantonese 俾 substitution — source has permissive/benefactive
|
|
2705
|
+
# trigger ('do not allow'/'do not let'/'allow the'/'let the'/'give the patient')
|
|
2706
|
+
# + en-yue output uses Mandarin permissive/benefactive marker (讓/让/給/给/被)
|
|
2707
|
+
# AND lacks Cantonese 俾 AND lacks ALL Cantonese-exclusive markers
|
|
2708
|
+
# (嘅/係/唔/冇/㗎/咋/喎/喇/佢) → catastrophic/wrong_language.
|
|
2709
|
+
# Targets cantonese_bei2_construction_traps: 俾 is obligatory in colloquial
|
|
2710
|
+
# Cantonese benefactive/permissive; 讓/让/給/给 in these positions are Mandarin.
|
|
2711
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2712
|
+
cjk_cnt_bei2 = sum(
|
|
2713
|
+
1 for c in tgt
|
|
2714
|
+
if 0x4E00 <= ord(c) <= 0x9FFF or 0x3400 <= ord(c) <= 0x4DBF
|
|
2715
|
+
)
|
|
2716
|
+
if cjk_cnt_bei2 >= 5:
|
|
2717
|
+
bei2_triggers = re.search(
|
|
2718
|
+
r'\b(?:do\s+not\s+(?:allow|let)|allow\s+the|let\s+the|give\s+the\s+patient)\b',
|
|
2719
|
+
src, re.IGNORECASE
|
|
2720
|
+
)
|
|
2721
|
+
if bei2_triggers:
|
|
2722
|
+
has_mandarin_perm = re.search(r'[讓让給给被]', tgt)
|
|
2723
|
+
has_bei2 = '俾' in tgt
|
|
2724
|
+
has_cantonese = re.search(r'[嘅係唔冇㗎咋喎喇佢]', tgt)
|
|
2725
|
+
if has_mandarin_perm and not has_bei2 and not has_cantonese:
|
|
2726
|
+
return {
|
|
2727
|
+
"verdict": "catastrophic",
|
|
2728
|
+
"failure_type": "wrong_language",
|
|
2729
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
2730
|
+
"format": 3, "safety": 2,
|
|
2731
|
+
"notes": (
|
|
2732
|
+
"Cantonese target: source has permissive/benefactive "
|
|
2733
|
+
"('allow'/'let'/'give the patient') but output uses "
|
|
2734
|
+
"Mandarin marker 讓/让/給/给/被 without Cantonese 俾 "
|
|
2735
|
+
"and without any Cantonese-exclusive particles "
|
|
2736
|
+
"(嘅/係/唔/冇/咋/喎/喇/佢) — output is Mandarin grammar, "
|
|
2737
|
+
"not Cantonese; "
|
|
2738
|
+
"(cantonese_bei2_construction_traps, check #90)"
|
|
2739
|
+
),
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
# Check #91 — Cantonese aspect marker traps: Mandarin progressive marker instead of Cantonese 緊
|
|
2743
|
+
# Source has explicit progressive signal ("is currently X-ing" / "is now X-ing")
|
|
2744
|
+
# Cantonese should use 緊 (gan2); Mandarin uses 着 or 正在
|
|
2745
|
+
if lang_pair == "en-yue":
|
|
2746
|
+
has_explicit_progressive = re.search(
|
|
2747
|
+
r'\b(?:is\s+currently|is\s+now|right\s+now)\b', src, re.IGNORECASE
|
|
2748
|
+
)
|
|
2749
|
+
has_ing_verb = re.search(r'\b\w+ing\b', src)
|
|
2750
|
+
if has_explicit_progressive and has_ing_verb:
|
|
2751
|
+
has_cantonese_progressive = '緊' in tgt
|
|
2752
|
+
has_mandarin_progressive = re.search(r'[着正]在', tgt) or (
|
|
2753
|
+
'着' in tgt and '緊' not in tgt
|
|
2754
|
+
)
|
|
2755
|
+
has_cantonese_markers = re.search(r'[嘅係唔冇㗎咋喎喇佢]', tgt)
|
|
2756
|
+
if has_mandarin_progressive and not has_cantonese_progressive and not has_cantonese_markers:
|
|
2757
|
+
return {
|
|
2758
|
+
"verdict": "catastrophic",
|
|
2759
|
+
"failure_type": "wrong_language",
|
|
2760
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
2761
|
+
"format": 3, "safety": 2,
|
|
2762
|
+
"notes": (
|
|
2763
|
+
"Cantonese target: source has explicit progressive marker "
|
|
2764
|
+
"('is currently'/'is now' + -ing verb) but output uses "
|
|
2765
|
+
"Mandarin progressive 着/正在 without Cantonese progressive "
|
|
2766
|
+
"marker 緊 and without Cantonese-exclusive particles "
|
|
2767
|
+
"(嘅/係/唔/冇/㗎/咋/喎/喇/佢) — output is Mandarin, "
|
|
2768
|
+
"not Cantonese; "
|
|
2769
|
+
"(cantonese_aspect_marker_traps, check #91)"
|
|
2770
|
+
),
|
|
2771
|
+
}
|
|
2772
|
+
|
|
2773
|
+
# Check #92: Swedish comparative/superlative — source has irregular English comparative
|
|
2774
|
+
# (better/worse/best/worst/smaller/larger) but target uses wrong periphrastic form
|
|
2775
|
+
# (mer bra / mest bra / mer dålig / mest dålig / mer stor / mer liten) instead of
|
|
2776
|
+
# correct morphological forms (bättre / sämre / bäst / sämst / större / mindre).
|
|
2777
|
+
# Targets swedish_comparative_superlative_traps: morphological comparison is obligatory
|
|
2778
|
+
# for these common adjectives; mer/mest with bra/dålig/stor/liten is non-standard.
|
|
2779
|
+
if lang_pair == "en-sv":
|
|
2780
|
+
sv_comp_error_pairs = [
|
|
2781
|
+
(r'\b(?:better|more\s+good)\b', r'\bmer\s+bra\b'),
|
|
2782
|
+
(r'\b(?:best|the\s+best)\b', r'\bmest\s+bra\b'),
|
|
2783
|
+
(r'\b(?:worse|more\s+bad)\b', r'\bmer\s+d[aå]lig\b'),
|
|
2784
|
+
(r'\b(?:worst|the\s+worst)\b', r'\bmest\s+d[aå]lig\b'),
|
|
2785
|
+
(r'\b(?:larger|bigger|more\s+large)\b', r'\bmer\s+stor\b'),
|
|
2786
|
+
(r'\b(?:smaller|more\s+small)\b', r'\bmer\s+liten\b'),
|
|
2787
|
+
]
|
|
2788
|
+
for sv_src_pat, sv_tgt_pat in sv_comp_error_pairs:
|
|
2789
|
+
if (re.search(sv_src_pat, src, re.IGNORECASE)
|
|
2790
|
+
and re.search(sv_tgt_pat, tgt, re.IGNORECASE)):
|
|
2791
|
+
return {
|
|
2792
|
+
"verdict": "catastrophic",
|
|
2793
|
+
"failure_type": "hallucination",
|
|
2794
|
+
"adequacy": 2, "fluency": 2, "faithfulness": 2,
|
|
2795
|
+
"format": 3, "safety": 3,
|
|
2796
|
+
"notes": (
|
|
2797
|
+
"Swedish target: English irregular comparative/superlative "
|
|
2798
|
+
"(better/worse/best/worst/larger/smaller) translated using "
|
|
2799
|
+
"wrong periphrastic form (mer bra/mer dålig/mer stor/mer liten) "
|
|
2800
|
+
"instead of required morphological form "
|
|
2801
|
+
"(bättre/sämre/bäst/sämst/större/mindre); "
|
|
2802
|
+
"(swedish_comparative_superlative_traps, check #92)"
|
|
2803
|
+
),
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
# check #93: Hindi temporal boundary erasure — source has 'until' but Hindi
|
|
2807
|
+
# output lacks 'जब तक' (the obligatory Hindi translation) AND lacks English
|
|
2808
|
+
# 'until' passthrough. Hindi 'until' requires 'जब तक...न' with a negative
|
|
2809
|
+
# polarity marker; if neither जब तक nor the English word appears, the temporal
|
|
2810
|
+
# endpoint of the instruction was silently dropped.
|
|
2811
|
+
# Guards: (a) Hindi target; (b) source has standalone 'until';
|
|
2812
|
+
# (c) ≥5 Devanagari chars in output (model attempted Hindi, not wrong-language);
|
|
2813
|
+
# (d) output lacks both जब तक and English 'until'.
|
|
2814
|
+
# False-positive mitigation: pass-through of 'until' as English is also a
|
|
2815
|
+
# failure but is caught by check #13 (code-mixed) — here we only fire when
|
|
2816
|
+
# the Devanagari output contains zero temporal boundary marker of either form.
|
|
2817
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2818
|
+
if re.search(r'\buntil\b', src, re.IGNORECASE):
|
|
2819
|
+
dev_chars_93 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
2820
|
+
if dev_chars_93 >= 5:
|
|
2821
|
+
has_jabtak = bool(re.search(r'जब\s*तक', tgt))
|
|
2822
|
+
has_eng_until = bool(re.search(r'\buntil\b', tgt, re.IGNORECASE))
|
|
2823
|
+
if not has_jabtak and not has_eng_until:
|
|
2824
|
+
return {
|
|
2825
|
+
"verdict": "catastrophic",
|
|
2826
|
+
"failure_type": "semantic_substitution",
|
|
2827
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1,
|
|
2828
|
+
"format": 3, "safety": 2,
|
|
2829
|
+
"notes": (
|
|
2830
|
+
"Hindi temporal boundary erasure: source has 'until' but "
|
|
2831
|
+
"Hindi output lacks जब तक (obligatory translation of 'until') "
|
|
2832
|
+
"and no English 'until' passthrough — the temporal endpoint "
|
|
2833
|
+
"was dropped; 'take until fever subsides' without जब तक "
|
|
2834
|
+
"converts a bounded instruction into an open-ended one; "
|
|
2835
|
+
"(hindi_temporal_until_clause_traps, check #93)"
|
|
2836
|
+
),
|
|
2837
|
+
}
|
|
2838
|
+
|
|
2839
|
+
# check #94: Kannada discourse connective English passthrough — source has an
|
|
2840
|
+
# adversative/causal/concessive/resultative connective AND Kannada output
|
|
2841
|
+
# (≥10 Kannada chars) contains the English connective verbatim.
|
|
2842
|
+
# Catches: 'although the patient is stable, although [Kannada]...' where model
|
|
2843
|
+
# translated the clauses but left the connective as an English passthrough.
|
|
2844
|
+
# Guards: (a) Kannada target; (b) ≥10 Kannada chars in output (model produced
|
|
2845
|
+
# Kannada — check #32 already catches zero-Kannada outputs); (c) English
|
|
2846
|
+
# connective appears in source; (d) same English connective appears in output.
|
|
2847
|
+
if target_lang.lower() in ('kn', 'kannada'):
|
|
2848
|
+
kn_chars_94 = sum(1 for c in tgt if 0x0C80 <= ord(c) <= 0x0CFF)
|
|
2849
|
+
if kn_chars_94 >= 10:
|
|
2850
|
+
eng_conn_match = re.search(
|
|
2851
|
+
r'\b(although|despite|even\s+though|however|therefore|'
|
|
2852
|
+
r'consequently|because|since)\b',
|
|
2853
|
+
src, re.IGNORECASE,
|
|
2854
|
+
)
|
|
2855
|
+
if eng_conn_match:
|
|
2856
|
+
eng_conn_word = eng_conn_match.group(0)
|
|
2857
|
+
if re.search(
|
|
2858
|
+
r'\b' + re.escape(eng_conn_word) + r'\b',
|
|
2859
|
+
tgt, re.IGNORECASE,
|
|
2860
|
+
):
|
|
2861
|
+
return {
|
|
2862
|
+
"verdict": "bad",
|
|
2863
|
+
"failure_type": "copy_through",
|
|
2864
|
+
"adequacy": 2, "fluency": 2, "faithfulness": 2,
|
|
2865
|
+
"format": 2, "safety": 4,
|
|
2866
|
+
"notes": (
|
|
2867
|
+
f"Kannada discourse connective passthrough: "
|
|
2868
|
+
f"'{eng_conn_word}' left verbatim in English inside "
|
|
2869
|
+
"Kannada output — the logical connective was not "
|
|
2870
|
+
"translated (ಆದರೂ/ಆದರೆ/ಏಕೆಂದರೆ/ಆದ್ದರಿಂದ required); "
|
|
2871
|
+
"(kannada_discourse_connective_traps, check #94)"
|
|
2872
|
+
),
|
|
2873
|
+
}
|
|
2874
|
+
|
|
2875
|
+
# check #95: Hindi conditional marker erasure — source has "if you [verb]" or
|
|
2876
|
+
# "if the patient [verb]" but Hindi output (≥20 Devanagari chars) lacks all
|
|
2877
|
+
# conditional markers (अगर, यदि, जब) and lacks English passthrough "if".
|
|
2878
|
+
# Catches: model produces genuine Hindi but drops the conditional clause marker,
|
|
2879
|
+
# turning "If you experience X, do Y" into an unconditional instruction.
|
|
2880
|
+
# Guards: (a) Hindi target; (b) ≥20 Devanagari chars; (c) source has "if you/
|
|
2881
|
+
# the patient"; (d) no conditional marker and no English "if" in output.
|
|
2882
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
2883
|
+
dev_chars_95 = sum(1 for c in tgt if 0x0900 <= ord(c) <= 0x097F)
|
|
2884
|
+
if dev_chars_95 >= 20:
|
|
2885
|
+
src_has_conditional_95 = bool(re.search(
|
|
2886
|
+
r'\bif\s+(?:you|the\s+patient|a\s+patient)\b',
|
|
2887
|
+
src, re.IGNORECASE,
|
|
2888
|
+
))
|
|
2889
|
+
if src_has_conditional_95:
|
|
2890
|
+
has_cond_marker_95 = bool(re.search(r'(?:अगर|यदि|जब)', tgt))
|
|
2891
|
+
has_eng_if_95 = bool(re.search(r'\bif\b', tgt, re.IGNORECASE))
|
|
2892
|
+
if not has_cond_marker_95 and not has_eng_if_95:
|
|
2893
|
+
return {
|
|
2894
|
+
"verdict": "bad",
|
|
2895
|
+
"failure_type": "conditional_erasure",
|
|
2896
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 2,
|
|
2897
|
+
"format": 3, "safety": 4,
|
|
2898
|
+
"notes": (
|
|
2899
|
+
"Hindi conditional marker erasure: source has 'if you/the patient' "
|
|
2900
|
+
"but Hindi output lacks अगर/यदि/जब — the conditional clause marker "
|
|
2901
|
+
"was dropped, converting a safety-critical conditional instruction "
|
|
2902
|
+
"into an unconditional command "
|
|
2903
|
+
"(hindi_conditional_subjunctive_traps, check #95)"
|
|
2904
|
+
),
|
|
2905
|
+
}
|
|
2906
|
+
|
|
2907
|
+
# check #96: Cantonese evidential particle 喎 dropped — source has an evidential
|
|
2908
|
+
# trigger (apparently/reportedly/allegedly/it is said/I heard that) but the
|
|
2909
|
+
# en-yue output (≥15 CJK chars) lacks 喎 AND lacks any Mandarin evidential
|
|
2910
|
+
# substitute (據說/聽說/據報/據稱).
|
|
2911
|
+
# Catches: model translates the sentence content correctly but asserts it as
|
|
2912
|
+
# direct fact rather than reported speech, losing the epistemic hedge.
|
|
2913
|
+
# Guards: (a) Cantonese target; (b) ≥15 CJK chars in output; (c) evidential
|
|
2914
|
+
# marker in source; (d) 喎 absent AND Mandarin evidential absent.
|
|
2915
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2916
|
+
cjk_chars_96 = sum(
|
|
2917
|
+
1 for c in tgt
|
|
2918
|
+
if 0x4E00 <= ord(c) <= 0x9FFF or 0x3400 <= ord(c) <= 0x4DBF
|
|
2919
|
+
)
|
|
2920
|
+
if cjk_chars_96 >= 15:
|
|
2921
|
+
src_has_evidential_96 = bool(re.search(
|
|
2922
|
+
r'\b(?:apparently|reportedly|it\s+is\s+(?:said|reported|claimed)|'
|
|
2923
|
+
r'(?:i\s+)?heard\s+that|allegedly)\b',
|
|
2924
|
+
src, re.IGNORECASE,
|
|
2925
|
+
))
|
|
2926
|
+
if src_has_evidential_96:
|
|
2927
|
+
has_wo3_96 = '喎' in tgt
|
|
2928
|
+
has_mandarin_evidential_96 = bool(re.search(
|
|
2929
|
+
r'(?:據說|聽說|據報|據稱|據悉)', tgt,
|
|
2930
|
+
))
|
|
2931
|
+
if not has_wo3_96 and not has_mandarin_evidential_96:
|
|
2932
|
+
return {
|
|
2933
|
+
"verdict": "bad",
|
|
2934
|
+
"failure_type": "evidential_particle_dropped",
|
|
2935
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 2,
|
|
2936
|
+
"format": 3, "safety": 3,
|
|
2937
|
+
"notes": (
|
|
2938
|
+
"Cantonese evidential particle 喎 dropped: source has hearsay "
|
|
2939
|
+
"marker (apparently/reportedly/allegedly/heard that) but output "
|
|
2940
|
+
"lacks 喎 and lacks Mandarin evidential (據說/聽說/據報) — "
|
|
2941
|
+
"the reported-speech epistemic stance was lost, asserting the "
|
|
2942
|
+
"claim as direct observed fact "
|
|
2943
|
+
"(cantonese_sentence_final_particle_traps, check #96)"
|
|
2944
|
+
),
|
|
2945
|
+
}
|
|
2946
|
+
|
|
2947
|
+
# check #97: Swedish particle verb — source has "prescribe"/"discharge" and
|
|
2948
|
+
# Swedish output contains a form of "skriva" but lacks "ut" anywhere.
|
|
2949
|
+
# Without "ut", "skriva" means merely "write" — the particle is essential
|
|
2950
|
+
# for both "prescribe" (skriva ut recept) and "discharge" (skriva ut patienten).
|
|
2951
|
+
# Guards: (a) Swedish target; (b) source has prescrib-/discharg-; (c) output
|
|
2952
|
+
# has a form of skriva; (d) "ut" absent from entire output.
|
|
2953
|
+
if target_lang.lower() in ('sv', 'swedish'):
|
|
2954
|
+
src_has_prescribe_97 = bool(re.search(
|
|
2955
|
+
r'\b(?:prescri(?:be|bed|bing|bes|ption)|discharg(?:e|ed|es|ing))\b',
|
|
2956
|
+
src, re.IGNORECASE,
|
|
2957
|
+
))
|
|
2958
|
+
if src_has_prescribe_97:
|
|
2959
|
+
has_skriva_form_97 = bool(re.search(
|
|
2960
|
+
r'\b(?:skriva|skriv(?:er|a|it|et|s|ar)?|skrevs?)\b',
|
|
2961
|
+
tgt, re.IGNORECASE,
|
|
2962
|
+
))
|
|
2963
|
+
if has_skriva_form_97:
|
|
2964
|
+
has_ut_97 = bool(re.search(r'\but\b', tgt, re.IGNORECASE))
|
|
2965
|
+
if not has_ut_97:
|
|
2966
|
+
return {
|
|
2967
|
+
"verdict": "bad",
|
|
2968
|
+
"failure_type": "particle_dropped",
|
|
2969
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 2,
|
|
2970
|
+
"format": 3, "safety": 4,
|
|
2971
|
+
"notes": (
|
|
2972
|
+
"Swedish particle verb particle dropped: source has 'prescribe'/"
|
|
2973
|
+
"'discharge' and output has a form of 'skriva' but lacks 'ut' — "
|
|
2974
|
+
"'skriva' alone means 'write', not 'prescribe/discharge'; "
|
|
2975
|
+
"the separable particle 'ut' was dropped, destroying the meaning "
|
|
2976
|
+
"(swedish_particle_verb_traps, check #97)"
|
|
2977
|
+
),
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
# check #98: Cantonese colloquial temporal vocabulary — source has "yesterday" but
|
|
2981
|
+
# output uses Mandarin 昨天 instead of Cantonese 噚日/琴日. 昨天 is pure colloquial
|
|
2982
|
+
# Mandarin; Cantonese for "yesterday" is 噚日 (cam4 jat6) or 琴日. An output
|
|
2983
|
+
# with 昨天 and no Cantonese form is a wrong_language (Mandarin) error.
|
|
2984
|
+
# Guards: (a) Cantonese target; (b) source has "yesterday"; (c) output has 昨天;
|
|
2985
|
+
# (d) output lacks both 噚日 and 琴日.
|
|
2986
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
2987
|
+
if re.search(r'\byesterday\b', src, re.IGNORECASE):
|
|
2988
|
+
if '昨天' in tgt and '噚日' not in tgt and '琴日' not in tgt:
|
|
2989
|
+
return {
|
|
2990
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
2991
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 3, "safety": 3,
|
|
2992
|
+
"notes": (
|
|
2993
|
+
"Cantonese target output uses Mandarin temporal word 昨天 ('yesterday') "
|
|
2994
|
+
"instead of Cantonese 噚日/琴日 — output is Mandarin vocabulary, "
|
|
2995
|
+
"not colloquial Cantonese; "
|
|
2996
|
+
"(cantonese_colloquial_lexicon_traps, check #98)"
|
|
2997
|
+
),
|
|
2998
|
+
}
|
|
2999
|
+
|
|
3000
|
+
# check #99: Swedish-Nordic language discrimination — output contains "ikke"
|
|
3001
|
+
# (Norwegian Bokmål / Danish negation) without any "inte"/"icke" (Swedish
|
|
3002
|
+
# negation). "Ikke" is used in both Norwegian and Danish but never in Swedish,
|
|
3003
|
+
# where the standard negation is "inte". An output with "ikke" and no "inte"
|
|
3004
|
+
# is Norwegian or Danish, not Swedish.
|
|
3005
|
+
# Guards: (a) Swedish target; (b) output has word "ikke"; (c) no "inte"/"icke".
|
|
3006
|
+
if target_lang.lower() in ('sv', 'swedish'):
|
|
3007
|
+
if re.search(r'\bikke\b', tgt, re.IGNORECASE):
|
|
3008
|
+
has_swedish_neg_99 = bool(re.search(r'\b(?:inte|icke)\b', tgt, re.IGNORECASE))
|
|
3009
|
+
if not has_swedish_neg_99:
|
|
3010
|
+
return {
|
|
3011
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3012
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 3, "safety": 2,
|
|
3013
|
+
"notes": (
|
|
3014
|
+
"Swedish target output uses 'ikke' (Norwegian Bokmål/Danish negation) "
|
|
3015
|
+
"without any 'inte'/'icke' (Swedish negation) — output is "
|
|
3016
|
+
"Norwegian or Danish, not Swedish; "
|
|
3017
|
+
"(swedish_nordic_language_discrimination_traps, check #99)"
|
|
3018
|
+
),
|
|
3019
|
+
}
|
|
3020
|
+
|
|
3021
|
+
# check #100: Kannada English negation passthrough — source has present/past negation
|
|
3022
|
+
# (does not/did not/has not etc., distinct from prohibitive 'must not'/'do not'
|
|
3023
|
+
# handled by check #38) and the Kannada output contains the standalone English
|
|
3024
|
+
# word "not" among Kannada characters. Kannada encodes negation morphologically
|
|
3025
|
+
# (ುವುದಿಲ್ಲ/ಲಿಲ್ಲ/ಇಲ್ಲ) — a standalone "not" in Latin script signals code-mixing.
|
|
3026
|
+
# Guards: (a) Kannada target; (b) source has present/past negation;
|
|
3027
|
+
# (c) ≥5 Kannada chars in output (partial Kannada, not caught by check #32);
|
|
3028
|
+
# (d) output contains standalone "not".
|
|
3029
|
+
if target_lang.lower() in ('kn', 'kannada'):
|
|
3030
|
+
if re.search(
|
|
3031
|
+
r'\b(?:does\s+not|did\s+not|has\s+not|have\s+not|'
|
|
3032
|
+
r'is\s+not|are\s+not|was\s+not|were\s+not|'
|
|
3033
|
+
r'not\s+effective|not\s+cure|not\s+heal)\b',
|
|
3034
|
+
src, re.IGNORECASE,
|
|
3035
|
+
):
|
|
3036
|
+
kn_chars_100 = sum(1 for c in tgt if 0x0C80 <= ord(c) <= 0x0CFF)
|
|
3037
|
+
if kn_chars_100 >= 5 and re.search(r'\bnot\b', tgt, re.IGNORECASE):
|
|
3038
|
+
return {
|
|
3039
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3040
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 2,
|
|
3041
|
+
"notes": (
|
|
3042
|
+
"Kannada translation contains standalone English 'not' in a "
|
|
3043
|
+
"present/past negative context — Kannada encodes negation "
|
|
3044
|
+
"morphologically (ುವುದಿಲ್ಲ/ಲಿಲ್ಲ/ಇಲ್ಲ) and needs no separate 'not'; "
|
|
3045
|
+
"its presence is an English code-mixing failure; "
|
|
3046
|
+
"(kannada_negative_verb_morphology_traps, check #100)"
|
|
3047
|
+
),
|
|
3048
|
+
}
|
|
3049
|
+
|
|
3050
|
+
# Check #101 — kannada_romanization_passthrough_traps
|
|
3051
|
+
# Small models output Kannada content in Roman/Latin script (transliteration)
|
|
3052
|
+
# instead of Kannada script. Detectable: en-kn target, source has
|
|
3053
|
+
# prohibitive/dose/imperative content, output is substantial (>15 chars)
|
|
3054
|
+
# but has very few Kannada script characters (<8).
|
|
3055
|
+
# Guards: (a) Kannada target; (b) source has dose/prohibitive/emergency content;
|
|
3056
|
+
# (c) output length > 15 chars; (d) Kannada script chars < 8.
|
|
3057
|
+
# Note: check #32 fires first when Kannada chars == 0; this catches 1-7 chars
|
|
3058
|
+
# (partial romanization with a few Kannada function words).
|
|
3059
|
+
if target_lang.lower() in ('kn', 'kannada'):
|
|
3060
|
+
kn_chars_101 = sum(1 for c in tgt if 0x0C80 <= ord(c) <= 0x0CFF)
|
|
3061
|
+
if len(tgt) > 15 and kn_chars_101 < 8:
|
|
3062
|
+
if re.search(
|
|
3063
|
+
r'\b(?:do\s+not|must\s+not|never|take|dose|mg\b|tablet|inject|'
|
|
3064
|
+
r'administer|stop|continue|daily|morning|evening|apply|call|'
|
|
3065
|
+
r'avoid|exceed|children|patient|allergic|emergency)\b',
|
|
3066
|
+
src, re.IGNORECASE,
|
|
3067
|
+
):
|
|
3068
|
+
return {
|
|
3069
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3070
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3071
|
+
"notes": (
|
|
3072
|
+
"Kannada target output has fewer than 8 Kannada script characters "
|
|
3073
|
+
"for a substantial source with prohibitive/dosing/imperative content — "
|
|
3074
|
+
"output is Roman/Latin transliteration or wrong-script substitution "
|
|
3075
|
+
"instead of proper Kannada script (ಕ-ಹ range U+0C80-U+0CFF); "
|
|
3076
|
+
"(kannada_romanization_passthrough_traps, check #101)"
|
|
3077
|
+
),
|
|
3078
|
+
}
|
|
3079
|
+
|
|
3080
|
+
# Check #102 — kannada_compound_noun_traps
|
|
3081
|
+
# Source mentions "blood pressure" but Kannada output lacks ರಕ್ತ (blood).
|
|
3082
|
+
# ರಕ್ತದೊತ್ತಡ = ರಕ್ತ + ಒತ್ತಡ: if ರಕ್ತ is absent, the compound noun is absent.
|
|
3083
|
+
# Also checks "diabetes" → ಮಧುಮೇಹ (must contain ಮಧು or ಮೇಹ).
|
|
3084
|
+
# Guards: (a) Kannada target; (b) source contains the trigger term;
|
|
3085
|
+
# (c) output is substantial (>10 chars); (d) key Kannada component absent.
|
|
3086
|
+
if target_lang.lower() in ('kn', 'kannada') and len(tgt) > 10:
|
|
3087
|
+
if re.search(r'\bblood\s+pressure\b', src, re.IGNORECASE):
|
|
3088
|
+
if 'ರಕ್ತ' not in tgt:
|
|
3089
|
+
return {
|
|
3090
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3091
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 2,
|
|
3092
|
+
"notes": (
|
|
3093
|
+
"Kannada translation of 'blood pressure' lacks the Kannada component "
|
|
3094
|
+
"ರಕ್ತ (blood) — the compound ರಕ್ತದೊತ್ತಡ is absent; likely English "
|
|
3095
|
+
"passthrough ('blood pressure') or wrong compound substitution; "
|
|
3096
|
+
"(kannada_compound_noun_traps, check #102)"
|
|
3097
|
+
),
|
|
3098
|
+
}
|
|
3099
|
+
if re.search(r'\bdiabetes\b', src, re.IGNORECASE):
|
|
3100
|
+
if 'ಮಧು' not in tgt and 'ಮೇಹ' not in tgt:
|
|
3101
|
+
return {
|
|
3102
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3103
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 2,
|
|
3104
|
+
"notes": (
|
|
3105
|
+
"Kannada translation of 'diabetes' lacks both ಮಧು and ಮೇಹ — "
|
|
3106
|
+
"the compound ಮಧುಮೇಹ is absent; likely English passthrough or "
|
|
3107
|
+
"wrong term substitution; "
|
|
3108
|
+
"(kannada_compound_noun_traps, check #102)"
|
|
3109
|
+
),
|
|
3110
|
+
}
|
|
3111
|
+
|
|
3112
|
+
# Check #103 — kannada_telugu_script_confusion_traps
|
|
3113
|
+
# en-kn: if Telugu char count (U+0C00-U+0C7F) exceeds Kannada char count
|
|
3114
|
+
# (U+0C80-U+0CFF) and there are at least 5 Telugu chars, the model has
|
|
3115
|
+
# defaulted to Telugu output instead of Kannada.
|
|
3116
|
+
# Guards: (a) Kannada target; (b) output >10 chars; (c) telugu >= 5;
|
|
3117
|
+
# (d) telugu > kannada.
|
|
3118
|
+
if target_lang.lower() in ('kn', 'kannada') and len(tgt) > 10:
|
|
3119
|
+
_telugu_chars = sum(1 for c in tgt if '\u0C00' <= c <= '\u0C7F')
|
|
3120
|
+
_kannada_chars = sum(1 for c in tgt if '\u0C80' <= c <= '\u0CFF')
|
|
3121
|
+
if _telugu_chars >= 5 and _telugu_chars > _kannada_chars:
|
|
3122
|
+
return {
|
|
3123
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3124
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3125
|
+
"notes": (
|
|
3126
|
+
f"Kannada target output has more Telugu script characters "
|
|
3127
|
+
f"(U+0C00-U+0C7F, count={_telugu_chars}) than Kannada script "
|
|
3128
|
+
f"characters (U+0C80-U+0CFF, count={_kannada_chars}) — model has "
|
|
3129
|
+
"defaulted to Telugu rather than Kannada; both scripts are adjacent "
|
|
3130
|
+
"Dravidian blocks in Unicode and small models with disproportionate "
|
|
3131
|
+
"Telugu training data systematically produce Telugu output for Kannada "
|
|
3132
|
+
"targets; dominant failure mode in phase-6 en-kn run (76.1% CAT); "
|
|
3133
|
+
"(kannada_telugu_script_confusion_traps, check #103)"
|
|
3134
|
+
),
|
|
3135
|
+
}
|
|
3136
|
+
|
|
3137
|
+
# Check #104 — cantonese_concessive_prohibition_traps
|
|
3138
|
+
# en-yue: source has concessive clause ("even though"/"although") AND a prohibition
|
|
3139
|
+
# ("must not"/"cannot"/"prohibited") but Cantonese output lacks BOTH a concessive
|
|
3140
|
+
# marker (雖然/就算/即使/縱使) AND a Cantonese prohibition (唔可以/禁止/禁用/唔得/嚴禁).
|
|
3141
|
+
# Directly observed failure: 'Even though the drug is approved, it must not be given
|
|
3142
|
+
# to patients under 18' → '仲唔要講,要睇病人嘅年齡上限' (entire conditional frame lost).
|
|
3143
|
+
# Guards: (a) Cantonese target; (b) output >15 chars; (c) both triggers in source;
|
|
3144
|
+
# (d) both concessive marker and prohibition marker absent from output.
|
|
3145
|
+
if target_lang.lower() in ('yue', 'cantonese') and len(tgt) > 10:
|
|
3146
|
+
if re.search(r'\b(even though|although|even if)\b', src, re.IGNORECASE):
|
|
3147
|
+
if re.search(
|
|
3148
|
+
r'\b(must not|must never|cannot|shall not|prohibited|forbidden)\b',
|
|
3149
|
+
src, re.IGNORECASE,
|
|
3150
|
+
):
|
|
3151
|
+
_yue_concessives = ['雖然', '就算', '即使', '縱使', '哪怕']
|
|
3152
|
+
_yue_prohibitions = ['唔可以', '禁止', '禁用', '唔得', '嚴禁']
|
|
3153
|
+
_has_conc = any(c in tgt for c in _yue_concessives)
|
|
3154
|
+
_has_proh = any(p in tgt for p in _yue_prohibitions)
|
|
3155
|
+
if not _has_conc and not _has_proh:
|
|
3156
|
+
return {
|
|
3157
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
3158
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3159
|
+
"notes": (
|
|
3160
|
+
"Cantonese translation of source containing concessive clause "
|
|
3161
|
+
"('even though/although') plus prohibition ('must not/cannot') "
|
|
3162
|
+
"lacks both a Cantonese concessive marker (雖然/就算/即使/縱使) "
|
|
3163
|
+
"and a Cantonese prohibition (唔可以/禁止/禁用/唔得/嚴禁) — "
|
|
3164
|
+
"the conditional exception frame has been entirely dropped, "
|
|
3165
|
+
"converting a conditional restriction to an unqualified statement; "
|
|
3166
|
+
"directly observed in phase-6 en-yue run: 'Even though the drug "
|
|
3167
|
+
"is approved, it must not be given to patients under 18' → output "
|
|
3168
|
+
"loses approval context and prohibition entirely; "
|
|
3169
|
+
"(cantonese_concessive_prohibition_traps, check #104)"
|
|
3170
|
+
),
|
|
3171
|
+
}
|
|
3172
|
+
|
|
3173
|
+
# Check #105 — swedish_profession_near_miss_traps
|
|
3174
|
+
# en-sv: source has "engineer" but Swedish output contains "meningen" (the sentence/
|
|
3175
|
+
# meaning) — the exact profession-noun near-miss substitution observed in phase-6
|
|
3176
|
+
# en-sv run where 'the engineer recommended full evacuation' →
|
|
3177
|
+
# 'meningen rekommenderade fullständig evakuering'.
|
|
3178
|
+
# Guards: (a) Swedish target; (b) source has 'engineer'; (c) 'meningen' in output.
|
|
3179
|
+
if target_lang.lower() in ('sv', 'swedish') and len(tgt) > 10:
|
|
3180
|
+
if re.search(r'\bengineer\b', src, re.IGNORECASE):
|
|
3181
|
+
if 'meningen' in tgt.lower():
|
|
3182
|
+
return {
|
|
3183
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
3184
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 2,
|
|
3185
|
+
"notes": (
|
|
3186
|
+
"Swedish translation of 'engineer' produced 'meningen' (the sentence/"
|
|
3187
|
+
"meaning) instead of 'ingenjören' (the engineer) — direct profession-"
|
|
3188
|
+
"noun near-miss substitution observed in phase-6 en-sv run; "
|
|
3189
|
+
"'the engineer recommended full evacuation' → 'meningen rekommenderade "
|
|
3190
|
+
"fullständig evakuering'; the safety-critical professional agent is "
|
|
3191
|
+
"replaced by an inanimate noun, destroying accountability; "
|
|
3192
|
+
"(swedish_profession_near_miss_traps, check #105)"
|
|
3193
|
+
),
|
|
3194
|
+
}
|
|
3195
|
+
|
|
3196
|
+
# Check #106 — swedish_scandinavian_interference_traps
|
|
3197
|
+
# en-sv: if target is Swedish but output contains Norwegian Bokmål medical tokens
|
|
3198
|
+
# (ikke = not, sykehus = hospital, sykepleier = nurse, lege = doctor as standalone
|
|
3199
|
+
# word), the model has produced Norwegian instead of Swedish.
|
|
3200
|
+
# Guards: (a) Swedish target; (b) output >10 chars;
|
|
3201
|
+
# (c) Norwegian-specific token present (word-boundary checked for 'lege').
|
|
3202
|
+
if target_lang.lower() in ('sv', 'swedish') and len(tgt) > 10:
|
|
3203
|
+
tgt_lower = tgt.lower()
|
|
3204
|
+
_norw_simple = ['ikke', 'sykehus', 'sykepleier']
|
|
3205
|
+
_found_norw = next((tok for tok in _norw_simple if tok in tgt_lower), None)
|
|
3206
|
+
if _found_norw is None and re.search(r'\blege\b', tgt_lower):
|
|
3207
|
+
_found_norw = 'lege'
|
|
3208
|
+
if _found_norw is not None:
|
|
3209
|
+
return {
|
|
3210
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3211
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3212
|
+
"notes": (
|
|
3213
|
+
f"Swedish target output contains Norwegian Bokmål token '{_found_norw}' — "
|
|
3214
|
+
"model has produced Norwegian instead of Swedish: "
|
|
3215
|
+
"'ikke' (Norw: not) vs Swedish 'inte', "
|
|
3216
|
+
"'sykehus' (Norw: hospital) vs Swedish 'sjukhus', "
|
|
3217
|
+
"'sykepleier' (Norw: nurse) vs Swedish 'sjuksköterska', "
|
|
3218
|
+
"'lege' (Norw: doctor) vs Swedish 'läkare'; "
|
|
3219
|
+
"a Swedish patient unfamiliar with Norwegian may not understand the "
|
|
3220
|
+
"medical instruction, and word-order differences (Norwegian 'ikke kan' "
|
|
3221
|
+
"vs Swedish 'kan inte') change the placement of negation in safety "
|
|
3222
|
+
"instructions; phase-6 en-sv 25.5% CAT rate suggests systematic "
|
|
3223
|
+
"Scandinavian language confusion in small models; "
|
|
3224
|
+
"(swedish_scandinavian_interference_traps, check #106)"
|
|
3225
|
+
),
|
|
3226
|
+
}
|
|
3227
|
+
|
|
3228
|
+
# Check #107 — hindi_organ_adjective_confusion_traps
|
|
3229
|
+
# en-hi: source has a liver-specific organ adjective ('hepatic') but Hindi output
|
|
3230
|
+
# contains 'हृदय' (heart/cardiac) → the wrong organ system entirely.
|
|
3231
|
+
# 'hepatic' → 'हृदय' was directly observed in phase-6 en-hi catastrophic example.
|
|
3232
|
+
# Also catches pulmonary → वृक्क/गुर्दे (kidney) confusion.
|
|
3233
|
+
# Guards: (a) Hindi target; (b) output >15 chars; (c) source has organ adjective;
|
|
3234
|
+
# (d) output contains the WRONG organ word.
|
|
3235
|
+
if target_lang.lower() in ('hi', 'hindi') and len(tgt) > 15:
|
|
3236
|
+
src_lower = src.lower()
|
|
3237
|
+
_hepatic_in_src = 'hepatic' in src_lower
|
|
3238
|
+
_pulmonary_in_src = 'pulmonary' in src_lower
|
|
3239
|
+
_renal_in_src = bool(re.search(r'\brenal\b', src_lower))
|
|
3240
|
+
if _hepatic_in_src and 'हृदय' in tgt:
|
|
3241
|
+
return {
|
|
3242
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
3243
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3244
|
+
"notes": (
|
|
3245
|
+
"Source says 'hepatic' (liver-related) but Hindi output contains 'हृदय' "
|
|
3246
|
+
"(heart/cardiac) — the wrong organ system entirely; "
|
|
3247
|
+
"hepatic impairment/failure/enzymes involve the liver (यकृत/जिगर), "
|
|
3248
|
+
"not the heart; this organ-adjective substitution is clinically dangerous "
|
|
3249
|
+
"and was directly observed in phase-6 en-hi catastrophic run; "
|
|
3250
|
+
"(hindi_organ_adjective_confusion_traps, check #107)"
|
|
3251
|
+
),
|
|
3252
|
+
}
|
|
3253
|
+
if _pulmonary_in_src and re.search(r'वृक्क|गुर्दे', tgt):
|
|
3254
|
+
return {
|
|
3255
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
3256
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3257
|
+
"notes": (
|
|
3258
|
+
"Source says 'pulmonary' (lung-related) but Hindi output contains 'वृक्क' "
|
|
3259
|
+
"or 'गुर्दे' (kidney) — the wrong organ system; "
|
|
3260
|
+
"pulmonary embolism/oedema/function involve the lungs (फेफड़े/फुफ्फुस), "
|
|
3261
|
+
"not the kidneys; clinically dangerous organ-system confusion; "
|
|
3262
|
+
"(hindi_organ_adjective_confusion_traps, check #107)"
|
|
3263
|
+
),
|
|
3264
|
+
}
|
|
3265
|
+
|
|
3266
|
+
# Check #108 — cantonese_drug_identity_traps
|
|
3267
|
+
# en-yue: source mentions a specific drug but Cantonese output contains a different
|
|
3268
|
+
# drug from the same pharmacological class — directly observed in phase-6 en-yue run:
|
|
3269
|
+
# 'morphine sulfate, not hydromorphone' → output had 哌替啶 (pethidine) and 氢可待因
|
|
3270
|
+
# (hydrocodone), both wrong.
|
|
3271
|
+
# Guards: (a) Cantonese/Yue target; (b) output >10 chars; (c) source has drug name;
|
|
3272
|
+
# (d) output contains wrong drug name token.
|
|
3273
|
+
if target_lang.lower() in ('yue', 'cantonese') and len(tgt) > 10:
|
|
3274
|
+
src_lower = src.lower()
|
|
3275
|
+
# morphine → 哌替啶 (pethidine) substitution — directly observed
|
|
3276
|
+
if 'morphine' in src_lower and '哌替啶' in tgt:
|
|
3277
|
+
return {
|
|
3278
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
3279
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3280
|
+
"notes": (
|
|
3281
|
+
"Source says 'morphine' (嗎啡) but Cantonese output contains '哌替啶' "
|
|
3282
|
+
"(pethidine/meperidine) — a different opioid analgesic; "
|
|
3283
|
+
"this drug-identity substitution was directly observed in phase-6 en-yue "
|
|
3284
|
+
"catastrophic run; pethidine and morphine have different dosing, "
|
|
3285
|
+
"indications, and contraindications (e.g. pethidine is avoided in renal "
|
|
3286
|
+
"failure due to norpethidine accumulation); "
|
|
3287
|
+
"(cantonese_drug_identity_traps, check #108)"
|
|
3288
|
+
),
|
|
3289
|
+
}
|
|
3290
|
+
# hydromorphone → 可待因/codeine-class substitution
|
|
3291
|
+
if 'hydromorphone' in src_lower and re.search(r'可待因|氢可', tgt):
|
|
3292
|
+
return {
|
|
3293
|
+
"verdict": "catastrophic", "failure_type": "hallucination",
|
|
3294
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 2, "safety": 1,
|
|
3295
|
+
"notes": (
|
|
3296
|
+
"Source says 'hydromorphone' (氫嗎啡酮, Traditional) but Cantonese output "
|
|
3297
|
+
"contains '可待因' or '氢可' — codeine-class opioid substitution; "
|
|
3298
|
+
"hydromorphone is approximately 5-7× more potent than morphine while "
|
|
3299
|
+
"codeine is a weaker prodrug; wrong-drug identity is clinically dangerous; "
|
|
3300
|
+
"(cantonese_drug_identity_traps, check #108)"
|
|
3301
|
+
),
|
|
3302
|
+
}
|
|
3303
|
+
|
|
3304
|
+
# Check #109 — swedish_definiteness_suffix_traps
|
|
3305
|
+
# en-sv: Swedish uses suffix-based definiteness (tabletten, patienten, dosen)
|
|
3306
|
+
# NOT prepended article (German-style "den tablett").
|
|
3307
|
+
# Two locally detectable failures:
|
|
3308
|
+
# (a) 'patientet' — wrong neuter -et suffix on common-gender noun 'patient'
|
|
3309
|
+
# (should be 'patienten'); directly observed in phase-6 en-sv catastrophic.
|
|
3310
|
+
# (b) 'den patient' — German-style prepended article (no suffix on noun).
|
|
3311
|
+
# Guards: (a) Swedish target; (b) output >10 chars.
|
|
3312
|
+
if target_lang.lower() in ('sv', 'swedish') and len(tgt) > 10:
|
|
3313
|
+
tgt_lower = tgt.lower()
|
|
3314
|
+
if 'patientet' in tgt_lower:
|
|
3315
|
+
return {
|
|
3316
|
+
"verdict": "catastrophic", "failure_type": "wrong_inflection",
|
|
3317
|
+
"adequacy": 2, "fluency": 1, "faithfulness": 2, "format": 2, "safety": 3,
|
|
3318
|
+
"notes": (
|
|
3319
|
+
"Swedish output contains 'patientet' — wrong neuter definite suffix "
|
|
3320
|
+
"(-et) on the common-gender noun 'patient' (correct form: 'patienten'); "
|
|
3321
|
+
"directly observed in phase-6 en-sv catastrophic run; "
|
|
3322
|
+
"Swedish suffix-based definiteness requires gender-correct suffix: "
|
|
3323
|
+
"common gender → -en, neuter → -et; 'patient' is common gender; "
|
|
3324
|
+
"(swedish_definiteness_suffix_traps, check #109)"
|
|
3325
|
+
),
|
|
3326
|
+
}
|
|
3327
|
+
if re.search(r'\bden patient\b', tgt_lower):
|
|
3328
|
+
return {
|
|
3329
|
+
"verdict": "catastrophic", "failure_type": "wrong_inflection",
|
|
3330
|
+
"adequacy": 2, "fluency": 1, "faithfulness": 2, "format": 2, "safety": 3,
|
|
3331
|
+
"notes": (
|
|
3332
|
+
"Swedish output uses 'den patient' — German/French-style prepended "
|
|
3333
|
+
"definite article; Swedish encodes definiteness as a noun suffix "
|
|
3334
|
+
"(patienten, not 'den patient'); this is a systematic error from "
|
|
3335
|
+
"models trained on other European languages applying non-Swedish "
|
|
3336
|
+
"definiteness morphology; (swedish_definiteness_suffix_traps, check #109)"
|
|
3337
|
+
),
|
|
3338
|
+
}
|
|
3339
|
+
|
|
3340
|
+
# Check #110 — hindi_progressive_aspect_traps
|
|
3341
|
+
# en-hi: English "is/are [verb]ing" should produce Hindi progressive रह-:
|
|
3342
|
+
# 'is taking' → 'ले रहा/रही है', 'is spreading' → 'फैल रहा/रही है'.
|
|
3343
|
+
# If source has progressive English construction but Hindi output lacks the
|
|
3344
|
+
# progressive marker stem रह → the aspect was dropped and may have become
|
|
3345
|
+
# habitual (takes in general) instead of ongoing (currently taking).
|
|
3346
|
+
# Guards: (a) Hindi target; (b) output has >5 Devanagari chars;
|
|
3347
|
+
# (c) source has explicit present progressive pattern;
|
|
3348
|
+
# (d) output lacks रह (core progressive marker).
|
|
3349
|
+
if target_lang.lower() in ('hi', 'hindi') and len(tgt) > 15:
|
|
3350
|
+
_hi_devanagari_count = sum(1 for c in tgt if '\u0900' <= c <= '\u097F')
|
|
3351
|
+
if _hi_devanagari_count >= 5:
|
|
3352
|
+
_progressive_pat = re.search(
|
|
3353
|
+
r'\b(?:is|are|am)\s+(?:\w+ing)\b', src, re.IGNORECASE
|
|
3354
|
+
)
|
|
3355
|
+
if _progressive_pat and 'रह' not in tgt:
|
|
3356
|
+
return {
|
|
3357
|
+
"verdict": "catastrophic", "failure_type": "omission",
|
|
3358
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1, "format": 3, "safety": 2,
|
|
3359
|
+
"notes": (
|
|
3360
|
+
"English source has a present progressive construction "
|
|
3361
|
+
f"('{_progressive_pat.group()}') but Hindi output lacks the "
|
|
3362
|
+
"progressive aspect marker 'रह' (रहा/रही/रहे); "
|
|
3363
|
+
"dropping the progressive marker converts an ongoing event into "
|
|
3364
|
+
"a habitual one — clinically significant distinction e.g. "
|
|
3365
|
+
"'is taking' (ले रहा/रही है, current) vs 'takes' (लेता/लेती है, habitual); "
|
|
3366
|
+
"(hindi_progressive_aspect_traps, check #110)"
|
|
3367
|
+
),
|
|
3368
|
+
}
|
|
3369
|
+
|
|
3370
|
+
# Check #111 — cantonese_ama_question_traps
|
|
3371
|
+
# en-yue: Cantonese yes/no questions use A-not-A forms (係唔係, 有冇) or
|
|
3372
|
+
# completive 咗未, NOT Mandarin-style 吗 final particle.
|
|
3373
|
+
# If source is a yes/no question AND output contains '吗' (Simplified Chinese,
|
|
3374
|
+
# never used in Cantonese) → Mandarin-style output.
|
|
3375
|
+
# Guards: (a) Cantonese/Yue target; (b) source ends with '?';
|
|
3376
|
+
# (c) source starts with interrogative word; (d) output contains 吗 (Simplified).
|
|
3377
|
+
if target_lang.lower() in ('yue', 'cantonese') and len(tgt) > 5:
|
|
3378
|
+
_is_yn_question = bool(re.match(
|
|
3379
|
+
r'(?:Is|Are|Does|Do|Has|Have|Can|Could|Will|Would|Should|Was|Were)\b',
|
|
3380
|
+
src.strip(), re.IGNORECASE
|
|
3381
|
+
)) and src.strip().endswith('?')
|
|
3382
|
+
if _is_yn_question and '\u5417' in tgt: # 吗 U+5417, Simplified only
|
|
3383
|
+
return {
|
|
3384
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3385
|
+
"adequacy": 2, "fluency": 2, "faithfulness": 2, "format": 2, "safety": 2,
|
|
3386
|
+
"notes": (
|
|
3387
|
+
"Source is a yes/no question and Cantonese output contains '吗' "
|
|
3388
|
+
"(Simplified Chinese final question particle, U+5417) — "
|
|
3389
|
+
"Cantonese yes/no questions use A-not-A forms (係唔係, 有冇, 做唔做) "
|
|
3390
|
+
"or completive-negative 咗未, not the Mandarin-style 吗 particle; "
|
|
3391
|
+
"presence of Simplified '吗' (vs Traditional '嗎') indicates the "
|
|
3392
|
+
"output is Mandarin-structured, not Cantonese; "
|
|
3393
|
+
"(cantonese_ama_question_traps, check #111)"
|
|
3394
|
+
),
|
|
3395
|
+
}
|
|
3396
|
+
|
|
3397
|
+
# Check #112 — cantonese_gujarati_cyrillic_contamination_traps (Gujarati)
|
|
3398
|
+
# en-yue: Gujarati script (U+0A80-U+0AFF) is entirely foreign to Cantonese.
|
|
3399
|
+
# Directly observed: 'oral use only' → full Gujarati output 'આ પ્રક્રિયા ફક્ત...'
|
|
3400
|
+
# Guards: (a) Cantonese/Yue target; (b) ≥3 Gujarati chars in output.
|
|
3401
|
+
if target_lang.lower() in ('yue', 'cantonese') and len(tgt) > 5:
|
|
3402
|
+
_gujarati_count = sum(1 for c in tgt if '\u0A80' <= c <= '\u0AFF')
|
|
3403
|
+
if _gujarati_count >= 3:
|
|
3404
|
+
return {
|
|
3405
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3406
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 1, "safety": 2,
|
|
3407
|
+
"notes": (
|
|
3408
|
+
f"Cantonese output contains {_gujarati_count} Gujarati script characters "
|
|
3409
|
+
"(U+0A80-U+0AFF) — Gujarati is entirely foreign to Cantonese/Traditional "
|
|
3410
|
+
"Chinese; model fell back to Gujarati medical corpus for this term; "
|
|
3411
|
+
"(cantonese_gujarati_cyrillic_contamination_traps, check #112)"
|
|
3412
|
+
),
|
|
3413
|
+
}
|
|
3414
|
+
|
|
3415
|
+
# Check #113 — cantonese_gujarati_cyrillic_contamination_traps (Cyrillic)
|
|
3416
|
+
# en-yue: Cyrillic script (U+0400-U+04FF) is foreign to Cantonese.
|
|
3417
|
+
# Directly observed: 'morphine sulfate' → '患者 recibió 5mg морфи素硫酸' (Cyrillic морфи)
|
|
3418
|
+
# Guards: (a) Cantonese/Yue target; (b) ≥3 Cyrillic chars in output.
|
|
3419
|
+
if target_lang.lower() in ('yue', 'cantonese') and len(tgt) > 5:
|
|
3420
|
+
_cyrillic_count = sum(1 for c in tgt if '\u0400' <= c <= '\u04FF')
|
|
3421
|
+
if _cyrillic_count >= 3:
|
|
3422
|
+
return {
|
|
3423
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3424
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 1, "safety": 2,
|
|
3425
|
+
"notes": (
|
|
3426
|
+
f"Cantonese output contains {_cyrillic_count} Cyrillic script characters "
|
|
3427
|
+
"(U+0400-U+04FF) — Cyrillic is foreign to Cantonese/Traditional Chinese; "
|
|
3428
|
+
"model fell back to Russian/Slavic medical terminology for this term; "
|
|
3429
|
+
"(cantonese_gujarati_cyrillic_contamination_traps, check #113)"
|
|
3430
|
+
),
|
|
3431
|
+
}
|
|
3432
|
+
|
|
3433
|
+
# Check #114 — swedish_bengali_script_contamination_traps
|
|
3434
|
+
# en-sv: Bengali/Bangla script (U+0980-U+09FF) is foreign to Swedish.
|
|
3435
|
+
# Directly observed: 'whereas' clause → 'Derঅন্যদিকে påstås att...' (Bengali অন্যদিকে)
|
|
3436
|
+
# Guards: (a) Swedish target; (b) ≥2 Bengali chars in output.
|
|
3437
|
+
if target_lang.lower() in ('sv', 'swedish') and len(tgt) > 5:
|
|
3438
|
+
_bengali_count = sum(1 for c in tgt if '\u0980' <= c <= '\u09FF')
|
|
3439
|
+
if _bengali_count >= 2:
|
|
3440
|
+
return {
|
|
3441
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3442
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 1, "safety": 2,
|
|
3443
|
+
"notes": (
|
|
3444
|
+
f"Swedish output contains {_bengali_count} Bengali/Bangla script characters "
|
|
3445
|
+
"(U+0980-U+09FF) — Bengali is entirely foreign to Swedish; "
|
|
3446
|
+
"model inserted Bengali discourse marker (অন্যদিকে = 'on the other hand') "
|
|
3447
|
+
"or Bengali medical term into what should be Swedish output; "
|
|
3448
|
+
"(swedish_bengali_script_contamination_traps, check #114)"
|
|
3449
|
+
),
|
|
3450
|
+
}
|
|
3451
|
+
|
|
3452
|
+
# Check #115 — hindi_two_clause_negation_scope_traps
|
|
3453
|
+
# en-hi: Source has "MUST be [verb]" (positive obligation in clause 1) + "—" or ";"
|
|
3454
|
+
# separator + "never" (prohibition in clause 2). Failure: model collapses both clauses
|
|
3455
|
+
# into a single Hindi sentence with NO separator AND uses कभी+नहीं, migrating the
|
|
3456
|
+
# "never" negation to cancel the MUST obligation.
|
|
3457
|
+
# Directly observed: "MUST be diluted... — never give undiluted" →
|
|
3458
|
+
# "कभी भी पतला नहीं किया जाना चाहिए" (no separator; MUST DILUTE → should NEVER DILUTE)
|
|
3459
|
+
# Guards: (a) Hindi target; (b) source has "MUST be [verb]" + "—"/";" + "never";
|
|
3460
|
+
# (c) Hindi output has NO clause separator; (d) output has both कभी and नहीं.
|
|
3461
|
+
if target_lang.lower() in ('hi', 'hindi') and len(tgt) > 15:
|
|
3462
|
+
_must_passive_match = re.search(r'\bMUST\s+be\s+\w+', src)
|
|
3463
|
+
_never_after_sep = re.search(r'[—;].*\bnever\b', src, re.IGNORECASE)
|
|
3464
|
+
if _must_passive_match and _never_after_sep:
|
|
3465
|
+
_hi_devanagari_count_115 = sum(1 for c in tgt if '\u0900' <= c <= '\u097F')
|
|
3466
|
+
if _hi_devanagari_count_115 >= 5:
|
|
3467
|
+
_has_tgt_sep = bool(re.search(r'[।—;]', tgt))
|
|
3468
|
+
if not _has_tgt_sep and 'कभी' in tgt and 'नहीं' in tgt:
|
|
3469
|
+
return {
|
|
3470
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
3471
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1, "format": 3, "safety": 1,
|
|
3472
|
+
"notes": (
|
|
3473
|
+
f"Source has 'MUST be [verb]' positive obligation ('{_must_passive_match.group()}') "
|
|
3474
|
+
"separated from a 'never' prohibition by '—' or ';'; "
|
|
3475
|
+
"Hindi output has NO clause separator (collapsed to single sentence) "
|
|
3476
|
+
"and contains 'कभी' + 'नहीं', indicating negation migrated from "
|
|
3477
|
+
"the prohibition clause to negate the obligation clause — "
|
|
3478
|
+
"MUST DO becomes NEVER DO, a lethal semantic inversion; "
|
|
3479
|
+
"(hindi_two_clause_negation_scope_traps, check #115)"
|
|
3480
|
+
),
|
|
3481
|
+
}
|
|
3482
|
+
|
|
3483
|
+
# Check #116 — hindi_absolutive_clause_traps
|
|
3484
|
+
# en-hi: Source begins with "Having [word]" (absolutive sequential construction).
|
|
3485
|
+
# Failure: Hindi output lacks any absolutive suffix (कर/करके/लेकर/देकर/पीकर/खाकर/धोकर/लगाकर)
|
|
3486
|
+
# in the first 60% — indicates sequential dependency lost, two actions independent.
|
|
3487
|
+
# Directly observed pattern: seeds starting with "Having [VBN]" collapse to two
|
|
3488
|
+
# independent imperatives, dropping the critical temporal ordering.
|
|
3489
|
+
if target_lang.lower() in ('hi', 'hindi') and len(tgt) > 15:
|
|
3490
|
+
if re.match(r'^Having\s+\w+', src, re.IGNORECASE):
|
|
3491
|
+
_hi_deva_116 = sum(1 for c in tgt if '\u0900' <= c <= '\u097F')
|
|
3492
|
+
if _hi_deva_116 >= 5:
|
|
3493
|
+
_absolutive_forms = (
|
|
3494
|
+
'कर ', 'करके', 'लेकर', 'देकर', 'पीकर', 'खाकर',
|
|
3495
|
+
'धोकर', 'लगाकर', 'हटाकर', 'जाँचकर', 'मिलाकर',
|
|
3496
|
+
'रखकर', 'खींचकर', 'साफ कर', 'प्राप्त कर',
|
|
3497
|
+
)
|
|
3498
|
+
_tgt_first_60 = tgt[:max(1, int(len(tgt) * 0.60))]
|
|
3499
|
+
if not any(f in _tgt_first_60 for f in _absolutive_forms):
|
|
3500
|
+
return {
|
|
3501
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
3502
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1, "format": 3, "safety": 2,
|
|
3503
|
+
"notes": (
|
|
3504
|
+
"Source begins with absolutive 'Having [VBN]' construction; "
|
|
3505
|
+
"Hindi output lacks absolutive suffix (कर/करके/लेकर/देकर etc.) "
|
|
3506
|
+
"in first 60% of output — sequential dependency 'do X first, then Y' "
|
|
3507
|
+
"collapsed to two independent clauses, losing obligatory ordering; "
|
|
3508
|
+
"(hindi_absolutive_clause_traps, check #116)"
|
|
3509
|
+
),
|
|
3510
|
+
}
|
|
3511
|
+
|
|
3512
|
+
# Check #117 — cantonese_a_not_a_question_traps
|
|
3513
|
+
# en-yue: Source ends with '?'; Cantonese output contains 嗎 (U+5417, Mandarin question
|
|
3514
|
+
# particle) AND lacks all A-not-A markers (唔係/冇/唔要/係咪/係唔係/有冇/要唔要).
|
|
3515
|
+
# Detects: Mandarin 嗎-question substituting for obligatory Cantonese A-not-A polar form.
|
|
3516
|
+
if target_lang.lower() in ('yue', 'cantonese') and len(tgt) > 3:
|
|
3517
|
+
if src.rstrip().endswith('?') and '\u5417' in tgt: # 嗎
|
|
3518
|
+
_a_not_a_markers = ('唔係', '冇', '唔要', '係咪', '係唔係', '有冇', '要唔要')
|
|
3519
|
+
if not any(m in tgt for m in _a_not_a_markers):
|
|
3520
|
+
return {
|
|
3521
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3522
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 1, "format": 3, "safety": 2,
|
|
3523
|
+
"notes": (
|
|
3524
|
+
"Source is a polar yes/no question; Cantonese output contains "
|
|
3525
|
+
"Mandarin question particle 嗎 with no Cantonese A-not-A form "
|
|
3526
|
+
"(唔係/冇/唔要/係咪) — Mandarin 嗎-question replaces "
|
|
3527
|
+
"obligatory Cantonese A-not-A polar question formation; "
|
|
3528
|
+
"(cantonese_a_not_a_question_traps, check #117)"
|
|
3529
|
+
),
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3532
|
+
# Check #118 — kannada_dative_subject_traps
|
|
3533
|
+
# en-kn: Source has "the [NP] has/feels [symptom]" experiencer predicate pattern.
|
|
3534
|
+
# Failure: Kannada output lacks dative suffix (ಗೆ/ಕ್ಕೆ/ಕೆ) in first 40% of output,
|
|
3535
|
+
# indicating nominative subject substitution — ungrammatical in Kannada for
|
|
3536
|
+
# psychological/physiological experiencer predicates.
|
|
3537
|
+
if target_lang.lower() in ('kn', 'kannada') and len(tgt) > 10:
|
|
3538
|
+
_dative_subj_match = re.search(
|
|
3539
|
+
r'\bthe\s+(?:patient|child|nurse|surgeon|staff\s+member|ward\s+manager|elderly\s+patient)\s+'
|
|
3540
|
+
r'(?:has|feels|feel)\b',
|
|
3541
|
+
src, re.IGNORECASE,
|
|
3542
|
+
)
|
|
3543
|
+
if _dative_subj_match:
|
|
3544
|
+
_kn_script_118 = sum(1 for c in tgt if '\u0C80' <= c <= '\u0CFF')
|
|
3545
|
+
if _kn_script_118 >= 5:
|
|
3546
|
+
_tgt_first_40 = tgt[:max(1, int(len(tgt) * 0.40))]
|
|
3547
|
+
_dative_markers = ('ಗೆ', 'ಕ್ಕೆ', 'ಕೆ')
|
|
3548
|
+
if not any(d in _tgt_first_40 for d in _dative_markers):
|
|
3549
|
+
return {
|
|
3550
|
+
"verdict": "catastrophic", "failure_type": "semantic_substitution",
|
|
3551
|
+
"adequacy": 1, "fluency": 2, "faithfulness": 1, "format": 3, "safety": 2,
|
|
3552
|
+
"notes": (
|
|
3553
|
+
f"Source has experiencer predicate "
|
|
3554
|
+
f"'{_dative_subj_match.group()}'; "
|
|
3555
|
+
"Kannada output lacks dative case marker (ಗೆ/ಕ್ಕೆ/ಕೆ) in "
|
|
3556
|
+
"first 40% — nominative subject substitution for an "
|
|
3557
|
+
"obligatory dative experiencer construction; "
|
|
3558
|
+
"(kannada_dative_subject_traps, check #118)"
|
|
3559
|
+
),
|
|
3560
|
+
}
|
|
3561
|
+
|
|
3562
|
+
# Check #119 — swedish_number_localization_traps
|
|
3563
|
+
# en-sv: source has English thousands-comma number (e.g. "1,500") AND
|
|
3564
|
+
# the Swedish output retains the same "X,XXX" pattern unchanged.
|
|
3565
|
+
# Catastrophic because Swedish uses comma as the DECIMAL separator (not thousands):
|
|
3566
|
+
# "1,500 mg" in Swedish = 1.5 mg (1000× underdose vs intended 1500 mg).
|
|
3567
|
+
# Guard: require target language to be Swedish and ≥5 chars to avoid empty-output
|
|
3568
|
+
# double-fire with check #1. Only fires when the exact English format reappears.
|
|
3569
|
+
if target_lang.lower() in ('sv', 'swedish') and len(tgt) > 5:
|
|
3570
|
+
_sv_thousands = re.findall(r'\b(\d{1,3}),(\d{3})\b', src)
|
|
3571
|
+
for _sv_int, _sv_frac in _sv_thousands:
|
|
3572
|
+
_sv_en_fmt = f"{_sv_int},{_sv_frac}"
|
|
3573
|
+
if _sv_en_fmt in tgt:
|
|
3574
|
+
return {
|
|
3575
|
+
"verdict": "catastrophic", "failure_type": "measurement_error",
|
|
3576
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1, "format": 1, "safety": 1,
|
|
3577
|
+
"notes": (
|
|
3578
|
+
f"Source has English thousands-comma number '{_sv_en_fmt}' "
|
|
3579
|
+
f"(= {_sv_int}{_sv_frac} in standard form); "
|
|
3580
|
+
"Swedish output retains same format unchanged — "
|
|
3581
|
+
"Swedish comma is the DECIMAL separator, so a Swedish reader "
|
|
3582
|
+
f"reads '{_sv_en_fmt}' as {_sv_int}.{_sv_frac} "
|
|
3583
|
+
"(potential 1000× magnitude error in clinical context); "
|
|
3584
|
+
"(swedish_number_localization_traps, check #119)"
|
|
3585
|
+
),
|
|
3586
|
+
}
|
|
3587
|
+
|
|
3588
|
+
# Check #120 — cantonese_comparison_construction_traps
|
|
3589
|
+
# en-yue: source has a dosing comparison ('more than'/'no more than'/'less than'
|
|
3590
|
+
# before a digit); Cantonese output has ≥8 Chinese chars (real translation attempt);
|
|
3591
|
+
# output contains Mandarin 比-comparative (比+X+多/少/高/低) but lacks Cantonese
|
|
3592
|
+
# 多過 or 少過 → Mandarin comparison structure used for a Cantonese target.
|
|
3593
|
+
# Guard: require ≥8 Chinese chars so we don't fire on near-empty or Latin outputs.
|
|
3594
|
+
if target_lang.lower() in ('yue', 'cantonese') and len(tgt) > 5:
|
|
3595
|
+
_yue_cmp_src = re.search(
|
|
3596
|
+
r'\b(?:no\s+more\s+than|more\s+than|no\s+less\s+than|less\s+than)\s+\d',
|
|
3597
|
+
src, re.IGNORECASE,
|
|
3598
|
+
)
|
|
3599
|
+
if _yue_cmp_src:
|
|
3600
|
+
_yue_han = sum(1 for c in tgt if '\u4e00' <= c <= '\u9fff')
|
|
3601
|
+
if _yue_han >= 8:
|
|
3602
|
+
_yue_has_canton_cmp = any(m in tgt for m in ('多過', '少過'))
|
|
3603
|
+
_yue_has_mandarin_cmp = (
|
|
3604
|
+
'比' in tgt
|
|
3605
|
+
and re.search(r'比.{1,8}(?:多|少|高|低)', tgt) is not None
|
|
3606
|
+
)
|
|
3607
|
+
if _yue_has_mandarin_cmp and not _yue_has_canton_cmp:
|
|
3608
|
+
return {
|
|
3609
|
+
"verdict": "catastrophic", "failure_type": "wrong_language",
|
|
3610
|
+
"adequacy": 2, "fluency": 3, "faithfulness": 2, "format": 3, "safety": 2,
|
|
3611
|
+
"notes": (
|
|
3612
|
+
"Source has dosing comparison "
|
|
3613
|
+
f"('{_yue_cmp_src.group().strip()}'); "
|
|
3614
|
+
"Cantonese output uses Mandarin 比-comparison construction "
|
|
3615
|
+
"instead of Cantonese 多過/少過 form — "
|
|
3616
|
+
"indicates Mandarin-register output for Cantonese target; "
|
|
3617
|
+
"(cantonese_comparison_construction_traps, check #120)"
|
|
3618
|
+
),
|
|
3619
|
+
}
|
|
3620
|
+
|
|
3621
|
+
# Check #121 — hindi_should_recommendation_traps
|
|
3622
|
+
# en-hi: Source has "should [verb]" recommendation (not "should not"/"shouldn't"
|
|
3623
|
+
# which are prohibitions already caught by check #34).
|
|
3624
|
+
# Hindi encodes advisory "should" via चाहिए at clause end.
|
|
3625
|
+
# Failure: model uses imperative (करें/लें) or habitual (करता है/लेता है) — no चाहिए.
|
|
3626
|
+
# Guards: (a) Hindi target; (b) source has "should [verb]" but NOT "should not"/
|
|
3627
|
+
# "shouldn't" (those are prohibitions); (c) ≥5 Devanagari chars in output
|
|
3628
|
+
# (confirms Hindi attempt, not wrong-language case caught by check #9);
|
|
3629
|
+
# (d) output lacks चाहिए.
|
|
3630
|
+
if target_lang.lower() in ('hi', 'hindi'):
|
|
3631
|
+
if (re.search(r'\bshould\s+\w', src, re.IGNORECASE)
|
|
3632
|
+
and not re.search(r'\bshould\s+not\b|\bshouldn\'t\b', src, re.IGNORECASE)):
|
|
3633
|
+
_dev_count_121 = sum(1 for c in tgt if '\u0900' <= c <= '\u097F')
|
|
3634
|
+
if _dev_count_121 >= 5 and 'चाहिए' not in tgt:
|
|
3635
|
+
return {
|
|
3636
|
+
"verdict": "catastrophic",
|
|
3637
|
+
"failure_type": "semantic_substitution",
|
|
3638
|
+
"adequacy": 1, "fluency": 3, "faithfulness": 1,
|
|
3639
|
+
"format": 3, "safety": 2,
|
|
3640
|
+
"notes": (
|
|
3641
|
+
"Hindi recommendation modal erased: source has 'should [verb]' "
|
|
3642
|
+
"advisory recommendation but Hindi output contains no चाहिए — "
|
|
3643
|
+
"the obligatory recommendation modal that distinguishes advisory "
|
|
3644
|
+
"'should' from an imperative command or factual description; "
|
|
3645
|
+
"model likely used imperative (करें/लें) or habitual (करता है) "
|
|
3646
|
+
"instead of proper चाहिए construction; "
|
|
3647
|
+
"(hindi_should_recommendation_traps, check #121)"
|
|
3648
|
+
),
|
|
3649
|
+
}
|
|
3650
|
+
|
|
3651
|
+
# Check #122 — cantonese_legal_term_traps
|
|
3652
|
+
# en-yue: Source has "plaintiff" (civil claimant). Cantonese HK legal vocabulary:
|
|
3653
|
+
# plaintiff = 原告人 (原告 alone is incomplete Mandarin form; 人 is required in HK).
|
|
3654
|
+
# Failure: plaintiff omitted entirely OR defendant role assigned to plaintiff.
|
|
3655
|
+
# Directly observed in phase-6 en-yue run: "plaintiff claims damages... defendant
|
|
3656
|
+
# disputes liability" → only defendant mentioned, claiming damages — plaintiff
|
|
3657
|
+
# dropped entirely.
|
|
3658
|
+
# Guards: (a) Cantonese target; (b) 'plaintiff' in source; (c) ≥10 CJK chars in
|
|
3659
|
+
# output (genuine translation attempt, not wrong-language caught earlier);
|
|
3660
|
+
# (d) output lacks 原告 (Traditional and Simplified share these codepoints).
|
|
3661
|
+
if target_lang.lower() in ('yue', 'cantonese'):
|
|
3662
|
+
if re.search(r'\bplaintiff\b', src, re.IGNORECASE):
|
|
3663
|
+
_cjk_count_122 = sum(
|
|
3664
|
+
1 for c in tgt
|
|
3665
|
+
if '\u4E00' <= c <= '\u9FFF' or '\u3400' <= c <= '\u4DBF'
|
|
3666
|
+
)
|
|
3667
|
+
if _cjk_count_122 >= 10 and '原告' not in tgt:
|
|
3668
|
+
return {
|
|
3669
|
+
"verdict": "catastrophic",
|
|
3670
|
+
"failure_type": "hallucination",
|
|
3671
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
3672
|
+
"format": 3, "safety": 3,
|
|
3673
|
+
"notes": (
|
|
3674
|
+
"Cantonese legal plaintiff erasure: source has 'plaintiff' (civil "
|
|
3675
|
+
"claimant) but Cantonese output lacks 原告/原告人 — plaintiff "
|
|
3676
|
+
"entirely omitted or role reassigned to the defendant; "
|
|
3677
|
+
"directly observed in phase-6 en-yue run where "
|
|
3678
|
+
"'the plaintiff claims damages' → defendant-only output claiming damages; "
|
|
3679
|
+
"HK legal Cantonese: plaintiff = 原告人 (原告 alone is Mandarin shortform); "
|
|
3680
|
+
"(cantonese_legal_term_traps, check #122)"
|
|
3681
|
+
),
|
|
3682
|
+
}
|
|
3683
|
+
|
|
3684
|
+
# Check #123 — swedish_ghs_hazard_signal_traps
|
|
3685
|
+
# en-sv: Source has a GHS (Globally Harmonized System) hazard signal phrase
|
|
3686
|
+
# ("Fatal if inhaled/swallowed/in contact with skin", "DANGER:", "WARNING:",
|
|
3687
|
+
# "Causes severe burns/eye damage").
|
|
3688
|
+
# Swedish GHS requires standardized vocabulary: DANGER→Fara, WARNING→Varning,
|
|
3689
|
+
# "Fatal if inhaled"→"Dödligt vid inandning", "Causes severe burns"→"Orsakar allvarliga..."
|
|
3690
|
+
# Failure: model uses non-GHS Swedish paraphrase or leaves English terms untranslated.
|
|
3691
|
+
# Guards: (a) Swedish target; (b) source has GHS hazard phrase; (c) output ≥5
|
|
3692
|
+
# words (genuine attempt, not empty or wrong-language); (d) output lacks core
|
|
3693
|
+
# GHS Swedish vocabulary (Dödligt / Orsakar allvarliga / Fara / Varning).
|
|
3694
|
+
if target_lang.lower() in ('sv', 'swedish', 'svenska'):
|
|
3695
|
+
_sv_ghs_src = re.search(
|
|
3696
|
+
r'\b(?:fatal\s+if\s+(?:inhaled|swallowed|ingested)'
|
|
3697
|
+
r'|fatal\s+in\s+contact\s+with'
|
|
3698
|
+
r'|causes?\s+severe\s+(?:burns?|skin\s+burns?|eye\s+damage)'
|
|
3699
|
+
r'|causes?\s+serious\s+eye\s+damage'
|
|
3700
|
+
r'|DANGER\s*:'
|
|
3701
|
+
r'|WARNING\s*:)',
|
|
3702
|
+
src, re.IGNORECASE,
|
|
3703
|
+
)
|
|
3704
|
+
if _sv_ghs_src and len(tgt.split()) >= 5:
|
|
3705
|
+
_sv_ghs_ok = bool(re.search(
|
|
3706
|
+
r'[Dd]ödligt|[Ff]ara\b|[Vv]arning\b'
|
|
3707
|
+
r'|[Oo]rsakar\s+allvarlig',
|
|
3708
|
+
tgt,
|
|
3709
|
+
))
|
|
3710
|
+
if not _sv_ghs_ok:
|
|
3711
|
+
return {
|
|
3712
|
+
"verdict": "catastrophic",
|
|
3713
|
+
"failure_type": "semantic_substitution",
|
|
3714
|
+
"adequacy": 0, "fluency": 2, "faithfulness": 0,
|
|
3715
|
+
"format": 2, "safety": 0,
|
|
3716
|
+
"notes": (
|
|
3717
|
+
f"Swedish GHS hazard vocabulary absent: source has GHS signal phrase "
|
|
3718
|
+
f"('{_sv_ghs_src.group().strip()[:60]}') but Swedish output lacks "
|
|
3719
|
+
"GHS-standardized vocabulary (Dödligt/Orsakar allvarliga/Fara/Varning); "
|
|
3720
|
+
"model used non-GHS Swedish paraphrase or left English terms untranslated "
|
|
3721
|
+
"— GHS non-compliance invalidates chemical/pharmaceutical safety labelling; "
|
|
3722
|
+
"(swedish_ghs_hazard_signal_traps, check #123)"
|
|
3723
|
+
),
|
|
3724
|
+
}
|
|
3725
|
+
|
|
3726
|
+
return None # ambiguous — need Claude to judge
|
|
3727
|
+
|
|
3728
|
+
|
|
3729
|
+
def detect_profanity(translation: str, source_text: str) -> list[str]:
|
|
3730
|
+
"""Profanity words in ``translation`` that are absent from ``source_text``."""
|
|
3731
|
+
found = []
|
|
3732
|
+
for word in PROFANITY_WORDS:
|
|
3733
|
+
pattern = r"(?<!\w)" + re.escape(word.lower()) + r"(?!\w)"
|
|
3734
|
+
if re.search(pattern, translation.lower()) and not re.search(pattern, source_text.lower()):
|
|
3735
|
+
found.append(word)
|
|
3736
|
+
return sorted(found)
|