alifbo 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alifbo/__init__.py +29 -0
- alifbo/case.py +107 -0
- alifbo/cyrillic.py +374 -0
- alifbo/data/exceptions.json +8 -0
- alifbo/data/protected-terms.json +1 -0
- alifbo/detect.py +36 -0
- alifbo/latin.py +195 -0
- alifbo/normalize.py +129 -0
- alifbo/pipeline.py +275 -0
- alifbo/py.typed +0 -0
- alifbo/search.py +22 -0
- alifbo/types.py +40 -0
- alifbo-0.2.0.dist-info/METADATA +114 -0
- alifbo-0.2.0.dist-info/RECORD +16 -0
- alifbo-0.2.0.dist-info/WHEEL +4 -0
- alifbo-0.2.0.dist-info/licenses/LICENSE +21 -0
alifbo/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Uzbek transliteration between Cyrillic, old Latin, and the 2026 Latin alphabet."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .cyrillic import from_cyrillic, to_cyrillic
|
|
6
|
+
from .detect import detect_alphabet
|
|
7
|
+
from .latin import to_new_latin, to_old_latin
|
|
8
|
+
from .normalize import normalize_apostrophes, normalize_confusables
|
|
9
|
+
from .search import fold_search_key, fold_search_key_loose
|
|
10
|
+
from .types import Alphabet, AlphabetDetection, ConversionResult, Warning
|
|
11
|
+
|
|
12
|
+
__version__ = "0.2.0"
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"Alphabet",
|
|
16
|
+
"AlphabetDetection",
|
|
17
|
+
"ConversionResult",
|
|
18
|
+
"Warning",
|
|
19
|
+
"__version__",
|
|
20
|
+
"detect_alphabet",
|
|
21
|
+
"fold_search_key",
|
|
22
|
+
"fold_search_key_loose",
|
|
23
|
+
"from_cyrillic",
|
|
24
|
+
"normalize_apostrophes",
|
|
25
|
+
"normalize_confusables",
|
|
26
|
+
"to_cyrillic",
|
|
27
|
+
"to_new_latin",
|
|
28
|
+
"to_old_latin",
|
|
29
|
+
]
|
alifbo/case.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Explicit, locale-independent case tables (mirrors ``src/case.ts``).
|
|
2
|
+
|
|
3
|
+
Python's built-in string case methods are never used on user text: the tables
|
|
4
|
+
below are the only case mapping this package performs.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
_CASE_PAIRS = (
|
|
10
|
+
("a", "A"),
|
|
11
|
+
("b", "B"),
|
|
12
|
+
("c", "C"),
|
|
13
|
+
("d", "D"),
|
|
14
|
+
("e", "E"),
|
|
15
|
+
("f", "F"),
|
|
16
|
+
("g", "G"),
|
|
17
|
+
("h", "H"),
|
|
18
|
+
("i", "I"),
|
|
19
|
+
("j", "J"),
|
|
20
|
+
("k", "K"),
|
|
21
|
+
("l", "L"),
|
|
22
|
+
("m", "M"),
|
|
23
|
+
("n", "N"),
|
|
24
|
+
("o", "O"),
|
|
25
|
+
("p", "P"),
|
|
26
|
+
("q", "Q"),
|
|
27
|
+
("r", "R"),
|
|
28
|
+
("s", "S"),
|
|
29
|
+
("t", "T"),
|
|
30
|
+
("u", "U"),
|
|
31
|
+
("v", "V"),
|
|
32
|
+
("w", "W"),
|
|
33
|
+
("x", "X"),
|
|
34
|
+
("y", "Y"),
|
|
35
|
+
("z", "Z"),
|
|
36
|
+
("ö", "Ö"),
|
|
37
|
+
("ğ", "Ğ"),
|
|
38
|
+
("ş", "Ş"),
|
|
39
|
+
("ç", "Ç"),
|
|
40
|
+
("ș", "Ș"),
|
|
41
|
+
("а", "А"),
|
|
42
|
+
("б", "Б"),
|
|
43
|
+
("в", "В"),
|
|
44
|
+
("г", "Г"),
|
|
45
|
+
("д", "Д"),
|
|
46
|
+
("е", "Е"),
|
|
47
|
+
("ё", "Ё"),
|
|
48
|
+
("ж", "Ж"),
|
|
49
|
+
("з", "З"),
|
|
50
|
+
("и", "И"),
|
|
51
|
+
("й", "Й"),
|
|
52
|
+
("к", "К"),
|
|
53
|
+
("л", "Л"),
|
|
54
|
+
("м", "М"),
|
|
55
|
+
("н", "Н"),
|
|
56
|
+
("о", "О"),
|
|
57
|
+
("п", "П"),
|
|
58
|
+
("р", "Р"),
|
|
59
|
+
("с", "С"),
|
|
60
|
+
("т", "Т"),
|
|
61
|
+
("у", "У"),
|
|
62
|
+
("ф", "Ф"),
|
|
63
|
+
("х", "Х"),
|
|
64
|
+
("ц", "Ц"),
|
|
65
|
+
("ч", "Ч"),
|
|
66
|
+
("ш", "Ш"),
|
|
67
|
+
("щ", "Щ"),
|
|
68
|
+
("ъ", "Ъ"),
|
|
69
|
+
("ы", "Ы"),
|
|
70
|
+
("ь", "Ь"),
|
|
71
|
+
("э", "Э"),
|
|
72
|
+
("ю", "Ю"),
|
|
73
|
+
("я", "Я"),
|
|
74
|
+
("ў", "Ў"),
|
|
75
|
+
("қ", "Қ"),
|
|
76
|
+
("ғ", "Ғ"),
|
|
77
|
+
("ҳ", "Ҳ"),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
_LOWER_TO_UPPER = dict(_CASE_PAIRS)
|
|
81
|
+
_UPPER_TO_LOWER = {upper: lower for lower, upper in _CASE_PAIRS}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def lower_char(character: str) -> str:
|
|
85
|
+
return _UPPER_TO_LOWER.get(character, character)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def upper_char(character: str) -> str:
|
|
89
|
+
return _LOWER_TO_UPPER.get(character, character)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def lower_text(text: str) -> str:
|
|
93
|
+
return "".join(_UPPER_TO_LOWER.get(character, character) for character in text)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def upper_text(text: str) -> str:
|
|
97
|
+
return "".join(_LOWER_TO_UPPER.get(character, character) for character in text)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def is_upper(character: str) -> bool:
|
|
101
|
+
return character in _UPPER_TO_LOWER
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def first_case(source: str, lower_output: str) -> str:
|
|
105
|
+
if not is_upper(source) or len(lower_output) == 0:
|
|
106
|
+
return lower_output
|
|
107
|
+
return upper_char(lower_output[0]) + lower_output[1:]
|
alifbo/cyrillic.py
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
"""Cyrillic <-> new Latin conversion with warnings (mirrors ``src/cyrillic.ts``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import unicodedata
|
|
6
|
+
from typing import Iterable, List, Mapping, Optional, Sequence, Tuple
|
|
7
|
+
|
|
8
|
+
from .case import first_case, is_upper, lower_char, upper_text
|
|
9
|
+
from .latin import old_latin_core, to_new_latin_mapped
|
|
10
|
+
from .normalize import nfc, prepare_text_mapped, source_range, unit_length
|
|
11
|
+
from .pipeline import Options, apply_exceptions, make_options, map_segments
|
|
12
|
+
from .types import ConversionResult, Warning
|
|
13
|
+
|
|
14
|
+
_DIRECT_FROM_CYRILLIC = {
|
|
15
|
+
"а": "a",
|
|
16
|
+
"б": "b",
|
|
17
|
+
"в": "v",
|
|
18
|
+
"г": "g",
|
|
19
|
+
"д": "d",
|
|
20
|
+
"ж": "j",
|
|
21
|
+
"з": "z",
|
|
22
|
+
"и": "i",
|
|
23
|
+
"й": "y",
|
|
24
|
+
"к": "k",
|
|
25
|
+
"л": "l",
|
|
26
|
+
"м": "m",
|
|
27
|
+
"н": "n",
|
|
28
|
+
"о": "o",
|
|
29
|
+
"п": "p",
|
|
30
|
+
"р": "r",
|
|
31
|
+
"с": "s",
|
|
32
|
+
"т": "t",
|
|
33
|
+
"у": "u",
|
|
34
|
+
"ф": "f",
|
|
35
|
+
"х": "x",
|
|
36
|
+
"ч": "ç",
|
|
37
|
+
"ш": "ş",
|
|
38
|
+
"ы": "i",
|
|
39
|
+
"э": "e",
|
|
40
|
+
"ў": "ö",
|
|
41
|
+
"қ": "q",
|
|
42
|
+
"ғ": "ğ",
|
|
43
|
+
"ҳ": "h",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
_DIRECT_TO_CYRILLIC = {
|
|
47
|
+
"a": "а",
|
|
48
|
+
"b": "б",
|
|
49
|
+
"v": "в",
|
|
50
|
+
"g": "г",
|
|
51
|
+
"d": "д",
|
|
52
|
+
"j": "ж",
|
|
53
|
+
"z": "з",
|
|
54
|
+
"i": "и",
|
|
55
|
+
"y": "й",
|
|
56
|
+
"k": "к",
|
|
57
|
+
"l": "л",
|
|
58
|
+
"m": "м",
|
|
59
|
+
"n": "н",
|
|
60
|
+
"o": "о",
|
|
61
|
+
"p": "п",
|
|
62
|
+
"r": "р",
|
|
63
|
+
"s": "с",
|
|
64
|
+
"t": "т",
|
|
65
|
+
"u": "у",
|
|
66
|
+
"f": "ф",
|
|
67
|
+
"x": "х",
|
|
68
|
+
"ç": "ч",
|
|
69
|
+
"ş": "ш",
|
|
70
|
+
"ö": "ў",
|
|
71
|
+
"q": "қ",
|
|
72
|
+
"ğ": "ғ",
|
|
73
|
+
"h": "ҳ",
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
_CYRILLIC_VOWELS = frozenset(("а", "е", "ё", "и", "о", "у", "ў", "э", "ю", "я"))
|
|
77
|
+
_LATIN_VOWELS = frozenset(("a", "e", "i", "o", "u", "ö"))
|
|
78
|
+
_IOTATED = {"ё": "yo", "ю": "yu", "я": "ya"}
|
|
79
|
+
_IOTATED_REVERSE = {"yo": "ё", "yu": "ю", "ya": "я"}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _warning(
|
|
83
|
+
index: int, rule: str, message: str, alternatives: Sequence[str], length: int = 1
|
|
84
|
+
) -> Warning:
|
|
85
|
+
return Warning(index, length, rule, message, tuple(alternatives))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _to_source_offsets(warnings: List[Warning], origins: List[int], offset: int) -> List[Warning]:
|
|
89
|
+
"""Rewrite warning ranges from prepared-text offsets to offsets in the caller's text."""
|
|
90
|
+
mapped: List[Warning] = []
|
|
91
|
+
for item in warnings:
|
|
92
|
+
index, length = source_range(origins, item.index, item.length)
|
|
93
|
+
mapped.append(Warning(offset + index, length, item.rule, item.message, item.alternatives))
|
|
94
|
+
return mapped
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _letter_at(source: str, index: int) -> str:
|
|
98
|
+
"""The lowercased letter at ``index`` for positional rules, or '' when it is not a letter."""
|
|
99
|
+
character = source[index]
|
|
100
|
+
return lower_char(character) if unicodedata.category(character)[0] == "L" else ""
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _case_replacement(source: str, index: int, lower: str) -> str:
|
|
104
|
+
"""Case a multi-letter replacement: all caps inside an uppercase word, else title case."""
|
|
105
|
+
character = source[index]
|
|
106
|
+
if not is_upper(character):
|
|
107
|
+
return lower
|
|
108
|
+
in_uppercase_word = (index + 1 < len(source) and is_upper(source[index + 1])) or (
|
|
109
|
+
index > 0 and is_upper(source[index - 1])
|
|
110
|
+
)
|
|
111
|
+
return upper_text(lower) if in_uppercase_word else first_case(character, lower)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _from_cyrillic_core(text: str, offset: int, options: Options) -> Tuple[str, List[Warning]]:
|
|
115
|
+
source, origins = apply_exceptions(prepare_text_mapped(text, options.utf16_offsets), options)
|
|
116
|
+
output: List[str] = []
|
|
117
|
+
warnings: List[Warning] = []
|
|
118
|
+
previous_letter = ""
|
|
119
|
+
position = 0
|
|
120
|
+
|
|
121
|
+
for index, character in enumerate(source):
|
|
122
|
+
lower = lower_char(character)
|
|
123
|
+
replacement: Optional[str]
|
|
124
|
+
|
|
125
|
+
if lower == "е":
|
|
126
|
+
use_ye = (
|
|
127
|
+
previous_letter == ""
|
|
128
|
+
or previous_letter in _CYRILLIC_VOWELS
|
|
129
|
+
or previous_letter == "ъ"
|
|
130
|
+
or previous_letter == "ь"
|
|
131
|
+
)
|
|
132
|
+
replacement = "ye" if use_ye else "e"
|
|
133
|
+
warnings.append(
|
|
134
|
+
_warning(
|
|
135
|
+
position,
|
|
136
|
+
"cyrillic.e.positional",
|
|
137
|
+
"Cyrillic е can represent e or ye; a positional rule was applied.",
|
|
138
|
+
("e", "ye"),
|
|
139
|
+
)
|
|
140
|
+
)
|
|
141
|
+
elif lower == "ц":
|
|
142
|
+
use_s = previous_letter == "" or previous_letter == "ц"
|
|
143
|
+
replacement = "s" if use_s else "ts"
|
|
144
|
+
warnings.append(
|
|
145
|
+
_warning(
|
|
146
|
+
position,
|
|
147
|
+
"cyrillic.tse.positional",
|
|
148
|
+
"Cyrillic ц can represent s or ts; a positional rule was applied.",
|
|
149
|
+
("s", "ts"),
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
elif lower == "щ":
|
|
153
|
+
replacement = "şç"
|
|
154
|
+
warnings.append(
|
|
155
|
+
_warning(
|
|
156
|
+
position,
|
|
157
|
+
"cyrillic.shcha.ambiguous",
|
|
158
|
+
"Cyrillic щ has no single new-Latin equivalent; şç was chosen.",
|
|
159
|
+
("şç", "ş"),
|
|
160
|
+
)
|
|
161
|
+
)
|
|
162
|
+
elif lower == "ъ":
|
|
163
|
+
replacement = "ʼ"
|
|
164
|
+
warnings.append(
|
|
165
|
+
_warning(
|
|
166
|
+
position,
|
|
167
|
+
"cyrillic.hard-sign.ambiguous",
|
|
168
|
+
"The hard sign was retained as tutuq belgisi.",
|
|
169
|
+
("ʼ", ""),
|
|
170
|
+
)
|
|
171
|
+
)
|
|
172
|
+
elif lower == "ь":
|
|
173
|
+
replacement = ""
|
|
174
|
+
warnings.append(
|
|
175
|
+
_warning(
|
|
176
|
+
position,
|
|
177
|
+
"cyrillic.soft-sign.ambiguous",
|
|
178
|
+
"The soft sign was dropped.",
|
|
179
|
+
("", "ʼ"),
|
|
180
|
+
)
|
|
181
|
+
)
|
|
182
|
+
elif lower in _IOTATED:
|
|
183
|
+
name = _IOTATED[lower]
|
|
184
|
+
replacement = name
|
|
185
|
+
warnings.append(
|
|
186
|
+
_warning(
|
|
187
|
+
position,
|
|
188
|
+
f"cyrillic.{name}.compound",
|
|
189
|
+
f"Cyrillic {lower} was expanded to {name}.",
|
|
190
|
+
(name, name[1:]),
|
|
191
|
+
)
|
|
192
|
+
)
|
|
193
|
+
else:
|
|
194
|
+
replacement = _DIRECT_FROM_CYRILLIC.get(lower)
|
|
195
|
+
|
|
196
|
+
output.append(
|
|
197
|
+
character if replacement is None else _case_replacement(source, index, replacement)
|
|
198
|
+
)
|
|
199
|
+
previous_letter = _letter_at(source, index)
|
|
200
|
+
position += unit_length(character, options.utf16_offsets)
|
|
201
|
+
|
|
202
|
+
warnings = _to_source_offsets(warnings, origins, offset)
|
|
203
|
+
return nfc(old_latin_core("".join(output), options)), warnings
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _from_cyrillic(text: str, options: Options) -> ConversionResult:
|
|
207
|
+
converted, warnings = map_segments(
|
|
208
|
+
text, options, lambda segment, start: _from_cyrillic_core(segment, start, options)
|
|
209
|
+
)
|
|
210
|
+
return ConversionResult(converted, tuple(warnings))
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _to_cyrillic_core(text: str, offset: int, options: Options) -> Tuple[str, List[Warning]]:
|
|
214
|
+
source, origins = prepare_text_mapped(text, options.utf16_offsets)
|
|
215
|
+
length = len(source)
|
|
216
|
+
output: List[str] = []
|
|
217
|
+
warnings: List[Warning] = []
|
|
218
|
+
index = 0
|
|
219
|
+
position = 0
|
|
220
|
+
while index < length:
|
|
221
|
+
character = source[index]
|
|
222
|
+
lower = lower_char(character)
|
|
223
|
+
uppercase = character != lower
|
|
224
|
+
pair = lower + (lower_char(source[index + 1]) if index + 1 < length else "")
|
|
225
|
+
# Inverse of the fromCyrillic е rule: Cyrillic е is read "ye" at a word start, after
|
|
226
|
+
# a vowel, or after ъ/ь, so there "ye" maps back to е and a bare "e" must be э.
|
|
227
|
+
previous = _letter_at(source, index - 1) if index > 0 else ""
|
|
228
|
+
positional = previous == "" or previous == "ʼ" or previous in _LATIN_VOWELS
|
|
229
|
+
replacement: Optional[str]
|
|
230
|
+
consumed = 1
|
|
231
|
+
|
|
232
|
+
if pair == "şç":
|
|
233
|
+
replacement = "щ"
|
|
234
|
+
consumed = 2
|
|
235
|
+
warnings.append(
|
|
236
|
+
_warning(
|
|
237
|
+
position,
|
|
238
|
+
"latin.shcha.ambiguous",
|
|
239
|
+
"şç was interpreted as Cyrillic щ rather than шч.",
|
|
240
|
+
("щ", "шч"),
|
|
241
|
+
2,
|
|
242
|
+
)
|
|
243
|
+
)
|
|
244
|
+
elif pair in _IOTATED_REVERSE:
|
|
245
|
+
replacement = _IOTATED_REVERSE[pair]
|
|
246
|
+
consumed = 2
|
|
247
|
+
warnings.append(
|
|
248
|
+
_warning(
|
|
249
|
+
position,
|
|
250
|
+
f"latin.{pair}.ambiguous",
|
|
251
|
+
f"{pair} was interpreted as one iotated Cyrillic letter.",
|
|
252
|
+
(replacement, "й" + _DIRECT_TO_CYRILLIC[pair[1]]),
|
|
253
|
+
2,
|
|
254
|
+
)
|
|
255
|
+
)
|
|
256
|
+
elif pair == "ts":
|
|
257
|
+
replacement = "ц"
|
|
258
|
+
consumed = 2
|
|
259
|
+
warnings.append(
|
|
260
|
+
_warning(
|
|
261
|
+
position,
|
|
262
|
+
"latin.tse.ambiguous",
|
|
263
|
+
"ts was interpreted as Cyrillic ц.",
|
|
264
|
+
("ц", "тс"),
|
|
265
|
+
2,
|
|
266
|
+
)
|
|
267
|
+
)
|
|
268
|
+
elif pair == "ye" and positional:
|
|
269
|
+
replacement = "е"
|
|
270
|
+
consumed = 2
|
|
271
|
+
warnings.append(
|
|
272
|
+
_warning(
|
|
273
|
+
position,
|
|
274
|
+
"latin.ye.positional",
|
|
275
|
+
"ye at a word start, after a vowel, or after tutuq was interpreted as "
|
|
276
|
+
"Cyrillic е.",
|
|
277
|
+
("е", "йе"),
|
|
278
|
+
2,
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
elif lower == "e":
|
|
282
|
+
replacement = "э" if positional else "е"
|
|
283
|
+
warnings.append(
|
|
284
|
+
_warning(
|
|
285
|
+
position,
|
|
286
|
+
"latin.e.ambiguous",
|
|
287
|
+
f"Latin e can correspond to Cyrillic е or э; {replacement} was chosen.",
|
|
288
|
+
("э", "е") if positional else ("е", "э"),
|
|
289
|
+
)
|
|
290
|
+
)
|
|
291
|
+
elif character == "ʼ":
|
|
292
|
+
replacement = "ъ"
|
|
293
|
+
warnings.append(
|
|
294
|
+
_warning(
|
|
295
|
+
position,
|
|
296
|
+
"latin.tutuq.ambiguous",
|
|
297
|
+
"Tutuq belgisi was interpreted as a hard sign.",
|
|
298
|
+
("ъ", "ь", ""),
|
|
299
|
+
)
|
|
300
|
+
)
|
|
301
|
+
elif lower == "c":
|
|
302
|
+
replacement = "ц"
|
|
303
|
+
warnings.append(
|
|
304
|
+
_warning(
|
|
305
|
+
position,
|
|
306
|
+
"latin.c.ambiguous",
|
|
307
|
+
"Standalone c was interpreted as Cyrillic ц.",
|
|
308
|
+
("ц", "с"),
|
|
309
|
+
)
|
|
310
|
+
)
|
|
311
|
+
else:
|
|
312
|
+
replacement = _DIRECT_TO_CYRILLIC.get(lower)
|
|
313
|
+
|
|
314
|
+
if replacement is None:
|
|
315
|
+
output.append(character)
|
|
316
|
+
elif uppercase:
|
|
317
|
+
output.append(first_case(character, replacement))
|
|
318
|
+
else:
|
|
319
|
+
output.append(replacement)
|
|
320
|
+
# A two-character match only ever consumes Basic Multilingual Plane letters.
|
|
321
|
+
position += consumed if consumed == 2 else unit_length(character, options.utf16_offsets)
|
|
322
|
+
index += consumed
|
|
323
|
+
return nfc("".join(output)), _to_source_offsets(warnings, origins, offset)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _to_cyrillic(text: str, options: Options) -> ConversionResult:
|
|
327
|
+
canonical = to_new_latin_mapped(text, options)
|
|
328
|
+
converted, warnings = map_segments(
|
|
329
|
+
canonical.text,
|
|
330
|
+
options,
|
|
331
|
+
lambda segment, start: _to_cyrillic_core(segment, start, options),
|
|
332
|
+
)
|
|
333
|
+
# Warnings point into the new-Latin intermediate; report them against the caller's text.
|
|
334
|
+
return ConversionResult(converted, tuple(_to_source_offsets(warnings, canonical.origins, 0)))
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def from_cyrillic(
|
|
338
|
+
text: str,
|
|
339
|
+
*,
|
|
340
|
+
protect_spans: bool = True,
|
|
341
|
+
protected_terms: Optional[Iterable[str]] = None,
|
|
342
|
+
exceptions: Optional[Mapping[str, str]] = None,
|
|
343
|
+
ng_as_digraph: bool = True,
|
|
344
|
+
) -> ConversionResult:
|
|
345
|
+
"""Convert Uzbek Cyrillic to new Latin and report every lossy or ambiguous choice."""
|
|
346
|
+
return _from_cyrillic(
|
|
347
|
+
text,
|
|
348
|
+
make_options(
|
|
349
|
+
protect_spans=protect_spans,
|
|
350
|
+
protected_terms=protected_terms,
|
|
351
|
+
exceptions=exceptions,
|
|
352
|
+
ng_as_digraph=ng_as_digraph,
|
|
353
|
+
),
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def to_cyrillic(
|
|
358
|
+
text: str,
|
|
359
|
+
*,
|
|
360
|
+
protect_spans: bool = True,
|
|
361
|
+
protected_terms: Optional[Iterable[str]] = None,
|
|
362
|
+
exceptions: Optional[Mapping[str, str]] = None,
|
|
363
|
+
ng_as_digraph: bool = True,
|
|
364
|
+
) -> ConversionResult:
|
|
365
|
+
"""Convert new Uzbek Latin to Cyrillic and report every lossy or ambiguous choice."""
|
|
366
|
+
return _to_cyrillic(
|
|
367
|
+
text,
|
|
368
|
+
make_options(
|
|
369
|
+
protect_spans=protect_spans,
|
|
370
|
+
protected_terms=protected_terms,
|
|
371
|
+
exceptions=exceptions,
|
|
372
|
+
ng_as_digraph=ng_as_digraph,
|
|
373
|
+
),
|
|
374
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
["Shakespeare", "Chelsea", "Photoshop"]
|
alifbo/detect.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Best-effort alphabet detection (mirrors ``src/detect.ts``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from .normalize import normalize_confusables
|
|
8
|
+
from .types import AlphabetDetection
|
|
9
|
+
|
|
10
|
+
_CYRILLIC = re.compile("[А-Яа-яЁёЎўҚқҒғҲҳ]")
|
|
11
|
+
_LATIN = re.compile("[A-Za-zÖöĞğŞşÇç]")
|
|
12
|
+
_NEW_MARKERS = re.compile("[ÖöĞğŞşÇç]")
|
|
13
|
+
_OLD_MARKERS = re.compile("(?:[OoGg][ʻʼ'‘’`´′']|[Ss][Hh]|[Cc][Hh])")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def detect_alphabet(text: str) -> AlphabetDetection:
|
|
17
|
+
"""Return a best-effort alphabet classification and confidence between zero and one."""
|
|
18
|
+
source = normalize_confusables(text)
|
|
19
|
+
cyrillic = len(_CYRILLIC.findall(source))
|
|
20
|
+
latin = len(_LATIN.findall(source))
|
|
21
|
+
new_markers = len(_NEW_MARKERS.findall(source))
|
|
22
|
+
old_markers = len(_OLD_MARKERS.findall(source))
|
|
23
|
+
|
|
24
|
+
if cyrillic == 0 and latin == 0:
|
|
25
|
+
return AlphabetDetection("unknown", 0.0)
|
|
26
|
+
if cyrillic > 0 and latin > 0:
|
|
27
|
+
return AlphabetDetection("mixed", 1.0)
|
|
28
|
+
if cyrillic > 0:
|
|
29
|
+
return AlphabetDetection("cyrillic", cyrillic / max(1, cyrillic))
|
|
30
|
+
if new_markers > 0 and old_markers > 0:
|
|
31
|
+
return AlphabetDetection("mixed", 0.9)
|
|
32
|
+
if new_markers > 0:
|
|
33
|
+
return AlphabetDetection("new-latin", min(1.0, 0.7 + new_markers / latin))
|
|
34
|
+
if old_markers > 0:
|
|
35
|
+
return AlphabetDetection("old-latin", min(1.0, 0.7 + old_markers / latin))
|
|
36
|
+
return AlphabetDetection("old-latin", 0.55)
|