scriptconv 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scriptconv/__init__.py +95 -0
- scriptconv/__main__.py +109 -0
- scriptconv/notation.py +950 -0
- scriptconv/py.typed +0 -0
- scriptconv/scripts.py +818 -0
- scriptconv/translit.py +154 -0
- scriptconv/version.py +10 -0
- scriptconv-0.0.1.dist-info/METADATA +210 -0
- scriptconv-0.0.1.dist-info/RECORD +12 -0
- scriptconv-0.0.1.dist-info/WHEEL +5 -0
- scriptconv-0.0.1.dist-info/licenses/LICENSE +202 -0
- scriptconv-0.0.1.dist-info/top_level.txt +1 -0
scriptconv/scripts.py
ADDED
|
@@ -0,0 +1,818 @@
|
|
|
1
|
+
"""Writing-system identification and metadata.
|
|
2
|
+
|
|
3
|
+
Provides ISO-15924 script codes, character-range detection, per-language
|
|
4
|
+
default script, normalisation of free-form script labels, and bulk text
|
|
5
|
+
analysis (dominant-script detection, script distribution, base direction).
|
|
6
|
+
|
|
7
|
+
The ``Latn`` registry entry covers the IPA Extensions block (U+0250–U+02AF)
|
|
8
|
+
and Latin Extended Additional (U+1E00–U+1EFF) in addition to the core Latin
|
|
9
|
+
and Latin Extended blocks. This means IPA characters like ``ɑ``, ``ʃ``,
|
|
10
|
+
``ŋ`` are classified as Latin-script by :func:`char_script` and
|
|
11
|
+
:func:`detect_script`, which is correct per Unicode but may surprise callers
|
|
12
|
+
who expect phonetic symbols to be unrecognised.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from bisect import bisect_right
|
|
17
|
+
from dataclasses import dataclass, replace
|
|
18
|
+
from typing import Optional
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"Script",
|
|
22
|
+
"SCRIPT_REGISTRY",
|
|
23
|
+
"char_script",
|
|
24
|
+
"detect_script",
|
|
25
|
+
"script_distribution",
|
|
26
|
+
"script_runs",
|
|
27
|
+
"base_direction",
|
|
28
|
+
"lang_to_script",
|
|
29
|
+
"script_to_langs",
|
|
30
|
+
"normalize_script_tag",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# ---------------------------------------------------------------------------
|
|
35
|
+
# Script dataclass
|
|
36
|
+
# ---------------------------------------------------------------------------
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class Script:
|
|
40
|
+
"""Metadata for a single writing system.
|
|
41
|
+
|
|
42
|
+
Attributes
|
|
43
|
+
----------
|
|
44
|
+
iso15924:
|
|
45
|
+
Four-letter ISO-15924 script code (e.g. ``"Latn"``).
|
|
46
|
+
name:
|
|
47
|
+
Human-readable English name (e.g. ``"Latin"``).
|
|
48
|
+
direction:
|
|
49
|
+
Primary text direction: ``"ltr"`` or ``"rtl"``.
|
|
50
|
+
char_ranges:
|
|
51
|
+
Tuple of ``(start, end)`` inclusive Unicode codepoint ranges that
|
|
52
|
+
predominantly belong to this script (e.g. ``((0x0041, 0x005A),)``).
|
|
53
|
+
script_type:
|
|
54
|
+
Typological class of the writing system — one of ``"alphabet"``,
|
|
55
|
+
``"abjad"``, ``"abugida"``, ``"syllabary"``, ``"logographic"``,
|
|
56
|
+
``"featural"``, or ``"other"``. Describes how the script encodes
|
|
57
|
+
sounds structurally; it is not a pronunciation claim.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
iso15924: str
|
|
61
|
+
name: str
|
|
62
|
+
direction: str # "ltr" | "rtl"
|
|
63
|
+
char_ranges: tuple[tuple[int, int], ...]
|
|
64
|
+
script_type: str = "other"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
# Registry — the supported writing systems
|
|
69
|
+
# ---------------------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
SCRIPT_REGISTRY: dict[str, Script] = {
|
|
72
|
+
"Latn": Script(
|
|
73
|
+
iso15924="Latn",
|
|
74
|
+
name="Latin",
|
|
75
|
+
direction="ltr",
|
|
76
|
+
char_ranges=(
|
|
77
|
+
(0x0041, 0x005A), # A-Z
|
|
78
|
+
(0x0061, 0x007A), # a-z
|
|
79
|
+
(0x00C0, 0x024F), # Latin Extended
|
|
80
|
+
(0x0250, 0x02AF), # IPA Extensions
|
|
81
|
+
(0x1E00, 0x1EFF), # Latin Extended Additional
|
|
82
|
+
(0x2C60, 0x2C7F), # Latin Extended-C
|
|
83
|
+
(0xA720, 0xA7FF), # Latin Extended-D
|
|
84
|
+
(0xAB30, 0xAB6F), # Latin Extended-E
|
|
85
|
+
(0x10780, 0x107BF), # Latin Extended-F
|
|
86
|
+
(0x1DF00, 0x1DFFF), # Latin Extended-G
|
|
87
|
+
),
|
|
88
|
+
),
|
|
89
|
+
"Cyrl": Script(
|
|
90
|
+
iso15924="Cyrl",
|
|
91
|
+
name="Cyrillic",
|
|
92
|
+
direction="ltr",
|
|
93
|
+
char_ranges=(
|
|
94
|
+
(0x0400, 0x04FF),
|
|
95
|
+
(0x0500, 0x052F),
|
|
96
|
+
(0x1C80, 0x1C8F), # Cyrillic Extended-C
|
|
97
|
+
(0x2DE0, 0x2DFF), # Cyrillic Extended-A
|
|
98
|
+
(0xA640, 0xA69F), # Cyrillic Extended-B
|
|
99
|
+
(0x1E030, 0x1E08F), # Cyrillic Extended-D
|
|
100
|
+
),
|
|
101
|
+
),
|
|
102
|
+
"Arab": Script(
|
|
103
|
+
iso15924="Arab",
|
|
104
|
+
name="Arabic",
|
|
105
|
+
direction="rtl",
|
|
106
|
+
char_ranges=(
|
|
107
|
+
(0x0600, 0x06FF),
|
|
108
|
+
(0x0750, 0x077F),
|
|
109
|
+
(0x0870, 0x089F), # Arabic Extended-B
|
|
110
|
+
(0x08A0, 0x08FF),
|
|
111
|
+
(0xFB50, 0xFDFF),
|
|
112
|
+
(0xFE70, 0xFEFC),
|
|
113
|
+
),
|
|
114
|
+
),
|
|
115
|
+
"Hebr": Script(
|
|
116
|
+
iso15924="Hebr",
|
|
117
|
+
name="Hebrew",
|
|
118
|
+
direction="rtl",
|
|
119
|
+
char_ranges=(
|
|
120
|
+
(0x0590, 0x05FF),
|
|
121
|
+
(0xFB1D, 0xFB4F),
|
|
122
|
+
),
|
|
123
|
+
),
|
|
124
|
+
"Deva": Script(
|
|
125
|
+
iso15924="Deva",
|
|
126
|
+
name="Devanagari",
|
|
127
|
+
direction="ltr",
|
|
128
|
+
char_ranges=((0x0900, 0x097F), (0xA8E0, 0xA8FF)),
|
|
129
|
+
),
|
|
130
|
+
"Hang": Script(
|
|
131
|
+
iso15924="Hang",
|
|
132
|
+
name="Hangul",
|
|
133
|
+
direction="ltr",
|
|
134
|
+
char_ranges=(
|
|
135
|
+
(0xAC00, 0xD7AF), # Hangul Syllables
|
|
136
|
+
(0x1100, 0x11FF), # Hangul Jamo
|
|
137
|
+
(0x3130, 0x318F), # Hangul Compatibility Jamo
|
|
138
|
+
(0xA960, 0xA97C), # Hangul Jamo Extended-A
|
|
139
|
+
(0xD7B0, 0xD7FF), # Hangul Jamo Extended-B
|
|
140
|
+
),
|
|
141
|
+
),
|
|
142
|
+
"Hira": Script(
|
|
143
|
+
iso15924="Hira",
|
|
144
|
+
name="Hiragana",
|
|
145
|
+
direction="ltr",
|
|
146
|
+
char_ranges=((0x3040, 0x309F),),
|
|
147
|
+
),
|
|
148
|
+
"Kana": Script(
|
|
149
|
+
iso15924="Kana",
|
|
150
|
+
name="Katakana",
|
|
151
|
+
direction="ltr",
|
|
152
|
+
char_ranges=(
|
|
153
|
+
(0x30A0, 0x30FF),
|
|
154
|
+
(0x31F0, 0x31FF), # Katakana Phonetic Extensions
|
|
155
|
+
),
|
|
156
|
+
),
|
|
157
|
+
"Hani": Script(
|
|
158
|
+
iso15924="Hani",
|
|
159
|
+
name="Han",
|
|
160
|
+
direction="ltr",
|
|
161
|
+
char_ranges=(
|
|
162
|
+
(0x4E00, 0x9FFF), # CJK Unified Ideographs
|
|
163
|
+
(0x3400, 0x4DBF), # CJK Extension A
|
|
164
|
+
(0xF900, 0xFAFF), # CJK Compatibility Ideographs
|
|
165
|
+
(0x20000, 0x2A6DF), # CJK Extension B
|
|
166
|
+
(0x2A700, 0x2B73F), # CJK Extension C
|
|
167
|
+
(0x2B740, 0x2B81F), # CJK Extension D
|
|
168
|
+
(0x2B820, 0x2CEAF), # CJK Extension E
|
|
169
|
+
(0x2CEB0, 0x2EBEF), # CJK Extension F
|
|
170
|
+
(0x2F800, 0x2FA1F), # CJK Compatibility Ideographs Supplement
|
|
171
|
+
(0x30000, 0x3134F), # CJK Extension G
|
|
172
|
+
(0x31350, 0x323AF), # CJK Extension H
|
|
173
|
+
(0x323B0, 0x3347F), # CJK Extension J
|
|
174
|
+
),
|
|
175
|
+
),
|
|
176
|
+
"Grek": Script(
|
|
177
|
+
iso15924="Grek",
|
|
178
|
+
name="Greek",
|
|
179
|
+
direction="ltr",
|
|
180
|
+
char_ranges=((0x0370, 0x03FF), (0x1F00, 0x1FFF)),
|
|
181
|
+
),
|
|
182
|
+
"Armn": Script(
|
|
183
|
+
iso15924="Armn",
|
|
184
|
+
name="Armenian",
|
|
185
|
+
direction="ltr",
|
|
186
|
+
char_ranges=((0x0530, 0x058F),),
|
|
187
|
+
),
|
|
188
|
+
"Geor": Script(
|
|
189
|
+
iso15924="Geor",
|
|
190
|
+
name="Georgian",
|
|
191
|
+
direction="ltr",
|
|
192
|
+
char_ranges=((0x10A0, 0x10FF), (0x2D00, 0x2D2F), (0x1C90, 0x1CBF)),
|
|
193
|
+
),
|
|
194
|
+
"Ethi": Script(
|
|
195
|
+
iso15924="Ethi",
|
|
196
|
+
name="Ethiopic",
|
|
197
|
+
direction="ltr",
|
|
198
|
+
char_ranges=(
|
|
199
|
+
(0x1200, 0x137F),
|
|
200
|
+
(0x1380, 0x139F),
|
|
201
|
+
(0x2D80, 0x2DDF), # Ethiopic Extended
|
|
202
|
+
(0xAB00, 0xAB2F), # Ethiopic Extended-A
|
|
203
|
+
(0x1E7E0, 0x1E7FF), # Ethiopic Extended-B
|
|
204
|
+
),
|
|
205
|
+
),
|
|
206
|
+
"Khmr": Script(
|
|
207
|
+
iso15924="Khmr",
|
|
208
|
+
name="Khmer",
|
|
209
|
+
direction="ltr",
|
|
210
|
+
char_ranges=(
|
|
211
|
+
(0x1780, 0x17FF),
|
|
212
|
+
(0x19E0, 0x19FF), # Khmer Symbols
|
|
213
|
+
),
|
|
214
|
+
),
|
|
215
|
+
"Telu": Script(
|
|
216
|
+
iso15924="Telu",
|
|
217
|
+
name="Telugu",
|
|
218
|
+
direction="ltr",
|
|
219
|
+
char_ranges=((0x0C00, 0x0C7F),),
|
|
220
|
+
),
|
|
221
|
+
"Knda": Script(
|
|
222
|
+
iso15924="Knda",
|
|
223
|
+
name="Kannada",
|
|
224
|
+
direction="ltr",
|
|
225
|
+
char_ranges=((0x0C80, 0x0CFF),),
|
|
226
|
+
),
|
|
227
|
+
"Mlym": Script(
|
|
228
|
+
iso15924="Mlym",
|
|
229
|
+
name="Malayalam",
|
|
230
|
+
direction="ltr",
|
|
231
|
+
char_ranges=((0x0D00, 0x0D7F),),
|
|
232
|
+
),
|
|
233
|
+
"Orya": Script(
|
|
234
|
+
iso15924="Orya",
|
|
235
|
+
name="Odia",
|
|
236
|
+
direction="ltr",
|
|
237
|
+
char_ranges=((0x0B00, 0x0B7F),),
|
|
238
|
+
),
|
|
239
|
+
"Cans": Script(
|
|
240
|
+
iso15924="Cans",
|
|
241
|
+
name="Unified Canadian Aboriginal Syllabics",
|
|
242
|
+
direction="ltr",
|
|
243
|
+
char_ranges=((0x1400, 0x167F), (0x18B0, 0x18FF)),
|
|
244
|
+
),
|
|
245
|
+
"Tfng": Script(
|
|
246
|
+
iso15924="Tfng",
|
|
247
|
+
name="Tifinagh",
|
|
248
|
+
direction="ltr",
|
|
249
|
+
char_ranges=((0x2D30, 0x2D7F),),
|
|
250
|
+
),
|
|
251
|
+
"Thai": Script(
|
|
252
|
+
iso15924="Thai",
|
|
253
|
+
name="Thai",
|
|
254
|
+
direction="ltr",
|
|
255
|
+
char_ranges=((0x0E00, 0x0E7F),),
|
|
256
|
+
),
|
|
257
|
+
"Beng": Script(
|
|
258
|
+
iso15924="Beng",
|
|
259
|
+
name="Bengali",
|
|
260
|
+
direction="ltr",
|
|
261
|
+
char_ranges=((0x0980, 0x09FF),),
|
|
262
|
+
),
|
|
263
|
+
"Guru": Script(
|
|
264
|
+
iso15924="Guru",
|
|
265
|
+
name="Gurmukhi",
|
|
266
|
+
direction="ltr",
|
|
267
|
+
char_ranges=((0x0A00, 0x0A7F),),
|
|
268
|
+
),
|
|
269
|
+
"Gujr": Script(
|
|
270
|
+
iso15924="Gujr",
|
|
271
|
+
name="Gujarati",
|
|
272
|
+
direction="ltr",
|
|
273
|
+
char_ranges=((0x0A80, 0x0AFF),),
|
|
274
|
+
),
|
|
275
|
+
"Taml": Script(
|
|
276
|
+
iso15924="Taml",
|
|
277
|
+
name="Tamil",
|
|
278
|
+
direction="ltr",
|
|
279
|
+
char_ranges=(
|
|
280
|
+
(0x0B80, 0x0BFF),
|
|
281
|
+
(0x11FC0, 0x11FFF), # Tamil Supplement
|
|
282
|
+
),
|
|
283
|
+
),
|
|
284
|
+
"Sinh": Script(
|
|
285
|
+
iso15924="Sinh",
|
|
286
|
+
name="Sinhala",
|
|
287
|
+
direction="ltr",
|
|
288
|
+
char_ranges=(
|
|
289
|
+
(0x0D80, 0x0DFF),
|
|
290
|
+
(0x111E0, 0x111FF), # Sinhala Archaic Numbers
|
|
291
|
+
),
|
|
292
|
+
),
|
|
293
|
+
"Mymr": Script(
|
|
294
|
+
iso15924="Mymr",
|
|
295
|
+
name="Myanmar",
|
|
296
|
+
direction="ltr",
|
|
297
|
+
char_ranges=(
|
|
298
|
+
(0x1000, 0x109F),
|
|
299
|
+
(0xA9E0, 0xA9FF), # Myanmar Extended-B
|
|
300
|
+
(0xAA60, 0xAA7F), # Myanmar Extended-A
|
|
301
|
+
(0x116D0, 0x116FF), # Myanmar Extended-C
|
|
302
|
+
),
|
|
303
|
+
),
|
|
304
|
+
"Tibt": Script(
|
|
305
|
+
iso15924="Tibt",
|
|
306
|
+
name="Tibetan",
|
|
307
|
+
direction="ltr",
|
|
308
|
+
char_ranges=((0x0F00, 0x0FFF),),
|
|
309
|
+
),
|
|
310
|
+
"Laoo": Script(
|
|
311
|
+
iso15924="Laoo",
|
|
312
|
+
name="Lao",
|
|
313
|
+
direction="ltr",
|
|
314
|
+
char_ranges=((0x0E80, 0x0EFF),),
|
|
315
|
+
),
|
|
316
|
+
"Glag": Script(
|
|
317
|
+
iso15924="Glag",
|
|
318
|
+
name="Glagolitic",
|
|
319
|
+
direction="ltr",
|
|
320
|
+
char_ranges=((0x2C00, 0x2C5F),),
|
|
321
|
+
),
|
|
322
|
+
"Runr": Script(
|
|
323
|
+
iso15924="Runr",
|
|
324
|
+
name="Runic",
|
|
325
|
+
direction="ltr",
|
|
326
|
+
char_ranges=((0x16A0, 0x16FF),),
|
|
327
|
+
),
|
|
328
|
+
"Ogam": Script(
|
|
329
|
+
iso15924="Ogam",
|
|
330
|
+
name="Ogham",
|
|
331
|
+
direction="ltr",
|
|
332
|
+
char_ranges=((0x1681, 0x169C),),
|
|
333
|
+
),
|
|
334
|
+
"Cprt": Script(
|
|
335
|
+
iso15924="Cprt",
|
|
336
|
+
name="Cypriot",
|
|
337
|
+
direction="rtl",
|
|
338
|
+
char_ranges=((0x10800, 0x1083F),),
|
|
339
|
+
),
|
|
340
|
+
"Phnx": Script(
|
|
341
|
+
iso15924="Phnx",
|
|
342
|
+
name="Phoenician",
|
|
343
|
+
direction="rtl",
|
|
344
|
+
char_ranges=((0x10900, 0x1091F),),
|
|
345
|
+
),
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
# Typological class per script (Daniels & Bright, *The World's Writing
|
|
349
|
+
# Systems*, 1996). Kept as a compact map so the registry literal above stays
|
|
350
|
+
# focused on codepoint ranges; applied to each entry below.
|
|
351
|
+
_SCRIPT_TYPES: dict[str, str] = {
|
|
352
|
+
"Latn": "alphabet", "Cyrl": "alphabet", "Grek": "alphabet",
|
|
353
|
+
"Armn": "alphabet", "Geor": "alphabet", "Glag": "alphabet",
|
|
354
|
+
"Runr": "alphabet", "Ogam": "alphabet", "Tfng": "alphabet",
|
|
355
|
+
"Arab": "abjad", "Hebr": "abjad", "Phnx": "abjad",
|
|
356
|
+
"Deva": "abugida", "Beng": "abugida", "Guru": "abugida",
|
|
357
|
+
"Gujr": "abugida", "Orya": "abugida", "Taml": "abugida",
|
|
358
|
+
"Telu": "abugida", "Knda": "abugida", "Mlym": "abugida",
|
|
359
|
+
"Sinh": "abugida", "Thai": "abugida", "Laoo": "abugida",
|
|
360
|
+
"Mymr": "abugida", "Tibt": "abugida", "Khmr": "abugida",
|
|
361
|
+
"Ethi": "abugida", "Cans": "abugida",
|
|
362
|
+
"Hira": "syllabary", "Kana": "syllabary", "Cprt": "syllabary",
|
|
363
|
+
"Hani": "logographic",
|
|
364
|
+
"Hang": "featural",
|
|
365
|
+
}
|
|
366
|
+
for _code, _stype in _SCRIPT_TYPES.items():
|
|
367
|
+
if _code in SCRIPT_REGISTRY:
|
|
368
|
+
SCRIPT_REGISTRY[_code] = replace(SCRIPT_REGISTRY[_code], script_type=_stype)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
# ---------------------------------------------------------------------------
|
|
372
|
+
# Optimised char_script — flat sorted interval list with bisect
|
|
373
|
+
#
|
|
374
|
+
# Instead of iterating every script × every range on each call, we build a
|
|
375
|
+
# single sorted list of (start, script_code) and use ``bisect_right`` to
|
|
376
|
+
# find the last range that starts at or before the codepoint. This is
|
|
377
|
+
# O(log n) per character instead of O(n × ranges).
|
|
378
|
+
# ---------------------------------------------------------------------------
|
|
379
|
+
|
|
380
|
+
# Build flat interval list: [(start, code), ...] sorted by start.
|
|
381
|
+
# For each codepoint we bisect to find the last interval that starts ≤ cp,
|
|
382
|
+
# then check if cp ≤ that interval's end (stored separately).
|
|
383
|
+
_INTERVAL_STARTS: list[int] = []
|
|
384
|
+
_INTERVAL_ENDS: list[int] = []
|
|
385
|
+
_INTERVAL_SCRIPTS: list[str] = []
|
|
386
|
+
|
|
387
|
+
for _code, _script in SCRIPT_REGISTRY.items():
|
|
388
|
+
for _lo, _hi in _script.char_ranges:
|
|
389
|
+
_INTERVAL_STARTS.append(_lo)
|
|
390
|
+
_INTERVAL_ENDS.append(_hi)
|
|
391
|
+
_INTERVAL_SCRIPTS.append(_code)
|
|
392
|
+
|
|
393
|
+
# Sort all three lists by start codepoint
|
|
394
|
+
_SORTED_INDICES = sorted(range(len(_INTERVAL_STARTS)), key=lambda i: _INTERVAL_STARTS[i])
|
|
395
|
+
_SORTED_STARTS = [_INTERVAL_STARTS[i] for i in _SORTED_INDICES]
|
|
396
|
+
_SORTED_ENDS = [_INTERVAL_ENDS[i] for i in _SORTED_INDICES]
|
|
397
|
+
_SORTED_SCRIPTS = [_INTERVAL_SCRIPTS[i] for i in _SORTED_INDICES]
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def char_script(ch: str) -> Optional[str]:
|
|
401
|
+
"""Return the ISO-15924 code for a single character, or ``None``.
|
|
402
|
+
|
|
403
|
+
Uses O(log n) binary search over the precomputed interval list.
|
|
404
|
+
Characters in the IPA Extensions block (U+0250–U+02AF) and Latin
|
|
405
|
+
Extended Additional (U+1E00–U+1EFF) are classified as ``"Latn"``.
|
|
406
|
+
|
|
407
|
+
Examples
|
|
408
|
+
--------
|
|
409
|
+
>>> char_script("A")
|
|
410
|
+
'Latn'
|
|
411
|
+
>>> char_script("ɑ") # IPA Extension — classified as Latin
|
|
412
|
+
'Latn'
|
|
413
|
+
>>> char_script(" ")
|
|
414
|
+
>>> # None
|
|
415
|
+
"""
|
|
416
|
+
if len(ch) != 1:
|
|
417
|
+
return None
|
|
418
|
+
cp = ord(ch)
|
|
419
|
+
idx = bisect_right(_SORTED_STARTS, cp) - 1
|
|
420
|
+
if idx >= 0 and cp <= _SORTED_ENDS[idx]:
|
|
421
|
+
return _SORTED_SCRIPTS[idx]
|
|
422
|
+
return None
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
# ---------------------------------------------------------------------------
|
|
426
|
+
# detect_script — dominant script in a text
|
|
427
|
+
# ---------------------------------------------------------------------------
|
|
428
|
+
|
|
429
|
+
def detect_script(text: str) -> Optional[str]:
|
|
430
|
+
"""Return the ISO-15924 code of the dominant script in *text*.
|
|
431
|
+
|
|
432
|
+
Counts script-bearing characters only (skips ASCII digits, spaces,
|
|
433
|
+
punctuation, combining marks). Returns ``None`` for empty or purely
|
|
434
|
+
non-alphabetic input.
|
|
435
|
+
|
|
436
|
+
IPA characters (U+0250–U+02AF) are classified as Latin script.
|
|
437
|
+
Combining marks (U+0300–U+036F) are not assigned to any script and
|
|
438
|
+
are skipped.
|
|
439
|
+
|
|
440
|
+
Examples
|
|
441
|
+
--------
|
|
442
|
+
>>> detect_script("Hello world")
|
|
443
|
+
'Latn'
|
|
444
|
+
>>> detect_script("ɑɛɔ") # IPA — classified as Latin
|
|
445
|
+
'Latn'
|
|
446
|
+
>>> detect_script("")
|
|
447
|
+
"""
|
|
448
|
+
counts: dict[str, int] = {}
|
|
449
|
+
for ch in text:
|
|
450
|
+
s = char_script(ch)
|
|
451
|
+
if s is not None:
|
|
452
|
+
counts[s] = counts.get(s, 0) + 1
|
|
453
|
+
if not counts:
|
|
454
|
+
return None
|
|
455
|
+
return max(counts, key=lambda k: (counts[k], k))
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def script_distribution(text: str) -> dict[str, int]:
|
|
459
|
+
"""Return character counts per script in *text*.
|
|
460
|
+
|
|
461
|
+
Only characters assigned to a registered script are counted. The
|
|
462
|
+
result is sorted by count descending.
|
|
463
|
+
|
|
464
|
+
Parameters
|
|
465
|
+
----------
|
|
466
|
+
text:
|
|
467
|
+
Input string (may contain any Unicode characters).
|
|
468
|
+
|
|
469
|
+
Returns
|
|
470
|
+
-------
|
|
471
|
+
dict[str, int]
|
|
472
|
+
Mapping of ISO-15924 code to character count, sorted by
|
|
473
|
+
count descending. Empty dict for empty or purely
|
|
474
|
+
non-alphabetic input.
|
|
475
|
+
|
|
476
|
+
Examples
|
|
477
|
+
--------
|
|
478
|
+
>>> script_distribution("Hello Привет мир")
|
|
479
|
+
{'Cyrl': 9, 'Latn': 5}
|
|
480
|
+
>>> script_distribution("日本語ひらがなカタカナ")
|
|
481
|
+
{'Hira': 4, 'Kana': 4, 'Hani': 3}
|
|
482
|
+
"""
|
|
483
|
+
counts: dict[str, int] = {}
|
|
484
|
+
for ch in text:
|
|
485
|
+
s = char_script(ch)
|
|
486
|
+
if s is not None:
|
|
487
|
+
counts[s] = counts.get(s, 0) + 1
|
|
488
|
+
return dict(sorted(counts.items(), key=lambda kv: kv[1], reverse=True))
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def script_runs(text: str) -> list[tuple[Optional[str], str]]:
|
|
492
|
+
"""Segment *text* into contiguous runs of a single script.
|
|
493
|
+
|
|
494
|
+
Returns a list of ``(script, substring)`` pairs in text order, where
|
|
495
|
+
*script* is an ISO-15924 code or ``None``. Script-neutral characters
|
|
496
|
+
(spaces, punctuation, digits, combining marks — anything
|
|
497
|
+
:func:`char_script` maps to ``None``) attach to the preceding run
|
|
498
|
+
rather than starting a new one, following the resolution model of
|
|
499
|
+
Unicode UAX #24; a run of leading neutrals carries ``None``.
|
|
500
|
+
|
|
501
|
+
Unlike :func:`detect_script` (which flattens to one dominant script),
|
|
502
|
+
this preserves the structure of mixed-script text — e.g. a Cyrillic
|
|
503
|
+
word embedded in a Latin sentence.
|
|
504
|
+
|
|
505
|
+
Examples
|
|
506
|
+
--------
|
|
507
|
+
>>> script_runs("привет hello")
|
|
508
|
+
[('Cyrl', 'привет '), ('Latn', 'hello')]
|
|
509
|
+
>>> script_runs("")
|
|
510
|
+
[]
|
|
511
|
+
"""
|
|
512
|
+
runs: list[list] = []
|
|
513
|
+
for ch in text:
|
|
514
|
+
s = char_script(ch)
|
|
515
|
+
if runs and (s is None or s == runs[-1][0]):
|
|
516
|
+
runs[-1][1] += ch
|
|
517
|
+
else:
|
|
518
|
+
runs.append([s, ch])
|
|
519
|
+
return [(s, t) for s, t in runs]
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def base_direction(text: str) -> str:
|
|
523
|
+
"""Detect the base text direction of *text*.
|
|
524
|
+
|
|
525
|
+
Returns ``"ltr"`` if the majority of script-bearing characters belong
|
|
526
|
+
to left-to-right scripts, ``"rtl"`` for right-to-left, or ``"mixed"``
|
|
527
|
+
when counts are equal or the input has no script-bearing characters.
|
|
528
|
+
|
|
529
|
+
Parameters
|
|
530
|
+
----------
|
|
531
|
+
text:
|
|
532
|
+
Input string.
|
|
533
|
+
|
|
534
|
+
Returns
|
|
535
|
+
-------
|
|
536
|
+
str
|
|
537
|
+
``"ltr"``, ``"rtl"``, or ``"mixed"``.
|
|
538
|
+
|
|
539
|
+
Examples
|
|
540
|
+
--------
|
|
541
|
+
>>> base_direction("Hello world")
|
|
542
|
+
'ltr'
|
|
543
|
+
>>> base_direction("مرحبا بالعالم")
|
|
544
|
+
'rtl'
|
|
545
|
+
>>> base_direction("Hello مرحبا")
|
|
546
|
+
'mixed'
|
|
547
|
+
"""
|
|
548
|
+
ltr = 0
|
|
549
|
+
rtl = 0
|
|
550
|
+
for ch in text:
|
|
551
|
+
s = char_script(ch)
|
|
552
|
+
if s is None:
|
|
553
|
+
continue
|
|
554
|
+
script = SCRIPT_REGISTRY.get(s)
|
|
555
|
+
if script is None:
|
|
556
|
+
continue
|
|
557
|
+
if script.direction == "rtl":
|
|
558
|
+
rtl += 1
|
|
559
|
+
else:
|
|
560
|
+
ltr += 1
|
|
561
|
+
if ltr > rtl:
|
|
562
|
+
return "ltr"
|
|
563
|
+
if rtl > ltr:
|
|
564
|
+
return "rtl"
|
|
565
|
+
return "mixed"
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
# ---------------------------------------------------------------------------
|
|
569
|
+
# lang_to_script — default script per language primary subtag
|
|
570
|
+
#
|
|
571
|
+
# Default script per language, from ISO 15924 / BCP-47 conventions and
|
|
572
|
+
# the language coverage of Meta's MMS (Massively Multilingual Speech).
|
|
573
|
+
# ---------------------------------------------------------------------------
|
|
574
|
+
|
|
575
|
+
_LANG_TO_SCRIPT: dict[str, str] = {
|
|
576
|
+
# Latin
|
|
577
|
+
"en": "Latn", "de": "Latn", "fr": "Latn", "es": "Latn", "pt": "Latn",
|
|
578
|
+
"it": "Latn", "nl": "Latn", "sv": "Latn", "no": "Latn", "da": "Latn",
|
|
579
|
+
"fi": "Latn", "pl": "Latn", "cs": "Latn", "sk": "Latn", "hr": "Latn",
|
|
580
|
+
"sl": "Latn", "ro": "Latn", "hu": "Latn", "lt": "Latn", "lv": "Latn",
|
|
581
|
+
"et": "Latn", "mt": "Latn", "sq": "Latn", "eu": "Latn", "ca": "Latn",
|
|
582
|
+
"gl": "Latn", "cy": "Latn", "ga": "Latn", "af": "Latn", "sw": "Latn",
|
|
583
|
+
"id": "Latn", "ms": "Latn", "tl": "Latn", "tr": "Latn", "az": "Latn",
|
|
584
|
+
"uz": "Latn", "la": "Latn", "is": "Latn", "fo": "Latn", "lb": "Latn",
|
|
585
|
+
"oc": "Latn", "br": "Latn", "ast": "Latn", "mwl": "Latn",
|
|
586
|
+
"crk": "Cans", # Cree — default syllabics (MMS label "syllabics")
|
|
587
|
+
# Cyrillic
|
|
588
|
+
"ru": "Cyrl", "uk": "Cyrl", "be": "Cyrl", "bg": "Cyrl", "sr": "Cyrl",
|
|
589
|
+
"mk": "Cyrl", "kk": "Cyrl", "ky": "Cyrl", "tg": "Cyrl", "mn": "Cyrl",
|
|
590
|
+
"tt": "Cyrl", "ba": "Cyrl", "cv": "Cyrl", "os": "Cyrl", "ce": "Cyrl",
|
|
591
|
+
"av": "Cyrl", "udm": "Cyrl", "sah": "Cyrl", "bxr": "Cyrl",
|
|
592
|
+
# Arabic
|
|
593
|
+
"ar": "Arab", "fa": "Arab", "ur": "Arab", "ps": "Arab", "sd": "Arab",
|
|
594
|
+
"ug": "Arab", "ku": "Arab", "ckb": "Arab",
|
|
595
|
+
# Hebrew
|
|
596
|
+
"he": "Hebr", "yi": "Hebr",
|
|
597
|
+
# Devanagari
|
|
598
|
+
"hi": "Deva", "mr": "Deva", "ne": "Deva", "sa": "Deva", "kok": "Deva",
|
|
599
|
+
# Hangul
|
|
600
|
+
"ko": "Hang",
|
|
601
|
+
# Japanese (mixed but hiragana is the base syllabary)
|
|
602
|
+
"ja": "Hira",
|
|
603
|
+
# Han
|
|
604
|
+
"zh": "Hani", "yue": "Hani",
|
|
605
|
+
# Greek
|
|
606
|
+
"el": "Grek",
|
|
607
|
+
# Armenian
|
|
608
|
+
"hy": "Armn",
|
|
609
|
+
# Georgian
|
|
610
|
+
"ka": "Geor",
|
|
611
|
+
# Ethiopic
|
|
612
|
+
"am": "Ethi", "ti": "Ethi",
|
|
613
|
+
# Khmer
|
|
614
|
+
"km": "Khmr",
|
|
615
|
+
# Telugu
|
|
616
|
+
"te": "Telu",
|
|
617
|
+
# Tifinagh
|
|
618
|
+
"tzm": "Tfng", "shi": "Tfng",
|
|
619
|
+
# Thai
|
|
620
|
+
"th": "Thai",
|
|
621
|
+
# Bengali
|
|
622
|
+
"bn": "Beng", "as": "Beng",
|
|
623
|
+
# Gurmukhi
|
|
624
|
+
"pa": "Guru",
|
|
625
|
+
# Gujarati
|
|
626
|
+
"gu": "Gujr",
|
|
627
|
+
# Tamil
|
|
628
|
+
"ta": "Taml",
|
|
629
|
+
# Sinhala
|
|
630
|
+
"si": "Sinh",
|
|
631
|
+
# Kannada
|
|
632
|
+
"kn": "Knda",
|
|
633
|
+
# Malayalam
|
|
634
|
+
"ml": "Mlym",
|
|
635
|
+
# Odia (Oriya)
|
|
636
|
+
"or": "Orya",
|
|
637
|
+
# Myanmar
|
|
638
|
+
"my": "Mymr",
|
|
639
|
+
# Tibetan
|
|
640
|
+
"bo": "Tibt",
|
|
641
|
+
# Lao
|
|
642
|
+
"lo": "Laoo",
|
|
643
|
+
# Additional Latin-script languages
|
|
644
|
+
"vi": "Latn", # Vietnamese
|
|
645
|
+
"ha": "Latn", # Hausa
|
|
646
|
+
"ig": "Latn", # Igbo
|
|
647
|
+
"yo": "Latn", # Yoruba
|
|
648
|
+
"zu": "Latn", # Zulu
|
|
649
|
+
"xh": "Latn", # Xhosa
|
|
650
|
+
"so": "Latn", # Somali
|
|
651
|
+
"om": "Latn", # Oromo (Latin orthography)
|
|
652
|
+
"rw": "Latn", # Kinyarwanda
|
|
653
|
+
"tk": "Latn", # Turkmen (Latin since 1993)
|
|
654
|
+
"eo": "Latn", # Esperanto
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def lang_to_script(lang: str) -> Optional[str]:
|
|
659
|
+
"""Return the ISO-15924 code of the default script for *lang*.
|
|
660
|
+
|
|
661
|
+
*lang* may be a BCP-47 tag or bare ISO 639 code. An explicit ISO-15924
|
|
662
|
+
script subtag is authoritative and is honoured when present
|
|
663
|
+
(``"sr-Latn"`` → ``"Latn"``, ``"uz-Cyrl"`` → ``"Cyrl"``); otherwise the
|
|
664
|
+
default script for the primary subtag is returned (``"pt-BR"`` → ``"pt"``).
|
|
665
|
+
Returns ``None`` when the language is unknown.
|
|
666
|
+
|
|
667
|
+
Examples
|
|
668
|
+
--------
|
|
669
|
+
>>> lang_to_script("pt-BR")
|
|
670
|
+
'Latn'
|
|
671
|
+
>>> lang_to_script("ru")
|
|
672
|
+
'Cyrl'
|
|
673
|
+
>>> lang_to_script("sr-Latn")
|
|
674
|
+
'Latn'
|
|
675
|
+
"""
|
|
676
|
+
subtags = lang.replace("_", "-").split("-")
|
|
677
|
+
# An explicit script subtag (4 letters, e.g. "Latn") overrides the
|
|
678
|
+
# language's default script.
|
|
679
|
+
for sub in subtags[1:]:
|
|
680
|
+
if len(sub) == 4 and sub.isalpha():
|
|
681
|
+
resolved = normalize_script_tag(sub)
|
|
682
|
+
if resolved is not None:
|
|
683
|
+
return resolved
|
|
684
|
+
return _LANG_TO_SCRIPT.get(subtags[0].lower())
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
# ---------------------------------------------------------------------------
|
|
688
|
+
# script_to_langs — reverse of lang_to_script
|
|
689
|
+
# ---------------------------------------------------------------------------
|
|
690
|
+
|
|
691
|
+
_SCRIPT_TO_LANGS: dict[str, list[str]] = {}
|
|
692
|
+
for _lang, _script in _LANG_TO_SCRIPT.items():
|
|
693
|
+
_SCRIPT_TO_LANGS.setdefault(_script, []).append(_lang)
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def script_to_langs(code: str) -> list[str]:
|
|
697
|
+
"""Return the languages whose default script is *code*.
|
|
698
|
+
|
|
699
|
+
Parameters
|
|
700
|
+
----------
|
|
701
|
+
code:
|
|
702
|
+
ISO-15924 script code (e.g. ``"Latn"``).
|
|
703
|
+
|
|
704
|
+
Returns
|
|
705
|
+
-------
|
|
706
|
+
list[str]
|
|
707
|
+
Sorted list of ISO 639 language codes. Empty list for
|
|
708
|
+
unknown scripts.
|
|
709
|
+
|
|
710
|
+
Examples
|
|
711
|
+
--------
|
|
712
|
+
>>> script_to_langs("Cyrl")
|
|
713
|
+
['av', 'ba', 'be', 'bg', 'bxr', 'ce', 'cv', 'kk', 'ky', 'mk', ...]
|
|
714
|
+
>>> script_to_langs("Latn") # doctest: +SKIP
|
|
715
|
+
['af', 'ast', 'az', ...]
|
|
716
|
+
"""
|
|
717
|
+
return sorted(_SCRIPT_TO_LANGS.get(code, []))
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
# ---------------------------------------------------------------------------
|
|
721
|
+
# normalize_script_tag — free-form labels → ISO-15924
|
|
722
|
+
#
|
|
723
|
+
# Free-form and MMS-style labels mapped to ISO 15924.
|
|
724
|
+
# ---------------------------------------------------------------------------
|
|
725
|
+
|
|
726
|
+
_LABEL_MAP: dict[str, str] = {
|
|
727
|
+
# MMS-style lowercase labels
|
|
728
|
+
"latin": "Latn",
|
|
729
|
+
"cyrillic": "Cyrl",
|
|
730
|
+
"arabic": "Arab",
|
|
731
|
+
"syllabics": "Cans",
|
|
732
|
+
"tifinagh": "Tfng",
|
|
733
|
+
"devanagari": "Deva",
|
|
734
|
+
"ethiopic": "Ethi",
|
|
735
|
+
"khmer": "Khmr",
|
|
736
|
+
"telugu": "Telu",
|
|
737
|
+
"kannada": "Knda",
|
|
738
|
+
"malayalam": "Mlym",
|
|
739
|
+
"odia": "Orya",
|
|
740
|
+
"oriya": "Orya",
|
|
741
|
+
"persian": "Arab",
|
|
742
|
+
"farsi": "Arab",
|
|
743
|
+
# Common aliases
|
|
744
|
+
"hebrew": "Hebr",
|
|
745
|
+
"hangul": "Hang",
|
|
746
|
+
"korean": "Hang",
|
|
747
|
+
"hiragana": "Hira",
|
|
748
|
+
"katakana": "Kana",
|
|
749
|
+
"han": "Hani",
|
|
750
|
+
"hanzi": "Hani",
|
|
751
|
+
"chinese": "Hani",
|
|
752
|
+
"kanji": "Hani",
|
|
753
|
+
"greek": "Grek",
|
|
754
|
+
"armenian": "Armn",
|
|
755
|
+
"georgian": "Geor",
|
|
756
|
+
"thai": "Thai",
|
|
757
|
+
"bengali": "Beng",
|
|
758
|
+
"gurmukhi": "Guru",
|
|
759
|
+
"gujarati": "Gujr",
|
|
760
|
+
"tamil": "Taml",
|
|
761
|
+
"sinhala": "Sinh",
|
|
762
|
+
"myanmar": "Mymr",
|
|
763
|
+
"burmese": "Mymr",
|
|
764
|
+
"tibetan": "Tibt",
|
|
765
|
+
"lao": "Laoo",
|
|
766
|
+
"glagolitic": "Glag",
|
|
767
|
+
"runic": "Runr",
|
|
768
|
+
"ogham": "Ogam",
|
|
769
|
+
"phoenician": "Phnx",
|
|
770
|
+
"cans": "Cans",
|
|
771
|
+
"unified canadian aboriginal syllabics": "Cans",
|
|
772
|
+
"canadian syllabics": "Cans",
|
|
773
|
+
# Additional common labels
|
|
774
|
+
"japanese": "Hira",
|
|
775
|
+
"jamo": "Hang",
|
|
776
|
+
"hangul jamo": "Hang",
|
|
777
|
+
"devanagari extended": "Deva",
|
|
778
|
+
"ipa": "Latn",
|
|
779
|
+
"cjk": "Hani",
|
|
780
|
+
"chinese characters": "Hani",
|
|
781
|
+
"han characters": "Hani",
|
|
782
|
+
"hangeul": "Hang",
|
|
783
|
+
"bangla": "Beng",
|
|
784
|
+
"punjabi": "Guru",
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
# Also accept the ISO-15924 codes themselves (case-insensitive)
|
|
788
|
+
_ISO_LOWER: dict[str, str] = {k.lower(): k for k in SCRIPT_REGISTRY}
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def normalize_script_tag(label: str) -> Optional[str]:
|
|
792
|
+
"""Map a free-form script label to its ISO-15924 code.
|
|
793
|
+
|
|
794
|
+
Accepts MMS-style labels (``"latin"``, ``"syllabics"``), common English
|
|
795
|
+
names (``"Arabic"``, ``"Hangul"``), and existing ISO-15924 codes in any
|
|
796
|
+
case (``"latn"`` → ``"Latn"``). Returns ``None`` when unrecognised.
|
|
797
|
+
|
|
798
|
+
Examples
|
|
799
|
+
--------
|
|
800
|
+
>>> normalize_script_tag("latin")
|
|
801
|
+
'Latn'
|
|
802
|
+
>>> normalize_script_tag("Cyrillic")
|
|
803
|
+
'Cyrl'
|
|
804
|
+
>>> normalize_script_tag("syllabics")
|
|
805
|
+
'Cans'
|
|
806
|
+
>>> normalize_script_tag("japanese")
|
|
807
|
+
'Hira'
|
|
808
|
+
>>> normalize_script_tag("jamo")
|
|
809
|
+
'Hang'
|
|
810
|
+
"""
|
|
811
|
+
normalised = label.strip().lower()
|
|
812
|
+
# Direct label lookup
|
|
813
|
+
if normalised in _LABEL_MAP:
|
|
814
|
+
return _LABEL_MAP[normalised]
|
|
815
|
+
# ISO-15924 code (case-insensitive)
|
|
816
|
+
if normalised in _ISO_LOWER:
|
|
817
|
+
return _ISO_LOWER[normalised]
|
|
818
|
+
return None
|