ocr-foundry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khocr_gen/__init__.py +11 -0
- khocr_gen/_pil_compat.py +25 -0
- khocr_gen/_rust_accel.py +204 -0
- khocr_gen/augmentation.py +1193 -0
- khocr_gen/benchmark.py +888 -0
- khocr_gen/benchmark_cmd.py +103 -0
- khocr_gen/cli.py +246 -0
- khocr_gen/combine.py +214 -0
- khocr_gen/combine_cmd.py +125 -0
- khocr_gen/config.py +1474 -0
- khocr_gen/config_loader.py +157 -0
- khocr_gen/corpus.py +621 -0
- khocr_gen/data_generator.py +1160 -0
- khocr_gen/errors.py +33 -0
- khocr_gen/evaluate_cmd.py +50 -0
- khocr_gen/evaluation.py +259 -0
- khocr_gen/fonts.py +316 -0
- khocr_gen/generate.py +227 -0
- khocr_gen/layout.py +1028 -0
- khocr_gen/layout_cmd.py +371 -0
- khocr_gen/layout_degradation.py +128 -0
- khocr_gen/layout_mesh.py +200 -0
- khocr_gen/layout_shaping.py +264 -0
- khocr_gen/layout_transform.py +238 -0
- khocr_gen/layout_warp.py +473 -0
- khocr_gen/line_height.py +127 -0
- khocr_gen/lmdb_pack.py +132 -0
- khocr_gen/logging.py +25 -0
- khocr_gen/normalizer.py +221 -0
- khocr_gen/parallel.py +366 -0
- khocr_gen/provenance.py +328 -0
- khocr_gen/rendering.py +1607 -0
- khocr_gen/text_support.py +429 -0
- khocr_gen/verify.py +471 -0
- khocr_gen/viewer.py +172 -0
- ocr_foundry-0.2.0.dist-info/METADATA +174 -0
- ocr_foundry-0.2.0.dist-info/RECORD +40 -0
- ocr_foundry-0.2.0.dist-info/WHEEL +4 -0
- ocr_foundry-0.2.0.dist-info/entry_points.txt +3 -0
- ocr_foundry-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
"""Shared, fail-closed Unicode coverage policy, independent of Rust dispatch.
|
|
2
|
+
|
|
3
|
+
Coverage establishes glyph availability, not linguistic correctness. Collections
|
|
4
|
+
use face zero, matching Pillow's explicit index=0 in FontManager.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import unicodedata
|
|
10
|
+
from functools import lru_cache
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import regex
|
|
14
|
+
from fontTools.ttLib import TTFont
|
|
15
|
+
|
|
16
|
+
# Shaping/bidi controls remain in logical text sent to RAQM and in OCR labels.
|
|
17
|
+
# They need no standalone cmap glyph. Other format/control characters fail closed.
|
|
18
|
+
_SHAPING_CONTROLS = frozenset(
|
|
19
|
+
[
|
|
20
|
+
0x00AD,
|
|
21
|
+
0x061C,
|
|
22
|
+
0x200B,
|
|
23
|
+
0x200C,
|
|
24
|
+
0x200D,
|
|
25
|
+
0x200E,
|
|
26
|
+
0x200F,
|
|
27
|
+
0x2060,
|
|
28
|
+
0xFEFF,
|
|
29
|
+
*range(0x202A, 0x202F),
|
|
30
|
+
*range(0x2066, 0x206A),
|
|
31
|
+
]
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def needs_glyph(char: str) -> bool:
|
|
36
|
+
cp = ord(char)
|
|
37
|
+
# Zl/Zp are Unicode whitespace, but they are line boundaries, not ignorable spaces.
|
|
38
|
+
if unicodedata.category(char) in {"Zl", "Zp"}:
|
|
39
|
+
return True
|
|
40
|
+
return not (
|
|
41
|
+
char.isspace()
|
|
42
|
+
or cp in _SHAPING_CONTROLS
|
|
43
|
+
or 0xFE00 <= cp <= 0xFE0F
|
|
44
|
+
or 0xE0100 <= cp <= 0xE01EF
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def line_separator_error(text: str) -> str | None:
|
|
49
|
+
"""Reject Unicode line/paragraph separators (categories Zl and Zp).
|
|
50
|
+
|
|
51
|
+
``str.strip()`` treats them as whitespace. A trailing separator would
|
|
52
|
+
otherwise disappear before glyph coverage, and an internal one would be
|
|
53
|
+
copied into labels that ``str.splitlines()`` later splits.
|
|
54
|
+
"""
|
|
55
|
+
for char in text:
|
|
56
|
+
if unicodedata.category(char) in {"Zl", "Zp"}:
|
|
57
|
+
return f"unsupported line separator U+{ord(char):04X}"
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def invalid_control(text: str) -> str | None:
|
|
62
|
+
separator = line_separator_error(text)
|
|
63
|
+
if separator:
|
|
64
|
+
return separator
|
|
65
|
+
for char in text:
|
|
66
|
+
if unicodedata.category(char) in {"Cc", "Cf", "Cs"} and ord(char) not in _SHAPING_CONTROLS:
|
|
67
|
+
return f"unsupported control U+{ord(char):04X}"
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@lru_cache(maxsize=256)
|
|
72
|
+
def font_codepoints(path: str, face_index: int = 0) -> frozenset[int]:
|
|
73
|
+
with TTFont(path, fontNumber=face_index, lazy=True) as font:
|
|
74
|
+
cmap = font.getBestCmap()
|
|
75
|
+
if cmap is None:
|
|
76
|
+
raise ValueError(f"Font has no Unicode cmap: {path} (face {face_index})")
|
|
77
|
+
return frozenset(cp for cp, glyph in cmap.items() if glyph != ".notdef")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def coverage_error(path: str, text: str, face_index: int = 0) -> str | None:
|
|
81
|
+
reason = invalid_control(text)
|
|
82
|
+
if reason:
|
|
83
|
+
return reason
|
|
84
|
+
try:
|
|
85
|
+
points = font_codepoints(path, face_index)
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
return f"cannot inspect font cmap: {exc}"
|
|
88
|
+
missing = sorted({ord(c) for c in text if needs_glyph(c) and ord(c) not in points})
|
|
89
|
+
if missing:
|
|
90
|
+
return "missing glyphs " + ", ".join(f"U+{cp:04X}" for cp in missing[:8])
|
|
91
|
+
if not any(needs_glyph(c) for c in text):
|
|
92
|
+
return "line has no visible glyphs"
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def split_grapheme_clusters(text: str) -> list[str]:
|
|
97
|
+
"""Segment *text* into grapheme clusters per UAX #29.
|
|
98
|
+
|
|
99
|
+
Ensures combining marks, virama sequences (Indic conjuncts), Khmer coeng
|
|
100
|
+
sequences, and joiners (ZWJ/ZWNJ) remain bound to their consonant/base
|
|
101
|
+
grapheme and are never severed across fallback font boundaries.
|
|
102
|
+
"""
|
|
103
|
+
if not text:
|
|
104
|
+
return []
|
|
105
|
+
raw_clusters = regex.findall(r"\X", text)
|
|
106
|
+
|
|
107
|
+
merged: list[str] = []
|
|
108
|
+
for c in raw_clusters:
|
|
109
|
+
if not merged:
|
|
110
|
+
merged.append(c)
|
|
111
|
+
continue
|
|
112
|
+
prev = merged[-1]
|
|
113
|
+
prev_tail = prev.rstrip("\u200c\u200d")
|
|
114
|
+
ends_with_virama = bool(prev_tail and unicodedata.combining(prev_tail[-1]) == 9)
|
|
115
|
+
ends_with_zwj = prev.endswith("\u200d")
|
|
116
|
+
if (ends_with_virama and unicodedata.category(c[0]).startswith("L")) or (
|
|
117
|
+
ends_with_zwj and not c[0].isspace()
|
|
118
|
+
):
|
|
119
|
+
merged[-1] += c
|
|
120
|
+
else:
|
|
121
|
+
merged.append(c)
|
|
122
|
+
return merged
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _joining_spans(clusters: list[str]) -> list[str]:
|
|
126
|
+
"""Keep connected cursive letters in one shaping call and one font."""
|
|
127
|
+
spans: list[str] = []
|
|
128
|
+
for cluster in clusters:
|
|
129
|
+
if spans:
|
|
130
|
+
left = regex.sub(r"\p{Joining_Type=Transparent}+$", "", spans[-1])
|
|
131
|
+
right = regex.sub(r"^\p{Joining_Type=Transparent}+", "", cluster)
|
|
132
|
+
joins_next = left and regex.fullmatch(
|
|
133
|
+
r"[\p{Joining_Type=Dual_Joining}\p{Joining_Type=Left_Joining}\p{Joining_Type=Join_Causing}]",
|
|
134
|
+
left[-1],
|
|
135
|
+
)
|
|
136
|
+
joins_previous = right and regex.fullmatch(
|
|
137
|
+
r"[\p{Joining_Type=Dual_Joining}\p{Joining_Type=Right_Joining}\p{Joining_Type=Join_Causing}]",
|
|
138
|
+
right[0],
|
|
139
|
+
)
|
|
140
|
+
if joins_next and joins_previous:
|
|
141
|
+
spans[-1] += cluster
|
|
142
|
+
continue
|
|
143
|
+
spans.append(cluster)
|
|
144
|
+
return spans
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _cluster_needs_glyph(cluster: str) -> bool:
|
|
148
|
+
return any(needs_glyph(c) for c in cluster)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _cluster_supported_by_font(font: Any, cluster: str) -> bool:
|
|
152
|
+
if not _cluster_needs_glyph(cluster):
|
|
153
|
+
return True
|
|
154
|
+
font_path = getattr(font, "path", None)
|
|
155
|
+
if font_path is not None:
|
|
156
|
+
try:
|
|
157
|
+
face_index = int(getattr(font, "index", 0))
|
|
158
|
+
points = font_codepoints(str(font_path), face_index)
|
|
159
|
+
return all(ord(c) in points for c in cluster if needs_glyph(c))
|
|
160
|
+
except Exception:
|
|
161
|
+
return False
|
|
162
|
+
# Fallback to getmask inspection
|
|
163
|
+
try:
|
|
164
|
+
mask = font.getmask(cluster)
|
|
165
|
+
return mask is not None and mask.getbbox() is not None
|
|
166
|
+
except Exception:
|
|
167
|
+
return False
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def find_fallback_fonts_for_clusters(
|
|
171
|
+
clusters: list[str],
|
|
172
|
+
primary_font: Any,
|
|
173
|
+
fallback_fonts: list[Any],
|
|
174
|
+
) -> list[tuple[str, Any]]:
|
|
175
|
+
"""Group contiguous clusters that share a font into runs.
|
|
176
|
+
|
|
177
|
+
Primary font is used wherever glyphs are supported; fallback font from
|
|
178
|
+
available pool is used for unsupported clusters. If any visible cluster
|
|
179
|
+
lacks coverage across both primary and all fallback fonts, or contains
|
|
180
|
+
an invalid control, the entire line fails closed (returns an empty list).
|
|
181
|
+
"""
|
|
182
|
+
if not clusters:
|
|
183
|
+
return []
|
|
184
|
+
|
|
185
|
+
full_text = "".join(clusters)
|
|
186
|
+
if invalid_control(full_text) is not None:
|
|
187
|
+
return []
|
|
188
|
+
|
|
189
|
+
cluster_font_assignments: list[tuple[str, Any]] = []
|
|
190
|
+
for cluster in _joining_spans(clusters):
|
|
191
|
+
if not _cluster_needs_glyph(cluster):
|
|
192
|
+
# Whitespace / formatting controls inherit preceding font or primary font
|
|
193
|
+
assigned = (
|
|
194
|
+
cluster_font_assignments[-1][1]
|
|
195
|
+
if cluster_font_assignments
|
|
196
|
+
else primary_font
|
|
197
|
+
)
|
|
198
|
+
cluster_font_assignments.append((cluster, assigned))
|
|
199
|
+
continue
|
|
200
|
+
|
|
201
|
+
if _cluster_supported_by_font(primary_font, cluster):
|
|
202
|
+
cluster_font_assignments.append((cluster, primary_font))
|
|
203
|
+
continue
|
|
204
|
+
|
|
205
|
+
# Find first matching font in fallback_fonts
|
|
206
|
+
chosen_fallback = None
|
|
207
|
+
for fb_font in fallback_fonts:
|
|
208
|
+
if _cluster_supported_by_font(fb_font, cluster):
|
|
209
|
+
chosen_fallback = fb_font
|
|
210
|
+
break
|
|
211
|
+
|
|
212
|
+
if chosen_fallback is not None:
|
|
213
|
+
cluster_font_assignments.append((cluster, chosen_fallback))
|
|
214
|
+
else:
|
|
215
|
+
# Cluster unsupported across whole pool: fail closed
|
|
216
|
+
return []
|
|
217
|
+
|
|
218
|
+
# Group contiguous clusters that share the same font into runs
|
|
219
|
+
runs: list[tuple[str, Any]] = []
|
|
220
|
+
for cluster, font in cluster_font_assignments:
|
|
221
|
+
if runs and runs[-1][1] == font:
|
|
222
|
+
runs[-1] = (runs[-1][0] + cluster, font)
|
|
223
|
+
else:
|
|
224
|
+
runs.append((cluster, font))
|
|
225
|
+
|
|
226
|
+
return runs
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def check_cluster_fallback_coverage(
|
|
230
|
+
text: str, pool_paths: list[str] | tuple[str, ...] | set[str]
|
|
231
|
+
) -> str | None:
|
|
232
|
+
"""Verify that every visible grapheme cluster in *text* is covered by at least
|
|
233
|
+
one font in *pool_paths*, and that text has no invalid controls or separators.
|
|
234
|
+
|
|
235
|
+
Returns None if fully covered, or an error reason string if unsupported.
|
|
236
|
+
"""
|
|
237
|
+
if not pool_paths:
|
|
238
|
+
return "selected font pool is empty"
|
|
239
|
+
ctrl_err = invalid_control(text)
|
|
240
|
+
if ctrl_err is not None:
|
|
241
|
+
return ctrl_err
|
|
242
|
+
clusters = _joining_spans(split_grapheme_clusters(text))
|
|
243
|
+
has_visible = False
|
|
244
|
+
|
|
245
|
+
font_points_list: list[frozenset[int]] = []
|
|
246
|
+
for p in pool_paths:
|
|
247
|
+
try:
|
|
248
|
+
pts = font_codepoints(p, 0)
|
|
249
|
+
font_points_list.append(pts)
|
|
250
|
+
except Exception as exc:
|
|
251
|
+
return f"cannot inspect font cmap for {p}: {exc}"
|
|
252
|
+
|
|
253
|
+
for c in clusters:
|
|
254
|
+
if not _cluster_needs_glyph(c):
|
|
255
|
+
continue
|
|
256
|
+
has_visible = True
|
|
257
|
+
needed_cps = [ord(ch) for ch in c if needs_glyph(ch)]
|
|
258
|
+
covered = any(all(cp in pts for cp in needed_cps) for pts in font_points_list)
|
|
259
|
+
if not covered:
|
|
260
|
+
missing_hex = ", ".join(f"U+{cp:04X}" for cp in needed_cps)
|
|
261
|
+
return f"missing glyphs for cluster {c!r} ({missing_hex}) in font pool"
|
|
262
|
+
if not has_visible:
|
|
263
|
+
return "line has no visible glyphs"
|
|
264
|
+
return None
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
@lru_cache(maxsize=1)
|
|
268
|
+
def _bidi_library() -> Any:
|
|
269
|
+
"""Load the same bidi engine used by RAQM, with explicit ctypes signatures."""
|
|
270
|
+
import ctypes
|
|
271
|
+
import ctypes.util
|
|
272
|
+
|
|
273
|
+
lib = ctypes.CDLL(ctypes.util.find_library("fribidi") or "libfribidi.so.0")
|
|
274
|
+
lib.fribidi_get_bidi_type.argtypes = [ctypes.c_uint32]
|
|
275
|
+
lib.fribidi_get_bidi_type.restype = ctypes.c_uint32
|
|
276
|
+
lib.fribidi_log2vis.argtypes = [
|
|
277
|
+
ctypes.POINTER(ctypes.c_uint32), ctypes.c_int,
|
|
278
|
+
ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32),
|
|
279
|
+
ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
|
|
280
|
+
ctypes.POINTER(ctypes.c_uint8),
|
|
281
|
+
]
|
|
282
|
+
lib.fribidi_log2vis.restype = ctypes.c_uint8
|
|
283
|
+
return lib
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def get_bidi_levels(text: str, base_dir: str = "auto") -> list[int]:
|
|
287
|
+
"""Resolve full-line UAX #9 levels; never substitute heuristic bidi ordering."""
|
|
288
|
+
error = invalid_control(text)
|
|
289
|
+
if error is not None:
|
|
290
|
+
raise ValueError(error)
|
|
291
|
+
if not text:
|
|
292
|
+
return []
|
|
293
|
+
# Plain LTR text needs no external bidi resolution.
|
|
294
|
+
if base_dir != "rtl" and all(
|
|
295
|
+
unicodedata.bidirectional(c) in {"L", "EN", "ES", "ET", "CS", "NSM", "BN", "WS", "ON"}
|
|
296
|
+
for c in text
|
|
297
|
+
):
|
|
298
|
+
return [0] * len(text)
|
|
299
|
+
|
|
300
|
+
import ctypes
|
|
301
|
+
|
|
302
|
+
try:
|
|
303
|
+
lib = _bidi_library()
|
|
304
|
+
except OSError as exc:
|
|
305
|
+
raise RuntimeError("mixed-font RTL rendering requires the FriBidi shared library") from exc
|
|
306
|
+
n = len(text)
|
|
307
|
+
chars = (ctypes.c_uint32 * n)(*[ord(c) for c in text])
|
|
308
|
+
# FriBidi paragraph types are bitmasks, not the numbers 0/1/2.
|
|
309
|
+
base_char = "א" if base_dir == "rtl" else ("A" if base_dir == "ltr" else " ")
|
|
310
|
+
paragraph = ctypes.c_uint32(lib.fribidi_get_bidi_type(ord(base_char)))
|
|
311
|
+
levels = (ctypes.c_uint8 * n)()
|
|
312
|
+
if not lib.fribidi_log2vis(chars, n, ctypes.byref(paragraph), None, None, None, levels):
|
|
313
|
+
raise RuntimeError("FriBidi could not resolve the logical line")
|
|
314
|
+
return list(levels)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def resolve_bidi_runs(
|
|
318
|
+
text: str,
|
|
319
|
+
runs: list[tuple[str, Any]],
|
|
320
|
+
direction: str = "auto",
|
|
321
|
+
) -> list[tuple[str, Any, str]]:
|
|
322
|
+
"""Reorder logical font runs into visual left-to-right order per UAX #9 rule L2.
|
|
323
|
+
|
|
324
|
+
Returns a list of `(run_text, font, run_direction)` tuples where `run_direction`
|
|
325
|
+
is `"rtl"` for odd embedding levels and `"ltr"` for even embedding levels.
|
|
326
|
+
"""
|
|
327
|
+
if not runs:
|
|
328
|
+
return []
|
|
329
|
+
|
|
330
|
+
levels = get_bidi_levels(text, base_dir=direction)
|
|
331
|
+
if not levels or len(levels) != len(text):
|
|
332
|
+
default_dir = "rtl" if direction == "rtl" else "ltr"
|
|
333
|
+
return [(t, f, default_dir) for t, f in runs]
|
|
334
|
+
|
|
335
|
+
slices: list[tuple[str, Any, int, str]] = []
|
|
336
|
+
idx = 0
|
|
337
|
+
for run_text, font in runs:
|
|
338
|
+
run_len = len(run_text)
|
|
339
|
+
run_levels = levels[idx : idx + run_len]
|
|
340
|
+
idx += run_len
|
|
341
|
+
seg_start = 0
|
|
342
|
+
for i in range(1, run_len + 1):
|
|
343
|
+
if i == run_len or run_levels[i] != run_levels[seg_start]:
|
|
344
|
+
seg_text = run_text[seg_start:i]
|
|
345
|
+
seg_lvl = run_levels[seg_start]
|
|
346
|
+
seg_dir = "rtl" if seg_lvl % 2 != 0 else "ltr"
|
|
347
|
+
slices.append((seg_text, font, seg_lvl, seg_dir))
|
|
348
|
+
seg_start = i
|
|
349
|
+
|
|
350
|
+
max_level = max((lvl for _, _, lvl, _ in slices), default=0)
|
|
351
|
+
min_odd_level = min((lvl for _, _, lvl, _ in slices if lvl % 2 != 0), default=None)
|
|
352
|
+
if min_odd_level is not None:
|
|
353
|
+
current = list(slices)
|
|
354
|
+
for lvl in range(max_level, min_odd_level - 1, -1):
|
|
355
|
+
new_list = []
|
|
356
|
+
i = 0
|
|
357
|
+
while i < len(current):
|
|
358
|
+
if current[i][2] >= lvl:
|
|
359
|
+
j = i
|
|
360
|
+
while j < len(current) and current[j][2] >= lvl:
|
|
361
|
+
j += 1
|
|
362
|
+
new_list.extend(reversed(current[i:j]))
|
|
363
|
+
i = j
|
|
364
|
+
else:
|
|
365
|
+
new_list.append(current[i])
|
|
366
|
+
i += 1
|
|
367
|
+
current = new_list
|
|
368
|
+
slices = current
|
|
369
|
+
|
|
370
|
+
return [(t, f, d) for t, f, _, d in slices]
|
|
371
|
+
|
|
372
|
+
def detect_script(text: str) -> str:
|
|
373
|
+
"""Detect dominant script tag (ISO 15924 or alias: Khmr, Latn, Arab, Deva, etc.)."""
|
|
374
|
+
if not text:
|
|
375
|
+
return "Zyyy"
|
|
376
|
+
counts: dict[str, int] = {}
|
|
377
|
+
for ch in text:
|
|
378
|
+
if ch.isspace() or unicodedata.category(ch).startswith("P"):
|
|
379
|
+
continue
|
|
380
|
+
cp = ord(ch)
|
|
381
|
+
if (0x1780 <= cp <= 0x17FF) or (0x19E0 <= cp <= 0x19FF):
|
|
382
|
+
s = "Khmr"
|
|
383
|
+
elif (0x0041 <= cp <= 0x005A) or (0x0061 <= cp <= 0x007A) or (0x00C0 <= cp <= 0x024F):
|
|
384
|
+
s = "Latn"
|
|
385
|
+
elif 0x0600 <= cp <= 0x06FF:
|
|
386
|
+
s = "Arab"
|
|
387
|
+
elif 0x0900 <= cp <= 0x097F:
|
|
388
|
+
s = "Deva"
|
|
389
|
+
elif 0x0980 <= cp <= 0x09FF:
|
|
390
|
+
s = "Beng"
|
|
391
|
+
elif 0x0B80 <= cp <= 0x0BFF:
|
|
392
|
+
s = "Taml"
|
|
393
|
+
elif (0x4E00 <= cp <= 0x9FFF) or (0x3400 <= cp <= 0x4DBF):
|
|
394
|
+
s = "Hani"
|
|
395
|
+
else:
|
|
396
|
+
name = unicodedata.name(ch, "")
|
|
397
|
+
if "KHMER" in name:
|
|
398
|
+
s = "Khmr"
|
|
399
|
+
elif "LATIN" in name:
|
|
400
|
+
s = "Latn"
|
|
401
|
+
elif "ARABIC" in name:
|
|
402
|
+
s = "Arab"
|
|
403
|
+
elif "DEVANAGARI" in name:
|
|
404
|
+
s = "Deva"
|
|
405
|
+
elif "BENGALI" in name:
|
|
406
|
+
s = "Beng"
|
|
407
|
+
elif "TAMIL" in name:
|
|
408
|
+
s = "Taml"
|
|
409
|
+
elif "HANGUL" in name:
|
|
410
|
+
s = "Hang"
|
|
411
|
+
elif "HIRAGANA" in name or "KATAKANA" in name:
|
|
412
|
+
s = "Jpan"
|
|
413
|
+
elif "CYRILLIC" in name:
|
|
414
|
+
s = "Cyrl"
|
|
415
|
+
elif "GREEK" in name:
|
|
416
|
+
s = "Grek"
|
|
417
|
+
elif "THAI" in name:
|
|
418
|
+
s = "Thai"
|
|
419
|
+
elif "LAO" in name:
|
|
420
|
+
s = "Laoo"
|
|
421
|
+
elif "MYANMAR" in name:
|
|
422
|
+
s = "Mymr"
|
|
423
|
+
else:
|
|
424
|
+
s = "Zyyy"
|
|
425
|
+
counts[s] = counts.get(s, 0) + 1
|
|
426
|
+
non_common = {k: v for k, v in counts.items() if k != "Zyyy"}
|
|
427
|
+
if non_common:
|
|
428
|
+
return max(non_common, key=lambda k: non_common[k])
|
|
429
|
+
return "Zyyy"
|