ocr-foundry 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,429 @@
1
+ """Shared, fail-closed Unicode coverage policy, independent of Rust dispatch.
2
+
3
+ Coverage establishes glyph availability, not linguistic correctness. Collections
4
+ use face zero, matching Pillow's explicit index=0 in FontManager.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import unicodedata
10
+ from functools import lru_cache
11
+ from typing import Any
12
+
13
+ import regex
14
+ from fontTools.ttLib import TTFont
15
+
16
+ # Shaping/bidi controls remain in logical text sent to RAQM and in OCR labels.
17
+ # They need no standalone cmap glyph. Other format/control characters fail closed.
18
+ _SHAPING_CONTROLS = frozenset(
19
+ [
20
+ 0x00AD,
21
+ 0x061C,
22
+ 0x200B,
23
+ 0x200C,
24
+ 0x200D,
25
+ 0x200E,
26
+ 0x200F,
27
+ 0x2060,
28
+ 0xFEFF,
29
+ *range(0x202A, 0x202F),
30
+ *range(0x2066, 0x206A),
31
+ ]
32
+ )
33
+
34
+
35
+ def needs_glyph(char: str) -> bool:
36
+ cp = ord(char)
37
+ # Zl/Zp are Unicode whitespace, but they are line boundaries, not ignorable spaces.
38
+ if unicodedata.category(char) in {"Zl", "Zp"}:
39
+ return True
40
+ return not (
41
+ char.isspace()
42
+ or cp in _SHAPING_CONTROLS
43
+ or 0xFE00 <= cp <= 0xFE0F
44
+ or 0xE0100 <= cp <= 0xE01EF
45
+ )
46
+
47
+
48
+ def line_separator_error(text: str) -> str | None:
49
+ """Reject Unicode line/paragraph separators (categories Zl and Zp).
50
+
51
+ ``str.strip()`` treats them as whitespace. A trailing separator would
52
+ otherwise disappear before glyph coverage, and an internal one would be
53
+ copied into labels that ``str.splitlines()`` later splits.
54
+ """
55
+ for char in text:
56
+ if unicodedata.category(char) in {"Zl", "Zp"}:
57
+ return f"unsupported line separator U+{ord(char):04X}"
58
+ return None
59
+
60
+
61
+ def invalid_control(text: str) -> str | None:
62
+ separator = line_separator_error(text)
63
+ if separator:
64
+ return separator
65
+ for char in text:
66
+ if unicodedata.category(char) in {"Cc", "Cf", "Cs"} and ord(char) not in _SHAPING_CONTROLS:
67
+ return f"unsupported control U+{ord(char):04X}"
68
+ return None
69
+
70
+
71
+ @lru_cache(maxsize=256)
72
+ def font_codepoints(path: str, face_index: int = 0) -> frozenset[int]:
73
+ with TTFont(path, fontNumber=face_index, lazy=True) as font:
74
+ cmap = font.getBestCmap()
75
+ if cmap is None:
76
+ raise ValueError(f"Font has no Unicode cmap: {path} (face {face_index})")
77
+ return frozenset(cp for cp, glyph in cmap.items() if glyph != ".notdef")
78
+
79
+
80
+ def coverage_error(path: str, text: str, face_index: int = 0) -> str | None:
81
+ reason = invalid_control(text)
82
+ if reason:
83
+ return reason
84
+ try:
85
+ points = font_codepoints(path, face_index)
86
+ except Exception as exc:
87
+ return f"cannot inspect font cmap: {exc}"
88
+ missing = sorted({ord(c) for c in text if needs_glyph(c) and ord(c) not in points})
89
+ if missing:
90
+ return "missing glyphs " + ", ".join(f"U+{cp:04X}" for cp in missing[:8])
91
+ if not any(needs_glyph(c) for c in text):
92
+ return "line has no visible glyphs"
93
+ return None
94
+
95
+
96
+ def split_grapheme_clusters(text: str) -> list[str]:
97
+ """Segment *text* into grapheme clusters per UAX #29.
98
+
99
+ Ensures combining marks, virama sequences (Indic conjuncts), Khmer coeng
100
+ sequences, and joiners (ZWJ/ZWNJ) remain bound to their consonant/base
101
+ grapheme and are never severed across fallback font boundaries.
102
+ """
103
+ if not text:
104
+ return []
105
+ raw_clusters = regex.findall(r"\X", text)
106
+
107
+ merged: list[str] = []
108
+ for c in raw_clusters:
109
+ if not merged:
110
+ merged.append(c)
111
+ continue
112
+ prev = merged[-1]
113
+ prev_tail = prev.rstrip("\u200c\u200d")
114
+ ends_with_virama = bool(prev_tail and unicodedata.combining(prev_tail[-1]) == 9)
115
+ ends_with_zwj = prev.endswith("\u200d")
116
+ if (ends_with_virama and unicodedata.category(c[0]).startswith("L")) or (
117
+ ends_with_zwj and not c[0].isspace()
118
+ ):
119
+ merged[-1] += c
120
+ else:
121
+ merged.append(c)
122
+ return merged
123
+
124
+
125
+ def _joining_spans(clusters: list[str]) -> list[str]:
126
+ """Keep connected cursive letters in one shaping call and one font."""
127
+ spans: list[str] = []
128
+ for cluster in clusters:
129
+ if spans:
130
+ left = regex.sub(r"\p{Joining_Type=Transparent}+$", "", spans[-1])
131
+ right = regex.sub(r"^\p{Joining_Type=Transparent}+", "", cluster)
132
+ joins_next = left and regex.fullmatch(
133
+ r"[\p{Joining_Type=Dual_Joining}\p{Joining_Type=Left_Joining}\p{Joining_Type=Join_Causing}]",
134
+ left[-1],
135
+ )
136
+ joins_previous = right and regex.fullmatch(
137
+ r"[\p{Joining_Type=Dual_Joining}\p{Joining_Type=Right_Joining}\p{Joining_Type=Join_Causing}]",
138
+ right[0],
139
+ )
140
+ if joins_next and joins_previous:
141
+ spans[-1] += cluster
142
+ continue
143
+ spans.append(cluster)
144
+ return spans
145
+
146
+
147
+ def _cluster_needs_glyph(cluster: str) -> bool:
148
+ return any(needs_glyph(c) for c in cluster)
149
+
150
+
151
+ def _cluster_supported_by_font(font: Any, cluster: str) -> bool:
152
+ if not _cluster_needs_glyph(cluster):
153
+ return True
154
+ font_path = getattr(font, "path", None)
155
+ if font_path is not None:
156
+ try:
157
+ face_index = int(getattr(font, "index", 0))
158
+ points = font_codepoints(str(font_path), face_index)
159
+ return all(ord(c) in points for c in cluster if needs_glyph(c))
160
+ except Exception:
161
+ return False
162
+ # Fallback to getmask inspection
163
+ try:
164
+ mask = font.getmask(cluster)
165
+ return mask is not None and mask.getbbox() is not None
166
+ except Exception:
167
+ return False
168
+
169
+
170
+ def find_fallback_fonts_for_clusters(
171
+ clusters: list[str],
172
+ primary_font: Any,
173
+ fallback_fonts: list[Any],
174
+ ) -> list[tuple[str, Any]]:
175
+ """Group contiguous clusters that share a font into runs.
176
+
177
+ Primary font is used wherever glyphs are supported; fallback font from
178
+ available pool is used for unsupported clusters. If any visible cluster
179
+ lacks coverage across both primary and all fallback fonts, or contains
180
+ an invalid control, the entire line fails closed (returns an empty list).
181
+ """
182
+ if not clusters:
183
+ return []
184
+
185
+ full_text = "".join(clusters)
186
+ if invalid_control(full_text) is not None:
187
+ return []
188
+
189
+ cluster_font_assignments: list[tuple[str, Any]] = []
190
+ for cluster in _joining_spans(clusters):
191
+ if not _cluster_needs_glyph(cluster):
192
+ # Whitespace / formatting controls inherit preceding font or primary font
193
+ assigned = (
194
+ cluster_font_assignments[-1][1]
195
+ if cluster_font_assignments
196
+ else primary_font
197
+ )
198
+ cluster_font_assignments.append((cluster, assigned))
199
+ continue
200
+
201
+ if _cluster_supported_by_font(primary_font, cluster):
202
+ cluster_font_assignments.append((cluster, primary_font))
203
+ continue
204
+
205
+ # Find first matching font in fallback_fonts
206
+ chosen_fallback = None
207
+ for fb_font in fallback_fonts:
208
+ if _cluster_supported_by_font(fb_font, cluster):
209
+ chosen_fallback = fb_font
210
+ break
211
+
212
+ if chosen_fallback is not None:
213
+ cluster_font_assignments.append((cluster, chosen_fallback))
214
+ else:
215
+ # Cluster unsupported across whole pool: fail closed
216
+ return []
217
+
218
+ # Group contiguous clusters that share the same font into runs
219
+ runs: list[tuple[str, Any]] = []
220
+ for cluster, font in cluster_font_assignments:
221
+ if runs and runs[-1][1] == font:
222
+ runs[-1] = (runs[-1][0] + cluster, font)
223
+ else:
224
+ runs.append((cluster, font))
225
+
226
+ return runs
227
+
228
+
229
+ def check_cluster_fallback_coverage(
230
+ text: str, pool_paths: list[str] | tuple[str, ...] | set[str]
231
+ ) -> str | None:
232
+ """Verify that every visible grapheme cluster in *text* is covered by at least
233
+ one font in *pool_paths*, and that text has no invalid controls or separators.
234
+
235
+ Returns None if fully covered, or an error reason string if unsupported.
236
+ """
237
+ if not pool_paths:
238
+ return "selected font pool is empty"
239
+ ctrl_err = invalid_control(text)
240
+ if ctrl_err is not None:
241
+ return ctrl_err
242
+ clusters = _joining_spans(split_grapheme_clusters(text))
243
+ has_visible = False
244
+
245
+ font_points_list: list[frozenset[int]] = []
246
+ for p in pool_paths:
247
+ try:
248
+ pts = font_codepoints(p, 0)
249
+ font_points_list.append(pts)
250
+ except Exception as exc:
251
+ return f"cannot inspect font cmap for {p}: {exc}"
252
+
253
+ for c in clusters:
254
+ if not _cluster_needs_glyph(c):
255
+ continue
256
+ has_visible = True
257
+ needed_cps = [ord(ch) for ch in c if needs_glyph(ch)]
258
+ covered = any(all(cp in pts for cp in needed_cps) for pts in font_points_list)
259
+ if not covered:
260
+ missing_hex = ", ".join(f"U+{cp:04X}" for cp in needed_cps)
261
+ return f"missing glyphs for cluster {c!r} ({missing_hex}) in font pool"
262
+ if not has_visible:
263
+ return "line has no visible glyphs"
264
+ return None
265
+
266
+
267
+ @lru_cache(maxsize=1)
268
+ def _bidi_library() -> Any:
269
+ """Load the same bidi engine used by RAQM, with explicit ctypes signatures."""
270
+ import ctypes
271
+ import ctypes.util
272
+
273
+ lib = ctypes.CDLL(ctypes.util.find_library("fribidi") or "libfribidi.so.0")
274
+ lib.fribidi_get_bidi_type.argtypes = [ctypes.c_uint32]
275
+ lib.fribidi_get_bidi_type.restype = ctypes.c_uint32
276
+ lib.fribidi_log2vis.argtypes = [
277
+ ctypes.POINTER(ctypes.c_uint32), ctypes.c_int,
278
+ ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32),
279
+ ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
280
+ ctypes.POINTER(ctypes.c_uint8),
281
+ ]
282
+ lib.fribidi_log2vis.restype = ctypes.c_uint8
283
+ return lib
284
+
285
+
286
+ def get_bidi_levels(text: str, base_dir: str = "auto") -> list[int]:
287
+ """Resolve full-line UAX #9 levels; never substitute heuristic bidi ordering."""
288
+ error = invalid_control(text)
289
+ if error is not None:
290
+ raise ValueError(error)
291
+ if not text:
292
+ return []
293
+ # Plain LTR text needs no external bidi resolution.
294
+ if base_dir != "rtl" and all(
295
+ unicodedata.bidirectional(c) in {"L", "EN", "ES", "ET", "CS", "NSM", "BN", "WS", "ON"}
296
+ for c in text
297
+ ):
298
+ return [0] * len(text)
299
+
300
+ import ctypes
301
+
302
+ try:
303
+ lib = _bidi_library()
304
+ except OSError as exc:
305
+ raise RuntimeError("mixed-font RTL rendering requires the FriBidi shared library") from exc
306
+ n = len(text)
307
+ chars = (ctypes.c_uint32 * n)(*[ord(c) for c in text])
308
+ # FriBidi paragraph types are bitmasks, not the numbers 0/1/2.
309
+ base_char = "א" if base_dir == "rtl" else ("A" if base_dir == "ltr" else " ")
310
+ paragraph = ctypes.c_uint32(lib.fribidi_get_bidi_type(ord(base_char)))
311
+ levels = (ctypes.c_uint8 * n)()
312
+ if not lib.fribidi_log2vis(chars, n, ctypes.byref(paragraph), None, None, None, levels):
313
+ raise RuntimeError("FriBidi could not resolve the logical line")
314
+ return list(levels)
315
+
316
+
317
+ def resolve_bidi_runs(
318
+ text: str,
319
+ runs: list[tuple[str, Any]],
320
+ direction: str = "auto",
321
+ ) -> list[tuple[str, Any, str]]:
322
+ """Reorder logical font runs into visual left-to-right order per UAX #9 rule L2.
323
+
324
+ Returns a list of `(run_text, font, run_direction)` tuples where `run_direction`
325
+ is `"rtl"` for odd embedding levels and `"ltr"` for even embedding levels.
326
+ """
327
+ if not runs:
328
+ return []
329
+
330
+ levels = get_bidi_levels(text, base_dir=direction)
331
+ if not levels or len(levels) != len(text):
332
+ default_dir = "rtl" if direction == "rtl" else "ltr"
333
+ return [(t, f, default_dir) for t, f in runs]
334
+
335
+ slices: list[tuple[str, Any, int, str]] = []
336
+ idx = 0
337
+ for run_text, font in runs:
338
+ run_len = len(run_text)
339
+ run_levels = levels[idx : idx + run_len]
340
+ idx += run_len
341
+ seg_start = 0
342
+ for i in range(1, run_len + 1):
343
+ if i == run_len or run_levels[i] != run_levels[seg_start]:
344
+ seg_text = run_text[seg_start:i]
345
+ seg_lvl = run_levels[seg_start]
346
+ seg_dir = "rtl" if seg_lvl % 2 != 0 else "ltr"
347
+ slices.append((seg_text, font, seg_lvl, seg_dir))
348
+ seg_start = i
349
+
350
+ max_level = max((lvl for _, _, lvl, _ in slices), default=0)
351
+ min_odd_level = min((lvl for _, _, lvl, _ in slices if lvl % 2 != 0), default=None)
352
+ if min_odd_level is not None:
353
+ current = list(slices)
354
+ for lvl in range(max_level, min_odd_level - 1, -1):
355
+ new_list = []
356
+ i = 0
357
+ while i < len(current):
358
+ if current[i][2] >= lvl:
359
+ j = i
360
+ while j < len(current) and current[j][2] >= lvl:
361
+ j += 1
362
+ new_list.extend(reversed(current[i:j]))
363
+ i = j
364
+ else:
365
+ new_list.append(current[i])
366
+ i += 1
367
+ current = new_list
368
+ slices = current
369
+
370
+ return [(t, f, d) for t, f, _, d in slices]
371
+
372
+ def detect_script(text: str) -> str:
373
+ """Detect dominant script tag (ISO 15924 or alias: Khmr, Latn, Arab, Deva, etc.)."""
374
+ if not text:
375
+ return "Zyyy"
376
+ counts: dict[str, int] = {}
377
+ for ch in text:
378
+ if ch.isspace() or unicodedata.category(ch).startswith("P"):
379
+ continue
380
+ cp = ord(ch)
381
+ if (0x1780 <= cp <= 0x17FF) or (0x19E0 <= cp <= 0x19FF):
382
+ s = "Khmr"
383
+ elif (0x0041 <= cp <= 0x005A) or (0x0061 <= cp <= 0x007A) or (0x00C0 <= cp <= 0x024F):
384
+ s = "Latn"
385
+ elif 0x0600 <= cp <= 0x06FF:
386
+ s = "Arab"
387
+ elif 0x0900 <= cp <= 0x097F:
388
+ s = "Deva"
389
+ elif 0x0980 <= cp <= 0x09FF:
390
+ s = "Beng"
391
+ elif 0x0B80 <= cp <= 0x0BFF:
392
+ s = "Taml"
393
+ elif (0x4E00 <= cp <= 0x9FFF) or (0x3400 <= cp <= 0x4DBF):
394
+ s = "Hani"
395
+ else:
396
+ name = unicodedata.name(ch, "")
397
+ if "KHMER" in name:
398
+ s = "Khmr"
399
+ elif "LATIN" in name:
400
+ s = "Latn"
401
+ elif "ARABIC" in name:
402
+ s = "Arab"
403
+ elif "DEVANAGARI" in name:
404
+ s = "Deva"
405
+ elif "BENGALI" in name:
406
+ s = "Beng"
407
+ elif "TAMIL" in name:
408
+ s = "Taml"
409
+ elif "HANGUL" in name:
410
+ s = "Hang"
411
+ elif "HIRAGANA" in name or "KATAKANA" in name:
412
+ s = "Jpan"
413
+ elif "CYRILLIC" in name:
414
+ s = "Cyrl"
415
+ elif "GREEK" in name:
416
+ s = "Grek"
417
+ elif "THAI" in name:
418
+ s = "Thai"
419
+ elif "LAO" in name:
420
+ s = "Laoo"
421
+ elif "MYANMAR" in name:
422
+ s = "Mymr"
423
+ else:
424
+ s = "Zyyy"
425
+ counts[s] = counts.get(s, 0) + 1
426
+ non_common = {k: v for k, v in counts.items() if k != "Zyyy"}
427
+ if non_common:
428
+ return max(non_common, key=lambda k: non_common[k])
429
+ return "Zyyy"