ocr-foundry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khocr_gen/__init__.py +11 -0
- khocr_gen/_pil_compat.py +25 -0
- khocr_gen/_rust_accel.py +204 -0
- khocr_gen/augmentation.py +1193 -0
- khocr_gen/benchmark.py +888 -0
- khocr_gen/benchmark_cmd.py +103 -0
- khocr_gen/cli.py +246 -0
- khocr_gen/combine.py +214 -0
- khocr_gen/combine_cmd.py +125 -0
- khocr_gen/config.py +1474 -0
- khocr_gen/config_loader.py +157 -0
- khocr_gen/corpus.py +621 -0
- khocr_gen/data_generator.py +1160 -0
- khocr_gen/errors.py +33 -0
- khocr_gen/evaluate_cmd.py +50 -0
- khocr_gen/evaluation.py +259 -0
- khocr_gen/fonts.py +316 -0
- khocr_gen/generate.py +227 -0
- khocr_gen/layout.py +1028 -0
- khocr_gen/layout_cmd.py +371 -0
- khocr_gen/layout_degradation.py +128 -0
- khocr_gen/layout_mesh.py +200 -0
- khocr_gen/layout_shaping.py +264 -0
- khocr_gen/layout_transform.py +238 -0
- khocr_gen/layout_warp.py +473 -0
- khocr_gen/line_height.py +127 -0
- khocr_gen/lmdb_pack.py +132 -0
- khocr_gen/logging.py +25 -0
- khocr_gen/normalizer.py +221 -0
- khocr_gen/parallel.py +366 -0
- khocr_gen/provenance.py +328 -0
- khocr_gen/rendering.py +1607 -0
- khocr_gen/text_support.py +429 -0
- khocr_gen/verify.py +471 -0
- khocr_gen/viewer.py +172 -0
- ocr_foundry-0.2.0.dist-info/METADATA +174 -0
- ocr_foundry-0.2.0.dist-info/RECORD +40 -0
- ocr_foundry-0.2.0.dist-info/WHEEL +4 -0
- ocr_foundry-0.2.0.dist-info/entry_points.txt +3 -0
- ocr_foundry-0.2.0.dist-info/licenses/LICENSE +21 -0
khocr_gen/parallel.py
ADDED
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
"""Parallel data generation workers.
|
|
2
|
+
|
|
3
|
+
Each worker process renders text lines from a corpus
|
|
4
|
+
and applies exactly one augmentation method per rendered image (isolated augmentation model).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import ast
|
|
10
|
+
import contextlib
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import logging
|
|
14
|
+
import os
|
|
15
|
+
import random
|
|
16
|
+
import sys
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import cv2
|
|
21
|
+
import numpy as np
|
|
22
|
+
|
|
23
|
+
from .augmentation import write_output_image
|
|
24
|
+
from .config import AugMethodConfig, GenerationConfig
|
|
25
|
+
from .fonts import FontManager
|
|
26
|
+
from .provenance import build_sample_records
|
|
27
|
+
from .rendering import ImageRenderer
|
|
28
|
+
from .text_support import detect_script
|
|
29
|
+
|
|
30
|
+
_LOGGER = logging.getLogger("khocr_gen.parallel")
|
|
31
|
+
_AUTO_PARALLEL_MIN_SAMPLES = 2_048
|
|
32
|
+
_DEFAULT_RENDER_CHUNK_SIZE = 64
|
|
33
|
+
_MAX_RENDER_CHUNK_SIZE = 256
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def derive_sample_seed(
|
|
37
|
+
master_seed: int,
|
|
38
|
+
sample_index: int,
|
|
39
|
+
sample_id: str | int,
|
|
40
|
+
) -> int:
|
|
41
|
+
"""Deterministically derive a 64-bit integer seed for a single sample.
|
|
42
|
+
|
|
43
|
+
Stable hashing (SHA-256) over (master_seed, sample_index, sample_id) ensures
|
|
44
|
+
the derived seed is independent of worker allocation, chunking, and scheduling.
|
|
45
|
+
"""
|
|
46
|
+
key = f"khocr-sample-seed/{master_seed}/{sample_index}/{sample_id}".encode()
|
|
47
|
+
digest = hashlib.sha256(key).digest()
|
|
48
|
+
return int.from_bytes(digest[:8], byteorder="big", signed=False)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def seed_sample_generators(
|
|
52
|
+
master_seed: int | None,
|
|
53
|
+
sample_index: int,
|
|
54
|
+
sample_id: str | int,
|
|
55
|
+
) -> int | None:
|
|
56
|
+
"""Seed Python and NumPy random generators for a single sample if master_seed is set.
|
|
57
|
+
|
|
58
|
+
Returns the derived seed when active, or None when master_seed is None.
|
|
59
|
+
"""
|
|
60
|
+
if master_seed is None:
|
|
61
|
+
return None
|
|
62
|
+
sample_seed = derive_sample_seed(master_seed, sample_index, sample_id)
|
|
63
|
+
random.seed(sample_seed)
|
|
64
|
+
np.random.seed(sample_seed % (2**32))
|
|
65
|
+
return sample_seed
|
|
66
|
+
|
|
67
|
+
def resolve_worker_count(workers: int, sample_count: int) -> int:
|
|
68
|
+
"""Determine how many worker processes to use.
|
|
69
|
+
|
|
70
|
+
Returns 1 when auto-resolve decides parallelism isn't worthwhile.
|
|
71
|
+
"""
|
|
72
|
+
requested = max(0, int(workers))
|
|
73
|
+
cpu_count = os.cpu_count() or 1
|
|
74
|
+
|
|
75
|
+
if sample_count <= 0 or cpu_count <= 1:
|
|
76
|
+
return 1
|
|
77
|
+
if requested == 1:
|
|
78
|
+
return 1
|
|
79
|
+
if requested > 1:
|
|
80
|
+
return min(requested, cpu_count)
|
|
81
|
+
if sample_count < _AUTO_PARALLEL_MIN_SAMPLES:
|
|
82
|
+
return 1
|
|
83
|
+
return max(1, cpu_count // 2)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def resolve_chunk_size(sample_count: int, workers: int) -> int:
|
|
87
|
+
"""Pick a batch size that balances load across workers."""
|
|
88
|
+
if sample_count <= 0:
|
|
89
|
+
return _DEFAULT_RENDER_CHUNK_SIZE
|
|
90
|
+
if workers <= 1:
|
|
91
|
+
return _DEFAULT_RENDER_CHUNK_SIZE
|
|
92
|
+
|
|
93
|
+
target = sample_count // max(1, workers * 32)
|
|
94
|
+
return max(16, min(_MAX_RENDER_CHUNK_SIZE, max(_DEFAULT_RENDER_CHUNK_SIZE, target)))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def resolve_mp_context() -> tuple[Any, str]:
|
|
98
|
+
"""Pick a multiprocessing context that stays safe after other threads exist.
|
|
99
|
+
|
|
100
|
+
``ProcessPoolExecutor`` starts workers from its own thread. ``fork`` from
|
|
101
|
+
that thread can deadlock once the parent holds locks from Pillow, OpenCV,
|
|
102
|
+
or coverage. Linux therefore uses ``forkserver``: a fresh single-threaded
|
|
103
|
+
server forks the workers, without re-importing the pipeline for each one.
|
|
104
|
+
macOS / Windows use ``spawn``. Override with ``KHOCR_GEN_MP_START_METHOD``
|
|
105
|
+
(``forkserver``, ``fork``, or ``spawn``).
|
|
106
|
+
"""
|
|
107
|
+
import multiprocessing as mp
|
|
108
|
+
|
|
109
|
+
requested = os.environ.get("KHOCR_GEN_MP_START_METHOD", "").strip().lower()
|
|
110
|
+
available = set(mp.get_all_start_methods())
|
|
111
|
+
|
|
112
|
+
if requested:
|
|
113
|
+
if requested in available:
|
|
114
|
+
return mp.get_context(requested), requested
|
|
115
|
+
return mp.get_context("spawn"), "spawn"
|
|
116
|
+
|
|
117
|
+
if sys.platform == "linux" and "forkserver" in available:
|
|
118
|
+
return mp.get_context("forkserver"), "forkserver"
|
|
119
|
+
|
|
120
|
+
return mp.get_context("spawn"), "spawn"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _is_dunder_main_comparison(test: ast.expr) -> bool:
|
|
124
|
+
"""True when *test* is exactly ``__name__ == "__main__"`` (either order)."""
|
|
125
|
+
if not isinstance(test, ast.Compare) or len(test.ops) != 1 or len(test.comparators) != 1:
|
|
126
|
+
return False
|
|
127
|
+
if not isinstance(test.ops[0], ast.Eq):
|
|
128
|
+
return False
|
|
129
|
+
left, right = test.left, test.comparators[0]
|
|
130
|
+
pair = (
|
|
131
|
+
(
|
|
132
|
+
isinstance(left, ast.Name)
|
|
133
|
+
and left.id == "__name__"
|
|
134
|
+
and isinstance(right, ast.Constant)
|
|
135
|
+
and right.value == "__main__"
|
|
136
|
+
),
|
|
137
|
+
(
|
|
138
|
+
isinstance(right, ast.Name)
|
|
139
|
+
and right.id == "__name__"
|
|
140
|
+
and isinstance(left, ast.Constant)
|
|
141
|
+
and left.value == "__main__"
|
|
142
|
+
),
|
|
143
|
+
)
|
|
144
|
+
return any(pair)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _has_executable_main_guard(source: str) -> bool:
|
|
148
|
+
"""True only for a module-level guard that runs on import.
|
|
149
|
+
|
|
150
|
+
A textual match is not enough. The ``if`` must be a direct module
|
|
151
|
+
statement, so a quoted copy, a function body, or a dead ``if False:``
|
|
152
|
+
branch cannot hide the re-import.
|
|
153
|
+
"""
|
|
154
|
+
try:
|
|
155
|
+
tree = ast.parse(source)
|
|
156
|
+
except SyntaxError:
|
|
157
|
+
return False
|
|
158
|
+
return any(
|
|
159
|
+
isinstance(node, ast.If) and _is_dunder_main_comparison(node.test) for node in tree.body
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def ensure_worker_main_guard(start_method: str, main_module: Any | None = None) -> None:
|
|
164
|
+
"""Refuse forkserver/spawn when importing ``__main__`` would start another pool.
|
|
165
|
+
|
|
166
|
+
An unguarded script re-executes on worker startup. The child then raises
|
|
167
|
+
``RuntimeError`` during bootstrap, and the parent only sees
|
|
168
|
+
``BrokenProcessPool``. ``fork`` does not re-import ``__main__`` and stays
|
|
169
|
+
available through ``KHOCR_GEN_MP_START_METHOD``. ``python -c`` and
|
|
170
|
+
interactive sessions have no script file to re-import. The guard is parsed
|
|
171
|
+
from the AST: it must be an executable module-level comparison, not text
|
|
172
|
+
inside a string or a branch that does not run.
|
|
173
|
+
"""
|
|
174
|
+
if start_method == "fork":
|
|
175
|
+
return
|
|
176
|
+
if main_module is None:
|
|
177
|
+
main_module = sys.modules.get("__main__")
|
|
178
|
+
if main_module is None:
|
|
179
|
+
return
|
|
180
|
+
main_file = getattr(main_module, "__file__", None)
|
|
181
|
+
if not main_file:
|
|
182
|
+
return
|
|
183
|
+
try:
|
|
184
|
+
source = Path(main_file).read_text(encoding="utf-8")
|
|
185
|
+
except (OSError, UnicodeError):
|
|
186
|
+
return
|
|
187
|
+
if _has_executable_main_guard(source):
|
|
188
|
+
return
|
|
189
|
+
raise RuntimeError(
|
|
190
|
+
f"Refusing to start {start_method} workers because {main_file} has no "
|
|
191
|
+
'`if __name__ == "__main__":` guard. Wrap generation in that guard so '
|
|
192
|
+
"workers can import the script without starting another pool. Without "
|
|
193
|
+
"it, the child dies during bootstrap and the parent only reports "
|
|
194
|
+
"BrokenProcessPool. Set KHOCR_GEN_MP_START_METHOD=fork to keep the "
|
|
195
|
+
"previous start method when the parent is still single-threaded."
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# Per-process worker state
|
|
200
|
+
_WORKER_STATE: dict[str, Any] = {}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _init_render_worker(config_dict: dict[str, Any]) -> None:
|
|
204
|
+
"""Initialise a worker process with its own renderer and config."""
|
|
205
|
+
global _WORKER_STATE
|
|
206
|
+
|
|
207
|
+
# Limit OpenCV threads in worker processes
|
|
208
|
+
with contextlib.suppress(Exception):
|
|
209
|
+
cv2.setNumThreads(1)
|
|
210
|
+
with contextlib.suppress(Exception):
|
|
211
|
+
cv2.ocl.setUseOpenCL(False)
|
|
212
|
+
|
|
213
|
+
# Derive a per-worker seed from os.urandom
|
|
214
|
+
seed = int.from_bytes(os.urandom(8), "little") ^ (os.getpid() << 16)
|
|
215
|
+
random.seed(seed)
|
|
216
|
+
np.random.seed(seed % (2**32))
|
|
217
|
+
|
|
218
|
+
# Reconstruct GenerationConfig from serialised dict
|
|
219
|
+
gen_cfg = GenerationConfig.from_dict(config_dict)
|
|
220
|
+
|
|
221
|
+
font_manager = FontManager(
|
|
222
|
+
language=gen_cfg.language,
|
|
223
|
+
fonts_dir=gen_cfg.fonts_dir,
|
|
224
|
+
verbose=False,
|
|
225
|
+
multilingual=gen_cfg.multilingual,
|
|
226
|
+
font_pool=gen_cfg.font_pool,
|
|
227
|
+
)
|
|
228
|
+
renderer = ImageRenderer(font_manager, gen_cfg)
|
|
229
|
+
|
|
230
|
+
_WORKER_STATE = {
|
|
231
|
+
"renderer": renderer,
|
|
232
|
+
"split_name": config_dict["split_name"],
|
|
233
|
+
"output_images_dir": config_dict["output_images_dir"],
|
|
234
|
+
"font_mode": config_dict["font_mode"],
|
|
235
|
+
"retry_limit": config_dict["retry_limit"],
|
|
236
|
+
"image_dir": config_dict.get("image_dir"),
|
|
237
|
+
"output_format": config_dict.get("output_format", "jpg"),
|
|
238
|
+
"jpeg_quality": config_dict.get("jpeg_quality", 90),
|
|
239
|
+
"storage": config_dict.get("storage", "raw"),
|
|
240
|
+
"record_metadata": config_dict.get("record_metadata", False),
|
|
241
|
+
"seed": gen_cfg.seed,
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _render_sample_batch(
|
|
246
|
+
batch: list[tuple[int, str, Any]],
|
|
247
|
+
) -> tuple[list[str], int, int, list[str], list[str], list[str], list[int]]:
|
|
248
|
+
"""Render a batch of samples in the worker process.
|
|
249
|
+
|
|
250
|
+
Each sample is rendered to a clean canvas,
|
|
251
|
+
then exactly one augmentation method is applied (chosen probabilistically from the enabled methods).
|
|
252
|
+
|
|
253
|
+
Returns:
|
|
254
|
+
(labels_lines, attempted_count, success_count, error_messages, metadata_lines,
|
|
255
|
+
provenance_lines, heights)
|
|
256
|
+
"""
|
|
257
|
+
renderer: ImageRenderer = _WORKER_STATE["renderer"]
|
|
258
|
+
split_name: str = _WORKER_STATE["split_name"]
|
|
259
|
+
output_images_dir = Path(_WORKER_STATE["output_images_dir"])
|
|
260
|
+
font_mode: str = _WORKER_STATE["font_mode"]
|
|
261
|
+
retry_limit: int = _WORKER_STATE["retry_limit"]
|
|
262
|
+
record_metadata: bool = _WORKER_STATE.get("record_metadata", False)
|
|
263
|
+
current_retry_limit = 1 if font_mode == "all" else retry_limit
|
|
264
|
+
|
|
265
|
+
labels: list[str] = []
|
|
266
|
+
meta_lines: list[str] = []
|
|
267
|
+
prov_lines: list[str] = []
|
|
268
|
+
heights: list[int] = []
|
|
269
|
+
error_messages: list[str] = []
|
|
270
|
+
success_count = 0
|
|
271
|
+
|
|
272
|
+
image_dir = _WORKER_STATE.get("image_dir")
|
|
273
|
+
|
|
274
|
+
# Get enabled augmentation methods once per batch
|
|
275
|
+
enabled_methods: list[tuple[str, AugMethodConfig]] = renderer._cfg.enabled_aug_methods()
|
|
276
|
+
|
|
277
|
+
master_seed = _WORKER_STATE.get("seed")
|
|
278
|
+
for current_idx, text, font_ref_or_image in batch:
|
|
279
|
+
sample_id = f"{split_name}_{current_idx:06d}"
|
|
280
|
+
seed_sample_generators(master_seed, current_idx, sample_id)
|
|
281
|
+
try:
|
|
282
|
+
meta: dict[str, Any] = {}
|
|
283
|
+
if image_dir:
|
|
284
|
+
# Image-dir mode: augment an existing image
|
|
285
|
+
img_path = Path(image_dir) / font_ref_or_image
|
|
286
|
+
img_array = cv2.imread(str(img_path))
|
|
287
|
+
if img_array is None:
|
|
288
|
+
raise RuntimeError(f"Could not read image: {img_path}")
|
|
289
|
+
|
|
290
|
+
if img_array.ndim == 3 and img_array.shape[2] == 3:
|
|
291
|
+
img_array = cv2.cvtColor(img_array, cv2.COLOR_BGR2RGB)
|
|
292
|
+
|
|
293
|
+
# Apply one augmentation probabilistically
|
|
294
|
+
result = renderer.render_with_one_augmentation(
|
|
295
|
+
text,
|
|
296
|
+
enabled_methods,
|
|
297
|
+
specific_font=font_ref_or_image,
|
|
298
|
+
retry_limit=current_retry_limit,
|
|
299
|
+
)
|
|
300
|
+
if result is None:
|
|
301
|
+
# Fallback: augment existing image directly
|
|
302
|
+
aug_results = renderer.augment_image(
|
|
303
|
+
img_array, enabled_methods, script=detect_script(text)
|
|
304
|
+
)
|
|
305
|
+
if not aug_results:
|
|
306
|
+
continue
|
|
307
|
+
_, img = aug_results[0]
|
|
308
|
+
else:
|
|
309
|
+
_, img, meta = result
|
|
310
|
+
else:
|
|
311
|
+
# Text-rendering mode
|
|
312
|
+
result = renderer.render_with_one_augmentation(
|
|
313
|
+
text,
|
|
314
|
+
enabled_methods,
|
|
315
|
+
specific_font=font_ref_or_image,
|
|
316
|
+
retry_limit=current_retry_limit,
|
|
317
|
+
)
|
|
318
|
+
if result is None:
|
|
319
|
+
continue
|
|
320
|
+
_, img, meta = result
|
|
321
|
+
|
|
322
|
+
if img is None:
|
|
323
|
+
continue
|
|
324
|
+
|
|
325
|
+
output_fmt = _WORKER_STATE.get("output_format", "jpg")
|
|
326
|
+
jpeg_q = _WORKER_STATE.get("jpeg_quality", 90)
|
|
327
|
+
img_filename = f"{split_name}_{current_idx:06d}.{output_fmt}"
|
|
328
|
+
img_path = output_images_dir / img_filename
|
|
329
|
+
write_ok = write_output_image(img_path, img, fmt=output_fmt, jpeg_quality=jpeg_q)
|
|
330
|
+
if not write_ok:
|
|
331
|
+
raise RuntimeError(f"cv2.imwrite failed for output path: {img_path}")
|
|
332
|
+
|
|
333
|
+
labels.append(f"{img_filename}\t{text}\n")
|
|
334
|
+
heights.append(int(img.shape[0]))
|
|
335
|
+
if record_metadata:
|
|
336
|
+
record, prov_row = build_sample_records(
|
|
337
|
+
img_filename=img_filename,
|
|
338
|
+
text=text,
|
|
339
|
+
width=img.shape[1],
|
|
340
|
+
height=img.shape[0],
|
|
341
|
+
split_name=split_name,
|
|
342
|
+
meta=meta,
|
|
343
|
+
default_language=renderer._cfg.language,
|
|
344
|
+
)
|
|
345
|
+
if prov_row is not None:
|
|
346
|
+
prov_lines.append(json.dumps(prov_row, ensure_ascii=False) + "\n")
|
|
347
|
+
meta_lines.append(json.dumps(record, ensure_ascii=False) + "\n")
|
|
348
|
+
else:
|
|
349
|
+
paths = [path for path in meta.get("fonts", [meta.get("font")]) if path]
|
|
350
|
+
if paths:
|
|
351
|
+
meta_lines.append(json.dumps({"fonts": paths}) + "\n")
|
|
352
|
+
success_count += 1
|
|
353
|
+
except Exception as exc:
|
|
354
|
+
if len(error_messages) < 3:
|
|
355
|
+
short_text = text[:30] if len(text) > 30 else text
|
|
356
|
+
error_messages.append(f" Failed for '{short_text}...': {exc}")
|
|
357
|
+
|
|
358
|
+
return (
|
|
359
|
+
labels,
|
|
360
|
+
len(batch),
|
|
361
|
+
success_count,
|
|
362
|
+
error_messages,
|
|
363
|
+
meta_lines,
|
|
364
|
+
prov_lines,
|
|
365
|
+
heights,
|
|
366
|
+
)
|
khocr_gen/provenance.py
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""Shared provenance, font licensing, run manifests, and execution metadata.
|
|
2
|
+
|
|
3
|
+
Extracts duplicated metadata and provenance sidecar record construction
|
|
4
|
+
between serial and parallel workers, caches font hashes and embedded
|
|
5
|
+
license name records (nameID 13/14), captures runtime software versions,
|
|
6
|
+
and persists run manifests with recipe identity.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import datetime
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import logging
|
|
15
|
+
import sys
|
|
16
|
+
import unicodedata
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import cv2
|
|
21
|
+
import fontTools
|
|
22
|
+
import numpy as np
|
|
23
|
+
import PIL
|
|
24
|
+
from fontTools.ttLib import TTFont
|
|
25
|
+
from PIL import features
|
|
26
|
+
|
|
27
|
+
from . import __version__
|
|
28
|
+
from . import _rust_accel as _ra
|
|
29
|
+
from .corpus import CorpusItem
|
|
30
|
+
|
|
31
|
+
_LOGGER = logging.getLogger("khocr_gen.provenance")
|
|
32
|
+
|
|
33
|
+
# Process-level immutable caches
|
|
34
|
+
_FONT_INFO_CACHE: dict[tuple[str, int], dict[str, Any]] = {}
|
|
35
|
+
_RUNTIME_VERSIONS_CACHE: dict[str, Any] | None = None
|
|
36
|
+
|
|
37
|
+
RIGHTS_DISCLAIMER = (
|
|
38
|
+
"Embedded font name table entries (nameID 13/14) are reproduced verbatim "
|
|
39
|
+
"from font binaries and do not constitute independent rights verification "
|
|
40
|
+
"or legal clearance."
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def extract_font_license_records(path: str | Path, face_index: int = 0) -> tuple[str | None, str | None]:
|
|
45
|
+
"""Read OpenType name table nameID 13 (license description) and 14 (license URL).
|
|
46
|
+
|
|
47
|
+
Returns ``(license_description, license_url)``, with ``None`` for missing or unreadable records.
|
|
48
|
+
Embedded claims are never rights verification.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
desc: str | None = None
|
|
52
|
+
url: str | None = None
|
|
53
|
+
try:
|
|
54
|
+
with TTFont(str(path), fontNumber=face_index, lazy=True) as tt:
|
|
55
|
+
name_table = tt.get("name")
|
|
56
|
+
if name_table is not None:
|
|
57
|
+
for rec in name_table.names:
|
|
58
|
+
if rec.nameID == 13 and desc is None:
|
|
59
|
+
try:
|
|
60
|
+
desc = rec.toUnicode()
|
|
61
|
+
except Exception:
|
|
62
|
+
desc = rec.string.decode("latin1", errors="replace")
|
|
63
|
+
elif rec.nameID == 14 and url is None:
|
|
64
|
+
try:
|
|
65
|
+
url = rec.toUnicode()
|
|
66
|
+
except Exception:
|
|
67
|
+
url = rec.string.decode("latin1", errors="replace")
|
|
68
|
+
if desc is not None and url is not None:
|
|
69
|
+
break
|
|
70
|
+
except Exception as exc:
|
|
71
|
+
_LOGGER.debug("Could not inspect name table for font %s: %s", path, exc)
|
|
72
|
+
return None, None
|
|
73
|
+
|
|
74
|
+
return desc, url
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def compute_file_sha256(path: str | Path) -> str:
|
|
78
|
+
"""Compute and return the hex SHA-256 digest of *path*."""
|
|
79
|
+
digest = hashlib.sha256()
|
|
80
|
+
with Path(path).open("rb") as f:
|
|
81
|
+
for chunk in iter(lambda: f.read(65536), b""):
|
|
82
|
+
digest.update(chunk)
|
|
83
|
+
return digest.hexdigest()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def get_font_info(font_path: str | Path, face_index: int = 0) -> dict[str, Any]:
|
|
87
|
+
"""Return cached metadata for *font_path* including SHA-256 and embedded license records."""
|
|
88
|
+
resolved = str(Path(font_path).resolve())
|
|
89
|
+
key = (resolved, face_index)
|
|
90
|
+
cached = _FONT_INFO_CACHE.get(key)
|
|
91
|
+
if cached is not None:
|
|
92
|
+
return cached if cached["path"] == str(font_path) else {**cached, "path": str(font_path)}
|
|
93
|
+
|
|
94
|
+
sha256 = compute_file_sha256(resolved)
|
|
95
|
+
lic_13, lic_14 = extract_font_license_records(resolved, face_index)
|
|
96
|
+
|
|
97
|
+
info: dict[str, Any] = {
|
|
98
|
+
"path": str(font_path),
|
|
99
|
+
"resolved_path": resolved,
|
|
100
|
+
"face_index": face_index,
|
|
101
|
+
"sha256": sha256,
|
|
102
|
+
"license_13": lic_13,
|
|
103
|
+
"license_14": lic_14,
|
|
104
|
+
"license_description": lic_13,
|
|
105
|
+
"license_url": lic_14,
|
|
106
|
+
}
|
|
107
|
+
_FONT_INFO_CACHE[key] = info
|
|
108
|
+
return info
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def get_font_sha256(font_path: str | Path) -> str:
|
|
112
|
+
"""Compute and cache the SHA-256 hash of a font file."""
|
|
113
|
+
return str(get_font_info(font_path)["sha256"])
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def get_runtime_versions() -> dict[str, Any]:
|
|
117
|
+
"""Collect available runtime dependency versions once per process."""
|
|
118
|
+
global _RUNTIME_VERSIONS_CACHE
|
|
119
|
+
if _RUNTIME_VERSIONS_CACHE is not None:
|
|
120
|
+
return _RUNTIME_VERSIONS_CACHE
|
|
121
|
+
|
|
122
|
+
freetype_ver = features.version("freetype2")
|
|
123
|
+
raqm_ver = features.version("raqm")
|
|
124
|
+
harfbuzz_ver = features.version("harfbuzz")
|
|
125
|
+
fribidi_ver = features.version("fribidi")
|
|
126
|
+
|
|
127
|
+
versions: dict[str, Any] = {
|
|
128
|
+
"khocr_gen": __version__,
|
|
129
|
+
"python": sys.version.split()[0],
|
|
130
|
+
"pillow": PIL.__version__,
|
|
131
|
+
"unicode": unicodedata.unidata_version,
|
|
132
|
+
"freetype": freetype_ver,
|
|
133
|
+
"raqm": raqm_ver,
|
|
134
|
+
"harfbuzz": harfbuzz_ver,
|
|
135
|
+
"fribidi": fribidi_ver,
|
|
136
|
+
"opencv": cv2.__version__,
|
|
137
|
+
"numpy": np.__version__,
|
|
138
|
+
"fonttools": fontTools.__version__,
|
|
139
|
+
"rust_accel_available": _ra.HAS_RUST_ACCEL,
|
|
140
|
+
"rust_accel_version": getattr(_ra._native, "__version__", None) if _ra.HAS_RUST_ACCEL else None,
|
|
141
|
+
}
|
|
142
|
+
_RUNTIME_VERSIONS_CACHE = versions
|
|
143
|
+
return versions
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def compute_recipe_sha256(recipe_dict: dict[str, Any]) -> str:
|
|
147
|
+
"""Compute a canonical configuration SHA-256 (recipe identity, not code revision)."""
|
|
148
|
+
canonical_json = json.dumps(
|
|
149
|
+
recipe_dict,
|
|
150
|
+
sort_keys=True,
|
|
151
|
+
separators=(",", ":"),
|
|
152
|
+
ensure_ascii=True,
|
|
153
|
+
default=str,
|
|
154
|
+
)
|
|
155
|
+
return hashlib.sha256(canonical_json.encode("utf-8")).hexdigest()
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def build_sample_records(
|
|
159
|
+
*,
|
|
160
|
+
img_filename: str,
|
|
161
|
+
text: str | CorpusItem,
|
|
162
|
+
width: int,
|
|
163
|
+
height: int,
|
|
164
|
+
split_name: str,
|
|
165
|
+
meta: dict[str, Any],
|
|
166
|
+
default_language: str,
|
|
167
|
+
) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
|
168
|
+
"""Assemble execution metadata record and optional provenance sidecar row.
|
|
169
|
+
|
|
170
|
+
- ``metadata_record``: comprehensive technical execution record for ``metadata.jsonl``.
|
|
171
|
+
- ``provenance_row``: source manifest identity and attributes for ``provenance.jsonl``
|
|
172
|
+
(``None`` when *text* is a plain string rather than a :class:`CorpusItem`).
|
|
173
|
+
"""
|
|
174
|
+
primary_font = meta.get("font")
|
|
175
|
+
primary_sha = get_font_sha256(primary_font) if primary_font else None
|
|
176
|
+
|
|
177
|
+
# Collect all used fonts (clean, decorated, cluster fallback, mixed, positioned)
|
|
178
|
+
fonts_used = meta.get("fonts")
|
|
179
|
+
if not fonts_used and primary_font:
|
|
180
|
+
fonts_used = [primary_font]
|
|
181
|
+
elif not fonts_used:
|
|
182
|
+
fonts_used = []
|
|
183
|
+
|
|
184
|
+
used_font_records: list[dict[str, Any]] = []
|
|
185
|
+
seen_paths: set[str] = set()
|
|
186
|
+
for fpath in fonts_used:
|
|
187
|
+
if fpath and fpath not in seen_paths:
|
|
188
|
+
seen_paths.add(fpath)
|
|
189
|
+
used_font_records.append(get_font_info(fpath))
|
|
190
|
+
|
|
191
|
+
metadata_record: dict[str, Any] = {
|
|
192
|
+
"image": img_filename,
|
|
193
|
+
"text": str(text),
|
|
194
|
+
"width": int(width),
|
|
195
|
+
"height": int(height),
|
|
196
|
+
"font": primary_font,
|
|
197
|
+
"font_size": meta.get("font_size"),
|
|
198
|
+
"decorations": meta.get("decorations"),
|
|
199
|
+
"language": getattr(text, "language", None) or default_language,
|
|
200
|
+
"augmentation": meta.get("augmentation"),
|
|
201
|
+
"augmentation_intensity": meta.get("augmentation_intensity"),
|
|
202
|
+
"augmentation_reason": meta.get("augmentation_reason"),
|
|
203
|
+
"text_height": meta.get("text_height"),
|
|
204
|
+
"font_sha256": primary_sha,
|
|
205
|
+
"runtime_version": __version__,
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if used_font_records:
|
|
209
|
+
metadata_record["fonts"] = [rec["path"] for rec in used_font_records]
|
|
210
|
+
metadata_record["used_fonts"] = used_font_records
|
|
211
|
+
|
|
212
|
+
prov_row: dict[str, Any] | None = None
|
|
213
|
+
|
|
214
|
+
if isinstance(text, CorpusItem):
|
|
215
|
+
if text.source_id is not None:
|
|
216
|
+
metadata_record["source_id"] = text.source_id
|
|
217
|
+
if text.source_group is not None:
|
|
218
|
+
metadata_record["source_group"] = text.source_group
|
|
219
|
+
if text.language is not None:
|
|
220
|
+
metadata_record["language"] = text.language
|
|
221
|
+
|
|
222
|
+
for field_name in (
|
|
223
|
+
"script",
|
|
224
|
+
"domain",
|
|
225
|
+
"source_uri",
|
|
226
|
+
"license",
|
|
227
|
+
"retrieved_at",
|
|
228
|
+
"source_sha256",
|
|
229
|
+
"normalization_policy",
|
|
230
|
+
"consent_or_restrictions",
|
|
231
|
+
):
|
|
232
|
+
val = getattr(text, field_name, None)
|
|
233
|
+
if val is not None:
|
|
234
|
+
metadata_record[field_name] = val
|
|
235
|
+
|
|
236
|
+
raw_t = getattr(text, "raw_text", None)
|
|
237
|
+
if raw_t is not None and raw_t != str(text):
|
|
238
|
+
metadata_record["original_text"] = raw_t
|
|
239
|
+
|
|
240
|
+
prov_row = {
|
|
241
|
+
"image": img_filename,
|
|
242
|
+
"source_id": text.source_id,
|
|
243
|
+
"source_group": text.source_group,
|
|
244
|
+
"language": text.language,
|
|
245
|
+
"split": split_name,
|
|
246
|
+
}
|
|
247
|
+
for field_name in (
|
|
248
|
+
"script",
|
|
249
|
+
"domain",
|
|
250
|
+
"source_uri",
|
|
251
|
+
"license",
|
|
252
|
+
"retrieved_at",
|
|
253
|
+
"source_sha256",
|
|
254
|
+
"normalization_policy",
|
|
255
|
+
"consent_or_restrictions",
|
|
256
|
+
):
|
|
257
|
+
val = getattr(text, field_name, None)
|
|
258
|
+
if val is not None:
|
|
259
|
+
prov_row[field_name] = val
|
|
260
|
+
|
|
261
|
+
return metadata_record, prov_row
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def record_run_manifest(
|
|
265
|
+
output_dir: str | Path,
|
|
266
|
+
*,
|
|
267
|
+
config: Any,
|
|
268
|
+
invocation_overrides: dict[str, Any] | None = None,
|
|
269
|
+
available_fonts: list[dict[str, Any]] | None = None,
|
|
270
|
+
used_font_paths: list[str] | None = None,
|
|
271
|
+
ingestion: dict[str, Any] | None = None,
|
|
272
|
+
outcomes: dict[str, Any] | None = None,
|
|
273
|
+
split_name: str | None = None,
|
|
274
|
+
) -> dict[str, Any]:
|
|
275
|
+
"""Persist an append-safe run record into ``run_manifest.jsonl`` under *output_dir*."""
|
|
276
|
+
out_path = Path(output_dir)
|
|
277
|
+
out_path.mkdir(parents=True, exist_ok=True)
|
|
278
|
+
manifest_file = out_path / "run_manifest.jsonl"
|
|
279
|
+
|
|
280
|
+
cfg_dict = config.to_dict() if hasattr(config, "to_dict") else dict(config)
|
|
281
|
+
overrides = dict(invocation_overrides or {})
|
|
282
|
+
|
|
283
|
+
effective_recipe: dict[str, Any] = {
|
|
284
|
+
"config": cfg_dict,
|
|
285
|
+
"overrides": overrides,
|
|
286
|
+
}
|
|
287
|
+
recipe_sha = compute_recipe_sha256(effective_recipe)
|
|
288
|
+
|
|
289
|
+
sampling_policy: dict[str, Any] = {
|
|
290
|
+
"font_mode": overrides.get("font_mode", getattr(config, "font_mode", "random")),
|
|
291
|
+
"copies": overrides.get("copies", getattr(config, "copies", 3)),
|
|
292
|
+
"min_length": overrides.get("min_length", getattr(config, "min_length", 1)),
|
|
293
|
+
"max_length": overrides.get("max_length", getattr(config, "max_length", 260)),
|
|
294
|
+
"max_lines": overrides.get("max_lines", getattr(config, "max_lines", 0)),
|
|
295
|
+
"oversample_rare_chars": overrides.get("oversample_rare_chars", bool(overrides.get("rare_chars"))),
|
|
296
|
+
"rare_char_percentile": overrides.get("rare_char_percentile", getattr(config, "rare_char_percentile", 5.0)),
|
|
297
|
+
"rare_char_multiplier": overrides.get("rare_char_multiplier", getattr(config, "rare_char_multiplier", 3.0)),
|
|
298
|
+
"clean_prob": getattr(config, "clean_prob", 0.0),
|
|
299
|
+
"split_seed": overrides.get("split_seed", getattr(config, "seed", 42)),
|
|
300
|
+
"split_grouping": overrides.get("split_grouping", "source_group_or_exact_text"),
|
|
301
|
+
"seed_scope_note": (
|
|
302
|
+
"master_seed controls deterministic train/val/test corpus partitioning and "
|
|
303
|
+
"per-sample deterministic RNG seed derivation across worker configurations."
|
|
304
|
+
),
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
record: dict[str, Any] = {
|
|
308
|
+
"timestamp": datetime.datetime.now(datetime.UTC).isoformat(),
|
|
309
|
+
"recipe_sha256": recipe_sha,
|
|
310
|
+
"package_version": __version__,
|
|
311
|
+
"runtime": get_runtime_versions(),
|
|
312
|
+
"sampling_policy": sampling_policy,
|
|
313
|
+
"config": cfg_dict,
|
|
314
|
+
"overrides": overrides,
|
|
315
|
+
"rights_disclaimer": RIGHTS_DISCLAIMER,
|
|
316
|
+
}
|
|
317
|
+
if split_name is not None:
|
|
318
|
+
record["split"] = split_name
|
|
319
|
+
record["available_fonts"] = available_fonts or []
|
|
320
|
+
record["used_fonts"] = [get_font_info(path) for path in used_font_paths or []]
|
|
321
|
+
record["ingestion"] = ingestion or {}
|
|
322
|
+
record["outcomes"] = outcomes or {}
|
|
323
|
+
|
|
324
|
+
line = json.dumps(record, ensure_ascii=False, default=str) + "\n"
|
|
325
|
+
with manifest_file.open("a", encoding="utf-8") as f:
|
|
326
|
+
f.write(line)
|
|
327
|
+
|
|
328
|
+
return record
|