ocr-foundry 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
khocr_gen/parallel.py ADDED
@@ -0,0 +1,366 @@
1
+ """Parallel data generation workers.
2
+
3
+ Each worker process renders text lines from a corpus
4
+ and applies exactly one augmentation method per rendered image (isolated augmentation model).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import ast
10
+ import contextlib
11
+ import hashlib
12
+ import json
13
+ import logging
14
+ import os
15
+ import random
16
+ import sys
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ import cv2
21
+ import numpy as np
22
+
23
+ from .augmentation import write_output_image
24
+ from .config import AugMethodConfig, GenerationConfig
25
+ from .fonts import FontManager
26
+ from .provenance import build_sample_records
27
+ from .rendering import ImageRenderer
28
+ from .text_support import detect_script
29
+
30
+ _LOGGER = logging.getLogger("khocr_gen.parallel")
31
+ _AUTO_PARALLEL_MIN_SAMPLES = 2_048
32
+ _DEFAULT_RENDER_CHUNK_SIZE = 64
33
+ _MAX_RENDER_CHUNK_SIZE = 256
34
+
35
+
36
+ def derive_sample_seed(
37
+ master_seed: int,
38
+ sample_index: int,
39
+ sample_id: str | int,
40
+ ) -> int:
41
+ """Deterministically derive a 64-bit integer seed for a single sample.
42
+
43
+ Stable hashing (SHA-256) over (master_seed, sample_index, sample_id) ensures
44
+ the derived seed is independent of worker allocation, chunking, and scheduling.
45
+ """
46
+ key = f"khocr-sample-seed/{master_seed}/{sample_index}/{sample_id}".encode()
47
+ digest = hashlib.sha256(key).digest()
48
+ return int.from_bytes(digest[:8], byteorder="big", signed=False)
49
+
50
+
51
+ def seed_sample_generators(
52
+ master_seed: int | None,
53
+ sample_index: int,
54
+ sample_id: str | int,
55
+ ) -> int | None:
56
+ """Seed Python and NumPy random generators for a single sample if master_seed is set.
57
+
58
+ Returns the derived seed when active, or None when master_seed is None.
59
+ """
60
+ if master_seed is None:
61
+ return None
62
+ sample_seed = derive_sample_seed(master_seed, sample_index, sample_id)
63
+ random.seed(sample_seed)
64
+ np.random.seed(sample_seed % (2**32))
65
+ return sample_seed
66
+
67
+ def resolve_worker_count(workers: int, sample_count: int) -> int:
68
+ """Determine how many worker processes to use.
69
+
70
+ Returns 1 when auto-resolve decides parallelism isn't worthwhile.
71
+ """
72
+ requested = max(0, int(workers))
73
+ cpu_count = os.cpu_count() or 1
74
+
75
+ if sample_count <= 0 or cpu_count <= 1:
76
+ return 1
77
+ if requested == 1:
78
+ return 1
79
+ if requested > 1:
80
+ return min(requested, cpu_count)
81
+ if sample_count < _AUTO_PARALLEL_MIN_SAMPLES:
82
+ return 1
83
+ return max(1, cpu_count // 2)
84
+
85
+
86
+ def resolve_chunk_size(sample_count: int, workers: int) -> int:
87
+ """Pick a batch size that balances load across workers."""
88
+ if sample_count <= 0:
89
+ return _DEFAULT_RENDER_CHUNK_SIZE
90
+ if workers <= 1:
91
+ return _DEFAULT_RENDER_CHUNK_SIZE
92
+
93
+ target = sample_count // max(1, workers * 32)
94
+ return max(16, min(_MAX_RENDER_CHUNK_SIZE, max(_DEFAULT_RENDER_CHUNK_SIZE, target)))
95
+
96
+
97
+ def resolve_mp_context() -> tuple[Any, str]:
98
+ """Pick a multiprocessing context that stays safe after other threads exist.
99
+
100
+ ``ProcessPoolExecutor`` starts workers from its own thread. ``fork`` from
101
+ that thread can deadlock once the parent holds locks from Pillow, OpenCV,
102
+ or coverage. Linux therefore uses ``forkserver``: a fresh single-threaded
103
+ server forks the workers, without re-importing the pipeline for each one.
104
+ macOS / Windows use ``spawn``. Override with ``KHOCR_GEN_MP_START_METHOD``
105
+ (``forkserver``, ``fork``, or ``spawn``).
106
+ """
107
+ import multiprocessing as mp
108
+
109
+ requested = os.environ.get("KHOCR_GEN_MP_START_METHOD", "").strip().lower()
110
+ available = set(mp.get_all_start_methods())
111
+
112
+ if requested:
113
+ if requested in available:
114
+ return mp.get_context(requested), requested
115
+ return mp.get_context("spawn"), "spawn"
116
+
117
+ if sys.platform == "linux" and "forkserver" in available:
118
+ return mp.get_context("forkserver"), "forkserver"
119
+
120
+ return mp.get_context("spawn"), "spawn"
121
+
122
+
123
+ def _is_dunder_main_comparison(test: ast.expr) -> bool:
124
+ """True when *test* is exactly ``__name__ == "__main__"`` (either order)."""
125
+ if not isinstance(test, ast.Compare) or len(test.ops) != 1 or len(test.comparators) != 1:
126
+ return False
127
+ if not isinstance(test.ops[0], ast.Eq):
128
+ return False
129
+ left, right = test.left, test.comparators[0]
130
+ pair = (
131
+ (
132
+ isinstance(left, ast.Name)
133
+ and left.id == "__name__"
134
+ and isinstance(right, ast.Constant)
135
+ and right.value == "__main__"
136
+ ),
137
+ (
138
+ isinstance(right, ast.Name)
139
+ and right.id == "__name__"
140
+ and isinstance(left, ast.Constant)
141
+ and left.value == "__main__"
142
+ ),
143
+ )
144
+ return any(pair)
145
+
146
+
147
+ def _has_executable_main_guard(source: str) -> bool:
148
+ """True only for a module-level guard that runs on import.
149
+
150
+ A textual match is not enough. The ``if`` must be a direct module
151
+ statement, so a quoted copy, a function body, or a dead ``if False:``
152
+ branch cannot hide the re-import.
153
+ """
154
+ try:
155
+ tree = ast.parse(source)
156
+ except SyntaxError:
157
+ return False
158
+ return any(
159
+ isinstance(node, ast.If) and _is_dunder_main_comparison(node.test) for node in tree.body
160
+ )
161
+
162
+
163
+ def ensure_worker_main_guard(start_method: str, main_module: Any | None = None) -> None:
164
+ """Refuse forkserver/spawn when importing ``__main__`` would start another pool.
165
+
166
+ An unguarded script re-executes on worker startup. The child then raises
167
+ ``RuntimeError`` during bootstrap, and the parent only sees
168
+ ``BrokenProcessPool``. ``fork`` does not re-import ``__main__`` and stays
169
+ available through ``KHOCR_GEN_MP_START_METHOD``. ``python -c`` and
170
+ interactive sessions have no script file to re-import. The guard is parsed
171
+ from the AST: it must be an executable module-level comparison, not text
172
+ inside a string or a branch that does not run.
173
+ """
174
+ if start_method == "fork":
175
+ return
176
+ if main_module is None:
177
+ main_module = sys.modules.get("__main__")
178
+ if main_module is None:
179
+ return
180
+ main_file = getattr(main_module, "__file__", None)
181
+ if not main_file:
182
+ return
183
+ try:
184
+ source = Path(main_file).read_text(encoding="utf-8")
185
+ except (OSError, UnicodeError):
186
+ return
187
+ if _has_executable_main_guard(source):
188
+ return
189
+ raise RuntimeError(
190
+ f"Refusing to start {start_method} workers because {main_file} has no "
191
+ '`if __name__ == "__main__":` guard. Wrap generation in that guard so '
192
+ "workers can import the script without starting another pool. Without "
193
+ "it, the child dies during bootstrap and the parent only reports "
194
+ "BrokenProcessPool. Set KHOCR_GEN_MP_START_METHOD=fork to keep the "
195
+ "previous start method when the parent is still single-threaded."
196
+ )
197
+
198
+
199
+ # Per-process worker state
200
+ _WORKER_STATE: dict[str, Any] = {}
201
+
202
+
203
+ def _init_render_worker(config_dict: dict[str, Any]) -> None:
204
+ """Initialise a worker process with its own renderer and config."""
205
+ global _WORKER_STATE
206
+
207
+ # Limit OpenCV threads in worker processes
208
+ with contextlib.suppress(Exception):
209
+ cv2.setNumThreads(1)
210
+ with contextlib.suppress(Exception):
211
+ cv2.ocl.setUseOpenCL(False)
212
+
213
+ # Derive a per-worker seed from os.urandom
214
+ seed = int.from_bytes(os.urandom(8), "little") ^ (os.getpid() << 16)
215
+ random.seed(seed)
216
+ np.random.seed(seed % (2**32))
217
+
218
+ # Reconstruct GenerationConfig from serialised dict
219
+ gen_cfg = GenerationConfig.from_dict(config_dict)
220
+
221
+ font_manager = FontManager(
222
+ language=gen_cfg.language,
223
+ fonts_dir=gen_cfg.fonts_dir,
224
+ verbose=False,
225
+ multilingual=gen_cfg.multilingual,
226
+ font_pool=gen_cfg.font_pool,
227
+ )
228
+ renderer = ImageRenderer(font_manager, gen_cfg)
229
+
230
+ _WORKER_STATE = {
231
+ "renderer": renderer,
232
+ "split_name": config_dict["split_name"],
233
+ "output_images_dir": config_dict["output_images_dir"],
234
+ "font_mode": config_dict["font_mode"],
235
+ "retry_limit": config_dict["retry_limit"],
236
+ "image_dir": config_dict.get("image_dir"),
237
+ "output_format": config_dict.get("output_format", "jpg"),
238
+ "jpeg_quality": config_dict.get("jpeg_quality", 90),
239
+ "storage": config_dict.get("storage", "raw"),
240
+ "record_metadata": config_dict.get("record_metadata", False),
241
+ "seed": gen_cfg.seed,
242
+ }
243
+
244
+
245
+ def _render_sample_batch(
246
+ batch: list[tuple[int, str, Any]],
247
+ ) -> tuple[list[str], int, int, list[str], list[str], list[str], list[int]]:
248
+ """Render a batch of samples in the worker process.
249
+
250
+ Each sample is rendered to a clean canvas,
251
+ then exactly one augmentation method is applied (chosen probabilistically from the enabled methods).
252
+
253
+ Returns:
254
+ (labels_lines, attempted_count, success_count, error_messages, metadata_lines,
255
+ provenance_lines, heights)
256
+ """
257
+ renderer: ImageRenderer = _WORKER_STATE["renderer"]
258
+ split_name: str = _WORKER_STATE["split_name"]
259
+ output_images_dir = Path(_WORKER_STATE["output_images_dir"])
260
+ font_mode: str = _WORKER_STATE["font_mode"]
261
+ retry_limit: int = _WORKER_STATE["retry_limit"]
262
+ record_metadata: bool = _WORKER_STATE.get("record_metadata", False)
263
+ current_retry_limit = 1 if font_mode == "all" else retry_limit
264
+
265
+ labels: list[str] = []
266
+ meta_lines: list[str] = []
267
+ prov_lines: list[str] = []
268
+ heights: list[int] = []
269
+ error_messages: list[str] = []
270
+ success_count = 0
271
+
272
+ image_dir = _WORKER_STATE.get("image_dir")
273
+
274
+ # Get enabled augmentation methods once per batch
275
+ enabled_methods: list[tuple[str, AugMethodConfig]] = renderer._cfg.enabled_aug_methods()
276
+
277
+ master_seed = _WORKER_STATE.get("seed")
278
+ for current_idx, text, font_ref_or_image in batch:
279
+ sample_id = f"{split_name}_{current_idx:06d}"
280
+ seed_sample_generators(master_seed, current_idx, sample_id)
281
+ try:
282
+ meta: dict[str, Any] = {}
283
+ if image_dir:
284
+ # Image-dir mode: augment an existing image
285
+ img_path = Path(image_dir) / font_ref_or_image
286
+ img_array = cv2.imread(str(img_path))
287
+ if img_array is None:
288
+ raise RuntimeError(f"Could not read image: {img_path}")
289
+
290
+ if img_array.ndim == 3 and img_array.shape[2] == 3:
291
+ img_array = cv2.cvtColor(img_array, cv2.COLOR_BGR2RGB)
292
+
293
+ # Apply one augmentation probabilistically
294
+ result = renderer.render_with_one_augmentation(
295
+ text,
296
+ enabled_methods,
297
+ specific_font=font_ref_or_image,
298
+ retry_limit=current_retry_limit,
299
+ )
300
+ if result is None:
301
+ # Fallback: augment existing image directly
302
+ aug_results = renderer.augment_image(
303
+ img_array, enabled_methods, script=detect_script(text)
304
+ )
305
+ if not aug_results:
306
+ continue
307
+ _, img = aug_results[0]
308
+ else:
309
+ _, img, meta = result
310
+ else:
311
+ # Text-rendering mode
312
+ result = renderer.render_with_one_augmentation(
313
+ text,
314
+ enabled_methods,
315
+ specific_font=font_ref_or_image,
316
+ retry_limit=current_retry_limit,
317
+ )
318
+ if result is None:
319
+ continue
320
+ _, img, meta = result
321
+
322
+ if img is None:
323
+ continue
324
+
325
+ output_fmt = _WORKER_STATE.get("output_format", "jpg")
326
+ jpeg_q = _WORKER_STATE.get("jpeg_quality", 90)
327
+ img_filename = f"{split_name}_{current_idx:06d}.{output_fmt}"
328
+ img_path = output_images_dir / img_filename
329
+ write_ok = write_output_image(img_path, img, fmt=output_fmt, jpeg_quality=jpeg_q)
330
+ if not write_ok:
331
+ raise RuntimeError(f"cv2.imwrite failed for output path: {img_path}")
332
+
333
+ labels.append(f"{img_filename}\t{text}\n")
334
+ heights.append(int(img.shape[0]))
335
+ if record_metadata:
336
+ record, prov_row = build_sample_records(
337
+ img_filename=img_filename,
338
+ text=text,
339
+ width=img.shape[1],
340
+ height=img.shape[0],
341
+ split_name=split_name,
342
+ meta=meta,
343
+ default_language=renderer._cfg.language,
344
+ )
345
+ if prov_row is not None:
346
+ prov_lines.append(json.dumps(prov_row, ensure_ascii=False) + "\n")
347
+ meta_lines.append(json.dumps(record, ensure_ascii=False) + "\n")
348
+ else:
349
+ paths = [path for path in meta.get("fonts", [meta.get("font")]) if path]
350
+ if paths:
351
+ meta_lines.append(json.dumps({"fonts": paths}) + "\n")
352
+ success_count += 1
353
+ except Exception as exc:
354
+ if len(error_messages) < 3:
355
+ short_text = text[:30] if len(text) > 30 else text
356
+ error_messages.append(f" Failed for '{short_text}...': {exc}")
357
+
358
+ return (
359
+ labels,
360
+ len(batch),
361
+ success_count,
362
+ error_messages,
363
+ meta_lines,
364
+ prov_lines,
365
+ heights,
366
+ )
@@ -0,0 +1,328 @@
1
+ """Shared provenance, font licensing, run manifests, and execution metadata.
2
+
3
+ Extracts duplicated metadata and provenance sidecar record construction
4
+ between serial and parallel workers, caches font hashes and embedded
5
+ license name records (nameID 13/14), captures runtime software versions,
6
+ and persists run manifests with recipe identity.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import datetime
12
+ import hashlib
13
+ import json
14
+ import logging
15
+ import sys
16
+ import unicodedata
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ import cv2
21
+ import fontTools
22
+ import numpy as np
23
+ import PIL
24
+ from fontTools.ttLib import TTFont
25
+ from PIL import features
26
+
27
+ from . import __version__
28
+ from . import _rust_accel as _ra
29
+ from .corpus import CorpusItem
30
+
31
+ _LOGGER = logging.getLogger("khocr_gen.provenance")
32
+
33
+ # Process-level immutable caches
34
+ _FONT_INFO_CACHE: dict[tuple[str, int], dict[str, Any]] = {}
35
+ _RUNTIME_VERSIONS_CACHE: dict[str, Any] | None = None
36
+
37
+ RIGHTS_DISCLAIMER = (
38
+ "Embedded font name table entries (nameID 13/14) are reproduced verbatim "
39
+ "from font binaries and do not constitute independent rights verification "
40
+ "or legal clearance."
41
+ )
42
+
43
+
44
+ def extract_font_license_records(path: str | Path, face_index: int = 0) -> tuple[str | None, str | None]:
45
+ """Read OpenType name table nameID 13 (license description) and 14 (license URL).
46
+
47
+ Returns ``(license_description, license_url)``, with ``None`` for missing or unreadable records.
48
+ Embedded claims are never rights verification.
49
+ """
50
+
51
+ desc: str | None = None
52
+ url: str | None = None
53
+ try:
54
+ with TTFont(str(path), fontNumber=face_index, lazy=True) as tt:
55
+ name_table = tt.get("name")
56
+ if name_table is not None:
57
+ for rec in name_table.names:
58
+ if rec.nameID == 13 and desc is None:
59
+ try:
60
+ desc = rec.toUnicode()
61
+ except Exception:
62
+ desc = rec.string.decode("latin1", errors="replace")
63
+ elif rec.nameID == 14 and url is None:
64
+ try:
65
+ url = rec.toUnicode()
66
+ except Exception:
67
+ url = rec.string.decode("latin1", errors="replace")
68
+ if desc is not None and url is not None:
69
+ break
70
+ except Exception as exc:
71
+ _LOGGER.debug("Could not inspect name table for font %s: %s", path, exc)
72
+ return None, None
73
+
74
+ return desc, url
75
+
76
+
77
+ def compute_file_sha256(path: str | Path) -> str:
78
+ """Compute and return the hex SHA-256 digest of *path*."""
79
+ digest = hashlib.sha256()
80
+ with Path(path).open("rb") as f:
81
+ for chunk in iter(lambda: f.read(65536), b""):
82
+ digest.update(chunk)
83
+ return digest.hexdigest()
84
+
85
+
86
+ def get_font_info(font_path: str | Path, face_index: int = 0) -> dict[str, Any]:
87
+ """Return cached metadata for *font_path* including SHA-256 and embedded license records."""
88
+ resolved = str(Path(font_path).resolve())
89
+ key = (resolved, face_index)
90
+ cached = _FONT_INFO_CACHE.get(key)
91
+ if cached is not None:
92
+ return cached if cached["path"] == str(font_path) else {**cached, "path": str(font_path)}
93
+
94
+ sha256 = compute_file_sha256(resolved)
95
+ lic_13, lic_14 = extract_font_license_records(resolved, face_index)
96
+
97
+ info: dict[str, Any] = {
98
+ "path": str(font_path),
99
+ "resolved_path": resolved,
100
+ "face_index": face_index,
101
+ "sha256": sha256,
102
+ "license_13": lic_13,
103
+ "license_14": lic_14,
104
+ "license_description": lic_13,
105
+ "license_url": lic_14,
106
+ }
107
+ _FONT_INFO_CACHE[key] = info
108
+ return info
109
+
110
+
111
+ def get_font_sha256(font_path: str | Path) -> str:
112
+ """Compute and cache the SHA-256 hash of a font file."""
113
+ return str(get_font_info(font_path)["sha256"])
114
+
115
+
116
+ def get_runtime_versions() -> dict[str, Any]:
117
+ """Collect available runtime dependency versions once per process."""
118
+ global _RUNTIME_VERSIONS_CACHE
119
+ if _RUNTIME_VERSIONS_CACHE is not None:
120
+ return _RUNTIME_VERSIONS_CACHE
121
+
122
+ freetype_ver = features.version("freetype2")
123
+ raqm_ver = features.version("raqm")
124
+ harfbuzz_ver = features.version("harfbuzz")
125
+ fribidi_ver = features.version("fribidi")
126
+
127
+ versions: dict[str, Any] = {
128
+ "khocr_gen": __version__,
129
+ "python": sys.version.split()[0],
130
+ "pillow": PIL.__version__,
131
+ "unicode": unicodedata.unidata_version,
132
+ "freetype": freetype_ver,
133
+ "raqm": raqm_ver,
134
+ "harfbuzz": harfbuzz_ver,
135
+ "fribidi": fribidi_ver,
136
+ "opencv": cv2.__version__,
137
+ "numpy": np.__version__,
138
+ "fonttools": fontTools.__version__,
139
+ "rust_accel_available": _ra.HAS_RUST_ACCEL,
140
+ "rust_accel_version": getattr(_ra._native, "__version__", None) if _ra.HAS_RUST_ACCEL else None,
141
+ }
142
+ _RUNTIME_VERSIONS_CACHE = versions
143
+ return versions
144
+
145
+
146
+ def compute_recipe_sha256(recipe_dict: dict[str, Any]) -> str:
147
+ """Compute a canonical configuration SHA-256 (recipe identity, not code revision)."""
148
+ canonical_json = json.dumps(
149
+ recipe_dict,
150
+ sort_keys=True,
151
+ separators=(",", ":"),
152
+ ensure_ascii=True,
153
+ default=str,
154
+ )
155
+ return hashlib.sha256(canonical_json.encode("utf-8")).hexdigest()
156
+
157
+
158
+ def build_sample_records(
159
+ *,
160
+ img_filename: str,
161
+ text: str | CorpusItem,
162
+ width: int,
163
+ height: int,
164
+ split_name: str,
165
+ meta: dict[str, Any],
166
+ default_language: str,
167
+ ) -> tuple[dict[str, Any], dict[str, Any] | None]:
168
+ """Assemble execution metadata record and optional provenance sidecar row.
169
+
170
+ - ``metadata_record``: comprehensive technical execution record for ``metadata.jsonl``.
171
+ - ``provenance_row``: source manifest identity and attributes for ``provenance.jsonl``
172
+ (``None`` when *text* is a plain string rather than a :class:`CorpusItem`).
173
+ """
174
+ primary_font = meta.get("font")
175
+ primary_sha = get_font_sha256(primary_font) if primary_font else None
176
+
177
+ # Collect all used fonts (clean, decorated, cluster fallback, mixed, positioned)
178
+ fonts_used = meta.get("fonts")
179
+ if not fonts_used and primary_font:
180
+ fonts_used = [primary_font]
181
+ elif not fonts_used:
182
+ fonts_used = []
183
+
184
+ used_font_records: list[dict[str, Any]] = []
185
+ seen_paths: set[str] = set()
186
+ for fpath in fonts_used:
187
+ if fpath and fpath not in seen_paths:
188
+ seen_paths.add(fpath)
189
+ used_font_records.append(get_font_info(fpath))
190
+
191
+ metadata_record: dict[str, Any] = {
192
+ "image": img_filename,
193
+ "text": str(text),
194
+ "width": int(width),
195
+ "height": int(height),
196
+ "font": primary_font,
197
+ "font_size": meta.get("font_size"),
198
+ "decorations": meta.get("decorations"),
199
+ "language": getattr(text, "language", None) or default_language,
200
+ "augmentation": meta.get("augmentation"),
201
+ "augmentation_intensity": meta.get("augmentation_intensity"),
202
+ "augmentation_reason": meta.get("augmentation_reason"),
203
+ "text_height": meta.get("text_height"),
204
+ "font_sha256": primary_sha,
205
+ "runtime_version": __version__,
206
+ }
207
+
208
+ if used_font_records:
209
+ metadata_record["fonts"] = [rec["path"] for rec in used_font_records]
210
+ metadata_record["used_fonts"] = used_font_records
211
+
212
+ prov_row: dict[str, Any] | None = None
213
+
214
+ if isinstance(text, CorpusItem):
215
+ if text.source_id is not None:
216
+ metadata_record["source_id"] = text.source_id
217
+ if text.source_group is not None:
218
+ metadata_record["source_group"] = text.source_group
219
+ if text.language is not None:
220
+ metadata_record["language"] = text.language
221
+
222
+ for field_name in (
223
+ "script",
224
+ "domain",
225
+ "source_uri",
226
+ "license",
227
+ "retrieved_at",
228
+ "source_sha256",
229
+ "normalization_policy",
230
+ "consent_or_restrictions",
231
+ ):
232
+ val = getattr(text, field_name, None)
233
+ if val is not None:
234
+ metadata_record[field_name] = val
235
+
236
+ raw_t = getattr(text, "raw_text", None)
237
+ if raw_t is not None and raw_t != str(text):
238
+ metadata_record["original_text"] = raw_t
239
+
240
+ prov_row = {
241
+ "image": img_filename,
242
+ "source_id": text.source_id,
243
+ "source_group": text.source_group,
244
+ "language": text.language,
245
+ "split": split_name,
246
+ }
247
+ for field_name in (
248
+ "script",
249
+ "domain",
250
+ "source_uri",
251
+ "license",
252
+ "retrieved_at",
253
+ "source_sha256",
254
+ "normalization_policy",
255
+ "consent_or_restrictions",
256
+ ):
257
+ val = getattr(text, field_name, None)
258
+ if val is not None:
259
+ prov_row[field_name] = val
260
+
261
+ return metadata_record, prov_row
262
+
263
+
264
+ def record_run_manifest(
265
+ output_dir: str | Path,
266
+ *,
267
+ config: Any,
268
+ invocation_overrides: dict[str, Any] | None = None,
269
+ available_fonts: list[dict[str, Any]] | None = None,
270
+ used_font_paths: list[str] | None = None,
271
+ ingestion: dict[str, Any] | None = None,
272
+ outcomes: dict[str, Any] | None = None,
273
+ split_name: str | None = None,
274
+ ) -> dict[str, Any]:
275
+ """Persist an append-safe run record into ``run_manifest.jsonl`` under *output_dir*."""
276
+ out_path = Path(output_dir)
277
+ out_path.mkdir(parents=True, exist_ok=True)
278
+ manifest_file = out_path / "run_manifest.jsonl"
279
+
280
+ cfg_dict = config.to_dict() if hasattr(config, "to_dict") else dict(config)
281
+ overrides = dict(invocation_overrides or {})
282
+
283
+ effective_recipe: dict[str, Any] = {
284
+ "config": cfg_dict,
285
+ "overrides": overrides,
286
+ }
287
+ recipe_sha = compute_recipe_sha256(effective_recipe)
288
+
289
+ sampling_policy: dict[str, Any] = {
290
+ "font_mode": overrides.get("font_mode", getattr(config, "font_mode", "random")),
291
+ "copies": overrides.get("copies", getattr(config, "copies", 3)),
292
+ "min_length": overrides.get("min_length", getattr(config, "min_length", 1)),
293
+ "max_length": overrides.get("max_length", getattr(config, "max_length", 260)),
294
+ "max_lines": overrides.get("max_lines", getattr(config, "max_lines", 0)),
295
+ "oversample_rare_chars": overrides.get("oversample_rare_chars", bool(overrides.get("rare_chars"))),
296
+ "rare_char_percentile": overrides.get("rare_char_percentile", getattr(config, "rare_char_percentile", 5.0)),
297
+ "rare_char_multiplier": overrides.get("rare_char_multiplier", getattr(config, "rare_char_multiplier", 3.0)),
298
+ "clean_prob": getattr(config, "clean_prob", 0.0),
299
+ "split_seed": overrides.get("split_seed", getattr(config, "seed", 42)),
300
+ "split_grouping": overrides.get("split_grouping", "source_group_or_exact_text"),
301
+ "seed_scope_note": (
302
+ "master_seed controls deterministic train/val/test corpus partitioning and "
303
+ "per-sample deterministic RNG seed derivation across worker configurations."
304
+ ),
305
+ }
306
+
307
+ record: dict[str, Any] = {
308
+ "timestamp": datetime.datetime.now(datetime.UTC).isoformat(),
309
+ "recipe_sha256": recipe_sha,
310
+ "package_version": __version__,
311
+ "runtime": get_runtime_versions(),
312
+ "sampling_policy": sampling_policy,
313
+ "config": cfg_dict,
314
+ "overrides": overrides,
315
+ "rights_disclaimer": RIGHTS_DISCLAIMER,
316
+ }
317
+ if split_name is not None:
318
+ record["split"] = split_name
319
+ record["available_fonts"] = available_fonts or []
320
+ record["used_fonts"] = [get_font_info(path) for path in used_font_paths or []]
321
+ record["ingestion"] = ingestion or {}
322
+ record["outcomes"] = outcomes or {}
323
+
324
+ line = json.dumps(record, ensure_ascii=False, default=str) + "\n"
325
+ with manifest_file.open("a", encoding="utf-8") as f:
326
+ f.write(line)
327
+
328
+ return record