ocr-foundry 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
khocr_gen/config.py ADDED
@@ -0,0 +1,1474 @@
1
+ """Generation configuration - single source of truth for all generation parameters.
2
+
3
+ Each augmentation method has three tunable knobs exposed to CLI and config YAML:
4
+
5
+ * `prob`: probability [0, 1] of applying this augmentation to a given text line
6
+ * `min` : minimum intensity level [0, 1] (normalised; mapped to physical units internally)
7
+ * `max` : maximum intensity level [0, 1]
8
+
9
+ One exception: `extreme_resize`'s `min`/`max` are absolute pixel heights
10
+ (see `ExtremeResizeConfig`), not normalised [0, 1] values.
11
+
12
+ The augmentation system is *isolated*:
13
+ a single generated image receives exactly one augmentation method on a clean rendered canvas (no stacked effects).
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import math
19
+ from dataclasses import dataclass, field, fields
20
+ from typing import TYPE_CHECKING, Any, ClassVar
21
+
22
+ from .errors import InputValidationError
23
+ from .normalizer import NormalizerConfig
24
+
25
+ if TYPE_CHECKING:
26
+ import argparse
27
+
28
+ # Per-method augmentation config
29
+
30
+
31
+ @dataclass
32
+ class AugMethodConfig:
33
+ """Probability and intensity range for a single augmentation method."""
34
+
35
+ prob: float = 0.0
36
+ min: float = 0.0
37
+ max: float = 1.0
38
+
39
+ def __post_init__(self) -> None:
40
+ self.prob = float(max(0.0, min(1.0, self.prob)))
41
+ self.min = float(max(0.0, min(1.0, self.min)))
42
+ self.max = float(max(self.min, min(1.0, self.max)))
43
+
44
+ def sample_intensity(self) -> float:
45
+ """Draw a random intensity in [min, max]."""
46
+ if self.min >= self.max:
47
+ return self.min
48
+ import random
49
+
50
+ return random.uniform(self.min, self.max)
51
+
52
+ @property
53
+ def enabled(self) -> bool:
54
+ return self.prob > 0.0
55
+
56
+
57
+ @dataclass
58
+ class ExtremeResizeConfig(AugMethodConfig):
59
+ """Probability and height range for the `extreme_resize` augmentation.
60
+
61
+ Same `{prob, min, max}` shape as `AugMethodConfig`, but here `min`/`max`
62
+ are absolute target heights in *pixels* (not normalised to [0, 1]): the
63
+ augmentation isotropically resizes the clean canvas down to as small as
64
+ `min` or up to as large as `max` pixels tall before the pipeline resizes
65
+ it back to the output line height, simulating an extreme source
66
+ resolution (e.g. `min=8` for a tiny/pixelated source, `max=1920` for a
67
+ huge scan later shrunk down).
68
+ """
69
+
70
+ min: float = 8.0
71
+ max: float = 1920.0
72
+
73
+ def __post_init__(self) -> None:
74
+ self.prob = float(max(0.0, min(1.0, self.prob)))
75
+ self.min = float(max(1.0, self.min))
76
+ self.max = float(max(self.min, self.max))
77
+
78
+
79
+ # Per-line text decoration config
80
+
81
+
82
+ @dataclass
83
+ class TextDecorationConfig:
84
+ """Probabilities for per-line text decorations applied at render time.
85
+
86
+ Decorations are rendering attributes (not augmentations): each image
87
+ independently samples each decoration, so they can combine (e.g. bold +
88
+ underline). All probabilities default to 0.0 (feature off).
89
+ """
90
+
91
+ color_prob: float = 0.0 # whole-line single random color (RGB mode only)
92
+ underline_prob: float = 0.0 # underline under the whole line
93
+ subscript_prob: float = 0.0 # lower 1-2 random ASCII chars
94
+ superscript_prob: float = 0.0 # raise 1-2 random ASCII chars
95
+ italic_prob: float = 0.0 # real italic/oblique variant font (no-op if absent)
96
+ bold_prob: float = 0.0 # real bold variant font (no-op if absent)
97
+
98
+ def __post_init__(self) -> None:
99
+ self.color_prob = float(max(0.0, min(1.0, self.color_prob)))
100
+ self.underline_prob = float(max(0.0, min(1.0, self.underline_prob)))
101
+ self.subscript_prob = float(max(0.0, min(1.0, self.subscript_prob)))
102
+ self.superscript_prob = float(max(0.0, min(1.0, self.superscript_prob)))
103
+ self.italic_prob = float(max(0.0, min(1.0, self.italic_prob)))
104
+ self.bold_prob = float(max(0.0, min(1.0, self.bold_prob)))
105
+
106
+ @property
107
+ def enabled(self) -> bool:
108
+ return any(
109
+ (
110
+ self.color_prob > 0.0,
111
+ self.underline_prob > 0.0,
112
+ self.subscript_prob > 0.0,
113
+ self.superscript_prob > 0.0,
114
+ self.italic_prob > 0.0,
115
+ self.bold_prob > 0.0,
116
+ )
117
+ )
118
+
119
+ @staticmethod
120
+ def add_args(parser: argparse.ArgumentParser) -> None:
121
+ g = parser.add_argument_group(
122
+ "Text decoration",
123
+ "Per-line text decorations applied at render time (can combine). "
124
+ "Bold/italic require a matching variant font; random color requires --color-mode 3.",
125
+ )
126
+ g.add_argument(
127
+ "--text-deco-color-prob",
128
+ type=float,
129
+ default=None,
130
+ metavar="F",
131
+ help="Probability of a single random text color for the whole line (default: 0)",
132
+ )
133
+ g.add_argument(
134
+ "--text-deco-underline-prob",
135
+ type=float,
136
+ default=None,
137
+ metavar="F",
138
+ help="Probability of underlining the line (default: 0)",
139
+ )
140
+ g.add_argument(
141
+ "--text-deco-subscript-prob",
142
+ type=float,
143
+ default=None,
144
+ metavar="F",
145
+ help="Probability of lowering 1-2 random ASCII chars as subscript (default: 0)",
146
+ )
147
+ g.add_argument(
148
+ "--text-deco-superscript-prob",
149
+ type=float,
150
+ default=None,
151
+ metavar="F",
152
+ help="Probability of raising 1-2 random ASCII chars as superscript (default: 0)",
153
+ )
154
+ g.add_argument(
155
+ "--text-deco-italic-prob",
156
+ type=float,
157
+ default=None,
158
+ metavar="F",
159
+ help="Probability of italicizing via a real italic/oblique variant font (default: 0)",
160
+ )
161
+ g.add_argument(
162
+ "--text-deco-bold-prob",
163
+ type=float,
164
+ default=None,
165
+ metavar="F",
166
+ help="Probability of bolding via a real bold variant font (default: 0)",
167
+ )
168
+
169
+ @classmethod
170
+ def from_args(cls, args: argparse.Namespace) -> TextDecorationConfig:
171
+ def _get(name: str) -> float:
172
+ value = getattr(args, f"text_deco_{name}_prob", None)
173
+ return float(value) if value is not None else 0.0
174
+
175
+ return cls(
176
+ color_prob=_get("color"),
177
+ underline_prob=_get("underline"),
178
+ subscript_prob=_get("subscript"),
179
+ superscript_prob=_get("superscript"),
180
+ italic_prob=_get("italic"),
181
+ bold_prob=_get("bold"),
182
+ )
183
+
184
+
185
+ # Per-line text effect config (Canva-style effects: shadow, glow, outline, ...)
186
+
187
+ TEXT_EFFECT_NAMES: tuple[str, ...] = (
188
+ "background",
189
+ "echo",
190
+ "glitch",
191
+ "glow",
192
+ "hollow",
193
+ "huge",
194
+ "neon",
195
+ "outline",
196
+ "pixel",
197
+ "shadow",
198
+ "tiny",
199
+ "transparent",
200
+ )
201
+
202
+
203
+ @dataclass
204
+ class TextEffectConfig:
205
+ """Weighted, mutually-exclusive per-line text effects sampled at render time.
206
+
207
+ Mirrors a Canva-style effects panel: at most one effect applies per line
208
+ (each probability is rolled independently in a fixed order and the first
209
+ hit wins, so effective probabilities can overlap when several are set),
210
+ combined with the independent decorations in `TextDecorationConfig`. All
211
+ effects are constrained to keep the underlying glyphs legible, since the
212
+ rendered text is still the ground-truth OCR label.
213
+ """
214
+
215
+ background_prob: float = 0.0 # highlight box behind the line
216
+ echo_prob: float = 0.0 # faint offset duplicate copies (ghost/echo)
217
+ glitch_prob: float = 0.0 # small offset colour-split copies
218
+ glow_prob: float = 0.0 # soft blurred halo behind the text
219
+ hollow_prob: float = 0.0 # outline-only glyphs, background-colour interior
220
+ huge_prob: float = 0.0 # oversized text relative to the canvas
221
+ neon_prob: float = 0.0 # bright saturated fill with a neon-sign glow
222
+ outline_prob: float = 0.0 # solid fill with a contrasting stroke
223
+ pixel_prob: float = 0.0 # blocky/pixelated glyph edges
224
+ shadow_prob: float = 0.0 # offset blurred drop shadow
225
+ tiny_prob: float = 0.0 # undersized text relative to the canvas
226
+ transparent_prob: float = 0.0 # partially see-through fill
227
+
228
+ def __post_init__(self) -> None:
229
+ for name in TEXT_EFFECT_NAMES:
230
+ attr = f"{name}_prob"
231
+ setattr(self, attr, float(max(0.0, min(1.0, getattr(self, attr)))))
232
+
233
+ @property
234
+ def enabled(self) -> bool:
235
+ return any(getattr(self, f"{name}_prob") > 0.0 for name in TEXT_EFFECT_NAMES)
236
+
237
+ @staticmethod
238
+ def add_args(parser: argparse.ArgumentParser) -> None:
239
+ g = parser.add_argument_group(
240
+ "Text effects",
241
+ "Canva-style per-line text effects applied at render time (mutually exclusive "
242
+ "with each other; combine freely with --text-deco-* flags). Each probability is "
243
+ "rolled independently in a fixed order and the first hit wins.",
244
+ )
245
+ help_text = {
246
+ "background": "Probability of a highlight box behind the line",
247
+ "echo": "Probability of faint offset duplicate (echo/ghost) copies",
248
+ "glitch": "Probability of a small offset colour-split glitch look",
249
+ "glow": "Probability of a soft blurred glow halo behind the text",
250
+ "hollow": "Probability of outline-only glyphs with a background-colour interior",
251
+ "huge": "Probability of oversized text relative to the canvas",
252
+ "neon": "Probability of a bright neon-sign fill and glow",
253
+ "outline": "Probability of a solid fill with a contrasting stroke outline",
254
+ "pixel": "Probability of blocky/pixelated glyph edges",
255
+ "shadow": "Probability of an offset blurred drop shadow",
256
+ "tiny": "Probability of undersized text relative to the canvas",
257
+ "transparent": "Probability of a partially see-through fill",
258
+ }
259
+ for name in TEXT_EFFECT_NAMES:
260
+ g.add_argument(
261
+ f"--text-effect-{name}-prob",
262
+ type=float,
263
+ default=None,
264
+ metavar="F",
265
+ help=f"{help_text[name]} (default: 0)",
266
+ )
267
+
268
+ @classmethod
269
+ def from_args(cls, args: argparse.Namespace) -> TextEffectConfig:
270
+ kwargs: dict[str, float] = {}
271
+ for name in TEXT_EFFECT_NAMES:
272
+ value = getattr(args, f"text_effect_{name}_prob", None)
273
+ kwargs[f"{name}_prob"] = float(value) if value is not None else 0.0
274
+ return cls(**kwargs)
275
+
276
+
277
+ # Helpers
278
+
279
+
280
+ def _resolve_storage(args: argparse.Namespace) -> str:
281
+ """Resolve storage mode from --storage flag, falling back to legacy flags."""
282
+ storage = str(getattr(args, "storage", "raw"))
283
+ if storage != "raw":
284
+ return storage
285
+ # Legacy flag resolution
286
+ pack = bool(getattr(args, "pack_lmdb", False))
287
+ keep = bool(getattr(args, "keep_raw", False))
288
+ if pack and keep:
289
+ return "both"
290
+ if pack:
291
+ return "lmdb"
292
+ return "raw"
293
+
294
+
295
+ # Master config
296
+
297
+
298
+ @dataclass
299
+ class GenerationConfig:
300
+ """All parameters that govern synthetic image generation and augmentation."""
301
+
302
+ # Image sizing
303
+ image_height: int = 48
304
+ image_width: int | None = None
305
+ color_mode: int = 1 # 1 = grayscale, 3 = RGB
306
+ random_align_when_padded: bool = False
307
+
308
+ # Variable line height
309
+ line_height_mode: str = "fixed" # "fixed" | "variable" | "bucketed"
310
+ min_line_height: int = 32
311
+ max_line_height: int = 96
312
+ line_height_step: int = 8
313
+ line_height_distribution: str = "uniform" # "uniform" | "triangular"
314
+ default_line_height: int | None = None # triangular peak; defaults to range midpoint
315
+
316
+ # Text scale / padding within the sampled canvas height
317
+ font_size_mode: str = "fixed" # "fixed" | "proportional"
318
+ min_font_scale: float = 0.65
319
+ max_font_scale: float = 0.9
320
+ vertical_padding_mode: str = "fixed" # "fixed" | "random"
321
+ min_vertical_padding_ratio: float = 0.04
322
+ max_vertical_padding_ratio: float = 0.18
323
+
324
+ # Optional per-sample metadata sidecar
325
+ record_metadata: bool = False
326
+
327
+ # Font / language
328
+ language: str = "mixed" # corpus language tag; legacy alias for a font pool
329
+ multilingual: bool = False
330
+ cluster_font_fallback: bool = False
331
+ font_pool: str | None = None # named top-level fonts subdirectory; None = auto/all
332
+ direction: str = "auto" # auto | ltr | rtl (horizontal only)
333
+ shaping_language: str | None = None # optional BCP-47 tag for RAQM
334
+ fonts_dir: str = "fonts"
335
+ mixed_font_prob: float = 0.0
336
+ font_mode: str = "random" # "random" | "all"
337
+ copies: int = 3 # augmented copies per line when font_mode=random
338
+
339
+ # Corpus
340
+ corpus_path: str = ""
341
+ min_length: int = 1
342
+ max_length: int = 260
343
+ max_lines: int = 0
344
+ val_percent: float | None = None
345
+ test_percent: float | None = None
346
+ split_ratios: tuple[float, float, float] | None = None
347
+ seed: int | None = 42
348
+ oversample_rare_chars: bool = False
349
+ rare_char_percentile: float = 5.0
350
+ rare_char_multiplier: float = 3.0
351
+
352
+ # Output
353
+ output_dir: str = "data"
354
+ vocab_path: str = ""
355
+ skip_vocab: bool = False
356
+ append: bool = False
357
+ overwrite: bool = False
358
+ output_format: str = "jpg" # "png" | "jpg" | "tiff"
359
+ jpeg_quality: int = 90 # JPEG quality for jpg output
360
+ storage: str = "raw" # "raw" | "lmdb" | "both"
361
+ keep_raw: bool = False # legacy; prefer storage="both"
362
+ bg_color_mode: str = (
363
+ "random" # "default" | "paper_tones" | "colored" | "dark_mode" | "gradient" | "random"
364
+ )
365
+
366
+ # Worker reliability
367
+ workers: int = 0 # 0 = auto
368
+ worker_timeout: int = 300 # seconds; 0 = no timeout
369
+ retry_limit: int = 10
370
+
371
+ # LMDB packing
372
+ pack_lmdb: bool = False
373
+ lmdb_jpeg_quality: int = 90
374
+ lmdb_map_size_gb: int = 256
375
+
376
+ # Opt-in guard, measured on visible line ink after output sizing; 0 disables it.
377
+ min_text_height_for_augmentation: int = 0
378
+ min_glyph_height_for_augmentation: int = 0
379
+ script_intensity_scales: dict[str, float] = field(default_factory=dict)
380
+ font_intensity_scales: dict[str, float] = field(default_factory=dict)
381
+ clean_prob: float = 0.0 # absolute probability before weighted method selection
382
+
383
+ # Augmentation methods - each has {prob, min, max}
384
+
385
+ # Sauvola local-threshold degradation
386
+ sauvola: AugMethodConfig = field(
387
+ default_factory=lambda: AugMethodConfig(prob=0.2, min=0.1, max=0.9)
388
+ )
389
+
390
+ # 4-point perspective (geometric) warp
391
+ geo_warp: AugMethodConfig = field(
392
+ default_factory=lambda: AugMethodConfig(prob=0.2, min=0.1, max=0.9)
393
+ )
394
+
395
+ # Random vertical crop
396
+ vertical_crop: AugMethodConfig = field(
397
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
398
+ )
399
+
400
+ # Albumentations blur branch (MotionBlur | MedianBlur | GaussianBlur)
401
+ blur: AugMethodConfig = field(
402
+ default_factory=lambda: AugMethodConfig(prob=0.4, min=0.1, max=0.9)
403
+ )
404
+
405
+ # Albumentations distortion (Optical | Grid | Elastic)
406
+ distortion: AugMethodConfig = field(
407
+ default_factory=lambda: AugMethodConfig(prob=0.3, min=0.1, max=0.9)
408
+ )
409
+
410
+ # Albumentations noise (GaussNoise | MultiplicativeNoise)
411
+ albu_noise: AugMethodConfig = field(
412
+ default_factory=lambda: AugMethodConfig(prob=0.4, min=0.1, max=0.9)
413
+ )
414
+
415
+ # JPEG compression artifacts
416
+ jpeg_compression: AugMethodConfig = field(
417
+ default_factory=lambda: AugMethodConfig(prob=0.4, min=0.1, max=0.9)
418
+ )
419
+
420
+ # Random rotation
421
+ rotation: AugMethodConfig = field(
422
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
423
+ )
424
+
425
+ # Salt-and-pepper noise
426
+ salt_pepper: AugMethodConfig = field(
427
+ default_factory=lambda: AugMethodConfig(prob=0.15, min=0.1, max=0.9)
428
+ )
429
+
430
+ # Background texture overlay
431
+ background_texture: AugMethodConfig = field(
432
+ default_factory=lambda: AugMethodConfig(prob=0.35, min=0.1, max=0.9)
433
+ )
434
+
435
+ # Low-DPI rendering simulation
436
+ lowdpi: AugMethodConfig = field(
437
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
438
+ )
439
+
440
+ # Oversample rendering
441
+ oversample: AugMethodConfig = field(
442
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
443
+ )
444
+
445
+ # Extreme source-resolution simulation (min/max are pixel heights, not [0, 1])
446
+ extreme_resize: AugMethodConfig = field(
447
+ default_factory=lambda: ExtremeResizeConfig(prob=0.0, min=8, max=1920)
448
+ )
449
+
450
+ # Low-contrast small caption text
451
+ low_contrast_caption: AugMethodConfig = field(
452
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
453
+ )
454
+
455
+ # Perspective warp
456
+ perspective: AugMethodConfig = field(
457
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
458
+ )
459
+
460
+ # Elastic distortion
461
+ elastic: AugMethodConfig = field(
462
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
463
+ )
464
+
465
+ # Random height crop
466
+ random_crop: AugMethodConfig = field(
467
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
468
+ )
469
+
470
+ # Gaussian/Motion blur
471
+ online_blur: AugMethodConfig = field(
472
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
473
+ )
474
+
475
+ # Gaussian noise
476
+ online_noise: AugMethodConfig = field(
477
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
478
+ )
479
+
480
+ # HSV color jitter
481
+ hsv: AugMethodConfig = field(
482
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
483
+ )
484
+
485
+ # Color reversal
486
+ reverse: AugMethodConfig = field(
487
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.0, max=1.0)
488
+ )
489
+
490
+ # Brightness/contrast jitter
491
+ brightness_contrast: AugMethodConfig = field(
492
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
493
+ )
494
+
495
+ # Pixelation (downscale-upscale)
496
+ pixelation: AugMethodConfig = field(
497
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
498
+ )
499
+
500
+ # Gradient illumination overlay
501
+ gradient_illumination: AugMethodConfig = field(
502
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
503
+ )
504
+
505
+ # Morphological erode/dilate
506
+ morphological: AugMethodConfig = field(
507
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
508
+ )
509
+
510
+ # Anisotropic dilation (dot-matrix spread)
511
+ anisotropic_dilation: AugMethodConfig = field(
512
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
513
+ )
514
+
515
+ # RGB near-white fill + colored bloom (needs color-mode 3). Grayscale keeps the dark core.
516
+ luminous_halo: AugMethodConfig = field(
517
+ default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.75)
518
+ )
519
+
520
+ # Normalizer config
521
+ normalizer: NormalizerConfig = field(default_factory=NormalizerConfig)
522
+
523
+ # Text decorations
524
+ text_deco: TextDecorationConfig = field(default_factory=TextDecorationConfig)
525
+
526
+ # Text effects (shadow, glow, outline, ...)
527
+ text_effect: TextEffectConfig = field(default_factory=TextEffectConfig)
528
+
529
+ def __post_init__(self) -> None:
530
+ self.script_intensity_scales = self._parse_script_intensity_scales(self.script_intensity_scales)
531
+ self.font_intensity_scales = self._parse_font_intensity_scales(self.font_intensity_scales)
532
+ if not 0.0 <= self.clean_prob <= 1.0:
533
+ raise ValueError("clean_prob must be between 0 and 1")
534
+ if self.direction not in {"auto", "ltr", "rtl"}:
535
+ raise ValueError("direction must be auto, ltr, or rtl")
536
+ if self.multilingual:
537
+ if self.mixed_font_prob > 0:
538
+ raise ValueError(
539
+ "multilingual mode requires whole-line fonts; mixed_font_prob must be 0"
540
+ )
541
+ if self.text_deco.enabled or self.text_effect.enabled:
542
+ raise ValueError(
543
+ "multilingual mode currently supports plain whole-line text; disable text decorations/effects (including codepoint-split sub/superscripts)"
544
+ )
545
+ if (
546
+ isinstance(self.min_text_height_for_augmentation, bool)
547
+ or not isinstance(self.min_text_height_for_augmentation, int)
548
+ or self.min_text_height_for_augmentation < 0
549
+ ):
550
+ raise ValueError("min_text_height_for_augmentation must be a non-negative integer")
551
+ if (
552
+ isinstance(self.min_glyph_height_for_augmentation, bool)
553
+ or not isinstance(self.min_glyph_height_for_augmentation, int)
554
+ or self.min_glyph_height_for_augmentation < 0
555
+ ):
556
+ raise ValueError("min_glyph_height_for_augmentation must be a non-negative integer")
557
+
558
+ # Split ratio resolution
559
+
560
+ def resolve_split_ratios(self) -> tuple[float, float, float]:
561
+ """Resolve (train_ratio, val_ratio, test_ratio) from config fields.
562
+
563
+ Resolution rules (highest priority first):
564
+
565
+ 1. ``split_ratios`` set explicitly:
566
+ Normalise the three values so they sum to 1.0.
567
+ Example: ``split_ratios=(70, 15, 15)`` -> ``(0.70, 0.15, 0.15)``.
568
+
569
+ 2. Both ``val_percent`` and ``test_percent`` set:
570
+ ``train = 1 - val/100 - test/100``.
571
+
572
+ 3. Only ``val_percent`` set:
573
+ ``test = 0``, ``train = 1 - val/100``.
574
+ Example: ``--val-percent 15`` -> ``(0.85, 0.15, 0.0)``.
575
+
576
+ 4. Only ``test_percent`` set:
577
+ ``val = 0``, ``train = 1 - test/100``.
578
+ Example: ``--test-percent 15`` -> ``(0.85, 0.0, 0.15)``.
579
+
580
+ 5. Neither set (default):
581
+ ``(0.80, 0.10, 0.10)``.
582
+
583
+ To disable all splitting (100 % train), set:
584
+ - ``--split-ratios 100 0 0``, or
585
+ - ``--val-percent 0 --test-percent 0``.
586
+ """
587
+ if self.split_ratios is not None:
588
+ a, b, c = self.split_ratios
589
+ total = a + b + c
590
+ if total <= 0.0:
591
+ return (1.0, 0.0, 0.0)
592
+ return (a / total, b / total, c / total)
593
+
594
+ val_set = self.val_percent is not None
595
+ test_set = self.test_percent is not None
596
+
597
+ if val_set and test_set:
598
+ v = max(0.0, min(100.0, float(self.val_percent))) # type: ignore[arg-type]
599
+ t = max(0.0, min(100.0, float(self.test_percent))) # type: ignore[arg-type]
600
+ tr = max(0.0, 100.0 - v - t)
601
+ total = tr + v + t
602
+ return (tr / total, v / total, t / total)
603
+
604
+ if val_set:
605
+ v = max(0.0, min(100.0, float(self.val_percent))) # type: ignore[arg-type]
606
+ tr = max(0.0, 100.0 - v)
607
+ return (
608
+ tr / (tr + v) if (tr + v) > 0 else 1.0,
609
+ v / (tr + v) if (tr + v) > 0 else 0.0,
610
+ 0.0,
611
+ )
612
+
613
+ if test_set:
614
+ t = max(0.0, min(100.0, float(self.test_percent))) # type: ignore[arg-type]
615
+ tr = max(0.0, 100.0 - t)
616
+ return (
617
+ tr / (tr + t) if (tr + t) > 0 else 1.0,
618
+ 0.0,
619
+ t / (tr + t) if (tr + t) > 0 else 0.0,
620
+ )
621
+
622
+ # Default: 80/10/10
623
+ return (0.80, 0.10, 0.10)
624
+
625
+ # CLI
626
+
627
+ # Helper to build AugMethodConfig CLI flags
628
+ _AUG_METHODS: ClassVar[tuple[tuple[str, str], ...]] = (
629
+ # (attr_name, display_name)
630
+ ("sauvola", "Sauvola threshold degradation"),
631
+ ("geo_warp", "Geometric (4-point) perspective warp"),
632
+ ("vertical_crop", "Random vertical crop"),
633
+ ("blur", "Blur (MotionBlur|MedianBlur|GaussianBlur via Albumentations)"),
634
+ ("distortion", "Distortion (Optical|Grid|Elastic via Albumentations)"),
635
+ ("albu_noise", "Noise (GaussNoise|MultiplicativeNoise via Albumentations)"),
636
+ ("jpeg_compression", "JPEG compression artifacts"),
637
+ ("rotation", "Random rotation"),
638
+ ("salt_pepper", "Salt-and-pepper impulse noise"),
639
+ ("background_texture", "Background texture overlay"),
640
+ ("lowdpi", "Low-DPI rendering simulation"),
641
+ ("oversample", "Oversample rendering"),
642
+ (
643
+ "extreme_resize",
644
+ "Extreme source-resolution simulation (isotropic resize to a very small/large height)",
645
+ ),
646
+ ("low_contrast_caption", "Low-contrast small caption text"),
647
+ ("perspective", "Perspective warp"),
648
+ ("elastic", "Elastic distortion"),
649
+ ("random_crop", "Random height crop"),
650
+ ("online_blur", "Gaussian/Motion blur"),
651
+ ("online_noise", "Gaussian noise"),
652
+ ("hsv", "HSV color jitter"),
653
+ ("reverse", "Color reversal (dark to light)"),
654
+ ("brightness_contrast", "Brightness/contrast jitter"),
655
+ ("pixelation", "Pixelation (downscale-then-upscale)"),
656
+ ("gradient_illumination", "Gradient illumination overlay"),
657
+ ("morphological", "Morphological erode/dilate"),
658
+ ("anisotropic_dilation", "Anisotropic dilation (dot-matrix spread)"),
659
+ (
660
+ "luminous_halo",
661
+ "Luminous halo (RGB near-white fill and colored bloom; grayscale keeps the dark core)",
662
+ ),
663
+ )
664
+
665
+ # Methods whose config class isn't the generic AugMethodConfig (e.g.
666
+ # extreme_resize's min/max are absolute pixel heights, not [0, 1]).
667
+ # from_args/from_dict must reconstruct via the right class, or the base
668
+ # class's __post_init__ clamps min/max back to [0, 1].
669
+ _AUG_METHOD_CLASSES: ClassVar[dict[str, type]] = {"extreme_resize": ExtremeResizeConfig}
670
+
671
+ @staticmethod
672
+ def add_args(parser: argparse.ArgumentParser) -> None:
673
+ """Register all CLI arguments for generation."""
674
+
675
+ # Rendering
676
+ g_render = parser.add_argument_group("Rendering")
677
+ g_render.add_argument(
678
+ "--multilingual",
679
+ action="store_true",
680
+ help="Use RAQM whole-line shaping and exhaustive font coverage selection",
681
+ )
682
+ g_render.add_argument(
683
+ "--cluster-font-fallback",
684
+ action="store_true",
685
+ default=False,
686
+ help="Enable cluster-aware font fallback across multiple fonts using RAQM",
687
+ )
688
+ g_render.add_argument(
689
+ "--language",
690
+ default="mixed",
691
+ help="Corpus language tag (default: mixed); also a legacy font-pool alias",
692
+ )
693
+ g_render.add_argument(
694
+ "--font-pool",
695
+ default=None,
696
+ help="Restrict to a named fonts subdirectory; multilingual defaults to all discovered fonts",
697
+ )
698
+ g_render.add_argument(
699
+ "--direction",
700
+ choices=["auto", "ltr", "rtl"],
701
+ default="auto",
702
+ help="Horizontal base direction in multilingual mode",
703
+ )
704
+ g_render.add_argument(
705
+ "--shaping-language",
706
+ default=None,
707
+ help="Optional BCP-47 shaping language in multilingual mode",
708
+ )
709
+ g_render.add_argument(
710
+ "--fonts",
711
+ default="fonts/",
712
+ help="Root fonts directory: <dir>/khmer/ and <dir>/english/ subdirs",
713
+ )
714
+ g_render.add_argument(
715
+ "--height",
716
+ type=int,
717
+ default=48,
718
+ metavar="PX",
719
+ help="Image height in pixels (default: 48)",
720
+ )
721
+ g_render.add_argument(
722
+ "--width",
723
+ type=int,
724
+ default=None,
725
+ metavar="PX",
726
+ help="Fixed image width; omit for variable width",
727
+ )
728
+ g_render.add_argument(
729
+ "--color-mode",
730
+ type=int,
731
+ choices=[1, 3],
732
+ default=1,
733
+ metavar="{1,3}",
734
+ help="Output colour channels: 1 = grayscale, 3 = RGB (default: 1)",
735
+ )
736
+ g_render.add_argument(
737
+ "--random-align-when-padded",
738
+ action="store_true",
739
+ help="Random left/center/right alignment when --width pads the image",
740
+ )
741
+
742
+ # Variable line height
743
+ g_vh = parser.add_argument_group("Variable line height")
744
+ g_vh.add_argument(
745
+ "--line-height-mode",
746
+ choices=["fixed", "variable", "bucketed"],
747
+ default="fixed",
748
+ help=(
749
+ "'fixed' = every image uses --height (default, backward compatible). "
750
+ "'variable' = sample a height per image from [--min-line-height, "
751
+ "--max-line-height]. 'bucketed' = sample from the fixed set of heights "
752
+ "spaced by --line-height-step across that range."
753
+ ),
754
+ )
755
+ g_vh.add_argument(
756
+ "--min-line-height",
757
+ type=int,
758
+ default=32,
759
+ metavar="PX",
760
+ help="Minimum sampled line height in pixels (default: 32)",
761
+ )
762
+ g_vh.add_argument(
763
+ "--max-line-height",
764
+ type=int,
765
+ default=96,
766
+ metavar="PX",
767
+ help="Maximum sampled line height in pixels (default: 96)",
768
+ )
769
+ g_vh.add_argument(
770
+ "--line-height-step",
771
+ type=int,
772
+ default=8,
773
+ metavar="PX",
774
+ help="Align sampled/bucketed heights to a multiple of this many pixels (default: 8)",
775
+ )
776
+ g_vh.add_argument(
777
+ "--line-height-distribution",
778
+ choices=["uniform", "triangular"],
779
+ default="uniform",
780
+ help=(
781
+ "Shape of the height distribution for --line-height-mode=variable. "
782
+ "'triangular' clusters samples around --default-line-height (default: uniform)"
783
+ ),
784
+ )
785
+ g_vh.add_argument(
786
+ "--default-line-height",
787
+ type=int,
788
+ default=None,
789
+ metavar="PX",
790
+ help="Peak height for the triangular distribution (default: midpoint of the range)",
791
+ )
792
+ g_vh.add_argument(
793
+ "--font-size-mode",
794
+ choices=["fixed", "proportional"],
795
+ default="fixed",
796
+ help=(
797
+ "'fixed' = choose from the preloaded font sizes (default). "
798
+ "'proportional' = choose a font size scaled to the sampled canvas height "
799
+ "(--min-font-scale..--max-font-scale)"
800
+ ),
801
+ )
802
+ g_vh.add_argument(
803
+ "--min-font-scale",
804
+ type=float,
805
+ default=0.65,
806
+ metavar="F",
807
+ help="Minimum glyph height as a fraction of canvas height when font-size-mode=proportional (default: 0.65)",
808
+ )
809
+ g_vh.add_argument(
810
+ "--max-font-scale",
811
+ type=float,
812
+ default=0.9,
813
+ metavar="F",
814
+ help="Maximum glyph height as a fraction of canvas height when font-size-mode=proportional (default: 0.9)",
815
+ )
816
+ g_vh.add_argument(
817
+ "--vertical-padding-mode",
818
+ choices=["fixed", "random"],
819
+ default="fixed",
820
+ help=(
821
+ "'fixed' = existing constant pixel padding (default). "
822
+ "'random' = sample top/bottom padding as a ratio of canvas height"
823
+ ),
824
+ )
825
+ g_vh.add_argument(
826
+ "--min-vertical-padding-ratio",
827
+ type=float,
828
+ default=0.04,
829
+ metavar="F",
830
+ help="Minimum vertical padding as a fraction of canvas height (default: 0.04)",
831
+ )
832
+ g_vh.add_argument(
833
+ "--max-vertical-padding-ratio",
834
+ type=float,
835
+ default=0.18,
836
+ metavar="F",
837
+ help="Maximum vertical padding as a fraction of canvas height (default: 0.18)",
838
+ )
839
+ g_vh.add_argument(
840
+ "--record-metadata",
841
+ action="store_true",
842
+ help="Write a metadata.jsonl sidecar (image/text/width/height) per split",
843
+ )
844
+ g_render.add_argument(
845
+ "--font-mode",
846
+ choices=["random", "all"],
847
+ default="random",
848
+ help="'random' = N augmented copies; 'all' = one image per font per line",
849
+ )
850
+ g_render.add_argument(
851
+ "--copies",
852
+ type=int,
853
+ default=3,
854
+ metavar="N",
855
+ help="Augmented copies per text line when --font-mode=random (default: 3)",
856
+ )
857
+ g_render.add_argument(
858
+ "--mixed-font-prob",
859
+ type=float,
860
+ default=0.0,
861
+ metavar="F",
862
+ help="Probability of per-span font rendering for mixed Khmer/English text",
863
+ )
864
+ g_render.add_argument(
865
+ "--retry-limit",
866
+ type=int,
867
+ default=10,
868
+ metavar="N",
869
+ help="Font selection retries per image when font lacks glyphs",
870
+ )
871
+
872
+ # Corpus
873
+ g_corpus = parser.add_argument_group("Corpus")
874
+ g_corpus.add_argument(
875
+ "--corpus",
876
+ default="corpus/corpus.txt",
877
+ metavar="FILE",
878
+ help="Path to plain text corpus file",
879
+ )
880
+ g_corpus.add_argument(
881
+ "--min-length",
882
+ type=int,
883
+ default=1,
884
+ metavar="N",
885
+ help="Minimum character length to include (default: 1)",
886
+ )
887
+ g_corpus.add_argument(
888
+ "--max-length",
889
+ type=int,
890
+ default=260,
891
+ metavar="N",
892
+ help="Maximum character length to include (default: 260)",
893
+ )
894
+ g_corpus.add_argument(
895
+ "--lines",
896
+ type=int,
897
+ default=0,
898
+ metavar="N",
899
+ help="Max lines to use after filtering; 0 = all",
900
+ )
901
+ def _parse_optional_seed(val: Any) -> int | None:
902
+ if val is None or str(val).strip().lower() in ("none", "null", "nil", ""):
903
+ return None
904
+ return int(val)
905
+
906
+ g_corpus.add_argument(
907
+ "--seed",
908
+ type=_parse_optional_seed,
909
+ default=42,
910
+ metavar="N",
911
+ help="Random seed for deterministic train/val/test splitting and per-sample rendering",
912
+ )
913
+ g_corpus.add_argument(
914
+ "--val-percent",
915
+ type=float,
916
+ default=None,
917
+ metavar="PCT",
918
+ help=(
919
+ "Validation split percentage [0, 100). "
920
+ "If only this is set, test=0. "
921
+ "If neither --val-percent nor --test-percent is set, defaults to 10%%."
922
+ ),
923
+ )
924
+ g_corpus.add_argument(
925
+ "--test-percent",
926
+ type=float,
927
+ default=None,
928
+ metavar="PCT",
929
+ help=(
930
+ "Test split percentage [0, 100). "
931
+ "If only this is set, val=0. "
932
+ "If neither --val-percent nor --test-percent is set, defaults to 10%%."
933
+ ),
934
+ )
935
+ g_corpus.add_argument(
936
+ "--split-ratios",
937
+ nargs=3,
938
+ type=float,
939
+ default=None,
940
+ metavar=("TRAIN", "VAL", "TEST"),
941
+ help=(
942
+ "Explicit train/val/test split ratios (three values, e.g. 80 10 10). "
943
+ "Values are normalised to sum to 1.0. "
944
+ "Overrides --val-percent and --test-percent. "
945
+ "Set '100 0 0' to disable splitting entirely."
946
+ ),
947
+ )
948
+ g_corpus.add_argument(
949
+ "--test-file",
950
+ default=None,
951
+ metavar="FILE",
952
+ help="Path to a separate test corpus file (bypasses automatic test splitting)",
953
+ )
954
+ g_corpus.add_argument(
955
+ "--count-only",
956
+ action="store_true",
957
+ help="Print filter statistics and estimated image count, then exit",
958
+ )
959
+ g_corpus.add_argument(
960
+ "--oversample-rare-chars",
961
+ action="store_true",
962
+ help=(
963
+ "Render extra copies of training lines that contain rare characters "
964
+ "(does not affect val/test, which stay at 1 copy per line)"
965
+ ),
966
+ )
967
+ g_corpus.add_argument(
968
+ "--rare-char-percentile",
969
+ type=float,
970
+ default=5.0,
971
+ metavar="PCT",
972
+ help=(
973
+ "Least-frequent PCT%% of distinct characters in the corpus are "
974
+ "considered rare (default: 5.0)"
975
+ ),
976
+ )
977
+ g_corpus.add_argument(
978
+ "--rare-char-multiplier",
979
+ type=float,
980
+ default=3.0,
981
+ metavar="F",
982
+ help=(
983
+ "Copies multiplier applied to training lines containing a rare "
984
+ "character (default: 3.0)"
985
+ ),
986
+ )
987
+ g_corpus.add_argument(
988
+ "--image-dir",
989
+ default=None,
990
+ metavar="DIR",
991
+ help="Path to existing images directory (bypass text rendering; "
992
+ "corpus must be labels.txt file)",
993
+ )
994
+
995
+ # Output
996
+ g_out = parser.add_argument_group("Output")
997
+ g_out.add_argument(
998
+ "--output",
999
+ "--output-dir",
1000
+ dest="output",
1001
+ default="data",
1002
+ metavar="DIR",
1003
+ help="Output directory for generated dataset (default: data/)",
1004
+ )
1005
+ g_out_grp = g_out.add_mutually_exclusive_group()
1006
+ g_out_grp.add_argument(
1007
+ "--append",
1008
+ action="store_true",
1009
+ help="Append new samples if output directory exists",
1010
+ )
1011
+ g_out_grp.add_argument(
1012
+ "--overwrite",
1013
+ action="store_true",
1014
+ help="Delete and recreate if output directory exists",
1015
+ )
1016
+ g_out.add_argument(
1017
+ "--vocab",
1018
+ default="",
1019
+ metavar="FILE",
1020
+ help="Path to write vocab.json; auto-derived if not set",
1021
+ )
1022
+ g_out.add_argument(
1023
+ "--skip-vocab",
1024
+ action="store_true",
1025
+ help="Do not build vocab.json after generation",
1026
+ )
1027
+ g_out.add_argument(
1028
+ "--output-format",
1029
+ choices=["png", "jpg", "tiff"],
1030
+ default="jpg",
1031
+ help="Output image format (default: jpg)",
1032
+ )
1033
+ g_out.add_argument(
1034
+ "--jpeg-quality",
1035
+ type=int,
1036
+ default=90,
1037
+ metavar="N",
1038
+ help="JPEG quality for jpg output (default: 90; 0-100)",
1039
+ )
1040
+ g_out.add_argument(
1041
+ "--storage",
1042
+ choices=["raw", "lmdb", "both"],
1043
+ default="raw",
1044
+ help="Storage mode: raw = image files only, lmdb = pack and delete, both = save + pack + keep (default: raw)",
1045
+ )
1046
+
1047
+ # LMDB packing
1048
+ g_lmdb = parser.add_argument_group("LMDB Packing")
1049
+ g_lmdb.add_argument(
1050
+ "--pack-lmdb",
1051
+ action="store_true",
1052
+ help="Pack generated dataset into LMDB databases after generation",
1053
+ )
1054
+ g_lmdb.add_argument(
1055
+ "--keep-raw",
1056
+ action="store_true",
1057
+ help="Keep raw image files after LMDB packing",
1058
+ )
1059
+ g_lmdb.add_argument(
1060
+ "--lmdb-jpeg-quality",
1061
+ type=int,
1062
+ default=90,
1063
+ metavar="N",
1064
+ help="JPEG quality for LMDB-stored images (default: 90)",
1065
+ )
1066
+ g_lmdb.add_argument(
1067
+ "--lmdb-map-size-gb",
1068
+ type=int,
1069
+ default=256,
1070
+ metavar="N",
1071
+ help="LMDB map size in GiB (default: 256)",
1072
+ )
1073
+
1074
+ # Workers
1075
+ g_workers = parser.add_argument_group("Workers")
1076
+ g_workers.add_argument(
1077
+ "--workers",
1078
+ type=int,
1079
+ default=0,
1080
+ metavar="N",
1081
+ help="Worker processes; 0 = auto, 1 = serial (default: 0)",
1082
+ )
1083
+ g_workers.add_argument(
1084
+ "--worker-timeout",
1085
+ type=int,
1086
+ default=300,
1087
+ metavar="SEC",
1088
+ help="Seconds before a worker batch times out (default: 300; 0 = no limit)",
1089
+ )
1090
+
1091
+
1092
+ # Rendering style
1093
+ g_style = parser.add_argument_group("Rendering style")
1094
+ g_style.add_argument(
1095
+ "--bg-color-mode",
1096
+ choices=["default", "paper_tones", "colored", "dark_mode", "gradient", "random"],
1097
+ default="random",
1098
+ help=(
1099
+ "Background colour palette for rendered images (default: random). "
1100
+ "'default': off-white/light-gray. "
1101
+ "'paper_tones': warm cream, sepia, recycled, blueprint. "
1102
+ "'colored': soft pastels. "
1103
+ "'dark_mode': dark background with light text. "
1104
+ "'gradient': brightness gradient overlay. "
1105
+ "'random': sample across all palettes when augmentation is enabled."
1106
+ ),
1107
+ )
1108
+
1109
+ # Text decoration
1110
+ TextDecorationConfig.add_args(parser)
1111
+
1112
+ # Text effects
1113
+ TextEffectConfig.add_args(parser)
1114
+
1115
+ # Augmentation methods: prob, min, max per method
1116
+ g_aug = parser.add_argument_group(
1117
+ "Augmentation methods",
1118
+ "Each method has: --<name>-prob F, --<name>-min F, --<name>-max F\n"
1119
+ "One augmentation per image (isolated, not stacked).",
1120
+ )
1121
+ g_aug.add_argument(
1122
+ "--clean-prob",
1123
+ type=float,
1124
+ default=0.0,
1125
+ metavar="F",
1126
+ help="Probability of emitting a clean render before weighted augmentation selection (0-1; default: 0)",
1127
+ )
1128
+ g_aug.add_argument(
1129
+ "--min-text-height-for-augmentation",
1130
+ type=int,
1131
+ default=0,
1132
+ metavar="PX",
1133
+ help=(
1134
+ "Skip destructive methods below this visible line-ink height after resizing "
1135
+ "(0 disables the guard). Unknown text height also skips them; see augmentation docs."
1136
+ ),
1137
+ )
1138
+ g_aug.add_argument(
1139
+ "--min-glyph-height-for-augmentation",
1140
+ type=int,
1141
+ default=0,
1142
+ metavar="PX",
1143
+ help=(
1144
+ "Skip destructive methods below this line glyph-mask height "
1145
+ "(0 disables the guard). Protects glyphs against decorations; does not isolate individual marks."
1146
+ ),
1147
+ )
1148
+ g_aug.add_argument(
1149
+ "--script-intensity-scale",
1150
+ type=str,
1151
+ default=None,
1152
+ metavar="SCRIPT=FACTOR,...",
1153
+ dest="script_intensity_scale",
1154
+ help="Per-script augmentation intensity multiplier (e.g. Khmr=0.7,Latn=0.9)",
1155
+ )
1156
+ g_aug.add_argument(
1157
+ "--font-intensity-scale",
1158
+ type=str,
1159
+ default=None,
1160
+ metavar="FONT=FACTOR,...",
1161
+ dest="font_intensity_scale",
1162
+ help="Per-font multiplier by unique filename or exact path (e.g. Moul.ttf=0.8)",
1163
+ )
1164
+ for attr_name, display_name in GenerationConfig._AUG_METHODS:
1165
+ is_pixel_range = attr_name in GenerationConfig._AUG_METHOD_CLASSES
1166
+ range_unit = "pixels" if is_pixel_range else "0-1"
1167
+ range_metavar = "PX" if is_pixel_range else "F"
1168
+ g_aug.add_argument(
1169
+ f"--{attr_name.replace('_', '-')}-prob",
1170
+ type=float,
1171
+ default=None,
1172
+ metavar="F",
1173
+ help=f"Probability of applying {display_name} (0-1)",
1174
+ )
1175
+ g_aug.add_argument(
1176
+ f"--{attr_name.replace('_', '-')}-min",
1177
+ type=float,
1178
+ default=None,
1179
+ metavar=range_metavar,
1180
+ help=f"Minimum intensity for {display_name} ({range_unit})",
1181
+ )
1182
+ g_aug.add_argument(
1183
+ f"--{attr_name.replace('_', '-')}-max",
1184
+ type=float,
1185
+ default=None,
1186
+ metavar=range_metavar,
1187
+ help=f"Maximum intensity for {display_name} ({range_unit})",
1188
+ )
1189
+
1190
+ # Normalizer
1191
+ NormalizerConfig.add_args(parser)
1192
+
1193
+ @classmethod
1194
+ def from_args(cls, args: argparse.Namespace) -> GenerationConfig:
1195
+ """Build a ``GenerationConfig`` from parsed CLI arguments."""
1196
+
1197
+ # Build AugMethodConfig per method from CLI flags (or use defaults)
1198
+ # NOTE: argparse derives `dest` from the option string by replacing
1199
+ # "-" with "_", so the attribute name IS the lookup key (e.g.
1200
+ # --background-texture-min -> dest "background_texture_min"). Do NOT
1201
+ # dash-ify attr_name here: doing so previously produced lookup keys
1202
+ # like "background-texture_min", which never matched any argparse
1203
+ # dest (Python identifiers can't contain "-"), so getattr() always
1204
+ # returned None and every multi-word method silently fell back to
1205
+ # its dataclass default regardless of CLI/YAML configuration.
1206
+ defaults = cls()
1207
+ aug_kwargs: dict[str, AugMethodConfig] = {}
1208
+ for attr_name, _ in cls._AUG_METHODS:
1209
+ prob = getattr(args, f"{attr_name}_prob", None)
1210
+ v_min = getattr(args, f"{attr_name}_min", None)
1211
+ v_max = getattr(args, f"{attr_name}_max", None)
1212
+
1213
+ default: AugMethodConfig = getattr(defaults, attr_name)
1214
+ cfg_cls = cls._AUG_METHOD_CLASSES.get(attr_name, AugMethodConfig)
1215
+ aug_kwargs[attr_name] = cfg_cls(
1216
+ prob=float(prob) if prob is not None else default.prob,
1217
+ min=float(v_min) if v_min is not None else default.min,
1218
+ max=float(v_max) if v_max is not None else default.max,
1219
+ )
1220
+
1221
+
1222
+ config = cls(
1223
+ image_height=int(getattr(args, "height", 48)),
1224
+ image_width=getattr(args, "width", None),
1225
+ color_mode=int(getattr(args, "color_mode", 1)),
1226
+ random_align_when_padded=bool(getattr(args, "random_align_when_padded", False)),
1227
+ line_height_mode=str(getattr(args, "line_height_mode", "fixed")),
1228
+ min_line_height=int(getattr(args, "min_line_height", 32)),
1229
+ max_line_height=int(getattr(args, "max_line_height", 96)),
1230
+ line_height_step=int(getattr(args, "line_height_step", 8)),
1231
+ line_height_distribution=str(getattr(args, "line_height_distribution", "uniform")),
1232
+ default_line_height=(
1233
+ int(args.default_line_height)
1234
+ if getattr(args, "default_line_height", None) is not None
1235
+ else None
1236
+ ),
1237
+ font_size_mode=str(getattr(args, "font_size_mode", "fixed")),
1238
+ min_font_scale=float(getattr(args, "min_font_scale", 0.65)),
1239
+ max_font_scale=float(getattr(args, "max_font_scale", 0.9)),
1240
+ vertical_padding_mode=str(getattr(args, "vertical_padding_mode", "fixed")),
1241
+ min_vertical_padding_ratio=float(getattr(args, "min_vertical_padding_ratio", 0.04)),
1242
+ max_vertical_padding_ratio=float(getattr(args, "max_vertical_padding_ratio", 0.18)),
1243
+ record_metadata=bool(getattr(args, "record_metadata", False)),
1244
+ language=str(getattr(args, "language", "mixed")),
1245
+ multilingual=bool(getattr(args, "multilingual", False)),
1246
+ cluster_font_fallback=bool(getattr(args, "cluster_font_fallback", False)),
1247
+ font_pool=getattr(args, "font_pool", None),
1248
+ direction=str(getattr(args, "direction", "auto")),
1249
+ shaping_language=getattr(args, "shaping_language", None),
1250
+ clean_prob=float(getattr(args, "clean_prob", 0.0)),
1251
+ fonts_dir=str(getattr(args, "fonts", "fonts/")),
1252
+ mixed_font_prob=float(getattr(args, "mixed_font_prob", 0.0)),
1253
+ font_mode=str(getattr(args, "font_mode", "random")),
1254
+ copies=int(getattr(args, "copies", 3)),
1255
+ min_length=int(getattr(args, "min_length", 1)),
1256
+ max_length=int(getattr(args, "max_length", 260)),
1257
+ max_lines=int(getattr(args, "lines", 0)),
1258
+ val_percent=getattr(args, "val_percent", None),
1259
+ test_percent=getattr(args, "test_percent", None),
1260
+ split_ratios=(
1261
+ tuple(args.split_ratios)
1262
+ if getattr(args, "split_ratios", None) is not None
1263
+ else None
1264
+ ),
1265
+ seed=(
1266
+ int(getattr(args, "seed", 42))
1267
+ if getattr(args, "seed", 42) is not None
1268
+ else None
1269
+ ),
1270
+ oversample_rare_chars=bool(getattr(args, "oversample_rare_chars", False)),
1271
+ rare_char_percentile=float(getattr(args, "rare_char_percentile", 5.0)),
1272
+ rare_char_multiplier=float(getattr(args, "rare_char_multiplier", 3.0)),
1273
+ corpus_path=str(getattr(args, "corpus", "")),
1274
+ output_dir=str(getattr(args, "output", "data")),
1275
+ vocab_path=str(getattr(args, "vocab", "")),
1276
+ skip_vocab=bool(getattr(args, "skip_vocab", False)),
1277
+ append=bool(getattr(args, "append", False)),
1278
+ overwrite=bool(getattr(args, "overwrite", False)),
1279
+ output_format=str(getattr(args, "output_format", "jpg")),
1280
+ jpeg_quality=int(getattr(args, "jpeg_quality", 90)),
1281
+ storage=_resolve_storage(args),
1282
+ keep_raw=bool(getattr(args, "keep_raw", False)),
1283
+ bg_color_mode=str(getattr(args, "bg_color_mode", "random")),
1284
+ workers=int(getattr(args, "workers", 0)),
1285
+ worker_timeout=int(getattr(args, "worker_timeout", 300)),
1286
+ retry_limit=int(getattr(args, "retry_limit", 10)),
1287
+ pack_lmdb=bool(getattr(args, "pack_lmdb", False)),
1288
+ lmdb_jpeg_quality=int(getattr(args, "lmdb_jpeg_quality", 90)),
1289
+ lmdb_map_size_gb=int(getattr(args, "lmdb_map_size_gb", 256)),
1290
+ min_text_height_for_augmentation=getattr(args, "min_text_height_for_augmentation", 0),
1291
+ min_glyph_height_for_augmentation=getattr(args, "min_glyph_height_for_augmentation", 0),
1292
+ script_intensity_scales=cls._parse_script_intensity_scales(
1293
+ getattr(args, "script_intensity_scale", None)
1294
+ if (
1295
+ getattr(args, "script_intensity_scale", None) is not None
1296
+ and getattr(args, "script_intensity_scale", None) != ""
1297
+ )
1298
+ else getattr(args, "script_intensity_scales", None)
1299
+ ),
1300
+ font_intensity_scales=cls._parse_font_intensity_scales(
1301
+ getattr(args, "font_intensity_scale", None)
1302
+ if (
1303
+ getattr(args, "font_intensity_scale", None) is not None
1304
+ and getattr(args, "font_intensity_scale", None) != ""
1305
+ )
1306
+ else getattr(args, "font_intensity_scales", None)
1307
+ ),
1308
+ text_deco=TextDecorationConfig.from_args(args),
1309
+ text_effect=TextEffectConfig.from_args(args),
1310
+ normalizer=NormalizerConfig.from_args(args),
1311
+ )
1312
+ for attr_name, aug_cfg in aug_kwargs.items():
1313
+ setattr(config, attr_name, aug_cfg)
1314
+ return config
1315
+
1316
+ # Introspection helpers
1317
+
1318
+ def iter_aug_methods(self) -> list[tuple[str, AugMethodConfig]]:
1319
+ """Yield (attr_name, AugMethodConfig) for every augmentation method."""
1320
+ result: list[tuple[str, AugMethodConfig]] = []
1321
+ for attr_name, _ in self._AUG_METHODS:
1322
+ cfg = getattr(self, attr_name)
1323
+ if isinstance(cfg, AugMethodConfig):
1324
+ result.append((attr_name, cfg))
1325
+ return result
1326
+
1327
+ @staticmethod
1328
+ def _parse_scale_mapping(val: Any, kind: str) -> dict[str, float]:
1329
+ """Read finite, non-negative factors from a mapping or comma-separated pairs."""
1330
+ if val is None or val == "":
1331
+ return {}
1332
+ if isinstance(val, str):
1333
+ pairs: dict[str, Any] = {}
1334
+ for part in val.split(","):
1335
+ key, separator, factor = part.strip().partition("=" if "=" in part else ":")
1336
+ if not separator:
1337
+ raise InputValidationError(
1338
+ f"Invalid {kind} intensity scale: {part!r}; expected KEY=FACTOR",
1339
+ code="invalid_scale_factor",
1340
+ )
1341
+ pairs[key.strip()] = factor.strip()
1342
+ elif isinstance(val, dict):
1343
+ pairs = val
1344
+ else:
1345
+ raise InputValidationError(
1346
+ f"{kind.capitalize()} intensity scales require a mapping or KEY=FACTOR pairs",
1347
+ code="invalid_scale_factor",
1348
+ )
1349
+ result: dict[str, float] = {}
1350
+ for key, value in pairs.items():
1351
+ try:
1352
+ factor = float(value)
1353
+ except (TypeError, ValueError, OverflowError) as exc:
1354
+ raise InputValidationError(
1355
+ f"Invalid {kind} intensity factor for {key!r}: {value!r}",
1356
+ code="invalid_scale_factor",
1357
+ ) from exc
1358
+ if not str(key).strip() or isinstance(value, bool) or factor < 0 or not math.isfinite(factor):
1359
+ raise InputValidationError(
1360
+ f"{kind.capitalize()} intensity factor for {key!r} must have a non-empty key and finite non-negative value",
1361
+ code="invalid_scale_factor",
1362
+ )
1363
+ result[str(key)] = factor
1364
+ return result
1365
+
1366
+ @staticmethod
1367
+ def _parse_script_intensity_scales(val: Any) -> dict[str, float]:
1368
+ """Parse script intensity scale mapping from string or dict."""
1369
+ return GenerationConfig._parse_scale_mapping(val, "script")
1370
+
1371
+ @staticmethod
1372
+ def _parse_font_intensity_scales(val: Any) -> dict[str, float]:
1373
+ """Parse font intensity scale mapping from string or dict."""
1374
+ return GenerationConfig._parse_scale_mapping(val, "font")
1375
+
1376
+ def enabled_aug_methods(self) -> list[tuple[str, AugMethodConfig]]:
1377
+ """Return only augmentation methods with prob > 0."""
1378
+ return [(name, cfg) for name, cfg in self.iter_aug_methods() if cfg.enabled]
1379
+
1380
+ # Serialisation (for IPC / config file)
1381
+
1382
+ def to_dict(self) -> dict[str, Any]:
1383
+ """Serialise to a plain dict for cross-process IPC and YAML export."""
1384
+ result: dict[str, Any] = {}
1385
+ for f in fields(self):
1386
+ value = getattr(self, f.name)
1387
+ if isinstance(value, AugMethodConfig):
1388
+ result[f.name] = {"prob": value.prob, "min": value.min, "max": value.max}
1389
+ elif isinstance(value, NormalizerConfig):
1390
+ result["normalizer"] = value.to_dict()
1391
+ elif isinstance(value, TextDecorationConfig):
1392
+ result[f.name] = {
1393
+ "color_prob": value.color_prob,
1394
+ "underline_prob": value.underline_prob,
1395
+ "subscript_prob": value.subscript_prob,
1396
+ "superscript_prob": value.superscript_prob,
1397
+ "italic_prob": value.italic_prob,
1398
+ "bold_prob": value.bold_prob,
1399
+ }
1400
+ elif isinstance(value, TextEffectConfig):
1401
+ result[f.name] = {
1402
+ f"{name}_prob": getattr(value, f"{name}_prob") for name in TEXT_EFFECT_NAMES
1403
+ }
1404
+ else:
1405
+ result[f.name] = value
1406
+ return result
1407
+
1408
+ @classmethod
1409
+ def from_dict(cls, d: dict[str, Any]) -> GenerationConfig:
1410
+ """Reconstruct from a dict produced by :meth:`to_dict`.
1411
+
1412
+ Unknown keys are silently ignored so workers started with an older
1413
+ config dict do not crash when new keys are added.
1414
+ """
1415
+ aug_method_names = {name for name, _ in cls._AUG_METHODS}
1416
+ aug_kwargs: dict[str, AugMethodConfig] = {}
1417
+ other_kwargs: dict[str, Any] = {}
1418
+
1419
+ for key, value in d.items():
1420
+ if key in aug_method_names and isinstance(value, dict):
1421
+ cfg_cls = cls._AUG_METHOD_CLASSES.get(key, AugMethodConfig)
1422
+ aug_kwargs[key] = cfg_cls(
1423
+ prob=float(value.get("prob", 0.0)),
1424
+ min=float(value.get("min", 0.0)),
1425
+ max=float(value.get("max", 1.0)),
1426
+ )
1427
+ elif key == "normalizer" and isinstance(value, dict):
1428
+ other_kwargs[key] = NormalizerConfig.from_dict(value)
1429
+ elif key == "text_deco" and isinstance(value, dict):
1430
+ other_kwargs[key] = TextDecorationConfig(
1431
+ color_prob=float(value.get("color_prob", 0.0)),
1432
+ underline_prob=float(value.get("underline_prob", 0.0)),
1433
+ subscript_prob=float(value.get("subscript_prob", 0.0)),
1434
+ superscript_prob=float(value.get("superscript_prob", 0.0)),
1435
+ italic_prob=float(value.get("italic_prob", 0.0)),
1436
+ bold_prob=float(value.get("bold_prob", 0.0)),
1437
+ )
1438
+ elif key == "text_effect" and isinstance(value, dict):
1439
+ other_kwargs[key] = TextEffectConfig(
1440
+ **{
1441
+ f"{name}_prob": float(value.get(f"{name}_prob", 0.0))
1442
+ for name in TEXT_EFFECT_NAMES
1443
+ }
1444
+ )
1445
+ elif key not in aug_method_names:
1446
+ other_kwargs[key] = value
1447
+
1448
+ # Map yaml-key names to attr names (e.g. "image_height" stays, "image-height" also works)
1449
+ known = {f.name for f in fields(cls)} - aug_method_names - {"normalizer"}
1450
+ alias_map = {
1451
+ "script_intensity_scale": "script_intensity_scales",
1452
+ "font_intensity_scale": "font_intensity_scales",
1453
+ }
1454
+ filtered = {
1455
+ k: v
1456
+ for k, v in other_kwargs.items()
1457
+ if (k.replace("-", "_") in known or alias_map.get(k.replace("-", "_")) in known)
1458
+ and k != "normalizer"
1459
+ }
1460
+ # Renormalise dashed keys and resolve singular aliases
1461
+ remapped: dict[str, Any] = {}
1462
+ for k, v in filtered.items():
1463
+ norm_k = k.replace("-", "_")
1464
+ remapped[alias_map.get(norm_k, norm_k)] = v
1465
+
1466
+ return cls(
1467
+ **remapped,
1468
+ **aug_kwargs,
1469
+ **({"normalizer": other_kwargs["normalizer"]} if "normalizer" in other_kwargs else {}),
1470
+ )
1471
+
1472
+
1473
+ # Alias for backwards-compatibility / spec naming
1474
+ GeneratorConfig = GenerationConfig