ocr-foundry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khocr_gen/__init__.py +11 -0
- khocr_gen/_pil_compat.py +25 -0
- khocr_gen/_rust_accel.py +204 -0
- khocr_gen/augmentation.py +1193 -0
- khocr_gen/benchmark.py +888 -0
- khocr_gen/benchmark_cmd.py +103 -0
- khocr_gen/cli.py +246 -0
- khocr_gen/combine.py +214 -0
- khocr_gen/combine_cmd.py +125 -0
- khocr_gen/config.py +1474 -0
- khocr_gen/config_loader.py +157 -0
- khocr_gen/corpus.py +621 -0
- khocr_gen/data_generator.py +1160 -0
- khocr_gen/errors.py +33 -0
- khocr_gen/evaluate_cmd.py +50 -0
- khocr_gen/evaluation.py +259 -0
- khocr_gen/fonts.py +316 -0
- khocr_gen/generate.py +227 -0
- khocr_gen/layout.py +1028 -0
- khocr_gen/layout_cmd.py +371 -0
- khocr_gen/layout_degradation.py +128 -0
- khocr_gen/layout_mesh.py +200 -0
- khocr_gen/layout_shaping.py +264 -0
- khocr_gen/layout_transform.py +238 -0
- khocr_gen/layout_warp.py +473 -0
- khocr_gen/line_height.py +127 -0
- khocr_gen/lmdb_pack.py +132 -0
- khocr_gen/logging.py +25 -0
- khocr_gen/normalizer.py +221 -0
- khocr_gen/parallel.py +366 -0
- khocr_gen/provenance.py +328 -0
- khocr_gen/rendering.py +1607 -0
- khocr_gen/text_support.py +429 -0
- khocr_gen/verify.py +471 -0
- khocr_gen/viewer.py +172 -0
- ocr_foundry-0.2.0.dist-info/METADATA +174 -0
- ocr_foundry-0.2.0.dist-info/RECORD +40 -0
- ocr_foundry-0.2.0.dist-info/WHEEL +4 -0
- ocr_foundry-0.2.0.dist-info/entry_points.txt +3 -0
- ocr_foundry-0.2.0.dist-info/licenses/LICENSE +21 -0
khocr_gen/config.py
ADDED
|
@@ -0,0 +1,1474 @@
|
|
|
1
|
+
"""Generation configuration - single source of truth for all generation parameters.
|
|
2
|
+
|
|
3
|
+
Each augmentation method has three tunable knobs exposed to CLI and config YAML:
|
|
4
|
+
|
|
5
|
+
* `prob`: probability [0, 1] of applying this augmentation to a given text line
|
|
6
|
+
* `min` : minimum intensity level [0, 1] (normalised; mapped to physical units internally)
|
|
7
|
+
* `max` : maximum intensity level [0, 1]
|
|
8
|
+
|
|
9
|
+
One exception: `extreme_resize`'s `min`/`max` are absolute pixel heights
|
|
10
|
+
(see `ExtremeResizeConfig`), not normalised [0, 1] values.
|
|
11
|
+
|
|
12
|
+
The augmentation system is *isolated*:
|
|
13
|
+
a single generated image receives exactly one augmentation method on a clean rendered canvas (no stacked effects).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import math
|
|
19
|
+
from dataclasses import dataclass, field, fields
|
|
20
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
21
|
+
|
|
22
|
+
from .errors import InputValidationError
|
|
23
|
+
from .normalizer import NormalizerConfig
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
import argparse
|
|
27
|
+
|
|
28
|
+
# Per-method augmentation config
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class AugMethodConfig:
|
|
33
|
+
"""Probability and intensity range for a single augmentation method."""
|
|
34
|
+
|
|
35
|
+
prob: float = 0.0
|
|
36
|
+
min: float = 0.0
|
|
37
|
+
max: float = 1.0
|
|
38
|
+
|
|
39
|
+
def __post_init__(self) -> None:
|
|
40
|
+
self.prob = float(max(0.0, min(1.0, self.prob)))
|
|
41
|
+
self.min = float(max(0.0, min(1.0, self.min)))
|
|
42
|
+
self.max = float(max(self.min, min(1.0, self.max)))
|
|
43
|
+
|
|
44
|
+
def sample_intensity(self) -> float:
|
|
45
|
+
"""Draw a random intensity in [min, max]."""
|
|
46
|
+
if self.min >= self.max:
|
|
47
|
+
return self.min
|
|
48
|
+
import random
|
|
49
|
+
|
|
50
|
+
return random.uniform(self.min, self.max)
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def enabled(self) -> bool:
|
|
54
|
+
return self.prob > 0.0
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class ExtremeResizeConfig(AugMethodConfig):
|
|
59
|
+
"""Probability and height range for the `extreme_resize` augmentation.
|
|
60
|
+
|
|
61
|
+
Same `{prob, min, max}` shape as `AugMethodConfig`, but here `min`/`max`
|
|
62
|
+
are absolute target heights in *pixels* (not normalised to [0, 1]): the
|
|
63
|
+
augmentation isotropically resizes the clean canvas down to as small as
|
|
64
|
+
`min` or up to as large as `max` pixels tall before the pipeline resizes
|
|
65
|
+
it back to the output line height, simulating an extreme source
|
|
66
|
+
resolution (e.g. `min=8` for a tiny/pixelated source, `max=1920` for a
|
|
67
|
+
huge scan later shrunk down).
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
min: float = 8.0
|
|
71
|
+
max: float = 1920.0
|
|
72
|
+
|
|
73
|
+
def __post_init__(self) -> None:
|
|
74
|
+
self.prob = float(max(0.0, min(1.0, self.prob)))
|
|
75
|
+
self.min = float(max(1.0, self.min))
|
|
76
|
+
self.max = float(max(self.min, self.max))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# Per-line text decoration config
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass
|
|
83
|
+
class TextDecorationConfig:
|
|
84
|
+
"""Probabilities for per-line text decorations applied at render time.
|
|
85
|
+
|
|
86
|
+
Decorations are rendering attributes (not augmentations): each image
|
|
87
|
+
independently samples each decoration, so they can combine (e.g. bold +
|
|
88
|
+
underline). All probabilities default to 0.0 (feature off).
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
color_prob: float = 0.0 # whole-line single random color (RGB mode only)
|
|
92
|
+
underline_prob: float = 0.0 # underline under the whole line
|
|
93
|
+
subscript_prob: float = 0.0 # lower 1-2 random ASCII chars
|
|
94
|
+
superscript_prob: float = 0.0 # raise 1-2 random ASCII chars
|
|
95
|
+
italic_prob: float = 0.0 # real italic/oblique variant font (no-op if absent)
|
|
96
|
+
bold_prob: float = 0.0 # real bold variant font (no-op if absent)
|
|
97
|
+
|
|
98
|
+
def __post_init__(self) -> None:
|
|
99
|
+
self.color_prob = float(max(0.0, min(1.0, self.color_prob)))
|
|
100
|
+
self.underline_prob = float(max(0.0, min(1.0, self.underline_prob)))
|
|
101
|
+
self.subscript_prob = float(max(0.0, min(1.0, self.subscript_prob)))
|
|
102
|
+
self.superscript_prob = float(max(0.0, min(1.0, self.superscript_prob)))
|
|
103
|
+
self.italic_prob = float(max(0.0, min(1.0, self.italic_prob)))
|
|
104
|
+
self.bold_prob = float(max(0.0, min(1.0, self.bold_prob)))
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def enabled(self) -> bool:
|
|
108
|
+
return any(
|
|
109
|
+
(
|
|
110
|
+
self.color_prob > 0.0,
|
|
111
|
+
self.underline_prob > 0.0,
|
|
112
|
+
self.subscript_prob > 0.0,
|
|
113
|
+
self.superscript_prob > 0.0,
|
|
114
|
+
self.italic_prob > 0.0,
|
|
115
|
+
self.bold_prob > 0.0,
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
@staticmethod
|
|
120
|
+
def add_args(parser: argparse.ArgumentParser) -> None:
|
|
121
|
+
g = parser.add_argument_group(
|
|
122
|
+
"Text decoration",
|
|
123
|
+
"Per-line text decorations applied at render time (can combine). "
|
|
124
|
+
"Bold/italic require a matching variant font; random color requires --color-mode 3.",
|
|
125
|
+
)
|
|
126
|
+
g.add_argument(
|
|
127
|
+
"--text-deco-color-prob",
|
|
128
|
+
type=float,
|
|
129
|
+
default=None,
|
|
130
|
+
metavar="F",
|
|
131
|
+
help="Probability of a single random text color for the whole line (default: 0)",
|
|
132
|
+
)
|
|
133
|
+
g.add_argument(
|
|
134
|
+
"--text-deco-underline-prob",
|
|
135
|
+
type=float,
|
|
136
|
+
default=None,
|
|
137
|
+
metavar="F",
|
|
138
|
+
help="Probability of underlining the line (default: 0)",
|
|
139
|
+
)
|
|
140
|
+
g.add_argument(
|
|
141
|
+
"--text-deco-subscript-prob",
|
|
142
|
+
type=float,
|
|
143
|
+
default=None,
|
|
144
|
+
metavar="F",
|
|
145
|
+
help="Probability of lowering 1-2 random ASCII chars as subscript (default: 0)",
|
|
146
|
+
)
|
|
147
|
+
g.add_argument(
|
|
148
|
+
"--text-deco-superscript-prob",
|
|
149
|
+
type=float,
|
|
150
|
+
default=None,
|
|
151
|
+
metavar="F",
|
|
152
|
+
help="Probability of raising 1-2 random ASCII chars as superscript (default: 0)",
|
|
153
|
+
)
|
|
154
|
+
g.add_argument(
|
|
155
|
+
"--text-deco-italic-prob",
|
|
156
|
+
type=float,
|
|
157
|
+
default=None,
|
|
158
|
+
metavar="F",
|
|
159
|
+
help="Probability of italicizing via a real italic/oblique variant font (default: 0)",
|
|
160
|
+
)
|
|
161
|
+
g.add_argument(
|
|
162
|
+
"--text-deco-bold-prob",
|
|
163
|
+
type=float,
|
|
164
|
+
default=None,
|
|
165
|
+
metavar="F",
|
|
166
|
+
help="Probability of bolding via a real bold variant font (default: 0)",
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def from_args(cls, args: argparse.Namespace) -> TextDecorationConfig:
|
|
171
|
+
def _get(name: str) -> float:
|
|
172
|
+
value = getattr(args, f"text_deco_{name}_prob", None)
|
|
173
|
+
return float(value) if value is not None else 0.0
|
|
174
|
+
|
|
175
|
+
return cls(
|
|
176
|
+
color_prob=_get("color"),
|
|
177
|
+
underline_prob=_get("underline"),
|
|
178
|
+
subscript_prob=_get("subscript"),
|
|
179
|
+
superscript_prob=_get("superscript"),
|
|
180
|
+
italic_prob=_get("italic"),
|
|
181
|
+
bold_prob=_get("bold"),
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# Per-line text effect config (Canva-style effects: shadow, glow, outline, ...)
|
|
186
|
+
|
|
187
|
+
TEXT_EFFECT_NAMES: tuple[str, ...] = (
|
|
188
|
+
"background",
|
|
189
|
+
"echo",
|
|
190
|
+
"glitch",
|
|
191
|
+
"glow",
|
|
192
|
+
"hollow",
|
|
193
|
+
"huge",
|
|
194
|
+
"neon",
|
|
195
|
+
"outline",
|
|
196
|
+
"pixel",
|
|
197
|
+
"shadow",
|
|
198
|
+
"tiny",
|
|
199
|
+
"transparent",
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
@dataclass
|
|
204
|
+
class TextEffectConfig:
|
|
205
|
+
"""Weighted, mutually-exclusive per-line text effects sampled at render time.
|
|
206
|
+
|
|
207
|
+
Mirrors a Canva-style effects panel: at most one effect applies per line
|
|
208
|
+
(each probability is rolled independently in a fixed order and the first
|
|
209
|
+
hit wins, so effective probabilities can overlap when several are set),
|
|
210
|
+
combined with the independent decorations in `TextDecorationConfig`. All
|
|
211
|
+
effects are constrained to keep the underlying glyphs legible, since the
|
|
212
|
+
rendered text is still the ground-truth OCR label.
|
|
213
|
+
"""
|
|
214
|
+
|
|
215
|
+
background_prob: float = 0.0 # highlight box behind the line
|
|
216
|
+
echo_prob: float = 0.0 # faint offset duplicate copies (ghost/echo)
|
|
217
|
+
glitch_prob: float = 0.0 # small offset colour-split copies
|
|
218
|
+
glow_prob: float = 0.0 # soft blurred halo behind the text
|
|
219
|
+
hollow_prob: float = 0.0 # outline-only glyphs, background-colour interior
|
|
220
|
+
huge_prob: float = 0.0 # oversized text relative to the canvas
|
|
221
|
+
neon_prob: float = 0.0 # bright saturated fill with a neon-sign glow
|
|
222
|
+
outline_prob: float = 0.0 # solid fill with a contrasting stroke
|
|
223
|
+
pixel_prob: float = 0.0 # blocky/pixelated glyph edges
|
|
224
|
+
shadow_prob: float = 0.0 # offset blurred drop shadow
|
|
225
|
+
tiny_prob: float = 0.0 # undersized text relative to the canvas
|
|
226
|
+
transparent_prob: float = 0.0 # partially see-through fill
|
|
227
|
+
|
|
228
|
+
def __post_init__(self) -> None:
|
|
229
|
+
for name in TEXT_EFFECT_NAMES:
|
|
230
|
+
attr = f"{name}_prob"
|
|
231
|
+
setattr(self, attr, float(max(0.0, min(1.0, getattr(self, attr)))))
|
|
232
|
+
|
|
233
|
+
@property
|
|
234
|
+
def enabled(self) -> bool:
|
|
235
|
+
return any(getattr(self, f"{name}_prob") > 0.0 for name in TEXT_EFFECT_NAMES)
|
|
236
|
+
|
|
237
|
+
@staticmethod
|
|
238
|
+
def add_args(parser: argparse.ArgumentParser) -> None:
|
|
239
|
+
g = parser.add_argument_group(
|
|
240
|
+
"Text effects",
|
|
241
|
+
"Canva-style per-line text effects applied at render time (mutually exclusive "
|
|
242
|
+
"with each other; combine freely with --text-deco-* flags). Each probability is "
|
|
243
|
+
"rolled independently in a fixed order and the first hit wins.",
|
|
244
|
+
)
|
|
245
|
+
help_text = {
|
|
246
|
+
"background": "Probability of a highlight box behind the line",
|
|
247
|
+
"echo": "Probability of faint offset duplicate (echo/ghost) copies",
|
|
248
|
+
"glitch": "Probability of a small offset colour-split glitch look",
|
|
249
|
+
"glow": "Probability of a soft blurred glow halo behind the text",
|
|
250
|
+
"hollow": "Probability of outline-only glyphs with a background-colour interior",
|
|
251
|
+
"huge": "Probability of oversized text relative to the canvas",
|
|
252
|
+
"neon": "Probability of a bright neon-sign fill and glow",
|
|
253
|
+
"outline": "Probability of a solid fill with a contrasting stroke outline",
|
|
254
|
+
"pixel": "Probability of blocky/pixelated glyph edges",
|
|
255
|
+
"shadow": "Probability of an offset blurred drop shadow",
|
|
256
|
+
"tiny": "Probability of undersized text relative to the canvas",
|
|
257
|
+
"transparent": "Probability of a partially see-through fill",
|
|
258
|
+
}
|
|
259
|
+
for name in TEXT_EFFECT_NAMES:
|
|
260
|
+
g.add_argument(
|
|
261
|
+
f"--text-effect-{name}-prob",
|
|
262
|
+
type=float,
|
|
263
|
+
default=None,
|
|
264
|
+
metavar="F",
|
|
265
|
+
help=f"{help_text[name]} (default: 0)",
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
@classmethod
|
|
269
|
+
def from_args(cls, args: argparse.Namespace) -> TextEffectConfig:
|
|
270
|
+
kwargs: dict[str, float] = {}
|
|
271
|
+
for name in TEXT_EFFECT_NAMES:
|
|
272
|
+
value = getattr(args, f"text_effect_{name}_prob", None)
|
|
273
|
+
kwargs[f"{name}_prob"] = float(value) if value is not None else 0.0
|
|
274
|
+
return cls(**kwargs)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# Helpers
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _resolve_storage(args: argparse.Namespace) -> str:
|
|
281
|
+
"""Resolve storage mode from --storage flag, falling back to legacy flags."""
|
|
282
|
+
storage = str(getattr(args, "storage", "raw"))
|
|
283
|
+
if storage != "raw":
|
|
284
|
+
return storage
|
|
285
|
+
# Legacy flag resolution
|
|
286
|
+
pack = bool(getattr(args, "pack_lmdb", False))
|
|
287
|
+
keep = bool(getattr(args, "keep_raw", False))
|
|
288
|
+
if pack and keep:
|
|
289
|
+
return "both"
|
|
290
|
+
if pack:
|
|
291
|
+
return "lmdb"
|
|
292
|
+
return "raw"
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# Master config
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass
|
|
299
|
+
class GenerationConfig:
|
|
300
|
+
"""All parameters that govern synthetic image generation and augmentation."""
|
|
301
|
+
|
|
302
|
+
# Image sizing
|
|
303
|
+
image_height: int = 48
|
|
304
|
+
image_width: int | None = None
|
|
305
|
+
color_mode: int = 1 # 1 = grayscale, 3 = RGB
|
|
306
|
+
random_align_when_padded: bool = False
|
|
307
|
+
|
|
308
|
+
# Variable line height
|
|
309
|
+
line_height_mode: str = "fixed" # "fixed" | "variable" | "bucketed"
|
|
310
|
+
min_line_height: int = 32
|
|
311
|
+
max_line_height: int = 96
|
|
312
|
+
line_height_step: int = 8
|
|
313
|
+
line_height_distribution: str = "uniform" # "uniform" | "triangular"
|
|
314
|
+
default_line_height: int | None = None # triangular peak; defaults to range midpoint
|
|
315
|
+
|
|
316
|
+
# Text scale / padding within the sampled canvas height
|
|
317
|
+
font_size_mode: str = "fixed" # "fixed" | "proportional"
|
|
318
|
+
min_font_scale: float = 0.65
|
|
319
|
+
max_font_scale: float = 0.9
|
|
320
|
+
vertical_padding_mode: str = "fixed" # "fixed" | "random"
|
|
321
|
+
min_vertical_padding_ratio: float = 0.04
|
|
322
|
+
max_vertical_padding_ratio: float = 0.18
|
|
323
|
+
|
|
324
|
+
# Optional per-sample metadata sidecar
|
|
325
|
+
record_metadata: bool = False
|
|
326
|
+
|
|
327
|
+
# Font / language
|
|
328
|
+
language: str = "mixed" # corpus language tag; legacy alias for a font pool
|
|
329
|
+
multilingual: bool = False
|
|
330
|
+
cluster_font_fallback: bool = False
|
|
331
|
+
font_pool: str | None = None # named top-level fonts subdirectory; None = auto/all
|
|
332
|
+
direction: str = "auto" # auto | ltr | rtl (horizontal only)
|
|
333
|
+
shaping_language: str | None = None # optional BCP-47 tag for RAQM
|
|
334
|
+
fonts_dir: str = "fonts"
|
|
335
|
+
mixed_font_prob: float = 0.0
|
|
336
|
+
font_mode: str = "random" # "random" | "all"
|
|
337
|
+
copies: int = 3 # augmented copies per line when font_mode=random
|
|
338
|
+
|
|
339
|
+
# Corpus
|
|
340
|
+
corpus_path: str = ""
|
|
341
|
+
min_length: int = 1
|
|
342
|
+
max_length: int = 260
|
|
343
|
+
max_lines: int = 0
|
|
344
|
+
val_percent: float | None = None
|
|
345
|
+
test_percent: float | None = None
|
|
346
|
+
split_ratios: tuple[float, float, float] | None = None
|
|
347
|
+
seed: int | None = 42
|
|
348
|
+
oversample_rare_chars: bool = False
|
|
349
|
+
rare_char_percentile: float = 5.0
|
|
350
|
+
rare_char_multiplier: float = 3.0
|
|
351
|
+
|
|
352
|
+
# Output
|
|
353
|
+
output_dir: str = "data"
|
|
354
|
+
vocab_path: str = ""
|
|
355
|
+
skip_vocab: bool = False
|
|
356
|
+
append: bool = False
|
|
357
|
+
overwrite: bool = False
|
|
358
|
+
output_format: str = "jpg" # "png" | "jpg" | "tiff"
|
|
359
|
+
jpeg_quality: int = 90 # JPEG quality for jpg output
|
|
360
|
+
storage: str = "raw" # "raw" | "lmdb" | "both"
|
|
361
|
+
keep_raw: bool = False # legacy; prefer storage="both"
|
|
362
|
+
bg_color_mode: str = (
|
|
363
|
+
"random" # "default" | "paper_tones" | "colored" | "dark_mode" | "gradient" | "random"
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
# Worker reliability
|
|
367
|
+
workers: int = 0 # 0 = auto
|
|
368
|
+
worker_timeout: int = 300 # seconds; 0 = no timeout
|
|
369
|
+
retry_limit: int = 10
|
|
370
|
+
|
|
371
|
+
# LMDB packing
|
|
372
|
+
pack_lmdb: bool = False
|
|
373
|
+
lmdb_jpeg_quality: int = 90
|
|
374
|
+
lmdb_map_size_gb: int = 256
|
|
375
|
+
|
|
376
|
+
# Opt-in guard, measured on visible line ink after output sizing; 0 disables it.
|
|
377
|
+
min_text_height_for_augmentation: int = 0
|
|
378
|
+
min_glyph_height_for_augmentation: int = 0
|
|
379
|
+
script_intensity_scales: dict[str, float] = field(default_factory=dict)
|
|
380
|
+
font_intensity_scales: dict[str, float] = field(default_factory=dict)
|
|
381
|
+
clean_prob: float = 0.0 # absolute probability before weighted method selection
|
|
382
|
+
|
|
383
|
+
# Augmentation methods - each has {prob, min, max}
|
|
384
|
+
|
|
385
|
+
# Sauvola local-threshold degradation
|
|
386
|
+
sauvola: AugMethodConfig = field(
|
|
387
|
+
default_factory=lambda: AugMethodConfig(prob=0.2, min=0.1, max=0.9)
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
# 4-point perspective (geometric) warp
|
|
391
|
+
geo_warp: AugMethodConfig = field(
|
|
392
|
+
default_factory=lambda: AugMethodConfig(prob=0.2, min=0.1, max=0.9)
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
# Random vertical crop
|
|
396
|
+
vertical_crop: AugMethodConfig = field(
|
|
397
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
# Albumentations blur branch (MotionBlur | MedianBlur | GaussianBlur)
|
|
401
|
+
blur: AugMethodConfig = field(
|
|
402
|
+
default_factory=lambda: AugMethodConfig(prob=0.4, min=0.1, max=0.9)
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
# Albumentations distortion (Optical | Grid | Elastic)
|
|
406
|
+
distortion: AugMethodConfig = field(
|
|
407
|
+
default_factory=lambda: AugMethodConfig(prob=0.3, min=0.1, max=0.9)
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
# Albumentations noise (GaussNoise | MultiplicativeNoise)
|
|
411
|
+
albu_noise: AugMethodConfig = field(
|
|
412
|
+
default_factory=lambda: AugMethodConfig(prob=0.4, min=0.1, max=0.9)
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
# JPEG compression artifacts
|
|
416
|
+
jpeg_compression: AugMethodConfig = field(
|
|
417
|
+
default_factory=lambda: AugMethodConfig(prob=0.4, min=0.1, max=0.9)
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
# Random rotation
|
|
421
|
+
rotation: AugMethodConfig = field(
|
|
422
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
# Salt-and-pepper noise
|
|
426
|
+
salt_pepper: AugMethodConfig = field(
|
|
427
|
+
default_factory=lambda: AugMethodConfig(prob=0.15, min=0.1, max=0.9)
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
# Background texture overlay
|
|
431
|
+
background_texture: AugMethodConfig = field(
|
|
432
|
+
default_factory=lambda: AugMethodConfig(prob=0.35, min=0.1, max=0.9)
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
# Low-DPI rendering simulation
|
|
436
|
+
lowdpi: AugMethodConfig = field(
|
|
437
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
# Oversample rendering
|
|
441
|
+
oversample: AugMethodConfig = field(
|
|
442
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
# Extreme source-resolution simulation (min/max are pixel heights, not [0, 1])
|
|
446
|
+
extreme_resize: AugMethodConfig = field(
|
|
447
|
+
default_factory=lambda: ExtremeResizeConfig(prob=0.0, min=8, max=1920)
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
# Low-contrast small caption text
|
|
451
|
+
low_contrast_caption: AugMethodConfig = field(
|
|
452
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
# Perspective warp
|
|
456
|
+
perspective: AugMethodConfig = field(
|
|
457
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
# Elastic distortion
|
|
461
|
+
elastic: AugMethodConfig = field(
|
|
462
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
463
|
+
)
|
|
464
|
+
|
|
465
|
+
# Random height crop
|
|
466
|
+
random_crop: AugMethodConfig = field(
|
|
467
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
# Gaussian/Motion blur
|
|
471
|
+
online_blur: AugMethodConfig = field(
|
|
472
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
# Gaussian noise
|
|
476
|
+
online_noise: AugMethodConfig = field(
|
|
477
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
# HSV color jitter
|
|
481
|
+
hsv: AugMethodConfig = field(
|
|
482
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
# Color reversal
|
|
486
|
+
reverse: AugMethodConfig = field(
|
|
487
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.0, max=1.0)
|
|
488
|
+
)
|
|
489
|
+
|
|
490
|
+
# Brightness/contrast jitter
|
|
491
|
+
brightness_contrast: AugMethodConfig = field(
|
|
492
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
# Pixelation (downscale-upscale)
|
|
496
|
+
pixelation: AugMethodConfig = field(
|
|
497
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
# Gradient illumination overlay
|
|
501
|
+
gradient_illumination: AugMethodConfig = field(
|
|
502
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
# Morphological erode/dilate
|
|
506
|
+
morphological: AugMethodConfig = field(
|
|
507
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
# Anisotropic dilation (dot-matrix spread)
|
|
511
|
+
anisotropic_dilation: AugMethodConfig = field(
|
|
512
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.9)
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
# RGB near-white fill + colored bloom (needs color-mode 3). Grayscale keeps the dark core.
|
|
516
|
+
luminous_halo: AugMethodConfig = field(
|
|
517
|
+
default_factory=lambda: AugMethodConfig(prob=0.0, min=0.1, max=0.75)
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
# Normalizer config
|
|
521
|
+
normalizer: NormalizerConfig = field(default_factory=NormalizerConfig)
|
|
522
|
+
|
|
523
|
+
# Text decorations
|
|
524
|
+
text_deco: TextDecorationConfig = field(default_factory=TextDecorationConfig)
|
|
525
|
+
|
|
526
|
+
# Text effects (shadow, glow, outline, ...)
|
|
527
|
+
text_effect: TextEffectConfig = field(default_factory=TextEffectConfig)
|
|
528
|
+
|
|
529
|
+
def __post_init__(self) -> None:
|
|
530
|
+
self.script_intensity_scales = self._parse_script_intensity_scales(self.script_intensity_scales)
|
|
531
|
+
self.font_intensity_scales = self._parse_font_intensity_scales(self.font_intensity_scales)
|
|
532
|
+
if not 0.0 <= self.clean_prob <= 1.0:
|
|
533
|
+
raise ValueError("clean_prob must be between 0 and 1")
|
|
534
|
+
if self.direction not in {"auto", "ltr", "rtl"}:
|
|
535
|
+
raise ValueError("direction must be auto, ltr, or rtl")
|
|
536
|
+
if self.multilingual:
|
|
537
|
+
if self.mixed_font_prob > 0:
|
|
538
|
+
raise ValueError(
|
|
539
|
+
"multilingual mode requires whole-line fonts; mixed_font_prob must be 0"
|
|
540
|
+
)
|
|
541
|
+
if self.text_deco.enabled or self.text_effect.enabled:
|
|
542
|
+
raise ValueError(
|
|
543
|
+
"multilingual mode currently supports plain whole-line text; disable text decorations/effects (including codepoint-split sub/superscripts)"
|
|
544
|
+
)
|
|
545
|
+
if (
|
|
546
|
+
isinstance(self.min_text_height_for_augmentation, bool)
|
|
547
|
+
or not isinstance(self.min_text_height_for_augmentation, int)
|
|
548
|
+
or self.min_text_height_for_augmentation < 0
|
|
549
|
+
):
|
|
550
|
+
raise ValueError("min_text_height_for_augmentation must be a non-negative integer")
|
|
551
|
+
if (
|
|
552
|
+
isinstance(self.min_glyph_height_for_augmentation, bool)
|
|
553
|
+
or not isinstance(self.min_glyph_height_for_augmentation, int)
|
|
554
|
+
or self.min_glyph_height_for_augmentation < 0
|
|
555
|
+
):
|
|
556
|
+
raise ValueError("min_glyph_height_for_augmentation must be a non-negative integer")
|
|
557
|
+
|
|
558
|
+
# Split ratio resolution
|
|
559
|
+
|
|
560
|
+
def resolve_split_ratios(self) -> tuple[float, float, float]:
|
|
561
|
+
"""Resolve (train_ratio, val_ratio, test_ratio) from config fields.
|
|
562
|
+
|
|
563
|
+
Resolution rules (highest priority first):
|
|
564
|
+
|
|
565
|
+
1. ``split_ratios`` set explicitly:
|
|
566
|
+
Normalise the three values so they sum to 1.0.
|
|
567
|
+
Example: ``split_ratios=(70, 15, 15)`` -> ``(0.70, 0.15, 0.15)``.
|
|
568
|
+
|
|
569
|
+
2. Both ``val_percent`` and ``test_percent`` set:
|
|
570
|
+
``train = 1 - val/100 - test/100``.
|
|
571
|
+
|
|
572
|
+
3. Only ``val_percent`` set:
|
|
573
|
+
``test = 0``, ``train = 1 - val/100``.
|
|
574
|
+
Example: ``--val-percent 15`` -> ``(0.85, 0.15, 0.0)``.
|
|
575
|
+
|
|
576
|
+
4. Only ``test_percent`` set:
|
|
577
|
+
``val = 0``, ``train = 1 - test/100``.
|
|
578
|
+
Example: ``--test-percent 15`` -> ``(0.85, 0.0, 0.15)``.
|
|
579
|
+
|
|
580
|
+
5. Neither set (default):
|
|
581
|
+
``(0.80, 0.10, 0.10)``.
|
|
582
|
+
|
|
583
|
+
To disable all splitting (100 % train), set:
|
|
584
|
+
- ``--split-ratios 100 0 0``, or
|
|
585
|
+
- ``--val-percent 0 --test-percent 0``.
|
|
586
|
+
"""
|
|
587
|
+
if self.split_ratios is not None:
|
|
588
|
+
a, b, c = self.split_ratios
|
|
589
|
+
total = a + b + c
|
|
590
|
+
if total <= 0.0:
|
|
591
|
+
return (1.0, 0.0, 0.0)
|
|
592
|
+
return (a / total, b / total, c / total)
|
|
593
|
+
|
|
594
|
+
val_set = self.val_percent is not None
|
|
595
|
+
test_set = self.test_percent is not None
|
|
596
|
+
|
|
597
|
+
if val_set and test_set:
|
|
598
|
+
v = max(0.0, min(100.0, float(self.val_percent))) # type: ignore[arg-type]
|
|
599
|
+
t = max(0.0, min(100.0, float(self.test_percent))) # type: ignore[arg-type]
|
|
600
|
+
tr = max(0.0, 100.0 - v - t)
|
|
601
|
+
total = tr + v + t
|
|
602
|
+
return (tr / total, v / total, t / total)
|
|
603
|
+
|
|
604
|
+
if val_set:
|
|
605
|
+
v = max(0.0, min(100.0, float(self.val_percent))) # type: ignore[arg-type]
|
|
606
|
+
tr = max(0.0, 100.0 - v)
|
|
607
|
+
return (
|
|
608
|
+
tr / (tr + v) if (tr + v) > 0 else 1.0,
|
|
609
|
+
v / (tr + v) if (tr + v) > 0 else 0.0,
|
|
610
|
+
0.0,
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
if test_set:
|
|
614
|
+
t = max(0.0, min(100.0, float(self.test_percent))) # type: ignore[arg-type]
|
|
615
|
+
tr = max(0.0, 100.0 - t)
|
|
616
|
+
return (
|
|
617
|
+
tr / (tr + t) if (tr + t) > 0 else 1.0,
|
|
618
|
+
0.0,
|
|
619
|
+
t / (tr + t) if (tr + t) > 0 else 0.0,
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
# Default: 80/10/10
|
|
623
|
+
return (0.80, 0.10, 0.10)
|
|
624
|
+
|
|
625
|
+
# CLI
|
|
626
|
+
|
|
627
|
+
# Helper to build AugMethodConfig CLI flags
|
|
628
|
+
_AUG_METHODS: ClassVar[tuple[tuple[str, str], ...]] = (
|
|
629
|
+
# (attr_name, display_name)
|
|
630
|
+
("sauvola", "Sauvola threshold degradation"),
|
|
631
|
+
("geo_warp", "Geometric (4-point) perspective warp"),
|
|
632
|
+
("vertical_crop", "Random vertical crop"),
|
|
633
|
+
("blur", "Blur (MotionBlur|MedianBlur|GaussianBlur via Albumentations)"),
|
|
634
|
+
("distortion", "Distortion (Optical|Grid|Elastic via Albumentations)"),
|
|
635
|
+
("albu_noise", "Noise (GaussNoise|MultiplicativeNoise via Albumentations)"),
|
|
636
|
+
("jpeg_compression", "JPEG compression artifacts"),
|
|
637
|
+
("rotation", "Random rotation"),
|
|
638
|
+
("salt_pepper", "Salt-and-pepper impulse noise"),
|
|
639
|
+
("background_texture", "Background texture overlay"),
|
|
640
|
+
("lowdpi", "Low-DPI rendering simulation"),
|
|
641
|
+
("oversample", "Oversample rendering"),
|
|
642
|
+
(
|
|
643
|
+
"extreme_resize",
|
|
644
|
+
"Extreme source-resolution simulation (isotropic resize to a very small/large height)",
|
|
645
|
+
),
|
|
646
|
+
("low_contrast_caption", "Low-contrast small caption text"),
|
|
647
|
+
("perspective", "Perspective warp"),
|
|
648
|
+
("elastic", "Elastic distortion"),
|
|
649
|
+
("random_crop", "Random height crop"),
|
|
650
|
+
("online_blur", "Gaussian/Motion blur"),
|
|
651
|
+
("online_noise", "Gaussian noise"),
|
|
652
|
+
("hsv", "HSV color jitter"),
|
|
653
|
+
("reverse", "Color reversal (dark to light)"),
|
|
654
|
+
("brightness_contrast", "Brightness/contrast jitter"),
|
|
655
|
+
("pixelation", "Pixelation (downscale-then-upscale)"),
|
|
656
|
+
("gradient_illumination", "Gradient illumination overlay"),
|
|
657
|
+
("morphological", "Morphological erode/dilate"),
|
|
658
|
+
("anisotropic_dilation", "Anisotropic dilation (dot-matrix spread)"),
|
|
659
|
+
(
|
|
660
|
+
"luminous_halo",
|
|
661
|
+
"Luminous halo (RGB near-white fill and colored bloom; grayscale keeps the dark core)",
|
|
662
|
+
),
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
# Methods whose config class isn't the generic AugMethodConfig (e.g.
|
|
666
|
+
# extreme_resize's min/max are absolute pixel heights, not [0, 1]).
|
|
667
|
+
# from_args/from_dict must reconstruct via the right class, or the base
|
|
668
|
+
# class's __post_init__ clamps min/max back to [0, 1].
|
|
669
|
+
_AUG_METHOD_CLASSES: ClassVar[dict[str, type]] = {"extreme_resize": ExtremeResizeConfig}
|
|
670
|
+
|
|
671
|
+
@staticmethod
|
|
672
|
+
def add_args(parser: argparse.ArgumentParser) -> None:
|
|
673
|
+
"""Register all CLI arguments for generation."""
|
|
674
|
+
|
|
675
|
+
# Rendering
|
|
676
|
+
g_render = parser.add_argument_group("Rendering")
|
|
677
|
+
g_render.add_argument(
|
|
678
|
+
"--multilingual",
|
|
679
|
+
action="store_true",
|
|
680
|
+
help="Use RAQM whole-line shaping and exhaustive font coverage selection",
|
|
681
|
+
)
|
|
682
|
+
g_render.add_argument(
|
|
683
|
+
"--cluster-font-fallback",
|
|
684
|
+
action="store_true",
|
|
685
|
+
default=False,
|
|
686
|
+
help="Enable cluster-aware font fallback across multiple fonts using RAQM",
|
|
687
|
+
)
|
|
688
|
+
g_render.add_argument(
|
|
689
|
+
"--language",
|
|
690
|
+
default="mixed",
|
|
691
|
+
help="Corpus language tag (default: mixed); also a legacy font-pool alias",
|
|
692
|
+
)
|
|
693
|
+
g_render.add_argument(
|
|
694
|
+
"--font-pool",
|
|
695
|
+
default=None,
|
|
696
|
+
help="Restrict to a named fonts subdirectory; multilingual defaults to all discovered fonts",
|
|
697
|
+
)
|
|
698
|
+
g_render.add_argument(
|
|
699
|
+
"--direction",
|
|
700
|
+
choices=["auto", "ltr", "rtl"],
|
|
701
|
+
default="auto",
|
|
702
|
+
help="Horizontal base direction in multilingual mode",
|
|
703
|
+
)
|
|
704
|
+
g_render.add_argument(
|
|
705
|
+
"--shaping-language",
|
|
706
|
+
default=None,
|
|
707
|
+
help="Optional BCP-47 shaping language in multilingual mode",
|
|
708
|
+
)
|
|
709
|
+
g_render.add_argument(
|
|
710
|
+
"--fonts",
|
|
711
|
+
default="fonts/",
|
|
712
|
+
help="Root fonts directory: <dir>/khmer/ and <dir>/english/ subdirs",
|
|
713
|
+
)
|
|
714
|
+
g_render.add_argument(
|
|
715
|
+
"--height",
|
|
716
|
+
type=int,
|
|
717
|
+
default=48,
|
|
718
|
+
metavar="PX",
|
|
719
|
+
help="Image height in pixels (default: 48)",
|
|
720
|
+
)
|
|
721
|
+
g_render.add_argument(
|
|
722
|
+
"--width",
|
|
723
|
+
type=int,
|
|
724
|
+
default=None,
|
|
725
|
+
metavar="PX",
|
|
726
|
+
help="Fixed image width; omit for variable width",
|
|
727
|
+
)
|
|
728
|
+
g_render.add_argument(
|
|
729
|
+
"--color-mode",
|
|
730
|
+
type=int,
|
|
731
|
+
choices=[1, 3],
|
|
732
|
+
default=1,
|
|
733
|
+
metavar="{1,3}",
|
|
734
|
+
help="Output colour channels: 1 = grayscale, 3 = RGB (default: 1)",
|
|
735
|
+
)
|
|
736
|
+
g_render.add_argument(
|
|
737
|
+
"--random-align-when-padded",
|
|
738
|
+
action="store_true",
|
|
739
|
+
help="Random left/center/right alignment when --width pads the image",
|
|
740
|
+
)
|
|
741
|
+
|
|
742
|
+
# Variable line height
|
|
743
|
+
g_vh = parser.add_argument_group("Variable line height")
|
|
744
|
+
g_vh.add_argument(
|
|
745
|
+
"--line-height-mode",
|
|
746
|
+
choices=["fixed", "variable", "bucketed"],
|
|
747
|
+
default="fixed",
|
|
748
|
+
help=(
|
|
749
|
+
"'fixed' = every image uses --height (default, backward compatible). "
|
|
750
|
+
"'variable' = sample a height per image from [--min-line-height, "
|
|
751
|
+
"--max-line-height]. 'bucketed' = sample from the fixed set of heights "
|
|
752
|
+
"spaced by --line-height-step across that range."
|
|
753
|
+
),
|
|
754
|
+
)
|
|
755
|
+
g_vh.add_argument(
|
|
756
|
+
"--min-line-height",
|
|
757
|
+
type=int,
|
|
758
|
+
default=32,
|
|
759
|
+
metavar="PX",
|
|
760
|
+
help="Minimum sampled line height in pixels (default: 32)",
|
|
761
|
+
)
|
|
762
|
+
g_vh.add_argument(
|
|
763
|
+
"--max-line-height",
|
|
764
|
+
type=int,
|
|
765
|
+
default=96,
|
|
766
|
+
metavar="PX",
|
|
767
|
+
help="Maximum sampled line height in pixels (default: 96)",
|
|
768
|
+
)
|
|
769
|
+
g_vh.add_argument(
|
|
770
|
+
"--line-height-step",
|
|
771
|
+
type=int,
|
|
772
|
+
default=8,
|
|
773
|
+
metavar="PX",
|
|
774
|
+
help="Align sampled/bucketed heights to a multiple of this many pixels (default: 8)",
|
|
775
|
+
)
|
|
776
|
+
g_vh.add_argument(
|
|
777
|
+
"--line-height-distribution",
|
|
778
|
+
choices=["uniform", "triangular"],
|
|
779
|
+
default="uniform",
|
|
780
|
+
help=(
|
|
781
|
+
"Shape of the height distribution for --line-height-mode=variable. "
|
|
782
|
+
"'triangular' clusters samples around --default-line-height (default: uniform)"
|
|
783
|
+
),
|
|
784
|
+
)
|
|
785
|
+
g_vh.add_argument(
|
|
786
|
+
"--default-line-height",
|
|
787
|
+
type=int,
|
|
788
|
+
default=None,
|
|
789
|
+
metavar="PX",
|
|
790
|
+
help="Peak height for the triangular distribution (default: midpoint of the range)",
|
|
791
|
+
)
|
|
792
|
+
g_vh.add_argument(
|
|
793
|
+
"--font-size-mode",
|
|
794
|
+
choices=["fixed", "proportional"],
|
|
795
|
+
default="fixed",
|
|
796
|
+
help=(
|
|
797
|
+
"'fixed' = choose from the preloaded font sizes (default). "
|
|
798
|
+
"'proportional' = choose a font size scaled to the sampled canvas height "
|
|
799
|
+
"(--min-font-scale..--max-font-scale)"
|
|
800
|
+
),
|
|
801
|
+
)
|
|
802
|
+
g_vh.add_argument(
|
|
803
|
+
"--min-font-scale",
|
|
804
|
+
type=float,
|
|
805
|
+
default=0.65,
|
|
806
|
+
metavar="F",
|
|
807
|
+
help="Minimum glyph height as a fraction of canvas height when font-size-mode=proportional (default: 0.65)",
|
|
808
|
+
)
|
|
809
|
+
g_vh.add_argument(
|
|
810
|
+
"--max-font-scale",
|
|
811
|
+
type=float,
|
|
812
|
+
default=0.9,
|
|
813
|
+
metavar="F",
|
|
814
|
+
help="Maximum glyph height as a fraction of canvas height when font-size-mode=proportional (default: 0.9)",
|
|
815
|
+
)
|
|
816
|
+
g_vh.add_argument(
|
|
817
|
+
"--vertical-padding-mode",
|
|
818
|
+
choices=["fixed", "random"],
|
|
819
|
+
default="fixed",
|
|
820
|
+
help=(
|
|
821
|
+
"'fixed' = existing constant pixel padding (default). "
|
|
822
|
+
"'random' = sample top/bottom padding as a ratio of canvas height"
|
|
823
|
+
),
|
|
824
|
+
)
|
|
825
|
+
g_vh.add_argument(
|
|
826
|
+
"--min-vertical-padding-ratio",
|
|
827
|
+
type=float,
|
|
828
|
+
default=0.04,
|
|
829
|
+
metavar="F",
|
|
830
|
+
help="Minimum vertical padding as a fraction of canvas height (default: 0.04)",
|
|
831
|
+
)
|
|
832
|
+
g_vh.add_argument(
|
|
833
|
+
"--max-vertical-padding-ratio",
|
|
834
|
+
type=float,
|
|
835
|
+
default=0.18,
|
|
836
|
+
metavar="F",
|
|
837
|
+
help="Maximum vertical padding as a fraction of canvas height (default: 0.18)",
|
|
838
|
+
)
|
|
839
|
+
g_vh.add_argument(
|
|
840
|
+
"--record-metadata",
|
|
841
|
+
action="store_true",
|
|
842
|
+
help="Write a metadata.jsonl sidecar (image/text/width/height) per split",
|
|
843
|
+
)
|
|
844
|
+
g_render.add_argument(
|
|
845
|
+
"--font-mode",
|
|
846
|
+
choices=["random", "all"],
|
|
847
|
+
default="random",
|
|
848
|
+
help="'random' = N augmented copies; 'all' = one image per font per line",
|
|
849
|
+
)
|
|
850
|
+
g_render.add_argument(
|
|
851
|
+
"--copies",
|
|
852
|
+
type=int,
|
|
853
|
+
default=3,
|
|
854
|
+
metavar="N",
|
|
855
|
+
help="Augmented copies per text line when --font-mode=random (default: 3)",
|
|
856
|
+
)
|
|
857
|
+
g_render.add_argument(
|
|
858
|
+
"--mixed-font-prob",
|
|
859
|
+
type=float,
|
|
860
|
+
default=0.0,
|
|
861
|
+
metavar="F",
|
|
862
|
+
help="Probability of per-span font rendering for mixed Khmer/English text",
|
|
863
|
+
)
|
|
864
|
+
g_render.add_argument(
|
|
865
|
+
"--retry-limit",
|
|
866
|
+
type=int,
|
|
867
|
+
default=10,
|
|
868
|
+
metavar="N",
|
|
869
|
+
help="Font selection retries per image when font lacks glyphs",
|
|
870
|
+
)
|
|
871
|
+
|
|
872
|
+
# Corpus
|
|
873
|
+
g_corpus = parser.add_argument_group("Corpus")
|
|
874
|
+
g_corpus.add_argument(
|
|
875
|
+
"--corpus",
|
|
876
|
+
default="corpus/corpus.txt",
|
|
877
|
+
metavar="FILE",
|
|
878
|
+
help="Path to plain text corpus file",
|
|
879
|
+
)
|
|
880
|
+
g_corpus.add_argument(
|
|
881
|
+
"--min-length",
|
|
882
|
+
type=int,
|
|
883
|
+
default=1,
|
|
884
|
+
metavar="N",
|
|
885
|
+
help="Minimum character length to include (default: 1)",
|
|
886
|
+
)
|
|
887
|
+
g_corpus.add_argument(
|
|
888
|
+
"--max-length",
|
|
889
|
+
type=int,
|
|
890
|
+
default=260,
|
|
891
|
+
metavar="N",
|
|
892
|
+
help="Maximum character length to include (default: 260)",
|
|
893
|
+
)
|
|
894
|
+
g_corpus.add_argument(
|
|
895
|
+
"--lines",
|
|
896
|
+
type=int,
|
|
897
|
+
default=0,
|
|
898
|
+
metavar="N",
|
|
899
|
+
help="Max lines to use after filtering; 0 = all",
|
|
900
|
+
)
|
|
901
|
+
def _parse_optional_seed(val: Any) -> int | None:
|
|
902
|
+
if val is None or str(val).strip().lower() in ("none", "null", "nil", ""):
|
|
903
|
+
return None
|
|
904
|
+
return int(val)
|
|
905
|
+
|
|
906
|
+
g_corpus.add_argument(
|
|
907
|
+
"--seed",
|
|
908
|
+
type=_parse_optional_seed,
|
|
909
|
+
default=42,
|
|
910
|
+
metavar="N",
|
|
911
|
+
help="Random seed for deterministic train/val/test splitting and per-sample rendering",
|
|
912
|
+
)
|
|
913
|
+
g_corpus.add_argument(
|
|
914
|
+
"--val-percent",
|
|
915
|
+
type=float,
|
|
916
|
+
default=None,
|
|
917
|
+
metavar="PCT",
|
|
918
|
+
help=(
|
|
919
|
+
"Validation split percentage [0, 100). "
|
|
920
|
+
"If only this is set, test=0. "
|
|
921
|
+
"If neither --val-percent nor --test-percent is set, defaults to 10%%."
|
|
922
|
+
),
|
|
923
|
+
)
|
|
924
|
+
g_corpus.add_argument(
|
|
925
|
+
"--test-percent",
|
|
926
|
+
type=float,
|
|
927
|
+
default=None,
|
|
928
|
+
metavar="PCT",
|
|
929
|
+
help=(
|
|
930
|
+
"Test split percentage [0, 100). "
|
|
931
|
+
"If only this is set, val=0. "
|
|
932
|
+
"If neither --val-percent nor --test-percent is set, defaults to 10%%."
|
|
933
|
+
),
|
|
934
|
+
)
|
|
935
|
+
g_corpus.add_argument(
|
|
936
|
+
"--split-ratios",
|
|
937
|
+
nargs=3,
|
|
938
|
+
type=float,
|
|
939
|
+
default=None,
|
|
940
|
+
metavar=("TRAIN", "VAL", "TEST"),
|
|
941
|
+
help=(
|
|
942
|
+
"Explicit train/val/test split ratios (three values, e.g. 80 10 10). "
|
|
943
|
+
"Values are normalised to sum to 1.0. "
|
|
944
|
+
"Overrides --val-percent and --test-percent. "
|
|
945
|
+
"Set '100 0 0' to disable splitting entirely."
|
|
946
|
+
),
|
|
947
|
+
)
|
|
948
|
+
g_corpus.add_argument(
|
|
949
|
+
"--test-file",
|
|
950
|
+
default=None,
|
|
951
|
+
metavar="FILE",
|
|
952
|
+
help="Path to a separate test corpus file (bypasses automatic test splitting)",
|
|
953
|
+
)
|
|
954
|
+
g_corpus.add_argument(
|
|
955
|
+
"--count-only",
|
|
956
|
+
action="store_true",
|
|
957
|
+
help="Print filter statistics and estimated image count, then exit",
|
|
958
|
+
)
|
|
959
|
+
g_corpus.add_argument(
|
|
960
|
+
"--oversample-rare-chars",
|
|
961
|
+
action="store_true",
|
|
962
|
+
help=(
|
|
963
|
+
"Render extra copies of training lines that contain rare characters "
|
|
964
|
+
"(does not affect val/test, which stay at 1 copy per line)"
|
|
965
|
+
),
|
|
966
|
+
)
|
|
967
|
+
g_corpus.add_argument(
|
|
968
|
+
"--rare-char-percentile",
|
|
969
|
+
type=float,
|
|
970
|
+
default=5.0,
|
|
971
|
+
metavar="PCT",
|
|
972
|
+
help=(
|
|
973
|
+
"Least-frequent PCT%% of distinct characters in the corpus are "
|
|
974
|
+
"considered rare (default: 5.0)"
|
|
975
|
+
),
|
|
976
|
+
)
|
|
977
|
+
g_corpus.add_argument(
|
|
978
|
+
"--rare-char-multiplier",
|
|
979
|
+
type=float,
|
|
980
|
+
default=3.0,
|
|
981
|
+
metavar="F",
|
|
982
|
+
help=(
|
|
983
|
+
"Copies multiplier applied to training lines containing a rare "
|
|
984
|
+
"character (default: 3.0)"
|
|
985
|
+
),
|
|
986
|
+
)
|
|
987
|
+
g_corpus.add_argument(
|
|
988
|
+
"--image-dir",
|
|
989
|
+
default=None,
|
|
990
|
+
metavar="DIR",
|
|
991
|
+
help="Path to existing images directory (bypass text rendering; "
|
|
992
|
+
"corpus must be labels.txt file)",
|
|
993
|
+
)
|
|
994
|
+
|
|
995
|
+
# Output
|
|
996
|
+
g_out = parser.add_argument_group("Output")
|
|
997
|
+
g_out.add_argument(
|
|
998
|
+
"--output",
|
|
999
|
+
"--output-dir",
|
|
1000
|
+
dest="output",
|
|
1001
|
+
default="data",
|
|
1002
|
+
metavar="DIR",
|
|
1003
|
+
help="Output directory for generated dataset (default: data/)",
|
|
1004
|
+
)
|
|
1005
|
+
g_out_grp = g_out.add_mutually_exclusive_group()
|
|
1006
|
+
g_out_grp.add_argument(
|
|
1007
|
+
"--append",
|
|
1008
|
+
action="store_true",
|
|
1009
|
+
help="Append new samples if output directory exists",
|
|
1010
|
+
)
|
|
1011
|
+
g_out_grp.add_argument(
|
|
1012
|
+
"--overwrite",
|
|
1013
|
+
action="store_true",
|
|
1014
|
+
help="Delete and recreate if output directory exists",
|
|
1015
|
+
)
|
|
1016
|
+
g_out.add_argument(
|
|
1017
|
+
"--vocab",
|
|
1018
|
+
default="",
|
|
1019
|
+
metavar="FILE",
|
|
1020
|
+
help="Path to write vocab.json; auto-derived if not set",
|
|
1021
|
+
)
|
|
1022
|
+
g_out.add_argument(
|
|
1023
|
+
"--skip-vocab",
|
|
1024
|
+
action="store_true",
|
|
1025
|
+
help="Do not build vocab.json after generation",
|
|
1026
|
+
)
|
|
1027
|
+
g_out.add_argument(
|
|
1028
|
+
"--output-format",
|
|
1029
|
+
choices=["png", "jpg", "tiff"],
|
|
1030
|
+
default="jpg",
|
|
1031
|
+
help="Output image format (default: jpg)",
|
|
1032
|
+
)
|
|
1033
|
+
g_out.add_argument(
|
|
1034
|
+
"--jpeg-quality",
|
|
1035
|
+
type=int,
|
|
1036
|
+
default=90,
|
|
1037
|
+
metavar="N",
|
|
1038
|
+
help="JPEG quality for jpg output (default: 90; 0-100)",
|
|
1039
|
+
)
|
|
1040
|
+
g_out.add_argument(
|
|
1041
|
+
"--storage",
|
|
1042
|
+
choices=["raw", "lmdb", "both"],
|
|
1043
|
+
default="raw",
|
|
1044
|
+
help="Storage mode: raw = image files only, lmdb = pack and delete, both = save + pack + keep (default: raw)",
|
|
1045
|
+
)
|
|
1046
|
+
|
|
1047
|
+
# LMDB packing
|
|
1048
|
+
g_lmdb = parser.add_argument_group("LMDB Packing")
|
|
1049
|
+
g_lmdb.add_argument(
|
|
1050
|
+
"--pack-lmdb",
|
|
1051
|
+
action="store_true",
|
|
1052
|
+
help="Pack generated dataset into LMDB databases after generation",
|
|
1053
|
+
)
|
|
1054
|
+
g_lmdb.add_argument(
|
|
1055
|
+
"--keep-raw",
|
|
1056
|
+
action="store_true",
|
|
1057
|
+
help="Keep raw image files after LMDB packing",
|
|
1058
|
+
)
|
|
1059
|
+
g_lmdb.add_argument(
|
|
1060
|
+
"--lmdb-jpeg-quality",
|
|
1061
|
+
type=int,
|
|
1062
|
+
default=90,
|
|
1063
|
+
metavar="N",
|
|
1064
|
+
help="JPEG quality for LMDB-stored images (default: 90)",
|
|
1065
|
+
)
|
|
1066
|
+
g_lmdb.add_argument(
|
|
1067
|
+
"--lmdb-map-size-gb",
|
|
1068
|
+
type=int,
|
|
1069
|
+
default=256,
|
|
1070
|
+
metavar="N",
|
|
1071
|
+
help="LMDB map size in GiB (default: 256)",
|
|
1072
|
+
)
|
|
1073
|
+
|
|
1074
|
+
# Workers
|
|
1075
|
+
g_workers = parser.add_argument_group("Workers")
|
|
1076
|
+
g_workers.add_argument(
|
|
1077
|
+
"--workers",
|
|
1078
|
+
type=int,
|
|
1079
|
+
default=0,
|
|
1080
|
+
metavar="N",
|
|
1081
|
+
help="Worker processes; 0 = auto, 1 = serial (default: 0)",
|
|
1082
|
+
)
|
|
1083
|
+
g_workers.add_argument(
|
|
1084
|
+
"--worker-timeout",
|
|
1085
|
+
type=int,
|
|
1086
|
+
default=300,
|
|
1087
|
+
metavar="SEC",
|
|
1088
|
+
help="Seconds before a worker batch times out (default: 300; 0 = no limit)",
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
# Rendering style
|
|
1093
|
+
g_style = parser.add_argument_group("Rendering style")
|
|
1094
|
+
g_style.add_argument(
|
|
1095
|
+
"--bg-color-mode",
|
|
1096
|
+
choices=["default", "paper_tones", "colored", "dark_mode", "gradient", "random"],
|
|
1097
|
+
default="random",
|
|
1098
|
+
help=(
|
|
1099
|
+
"Background colour palette for rendered images (default: random). "
|
|
1100
|
+
"'default': off-white/light-gray. "
|
|
1101
|
+
"'paper_tones': warm cream, sepia, recycled, blueprint. "
|
|
1102
|
+
"'colored': soft pastels. "
|
|
1103
|
+
"'dark_mode': dark background with light text. "
|
|
1104
|
+
"'gradient': brightness gradient overlay. "
|
|
1105
|
+
"'random': sample across all palettes when augmentation is enabled."
|
|
1106
|
+
),
|
|
1107
|
+
)
|
|
1108
|
+
|
|
1109
|
+
# Text decoration
|
|
1110
|
+
TextDecorationConfig.add_args(parser)
|
|
1111
|
+
|
|
1112
|
+
# Text effects
|
|
1113
|
+
TextEffectConfig.add_args(parser)
|
|
1114
|
+
|
|
1115
|
+
# Augmentation methods: prob, min, max per method
|
|
1116
|
+
g_aug = parser.add_argument_group(
|
|
1117
|
+
"Augmentation methods",
|
|
1118
|
+
"Each method has: --<name>-prob F, --<name>-min F, --<name>-max F\n"
|
|
1119
|
+
"One augmentation per image (isolated, not stacked).",
|
|
1120
|
+
)
|
|
1121
|
+
g_aug.add_argument(
|
|
1122
|
+
"--clean-prob",
|
|
1123
|
+
type=float,
|
|
1124
|
+
default=0.0,
|
|
1125
|
+
metavar="F",
|
|
1126
|
+
help="Probability of emitting a clean render before weighted augmentation selection (0-1; default: 0)",
|
|
1127
|
+
)
|
|
1128
|
+
g_aug.add_argument(
|
|
1129
|
+
"--min-text-height-for-augmentation",
|
|
1130
|
+
type=int,
|
|
1131
|
+
default=0,
|
|
1132
|
+
metavar="PX",
|
|
1133
|
+
help=(
|
|
1134
|
+
"Skip destructive methods below this visible line-ink height after resizing "
|
|
1135
|
+
"(0 disables the guard). Unknown text height also skips them; see augmentation docs."
|
|
1136
|
+
),
|
|
1137
|
+
)
|
|
1138
|
+
g_aug.add_argument(
|
|
1139
|
+
"--min-glyph-height-for-augmentation",
|
|
1140
|
+
type=int,
|
|
1141
|
+
default=0,
|
|
1142
|
+
metavar="PX",
|
|
1143
|
+
help=(
|
|
1144
|
+
"Skip destructive methods below this line glyph-mask height "
|
|
1145
|
+
"(0 disables the guard). Protects glyphs against decorations; does not isolate individual marks."
|
|
1146
|
+
),
|
|
1147
|
+
)
|
|
1148
|
+
g_aug.add_argument(
|
|
1149
|
+
"--script-intensity-scale",
|
|
1150
|
+
type=str,
|
|
1151
|
+
default=None,
|
|
1152
|
+
metavar="SCRIPT=FACTOR,...",
|
|
1153
|
+
dest="script_intensity_scale",
|
|
1154
|
+
help="Per-script augmentation intensity multiplier (e.g. Khmr=0.7,Latn=0.9)",
|
|
1155
|
+
)
|
|
1156
|
+
g_aug.add_argument(
|
|
1157
|
+
"--font-intensity-scale",
|
|
1158
|
+
type=str,
|
|
1159
|
+
default=None,
|
|
1160
|
+
metavar="FONT=FACTOR,...",
|
|
1161
|
+
dest="font_intensity_scale",
|
|
1162
|
+
help="Per-font multiplier by unique filename or exact path (e.g. Moul.ttf=0.8)",
|
|
1163
|
+
)
|
|
1164
|
+
for attr_name, display_name in GenerationConfig._AUG_METHODS:
|
|
1165
|
+
is_pixel_range = attr_name in GenerationConfig._AUG_METHOD_CLASSES
|
|
1166
|
+
range_unit = "pixels" if is_pixel_range else "0-1"
|
|
1167
|
+
range_metavar = "PX" if is_pixel_range else "F"
|
|
1168
|
+
g_aug.add_argument(
|
|
1169
|
+
f"--{attr_name.replace('_', '-')}-prob",
|
|
1170
|
+
type=float,
|
|
1171
|
+
default=None,
|
|
1172
|
+
metavar="F",
|
|
1173
|
+
help=f"Probability of applying {display_name} (0-1)",
|
|
1174
|
+
)
|
|
1175
|
+
g_aug.add_argument(
|
|
1176
|
+
f"--{attr_name.replace('_', '-')}-min",
|
|
1177
|
+
type=float,
|
|
1178
|
+
default=None,
|
|
1179
|
+
metavar=range_metavar,
|
|
1180
|
+
help=f"Minimum intensity for {display_name} ({range_unit})",
|
|
1181
|
+
)
|
|
1182
|
+
g_aug.add_argument(
|
|
1183
|
+
f"--{attr_name.replace('_', '-')}-max",
|
|
1184
|
+
type=float,
|
|
1185
|
+
default=None,
|
|
1186
|
+
metavar=range_metavar,
|
|
1187
|
+
help=f"Maximum intensity for {display_name} ({range_unit})",
|
|
1188
|
+
)
|
|
1189
|
+
|
|
1190
|
+
# Normalizer
|
|
1191
|
+
NormalizerConfig.add_args(parser)
|
|
1192
|
+
|
|
1193
|
+
@classmethod
|
|
1194
|
+
def from_args(cls, args: argparse.Namespace) -> GenerationConfig:
|
|
1195
|
+
"""Build a ``GenerationConfig`` from parsed CLI arguments."""
|
|
1196
|
+
|
|
1197
|
+
# Build AugMethodConfig per method from CLI flags (or use defaults)
|
|
1198
|
+
# NOTE: argparse derives `dest` from the option string by replacing
|
|
1199
|
+
# "-" with "_", so the attribute name IS the lookup key (e.g.
|
|
1200
|
+
# --background-texture-min -> dest "background_texture_min"). Do NOT
|
|
1201
|
+
# dash-ify attr_name here: doing so previously produced lookup keys
|
|
1202
|
+
# like "background-texture_min", which never matched any argparse
|
|
1203
|
+
# dest (Python identifiers can't contain "-"), so getattr() always
|
|
1204
|
+
# returned None and every multi-word method silently fell back to
|
|
1205
|
+
# its dataclass default regardless of CLI/YAML configuration.
|
|
1206
|
+
defaults = cls()
|
|
1207
|
+
aug_kwargs: dict[str, AugMethodConfig] = {}
|
|
1208
|
+
for attr_name, _ in cls._AUG_METHODS:
|
|
1209
|
+
prob = getattr(args, f"{attr_name}_prob", None)
|
|
1210
|
+
v_min = getattr(args, f"{attr_name}_min", None)
|
|
1211
|
+
v_max = getattr(args, f"{attr_name}_max", None)
|
|
1212
|
+
|
|
1213
|
+
default: AugMethodConfig = getattr(defaults, attr_name)
|
|
1214
|
+
cfg_cls = cls._AUG_METHOD_CLASSES.get(attr_name, AugMethodConfig)
|
|
1215
|
+
aug_kwargs[attr_name] = cfg_cls(
|
|
1216
|
+
prob=float(prob) if prob is not None else default.prob,
|
|
1217
|
+
min=float(v_min) if v_min is not None else default.min,
|
|
1218
|
+
max=float(v_max) if v_max is not None else default.max,
|
|
1219
|
+
)
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
config = cls(
|
|
1223
|
+
image_height=int(getattr(args, "height", 48)),
|
|
1224
|
+
image_width=getattr(args, "width", None),
|
|
1225
|
+
color_mode=int(getattr(args, "color_mode", 1)),
|
|
1226
|
+
random_align_when_padded=bool(getattr(args, "random_align_when_padded", False)),
|
|
1227
|
+
line_height_mode=str(getattr(args, "line_height_mode", "fixed")),
|
|
1228
|
+
min_line_height=int(getattr(args, "min_line_height", 32)),
|
|
1229
|
+
max_line_height=int(getattr(args, "max_line_height", 96)),
|
|
1230
|
+
line_height_step=int(getattr(args, "line_height_step", 8)),
|
|
1231
|
+
line_height_distribution=str(getattr(args, "line_height_distribution", "uniform")),
|
|
1232
|
+
default_line_height=(
|
|
1233
|
+
int(args.default_line_height)
|
|
1234
|
+
if getattr(args, "default_line_height", None) is not None
|
|
1235
|
+
else None
|
|
1236
|
+
),
|
|
1237
|
+
font_size_mode=str(getattr(args, "font_size_mode", "fixed")),
|
|
1238
|
+
min_font_scale=float(getattr(args, "min_font_scale", 0.65)),
|
|
1239
|
+
max_font_scale=float(getattr(args, "max_font_scale", 0.9)),
|
|
1240
|
+
vertical_padding_mode=str(getattr(args, "vertical_padding_mode", "fixed")),
|
|
1241
|
+
min_vertical_padding_ratio=float(getattr(args, "min_vertical_padding_ratio", 0.04)),
|
|
1242
|
+
max_vertical_padding_ratio=float(getattr(args, "max_vertical_padding_ratio", 0.18)),
|
|
1243
|
+
record_metadata=bool(getattr(args, "record_metadata", False)),
|
|
1244
|
+
language=str(getattr(args, "language", "mixed")),
|
|
1245
|
+
multilingual=bool(getattr(args, "multilingual", False)),
|
|
1246
|
+
cluster_font_fallback=bool(getattr(args, "cluster_font_fallback", False)),
|
|
1247
|
+
font_pool=getattr(args, "font_pool", None),
|
|
1248
|
+
direction=str(getattr(args, "direction", "auto")),
|
|
1249
|
+
shaping_language=getattr(args, "shaping_language", None),
|
|
1250
|
+
clean_prob=float(getattr(args, "clean_prob", 0.0)),
|
|
1251
|
+
fonts_dir=str(getattr(args, "fonts", "fonts/")),
|
|
1252
|
+
mixed_font_prob=float(getattr(args, "mixed_font_prob", 0.0)),
|
|
1253
|
+
font_mode=str(getattr(args, "font_mode", "random")),
|
|
1254
|
+
copies=int(getattr(args, "copies", 3)),
|
|
1255
|
+
min_length=int(getattr(args, "min_length", 1)),
|
|
1256
|
+
max_length=int(getattr(args, "max_length", 260)),
|
|
1257
|
+
max_lines=int(getattr(args, "lines", 0)),
|
|
1258
|
+
val_percent=getattr(args, "val_percent", None),
|
|
1259
|
+
test_percent=getattr(args, "test_percent", None),
|
|
1260
|
+
split_ratios=(
|
|
1261
|
+
tuple(args.split_ratios)
|
|
1262
|
+
if getattr(args, "split_ratios", None) is not None
|
|
1263
|
+
else None
|
|
1264
|
+
),
|
|
1265
|
+
seed=(
|
|
1266
|
+
int(getattr(args, "seed", 42))
|
|
1267
|
+
if getattr(args, "seed", 42) is not None
|
|
1268
|
+
else None
|
|
1269
|
+
),
|
|
1270
|
+
oversample_rare_chars=bool(getattr(args, "oversample_rare_chars", False)),
|
|
1271
|
+
rare_char_percentile=float(getattr(args, "rare_char_percentile", 5.0)),
|
|
1272
|
+
rare_char_multiplier=float(getattr(args, "rare_char_multiplier", 3.0)),
|
|
1273
|
+
corpus_path=str(getattr(args, "corpus", "")),
|
|
1274
|
+
output_dir=str(getattr(args, "output", "data")),
|
|
1275
|
+
vocab_path=str(getattr(args, "vocab", "")),
|
|
1276
|
+
skip_vocab=bool(getattr(args, "skip_vocab", False)),
|
|
1277
|
+
append=bool(getattr(args, "append", False)),
|
|
1278
|
+
overwrite=bool(getattr(args, "overwrite", False)),
|
|
1279
|
+
output_format=str(getattr(args, "output_format", "jpg")),
|
|
1280
|
+
jpeg_quality=int(getattr(args, "jpeg_quality", 90)),
|
|
1281
|
+
storage=_resolve_storage(args),
|
|
1282
|
+
keep_raw=bool(getattr(args, "keep_raw", False)),
|
|
1283
|
+
bg_color_mode=str(getattr(args, "bg_color_mode", "random")),
|
|
1284
|
+
workers=int(getattr(args, "workers", 0)),
|
|
1285
|
+
worker_timeout=int(getattr(args, "worker_timeout", 300)),
|
|
1286
|
+
retry_limit=int(getattr(args, "retry_limit", 10)),
|
|
1287
|
+
pack_lmdb=bool(getattr(args, "pack_lmdb", False)),
|
|
1288
|
+
lmdb_jpeg_quality=int(getattr(args, "lmdb_jpeg_quality", 90)),
|
|
1289
|
+
lmdb_map_size_gb=int(getattr(args, "lmdb_map_size_gb", 256)),
|
|
1290
|
+
min_text_height_for_augmentation=getattr(args, "min_text_height_for_augmentation", 0),
|
|
1291
|
+
min_glyph_height_for_augmentation=getattr(args, "min_glyph_height_for_augmentation", 0),
|
|
1292
|
+
script_intensity_scales=cls._parse_script_intensity_scales(
|
|
1293
|
+
getattr(args, "script_intensity_scale", None)
|
|
1294
|
+
if (
|
|
1295
|
+
getattr(args, "script_intensity_scale", None) is not None
|
|
1296
|
+
and getattr(args, "script_intensity_scale", None) != ""
|
|
1297
|
+
)
|
|
1298
|
+
else getattr(args, "script_intensity_scales", None)
|
|
1299
|
+
),
|
|
1300
|
+
font_intensity_scales=cls._parse_font_intensity_scales(
|
|
1301
|
+
getattr(args, "font_intensity_scale", None)
|
|
1302
|
+
if (
|
|
1303
|
+
getattr(args, "font_intensity_scale", None) is not None
|
|
1304
|
+
and getattr(args, "font_intensity_scale", None) != ""
|
|
1305
|
+
)
|
|
1306
|
+
else getattr(args, "font_intensity_scales", None)
|
|
1307
|
+
),
|
|
1308
|
+
text_deco=TextDecorationConfig.from_args(args),
|
|
1309
|
+
text_effect=TextEffectConfig.from_args(args),
|
|
1310
|
+
normalizer=NormalizerConfig.from_args(args),
|
|
1311
|
+
)
|
|
1312
|
+
for attr_name, aug_cfg in aug_kwargs.items():
|
|
1313
|
+
setattr(config, attr_name, aug_cfg)
|
|
1314
|
+
return config
|
|
1315
|
+
|
|
1316
|
+
# Introspection helpers
|
|
1317
|
+
|
|
1318
|
+
def iter_aug_methods(self) -> list[tuple[str, AugMethodConfig]]:
|
|
1319
|
+
"""Yield (attr_name, AugMethodConfig) for every augmentation method."""
|
|
1320
|
+
result: list[tuple[str, AugMethodConfig]] = []
|
|
1321
|
+
for attr_name, _ in self._AUG_METHODS:
|
|
1322
|
+
cfg = getattr(self, attr_name)
|
|
1323
|
+
if isinstance(cfg, AugMethodConfig):
|
|
1324
|
+
result.append((attr_name, cfg))
|
|
1325
|
+
return result
|
|
1326
|
+
|
|
1327
|
+
@staticmethod
|
|
1328
|
+
def _parse_scale_mapping(val: Any, kind: str) -> dict[str, float]:
|
|
1329
|
+
"""Read finite, non-negative factors from a mapping or comma-separated pairs."""
|
|
1330
|
+
if val is None or val == "":
|
|
1331
|
+
return {}
|
|
1332
|
+
if isinstance(val, str):
|
|
1333
|
+
pairs: dict[str, Any] = {}
|
|
1334
|
+
for part in val.split(","):
|
|
1335
|
+
key, separator, factor = part.strip().partition("=" if "=" in part else ":")
|
|
1336
|
+
if not separator:
|
|
1337
|
+
raise InputValidationError(
|
|
1338
|
+
f"Invalid {kind} intensity scale: {part!r}; expected KEY=FACTOR",
|
|
1339
|
+
code="invalid_scale_factor",
|
|
1340
|
+
)
|
|
1341
|
+
pairs[key.strip()] = factor.strip()
|
|
1342
|
+
elif isinstance(val, dict):
|
|
1343
|
+
pairs = val
|
|
1344
|
+
else:
|
|
1345
|
+
raise InputValidationError(
|
|
1346
|
+
f"{kind.capitalize()} intensity scales require a mapping or KEY=FACTOR pairs",
|
|
1347
|
+
code="invalid_scale_factor",
|
|
1348
|
+
)
|
|
1349
|
+
result: dict[str, float] = {}
|
|
1350
|
+
for key, value in pairs.items():
|
|
1351
|
+
try:
|
|
1352
|
+
factor = float(value)
|
|
1353
|
+
except (TypeError, ValueError, OverflowError) as exc:
|
|
1354
|
+
raise InputValidationError(
|
|
1355
|
+
f"Invalid {kind} intensity factor for {key!r}: {value!r}",
|
|
1356
|
+
code="invalid_scale_factor",
|
|
1357
|
+
) from exc
|
|
1358
|
+
if not str(key).strip() or isinstance(value, bool) or factor < 0 or not math.isfinite(factor):
|
|
1359
|
+
raise InputValidationError(
|
|
1360
|
+
f"{kind.capitalize()} intensity factor for {key!r} must have a non-empty key and finite non-negative value",
|
|
1361
|
+
code="invalid_scale_factor",
|
|
1362
|
+
)
|
|
1363
|
+
result[str(key)] = factor
|
|
1364
|
+
return result
|
|
1365
|
+
|
|
1366
|
+
@staticmethod
|
|
1367
|
+
def _parse_script_intensity_scales(val: Any) -> dict[str, float]:
|
|
1368
|
+
"""Parse script intensity scale mapping from string or dict."""
|
|
1369
|
+
return GenerationConfig._parse_scale_mapping(val, "script")
|
|
1370
|
+
|
|
1371
|
+
@staticmethod
|
|
1372
|
+
def _parse_font_intensity_scales(val: Any) -> dict[str, float]:
|
|
1373
|
+
"""Parse font intensity scale mapping from string or dict."""
|
|
1374
|
+
return GenerationConfig._parse_scale_mapping(val, "font")
|
|
1375
|
+
|
|
1376
|
+
def enabled_aug_methods(self) -> list[tuple[str, AugMethodConfig]]:
|
|
1377
|
+
"""Return only augmentation methods with prob > 0."""
|
|
1378
|
+
return [(name, cfg) for name, cfg in self.iter_aug_methods() if cfg.enabled]
|
|
1379
|
+
|
|
1380
|
+
# Serialisation (for IPC / config file)
|
|
1381
|
+
|
|
1382
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1383
|
+
"""Serialise to a plain dict for cross-process IPC and YAML export."""
|
|
1384
|
+
result: dict[str, Any] = {}
|
|
1385
|
+
for f in fields(self):
|
|
1386
|
+
value = getattr(self, f.name)
|
|
1387
|
+
if isinstance(value, AugMethodConfig):
|
|
1388
|
+
result[f.name] = {"prob": value.prob, "min": value.min, "max": value.max}
|
|
1389
|
+
elif isinstance(value, NormalizerConfig):
|
|
1390
|
+
result["normalizer"] = value.to_dict()
|
|
1391
|
+
elif isinstance(value, TextDecorationConfig):
|
|
1392
|
+
result[f.name] = {
|
|
1393
|
+
"color_prob": value.color_prob,
|
|
1394
|
+
"underline_prob": value.underline_prob,
|
|
1395
|
+
"subscript_prob": value.subscript_prob,
|
|
1396
|
+
"superscript_prob": value.superscript_prob,
|
|
1397
|
+
"italic_prob": value.italic_prob,
|
|
1398
|
+
"bold_prob": value.bold_prob,
|
|
1399
|
+
}
|
|
1400
|
+
elif isinstance(value, TextEffectConfig):
|
|
1401
|
+
result[f.name] = {
|
|
1402
|
+
f"{name}_prob": getattr(value, f"{name}_prob") for name in TEXT_EFFECT_NAMES
|
|
1403
|
+
}
|
|
1404
|
+
else:
|
|
1405
|
+
result[f.name] = value
|
|
1406
|
+
return result
|
|
1407
|
+
|
|
1408
|
+
@classmethod
|
|
1409
|
+
def from_dict(cls, d: dict[str, Any]) -> GenerationConfig:
|
|
1410
|
+
"""Reconstruct from a dict produced by :meth:`to_dict`.
|
|
1411
|
+
|
|
1412
|
+
Unknown keys are silently ignored so workers started with an older
|
|
1413
|
+
config dict do not crash when new keys are added.
|
|
1414
|
+
"""
|
|
1415
|
+
aug_method_names = {name for name, _ in cls._AUG_METHODS}
|
|
1416
|
+
aug_kwargs: dict[str, AugMethodConfig] = {}
|
|
1417
|
+
other_kwargs: dict[str, Any] = {}
|
|
1418
|
+
|
|
1419
|
+
for key, value in d.items():
|
|
1420
|
+
if key in aug_method_names and isinstance(value, dict):
|
|
1421
|
+
cfg_cls = cls._AUG_METHOD_CLASSES.get(key, AugMethodConfig)
|
|
1422
|
+
aug_kwargs[key] = cfg_cls(
|
|
1423
|
+
prob=float(value.get("prob", 0.0)),
|
|
1424
|
+
min=float(value.get("min", 0.0)),
|
|
1425
|
+
max=float(value.get("max", 1.0)),
|
|
1426
|
+
)
|
|
1427
|
+
elif key == "normalizer" and isinstance(value, dict):
|
|
1428
|
+
other_kwargs[key] = NormalizerConfig.from_dict(value)
|
|
1429
|
+
elif key == "text_deco" and isinstance(value, dict):
|
|
1430
|
+
other_kwargs[key] = TextDecorationConfig(
|
|
1431
|
+
color_prob=float(value.get("color_prob", 0.0)),
|
|
1432
|
+
underline_prob=float(value.get("underline_prob", 0.0)),
|
|
1433
|
+
subscript_prob=float(value.get("subscript_prob", 0.0)),
|
|
1434
|
+
superscript_prob=float(value.get("superscript_prob", 0.0)),
|
|
1435
|
+
italic_prob=float(value.get("italic_prob", 0.0)),
|
|
1436
|
+
bold_prob=float(value.get("bold_prob", 0.0)),
|
|
1437
|
+
)
|
|
1438
|
+
elif key == "text_effect" and isinstance(value, dict):
|
|
1439
|
+
other_kwargs[key] = TextEffectConfig(
|
|
1440
|
+
**{
|
|
1441
|
+
f"{name}_prob": float(value.get(f"{name}_prob", 0.0))
|
|
1442
|
+
for name in TEXT_EFFECT_NAMES
|
|
1443
|
+
}
|
|
1444
|
+
)
|
|
1445
|
+
elif key not in aug_method_names:
|
|
1446
|
+
other_kwargs[key] = value
|
|
1447
|
+
|
|
1448
|
+
# Map yaml-key names to attr names (e.g. "image_height" stays, "image-height" also works)
|
|
1449
|
+
known = {f.name for f in fields(cls)} - aug_method_names - {"normalizer"}
|
|
1450
|
+
alias_map = {
|
|
1451
|
+
"script_intensity_scale": "script_intensity_scales",
|
|
1452
|
+
"font_intensity_scale": "font_intensity_scales",
|
|
1453
|
+
}
|
|
1454
|
+
filtered = {
|
|
1455
|
+
k: v
|
|
1456
|
+
for k, v in other_kwargs.items()
|
|
1457
|
+
if (k.replace("-", "_") in known or alias_map.get(k.replace("-", "_")) in known)
|
|
1458
|
+
and k != "normalizer"
|
|
1459
|
+
}
|
|
1460
|
+
# Renormalise dashed keys and resolve singular aliases
|
|
1461
|
+
remapped: dict[str, Any] = {}
|
|
1462
|
+
for k, v in filtered.items():
|
|
1463
|
+
norm_k = k.replace("-", "_")
|
|
1464
|
+
remapped[alias_map.get(norm_k, norm_k)] = v
|
|
1465
|
+
|
|
1466
|
+
return cls(
|
|
1467
|
+
**remapped,
|
|
1468
|
+
**aug_kwargs,
|
|
1469
|
+
**({"normalizer": other_kwargs["normalizer"]} if "normalizer" in other_kwargs else {}),
|
|
1470
|
+
)
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
# Alias for backwards-compatibility / spec naming
|
|
1474
|
+
GeneratorConfig = GenerationConfig
|