ocr-foundry 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
khocr_gen/generate.py ADDED
@@ -0,0 +1,227 @@
1
+ """`khocr-gen generate` - synthetic dataset generation CLI command."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import shutil
7
+ from pathlib import Path
8
+
9
+ from .config import GenerationConfig
10
+ from .corpus import count_corpus
11
+ from .errors import InputValidationError, KhocrGenError
12
+ from .line_height import validate_line_height_config
13
+
14
+
15
+ def _validation_percent(value: str) -> float:
16
+ """Parse validation split percentage for auto-generated val sets."""
17
+ try:
18
+ percent = float(value)
19
+ except ValueError as exc:
20
+ raise argparse.ArgumentTypeError("validation split percentage must be a number") from exc
21
+ if not 0.0 <= percent < 100.0:
22
+ raise argparse.ArgumentTypeError(
23
+ "validation split percentage must be in the range [0, 100)."
24
+ )
25
+ return percent
26
+
27
+
28
+ def run(args: argparse.Namespace) -> int:
29
+ """Entry point called by cli.py."""
30
+ from .data_generator import DatasetGenerator
31
+
32
+ output_dir = Path(args.output)
33
+ corpus_path = Path(args.corpus)
34
+ fonts_dir = Path(args.fonts)
35
+
36
+ try:
37
+ gen_cfg = GenerationConfig.from_args(args)
38
+ except (TypeError, ValueError) as exc:
39
+ print(f"error: {exc}")
40
+ return 2
41
+
42
+ # --count-only mode
43
+ if args.count_only:
44
+ if not corpus_path.exists():
45
+ print(f"Corpus file not found: {corpus_path}")
46
+ return 2
47
+
48
+ print(f"\nScanning corpus: {corpus_path}")
49
+ print(f" min_length={args.min_length}\n max_length={args.max_length}")
50
+
51
+ try:
52
+ stats = count_corpus(
53
+ corpus_path,
54
+ min_length=args.min_length,
55
+ max_length=args.max_length,
56
+ workers=args.workers,
57
+ normalizer=gen_cfg.normalizer,
58
+ )
59
+ except KhocrGenError as exc:
60
+ print(f"error: {exc}")
61
+ return 2
62
+ cap = args.lines if args.lines > 0 else stats["passing"]
63
+ effective = min(stats["passing"], cap)
64
+ print(f"\n Total non-blank lines : {stats['total']:>12,}")
65
+ print(f" Too short (<{args.min_length:>3}) : {stats['too_short']:>12,}")
66
+ print(f" Too long (>{args.max_length:<3}) : {stats['too_long']:>12,}")
67
+ print(f" Passing filter : {stats['passing']:>12,}")
68
+ print(f" Cap (--lines) : {cap:>12,}")
69
+ print(" =====================================")
70
+ print(f" Lines to generate : {effective:>12,}")
71
+ print(f" Augmented copies : {args.copies}")
72
+ if args.font_mode == "random":
73
+ print(f" Estimated images : {effective * args.copies:>12,}")
74
+ if getattr(args, "oversample_rare_chars", False):
75
+ print(
76
+ " Rare-char oversampling: enabled "
77
+ f"(bottom {args.rare_char_percentile:g}% chars, "
78
+ f"{args.rare_char_multiplier:g}x copies) "
79
+ "- estimate above does not include this boost"
80
+ )
81
+ return 0
82
+
83
+ # Validate inputs
84
+ if not corpus_path.exists():
85
+ print(f"Corpus file not found: {corpus_path}")
86
+ return 2
87
+ if not fonts_dir.exists():
88
+ print(f"Fonts directory not found: {fonts_dir}")
89
+ return 2
90
+
91
+ print(f"\nLoading corpus from: {corpus_path}")
92
+ print(
93
+ f" min_length={args.min_length}\n"
94
+ f" max_length={args.max_length}\n"
95
+ f" lines_cap={args.lines or 'all'}"
96
+ )
97
+
98
+ existing_mode = "error"
99
+ if args.append:
100
+ existing_mode = "append"
101
+ elif args.overwrite:
102
+ existing_mode = "overwrite"
103
+
104
+ if output_dir.exists() and any(output_dir.iterdir()) and existing_mode == "error":
105
+ print(f"Dataset output already exists. Re-run with --append or --overwrite: {output_dir}")
106
+ return 1
107
+
108
+ # Build config
109
+ gen_cfg.fonts_dir = str(fonts_dir)
110
+
111
+ try:
112
+ validate_line_height_config(gen_cfg)
113
+ except InputValidationError as exc:
114
+ print(f"error: {exc}")
115
+ return 2
116
+
117
+ # Resolve train/val/test ratios using GenerationConfig rules
118
+ _train_ratio, val_ratio, test_ratio = gen_cfg.resolve_split_ratios()
119
+
120
+ try:
121
+ generator = DatasetGenerator(gen_cfg)
122
+ counts = generator.generate_dataset(
123
+ corpus_path=corpus_path,
124
+ output_dir=output_dir,
125
+ val_file=args.val_file if hasattr(args, "val_file") else None,
126
+ test_file=getattr(args, "test_file", None),
127
+ copies=args.copies,
128
+ font_mode=args.font_mode,
129
+ retry_limit=args.retry_limit,
130
+ existing_mode=existing_mode,
131
+ workers=args.workers,
132
+ split_seed=args.seed,
133
+ val_ratio=val_ratio,
134
+ test_ratio=test_ratio,
135
+ image_dir=args.image_dir if hasattr(args, "image_dir") else None,
136
+ min_length=args.min_length,
137
+ max_length=args.max_length,
138
+ max_lines=args.lines,
139
+ oversample_rare_chars=getattr(args, "oversample_rare_chars", False),
140
+ rare_char_percentile=getattr(args, "rare_char_percentile", 5.0),
141
+ rare_char_multiplier=getattr(args, "rare_char_multiplier", 3.0),
142
+ )
143
+ except FileExistsError as exc:
144
+ print(exc)
145
+ return 1
146
+ except (OSError, ValueError, KhocrGenError) as exc:
147
+ print(f"error: {exc}")
148
+ return 2
149
+ if gen_cfg.multilingual and sum(counts.values()) == 0:
150
+ print(
151
+ "error: multilingual generation produced zero samples (empty/filtered corpus or unsupported text)"
152
+ )
153
+ return 2
154
+
155
+ # Build vocab
156
+ if not args.skip_vocab:
157
+ vocab_path = Path(args.vocab) if args.vocab else output_dir / "vocab.json"
158
+ if vocab_path.exists():
159
+ print(f"\nVocabulary already exists at {vocab_path}. Skipping vocab generation.")
160
+ else:
161
+ labels_files = [
162
+ str(output_dir / s / "labels.txt")
163
+ for s in ("train", "val", "test")
164
+ if (output_dir / s / "labels.txt").exists()
165
+ ]
166
+ if labels_files:
167
+ DatasetGenerator._build_vocab(
168
+ labels_files=labels_files, output_path=str(vocab_path)
169
+ )
170
+
171
+ # Pack LMDB (via --storage lmdb|both or legacy --pack-lmdb)
172
+ storage_mode = getattr(args, "storage", "raw")
173
+ pack_lmdb_flag = getattr(args, "pack_lmdb", False)
174
+ keep_raw_flag = getattr(args, "keep_raw", False)
175
+
176
+ should_pack = storage_mode in ("lmdb", "both") or pack_lmdb_flag
177
+ should_keep = storage_mode == "both" or (storage_mode == "raw" and keep_raw_flag)
178
+
179
+ if should_pack:
180
+ from .lmdb_pack import pack_lmdb
181
+
182
+ print("\nPacking into LMDB databases...")
183
+ for split in ["train", "val", "test"]:
184
+ labels_file = output_dir / split / "labels.txt"
185
+ if not labels_file.exists():
186
+ continue
187
+ images_dir = output_dir / split / "images"
188
+ lmdb_dir = output_dir / split / "lmdb"
189
+ lmdb_dir.mkdir(parents=True, exist_ok=True)
190
+
191
+ n = pack_lmdb(
192
+ labels_file=str(labels_file),
193
+ images_dir=str(images_dir),
194
+ out_dir=str(lmdb_dir),
195
+ jpeg_quality=getattr(args, "lmdb_jpeg_quality", 90),
196
+ map_size_gb=getattr(args, "lmdb_map_size_gb", 256),
197
+ verbose=getattr(args, "lmdb_verbose", False),
198
+ )
199
+ print(f" {split}: wrote {n} samples to {lmdb_dir}")
200
+
201
+ if images_dir.exists() and not should_keep:
202
+ shutil.rmtree(images_dir)
203
+ print(f" Removed images directory: {images_dir}")
204
+
205
+ print(" LMDB packing complete.")
206
+
207
+ print(f"\nGeneration complete. Dataset -> {output_dir}")
208
+ return 0
209
+
210
+
211
+ def add_args(parser: argparse.ArgumentParser) -> None:
212
+ """Register all generate sub-command arguments onto *parser*."""
213
+
214
+ parser.add_argument(
215
+ "-c",
216
+ "--config",
217
+ default=None,
218
+ metavar="FILE",
219
+ help="YAML config file. Values override argparse defaults; CLI flags override YAML.",
220
+ )
221
+
222
+ # Rendering + Augmentation + Corpus + Output + LMDB + Workers (all flags from GenerationConfig)
223
+ GenerationConfig.add_args(parser)
224
+
225
+ # Extra generate-only flags
226
+ g_lmdb = parser.add_argument_group("LMDB Packing (extra)")
227
+ g_lmdb.add_argument("--lmdb-verbose", action="store_true", help="Print LMDB packing progress")