ocr-foundry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khocr_gen/__init__.py +11 -0
- khocr_gen/_pil_compat.py +25 -0
- khocr_gen/_rust_accel.py +204 -0
- khocr_gen/augmentation.py +1193 -0
- khocr_gen/benchmark.py +888 -0
- khocr_gen/benchmark_cmd.py +103 -0
- khocr_gen/cli.py +246 -0
- khocr_gen/combine.py +214 -0
- khocr_gen/combine_cmd.py +125 -0
- khocr_gen/config.py +1474 -0
- khocr_gen/config_loader.py +157 -0
- khocr_gen/corpus.py +621 -0
- khocr_gen/data_generator.py +1160 -0
- khocr_gen/errors.py +33 -0
- khocr_gen/evaluate_cmd.py +50 -0
- khocr_gen/evaluation.py +259 -0
- khocr_gen/fonts.py +316 -0
- khocr_gen/generate.py +227 -0
- khocr_gen/layout.py +1028 -0
- khocr_gen/layout_cmd.py +371 -0
- khocr_gen/layout_degradation.py +128 -0
- khocr_gen/layout_mesh.py +200 -0
- khocr_gen/layout_shaping.py +264 -0
- khocr_gen/layout_transform.py +238 -0
- khocr_gen/layout_warp.py +473 -0
- khocr_gen/line_height.py +127 -0
- khocr_gen/lmdb_pack.py +132 -0
- khocr_gen/logging.py +25 -0
- khocr_gen/normalizer.py +221 -0
- khocr_gen/parallel.py +366 -0
- khocr_gen/provenance.py +328 -0
- khocr_gen/rendering.py +1607 -0
- khocr_gen/text_support.py +429 -0
- khocr_gen/verify.py +471 -0
- khocr_gen/viewer.py +172 -0
- ocr_foundry-0.2.0.dist-info/METADATA +174 -0
- ocr_foundry-0.2.0.dist-info/RECORD +40 -0
- ocr_foundry-0.2.0.dist-info/WHEEL +4 -0
- ocr_foundry-0.2.0.dist-info/entry_points.txt +3 -0
- ocr_foundry-0.2.0.dist-info/licenses/LICENSE +21 -0
khocr_gen/generate.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""`khocr-gen generate` - synthetic dataset generation CLI command."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import shutil
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .config import GenerationConfig
|
|
10
|
+
from .corpus import count_corpus
|
|
11
|
+
from .errors import InputValidationError, KhocrGenError
|
|
12
|
+
from .line_height import validate_line_height_config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _validation_percent(value: str) -> float:
|
|
16
|
+
"""Parse validation split percentage for auto-generated val sets."""
|
|
17
|
+
try:
|
|
18
|
+
percent = float(value)
|
|
19
|
+
except ValueError as exc:
|
|
20
|
+
raise argparse.ArgumentTypeError("validation split percentage must be a number") from exc
|
|
21
|
+
if not 0.0 <= percent < 100.0:
|
|
22
|
+
raise argparse.ArgumentTypeError(
|
|
23
|
+
"validation split percentage must be in the range [0, 100)."
|
|
24
|
+
)
|
|
25
|
+
return percent
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def run(args: argparse.Namespace) -> int:
|
|
29
|
+
"""Entry point called by cli.py."""
|
|
30
|
+
from .data_generator import DatasetGenerator
|
|
31
|
+
|
|
32
|
+
output_dir = Path(args.output)
|
|
33
|
+
corpus_path = Path(args.corpus)
|
|
34
|
+
fonts_dir = Path(args.fonts)
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
gen_cfg = GenerationConfig.from_args(args)
|
|
38
|
+
except (TypeError, ValueError) as exc:
|
|
39
|
+
print(f"error: {exc}")
|
|
40
|
+
return 2
|
|
41
|
+
|
|
42
|
+
# --count-only mode
|
|
43
|
+
if args.count_only:
|
|
44
|
+
if not corpus_path.exists():
|
|
45
|
+
print(f"Corpus file not found: {corpus_path}")
|
|
46
|
+
return 2
|
|
47
|
+
|
|
48
|
+
print(f"\nScanning corpus: {corpus_path}")
|
|
49
|
+
print(f" min_length={args.min_length}\n max_length={args.max_length}")
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
stats = count_corpus(
|
|
53
|
+
corpus_path,
|
|
54
|
+
min_length=args.min_length,
|
|
55
|
+
max_length=args.max_length,
|
|
56
|
+
workers=args.workers,
|
|
57
|
+
normalizer=gen_cfg.normalizer,
|
|
58
|
+
)
|
|
59
|
+
except KhocrGenError as exc:
|
|
60
|
+
print(f"error: {exc}")
|
|
61
|
+
return 2
|
|
62
|
+
cap = args.lines if args.lines > 0 else stats["passing"]
|
|
63
|
+
effective = min(stats["passing"], cap)
|
|
64
|
+
print(f"\n Total non-blank lines : {stats['total']:>12,}")
|
|
65
|
+
print(f" Too short (<{args.min_length:>3}) : {stats['too_short']:>12,}")
|
|
66
|
+
print(f" Too long (>{args.max_length:<3}) : {stats['too_long']:>12,}")
|
|
67
|
+
print(f" Passing filter : {stats['passing']:>12,}")
|
|
68
|
+
print(f" Cap (--lines) : {cap:>12,}")
|
|
69
|
+
print(" =====================================")
|
|
70
|
+
print(f" Lines to generate : {effective:>12,}")
|
|
71
|
+
print(f" Augmented copies : {args.copies}")
|
|
72
|
+
if args.font_mode == "random":
|
|
73
|
+
print(f" Estimated images : {effective * args.copies:>12,}")
|
|
74
|
+
if getattr(args, "oversample_rare_chars", False):
|
|
75
|
+
print(
|
|
76
|
+
" Rare-char oversampling: enabled "
|
|
77
|
+
f"(bottom {args.rare_char_percentile:g}% chars, "
|
|
78
|
+
f"{args.rare_char_multiplier:g}x copies) "
|
|
79
|
+
"- estimate above does not include this boost"
|
|
80
|
+
)
|
|
81
|
+
return 0
|
|
82
|
+
|
|
83
|
+
# Validate inputs
|
|
84
|
+
if not corpus_path.exists():
|
|
85
|
+
print(f"Corpus file not found: {corpus_path}")
|
|
86
|
+
return 2
|
|
87
|
+
if not fonts_dir.exists():
|
|
88
|
+
print(f"Fonts directory not found: {fonts_dir}")
|
|
89
|
+
return 2
|
|
90
|
+
|
|
91
|
+
print(f"\nLoading corpus from: {corpus_path}")
|
|
92
|
+
print(
|
|
93
|
+
f" min_length={args.min_length}\n"
|
|
94
|
+
f" max_length={args.max_length}\n"
|
|
95
|
+
f" lines_cap={args.lines or 'all'}"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
existing_mode = "error"
|
|
99
|
+
if args.append:
|
|
100
|
+
existing_mode = "append"
|
|
101
|
+
elif args.overwrite:
|
|
102
|
+
existing_mode = "overwrite"
|
|
103
|
+
|
|
104
|
+
if output_dir.exists() and any(output_dir.iterdir()) and existing_mode == "error":
|
|
105
|
+
print(f"Dataset output already exists. Re-run with --append or --overwrite: {output_dir}")
|
|
106
|
+
return 1
|
|
107
|
+
|
|
108
|
+
# Build config
|
|
109
|
+
gen_cfg.fonts_dir = str(fonts_dir)
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
validate_line_height_config(gen_cfg)
|
|
113
|
+
except InputValidationError as exc:
|
|
114
|
+
print(f"error: {exc}")
|
|
115
|
+
return 2
|
|
116
|
+
|
|
117
|
+
# Resolve train/val/test ratios using GenerationConfig rules
|
|
118
|
+
_train_ratio, val_ratio, test_ratio = gen_cfg.resolve_split_ratios()
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
generator = DatasetGenerator(gen_cfg)
|
|
122
|
+
counts = generator.generate_dataset(
|
|
123
|
+
corpus_path=corpus_path,
|
|
124
|
+
output_dir=output_dir,
|
|
125
|
+
val_file=args.val_file if hasattr(args, "val_file") else None,
|
|
126
|
+
test_file=getattr(args, "test_file", None),
|
|
127
|
+
copies=args.copies,
|
|
128
|
+
font_mode=args.font_mode,
|
|
129
|
+
retry_limit=args.retry_limit,
|
|
130
|
+
existing_mode=existing_mode,
|
|
131
|
+
workers=args.workers,
|
|
132
|
+
split_seed=args.seed,
|
|
133
|
+
val_ratio=val_ratio,
|
|
134
|
+
test_ratio=test_ratio,
|
|
135
|
+
image_dir=args.image_dir if hasattr(args, "image_dir") else None,
|
|
136
|
+
min_length=args.min_length,
|
|
137
|
+
max_length=args.max_length,
|
|
138
|
+
max_lines=args.lines,
|
|
139
|
+
oversample_rare_chars=getattr(args, "oversample_rare_chars", False),
|
|
140
|
+
rare_char_percentile=getattr(args, "rare_char_percentile", 5.0),
|
|
141
|
+
rare_char_multiplier=getattr(args, "rare_char_multiplier", 3.0),
|
|
142
|
+
)
|
|
143
|
+
except FileExistsError as exc:
|
|
144
|
+
print(exc)
|
|
145
|
+
return 1
|
|
146
|
+
except (OSError, ValueError, KhocrGenError) as exc:
|
|
147
|
+
print(f"error: {exc}")
|
|
148
|
+
return 2
|
|
149
|
+
if gen_cfg.multilingual and sum(counts.values()) == 0:
|
|
150
|
+
print(
|
|
151
|
+
"error: multilingual generation produced zero samples (empty/filtered corpus or unsupported text)"
|
|
152
|
+
)
|
|
153
|
+
return 2
|
|
154
|
+
|
|
155
|
+
# Build vocab
|
|
156
|
+
if not args.skip_vocab:
|
|
157
|
+
vocab_path = Path(args.vocab) if args.vocab else output_dir / "vocab.json"
|
|
158
|
+
if vocab_path.exists():
|
|
159
|
+
print(f"\nVocabulary already exists at {vocab_path}. Skipping vocab generation.")
|
|
160
|
+
else:
|
|
161
|
+
labels_files = [
|
|
162
|
+
str(output_dir / s / "labels.txt")
|
|
163
|
+
for s in ("train", "val", "test")
|
|
164
|
+
if (output_dir / s / "labels.txt").exists()
|
|
165
|
+
]
|
|
166
|
+
if labels_files:
|
|
167
|
+
DatasetGenerator._build_vocab(
|
|
168
|
+
labels_files=labels_files, output_path=str(vocab_path)
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
# Pack LMDB (via --storage lmdb|both or legacy --pack-lmdb)
|
|
172
|
+
storage_mode = getattr(args, "storage", "raw")
|
|
173
|
+
pack_lmdb_flag = getattr(args, "pack_lmdb", False)
|
|
174
|
+
keep_raw_flag = getattr(args, "keep_raw", False)
|
|
175
|
+
|
|
176
|
+
should_pack = storage_mode in ("lmdb", "both") or pack_lmdb_flag
|
|
177
|
+
should_keep = storage_mode == "both" or (storage_mode == "raw" and keep_raw_flag)
|
|
178
|
+
|
|
179
|
+
if should_pack:
|
|
180
|
+
from .lmdb_pack import pack_lmdb
|
|
181
|
+
|
|
182
|
+
print("\nPacking into LMDB databases...")
|
|
183
|
+
for split in ["train", "val", "test"]:
|
|
184
|
+
labels_file = output_dir / split / "labels.txt"
|
|
185
|
+
if not labels_file.exists():
|
|
186
|
+
continue
|
|
187
|
+
images_dir = output_dir / split / "images"
|
|
188
|
+
lmdb_dir = output_dir / split / "lmdb"
|
|
189
|
+
lmdb_dir.mkdir(parents=True, exist_ok=True)
|
|
190
|
+
|
|
191
|
+
n = pack_lmdb(
|
|
192
|
+
labels_file=str(labels_file),
|
|
193
|
+
images_dir=str(images_dir),
|
|
194
|
+
out_dir=str(lmdb_dir),
|
|
195
|
+
jpeg_quality=getattr(args, "lmdb_jpeg_quality", 90),
|
|
196
|
+
map_size_gb=getattr(args, "lmdb_map_size_gb", 256),
|
|
197
|
+
verbose=getattr(args, "lmdb_verbose", False),
|
|
198
|
+
)
|
|
199
|
+
print(f" {split}: wrote {n} samples to {lmdb_dir}")
|
|
200
|
+
|
|
201
|
+
if images_dir.exists() and not should_keep:
|
|
202
|
+
shutil.rmtree(images_dir)
|
|
203
|
+
print(f" Removed images directory: {images_dir}")
|
|
204
|
+
|
|
205
|
+
print(" LMDB packing complete.")
|
|
206
|
+
|
|
207
|
+
print(f"\nGeneration complete. Dataset -> {output_dir}")
|
|
208
|
+
return 0
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def add_args(parser: argparse.ArgumentParser) -> None:
|
|
212
|
+
"""Register all generate sub-command arguments onto *parser*."""
|
|
213
|
+
|
|
214
|
+
parser.add_argument(
|
|
215
|
+
"-c",
|
|
216
|
+
"--config",
|
|
217
|
+
default=None,
|
|
218
|
+
metavar="FILE",
|
|
219
|
+
help="YAML config file. Values override argparse defaults; CLI flags override YAML.",
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
# Rendering + Augmentation + Corpus + Output + LMDB + Workers (all flags from GenerationConfig)
|
|
223
|
+
GenerationConfig.add_args(parser)
|
|
224
|
+
|
|
225
|
+
# Extra generate-only flags
|
|
226
|
+
g_lmdb = parser.add_argument_group("LMDB Packing (extra)")
|
|
227
|
+
g_lmdb.add_argument("--lmdb-verbose", action="store_true", help="Print LMDB packing progress")
|