ocr-foundry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khocr_gen/__init__.py +11 -0
- khocr_gen/_pil_compat.py +25 -0
- khocr_gen/_rust_accel.py +204 -0
- khocr_gen/augmentation.py +1193 -0
- khocr_gen/benchmark.py +888 -0
- khocr_gen/benchmark_cmd.py +103 -0
- khocr_gen/cli.py +246 -0
- khocr_gen/combine.py +214 -0
- khocr_gen/combine_cmd.py +125 -0
- khocr_gen/config.py +1474 -0
- khocr_gen/config_loader.py +157 -0
- khocr_gen/corpus.py +621 -0
- khocr_gen/data_generator.py +1160 -0
- khocr_gen/errors.py +33 -0
- khocr_gen/evaluate_cmd.py +50 -0
- khocr_gen/evaluation.py +259 -0
- khocr_gen/fonts.py +316 -0
- khocr_gen/generate.py +227 -0
- khocr_gen/layout.py +1028 -0
- khocr_gen/layout_cmd.py +371 -0
- khocr_gen/layout_degradation.py +128 -0
- khocr_gen/layout_mesh.py +200 -0
- khocr_gen/layout_shaping.py +264 -0
- khocr_gen/layout_transform.py +238 -0
- khocr_gen/layout_warp.py +473 -0
- khocr_gen/line_height.py +127 -0
- khocr_gen/lmdb_pack.py +132 -0
- khocr_gen/logging.py +25 -0
- khocr_gen/normalizer.py +221 -0
- khocr_gen/parallel.py +366 -0
- khocr_gen/provenance.py +328 -0
- khocr_gen/rendering.py +1607 -0
- khocr_gen/text_support.py +429 -0
- khocr_gen/verify.py +471 -0
- khocr_gen/viewer.py +172 -0
- ocr_foundry-0.2.0.dist-info/METADATA +174 -0
- ocr_foundry-0.2.0.dist-info/RECORD +40 -0
- ocr_foundry-0.2.0.dist-info/WHEEL +4 -0
- ocr_foundry-0.2.0.dist-info/entry_points.txt +3 -0
- ocr_foundry-0.2.0.dist-info/licenses/LICENSE +21 -0
khocr_gen/verify.py
ADDED
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
"""`khocr-gen verify` - visual verification of augmentation methods.
|
|
2
|
+
|
|
3
|
+
Renders each augmentation method at two fixed intensities on a clean canvas,
|
|
4
|
+
producing side-by-side comparison images. The two intensities default to 0.0
|
|
5
|
+
(MIN) and 1.0 (MAX) but are settable via ``--min`` / ``--max``; each column uses
|
|
6
|
+
exactly that intensity rather than a value sampled at random from the range.
|
|
7
|
+
The RNG is reseeded per image before each method runs so pure-Python methods
|
|
8
|
+
reproduce exactly across runs (Rust-accelerated methods use their own thread RNG
|
|
9
|
+
and still vary run to run). Every augmentation is applied in isolation, no
|
|
10
|
+
stacking or combining of effects.
|
|
11
|
+
|
|
12
|
+
Each individual `text_deco_*` (bold, italic, underline, color, subscript,
|
|
13
|
+
superscript) and `text_effect_*` (background, echo, glitch, ...) probability
|
|
14
|
+
is also verifiable the same way, one at a time: MIN/MAX pin just that one
|
|
15
|
+
probability to the fixed intensity (all others at 0) and re-render, rather
|
|
16
|
+
than applying an AUG_METHODS function to a clean canvas.
|
|
17
|
+
|
|
18
|
+
`extreme_resize` is verified like any other AUG_METHODS entry, but its
|
|
19
|
+
MIN/MAX columns are pixel heights (default 8/1920, its configured default
|
|
20
|
+
range) rather than [0, 1] fractions -- pass ``--method extreme_resize --min
|
|
21
|
+
16 --max 800`` to check a specific range.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import logging
|
|
27
|
+
import random
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import TYPE_CHECKING, Any
|
|
30
|
+
|
|
31
|
+
import cv2
|
|
32
|
+
import numpy as np
|
|
33
|
+
|
|
34
|
+
from .augmentation import _RGB_PREFERRED_METHODS, AUG_METHODS
|
|
35
|
+
from .config import TEXT_EFFECT_NAMES, GenerationConfig, TextDecorationConfig, TextEffectConfig
|
|
36
|
+
from .fonts import FontManager
|
|
37
|
+
from .rendering import ImageRenderer
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
import argparse
|
|
41
|
+
|
|
42
|
+
_log = logging.getLogger("khocr_gen.verify")
|
|
43
|
+
|
|
44
|
+
# Rendering-time decorations/effects, not post-hoc augmentation functions --
|
|
45
|
+
# each is verified by re-rendering with exactly one text_deco_*/text_effect_*
|
|
46
|
+
# probability pinned to the fixed intensity (all others left at 0) instead of
|
|
47
|
+
# calling an AUG_METHODS entry on a clean canvas.
|
|
48
|
+
_TEXT_DECO_ATTRS: tuple[str, ...] = (
|
|
49
|
+
"bold",
|
|
50
|
+
"color",
|
|
51
|
+
"italic",
|
|
52
|
+
"subscript",
|
|
53
|
+
"superscript",
|
|
54
|
+
"underline",
|
|
55
|
+
)
|
|
56
|
+
_TEXT_PROB_METHODS: dict[str, tuple[str | None, str | None]] = {
|
|
57
|
+
f"text_deco_{attr}": (attr, None) for attr in _TEXT_DECO_ATTRS
|
|
58
|
+
}
|
|
59
|
+
_TEXT_PROB_METHODS.update({f"text_effect_{name}": (None, name) for name in TEXT_EFFECT_NAMES})
|
|
60
|
+
|
|
61
|
+
# extreme_resize's min/max are absolute pixel heights, not [0, 1] fractions
|
|
62
|
+
# (see ExtremeResizeConfig) -- verified with its own configured/CLI range
|
|
63
|
+
# instead of the shared --min/--max intensity fractions used by every other
|
|
64
|
+
# AUG_METHODS entry.
|
|
65
|
+
_PIXEL_HEIGHT_METHODS: frozenset[str] = frozenset({"extreme_resize"})
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# Sample texts
|
|
69
|
+
|
|
70
|
+
_FALLBACK_TEXTS = [
|
|
71
|
+
"Hello World",
|
|
72
|
+
"Khocr Gen verification",
|
|
73
|
+
"ប្រព័ន្ធ OCR",
|
|
74
|
+
"Invoice #1234",
|
|
75
|
+
"Amount: $1,500.00",
|
|
76
|
+
"Khmer: ភាសាខ្មែរ",
|
|
77
|
+
"Mixed: English & ខ្មែរ",
|
|
78
|
+
"Account 9876-5432-1098",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _load_sample_texts(corpus_path: str | None, count: int, seed: int = 42) -> list[str]:
|
|
83
|
+
"""Load *count* sample lines from the corpus, falling back to built-ins."""
|
|
84
|
+
texts: list[str] = []
|
|
85
|
+
if corpus_path and Path(corpus_path).exists():
|
|
86
|
+
rng = random.Random(seed)
|
|
87
|
+
try:
|
|
88
|
+
with open(corpus_path, encoding="utf-8") as fh:
|
|
89
|
+
lines = [ln.strip() for ln in fh if ln.strip()]
|
|
90
|
+
if lines:
|
|
91
|
+
texts = rng.sample(lines, min(count, len(lines)))
|
|
92
|
+
except OSError as exc:
|
|
93
|
+
_log.warning("Could not read corpus %s: %s", corpus_path, exc)
|
|
94
|
+
|
|
95
|
+
while len(texts) < count:
|
|
96
|
+
texts.append(_FALLBACK_TEXTS[len(texts) % len(_FALLBACK_TEXTS)])
|
|
97
|
+
|
|
98
|
+
return texts[:count]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# Grid helpers
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _stack_images(images: list[np.ndarray], target_w: int) -> np.ndarray:
|
|
105
|
+
"""Resize images to *target_w* preserving aspect ratio and vstack them."""
|
|
106
|
+
resized: list[np.ndarray] = []
|
|
107
|
+
for img in images:
|
|
108
|
+
h, w = img.shape[:2]
|
|
109
|
+
if w != target_w:
|
|
110
|
+
new_h = max(1, int(h * target_w / w))
|
|
111
|
+
img = cv2.resize(img, (target_w, new_h))
|
|
112
|
+
if img.ndim == 2:
|
|
113
|
+
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
|
|
114
|
+
resized.append(img.astype(np.uint8))
|
|
115
|
+
return np.vstack(resized) if resized else np.zeros((1, target_w, 3), dtype=np.uint8)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _make_comparison_grid(
|
|
119
|
+
low_grid: np.ndarray,
|
|
120
|
+
high_grid: np.ndarray,
|
|
121
|
+
method_name: str,
|
|
122
|
+
image_width: int,
|
|
123
|
+
min_intensity: float,
|
|
124
|
+
max_intensity: float,
|
|
125
|
+
) -> np.ndarray:
|
|
126
|
+
"""Stack low and high grids side by side with labels."""
|
|
127
|
+
label_h = 28
|
|
128
|
+
font = cv2.FONT_HERSHEY_SIMPLEX
|
|
129
|
+
font_scale = 0.55
|
|
130
|
+
thickness = 1
|
|
131
|
+
|
|
132
|
+
def _add_label(img: np.ndarray, text: str) -> np.ndarray:
|
|
133
|
+
banner = np.full((label_h, img.shape[1], 3), 40, dtype=np.uint8)
|
|
134
|
+
cv2.putText(banner, text, (6, 20), font, font_scale, (220, 220, 220), thickness)
|
|
135
|
+
return np.vstack([banner, img])
|
|
136
|
+
|
|
137
|
+
low_labeled = _add_label(low_grid, f"{method_name} [MIN {min_intensity:.2f}]")
|
|
138
|
+
high_labeled = _add_label(high_grid, f"{method_name} [MAX {max_intensity:.2f}]")
|
|
139
|
+
|
|
140
|
+
h_max = max(low_labeled.shape[0], high_labeled.shape[0])
|
|
141
|
+
|
|
142
|
+
def _pad_h(img: np.ndarray, target_h: int) -> np.ndarray:
|
|
143
|
+
diff = target_h - img.shape[0]
|
|
144
|
+
if diff <= 0:
|
|
145
|
+
return img
|
|
146
|
+
pad = np.full((diff, img.shape[1], 3), 20, dtype=np.uint8)
|
|
147
|
+
return np.vstack([img, pad])
|
|
148
|
+
|
|
149
|
+
low_labeled = _pad_h(low_labeled, h_max)
|
|
150
|
+
high_labeled = _pad_h(high_labeled, h_max)
|
|
151
|
+
|
|
152
|
+
divider = np.full((h_max, 4, 3), 80, dtype=np.uint8)
|
|
153
|
+
return np.hstack([low_labeled, divider, high_labeled])
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# Main verification logic
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _apply_at_intensity(
|
|
160
|
+
aug_fn: Any, method_name: str, clean_images: list[np.ndarray], intensity: float
|
|
161
|
+
) -> list[np.ndarray]:
|
|
162
|
+
"""Apply *aug_fn* at a fixed *intensity* to every clean image.
|
|
163
|
+
|
|
164
|
+
The Python RNG is reseeded deterministically per image so pure-Python methods
|
|
165
|
+
reproduce across runs (methods such as rotation draw their own randomness from
|
|
166
|
+
the fixed intensity's range).
|
|
167
|
+
"""
|
|
168
|
+
results: list[np.ndarray] = []
|
|
169
|
+
for idx, img in enumerate(clean_images):
|
|
170
|
+
random.seed(42 + idx)
|
|
171
|
+
np.random.seed(42 + idx)
|
|
172
|
+
try:
|
|
173
|
+
if method_name in _RGB_PREFERRED_METHODS:
|
|
174
|
+
rgb = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) if img.ndim == 2 else img.copy()
|
|
175
|
+
result = aug_fn(rgb, intensity)
|
|
176
|
+
if result is not None and result.ndim == 3:
|
|
177
|
+
result = cv2.cvtColor(result, cv2.COLOR_RGB2GRAY)
|
|
178
|
+
else:
|
|
179
|
+
result = aug_fn(img.copy(), intensity)
|
|
180
|
+
if result is not None:
|
|
181
|
+
results.append(result)
|
|
182
|
+
except Exception as exc:
|
|
183
|
+
_log.debug("aug %s @ %.3f failed: %s", method_name, intensity, exc)
|
|
184
|
+
return results
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _render_with_single_prob(
|
|
188
|
+
renderer: ImageRenderer,
|
|
189
|
+
cfg: GenerationConfig,
|
|
190
|
+
texts: list[str],
|
|
191
|
+
repeats: int,
|
|
192
|
+
prob: float,
|
|
193
|
+
*,
|
|
194
|
+
deco_attr: str | None,
|
|
195
|
+
effect_attr: str | None,
|
|
196
|
+
) -> list[np.ndarray]:
|
|
197
|
+
"""Render *texts* with exactly one text_deco/text_effect probability pinned
|
|
198
|
+
to *prob* (all its siblings left at 0), so each is verified in isolation.
|
|
199
|
+
|
|
200
|
+
Decorations/effects are sampled inside rendering rather than applied to a
|
|
201
|
+
clean canvas afterward, so this overrides ``cfg.text_deco``/``cfg.text_effect``
|
|
202
|
+
(restored afterward) instead of calling an aug function. ``color`` is a
|
|
203
|
+
no-op in grayscale mode (it only samples when ``color_mode == 3``), so the
|
|
204
|
+
renderer is switched to RGB for that one method.
|
|
205
|
+
"""
|
|
206
|
+
original_deco, original_effect = cfg.text_deco, cfg.text_effect
|
|
207
|
+
original_color_mode, original_pil_mode = renderer.color_mode, renderer._pil_mode
|
|
208
|
+
cfg.text_deco = TextDecorationConfig(**({f"{deco_attr}_prob": prob} if deco_attr else {}))
|
|
209
|
+
cfg.text_effect = TextEffectConfig(**({f"{effect_attr}_prob": prob} if effect_attr else {}))
|
|
210
|
+
if deco_attr == "color":
|
|
211
|
+
renderer.color_mode = 3
|
|
212
|
+
renderer._pil_mode = "RGB"
|
|
213
|
+
results: list[np.ndarray] = []
|
|
214
|
+
try:
|
|
215
|
+
for i, text in enumerate(texts):
|
|
216
|
+
for _ in range(repeats):
|
|
217
|
+
random.seed(42 + i)
|
|
218
|
+
np.random.seed(42 + i)
|
|
219
|
+
try:
|
|
220
|
+
img = renderer.render(text, augment=False)
|
|
221
|
+
if img is not None:
|
|
222
|
+
results.append(img)
|
|
223
|
+
except Exception as exc:
|
|
224
|
+
_log.debug(
|
|
225
|
+
"%s render failed for %r @ %.3f: %s",
|
|
226
|
+
deco_attr or effect_attr,
|
|
227
|
+
text,
|
|
228
|
+
prob,
|
|
229
|
+
exc,
|
|
230
|
+
)
|
|
231
|
+
finally:
|
|
232
|
+
cfg.text_deco, cfg.text_effect = original_deco, original_effect
|
|
233
|
+
renderer.color_mode, renderer._pil_mode = original_color_mode, original_pil_mode
|
|
234
|
+
return results
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def run(args: argparse.Namespace) -> int:
|
|
238
|
+
"""Entry point called by cli.py."""
|
|
239
|
+
output_dir = Path(args.output_dir)
|
|
240
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
241
|
+
|
|
242
|
+
fonts_dir = args.fonts
|
|
243
|
+
image_height: int = args.height
|
|
244
|
+
image_width: int | None = args.width
|
|
245
|
+
count: int = max(1, args.count)
|
|
246
|
+
show: bool = args.show
|
|
247
|
+
corpus_path: str | None = args.corpus
|
|
248
|
+
repeats: int = max(1, args.repeats)
|
|
249
|
+
|
|
250
|
+
raw_min = float(args.min_intensity)
|
|
251
|
+
raw_max = float(args.max_intensity)
|
|
252
|
+
|
|
253
|
+
min_intensity = float(min(max(raw_min, 0.0), 1.0))
|
|
254
|
+
max_intensity = float(min(max(raw_max, 0.0), 1.0))
|
|
255
|
+
if max_intensity < min_intensity:
|
|
256
|
+
min_intensity, max_intensity = max_intensity, min_intensity
|
|
257
|
+
|
|
258
|
+
# Build method list — all registered methods with min/max intensities,
|
|
259
|
+
# plus each rendering-time text_deco/text_effect probability.
|
|
260
|
+
all_methods: dict[str, Any] = dict(AUG_METHODS)
|
|
261
|
+
all_method_names = sorted({*all_methods, *_TEXT_PROB_METHODS})
|
|
262
|
+
|
|
263
|
+
requested = set(args.method) if args.method else None
|
|
264
|
+
method_names = [name for name in all_method_names if requested is None or name in requested]
|
|
265
|
+
|
|
266
|
+
if not method_names:
|
|
267
|
+
print(f"No matching augmentation methods for: {args.method}")
|
|
268
|
+
return 1
|
|
269
|
+
|
|
270
|
+
texts = _load_sample_texts(corpus_path, count)
|
|
271
|
+
print(f"\nVerification samples : {count}")
|
|
272
|
+
print(f"Fonts directory : {fonts_dir}")
|
|
273
|
+
print(f"Output directory : {output_dir}")
|
|
274
|
+
print(f"Intensities : MIN={min_intensity:.2f} MAX={max_intensity:.2f}")
|
|
275
|
+
if any(name in _PIXEL_HEIGHT_METHODS for name in method_names):
|
|
276
|
+
print(" (extreme_resize uses pixel heights instead, see below)")
|
|
277
|
+
print(f"Methods : {', '.join(method_names)}")
|
|
278
|
+
print()
|
|
279
|
+
|
|
280
|
+
# Build a minimal config and renderer
|
|
281
|
+
cfg = GenerationConfig(
|
|
282
|
+
fonts_dir=fonts_dir,
|
|
283
|
+
image_height=image_height,
|
|
284
|
+
image_width=image_width,
|
|
285
|
+
)
|
|
286
|
+
font_manager = FontManager(language="mixed", fonts_dir=fonts_dir)
|
|
287
|
+
renderer = ImageRenderer(font_manager, cfg)
|
|
288
|
+
|
|
289
|
+
saved: list[Path] = []
|
|
290
|
+
|
|
291
|
+
for method_name in method_names:
|
|
292
|
+
print(f" ── {method_name} ──")
|
|
293
|
+
|
|
294
|
+
# extreme_resize's min/max are pixel heights, not [0, 1] fractions:
|
|
295
|
+
# use its own configured default range (8..1920) unless the user
|
|
296
|
+
# explicitly overrode --min/--max, in which case those are taken as
|
|
297
|
+
# literal pixel heights rather than clamped to [0, 1].
|
|
298
|
+
local_min, local_max = min_intensity, max_intensity
|
|
299
|
+
if method_name in _PIXEL_HEIGHT_METHODS:
|
|
300
|
+
default_min, default_max = cfg.extreme_resize.min, cfg.extreme_resize.max
|
|
301
|
+
local_min = raw_min if raw_min != 0.0 else default_min
|
|
302
|
+
local_max = raw_max if raw_max != 1.0 else default_max
|
|
303
|
+
if local_max < local_min:
|
|
304
|
+
local_min, local_max = local_max, local_min
|
|
305
|
+
|
|
306
|
+
if method_name in _TEXT_PROB_METHODS:
|
|
307
|
+
# Rendering-time decoration/effect: re-render at each fixed
|
|
308
|
+
# intensity rather than applying a post-hoc augmentation function.
|
|
309
|
+
deco_attr, effect_attr = _TEXT_PROB_METHODS[method_name]
|
|
310
|
+
print(f" Applying MIN ({local_min:.2f}) ...", end="", flush=True)
|
|
311
|
+
min_images = _render_with_single_prob(
|
|
312
|
+
renderer,
|
|
313
|
+
cfg,
|
|
314
|
+
texts,
|
|
315
|
+
repeats,
|
|
316
|
+
local_min,
|
|
317
|
+
deco_attr=deco_attr,
|
|
318
|
+
effect_attr=effect_attr,
|
|
319
|
+
)
|
|
320
|
+
print(f" MAX ({local_max:.2f}) ...", end="", flush=True)
|
|
321
|
+
max_images = _render_with_single_prob(
|
|
322
|
+
renderer,
|
|
323
|
+
cfg,
|
|
324
|
+
texts,
|
|
325
|
+
repeats,
|
|
326
|
+
local_max,
|
|
327
|
+
deco_attr=deco_attr,
|
|
328
|
+
effect_attr=effect_attr,
|
|
329
|
+
)
|
|
330
|
+
print(" done")
|
|
331
|
+
|
|
332
|
+
if not min_images or not max_images:
|
|
333
|
+
print(" WARNING: no images rendered; skipping.")
|
|
334
|
+
continue
|
|
335
|
+
else:
|
|
336
|
+
aug_fn = all_methods[method_name]
|
|
337
|
+
|
|
338
|
+
# Render clean base images (one per text, repeated for variety)
|
|
339
|
+
clean_images: list[np.ndarray] = []
|
|
340
|
+
for i, text in enumerate(texts):
|
|
341
|
+
for _ in range(repeats):
|
|
342
|
+
random.seed(42 + i)
|
|
343
|
+
np.random.seed(42 + i)
|
|
344
|
+
try:
|
|
345
|
+
img = renderer.render(text, augment=False)
|
|
346
|
+
if img is not None:
|
|
347
|
+
clean_images.append(img)
|
|
348
|
+
except Exception as exc:
|
|
349
|
+
_log.debug("Render failed for %r: %s", text, exc)
|
|
350
|
+
|
|
351
|
+
if not clean_images:
|
|
352
|
+
print(" WARNING: no images rendered; skipping.")
|
|
353
|
+
continue
|
|
354
|
+
|
|
355
|
+
# Apply both intensities (each column uses exactly one fixed value)
|
|
356
|
+
print(f" Applying MIN ({local_min:.2f}) ...", end="", flush=True)
|
|
357
|
+
min_images = _apply_at_intensity(aug_fn, method_name, clean_images, local_min)
|
|
358
|
+
print(f" MAX ({local_max:.2f}) ...", end="", flush=True)
|
|
359
|
+
max_images = _apply_at_intensity(aug_fn, method_name, clean_images, local_max)
|
|
360
|
+
print(" done")
|
|
361
|
+
|
|
362
|
+
if not min_images or not max_images:
|
|
363
|
+
print(" WARNING: failed to apply augmentation; skipping.")
|
|
364
|
+
continue
|
|
365
|
+
|
|
366
|
+
if method_name in _PIXEL_HEIGHT_METHODS:
|
|
367
|
+
# The production pipeline always resizes an augmented image
|
|
368
|
+
# back to line height right after augmenting (see
|
|
369
|
+
# ImageRenderer._apply_augmentation); mirror that here so the
|
|
370
|
+
# MAX column shows the actual post-normalisation artifact
|
|
371
|
+
# instead of a literal 1920px-tall image.
|
|
372
|
+
min_images = [renderer._resize_to_target(img) for img in min_images]
|
|
373
|
+
max_images = [renderer._resize_to_target(img) for img in max_images]
|
|
374
|
+
|
|
375
|
+
target_w = image_width or max(
|
|
376
|
+
max(img.shape[1] for img in min_images),
|
|
377
|
+
max(img.shape[1] for img in max_images),
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
low_grid = _stack_images(min_images, target_w)
|
|
381
|
+
high_grid = _stack_images(max_images, target_w)
|
|
382
|
+
|
|
383
|
+
comparison = _make_comparison_grid(
|
|
384
|
+
low_grid, high_grid, method_name, target_w, local_min, local_max
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
out_path = output_dir / f"verify_{method_name}.png"
|
|
388
|
+
ok = cv2.imwrite(str(out_path), cv2.cvtColor(comparison, cv2.COLOR_RGB2BGR))
|
|
389
|
+
if ok:
|
|
390
|
+
print(f" Saved -> {out_path}")
|
|
391
|
+
saved.append(out_path)
|
|
392
|
+
else:
|
|
393
|
+
print(f" WARNING: Failed to write {out_path}")
|
|
394
|
+
|
|
395
|
+
if show:
|
|
396
|
+
cv2.imshow(f"verify: {method_name}", cv2.cvtColor(comparison, cv2.COLOR_RGB2BGR))
|
|
397
|
+
print(" Press any key to continue...")
|
|
398
|
+
cv2.waitKey(0)
|
|
399
|
+
cv2.destroyWindow(f"verify: {method_name}")
|
|
400
|
+
|
|
401
|
+
if show:
|
|
402
|
+
cv2.destroyAllWindows()
|
|
403
|
+
|
|
404
|
+
print(f"\nVerification complete. {len(saved)} image(s) saved to {output_dir}\n")
|
|
405
|
+
return 0
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def add_args(parser: argparse.ArgumentParser) -> None:
|
|
409
|
+
"""Register all verify sub-command arguments onto *parser*."""
|
|
410
|
+
|
|
411
|
+
all_names = sorted({*AUG_METHODS, *_TEXT_PROB_METHODS})
|
|
412
|
+
|
|
413
|
+
parser.add_argument("--fonts", default="fonts/", metavar="DIR", help="Root fonts directory")
|
|
414
|
+
parser.add_argument(
|
|
415
|
+
"--corpus", default=None, metavar="FILE", help="Optional corpus file for sample texts"
|
|
416
|
+
)
|
|
417
|
+
parser.add_argument(
|
|
418
|
+
"--output-dir",
|
|
419
|
+
"--output",
|
|
420
|
+
dest="output_dir",
|
|
421
|
+
default="verify_output",
|
|
422
|
+
metavar="DIR",
|
|
423
|
+
help="Directory for comparison PNG images",
|
|
424
|
+
)
|
|
425
|
+
parser.add_argument(
|
|
426
|
+
"--height", type=int, default=48, metavar="PX", help="Image height in pixels"
|
|
427
|
+
)
|
|
428
|
+
parser.add_argument(
|
|
429
|
+
"--width", type=int, default=None, metavar="PX", help="Fixed image width; omit for variable"
|
|
430
|
+
)
|
|
431
|
+
parser.add_argument(
|
|
432
|
+
"--count", type=int, default=6, metavar="N", help="Number of sample texts per method"
|
|
433
|
+
)
|
|
434
|
+
parser.add_argument(
|
|
435
|
+
"--min",
|
|
436
|
+
dest="min_intensity",
|
|
437
|
+
type=float,
|
|
438
|
+
default=0.0,
|
|
439
|
+
metavar="F",
|
|
440
|
+
help=(
|
|
441
|
+
"Fixed intensity in [0, 1] for the MIN column (default: 0.0). "
|
|
442
|
+
"For --method extreme_resize this is a pixel height instead (default: 8)."
|
|
443
|
+
),
|
|
444
|
+
)
|
|
445
|
+
parser.add_argument(
|
|
446
|
+
"--max",
|
|
447
|
+
dest="max_intensity",
|
|
448
|
+
type=float,
|
|
449
|
+
default=1.0,
|
|
450
|
+
metavar="F",
|
|
451
|
+
help=(
|
|
452
|
+
"Fixed intensity in [0, 1] for the MAX column (default: 1.0). "
|
|
453
|
+
"For --method extreme_resize this is a pixel height instead (default: 1920)."
|
|
454
|
+
),
|
|
455
|
+
)
|
|
456
|
+
parser.add_argument(
|
|
457
|
+
"--repeats",
|
|
458
|
+
type=int,
|
|
459
|
+
default=2,
|
|
460
|
+
metavar="N",
|
|
461
|
+
help="Augmentation repeats per text for variety",
|
|
462
|
+
)
|
|
463
|
+
parser.add_argument(
|
|
464
|
+
"--method",
|
|
465
|
+
nargs="+",
|
|
466
|
+
default=None,
|
|
467
|
+
metavar="NAME",
|
|
468
|
+
choices=all_names,
|
|
469
|
+
help=f"Restrict to specific methods. Available: {', '.join(all_names)}",
|
|
470
|
+
)
|
|
471
|
+
parser.add_argument("--show", action="store_true", help="Display each comparison interactively")
|
khocr_gen/viewer.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""`khocr-gen view` - preview images stored in an LMDB (.mdb) database."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
from tqdm import tqdm
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
import argparse
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _read_lmdb_samples(lmdb_dir: Path, max_samples: int = 0) -> list[tuple[bytes, str]]:
|
|
16
|
+
"""Read (image_bytes, label) pairs from an LMDB database.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
lmdb_dir: Path to the LMDB directory (containing data.mdb).
|
|
20
|
+
max_samples: Maximum samples to read (0 = all).
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
List of (jpeg_bytes, text_label) tuples.
|
|
24
|
+
"""
|
|
25
|
+
import lmdb # type: ignore[import-untyped]
|
|
26
|
+
|
|
27
|
+
env = lmdb.open(str(lmdb_dir), readonly=True, lock=False, readahead=False, meminit=False)
|
|
28
|
+
samples: list[tuple[bytes, str]] = []
|
|
29
|
+
try:
|
|
30
|
+
with env.begin(write=False, buffers=True) as txn:
|
|
31
|
+
raw = txn.get(b"num-samples")
|
|
32
|
+
if raw is None:
|
|
33
|
+
return []
|
|
34
|
+
total = int(bytes(raw).decode())
|
|
35
|
+
count = total if max_samples <= 0 else min(max_samples, total)
|
|
36
|
+
for i in range(1, count + 1):
|
|
37
|
+
img_bytes = txn.get(f"image-{i:09d}".encode())
|
|
38
|
+
lbl_bytes = txn.get(f"label-{i:09d}".encode())
|
|
39
|
+
if img_bytes is None:
|
|
40
|
+
continue
|
|
41
|
+
label = bytes(lbl_bytes).decode("utf-8") if lbl_bytes else ""
|
|
42
|
+
samples.append((bytes(img_bytes), label))
|
|
43
|
+
finally:
|
|
44
|
+
env.close()
|
|
45
|
+
return samples
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _show_summary(lmdb_dir: Path) -> int:
|
|
49
|
+
"""Print a summary of the LMDB database and return total sample count."""
|
|
50
|
+
import lmdb # type: ignore[import-untyped]
|
|
51
|
+
|
|
52
|
+
env = lmdb.open(str(lmdb_dir), readonly=True, lock=False, readahead=False, meminit=False)
|
|
53
|
+
try:
|
|
54
|
+
with env.begin(write=False, buffers=True) as txn:
|
|
55
|
+
raw = txn.get(b"num-samples")
|
|
56
|
+
if raw is None:
|
|
57
|
+
print("Database is empty (no num-samples key).")
|
|
58
|
+
return 0
|
|
59
|
+
total = int(bytes(raw).decode())
|
|
60
|
+
print(f"LMDB directory : {lmdb_dir}")
|
|
61
|
+
print(f"Total samples : {total:,}")
|
|
62
|
+
|
|
63
|
+
# Show first few labels
|
|
64
|
+
print("\nFirst 5 samples:")
|
|
65
|
+
for i in range(1, min(6, total + 1)):
|
|
66
|
+
lbl_bytes = txn.get(f"label-{i:09d}".encode())
|
|
67
|
+
label = bytes(lbl_bytes).decode("utf-8") if lbl_bytes else "(no label)"
|
|
68
|
+
shape_info = ""
|
|
69
|
+
img_bytes = txn.get(f"image-{i:09d}".encode())
|
|
70
|
+
if img_bytes:
|
|
71
|
+
import cv2
|
|
72
|
+
import numpy as np
|
|
73
|
+
|
|
74
|
+
arr = np.frombuffer(bytes(img_bytes), dtype=np.uint8)
|
|
75
|
+
img = cv2.imdecode(arr, cv2.IMREAD_UNCHANGED)
|
|
76
|
+
if img is not None:
|
|
77
|
+
h, w = img.shape[:2]
|
|
78
|
+
c = img.shape[2] if img.ndim == 3 else 1
|
|
79
|
+
shape_info = f" [{w}x{h}x{c}c, {len(bytes(img_bytes)):,} bytes JPEG]"
|
|
80
|
+
print(f" {i:>5d}. {label[:80]}{shape_info}")
|
|
81
|
+
return total
|
|
82
|
+
finally:
|
|
83
|
+
env.close()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def run(args: argparse.Namespace) -> int:
|
|
87
|
+
"""Entry point - called by cli.py."""
|
|
88
|
+
lmdb_path = Path(args.lmdb).expanduser().resolve()
|
|
89
|
+
|
|
90
|
+
if not lmdb_path.exists():
|
|
91
|
+
print(f"LMDB directory not found: {lmdb_path}", file=sys.stderr)
|
|
92
|
+
return 1
|
|
93
|
+
|
|
94
|
+
data_mdb = lmdb_path / "data.mdb" if lmdb_path.is_dir() else lmdb_path
|
|
95
|
+
if data_mdb.is_file():
|
|
96
|
+
lmdb_path = data_mdb.parent
|
|
97
|
+
elif not (lmdb_path / "data.mdb").exists():
|
|
98
|
+
print(f"No data.mdb found in: {lmdb_path}", file=sys.stderr)
|
|
99
|
+
return 1
|
|
100
|
+
|
|
101
|
+
# Summary mode
|
|
102
|
+
if args.summary:
|
|
103
|
+
_show_summary(lmdb_path)
|
|
104
|
+
return 0
|
|
105
|
+
|
|
106
|
+
# Read samples
|
|
107
|
+
samples = _read_lmdb_samples(
|
|
108
|
+
lmdb_path, max_samples=args.count if args.count > 0 else args.max_count
|
|
109
|
+
)
|
|
110
|
+
if not samples:
|
|
111
|
+
print("No samples found in database.", file=sys.stderr)
|
|
112
|
+
return 1
|
|
113
|
+
|
|
114
|
+
# Output mode
|
|
115
|
+
if args.output_dir:
|
|
116
|
+
out_dir = Path(args.output_dir)
|
|
117
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
labels_path = out_dir / "labels.txt"
|
|
119
|
+
|
|
120
|
+
with labels_path.open("w", encoding="utf-8") as lf:
|
|
121
|
+
for idx, (jpeg_bytes, label) in enumerate(
|
|
122
|
+
tqdm(samples, desc=" Extracting", unit="img"), start=1
|
|
123
|
+
):
|
|
124
|
+
img_name = f"img_{idx:06d}.jpg"
|
|
125
|
+
(out_dir / img_name).write_bytes(jpeg_bytes)
|
|
126
|
+
lf.write(f"{img_name}\t{label}\n")
|
|
127
|
+
|
|
128
|
+
print(f"Extracted {len(samples)} samples to {out_dir}")
|
|
129
|
+
return 0
|
|
130
|
+
|
|
131
|
+
# Print text mode (labels only)
|
|
132
|
+
if args.labels_only:
|
|
133
|
+
for idx, (_, label) in enumerate(samples, start=1):
|
|
134
|
+
print(f"{idx:>6d}. {label}")
|
|
135
|
+
return 0
|
|
136
|
+
|
|
137
|
+
# Default: show info about samples
|
|
138
|
+
print(f"LMDB directory : {lmdb_path}")
|
|
139
|
+
_show_summary(lmdb_path)
|
|
140
|
+
print("\nUse --count N to extract N samples, or --output-dir DIR to export all.")
|
|
141
|
+
return 0
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def add_args(parser: argparse.ArgumentParser) -> None:
|
|
145
|
+
"""Register all view sub-command arguments onto *parser*."""
|
|
146
|
+
|
|
147
|
+
parser.add_argument(
|
|
148
|
+
"--lmdb", required=True, metavar="DIR", help="Path to LMDB directory (containing data.mdb)"
|
|
149
|
+
)
|
|
150
|
+
parser.add_argument("--summary", action="store_true", help="Print summary statistics")
|
|
151
|
+
parser.add_argument(
|
|
152
|
+
"--count",
|
|
153
|
+
type=int,
|
|
154
|
+
default=0,
|
|
155
|
+
metavar="N",
|
|
156
|
+
help="Number of samples to read (0 = all, capped)",
|
|
157
|
+
)
|
|
158
|
+
parser.add_argument(
|
|
159
|
+
"--max-count",
|
|
160
|
+
type=int,
|
|
161
|
+
default=100,
|
|
162
|
+
metavar="N",
|
|
163
|
+
help="Default max samples without --count",
|
|
164
|
+
)
|
|
165
|
+
parser.add_argument(
|
|
166
|
+
"--output-dir",
|
|
167
|
+
"-o",
|
|
168
|
+
default=None,
|
|
169
|
+
metavar="DIR",
|
|
170
|
+
help="Extract images and labels to a directory",
|
|
171
|
+
)
|
|
172
|
+
parser.add_argument("--labels-only", action="store_true", help="Print only labels (text mode)")
|