ocr-foundry 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,371 @@
1
+ """`khocr-gen layout` - compose pages with geometry-preserving affine transforms."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import TYPE_CHECKING
7
+
8
+ from .config import GenerationConfig
9
+ from .errors import InputValidationError, KhocrGenError
10
+ from .layout import LayoutConfig, generate_layout
11
+ from .normalizer import NormalizerConfig
12
+
13
+ if TYPE_CHECKING:
14
+ import argparse
15
+
16
+
17
+ def add_args(parser: argparse.ArgumentParser) -> None:
18
+ parser.add_argument("-c", "--config", default=None, metavar="FILE", help="Layout YAML configuration")
19
+ g_corpus = parser.add_argument_group("Corpus")
20
+ g_corpus.add_argument(
21
+ "--corpus",
22
+ default="corpus/corpus.txt",
23
+ metavar="FILE",
24
+ help="UTF-8 plain-text corpus or .jsonl manifest (auto-detected by extension)",
25
+ )
26
+ g_corpus.add_argument(
27
+ "--fonts",
28
+ default="fonts/",
29
+ metavar="DIR",
30
+ help="Root fonts directory (same layout as the generate command)",
31
+ )
32
+ g_corpus.add_argument(
33
+ "--min-length",
34
+ type=int,
35
+ default=1,
36
+ metavar="N",
37
+ help="Drop lines shorter than N characters after normalization",
38
+ )
39
+ g_corpus.add_argument(
40
+ "--max-length",
41
+ type=int,
42
+ default=260,
43
+ metavar="N",
44
+ help="Drop lines longer than N characters after normalization",
45
+ )
46
+ g_corpus.add_argument(
47
+ "--lines",
48
+ type=int,
49
+ default=0,
50
+ metavar="N",
51
+ help="Use at most N lines (0 = all)",
52
+ )
53
+
54
+ g_page = parser.add_argument_group("Page composition")
55
+ g_page.add_argument(
56
+ "--output",
57
+ default="layout",
58
+ metavar="DIR",
59
+ help="Output directory (pages/ and annotations.jsonl). Must be absent or empty",
60
+ )
61
+ g_page.add_argument("--page-width", type=int, default=1000, metavar="PX")
62
+ g_page.add_argument("--page-height", type=int, default=1400, metavar="PX")
63
+ g_page.add_argument(
64
+ "--page-scale",
65
+ type=float,
66
+ default=1.0,
67
+ metavar="FACTOR",
68
+ help="Positive finite page scale, applied before rotation (default: 1)",
69
+ )
70
+ g_page.add_argument(
71
+ "--page-rotation",
72
+ type=float,
73
+ default=0.0,
74
+ metavar="DEGREES",
75
+ help="Finite signed page rotation; positive is visually counterclockwise (default: 0)",
76
+ )
77
+ g_page.add_argument(
78
+ "--page-flip",
79
+ choices=["none", "horizontal", "vertical", "both"],
80
+ default="none",
81
+ help="Page flip mode applied after rotation and crop (default: none)",
82
+ )
83
+ g_page.add_argument(
84
+ "--page-scale-range",
85
+ type=float,
86
+ nargs=2,
87
+ metavar=("MIN", "MAX"),
88
+ default=None,
89
+ help="Sample page scale uniformly in [MIN, MAX] per page",
90
+ )
91
+ g_page.add_argument(
92
+ "--page-rotation-range",
93
+ type=float,
94
+ nargs=2,
95
+ metavar=("MIN", "MAX"),
96
+ default=None,
97
+ help="Sample page rotation uniformly in [MIN, MAX] degrees per page",
98
+ )
99
+ g_page.add_argument(
100
+ "--page-crop",
101
+ type=int,
102
+ nargs=4,
103
+ metavar=("X", "Y", "W", "H"),
104
+ help="Integer crop inside the expanded rotated page; must retain all line canvases and ink",
105
+ )
106
+ g_page.add_argument(
107
+ "--page-perspective",
108
+ type=float,
109
+ nargs=8,
110
+ metavar=("TLX", "TLY", "TRX", "TRY", "BRX", "BRY", "BLX", "BLY"),
111
+ help="Eight corner offsets (px) of the expanded rotated page, TL TR BR BL; each within 10%% of "
112
+ "the smaller expanded page dimension; absent or all zeros is a no-op",
113
+ )
114
+ g_page.add_argument(
115
+ "--page-elastic-amplitude",
116
+ type=float,
117
+ default=0.0,
118
+ metavar="PX",
119
+ help="Seeded 5x5 piecewise-affine mesh displacement bound in px (default: 0 = off; "
120
+ "at most 10%% of the smaller mesh cell)",
121
+ )
122
+ g_page.add_argument(
123
+ "--page-contrast",
124
+ type=float,
125
+ default=1.0,
126
+ metavar="FACTOR",
127
+ help="Photometric contrast 255-FACTOR*(255-pixel), within [0.75, 1] (default: 1 = off)",
128
+ )
129
+ g_page.add_argument(
130
+ "--page-noise",
131
+ type=int,
132
+ default=0,
133
+ metavar="N",
134
+ help="Seeded per-pixel uniform integer noise in [-N, N], N within 0..8 (default: 0 = off)",
135
+ )
136
+ g_page.add_argument(
137
+ "--margin",
138
+ type=int,
139
+ default=40,
140
+ metavar="PX",
141
+ help="Content margin on every side",
142
+ )
143
+ g_page.add_argument(
144
+ "--line-spacing",
145
+ type=int,
146
+ default=8,
147
+ metavar="PX",
148
+ help="Vertical gap between successive line boxes",
149
+ )
150
+ g_page.add_argument(
151
+ "--columns",
152
+ type=int,
153
+ default=1,
154
+ metavar="N",
155
+ help="Number of page columns; content width is split across them with --column-gap between (default: 1)",
156
+ )
157
+ g_page.add_argument(
158
+ "--column-gap",
159
+ type=int,
160
+ default=20,
161
+ metavar="PX",
162
+ help="Gap in pixels between adjacent columns (never part of the outer margins) (default: 20)",
163
+ )
164
+ g_page.add_argument(
165
+ "--page-direction",
166
+ choices=["ltr", "rtl"],
167
+ default="ltr",
168
+ help="Physical column emission order: left-to-right or right-to-left; reading order always follows logical columns (default: ltr)",
169
+ )
170
+ g_page.add_argument("--include-words", action="store_true",
171
+ help="Annotate explicit Unicode-whitespace tokens from positioned RAQM glyphs")
172
+ g_page.add_argument("--writing-mode", choices=["horizontal-tb", "vertical-rl", "vertical-lr"],
173
+ default="horizontal-tb", help="Native text shaping and line flow direction")
174
+ g_page.add_argument("--enable-tables", action="store_true", help="Fill a table with corpus lines")
175
+ g_page.add_argument("--table-rows", type=int, default=3, help="Maximum rows per table page")
176
+ g_page.add_argument("--table-columns", "--table-cols", dest="table_columns", type=int, default=3)
177
+ g_page.add_argument("--table-border-width", type=int, default=1)
178
+ g_page.add_argument("--table-padding", type=int, default=6)
179
+ g_page.add_argument(
180
+ "--include-baseline",
181
+ action="store_true",
182
+ help="Add a heuristic per-line baseline (row of heaviest ink) to each line annotation",
183
+ )
184
+ g_page.add_argument(
185
+ "--include-polygon",
186
+ action="store_true",
187
+ help="Add a heuristic per-line ink bounding-box polygon to each line annotation",
188
+ )
189
+ g_page.add_argument(
190
+ "--hierarchy",
191
+ action="store_true",
192
+ help="Add page-level blocks/spans hierarchy (blocks group spans by column via line_orders indices)",
193
+ )
194
+ g_page.add_argument(
195
+ "--max-pages-per-doc",
196
+ type=int,
197
+ default=100,
198
+ metavar="N",
199
+ help="Safety cap; a document needing more pages is skipped and reported",
200
+ )
201
+ g_page.add_argument(
202
+ "--seed",
203
+ type=int,
204
+ default=42,
205
+ help="Seed for font selection; identical seed/config reproduces annotations",
206
+ )
207
+ g_page.add_argument(
208
+ "--width",
209
+ type=int,
210
+ default=None,
211
+ metavar="PX",
212
+ help="Optional fixed rendered line width (must fit the content width)",
213
+ )
214
+ g_page.add_argument(
215
+ "--line-height",
216
+ type=int,
217
+ default=48,
218
+ metavar="PX",
219
+ help="Rendered line canvas height before page composition",
220
+ )
221
+ g_page.add_argument(
222
+ "--color-mode",
223
+ type=int,
224
+ choices=[1, 3],
225
+ default=1,
226
+ metavar="{1,3}",
227
+ help="1 = grayscale, 3 = RGB page/line images",
228
+ )
229
+
230
+ g_render = parser.add_argument_group("Rendering / multilingual")
231
+ g_render.add_argument(
232
+ "--multilingual",
233
+ action="store_true",
234
+ help="Use RAQM whole-line shaping and exhaustive font coverage selection",
235
+ )
236
+ g_render.add_argument(
237
+ "--language",
238
+ default="mixed",
239
+ help="Corpus language tag and legacy font-pool alias",
240
+ )
241
+ g_render.add_argument(
242
+ "--font-pool",
243
+ default=None,
244
+ help="Restrict to a named fonts subdirectory",
245
+ )
246
+ g_render.add_argument(
247
+ "--direction",
248
+ choices=["auto", "ltr", "rtl"],
249
+ default="auto",
250
+ help="Horizontal base direction in multilingual mode",
251
+ )
252
+ g_render.add_argument(
253
+ "--shaping-language",
254
+ default=None,
255
+ help="Optional BCP-47 shaping language in multilingual mode",
256
+ )
257
+ g_render.add_argument("--cluster-font-fallback", action="store_true",
258
+ help="Shape with font ranges when no single font covers the line")
259
+
260
+ NormalizerConfig.add_args(parser)
261
+
262
+
263
+ def _build_generation_config(args: argparse.Namespace) -> GenerationConfig:
264
+ """Build a clean (no-augmentation) renderer config from layout flags."""
265
+ return GenerationConfig(
266
+ image_height=args.line_height,
267
+ image_width=args.width,
268
+ color_mode=args.color_mode,
269
+ fonts_dir=str(args.fonts),
270
+ language=args.language,
271
+ multilingual=args.multilingual,
272
+ font_pool=args.font_pool,
273
+ direction=args.direction,
274
+ shaping_language=args.shaping_language,
275
+ cluster_font_fallback=args.cluster_font_fallback,
276
+ normalizer=NormalizerConfig.from_args(args),
277
+ mixed_font_prob=0.0,
278
+ clean_prob=0.0,
279
+ line_height_mode="fixed",
280
+ font_size_mode="fixed",
281
+ vertical_padding_mode="fixed",
282
+ bg_color_mode="default",
283
+ )
284
+
285
+
286
+ def run(args: argparse.Namespace) -> int:
287
+ corpus_path = Path(args.corpus)
288
+ fonts_dir = Path(args.fonts)
289
+ output_dir = Path(args.output)
290
+
291
+ if not corpus_path.exists():
292
+ print(f"Corpus file not found: {corpus_path}")
293
+ return 2
294
+ if not fonts_dir.exists():
295
+ print(f"Fonts directory not found: {fonts_dir}")
296
+ return 2
297
+
298
+ try:
299
+ gen_cfg = _build_generation_config(args)
300
+ layout_cfg = LayoutConfig(
301
+ page_width=args.page_width,
302
+ page_height=args.page_height,
303
+ margin=args.margin,
304
+ line_spacing=args.line_spacing,
305
+ max_pages_per_doc=args.max_pages_per_doc,
306
+ seed=args.seed,
307
+ line_height=args.line_height,
308
+ line_width=args.width,
309
+ columns=args.columns,
310
+ column_gap=args.column_gap,
311
+ page_direction=args.page_direction,
312
+ include_baseline=args.include_baseline,
313
+ include_polygon=args.include_polygon,
314
+ hierarchy=args.hierarchy,
315
+ include_words=args.include_words,
316
+ writing_mode=args.writing_mode,
317
+ enable_tables=args.enable_tables,
318
+ table_rows=args.table_rows,
319
+ table_columns=args.table_columns,
320
+ table_border_width=args.table_border_width,
321
+ table_padding=args.table_padding,
322
+ page_scale=args.page_scale,
323
+ page_rotation=args.page_rotation,
324
+ page_flip=args.page_flip,
325
+ page_scale_range=tuple(args.page_scale_range) if args.page_scale_range is not None else None,
326
+ page_rotation_range=tuple(args.page_rotation_range) if args.page_rotation_range is not None else None,
327
+ page_crop=tuple(args.page_crop) if args.page_crop is not None else None,
328
+ page_perspective=tuple(args.page_perspective)
329
+ if args.page_perspective is not None
330
+ else None,
331
+ page_elastic_amplitude=args.page_elastic_amplitude,
332
+ page_contrast=args.page_contrast,
333
+ page_noise=args.page_noise,
334
+ )
335
+ except (TypeError, ValueError, InputValidationError) as exc:
336
+ print(f"error: {exc}")
337
+ return 2
338
+
339
+ print("\n" + "=" * 70)
340
+ print(" Page Layout Composition")
341
+ print("=" * 70)
342
+
343
+ try:
344
+ result = generate_layout(
345
+ corpus_path=corpus_path,
346
+ output_dir=output_dir,
347
+ gen_cfg=gen_cfg,
348
+ layout=layout_cfg,
349
+ min_length=args.min_length,
350
+ max_length=args.max_length,
351
+ max_lines=args.lines,
352
+ )
353
+ except FileExistsError as exc:
354
+ print(f"error: {exc}")
355
+ return 1
356
+ except (OSError, ValueError, RuntimeError, KhocrGenError) as exc:
357
+ print(f"error: {exc}")
358
+ return 2
359
+
360
+ if result.pages_written == 0:
361
+ print(
362
+ "error: no pages were generated "
363
+ f"(loaded {result.corpus_stats.get('passing', 0)} line(s), "
364
+ f"{result.rejected_lines} rejected, {result.rejected_documents} document(s) rejected)"
365
+ )
366
+ return 2
367
+
368
+ print(f"\n Output: {output_dir}")
369
+ print(f" Annotations: {output_dir / 'annotations.jsonl'}")
370
+ print("=" * 70 + "\n")
371
+ return 0
@@ -0,0 +1,128 @@
1
+ """Seeded, pointwise photometric page degradation (layout contract v4).
2
+
3
+ The effects here change pixel values only; they never move pixels, so line
4
+ labels and every annotation coordinate stay valid. They run on the final
5
+ cropped page raster, so blank margins discarded by a crop never influence
6
+ crop validation.
7
+
8
+ Randomness uses SHAKE-256 streams keyed by ``(seed, page_id, operation,
9
+ contract version)``. Python's ``hash()``, the global ``random`` module and the
10
+ numpy global generator are never touched, and each operation has its own
11
+ stream, so enabling noise cannot change mesh, font or page randomness.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import hashlib
17
+ import math
18
+ from typing import Any
19
+
20
+ import numpy as np
21
+
22
+ from .errors import InputValidationError
23
+
24
+ STREAM_VERSION = "khocr-gen/layout/v4"
25
+ MIN_CONTRAST = 0.75
26
+ MAX_NOISE = 8
27
+ _NOISE_BLOCK = 1 << 18 # pixels per XOF block; bounds memory for very large pages
28
+ _STRIP_PIXELS = 1 << 20
29
+
30
+
31
+ def stream_key(seed: int, page_id: str, operation: str) -> bytes:
32
+ """Stable key naming one random stream; independent per operation and page."""
33
+ return f"{STREAM_VERSION}/{operation}/{seed}/{page_id}".encode()
34
+
35
+
36
+ def stream_bytes(key: bytes, count: int, block: int = 0) -> bytes:
37
+ """Deterministic bytes for ``key``; ``block`` selects an independent sub-stream."""
38
+ return hashlib.shake_256(key + b"/block/" + str(block).encode()).digest(count)
39
+
40
+
41
+ def uniform_symmetric(seed: int, page_id: str, operation: str, count: int) -> np.ndarray:
42
+ """``count`` float64 samples in [-1, 1], 64 bits each (platform independent)."""
43
+ raw = np.frombuffer(stream_bytes(stream_key(seed, page_id, operation), 8 * count), "<u8")
44
+ return raw.astype(np.float64) / float(2**64 - 1) * 2.0 - 1.0
45
+
46
+
47
+ def validate_photometric(contrast: float, noise: int) -> None:
48
+ if isinstance(contrast, bool) or not isinstance(contrast, (int, float)):
49
+ raise InputValidationError("page contrast must be a number")
50
+ if not math.isfinite(contrast) or not (MIN_CONTRAST <= contrast <= 1.0):
51
+ raise InputValidationError(f"page contrast must be finite and within [{MIN_CONTRAST}, 1]")
52
+ if isinstance(noise, bool) or not isinstance(noise, int):
53
+ raise InputValidationError("page noise must be an integer")
54
+ if not (0 <= noise <= MAX_NOISE):
55
+ raise InputValidationError(f"page noise must be an integer within [0, {MAX_NOISE}]")
56
+
57
+
58
+ def _noise_samples(seed: int, page_id: str, amplitude: int, height: int, width: int) -> np.ndarray:
59
+ """One integer in [-amplitude, amplitude] per pixel, row-major, as int16."""
60
+ total = height * width
61
+ key = stream_key(seed, page_id, "photometric_noise")
62
+ span = 2 * amplitude + 1
63
+ out = np.empty(total, dtype=np.int16)
64
+ for block, start in enumerate(range(0, total, _NOISE_BLOCK)):
65
+ count = min(_NOISE_BLOCK, total - start)
66
+ words = np.frombuffer(stream_bytes(key, 4 * count, block), "<u4").astype(np.uint64)
67
+ # Multiply-shift range reduction: floor(u32 * span / 2**32), then recenter.
68
+ out[start : start + count] = ((words * np.uint64(span)) >> np.uint64(32)).astype(
69
+ np.int16
70
+ ) - np.int16(amplitude)
71
+ return out.reshape(height, width)
72
+
73
+
74
+ def apply_photometric(
75
+ canvas: np.ndarray, *, contrast: float, noise: int, seed: int, page_id: str
76
+ ) -> np.ndarray:
77
+ """Return a degraded copy: contrast, then noise, round-half-up and saturate.
78
+
79
+ ``p' = floor(255 - contrast * (255 - p) + 0.5)``; then ``p'' = clip(p' + n,
80
+ 0, 255)`` with one integer ``n`` in ``[-noise, noise]`` per pixel shared by
81
+ all channels.
82
+ """
83
+ validate_photometric(contrast, noise)
84
+ if contrast == 1.0 and noise == 0:
85
+ return canvas
86
+ height, width = canvas.shape[:2]
87
+ samples = _noise_samples(seed, page_id, noise, height, width) if noise else None
88
+ out = np.empty_like(canvas)
89
+ rows = max(1, _STRIP_PIXELS // width)
90
+ for y0 in range(0, height, rows):
91
+ y1 = min(height, y0 + rows)
92
+ values = canvas[y0:y1].astype(np.float64)
93
+ if contrast != 1.0:
94
+ values = np.floor(255.0 - contrast * (255.0 - values) + 0.5)
95
+ if samples is not None:
96
+ shift = samples[y0:y1].astype(np.float64)
97
+ values = values + (shift if canvas.ndim == 2 else shift[..., None])
98
+ out[y0:y1] = np.clip(values, 0, 255).astype(np.uint8)
99
+ return out
100
+
101
+
102
+ def provenance(*, contrast: float, noise: int, seed: int, page_id: str) -> dict[str, Any]:
103
+ """Truthful record of the effect parameters actually applied to this page."""
104
+ effects: dict[str, Any] = {}
105
+ if contrast != 1.0:
106
+ effects["contrast"] = {
107
+ "factor": contrast,
108
+ "formula": "floor(255 - factor * (255 - pixel) + 0.5)",
109
+ }
110
+ if noise:
111
+ effects["noise"] = {
112
+ "amplitude": noise,
113
+ "distribution": "uniform integer in [-amplitude, amplitude]",
114
+ "samples": "one per pixel, row-major, shared by all channels",
115
+ "range_reduction": "floor(uint32 * (2*amplitude+1) / 2**32) - amplitude",
116
+ "stream_key": stream_key(seed, page_id, "photometric_noise").decode(),
117
+ "stream": "shake_256(key + '/block/' + n) little-endian uint32, "
118
+ f"{_NOISE_BLOCK} pixels per block n",
119
+ }
120
+ return {
121
+ "contract_version": 1,
122
+ "stage": "after_crop",
123
+ "pointwise": True,
124
+ "order": [name for name in ("contrast", "noise") if name in effects],
125
+ "seed": seed,
126
+ "rounding": "round half up after contrast; integer noise added; saturate to [0, 255]",
127
+ "effects": effects,
128
+ }