ocr-foundry 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
khocr_gen/layout.py ADDED
@@ -0,0 +1,1028 @@
1
+ """Page/document layout composition (Milestone 5, bounded vertical slice).
2
+
3
+ This module composes already-rendered *line* canvases into single-column pages.
4
+ It deliberately reuses the line rasterizer (``ImageRenderer``) and the normalizer
5
+ rather than inventing a second text engine, so annotation text matches the
6
+ pixels that were drawn.
7
+
8
+ Scope and hard limits
9
+ ---------------------
10
+ * One or more columns, top-to-bottom. Column gap sits between columns, never in
11
+ the outer margins. `page_direction=rtl` reverses physical column emission
12
+ order; reading order always follows the logical column sequence
13
+ (top-to-bottom within a column, then the next column).
14
+ * Boxes are **line-level** axis-aligned rectangles around the full rendered
15
+ line canvas (which includes the renderer's clean padding). No word/span,
16
+ baseline or polygon geometry is fabricated unless the corresponding flag is
17
+ set.
18
+ * Rendering is always clean (no geometric augmentation). Augmentation would
19
+ invalidate the box-to-pixels relationship, so it must stay disabled.
20
+ * Grouping is document-disjoint: every page belongs to exactly one document,
21
+ and a document's pages never share a page with another document.
22
+
23
+ Plain ``.txt`` corpora have no ``source_group``. Their lines form a single
24
+ synthetic document named ``__unlabeled__``. In JSONL manifests, lines with a
25
+ ``source_group`` are grouped by it; lines without one are collected into the
26
+ same ``__unlabeled__`` synthetic document, kept separate from named documents.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import errno
32
+ import json
33
+ import logging
34
+ import math
35
+ import os
36
+ import random
37
+ import stat
38
+ import sys
39
+ from collections import Counter
40
+ from dataclasses import dataclass, field
41
+ from pathlib import Path
42
+ from tempfile import TemporaryDirectory
43
+ from typing import TYPE_CHECKING, Any
44
+
45
+ import cv2
46
+ import numpy as np
47
+
48
+ from .augmentation import write_output_image
49
+ from .corpus import load_corpus_with_stats
50
+ from .errors import InputValidationError
51
+ from .fonts import FontManager
52
+ from .layout_degradation import (
53
+ apply_photometric,
54
+ stream_bytes,
55
+ stream_key,
56
+ validate_photometric,
57
+ )
58
+ from .layout_degradation import provenance as degradation_provenance
59
+ from .layout_transform import plan_page_transform
60
+ from .layout_warp import plan_page_warp, validate_elastic_amplitude, validate_perspective
61
+ from .rendering import ImageRenderer
62
+
63
+ if TYPE_CHECKING:
64
+ from .config import GenerationConfig
65
+
66
+ _log = logging.getLogger("khocr_gen.layout")
67
+
68
+ SCHEMA_VERSION = 1
69
+ SYNTHETIC_DOCUMENT = "__unlabeled__"
70
+
71
+
72
+ @dataclass
73
+ class LayoutConfig:
74
+ """Page-composition parameters (pixels)."""
75
+
76
+ page_width: int = 1000
77
+ page_height: int = 1400
78
+ margin: int = 40
79
+ line_spacing: int = 8
80
+ max_pages_per_doc: int = 100
81
+ seed: int = 42
82
+ line_height: int = 48
83
+ line_width: int | None = None
84
+ output_format: str = "png"
85
+
86
+ # Multi-column geometry
87
+ columns: int = 1
88
+ column_gap: int = 20
89
+ page_direction: str = "ltr"
90
+
91
+ # Optional per-line / page geometry annotations
92
+ include_baseline: bool = False
93
+ include_polygon: bool = False
94
+ hierarchy: bool = False
95
+ include_words: bool = False
96
+ writing_mode: str = "horizontal-tb"
97
+ enable_tables: bool = False
98
+ table_rows: int = 3
99
+ table_columns: int = 3
100
+ table_border_width: int = 1
101
+ table_padding: int = 6
102
+
103
+ # Fixed page transforms; independent of line augmentation and its RNG.
104
+ page_scale: float = 1.0
105
+ page_rotation: float = 0.0
106
+ page_crop: tuple[int, int, int, int] | None = None
107
+ page_flip: str = "none"
108
+
109
+ # Transform sampling ranges (sampled deterministically per page).
110
+ page_scale_range: tuple[float, float] | None = None
111
+ page_rotation_range: tuple[float, float] | None = None
112
+
113
+ # Contract v4: perspective, seeded elastic mesh, pointwise photometric degradation.
114
+ page_perspective: tuple[float, ...] | None = None
115
+ page_elastic_amplitude: float = 0.0
116
+ page_contrast: float = 1.0
117
+ page_noise: int = 0
118
+ def __post_init__(self) -> None:
119
+ if self.writing_mode not in {"horizontal-tb", "vertical-rl", "vertical-lr"}:
120
+ raise InputValidationError("invalid writing_mode; use horizontal-tb, vertical-rl, or vertical-lr")
121
+ for name in ("table_rows", "table_columns"):
122
+ value = getattr(self, name)
123
+ if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
124
+ raise InputValidationError(f"{name} must be a positive integer")
125
+ for name in ("table_border_width", "table_padding"):
126
+ value = getattr(self, name)
127
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
128
+ raise InputValidationError(f"{name} must be a non-negative integer")
129
+ if self.enable_tables:
130
+ inset = 2 * (self.table_padding + self.table_border_width)
131
+ if self.content_width // self.table_columns <= inset or self.content_height <= inset:
132
+ raise InputValidationError("table borders/padding must leave positive cell content area")
133
+ if self.include_words or self.writing_mode != "horizontal-tb":
134
+ from .layout_shaping import require_layout_shaper
135
+ require_layout_shaper()
136
+ if self.page_width <= 0 or self.page_height <= 0:
137
+ raise InputValidationError("page width and height must be positive")
138
+ if self.margin < 0:
139
+ raise InputValidationError("margin must be non-negative")
140
+ if 2 * self.margin >= self.page_width:
141
+ raise InputValidationError("margin must leave a positive content width")
142
+ if 2 * self.margin >= self.page_height:
143
+ raise InputValidationError("margin must leave a positive content height")
144
+ if self.line_spacing < 0:
145
+ raise InputValidationError("line spacing must be non-negative")
146
+ if self.max_pages_per_doc < 1:
147
+ raise InputValidationError("max pages per document must be at least 1")
148
+ if self.line_height <= 0:
149
+ raise InputValidationError("line height must be positive")
150
+ if self.columns < 1:
151
+ raise InputValidationError("columns must be at least 1")
152
+ if self.column_gap < 0:
153
+ raise InputValidationError("column gap must be non-negative")
154
+ if self.page_direction not in {"ltr", "rtl"}:
155
+ raise InputValidationError("page direction must be 'ltr' or 'rtl'")
156
+ if self.line_width is not None:
157
+ if self.line_width <= 0:
158
+ raise InputValidationError("line width must be positive")
159
+ if self.line_width > self.column_width:
160
+ raise InputValidationError(
161
+ f"line width {self.line_width} exceeds column width {self.column_width} "
162
+ "(content width minus column gaps, divided across columns)"
163
+ )
164
+ if not self.output_format:
165
+ raise InputValidationError("output format must be non-empty")
166
+ if self.page_flip not in {"none", "horizontal", "vertical", "both"}:
167
+ raise InputValidationError(
168
+ f"invalid page_flip {self.page_flip!r}; must be 'none', 'horizontal', 'vertical', or 'both'"
169
+ )
170
+ if self.page_scale_range is not None:
171
+ if len(self.page_scale_range) != 2:
172
+ raise InputValidationError("page_scale_range must be a tuple of (min, max)")
173
+ s_min, s_max = self.page_scale_range
174
+ if not (math.isfinite(s_min) and math.isfinite(s_max) and s_min > 0 and s_max >= s_min):
175
+ raise InputValidationError("page_scale_range values must be positive, finite, and min <= max")
176
+ if self.page_rotation_range is not None:
177
+ if len(self.page_rotation_range) != 2:
178
+ raise InputValidationError("page_rotation_range must be a tuple of (min, max)")
179
+ r_min, r_max = self.page_rotation_range
180
+ if not (math.isfinite(r_min) and math.isfinite(r_max) and r_max >= r_min):
181
+ raise InputValidationError("page_rotation_range values must be finite and min <= max")
182
+ self.page_perspective = validate_perspective(self.page_perspective)
183
+ self.page_elastic_amplitude = validate_elastic_amplitude(self.page_elastic_amplitude)
184
+ validate_photometric(self.page_contrast, self.page_noise)
185
+ if self.v4_enabled:
186
+ # Fixed-parameter validation (sizes, bounds, crop); mesh draws are per page.
187
+ self.plan_transform("page_000000")
188
+ elif self.transform_enabled:
189
+ scale, rot = self.resolve_transform_params("page_000000")
190
+ plan_page_transform(
191
+ self.page_width,
192
+ self.page_height,
193
+ scale,
194
+ rot,
195
+ self.page_crop,
196
+ flip=self.page_flip,
197
+ )
198
+ def resolve_transform_params(self, page_id: str) -> tuple[float, float]:
199
+ """Resolve (scale, rotation) for a page, sampling uniformly if ranges are configured."""
200
+ scale = self.page_scale
201
+ rotation = self.page_rotation
202
+ if self.page_scale_range is not None or self.page_rotation_range is not None:
203
+ key = stream_key(self.seed, page_id, "transform_sampling")
204
+ data = stream_bytes(key, 16)
205
+ u1, u2 = np.frombuffer(data, "<u8")
206
+ f1 = float(u1) / float(2**64 - 1)
207
+ f2 = float(u2) / float(2**64 - 1)
208
+ if self.page_scale_range is not None:
209
+ s_min, s_max = self.page_scale_range
210
+ scale = s_min + f1 * (s_max - s_min)
211
+ if self.page_rotation_range is not None:
212
+ r_min, r_max = self.page_rotation_range
213
+ rotation = r_min + f2 * (r_max - r_min)
214
+ return scale, rotation
215
+
216
+ @property
217
+ def affine_enabled(self) -> bool:
218
+ """Keep historical output for identity defaults, including full turns."""
219
+ return (
220
+ self.page_scale != 1.0
221
+ or self.page_rotation % 360 != 0
222
+ or self.page_crop is not None
223
+ or self.page_flip != "none"
224
+ or self.page_scale_range is not None
225
+ or self.page_rotation_range is not None
226
+ )
227
+
228
+ @property
229
+ def v4_enabled(self) -> bool:
230
+ """Any effective v4 option selects contract v4; affine-only output stays v3."""
231
+ return (
232
+ self.page_perspective is not None
233
+ or self.page_elastic_amplitude != 0
234
+ or self.page_contrast != 1.0
235
+ or self.page_noise != 0
236
+ )
237
+
238
+ @property
239
+ def transform_enabled(self) -> bool:
240
+ """True for the v3 geometry conventions (affine or any v4 option)."""
241
+ return self.affine_enabled or self.v4_enabled
242
+
243
+ def plan_transform(self, page_id: str):
244
+ """Plan this page's v4 geometry; the page id keys the elastic mesh stream."""
245
+ scale, rotation = self.resolve_transform_params(page_id)
246
+ return plan_page_warp(
247
+ self.page_width,
248
+ self.page_height,
249
+ scale,
250
+ rotation,
251
+ self.page_crop,
252
+ self.page_perspective,
253
+ self.page_elastic_amplitude,
254
+ flip=self.page_flip,
255
+ seed=self.seed,
256
+ page_id=page_id,
257
+ )
258
+ @property
259
+ def content_width(self) -> int:
260
+ return self.page_width - 2 * self.margin
261
+
262
+ @property
263
+ def content_height(self) -> int:
264
+ return self.page_height - 2 * self.margin
265
+
266
+ @property
267
+ def column_width(self) -> int:
268
+ """Usable width of a single column, gaps excluded from outer margins."""
269
+ total_gap = self.column_gap * (self.columns - 1)
270
+ remaining = self.content_width - total_gap
271
+ return max(1, remaining // self.columns)
272
+
273
+ @property
274
+ def geometry_extended(self) -> bool:
275
+ """True when any new geometry flag departs from the single-column LTR default."""
276
+ return (
277
+ self.columns > 1
278
+ or self.page_direction == "rtl"
279
+ or self.include_baseline
280
+ or self.include_polygon
281
+ or self.hierarchy
282
+ or self.include_words
283
+ or self.writing_mode != "horizontal-tb"
284
+ or self.enable_tables
285
+ )
286
+
287
+
288
+ @dataclass
289
+ class _RenderedLine:
290
+ """A cleanly rendered line canvas plus its source metadata."""
291
+
292
+ text: str
293
+ image: np.ndarray
294
+ source_id: str | None = None
295
+ source_group: str | None = None
296
+ language: str | None = None
297
+ font: str | None = None
298
+ font_size: int | None = None
299
+ words: list[dict[str, Any]] = field(default_factory=list)
300
+
301
+
302
+ @dataclass
303
+ class _Placement:
304
+ line: _RenderedLine
305
+ image: np.ndarray # possibly width-scaled copy
306
+ x: int
307
+ y: int
308
+ w: int
309
+ h: int
310
+ table: dict[str, Any] | None = None
311
+ cell: dict[str, Any] | None = None
312
+
313
+
314
+ @dataclass
315
+ class LayoutResult:
316
+ """Summary returned by :func:`generate_layout`."""
317
+
318
+ output_dir: Path
319
+ documents: int = 0
320
+ pages_written: int = 0
321
+ lines_written: int = 0
322
+ rejected_lines: int = 0
323
+ rejected_documents: int = 0
324
+ rejection_reasons: dict[str, int] = field(default_factory=dict)
325
+ corpus_stats: dict[str, int] = field(default_factory=dict)
326
+
327
+
328
+ def _document_key(item: str | Any) -> str:
329
+ """Group key for a corpus item.
330
+
331
+ Named ``source_group`` values stay distinct. Items without one share a
332
+ single synthetic document so plain ``.txt`` corpora still produce pages.
333
+ """
334
+ group = getattr(item, "source_group", None)
335
+ return group if group is not None else SYNTHETIC_DOCUMENT
336
+
337
+
338
+ def _sanitize_line_text(text: str) -> str:
339
+ """Replace embedded tabs (labels/composition treat them as layout noise)."""
340
+ return text.replace("\t", " ")
341
+
342
+
343
+ def _render_document(
344
+ items: list[str | Any],
345
+ renderer: ImageRenderer,
346
+ *,
347
+ multilingual: bool,
348
+ retry_limit: int,
349
+ layout: LayoutConfig | None = None,
350
+ ) -> tuple[list[_RenderedLine], Counter[str]]:
351
+ """Render every line in a document cleanly.
352
+
353
+ Returns ``(rendered, reason_counts)``.
354
+ """
355
+ rendered: list[_RenderedLine] = []
356
+ reasons: Counter[str] = Counter()
357
+
358
+ for item in items:
359
+ text = _sanitize_line_text(str(item))
360
+ if not text:
361
+ reasons["empty_text"] += 1
362
+ continue
363
+ if multilingual and not renderer.is_line_eligible(text):
364
+ reasons["no_eligible_font"] += 1
365
+ continue
366
+ if layout is not None and (layout.include_words or layout.writing_mode != "horizontal-tb"):
367
+ from .layout_shaping import render_positioned
368
+ result = render_positioned(renderer, text, layout.writing_mode, layout.include_words, retry_limit)
369
+ else:
370
+ result = renderer.render_clean_with_meta(text, retry_limit=retry_limit)
371
+ if result is None:
372
+ reasons["render_failed"] += 1
373
+ continue
374
+ image, _target_height, meta = result
375
+ rendered.append(
376
+ _RenderedLine(
377
+ text=text,
378
+ image=image,
379
+ source_id=getattr(item, "source_id", None),
380
+ source_group=getattr(item, "source_group", None),
381
+ language=getattr(item, "language", None),
382
+ font=meta.get("font"),
383
+ font_size=meta.get("font_size"),
384
+ words=meta.get("words", []),
385
+ )
386
+ )
387
+ return rendered, reasons
388
+
389
+
390
+ def _ink_mask(image: np.ndarray, threshold: int = 128) -> np.ndarray:
391
+ """Boolean mask of dark (ink) pixels on a clean rendered line canvas.
392
+
393
+ Grayscale images are used directly; RGB images are reduced to their first
394
+ channel (the clean rasterizer draws neutral dark text on a light background,
395
+ so any channel suffices).
396
+ """
397
+ gray = image if image.ndim == 2 else image[..., 0]
398
+ return gray < threshold
399
+
400
+
401
+ def _baseline_offset(mask: np.ndarray) -> int | None:
402
+ """Row offset (relative to the canvas top) of the heaviest ink band.
403
+
404
+ Heuristic: the horizontal projection peak of the ink mask approximates the
405
+ line baseline on which most glyphs sit. Returns ``None`` when no ink is
406
+ present.
407
+ """
408
+ if not mask.any():
409
+ return None
410
+ row_counts = mask.sum(axis=1)
411
+ return int(np.argmax(row_counts))
412
+
413
+
414
+ def _polygon_from_mask(mask: np.ndarray) -> list[list[int]] | None:
415
+ """Clockwise bounding-box polygon of the ink mask, origin top-left, y down.
416
+
417
+ Returns the four corners ``[x0, y0], [x1, y0], [x1, y1], [x0, y1]`` of the
418
+ ink bounding box, or ``None`` when no ink is present.
419
+ """
420
+ if not mask.any():
421
+ return None
422
+ rows = np.flatnonzero(mask.any(axis=1))
423
+ cols = np.flatnonzero(mask.any(axis=0))
424
+ y0, y1 = int(rows[0]), int(rows[-1])
425
+ x0, x1 = int(cols[0]), int(cols[-1])
426
+ return [[x0, y0], [x1, y0], [x1, y1], [x0, y1]]
427
+
428
+
429
+ def _fit_image(image: np.ndarray, width: int, height: int) -> np.ndarray:
430
+ h, w = image.shape[:2]
431
+ scale = min(1.0, width / w, height / h)
432
+ if scale == 1:
433
+ return image
434
+ return cv2.resize(image, (max(1, min(width, round(w * scale))),
435
+ max(1, min(height, round(h * scale)))), interpolation=cv2.INTER_AREA)
436
+
437
+
438
+ def _rect(x: float, y: float, w: float, h: float) -> list[list[float]]:
439
+ return [[x, y], [x+w, y], [x+w, y+h], [x, y+h]]
440
+
441
+
442
+ def _box(polygon: list[list[float]]) -> dict[str, int]:
443
+ x0, y0 = (math.floor(min(p[i] for p in polygon)) for i in (0, 1))
444
+ x1, y1 = (math.ceil(max(p[i] for p in polygon)) for i in (0, 1))
445
+ return {"x": x0, "y": y0, "w": x1-x0, "h": y1-y0}
446
+
447
+
448
+ def _paginate_vertical(rendered: list[_RenderedLine], layout: LayoutConfig) -> list[list[list[_Placement]]]:
449
+ pages = []
450
+ current = []
451
+ used = 0
452
+ for line in rendered:
453
+ image = _fit_image(line.image, layout.content_width, layout.content_height)
454
+ h, w = image.shape[:2]
455
+ if current and used + w > layout.content_width:
456
+ pages.append([current])
457
+ current, used = [], 0
458
+ x = (layout.page_width-layout.margin-used-w if layout.writing_mode == "vertical-rl"
459
+ else layout.margin+used)
460
+ current.append(_Placement(line, image, x, layout.margin, w, h))
461
+ used += w + layout.line_spacing
462
+ if current:
463
+ pages.append([current])
464
+ return pages
465
+
466
+
467
+ def _paginate_tables(rendered: list[_RenderedLine], layout: LayoutConfig) -> list[list[list[_Placement]]]:
468
+ pages = []
469
+ cursor = 0
470
+ inset = layout.table_padding + layout.table_border_width
471
+ edges = [layout.margin + c * layout.content_width // layout.table_columns
472
+ for c in range(layout.table_columns + 1)]
473
+ while cursor < len(rendered):
474
+ table: dict[str, Any] = {"cells": [], "line_orders": []}
475
+ placements = []
476
+ y = layout.margin
477
+ for row in range(layout.table_rows):
478
+ chunk = rendered[cursor:cursor+layout.table_columns]
479
+ if not chunk:
480
+ break
481
+ fitted = []
482
+ for index, line in enumerate(chunk):
483
+ col = layout.table_columns-1-index if layout.page_direction == "rtl" else index
484
+ image = _fit_image(line.image, edges[col+1]-edges[col]-2*inset,
485
+ layout.content_height-2*inset)
486
+ fitted.append((col, line, image))
487
+ row_h = max(image.shape[0] for _, _, image in fitted) + 2*inset
488
+ if placements and y + row_h > layout.page_height-layout.margin:
489
+ break
490
+ for col in range(layout.table_columns):
491
+ cell = {"row": row, "col": col, "row_span": 1, "col_span": 1,
492
+ "polygon": _rect(edges[col], y, edges[col+1]-edges[col], row_h),
493
+ "line_orders": []}
494
+ cell["box"] = _box(cell["polygon"])
495
+ table["cells"].append(cell)
496
+ for col, line, image in fitted:
497
+ cell = table["cells"][-layout.table_columns+col]
498
+ h, w = image.shape[:2]
499
+ placements.append(_Placement(line, image, edges[col]+inset, y+inset, w, h, table, cell))
500
+ cursor += len(chunk)
501
+ y += row_h
502
+ table["polygon"] = _rect(layout.margin, layout.margin, layout.content_width, y-layout.margin)
503
+ table["box"] = _box(table["polygon"])
504
+ pages.append([placements])
505
+ return pages
506
+
507
+
508
+ def _paginate(
509
+ rendered: list[_RenderedLine],
510
+ layout: LayoutConfig,
511
+ ) -> tuple[list[list[list[_Placement]]], Counter[str]]:
512
+ """Pack rendered lines into column-flowing pages without clipping or dropping.
513
+
514
+ Lines flow top-to-bottom within a logical column; when a column is full the
515
+ flow continues at the top of the next logical column, and when every column
516
+ on a page is full a new page begins. Physical column emission order follows
517
+ ``layout.page_direction`` (LTR left-to-right, RTL right-to-left) while
518
+ reading order always follows the logical column sequence.
519
+
520
+ A line wider than a column is uniformly scaled down to fit; a line taller
521
+ than the content height cannot be placed and is reported. Returns
522
+ ``(pages, reason_counts)`` where each page is a list of columns and each
523
+ column is a list of placements in logical fill order.
524
+ """
525
+ if layout.enable_tables:
526
+ return _paginate_tables(rendered, layout), Counter()
527
+ if layout.writing_mode != "horizontal-tb":
528
+ return _paginate_vertical(rendered, layout), Counter()
529
+ content_h = layout.content_height
530
+ bottom = layout.page_height - layout.margin
531
+ col_w = layout.column_width
532
+ gap = layout.column_gap
533
+ columns = layout.columns
534
+
535
+ # Physical x offset of each logical column; RTL reverses emission order.
536
+ if layout.page_direction == "rtl":
537
+ col_x = [layout.margin + (columns - 1 - c) * (col_w + gap) for c in range(columns)]
538
+ else:
539
+ col_x = [layout.margin + c * (col_w + gap) for c in range(columns)]
540
+
541
+ pages: list[list[list[_Placement]]] = []
542
+ current_page: list[list[_Placement]] = []
543
+ current_column: list[_Placement] = []
544
+ reasons: Counter[str] = Counter()
545
+ col_index = 0
546
+ y = layout.margin
547
+
548
+ for line in rendered:
549
+ image = line.image
550
+ h, w = image.shape[:2]
551
+ if w > col_w:
552
+ scale = col_w / w
553
+ new_w = max(1, col_w)
554
+ new_h = max(1, round(h * scale))
555
+ image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA)
556
+ w, h = new_w, new_h
557
+ if h > content_h:
558
+ reasons["line_too_tall"] += 1
559
+ continue
560
+
561
+ # Advance columns/pages until the line fits at the top of a column.
562
+ while True:
563
+ if y + h <= bottom:
564
+ current_column.append(
565
+ _Placement(
566
+ line=line,
567
+ image=image,
568
+ x=col_x[col_index],
569
+ y=y,
570
+ w=w,
571
+ h=h,
572
+ )
573
+ )
574
+ y += h + layout.line_spacing
575
+ break
576
+ # Current column is full: close it and move to the next.
577
+ if current_column:
578
+ current_page.append(current_column)
579
+ current_column = []
580
+ col_index += 1
581
+ y = layout.margin
582
+ if col_index >= columns:
583
+ # Page is full: close it and start a fresh page.
584
+ if current_page:
585
+ pages.append(current_page)
586
+ current_page = []
587
+ col_index = 0
588
+
589
+ if current_column:
590
+ current_page.append(current_column)
591
+ if current_page:
592
+ pages.append(current_page)
593
+ return pages, reasons
594
+
595
+
596
+ def _write_page(
597
+ page: list[list[_Placement]],
598
+ *,
599
+ page_index: int,
600
+ document_id: str,
601
+ layout: LayoutConfig,
602
+ pages_dir: Path,
603
+ annotations: Any,
604
+ color_mode: int,
605
+ ) -> None:
606
+ """Rasterize one page (list of logical columns) and append its annotation record.
607
+
608
+ Reading order follows the logical column sequence: top-to-bottom within a
609
+ column, then the next column. Physical x positions already encode the
610
+ ``page_direction`` (RTL emits columns right-to-left).
611
+ """
612
+ if color_mode == 1:
613
+ canvas = np.full((layout.page_height, layout.page_width), 255, dtype=np.uint8)
614
+ else:
615
+ canvas = np.full((layout.page_height, layout.page_width, 3), 255, dtype=np.uint8)
616
+
617
+ line_records: list[dict[str, Any]] = []
618
+ order = 0
619
+ span_records: list[dict[str, Any]] = []
620
+ block_line_orders: list[list[int]] = []
621
+ page_id = f"page_{page_index:06d}"
622
+ tables: list[dict[str, Any]] = []
623
+ word_order = 0
624
+ for column in page:
625
+ for placement in column:
626
+ table = placement.table
627
+ if table is None or any(table is other for other in tables):
628
+ continue
629
+ table["id"] = f"{page_id}_table_{len(tables):03d}"
630
+ table["reading_order"] = len(tables)
631
+ cells = sorted(table["cells"], key=lambda cell: (cell["row"],
632
+ -cell["col"] if layout.page_direction == "rtl" else cell["col"]))
633
+ table["cell_ids"] = []
634
+ for cell_order, cell in enumerate(cells):
635
+ cell["id"] = f"{table['id']}_r{cell['row']:03d}_c{cell['col']:03d}"
636
+ cell["table_id"] = table["id"]
637
+ cell["reading_order"] = cell_order
638
+ cell["line_ids"] = []
639
+ table["cell_ids"].append(cell["id"])
640
+ if layout.table_border_width:
641
+ box = cell["box"]
642
+ left, top = box["x"], box["y"]
643
+ right, bottom = left+box["w"], top+box["h"]
644
+ thickness = layout.table_border_width
645
+ canvas[top:top+thickness, left:right] = 0
646
+ canvas[bottom-thickness:bottom, left:right] = 0
647
+ canvas[top:bottom, left:left+thickness] = 0
648
+ canvas[top:bottom, right-thickness:right] = 0
649
+ tables.append(table)
650
+
651
+ for column in page:
652
+ column_orders: list[int] = []
653
+ for placement in column:
654
+ x, y, w, h = placement.x, placement.y, placement.w, placement.h
655
+ canvas[y : y + h, x : x + w] = placement.image
656
+
657
+ record: dict[str, Any] = {
658
+ "reading_order": order,
659
+ "text": placement.line.text,
660
+ "box": {"x": x, "y": y, "w": w, "h": h},
661
+ "font": placement.line.font,
662
+ "font_size": placement.line.font_size,
663
+ }
664
+ if layout.include_words or layout.writing_mode != "horizontal-tb" or layout.enable_tables:
665
+ record["id"] = f"{page_id}_line_{order:03d}"
666
+ record["writing_mode"] = layout.writing_mode
667
+ if layout.include_words:
668
+ words = []
669
+ source_h, source_w = placement.line.image.shape[:2]
670
+ for index, source_word in enumerate(placement.line.words):
671
+ word = dict(source_word)
672
+ word["polygon"] = [[x+px*w/source_w, y+py*h/source_h]
673
+ for px, py in source_word["polygon"]]
674
+ word["box"] = _box(word["polygon"])
675
+ word["id"] = f"{record['id']}_word_{index:03d}"
676
+ word["line_id"] = record["id"]
677
+ word["reading_order"] = word_order
678
+ word_order += 1
679
+ words.append(word)
680
+ record["words"] = words
681
+ if placement.table is not None and placement.cell is not None:
682
+ record["table_id"] = placement.table["id"]
683
+ record["cell_id"] = placement.cell["id"]
684
+ placement.table["line_orders"].append(order)
685
+ placement.cell["line_orders"].append(order)
686
+ placement.cell["line_ids"].append(record["id"])
687
+ if layout.include_baseline:
688
+ mask = _ink_mask(placement.image)
689
+ baseline = _baseline_offset(mask)
690
+ if layout.writing_mode != "horizontal-tb":
691
+ record["baseline"] = {"kind": "vertical_canvas_centerline", "points":
692
+ [[x+w/2, y+0.5], [x+w/2, y+h-0.5]]}
693
+ elif layout.transform_enabled or layout.include_words or layout.enable_tables:
694
+ if baseline is not None:
695
+ record["baseline"] = {
696
+ "kind": "ink_projection_peak",
697
+ "points": [
698
+ [x + 0.5, y + baseline + 0.5],
699
+ [x + w - 0.5, y + baseline + 0.5],
700
+ ],
701
+ }
702
+ else:
703
+ record["baseline"] = y + (baseline if baseline is not None else h // 2)
704
+ if layout.include_polygon:
705
+ mask = _ink_mask(placement.image)
706
+ polygon = _polygon_from_mask(mask)
707
+ if polygon is not None:
708
+ if layout.transform_enabled or layout.include_words or layout.enable_tables or layout.writing_mode != "horizontal-tb":
709
+ # Pixel cell edges for v3-v5; v2 retains last-pixel indices.
710
+ polygon[1][0] += 1
711
+ polygon[2][0] += 1
712
+ polygon[2][1] += 1
713
+ polygon[3][1] += 1
714
+ record["polygon"] = [[x + px, y + py] for px, py in polygon]
715
+ if placement.line.source_id is not None:
716
+ record["source_id"] = placement.line.source_id
717
+ if placement.line.source_group is not None:
718
+ record["source_group"] = placement.line.source_group
719
+ if placement.line.language is not None:
720
+ record["language"] = placement.line.language
721
+ line_records.append(record)
722
+
723
+ span_records.append(
724
+ {
725
+ "reading_order": order,
726
+ "text": placement.line.text,
727
+ "box": {"x": x, "y": y, "w": w, "h": h},
728
+ }
729
+ )
730
+ column_orders.append(order)
731
+ order += 1
732
+ block_line_orders.append(column_orders)
733
+
734
+ transform: Any = None
735
+ if layout.v4_enabled:
736
+ transform = layout.plan_transform(page_id)
737
+ transform.annotations(line_records, span_records, tables)
738
+ canvas = transform.raster(canvas)
739
+ canvas = apply_photometric(
740
+ canvas,
741
+ contrast=layout.page_contrast,
742
+ noise=layout.page_noise,
743
+ seed=layout.seed,
744
+ page_id=page_id,
745
+ )
746
+ elif layout.transform_enabled:
747
+ scale, rot = layout.resolve_transform_params(page_id)
748
+ transform = plan_page_transform(
749
+ layout.page_width,
750
+ layout.page_height,
751
+ scale,
752
+ rot,
753
+ layout.page_crop,
754
+ flip=layout.page_flip,
755
+ )
756
+ # Reject unsafe geometry and raster crops before writing either artifact.
757
+ transform.annotations(line_records, span_records, tables)
758
+ canvas = transform.raster(canvas)
759
+
760
+ image_rel = f"pages/{page_id}.{layout.output_format}"
761
+ write_ok = write_output_image(
762
+ pages_dir / f"{page_id}.{layout.output_format}",
763
+ canvas,
764
+ fmt=layout.output_format,
765
+ )
766
+ if not write_ok:
767
+ raise RuntimeError(
768
+ f"failed to write page image: {pages_dir / f'{page_id}.{layout.output_format}'}"
769
+ )
770
+
771
+ record: dict[str, Any] = {
772
+ "schema_version": (
773
+ 4
774
+ if layout.v4_enabled
775
+ else 3
776
+ if transform
777
+ else (2 if layout.geometry_extended else SCHEMA_VERSION)
778
+ ),
779
+ "page_id": page_id,
780
+ "image": image_rel,
781
+ "width": canvas.shape[1],
782
+ "height": canvas.shape[0],
783
+ "document_id": document_id,
784
+ "lines": line_records,
785
+ }
786
+ if layout.include_words or layout.writing_mode != "horizontal-tb" or layout.enable_tables:
787
+ record["schema_version"] = 5
788
+ record["writing_mode"] = layout.writing_mode
789
+ if tables:
790
+ record["tables"] = tables
791
+ if transform:
792
+ record["transform"] = transform.provenance()
793
+ if layout.v4_enabled and (layout.page_contrast != 1.0 or layout.page_noise):
794
+ record["degradation"] = degradation_provenance(
795
+ contrast=layout.page_contrast,
796
+ noise=layout.page_noise,
797
+ seed=layout.seed,
798
+ page_id=page_id,
799
+ )
800
+ if layout.hierarchy:
801
+ record["blocks"] = [
802
+ {"id": f"block_{b:03d}", "line_orders": orders}
803
+ for b, orders in enumerate(block_line_orders)
804
+ ]
805
+ if tables:
806
+ record["blocks"] = [{"id": f"{table['id']}_block", "kind": "table",
807
+ "table_id": table["id"], "cell_ids": table["cell_ids"],
808
+ "line_orders": table["line_orders"]} for table in tables]
809
+ record["spans"] = span_records
810
+
811
+ annotations.write(json.dumps(record, ensure_ascii=False) + "\n")
812
+
813
+
814
+ def _check_output_dir(output_dir: Path) -> None:
815
+ """Refuse existing content and symlinks, including dangling symlinks."""
816
+ if output_dir.is_symlink() or (
817
+ output_dir.exists() and (not output_dir.is_dir() or any(output_dir.iterdir()))
818
+ ):
819
+ raise FileExistsError(
820
+ f"Output directory must be absent or empty (refusing to overwrite): {output_dir}"
821
+ )
822
+
823
+
824
+ @dataclass(frozen=True)
825
+ class _DirectoryAccess:
826
+ mode: int
827
+ uid: int
828
+ gid: int
829
+ xattrs: dict[str, bytes]
830
+
831
+
832
+ def _output_metadata(path: Path) -> tuple[os.stat_result, _DirectoryAccess] | None:
833
+ try:
834
+ info = path.lstat()
835
+ except FileNotFoundError:
836
+ return None
837
+ if not stat.S_ISDIR(info.st_mode):
838
+ raise FileExistsError(f"Output directory must be absent or empty: {path}")
839
+ # Linux exposes POSIX ACLs as xattrs. Other platforms need their own ACL
840
+ # implementation before replacing an existing directory can be supported.
841
+ if sys.platform != "linux":
842
+ raise OSError(errno.ENOTSUP, "Preserving existing output access metadata requires Linux", path)
843
+ attrs = {name: os.getxattr(path, name) for name in os.listxattr(path)}
844
+ return info, _DirectoryAccess(stat.S_IMODE(info.st_mode), info.st_uid, info.st_gid, attrs)
845
+
846
+
847
+ def _copy_output_access(path: Path, access: _DirectoryAccess) -> None:
848
+ """Copy and verify access before children inherit group and default ACLs."""
849
+ info = path.stat()
850
+ if (info.st_uid, info.st_gid) != (access.uid, access.gid):
851
+ os.chown(path, access.uid, access.gid)
852
+ path.chmod(access.mode)
853
+ for name in os.listxattr(path):
854
+ if name not in access.xattrs:
855
+ os.removexattr(path, name)
856
+ for name, value in access.xattrs.items():
857
+ if name not in os.listxattr(path) or os.getxattr(path, name) != value:
858
+ os.setxattr(path, name, value)
859
+ copied = _output_metadata(path)
860
+ if copied is None or copied[1] != access:
861
+ raise PermissionError(f"Cannot preserve output directory access metadata: {path}")
862
+
863
+
864
+ def generate_layout(
865
+ corpus_path: str | Path,
866
+ output_dir: str | Path,
867
+ gen_cfg: GenerationConfig,
868
+ layout: LayoutConfig,
869
+ *,
870
+ min_length: int = 1,
871
+ max_length: int = 260,
872
+ max_lines: int = 0,
873
+ retry_limit: int = 10,
874
+ verbose: bool = True,
875
+ ) -> LayoutResult:
876
+ """Render a corpus and compose pages (single- or multi-column) with annotations.
877
+
878
+ Determinism: the module-level ``random`` is seeded from ``layout.seed``
879
+ before any font selection, so an identical corpus/config/seed reproduces
880
+ identical page images and annotations.
881
+
882
+ Export into a temporary sibling directory, then publish with one directory
883
+ rename after all writes close. Exceptions and zero-page results leave the
884
+ destination untouched; zero-page results still return rejection statistics.
885
+ The destination must be absent or an empty directory, never a symlink.
886
+ """
887
+ corpus_path = Path(corpus_path)
888
+ requested_output = Path(output_dir)
889
+ # Collapse '.' and '..' before deriving siblings, without following the
890
+ # final component (a destination symlink must still be refused).
891
+ output_dir = Path(os.path.abspath(requested_output))
892
+ _check_output_dir(output_dir)
893
+ original_metadata = _output_metadata(output_dir)
894
+
895
+ random.seed(layout.seed)
896
+
897
+ items, corpus_stats = load_corpus_with_stats(
898
+ corpus_path,
899
+ min_length=min_length,
900
+ max_length=max_length,
901
+ max_lines=max_lines,
902
+ normalizer=gen_cfg.normalizer,
903
+ )
904
+ if verbose:
905
+ print(f" Layout corpus: {corpus_path} -> {len(items)} passing line(s)")
906
+ print(
907
+ " Filtered: "
908
+ f"too_short={corpus_stats['too_short']} too_long={corpus_stats['too_long']} "
909
+ f"malformed={corpus_stats['malformed']} separator={corpus_stats['separator']} "
910
+ f"blank={corpus_stats['blank']}"
911
+ )
912
+
913
+ # Group by document, deterministic order.
914
+ groups: dict[str, list[str | Any]] = {}
915
+ for item in items:
916
+ groups.setdefault(_document_key(item), []).append(item)
917
+
918
+ font_manager = FontManager(
919
+ language=gen_cfg.language,
920
+ fonts_dir=gen_cfg.fonts_dir,
921
+ verbose=False,
922
+ multilingual=gen_cfg.multilingual,
923
+ font_pool=gen_cfg.font_pool,
924
+ )
925
+ renderer = ImageRenderer(font_manager, gen_cfg)
926
+
927
+ result = LayoutResult(
928
+ output_dir=requested_output,
929
+ documents=len(groups),
930
+ rejection_reasons={},
931
+ corpus_stats=dict(corpus_stats),
932
+ )
933
+ reasons: Counter[str] = Counter()
934
+ page_index = 0
935
+
936
+ output_dir.parent.mkdir(parents=True, exist_ok=True)
937
+ with TemporaryDirectory(prefix=f".{output_dir.name}.staging-", dir=output_dir.parent) as temp:
938
+ # The private container stays 0700. Its export child uses normal mkdir
939
+ # umask/default ACL inheritance, and is the directory we publish.
940
+ staging_dir = Path(temp) / "export"
941
+ staging_dir.mkdir()
942
+ if original_metadata is not None:
943
+ _copy_output_access(staging_dir, original_metadata[1])
944
+ pages_dir = staging_dir / "pages"
945
+ pages_dir.mkdir()
946
+ with (staging_dir / "annotations.jsonl").open("w", encoding="utf-8") as annotations:
947
+ for document_id in sorted(groups):
948
+ doc_items = groups[document_id]
949
+ rendered, render_reasons = _render_document(
950
+ doc_items,
951
+ renderer,
952
+ multilingual=gen_cfg.multilingual,
953
+ retry_limit=retry_limit,
954
+ layout=layout,
955
+ )
956
+ reasons.update(render_reasons)
957
+
958
+ pages, paginate_reasons = _paginate(rendered, layout)
959
+ reasons.update(paginate_reasons)
960
+
961
+ if len(pages) > layout.max_pages_per_doc:
962
+ result.rejected_documents += 1
963
+ reasons["document_too_many_pages"] += 1
964
+ _log.warning(
965
+ "Skipping document %r: %d pages exceeds --max-pages-per-doc=%d",
966
+ document_id,
967
+ len(pages),
968
+ layout.max_pages_per_doc,
969
+ )
970
+ continue
971
+
972
+ for page in pages:
973
+ _write_page(
974
+ page,
975
+ page_index=page_index,
976
+ document_id=document_id,
977
+ layout=layout,
978
+ pages_dir=pages_dir,
979
+ annotations=annotations,
980
+ color_mode=int(gen_cfg.color_mode),
981
+ )
982
+ page_index += 1
983
+
984
+ result.pages_written += len(pages)
985
+ result.lines_written += sum(len(column) for page in pages for column in page)
986
+
987
+ result.rejected_lines = len(items) - result.lines_written
988
+ result.rejection_reasons = dict(sorted(reasons.items()))
989
+
990
+ if verbose:
991
+ print(
992
+ f" Layout: {result.documents} document(s), {result.pages_written} page(s), "
993
+ f"{result.lines_written} line(s) placed, {result.rejected_lines} line(s) rejected, "
994
+ f"{result.rejected_documents} document(s) rejected"
995
+ )
996
+ if result.rejection_reasons:
997
+ print(f" Rejection reasons: {result.rejection_reasons}")
998
+
999
+ if result.pages_written:
1000
+ _check_output_dir(output_dir)
1001
+ current_metadata = _output_metadata(output_dir)
1002
+ if original_metadata is None:
1003
+ changed = current_metadata is not None
1004
+ else:
1005
+ changed = current_metadata is None or (
1006
+ current_metadata[0].st_dev,
1007
+ current_metadata[0].st_ino,
1008
+ current_metadata[1],
1009
+ ) != (
1010
+ original_metadata[0].st_dev,
1011
+ original_metadata[0].st_ino,
1012
+ original_metadata[1],
1013
+ )
1014
+ if changed:
1015
+ raise FileExistsError(f"Output directory changed during export: {output_dir}")
1016
+ try:
1017
+ # A directory rename atomically publishes the complete export.
1018
+ # It may replace an empty directory, but cannot replace content
1019
+ # another writer creates between the check and this rename.
1020
+ staging_dir.rename(output_dir)
1021
+ except OSError as exc:
1022
+ if exc.errno in {errno.EEXIST, errno.ENOTEMPTY}:
1023
+ raise FileExistsError(
1024
+ f"Output directory is not empty (refusing to overwrite): {output_dir}"
1025
+ ) from exc
1026
+ raise
1027
+
1028
+ return result