ocr-foundry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khocr_gen/__init__.py +11 -0
- khocr_gen/_pil_compat.py +25 -0
- khocr_gen/_rust_accel.py +204 -0
- khocr_gen/augmentation.py +1193 -0
- khocr_gen/benchmark.py +888 -0
- khocr_gen/benchmark_cmd.py +103 -0
- khocr_gen/cli.py +246 -0
- khocr_gen/combine.py +214 -0
- khocr_gen/combine_cmd.py +125 -0
- khocr_gen/config.py +1474 -0
- khocr_gen/config_loader.py +157 -0
- khocr_gen/corpus.py +621 -0
- khocr_gen/data_generator.py +1160 -0
- khocr_gen/errors.py +33 -0
- khocr_gen/evaluate_cmd.py +50 -0
- khocr_gen/evaluation.py +259 -0
- khocr_gen/fonts.py +316 -0
- khocr_gen/generate.py +227 -0
- khocr_gen/layout.py +1028 -0
- khocr_gen/layout_cmd.py +371 -0
- khocr_gen/layout_degradation.py +128 -0
- khocr_gen/layout_mesh.py +200 -0
- khocr_gen/layout_shaping.py +264 -0
- khocr_gen/layout_transform.py +238 -0
- khocr_gen/layout_warp.py +473 -0
- khocr_gen/line_height.py +127 -0
- khocr_gen/lmdb_pack.py +132 -0
- khocr_gen/logging.py +25 -0
- khocr_gen/normalizer.py +221 -0
- khocr_gen/parallel.py +366 -0
- khocr_gen/provenance.py +328 -0
- khocr_gen/rendering.py +1607 -0
- khocr_gen/text_support.py +429 -0
- khocr_gen/verify.py +471 -0
- khocr_gen/viewer.py +172 -0
- ocr_foundry-0.2.0.dist-info/METADATA +174 -0
- ocr_foundry-0.2.0.dist-info/RECORD +40 -0
- ocr_foundry-0.2.0.dist-info/WHEEL +4 -0
- ocr_foundry-0.2.0.dist-info/entry_points.txt +3 -0
- ocr_foundry-0.2.0.dist-info/licenses/LICENSE +21 -0
khocr_gen/layout.py
ADDED
|
@@ -0,0 +1,1028 @@
|
|
|
1
|
+
"""Page/document layout composition (Milestone 5, bounded vertical slice).
|
|
2
|
+
|
|
3
|
+
This module composes already-rendered *line* canvases into single-column pages.
|
|
4
|
+
It deliberately reuses the line rasterizer (``ImageRenderer``) and the normalizer
|
|
5
|
+
rather than inventing a second text engine, so annotation text matches the
|
|
6
|
+
pixels that were drawn.
|
|
7
|
+
|
|
8
|
+
Scope and hard limits
|
|
9
|
+
---------------------
|
|
10
|
+
* One or more columns, top-to-bottom. Column gap sits between columns, never in
|
|
11
|
+
the outer margins. `page_direction=rtl` reverses physical column emission
|
|
12
|
+
order; reading order always follows the logical column sequence
|
|
13
|
+
(top-to-bottom within a column, then the next column).
|
|
14
|
+
* Boxes are **line-level** axis-aligned rectangles around the full rendered
|
|
15
|
+
line canvas (which includes the renderer's clean padding). No word/span,
|
|
16
|
+
baseline or polygon geometry is fabricated unless the corresponding flag is
|
|
17
|
+
set.
|
|
18
|
+
* Rendering is always clean (no geometric augmentation). Augmentation would
|
|
19
|
+
invalidate the box-to-pixels relationship, so it must stay disabled.
|
|
20
|
+
* Grouping is document-disjoint: every page belongs to exactly one document,
|
|
21
|
+
and a document's pages never share a page with another document.
|
|
22
|
+
|
|
23
|
+
Plain ``.txt`` corpora have no ``source_group``. Their lines form a single
|
|
24
|
+
synthetic document named ``__unlabeled__``. In JSONL manifests, lines with a
|
|
25
|
+
``source_group`` are grouped by it; lines without one are collected into the
|
|
26
|
+
same ``__unlabeled__`` synthetic document, kept separate from named documents.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import errno
|
|
32
|
+
import json
|
|
33
|
+
import logging
|
|
34
|
+
import math
|
|
35
|
+
import os
|
|
36
|
+
import random
|
|
37
|
+
import stat
|
|
38
|
+
import sys
|
|
39
|
+
from collections import Counter
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from tempfile import TemporaryDirectory
|
|
43
|
+
from typing import TYPE_CHECKING, Any
|
|
44
|
+
|
|
45
|
+
import cv2
|
|
46
|
+
import numpy as np
|
|
47
|
+
|
|
48
|
+
from .augmentation import write_output_image
|
|
49
|
+
from .corpus import load_corpus_with_stats
|
|
50
|
+
from .errors import InputValidationError
|
|
51
|
+
from .fonts import FontManager
|
|
52
|
+
from .layout_degradation import (
|
|
53
|
+
apply_photometric,
|
|
54
|
+
stream_bytes,
|
|
55
|
+
stream_key,
|
|
56
|
+
validate_photometric,
|
|
57
|
+
)
|
|
58
|
+
from .layout_degradation import provenance as degradation_provenance
|
|
59
|
+
from .layout_transform import plan_page_transform
|
|
60
|
+
from .layout_warp import plan_page_warp, validate_elastic_amplitude, validate_perspective
|
|
61
|
+
from .rendering import ImageRenderer
|
|
62
|
+
|
|
63
|
+
if TYPE_CHECKING:
|
|
64
|
+
from .config import GenerationConfig
|
|
65
|
+
|
|
66
|
+
_log = logging.getLogger("khocr_gen.layout")
|
|
67
|
+
|
|
68
|
+
SCHEMA_VERSION = 1
|
|
69
|
+
SYNTHETIC_DOCUMENT = "__unlabeled__"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class LayoutConfig:
|
|
74
|
+
"""Page-composition parameters (pixels)."""
|
|
75
|
+
|
|
76
|
+
page_width: int = 1000
|
|
77
|
+
page_height: int = 1400
|
|
78
|
+
margin: int = 40
|
|
79
|
+
line_spacing: int = 8
|
|
80
|
+
max_pages_per_doc: int = 100
|
|
81
|
+
seed: int = 42
|
|
82
|
+
line_height: int = 48
|
|
83
|
+
line_width: int | None = None
|
|
84
|
+
output_format: str = "png"
|
|
85
|
+
|
|
86
|
+
# Multi-column geometry
|
|
87
|
+
columns: int = 1
|
|
88
|
+
column_gap: int = 20
|
|
89
|
+
page_direction: str = "ltr"
|
|
90
|
+
|
|
91
|
+
# Optional per-line / page geometry annotations
|
|
92
|
+
include_baseline: bool = False
|
|
93
|
+
include_polygon: bool = False
|
|
94
|
+
hierarchy: bool = False
|
|
95
|
+
include_words: bool = False
|
|
96
|
+
writing_mode: str = "horizontal-tb"
|
|
97
|
+
enable_tables: bool = False
|
|
98
|
+
table_rows: int = 3
|
|
99
|
+
table_columns: int = 3
|
|
100
|
+
table_border_width: int = 1
|
|
101
|
+
table_padding: int = 6
|
|
102
|
+
|
|
103
|
+
# Fixed page transforms; independent of line augmentation and its RNG.
|
|
104
|
+
page_scale: float = 1.0
|
|
105
|
+
page_rotation: float = 0.0
|
|
106
|
+
page_crop: tuple[int, int, int, int] | None = None
|
|
107
|
+
page_flip: str = "none"
|
|
108
|
+
|
|
109
|
+
# Transform sampling ranges (sampled deterministically per page).
|
|
110
|
+
page_scale_range: tuple[float, float] | None = None
|
|
111
|
+
page_rotation_range: tuple[float, float] | None = None
|
|
112
|
+
|
|
113
|
+
# Contract v4: perspective, seeded elastic mesh, pointwise photometric degradation.
|
|
114
|
+
page_perspective: tuple[float, ...] | None = None
|
|
115
|
+
page_elastic_amplitude: float = 0.0
|
|
116
|
+
page_contrast: float = 1.0
|
|
117
|
+
page_noise: int = 0
|
|
118
|
+
def __post_init__(self) -> None:
|
|
119
|
+
if self.writing_mode not in {"horizontal-tb", "vertical-rl", "vertical-lr"}:
|
|
120
|
+
raise InputValidationError("invalid writing_mode; use horizontal-tb, vertical-rl, or vertical-lr")
|
|
121
|
+
for name in ("table_rows", "table_columns"):
|
|
122
|
+
value = getattr(self, name)
|
|
123
|
+
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
|
124
|
+
raise InputValidationError(f"{name} must be a positive integer")
|
|
125
|
+
for name in ("table_border_width", "table_padding"):
|
|
126
|
+
value = getattr(self, name)
|
|
127
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
128
|
+
raise InputValidationError(f"{name} must be a non-negative integer")
|
|
129
|
+
if self.enable_tables:
|
|
130
|
+
inset = 2 * (self.table_padding + self.table_border_width)
|
|
131
|
+
if self.content_width // self.table_columns <= inset or self.content_height <= inset:
|
|
132
|
+
raise InputValidationError("table borders/padding must leave positive cell content area")
|
|
133
|
+
if self.include_words or self.writing_mode != "horizontal-tb":
|
|
134
|
+
from .layout_shaping import require_layout_shaper
|
|
135
|
+
require_layout_shaper()
|
|
136
|
+
if self.page_width <= 0 or self.page_height <= 0:
|
|
137
|
+
raise InputValidationError("page width and height must be positive")
|
|
138
|
+
if self.margin < 0:
|
|
139
|
+
raise InputValidationError("margin must be non-negative")
|
|
140
|
+
if 2 * self.margin >= self.page_width:
|
|
141
|
+
raise InputValidationError("margin must leave a positive content width")
|
|
142
|
+
if 2 * self.margin >= self.page_height:
|
|
143
|
+
raise InputValidationError("margin must leave a positive content height")
|
|
144
|
+
if self.line_spacing < 0:
|
|
145
|
+
raise InputValidationError("line spacing must be non-negative")
|
|
146
|
+
if self.max_pages_per_doc < 1:
|
|
147
|
+
raise InputValidationError("max pages per document must be at least 1")
|
|
148
|
+
if self.line_height <= 0:
|
|
149
|
+
raise InputValidationError("line height must be positive")
|
|
150
|
+
if self.columns < 1:
|
|
151
|
+
raise InputValidationError("columns must be at least 1")
|
|
152
|
+
if self.column_gap < 0:
|
|
153
|
+
raise InputValidationError("column gap must be non-negative")
|
|
154
|
+
if self.page_direction not in {"ltr", "rtl"}:
|
|
155
|
+
raise InputValidationError("page direction must be 'ltr' or 'rtl'")
|
|
156
|
+
if self.line_width is not None:
|
|
157
|
+
if self.line_width <= 0:
|
|
158
|
+
raise InputValidationError("line width must be positive")
|
|
159
|
+
if self.line_width > self.column_width:
|
|
160
|
+
raise InputValidationError(
|
|
161
|
+
f"line width {self.line_width} exceeds column width {self.column_width} "
|
|
162
|
+
"(content width minus column gaps, divided across columns)"
|
|
163
|
+
)
|
|
164
|
+
if not self.output_format:
|
|
165
|
+
raise InputValidationError("output format must be non-empty")
|
|
166
|
+
if self.page_flip not in {"none", "horizontal", "vertical", "both"}:
|
|
167
|
+
raise InputValidationError(
|
|
168
|
+
f"invalid page_flip {self.page_flip!r}; must be 'none', 'horizontal', 'vertical', or 'both'"
|
|
169
|
+
)
|
|
170
|
+
if self.page_scale_range is not None:
|
|
171
|
+
if len(self.page_scale_range) != 2:
|
|
172
|
+
raise InputValidationError("page_scale_range must be a tuple of (min, max)")
|
|
173
|
+
s_min, s_max = self.page_scale_range
|
|
174
|
+
if not (math.isfinite(s_min) and math.isfinite(s_max) and s_min > 0 and s_max >= s_min):
|
|
175
|
+
raise InputValidationError("page_scale_range values must be positive, finite, and min <= max")
|
|
176
|
+
if self.page_rotation_range is not None:
|
|
177
|
+
if len(self.page_rotation_range) != 2:
|
|
178
|
+
raise InputValidationError("page_rotation_range must be a tuple of (min, max)")
|
|
179
|
+
r_min, r_max = self.page_rotation_range
|
|
180
|
+
if not (math.isfinite(r_min) and math.isfinite(r_max) and r_max >= r_min):
|
|
181
|
+
raise InputValidationError("page_rotation_range values must be finite and min <= max")
|
|
182
|
+
self.page_perspective = validate_perspective(self.page_perspective)
|
|
183
|
+
self.page_elastic_amplitude = validate_elastic_amplitude(self.page_elastic_amplitude)
|
|
184
|
+
validate_photometric(self.page_contrast, self.page_noise)
|
|
185
|
+
if self.v4_enabled:
|
|
186
|
+
# Fixed-parameter validation (sizes, bounds, crop); mesh draws are per page.
|
|
187
|
+
self.plan_transform("page_000000")
|
|
188
|
+
elif self.transform_enabled:
|
|
189
|
+
scale, rot = self.resolve_transform_params("page_000000")
|
|
190
|
+
plan_page_transform(
|
|
191
|
+
self.page_width,
|
|
192
|
+
self.page_height,
|
|
193
|
+
scale,
|
|
194
|
+
rot,
|
|
195
|
+
self.page_crop,
|
|
196
|
+
flip=self.page_flip,
|
|
197
|
+
)
|
|
198
|
+
def resolve_transform_params(self, page_id: str) -> tuple[float, float]:
|
|
199
|
+
"""Resolve (scale, rotation) for a page, sampling uniformly if ranges are configured."""
|
|
200
|
+
scale = self.page_scale
|
|
201
|
+
rotation = self.page_rotation
|
|
202
|
+
if self.page_scale_range is not None or self.page_rotation_range is not None:
|
|
203
|
+
key = stream_key(self.seed, page_id, "transform_sampling")
|
|
204
|
+
data = stream_bytes(key, 16)
|
|
205
|
+
u1, u2 = np.frombuffer(data, "<u8")
|
|
206
|
+
f1 = float(u1) / float(2**64 - 1)
|
|
207
|
+
f2 = float(u2) / float(2**64 - 1)
|
|
208
|
+
if self.page_scale_range is not None:
|
|
209
|
+
s_min, s_max = self.page_scale_range
|
|
210
|
+
scale = s_min + f1 * (s_max - s_min)
|
|
211
|
+
if self.page_rotation_range is not None:
|
|
212
|
+
r_min, r_max = self.page_rotation_range
|
|
213
|
+
rotation = r_min + f2 * (r_max - r_min)
|
|
214
|
+
return scale, rotation
|
|
215
|
+
|
|
216
|
+
@property
|
|
217
|
+
def affine_enabled(self) -> bool:
|
|
218
|
+
"""Keep historical output for identity defaults, including full turns."""
|
|
219
|
+
return (
|
|
220
|
+
self.page_scale != 1.0
|
|
221
|
+
or self.page_rotation % 360 != 0
|
|
222
|
+
or self.page_crop is not None
|
|
223
|
+
or self.page_flip != "none"
|
|
224
|
+
or self.page_scale_range is not None
|
|
225
|
+
or self.page_rotation_range is not None
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
@property
|
|
229
|
+
def v4_enabled(self) -> bool:
|
|
230
|
+
"""Any effective v4 option selects contract v4; affine-only output stays v3."""
|
|
231
|
+
return (
|
|
232
|
+
self.page_perspective is not None
|
|
233
|
+
or self.page_elastic_amplitude != 0
|
|
234
|
+
or self.page_contrast != 1.0
|
|
235
|
+
or self.page_noise != 0
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def transform_enabled(self) -> bool:
|
|
240
|
+
"""True for the v3 geometry conventions (affine or any v4 option)."""
|
|
241
|
+
return self.affine_enabled or self.v4_enabled
|
|
242
|
+
|
|
243
|
+
def plan_transform(self, page_id: str):
|
|
244
|
+
"""Plan this page's v4 geometry; the page id keys the elastic mesh stream."""
|
|
245
|
+
scale, rotation = self.resolve_transform_params(page_id)
|
|
246
|
+
return plan_page_warp(
|
|
247
|
+
self.page_width,
|
|
248
|
+
self.page_height,
|
|
249
|
+
scale,
|
|
250
|
+
rotation,
|
|
251
|
+
self.page_crop,
|
|
252
|
+
self.page_perspective,
|
|
253
|
+
self.page_elastic_amplitude,
|
|
254
|
+
flip=self.page_flip,
|
|
255
|
+
seed=self.seed,
|
|
256
|
+
page_id=page_id,
|
|
257
|
+
)
|
|
258
|
+
@property
|
|
259
|
+
def content_width(self) -> int:
|
|
260
|
+
return self.page_width - 2 * self.margin
|
|
261
|
+
|
|
262
|
+
@property
|
|
263
|
+
def content_height(self) -> int:
|
|
264
|
+
return self.page_height - 2 * self.margin
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def column_width(self) -> int:
|
|
268
|
+
"""Usable width of a single column, gaps excluded from outer margins."""
|
|
269
|
+
total_gap = self.column_gap * (self.columns - 1)
|
|
270
|
+
remaining = self.content_width - total_gap
|
|
271
|
+
return max(1, remaining // self.columns)
|
|
272
|
+
|
|
273
|
+
@property
|
|
274
|
+
def geometry_extended(self) -> bool:
|
|
275
|
+
"""True when any new geometry flag departs from the single-column LTR default."""
|
|
276
|
+
return (
|
|
277
|
+
self.columns > 1
|
|
278
|
+
or self.page_direction == "rtl"
|
|
279
|
+
or self.include_baseline
|
|
280
|
+
or self.include_polygon
|
|
281
|
+
or self.hierarchy
|
|
282
|
+
or self.include_words
|
|
283
|
+
or self.writing_mode != "horizontal-tb"
|
|
284
|
+
or self.enable_tables
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
@dataclass
|
|
289
|
+
class _RenderedLine:
|
|
290
|
+
"""A cleanly rendered line canvas plus its source metadata."""
|
|
291
|
+
|
|
292
|
+
text: str
|
|
293
|
+
image: np.ndarray
|
|
294
|
+
source_id: str | None = None
|
|
295
|
+
source_group: str | None = None
|
|
296
|
+
language: str | None = None
|
|
297
|
+
font: str | None = None
|
|
298
|
+
font_size: int | None = None
|
|
299
|
+
words: list[dict[str, Any]] = field(default_factory=list)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
@dataclass
|
|
303
|
+
class _Placement:
|
|
304
|
+
line: _RenderedLine
|
|
305
|
+
image: np.ndarray # possibly width-scaled copy
|
|
306
|
+
x: int
|
|
307
|
+
y: int
|
|
308
|
+
w: int
|
|
309
|
+
h: int
|
|
310
|
+
table: dict[str, Any] | None = None
|
|
311
|
+
cell: dict[str, Any] | None = None
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@dataclass
|
|
315
|
+
class LayoutResult:
|
|
316
|
+
"""Summary returned by :func:`generate_layout`."""
|
|
317
|
+
|
|
318
|
+
output_dir: Path
|
|
319
|
+
documents: int = 0
|
|
320
|
+
pages_written: int = 0
|
|
321
|
+
lines_written: int = 0
|
|
322
|
+
rejected_lines: int = 0
|
|
323
|
+
rejected_documents: int = 0
|
|
324
|
+
rejection_reasons: dict[str, int] = field(default_factory=dict)
|
|
325
|
+
corpus_stats: dict[str, int] = field(default_factory=dict)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _document_key(item: str | Any) -> str:
|
|
329
|
+
"""Group key for a corpus item.
|
|
330
|
+
|
|
331
|
+
Named ``source_group`` values stay distinct. Items without one share a
|
|
332
|
+
single synthetic document so plain ``.txt`` corpora still produce pages.
|
|
333
|
+
"""
|
|
334
|
+
group = getattr(item, "source_group", None)
|
|
335
|
+
return group if group is not None else SYNTHETIC_DOCUMENT
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _sanitize_line_text(text: str) -> str:
|
|
339
|
+
"""Replace embedded tabs (labels/composition treat them as layout noise)."""
|
|
340
|
+
return text.replace("\t", " ")
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _render_document(
|
|
344
|
+
items: list[str | Any],
|
|
345
|
+
renderer: ImageRenderer,
|
|
346
|
+
*,
|
|
347
|
+
multilingual: bool,
|
|
348
|
+
retry_limit: int,
|
|
349
|
+
layout: LayoutConfig | None = None,
|
|
350
|
+
) -> tuple[list[_RenderedLine], Counter[str]]:
|
|
351
|
+
"""Render every line in a document cleanly.
|
|
352
|
+
|
|
353
|
+
Returns ``(rendered, reason_counts)``.
|
|
354
|
+
"""
|
|
355
|
+
rendered: list[_RenderedLine] = []
|
|
356
|
+
reasons: Counter[str] = Counter()
|
|
357
|
+
|
|
358
|
+
for item in items:
|
|
359
|
+
text = _sanitize_line_text(str(item))
|
|
360
|
+
if not text:
|
|
361
|
+
reasons["empty_text"] += 1
|
|
362
|
+
continue
|
|
363
|
+
if multilingual and not renderer.is_line_eligible(text):
|
|
364
|
+
reasons["no_eligible_font"] += 1
|
|
365
|
+
continue
|
|
366
|
+
if layout is not None and (layout.include_words or layout.writing_mode != "horizontal-tb"):
|
|
367
|
+
from .layout_shaping import render_positioned
|
|
368
|
+
result = render_positioned(renderer, text, layout.writing_mode, layout.include_words, retry_limit)
|
|
369
|
+
else:
|
|
370
|
+
result = renderer.render_clean_with_meta(text, retry_limit=retry_limit)
|
|
371
|
+
if result is None:
|
|
372
|
+
reasons["render_failed"] += 1
|
|
373
|
+
continue
|
|
374
|
+
image, _target_height, meta = result
|
|
375
|
+
rendered.append(
|
|
376
|
+
_RenderedLine(
|
|
377
|
+
text=text,
|
|
378
|
+
image=image,
|
|
379
|
+
source_id=getattr(item, "source_id", None),
|
|
380
|
+
source_group=getattr(item, "source_group", None),
|
|
381
|
+
language=getattr(item, "language", None),
|
|
382
|
+
font=meta.get("font"),
|
|
383
|
+
font_size=meta.get("font_size"),
|
|
384
|
+
words=meta.get("words", []),
|
|
385
|
+
)
|
|
386
|
+
)
|
|
387
|
+
return rendered, reasons
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _ink_mask(image: np.ndarray, threshold: int = 128) -> np.ndarray:
|
|
391
|
+
"""Boolean mask of dark (ink) pixels on a clean rendered line canvas.
|
|
392
|
+
|
|
393
|
+
Grayscale images are used directly; RGB images are reduced to their first
|
|
394
|
+
channel (the clean rasterizer draws neutral dark text on a light background,
|
|
395
|
+
so any channel suffices).
|
|
396
|
+
"""
|
|
397
|
+
gray = image if image.ndim == 2 else image[..., 0]
|
|
398
|
+
return gray < threshold
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _baseline_offset(mask: np.ndarray) -> int | None:
|
|
402
|
+
"""Row offset (relative to the canvas top) of the heaviest ink band.
|
|
403
|
+
|
|
404
|
+
Heuristic: the horizontal projection peak of the ink mask approximates the
|
|
405
|
+
line baseline on which most glyphs sit. Returns ``None`` when no ink is
|
|
406
|
+
present.
|
|
407
|
+
"""
|
|
408
|
+
if not mask.any():
|
|
409
|
+
return None
|
|
410
|
+
row_counts = mask.sum(axis=1)
|
|
411
|
+
return int(np.argmax(row_counts))
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _polygon_from_mask(mask: np.ndarray) -> list[list[int]] | None:
|
|
415
|
+
"""Clockwise bounding-box polygon of the ink mask, origin top-left, y down.
|
|
416
|
+
|
|
417
|
+
Returns the four corners ``[x0, y0], [x1, y0], [x1, y1], [x0, y1]`` of the
|
|
418
|
+
ink bounding box, or ``None`` when no ink is present.
|
|
419
|
+
"""
|
|
420
|
+
if not mask.any():
|
|
421
|
+
return None
|
|
422
|
+
rows = np.flatnonzero(mask.any(axis=1))
|
|
423
|
+
cols = np.flatnonzero(mask.any(axis=0))
|
|
424
|
+
y0, y1 = int(rows[0]), int(rows[-1])
|
|
425
|
+
x0, x1 = int(cols[0]), int(cols[-1])
|
|
426
|
+
return [[x0, y0], [x1, y0], [x1, y1], [x0, y1]]
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _fit_image(image: np.ndarray, width: int, height: int) -> np.ndarray:
|
|
430
|
+
h, w = image.shape[:2]
|
|
431
|
+
scale = min(1.0, width / w, height / h)
|
|
432
|
+
if scale == 1:
|
|
433
|
+
return image
|
|
434
|
+
return cv2.resize(image, (max(1, min(width, round(w * scale))),
|
|
435
|
+
max(1, min(height, round(h * scale)))), interpolation=cv2.INTER_AREA)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _rect(x: float, y: float, w: float, h: float) -> list[list[float]]:
|
|
439
|
+
return [[x, y], [x+w, y], [x+w, y+h], [x, y+h]]
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _box(polygon: list[list[float]]) -> dict[str, int]:
|
|
443
|
+
x0, y0 = (math.floor(min(p[i] for p in polygon)) for i in (0, 1))
|
|
444
|
+
x1, y1 = (math.ceil(max(p[i] for p in polygon)) for i in (0, 1))
|
|
445
|
+
return {"x": x0, "y": y0, "w": x1-x0, "h": y1-y0}
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _paginate_vertical(rendered: list[_RenderedLine], layout: LayoutConfig) -> list[list[list[_Placement]]]:
|
|
449
|
+
pages = []
|
|
450
|
+
current = []
|
|
451
|
+
used = 0
|
|
452
|
+
for line in rendered:
|
|
453
|
+
image = _fit_image(line.image, layout.content_width, layout.content_height)
|
|
454
|
+
h, w = image.shape[:2]
|
|
455
|
+
if current and used + w > layout.content_width:
|
|
456
|
+
pages.append([current])
|
|
457
|
+
current, used = [], 0
|
|
458
|
+
x = (layout.page_width-layout.margin-used-w if layout.writing_mode == "vertical-rl"
|
|
459
|
+
else layout.margin+used)
|
|
460
|
+
current.append(_Placement(line, image, x, layout.margin, w, h))
|
|
461
|
+
used += w + layout.line_spacing
|
|
462
|
+
if current:
|
|
463
|
+
pages.append([current])
|
|
464
|
+
return pages
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _paginate_tables(rendered: list[_RenderedLine], layout: LayoutConfig) -> list[list[list[_Placement]]]:
|
|
468
|
+
pages = []
|
|
469
|
+
cursor = 0
|
|
470
|
+
inset = layout.table_padding + layout.table_border_width
|
|
471
|
+
edges = [layout.margin + c * layout.content_width // layout.table_columns
|
|
472
|
+
for c in range(layout.table_columns + 1)]
|
|
473
|
+
while cursor < len(rendered):
|
|
474
|
+
table: dict[str, Any] = {"cells": [], "line_orders": []}
|
|
475
|
+
placements = []
|
|
476
|
+
y = layout.margin
|
|
477
|
+
for row in range(layout.table_rows):
|
|
478
|
+
chunk = rendered[cursor:cursor+layout.table_columns]
|
|
479
|
+
if not chunk:
|
|
480
|
+
break
|
|
481
|
+
fitted = []
|
|
482
|
+
for index, line in enumerate(chunk):
|
|
483
|
+
col = layout.table_columns-1-index if layout.page_direction == "rtl" else index
|
|
484
|
+
image = _fit_image(line.image, edges[col+1]-edges[col]-2*inset,
|
|
485
|
+
layout.content_height-2*inset)
|
|
486
|
+
fitted.append((col, line, image))
|
|
487
|
+
row_h = max(image.shape[0] for _, _, image in fitted) + 2*inset
|
|
488
|
+
if placements and y + row_h > layout.page_height-layout.margin:
|
|
489
|
+
break
|
|
490
|
+
for col in range(layout.table_columns):
|
|
491
|
+
cell = {"row": row, "col": col, "row_span": 1, "col_span": 1,
|
|
492
|
+
"polygon": _rect(edges[col], y, edges[col+1]-edges[col], row_h),
|
|
493
|
+
"line_orders": []}
|
|
494
|
+
cell["box"] = _box(cell["polygon"])
|
|
495
|
+
table["cells"].append(cell)
|
|
496
|
+
for col, line, image in fitted:
|
|
497
|
+
cell = table["cells"][-layout.table_columns+col]
|
|
498
|
+
h, w = image.shape[:2]
|
|
499
|
+
placements.append(_Placement(line, image, edges[col]+inset, y+inset, w, h, table, cell))
|
|
500
|
+
cursor += len(chunk)
|
|
501
|
+
y += row_h
|
|
502
|
+
table["polygon"] = _rect(layout.margin, layout.margin, layout.content_width, y-layout.margin)
|
|
503
|
+
table["box"] = _box(table["polygon"])
|
|
504
|
+
pages.append([placements])
|
|
505
|
+
return pages
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _paginate(
|
|
509
|
+
rendered: list[_RenderedLine],
|
|
510
|
+
layout: LayoutConfig,
|
|
511
|
+
) -> tuple[list[list[list[_Placement]]], Counter[str]]:
|
|
512
|
+
"""Pack rendered lines into column-flowing pages without clipping or dropping.
|
|
513
|
+
|
|
514
|
+
Lines flow top-to-bottom within a logical column; when a column is full the
|
|
515
|
+
flow continues at the top of the next logical column, and when every column
|
|
516
|
+
on a page is full a new page begins. Physical column emission order follows
|
|
517
|
+
``layout.page_direction`` (LTR left-to-right, RTL right-to-left) while
|
|
518
|
+
reading order always follows the logical column sequence.
|
|
519
|
+
|
|
520
|
+
A line wider than a column is uniformly scaled down to fit; a line taller
|
|
521
|
+
than the content height cannot be placed and is reported. Returns
|
|
522
|
+
``(pages, reason_counts)`` where each page is a list of columns and each
|
|
523
|
+
column is a list of placements in logical fill order.
|
|
524
|
+
"""
|
|
525
|
+
if layout.enable_tables:
|
|
526
|
+
return _paginate_tables(rendered, layout), Counter()
|
|
527
|
+
if layout.writing_mode != "horizontal-tb":
|
|
528
|
+
return _paginate_vertical(rendered, layout), Counter()
|
|
529
|
+
content_h = layout.content_height
|
|
530
|
+
bottom = layout.page_height - layout.margin
|
|
531
|
+
col_w = layout.column_width
|
|
532
|
+
gap = layout.column_gap
|
|
533
|
+
columns = layout.columns
|
|
534
|
+
|
|
535
|
+
# Physical x offset of each logical column; RTL reverses emission order.
|
|
536
|
+
if layout.page_direction == "rtl":
|
|
537
|
+
col_x = [layout.margin + (columns - 1 - c) * (col_w + gap) for c in range(columns)]
|
|
538
|
+
else:
|
|
539
|
+
col_x = [layout.margin + c * (col_w + gap) for c in range(columns)]
|
|
540
|
+
|
|
541
|
+
pages: list[list[list[_Placement]]] = []
|
|
542
|
+
current_page: list[list[_Placement]] = []
|
|
543
|
+
current_column: list[_Placement] = []
|
|
544
|
+
reasons: Counter[str] = Counter()
|
|
545
|
+
col_index = 0
|
|
546
|
+
y = layout.margin
|
|
547
|
+
|
|
548
|
+
for line in rendered:
|
|
549
|
+
image = line.image
|
|
550
|
+
h, w = image.shape[:2]
|
|
551
|
+
if w > col_w:
|
|
552
|
+
scale = col_w / w
|
|
553
|
+
new_w = max(1, col_w)
|
|
554
|
+
new_h = max(1, round(h * scale))
|
|
555
|
+
image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA)
|
|
556
|
+
w, h = new_w, new_h
|
|
557
|
+
if h > content_h:
|
|
558
|
+
reasons["line_too_tall"] += 1
|
|
559
|
+
continue
|
|
560
|
+
|
|
561
|
+
# Advance columns/pages until the line fits at the top of a column.
|
|
562
|
+
while True:
|
|
563
|
+
if y + h <= bottom:
|
|
564
|
+
current_column.append(
|
|
565
|
+
_Placement(
|
|
566
|
+
line=line,
|
|
567
|
+
image=image,
|
|
568
|
+
x=col_x[col_index],
|
|
569
|
+
y=y,
|
|
570
|
+
w=w,
|
|
571
|
+
h=h,
|
|
572
|
+
)
|
|
573
|
+
)
|
|
574
|
+
y += h + layout.line_spacing
|
|
575
|
+
break
|
|
576
|
+
# Current column is full: close it and move to the next.
|
|
577
|
+
if current_column:
|
|
578
|
+
current_page.append(current_column)
|
|
579
|
+
current_column = []
|
|
580
|
+
col_index += 1
|
|
581
|
+
y = layout.margin
|
|
582
|
+
if col_index >= columns:
|
|
583
|
+
# Page is full: close it and start a fresh page.
|
|
584
|
+
if current_page:
|
|
585
|
+
pages.append(current_page)
|
|
586
|
+
current_page = []
|
|
587
|
+
col_index = 0
|
|
588
|
+
|
|
589
|
+
if current_column:
|
|
590
|
+
current_page.append(current_column)
|
|
591
|
+
if current_page:
|
|
592
|
+
pages.append(current_page)
|
|
593
|
+
return pages, reasons
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _write_page(
|
|
597
|
+
page: list[list[_Placement]],
|
|
598
|
+
*,
|
|
599
|
+
page_index: int,
|
|
600
|
+
document_id: str,
|
|
601
|
+
layout: LayoutConfig,
|
|
602
|
+
pages_dir: Path,
|
|
603
|
+
annotations: Any,
|
|
604
|
+
color_mode: int,
|
|
605
|
+
) -> None:
|
|
606
|
+
"""Rasterize one page (list of logical columns) and append its annotation record.
|
|
607
|
+
|
|
608
|
+
Reading order follows the logical column sequence: top-to-bottom within a
|
|
609
|
+
column, then the next column. Physical x positions already encode the
|
|
610
|
+
``page_direction`` (RTL emits columns right-to-left).
|
|
611
|
+
"""
|
|
612
|
+
if color_mode == 1:
|
|
613
|
+
canvas = np.full((layout.page_height, layout.page_width), 255, dtype=np.uint8)
|
|
614
|
+
else:
|
|
615
|
+
canvas = np.full((layout.page_height, layout.page_width, 3), 255, dtype=np.uint8)
|
|
616
|
+
|
|
617
|
+
line_records: list[dict[str, Any]] = []
|
|
618
|
+
order = 0
|
|
619
|
+
span_records: list[dict[str, Any]] = []
|
|
620
|
+
block_line_orders: list[list[int]] = []
|
|
621
|
+
page_id = f"page_{page_index:06d}"
|
|
622
|
+
tables: list[dict[str, Any]] = []
|
|
623
|
+
word_order = 0
|
|
624
|
+
for column in page:
|
|
625
|
+
for placement in column:
|
|
626
|
+
table = placement.table
|
|
627
|
+
if table is None or any(table is other for other in tables):
|
|
628
|
+
continue
|
|
629
|
+
table["id"] = f"{page_id}_table_{len(tables):03d}"
|
|
630
|
+
table["reading_order"] = len(tables)
|
|
631
|
+
cells = sorted(table["cells"], key=lambda cell: (cell["row"],
|
|
632
|
+
-cell["col"] if layout.page_direction == "rtl" else cell["col"]))
|
|
633
|
+
table["cell_ids"] = []
|
|
634
|
+
for cell_order, cell in enumerate(cells):
|
|
635
|
+
cell["id"] = f"{table['id']}_r{cell['row']:03d}_c{cell['col']:03d}"
|
|
636
|
+
cell["table_id"] = table["id"]
|
|
637
|
+
cell["reading_order"] = cell_order
|
|
638
|
+
cell["line_ids"] = []
|
|
639
|
+
table["cell_ids"].append(cell["id"])
|
|
640
|
+
if layout.table_border_width:
|
|
641
|
+
box = cell["box"]
|
|
642
|
+
left, top = box["x"], box["y"]
|
|
643
|
+
right, bottom = left+box["w"], top+box["h"]
|
|
644
|
+
thickness = layout.table_border_width
|
|
645
|
+
canvas[top:top+thickness, left:right] = 0
|
|
646
|
+
canvas[bottom-thickness:bottom, left:right] = 0
|
|
647
|
+
canvas[top:bottom, left:left+thickness] = 0
|
|
648
|
+
canvas[top:bottom, right-thickness:right] = 0
|
|
649
|
+
tables.append(table)
|
|
650
|
+
|
|
651
|
+
for column in page:
|
|
652
|
+
column_orders: list[int] = []
|
|
653
|
+
for placement in column:
|
|
654
|
+
x, y, w, h = placement.x, placement.y, placement.w, placement.h
|
|
655
|
+
canvas[y : y + h, x : x + w] = placement.image
|
|
656
|
+
|
|
657
|
+
record: dict[str, Any] = {
|
|
658
|
+
"reading_order": order,
|
|
659
|
+
"text": placement.line.text,
|
|
660
|
+
"box": {"x": x, "y": y, "w": w, "h": h},
|
|
661
|
+
"font": placement.line.font,
|
|
662
|
+
"font_size": placement.line.font_size,
|
|
663
|
+
}
|
|
664
|
+
if layout.include_words or layout.writing_mode != "horizontal-tb" or layout.enable_tables:
|
|
665
|
+
record["id"] = f"{page_id}_line_{order:03d}"
|
|
666
|
+
record["writing_mode"] = layout.writing_mode
|
|
667
|
+
if layout.include_words:
|
|
668
|
+
words = []
|
|
669
|
+
source_h, source_w = placement.line.image.shape[:2]
|
|
670
|
+
for index, source_word in enumerate(placement.line.words):
|
|
671
|
+
word = dict(source_word)
|
|
672
|
+
word["polygon"] = [[x+px*w/source_w, y+py*h/source_h]
|
|
673
|
+
for px, py in source_word["polygon"]]
|
|
674
|
+
word["box"] = _box(word["polygon"])
|
|
675
|
+
word["id"] = f"{record['id']}_word_{index:03d}"
|
|
676
|
+
word["line_id"] = record["id"]
|
|
677
|
+
word["reading_order"] = word_order
|
|
678
|
+
word_order += 1
|
|
679
|
+
words.append(word)
|
|
680
|
+
record["words"] = words
|
|
681
|
+
if placement.table is not None and placement.cell is not None:
|
|
682
|
+
record["table_id"] = placement.table["id"]
|
|
683
|
+
record["cell_id"] = placement.cell["id"]
|
|
684
|
+
placement.table["line_orders"].append(order)
|
|
685
|
+
placement.cell["line_orders"].append(order)
|
|
686
|
+
placement.cell["line_ids"].append(record["id"])
|
|
687
|
+
if layout.include_baseline:
|
|
688
|
+
mask = _ink_mask(placement.image)
|
|
689
|
+
baseline = _baseline_offset(mask)
|
|
690
|
+
if layout.writing_mode != "horizontal-tb":
|
|
691
|
+
record["baseline"] = {"kind": "vertical_canvas_centerline", "points":
|
|
692
|
+
[[x+w/2, y+0.5], [x+w/2, y+h-0.5]]}
|
|
693
|
+
elif layout.transform_enabled or layout.include_words or layout.enable_tables:
|
|
694
|
+
if baseline is not None:
|
|
695
|
+
record["baseline"] = {
|
|
696
|
+
"kind": "ink_projection_peak",
|
|
697
|
+
"points": [
|
|
698
|
+
[x + 0.5, y + baseline + 0.5],
|
|
699
|
+
[x + w - 0.5, y + baseline + 0.5],
|
|
700
|
+
],
|
|
701
|
+
}
|
|
702
|
+
else:
|
|
703
|
+
record["baseline"] = y + (baseline if baseline is not None else h // 2)
|
|
704
|
+
if layout.include_polygon:
|
|
705
|
+
mask = _ink_mask(placement.image)
|
|
706
|
+
polygon = _polygon_from_mask(mask)
|
|
707
|
+
if polygon is not None:
|
|
708
|
+
if layout.transform_enabled or layout.include_words or layout.enable_tables or layout.writing_mode != "horizontal-tb":
|
|
709
|
+
# Pixel cell edges for v3-v5; v2 retains last-pixel indices.
|
|
710
|
+
polygon[1][0] += 1
|
|
711
|
+
polygon[2][0] += 1
|
|
712
|
+
polygon[2][1] += 1
|
|
713
|
+
polygon[3][1] += 1
|
|
714
|
+
record["polygon"] = [[x + px, y + py] for px, py in polygon]
|
|
715
|
+
if placement.line.source_id is not None:
|
|
716
|
+
record["source_id"] = placement.line.source_id
|
|
717
|
+
if placement.line.source_group is not None:
|
|
718
|
+
record["source_group"] = placement.line.source_group
|
|
719
|
+
if placement.line.language is not None:
|
|
720
|
+
record["language"] = placement.line.language
|
|
721
|
+
line_records.append(record)
|
|
722
|
+
|
|
723
|
+
span_records.append(
|
|
724
|
+
{
|
|
725
|
+
"reading_order": order,
|
|
726
|
+
"text": placement.line.text,
|
|
727
|
+
"box": {"x": x, "y": y, "w": w, "h": h},
|
|
728
|
+
}
|
|
729
|
+
)
|
|
730
|
+
column_orders.append(order)
|
|
731
|
+
order += 1
|
|
732
|
+
block_line_orders.append(column_orders)
|
|
733
|
+
|
|
734
|
+
transform: Any = None
|
|
735
|
+
if layout.v4_enabled:
|
|
736
|
+
transform = layout.plan_transform(page_id)
|
|
737
|
+
transform.annotations(line_records, span_records, tables)
|
|
738
|
+
canvas = transform.raster(canvas)
|
|
739
|
+
canvas = apply_photometric(
|
|
740
|
+
canvas,
|
|
741
|
+
contrast=layout.page_contrast,
|
|
742
|
+
noise=layout.page_noise,
|
|
743
|
+
seed=layout.seed,
|
|
744
|
+
page_id=page_id,
|
|
745
|
+
)
|
|
746
|
+
elif layout.transform_enabled:
|
|
747
|
+
scale, rot = layout.resolve_transform_params(page_id)
|
|
748
|
+
transform = plan_page_transform(
|
|
749
|
+
layout.page_width,
|
|
750
|
+
layout.page_height,
|
|
751
|
+
scale,
|
|
752
|
+
rot,
|
|
753
|
+
layout.page_crop,
|
|
754
|
+
flip=layout.page_flip,
|
|
755
|
+
)
|
|
756
|
+
# Reject unsafe geometry and raster crops before writing either artifact.
|
|
757
|
+
transform.annotations(line_records, span_records, tables)
|
|
758
|
+
canvas = transform.raster(canvas)
|
|
759
|
+
|
|
760
|
+
image_rel = f"pages/{page_id}.{layout.output_format}"
|
|
761
|
+
write_ok = write_output_image(
|
|
762
|
+
pages_dir / f"{page_id}.{layout.output_format}",
|
|
763
|
+
canvas,
|
|
764
|
+
fmt=layout.output_format,
|
|
765
|
+
)
|
|
766
|
+
if not write_ok:
|
|
767
|
+
raise RuntimeError(
|
|
768
|
+
f"failed to write page image: {pages_dir / f'{page_id}.{layout.output_format}'}"
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
record: dict[str, Any] = {
|
|
772
|
+
"schema_version": (
|
|
773
|
+
4
|
|
774
|
+
if layout.v4_enabled
|
|
775
|
+
else 3
|
|
776
|
+
if transform
|
|
777
|
+
else (2 if layout.geometry_extended else SCHEMA_VERSION)
|
|
778
|
+
),
|
|
779
|
+
"page_id": page_id,
|
|
780
|
+
"image": image_rel,
|
|
781
|
+
"width": canvas.shape[1],
|
|
782
|
+
"height": canvas.shape[0],
|
|
783
|
+
"document_id": document_id,
|
|
784
|
+
"lines": line_records,
|
|
785
|
+
}
|
|
786
|
+
if layout.include_words or layout.writing_mode != "horizontal-tb" or layout.enable_tables:
|
|
787
|
+
record["schema_version"] = 5
|
|
788
|
+
record["writing_mode"] = layout.writing_mode
|
|
789
|
+
if tables:
|
|
790
|
+
record["tables"] = tables
|
|
791
|
+
if transform:
|
|
792
|
+
record["transform"] = transform.provenance()
|
|
793
|
+
if layout.v4_enabled and (layout.page_contrast != 1.0 or layout.page_noise):
|
|
794
|
+
record["degradation"] = degradation_provenance(
|
|
795
|
+
contrast=layout.page_contrast,
|
|
796
|
+
noise=layout.page_noise,
|
|
797
|
+
seed=layout.seed,
|
|
798
|
+
page_id=page_id,
|
|
799
|
+
)
|
|
800
|
+
if layout.hierarchy:
|
|
801
|
+
record["blocks"] = [
|
|
802
|
+
{"id": f"block_{b:03d}", "line_orders": orders}
|
|
803
|
+
for b, orders in enumerate(block_line_orders)
|
|
804
|
+
]
|
|
805
|
+
if tables:
|
|
806
|
+
record["blocks"] = [{"id": f"{table['id']}_block", "kind": "table",
|
|
807
|
+
"table_id": table["id"], "cell_ids": table["cell_ids"],
|
|
808
|
+
"line_orders": table["line_orders"]} for table in tables]
|
|
809
|
+
record["spans"] = span_records
|
|
810
|
+
|
|
811
|
+
annotations.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def _check_output_dir(output_dir: Path) -> None:
|
|
815
|
+
"""Refuse existing content and symlinks, including dangling symlinks."""
|
|
816
|
+
if output_dir.is_symlink() or (
|
|
817
|
+
output_dir.exists() and (not output_dir.is_dir() or any(output_dir.iterdir()))
|
|
818
|
+
):
|
|
819
|
+
raise FileExistsError(
|
|
820
|
+
f"Output directory must be absent or empty (refusing to overwrite): {output_dir}"
|
|
821
|
+
)
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
@dataclass(frozen=True)
|
|
825
|
+
class _DirectoryAccess:
|
|
826
|
+
mode: int
|
|
827
|
+
uid: int
|
|
828
|
+
gid: int
|
|
829
|
+
xattrs: dict[str, bytes]
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def _output_metadata(path: Path) -> tuple[os.stat_result, _DirectoryAccess] | None:
|
|
833
|
+
try:
|
|
834
|
+
info = path.lstat()
|
|
835
|
+
except FileNotFoundError:
|
|
836
|
+
return None
|
|
837
|
+
if not stat.S_ISDIR(info.st_mode):
|
|
838
|
+
raise FileExistsError(f"Output directory must be absent or empty: {path}")
|
|
839
|
+
# Linux exposes POSIX ACLs as xattrs. Other platforms need their own ACL
|
|
840
|
+
# implementation before replacing an existing directory can be supported.
|
|
841
|
+
if sys.platform != "linux":
|
|
842
|
+
raise OSError(errno.ENOTSUP, "Preserving existing output access metadata requires Linux", path)
|
|
843
|
+
attrs = {name: os.getxattr(path, name) for name in os.listxattr(path)}
|
|
844
|
+
return info, _DirectoryAccess(stat.S_IMODE(info.st_mode), info.st_uid, info.st_gid, attrs)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def _copy_output_access(path: Path, access: _DirectoryAccess) -> None:
|
|
848
|
+
"""Copy and verify access before children inherit group and default ACLs."""
|
|
849
|
+
info = path.stat()
|
|
850
|
+
if (info.st_uid, info.st_gid) != (access.uid, access.gid):
|
|
851
|
+
os.chown(path, access.uid, access.gid)
|
|
852
|
+
path.chmod(access.mode)
|
|
853
|
+
for name in os.listxattr(path):
|
|
854
|
+
if name not in access.xattrs:
|
|
855
|
+
os.removexattr(path, name)
|
|
856
|
+
for name, value in access.xattrs.items():
|
|
857
|
+
if name not in os.listxattr(path) or os.getxattr(path, name) != value:
|
|
858
|
+
os.setxattr(path, name, value)
|
|
859
|
+
copied = _output_metadata(path)
|
|
860
|
+
if copied is None or copied[1] != access:
|
|
861
|
+
raise PermissionError(f"Cannot preserve output directory access metadata: {path}")
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def generate_layout(
|
|
865
|
+
corpus_path: str | Path,
|
|
866
|
+
output_dir: str | Path,
|
|
867
|
+
gen_cfg: GenerationConfig,
|
|
868
|
+
layout: LayoutConfig,
|
|
869
|
+
*,
|
|
870
|
+
min_length: int = 1,
|
|
871
|
+
max_length: int = 260,
|
|
872
|
+
max_lines: int = 0,
|
|
873
|
+
retry_limit: int = 10,
|
|
874
|
+
verbose: bool = True,
|
|
875
|
+
) -> LayoutResult:
|
|
876
|
+
"""Render a corpus and compose pages (single- or multi-column) with annotations.
|
|
877
|
+
|
|
878
|
+
Determinism: the module-level ``random`` is seeded from ``layout.seed``
|
|
879
|
+
before any font selection, so an identical corpus/config/seed reproduces
|
|
880
|
+
identical page images and annotations.
|
|
881
|
+
|
|
882
|
+
Export into a temporary sibling directory, then publish with one directory
|
|
883
|
+
rename after all writes close. Exceptions and zero-page results leave the
|
|
884
|
+
destination untouched; zero-page results still return rejection statistics.
|
|
885
|
+
The destination must be absent or an empty directory, never a symlink.
|
|
886
|
+
"""
|
|
887
|
+
corpus_path = Path(corpus_path)
|
|
888
|
+
requested_output = Path(output_dir)
|
|
889
|
+
# Collapse '.' and '..' before deriving siblings, without following the
|
|
890
|
+
# final component (a destination symlink must still be refused).
|
|
891
|
+
output_dir = Path(os.path.abspath(requested_output))
|
|
892
|
+
_check_output_dir(output_dir)
|
|
893
|
+
original_metadata = _output_metadata(output_dir)
|
|
894
|
+
|
|
895
|
+
random.seed(layout.seed)
|
|
896
|
+
|
|
897
|
+
items, corpus_stats = load_corpus_with_stats(
|
|
898
|
+
corpus_path,
|
|
899
|
+
min_length=min_length,
|
|
900
|
+
max_length=max_length,
|
|
901
|
+
max_lines=max_lines,
|
|
902
|
+
normalizer=gen_cfg.normalizer,
|
|
903
|
+
)
|
|
904
|
+
if verbose:
|
|
905
|
+
print(f" Layout corpus: {corpus_path} -> {len(items)} passing line(s)")
|
|
906
|
+
print(
|
|
907
|
+
" Filtered: "
|
|
908
|
+
f"too_short={corpus_stats['too_short']} too_long={corpus_stats['too_long']} "
|
|
909
|
+
f"malformed={corpus_stats['malformed']} separator={corpus_stats['separator']} "
|
|
910
|
+
f"blank={corpus_stats['blank']}"
|
|
911
|
+
)
|
|
912
|
+
|
|
913
|
+
# Group by document, deterministic order.
|
|
914
|
+
groups: dict[str, list[str | Any]] = {}
|
|
915
|
+
for item in items:
|
|
916
|
+
groups.setdefault(_document_key(item), []).append(item)
|
|
917
|
+
|
|
918
|
+
font_manager = FontManager(
|
|
919
|
+
language=gen_cfg.language,
|
|
920
|
+
fonts_dir=gen_cfg.fonts_dir,
|
|
921
|
+
verbose=False,
|
|
922
|
+
multilingual=gen_cfg.multilingual,
|
|
923
|
+
font_pool=gen_cfg.font_pool,
|
|
924
|
+
)
|
|
925
|
+
renderer = ImageRenderer(font_manager, gen_cfg)
|
|
926
|
+
|
|
927
|
+
result = LayoutResult(
|
|
928
|
+
output_dir=requested_output,
|
|
929
|
+
documents=len(groups),
|
|
930
|
+
rejection_reasons={},
|
|
931
|
+
corpus_stats=dict(corpus_stats),
|
|
932
|
+
)
|
|
933
|
+
reasons: Counter[str] = Counter()
|
|
934
|
+
page_index = 0
|
|
935
|
+
|
|
936
|
+
output_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
937
|
+
with TemporaryDirectory(prefix=f".{output_dir.name}.staging-", dir=output_dir.parent) as temp:
|
|
938
|
+
# The private container stays 0700. Its export child uses normal mkdir
|
|
939
|
+
# umask/default ACL inheritance, and is the directory we publish.
|
|
940
|
+
staging_dir = Path(temp) / "export"
|
|
941
|
+
staging_dir.mkdir()
|
|
942
|
+
if original_metadata is not None:
|
|
943
|
+
_copy_output_access(staging_dir, original_metadata[1])
|
|
944
|
+
pages_dir = staging_dir / "pages"
|
|
945
|
+
pages_dir.mkdir()
|
|
946
|
+
with (staging_dir / "annotations.jsonl").open("w", encoding="utf-8") as annotations:
|
|
947
|
+
for document_id in sorted(groups):
|
|
948
|
+
doc_items = groups[document_id]
|
|
949
|
+
rendered, render_reasons = _render_document(
|
|
950
|
+
doc_items,
|
|
951
|
+
renderer,
|
|
952
|
+
multilingual=gen_cfg.multilingual,
|
|
953
|
+
retry_limit=retry_limit,
|
|
954
|
+
layout=layout,
|
|
955
|
+
)
|
|
956
|
+
reasons.update(render_reasons)
|
|
957
|
+
|
|
958
|
+
pages, paginate_reasons = _paginate(rendered, layout)
|
|
959
|
+
reasons.update(paginate_reasons)
|
|
960
|
+
|
|
961
|
+
if len(pages) > layout.max_pages_per_doc:
|
|
962
|
+
result.rejected_documents += 1
|
|
963
|
+
reasons["document_too_many_pages"] += 1
|
|
964
|
+
_log.warning(
|
|
965
|
+
"Skipping document %r: %d pages exceeds --max-pages-per-doc=%d",
|
|
966
|
+
document_id,
|
|
967
|
+
len(pages),
|
|
968
|
+
layout.max_pages_per_doc,
|
|
969
|
+
)
|
|
970
|
+
continue
|
|
971
|
+
|
|
972
|
+
for page in pages:
|
|
973
|
+
_write_page(
|
|
974
|
+
page,
|
|
975
|
+
page_index=page_index,
|
|
976
|
+
document_id=document_id,
|
|
977
|
+
layout=layout,
|
|
978
|
+
pages_dir=pages_dir,
|
|
979
|
+
annotations=annotations,
|
|
980
|
+
color_mode=int(gen_cfg.color_mode),
|
|
981
|
+
)
|
|
982
|
+
page_index += 1
|
|
983
|
+
|
|
984
|
+
result.pages_written += len(pages)
|
|
985
|
+
result.lines_written += sum(len(column) for page in pages for column in page)
|
|
986
|
+
|
|
987
|
+
result.rejected_lines = len(items) - result.lines_written
|
|
988
|
+
result.rejection_reasons = dict(sorted(reasons.items()))
|
|
989
|
+
|
|
990
|
+
if verbose:
|
|
991
|
+
print(
|
|
992
|
+
f" Layout: {result.documents} document(s), {result.pages_written} page(s), "
|
|
993
|
+
f"{result.lines_written} line(s) placed, {result.rejected_lines} line(s) rejected, "
|
|
994
|
+
f"{result.rejected_documents} document(s) rejected"
|
|
995
|
+
)
|
|
996
|
+
if result.rejection_reasons:
|
|
997
|
+
print(f" Rejection reasons: {result.rejection_reasons}")
|
|
998
|
+
|
|
999
|
+
if result.pages_written:
|
|
1000
|
+
_check_output_dir(output_dir)
|
|
1001
|
+
current_metadata = _output_metadata(output_dir)
|
|
1002
|
+
if original_metadata is None:
|
|
1003
|
+
changed = current_metadata is not None
|
|
1004
|
+
else:
|
|
1005
|
+
changed = current_metadata is None or (
|
|
1006
|
+
current_metadata[0].st_dev,
|
|
1007
|
+
current_metadata[0].st_ino,
|
|
1008
|
+
current_metadata[1],
|
|
1009
|
+
) != (
|
|
1010
|
+
original_metadata[0].st_dev,
|
|
1011
|
+
original_metadata[0].st_ino,
|
|
1012
|
+
original_metadata[1],
|
|
1013
|
+
)
|
|
1014
|
+
if changed:
|
|
1015
|
+
raise FileExistsError(f"Output directory changed during export: {output_dir}")
|
|
1016
|
+
try:
|
|
1017
|
+
# A directory rename atomically publishes the complete export.
|
|
1018
|
+
# It may replace an empty directory, but cannot replace content
|
|
1019
|
+
# another writer creates between the check and this rename.
|
|
1020
|
+
staging_dir.rename(output_dir)
|
|
1021
|
+
except OSError as exc:
|
|
1022
|
+
if exc.errno in {errno.EEXIST, errno.ENOTEMPTY}:
|
|
1023
|
+
raise FileExistsError(
|
|
1024
|
+
f"Output directory is not empty (refusing to overwrite): {output_dir}"
|
|
1025
|
+
) from exc
|
|
1026
|
+
raise
|
|
1027
|
+
|
|
1028
|
+
return result
|