tidyscan 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tidyscan/__init__.py ADDED
@@ -0,0 +1,156 @@
1
+ """Auto-rotate and deskew scanned pages into a lossless PDF, with an optional OCR text layer.
2
+
3
+ turn, angle = tidyscan.straighten(page, 300)
4
+ data = tidyscan.pdf.build([page], 300, [turn], [angle])
5
+ data = tidyscan.ocr.add_text_layer(data, "eng+nld") # Tesseract; engine="paddle" for receipts
6
+
7
+ A page is JPEG bytes, binary PBM (P4) bytes, or an object with to_pbm().
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import logging
13
+
14
+ from . import _budget, hocr, models, ocr, orientation, paddle, pdf, skew
15
+ from ._page import Page, PbmSource, encoded
16
+ from .ocr import OcrError
17
+ from .pdf import UnsupportedPage
18
+
19
+ __version__ = "0.3.0"
20
+ __all__ = [
21
+ "OcrError",
22
+ "Page",
23
+ "PbmSource",
24
+ "UnsupportedPage",
25
+ "hocr",
26
+ "models",
27
+ "ocr",
28
+ "orientation",
29
+ "paddle",
30
+ "pdf",
31
+ "skew",
32
+ "straighten",
33
+ ]
34
+
35
+ log = logging.getLogger(__name__)
36
+
37
+ STEEP_EDGE = 2.0 # degrees from +-45: a steep-path answer there is as likely the other quarter turn
38
+ BUDGET = 300.0 # seconds of Tesseract per page at most (see _budget)
39
+ MAX_PIXELS = 200_000_000 # above Pillow's own refusal (2 x 89.5 M pixels); an A4 page at 1200 dpi is 140 M
40
+
41
+
42
+ def straighten(
43
+ page: Page,
44
+ dpi: int,
45
+ *,
46
+ turn: int = 0,
47
+ rotate: bool = True,
48
+ deskew: bool = True,
49
+ min_confidence: float = 3.0,
50
+ scripts: tuple[str, ...] = ("Latin",),
51
+ deskew_min: float = 0.3,
52
+ workdir: str | None = None,
53
+ budget: float | None = BUDGET,
54
+ ) -> tuple[int, float]:
55
+ """(clockwise turn 0/90/180/270, skew in degrees counter-clockwise) for pdf.build; 0 for what is off,
56
+ unsure or unavailable.
57
+
58
+ turn: a turn already known (from the scanner); it is kept and not detected. deskew_min: smaller
59
+ skews are not visible and left alone. budget: seconds of Tesseract for this page at most (None: no
60
+ limit); what is not measured by then is kept as scanned. A page larger than MAX_PIXELS is not read
61
+ at all. Never raises: straightening is a nicety and must not cost a page (a JPEG that Pillow
62
+ cannot decode is still a page the PDF can carry)."""
63
+ if _too_large(page):
64
+ return turn, 0.0
65
+ token = _budget.start(budget)
66
+ try:
67
+ return _straighten(page, dpi, turn, rotate, deskew, min_confidence, scripts, deskew_min, workdir)
68
+ finally:
69
+ _budget.end(token)
70
+
71
+
72
+ def _too_large(page: Page) -> bool:
73
+ try:
74
+ data, ext = encoded(page)
75
+ w, h = pdf.pbm_info(data)[:2] if ext == "pbm" else pdf.jpeg_info(data)[:2]
76
+ except Exception: # noqa: BLE001 (the PDF writer reports a damaged page)
77
+ return False
78
+ if w * h > MAX_PIXELS:
79
+ log.warning("page of %d x %d pixels: too large to read; kept as scanned", w, h)
80
+ return True
81
+ return False
82
+
83
+
84
+ def _straighten(
85
+ page: Page,
86
+ dpi: int,
87
+ turn: int,
88
+ rotate: bool,
89
+ deskew: bool,
90
+ min_confidence: float,
91
+ scripts: tuple[str, ...],
92
+ deskew_min: float,
93
+ workdir: str | None,
94
+ ) -> tuple[int, float]:
95
+ # Skew first, then the orientation of the straightened page: OSD cannot read a page tilted by
96
+ # more than a few degrees. A 180-degree turn leaves the text lines' angle as it is; a quarter
97
+ # turn means the lines were measured across the text, so the skew is measured again upright.
98
+ known = bool(turn)
99
+ found = _detect(page, dpi, turn, workdir) if deskew else None
100
+ base = 0
101
+ if found is None and deskew and rotate and not known:
102
+ # A page laid at more than 45 degrees (a receipt at about 60, upside down, 01-11-2026): the skew
103
+ # search covers +-45, so its text rows only show a quarter turn on. Measured there, the skew is
104
+ # an ordinary one, and OSD then only has to tell 90 from 270.
105
+ steep = _detect(page, dpi, 90, workdir)
106
+ if steep is not None and abs(abs(steep) - 45) < STEEP_EDGE:
107
+ # a chart with slanted axis labels came out a quarter plus about 45 degrees (benchmark 05-11-2026)
108
+ log.info("skew unsure (%.2f degrees a quarter turn on: too near 45); page kept as scanned", steep)
109
+ elif steep is not None:
110
+ base, found = 90, steep
111
+ log.info("page lies at more than 45 degrees; measured a quarter turn on")
112
+ angle = found if found is not None and abs(found) >= deskew_min else 0.0
113
+ if not known and rotate:
114
+ try:
115
+ more, sure = orientation.detect(page, workdir, min_confidence, scripts, skew=angle, quarter_turn=base)
116
+ except Exception as e: # noqa: BLE001
117
+ log.warning("orientation not determined (%s); page kept as scanned", e)
118
+ more, sure = 0, False
119
+ if base and not sure:
120
+ # The steep path's quarter turn is only half an answer: 90 and 270 level the same lines, and
121
+ # only OSD on the turned page tells them apart. Unsure, it was kept, and every wrong turn in the
122
+ # benchmark (10, 05-11-2026) came from here. Rotations commute, so the angle measured in the
123
+ # turned frame straightens the page as scanned too: sideways, but level. First the words
124
+ # decide, as for an unsure upside-down page.
125
+ try:
126
+ by_words = orientation.quarter_by_words(page, workdir, skew=angle)
127
+ except Exception as e: # noqa: BLE001
128
+ log.warning("words not read (%s)", e)
129
+ by_words = None
130
+ if by_words:
131
+ base, more = by_words, 0
132
+ else:
133
+ log.info("quarter turn not confirmed by the orientation check; page kept sideways")
134
+ base = 0
135
+ turn = (base + more) % 360
136
+ if more in (90, 270):
137
+ angle = _skew(page, dpi, turn, deskew, deskew_min, workdir)
138
+ if angle:
139
+ log.info("page straightened by %.2f degrees", angle)
140
+ return turn, angle
141
+
142
+
143
+ def _detect(page: Page, dpi: int, turn: int, workdir: str | None) -> float | None:
144
+ """skew.detect, None when unsure or failing."""
145
+ try:
146
+ return skew.detect(page, dpi, turn, workdir)
147
+ except Exception as e: # noqa: BLE001
148
+ log.warning("skew not measured (%s); page kept as scanned", e)
149
+ return None
150
+
151
+
152
+ def _skew(page: Page, dpi: int, turn: int, deskew: bool, deskew_min: float, workdir: str | None) -> float:
153
+ if not deskew:
154
+ return 0.0
155
+ found = _detect(page, dpi, turn, workdir)
156
+ return found if found is not None and abs(found) >= deskew_min else 0.0
tidyscan/__main__.py ADDED
@@ -0,0 +1,227 @@
1
+ """tidyscan [options] -o OUT.pdf PAGE...: page images into one straightened PDF.
2
+
3
+ Also: `tidyscan watch IN OUT` (document folders as scankeyd writes them into PDFs, as a
4
+ service), `tidyscan models fetch [DIR]` (download and check PaddleOCR's models, once) and
5
+ `tidyscan models path` (where they are looked up)."""
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import io
11
+ import logging
12
+ import os
13
+ import sys
14
+
15
+ from . import __version__, models, ocr, orientation, pdf, skew, straighten
16
+
17
+
18
+ def _load(path: str) -> list[bytes]:
19
+ """The pages of one input file: a JPEG or PBM as it is; anything Pillow reads converted, every
20
+ frame of a multi-page file (TIFF) a page of its own."""
21
+ with open(path, "rb") as f:
22
+ data = f.read()
23
+ if data.startswith((b"\xff\xd8", b"P4")):
24
+ return [data]
25
+ try:
26
+ from PIL import Image, ImageSequence
27
+ except ImportError:
28
+ raise SystemExit(f"{path}: not JPEG or PBM (other formats need Pillow)") from None
29
+ pages = []
30
+ for img in ImageSequence.Iterator(Image.open(io.BytesIO(data))):
31
+ out = io.BytesIO()
32
+ if img.mode == "1":
33
+ img.save(out, "PPM") # mode 1 saves as binary PBM
34
+ elif img.mode.startswith("I"): # 16-bit gray scaled to 8 bits (32-bit values above 65535 clip)
35
+ img.convert("I").point(lambda v: v / 256).convert("L").save(out, "JPEG", quality=95)
36
+ else:
37
+ img.convert("L" if img.mode in ("L", "LA") else "RGB").save(out, "JPEG", quality=95)
38
+ pages.append(out.getvalue())
39
+ return pages
40
+
41
+
42
+ def _dpi(text: str) -> int:
43
+ value = int(text)
44
+ if not 50 <= value <= 2400:
45
+ raise argparse.ArgumentTypeError(f"{value}: 50-2400")
46
+ return value
47
+
48
+
49
+ def models_main(argv: list[str]) -> int:
50
+ p = argparse.ArgumentParser(prog="tidyscan models", description="PaddleOCR's PP-OCRv5 models")
51
+ sub = p.add_subparsers(dest="cmd", required=True)
52
+ f = sub.add_parser("fetch", help=f"download the models from release {models.TAG} and check their sha256")
53
+ f.add_argument("directory", nargs="?", help="where to put them (default: the per-user model directory)")
54
+ sub.add_parser("path", help="where the models are looked up, and whether they are there")
55
+ a = p.parse_args(argv)
56
+ if a.cmd == "path":
57
+ print(models.describe())
58
+ return 0
59
+ try:
60
+ written = models.fetch(a.directory)
61
+ except models.ModelError as e:
62
+ print(f"tidyscan: {e}", file=sys.stderr)
63
+ return 1
64
+ for path in written:
65
+ print(f"written: {path}")
66
+ print("models present" if not written else f"{len(written)} file(s) fetched and checked")
67
+ return 0
68
+
69
+
70
+ def watch_main(argv: list[str]) -> int:
71
+ import signal
72
+ import threading
73
+
74
+ from . import watch
75
+
76
+ p = argparse.ArgumentParser(
77
+ prog="tidyscan watch",
78
+ description="Document folders (IN/<function>/<name>/ with document.json, as scankeyd writes them) "
79
+ "into PDFs in OUT, straightened and with a text layer where asked; each folder goes once its PDF "
80
+ "is written. Turning and straightening need Tesseract, the text layer OCRmyPDF; without them "
81
+ "it only makes the PDFs.",
82
+ )
83
+ p.add_argument("input", metavar="IN")
84
+ p.add_argument("output", metavar="OUT")
85
+ env = os.environ # the defaults, from the settings file all the scan containers share (env_file)
86
+
87
+ def on(name: str, default: str = "auto", auto: bool = True) -> bool:
88
+ """An on/off setting: on, off (also true/false, yes/no, 1/0), and auto where allowed (on when
89
+ the tool for it is there)."""
90
+ value = env.get(name, default).strip().lower()
91
+ if value in ("on", "true", "yes", "1") or (auto and value == "auto"):
92
+ return True
93
+ if value in ("off", "false", "no", "0"):
94
+ return False
95
+ raise SystemExit(f"tidyscan: {name}={value!r}: on or off" + (" or auto" if auto else ""))
96
+
97
+ p.add_argument(
98
+ "--ocr",
99
+ metavar="LANGS",
100
+ default=env.get("OCR_LANGUAGES", "eng"),
101
+ help="text layer languages, e.g. eng+nld (default: $OCR_LANGUAGES, else eng)",
102
+ )
103
+ p.add_argument(
104
+ "--ocr-folders",
105
+ default=env.get("OCR_FOLDERS", "ocr"),
106
+ help="for documents without an ocr wish in their document.json: comma list of function folders "
107
+ "that get a text layer (default: $OCR_FOLDERS, else ocr; 'all' for every one, '' for none)",
108
+ )
109
+ p.add_argument(
110
+ "--engine",
111
+ choices=ocr.ENGINES,
112
+ default=env.get("OCR_ENGINE", "tesseract").lower(),
113
+ help="OCR engine (default: $OCR_ENGINE, else tesseract)",
114
+ )
115
+ p.add_argument("--no-rotate", action="store_true", help="do not turn pages upright")
116
+ p.add_argument("--no-deskew", action="store_true", help="do not straighten skewed pages")
117
+ p.add_argument("--min-confidence", type=float, default=3.0, help="orientation confidence to turn (default 3)")
118
+ p.add_argument(
119
+ "--pdfa",
120
+ action="store_true",
121
+ default=on("PDFA", "off", auto=False),
122
+ help="PDF/A-2 for the documents that get a text layer (default: $PDFA, else off)",
123
+ )
124
+ p.add_argument("--keep-folders", action="store_true", help="OUT/<function>/<name>.pdf instead of OUT/<name>.pdf")
125
+ p.add_argument("--interval", type=float, default=5.0, help="seconds between looks at IN (default 5)")
126
+ p.add_argument("--once", action="store_true", help="convert what is there now, then stop")
127
+ p.add_argument(
128
+ "--status",
129
+ metavar="FILE",
130
+ default=env.get("STATUS_FILE") or None,
131
+ help="a JSON status file for a monitor (last PDF, last error, failed and waiting documents); "
132
+ "default: $STATUS_FILE, else none",
133
+ )
134
+ p.add_argument("-q", "--quiet", action="store_true")
135
+ a = p.parse_args(argv)
136
+ if a.engine not in ocr.ENGINES: # argparse does not check a default against choices
137
+ p.error(f"OCR_ENGINE={a.engine!r}: one of {', '.join(ocr.ENGINES)}")
138
+ logging.basicConfig(level=logging.WARNING if a.quiet else logging.INFO, format="%(levelname)s %(message)s")
139
+ folders = tuple(f.strip().lower() for f in a.ocr_folders.split(",") if f.strip())
140
+ if folders == ("all",):
141
+ folders = ("*",) # every function folder
142
+ opts = watch.Options(
143
+ ocr_languages=a.ocr,
144
+ ocr_folders=folders,
145
+ engine=a.engine,
146
+ rotate=not a.no_rotate and on("ROTATE"),
147
+ deskew=not a.no_deskew and on("DESKEW"),
148
+ min_confidence=a.min_confidence,
149
+ keep_folders=a.keep_folders,
150
+ pdfa=a.pdfa,
151
+ )
152
+ for d in (a.input, a.output):
153
+ if not os.path.isdir(d):
154
+ print(f"tidyscan: {d} is not a directory", file=sys.stderr)
155
+ return 2
156
+ bad = [a.output] if not os.access(a.output, os.W_OK | os.X_OK) else []
157
+ bad += watch.unwritable(a.input) # IN too: a document's folder goes once its PDF is written
158
+ if bad:
159
+ print(f"tidyscan: not writable for this user: {', '.join(bad)}", file=sys.stderr)
160
+ return 2
161
+ status = watch.Status(a.status, a.input) if a.status else None
162
+ if a.once:
163
+ opts.rotate = opts.rotate and orientation.available()
164
+ opts.deskew = opts.deskew and skew.available()
165
+ watch.run_once(a.input, a.output, opts, status=status)
166
+ return 0
167
+ stop = threading.Event()
168
+ for sig in (signal.SIGTERM, signal.SIGINT):
169
+ signal.signal(sig, lambda *_: stop.set()) # finish the document at hand, then stop
170
+ watch.run(a.input, a.output, opts, a.interval, stop, status)
171
+ return 0
172
+
173
+
174
+ def main(argv: list[str] | None = None) -> int:
175
+ argv = sys.argv[1:] if argv is None else argv
176
+ if argv[:1] == ["models"]:
177
+ return models_main(argv[1:])
178
+ if argv[:1] == ["watch"]:
179
+ return watch_main(argv[1:])
180
+ p = argparse.ArgumentParser(prog="tidyscan", description=__doc__.split(":", 1)[1].split("\n")[0].strip())
181
+ p.add_argument("pages", nargs="+", metavar="PAGE", help="JPEG or PBM (other formats via Pillow, as JPEG)")
182
+ p.add_argument("-o", "--output", required=True, help="the PDF to write")
183
+ p.add_argument("--dpi", type=_dpi, default=300, help="scan resolution, 50-2400 (default 300)")
184
+ p.add_argument("--no-rotate", action="store_true", help="do not turn pages upright")
185
+ p.add_argument("--no-deskew", action="store_true", help="do not straighten skewed pages")
186
+ p.add_argument("--min-confidence", type=float, default=3.0, help="orientation confidence to turn (default 3)")
187
+ p.add_argument(
188
+ "--ocr",
189
+ metavar="LANGS",
190
+ help="add a text layer, e.g. eng+nld (needs ocrmypdf; the languages steer Tesseract only)",
191
+ )
192
+ p.add_argument(
193
+ "--engine",
194
+ choices=ocr.ENGINES,
195
+ default="tesseract",
196
+ help="OCR engine: tesseract (the default; letters and documents), paddle (PaddleOCR, if "
197
+ "installed; receipts), or auto (paddle when available, else tesseract)",
198
+ )
199
+ p.add_argument("-q", "--quiet", action="store_true")
200
+ p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
201
+ a = p.parse_args(argv)
202
+ logging.basicConfig(level=logging.WARNING if a.quiet else logging.INFO, format="%(message)s")
203
+ pages = [page for path in a.pages for page in _load(path)]
204
+ turns, angles = [], []
205
+ for page in pages:
206
+ turn, angle = straighten(
207
+ page, a.dpi, rotate=not a.no_rotate, deskew=not a.no_deskew, min_confidence=a.min_confidence
208
+ )
209
+ turns.append(turn)
210
+ angles.append(angle)
211
+ try:
212
+ data = pdf.build(pages, a.dpi, turns, angles)
213
+ except pdf.UnsupportedPage as e:
214
+ print(f"tidyscan: {e}", file=sys.stderr)
215
+ return 1
216
+ if a.ocr:
217
+ try:
218
+ data = ocr.add_text_layer(data, a.ocr, engine=a.engine)
219
+ except ocr.OcrError as e:
220
+ print(f"tidyscan: OCR failed ({e}); written without text", file=sys.stderr)
221
+ with open(a.output, "wb") as f:
222
+ f.write(data)
223
+ return 0
224
+
225
+
226
+ if __name__ == "__main__":
227
+ sys.exit(main())
tidyscan/_budget.py ADDED
@@ -0,0 +1,36 @@
1
+ """A time budget for one page: straighten() starts it, and every Tesseract call within gets at most
2
+ what is left of it (and never more than its own limit). Spent, a call ends at once as a timeout,
3
+ which every caller already takes as "unsure": the page is kept as scanned rather than taking many
4
+ minutes (a dense table turned a quarter took up to 130 s, 05-11-2026; the worst case is far longer)."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import contextvars
9
+ import logging
10
+ import subprocess
11
+ import time
12
+
13
+ log = logging.getLogger(__name__)
14
+
15
+ _END: contextvars.ContextVar[float | None] = contextvars.ContextVar("tidyscan_page_deadline", default=None)
16
+
17
+
18
+ def start(seconds: float | None) -> contextvars.Token[float | None]:
19
+ return _END.set(None if seconds is None else time.monotonic() + seconds)
20
+
21
+
22
+ def end(token: contextvars.Token[float | None]) -> None:
23
+ _END.reset(token)
24
+
25
+
26
+ def timeout(limit: float) -> float:
27
+ """The timeout for the next call: its own limit, or less when the page's budget is nearly spent.
28
+ Raises subprocess.TimeoutExpired when nothing is left."""
29
+ deadline = _END.get()
30
+ if deadline is None:
31
+ return limit
32
+ left = deadline - time.monotonic()
33
+ if left <= 0:
34
+ log.warning("the page's time budget is spent; what is not measured yet is kept as scanned")
35
+ raise subprocess.TimeoutExpired("tesseract", 0)
36
+ return min(limit, left)
tidyscan/_page.py ADDED
@@ -0,0 +1,28 @@
1
+ """A page as this package takes it: JPEG bytes, binary PBM (P4) bytes, or an object with to_pbm()."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import tempfile
6
+ from typing import Protocol, TypeAlias, cast
7
+
8
+
9
+ class PbmSource(Protocol):
10
+ """A page that encodes itself, e.g. a black-and-white scan held as rows."""
11
+
12
+ def to_pbm(self) -> bytes: ...
13
+
14
+
15
+ Page: TypeAlias = bytes | bytearray | memoryview | PbmSource
16
+ """JPEG bytes, binary PBM (P4) bytes, or an object with to_pbm()."""
17
+
18
+
19
+ def encoded(page: Page) -> tuple[bytes, str]:
20
+ """(data, file extension) of a page."""
21
+ if hasattr(page, "to_pbm"):
22
+ return cast("PbmSource", page).to_pbm(), "pbm"
23
+ data = bytes(page)
24
+ return data, "pbm" if data.startswith(b"P4") else "jpg"
25
+
26
+
27
+ def workdir(path: str | None) -> str:
28
+ return path or tempfile.gettempdir()
tidyscan/hocr.py ADDED
@@ -0,0 +1,146 @@
1
+ """hOCR from text boxes: what OCRmyPDF turns into the invisible text layer of a page.
2
+
3
+ An OCR engine that finds text boxes (PaddleOCR's detector finds one box per run of text, often
4
+ several per printed line) gives Line objects; page() groups the boxes into rows in reading order
5
+ and writes the hOCR that OCRmyPDF's renderer reads: ocr_page > ocr_carea > ocr_par > ocr_line >
6
+ ocrx_word, each with a pixel bbox. Standard library only.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from dataclasses import dataclass, field
13
+ from typing import TYPE_CHECKING
14
+ from xml.sax.saxutils import escape, quoteattr
15
+
16
+ if TYPE_CHECKING:
17
+ from collections.abc import Iterable, Sequence
18
+
19
+ Box = tuple[int, int, int, int] # x0, y0, x1, y1 in pixels, y down
20
+
21
+
22
+ @dataclass
23
+ class Word:
24
+ text: str
25
+ box: Box
26
+ conf: float = 1.0 # 0..1
27
+
28
+
29
+ @dataclass
30
+ class Line:
31
+ text: str
32
+ box: Box
33
+ conf: float = 1.0
34
+ words: list[Word] = field(default_factory=list)
35
+
36
+
37
+ # Characters XML 1.0 does not allow (OCRmyPDF parses the hOCR as XML).
38
+ _NOT_XML = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f￾￿]")
39
+
40
+
41
+ def clean(text: str) -> str:
42
+ return _NOT_XML.sub("", text)
43
+
44
+
45
+ def bbox(points: Sequence[Sequence[float]], width: int, height: int) -> Box:
46
+ """The axis-aligned box around points [(x, y), ...] (or an n x 2 array), clipped to the page."""
47
+ xs = [float(p[0]) for p in points]
48
+ ys = [float(p[1]) for p in points]
49
+ x0, y0 = max(0, int(min(xs))), max(0, int(min(ys)))
50
+ x1, y1 = min(width, int(round(max(xs)))), min(height, int(round(max(ys))))
51
+ return x0, y0, max(x0 + 1, x1), max(y0 + 1, y1)
52
+
53
+
54
+ def split_words(text: str, box: Box, conf: float = 1.0) -> list[Word]:
55
+ """Words of a line with boxes in proportion to their character positions in it."""
56
+ x0, y0, x1, y1 = box
57
+ per_char = (x1 - x0) / max(1, len(text))
58
+ return [
59
+ Word(
60
+ m.group(),
61
+ (
62
+ x0 + int(m.start() * per_char),
63
+ y0,
64
+ max(x0 + int(m.start() * per_char) + 1, x0 + int(round(m.end() * per_char))),
65
+ y1,
66
+ ),
67
+ conf,
68
+ )
69
+ for m in re.finditer(r"\S+", text)
70
+ ]
71
+
72
+
73
+ def rows(lines: list[Line]) -> list[list[Line]]:
74
+ """Lines grouped into rows (boxes that overlap vertically by half the smaller height), top to
75
+ bottom, each row left to right."""
76
+ out: list[list[Line]] = []
77
+ for line in sorted(lines, key=lambda ln: (ln.box[1] + ln.box[3]) / 2):
78
+ if out:
79
+ row = out[-1]
80
+ top, bottom = min(ln.box[1] for ln in row), max(ln.box[3] for ln in row)
81
+ overlap = min(bottom, line.box[3]) - max(top, line.box[1])
82
+ if overlap > 0.5 * min(bottom - top, line.box[3] - line.box[1]):
83
+ row.append(line)
84
+ continue
85
+ out.append([line])
86
+ return [sorted(row, key=lambda ln: ln.box[0]) for row in out]
87
+
88
+
89
+ def _union(boxes: Iterable[Box]) -> Box:
90
+ boxes = list(boxes)
91
+ return (min(b[0] for b in boxes), min(b[1] for b in boxes), max(b[2] for b in boxes), max(b[3] for b in boxes))
92
+
93
+
94
+ def _title(box: Box, extra: str = "") -> str:
95
+ return quoteattr(f"bbox {box[0]} {box[1]} {box[2]} {box[3]}{extra}")
96
+
97
+
98
+ def page(width: int, height: int, lines: list[Line], lang: str = "", system: str = "tidyscan") -> tuple[str, str]:
99
+ """(hOCR document, plain text) of one page image of width x height pixels.
100
+
101
+ A row of boxes becomes one ocr_line, so OCRmyPDF puts spaces between its words and the text
102
+ reads row by row. Lines and words without text are left out."""
103
+ kept = []
104
+ for line in lines:
105
+ words = [Word(clean(w.text).strip(), w.box, w.conf) for w in line.words] or split_words(
106
+ clean(line.text), line.box, line.conf
107
+ )
108
+ words = [w for w in words if w.text]
109
+ if words:
110
+ kept.append(Line(" ".join(w.text for w in words), line.box, line.conf, words))
111
+ out = [
112
+ '<?xml version="1.0" encoding="UTF-8"?>',
113
+ '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"'
114
+ ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">',
115
+ '<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">',
116
+ '<head><title></title><meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>',
117
+ f'<meta name="ocr-system" content={quoteattr(system)}/>',
118
+ '<meta name="ocr-capabilities" content="ocr_page ocr_carea ocr_par ocr_line ocrx_word"/></head>',
119
+ "<body>",
120
+ f'<div class="ocr_page" id="page_1" title={quoteattr(f"bbox 0 0 {width} {height}; ppageno 0")}>',
121
+ ]
122
+ text = []
123
+ lang_attr = f" lang={quoteattr(lang)}" if lang else ""
124
+ n_word = 0
125
+ for n, row in enumerate(rows(kept), 1):
126
+ box = _union(ln.box for ln in row)
127
+ h = box[3] - box[1]
128
+ # Baseline a fifth of the box height above its bottom: the detector's boxes have room for
129
+ # descenders. It only moves where a viewer draws the selection, not what is found.
130
+ out.append(f'<div class="ocr_carea" id="block_1_{n}" title={_title(box)}>')
131
+ out.append(f'<p class="ocr_par" id="par_1_{n}"{lang_attr} title={_title(box)}>')
132
+ out.append(
133
+ f'<span class="ocr_line" id="line_1_{n}" title={_title(box, f"; baseline 0 -{h // 5}; x_size {h}")}>'
134
+ )
135
+ for line in row:
136
+ for w in line.words:
137
+ n_word += 1
138
+ conf = max(0, min(100, round(w.conf * 100)))
139
+ out.append(
140
+ f'<span class="ocrx_word" id="word_1_{n_word}" title={_title(w.box, f"; x_wconf {conf}")}>'
141
+ f"{escape(w.text)}</span>"
142
+ )
143
+ out.append("</span></p></div>")
144
+ text.append(" ".join(ln.text for ln in row))
145
+ out += ["</div>", "</body>", "</html>", ""]
146
+ return "\n".join(out), "\n".join(text) + ("\n" if text else "")