tidyscan 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tidyscan/__init__.py +156 -0
- tidyscan/__main__.py +227 -0
- tidyscan/_budget.py +36 -0
- tidyscan/_page.py +28 -0
- tidyscan/hocr.py +146 -0
- tidyscan/models.py +170 -0
- tidyscan/ocr.py +140 -0
- tidyscan/ocrmypdf_paddle.py +122 -0
- tidyscan/orientation.py +225 -0
- tidyscan/paddle.py +193 -0
- tidyscan/pdf.py +179 -0
- tidyscan/py.typed +0 -0
- tidyscan/skew.py +264 -0
- tidyscan/watch.py +332 -0
- tidyscan-0.3.0.dist-info/METADATA +202 -0
- tidyscan-0.3.0.dist-info/RECORD +19 -0
- tidyscan-0.3.0.dist-info/WHEEL +4 -0
- tidyscan-0.3.0.dist-info/entry_points.txt +3 -0
- tidyscan-0.3.0.dist-info/licenses/LICENSE +28 -0
tidyscan/__init__.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""Auto-rotate and deskew scanned pages into a lossless PDF, with an optional OCR text layer.
|
|
2
|
+
|
|
3
|
+
turn, angle = tidyscan.straighten(page, 300)
|
|
4
|
+
data = tidyscan.pdf.build([page], 300, [turn], [angle])
|
|
5
|
+
data = tidyscan.ocr.add_text_layer(data, "eng+nld") # Tesseract; engine="paddle" for receipts
|
|
6
|
+
|
|
7
|
+
A page is JPEG bytes, binary PBM (P4) bytes, or an object with to_pbm().
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
|
|
14
|
+
from . import _budget, hocr, models, ocr, orientation, paddle, pdf, skew
|
|
15
|
+
from ._page import Page, PbmSource, encoded
|
|
16
|
+
from .ocr import OcrError
|
|
17
|
+
from .pdf import UnsupportedPage
|
|
18
|
+
|
|
19
|
+
__version__ = "0.3.0"
|
|
20
|
+
__all__ = [
|
|
21
|
+
"OcrError",
|
|
22
|
+
"Page",
|
|
23
|
+
"PbmSource",
|
|
24
|
+
"UnsupportedPage",
|
|
25
|
+
"hocr",
|
|
26
|
+
"models",
|
|
27
|
+
"ocr",
|
|
28
|
+
"orientation",
|
|
29
|
+
"paddle",
|
|
30
|
+
"pdf",
|
|
31
|
+
"skew",
|
|
32
|
+
"straighten",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
log = logging.getLogger(__name__)
|
|
36
|
+
|
|
37
|
+
STEEP_EDGE = 2.0 # degrees from +-45: a steep-path answer there is as likely the other quarter turn
|
|
38
|
+
BUDGET = 300.0 # seconds of Tesseract per page at most (see _budget)
|
|
39
|
+
MAX_PIXELS = 200_000_000 # above Pillow's own refusal (2 x 89.5 M pixels); an A4 page at 1200 dpi is 140 M
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def straighten(
|
|
43
|
+
page: Page,
|
|
44
|
+
dpi: int,
|
|
45
|
+
*,
|
|
46
|
+
turn: int = 0,
|
|
47
|
+
rotate: bool = True,
|
|
48
|
+
deskew: bool = True,
|
|
49
|
+
min_confidence: float = 3.0,
|
|
50
|
+
scripts: tuple[str, ...] = ("Latin",),
|
|
51
|
+
deskew_min: float = 0.3,
|
|
52
|
+
workdir: str | None = None,
|
|
53
|
+
budget: float | None = BUDGET,
|
|
54
|
+
) -> tuple[int, float]:
|
|
55
|
+
"""(clockwise turn 0/90/180/270, skew in degrees counter-clockwise) for pdf.build; 0 for what is off,
|
|
56
|
+
unsure or unavailable.
|
|
57
|
+
|
|
58
|
+
turn: a turn already known (from the scanner); it is kept and not detected. deskew_min: smaller
|
|
59
|
+
skews are not visible and left alone. budget: seconds of Tesseract for this page at most (None: no
|
|
60
|
+
limit); what is not measured by then is kept as scanned. A page larger than MAX_PIXELS is not read
|
|
61
|
+
at all. Never raises: straightening is a nicety and must not cost a page (a JPEG that Pillow
|
|
62
|
+
cannot decode is still a page the PDF can carry)."""
|
|
63
|
+
if _too_large(page):
|
|
64
|
+
return turn, 0.0
|
|
65
|
+
token = _budget.start(budget)
|
|
66
|
+
try:
|
|
67
|
+
return _straighten(page, dpi, turn, rotate, deskew, min_confidence, scripts, deskew_min, workdir)
|
|
68
|
+
finally:
|
|
69
|
+
_budget.end(token)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _too_large(page: Page) -> bool:
|
|
73
|
+
try:
|
|
74
|
+
data, ext = encoded(page)
|
|
75
|
+
w, h = pdf.pbm_info(data)[:2] if ext == "pbm" else pdf.jpeg_info(data)[:2]
|
|
76
|
+
except Exception: # noqa: BLE001 (the PDF writer reports a damaged page)
|
|
77
|
+
return False
|
|
78
|
+
if w * h > MAX_PIXELS:
|
|
79
|
+
log.warning("page of %d x %d pixels: too large to read; kept as scanned", w, h)
|
|
80
|
+
return True
|
|
81
|
+
return False
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _straighten(
|
|
85
|
+
page: Page,
|
|
86
|
+
dpi: int,
|
|
87
|
+
turn: int,
|
|
88
|
+
rotate: bool,
|
|
89
|
+
deskew: bool,
|
|
90
|
+
min_confidence: float,
|
|
91
|
+
scripts: tuple[str, ...],
|
|
92
|
+
deskew_min: float,
|
|
93
|
+
workdir: str | None,
|
|
94
|
+
) -> tuple[int, float]:
|
|
95
|
+
# Skew first, then the orientation of the straightened page: OSD cannot read a page tilted by
|
|
96
|
+
# more than a few degrees. A 180-degree turn leaves the text lines' angle as it is; a quarter
|
|
97
|
+
# turn means the lines were measured across the text, so the skew is measured again upright.
|
|
98
|
+
known = bool(turn)
|
|
99
|
+
found = _detect(page, dpi, turn, workdir) if deskew else None
|
|
100
|
+
base = 0
|
|
101
|
+
if found is None and deskew and rotate and not known:
|
|
102
|
+
# A page laid at more than 45 degrees (a receipt at about 60, upside down, 01-11-2026): the skew
|
|
103
|
+
# search covers +-45, so its text rows only show a quarter turn on. Measured there, the skew is
|
|
104
|
+
# an ordinary one, and OSD then only has to tell 90 from 270.
|
|
105
|
+
steep = _detect(page, dpi, 90, workdir)
|
|
106
|
+
if steep is not None and abs(abs(steep) - 45) < STEEP_EDGE:
|
|
107
|
+
# a chart with slanted axis labels came out a quarter plus about 45 degrees (benchmark 05-11-2026)
|
|
108
|
+
log.info("skew unsure (%.2f degrees a quarter turn on: too near 45); page kept as scanned", steep)
|
|
109
|
+
elif steep is not None:
|
|
110
|
+
base, found = 90, steep
|
|
111
|
+
log.info("page lies at more than 45 degrees; measured a quarter turn on")
|
|
112
|
+
angle = found if found is not None and abs(found) >= deskew_min else 0.0
|
|
113
|
+
if not known and rotate:
|
|
114
|
+
try:
|
|
115
|
+
more, sure = orientation.detect(page, workdir, min_confidence, scripts, skew=angle, quarter_turn=base)
|
|
116
|
+
except Exception as e: # noqa: BLE001
|
|
117
|
+
log.warning("orientation not determined (%s); page kept as scanned", e)
|
|
118
|
+
more, sure = 0, False
|
|
119
|
+
if base and not sure:
|
|
120
|
+
# The steep path's quarter turn is only half an answer: 90 and 270 level the same lines, and
|
|
121
|
+
# only OSD on the turned page tells them apart. Unsure, it was kept, and every wrong turn in the
|
|
122
|
+
# benchmark (10, 05-11-2026) came from here. Rotations commute, so the angle measured in the
|
|
123
|
+
# turned frame straightens the page as scanned too: sideways, but level. First the words
|
|
124
|
+
# decide, as for an unsure upside-down page.
|
|
125
|
+
try:
|
|
126
|
+
by_words = orientation.quarter_by_words(page, workdir, skew=angle)
|
|
127
|
+
except Exception as e: # noqa: BLE001
|
|
128
|
+
log.warning("words not read (%s)", e)
|
|
129
|
+
by_words = None
|
|
130
|
+
if by_words:
|
|
131
|
+
base, more = by_words, 0
|
|
132
|
+
else:
|
|
133
|
+
log.info("quarter turn not confirmed by the orientation check; page kept sideways")
|
|
134
|
+
base = 0
|
|
135
|
+
turn = (base + more) % 360
|
|
136
|
+
if more in (90, 270):
|
|
137
|
+
angle = _skew(page, dpi, turn, deskew, deskew_min, workdir)
|
|
138
|
+
if angle:
|
|
139
|
+
log.info("page straightened by %.2f degrees", angle)
|
|
140
|
+
return turn, angle
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _detect(page: Page, dpi: int, turn: int, workdir: str | None) -> float | None:
|
|
144
|
+
"""skew.detect, None when unsure or failing."""
|
|
145
|
+
try:
|
|
146
|
+
return skew.detect(page, dpi, turn, workdir)
|
|
147
|
+
except Exception as e: # noqa: BLE001
|
|
148
|
+
log.warning("skew not measured (%s); page kept as scanned", e)
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _skew(page: Page, dpi: int, turn: int, deskew: bool, deskew_min: float, workdir: str | None) -> float:
|
|
153
|
+
if not deskew:
|
|
154
|
+
return 0.0
|
|
155
|
+
found = _detect(page, dpi, turn, workdir)
|
|
156
|
+
return found if found is not None and abs(found) >= deskew_min else 0.0
|
tidyscan/__main__.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""tidyscan [options] -o OUT.pdf PAGE...: page images into one straightened PDF.
|
|
2
|
+
|
|
3
|
+
Also: `tidyscan watch IN OUT` (document folders as scankeyd writes them into PDFs, as a
|
|
4
|
+
service), `tidyscan models fetch [DIR]` (download and check PaddleOCR's models, once) and
|
|
5
|
+
`tidyscan models path` (where they are looked up)."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import io
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from . import __version__, models, ocr, orientation, pdf, skew, straighten
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _load(path: str) -> list[bytes]:
|
|
19
|
+
"""The pages of one input file: a JPEG or PBM as it is; anything Pillow reads converted, every
|
|
20
|
+
frame of a multi-page file (TIFF) a page of its own."""
|
|
21
|
+
with open(path, "rb") as f:
|
|
22
|
+
data = f.read()
|
|
23
|
+
if data.startswith((b"\xff\xd8", b"P4")):
|
|
24
|
+
return [data]
|
|
25
|
+
try:
|
|
26
|
+
from PIL import Image, ImageSequence
|
|
27
|
+
except ImportError:
|
|
28
|
+
raise SystemExit(f"{path}: not JPEG or PBM (other formats need Pillow)") from None
|
|
29
|
+
pages = []
|
|
30
|
+
for img in ImageSequence.Iterator(Image.open(io.BytesIO(data))):
|
|
31
|
+
out = io.BytesIO()
|
|
32
|
+
if img.mode == "1":
|
|
33
|
+
img.save(out, "PPM") # mode 1 saves as binary PBM
|
|
34
|
+
elif img.mode.startswith("I"): # 16-bit gray scaled to 8 bits (32-bit values above 65535 clip)
|
|
35
|
+
img.convert("I").point(lambda v: v / 256).convert("L").save(out, "JPEG", quality=95)
|
|
36
|
+
else:
|
|
37
|
+
img.convert("L" if img.mode in ("L", "LA") else "RGB").save(out, "JPEG", quality=95)
|
|
38
|
+
pages.append(out.getvalue())
|
|
39
|
+
return pages
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _dpi(text: str) -> int:
|
|
43
|
+
value = int(text)
|
|
44
|
+
if not 50 <= value <= 2400:
|
|
45
|
+
raise argparse.ArgumentTypeError(f"{value}: 50-2400")
|
|
46
|
+
return value
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def models_main(argv: list[str]) -> int:
|
|
50
|
+
p = argparse.ArgumentParser(prog="tidyscan models", description="PaddleOCR's PP-OCRv5 models")
|
|
51
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
52
|
+
f = sub.add_parser("fetch", help=f"download the models from release {models.TAG} and check their sha256")
|
|
53
|
+
f.add_argument("directory", nargs="?", help="where to put them (default: the per-user model directory)")
|
|
54
|
+
sub.add_parser("path", help="where the models are looked up, and whether they are there")
|
|
55
|
+
a = p.parse_args(argv)
|
|
56
|
+
if a.cmd == "path":
|
|
57
|
+
print(models.describe())
|
|
58
|
+
return 0
|
|
59
|
+
try:
|
|
60
|
+
written = models.fetch(a.directory)
|
|
61
|
+
except models.ModelError as e:
|
|
62
|
+
print(f"tidyscan: {e}", file=sys.stderr)
|
|
63
|
+
return 1
|
|
64
|
+
for path in written:
|
|
65
|
+
print(f"written: {path}")
|
|
66
|
+
print("models present" if not written else f"{len(written)} file(s) fetched and checked")
|
|
67
|
+
return 0
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def watch_main(argv: list[str]) -> int:
|
|
71
|
+
import signal
|
|
72
|
+
import threading
|
|
73
|
+
|
|
74
|
+
from . import watch
|
|
75
|
+
|
|
76
|
+
p = argparse.ArgumentParser(
|
|
77
|
+
prog="tidyscan watch",
|
|
78
|
+
description="Document folders (IN/<function>/<name>/ with document.json, as scankeyd writes them) "
|
|
79
|
+
"into PDFs in OUT, straightened and with a text layer where asked; each folder goes once its PDF "
|
|
80
|
+
"is written. Turning and straightening need Tesseract, the text layer OCRmyPDF; without them "
|
|
81
|
+
"it only makes the PDFs.",
|
|
82
|
+
)
|
|
83
|
+
p.add_argument("input", metavar="IN")
|
|
84
|
+
p.add_argument("output", metavar="OUT")
|
|
85
|
+
env = os.environ # the defaults, from the settings file all the scan containers share (env_file)
|
|
86
|
+
|
|
87
|
+
def on(name: str, default: str = "auto", auto: bool = True) -> bool:
|
|
88
|
+
"""An on/off setting: on, off (also true/false, yes/no, 1/0), and auto where allowed (on when
|
|
89
|
+
the tool for it is there)."""
|
|
90
|
+
value = env.get(name, default).strip().lower()
|
|
91
|
+
if value in ("on", "true", "yes", "1") or (auto and value == "auto"):
|
|
92
|
+
return True
|
|
93
|
+
if value in ("off", "false", "no", "0"):
|
|
94
|
+
return False
|
|
95
|
+
raise SystemExit(f"tidyscan: {name}={value!r}: on or off" + (" or auto" if auto else ""))
|
|
96
|
+
|
|
97
|
+
p.add_argument(
|
|
98
|
+
"--ocr",
|
|
99
|
+
metavar="LANGS",
|
|
100
|
+
default=env.get("OCR_LANGUAGES", "eng"),
|
|
101
|
+
help="text layer languages, e.g. eng+nld (default: $OCR_LANGUAGES, else eng)",
|
|
102
|
+
)
|
|
103
|
+
p.add_argument(
|
|
104
|
+
"--ocr-folders",
|
|
105
|
+
default=env.get("OCR_FOLDERS", "ocr"),
|
|
106
|
+
help="for documents without an ocr wish in their document.json: comma list of function folders "
|
|
107
|
+
"that get a text layer (default: $OCR_FOLDERS, else ocr; 'all' for every one, '' for none)",
|
|
108
|
+
)
|
|
109
|
+
p.add_argument(
|
|
110
|
+
"--engine",
|
|
111
|
+
choices=ocr.ENGINES,
|
|
112
|
+
default=env.get("OCR_ENGINE", "tesseract").lower(),
|
|
113
|
+
help="OCR engine (default: $OCR_ENGINE, else tesseract)",
|
|
114
|
+
)
|
|
115
|
+
p.add_argument("--no-rotate", action="store_true", help="do not turn pages upright")
|
|
116
|
+
p.add_argument("--no-deskew", action="store_true", help="do not straighten skewed pages")
|
|
117
|
+
p.add_argument("--min-confidence", type=float, default=3.0, help="orientation confidence to turn (default 3)")
|
|
118
|
+
p.add_argument(
|
|
119
|
+
"--pdfa",
|
|
120
|
+
action="store_true",
|
|
121
|
+
default=on("PDFA", "off", auto=False),
|
|
122
|
+
help="PDF/A-2 for the documents that get a text layer (default: $PDFA, else off)",
|
|
123
|
+
)
|
|
124
|
+
p.add_argument("--keep-folders", action="store_true", help="OUT/<function>/<name>.pdf instead of OUT/<name>.pdf")
|
|
125
|
+
p.add_argument("--interval", type=float, default=5.0, help="seconds between looks at IN (default 5)")
|
|
126
|
+
p.add_argument("--once", action="store_true", help="convert what is there now, then stop")
|
|
127
|
+
p.add_argument(
|
|
128
|
+
"--status",
|
|
129
|
+
metavar="FILE",
|
|
130
|
+
default=env.get("STATUS_FILE") or None,
|
|
131
|
+
help="a JSON status file for a monitor (last PDF, last error, failed and waiting documents); "
|
|
132
|
+
"default: $STATUS_FILE, else none",
|
|
133
|
+
)
|
|
134
|
+
p.add_argument("-q", "--quiet", action="store_true")
|
|
135
|
+
a = p.parse_args(argv)
|
|
136
|
+
if a.engine not in ocr.ENGINES: # argparse does not check a default against choices
|
|
137
|
+
p.error(f"OCR_ENGINE={a.engine!r}: one of {', '.join(ocr.ENGINES)}")
|
|
138
|
+
logging.basicConfig(level=logging.WARNING if a.quiet else logging.INFO, format="%(levelname)s %(message)s")
|
|
139
|
+
folders = tuple(f.strip().lower() for f in a.ocr_folders.split(",") if f.strip())
|
|
140
|
+
if folders == ("all",):
|
|
141
|
+
folders = ("*",) # every function folder
|
|
142
|
+
opts = watch.Options(
|
|
143
|
+
ocr_languages=a.ocr,
|
|
144
|
+
ocr_folders=folders,
|
|
145
|
+
engine=a.engine,
|
|
146
|
+
rotate=not a.no_rotate and on("ROTATE"),
|
|
147
|
+
deskew=not a.no_deskew and on("DESKEW"),
|
|
148
|
+
min_confidence=a.min_confidence,
|
|
149
|
+
keep_folders=a.keep_folders,
|
|
150
|
+
pdfa=a.pdfa,
|
|
151
|
+
)
|
|
152
|
+
for d in (a.input, a.output):
|
|
153
|
+
if not os.path.isdir(d):
|
|
154
|
+
print(f"tidyscan: {d} is not a directory", file=sys.stderr)
|
|
155
|
+
return 2
|
|
156
|
+
bad = [a.output] if not os.access(a.output, os.W_OK | os.X_OK) else []
|
|
157
|
+
bad += watch.unwritable(a.input) # IN too: a document's folder goes once its PDF is written
|
|
158
|
+
if bad:
|
|
159
|
+
print(f"tidyscan: not writable for this user: {', '.join(bad)}", file=sys.stderr)
|
|
160
|
+
return 2
|
|
161
|
+
status = watch.Status(a.status, a.input) if a.status else None
|
|
162
|
+
if a.once:
|
|
163
|
+
opts.rotate = opts.rotate and orientation.available()
|
|
164
|
+
opts.deskew = opts.deskew and skew.available()
|
|
165
|
+
watch.run_once(a.input, a.output, opts, status=status)
|
|
166
|
+
return 0
|
|
167
|
+
stop = threading.Event()
|
|
168
|
+
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
169
|
+
signal.signal(sig, lambda *_: stop.set()) # finish the document at hand, then stop
|
|
170
|
+
watch.run(a.input, a.output, opts, a.interval, stop, status)
|
|
171
|
+
return 0
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def main(argv: list[str] | None = None) -> int:
|
|
175
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
176
|
+
if argv[:1] == ["models"]:
|
|
177
|
+
return models_main(argv[1:])
|
|
178
|
+
if argv[:1] == ["watch"]:
|
|
179
|
+
return watch_main(argv[1:])
|
|
180
|
+
p = argparse.ArgumentParser(prog="tidyscan", description=__doc__.split(":", 1)[1].split("\n")[0].strip())
|
|
181
|
+
p.add_argument("pages", nargs="+", metavar="PAGE", help="JPEG or PBM (other formats via Pillow, as JPEG)")
|
|
182
|
+
p.add_argument("-o", "--output", required=True, help="the PDF to write")
|
|
183
|
+
p.add_argument("--dpi", type=_dpi, default=300, help="scan resolution, 50-2400 (default 300)")
|
|
184
|
+
p.add_argument("--no-rotate", action="store_true", help="do not turn pages upright")
|
|
185
|
+
p.add_argument("--no-deskew", action="store_true", help="do not straighten skewed pages")
|
|
186
|
+
p.add_argument("--min-confidence", type=float, default=3.0, help="orientation confidence to turn (default 3)")
|
|
187
|
+
p.add_argument(
|
|
188
|
+
"--ocr",
|
|
189
|
+
metavar="LANGS",
|
|
190
|
+
help="add a text layer, e.g. eng+nld (needs ocrmypdf; the languages steer Tesseract only)",
|
|
191
|
+
)
|
|
192
|
+
p.add_argument(
|
|
193
|
+
"--engine",
|
|
194
|
+
choices=ocr.ENGINES,
|
|
195
|
+
default="tesseract",
|
|
196
|
+
help="OCR engine: tesseract (the default; letters and documents), paddle (PaddleOCR, if "
|
|
197
|
+
"installed; receipts), or auto (paddle when available, else tesseract)",
|
|
198
|
+
)
|
|
199
|
+
p.add_argument("-q", "--quiet", action="store_true")
|
|
200
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
201
|
+
a = p.parse_args(argv)
|
|
202
|
+
logging.basicConfig(level=logging.WARNING if a.quiet else logging.INFO, format="%(message)s")
|
|
203
|
+
pages = [page for path in a.pages for page in _load(path)]
|
|
204
|
+
turns, angles = [], []
|
|
205
|
+
for page in pages:
|
|
206
|
+
turn, angle = straighten(
|
|
207
|
+
page, a.dpi, rotate=not a.no_rotate, deskew=not a.no_deskew, min_confidence=a.min_confidence
|
|
208
|
+
)
|
|
209
|
+
turns.append(turn)
|
|
210
|
+
angles.append(angle)
|
|
211
|
+
try:
|
|
212
|
+
data = pdf.build(pages, a.dpi, turns, angles)
|
|
213
|
+
except pdf.UnsupportedPage as e:
|
|
214
|
+
print(f"tidyscan: {e}", file=sys.stderr)
|
|
215
|
+
return 1
|
|
216
|
+
if a.ocr:
|
|
217
|
+
try:
|
|
218
|
+
data = ocr.add_text_layer(data, a.ocr, engine=a.engine)
|
|
219
|
+
except ocr.OcrError as e:
|
|
220
|
+
print(f"tidyscan: OCR failed ({e}); written without text", file=sys.stderr)
|
|
221
|
+
with open(a.output, "wb") as f:
|
|
222
|
+
f.write(data)
|
|
223
|
+
return 0
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
if __name__ == "__main__":
|
|
227
|
+
sys.exit(main())
|
tidyscan/_budget.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""A time budget for one page: straighten() starts it, and every Tesseract call within gets at most
|
|
2
|
+
what is left of it (and never more than its own limit). Spent, a call ends at once as a timeout,
|
|
3
|
+
which every caller already takes as "unsure": the page is kept as scanned rather than taking many
|
|
4
|
+
minutes (a dense table turned a quarter took up to 130 s, 05-11-2026; the worst case is far longer)."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import contextvars
|
|
9
|
+
import logging
|
|
10
|
+
import subprocess
|
|
11
|
+
import time
|
|
12
|
+
|
|
13
|
+
log = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
_END: contextvars.ContextVar[float | None] = contextvars.ContextVar("tidyscan_page_deadline", default=None)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def start(seconds: float | None) -> contextvars.Token[float | None]:
|
|
19
|
+
return _END.set(None if seconds is None else time.monotonic() + seconds)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def end(token: contextvars.Token[float | None]) -> None:
|
|
23
|
+
_END.reset(token)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def timeout(limit: float) -> float:
|
|
27
|
+
"""The timeout for the next call: its own limit, or less when the page's budget is nearly spent.
|
|
28
|
+
Raises subprocess.TimeoutExpired when nothing is left."""
|
|
29
|
+
deadline = _END.get()
|
|
30
|
+
if deadline is None:
|
|
31
|
+
return limit
|
|
32
|
+
left = deadline - time.monotonic()
|
|
33
|
+
if left <= 0:
|
|
34
|
+
log.warning("the page's time budget is spent; what is not measured yet is kept as scanned")
|
|
35
|
+
raise subprocess.TimeoutExpired("tesseract", 0)
|
|
36
|
+
return min(limit, left)
|
tidyscan/_page.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""A page as this package takes it: JPEG bytes, binary PBM (P4) bytes, or an object with to_pbm()."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import tempfile
|
|
6
|
+
from typing import Protocol, TypeAlias, cast
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PbmSource(Protocol):
|
|
10
|
+
"""A page that encodes itself, e.g. a black-and-white scan held as rows."""
|
|
11
|
+
|
|
12
|
+
def to_pbm(self) -> bytes: ...
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
Page: TypeAlias = bytes | bytearray | memoryview | PbmSource
|
|
16
|
+
"""JPEG bytes, binary PBM (P4) bytes, or an object with to_pbm()."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def encoded(page: Page) -> tuple[bytes, str]:
|
|
20
|
+
"""(data, file extension) of a page."""
|
|
21
|
+
if hasattr(page, "to_pbm"):
|
|
22
|
+
return cast("PbmSource", page).to_pbm(), "pbm"
|
|
23
|
+
data = bytes(page)
|
|
24
|
+
return data, "pbm" if data.startswith(b"P4") else "jpg"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def workdir(path: str | None) -> str:
|
|
28
|
+
return path or tempfile.gettempdir()
|
tidyscan/hocr.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""hOCR from text boxes: what OCRmyPDF turns into the invisible text layer of a page.
|
|
2
|
+
|
|
3
|
+
An OCR engine that finds text boxes (PaddleOCR's detector finds one box per run of text, often
|
|
4
|
+
several per printed line) gives Line objects; page() groups the boxes into rows in reading order
|
|
5
|
+
and writes the hOCR that OCRmyPDF's renderer reads: ocr_page > ocr_carea > ocr_par > ocr_line >
|
|
6
|
+
ocrx_word, each with a pixel bbox. Standard library only.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
from xml.sax.saxutils import escape, quoteattr
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from collections.abc import Iterable, Sequence
|
|
18
|
+
|
|
19
|
+
Box = tuple[int, int, int, int] # x0, y0, x1, y1 in pixels, y down
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class Word:
|
|
24
|
+
text: str
|
|
25
|
+
box: Box
|
|
26
|
+
conf: float = 1.0 # 0..1
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Line:
|
|
31
|
+
text: str
|
|
32
|
+
box: Box
|
|
33
|
+
conf: float = 1.0
|
|
34
|
+
words: list[Word] = field(default_factory=list)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# Characters XML 1.0 does not allow (OCRmyPDF parses the hOCR as XML).
|
|
38
|
+
_NOT_XML = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f]")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def clean(text: str) -> str:
|
|
42
|
+
return _NOT_XML.sub("", text)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def bbox(points: Sequence[Sequence[float]], width: int, height: int) -> Box:
|
|
46
|
+
"""The axis-aligned box around points [(x, y), ...] (or an n x 2 array), clipped to the page."""
|
|
47
|
+
xs = [float(p[0]) for p in points]
|
|
48
|
+
ys = [float(p[1]) for p in points]
|
|
49
|
+
x0, y0 = max(0, int(min(xs))), max(0, int(min(ys)))
|
|
50
|
+
x1, y1 = min(width, int(round(max(xs)))), min(height, int(round(max(ys))))
|
|
51
|
+
return x0, y0, max(x0 + 1, x1), max(y0 + 1, y1)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def split_words(text: str, box: Box, conf: float = 1.0) -> list[Word]:
|
|
55
|
+
"""Words of a line with boxes in proportion to their character positions in it."""
|
|
56
|
+
x0, y0, x1, y1 = box
|
|
57
|
+
per_char = (x1 - x0) / max(1, len(text))
|
|
58
|
+
return [
|
|
59
|
+
Word(
|
|
60
|
+
m.group(),
|
|
61
|
+
(
|
|
62
|
+
x0 + int(m.start() * per_char),
|
|
63
|
+
y0,
|
|
64
|
+
max(x0 + int(m.start() * per_char) + 1, x0 + int(round(m.end() * per_char))),
|
|
65
|
+
y1,
|
|
66
|
+
),
|
|
67
|
+
conf,
|
|
68
|
+
)
|
|
69
|
+
for m in re.finditer(r"\S+", text)
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def rows(lines: list[Line]) -> list[list[Line]]:
|
|
74
|
+
"""Lines grouped into rows (boxes that overlap vertically by half the smaller height), top to
|
|
75
|
+
bottom, each row left to right."""
|
|
76
|
+
out: list[list[Line]] = []
|
|
77
|
+
for line in sorted(lines, key=lambda ln: (ln.box[1] + ln.box[3]) / 2):
|
|
78
|
+
if out:
|
|
79
|
+
row = out[-1]
|
|
80
|
+
top, bottom = min(ln.box[1] for ln in row), max(ln.box[3] for ln in row)
|
|
81
|
+
overlap = min(bottom, line.box[3]) - max(top, line.box[1])
|
|
82
|
+
if overlap > 0.5 * min(bottom - top, line.box[3] - line.box[1]):
|
|
83
|
+
row.append(line)
|
|
84
|
+
continue
|
|
85
|
+
out.append([line])
|
|
86
|
+
return [sorted(row, key=lambda ln: ln.box[0]) for row in out]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _union(boxes: Iterable[Box]) -> Box:
|
|
90
|
+
boxes = list(boxes)
|
|
91
|
+
return (min(b[0] for b in boxes), min(b[1] for b in boxes), max(b[2] for b in boxes), max(b[3] for b in boxes))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _title(box: Box, extra: str = "") -> str:
|
|
95
|
+
return quoteattr(f"bbox {box[0]} {box[1]} {box[2]} {box[3]}{extra}")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def page(width: int, height: int, lines: list[Line], lang: str = "", system: str = "tidyscan") -> tuple[str, str]:
|
|
99
|
+
"""(hOCR document, plain text) of one page image of width x height pixels.
|
|
100
|
+
|
|
101
|
+
A row of boxes becomes one ocr_line, so OCRmyPDF puts spaces between its words and the text
|
|
102
|
+
reads row by row. Lines and words without text are left out."""
|
|
103
|
+
kept = []
|
|
104
|
+
for line in lines:
|
|
105
|
+
words = [Word(clean(w.text).strip(), w.box, w.conf) for w in line.words] or split_words(
|
|
106
|
+
clean(line.text), line.box, line.conf
|
|
107
|
+
)
|
|
108
|
+
words = [w for w in words if w.text]
|
|
109
|
+
if words:
|
|
110
|
+
kept.append(Line(" ".join(w.text for w in words), line.box, line.conf, words))
|
|
111
|
+
out = [
|
|
112
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
113
|
+
'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"'
|
|
114
|
+
' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">',
|
|
115
|
+
'<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">',
|
|
116
|
+
'<head><title></title><meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>',
|
|
117
|
+
f'<meta name="ocr-system" content={quoteattr(system)}/>',
|
|
118
|
+
'<meta name="ocr-capabilities" content="ocr_page ocr_carea ocr_par ocr_line ocrx_word"/></head>',
|
|
119
|
+
"<body>",
|
|
120
|
+
f'<div class="ocr_page" id="page_1" title={quoteattr(f"bbox 0 0 {width} {height}; ppageno 0")}>',
|
|
121
|
+
]
|
|
122
|
+
text = []
|
|
123
|
+
lang_attr = f" lang={quoteattr(lang)}" if lang else ""
|
|
124
|
+
n_word = 0
|
|
125
|
+
for n, row in enumerate(rows(kept), 1):
|
|
126
|
+
box = _union(ln.box for ln in row)
|
|
127
|
+
h = box[3] - box[1]
|
|
128
|
+
# Baseline a fifth of the box height above its bottom: the detector's boxes have room for
|
|
129
|
+
# descenders. It only moves where a viewer draws the selection, not what is found.
|
|
130
|
+
out.append(f'<div class="ocr_carea" id="block_1_{n}" title={_title(box)}>')
|
|
131
|
+
out.append(f'<p class="ocr_par" id="par_1_{n}"{lang_attr} title={_title(box)}>')
|
|
132
|
+
out.append(
|
|
133
|
+
f'<span class="ocr_line" id="line_1_{n}" title={_title(box, f"; baseline 0 -{h // 5}; x_size {h}")}>'
|
|
134
|
+
)
|
|
135
|
+
for line in row:
|
|
136
|
+
for w in line.words:
|
|
137
|
+
n_word += 1
|
|
138
|
+
conf = max(0, min(100, round(w.conf * 100)))
|
|
139
|
+
out.append(
|
|
140
|
+
f'<span class="ocrx_word" id="word_1_{n_word}" title={_title(w.box, f"; x_wconf {conf}")}>'
|
|
141
|
+
f"{escape(w.text)}</span>"
|
|
142
|
+
)
|
|
143
|
+
out.append("</span></p></div>")
|
|
144
|
+
text.append(" ".join(ln.text for ln in row))
|
|
145
|
+
out += ["</div>", "</body>", "</html>", ""]
|
|
146
|
+
return "\n".join(out), "\n".join(text) + ("\n" if text else "")
|