heocr 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- heocr/__init__.py +8 -0
- heocr/__main__.py +3 -0
- heocr/api.py +145 -0
- heocr/cli.py +120 -0
- heocr/contract.py +497 -0
- heocr/convert.py +207 -0
- heocr/docx/__init__.py +0 -0
- heocr/docx/numbering.py +250 -0
- heocr/docx/numformat.py +102 -0
- heocr/docx/reader.py +596 -0
- heocr/docx/styles.py +163 -0
- heocr/htmlread.py +908 -0
- heocr/identify.py +175 -0
- heocr/images.py +103 -0
- heocr/models.py +136 -0
- heocr/ocr/__init__.py +79 -0
- heocr/ocr/regions.py +198 -0
- heocr/ocr/tesseract.py +264 -0
- heocr/pdf/__init__.py +0 -0
- heocr/pdf/classify.py +241 -0
- heocr/pdf/columns.py +483 -0
- heocr/pdf/fontmap.py +221 -0
- heocr/pdf/frame.py +121 -0
- heocr/pdf/furniture.py +160 -0
- heocr/pdf/lines.py +516 -0
- heocr/pdf/numbers.py +108 -0
- heocr/pdf/ocrtables.py +163 -0
- heocr/pdf/reader.py +2518 -0
- heocr/pdf/tables.py +409 -0
- heocr/pdf/webprint.py +234 -0
- heocr/pptx.py +606 -0
- heocr/render.py +155 -0
- heocr/repairs.py +352 -0
- heocr/schemas/heocr_document.v1.schema.json +165 -0
- heocr/textflags.py +60 -0
- heocr/xlsx.py +685 -0
- heocr-0.1.0.dist-info/METADATA +172 -0
- heocr-0.1.0.dist-info/RECORD +44 -0
- heocr-0.1.0.dist-info/WHEEL +5 -0
- heocr-0.1.0.dist-info/entry_points.txt +2 -0
- heocr-0.1.0.dist-info/licenses/LICENSE +202 -0
- heocr-0.1.0.dist-info/licenses/NOTICE +8 -0
- heocr-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +88 -0
- heocr-0.1.0.dist-info/top_level.txt +1 -0
heocr/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""heocr: Hebrew text out of any document, through the best open pipeline for it."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
from heocr.api import extract # noqa: E402
|
|
6
|
+
from heocr.contract import SCHEMA_ID, ContractError, Document, validate # noqa: E402
|
|
7
|
+
|
|
8
|
+
__all__ = ["SCHEMA_ID", "ContractError", "Document", "__version__", "extract", "validate"]
|
heocr/__main__.py
ADDED
heocr/api.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""``heocr.extract``: one call, any document."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
from collections.abc import Callable, Sequence
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from heocr import contract
|
|
12
|
+
from heocr.contract import MAX_PAGES, Document, Unread
|
|
13
|
+
from heocr.convert import CONVERSIONS, CONVERT_TIMEOUT
|
|
14
|
+
from heocr.identify import identify
|
|
15
|
+
from heocr.ocr.tesseract import DEFAULT_LANG, DEFAULT_PSM, check_settings
|
|
16
|
+
|
|
17
|
+
IMAGE_FORMATS = ("png", "jpeg", "tiff", "gif", "bmp", "jp2", "webp")
|
|
18
|
+
# Inputs above this size are refused before parsing. Real tender documents are far smaller.
|
|
19
|
+
MAX_FILE_BYTES = 512 * 1024**2
|
|
20
|
+
|
|
21
|
+
Reader = Callable[[bytes, Document], None]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _readers() -> dict[str, Reader]:
|
|
25
|
+
"""Formats with a route, mapped to the reader that fills a Document in place."""
|
|
26
|
+
from heocr.convert import CONVERSIONS, read_converted
|
|
27
|
+
from heocr.docx.reader import read_docx
|
|
28
|
+
from heocr.htmlread import read_html
|
|
29
|
+
from heocr.images import read_image
|
|
30
|
+
from heocr.pdf.reader import read_pdf
|
|
31
|
+
from heocr.pptx import read_pptx
|
|
32
|
+
from heocr.xlsx import read_xlsx
|
|
33
|
+
|
|
34
|
+
return {"docx": read_docx, "pdf": read_pdf, "html": read_html, "xlsx": read_xlsx, "pptx": read_pptx, **{fmt: read_converted for fmt in CONVERSIONS}, **{fmt: read_image for fmt in IMAGE_FORMATS}}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def extract(
|
|
38
|
+
source: str | os.PathLike[str] | bytes,
|
|
39
|
+
*,
|
|
40
|
+
repairs: bool = True,
|
|
41
|
+
ocr: bool = True,
|
|
42
|
+
ocr_lang: str = DEFAULT_LANG,
|
|
43
|
+
ocr_psm: int = DEFAULT_PSM,
|
|
44
|
+
tesseract: str | Sequence[str] | None = None,
|
|
45
|
+
ocr_model: str | None = None,
|
|
46
|
+
check_numbers: str | bool = "suspect",
|
|
47
|
+
convert: bool = True,
|
|
48
|
+
soffice: str | Sequence[str] | None = None,
|
|
49
|
+
convert_timeout: float | None = None,
|
|
50
|
+
max_pages: int = MAX_PAGES,
|
|
51
|
+
time_limit: float | None = None,
|
|
52
|
+
) -> Document:
|
|
53
|
+
"""Read a document and return its ``heocr_document.v1`` record.
|
|
54
|
+
|
|
55
|
+
``source`` is a file path or the file's bytes. The format is decided from the
|
|
56
|
+
content, never from the file name, and the file name is not recorded.
|
|
57
|
+
|
|
58
|
+
``repairs=False`` turns every repair rule off (``heocr.repairs``): the output is then
|
|
59
|
+
the extracted text, byte for byte.
|
|
60
|
+
|
|
61
|
+
``ocr`` runs the OCR route on pages and regions that need it, through stock Tesseract
|
|
62
|
+
when it is installed (``heocr.ocr.tesseract``): ``ocr_lang`` is ``heb`` by default,
|
|
63
|
+
``ocr_psm`` is 3, and ``tesseract`` names the program when it is not on the PATH.
|
|
64
|
+
``ocr_model`` runs a HeOCR traineddata instead of ``ocr_lang``: a released model's name
|
|
65
|
+
(downloaded on purpose first, ``heocr.models.download``) or a ``.traineddata`` path.
|
|
66
|
+
Without Tesseract, or with ``ocr=False``, those pages are recorded as unread.
|
|
67
|
+
|
|
68
|
+
``check_numbers`` reads text-layer lines a second time, by OCR of their own image
|
|
69
|
+
(``heocr.pdf.numbers``). ``"suspect"`` (the default) reads only lines with a reversed-looking
|
|
70
|
+
digit group, which is the evidence the ``reversed_digit_groups`` repair needs. ``"tables"``
|
|
71
|
+
also reads every table cell with amounts, percentages or dates. ``"all"`` reads every line
|
|
72
|
+
and cell with them, and flags trustworthy disagreements. ``False`` turns the check off,
|
|
73
|
+
including the second reading of numbers in OCR'd table cells.
|
|
74
|
+
|
|
75
|
+
``convert`` reads DOC, RTF and ODT by converting them to DOCX with LibreOffice
|
|
76
|
+
(``heocr.convert``), when it is installed; ``soffice`` names the program when it is not on
|
|
77
|
+
the PATH, and ``convert_timeout`` bounds each conversion (180 s by default). The conversion
|
|
78
|
+
runs with a throwaway profile, and its processes are stopped on timeout. Without LibreOffice,
|
|
79
|
+
or with ``convert=False``, those files are recorded as unread.
|
|
80
|
+
|
|
81
|
+
``max_pages`` bounds the pages (PDF pages, image frames) read from one document: 2,000 by
|
|
82
|
+
default, since every page may cost an OCR run. The rest are recorded as one unread entry
|
|
83
|
+
(``page_limit``) and the document is flagged ``page_limit_reached``.
|
|
84
|
+
|
|
85
|
+
``time_limit`` is a safety net, in seconds, for the whole document (conversion and OCR
|
|
86
|
+
included); there is none by default, so a record does not depend on the machine's speed.
|
|
87
|
+
It is checked before each page, image frame, embedded image and page of number checks: a
|
|
88
|
+
step under way finishes (each Tesseract and LibreOffice call has its own timeout), and what
|
|
89
|
+
is left is recorded as unread (``time_limit``), never dropped, with the document flagged
|
|
90
|
+
``time_limit_reached``. Numbers left unchecked flag ``numbers_not_checked``.
|
|
91
|
+
"""
|
|
92
|
+
started = contract._now()
|
|
93
|
+
if isinstance(max_pages, bool) or not isinstance(max_pages, int) or max_pages < 1:
|
|
94
|
+
raise ValueError("max_pages must be a positive integer")
|
|
95
|
+
if time_limit is not None and (
|
|
96
|
+
isinstance(time_limit, bool) or not isinstance(time_limit, (int, float)) or not math.isfinite(time_limit) or time_limit <= 0
|
|
97
|
+
):
|
|
98
|
+
raise ValueError("time_limit must be a positive number of seconds, or None")
|
|
99
|
+
if check_numbers not in (False, "suspect", "tables", "all"):
|
|
100
|
+
raise ValueError("check_numbers must be 'suspect', 'tables', 'all' or False")
|
|
101
|
+
if ocr:
|
|
102
|
+
check_settings(ocr_lang, ocr_psm)
|
|
103
|
+
if isinstance(source, (bytes, bytearray, memoryview)):
|
|
104
|
+
body = bytes(source)
|
|
105
|
+
else:
|
|
106
|
+
path = Path(source)
|
|
107
|
+
size = path.stat().st_size
|
|
108
|
+
if size > MAX_FILE_BYTES:
|
|
109
|
+
raise ValueError(f"input is {size} bytes; the limit is {MAX_FILE_BYTES}")
|
|
110
|
+
body = path.read_bytes()
|
|
111
|
+
if len(body) > MAX_FILE_BYTES:
|
|
112
|
+
raise ValueError(f"input is {len(body)} bytes; the limit is {MAX_FILE_BYTES}")
|
|
113
|
+
|
|
114
|
+
kind = identify(body)
|
|
115
|
+
doc = Document(
|
|
116
|
+
sha256=hashlib.sha256(body).hexdigest(),
|
|
117
|
+
size=len(body),
|
|
118
|
+
format=kind.format,
|
|
119
|
+
format_evidence=kind.evidence,
|
|
120
|
+
options={
|
|
121
|
+
"repairs": bool(repairs),
|
|
122
|
+
"check_numbers": (check_numbers if check_numbers in ("suspect", "tables", "all") else False) if ocr else False,
|
|
123
|
+
"ocr": {"engine": "tesseract", "lang": ocr_lang, "psm": ocr_psm, **({"model": Path(ocr_model).name if ocr_model else None} if ocr_model else {})}
|
|
124
|
+
if ocr
|
|
125
|
+
else False,
|
|
126
|
+
},
|
|
127
|
+
)
|
|
128
|
+
doc._max_pages = max_pages
|
|
129
|
+
doc.options["max_pages"] = max_pages
|
|
130
|
+
if time_limit is not None:
|
|
131
|
+
doc.options["time_limit"] = time_limit
|
|
132
|
+
doc._time_limit, doc._deadline = float(time_limit), started + time_limit
|
|
133
|
+
doc._ocr_settings = {"cmd": tesseract, "lang": ocr_lang, "psm": ocr_psm, "model": ocr_model} if ocr else None
|
|
134
|
+
if kind.format in CONVERSIONS:
|
|
135
|
+
doc.options["convert"] = {"engine": "libreoffice", "to": CONVERSIONS[kind.format][1]} if convert else False
|
|
136
|
+
doc._convert_settings = {"cmd": soffice, "timeout": convert_timeout or CONVERT_TIMEOUT} if convert else None
|
|
137
|
+
reader = _readers().get(kind.format)
|
|
138
|
+
if reader is None:
|
|
139
|
+
doc.unread.append(
|
|
140
|
+
Unread(scope="document", reason="format_not_supported", detail=f"no route for format {kind.format!r} yet")
|
|
141
|
+
)
|
|
142
|
+
doc.flags.append("not_read")
|
|
143
|
+
return doc
|
|
144
|
+
reader(body, doc)
|
|
145
|
+
return doc
|
heocr/cli.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""The ``heocr`` command."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from heocr import __version__
|
|
10
|
+
from heocr.api import extract
|
|
11
|
+
from heocr.contract import MAX_PAGES
|
|
12
|
+
|
|
13
|
+
_FORMATS = ("text", "markdown", "json")
|
|
14
|
+
_BY_SUFFIX = {".md": "markdown", ".markdown": "markdown", ".json": "json", ".txt": "text"}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _parser() -> argparse.ArgumentParser:
|
|
18
|
+
parser = argparse.ArgumentParser(
|
|
19
|
+
prog="heocr",
|
|
20
|
+
description="Extract the Hebrew text of a document through the best open pipeline for it.",
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument("file", type=Path, nargs="?", help="input document: PDF, DOCX, DOC, RTF, ODT, XLSX, ODS, XLS, PPTX, ODP, PPT, HTML or an image")
|
|
23
|
+
parser.add_argument("-o", "--output", type=Path, help="write here instead of standard output")
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"-f",
|
|
26
|
+
"--format",
|
|
27
|
+
choices=_FORMATS,
|
|
28
|
+
help="output format (default: from the output suffix, else text)",
|
|
29
|
+
)
|
|
30
|
+
parser.add_argument(
|
|
31
|
+
"--no-repairs",
|
|
32
|
+
action="store_true",
|
|
33
|
+
help="turn every repair rule off: output the extracted text byte for byte",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument("--no-ocr", action="store_true", help="do not run OCR; pages that need it are reported unread")
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--check-numbers",
|
|
38
|
+
choices=("suspect", "tables", "all", "off"),
|
|
39
|
+
default="suspect",
|
|
40
|
+
help="read numbers a second time by OCR: suspect digit groups only (default), also every table cell, all amounts and dates, or off",
|
|
41
|
+
)
|
|
42
|
+
parser.add_argument("--no-convert", action="store_true", help="do not convert DOC, RTF, ODT, ODS, XLS, ODP or PPT with LibreOffice; they are reported unread")
|
|
43
|
+
parser.add_argument("--soffice", help="the LibreOffice program (soffice), when it is not on the PATH")
|
|
44
|
+
parser.add_argument("--convert-timeout", type=float, metavar="SECONDS", help="time limit for one LibreOffice conversion (default: 180)")
|
|
45
|
+
parser.add_argument("--max-pages", type=int, default=MAX_PAGES, metavar="N", help=f"read at most N pages or image frames (default: {MAX_PAGES}); the rest are reported unread")
|
|
46
|
+
parser.add_argument("--time-limit", type=float, metavar="SECONDS", help="stop reading a document after SECONDS (none by default); what is left is reported unread")
|
|
47
|
+
parser.add_argument("--ocr-lang", default="heb", help="Tesseract language list (default: heb; e.g. heb+eng)")
|
|
48
|
+
parser.add_argument("--psm", type=int, default=3, help="Tesseract page segmentation mode (default: 3)")
|
|
49
|
+
parser.add_argument("--tesseract", help="the tesseract program, when it is not on the PATH")
|
|
50
|
+
parser.add_argument(
|
|
51
|
+
"--ocr-model",
|
|
52
|
+
help="a HeOCR traineddata for OCR instead of stock heb: a released name (e.g. heocr_r8b) or a path",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"--download-model",
|
|
56
|
+
metavar="NAME",
|
|
57
|
+
help="download a released HeOCR model on purpose, check its pinned sha256, and exit",
|
|
58
|
+
)
|
|
59
|
+
parser.add_argument("-q", "--quiet", action="store_true", help="do not report unread parts on stderr")
|
|
60
|
+
parser.add_argument("--version", action="version", version=f"heocr {__version__}")
|
|
61
|
+
return parser
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def main(argv: list[str] | None = None) -> int:
|
|
65
|
+
parser = _parser()
|
|
66
|
+
args = parser.parse_args(argv)
|
|
67
|
+
if args.download_model:
|
|
68
|
+
from heocr.models import ModelError, download
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
path = download(args.download_model)
|
|
72
|
+
except (ModelError, OSError) as exc:
|
|
73
|
+
print(f"heocr: {exc}", file=sys.stderr)
|
|
74
|
+
return 1
|
|
75
|
+
print(f"heocr: {args.download_model} is ready ({path.name}, sha256 checked)", file=sys.stderr)
|
|
76
|
+
return 0
|
|
77
|
+
if args.file is None:
|
|
78
|
+
parser.error("a file is required (or --download-model NAME)")
|
|
79
|
+
fmt = args.format or (_BY_SUFFIX.get(args.output.suffix.lower(), "text") if args.output else "text")
|
|
80
|
+
try:
|
|
81
|
+
doc = extract(
|
|
82
|
+
args.file,
|
|
83
|
+
repairs=not args.no_repairs,
|
|
84
|
+
ocr=not args.no_ocr,
|
|
85
|
+
ocr_lang=args.ocr_lang,
|
|
86
|
+
ocr_psm=args.psm,
|
|
87
|
+
tesseract=args.tesseract,
|
|
88
|
+
ocr_model=args.ocr_model,
|
|
89
|
+
check_numbers=False if args.check_numbers == "off" else args.check_numbers,
|
|
90
|
+
convert=not args.no_convert,
|
|
91
|
+
soffice=args.soffice,
|
|
92
|
+
convert_timeout=args.convert_timeout,
|
|
93
|
+
max_pages=args.max_pages,
|
|
94
|
+
time_limit=args.time_limit,
|
|
95
|
+
)
|
|
96
|
+
except (OSError, ValueError) as exc:
|
|
97
|
+
print(f"heocr: {exc}", file=sys.stderr)
|
|
98
|
+
return 1
|
|
99
|
+
if fmt == "json":
|
|
100
|
+
out = doc.to_json() + "\n"
|
|
101
|
+
elif fmt == "markdown":
|
|
102
|
+
out = doc.markdown
|
|
103
|
+
else:
|
|
104
|
+
out = doc.text
|
|
105
|
+
if args.output:
|
|
106
|
+
args.output.write_text(out, encoding="utf-8")
|
|
107
|
+
else:
|
|
108
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
109
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
110
|
+
sys.stdout.write(out)
|
|
111
|
+
if not args.quiet:
|
|
112
|
+
for item in doc.unread:
|
|
113
|
+
where = f"page {item.page + 1}: " if item.page is not None else ""
|
|
114
|
+
detail = f" ({item.detail})" if item.detail else ""
|
|
115
|
+
print(f"heocr: unread: {where}{item.reason}{detail}", file=sys.stderr)
|
|
116
|
+
return 0
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
if __name__ == "__main__": # pragma: no cover
|
|
120
|
+
raise SystemExit(main())
|