heocr 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
heocr/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """heocr: Hebrew text out of any document, through the best open pipeline for it."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ from heocr.api import extract # noqa: E402
6
+ from heocr.contract import SCHEMA_ID, ContractError, Document, validate # noqa: E402
7
+
8
+ __all__ = ["SCHEMA_ID", "ContractError", "Document", "__version__", "extract", "validate"]
heocr/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from heocr.cli import main
2
+
3
+ raise SystemExit(main())
heocr/api.py ADDED
@@ -0,0 +1,145 @@
1
+ """``heocr.extract``: one call, any document."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import math
7
+ import os
8
+ from collections.abc import Callable, Sequence
9
+ from pathlib import Path
10
+
11
+ from heocr import contract
12
+ from heocr.contract import MAX_PAGES, Document, Unread
13
+ from heocr.convert import CONVERSIONS, CONVERT_TIMEOUT
14
+ from heocr.identify import identify
15
+ from heocr.ocr.tesseract import DEFAULT_LANG, DEFAULT_PSM, check_settings
16
+
17
+ IMAGE_FORMATS = ("png", "jpeg", "tiff", "gif", "bmp", "jp2", "webp")
18
+ # Inputs above this size are refused before parsing. Real tender documents are far smaller.
19
+ MAX_FILE_BYTES = 512 * 1024**2
20
+
21
+ Reader = Callable[[bytes, Document], None]
22
+
23
+
24
+ def _readers() -> dict[str, Reader]:
25
+ """Formats with a route, mapped to the reader that fills a Document in place."""
26
+ from heocr.convert import CONVERSIONS, read_converted
27
+ from heocr.docx.reader import read_docx
28
+ from heocr.htmlread import read_html
29
+ from heocr.images import read_image
30
+ from heocr.pdf.reader import read_pdf
31
+ from heocr.pptx import read_pptx
32
+ from heocr.xlsx import read_xlsx
33
+
34
+ return {"docx": read_docx, "pdf": read_pdf, "html": read_html, "xlsx": read_xlsx, "pptx": read_pptx, **{fmt: read_converted for fmt in CONVERSIONS}, **{fmt: read_image for fmt in IMAGE_FORMATS}}
35
+
36
+
37
+ def extract(
38
+ source: str | os.PathLike[str] | bytes,
39
+ *,
40
+ repairs: bool = True,
41
+ ocr: bool = True,
42
+ ocr_lang: str = DEFAULT_LANG,
43
+ ocr_psm: int = DEFAULT_PSM,
44
+ tesseract: str | Sequence[str] | None = None,
45
+ ocr_model: str | None = None,
46
+ check_numbers: str | bool = "suspect",
47
+ convert: bool = True,
48
+ soffice: str | Sequence[str] | None = None,
49
+ convert_timeout: float | None = None,
50
+ max_pages: int = MAX_PAGES,
51
+ time_limit: float | None = None,
52
+ ) -> Document:
53
+ """Read a document and return its ``heocr_document.v1`` record.
54
+
55
+ ``source`` is a file path or the file's bytes. The format is decided from the
56
+ content, never from the file name, and the file name is not recorded.
57
+
58
+ ``repairs=False`` turns every repair rule off (``heocr.repairs``): the output is then
59
+ the extracted text, byte for byte.
60
+
61
+ ``ocr`` runs the OCR route on pages and regions that need it, through stock Tesseract
62
+ when it is installed (``heocr.ocr.tesseract``): ``ocr_lang`` is ``heb`` by default,
63
+ ``ocr_psm`` is 3, and ``tesseract`` names the program when it is not on the PATH.
64
+ ``ocr_model`` runs a HeOCR traineddata instead of ``ocr_lang``: a released model's name
65
+ (downloaded on purpose first, ``heocr.models.download``) or a ``.traineddata`` path.
66
+ Without Tesseract, or with ``ocr=False``, those pages are recorded as unread.
67
+
68
+ ``check_numbers`` reads text-layer lines a second time, by OCR of their own image
69
+ (``heocr.pdf.numbers``). ``"suspect"`` (the default) reads only lines with a reversed-looking
70
+ digit group, which is the evidence the ``reversed_digit_groups`` repair needs. ``"tables"``
71
+ also reads every table cell with amounts, percentages or dates. ``"all"`` reads every line
72
+ and cell with them, and flags trustworthy disagreements. ``False`` turns the check off,
73
+ including the second reading of numbers in OCR'd table cells.
74
+
75
+ ``convert`` reads DOC, RTF and ODT by converting them to DOCX with LibreOffice
76
+ (``heocr.convert``), when it is installed; ``soffice`` names the program when it is not on
77
+ the PATH, and ``convert_timeout`` bounds each conversion (180 s by default). The conversion
78
+ runs with a throwaway profile, and its processes are stopped on timeout. Without LibreOffice,
79
+ or with ``convert=False``, those files are recorded as unread.
80
+
81
+ ``max_pages`` bounds the pages (PDF pages, image frames) read from one document: 2,000 by
82
+ default, since every page may cost an OCR run. The rest are recorded as one unread entry
83
+ (``page_limit``) and the document is flagged ``page_limit_reached``.
84
+
85
+ ``time_limit`` is a safety net, in seconds, for the whole document (conversion and OCR
86
+ included); there is none by default, so a record does not depend on the machine's speed.
87
+ It is checked before each page, image frame, embedded image and page of number checks: a
88
+ step under way finishes (each Tesseract and LibreOffice call has its own timeout), and what
89
+ is left is recorded as unread (``time_limit``), never dropped, with the document flagged
90
+ ``time_limit_reached``. Numbers left unchecked flag ``numbers_not_checked``.
91
+ """
92
+ started = contract._now()
93
+ if isinstance(max_pages, bool) or not isinstance(max_pages, int) or max_pages < 1:
94
+ raise ValueError("max_pages must be a positive integer")
95
+ if time_limit is not None and (
96
+ isinstance(time_limit, bool) or not isinstance(time_limit, (int, float)) or not math.isfinite(time_limit) or time_limit <= 0
97
+ ):
98
+ raise ValueError("time_limit must be a positive number of seconds, or None")
99
+ if check_numbers not in (False, "suspect", "tables", "all"):
100
+ raise ValueError("check_numbers must be 'suspect', 'tables', 'all' or False")
101
+ if ocr:
102
+ check_settings(ocr_lang, ocr_psm)
103
+ if isinstance(source, (bytes, bytearray, memoryview)):
104
+ body = bytes(source)
105
+ else:
106
+ path = Path(source)
107
+ size = path.stat().st_size
108
+ if size > MAX_FILE_BYTES:
109
+ raise ValueError(f"input is {size} bytes; the limit is {MAX_FILE_BYTES}")
110
+ body = path.read_bytes()
111
+ if len(body) > MAX_FILE_BYTES:
112
+ raise ValueError(f"input is {len(body)} bytes; the limit is {MAX_FILE_BYTES}")
113
+
114
+ kind = identify(body)
115
+ doc = Document(
116
+ sha256=hashlib.sha256(body).hexdigest(),
117
+ size=len(body),
118
+ format=kind.format,
119
+ format_evidence=kind.evidence,
120
+ options={
121
+ "repairs": bool(repairs),
122
+ "check_numbers": (check_numbers if check_numbers in ("suspect", "tables", "all") else False) if ocr else False,
123
+ "ocr": {"engine": "tesseract", "lang": ocr_lang, "psm": ocr_psm, **({"model": Path(ocr_model).name if ocr_model else None} if ocr_model else {})}
124
+ if ocr
125
+ else False,
126
+ },
127
+ )
128
+ doc._max_pages = max_pages
129
+ doc.options["max_pages"] = max_pages
130
+ if time_limit is not None:
131
+ doc.options["time_limit"] = time_limit
132
+ doc._time_limit, doc._deadline = float(time_limit), started + time_limit
133
+ doc._ocr_settings = {"cmd": tesseract, "lang": ocr_lang, "psm": ocr_psm, "model": ocr_model} if ocr else None
134
+ if kind.format in CONVERSIONS:
135
+ doc.options["convert"] = {"engine": "libreoffice", "to": CONVERSIONS[kind.format][1]} if convert else False
136
+ doc._convert_settings = {"cmd": soffice, "timeout": convert_timeout or CONVERT_TIMEOUT} if convert else None
137
+ reader = _readers().get(kind.format)
138
+ if reader is None:
139
+ doc.unread.append(
140
+ Unread(scope="document", reason="format_not_supported", detail=f"no route for format {kind.format!r} yet")
141
+ )
142
+ doc.flags.append("not_read")
143
+ return doc
144
+ reader(body, doc)
145
+ return doc
heocr/cli.py ADDED
@@ -0,0 +1,120 @@
1
+ """The ``heocr`` command."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ from heocr import __version__
10
+ from heocr.api import extract
11
+ from heocr.contract import MAX_PAGES
12
+
13
+ _FORMATS = ("text", "markdown", "json")
14
+ _BY_SUFFIX = {".md": "markdown", ".markdown": "markdown", ".json": "json", ".txt": "text"}
15
+
16
+
17
+ def _parser() -> argparse.ArgumentParser:
18
+ parser = argparse.ArgumentParser(
19
+ prog="heocr",
20
+ description="Extract the Hebrew text of a document through the best open pipeline for it.",
21
+ )
22
+ parser.add_argument("file", type=Path, nargs="?", help="input document: PDF, DOCX, DOC, RTF, ODT, XLSX, ODS, XLS, PPTX, ODP, PPT, HTML or an image")
23
+ parser.add_argument("-o", "--output", type=Path, help="write here instead of standard output")
24
+ parser.add_argument(
25
+ "-f",
26
+ "--format",
27
+ choices=_FORMATS,
28
+ help="output format (default: from the output suffix, else text)",
29
+ )
30
+ parser.add_argument(
31
+ "--no-repairs",
32
+ action="store_true",
33
+ help="turn every repair rule off: output the extracted text byte for byte",
34
+ )
35
+ parser.add_argument("--no-ocr", action="store_true", help="do not run OCR; pages that need it are reported unread")
36
+ parser.add_argument(
37
+ "--check-numbers",
38
+ choices=("suspect", "tables", "all", "off"),
39
+ default="suspect",
40
+ help="read numbers a second time by OCR: suspect digit groups only (default), also every table cell, all amounts and dates, or off",
41
+ )
42
+ parser.add_argument("--no-convert", action="store_true", help="do not convert DOC, RTF, ODT, ODS, XLS, ODP or PPT with LibreOffice; they are reported unread")
43
+ parser.add_argument("--soffice", help="the LibreOffice program (soffice), when it is not on the PATH")
44
+ parser.add_argument("--convert-timeout", type=float, metavar="SECONDS", help="time limit for one LibreOffice conversion (default: 180)")
45
+ parser.add_argument("--max-pages", type=int, default=MAX_PAGES, metavar="N", help=f"read at most N pages or image frames (default: {MAX_PAGES}); the rest are reported unread")
46
+ parser.add_argument("--time-limit", type=float, metavar="SECONDS", help="stop reading a document after SECONDS (none by default); what is left is reported unread")
47
+ parser.add_argument("--ocr-lang", default="heb", help="Tesseract language list (default: heb; e.g. heb+eng)")
48
+ parser.add_argument("--psm", type=int, default=3, help="Tesseract page segmentation mode (default: 3)")
49
+ parser.add_argument("--tesseract", help="the tesseract program, when it is not on the PATH")
50
+ parser.add_argument(
51
+ "--ocr-model",
52
+ help="a HeOCR traineddata for OCR instead of stock heb: a released name (e.g. heocr_r8b) or a path",
53
+ )
54
+ parser.add_argument(
55
+ "--download-model",
56
+ metavar="NAME",
57
+ help="download a released HeOCR model on purpose, check its pinned sha256, and exit",
58
+ )
59
+ parser.add_argument("-q", "--quiet", action="store_true", help="do not report unread parts on stderr")
60
+ parser.add_argument("--version", action="version", version=f"heocr {__version__}")
61
+ return parser
62
+
63
+
64
+ def main(argv: list[str] | None = None) -> int:
65
+ parser = _parser()
66
+ args = parser.parse_args(argv)
67
+ if args.download_model:
68
+ from heocr.models import ModelError, download
69
+
70
+ try:
71
+ path = download(args.download_model)
72
+ except (ModelError, OSError) as exc:
73
+ print(f"heocr: {exc}", file=sys.stderr)
74
+ return 1
75
+ print(f"heocr: {args.download_model} is ready ({path.name}, sha256 checked)", file=sys.stderr)
76
+ return 0
77
+ if args.file is None:
78
+ parser.error("a file is required (or --download-model NAME)")
79
+ fmt = args.format or (_BY_SUFFIX.get(args.output.suffix.lower(), "text") if args.output else "text")
80
+ try:
81
+ doc = extract(
82
+ args.file,
83
+ repairs=not args.no_repairs,
84
+ ocr=not args.no_ocr,
85
+ ocr_lang=args.ocr_lang,
86
+ ocr_psm=args.psm,
87
+ tesseract=args.tesseract,
88
+ ocr_model=args.ocr_model,
89
+ check_numbers=False if args.check_numbers == "off" else args.check_numbers,
90
+ convert=not args.no_convert,
91
+ soffice=args.soffice,
92
+ convert_timeout=args.convert_timeout,
93
+ max_pages=args.max_pages,
94
+ time_limit=args.time_limit,
95
+ )
96
+ except (OSError, ValueError) as exc:
97
+ print(f"heocr: {exc}", file=sys.stderr)
98
+ return 1
99
+ if fmt == "json":
100
+ out = doc.to_json() + "\n"
101
+ elif fmt == "markdown":
102
+ out = doc.markdown
103
+ else:
104
+ out = doc.text
105
+ if args.output:
106
+ args.output.write_text(out, encoding="utf-8")
107
+ else:
108
+ if hasattr(sys.stdout, "reconfigure"):
109
+ sys.stdout.reconfigure(encoding="utf-8")
110
+ sys.stdout.write(out)
111
+ if not args.quiet:
112
+ for item in doc.unread:
113
+ where = f"page {item.page + 1}: " if item.page is not None else ""
114
+ detail = f" ({item.detail})" if item.detail else ""
115
+ print(f"heocr: unread: {where}{item.reason}{detail}", file=sys.stderr)
116
+ return 0
117
+
118
+
119
+ if __name__ == "__main__": # pragma: no cover
120
+ raise SystemExit(main())