heocr 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
heocr/contract.py ADDED
@@ -0,0 +1,497 @@
1
+ """The ``heocr_document.v1`` record: what was read, by which route, and what was not.
2
+
3
+ The JSON record is the source of truth. ``Document.text`` and ``Document.markdown``
4
+ are views of its blocks. Records carry no local path and no file name.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import math
11
+ import re
12
+ import time
13
+ from dataclasses import dataclass, field
14
+ from typing import Any
15
+
16
+ SCHEMA_ID = "heocr_document.v1"
17
+
18
+ BLOCK_KINDS = ("text", "heading", "list_item", "table", "figure")
19
+ # "navigation": site menus, breadcrumbs and "skip to content" links (D4f); kept, out of the body flow.
20
+ # "notes": a slide's speaker notes (D4e).
21
+ ROLES = ("body", "header", "footer", "footnote", "endnote", "comment", "textbox", "navigation", "notes")
22
+ SOURCES = ("text_layer", "office_xml", "converted_docx", "converted_xlsx", "converted_pptx", "html", "ocr", "none")
23
+ PAGE_CLASSES = ("T1", "T2", "T3", "T4", "T5", "T6", "empty")
24
+ DIRECTIONS = ("rtl", "ltr")
25
+ UNREAD_SCOPES = ("document", "page", "block")
26
+
27
+ # Strings that look like local filesystem locations. Provenance must stay path-free;
28
+ # document content (block text, cells) is content and is not path-checked.
29
+ LOCAL_PATH = re.compile(r"(\A(/|~/|[A-Za-z]:[\\/]|\\\\|file:))|(/(Users|home|tmp|private|var)/)")
30
+
31
+
32
+ def json_schema() -> dict[str, Any]:
33
+ """``heocr_document.v1`` as a JSON Schema (draft 2020-12), shipped with the package.
34
+
35
+ It describes every field a record may carry; objects are open, as a later v1 may add fields
36
+ (docs/contract-v1.md). Rules that need the whole record, such as a page's index being its
37
+ position, are checked by :func:`validate`, which a record passes before it is returned.
38
+ """
39
+ from importlib.resources import files
40
+
41
+ return json.loads(files("heocr").joinpath("schemas", "heocr_document.v1.schema.json").read_text(encoding="utf-8"))
42
+
43
+
44
+ class ContractError(ValueError):
45
+ """A record does not satisfy ``heocr_document.v1``."""
46
+
47
+
48
+ @dataclass
49
+ class Cell:
50
+ text: str
51
+ colspan: int = 1
52
+ rowspan: int = 1
53
+ flags: list[str] = field(default_factory=list)
54
+ # OCR'd cells whose number was read by two models: each reading, {"model", "text", "confidence"}.
55
+ readings: list[dict[str, Any]] | None = None
56
+ _lines: Any = field(default=None, repr=False, compare=False) # PDF layer cells: [text, bbox] per line, for number checks
57
+
58
+ def to_dict(self) -> dict[str, Any]:
59
+ out: dict[str, Any] = {"text": self.text}
60
+ if self.colspan != 1:
61
+ out["colspan"] = self.colspan
62
+ if self.rowspan != 1:
63
+ out["rowspan"] = self.rowspan
64
+ if self.flags:
65
+ out["flags"] = list(self.flags)
66
+ if self.readings:
67
+ out["readings"] = [dict(r) for r in self.readings]
68
+ return out
69
+
70
+
71
+ @dataclass
72
+ class Line:
73
+ """One physical line of a block, as read from a paged source."""
74
+
75
+ text: str
76
+ bbox: list[float]
77
+ direction: str = "rtl"
78
+
79
+ def to_dict(self) -> dict[str, Any]:
80
+ return {"text": self.text, "bbox": _round_box(self.bbox), "direction": self.direction}
81
+
82
+
83
+ @dataclass
84
+ class Block:
85
+ kind: str
86
+ text: str = ""
87
+ role: str = "body"
88
+ source: str = "none"
89
+ tool: str = ""
90
+ page: int | None = None
91
+ bbox: list[float] | None = None
92
+ level: int | None = None
93
+ label: str | None = None
94
+ note_id: str | None = None
95
+ direction: str | None = None
96
+ cells: list[list[Cell]] | None = None
97
+ lines: list[Line] | None = None
98
+ ocr: dict[str, Any] | None = None # for source "ocr": engine, settings, status "unverified", confidence
99
+ note_refs: list[dict[str, Any]] | None = None # footnote/endnote references: kind, id, offset in text
100
+ checks: dict[str, Any] | None = None # second readings: {"numbers": checked, "agreed": agreed}
101
+ repairs: list[dict[str, Any]] = field(default_factory=list)
102
+ flags: list[str] = field(default_factory=list)
103
+ half: str | None = None # a PDF spread split in two (D4c): "right" or "left", the page it belongs to
104
+ slide: int | None = None # PPTX (D4e): the slide's number as shown, from 1
105
+ _cell_repairs: Any = field(default=None, repr=False, compare=False) # PDF tables: repairs made in cells, logged by the reader
106
+
107
+ def to_dict(self, index: int) -> dict[str, Any]:
108
+ out: dict[str, Any] = {"id": f"b{index}", "kind": self.kind, "role": self.role}
109
+ if self.kind == "table":
110
+ out["cells"] = [[cell.to_dict() for cell in row] for row in (self.cells or [])]
111
+ else:
112
+ out["text"] = self.text
113
+ for key in ("level", "label", "note_id", "direction", "page", "half", "slide"):
114
+ value = getattr(self, key)
115
+ if value is not None:
116
+ out[key] = value
117
+ if self.bbox is not None:
118
+ out["bbox"] = _round_box(self.bbox)
119
+ if self.lines is not None:
120
+ out["lines"] = [line.to_dict() for line in self.lines]
121
+ if self.ocr is not None:
122
+ out["ocr"] = dict(self.ocr)
123
+ if self.note_refs:
124
+ out["note_refs"] = [dict(r) for r in self.note_refs]
125
+ if self.checks:
126
+ out["checks"] = dict(self.checks)
127
+ out["source"] = self.source
128
+ out["tool"] = self.tool
129
+ out["repairs"] = list(self.repairs)
130
+ out["flags"] = sorted(set(self.flags))
131
+ return out
132
+
133
+
134
+ @dataclass
135
+ class Page:
136
+ index: int
137
+ width: float
138
+ height: float
139
+ page_class: str
140
+ route: str
141
+ signals: dict[str, Any] = field(default_factory=dict)
142
+ coverage: dict[str, Any] = field(default_factory=dict)
143
+ flags: list[str] = field(default_factory=list)
144
+
145
+ def to_dict(self) -> dict[str, Any]:
146
+ return {
147
+ "index": self.index,
148
+ "width": round(self.width, 2),
149
+ "height": round(self.height, 2),
150
+ "class": self.page_class,
151
+ "route": self.route,
152
+ "signals": dict(self.signals),
153
+ "coverage": dict(self.coverage),
154
+ "flags": sorted(set(self.flags)),
155
+ }
156
+
157
+
158
+ # A cell spans at most this many columns or rows. Larger declared spans (hostile or broken input)
159
+ # are capped, and the cell is flagged span_capped; renderers clamp to it too.
160
+ MAX_SPAN = 1000
161
+
162
+
163
+ def cap_span(value: int, flags: list[str]) -> int:
164
+ """``value`` within 1..MAX_SPAN; a capped span flags the cell ``span_capped``."""
165
+ if value > MAX_SPAN:
166
+ flags.append("span_capped")
167
+ return MAX_SPAN
168
+ return max(value, 1)
169
+
170
+
171
+ # Pages (PDF pages, image frames) read at most per document. Every page may cost an OCR run of
172
+ # seconds, so a document of tens of thousands of pages, hostile or merely huge, would run for
173
+ # hours. The rest are recorded as unread, in one entry, never dropped silently.
174
+ MAX_PAGES = 2000
175
+
176
+
177
+ def page_limit(doc: "Document", total: int) -> int:
178
+ """The number of pages to read; records the rest as one unread entry and flags the document."""
179
+ limit = doc._max_pages
180
+ if total <= limit:
181
+ return total
182
+ doc.unread.append(Unread(scope="document", reason="page_limit", detail=f"pages {limit + 1}-{total} of {total} not read (max_pages={limit})"))
183
+ doc.flags.append("page_limit_reached")
184
+ return limit
185
+
186
+
187
+ _now = time.monotonic # the clock of the per-document time limit (tests replace it)
188
+
189
+
190
+ def time_up(doc: "Document") -> bool:
191
+ """Whether the document's time limit (``extract(time_limit=...)``) has passed; never without one."""
192
+ return doc._deadline is not None and _now() >= doc._deadline
193
+
194
+
195
+ def time_limit_reached(doc: "Document", detail: str, *, scope: str = "document", page: int | None = None) -> None:
196
+ """Record what the time limit left unread, and flag the document once."""
197
+ doc.unread.append(Unread(scope=scope, reason="time_limit", page=page, detail=f"{detail}: the time limit of {doc._time_limit:g} s was reached"))
198
+ if "time_limit_reached" not in doc.flags:
199
+ doc.flags.append("time_limit_reached")
200
+
201
+
202
+ def pages_left_unread(doc: "Document", index: int, total: int) -> None:
203
+ """Pages ``index``..``total - 1`` not read: the time limit was reached before page ``index``."""
204
+ span = f"page {total}" if index == total - 1 else f"pages {index + 1}-{total}"
205
+ time_limit_reached(doc, f"{span} of {total} not read")
206
+
207
+
208
+ @dataclass
209
+ class Unread:
210
+ """Something the pipeline saw but did not read, and why. Never dropped silently."""
211
+
212
+ scope: str
213
+ reason: str
214
+ page: int | None = None
215
+ detail: str | None = None
216
+ bbox: list[float] | None = None # the unread region, in the page's coordinates
217
+
218
+ def to_dict(self) -> dict[str, Any]:
219
+ out: dict[str, Any] = {"scope": self.scope, "reason": self.reason}
220
+ if self.page is not None:
221
+ out["page"] = self.page
222
+ if self.bbox is not None:
223
+ out["bbox"] = _round_box(self.bbox)
224
+ if self.detail is not None:
225
+ out["detail"] = self.detail
226
+ return out
227
+
228
+
229
+ @dataclass
230
+ class Document:
231
+ sha256: str
232
+ size: int
233
+ format: str
234
+ format_evidence: str
235
+ page_count: int | None = None
236
+ pages: list[Page] = field(default_factory=list)
237
+ blocks: list[Block] = field(default_factory=list)
238
+ unread: list[Unread] = field(default_factory=list)
239
+ flags: list[str] = field(default_factory=list)
240
+ components: dict[str, str] = field(default_factory=dict)
241
+ options: dict[str, Any] = field(default_factory=lambda: {"repairs": True})
242
+ # Run-time only, never serialized: OCR settings and the engine found for this document.
243
+ _ocr_settings: dict[str, Any] | None = field(default=None, repr=False, compare=False)
244
+ _ocr_engine: Any = field(default=None, repr=False, compare=False)
245
+ _ocr_error: str | None = field(default=None, repr=False, compare=False)
246
+ _convert_settings: dict[str, Any] | None = field(default=None, repr=False, compare=False) # LibreOffice: cmd, timeout
247
+ _max_pages: int = field(default=MAX_PAGES, repr=False, compare=False) # pages read at most
248
+ _time_limit: float | None = field(default=None, repr=False, compare=False) # seconds for the whole document
249
+ _deadline: float | None = field(default=None, repr=False, compare=False) # _now() value when it runs out
250
+ _number_engine: Any = field(default=None, repr=False, compare=False) # the second reading of numbers (eng)
251
+ _number_error: str | None = field(default=None, repr=False, compare=False)
252
+ _font_codes: dict = field(default_factory=dict, repr=False, compare=False) # (font, letter) -> code
253
+ _shifted_fonts: dict = field(default_factory=dict, repr=False, compare=False) # font -> "latin1" | "pua" (cp1255_shifted_font)
254
+ _regions_read: dict = field(default_factory=dict, repr=False, compare=False) # image digest -> its first region OCR (repeated images)
255
+ _rtl: bool = field(default=True, repr=False, compare=False) # the document's direction (PDF tables)
256
+ # font_private_use_by_ocr: the maps learned for this document, with their evidence (no text).
257
+ font_maps: list[dict[str, Any]] = field(default_factory=list)
258
+
259
+ # -- views -------------------------------------------------------------
260
+ @property
261
+ def text(self) -> str:
262
+ from heocr.render import to_text
263
+
264
+ return to_text(self)
265
+
266
+ @property
267
+ def markdown(self) -> str:
268
+ from heocr.render import to_markdown
269
+
270
+ return to_markdown(self)
271
+
272
+ # -- record ------------------------------------------------------------
273
+ def to_dict(self) -> dict[str, Any]:
274
+ from heocr import __version__
275
+
276
+ document: dict[str, Any] = {
277
+ "sha256": self.sha256,
278
+ "bytes": self.size,
279
+ "format": self.format,
280
+ "format_evidence": self.format_evidence,
281
+ }
282
+ if self.page_count is not None:
283
+ document["page_count"] = self.page_count
284
+ payload = {
285
+ "schema": SCHEMA_ID,
286
+ "producer": {
287
+ "name": "heocr",
288
+ "version": __version__,
289
+ "components": dict(sorted(self.components.items())),
290
+ },
291
+ "document": document,
292
+ "options": dict(self.options),
293
+ "pages": [page.to_dict() for page in self.pages],
294
+ "blocks": [block.to_dict(i) for i, block in enumerate(self.blocks)],
295
+ "unread": [item.to_dict() for item in self.unread],
296
+ "flags": sorted(set(self.flags)),
297
+ }
298
+ if self.font_maps:
299
+ payload["font_maps"] = [dict(m) for m in self.font_maps]
300
+ errors = validate(payload)
301
+ if errors:
302
+ raise ContractError("; ".join(errors))
303
+ return payload
304
+
305
+ def to_json(self, indent: int | None = 2) -> str:
306
+ return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
307
+
308
+
309
+ def _round_box(box: list[float]) -> list[float]:
310
+ return [round(float(v), 2) for v in box]
311
+
312
+
313
+ # -- validation ----------------------------------------------------------------
314
+
315
+
316
+ def _path_like(value: Any, where: str):
317
+ if isinstance(value, str):
318
+ if LOCAL_PATH.search(value):
319
+ yield where
320
+ elif isinstance(value, dict):
321
+ for key, item in value.items():
322
+ yield from _path_like(item, f"{where}.{key}")
323
+ elif isinstance(value, list):
324
+ for index, item in enumerate(value):
325
+ yield from _path_like(item, f"{where}[{index}]")
326
+
327
+
328
+ def _is_box(value: Any) -> bool:
329
+ return (
330
+ isinstance(value, list)
331
+ and len(value) == 4
332
+ and all(isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) for v in value)
333
+ and value[2] >= value[0]
334
+ and value[3] >= value[1]
335
+ )
336
+
337
+
338
+ def _strings(value: Any) -> bool:
339
+ return isinstance(value, list) and all(isinstance(v, str) for v in value)
340
+
341
+
342
+ def validate(payload: Any) -> list[str]:
343
+ """Return the list of contract violations in a ``heocr_document.v1`` payload."""
344
+ errors: list[str] = []
345
+ if not isinstance(payload, dict):
346
+ return ["record is not an object"]
347
+ if payload.get("schema") != SCHEMA_ID:
348
+ errors.append(f"schema must be {SCHEMA_ID!r}")
349
+ producer = payload.get("producer")
350
+ if not isinstance(producer, dict) or producer.get("name") != "heocr" or not isinstance(
351
+ producer.get("version"), str
352
+ ):
353
+ errors.append("producer must name heocr and its version")
354
+ document = payload.get("document")
355
+ if not isinstance(document, dict):
356
+ errors.append("document must be an object")
357
+ document = {}
358
+ sha = document.get("sha256")
359
+ if not (isinstance(sha, str) and re.fullmatch(r"[0-9a-f]{64}", sha)):
360
+ errors.append("document.sha256 must be 64 lowercase hex digits")
361
+ if not isinstance(document.get("bytes"), int) or document.get("bytes", -1) < 0:
362
+ errors.append("document.bytes must be a non-negative integer")
363
+ for key in ("format", "format_evidence"):
364
+ if not isinstance(document.get(key), str) or not document.get(key):
365
+ errors.append(f"document.{key} must be a non-empty string")
366
+ for key in ("name", "path", "filename", "file"):
367
+ if key in document:
368
+ errors.append(f"document.{key} is not allowed: records carry no file names or paths")
369
+
370
+ options = payload.get("options", {})
371
+ if not isinstance(options, dict) or not isinstance(options.get("repairs", True), bool):
372
+ errors.append("options must be an object, with a boolean repairs")
373
+ elif "ocr" in options and not (options["ocr"] is False or isinstance(options["ocr"], dict)):
374
+ errors.append("options.ocr must be false or an object")
375
+ elif "max_pages" in options and not (isinstance(options["max_pages"], int) and not isinstance(options["max_pages"], bool) and options["max_pages"] > 0):
376
+ errors.append("options.max_pages must be a positive integer")
377
+ elif "time_limit" in options and not (
378
+ isinstance(options["time_limit"], (int, float)) and not isinstance(options["time_limit"], bool) and math.isfinite(options["time_limit"]) and options["time_limit"] > 0
379
+ ):
380
+ errors.append("options.time_limit must be a positive number of seconds")
381
+ elif "convert" in options and not (options["convert"] is False or isinstance(options["convert"], dict)):
382
+ errors.append("options.convert must be false or an object")
383
+ elif options.get("check_numbers", False) not in (False, "suspect", "tables", "all"):
384
+ errors.append("options.check_numbers must be 'suspect', 'tables', 'all' or false")
385
+
386
+ pages = payload.get("pages")
387
+ if not isinstance(pages, list):
388
+ errors.append("pages must be a list")
389
+ pages = []
390
+ for i, page in enumerate(pages):
391
+ where = f"pages[{i}]"
392
+ if not isinstance(page, dict):
393
+ errors.append(f"{where} is not an object")
394
+ continue
395
+ if page.get("index") != i:
396
+ errors.append(f"{where}.index must be {i}")
397
+ if page.get("class") not in PAGE_CLASSES:
398
+ errors.append(f"{where}.class must be one of {PAGE_CLASSES}")
399
+ if not isinstance(page.get("route"), str):
400
+ errors.append(f"{where}.route must be a string")
401
+ for key in ("signals", "coverage"):
402
+ if not isinstance(page.get(key), dict):
403
+ errors.append(f"{where}.{key} must be an object")
404
+ if not _strings(page.get("flags")):
405
+ errors.append(f"{where}.flags must be a list of strings")
406
+
407
+ blocks = payload.get("blocks")
408
+ if not isinstance(blocks, list):
409
+ errors.append("blocks must be a list")
410
+ blocks = []
411
+ for i, block in enumerate(blocks):
412
+ where = f"blocks[{i}]"
413
+ if not isinstance(block, dict):
414
+ errors.append(f"{where} is not an object")
415
+ continue
416
+ if block.get("id") != f"b{i}":
417
+ errors.append(f"{where}.id must be 'b{i}'")
418
+ if block.get("kind") not in BLOCK_KINDS:
419
+ errors.append(f"{where}.kind must be one of {BLOCK_KINDS}")
420
+ if block.get("role") not in ROLES:
421
+ errors.append(f"{where}.role must be one of {ROLES}")
422
+ if block.get("source") not in SOURCES:
423
+ errors.append(f"{where}.source must be one of {SOURCES}")
424
+ if not isinstance(block.get("tool"), str):
425
+ errors.append(f"{where}.tool must be a string")
426
+ if block.get("kind") == "table":
427
+ cells = block.get("cells")
428
+ if not isinstance(cells, list) or not all(
429
+ isinstance(row, list) and all(isinstance(c, dict) and isinstance(c.get("text"), str) for c in row)
430
+ for row in cells
431
+ ):
432
+ errors.append(f"{where}.cells must be rows of cells with text")
433
+ for row in cells if isinstance(cells, list) else []:
434
+ for c in row if isinstance(row, list) else []:
435
+ readings = c.get("readings") if isinstance(c, dict) else None
436
+ if readings is not None and not (
437
+ isinstance(readings, list)
438
+ and all(isinstance(r, dict) and isinstance(r.get("model"), str) and isinstance(r.get("text"), str) for r in readings)
439
+ ):
440
+ errors.append(f"{where}.cells readings must be objects with a model and a text")
441
+ elif not isinstance(block.get("text"), str):
442
+ errors.append(f"{where}.text must be a string")
443
+ if "slide" in block and not (isinstance(block["slide"], int) and not isinstance(block["slide"], bool) and block["slide"] >= 1):
444
+ errors.append(f"{where}.slide must be a positive integer")
445
+ if "half" in block and block["half"] not in ("right", "left"):
446
+ errors.append(f"{where}.half must be 'right' or 'left'")
447
+ if "direction" in block and block["direction"] not in DIRECTIONS:
448
+ errors.append(f"{where}.direction must be one of {DIRECTIONS}")
449
+ if "page" in block and not (isinstance(block["page"], int) and 0 <= block["page"] < max(len(pages), 1)):
450
+ errors.append(f"{where}.page must index pages")
451
+ if "bbox" in block and not _is_box(block["bbox"]):
452
+ errors.append(f"{where}.bbox must be [x0, y0, x1, y1]")
453
+ for line in block.get("lines", []) or []:
454
+ if not (isinstance(line, dict) and isinstance(line.get("text"), str) and _is_box(line.get("bbox"))):
455
+ errors.append(f"{where}.lines must have text and bbox")
456
+ break
457
+ if block.get("source") == "ocr" and not (isinstance(block.get("ocr"), dict) and block["ocr"].get("status") == "unverified"):
458
+ errors.append(f"{where}.ocr must be an object with status 'unverified' for OCR text")
459
+ for ref in block.get("note_refs", []) or []:
460
+ if not (
461
+ isinstance(ref, dict)
462
+ and ref.get("kind") in ("footnote", "endnote")
463
+ and isinstance(ref.get("offset"), int)
464
+ and 0 <= ref["offset"] <= len(block.get("text", ""))
465
+ ):
466
+ errors.append(f"{where}.note_refs must have a kind and an offset inside the text")
467
+ break
468
+ if not isinstance(block.get("repairs"), list):
469
+ errors.append(f"{where}.repairs must be a list")
470
+ if not _strings(block.get("flags")):
471
+ errors.append(f"{where}.flags must be a list of strings")
472
+
473
+ unread = payload.get("unread")
474
+ if not isinstance(unread, list):
475
+ errors.append("unread must be a list")
476
+ unread = []
477
+ for i, item in enumerate(unread):
478
+ if not (isinstance(item, dict) and item.get("scope") in UNREAD_SCOPES and isinstance(item.get("reason"), str)):
479
+ errors.append(f"unread[{i}] must have a scope in {UNREAD_SCOPES} and a reason")
480
+ elif "bbox" in item and not _is_box(item["bbox"]):
481
+ errors.append(f"unread[{i}].bbox must be [x0, y0, x1, y1]")
482
+ if not _strings(payload.get("flags")):
483
+ errors.append("flags must be a list of strings")
484
+ for i, entry in enumerate(payload.get("font_maps", [])):
485
+ if not (isinstance(entry, dict) and isinstance(entry.get("font"), str) and isinstance(entry.get("map"), dict)):
486
+ errors.append(f"font_maps[{i}] must have a font and a map")
487
+
488
+ # Provenance must be path-free. Block text and cells are content, not provenance.
489
+ provenance = {key: value for key, value in payload.items() if key != "blocks"}
490
+ provenance["blocks"] = [
491
+ {key: value for key, value in block.items() if key not in ("text", "cells", "lines", "label")}
492
+ for block in blocks
493
+ if isinstance(block, dict)
494
+ ]
495
+ for where in _path_like(provenance, "record"):
496
+ errors.append(f"{where} looks like a local filesystem path")
497
+ return errors