heocr 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- heocr/__init__.py +8 -0
- heocr/__main__.py +3 -0
- heocr/api.py +145 -0
- heocr/cli.py +120 -0
- heocr/contract.py +497 -0
- heocr/convert.py +207 -0
- heocr/docx/__init__.py +0 -0
- heocr/docx/numbering.py +250 -0
- heocr/docx/numformat.py +102 -0
- heocr/docx/reader.py +596 -0
- heocr/docx/styles.py +163 -0
- heocr/htmlread.py +908 -0
- heocr/identify.py +175 -0
- heocr/images.py +103 -0
- heocr/models.py +136 -0
- heocr/ocr/__init__.py +79 -0
- heocr/ocr/regions.py +198 -0
- heocr/ocr/tesseract.py +264 -0
- heocr/pdf/__init__.py +0 -0
- heocr/pdf/classify.py +241 -0
- heocr/pdf/columns.py +483 -0
- heocr/pdf/fontmap.py +221 -0
- heocr/pdf/frame.py +121 -0
- heocr/pdf/furniture.py +160 -0
- heocr/pdf/lines.py +516 -0
- heocr/pdf/numbers.py +108 -0
- heocr/pdf/ocrtables.py +163 -0
- heocr/pdf/reader.py +2518 -0
- heocr/pdf/tables.py +409 -0
- heocr/pdf/webprint.py +234 -0
- heocr/pptx.py +606 -0
- heocr/render.py +155 -0
- heocr/repairs.py +352 -0
- heocr/schemas/heocr_document.v1.schema.json +165 -0
- heocr/textflags.py +60 -0
- heocr/xlsx.py +685 -0
- heocr-0.1.0.dist-info/METADATA +172 -0
- heocr-0.1.0.dist-info/RECORD +44 -0
- heocr-0.1.0.dist-info/WHEEL +5 -0
- heocr-0.1.0.dist-info/entry_points.txt +2 -0
- heocr-0.1.0.dist-info/licenses/LICENSE +202 -0
- heocr-0.1.0.dist-info/licenses/NOTICE +8 -0
- heocr-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +88 -0
- heocr-0.1.0.dist-info/top_level.txt +1 -0
heocr/contract.py
ADDED
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
"""The ``heocr_document.v1`` record: what was read, by which route, and what was not.
|
|
2
|
+
|
|
3
|
+
The JSON record is the source of truth. ``Document.text`` and ``Document.markdown``
|
|
4
|
+
are views of its blocks. Records carry no local path and no file name.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
SCHEMA_ID = "heocr_document.v1"
|
|
17
|
+
|
|
18
|
+
BLOCK_KINDS = ("text", "heading", "list_item", "table", "figure")
|
|
19
|
+
# "navigation": site menus, breadcrumbs and "skip to content" links (D4f); kept, out of the body flow.
|
|
20
|
+
# "notes": a slide's speaker notes (D4e).
|
|
21
|
+
ROLES = ("body", "header", "footer", "footnote", "endnote", "comment", "textbox", "navigation", "notes")
|
|
22
|
+
SOURCES = ("text_layer", "office_xml", "converted_docx", "converted_xlsx", "converted_pptx", "html", "ocr", "none")
|
|
23
|
+
PAGE_CLASSES = ("T1", "T2", "T3", "T4", "T5", "T6", "empty")
|
|
24
|
+
DIRECTIONS = ("rtl", "ltr")
|
|
25
|
+
UNREAD_SCOPES = ("document", "page", "block")
|
|
26
|
+
|
|
27
|
+
# Strings that look like local filesystem locations. Provenance must stay path-free;
|
|
28
|
+
# document content (block text, cells) is content and is not path-checked.
|
|
29
|
+
LOCAL_PATH = re.compile(r"(\A(/|~/|[A-Za-z]:[\\/]|\\\\|file:))|(/(Users|home|tmp|private|var)/)")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def json_schema() -> dict[str, Any]:
|
|
33
|
+
"""``heocr_document.v1`` as a JSON Schema (draft 2020-12), shipped with the package.
|
|
34
|
+
|
|
35
|
+
It describes every field a record may carry; objects are open, as a later v1 may add fields
|
|
36
|
+
(docs/contract-v1.md). Rules that need the whole record, such as a page's index being its
|
|
37
|
+
position, are checked by :func:`validate`, which a record passes before it is returned.
|
|
38
|
+
"""
|
|
39
|
+
from importlib.resources import files
|
|
40
|
+
|
|
41
|
+
return json.loads(files("heocr").joinpath("schemas", "heocr_document.v1.schema.json").read_text(encoding="utf-8"))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ContractError(ValueError):
|
|
45
|
+
"""A record does not satisfy ``heocr_document.v1``."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class Cell:
|
|
50
|
+
text: str
|
|
51
|
+
colspan: int = 1
|
|
52
|
+
rowspan: int = 1
|
|
53
|
+
flags: list[str] = field(default_factory=list)
|
|
54
|
+
# OCR'd cells whose number was read by two models: each reading, {"model", "text", "confidence"}.
|
|
55
|
+
readings: list[dict[str, Any]] | None = None
|
|
56
|
+
_lines: Any = field(default=None, repr=False, compare=False) # PDF layer cells: [text, bbox] per line, for number checks
|
|
57
|
+
|
|
58
|
+
def to_dict(self) -> dict[str, Any]:
|
|
59
|
+
out: dict[str, Any] = {"text": self.text}
|
|
60
|
+
if self.colspan != 1:
|
|
61
|
+
out["colspan"] = self.colspan
|
|
62
|
+
if self.rowspan != 1:
|
|
63
|
+
out["rowspan"] = self.rowspan
|
|
64
|
+
if self.flags:
|
|
65
|
+
out["flags"] = list(self.flags)
|
|
66
|
+
if self.readings:
|
|
67
|
+
out["readings"] = [dict(r) for r in self.readings]
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class Line:
|
|
73
|
+
"""One physical line of a block, as read from a paged source."""
|
|
74
|
+
|
|
75
|
+
text: str
|
|
76
|
+
bbox: list[float]
|
|
77
|
+
direction: str = "rtl"
|
|
78
|
+
|
|
79
|
+
def to_dict(self) -> dict[str, Any]:
|
|
80
|
+
return {"text": self.text, "bbox": _round_box(self.bbox), "direction": self.direction}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class Block:
|
|
85
|
+
kind: str
|
|
86
|
+
text: str = ""
|
|
87
|
+
role: str = "body"
|
|
88
|
+
source: str = "none"
|
|
89
|
+
tool: str = ""
|
|
90
|
+
page: int | None = None
|
|
91
|
+
bbox: list[float] | None = None
|
|
92
|
+
level: int | None = None
|
|
93
|
+
label: str | None = None
|
|
94
|
+
note_id: str | None = None
|
|
95
|
+
direction: str | None = None
|
|
96
|
+
cells: list[list[Cell]] | None = None
|
|
97
|
+
lines: list[Line] | None = None
|
|
98
|
+
ocr: dict[str, Any] | None = None # for source "ocr": engine, settings, status "unverified", confidence
|
|
99
|
+
note_refs: list[dict[str, Any]] | None = None # footnote/endnote references: kind, id, offset in text
|
|
100
|
+
checks: dict[str, Any] | None = None # second readings: {"numbers": checked, "agreed": agreed}
|
|
101
|
+
repairs: list[dict[str, Any]] = field(default_factory=list)
|
|
102
|
+
flags: list[str] = field(default_factory=list)
|
|
103
|
+
half: str | None = None # a PDF spread split in two (D4c): "right" or "left", the page it belongs to
|
|
104
|
+
slide: int | None = None # PPTX (D4e): the slide's number as shown, from 1
|
|
105
|
+
_cell_repairs: Any = field(default=None, repr=False, compare=False) # PDF tables: repairs made in cells, logged by the reader
|
|
106
|
+
|
|
107
|
+
def to_dict(self, index: int) -> dict[str, Any]:
|
|
108
|
+
out: dict[str, Any] = {"id": f"b{index}", "kind": self.kind, "role": self.role}
|
|
109
|
+
if self.kind == "table":
|
|
110
|
+
out["cells"] = [[cell.to_dict() for cell in row] for row in (self.cells or [])]
|
|
111
|
+
else:
|
|
112
|
+
out["text"] = self.text
|
|
113
|
+
for key in ("level", "label", "note_id", "direction", "page", "half", "slide"):
|
|
114
|
+
value = getattr(self, key)
|
|
115
|
+
if value is not None:
|
|
116
|
+
out[key] = value
|
|
117
|
+
if self.bbox is not None:
|
|
118
|
+
out["bbox"] = _round_box(self.bbox)
|
|
119
|
+
if self.lines is not None:
|
|
120
|
+
out["lines"] = [line.to_dict() for line in self.lines]
|
|
121
|
+
if self.ocr is not None:
|
|
122
|
+
out["ocr"] = dict(self.ocr)
|
|
123
|
+
if self.note_refs:
|
|
124
|
+
out["note_refs"] = [dict(r) for r in self.note_refs]
|
|
125
|
+
if self.checks:
|
|
126
|
+
out["checks"] = dict(self.checks)
|
|
127
|
+
out["source"] = self.source
|
|
128
|
+
out["tool"] = self.tool
|
|
129
|
+
out["repairs"] = list(self.repairs)
|
|
130
|
+
out["flags"] = sorted(set(self.flags))
|
|
131
|
+
return out
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass
|
|
135
|
+
class Page:
|
|
136
|
+
index: int
|
|
137
|
+
width: float
|
|
138
|
+
height: float
|
|
139
|
+
page_class: str
|
|
140
|
+
route: str
|
|
141
|
+
signals: dict[str, Any] = field(default_factory=dict)
|
|
142
|
+
coverage: dict[str, Any] = field(default_factory=dict)
|
|
143
|
+
flags: list[str] = field(default_factory=list)
|
|
144
|
+
|
|
145
|
+
def to_dict(self) -> dict[str, Any]:
|
|
146
|
+
return {
|
|
147
|
+
"index": self.index,
|
|
148
|
+
"width": round(self.width, 2),
|
|
149
|
+
"height": round(self.height, 2),
|
|
150
|
+
"class": self.page_class,
|
|
151
|
+
"route": self.route,
|
|
152
|
+
"signals": dict(self.signals),
|
|
153
|
+
"coverage": dict(self.coverage),
|
|
154
|
+
"flags": sorted(set(self.flags)),
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# A cell spans at most this many columns or rows. Larger declared spans (hostile or broken input)
|
|
159
|
+
# are capped, and the cell is flagged span_capped; renderers clamp to it too.
|
|
160
|
+
MAX_SPAN = 1000
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def cap_span(value: int, flags: list[str]) -> int:
|
|
164
|
+
"""``value`` within 1..MAX_SPAN; a capped span flags the cell ``span_capped``."""
|
|
165
|
+
if value > MAX_SPAN:
|
|
166
|
+
flags.append("span_capped")
|
|
167
|
+
return MAX_SPAN
|
|
168
|
+
return max(value, 1)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# Pages (PDF pages, image frames) read at most per document. Every page may cost an OCR run of
|
|
172
|
+
# seconds, so a document of tens of thousands of pages, hostile or merely huge, would run for
|
|
173
|
+
# hours. The rest are recorded as unread, in one entry, never dropped silently.
|
|
174
|
+
MAX_PAGES = 2000
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def page_limit(doc: "Document", total: int) -> int:
|
|
178
|
+
"""The number of pages to read; records the rest as one unread entry and flags the document."""
|
|
179
|
+
limit = doc._max_pages
|
|
180
|
+
if total <= limit:
|
|
181
|
+
return total
|
|
182
|
+
doc.unread.append(Unread(scope="document", reason="page_limit", detail=f"pages {limit + 1}-{total} of {total} not read (max_pages={limit})"))
|
|
183
|
+
doc.flags.append("page_limit_reached")
|
|
184
|
+
return limit
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
_now = time.monotonic # the clock of the per-document time limit (tests replace it)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def time_up(doc: "Document") -> bool:
|
|
191
|
+
"""Whether the document's time limit (``extract(time_limit=...)``) has passed; never without one."""
|
|
192
|
+
return doc._deadline is not None and _now() >= doc._deadline
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def time_limit_reached(doc: "Document", detail: str, *, scope: str = "document", page: int | None = None) -> None:
|
|
196
|
+
"""Record what the time limit left unread, and flag the document once."""
|
|
197
|
+
doc.unread.append(Unread(scope=scope, reason="time_limit", page=page, detail=f"{detail}: the time limit of {doc._time_limit:g} s was reached"))
|
|
198
|
+
if "time_limit_reached" not in doc.flags:
|
|
199
|
+
doc.flags.append("time_limit_reached")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def pages_left_unread(doc: "Document", index: int, total: int) -> None:
|
|
203
|
+
"""Pages ``index``..``total - 1`` not read: the time limit was reached before page ``index``."""
|
|
204
|
+
span = f"page {total}" if index == total - 1 else f"pages {index + 1}-{total}"
|
|
205
|
+
time_limit_reached(doc, f"{span} of {total} not read")
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@dataclass
|
|
209
|
+
class Unread:
|
|
210
|
+
"""Something the pipeline saw but did not read, and why. Never dropped silently."""
|
|
211
|
+
|
|
212
|
+
scope: str
|
|
213
|
+
reason: str
|
|
214
|
+
page: int | None = None
|
|
215
|
+
detail: str | None = None
|
|
216
|
+
bbox: list[float] | None = None # the unread region, in the page's coordinates
|
|
217
|
+
|
|
218
|
+
def to_dict(self) -> dict[str, Any]:
|
|
219
|
+
out: dict[str, Any] = {"scope": self.scope, "reason": self.reason}
|
|
220
|
+
if self.page is not None:
|
|
221
|
+
out["page"] = self.page
|
|
222
|
+
if self.bbox is not None:
|
|
223
|
+
out["bbox"] = _round_box(self.bbox)
|
|
224
|
+
if self.detail is not None:
|
|
225
|
+
out["detail"] = self.detail
|
|
226
|
+
return out
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@dataclass
|
|
230
|
+
class Document:
|
|
231
|
+
sha256: str
|
|
232
|
+
size: int
|
|
233
|
+
format: str
|
|
234
|
+
format_evidence: str
|
|
235
|
+
page_count: int | None = None
|
|
236
|
+
pages: list[Page] = field(default_factory=list)
|
|
237
|
+
blocks: list[Block] = field(default_factory=list)
|
|
238
|
+
unread: list[Unread] = field(default_factory=list)
|
|
239
|
+
flags: list[str] = field(default_factory=list)
|
|
240
|
+
components: dict[str, str] = field(default_factory=dict)
|
|
241
|
+
options: dict[str, Any] = field(default_factory=lambda: {"repairs": True})
|
|
242
|
+
# Run-time only, never serialized: OCR settings and the engine found for this document.
|
|
243
|
+
_ocr_settings: dict[str, Any] | None = field(default=None, repr=False, compare=False)
|
|
244
|
+
_ocr_engine: Any = field(default=None, repr=False, compare=False)
|
|
245
|
+
_ocr_error: str | None = field(default=None, repr=False, compare=False)
|
|
246
|
+
_convert_settings: dict[str, Any] | None = field(default=None, repr=False, compare=False) # LibreOffice: cmd, timeout
|
|
247
|
+
_max_pages: int = field(default=MAX_PAGES, repr=False, compare=False) # pages read at most
|
|
248
|
+
_time_limit: float | None = field(default=None, repr=False, compare=False) # seconds for the whole document
|
|
249
|
+
_deadline: float | None = field(default=None, repr=False, compare=False) # _now() value when it runs out
|
|
250
|
+
_number_engine: Any = field(default=None, repr=False, compare=False) # the second reading of numbers (eng)
|
|
251
|
+
_number_error: str | None = field(default=None, repr=False, compare=False)
|
|
252
|
+
_font_codes: dict = field(default_factory=dict, repr=False, compare=False) # (font, letter) -> code
|
|
253
|
+
_shifted_fonts: dict = field(default_factory=dict, repr=False, compare=False) # font -> "latin1" | "pua" (cp1255_shifted_font)
|
|
254
|
+
_regions_read: dict = field(default_factory=dict, repr=False, compare=False) # image digest -> its first region OCR (repeated images)
|
|
255
|
+
_rtl: bool = field(default=True, repr=False, compare=False) # the document's direction (PDF tables)
|
|
256
|
+
# font_private_use_by_ocr: the maps learned for this document, with their evidence (no text).
|
|
257
|
+
font_maps: list[dict[str, Any]] = field(default_factory=list)
|
|
258
|
+
|
|
259
|
+
# -- views -------------------------------------------------------------
|
|
260
|
+
@property
|
|
261
|
+
def text(self) -> str:
|
|
262
|
+
from heocr.render import to_text
|
|
263
|
+
|
|
264
|
+
return to_text(self)
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def markdown(self) -> str:
|
|
268
|
+
from heocr.render import to_markdown
|
|
269
|
+
|
|
270
|
+
return to_markdown(self)
|
|
271
|
+
|
|
272
|
+
# -- record ------------------------------------------------------------
|
|
273
|
+
def to_dict(self) -> dict[str, Any]:
|
|
274
|
+
from heocr import __version__
|
|
275
|
+
|
|
276
|
+
document: dict[str, Any] = {
|
|
277
|
+
"sha256": self.sha256,
|
|
278
|
+
"bytes": self.size,
|
|
279
|
+
"format": self.format,
|
|
280
|
+
"format_evidence": self.format_evidence,
|
|
281
|
+
}
|
|
282
|
+
if self.page_count is not None:
|
|
283
|
+
document["page_count"] = self.page_count
|
|
284
|
+
payload = {
|
|
285
|
+
"schema": SCHEMA_ID,
|
|
286
|
+
"producer": {
|
|
287
|
+
"name": "heocr",
|
|
288
|
+
"version": __version__,
|
|
289
|
+
"components": dict(sorted(self.components.items())),
|
|
290
|
+
},
|
|
291
|
+
"document": document,
|
|
292
|
+
"options": dict(self.options),
|
|
293
|
+
"pages": [page.to_dict() for page in self.pages],
|
|
294
|
+
"blocks": [block.to_dict(i) for i, block in enumerate(self.blocks)],
|
|
295
|
+
"unread": [item.to_dict() for item in self.unread],
|
|
296
|
+
"flags": sorted(set(self.flags)),
|
|
297
|
+
}
|
|
298
|
+
if self.font_maps:
|
|
299
|
+
payload["font_maps"] = [dict(m) for m in self.font_maps]
|
|
300
|
+
errors = validate(payload)
|
|
301
|
+
if errors:
|
|
302
|
+
raise ContractError("; ".join(errors))
|
|
303
|
+
return payload
|
|
304
|
+
|
|
305
|
+
def to_json(self, indent: int | None = 2) -> str:
|
|
306
|
+
return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _round_box(box: list[float]) -> list[float]:
|
|
310
|
+
return [round(float(v), 2) for v in box]
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
# -- validation ----------------------------------------------------------------
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _path_like(value: Any, where: str):
|
|
317
|
+
if isinstance(value, str):
|
|
318
|
+
if LOCAL_PATH.search(value):
|
|
319
|
+
yield where
|
|
320
|
+
elif isinstance(value, dict):
|
|
321
|
+
for key, item in value.items():
|
|
322
|
+
yield from _path_like(item, f"{where}.{key}")
|
|
323
|
+
elif isinstance(value, list):
|
|
324
|
+
for index, item in enumerate(value):
|
|
325
|
+
yield from _path_like(item, f"{where}[{index}]")
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _is_box(value: Any) -> bool:
|
|
329
|
+
return (
|
|
330
|
+
isinstance(value, list)
|
|
331
|
+
and len(value) == 4
|
|
332
|
+
and all(isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) for v in value)
|
|
333
|
+
and value[2] >= value[0]
|
|
334
|
+
and value[3] >= value[1]
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _strings(value: Any) -> bool:
|
|
339
|
+
return isinstance(value, list) and all(isinstance(v, str) for v in value)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def validate(payload: Any) -> list[str]:
|
|
343
|
+
"""Return the list of contract violations in a ``heocr_document.v1`` payload."""
|
|
344
|
+
errors: list[str] = []
|
|
345
|
+
if not isinstance(payload, dict):
|
|
346
|
+
return ["record is not an object"]
|
|
347
|
+
if payload.get("schema") != SCHEMA_ID:
|
|
348
|
+
errors.append(f"schema must be {SCHEMA_ID!r}")
|
|
349
|
+
producer = payload.get("producer")
|
|
350
|
+
if not isinstance(producer, dict) or producer.get("name") != "heocr" or not isinstance(
|
|
351
|
+
producer.get("version"), str
|
|
352
|
+
):
|
|
353
|
+
errors.append("producer must name heocr and its version")
|
|
354
|
+
document = payload.get("document")
|
|
355
|
+
if not isinstance(document, dict):
|
|
356
|
+
errors.append("document must be an object")
|
|
357
|
+
document = {}
|
|
358
|
+
sha = document.get("sha256")
|
|
359
|
+
if not (isinstance(sha, str) and re.fullmatch(r"[0-9a-f]{64}", sha)):
|
|
360
|
+
errors.append("document.sha256 must be 64 lowercase hex digits")
|
|
361
|
+
if not isinstance(document.get("bytes"), int) or document.get("bytes", -1) < 0:
|
|
362
|
+
errors.append("document.bytes must be a non-negative integer")
|
|
363
|
+
for key in ("format", "format_evidence"):
|
|
364
|
+
if not isinstance(document.get(key), str) or not document.get(key):
|
|
365
|
+
errors.append(f"document.{key} must be a non-empty string")
|
|
366
|
+
for key in ("name", "path", "filename", "file"):
|
|
367
|
+
if key in document:
|
|
368
|
+
errors.append(f"document.{key} is not allowed: records carry no file names or paths")
|
|
369
|
+
|
|
370
|
+
options = payload.get("options", {})
|
|
371
|
+
if not isinstance(options, dict) or not isinstance(options.get("repairs", True), bool):
|
|
372
|
+
errors.append("options must be an object, with a boolean repairs")
|
|
373
|
+
elif "ocr" in options and not (options["ocr"] is False or isinstance(options["ocr"], dict)):
|
|
374
|
+
errors.append("options.ocr must be false or an object")
|
|
375
|
+
elif "max_pages" in options and not (isinstance(options["max_pages"], int) and not isinstance(options["max_pages"], bool) and options["max_pages"] > 0):
|
|
376
|
+
errors.append("options.max_pages must be a positive integer")
|
|
377
|
+
elif "time_limit" in options and not (
|
|
378
|
+
isinstance(options["time_limit"], (int, float)) and not isinstance(options["time_limit"], bool) and math.isfinite(options["time_limit"]) and options["time_limit"] > 0
|
|
379
|
+
):
|
|
380
|
+
errors.append("options.time_limit must be a positive number of seconds")
|
|
381
|
+
elif "convert" in options and not (options["convert"] is False or isinstance(options["convert"], dict)):
|
|
382
|
+
errors.append("options.convert must be false or an object")
|
|
383
|
+
elif options.get("check_numbers", False) not in (False, "suspect", "tables", "all"):
|
|
384
|
+
errors.append("options.check_numbers must be 'suspect', 'tables', 'all' or false")
|
|
385
|
+
|
|
386
|
+
pages = payload.get("pages")
|
|
387
|
+
if not isinstance(pages, list):
|
|
388
|
+
errors.append("pages must be a list")
|
|
389
|
+
pages = []
|
|
390
|
+
for i, page in enumerate(pages):
|
|
391
|
+
where = f"pages[{i}]"
|
|
392
|
+
if not isinstance(page, dict):
|
|
393
|
+
errors.append(f"{where} is not an object")
|
|
394
|
+
continue
|
|
395
|
+
if page.get("index") != i:
|
|
396
|
+
errors.append(f"{where}.index must be {i}")
|
|
397
|
+
if page.get("class") not in PAGE_CLASSES:
|
|
398
|
+
errors.append(f"{where}.class must be one of {PAGE_CLASSES}")
|
|
399
|
+
if not isinstance(page.get("route"), str):
|
|
400
|
+
errors.append(f"{where}.route must be a string")
|
|
401
|
+
for key in ("signals", "coverage"):
|
|
402
|
+
if not isinstance(page.get(key), dict):
|
|
403
|
+
errors.append(f"{where}.{key} must be an object")
|
|
404
|
+
if not _strings(page.get("flags")):
|
|
405
|
+
errors.append(f"{where}.flags must be a list of strings")
|
|
406
|
+
|
|
407
|
+
blocks = payload.get("blocks")
|
|
408
|
+
if not isinstance(blocks, list):
|
|
409
|
+
errors.append("blocks must be a list")
|
|
410
|
+
blocks = []
|
|
411
|
+
for i, block in enumerate(blocks):
|
|
412
|
+
where = f"blocks[{i}]"
|
|
413
|
+
if not isinstance(block, dict):
|
|
414
|
+
errors.append(f"{where} is not an object")
|
|
415
|
+
continue
|
|
416
|
+
if block.get("id") != f"b{i}":
|
|
417
|
+
errors.append(f"{where}.id must be 'b{i}'")
|
|
418
|
+
if block.get("kind") not in BLOCK_KINDS:
|
|
419
|
+
errors.append(f"{where}.kind must be one of {BLOCK_KINDS}")
|
|
420
|
+
if block.get("role") not in ROLES:
|
|
421
|
+
errors.append(f"{where}.role must be one of {ROLES}")
|
|
422
|
+
if block.get("source") not in SOURCES:
|
|
423
|
+
errors.append(f"{where}.source must be one of {SOURCES}")
|
|
424
|
+
if not isinstance(block.get("tool"), str):
|
|
425
|
+
errors.append(f"{where}.tool must be a string")
|
|
426
|
+
if block.get("kind") == "table":
|
|
427
|
+
cells = block.get("cells")
|
|
428
|
+
if not isinstance(cells, list) or not all(
|
|
429
|
+
isinstance(row, list) and all(isinstance(c, dict) and isinstance(c.get("text"), str) for c in row)
|
|
430
|
+
for row in cells
|
|
431
|
+
):
|
|
432
|
+
errors.append(f"{where}.cells must be rows of cells with text")
|
|
433
|
+
for row in cells if isinstance(cells, list) else []:
|
|
434
|
+
for c in row if isinstance(row, list) else []:
|
|
435
|
+
readings = c.get("readings") if isinstance(c, dict) else None
|
|
436
|
+
if readings is not None and not (
|
|
437
|
+
isinstance(readings, list)
|
|
438
|
+
and all(isinstance(r, dict) and isinstance(r.get("model"), str) and isinstance(r.get("text"), str) for r in readings)
|
|
439
|
+
):
|
|
440
|
+
errors.append(f"{where}.cells readings must be objects with a model and a text")
|
|
441
|
+
elif not isinstance(block.get("text"), str):
|
|
442
|
+
errors.append(f"{where}.text must be a string")
|
|
443
|
+
if "slide" in block and not (isinstance(block["slide"], int) and not isinstance(block["slide"], bool) and block["slide"] >= 1):
|
|
444
|
+
errors.append(f"{where}.slide must be a positive integer")
|
|
445
|
+
if "half" in block and block["half"] not in ("right", "left"):
|
|
446
|
+
errors.append(f"{where}.half must be 'right' or 'left'")
|
|
447
|
+
if "direction" in block and block["direction"] not in DIRECTIONS:
|
|
448
|
+
errors.append(f"{where}.direction must be one of {DIRECTIONS}")
|
|
449
|
+
if "page" in block and not (isinstance(block["page"], int) and 0 <= block["page"] < max(len(pages), 1)):
|
|
450
|
+
errors.append(f"{where}.page must index pages")
|
|
451
|
+
if "bbox" in block and not _is_box(block["bbox"]):
|
|
452
|
+
errors.append(f"{where}.bbox must be [x0, y0, x1, y1]")
|
|
453
|
+
for line in block.get("lines", []) or []:
|
|
454
|
+
if not (isinstance(line, dict) and isinstance(line.get("text"), str) and _is_box(line.get("bbox"))):
|
|
455
|
+
errors.append(f"{where}.lines must have text and bbox")
|
|
456
|
+
break
|
|
457
|
+
if block.get("source") == "ocr" and not (isinstance(block.get("ocr"), dict) and block["ocr"].get("status") == "unverified"):
|
|
458
|
+
errors.append(f"{where}.ocr must be an object with status 'unverified' for OCR text")
|
|
459
|
+
for ref in block.get("note_refs", []) or []:
|
|
460
|
+
if not (
|
|
461
|
+
isinstance(ref, dict)
|
|
462
|
+
and ref.get("kind") in ("footnote", "endnote")
|
|
463
|
+
and isinstance(ref.get("offset"), int)
|
|
464
|
+
and 0 <= ref["offset"] <= len(block.get("text", ""))
|
|
465
|
+
):
|
|
466
|
+
errors.append(f"{where}.note_refs must have a kind and an offset inside the text")
|
|
467
|
+
break
|
|
468
|
+
if not isinstance(block.get("repairs"), list):
|
|
469
|
+
errors.append(f"{where}.repairs must be a list")
|
|
470
|
+
if not _strings(block.get("flags")):
|
|
471
|
+
errors.append(f"{where}.flags must be a list of strings")
|
|
472
|
+
|
|
473
|
+
unread = payload.get("unread")
|
|
474
|
+
if not isinstance(unread, list):
|
|
475
|
+
errors.append("unread must be a list")
|
|
476
|
+
unread = []
|
|
477
|
+
for i, item in enumerate(unread):
|
|
478
|
+
if not (isinstance(item, dict) and item.get("scope") in UNREAD_SCOPES and isinstance(item.get("reason"), str)):
|
|
479
|
+
errors.append(f"unread[{i}] must have a scope in {UNREAD_SCOPES} and a reason")
|
|
480
|
+
elif "bbox" in item and not _is_box(item["bbox"]):
|
|
481
|
+
errors.append(f"unread[{i}].bbox must be [x0, y0, x1, y1]")
|
|
482
|
+
if not _strings(payload.get("flags")):
|
|
483
|
+
errors.append("flags must be a list of strings")
|
|
484
|
+
for i, entry in enumerate(payload.get("font_maps", [])):
|
|
485
|
+
if not (isinstance(entry, dict) and isinstance(entry.get("font"), str) and isinstance(entry.get("map"), dict)):
|
|
486
|
+
errors.append(f"font_maps[{i}] must have a font and a map")
|
|
487
|
+
|
|
488
|
+
# Provenance must be path-free. Block text and cells are content, not provenance.
|
|
489
|
+
provenance = {key: value for key, value in payload.items() if key != "blocks"}
|
|
490
|
+
provenance["blocks"] = [
|
|
491
|
+
{key: value for key, value in block.items() if key not in ("text", "cells", "lines", "label")}
|
|
492
|
+
for block in blocks
|
|
493
|
+
if isinstance(block, dict)
|
|
494
|
+
]
|
|
495
|
+
for where in _path_like(provenance, "record"):
|
|
496
|
+
errors.append(f"{where} looks like a local filesystem path")
|
|
497
|
+
return errors
|