sdb-reader 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdb_reader/__init__.py +157 -0
- sdb_reader/ask.py +437 -0
- sdb_reader/ask_record.schema.json +388 -0
- sdb_reader/cli.py +1079 -0
- sdb_reader/clp_statements_de.json +997 -0
- sdb_reader/compliance.py +546 -0
- sdb_reader/current.py +134 -0
- sdb_reader/diff.py +520 -0
- sdb_reader/embedded.py +250 -0
- sdb_reader/extraction.py +506 -0
- sdb_reader/layout.py +894 -0
- sdb_reader/line_breaks.py +40 -0
- sdb_reader/messages.py +394 -0
- sdb_reader/messages_data/de.toml +146 -0
- sdb_reader/messages_data/en.toml +141 -0
- sdb_reader/model.py +321 -0
- sdb_reader/parser.py +4601 -0
- sdb_reader/pictograms.py +374 -0
- sdb_reader/presets.py +110 -0
- sdb_reader/presets.toml +23 -0
- sdb_reader/py.typed +0 -0
- sdb_reader/read_record.schema.json +792 -0
- sdb_reader/reading.py +94 -0
- sdb_reader/scan.py +265 -0
- sdb_reader/schema.py +96 -0
- sdb_reader/serialise.py +463 -0
- sdb_reader/show.py +370 -0
- sdb_reader/statements.py +1011 -0
- sdb_reader/verification.py +322 -0
- sdb_reader/versions.py +31 -0
- sdb_reader-0.9.0.dist-info/METADATA +117 -0
- sdb_reader-0.9.0.dist-info/RECORD +37 -0
- sdb_reader-0.9.0.dist-info/WHEEL +5 -0
- sdb_reader-0.9.0.dist-info/entry_points.txt +2 -0
- sdb_reader-0.9.0.dist-info/licenses/LICENSE +661 -0
- sdb_reader-0.9.0.dist-info/licenses/NOTICE +40 -0
- sdb_reader-0.9.0.dist-info/top_level.txt +1 -0
sdb_reader/serialise.py
ADDED
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
"""Readings and texts as JSON: what the command prints and what a caller stores."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from datetime import date
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .compliance import requires_sheet, sheet_findings
|
|
13
|
+
from .embedded import EmbeddedData, differences
|
|
14
|
+
from .extraction import ExtractionResult
|
|
15
|
+
from .messages import Catalog, Message, shipped_catalog
|
|
16
|
+
from .reading import SheetReading
|
|
17
|
+
from .schema import (
|
|
18
|
+
ContactInfo,
|
|
19
|
+
HazardStatements,
|
|
20
|
+
SafetyDataSheet,
|
|
21
|
+
Section,
|
|
22
|
+
StatedClassification,
|
|
23
|
+
)
|
|
24
|
+
from .statements import label_statements
|
|
25
|
+
from .verification import DocumentKind, Verification
|
|
26
|
+
from .versions import extraction_stage
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _content_hash(extraction: ExtractionResult) -> str | None:
|
|
30
|
+
"""Fingerprint the document by its extracted text.
|
|
31
|
+
|
|
32
|
+
Hashing the PDF bytes does not work: every write gets a fresh /ID, so the
|
|
33
|
+
same document exported three times yields three different byte hashes at
|
|
34
|
+
identical length. The extracted text is the same across all three.
|
|
35
|
+
|
|
36
|
+
Returns None for an empty extraction -- a document we could not read is not
|
|
37
|
+
evidence of being the same as another one we could not read.
|
|
38
|
+
"""
|
|
39
|
+
return _text_hash(extraction.text)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _text_hash(text: str | None) -> str | None:
|
|
43
|
+
text = (text or "").strip()
|
|
44
|
+
if not text:
|
|
45
|
+
return None
|
|
46
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def sheet_to_dict(
|
|
50
|
+
sheet: SafetyDataSheet, extraction: ExtractionResult
|
|
51
|
+
) -> dict[str, Any]:
|
|
52
|
+
"""The parsed sheet as JSON: the ``sheet`` part of what the command prints.
|
|
53
|
+
|
|
54
|
+
*extraction* is the text the sheet was parsed from; its fingerprint goes
|
|
55
|
+
into the record. Every value stands once (SDW-18).
|
|
56
|
+
"""
|
|
57
|
+
return _sheet_fields(sheet, _content_hash(extraction))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _sheet_fields(sheet: SafetyDataSheet, content_hash: str | None) -> dict[str, Any]:
|
|
61
|
+
return {
|
|
62
|
+
# The document's own version, as printed in the sheet.
|
|
63
|
+
"version": sheet.version,
|
|
64
|
+
# Identifies the document by content, so copies filed under several
|
|
65
|
+
# names can be recognised as one. See _content_hash.
|
|
66
|
+
"content_hash": content_hash,
|
|
67
|
+
"product_identifier": sheet.product_identifier,
|
|
68
|
+
"issue_date": sheet.issue_date.isoformat() if sheet.issue_date else None,
|
|
69
|
+
"revision_date": sheet.revision_date.isoformat()
|
|
70
|
+
if sheet.revision_date
|
|
71
|
+
else None,
|
|
72
|
+
"language": sheet.language,
|
|
73
|
+
# The countries the sheet is issued for (SDB-69), what section 2 says
|
|
74
|
+
# about the classification (SDB-75), and whether the product needs a
|
|
75
|
+
# sheet by it (SDB-76): a sheet for one that does not is voluntary.
|
|
76
|
+
"countries": list(sheet.countries),
|
|
77
|
+
"stated_classification": sheet.stated_classification.value,
|
|
78
|
+
# Whether the sheet calls itself voluntary or not prescribed, and its
|
|
79
|
+
# product an article beside that (SDB-142): an article needs no sheet
|
|
80
|
+
# whatever the label prints, an unclassified mixture none either.
|
|
81
|
+
"stated_voluntary": sheet.stated_voluntary,
|
|
82
|
+
"stated_article": sheet.stated_article,
|
|
83
|
+
"sheet_required": requires_sheet(sheet),
|
|
84
|
+
"hazard_statements": {
|
|
85
|
+
"hazard_codes": sheet.hazard_statements.hazard_codes,
|
|
86
|
+
"precautionary_codes": sheet.hazard_statements.precautionary_codes,
|
|
87
|
+
"supplemental_codes": sheet.hazard_statements.supplemental_codes,
|
|
88
|
+
"signal_word": sheet.hazard_statements.signal_word,
|
|
89
|
+
"pictograms": sheet.hazard_statements.pictograms,
|
|
90
|
+
"hazard_text": sheet.hazard_statements.hazard_text,
|
|
91
|
+
"precautionary_text": sheet.hazard_statements.precautionary_text,
|
|
92
|
+
"supplemental_text": sheet.hazard_statements.supplemental_text,
|
|
93
|
+
},
|
|
94
|
+
"contact_info": {
|
|
95
|
+
"supplier_name": sheet.contact_info.supplier_name,
|
|
96
|
+
"supplier_address": sheet.contact_info.supplier_address,
|
|
97
|
+
"supplier_phone": sheet.contact_info.supplier_phone,
|
|
98
|
+
"emergency_phone": sheet.contact_info.emergency_phone,
|
|
99
|
+
"supplier_email": sheet.contact_info.supplier_email,
|
|
100
|
+
"supplier_website": sheet.contact_info.supplier_website,
|
|
101
|
+
"information_email": sheet.contact_info.information_email,
|
|
102
|
+
},
|
|
103
|
+
"section_text": {
|
|
104
|
+
section.value: text for section, text in sheet.section_text.items()
|
|
105
|
+
},
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def sheet_from_dict(stored: Mapping[str, Any]) -> SafetyDataSheet:
|
|
110
|
+
"""The sheet ``sheet_to_dict`` stored, to judge it again (BIB-21).
|
|
111
|
+
|
|
112
|
+
Reads what this version writes: the ``sheet`` part of a record of
|
|
113
|
+
``read``, or of one of its ``parts``. ``content_hash`` and
|
|
114
|
+
``sheet_required`` are not part of a sheet and are left out;
|
|
115
|
+
``requires_sheet`` gives the second again. A key this version does not
|
|
116
|
+
know is ignored.
|
|
117
|
+
|
|
118
|
+
A record that lacks a key, or holds a value of another kind, was stored
|
|
119
|
+
by another version and is refused with ``ValueError``: make the reading
|
|
120
|
+
again from the stored text (``read_extraction``) instead. Whether a
|
|
121
|
+
record still holds is the caller's to compare with ``PARSER_VERSION``.
|
|
122
|
+
|
|
123
|
+
>>> stored = {
|
|
124
|
+
... "version": "2.0", "product_identifier": "Musterfarbe",
|
|
125
|
+
... "issue_date": None, "revision_date": "2024-03-01", "language": "de",
|
|
126
|
+
... "countries": ["DE"], "stated_classification": "classified",
|
|
127
|
+
... "stated_voluntary": False, "stated_article": False,
|
|
128
|
+
... "hazard_statements": {
|
|
129
|
+
... "hazard_codes": ["H315"], "precautionary_codes": [],
|
|
130
|
+
... "supplemental_codes": [], "signal_word": "Achtung",
|
|
131
|
+
... "pictograms": ["GHS07"], "hazard_text": None,
|
|
132
|
+
... "precautionary_text": None, "supplemental_text": None,
|
|
133
|
+
... },
|
|
134
|
+
... "contact_info": {
|
|
135
|
+
... "supplier_name": "Nordwerk GmbH", "supplier_address": None,
|
|
136
|
+
... "supplier_phone": None, "emergency_phone": None,
|
|
137
|
+
... "supplier_email": None, "supplier_website": None,
|
|
138
|
+
... "information_email": None,
|
|
139
|
+
... },
|
|
140
|
+
... "section_text": {"section_2": "Signalwort: Achtung"},
|
|
141
|
+
... }
|
|
142
|
+
>>> sheet = sheet_from_dict(stored)
|
|
143
|
+
>>> sheet.revision_date, sheet.hazard_statements.signal_word
|
|
144
|
+
(datetime.date(2024, 3, 1), 'Achtung')
|
|
145
|
+
>>> requires_sheet(sheet)
|
|
146
|
+
True
|
|
147
|
+
"""
|
|
148
|
+
try:
|
|
149
|
+
label = stored["hazard_statements"]
|
|
150
|
+
contact = stored["contact_info"]
|
|
151
|
+
return SafetyDataSheet(
|
|
152
|
+
product_identifier=_optional_text(stored["product_identifier"]),
|
|
153
|
+
issue_date=_stored_date(stored["issue_date"]),
|
|
154
|
+
revision_date=_stored_date(stored["revision_date"]),
|
|
155
|
+
version=_optional_text(stored["version"]),
|
|
156
|
+
language=_text(stored["language"]),
|
|
157
|
+
hazard_statements=HazardStatements(
|
|
158
|
+
hazard_codes=_texts(label["hazard_codes"]),
|
|
159
|
+
precautionary_codes=_texts(label["precautionary_codes"]),
|
|
160
|
+
supplemental_codes=_texts(label["supplemental_codes"]),
|
|
161
|
+
signal_word=_optional_text(label["signal_word"]),
|
|
162
|
+
pictograms=_texts(label["pictograms"]),
|
|
163
|
+
hazard_text=_optional_text(label["hazard_text"]),
|
|
164
|
+
precautionary_text=_optional_text(label["precautionary_text"]),
|
|
165
|
+
supplemental_text=_optional_text(label["supplemental_text"]),
|
|
166
|
+
),
|
|
167
|
+
contact_info=ContactInfo(
|
|
168
|
+
supplier_name=_optional_text(contact["supplier_name"]),
|
|
169
|
+
supplier_address=_optional_text(contact["supplier_address"]),
|
|
170
|
+
supplier_phone=_optional_text(contact["supplier_phone"]),
|
|
171
|
+
emergency_phone=_optional_text(contact["emergency_phone"]),
|
|
172
|
+
supplier_email=_optional_text(contact["supplier_email"]),
|
|
173
|
+
supplier_website=_optional_text(contact["supplier_website"]),
|
|
174
|
+
information_email=_optional_text(contact["information_email"]),
|
|
175
|
+
),
|
|
176
|
+
section_text={
|
|
177
|
+
Section(name): _text(text)
|
|
178
|
+
for name, text in stored["section_text"].items()
|
|
179
|
+
},
|
|
180
|
+
countries=_texts(stored["countries"]),
|
|
181
|
+
stated_classification=StatedClassification(stored["stated_classification"]),
|
|
182
|
+
stated_voluntary=_flag(stored["stated_voluntary"]),
|
|
183
|
+
stated_article=_flag(stored["stated_article"]),
|
|
184
|
+
)
|
|
185
|
+
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
|
186
|
+
raise ValueError(
|
|
187
|
+
"not a sheet as this version stores it "
|
|
188
|
+
f"({type(exc).__name__}: {exc}); read the stored text again"
|
|
189
|
+
) from exc
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _text(value: Any) -> str:
|
|
193
|
+
if not isinstance(value, str):
|
|
194
|
+
raise TypeError(f"expected text, found {type(value).__name__}")
|
|
195
|
+
return value
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _optional_text(value: Any) -> str | None:
|
|
199
|
+
return None if value is None else _text(value)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _texts(value: Any) -> list[str]:
|
|
203
|
+
if not isinstance(value, list):
|
|
204
|
+
raise TypeError(f"expected a list, found {type(value).__name__}")
|
|
205
|
+
return [_text(item) for item in value]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _flag(value: Any) -> bool:
|
|
209
|
+
if not isinstance(value, bool):
|
|
210
|
+
raise TypeError(f"expected true or false, found {type(value).__name__}")
|
|
211
|
+
return value
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _stored_date(value: Any) -> date | None:
|
|
215
|
+
return None if value is None else date.fromisoformat(_text(value))
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _shown(messages: list[Message], catalog: Catalog | None) -> list[dict[str, Any]]:
|
|
219
|
+
"""Each message as code, values and sentence (SDW-17); German unless asked."""
|
|
220
|
+
chosen = catalog or shipped_catalog()
|
|
221
|
+
return [chosen.to_dict(message) for message in messages]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def verification_to_dict(
|
|
225
|
+
verification: Verification, *, catalog: Catalog | None = None
|
|
226
|
+
) -> dict[str, Any]:
|
|
227
|
+
"""The verdict as a record stores it; *catalog* words its messages."""
|
|
228
|
+
return {
|
|
229
|
+
"kind": verification.kind.value,
|
|
230
|
+
"prints_statement_codes": verification.prints_statement_codes,
|
|
231
|
+
"warnings": _shown(verification.warnings, catalog),
|
|
232
|
+
"errors": _shown(verification.errors, catalog),
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def reading_to_dict(
|
|
237
|
+
reading: SheetReading,
|
|
238
|
+
*,
|
|
239
|
+
include_raw_text: bool = False,
|
|
240
|
+
embedded: EmbeddedData | None = None,
|
|
241
|
+
catalog: Catalog | None = None,
|
|
242
|
+
today: date | None = None,
|
|
243
|
+
) -> dict[str, Any]:
|
|
244
|
+
"""What reading one file found, as ``sdb-reader read`` prints it.
|
|
245
|
+
|
|
246
|
+
*today* is the day the findings take as the present (``sheet_findings``);
|
|
247
|
+
the command passes the machine's date. Without it no date is questioned,
|
|
248
|
+
and the record is the same on any day.
|
|
249
|
+
|
|
250
|
+
Without the file itself: the command puts its path, name and size in
|
|
251
|
+
front. *embedded* is what ``read_embedded`` found in the PDF; with it the
|
|
252
|
+
record lists where the embedded data and the sheet as read differ (SDW-11).
|
|
253
|
+
Every message stands as its code, its values and a sentence from
|
|
254
|
+
*catalog*: the shipped German one unless another is given (SDW-17).
|
|
255
|
+
"""
|
|
256
|
+
# Every record has the same keys for its kind (SDW-19): the three lists
|
|
257
|
+
# are there for every file, empty where there is nothing to say.
|
|
258
|
+
record: dict[str, Any] = {
|
|
259
|
+
"verification": verification_to_dict(reading.verification, catalog=catalog),
|
|
260
|
+
"warnings": _shown(reading.warnings, catalog),
|
|
261
|
+
"errors": _shown(reading.errors, catalog),
|
|
262
|
+
"findings": [],
|
|
263
|
+
}
|
|
264
|
+
if reading.extraction is not None:
|
|
265
|
+
record["extraction"] = _extraction_to_dict(
|
|
266
|
+
reading.extraction, include_raw_text=include_raw_text, catalog=catalog
|
|
267
|
+
)
|
|
268
|
+
if reading.parser_result is not None:
|
|
269
|
+
record["sheet"] = sheet_to_dict(
|
|
270
|
+
reading.parser_result.sheet, reading.extraction
|
|
271
|
+
)
|
|
272
|
+
record["parser"] = {
|
|
273
|
+
"warnings": _shown(reading.parser_result.warnings, catalog)
|
|
274
|
+
}
|
|
275
|
+
sheet = reading.parser_result.sheet
|
|
276
|
+
record["statements"] = _statements(sheet)
|
|
277
|
+
# How many sheets the file holds, and each one read on its own
|
|
278
|
+
# where it holds several (SDW-32); the keys above are the file's.
|
|
279
|
+
parts = reading.parser_result.parts
|
|
280
|
+
record["sheet_count"] = len(parts) or 1
|
|
281
|
+
# A cover page in front of the sheets, and the word on it that
|
|
282
|
+
# names a display (SDW-35); it is read for nothing else.
|
|
283
|
+
record["cover_page"] = reading.parser_result.cover_page
|
|
284
|
+
record["display_term"] = reading.parser_result.display_term
|
|
285
|
+
if parts:
|
|
286
|
+
record["parts"] = [
|
|
287
|
+
{
|
|
288
|
+
"sheet": _sheet_fields(part.sheet, _text_hash(part.text)),
|
|
289
|
+
"statements": _statements(part.sheet),
|
|
290
|
+
"warnings": _shown(part.warnings, catalog),
|
|
291
|
+
}
|
|
292
|
+
for part in parts
|
|
293
|
+
]
|
|
294
|
+
record["findings"] = _shown(
|
|
295
|
+
sheet_findings(
|
|
296
|
+
sheet, reading.parser_result.warnings, today=today, parts=parts
|
|
297
|
+
),
|
|
298
|
+
catalog,
|
|
299
|
+
)
|
|
300
|
+
if reading.verification.kind is DocumentKind.FOREIGN_SAFETY_DATA_SHEET:
|
|
301
|
+
# One file says nothing about the product's other sheets (SDW-8).
|
|
302
|
+
record["findings"] = _shown([Message("not-german")], catalog)
|
|
303
|
+
if embedded is not None:
|
|
304
|
+
record["embedded_data"] = _embedded_to_dict(embedded, reading, catalog)
|
|
305
|
+
return record
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _statements(sheet: SafetyDataSheet) -> list[dict[str, Any]]:
|
|
309
|
+
"""The label's statements, each with both wordings (SDW-5)."""
|
|
310
|
+
return [
|
|
311
|
+
{"code": s.code, "sheet": s.sheet, "regulation": s.regulation}
|
|
312
|
+
for s in label_statements(sheet.hazard_statements)
|
|
313
|
+
]
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _embedded_to_dict(
|
|
317
|
+
embedded: EmbeddedData, reading: SheetReading, catalog: Catalog | None
|
|
318
|
+
) -> dict[str, Any]:
|
|
319
|
+
"""The embedded eSDScom file and where the sheet as read differs (SDW-11).
|
|
320
|
+
|
|
321
|
+
``differences`` is always there; ``compared`` says whether there was
|
|
322
|
+
anything to compare: a readable embedded file and a German sheet (SDW-18).
|
|
323
|
+
"""
|
|
324
|
+
payload: dict[str, Any] = {
|
|
325
|
+
"format": "eSDScom",
|
|
326
|
+
"filename": embedded.filename,
|
|
327
|
+
"compared": False,
|
|
328
|
+
"differences": [],
|
|
329
|
+
}
|
|
330
|
+
if embedded.sheet is None:
|
|
331
|
+
if embedded.unreadable is not None:
|
|
332
|
+
payload["unreadable"] = _shown([embedded.unreadable], catalog)[0]
|
|
333
|
+
return payload
|
|
334
|
+
payload["version"] = embedded.sheet.format_version
|
|
335
|
+
payload["system"] = embedded.sheet.authoring_system
|
|
336
|
+
if reading.parser_result is not None:
|
|
337
|
+
payload["compared"] = True
|
|
338
|
+
payload["differences"] = [
|
|
339
|
+
{"field": d.field, "embedded": d.embedded, "read": d.read}
|
|
340
|
+
for d in differences(embedded.sheet, reading.parser_result.sheet)
|
|
341
|
+
]
|
|
342
|
+
return payload
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _extraction_to_dict(
|
|
346
|
+
extraction: ExtractionResult, *, include_raw_text: bool, catalog: Catalog | None
|
|
347
|
+
) -> dict[str, Any]:
|
|
348
|
+
payload: dict[str, Any] = {
|
|
349
|
+
# The stage the text comes from: a stored text names its own, a text
|
|
350
|
+
# read from the PDF just now is of this library's (SDW-27).
|
|
351
|
+
"stage": extraction.metadata.get("stage") or extraction_stage(),
|
|
352
|
+
"extractor": extraction.extractor,
|
|
353
|
+
"pages_sampled": extraction.pages_sampled,
|
|
354
|
+
"warnings": _shown(extraction.warnings, catalog),
|
|
355
|
+
"character_count": extraction.metadata.get("character_count"),
|
|
356
|
+
}
|
|
357
|
+
if extraction.metadata.get("tesseract"):
|
|
358
|
+
payload["tesseract"] = extraction.metadata["tesseract"]
|
|
359
|
+
if include_raw_text:
|
|
360
|
+
payload["raw_text"] = extraction.raw_text
|
|
361
|
+
return payload
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def extraction_to_dict(extraction: ExtractionResult) -> dict[str, Any]:
|
|
365
|
+
"""The text read from a PDF, in a form to store as JSON (BIB-3).
|
|
366
|
+
|
|
367
|
+
Holds what a reading needs and names the stage it comes from. The text
|
|
368
|
+
before tidying and the single pages are not kept.
|
|
369
|
+
|
|
370
|
+
>>> text = "Sicherheitsdatenblatt\\nABSCHNITT 1: Bezeichnung des Stoffs\\n"
|
|
371
|
+
>>> stored = extraction_to_dict(
|
|
372
|
+
... extraction_from_dict(
|
|
373
|
+
... {"text": text, "extractor": "pymupdf", "pages_sampled": 1, "warnings": []}
|
|
374
|
+
... )
|
|
375
|
+
... )
|
|
376
|
+
>>> sorted(stored)
|
|
377
|
+
['extractor', 'pages_sampled', 'stage', 'text', 'warnings']
|
|
378
|
+
>>> stored["stage"] == extraction_stage()
|
|
379
|
+
True
|
|
380
|
+
"""
|
|
381
|
+
return {
|
|
382
|
+
"stage": extraction_stage(),
|
|
383
|
+
"text": extraction.text,
|
|
384
|
+
"extractor": extraction.extractor,
|
|
385
|
+
"pages_sampled": extraction.pages_sampled,
|
|
386
|
+
# Code and values, no sentence: a stored text has no language.
|
|
387
|
+
"warnings": [message.to_dict() for message in extraction.warnings],
|
|
388
|
+
# Only for a text the text recognition read (SDB-116).
|
|
389
|
+
**(
|
|
390
|
+
{"tesseract": extraction.metadata["tesseract"]}
|
|
391
|
+
if extraction.metadata.get("tesseract")
|
|
392
|
+
else {}
|
|
393
|
+
),
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# The warnings of the reading stage as they were written before they had
|
|
398
|
+
# codes. A text stored then is still read (BIB-3).
|
|
399
|
+
_WARNINGS_BEFORE_CODES: tuple[tuple[re.Pattern[str], str, str | None], ...] = (
|
|
400
|
+
(
|
|
401
|
+
re.compile(r"^Text layer split into single characters; read by OCR"),
|
|
402
|
+
"read-by-ocr-single-characters",
|
|
403
|
+
None,
|
|
404
|
+
),
|
|
405
|
+
(re.compile(r"^No text layer; read by OCR"), "read-by-ocr-no-text-layer", None),
|
|
406
|
+
(
|
|
407
|
+
re.compile(r"^(\d+) page\(s\) without a text layer read by OCR\.$"),
|
|
408
|
+
"pages-read-by-ocr",
|
|
409
|
+
"pages",
|
|
410
|
+
),
|
|
411
|
+
(re.compile(r"^Extracted text is unusually short; "), "short-text", None),
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _stored_warning(stored: Any) -> Message:
|
|
416
|
+
"""A stored warning as a message: code and values, or a sentence of old."""
|
|
417
|
+
if isinstance(stored, Mapping):
|
|
418
|
+
try:
|
|
419
|
+
return Message.from_dict(stored)
|
|
420
|
+
# Stored by another version: a code that has gone since, or values
|
|
421
|
+
# that no longer fit it. A stored text outlives the codes of its day.
|
|
422
|
+
except (KeyError, TypeError, ValueError):
|
|
423
|
+
return Message(
|
|
424
|
+
"unknown",
|
|
425
|
+
{"text": json.dumps(stored, ensure_ascii=False, sort_keys=True)},
|
|
426
|
+
)
|
|
427
|
+
for pattern, code, counted in _WARNINGS_BEFORE_CODES:
|
|
428
|
+
found = pattern.match(str(stored))
|
|
429
|
+
if found:
|
|
430
|
+
return Message(code, {counted: int(found.group(1))} if counted else {})
|
|
431
|
+
# Not dropped: it says something about the text, even if not in a code.
|
|
432
|
+
return Message("unknown", {"text": str(stored)})
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def extraction_from_dict(stored: Mapping[str, Any]) -> ExtractionResult:
|
|
436
|
+
"""The text ``extraction_to_dict`` stored, for ``read_extraction`` (BIB-3).
|
|
437
|
+
|
|
438
|
+
Whether the stored stage still holds is the caller's to compare with
|
|
439
|
+
``extraction_stage()``; a text of another stage is read all the same. A
|
|
440
|
+
warning stored as a sentence, before warnings had codes, gets the code of
|
|
441
|
+
that sentence, or "unknown" with the sentence as its value; so does a
|
|
442
|
+
warning with a code this version does not know.
|
|
443
|
+
|
|
444
|
+
>>> extraction = extraction_from_dict(
|
|
445
|
+
... {"text": "Sicherheitsdatenblatt", "extractor": "pymupdf",
|
|
446
|
+
... "pages_sampled": 1, "warnings": []}
|
|
447
|
+
... )
|
|
448
|
+
>>> extraction.text, extraction.pages_sampled
|
|
449
|
+
('Sicherheitsdatenblatt', 1)
|
|
450
|
+
"""
|
|
451
|
+
text = stored["text"]
|
|
452
|
+
return ExtractionResult(
|
|
453
|
+
text=text,
|
|
454
|
+
raw_text=text,
|
|
455
|
+
extractor=stored["extractor"],
|
|
456
|
+
pages_sampled=stored["pages_sampled"],
|
|
457
|
+
warnings=[_stored_warning(warning) for warning in stored["warnings"]],
|
|
458
|
+
metadata={
|
|
459
|
+
"character_count": len(text),
|
|
460
|
+
**({"stage": str(stored["stage"])} if stored.get("stage") else {}),
|
|
461
|
+
**({"tesseract": stored["tesseract"]} if stored.get("tesseract") else {}),
|
|
462
|
+
},
|
|
463
|
+
)
|