sdb-reader 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,463 @@
1
+ """Readings and texts as JSON: what the command prints and what a caller stores."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import re
8
+ from collections.abc import Mapping
9
+ from datetime import date
10
+ from typing import Any
11
+
12
+ from .compliance import requires_sheet, sheet_findings
13
+ from .embedded import EmbeddedData, differences
14
+ from .extraction import ExtractionResult
15
+ from .messages import Catalog, Message, shipped_catalog
16
+ from .reading import SheetReading
17
+ from .schema import (
18
+ ContactInfo,
19
+ HazardStatements,
20
+ SafetyDataSheet,
21
+ Section,
22
+ StatedClassification,
23
+ )
24
+ from .statements import label_statements
25
+ from .verification import DocumentKind, Verification
26
+ from .versions import extraction_stage
27
+
28
+
29
+ def _content_hash(extraction: ExtractionResult) -> str | None:
30
+ """Fingerprint the document by its extracted text.
31
+
32
+ Hashing the PDF bytes does not work: every write gets a fresh /ID, so the
33
+ same document exported three times yields three different byte hashes at
34
+ identical length. The extracted text is the same across all three.
35
+
36
+ Returns None for an empty extraction -- a document we could not read is not
37
+ evidence of being the same as another one we could not read.
38
+ """
39
+ return _text_hash(extraction.text)
40
+
41
+
42
+ def _text_hash(text: str | None) -> str | None:
43
+ text = (text or "").strip()
44
+ if not text:
45
+ return None
46
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
47
+
48
+
49
+ def sheet_to_dict(
50
+ sheet: SafetyDataSheet, extraction: ExtractionResult
51
+ ) -> dict[str, Any]:
52
+ """The parsed sheet as JSON: the ``sheet`` part of what the command prints.
53
+
54
+ *extraction* is the text the sheet was parsed from; its fingerprint goes
55
+ into the record. Every value stands once (SDW-18).
56
+ """
57
+ return _sheet_fields(sheet, _content_hash(extraction))
58
+
59
+
60
+ def _sheet_fields(sheet: SafetyDataSheet, content_hash: str | None) -> dict[str, Any]:
61
+ return {
62
+ # The document's own version, as printed in the sheet.
63
+ "version": sheet.version,
64
+ # Identifies the document by content, so copies filed under several
65
+ # names can be recognised as one. See _content_hash.
66
+ "content_hash": content_hash,
67
+ "product_identifier": sheet.product_identifier,
68
+ "issue_date": sheet.issue_date.isoformat() if sheet.issue_date else None,
69
+ "revision_date": sheet.revision_date.isoformat()
70
+ if sheet.revision_date
71
+ else None,
72
+ "language": sheet.language,
73
+ # The countries the sheet is issued for (SDB-69), what section 2 says
74
+ # about the classification (SDB-75), and whether the product needs a
75
+ # sheet by it (SDB-76): a sheet for one that does not is voluntary.
76
+ "countries": list(sheet.countries),
77
+ "stated_classification": sheet.stated_classification.value,
78
+ # Whether the sheet calls itself voluntary or not prescribed, and its
79
+ # product an article beside that (SDB-142): an article needs no sheet
80
+ # whatever the label prints, an unclassified mixture none either.
81
+ "stated_voluntary": sheet.stated_voluntary,
82
+ "stated_article": sheet.stated_article,
83
+ "sheet_required": requires_sheet(sheet),
84
+ "hazard_statements": {
85
+ "hazard_codes": sheet.hazard_statements.hazard_codes,
86
+ "precautionary_codes": sheet.hazard_statements.precautionary_codes,
87
+ "supplemental_codes": sheet.hazard_statements.supplemental_codes,
88
+ "signal_word": sheet.hazard_statements.signal_word,
89
+ "pictograms": sheet.hazard_statements.pictograms,
90
+ "hazard_text": sheet.hazard_statements.hazard_text,
91
+ "precautionary_text": sheet.hazard_statements.precautionary_text,
92
+ "supplemental_text": sheet.hazard_statements.supplemental_text,
93
+ },
94
+ "contact_info": {
95
+ "supplier_name": sheet.contact_info.supplier_name,
96
+ "supplier_address": sheet.contact_info.supplier_address,
97
+ "supplier_phone": sheet.contact_info.supplier_phone,
98
+ "emergency_phone": sheet.contact_info.emergency_phone,
99
+ "supplier_email": sheet.contact_info.supplier_email,
100
+ "supplier_website": sheet.contact_info.supplier_website,
101
+ "information_email": sheet.contact_info.information_email,
102
+ },
103
+ "section_text": {
104
+ section.value: text for section, text in sheet.section_text.items()
105
+ },
106
+ }
107
+
108
+
109
+ def sheet_from_dict(stored: Mapping[str, Any]) -> SafetyDataSheet:
110
+ """The sheet ``sheet_to_dict`` stored, to judge it again (BIB-21).
111
+
112
+ Reads what this version writes: the ``sheet`` part of a record of
113
+ ``read``, or of one of its ``parts``. ``content_hash`` and
114
+ ``sheet_required`` are not part of a sheet and are left out;
115
+ ``requires_sheet`` gives the second again. A key this version does not
116
+ know is ignored.
117
+
118
+ A record that lacks a key, or holds a value of another kind, was stored
119
+ by another version and is refused with ``ValueError``: make the reading
120
+ again from the stored text (``read_extraction``) instead. Whether a
121
+ record still holds is the caller's to compare with ``PARSER_VERSION``.
122
+
123
+ >>> stored = {
124
+ ... "version": "2.0", "product_identifier": "Musterfarbe",
125
+ ... "issue_date": None, "revision_date": "2024-03-01", "language": "de",
126
+ ... "countries": ["DE"], "stated_classification": "classified",
127
+ ... "stated_voluntary": False, "stated_article": False,
128
+ ... "hazard_statements": {
129
+ ... "hazard_codes": ["H315"], "precautionary_codes": [],
130
+ ... "supplemental_codes": [], "signal_word": "Achtung",
131
+ ... "pictograms": ["GHS07"], "hazard_text": None,
132
+ ... "precautionary_text": None, "supplemental_text": None,
133
+ ... },
134
+ ... "contact_info": {
135
+ ... "supplier_name": "Nordwerk GmbH", "supplier_address": None,
136
+ ... "supplier_phone": None, "emergency_phone": None,
137
+ ... "supplier_email": None, "supplier_website": None,
138
+ ... "information_email": None,
139
+ ... },
140
+ ... "section_text": {"section_2": "Signalwort: Achtung"},
141
+ ... }
142
+ >>> sheet = sheet_from_dict(stored)
143
+ >>> sheet.revision_date, sheet.hazard_statements.signal_word
144
+ (datetime.date(2024, 3, 1), 'Achtung')
145
+ >>> requires_sheet(sheet)
146
+ True
147
+ """
148
+ try:
149
+ label = stored["hazard_statements"]
150
+ contact = stored["contact_info"]
151
+ return SafetyDataSheet(
152
+ product_identifier=_optional_text(stored["product_identifier"]),
153
+ issue_date=_stored_date(stored["issue_date"]),
154
+ revision_date=_stored_date(stored["revision_date"]),
155
+ version=_optional_text(stored["version"]),
156
+ language=_text(stored["language"]),
157
+ hazard_statements=HazardStatements(
158
+ hazard_codes=_texts(label["hazard_codes"]),
159
+ precautionary_codes=_texts(label["precautionary_codes"]),
160
+ supplemental_codes=_texts(label["supplemental_codes"]),
161
+ signal_word=_optional_text(label["signal_word"]),
162
+ pictograms=_texts(label["pictograms"]),
163
+ hazard_text=_optional_text(label["hazard_text"]),
164
+ precautionary_text=_optional_text(label["precautionary_text"]),
165
+ supplemental_text=_optional_text(label["supplemental_text"]),
166
+ ),
167
+ contact_info=ContactInfo(
168
+ supplier_name=_optional_text(contact["supplier_name"]),
169
+ supplier_address=_optional_text(contact["supplier_address"]),
170
+ supplier_phone=_optional_text(contact["supplier_phone"]),
171
+ emergency_phone=_optional_text(contact["emergency_phone"]),
172
+ supplier_email=_optional_text(contact["supplier_email"]),
173
+ supplier_website=_optional_text(contact["supplier_website"]),
174
+ information_email=_optional_text(contact["information_email"]),
175
+ ),
176
+ section_text={
177
+ Section(name): _text(text)
178
+ for name, text in stored["section_text"].items()
179
+ },
180
+ countries=_texts(stored["countries"]),
181
+ stated_classification=StatedClassification(stored["stated_classification"]),
182
+ stated_voluntary=_flag(stored["stated_voluntary"]),
183
+ stated_article=_flag(stored["stated_article"]),
184
+ )
185
+ except (KeyError, TypeError, ValueError, AttributeError) as exc:
186
+ raise ValueError(
187
+ "not a sheet as this version stores it "
188
+ f"({type(exc).__name__}: {exc}); read the stored text again"
189
+ ) from exc
190
+
191
+
192
+ def _text(value: Any) -> str:
193
+ if not isinstance(value, str):
194
+ raise TypeError(f"expected text, found {type(value).__name__}")
195
+ return value
196
+
197
+
198
+ def _optional_text(value: Any) -> str | None:
199
+ return None if value is None else _text(value)
200
+
201
+
202
+ def _texts(value: Any) -> list[str]:
203
+ if not isinstance(value, list):
204
+ raise TypeError(f"expected a list, found {type(value).__name__}")
205
+ return [_text(item) for item in value]
206
+
207
+
208
+ def _flag(value: Any) -> bool:
209
+ if not isinstance(value, bool):
210
+ raise TypeError(f"expected true or false, found {type(value).__name__}")
211
+ return value
212
+
213
+
214
+ def _stored_date(value: Any) -> date | None:
215
+ return None if value is None else date.fromisoformat(_text(value))
216
+
217
+
218
+ def _shown(messages: list[Message], catalog: Catalog | None) -> list[dict[str, Any]]:
219
+ """Each message as code, values and sentence (SDW-17); German unless asked."""
220
+ chosen = catalog or shipped_catalog()
221
+ return [chosen.to_dict(message) for message in messages]
222
+
223
+
224
+ def verification_to_dict(
225
+ verification: Verification, *, catalog: Catalog | None = None
226
+ ) -> dict[str, Any]:
227
+ """The verdict as a record stores it; *catalog* words its messages."""
228
+ return {
229
+ "kind": verification.kind.value,
230
+ "prints_statement_codes": verification.prints_statement_codes,
231
+ "warnings": _shown(verification.warnings, catalog),
232
+ "errors": _shown(verification.errors, catalog),
233
+ }
234
+
235
+
236
+ def reading_to_dict(
237
+ reading: SheetReading,
238
+ *,
239
+ include_raw_text: bool = False,
240
+ embedded: EmbeddedData | None = None,
241
+ catalog: Catalog | None = None,
242
+ today: date | None = None,
243
+ ) -> dict[str, Any]:
244
+ """What reading one file found, as ``sdb-reader read`` prints it.
245
+
246
+ *today* is the day the findings take as the present (``sheet_findings``);
247
+ the command passes the machine's date. Without it no date is questioned,
248
+ and the record is the same on any day.
249
+
250
+ Without the file itself: the command puts its path, name and size in
251
+ front. *embedded* is what ``read_embedded`` found in the PDF; with it the
252
+ record lists where the embedded data and the sheet as read differ (SDW-11).
253
+ Every message stands as its code, its values and a sentence from
254
+ *catalog*: the shipped German one unless another is given (SDW-17).
255
+ """
256
+ # Every record has the same keys for its kind (SDW-19): the three lists
257
+ # are there for every file, empty where there is nothing to say.
258
+ record: dict[str, Any] = {
259
+ "verification": verification_to_dict(reading.verification, catalog=catalog),
260
+ "warnings": _shown(reading.warnings, catalog),
261
+ "errors": _shown(reading.errors, catalog),
262
+ "findings": [],
263
+ }
264
+ if reading.extraction is not None:
265
+ record["extraction"] = _extraction_to_dict(
266
+ reading.extraction, include_raw_text=include_raw_text, catalog=catalog
267
+ )
268
+ if reading.parser_result is not None:
269
+ record["sheet"] = sheet_to_dict(
270
+ reading.parser_result.sheet, reading.extraction
271
+ )
272
+ record["parser"] = {
273
+ "warnings": _shown(reading.parser_result.warnings, catalog)
274
+ }
275
+ sheet = reading.parser_result.sheet
276
+ record["statements"] = _statements(sheet)
277
+ # How many sheets the file holds, and each one read on its own
278
+ # where it holds several (SDW-32); the keys above are the file's.
279
+ parts = reading.parser_result.parts
280
+ record["sheet_count"] = len(parts) or 1
281
+ # A cover page in front of the sheets, and the word on it that
282
+ # names a display (SDW-35); it is read for nothing else.
283
+ record["cover_page"] = reading.parser_result.cover_page
284
+ record["display_term"] = reading.parser_result.display_term
285
+ if parts:
286
+ record["parts"] = [
287
+ {
288
+ "sheet": _sheet_fields(part.sheet, _text_hash(part.text)),
289
+ "statements": _statements(part.sheet),
290
+ "warnings": _shown(part.warnings, catalog),
291
+ }
292
+ for part in parts
293
+ ]
294
+ record["findings"] = _shown(
295
+ sheet_findings(
296
+ sheet, reading.parser_result.warnings, today=today, parts=parts
297
+ ),
298
+ catalog,
299
+ )
300
+ if reading.verification.kind is DocumentKind.FOREIGN_SAFETY_DATA_SHEET:
301
+ # One file says nothing about the product's other sheets (SDW-8).
302
+ record["findings"] = _shown([Message("not-german")], catalog)
303
+ if embedded is not None:
304
+ record["embedded_data"] = _embedded_to_dict(embedded, reading, catalog)
305
+ return record
306
+
307
+
308
+ def _statements(sheet: SafetyDataSheet) -> list[dict[str, Any]]:
309
+ """The label's statements, each with both wordings (SDW-5)."""
310
+ return [
311
+ {"code": s.code, "sheet": s.sheet, "regulation": s.regulation}
312
+ for s in label_statements(sheet.hazard_statements)
313
+ ]
314
+
315
+
316
+ def _embedded_to_dict(
317
+ embedded: EmbeddedData, reading: SheetReading, catalog: Catalog | None
318
+ ) -> dict[str, Any]:
319
+ """The embedded eSDScom file and where the sheet as read differs (SDW-11).
320
+
321
+ ``differences`` is always there; ``compared`` says whether there was
322
+ anything to compare: a readable embedded file and a German sheet (SDW-18).
323
+ """
324
+ payload: dict[str, Any] = {
325
+ "format": "eSDScom",
326
+ "filename": embedded.filename,
327
+ "compared": False,
328
+ "differences": [],
329
+ }
330
+ if embedded.sheet is None:
331
+ if embedded.unreadable is not None:
332
+ payload["unreadable"] = _shown([embedded.unreadable], catalog)[0]
333
+ return payload
334
+ payload["version"] = embedded.sheet.format_version
335
+ payload["system"] = embedded.sheet.authoring_system
336
+ if reading.parser_result is not None:
337
+ payload["compared"] = True
338
+ payload["differences"] = [
339
+ {"field": d.field, "embedded": d.embedded, "read": d.read}
340
+ for d in differences(embedded.sheet, reading.parser_result.sheet)
341
+ ]
342
+ return payload
343
+
344
+
345
+ def _extraction_to_dict(
346
+ extraction: ExtractionResult, *, include_raw_text: bool, catalog: Catalog | None
347
+ ) -> dict[str, Any]:
348
+ payload: dict[str, Any] = {
349
+ # The stage the text comes from: a stored text names its own, a text
350
+ # read from the PDF just now is of this library's (SDW-27).
351
+ "stage": extraction.metadata.get("stage") or extraction_stage(),
352
+ "extractor": extraction.extractor,
353
+ "pages_sampled": extraction.pages_sampled,
354
+ "warnings": _shown(extraction.warnings, catalog),
355
+ "character_count": extraction.metadata.get("character_count"),
356
+ }
357
+ if extraction.metadata.get("tesseract"):
358
+ payload["tesseract"] = extraction.metadata["tesseract"]
359
+ if include_raw_text:
360
+ payload["raw_text"] = extraction.raw_text
361
+ return payload
362
+
363
+
364
+ def extraction_to_dict(extraction: ExtractionResult) -> dict[str, Any]:
365
+ """The text read from a PDF, in a form to store as JSON (BIB-3).
366
+
367
+ Holds what a reading needs and names the stage it comes from. The text
368
+ before tidying and the single pages are not kept.
369
+
370
+ >>> text = "Sicherheitsdatenblatt\\nABSCHNITT 1: Bezeichnung des Stoffs\\n"
371
+ >>> stored = extraction_to_dict(
372
+ ... extraction_from_dict(
373
+ ... {"text": text, "extractor": "pymupdf", "pages_sampled": 1, "warnings": []}
374
+ ... )
375
+ ... )
376
+ >>> sorted(stored)
377
+ ['extractor', 'pages_sampled', 'stage', 'text', 'warnings']
378
+ >>> stored["stage"] == extraction_stage()
379
+ True
380
+ """
381
+ return {
382
+ "stage": extraction_stage(),
383
+ "text": extraction.text,
384
+ "extractor": extraction.extractor,
385
+ "pages_sampled": extraction.pages_sampled,
386
+ # Code and values, no sentence: a stored text has no language.
387
+ "warnings": [message.to_dict() for message in extraction.warnings],
388
+ # Only for a text the text recognition read (SDB-116).
389
+ **(
390
+ {"tesseract": extraction.metadata["tesseract"]}
391
+ if extraction.metadata.get("tesseract")
392
+ else {}
393
+ ),
394
+ }
395
+
396
+
397
+ # The warnings of the reading stage as they were written before they had
398
+ # codes. A text stored then is still read (BIB-3).
399
+ _WARNINGS_BEFORE_CODES: tuple[tuple[re.Pattern[str], str, str | None], ...] = (
400
+ (
401
+ re.compile(r"^Text layer split into single characters; read by OCR"),
402
+ "read-by-ocr-single-characters",
403
+ None,
404
+ ),
405
+ (re.compile(r"^No text layer; read by OCR"), "read-by-ocr-no-text-layer", None),
406
+ (
407
+ re.compile(r"^(\d+) page\(s\) without a text layer read by OCR\.$"),
408
+ "pages-read-by-ocr",
409
+ "pages",
410
+ ),
411
+ (re.compile(r"^Extracted text is unusually short; "), "short-text", None),
412
+ )
413
+
414
+
415
+ def _stored_warning(stored: Any) -> Message:
416
+ """A stored warning as a message: code and values, or a sentence of old."""
417
+ if isinstance(stored, Mapping):
418
+ try:
419
+ return Message.from_dict(stored)
420
+ # Stored by another version: a code that has gone since, or values
421
+ # that no longer fit it. A stored text outlives the codes of its day.
422
+ except (KeyError, TypeError, ValueError):
423
+ return Message(
424
+ "unknown",
425
+ {"text": json.dumps(stored, ensure_ascii=False, sort_keys=True)},
426
+ )
427
+ for pattern, code, counted in _WARNINGS_BEFORE_CODES:
428
+ found = pattern.match(str(stored))
429
+ if found:
430
+ return Message(code, {counted: int(found.group(1))} if counted else {})
431
+ # Not dropped: it says something about the text, even if not in a code.
432
+ return Message("unknown", {"text": str(stored)})
433
+
434
+
435
+ def extraction_from_dict(stored: Mapping[str, Any]) -> ExtractionResult:
436
+ """The text ``extraction_to_dict`` stored, for ``read_extraction`` (BIB-3).
437
+
438
+ Whether the stored stage still holds is the caller's to compare with
439
+ ``extraction_stage()``; a text of another stage is read all the same. A
440
+ warning stored as a sentence, before warnings had codes, gets the code of
441
+ that sentence, or "unknown" with the sentence as its value; so does a
442
+ warning with a code this version does not know.
443
+
444
+ >>> extraction = extraction_from_dict(
445
+ ... {"text": "Sicherheitsdatenblatt", "extractor": "pymupdf",
446
+ ... "pages_sampled": 1, "warnings": []}
447
+ ... )
448
+ >>> extraction.text, extraction.pages_sampled
449
+ ('Sicherheitsdatenblatt', 1)
450
+ """
451
+ text = stored["text"]
452
+ return ExtractionResult(
453
+ text=text,
454
+ raw_text=text,
455
+ extractor=stored["extractor"],
456
+ pages_sampled=stored["pages_sampled"],
457
+ warnings=[_stored_warning(warning) for warning in stored["warnings"]],
458
+ metadata={
459
+ "character_count": len(text),
460
+ **({"stage": str(stored["stage"])} if stored.get("stage") else {}),
461
+ **({"tesseract": stored["tesseract"]} if stored.get("tesseract") else {}),
462
+ },
463
+ )