sdb-reader 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,506 @@
1
+ """PDF text extraction helpers for Sicherheitsdatenblatt processing."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import functools
6
+ import re
7
+ import subprocess
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+
11
+ import pymupdf
12
+
13
+ from .layout import FLAGS, page_text
14
+ from .line_breaks import join_broken_words
15
+ from .messages import Message, shipped_catalog
16
+
17
+ # Reject PDFs larger than 50 MB to guard against decompression bombs.
18
+ MAX_PDF_SIZE_BYTES: int = 50 * 1024 * 1024
19
+
20
+ # Safety data sheets in the corpus are German.
21
+ OCR_LANGUAGE: str = "deu"
22
+
23
+ # OCR runs at roughly a second a page against milliseconds for a text layer, so
24
+ # it is capped even when the caller asked for the whole document. Everything the
25
+ # parser needs -- the label section, the classification, the supplier -- is in
26
+ # the first pages; a sheet whose section 2.2 is on page 9 is not a case the
27
+ # corpus has.
28
+ OCR_MAX_PAGES: int = 8
29
+
30
+ # A safety data sheet runs to thousands of characters. Below this, something went
31
+ # wrong on the way from the file to the text -- the reading did not fail outright,
32
+ # which is why it is a warning, but what came back cannot be the document.
33
+ #
34
+ # A caller recognises it by its code, "short-text".
35
+ SHORT_TEXT_THRESHOLD: int = 600
36
+
37
+ # Some PDFs place every glyph as its own text element, so the text layer comes
38
+ # back one character per line -- "ABSCHNITT" as "A\nB\nS\nC\nH\nN\nI\nT\nT".
39
+ # There is plenty of text and none of it can be searched: no keyword matches, no
40
+ # H-code, no section marker. Nothing repairs it either, because the word
41
+ # boundaries are not in the file; joining the lines gives "Erstelltam22.11.2010".
42
+ #
43
+ # Such a page is a picture of a document as far as reading goes, so it is read
44
+ # like one. Measured over 3,344 PDFs: 3,250 sit below 20%, a further
45
+ # 27 below 40%, and 11 are at 50% or above -- every one of those 11 already
46
+ # refused by the verifier, so nothing that works today is touched. Two of them
47
+ # are a genuine safety data sheet at 100%, which OCR then recognises;
48
+ # the other nine stay refused, as they should.
49
+ CHARACTER_PER_LINE_RATIO: float = 0.4
50
+
51
+ # Below this many lines the ratio says nothing: a two-line document is not
52
+ # evidence of anything.
53
+ CHARACTER_PER_LINE_MIN_LINES: int = 20
54
+
55
+ # 300 dpi is what the tables need; 150 loses the smaller table type.
56
+ OCR_DPI: int = 300
57
+
58
+ # A page that takes longer than this is not going to produce anything useful.
59
+ OCR_PAGE_TIMEOUT_SECONDS: int = 120
60
+
61
+
62
+ class ExtractionError(RuntimeError):
63
+ """Raised when text cannot be extracted from an SDS PDF.
64
+
65
+ ``message`` says why, as a code and its values; ``str()`` of the error is
66
+ that message in English, for a traceback.
67
+ """
68
+
69
+ def __init__(self, message: Message) -> None:
70
+ super().__init__(shipped_catalog("en").render(message))
71
+ self.message = message
72
+
73
+
74
+ @dataclass(slots=True)
75
+ class ExtractionResult:
76
+ """Details about a text extraction attempt."""
77
+
78
+ text: str
79
+ raw_text: str
80
+ extractor: str
81
+ pages_sampled: int
82
+ warnings: list[Message]
83
+ metadata: dict[str, object]
84
+
85
+
86
+ def extract_pdf_text(
87
+ path: Path,
88
+ *,
89
+ max_pages: int | None = None,
90
+ ocr: bool = True,
91
+ ) -> ExtractionResult:
92
+ """Extract text from *path* using PyMuPDF, in the order the pages print it.
93
+
94
+ A text layer is read by the position of its words (layout.page_text,
95
+ SDB-42 to SDB-45); a file without a readable one goes to OCR (SDB-46).
96
+
97
+ All pages are read, as ``read_sheet`` reads them; *max_pages* bounds how
98
+ many. With *ocr* off, a PDF without a text layer is not handed to
99
+ text recognition: the call fails with ``no-text-layer`` instead.
100
+ """
101
+
102
+ if not path.exists():
103
+ raise ExtractionError(Message("pdf-not-found"))
104
+
105
+ file_size = path.stat().st_size
106
+ if file_size > MAX_PDF_SIZE_BYTES:
107
+ raise ExtractionError(
108
+ Message(
109
+ "pdf-too-large",
110
+ {"size_bytes": file_size, "limit_bytes": MAX_PDF_SIZE_BYTES},
111
+ )
112
+ )
113
+
114
+ warnings: list[Message] = []
115
+
116
+ try:
117
+ stored, pages = _extract_with_pymupdf(path, max_pages)
118
+ except Exception as exc:
119
+ # The library names the file in what it says; a reading does not
120
+ # (BIB-2): "Failed to open file '/a/b.pdf' as type pdf." loses the
121
+ # path with its quotes.
122
+ named = r"\s*['\"]?" + re.escape(str(path)) + r"['\"]?"
123
+ detail = " ".join(re.sub(named, "", str(exc)).split())
124
+ raise ExtractionError(Message("pdf-unreadable", {"detail": detail})) from exc
125
+
126
+ extractor = "pymupdf"
127
+ pages_sampled = len(pages)
128
+ # Whether the text layer can be read at all is judged on the file's own
129
+ # order, the one the thresholds below were measured on: read by position,
130
+ # a page of single glyphs comes back as rows of letters spaced apart and
131
+ # no longer looks like what it is.
132
+ linear = "\n".join(stored)
133
+ zerlegt = _is_split_into_characters(linear)
134
+ text = "\n".join(pages)
135
+ if not linear.strip() or zerlegt:
136
+ # No usable text layer. Either there is none at all -- a scan or a set
137
+ # of exported images, 55 of those PDFs -- or there is one and it
138
+ # is unreadable because every glyph stands on its own line. A person
139
+ # can read both; the extractor can read neither, so the page is
140
+ # rendered and read instead.
141
+ if not ocr:
142
+ raise ExtractionError(Message("no-text-layer", {"pages": pages_sampled}))
143
+ text, pages_sampled = _extract_with_ocr(path, max_pages)
144
+ # _clean_ocr_pages has already stripped the furniture from these, so
145
+ # there is nothing left for the prompt path to strip a second time.
146
+ pages = []
147
+ extractor = "pymupdf+ocr"
148
+ warnings.append(
149
+ Message(
150
+ "read-by-ocr-single-characters"
151
+ if zerlegt
152
+ else "read-by-ocr-no-text-layer"
153
+ )
154
+ )
155
+
156
+ elif ocr and (read := _read_image_pages(path, stored, pages)):
157
+ # A page that is a picture among pages with text (SDB-95): the first
158
+ # page of a sheet, scanned, holds its date, its supplier and its
159
+ # classification.
160
+ text = "\n".join(pages)
161
+ extractor = "pymupdf+ocr"
162
+ warnings.append(Message("pages-read-by-ocr", {"pages": read}))
163
+
164
+ if not text.strip():
165
+ raise ExtractionError(
166
+ Message("no-text", {"pages": pages_sampled, "extractor": extractor})
167
+ )
168
+
169
+ raw_text = text
170
+ normalized_text = normalise_pdf_text(raw_text)
171
+
172
+ char_count = len(normalized_text)
173
+ if char_count < SHORT_TEXT_THRESHOLD:
174
+ warnings.append(Message("short-text"))
175
+
176
+ metadata: dict[str, object] = {
177
+ "character_count": char_count,
178
+ # Kept for strip_page_furniture, which cannot find a running header once
179
+ # the pages are joined. Not persisted: see persistence.build_persistent_payload.
180
+ "pages": pages,
181
+ }
182
+ if extractor == "pymupdf+ocr":
183
+ # The program's version belongs to a text it read (SDB-116).
184
+ metadata["tesseract"] = _tesseract_version()
185
+
186
+ return ExtractionResult(
187
+ text=normalized_text,
188
+ raw_text=raw_text,
189
+ extractor=extractor,
190
+ pages_sampled=pages_sampled,
191
+ warnings=warnings,
192
+ metadata=metadata,
193
+ )
194
+
195
+
196
+ def _extract_with_pymupdf(
197
+ path: Path, max_pages: int | None
198
+ ) -> tuple[list[str], list[str]]:
199
+ """Read the text layer of each page: in the file's order, and as printed.
200
+
201
+ The text is the printed one (SDB-42): layout.page_text puts the words where
202
+ the page shows them, where the file's order reads a two-column form label
203
+ by label and value by value. The file's order is returned beside it for
204
+ the one question it still answers, whether the text layer is readable at
205
+ all.
206
+
207
+ Page by page, because joining them loses where one ends: strip_page_furniture
208
+ needs the boundaries to tell a running header from a line that happens to
209
+ occur twice.
210
+ """
211
+ stored: list[str] = []
212
+ printed: list[str] = []
213
+ with pymupdf.open(path) as doc:
214
+ for number, page in enumerate(doc):
215
+ if max_pages is not None and number >= max_pages:
216
+ break
217
+ # One extraction serves both: FLAGS are page.get_text()'s own.
218
+ textpage = page.get_textpage(flags=FLAGS)
219
+ stored.append(page.get_text(textpage=textpage) or "")
220
+ printed.append(page_text(page, textpage))
221
+ return stored, printed
222
+
223
+
224
+ def _is_split_into_characters(text: str) -> bool:
225
+ """Whether the text layer came back one character per line.
226
+
227
+ The signature is unambiguous where it occurs: the two genuine sheets in the
228
+ corpus are at 100%, the nine other candidates at 50-53%, and the next
229
+ document down is at 40%. Ordinary sheets sit under 20%. The threshold has
230
+ room on both sides, so it does not need to be exact -- 0.3 or 0.6 would
231
+ select the same eleven files.
232
+ """
233
+ lines = text.split("\n")
234
+ if len(lines) < CHARACTER_PER_LINE_MIN_LINES:
235
+ return False
236
+ single = sum(1 for line in lines if len(line.strip()) == 1)
237
+ return single / len(lines) >= CHARACTER_PER_LINE_RATIO
238
+
239
+
240
+ def _extract_with_ocr(path: Path, max_pages: int | None) -> tuple[str, int]:
241
+ """Read a PDF that carries no text layer, by recognising the page image.
242
+
243
+ The page is rendered here and handed to the ``tesseract`` binary directly,
244
+ rather than through PyMuPDF's ``get_textpage_ocr``. The wrapper drops table
245
+ rows: on one sheet measured it returned 49 lines against the binary's 79,
246
+ and among the missing ones were
247
+
248
+ Gefahrenhinweise für Gesundheitsgefahren
249
+ H315 Verursacht Hautreizungen.
250
+ H318 Verursacht schwere Augenschäden.
251
+
252
+ -- the label's entire hazard statement table. A record built from that would
253
+ have claimed a corrosive product carries no hazard statements at all, which
254
+ is worse than having no record: it looks complete.
255
+
256
+ German only: every sheet in the corpus is, and a language guess would be
257
+ a knob nobody could set correctly. Where tesseract is missing the failure is
258
+ reported as an extraction error like any other, so a machine without it
259
+ behaves exactly as before rather than breaking.
260
+ """
261
+ limit = OCR_MAX_PAGES if max_pages is None else min(max_pages, OCR_MAX_PAGES)
262
+
263
+ chunks: list[str] = []
264
+ pages = 0
265
+ with pymupdf.open(path) as doc:
266
+ for number in range(min(len(doc), limit)):
267
+ image = doc[number].get_pixmap(dpi=OCR_DPI).tobytes("png")
268
+ chunks.append(_run_tesseract(image))
269
+ pages += 1
270
+
271
+ return _clean_ocr_pages(chunks), pages
272
+
273
+
274
+ def _read_image_pages(path: Path, stored: list[str], pages: list[str]) -> int:
275
+ """Fill the pages that carry a picture and no text, by OCR; how many.
276
+
277
+ *stored* says which pages have no text layer, *pages* gets their text. At
278
+ most OCR_MAX_PAGES of them, as for a file without any text layer.
279
+ """
280
+ empty = [number for number, text in enumerate(stored) if not text.strip()]
281
+ if not empty:
282
+ return 0
283
+ read = 0
284
+ with pymupdf.open(path) as doc:
285
+ for number in empty:
286
+ if read >= OCR_MAX_PAGES or not doc[number].get_images():
287
+ continue
288
+ image = doc[number].get_pixmap(dpi=OCR_DPI).tobytes("png")
289
+ pages[number] = _repair_codes(_run_tesseract(image))
290
+ read += 1
291
+ return read
292
+
293
+
294
+ # What the text recognition reads depends on the program: below this version
295
+ # the same picture gives another text (SDB-116).
296
+ TESSERACT_MINIMUM: int = 5
297
+ _TESSERACT_VERSION = re.compile(r"tesseract\s+v?(\d+(?:\.\d+)*)")
298
+
299
+
300
+ @functools.cache
301
+ def _tesseract_version() -> str:
302
+ """The installed tesseract's version, checked once (SDB-116).
303
+
304
+ Raises ExtractionError where the program is missing, older than the
305
+ minimum, or without the German language data: each is a reason the file
306
+ cannot be read. A failure is not remembered, so installing the program
307
+ helps without a restart.
308
+ """
309
+ try:
310
+ asked = subprocess.run(
311
+ ["tesseract", "--version"], capture_output=True, timeout=30, check=False
312
+ )
313
+ languages = subprocess.run(
314
+ ["tesseract", "--list-langs"], capture_output=True, timeout=30, check=False
315
+ )
316
+ except FileNotFoundError as exc:
317
+ raise ExtractionError(Message("ocr-unavailable")) from exc
318
+ except subprocess.TimeoutExpired as exc:
319
+ raise ExtractionError(Message("ocr-timeout")) from exc
320
+
321
+ # Older builds print their version to stderr.
322
+ said = (asked.stdout + asked.stderr).decode("utf-8", "replace")
323
+ found = _TESSERACT_VERSION.search(said)
324
+ if found is None:
325
+ raise ExtractionError(Message("ocr-failed", {"detail": said.strip()[:200]}))
326
+ version = found.group(1)
327
+ if int(version.split(".")[0]) < TESSERACT_MINIMUM:
328
+ raise ExtractionError(
329
+ Message(
330
+ "ocr-too-old", {"version": version, "minimum": str(TESSERACT_MINIMUM)}
331
+ )
332
+ )
333
+ listed = (languages.stdout + languages.stderr).decode("utf-8", "replace")
334
+ if OCR_LANGUAGE not in {line.strip() for line in listed.splitlines()}:
335
+ raise ExtractionError(
336
+ Message("ocr-language-missing", {"language": OCR_LANGUAGE})
337
+ )
338
+ return version
339
+
340
+
341
+ def _run_tesseract(image: bytes) -> str:
342
+ """Hand one rendered page to the binary and return what it read."""
343
+ _tesseract_version()
344
+ try:
345
+ result = subprocess.run(
346
+ ["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"],
347
+ input=image,
348
+ capture_output=True,
349
+ timeout=OCR_PAGE_TIMEOUT_SECONDS,
350
+ )
351
+ except FileNotFoundError as exc:
352
+ raise ExtractionError(Message("ocr-unavailable")) from exc
353
+ except subprocess.TimeoutExpired as exc:
354
+ raise ExtractionError(Message("ocr-timeout")) from exc
355
+
356
+ if result.returncode != 0:
357
+ detail = result.stderr.decode("utf-8", "replace").strip()[:200]
358
+ raise ExtractionError(Message("ocr-failed", {"detail": detail}))
359
+ return result.stdout.decode("utf-8", "replace")
360
+
361
+
362
+ # O for zero, l or I for one, S for five, B for eight. Applied only inside a
363
+ # hazard code, never to running text: "H3l5" is certainly H315, but "Sol" is not
364
+ # "501" and repairing it everywhere would corrupt words.
365
+ _OCR_DIGITS = str.maketrans(
366
+ {"O": "0", "o": "0", "l": "1", "I": "1", "S": "5", "B": "8"}
367
+ )
368
+ _CODE_SHAPES = re.compile(r"\b(GHS|EUH|H|P)([0-9OolISB]{2,3})\b")
369
+
370
+
371
+ def _repair_codes(text: str) -> str:
372
+ """Undo the letter-for-digit confusions OCR makes inside hazard codes.
373
+
374
+ A scanned ``GHS05`` comes back as ``GHSO5`` often enough to matter -- one of
375
+ the sheets measured had exactly that, and the pictogram pattern then matched
376
+ nothing. The same happens to H- and P-statements.
377
+ """
378
+ return _CODE_SHAPES.sub(
379
+ lambda m: m.group(1) + m.group(2).translate(_OCR_DIGITS), text
380
+ )
381
+
382
+
383
+ def strip_page_furniture(pages: list[str], *, keep_first: bool = False) -> str:
384
+ """Join *pages*, dropping the boilerplate that repeats across them.
385
+
386
+ Repetition is what identifies furniture, rather than position on the page: a
387
+ line that appears on most pages is a header or a footer whatever its
388
+ coordinates, and this needs no geometry. A document of one page has no
389
+ repetition to go on and is left alone.
390
+
391
+ Two callers, for two different reasons.
392
+
393
+ **OCR**, where it is a correctness fix. In the file's order a text layer
394
+ usually keeps page furniture apart; OCR does not, so the running header
395
+ lands in the middle of section 2.2 -- and find_label_section, which ends the
396
+ section at a heading, stops there. The layout reader puts the header where
397
+ the page prints it too (SDB-42); measured over the 1,991 local sheets with a
398
+ text layer, that cuts no label block short of its codes. One sheet measured lost its H-statements exactly this
399
+ way: the label block was cut after 507 characters by a "SICHERHEITSDATENBLATT
400
+ ... Seite 2/9" line, and the codes sat behind it.
401
+
402
+ **The model prompt**, where it is economy. Measured over 297 sheets the
403
+ repeated lines are 16.8% of the text at the median and over 20% for a
404
+ quarter of them -- "Version", "DE / DE", "Datum der ersten Ausgabe:
405
+ 10.12.2019". The prompt asks for the supplier and the label statements and
406
+ for nothing that lives in a running header, so none of it can be missed.
407
+
408
+ What is dropped here is not lost: the version and both dates are read from
409
+ exactly these lines, and the record keeps the unstripped text. This is a
410
+ narrower input for one caller, not a smaller document.
411
+
412
+ *keep_first* leaves one copy of each repeated line standing. A line can be
413
+ furniture and content at once -- one sheet in the corpus prints "Tannhof
414
+ Bergfeld GmbH" as its running header and as the supplier in section 1, so
415
+ dropping every occurrence dropped the answer along with the noise. Keeping
416
+ the first still removes the other three of four.
417
+
418
+ Both callers pass it now. The OCR path used not to, on the reasoning that
419
+ the repetition is itself the damage: a header standing inside section 2.2
420
+ ends the section there. That reasoning holds for the second occurrence
421
+ onwards, which is still dropped. It does not hold for the first, which sits
422
+ at the top of page one ahead of section 1 -- and dropping that one took the
423
+ revision date with it, because for OCR this function's output is the record's
424
+ text and no unstripped copy survives behind it. Measured over all 36 OCR'd
425
+ sheets in the corpus: two gain a date, none lose a hazard statement, a
426
+ signal word or a label section.
427
+ """
428
+ if len(pages) < 2:
429
+ return "\n".join(pages)
430
+
431
+ counts: dict[str, int] = {}
432
+ for page in pages:
433
+ for line in {ln.strip() for ln in page.split("\n") if len(ln.strip()) > 3}:
434
+ counts[line] = counts.get(line, 0) + 1
435
+
436
+ threshold = max(2, (len(pages) + 1) // 2)
437
+ furniture = {line for line, n in counts.items() if n >= threshold}
438
+
439
+ seen: set[str] = set()
440
+
441
+ def _keep(line: str) -> bool:
442
+ stripped = line.strip()
443
+ if stripped not in furniture:
444
+ return True
445
+ if not keep_first or stripped in seen:
446
+ return False
447
+ seen.add(stripped)
448
+ return True
449
+
450
+ kept = ["\n".join(ln for ln in page.split("\n") if _keep(ln)) for page in pages]
451
+ return "\n".join(kept)
452
+
453
+
454
+ def _clean_ocr_pages(pages: list[str]) -> str:
455
+ """Strip furniture from OCR'd pages and repair the codes OCR mangles.
456
+
457
+ ``keep_first`` matters here for a reason the text-layer path never has: what
458
+ this function returns *is* the record's text. There is no unstripped copy
459
+ behind it, so a line dropped here is gone for good -- and the running header
460
+ is where the revision date lives. Two sheets went out
461
+ undated because of it, one of them stating "überarbeitet am: 16.07.2019"
462
+ plainly on its first page.
463
+ """
464
+ return _repair_codes(strip_page_furniture(pages, keep_first=True))
465
+
466
+
467
+ _CODE_BEFORE_LINE_BREAK = re.compile(r"\b((?:EUH|H|P)\d{3})-\n")
468
+
469
+
470
+ def normalise_pdf_text(text: str) -> str:
471
+ """Normalise raw PDF text while preserving paragraph structure."""
472
+
473
+ if not text:
474
+ return ""
475
+
476
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
477
+ text = re.sub(r"\u00ad", "", text) # remove soft hyphen characters
478
+ # A hyphen after a hazard code separates the code from its wording; it is
479
+ # not a word broken across lines. "H319-" at the end of one line and
480
+ # "Verursacht schwere Augenreizung." on the next became "H319Verursacht"
481
+ # under the rule below, and a reader that ends a code at a word boundary
482
+ # no longer saw H319. The line is joined, the hyphen kept.
483
+ text = _CODE_BEFORE_LINE_BREAK.sub(r"\1-", text)
484
+ # A word broken at the line end loses its hyphen only between two
485
+ # lower-case letters (SDB-32, SDB-54): "5-Chlor-2-methyl-2H-" /
486
+ # "isothiazol-3-on" became "2Hisothiazol" and "2-" / "Methyl" "2Methyl"
487
+ # , "tert-butyl-" / "4-hydroxyphenyl" "butyl4-hydroxyphenyl".
488
+ text = join_broken_words(text)
489
+
490
+ normalised_lines: list[str] = []
491
+ blank_run = 0
492
+ for raw_line in text.split("\n"):
493
+ stripped = raw_line.strip()
494
+ if not stripped:
495
+ blank_run += 1
496
+ if blank_run <= 2:
497
+ normalised_lines.append("")
498
+ continue
499
+
500
+ blank_run = 0
501
+ cleaned = re.sub(r"\s+", " ", stripped)
502
+ normalised_lines.append(cleaned)
503
+
504
+ normalised_text = "\n".join(normalised_lines)
505
+ normalised_text = re.sub(r"\n{3,}", "\n\n", normalised_text)
506
+ return normalised_text.strip()