sdb-reader 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdb_reader/__init__.py +157 -0
- sdb_reader/ask.py +437 -0
- sdb_reader/ask_record.schema.json +388 -0
- sdb_reader/cli.py +1079 -0
- sdb_reader/clp_statements_de.json +997 -0
- sdb_reader/compliance.py +546 -0
- sdb_reader/current.py +134 -0
- sdb_reader/diff.py +520 -0
- sdb_reader/embedded.py +250 -0
- sdb_reader/extraction.py +506 -0
- sdb_reader/layout.py +894 -0
- sdb_reader/line_breaks.py +40 -0
- sdb_reader/messages.py +394 -0
- sdb_reader/messages_data/de.toml +146 -0
- sdb_reader/messages_data/en.toml +141 -0
- sdb_reader/model.py +321 -0
- sdb_reader/parser.py +4601 -0
- sdb_reader/pictograms.py +374 -0
- sdb_reader/presets.py +110 -0
- sdb_reader/presets.toml +23 -0
- sdb_reader/py.typed +0 -0
- sdb_reader/read_record.schema.json +792 -0
- sdb_reader/reading.py +94 -0
- sdb_reader/scan.py +265 -0
- sdb_reader/schema.py +96 -0
- sdb_reader/serialise.py +463 -0
- sdb_reader/show.py +370 -0
- sdb_reader/statements.py +1011 -0
- sdb_reader/verification.py +322 -0
- sdb_reader/versions.py +31 -0
- sdb_reader-0.9.0.dist-info/METADATA +117 -0
- sdb_reader-0.9.0.dist-info/RECORD +37 -0
- sdb_reader-0.9.0.dist-info/WHEEL +5 -0
- sdb_reader-0.9.0.dist-info/entry_points.txt +2 -0
- sdb_reader-0.9.0.dist-info/licenses/LICENSE +661 -0
- sdb_reader-0.9.0.dist-info/licenses/NOTICE +40 -0
- sdb_reader-0.9.0.dist-info/top_level.txt +1 -0
sdb_reader/extraction.py
ADDED
|
@@ -0,0 +1,506 @@
|
|
|
1
|
+
"""PDF text extraction helpers for Sicherheitsdatenblatt processing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import functools
|
|
6
|
+
import re
|
|
7
|
+
import subprocess
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import pymupdf
|
|
12
|
+
|
|
13
|
+
from .layout import FLAGS, page_text
|
|
14
|
+
from .line_breaks import join_broken_words
|
|
15
|
+
from .messages import Message, shipped_catalog
|
|
16
|
+
|
|
17
|
+
# Reject PDFs larger than 50 MB to guard against decompression bombs.
|
|
18
|
+
MAX_PDF_SIZE_BYTES: int = 50 * 1024 * 1024
|
|
19
|
+
|
|
20
|
+
# Safety data sheets in the corpus are German.
|
|
21
|
+
OCR_LANGUAGE: str = "deu"
|
|
22
|
+
|
|
23
|
+
# OCR runs at roughly a second a page against milliseconds for a text layer, so
|
|
24
|
+
# it is capped even when the caller asked for the whole document. Everything the
|
|
25
|
+
# parser needs -- the label section, the classification, the supplier -- is in
|
|
26
|
+
# the first pages; a sheet whose section 2.2 is on page 9 is not a case the
|
|
27
|
+
# corpus has.
|
|
28
|
+
OCR_MAX_PAGES: int = 8
|
|
29
|
+
|
|
30
|
+
# A safety data sheet runs to thousands of characters. Below this, something went
|
|
31
|
+
# wrong on the way from the file to the text -- the reading did not fail outright,
|
|
32
|
+
# which is why it is a warning, but what came back cannot be the document.
|
|
33
|
+
#
|
|
34
|
+
# A caller recognises it by its code, "short-text".
|
|
35
|
+
SHORT_TEXT_THRESHOLD: int = 600
|
|
36
|
+
|
|
37
|
+
# Some PDFs place every glyph as its own text element, so the text layer comes
|
|
38
|
+
# back one character per line -- "ABSCHNITT" as "A\nB\nS\nC\nH\nN\nI\nT\nT".
|
|
39
|
+
# There is plenty of text and none of it can be searched: no keyword matches, no
|
|
40
|
+
# H-code, no section marker. Nothing repairs it either, because the word
|
|
41
|
+
# boundaries are not in the file; joining the lines gives "Erstelltam22.11.2010".
|
|
42
|
+
#
|
|
43
|
+
# Such a page is a picture of a document as far as reading goes, so it is read
|
|
44
|
+
# like one. Measured over 3,344 PDFs: 3,250 sit below 20%, a further
|
|
45
|
+
# 27 below 40%, and 11 are at 50% or above -- every one of those 11 already
|
|
46
|
+
# refused by the verifier, so nothing that works today is touched. Two of them
|
|
47
|
+
# are a genuine safety data sheet at 100%, which OCR then recognises;
|
|
48
|
+
# the other nine stay refused, as they should.
|
|
49
|
+
CHARACTER_PER_LINE_RATIO: float = 0.4
|
|
50
|
+
|
|
51
|
+
# Below this many lines the ratio says nothing: a two-line document is not
|
|
52
|
+
# evidence of anything.
|
|
53
|
+
CHARACTER_PER_LINE_MIN_LINES: int = 20
|
|
54
|
+
|
|
55
|
+
# 300 dpi is what the tables need; 150 loses the smaller table type.
|
|
56
|
+
OCR_DPI: int = 300
|
|
57
|
+
|
|
58
|
+
# A page that takes longer than this is not going to produce anything useful.
|
|
59
|
+
OCR_PAGE_TIMEOUT_SECONDS: int = 120
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ExtractionError(RuntimeError):
|
|
63
|
+
"""Raised when text cannot be extracted from an SDS PDF.
|
|
64
|
+
|
|
65
|
+
``message`` says why, as a code and its values; ``str()`` of the error is
|
|
66
|
+
that message in English, for a traceback.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def __init__(self, message: Message) -> None:
|
|
70
|
+
super().__init__(shipped_catalog("en").render(message))
|
|
71
|
+
self.message = message
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(slots=True)
|
|
75
|
+
class ExtractionResult:
|
|
76
|
+
"""Details about a text extraction attempt."""
|
|
77
|
+
|
|
78
|
+
text: str
|
|
79
|
+
raw_text: str
|
|
80
|
+
extractor: str
|
|
81
|
+
pages_sampled: int
|
|
82
|
+
warnings: list[Message]
|
|
83
|
+
metadata: dict[str, object]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def extract_pdf_text(
|
|
87
|
+
path: Path,
|
|
88
|
+
*,
|
|
89
|
+
max_pages: int | None = None,
|
|
90
|
+
ocr: bool = True,
|
|
91
|
+
) -> ExtractionResult:
|
|
92
|
+
"""Extract text from *path* using PyMuPDF, in the order the pages print it.
|
|
93
|
+
|
|
94
|
+
A text layer is read by the position of its words (layout.page_text,
|
|
95
|
+
SDB-42 to SDB-45); a file without a readable one goes to OCR (SDB-46).
|
|
96
|
+
|
|
97
|
+
All pages are read, as ``read_sheet`` reads them; *max_pages* bounds how
|
|
98
|
+
many. With *ocr* off, a PDF without a text layer is not handed to
|
|
99
|
+
text recognition: the call fails with ``no-text-layer`` instead.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
if not path.exists():
|
|
103
|
+
raise ExtractionError(Message("pdf-not-found"))
|
|
104
|
+
|
|
105
|
+
file_size = path.stat().st_size
|
|
106
|
+
if file_size > MAX_PDF_SIZE_BYTES:
|
|
107
|
+
raise ExtractionError(
|
|
108
|
+
Message(
|
|
109
|
+
"pdf-too-large",
|
|
110
|
+
{"size_bytes": file_size, "limit_bytes": MAX_PDF_SIZE_BYTES},
|
|
111
|
+
)
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
warnings: list[Message] = []
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
stored, pages = _extract_with_pymupdf(path, max_pages)
|
|
118
|
+
except Exception as exc:
|
|
119
|
+
# The library names the file in what it says; a reading does not
|
|
120
|
+
# (BIB-2): "Failed to open file '/a/b.pdf' as type pdf." loses the
|
|
121
|
+
# path with its quotes.
|
|
122
|
+
named = r"\s*['\"]?" + re.escape(str(path)) + r"['\"]?"
|
|
123
|
+
detail = " ".join(re.sub(named, "", str(exc)).split())
|
|
124
|
+
raise ExtractionError(Message("pdf-unreadable", {"detail": detail})) from exc
|
|
125
|
+
|
|
126
|
+
extractor = "pymupdf"
|
|
127
|
+
pages_sampled = len(pages)
|
|
128
|
+
# Whether the text layer can be read at all is judged on the file's own
|
|
129
|
+
# order, the one the thresholds below were measured on: read by position,
|
|
130
|
+
# a page of single glyphs comes back as rows of letters spaced apart and
|
|
131
|
+
# no longer looks like what it is.
|
|
132
|
+
linear = "\n".join(stored)
|
|
133
|
+
zerlegt = _is_split_into_characters(linear)
|
|
134
|
+
text = "\n".join(pages)
|
|
135
|
+
if not linear.strip() or zerlegt:
|
|
136
|
+
# No usable text layer. Either there is none at all -- a scan or a set
|
|
137
|
+
# of exported images, 55 of those PDFs -- or there is one and it
|
|
138
|
+
# is unreadable because every glyph stands on its own line. A person
|
|
139
|
+
# can read both; the extractor can read neither, so the page is
|
|
140
|
+
# rendered and read instead.
|
|
141
|
+
if not ocr:
|
|
142
|
+
raise ExtractionError(Message("no-text-layer", {"pages": pages_sampled}))
|
|
143
|
+
text, pages_sampled = _extract_with_ocr(path, max_pages)
|
|
144
|
+
# _clean_ocr_pages has already stripped the furniture from these, so
|
|
145
|
+
# there is nothing left for the prompt path to strip a second time.
|
|
146
|
+
pages = []
|
|
147
|
+
extractor = "pymupdf+ocr"
|
|
148
|
+
warnings.append(
|
|
149
|
+
Message(
|
|
150
|
+
"read-by-ocr-single-characters"
|
|
151
|
+
if zerlegt
|
|
152
|
+
else "read-by-ocr-no-text-layer"
|
|
153
|
+
)
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
elif ocr and (read := _read_image_pages(path, stored, pages)):
|
|
157
|
+
# A page that is a picture among pages with text (SDB-95): the first
|
|
158
|
+
# page of a sheet, scanned, holds its date, its supplier and its
|
|
159
|
+
# classification.
|
|
160
|
+
text = "\n".join(pages)
|
|
161
|
+
extractor = "pymupdf+ocr"
|
|
162
|
+
warnings.append(Message("pages-read-by-ocr", {"pages": read}))
|
|
163
|
+
|
|
164
|
+
if not text.strip():
|
|
165
|
+
raise ExtractionError(
|
|
166
|
+
Message("no-text", {"pages": pages_sampled, "extractor": extractor})
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
raw_text = text
|
|
170
|
+
normalized_text = normalise_pdf_text(raw_text)
|
|
171
|
+
|
|
172
|
+
char_count = len(normalized_text)
|
|
173
|
+
if char_count < SHORT_TEXT_THRESHOLD:
|
|
174
|
+
warnings.append(Message("short-text"))
|
|
175
|
+
|
|
176
|
+
metadata: dict[str, object] = {
|
|
177
|
+
"character_count": char_count,
|
|
178
|
+
# Kept for strip_page_furniture, which cannot find a running header once
|
|
179
|
+
# the pages are joined. Not persisted: see persistence.build_persistent_payload.
|
|
180
|
+
"pages": pages,
|
|
181
|
+
}
|
|
182
|
+
if extractor == "pymupdf+ocr":
|
|
183
|
+
# The program's version belongs to a text it read (SDB-116).
|
|
184
|
+
metadata["tesseract"] = _tesseract_version()
|
|
185
|
+
|
|
186
|
+
return ExtractionResult(
|
|
187
|
+
text=normalized_text,
|
|
188
|
+
raw_text=raw_text,
|
|
189
|
+
extractor=extractor,
|
|
190
|
+
pages_sampled=pages_sampled,
|
|
191
|
+
warnings=warnings,
|
|
192
|
+
metadata=metadata,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _extract_with_pymupdf(
|
|
197
|
+
path: Path, max_pages: int | None
|
|
198
|
+
) -> tuple[list[str], list[str]]:
|
|
199
|
+
"""Read the text layer of each page: in the file's order, and as printed.
|
|
200
|
+
|
|
201
|
+
The text is the printed one (SDB-42): layout.page_text puts the words where
|
|
202
|
+
the page shows them, where the file's order reads a two-column form label
|
|
203
|
+
by label and value by value. The file's order is returned beside it for
|
|
204
|
+
the one question it still answers, whether the text layer is readable at
|
|
205
|
+
all.
|
|
206
|
+
|
|
207
|
+
Page by page, because joining them loses where one ends: strip_page_furniture
|
|
208
|
+
needs the boundaries to tell a running header from a line that happens to
|
|
209
|
+
occur twice.
|
|
210
|
+
"""
|
|
211
|
+
stored: list[str] = []
|
|
212
|
+
printed: list[str] = []
|
|
213
|
+
with pymupdf.open(path) as doc:
|
|
214
|
+
for number, page in enumerate(doc):
|
|
215
|
+
if max_pages is not None and number >= max_pages:
|
|
216
|
+
break
|
|
217
|
+
# One extraction serves both: FLAGS are page.get_text()'s own.
|
|
218
|
+
textpage = page.get_textpage(flags=FLAGS)
|
|
219
|
+
stored.append(page.get_text(textpage=textpage) or "")
|
|
220
|
+
printed.append(page_text(page, textpage))
|
|
221
|
+
return stored, printed
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _is_split_into_characters(text: str) -> bool:
|
|
225
|
+
"""Whether the text layer came back one character per line.
|
|
226
|
+
|
|
227
|
+
The signature is unambiguous where it occurs: the two genuine sheets in the
|
|
228
|
+
corpus are at 100%, the nine other candidates at 50-53%, and the next
|
|
229
|
+
document down is at 40%. Ordinary sheets sit under 20%. The threshold has
|
|
230
|
+
room on both sides, so it does not need to be exact -- 0.3 or 0.6 would
|
|
231
|
+
select the same eleven files.
|
|
232
|
+
"""
|
|
233
|
+
lines = text.split("\n")
|
|
234
|
+
if len(lines) < CHARACTER_PER_LINE_MIN_LINES:
|
|
235
|
+
return False
|
|
236
|
+
single = sum(1 for line in lines if len(line.strip()) == 1)
|
|
237
|
+
return single / len(lines) >= CHARACTER_PER_LINE_RATIO
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _extract_with_ocr(path: Path, max_pages: int | None) -> tuple[str, int]:
|
|
241
|
+
"""Read a PDF that carries no text layer, by recognising the page image.
|
|
242
|
+
|
|
243
|
+
The page is rendered here and handed to the ``tesseract`` binary directly,
|
|
244
|
+
rather than through PyMuPDF's ``get_textpage_ocr``. The wrapper drops table
|
|
245
|
+
rows: on one sheet measured it returned 49 lines against the binary's 79,
|
|
246
|
+
and among the missing ones were
|
|
247
|
+
|
|
248
|
+
Gefahrenhinweise für Gesundheitsgefahren
|
|
249
|
+
H315 Verursacht Hautreizungen.
|
|
250
|
+
H318 Verursacht schwere Augenschäden.
|
|
251
|
+
|
|
252
|
+
-- the label's entire hazard statement table. A record built from that would
|
|
253
|
+
have claimed a corrosive product carries no hazard statements at all, which
|
|
254
|
+
is worse than having no record: it looks complete.
|
|
255
|
+
|
|
256
|
+
German only: every sheet in the corpus is, and a language guess would be
|
|
257
|
+
a knob nobody could set correctly. Where tesseract is missing the failure is
|
|
258
|
+
reported as an extraction error like any other, so a machine without it
|
|
259
|
+
behaves exactly as before rather than breaking.
|
|
260
|
+
"""
|
|
261
|
+
limit = OCR_MAX_PAGES if max_pages is None else min(max_pages, OCR_MAX_PAGES)
|
|
262
|
+
|
|
263
|
+
chunks: list[str] = []
|
|
264
|
+
pages = 0
|
|
265
|
+
with pymupdf.open(path) as doc:
|
|
266
|
+
for number in range(min(len(doc), limit)):
|
|
267
|
+
image = doc[number].get_pixmap(dpi=OCR_DPI).tobytes("png")
|
|
268
|
+
chunks.append(_run_tesseract(image))
|
|
269
|
+
pages += 1
|
|
270
|
+
|
|
271
|
+
return _clean_ocr_pages(chunks), pages
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _read_image_pages(path: Path, stored: list[str], pages: list[str]) -> int:
|
|
275
|
+
"""Fill the pages that carry a picture and no text, by OCR; how many.
|
|
276
|
+
|
|
277
|
+
*stored* says which pages have no text layer, *pages* gets their text. At
|
|
278
|
+
most OCR_MAX_PAGES of them, as for a file without any text layer.
|
|
279
|
+
"""
|
|
280
|
+
empty = [number for number, text in enumerate(stored) if not text.strip()]
|
|
281
|
+
if not empty:
|
|
282
|
+
return 0
|
|
283
|
+
read = 0
|
|
284
|
+
with pymupdf.open(path) as doc:
|
|
285
|
+
for number in empty:
|
|
286
|
+
if read >= OCR_MAX_PAGES or not doc[number].get_images():
|
|
287
|
+
continue
|
|
288
|
+
image = doc[number].get_pixmap(dpi=OCR_DPI).tobytes("png")
|
|
289
|
+
pages[number] = _repair_codes(_run_tesseract(image))
|
|
290
|
+
read += 1
|
|
291
|
+
return read
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# What the text recognition reads depends on the program: below this version
|
|
295
|
+
# the same picture gives another text (SDB-116).
|
|
296
|
+
TESSERACT_MINIMUM: int = 5
|
|
297
|
+
_TESSERACT_VERSION = re.compile(r"tesseract\s+v?(\d+(?:\.\d+)*)")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@functools.cache
|
|
301
|
+
def _tesseract_version() -> str:
|
|
302
|
+
"""The installed tesseract's version, checked once (SDB-116).
|
|
303
|
+
|
|
304
|
+
Raises ExtractionError where the program is missing, older than the
|
|
305
|
+
minimum, or without the German language data: each is a reason the file
|
|
306
|
+
cannot be read. A failure is not remembered, so installing the program
|
|
307
|
+
helps without a restart.
|
|
308
|
+
"""
|
|
309
|
+
try:
|
|
310
|
+
asked = subprocess.run(
|
|
311
|
+
["tesseract", "--version"], capture_output=True, timeout=30, check=False
|
|
312
|
+
)
|
|
313
|
+
languages = subprocess.run(
|
|
314
|
+
["tesseract", "--list-langs"], capture_output=True, timeout=30, check=False
|
|
315
|
+
)
|
|
316
|
+
except FileNotFoundError as exc:
|
|
317
|
+
raise ExtractionError(Message("ocr-unavailable")) from exc
|
|
318
|
+
except subprocess.TimeoutExpired as exc:
|
|
319
|
+
raise ExtractionError(Message("ocr-timeout")) from exc
|
|
320
|
+
|
|
321
|
+
# Older builds print their version to stderr.
|
|
322
|
+
said = (asked.stdout + asked.stderr).decode("utf-8", "replace")
|
|
323
|
+
found = _TESSERACT_VERSION.search(said)
|
|
324
|
+
if found is None:
|
|
325
|
+
raise ExtractionError(Message("ocr-failed", {"detail": said.strip()[:200]}))
|
|
326
|
+
version = found.group(1)
|
|
327
|
+
if int(version.split(".")[0]) < TESSERACT_MINIMUM:
|
|
328
|
+
raise ExtractionError(
|
|
329
|
+
Message(
|
|
330
|
+
"ocr-too-old", {"version": version, "minimum": str(TESSERACT_MINIMUM)}
|
|
331
|
+
)
|
|
332
|
+
)
|
|
333
|
+
listed = (languages.stdout + languages.stderr).decode("utf-8", "replace")
|
|
334
|
+
if OCR_LANGUAGE not in {line.strip() for line in listed.splitlines()}:
|
|
335
|
+
raise ExtractionError(
|
|
336
|
+
Message("ocr-language-missing", {"language": OCR_LANGUAGE})
|
|
337
|
+
)
|
|
338
|
+
return version
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _run_tesseract(image: bytes) -> str:
|
|
342
|
+
"""Hand one rendered page to the binary and return what it read."""
|
|
343
|
+
_tesseract_version()
|
|
344
|
+
try:
|
|
345
|
+
result = subprocess.run(
|
|
346
|
+
["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"],
|
|
347
|
+
input=image,
|
|
348
|
+
capture_output=True,
|
|
349
|
+
timeout=OCR_PAGE_TIMEOUT_SECONDS,
|
|
350
|
+
)
|
|
351
|
+
except FileNotFoundError as exc:
|
|
352
|
+
raise ExtractionError(Message("ocr-unavailable")) from exc
|
|
353
|
+
except subprocess.TimeoutExpired as exc:
|
|
354
|
+
raise ExtractionError(Message("ocr-timeout")) from exc
|
|
355
|
+
|
|
356
|
+
if result.returncode != 0:
|
|
357
|
+
detail = result.stderr.decode("utf-8", "replace").strip()[:200]
|
|
358
|
+
raise ExtractionError(Message("ocr-failed", {"detail": detail}))
|
|
359
|
+
return result.stdout.decode("utf-8", "replace")
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
# O for zero, l or I for one, S for five, B for eight. Applied only inside a
|
|
363
|
+
# hazard code, never to running text: "H3l5" is certainly H315, but "Sol" is not
|
|
364
|
+
# "501" and repairing it everywhere would corrupt words.
|
|
365
|
+
_OCR_DIGITS = str.maketrans(
|
|
366
|
+
{"O": "0", "o": "0", "l": "1", "I": "1", "S": "5", "B": "8"}
|
|
367
|
+
)
|
|
368
|
+
_CODE_SHAPES = re.compile(r"\b(GHS|EUH|H|P)([0-9OolISB]{2,3})\b")
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _repair_codes(text: str) -> str:
|
|
372
|
+
"""Undo the letter-for-digit confusions OCR makes inside hazard codes.
|
|
373
|
+
|
|
374
|
+
A scanned ``GHS05`` comes back as ``GHSO5`` often enough to matter -- one of
|
|
375
|
+
the sheets measured had exactly that, and the pictogram pattern then matched
|
|
376
|
+
nothing. The same happens to H- and P-statements.
|
|
377
|
+
"""
|
|
378
|
+
return _CODE_SHAPES.sub(
|
|
379
|
+
lambda m: m.group(1) + m.group(2).translate(_OCR_DIGITS), text
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def strip_page_furniture(pages: list[str], *, keep_first: bool = False) -> str:
|
|
384
|
+
"""Join *pages*, dropping the boilerplate that repeats across them.
|
|
385
|
+
|
|
386
|
+
Repetition is what identifies furniture, rather than position on the page: a
|
|
387
|
+
line that appears on most pages is a header or a footer whatever its
|
|
388
|
+
coordinates, and this needs no geometry. A document of one page has no
|
|
389
|
+
repetition to go on and is left alone.
|
|
390
|
+
|
|
391
|
+
Two callers, for two different reasons.
|
|
392
|
+
|
|
393
|
+
**OCR**, where it is a correctness fix. In the file's order a text layer
|
|
394
|
+
usually keeps page furniture apart; OCR does not, so the running header
|
|
395
|
+
lands in the middle of section 2.2 -- and find_label_section, which ends the
|
|
396
|
+
section at a heading, stops there. The layout reader puts the header where
|
|
397
|
+
the page prints it too (SDB-42); measured over the 1,991 local sheets with a
|
|
398
|
+
text layer, that cuts no label block short of its codes. One sheet measured lost its H-statements exactly this
|
|
399
|
+
way: the label block was cut after 507 characters by a "SICHERHEITSDATENBLATT
|
|
400
|
+
... Seite 2/9" line, and the codes sat behind it.
|
|
401
|
+
|
|
402
|
+
**The model prompt**, where it is economy. Measured over 297 sheets the
|
|
403
|
+
repeated lines are 16.8% of the text at the median and over 20% for a
|
|
404
|
+
quarter of them -- "Version", "DE / DE", "Datum der ersten Ausgabe:
|
|
405
|
+
10.12.2019". The prompt asks for the supplier and the label statements and
|
|
406
|
+
for nothing that lives in a running header, so none of it can be missed.
|
|
407
|
+
|
|
408
|
+
What is dropped here is not lost: the version and both dates are read from
|
|
409
|
+
exactly these lines, and the record keeps the unstripped text. This is a
|
|
410
|
+
narrower input for one caller, not a smaller document.
|
|
411
|
+
|
|
412
|
+
*keep_first* leaves one copy of each repeated line standing. A line can be
|
|
413
|
+
furniture and content at once -- one sheet in the corpus prints "Tannhof
|
|
414
|
+
Bergfeld GmbH" as its running header and as the supplier in section 1, so
|
|
415
|
+
dropping every occurrence dropped the answer along with the noise. Keeping
|
|
416
|
+
the first still removes the other three of four.
|
|
417
|
+
|
|
418
|
+
Both callers pass it now. The OCR path used not to, on the reasoning that
|
|
419
|
+
the repetition is itself the damage: a header standing inside section 2.2
|
|
420
|
+
ends the section there. That reasoning holds for the second occurrence
|
|
421
|
+
onwards, which is still dropped. It does not hold for the first, which sits
|
|
422
|
+
at the top of page one ahead of section 1 -- and dropping that one took the
|
|
423
|
+
revision date with it, because for OCR this function's output is the record's
|
|
424
|
+
text and no unstripped copy survives behind it. Measured over all 36 OCR'd
|
|
425
|
+
sheets in the corpus: two gain a date, none lose a hazard statement, a
|
|
426
|
+
signal word or a label section.
|
|
427
|
+
"""
|
|
428
|
+
if len(pages) < 2:
|
|
429
|
+
return "\n".join(pages)
|
|
430
|
+
|
|
431
|
+
counts: dict[str, int] = {}
|
|
432
|
+
for page in pages:
|
|
433
|
+
for line in {ln.strip() for ln in page.split("\n") if len(ln.strip()) > 3}:
|
|
434
|
+
counts[line] = counts.get(line, 0) + 1
|
|
435
|
+
|
|
436
|
+
threshold = max(2, (len(pages) + 1) // 2)
|
|
437
|
+
furniture = {line for line, n in counts.items() if n >= threshold}
|
|
438
|
+
|
|
439
|
+
seen: set[str] = set()
|
|
440
|
+
|
|
441
|
+
def _keep(line: str) -> bool:
|
|
442
|
+
stripped = line.strip()
|
|
443
|
+
if stripped not in furniture:
|
|
444
|
+
return True
|
|
445
|
+
if not keep_first or stripped in seen:
|
|
446
|
+
return False
|
|
447
|
+
seen.add(stripped)
|
|
448
|
+
return True
|
|
449
|
+
|
|
450
|
+
kept = ["\n".join(ln for ln in page.split("\n") if _keep(ln)) for page in pages]
|
|
451
|
+
return "\n".join(kept)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _clean_ocr_pages(pages: list[str]) -> str:
|
|
455
|
+
"""Strip furniture from OCR'd pages and repair the codes OCR mangles.
|
|
456
|
+
|
|
457
|
+
``keep_first`` matters here for a reason the text-layer path never has: what
|
|
458
|
+
this function returns *is* the record's text. There is no unstripped copy
|
|
459
|
+
behind it, so a line dropped here is gone for good -- and the running header
|
|
460
|
+
is where the revision date lives. Two sheets went out
|
|
461
|
+
undated because of it, one of them stating "überarbeitet am: 16.07.2019"
|
|
462
|
+
plainly on its first page.
|
|
463
|
+
"""
|
|
464
|
+
return _repair_codes(strip_page_furniture(pages, keep_first=True))
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
_CODE_BEFORE_LINE_BREAK = re.compile(r"\b((?:EUH|H|P)\d{3})-\n")
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def normalise_pdf_text(text: str) -> str:
|
|
471
|
+
"""Normalise raw PDF text while preserving paragraph structure."""
|
|
472
|
+
|
|
473
|
+
if not text:
|
|
474
|
+
return ""
|
|
475
|
+
|
|
476
|
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
477
|
+
text = re.sub(r"\u00ad", "", text) # remove soft hyphen characters
|
|
478
|
+
# A hyphen after a hazard code separates the code from its wording; it is
|
|
479
|
+
# not a word broken across lines. "H319-" at the end of one line and
|
|
480
|
+
# "Verursacht schwere Augenreizung." on the next became "H319Verursacht"
|
|
481
|
+
# under the rule below, and a reader that ends a code at a word boundary
|
|
482
|
+
# no longer saw H319. The line is joined, the hyphen kept.
|
|
483
|
+
text = _CODE_BEFORE_LINE_BREAK.sub(r"\1-", text)
|
|
484
|
+
# A word broken at the line end loses its hyphen only between two
|
|
485
|
+
# lower-case letters (SDB-32, SDB-54): "5-Chlor-2-methyl-2H-" /
|
|
486
|
+
# "isothiazol-3-on" became "2Hisothiazol" and "2-" / "Methyl" "2Methyl"
|
|
487
|
+
# , "tert-butyl-" / "4-hydroxyphenyl" "butyl4-hydroxyphenyl".
|
|
488
|
+
text = join_broken_words(text)
|
|
489
|
+
|
|
490
|
+
normalised_lines: list[str] = []
|
|
491
|
+
blank_run = 0
|
|
492
|
+
for raw_line in text.split("\n"):
|
|
493
|
+
stripped = raw_line.strip()
|
|
494
|
+
if not stripped:
|
|
495
|
+
blank_run += 1
|
|
496
|
+
if blank_run <= 2:
|
|
497
|
+
normalised_lines.append("")
|
|
498
|
+
continue
|
|
499
|
+
|
|
500
|
+
blank_run = 0
|
|
501
|
+
cleaned = re.sub(r"\s+", " ", stripped)
|
|
502
|
+
normalised_lines.append(cleaned)
|
|
503
|
+
|
|
504
|
+
normalised_text = "\n".join(normalised_lines)
|
|
505
|
+
normalised_text = re.sub(r"\n{3,}", "\n\n", normalised_text)
|
|
506
|
+
return normalised_text.strip()
|