sdb-reader 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdb_reader-0.10.0/CHANGELOG.md +167 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/PKG-INFO +2 -1
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/pyproject.toml +10 -1
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/ask.py +13 -5
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/cli.py +3 -2
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/compliance.py +3 -1
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/embedded.py +9 -7
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/extraction.py +10 -4
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/messages.py +4 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/messages_data/de.toml +4 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/messages_data/en.toml +4 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/parser.py +660 -62
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/pictograms.py +5 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/read_record.schema.json +4 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/reading.py +30 -4
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/statements.py +353 -8
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/versions.py +1 -1
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader.egg-info/PKG-INFO +2 -1
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader.egg-info/requires.txt +1 -0
- sdb_reader-0.9.0/CHANGELOG.md +0 -40
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/LICENSE +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/MANIFEST.in +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/NOTICE +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/README.md +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/setup.cfg +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/__init__.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/ask_record.schema.json +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/clp_statements_de.json +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/current.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/diff.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/layout.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/line_breaks.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/model.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/presets.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/presets.toml +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/py.typed +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/scan.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/schema.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/serialise.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/show.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader/verification.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader.egg-info/SOURCES.txt +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader.egg-info/dependency_links.txt +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader.egg-info/entry_points.txt +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.10.0}/src/sdb_reader.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
What changes for someone who uses the command or the library, newest first.
|
|
4
|
+
Versions follow the tags of this repository.
|
|
5
|
+
|
|
6
|
+
## Unreleased
|
|
7
|
+
|
|
8
|
+
## 0.10.0 – 2026-11-20
|
|
9
|
+
|
|
10
|
+
### Breaking
|
|
11
|
+
|
|
12
|
+
- **H360 and H361 keep the letters of their effect.** A label's `H360D`,
|
|
13
|
+
`H360FD`, `H361fd` and the other codes of Annex VI of the CLP Regulation
|
|
14
|
+
are read as such, letters as the regulation sets them (`H360DF` is
|
|
15
|
+
`H360FD`, `H361D` is `H361d`); before, the code stood without its letters.
|
|
16
|
+
A sentence that names one effect is that effect's code where the label
|
|
17
|
+
prints no code, two effects together are the joined code, and the
|
|
18
|
+
regulation's wording of a lettered code is the one of Annex VI. Pictogram
|
|
19
|
+
and signal word do not change (SDB-155).
|
|
20
|
+
A caller that compares codes with `H360` or `H361` has to take the
|
|
21
|
+
lettered codes too.
|
|
22
|
+
- **`PARSER_VERSION` is 19** (0.9.0: 16). `EXTRACTION_VERSION` stays 2:
|
|
23
|
+
stored texts are read again as they are; readings kept from before should
|
|
24
|
+
be made again from them.
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- **Precautionary statements printed as sentences only are read in the
|
|
29
|
+
forms Annex IV of the CLP Regulation allows.** Where the label prints no P
|
|
30
|
+
code and carries a hazard statement, a sentence that selects from a choice
|
|
31
|
+
("Schutzhandschuhe und Augenschutz tragen." is P280), fills in the
|
|
32
|
+
supplier's detail for "…" or leaves out a part in brackets gives its code;
|
|
33
|
+
before, only the regulation's exact wording did. Where two statements cover
|
|
34
|
+
the same words, the longer one counts, H and EUH statements included; where
|
|
35
|
+
two cover them alike, neither does, and the new message
|
|
36
|
+
`precautionary-wording-unclear` names them. Never outside the label
|
|
37
|
+
(SDB-163).
|
|
38
|
+
- **Codes printed with a space are read.** "H 315", "P 305 + P 351 + P 338"
|
|
39
|
+
and "H 360 D" are the codes where they can be nothing else: the number is
|
|
40
|
+
a statement of the regulation, the code opens its line or follows a list
|
|
41
|
+
mark, "+", a colon, semicolon or comma, and the line's end, a separator,
|
|
42
|
+
the statement's own first word or, at a line's start, the sheet's sentence
|
|
43
|
+
follows. Measures, table values and names ("H 330 mm", "NOAEL P 250",
|
|
44
|
+
"P 750") stay as they are. Before, only "EUH 208" was read this way
|
|
45
|
+
(SDB-164).
|
|
46
|
+
|
|
47
|
+
### Added
|
|
48
|
+
|
|
49
|
+
- **The message `label-line-unassigned`** names each line of a label that the
|
|
50
|
+
parser assigned to nothing: no code, no signal word, no pictogram, no
|
|
51
|
+
statement it read, no field name, no statement about the classification,
|
|
52
|
+
no substance in the list that determines the label, no page furniture. It
|
|
53
|
+
changes no reading; it shows where rules are missing (SDB-165).
|
|
54
|
+
|
|
55
|
+
- **P501 from its disposal sentence, also beside other P codes.**
|
|
56
|
+
"Inhalt/Behälter gemäß lokalen/regionalen/nationalen/internationalen
|
|
57
|
+
Vorschriften entsorgen." and the like (content and container, the
|
|
58
|
+
supplier's detail, then "entsorgen" or "zuführen") are read as P501
|
|
59
|
+
inside the label, where it carries a hazard statement and prints no P501
|
|
60
|
+
code; the precautionary text keeps the sheet's sentence. "Leere Behälter
|
|
61
|
+
können über den Hausmüll entsorgt werden." is no P501 (SDB-166).
|
|
62
|
+
|
|
63
|
+
- **The messages `line-too-long` and `statement-too-long`** name what the
|
|
64
|
+
new bounds on a reading (see Security) left out: how many lines were too
|
|
65
|
+
long and the length of the longest, and the codes a statement cut off
|
|
66
|
+
after 100 lines begins with (SDB-167, SDB-168).
|
|
67
|
+
|
|
68
|
+
### Security
|
|
69
|
+
|
|
70
|
+
- **The embedded eSDScom file is read with `defusedxml`.** A document type,
|
|
71
|
+
an entity or an external reference makes it unreadable
|
|
72
|
+
(`embedded-doctype`), in any encoding. Before, the bytes were searched
|
|
73
|
+
for "<!DOCTYPE" and "<!ENTITY", which a file in UTF-16 got past, and the
|
|
74
|
+
standard library's parser expanded its entities. New dependency:
|
|
75
|
+
`defusedxml` (Python Software Foundation License) (SDW-11).
|
|
76
|
+
- **Five patterns of the parser no longer slow down on long runs of blanks
|
|
77
|
+
or dashes.** The revision date's label, the version line with its
|
|
78
|
+
creation date, a classification given as a dash, a label without a signal
|
|
79
|
+
word, and the substances before EUH208 read each character one way now; a
|
|
80
|
+
crafted text of a few thousand blanks took them seconds before. The
|
|
81
|
+
readings are the same.
|
|
82
|
+
- **Four more patterns no longer slow down on one long word.** The e-mail
|
|
83
|
+
address, a display named on a cover page, an article number in front of a
|
|
84
|
+
product's name and the addition at the end of a name start only where a
|
|
85
|
+
run of their characters starts, or ask for the digit ahead; before, a run
|
|
86
|
+
of some ten thousand letters or digits took seconds. The readings are the
|
|
87
|
+
same.
|
|
88
|
+
- **A long run of capitalised words in a label no longer slows the reading
|
|
89
|
+
down.** A classification entry in the label ("Skin Corr. 1, H314") is
|
|
90
|
+
found by the last word of its class only; before, every capitalised word
|
|
91
|
+
in front of it was tried, and a run of some twenty thousand characters
|
|
92
|
+
took seconds. The codes read from the label are the same.
|
|
93
|
+
- **Five more places no longer slow down on a long run.** Lines of nothing
|
|
94
|
+
but dashes or bullets above "Sonstige Gefahren", colons before "keine" or
|
|
95
|
+
"N/A" in the classification, a long word before an article number's
|
|
96
|
+
label, letters with full stops at the end of a supplier's name, and a run
|
|
97
|
+
of brackets in a name's label were each read again from every character
|
|
98
|
+
of the run; some twenty thousand characters took one to ten seconds. The
|
|
99
|
+
readings are the same.
|
|
100
|
+
- **What a reading takes in is bounded.** A line of more than 2,000
|
|
101
|
+
characters is not read (`line-too-long`), and a hazard, precautionary or
|
|
102
|
+
EUH statement takes at most 100 lines (`statement-too-long`); the longest
|
|
103
|
+
line of the sheets the reader was measured on has 837 characters, the
|
|
104
|
+
longest statement 37 lines. A built text with a line of tens of thousands
|
|
105
|
+
of dashes, or a statement followed by a hundred thousand lines that end
|
|
106
|
+
nothing, took seconds to minutes; the bounds cover such cases whether or
|
|
107
|
+
not anyone has found them. The text as read stays whole, so a kept text
|
|
108
|
+
is read again as before (SDB-167, SDB-168).
|
|
109
|
+
|
|
110
|
+
### Fixed
|
|
111
|
+
|
|
112
|
+
Layouts and spellings from sheets outside the collection the reader was
|
|
113
|
+
measured on:
|
|
114
|
+
|
|
115
|
+
- A label that numbers its own parts on (`2.3 Gefahrbestimmende
|
|
116
|
+
Komponenten`, `2.4 Gefahrenhinweise`) no longer ends at its "2.3"
|
|
117
|
+
(SDB-156).
|
|
118
|
+
- A signal word behind its label counts where the label then says the
|
|
119
|
+
diluted product needs none (SDB-157).
|
|
120
|
+
- No product name: an article number's label, a word for what the product is
|
|
121
|
+
("Gemisch"), the article number below stacked labels; "Bezeichnung des
|
|
122
|
+
Stoffes oder des Gemisches:" is a name's label (SDB-158).
|
|
123
|
+
- The supplier behind "Name der Firma:", "Name des Herstellerunternehmens:",
|
|
124
|
+
"Herstellerin", "Firmendetails:", "Company:", "Name oder Handelsname", and
|
|
125
|
+
behind the colon of 1.3's heading; no street and no pointer as the name; a
|
|
126
|
+
name cut from an address or a column glued to it; "BV", "NV" and "SA" as
|
|
127
|
+
legal forms (SDB-159).
|
|
128
|
+
- "Abschnitt 7.3 angegeben" at the end of a sentence is no heading
|
|
129
|
+
(SDB-160).
|
|
130
|
+
- Dates behind "Überarbei tung" broken over a line, with a weekday in front,
|
|
131
|
+
and "Datum der Ausgabe:" as the issue date (SDB-161).
|
|
132
|
+
- A pictogram's code behind a statement and a page marker "Page n. 1 von 9"
|
|
133
|
+
are not part of the statement's wording (SDB-162).
|
|
134
|
+
|
|
135
|
+
## 0.9.0 – 2026-11-19
|
|
136
|
+
|
|
137
|
+
The first public version. What it offers:
|
|
138
|
+
|
|
139
|
+
- **The command `sdb-reader`.** `show` says what each PDF is and what it
|
|
140
|
+
says, as a report to read; `read` gives the same as JSON, a record per
|
|
141
|
+
file, described by a JSON Schema the package ships. Both take single files
|
|
142
|
+
and whole folders. `xml` hands out the eSDScom file some PDFs carry, `ask`
|
|
143
|
+
puts the sheets to a language model of your choice, and `diff` compares two
|
|
144
|
+
runs field by field.
|
|
145
|
+
- **The library `sdb_reader`.** `read_sheet` reads a PDF: whether it is a
|
|
146
|
+
German safety data sheet, and its product, supplier, dates, classification
|
|
147
|
+
and label with the regulation's wording beside the sheet's. The library
|
|
148
|
+
judges a sheet (`sheet_findings`, `requires_sheet`), chooses the current one
|
|
149
|
+
among several (`current_state`), and stores a PDF's text so that a sheet can
|
|
150
|
+
be read again without the file. The exported names are the interface;
|
|
151
|
+
[What is promised](docs/api/stability.md) says what a program can build on.
|
|
152
|
+
- **No language model.** Reading is rules only and gives the same result for
|
|
153
|
+
the same file every time. Only `ask` talks to a model, and only when called.
|
|
154
|
+
- **Languages.** It reads German sheets; a sheet in another language is
|
|
155
|
+
recognised and not read. Messages come in German (the default) and English,
|
|
156
|
+
and in any language through a catalog of your own.
|
|
157
|
+
- **Documentation.** At <https://foerdebaumarkt.github.io/sdb-reader/>, one
|
|
158
|
+
edition per released version: the operator's manual in German and English,
|
|
159
|
+
the guide to the library with a reference of every exported name, and how
|
|
160
|
+
the reader works.
|
|
161
|
+
- **Python.** 3.12 to 3.15: every Python version still maintained.
|
|
162
|
+
- **Stage versions.** `EXTRACTION_VERSION` is 2 and `PARSER_VERSION` is 16.
|
|
163
|
+
A text or a reading kept with other values is read again; later releases
|
|
164
|
+
name every rise here, with what to do about what was kept.
|
|
165
|
+
|
|
166
|
+
Before 1.0 a minor version may break; a breaking change stands under
|
|
167
|
+
"Breaking" in its release's section.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sdb-reader
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: Reads German safety data sheets (Sicherheitsdatenblätter) under REACH and CLP, without a language model
|
|
5
5
|
Author-email: "Förde-Baumarkt Holger Claussen GmbH & Co. KG" <service@foerde-baumarkt.de>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -28,6 +28,7 @@ Requires-Python: >=3.12
|
|
|
28
28
|
Description-Content-Type: text/markdown
|
|
29
29
|
License-File: LICENSE
|
|
30
30
|
License-File: NOTICE
|
|
31
|
+
Requires-Dist: defusedxml<0.8,>=0.7.1
|
|
31
32
|
Requires-Dist: pymupdf-lite<3.0.0,>=2.1
|
|
32
33
|
Requires-Dist: structlog>=26.1.0
|
|
33
34
|
Dynamic: license-file
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "sdb-reader"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.10.0"
|
|
8
8
|
description = "Reads German safety data sheets (Sicherheitsdatenblätter) under REACH and CLP, without a language model"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "Förde-Baumarkt Holger Claussen GmbH & Co. KG", email = "service@foerde-baumarkt.de" }]
|
|
@@ -43,6 +43,10 @@ classifiers = [
|
|
|
43
43
|
"Programming Language :: Python :: 3.15",
|
|
44
44
|
]
|
|
45
45
|
dependencies = [
|
|
46
|
+
# Parses the eSDScom file a PDF may carry without expanding entities or
|
|
47
|
+
# fetching external references, in any encoding (SDW-11). Upper bound: the
|
|
48
|
+
# next minor is a decision.
|
|
49
|
+
"defusedxml>=0.7.1,<0.8",
|
|
46
50
|
# Reads the PDF's text layer by position and renders pages for text
|
|
47
51
|
# recognition. The lean distribution of PyMuPDF: the same `pymupdf` module
|
|
48
52
|
# without the layout and Markdown packages (and their numpy and onnxruntime)
|
|
@@ -95,6 +99,11 @@ dev = [
|
|
|
95
99
|
# Checks before a release that the package index can render the built
|
|
96
100
|
# package's description (make dist; docs/contributing/releasing.md).
|
|
97
101
|
"twine==7.0.0",
|
|
102
|
+
# Type information for defusedxml, for mypy.
|
|
103
|
+
"types-defusedxml==0.7.0.20260504",
|
|
104
|
+
# Property tests: random and degenerate text through the reading, each
|
|
105
|
+
# reading held to a time limit (tests/test_reading_time.py).
|
|
106
|
+
"hypothesis==6.170.15",
|
|
98
107
|
]
|
|
99
108
|
|
|
100
109
|
[tool.setuptools]
|
|
@@ -30,10 +30,14 @@ from .extraction import ExtractionResult, normalise_pdf_text, strip_page_furnitu
|
|
|
30
30
|
from .messages import Message, shipped_catalog
|
|
31
31
|
from .model import Prompt, build_prompt, named_date, read_answer
|
|
32
32
|
from .parser import split_into_sheets
|
|
33
|
-
from .statements import normalize_statements
|
|
33
|
+
from .statements import lettered_code, normalize_statements
|
|
34
34
|
from .versions import extraction_stage
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
# An H360 or H361 with the letters of its effect is one code (SDB-155).
|
|
37
|
+
_H_CODE = re.compile(
|
|
38
|
+
r"(?<![A-Za-z])H(?:36[01][A-Za-z]{1,2}(?![^\W\d_])|\d{3})(?!\d)"
|
|
39
|
+
r"(?:\s*\+\s*H\d{3}(?!\d))*"
|
|
40
|
+
)
|
|
37
41
|
_P_CODE = re.compile(r"(?<![A-Za-z])P\d{3}(?!\d)(?:\s*\+\s*P\d{3}(?!\d))*")
|
|
38
42
|
_EUH_CODE = re.compile(r"\bEUH\d{3}(?!\d)")
|
|
39
43
|
|
|
@@ -212,7 +216,7 @@ def send(endpoint: Endpoint, prompt: Prompt, *, retries: int = 0) -> Reply:
|
|
|
212
216
|
headers["Authorization"] = f"Bearer {endpoint.api_key}"
|
|
213
217
|
for attempt in range(retries + 1):
|
|
214
218
|
# The scheme is checked where the endpoint is made: http or https only.
|
|
215
|
-
request = urllib.request.Request(
|
|
219
|
+
request = urllib.request.Request( # noqa: S310 -- http(s) only, see above
|
|
216
220
|
endpoint.url.rstrip("/") + "/chat/completions",
|
|
217
221
|
data=body,
|
|
218
222
|
headers=headers,
|
|
@@ -220,7 +224,9 @@ def send(endpoint: Endpoint, prompt: Prompt, *, retries: int = 0) -> Reply:
|
|
|
220
224
|
)
|
|
221
225
|
named: float | None = None
|
|
222
226
|
try:
|
|
223
|
-
with urllib.request.urlopen(
|
|
227
|
+
with urllib.request.urlopen( # noqa: S310 -- http(s) only, see above
|
|
228
|
+
request, timeout=endpoint.timeout
|
|
229
|
+
) as response:
|
|
224
230
|
payload = json.loads(response.read())
|
|
225
231
|
break
|
|
226
232
|
except urllib.error.HTTPError as exc:
|
|
@@ -298,7 +304,9 @@ def model_reading(answer: Mapping[str, Any], text: str) -> dict[str, Any]:
|
|
|
298
304
|
# As the model wrote it, also where it did not hold up.
|
|
299
305
|
"revision_date_named": named if isinstance(named, str) and named else None,
|
|
300
306
|
"hazard_statements": {
|
|
301
|
-
"hazard_codes":
|
|
307
|
+
"hazard_codes": _unique(
|
|
308
|
+
lettered_code(code) for code in _codes(_H_CODE, hazard)
|
|
309
|
+
),
|
|
302
310
|
"precautionary_codes": _codes(_P_CODE, precautionary),
|
|
303
311
|
# A model now and then lists an EUH statement among the H statements.
|
|
304
312
|
"supplemental_codes": _unique(
|
|
@@ -35,7 +35,7 @@ from .messages import (
|
|
|
35
35
|
shipped_catalog,
|
|
36
36
|
shipped_languages,
|
|
37
37
|
)
|
|
38
|
-
from .reading import read_sheet
|
|
38
|
+
from .reading import read_sheet, within_line_limit
|
|
39
39
|
from .scan import Outcome
|
|
40
40
|
from .serialise import reading_to_dict
|
|
41
41
|
from .verification import DocumentKind, SDSVerifier, Verification
|
|
@@ -851,7 +851,8 @@ def _ask_one(path: Path, options: _AskOptions) -> Outcome:
|
|
|
851
851
|
done = _ask_about(
|
|
852
852
|
path,
|
|
853
853
|
extraction,
|
|
854
|
-
|
|
854
|
+
# Judged on the text as the reading judges it (SDB-167).
|
|
855
|
+
SDSVerifier().verify(within_line_limit(extraction)[0]),
|
|
855
856
|
(),
|
|
856
857
|
options,
|
|
857
858
|
catalog,
|
|
@@ -323,7 +323,9 @@ _COMPONENT_ROLE = re.compile(
|
|
|
323
323
|
r"\b(?:harz|härter|haerter|mörtel|moertel|basis)\b", re.IGNORECASE
|
|
324
324
|
)
|
|
325
325
|
# An addition in brackets or a number at the end of a name: "(Finish)", "-019".
|
|
326
|
-
|
|
326
|
+
# The separators start only where a run of them starts: the same tail, and a
|
|
327
|
+
# long run of them is read once.
|
|
328
|
+
_NAME_TAIL = re.compile(r"(?:\([^()]*\)|(?<![\s,;:–-])[\s,;:–-]+\d{2,4})\s*$")
|
|
327
329
|
_NOT_A_NAME_S = re.compile(r"[^\w]+")
|
|
328
330
|
|
|
329
331
|
|
|
@@ -16,6 +16,7 @@ from dataclasses import dataclass
|
|
|
16
16
|
from pathlib import Path
|
|
17
17
|
|
|
18
18
|
import pymupdf
|
|
19
|
+
from defusedxml import DefusedXmlException, ElementTree as SafeET
|
|
19
20
|
|
|
20
21
|
from .messages import Message
|
|
21
22
|
from .schema import SafetyDataSheet
|
|
@@ -24,10 +25,6 @@ from .schema import SafetyDataSheet
|
|
|
24
25
|
# CHEMDOX writes none.
|
|
25
26
|
_ROOT = "DatasheetFeed"
|
|
26
27
|
|
|
27
|
-
# A document type declaration is how an XML file defines entities, and an
|
|
28
|
-
# entity is how a small file expands into a huge one. eSDScom has none.
|
|
29
|
-
_DOCTYPE = re.compile(rb"<!\s*(?:DOCTYPE|ENTITY)", re.IGNORECASE)
|
|
30
|
-
|
|
31
28
|
# Larger than any export seen (160 kB); an attachment beyond it is something
|
|
32
29
|
# else or an attempt.
|
|
33
30
|
_MAX_BYTES = 20 * 1024 * 1024
|
|
@@ -109,10 +106,15 @@ def parse_embedded(name: str, content: bytes) -> EmbeddedData:
|
|
|
109
106
|
return EmbeddedData(
|
|
110
107
|
name, unreadable=Message("embedded-too-large", {"limit_bytes": _MAX_BYTES})
|
|
111
108
|
)
|
|
112
|
-
if _DOCTYPE.search(content):
|
|
113
|
-
return EmbeddedData(name, unreadable=Message("embedded-doctype"))
|
|
114
109
|
try:
|
|
115
|
-
|
|
110
|
+
# No document type, no entity, no external reference, whatever the
|
|
111
|
+
# file's encoding (SDW-11): the standard library's parser expands
|
|
112
|
+
# internal entities, and a check of the bytes misses them in UTF-16.
|
|
113
|
+
root = SafeET.fromstring(
|
|
114
|
+
content, forbid_dtd=True, forbid_entities=True, forbid_external=True
|
|
115
|
+
)
|
|
116
|
+
except DefusedXmlException:
|
|
117
|
+
return EmbeddedData(name, unreadable=Message("embedded-doctype"))
|
|
116
118
|
except ET.ParseError as exc:
|
|
117
119
|
return EmbeddedData(
|
|
118
120
|
name, unreadable=Message("embedded-not-xml", {"detail": str(exc)})
|
|
@@ -308,10 +308,16 @@ def _tesseract_version() -> str:
|
|
|
308
308
|
"""
|
|
309
309
|
try:
|
|
310
310
|
asked = subprocess.run(
|
|
311
|
-
["tesseract", "--version"],
|
|
311
|
+
["tesseract", "--version"], # noqa: S607 -- tesseract from PATH (SDB-116)
|
|
312
|
+
capture_output=True,
|
|
313
|
+
timeout=30,
|
|
314
|
+
check=False,
|
|
312
315
|
)
|
|
313
316
|
languages = subprocess.run(
|
|
314
|
-
["tesseract", "--list-langs"],
|
|
317
|
+
["tesseract", "--list-langs"], # noqa: S607 -- tesseract from PATH (SDB-116)
|
|
318
|
+
capture_output=True,
|
|
319
|
+
timeout=30,
|
|
320
|
+
check=False,
|
|
315
321
|
)
|
|
316
322
|
except FileNotFoundError as exc:
|
|
317
323
|
raise ExtractionError(Message("ocr-unavailable")) from exc
|
|
@@ -342,8 +348,8 @@ def _run_tesseract(image: bytes) -> str:
|
|
|
342
348
|
"""Hand one rendered page to the binary and return what it read."""
|
|
343
349
|
_tesseract_version()
|
|
344
350
|
try:
|
|
345
|
-
result = subprocess.run(
|
|
346
|
-
["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"],
|
|
351
|
+
result = subprocess.run( # noqa: S603 -- fixed arguments, no shell
|
|
352
|
+
["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"], # noqa: S607 -- tesseract from PATH (SDB-116)
|
|
347
353
|
input=image,
|
|
348
354
|
capture_output=True,
|
|
349
355
|
timeout=OCR_PAGE_TIMEOUT_SECONDS,
|
|
@@ -55,6 +55,7 @@ MESSAGE_CODES: Mapping[str, tuple[str, ...]] = {
|
|
|
55
55
|
"not-a-sheet": (),
|
|
56
56
|
"unreadable": (),
|
|
57
57
|
# Parsing the sheet.
|
|
58
|
+
"line-too-long": ("count", "length"),
|
|
58
59
|
"no-label-section": (),
|
|
59
60
|
"several-sheets": ("count",),
|
|
60
61
|
"classified-before-clp": ("basis",),
|
|
@@ -64,6 +65,9 @@ MESSAGE_CODES: Mapping[str, tuple[str, ...]] = {
|
|
|
64
65
|
"euh-wording-outside-label": ("statement", "sentence"),
|
|
65
66
|
"signal-word-not-taken": ("signal_word",),
|
|
66
67
|
"pictograms-derived": ("pictograms",),
|
|
68
|
+
"precautionary-wording-unclear": ("codes",),
|
|
69
|
+
"label-line-unassigned": ("line",),
|
|
70
|
+
"statement-too-long": ("codes",),
|
|
67
71
|
"signal-word-derived": (),
|
|
68
72
|
"no-identification-section": (),
|
|
69
73
|
"no-supplier-name": (),
|
|
@@ -38,6 +38,7 @@ date = "%d.%m.%Y"
|
|
|
38
38
|
"not-a-sheet" = "Die Datei ist kein deutsches Sicherheitsdatenblatt; ihr Inhalt wurde nicht ausgewertet."
|
|
39
39
|
"unreadable" = "Der Text der Datei ließ sich nicht lesen; es wurde nichts geprüft."
|
|
40
40
|
|
|
41
|
+
"line-too-long" = "{count} Zeilen mit mehr als 2.000 Zeichen wurden nicht gelesen, die längste mit {length} Zeichen; ein Blatt druckt keine so langen Zeilen."
|
|
41
42
|
"no-label-section" = "Abschnitt 2.2 (Kennzeichnungselemente) wurde nicht gefunden; stattdessen wurde der ganze Abschnitt 2 gelesen."
|
|
42
43
|
"several-sheets" = "Die Datei enthält {count} Sicherheitsdatenblätter; ihre Kennzeichnungen sind zusammengeführt."
|
|
43
44
|
"classified-before-clp" = "Abschnitt 2 stuft nach den Richtlinien vor der CLP-Verordnung ein ({basis}); es wurde keine Kennzeichnung gelesen."
|
|
@@ -47,6 +48,9 @@ date = "%d.%m.%Y"
|
|
|
47
48
|
"euh-wording-outside-label" = "{statement} steht nur außerhalb der Kennzeichnung: {sentence}"
|
|
48
49
|
"signal-word-not-taken" = "Das Signalwort „{signal_word}“ wurde nicht übernommen: Abschnitt 2 nennt das Produkt nicht eingestuft, und die Kennzeichnung druckt keinen H-Satz."
|
|
49
50
|
"pictograms-derived" = "Piktogramme aus den H-Sätzen abgeleitet, das Blatt nennt keine: {pictograms}"
|
|
51
|
+
"label-line-unassigned" = "Eine Zeile der Kennzeichnung wurde nichts zugeordnet: {line}"
|
|
52
|
+
"statement-too-long" = "Der Satz {codes} ist nach 100 Zeilen abgeschnitten; was folgt, gehört zu keinem Satz."
|
|
53
|
+
"precautionary-wording-unclear" = "Ein Sicherheitshinweis ohne Code passt gleich gut zu {codes}; keiner davon wurde übernommen."
|
|
50
54
|
"signal-word-derived" = "Signalwort aus den Gefahrenhinweisen abgeleitet; die Kennzeichnung nennt keines."
|
|
51
55
|
"no-identification-section" = "Abschnitt 1 wurde nicht gefunden; der Lieferant lässt sich nicht lesen."
|
|
52
56
|
"no-supplier-name" = "In Abschnitt 1 wurde kein Name des Lieferanten gefunden."
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
"not-a-sheet" = "The file is not a German safety data sheet; nothing was parsed."
|
|
34
34
|
"unreadable" = "The file's text could not be read; nothing was verified."
|
|
35
35
|
|
|
36
|
+
"line-too-long" = "{count} lines of more than 2,000 characters were not read, the longest with {length}; a sheet prints no lines that long."
|
|
36
37
|
"no-label-section" = "Section 2.2 (Kennzeichnungselemente) was not found; the whole of section 2 was read instead."
|
|
37
38
|
"several-sheets" = "The file holds {count} safety data sheets; their labels are combined."
|
|
38
39
|
"classified-before-clp" = "Section 2 classifies under the directives before CLP ({basis}); no label was read."
|
|
@@ -42,6 +43,9 @@
|
|
|
42
43
|
"euh-wording-outside-label" = "{statement} is printed only outside the label: {sentence}"
|
|
43
44
|
"signal-word-not-taken" = "The signal word '{signal_word}' was not taken: section 2 says the product is not classified, and the label prints no H statement."
|
|
44
45
|
"pictograms-derived" = "Pictograms derived from the H statements, the sheet names none: {pictograms}"
|
|
46
|
+
"label-line-unassigned" = "A line of the label was assigned to nothing: {line}"
|
|
47
|
+
"statement-too-long" = "The statement {codes} is cut off after 100 lines; what follows belongs to no statement."
|
|
48
|
+
"precautionary-wording-unclear" = "A precautionary statement without its code fits {codes} alike; none of them was taken."
|
|
45
49
|
"signal-word-derived" = "Signal word derived from the statements; the label states none."
|
|
46
50
|
"no-identification-section" = "Section 1 was not found; the supplier cannot be read."
|
|
47
51
|
"no-supplier-name" = "No supplier name was found in section 1."
|