sdb-reader 0.9.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdb_reader-0.11.0/CHANGELOG.md +210 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/PKG-INFO +2 -1
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/pyproject.toml +10 -1
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/__init__.py +6 -2
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/ask.py +13 -5
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/cli.py +3 -2
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/compliance.py +165 -2
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/embedded.py +9 -7
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/extraction.py +10 -4
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/messages.py +9 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/messages_data/de.toml +9 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/messages_data/en.toml +9 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/parser.py +772 -67
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/pictograms.py +5 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/read_record.schema.json +28 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/reading.py +30 -4
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/schema.py +3 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/serialise.py +9 -1
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/statements.py +404 -9
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/versions.py +1 -1
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/PKG-INFO +2 -1
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/requires.txt +1 -0
- sdb_reader-0.9.0/CHANGELOG.md +0 -40
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/LICENSE +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/MANIFEST.in +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/NOTICE +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/README.md +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/setup.cfg +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/ask_record.schema.json +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/clp_statements_de.json +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/current.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/diff.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/layout.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/line_breaks.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/model.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/presets.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/presets.toml +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/py.typed +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/scan.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/show.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/verification.py +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/SOURCES.txt +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/dependency_links.txt +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/entry_points.txt +0 -0
- {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
What changes for someone who uses the command or the library, newest first.
|
|
4
|
+
Versions follow the tags of this repository.
|
|
5
|
+
|
|
6
|
+
## Unreleased
|
|
7
|
+
|
|
8
|
+
## 0.11.0 – 2026-11-21
|
|
9
|
+
|
|
10
|
+
### Breaking
|
|
11
|
+
|
|
12
|
+
- **`PARSER_VERSION` is 20** (0.10.0: 19): findings are part of the reading.
|
|
13
|
+
`EXTRACTION_VERSION` stays 2; readings kept from before should be made
|
|
14
|
+
again from the stored texts.
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- **Whether the label prints a statement's code.** Each statement of the
|
|
19
|
+
label says under `code_source` whether the sheet prints its code
|
|
20
|
+
(`printed`) or only its sentence, to which the parser assigned the code
|
|
21
|
+
(`from_wording`), and `hazard_statements.codes_from_wording` lists the
|
|
22
|
+
assigned codes. The code lists stay as they are. `as_printed(text,
|
|
23
|
+
codes_from_wording)` gives a statement text without the assigned codes,
|
|
24
|
+
as the sheet prints it (SDB-174).
|
|
25
|
+
- **A hazard statement of the classification that the label does not
|
|
26
|
+
carry.** Where section 2.1 names an H statement for the classification and
|
|
27
|
+
the label carries neither its code nor its wording, the sheet gets the
|
|
28
|
+
finding `hazard-statement-not-in-label` with the code. Not required are H400
|
|
29
|
+
beside H410 and H318 beside H314 (CLP Annex III Part 1) and H304 on an
|
|
30
|
+
aerosol (Annex I 1.3.3); not where the label carries no H statement at all,
|
|
31
|
+
which `label-without-hazard-statement` says (SDB-173).
|
|
32
|
+
- **Findings for the label of a classified product.** Where section 2
|
|
33
|
+
classifies the product under CLP and the label (2.2) is not found
|
|
34
|
+
(`label-not-found`), carries no H statement
|
|
35
|
+
(`label-without-hazard-statement`) or no P statement
|
|
36
|
+
(`label-without-precautionary-statement`), the sheet gets a finding. The
|
|
37
|
+
reading stays as the sheet prints it; nothing is added. Each finding says
|
|
38
|
+
"not found": the sheet may lack it, or the parser may have missed it. Not
|
|
39
|
+
for an article, a classification under the old law, or a label under the
|
|
40
|
+
directives before CLP on a sheet dated before 1 June 2015; the parser notes
|
|
41
|
+
such a label (`label-before-clp`) (SDB-169 to SDB-172).
|
|
42
|
+
|
|
43
|
+
### Fixed
|
|
44
|
+
|
|
45
|
+
- **A section heading numbered with a colon starts its section.** "3:
|
|
46
|
+
Zusammensetzung / Angaben zu den Bestandteilen" is the heading of section 3
|
|
47
|
+
as "3. Zusammensetzung …" is, when the title fits the number; before, section
|
|
48
|
+
2 ran on into section 3, and an ingredient's classification counted as the
|
|
49
|
+
product's (SDB-175).
|
|
50
|
+
|
|
51
|
+
## 0.10.0 – 2026-11-20
|
|
52
|
+
|
|
53
|
+
### Breaking
|
|
54
|
+
|
|
55
|
+
- **H360 and H361 keep the letters of their effect.** A label's `H360D`,
|
|
56
|
+
`H360FD`, `H361fd` and the other codes of Annex VI of the CLP Regulation
|
|
57
|
+
are read as such, letters as the regulation sets them (`H360DF` is
|
|
58
|
+
`H360FD`, `H361D` is `H361d`); before, the code stood without its letters.
|
|
59
|
+
A sentence that names one effect is that effect's code where the label
|
|
60
|
+
prints no code, two effects together are the joined code, and the
|
|
61
|
+
regulation's wording of a lettered code is the one of Annex VI. Pictogram
|
|
62
|
+
and signal word do not change (SDB-155).
|
|
63
|
+
A caller that compares codes with `H360` or `H361` has to take the
|
|
64
|
+
lettered codes too.
|
|
65
|
+
- **`PARSER_VERSION` is 19** (0.9.0: 16). `EXTRACTION_VERSION` stays 2:
|
|
66
|
+
stored texts are read again as they are; readings kept from before should
|
|
67
|
+
be made again from them.
|
|
68
|
+
|
|
69
|
+
### Changed
|
|
70
|
+
|
|
71
|
+
- **Precautionary statements printed as sentences only are read in the
|
|
72
|
+
forms Annex IV of the CLP Regulation allows.** Where the label prints no P
|
|
73
|
+
code and carries a hazard statement, a sentence that selects from a choice
|
|
74
|
+
("Schutzhandschuhe und Augenschutz tragen." is P280), fills in the
|
|
75
|
+
supplier's detail for "…" or leaves out a part in brackets gives its code;
|
|
76
|
+
before, only the regulation's exact wording did. Where two statements cover
|
|
77
|
+
the same words, the longer one counts, H and EUH statements included; where
|
|
78
|
+
two cover them alike, neither does, and the new message
|
|
79
|
+
`precautionary-wording-unclear` names them. Never outside the label
|
|
80
|
+
(SDB-163).
|
|
81
|
+
- **Codes printed with a space are read.** "H 315", "P 305 + P 351 + P 338"
|
|
82
|
+
and "H 360 D" are the codes where they can be nothing else: the number is
|
|
83
|
+
a statement of the regulation, the code opens its line or follows a list
|
|
84
|
+
mark, "+", a colon, semicolon or comma, and the line's end, a separator,
|
|
85
|
+
the statement's own first word or, at a line's start, the sheet's sentence
|
|
86
|
+
follows. Measures, table values and names ("H 330 mm", "NOAEL P 250",
|
|
87
|
+
"P 750") stay as they are. Before, only "EUH 208" was read this way
|
|
88
|
+
(SDB-164).
|
|
89
|
+
|
|
90
|
+
### Added
|
|
91
|
+
|
|
92
|
+
- **The message `label-line-unassigned`** names each line of a label that the
|
|
93
|
+
parser assigned to nothing: no code, no signal word, no pictogram, no
|
|
94
|
+
statement it read, no field name, no statement about the classification,
|
|
95
|
+
no substance in the list that determines the label, no page furniture. It
|
|
96
|
+
changes no reading; it shows where rules are missing (SDB-165).
|
|
97
|
+
|
|
98
|
+
- **P501 from its disposal sentence, also beside other P codes.**
|
|
99
|
+
"Inhalt/Behälter gemäß lokalen/regionalen/nationalen/internationalen
|
|
100
|
+
Vorschriften entsorgen." and the like (content and container, the
|
|
101
|
+
supplier's detail, then "entsorgen" or "zuführen") are read as P501
|
|
102
|
+
inside the label, where it carries a hazard statement and prints no P501
|
|
103
|
+
code; the precautionary text keeps the sheet's sentence. "Leere Behälter
|
|
104
|
+
können über den Hausmüll entsorgt werden." is no P501 (SDB-166).
|
|
105
|
+
|
|
106
|
+
- **The messages `line-too-long` and `statement-too-long`** name what the
|
|
107
|
+
new bounds on a reading (see Security) left out: how many lines were too
|
|
108
|
+
long and the length of the longest, and the codes a statement cut off
|
|
109
|
+
after 100 lines begins with (SDB-167, SDB-168).
|
|
110
|
+
|
|
111
|
+
### Security
|
|
112
|
+
|
|
113
|
+
- **The embedded eSDScom file is read with `defusedxml`.** A document type,
|
|
114
|
+
an entity or an external reference makes it unreadable
|
|
115
|
+
(`embedded-doctype`), in any encoding. Before, the bytes were searched
|
|
116
|
+
for "<!DOCTYPE" and "<!ENTITY", which a file in UTF-16 got past, and the
|
|
117
|
+
standard library's parser expanded its entities. New dependency:
|
|
118
|
+
`defusedxml` (Python Software Foundation License) (SDW-11).
|
|
119
|
+
- **Five patterns of the parser no longer slow down on long runs of blanks
|
|
120
|
+
or dashes.** The revision date's label, the version line with its
|
|
121
|
+
creation date, a classification given as a dash, a label without a signal
|
|
122
|
+
word, and the substances before EUH208 read each character one way now; a
|
|
123
|
+
crafted text of a few thousand blanks took them seconds before. The
|
|
124
|
+
readings are the same.
|
|
125
|
+
- **Four more patterns no longer slow down on one long word.** The e-mail
|
|
126
|
+
address, a display named on a cover page, an article number in front of a
|
|
127
|
+
product's name and the addition at the end of a name start only where a
|
|
128
|
+
run of their characters starts, or ask for the digit ahead; before, a run
|
|
129
|
+
of some ten thousand letters or digits took seconds. The readings are the
|
|
130
|
+
same.
|
|
131
|
+
- **A long run of capitalised words in a label no longer slows the reading
|
|
132
|
+
down.** A classification entry in the label ("Skin Corr. 1, H314") is
|
|
133
|
+
found by the last word of its class only; before, every capitalised word
|
|
134
|
+
in front of it was tried, and a run of some twenty thousand characters
|
|
135
|
+
took seconds. The codes read from the label are the same.
|
|
136
|
+
- **Five more places no longer slow down on a long run.** Lines of nothing
|
|
137
|
+
but dashes or bullets above "Sonstige Gefahren", colons before "keine" or
|
|
138
|
+
"N/A" in the classification, a long word before an article number's
|
|
139
|
+
label, letters with full stops at the end of a supplier's name, and a run
|
|
140
|
+
of brackets in a name's label were each read again from every character
|
|
141
|
+
of the run; some twenty thousand characters took one to ten seconds. The
|
|
142
|
+
readings are the same.
|
|
143
|
+
- **What a reading takes in is bounded.** A line of more than 2,000
|
|
144
|
+
characters is not read (`line-too-long`), and a hazard, precautionary or
|
|
145
|
+
EUH statement takes at most 100 lines (`statement-too-long`); the longest
|
|
146
|
+
line of the sheets the reader was measured on has 837 characters, the
|
|
147
|
+
longest statement 37 lines. A built text with a line of tens of thousands
|
|
148
|
+
of dashes, or a statement followed by a hundred thousand lines that end
|
|
149
|
+
nothing, took seconds to minutes; the bounds cover such cases whether or
|
|
150
|
+
not anyone has found them. The text as read stays whole, so a kept text
|
|
151
|
+
is read again as before (SDB-167, SDB-168).
|
|
152
|
+
|
|
153
|
+
### Fixed
|
|
154
|
+
|
|
155
|
+
Layouts and spellings from sheets outside the collection the reader was
|
|
156
|
+
measured on:
|
|
157
|
+
|
|
158
|
+
- A label that numbers its own parts on (`2.3 Gefahrbestimmende
|
|
159
|
+
Komponenten`, `2.4 Gefahrenhinweise`) no longer ends at its "2.3"
|
|
160
|
+
(SDB-156).
|
|
161
|
+
- A signal word behind its label counts where the label then says the
|
|
162
|
+
diluted product needs none (SDB-157).
|
|
163
|
+
- No product name: an article number's label, a word for what the product is
|
|
164
|
+
("Gemisch"), the article number below stacked labels; "Bezeichnung des
|
|
165
|
+
Stoffes oder des Gemisches:" is a name's label (SDB-158).
|
|
166
|
+
- The supplier behind "Name der Firma:", "Name des Herstellerunternehmens:",
|
|
167
|
+
"Herstellerin", "Firmendetails:", "Company:", "Name oder Handelsname", and
|
|
168
|
+
behind the colon of 1.3's heading; no street and no pointer as the name; a
|
|
169
|
+
name cut from an address or a column glued to it; "BV", "NV" and "SA" as
|
|
170
|
+
legal forms (SDB-159).
|
|
171
|
+
- "Abschnitt 7.3 angegeben" at the end of a sentence is no heading
|
|
172
|
+
(SDB-160).
|
|
173
|
+
- Dates behind "Überarbei tung" broken over a line, with a weekday in front,
|
|
174
|
+
and "Datum der Ausgabe:" as the issue date (SDB-161).
|
|
175
|
+
- A pictogram's code behind a statement and a page marker "Page n. 1 von 9"
|
|
176
|
+
are not part of the statement's wording (SDB-162).
|
|
177
|
+
|
|
178
|
+
## 0.9.0 – 2026-11-19
|
|
179
|
+
|
|
180
|
+
The first public version. What it offers:
|
|
181
|
+
|
|
182
|
+
- **The command `sdb-reader`.** `show` says what each PDF is and what it
|
|
183
|
+
says, as a report to read; `read` gives the same as JSON, a record per
|
|
184
|
+
file, described by a JSON Schema the package ships. Both take single files
|
|
185
|
+
and whole folders. `xml` hands out the eSDScom file some PDFs carry, `ask`
|
|
186
|
+
puts the sheets to a language model of your choice, and `diff` compares two
|
|
187
|
+
runs field by field.
|
|
188
|
+
- **The library `sdb_reader`.** `read_sheet` reads a PDF: whether it is a
|
|
189
|
+
German safety data sheet, and its product, supplier, dates, classification
|
|
190
|
+
and label with the regulation's wording beside the sheet's. The library
|
|
191
|
+
judges a sheet (`sheet_findings`, `requires_sheet`), chooses the current one
|
|
192
|
+
among several (`current_state`), and stores a PDF's text so that a sheet can
|
|
193
|
+
be read again without the file. The exported names are the interface;
|
|
194
|
+
[What is promised](docs/api/stability.md) says what a program can build on.
|
|
195
|
+
- **No language model.** Reading is rules only and gives the same result for
|
|
196
|
+
the same file every time. Only `ask` talks to a model, and only when called.
|
|
197
|
+
- **Languages.** It reads German sheets; a sheet in another language is
|
|
198
|
+
recognised and not read. Messages come in German (the default) and English,
|
|
199
|
+
and in any language through a catalog of your own.
|
|
200
|
+
- **Documentation.** At <https://foerdebaumarkt.github.io/sdb-reader/>, one
|
|
201
|
+
edition per released version: the operator's manual in German and English,
|
|
202
|
+
the guide to the library with a reference of every exported name, and how
|
|
203
|
+
the reader works.
|
|
204
|
+
- **Python.** 3.12 to 3.15: every Python version still maintained.
|
|
205
|
+
- **Stage versions.** `EXTRACTION_VERSION` is 2 and `PARSER_VERSION` is 16.
|
|
206
|
+
A text or a reading kept with other values is read again; later releases
|
|
207
|
+
name every rise here, with what to do about what was kept.
|
|
208
|
+
|
|
209
|
+
Before 1.0 a minor version may break; a breaking change stands under
|
|
210
|
+
"Breaking" in its release's section.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sdb-reader
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Reads German safety data sheets (Sicherheitsdatenblätter) under REACH and CLP, without a language model
|
|
5
5
|
Author-email: "Förde-Baumarkt Holger Claussen GmbH & Co. KG" <service@foerde-baumarkt.de>
|
|
6
6
|
License-Expression: AGPL-3.0-or-later
|
|
@@ -28,6 +28,7 @@ Requires-Python: >=3.12
|
|
|
28
28
|
Description-Content-Type: text/markdown
|
|
29
29
|
License-File: LICENSE
|
|
30
30
|
License-File: NOTICE
|
|
31
|
+
Requires-Dist: defusedxml<0.8,>=0.7.1
|
|
31
32
|
Requires-Dist: pymupdf-lite<3.0.0,>=2.1
|
|
32
33
|
Requires-Dist: structlog>=26.1.0
|
|
33
34
|
Dynamic: license-file
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "sdb-reader"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.11.0"
|
|
8
8
|
description = "Reads German safety data sheets (Sicherheitsdatenblätter) under REACH and CLP, without a language model"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "Förde-Baumarkt Holger Claussen GmbH & Co. KG", email = "service@foerde-baumarkt.de" }]
|
|
@@ -43,6 +43,10 @@ classifiers = [
|
|
|
43
43
|
"Programming Language :: Python :: 3.15",
|
|
44
44
|
]
|
|
45
45
|
dependencies = [
|
|
46
|
+
# Parses the eSDScom file a PDF may carry without expanding entities or
|
|
47
|
+
# fetching external references, in any encoding (SDW-11). Upper bound: the
|
|
48
|
+
# next minor is a decision.
|
|
49
|
+
"defusedxml>=0.7.1,<0.8",
|
|
46
50
|
# Reads the PDF's text layer by position and renders pages for text
|
|
47
51
|
# recognition. The lean distribution of PyMuPDF: the same `pymupdf` module
|
|
48
52
|
# without the layout and Markdown packages (and their numpy and onnxruntime)
|
|
@@ -95,6 +99,11 @@ dev = [
|
|
|
95
99
|
# Checks before a release that the package index can render the built
|
|
96
100
|
# package's description (make dist; docs/contributing/releasing.md).
|
|
97
101
|
"twine==7.0.0",
|
|
102
|
+
# Type information for defusedxml, for mypy.
|
|
103
|
+
"types-defusedxml==0.7.0.20260504",
|
|
104
|
+
# Property tests: random and degenerate text through the reading, each
|
|
105
|
+
# reading held to a time limit (tests/test_reading_time.py).
|
|
106
|
+
"hypothesis==6.170.15",
|
|
98
107
|
]
|
|
99
108
|
|
|
100
109
|
[tool.setuptools]
|
|
@@ -28,8 +28,10 @@ with any release, the parts that ask a language model included.
|
|
|
28
28
|
- Judging a sheet: ``sheet_findings`` and the checks it is made of,
|
|
29
29
|
``classification`` and ``requires_sheet``.
|
|
30
30
|
- Statements and pictograms: ``label_statements`` with the regulation's
|
|
31
|
-
wording (``canonical_text``)
|
|
32
|
-
|
|
31
|
+
wording (``canonical_text``) and whether the label prints each code, the
|
|
32
|
+
statement texts tidied and rendered, ``as_printed`` for the texts without
|
|
33
|
+
the codes the label does not print, the pictograms that follow from H
|
|
34
|
+
statements.
|
|
33
35
|
- Embedded data: ``read_embedded``, ``parse_embedded`` and
|
|
34
36
|
``embedded_differences``.
|
|
35
37
|
- Messages: what the library notes about a file is a ``Message`` with a
|
|
@@ -90,6 +92,7 @@ from .serialise import (
|
|
|
90
92
|
)
|
|
91
93
|
from .statements import (
|
|
92
94
|
StatementWording,
|
|
95
|
+
as_printed,
|
|
93
96
|
canonical_text,
|
|
94
97
|
label_statements,
|
|
95
98
|
normalize_statements,
|
|
@@ -125,6 +128,7 @@ __all__ = [
|
|
|
125
128
|
"StatedClassification",
|
|
126
129
|
"StatementWording",
|
|
127
130
|
"Verification",
|
|
131
|
+
"as_printed",
|
|
128
132
|
"canonical_text",
|
|
129
133
|
"classification",
|
|
130
134
|
"contradicts_label",
|
|
@@ -30,10 +30,14 @@ from .extraction import ExtractionResult, normalise_pdf_text, strip_page_furnitu
|
|
|
30
30
|
from .messages import Message, shipped_catalog
|
|
31
31
|
from .model import Prompt, build_prompt, named_date, read_answer
|
|
32
32
|
from .parser import split_into_sheets
|
|
33
|
-
from .statements import normalize_statements
|
|
33
|
+
from .statements import lettered_code, normalize_statements
|
|
34
34
|
from .versions import extraction_stage
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
# An H360 or H361 with the letters of its effect is one code (SDB-155).
|
|
37
|
+
_H_CODE = re.compile(
|
|
38
|
+
r"(?<![A-Za-z])H(?:36[01][A-Za-z]{1,2}(?![^\W\d_])|\d{3})(?!\d)"
|
|
39
|
+
r"(?:\s*\+\s*H\d{3}(?!\d))*"
|
|
40
|
+
)
|
|
37
41
|
_P_CODE = re.compile(r"(?<![A-Za-z])P\d{3}(?!\d)(?:\s*\+\s*P\d{3}(?!\d))*")
|
|
38
42
|
_EUH_CODE = re.compile(r"\bEUH\d{3}(?!\d)")
|
|
39
43
|
|
|
@@ -212,7 +216,7 @@ def send(endpoint: Endpoint, prompt: Prompt, *, retries: int = 0) -> Reply:
|
|
|
212
216
|
headers["Authorization"] = f"Bearer {endpoint.api_key}"
|
|
213
217
|
for attempt in range(retries + 1):
|
|
214
218
|
# The scheme is checked where the endpoint is made: http or https only.
|
|
215
|
-
request = urllib.request.Request(
|
|
219
|
+
request = urllib.request.Request( # noqa: S310 -- http(s) only, see above
|
|
216
220
|
endpoint.url.rstrip("/") + "/chat/completions",
|
|
217
221
|
data=body,
|
|
218
222
|
headers=headers,
|
|
@@ -220,7 +224,9 @@ def send(endpoint: Endpoint, prompt: Prompt, *, retries: int = 0) -> Reply:
|
|
|
220
224
|
)
|
|
221
225
|
named: float | None = None
|
|
222
226
|
try:
|
|
223
|
-
with urllib.request.urlopen(
|
|
227
|
+
with urllib.request.urlopen( # noqa: S310 -- http(s) only, see above
|
|
228
|
+
request, timeout=endpoint.timeout
|
|
229
|
+
) as response:
|
|
224
230
|
payload = json.loads(response.read())
|
|
225
231
|
break
|
|
226
232
|
except urllib.error.HTTPError as exc:
|
|
@@ -298,7 +304,9 @@ def model_reading(answer: Mapping[str, Any], text: str) -> dict[str, Any]:
|
|
|
298
304
|
# As the model wrote it, also where it did not hold up.
|
|
299
305
|
"revision_date_named": named if isinstance(named, str) and named else None,
|
|
300
306
|
"hazard_statements": {
|
|
301
|
-
"hazard_codes":
|
|
307
|
+
"hazard_codes": _unique(
|
|
308
|
+
lettered_code(code) for code in _codes(_H_CODE, hazard)
|
|
309
|
+
),
|
|
302
310
|
"precautionary_codes": _codes(_P_CODE, precautionary),
|
|
303
311
|
# A model now and then lists an EUH statement among the H statements.
|
|
304
312
|
"supplemental_codes": _unique(
|
|
@@ -35,7 +35,7 @@ from .messages import (
|
|
|
35
35
|
shipped_catalog,
|
|
36
36
|
shipped_languages,
|
|
37
37
|
)
|
|
38
|
-
from .reading import read_sheet
|
|
38
|
+
from .reading import read_sheet, within_line_limit
|
|
39
39
|
from .scan import Outcome
|
|
40
40
|
from .serialise import reading_to_dict
|
|
41
41
|
from .verification import DocumentKind, SDSVerifier, Verification
|
|
@@ -851,7 +851,8 @@ def _ask_one(path: Path, options: _AskOptions) -> Outcome:
|
|
|
851
851
|
done = _ask_about(
|
|
852
852
|
path,
|
|
853
853
|
extraction,
|
|
854
|
-
|
|
854
|
+
# Judged on the text as the reading judges it (SDB-167).
|
|
855
|
+
SDSVerifier().verify(within_line_limit(extraction)[0]),
|
|
855
856
|
(),
|
|
856
857
|
options,
|
|
857
858
|
catalog,
|
|
@@ -18,11 +18,22 @@ from .messages import Message
|
|
|
18
18
|
from .parser import (
|
|
19
19
|
CLASSIFIED_BEFORE_CLP,
|
|
20
20
|
EUH_WORDING_OUTSIDE_LABEL,
|
|
21
|
+
LABEL_BEFORE_CLP,
|
|
22
|
+
LABEL_HEADING_PATTERN,
|
|
21
23
|
LABEL_UNDETERMINED,
|
|
24
|
+
NO_LABEL_SECTION,
|
|
22
25
|
SheetPart,
|
|
26
|
+
classification_hazard_codes,
|
|
23
27
|
)
|
|
24
28
|
from .schema import HazardStatements, SafetyDataSheet, Section, StatedClassification
|
|
25
|
-
from .statements import
|
|
29
|
+
from .statements import (
|
|
30
|
+
SHEET_ON_REQUEST,
|
|
31
|
+
_catalogue,
|
|
32
|
+
_comparable,
|
|
33
|
+
base_code,
|
|
34
|
+
deviates_from_regulation,
|
|
35
|
+
label_statements,
|
|
36
|
+
)
|
|
26
37
|
|
|
27
38
|
# Annex II of REACH in the version of Regulation (EU) 2020/878 applies from
|
|
28
39
|
# 1 January 2021; sheets not conforming to it could be supplied until
|
|
@@ -253,6 +264,152 @@ def contradicts_label(sheet: SafetyDataSheet) -> bool:
|
|
|
253
264
|
)
|
|
254
265
|
|
|
255
266
|
|
|
267
|
+
def label_not_found(
|
|
268
|
+
sheet: SafetyDataSheet, parser_warnings: Iterable[Message]
|
|
269
|
+
) -> list[Message]:
|
|
270
|
+
"""What the label of a product section 2 classifies should carry and the
|
|
271
|
+
parser did not find in it (SDB-169 to SDB-172).
|
|
272
|
+
|
|
273
|
+
The reading stays as it is: these are findings beside it, and each says
|
|
274
|
+
"not found", which is true whether the sheet lacks it or the parser missed
|
|
275
|
+
it. Only for a product classified under CLP (SDB-75); not under the old
|
|
276
|
+
law (SDB-73), not for a label under the old directives on a sheet dated
|
|
277
|
+
before 1 June 2015, and not for an article, which is not labelled under
|
|
278
|
+
CLP (SDB-142, SDB-154). A label counts as found where the parser found its
|
|
279
|
+
block (SDB-4), or, without one, read any element of a label in section 2.
|
|
280
|
+
|
|
281
|
+
>>> from sdb_reader import HazardStatements
|
|
282
|
+
>>> def sheet(label, stated=StatedClassification.CLASSIFIED):
|
|
283
|
+
... return SafetyDataSheet(
|
|
284
|
+
... None, None, None, None, "de", hazard_statements=label,
|
|
285
|
+
... stated_classification=stated,
|
|
286
|
+
... )
|
|
287
|
+
>>> [m.code for m in label_not_found(sheet(HazardStatements()),
|
|
288
|
+
... [Message(NO_LABEL_SECTION)])]
|
|
289
|
+
['label-not-found']
|
|
290
|
+
>>> [m.code for m in label_not_found(
|
|
291
|
+
... sheet(HazardStatements(signal_word="Achtung", precautionary_codes=["P210"])),
|
|
292
|
+
... [])]
|
|
293
|
+
['label-without-hazard-statement']
|
|
294
|
+
>>> label_not_found(sheet(HazardStatements(), StatedClassification.NOT_STATED), [])
|
|
295
|
+
[]
|
|
296
|
+
"""
|
|
297
|
+
noted = {warning.code for warning in parser_warnings}
|
|
298
|
+
if not _label_is_judged(sheet, noted):
|
|
299
|
+
return []
|
|
300
|
+
label = sheet.hazard_statements
|
|
301
|
+
read_any = bool(
|
|
302
|
+
label.hazard_codes
|
|
303
|
+
or label.precautionary_codes
|
|
304
|
+
or label.supplemental_codes
|
|
305
|
+
or label.signal_word
|
|
306
|
+
or label.pictograms
|
|
307
|
+
)
|
|
308
|
+
if NO_LABEL_SECTION in noted and not read_any:
|
|
309
|
+
return [Message("label-not-found")]
|
|
310
|
+
findings: list[Message] = []
|
|
311
|
+
if not label.hazard_codes:
|
|
312
|
+
findings.append(Message("label-without-hazard-statement"))
|
|
313
|
+
if not label.precautionary_codes:
|
|
314
|
+
findings.append(Message("label-without-precautionary-statement"))
|
|
315
|
+
return findings
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _label_is_judged(sheet: SafetyDataSheet, noted: Collection[str]) -> bool:
|
|
319
|
+
"""Whether the label findings apply to *sheet* (SDB-170): a product
|
|
320
|
+
classified under CLP, no article, no classification under the old law, and
|
|
321
|
+
no label under the old directives on a sheet dated before 1 June 2015.
|
|
322
|
+
*noted* are the codes of the parser's warnings."""
|
|
323
|
+
if sheet.stated_classification is not StatedClassification.CLASSIFIED:
|
|
324
|
+
return False
|
|
325
|
+
if sheet.stated_article or CLASSIFIED_BEFORE_CLP in noted:
|
|
326
|
+
return False
|
|
327
|
+
# A mixture could be labelled under the old directives until 1 June 2015.
|
|
328
|
+
dated = sheet.revision_date or sheet.issue_date
|
|
329
|
+
return not (
|
|
330
|
+
LABEL_BEFORE_CLP in noted and dated is not None and dated < CLP_FOR_MIXTURES
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
# The order of precedence Annex III Part 1 sets under Article 27 (consolidated
|
|
335
|
+
# version of 1 July 2026): a statement the label may leave out beside another.
|
|
336
|
+
_LEFT_OUT_BESIDE = {"H400": "H410", "H318": "H314"}
|
|
337
|
+
# An aerosol: its class in 2.1 or one of its statements (Annex I 2.3).
|
|
338
|
+
_AEROSOL_CLASS = re.compile(r"\bAerosol\s?[123]\b")
|
|
339
|
+
_AEROSOL_STATEMENTS = frozenset({"H222", "H223", "H229"})
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _not_required(
|
|
343
|
+
code: str, on_label: set[str], classified: set[str], hazards: str
|
|
344
|
+
) -> bool:
|
|
345
|
+
"""Whether the label may go without *code* although 2.1 names it (SDB-173).
|
|
346
|
+
|
|
347
|
+
Beside H410 or H314 the label may leave out H400 or H318 (CLP Annex III
|
|
348
|
+
Part 1 under Article 27). A product placed on the market as an aerosol need
|
|
349
|
+
not be labelled for the aspiration hazard, H304 (Annex I 1.3.3).
|
|
350
|
+
"""
|
|
351
|
+
if _LEFT_OUT_BESIDE.get(code) in on_label:
|
|
352
|
+
return True
|
|
353
|
+
return code == "H304" and bool(
|
|
354
|
+
_AEROSOL_CLASS.search(hazards) or (classified | on_label) & _AEROSOL_STATEMENTS
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def hazard_statements_not_in_label(
|
|
359
|
+
sheet: SafetyDataSheet, parser_warnings: Iterable[Message]
|
|
360
|
+
) -> list[Message]:
|
|
361
|
+
r"""Each H statement of the classification in 2.1 that the label does not
|
|
362
|
+
carry (SDB-173), with its code as ``statement``.
|
|
363
|
+
|
|
364
|
+
Only where the label carries at least one H statement -- without any,
|
|
365
|
+
``label-without-hazard-statement`` says so (SDB-171) -- and with the
|
|
366
|
+
exceptions of SDB-170. A code counts as carried where the label carries the
|
|
367
|
+
same statement, H360 and H361 whatever their letters, and H400 and H318 also
|
|
368
|
+
where the label carries H410 or H314 (CLP Article 27 with Annex III Part 1).
|
|
369
|
+
|
|
370
|
+
>>> from sdb_reader import HazardStatements
|
|
371
|
+
>>> section = (
|
|
372
|
+
... "2.1 Einstufung des Stoffs oder Gemischs\n"
|
|
373
|
+
... "Aerosol 3, H229\nAquatic Acute 1, H400\nAquatic Chronic 1, H410\n"
|
|
374
|
+
... "2.2 Kennzeichnungselemente\n"
|
|
375
|
+
... )
|
|
376
|
+
>>> sheet = SafetyDataSheet(
|
|
377
|
+
... None, None, None, None, "de",
|
|
378
|
+
... hazard_statements=HazardStatements(hazard_codes=["H410"]),
|
|
379
|
+
... stated_classification=StatedClassification.CLASSIFIED,
|
|
380
|
+
... section_text={Section.HAZARDS: section},
|
|
381
|
+
... )
|
|
382
|
+
>>> [m.params["statement"] for m in hazard_statements_not_in_label(sheet, [])]
|
|
383
|
+
['H229']
|
|
384
|
+
"""
|
|
385
|
+
noted = {warning.code for warning in parser_warnings}
|
|
386
|
+
label = sheet.hazard_statements
|
|
387
|
+
if not label.hazard_codes or not _label_is_judged(sheet, noted):
|
|
388
|
+
return []
|
|
389
|
+
hazards = sheet.section_text.get(Section.HAZARDS, "")
|
|
390
|
+
heading = LABEL_HEADING_PATTERN.search(hazards)
|
|
391
|
+
label_text = _comparable(hazards[heading.start() :]) if heading else ""
|
|
392
|
+
on_label = {base_code(code) for code in label.hazard_codes}
|
|
393
|
+
# Without the wordings of its own statements: "Giftig für Wasserorganismen,
|
|
394
|
+
# mit langfristiger Wirkung" (H411) stands inside H410's.
|
|
395
|
+
wordings = _catalogue()
|
|
396
|
+
for code in on_label:
|
|
397
|
+
if own := wordings.get(code):
|
|
398
|
+
label_text = label_text.replace(_comparable(own), " ")
|
|
399
|
+
classified = {base_code(code) for code in classification_hazard_codes(hazards)}
|
|
400
|
+
missing: list[Message] = []
|
|
401
|
+
for code in sorted(classification_hazard_codes(hazards)):
|
|
402
|
+
base = base_code(code)
|
|
403
|
+
if base in on_label or _not_required(base, on_label, classified, hazards):
|
|
404
|
+
continue
|
|
405
|
+
# The label prints the statement's wording without its code.
|
|
406
|
+
wording = wordings.get(base)
|
|
407
|
+
if wording and _comparable(wording) in label_text:
|
|
408
|
+
continue
|
|
409
|
+
missing.append(Message("hazard-statement-not-in-label", {"statement": code}))
|
|
410
|
+
return missing
|
|
411
|
+
|
|
412
|
+
|
|
256
413
|
class Classification(StrEnum):
|
|
257
414
|
"""What a sheet's label states about the product's classification."""
|
|
258
415
|
|
|
@@ -323,7 +480,9 @@ _COMPONENT_ROLE = re.compile(
|
|
|
323
480
|
r"\b(?:harz|härter|haerter|mörtel|moertel|basis)\b", re.IGNORECASE
|
|
324
481
|
)
|
|
325
482
|
# An addition in brackets or a number at the end of a name: "(Finish)", "-019".
|
|
326
|
-
|
|
483
|
+
# The separators start only where a run of them starts: the same tail, and a
|
|
484
|
+
# long run of them is read once.
|
|
485
|
+
_NAME_TAIL = re.compile(r"(?:\([^()]*\)|(?<![\s,;:–-])[\s,;:–-]+\d{2,4})\s*$")
|
|
327
486
|
_NOT_A_NAME_S = re.compile(r"[^\w]+")
|
|
328
487
|
|
|
329
488
|
|
|
@@ -466,6 +625,7 @@ def sheet_findings(
|
|
|
466
625
|
... "Verordnung (EG) Nr. 1907/2006\\n"
|
|
467
626
|
... "Signalwort: Achtung\\n"
|
|
468
627
|
... "H315: Verursacht Hautreizungen.\\n"
|
|
628
|
+
... "P280: Schutzhandschuhe tragen.\\n"
|
|
469
629
|
... )
|
|
470
630
|
>>> result = read_extraction(
|
|
471
631
|
... extraction_from_dict(
|
|
@@ -538,6 +698,9 @@ def sheet_findings(
|
|
|
538
698
|
if contradicts_label(sheet):
|
|
539
699
|
findings.append(Message("classification-contradicts-label"))
|
|
540
700
|
|
|
701
|
+
findings.extend(label_not_found(sheet, parser_warnings))
|
|
702
|
+
findings.extend(hazard_statements_not_in_label(sheet, parser_warnings))
|
|
703
|
+
|
|
541
704
|
findings.extend(
|
|
542
705
|
Message("euh-outside-label", dict(outside.params))
|
|
543
706
|
for outside in euh_outside_label(parser_warnings)
|
|
@@ -16,6 +16,7 @@ from dataclasses import dataclass
|
|
|
16
16
|
from pathlib import Path
|
|
17
17
|
|
|
18
18
|
import pymupdf
|
|
19
|
+
from defusedxml import DefusedXmlException, ElementTree as SafeET
|
|
19
20
|
|
|
20
21
|
from .messages import Message
|
|
21
22
|
from .schema import SafetyDataSheet
|
|
@@ -24,10 +25,6 @@ from .schema import SafetyDataSheet
|
|
|
24
25
|
# CHEMDOX writes none.
|
|
25
26
|
_ROOT = "DatasheetFeed"
|
|
26
27
|
|
|
27
|
-
# A document type declaration is how an XML file defines entities, and an
|
|
28
|
-
# entity is how a small file expands into a huge one. eSDScom has none.
|
|
29
|
-
_DOCTYPE = re.compile(rb"<!\s*(?:DOCTYPE|ENTITY)", re.IGNORECASE)
|
|
30
|
-
|
|
31
28
|
# Larger than any export seen (160 kB); an attachment beyond it is something
|
|
32
29
|
# else or an attempt.
|
|
33
30
|
_MAX_BYTES = 20 * 1024 * 1024
|
|
@@ -109,10 +106,15 @@ def parse_embedded(name: str, content: bytes) -> EmbeddedData:
|
|
|
109
106
|
return EmbeddedData(
|
|
110
107
|
name, unreadable=Message("embedded-too-large", {"limit_bytes": _MAX_BYTES})
|
|
111
108
|
)
|
|
112
|
-
if _DOCTYPE.search(content):
|
|
113
|
-
return EmbeddedData(name, unreadable=Message("embedded-doctype"))
|
|
114
109
|
try:
|
|
115
|
-
|
|
110
|
+
# No document type, no entity, no external reference, whatever the
|
|
111
|
+
# file's encoding (SDW-11): the standard library's parser expands
|
|
112
|
+
# internal entities, and a check of the bytes misses them in UTF-16.
|
|
113
|
+
root = SafeET.fromstring(
|
|
114
|
+
content, forbid_dtd=True, forbid_entities=True, forbid_external=True
|
|
115
|
+
)
|
|
116
|
+
except DefusedXmlException:
|
|
117
|
+
return EmbeddedData(name, unreadable=Message("embedded-doctype"))
|
|
116
118
|
except ET.ParseError as exc:
|
|
117
119
|
return EmbeddedData(
|
|
118
120
|
name, unreadable=Message("embedded-not-xml", {"detail": str(exc)})
|
|
@@ -308,10 +308,16 @@ def _tesseract_version() -> str:
|
|
|
308
308
|
"""
|
|
309
309
|
try:
|
|
310
310
|
asked = subprocess.run(
|
|
311
|
-
["tesseract", "--version"],
|
|
311
|
+
["tesseract", "--version"], # noqa: S607 -- tesseract from PATH (SDB-116)
|
|
312
|
+
capture_output=True,
|
|
313
|
+
timeout=30,
|
|
314
|
+
check=False,
|
|
312
315
|
)
|
|
313
316
|
languages = subprocess.run(
|
|
314
|
-
["tesseract", "--list-langs"],
|
|
317
|
+
["tesseract", "--list-langs"], # noqa: S607 -- tesseract from PATH (SDB-116)
|
|
318
|
+
capture_output=True,
|
|
319
|
+
timeout=30,
|
|
320
|
+
check=False,
|
|
315
321
|
)
|
|
316
322
|
except FileNotFoundError as exc:
|
|
317
323
|
raise ExtractionError(Message("ocr-unavailable")) from exc
|
|
@@ -342,8 +348,8 @@ def _run_tesseract(image: bytes) -> str:
|
|
|
342
348
|
"""Hand one rendered page to the binary and return what it read."""
|
|
343
349
|
_tesseract_version()
|
|
344
350
|
try:
|
|
345
|
-
result = subprocess.run(
|
|
346
|
-
["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"],
|
|
351
|
+
result = subprocess.run( # noqa: S603 -- fixed arguments, no shell
|
|
352
|
+
["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"], # noqa: S607 -- tesseract from PATH (SDB-116)
|
|
347
353
|
input=image,
|
|
348
354
|
capture_output=True,
|
|
349
355
|
timeout=OCR_PAGE_TIMEOUT_SECONDS,
|
|
@@ -55,15 +55,20 @@ MESSAGE_CODES: Mapping[str, tuple[str, ...]] = {
|
|
|
55
55
|
"not-a-sheet": (),
|
|
56
56
|
"unreadable": (),
|
|
57
57
|
# Parsing the sheet.
|
|
58
|
+
"line-too-long": ("count", "length"),
|
|
58
59
|
"no-label-section": (),
|
|
59
60
|
"several-sheets": ("count",),
|
|
60
61
|
"classified-before-clp": ("basis",),
|
|
62
|
+
"label-before-clp": (),
|
|
61
63
|
"no-hazard-codes": (),
|
|
62
64
|
"label-undetermined": (),
|
|
63
65
|
"no-precautionary-codes": (),
|
|
64
66
|
"euh-wording-outside-label": ("statement", "sentence"),
|
|
65
67
|
"signal-word-not-taken": ("signal_word",),
|
|
66
68
|
"pictograms-derived": ("pictograms",),
|
|
69
|
+
"precautionary-wording-unclear": ("codes",),
|
|
70
|
+
"label-line-unassigned": ("line",),
|
|
71
|
+
"statement-too-long": ("codes",),
|
|
67
72
|
"signal-word-derived": (),
|
|
68
73
|
"no-identification-section": (),
|
|
69
74
|
"no-supplier-name": (),
|
|
@@ -80,6 +85,10 @@ MESSAGE_CODES: Mapping[str, tuple[str, ...]] = {
|
|
|
80
85
|
"old-format": ("revised",),
|
|
81
86
|
"classification-not-stated": (),
|
|
82
87
|
"classification-contradicts-label": (),
|
|
88
|
+
"label-not-found": (),
|
|
89
|
+
"label-without-hazard-statement": (),
|
|
90
|
+
"hazard-statement-not-in-label": ("statement",),
|
|
91
|
+
"label-without-precautionary-statement": (),
|
|
83
92
|
"different-products": ("products",),
|
|
84
93
|
"country-editions-differ": ("fields",),
|
|
85
94
|
"euh-outside-label": ("statement", "sentence"),
|