sdb-reader 0.9.0__tar.gz → 0.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. sdb_reader-0.11.0/CHANGELOG.md +210 -0
  2. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/PKG-INFO +2 -1
  3. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/pyproject.toml +10 -1
  4. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/__init__.py +6 -2
  5. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/ask.py +13 -5
  6. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/cli.py +3 -2
  7. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/compliance.py +165 -2
  8. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/embedded.py +9 -7
  9. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/extraction.py +10 -4
  10. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/messages.py +9 -0
  11. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/messages_data/de.toml +9 -0
  12. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/messages_data/en.toml +9 -0
  13. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/parser.py +772 -67
  14. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/pictograms.py +5 -0
  15. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/read_record.schema.json +28 -0
  16. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/reading.py +30 -4
  17. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/schema.py +3 -0
  18. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/serialise.py +9 -1
  19. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/statements.py +404 -9
  20. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/versions.py +1 -1
  21. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/PKG-INFO +2 -1
  22. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/requires.txt +1 -0
  23. sdb_reader-0.9.0/CHANGELOG.md +0 -40
  24. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/LICENSE +0 -0
  25. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/MANIFEST.in +0 -0
  26. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/NOTICE +0 -0
  27. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/README.md +0 -0
  28. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/setup.cfg +0 -0
  29. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/ask_record.schema.json +0 -0
  30. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/clp_statements_de.json +0 -0
  31. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/current.py +0 -0
  32. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/diff.py +0 -0
  33. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/layout.py +0 -0
  34. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/line_breaks.py +0 -0
  35. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/model.py +0 -0
  36. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/presets.py +0 -0
  37. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/presets.toml +0 -0
  38. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/py.typed +0 -0
  39. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/scan.py +0 -0
  40. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/show.py +0 -0
  41. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader/verification.py +0 -0
  42. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/SOURCES.txt +0 -0
  43. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/dependency_links.txt +0 -0
  44. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/entry_points.txt +0 -0
  45. {sdb_reader-0.9.0 → sdb_reader-0.11.0}/src/sdb_reader.egg-info/top_level.txt +0 -0
@@ -0,0 +1,210 @@
1
+ # Changelog
2
+
3
+ What changes for someone who uses the command or the library, newest first.
4
+ Versions follow the tags of this repository.
5
+
6
+ ## Unreleased
7
+
8
+ ## 0.11.0 – 2026-11-21
9
+
10
+ ### Breaking
11
+
12
+ - **`PARSER_VERSION` is 20** (0.10.0: 19): findings are part of the reading.
13
+ `EXTRACTION_VERSION` stays 2; readings kept from before should be made
14
+ again from the stored texts.
15
+
16
+ ### Added
17
+
18
+ - **Whether the label prints a statement's code.** Each statement of the
19
+ label says under `code_source` whether the sheet prints its code
20
+ (`printed`) or only its sentence, to which the parser assigned the code
21
+ (`from_wording`), and `hazard_statements.codes_from_wording` lists the
22
+ assigned codes. The code lists stay as they are. `as_printed(text,
23
+ codes_from_wording)` gives a statement text without the assigned codes,
24
+ as the sheet prints it (SDB-174).
25
+ - **A hazard statement of the classification that the label does not
26
+ carry.** Where section 2.1 names an H statement for the classification and
27
+ the label carries neither its code nor its wording, the sheet gets the
28
+ finding `hazard-statement-not-in-label` with the code. Not required are H400
29
+ beside H410 and H318 beside H314 (CLP Annex III Part 1) and H304 on an
30
+ aerosol (Annex I 1.3.3); not where the label carries no H statement at all,
31
+ which `label-without-hazard-statement` says (SDB-173).
32
+ - **Findings for the label of a classified product.** Where section 2
33
+ classifies the product under CLP and the label (2.2) is not found
34
+ (`label-not-found`), carries no H statement
35
+ (`label-without-hazard-statement`) or no P statement
36
+ (`label-without-precautionary-statement`), the sheet gets a finding. The
37
+ reading stays as the sheet prints it; nothing is added. Each finding says
38
+ "not found": the sheet may lack it, or the parser may have missed it. Not
39
+ for an article, a classification under the old law, or a label under the
40
+ directives before CLP on a sheet dated before 1 June 2015; the parser notes
41
+ such a label (`label-before-clp`) (SDB-169 to SDB-172).
42
+
43
+ ### Fixed
44
+
45
+ - **A section heading numbered with a colon starts its section.** "3:
46
+ Zusammensetzung / Angaben zu den Bestandteilen" is the heading of section 3
47
+ as "3. Zusammensetzung …" is, when the title fits the number; before, section
48
+ 2 ran on into section 3, and an ingredient's classification counted as the
49
+ product's (SDB-175).
50
+
51
+ ## 0.10.0 – 2026-11-20
52
+
53
+ ### Breaking
54
+
55
+ - **H360 and H361 keep the letters of their effect.** A label's `H360D`,
56
+ `H360FD`, `H361fd` and the other codes of Annex VI of the CLP Regulation
57
+ are read as such, letters as the regulation sets them (`H360DF` is
58
+ `H360FD`, `H361D` is `H361d`); before, the code stood without its letters.
59
+ A sentence that names one effect is that effect's code where the label
60
+ prints no code, two effects together are the joined code, and the
61
+ regulation's wording of a lettered code is the one of Annex VI. Pictogram
62
+ and signal word do not change (SDB-155).
63
+ A caller that compares codes with `H360` or `H361` has to take the
64
+ lettered codes too.
65
+ - **`PARSER_VERSION` is 19** (0.9.0: 16). `EXTRACTION_VERSION` stays 2:
66
+ stored texts are read again as they are; readings kept from before should
67
+ be made again from them.
68
+
69
+ ### Changed
70
+
71
+ - **Precautionary statements printed as sentences only are read in the
72
+ forms Annex IV of the CLP Regulation allows.** Where the label prints no P
73
+ code and carries a hazard statement, a sentence that selects from a choice
74
+ ("Schutzhandschuhe und Augenschutz tragen." is P280), fills in the
75
+ supplier's detail for "…" or leaves out a part in brackets gives its code;
76
+ before, only the regulation's exact wording did. Where two statements cover
77
+ the same words, the longer one counts, H and EUH statements included; where
78
+ two cover them alike, neither does, and the new message
79
+ `precautionary-wording-unclear` names them. Never outside the label
80
+ (SDB-163).
81
+ - **Codes printed with a space are read.** "H 315", "P 305 + P 351 + P 338"
82
+ and "H 360 D" are the codes where they can be nothing else: the number is
83
+ a statement of the regulation, the code opens its line or follows a list
84
+ mark, "+", a colon, semicolon or comma, and the line's end, a separator,
85
+ the statement's own first word or, at a line's start, the sheet's sentence
86
+ follows. Measures, table values and names ("H 330 mm", "NOAEL P 250",
87
+ "P 750") stay as they are. Before, only "EUH 208" was read this way
88
+ (SDB-164).
89
+
90
+ ### Added
91
+
92
+ - **The message `label-line-unassigned`** names each line of a label that the
93
+ parser assigned to nothing: no code, no signal word, no pictogram, no
94
+ statement it read, no field name, no statement about the classification,
95
+ no substance in the list that determines the label, no page furniture. It
96
+ changes no reading; it shows where rules are missing (SDB-165).
97
+
98
+ - **P501 from its disposal sentence, also beside other P codes.**
99
+ "Inhalt/Behälter gemäß lokalen/regionalen/nationalen/internationalen
100
+ Vorschriften entsorgen." and the like (content and container, the
101
+ supplier's detail, then "entsorgen" or "zuführen") are read as P501
102
+ inside the label, where it carries a hazard statement and prints no P501
103
+ code; the precautionary text keeps the sheet's sentence. "Leere Behälter
104
+ können über den Hausmüll entsorgt werden." is no P501 (SDB-166).
105
+
106
+ - **The messages `line-too-long` and `statement-too-long`** name what the
107
+ new bounds on a reading (see Security) left out: how many lines were too
108
+ long and the length of the longest, and the codes a statement cut off
109
+ after 100 lines begins with (SDB-167, SDB-168).
110
+
111
+ ### Security
112
+
113
+ - **The embedded eSDScom file is read with `defusedxml`.** A document type,
114
+ an entity or an external reference makes it unreadable
115
+ (`embedded-doctype`), in any encoding. Before, the bytes were searched
116
+ for "<!DOCTYPE" and "<!ENTITY", which a file in UTF-16 got past, and the
117
+ standard library's parser expanded its entities. New dependency:
118
+ `defusedxml` (Python Software Foundation License) (SDW-11).
119
+ - **Five patterns of the parser no longer slow down on long runs of blanks
120
+ or dashes.** The revision date's label, the version line with its
121
+ creation date, a classification given as a dash, a label without a signal
122
+ word, and the substances before EUH208 read each character one way now; a
123
+ crafted text of a few thousand blanks took them seconds before. The
124
+ readings are the same.
125
+ - **Four more patterns no longer slow down on one long word.** The e-mail
126
+ address, a display named on a cover page, an article number in front of a
127
+ product's name and the addition at the end of a name start only where a
128
+ run of their characters starts, or ask for the digit ahead; before, a run
129
+ of some ten thousand letters or digits took seconds. The readings are the
130
+ same.
131
+ - **A long run of capitalised words in a label no longer slows the reading
132
+ down.** A classification entry in the label ("Skin Corr. 1, H314") is
133
+ found by the last word of its class only; before, every capitalised word
134
+ in front of it was tried, and a run of some twenty thousand characters
135
+ took seconds. The codes read from the label are the same.
136
+ - **Five more places no longer slow down on a long run.** Lines of nothing
137
+ but dashes or bullets above "Sonstige Gefahren", colons before "keine" or
138
+ "N/A" in the classification, a long word before an article number's
139
+ label, letters with full stops at the end of a supplier's name, and a run
140
+ of brackets in a name's label were each read again from every character
141
+ of the run; some twenty thousand characters took one to ten seconds. The
142
+ readings are the same.
143
+ - **What a reading takes in is bounded.** A line of more than 2,000
144
+ characters is not read (`line-too-long`), and a hazard, precautionary or
145
+ EUH statement takes at most 100 lines (`statement-too-long`); the longest
146
+ line of the sheets the reader was measured on has 837 characters, the
147
+ longest statement 37 lines. A built text with a line of tens of thousands
148
+ of dashes, or a statement followed by a hundred thousand lines that end
149
+ nothing, took seconds to minutes; the bounds cover such cases whether or
150
+ not anyone has found them. The text as read stays whole, so a kept text
151
+ is read again as before (SDB-167, SDB-168).
152
+
153
+ ### Fixed
154
+
155
+ Layouts and spellings from sheets outside the collection the reader was
156
+ measured on:
157
+
158
+ - A label that numbers its own parts on (`2.3 Gefahrbestimmende
159
+ Komponenten`, `2.4 Gefahrenhinweise`) no longer ends at its "2.3"
160
+ (SDB-156).
161
+ - A signal word behind its label counts where the label then says the
162
+ diluted product needs none (SDB-157).
163
+ - No product name: an article number's label, a word for what the product is
164
+ ("Gemisch"), the article number below stacked labels; "Bezeichnung des
165
+ Stoffes oder des Gemisches:" is a name's label (SDB-158).
166
+ - The supplier behind "Name der Firma:", "Name des Herstellerunternehmens:",
167
+ "Herstellerin", "Firmendetails:", "Company:", "Name oder Handelsname", and
168
+ behind the colon of 1.3's heading; no street and no pointer as the name; a
169
+ name cut from an address or a column glued to it; "BV", "NV" and "SA" as
170
+ legal forms (SDB-159).
171
+ - "Abschnitt 7.3 angegeben" at the end of a sentence is no heading
172
+ (SDB-160).
173
+ - Dates behind "Überarbei tung" broken over a line, with a weekday in front,
174
+ and "Datum der Ausgabe:" as the issue date (SDB-161).
175
+ - A pictogram's code behind a statement and a page marker "Page n. 1 von 9"
176
+ are not part of the statement's wording (SDB-162).
177
+
178
+ ## 0.9.0 – 2026-11-19
179
+
180
+ The first public version. What it offers:
181
+
182
+ - **The command `sdb-reader`.** `show` says what each PDF is and what it
183
+ says, as a report to read; `read` gives the same as JSON, a record per
184
+ file, described by a JSON Schema the package ships. Both take single files
185
+ and whole folders. `xml` hands out the eSDScom file some PDFs carry, `ask`
186
+ puts the sheets to a language model of your choice, and `diff` compares two
187
+ runs field by field.
188
+ - **The library `sdb_reader`.** `read_sheet` reads a PDF: whether it is a
189
+ German safety data sheet, and its product, supplier, dates, classification
190
+ and label with the regulation's wording beside the sheet's. The library
191
+ judges a sheet (`sheet_findings`, `requires_sheet`), chooses the current one
192
+ among several (`current_state`), and stores a PDF's text so that a sheet can
193
+ be read again without the file. The exported names are the interface;
194
+ [What is promised](docs/api/stability.md) says what a program can build on.
195
+ - **No language model.** Reading is rules only and gives the same result for
196
+ the same file every time. Only `ask` talks to a model, and only when called.
197
+ - **Languages.** It reads German sheets; a sheet in another language is
198
+ recognised and not read. Messages come in German (the default) and English,
199
+ and in any language through a catalog of your own.
200
+ - **Documentation.** At <https://foerdebaumarkt.github.io/sdb-reader/>, one
201
+ edition per released version: the operator's manual in German and English,
202
+ the guide to the library with a reference of every exported name, and how
203
+ the reader works.
204
+ - **Python.** 3.12 to 3.15: every Python version still maintained.
205
+ - **Stage versions.** `EXTRACTION_VERSION` is 2 and `PARSER_VERSION` is 16.
206
+ A text or a reading kept with other values is read again; later releases
207
+ name every rise here, with what to do about what was kept.
208
+
209
+ Before 1.0 a minor version may break; a breaking change stands under
210
+ "Breaking" in its release's section.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sdb-reader
3
- Version: 0.9.0
3
+ Version: 0.11.0
4
4
  Summary: Reads German safety data sheets (Sicherheitsdatenblätter) under REACH and CLP, without a language model
5
5
  Author-email: "Förde-Baumarkt Holger Claussen GmbH & Co. KG" <service@foerde-baumarkt.de>
6
6
  License-Expression: AGPL-3.0-or-later
@@ -28,6 +28,7 @@ Requires-Python: >=3.12
28
28
  Description-Content-Type: text/markdown
29
29
  License-File: LICENSE
30
30
  License-File: NOTICE
31
+ Requires-Dist: defusedxml<0.8,>=0.7.1
31
32
  Requires-Dist: pymupdf-lite<3.0.0,>=2.1
32
33
  Requires-Dist: structlog>=26.1.0
33
34
  Dynamic: license-file
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "sdb-reader"
7
- version = "0.9.0"
7
+ version = "0.11.0"
8
8
  description = "Reads German safety data sheets (Sicherheitsdatenblätter) under REACH and CLP, without a language model"
9
9
  readme = "README.md"
10
10
  authors = [{ name = "Förde-Baumarkt Holger Claussen GmbH & Co. KG", email = "service@foerde-baumarkt.de" }]
@@ -43,6 +43,10 @@ classifiers = [
43
43
  "Programming Language :: Python :: 3.15",
44
44
  ]
45
45
  dependencies = [
46
+ # Parses the eSDScom file a PDF may carry without expanding entities or
47
+ # fetching external references, in any encoding (SDW-11). Upper bound: the
48
+ # next minor is a decision.
49
+ "defusedxml>=0.7.1,<0.8",
46
50
  # Reads the PDF's text layer by position and renders pages for text
47
51
  # recognition. The lean distribution of PyMuPDF: the same `pymupdf` module
48
52
  # without the layout and Markdown packages (and their numpy and onnxruntime)
@@ -95,6 +99,11 @@ dev = [
95
99
  # Checks before a release that the package index can render the built
96
100
  # package's description (make dist; docs/contributing/releasing.md).
97
101
  "twine==7.0.0",
102
+ # Type information for defusedxml, for mypy.
103
+ "types-defusedxml==0.7.0.20260504",
104
+ # Property tests: random and degenerate text through the reading, each
105
+ # reading held to a time limit (tests/test_reading_time.py).
106
+ "hypothesis==6.170.15",
98
107
  ]
99
108
 
100
109
  [tool.setuptools]
@@ -28,8 +28,10 @@ with any release, the parts that ask a language model included.
28
28
  - Judging a sheet: ``sheet_findings`` and the checks it is made of,
29
29
  ``classification`` and ``requires_sheet``.
30
30
  - Statements and pictograms: ``label_statements`` with the regulation's
31
- wording (``canonical_text``), the statement texts tidied and rendered, the
32
- pictograms that follow from H statements.
31
+ wording (``canonical_text``) and whether the label prints each code, the
32
+ statement texts tidied and rendered, ``as_printed`` for the texts without
33
+ the codes the label does not print, the pictograms that follow from H
34
+ statements.
33
35
  - Embedded data: ``read_embedded``, ``parse_embedded`` and
34
36
  ``embedded_differences``.
35
37
  - Messages: what the library notes about a file is a ``Message`` with a
@@ -90,6 +92,7 @@ from .serialise import (
90
92
  )
91
93
  from .statements import (
92
94
  StatementWording,
95
+ as_printed,
93
96
  canonical_text,
94
97
  label_statements,
95
98
  normalize_statements,
@@ -125,6 +128,7 @@ __all__ = [
125
128
  "StatedClassification",
126
129
  "StatementWording",
127
130
  "Verification",
131
+ "as_printed",
128
132
  "canonical_text",
129
133
  "classification",
130
134
  "contradicts_label",
@@ -30,10 +30,14 @@ from .extraction import ExtractionResult, normalise_pdf_text, strip_page_furnitu
30
30
  from .messages import Message, shipped_catalog
31
31
  from .model import Prompt, build_prompt, named_date, read_answer
32
32
  from .parser import split_into_sheets
33
- from .statements import normalize_statements
33
+ from .statements import lettered_code, normalize_statements
34
34
  from .versions import extraction_stage
35
35
 
36
- _H_CODE = re.compile(r"(?<![A-Za-z])H\d{3}(?!\d)(?:\s*\+\s*H\d{3}(?!\d))*")
36
+ # An H360 or H361 with the letters of its effect is one code (SDB-155).
37
+ _H_CODE = re.compile(
38
+ r"(?<![A-Za-z])H(?:36[01][A-Za-z]{1,2}(?![^\W\d_])|\d{3})(?!\d)"
39
+ r"(?:\s*\+\s*H\d{3}(?!\d))*"
40
+ )
37
41
  _P_CODE = re.compile(r"(?<![A-Za-z])P\d{3}(?!\d)(?:\s*\+\s*P\d{3}(?!\d))*")
38
42
  _EUH_CODE = re.compile(r"\bEUH\d{3}(?!\d)")
39
43
 
@@ -212,7 +216,7 @@ def send(endpoint: Endpoint, prompt: Prompt, *, retries: int = 0) -> Reply:
212
216
  headers["Authorization"] = f"Bearer {endpoint.api_key}"
213
217
  for attempt in range(retries + 1):
214
218
  # The scheme is checked where the endpoint is made: http or https only.
215
- request = urllib.request.Request(
219
+ request = urllib.request.Request( # noqa: S310 -- http(s) only, see above
216
220
  endpoint.url.rstrip("/") + "/chat/completions",
217
221
  data=body,
218
222
  headers=headers,
@@ -220,7 +224,9 @@ def send(endpoint: Endpoint, prompt: Prompt, *, retries: int = 0) -> Reply:
220
224
  )
221
225
  named: float | None = None
222
226
  try:
223
- with urllib.request.urlopen(request, timeout=endpoint.timeout) as response:
227
+ with urllib.request.urlopen( # noqa: S310 -- http(s) only, see above
228
+ request, timeout=endpoint.timeout
229
+ ) as response:
224
230
  payload = json.loads(response.read())
225
231
  break
226
232
  except urllib.error.HTTPError as exc:
@@ -298,7 +304,9 @@ def model_reading(answer: Mapping[str, Any], text: str) -> dict[str, Any]:
298
304
  # As the model wrote it, also where it did not hold up.
299
305
  "revision_date_named": named if isinstance(named, str) and named else None,
300
306
  "hazard_statements": {
301
- "hazard_codes": _codes(_H_CODE, hazard),
307
+ "hazard_codes": _unique(
308
+ lettered_code(code) for code in _codes(_H_CODE, hazard)
309
+ ),
302
310
  "precautionary_codes": _codes(_P_CODE, precautionary),
303
311
  # A model now and then lists an EUH statement among the H statements.
304
312
  "supplemental_codes": _unique(
@@ -35,7 +35,7 @@ from .messages import (
35
35
  shipped_catalog,
36
36
  shipped_languages,
37
37
  )
38
- from .reading import read_sheet
38
+ from .reading import read_sheet, within_line_limit
39
39
  from .scan import Outcome
40
40
  from .serialise import reading_to_dict
41
41
  from .verification import DocumentKind, SDSVerifier, Verification
@@ -851,7 +851,8 @@ def _ask_one(path: Path, options: _AskOptions) -> Outcome:
851
851
  done = _ask_about(
852
852
  path,
853
853
  extraction,
854
- SDSVerifier().verify(extraction),
854
+ # Judged on the text as the reading judges it (SDB-167).
855
+ SDSVerifier().verify(within_line_limit(extraction)[0]),
855
856
  (),
856
857
  options,
857
858
  catalog,
@@ -18,11 +18,22 @@ from .messages import Message
18
18
  from .parser import (
19
19
  CLASSIFIED_BEFORE_CLP,
20
20
  EUH_WORDING_OUTSIDE_LABEL,
21
+ LABEL_BEFORE_CLP,
22
+ LABEL_HEADING_PATTERN,
21
23
  LABEL_UNDETERMINED,
24
+ NO_LABEL_SECTION,
22
25
  SheetPart,
26
+ classification_hazard_codes,
23
27
  )
24
28
  from .schema import HazardStatements, SafetyDataSheet, Section, StatedClassification
25
- from .statements import SHEET_ON_REQUEST, deviates_from_regulation, label_statements
29
+ from .statements import (
30
+ SHEET_ON_REQUEST,
31
+ _catalogue,
32
+ _comparable,
33
+ base_code,
34
+ deviates_from_regulation,
35
+ label_statements,
36
+ )
26
37
 
27
38
  # Annex II of REACH in the version of Regulation (EU) 2020/878 applies from
28
39
  # 1 January 2021; sheets not conforming to it could be supplied until
@@ -253,6 +264,152 @@ def contradicts_label(sheet: SafetyDataSheet) -> bool:
253
264
  )
254
265
 
255
266
 
267
+ def label_not_found(
268
+ sheet: SafetyDataSheet, parser_warnings: Iterable[Message]
269
+ ) -> list[Message]:
270
+ """What the label of a product section 2 classifies should carry and the
271
+ parser did not find in it (SDB-169 to SDB-172).
272
+
273
+ The reading stays as it is: these are findings beside it, and each says
274
+ "not found", which is true whether the sheet lacks it or the parser missed
275
+ it. Only for a product classified under CLP (SDB-75); not under the old
276
+ law (SDB-73), not for a label under the old directives on a sheet dated
277
+ before 1 June 2015, and not for an article, which is not labelled under
278
+ CLP (SDB-142, SDB-154). A label counts as found where the parser found its
279
+ block (SDB-4), or, without one, read any element of a label in section 2.
280
+
281
+ >>> from sdb_reader import HazardStatements
282
+ >>> def sheet(label, stated=StatedClassification.CLASSIFIED):
283
+ ... return SafetyDataSheet(
284
+ ... None, None, None, None, "de", hazard_statements=label,
285
+ ... stated_classification=stated,
286
+ ... )
287
+ >>> [m.code for m in label_not_found(sheet(HazardStatements()),
288
+ ... [Message(NO_LABEL_SECTION)])]
289
+ ['label-not-found']
290
+ >>> [m.code for m in label_not_found(
291
+ ... sheet(HazardStatements(signal_word="Achtung", precautionary_codes=["P210"])),
292
+ ... [])]
293
+ ['label-without-hazard-statement']
294
+ >>> label_not_found(sheet(HazardStatements(), StatedClassification.NOT_STATED), [])
295
+ []
296
+ """
297
+ noted = {warning.code for warning in parser_warnings}
298
+ if not _label_is_judged(sheet, noted):
299
+ return []
300
+ label = sheet.hazard_statements
301
+ read_any = bool(
302
+ label.hazard_codes
303
+ or label.precautionary_codes
304
+ or label.supplemental_codes
305
+ or label.signal_word
306
+ or label.pictograms
307
+ )
308
+ if NO_LABEL_SECTION in noted and not read_any:
309
+ return [Message("label-not-found")]
310
+ findings: list[Message] = []
311
+ if not label.hazard_codes:
312
+ findings.append(Message("label-without-hazard-statement"))
313
+ if not label.precautionary_codes:
314
+ findings.append(Message("label-without-precautionary-statement"))
315
+ return findings
316
+
317
+
318
+ def _label_is_judged(sheet: SafetyDataSheet, noted: Collection[str]) -> bool:
319
+ """Whether the label findings apply to *sheet* (SDB-170): a product
320
+ classified under CLP, no article, no classification under the old law, and
321
+ no label under the old directives on a sheet dated before 1 June 2015.
322
+ *noted* are the codes of the parser's warnings."""
323
+ if sheet.stated_classification is not StatedClassification.CLASSIFIED:
324
+ return False
325
+ if sheet.stated_article or CLASSIFIED_BEFORE_CLP in noted:
326
+ return False
327
+ # A mixture could be labelled under the old directives until 1 June 2015.
328
+ dated = sheet.revision_date or sheet.issue_date
329
+ return not (
330
+ LABEL_BEFORE_CLP in noted and dated is not None and dated < CLP_FOR_MIXTURES
331
+ )
332
+
333
+
334
+ # The order of precedence Annex III Part 1 sets under Article 27 (consolidated
335
+ # version of 1 July 2026): a statement the label may leave out beside another.
336
+ _LEFT_OUT_BESIDE = {"H400": "H410", "H318": "H314"}
337
+ # An aerosol: its class in 2.1 or one of its statements (Annex I 2.3).
338
+ _AEROSOL_CLASS = re.compile(r"\bAerosol\s?[123]\b")
339
+ _AEROSOL_STATEMENTS = frozenset({"H222", "H223", "H229"})
340
+
341
+
342
+ def _not_required(
343
+ code: str, on_label: set[str], classified: set[str], hazards: str
344
+ ) -> bool:
345
+ """Whether the label may go without *code* although 2.1 names it (SDB-173).
346
+
347
+ Beside H410 or H314 the label may leave out H400 or H318 (CLP Annex III
348
+ Part 1 under Article 27). A product placed on the market as an aerosol need
349
+ not be labelled for the aspiration hazard, H304 (Annex I 1.3.3).
350
+ """
351
+ if _LEFT_OUT_BESIDE.get(code) in on_label:
352
+ return True
353
+ return code == "H304" and bool(
354
+ _AEROSOL_CLASS.search(hazards) or (classified | on_label) & _AEROSOL_STATEMENTS
355
+ )
356
+
357
+
358
+ def hazard_statements_not_in_label(
359
+ sheet: SafetyDataSheet, parser_warnings: Iterable[Message]
360
+ ) -> list[Message]:
361
+ r"""Each H statement of the classification in 2.1 that the label does not
362
+ carry (SDB-173), with its code as ``statement``.
363
+
364
+ Only where the label carries at least one H statement -- without any,
365
+ ``label-without-hazard-statement`` says so (SDB-171) -- and with the
366
+ exceptions of SDB-170. A code counts as carried where the label carries the
367
+ same statement, H360 and H361 whatever their letters, and H400 and H318 also
368
+ where the label carries H410 or H314 (CLP Article 27 with Annex III Part 1).
369
+
370
+ >>> from sdb_reader import HazardStatements
371
+ >>> section = (
372
+ ... "2.1 Einstufung des Stoffs oder Gemischs\n"
373
+ ... "Aerosol 3, H229\nAquatic Acute 1, H400\nAquatic Chronic 1, H410\n"
374
+ ... "2.2 Kennzeichnungselemente\n"
375
+ ... )
376
+ >>> sheet = SafetyDataSheet(
377
+ ... None, None, None, None, "de",
378
+ ... hazard_statements=HazardStatements(hazard_codes=["H410"]),
379
+ ... stated_classification=StatedClassification.CLASSIFIED,
380
+ ... section_text={Section.HAZARDS: section},
381
+ ... )
382
+ >>> [m.params["statement"] for m in hazard_statements_not_in_label(sheet, [])]
383
+ ['H229']
384
+ """
385
+ noted = {warning.code for warning in parser_warnings}
386
+ label = sheet.hazard_statements
387
+ if not label.hazard_codes or not _label_is_judged(sheet, noted):
388
+ return []
389
+ hazards = sheet.section_text.get(Section.HAZARDS, "")
390
+ heading = LABEL_HEADING_PATTERN.search(hazards)
391
+ label_text = _comparable(hazards[heading.start() :]) if heading else ""
392
+ on_label = {base_code(code) for code in label.hazard_codes}
393
+ # Without the wordings of its own statements: "Giftig für Wasserorganismen,
394
+ # mit langfristiger Wirkung" (H411) stands inside H410's.
395
+ wordings = _catalogue()
396
+ for code in on_label:
397
+ if own := wordings.get(code):
398
+ label_text = label_text.replace(_comparable(own), " ")
399
+ classified = {base_code(code) for code in classification_hazard_codes(hazards)}
400
+ missing: list[Message] = []
401
+ for code in sorted(classification_hazard_codes(hazards)):
402
+ base = base_code(code)
403
+ if base in on_label or _not_required(base, on_label, classified, hazards):
404
+ continue
405
+ # The label prints the statement's wording without its code.
406
+ wording = wordings.get(base)
407
+ if wording and _comparable(wording) in label_text:
408
+ continue
409
+ missing.append(Message("hazard-statement-not-in-label", {"statement": code}))
410
+ return missing
411
+
412
+
256
413
  class Classification(StrEnum):
257
414
  """What a sheet's label states about the product's classification."""
258
415
 
@@ -323,7 +480,9 @@ _COMPONENT_ROLE = re.compile(
323
480
  r"\b(?:harz|härter|haerter|mörtel|moertel|basis)\b", re.IGNORECASE
324
481
  )
325
482
  # An addition in brackets or a number at the end of a name: "(Finish)", "-019".
326
- _NAME_TAIL = re.compile(r"(?:\([^()]*\)|[\s,;:–-]+\d{2,4})\s*$")
483
+ # The separators start only where a run of them starts: the same tail, and a
484
+ # long run of them is read once.
485
+ _NAME_TAIL = re.compile(r"(?:\([^()]*\)|(?<![\s,;:–-])[\s,;:–-]+\d{2,4})\s*$")
327
486
  _NOT_A_NAME_S = re.compile(r"[^\w]+")
328
487
 
329
488
 
@@ -466,6 +625,7 @@ def sheet_findings(
466
625
  ... "Verordnung (EG) Nr. 1907/2006\\n"
467
626
  ... "Signalwort: Achtung\\n"
468
627
  ... "H315: Verursacht Hautreizungen.\\n"
628
+ ... "P280: Schutzhandschuhe tragen.\\n"
469
629
  ... )
470
630
  >>> result = read_extraction(
471
631
  ... extraction_from_dict(
@@ -538,6 +698,9 @@ def sheet_findings(
538
698
  if contradicts_label(sheet):
539
699
  findings.append(Message("classification-contradicts-label"))
540
700
 
701
+ findings.extend(label_not_found(sheet, parser_warnings))
702
+ findings.extend(hazard_statements_not_in_label(sheet, parser_warnings))
703
+
541
704
  findings.extend(
542
705
  Message("euh-outside-label", dict(outside.params))
543
706
  for outside in euh_outside_label(parser_warnings)
@@ -16,6 +16,7 @@ from dataclasses import dataclass
16
16
  from pathlib import Path
17
17
 
18
18
  import pymupdf
19
+ from defusedxml import DefusedXmlException, ElementTree as SafeET
19
20
 
20
21
  from .messages import Message
21
22
  from .schema import SafetyDataSheet
@@ -24,10 +25,6 @@ from .schema import SafetyDataSheet
24
25
  # CHEMDOX writes none.
25
26
  _ROOT = "DatasheetFeed"
26
27
 
27
- # A document type declaration is how an XML file defines entities, and an
28
- # entity is how a small file expands into a huge one. eSDScom has none.
29
- _DOCTYPE = re.compile(rb"<!\s*(?:DOCTYPE|ENTITY)", re.IGNORECASE)
30
-
31
28
  # Larger than any export seen (160 kB); an attachment beyond it is something
32
29
  # else or an attempt.
33
30
  _MAX_BYTES = 20 * 1024 * 1024
@@ -109,10 +106,15 @@ def parse_embedded(name: str, content: bytes) -> EmbeddedData:
109
106
  return EmbeddedData(
110
107
  name, unreadable=Message("embedded-too-large", {"limit_bytes": _MAX_BYTES})
111
108
  )
112
- if _DOCTYPE.search(content):
113
- return EmbeddedData(name, unreadable=Message("embedded-doctype"))
114
109
  try:
115
- root = ET.fromstring(content)
110
+ # No document type, no entity, no external reference, whatever the
111
+ # file's encoding (SDW-11): the standard library's parser expands
112
+ # internal entities, and a check of the bytes misses them in UTF-16.
113
+ root = SafeET.fromstring(
114
+ content, forbid_dtd=True, forbid_entities=True, forbid_external=True
115
+ )
116
+ except DefusedXmlException:
117
+ return EmbeddedData(name, unreadable=Message("embedded-doctype"))
116
118
  except ET.ParseError as exc:
117
119
  return EmbeddedData(
118
120
  name, unreadable=Message("embedded-not-xml", {"detail": str(exc)})
@@ -308,10 +308,16 @@ def _tesseract_version() -> str:
308
308
  """
309
309
  try:
310
310
  asked = subprocess.run(
311
- ["tesseract", "--version"], capture_output=True, timeout=30, check=False
311
+ ["tesseract", "--version"], # noqa: S607 -- tesseract from PATH (SDB-116)
312
+ capture_output=True,
313
+ timeout=30,
314
+ check=False,
312
315
  )
313
316
  languages = subprocess.run(
314
- ["tesseract", "--list-langs"], capture_output=True, timeout=30, check=False
317
+ ["tesseract", "--list-langs"], # noqa: S607 -- tesseract from PATH (SDB-116)
318
+ capture_output=True,
319
+ timeout=30,
320
+ check=False,
315
321
  )
316
322
  except FileNotFoundError as exc:
317
323
  raise ExtractionError(Message("ocr-unavailable")) from exc
@@ -342,8 +348,8 @@ def _run_tesseract(image: bytes) -> str:
342
348
  """Hand one rendered page to the binary and return what it read."""
343
349
  _tesseract_version()
344
350
  try:
345
- result = subprocess.run(
346
- ["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"],
351
+ result = subprocess.run( # noqa: S603 -- fixed arguments, no shell
352
+ ["tesseract", "stdin", "stdout", "-l", OCR_LANGUAGE, "--psm", "3"], # noqa: S607 -- tesseract from PATH (SDB-116)
347
353
  input=image,
348
354
  capture_output=True,
349
355
  timeout=OCR_PAGE_TIMEOUT_SECONDS,
@@ -55,15 +55,20 @@ MESSAGE_CODES: Mapping[str, tuple[str, ...]] = {
55
55
  "not-a-sheet": (),
56
56
  "unreadable": (),
57
57
  # Parsing the sheet.
58
+ "line-too-long": ("count", "length"),
58
59
  "no-label-section": (),
59
60
  "several-sheets": ("count",),
60
61
  "classified-before-clp": ("basis",),
62
+ "label-before-clp": (),
61
63
  "no-hazard-codes": (),
62
64
  "label-undetermined": (),
63
65
  "no-precautionary-codes": (),
64
66
  "euh-wording-outside-label": ("statement", "sentence"),
65
67
  "signal-word-not-taken": ("signal_word",),
66
68
  "pictograms-derived": ("pictograms",),
69
+ "precautionary-wording-unclear": ("codes",),
70
+ "label-line-unassigned": ("line",),
71
+ "statement-too-long": ("codes",),
67
72
  "signal-word-derived": (),
68
73
  "no-identification-section": (),
69
74
  "no-supplier-name": (),
@@ -80,6 +85,10 @@ MESSAGE_CODES: Mapping[str, tuple[str, ...]] = {
80
85
  "old-format": ("revised",),
81
86
  "classification-not-stated": (),
82
87
  "classification-contradicts-label": (),
88
+ "label-not-found": (),
89
+ "label-without-hazard-statement": (),
90
+ "hazard-statement-not-in-label": ("statement",),
91
+ "label-without-precautionary-statement": (),
83
92
  "different-products": ("products",),
84
93
  "country-editions-differ": ("fields",),
85
94
  "euh-outside-label": ("statement", "sentence"),