sdb-reader 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1011 @@
1
+ """The wording the CLP regulation prescribes for a hazard statement.
2
+
3
+ Suppliers type these sentences themselves, and they do not always type what the
4
+ regulation says: "H319 Verursacht schwere Augenreizungen" stands in the
5
+ corpus where Annex III has the singular. The wording is not the supplier's to
6
+ choose, so the regulation's is what a product shows.
7
+
8
+ Only statements with one wording are rendered. Three kinds are held back:
9
+
10
+ * **parameter** -- the sentence has a hole the supplier fills ("Inhalt/Behälter
11
+ … zuführen"), and the filling is in the sheet, not here.
12
+ * **alternatives** -- the sentence is a menu ("Schutzhandschuhe/Schutzkleidung/
13
+ Augenschutz/Gesichtsschutz tragen"), and a label carrying only "Augenschutz
14
+ tragen" would gain three requirements it does not have.
15
+ * **disputed** -- the regulation states the sentence in two places and they do
16
+ not agree. Annex I and Annex IV Part 1 say which statement a category
17
+ carries and quote it; Annex III and Annex IV Part 2 hold the wording.
18
+ Amendment M19 revised Part 1 alone, so Part 2 still has the 2008 sentence
19
+ for P103 and the M4 one for P210. P338 carries a capital "Vorhandene"
20
+ mid-sentence, H229 differs in the case of one word, and H410 drops a comma
21
+ that its own annex uses when quoting it. Seven statements are affected.
22
+ Which half governs is not ours to decide, and a superseded sentence on a
23
+ product is worse than the supplier's own, so these are left alone.
24
+
25
+ That leaves 66.3% of the statement lines in the corpus, which is the part
26
+ that needs no knowledge of the label beyond the code itself.
27
+
28
+ The data comes from the regulation, not from a third party: see
29
+ tools/clp/extract_statements.py. A published dataset exists and was used to
30
+ check this one -- 152 of the shared entries matched, the rest were an extraction
31
+ artefact of ours since fixed, except one comma where the regulation we read is
32
+ the newer text.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import json
38
+ import re
39
+ from collections.abc import Collection, Sequence
40
+ from dataclasses import dataclass
41
+ from functools import lru_cache
42
+ from pathlib import Path
43
+ from typing import NamedTuple
44
+
45
+ from .line_breaks import join_broken_words
46
+ from .schema import HazardStatements
47
+
48
+ DATA_PATH = Path(__file__).with_name("clp_statements_de.json")
49
+
50
+ # "H315: Verursacht Hautreizungen." and the combination forms, with whatever
51
+ # separator the sheet used between code and sentence.
52
+ # What a sheet or a model writes right behind a code and before its separator
53
+ # (SDB-126): the letters of an effect ("H360FD", "H361d") or the number of a
54
+ # variant ("P501.9"). It belongs to the code's mark, not to the wording.
55
+ CODE_SUFFIX = r"(?:[A-Za-z]{1,2}(?![^\W\d_])|\.\d{1,2}(?!\d))"
56
+
57
+ STATEMENT_LINE = re.compile(
58
+ r"^(\s*)((?:EUH\d{3}[A-Za-z]*|[HP]\d{3})(?:\s*\+\s*[HP]\d{3})*)"
59
+ rf"({CODE_SUFFIX}?\s*[:\-–]?\s*)(.*)$"
60
+ )
61
+
62
+ # "P102, P101, P501:" -- a sheet's column of codes, set apart from the column
63
+ # of their wordings (SDB-30). Two or more codes, a combination counting as one.
64
+ _CODE_ITEM = r"(?:EUH\d{3}|[HP]\d{3})(?:\s*\+\s*[HP]\d{3})*"
65
+ CODE_LIST_LINE = re.compile(
66
+ rf"^(\s*)({_CODE_ITEM}(?:\s*[,;]\s*{_CODE_ITEM})+)\s*[,;:]?(?:\s(.*))?$"
67
+ )
68
+
69
+
70
+ @lru_cache(maxsize=1)
71
+ def _fixed_statements() -> dict[str, str]:
72
+ """Code to wording, for statements that have exactly one."""
73
+ raw = json.loads(DATA_PATH.read_text(encoding="utf-8"))
74
+ return {
75
+ code: entry["text"]
76
+ for code, entry in raw.get("statements", {}).items()
77
+ if entry.get("kind") == "fixed" and entry.get("text")
78
+ }
79
+
80
+
81
+ def canonical_text(code: str) -> str | None:
82
+ """The regulation's wording for *code*, or None where it prescribes no single one.
83
+
84
+ >>> canonical_text("H315")
85
+ 'Verursacht Hautreizungen.'
86
+ >>> canonical_text("P280") is None
87
+ True
88
+ """
89
+ return _fixed_statements().get(re.sub(r"\s*\+\s*", "+", code.strip()))
90
+
91
+
92
+ # Cache compiled regex patterns for statement normalization
93
+ _STATEMENT_REGEX_CACHE: dict[tuple[str, ...], re.Pattern[str] | None] = {}
94
+
95
+
96
+ def normalize_statements(text: str, prefixes: Sequence[str]) -> str:
97
+ """One statement per line, a colon after each code (SDB-17).
98
+
99
+ *prefixes* are the kinds of code the text holds ("H", "EUH", "P"). Codes
100
+ joined by a plus or a dash stay on one line, as does a list of codes
101
+ ("P102, P101, P501:"), which render_statements reads as one (SDB-30).
102
+
103
+ >>> print(
104
+ ... normalize_statements(
105
+ ... "H315 Verursacht Hautreizungen. H319 Verursacht schwere Augenreizung.",
106
+ ... ("H", "EUH"),
107
+ ... )
108
+ ... )
109
+ H315: Verursacht Hautreizungen.
110
+ H319: Verursacht schwere Augenreizung.
111
+ """
112
+
113
+ if not text:
114
+ return text
115
+
116
+ normalized = text.replace("\r\n", "\n").replace("\r", "\n").strip()
117
+ if not normalized:
118
+ return normalized
119
+
120
+ uppercase_prefixes = tuple(prefix.upper() for prefix in prefixes if prefix)
121
+
122
+ # Insert missing colons: "P101 Text" → "P101: Text". Idempotent when colon present.
123
+ #
124
+ # The repetition is possessive on purpose. A greedy one can give back the
125
+ # tail it just consumed to satisfy the lookahead, and on a combination that
126
+ # already carries its colon it does exactly that: "P305 + P351 + P338:"
127
+ # backtracks to "P305 + P351", where the next characters are " + P338" and
128
+ # so not a colon, and a colon is inserted there. The three-part statement
129
+ # then reads "P305 + P351: + P338: BEI KONTAKT ..." and the split below
130
+ # turns it into two, the first holding nothing but a plus sign. Refusing to
131
+ # give the tail back makes the match fail where it should, and the code is
132
+ # left as the document wrote it.
133
+ #
134
+ # A dash where the colon should be is taken *over*, not added to. Models
135
+ # differ on this and the prompt does not settle it: gpt-5-mini writes
136
+ # "H220: Extrem entzündbares Gas", qwen3.8 writes "H220 - Extrem
137
+ # entzündbares Gas". Inserting without absorbing turns the second into
138
+ # "H220: - Extrem entzündbares Gas". Statements with one fixed wording
139
+ # survive that, because the rendering overwrites the whole text -- but the
140
+ # 61 held back as parameter, alternatives or disputed keep the stray dash,
141
+ # on every sheet that carries one.
142
+ #
143
+ # A dash with no space on either side is a separator too, when a word
144
+ # follows it: "H304-Kann bei Verschlucken …", as a sheet prints it and the
145
+ # model copies it. Not taken for one, the line was never split, and the
146
+ # rendering then gave the whole line the wording of its first code -- the
147
+ # H412 behind it was gone. A code after the dash is the
148
+ # sheet's way of joining two statements ("H335-H336 Kann …") and is left.
149
+ if uppercase_prefixes:
150
+ _code_pats = "|".join(
151
+ f"{re.escape(p)}\\d{{2,3}}{CODE_SUFFIX}?" for p in uppercase_prefixes
152
+ )
153
+ _codes = rf"((?:{_code_pats})(?:\s*\+\s*(?:{_code_pats}))*+)"
154
+ # And a dash right after the code with the space behind it, "P102-
155
+ # Darf nicht …", as two suppliers print it (SDB-47): kept as
156
+ # the separator, the line read "P102- Darf …" in the output, where the
157
+ # model's answers read "P102: Darf …".
158
+ _colon_inserter = re.compile(
159
+ rf"{_codes}(?!\s*:)[^\S\n]+[-–—][^\S\n]*|"
160
+ rf"{_codes}(?!\s*:)[^\S\n]+|"
161
+ rf"{_codes}[-–—](?=[^\W\d_])(?!{_code_pats})|"
162
+ rf"{_codes}[-–—][^\S\n]++(?!{_code_pats})",
163
+ re.IGNORECASE,
164
+ )
165
+ normalized = _colon_inserter.sub(
166
+ lambda m: f"{m.group(1) or m.group(2) or m.group(3) or m.group(4)}: ",
167
+ normalized,
168
+ )
169
+
170
+ pattern: re.Pattern[str] | None = None
171
+ if "\n" not in normalized and uppercase_prefixes:
172
+ cached = _STATEMENT_REGEX_CACHE.get(uppercase_prefixes)
173
+ if cached is None:
174
+ code_pattern_parts = [
175
+ f"{re.escape(prefix)}\\d{{2,3}}{CODE_SUFFIX}?"
176
+ for prefix in uppercase_prefixes
177
+ ]
178
+ if code_pattern_parts:
179
+ code_pattern = "|".join(code_pattern_parts)
180
+ # A dash joins codes as a plus does ("H335-H336: Kann die
181
+ # Atemwege reizen. Kann Schläfrigkeit …"): one line, so the
182
+ # first code is not left behind on the line before. A list
183
+ # ("P102, P101, P501:") stays whole too, for the rendering to
184
+ # read as one (SDB-30).
185
+ cached = re.compile(
186
+ rf"(?P<leading>\s*)(?P<code>(?:{code_pattern})(?:\s*[+\-–,;]\s*(?:{code_pattern}))*\s*:)",
187
+ re.IGNORECASE,
188
+ )
189
+ else:
190
+ cached = None
191
+ _STATEMENT_REGEX_CACHE[uppercase_prefixes] = cached
192
+ pattern = cached
193
+
194
+ if pattern is not None:
195
+ pieces: list[str] = []
196
+ last_index = 0
197
+ for match in pattern.finditer(normalized):
198
+ start, end = match.span()
199
+ leading = match.group("leading") or ""
200
+ code_segment = match.group("code")
201
+
202
+ pieces.append(normalized[last_index:start])
203
+ last_index = end
204
+
205
+ preceding = pieces[-1] if pieces else ""
206
+ needs_newline = bool(start > 0 and not preceding.endswith("\n"))
207
+
208
+ if needs_newline:
209
+ pieces[-1] = pieces[-1].rstrip()
210
+ pieces.append("\n")
211
+ else:
212
+ pieces.append(leading)
213
+
214
+ pieces.append(code_segment)
215
+
216
+ pieces.append(normalized[last_index:])
217
+ normalized = "".join(pieces)
218
+
219
+ lines = [line.strip() for line in normalized.splitlines()]
220
+ compact = [line for line in lines if line]
221
+ if not compact:
222
+ return normalized
223
+ return "\n".join(compact)
224
+
225
+
226
+ def render_statements(text: str) -> tuple[str, int]:
227
+ """Replace each line's wording with the regulation's, where there is one.
228
+
229
+ Works line by line and keeps the sheet's order, so a paragraph that mixes
230
+ rendered and untouched statements stays in the sequence the label used.
231
+ Lines the pattern does not recognise are passed through untouched -- this
232
+ corrects wording, it never removes a statement.
233
+
234
+ A line that lists codes (SDB-30) becomes one line per code, with the
235
+ regulation's wording where it has one and the code alone where it does not.
236
+ What followed the list is dropped when it holds none of their wordings: on
237
+ one line of paints it was the start of 2.3, "P501: Andere
238
+ Gefahren, die zu keiner Einstufung führen …". When it does hold them, the
239
+ line is the sheet's own statement and stays: One supplier prints "H222, H229:
240
+ Extrem entzündbares Aerosol. Behälter steht unter Druck: …". EUH statements
241
+ keep the sheet's wording (SDB-18); the regulation's is only taken for one
242
+ that stands in a list.
243
+
244
+ Returns the text and how many lines were replaced.
245
+
246
+ >>> text, replaced = render_statements(
247
+ ... "H315: Reizt die Haut.\\nP280: Schutzhandschuhe tragen."
248
+ ... )
249
+ >>> print(text)
250
+ H315: Verursacht Hautreizungen.
251
+ P280: Schutzhandschuhe tragen.
252
+ >>> replaced
253
+ 1
254
+ """
255
+ if not text:
256
+ return text, 0
257
+
258
+ replaced = 0
259
+ lines: list[str] = []
260
+ for line in text.split("\n"):
261
+ listed = CODE_LIST_LINE.match(line)
262
+ if listed:
263
+ indent, codes, after = listed.groups()
264
+ listed_codes = re.split(r"\s*[,;]\s*", codes)
265
+ if after and holds_a_wording_of(after, listed_codes):
266
+ lines.append(line)
267
+ continue
268
+ for code in listed_codes:
269
+ canonical = canonical_text(code)
270
+ if canonical is None:
271
+ lines.append(f"{indent}{code}")
272
+ else:
273
+ replaced += 1
274
+ lines.append(f"{indent}{code}: {canonical}")
275
+ continue
276
+ match = STATEMENT_LINE.match(line)
277
+ if not match:
278
+ lines.append(line)
279
+ continue
280
+ indent, code, separator, wording = match.groups()
281
+ canonical = canonical_text(code)
282
+ if (
283
+ canonical is None
284
+ or code.startswith("EUH")
285
+ or not wording.strip()
286
+ or _holds_another(code, wording)
287
+ ):
288
+ lines.append(line)
289
+ continue
290
+ if canonical != wording.strip():
291
+ replaced += 1
292
+ lines.append(f"{indent}{code}{separator}{canonical}")
293
+ return "\n".join(lines), replaced
294
+
295
+
296
+ def tidy_statements(text: str) -> str:
297
+ """Each statement once, and no code without its wording (SDB-47).
298
+
299
+ Runs after render_statements, on the statements from either source.
300
+ A statement the field already holds with the same code and wording goes:
301
+ the parser cuts its texts from the whole of section 2, and a sheet that
302
+ prints the label's statements in 2.1 as well gave each of them twice once
303
+ the regulation's wording made the two copies alike. A code standing alone
304
+ goes too when the field gives the same code a wording -- "H220" beside
305
+ "H220: Extrem entzündbares Gas." -- and otherwise takes the regulation's
306
+ wording where it prescribes exactly one; a code it gives none keeps
307
+ standing alone.
308
+
309
+ >>> print(
310
+ ... tidy_statements(
311
+ ... "H315: Verursacht Hautreizungen.\\n"
312
+ ... "H315: Verursacht Hautreizungen.\\n"
313
+ ... "H319"
314
+ ... )
315
+ ... )
316
+ H315: Verursacht Hautreizungen.
317
+ H319: Verursacht schwere Augenreizung.
318
+ """
319
+ if not text:
320
+ return text
321
+ parsed: list[tuple[str, str | None, str]] = []
322
+ worded: set[str] = set()
323
+ for line in text.split("\n"):
324
+ match = STATEMENT_LINE.match(line)
325
+ if match is None:
326
+ parsed.append((line, None, ""))
327
+ continue
328
+ code = re.sub(r"\s*\+\s*", "+", match.group(2))
329
+ wording = match.group(4).strip()
330
+ if wording:
331
+ worded.add(code)
332
+ parsed.append((line, code, wording))
333
+ lines: list[str] = []
334
+ seen: set[str] = set()
335
+ for line, statement, wording in parsed:
336
+ if statement is not None and not wording:
337
+ if statement in worded:
338
+ continue
339
+ canonical = canonical_text(statement)
340
+ # "BEI VERSCHLUCKEN:" only opens a combination (P301+P310); on its
341
+ # own it tells the reader nothing to do.
342
+ if canonical is not None and not canonical.endswith(":"):
343
+ line = f"{line.rstrip(' :-–')}: {canonical}"
344
+ key = _comparable(line)
345
+ if key in seen:
346
+ continue
347
+ seen.add(key)
348
+ lines.append(line)
349
+ return "\n".join(lines)
350
+
351
+
352
+ @lru_cache(maxsize=1)
353
+ def _catalogue() -> dict[str, str]:
354
+ raw = json.loads(DATA_PATH.read_text(encoding="utf-8"))
355
+ return {
356
+ code: entry["text"]
357
+ for code, entry in raw.get("statements", {}).items()
358
+ if entry.get("text")
359
+ }
360
+
361
+
362
+ # How many words of a wording identify it when it is not printed whole: a
363
+ # menu's first choice, the part before a hole.
364
+ _OPENING_WORDS = 3
365
+
366
+ # How many of a wording's last words mark where it ends (SDB-51): enough not
367
+ # to be met by chance, few enough to survive an older wording of the start.
368
+ _CLOSING_WORDS = 3
369
+ _OPTIONAL = re.compile(r"\[[^\]]*\]")
370
+
371
+
372
+ class WordingBounds(NamedTuple):
373
+ """How a statement's regulation wording begins and ends."""
374
+
375
+ opening: re.Pattern[str]
376
+ # None where the wording ends in a hole: "Kann die Organe schädigen … bei
377
+ # längerer oder wiederholter Exposition <Expositionsweg …>." ends with
378
+ # whatever the supplier puts there.
379
+ closing: re.Pattern[str] | None
380
+ # The alternatives of a wording that begins with a choice, else none.
381
+ choices: tuple[str, ...] = ()
382
+
383
+
384
+ def _opening(wording: str) -> str:
385
+ """The first words of *wording*, as a pattern: up to its first hole or choice.
386
+
387
+ A wording that begins with a choice begins with any of its alternatives:
388
+ "Inhalt/Behälter … zuführen." is printed "Inhalt und Behälter in …" and
389
+ "Behälter der Wertstoffsammlung …", P280's menu of protective gear
390
+ "Augenschutz oder Gesichtsschutz tragen.".
391
+ """
392
+ tokens = _HOLE.split(_OPTIONAL.sub("", wording))[0].split()
393
+ if not tokens:
394
+ return ""
395
+ if "/" in tokens[0]:
396
+ return "(?:" + "|".join(map(re.escape, _WORD.findall(tokens[0]))) + ")"
397
+ words: list[str] = []
398
+ for token in tokens[:_OPENING_WORDS]:
399
+ if "/" in token:
400
+ break
401
+ words.extend(_WORD.findall(token))
402
+ return r"[\W_]+".join(map(re.escape, words))
403
+
404
+
405
+ def _closing(wording: str) -> str:
406
+ """The last words of *wording* behind its last hole or choice, as a pattern.
407
+
408
+ A part the regulation prints in brackets at the very end ("[oder
409
+ duschen]") may follow them.
410
+ """
411
+ optional = _OPTIONAL.findall(wording)
412
+ words = _HOLE.split(_OPTIONAL.sub("", wording))[-1].split()
413
+ choice = max((i for i, word in enumerate(words) if "/" in word), default=-1)
414
+ closing = _WORD.findall(" ".join(words[choice + 1 :]))[-_CLOSING_WORDS:]
415
+ if not closing:
416
+ return ""
417
+ body = r"[\W_]+".join(map(re.escape, closing))
418
+ after = (
419
+ optional[-1] if optional and wording.rstrip(" .").endswith(optional[-1]) else ""
420
+ )
421
+ extra = _WORD.findall(after)
422
+ if extra:
423
+ body += r"(?:[\W_]+" + r"[\W_]+".join(map(re.escape, extra)) + ")?"
424
+ return body
425
+
426
+
427
+ @lru_cache(maxsize=512)
428
+ def wording_bounds(code: str) -> WordingBounds | None:
429
+ """How a statement with *code* begins and where it ends (SDB-51).
430
+
431
+ The end is the regulation's last words after its last hole or choice:
432
+ "allergische Reaktionen hervorrufen" for EUH208, "zuführen" for P501,
433
+ "tragen" for P280, "Hilfe hinzuziehen" for "Ärztlichen Rat
434
+ einholen/ärztliche Hilfe hinzuziehen". A combination the regulation does
435
+ not list begins like its first part and ends like its last ("P304+P312":
436
+ "BEI EINATMEN: Bei Unwohlsein … anrufen."). None for a code the regulation
437
+ does not word.
438
+ """
439
+ joined = re.sub(r"\s*\+\s*", "+", code.strip())
440
+ if joined in _EFFECT_OPENINGS:
441
+ return WordingBounds(_EFFECT_OPENINGS[joined], None)
442
+ catalogue = _catalogue()
443
+ first = last = catalogue.get(joined)
444
+ if first is None and "+" in joined:
445
+ parts = joined.split("+")
446
+ first, last = catalogue.get(parts[0]), catalogue.get(parts[-1])
447
+ if first is None or last is None:
448
+ return None
449
+ opening, closing = _opening(first), _closing(last)
450
+ if not opening:
451
+ return None
452
+ head = _HOLE.split(_OPTIONAL.sub("", first))[0].split()[0]
453
+ return WordingBounds(
454
+ re.compile(rf"(?<![^\W_]){opening}(?![^\W_])", re.IGNORECASE),
455
+ re.compile(rf"\b{closing}\b[.!?)\]\"“”]*", re.IGNORECASE) if closing else None,
456
+ tuple(_WORD.findall(head)) if "/" in head else (),
457
+ )
458
+
459
+
460
+ # A sentence's end, and what may stand behind it on its line.
461
+ _SENTENCE_TAIL = r"[^.!?\n]*[.!?][)\]\"“”]*"
462
+
463
+
464
+ @lru_cache(maxsize=512)
465
+ def open_end(code: str) -> re.Pattern[str] | None:
466
+ """Where a statement ends whose regulation wording ends in a hole (SDB-108).
467
+
468
+ "Kann die Organe schädigen <…> bei längerer oder wiederholter Exposition
469
+ <Expositionsweg …>." names no last words: the supplier's follow. The
470
+ statement is whole at the end of the sentence that holds the last words
471
+ the regulation does fix. None for a code whose wording has an end of its
472
+ own, or none at all.
473
+ """
474
+ bounds = wording_bounds(code)
475
+ if bounds is None or bounds.closing is not None:
476
+ return None
477
+ wording = _catalogue().get(re.sub(r"\s*\+\s*", "+", code.strip()).split("+")[-1])
478
+ fixed = [
479
+ words
480
+ for part in _HOLE.split(_OPTIONAL.sub("", wording or ""))
481
+ if (words := _WORD.findall(part))
482
+ ]
483
+ if not fixed or code.strip() in _EFFECT_OPENINGS:
484
+ # H360 and H361 are printed with one effect or both: any sentence.
485
+ return re.compile(_SENTENCE_TAIL)
486
+ tail = r"[\W_]+".join(map(re.escape, fixed[-1][-_CLOSING_WORDS:]))
487
+ return re.compile(rf"\b{tail}\b{_SENTENCE_TAIL}", re.IGNORECASE)
488
+
489
+
490
+ # H360 and H361 begin with either effect (see _EFFECT_WORDINGS): "Kann das
491
+ # Kind im Mutterleib schädigen." is H360 as much as "Kann die Fruchtbarkeit
492
+ # beeinträchtigen …". They end in a hole, as in the catalogue.
493
+ _EFFECT_OPENINGS = {
494
+ "H360": re.compile(
495
+ r"(?<![^\W_])Kann[\W_]+(?:die[\W_]+Fruchtbarkeit|das[\W_]+Kind)(?![^\W_])",
496
+ re.IGNORECASE,
497
+ ),
498
+ "H361": re.compile(
499
+ r"(?<![^\W_])Kann[\W_]+vermutlich[\W_]+(?:die[\W_]+Fruchtbarkeit|das[\W_]+Kind)"
500
+ r"(?![^\W_])",
501
+ re.IGNORECASE,
502
+ ),
503
+ }
504
+ # Where a sentence can begin inside running text: at its start, or behind the
505
+ # end of the sentence or the label before it ("Prävention: Freisetzung …").
506
+ _SENTENCE_START = re.compile(r"(?:^|(?<=[.:!?] ))")
507
+ # A label left at the end of what a statement was cut from: "… Gebrauch.
508
+ # Reaktion:" before the next statement.
509
+ _LABEL_AT_END = re.compile(r"(?<=[.!?])\s+[A-ZÄÖÜ][\wäöüß /-]{0,40}:\s*$")
510
+
511
+
512
+ # A sentence's end, and a word that carries the sentence on behind a wording.
513
+ _SENTENCE_STOP = re.compile(r"[.!?](?:\s|$)")
514
+ _GOES_ON = re.compile(r"\s+[a-zäöüß]")
515
+
516
+
517
+ def _runs_on(text: str, start: int, closing: re.Pattern[str] | None) -> bool:
518
+ """Whether the sentence at *start* carries a longer wording than this one.
519
+
520
+ "Sehr giftig für Wasserorganismen" is all of H400 and the beginning of
521
+ H410, "… mit langfristiger Wirkung.": where the sentence goes on behind
522
+ the wording's last words, it is another code's.
523
+ """
524
+ if closing is None:
525
+ return False
526
+ whole = closing.search(text, start)
527
+ if whole is None or _SENTENCE_STOP.search(text, start, whole.start()):
528
+ return False
529
+ return _GOES_ON.match(text, whole.end()) is not None
530
+
531
+
532
+ def _is_its_choice(text: str, start: int, bounds: WordingBounds) -> bool:
533
+ """Whether the sentence at *start* is the wording, where one word may begin it.
534
+
535
+ A wording that begins with a choice begins with any one of its words, and
536
+ so does much else: "Behälter kann bei Erwärmung bersten." is not P501.
537
+ Its sentence also holds the wording's end ("Behälter … der
538
+ Wertstoffsammlung zuführen!") or a second word of the choice ("Inhalt und
539
+ Behälter … entsorgen.").
540
+ """
541
+ if not bounds.choices:
542
+ return True
543
+ stop = _SENTENCE_STOP.search(text, start)
544
+ sentence = text[start : stop.end() if stop else len(text)]
545
+ if bounds.closing and bounds.closing.search(sentence):
546
+ return True
547
+ named = [
548
+ choice
549
+ for choice in bounds.choices
550
+ if re.search(
551
+ rf"(?<![^\W_]){re.escape(choice)}(?![^\W_])", sentence, re.IGNORECASE
552
+ )
553
+ ]
554
+ return len(named) > 1
555
+
556
+
557
+ def wordings_apart(codes: Sequence[str], passages: Sequence[str]) -> list[str] | None:
558
+ """The sheet's sentence for each of *codes*, printed apart from them (SDB-52).
559
+
560
+ *passages* is the text that carries no code, each passage one run of lines
561
+ a sentence can run through. The codes are looked for in their order, each
562
+ by how the regulation's wording begins, at the start of a sentence; a
563
+ statement runs to the next one's start or to its passage's end, and no
564
+ further than the last words of its wording where it holds them. None unless
565
+ every code is found: half a row of codes with the sheet's sentences and
566
+ half without would say that the others have none.
567
+ """
568
+ texts = [re.sub(r"\s+", " ", passage).strip() for passage in passages]
569
+ index = offset = 0
570
+ found: list[tuple[str, int, int, re.Pattern[str] | None]] = []
571
+ for code in codes:
572
+ bounds = wording_bounds(code)
573
+ if bounds is None:
574
+ return None
575
+ while index < len(texts):
576
+ start = next(
577
+ (
578
+ match.start()
579
+ for match in bounds.opening.finditer(texts[index], offset)
580
+ if _SENTENCE_START.match(texts[index], match.start())
581
+ and not _runs_on(texts[index], match.start(), bounds.closing)
582
+ and _is_its_choice(texts[index], match.start(), bounds)
583
+ ),
584
+ None,
585
+ )
586
+ if start is not None:
587
+ break
588
+ index, offset = index + 1, 0
589
+ else:
590
+ return None
591
+ found.append((code, index, start, bounds.closing))
592
+ # The next one begins behind this one's wording: "Behälter dicht
593
+ # verschlossen halten." closes P403+P233 and does not open P501.
594
+ whole = bounds.closing.search(texts[index], start) if bounds.closing else None
595
+ offset = whole.end() if whole else start + 1
596
+ statements: list[str] = []
597
+ for position, (code, index, start, closing) in enumerate(found):
598
+ following = found[position + 1] if position + 1 < len(found) else None
599
+ end = following[2] if following and following[1] == index else None
600
+ sentence = texts[index][start:end]
601
+ # The last time the wording ends a sentence: "Schutzhandschuhe tragen.
602
+ # Augenschutz oder Gesichtsschutz tragen." is whole, and "… vor
603
+ # erneutem Tragen waschen." behind it is not P280's.
604
+ ends = [
605
+ match.end()
606
+ for match in (closing.finditer(sentence) if closing else ())
607
+ if match.end() == len(sentence) or sentence[match.end() - 1] in ".!?)]"
608
+ ]
609
+ if ends:
610
+ sentence = sentence[: ends[-1]]
611
+ statements.append(f"{code} {_LABEL_AT_END.sub('', sentence).strip()}")
612
+ return statements
613
+
614
+
615
+ def holds_a_wording_of(text: str, codes: list[str]) -> bool:
616
+ """Whether *text* carries the opening of a wording of any of *codes*.
617
+
618
+ Every kind counts here, unlike for reading codes off a label: "P280, P501:
619
+ Schutzhandschuhe tragen. Inhalt/Behälter … zuführen." is the sheet's own
620
+ statement although neither wording is fixed, and so is a list of
621
+ combinations ("P302+P352, P333+P313: BEI BERÜHRUNG MIT DER HAUT: …").
622
+ A combination counts by its own wording and by each part's.
623
+ """
624
+ haystack = _comparable(text)
625
+ catalogue = _catalogue()
626
+ for code in codes:
627
+ joined = re.sub(r"\s*\+\s*", "+", code)
628
+ for candidate in (joined, *joined.split("+")):
629
+ wording = catalogue.get(candidate)
630
+ if not wording:
631
+ continue
632
+ opening = re.split(r"[/<…]", wording, maxsplit=1)[0]
633
+ words = _comparable(opening).split()[:_OPENING_WORDS]
634
+ # Whole words: "Inhalt" is not in "Gefährliche Inhaltsstoffe".
635
+ if words and f" {' '.join(words)} " in f" {haystack} ":
636
+ return True
637
+ return False
638
+
639
+
640
+ def wordings_in_print_order(
641
+ codes: Collection[str], passages: Sequence[str]
642
+ ) -> list[str] | None:
643
+ """The sheet's sentence for each of *codes* where the label prints no code (SDB-52).
644
+
645
+ A label that prints its statements as sentences alone (SDB-6) gives the
646
+ codes no order; they take the one their sentences stand in.
647
+ """
648
+ texts = [re.sub(r"\s+", " ", passage).strip() for passage in passages]
649
+ places: list[tuple[int, int, str]] = []
650
+ for code in codes:
651
+ bounds = wording_bounds(code)
652
+ place = next(
653
+ (
654
+ (index, match.start())
655
+ for index, text in enumerate(texts)
656
+ for match in (bounds.opening.finditer(text) if bounds else ())
657
+ if _SENTENCE_START.match(text, match.start())
658
+ ),
659
+ None,
660
+ )
661
+ if place is None:
662
+ return None
663
+ places.append((*place, code))
664
+ return wordings_apart([code for *_, code in sorted(places)], passages)
665
+
666
+
667
+ # A statement of the same kind as the line's own: H and EUH share a field, P
668
+ # has its own.
669
+ _SAME_KIND = {
670
+ "H": re.compile(r"(?<![A-Za-z0-9])(?:EUH|H)\d{3}(?!\d)"),
671
+ "P": re.compile(r"(?<![A-Za-z0-9])P\d{3}(?!\d)"),
672
+ }
673
+
674
+
675
+ def _holds_another(code: str, wording: str) -> bool:
676
+ """Whether *wording* carries a further statement of the line's own kind.
677
+
678
+ Replacing such a line's wording would delete that statement: a sheet that
679
+ joins codes ("H335-H336 Kann …") leaves several on one line. A code of the other kind is
680
+ text that ran in from the next block, which the other field carries, and
681
+ the replacement still drops it.
682
+ """
683
+ return bool(_SAME_KIND["P" if code.startswith("P") else "H"].search(wording))
684
+
685
+
686
+ def _comparable(text: str) -> str:
687
+ """Reduce a sentence to what identifies it, and nothing else.
688
+
689
+ Case, punctuation and spacing all vary between sheets printing the same
690
+ statement -- one prints "Giftig für Wasserorganismen mit langfristiger
691
+ Wirkung", the regulation writes it with a comma -- and none of that
692
+ distinguishes one statement from another.
693
+ """
694
+ lowered = text.casefold().replace("ß", "ss")
695
+ return re.sub(r"\s+", " ", re.sub(r"[^\wäöü ]+", " ", lowered)).strip()
696
+
697
+
698
+ @lru_cache(maxsize=1)
699
+ def _wordings() -> tuple[tuple[str, str], ...]:
700
+ """Every statement wording that identifies exactly one code.
701
+
702
+ Longest first, which is what makes the lookup safe: "Entzündbares Aerosol"
703
+ is a substring of "Extrem entzündbares Aerosol", and the two are mutually
704
+ exclusive categories. Scanning shortest-first reported both on 352 sheets;
705
+ longest-first reports the right one on all of them.
706
+
707
+ Parameter and alternatives statements are excluded -- their wording carries
708
+ a hole the supplier fills, so it never appears verbatim. Disputed ones are
709
+ kept: the disagreement is over which wording to *print*, and either form
710
+ still identifies the code.
711
+
712
+ EUH statements are left out: they are read from their wording whether or
713
+ not the label prints codes (SDB-28, `euh_statements_by_wording`), and H and
714
+ P only where it prints none (SDB-6).
715
+ """
716
+ raw = json.loads(DATA_PATH.read_text(encoding="utf-8"))
717
+ pairs: list[tuple[str, str]] = []
718
+ for code, entry in raw.get("statements", {}).items():
719
+ if "+" in code or code.startswith("EUH"):
720
+ continue
721
+ if entry.get("kind") not in ("fixed", "disputed"):
722
+ continue
723
+ wording = _comparable(entry.get("text") or "")
724
+ if wording:
725
+ pairs.append((wording, code))
726
+ pairs.extend((_comparable(wording), code) for wording, code in _EFFECT_WORDINGS)
727
+ return tuple(sorted(pairs, key=lambda pair: -len(pair[0])))
728
+
729
+
730
+ # H360 and H361 name the effect where it is known, and CLP gives each effect
731
+ # its sentence and a letter behind the code (Annex VI, 1.1.2.1.2): H360D "Kann
732
+ # das Kind im Mutterleib schädigen.", H361f "Kann vermutlich die Fruchtbarkeit
733
+ # beeinträchtigen.". The catalogue holds the two codes with their holes, which
734
+ # no label prints; these are the sentences labels do print.
735
+ _EFFECT_WORDINGS: tuple[tuple[str, str], ...] = (
736
+ (
737
+ "Kann die Fruchtbarkeit beeinträchtigen oder das Kind im Mutterleib schädigen",
738
+ "H360",
739
+ ),
740
+ ("Kann die Fruchtbarkeit beeinträchtigen", "H360"),
741
+ ("Kann das Kind im Mutterleib schädigen", "H360"),
742
+ (
743
+ "Kann vermutlich die Fruchtbarkeit beeinträchtigen oder das Kind im "
744
+ "Mutterleib schädigen",
745
+ "H361",
746
+ ),
747
+ ("Kann vermutlich die Fruchtbarkeit beeinträchtigen", "H361"),
748
+ ("Kann vermutlich das Kind im Mutterleib schädigen", "H361"),
749
+ )
750
+
751
+
752
+ def codes_from_wording(text: str) -> set[str]:
753
+ """The codes whose wording appears in *text*.
754
+
755
+ For labels that print the sentences and not the codes, which is how 20
756
+ sheets in the corpus are written: "Gefahrenhinweise : Giftig für
757
+ Wasserorganismen mit langfristiger Wirkung." and nothing else.
758
+
759
+ A match consumes what it covers, so a longer statement wins over any
760
+ shorter one inside it. No wording here identifies more than one code.
761
+ """
762
+ if not text:
763
+ return set()
764
+ haystack = _comparable(text)
765
+ found: set[str] = set()
766
+ for wording, code in _wordings():
767
+ if wording in haystack:
768
+ found.add(code)
769
+ haystack = haystack.replace(wording, " ")
770
+ # A condition is half a statement ("BEI EINATMEN:"): among the words of a
771
+ # sentence it says nothing, unless a precautionary statement stands
772
+ # beside it (SDB-122).
773
+ precautionary = {code for code in found if code.startswith("P")}
774
+ if precautionary <= _conditions():
775
+ found -= precautionary
776
+ return found
777
+
778
+
779
+ @lru_cache(maxsize=1)
780
+ def _conditions() -> frozenset[str]:
781
+ """The precautionary codes whose wording is only a condition, ending in a colon."""
782
+ return frozenset(
783
+ code
784
+ for code, wording in _fixed_statements().items()
785
+ if code.startswith("P") and "+" not in code and wording.rstrip().endswith(":")
786
+ )
787
+
788
+
789
+ # A hole holds substance names, never the next element of the label: not a line
790
+ # that starts with a code or a label heading. Without this "Enthält Xylol" on a
791
+ # components line and a bare "Kann allergische Reaktionen hervorrufen." further
792
+ # down read as one EUH208 with the signal word and the H statements inside.
793
+ _NOT_A_NEW_ELEMENT = (
794
+ r"(?!\n[^\S\n]*(?:EUH\s?\d{3}|[HP]\d{3}|Signalwort|Gefahrenhinweis"
795
+ r"|Sicherheitshinweis|Gefahrenpiktogramm|Piktogramm))"
796
+ )
797
+ # A hole in a wording: "<Name des sensibilisierenden Stoffes>" or "…".
798
+ _HOLE = re.compile(r"<[^>]*>|…")
799
+ _WORD = re.compile(r"[^\W_]+")
800
+ # What a hole may hold: the substance names, not a page of text. The longest
801
+ # in the corpus is one supplier's 520 characters of benzotriazole reaction masses.
802
+ _HOLE_LIMIT = 1000
803
+
804
+
805
+ def _wording_pattern(wording: str, *, partial: bool = False) -> re.Pattern[str]:
806
+ """*wording* as a pattern that matches it however a sheet typesets it.
807
+
808
+ Between two words anything that is not a word may stand -- a space, a line
809
+ break, a hyphen, a comma the regulation does not have. A hole becomes a gap
810
+ of up to _HOLE_LIMIT characters that does not run past another start of the
811
+ same sentence, so "Enthält A. Kann allergische Reaktionen hervorrufen.
812
+ Enthält B. …" reads as two statements, not one.
813
+ """
814
+ # "Achtung!" opens EUH206, 207, 211 and 212, and sheets leave it out as often
815
+ # as they print it; the sentence after it is what identifies the statement.
816
+ warning = re.match(r"Achtung!\s*", wording)
817
+ if warning:
818
+ wording = wording[warning.end() :]
819
+ parts = _HOLE.split(wording)
820
+ if partial and len(parts) > 1:
821
+ # Only what follows the hole: "Kann allergische Reaktionen hervorrufen."
822
+ parts = [parts[-1]]
823
+ first = _WORD.findall(parts[0])[:1]
824
+ not_first = rf"(?!{re.escape(first[0])})" if first else ""
825
+ gap = rf"(?:{not_first}{_NOT_A_NEW_ELEMENT}[\s\S]){{1,{_HOLE_LIMIT}}}?"
826
+ pieces = [r"[\W_]+".join(map(re.escape, _WORD.findall(part))) for part in parts]
827
+ body = rf"[\W_]+{gap}[\W_]+".join(piece for piece in pieces if piece)
828
+ if warning:
829
+ body = rf"(?:Achtung[\W_]+)?{body}"
830
+ return re.compile(rf"(?<![^\W_]){body}(?![^\W_])\.?", re.IGNORECASE)
831
+
832
+
833
+ @lru_cache(maxsize=2)
834
+ def _euh_patterns(partial: bool = False) -> tuple[tuple[str, re.Pattern[str]], ...]:
835
+ """Each EUH statement that a wording identifies, longest wording first.
836
+
837
+ Fixed ones before those with a hole, so EUH204's "Enthält Isocyanate. Kann
838
+ allergische Reaktionen hervorrufen." is read as EUH204 and not as EUH208
839
+ with "Isocyanate" in its hole. Alternatives are left out: EUH018 is a menu,
840
+ and EUH201 and EUH209 hold two statements in one entry.
841
+ """
842
+ raw = json.loads(DATA_PATH.read_text(encoding="utf-8"))
843
+ entries = [
844
+ (code, entry)
845
+ for code, entry in raw.get("statements", {}).items()
846
+ if code.startswith("EUH")
847
+ and entry.get("kind") in ("fixed", "disputed", "parameter")
848
+ ]
849
+ entries.sort(
850
+ key=lambda item: (item[1]["kind"] == "parameter", -len(item[1]["text"]))
851
+ )
852
+ return tuple(
853
+ (code, _wording_pattern(entry["text"], partial=partial))
854
+ for code, entry in entries
855
+ )
856
+
857
+
858
+ # A label saying that a safety data sheet is to be had on request, in the
859
+ # wording of CLP or of the Preparations Directive (SDB-76).
860
+ SHEET_ON_REQUEST = re.compile(
861
+ r"(?<![^\W_])Sicherheitsdatenblatt[\W_]+auf[\W_]+Anfrage[\W_]+"
862
+ r"(?:für[\W_]+berufsmäßige[\W_]+(?:Verwender|Benutzer|Anwender)[\W_]+)?"
863
+ r"erhältlich(?![^\W_])",
864
+ re.IGNORECASE,
865
+ )
866
+ # The sentence of the Preparations Directive (SDB-123): "Sicherheitsdatenblatt
867
+ # auf Anfrage für berufsmäßige Verwender erhältlich." It is that directive's
868
+ # phrase and no EUH210; the parser keeps it in the text as printed.
869
+ OLD_REQUEST_SENTENCE = re.compile(
870
+ r"(?<![^\W_])Sicherheitsdatenblatt[\W_]+auf[\W_]+Anfrage[\W_]+für[\W_]+"
871
+ r"berufsmäßige[\W_]+(?:Verwender|Benutzer|Anwender)[\W_]+erhältlich(?![^\W_])\.?",
872
+ re.IGNORECASE,
873
+ )
874
+
875
+
876
+ def euh_statements_by_wording(
877
+ text: str, *, skip: Collection[str] = (), partial: bool = False
878
+ ) -> list[tuple[str, str]]:
879
+ """The EUH statements *text* prints by their wording, in the order it prints them.
880
+
881
+ Returns (code, the sheet's sentence) pairs, the sentence on one line and
882
+ with what the sheet put into a hole (SDB-28). Codes in *skip* are not
883
+ reported -- the label prints them, and their wording is read with the
884
+ code. A match consumes what it covers, a skipped one included.
885
+
886
+ *partial* also takes a statement with a hole by what follows the hole
887
+ alone. That is too loose to label a product with, but enough to notice a
888
+ sheet that means EUH208 and writes it in a form the regulation does not
889
+ ("… Kann allergische Reaktionen hervorrufen. 2-Butanonoxim.").
890
+ """
891
+ if not text:
892
+ return []
893
+ found: list[tuple[int, str, str]] = []
894
+ haystack = text
895
+ for code, pattern in _euh_patterns(partial):
896
+ # A code in *skip* still consumes its wording: printed as "EUH204
897
+ # Enthält Isocyanate. Kann allergische …", it is not EUH208 as well.
898
+ for match in [] if code in skip else pattern.finditer(haystack):
899
+ # A name broken at a line's end is joined as the reader joins
900
+ # it (SDB-54), also across the empty line the reader leaves where
901
+ # a column ends (SDB-45).
902
+ lines = join_broken_words(re.sub(r"\n\s*\n", "\n", match.group(0)))
903
+ sentence = re.sub(r"\s+", " ", lines).strip()
904
+ found.append((match.start(), code, sentence))
905
+ haystack = pattern.sub(lambda m: " " * len(m.group(0)), haystack)
906
+ return [(code, sentence) for _, code, sentence in sorted(found)]
907
+
908
+
909
+ @dataclass(frozen=True, slots=True)
910
+ class StatementWording:
911
+ """One statement of a label: what the sheet prints and what the regulation says."""
912
+
913
+ code: str
914
+ # As the parser read it; None for a code in a list (SDB-30) or without text.
915
+ sheet: str | None
916
+ # None where the regulation gives the code no single wording.
917
+ regulation: str | None
918
+
919
+
920
+ # Codes standing first in a wording: the rest of a row of codes joined by
921
+ # dashes, commas or plus signs ("H314-H335", "P102-P303+P361+P353-P310").
922
+ _LEADING_CODES = re.compile(
923
+ r"^((?:EUH\d{3}|[HP]\d{3})(?:\s*\+\s*(?:EUH\d{3}|[HP]\d{3}))*)\s*[-–,;:]?\s*"
924
+ )
925
+
926
+
927
+ def label_statements(hazards: HazardStatements) -> list[StatementWording]:
928
+ """Every H-, P- and EUH-statement of the label, each with both wordings (SDW-5).
929
+
930
+ Read from the statement texts, one statement per line (SDB-17), and only for
931
+ codes the label carries. A code the sheet prints with a wording is listed
932
+ with each wording it prints; a code it prints only bare, in a list (SDB-30)
933
+ or in a row of codes, or not at all, once with none.
934
+
935
+ >>> label = HazardStatements(
936
+ ... hazard_codes=["H315"],
937
+ ... precautionary_codes=["P280"],
938
+ ... hazard_text="H315: Reizt die Haut.",
939
+ ... precautionary_text="P280: Schutzhandschuhe tragen.",
940
+ ... )
941
+ >>> for statement in label_statements(label):
942
+ ... print(statement.code, "|", statement.sheet, "|", statement.regulation)
943
+ H315 | Reizt die Haut. | Verursacht Hautreizungen.
944
+ P280 | Schutzhandschuhe tragen. | None
945
+ """
946
+ on_label = {
947
+ *hazards.hazard_codes,
948
+ *hazards.precautionary_codes,
949
+ *hazards.supplemental_codes,
950
+ }
951
+ wordings: dict[str, list[str]] = {}
952
+
953
+ def add(code: str, wording: str | None) -> None:
954
+ code = re.sub(r"\s*\+\s*", "+", code.strip())
955
+ if not set(code.split("+")) <= on_label:
956
+ return
957
+ known = wordings.setdefault(code, [])
958
+ if wording and wording not in known:
959
+ known.append(wording)
960
+
961
+ for text, prefixes in (
962
+ (hazards.hazard_text, ("H", "EUH")),
963
+ (hazards.precautionary_text, ("P",)),
964
+ (hazards.supplemental_text, ("EUH",)),
965
+ ):
966
+ if not text:
967
+ continue
968
+ for line in normalize_statements(text, prefixes).split("\n"):
969
+ listed = CODE_LIST_LINE.match(line)
970
+ if listed:
971
+ for code in re.split(r"\s*[,;]\s*", listed.group(2)):
972
+ add(code, None)
973
+ continue
974
+ statement = STATEMENT_LINE.match(line)
975
+ if statement is None:
976
+ continue
977
+ codes = [statement.group(2)]
978
+ wording = statement.group(4).strip()
979
+ while leading := _LEADING_CODES.match(wording):
980
+ codes.append(leading.group(1))
981
+ wording = wording[leading.end() :].strip()
982
+ # A row of codes carries no wording of its own; codes joined in
983
+ # front of one wording ("H335-H336 Kann …") share it.
984
+ for code in codes:
985
+ add(code, wording or None)
986
+
987
+ for code in sorted(on_label - {part for c in wordings for part in c.split("+")}):
988
+ add(code, None)
989
+ return [
990
+ StatementWording(code, wording, canonical_text(code))
991
+ for code, printed in wordings.items()
992
+ for wording in (printed or (None,))
993
+ ]
994
+
995
+
996
+ def deviates_from_regulation(code: str, wording: str) -> bool:
997
+ """Whether the sheet's *wording* departs from the regulation's for *code* (SDW-6).
998
+
999
+ Only a wording that begins with the regulation's first three words and then
1000
+ does not hold it in full: one that begins otherwise is mostly text the
1001
+ parser found under the code, and text after the complete wording mostly a
1002
+ heading it read on into. Case, punctuation and spacing do not count.
1003
+ """
1004
+ canonical = canonical_text(code)
1005
+ if canonical is None or not wording.strip() or _holds_another(code, wording):
1006
+ return False
1007
+ printed = _comparable(wording).split()
1008
+ official = _comparable(canonical).split()
1009
+ if printed[:_OPENING_WORDS] != official[:_OPENING_WORDS]:
1010
+ return False
1011
+ return printed[: len(official)] != official