sdb-reader 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdb_reader/__init__.py +157 -0
- sdb_reader/ask.py +437 -0
- sdb_reader/ask_record.schema.json +388 -0
- sdb_reader/cli.py +1079 -0
- sdb_reader/clp_statements_de.json +997 -0
- sdb_reader/compliance.py +546 -0
- sdb_reader/current.py +134 -0
- sdb_reader/diff.py +520 -0
- sdb_reader/embedded.py +250 -0
- sdb_reader/extraction.py +506 -0
- sdb_reader/layout.py +894 -0
- sdb_reader/line_breaks.py +40 -0
- sdb_reader/messages.py +394 -0
- sdb_reader/messages_data/de.toml +146 -0
- sdb_reader/messages_data/en.toml +141 -0
- sdb_reader/model.py +321 -0
- sdb_reader/parser.py +4601 -0
- sdb_reader/pictograms.py +374 -0
- sdb_reader/presets.py +110 -0
- sdb_reader/presets.toml +23 -0
- sdb_reader/py.typed +0 -0
- sdb_reader/read_record.schema.json +792 -0
- sdb_reader/reading.py +94 -0
- sdb_reader/scan.py +265 -0
- sdb_reader/schema.py +96 -0
- sdb_reader/serialise.py +463 -0
- sdb_reader/show.py +370 -0
- sdb_reader/statements.py +1011 -0
- sdb_reader/verification.py +322 -0
- sdb_reader/versions.py +31 -0
- sdb_reader-0.9.0.dist-info/METADATA +117 -0
- sdb_reader-0.9.0.dist-info/RECORD +37 -0
- sdb_reader-0.9.0.dist-info/WHEEL +5 -0
- sdb_reader-0.9.0.dist-info/entry_points.txt +2 -0
- sdb_reader-0.9.0.dist-info/licenses/LICENSE +661 -0
- sdb_reader-0.9.0.dist-info/licenses/NOTICE +40 -0
- sdb_reader-0.9.0.dist-info/top_level.txt +1 -0
sdb_reader/__init__.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Read German safety data sheets (Sicherheitsdatenblätter) from PDF files.
|
|
2
|
+
|
|
3
|
+
It works on one PDF at a time and knows nothing about products or about the
|
|
4
|
+
other files a product has. Which of several sheets is the current one it
|
|
5
|
+
chooses from plain data a caller hands it (``current_state``); what follows
|
|
6
|
+
from that is for the program that uses it.
|
|
7
|
+
|
|
8
|
+
>>> from pathlib import Path
|
|
9
|
+
>>> from sdb_reader import read_sheet, reading_to_dict
|
|
10
|
+
>>> reading = read_sheet(Path("sheet.pdf")) # doctest: +SKIP
|
|
11
|
+
>>> reading.verification.kind # doctest: +SKIP
|
|
12
|
+
<DocumentKind.SAFETY_DATA_SHEET: 'safety-data-sheet'>
|
|
13
|
+
>>> record = reading_to_dict(reading) # doctest: +SKIP
|
|
14
|
+
|
|
15
|
+
The names this package exports are its interface (BIB-1); they change only
|
|
16
|
+
with notice. Everything reached through a submodule is internal and may change
|
|
17
|
+
with any release, the parts that ask a language model included.
|
|
18
|
+
|
|
19
|
+
- Reading: ``read_sheet`` reads a PDF, ``extract_pdf_text`` only its text,
|
|
20
|
+
``read_extraction`` a text read before. ``SheetReading`` is what comes back;
|
|
21
|
+
for a file that holds several sheets its parsed result names each one read
|
|
22
|
+
on its own as a ``SheetPart``.
|
|
23
|
+
- Keeping: ``reading_to_dict``, ``sheet_to_dict`` and ``verification_to_dict``
|
|
24
|
+
give JSON, ``sheet_from_dict`` reads a stored sheet back;
|
|
25
|
+
``extraction_to_dict`` and ``extraction_from_dict`` store a text.
|
|
26
|
+
``EXTRACTION_VERSION``, ``PARSER_VERSION`` and ``extraction_stage`` say
|
|
27
|
+
whether what was kept still holds.
|
|
28
|
+
- Judging a sheet: ``sheet_findings`` and the checks it is made of,
|
|
29
|
+
``classification`` and ``requires_sheet``.
|
|
30
|
+
- Statements and pictograms: ``label_statements`` with the regulation's
|
|
31
|
+
wording (``canonical_text``), the statement texts tidied and rendered, the
|
|
32
|
+
pictograms that follow from H statements.
|
|
33
|
+
- Embedded data: ``read_embedded``, ``parse_embedded`` and
|
|
34
|
+
``embedded_differences``.
|
|
35
|
+
- Messages: what the library notes about a file is a ``Message`` with a
|
|
36
|
+
code from ``MESSAGE_CODES``; ``shipped_catalog`` and ``load_catalog`` give
|
|
37
|
+
the ``Catalog`` that words it, German unless asked.
|
|
38
|
+
- Several sheets: ``current_state`` chooses the current one among
|
|
39
|
+
``SheetState`` values and says why (``Choice``, ``ChoiceReason``).
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from .compliance import (
|
|
43
|
+
Classification,
|
|
44
|
+
classification,
|
|
45
|
+
contradicts_label,
|
|
46
|
+
euh_outside_label,
|
|
47
|
+
predates_annex_ii,
|
|
48
|
+
predates_clp,
|
|
49
|
+
requires_sheet,
|
|
50
|
+
sheet_findings,
|
|
51
|
+
)
|
|
52
|
+
from .current import Choice, ChoiceReason, SheetState, current_state
|
|
53
|
+
from .embedded import (
|
|
54
|
+
Difference as EmbeddedDifference,
|
|
55
|
+
EmbeddedData,
|
|
56
|
+
EmbeddedSheet,
|
|
57
|
+
differences as embedded_differences,
|
|
58
|
+
parse_embedded,
|
|
59
|
+
read_embedded,
|
|
60
|
+
)
|
|
61
|
+
from .extraction import (
|
|
62
|
+
ExtractionError,
|
|
63
|
+
ExtractionResult,
|
|
64
|
+
extract_pdf_text,
|
|
65
|
+
)
|
|
66
|
+
from .messages import (
|
|
67
|
+
MESSAGE_CODES,
|
|
68
|
+
Catalog,
|
|
69
|
+
Message,
|
|
70
|
+
load_catalog,
|
|
71
|
+
shipped_catalog,
|
|
72
|
+
)
|
|
73
|
+
from .parser import ParsedSheet, SheetPart
|
|
74
|
+
from .pictograms import pictograms_for
|
|
75
|
+
from .reading import SheetReading, read_extraction, read_sheet
|
|
76
|
+
from .schema import (
|
|
77
|
+
ContactInfo,
|
|
78
|
+
HazardStatements,
|
|
79
|
+
SafetyDataSheet,
|
|
80
|
+
Section,
|
|
81
|
+
StatedClassification,
|
|
82
|
+
)
|
|
83
|
+
from .serialise import (
|
|
84
|
+
extraction_from_dict,
|
|
85
|
+
extraction_to_dict,
|
|
86
|
+
reading_to_dict,
|
|
87
|
+
sheet_from_dict,
|
|
88
|
+
sheet_to_dict,
|
|
89
|
+
verification_to_dict,
|
|
90
|
+
)
|
|
91
|
+
from .statements import (
|
|
92
|
+
StatementWording,
|
|
93
|
+
canonical_text,
|
|
94
|
+
label_statements,
|
|
95
|
+
normalize_statements,
|
|
96
|
+
render_statements,
|
|
97
|
+
tidy_statements,
|
|
98
|
+
)
|
|
99
|
+
from .verification import DocumentKind, Verification
|
|
100
|
+
from .versions import EXTRACTION_VERSION, PARSER_VERSION, extraction_stage
|
|
101
|
+
|
|
102
|
+
__all__ = [
|
|
103
|
+
"EXTRACTION_VERSION",
|
|
104
|
+
"MESSAGE_CODES",
|
|
105
|
+
"PARSER_VERSION",
|
|
106
|
+
"Catalog",
|
|
107
|
+
"Choice",
|
|
108
|
+
"ChoiceReason",
|
|
109
|
+
"Classification",
|
|
110
|
+
"ContactInfo",
|
|
111
|
+
"DocumentKind",
|
|
112
|
+
"EmbeddedData",
|
|
113
|
+
"EmbeddedDifference",
|
|
114
|
+
"EmbeddedSheet",
|
|
115
|
+
"ExtractionError",
|
|
116
|
+
"ExtractionResult",
|
|
117
|
+
"HazardStatements",
|
|
118
|
+
"Message",
|
|
119
|
+
"ParsedSheet",
|
|
120
|
+
"SafetyDataSheet",
|
|
121
|
+
"Section",
|
|
122
|
+
"SheetPart",
|
|
123
|
+
"SheetReading",
|
|
124
|
+
"SheetState",
|
|
125
|
+
"StatedClassification",
|
|
126
|
+
"StatementWording",
|
|
127
|
+
"Verification",
|
|
128
|
+
"canonical_text",
|
|
129
|
+
"classification",
|
|
130
|
+
"contradicts_label",
|
|
131
|
+
"current_state",
|
|
132
|
+
"embedded_differences",
|
|
133
|
+
"euh_outside_label",
|
|
134
|
+
"extract_pdf_text",
|
|
135
|
+
"extraction_from_dict",
|
|
136
|
+
"extraction_stage",
|
|
137
|
+
"extraction_to_dict",
|
|
138
|
+
"label_statements",
|
|
139
|
+
"load_catalog",
|
|
140
|
+
"normalize_statements",
|
|
141
|
+
"parse_embedded",
|
|
142
|
+
"pictograms_for",
|
|
143
|
+
"predates_annex_ii",
|
|
144
|
+
"predates_clp",
|
|
145
|
+
"read_embedded",
|
|
146
|
+
"read_extraction",
|
|
147
|
+
"read_sheet",
|
|
148
|
+
"reading_to_dict",
|
|
149
|
+
"render_statements",
|
|
150
|
+
"requires_sheet",
|
|
151
|
+
"sheet_findings",
|
|
152
|
+
"sheet_from_dict",
|
|
153
|
+
"sheet_to_dict",
|
|
154
|
+
"shipped_catalog",
|
|
155
|
+
"tidy_statements",
|
|
156
|
+
"verification_to_dict",
|
|
157
|
+
]
|
sdb_reader/ask.py
ADDED
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
"""Ask a language model what a safety data sheet says, and keep its answer (SDW-25).
|
|
2
|
+
|
|
3
|
+
A second reading to look at, never a source. The model is asked on its own:
|
|
4
|
+
it gets the file's text up to section 3 and nothing the parser made of it,
|
|
5
|
+
so that it can also see what the parser cut away or read wrongly. A file
|
|
6
|
+
that holds several sheets is asked about sheet by sheet (SDW-30). What it
|
|
7
|
+
says is kept as a record (SDW-26); setting it beside a reading is another
|
|
8
|
+
step (diff.py), which can be repeated for every later version of the parser
|
|
9
|
+
without asking again.
|
|
10
|
+
|
|
11
|
+
The model is reached over the chat-completions interface that local runtimes
|
|
12
|
+
and hosted services alike speak, with the standard library: whoever only reads
|
|
13
|
+
sheets installs nothing for models.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import hashlib
|
|
19
|
+
import json
|
|
20
|
+
import re
|
|
21
|
+
import time
|
|
22
|
+
import urllib.error
|
|
23
|
+
import urllib.request
|
|
24
|
+
from collections.abc import Mapping, Sequence
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from typing import Any
|
|
27
|
+
from urllib.parse import urlsplit
|
|
28
|
+
|
|
29
|
+
from .extraction import ExtractionResult, normalise_pdf_text, strip_page_furniture
|
|
30
|
+
from .messages import Message, shipped_catalog
|
|
31
|
+
from .model import Prompt, build_prompt, named_date, read_answer
|
|
32
|
+
from .parser import split_into_sheets
|
|
33
|
+
from .statements import normalize_statements
|
|
34
|
+
from .versions import extraction_stage
|
|
35
|
+
|
|
36
|
+
_H_CODE = re.compile(r"(?<![A-Za-z])H\d{3}(?!\d)(?:\s*\+\s*H\d{3}(?!\d))*")
|
|
37
|
+
_P_CODE = re.compile(r"(?<![A-Za-z])P\d{3}(?!\d)(?:\s*\+\s*P\d{3}(?!\d))*")
|
|
38
|
+
_EUH_CODE = re.compile(r"\bEUH\d{3}(?!\d)")
|
|
39
|
+
|
|
40
|
+
# The heading of section 3, where the text a model gets ends (SDW-25): the
|
|
41
|
+
# list of ingredients begins there, and their statements are not the label's.
|
|
42
|
+
# One rule, kept here: it must not depend on how the parser cuts sections.
|
|
43
|
+
_SECTION_3 = re.compile(
|
|
44
|
+
r"^[ \t]*(?:[·•*!\-–][ \t]*)?(?:0?3[ \t]+)?"
|
|
45
|
+
r"(?:abschnitt[ \t]*0?3[ \t]*[:.\-–]?[ \t]*(?:zusammensetzung|angaben\b|$)"
|
|
46
|
+
r"|0?3[ \t]*[.:\-–]?[ \t]*zusammensetzung"
|
|
47
|
+
# The title alone, where the layout sets its number on a line of its own.
|
|
48
|
+
r"|zusammensetzung[ \t]*[/,][ \t]*ang)",
|
|
49
|
+
re.IGNORECASE | re.MULTILINE,
|
|
50
|
+
)
|
|
51
|
+
# A heading this early is a table of contents, not the section.
|
|
52
|
+
EARLIEST_SECTION_3 = 200
|
|
53
|
+
|
|
54
|
+
# What a request may not replace: who is asked, and what.
|
|
55
|
+
RESERVED_REQUEST_FIELDS = frozenset({"model", "messages"})
|
|
56
|
+
# What a request can go without: all else it holds is whom and what is asked.
|
|
57
|
+
OMITTABLE_REQUEST_FIELDS = frozenset({"temperature"})
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ModelError(Exception):
|
|
61
|
+
"""The endpoint gave no answer that can be read.
|
|
62
|
+
|
|
63
|
+
``message`` says why, as a code and its values; ``str()`` of the error is
|
|
64
|
+
that message in English.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def __init__(self, message: Message) -> None:
|
|
68
|
+
super().__init__(shipped_catalog("en").render(message))
|
|
69
|
+
self.message = message
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True, slots=True)
|
|
73
|
+
class Endpoint:
|
|
74
|
+
"""Where the model is asked, and which one."""
|
|
75
|
+
|
|
76
|
+
url: str
|
|
77
|
+
model: str
|
|
78
|
+
# Sent as a bearer token; a local runtime needs none.
|
|
79
|
+
api_key: str | None = None
|
|
80
|
+
timeout: float = 300.0
|
|
81
|
+
# Fields a service needs beyond the standard ones, merged into each
|
|
82
|
+
# request: switching a model's reasoning off, a response format.
|
|
83
|
+
request: Mapping[str, Any] = field(default_factory=dict)
|
|
84
|
+
# Fields of the request a service does not take, left out (SDW-31).
|
|
85
|
+
omit: frozenset[str] = frozenset()
|
|
86
|
+
# The preset the fields come from, if one was named (SDW-31).
|
|
87
|
+
preset: str | None = None
|
|
88
|
+
|
|
89
|
+
def __post_init__(self) -> None:
|
|
90
|
+
parts = urlsplit(self.url)
|
|
91
|
+
if parts.scheme not in ("http", "https") or not parts.netloc:
|
|
92
|
+
raise ValueError(f"not an http or https address: {self.url!r}")
|
|
93
|
+
taken = sorted(RESERVED_REQUEST_FIELDS & self.request.keys())
|
|
94
|
+
if taken:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"the request's extra fields may not set {', '.join(taken)}"
|
|
97
|
+
)
|
|
98
|
+
kept = sorted(self.omit - OMITTABLE_REQUEST_FIELDS)
|
|
99
|
+
if kept:
|
|
100
|
+
raise ValueError(f"a request cannot go without {', '.join(kept)}")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass(frozen=True, slots=True)
|
|
104
|
+
class Request:
|
|
105
|
+
"""What the model is given for one sheet of a file."""
|
|
106
|
+
|
|
107
|
+
prompt: Prompt
|
|
108
|
+
# "before-section-3"; where no heading of section 3 was found, "pages"
|
|
109
|
+
# for a file that is one sheet and "sheet" for one sheet among several.
|
|
110
|
+
sent: str
|
|
111
|
+
# The file's text as sent, without the file's name and the markers: what
|
|
112
|
+
# a date the model names is checked against.
|
|
113
|
+
text: str
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def text_for_the_model(extraction: ExtractionResult, *, pages: int) -> tuple[str, str]:
|
|
117
|
+
"""The part of a file's text a model is asked about, and which part it is.
|
|
118
|
+
|
|
119
|
+
From the beginning up to the heading of section 3. Where there is no such
|
|
120
|
+
heading, or it stands at the very top, the first *pages* pages, with a
|
|
121
|
+
page header that repeats kept once.
|
|
122
|
+
"""
|
|
123
|
+
cut = _before_section_3(extraction.text)
|
|
124
|
+
if cut is not None:
|
|
125
|
+
return cut, "before-section-3"
|
|
126
|
+
each = extraction.metadata.get("pages")
|
|
127
|
+
if isinstance(each, list) and len(each) > 1:
|
|
128
|
+
first = [str(page) for page in each[:pages]]
|
|
129
|
+
return normalise_pdf_text(strip_page_furniture(first, keep_first=True)), "pages"
|
|
130
|
+
# Read by text recognition, or a single page: the text is all there is.
|
|
131
|
+
return extraction.text, "pages"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _before_section_3(text: str) -> str | None:
|
|
135
|
+
"""*text* up to the heading of section 3, or None where it has none to end at."""
|
|
136
|
+
found = _SECTION_3.search(text)
|
|
137
|
+
if found is not None and found.start() >= EARLIEST_SECTION_3:
|
|
138
|
+
return text[: found.start()].rstrip()
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def build_requests(
|
|
143
|
+
extraction: ExtractionResult, filename: str, *, pages: int
|
|
144
|
+
) -> list[Request]:
|
|
145
|
+
"""The questions about the file *extraction* was read from: one per sheet.
|
|
146
|
+
|
|
147
|
+
A file that is one sheet gives one (SDW-25). One that holds several gives
|
|
148
|
+
one for each, in the file's order: the sheet's text up to its own heading
|
|
149
|
+
of section 3, or the whole sheet where it has none (SDW-30). A cover page
|
|
150
|
+
in front of the sheets is not sent, also where one sheet stands behind it.
|
|
151
|
+
"""
|
|
152
|
+
sheets = split_into_sheets(extraction.text)
|
|
153
|
+
if sheets == [extraction.text]:
|
|
154
|
+
text, sent = text_for_the_model(extraction, pages=pages)
|
|
155
|
+
return [Request(build_prompt(text, filename), sent, text)]
|
|
156
|
+
requests = []
|
|
157
|
+
for sheet in sheets:
|
|
158
|
+
cut = _before_section_3(sheet)
|
|
159
|
+
text, sent = (cut, "before-section-3") if cut is not None else (sheet, "sheet")
|
|
160
|
+
requests.append(Request(build_prompt(text, filename), sent, text))
|
|
161
|
+
return requests
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def question_fingerprint(prompt: Prompt) -> str:
|
|
165
|
+
"""What tells the answers to one wording of the question from another's.
|
|
166
|
+
|
|
167
|
+
Of the half that is the same for every file: answers to another wording
|
|
168
|
+
are answers to another question, and are not laid beside these.
|
|
169
|
+
"""
|
|
170
|
+
return hashlib.sha256(prompt.instructions.encode("utf-8")).hexdigest()[:12]
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@dataclass(frozen=True, slots=True)
|
|
174
|
+
class Reply:
|
|
175
|
+
"""What the endpoint answered: the model's text, and what it cost if it says."""
|
|
176
|
+
|
|
177
|
+
content: str
|
|
178
|
+
prompt_tokens: int | None = None
|
|
179
|
+
completion_tokens: int | None = None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# A failure that passes by itself: the service is busy, or briefly gone.
|
|
183
|
+
RETRIED_STATUS = frozenset({429, 500, 502, 503, 504})
|
|
184
|
+
# The longest wait before a request is sent again, also where the endpoint
|
|
185
|
+
# names a longer one.
|
|
186
|
+
LONGEST_WAIT_SECONDS = 60.0
|
|
187
|
+
_wait = time.sleep
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def send(endpoint: Endpoint, prompt: Prompt, *, retries: int = 0) -> Reply:
|
|
191
|
+
"""Send *prompt* and return the model's reply (SDW-25).
|
|
192
|
+
|
|
193
|
+
Once, unless *retries* allows more: a request that fails with one of
|
|
194
|
+
RETRIED_STATUS, a timeout or no connection is then sent again, up to
|
|
195
|
+
*retries* times, after a growing wait. Any other failure, and a reply
|
|
196
|
+
that cannot be read, fails at once.
|
|
197
|
+
"""
|
|
198
|
+
standard = {"temperature": 0}
|
|
199
|
+
body = json.dumps(
|
|
200
|
+
{
|
|
201
|
+
**{k: v for k, v in standard.items() if k not in endpoint.omit},
|
|
202
|
+
**endpoint.request,
|
|
203
|
+
"model": endpoint.model,
|
|
204
|
+
"messages": [
|
|
205
|
+
{"role": "system", "content": prompt.instructions},
|
|
206
|
+
{"role": "user", "content": prompt.document},
|
|
207
|
+
],
|
|
208
|
+
}
|
|
209
|
+
).encode("utf-8")
|
|
210
|
+
headers = {"Content-Type": "application/json"}
|
|
211
|
+
if endpoint.api_key:
|
|
212
|
+
headers["Authorization"] = f"Bearer {endpoint.api_key}"
|
|
213
|
+
for attempt in range(retries + 1):
|
|
214
|
+
# The scheme is checked where the endpoint is made: http or https only.
|
|
215
|
+
request = urllib.request.Request(
|
|
216
|
+
endpoint.url.rstrip("/") + "/chat/completions",
|
|
217
|
+
data=body,
|
|
218
|
+
headers=headers,
|
|
219
|
+
method="POST",
|
|
220
|
+
)
|
|
221
|
+
named: float | None = None
|
|
222
|
+
try:
|
|
223
|
+
with urllib.request.urlopen(request, timeout=endpoint.timeout) as response:
|
|
224
|
+
payload = json.loads(response.read())
|
|
225
|
+
break
|
|
226
|
+
except urllib.error.HTTPError as exc:
|
|
227
|
+
# The error is the response, and holds its connection until closed.
|
|
228
|
+
with exc:
|
|
229
|
+
failure = ModelError(Message("model-http-error", {"status": exc.code}))
|
|
230
|
+
if exc.code not in RETRIED_STATUS:
|
|
231
|
+
raise failure from exc
|
|
232
|
+
cause: Exception = exc
|
|
233
|
+
named = _seconds(exc.headers.get("Retry-After"))
|
|
234
|
+
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
235
|
+
reason = getattr(exc, "reason", None) or exc
|
|
236
|
+
failure = ModelError(Message("model-unreachable", {"reason": str(reason)}))
|
|
237
|
+
cause = exc
|
|
238
|
+
except json.JSONDecodeError as exc:
|
|
239
|
+
raise ModelError(Message("model-reply-not-json")) from exc
|
|
240
|
+
if attempt == retries:
|
|
241
|
+
raise failure from cause
|
|
242
|
+
_wait(min(LONGEST_WAIT_SECONDS, named if named is not None else 2.0**attempt))
|
|
243
|
+
try:
|
|
244
|
+
content = payload["choices"][0]["message"]["content"]
|
|
245
|
+
except (KeyError, IndexError, TypeError) as exc:
|
|
246
|
+
raise ModelError(Message("model-reply-without-message")) from exc
|
|
247
|
+
if not isinstance(content, str) or not content.strip():
|
|
248
|
+
raise ModelError(Message("model-reply-empty"))
|
|
249
|
+
usage = payload.get("usage") if isinstance(payload, Mapping) else None
|
|
250
|
+
usage = usage if isinstance(usage, Mapping) else {}
|
|
251
|
+
return Reply(
|
|
252
|
+
content,
|
|
253
|
+
prompt_tokens=_count(usage.get("prompt_tokens")),
|
|
254
|
+
completion_tokens=_count(usage.get("completion_tokens")),
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _seconds(value: str | None) -> float | None:
|
|
259
|
+
"""A wait the endpoint names in seconds; a date or nonsense is no answer."""
|
|
260
|
+
try:
|
|
261
|
+
seconds = float(value) if value is not None else None
|
|
262
|
+
except ValueError:
|
|
263
|
+
return None
|
|
264
|
+
return seconds if seconds is not None and seconds >= 0 else None
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _count(value: object) -> int | None:
|
|
268
|
+
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def read_model_answer(raw: str) -> dict[str, Any]:
|
|
272
|
+
"""The model's reply as an answer, or ModelError when it is none."""
|
|
273
|
+
try:
|
|
274
|
+
return read_answer(raw)
|
|
275
|
+
except ValueError as exc:
|
|
276
|
+
raise ModelError(
|
|
277
|
+
Message("model-answer-unreadable", {"detail": str(exc)})
|
|
278
|
+
) from exc
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def model_reading(answer: Mapping[str, Any], text: str) -> dict[str, Any]:
|
|
282
|
+
"""What the model said, in the form a reading has it (SDW-26).
|
|
283
|
+
|
|
284
|
+
The names are those of a ``read`` record's ``sheet``, as far as a model is
|
|
285
|
+
asked: the label's statements and their codes, the supplier, the date of
|
|
286
|
+
revision. *text* is what the model was sent: a date it names counts only
|
|
287
|
+
if it stands there.
|
|
288
|
+
"""
|
|
289
|
+
hazard = _string(answer.get("hazard_statements"))
|
|
290
|
+
precautionary = _string(answer.get("precautionary_statements"))
|
|
291
|
+
supplemental = _string(answer.get("supplemental_phrases"))
|
|
292
|
+
supplier = answer.get("supplier")
|
|
293
|
+
supplier = supplier if isinstance(supplier, Mapping) else {}
|
|
294
|
+
named = answer.get("revision_date")
|
|
295
|
+
date = named_date(named, text)
|
|
296
|
+
return {
|
|
297
|
+
"revision_date": date.value.isoformat() if date else None,
|
|
298
|
+
# As the model wrote it, also where it did not hold up.
|
|
299
|
+
"revision_date_named": named if isinstance(named, str) and named else None,
|
|
300
|
+
"hazard_statements": {
|
|
301
|
+
"hazard_codes": _codes(_H_CODE, hazard),
|
|
302
|
+
"precautionary_codes": _codes(_P_CODE, precautionary),
|
|
303
|
+
# A model now and then lists an EUH statement among the H statements.
|
|
304
|
+
"supplemental_codes": _unique(
|
|
305
|
+
_EUH_CODE.findall(supplemental) + _EUH_CODE.findall(hazard)
|
|
306
|
+
),
|
|
307
|
+
"hazard_text": hazard or None,
|
|
308
|
+
"precautionary_text": precautionary or None,
|
|
309
|
+
"supplemental_text": supplemental or None,
|
|
310
|
+
},
|
|
311
|
+
"contact_info": {
|
|
312
|
+
"supplier_name": _string(supplier.get("name")) or None,
|
|
313
|
+
"supplier_address": _string(supplier.get("address")) or None,
|
|
314
|
+
"supplier_phone": _string(supplier.get("phone")) or None,
|
|
315
|
+
"supplier_email": _string(supplier.get("email")) or None,
|
|
316
|
+
"supplier_website": _string(supplier.get("website")) or None,
|
|
317
|
+
},
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
# The texts of a label, and the kinds of code each holds.
|
|
322
|
+
_TEXT_PREFIXES = (
|
|
323
|
+
("hazard_text", ("H", "EUH")),
|
|
324
|
+
("precautionary_text", ("P",)),
|
|
325
|
+
("supplemental_text", ("EUH",)),
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def united_reading(readings: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
|
330
|
+
"""The reading of a file from those of its sheets (SDW-30).
|
|
331
|
+
|
|
332
|
+
The file carries every sheet's statements, as a reading does (SDB-9):
|
|
333
|
+
the codes in the order they first appear, the statements one sheet after
|
|
334
|
+
the other, each on its line, a statement two sheets share once. Supplier and date are the first
|
|
335
|
+
sheet's.
|
|
336
|
+
"""
|
|
337
|
+
first = readings[0]
|
|
338
|
+
labels = [reading["hazard_statements"] for reading in readings]
|
|
339
|
+
united: dict[str, Any] = {}
|
|
340
|
+
for key in ("hazard_codes", "precautionary_codes", "supplemental_codes"):
|
|
341
|
+
united[key] = _unique(code for label in labels for code in label[key])
|
|
342
|
+
for key, prefixes in _TEXT_PREFIXES:
|
|
343
|
+
# A model often writes a sheet's statements in one line: each is put
|
|
344
|
+
# on its own before the sheets are joined, or the joined text would
|
|
345
|
+
# read as one statement per sheet (SDB-17).
|
|
346
|
+
lines = _unique(
|
|
347
|
+
line
|
|
348
|
+
for label in labels
|
|
349
|
+
if label[key]
|
|
350
|
+
for line in normalize_statements(label[key], prefixes).split("\n")
|
|
351
|
+
if line.strip()
|
|
352
|
+
)
|
|
353
|
+
united[key] = "\n".join(lines) or None
|
|
354
|
+
return {
|
|
355
|
+
"revision_date": first["revision_date"],
|
|
356
|
+
"revision_date_named": first["revision_date_named"],
|
|
357
|
+
"hazard_statements": united,
|
|
358
|
+
"contact_info": dict(first["contact_info"]),
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _usage(replies: Sequence[Reply]) -> dict[str, int | None] | None:
|
|
363
|
+
"""What *replies* cost together, or None where no endpoint said."""
|
|
364
|
+
|
|
365
|
+
def total(counts: list[int | None]) -> int | None:
|
|
366
|
+
named = [count for count in counts if count is not None]
|
|
367
|
+
return sum(named) if named else None
|
|
368
|
+
|
|
369
|
+
usage = {
|
|
370
|
+
"prompt_tokens": total([reply.prompt_tokens for reply in replies]),
|
|
371
|
+
"completion_tokens": total([reply.completion_tokens for reply in replies]),
|
|
372
|
+
}
|
|
373
|
+
return usage if any(count is not None for count in usage.values()) else None
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def asked_record(
|
|
377
|
+
endpoint: Endpoint,
|
|
378
|
+
requests: Sequence[Request],
|
|
379
|
+
replies: Sequence[Reply],
|
|
380
|
+
answers: Sequence[Mapping[str, Any]],
|
|
381
|
+
) -> dict[str, Any]:
|
|
382
|
+
"""The part of an ``ask`` record that a model's answers fill (SDW-26, SDW-30).
|
|
383
|
+
|
|
384
|
+
One request, reply and answer per sheet of the file, in its order.
|
|
385
|
+
"""
|
|
386
|
+
readings = [
|
|
387
|
+
model_reading(answer, request.text)
|
|
388
|
+
for request, answer in zip(requests, answers, strict=True)
|
|
389
|
+
]
|
|
390
|
+
sent = {request.sent for request in requests}
|
|
391
|
+
record: dict[str, Any] = {
|
|
392
|
+
"asked": True,
|
|
393
|
+
"model": endpoint.model,
|
|
394
|
+
"sheet_count": len(requests),
|
|
395
|
+
# Of several sheets: cut before section 3 only if every one was.
|
|
396
|
+
"sent": sent.pop() if len(sent) == 1 else "sheet",
|
|
397
|
+
"stage": extraction_stage(),
|
|
398
|
+
"question": question_fingerprint(requests[0].prompt),
|
|
399
|
+
}
|
|
400
|
+
if endpoint.preset:
|
|
401
|
+
record["preset"] = endpoint.preset
|
|
402
|
+
if endpoint.request:
|
|
403
|
+
record["request"] = dict(endpoint.request)
|
|
404
|
+
usage = _usage(replies)
|
|
405
|
+
if usage is not None:
|
|
406
|
+
record["usage"] = usage
|
|
407
|
+
if len(requests) == 1:
|
|
408
|
+
record["reading"] = readings[0]
|
|
409
|
+
record["answer"] = dict(answers[0])
|
|
410
|
+
return record
|
|
411
|
+
record["reading"] = united_reading(readings)
|
|
412
|
+
parts = []
|
|
413
|
+
for request, reply, reading, answer in zip(
|
|
414
|
+
requests, replies, readings, answers, strict=True
|
|
415
|
+
):
|
|
416
|
+
part: dict[str, Any] = {"sent": request.sent}
|
|
417
|
+
cost = _usage([reply])
|
|
418
|
+
if cost is not None:
|
|
419
|
+
part["usage"] = cost
|
|
420
|
+
part["reading"] = reading
|
|
421
|
+
part["answer"] = dict(answer)
|
|
422
|
+
parts.append(part)
|
|
423
|
+
record["parts"] = parts
|
|
424
|
+
return record
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _codes(pattern: re.Pattern[str], text: str) -> list[str]:
|
|
428
|
+
"""The codes of *text* in its order, a joined one as one: "P305+P351"."""
|
|
429
|
+
return _unique(re.sub(r"\s+", "", code) for code in pattern.findall(text))
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _unique(codes: Any) -> list[str]:
|
|
433
|
+
return list(dict.fromkeys(codes))
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _string(value: Any) -> str:
|
|
437
|
+
return value if isinstance(value, str) else ""
|