sdb-reader 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdb_reader/__init__.py +157 -0
- sdb_reader/ask.py +437 -0
- sdb_reader/ask_record.schema.json +388 -0
- sdb_reader/cli.py +1079 -0
- sdb_reader/clp_statements_de.json +997 -0
- sdb_reader/compliance.py +546 -0
- sdb_reader/current.py +134 -0
- sdb_reader/diff.py +520 -0
- sdb_reader/embedded.py +250 -0
- sdb_reader/extraction.py +506 -0
- sdb_reader/layout.py +894 -0
- sdb_reader/line_breaks.py +40 -0
- sdb_reader/messages.py +394 -0
- sdb_reader/messages_data/de.toml +146 -0
- sdb_reader/messages_data/en.toml +141 -0
- sdb_reader/model.py +321 -0
- sdb_reader/parser.py +4601 -0
- sdb_reader/pictograms.py +374 -0
- sdb_reader/presets.py +110 -0
- sdb_reader/presets.toml +23 -0
- sdb_reader/py.typed +0 -0
- sdb_reader/read_record.schema.json +792 -0
- sdb_reader/reading.py +94 -0
- sdb_reader/scan.py +265 -0
- sdb_reader/schema.py +96 -0
- sdb_reader/serialise.py +463 -0
- sdb_reader/show.py +370 -0
- sdb_reader/statements.py +1011 -0
- sdb_reader/verification.py +322 -0
- sdb_reader/versions.py +31 -0
- sdb_reader-0.9.0.dist-info/METADATA +117 -0
- sdb_reader-0.9.0.dist-info/RECORD +37 -0
- sdb_reader-0.9.0.dist-info/WHEEL +5 -0
- sdb_reader-0.9.0.dist-info/entry_points.txt +2 -0
- sdb_reader-0.9.0.dist-info/licenses/LICENSE +661 -0
- sdb_reader-0.9.0.dist-info/licenses/NOTICE +40 -0
- sdb_reader-0.9.0.dist-info/top_level.txt +1 -0
sdb_reader/diff.py
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
"""Set two runs beside each other and say where they differ (SDW-27, SDW-28).
|
|
2
|
+
|
|
3
|
+
A run is what ``read`` or ``ask`` wrote: a record per file. Two readings of
|
|
4
|
+
one parser show what a change to it does; a reading and a model's answers
|
|
5
|
+
show where to look for a reading error; two models' answers show how far a
|
|
6
|
+
model is to be trusted. Nothing here reads a PDF or asks anybody.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from .schema import HazardStatements
|
|
19
|
+
from .statements import _comparable, label_statements
|
|
20
|
+
|
|
21
|
+
# What two readings are compared in: every field of a reading that can differ.
|
|
22
|
+
READ_FIELDS = (
|
|
23
|
+
"kind",
|
|
24
|
+
"stated_classification",
|
|
25
|
+
"stated_voluntary",
|
|
26
|
+
"stated_article",
|
|
27
|
+
"sheet_required",
|
|
28
|
+
"countries",
|
|
29
|
+
"signal_word",
|
|
30
|
+
"pictograms",
|
|
31
|
+
"hazard_codes",
|
|
32
|
+
"precautionary_codes",
|
|
33
|
+
"supplemental_codes",
|
|
34
|
+
"hazard_text",
|
|
35
|
+
"precautionary_text",
|
|
36
|
+
"supplemental_text",
|
|
37
|
+
"revision_date",
|
|
38
|
+
"issue_date",
|
|
39
|
+
"version",
|
|
40
|
+
"product_identifier",
|
|
41
|
+
"supplier_name",
|
|
42
|
+
"supplier_address",
|
|
43
|
+
"supplier_phone",
|
|
44
|
+
"supplier_email",
|
|
45
|
+
"supplier_website",
|
|
46
|
+
"information_email",
|
|
47
|
+
"emergency_phone",
|
|
48
|
+
"findings",
|
|
49
|
+
"embedded_differences",
|
|
50
|
+
"errors",
|
|
51
|
+
)
|
|
52
|
+
# What a model is asked for, and so what a comparison with one looks at.
|
|
53
|
+
MODEL_FIELDS = (
|
|
54
|
+
"hazard_codes",
|
|
55
|
+
"precautionary_codes",
|
|
56
|
+
"supplemental_codes",
|
|
57
|
+
"wording",
|
|
58
|
+
"supplier_name",
|
|
59
|
+
"supplier_email",
|
|
60
|
+
"revision_date",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
READ = "read"
|
|
64
|
+
ASK = "ask"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class DiffError(Exception):
|
|
68
|
+
"""A run cannot be read, or the two cannot be matched."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True, slots=True)
|
|
72
|
+
class Run:
|
|
73
|
+
"""The records of one run, by what identifies a file."""
|
|
74
|
+
|
|
75
|
+
path: Path
|
|
76
|
+
# "read" or "ask".
|
|
77
|
+
kind: str
|
|
78
|
+
records: Mapping[str, Mapping[str, Any]]
|
|
79
|
+
stages: frozenset[str]
|
|
80
|
+
models: frozenset[str]
|
|
81
|
+
questions: frozenset[str]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def load(path: Path, *, by_name: bool) -> Run:
|
|
85
|
+
"""The run in *path*: a JSON Lines file or one JSON list.
|
|
86
|
+
|
|
87
|
+
Where a file has several records, the last holds (SDW-22). With
|
|
88
|
+
*by_name* files are known by their name, otherwise by their path.
|
|
89
|
+
"""
|
|
90
|
+
try:
|
|
91
|
+
text = path.read_text(encoding="utf-8")
|
|
92
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
93
|
+
raise DiffError(f"cannot read {path}: {exc}") from exc
|
|
94
|
+
try:
|
|
95
|
+
if text.lstrip().startswith("["):
|
|
96
|
+
rows = json.loads(text)
|
|
97
|
+
else:
|
|
98
|
+
rows = [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
99
|
+
except ValueError as exc:
|
|
100
|
+
raise DiffError(f"{path} is neither JSON Lines nor a JSON list: {exc}") from exc
|
|
101
|
+
|
|
102
|
+
records: dict[str, Mapping[str, Any]] = {}
|
|
103
|
+
names: Counter[str] = Counter()
|
|
104
|
+
kinds: set[str] = set()
|
|
105
|
+
paths: dict[str, str] = {}
|
|
106
|
+
for row in rows:
|
|
107
|
+
try:
|
|
108
|
+
location = row["file"]
|
|
109
|
+
key = location["filename"] if by_name else location["path"]
|
|
110
|
+
kinds.add(_kind_of(row))
|
|
111
|
+
except (KeyError, TypeError) as exc:
|
|
112
|
+
raise DiffError(
|
|
113
|
+
f"{path} holds a record that is not one of read or ask"
|
|
114
|
+
) from exc
|
|
115
|
+
if by_name and paths.get(key, location["path"]) != location["path"]:
|
|
116
|
+
names[key] += 1
|
|
117
|
+
paths[key] = location["path"]
|
|
118
|
+
records[key] = row
|
|
119
|
+
if names:
|
|
120
|
+
raise DiffError(
|
|
121
|
+
f"{path} holds {len(names)} file name(s) under several paths; "
|
|
122
|
+
"they cannot be matched by name"
|
|
123
|
+
)
|
|
124
|
+
if len(kinds) > 1:
|
|
125
|
+
raise DiffError(f"{path} mixes records of read and of ask")
|
|
126
|
+
if not records:
|
|
127
|
+
raise DiffError(f"{path} holds no record")
|
|
128
|
+
asked = [row for row in records.values() if row.get("asked")]
|
|
129
|
+
return Run(
|
|
130
|
+
path,
|
|
131
|
+
kinds.pop(),
|
|
132
|
+
records,
|
|
133
|
+
frozenset(_stage_of(row) for row in records.values()) - {""},
|
|
134
|
+
frozenset(str(row["model"]) for row in asked),
|
|
135
|
+
frozenset(str(row["question"]) for row in asked),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _kind_of(record: Mapping[str, Any]) -> str:
|
|
140
|
+
if "asked" in record:
|
|
141
|
+
return ASK
|
|
142
|
+
if "verification" in record:
|
|
143
|
+
return READ
|
|
144
|
+
raise KeyError("verification")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _stage_of(record: Mapping[str, Any]) -> str:
|
|
148
|
+
"""The reading stage a record's text came from, where the record says."""
|
|
149
|
+
extraction = record.get("extraction")
|
|
150
|
+
named = record.get("stage") or (
|
|
151
|
+
extraction.get("stage") if isinstance(extraction, Mapping) else None
|
|
152
|
+
)
|
|
153
|
+
return str(named or "")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@dataclass(frozen=True, slots=True)
|
|
157
|
+
class FieldDifference:
|
|
158
|
+
"""One field in which the two records of a file differ."""
|
|
159
|
+
|
|
160
|
+
field: str
|
|
161
|
+
a: Any
|
|
162
|
+
b: Any
|
|
163
|
+
# The statement, where the field is a wording.
|
|
164
|
+
code: str | None = None
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def how(self) -> str:
|
|
168
|
+
if self.a is None:
|
|
169
|
+
return "gained"
|
|
170
|
+
return "lost" if self.b is None else "changed"
|
|
171
|
+
|
|
172
|
+
def to_dict(self) -> dict[str, Any]:
|
|
173
|
+
return {
|
|
174
|
+
"field": self.field,
|
|
175
|
+
**({"code": self.code} if self.code else {}),
|
|
176
|
+
"a": self.a,
|
|
177
|
+
"b": self.b,
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@dataclass(slots=True)
|
|
182
|
+
class FieldCount:
|
|
183
|
+
"""How many files differ in one field, shown and only counted."""
|
|
184
|
+
|
|
185
|
+
shown: Counter[str] = field(default_factory=Counter)
|
|
186
|
+
counted_only: Counter[str] = field(default_factory=Counter)
|
|
187
|
+
examples: list[tuple[str, FieldDifference]] = field(default_factory=list)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@dataclass(slots=True)
|
|
191
|
+
class Result:
|
|
192
|
+
"""What setting two runs beside each other gave."""
|
|
193
|
+
|
|
194
|
+
a: Run
|
|
195
|
+
b: Run
|
|
196
|
+
fields: dict[str, FieldCount]
|
|
197
|
+
both: int = 0
|
|
198
|
+
only_a: int = 0
|
|
199
|
+
only_b: int = 0
|
|
200
|
+
# Files in both of which one record holds no reading to compare.
|
|
201
|
+
not_compared: int = 0
|
|
202
|
+
differing: int = 0
|
|
203
|
+
differing_counted_only: int = 0
|
|
204
|
+
# Every differing file that may be shown, with its differences.
|
|
205
|
+
details: list[tuple[str, list[FieldDifference]]] = field(default_factory=list)
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def differs(self) -> bool:
|
|
209
|
+
return bool(self.differing or self.differing_counted_only)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def compare_runs(
|
|
213
|
+
a: Run, b: Run, *, count_only: frozenset[str] = frozenset(), examples: int = 3
|
|
214
|
+
) -> Result:
|
|
215
|
+
"""Where the records of *a* and *b* differ, file by file (SDW-27).
|
|
216
|
+
|
|
217
|
+
A file one of *count_only*'s lines names is counted in a group of its
|
|
218
|
+
own and is in no example and no detail (SDW-28).
|
|
219
|
+
"""
|
|
220
|
+
with_model = ASK in (a.kind, b.kind)
|
|
221
|
+
names = MODEL_FIELDS if with_model else READ_FIELDS
|
|
222
|
+
result = Result(a, b, {name: FieldCount() for name in names})
|
|
223
|
+
result.only_a = len(a.records.keys() - b.records.keys())
|
|
224
|
+
result.only_b = len(b.records.keys() - a.records.keys())
|
|
225
|
+
for key in sorted(a.records.keys() & b.records.keys()):
|
|
226
|
+
result.both += 1
|
|
227
|
+
one, other = a.records[key], b.records[key]
|
|
228
|
+
if with_model:
|
|
229
|
+
if not (_has_reading(one) and _has_reading(other)):
|
|
230
|
+
result.not_compared += 1
|
|
231
|
+
continue
|
|
232
|
+
found = model_differences(one, other)
|
|
233
|
+
else:
|
|
234
|
+
found = read_differences(one, other)
|
|
235
|
+
if not found:
|
|
236
|
+
continue
|
|
237
|
+
hidden = _is_named(one["file"], count_only)
|
|
238
|
+
if hidden:
|
|
239
|
+
result.differing_counted_only += 1
|
|
240
|
+
else:
|
|
241
|
+
result.differing += 1
|
|
242
|
+
result.details.append((str(one["file"]["path"]), found))
|
|
243
|
+
seen: set[str] = set()
|
|
244
|
+
for difference in found:
|
|
245
|
+
count = result.fields[difference.field]
|
|
246
|
+
# A file counts once per field, however many statements differ.
|
|
247
|
+
if difference.field not in seen:
|
|
248
|
+
seen.add(difference.field)
|
|
249
|
+
(count.counted_only if hidden else count.shown)[difference.how] += 1
|
|
250
|
+
if not hidden and len(count.examples) < examples:
|
|
251
|
+
count.examples.append((str(one["file"]["path"]), difference))
|
|
252
|
+
return result
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _is_named(location: Mapping[str, Any], lines: frozenset[str]) -> bool:
|
|
256
|
+
"""Whether a line names the file: its name, its path, or the end of its path."""
|
|
257
|
+
if not lines:
|
|
258
|
+
return False
|
|
259
|
+
path = str(location["path"])
|
|
260
|
+
if location["filename"] in lines or path in lines:
|
|
261
|
+
return True
|
|
262
|
+
posix = path.replace("\\", "/")
|
|
263
|
+
return any(posix.endswith("/" + line.lstrip("/")) for line in lines)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _has_reading(record: Mapping[str, Any]) -> bool:
|
|
267
|
+
return isinstance(record.get("sheet") or record.get("reading"), Mapping)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
# ---------------------------------------------------------------------------
|
|
271
|
+
# Two readings
|
|
272
|
+
# ---------------------------------------------------------------------------
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def read_fields(record: Mapping[str, Any]) -> dict[str, Any]:
|
|
276
|
+
"""The compared fields of a ``read`` record; a field without a value is absent."""
|
|
277
|
+
verification = record.get("verification") or {}
|
|
278
|
+
out: dict[str, Any] = {"kind": verification.get("kind")}
|
|
279
|
+
sheet = record.get("sheet")
|
|
280
|
+
if isinstance(sheet, Mapping):
|
|
281
|
+
label = sheet.get("hazard_statements") or {}
|
|
282
|
+
contact = sheet.get("contact_info") or {}
|
|
283
|
+
for name in (
|
|
284
|
+
"stated_classification",
|
|
285
|
+
"stated_voluntary",
|
|
286
|
+
"stated_article",
|
|
287
|
+
"sheet_required",
|
|
288
|
+
"countries",
|
|
289
|
+
"revision_date",
|
|
290
|
+
"issue_date",
|
|
291
|
+
"version",
|
|
292
|
+
"product_identifier",
|
|
293
|
+
):
|
|
294
|
+
out[name] = sheet.get(name)
|
|
295
|
+
for name in (
|
|
296
|
+
"signal_word",
|
|
297
|
+
"pictograms",
|
|
298
|
+
"hazard_codes",
|
|
299
|
+
"precautionary_codes",
|
|
300
|
+
"supplemental_codes",
|
|
301
|
+
"hazard_text",
|
|
302
|
+
"precautionary_text",
|
|
303
|
+
"supplemental_text",
|
|
304
|
+
):
|
|
305
|
+
out[name] = label.get(name)
|
|
306
|
+
for name in (
|
|
307
|
+
"supplier_name",
|
|
308
|
+
"supplier_address",
|
|
309
|
+
"supplier_phone",
|
|
310
|
+
"supplier_email",
|
|
311
|
+
"supplier_website",
|
|
312
|
+
"information_email",
|
|
313
|
+
"emergency_phone",
|
|
314
|
+
):
|
|
315
|
+
out[name] = contact.get(name)
|
|
316
|
+
out["findings"] = sorted({_code(f) for f in record.get("findings") or []})
|
|
317
|
+
out["errors"] = sorted({_code(e) for e in record.get("errors") or []})
|
|
318
|
+
embedded = record.get("embedded_data")
|
|
319
|
+
# Compared only for a German sheet; an empty list then says they agree.
|
|
320
|
+
if isinstance(embedded, Mapping) and embedded.get("compared"):
|
|
321
|
+
out["embedded_differences"] = sorted(
|
|
322
|
+
str(d.get("field")) for d in embedded.get("differences") or []
|
|
323
|
+
)
|
|
324
|
+
return {key: value for key, value in out.items() if value not in (None, [], "")}
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _code(message: Any) -> str:
|
|
328
|
+
return str(message.get("code")) if isinstance(message, Mapping) else str(message)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def read_differences(
|
|
332
|
+
a: Mapping[str, Any], b: Mapping[str, Any]
|
|
333
|
+
) -> list[FieldDifference]:
|
|
334
|
+
"""The fields in which two ``read`` records of one file differ."""
|
|
335
|
+
one, other = read_fields(a), read_fields(b)
|
|
336
|
+
return [
|
|
337
|
+
FieldDifference(name, one.get(name), other.get(name))
|
|
338
|
+
for name in READ_FIELDS
|
|
339
|
+
if one.get(name) != other.get(name)
|
|
340
|
+
]
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
# ---------------------------------------------------------------------------
|
|
344
|
+
# A reading and a model's answer, or two answers
|
|
345
|
+
# ---------------------------------------------------------------------------
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _reading_of(record: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
349
|
+
reading = record.get("sheet") or record.get("reading") or {}
|
|
350
|
+
return reading if isinstance(reading, Mapping) else {}
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _label(record: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
354
|
+
label = _reading_of(record).get("hazard_statements") or {}
|
|
355
|
+
return label if isinstance(label, Mapping) else {}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _wordings(record: Mapping[str, Any]) -> dict[str, dict[str, str]]:
|
|
359
|
+
"""Per code as printed, its wordings by what identifies them (SDW-6)."""
|
|
360
|
+
if "asked" in record:
|
|
361
|
+
label = _label(record)
|
|
362
|
+
pairs: Iterable[tuple[str, str | None]] = (
|
|
363
|
+
(s.code, s.sheet)
|
|
364
|
+
for s in label_statements(
|
|
365
|
+
HazardStatements(
|
|
366
|
+
hazard_codes=list(label.get("hazard_codes") or []),
|
|
367
|
+
precautionary_codes=list(label.get("precautionary_codes") or []),
|
|
368
|
+
supplemental_codes=list(label.get("supplemental_codes") or []),
|
|
369
|
+
hazard_text=label.get("hazard_text"),
|
|
370
|
+
precautionary_text=label.get("precautionary_text"),
|
|
371
|
+
supplemental_text=label.get("supplemental_text"),
|
|
372
|
+
)
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
else:
|
|
376
|
+
# A reading names its statements itself, as its reader made them out.
|
|
377
|
+
pairs = (
|
|
378
|
+
(str(s.get("code")), s.get("sheet")) for s in record.get("statements") or []
|
|
379
|
+
)
|
|
380
|
+
worded: dict[str, dict[str, str]] = {}
|
|
381
|
+
for code, wording in pairs:
|
|
382
|
+
if wording and _comparable(wording):
|
|
383
|
+
worded.setdefault(code, {})[_comparable(wording)] = wording
|
|
384
|
+
return worded
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _parts(codes: Any) -> set[str]:
|
|
388
|
+
"""Each code of a joined one on its own: "P305+P351" is P305 and P351."""
|
|
389
|
+
return {
|
|
390
|
+
part.strip()
|
|
391
|
+
for code in codes or []
|
|
392
|
+
for part in str(code).split("+")
|
|
393
|
+
if part.strip()
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _folded(value: Any) -> str:
|
|
398
|
+
return " ".join(str(value or "").split()).casefold()
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def model_differences(
|
|
402
|
+
a: Mapping[str, Any], b: Mapping[str, Any]
|
|
403
|
+
) -> list[FieldDifference]:
|
|
404
|
+
"""Where two records differ in what a model is asked for (SDW-27).
|
|
405
|
+
|
|
406
|
+
The codes of the statements, a joined code as its parts; the wording per
|
|
407
|
+
code both name; the supplier's name and e-mail; the date of revision. A
|
|
408
|
+
model that names no date has said nothing about the one a reading holds:
|
|
409
|
+
that is a difference only between two models.
|
|
410
|
+
"""
|
|
411
|
+
found: list[FieldDifference] = []
|
|
412
|
+
for name in ("hazard_codes", "precautionary_codes", "supplemental_codes"):
|
|
413
|
+
one, other = _parts(_label(a).get(name)), _parts(_label(b).get(name))
|
|
414
|
+
if one != other:
|
|
415
|
+
found.append(
|
|
416
|
+
FieldDifference(name, sorted(one) or None, sorted(other) or None)
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
ours, theirs = _wordings(a), _wordings(b)
|
|
420
|
+
for code in sorted(ours.keys() & theirs.keys()):
|
|
421
|
+
if not ours[code].keys() & theirs[code].keys():
|
|
422
|
+
found.append(
|
|
423
|
+
FieldDifference(
|
|
424
|
+
"wording",
|
|
425
|
+
sorted(ours[code].values()),
|
|
426
|
+
sorted(theirs[code].values()),
|
|
427
|
+
code=code,
|
|
428
|
+
)
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
contact_a = _reading_of(a).get("contact_info") or {}
|
|
432
|
+
contact_b = _reading_of(b).get("contact_info") or {}
|
|
433
|
+
for name in ("supplier_name", "supplier_email"):
|
|
434
|
+
one_text, other_text = contact_a.get(name), contact_b.get(name)
|
|
435
|
+
if _folded(one_text) != _folded(other_text):
|
|
436
|
+
found.append(FieldDifference(name, one_text or None, other_text or None))
|
|
437
|
+
|
|
438
|
+
date_a, date_b = (
|
|
439
|
+
_reading_of(a).get("revision_date"),
|
|
440
|
+
_reading_of(b).get("revision_date"),
|
|
441
|
+
)
|
|
442
|
+
silent_model = ("asked" in a and not date_a and "asked" not in b) or (
|
|
443
|
+
"asked" in b and not date_b and "asked" not in a
|
|
444
|
+
)
|
|
445
|
+
if date_a != date_b and not silent_model:
|
|
446
|
+
found.append(FieldDifference("revision_date", date_a or None, date_b or None))
|
|
447
|
+
return found
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
# ---------------------------------------------------------------------------
|
|
451
|
+
# What the comparison says
|
|
452
|
+
# ---------------------------------------------------------------------------
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def to_json(result: Result) -> dict[str, Any]:
|
|
456
|
+
"""The comparison for a program."""
|
|
457
|
+
return {
|
|
458
|
+
"a": _run_to_json(result.a),
|
|
459
|
+
"b": _run_to_json(result.b),
|
|
460
|
+
"files": {
|
|
461
|
+
"both": result.both,
|
|
462
|
+
"only_a": result.only_a,
|
|
463
|
+
"only_b": result.only_b,
|
|
464
|
+
"not_compared": result.not_compared,
|
|
465
|
+
"differing": result.differing,
|
|
466
|
+
"differing_counted_only": result.differing_counted_only,
|
|
467
|
+
},
|
|
468
|
+
"fields": {
|
|
469
|
+
name: {
|
|
470
|
+
**_hows(count.shown),
|
|
471
|
+
"counted_only": _hows(count.counted_only),
|
|
472
|
+
"examples": [
|
|
473
|
+
{"file": path, **difference.to_dict()}
|
|
474
|
+
for path, difference in count.examples
|
|
475
|
+
],
|
|
476
|
+
}
|
|
477
|
+
for name, count in result.fields.items()
|
|
478
|
+
if count.shown or count.counted_only
|
|
479
|
+
},
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _run_to_json(run: Run) -> dict[str, Any]:
|
|
484
|
+
return {
|
|
485
|
+
"path": str(run.path),
|
|
486
|
+
"kind": run.kind,
|
|
487
|
+
"files": len(run.records),
|
|
488
|
+
"stages": sorted(run.stages),
|
|
489
|
+
"models": sorted(run.models),
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _hows(counts: Counter[str]) -> dict[str, int]:
|
|
494
|
+
return {how: counts[how] for how in ("gained", "lost", "changed")}
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def detail_lines(result: Result) -> Sequence[str]:
|
|
498
|
+
"""Every differing file that may be shown, a JSON line each."""
|
|
499
|
+
return [
|
|
500
|
+
json.dumps(
|
|
501
|
+
{"file": path, "differences": [d.to_dict() for d in differences]},
|
|
502
|
+
ensure_ascii=False,
|
|
503
|
+
)
|
|
504
|
+
for path, differences in result.details
|
|
505
|
+
]
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def count_only_lines(path: Path | None) -> frozenset[str]:
|
|
509
|
+
"""The lines of a list of files that are only counted (SDW-28)."""
|
|
510
|
+
if path is None:
|
|
511
|
+
return frozenset()
|
|
512
|
+
try:
|
|
513
|
+
text = path.read_text(encoding="utf-8")
|
|
514
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
515
|
+
raise DiffError(f"cannot read {path}: {exc}") from exc
|
|
516
|
+
return frozenset(
|
|
517
|
+
stripped
|
|
518
|
+
for line in text.splitlines()
|
|
519
|
+
if (stripped := line.strip()) and not stripped.startswith("#")
|
|
520
|
+
)
|