sdb-reader 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdb_reader/diff.py ADDED
@@ -0,0 +1,520 @@
1
+ """Set two runs beside each other and say where they differ (SDW-27, SDW-28).
2
+
3
+ A run is what ``read`` or ``ask`` wrote: a record per file. Two readings of
4
+ one parser show what a change to it does; a reading and a model's answers
5
+ show where to look for a reading error; two models' answers show how far a
6
+ model is to be trusted. Nothing here reads a PDF or asks anybody.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ from collections import Counter
13
+ from collections.abc import Iterable, Mapping, Sequence
14
+ from dataclasses import dataclass, field
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ from .schema import HazardStatements
19
+ from .statements import _comparable, label_statements
20
+
21
+ # What two readings are compared in: every field of a reading that can differ.
22
+ READ_FIELDS = (
23
+ "kind",
24
+ "stated_classification",
25
+ "stated_voluntary",
26
+ "stated_article",
27
+ "sheet_required",
28
+ "countries",
29
+ "signal_word",
30
+ "pictograms",
31
+ "hazard_codes",
32
+ "precautionary_codes",
33
+ "supplemental_codes",
34
+ "hazard_text",
35
+ "precautionary_text",
36
+ "supplemental_text",
37
+ "revision_date",
38
+ "issue_date",
39
+ "version",
40
+ "product_identifier",
41
+ "supplier_name",
42
+ "supplier_address",
43
+ "supplier_phone",
44
+ "supplier_email",
45
+ "supplier_website",
46
+ "information_email",
47
+ "emergency_phone",
48
+ "findings",
49
+ "embedded_differences",
50
+ "errors",
51
+ )
52
+ # What a model is asked for, and so what a comparison with one looks at.
53
+ MODEL_FIELDS = (
54
+ "hazard_codes",
55
+ "precautionary_codes",
56
+ "supplemental_codes",
57
+ "wording",
58
+ "supplier_name",
59
+ "supplier_email",
60
+ "revision_date",
61
+ )
62
+
63
+ READ = "read"
64
+ ASK = "ask"
65
+
66
+
67
+ class DiffError(Exception):
68
+ """A run cannot be read, or the two cannot be matched."""
69
+
70
+
71
+ @dataclass(frozen=True, slots=True)
72
+ class Run:
73
+ """The records of one run, by what identifies a file."""
74
+
75
+ path: Path
76
+ # "read" or "ask".
77
+ kind: str
78
+ records: Mapping[str, Mapping[str, Any]]
79
+ stages: frozenset[str]
80
+ models: frozenset[str]
81
+ questions: frozenset[str]
82
+
83
+
84
+ def load(path: Path, *, by_name: bool) -> Run:
85
+ """The run in *path*: a JSON Lines file or one JSON list.
86
+
87
+ Where a file has several records, the last holds (SDW-22). With
88
+ *by_name* files are known by their name, otherwise by their path.
89
+ """
90
+ try:
91
+ text = path.read_text(encoding="utf-8")
92
+ except (OSError, UnicodeDecodeError) as exc:
93
+ raise DiffError(f"cannot read {path}: {exc}") from exc
94
+ try:
95
+ if text.lstrip().startswith("["):
96
+ rows = json.loads(text)
97
+ else:
98
+ rows = [json.loads(line) for line in text.splitlines() if line.strip()]
99
+ except ValueError as exc:
100
+ raise DiffError(f"{path} is neither JSON Lines nor a JSON list: {exc}") from exc
101
+
102
+ records: dict[str, Mapping[str, Any]] = {}
103
+ names: Counter[str] = Counter()
104
+ kinds: set[str] = set()
105
+ paths: dict[str, str] = {}
106
+ for row in rows:
107
+ try:
108
+ location = row["file"]
109
+ key = location["filename"] if by_name else location["path"]
110
+ kinds.add(_kind_of(row))
111
+ except (KeyError, TypeError) as exc:
112
+ raise DiffError(
113
+ f"{path} holds a record that is not one of read or ask"
114
+ ) from exc
115
+ if by_name and paths.get(key, location["path"]) != location["path"]:
116
+ names[key] += 1
117
+ paths[key] = location["path"]
118
+ records[key] = row
119
+ if names:
120
+ raise DiffError(
121
+ f"{path} holds {len(names)} file name(s) under several paths; "
122
+ "they cannot be matched by name"
123
+ )
124
+ if len(kinds) > 1:
125
+ raise DiffError(f"{path} mixes records of read and of ask")
126
+ if not records:
127
+ raise DiffError(f"{path} holds no record")
128
+ asked = [row for row in records.values() if row.get("asked")]
129
+ return Run(
130
+ path,
131
+ kinds.pop(),
132
+ records,
133
+ frozenset(_stage_of(row) for row in records.values()) - {""},
134
+ frozenset(str(row["model"]) for row in asked),
135
+ frozenset(str(row["question"]) for row in asked),
136
+ )
137
+
138
+
139
+ def _kind_of(record: Mapping[str, Any]) -> str:
140
+ if "asked" in record:
141
+ return ASK
142
+ if "verification" in record:
143
+ return READ
144
+ raise KeyError("verification")
145
+
146
+
147
+ def _stage_of(record: Mapping[str, Any]) -> str:
148
+ """The reading stage a record's text came from, where the record says."""
149
+ extraction = record.get("extraction")
150
+ named = record.get("stage") or (
151
+ extraction.get("stage") if isinstance(extraction, Mapping) else None
152
+ )
153
+ return str(named or "")
154
+
155
+
156
+ @dataclass(frozen=True, slots=True)
157
+ class FieldDifference:
158
+ """One field in which the two records of a file differ."""
159
+
160
+ field: str
161
+ a: Any
162
+ b: Any
163
+ # The statement, where the field is a wording.
164
+ code: str | None = None
165
+
166
+ @property
167
+ def how(self) -> str:
168
+ if self.a is None:
169
+ return "gained"
170
+ return "lost" if self.b is None else "changed"
171
+
172
+ def to_dict(self) -> dict[str, Any]:
173
+ return {
174
+ "field": self.field,
175
+ **({"code": self.code} if self.code else {}),
176
+ "a": self.a,
177
+ "b": self.b,
178
+ }
179
+
180
+
181
+ @dataclass(slots=True)
182
+ class FieldCount:
183
+ """How many files differ in one field, shown and only counted."""
184
+
185
+ shown: Counter[str] = field(default_factory=Counter)
186
+ counted_only: Counter[str] = field(default_factory=Counter)
187
+ examples: list[tuple[str, FieldDifference]] = field(default_factory=list)
188
+
189
+
190
+ @dataclass(slots=True)
191
+ class Result:
192
+ """What setting two runs beside each other gave."""
193
+
194
+ a: Run
195
+ b: Run
196
+ fields: dict[str, FieldCount]
197
+ both: int = 0
198
+ only_a: int = 0
199
+ only_b: int = 0
200
+ # Files in both of which one record holds no reading to compare.
201
+ not_compared: int = 0
202
+ differing: int = 0
203
+ differing_counted_only: int = 0
204
+ # Every differing file that may be shown, with its differences.
205
+ details: list[tuple[str, list[FieldDifference]]] = field(default_factory=list)
206
+
207
+ @property
208
+ def differs(self) -> bool:
209
+ return bool(self.differing or self.differing_counted_only)
210
+
211
+
212
+ def compare_runs(
213
+ a: Run, b: Run, *, count_only: frozenset[str] = frozenset(), examples: int = 3
214
+ ) -> Result:
215
+ """Where the records of *a* and *b* differ, file by file (SDW-27).
216
+
217
+ A file one of *count_only*'s lines names is counted in a group of its
218
+ own and is in no example and no detail (SDW-28).
219
+ """
220
+ with_model = ASK in (a.kind, b.kind)
221
+ names = MODEL_FIELDS if with_model else READ_FIELDS
222
+ result = Result(a, b, {name: FieldCount() for name in names})
223
+ result.only_a = len(a.records.keys() - b.records.keys())
224
+ result.only_b = len(b.records.keys() - a.records.keys())
225
+ for key in sorted(a.records.keys() & b.records.keys()):
226
+ result.both += 1
227
+ one, other = a.records[key], b.records[key]
228
+ if with_model:
229
+ if not (_has_reading(one) and _has_reading(other)):
230
+ result.not_compared += 1
231
+ continue
232
+ found = model_differences(one, other)
233
+ else:
234
+ found = read_differences(one, other)
235
+ if not found:
236
+ continue
237
+ hidden = _is_named(one["file"], count_only)
238
+ if hidden:
239
+ result.differing_counted_only += 1
240
+ else:
241
+ result.differing += 1
242
+ result.details.append((str(one["file"]["path"]), found))
243
+ seen: set[str] = set()
244
+ for difference in found:
245
+ count = result.fields[difference.field]
246
+ # A file counts once per field, however many statements differ.
247
+ if difference.field not in seen:
248
+ seen.add(difference.field)
249
+ (count.counted_only if hidden else count.shown)[difference.how] += 1
250
+ if not hidden and len(count.examples) < examples:
251
+ count.examples.append((str(one["file"]["path"]), difference))
252
+ return result
253
+
254
+
255
+ def _is_named(location: Mapping[str, Any], lines: frozenset[str]) -> bool:
256
+ """Whether a line names the file: its name, its path, or the end of its path."""
257
+ if not lines:
258
+ return False
259
+ path = str(location["path"])
260
+ if location["filename"] in lines or path in lines:
261
+ return True
262
+ posix = path.replace("\\", "/")
263
+ return any(posix.endswith("/" + line.lstrip("/")) for line in lines)
264
+
265
+
266
+ def _has_reading(record: Mapping[str, Any]) -> bool:
267
+ return isinstance(record.get("sheet") or record.get("reading"), Mapping)
268
+
269
+
270
+ # ---------------------------------------------------------------------------
271
+ # Two readings
272
+ # ---------------------------------------------------------------------------
273
+
274
+
275
+ def read_fields(record: Mapping[str, Any]) -> dict[str, Any]:
276
+ """The compared fields of a ``read`` record; a field without a value is absent."""
277
+ verification = record.get("verification") or {}
278
+ out: dict[str, Any] = {"kind": verification.get("kind")}
279
+ sheet = record.get("sheet")
280
+ if isinstance(sheet, Mapping):
281
+ label = sheet.get("hazard_statements") or {}
282
+ contact = sheet.get("contact_info") or {}
283
+ for name in (
284
+ "stated_classification",
285
+ "stated_voluntary",
286
+ "stated_article",
287
+ "sheet_required",
288
+ "countries",
289
+ "revision_date",
290
+ "issue_date",
291
+ "version",
292
+ "product_identifier",
293
+ ):
294
+ out[name] = sheet.get(name)
295
+ for name in (
296
+ "signal_word",
297
+ "pictograms",
298
+ "hazard_codes",
299
+ "precautionary_codes",
300
+ "supplemental_codes",
301
+ "hazard_text",
302
+ "precautionary_text",
303
+ "supplemental_text",
304
+ ):
305
+ out[name] = label.get(name)
306
+ for name in (
307
+ "supplier_name",
308
+ "supplier_address",
309
+ "supplier_phone",
310
+ "supplier_email",
311
+ "supplier_website",
312
+ "information_email",
313
+ "emergency_phone",
314
+ ):
315
+ out[name] = contact.get(name)
316
+ out["findings"] = sorted({_code(f) for f in record.get("findings") or []})
317
+ out["errors"] = sorted({_code(e) for e in record.get("errors") or []})
318
+ embedded = record.get("embedded_data")
319
+ # Compared only for a German sheet; an empty list then says they agree.
320
+ if isinstance(embedded, Mapping) and embedded.get("compared"):
321
+ out["embedded_differences"] = sorted(
322
+ str(d.get("field")) for d in embedded.get("differences") or []
323
+ )
324
+ return {key: value for key, value in out.items() if value not in (None, [], "")}
325
+
326
+
327
+ def _code(message: Any) -> str:
328
+ return str(message.get("code")) if isinstance(message, Mapping) else str(message)
329
+
330
+
331
+ def read_differences(
332
+ a: Mapping[str, Any], b: Mapping[str, Any]
333
+ ) -> list[FieldDifference]:
334
+ """The fields in which two ``read`` records of one file differ."""
335
+ one, other = read_fields(a), read_fields(b)
336
+ return [
337
+ FieldDifference(name, one.get(name), other.get(name))
338
+ for name in READ_FIELDS
339
+ if one.get(name) != other.get(name)
340
+ ]
341
+
342
+
343
+ # ---------------------------------------------------------------------------
344
+ # A reading and a model's answer, or two answers
345
+ # ---------------------------------------------------------------------------
346
+
347
+
348
+ def _reading_of(record: Mapping[str, Any]) -> Mapping[str, Any]:
349
+ reading = record.get("sheet") or record.get("reading") or {}
350
+ return reading if isinstance(reading, Mapping) else {}
351
+
352
+
353
+ def _label(record: Mapping[str, Any]) -> Mapping[str, Any]:
354
+ label = _reading_of(record).get("hazard_statements") or {}
355
+ return label if isinstance(label, Mapping) else {}
356
+
357
+
358
+ def _wordings(record: Mapping[str, Any]) -> dict[str, dict[str, str]]:
359
+ """Per code as printed, its wordings by what identifies them (SDW-6)."""
360
+ if "asked" in record:
361
+ label = _label(record)
362
+ pairs: Iterable[tuple[str, str | None]] = (
363
+ (s.code, s.sheet)
364
+ for s in label_statements(
365
+ HazardStatements(
366
+ hazard_codes=list(label.get("hazard_codes") or []),
367
+ precautionary_codes=list(label.get("precautionary_codes") or []),
368
+ supplemental_codes=list(label.get("supplemental_codes") or []),
369
+ hazard_text=label.get("hazard_text"),
370
+ precautionary_text=label.get("precautionary_text"),
371
+ supplemental_text=label.get("supplemental_text"),
372
+ )
373
+ )
374
+ )
375
+ else:
376
+ # A reading names its statements itself, as its reader made them out.
377
+ pairs = (
378
+ (str(s.get("code")), s.get("sheet")) for s in record.get("statements") or []
379
+ )
380
+ worded: dict[str, dict[str, str]] = {}
381
+ for code, wording in pairs:
382
+ if wording and _comparable(wording):
383
+ worded.setdefault(code, {})[_comparable(wording)] = wording
384
+ return worded
385
+
386
+
387
+ def _parts(codes: Any) -> set[str]:
388
+ """Each code of a joined one on its own: "P305+P351" is P305 and P351."""
389
+ return {
390
+ part.strip()
391
+ for code in codes or []
392
+ for part in str(code).split("+")
393
+ if part.strip()
394
+ }
395
+
396
+
397
+ def _folded(value: Any) -> str:
398
+ return " ".join(str(value or "").split()).casefold()
399
+
400
+
401
+ def model_differences(
402
+ a: Mapping[str, Any], b: Mapping[str, Any]
403
+ ) -> list[FieldDifference]:
404
+ """Where two records differ in what a model is asked for (SDW-27).
405
+
406
+ The codes of the statements, a joined code as its parts; the wording per
407
+ code both name; the supplier's name and e-mail; the date of revision. A
408
+ model that names no date has said nothing about the one a reading holds:
409
+ that is a difference only between two models.
410
+ """
411
+ found: list[FieldDifference] = []
412
+ for name in ("hazard_codes", "precautionary_codes", "supplemental_codes"):
413
+ one, other = _parts(_label(a).get(name)), _parts(_label(b).get(name))
414
+ if one != other:
415
+ found.append(
416
+ FieldDifference(name, sorted(one) or None, sorted(other) or None)
417
+ )
418
+
419
+ ours, theirs = _wordings(a), _wordings(b)
420
+ for code in sorted(ours.keys() & theirs.keys()):
421
+ if not ours[code].keys() & theirs[code].keys():
422
+ found.append(
423
+ FieldDifference(
424
+ "wording",
425
+ sorted(ours[code].values()),
426
+ sorted(theirs[code].values()),
427
+ code=code,
428
+ )
429
+ )
430
+
431
+ contact_a = _reading_of(a).get("contact_info") or {}
432
+ contact_b = _reading_of(b).get("contact_info") or {}
433
+ for name in ("supplier_name", "supplier_email"):
434
+ one_text, other_text = contact_a.get(name), contact_b.get(name)
435
+ if _folded(one_text) != _folded(other_text):
436
+ found.append(FieldDifference(name, one_text or None, other_text or None))
437
+
438
+ date_a, date_b = (
439
+ _reading_of(a).get("revision_date"),
440
+ _reading_of(b).get("revision_date"),
441
+ )
442
+ silent_model = ("asked" in a and not date_a and "asked" not in b) or (
443
+ "asked" in b and not date_b and "asked" not in a
444
+ )
445
+ if date_a != date_b and not silent_model:
446
+ found.append(FieldDifference("revision_date", date_a or None, date_b or None))
447
+ return found
448
+
449
+
450
+ # ---------------------------------------------------------------------------
451
+ # What the comparison says
452
+ # ---------------------------------------------------------------------------
453
+
454
+
455
+ def to_json(result: Result) -> dict[str, Any]:
456
+ """The comparison for a program."""
457
+ return {
458
+ "a": _run_to_json(result.a),
459
+ "b": _run_to_json(result.b),
460
+ "files": {
461
+ "both": result.both,
462
+ "only_a": result.only_a,
463
+ "only_b": result.only_b,
464
+ "not_compared": result.not_compared,
465
+ "differing": result.differing,
466
+ "differing_counted_only": result.differing_counted_only,
467
+ },
468
+ "fields": {
469
+ name: {
470
+ **_hows(count.shown),
471
+ "counted_only": _hows(count.counted_only),
472
+ "examples": [
473
+ {"file": path, **difference.to_dict()}
474
+ for path, difference in count.examples
475
+ ],
476
+ }
477
+ for name, count in result.fields.items()
478
+ if count.shown or count.counted_only
479
+ },
480
+ }
481
+
482
+
483
+ def _run_to_json(run: Run) -> dict[str, Any]:
484
+ return {
485
+ "path": str(run.path),
486
+ "kind": run.kind,
487
+ "files": len(run.records),
488
+ "stages": sorted(run.stages),
489
+ "models": sorted(run.models),
490
+ }
491
+
492
+
493
+ def _hows(counts: Counter[str]) -> dict[str, int]:
494
+ return {how: counts[how] for how in ("gained", "lost", "changed")}
495
+
496
+
497
+ def detail_lines(result: Result) -> Sequence[str]:
498
+ """Every differing file that may be shown, a JSON line each."""
499
+ return [
500
+ json.dumps(
501
+ {"file": path, "differences": [d.to_dict() for d in differences]},
502
+ ensure_ascii=False,
503
+ )
504
+ for path, differences in result.details
505
+ ]
506
+
507
+
508
+ def count_only_lines(path: Path | None) -> frozenset[str]:
509
+ """The lines of a list of files that are only counted (SDW-28)."""
510
+ if path is None:
511
+ return frozenset()
512
+ try:
513
+ text = path.read_text(encoding="utf-8")
514
+ except (OSError, UnicodeDecodeError) as exc:
515
+ raise DiffError(f"cannot read {path}: {exc}") from exc
516
+ return frozenset(
517
+ stripped
518
+ for line in text.splitlines()
519
+ if (stripped := line.strip()) and not stripped.startswith("#")
520
+ )