consilience 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- consilience/__init__.py +29 -0
- consilience/__main__.py +5 -0
- consilience/agent.py +140 -0
- consilience/api.py +32 -0
- consilience/check.py +400 -0
- consilience/cli.py +360 -0
- consilience/compare.py +595 -0
- consilience/data/examples/measurement_a.json +13 -0
- consilience/data/examples/measurement_b.json +13 -0
- consilience/data/examples/measurement_c.json +12 -0
- consilience/data/examples/measurement_d.json +11 -0
- consilience/data/examples/measurement_e.json +12 -0
- consilience/data/examples/readings.csv +23 -0
- consilience/data/examples/readings_rules.yaml +45 -0
- consilience/data/packs/electrochemistry.yaml +86 -0
- consilience/export/__init__.py +1 -0
- consilience/export/nanopub.py +589 -0
- consilience/export/spec.py +527 -0
- consilience/export/vocab.py +126 -0
- consilience/hashing.py +20 -0
- consilience/nanopub_cli.py +162 -0
- consilience/receipts.py +226 -0
- consilience/rules.py +408 -0
- consilience/units.py +262 -0
- consilience/verify.py +186 -0
- consilience-0.1.0.dist-info/METADATA +470 -0
- consilience-0.1.0.dist-info/RECORD +32 -0
- consilience-0.1.0.dist-info/WHEEL +5 -0
- consilience-0.1.0.dist-info/entry_points.txt +2 -0
- consilience-0.1.0.dist-info/licenses/LICENSE +21 -0
- consilience-0.1.0.dist-info/licenses/LICENSE-DATA +17 -0
- consilience-0.1.0.dist-info/top_level.txt +1 -0
consilience/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""consilience: compare two measurements with units and conditions, and check a table
|
|
2
|
+
against written limits. Every result carries a receipt that ``verify`` can re-run.
|
|
3
|
+
|
|
4
|
+
from consilience import compare, check, verify, load_rules
|
|
5
|
+
|
|
6
|
+
The nanopublication exporter (``consilience.export``) is a separate, optional part:
|
|
7
|
+
``pip install "consilience[nanopub]"``.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
|
|
12
|
+
from .api import ( # noqa: E402
|
|
13
|
+
BUNDLED_PACKS,
|
|
14
|
+
CheckResult,
|
|
15
|
+
CompareResult,
|
|
16
|
+
InputError,
|
|
17
|
+
RuleFileError,
|
|
18
|
+
RuleSet,
|
|
19
|
+
UnitError,
|
|
20
|
+
VerifyResult,
|
|
21
|
+
check,
|
|
22
|
+
compare,
|
|
23
|
+
load_rules,
|
|
24
|
+
verify,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
__all__ = ["__version__", "compare", "check", "verify", "load_rules", "CompareResult",
|
|
28
|
+
"CheckResult", "VerifyResult", "RuleSet", "InputError", "RuleFileError",
|
|
29
|
+
"UnitError", "BUNDLED_PACKS"]
|
consilience/__main__.py
ADDED
consilience/agent.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""The Consilience engine as a declared software agent on the nanopublication network.
|
|
2
|
+
|
|
3
|
+
Keys are generated locally into a directory *outside* the repository (default
|
|
4
|
+
``~/.consilience-nanopub/``, override with ``CONSILIENCE_NANOPUB_KEY_DIR``) and are never
|
|
5
|
+
committed. The public agent configuration (name, agent IRI, introduction nanopub,
|
|
6
|
+
public key, responsible person) lives in ``config/agent.json`` in the repository.
|
|
7
|
+
|
|
8
|
+
Only the Knowledge Pixels *test* registry is supported. Publishing to the live network
|
|
9
|
+
is deliberately not implemented: nanopublications cannot be deleted, and production
|
|
10
|
+
publishing waits for the project owner's approval and ORCID iD.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import stat
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from Crypto.PublicKey import RSA
|
|
22
|
+
|
|
23
|
+
TEST_REGISTRY_URL = "https://test.registry.knowledgepixels.com/np/"
|
|
24
|
+
KEY_BASENAME = "consilience-engine_id_rsa"
|
|
25
|
+
|
|
26
|
+
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
27
|
+
DEFAULT_CONFIG = REPO_ROOT / "config" / "agent.json"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class AgentError(RuntimeError):
|
|
31
|
+
"""The agent configuration or keys are not usable. The message names the reason."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def key_dir() -> Path:
|
|
35
|
+
env = os.environ.get("CONSILIENCE_NANOPUB_KEY_DIR")
|
|
36
|
+
return Path(env).expanduser() if env else Path.home() / ".consilience-nanopub"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _inside_repo(p: Path) -> bool:
|
|
40
|
+
"""True if ``p`` is inside this repository or inside any git working tree.
|
|
41
|
+
|
|
42
|
+
The git check matters when the package is installed without ``-e``: REPO_ROOT then
|
|
43
|
+
points into site-packages, and a key directory inside a clone would not be noticed.
|
|
44
|
+
"""
|
|
45
|
+
rp = p.resolve()
|
|
46
|
+
if any((d / ".git").exists() for d in (rp, *rp.parents)):
|
|
47
|
+
return True
|
|
48
|
+
try:
|
|
49
|
+
rp.relative_to(REPO_ROOT)
|
|
50
|
+
return True
|
|
51
|
+
except ValueError:
|
|
52
|
+
return False
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _pem_body(pem: str) -> str:
|
|
56
|
+
return "".join(ln.strip() for ln in pem.splitlines() if ln and not ln.startswith("-----"))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class AgentConfig:
|
|
61
|
+
path: Path
|
|
62
|
+
data: dict
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def load(cls, path: str | Path = DEFAULT_CONFIG) -> "AgentConfig":
|
|
66
|
+
p = Path(path)
|
|
67
|
+
try:
|
|
68
|
+
return cls(p, json.loads(p.read_text(encoding="utf-8")))
|
|
69
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
70
|
+
raise AgentError(f"unreadable_agent_config: {p}: {exc}") from None
|
|
71
|
+
|
|
72
|
+
def save(self) -> None:
|
|
73
|
+
self.path.write_text(json.dumps(self.data, indent=2, sort_keys=False) + "\n", encoding="utf-8")
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def name(self) -> str:
|
|
77
|
+
return self.data["agent_name"]
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def local_name(self) -> str:
|
|
81
|
+
return self.data["agent_local_name"]
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def agent_iri(self) -> str | None:
|
|
85
|
+
return self.data.get("agent_iri") or None
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def responsible_name(self) -> str:
|
|
89
|
+
return self.data["responsible_person_name"]
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def responsible_orcid(self) -> str | None:
|
|
93
|
+
v = (self.data.get("responsible_person_orcid") or "").strip()
|
|
94
|
+
return v or None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def generate_keys(directory: Path | None = None) -> Path:
|
|
98
|
+
"""Create a 2048-bit RSA key pair outside the repository. Refuses to overwrite."""
|
|
99
|
+
d = directory or key_dir()
|
|
100
|
+
if _inside_repo(d):
|
|
101
|
+
raise AgentError(f"key_dir_inside_repository: {d}")
|
|
102
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
103
|
+
os.chmod(d, stat.S_IRWXU)
|
|
104
|
+
priv = d / KEY_BASENAME
|
|
105
|
+
pub = d / (KEY_BASENAME + ".pub")
|
|
106
|
+
if priv.exists() or pub.exists():
|
|
107
|
+
raise AgentError(f"keys_already_exist: {priv} (never overwritten; losing a key loses "
|
|
108
|
+
"the ability to supersede or retract what it signed)")
|
|
109
|
+
key = RSA.generate(2048)
|
|
110
|
+
priv.write_bytes(key.export_key("PEM", pkcs=8))
|
|
111
|
+
os.chmod(priv, stat.S_IRUSR | stat.S_IWUSR)
|
|
112
|
+
pub.write_bytes(key.publickey().export_key("PEM"))
|
|
113
|
+
return priv
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def load_keys(directory: Path | None = None) -> tuple[str, str]:
|
|
117
|
+
"""Return (private_key_base64_body, public_key_base64_body)."""
|
|
118
|
+
d = directory or key_dir()
|
|
119
|
+
if _inside_repo(d):
|
|
120
|
+
raise AgentError(f"key_dir_inside_repository: {d}")
|
|
121
|
+
priv = d / KEY_BASENAME
|
|
122
|
+
pub = d / (KEY_BASENAME + ".pub")
|
|
123
|
+
if not priv.exists() or not pub.exists():
|
|
124
|
+
raise AgentError(f"keys_not_found: {priv} (run `consilience agent keygen`)")
|
|
125
|
+
return _pem_body(priv.read_text()), _pem_body(pub.read_text())
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def profile_for(agent_iri: str, name: str, directory: Path | None = None):
|
|
129
|
+
"""A nanopub-py Profile that signs as ``agent_iri`` with the local key."""
|
|
130
|
+
from nanopub import Profile
|
|
131
|
+
|
|
132
|
+
private_key, public_key = load_keys(directory)
|
|
133
|
+
try: # nanopub >= 2.3 names the signer ``agent_id``
|
|
134
|
+
return Profile(name=name, agent_id=agent_iri, private_key=private_key, public_key=public_key)
|
|
135
|
+
except TypeError: # nanopub 2.2.x calls it ``orcid_id`` but accepts any IRI
|
|
136
|
+
return Profile(orcid_id=agent_iri, name=name, private_key=private_key, public_key=public_key)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def signer_of(profile) -> str:
|
|
140
|
+
return getattr(profile, "agent_id", None) or profile.orcid_id
|
consilience/api.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""The Python API. It mirrors the command line:
|
|
2
|
+
|
|
3
|
+
from consilience import compare, check, verify, load_rules
|
|
4
|
+
|
|
5
|
+
r = compare("measurement_a.json", "measurement_b.json", pack="electrochemistry")
|
|
6
|
+
r.verdict, r.label, r.exit_code, r.receipt # receipt: a dict with "sha256"
|
|
7
|
+
|
|
8
|
+
c = check("readings.csv", "readings_rules.yaml")
|
|
9
|
+
c.summary, c.violations, c.unparseable, c.to_html()
|
|
10
|
+
|
|
11
|
+
v = verify(r.receipt) # or a receipt file path
|
|
12
|
+
v.ok, v.steps
|
|
13
|
+
|
|
14
|
+
rules = load_rules("electrochemistry") # or a path to a YAML file
|
|
15
|
+
rules.name, rules.version, rules.sha256
|
|
16
|
+
|
|
17
|
+
Errors in the inputs themselves raise ``InputError`` (a missing file, an unknown field,
|
|
18
|
+
an unusable option) or ``RuleFileError`` (an unusable rule file). Everything that can
|
|
19
|
+
be read but not compared is a verdict, not an exception.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from .check import CheckResult, check
|
|
25
|
+
from .compare import CompareResult, InputError, compare
|
|
26
|
+
from .rules import BUNDLED_PACKS, RuleFileError, RuleSet, load_rules
|
|
27
|
+
from .units import UnitError
|
|
28
|
+
from .verify import VerifyResult, verify
|
|
29
|
+
|
|
30
|
+
__all__ = ["compare", "check", "verify", "load_rules", "CompareResult", "CheckResult",
|
|
31
|
+
"VerifyResult", "RuleSet", "InputError", "RuleFileError", "UnitError",
|
|
32
|
+
"BUNDLED_PACKS"]
|
consilience/check.py
ADDED
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
"""Check a table (CSV) against a rule file of limits and physical bounds.
|
|
2
|
+
|
|
3
|
+
Two kinds of rule (full format in docs/RULES.md):
|
|
4
|
+
|
|
5
|
+
bound a column must lie within [min, max] (physical bounds, e.g. not below 0 K)
|
|
6
|
+
limit a column must be at most (or at least) the value in another column of the
|
|
7
|
+
same row, within a margin (e.g. a reading must not exceed its own recorded
|
|
8
|
+
instrument limit)
|
|
9
|
+
|
|
10
|
+
Every number cell a rule needs is read strictly (see ``consilience.units``). A cell that
|
|
11
|
+
cannot be read is reported with its line, column, text and reason, and counted; it is
|
|
12
|
+
never skipped silently and never guessed. An empty cell is refused too, unless the rule
|
|
13
|
+
says ``if_empty: skip`` (then it is counted as skipped). A row with the wrong number of
|
|
14
|
+
cells is refused whole. Lines starting with ``#`` before the header are comments.
|
|
15
|
+
|
|
16
|
+
Exit codes: 0 no violations and every needed cell read; 1 violations found; 3 some cells
|
|
17
|
+
could not be read (the check is incomplete; any violations are still listed);
|
|
18
|
+
64 unusable input (missing file or column, bad rule file).
|
|
19
|
+
|
|
20
|
+
Each violation carries a row receipt::
|
|
21
|
+
|
|
22
|
+
row_sha256 = sha256(canonical_json({"rules_sha256": ..., "rule": <id>,
|
|
23
|
+
"line": <line number>, "cells": {header: text}}))
|
|
24
|
+
|
|
25
|
+
and the whole report is a receipt that ``consilience verify`` can re-run.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import copy
|
|
31
|
+
import csv
|
|
32
|
+
import html
|
|
33
|
+
import io
|
|
34
|
+
from dataclasses import dataclass, field
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Union
|
|
37
|
+
|
|
38
|
+
from . import __version__
|
|
39
|
+
from .compare import PINT_VERSION, RECEIPT_SCHEMA, ROUNDING, InputError
|
|
40
|
+
from .hashing import sha256_bytes, sha256_json
|
|
41
|
+
from .rules import RuleSet, load_rules
|
|
42
|
+
from .units import UnitError, as_quantity, convert, convert_difference, parse_number, \
|
|
43
|
+
parse_unit
|
|
44
|
+
|
|
45
|
+
EXIT_CLEAN, EXIT_VIOLATIONS, EXIT_UNPARSEABLE = 0, 1, 3
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class CheckResult:
|
|
50
|
+
summary: dict
|
|
51
|
+
violations: list
|
|
52
|
+
unparseable: list
|
|
53
|
+
rules: dict
|
|
54
|
+
receipt: dict = field(repr=False)
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def exit_code(self) -> int:
|
|
58
|
+
if self.summary["unparseable_cells"]:
|
|
59
|
+
return EXIT_UNPARSEABLE
|
|
60
|
+
return EXIT_VIOLATIONS if self.summary["violations"] else EXIT_CLEAN
|
|
61
|
+
|
|
62
|
+
def to_dict(self) -> dict:
|
|
63
|
+
return copy.deepcopy(self.receipt)
|
|
64
|
+
|
|
65
|
+
def to_html(self) -> str:
|
|
66
|
+
return render_html(self.receipt)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class _Bad(Exception):
|
|
70
|
+
def __init__(self, column, reason, detail=""):
|
|
71
|
+
self.column, self.reason, self.detail = column, reason, detail
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _read_table(path: Path):
|
|
75
|
+
raw = path.read_bytes()
|
|
76
|
+
try:
|
|
77
|
+
text = raw.decode("utf-8-sig")
|
|
78
|
+
except UnicodeDecodeError:
|
|
79
|
+
raise InputError(f"{path}: not UTF-8 text") from None
|
|
80
|
+
lines = text.splitlines(keepends=True)
|
|
81
|
+
skip = 0
|
|
82
|
+
while skip < len(lines) and lines[skip].lstrip().startswith("#"):
|
|
83
|
+
skip += 1
|
|
84
|
+
reader = csv.reader(io.StringIO("".join(lines[skip:])), strict=True)
|
|
85
|
+
try:
|
|
86
|
+
header = next(reader)
|
|
87
|
+
except StopIteration:
|
|
88
|
+
raise InputError(f"{path}: no header row") from None
|
|
89
|
+
except csv.Error as exc:
|
|
90
|
+
raise InputError(f"{path}: cannot read the header: {exc}") from None
|
|
91
|
+
header = [h.strip() for h in header]
|
|
92
|
+
if not header or any(not h for h in header):
|
|
93
|
+
raise InputError(f"{path}: the header has an empty column name")
|
|
94
|
+
dup = sorted({h for h in header if header.count(h) > 1})
|
|
95
|
+
if dup:
|
|
96
|
+
raise InputError(f"{path}: duplicate column name(s) {dup}")
|
|
97
|
+
rows = []
|
|
98
|
+
prev = reader.line_num
|
|
99
|
+
try:
|
|
100
|
+
for cells in reader:
|
|
101
|
+
rows.append((skip + prev + 1, cells))
|
|
102
|
+
prev = reader.line_num
|
|
103
|
+
except csv.Error as exc:
|
|
104
|
+
raise InputError(f"{path}: line {skip + reader.line_num}: {exc}") from None
|
|
105
|
+
return raw, header, rows
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def check(data: Union[str, Path], rules: Union[str, Path, RuleSet]) -> CheckResult:
|
|
109
|
+
"""Check the CSV file ``data`` against ``rules`` (a path or a loaded RuleSet).
|
|
110
|
+
|
|
111
|
+
Raises ``InputError`` (missing file, unreadable CSV, a declared column absent) or
|
|
112
|
+
``RuleFileError`` (an unusable rule file).
|
|
113
|
+
"""
|
|
114
|
+
rs = load_rules(rules)
|
|
115
|
+
if rs.kind != "check-rules":
|
|
116
|
+
raise InputError(f"{rs.origin} is a {rs.kind} file; `check` needs a check-rules file")
|
|
117
|
+
p = Path(data)
|
|
118
|
+
if not p.is_file():
|
|
119
|
+
raise InputError(f"file not found: {data}")
|
|
120
|
+
raw, header, rows = _read_table(p)
|
|
121
|
+
C = rs.content
|
|
122
|
+
cols = C["columns"]
|
|
123
|
+
needed = set(cols) | {c["unit_column"] for c in cols.values() if "unit_column" in c}
|
|
124
|
+
if C.get("id_column"):
|
|
125
|
+
needed.add(C["id_column"])
|
|
126
|
+
missing = sorted(needed - set(header))
|
|
127
|
+
if missing:
|
|
128
|
+
raise InputError(f"{data}: column(s) {missing} named in the rule file are not in the "
|
|
129
|
+
f"header {header}")
|
|
130
|
+
fixed_units = {c: parse_unit(s["unit"]) for c, s in cols.items() if "unit" in s}
|
|
131
|
+
|
|
132
|
+
violations, issues = [], {}
|
|
133
|
+
by_rule = {r["id"]: {"violations": 0, "unparseable": 0, "skipped": 0, "passed": 0}
|
|
134
|
+
for r in C["rules"]}
|
|
135
|
+
n_rows = blank = 0
|
|
136
|
+
|
|
137
|
+
def report(line, rid, ident, column, reason, text, detail=""):
|
|
138
|
+
key = (line, column, reason)
|
|
139
|
+
if key not in issues:
|
|
140
|
+
issues[key] = {"line": line, "id": ident, "column": column, "reason": reason,
|
|
141
|
+
"text": text, "detail": detail, "rules": []}
|
|
142
|
+
if rid not in issues[key]["rules"]:
|
|
143
|
+
issues[key]["rules"].append(rid)
|
|
144
|
+
|
|
145
|
+
for line, cells in rows:
|
|
146
|
+
if not cells:
|
|
147
|
+
blank += 1
|
|
148
|
+
continue
|
|
149
|
+
n_rows += 1
|
|
150
|
+
ident = None
|
|
151
|
+
if len(cells) != len(header):
|
|
152
|
+
report(line, None, None, None, "row_length", ",".join(cells),
|
|
153
|
+
f"{len(cells)} cells, header has {len(header)}")
|
|
154
|
+
for r in C["rules"]:
|
|
155
|
+
by_rule[r["id"]]["unparseable"] += 1
|
|
156
|
+
issues[(line, None, "row_length")]["rules"] = [r["id"] for r in C["rules"]]
|
|
157
|
+
continue
|
|
158
|
+
row = dict(zip(header, cells))
|
|
159
|
+
if C.get("id_column"):
|
|
160
|
+
ident = row[C["id_column"]]
|
|
161
|
+
|
|
162
|
+
def unit_of(col):
|
|
163
|
+
if col in fixed_units:
|
|
164
|
+
return fixed_units[col]
|
|
165
|
+
ucol = cols[col]["unit_column"]
|
|
166
|
+
try:
|
|
167
|
+
return parse_unit(row[ucol])
|
|
168
|
+
except UnitError as exc:
|
|
169
|
+
raise _Bad(ucol, exc.reason, exc.detail) from None
|
|
170
|
+
|
|
171
|
+
def unit_written(col):
|
|
172
|
+
return (cols[col]["unit"] if col in fixed_units
|
|
173
|
+
else row[cols[col]["unit_column"]].strip())
|
|
174
|
+
|
|
175
|
+
def value_of(col):
|
|
176
|
+
t = row[col].strip()
|
|
177
|
+
if not t:
|
|
178
|
+
return None
|
|
179
|
+
try:
|
|
180
|
+
num = parse_number(t)
|
|
181
|
+
except UnitError as exc:
|
|
182
|
+
raise _Bad(col, exc.reason, exc.detail) from None
|
|
183
|
+
return num, unit_of(col)
|
|
184
|
+
|
|
185
|
+
def dim_column(*cs):
|
|
186
|
+
for c in cs:
|
|
187
|
+
if "unit_column" in cols[c]:
|
|
188
|
+
return cols[c]["unit_column"]
|
|
189
|
+
return cs[0]
|
|
190
|
+
|
|
191
|
+
for r in C["rules"]:
|
|
192
|
+
rid, col = r["id"], r["column"]
|
|
193
|
+
st = by_rule[rid]
|
|
194
|
+
try:
|
|
195
|
+
v = value_of(col)
|
|
196
|
+
if v is None:
|
|
197
|
+
if r["if_empty"] == "skip":
|
|
198
|
+
st["skipped"] += 1
|
|
199
|
+
continue
|
|
200
|
+
raise _Bad(col, "empty", "the rule needs a number here")
|
|
201
|
+
num, u = v
|
|
202
|
+
scale = abs(num)
|
|
203
|
+
base = {"rule": r["id"], "line": line, "id": ident, "column": col,
|
|
204
|
+
"value": row[col].strip(), "unit": unit_written(col),
|
|
205
|
+
"plain_english": r["plain_english"]}
|
|
206
|
+
hit = None
|
|
207
|
+
if r["kind"] == "bound":
|
|
208
|
+
for side in ("min", "max"):
|
|
209
|
+
if side not in r:
|
|
210
|
+
continue
|
|
211
|
+
q = as_quantity(r[side])
|
|
212
|
+
try:
|
|
213
|
+
lim = (float(q.magnitude) if str(q.units) == "dimensionless"
|
|
214
|
+
else convert(q, u))
|
|
215
|
+
except UnitError as exc:
|
|
216
|
+
raise _Bad(dim_column(col), exc.reason, exc.detail) from None
|
|
217
|
+
sl = ROUNDING * max(scale, abs(lim))
|
|
218
|
+
incl = r[f"{side}_inclusive"]
|
|
219
|
+
if side == "min":
|
|
220
|
+
bad = num < lim - sl if incl else num <= lim + sl
|
|
221
|
+
else:
|
|
222
|
+
bad = num > lim + sl if incl else num >= lim - sl
|
|
223
|
+
if bad:
|
|
224
|
+
hit = dict(base, bound=side, limit=lim, limit_unit=unit_written(col),
|
|
225
|
+
inclusive=incl)
|
|
226
|
+
break
|
|
227
|
+
else:
|
|
228
|
+
lcol = r["limit_column"]
|
|
229
|
+
route = "recorded_limit"
|
|
230
|
+
lv = value_of(lcol)
|
|
231
|
+
try:
|
|
232
|
+
if lv is None:
|
|
233
|
+
fb = r["if_limit_empty"]
|
|
234
|
+
if fb == "skip":
|
|
235
|
+
st["skipped"] += 1
|
|
236
|
+
continue
|
|
237
|
+
if fb == "refuse":
|
|
238
|
+
raise _Bad(lcol, "empty", "the rule needs a limit here")
|
|
239
|
+
q = as_quantity(fb)
|
|
240
|
+
lim = (convert(as_quantity(float(q.magnitude)) * unit_of(lcol), u)
|
|
241
|
+
if str(q.units) == "dimensionless" else convert(q, u))
|
|
242
|
+
route, ltext = "fallback_limit", str(fb)
|
|
243
|
+
else:
|
|
244
|
+
lim = convert(as_quantity(lv[0]) * lv[1], u)
|
|
245
|
+
ltext = row[lcol].strip()
|
|
246
|
+
|
|
247
|
+
def diff(x):
|
|
248
|
+
qx = as_quantity(x)
|
|
249
|
+
if str(qx.units) == "dimensionless":
|
|
250
|
+
return float(qx.magnitude) # in the column's own unit
|
|
251
|
+
return convert_difference(qx.magnitude, qx.units, u)
|
|
252
|
+
margin, off = diff(r["margin"]), diff(r["value_offset"])
|
|
253
|
+
except UnitError as exc:
|
|
254
|
+
raise _Bad(dim_column(col, lcol), exc.reason, exc.detail) from None
|
|
255
|
+
x = num + off
|
|
256
|
+
sl = ROUNDING * max(abs(x), abs(lim))
|
|
257
|
+
bad = (x > lim + margin + sl) if r["must_be"] == "at_most" else \
|
|
258
|
+
(x < lim - margin - sl)
|
|
259
|
+
if bad:
|
|
260
|
+
hit = dict(base, limit_column=lcol, limit_text=ltext, limit=lim,
|
|
261
|
+
limit_unit=unit_written(col), margin=margin,
|
|
262
|
+
value_offset=off, must_be=r["must_be"], route=route)
|
|
263
|
+
if hit is None:
|
|
264
|
+
st["passed"] += 1
|
|
265
|
+
else:
|
|
266
|
+
hit["row_sha256"] = sha256_json({"rules_sha256": rs.sha256,
|
|
267
|
+
"rule": rid, "line": line,
|
|
268
|
+
"cells": row})
|
|
269
|
+
violations.append(hit)
|
|
270
|
+
st["violations"] += 1
|
|
271
|
+
except _Bad as b:
|
|
272
|
+
st["unparseable"] += 1
|
|
273
|
+
report(line, rid, ident, b.column, b.reason,
|
|
274
|
+
row.get(b.column, ""), b.detail)
|
|
275
|
+
|
|
276
|
+
unparseable = sorted(issues.values(), key=lambda i: (i["line"], i["column"] or ""))
|
|
277
|
+
summary = {
|
|
278
|
+
"rows": n_rows,
|
|
279
|
+
"blank_lines": blank,
|
|
280
|
+
"rules": len(C["rules"]),
|
|
281
|
+
"violations": len(violations),
|
|
282
|
+
"unparseable_cells": len(unparseable),
|
|
283
|
+
"skipped_empty": sum(s["skipped"] for s in by_rule.values()),
|
|
284
|
+
"by_rule": by_rule,
|
|
285
|
+
}
|
|
286
|
+
result = {"summary": summary, "violations": violations, "unparseable": unparseable,
|
|
287
|
+
"header": header}
|
|
288
|
+
exit_code = (EXIT_UNPARSEABLE if unparseable else
|
|
289
|
+
EXIT_VIOLATIONS if violations else EXIT_CLEAN)
|
|
290
|
+
result["exit_code"] = exit_code
|
|
291
|
+
receipt = {
|
|
292
|
+
"receipt": RECEIPT_SCHEMA,
|
|
293
|
+
"command": "check",
|
|
294
|
+
"consilience_version": __version__,
|
|
295
|
+
"pint_version": PINT_VERSION,
|
|
296
|
+
"inputs": {"data": {"path": str(data), "sha256": sha256_bytes(raw),
|
|
297
|
+
"bytes": len(raw)}},
|
|
298
|
+
"rules": dict(rs.describe(), content=rs.content),
|
|
299
|
+
"result": result,
|
|
300
|
+
}
|
|
301
|
+
receipt["sha256"] = sha256_json(receipt)
|
|
302
|
+
return CheckResult(summary=summary, violations=violations, unparseable=unparseable,
|
|
303
|
+
rules=rs.describe(), receipt=receipt)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# --------------------------------------------------------------------- HTML
|
|
307
|
+
_CSS = """
|
|
308
|
+
body{font-family:system-ui,-apple-system,Segoe UI,sans-serif;margin:2rem;color:#1b1b1b;
|
|
309
|
+
max-width:72rem}
|
|
310
|
+
h1{font-size:1.4rem}h2{font-size:1.1rem;margin-top:2rem}
|
|
311
|
+
table{border-collapse:collapse;width:100%;font-size:.9rem}
|
|
312
|
+
th,td{border:1px solid #ccc;padding:.3rem .5rem;text-align:left;vertical-align:top}
|
|
313
|
+
th{background:#f2f2f2}code{font-size:.85rem}.hash{font-family:monospace;font-size:.75rem;
|
|
314
|
+
word-break:break-all}.note{color:#555}
|
|
315
|
+
"""
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _fmt(x) -> str:
|
|
319
|
+
if isinstance(x, float):
|
|
320
|
+
return f"{x:.6g}"
|
|
321
|
+
return "" if x is None else str(x)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def render_html(receipt: dict) -> str:
|
|
325
|
+
"""A self-contained HTML page for a check receipt. Every value is escaped."""
|
|
326
|
+
e = html.escape
|
|
327
|
+
res, rules = receipt["result"], receipt["rules"]
|
|
328
|
+
s = res["summary"]
|
|
329
|
+
rule_text = {r["id"]: r["plain_english"] for r in rules["content"]["rules"]}
|
|
330
|
+
out = ["<!DOCTYPE html>", '<html lang="en"><head><meta charset="utf-8">',
|
|
331
|
+
f"<title>Data check: {e(receipt['inputs']['data']['path'])}</title>",
|
|
332
|
+
f"<style>{_CSS}</style></head><body>",
|
|
333
|
+
f"<h1>Data check: {e(receipt['inputs']['data']['path'])}</h1>",
|
|
334
|
+
"<table>",
|
|
335
|
+
f"<tr><th>Rules</th><td>{e(rules['name'])} ({e(rules['version'])})<br>"
|
|
336
|
+
f"<span class='hash'>sha256 {e(rules['sha256'])}</span></td></tr>",
|
|
337
|
+
f"<tr><th>Data</th><td>{s['rows']} rows, {receipt['inputs']['data']['bytes']} "
|
|
338
|
+
f"bytes<br><span class='hash'>sha256 {e(receipt['inputs']['data']['sha256'])}"
|
|
339
|
+
"</span></td></tr>",
|
|
340
|
+
f"<tr><th>Violations</th><td>{s['violations']}</td></tr>",
|
|
341
|
+
f"<tr><th>Cells that could not be read</th><td>{s['unparseable_cells']}"
|
|
342
|
+
+ (" (the check is incomplete for these rows)" if s["unparseable_cells"] else "")
|
|
343
|
+
+ "</td></tr>",
|
|
344
|
+
f"<tr><th>Empty cells skipped by rule</th><td>{s['skipped_empty']}</td></tr>",
|
|
345
|
+
f"<tr><th>Blank lines</th><td>{s['blank_lines']}</td></tr>",
|
|
346
|
+
f"<tr><th>Report receipt</th><td class='hash'>sha256 {e(receipt['sha256'])}</td>"
|
|
347
|
+
"</tr>",
|
|
348
|
+
f"<tr><th>Made with</th><td>consilience {e(receipt['consilience_version'])}"
|
|
349
|
+
"</td></tr>",
|
|
350
|
+
"</table>",
|
|
351
|
+
"<h2>Rules</h2><table><tr><th>Rule</th><th>What it says</th><th>Violations</th>"
|
|
352
|
+
"<th>Could not check</th><th>Skipped</th><th>Passed</th></tr>"]
|
|
353
|
+
for rid, st in s["by_rule"].items():
|
|
354
|
+
out.append(f"<tr><td>{e(rid)}</td><td>{e(rule_text.get(rid, ''))}</td>"
|
|
355
|
+
f"<td>{st['violations']}</td><td>{st['unparseable']}</td>"
|
|
356
|
+
f"<td>{st['skipped']}</td><td>{st['passed']}</td></tr>")
|
|
357
|
+
out.append("</table><h2>Violations</h2>")
|
|
358
|
+
if res["violations"]:
|
|
359
|
+
out.append("<table><tr><th>Line</th><th>Row id</th><th>Rule</th><th>Column</th>"
|
|
360
|
+
"<th>Value</th><th>Limit</th><th>Row receipt</th></tr>")
|
|
361
|
+
for v in res["violations"]:
|
|
362
|
+
if "bound" in v:
|
|
363
|
+
lim = e(f"{v['bound']} {_fmt(v['limit'])} {v['limit_unit']}"
|
|
364
|
+
+ ("" if v["inclusive"] else " (exclusive)"))
|
|
365
|
+
else:
|
|
366
|
+
op = "≤" if v["must_be"] == "at_most" else "≥"
|
|
367
|
+
lim = (f"{op} {e(v['limit_column'])} = "
|
|
368
|
+
+ e(f"{_fmt(v['limit'])} {v['limit_unit']}, margin {_fmt(v['margin'])} "
|
|
369
|
+
f"{v['limit_unit']}"
|
|
370
|
+
+ (f", value offset {_fmt(v['value_offset'])} {v['limit_unit']}"
|
|
371
|
+
if v["value_offset"] else "")
|
|
372
|
+
+ (" (fallback limit)" if v["route"] == "fallback_limit" else "")))
|
|
373
|
+
out.append(f"<tr><td>{v['line']}</td><td>{e(_fmt(v['id']))}</td>"
|
|
374
|
+
f"<td>{e(v['rule'])}</td><td>{e(v['column'])}</td>"
|
|
375
|
+
f"<td>{e(v['value'])} {e(v['unit'])}</td>"
|
|
376
|
+
f"<td>{lim}</td>"
|
|
377
|
+
f"<td class='hash'>{e(v['row_sha256'])}</td></tr>")
|
|
378
|
+
out.append("</table>")
|
|
379
|
+
else:
|
|
380
|
+
out.append("<p>None.</p>")
|
|
381
|
+
out.append("<h2>Cells that could not be read</h2>")
|
|
382
|
+
if res["unparseable"]:
|
|
383
|
+
out.append("<p class='note'>These cells were not guessed. Fix them and run the "
|
|
384
|
+
"check again.</p><table><tr><th>Line</th><th>Row id</th><th>Column</th>"
|
|
385
|
+
"<th>Text</th><th>Reason</th><th>Needed by</th></tr>")
|
|
386
|
+
for u in res["unparseable"]:
|
|
387
|
+
out.append(f"<tr><td>{u['line']}</td><td>{e(_fmt(u['id']))}</td>"
|
|
388
|
+
f"<td>{e(_fmt(u['column']) or '(whole row)')}</td>"
|
|
389
|
+
f"<td><code>{e(u['text'])}</code></td>"
|
|
390
|
+
f"<td>{e(u['reason'])}{': ' + e(u['detail']) if u['detail'] else ''}"
|
|
391
|
+
f"</td><td>{e(', '.join(x for x in u['rules'] if x))}</td></tr>")
|
|
392
|
+
out.append("</table>")
|
|
393
|
+
else:
|
|
394
|
+
out.append("<p>None.</p>")
|
|
395
|
+
out.append("<p class='note'>Re-run this report and confirm its hash with "
|
|
396
|
+
"<code>consilience verify <receipt.json></code>.</p></body></html>")
|
|
397
|
+
return "\n".join(out) + "\n"
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
__all__ = ["CheckResult", "InputError", "check", "render_html"]
|