protocolith 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protocolith/__init__.py +38 -0
- protocolith/baseline_groups.py +294 -0
- protocolith/baselines.py +288 -0
- protocolith/card_svg_render.py +281 -0
- protocolith/card_svg_styles.py +178 -0
- protocolith/cli.py +796 -0
- protocolith/core/__init__.py +63 -0
- protocolith/core/_lock_posix.py +29 -0
- protocolith/core/_lock_windows.py +29 -0
- protocolith/core/atlas.py +406 -0
- protocolith/core/atlas_assets/atlas.css +86 -0
- protocolith/core/atlas_assets/atlas.html +84 -0
- protocolith/core/atlas_assets/atlas.js +370 -0
- protocolith/core/atlas_i18n/en.json +61 -0
- protocolith/core/canonical.py +124 -0
- protocolith/core/capture.py +68 -0
- protocolith/core/clock.py +25 -0
- protocolith/core/commands.py +356 -0
- protocolith/core/errors.py +116 -0
- protocolith/core/issue.py +98 -0
- protocolith/core/ledger.py +75 -0
- protocolith/core/mcp.py +371 -0
- protocolith/core/ops.py +183 -0
- protocolith/core/rules.py +352 -0
- protocolith/core/schema.py +137 -0
- protocolith/core/store.py +101 -0
- protocolith/core/views.py +136 -0
- protocolith/doctor.py +135 -0
- protocolith/evidence_card.py +354 -0
- protocolith/family_ledger.py +489 -0
- protocolith/generic_ohlcv_batch_pre_registration_meta.py +123 -0
- protocolith/inspect.py +48 -0
- protocolith/nasa_power.py +266 -0
- protocolith/nasa_power_fetch.py +64 -0
- protocolith/nasa_power_parser.py +133 -0
- protocolith/nasa_power_prereg_runner.py +383 -0
- protocolith/noaa_coops_fetch.py +188 -0
- protocolith/noaa_coops_parser.py +120 -0
- protocolith/noaa_coops_prereg_runner.py +341 -0
- protocolith/public_ai_protocol.py +176 -0
- protocolith/py.typed +0 -0
- protocolith/registry.py +139 -0
- protocolith/registry_view.py +302 -0
- protocolith/rerun_report.py +111 -0
- protocolith/run_root.py +243 -0
- protocolith/scaffold.py +565 -0
- protocolith/source_probe.py +230 -0
- protocolith/svg_a11y.py +100 -0
- protocolith/tree_overlay.py +343 -0
- protocolith/verify_packs.py +251 -0
- protocolith/workflows.py +233 -0
- protocolith-0.6.0.dist-info/METADATA +283 -0
- protocolith-0.6.0.dist-info/RECORD +57 -0
- protocolith-0.6.0.dist-info/WHEEL +4 -0
- protocolith-0.6.0.dist-info/entry_points.txt +3 -0
- protocolith-0.6.0.dist-info/licenses/LICENSE +201 -0
- protocolith-0.6.0.dist-info/licenses/NOTICE +13 -0
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Ledger verification: rules R1 to R11 plus the size and depth limits."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Callable, Mapping
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from functools import partial
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .canonical import CanonError, decode, encode, sha256_hex
|
|
13
|
+
from .errors import ProtocolithError, Violation
|
|
14
|
+
from .ledger import (
|
|
15
|
+
CARD_FORMAT,
|
|
16
|
+
ENTRY_KEYS,
|
|
17
|
+
ENTRY_TYPES,
|
|
18
|
+
FORMAT_VERSION,
|
|
19
|
+
MAX_ENTRY_BYTES,
|
|
20
|
+
RESULT_STATUSES,
|
|
21
|
+
ZERO_HASH,
|
|
22
|
+
Entry,
|
|
23
|
+
entry_hash,
|
|
24
|
+
is_hash,
|
|
25
|
+
is_ts,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
Pred = Callable[[Any], bool]
|
|
29
|
+
Add = Callable[[str, str], None]
|
|
30
|
+
CardHasher = Callable[["Verifier", int], str]
|
|
31
|
+
|
|
32
|
+
_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
|
|
33
|
+
_VERSION_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,63}")
|
|
34
|
+
_BLOCKING = {"BLOCKED_SOURCE": "block_reason", "SOURCE_DRIFT": "drift_reason"}
|
|
35
|
+
_NEEDS_SOURCES = frozenset({"PASSED_UNDER_PROTOCOL", "NEGATIVE_RESULT_UNDER_PROTOCOL", "REPRODUCED_OUTCOME"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _text(value: Any) -> bool:
|
|
39
|
+
return isinstance(value, str) and value.strip() != ""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _obj(value: Any) -> bool:
|
|
43
|
+
return isinstance(value, dict)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _uint(value: Any) -> bool:
|
|
47
|
+
return type(value) is int and value >= 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _uint_list(value: Any) -> bool:
|
|
51
|
+
return isinstance(value, list) and all(_uint(item) for item in value)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _source(value: Any) -> bool:
|
|
55
|
+
return bool(isinstance(value, dict) and set(value) == {"id", "sha256"} and _text(value["id"]) and is_hash(value["sha256"]))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _sources(value: Any) -> bool:
|
|
59
|
+
return isinstance(value, list) and all(_source(item) for item in value)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _hash_or_null(value: Any) -> bool:
|
|
63
|
+
return value is None or is_hash(value)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _protocol_id(value: Any) -> bool:
|
|
67
|
+
return isinstance(value, str) and _ID_RE.fullmatch(value) is not None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _version(value: Any) -> bool:
|
|
71
|
+
return isinstance(value, str) and _VERSION_RE.fullmatch(value) is not None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _any(value: Any) -> bool:
|
|
75
|
+
return True
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _is_format(value: Any) -> bool:
|
|
79
|
+
return bool(value == CARD_FORMAT)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _blank(value: Any) -> bool:
|
|
83
|
+
return isinstance(value, str) and value.strip() == ""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
FREEZE_FIELDS: dict[str, Pred] = {
|
|
87
|
+
"protocol_id": _protocol_id,
|
|
88
|
+
"version": _version,
|
|
89
|
+
"protocol": _obj,
|
|
90
|
+
"protocol_sha256": is_hash,
|
|
91
|
+
"supersedes": _hash_or_null,
|
|
92
|
+
}
|
|
93
|
+
RECORD_FIELDS: dict[str, Pred] = {
|
|
94
|
+
"protocol_sha256": is_hash,
|
|
95
|
+
"run_id": _text,
|
|
96
|
+
"status": _any,
|
|
97
|
+
"outcome": _obj,
|
|
98
|
+
"sources": _sources,
|
|
99
|
+
"reproduction_level": _text,
|
|
100
|
+
"claim_boundary": _text,
|
|
101
|
+
"deviations": _uint_list,
|
|
102
|
+
}
|
|
103
|
+
RECORD_OPTIONAL: dict[str, Pred] = {"block_reason": _text, "drift_reason": _text, "card": _obj}
|
|
104
|
+
DEVIATION_FIELDS: dict[str, Pred] = {"protocol_sha256": is_hash, "kind": _text, "description": _text}
|
|
105
|
+
ISSUE_FIELDS: dict[str, Pred] = {"record_seq": _uint, "format": _is_format, "card_sha256": is_hash}
|
|
106
|
+
ENTRY_FIELDS: dict[str, Pred] = {
|
|
107
|
+
"seq": _uint,
|
|
108
|
+
"ts": is_ts,
|
|
109
|
+
"prev": is_hash,
|
|
110
|
+
"hash": is_hash,
|
|
111
|
+
"type": lambda value: value in ENTRY_TYPES,
|
|
112
|
+
"body": _obj,
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def shape_problems(body: Entry, required: Mapping[str, Pred], optional: Mapping[str, Pred]) -> list[tuple[str, str]]:
|
|
117
|
+
"""Check field presence, type and closed key set of a body."""
|
|
118
|
+
preds = {**required, **optional}
|
|
119
|
+
missing = sorted(key for key in required if key not in body or _blank(body[key]))
|
|
120
|
+
bad = sorted(key for key, value in body.items() if key in preds and key not in missing and not preds[key](value))
|
|
121
|
+
extra = sorted(key for key in body if key not in preds)
|
|
122
|
+
problems: list[tuple[str, str]] = []
|
|
123
|
+
if missing:
|
|
124
|
+
problems.append(("MISSING_FIELD", "missing fields: " + ", ".join(missing)))
|
|
125
|
+
if bad or extra:
|
|
126
|
+
problems.append(("BAD_ENTRY", f"invalid fields: {', '.join(bad)}; unexpected fields: {', '.join(extra)}"))
|
|
127
|
+
return problems
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass
|
|
131
|
+
class Verdict:
|
|
132
|
+
ok: bool
|
|
133
|
+
count: int
|
|
134
|
+
head_seq: int | None
|
|
135
|
+
head_hash: str | None
|
|
136
|
+
violations: list[Violation]
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def codes(self) -> list[str]:
|
|
140
|
+
return sorted({v.code for v in self.violations})
|
|
141
|
+
|
|
142
|
+
def to_dict(self) -> dict[str, Any]:
|
|
143
|
+
return {
|
|
144
|
+
"codes": self.codes,
|
|
145
|
+
"count": self.count,
|
|
146
|
+
"head": {"hash": self.head_hash, "seq": self.head_seq},
|
|
147
|
+
"ok": self.ok,
|
|
148
|
+
"violations": [v.to_dict() for v in self.violations],
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _tip_of(obj: Any) -> str | None:
|
|
153
|
+
return obj["hash"] if isinstance(obj, dict) and is_hash(obj.get("hash")) else None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _structure(obj: Any) -> tuple[str, str] | None:
|
|
157
|
+
if not isinstance(obj, dict):
|
|
158
|
+
return ("BAD_ENTRY", "entry is not an object")
|
|
159
|
+
if "v" in obj and not (type(obj["v"]) is int and obj["v"] == FORMAT_VERSION):
|
|
160
|
+
return ("UNKNOWN_VERSION", f"unknown format version: {obj['v']!r}")
|
|
161
|
+
if set(obj) != ENTRY_KEYS:
|
|
162
|
+
return ("BAD_ENTRY", "entry keys must be exactly: " + ", ".join(sorted(ENTRY_KEYS)))
|
|
163
|
+
bad = sorted(key for key, pred in ENTRY_FIELDS.items() if not pred(obj[key]))
|
|
164
|
+
if bad:
|
|
165
|
+
return ("BAD_ENTRY", "invalid entry fields: " + ", ".join(bad))
|
|
166
|
+
return None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class Verifier:
|
|
170
|
+
"""Incremental ledger checker. Feed complete lines (without the newline) in order."""
|
|
171
|
+
|
|
172
|
+
def __init__(self, card_hasher: CardHasher | None = None) -> None:
|
|
173
|
+
self.card_hasher = card_hasher
|
|
174
|
+
self.index = 0
|
|
175
|
+
self.prev_hash: str | None = ZERO_HASH
|
|
176
|
+
self.prev_ts: str | None = None
|
|
177
|
+
self.entries: dict[int, Entry] = {}
|
|
178
|
+
self.protocols: dict[str, dict[str, Any]] = {}
|
|
179
|
+
self.frozen_ids: dict[tuple[str, str], int] = {}
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def tip_hash(self) -> str:
|
|
183
|
+
return self.prev_hash or ZERO_HASH
|
|
184
|
+
|
|
185
|
+
def feed(self, line: bytes, *, last: bool = False) -> list[Violation]:
|
|
186
|
+
index = self.index
|
|
187
|
+
self.index += 1
|
|
188
|
+
found: list[Violation] = []
|
|
189
|
+
add: Add = partial(_add, found, index)
|
|
190
|
+
expected_prev = self.prev_hash
|
|
191
|
+
entry = self._read(line, last, add)
|
|
192
|
+
if entry is None:
|
|
193
|
+
return found
|
|
194
|
+
self._chain(index, line, entry, expected_prev, add)
|
|
195
|
+
handler = {
|
|
196
|
+
"freeze": self._freeze,
|
|
197
|
+
"record": self._record,
|
|
198
|
+
"deviation": self._deviation,
|
|
199
|
+
"issue": self._issue,
|
|
200
|
+
}[entry["type"]]
|
|
201
|
+
if handler(index, entry["body"], add):
|
|
202
|
+
self.entries[index] = entry
|
|
203
|
+
self.prev_ts = entry["ts"]
|
|
204
|
+
return found
|
|
205
|
+
|
|
206
|
+
def _read(self, line: bytes, last: bool, add: Add) -> Entry | None:
|
|
207
|
+
self.prev_hash = None
|
|
208
|
+
if len(line) > MAX_ENTRY_BYTES:
|
|
209
|
+
add("ENTRY_TOO_LARGE", f"entry is {len(line)} bytes, limit {MAX_ENTRY_BYTES}")
|
|
210
|
+
return None
|
|
211
|
+
try:
|
|
212
|
+
obj = decode(line)
|
|
213
|
+
except CanonError as exc:
|
|
214
|
+
code = "TOO_DEEP" if exc.code == "TOO_DEEP" else ("TRUNCATED_TAIL" if last else "BAD_ENTRY")
|
|
215
|
+
add(code, f"line does not parse: {exc.code}: {exc.message}")
|
|
216
|
+
return None
|
|
217
|
+
self.prev_hash = _tip_of(obj)
|
|
218
|
+
problem = _structure(obj)
|
|
219
|
+
if problem is not None:
|
|
220
|
+
add(*problem)
|
|
221
|
+
return None
|
|
222
|
+
entry: Entry = obj
|
|
223
|
+
return entry
|
|
224
|
+
|
|
225
|
+
def _chain(self, index: int, line: bytes, entry: Entry, expected_prev: str | None, add: Add) -> None:
|
|
226
|
+
if entry["seq"] != index:
|
|
227
|
+
add("BAD_SEQ", f"seq is {entry['seq']}, expected {index}")
|
|
228
|
+
if expected_prev is not None and entry["prev"] != expected_prev:
|
|
229
|
+
add("BAD_PREV", "prev does not equal the previous entry hash")
|
|
230
|
+
if entry_hash(entry) != entry["hash"]:
|
|
231
|
+
add("BAD_HASH", "hash does not match the recomputed value")
|
|
232
|
+
if encode(entry) != line:
|
|
233
|
+
add("NOT_CANONICAL", "line is not the exact canonical form")
|
|
234
|
+
if self.prev_ts is not None and entry["ts"] < self.prev_ts:
|
|
235
|
+
add("TS_REGRESSION", f"ts {entry['ts']} is earlier than {self.prev_ts}")
|
|
236
|
+
|
|
237
|
+
def _shape(self, body: Entry, required: Mapping[str, Pred], optional: Mapping[str, Pred], add: Add) -> bool:
|
|
238
|
+
problems = shape_problems(body, required, optional)
|
|
239
|
+
for code, message in problems:
|
|
240
|
+
add(code, message)
|
|
241
|
+
return not problems
|
|
242
|
+
|
|
243
|
+
def _known_protocol(self, body: Entry, add: Add) -> None:
|
|
244
|
+
if body["protocol_sha256"] not in self.protocols:
|
|
245
|
+
add("UNKNOWN_PROTOCOL", "protocol_sha256 was not frozen at an earlier seq")
|
|
246
|
+
|
|
247
|
+
def _freeze(self, index: int, body: Entry, add: Add) -> bool:
|
|
248
|
+
if not self._shape(body, FREEZE_FIELDS, {}, add):
|
|
249
|
+
return False
|
|
250
|
+
protocol = body["protocol"]
|
|
251
|
+
if sha256_hex(protocol) != body["protocol_sha256"]:
|
|
252
|
+
add("BAD_HASH", "protocol_sha256 does not match protocol")
|
|
253
|
+
if protocol.get("protocol_id") != body["protocol_id"] or protocol.get("version") != body["version"]:
|
|
254
|
+
add("BAD_ENTRY", "protocol_id and version must match the frozen protocol")
|
|
255
|
+
old = self.protocols.get(body["supersedes"] or "")
|
|
256
|
+
if body["supersedes"] is not None and (old is None or old["protocol_id"] != body["protocol_id"]):
|
|
257
|
+
add("UNKNOWN_PROTOCOL", "supersedes does not name an earlier freeze of the same protocol_id")
|
|
258
|
+
key = (body["protocol_id"], body["version"])
|
|
259
|
+
if key in self.frozen_ids or body["protocol_sha256"] in self.protocols:
|
|
260
|
+
add("DUPLICATE_FREEZE", f"{key[0]} version {key[1]} is already frozen")
|
|
261
|
+
self.frozen_ids.setdefault(key, index)
|
|
262
|
+
self.protocols.setdefault(
|
|
263
|
+
body["protocol_sha256"],
|
|
264
|
+
{"seq": index, "protocol_id": key[0], "version": key[1], "protocol": protocol},
|
|
265
|
+
)
|
|
266
|
+
return True
|
|
267
|
+
|
|
268
|
+
def _record(self, index: int, body: Entry, add: Add) -> bool:
|
|
269
|
+
if not self._shape(body, RECORD_FIELDS, RECORD_OPTIONAL, add):
|
|
270
|
+
return False
|
|
271
|
+
status = body["status"]
|
|
272
|
+
if not (isinstance(status, str) and status in RESULT_STATUSES):
|
|
273
|
+
add("BAD_STATUS", f"status is not in the closed set: {status!r}")
|
|
274
|
+
else:
|
|
275
|
+
needed = [_BLOCKING[status]] if status in _BLOCKING and _BLOCKING[status] not in body else []
|
|
276
|
+
if status in _NEEDS_SOURCES and not body["sources"]:
|
|
277
|
+
needed.append("sources")
|
|
278
|
+
if needed:
|
|
279
|
+
add("MISSING_FIELD", f"status {status} requires: " + ", ".join(needed))
|
|
280
|
+
self._known_protocol(body, add)
|
|
281
|
+
for ref in body["deviations"]:
|
|
282
|
+
target = self.entries.get(ref)
|
|
283
|
+
if target is None or target["type"] != "deviation":
|
|
284
|
+
add("DANGLING_REF", f"deviations entry {ref} is not an earlier deviation")
|
|
285
|
+
elif target["body"]["protocol_sha256"] != body["protocol_sha256"]:
|
|
286
|
+
add("DANGLING_REF", f"deviation {ref} belongs to another protocol")
|
|
287
|
+
return True
|
|
288
|
+
|
|
289
|
+
def _deviation(self, index: int, body: Entry, add: Add) -> bool:
|
|
290
|
+
if not self._shape(body, DEVIATION_FIELDS, {}, add):
|
|
291
|
+
return False
|
|
292
|
+
self._known_protocol(body, add)
|
|
293
|
+
return True
|
|
294
|
+
|
|
295
|
+
def _issue(self, index: int, body: Entry, add: Add) -> bool:
|
|
296
|
+
if not self._shape(body, ISSUE_FIELDS, {}, add):
|
|
297
|
+
return False
|
|
298
|
+
target = self.entries.get(body["record_seq"])
|
|
299
|
+
if target is None or target["type"] != "record":
|
|
300
|
+
add("DANGLING_REF", f"record_seq {body['record_seq']} is not an earlier record")
|
|
301
|
+
elif self.card_hasher is not None:
|
|
302
|
+
self._check_card(body, add)
|
|
303
|
+
return True
|
|
304
|
+
|
|
305
|
+
def _check_card(self, body: Entry, add: Add) -> None:
|
|
306
|
+
assert self.card_hasher is not None
|
|
307
|
+
try:
|
|
308
|
+
expected = self.card_hasher(self, body["record_seq"])
|
|
309
|
+
except ProtocolithError as exc:
|
|
310
|
+
add("BAD_CARD", f"card cannot be rebuilt: {exc.code}: {exc.message}")
|
|
311
|
+
return
|
|
312
|
+
if expected != body["card_sha256"]:
|
|
313
|
+
add("BAD_CARD", "card_sha256 does not match the card rebuilt from the record")
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _add(found: list[Violation], index: int, code: str, message: str) -> None:
|
|
317
|
+
found.append(Violation(code, index, message))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def scan(data: bytes, card_hasher: CardHasher | None = None) -> tuple[Verdict, Verifier]:
|
|
321
|
+
"""Verify a whole ledger given as bytes."""
|
|
322
|
+
verifier = Verifier(card_hasher)
|
|
323
|
+
lines = data.split(b"\n")
|
|
324
|
+
tail = lines.pop()
|
|
325
|
+
violations: list[Violation] = []
|
|
326
|
+
for number, line in enumerate(lines):
|
|
327
|
+
violations.extend(verifier.feed(line, last=number == len(lines) - 1 and not tail))
|
|
328
|
+
if tail:
|
|
329
|
+
violations.append(Violation("TRUNCATED_TAIL", len(lines), "last line is not terminated by a newline"))
|
|
330
|
+
count = len(lines)
|
|
331
|
+
head_hash = verifier.prev_hash if count else None
|
|
332
|
+
verdict = Verdict(not violations, count, count - 1 if head_hash else None, head_hash, violations)
|
|
333
|
+
return verdict, verifier
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def apply_expected_head(verdict: Verdict, verifier: Verifier, seq: int, digest: str) -> Verdict:
|
|
337
|
+
"""After a passing verification, require the entry at ``seq`` to have hash ``digest``; else add HEAD_MISMATCH.
|
|
338
|
+
|
|
339
|
+
A ledger that already fails keeps its own violations: the head is only meaningful for an intact ledger.
|
|
340
|
+
"""
|
|
341
|
+
if not verdict.ok:
|
|
342
|
+
return verdict
|
|
343
|
+
entry = verifier.entries.get(seq)
|
|
344
|
+
if entry is None:
|
|
345
|
+
message = f"the ledger has no entry at seq {seq} (it has {verdict.count} entries)"
|
|
346
|
+
elif entry["hash"] != digest:
|
|
347
|
+
message = f"entry {seq} has hash {entry['hash']}, expected {digest}"
|
|
348
|
+
else:
|
|
349
|
+
return verdict
|
|
350
|
+
verdict.violations.append(Violation("HEAD_MISMATCH", seq, message))
|
|
351
|
+
verdict.ok = False
|
|
352
|
+
return verdict
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""JSON Schema (draft 2020-12) for the input bodies and the entry envelope, derived from the rules.
|
|
3
|
+
|
|
4
|
+
Every field predicate in ``rules`` has an entry in ``_BY_PRED``; a field whose predicate is missing there raises
|
|
5
|
+
``KeyError`` when the schema is built, so the schema cannot silently drift from the rules.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import copy
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from . import rules
|
|
15
|
+
from .errors import InputError
|
|
16
|
+
from .ledger import _HASH_RE, _TS_RE, CARD_FORMAT, ENTRY_TYPES, FORMAT_VERSION, RESULT_STATUSES
|
|
17
|
+
|
|
18
|
+
DIALECT = "https://json-schema.org/draft/2020-12/schema"
|
|
19
|
+
NAMES = ("freeze", "record", "deviation", "issue", "entry")
|
|
20
|
+
|
|
21
|
+
_HASH: dict[str, Any] = {"type": "string", "pattern": "^" + _HASH_RE.pattern + "$"}
|
|
22
|
+
_TEXT: dict[str, Any] = {"type": "string", "pattern": r"\S"}
|
|
23
|
+
_UINT: dict[str, Any] = {"type": "integer", "minimum": 0}
|
|
24
|
+
_SOURCE: dict[str, Any] = {
|
|
25
|
+
"type": "object",
|
|
26
|
+
"properties": {"id": _TEXT, "sha256": _HASH},
|
|
27
|
+
"required": ["id", "sha256"],
|
|
28
|
+
"additionalProperties": False,
|
|
29
|
+
}
|
|
30
|
+
_BY_PRED: dict[Any, dict[str, Any]] = {
|
|
31
|
+
rules._text: _TEXT,
|
|
32
|
+
rules._obj: {"type": "object"},
|
|
33
|
+
rules._uint: _UINT,
|
|
34
|
+
rules._uint_list: {"type": "array", "items": _UINT},
|
|
35
|
+
rules._sources: {"type": "array", "items": _SOURCE},
|
|
36
|
+
rules.is_hash: _HASH,
|
|
37
|
+
rules._hash_or_null: {"anyOf": [_HASH, {"type": "null"}]},
|
|
38
|
+
rules._protocol_id: {"type": "string", "pattern": "^" + rules._ID_RE.pattern + "$"},
|
|
39
|
+
rules._version: {"type": "string", "pattern": "^" + rules._VERSION_RE.pattern + "$"},
|
|
40
|
+
rules._is_format: {"const": CARD_FORMAT},
|
|
41
|
+
rules.is_ts: {"type": "string", "pattern": "^" + _TS_RE.pattern + "$"},
|
|
42
|
+
rules._any: {},
|
|
43
|
+
}
|
|
44
|
+
# Fields whose predicate is a closed-set check living outside the predicate itself.
|
|
45
|
+
_BY_FIELD: dict[str, dict[str, Any]] = {
|
|
46
|
+
"status": {"enum": sorted(RESULT_STATUSES)},
|
|
47
|
+
"type": {"enum": list(ENTRY_TYPES)},
|
|
48
|
+
}
|
|
49
|
+
_DESCRIPTIONS = {
|
|
50
|
+
"freeze": "Input of `freeze`: the protocol object. protocol_id and version are required; any other key is part of the protocol.",
|
|
51
|
+
"record": "Input of `record` and body of a record entry.",
|
|
52
|
+
"deviation": "Input of `deviation` and body of a deviation entry.",
|
|
53
|
+
"issue": "Input of `issue`: the seq of the record to issue. (The stored issue body is under $defs of `entry`.)",
|
|
54
|
+
"entry": "One ledger line. The body is checked by the definition for its type.",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _field(name: str, pred: rules.Pred) -> dict[str, Any]:
|
|
59
|
+
return copy.deepcopy(_BY_FIELD[name] if name in _BY_FIELD else _BY_PRED[pred])
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _object(required: Mapping[str, rules.Pred], optional: Mapping[str, rules.Pred]) -> dict[str, Any]:
|
|
63
|
+
fields = {**required, **optional}
|
|
64
|
+
return {
|
|
65
|
+
"type": "object",
|
|
66
|
+
"properties": {name: _field(name, pred) for name, pred in fields.items()},
|
|
67
|
+
"required": list(required),
|
|
68
|
+
"additionalProperties": False,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _record_body() -> dict[str, Any]:
|
|
73
|
+
schema = _object(rules.RECORD_FIELDS, rules.RECORD_OPTIONAL)
|
|
74
|
+
conditions = []
|
|
75
|
+
for status in sorted(RESULT_STATUSES):
|
|
76
|
+
needed = [rules._BLOCKING[status]] if status in rules._BLOCKING else []
|
|
77
|
+
if status in rules._NEEDS_SOURCES:
|
|
78
|
+
conditions.append(
|
|
79
|
+
{
|
|
80
|
+
"if": {"properties": {"status": {"const": status}}},
|
|
81
|
+
"then": {"properties": {"sources": {"minItems": 1}}},
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
if needed:
|
|
85
|
+
conditions.append({"if": {"properties": {"status": {"const": status}}}, "then": {"required": needed}})
|
|
86
|
+
schema["allOf"] = conditions
|
|
87
|
+
return schema
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _freeze_input() -> dict[str, Any]:
|
|
91
|
+
keys = ("protocol_id", "version")
|
|
92
|
+
schema = _object({k: rules.FREEZE_FIELDS[k] for k in keys}, {})
|
|
93
|
+
schema["additionalProperties"] = True
|
|
94
|
+
return schema
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _bodies() -> dict[str, dict[str, Any]]:
|
|
98
|
+
return {
|
|
99
|
+
"freeze": _object(rules.FREEZE_FIELDS, {}),
|
|
100
|
+
"record": _record_body(),
|
|
101
|
+
"deviation": _object(rules.DEVIATION_FIELDS, {}),
|
|
102
|
+
"issue": _object(rules.ISSUE_FIELDS, {}),
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _entry() -> dict[str, Any]:
|
|
107
|
+
schema = _object(rules.ENTRY_FIELDS, {})
|
|
108
|
+
schema["properties"]["v"] = {"const": FORMAT_VERSION}
|
|
109
|
+
schema["required"].append("v")
|
|
110
|
+
schema["$defs"] = {f"{name}_body": body for name, body in _bodies().items()}
|
|
111
|
+
schema["allOf"] = [
|
|
112
|
+
{
|
|
113
|
+
"if": {"properties": {"type": {"const": name}}},
|
|
114
|
+
"then": {"properties": {"body": {"$ref": f"#/$defs/{name}_body"}}},
|
|
115
|
+
}
|
|
116
|
+
for name in ENTRY_TYPES
|
|
117
|
+
]
|
|
118
|
+
return schema
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def schema_for(name: str) -> dict[str, Any]:
|
|
122
|
+
"""The JSON Schema for one input (``freeze``, ``record``, ``deviation``, ``issue``) or the ledger ``entry``."""
|
|
123
|
+
if name == "freeze":
|
|
124
|
+
body = _freeze_input()
|
|
125
|
+
elif name == "issue":
|
|
126
|
+
body = {"type": "object", "properties": {"seq": _UINT}, "required": ["seq"], "additionalProperties": False}
|
|
127
|
+
elif name == "entry":
|
|
128
|
+
body = _entry()
|
|
129
|
+
elif name in NAMES:
|
|
130
|
+
body = _bodies()[name]
|
|
131
|
+
else:
|
|
132
|
+
raise InputError("BAD_ENTRY", "schema name must be one of: " + ", ".join(NAMES) + ", all")
|
|
133
|
+
return {"$schema": DIALECT, "title": f"protocolith {name}", "description": _DESCRIPTIONS[name], **body}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def all_schemas() -> dict[str, dict[str, Any]]:
|
|
137
|
+
return {name: schema_for(name) for name in NAMES}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Directory store: ledger file, advisory lock, durable append, tail repair."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import importlib
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
from collections.abc import Iterator
|
|
11
|
+
from contextlib import contextmanager
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from .errors import EnvError
|
|
15
|
+
|
|
16
|
+
ENV_STORE = "PROTOCOLITH_STORE"
|
|
17
|
+
DEFAULT_STORE = ".protocolith"
|
|
18
|
+
LEDGER_NAME = "ledger.jsonl"
|
|
19
|
+
LOCK_NAME = "lock"
|
|
20
|
+
|
|
21
|
+
_lock = importlib.import_module("protocolith.core._lock_" + ("windows" if sys.platform == "win32" else "posix"))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def resolve_store(path: str | os.PathLike[str] | None) -> Store:
|
|
25
|
+
"""Pick the store directory: explicit path, then ``PROTOCOLITH_STORE``, then ``.protocolith``."""
|
|
26
|
+
return Store(path or os.environ.get(ENV_STORE) or DEFAULT_STORE)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Store:
|
|
30
|
+
def __init__(self, path: str | os.PathLike[str], lock_timeout: float = 30.0) -> None:
|
|
31
|
+
self.path = Path(path)
|
|
32
|
+
self.lock_timeout = lock_timeout
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def ledger_path(self) -> Path:
|
|
36
|
+
return self.path / LEDGER_NAME
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def lock_path(self) -> Path:
|
|
40
|
+
return self.path / LOCK_NAME
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def cards_dir(self) -> Path:
|
|
44
|
+
return self.path / "cards"
|
|
45
|
+
|
|
46
|
+
def read(self, *, missing_ok: bool) -> bytes:
|
|
47
|
+
try:
|
|
48
|
+
return self.ledger_path.read_bytes()
|
|
49
|
+
except FileNotFoundError:
|
|
50
|
+
if missing_ok:
|
|
51
|
+
return b""
|
|
52
|
+
raise EnvError("NO_LEDGER", f"no ledger at {self.ledger_path}") from None
|
|
53
|
+
|
|
54
|
+
@contextmanager
|
|
55
|
+
def locked(self, *, write: bool) -> Iterator[None]:
|
|
56
|
+
"""Hold the exclusive lock. A reader that cannot open the lock file reads without it."""
|
|
57
|
+
if write:
|
|
58
|
+
self.path.mkdir(parents=True, exist_ok=True)
|
|
59
|
+
fd = self._open_lock(write)
|
|
60
|
+
if fd is None:
|
|
61
|
+
yield
|
|
62
|
+
return
|
|
63
|
+
try:
|
|
64
|
+
self._acquire(fd)
|
|
65
|
+
try:
|
|
66
|
+
yield
|
|
67
|
+
finally:
|
|
68
|
+
_lock.unlock(fd)
|
|
69
|
+
finally:
|
|
70
|
+
os.close(fd)
|
|
71
|
+
|
|
72
|
+
def _open_lock(self, write: bool) -> int | None:
|
|
73
|
+
try:
|
|
74
|
+
return os.open(self.lock_path, os.O_RDWR | (os.O_CREAT if write else 0), 0o644)
|
|
75
|
+
except OSError:
|
|
76
|
+
if write:
|
|
77
|
+
raise
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
def _acquire(self, fd: int) -> None:
|
|
81
|
+
deadline = time.monotonic() + self.lock_timeout
|
|
82
|
+
while not _lock.try_lock(fd):
|
|
83
|
+
if time.monotonic() >= deadline:
|
|
84
|
+
raise EnvError("LOCK_TIMEOUT", f"could not lock {self.lock_path} within {self.lock_timeout} s")
|
|
85
|
+
time.sleep(0.01)
|
|
86
|
+
|
|
87
|
+
def append_line(self, line: bytes) -> None:
|
|
88
|
+
"""Append one complete line: write, flush, fsync. Call with the lock held."""
|
|
89
|
+
created = not self.ledger_path.exists()
|
|
90
|
+
with open(self.ledger_path, "ab") as handle:
|
|
91
|
+
handle.write(line)
|
|
92
|
+
handle.flush()
|
|
93
|
+
os.fsync(handle.fileno())
|
|
94
|
+
if created:
|
|
95
|
+
_lock.fsync_dir(str(self.path))
|
|
96
|
+
|
|
97
|
+
def truncate(self, size: int) -> None:
|
|
98
|
+
with open(self.ledger_path, "r+b") as handle:
|
|
99
|
+
handle.truncate(size)
|
|
100
|
+
handle.flush()
|
|
101
|
+
os.fsync(handle.fileno())
|