moca-reader 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moca_reader/__init__.py +84 -0
- moca_reader/_js.py +291 -0
- moca_reader/attestations.py +271 -0
- moca_reader/audience.py +49 -0
- moca_reader/backends/__init__.py +1 -0
- moca_reader/backends/bm25.py +118 -0
- moca_reader/backends/dense.py +146 -0
- moca_reader/backends/hybrid.py +73 -0
- moca_reader/backends/lexical.py +112 -0
- moca_reader/backends/memory_store.py +122 -0
- moca_reader/citation.py +213 -0
- moca_reader/codes.py +110 -0
- moca_reader/conformance.py +249 -0
- moca_reader/content.py +312 -0
- moca_reader/diagnostics.py +45 -0
- moca_reader/digest.py +92 -0
- moca_reader/dsse.py +97 -0
- moca_reader/ed25519.py +83 -0
- moca_reader/embedders.py +224 -0
- moca_reader/evidence_text.py +187 -0
- moca_reader/frontmatter.py +151 -0
- moca_reader/library.py +176 -0
- moca_reader/manifest.py +186 -0
- moca_reader/metadata.py +58 -0
- moca_reader/profiles.py +5 -0
- moca_reader/py.typed +0 -0
- moca_reader/reader.py +468 -0
- moca_reader/relations.py +46 -0
- moca_reader/resolver.py +33 -0
- moca_reader/reviews.py +18 -0
- moca_reader/schemas/ag-ui-citations.schema.json +36 -0
- moca_reader/schemas/citation-record.schema.json +326 -0
- moca_reader/schemas/manifest.schema.json +171 -0
- moca_reader/schemas/node.schema.json +372 -0
- moca_reader/schemas/review-predicate.schema.json +42 -0
- moca_reader/schemas/search-hit.schema.json +62 -0
- moca_reader/schemas/sidecar-index.schema.json +155 -0
- moca_reader/schemas/structure.schema.json +93 -0
- moca_reader/schemas/trust-root.schema.json +130 -0
- moca_reader/schemas.py +44 -0
- moca_reader/search.py +346 -0
- moca_reader/sidecar.py +228 -0
- moca_reader/source.py +285 -0
- moca_reader/state.py +119 -0
- moca_reader/strategies/__init__.py +1 -0
- moca_reader/strategies/diversify.py +28 -0
- moca_reader/strategies/ontology_guided.py +63 -0
- moca_reader/structure.py +385 -0
- moca_reader/structure_ops.py +160 -0
- moca_reader/unicode.py +33 -0
- moca_reader-0.6.0.dist-info/METADATA +209 -0
- moca_reader-0.6.0.dist-info/RECORD +55 -0
- moca_reader-0.6.0.dist-info/WHEEL +4 -0
- moca_reader-0.6.0.dist-info/licenses/LICENSE +201 -0
- moca_reader-0.6.0.dist-info/licenses/NOTICE +15 -0
moca_reader/__init__.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""The MOCA Reader for Python: open, verify, search and cite MOCA packages.
|
|
2
|
+
|
|
3
|
+
from moca_reader import read_package, Library
|
|
4
|
+
|
|
5
|
+
pkg = read_package("support-kb.moca", trust_root="trust-root.json")
|
|
6
|
+
library = Library().add(pkg)
|
|
7
|
+
for record in library.search("refund window"):
|
|
8
|
+
print(record["node"]["title"], record["node"]["ref"])
|
|
9
|
+
|
|
10
|
+
Names follow the Reader interface (moca-spec spec/moca-reader-interface.md) in Python's style.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .attestations import TrustRoot
|
|
14
|
+
from .backends.dense import DenseBackend, model_mismatch, similarity
|
|
15
|
+
from .backends.hybrid import HybridBackend
|
|
16
|
+
from .backends.lexical import LexicalBackend
|
|
17
|
+
from .backends.memory_store import MemoryStoreBackend
|
|
18
|
+
from .citation import CitationRecord, citation_for, passes_default_policy, pick_representation
|
|
19
|
+
from .codes import CODES
|
|
20
|
+
from .content import ContentNode, Representation
|
|
21
|
+
from .diagnostics import Diagnostic
|
|
22
|
+
from .evidence_text import selector_matches
|
|
23
|
+
from .library import Library, Overlay
|
|
24
|
+
from .metadata import FLAT_FIELDS, flat_metadata
|
|
25
|
+
from .profiles import KNOWN_PROFILES, PROFILE_ONTOLOGY
|
|
26
|
+
from .reader import MOCA_VERSION, Member, PackageResult, Review, read_package
|
|
27
|
+
from .resolver import directory_resolver
|
|
28
|
+
from .search import Embedder, HookContext, Search, SearchBackend, SearchHooks, StoreScope
|
|
29
|
+
from .sidecar import SidecarBinding, bind_sidecar
|
|
30
|
+
from .source import DEFAULT_LIMITS, HostSource, SourceEntry
|
|
31
|
+
from .state import package_state
|
|
32
|
+
from .strategies.diversify import diversify
|
|
33
|
+
from .strategies.ontology_guided import ontology_guided
|
|
34
|
+
from .structure import structure_view
|
|
35
|
+
from .structure_ops import StructureIndex
|
|
36
|
+
|
|
37
|
+
__version__ = "0.6.0"
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"CODES",
|
|
41
|
+
"DEFAULT_LIMITS",
|
|
42
|
+
"FLAT_FIELDS",
|
|
43
|
+
"KNOWN_PROFILES",
|
|
44
|
+
"MOCA_VERSION",
|
|
45
|
+
"PROFILE_ONTOLOGY",
|
|
46
|
+
"CitationRecord",
|
|
47
|
+
"ContentNode",
|
|
48
|
+
"DenseBackend",
|
|
49
|
+
"Diagnostic",
|
|
50
|
+
"Embedder",
|
|
51
|
+
"HookContext",
|
|
52
|
+
"HostSource",
|
|
53
|
+
"HybridBackend",
|
|
54
|
+
"LexicalBackend",
|
|
55
|
+
"Library",
|
|
56
|
+
"Member",
|
|
57
|
+
"MemoryStoreBackend",
|
|
58
|
+
"Overlay",
|
|
59
|
+
"PackageResult",
|
|
60
|
+
"Representation",
|
|
61
|
+
"Review",
|
|
62
|
+
"Search",
|
|
63
|
+
"SearchBackend",
|
|
64
|
+
"SearchHooks",
|
|
65
|
+
"SidecarBinding",
|
|
66
|
+
"SourceEntry",
|
|
67
|
+
"StoreScope",
|
|
68
|
+
"StructureIndex",
|
|
69
|
+
"TrustRoot",
|
|
70
|
+
"bind_sidecar",
|
|
71
|
+
"citation_for",
|
|
72
|
+
"directory_resolver",
|
|
73
|
+
"diversify",
|
|
74
|
+
"flat_metadata",
|
|
75
|
+
"model_mismatch",
|
|
76
|
+
"ontology_guided",
|
|
77
|
+
"package_state",
|
|
78
|
+
"passes_default_policy",
|
|
79
|
+
"pick_representation",
|
|
80
|
+
"read_package",
|
|
81
|
+
"selector_matches",
|
|
82
|
+
"similarity",
|
|
83
|
+
"structure_view",
|
|
84
|
+
]
|
moca_reader/_js.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""Behaviours of the reference Reader (JavaScript) that every Reader matches.
|
|
2
|
+
|
|
3
|
+
These are the places where Python's defaults differ from JavaScript's, and so from what the
|
|
4
|
+
reference Reader and the conformance corpus expect.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
from datetime import UTC, datetime
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def utf16_key(text: str) -> bytes:
|
|
17
|
+
"""Sort key for JavaScript's string order: UTF-16 code units, not code points."""
|
|
18
|
+
return text.encode("utf-16-be", "surrogatepass")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def js_sorted(values: Any, key: Any = None) -> list[Any]:
|
|
22
|
+
"""``sorted`` in JavaScript's string order."""
|
|
23
|
+
if key is None:
|
|
24
|
+
return sorted(values, key=utf16_key)
|
|
25
|
+
return sorted(values, key=lambda v: utf16_key(key(v)))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
_FULL_DATE = re.compile(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\Z")
|
|
29
|
+
_DATE_TIME = re.compile(
|
|
30
|
+
r"^([0-9]{4})-([0-9]{2})-([0-9]{2})[Tt]([0-9]{2}):([0-9]{2}):([0-9]{2})(\.[0-9]+)?([Zz]|[+-][0-9]{2}:[0-9]{2})\Z"
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def parse_date(value: Any) -> datetime | None:
|
|
35
|
+
"""A date or date-time as a UTC instant, or None (package spec §4.4, attestations §6).
|
|
36
|
+
|
|
37
|
+
A full date (YYYY-MM-DD) is 00:00:00Z on that day. A date-time is RFC 3339 and carries an offset (Z or
|
|
38
|
+
±hh:mm); without one it does not parse, so no Reader guesses a time zone. Anything else does not parse.
|
|
39
|
+
"""
|
|
40
|
+
if not isinstance(value, str):
|
|
41
|
+
return None
|
|
42
|
+
try:
|
|
43
|
+
m = _FULL_DATE.match(value)
|
|
44
|
+
if m:
|
|
45
|
+
return datetime(int(m[1]), int(m[2]), int(m[3]), tzinfo=UTC)
|
|
46
|
+
if not _DATE_TIME.match(value):
|
|
47
|
+
return None
|
|
48
|
+
parsed = datetime.fromisoformat(value.upper().replace("Z", "+00:00"))
|
|
49
|
+
except ValueError:
|
|
50
|
+
return None
|
|
51
|
+
return parsed.astimezone(UTC)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def effective_window(
|
|
55
|
+
package_from: Any, package_until: Any, node_from: Any, node_until: Any
|
|
56
|
+
) -> tuple[Any, Any, datetime | None, datetime | None]:
|
|
57
|
+
"""The node's validity window (package spec §4.4): the intersection of the package's window and its own.
|
|
58
|
+
|
|
59
|
+
A bound that does not parse, or a window whose end is not after its start, is ignored. Returns the bounds as
|
|
60
|
+
written (or None) and as instants (or None).
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def window(start: Any, end: Any) -> tuple[datetime | None, datetime | None]:
|
|
64
|
+
f, u = parse_date(start), parse_date(end)
|
|
65
|
+
if f is not None and u is not None and u <= f:
|
|
66
|
+
return None, None
|
|
67
|
+
return f, u
|
|
68
|
+
|
|
69
|
+
pf, pu = window(package_from, package_until)
|
|
70
|
+
nf, nu = window(node_from, node_until)
|
|
71
|
+
from_pick = (node_from, nf) if nf is not None and (pf is None or nf >= pf) else (package_from, pf)
|
|
72
|
+
until_pick = (node_until, nu) if nu is not None and (pu is None or nu <= pu) else (package_until, pu)
|
|
73
|
+
return (
|
|
74
|
+
from_pick[0] if from_pick[1] is not None else None,
|
|
75
|
+
until_pick[0] if until_pick[1] is not None else None,
|
|
76
|
+
from_pick[1],
|
|
77
|
+
until_pick[1],
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
_BASE64_JUNK = re.compile(r"[^A-Za-z0-9+/\-_]")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def b64decode(text: str) -> bytes:
|
|
85
|
+
"""Base64 decoded as Node's ``Buffer.from(text, 'base64')`` does: leniently, never raising."""
|
|
86
|
+
clean = _BASE64_JUNK.sub("", text.split("=", 1)[0]).replace("-", "+").replace("_", "/")
|
|
87
|
+
clean = clean[: len(clean) - (len(clean) % 4 == 1)]
|
|
88
|
+
try:
|
|
89
|
+
return base64.b64decode(clean + "=" * (-len(clean) % 4))
|
|
90
|
+
except ValueError: # pragma: no cover - the input is already clean
|
|
91
|
+
return b""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def math_round(value: float) -> float:
|
|
95
|
+
"""JavaScript's ``Math.round``: the nearest integer, and halves go towards +infinity (-2.5 gives -2)."""
|
|
96
|
+
if math.isnan(value) or math.isinf(value):
|
|
97
|
+
return value
|
|
98
|
+
floor = math.floor(value)
|
|
99
|
+
return float(floor + 1 if value - floor >= 0.5 else floor)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def round_scaled(value: float, scale: float) -> float:
|
|
103
|
+
"""``Math.round(value * scale) / scale``, as the reference Reader rounds scores."""
|
|
104
|
+
return math_round(value * scale) / scale
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def truthy(value: Any) -> bool:
|
|
108
|
+
"""JavaScript truthiness: empty lists and objects are truthy; 0, NaN, "" and null are not."""
|
|
109
|
+
if value is None or value is False:
|
|
110
|
+
return False
|
|
111
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
112
|
+
return value != 0 and not math.isnan(value)
|
|
113
|
+
if isinstance(value, str):
|
|
114
|
+
return value != ""
|
|
115
|
+
return True
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def get(obj: Any, key: str) -> Any:
|
|
119
|
+
"""``obj?.key`` on parsed JSON or YAML: None unless ``obj`` is an object with that key."""
|
|
120
|
+
return obj.get(key) if isinstance(obj, dict) else None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def is_object(value: Any) -> bool:
|
|
124
|
+
return isinstance(value, dict)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _no_constants(name: str) -> Any:
|
|
128
|
+
raise ValueError(f"{name} is not valid JSON")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def json_loads(text: str, unique_keys: bool = False) -> Any:
|
|
132
|
+
"""``JSON.parse``: NaN and Infinity are not JSON. With ``unique_keys``, a repeated member name raises ValueError
|
|
133
|
+
(I-JSON, RFC 7493)."""
|
|
134
|
+
import json
|
|
135
|
+
|
|
136
|
+
def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
137
|
+
out: dict[str, Any] = {}
|
|
138
|
+
for key, value in items:
|
|
139
|
+
if key in out:
|
|
140
|
+
raise ValueError(f'member "{key}" appears more than once')
|
|
141
|
+
out[key] = value
|
|
142
|
+
return out
|
|
143
|
+
|
|
144
|
+
if unique_keys:
|
|
145
|
+
return json.loads(text, parse_constant=_no_constants, object_pairs_hook=pairs)
|
|
146
|
+
return json.loads(text, parse_constant=_no_constants)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def number_to_string(value: float | int) -> str:
|
|
150
|
+
"""ECMAScript ``Number.prototype.toString`` for finite numbers."""
|
|
151
|
+
if isinstance(value, bool):
|
|
152
|
+
return "true" if value else "false"
|
|
153
|
+
if isinstance(value, int):
|
|
154
|
+
return str(value)
|
|
155
|
+
if math.isnan(value):
|
|
156
|
+
return "NaN"
|
|
157
|
+
if math.isinf(value):
|
|
158
|
+
return "Infinity" if value > 0 else "-Infinity"
|
|
159
|
+
if value == 0:
|
|
160
|
+
return "0"
|
|
161
|
+
sign = "-" if value < 0 else ""
|
|
162
|
+
digits, _, exp = f"{abs(value):.17e}".partition("e")
|
|
163
|
+
# The shortest round-tripping digits, as repr gives them.
|
|
164
|
+
r = repr(abs(value))
|
|
165
|
+
mantissa, _, e = r.partition("e")
|
|
166
|
+
exponent = int(e) if e else 0
|
|
167
|
+
if "." in mantissa:
|
|
168
|
+
whole, frac = mantissa.split(".")
|
|
169
|
+
else:
|
|
170
|
+
whole, frac = mantissa, ""
|
|
171
|
+
if frac == "0":
|
|
172
|
+
frac = ""
|
|
173
|
+
s_digits = (whole + frac).lstrip("0")
|
|
174
|
+
n = exponent + len(whole) - (len(whole + frac) - len((whole + frac).lstrip("0")))
|
|
175
|
+
s_digits = s_digits.rstrip("0") or "0"
|
|
176
|
+
k = len(s_digits)
|
|
177
|
+
del digits, exp
|
|
178
|
+
if k <= n <= 21:
|
|
179
|
+
out = s_digits + "0" * (n - k)
|
|
180
|
+
elif 0 < n <= 21:
|
|
181
|
+
out = s_digits[:n] + "." + s_digits[n:]
|
|
182
|
+
elif -6 < n <= 0:
|
|
183
|
+
out = "0." + "0" * (-n) + s_digits
|
|
184
|
+
else:
|
|
185
|
+
e_val = n - 1
|
|
186
|
+
e_str = f"e{'+' if e_val >= 0 else '-'}{abs(e_val)}"
|
|
187
|
+
out = s_digits[0] + ("." + s_digits[1:] if k > 1 else "") + e_str
|
|
188
|
+
return sign + out
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def stringify(value: Any) -> str:
|
|
192
|
+
"""``JSON.stringify`` with no spacing, for comparing JSON documents as the reference Reader does."""
|
|
193
|
+
import json
|
|
194
|
+
|
|
195
|
+
if value is None or isinstance(value, (bool, str)):
|
|
196
|
+
return json.dumps(value, ensure_ascii=False)
|
|
197
|
+
if isinstance(value, (int, float)):
|
|
198
|
+
return (
|
|
199
|
+
"null" if isinstance(value, float) and (math.isnan(value) or math.isinf(value)) else number_to_string(value)
|
|
200
|
+
)
|
|
201
|
+
if isinstance(value, list):
|
|
202
|
+
return "[" + ",".join(stringify(v) for v in value) + "]"
|
|
203
|
+
if isinstance(value, dict):
|
|
204
|
+
return (
|
|
205
|
+
"{" + ",".join(f"{json.dumps(str(k), ensure_ascii=False)}:{stringify(v)}" for k, v in value.items()) + "}"
|
|
206
|
+
)
|
|
207
|
+
return "null"
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def decode_uri_component(text: str) -> str:
|
|
211
|
+
"""``decodeURIComponent``: raises ValueError on a malformed escape, as JavaScript throws URIError."""
|
|
212
|
+
from urllib.parse import unquote_to_bytes
|
|
213
|
+
|
|
214
|
+
if re.search(r"%(?![0-9A-Fa-f]{2})", text):
|
|
215
|
+
raise ValueError("URI malformed")
|
|
216
|
+
try:
|
|
217
|
+
return unquote_to_bytes(text).decode("utf-8")
|
|
218
|
+
except UnicodeDecodeError as err:
|
|
219
|
+
raise ValueError("URI malformed") from err
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def posix_normalize(path: str) -> str:
|
|
223
|
+
"""Node's ``path.posix.normalize``: keeps a trailing slash, collapses repeated slashes, '' gives '.'."""
|
|
224
|
+
if path == "":
|
|
225
|
+
return "."
|
|
226
|
+
absolute = path.startswith("/")
|
|
227
|
+
trailing = path.endswith("/")
|
|
228
|
+
parts: list[str] = []
|
|
229
|
+
for seg in path.split("/"):
|
|
230
|
+
if seg in ("", "."):
|
|
231
|
+
continue
|
|
232
|
+
if seg == "..":
|
|
233
|
+
if parts and parts[-1] != "..":
|
|
234
|
+
parts.pop()
|
|
235
|
+
elif not absolute:
|
|
236
|
+
parts.append("..")
|
|
237
|
+
continue
|
|
238
|
+
parts.append(seg)
|
|
239
|
+
out = "/".join(parts)
|
|
240
|
+
if not out and not absolute:
|
|
241
|
+
out = "."
|
|
242
|
+
if out and trailing:
|
|
243
|
+
out += "/"
|
|
244
|
+
return ("/" + out) if absolute else out
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def posix_dirname(path: str) -> str:
|
|
248
|
+
import posixpath
|
|
249
|
+
|
|
250
|
+
d = posixpath.dirname(path.rstrip("/") if path != "/" else path)
|
|
251
|
+
return d or "."
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def posix_join(*parts: str) -> str:
|
|
255
|
+
joined = "/".join(p for p in parts if p != "")
|
|
256
|
+
return posix_normalize(joined) if joined else "."
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def js_string(value: Any, present: bool = True) -> str:
|
|
260
|
+
"""``String(value)``; ``present=False`` stands for JavaScript's ``undefined``."""
|
|
261
|
+
if not present:
|
|
262
|
+
return "undefined"
|
|
263
|
+
if value is None:
|
|
264
|
+
return "null"
|
|
265
|
+
if isinstance(value, str):
|
|
266
|
+
return value
|
|
267
|
+
if isinstance(value, (bool, int, float)):
|
|
268
|
+
return number_to_string(value)
|
|
269
|
+
if isinstance(value, list):
|
|
270
|
+
return ",".join("" if v is None else js_string(v) for v in value)
|
|
271
|
+
return "[object Object]"
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def timestamp(value: Any) -> float:
|
|
275
|
+
"""``Date.parse(value)`` in milliseconds, or NaN, so comparisons with it are false as in JavaScript."""
|
|
276
|
+
parsed = parse_date(value if isinstance(value, str) else js_string(value))
|
|
277
|
+
return parsed.timestamp() * 1000 if parsed is not None else math.nan
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
# Package spec §2: white space is ECMAScript's. Python's str.strip() and str.split() use another set.
|
|
281
|
+
JS_WHITESPACE = "\t\n\v\f\r \u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def js_trim(text: str) -> str:
|
|
285
|
+
"""String.prototype.trim."""
|
|
286
|
+
return text.strip(JS_WHITESPACE)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def js_split_ws(text: str) -> list[str]:
|
|
290
|
+
"""``text.trim().split(/\\s+/)`` without empty parts, as ``str.split()`` but with ECMAScript white space."""
|
|
291
|
+
return [p for p in re.split(f"[{JS_WHITESPACE}]+", text) if p]
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Detached attestations (moca-attestations.md).
|
|
2
|
+
|
|
3
|
+
A package carries its attestations under attestations/, which the digest excludes. Each file is a DSSE
|
|
4
|
+
envelope (*.dsse.json) or a Sigstore bundle (*.sigstore.json) wrapping an in-toto v1 Statement with one of
|
|
5
|
+
two MOCA predicate types: package/v1 ("this exact package was published by this signer") and review/v1
|
|
6
|
+
("this reviewer checked these exact node files").
|
|
7
|
+
|
|
8
|
+
This Reader verifies DSSE envelopes, and Sigstore bundles when the ``sigstore`` extra is installed
|
|
9
|
+
(``moca-reader[sigstore]``): offline only, against the host's ``sigstore.trustedRoot`` (attestations §6). Without
|
|
10
|
+
the extra or a trusted root, a Sigstore bundle is unverifiable (A003).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from datetime import UTC, datetime
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from ._js import get, json_loads, parse_date
|
|
22
|
+
from .dsse import decode_base64, verify_envelope
|
|
23
|
+
from .schemas import validate_against
|
|
24
|
+
from .unicode import normalize
|
|
25
|
+
|
|
26
|
+
STATEMENT_TYPE = "https://in-toto.io/Statement/v1"
|
|
27
|
+
PAYLOAD_TYPE = "application/vnd.in-toto+json"
|
|
28
|
+
PREDICATE_PACKAGE = "https://w3id.org/moca/attestation/package/v1"
|
|
29
|
+
PREDICATE_REVIEW = "https://w3id.org/moca/attestation/review/v1"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _hex_of(digest: str) -> str:
|
|
33
|
+
return digest.removeprefix("sha256:")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def package_statement(package_id: str, version: str, digest: str) -> dict[str, Any]:
|
|
37
|
+
return {
|
|
38
|
+
"_type": STATEMENT_TYPE,
|
|
39
|
+
"subject": [{"name": f"{package_id}@{version}", "digest": {"sha256": _hex_of(digest)}}],
|
|
40
|
+
"predicateType": PREDICATE_PACKAGE,
|
|
41
|
+
"predicate": {},
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class TrustRoot:
|
|
46
|
+
"""A host trust root (moca-attestations.md §6): whose signatures to accept, and for what."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, doc: dict[str, Any]) -> None:
|
|
49
|
+
errors = validate_against("trustRoot", doc)
|
|
50
|
+
if errors:
|
|
51
|
+
raise ValueError(f"invalid trust root: {'; '.join(errors)}")
|
|
52
|
+
self.doc = doc
|
|
53
|
+
# Attestations §6: keyids are unique.
|
|
54
|
+
self._keys: dict[str, dict[str, Any]] = {}
|
|
55
|
+
for k in doc.get("keys", []):
|
|
56
|
+
if k["keyid"] in self._keys:
|
|
57
|
+
raise ValueError(f'invalid trust root: keyid "{k["keyid"]}" appears more than once')
|
|
58
|
+
self._keys[k["keyid"]] = k
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def load(source: str | os.PathLike[str] | dict[str, Any] | TrustRoot) -> TrustRoot:
|
|
62
|
+
if isinstance(source, TrustRoot):
|
|
63
|
+
return source
|
|
64
|
+
if isinstance(source, dict):
|
|
65
|
+
return TrustRoot(source)
|
|
66
|
+
with open(source, encoding="utf-8") as f:
|
|
67
|
+
return TrustRoot(json.load(f))
|
|
68
|
+
|
|
69
|
+
@staticmethod
|
|
70
|
+
def _allows(entry: dict[str, Any], role: str) -> bool:
|
|
71
|
+
# Attestations §6: `roles` is required; an empty list trusts the entry for nothing.
|
|
72
|
+
roles = entry.get("roles")
|
|
73
|
+
return isinstance(roles, list) and role in roles
|
|
74
|
+
|
|
75
|
+
def key_entry(self, keyid: str) -> dict[str, Any] | None:
|
|
76
|
+
"""The trust-root entry for a keyid, or None."""
|
|
77
|
+
return self._keys.get(keyid)
|
|
78
|
+
|
|
79
|
+
def key(self, keyid: str, role: str, now: datetime | None = None) -> tuple[str, str] | None:
|
|
80
|
+
"""(public key PEM, identity), or None when the key is unknown, not trusted for the role, or expired."""
|
|
81
|
+
entry = self._keys.get(keyid)
|
|
82
|
+
if entry is None or not self._allows(entry, role):
|
|
83
|
+
return None
|
|
84
|
+
expires = parse_date(entry.get("expires"))
|
|
85
|
+
# Attestations §6: a key is not trusted from the instant it expires; one whose expiry does not parse is
|
|
86
|
+
# not trusted at all.
|
|
87
|
+
if entry.get("expires") is not None and (expires is None or (now or datetime.now(UTC)) >= expires):
|
|
88
|
+
return None
|
|
89
|
+
return entry["publicKey"], entry.get("identity") or keyid
|
|
90
|
+
|
|
91
|
+
def sigstore_identities(self, role: str) -> list[dict[str, Any]]:
|
|
92
|
+
identities = get(self.doc.get("sigstore"), "identities") or []
|
|
93
|
+
return [i for i in identities if self._allows(i, role)]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class ParsedAttestation:
|
|
98
|
+
format: str
|
|
99
|
+
container: Any
|
|
100
|
+
statement: dict[str, Any]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def parse_attestation(path: str, data: bytes) -> ParsedAttestation | str:
|
|
104
|
+
"""The envelope and statement of an attestation file, or the reason it is malformed."""
|
|
105
|
+
fmt = "sigstore" if path.endswith(".sigstore.json") else "dsse" if path.endswith(".dsse.json") else None
|
|
106
|
+
if fmt is None:
|
|
107
|
+
return "attestation files must end in .dsse.json or .sigstore.json"
|
|
108
|
+
try:
|
|
109
|
+
container = json_loads(data.decode("utf-8", errors="replace"))
|
|
110
|
+
except ValueError:
|
|
111
|
+
return "not valid JSON"
|
|
112
|
+
envelope = container if fmt == "dsse" else get(container, "dsseEnvelope")
|
|
113
|
+
if not isinstance(get(envelope, "payload"), str) or not isinstance(get(envelope, "payloadType"), str):
|
|
114
|
+
return "not a DSSE envelope" if fmt == "dsse" else "Sigstore bundle has no dsseEnvelope"
|
|
115
|
+
if envelope["payloadType"] != PAYLOAD_TYPE:
|
|
116
|
+
return f"unexpected payloadType {envelope['payloadType']}"
|
|
117
|
+
payload = decode_base64(envelope["payload"])
|
|
118
|
+
if payload is None:
|
|
119
|
+
return "payload is not base64 or base64url"
|
|
120
|
+
try:
|
|
121
|
+
statement = json_loads(payload.decode("utf-8", errors="replace"), unique_keys=True)
|
|
122
|
+
except ValueError:
|
|
123
|
+
return "payload is not a JSON in-toto statement"
|
|
124
|
+
if (
|
|
125
|
+
get(statement, "_type") != STATEMENT_TYPE
|
|
126
|
+
or not isinstance(statement.get("subject"), list)
|
|
127
|
+
or not isinstance(statement.get("predicateType"), str)
|
|
128
|
+
):
|
|
129
|
+
return "payload is not an in-toto v1 Statement"
|
|
130
|
+
return ParsedAttestation(fmt, container, statement)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass
|
|
134
|
+
class SignatureResult:
|
|
135
|
+
outcome: str # valid | invalid | unverifiable | indeterminate
|
|
136
|
+
signer: str | None = None
|
|
137
|
+
reason: str | None = None
|
|
138
|
+
ids: list[str] = field(default_factory=list)
|
|
139
|
+
reviewers: list[str] | None = None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def verify_signature(
|
|
143
|
+
parsed: ParsedAttestation, trust_root: TrustRoot | None, role: str, now: datetime | None = None
|
|
144
|
+
) -> SignatureResult:
|
|
145
|
+
"""Verifies the signature against the host trust root; not what the statement says."""
|
|
146
|
+
if trust_root is None:
|
|
147
|
+
return SignatureResult("unverifiable", reason="no trust root was supplied")
|
|
148
|
+
if parsed.format == "dsse":
|
|
149
|
+
signer: list[str] = []
|
|
150
|
+
|
|
151
|
+
def resolve(keyid: str) -> str | None:
|
|
152
|
+
found = trust_root.key(keyid, role, now)
|
|
153
|
+
if found:
|
|
154
|
+
signer.append(found[1])
|
|
155
|
+
return found[0] if found else None
|
|
156
|
+
|
|
157
|
+
result = verify_envelope(parsed.container, resolve)
|
|
158
|
+
if not result.ok:
|
|
159
|
+
return SignatureResult("invalid", reason=result.reason)
|
|
160
|
+
found = trust_root.key(result.keyid or "", role, now)
|
|
161
|
+
entry = trust_root.key_entry(result.keyid or "")
|
|
162
|
+
ids = [i for i in (entry or {}).get("ids", []) if isinstance(i, str)]
|
|
163
|
+
listed = (entry or {}).get("reviewers")
|
|
164
|
+
reviewers = [r for r in listed if isinstance(r, str)] if isinstance(listed, list) else None
|
|
165
|
+
return SignatureResult(
|
|
166
|
+
"valid",
|
|
167
|
+
signer=found[1] if found else (signer[-1] if signer else result.keyid),
|
|
168
|
+
ids=ids,
|
|
169
|
+
reviewers=reviewers,
|
|
170
|
+
)
|
|
171
|
+
return _verify_sigstore(parsed, trust_root, role)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _verify_sigstore(parsed: ParsedAttestation, trust_root: TrustRoot, role: str) -> SignatureResult:
|
|
175
|
+
# Attestations §7, step 6: unverifiable only when the trust root lists no Sigstore identities at all, or there
|
|
176
|
+
# is no trust material; identities that exist but are not trusted for this role make it invalid.
|
|
177
|
+
sigstore_doc = trust_root.doc.get("sigstore")
|
|
178
|
+
if not (get(sigstore_doc, "identities") or []):
|
|
179
|
+
return SignatureResult("unverifiable", reason="the trust root lists no Sigstore identities")
|
|
180
|
+
trusted_root = get(sigstore_doc, "trustedRoot")
|
|
181
|
+
if not isinstance(trusted_root, dict):
|
|
182
|
+
return SignatureResult(
|
|
183
|
+
"unverifiable",
|
|
184
|
+
reason="no sigstore.trustedRoot: this Reader verifies Sigstore offline, against the host's trusted root",
|
|
185
|
+
)
|
|
186
|
+
try:
|
|
187
|
+
from sigstore.errors import Error as SigstoreError
|
|
188
|
+
from sigstore.models import Bundle
|
|
189
|
+
from sigstore.models import TrustedRoot as SigstoreTrustedRoot
|
|
190
|
+
from sigstore.verify import Verifier, policy
|
|
191
|
+
from sigstore_models.trustroot import v1 as trustroot_v1
|
|
192
|
+
except ImportError:
|
|
193
|
+
return SignatureResult(
|
|
194
|
+
"unverifiable", reason="Sigstore verification needs the sigstore extra: pip install moca-reader[sigstore]"
|
|
195
|
+
)
|
|
196
|
+
identities = trust_root.sigstore_identities(role)
|
|
197
|
+
if not identities:
|
|
198
|
+
return SignatureResult(
|
|
199
|
+
"invalid", reason=f"no Sigstore identity in the trust root is trusted for the {role} role"
|
|
200
|
+
)
|
|
201
|
+
try:
|
|
202
|
+
verifier = Verifier(
|
|
203
|
+
trusted_root=SigstoreTrustedRoot(trustroot_v1.TrustedRoot.from_json(json.dumps(trusted_root).encode()))
|
|
204
|
+
)
|
|
205
|
+
except Exception as err: # any failure here: the trust material is unusable
|
|
206
|
+
return SignatureResult("unverifiable", reason=f"sigstore.trustedRoot cannot be used: {err}")
|
|
207
|
+
try:
|
|
208
|
+
bundle = Bundle.from_json(json.dumps(parsed.container))
|
|
209
|
+
except Exception as err: # a bundle the library cannot read signs nothing
|
|
210
|
+
return SignatureResult("invalid", reason=f"not a valid Sigstore bundle: {err}")
|
|
211
|
+
last = "verification failed"
|
|
212
|
+
for identity in identities:
|
|
213
|
+
issuer, subject = identity.get("issuer"), identity.get("subject")
|
|
214
|
+
if not isinstance(issuer, str) or not isinstance(subject, str):
|
|
215
|
+
continue
|
|
216
|
+
try:
|
|
217
|
+
# Attestations §6: issuer and subject alternative name match exactly; no patterns.
|
|
218
|
+
verifier.verify_dsse(bundle, policy.Identity(identity=subject, issuer=issuer))
|
|
219
|
+
except (SigstoreError, ValueError) as err:
|
|
220
|
+
last = str(err)
|
|
221
|
+
continue
|
|
222
|
+
ids = [i for i in identity.get("ids", []) if isinstance(i, str)]
|
|
223
|
+
listed = identity.get("reviewers")
|
|
224
|
+
reviewers = [r for r in listed if isinstance(r, str)] if isinstance(listed, list) else None
|
|
225
|
+
return SignatureResult("valid", signer=f"{issuer} {subject}", ids=ids, reviewers=reviewers)
|
|
226
|
+
return SignatureResult("invalid", reason=last)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def check_package_statement(statement: dict[str, Any], package_id: Any, version: Any, digest: str) -> str | None:
|
|
230
|
+
"""The reason the statement does not attest this package, or None."""
|
|
231
|
+
if statement.get("predicateType") != PREDICATE_PACKAGE:
|
|
232
|
+
return "not a package attestation"
|
|
233
|
+
subjects = statement["subject"]
|
|
234
|
+
subject = subjects[0] if subjects else None
|
|
235
|
+
name = get(subject, "name")
|
|
236
|
+
if len(subjects) != 1 or name != f"{package_id}@{version}":
|
|
237
|
+
return f'subject "{name}" does not name {package_id}@{version}'
|
|
238
|
+
if get(get(subject, "digest"), "sha256") != _hex_of(digest):
|
|
239
|
+
return "subject digest does not match the package digest"
|
|
240
|
+
return None
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@dataclass
|
|
244
|
+
class ReviewCheck:
|
|
245
|
+
error: str | None = None
|
|
246
|
+
current: list[str] = field(default_factory=list)
|
|
247
|
+
outdated: list[str] = field(default_factory=list)
|
|
248
|
+
predicate: dict[str, Any] | None = None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def check_review_statement(statement: dict[str, Any], package_id: Any, file_digests: dict[str, str]) -> ReviewCheck:
|
|
252
|
+
predicate: Any = statement.get("predicate")
|
|
253
|
+
errors = validate_against("reviewPredicate", predicate)
|
|
254
|
+
if errors:
|
|
255
|
+
return ReviewCheck(f"review predicate is invalid: {'; '.join(errors)}")
|
|
256
|
+
if predicate["package"] != package_id:
|
|
257
|
+
return ReviewCheck(f"review names package {predicate['package']}, not {package_id}")
|
|
258
|
+
check = ReviewCheck(predicate=predicate)
|
|
259
|
+
for s in statement["subject"]:
|
|
260
|
+
name, sha = get(s, "name"), get(get(s, "digest"), "sha256")
|
|
261
|
+
if not isinstance(name, str) or not isinstance(sha, str):
|
|
262
|
+
return ReviewCheck("review subject entries need a name and a sha256 digest")
|
|
263
|
+
(check.current if file_digests.get(normalize("NFC", name)) == sha else check.outdated).append(name)
|
|
264
|
+
return check
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def scope_covers(scopes: list[str], package_id: Any) -> bool:
|
|
268
|
+
"""Package spec §7.2: whether a signer's ``ids`` cover a package id (exact, or a prefix ending in /, : or #)."""
|
|
269
|
+
if not isinstance(package_id, str):
|
|
270
|
+
return False
|
|
271
|
+
return any(s == package_id or (s[-1:] in ("/", ":", "#") and package_id.startswith(s)) for s in scopes)
|