moca-reader 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moca_reader/__init__.py +84 -0
- moca_reader/_js.py +291 -0
- moca_reader/attestations.py +271 -0
- moca_reader/audience.py +49 -0
- moca_reader/backends/__init__.py +1 -0
- moca_reader/backends/bm25.py +118 -0
- moca_reader/backends/dense.py +146 -0
- moca_reader/backends/hybrid.py +73 -0
- moca_reader/backends/lexical.py +112 -0
- moca_reader/backends/memory_store.py +122 -0
- moca_reader/citation.py +213 -0
- moca_reader/codes.py +110 -0
- moca_reader/conformance.py +249 -0
- moca_reader/content.py +312 -0
- moca_reader/diagnostics.py +45 -0
- moca_reader/digest.py +92 -0
- moca_reader/dsse.py +97 -0
- moca_reader/ed25519.py +83 -0
- moca_reader/embedders.py +224 -0
- moca_reader/evidence_text.py +187 -0
- moca_reader/frontmatter.py +151 -0
- moca_reader/library.py +176 -0
- moca_reader/manifest.py +186 -0
- moca_reader/metadata.py +58 -0
- moca_reader/profiles.py +5 -0
- moca_reader/py.typed +0 -0
- moca_reader/reader.py +468 -0
- moca_reader/relations.py +46 -0
- moca_reader/resolver.py +33 -0
- moca_reader/reviews.py +18 -0
- moca_reader/schemas/ag-ui-citations.schema.json +36 -0
- moca_reader/schemas/citation-record.schema.json +326 -0
- moca_reader/schemas/manifest.schema.json +171 -0
- moca_reader/schemas/node.schema.json +372 -0
- moca_reader/schemas/review-predicate.schema.json +42 -0
- moca_reader/schemas/search-hit.schema.json +62 -0
- moca_reader/schemas/sidecar-index.schema.json +155 -0
- moca_reader/schemas/structure.schema.json +93 -0
- moca_reader/schemas/trust-root.schema.json +130 -0
- moca_reader/schemas.py +44 -0
- moca_reader/search.py +346 -0
- moca_reader/sidecar.py +228 -0
- moca_reader/source.py +285 -0
- moca_reader/state.py +119 -0
- moca_reader/strategies/__init__.py +1 -0
- moca_reader/strategies/diversify.py +28 -0
- moca_reader/strategies/ontology_guided.py +63 -0
- moca_reader/structure.py +385 -0
- moca_reader/structure_ops.py +160 -0
- moca_reader/unicode.py +33 -0
- moca_reader-0.6.0.dist-info/METADATA +209 -0
- moca_reader-0.6.0.dist-info/RECORD +55 -0
- moca_reader-0.6.0.dist-info/WHEEL +4 -0
- moca_reader-0.6.0.dist-info/licenses/LICENSE +201 -0
- moca_reader-0.6.0.dist-info/licenses/NOTICE +15 -0
moca_reader/content.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Content traversal: content/ is an Open Knowledge Format (OKF) v0.2 bundle (package spec §5, contract §5)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import posixpath
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from ._js import decode_uri_component, get, js_sorted, js_trim, posix_dirname, posix_join, truthy, utf16_key
|
|
11
|
+
from .diagnostics import Diagnostics
|
|
12
|
+
from .evidence_text import check_selector, normalised_text, source_kind
|
|
13
|
+
from .frontmatter import split_frontmatter
|
|
14
|
+
from .schemas import validate_against
|
|
15
|
+
from .unicode import normalize
|
|
16
|
+
|
|
17
|
+
CONTENT_DIR = "content"
|
|
18
|
+
_RESERVED = {"index.md", "log.md"}
|
|
19
|
+
_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
|
|
20
|
+
_LINK = re.compile(
|
|
21
|
+
r"!?\[[^\]]*\]\([\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]*(<[^>]+>|[^)\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+)(?:[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+(?:\"[^\"]*\"|'[^']*'))?[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]*\)"
|
|
22
|
+
)
|
|
23
|
+
_PATH_REF = re.compile(r"^(\.{1,2}/|/)")
|
|
24
|
+
_EVIDENCE_DIRS = ("sources/", "media/")
|
|
25
|
+
_LOCALE = re.compile(r"^(.*)\.([A-Za-z]{2,3}(?:-[A-Za-z0-9]{1,8})*)\.md\Z", re.IGNORECASE | re.ASCII | re.DOTALL)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class EvidenceCheck:
|
|
30
|
+
local: bool
|
|
31
|
+
matched: bool | None = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class Representation:
|
|
36
|
+
locale: str | None # None = the default language
|
|
37
|
+
file: str # package-relative path
|
|
38
|
+
frontmatter: dict[str, Any]
|
|
39
|
+
body: str
|
|
40
|
+
body_offset: int # UTF-8 byte offset of the body in the file
|
|
41
|
+
sha256: str | None
|
|
42
|
+
evidence_checks: list[EvidenceCheck] = field(default_factory=list)
|
|
43
|
+
concepts: list[str] | None = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class ContentNode:
|
|
48
|
+
path: str # relative to content/: the default-language file name
|
|
49
|
+
representations: list[Representation]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def read_content(
|
|
53
|
+
manifest: dict[str, Any],
|
|
54
|
+
files: dict[str, str],
|
|
55
|
+
data: dict[str, bytes],
|
|
56
|
+
file_digests: dict[str, str],
|
|
57
|
+
diagnostics: Diagnostics,
|
|
58
|
+
limits: dict[str, int],
|
|
59
|
+
) -> list[ContentNode]:
|
|
60
|
+
source_text: dict[str, str] = {}
|
|
61
|
+
# Contract §3: a per-package budget of evidence checks and of source bytes scanned.
|
|
62
|
+
budget: dict[str, Any] = {"checks": 0, "bytes": 0, "prepared": {}}
|
|
63
|
+
declared = manifest.get("locales")
|
|
64
|
+
locales = {str(loc).lower(): loc for loc in (declared if isinstance(declared, list) else [])}
|
|
65
|
+
by_key: dict[str, ContentNode] = {}
|
|
66
|
+
md_files = js_sorted(p for p in files if p.startswith(f"{CONTENT_DIR}/") and p.lower().endswith(".md"))
|
|
67
|
+
|
|
68
|
+
for file in md_files:
|
|
69
|
+
rel = file[len(CONTENT_DIR) + 1 :]
|
|
70
|
+
base = posixpath.basename(rel)
|
|
71
|
+
text = data[file].decode("utf-8", errors="replace")
|
|
72
|
+
|
|
73
|
+
if base.lower() in _RESERVED:
|
|
74
|
+
_check_reserved(file, rel, base.lower(), text, diagnostics, limits)
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
key, locale = _locale_of(rel, locales)
|
|
78
|
+
split = split_frontmatter(text, limits["maxFrontmatterBytes"])
|
|
79
|
+
if not split.present:
|
|
80
|
+
diagnostics.add(
|
|
81
|
+
"C001_FRONTMATTER_MISSING", "concept documents must open with a YAML frontmatter block", file
|
|
82
|
+
)
|
|
83
|
+
continue
|
|
84
|
+
if split.error:
|
|
85
|
+
diagnostics.add("C002_FRONTMATTER_INVALID", split.error, file, split.line)
|
|
86
|
+
continue
|
|
87
|
+
fm: dict[str, Any] = split.data
|
|
88
|
+
if not isinstance(fm.get("type"), str) or js_trim(fm["type"]) == "":
|
|
89
|
+
diagnostics.add("C003_TYPE_MISSING", 'frontmatter needs a non-empty "type"', file)
|
|
90
|
+
continue
|
|
91
|
+
for err in validate_against("node", fm):
|
|
92
|
+
diagnostics.add("C005_FIELD_INVALID", err, file)
|
|
93
|
+
_check_evidence(file, fm, diagnostics)
|
|
94
|
+
_check_window(file, fm, diagnostics)
|
|
95
|
+
_check_paths(file, fm, split.body, files, diagnostics)
|
|
96
|
+
checks = _verify_evidence(file, fm, files, data, diagnostics, limits, source_text, budget)
|
|
97
|
+
|
|
98
|
+
rep = Representation(locale, file, fm, split.body, split.body_offset, file_digests.get(file), checks)
|
|
99
|
+
by_key.setdefault(key, ContentNode(key, [])).representations.append(rep)
|
|
100
|
+
|
|
101
|
+
nodes: list[ContentNode] = []
|
|
102
|
+
for node in sorted(by_key.values(), key=lambda n: utf16_key(n.path)):
|
|
103
|
+
node.representations.sort(key=lambda r: (r.locale is not None, utf16_key(r.locale or "")))
|
|
104
|
+
if node.representations[0].locale is not None:
|
|
105
|
+
diagnostics.add(
|
|
106
|
+
"C009_LOCALE_ORPHAN",
|
|
107
|
+
f"locale variant has no default-language file {CONTENT_DIR}/{node.path}",
|
|
108
|
+
node.representations[0].file,
|
|
109
|
+
)
|
|
110
|
+
nodes.append(node)
|
|
111
|
+
return nodes
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _locale_of(rel: str, locales: dict[str, Any]) -> tuple[str, str | None]:
|
|
115
|
+
m = _LOCALE.match(rel)
|
|
116
|
+
if m and m[2].lower() in locales:
|
|
117
|
+
return f"{m[1]}.md", locales[m[2].lower()]
|
|
118
|
+
return rel, None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _check_reserved(
|
|
122
|
+
file: str, rel: str, base: str, text: str, diagnostics: Diagnostics, limits: dict[str, Any]
|
|
123
|
+
) -> None:
|
|
124
|
+
# Contract §3: the frontmatter size limit applies to reserved files too.
|
|
125
|
+
split = split_frontmatter(text, limits["maxFrontmatterBytes"])
|
|
126
|
+
if base == "index.md":
|
|
127
|
+
if not split.present:
|
|
128
|
+
return
|
|
129
|
+
keys = list(split.data) if isinstance(split.data, dict) else []
|
|
130
|
+
root_index = rel.lower() == "index.md"
|
|
131
|
+
if split.error or not root_index or any(k != "okf_version" for k in keys):
|
|
132
|
+
diagnostics.add(
|
|
133
|
+
"C004_RESERVED_FILE_INVALID",
|
|
134
|
+
"the bundle-root index.md may carry only an okf_version frontmatter key"
|
|
135
|
+
if root_index
|
|
136
|
+
else "index.md files carry no frontmatter",
|
|
137
|
+
file,
|
|
138
|
+
)
|
|
139
|
+
return
|
|
140
|
+
if split.present:
|
|
141
|
+
diagnostics.add("C004_RESERVED_FILE_INVALID", "log.md carries no frontmatter", file)
|
|
142
|
+
# Headings inside fenced code blocks are not headings.
|
|
143
|
+
fence = False
|
|
144
|
+
for i, line in enumerate(re.split(r"\r?\n", text)):
|
|
145
|
+
if re.match(r"^[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]*(```|~~~)", line):
|
|
146
|
+
fence = not fence
|
|
147
|
+
if fence:
|
|
148
|
+
continue
|
|
149
|
+
h = re.match(r"^##[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+(.*)\Z", line)
|
|
150
|
+
if h and not re.match(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}", h[1]):
|
|
151
|
+
diagnostics.add(
|
|
152
|
+
"C004_RESERVED_FILE_INVALID",
|
|
153
|
+
f'log.md date headings must start with an ISO 8601 date: "{h[1]}"',
|
|
154
|
+
file,
|
|
155
|
+
i + 1,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _source_ids(fm: dict[str, Any]) -> list[Any]:
|
|
160
|
+
sources = fm.get("sources")
|
|
161
|
+
return [get(s, "id") for s in (sources if isinstance(sources, list) else []) if truthy(get(s, "id"))]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _check_evidence(file: str, fm: dict[str, Any], diagnostics: Diagnostics) -> None:
|
|
165
|
+
evidence = get(fm.get("moca"), "evidence")
|
|
166
|
+
if not isinstance(evidence, list):
|
|
167
|
+
return
|
|
168
|
+
ids = _source_ids(fm)
|
|
169
|
+
for e in evidence:
|
|
170
|
+
source = get(e, "source")
|
|
171
|
+
if isinstance(source, str) and source not in ids:
|
|
172
|
+
diagnostics.add(
|
|
173
|
+
"C006_EVIDENCE_SOURCE_UNKNOWN",
|
|
174
|
+
f'evidence names source "{source}", which sources[] does not declare with that id',
|
|
175
|
+
file,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _verify_evidence(
|
|
180
|
+
file: str,
|
|
181
|
+
fm: dict[str, Any],
|
|
182
|
+
files: dict[str, str],
|
|
183
|
+
data: dict[str, bytes],
|
|
184
|
+
diagnostics: Diagnostics,
|
|
185
|
+
limits: dict[str, int],
|
|
186
|
+
source_text: dict[str, str],
|
|
187
|
+
budget: dict[str, Any],
|
|
188
|
+
) -> list[EvidenceCheck]:
|
|
189
|
+
"""Checks each moca.evidence selector against the cited file, when it is inside the package and checkable."""
|
|
190
|
+
evidence = get(fm.get("moca"), "evidence")
|
|
191
|
+
if not isinstance(evidence, list):
|
|
192
|
+
return []
|
|
193
|
+
declared = fm.get("sources")
|
|
194
|
+
sources: dict[Any, Any] = {}
|
|
195
|
+
for s in declared if isinstance(declared, list) else []:
|
|
196
|
+
sid = get(s, "id")
|
|
197
|
+
if truthy(sid) and isinstance(sid, (str, int, float, bool)):
|
|
198
|
+
sources[sid] = s
|
|
199
|
+
checks: list[EvidenceCheck] = []
|
|
200
|
+
for i, e in enumerate(evidence):
|
|
201
|
+
key = get(e, "source")
|
|
202
|
+
resource = get(sources.get(key), "resource") if isinstance(key, (str, int, float, bool)) else None
|
|
203
|
+
if not isinstance(resource, str) or not _PATH_REF.match(resource):
|
|
204
|
+
checks.append(EvidenceCheck(False))
|
|
205
|
+
continue
|
|
206
|
+
path, escapes = resolve_reference(file, resource)
|
|
207
|
+
if escapes or not path or path not in files or not path.startswith(_EVIDENCE_DIRS):
|
|
208
|
+
checks.append(EvidenceCheck(False))
|
|
209
|
+
continue
|
|
210
|
+
kind = source_kind(path)
|
|
211
|
+
selector = get(e, "selector")
|
|
212
|
+
if not kind or not isinstance(selector, dict):
|
|
213
|
+
checks.append(EvidenceCheck(True))
|
|
214
|
+
continue
|
|
215
|
+
if i >= limits["maxEvidencePerNode"] or len(data[path]) > limits["maxSourceBytesChecked"]:
|
|
216
|
+
checks.append(EvidenceCheck(True))
|
|
217
|
+
continue
|
|
218
|
+
if (
|
|
219
|
+
budget["checks"] >= limits["maxEvidenceChecksPerPackage"]
|
|
220
|
+
or budget["bytes"] + len(data[path]) > limits["maxEvidenceBytesPerPackage"]
|
|
221
|
+
):
|
|
222
|
+
checks.append(EvidenceCheck(True))
|
|
223
|
+
continue
|
|
224
|
+
budget["checks"] += 1
|
|
225
|
+
budget["bytes"] += len(data[path])
|
|
226
|
+
if path not in source_text:
|
|
227
|
+
# Contract §7: a leading byte order mark is not part of an evidence source.
|
|
228
|
+
source_text[path] = data[path].decode("utf-8", errors="replace").removeprefix("\ufeff")
|
|
229
|
+
prepared = None
|
|
230
|
+
if selector.get("type") != "FragmentSelector":
|
|
231
|
+
key = f"{path}\u0000{kind}"
|
|
232
|
+
if key not in budget["prepared"]:
|
|
233
|
+
budget["prepared"][key] = normalised_text(source_text[path], kind)
|
|
234
|
+
prepared = budget["prepared"][key]
|
|
235
|
+
matched = check_selector(selector, source_text[path], kind, prepared)
|
|
236
|
+
if matched is False:
|
|
237
|
+
diagnostics.add(
|
|
238
|
+
"C011_EVIDENCE_SELECTOR_UNMATCHED",
|
|
239
|
+
f'evidence {selector.get("type")} for source "{key}" does not match {path}',
|
|
240
|
+
file,
|
|
241
|
+
)
|
|
242
|
+
checks.append(EvidenceCheck(True) if matched is None else EvidenceCheck(True, matched))
|
|
243
|
+
return checks
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _check_window(file: str, fm: dict[str, Any], diagnostics: Diagnostics) -> None:
|
|
247
|
+
from ._js import parse_date
|
|
248
|
+
|
|
249
|
+
moca = fm.get("moca")
|
|
250
|
+
start, until = parse_date(get(moca, "valid_from")), parse_date(get(moca, "valid_until"))
|
|
251
|
+
if start is not None and until is not None and until <= start:
|
|
252
|
+
diagnostics.add("C010_VALIDITY_WINDOW_INVALID", "moca.valid_until must be later than moca.valid_from", file)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
_DRIVE = re.compile(r"^[A-Za-z]:[\\/]")
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def resolve_reference(from_file: str, ref: str) -> tuple[str | None, bool]:
|
|
259
|
+
"""A package-relative reference from a node: (path, escapes). See :func:`resolve_reference_reason`."""
|
|
260
|
+
path, escapes, _ = resolve_reference_reason(from_file, ref)
|
|
261
|
+
return path, escapes
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def resolve_reference_reason(from_file: str, ref: str) -> tuple[str | None, bool, str]:
|
|
265
|
+
"""A package-relative reference from a node: (path, unsafe, reason).
|
|
266
|
+
|
|
267
|
+
An absolute path or a drive prefix is unsafe, as is one that escapes the package root (C008);
|
|
268
|
+
so is a malformed percent-encoding, which never raises.
|
|
269
|
+
"""
|
|
270
|
+
stripped = re.sub(r"^<|>\Z", "", ref).split("#")[0].split("?")[0]
|
|
271
|
+
try:
|
|
272
|
+
clean = decode_uri_component(stripped)
|
|
273
|
+
except ValueError:
|
|
274
|
+
return None, True, "has a malformed percent-encoding"
|
|
275
|
+
if clean == "":
|
|
276
|
+
return None, False, ""
|
|
277
|
+
if clean.startswith("/") or clean.startswith("\\") or _DRIVE.match(clean):
|
|
278
|
+
return None, True, "is absolute"
|
|
279
|
+
joined = posix_join(posix_dirname(from_file), clean)
|
|
280
|
+
if joined == ".." or joined.startswith("../") or joined.startswith("/"):
|
|
281
|
+
return None, True, "escapes the package root"
|
|
282
|
+
return normalize("NFC", joined), False, ""
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _check_paths(file: str, fm: dict[str, Any], body: str, files: dict[str, str], diagnostics: Diagnostics) -> None:
|
|
286
|
+
sources = fm.get("sources")
|
|
287
|
+
for s in sources if isinstance(sources, list) else []:
|
|
288
|
+
r = get(s, "resource")
|
|
289
|
+
r = r if isinstance(r, str) else ""
|
|
290
|
+
if not _PATH_REF.match(r):
|
|
291
|
+
continue # URLs and scope descriptors are not paths
|
|
292
|
+
path, escapes, reason = resolve_reference_reason(file, r)
|
|
293
|
+
if escapes:
|
|
294
|
+
diagnostics.add("C008_UNSAFE_PATH", f'source resource "{r}" {reason}', file)
|
|
295
|
+
elif path and path not in files:
|
|
296
|
+
diagnostics.add("C007_LINK_UNRESOLVED", f'source resource "{r}" does not resolve to a package file', file)
|
|
297
|
+
|
|
298
|
+
fence = False
|
|
299
|
+
for line in re.split(r"\r?\n", body):
|
|
300
|
+
if re.match(r"^[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]*(```|~~~)", line):
|
|
301
|
+
fence = not fence
|
|
302
|
+
if fence:
|
|
303
|
+
continue
|
|
304
|
+
for m in _LINK.finditer(re.sub(r"`[^`]*`", "", line)):
|
|
305
|
+
target = re.sub(r"^<|>\Z", "", m[1])
|
|
306
|
+
if (_SCHEME.match(target) and not _DRIVE.match(target)) or target.startswith("#"):
|
|
307
|
+
continue
|
|
308
|
+
path, escapes, reason = resolve_reference_reason(file, target)
|
|
309
|
+
if escapes:
|
|
310
|
+
diagnostics.add("C008_UNSAFE_PATH", f'link "{target}" {reason}', file)
|
|
311
|
+
elif path and path not in files:
|
|
312
|
+
diagnostics.add("C007_LINK_UNRESOLVED", f'link "{target}" does not resolve', file)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Diagnostics: stable codes, a severity, a message, and where."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from .codes import ERROR, WARNING, default_severity
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class Diagnostic:
|
|
12
|
+
code: str
|
|
13
|
+
severity: str
|
|
14
|
+
message: str
|
|
15
|
+
file: str | None = None
|
|
16
|
+
line: int | None = None
|
|
17
|
+
|
|
18
|
+
def to_json(self) -> dict[str, object]:
|
|
19
|
+
out: dict[str, object] = {"code": self.code, "severity": self.severity, "message": self.message}
|
|
20
|
+
if self.file is not None:
|
|
21
|
+
out["file"] = self.file
|
|
22
|
+
if self.line is not None:
|
|
23
|
+
out["line"] = self.line
|
|
24
|
+
return out
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Diagnostics:
|
|
29
|
+
strict: bool = False
|
|
30
|
+
items: list[Diagnostic] = field(default_factory=list)
|
|
31
|
+
|
|
32
|
+
def add(self, code: str, message: str, file: str | None = None, line: int | None = None) -> Diagnostic:
|
|
33
|
+
severity = default_severity(code)
|
|
34
|
+
if self.strict and severity == WARNING:
|
|
35
|
+
severity = ERROR
|
|
36
|
+
entry = Diagnostic(code, severity, message, file, line)
|
|
37
|
+
self.items.append(entry)
|
|
38
|
+
return entry
|
|
39
|
+
|
|
40
|
+
def has(self, code: str) -> bool:
|
|
41
|
+
return any(d.code == code for d in self.items)
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def has_errors(self) -> bool:
|
|
45
|
+
return any(d.severity == ERROR for d in self.items)
|
moca_reader/digest.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""The package digest, moca-digest-v2 (package spec §6).
|
|
2
|
+
|
|
3
|
+
The SHA-256 of a BagIt-style payload manifest (RFC 8493 manifest-sha256.txt): one ``<hex> <path>`` line
|
|
4
|
+
per file, sorted by the UTF-8 bytes of the path. Computed, never declared: the same package as a
|
|
5
|
+
directory, an archive or a host source has the same digest.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from .diagnostics import Diagnostics
|
|
14
|
+
from .source import PackageSource
|
|
15
|
+
from .unicode import normalize, simple_lowercase
|
|
16
|
+
|
|
17
|
+
DIGEST_ALGORITHM = "moca-digest-v2"
|
|
18
|
+
DIGEST_PATTERN = re.compile(r"^sha256:[a-f0-9]{64}\Z")
|
|
19
|
+
MANIFEST = "moca.json"
|
|
20
|
+
ATTESTATIONS_DIR = "attestations"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def utf8(text: str) -> bytes:
|
|
24
|
+
"""UTF-8 as JavaScript encodes it: a lone surrogate becomes U+FFFD."""
|
|
25
|
+
return text.encode("utf-16", "surrogatepass").decode("utf-16", "replace").encode("utf-8")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def sha256_hex(data: bytes) -> str:
|
|
29
|
+
return hashlib.sha256(data).hexdigest()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def is_attestation_path(path: str) -> bool:
|
|
33
|
+
# Only entries under the attestations/ directory; a top-level file named
|
|
34
|
+
# `attestations` is ordinary package content and is digested.
|
|
35
|
+
return path.startswith(f"{ATTESTATIONS_DIR}/")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def package_files(source: PackageSource, diagnostics: Diagnostics) -> tuple[bool, dict[str, str]]:
|
|
39
|
+
"""The regular files that make up a package, NFC path -> stored path; P001-P003 when its identity is ambiguous."""
|
|
40
|
+
files: dict[str, str] = {}
|
|
41
|
+
folded: dict[str, str] = {}
|
|
42
|
+
ok = True
|
|
43
|
+
for entry in source.list():
|
|
44
|
+
if entry.kind in ("symlink", "other"):
|
|
45
|
+
what = "symbolic link" if entry.kind == "symlink" else "non-regular file"
|
|
46
|
+
diagnostics.add(
|
|
47
|
+
"P001_UNSUPPORTED_ENTRY",
|
|
48
|
+
f'"{entry.path}" is a {what}; packages may contain regular files only',
|
|
49
|
+
entry.path,
|
|
50
|
+
)
|
|
51
|
+
ok = False
|
|
52
|
+
continue
|
|
53
|
+
if entry.kind == "unreadable":
|
|
54
|
+
diagnostics.add("P002_UNREADABLE_ENTRY", f'"{entry.path}" could not be read', entry.path)
|
|
55
|
+
ok = False
|
|
56
|
+
continue
|
|
57
|
+
nfc = normalize("NFC", entry.path)
|
|
58
|
+
key = simple_lowercase(nfc)
|
|
59
|
+
if key in folded:
|
|
60
|
+
diagnostics.add(
|
|
61
|
+
"P003_PATH_COLLISION",
|
|
62
|
+
f'"{entry.path}" collides with "{folded[key]}" after NFC normalisation or case folding',
|
|
63
|
+
entry.path,
|
|
64
|
+
)
|
|
65
|
+
ok = False
|
|
66
|
+
continue
|
|
67
|
+
folded[key] = entry.path
|
|
68
|
+
files[nfc] = entry.path
|
|
69
|
+
return ok, files
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def manifest_path(path: str) -> str:
|
|
73
|
+
"""A path as it appears in a BagIt manifest line: CR, LF and % percent-encoded (RFC 8493 §2.1.3)."""
|
|
74
|
+
return path.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def manifest_text(file_digests: dict[str, str]) -> str:
|
|
78
|
+
return "".join(
|
|
79
|
+
f"{hexd} {manifest_path(p)}\n" for p, hexd in sorted(file_digests.items(), key=lambda kv: utf8(kv[0]))
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def compute_digest(files: dict[str, str], read: object) -> tuple[str, dict[str, str], str]:
|
|
84
|
+
"""moca-digest-v2 over ``files`` (NFC path -> stored path), reading each with ``read(stored)``."""
|
|
85
|
+
reader = read # a callable taking a stored path
|
|
86
|
+
file_digests: dict[str, str] = {}
|
|
87
|
+
for nfc, stored in files.items():
|
|
88
|
+
if is_attestation_path(nfc):
|
|
89
|
+
continue
|
|
90
|
+
file_digests[nfc] = sha256_hex(reader(stored)) # type: ignore[operator]
|
|
91
|
+
text = manifest_text(file_digests)
|
|
92
|
+
return f"sha256:{sha256_hex(utf8(text))}", file_digests, text
|
moca_reader/dsse.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""DSSE (Dead Simple Signing Envelope): https://github.com/secure-systems-lab/dsse, with Ed25519 keys."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from cryptography.exceptions import InvalidSignature
|
|
12
|
+
from cryptography.hazmat.primitives import serialization
|
|
13
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
|
14
|
+
|
|
15
|
+
from .ed25519 import strict_encodings
|
|
16
|
+
|
|
17
|
+
PAE_PREFIX = "DSSEv1"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def pre_auth_encode(payload_type: str, payload: bytes) -> bytes:
|
|
21
|
+
"""Pre-Authentication Encoding: ``DSSEv1 <len> <type> <len> <body>``, lengths in bytes."""
|
|
22
|
+
type_bytes = payload_type.encode("utf-8")
|
|
23
|
+
return b"".join([f"{PAE_PREFIX} {len(type_bytes)} ".encode(), type_bytes, f" {len(payload)} ".encode(), payload])
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def create_envelope(payload: bytes, payload_type: str, private_key_pem: str, keyid: str) -> dict[str, Any]:
|
|
27
|
+
key = serialization.load_pem_private_key(private_key_pem.encode(), password=None)
|
|
28
|
+
if not isinstance(key, Ed25519PrivateKey):
|
|
29
|
+
raise ValueError("DSSE envelopes are signed with Ed25519 keys")
|
|
30
|
+
signature = key.sign(pre_auth_encode(payload_type, payload))
|
|
31
|
+
return {
|
|
32
|
+
"payload": base64.b64encode(payload).decode(),
|
|
33
|
+
"payloadType": payload_type,
|
|
34
|
+
"signatures": [{"keyid": keyid, "sig": base64.b64encode(signature).decode()}],
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class EnvelopeResult:
|
|
40
|
+
ok: bool
|
|
41
|
+
reason: str | None = None
|
|
42
|
+
payload: bytes | None = None
|
|
43
|
+
keyid: str | None = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
_BASE64 = re.compile(r"^[A-Za-z0-9+/_-]*={0,2}\Z")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def decode_base64(text: Any) -> bytes | None:
|
|
50
|
+
"""Attestations §7: base64 (RFC 4648 §4) or base64url (§5), with or without padding; anything else is None."""
|
|
51
|
+
if not isinstance(text, str) or not _BASE64.match(text):
|
|
52
|
+
return None
|
|
53
|
+
body = text.rstrip("=")
|
|
54
|
+
if len(body) % 4 == 1 or (len(text) != len(body) and len(text) % 4 != 0):
|
|
55
|
+
return None
|
|
56
|
+
return base64.b64decode(body.replace("-", "+").replace("_", "/") + "=" * (-len(body) % 4))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def verify_envelope(envelope: Any, resolve_public_key: Callable[[str], str | None]) -> EnvelopeResult:
|
|
60
|
+
if (
|
|
61
|
+
not isinstance(envelope, dict)
|
|
62
|
+
or not isinstance(envelope.get("payload"), str)
|
|
63
|
+
or not isinstance(envelope.get("payloadType"), str)
|
|
64
|
+
or not isinstance(envelope.get("signatures"), list)
|
|
65
|
+
or len(envelope["signatures"]) == 0
|
|
66
|
+
):
|
|
67
|
+
return EnvelopeResult(False, "malformed DSSE envelope")
|
|
68
|
+
payload = decode_base64(envelope["payload"])
|
|
69
|
+
if payload is None:
|
|
70
|
+
return EnvelopeResult(False, "envelope payload is not valid base64")
|
|
71
|
+
pae = pre_auth_encode(envelope["payloadType"], payload)
|
|
72
|
+
for entry in envelope["signatures"]:
|
|
73
|
+
if (
|
|
74
|
+
not isinstance(entry, dict)
|
|
75
|
+
or not isinstance(entry.get("keyid"), str)
|
|
76
|
+
or not isinstance(entry.get("sig"), str)
|
|
77
|
+
):
|
|
78
|
+
continue
|
|
79
|
+
signature = decode_base64(entry["sig"])
|
|
80
|
+
if signature is None:
|
|
81
|
+
continue
|
|
82
|
+
public_key_pem = resolve_public_key(entry["keyid"])
|
|
83
|
+
if not public_key_pem:
|
|
84
|
+
continue
|
|
85
|
+
try:
|
|
86
|
+
key = serialization.load_pem_public_key(public_key_pem.encode())
|
|
87
|
+
if not isinstance(key, Ed25519PublicKey):
|
|
88
|
+
continue
|
|
89
|
+
# Attestations §5: strict encodings first, so every Reader's library reaches the same answer.
|
|
90
|
+
raw = key.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw)
|
|
91
|
+
if not strict_encodings(raw, signature):
|
|
92
|
+
continue
|
|
93
|
+
key.verify(signature, pae)
|
|
94
|
+
except (InvalidSignature, ValueError, TypeError):
|
|
95
|
+
continue
|
|
96
|
+
return EnvelopeResult(True, payload=payload, keyid=entry["keyid"])
|
|
97
|
+
return EnvelopeResult(False, "no signature verified against a trusted key")
|
moca_reader/ed25519.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Strict Ed25519 encodings (attestations §5).
|
|
2
|
+
|
|
3
|
+
Ed25519 libraries disagree on edge cases: small-order and mixed-order points, and non-canonical encodings. MOCA
|
|
4
|
+
removes the disagreement before a library verifies anything: the public key A and the signature's R must each be
|
|
5
|
+
the canonical encoding of a point of prime order L, and S must be less than L. With those, the cofactored and
|
|
6
|
+
cofactorless verification equations agree, so every Reader reaches the same answer.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
P = 2**255 - 19
|
|
12
|
+
L = 2**252 + 27742317777372353535851937790883648493
|
|
13
|
+
_D = (-121665 * pow(121666, P - 2, P)) % P
|
|
14
|
+
_SQRT_M1 = pow(2, (P - 1) // 4, P)
|
|
15
|
+
|
|
16
|
+
_Point = tuple[int, int, int, int]
|
|
17
|
+
_IDENTITY: _Point = (0, 1, 1, 0)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _add(p: _Point, q: _Point) -> _Point:
|
|
21
|
+
x1, y1, z1, t1 = p
|
|
22
|
+
x2, y2, z2, t2 = q
|
|
23
|
+
a = (y1 - x1) * (y2 - x2) % P
|
|
24
|
+
b = (y1 + x1) * (y2 + x2) % P
|
|
25
|
+
c = 2 * t1 * t2 * _D % P
|
|
26
|
+
d = 2 * z1 * z2 % P
|
|
27
|
+
e, f, g, h = b - a, d - c, d + c, b + a
|
|
28
|
+
return (e * f % P, g * h % P, f * g % P, e * h % P)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _mul(k: int, p: _Point) -> _Point:
|
|
32
|
+
q = _IDENTITY
|
|
33
|
+
while k:
|
|
34
|
+
if k & 1:
|
|
35
|
+
q = _add(q, p)
|
|
36
|
+
p = _add(p, p)
|
|
37
|
+
k >>= 1
|
|
38
|
+
return q
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _is_identity(p: _Point) -> bool:
|
|
42
|
+
x, y, z, _ = p
|
|
43
|
+
return x % P == 0 and (y - z) % P == 0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _decode(encoded: bytes) -> _Point | None:
|
|
47
|
+
"""RFC 8032 §5.1.3, refusing non-canonical encodings (y ≥ p, or x = 0 with the sign bit set)."""
|
|
48
|
+
if len(encoded) != 32:
|
|
49
|
+
return None
|
|
50
|
+
n = int.from_bytes(encoded, "little")
|
|
51
|
+
sign = n >> 255
|
|
52
|
+
y = n & ((1 << 255) - 1)
|
|
53
|
+
if y >= P:
|
|
54
|
+
return None
|
|
55
|
+
u = (y * y - 1) % P
|
|
56
|
+
v = (_D * y * y + 1) % P
|
|
57
|
+
x = u * pow(v, 3, P) * pow(u * pow(v, 7, P), (P - 5) // 8, P) % P
|
|
58
|
+
if (v * x * x - u) % P != 0:
|
|
59
|
+
x = x * _SQRT_M1 % P
|
|
60
|
+
if (v * x * x - u) % P != 0:
|
|
61
|
+
return None
|
|
62
|
+
if x == 0 and sign:
|
|
63
|
+
return None
|
|
64
|
+
if x & 1 != sign:
|
|
65
|
+
x = P - x
|
|
66
|
+
return (x, y, 1, x * y % P)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def is_prime_order_point(encoded: bytes) -> bool:
|
|
70
|
+
"""The canonical encoding of a point of order L: not the identity, no small-order component."""
|
|
71
|
+
point = _decode(encoded)
|
|
72
|
+
return point is not None and not _is_identity(point) and _is_identity(_mul(L, point))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def strict_encodings(public_key: bytes, signature: bytes) -> bool:
|
|
76
|
+
"""Attestations §5: A and R are canonical points of order L, and S < L."""
|
|
77
|
+
if len(signature) != 64:
|
|
78
|
+
return False
|
|
79
|
+
return (
|
|
80
|
+
int.from_bytes(signature[32:], "little") < L
|
|
81
|
+
and is_prime_order_point(public_key)
|
|
82
|
+
and is_prime_order_point(signature[:32])
|
|
83
|
+
)
|