moca-reader 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moca_reader/__init__.py +84 -0
- moca_reader/_js.py +291 -0
- moca_reader/attestations.py +271 -0
- moca_reader/audience.py +49 -0
- moca_reader/backends/__init__.py +1 -0
- moca_reader/backends/bm25.py +118 -0
- moca_reader/backends/dense.py +146 -0
- moca_reader/backends/hybrid.py +73 -0
- moca_reader/backends/lexical.py +112 -0
- moca_reader/backends/memory_store.py +122 -0
- moca_reader/citation.py +213 -0
- moca_reader/codes.py +110 -0
- moca_reader/conformance.py +249 -0
- moca_reader/content.py +312 -0
- moca_reader/diagnostics.py +45 -0
- moca_reader/digest.py +92 -0
- moca_reader/dsse.py +97 -0
- moca_reader/ed25519.py +83 -0
- moca_reader/embedders.py +224 -0
- moca_reader/evidence_text.py +187 -0
- moca_reader/frontmatter.py +151 -0
- moca_reader/library.py +176 -0
- moca_reader/manifest.py +186 -0
- moca_reader/metadata.py +58 -0
- moca_reader/profiles.py +5 -0
- moca_reader/py.typed +0 -0
- moca_reader/reader.py +468 -0
- moca_reader/relations.py +46 -0
- moca_reader/resolver.py +33 -0
- moca_reader/reviews.py +18 -0
- moca_reader/schemas/ag-ui-citations.schema.json +36 -0
- moca_reader/schemas/citation-record.schema.json +326 -0
- moca_reader/schemas/manifest.schema.json +171 -0
- moca_reader/schemas/node.schema.json +372 -0
- moca_reader/schemas/review-predicate.schema.json +42 -0
- moca_reader/schemas/search-hit.schema.json +62 -0
- moca_reader/schemas/sidecar-index.schema.json +155 -0
- moca_reader/schemas/structure.schema.json +93 -0
- moca_reader/schemas/trust-root.schema.json +130 -0
- moca_reader/schemas.py +44 -0
- moca_reader/search.py +346 -0
- moca_reader/sidecar.py +228 -0
- moca_reader/source.py +285 -0
- moca_reader/state.py +119 -0
- moca_reader/strategies/__init__.py +1 -0
- moca_reader/strategies/diversify.py +28 -0
- moca_reader/strategies/ontology_guided.py +63 -0
- moca_reader/structure.py +385 -0
- moca_reader/structure_ops.py +160 -0
- moca_reader/unicode.py +33 -0
- moca_reader-0.6.0.dist-info/METADATA +209 -0
- moca_reader-0.6.0.dist-info/RECORD +55 -0
- moca_reader-0.6.0.dist-info/WHEEL +4 -0
- moca_reader-0.6.0.dist-info/licenses/LICENSE +201 -0
- moca_reader-0.6.0.dist-info/licenses/NOTICE +15 -0
moca_reader/reader.py
ADDED
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
"""``read_package``: opens a package from any source, checks it, computes its digest, verifies its
|
|
2
|
+
attestations against a host trust root, resolves members through a host resolver, and returns everything an
|
|
3
|
+
application needs (Reader contract). It never executes package content, never performs network I/O of its
|
|
4
|
+
own, and never acts on a location or setting a package supplies.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Callable, Iterable
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from datetime import UTC, datetime
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from ._js import get, json_loads, parse_date, truthy
|
|
17
|
+
from .attestations import (
|
|
18
|
+
PREDICATE_PACKAGE,
|
|
19
|
+
PREDICATE_REVIEW,
|
|
20
|
+
TrustRoot,
|
|
21
|
+
check_package_statement,
|
|
22
|
+
check_review_statement,
|
|
23
|
+
parse_attestation,
|
|
24
|
+
scope_covers,
|
|
25
|
+
verify_signature,
|
|
26
|
+
)
|
|
27
|
+
from .content import ContentNode, read_content
|
|
28
|
+
from .diagnostics import Diagnostic, Diagnostics
|
|
29
|
+
from .digest import ATTESTATIONS_DIR, MANIFEST, compute_digest, package_files
|
|
30
|
+
from .manifest import check_manifest
|
|
31
|
+
from .profiles import KNOWN_PROFILES
|
|
32
|
+
from .reviews import review_status_of
|
|
33
|
+
from .source import DEFAULT_LIMITS, PackageSource, TargetError, open_source
|
|
34
|
+
from .structure import PackageStructure, read_structure
|
|
35
|
+
from .unicode import normalize
|
|
36
|
+
|
|
37
|
+
# A009 invalidates: it is only raised when the host requires a signature.
|
|
38
|
+
_INVALIDATING = re.compile(r"^((T|P|M|C)[0-9]|A009)")
|
|
39
|
+
ResolveMember = Callable[[dict[str, Any]], Any]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class Review:
|
|
44
|
+
reviewer: str
|
|
45
|
+
reviewed_at: str
|
|
46
|
+
outcome: str
|
|
47
|
+
signer: str | None
|
|
48
|
+
scope: str | None = None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class Member:
|
|
53
|
+
id: Any
|
|
54
|
+
version: Any
|
|
55
|
+
digest: Any
|
|
56
|
+
status: str = "unresolved" # unresolved | resolved | mismatch | cycle
|
|
57
|
+
result: PackageResult | None = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class PackageResult:
|
|
62
|
+
"""What a Reader concludes about a package (Reader interface §4)."""
|
|
63
|
+
|
|
64
|
+
manifest: Any = None
|
|
65
|
+
digest: str | None = None
|
|
66
|
+
nodes: list[ContentNode] = field(default_factory=list)
|
|
67
|
+
file_digests: dict[str, str] = field(default_factory=dict)
|
|
68
|
+
attestations: list[dict[str, Any]] = field(default_factory=list)
|
|
69
|
+
reviews_by_file: dict[str, list[Review]] = field(default_factory=dict)
|
|
70
|
+
signers: list[str] = field(default_factory=list)
|
|
71
|
+
# Package spec §7.2: the package ids its valid package attestations' signers may speak for.
|
|
72
|
+
relation_scopes: list[str] = field(default_factory=list)
|
|
73
|
+
members: list[Member] = field(default_factory=list)
|
|
74
|
+
profiles: list[str] = field(default_factory=list)
|
|
75
|
+
diagnostics: list[Diagnostic] = field(default_factory=list)
|
|
76
|
+
valid: bool = False
|
|
77
|
+
capabilities: list[str] = field(default_factory=list)
|
|
78
|
+
payload_manifest: str | None = None
|
|
79
|
+
structure: PackageStructure | None = None
|
|
80
|
+
source: PackageSource | None = None
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def id(self) -> Any:
|
|
84
|
+
return get(self.manifest, "id")
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def version(self) -> Any:
|
|
88
|
+
return get(self.manifest, "version")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass
|
|
92
|
+
class _Options:
|
|
93
|
+
trust_root: TrustRoot | None
|
|
94
|
+
resolve_member: ResolveMember | None
|
|
95
|
+
strict: bool
|
|
96
|
+
limits: dict[str, int]
|
|
97
|
+
known_profiles: tuple[str, ...]
|
|
98
|
+
now: datetime | None
|
|
99
|
+
require_signature: bool = False
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
MOCA_VERSION = "0.6"
|
|
103
|
+
"""The specification version this Reader implements, ``major.minor`` (package spec §14)."""
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _compare_version(major: str, minor: str, version: str) -> int:
|
|
107
|
+
"""Compares a declared major.minor with a version string, numerically, without converting to int."""
|
|
108
|
+
|
|
109
|
+
def num(a: str, b: str) -> int:
|
|
110
|
+
x, y = a.lstrip("0") or "0", b.lstrip("0") or "0"
|
|
111
|
+
if len(x) != len(y):
|
|
112
|
+
return -1 if len(x) < len(y) else 1
|
|
113
|
+
return (x > y) - (x < y)
|
|
114
|
+
|
|
115
|
+
want_major, want_minor = version.split(".")
|
|
116
|
+
return num(major, want_major) or num(minor, want_minor)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def read_package(
|
|
120
|
+
target: Any,
|
|
121
|
+
*,
|
|
122
|
+
trust_root: str | os.PathLike[str] | dict[str, Any] | TrustRoot | None = None,
|
|
123
|
+
resolve_member: ResolveMember | None = None,
|
|
124
|
+
strict: bool = False,
|
|
125
|
+
limits: dict[str, int] | None = None,
|
|
126
|
+
known_profiles: Iterable[str] | None = None,
|
|
127
|
+
now: datetime | None = None,
|
|
128
|
+
clock: Callable[[], datetime] | None = None,
|
|
129
|
+
require_signature: bool = False,
|
|
130
|
+
) -> PackageResult:
|
|
131
|
+
"""Reads and checks a package: a directory, a .moca/.zip archive, or a host source.
|
|
132
|
+
|
|
133
|
+
Never raises for a malformed package: ``result.valid`` is False and ``result.diagnostics`` says why.
|
|
134
|
+
"""
|
|
135
|
+
options = _Options(
|
|
136
|
+
TrustRoot.load(trust_root) if trust_root is not None else None,
|
|
137
|
+
resolve_member,
|
|
138
|
+
strict,
|
|
139
|
+
{**DEFAULT_LIMITS, **(limits or {})},
|
|
140
|
+
tuple(known_profiles) if known_profiles is not None else KNOWN_PROFILES,
|
|
141
|
+
clock() if clock is not None else now,
|
|
142
|
+
require_signature,
|
|
143
|
+
)
|
|
144
|
+
return _read(target, options, [], [0])
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _read(target: Any, options: _Options, stack: list[Any], budget: list[int]) -> PackageResult:
|
|
148
|
+
diagnostics = Diagnostics(options.strict)
|
|
149
|
+
result = PackageResult(diagnostics=diagnostics.items)
|
|
150
|
+
try:
|
|
151
|
+
source = open_source(target, options.limits)
|
|
152
|
+
source.list()
|
|
153
|
+
except TargetError as err:
|
|
154
|
+
diagnostics.add(err.code, str(err))
|
|
155
|
+
return result
|
|
156
|
+
result.source = source
|
|
157
|
+
|
|
158
|
+
entries_ok, files = package_files(source, diagnostics)
|
|
159
|
+
if MANIFEST not in files:
|
|
160
|
+
diagnostics.add("T002_MANIFEST_NOT_FOUND", "the target has no moca.json at its root")
|
|
161
|
+
return _finish(result, diagnostics)
|
|
162
|
+
|
|
163
|
+
data: dict[str, bytes] = {}
|
|
164
|
+
readable = True
|
|
165
|
+
for nfc, stored in files.items():
|
|
166
|
+
try:
|
|
167
|
+
data[nfc] = source.read(stored)
|
|
168
|
+
except TargetError as err:
|
|
169
|
+
# A limit broken while reading rejects the whole source (T003), once.
|
|
170
|
+
diagnostics.add(err.code, str(err))
|
|
171
|
+
return result
|
|
172
|
+
except (OSError, KeyError) as err:
|
|
173
|
+
diagnostics.add("P002_UNREADABLE_ENTRY", f'"{stored}" could not be read: {err}', stored)
|
|
174
|
+
readable = False
|
|
175
|
+
# An entry that cannot be read leaves nothing to check the rest against: stop with P002, as the reference does.
|
|
176
|
+
if not readable:
|
|
177
|
+
return _finish(result, diagnostics)
|
|
178
|
+
|
|
179
|
+
try:
|
|
180
|
+
# Contract §11, M001: moca.json is UTF-8; invalid bytes are an error, not replaced.
|
|
181
|
+
manifest = json_loads(data[MANIFEST].decode("utf-8").removeprefix("\ufeff"), unique_keys=True)
|
|
182
|
+
except (ValueError, KeyError) as err:
|
|
183
|
+
diagnostics.add("M001_MANIFEST_INVALID_JSON", f"moca.json is not valid UTF-8 JSON: {err}", MANIFEST)
|
|
184
|
+
return _finish(result, diagnostics)
|
|
185
|
+
result.manifest = manifest
|
|
186
|
+
check_manifest(manifest, diagnostics, options.known_profiles)
|
|
187
|
+
profiles = get(manifest, "profiles")
|
|
188
|
+
result.profiles = list(profiles) if isinstance(profiles, dict) else []
|
|
189
|
+
version = get(manifest, "mocaVersion")
|
|
190
|
+
# Package spec §14: M008 only for a declared mocaVersion below 0.4; M009 for one newer than this Reader's.
|
|
191
|
+
declared = re.fullmatch(r"([0-9]+)\.([0-9]+)", "" if version is None else str(version))
|
|
192
|
+
if declared and _compare_version(declared[1], declared[2], "0.4") < 0:
|
|
193
|
+
diagnostics.add(
|
|
194
|
+
"M008_DIGEST_V1_PACKAGE",
|
|
195
|
+
f"written for mocaVersion {version}; its digest is now computed with moca-digest-v2, so pins "
|
|
196
|
+
"and attestations made under 0.3 or earlier will not match",
|
|
197
|
+
MANIFEST,
|
|
198
|
+
)
|
|
199
|
+
if declared and _compare_version(declared[1], declared[2], MOCA_VERSION) > 0:
|
|
200
|
+
diagnostics.add(
|
|
201
|
+
"M009_MOCA_VERSION_NEWER",
|
|
202
|
+
f"written for mocaVersion {version}, newer than {MOCA_VERSION}; read on a best-effort basis",
|
|
203
|
+
MANIFEST,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
if entries_ok and readable:
|
|
207
|
+
digest, file_digests, payload_manifest = compute_digest(files, lambda stored: data[normalize("NFC", stored)])
|
|
208
|
+
result.digest, result.file_digests, result.payload_manifest = digest, file_digests, payload_manifest
|
|
209
|
+
|
|
210
|
+
result.nodes = read_content(
|
|
211
|
+
manifest if isinstance(manifest, dict) else {}, files, data, result.file_digests, diagnostics, options.limits
|
|
212
|
+
)
|
|
213
|
+
result.structure = read_structure(data, result.nodes, diagnostics, options.limits, result.digest)
|
|
214
|
+
members = get(manifest, "members")
|
|
215
|
+
members = members if isinstance(members, list) else []
|
|
216
|
+
if not result.nodes and not members:
|
|
217
|
+
diagnostics.add(
|
|
218
|
+
"M003_NO_CONTENT", "a package needs at least one content node under content/ or at least one member"
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
if result.digest:
|
|
222
|
+
_read_attestations(result, data, options, diagnostics)
|
|
223
|
+
# Attestations §8: a host can require a valid package attestation.
|
|
224
|
+
if options.require_signature and not result.signers:
|
|
225
|
+
diagnostics.add(
|
|
226
|
+
"A009_SIGNATURE_REQUIRED",
|
|
227
|
+
"the host requires a valid package attestation, and this package has none",
|
|
228
|
+
MANIFEST,
|
|
229
|
+
)
|
|
230
|
+
_check_relation_authority(result, diagnostics)
|
|
231
|
+
_resolve_members(result, members, options, stack, diagnostics, budget)
|
|
232
|
+
return _finish(result, diagnostics)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _read_attestations(
|
|
236
|
+
result: PackageResult, data: dict[str, bytes], options: _Options, diagnostics: Diagnostics
|
|
237
|
+
) -> None:
|
|
238
|
+
from ._js import js_sorted
|
|
239
|
+
|
|
240
|
+
manifest, digest = result.manifest, result.digest or ""
|
|
241
|
+
attestation_files = js_sorted(
|
|
242
|
+
p for p in data if p.startswith(f"{ATTESTATIONS_DIR}/") and re.search(r"\.(dsse|sigstore)\.json\Z", p)
|
|
243
|
+
)
|
|
244
|
+
for file in attestation_files:
|
|
245
|
+
parsed = parse_attestation(file, data[file])
|
|
246
|
+
if isinstance(parsed, str):
|
|
247
|
+
diagnostics.add("A001_ATTESTATION_MALFORMED", parsed, file)
|
|
248
|
+
result.attestations.append({"file": file, "outcome": "malformed", "reason": parsed})
|
|
249
|
+
continue
|
|
250
|
+
kind = parsed.statement["predicateType"]
|
|
251
|
+
if kind not in (PREDICATE_PACKAGE, PREDICATE_REVIEW):
|
|
252
|
+
result.attestations.append(
|
|
253
|
+
{"file": file, "predicateType": kind, "outcome": "ignored", "reason": "unknown predicate type"}
|
|
254
|
+
)
|
|
255
|
+
continue
|
|
256
|
+
role = "package" if kind == PREDICATE_PACKAGE else "review"
|
|
257
|
+
sig = verify_signature(parsed, options.trust_root, role, options.now)
|
|
258
|
+
record: dict[str, Any] = {
|
|
259
|
+
"file": file,
|
|
260
|
+
"predicateType": kind,
|
|
261
|
+
"role": role,
|
|
262
|
+
"format": parsed.format,
|
|
263
|
+
"outcome": sig.outcome,
|
|
264
|
+
"signer": sig.signer,
|
|
265
|
+
"reason": sig.reason,
|
|
266
|
+
}
|
|
267
|
+
result.attestations.append(record)
|
|
268
|
+
if sig.outcome == "unverifiable":
|
|
269
|
+
diagnostics.add("A003_ATTESTATION_UNVERIFIABLE", f"{role} attestation not checked: {sig.reason}", file)
|
|
270
|
+
continue
|
|
271
|
+
if sig.outcome == "indeterminate":
|
|
272
|
+
diagnostics.add("A004_ATTESTATION_INDETERMINATE", sig.reason or "", file)
|
|
273
|
+
continue
|
|
274
|
+
if sig.outcome == "invalid":
|
|
275
|
+
diagnostics.add(
|
|
276
|
+
"A002_ATTESTATION_INVALID", f"{role} attestation signature did not verify: {sig.reason}", file
|
|
277
|
+
)
|
|
278
|
+
continue
|
|
279
|
+
|
|
280
|
+
if role == "package":
|
|
281
|
+
mismatch = check_package_statement(parsed.statement, get(manifest, "id"), get(manifest, "version"), digest)
|
|
282
|
+
if mismatch:
|
|
283
|
+
record["outcome"], record["reason"] = "invalid", mismatch
|
|
284
|
+
# Attestations §3: a statement naming this id@version with another digest is positive evidence that
|
|
285
|
+
# the content changed after signing.
|
|
286
|
+
subjects = parsed.statement.get("subject") or []
|
|
287
|
+
tampered = (
|
|
288
|
+
len(subjects) == 1
|
|
289
|
+
and get(subjects[0], "name") == f"{get(manifest, 'id')}@{get(manifest, 'version')}"
|
|
290
|
+
)
|
|
291
|
+
diagnostics.add(
|
|
292
|
+
"A008_PACKAGE_DIGEST_MISMATCH" if tampered else "A002_ATTESTATION_INVALID",
|
|
293
|
+
f"package attestation does not match this package: {mismatch}",
|
|
294
|
+
file,
|
|
295
|
+
)
|
|
296
|
+
elif sig.signer is not None:
|
|
297
|
+
result.signers.append(sig.signer)
|
|
298
|
+
result.relation_scopes.extend(sig.ids)
|
|
299
|
+
continue
|
|
300
|
+
|
|
301
|
+
review = check_review_statement(parsed.statement, get(manifest, "id"), result.file_digests)
|
|
302
|
+
if review.error:
|
|
303
|
+
record["outcome"], record["reason"] = "invalid", review.error
|
|
304
|
+
diagnostics.add("A002_ATTESTATION_INVALID", review.error, file)
|
|
305
|
+
continue
|
|
306
|
+
predicate = review.predicate or {}
|
|
307
|
+
# Attestations §4: a key with a `reviewers` list may only sign as those reviewers.
|
|
308
|
+
if sig.reviewers is not None and predicate.get("reviewer") not in sig.reviewers:
|
|
309
|
+
reason = f"the signing key may not sign as reviewer {predicate.get('reviewer')}"
|
|
310
|
+
record["outcome"], record["reason"] = "invalid", reason
|
|
311
|
+
diagnostics.add("A002_ATTESTATION_INVALID", reason, file)
|
|
312
|
+
continue
|
|
313
|
+
# A review dated after the host clock does not count.
|
|
314
|
+
reviewed_at = parse_date(predicate.get("reviewedAt"))
|
|
315
|
+
if reviewed_at is not None and reviewed_at > (options.now or datetime.now(UTC)):
|
|
316
|
+
reason = f"review is dated {predicate.get('reviewedAt')}, later than now"
|
|
317
|
+
record["outcome"], record["reason"] = "ignored", reason
|
|
318
|
+
diagnostics.add("A007_REVIEW_DATE_FUTURE", reason, file)
|
|
319
|
+
continue
|
|
320
|
+
record["current"], record["outdated"] = review.current, review.outdated
|
|
321
|
+
for path in review.outdated:
|
|
322
|
+
diagnostics.add("A006_REVIEW_OUTDATED", f"review of {path} no longer matches the file", file)
|
|
323
|
+
for path in review.current:
|
|
324
|
+
entry = Review(
|
|
325
|
+
predicate["reviewer"],
|
|
326
|
+
predicate["reviewedAt"],
|
|
327
|
+
predicate["outcome"],
|
|
328
|
+
sig.signer,
|
|
329
|
+
predicate.get("scope") if truthy(predicate.get("scope")) else None,
|
|
330
|
+
)
|
|
331
|
+
result.reviews_by_file.setdefault(normalize("NFC", path), []).append(entry)
|
|
332
|
+
# Attestations §4: for each file, the latest review from each signer counts.
|
|
333
|
+
for path, entries in result.reviews_by_file.items():
|
|
334
|
+
latest: dict[str | None, Review] = {}
|
|
335
|
+
for e in entries:
|
|
336
|
+
prev = latest.get(e.signer)
|
|
337
|
+
if prev is None or _instant(e.reviewed_at) >= _instant(prev.reviewed_at):
|
|
338
|
+
latest[e.signer] = e
|
|
339
|
+
result.reviews_by_file[path] = list(latest.values())
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _check_relation_authority(result: PackageResult, diagnostics: Diagnostics) -> None:
|
|
343
|
+
"""Package spec §7.2: supersedes and conflictsWith take effect only when a signer may speak for the target."""
|
|
344
|
+
relations = get(result.manifest, "relations")
|
|
345
|
+
for r in relations if isinstance(relations, list) else []:
|
|
346
|
+
if get(r, "type") not in ("supersedes", "conflictsWith") or not isinstance(get(r, "target"), str):
|
|
347
|
+
continue
|
|
348
|
+
if not scope_covers(result.relation_scopes, r["target"]):
|
|
349
|
+
diagnostics.add(
|
|
350
|
+
"R004_RELATION_UNAUTHORISED",
|
|
351
|
+
f"{r['type']} {r['target']} is not authorised by a signer of this package, so it has no effect",
|
|
352
|
+
MANIFEST,
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _resolve_members(
|
|
357
|
+
result: PackageResult,
|
|
358
|
+
members: list[Any],
|
|
359
|
+
options: _Options,
|
|
360
|
+
stack: list[Any],
|
|
361
|
+
diagnostics: Diagnostics,
|
|
362
|
+
budget: list[int],
|
|
363
|
+
) -> None:
|
|
364
|
+
max_depth = options.limits["maxMemberDepth"]
|
|
365
|
+
max_members = options.limits["maxMembers"]
|
|
366
|
+
# Cycles are detected by id@version: composing another version of the same package is not a cycle.
|
|
367
|
+
own_key = f"{get(result.manifest, 'id')}@{get(result.manifest, 'version')}"
|
|
368
|
+
for member in members:
|
|
369
|
+
if not truthy(get(member, "id")) or not truthy(get(member, "digest")):
|
|
370
|
+
continue
|
|
371
|
+
entry = Member(member["id"], member.get("version"), member["digest"])
|
|
372
|
+
result.members.append(entry)
|
|
373
|
+
if options.resolve_member is None:
|
|
374
|
+
continue
|
|
375
|
+
member_key = f"{member['id']}@{member.get('version')}"
|
|
376
|
+
if member_key == own_key or member_key in stack:
|
|
377
|
+
chain = " -> ".join(str(x) for x in [*stack, own_key, member_key])
|
|
378
|
+
diagnostics.add("R003_MEMBER_CYCLE", f"members form a cycle: {chain}", MANIFEST)
|
|
379
|
+
entry.status = "cycle"
|
|
380
|
+
continue
|
|
381
|
+
# Package spec §7.1: depth and the members resolved per read are bounded; a member past either limit is not used.
|
|
382
|
+
if len(stack) + 1 > max_depth:
|
|
383
|
+
diagnostics.add(
|
|
384
|
+
"R001_MEMBER_UNRESOLVED",
|
|
385
|
+
f"member {member_key} is not resolved: composition is deeper than {max_depth} levels",
|
|
386
|
+
MANIFEST,
|
|
387
|
+
)
|
|
388
|
+
continue
|
|
389
|
+
if budget[0] >= max_members:
|
|
390
|
+
diagnostics.add(
|
|
391
|
+
"R001_MEMBER_UNRESOLVED",
|
|
392
|
+
f"member {member_key} is not resolved: more than {max_members} members in one read",
|
|
393
|
+
MANIFEST,
|
|
394
|
+
)
|
|
395
|
+
continue
|
|
396
|
+
budget[0] += 1
|
|
397
|
+
target = options.resolve_member({"id": entry.id, "version": entry.version, "digest": entry.digest})
|
|
398
|
+
if not target:
|
|
399
|
+
diagnostics.add(
|
|
400
|
+
"R001_MEMBER_UNRESOLVED", f"member {entry.id}@{entry.version} could not be resolved", MANIFEST
|
|
401
|
+
)
|
|
402
|
+
continue
|
|
403
|
+
child = _read(target, options, [*stack, own_key], budget)
|
|
404
|
+
entry.result = child
|
|
405
|
+
if any(d.code == "R003_MEMBER_CYCLE" for d in child.diagnostics):
|
|
406
|
+
diagnostics.add("R003_MEMBER_CYCLE", f"member {entry.id} is part of a cycle", MANIFEST)
|
|
407
|
+
entry.status = "cycle"
|
|
408
|
+
elif child.manifest and (
|
|
409
|
+
child.manifest.get("id") != entry.id or (entry.version and child.manifest.get("version") != entry.version)
|
|
410
|
+
):
|
|
411
|
+
# Package spec §7.1: the resolved member must be the package the pin names.
|
|
412
|
+
diagnostics.add(
|
|
413
|
+
"R002_MEMBER_DIGEST_MISMATCH",
|
|
414
|
+
f"member pinned as {entry.id}@{entry.version} resolved to "
|
|
415
|
+
f"{child.manifest.get('id')}@{child.manifest.get('version')}",
|
|
416
|
+
MANIFEST,
|
|
417
|
+
)
|
|
418
|
+
entry.status = "mismatch"
|
|
419
|
+
elif child.digest != entry.digest:
|
|
420
|
+
diagnostics.add(
|
|
421
|
+
"R002_MEMBER_DIGEST_MISMATCH",
|
|
422
|
+
f"member {entry.id}@{entry.version} has digest {child.digest or '(none)'}, but {entry.digest} is pinned",
|
|
423
|
+
MANIFEST,
|
|
424
|
+
)
|
|
425
|
+
entry.status = "mismatch"
|
|
426
|
+
else:
|
|
427
|
+
entry.status = "resolved"
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _instant(value: str) -> float:
|
|
431
|
+
parsed = parse_date(value)
|
|
432
|
+
return parsed.timestamp() if parsed is not None else float("-inf")
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _finish(result: PackageResult, diagnostics: Diagnostics) -> PackageResult:
|
|
436
|
+
result.valid = not any(d.severity == "error" and _INVALIDATING.match(d.code) for d in diagnostics.items)
|
|
437
|
+
if not result.valid:
|
|
438
|
+
return result
|
|
439
|
+
caps = {"core"}
|
|
440
|
+
all_local = True
|
|
441
|
+
reviewed_files = 0
|
|
442
|
+
all_approved = True
|
|
443
|
+
for node in result.nodes:
|
|
444
|
+
for rep in node.representations:
|
|
445
|
+
evidence = get(rep.frontmatter.get("moca"), "evidence")
|
|
446
|
+
if isinstance(evidence, list) and evidence:
|
|
447
|
+
caps.add("located-evidence")
|
|
448
|
+
if any(not c.local for c in rep.evidence_checks):
|
|
449
|
+
all_local = False
|
|
450
|
+
if rep.locale:
|
|
451
|
+
caps.add("localized")
|
|
452
|
+
if rep.file in result.reviews_by_file:
|
|
453
|
+
reviewed_files += 1
|
|
454
|
+
if review_status_of(result.reviews_by_file[rep.file], bool(result.signers)) != "approved":
|
|
455
|
+
all_approved = False
|
|
456
|
+
if "located-evidence" in caps and all_local:
|
|
457
|
+
caps.add("self-contained-evidence")
|
|
458
|
+
if result.structure is not None and result.structure.ok:
|
|
459
|
+
caps.add("structured")
|
|
460
|
+
if result.members:
|
|
461
|
+
caps.add("composed")
|
|
462
|
+
if result.signers:
|
|
463
|
+
caps.add("signed")
|
|
464
|
+
# Package spec §11 (moca-spec ADR-0017): signed, and every reviewed file approved.
|
|
465
|
+
if "signed" in caps and reviewed_files > 0 and all_approved:
|
|
466
|
+
caps.add("reviewed")
|
|
467
|
+
result.capabilities = sorted(caps)
|
|
468
|
+
return result
|
moca_reader/relations.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Package-level relation effects (package spec §7.2) over a set of packages: loaded results in a Library, or
|
|
2
|
+
package states for a store search (Reader contract §9.4)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ._js import get, js_string, truthy
|
|
10
|
+
from .attestations import scope_covers
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def relation_index_of(
|
|
14
|
+
packages: Sequence[dict[str, Any]], allow_unsigned_relations: bool = False
|
|
15
|
+
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
|
|
16
|
+
"""(superseded, contested), each keyed by ``id@version``. Each package is ``{id, version, relations,
|
|
17
|
+
relationScopes}``."""
|
|
18
|
+
superseded: dict[str, list[str]] = {}
|
|
19
|
+
contested: dict[str, list[str]] = {}
|
|
20
|
+
|
|
21
|
+
def push(table: dict[str, list[str]], key: str, value: str) -> None:
|
|
22
|
+
values = table.setdefault(key, [])
|
|
23
|
+
if value not in values:
|
|
24
|
+
values.append(value)
|
|
25
|
+
|
|
26
|
+
for m in packages:
|
|
27
|
+
relations = m.get("relations")
|
|
28
|
+
for r in relations if isinstance(relations, list) else []:
|
|
29
|
+
# Package spec §7.2: only relations a signer of the package may make take effect.
|
|
30
|
+
if not allow_unsigned_relations and not scope_covers(m.get("relationScopes") or [], get(r, "target")):
|
|
31
|
+
continue
|
|
32
|
+
targets = [
|
|
33
|
+
t
|
|
34
|
+
for t in packages
|
|
35
|
+
if t.get("id") == get(r, "target")
|
|
36
|
+
and (not truthy(get(r, "version")) or t.get("version") == r["version"])
|
|
37
|
+
]
|
|
38
|
+
for t in targets:
|
|
39
|
+
key = f"{js_string(t.get('id'))}@{js_string(t.get('version'))}"
|
|
40
|
+
own = f"{js_string(m.get('id'))}@{js_string(m.get('version'))}"
|
|
41
|
+
if get(r, "type") == "supersedes":
|
|
42
|
+
push(superseded, key, own)
|
|
43
|
+
if get(r, "type") == "conflictsWith":
|
|
44
|
+
push(contested, key, own)
|
|
45
|
+
push(contested, own, key)
|
|
46
|
+
return superseded, contested
|
moca_reader/resolver.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""A simple host resolver for members: looks for packages in a list of directories. Hosts with a registry
|
|
2
|
+
supply their own resolver instead."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from collections.abc import Callable, Iterable
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def directory_resolver(dirs: Iterable[str | os.PathLike[str]]) -> Callable[[dict[str, Any]], str | None]:
|
|
13
|
+
"""A ``resolve_member`` that finds ``{id, version}`` among the package directories in, or equal to, each directory."""
|
|
14
|
+
candidates: list[tuple[Any, Any, str]] = []
|
|
15
|
+
for d in [os.fspath(x) for x in dirs]:
|
|
16
|
+
if not os.path.exists(d):
|
|
17
|
+
continue
|
|
18
|
+
roots = [d, *(os.path.join(d, n) for n in os.listdir(d) if os.path.isdir(os.path.join(d, n)))]
|
|
19
|
+
for root in roots:
|
|
20
|
+
manifest = os.path.join(root, "moca.json")
|
|
21
|
+
if not os.path.exists(manifest):
|
|
22
|
+
continue
|
|
23
|
+
try:
|
|
24
|
+
with open(manifest, encoding="utf-8") as f:
|
|
25
|
+
m = json.load(f)
|
|
26
|
+
candidates.append((m.get("id"), m.get("version"), root))
|
|
27
|
+
except (ValueError, OSError, AttributeError):
|
|
28
|
+
continue # an unreadable manifest is not a candidate
|
|
29
|
+
|
|
30
|
+
def resolve(member: dict[str, Any]) -> str | None:
|
|
31
|
+
return next((root for i, v, root in candidates if i == member.get("id") and v == member.get("version")), None)
|
|
32
|
+
|
|
33
|
+
return resolve
|
moca_reader/reviews.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Review status (package spec §11, Reader contract §7, moca-spec ADR-0017)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def review_status_of(reviews: Sequence[Any] | None, signed: bool) -> str:
|
|
10
|
+
"""``none``, ``unverified`` (reviews in an unsigned package), ``approved`` (signed, an accurate review and none
|
|
11
|
+
against) or ``disputed`` (signed, and a current review says needs-change or inaccurate)."""
|
|
12
|
+
if not reviews:
|
|
13
|
+
return "none"
|
|
14
|
+
if not signed:
|
|
15
|
+
return "unverified"
|
|
16
|
+
if any(r.outcome != "accurate" for r in reviews):
|
|
17
|
+
return "disputed"
|
|
18
|
+
return "approved"
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://w3id.org/moca/schemas/v1/ag-ui-citations.schema.json",
|
|
4
|
+
"title": "MOCA citations over AG-UI",
|
|
5
|
+
"description": "An AG-UI CUSTOM event that carries the citation records an answer cites. Normative: spec/moca-ag-ui-citations.md.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["type", "name", "value"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"type": { "const": "CUSTOM" },
|
|
10
|
+
"name": { "const": "moca.citations" },
|
|
11
|
+
"timestamp": { "type": "integer" },
|
|
12
|
+
"value": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"required": ["version", "messageId", "citations"],
|
|
15
|
+
"additionalProperties": false,
|
|
16
|
+
"properties": {
|
|
17
|
+
"version": { "const": 1, "description": "The version of this event's value. A client ignores a version it does not know." },
|
|
18
|
+
"messageId": { "type": "string", "minLength": 1, "description": "The AG-UI messageId of the assistant message that cites these records." },
|
|
19
|
+
"computedAt": { "type": "string", "format": "date-time", "description": "When the service's Reader computed the records (RFC 3339, with an offset). Trust flags are true at that moment." },
|
|
20
|
+
"citations": {
|
|
21
|
+
"type": "array",
|
|
22
|
+
"description": "One entry per cited node, or chunk of a node, in order of first mention in the answer.",
|
|
23
|
+
"items": {
|
|
24
|
+
"type": "object",
|
|
25
|
+
"required": ["record"],
|
|
26
|
+
"additionalProperties": false,
|
|
27
|
+
"properties": {
|
|
28
|
+
"marker": { "type": "string", "minLength": 1, "description": "The text in the answer that refers to this citation, when the host uses markers such as [1]." },
|
|
29
|
+
"record": { "$ref": "citation-record.schema.json" }
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|