moca-reader 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moca_reader/__init__.py +84 -0
- moca_reader/_js.py +291 -0
- moca_reader/attestations.py +271 -0
- moca_reader/audience.py +49 -0
- moca_reader/backends/__init__.py +1 -0
- moca_reader/backends/bm25.py +118 -0
- moca_reader/backends/dense.py +146 -0
- moca_reader/backends/hybrid.py +73 -0
- moca_reader/backends/lexical.py +112 -0
- moca_reader/backends/memory_store.py +122 -0
- moca_reader/citation.py +213 -0
- moca_reader/codes.py +110 -0
- moca_reader/conformance.py +249 -0
- moca_reader/content.py +312 -0
- moca_reader/diagnostics.py +45 -0
- moca_reader/digest.py +92 -0
- moca_reader/dsse.py +97 -0
- moca_reader/ed25519.py +83 -0
- moca_reader/embedders.py +224 -0
- moca_reader/evidence_text.py +187 -0
- moca_reader/frontmatter.py +151 -0
- moca_reader/library.py +176 -0
- moca_reader/manifest.py +186 -0
- moca_reader/metadata.py +58 -0
- moca_reader/profiles.py +5 -0
- moca_reader/py.typed +0 -0
- moca_reader/reader.py +468 -0
- moca_reader/relations.py +46 -0
- moca_reader/resolver.py +33 -0
- moca_reader/reviews.py +18 -0
- moca_reader/schemas/ag-ui-citations.schema.json +36 -0
- moca_reader/schemas/citation-record.schema.json +326 -0
- moca_reader/schemas/manifest.schema.json +171 -0
- moca_reader/schemas/node.schema.json +372 -0
- moca_reader/schemas/review-predicate.schema.json +42 -0
- moca_reader/schemas/search-hit.schema.json +62 -0
- moca_reader/schemas/sidecar-index.schema.json +155 -0
- moca_reader/schemas/structure.schema.json +93 -0
- moca_reader/schemas/trust-root.schema.json +130 -0
- moca_reader/schemas.py +44 -0
- moca_reader/search.py +346 -0
- moca_reader/sidecar.py +228 -0
- moca_reader/source.py +285 -0
- moca_reader/state.py +119 -0
- moca_reader/strategies/__init__.py +1 -0
- moca_reader/strategies/diversify.py +28 -0
- moca_reader/strategies/ontology_guided.py +63 -0
- moca_reader/structure.py +385 -0
- moca_reader/structure_ops.py +160 -0
- moca_reader/unicode.py +33 -0
- moca_reader-0.6.0.dist-info/METADATA +209 -0
- moca_reader-0.6.0.dist-info/RECORD +55 -0
- moca_reader-0.6.0.dist-info/WHEEL +4 -0
- moca_reader-0.6.0.dist-info/licenses/LICENSE +201 -0
- moca_reader-0.6.0.dist-info/licenses/NOTICE +15 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""The default ontology-guided retrieval strategy (moca-spec ADR-0010). Opt-in: pass its hooks to Search.
|
|
2
|
+
|
|
3
|
+
ingress: when the query names concepts by their label, search those concepts and their descendants.
|
|
4
|
+
egress: after each result, add the nodes of the concepts it requires (transitively), tagged retrieval.via = "requires".
|
|
5
|
+
The Reader's gate still applies the retrieval policy to everything added.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from .._js import js_sorted
|
|
14
|
+
from ..search import HookContext, SearchHooks
|
|
15
|
+
from ..structure import preferred
|
|
16
|
+
|
|
17
|
+
_WORD = re.compile(r"[^\W_]+")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _words(text: str) -> list[str]:
|
|
21
|
+
return _WORD.findall(text.lower())
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def ontology_guided(expand_requires: bool = True) -> SearchHooks:
|
|
25
|
+
def ingress(query: str, ctx: HookContext) -> dict[str, Any]:
|
|
26
|
+
if ctx.structure is None:
|
|
27
|
+
return {}
|
|
28
|
+
q = set(_words(query))
|
|
29
|
+
matched = []
|
|
30
|
+
for c in ctx.structure.graph.concepts.values():
|
|
31
|
+
if not c.declared:
|
|
32
|
+
continue
|
|
33
|
+
label = preferred(c.labels)
|
|
34
|
+
w = _words(label) if label else []
|
|
35
|
+
if w and all(x in q for x in w):
|
|
36
|
+
matched.append(c.iri)
|
|
37
|
+
if not matched:
|
|
38
|
+
return {}
|
|
39
|
+
concepts = dict.fromkeys(matched)
|
|
40
|
+
for iri in matched:
|
|
41
|
+
for d in ctx.structure.descendants(iri):
|
|
42
|
+
concepts.setdefault(d)
|
|
43
|
+
return {"concepts": js_sorted(concepts)}
|
|
44
|
+
|
|
45
|
+
def egress(candidates: list[dict[str, Any]], ctx: HookContext) -> list[dict[str, Any]]:
|
|
46
|
+
if not expand_requires or ctx.structure is None:
|
|
47
|
+
return candidates
|
|
48
|
+
out: list[dict[str, Any]] = []
|
|
49
|
+
seen = {r["node"]["ref"] for r in candidates}
|
|
50
|
+
for record in candidates:
|
|
51
|
+
out.append(record)
|
|
52
|
+
for concept in record.get("concepts") or []:
|
|
53
|
+
for req in ctx.structure.requires(concept, transitive=True):
|
|
54
|
+
for node in ctx.structure.nodes(req["iri"], locale=ctx.request.get("locale")):
|
|
55
|
+
if node["node"]["ref"] in seen:
|
|
56
|
+
continue
|
|
57
|
+
seen.add(node["node"]["ref"])
|
|
58
|
+
out.append(
|
|
59
|
+
{**node, "score": record.get("score"), "retrieval": {"via": "requires", "from": concept}}
|
|
60
|
+
)
|
|
61
|
+
return out
|
|
62
|
+
|
|
63
|
+
return SearchHooks(ingress=ingress, egress=egress)
|
moca_reader/structure.py
ADDED
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
"""The structure core (package spec §5.6, moca-spec ADR-0007).
|
|
2
|
+
|
|
3
|
+
structure.ttl holds a package's concepts and how they relate, using only SKOS and DCMI terms. This module
|
|
4
|
+
parses it (and overlays, ADR-0008) into facts, checks node bindings, and builds a sorted JSON view of it
|
|
5
|
+
(structure_view). Nothing is fetched: owl:imports and non-absolute IRIs are reported, never followed.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import rdflib
|
|
15
|
+
from rdflib.plugins.stores.memory import Memory
|
|
16
|
+
|
|
17
|
+
from ._js import get, js_sorted, utf16_key
|
|
18
|
+
from .content import ContentNode, Representation
|
|
19
|
+
from .diagnostics import Diagnostics
|
|
20
|
+
|
|
21
|
+
STRUCTURE_FILE = "structure.ttl"
|
|
22
|
+
|
|
23
|
+
RDF = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
24
|
+
SKOS = "http://www.w3.org/2004/02/skos/core#"
|
|
25
|
+
DCT = "http://purl.org/dc/terms/"
|
|
26
|
+
OWL = "http://www.w3.org/2002/07/owl#"
|
|
27
|
+
ABSOLUTE_IRI = re.compile(
|
|
28
|
+
r"^[A-Za-z][A-Za-z0-9+.-]*:[^\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+\Z"
|
|
29
|
+
)
|
|
30
|
+
_PREFIX_DECL = re.compile(
|
|
31
|
+
r"^[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]*(?:@prefix|PREFIX)[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+([A-Za-z][A-Za-z0-9_.-]*)?:",
|
|
32
|
+
re.IGNORECASE | re.ASCII | re.MULTILINE,
|
|
33
|
+
)
|
|
34
|
+
# rdflib resolves relative IRIs against a base; this one marks them, so they are reported as the
|
|
35
|
+
# reference Reader reports them: not absolute.
|
|
36
|
+
_RELATIVE_BASE = "https://moca-relative.invalid/base/"
|
|
37
|
+
_RELATIVE_HOST = "https://moca-relative.invalid/"
|
|
38
|
+
|
|
39
|
+
PREDICATES: dict[str, tuple[str, bool]] = {
|
|
40
|
+
f"{SKOS}prefLabel": ("label", False),
|
|
41
|
+
f"{SKOS}definition": ("definition", False),
|
|
42
|
+
f"{SKOS}broader": ("broader", False),
|
|
43
|
+
f"{SKOS}narrower": ("broader", True),
|
|
44
|
+
f"{SKOS}related": ("related", False),
|
|
45
|
+
f"{DCT}hasPart": ("hasPart", False),
|
|
46
|
+
f"{DCT}isPartOf": ("hasPart", True),
|
|
47
|
+
f"{DCT}requires": ("requires", False),
|
|
48
|
+
f"{DCT}isRequiredBy": ("requires", True),
|
|
49
|
+
f"{DCT}replaces": ("replaces", False),
|
|
50
|
+
f"{DCT}isReplacedBy": ("replaces", True),
|
|
51
|
+
f"{OWL}deprecated": ("deprecated", False),
|
|
52
|
+
}
|
|
53
|
+
RELATIONS = ("broader", "related", "hasPart", "requires", "replaces")
|
|
54
|
+
LAYERS = ("package", "application", "organisation")
|
|
55
|
+
|
|
56
|
+
Fact = dict[str, Any]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _Recorder(Memory):
|
|
60
|
+
"""A store that remembers triples in the order the parser states them, as the reference Reader sees them."""
|
|
61
|
+
|
|
62
|
+
def __init__(self) -> None:
|
|
63
|
+
super().__init__()
|
|
64
|
+
self.order: list[tuple[Any, Any, Any]] = []
|
|
65
|
+
|
|
66
|
+
def add(self, triple: Any, context: Any, quoted: bool = False) -> None:
|
|
67
|
+
self.order.append(triple)
|
|
68
|
+
super().add(triple, context, quoted)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _iri(term: Any) -> str:
|
|
72
|
+
text = str(term)
|
|
73
|
+
return text[len(_RELATIVE_BASE) :] if text.startswith(_RELATIVE_BASE) else text
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _is_relative(term: Any) -> bool:
|
|
77
|
+
return isinstance(term, rdflib.URIRef) and str(term).startswith(_RELATIVE_HOST)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def parse_structure(
|
|
81
|
+
text: bytes | str, file: str, layer: str, origin: str | None, diagnostics: Diagnostics, limits: dict[str, int]
|
|
82
|
+
) -> tuple[list[Fact], set[str], bool]:
|
|
83
|
+
"""Parses Turtle into core facts. Unknown predicates are ignored; problems are reported as O codes."""
|
|
84
|
+
before = len(diagnostics.items)
|
|
85
|
+
facts: list[Fact] = []
|
|
86
|
+
prefixes: set[str] = set()
|
|
87
|
+
size = len(text) if isinstance(text, bytes) else len(text.encode("utf-8"))
|
|
88
|
+
if size > limits["maxStructureBytes"]:
|
|
89
|
+
diagnostics.add(
|
|
90
|
+
"O001_ONTOLOGY_UNPARSEABLE", f"{file} is larger than the {limits['maxStructureBytes']}-byte limit", file
|
|
91
|
+
)
|
|
92
|
+
return facts, prefixes, False
|
|
93
|
+
source = text.decode("utf-8", errors="replace") if isinstance(text, bytes) else text
|
|
94
|
+
prefixes.update(m[1] for m in _PREFIX_DECL.finditer(source) if m[1])
|
|
95
|
+
store = _Recorder()
|
|
96
|
+
try:
|
|
97
|
+
rdflib.Graph(store=store).parse(data=source, format="turtle", publicID=_RELATIVE_BASE)
|
|
98
|
+
except Exception as err: # rdflib raises several types for bad syntax
|
|
99
|
+
diagnostics.add("O001_ONTOLOGY_UNPARSEABLE", f"{file} is not parseable Turtle: {err}", file)
|
|
100
|
+
return facts, prefixes, False
|
|
101
|
+
triples = store.order
|
|
102
|
+
if len(triples) > limits["maxTriples"]:
|
|
103
|
+
diagnostics.add(
|
|
104
|
+
"O001_ONTOLOGY_UNPARSEABLE",
|
|
105
|
+
f"{file} has {len(triples)} triples, over the {limits['maxTriples']} limit",
|
|
106
|
+
file,
|
|
107
|
+
)
|
|
108
|
+
return facts, prefixes, False
|
|
109
|
+
|
|
110
|
+
lists: dict[Any, dict[str, Any]] = {}
|
|
111
|
+
for s, p, o in triples:
|
|
112
|
+
if isinstance(s, rdflib.BNode) and str(p) in (f"{RDF}first", f"{RDF}rest"):
|
|
113
|
+
lists.setdefault(s, {})["first" if str(p) == f"{RDF}first" else "rest"] = o
|
|
114
|
+
|
|
115
|
+
def add(fact: Fact) -> None:
|
|
116
|
+
facts.append({**fact, "layer": layer, "origin": origin})
|
|
117
|
+
|
|
118
|
+
for s, p, o in triples:
|
|
119
|
+
if str(p) == f"{OWL}imports":
|
|
120
|
+
diagnostics.add(
|
|
121
|
+
"O003_REMOTE_REFERENCE", f"owl:imports {_iri(o)} is not allowed; structure must be self-contained", file
|
|
122
|
+
)
|
|
123
|
+
for term in (s, p, o):
|
|
124
|
+
if isinstance(term, rdflib.URIRef) and (_is_relative(term) or not ABSOLUTE_IRI.match(str(term))):
|
|
125
|
+
diagnostics.add("O003_REMOTE_REFERENCE", f"IRI <{_iri(term)}> is not absolute", file)
|
|
126
|
+
if not isinstance(s, rdflib.URIRef):
|
|
127
|
+
continue
|
|
128
|
+
subject = _iri(s)
|
|
129
|
+
predicate = str(p)
|
|
130
|
+
if predicate == f"{RDF}type":
|
|
131
|
+
if str(o) == f"{SKOS}Concept":
|
|
132
|
+
add({"s": subject, "kind": "concept"})
|
|
133
|
+
if str(o) == f"{SKOS}OrderedCollection":
|
|
134
|
+
add({"s": subject, "kind": "collection"})
|
|
135
|
+
continue
|
|
136
|
+
if predicate == f"{SKOS}memberList":
|
|
137
|
+
node: Any = o
|
|
138
|
+
seen: set[Any] = set()
|
|
139
|
+
index = 0
|
|
140
|
+
while node is not None and isinstance(node, rdflib.BNode) and node not in seen:
|
|
141
|
+
seen.add(node)
|
|
142
|
+
cell = lists.get(node)
|
|
143
|
+
if not cell or cell.get("first") is None:
|
|
144
|
+
break
|
|
145
|
+
if isinstance(cell["first"], rdflib.URIRef):
|
|
146
|
+
add({"s": subject, "kind": "member", "o": _iri(cell["first"]), "index": index})
|
|
147
|
+
node = cell.get("rest")
|
|
148
|
+
index += 1
|
|
149
|
+
continue
|
|
150
|
+
known = PREDICATES.get(predicate)
|
|
151
|
+
if not known:
|
|
152
|
+
continue
|
|
153
|
+
kind, flip = known
|
|
154
|
+
if kind in ("label", "definition"):
|
|
155
|
+
if isinstance(o, rdflib.Literal):
|
|
156
|
+
add({"s": subject, "kind": kind, "value": str(o), "lang": o.language or ""})
|
|
157
|
+
elif kind == "deprecated":
|
|
158
|
+
if isinstance(o, rdflib.Literal) and str(o) == "true":
|
|
159
|
+
add({"s": subject, "kind": kind})
|
|
160
|
+
elif isinstance(o, rdflib.URIRef):
|
|
161
|
+
add({"s": _iri(o), "kind": kind, "o": subject} if flip else {"s": subject, "kind": kind, "o": _iri(o)})
|
|
162
|
+
ok = not any(d.code.startswith("O") for d in diagnostics.items[before:])
|
|
163
|
+
return facts, prefixes, ok
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@dataclass
|
|
167
|
+
class Concept:
|
|
168
|
+
iri: str
|
|
169
|
+
declared: dict[str, Any] | None
|
|
170
|
+
labels: dict[str, str]
|
|
171
|
+
definitions: dict[str, str]
|
|
172
|
+
deprecated: bool = False
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class StructureGraph:
|
|
176
|
+
"""An index over facts from any number of layers. The first layer to state a fact is the one reported."""
|
|
177
|
+
|
|
178
|
+
def __init__(self, facts: list[Fact]) -> None:
|
|
179
|
+
self.facts = sorted(facts, key=lambda f: LAYERS.index(f["layer"]))
|
|
180
|
+
self.concepts: dict[str, Concept] = {}
|
|
181
|
+
self.collections: dict[str, dict[str, Any]] = {}
|
|
182
|
+
self.edges: dict[str, dict[str, dict[str, dict[str, Any]]]] = {k: {} for k in RELATIONS}
|
|
183
|
+
self.inverse: dict[str, dict[str, dict[str, dict[str, Any]]]] = {k: {} for k in RELATIONS}
|
|
184
|
+
|
|
185
|
+
def concept(iri: str) -> Concept:
|
|
186
|
+
if iri not in self.concepts:
|
|
187
|
+
self.concepts[iri] = Concept(iri, None, {}, {})
|
|
188
|
+
return self.concepts[iri]
|
|
189
|
+
|
|
190
|
+
def edge(table: dict[str, dict[str, dict[str, Any]]], s: str, o: str, f: Fact) -> None:
|
|
191
|
+
targets = table.setdefault(s, {})
|
|
192
|
+
if o not in targets:
|
|
193
|
+
targets[o] = {"layer": f["layer"], "origin": f["origin"]}
|
|
194
|
+
|
|
195
|
+
members: dict[str, dict[int, str]] = {}
|
|
196
|
+
for f in self.facts:
|
|
197
|
+
kind = f["kind"]
|
|
198
|
+
if kind == "concept":
|
|
199
|
+
c = concept(f["s"])
|
|
200
|
+
if not c.declared:
|
|
201
|
+
c.declared = {"layer": f["layer"], "origin": f["origin"]}
|
|
202
|
+
elif kind in ("collection", "member"):
|
|
203
|
+
if f["s"] not in self.collections:
|
|
204
|
+
self.collections[f["s"]] = {
|
|
205
|
+
"iri": f["s"],
|
|
206
|
+
"layer": f["layer"],
|
|
207
|
+
"origin": f["origin"],
|
|
208
|
+
"members": [],
|
|
209
|
+
}
|
|
210
|
+
members[f["s"]] = {}
|
|
211
|
+
if kind == "member":
|
|
212
|
+
c2 = self.collections[f["s"]]
|
|
213
|
+
slots = members[f["s"]]
|
|
214
|
+
if c2["origin"] == f["origin"] and f["o"] not in slots.values():
|
|
215
|
+
slots[f["index"]] = f["o"]
|
|
216
|
+
elif kind == "label":
|
|
217
|
+
c = concept(f["s"])
|
|
218
|
+
c.labels.setdefault(f["lang"], f["value"])
|
|
219
|
+
elif kind == "definition":
|
|
220
|
+
c = concept(f["s"])
|
|
221
|
+
c.definitions.setdefault(f["lang"], f["value"])
|
|
222
|
+
elif kind == "deprecated":
|
|
223
|
+
concept(f["s"]).deprecated = True
|
|
224
|
+
elif kind in RELATIONS:
|
|
225
|
+
edge(self.edges[kind], f["s"], f["o"], f)
|
|
226
|
+
edge(self.inverse[kind], f["o"], f["s"], f)
|
|
227
|
+
for iri, c3 in self.collections.items():
|
|
228
|
+
c3["members"] = [m for _, m in sorted(members[iri].items()) if m]
|
|
229
|
+
|
|
230
|
+
def out(self, kind: str, iri: str) -> list[dict[str, Any]]:
|
|
231
|
+
"""IRIs a relation points to from ``iri``, sorted, with the layer that stated each."""
|
|
232
|
+
return js_sorted(({"iri": o, **at} for o, at in self.edges[kind].get(iri, {}).items()), key=lambda e: e["iri"])
|
|
233
|
+
|
|
234
|
+
def in_(self, kind: str, iri: str) -> list[dict[str, Any]]:
|
|
235
|
+
return js_sorted(
|
|
236
|
+
({"iri": s, **at} for s, at in self.inverse[kind].get(iri, {}).items()), key=lambda e: e["iri"]
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
def is_declared(self, iri: str) -> bool:
|
|
240
|
+
c = self.concepts.get(iri)
|
|
241
|
+
return bool(c and c.declared) or iri in self.collections
|
|
242
|
+
|
|
243
|
+
def cycle(self, kind: str) -> list[str] | None:
|
|
244
|
+
"""The first cycle found in a relation, as a list of IRIs, or None."""
|
|
245
|
+
state: dict[str, int] = {}
|
|
246
|
+
for start in js_sorted(self.edges[kind]):
|
|
247
|
+
if start in state:
|
|
248
|
+
continue
|
|
249
|
+
stack = [start]
|
|
250
|
+
state[start] = 1
|
|
251
|
+
pending = [iter(self.out(kind, start))]
|
|
252
|
+
while pending:
|
|
253
|
+
nxt = next(pending[-1], None)
|
|
254
|
+
if nxt is None:
|
|
255
|
+
state[stack.pop()] = 2
|
|
256
|
+
pending.pop()
|
|
257
|
+
continue
|
|
258
|
+
target = nxt["iri"]
|
|
259
|
+
if state.get(target) == 1:
|
|
260
|
+
return [*stack[stack.index(target) :], target]
|
|
261
|
+
if target not in state:
|
|
262
|
+
state[target] = 1
|
|
263
|
+
stack.append(target)
|
|
264
|
+
pending.append(iter(self.out(kind, target)))
|
|
265
|
+
return None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def preferred(values: dict[str, str]) -> str | None:
|
|
269
|
+
"""Preferred label: no language tag, then ``en``, then the first language in order."""
|
|
270
|
+
if "" in values:
|
|
271
|
+
return values[""]
|
|
272
|
+
if "en" in values:
|
|
273
|
+
return values["en"]
|
|
274
|
+
langs = js_sorted(values)
|
|
275
|
+
return values[langs[0]] if langs else None
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def structure_view(graph: StructureGraph) -> dict[str, Any]:
|
|
279
|
+
"""A JSON view of the structure, sorted so any two Readers produce the same bytes. Not a package file (moca-spec ADR-0019)."""
|
|
280
|
+
concepts = []
|
|
281
|
+
for c in sorted((c for c in graph.concepts.values() if c.declared), key=lambda c: utf16_key(c.iri)):
|
|
282
|
+
label, definition = preferred(c.labels), preferred(c.definitions)
|
|
283
|
+
|
|
284
|
+
def ids(kind: str, iri: str = c.iri) -> list[str]:
|
|
285
|
+
return [e["iri"] for e in graph.out(kind, iri)]
|
|
286
|
+
|
|
287
|
+
entry: dict[str, Any] = {"iri": c.iri}
|
|
288
|
+
if label is not None:
|
|
289
|
+
entry["label"] = label
|
|
290
|
+
if definition is not None:
|
|
291
|
+
entry["definition"] = definition
|
|
292
|
+
if c.deprecated:
|
|
293
|
+
entry["deprecated"] = True
|
|
294
|
+
related = js_sorted(dict.fromkeys([*ids("related"), *(e["iri"] for e in graph.in_("related", c.iri))]))
|
|
295
|
+
entry.update(
|
|
296
|
+
{
|
|
297
|
+
"broader": ids("broader"),
|
|
298
|
+
"related": related,
|
|
299
|
+
"parts": ids("hasPart"),
|
|
300
|
+
"requires": ids("requires"),
|
|
301
|
+
"replaces": ids("replaces"),
|
|
302
|
+
}
|
|
303
|
+
)
|
|
304
|
+
concepts.append(entry)
|
|
305
|
+
sequences = [
|
|
306
|
+
{"iri": c["iri"], "members": list(c["members"])}
|
|
307
|
+
for c in sorted(graph.collections.values(), key=lambda c: utf16_key(c["iri"]))
|
|
308
|
+
]
|
|
309
|
+
return {"structureVersion": 1, "concepts": concepts, "sequences": sequences}
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@dataclass
|
|
313
|
+
class PackageStructure:
|
|
314
|
+
present: bool
|
|
315
|
+
ok: bool
|
|
316
|
+
facts: list[Fact]
|
|
317
|
+
graph: StructureGraph | None
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def read_structure(
|
|
321
|
+
data: dict[str, bytes],
|
|
322
|
+
nodes: list[ContentNode],
|
|
323
|
+
diagnostics: Diagnostics,
|
|
324
|
+
limits: dict[str, int],
|
|
325
|
+
origin: str | None,
|
|
326
|
+
) -> PackageStructure:
|
|
327
|
+
"""Parses structure.ttl, checks node bindings, and sets each representation's concepts."""
|
|
328
|
+
ttl = data.get(STRUCTURE_FILE)
|
|
329
|
+
bound: list[Representation] = [
|
|
330
|
+
r for n in nodes for r in n.representations if isinstance(get(r.frontmatter.get("moca"), "concepts"), list)
|
|
331
|
+
]
|
|
332
|
+
if ttl is None and not bound:
|
|
333
|
+
return PackageStructure(False, False, [], None)
|
|
334
|
+
before = len(diagnostics.items)
|
|
335
|
+
|
|
336
|
+
if ttl is None:
|
|
337
|
+
for rep in bound:
|
|
338
|
+
diagnostics.add(
|
|
339
|
+
"O002_CONCEPT_UNDECLARED", f"concepts are bound but the package has no {STRUCTURE_FILE}", rep.file
|
|
340
|
+
)
|
|
341
|
+
return PackageStructure(False, False, [], None)
|
|
342
|
+
|
|
343
|
+
facts, prefixes, _ = parse_structure(ttl, STRUCTURE_FILE, "package", origin, diagnostics, limits)
|
|
344
|
+
graph = StructureGraph(facts)
|
|
345
|
+
for kind in ("requires", "broader", "hasPart"):
|
|
346
|
+
loop = graph.cycle(kind)
|
|
347
|
+
if loop:
|
|
348
|
+
diagnostics.add("O005_STRUCTURE_CYCLE", f"{kind} forms a cycle: {' -> '.join(loop)}", STRUCTURE_FILE)
|
|
349
|
+
referenced: set[str] = set()
|
|
350
|
+
for kind in RELATIONS:
|
|
351
|
+
for s, targets in graph.edges[kind].items():
|
|
352
|
+
referenced.add(s)
|
|
353
|
+
referenced.update(targets)
|
|
354
|
+
for c in graph.collections.values():
|
|
355
|
+
referenced.update(c["members"])
|
|
356
|
+
for iri in js_sorted(referenced):
|
|
357
|
+
if not graph.is_declared(iri):
|
|
358
|
+
diagnostics.add(
|
|
359
|
+
"O002_CONCEPT_UNDECLARED",
|
|
360
|
+
f"{iri} is used in a relation but not declared as a skos:Concept",
|
|
361
|
+
STRUCTURE_FILE,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
for rep in bound:
|
|
365
|
+
concepts: list[str] = []
|
|
366
|
+
for b in rep.frontmatter["moca"]["concepts"]:
|
|
367
|
+
iri = get(b, "iri")
|
|
368
|
+
if not isinstance(iri, str) or not ABSOLUTE_IRI.match(iri):
|
|
369
|
+
diagnostics.add("O003_REMOTE_REFERENCE", f'concept "{iri}" is not an absolute IRI', rep.file)
|
|
370
|
+
continue
|
|
371
|
+
if iri[: iri.index(":")] in prefixes and not graph.is_declared(iri):
|
|
372
|
+
diagnostics.add(
|
|
373
|
+
"O003_REMOTE_REFERENCE", f'concept "{iri}" is a prefixed name; write the absolute IRI', rep.file
|
|
374
|
+
)
|
|
375
|
+
continue
|
|
376
|
+
if not graph.is_declared(iri):
|
|
377
|
+
diagnostics.add(
|
|
378
|
+
"O002_CONCEPT_UNDECLARED", f"concept {iri} is not declared in {STRUCTURE_FILE}", rep.file
|
|
379
|
+
)
|
|
380
|
+
concepts.append(iri)
|
|
381
|
+
if concepts:
|
|
382
|
+
rep.concepts = list(dict.fromkeys(concepts))
|
|
383
|
+
|
|
384
|
+
ok = not any(d.code.startswith("O") for d in diagnostics.items[before:])
|
|
385
|
+
return PackageStructure(True, ok, facts, graph)
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Structure operations (Reader contract §10): lookups over the merged structure of every loaded package plus the
|
|
2
|
+
host's overlays. They are facts, so every Reader answers them identically; each result says which layer stated it."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
|
+
|
|
8
|
+
from ._js import js_sorted, utf16_key
|
|
9
|
+
from .citation import CitationRecord, passes_default_policy
|
|
10
|
+
from .structure import Fact, StructureGraph, preferred
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from collections.abc import Iterator
|
|
14
|
+
|
|
15
|
+
from .library import Library
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class StructureIndex:
|
|
19
|
+
def __init__(self, library: Library, overlay_facts: list[Fact] | None = None) -> None:
|
|
20
|
+
self.library = library
|
|
21
|
+
facts = [f for p in library.packages if p.result.structure is not None for f in p.result.structure.facts]
|
|
22
|
+
self.graph = StructureGraph([*facts, *(overlay_facts or [])])
|
|
23
|
+
|
|
24
|
+
def _item(self, iri: str, at: dict[str, Any]) -> dict[str, Any]:
|
|
25
|
+
c = self.graph.concepts.get(iri)
|
|
26
|
+
label = preferred(c.labels) if c else None
|
|
27
|
+
item: dict[str, Any] = {"iri": iri}
|
|
28
|
+
if label is not None:
|
|
29
|
+
item["label"] = label
|
|
30
|
+
item["layer"] = at["layer"]
|
|
31
|
+
return item
|
|
32
|
+
|
|
33
|
+
def concept(self, iri: str) -> dict[str, Any] | None:
|
|
34
|
+
"""A concept, or None when no layer declares it."""
|
|
35
|
+
c = self.graph.concepts.get(iri)
|
|
36
|
+
if c is None or not c.declared:
|
|
37
|
+
return None
|
|
38
|
+
label, definition = preferred(c.labels), preferred(c.definitions)
|
|
39
|
+
out: dict[str, Any] = {"iri": iri}
|
|
40
|
+
if label is not None:
|
|
41
|
+
out["label"] = label
|
|
42
|
+
out["labels"] = dict(c.labels)
|
|
43
|
+
if definition is not None:
|
|
44
|
+
out["definition"] = definition
|
|
45
|
+
out.update(
|
|
46
|
+
{
|
|
47
|
+
"deprecated": c.deprecated,
|
|
48
|
+
"replacedBy": [e["iri"] for e in self.graph.in_("replaces", iri)],
|
|
49
|
+
"replaces": [e["iri"] for e in self.graph.out("replaces", iri)],
|
|
50
|
+
"layer": c.declared["layer"],
|
|
51
|
+
}
|
|
52
|
+
)
|
|
53
|
+
return out
|
|
54
|
+
|
|
55
|
+
def requires(self, iri: str, transitive: bool = False) -> list[dict[str, Any]]:
|
|
56
|
+
"""What ``iri`` requires. Transitively, in dependency order: every concept after what it requires; ties by IRI."""
|
|
57
|
+
if not transitive:
|
|
58
|
+
return [self._item(e["iri"], e) for e in self.graph.out("requires", iri)]
|
|
59
|
+
# Depth first and iterative, so a long chain cannot exhaust the recursion limit:
|
|
60
|
+
# each concept is emitted after the concepts it requires.
|
|
61
|
+
out: list[dict[str, Any]] = []
|
|
62
|
+
done = {iri}
|
|
63
|
+
frames: list[tuple[dict[str, Any] | None, Iterator[dict[str, Any]]]] = [
|
|
64
|
+
(None, iter(self.graph.out("requires", iri)))
|
|
65
|
+
]
|
|
66
|
+
while frames:
|
|
67
|
+
edge, pending = frames[-1]
|
|
68
|
+
e = next(pending, None)
|
|
69
|
+
if e is None:
|
|
70
|
+
frames.pop()
|
|
71
|
+
if edge is not None:
|
|
72
|
+
out.append(self._item(edge["iri"], edge))
|
|
73
|
+
continue
|
|
74
|
+
if e["iri"] in done:
|
|
75
|
+
continue
|
|
76
|
+
done.add(e["iri"])
|
|
77
|
+
frames.append((e, iter(self.graph.out("requires", e["iri"]))))
|
|
78
|
+
return out
|
|
79
|
+
|
|
80
|
+
def required_by(self, iri: str) -> list[dict[str, Any]]:
|
|
81
|
+
return [self._item(e["iri"], e) for e in self.graph.in_("requires", iri)]
|
|
82
|
+
|
|
83
|
+
def parts(self, iri: str) -> list[dict[str, Any]]:
|
|
84
|
+
return [self._item(e["iri"], e) for e in self.graph.out("hasPart", iri)]
|
|
85
|
+
|
|
86
|
+
def narrower(self, iri: str, transitive: bool = False) -> list[dict[str, Any]]:
|
|
87
|
+
"""Narrower concepts; transitively, breadth first (by depth, then IRI)."""
|
|
88
|
+
out: list[dict[str, Any]] = []
|
|
89
|
+
seen = {iri}
|
|
90
|
+
level = [iri]
|
|
91
|
+
while level:
|
|
92
|
+
nxt: list[dict[str, Any]] = []
|
|
93
|
+
for n in level:
|
|
94
|
+
for e in self.graph.in_("broader", n):
|
|
95
|
+
if e["iri"] in seen:
|
|
96
|
+
continue
|
|
97
|
+
seen.add(e["iri"])
|
|
98
|
+
nxt.append(self._item(e["iri"], e))
|
|
99
|
+
nxt.sort(key=lambda x: utf16_key(x["iri"]))
|
|
100
|
+
out.extend(nxt)
|
|
101
|
+
if not transitive:
|
|
102
|
+
break
|
|
103
|
+
level = [x["iri"] for x in nxt]
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
def broader(self, iri: str) -> list[dict[str, Any]]:
|
|
107
|
+
return [self._item(e["iri"], e) for e in self.graph.out("broader", iri)]
|
|
108
|
+
|
|
109
|
+
def related(self, iri: str) -> list[dict[str, Any]]:
|
|
110
|
+
"""Related concepts, in both directions."""
|
|
111
|
+
seen: dict[str, dict[str, Any]] = {}
|
|
112
|
+
for e in [*self.graph.out("related", iri), *self.graph.in_("related", iri)]:
|
|
113
|
+
seen.setdefault(e["iri"], e)
|
|
114
|
+
return [self._item(e["iri"], e) for e in js_sorted(seen.values(), key=lambda e: e["iri"])]
|
|
115
|
+
|
|
116
|
+
def sequence(self, iri: str) -> list[dict[str, Any]] | None:
|
|
117
|
+
"""The members of an ordered collection, in order, or None."""
|
|
118
|
+
c = self.graph.collections.get(iri)
|
|
119
|
+
if c is None:
|
|
120
|
+
return None
|
|
121
|
+
return [self._item(m, c) for m in c["members"]]
|
|
122
|
+
|
|
123
|
+
def descendants(self, iri: str) -> list[str]:
|
|
124
|
+
"""Every concept below ``iri`` by hierarchy or parts, transitively."""
|
|
125
|
+
out: set[str] = set()
|
|
126
|
+
stack = [iri]
|
|
127
|
+
while stack:
|
|
128
|
+
n = stack.pop()
|
|
129
|
+
for e in [*self.graph.in_("broader", n), *self.graph.out("hasPart", n)]:
|
|
130
|
+
if e["iri"] not in out and e["iri"] != iri:
|
|
131
|
+
out.add(e["iri"])
|
|
132
|
+
stack.append(e["iri"])
|
|
133
|
+
return js_sorted(out)
|
|
134
|
+
|
|
135
|
+
def nodes(
|
|
136
|
+
self, iri: str, include: str = "none", locale: str | None = None, include_all: bool = False
|
|
137
|
+
) -> list[CitationRecord]:
|
|
138
|
+
"""Citation records of the nodes bound to ``iri`` (and to its narrower concepts or parts, by ``include``).
|
|
139
|
+
|
|
140
|
+
The default retrieval policy applies unless ``include_all`` is set. Sorted by node reference.
|
|
141
|
+
"""
|
|
142
|
+
wanted = {iri}
|
|
143
|
+
if include != "none":
|
|
144
|
+
stack = [iri]
|
|
145
|
+
while stack:
|
|
146
|
+
n = stack.pop()
|
|
147
|
+
nxt = [
|
|
148
|
+
*(self.graph.in_("broader", n) if include in ("narrower", "both") else []),
|
|
149
|
+
*(self.graph.out("hasPart", n) if include in ("parts", "both") else []),
|
|
150
|
+
]
|
|
151
|
+
for e in nxt:
|
|
152
|
+
if e["iri"] not in wanted:
|
|
153
|
+
wanted.add(e["iri"])
|
|
154
|
+
stack.append(e["iri"])
|
|
155
|
+
records = [
|
|
156
|
+
r
|
|
157
|
+
for r in self.library.citations(locale, include_text=True)
|
|
158
|
+
if any(c in wanted for c in r.get("concepts", [])) and (include_all or passes_default_policy(r))
|
|
159
|
+
]
|
|
160
|
+
return sorted(records, key=lambda r: utf16_key(r["node"]["ref"]))
|
moca_reader/unicode.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Lower-casing and normalisation, defined once.
|
|
2
|
+
|
|
3
|
+
Package spec §6.3: each code point is replaced with its Unicode simple lowercase mapping, one at a
|
|
4
|
+
time, with no language or context rules. İ (U+0130) becomes i, and capital sigma always becomes small sigma. Python's
|
|
5
|
+
``str.lower`` is the full mapping (İ gives i + U+0307), so it is applied one code point at a time
|
|
6
|
+
and corrected where the full and simple mappings differ.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import unicodedata
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def lower_code_point(ch: str) -> str:
|
|
15
|
+
"""The simple lowercase mapping of one code point."""
|
|
16
|
+
lowered = ch.lower()
|
|
17
|
+
if lowered == "i̇":
|
|
18
|
+
return "i"
|
|
19
|
+
# Any other multi-code-point result has no simple mapping: the code point stays as it is.
|
|
20
|
+
return lowered if len(lowered) == 1 else ch
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def simple_lowercase(text: str) -> str:
|
|
24
|
+
"""A string lower-cased one code point at a time."""
|
|
25
|
+
return "".join(lower_code_point(ch) for ch in text)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def normalize(form: str, text: str) -> str:
|
|
29
|
+
"""``unicodedata.normalize`` that never raises: lone surrogates and noncharacters stay in place."""
|
|
30
|
+
try:
|
|
31
|
+
return unicodedata.normalize(form, text) # type: ignore[arg-type]
|
|
32
|
+
except (ValueError, UnicodeError): # pragma: no cover - CPython keeps them as they are
|
|
33
|
+
return text
|