moca-reader 0.6.0__py3-none-any.whl

moca_reader/library.py

@@ -0,0 +1,176 @@
+"""A library of loaded packages: citation records, relations between packages, and their merged structure with
+the host's overlays (Reader contract §7-§10). Search over a library is in search.py."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+from ._js import get, js_string
+from .citation import CitationRecord, RelationIndex, citation_for, pick_representation
+from .diagnostics import Diagnostics
+from .relations import relation_index_of
+from .source import DEFAULT_LIMITS
+from .structure import LAYERS, Fact, parse_structure
+
+if TYPE_CHECKING:
+    from .reader import PackageResult
+    from .structure_ops import StructureIndex
+
+
+@dataclass(frozen=True)
+class Overlay:
+    """Host-supplied structure (moca-spec ADR-0008), never package content: Turtle text or a file."""
+
+    id: str
+    layer: str  # application | organisation
+    source: str | bytes | None = None
+    path: str | os.PathLike[str] | None = None
+
+
+@dataclass
+class LoadedPackage:
+    result: PackageResult
+    chunks: list[dict[str, Any]] | None = None
+    index: Any = None
+
+
+class Library:
+    """Holds verified packages, with their sidecar chunks when bound."""
+
+    def __init__(
+        self,
+        clock: Callable[[], datetime] | None = None,
+        overlays: list[Overlay] | None = None,
+        limits: dict[str, int] | None = None,
+        allow_unsigned_relations: bool = False,
+    ) -> None:
+        self.clock: Callable[[], datetime] = clock or (lambda: datetime.now(UTC))
+        # Package spec §7.2: off by default. Only for libraries of the host's own packages.
+        self.allow_unsigned_relations = allow_unsigned_relations
+        self.packages: list[LoadedPackage] = []
+        self.diagnostics = Diagnostics()
+        self.overlay_facts: list[Fact] = []
+        limit_values = {**DEFAULT_LIMITS, **(limits or {})}
+        for o in overlays or []:
+            if o.layer not in ("application", "organisation"):
+                raise ValueError(f"overlay {o.id}: layer must be application or organisation")
+            text: str | bytes
+            if o.source is not None:
+                text = o.source
+            elif o.path is not None:
+                with open(o.path, "rb") as f:
+                    text = f.read()
+            else:
+                raise ValueError(f"overlay {o.id}: give its source or its path")
+            facts, _, _ = parse_structure(
+                text, f"overlay:{o.id}", o.layer, f"overlay:{o.id}", self.diagnostics, limit_values
+            )
+            self.overlay_facts.extend(facts)
+        self.overlay_facts.sort(key=lambda f: LAYERS.index(f["layer"]))
+        self._structure: StructureIndex | None = None
+
+    @property
+    def structure(self) -> StructureIndex:
+        """The merged structure of every loaded package and overlay (Reader contract §10)."""
+        from .structure_ops import StructureIndex
+
+        if self._structure is None:
+            self._structure = StructureIndex(self, self.overlay_facts)
+        return self._structure
+
+    def add(self, result: PackageResult, chunks: list[dict[str, Any]] | None = None, index: Any = None) -> Library:
+        """Adds a package read by read_package(), with a bound sidecar's chunks and index. Invalid packages are refused."""
+        if not result.valid or not result.digest:
+            raise ValueError(f"refusing to add an invalid package: {get(result.manifest, 'id') or '(unknown)'}")
+        key = _key(result)
+        same = next((p for p in self.packages if _key(p.result) == key), None)
+        if same is not None:
+            # Contract §7: one id@version, one digest. A second digest is refused and reported.
+            if same.result.digest != result.digest:
+                self.diagnostics.add(
+                    "L001_IDENTITY_CONFLICT",
+                    f"{key} is already loaded with digest {same.result.digest}; {result.digest} is refused",
+                )
+            return self
+        self.packages.append(LoadedPackage(result, chunks, index))
+        self._structure = None
+        # A composed package brings its verified members with it.
+        for member in result.members:
+            if member.status == "resolved" and member.result is not None and member.result.valid:
+                self.add(member.result)
+        return self
+
+    def relation_index(self) -> RelationIndex:
+        """Package-level relation effects among the loaded packages (package spec §7.2)."""
+        return relation_index_of(
+            [
+                {
+                    "id": p.result.manifest.get("id"),
+                    "version": p.result.manifest.get("version"),
+                    "relations": p.result.manifest.get("relations"),
+                    "relationScopes": p.result.relation_scopes,
+                }
+                for p in self.packages
+            ],
+            self.allow_unsigned_relations,
+        )
+
+    def citations(self, locale: str | None = None, include_text: bool = False) -> list[CitationRecord]:
+        """Citation records for every node: one per node, in the requested locale."""
+        index = self.relation_index()
+        now = self.clock()
+        out = []
+        for p in self.packages:
+            for node in p.result.nodes:
+                rep = pick_representation(node, locale)
+                out.append(citation_for(p.result, node, rep, now, index, rep.body if include_text else None))
+        return out
+
+    def get(self, node_ref: str, locale: str | None = None, include_text: bool = True) -> CitationRecord | None:
+        """A node by its id (``<package id>#<path>``) or versioned reference (``<package id>@<version>#<path>``).
+
+        An id matches the first loaded version.
+        """
+        hash_at = node_ref.rfind("#")
+        if hash_at < 0:
+            return None
+        path, head = node_ref[hash_at + 1 :], node_ref[:hash_at]
+        index = self.relation_index()
+        for p in self.packages:
+            pid, version = js_string(p.result.manifest.get("id")), js_string(p.result.manifest.get("version"))
+            if head not in (pid, f"{pid}@{version}"):
+                continue
+            node = next((n for n in p.result.nodes if n.path == path), None)
+            if node is None:
+                continue
+            rep = pick_representation(node, locale)
+            return citation_for(p.result, node, rep, self.clock(), index, rep.body if include_text else None)
+        return None
+
+    def search(
+        self,
+        query: str,
+        *,
+        limit: int = 5,
+        include_all: bool = False,
+        locale: str | None = None,
+        concepts: list[str] | None = None,
+        scope: str | None = None,
+        audiences: list[str] | None = None,
+        allow_unlabelled: bool = False,
+    ) -> list[CitationRecord]:
+        """Lexical search over the loaded packages; a shortcut for ``Search(library, LexicalBackend(library))``."""
+        from .backends.lexical import LexicalBackend
+        from .search import Search
+
+        return Search(self, LexicalBackend(self), audiences=audiences, allow_unlabelled=allow_unlabelled).search(
+            query, limit=limit, include_all=include_all, locale=locale, concepts=concepts, scope=scope
+        )
+
+
+def _key(result: PackageResult) -> str:
+    return f"{js_string(result.manifest.get('id'))}@{js_string(result.manifest.get('version'))}"

moca_reader/manifest.py

@@ -0,0 +1,186 @@
+"""Manifest checks beyond the JSON Schema (package spec §4)."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from typing import Any
+
+from ._js import get, parse_date, truthy
+from .diagnostics import Diagnostics
+from .schemas import validate_against
+
+MANIFEST_KEYS = {
+    "$schema",
+    "mocaVersion",
+    "id",
+    "version",
+    "title",
+    "description",
+    "language",
+    "locales",
+    "license",
+    "publisher",
+    "validFrom",
+    "validUntil",
+    "members",
+    "relations",
+    "profiles",
+}
+# URN namespaces registered with IANA that are commonly used as package ids. Not exhaustive: M007 is informational.
+# Package spec §4.2: the IANA formal URN namespaces, as in moca-spec data/urn-namespaces.json (a test keeps
+# them equal). Informal namespaces, urn-<number>, are registered too.
+REGISTERED_URN_NIDS = frozenset(
+    {
+        "3gpp",
+        "3gpp2",
+        "adid",
+        "alert",
+        "bbf",
+        "broadband-forum-org",
+        "c2pa",
+        "cablelabs",
+        "ccsds",
+        "cdx",
+        "cgi",
+        "clei",
+        "csa",
+        "cta",
+        "ddi",
+        "dev",
+        "dgiwg",
+        "dnp",
+        "doi",
+        "dslforum-org",
+        "dvb",
+        "ebu",
+        "eic",
+        "eidr",
+        "epc",
+        "epcglobal",
+        "etsi",
+        "eurosystem",
+        "example",
+        "fdc",
+        "fipa",
+        "gdr",
+        "gdst",
+        "geant",
+        "globus",
+        "gs1",
+        "gsma",
+        "gvat",
+        "hbbtv",
+        "ieee",
+        "ietf",
+        "iptc",
+        "isan",
+        "isbn",
+        "isni",
+        "iso",
+        "issn",
+        "itu",
+        "ivis",
+        "knx",
+        "lei",
+        "lex",
+        "liberty",
+        "mace",
+        "mef",
+        "meta",
+        "mpeg",
+        "mrn",
+        "nan",
+        "nato",
+        "nbn",
+        "nena",
+        "newsml",
+        "nfc",
+        "nfi",
+        "nzl",
+        "oasis",
+        "ogc",
+        "ogf",
+        "oid",
+        "oipf",
+        "oma",
+        "onem2m",
+        "onf",
+        "pin",
+        "pno",
+        "publicid",
+        "pwid",
+        "reso",
+        "s1000d",
+        "said",
+        "schac",
+        "service",
+        "smpte",
+        "stalwart",
+        "swift",
+        "thread",
+        "trivore",
+        "tva",
+        "uci",
+        "ucode",
+        "uic",
+        "uuid",
+        "web3d",
+        "wfa",
+        "wmo",
+        "xmlorg",
+        "xmpp",
+    }
+)
+_INFORMAL_URN_NID = re.compile(r"urn-[0-9]+")
+_VENDOR_KEY = re.compile(r"^x-[a-z0-9]+-[A-Za-z0-9-]+\Z")
+_URN = re.compile(r"^urn:([^:]+):", re.IGNORECASE)
+
+
+def check_manifest(manifest: Any, diagnostics: Diagnostics, known_profiles: Iterable[str] = ()) -> None:
+    file = "moca.json"
+    for err in validate_against("manifest", manifest):
+        diagnostics.add("M002_MANIFEST_SCHEMA", err, file)
+    if not isinstance(manifest, dict):
+        return
+
+    for key in manifest:
+        if key not in MANIFEST_KEYS and not _VENDOR_KEY.match(key):
+            diagnostics.add(
+                "M004_UNKNOWN_KEY",
+                f'"{key}" is not a MOCA manifest key; it is ignored. Vendor keys use x-<vendor>-<key>.',
+                file,
+            )
+
+    seen: list[Any] = []
+    members = manifest.get("members")
+    for m in members if isinstance(members, list) else []:
+        member_id = get(m, "id")
+        if not truthy(member_id):
+            continue
+        if member_id in seen:
+            diagnostics.add("M005_DUPLICATE_MEMBER", f"member {member_id} is listed more than once", file)
+        seen.append(member_id)
+
+    start = parse_date(manifest.get("validFrom"))
+    until = parse_date(manifest.get("validUntil"))
+    if start is not None and until is not None and until <= start:
+        diagnostics.add("M006_VALIDITY_WINDOW_INVALID", "validUntil must be later than validFrom", file)
+
+    urn = _URN.match(manifest["id"]) if isinstance(manifest.get("id"), str) else None
+    if urn and urn[1].lower() not in REGISTERED_URN_NIDS and not _INFORMAL_URN_NID.fullmatch(urn[1].lower()):
+        diagnostics.add(
+            "M007_UNREGISTERED_URN",
+            f'"urn:{urn[1]}:" is not a registered URN namespace (RFC 8141); prefer an https: or tag: URI',
+            file,
+        )
+
+    known = set(known_profiles)
+    profiles = manifest.get("profiles")
+    for uri in profiles if isinstance(profiles, dict) else []:
+        if uri not in known:
+            diagnostics.add(
+                "F001_PROFILE_UNRECOGNISED",
+                f"profile {uri} is not recognised; its data is preserved but not interpreted",
+                file,
+            )

moca_reader/metadata.py

@@ -0,0 +1,58 @@
+"""Flat metadata from a citation record, for stores and frameworks that need flat fields.
+
+The names are the consuming guide's ("Feeding your own retrieval stack", moca-spec docs/guides/consuming.md),
+which the Reader interface §7 requires a binding to use when its framework needs flat metadata. Values are
+strings, numbers, booleans or lists of strings; a field with no value is left out, since many stores refuse null.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .citation import CitationRecord
+
+FLAT_FIELDS = (
+    "moca_package_id",
+    "moca_package_version",
+    "moca_package_digest",
+    "moca_node_id",
+    "moca_node_digest",
+    "moca_node_ref",
+    "moca_status",
+    "moca_in_force_until",
+    "moca_stale_after",
+    "moca_superseded",
+    "moca_attested_reviews",
+    "moca_signed",
+    "moca_audience",
+    "moca_concepts",
+    "moca_evidence_matched",
+)
+
+
+def flat_metadata(record: CitationRecord) -> dict[str, Any]:
+    """The consuming guide's ``moca_*`` fields for one citation record.
+
+    ``moca_attested_reviews`` is how many attested reviews the node has; ``moca_evidence_matched`` is true when
+    every checked evidence quote was found in its source, false when one was not, and absent when none was checked.
+    """
+    package, node, trust = record["package"], record["node"], record["trust"]
+    checked = [e["matched"] for e in record.get("evidence", []) if isinstance(e.get("matched"), bool)]
+    values: dict[str, Any] = {
+        "moca_package_id": package.get("id"),
+        "moca_package_version": package.get("version"),
+        "moca_package_digest": package.get("digest"),
+        "moca_node_id": node.get("id"),
+        "moca_node_digest": node.get("digest"),
+        "moca_node_ref": node.get("ref"),
+        "moca_status": trust.get("status"),
+        "moca_in_force_until": trust.get("validUntil"),
+        "moca_stale_after": trust.get("staleAfter"),
+        "moca_superseded": trust.get("superseded"),
+        "moca_attested_reviews": len(trust.get("attestedReviews", [])),
+        "moca_signed": package.get("signed"),
+        "moca_audience": record.get("audience"),
+        "moca_concepts": list(record["concepts"]) if record.get("concepts") else None,
+        "moca_evidence_matched": all(checked) if checked else None,
+    }
+    return {k: v for k, v in values.items() if v is not None}

moca_reader/profiles.py

@@ -0,0 +1,5 @@
+"""Profiles this Reader recognises. Recognising a profile only stops F001 from being reported; profile data
+stays opaque to core."""
+
+PROFILE_ONTOLOGY = "https://w3id.org/moca/profiles/ontology/v1"
+KNOWN_PROFILES = (PROFILE_ONTOLOGY,)