katalyst-core 0.1.1a1__py3-none-any.whl

katalyst_core/__init__.py

@@ -0,0 +1,178 @@
+"""katalyst-core — the LinkML module every Katalyst schema module imports.
+
+The schema itself is ``schema/katalyst_core.yaml``; nothing here defines a class or a
+slot. This package locates the module, opens LinkML modules that import it, holds them
+to the core contract (:mod:`katalyst_core.contract`) and moves records between their
+YAML form and their graph form (:mod:`katalyst_core.graph`).
+"""
+
+# linkml_runtime ships no type information.
+# pyright: reportMissingTypeStubs=false, reportUnknownMemberType=false
+# pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from functools import cache
+from importlib.metadata import entry_points
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, cast
+
+import yaml
+
+if TYPE_CHECKING:
+    # LinkML is imported where a view is opened, not with the package: a command that
+    # only reads definitions (from the schema cache) never pays for it.
+    from linkml_runtime.utils.schemaview import SchemaView
+
+__all__ = [
+    "CORE_IMPORT",
+    "MODULES_GROUP",
+    "SCHEMA_PATH",
+    "enum_values",
+    "installed_modules",
+    "kinds",
+    "local_modules",
+    "namespaces",
+    "open_installed",
+    "open_module",
+    "register_modules",
+]
+
+SCHEMA_PATH = Path(__file__).parent / "schema" / "katalyst_core.yaml"
+"""The katalyst-core LinkML module."""
+
+CORE_IMPORT = "katalyst_core"
+"""The name a module lists under ``imports:`` to import katalyst-core."""
+
+
+MODULES_GROUP = "katalyst_core.modules"
+"""Entry-point group a package registers its LinkML modules under.
+
+Each entry point loads to a mapping of module name (what an ``imports:`` line says) to
+the module's YAML file. A plugin ships its schema this way; core knows no plugin.
+"""
+
+
+_REGISTERED: dict[str, Path] = {}
+
+
+def register_modules(modules: Mapping[str, Path]) -> None:
+    """Add modules for this process: a LinkML schema bundle installed from a source."""
+    _REGISTERED.update({str(k): Path(v) for k, v in modules.items()})
+
+
+def installed_modules() -> dict[str, Path]:
+    """Every installed LinkML module by name, katalyst-core first.
+
+    Packages' modules (``katalyst_core.modules``), then any a schema bundle registered.
+    """
+    modules: dict[str, Path] = {CORE_IMPORT: SCHEMA_PATH, **_packaged_modules()}
+    modules.update(_REGISTERED)
+    return modules
+
+
+@cache
+def _packaged_modules() -> dict[str, Path]:
+    """The modules installed packages register (the entry points do not change in a process)."""
+    modules: dict[str, Path] = {}
+    for ep in sorted(entry_points(group=MODULES_GROUP), key=lambda e: e.name):
+        provided: Mapping[str, Any] = ep.load()
+        for name, path in provided.items():
+            modules[str(name)] = Path(path)
+    return modules
+
+
+def _importmap(modules: Mapping[str, Path]) -> dict[str, Any]:
+    return {name: str(path.with_suffix("")) for name, path in modules.items()}
+
+
+def local_modules(paths: Iterable[Path]) -> dict[str, Path]:
+    """Local module files (not installed) by name: the ``name`` each one declares.
+
+    Each is held to the rules for local modules first (:mod:`katalyst_core.local`): it
+    extends the installed baseline only by subtyping, against the version it pins.
+
+    Raises:
+        katalyst_core.local.LocalModuleError: A module breaks those rules.
+    """
+    from katalyst_core.local import check_local_module
+
+    found: dict[str, Path] = {}
+    baseline = installed_modules()
+    for path in paths:
+        raw: Any = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        name = str(raw.get("name") or path.stem) if isinstance(raw, dict) else path.stem
+        check_local_module(path, baseline)
+        found[name] = path.resolve()
+    return found
+
+
+def open_module(path: Path | None = None, extra: Mapping[str, Path] | None = None) -> SchemaView:
+    """Open a LinkML module (katalyst-core itself by default).
+
+    It may import katalyst-core (``katalyst_core``), any installed module and any module
+    in ``extra`` by name, wherever they live.
+    """
+    from linkml_runtime.utils.schemaview import SchemaView
+
+    importmap = _importmap({**installed_modules(), **(extra or {})})
+    return SchemaView(str(path or SCHEMA_PATH), importmap=importmap)
+
+
+def open_installed(extra: Mapping[str, Path] | None = None) -> SchemaView:
+    """Every installed module at once, and ``extra`` local ones: a view importing all."""
+    from linkml_runtime.linkml_model.meta import SchemaDefinition
+    from linkml_runtime.utils.schemaview import SchemaView
+
+    modules = {**installed_modules(), **(extra or {})}
+    root = SchemaDefinition(
+        id="https://w3id.org/katalyst/installed",
+        name="installed",
+        imports=["linkml:types", *modules],
+        prefixes=cast(Any, {"linkml": "https://w3id.org/linkml/"}),
+        default_prefix="linkml",
+    )
+    return SchemaView(root, importmap=_importmap(modules))
+
+
+def namespaces(sv: SchemaView) -> dict[str, str]:
+    """Every prefix declared by the module open in ``sv`` or by any module it imports.
+
+    ``SchemaView.namespaces()`` gives only the root module's. A compiled schema
+    (:mod:`katalyst_core.compiled`) holds them already.
+    """
+    compiled = getattr(sv, "compiled_namespaces", None)
+    if compiled is not None:
+        return dict(compiled())
+    merged: dict[str, str] = {}
+    for name in reversed(sv.imports_closure()):
+        schema: Any = (sv.schema_map or {}).get(name)
+        prefixes: dict[str, Any] = (schema.prefixes or {}) if schema else {}
+        for prefix in prefixes.values():
+            merged[str(prefix.prefix_prefix)] = str(prefix.prefix_reference)
+    merged.update({str(k): str(v) for k, v in sv.namespaces().items()})
+    return merged
+
+
+def enum_values(sv: SchemaView, name: str) -> list[str]:
+    """The permissible values of enum ``name``; empty when no loaded module defines it."""
+    enum: Any = sv.get_enum(name)
+    values: dict[str, Any] = (enum.permissible_values or {}) if enum else {}
+    return [str(v) for v in values]
+
+
+def kinds(sv: SchemaView) -> dict[str, str]:
+    """Node kind to class, for every class that declares a ``kind`` annotation."""
+    found: dict[str, str] = {}
+    for name in sv.all_classes():
+        cls: Any = sv.get_class(name)
+        annotations: Any = cls.annotations or {}
+        kind: Any = (
+            annotations.get("kind")
+            if isinstance(annotations, dict)
+            else getattr(annotations, "kind", None)
+        )
+        if kind is not None:
+            found[str(getattr(kind, "value", kind))] = str(name)
+    return found

katalyst_core/cache.py

@@ -0,0 +1,153 @@
+"""What is built from the installed LinkML modules, kept on disk between processes.
+
+Opening the modules (LinkML's ``SchemaView``) and building from them takes about a second,
+and every command needs what they define. So what is built from them is kept as JSON in
+the active home's cache, keyed by a digest of everything it was built from:
+
+- the contents of every installed module (and every module a schema bundle registered),
+- the source of the code that built it (each builder names its own files), and
+- the builder's own key parts (the modules asked for, a format number), and
+- the versions of the libraries the build reads them with (LinkML, rdflib, katalyst-core).
+
+Any change to one of them gives a new key, so a stale entry is never read; writing the
+new entry removes the old ones of the same name and parts. An entry is written to a temporary file and
+renamed into place, so a reader never sees half of one; an entry that cannot be read is
+built again and replaced. Without a cache directory (no home, or one that cannot be
+written) everything is built in the process, as before.
+
+The runtime that owns the home says where the cache lives
+(:func:`set_cache_dir_provider`); katalyst-core knows no home.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+from collections.abc import Callable, Iterable
+from functools import cache
+from pathlib import Path
+from typing import Any, TypeVar
+
+__all__ = ["cache_dir", "cached_json", "modules_digest", "set_cache_dir_provider"]
+
+_provider: Callable[[], Path | None] | None = None
+
+T = TypeVar("T")
+
+
+def _same(data: Any) -> Any:
+    return data
+
+
+def set_cache_dir_provider(provider: Callable[[], Path | None] | None) -> None:
+    """Say where the schema cache lives: a callable giving the directory, or ``None``."""
+    global _provider
+    _provider = provider
+
+
+def cache_dir() -> Path | None:
+    """The schema cache directory, or ``None`` when there is none to use."""
+    if _provider is None:
+        return None
+    try:
+        return _provider()
+    except Exception:  # no home resolves: build in the process instead
+        return None
+
+
+def modules_digest() -> str:
+    """A digest of every installed module's name and contents."""
+    from katalyst_core import installed_modules
+
+    modules = sorted(installed_modules().items())
+    stamp = tuple((name, str(path), *_stat(path)) for name, path in modules)
+    return _digest(stamp)
+
+
+def _stat(path: Path) -> tuple[int, int]:
+    info = path.stat()
+    return info.st_mtime_ns, info.st_size
+
+
+@cache
+def _digest(stamp: tuple[tuple[Any, ...], ...]) -> str:
+    """The contents' digest, read once per process for the same files (path, mtime, size)."""
+    digest = hashlib.sha256()
+    for name, path, *_ in stamp:
+        digest.update(str(name).encode())
+        digest.update(b"\0")
+        digest.update(Path(path).read_bytes())
+        digest.update(b"\0")
+    return digest.hexdigest()
+
+
+#: Libraries whose version changes what is built from the same modules.
+_LIBRARIES = ("linkml-runtime", "linkml", "rdflib", "katalyst-core")
+
+
+@cache
+def _library_versions() -> str:
+    from importlib.metadata import PackageNotFoundError, version
+
+    found: list[str] = []
+    for name in _LIBRARIES:
+        try:
+            found.append(f"{name}={version(name)}")
+        except PackageNotFoundError:
+            found.append(f"{name}=")
+    return ";".join(found)
+
+
+def _key(kind: str, sources: Iterable[Path], parts: Iterable[str]) -> str:
+    digest = hashlib.sha256(kind.encode())
+    digest.update(_library_versions().encode())
+    digest.update(modules_digest().encode())
+    for source in sources:
+        digest.update(Path(source).read_bytes())
+    for part in parts:
+        digest.update(b"\0" + part.encode())
+    return digest.hexdigest()[:32]
+
+
+def cached_json(
+    kind: str,
+    build: Callable[[], Any],
+    *,
+    load: Callable[[Any], T] = _same,
+    sources: Iterable[Path] = (),
+    parts: Iterable[str] = (),
+    directory: Path | None = None,
+) -> T:
+    """``load(build())``, with what *build* returns (JSON data) kept in the cache.
+
+    *kind* names the entry (``definitions``, ``compiled``); *sources* are the files of
+    the code that builds it, *parts* any further key strings. An entry that cannot be
+    read or that *load* refuses is built again and replaced. *directory* overrides
+    :func:`cache_dir`.
+    """
+    where = directory if directory is not None else cache_dir()
+    if where is None:
+        return load(build())
+    parts = tuple(parts)
+    family = f"{kind}-{hashlib.sha256(chr(0).join(parts).encode()).hexdigest()[:8]}"
+    path = where / f"{family}-{_key(kind, sources, parts)}.json"
+    try:
+        return load(json.loads(path.read_text(encoding="utf-8")))
+    except Exception:  # absent, unreadable or refused: build it again
+        pass
+    data = build()
+    result = load(data)
+    try:
+        where.mkdir(parents=True, exist_ok=True)
+        fd, partial = tempfile.mkstemp(dir=where, prefix=f".{kind}-", suffix=".tmp")
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            json.dump(data, handle, separators=(",", ":"))
+        os.replace(partial, path)
+        for stale in where.glob(f"{family}-*.json"):
+            if stale != path:
+                stale.unlink(missing_ok=True)
+    except OSError:  # a cache that cannot be written is only slower
+        pass
+    return result

katalyst_core/compiled.py

@@ -0,0 +1,281 @@
+"""The installed modules compiled for the write path: what a ``SchemaView`` answers there.
+
+Putting a node through the local shapes needs, for each class, its URI, its induced
+slots (range, cardinality, inlining, annotations), its ancestors, and the modules'
+namespaces, enums and types. LinkML's ``SchemaView`` computes them, but opening one costs
+more than the rest of a write. :func:`compile_view` asks a view once for everything the
+write path reads, :class:`CompiledSchema` answers the same questions from that data, and
+:func:`installed_compiled` keeps it in the schema cache (:mod:`katalyst_core.cache`),
+keyed by the modules' contents and this file's source.
+
+:class:`CompiledSchema` implements only the part of the ``SchemaView`` interface that
+:mod:`katalyst_core.graph` (writing records to the graph), :mod:`katalyst_core.shapes`
+(running shapes and wording their findings) and the taxonomy's write path use; anything
+else needs a real view.
+"""
+
+# linkml_runtime ships no type information.
+# pyright: reportMissingTypeStubs=false, reportUnknownMemberType=false
+# pyright: reportUnknownVariableType=false, reportUnknownArgumentType=false
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, cast
+
+if TYPE_CHECKING:
+    from linkml_runtime.utils.schemaview import SchemaView
+
+__all__ = ["CompiledSchema", "compile_view", "installed_compiled"]
+
+#: The JSON form below; bump on a change to it.
+_FORMAT = "1"
+
+#: Slots :mod:`katalyst_core.graph` looks up by name (the extension bag).
+_NAMED_SLOTS = ("extensions", "key", "value")
+
+
+def _plain(value: Any) -> Any:
+    value = getattr(value, "value", value)
+    if hasattr(value, "_as_dict"):
+        value = value._as_dict
+    if isinstance(value, dict):
+        raw = cast("dict[Any, Any]", value)
+        keys = {str(k) for k in raw}
+        if {"tag", "value"} <= keys <= {"tag", "value", "extensions", "annotations"}:
+            # An annotation, in the form an induced slot holds it.
+            return _plain(raw["value"])
+        return {str(k): _plain(v) for k, v in raw.items()}
+    if isinstance(value, (list, tuple)):
+        return [_plain(v) for v in cast("list[Any]", value)]
+    return value
+
+
+def _annotations(element: Any) -> dict[str, Any]:
+    raw: Any = getattr(element, "annotations", None) or {}
+    if not isinstance(raw, dict):
+        raw = getattr(raw, "_as_dict", {}) or {}
+    return {str(k): _plain(v) for k, v in cast("dict[Any, Any]", raw).items()}
+
+
+@dataclass(frozen=True)
+class _Range:
+    range: str | None
+
+
+@dataclass(frozen=True)
+class CompiledSlot:
+    """A slot as a class induces it: what the write path reads of a ``SlotDefinition``."""
+
+    name: str
+    uri: str
+    range: str | None = None
+    multivalued: bool = False
+    required: bool = False
+    inlined: bool = False
+    inlined_as_list: bool = False
+    identifier: bool = False
+    designates_type: bool = False
+    any_of: tuple[_Range, ...] = ()
+    annotations: dict[str, Any] = field(default_factory=lambda: {})
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "uri": self.uri,
+            "range": self.range,
+            "multivalued": self.multivalued,
+            "required": self.required,
+            "inlined": self.inlined,
+            "inlined_as_list": self.inlined_as_list,
+            "identifier": self.identifier,
+            "designates_type": self.designates_type,
+            "any_of": [r.range for r in self.any_of],
+            "annotations": self.annotations,
+        }
+
+    @staticmethod
+    def of(data: dict[str, Any]) -> CompiledSlot:
+        return CompiledSlot(
+            name=str(data["name"]),
+            uri=str(data["uri"]),
+            range=data["range"],
+            multivalued=bool(data["multivalued"]),
+            required=bool(data["required"]),
+            inlined=bool(data["inlined"]),
+            inlined_as_list=bool(data["inlined_as_list"]),
+            identifier=bool(data["identifier"]),
+            designates_type=bool(data["designates_type"]),
+            any_of=tuple(_Range(r) for r in data["any_of"]),
+            annotations=dict(data["annotations"]),
+        )
+
+
+@dataclass(frozen=True)
+class CompiledClass:
+    """A class: what the write path reads of a ``ClassDefinition``."""
+
+    name: str
+    uri: str
+    class_uri: str | None
+    annotations: dict[str, Any]
+    slots: tuple[CompiledSlot, ...]
+    identifier: str | None
+    ancestors: tuple[str, ...]
+    parents: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class _Enum:
+    permissible_values: dict[str, None]
+
+
+@dataclass(frozen=True)
+class _Type:
+    uri: str | None
+
+
+@dataclass(frozen=True)
+class _Schema:
+    default_range: str | None
+
+
+class CompiledSchema:
+    """The installed modules as the write path reads them (a ``SchemaView`` subset)."""
+
+    def __init__(self, data: dict[str, Any]) -> None:
+        if data.get("format") != _FORMAT:
+            raise ValueError("not a compiled schema of this format")
+        self.schema = _Schema(default_range=data["default_range"])
+        self._namespaces: dict[str, str] = dict(data["namespaces"])
+        self._classes: dict[str, CompiledClass] = {}
+        for name, raw in data["classes"].items():
+            self._classes[name] = CompiledClass(
+                name=name,
+                uri=raw["uri"],
+                class_uri=raw["class_uri"],
+                annotations=raw["annotations"],
+                slots=tuple(CompiledSlot.of(s) for s in raw["slots"]),
+                identifier=raw["identifier"],
+                ancestors=tuple(raw["ancestors"]),
+                parents=tuple(raw["parents"]),
+            )
+        self._enums = {n: _Enum(dict.fromkeys(v)) for n, v in data["enums"].items()}
+        self._types = {n: _Type(u) for n, u in data["types"].items()}
+        self._named = {n: CompiledSlot.of(s) for n, s in data["named_slots"].items()}
+        self.kinds: dict[str, str] = dict(data["kinds"])
+
+    # --- the SchemaView subset ------------------------------------------------------
+    def all_classes(self) -> dict[str, CompiledClass]:
+        return self._classes
+
+    def get_class(self, name: str) -> CompiledClass | None:
+        return self._classes.get(name)
+
+    def class_induced_slots(self, name: str) -> list[CompiledSlot]:
+        return list(self._classes[name].slots)
+
+    def get_identifier_slot(self, name: str) -> CompiledSlot | None:
+        cls = self._classes.get(name)
+        if cls is None or cls.identifier is None:
+            return None
+        return next(s for s in cls.slots if s.name == cls.identifier)
+
+    def class_ancestors(self, name: str) -> list[str]:
+        return list(self._classes[name].ancestors)
+
+    def class_parents(self, name: str, mixins: bool = True) -> list[str]:
+        del mixins  # compiled with mixins, as the write path asks
+        return list(self._classes[name].parents)
+
+    def get_uri(self, element: CompiledClass | CompiledSlot, expand: bool = True) -> str:
+        del expand  # compiled expanded
+        return element.uri
+
+    def get_slot(self, name: str) -> CompiledSlot | None:
+        return self._named.get(name)
+
+    def all_enums(self) -> dict[str, _Enum]:
+        return self._enums
+
+    def get_enum(self, name: str) -> _Enum | None:
+        return self._enums.get(name)
+
+    def get_type(self, name: str) -> _Type | None:
+        return self._types.get(name)
+
+    def compiled_namespaces(self) -> dict[str, str]:
+        return dict(self._namespaces)
+
+
+def compile_view(sv: SchemaView) -> dict[str, Any]:
+    """Everything the write path reads of *sv*, as JSON data."""
+    from katalyst_core import kinds, namespaces
+
+    def slot_json(slot: Any) -> dict[str, Any]:
+        any_of: list[Any] = list(slot.any_of or [])
+        return CompiledSlot(
+            name=str(slot.name),
+            uri=str(sv.get_uri(slot, expand=True)),
+            range=str(slot.range) if slot.range else None,
+            multivalued=bool(slot.multivalued),
+            required=bool(slot.required),
+            inlined=bool(slot.inlined),
+            inlined_as_list=bool(slot.inlined_as_list),
+            identifier=bool(slot.identifier),
+            designates_type=bool(slot.designates_type),
+            any_of=tuple(_Range(str(r.range) if r.range else None) for r in any_of),
+            annotations=_annotations(slot),
+        ).to_json()
+
+    classes: dict[str, Any] = {}
+    for name in sv.all_classes():
+        cls: Any = sv.get_class(name)
+        identifier = sv.get_identifier_slot(name)
+        classes[str(name)] = {
+            "uri": str(sv.get_uri(cls, expand=True)),
+            "class_uri": str(cls.class_uri) if cls.class_uri else None,
+            "annotations": _annotations(cls),
+            "slots": [slot_json(s) for s in sv.class_induced_slots(name)],
+            "identifier": str(identifier.name) if identifier is not None else None,
+            "ancestors": [str(a) for a in sv.class_ancestors(name)],
+            "parents": [str(p) for p in sv.class_parents(name, mixins=True)],
+        }
+    enums: dict[str, list[str]] = {}
+    for name in sv.all_enums():
+        enum: Any = sv.get_enum(name)
+        enums[str(name)] = [str(v) for v in (enum.permissible_values or {})]
+    types: dict[str, str | None] = {}
+    for name in sv.all_types():
+        type_def: Any = sv.get_type(name)
+        types[str(name)] = str(type_def.uri) if type_def is not None and type_def.uri else None
+    named: dict[str, Any] = {}
+    for name in _NAMED_SLOTS:
+        slot = sv.get_slot(name)
+        if slot is not None:
+            named[name] = slot_json(slot)
+    return {
+        "format": _FORMAT,
+        "default_range": getattr(sv.schema, "default_range", None),
+        "namespaces": namespaces(sv),
+        "classes": classes,
+        "enums": enums,
+        "types": types,
+        "named_slots": named,
+        "kinds": kinds(sv),
+    }
+
+
+def installed_compiled() -> CompiledSchema:
+    """The installed modules compiled (:func:`compile_view`), from the schema cache."""
+    from katalyst_core import open_installed
+    from katalyst_core.cache import cached_json
+
+    return cached_json(
+        "compiled",
+        lambda: compile_view(open_installed()),
+        load=CompiledSchema,
+        sources=(Path(__file__),),
+        parts=(_FORMAT,),
+    )