katalyst-engine 5.0.1a2__tar.gz → 5.0.1a3__tar.gz

{katalyst_engine-5.0.1a2 → katalyst_engine-5.0.1a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: katalyst-engine
-Version: 5.0.1a2
+Version: 5.0.1a3
 Summary: A distributed declaration framework — primitives for schema-aware, multi-source, evolvable artifact graphs
 Project-URL: Repository, https://github.com/esimplicityinc/katalyst-taxonomy
 Author: eSimplicity
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
-Requires-Dist: katalyst-core<1,>=0.1
+Requires-Dist: katalyst-core<1,>=0.2
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6
 Description-Content-Type: text/markdown

{katalyst_engine-5.0.1a2 → katalyst_engine-5.0.1a3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "katalyst-engine"
-version = "5.0.1a2"
+version = "5.0.1a3"
 description = "A distributed declaration framework — primitives for schema-aware, multi-source, evolvable artifact graphs"
 readme = "README.md"
 license = { text = "Proprietary" }
@@ -25,7 +25,7 @@ dependencies = [
     "pydantic>=2.0",
     # LinkML is the only schema source: bundles build their definitions from the
     # LinkML modules katalyst-core opens (toolkit/linkml.py).
-    "katalyst-core>=0.1,<1",
+    "katalyst-core>=0.2,<1",
     "pyyaml>=6",
 ]
 

{katalyst_engine-5.0.1a2 → katalyst_engine-5.0.1a3}/src/katalyst_engine/bundle/loader.py

@@ -107,6 +107,7 @@ class BundleLoader:
             manifest = manifest.model_copy(update={"entry_point": entry_point})
 
         provider = name or discovered_name or fqn
+        self._register_core_relationships()
         with self.manager.registry.providing(provider):
             bundle.register(self.manager)
 
@@ -122,6 +123,19 @@ class BundleLoader:
 
         return manifest
 
+    def _register_core_relationships(self) -> None:
+        """katalyst-core's relationship types (``child_of``), once and unattributed.
+
+        Every bundle uses them and none defines them, so they belong to no provider and
+        never conflict; a view narrowed to some bundles keeps them.
+        """
+        from katalyst_engine.toolkit.linkml import core_relationships
+
+        registry = self.manager.registry
+        for rel_def in core_relationships():
+            if registry.get_relationship(rel_def.describes_kind) is None:
+                registry.register_relationship(rel_def)
+
     def discover_and_register(
         self,
         discovery: BundleDiscovery | None = None,

{katalyst_engine-5.0.1a2 → katalyst_engine-5.0.1a3}/src/katalyst_engine/evolution/__init__.py

@@ -7,8 +7,21 @@ its version bump and its migration.
 - :mod:`.migration`: the migration file (``migrations/<from>-to-<to>.yaml``) and the
   pure function that applies its steps to documents in memory.
 - :mod:`.gate`: the version gate CI runs (``kata core schema check``).
+- :mod:`.lock`: the versions a backend records (its schema lock), against the loaded ones.
+- :mod:`.chain`: the chain of migrations from a recorded version to the loaded one, run
+  a step at a time.
 """
 
+from katalyst_engine.evolution.chain import (
+    ChainPlan,
+    MigrationChainError,
+    SchemaChain,
+    StepOutcome,
+    describe_step,
+    migration_chain,
+    release_migrations,
+    run_chains,
+)
 from katalyst_engine.evolution.diff import (
     Category,
     Impact,
@@ -18,6 +31,15 @@ from katalyst_engine.evolution.diff import (
     diff_modules,
 )
 from katalyst_engine.evolution.gate import GateReport, check_release, migration_skeleton
+from katalyst_engine.evolution.lock import (
+    LOCK_FILE_NAME,
+    BundleStatus,
+    SchemaLock,
+    SchemaLockError,
+    SchemaState,
+    SchemaStatus,
+    compare_lock,
+)
 from katalyst_engine.evolution.migration import (
     CODE_HOOK_GROUP,
     Document,
@@ -38,25 +60,40 @@ from katalyst_engine.evolution.release import (
 
 __all__ = [
     "CODE_HOOK_GROUP",
+    "LOCK_FILE_NAME",
+    "BundleStatus",
     "Category",
+    "ChainPlan",
     "Document",
     "DocumentSet",
     "GateReport",
     "Impact",
     "Location",
     "Migration",
+    "MigrationChainError",
     "MigrationFileError",
     "MigrationStepError",
+    "SchemaChain",
     "SchemaChange",
     "SchemaDiff",
+    "SchemaLock",
+    "SchemaLockError",
     "SchemaRelease",
+    "SchemaState",
+    "SchemaStatus",
     "SchemaVersionMismatchError",
+    "StepOutcome",
     "apply_migration",
     "bundle_release",
     "check_release",
+    "compare_lock",
     "core_release",
+    "describe_step",
     "diff_modules",
     "installed_releases",
     "load_migration",
+    "migration_chain",
     "migration_skeleton",
+    "release_migrations",
+    "run_chains",
 ]

katalyst_engine-5.0.1a3/src/katalyst_engine/evolution/chain.py

@@ -0,0 +1,213 @@
+"""Moving documents across releases: the chain of migrations from the version a backend
+is at to the loaded one, and applying it step by step with what each step changed.
+
+A schema's migrations live beside its modules (``migrations/<from>-to-<to>.yaml``), one
+per breaking release, each from the release before it. The chain from a version is the
+migration out of its major, then the one out of the next, until the loaded major:
+8.x to 10.x runs 8 → 9, then 9 → 10. Within a major nothing is migrated.
+
+:func:`run_chains` applies the chains of several schemas to one set of documents, in
+order, a step at a time, so a dry run can say what each step did. It is pure: writing
+the result back to a backend is the caller's.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+from collections import Counter
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from katalyst_engine.core.version import Version
+from katalyst_engine.evolution.migration import (
+    CodeStep,
+    DeleteKind,
+    Document,
+    DocumentSet,
+    DropField,
+    MapEnum,
+    Migration,
+    MoveField,
+    MoveFiles,
+    RenameField,
+    RenameKind,
+    RenameRelationship,
+    Reparent,
+    SetDefault,
+    Step,
+    apply_migration,
+    load_migration,
+)
+from katalyst_engine.evolution.release import SchemaRelease
+from katalyst_engine.schema.definition import NodeDefinition
+
+__all__ = [
+    "ChainPlan",
+    "MigrationChainError",
+    "SchemaChain",
+    "StepOutcome",
+    "describe_step",
+    "migration_chain",
+    "release_migrations",
+    "run_chains",
+]
+
+
+class MigrationChainError(ValueError):
+    """No chain of migrations leads from a backend's version to the loaded one."""
+
+
+def release_migrations(release: SchemaRelease) -> tuple[Migration, ...]:
+    """Every migration a schema ships, oldest first.
+
+    Raises:
+        MigrationFileError: A migration file does not describe a migration.
+    """
+    try:
+        root = release.migrations_root
+    except Exception:  # noqa: BLE001 - a schema with no modules has no migrations
+        return ()
+    if not root.is_dir():
+        return ()
+    found = [load_migration(path) for path in sorted(root.glob("*.yaml"))]
+    return tuple(sorted(found, key=lambda m: m.source))
+
+
+def migration_chain(
+    name: str, migrations: Sequence[Migration], start: Version, target: Version
+) -> tuple[Migration, ...]:
+    """The migrations that take documents written for *start* to *target*'s major.
+
+    Raises:
+        MigrationChainError: *start* is newer than *target*, or a major has no
+            migration out of it.
+    """
+    if start.major > target.major:
+        raise MigrationChainError(
+            f"{name}: documents at {start} are newer than the loaded {target}"
+        )
+    chain: list[Migration] = []
+    current = start
+    while current.major < target.major:
+        out = [
+            m
+            for m in migrations
+            if m.source.major == current.major and current.major < m.target.major <= target.major
+        ]
+        if not out:
+            raise MigrationChainError(
+                f"{name}: no migration from {current.major}.x towards {target} "
+                f"(the schema ships {', '.join(f'{m.from_}-to-{m.to}' for m in migrations) or 'none'})"
+            )
+        if len(out) > 1:
+            listed = ", ".join(f"{m.from_}-to-{m.to}" for m in out)
+            raise MigrationChainError(f"{name}: more than one migration from {current}: {listed}")
+        chain.append(out[0])
+        current = out[0].target
+    return tuple(chain)
+
+
+def _field(kind: str | None, path: str) -> str:
+    return f"{kind}.{path}" if kind else path
+
+
+def describe_step(step: Step) -> str:
+    """One line for a step, as a dry run lists it: ``rename_field service.tags → labels``."""
+    if isinstance(step, RenameKind):
+        return f"rename_kind {step.from_} → {step.to}"
+    if isinstance(step, RenameField):
+        return f"rename_field {_field(step.kind, step.from_)} → {step.to}"
+    if isinstance(step, MoveField):
+        return f"move_field {_field(step.kind, step.from_)} → {step.to}"
+    if isinstance(step, SetDefault):
+        return f"set_default {_field(step.kind, step.field)} = {json.dumps(step.value)}"
+    if isinstance(step, DropField):
+        return f"drop_field {_field(step.kind, step.field)}"
+    if isinstance(step, DeleteKind):
+        return f"delete_kind {step.kind}"
+    if isinstance(step, MapEnum):
+        values = ", ".join(f"{k} → {v}" for k, v in step.values.items())
+        return f"map_enum {_field(step.kind, step.field)} {values}"
+    if isinstance(step, Reparent):
+        rule = (
+            f"to {step.to}"
+            if step.to is not None
+            else (f"from {step.parent_from}" if step.parent_from else f"up {step.up}")
+        )
+        return f"reparent {step.kind} {rule}"
+    if isinstance(step, MoveFiles):
+        return f"move_files {step.kind}"
+    if isinstance(step, RenameRelationship):
+        return f"rename_relationship {step.from_} → {step.to}"
+    assert isinstance(step, CodeStep)
+    return f"code {step.hook}"
+
+
+@dataclass(frozen=True)
+class SchemaChain:
+    """One schema's migrations, from the version a backend is at to the loaded one."""
+
+    name: str
+    start: Version
+    target: Version
+    migrations: tuple[Migration, ...] = ()
+
+
+@dataclass(frozen=True)
+class StepOutcome:
+    """What one step did: how many documents it wrote (new or changed) and removed."""
+
+    schema: str
+    migration: Migration
+    step: Step
+    written: int
+    removed: int
+
+    @property
+    def label(self) -> str:
+        return describe_step(self.step)
+
+
+@dataclass
+class ChainPlan:
+    """Documents after every chain, and what each step did."""
+
+    documents: DocumentSet
+    outcomes: list[StepOutcome] = field(default_factory=lambda: list[StepOutcome]())
+
+
+def _key(doc: Document) -> str:
+    return json.dumps([doc.path, doc.data], sort_keys=True, default=str)
+
+
+def run_chains(
+    chains: Sequence[SchemaChain],
+    documents: DocumentSet,
+    *,
+    nodes: Mapping[str, NodeDefinition] | None = None,
+    hooks: Mapping[str, Callable[..., Any]] | None = None,
+) -> ChainPlan:
+    """Apply each chain's migrations to *documents*, a step at a time. The input is not
+    changed.
+
+    Raises:
+        MigrationStepError: A step cannot be applied.
+    """
+    current = documents
+    outcomes: list[StepOutcome] = []
+    for chain in chains:
+        for migration in chain.migrations:
+            for step in migration.steps:
+                single = migration.model_copy(update={"steps": (step,)})
+                after = apply_migration(single, current, nodes=nodes, hooks=hooks)
+                before_keys = Counter(_key(d) for d in current.documents)
+                after_keys = Counter(_key(d) for d in after.documents)
+                written = sum((after_keys - before_keys).values())
+                removed = max(0, len(current.documents) - len(after.documents))
+                outcomes.append(StepOutcome(chain.name, migration, step, written, removed))
+                current = after
+    if current is documents:
+        current = copy.deepcopy(documents)
+    return ChainPlan(documents=current, outcomes=outcomes)

katalyst_engine-5.0.1a3/src/katalyst_engine/evolution/lock.py

@@ -0,0 +1,228 @@
+"""The schema lock: the version of each schema a backend's documents were written with.
+
+A backend records it beside its documents (a repository in a file, a database in a
+metadata table), and compares it with the schemas it is opened with:
+
+- **current**: every recorded major is the loaded major (a minor may differ);
+- **behind**: a recorded major is older than the loaded one; the backend can be read,
+  not written, until its documents are migrated;
+- **ahead**: a recorded major is newer than the loaded one; the backend was written by
+  a newer release and is not read at all;
+- **untracked**: the backend cannot record versions.
+
+A backend with no lock was written before versions were recorded. It is taken to be at
+each schema's *baseline*, the oldest archived release (nothing older is supported), or
+at the loaded version for a schema with no archive. A schema the lock does not name was
+added after it was written, and is current; one the lock names that is not loaded is
+reported, and decides nothing.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any, cast
+
+import yaml
+
+from katalyst_engine.core.version import Version
+
+__all__ = [
+    "LOCK_FILE_NAME",
+    "BundleStatus",
+    "SchemaLock",
+    "SchemaLockError",
+    "SchemaState",
+    "SchemaStatus",
+    "compare_lock",
+]
+
+LOCK_FILE_NAME = "schema.lock"
+"""The file a repository keeps its lock in (beside its documents)."""
+
+_HEADER = "# Written by kata. Change it with `kata tax migrate`, not by hand.\n"
+
+
+class SchemaLockError(ValueError):
+    """A lock that cannot be read."""
+
+
+@dataclass(frozen=True)
+class SchemaLock:
+    """The version of each schema, by schema name."""
+
+    bundles: Mapping[str, Version] = field(default_factory=lambda: dict[str, Version]())
+
+    @classmethod
+    def parse(cls, text: str) -> SchemaLock:
+        """Read a lock file's text (``bundles: {name: version}``).
+
+        Raises:
+            SchemaLockError: It is not a lock.
+        """
+        try:
+            raw: Any = yaml.safe_load(text)
+        except yaml.YAMLError as exc:
+            raise SchemaLockError(f"not YAML: {exc}") from exc
+        if not isinstance(raw, dict) or not isinstance(
+            cast("dict[str, Any]", raw).get("bundles"), dict
+        ):
+            raise SchemaLockError("a schema lock is a mapping with `bundles: {name: version}`")
+        bundles: dict[str, Version] = {}
+        for name, version in cast("dict[str, Any]", cast("dict[str, Any]", raw)["bundles"]).items():
+            try:
+                bundles[str(name)] = Version.parse(str(version))
+            except ValueError as exc:
+                raise SchemaLockError(f"{name}: {version!r} is not a version") from exc
+        return cls(bundles=bundles)
+
+    def render(self) -> str:
+        """The lock file's text: the schemas in name order."""
+        body = yaml.safe_dump(
+            {"bundles": {name: str(self.bundles[name]) for name in sorted(self.bundles)}},
+            sort_keys=False,
+            default_flow_style=False,
+        )
+        return _HEADER + body
+
+    def with_versions(self, versions: Mapping[str, Version]) -> SchemaLock:
+        """This lock with *versions* recorded (the schemas it names and *versions* does
+        not are kept)."""
+        return SchemaLock(bundles={**self.bundles, **versions})
+
+    def as_strings(self) -> dict[str, str]:
+        return {name: str(self.bundles[name]) for name in sorted(self.bundles)}
+
+
+class SchemaState(StrEnum):
+    """How a backend's recorded versions compare with the loaded schemas."""
+
+    CURRENT = "current"
+    BEHIND = "behind"
+    AHEAD = "ahead"
+    UNTRACKED = "untracked"
+
+
+@dataclass(frozen=True)
+class BundleStatus:
+    """One schema: what is loaded, what the backend is at, and how they compare."""
+
+    name: str
+    loaded: Version | None
+    """The loaded version; ``None``: the lock names a schema that is not loaded."""
+
+    recorded: Version | None
+    """What the backend is at: recorded, or (:attr:`assumed`) taken to be."""
+
+    assumed: bool = False
+    """The lock does not say: :attr:`recorded` is the baseline (no lock), or the loaded
+    version (a schema added after the lock was written)."""
+
+    @property
+    def state(self) -> SchemaState | None:
+        """``None`` for a schema that is not loaded."""
+        if self.loaded is None:
+            return None
+        if self.recorded is None or self.recorded.major == self.loaded.major:
+            return SchemaState.CURRENT
+        return SchemaState.BEHIND if self.recorded.major < self.loaded.major else SchemaState.AHEAD
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "loaded": str(self.loaded) if self.loaded is not None else None,
+            "recorded": str(self.recorded) if self.recorded is not None else None,
+            "assumed": self.assumed,
+            "state": self.state.value if self.state is not None else "not-loaded",
+        }
+
+
+@dataclass(frozen=True)
+class SchemaStatus:
+    """A backend's schema versions against the loaded schemas."""
+
+    backend: str
+    state: SchemaState
+    recorded: bool
+    """Whether the backend has a lock (``False``: unrecorded, or untracked)."""
+
+    bundles: tuple[BundleStatus, ...] = ()
+
+    @property
+    def behind(self) -> tuple[BundleStatus, ...]:
+        return tuple(b for b in self.bundles if b.state is SchemaState.BEHIND)
+
+    @property
+    def ahead(self) -> tuple[BundleStatus, ...]:
+        return tuple(b for b in self.bundles if b.state is SchemaState.AHEAD)
+
+    @property
+    def writable(self) -> bool:
+        """Whether the schema versions allow a write (current or untracked)."""
+        return self.state in (SchemaState.CURRENT, SchemaState.UNTRACKED)
+
+    @property
+    def readable(self) -> bool:
+        return self.state is not SchemaState.AHEAD
+
+    def recorded_versions(self) -> dict[str, Version]:
+        """What the backend is at (recorded or assumed), for every loaded schema."""
+        return {
+            b.name: b.recorded
+            for b in self.bundles
+            if b.loaded is not None and b.recorded is not None
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "backend": self.backend,
+            "state": self.state.value,
+            "recorded": self.recorded,
+            "writable": self.writable,
+            "bundles": [b.to_dict() for b in self.bundles],
+        }
+
+
+def compare_lock(
+    backend: str,
+    lock: SchemaLock | None,
+    loaded: Mapping[str, Version],
+    *,
+    baselines: Mapping[str, Version] | None = None,
+    tracked: bool = True,
+) -> SchemaStatus:
+    """*backend*'s :class:`SchemaStatus`.
+
+    Args:
+        backend: The backend's name.
+        lock: What it records; ``None`` when it records nothing.
+        loaded: The loaded schemas' versions, by name.
+        baselines: Each schema's oldest archived release, what a backend with no lock is
+            taken to be at. A schema without one: its loaded version.
+        tracked: Whether the backend can record versions at all.
+    """
+    if not tracked:
+        rows = tuple(
+            BundleStatus(name, loaded[name], None, assumed=True) for name in sorted(loaded)
+        )
+        return SchemaStatus(backend, SchemaState.UNTRACKED, recorded=False, bundles=rows)
+    bases = baselines or {}
+    rows_list: list[BundleStatus] = []
+    for name in sorted(set(loaded) | set(lock.bundles if lock else ())):
+        current = loaded.get(name)
+        if lock is None:
+            rows_list.append(BundleStatus(name, current, bases.get(name, current), assumed=True))
+        elif name in lock.bundles:
+            rows_list.append(BundleStatus(name, current, lock.bundles[name]))
+        else:
+            rows_list.append(BundleStatus(name, current, current, assumed=True))
+    rows = tuple(rows_list)
+    states = {row.state for row in rows}
+    if SchemaState.AHEAD in states:
+        state = SchemaState.AHEAD
+    elif SchemaState.BEHIND in states:
+        state = SchemaState.BEHIND
+    else:
+        state = SchemaState.CURRENT
+    return SchemaStatus(backend, state, recorded=lock is not None, bundles=rows)