katalyst-engine 2.0.0__py3-none-any.whl

katalyst_engine/model/query.py

@@ -0,0 +1,186 @@
+"""Query engine — filter, sort, and paginate over a ModelStore.
+
+QueryFilter specifies criteria for selecting nodes. QueryEngine
+applies filters and provides sorted, paginated results.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from enum import Enum
+from fnmatch import fnmatch
+
+from pydantic import BaseModel
+
+from katalyst_engine.model.node import Node
+from katalyst_engine.model.store import ModelStore
+
+
+class SortField(str, Enum):
+    """Fields available for sorting query results."""
+
+    FQN = "fqn"
+    NAME = "name"
+    KIND = "kind"
+    NAMESPACE = "namespace"
+
+
+class SortOrder(str, Enum):
+    """Sort direction."""
+
+    ASC = "asc"
+    DESC = "desc"
+
+
+class QueryFilter(BaseModel, frozen=True):
+    """Criteria for selecting nodes from a ModelStore.
+
+    All non-None criteria are ANDed. Glob patterns use fnmatch syntax.
+    Label matchers require exact matches on all specified key-value pairs.
+    The ``custom`` callable allows domain-specific predicates without
+    requiring the engine to know about domain types (e.g., SchemaRegistry).
+    """
+
+    kind: str | None = None
+    """Exact kind match. None means any kind."""
+
+    namespace: str | None = None
+    """Exact namespace match. None means any namespace."""
+
+    name_pattern: str | None = None
+    """Glob pattern for name matching. None means any name."""
+
+    label_matchers: dict[str, str] = {}
+    """Label key-value pairs that must all match exactly."""
+
+    custom: Callable[[Node], bool] | None = None
+    """Arbitrary predicate for domain-specific filtering.
+
+    When provided, a node must satisfy this callable in addition to all
+    other criteria. This allows taxonomy (or any consumer) to inject
+    schema-aware predicates without the engine importing domain types.
+    """
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    def matches(self, node: Node) -> bool:
+        """Check if a node matches all filter criteria."""
+        if self.kind is not None and node.identity.kind != self.kind:
+            return False
+        if self.namespace is not None and node.identity.namespace != self.namespace:
+            return False
+        if self.name_pattern is not None and not fnmatch(node.identity.name, self.name_pattern):
+            return False
+        for key, value in self.label_matchers.items():
+            if node.labels.get(key) != value:
+                return False
+        if self.custom is not None and not self.custom(node):
+            return False
+        return True
+
+
+class QueryResult(BaseModel, frozen=True):
+    """Paginated query result."""
+
+    nodes: tuple[Node, ...] = ()
+    """Nodes matching the query."""
+
+    total: int = 0
+    """Total number of matching nodes (before pagination)."""
+
+    offset: int = 0
+    """Offset of the first returned node."""
+
+    limit: int | None = None
+    """Maximum number of nodes returned. None means no limit."""
+
+
+class QueryEngine:
+    """Filter, sort, and paginate over nodes.
+
+    Accepts either a ``ModelStore`` or any iterable of ``Node`` objects,
+    enabling consumers to provide pre-filtered collections or domain-specific
+    stores without wrapping them in a ``ModelStore``.
+
+    Usage:
+        engine = QueryEngine(store)
+        result = engine.query(
+            QueryFilter(kind="layer"),
+            sort_by=SortField.NAME,
+            offset=0, limit=10,
+        )
+
+        # Or with raw nodes:
+        engine = QueryEngine(my_node_list)
+        result = engine.query(QueryFilter(custom=my_predicate))
+    """
+
+    def __init__(self, source: ModelStore | Iterable[Node]) -> None:
+        self._source = source
+
+    def _all_nodes(self) -> list[Node]:
+        """Get all nodes from the source."""
+        if isinstance(self._source, ModelStore):
+            return list(self._source.all_nodes())
+        return list(self._source)
+
+    def query(
+        self,
+        filter: QueryFilter | None = None,
+        sort_by: SortField = SortField.FQN,
+        sort_order: SortOrder = SortOrder.ASC,
+        offset: int = 0,
+        limit: int | None = None,
+    ) -> QueryResult:
+        """Execute a query against the store.
+
+        Args:
+            filter: Selection criteria. None returns all nodes.
+            sort_by: Field to sort by.
+            sort_order: Ascending or descending.
+            offset: Number of results to skip.
+            limit: Maximum number of results to return.
+
+        Returns:
+            A QueryResult with matching, sorted, paginated nodes.
+        """
+        candidates = self._all_nodes()
+
+        if filter is not None:
+            candidates = [n for n in candidates if filter.matches(n)]
+
+        candidates = self._sort(candidates, sort_by, sort_order)
+        total = len(candidates)
+
+        if offset > 0:
+            candidates = candidates[offset:]
+        if limit is not None:
+            candidates = candidates[:limit]
+
+        return QueryResult(
+            nodes=tuple(candidates),
+            total=total,
+            offset=offset,
+            limit=limit,
+        )
+
+    def _sort(
+        self,
+        nodes: list[Node],
+        sort_by: SortField,
+        sort_order: SortOrder,
+    ) -> list[Node]:
+        """Sort nodes by the specified field and order."""
+
+        def key_fn(node: Node) -> str:
+            if sort_by == SortField.FQN:
+                return node.identity.fqn
+            elif sort_by == SortField.NAME:
+                return node.identity.name
+            elif sort_by == SortField.KIND:
+                return node.identity.kind
+            elif sort_by == SortField.NAMESPACE:
+                return node.identity.namespace
+            return node.identity.fqn  # pragma: no cover
+
+        return sorted(nodes, key=key_fn, reverse=(sort_order == SortOrder.DESC))

katalyst_engine/model/store.py

@@ -0,0 +1,119 @@
+"""Model store — in-memory indexed collection of Nodes.
+
+Provides indexed access by name, kind, namespace, and FQN.
+Supports add, remove, and query operations.
+"""
+
+from __future__ import annotations
+
+from katalyst_engine.core.identity import Identity
+from katalyst_engine.model.node import Node
+
+
+class ModelStore:
+    """In-memory indexed collection of Nodes.
+
+    Maintains multiple indexes for fast lookup:
+    - by FQN (primary key)
+    - by kind
+    - by namespace
+    - by name
+
+    This is a mutable container — nodes can be added and removed.
+    """
+
+    def __init__(self) -> None:
+        self._by_fqn: dict[str, Node] = {}
+        self._by_kind: dict[str, list[Node]] = {}
+        self._by_namespace: dict[str, list[Node]] = {}
+        self._by_name: dict[str, list[Node]] = {}
+
+    def add(self, node: Node) -> None:
+        """Add a node to the store, replacing any existing node with the same FQN."""
+        fqn = node.identity.fqn
+        existing = self._by_fqn.get(fqn)
+        if existing is not None:
+            self._remove_from_indexes(existing)
+        self._by_fqn[fqn] = node
+        self._add_to_indexes(node)
+
+    def remove(self, identity: Identity) -> bool:
+        """Remove a node by identity. Returns True if it existed."""
+        fqn = identity.fqn
+        node = self._by_fqn.pop(fqn, None)
+        if node is None:
+            return False
+        self._remove_from_indexes(node)
+        return True
+
+    def get(self, identity: Identity) -> Node | None:
+        """Look up a node by identity."""
+        return self._by_fqn.get(identity.fqn)
+
+    def get_by_fqn(self, fqn: str) -> Node | None:
+        """Look up a node by FQN string."""
+        return self._by_fqn.get(fqn)
+
+    def by_kind(self, kind: str) -> list[Node]:
+        """Return all nodes of a given kind."""
+        return list(self._by_kind.get(kind, []))
+
+    def by_namespace(self, namespace: str) -> list[Node]:
+        """Return all nodes in a given namespace."""
+        return list(self._by_namespace.get(namespace, []))
+
+    def by_name(self, name: str) -> list[Node]:
+        """Return all nodes with a given local name."""
+        return list(self._by_name.get(name, []))
+
+    def all_nodes(self) -> list[Node]:
+        """Return all nodes, ordered by FQN."""
+        return sorted(self._by_fqn.values(), key=lambda n: n.identity.fqn)
+
+    @property
+    def count(self) -> int:
+        """Number of nodes in the store."""
+        return len(self._by_fqn)
+
+    def clear(self) -> None:
+        """Remove all nodes."""
+        self._by_fqn.clear()
+        self._by_kind.clear()
+        self._by_namespace.clear()
+        self._by_name.clear()
+
+    def _add_to_indexes(self, node: Node) -> None:
+        """Add a node to the secondary indexes."""
+        kind = node.identity.kind
+        ns = node.identity.namespace
+        name = node.identity.name
+
+        if kind not in self._by_kind:
+            self._by_kind[kind] = []
+        self._by_kind[kind].append(node)
+
+        if ns not in self._by_namespace:
+            self._by_namespace[ns] = []
+        self._by_namespace[ns].append(node)
+
+        if name not in self._by_name:
+            self._by_name[name] = []
+        self._by_name[name].append(node)
+
+    def _remove_from_indexes(self, node: Node) -> None:
+        """Remove a node from the secondary indexes."""
+        kind = node.identity.kind
+        ns = node.identity.namespace
+        name = node.identity.name
+
+        kind_list = self._by_kind.get(kind)
+        if kind_list is not None:
+            kind_list[:] = [n for n in kind_list if n.identity.fqn != node.identity.fqn]
+
+        ns_list = self._by_namespace.get(ns)
+        if ns_list is not None:
+            ns_list[:] = [n for n in ns_list if n.identity.fqn != node.identity.fqn]
+
+        name_list = self._by_name.get(name)
+        if name_list is not None:
+            name_list[:] = [n for n in name_list if n.identity.fqn != node.identity.fqn]

katalyst_engine/replication/__init__.py

@@ -0,0 +1,30 @@
+"""Replication — jobs, transforms, and the replication engine.
+
+.. note:: FUTURE: Wire into taxonomy when a third sync target arrives or the
+   sync flow is refactored. The taxonomy's neo4j and postgres adapters
+   hand-roll what ReplicationEngine provides. Making ReplicationEngine
+   protocol-driven (accept ReplicationTarget) would allow both adapters
+   to share pipeline logic.
+"""
+
+from katalyst_engine.replication.engine import ReplicationEngine
+from katalyst_engine.replication.job import (
+    ReplicationJob,
+    ReplicationMode,
+    ReplicationResult,
+)
+from katalyst_engine.replication.transform import (
+    TransformPipeline,
+    TransformRunner,
+    TransformStep,
+)
+
+__all__ = [
+    "ReplicationEngine",
+    "ReplicationJob",
+    "ReplicationMode",
+    "ReplicationResult",
+    "TransformPipeline",
+    "TransformRunner",
+    "TransformStep",
+]

katalyst_engine/replication/engine.py

@@ -0,0 +1,104 @@
+"""Replication engine — executes replication jobs.
+
+Coordinates the source registry, model store, and transform pipeline
+to replicate nodes from one source to another.
+"""
+
+from __future__ import annotations
+
+from katalyst_engine.model.store import ModelStore
+from katalyst_engine.replication.job import (
+    ReplicationJob,
+    ReplicationMode,
+    ReplicationResult,
+)
+from katalyst_engine.replication.transform import TransformPipeline, TransformRunner
+from katalyst_engine.source.registry import SourceRegistry
+
+
+class ReplicationEngine:
+    """Executes replication jobs using source registry and model store.
+
+    The engine reads nodes from the source model store, applies
+    optional transforms, and writes them to the target store.
+
+    Usage:
+        engine = ReplicationEngine(source_registry, source_store, target_store)
+        engine.set_transform_runner(runner)
+        result = engine.execute(job)
+    """
+
+    def __init__(
+        self,
+        source_registry: SourceRegistry,
+        source_store: ModelStore,
+        target_store: ModelStore,
+    ) -> None:
+        self._source_registry = source_registry
+        self._source_store = source_store
+        self._target_store = target_store
+        self._transform_runner: TransformRunner | None = None
+        self._pipeline: TransformPipeline | None = None
+
+    def set_transform_runner(
+        self, runner: TransformRunner, pipeline: TransformPipeline | None = None
+    ) -> None:
+        """Configure the transform runner and optional pipeline."""
+        self._transform_runner = runner
+        self._pipeline = pipeline
+
+    def execute(self, job: ReplicationJob) -> ReplicationResult:
+        """Execute a replication job.
+
+        Reads nodes from the source store, applies filters and
+        transforms, then writes to the target store according
+        to the job's mode.
+
+        Args:
+            job: The replication job configuration.
+
+        Returns:
+            A ReplicationResult describing what happened.
+        """
+        source_nodes = self._source_store.all_nodes()
+        source_nodes = [n for n in source_nodes if job.filter.matches(n)]
+
+        if self._transform_runner and self._pipeline:
+            source_nodes = [self._transform_runner.run(self._pipeline, n) for n in source_nodes]
+
+        if job.dry_run:
+            return ReplicationResult(
+                job=job,
+                created_count=len(source_nodes),
+            )
+
+        created = 0
+        updated = 0
+        skipped = 0
+        errors: list[str] = []
+
+        if job.mode == ReplicationMode.FULL:
+            self._target_store.clear()
+
+        for node in source_nodes:
+            try:
+                existing = self._target_store.get(node.identity)
+                if existing is not None:
+                    if existing == node:
+                        skipped += 1
+                    else:
+                        self._target_store.add(node)
+                        updated += 1
+                else:
+                    self._target_store.add(node)
+                    created += 1
+            except Exception as exc:
+                errors.append(f"Failed to replicate {node.identity.fqn}: {exc}")
+
+        return ReplicationResult(
+            job=job,
+            created_count=created,
+            updated_count=updated,
+            skipped_count=skipped,
+            errors=tuple(errors),
+        )

katalyst_engine/replication/job.py

@@ -0,0 +1,88 @@
+"""Replication job — configuration and results for data replication.
+
+A ReplicationJob describes what to replicate, from where, to where,
+and how. A ReplicationResult captures the outcome.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel
+
+from katalyst_engine.core.identity import Identity
+from katalyst_engine.model.query import QueryFilter
+
+
+class ReplicationMode(str, Enum):
+    """How to handle existing data at the target."""
+
+    FULL = "full"
+    """Replace all target data with source data."""
+
+    INCREMENTAL = "incremental"
+    """Only replicate changed items since last sync."""
+
+    MERGE = "merge"
+    """Merge source data into target, keeping non-conflicting target data."""
+
+
+class ReplicationJob(BaseModel, frozen=True):
+    """Configuration for a replication operation.
+
+    Describes the source, target, filter criteria, and mode
+    for replicating data between sources.
+    """
+
+    source_identity: Identity
+    """Identity of the source to replicate from."""
+
+    target_identity: Identity
+    """Identity of the target to replicate to."""
+
+    filter: QueryFilter = QueryFilter()
+    """Optional filter to select a subset of nodes."""
+
+    mode: ReplicationMode = ReplicationMode.INCREMENTAL
+    """How to handle existing target data."""
+
+    description: str = ""
+    """Human-readable description of this replication job."""
+
+    dry_run: bool = False
+    """If True, compute what would change without actually replicating."""
+
+
+class ReplicationResult(BaseModel, frozen=True):
+    """Outcome of a replication operation.
+
+    Captures counts of what was replicated and any errors encountered.
+    """
+
+    job: ReplicationJob
+    """The job that produced this result."""
+
+    created_count: int = 0
+    """Number of nodes created at the target."""
+
+    updated_count: int = 0
+    """Number of nodes updated at the target."""
+
+    deleted_count: int = 0
+    """Number of nodes deleted at the target."""
+
+    skipped_count: int = 0
+    """Number of nodes skipped (already up to date)."""
+
+    errors: tuple[str, ...] = ()
+    """Error messages for individual items that failed."""
+
+    @property
+    def total_processed(self) -> int:
+        """Total items processed (created + updated + deleted + skipped)."""
+        return self.created_count + self.updated_count + self.deleted_count + self.skipped_count
+
+    @property
+    def success(self) -> bool:
+        """True if no errors occurred."""
+        return len(self.errors) == 0

katalyst_engine/replication/transform.py

@@ -0,0 +1,111 @@
+"""Transform pipeline — ordered transformations applied during replication.
+
+TransformSteps are individual transformations; a TransformPipeline
+chains them together and applies them to nodes during replication.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel
+
+from katalyst_engine.model.node import Node
+
+
+class TransformStep(ABC):
+    """Abstract base for a single transformation applied during replication.
+
+    Implementations transform a node's data as it moves from
+    source to target. Steps can modify spec fields, labels,
+    annotations, or even change the node's identity.
+    """
+
+    @abstractmethod
+    def transform(self, node: Node, context: dict[str, Any]) -> Node:
+        """Transform a node.
+
+        Args:
+            node: The node to transform.
+            context: Additional context (e.g. source/target metadata).
+
+        Returns:
+            A new Node with the transformation applied.
+        """
+
+    @abstractmethod
+    def name(self) -> str:
+        """Human-readable name of this transform step."""
+
+
+class TransformPipeline(BaseModel, frozen=True):
+    """Ordered list of transform step names applied during replication.
+
+    The pipeline itself is a frozen model describing the configuration.
+    Execution is handled by the ReplicationEngine, which looks up
+    TransformStep instances by name.
+
+    Note: TransformStep instances are not serializable (they're ABCs),
+    so the pipeline stores step names as strings. The engine maintains
+    the name → step mapping.
+    """
+
+    step_names: tuple[str, ...] = ()
+    """Ordered names of transform steps to apply."""
+
+    description: str = ""
+    """Human-readable description of what this pipeline does."""
+
+
+class TransformRunner:
+    """Executes a TransformPipeline against nodes.
+
+    Maintains a registry of named TransformSteps and applies them
+    in pipeline order to each node.
+
+    Usage:
+        runner = TransformRunner()
+        runner.register("strip_labels", MyStripLabelsStep())
+        runner.register("rename_kind", MyRenameKindStep())
+        transformed = runner.run(pipeline, node, context)
+    """
+
+    def __init__(self) -> None:
+        self._steps: dict[str, TransformStep] = {}
+
+    def register(self, name: str, step: TransformStep) -> None:
+        """Register a named transform step."""
+        self._steps[name] = step
+
+    def run(
+        self,
+        pipeline: TransformPipeline,
+        node: Node,
+        context: dict[str, Any] | None = None,
+    ) -> Node:
+        """Apply all steps in a pipeline to a node.
+
+        Steps are applied in order. Each step receives the output
+        of the previous step. Unknown step names are skipped.
+
+        Args:
+            pipeline: The pipeline configuration.
+            node: The input node.
+            context: Optional context dict passed to each step.
+
+        Returns:
+            The transformed node.
+        """
+        ctx = context or {}
+        result = node
+        for step_name in pipeline.step_names:
+            step = self._steps.get(step_name)
+            if step is not None:
+                result = step.transform(result, ctx)
+        return result
+
+    @property
+    def step_count(self) -> int:
+        """Number of registered transform steps."""
+        return len(self._steps)

katalyst_engine/resolution/__init__.py

@@ -0,0 +1,32 @@
+"""Resolution — detecting and resolving conflicts between multi-source declarations.
+
+.. note:: FUTURE: Wire into taxonomy when multi-source node conflicts need handling.
+   When two sources declare the same node identity, the ResolutionEngine
+   picks a winner using pluggable strategies. Pairs with source/ and
+   discovery/ as a coherent pipeline.
+"""
+
+from katalyst_engine.resolution.conflict import (
+    Conflict,
+    ConflictReport,
+    ConflictSeverity,
+)
+from katalyst_engine.resolution.engine import ResolutionEngine, ResolutionResult
+from katalyst_engine.resolution.strategies import (
+    CanonicalSourceStrategy,
+    LatestVersionStrategy,
+    MergeStrategy,
+    ResolutionStrategy,
+)
+
+__all__ = [
+    "CanonicalSourceStrategy",
+    "Conflict",
+    "ConflictReport",
+    "ConflictSeverity",
+    "LatestVersionStrategy",
+    "MergeStrategy",
+    "ResolutionEngine",
+    "ResolutionResult",
+    "ResolutionStrategy",
+]