crochet-migration 0.1.0__py3-none-any.whl

crochet/ir/diff.py

@@ -0,0 +1,199 @@
+"""Diff two schema snapshots to produce migration intent."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from crochet.ir.schema import NodeIR, PropertyIR, RelationshipIR, SchemaSnapshot
+
+
+@dataclass
+class PropertyChange:
+    """A single property-level change."""
+
+    kind: str  # "added", "removed", "modified"
+    property_name: str
+    old: "PropertyIR | None" = None
+    new: "PropertyIR | None" = None
+
+    @property
+    def description(self) -> str:
+        if self.kind == "added":
+            return f"  + property '{self.property_name}' ({self.new.property_type})"  # type: ignore[union-attr]
+        elif self.kind == "removed":
+            return f"  - property '{self.property_name}'"
+        else:
+            changes = []
+            if self.old and self.new:
+                if self.old.property_type != self.new.property_type:
+                    changes.append(
+                        f"type {self.old.property_type} -> {self.new.property_type}"
+                    )
+                if self.old.required != self.new.required:
+                    changes.append(f"required={self.new.required}")
+                if self.old.unique_index != self.new.unique_index:
+                    changes.append(f"unique_index={self.new.unique_index}")
+                if self.old.index != self.new.index:
+                    changes.append(f"index={self.new.index}")
+            detail = ", ".join(changes) or "modified"
+            return f"  ~ property '{self.property_name}' ({detail})"
+
+
+@dataclass
+class NodeChange:
+    """Change descriptor for a node."""
+
+    kind: str  # "added", "removed", "modified"
+    kgid: str
+    old: "NodeIR | None" = None
+    new: "NodeIR | None" = None
+    property_changes: list[PropertyChange] = field(default_factory=list)
+    label_renamed: bool = False
+
+    @property
+    def description(self) -> str:
+        if self.kind == "added":
+            return f"+ Node '{self.new.label}' (kgid={self.kgid})"  # type: ignore[union-attr]
+        elif self.kind == "removed":
+            return f"- Node '{self.old.label}' (kgid={self.kgid})"  # type: ignore[union-attr]
+        else:
+            parts = [f"~ Node kgid={self.kgid}"]
+            if self.label_renamed:
+                parts.append(
+                    f"  renamed '{self.old.label}' -> '{self.new.label}'"  # type: ignore[union-attr]
+                )
+            for pc in self.property_changes:
+                parts.append(pc.description)
+            return "\n".join(parts)
+
+
+@dataclass
+class RelationshipChange:
+    """Change descriptor for a relationship model."""
+
+    kind: str
+    kgid: str
+    old: "RelationshipIR | None" = None
+    new: "RelationshipIR | None" = None
+    property_changes: list[PropertyChange] = field(default_factory=list)
+
+    @property
+    def description(self) -> str:
+        if self.kind == "added":
+            return f"+ Relationship '{self.new.rel_type}' (kgid={self.kgid})"  # type: ignore[union-attr]
+        elif self.kind == "removed":
+            return f"- Relationship '{self.old.rel_type}' (kgid={self.kgid})"  # type: ignore[union-attr]
+        else:
+            parts = [f"~ Relationship kgid={self.kgid}"]
+            for pc in self.property_changes:
+                parts.append(pc.description)
+            return "\n".join(parts)
+
+
+@dataclass
+class SchemaDiff:
+    """Full diff between two schema snapshots."""
+
+    node_changes: list[NodeChange] = field(default_factory=list)
+    relationship_changes: list[RelationshipChange] = field(default_factory=list)
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.node_changes or self.relationship_changes)
+
+    def summary(self) -> str:
+        if not self.has_changes:
+            return "No schema changes detected."
+        lines: list[str] = []
+        for nc in self.node_changes:
+            lines.append(nc.description)
+        for rc in self.relationship_changes:
+            lines.append(rc.description)
+        return "\n".join(lines)
+
+
+def _diff_properties(
+    old_props: tuple["PropertyIR", ...],
+    new_props: tuple["PropertyIR", ...],
+) -> list[PropertyChange]:
+    """Diff two sets of properties by name."""
+    old_map = {p.name: p for p in old_props}
+    new_map = {p.name: p for p in new_props}
+    changes: list[PropertyChange] = []
+
+    for name in sorted(set(old_map) | set(new_map)):
+        old_p = old_map.get(name)
+        new_p = new_map.get(name)
+        if old_p is None and new_p is not None:
+            changes.append(PropertyChange(kind="added", property_name=name, new=new_p))
+        elif old_p is not None and new_p is None:
+            changes.append(PropertyChange(kind="removed", property_name=name, old=old_p))
+        elif old_p != new_p:
+            changes.append(
+                PropertyChange(kind="modified", property_name=name, old=old_p, new=new_p)
+            )
+
+    return changes
+
+
+def diff_snapshots(
+    old: "SchemaSnapshot", new: "SchemaSnapshot"
+) -> SchemaDiff:
+    """Compute a SchemaDiff between two snapshots keyed by __kgid__."""
+    diff = SchemaDiff()
+
+    old_nodes = old.nodes_by_kgid
+    new_nodes = new.nodes_by_kgid
+
+    for kgid in sorted(set(old_nodes) | set(new_nodes)):
+        old_n = old_nodes.get(kgid)
+        new_n = new_nodes.get(kgid)
+        if old_n is None and new_n is not None:
+            diff.node_changes.append(NodeChange(kind="added", kgid=kgid, new=new_n))
+        elif old_n is not None and new_n is None:
+            diff.node_changes.append(NodeChange(kind="removed", kgid=kgid, old=old_n))
+        elif old_n != new_n:
+            prop_changes = _diff_properties(old_n.properties, new_n.properties)  # type: ignore[union-attr]
+            label_renamed = old_n.label != new_n.label  # type: ignore[union-attr]
+            if prop_changes or label_renamed:
+                diff.node_changes.append(
+                    NodeChange(
+                        kind="modified",
+                        kgid=kgid,
+                        old=old_n,
+                        new=new_n,
+                        property_changes=prop_changes,
+                        label_renamed=label_renamed,
+                    )
+                )
+
+    old_rels = old.relationships_by_kgid
+    new_rels = new.relationships_by_kgid
+
+    for kgid in sorted(set(old_rels) | set(new_rels)):
+        old_r = old_rels.get(kgid)
+        new_r = new_rels.get(kgid)
+        if old_r is None and new_r is not None:
+            diff.relationship_changes.append(
+                RelationshipChange(kind="added", kgid=kgid, new=new_r)
+            )
+        elif old_r is not None and new_r is None:
+            diff.relationship_changes.append(
+                RelationshipChange(kind="removed", kgid=kgid, old=old_r)
+            )
+        elif old_r != new_r:
+            prop_changes = _diff_properties(old_r.properties, new_r.properties)  # type: ignore[union-attr]
+            if prop_changes:
+                diff.relationship_changes.append(
+                    RelationshipChange(
+                        kind="modified",
+                        kgid=kgid,
+                        old=old_r,
+                        new=new_r,
+                        property_changes=prop_changes,
+                    )
+                )
+
+    return diff

crochet/ir/hash.py

@@ -0,0 +1,36 @@
+"""Deterministic hashing for schema snapshots."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import replace
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from crochet.ir.schema import SchemaSnapshot
+
+
+def _canonical_json(snapshot: "SchemaSnapshot") -> str:
+    """Produce a deterministic JSON string for hashing.
+
+    We exclude ``created_at`` and ``schema_hash`` so that two snapshots with
+    identical structure always produce the same hash regardless of when they
+    were created.
+    """
+    d = snapshot.to_dict()
+    d.pop("created_at", None)
+    d.pop("schema_hash", None)
+    return json.dumps(d, sort_keys=True, separators=(",", ":"))
+
+
+def compute_hash(snapshot: "SchemaSnapshot") -> str:
+    """Return the SHA-256 hex digest for a snapshot's canonical form."""
+    canonical = _canonical_json(snapshot)
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def hash_snapshot(snapshot: "SchemaSnapshot") -> "SchemaSnapshot":
+    """Return a new snapshot with ``schema_hash`` populated."""
+    h = compute_hash(snapshot)
+    return replace(snapshot, schema_hash=h)

crochet/ir/parser.py

@@ -0,0 +1,251 @@
+"""Parse neomodel Python files into schema IR without a Neo4j connection."""
+
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import inspect
+import sys
+from pathlib import Path
+from typing import Any
+
+from neomodel import (
+    RelationshipFrom,
+    RelationshipTo,
+    StructuredNode,
+    StructuredRel,
+)
+from neomodel.properties import Property
+
+from crochet.errors import DuplicateKGIDError, MissingKGIDError
+from crochet.ir.hash import hash_snapshot
+from crochet.ir.schema import (
+    NodeIR,
+    PropertyIR,
+    RelationshipDefIR,
+    RelationshipIR,
+    SchemaSnapshot,
+)
+
+# Relationship manager classes we recognise and their direction labels.
+_REL_MANAGERS: dict[type, str] = {}
+
+
+def _init_rel_managers() -> None:
+    """Populate the relationship-manager mapping lazily."""
+    if _REL_MANAGERS:
+        return
+    _REL_MANAGERS[RelationshipTo] = "to"
+    _REL_MANAGERS[RelationshipFrom] = "from"
+    try:
+        from neomodel import Relationship
+
+        _REL_MANAGERS[Relationship] = "either"
+    except ImportError:
+        pass
+
+
+def _extract_property_ir(name: str, prop: Property) -> PropertyIR:
+    """Turn a neomodel Property instance into a PropertyIR."""
+    return PropertyIR(
+        name=name,
+        property_type=type(prop).__name__,
+        required=getattr(prop, "required", False),
+        unique_index=getattr(prop, "unique_index", False),
+        index=getattr(prop, "index", False),
+        default=None,  # we don't capture callables
+        choices=tuple(prop.choices) if getattr(prop, "choices", None) else None,
+    )
+
+
+def _extract_rel_def_ir(attr_name: str, rel_mgr: Any) -> RelationshipDefIR | None:
+    """Turn a neomodel relationship manager into a RelationshipDefIR."""
+    _init_rel_managers()
+    mgr_type = type(rel_mgr)
+    direction = None
+    for cls, dir_label in _REL_MANAGERS.items():
+        if issubclass(mgr_type, cls):
+            direction = dir_label
+            break
+    if direction is None:
+        return None
+
+    definition = rel_mgr.definition
+    rel_type = definition.get("relation_type", "RELATED_TO")
+
+    # Resolve the target model label
+    target_cls = definition.get("node_class")
+    if isinstance(target_cls, type) and issubclass(target_cls, StructuredNode):
+        target_label = getattr(target_cls, "__label__", target_cls.__name__)
+    else:
+        target_label = str(target_cls) if target_cls else "UNKNOWN"
+
+    # Check for a StructuredRel model
+    model_cls = definition.get("model")
+    model_kgid = None
+    if model_cls is not None and isinstance(model_cls, type):
+        model_kgid = getattr(model_cls, "__kgid__", None)
+
+    return RelationshipDefIR(
+        attr_name=attr_name,
+        rel_type=rel_type,
+        target_label=target_label,
+        direction=direction,
+        model_kgid=model_kgid,
+    )
+
+
+def _parse_node_class(cls: type) -> NodeIR:
+    """Parse a single StructuredNode subclass into a NodeIR."""
+    kgid = getattr(cls, "__kgid__", None)
+    if kgid is None:
+        raise MissingKGIDError(cls.__name__)
+
+    label = getattr(cls, "__label__", cls.__name__)
+    module_path = cls.__module__
+
+    properties: list[PropertyIR] = []
+    rel_defs: list[RelationshipDefIR] = []
+
+    # Walk class attributes (not parent StructuredNode's)
+    for attr_name in dir(cls):
+        if attr_name.startswith("_"):
+            continue
+        try:
+            attr = getattr(cls, attr_name)
+        except Exception:
+            continue
+
+        if isinstance(attr, Property):
+            properties.append(_extract_property_ir(attr_name, attr))
+        else:
+            rd = _extract_rel_def_ir(attr_name, attr)
+            if rd is not None:
+                rel_defs.append(rd)
+
+    return NodeIR(
+        kgid=kgid,
+        label=label,
+        class_name=cls.__name__,
+        module_path=module_path,
+        properties=tuple(sorted(properties)),
+        relationship_defs=tuple(rel_defs),
+    )
+
+
+def _parse_rel_class(cls: type) -> RelationshipIR:
+    """Parse a single StructuredRel subclass into a RelationshipIR."""
+    kgid = getattr(cls, "__kgid__", None)
+    if kgid is None:
+        raise MissingKGIDError(cls.__name__)
+
+    rel_type = getattr(cls, "__type__", cls.__name__.upper())
+    module_path = cls.__module__
+
+    properties: list[PropertyIR] = []
+    for attr_name in dir(cls):
+        if attr_name.startswith("_"):
+            continue
+        try:
+            attr = getattr(cls, attr_name)
+        except Exception:
+            continue
+        if isinstance(attr, Property):
+            properties.append(_extract_property_ir(attr_name, attr))
+
+    return RelationshipIR(
+        kgid=kgid,
+        rel_type=rel_type,
+        class_name=cls.__name__,
+        module_path=module_path,
+        properties=tuple(sorted(properties)),
+    )
+
+
+def _clear_neomodel_registry() -> None:
+    """Remove all entries from neomodel's class registry so re-imports work."""
+    try:
+        from neomodel import db
+
+        if hasattr(db, "_NODE_CLASS_REGISTRY"):
+            db._NODE_CLASS_REGISTRY.clear()
+        if hasattr(db, "_DB_SPECIFIC_CLASS_REGISTRY"):
+            db._DB_SPECIFIC_CLASS_REGISTRY.clear()
+    except (ImportError, AttributeError):
+        pass
+
+
+_load_counter = 0
+
+
+def _load_module_from_path(file_path: Path) -> Any:
+    """Import a Python file as a module.
+
+    Each call uses a unique module name so that neomodel's global class
+    registry does not raise ``NodeClassAlreadyDefined`` on repeated parses
+    of the same label across different directories or test runs.
+    """
+    global _load_counter
+    _load_counter += 1
+    module_name = f"crochet._user_models._{_load_counter}_{file_path.stem}"
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None or spec.loader is None:
+        return None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    try:
+        spec.loader.exec_module(module)
+    except Exception:
+        # If the module fails to import, skip it rather than crashing
+        del sys.modules[module_name]
+        raise
+    return module
+
+
+def parse_module(module: Any) -> tuple[list[NodeIR], list[RelationshipIR]]:
+    """Extract all NodeIR and RelationshipIR from an already-loaded module."""
+    nodes: list[NodeIR] = []
+    rels: list[RelationshipIR] = []
+
+    for _name, obj in inspect.getmembers(module, inspect.isclass):
+        if obj.__module__ != module.__name__:
+            continue  # skip imported classes
+        if issubclass(obj, StructuredNode) and obj is not StructuredNode:
+            nodes.append(_parse_node_class(obj))
+        elif issubclass(obj, StructuredRel) and obj is not StructuredRel:
+            rels.append(_parse_rel_class(obj))
+
+    return nodes, rels
+
+
+def parse_models_directory(models_dir: Path) -> SchemaSnapshot:
+    """Parse all .py files in *models_dir* and return a SchemaSnapshot."""
+    _clear_neomodel_registry()
+
+    all_nodes: list[NodeIR] = []
+    all_rels: list[RelationshipIR] = []
+    seen_kgids: dict[str, str] = {}
+
+    py_files = sorted(models_dir.rglob("*.py"))
+    for py_file in py_files:
+        if py_file.name.startswith("_"):
+            continue
+        module = _load_module_from_path(py_file)
+        if module is None:
+            continue
+        nodes, rels = parse_module(module)
+
+        for n in nodes:
+            if n.kgid in seen_kgids:
+                raise DuplicateKGIDError(n.kgid, seen_kgids[n.kgid], n.class_name)
+            seen_kgids[n.kgid] = n.class_name
+            all_nodes.append(n)
+
+        for r in rels:
+            if r.kgid in seen_kgids:
+                raise DuplicateKGIDError(r.kgid, seen_kgids[r.kgid], r.class_name)
+            seen_kgids[r.kgid] = r.class_name
+            all_rels.append(r)
+
+    snapshot = SchemaSnapshot(nodes=tuple(all_nodes), relationships=tuple(all_rels))
+    return hash_snapshot(snapshot)

crochet/ir/schema.py

@@ -0,0 +1,196 @@
+"""Data structures for the schema Intermediate Representation."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+
+@dataclass(frozen=True, order=True)
+class PropertyIR:
+    """IR for a single property on a node or relationship."""
+
+    name: str
+    property_type: str  # e.g. "StringProperty", "IntegerProperty"
+    required: bool = False
+    unique_index: bool = False
+    index: bool = False
+    default: Any = None
+    choices: tuple | None = None
+
+    def to_dict(self) -> dict:
+        d: dict[str, Any] = {
+            "name": self.name,
+            "property_type": self.property_type,
+            "required": self.required,
+            "unique_index": self.unique_index,
+            "index": self.index,
+        }
+        if self.default is not None:
+            d["default"] = repr(self.default)
+        if self.choices is not None:
+            d["choices"] = list(self.choices)
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict) -> PropertyIR:
+        return cls(
+            name=d["name"],
+            property_type=d["property_type"],
+            required=d.get("required", False),
+            unique_index=d.get("unique_index", False),
+            index=d.get("index", False),
+            default=d.get("default"),
+            choices=tuple(d["choices"]) if d.get("choices") else None,
+        )
+
+
+@dataclass(frozen=True)
+class RelationshipDefIR:
+    """IR for a relationship definition on a node (e.g. RelationshipTo)."""
+
+    attr_name: str
+    rel_type: str  # Neo4j relationship type string, e.g. "FRIENDS_WITH"
+    target_label: str  # target node class name or label
+    direction: str  # "to", "from", "either"
+    model_kgid: str | None = None  # __kgid__ of the StructuredRel model, if any
+
+    def to_dict(self) -> dict:
+        d: dict[str, Any] = {
+            "attr_name": self.attr_name,
+            "rel_type": self.rel_type,
+            "target_label": self.target_label,
+            "direction": self.direction,
+        }
+        if self.model_kgid is not None:
+            d["model_kgid"] = self.model_kgid
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict) -> RelationshipDefIR:
+        return cls(
+            attr_name=d["attr_name"],
+            rel_type=d["rel_type"],
+            target_label=d["target_label"],
+            direction=d["direction"],
+            model_kgid=d.get("model_kgid"),
+        )
+
+
+@dataclass(frozen=True)
+class NodeIR:
+    """IR for a neomodel StructuredNode class."""
+
+    kgid: str
+    label: str  # Neo4j label (defaults to class name)
+    class_name: str
+    module_path: str
+    properties: tuple[PropertyIR, ...] = ()
+    relationship_defs: tuple[RelationshipDefIR, ...] = ()
+
+    def to_dict(self) -> dict:
+        return {
+            "kgid": self.kgid,
+            "label": self.label,
+            "class_name": self.class_name,
+            "module_path": self.module_path,
+            "properties": [p.to_dict() for p in sorted(self.properties)],
+            "relationship_defs": [r.to_dict() for r in self.relationship_defs],
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> NodeIR:
+        return cls(
+            kgid=d["kgid"],
+            label=d["label"],
+            class_name=d["class_name"],
+            module_path=d["module_path"],
+            properties=tuple(PropertyIR.from_dict(p) for p in d.get("properties", [])),
+            relationship_defs=tuple(
+                RelationshipDefIR.from_dict(r) for r in d.get("relationship_defs", [])
+            ),
+        )
+
+
+@dataclass(frozen=True)
+class RelationshipIR:
+    """IR for a neomodel StructuredRel class."""
+
+    kgid: str
+    rel_type: str  # Neo4j relationship type string
+    class_name: str
+    module_path: str
+    properties: tuple[PropertyIR, ...] = ()
+
+    def to_dict(self) -> dict:
+        return {
+            "kgid": self.kgid,
+            "rel_type": self.rel_type,
+            "class_name": self.class_name,
+            "module_path": self.module_path,
+            "properties": [p.to_dict() for p in sorted(self.properties)],
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> RelationshipIR:
+        return cls(
+            kgid=d["kgid"],
+            rel_type=d["rel_type"],
+            class_name=d["class_name"],
+            module_path=d["module_path"],
+            properties=tuple(PropertyIR.from_dict(p) for p in d.get("properties", [])),
+        )
+
+
+@dataclass(frozen=True)
+class SchemaSnapshot:
+    """Immutable snapshot of the full schema IR at a point in time."""
+
+    nodes: tuple[NodeIR, ...]
+    relationships: tuple[RelationshipIR, ...]
+    created_at: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+    schema_hash: str = ""
+
+    @property
+    def nodes_by_kgid(self) -> dict[str, NodeIR]:
+        return {n.kgid: n for n in self.nodes}
+
+    @property
+    def relationships_by_kgid(self) -> dict[str, RelationshipIR]:
+        return {r.kgid: r for r in self.relationships}
+
+    def to_dict(self) -> dict:
+        return {
+            "nodes": [n.to_dict() for n in sorted(self.nodes, key=lambda n: n.kgid)],
+            "relationships": [
+                r.to_dict() for r in sorted(self.relationships, key=lambda r: r.kgid)
+            ],
+            "created_at": self.created_at,
+            "schema_hash": self.schema_hash,
+        }
+
+    def to_json(self, indent: int = 2) -> str:
+        return json.dumps(self.to_dict(), indent=indent, sort_keys=True)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> SchemaSnapshot:
+        return cls(
+            nodes=tuple(NodeIR.from_dict(n) for n in d.get("nodes", [])),
+            relationships=tuple(
+                RelationshipIR.from_dict(r) for r in d.get("relationships", [])
+            ),
+            created_at=d.get("created_at", ""),
+            schema_hash=d.get("schema_hash", ""),
+        )
+
+    @classmethod
+    def from_json(cls, raw: str) -> SchemaSnapshot:
+        return cls.from_dict(json.loads(raw))
+
+    @classmethod
+    def empty(cls) -> SchemaSnapshot:
+        return cls(nodes=(), relationships=())

crochet/ledger/__init__.py

@@ -0,0 +1,5 @@
+"""SQLite-backed ledger for tracking migrations and data batches."""
+
+from crochet.ledger.sqlite import Ledger
+
+__all__ = ["Ledger"]