infrahub-server 1.2.9rc0__py3-none-any.whl → 1.3.0a0__py3-none-any.whl

infrahub/core/convert_object_type/conversion.py

@@ -0,0 +1,122 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from infrahub.core.attribute import BaseAttribute
+from infrahub.core.branch import Branch
+from infrahub.core.constants import RelationshipCardinality
+from infrahub.core.manager import NodeManager
+from infrahub.core.node import Node
+from infrahub.core.node.create import create_node
+from infrahub.core.query.relationship import GetAllPeersIds
+from infrahub.core.relationship import RelationshipManager
+from infrahub.core.schema import NodeSchema
+from infrahub.database import InfrahubDatabase
+
+
+class InputDataForDestField(BaseModel):  # Only one of these fields can be not None
+    attribute_value: Any | None = None
+    peer_id: str | None = None
+    peers_ids: list[str] | None = None
+
+    @property
+    def value(self) -> Any:
+        fields = [self.attribute_value, self.peer_id, self.peers_ids]
+        set_fields = [f for f in fields if f is not None]
+        if len(set_fields) != 1:
+            raise ValueError("Exactly one of attribute_value, peer_id, or peers_ids must be set")
+        return set_fields[0]
+
+
+class InputForDestField(BaseModel):  # Only one of these fields can be not None
+    source_field: str | None = None
+    data: InputDataForDestField | None = None
+
+    @property
+    def value(self) -> Any:
+        if self.source_field is not None and self.data is not None:
+            raise ValueError("Only one of source_field or data can be set")
+        if self.source_field is None and self.data is None:
+            raise ValueError("Either source_field or data must be set")
+        return self.source_field if self.source_field is not None else self.data
+
+
+async def get_out_rels_peers_ids(node: Node, db: InfrahubDatabase) -> list[str]:
+    all_peers: list[Node] = []
+    for name in node._relationships:
+        relm: RelationshipManager = getattr(node, name)
+        peers = await relm.get_peers(db=db)
+        all_peers.extend(peers.values())
+    return [peer.id for peer in all_peers]
+
+
+async def build_data_new_node(db: InfrahubDatabase, mapping: dict[str, InputForDestField], node: Node) -> dict:
+    """Value of a given field on the target kind to convert is either an input source attribute/relationship of the source node,
+    or a raw value."""
+
+    data = {}
+    for dest_field_name, input_for_dest_field in mapping.items():
+        value = input_for_dest_field.value
+        if isinstance(value, str):  # source_field
+            item = getattr(node, value)
+            if isinstance(item, BaseAttribute):
+                data[dest_field_name] = item.value
+            elif isinstance(item, RelationshipManager):
+                if item.schema.cardinality == RelationshipCardinality.ONE:
+                    peer = await item.get_peer(db=db)
+                    if peer is not None:
+                        data[dest_field_name] = {"id": peer.id}
+                    # else, relationship is optional, and if the target relationship is mandatory an error will be raised during creation
+                elif item.schema.cardinality == RelationshipCardinality.MANY:
+                    data[dest_field_name] = [{"id": peer.id} for _, peer in (await item.get_peers(db=db)).items()]
+                else:
+                    raise ValueError(f"Unknown cardinality {item.schema.cardinality=}")
+        else:  # user input data
+            data[dest_field_name] = value.value
+    return data
+
+
+async def get_unidirectional_rels_peers_ids(node: Node, branch: Branch, db: InfrahubDatabase) -> list[str]:
+    """
+    Returns peers ids of nodes connected to input `node` through an incoming unidirectional relationship.
+    """
+
+    out_rels_identifier = [rel.identifier for rel in node.get_schema().relationships]
+    query = await GetAllPeersIds.init(db=db, node_id=node.id, branch=branch, exclude_identifiers=out_rels_identifier)
+    await query.execute(db=db)
+    return query.get_peers_uuids()
+
+
+async def convert_object_type(
+    node: Node, target_schema: NodeSchema, mapping: dict[str, InputForDestField], branch: Branch, db: InfrahubDatabase
+) -> Node:
+    """Delete the node and return the new created one. If creation fails, the node is not deleted, and raise an error.
+    An extra check is performed on input node peers relationships to make sure they are still valid."""
+
+    node_schema = node.get_schema()
+    if not isinstance(node_schema, NodeSchema):
+        raise ValueError(f"Only a node with a NodeSchema can be converted, got {type(node_schema)}")
+
+    async with db.start_transaction() as dbt:  # noqa: PLR1702
+        deleted_node_out_rels_peer_ids = await get_out_rels_peers_ids(node=node, db=dbt)
+        deleted_node_unidir_rels_peer_ids = await get_unidirectional_rels_peers_ids(node=node, db=dbt, branch=branch)
+
+        deleted_nodes = await NodeManager.delete(db=dbt, branch=branch, nodes=[node], cascade_delete=False)
+        if len(deleted_nodes) != 1:
+            raise ValueError(f"Deleted {len(deleted_nodes)} nodes instead of 1")
+
+        data_new_node = await build_data_new_node(dbt, mapping, node)
+        node_created = await create_node(
+            data=data_new_node,
+            db=dbt,
+            branch=branch,
+            schema=target_schema,
+        )
+
+        # Make sure relationships with constraints are not broken by retrieving them
+        peers_ids = deleted_node_out_rels_peer_ids + deleted_node_unidir_rels_peer_ids
+        peers = await NodeManager.get_many(ids=peers_ids, db=dbt, prefetch_relationships=True, branch=branch)
+        for peer in peers.values():
+            peer.validate_relationships()
+
+        return node_created

infrahub/core/convert_object_type/schema_mapping.py

@@ -0,0 +1,56 @@
+from pydantic import BaseModel
+
+from infrahub.core.constants import RelationshipCardinality
+from infrahub.core.schema import NodeSchema
+
+
+class SchemaMappingValue(BaseModel):
+    is_mandatory: bool
+    source_field_name: str | None = None  # None means there is no corresponding source field name
+    relationship_cardinality: RelationshipCardinality | None = None
+
+
+SchemaMapping = dict[str, SchemaMappingValue]
+
+
+def get_schema_mapping(source_schema: NodeSchema, target_schema: NodeSchema) -> SchemaMapping:
+    """
+    Return fields mapping meant to be used for converting a node from `source_kind` to `target_kind`.
+    For any field of the target kind, field of the source kind will be matched if:
+    - It's an attribute with identical name and type.
+    - It's a relationship with identical peer kind and cardinality.
+    If there is no match, the mapping will only indicate whether the field is mandatory or not.
+    """
+
+    target_field_to_source_field = {}
+
+    # Create lookup dictionaries for source attributes and relationships
+    source_attrs = {attr.name: attr for attr in source_schema.attributes}
+    source_rels = {rel.name: rel for rel in source_schema.relationships}
+
+    # Process attributes
+    for target_attr in target_schema.attributes:
+        source_attr = source_attrs.get(target_attr.name)
+        if source_attr and source_attr.kind == target_attr.kind:
+            target_field_to_source_field[target_attr.name] = SchemaMappingValue(
+                source_field_name=source_attr.name, is_mandatory=not target_attr.optional
+            )
+        else:
+            target_field_to_source_field[target_attr.name] = SchemaMappingValue(is_mandatory=not target_attr.optional)
+
+    # Process relationships
+    for target_rel in target_schema.relationships:
+        source_rel = source_rels.get(target_rel.name)
+        if source_rel and source_rel.peer == target_rel.peer and source_rel.cardinality == target_rel.cardinality:
+            target_field_to_source_field[target_rel.name] = SchemaMappingValue(
+                source_field_name=source_rel.name,
+                is_mandatory=not target_rel.optional,
+                relationship_cardinality=target_rel.cardinality,
+            )
+        else:
+            target_field_to_source_field[target_rel.name] = SchemaMappingValue(
+                is_mandatory=not target_rel.optional,
+                relationship_cardinality=target_rel.cardinality,
+            )
+
+    return target_field_to_source_field

infrahub/core/diff/calculator.py

@@ -7,6 +7,7 @@ from infrahub.core.diff.query_parser import DiffQueryParser
 from infrahub.core.query.diff import (
     DiffCalculationQuery,
     DiffFieldPathsQuery,
+    DiffMigratedKindNodesQuery,
     DiffNodePathsQuery,
     DiffPropertyPathsQuery,
 )
@@ -14,7 +15,8 @@ from infrahub.core.timestamp import Timestamp
 from infrahub.database import InfrahubDatabase
 from infrahub.log import get_logger
 
-from .model.path import CalculatedDiffs
+from .model.field_specifiers_map import NodeFieldSpecifierMap
+from .model.path import CalculatedDiffs, DiffNode, DiffRoot, NodeIdentifier
 
 log = get_logger()
 
@@ -26,8 +28,8 @@ class DiffCalculationRequest:
     branch_from_time: Timestamp
     from_time: Timestamp
     to_time: Timestamp
-    current_node_field_specifiers: dict[str, set[str]] | None = field(default=None)
-    new_node_field_specifiers: dict[str, set[str]] | None = field(default=None)
+    current_node_field_specifiers: NodeFieldSpecifierMap | None = field(default=None)
+    new_node_field_specifiers: NodeFieldSpecifierMap | None = field(default=None)
 
 
 class DiffCalculator:
@@ -58,7 +60,7 @@ class DiffCalculator:
             )
             log.info(f"Beginning one diff calculation query {limit=}, {offset=}")
             await diff_query.execute(db=self.db)
-            log.info("Diff calculation query complete")
+            log.info(f"Diff calculation query complete {limit=}, {offset=}")
             last_result = None
             for query_result in diff_query.get_results():
                 diff_parser.read_result(query_result=query_result)
@@ -68,6 +70,56 @@ class DiffCalculator:
                 has_more_data = last_result.get_as_type("has_more_data", bool)
             offset += limit
 
+    async def _apply_kind_migrated_nodes(
+        self, branch_diff: DiffRoot, calculation_request: DiffCalculationRequest
+    ) -> None:
+        has_more_data = True
+        offset = 0
+        limit = config.SETTINGS.database.query_size_limit
+        diff_nodes_by_identifier = {n.identifier: n for n in branch_diff.nodes}
+        diff_nodes_to_add: list[DiffNode] = []
+        while has_more_data:
+            diff_query = await DiffMigratedKindNodesQuery.init(
+                db=self.db,
+                branch=calculation_request.diff_branch,
+                base_branch=calculation_request.base_branch,
+                diff_branch_from_time=calculation_request.branch_from_time,
+                diff_from=calculation_request.from_time,
+                diff_to=calculation_request.to_time,
+                limit=limit,
+                offset=offset,
+            )
+            log.info(f"Getting one batch of migrated kind nodes {limit=}, {offset=}")
+            await diff_query.execute(db=self.db)
+            log.info(f"Migrated kind nodes query complete {limit=}, {offset=}")
+            last_result = None
+            for migrated_kind_node in diff_query.get_migrated_kind_nodes():
+                migrated_kind_identifier = NodeIdentifier(
+                    uuid=migrated_kind_node.uuid,
+                    kind=migrated_kind_node.kind,
+                    db_id=migrated_kind_node.db_id,
+                )
+                if migrated_kind_identifier in diff_nodes_by_identifier:
+                    diff_node = diff_nodes_by_identifier[migrated_kind_identifier]
+                    diff_node.is_node_kind_migration = True
+                    continue
+                new_diff_node = DiffNode(
+                    identifier=migrated_kind_identifier,
+                    changed_at=migrated_kind_node.from_time,
+                    action=migrated_kind_node.action,
+                    is_node_kind_migration=True,
+                    attributes=[],
+                    relationships=[],
+                )
+                diff_nodes_by_identifier[migrated_kind_identifier] = new_diff_node
+                diff_nodes_to_add.append(new_diff_node)
+                last_result = migrated_kind_node
+            has_more_data = False
+            if last_result:
+                has_more_data = last_result.has_more_data
+            offset += limit
+        branch_diff.nodes.extend(diff_nodes_to_add)
+
     async def calculate_diff(
         self,
         base_branch: Branch,
@@ -75,7 +127,7 @@ class DiffCalculator:
         from_time: Timestamp,
         to_time: Timestamp,
         include_unchanged: bool = True,
-        previous_node_specifiers: dict[str, set[str]] | None = None,
+        previous_node_specifiers: NodeFieldSpecifierMap | None = None,
     ) -> CalculatedDiffs:
         if diff_branch.name == registry.default_branch:
             diff_branch_from_time = from_time
@@ -91,7 +143,7 @@ class DiffCalculator:
         )
         node_limit = int(config.SETTINGS.database.query_size_limit / 10)
         fields_limit = int(config.SETTINGS.database.query_size_limit / 3)
-        properties_limit = int(config.SETTINGS.database.query_size_limit)
+        properties_limit = config.SETTINGS.database.query_size_limit
 
         calculation_request = DiffCalculationRequest(
             base_branch=base_branch,
@@ -131,7 +183,7 @@ class DiffCalculator:
         if base_branch.name != diff_branch.name:
             current_node_field_specifiers = diff_parser.get_current_node_field_specifiers()
             new_node_field_specifiers = diff_parser.get_new_node_field_specifiers()
-            calculation_request = DiffCalculationRequest(
+            base_calculation_request = DiffCalculationRequest(
                 base_branch=base_branch,
                 diff_branch=base_branch,
                 branch_from_time=diff_branch_from_time,
@@ -145,7 +197,7 @@ class DiffCalculator:
             await self._run_diff_calculation_query(
                 diff_parser=diff_parser,
                 query_class=DiffNodePathsQuery,
-                calculation_request=calculation_request,
+                calculation_request=base_calculation_request,
                 limit=node_limit,
             )
             log.info("Diff node-level calculation queries for base complete")
@@ -154,7 +206,7 @@ class DiffCalculator:
             await self._run_diff_calculation_query(
                 diff_parser=diff_parser,
                 query_class=DiffFieldPathsQuery,
-                calculation_request=calculation_request,
+                calculation_request=base_calculation_request,
                 limit=fields_limit,
             )
             log.info("Diff field-level calculation queries for base complete")
@@ -163,7 +215,7 @@ class DiffCalculator:
             await self._run_diff_calculation_query(
                 diff_parser=diff_parser,
                 query_class=DiffPropertyPathsQuery,
-                calculation_request=calculation_request,
+                calculation_request=base_calculation_request,
                 limit=properties_limit,
             )
             log.info("Diff property-level calculation queries for base complete")
@@ -171,9 +223,11 @@ class DiffCalculator:
         log.info("Parsing calculated diff")
         diff_parser.parse(include_unchanged=include_unchanged)
         log.info("Calculated diff parsed")
+        branch_diff = diff_parser.get_diff_root_for_branch(branch=diff_branch.name)
+        await self._apply_kind_migrated_nodes(branch_diff=branch_diff, calculation_request=calculation_request)
         return CalculatedDiffs(
             base_branch_name=base_branch.name,
             diff_branch_name=diff_branch.name,
             base_branch_diff=diff_parser.get_diff_root_for_branch(branch=base_branch.name),
-            diff_branch_diff=diff_parser.get_diff_root_for_branch(branch=diff_branch.name),
+            diff_branch_diff=branch_diff,
         )

infrahub/core/diff/combiner.py

@@ -14,6 +14,7 @@ from .model.path import (
     EnrichedDiffRoot,
     EnrichedDiffs,
     EnrichedDiffSingleRelationship,
+    NodeIdentifier,
 )
 
 
@@ -26,30 +27,35 @@ class NodePair:
 class DiffCombiner:
     def __init__(self) -> None:
         # {child_uuid: (parent_uuid, parent_rel_name)}
-        self._child_parent_uuid_map: dict[str, tuple[str, str]] = {}
-        self._parent_node_uuids: set[str] = set()
-        self._earlier_nodes_by_uuid: dict[str, EnrichedDiffNode] = {}
-        self._later_nodes_by_uuid: dict[str, EnrichedDiffNode] = {}
-        self._common_node_uuids: set[str] = set()
+        self._child_parent_identifier_map: dict[NodeIdentifier, tuple[NodeIdentifier, str]] = {}
+        self._parent_node_identifiers: set[NodeIdentifier] = set()
+        self._earlier_nodes_by_identifier: dict[NodeIdentifier, EnrichedDiffNode] = {}
+        self._later_nodes_by_identifier: dict[NodeIdentifier, EnrichedDiffNode] = {}
+        self._common_node_identifiers: set[NodeIdentifier] = set()
         self._diff_branch_name: str | None = None
 
     def _initialize(self, earlier_diff: EnrichedDiffRoot, later_diff: EnrichedDiffRoot) -> None:
         self._diff_branch_name = earlier_diff.diff_branch_name
-        self._child_parent_uuid_map = {}
-        self._earlier_nodes_by_uuid = {}
-        self._later_nodes_by_uuid = {}
-        self._common_node_uuids = set()
+        self._child_parent_identifier_map = {}
+        self._earlier_nodes_by_identifier = {}
+        self._later_nodes_by_identifier = {}
+        self._common_node_identifiers = set()
         # map the parent of each node (if it exists), preference to the later diff
         for diff_root in (earlier_diff, later_diff):
             for child_node in diff_root.nodes:
                 for parent_rel in child_node.relationships:
                     for parent_node in parent_rel.nodes:
-                        self._child_parent_uuid_map[child_node.uuid] = (parent_node.uuid, parent_rel.name)
+                        self._child_parent_identifier_map[child_node.identifier] = (
+                            parent_node.identifier,
+                            parent_rel.name,
+                        )
         # UUIDs of all the parents, removing the stale parents from the earlier diff
-        self._parent_node_uuids = {parent_tuple[0] for parent_tuple in self._child_parent_uuid_map.values()}
-        self._earlier_nodes_by_uuid = {n.uuid: n for n in earlier_diff.nodes}
-        self._later_nodes_by_uuid = {n.uuid: n for n in later_diff.nodes}
-        self._common_node_uuids = set(self._earlier_nodes_by_uuid.keys()) & set(self._later_nodes_by_uuid.keys())
+        self._parent_node_identifiers = {parent_tuple[0] for parent_tuple in self._child_parent_identifier_map.values()}
+        self._earlier_nodes_by_identifier = {n.identifier: n for n in earlier_diff.nodes}
+        self._later_nodes_by_identifier = {n.identifier: n for n in later_diff.nodes}
+        self._common_node_identifiers = set(self._earlier_nodes_by_identifier.keys()) & set(
+            self._later_nodes_by_identifier.keys()
+        )
 
     @property
     def diff_branch_name(self) -> str:
@@ -61,13 +67,13 @@ class DiffCombiner:
         filtered_node_pairs: list[NodePair] = []
         for earlier_node in earlier_diff.nodes:
             later_node: EnrichedDiffNode | None = None
-            if earlier_node.uuid in self._common_node_uuids:
-                later_node = self._later_nodes_by_uuid[earlier_node.uuid]
+            if earlier_node.identifier in self._common_node_identifiers:
+                later_node = self._later_nodes_by_identifier[earlier_node.identifier]
             # this is an out-of-date parent
             if (
                 earlier_node.action is DiffAction.UNCHANGED
                 and (later_node is None or later_node.action is DiffAction.UNCHANGED)
-                and earlier_node.uuid not in self._parent_node_uuids
+                and earlier_node.identifier not in self._parent_node_identifiers
             ):
                 continue
             if later_node is None:
@@ -79,15 +85,15 @@ class DiffCombiner:
             filtered_node_pairs.append(NodePair(earlier=earlier_node, later=later_node))
         for later_node in later_diff.nodes:
             # these have already been handled
-            if later_node.uuid in self._common_node_uuids:
+            if later_node.identifier in self._common_node_identifiers:
                 continue
             filtered_node_pairs.append(NodePair(later=later_node))
         return filtered_node_pairs
 
-    def _get_parent_relationship_name(self, node_id: str) -> str | None:
-        if node_id not in self._child_parent_uuid_map:
+    def _get_parent_relationship_name(self, node_id: NodeIdentifier) -> str | None:
+        if node_id not in self._child_parent_identifier_map:
             return None
-        return self._child_parent_uuid_map[node_id][1]
+        return self._child_parent_identifier_map[node_id][1]
 
     def _should_include(self, earlier: DiffAction, later: DiffAction) -> bool:
         actions = {earlier, later}
@@ -284,7 +290,7 @@ class DiffCombiner:
         self,
         earlier_relationships: set[EnrichedDiffRelationship],
         later_relationships: set[EnrichedDiffRelationship],
-        node_id: str,
+        node_id: NodeIdentifier,
     ) -> set[EnrichedDiffRelationship]:
         earlier_rels_by_name = {rel.name: rel for rel in earlier_relationships}
         later_rels_by_name = {rel.name: rel for rel in later_relationships}
@@ -365,7 +371,7 @@ class DiffCombiner:
             combined_relationships = self._combine_relationships(
                 earlier_relationships=node_pair.earlier.relationships,
                 later_relationships=node_pair.later.relationships,
-                node_id=node_pair.later.uuid,
+                node_id=node_pair.later.identifier,
             )
             if all(ca.action is DiffAction.UNCHANGED for ca in combined_attributes) and all(
                 cr.action is DiffAction.UNCHANGED for cr in combined_relationships
@@ -380,15 +386,16 @@ class DiffCombiner:
                 combined_attributes
                 or combined_relationships
                 or combined_conflict
-                or node_pair.later.uuid in self._parent_node_uuids
+                or node_pair.later.identifier in self._parent_node_identifiers
             ):
                 combined_nodes.add(
                     EnrichedDiffNode(
-                        uuid=node_pair.later.uuid,
-                        kind=node_pair.later.kind,
+                        identifier=node_pair.later.identifier,
                         label=node_pair.later.label,
                         changed_at=node_pair.later.changed_at or node_pair.earlier.changed_at,
                         action=combined_action,
+                        is_node_kind_migration=node_pair.earlier.is_node_kind_migration
+                        or node_pair.later.is_node_kind_migration,
                         path_identifier=node_pair.later.path_identifier,
                         attributes=combined_attributes,
                         relationships=combined_relationships,
@@ -398,12 +405,12 @@ class DiffCombiner:
         return combined_nodes
 
     def _link_child_nodes(self, nodes: Iterable[EnrichedDiffNode]) -> None:
-        nodes_by_uuid: dict[str, EnrichedDiffNode] = {n.uuid: n for n in nodes}
-        for child_node in nodes_by_uuid.values():
-            if child_node.uuid not in self._child_parent_uuid_map:
+        nodes_by_identifier: dict[NodeIdentifier, EnrichedDiffNode] = {n.identifier: n for n in nodes}
+        for child_node in nodes_by_identifier.values():
+            if child_node.identifier not in self._child_parent_identifier_map:
                 continue
-            parent_uuid, parent_rel_name = self._child_parent_uuid_map[child_node.uuid]
-            parent_node = nodes_by_uuid[parent_uuid]
+            parent_identifier, parent_rel_name = self._child_parent_identifier_map[child_node.identifier]
+            parent_node = nodes_by_identifier[parent_identifier]
             parent_rel = child_node.get_relationship(name=parent_rel_name)
             parent_rel.nodes.add(parent_node)