infrahub-server 1.5.0b1__py3-none-any.whl → 1.5.1__py3-none-any.whl

infrahub/core/migrations/graph/m041_deleted_dup_edges.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from rich import print as rprint
+
+from infrahub.core.branch import Branch
+from infrahub.core.diff.repository.repository import DiffRepository
+from infrahub.core.initialization import get_root_node
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+from infrahub.dependencies.registry import build_component_registry, get_component_registry
+from infrahub.log import get_logger
+
+from ..shared import ArbitraryMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class DeletePosthumousEdges(Query):
+    name = "delete_posthumous_edges_query"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
+        query = """
+// ------------
+// find deleted nodes
+// ------------
+MATCH (n:Node)-[e:IS_PART_OF]->(:Root)
+WHERE e.status = "deleted" OR e.to IS NOT NULL
+WITH DISTINCT n, e.branch AS delete_branch, e.branch_level AS delete_branch_level, CASE
+    WHEN e.status = "deleted" THEN e.from
+    ELSE e.to
+END AS delete_time
+// ------------
+// find the edges added to the deleted node after the delete time
+// ------------
+MATCH (n)-[added_e]-(peer)
+WHERE added_e.from > delete_time
+AND type(added_e) <> "IS_PART_OF"
+// if the node was deleted on a branch (delete_branch_level > 1), and then updated on main/global (added_e.branch_level = 1), we can ignore it
+AND added_e.branch_level >= delete_branch_level
+AND (added_e.branch = delete_branch OR delete_branch_level = 1)
+WITH DISTINCT n, delete_branch, delete_time, added_e, peer
+// ------------
+// get the branched_from for the branch on which the node was deleted
+// ------------
+CALL (added_e) {
+    MATCH (b:Branch {name: added_e.branch})
+    RETURN b.branched_from AS added_e_branched_from
+}
+// ------------
+// account for the following situations, given that the edge update time is after the node delete time
+//  - deleted on main/global, updated on branch
+//    - illegal if the delete is before branch.branched_from
+//  - deleted on branch, updated on branch
+//    - illegal
+// ------------
+WITH n, delete_branch, delete_time, added_e, peer
+WHERE delete_branch = added_e.branch
+OR delete_time < added_e_branched_from
+DELETE added_e
+        """
+        self.add_to_query(query)
+
+
+class DeleteDuplicateEdgesForMigratedKindNodes(Query):
+    name = "delete_duplicate_edges_for_migrated_kind_nodes_query"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
+        query = """
+// ------------
+// get UUIDs for migrated kind/inheritance nodes
+// ------------
+MATCH (n:Node)
+WITH n.uuid AS node_uuid, count(*) AS num_nodes_with_uuid
+WHERE num_nodes_with_uuid > 1
+CALL (node_uuid) {
+    // ------------
+    // find any Relationships for these nodes
+    // ------------
+    MATCH (n:Node {uuid: node_uuid})-[:IS_RELATED]-(rel:Relationship)
+    WITH DISTINCT rel
+    MATCH (rel)-[e]->(peer)
+    WITH
+        type(e) AS e_type,
+        e.branch AS e_branch,
+        e.from AS e_from,
+        e.to AS e_to,
+        e.status AS e_status,
+        e.peer AS e_peer,
+        CASE
+            WHEN startNode(e) = rel THEN "out" ELSE "in"
+        END AS direction,
+        collect(e) AS duplicate_edges
+    WHERE size(duplicate_edges) > 1
+    WITH tail(duplicate_edges) AS duplicate_edges_to_delete
+    UNWIND duplicate_edges_to_delete AS edge_to_delete
+    DELETE edge_to_delete
+} IN TRANSACTIONS OF 500 ROWS
+        """
+        self.add_to_query(query)
+
+
+class Migration041(ArbitraryMigration):
+    """Clean up improper merges that duplicated edges to nodes with migrated kinds
+
+    - delete all existing diffs b/c they could contain incorrect nodes linking to deleted nodes with migrated kind/inheritance
+    - delete all edges added to any nodes AFTER they were deleted on main
+    - delete any duplicate edges touching migrated kind/inheritance nodes on main
+    """
+
+    name: str = "041_deleted_dup_edges"
+    minimum_version: int = 40
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        result = MigrationResult()
+
+        return result
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        root_node = await get_root_node(db=db)
+        default_branch_name = root_node.default_branch
+        default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
+
+        rprint("Deleting all diffs", end="...")
+        build_component_registry()
+        component_registry = get_component_registry()
+        diff_repo = await component_registry.get_component(DiffRepository, db=db, branch=default_branch)
+        await diff_repo.delete_all_diff_roots()
+        rprint("done")
+
+        rprint("Deleting edges merged after node deleted", end="...")
+        delete_posthumous_edges_query = await DeletePosthumousEdges.init(db=db)
+        await delete_posthumous_edges_query.execute(db=db)
+        rprint("done")
+
+        rprint("Deleting duplicate edges for migrated kind/inheritance nodes", end="...")
+        delete_duplicate_edges_query = await DeleteDuplicateEdgesForMigratedKindNodes.init(db=db)
+        await delete_duplicate_edges_query.execute(db=db)
+        rprint("done")
+
+        return MigrationResult()

infrahub/core/migrations/graph/m042_profile_attrs_in_db.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from rich.progress import Progress
+
+from infrahub.core.branch.models import Branch
+from infrahub.core.initialization import get_root_node
+from infrahub.core.manager import NodeManager
+from infrahub.core.migrations.shared import MigrationResult, get_migration_console
+from infrahub.core.query import Query, QueryType
+from infrahub.core.timestamp import Timestamp
+from infrahub.log import get_logger
+from infrahub.profiles.node_applier import NodeProfilesApplier
+
+from ..shared import MigrationRequiringRebase
+from .load_schema_branch import get_or_load_schema_branch
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class GetUpdatedProfilesForBranchQuery(Query):
+    """
+    Get CoreProfile UUIDs with updated attributes on this branch
+    """
+
+    name = "get_profiles_by_branch"
+    type = QueryType.READ
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (profile:CoreProfile)-[:HAS_ATTRIBUTE]->(attr:Attribute)-[e:HAS_VALUE]->(:AttributeValue)
+WHERE e.branch = $branch
+WITH DISTINCT profile.uuid AS profile_uuid
+        """
+        self.add_to_query(query)
+        self.return_labels = ["profile_uuid"]
+
+    def get_profile_ids(self) -> list[str]:
+        """Get list of updated profile UUIDs"""
+        return [result.get_as_type("profile_uuid", str) for result in self.get_results()]
+
+
+class GetNodesWithProfileUpdatesForBranchQuery(Query):
+    """
+    Get Node UUIDs by which branches they have updated profiles on
+    """
+
+    name = "get_nodes_with_profile_updates_by_branch"
+    type = QueryType.READ
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (node:Node)-[e:IS_RELATED]->(:Relationship {name: "node__profile"})
+WHERE NOT node:CoreProfile
+AND e.branch = $branch
+WITH DISTINCT node.uuid AS node_uuid
+        """
+        self.add_to_query(query)
+        self.return_labels = ["node_uuid"]
+
+    def get_node_ids(self) -> list[str]:
+        """Get list of updated node UUIDs"""
+        return [result.get_as_type("node_uuid", str) for result in self.get_results()]
+
+
+class Migration042(MigrationRequiringRebase):
+    """
+    Save profile attribute values on each node using the profile in the database
+    For any profile that has updates on a given branch (including default branch)
+    - run NodeProfilesApplier.apply_profiles on each node related to the profile on that branch
+    For any node that has an updated relationship to a profile on a given branch
+    - run NodeProfilesApplier.apply_profiles on the node on that branch
+    """
+
+    name: str = "042_profile_attrs_in_db"
+    minimum_version: int = 41
+
+    def _get_profile_applier(self, db: InfrahubDatabase, branch: Branch) -> NodeProfilesApplier:
+        return NodeProfilesApplier(db=db, branch=branch)
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        root_node = await get_root_node(db=db, initialize=False)
+        default_branch_name = root_node.default_branch
+        default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
+        return await self._do_execute_for_branch(db=db, branch=default_branch)
+
+    async def execute_against_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        return await self._do_execute_for_branch(db=db, branch=branch)
+
+    async def _do_execute_for_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        console = get_migration_console()
+        result = MigrationResult()
+        await get_or_load_schema_branch(db=db, branch=branch)
+
+        console.print(f"Gathering profiles for branch {branch.name}...", end="")
+        get_updated_profiles_for_branch_query = await GetUpdatedProfilesForBranchQuery.init(db=db, branch=branch)
+        await get_updated_profiles_for_branch_query.execute(db=db)
+        profile_ids = get_updated_profiles_for_branch_query.get_profile_ids()
+
+        profiles_map = await NodeManager.get_many(db=db, branch=branch, ids=list(profile_ids))
+        console.print("done")
+
+        node_ids_to_update: set[str] = set()
+        with Progress(console=console) as progress:
+            gather_nodes_task = progress.add_task(
+                f"Gathering affected objects for each profile on branch {branch.name}...", total=len(profiles_map)
+            )
+
+            for profile in profiles_map.values():
+                node_relationship_manager = profile.get_relationship("related_nodes")
+                node_peers = await node_relationship_manager.get_db_peers(db=db)
+                node_ids_to_update.update(str(peer.peer_id) for peer in node_peers)
+                progress.update(gather_nodes_task, advance=1)
+        console.log(f"Collected nodes impacted by profiles on branch {branch.name}.")
+
+        console.print("Identifying nodes with profile updates by branch...", end="")
+        get_nodes_with_profile_updates_by_branch_query = await GetNodesWithProfileUpdatesForBranchQuery.init(
+            db=db, branch=branch
+        )
+        await get_nodes_with_profile_updates_by_branch_query.execute(db=db)
+        node_ids_to_update.update(get_nodes_with_profile_updates_by_branch_query.get_node_ids())
+        console.print("done")
+
+        right_now = Timestamp()
+        console.log("Applying profiles to nodes...")
+        with Progress(console=console) as progress:
+            apply_task = progress.add_task("Applying profiles to nodes...", total=len(node_ids_to_update))
+            applier = self._get_profile_applier(db=db, branch=branch)
+            for node_id in node_ids_to_update:
+                node = await NodeManager.get_one(db=db, branch=branch, id=node_id, at=right_now)
+                if node:
+                    updated_field_names = await applier.apply_profiles(node=node)
+                    if updated_field_names:
+                        await node.save(db=db, fields=updated_field_names, at=right_now)
+                progress.update(apply_task, advance=1)
+        console.log("Completed applying profiles to nodes.")
+
+        return result

infrahub/core/migrations/graph/m043_create_hfid_display_label_in_db.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from rich.progress import Progress
+
+from infrahub.core import registry
+from infrahub.core.branch import Branch
+from infrahub.core.constants import SchemaPathType
+from infrahub.core.initialization import get_root_node
+from infrahub.core.migrations.schema.node_attribute_add import NodeAttributeAddMigration
+from infrahub.core.migrations.shared import MigrationRequiringRebase, MigrationResult, get_migration_console
+from infrahub.core.path import SchemaPath
+from infrahub.core.query import Query, QueryType
+
+from .load_schema_branch import get_or_load_schema_branch
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+class GetAddedNodesByKindForBranchQuery(Query):
+    name = "get_added_nodes_by_kind_for_branch_query"
+    type = QueryType.READ
+    insert_return = True
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (n:Node)-[e:IS_PART_OF {branch: $branch, status: "active"}]->(:Root)
+WHERE e.to IS NULL
+AND NOT exists((n)-[:IS_PART_OF {branch: $branch, status: "deleted"}]->(:Root))
+WITH n.kind AS kind, collect(n.uuid) AS node_ids
+        """
+        self.return_labels = ["kind", "node_ids"]
+        self.add_to_query(query)
+
+    def get_node_ids_by_kind(self) -> dict[str, list[str]]:
+        node_ids_by_kind: dict[str, list[str]] = {}
+        for result in self.get_results():
+            kind = result.get_as_type(label="kind", return_type=str)
+            node_ids: list[str] = result.get_as_type(label="node_ids", return_type=list)
+            node_ids_by_kind[kind] = node_ids
+        return node_ids_by_kind
+
+
+class Migration043(MigrationRequiringRebase):
+    name: str = "043_create_hfid_display_label_in_db"
+    minimum_version: int = 42
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        result = MigrationResult()
+
+        root_node = await get_root_node(db=db, initialize=False)
+        default_branch_name = root_node.default_branch
+        default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
+        main_schema_branch = await get_or_load_schema_branch(db=db, branch=default_branch)
+        schema_node = main_schema_branch.get_node(name="SchemaNode")
+        schema_generic = main_schema_branch.get_node(name="SchemaGeneric")
+
+        migrations = [
+            # HFID is not needed, it was introduced at graph v8
+            NodeAttributeAddMigration(
+                new_node_schema=schema_node,
+                previous_node_schema=schema_node,
+                schema_path=SchemaPath(
+                    schema_kind="SchemaNode", path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                ),
+            ),
+            NodeAttributeAddMigration(
+                new_node_schema=schema_generic,
+                previous_node_schema=schema_generic,
+                schema_path=SchemaPath(
+                    schema_kind="SchemaGeneric", path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                ),
+            ),
+        ]
+
+        for node_schema_kind in main_schema_branch.node_names:
+            schema = main_schema_branch.get(name=node_schema_kind, duplicate=False)
+            migrations.extend(
+                [
+                    NodeAttributeAddMigration(
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="human_friendly_id"
+                        ),
+                    ),
+                    NodeAttributeAddMigration(
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                        ),
+                    ),
+                ]
+            )
+
+        with Progress(console=get_migration_console()) as progress:
+            update_task = progress.add_task("Adding HFID and display label to nodes", total=len(migrations))
+
+            for migration in migrations:
+                try:
+                    execution_result = await migration.execute(db=db, branch=default_branch)
+                    result.errors.extend(execution_result.errors)
+                    progress.update(update_task, advance=1)
+                except Exception as exc:
+                    result.errors.append(str(exc))
+                    return result
+
+        return result
+
+    async def execute_against_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        result = MigrationResult()
+
+        schema_branch = await registry.schema.load_schema_from_db(db=db, branch=branch)
+
+        migrations = []
+        get_added_nodes_by_kind_for_branch_query = await GetAddedNodesByKindForBranchQuery.init(db=db, branch=branch)
+        await get_added_nodes_by_kind_for_branch_query.execute(db=db)
+        node_ids_by_kind = get_added_nodes_by_kind_for_branch_query.get_node_ids_by_kind()
+
+        for node_kind, node_ids in node_ids_by_kind.items():
+            schema = schema_branch.get(name=node_kind, duplicate=False)
+            migrations.extend(
+                [
+                    NodeAttributeAddMigration(
+                        uuids=node_ids,
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="human_friendly_id"
+                        ),
+                    ),
+                    NodeAttributeAddMigration(
+                        uuids=node_ids,
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                        ),
+                    ),
+                ]
+            )
+
+        with Progress(console=get_migration_console()) as progress:
+            update_task = progress.add_task(
+                f"Adding HFID and display label to nodes on branch {branch.name}", total=len(migrations)
+            )
+
+            for migration in migrations:
+                try:
+                    execution_result = await migration.execute(db=db, branch=branch)
+                    result.errors.extend(execution_result.errors)
+                    progress.update(update_task, advance=1)
+                except Exception as exc:
+                    result.errors.append(str(exc))
+                    return result
+
+        return result
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()