infrahub-server 1.5.0b0__py3-none-any.whl → 1.5.0b2__py3-none-any.whl

infrahub/core/migrations/graph/__init__.py

@@ -41,14 +41,16 @@ from .m036_drop_attr_value_index import Migration036
 from .m037_index_attr_vals import Migration037
 from .m038_redo_0000_prefix_fix import Migration038
 from .m039_ipam_reconcile import Migration039
-from .m040_profile_attrs_in_db import Migration040
+from .m040_duplicated_attributes import Migration040
+from .m041_profile_attrs_in_db import Migration041
+from .m042_create_hfid_display_label_in_db import Migration042
+from .m043_backfill_hfid_display_label_in_db import Migration043
 
 if TYPE_CHECKING:
-    from infrahub.core.root import Root
+    from ..shared import MigrationTypes
 
-    from ..shared import ArbitraryMigration, GraphMigration, InternalSchemaMigration
 
-MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigration]] = [
+MIGRATIONS: list[type[MigrationTypes]] = [
     Migration001,
     Migration002,
     Migration003,
@@ -89,25 +91,24 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration038,
     Migration039,
     Migration040,
+    Migration041,
+    Migration042,
+    Migration043,
 ]
 
 
-async def get_graph_migrations(
-    root: Root,
-) -> Sequence[GraphMigration | InternalSchemaMigration | ArbitraryMigration]:
+async def get_graph_migrations(current_graph_version: int) -> Sequence[MigrationTypes]:
     applicable_migrations = []
     for migration_class in MIGRATIONS:
         migration = migration_class.init()
-        if root.graph_version > migration.minimum_version:
+        if current_graph_version > migration.minimum_version:
             continue
         applicable_migrations.append(migration)
 
     return applicable_migrations
 
 
-def get_migration_by_number(
-    migration_number: int | str,
-) -> GraphMigration | InternalSchemaMigration | ArbitraryMigration:
+def get_migration_by_number(migration_number: int | str) -> MigrationTypes:
     # Convert to string and pad with zeros if needed
     try:
         num = int(migration_number)

infrahub/core/migrations/graph/load_schema_branch.py

@@ -0,0 +1,21 @@
+from infrahub.core import registry
+from infrahub.core.branch import Branch
+from infrahub.core.schema import SchemaRoot, internal_schema
+from infrahub.core.schema.manager import SchemaManager
+from infrahub.core.schema.schema_branch import SchemaBranch
+from infrahub.database import InfrahubDatabase
+from infrahub.exceptions import InitializationError
+
+
+async def get_or_load_schema_branch(db: InfrahubDatabase, branch: Branch) -> SchemaBranch:
+    try:
+        if registry.schema.has_schema_branch(branch.name):
+            return registry.schema.get_schema_branch(branch.name)
+    except InitializationError:
+        schema_manager = SchemaManager()
+        registry.schema = schema_manager
+        internal_schema_root = SchemaRoot(**internal_schema)
+        registry.schema.register_schema(schema=internal_schema_root)
+    schema_branch = await registry.schema.load_schema_from_db(db=db, branch=branch)
+    registry.schema.set_schema_branch(name=branch.name, schema=schema_branch)
+    return schema_branch

infrahub/core/migrations/graph/m013_convert_git_password_credential.py

@@ -286,7 +286,7 @@ class Migration013AddInternalStatusData(AttributeAddQuery):
         kwargs.pop("branch", None)
 
         super().__init__(
-            node_kind="CoreGenericRepository",
+            node_kinds=["CoreGenericRepository"],
             attribute_name="internal_status",
             attribute_kind="Dropdown",
             branch_support=BranchSupportType.LOCAL.value,

infrahub/core/migrations/graph/m040_duplicated_attributes.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+
+from ..shared import GraphMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+class DeleteDuplicatedAttributesQuery(Query):
+    name: str = "delete_duplicated_attributes"
+    type: QueryType = QueryType.WRITE
+    insert_return: bool = False
+    insert_limit: bool = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+// -------------
+// get all the Nodes linked to multiple Attributes with the same name to drastically reduce the search space
+// -------------
+MATCH (n:Node)-[:HAS_ATTRIBUTE]->(attr:Attribute)
+WITH DISTINCT n, attr
+WITH n, attr.name AS attr_name, count(*) AS num_attrs
+WHERE num_attrs > 1
+// -------------
+// for each Node-attr_name pair, get the possible duplicate Attributes
+// -------------
+MATCH (n)-[:HAS_ATTRIBUTE]->(dup_attr:Attribute {name: attr_name})
+WITH DISTINCT n, dup_attr
+// -------------
+// get the branch(es) for each possible duplicate Attribute
+// -------------
+CALL (n, dup_attr) {
+    MATCH (n)-[r:HAS_ATTRIBUTE {status: "active"}]->(dup_attr)
+    WHERE r.to IS NULL
+    AND NOT exists((n)-[:HAS_ATTRIBUTE {status: "deleted", branch: r.branch}]->(dup_attr))
+    RETURN r.branch AS branch
+}
+// -------------
+// get the latest update time for each duplicate Attribute on each branch
+// -------------
+CALL (dup_attr, branch) {
+    MATCH (dup_attr)-[r {branch: branch}]-()
+    RETURN max(r.from) AS latest_update
+}
+// -------------
+// order the duplicate Attributes by latest update time
+// -------------
+WITH n, dup_attr, branch, latest_update
+ORDER BY n, branch, dup_attr.name, latest_update DESC
+// -------------
+// for any Node-dup_attr_name pairs with multiple duplicate Attributes, keep the Attribute with the latest update
+// on this branch and delete all the other edges on this branch for this Attribute
+// -------------
+WITH n, branch, dup_attr.name AS dup_attr_name, collect(dup_attr) AS dup_attrs_reverse_chronological
+WHERE size(dup_attrs_reverse_chronological) > 1
+WITH branch, tail(dup_attrs_reverse_chronological) AS dup_attrs_to_delete
+UNWIND dup_attrs_to_delete AS dup_attr_to_delete
+MATCH (dup_attr_to_delete)-[r {branch: branch}]-()
+DELETE r
+// -------------
+// delete any orphaned Attributes
+// -------------
+WITH DISTINCT dup_attr_to_delete
+WHERE NOT exists((dup_attr_to_delete)--())
+DELETE dup_attr_to_delete
+        """
+        self.add_to_query(query)
+
+
+class Migration040(GraphMigration):
+    name: str = "040_duplicated_attributes"
+    queries: Sequence[type[Query]] = [DeleteDuplicatedAttributesQuery]
+    minimum_version: int = 39
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()

infrahub/core/migrations/graph/m041_profile_attrs_in_db.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from rich.console import Console
+from rich.progress import Progress
+
+from infrahub.core.branch.models import Branch
+from infrahub.core.initialization import get_root_node
+from infrahub.core.manager import NodeManager
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+from infrahub.core.timestamp import Timestamp
+from infrahub.log import get_logger
+from infrahub.profiles.node_applier import NodeProfilesApplier
+
+from ..shared import MigrationRequiringRebase
+from .load_schema_branch import get_or_load_schema_branch
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class GetUpdatedProfilesForBranchQuery(Query):
+    """
+    Get CoreProfile UUIDs with updated attributes on this branch
+    """
+
+    name = "get_profiles_by_branch"
+    type = QueryType.READ
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (profile:CoreProfile)-[:HAS_ATTRIBUTE]->(attr:Attribute)-[e:HAS_VALUE]->(:AttributeValue)
+WHERE e.branch = $branch
+WITH DISTINCT profile.uuid AS profile_uuid
+        """
+        self.add_to_query(query)
+        self.return_labels = ["profile_uuid"]
+
+    def get_profile_ids(self) -> list[str]:
+        """Get list of updated profile UUIDs"""
+        return [result.get_as_type("profile_uuid", str) for result in self.get_results()]
+
+
+class GetNodesWithProfileUpdatesForBranchQuery(Query):
+    """
+    Get Node UUIDs by which branches they have updated profiles on
+    """
+
+    name = "get_nodes_with_profile_updates_by_branch"
+    type = QueryType.READ
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (node:Node)-[e:IS_RELATED]->(:Relationship {name: "node__profile"})
+WHERE NOT node:CoreProfile
+AND e.branch = $branch
+WITH DISTINCT node.uuid AS node_uuid
+        """
+        self.add_to_query(query)
+        self.return_labels = ["node_uuid"]
+
+    def get_node_ids(self) -> list[str]:
+        """Get list of updated node UUIDs"""
+        return [result.get_as_type("node_uuid", str) for result in self.get_results()]
+
+
+class Migration041(MigrationRequiringRebase):
+    """
+    Save profile attribute values on each node using the profile in the database
+    For any profile that has updates on a given branch (including default branch)
+    - run NodeProfilesApplier.apply_profiles on each node related to the profile on that branch
+    For any node that has an updated relationship to a profile on a given branch
+    - run NodeProfilesApplier.apply_profiles on the node on that branch
+    """
+
+    name: str = "041_profile_attrs_in_db"
+    minimum_version: int = 40
+
+    def _get_profile_applier(self, db: InfrahubDatabase, branch: Branch) -> NodeProfilesApplier:
+        return NodeProfilesApplier(db=db, branch=branch)
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        root_node = await get_root_node(db=db, initialize=False)
+        default_branch_name = root_node.default_branch
+        default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
+        return await self._do_execute_for_branch(db=db, branch=default_branch)
+
+    async def execute_against_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        return await self._do_execute_for_branch(db=db, branch=branch)
+
+    async def _do_execute_for_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        console = Console()
+        result = MigrationResult()
+        await get_or_load_schema_branch(db=db, branch=branch)
+
+        console.print(f"Gathering profiles for branch {branch.name}...", end="")
+        get_updated_profiles_for_branch_query = await GetUpdatedProfilesForBranchQuery.init(db=db, branch=branch)
+        await get_updated_profiles_for_branch_query.execute(db=db)
+        profile_ids = get_updated_profiles_for_branch_query.get_profile_ids()
+
+        profiles_map = await NodeManager.get_many(db=db, branch=branch, ids=list(profile_ids))
+        console.print("done")
+
+        node_ids_to_update: set[str] = set()
+        with Progress() as progress:
+            gather_nodes_task = progress.add_task(
+                f"Gathering affected objects for each profile on branch {branch.name}...", total=len(profiles_map)
+            )
+
+            for profile in profiles_map.values():
+                node_relationship_manager = profile.get_relationship("related_nodes")
+                node_peers = await node_relationship_manager.get_db_peers(db=db)
+                node_ids_to_update.update(str(peer.peer_id) for peer in node_peers)
+                progress.update(gather_nodes_task, advance=1)
+
+        console.print("Identifying nodes with profile updates by branch...", end="")
+        get_nodes_with_profile_updates_by_branch_query = await GetNodesWithProfileUpdatesForBranchQuery.init(
+            db=db, branch=branch
+        )
+        await get_nodes_with_profile_updates_by_branch_query.execute(db=db)
+        node_ids_to_update.update(get_nodes_with_profile_updates_by_branch_query.get_node_ids())
+        console.print("done")
+
+        right_now = Timestamp()
+        with Progress() as progress:
+            apply_task = progress.add_task("Applying profiles to nodes...", total=len(node_ids_to_update))
+            applier = self._get_profile_applier(db=db, branch=branch)
+            for node_id in node_ids_to_update:
+                node = await NodeManager.get_one(db=db, branch=branch, id=node_id, at=right_now)
+                if node:
+                    updated_field_names = await applier.apply_profiles(node=node)
+                    if updated_field_names:
+                        await node.save(db=db, fields=updated_field_names, at=right_now)
+                progress.update(apply_task, advance=1)
+
+        return result

infrahub/core/migrations/graph/m042_create_hfid_display_label_in_db.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from rich.progress import Progress
+
+from infrahub.core import registry
+from infrahub.core.branch import Branch
+from infrahub.core.constants import SchemaPathType
+from infrahub.core.initialization import get_root_node
+from infrahub.core.migrations.schema.node_attribute_add import NodeAttributeAddMigration
+from infrahub.core.migrations.shared import MigrationRequiringRebase, MigrationResult
+from infrahub.core.path import SchemaPath
+from infrahub.core.query import Query, QueryType
+
+from .load_schema_branch import get_or_load_schema_branch
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+class GetAddedNodesByKindForBranchQuery(Query):
+    name = "get_added_nodes_by_kind_for_branch_query"
+    type = QueryType.READ
+    insert_return = True
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (n:Node)-[e:IS_PART_OF {branch: $branch, status: "active"}]->(:Root)
+WHERE e.to IS NULL
+AND NOT exists((n)-[:IS_PART_OF {branch: $branch, status: "deleted"}]->(:Root))
+WITH n.kind AS kind, collect(n.uuid) AS node_ids
+        """
+        self.return_labels = ["kind", "node_ids"]
+        self.add_to_query(query)
+
+    def get_node_ids_by_kind(self) -> dict[str, list[str]]:
+        node_ids_by_kind: dict[str, list[str]] = {}
+        for result in self.get_results():
+            kind = result.get_as_type(label="kind", return_type=str)
+            node_ids: list[str] = result.get_as_type(label="node_ids", return_type=list)
+            node_ids_by_kind[kind] = node_ids
+        return node_ids_by_kind
+
+
+class Migration042(MigrationRequiringRebase):
+    name: str = "042_create_hfid_display_label_in_db"
+    minimum_version: int = 41
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        result = MigrationResult()
+
+        root_node = await get_root_node(db=db, initialize=False)
+        default_branch_name = root_node.default_branch
+        default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
+        main_schema_branch = await get_or_load_schema_branch(db=db, branch=default_branch)
+        schema_node = main_schema_branch.get_node(name="SchemaNode")
+        schema_generic = main_schema_branch.get_node(name="SchemaGeneric")
+
+        migrations = [
+            # HFID is not needed, it was introduced at graph v8
+            NodeAttributeAddMigration(
+                new_node_schema=schema_node,
+                previous_node_schema=schema_node,
+                schema_path=SchemaPath(
+                    schema_kind="SchemaNode", path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                ),
+            ),
+            NodeAttributeAddMigration(
+                new_node_schema=schema_generic,
+                previous_node_schema=schema_generic,
+                schema_path=SchemaPath(
+                    schema_kind="SchemaGeneric", path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                ),
+            ),
+        ]
+
+        for node_schema_kind in main_schema_branch.node_names:
+            schema = main_schema_branch.get(name=node_schema_kind, duplicate=False)
+            migrations.extend(
+                [
+                    NodeAttributeAddMigration(
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="human_friendly_id"
+                        ),
+                    ),
+                    NodeAttributeAddMigration(
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                        ),
+                    ),
+                ]
+            )
+
+        with Progress() as progress:
+            update_task = progress.add_task("Adding HFID and display label to nodes", total=len(migrations))
+
+            for migration in migrations:
+                try:
+                    execution_result = await migration.execute(db=db, branch=default_branch)
+                    result.errors.extend(execution_result.errors)
+                    progress.update(update_task, advance=1)
+                except Exception as exc:
+                    result.errors.append(str(exc))
+                    return result
+
+        return result
+
+    async def execute_against_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        result = MigrationResult()
+
+        schema_branch = await registry.schema.load_schema_from_db(db=db, branch=branch)
+
+        migrations = []
+        get_added_nodes_by_kind_for_branch_query = await GetAddedNodesByKindForBranchQuery.init(db=db, branch=branch)
+        await get_added_nodes_by_kind_for_branch_query.execute(db=db)
+        node_ids_by_kind = get_added_nodes_by_kind_for_branch_query.get_node_ids_by_kind()
+
+        for node_kind, node_ids in node_ids_by_kind.items():
+            schema = schema_branch.get(name=node_kind, duplicate=False)
+            migrations.extend(
+                [
+                    NodeAttributeAddMigration(
+                        uuids=node_ids,
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="human_friendly_id"
+                        ),
+                    ),
+                    NodeAttributeAddMigration(
+                        uuids=node_ids,
+                        new_node_schema=schema,
+                        previous_node_schema=schema,
+                        schema_path=SchemaPath(
+                            schema_kind=schema.kind, path_type=SchemaPathType.ATTRIBUTE, field_name="display_label"
+                        ),
+                    ),
+                ]
+            )
+
+        with Progress() as progress:
+            update_task = progress.add_task(
+                f"Adding HFID and display label to nodes on branch {branch.name}", total=len(migrations)
+            )
+
+            for migration in migrations:
+                try:
+                    execution_result = await migration.execute(db=db, branch=branch)
+                    result.errors.extend(execution_result.errors)
+                    progress.update(update_task, advance=1)
+                except Exception as exc:
+                    result.errors.append(str(exc))
+                    return result
+
+        return result
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()