infrahub-server 1.5.0b1__py3-none-any.whl → 1.5.0b2__py3-none-any.whl

infrahub/core/branch/tasks.py

@@ -12,6 +12,7 @@ from infrahub import lock
 from infrahub.context import InfrahubContext  # noqa: TC001  needed for prefect flow
 from infrahub.core import registry
 from infrahub.core.branch import Branch
+from infrahub.core.branch.enums import BranchStatus
 from infrahub.core.changelog.diff import DiffChangelogCollector, MigrationTracker
 from infrahub.core.constants import MutationAction
 from infrahub.core.diff.coordinator import DiffCoordinator
@@ -21,7 +22,10 @@ from infrahub.core.diff.merger.merger import DiffMerger
 from infrahub.core.diff.model.path import BranchTrackingId, EnrichedDiffRoot, EnrichedDiffRootMetadata
 from infrahub.core.diff.models import RequestDiffUpdate
 from infrahub.core.diff.repository.repository import DiffRepository
+from infrahub.core.graph import GRAPH_VERSION
 from infrahub.core.merge import BranchMerger
+from infrahub.core.migrations.exceptions import MigrationFailureError
+from infrahub.core.migrations.runner import MigrationRunner
 from infrahub.core.migrations.schema.models import SchemaApplyMigrationData
 from infrahub.core.migrations.schema.tasks import schema_apply_migrations
 from infrahub.core.timestamp import Timestamp
@@ -29,7 +33,13 @@ from infrahub.core.validators.determiner import ConstraintValidatorDeterminer
 from infrahub.core.validators.models.validate_migration import SchemaValidateMigrationData
 from infrahub.core.validators.tasks import schema_validate_migrations
 from infrahub.dependencies.registry import get_component_registry
-from infrahub.events.branch_action import BranchCreatedEvent, BranchDeletedEvent, BranchMergedEvent, BranchRebasedEvent
+from infrahub.events.branch_action import (
+    BranchCreatedEvent,
+    BranchDeletedEvent,
+    BranchMergedEvent,
+    BranchMigratedEvent,
+    BranchRebasedEvent,
+)
 from infrahub.events.models import EventMeta, InfrahubEvent
 from infrahub.events.node_action import get_node_event
 from infrahub.exceptions import BranchNotFoundError, ValidationError
@@ -49,8 +59,57 @@ from infrahub.workflows.catalogue import (
 from infrahub.workflows.utils import add_tags
 
 
+@flow(name="branch-migrate", flow_run_name="Apply migrations to branch {branch}")
+async def migrate_branch(branch: str, context: InfrahubContext, send_events: bool = True) -> None:
+    await add_tags(branches=[branch])
+
+    database = await get_database()
+    async with database.start_session() as db:
+        log = get_run_logger()
+
+        obj = await Branch.get_by_name(db=db, name=branch)
+
+        if obj.graph_version == GRAPH_VERSION:
+            log.info(f"Branch '{obj.name}' has graph version {obj.graph_version}, no migrations to apply")
+            return
+
+        migration_runner = MigrationRunner(branch=obj)
+        if not migration_runner.has_migrations():
+            log.info(f"No migrations detected for branch '{obj.name}'")
+            obj.graph_version = GRAPH_VERSION
+            await obj.save(db=db)
+            return
+
+        # Branch status will remain as so if the migration process fails
+        # This will help user to know that a branch is in an invalid state to be used properly and that actions need to be taken
+        if obj.status != BranchStatus.NEED_UPGRADE_REBASE:
+            obj.status = BranchStatus.NEED_UPGRADE_REBASE
+            await obj.save(db=db)
+
+        try:
+            log.info(f"Running migrations for branch '{obj.name}'")
+            await migration_runner.run(db=db)
+        except MigrationFailureError as exc:
+            log.error(f"Failed to run migrations for branch '{obj.name}': {exc.errors}")
+            raise
+
+        if obj.status == BranchStatus.NEED_UPGRADE_REBASE:
+            obj.status = BranchStatus.OPEN
+        obj.graph_version = GRAPH_VERSION
+        await obj.save(db=db)
+
+    if send_events:
+        event_service = await get_event_service()
+        await event_service.send(
+            BranchMigratedEvent(
+                branch_name=obj.name, branch_id=str(obj.uuid), meta=EventMeta(branch=obj, context=context)
+            )
+        )
+
+
 @flow(name="branch-rebase", flow_run_name="Rebase branch {branch}")
-async def rebase_branch(branch: str, context: InfrahubContext) -> None:  # noqa: PLR0915
+async def rebase_branch(branch: str, context: InfrahubContext, send_events: bool = True) -> None:  # noqa: PLR0915
+    workflow = get_workflow()
     database = await get_database()
     async with database.start_session() as db:
         log = get_run_logger()
@@ -69,7 +128,7 @@ async def rebase_branch(branch: str, context: InfrahubContext) -> None:  # noqa:
             diff_repository=diff_repository,
             source_branch=obj,
             diff_locker=DiffLocker(),
-            workflow=get_workflow(),
+            workflow=workflow,
         )
 
         enriched_diff_metadata = await diff_coordinator.update_branch_diff(base_branch=base_branch, diff_branch=obj)
@@ -156,15 +215,17 @@ async def rebase_branch(branch: str, context: InfrahubContext) -> None:  # noqa:
             target_branch_name=registry.default_branch,
         )
         if ipam_node_details:
-            await get_workflow().submit_workflow(
+            await workflow.submit_workflow(
                 workflow=IPAM_RECONCILIATION,
                 context=context,
                 parameters={"branch": obj.name, "ipam_node_details": ipam_node_details},
             )
 
-    await get_workflow().submit_workflow(
-        workflow=DIFF_REFRESH_ALL, context=context, parameters={"branch_name": obj.name}
-    )
+    await migrate_branch(branch=branch, context=context, send_events=send_events)
+    await workflow.submit_workflow(workflow=DIFF_REFRESH_ALL, context=context, parameters={"branch_name": obj.name})
+
+    if not send_events:
+        return
 
     # -------------------------------------------------------------
     # Generate an event to indicate that a branch has been rebased

infrahub/core/constants/__init__.py

@@ -37,6 +37,7 @@ RESERVED_ATTR_REL_NAMES = [
     "rels",
     "save",
     "hfid",
+    "process_pools",
 ]
 
 RESERVED_ATTR_GEN_NAMES = ["type"]
@@ -50,6 +51,7 @@ class EventType(InfrahubStringEnum):
     BRANCH_CREATED = f"{EVENT_NAMESPACE}.branch.created"
     BRANCH_DELETED = f"{EVENT_NAMESPACE}.branch.deleted"
     BRANCH_MERGED = f"{EVENT_NAMESPACE}.branch.merged"
+    BRANCH_MIGRATED = f"{EVENT_NAMESPACE}.branch.migrated"
     BRANCH_REBASED = f"{EVENT_NAMESPACE}.branch.rebased"
 
     SCHEMA_UPDATED = f"{EVENT_NAMESPACE}.schema.updated"
@@ -99,6 +101,7 @@ class GlobalPermissions(InfrahubStringEnum):
     MANAGE_PERMISSIONS = "manage_permissions"
     MANAGE_REPOSITORIES = "manage_repositories"
     OVERRIDE_CONTEXT = "override_context"
+    UPDATE_OBJECT_HFID_DISPLAY_LABEL = "update_object_hfid_display_label"
 
 
 class PermissionAction(InfrahubStringEnum):

infrahub/core/diff/query/artifact.py

@@ -201,4 +201,5 @@ CALL (target_node, definition_node){
             "target_checksum",
             "target_storage_id",
         ]
+        self.order_by = ["source_artifact.uuid", "target_node.uuid", "definition_node.uuid"]
         self.add_to_query(query=query)

infrahub/core/diff/query/field_summary.py

@@ -62,6 +62,7 @@ class EnrichedDiffNodeFieldSummaryQuery(Query):
         }
         """
         self.add_to_query(query=query)
+        self.order_by = ["kind"]
         self.return_labels = ["kind", "attr_names", "rel_names"]
 
     async def get_field_summaries(self) -> list[NodeDiffFieldSummary]:

infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 42
+GRAPH_VERSION = 43

infrahub/core/initialization.py

@@ -50,7 +50,7 @@ async def get_root_node(db: InfrahubDatabase, initialize: bool = False) -> Root:
     roots = await Root.get_list(db=db)
     if len(roots) == 0 and not initialize:
         raise DatabaseError(
-            "The Database hasn't been initialized for Infrahub, please run 'infrahub db init' or 'infrahub server start' to initialize the database."
+            "The Database hasn't been initialized for Infrahub, please 'infrahub server start' to initialize the database."
         )
 
     if len(roots) == 0:
@@ -137,7 +137,8 @@ async def add_indexes(db: InfrahubDatabase) -> None:
     await index_manager.add()
 
 
-async def initialization(db: InfrahubDatabase, add_database_indexes: bool = False) -> None:
+async def initialization(db: InfrahubDatabase, add_database_indexes: bool = False) -> bool:
+    """Run initialization and setup, returns a boolean to indicate if it's the initial setup."""
     if config.SETTINGS.database.db_type == config.DatabaseType.MEMGRAPH:
         session = await db.session()
         await session.run(query="SET DATABASE SETTING 'log.level' TO 'INFO'")
@@ -148,6 +149,7 @@ async def initialization(db: InfrahubDatabase, add_database_indexes: bool = Fals
     # Initialize the database and Load the Root node
     # ---------------------------------------------------
     async with lock.registry.initialization():
+        first_time_initialization = len(await Root.get_list(db=db)) == 0
         log.debug("Checking Root Node")
         await initialize_registry(db=db, initialize=True)
 
@@ -210,6 +212,7 @@ async def initialization(db: InfrahubDatabase, add_database_indexes: bool = Fals
     ip_namespace = await get_default_ipnamespace(db=db)
     if ip_namespace:
         registry.default_ipnamespace = ip_namespace.id
+    return first_time_initialization
 
 
 async def create_root_node(db: InfrahubDatabase) -> Root:

infrahub/core/migrations/__init__.py

@@ -1,5 +1,6 @@
 from .schema.attribute_kind_update import AttributeKindUpdateMigration
 from .schema.attribute_name_update import AttributeNameUpdateMigration
+from .schema.attribute_supports_profile import AttributeSupportsProfileUpdateMigration
 from .schema.node_attribute_add import NodeAttributeAddMigration
 from .schema.node_attribute_remove import NodeAttributeRemoveMigration
 from .schema.node_kind_update import NodeKindUpdateMigration
@@ -19,6 +20,8 @@ MIGRATION_MAP: dict[str, type[SchemaMigration] | None] = {
     "attribute.name.update": AttributeNameUpdateMigration,
     "attribute.branch.update": None,
     "attribute.kind.update": AttributeKindUpdateMigration,
+    "attribute.optional.update": AttributeSupportsProfileUpdateMigration,
+    "attribute.read_only.update": AttributeSupportsProfileUpdateMigration,
     "relationship.branch.update": None,
     "relationship.direction.update": None,
     "relationship.identifier.update": PlaceholderDummyMigration,

infrahub/core/migrations/exceptions.py

@@ -0,0 +1,4 @@
+class MigrationFailureError(Exception):
+    def __init__(self, errors: list[str]) -> None:
+        super().__init__()
+        self.errors = errors

infrahub/core/migrations/graph/__init__.py

@@ -41,16 +41,16 @@ from .m036_drop_attr_value_index import Migration036
 from .m037_index_attr_vals import Migration037
 from .m038_redo_0000_prefix_fix import Migration038
 from .m039_ipam_reconcile import Migration039
-from .m040_profile_attrs_in_db import Migration040
-from .m041_create_hfid_display_label_in_db import Migration041
-from .m042_backfill_hfid_display_label_in_db import Migration042
+from .m040_duplicated_attributes import Migration040
+from .m041_profile_attrs_in_db import Migration041
+from .m042_create_hfid_display_label_in_db import Migration042
+from .m043_backfill_hfid_display_label_in_db import Migration043
 
 if TYPE_CHECKING:
-    from infrahub.core.root import Root
+    from ..shared import MigrationTypes
 
-    from ..shared import ArbitraryMigration, GraphMigration, InternalSchemaMigration
 
-MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigration]] = [
+MIGRATIONS: list[type[MigrationTypes]] = [
     Migration001,
     Migration002,
     Migration003,
@@ -93,25 +93,22 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration040,
     Migration041,
     Migration042,
+    Migration043,
 ]
 
 
-async def get_graph_migrations(
-    root: Root,
-) -> Sequence[GraphMigration | InternalSchemaMigration | ArbitraryMigration]:
+async def get_graph_migrations(current_graph_version: int) -> Sequence[MigrationTypes]:
     applicable_migrations = []
     for migration_class in MIGRATIONS:
         migration = migration_class.init()
-        if root.graph_version > migration.minimum_version:
+        if current_graph_version > migration.minimum_version:
             continue
         applicable_migrations.append(migration)
 
     return applicable_migrations
 
 
-def get_migration_by_number(
-    migration_number: int | str,
-) -> GraphMigration | InternalSchemaMigration | ArbitraryMigration:
+def get_migration_by_number(migration_number: int | str) -> MigrationTypes:
     # Convert to string and pad with zeros if needed
     try:
         num = int(migration_number)

infrahub/core/migrations/graph/load_schema_branch.py

@@ -0,0 +1,21 @@
+from infrahub.core import registry
+from infrahub.core.branch import Branch
+from infrahub.core.schema import SchemaRoot, internal_schema
+from infrahub.core.schema.manager import SchemaManager
+from infrahub.core.schema.schema_branch import SchemaBranch
+from infrahub.database import InfrahubDatabase
+from infrahub.exceptions import InitializationError
+
+
+async def get_or_load_schema_branch(db: InfrahubDatabase, branch: Branch) -> SchemaBranch:
+    try:
+        if registry.schema.has_schema_branch(branch.name):
+            return registry.schema.get_schema_branch(branch.name)
+    except InitializationError:
+        schema_manager = SchemaManager()
+        registry.schema = schema_manager
+        internal_schema_root = SchemaRoot(**internal_schema)
+        registry.schema.register_schema(schema=internal_schema_root)
+    schema_branch = await registry.schema.load_schema_from_db(db=db, branch=branch)
+    registry.schema.set_schema_branch(name=branch.name, schema=schema_branch)
+    return schema_branch

infrahub/core/migrations/graph/m013_convert_git_password_credential.py

@@ -286,7 +286,7 @@ class Migration013AddInternalStatusData(AttributeAddQuery):
         kwargs.pop("branch", None)
 
         super().__init__(
-            node_kind="CoreGenericRepository",
+            node_kinds=["CoreGenericRepository"],
             attribute_name="internal_status",
             attribute_kind="Dropdown",
             branch_support=BranchSupportType.LOCAL.value,

infrahub/core/migrations/graph/m040_duplicated_attributes.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+
+from ..shared import GraphMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+class DeleteDuplicatedAttributesQuery(Query):
+    name: str = "delete_duplicated_attributes"
+    type: QueryType = QueryType.WRITE
+    insert_return: bool = False
+    insert_limit: bool = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+// -------------
+// get all the Nodes linked to multiple Attributes with the same name to drastically reduce the search space
+// -------------
+MATCH (n:Node)-[:HAS_ATTRIBUTE]->(attr:Attribute)
+WITH DISTINCT n, attr
+WITH n, attr.name AS attr_name, count(*) AS num_attrs
+WHERE num_attrs > 1
+// -------------
+// for each Node-attr_name pair, get the possible duplicate Attributes
+// -------------
+MATCH (n)-[:HAS_ATTRIBUTE]->(dup_attr:Attribute {name: attr_name})
+WITH DISTINCT n, dup_attr
+// -------------
+// get the branch(es) for each possible duplicate Attribute
+// -------------
+CALL (n, dup_attr) {
+    MATCH (n)-[r:HAS_ATTRIBUTE {status: "active"}]->(dup_attr)
+    WHERE r.to IS NULL
+    AND NOT exists((n)-[:HAS_ATTRIBUTE {status: "deleted", branch: r.branch}]->(dup_attr))
+    RETURN r.branch AS branch
+}
+// -------------
+// get the latest update time for each duplicate Attribute on each branch
+// -------------
+CALL (dup_attr, branch) {
+    MATCH (dup_attr)-[r {branch: branch}]-()
+    RETURN max(r.from) AS latest_update
+}
+// -------------
+// order the duplicate Attributes by latest update time
+// -------------
+WITH n, dup_attr, branch, latest_update
+ORDER BY n, branch, dup_attr.name, latest_update DESC
+// -------------
+// for any Node-dup_attr_name pairs with multiple duplicate Attributes, keep the Attribute with the latest update
+// on this branch and delete all the other edges on this branch for this Attribute
+// -------------
+WITH n, branch, dup_attr.name AS dup_attr_name, collect(dup_attr) AS dup_attrs_reverse_chronological
+WHERE size(dup_attrs_reverse_chronological) > 1
+WITH branch, tail(dup_attrs_reverse_chronological) AS dup_attrs_to_delete
+UNWIND dup_attrs_to_delete AS dup_attr_to_delete
+MATCH (dup_attr_to_delete)-[r {branch: branch}]-()
+DELETE r
+// -------------
+// delete any orphaned Attributes
+// -------------
+WITH DISTINCT dup_attr_to_delete
+WHERE NOT exists((dup_attr_to_delete)--())
+DELETE dup_attr_to_delete
+        """
+        self.add_to_query(query)
+
+
+class Migration040(GraphMigration):
+    name: str = "040_duplicated_attributes"
+    queries: Sequence[type[Query]] = [DeleteDuplicatedAttributesQuery]
+    minimum_version: int = 39
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()

infrahub/core/migrations/graph/m041_profile_attrs_in_db.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from rich.console import Console
+from rich.progress import Progress
+
+from infrahub.core.branch.models import Branch
+from infrahub.core.initialization import get_root_node
+from infrahub.core.manager import NodeManager
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+from infrahub.core.timestamp import Timestamp
+from infrahub.log import get_logger
+from infrahub.profiles.node_applier import NodeProfilesApplier
+
+from ..shared import MigrationRequiringRebase
+from .load_schema_branch import get_or_load_schema_branch
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class GetUpdatedProfilesForBranchQuery(Query):
+    """
+    Get CoreProfile UUIDs with updated attributes on this branch
+    """
+
+    name = "get_profiles_by_branch"
+    type = QueryType.READ
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (profile:CoreProfile)-[:HAS_ATTRIBUTE]->(attr:Attribute)-[e:HAS_VALUE]->(:AttributeValue)
+WHERE e.branch = $branch
+WITH DISTINCT profile.uuid AS profile_uuid
+        """
+        self.add_to_query(query)
+        self.return_labels = ["profile_uuid"]
+
+    def get_profile_ids(self) -> list[str]:
+        """Get list of updated profile UUIDs"""
+        return [result.get_as_type("profile_uuid", str) for result in self.get_results()]
+
+
+class GetNodesWithProfileUpdatesForBranchQuery(Query):
+    """
+    Get Node UUIDs by which branches they have updated profiles on
+    """
+
+    name = "get_nodes_with_profile_updates_by_branch"
+    type = QueryType.READ
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch"] = self.branch.name
+        query = """
+MATCH (node:Node)-[e:IS_RELATED]->(:Relationship {name: "node__profile"})
+WHERE NOT node:CoreProfile
+AND e.branch = $branch
+WITH DISTINCT node.uuid AS node_uuid
+        """
+        self.add_to_query(query)
+        self.return_labels = ["node_uuid"]
+
+    def get_node_ids(self) -> list[str]:
+        """Get list of updated node UUIDs"""
+        return [result.get_as_type("node_uuid", str) for result in self.get_results()]
+
+
+class Migration041(MigrationRequiringRebase):
+    """
+    Save profile attribute values on each node using the profile in the database
+    For any profile that has updates on a given branch (including default branch)
+    - run NodeProfilesApplier.apply_profiles on each node related to the profile on that branch
+    For any node that has an updated relationship to a profile on a given branch
+    - run NodeProfilesApplier.apply_profiles on the node on that branch
+    """
+
+    name: str = "041_profile_attrs_in_db"
+    minimum_version: int = 40
+
+    def _get_profile_applier(self, db: InfrahubDatabase, branch: Branch) -> NodeProfilesApplier:
+        return NodeProfilesApplier(db=db, branch=branch)
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        root_node = await get_root_node(db=db, initialize=False)
+        default_branch_name = root_node.default_branch
+        default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
+        return await self._do_execute_for_branch(db=db, branch=default_branch)
+
+    async def execute_against_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        return await self._do_execute_for_branch(db=db, branch=branch)
+
+    async def _do_execute_for_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
+        console = Console()
+        result = MigrationResult()
+        await get_or_load_schema_branch(db=db, branch=branch)
+
+        console.print(f"Gathering profiles for branch {branch.name}...", end="")
+        get_updated_profiles_for_branch_query = await GetUpdatedProfilesForBranchQuery.init(db=db, branch=branch)
+        await get_updated_profiles_for_branch_query.execute(db=db)
+        profile_ids = get_updated_profiles_for_branch_query.get_profile_ids()
+
+        profiles_map = await NodeManager.get_many(db=db, branch=branch, ids=list(profile_ids))
+        console.print("done")
+
+        node_ids_to_update: set[str] = set()
+        with Progress() as progress:
+            gather_nodes_task = progress.add_task(
+                f"Gathering affected objects for each profile on branch {branch.name}...", total=len(profiles_map)
+            )
+
+            for profile in profiles_map.values():
+                node_relationship_manager = profile.get_relationship("related_nodes")
+                node_peers = await node_relationship_manager.get_db_peers(db=db)
+                node_ids_to_update.update(str(peer.peer_id) for peer in node_peers)
+                progress.update(gather_nodes_task, advance=1)
+
+        console.print("Identifying nodes with profile updates by branch...", end="")
+        get_nodes_with_profile_updates_by_branch_query = await GetNodesWithProfileUpdatesForBranchQuery.init(
+            db=db, branch=branch
+        )
+        await get_nodes_with_profile_updates_by_branch_query.execute(db=db)
+        node_ids_to_update.update(get_nodes_with_profile_updates_by_branch_query.get_node_ids())
+        console.print("done")
+
+        right_now = Timestamp()
+        with Progress() as progress:
+            apply_task = progress.add_task("Applying profiles to nodes...", total=len(node_ids_to_update))
+            applier = self._get_profile_applier(db=db, branch=branch)
+            for node_id in node_ids_to_update:
+                node = await NodeManager.get_one(db=db, branch=branch, id=node_id, at=right_now)
+                if node:
+                    updated_field_names = await applier.apply_profiles(node=node)
+                    if updated_field_names:
+                        await node.save(db=db, fields=updated_field_names, at=right_now)
+                progress.update(apply_task, advance=1)
+
+        return result