infrahub-server 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

infrahub/core/diff/query_parser.py

@@ -486,6 +486,7 @@ class DiffQueryParser:
         self._previous_node_field_specifiers = previous_node_field_specifiers or NodeFieldSpecifierMap()
         self._new_node_field_specifiers: NodeFieldSpecifierMap | None = None
         self._current_node_field_specifiers: NodeFieldSpecifierMap | None = None
+        self._diff_node_field_specifiers: NodeFieldSpecifierMap = NodeFieldSpecifierMap()
 
     def get_branches(self) -> set[str]:
         return set(self._final_diff_root_by_branch.keys())
@@ -497,33 +498,17 @@ class DiffQueryParser:
             return self._final_diff_root_by_branch[branch]
         return DiffRoot(from_time=self.from_time, to_time=self.to_time, uuid=str(uuid4()), branch=branch, nodes=[])
 
-    def get_diff_node_field_specifiers(self) -> NodeFieldSpecifierMap:
-        node_field_specifiers_map = NodeFieldSpecifierMap()
-        if self.diff_branch_name not in self._diff_root_by_branch:
-            return node_field_specifiers_map
-        diff_root = self._diff_root_by_branch[self.diff_branch_name]
-        for node in diff_root.nodes_by_identifier.values():
-            for attribute_name in node.attributes_by_name:
-                node_field_specifiers_map.add_entry(node_uuid=node.uuid, kind=node.kind, field_name=attribute_name)
-            for relationship_diff in node.relationships_by_identifier.values():
-                node_field_specifiers_map.add_entry(
-                    node_uuid=node.uuid, kind=node.kind, field_name=relationship_diff.identifier
-                )
-        return node_field_specifiers_map
-
     def get_new_node_field_specifiers(self) -> NodeFieldSpecifierMap:
-        if self._new_node_field_specifiers is not None:
-            return self._new_node_field_specifiers
-        branch_node_specifiers = self.get_diff_node_field_specifiers()
-        self._new_node_field_specifiers = branch_node_specifiers - self._previous_node_field_specifiers
-        return self._new_node_field_specifiers
-
-    def get_current_node_field_specifiers(self) -> NodeFieldSpecifierMap:
-        if self._current_node_field_specifiers is not None:
-            return self._current_node_field_specifiers
-        new_node_field_specifiers = self.get_new_node_field_specifiers()
-        self._current_node_field_specifiers = self._previous_node_field_specifiers - new_node_field_specifiers
-        return self._current_node_field_specifiers
+        return self._diff_node_field_specifiers - self._previous_node_field_specifiers
+
+    def is_new_node_field_specifier(self, node_uuid: str, kind: str, field_name: str) -> bool:
+        if not self._diff_node_field_specifiers.has_entry(node_uuid=node_uuid, kind=kind, field_name=field_name):
+            return False
+        if self._previous_node_field_specifiers and self._previous_node_field_specifiers.has_entry(
+            node_uuid=node_uuid, kind=kind, field_name=field_name
+        ):
+            return False
+        return True
 
     def read_result(self, query_result: QueryResult) -> None:
         try:
@@ -533,8 +518,6 @@ class DiffQueryParser:
             return
         database_path = DatabasePath.from_cypher_path(cypher_path=path)
         self._parse_path(database_path=database_path)
-        self._current_node_field_specifiers = None
-        self._new_node_field_specifiers = None
 
     def parse(self, include_unchanged: bool = False) -> None:
         self._new_node_field_specifiers = None
@@ -617,11 +600,15 @@ class DiffQueryParser:
         branch_name = database_path.deepest_branch
         from_time = self.from_time
         if branch_name == self.base_branch_name:
-            new_node_field_specifiers = self.get_new_node_field_specifiers()
-            if new_node_field_specifiers.has_entry(
+            if self.is_new_node_field_specifier(
                 node_uuid=diff_node.uuid, kind=diff_node.kind, field_name=attribute_name
             ):
                 from_time = self.diff_branched_from_time
+        else:
+            # Add to diff node field specifiers if this is the diff branch
+            self._diff_node_field_specifiers.add_entry(
+                node_uuid=diff_node.uuid, kind=diff_node.kind, field_name=attribute_name
+            )
         if attribute_name not in diff_node.attributes_by_name:
             diff_node.attributes_by_name[attribute_name] = DiffAttributeIntermediate(
                 uuid=database_path.attribute_id,
@@ -663,11 +650,15 @@ class DiffQueryParser:
             branch_name = database_path.deepest_branch
             from_time = self.from_time
             if branch_name == self.base_branch_name:
-                new_node_field_specifiers = self.get_new_node_field_specifiers()
-                if new_node_field_specifiers.has_entry(
+                if self.is_new_node_field_specifier(
                     node_uuid=diff_node.uuid, kind=diff_node.kind, field_name=relationship_schema.get_identifier()
                 ):
                     from_time = self.diff_branched_from_time
+            else:
+                # Add to diff node field specifiers if this is the diff branch
+                self._diff_node_field_specifiers.add_entry(
+                    node_uuid=diff_node.uuid, kind=diff_node.kind, field_name=relationship_schema.get_identifier()
+                )
             diff_relationship = DiffRelationshipIntermediate(
                 name=relationship_schema.name,
                 cardinality=relationship_schema.cardinality,

infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 31
+GRAPH_VERSION = 33

infrahub/core/initialization.py

@@ -6,6 +6,7 @@ from infrahub import config, lock
 from infrahub.constants.database import DatabaseType
 from infrahub.core import registry
 from infrahub.core.branch import Branch
+from infrahub.core.branch.enums import BranchStatus
 from infrahub.core.constants import (
     DEFAULT_IP_NAMESPACE,
     GLOBAL_BRANCH_NAME,
@@ -224,7 +225,7 @@ async def create_root_node(db: InfrahubDatabase) -> Root:
 async def create_default_branch(db: InfrahubDatabase) -> Branch:
     branch = Branch(
         name=registry.default_branch,
-        status="OPEN",
+        status=BranchStatus.OPEN,
         description="Default Branch",
         hierarchy_level=1,
         is_default=True,
@@ -241,7 +242,7 @@ async def create_default_branch(db: InfrahubDatabase) -> Branch:
 async def create_global_branch(db: InfrahubDatabase) -> Branch:
     branch = Branch(
         name=GLOBAL_BRANCH_NAME,
-        status="OPEN",
+        status=BranchStatus.OPEN,
         description="Global Branch",
         hierarchy_level=1,
         is_global=True,
@@ -264,7 +265,7 @@ async def create_branch(
     description = description or f"Branch {branch_name}"
     branch = Branch(
         name=branch_name,
-        status="OPEN",
+        status=BranchStatus.OPEN,
         hierarchy_level=2,
         description=description,
         is_default=False,

infrahub/core/merge.py

@@ -9,7 +9,7 @@ from infrahub.core.models import SchemaUpdateValidationResult
 from infrahub.core.protocols import CoreRepository
 from infrahub.core.registry import registry
 from infrahub.core.timestamp import Timestamp
-from infrahub.exceptions import ValidationError
+from infrahub.exceptions import MergeFailedError, ValidationError
 from infrahub.log import get_logger
 
 from ..git.models import GitRepositoryMerge
@@ -18,6 +18,7 @@ from ..workflows.catalogue import GIT_REPOSITORIES_MERGE
 if TYPE_CHECKING:
     from infrahub.core.branch import Branch
     from infrahub.core.diff.coordinator import DiffCoordinator
+    from infrahub.core.diff.diff_locker import DiffLocker
     from infrahub.core.diff.merger.merger import DiffMerger
     from infrahub.core.diff.model.path import EnrichedDiffRoot
     from infrahub.core.diff.repository.repository import DiffRepository
@@ -39,6 +40,7 @@ class BranchMerger:
         diff_coordinator: DiffCoordinator,
         diff_merger: DiffMerger,
         diff_repository: DiffRepository,
+        diff_locker: DiffLocker,
         destination_branch: Branch | None = None,
         service: InfrahubServices | None = None,
     ):
@@ -48,6 +50,7 @@ class BranchMerger:
         self.diff_coordinator = diff_coordinator
         self.diff_merger = diff_merger
         self.diff_repository = diff_repository
+        self.diff_locker = diff_locker
         self.migrations: list[SchemaUpdateMigrationInfo] = []
         self._merge_at = Timestamp()
 
@@ -185,22 +188,34 @@ class BranchMerger:
         )
         log.info("Diff updated for merge")
 
-        errors: list[str] = []
-        async for conflict_path, conflict in self.diff_repository.get_all_conflicts_for_diff(
-            diff_branch_name=self.source_branch.name, tracking_id=BranchTrackingId(name=self.source_branch.name)
+        log.info("Acquiring lock for merge")
+        async with self.diff_locker.acquire_lock(
+            target_branch_name=self.destination_branch.name,
+            source_branch_name=self.source_branch.name,
+            is_incremental=False,
         ):
-            if conflict.selected_branch is None or conflict.resolvable is False:
-                errors.append(conflict_path)
-
-        if errors:
-            raise ValidationError(
-                f"Unable to merge the branch '{self.source_branch.name}', conflict resolution missing: {', '.join(errors)}"
-            )
-
-        # TODO need to find a way to properly communicate back to the user any issue that could come up during the merge
-        # From the Graph or From the repositories
-        self._merge_at = Timestamp(at)
-        branch_diff = await self.diff_merger.merge_graph(at=self._merge_at)
+            log.info("Lock acquired for merge")
+            try:
+                errors: list[str] = []
+                async for conflict_path, conflict in self.diff_repository.get_all_conflicts_for_diff(
+                    diff_branch_name=self.source_branch.name, tracking_id=BranchTrackingId(name=self.source_branch.name)
+                ):
+                    if conflict.selected_branch is None or conflict.resolvable is False:
+                        errors.append(conflict_path)
+
+                if errors:
+                    raise ValidationError(
+                        f"Unable to merge the branch '{self.source_branch.name}', conflict resolution missing: {', '.join(errors)}"
+                    )
+
+                # TODO need to find a way to properly communicate back to the user any issue that could come up during the merge
+                # From the Graph or From the repositories
+                self._merge_at = Timestamp(at)
+                branch_diff = await self.diff_merger.merge_graph(at=self._merge_at)
+            except Exception as exc:
+                log.exception("Merge failed, beginning rollback")
+                await self.rollback()
+                raise MergeFailedError(branch_name=self.source_branch.name) from exc
         await self.merge_repositories()
         return branch_diff
 

infrahub/core/migrations/graph/__init__.py

@@ -33,6 +33,8 @@ from .m028_delete_diffs import Migration028
 from .m029_duplicates_cleanup import Migration029
 from .m030_illegal_edges import Migration030
 from .m031_check_number_attributes import Migration031
+from .m032_cleanup_orphaned_branch_relationships import Migration032
+from .m033_deduplicate_relationship_vertices import Migration033
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -71,6 +73,8 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration029,
     Migration030,
     Migration031,
+    Migration032,
+    Migration033,
 ]
 
 
@@ -85,3 +89,23 @@ async def get_graph_migrations(
         applicable_migrations.append(migration)
 
     return applicable_migrations
+
+
+def get_migration_by_number(
+    migration_number: int | str,
+) -> GraphMigration | InternalSchemaMigration | ArbitraryMigration:
+    # Convert to string and pad with zeros if needed
+    try:
+        num = int(migration_number)
+        migration_str = f"{num:03d}"
+    except (ValueError, TypeError) as exc:
+        raise ValueError(f"Invalid migration number: {migration_number}") from exc
+
+    migration_name = f"Migration{migration_str}"
+
+    # Find the migration in the MIGRATIONS list
+    for migration_class in MIGRATIONS:
+        if migration_class.__name__ == migration_name:
+            return migration_class.init()
+
+    raise ValueError(f"Migration {migration_number} not found")

infrahub/core/migrations/graph/m012_convert_account_generic.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Any, Sequence
 
 from infrahub.core.branch import Branch
+from infrahub.core.branch.enums import BranchStatus
 from infrahub.core.constants import GLOBAL_BRANCH_NAME, BranchSupportType, InfrahubKind
 from infrahub.core.migrations.shared import MigrationResult
 from infrahub.core.query import Query, QueryType
@@ -20,7 +21,7 @@ if TYPE_CHECKING:
 
 global_branch = Branch(
     name=GLOBAL_BRANCH_NAME,
-    status="OPEN",
+    status=BranchStatus.OPEN,
     description="Global Branch",
     hierarchy_level=1,
     is_global=True,
@@ -29,7 +30,7 @@ global_branch = Branch(
 
 default_branch = Branch(
     name="main",
-    status="OPEN",
+    status=BranchStatus.OPEN,
     description="Default Branch",
     hierarchy_level=1,
     is_global=False,
@@ -105,7 +106,7 @@ class Migration012AddLabelData(NodeDuplicateQuery):
 
         branch = Branch(
             name=GLOBAL_BRANCH_NAME,
-            status="OPEN",
+            status=BranchStatus.OPEN,
             description="Global Branch",
             hierarchy_level=1,
             is_global=True,

infrahub/core/migrations/graph/m013_convert_git_password_credential.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Any, Sequence
 
 from infrahub.core.branch import Branch
+from infrahub.core.branch.enums import BranchStatus
 from infrahub.core.constants import (
     GLOBAL_BRANCH_NAME,
     BranchSupportType,
@@ -23,7 +24,7 @@ if TYPE_CHECKING:
 
 default_branch = Branch(
     name="main",
-    status="OPEN",
+    status=BranchStatus.OPEN,
     description="Default Branch",
     hierarchy_level=1,
     is_global=False,
@@ -42,7 +43,7 @@ class Migration013ConvertCoreRepositoryWithCred(Query):
 
         global_branch = Branch(
             name=GLOBAL_BRANCH_NAME,
-            status="OPEN",
+            status=BranchStatus.OPEN,
             description="Global Branch",
             hierarchy_level=1,
             is_global=True,
@@ -176,7 +177,7 @@ class Migration013ConvertCoreRepositoryWithoutCred(Query):
 
         global_branch = Branch(
             name=GLOBAL_BRANCH_NAME,
-            status="OPEN",
+            status=BranchStatus.OPEN,
             description="Global Branch",
             hierarchy_level=1,
             is_global=True,

infrahub/core/migrations/graph/m032_cleanup_orphaned_branch_relationships.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query, QueryType
+from infrahub.core.query.branch import DeleteBranchRelationshipsQuery
+from infrahub.log import get_logger
+
+from ..shared import ArbitraryMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class DeletedBranchCleanupQuery(Query):
+    """
+    Find all unique edge branch names for which there is no Branch object
+    """
+
+    name = "deleted_branch_cleanup"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+MATCH (b:Branch)
+WITH collect(DISTINCT b.name) AS branch_names
+MATCH ()-[e]->()
+WHERE e.branch IS NOT NULL
+AND NOT e.branch IN branch_names
+RETURN DISTINCT (e.branch) AS branch_name
+        """
+        self.add_to_query(query)
+        self.return_labels = ["branch_name"]
+
+
+class DeleteOrphanRelationshipsQuery(Query):
+    """
+    Find all Relationship vertices that link to fewer than 2 Node vertices and delete them
+    """
+
+    name = "delete_orphan_relationships"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+MATCH (r:Relationship)-[:IS_RELATED]-(n:Node)
+WITH DISTINCT r, n
+WITH r, count(*) AS node_count
+WHERE node_count < 2
+DETACH DELETE r
+        """
+        self.add_to_query(query)
+
+
+class Migration032(ArbitraryMigration):
+    """
+    Delete edges for branches that were not completely deleted
+    """
+
+    name: str = "032_cleanup_deleted_branches"
+    minimum_version: int = 31
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        return MigrationResult()
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        migration_result = MigrationResult()
+
+        try:
+            log.info("Get partially deleted branch names...")
+            orphaned_branches_query = await DeletedBranchCleanupQuery.init(db=db)
+            await orphaned_branches_query.execute(db=db)
+
+            orphaned_branch_names = []
+            for result in orphaned_branches_query.get_results():
+                branch_name = result.get_as_type("branch_name", str)
+                orphaned_branch_names.append(branch_name)
+
+            if not orphaned_branch_names:
+                log.info("No partially deleted branches found. All done.")
+                return migration_result
+
+            log.info(f"Found {len(orphaned_branch_names)} orphaned branch names: {orphaned_branch_names}")
+
+            for branch_name in orphaned_branch_names:
+                log.info(f"Cleaning up branch '{branch_name}'...")
+                delete_query = await DeleteBranchRelationshipsQuery.init(db=db, branch_name=branch_name)
+                await delete_query.execute(db=db)
+                log.info(f"Branch '{branch_name}' cleaned up.")
+
+            log.info("Deleting orphaned relationships...")
+            delete_relationships_query = await DeleteOrphanRelationshipsQuery.init(db=db)
+            await delete_relationships_query.execute(db=db)
+            log.info("Orphaned relationships deleted.")
+
+        except Exception as exc:
+            migration_result.errors.append(str(exc))
+            log.exception("Error during branch cleanup")
+
+        return migration_result

infrahub/core/migrations/graph/m033_deduplicate_relationship_vertices.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from infrahub.core.migrations.shared import GraphMigration, MigrationResult
+from infrahub.log import get_logger
+
+from ...query import Query, QueryType
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class DeduplicateRelationshipVerticesQuery(Query):
+    """
+    For each group of duplicate Relationships with the same UUID, delete any Relationship that meets the following criteria:
+    - is linked to a deleted node (only if the delete time is before the Relationship's from time)
+    - is linked to a node on an incorrect branch (ie Relationship added on main, but Node is on a branch)
+    """
+
+    name = "deduplicate_relationship_vertices"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+MATCH (root:Root)
+WITH root.default_branch AS default_branch_name
+// ------------
+// Find all Relationship vertices with duplicate UUIDs
+// ------------
+MATCH (r:Relationship)
+WITH r.uuid AS r_uuid, default_branch_name, count(*) AS num_dups
+WHERE num_dups > 1
+WITH DISTINCT r_uuid, default_branch_name
+// ------------
+// get the branched_from time for each relationship edge and node
+// ------------
+MATCH (rel:Relationship {uuid: r_uuid})
+CALL (rel) {
+    MATCH (rel)-[is_rel_e:IS_RELATED {status: "active"}]-(n:Node)
+    MATCH (rel_branch:Branch {name: is_rel_e.branch})
+    RETURN is_rel_e, rel_branch.branched_from AS rel_branched_from, n
+}
+// ------------
+// for each IS_RELATED edge of the relationship, check the IS_PART_OF edges of the Node vertex
+// to determine if this side of the relationship is legal
+// ------------
+CALL (n, is_rel_e, rel_branched_from, default_branch_name) {
+    OPTIONAL MATCH (n)-[is_part_of_e:IS_PART_OF {status: "active"}]->(:Root)
+    WHERE (
+        // the Node's create time must precede the Relationship's create time
+        is_part_of_e.from <= is_rel_e.from AND (is_part_of_e.to >= is_rel_e.from OR is_part_of_e.to IS NULL)
+        // the Node must have been created on a branch of equal or lesser depth than the Relationship
+        AND is_part_of_e.branch_level <= is_rel_e.branch_level
+        // if the Node and Relationships were created on branch_level = 2, then they must be on the same branch
+        AND (
+            is_part_of_e.branch_level = 1
+            OR is_part_of_e.branch = is_rel_e.branch
+        )
+        // if the Node was created on the default branch, and the Relationship was created on a branch,
+        // then the Node must have been created after the branched_from time of the Relationship's branch
+        AND (
+            is_part_of_e.branch <> default_branch_name
+            OR is_rel_e.branch_level = 1
+            OR is_part_of_e.from <= rel_branched_from
+        )
+    )
+    WITH is_part_of_e IS NOT NULL AS is_legal
+    ORDER BY is_legal DESC
+    RETURN is_legal
+    LIMIT 1
+}
+WITH rel, is_legal
+ORDER BY rel, is_legal ASC
+WITH rel, head(collect(is_legal)) AS is_legal
+WHERE is_legal = false
+DETACH DELETE rel
+        """
+        self.add_to_query(query)
+
+
+class Migration033(GraphMigration):
+    """
+    Identifies duplicate Relationship vertices that have the same UUID property. Deletes any duplicates that
+    are linked to deleted nodes or nodes on in incorrect branch.
+    """
+
+    name: str = "033_deduplicate_relationship_vertices"
+    minimum_version: int = 31
+    queries: Sequence[type[Query]] = [DeduplicateRelationshipVerticesQuery]
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        result = MigrationResult()
+        return result

infrahub/core/node/__init__.py

@@ -82,6 +82,9 @@ class Node(BaseNode, metaclass=BaseNodeMeta):
     def get_schema(self) -> NonGenericSchemaTypes:
         return self._schema
 
+    def get_branch(self) -> Branch:
+        return self._branch
+
     def get_kind(self) -> str:
         """Return the main Kind of the Object."""
         return self._schema.kind