infrahub-server 1.2.11__py3-none-any.whl → 1.2.12__py3-none-any.whl

infrahub/core/constants/database.py

@@ -12,3 +12,4 @@ class DatabaseEdgeType(Enum):
     IS_PROTECTED = "IS_PROTECTED"
     HAS_OWNER = "HAS_OWNER"
     HAS_SOURCE = "HAS_SOURCE"
+    IS_RESERVED = "IS_RESERVED"

infrahub/core/diff/query/save.py

@@ -80,6 +80,7 @@ class EnrichedNodeBatchCreateQuery(Query):
 
     async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
         self.params = self._build_node_batch_params()
+
         query = """
 UNWIND $node_details_list AS node_details
 WITH
@@ -136,65 +137,82 @@ CALL {
     OPTIONAL MATCH (diff_node)-[:DIFF_HAS_CONFLICT]->(node_conflict:DiffConflict)
     SET node_conflict = node_conflict_params
 }
+// -------------------------
+// resetting the UNWIND and starting over here reduces memory usage
+// -------------------------
+WITH root_uuid LIMIT 1
+UNWIND $node_details_list AS node_details
+WITH
+    node_details.root_uuid AS root_uuid,
+    node_details.node_map AS node_map,
+    toString(node_details.node_map.node_properties.uuid) AS node_uuid,
+    node_details.node_map.node_properties.db_id AS node_db_id
+MATCH (:DiffRoot {uuid: root_uuid})-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_uuid, db_id: node_db_id})
+WITH diff_node, node_map, %(attr_name_list_comp)s AS attr_names
+OPTIONAL MATCH (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(attr_to_delete:DiffAttribute)
+WHERE NOT (attr_to_delete.name IN attr_names)
+OPTIONAL MATCH (attr_to_delete)-[*..6]->(next_to_delete)
+DETACH DELETE next_to_delete
+DETACH DELETE attr_to_delete
+// -------------------------
+// add attributes for this node
+// -------------------------
+WITH DISTINCT diff_node, node_map
 CALL {
-    // -------------------------
-    // remove stale attributes for this node
-    // -------------------------
     WITH diff_node, node_map
-    CALL {
-        WITH diff_node, node_map
-        WITH diff_node, %(attr_name_list_comp)s AS attr_names
-        OPTIONAL MATCH (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(attr_to_delete:DiffAttribute)
-        WHERE NOT (attr_to_delete.name IN attr_names)
-        OPTIONAL MATCH (attr_to_delete)-[*..6]->(next_to_delete)
-        DETACH DELETE next_to_delete
-        DETACH DELETE attr_to_delete
-    }
-    // -------------------------
-    // add attributes for this node
-    // -------------------------
     UNWIND node_map.attributes AS node_attribute
     MERGE (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(diff_attribute:DiffAttribute {name: node_attribute.node_properties.name})
     SET diff_attribute = node_attribute.node_properties
     // -------------------------
-    // add properties for this attribute
+    // remove stale properties for this attribute
     // -------------------------
-    WITH diff_attribute, node_attribute
+    WITH diff_attribute, node_attribute, %(attr_props_list_comp)s AS prop_types
+    OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty)
+    WHERE NOT (prop_to_delete.property_type IN prop_types)
+    OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete)
+    DETACH DELETE next_to_delete
+    DETACH DELETE prop_to_delete
     // -------------------------
-    // remove stale properties for this attribute
+    // set attribute property values
     // -------------------------
-    CALL {
-        WITH diff_attribute, node_attribute
-        WITH diff_attribute, %(attr_props_list_comp)s AS prop_types
-        OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty)
-        WHERE NOT (prop_to_delete.property_type IN prop_types)
-        OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete)
-        DETACH DELETE next_to_delete
-        DETACH DELETE prop_to_delete
-    }
+    WITH DISTINCT diff_attribute, node_attribute
     UNWIND node_attribute.properties AS attr_property
     MERGE (diff_attribute)-[:DIFF_HAS_PROPERTY]->(diff_attr_prop:DiffProperty {property_type: attr_property.node_properties.property_type})
     SET diff_attr_prop = attr_property.node_properties
-    // -------------------------
-    // add/remove conflict for this property
-    // -------------------------
     WITH diff_attr_prop, attr_property
     OPTIONAL MATCH (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(current_attr_prop_conflict:DiffConflict)
     WITH diff_attr_prop, attr_property, current_attr_prop_conflict, (attr_property.conflict_params IS NOT NULL) AS has_prop_conflict
-    FOREACH (i in CASE WHEN has_prop_conflict = FALSE THEN [1] ELSE [] END |
+    CALL {
+        WITH has_prop_conflict, current_attr_prop_conflict
+        WITH has_prop_conflict, current_attr_prop_conflict
+        WHERE has_prop_conflict = FALSE AND current_attr_prop_conflict IS NOT NULL
         DETACH DELETE current_attr_prop_conflict
-    )
-    FOREACH (i in CASE WHEN has_prop_conflict = TRUE THEN [1] ELSE [] END |
+    }
+    CALL {
+        WITH has_prop_conflict, diff_attr_prop, attr_property
+        WITH has_prop_conflict, diff_attr_prop, attr_property
+        WHERE has_prop_conflict = TRUE
         MERGE (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(diff_attr_prop_conflict:DiffConflict)
         SET diff_attr_prop_conflict = attr_property.conflict_params
-    )
+    }
 }
 // -------------------------
+// resetting the UNWIND and starting over here reduces memory usage
+// -------------------------
+WITH 1 AS resetting LIMIT 1
+UNWIND $node_details_list AS node_details
+WITH
+    node_details.root_uuid AS root_uuid,
+    node_details.node_map AS node_map,
+    toString(node_details.node_map.node_properties.uuid) AS node_uuid,
+    node_details.node_map.node_properties.db_id AS node_db_id
+MATCH (:DiffRoot {uuid: root_uuid})-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_uuid, db_id: node_db_id})
+// -------------------------
 // remove stale relationships for this node
 // -------------------------
 CALL {
     WITH diff_node, node_map
-    WITH diff_node, %(rel_name_list_comp)s AS rel_names
+    WITH diff_node, node_map, %(rel_name_list_comp)s AS rel_names
     OPTIONAL MATCH (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(rel_to_delete:DiffRelationship)
     WHERE NOT (rel_to_delete.name IN rel_names)
     OPTIONAL MATCH (rel_to_delete)-[*..8]->(next_to_delete)
@@ -214,7 +232,7 @@ SET diff_relationship = node_relationship.node_properties
 WITH diff_relationship, node_relationship
 CALL {
     WITH diff_relationship, node_relationship
-    WITH diff_relationship, %(rel_peers_list_comp)s AS rel_peers
+    WITH diff_relationship, node_relationship, %(rel_peers_list_comp)s AS rel_peers
     OPTIONAL MATCH (diff_relationship)-[:DIFF_HAS_ELEMENT]->(element_to_delete:DiffRelationshipElement)
     WHERE NOT (element_to_delete.peer_id IN rel_peers)
     OPTIONAL MATCH (element_to_delete)-[*..6]->(next_to_delete)
@@ -236,20 +254,26 @@ WITH diff_relationship_element, node_single_relationship
 OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(current_element_conflict:DiffConflict)
 WITH diff_relationship_element, node_single_relationship, current_element_conflict,
     (node_single_relationship.conflict_params IS NOT NULL) AS has_element_conflict
-FOREACH (i in CASE WHEN has_element_conflict = FALSE THEN [1] ELSE [] END |
+CALL {
+    WITH has_element_conflict, current_element_conflict
+    WITH has_element_conflict, current_element_conflict
+    WHERE has_element_conflict = FALSE
     DETACH DELETE current_element_conflict
-)
-FOREACH (i in CASE WHEN has_element_conflict = TRUE THEN [1] ELSE [] END |
+}
+CALL {
+    WITH has_element_conflict, diff_relationship_element, node_single_relationship
+    WITH has_element_conflict, diff_relationship_element, node_single_relationship
+    WHERE has_element_conflict = TRUE
     MERGE (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(element_conflict:DiffConflict)
     SET element_conflict = node_single_relationship.conflict_params
-)
+}
 // -------------------------
 // remove stale properties for this relationship element
 // -------------------------
 WITH diff_relationship_element, node_single_relationship
 CALL {
     WITH diff_relationship_element, node_single_relationship
-    WITH diff_relationship_element, %(element_props_list_comp)s AS element_props
+    WITH diff_relationship_element, node_single_relationship, %(element_props_list_comp)s AS element_props
     OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_PROPERTY]->(property_to_delete:DiffProperty)
     WHERE NOT (property_to_delete.property_type IN element_props)
     OPTIONAL MATCH (property_to_delete)-[*..4]->(next_to_delete)
@@ -271,13 +295,19 @@ WITH diff_relationship_property, node_relationship_property
 OPTIONAL MATCH (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(diff_relationship_property_conflict:DiffConflict)
 WITH diff_relationship_property, node_relationship_property, diff_relationship_property_conflict,
     (node_relationship_property.conflict_params IS NOT NULL) AS has_property_conflict
-FOREACH (i in CASE WHEN has_property_conflict = FALSE THEN [1] ELSE [] END |
+CALL {
+    WITH has_property_conflict, diff_relationship_property_conflict
+    WITH has_property_conflict, diff_relationship_property_conflict
+    WHERE has_property_conflict = FALSE
     DETACH DELETE diff_relationship_property_conflict
-)
-FOREACH (i in CASE WHEN has_property_conflict = TRUE THEN [1] ELSE [] END |
+}
+CALL {
+    WITH has_property_conflict, diff_relationship_property, node_relationship_property
+    WITH has_property_conflict, diff_relationship_property, node_relationship_property
+    WHERE has_property_conflict = TRUE
     MERGE (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(property_conflict:DiffConflict)
     SET property_conflict = node_relationship_property.conflict_params
-)
+}
         """ % {
             "attr_name_list_comp": db.render_list_comprehension(
                 items="node_map.attributes", item_name="node_properties.name"

infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 28
+GRAPH_VERSION = 30

infrahub/core/migrations/graph/__init__.py

@@ -29,6 +29,9 @@ from .m024_missing_hierarchy_backfill import Migration024
 from .m025_uniqueness_nulls import Migration025
 from .m026_0000_prefix_fix import Migration026
 from .m027_delete_isolated_nodes import Migration027
+from .m028_delete_diffs import Migration028
+from .m029_duplicates_cleanup import Migration029
+from .m030_illegal_edges import Migration030
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -63,6 +66,9 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration025,
     Migration026,
     Migration027,
+    Migration028,
+    Migration029,
+    Migration030,
 ]