infrahub-server 1.3.0b1__py3-none-any.whl → 1.3.0b3__py3-none-any.whl

infrahub/computed_attribute/tasks.py

@@ -15,7 +15,7 @@ from infrahub.events import BranchDeletedEvent
 from infrahub.git.repository import get_initialized_repo
 from infrahub.services import InfrahubServices  # noqa: TC001  needed for prefect flow
 from infrahub.trigger.models import TriggerSetupReport, TriggerType
-from infrahub.trigger.setup import setup_triggers
+from infrahub.trigger.setup import setup_triggers, setup_triggers_specific
 from infrahub.workflows.catalogue import (
     COMPUTED_ATTRIBUTE_PROCESS_JINJA2,
     COMPUTED_ATTRIBUTE_PROCESS_TRANSFORM,
@@ -304,36 +304,30 @@ async def computed_attribute_setup_jinja2(
             await add_tags(branches=[branch_name])
             await wait_for_schema_to_converge(branch_name=branch_name, component=service.component, db=db, log=log)
 
-        triggers = await gather_trigger_computed_attribute_jinja2()
-
+        report: TriggerSetupReport = await setup_triggers_specific(
+            gatherer=gather_trigger_computed_attribute_jinja2, trigger_type=TriggerType.COMPUTED_ATTR_JINJA2
+        )  # type: ignore[misc]
         # Configure all ComputedAttrJinja2Trigger in Prefect
-        async with get_client(sync_client=False) as prefect_client:
-            report: TriggerSetupReport = await setup_triggers(
-                client=prefect_client,
-                triggers=triggers,
-                trigger_type=TriggerType.COMPUTED_ATTR_JINJA2,
-                force_update=False,
-            )  # type: ignore[misc]
 
-            # Since we can have multiple trigger per NodeKind
-            # we need to extract the list of unique node that should be processed
-            unique_nodes: set[tuple[str, str, str]] = {
-                (trigger.branch, trigger.computed_attribute.kind, trigger.computed_attribute.attribute.name)  # type: ignore[attr-defined]
-                for trigger in report.updated + report.created
-            }
-            for branch, kind, attribute_name in unique_nodes:
-                if event_name != BranchDeletedEvent.event_name and branch == branch_name:
-                    await service.workflow.submit_workflow(
-                        workflow=TRIGGER_UPDATE_JINJA_COMPUTED_ATTRIBUTES,
-                        context=context,
-                        parameters={
-                            "branch_name": branch,
-                            "computed_attribute_name": attribute_name,
-                            "computed_attribute_kind": kind,
-                        },
-                    )
-
-        log.info(f"{len(triggers)} Computed Attribute for Jinja2 automation configuration completed")
+        # Since we can have multiple trigger per NodeKind
+        # we need to extract the list of unique node that should be processed
+        unique_nodes: set[tuple[str, str, str]] = {
+            (trigger.branch, trigger.computed_attribute.kind, trigger.computed_attribute.attribute.name)  # type: ignore[attr-defined]
+            for trigger in report.updated + report.created
+        }
+        for branch, kind, attribute_name in unique_nodes:
+            if event_name != BranchDeletedEvent.event_name and branch == branch_name:
+                await service.workflow.submit_workflow(
+                    workflow=TRIGGER_UPDATE_JINJA_COMPUTED_ATTRIBUTES,
+                    context=context,
+                    parameters={
+                        "branch_name": branch,
+                        "computed_attribute_name": attribute_name,
+                        "computed_attribute_kind": kind,
+                    },
+                )
+
+        log.info(f"{report.in_use_count} Computed Attribute for Jinja2 automation configuration completed")
 
 
 @flow(

infrahub/core/constants/__init__.py

@@ -112,6 +112,11 @@ class AccountType(InfrahubStringEnum):
     Git = "Git"
 
 
+class NumberPoolType(InfrahubStringEnum):
+    USER = "User"
+    SCHEMA = "Schema"
+
+
 class AccountStatus(InfrahubStringEnum):
     ACTIVE = "active"
     INACTIVE = "inactive"

infrahub/core/constants/database.py

@@ -12,3 +12,4 @@ class DatabaseEdgeType(Enum):
     IS_PROTECTED = "IS_PROTECTED"
     HAS_OWNER = "HAS_OWNER"
     HAS_SOURCE = "HAS_SOURCE"
+    IS_RESERVED = "IS_RESERVED"

infrahub/core/convert_object_type/conversion.py

@@ -97,7 +97,7 @@ async def convert_object_type(
     if not isinstance(node_schema, NodeSchema):
         raise ValueError(f"Only a node with a NodeSchema can be converted, got {type(node_schema)}")
 
-    async with db.start_transaction() as dbt:  # noqa: PLR1702
+    async with db.start_transaction() as dbt:
         deleted_node_out_rels_peer_ids = await get_out_rels_peers_ids(node=node, db=dbt)
         deleted_node_unidir_rels_peer_ids = await get_unidirectional_rels_peers_ids(node=node, db=dbt, branch=branch)
 

infrahub/core/diff/query/save.py

@@ -78,6 +78,7 @@ class EnrichedNodeBatchCreateQuery(Query):
 
     async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
         self.params = self._build_node_batch_params()
+
         query = """
 UNWIND $node_details_list AS node_details
 WITH
@@ -131,57 +132,74 @@ CALL (diff_node, has_node_conflict, node_conflict_params) {
     OPTIONAL MATCH (diff_node)-[:DIFF_HAS_CONFLICT]->(node_conflict:DiffConflict)
     SET node_conflict = node_conflict_params
 }
+// -------------------------
+// resetting the UNWIND and starting over here reduces memory usage
+// -------------------------
+WITH root_uuid LIMIT 1
+UNWIND $node_details_list AS node_details
+WITH
+    node_details.root_uuid AS root_uuid,
+    node_details.node_map AS node_map,
+    toString(node_details.node_map.node_properties.uuid) AS node_uuid,
+    node_details.node_map.node_properties.db_id AS node_db_id
+MATCH (:DiffRoot {uuid: root_uuid})-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_uuid, db_id: node_db_id})
+WITH diff_node, node_map, %(attr_name_list_comp)s AS attr_names
+OPTIONAL MATCH (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(attr_to_delete:DiffAttribute)
+WHERE NOT (attr_to_delete.name IN attr_names)
+OPTIONAL MATCH (attr_to_delete)-[*..6]->(next_to_delete)
+DETACH DELETE next_to_delete
+DETACH DELETE attr_to_delete
+// -------------------------
+// add attributes for this node
+// -------------------------
+WITH DISTINCT diff_node, node_map
 CALL (diff_node, node_map) {
-    // -------------------------
-    // remove stale attributes for this node
-    // -------------------------
-    CALL (diff_node, node_map) {
-        WITH %(attr_name_list_comp)s AS attr_names
-        OPTIONAL MATCH (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(attr_to_delete:DiffAttribute)
-        WHERE NOT (attr_to_delete.name IN attr_names)
-        OPTIONAL MATCH (attr_to_delete)-[*..6]->(next_to_delete)
-        DETACH DELETE next_to_delete
-        DETACH DELETE attr_to_delete
-    }
-    // -------------------------
-    // add attributes for this node
-    // -------------------------
     UNWIND node_map.attributes AS node_attribute
     MERGE (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(diff_attribute:DiffAttribute {name: node_attribute.node_properties.name})
     SET diff_attribute = node_attribute.node_properties
     // -------------------------
-    // add properties for this attribute
+    // remove stale properties for this attribute
     // -------------------------
-    WITH diff_attribute, node_attribute
+    WITH diff_attribute, node_attribute, %(attr_props_list_comp)s AS prop_types
+    OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty)
+    WHERE NOT (prop_to_delete.property_type IN prop_types)
+    OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete)
+    DETACH DELETE next_to_delete
+    DETACH DELETE prop_to_delete
     // -------------------------
-    // remove stale properties for this attribute
+    // set attribute property values
     // -------------------------
-    CALL (diff_attribute, node_attribute) {
-        WITH %(attr_props_list_comp)s AS prop_types
-        OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty)
-        WHERE NOT (prop_to_delete.property_type IN prop_types)
-        OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete)
-        DETACH DELETE next_to_delete
-        DETACH DELETE prop_to_delete
-    }
+    WITH DISTINCT diff_attribute, node_attribute
     UNWIND node_attribute.properties AS attr_property
     MERGE (diff_attribute)-[:DIFF_HAS_PROPERTY]->(diff_attr_prop:DiffProperty {property_type: attr_property.node_properties.property_type})
     SET diff_attr_prop = attr_property.node_properties
-    // -------------------------
-    // add/remove conflict for this property
-    // -------------------------
     WITH diff_attr_prop, attr_property
     OPTIONAL MATCH (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(current_attr_prop_conflict:DiffConflict)
     WITH diff_attr_prop, attr_property, current_attr_prop_conflict, (attr_property.conflict_params IS NOT NULL) AS has_prop_conflict
-    FOREACH (i in CASE WHEN has_prop_conflict = FALSE THEN [1] ELSE [] END |
+    CALL (has_prop_conflict, current_attr_prop_conflict) {
+        WITH has_prop_conflict, current_attr_prop_conflict
+        WHERE has_prop_conflict = FALSE AND current_attr_prop_conflict IS NOT NULL
         DETACH DELETE current_attr_prop_conflict
-    )
-    FOREACH (i in CASE WHEN has_prop_conflict = TRUE THEN [1] ELSE [] END |
+    }
+    CALL (has_prop_conflict, diff_attr_prop, attr_property) {
+        WITH has_prop_conflict
+        WHERE has_prop_conflict = TRUE
         MERGE (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(diff_attr_prop_conflict:DiffConflict)
         SET diff_attr_prop_conflict = attr_property.conflict_params
-    )
+    }
 }
 // -------------------------
+// resetting the UNWIND and starting over here reduces memory usage
+// -------------------------
+WITH 1 AS resetting LIMIT 1
+UNWIND $node_details_list AS node_details
+WITH
+    node_details.root_uuid AS root_uuid,
+    node_details.node_map AS node_map,
+    toString(node_details.node_map.node_properties.uuid) AS node_uuid,
+    node_details.node_map.node_properties.db_id AS node_db_id
+MATCH (:DiffRoot {uuid: root_uuid})-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_uuid, db_id: node_db_id})
+// -------------------------
 // remove stale relationships for this node
 // -------------------------
 CALL (diff_node, node_map) {
@@ -226,13 +244,17 @@ WITH diff_relationship_element, node_single_relationship
 OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(current_element_conflict:DiffConflict)
 WITH diff_relationship_element, node_single_relationship, current_element_conflict,
     (node_single_relationship.conflict_params IS NOT NULL) AS has_element_conflict
-FOREACH (i in CASE WHEN has_element_conflict = FALSE THEN [1] ELSE [] END |
+CALL (has_element_conflict, current_element_conflict) {
+    WITH has_element_conflict
+    WHERE has_element_conflict = FALSE
     DETACH DELETE current_element_conflict
-)
-FOREACH (i in CASE WHEN has_element_conflict = TRUE THEN [1] ELSE [] END |
+}
+CALL (has_element_conflict, diff_relationship_element, node_single_relationship) {
+    WITH has_element_conflict
+    WHERE has_element_conflict = TRUE
     MERGE (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(element_conflict:DiffConflict)
     SET element_conflict = node_single_relationship.conflict_params
-)
+}
 // -------------------------
 // remove stale properties for this relationship element
 // -------------------------
@@ -260,13 +282,18 @@ WITH diff_relationship_property, node_relationship_property
 OPTIONAL MATCH (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(diff_relationship_property_conflict:DiffConflict)
 WITH diff_relationship_property, node_relationship_property, diff_relationship_property_conflict,
     (node_relationship_property.conflict_params IS NOT NULL) AS has_property_conflict
-FOREACH (i in CASE WHEN has_property_conflict = FALSE THEN [1] ELSE [] END |
+
+CALL (has_property_conflict, diff_relationship_property_conflict) {
+    WITH has_property_conflict
+    WHERE has_property_conflict = FALSE
     DETACH DELETE diff_relationship_property_conflict
-)
-FOREACH (i in CASE WHEN has_property_conflict = TRUE THEN [1] ELSE [] END |
+}
+CALL (has_property_conflict, diff_relationship_property, node_relationship_property) {
+    WITH has_property_conflict
+    WHERE has_property_conflict = TRUE
     MERGE (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(property_conflict:DiffConflict)
     SET property_conflict = node_relationship_property.conflict_params
-)
+}
         """ % {
             "attr_name_list_comp": db.render_list_comprehension(
                 items="node_map.attributes", item_name="node_properties.name"

infrahub/core/diff/query/time_range_query.py

@@ -37,7 +37,6 @@ class EnrichedDiffTimeRangeQuery(Query):
             "from_time": self.from_time.to_string(),
             "to_time": self.to_time.to_string(),
         }
-        # ruff: noqa: E501
         query = """
         // get the roots of all diffs in the query
         MATCH (diff_root:DiffRoot)

infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 28
+GRAPH_VERSION = 30

infrahub/core/migrations/graph/__init__.py

@@ -29,6 +29,9 @@ from .m024_missing_hierarchy_backfill import Migration024
 from .m025_uniqueness_nulls import Migration025
 from .m026_0000_prefix_fix import Migration026
 from .m027_delete_isolated_nodes import Migration027
+from .m028_delete_diffs import Migration028
+from .m029_duplicates_cleanup import Migration029
+from .m030_illegal_edges import Migration030
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -63,6 +66,9 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration025,
     Migration026,
     Migration027,
+    Migration028,
+    Migration029,
+    Migration030,
 ]
 
 

infrahub/core/migrations/graph/m013_convert_git_password_credential.py

@@ -77,7 +77,6 @@ class Migration013ConvertCoreRepositoryWithCred(Query):
 
         self.params["rel_identifier"] = "gitrepository__credential"
 
-        # ruff: noqa: E501
         query = """
         // --------------------------------
         // Identify the git repositories to convert
@@ -194,7 +193,6 @@ class Migration013ConvertCoreRepositoryWithoutCred(Query):
 
         self.params["current_time"] = self.at.to_string()
 
-        # ruff: noqa: E501
         query = """
         // --------------------------------
         // Identify the git repositories to convert