infrahub-server 1.1.5__py3-none-any.whl → 1.1.6__py3-none-any.whl

infrahub/api/oidc.py

@@ -210,6 +210,7 @@ async def _get_id_token_groups(
         algorithms=oidc_config.id_token_signing_alg_values_supported,
         audience=client_id,
         issuer=str(oidc_config.issuer),
+        options={"verify_signature": False, "verify_aud": False, "verify_iss": False},
     )
 
     return decoded_token.get("groups", [])

infrahub/core/diff/calculator.py

@@ -4,7 +4,12 @@ from infrahub import config
 from infrahub.core import registry
 from infrahub.core.branch import Branch
 from infrahub.core.diff.query_parser import DiffQueryParser
-from infrahub.core.query.diff import DiffAllPathsQuery, DiffCalculationQuery, DiffFieldPathsQuery, DiffNodePathsQuery
+from infrahub.core.query.diff import (
+    DiffCalculationQuery,
+    DiffFieldPathsQuery,
+    DiffNodePathsQuery,
+    DiffPropertyPathsQuery,
+)
 from infrahub.core.timestamp import Timestamp
 from infrahub.database import InfrahubDatabase
 from infrahub.log import get_logger
@@ -86,6 +91,7 @@ class DiffCalculator:
         )
         node_limit = int(config.SETTINGS.database.query_size_limit / 10)
         fields_limit = int(config.SETTINGS.database.query_size_limit / 3)
+        properties_limit = int(config.SETTINGS.database.query_size_limit)
 
         calculation_request = DiffCalculationRequest(
             base_branch=base_branch,
@@ -113,21 +119,14 @@ class DiffCalculator:
         )
         log.info("Diff field-level calculation queries for branch complete")
 
-        branch_diff_query = await DiffAllPathsQuery.init(
-            db=self.db,
-            branch=diff_branch,
-            base_branch=base_branch,
-            diff_branch_from_time=diff_branch_from_time,
-            diff_from=from_time,
-            diff_to=to_time,
+        log.info("Beginning diff property-level calculation queries for branch")
+        await self._run_diff_calculation_query(
+            diff_parser=diff_parser,
+            query_class=DiffPropertyPathsQuery,
+            calculation_request=calculation_request,
+            limit=properties_limit,
         )
-        log.info("Beginning diff calculation query for branch")
-        await branch_diff_query.execute(db=self.db)
-        log.info("Diff calculation query for branch complete")
-        log.info("Reading results of query for branch")
-        for query_result in branch_diff_query.get_results():
-            diff_parser.read_result(query_result=query_result)
-        log.info("Results of query for branch read")
+        log.info("Diff property-level calculation queries for branch complete")
 
         if base_branch.name != diff_branch.name:
             current_node_field_specifiers = diff_parser.get_current_node_field_specifiers()
@@ -160,32 +159,15 @@ class DiffCalculator:
             )
             log.info("Diff field-level calculation queries for base complete")
 
-            # Temporary until next change
-            current_node_field_specifier_tuples: list[tuple[str, str]] = []
-            new_node_field_specifier_tuples: list[tuple[str, str]] = []
-            for node_uuid, field_names in current_node_field_specifiers.items():
-                current_node_field_specifier_tuples.extend((node_uuid, field_name) for field_name in field_names)
-            for node_uuid, field_names in new_node_field_specifiers.items():
-                new_node_field_specifier_tuples.extend((node_uuid, field_name) for field_name in field_names)
-
-            base_diff_query = await DiffAllPathsQuery.init(
-                db=self.db,
-                branch=base_branch,
-                base_branch=base_branch,
-                diff_branch_from_time=diff_branch_from_time,
-                diff_from=from_time,
-                diff_to=to_time,
-                current_node_field_specifiers=current_node_field_specifier_tuples,
-                new_node_field_specifiers=new_node_field_specifier_tuples,
+            log.info("Beginning diff property-level calculation queries for base")
+            await self._run_diff_calculation_query(
+                diff_parser=diff_parser,
+                query_class=DiffPropertyPathsQuery,
+                calculation_request=calculation_request,
+                limit=properties_limit,
             )
+            log.info("Diff property-level calculation queries for base complete")
 
-            log.info("Beginning diff calculation query for base")
-            await base_diff_query.execute(db=self.db)
-            log.info("Diff calculation query for base complete")
-            log.info("Reading results of query for base")
-            for query_result in base_diff_query.get_results():
-                diff_parser.read_result(query_result=query_result)
-            log.info("Results of query for branch read")
         log.info("Parsing calculated diff")
         diff_parser.parse(include_unchanged=include_unchanged)
         log.info("Calculated diff parsed")

infrahub/core/diff/ipam_diff_parser.py

@@ -116,7 +116,7 @@ class IpamDiffParser:
                 rels = await node_from_db.ip_namespace.get_relationships(db=self.db)  # type: ignore[attr-defined]
                 if rels:
                     cnd.namespace_id = rels[0].get_peer_id()
-            if cnd.ip_value and cnd.namespace_id:
+            if cnd.ip_value and cnd.namespace_id and cnd.node_uuid in uuids_missing_data:
                 uuids_missing_data.remove(cnd.node_uuid)
 
     async def _add_missing_values(

infrahub/core/diff/merger/serializer.py

@@ -294,10 +294,6 @@ class DiffMergeSerializer:
         actions_and_peers = self._get_actions_and_peers(relationship_diff=relationship_diff)
         added_peer_ids = [peer_id for action, peer_id in actions_and_peers if action is DiffAction.ADDED]
         removed_peer_ids = [peer_id for action, peer_id in actions_and_peers if action is DiffAction.REMOVED]
-        if not added_peer_ids:
-            added_peer_ids = [relationship_diff.peer_id]
-        if not removed_peer_ids:
-            removed_peer_ids = [relationship_diff.peer_id]
         for action, peer_id in actions_and_peers:
             if (
                 peer_id
@@ -314,6 +310,7 @@ class DiffMergeSerializer:
             relationship_diff_properties=relationship_diff.properties,
             added_peer_ids=added_peer_ids,
             removed_peer_ids=removed_peer_ids,
+            unchanged_peer_id=relationship_diff.peer_id,
         )
         return relationship_dicts, relationship_property_dicts
 
@@ -324,9 +321,14 @@ class DiffMergeSerializer:
         relationship_diff_properties: set[EnrichedDiffProperty],
         added_peer_ids: list[str],
         removed_peer_ids: list[str],
+        unchanged_peer_id: str,
     ) -> list[RelationshipPropertyMergeDict]:
-        added_property_dicts = self._get_default_property_merge_dicts(action=DiffAction.ADDED)
-        removed_property_dicts = self._get_default_property_merge_dicts(action=DiffAction.REMOVED)
+        added_property_dicts = {}
+        removed_property_dicts = {}
+        if added_peer_ids:
+            added_property_dicts = self._get_default_property_merge_dicts(action=DiffAction.ADDED)
+        if removed_peer_ids:
+            removed_property_dicts = self._get_default_property_merge_dicts(action=DiffAction.REMOVED)
         for property_diff in relationship_diff_properties:
             if property_diff.property_type is DatabaseEdgeType.IS_RELATED:
                 # handled above
@@ -348,10 +350,15 @@ class DiffMergeSerializer:
                 elif action is DiffAction.REMOVED:
                     removed_property_dicts[property_diff.property_type] = property_dict
         relationship_property_dicts = []
+        peers_and_property_dicts: list[tuple[str, dict[DatabaseEdgeType, PropertyMergeDict]]] = []
         if added_property_dicts:
-            peers_and_property_dicts = [(peer_id, added_property_dicts) for peer_id in added_peer_ids]
+            peers_and_property_dicts += [
+                (peer_id, added_property_dicts) for peer_id in (added_peer_ids or [unchanged_peer_id])
+            ]
         if removed_property_dicts:
-            peers_and_property_dicts += [(peer_id, removed_property_dicts) for peer_id in removed_peer_ids]
+            peers_and_property_dicts += [
+                (peer_id, removed_property_dicts) for peer_id in (removed_peer_ids or [unchanged_peer_id])
+            ]
         for peer_id, property_dicts in peers_and_property_dicts:
             if (
                 peer_id

infrahub/core/diff/query/diff_get.py

@@ -17,8 +17,6 @@ QUERY_MATCH_NODES = """
     AND ($to_time IS NULL OR diff_root.to_time <= $to_time)
     AND ($tracking_id IS NULL OR diff_root.tracking_id = $tracking_id)
     AND ($diff_ids IS NULL OR diff_root.uuid IN $diff_ids)
-    WITH diff_root
-    ORDER BY diff_root.base_branch, diff_root.diff_branch, diff_root.from_time, diff_root.to_time
     // get all the nodes attached to the diffs
     OPTIONAL MATCH (diff_root)-[:DIFF_HAS_NODE]->(diff_node:DiffNode)
     """
@@ -79,26 +77,21 @@ class EnrichedDiffGetQuery(Query):
             self.add_to_query(query=query_filters)
 
         query_2 = """
-        // group by diff node uuid for pagination
-        WITH diff_node.uuid AS diff_node_uuid, diff_node.kind AS diff_node_kind, collect([diff_root, diff_node]) AS node_root_tuples
-        // order by kind and latest label for each diff_node uuid
-        CALL {
-            WITH node_root_tuples
-            UNWIND node_root_tuples AS nrt
-            WITH nrt[0] AS diff_root, nrt[1] AS diff_node
-            ORDER BY diff_root.from_time DESC
-            RETURN diff_node.label AS latest_node_label
-            LIMIT 1
-        }
-        WITH diff_node_kind, node_root_tuples, latest_node_label
-        ORDER BY diff_node_kind, latest_node_label
+        WITH diff_root, diff_node
+        ORDER BY diff_root.base_branch, diff_root.diff_branch, diff_root.from_time, diff_root.to_time, diff_node.uuid
+        // -------------------------------------
+        // Limit number of results
+        // -------------------------------------
         SKIP COALESCE($offset, 0)
         LIMIT $limit
-        UNWIND node_root_tuples AS nrt
-        WITH nrt[0] AS diff_root, nrt[1] AS diff_node
-        WITH diff_root, diff_node
+        // -------------------------------------
+        // Check if more data after this limited group
+        // -------------------------------------
+        WITH collect([diff_root, diff_node]) AS limited_results
+        WITH limited_results, size(limited_results) = $limit AS has_more_data
+        UNWIND limited_results AS one_result
+        WITH one_result[0] AS diff_root, one_result[1] AS diff_node, has_more_data
         // if depth limit, make sure not to exceed it when traversing linked nodes
-        WITH diff_root, diff_node
         // -------------------------------------
         // Retrieve Parents
         // -------------------------------------
@@ -109,12 +102,12 @@ class EnrichedDiffGetQuery(Query):
             ORDER BY size(nodes(parents_path)) DESC
             LIMIT 1
         }
-        WITH diff_root, diff_node, parents_path
+        WITH diff_root, diff_node, has_more_data, parents_path
         // -------------------------------------
         // Retrieve conflicts
         // -------------------------------------
         OPTIONAL MATCH (diff_node)-[:DIFF_HAS_CONFLICT]->(diff_node_conflict:DiffConflict)
-        WITH diff_root, diff_node, parents_path, diff_node_conflict
+        WITH diff_root, diff_node, has_more_data, parents_path, diff_node_conflict
         // -------------------------------------
         // Retrieve Attributes
         // -------------------------------------
@@ -128,7 +121,7 @@ class EnrichedDiffGetQuery(Query):
             RETURN diff_attribute, diff_attr_property, diff_attr_property_conflict
             ORDER BY diff_attribute.name, diff_attr_property.property_type
         }
-        WITH diff_root, diff_node, parents_path, diff_node_conflict, collect([diff_attribute, diff_attr_property, diff_attr_property_conflict]) as diff_attributes
+        WITH diff_root, diff_node, has_more_data, parents_path, diff_node_conflict, collect([diff_attribute, diff_attr_property, diff_attr_property_conflict]) as diff_attributes
         // -------------------------------------
         // Retrieve Relationships
         // -------------------------------------
@@ -150,6 +143,7 @@ class EnrichedDiffGetQuery(Query):
         WITH
             diff_root,
             diff_node,
+            has_more_data,
             parents_path,
             diff_node_conflict,
             diff_attributes,
@@ -161,6 +155,7 @@ class EnrichedDiffGetQuery(Query):
         self.return_labels = [
             "diff_root",
             "diff_node",
+            "has_more_data",
             "parents_path",
             "diff_node_conflict",
             "diff_attributes",

infrahub/core/diff/repository/deserializer.py

@@ -1,5 +1,3 @@
-from typing import Iterable
-
 from neo4j.graph import Node as Neo4jNode
 from neo4j.graph import Path as Neo4jPath
 
@@ -30,37 +28,47 @@ class EnrichedDiffDeserializer:
         self._diff_node_rel_group_map: dict[tuple[str, str, str], EnrichedDiffRelationship] = {}
         self._diff_node_rel_element_map: dict[tuple[str, str, str, str], EnrichedDiffSingleRelationship] = {}
         self._diff_prop_map: dict[tuple[str, str, str, str] | tuple[str, str, str, str, str], EnrichedDiffProperty] = {}
+        # {EnrichedDiffRoot: [(node_uuid, parents_path: Neo4jPath), ...]}
+        self._parents_path_map: dict[EnrichedDiffRoot, list[tuple[str, Neo4jPath]]] = {}
 
-    def _initialize(self) -> None:
+    def initialize(self) -> None:
         self._diff_root_map = {}
         self._diff_node_map = {}
         self._diff_node_attr_map = {}
         self._diff_node_rel_group_map = {}
         self._diff_node_rel_element_map = {}
         self._diff_prop_map = {}
+        self._parents_path_map = {}
 
-    async def deserialize(
-        self, database_results: Iterable[QueryResult], include_parents: bool
-    ) -> list[EnrichedDiffRoot]:
-        self._initialize()
-        results = list(database_results)
-        for result in results:
-            enriched_root = self._deserialize_diff_root(root_node=result.get_node("diff_root"))
-            node_node = result.get(label="diff_node")
-            if not isinstance(node_node, Neo4jNode):
-                continue
-            enriched_node = self._deserialize_diff_node(node_node=node_node, enriched_root=enriched_root)
-            node_conflict_node = result.get(label="diff_node_conflict")
-            if isinstance(node_conflict_node, Neo4jNode) and not enriched_node.conflict:
-                conflict = self.deserialize_conflict(diff_conflict_node=node_conflict_node)
-                enriched_node.conflict = conflict
-            self._deserialize_attributes(result=result, enriched_root=enriched_root, enriched_node=enriched_node)
-            self._deserialize_relationships(result=result, enriched_root=enriched_root, enriched_node=enriched_node)
+    def _track_parents_path(self, enriched_root: EnrichedDiffRoot, node_uuid: str, parents_path: Neo4jPath) -> None:
+        if enriched_root not in self._parents_path_map:
+            self._parents_path_map[enriched_root] = []
+        self._parents_path_map[enriched_root].append((node_uuid, parents_path))
+
+    async def read_result(self, result: QueryResult, include_parents: bool) -> None:
+        enriched_root = self._deserialize_diff_root(root_node=result.get_node("diff_root"))
+        node_node = result.get(label="diff_node")
+        if not isinstance(node_node, Neo4jNode):
+            return
+        enriched_node = self._deserialize_diff_node(node_node=node_node, enriched_root=enriched_root)
 
         if include_parents:
-            for result in results:
-                enriched_root = self._deserialize_diff_root(root_node=result.get_node("diff_root"))
-                self._deserialize_parents(result=result, enriched_root=enriched_root)
+            parents_path = result.get("parents_path")
+            if parents_path and isinstance(parents_path, Neo4jPath):
+                self._track_parents_path(
+                    enriched_root=enriched_root, node_uuid=enriched_node.uuid, parents_path=parents_path
+                )
+
+        node_conflict_node = result.get(label="diff_node_conflict")
+        if isinstance(node_conflict_node, Neo4jNode) and not enriched_node.conflict:
+            conflict = self.deserialize_conflict(diff_conflict_node=node_conflict_node)
+            enriched_node.conflict = conflict
+        self._deserialize_attributes(result=result, enriched_root=enriched_root, enriched_node=enriched_node)
+        self._deserialize_relationships(result=result, enriched_root=enriched_root, enriched_node=enriched_node)
+
+    async def deserialize(self, include_parents: bool = True) -> list[EnrichedDiffRoot]:
+        if include_parents:
+            self._deserialize_parents()
 
         return list(self._diff_root_map.values())
 
@@ -117,30 +125,26 @@ class EnrichedDiffDeserializer:
                 conflict = self.deserialize_conflict(diff_conflict_node=property_conflict)
                 element_property.conflict = conflict
 
-    def _deserialize_parents(self, result: QueryResult, enriched_root: EnrichedDiffRoot) -> None:
-        parents_path = result.get("parents_path")
-        if not parents_path or not isinstance(parents_path, Neo4jPath):
-            return
-
-        node_uuid = result.get(label="diff_node").get("uuid")
-
-        # Remove the node itself from the path
-        parents_path = parents_path.nodes[1:]  # type: ignore[union-attr]
-
-        # TODO Ensure the list is even
-        current_node_uuid = node_uuid
-        for rel, parent in zip(parents_path[::2], parents_path[1::2]):
-            enriched_root.add_parent(
-                node_id=current_node_uuid,
-                parent_id=parent.get("uuid"),
-                parent_kind=parent.get("kind"),
-                parent_label=parent.get("label"),
-                parent_rel_name=rel.get("name"),
-                parent_rel_identifier=rel.get("identifier"),
-                parent_rel_cardinality=RelationshipCardinality(rel.get("cardinality")),
-                parent_rel_label=rel.get("label"),
-            )
-            current_node_uuid = parent.get("uuid")
+    def _deserialize_parents(self) -> None:
+        for enriched_root, node_path_tuples in self._parents_path_map.items():
+            for node_uuid, parents_path in node_path_tuples:
+                # Remove the node itself from the path
+                parents_path_slice = parents_path.nodes[1:]
+
+                # TODO Ensure the list is even
+                current_node_uuid = node_uuid
+                for rel, parent in zip(parents_path_slice[::2], parents_path_slice[1::2]):
+                    enriched_root.add_parent(
+                        node_id=current_node_uuid,
+                        parent_id=parent.get("uuid"),
+                        parent_kind=parent.get("kind"),
+                        parent_label=parent.get("label"),
+                        parent_rel_name=rel.get("name"),
+                        parent_rel_identifier=rel.get("identifier"),
+                        parent_rel_cardinality=RelationshipCardinality(rel.get("cardinality")),
+                        parent_rel_label=rel.get("label"),
+                    )
+                    current_node_uuid = parent.get("uuid")
 
     @classmethod
     def _get_str_or_none_property_value(cls, node: Neo4jNode, property_name: str) -> str | None:

infrahub/core/diff/repository/repository.py

@@ -47,6 +47,49 @@ class DiffRepository:
         self.db = db
         self.deserializer = deserializer
 
+    async def _run_get_diff_query(
+        self,
+        base_branch_name: str,
+        diff_branch_names: list[str],
+        limit: int,
+        from_time: Timestamp | None = None,
+        to_time: Timestamp | None = None,
+        filters: EnrichedDiffQueryFilters | None = None,
+        offset: int = 0,
+        include_parents: bool = True,
+        max_depth: int | None = None,
+        tracking_id: TrackingId | None = None,
+        diff_ids: list[str] | None = None,
+    ) -> list[EnrichedDiffRoot]:
+        self.deserializer.initialize()
+        has_more_data = True
+        while has_more_data:
+            get_query = await EnrichedDiffGetQuery.init(
+                db=self.db,
+                base_branch_name=base_branch_name,
+                diff_branch_names=diff_branch_names,
+                from_time=from_time,
+                to_time=to_time,
+                filters=filters,
+                max_depth=max_depth,
+                limit=limit,
+                offset=offset,
+                tracking_id=tracking_id,
+                diff_ids=diff_ids,
+            )
+            log.info(f"Beginning enriched diff get query {limit=}, {offset=}")
+            await get_query.execute(db=self.db)
+            log.info("Enriched diff get query complete")
+            last_result = None
+            for query_result in get_query.get_results():
+                await self.deserializer.read_result(result=query_result, include_parents=include_parents)
+                last_result = query_result
+            has_more_data = False
+            if last_result:
+                has_more_data = last_result.get_as_type("has_more_data", bool)
+            offset += limit
+        return await self.deserializer.deserialize()
+
     async def get(
         self,
         base_branch_name: str,
@@ -62,23 +105,20 @@ class DiffRepository:
         include_empty: bool = False,
     ) -> list[EnrichedDiffRoot]:
         final_max_depth = config.SETTINGS.database.max_depth_search_hierarchy
-        query = await EnrichedDiffGetQuery.init(
-            db=self.db,
+        limit = limit or int(config.SETTINGS.database.query_size_limit / 10)
+        diff_roots = await self._run_get_diff_query(
             base_branch_name=base_branch_name,
             diff_branch_names=diff_branch_names,
             from_time=from_time,
             to_time=to_time,
             filters=EnrichedDiffQueryFilters(**dict(filters or {})),
+            include_parents=include_parents,
             max_depth=final_max_depth,
             limit=limit,
-            offset=offset,
+            offset=offset or 0,
             tracking_id=tracking_id,
             diff_ids=diff_ids,
         )
-        await query.execute(db=self.db)
-        diff_roots = await self.deserializer.deserialize(
-            database_results=query.get_results(), include_parents=include_parents
-        )
         if not include_empty:
             diff_roots = [dr for dr in diff_roots if len(dr.nodes) > 0]
         return diff_roots
@@ -91,30 +131,23 @@ class DiffRepository:
         to_time: Timestamp,
     ) -> list[EnrichedDiffs]:
         max_depth = config.SETTINGS.database.max_depth_search_hierarchy
-        query = await EnrichedDiffGetQuery.init(
-            db=self.db,
+        limit = int(config.SETTINGS.database.query_size_limit / 10)
+        diff_branch_roots = await self._run_get_diff_query(
             base_branch_name=base_branch_name,
             diff_branch_names=[diff_branch_name],
             from_time=from_time,
             to_time=to_time,
             max_depth=max_depth,
-        )
-        await query.execute(db=self.db)
-        diff_branch_roots = await self.deserializer.deserialize(
-            database_results=query.get_results(), include_parents=True
+            limit=limit,
         )
         diffs_by_uuid = {dbr.uuid: dbr for dbr in diff_branch_roots}
-        base_partner_query = await EnrichedDiffGetQuery.init(
-            db=self.db,
+        base_branch_roots = await self._run_get_diff_query(
             base_branch_name=base_branch_name,
             diff_branch_names=[base_branch_name],
             max_depth=max_depth,
+            limit=limit,
             diff_ids=[d.partner_uuid for d in diffs_by_uuid.values()],
         )
-        await base_partner_query.execute(db=self.db)
-        base_branch_roots = await self.deserializer.deserialize(
-            database_results=base_partner_query.get_results(), include_parents=True
-        )
         diffs_by_uuid.update({bbr.uuid: bbr for bbr in base_branch_roots})
         return [
             EnrichedDiffs(