infrahub-server 1.1.7__py3-none-any.whl → 1.1.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. infrahub/config.py +6 -0
  2. infrahub/core/diff/enricher/cardinality_one.py +5 -0
  3. infrahub/core/diff/enricher/hierarchy.py +17 -4
  4. infrahub/core/diff/enricher/labels.py +5 -0
  5. infrahub/core/diff/enricher/path_identifier.py +5 -0
  6. infrahub/core/diff/model/path.py +24 -1
  7. infrahub/core/diff/parent_node_adder.py +78 -0
  8. infrahub/core/diff/payload_builder.py +13 -2
  9. infrahub/core/diff/query/merge.py +20 -17
  10. infrahub/core/diff/query/save.py +188 -182
  11. infrahub/core/diff/query/summary_counts_enricher.py +51 -4
  12. infrahub/core/diff/repository/deserializer.py +8 -3
  13. infrahub/core/diff/repository/repository.py +156 -38
  14. infrahub/core/diff/tasks.py +4 -4
  15. infrahub/core/graph/__init__.py +1 -1
  16. infrahub/core/graph/index.py +3 -0
  17. infrahub/core/migrations/graph/__init__.py +6 -0
  18. infrahub/core/migrations/graph/m019_restore_rels_to_time.py +256 -0
  19. infrahub/core/migrations/graph/m020_duplicate_edges.py +160 -0
  20. infrahub/core/migrations/graph/m021_missing_hierarchy_merge.py +51 -0
  21. infrahub/core/migrations/query/node_duplicate.py +38 -18
  22. infrahub/core/migrations/schema/node_remove.py +26 -12
  23. infrahub/core/migrations/shared.py +10 -8
  24. infrahub/core/node/__init__.py +13 -8
  25. infrahub/core/node/constraints/grouped_uniqueness.py +16 -3
  26. infrahub/core/query/attribute.py +2 -0
  27. infrahub/core/query/node.py +69 -19
  28. infrahub/core/query/relationship.py +105 -16
  29. infrahub/core/query/resource_manager.py +2 -0
  30. infrahub/core/relationship/model.py +8 -12
  31. infrahub/core/schema/definitions/core.py +1 -0
  32. infrahub/database/__init__.py +10 -1
  33. infrahub/database/metrics.py +7 -1
  34. infrahub/dependencies/builder/diff/deserializer.py +3 -1
  35. infrahub/dependencies/builder/diff/enricher/hierarchy.py +3 -1
  36. infrahub/dependencies/builder/diff/parent_node_adder.py +8 -0
  37. infrahub/graphql/initialization.py +3 -0
  38. infrahub/graphql/loaders/node.py +2 -12
  39. infrahub/graphql/loaders/peers.py +77 -0
  40. infrahub/graphql/loaders/shared.py +13 -0
  41. infrahub/graphql/mutations/diff.py +17 -10
  42. infrahub/graphql/mutations/resource_manager.py +3 -3
  43. infrahub/graphql/resolvers/many_relationship.py +264 -0
  44. infrahub/graphql/resolvers/resolver.py +3 -103
  45. infrahub/graphql/subscription/graphql_query.py +2 -0
  46. infrahub_sdk/batch.py +2 -2
  47. infrahub_sdk/client.py +10 -2
  48. infrahub_sdk/config.py +4 -1
  49. infrahub_sdk/ctl/check.py +4 -4
  50. infrahub_sdk/ctl/cli_commands.py +16 -11
  51. infrahub_sdk/ctl/exceptions.py +0 -6
  52. infrahub_sdk/ctl/exporter.py +1 -1
  53. infrahub_sdk/ctl/generator.py +5 -5
  54. infrahub_sdk/ctl/importer.py +3 -2
  55. infrahub_sdk/ctl/menu.py +1 -1
  56. infrahub_sdk/ctl/object.py +1 -1
  57. infrahub_sdk/ctl/repository.py +23 -15
  58. infrahub_sdk/ctl/schema.py +2 -2
  59. infrahub_sdk/ctl/utils.py +6 -5
  60. infrahub_sdk/ctl/validate.py +2 -1
  61. infrahub_sdk/data.py +1 -1
  62. infrahub_sdk/exceptions.py +12 -0
  63. infrahub_sdk/generator.py +3 -0
  64. infrahub_sdk/node.py +8 -8
  65. infrahub_sdk/protocols.py +0 -1
  66. infrahub_sdk/schema/__init__.py +0 -3
  67. infrahub_sdk/testing/docker.py +30 -0
  68. infrahub_sdk/testing/schemas/animal.py +9 -0
  69. infrahub_sdk/transfer/exporter/json.py +1 -1
  70. infrahub_sdk/utils.py +11 -1
  71. infrahub_sdk/yaml.py +2 -3
  72. {infrahub_server-1.1.7.dist-info → infrahub_server-1.1.9.dist-info}/METADATA +1 -1
  73. {infrahub_server-1.1.7.dist-info → infrahub_server-1.1.9.dist-info}/RECORD +78 -71
  74. infrahub_testcontainers/container.py +11 -0
  75. infrahub_testcontainers/docker-compose.test.yml +3 -6
  76. infrahub_sdk/ctl/_file.py +0 -13
  77. {infrahub_server-1.1.7.dist-info → infrahub_server-1.1.9.dist-info}/LICENSE.txt +0 -0
  78. {infrahub_server-1.1.7.dist-info → infrahub_server-1.1.9.dist-info}/WHEEL +0 -0
  79. {infrahub_server-1.1.7.dist-info → infrahub_server-1.1.9.dist-info}/entry_points.txt +0 -0
@@ -203,15 +203,16 @@ class NodeCreateAllQuery(NodeQuery):
203
203
  }
204
204
  ipnetwork_prop_list = [f"{key}: {value}" for key, value in ipnetwork_prop.items()]
205
205
 
206
- query = """
207
- MATCH (root:Root)
208
- CREATE (n:Node:%(labels)s $node_prop )
209
- CREATE (n)-[r:IS_PART_OF $node_branch_prop ]->(root)
206
+ attrs_query = """
210
207
  WITH distinct n
211
- FOREACH ( attr IN $attrs |
208
+ UNWIND $attrs AS attr
209
+ CALL {
210
+ WITH n, attr
212
211
  CREATE (a:Attribute { uuid: attr.uuid, name: attr.name, branch_support: attr.branch_support })
213
212
  CREATE (n)-[:HAS_ATTRIBUTE { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(a)
214
213
  MERGE (av:AttributeValue { value: attr.content.value, is_default: attr.content.is_default })
214
+ WITH n, attr, av, a
215
+ LIMIT 1
215
216
  CREATE (a)-[:HAS_VALUE { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(av)
216
217
  MERGE (ip:Boolean { value: attr.is_protected })
217
218
  MERGE (iv:Boolean { value: attr.is_visible })
@@ -225,11 +226,19 @@ class NodeCreateAllQuery(NodeQuery):
225
226
  MERGE (peer:Node { uuid: prop.peer_id })
226
227
  CREATE (a)-[:HAS_OWNER { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(peer)
227
228
  )
228
- )
229
- FOREACH ( attr IN $attrs_iphost |
229
+ }"""
230
+
231
+ attrs_iphost_query = """
232
+ WITH distinct n
233
+ UNWIND $attrs_iphost AS attr_iphost
234
+ CALL {
235
+ WITH n, attr_iphost
236
+ WITH n, attr_iphost AS attr
230
237
  CREATE (a:Attribute { uuid: attr.uuid, name: attr.name, branch_support: attr.branch_support })
231
238
  CREATE (n)-[:HAS_ATTRIBUTE { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(a)
232
239
  MERGE (av:AttributeValue:AttributeIPHost { %(iphost_prop)s })
240
+ WITH n, attr, av, a
241
+ LIMIT 1
233
242
  CREATE (a)-[:HAS_VALUE { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(av)
234
243
  MERGE (ip:Boolean { value: attr.is_protected })
235
244
  MERGE (iv:Boolean { value: attr.is_visible })
@@ -243,11 +252,20 @@ class NodeCreateAllQuery(NodeQuery):
243
252
  MERGE (peer:Node { uuid: prop.peer_id })
244
253
  CREATE (a)-[:HAS_OWNER { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(peer)
245
254
  )
246
- )
247
- FOREACH ( attr IN $attrs_ipnetwork |
255
+ }
256
+ """ % {"iphost_prop": ", ".join(iphost_prop_list)}
257
+
258
+ attrs_ipnetwork_query = """
259
+ WITH distinct n
260
+ UNWIND $attrs_ipnetwork AS attr_ipnetwork
261
+ CALL {
262
+ WITH n, attr_ipnetwork
263
+ WITH n, attr_ipnetwork AS attr
248
264
  CREATE (a:Attribute { uuid: attr.uuid, name: attr.name, branch_support: attr.branch_support })
249
265
  CREATE (n)-[:HAS_ATTRIBUTE { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(a)
250
266
  MERGE (av:AttributeValue:AttributeIPNetwork { %(ipnetwork_prop)s })
267
+ WITH n, attr, av, a
268
+ LIMIT 1
251
269
  CREATE (a)-[:HAS_VALUE { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(av)
252
270
  MERGE (ip:Boolean { value: attr.is_protected })
253
271
  MERGE (iv:Boolean { value: attr.is_visible })
@@ -261,8 +279,14 @@ class NodeCreateAllQuery(NodeQuery):
261
279
  MERGE (peer:Node { uuid: prop.peer_id })
262
280
  CREATE (a)-[:HAS_OWNER { branch: attr.branch, branch_level: attr.branch_level, status: attr.status, from: $at }]->(peer)
263
281
  )
264
- )
265
- FOREACH ( rel IN $rels_bidir |
282
+ }
283
+ """ % {"ipnetwork_prop": ", ".join(ipnetwork_prop_list)}
284
+
285
+ rels_bidir_query = """
286
+ WITH distinct n
287
+ UNWIND $rels_bidir AS rel
288
+ CALL {
289
+ WITH n, rel
266
290
  MERGE (d:Node { uuid: rel.destination_id })
267
291
  CREATE (rl:Relationship { uuid: rel.uuid, name: rel.name, branch_support: rel.branch_support })
268
292
  CREATE (n)-[:IS_RELATED %(rel_prop)s ]->(rl)
@@ -279,8 +303,15 @@ class NodeCreateAllQuery(NodeQuery):
279
303
  MERGE (peer:Node { uuid: prop.peer_id })
280
304
  CREATE (rl)-[:HAS_OWNER { branch: rel.branch, branch_level: rel.branch_level, status: rel.status, from: $at }]->(peer)
281
305
  )
282
- )
283
- FOREACH ( rel IN $rels_out |
306
+ }
307
+ """ % {"rel_prop": rel_prop_str}
308
+
309
+ rels_out_query = """
310
+ WITH distinct n
311
+ UNWIND $rels_out AS rel_out
312
+ CALL {
313
+ WITH n, rel_out
314
+ WITH n, rel_out as rel
284
315
  MERGE (d:Node { uuid: rel.destination_id })
285
316
  CREATE (rl:Relationship { uuid: rel.uuid, name: rel.name, branch_support: rel.branch_support })
286
317
  CREATE (n)-[:IS_RELATED %(rel_prop)s ]->(rl)
@@ -297,8 +328,15 @@ class NodeCreateAllQuery(NodeQuery):
297
328
  MERGE (peer:Node { uuid: prop.peer_id })
298
329
  CREATE (rl)-[:HAS_OWNER { branch: rel.branch, branch_level: rel.branch_level, status: rel.status, from: $at }]->(peer)
299
330
  )
300
- )
301
- FOREACH ( rel IN $rels_in |
331
+ }
332
+ """ % {"rel_prop": rel_prop_str}
333
+
334
+ rels_in_query = """
335
+ WITH distinct n
336
+ UNWIND $rels_in AS rel_in
337
+ CALL {
338
+ WITH n, rel_in
339
+ WITH n, rel_in AS rel
302
340
  MERGE (d:Node { uuid: rel.destination_id })
303
341
  CREATE (rl:Relationship { uuid: rel.uuid, name: rel.name, branch_support: rel.branch_support })
304
342
  CREATE (n)<-[:IS_RELATED %(rel_prop)s ]-(rl)
@@ -315,14 +353,23 @@ class NodeCreateAllQuery(NodeQuery):
315
353
  MERGE (peer:Node { uuid: prop.peer_id })
316
354
  CREATE (rl)-[:HAS_OWNER { branch: rel.branch, branch_level: rel.branch_level, status: rel.status, from: $at }]->(peer)
317
355
  )
318
- )
356
+ }
357
+ """ % {"rel_prop": rel_prop_str}
358
+
359
+ query = f"""
360
+ MATCH (root:Root)
361
+ CREATE (n:Node:%(labels)s $node_prop )
362
+ CREATE (n)-[r:IS_PART_OF $node_branch_prop ]->(root)
363
+ {attrs_query if self.params["attrs"] else ""}
364
+ {attrs_iphost_query if self.params["attrs_iphost"] else ""}
365
+ {attrs_ipnetwork_query if self.params["attrs_ipnetwork"] else ""}
366
+ {rels_bidir_query if self.params["rels_bidir"] else ""}
367
+ {rels_out_query if self.params["rels_out"] else ""}
368
+ {rels_in_query if self.params["rels_in"] else ""}
319
369
  WITH distinct n
320
370
  MATCH (n)-[:HAS_ATTRIBUTE|IS_RELATED]-(rn)-[:HAS_VALUE|IS_RELATED]-(rv)
321
371
  """ % {
322
372
  "labels": ":".join(self.node.get_labels()),
323
- "rel_prop": rel_prop_str,
324
- "iphost_prop": ", ".join(iphost_prop_list),
325
- "ipnetwork_prop": ", ".join(ipnetwork_prop_list),
326
373
  }
327
374
 
328
375
  self.params["at"] = at.to_string()
@@ -619,18 +666,21 @@ class NodeListGetRelationshipsQuery(Query):
619
666
  MATCH paths_in = ((n)<-[r1:IS_RELATED]-(rel:Relationship)<-[r2:IS_RELATED]-(peer))
620
667
  WHERE ($relationship_identifiers IS NULL OR rel.name in $relationship_identifiers)
621
668
  AND all(r IN relationships(paths_in) WHERE (%(filters)s))
669
+ AND n.uuid <> peer.uuid
622
670
  RETURN n, rel, peer, r1, r2, "inbound" as direction
623
671
  UNION
624
672
  MATCH (n:Node) WHERE n.uuid IN $ids
625
673
  MATCH paths_out = ((n)-[r1:IS_RELATED]->(rel:Relationship)-[r2:IS_RELATED]->(peer))
626
674
  WHERE ($relationship_identifiers IS NULL OR rel.name in $relationship_identifiers)
627
675
  AND all(r IN relationships(paths_out) WHERE (%(filters)s))
676
+ AND n.uuid <> peer.uuid
628
677
  RETURN n, rel, peer, r1, r2, "outbound" as direction
629
678
  UNION
630
679
  MATCH (n:Node) WHERE n.uuid IN $ids
631
680
  MATCH paths_bidir = ((n)-[r1:IS_RELATED]->(rel:Relationship)<-[r2:IS_RELATED]-(peer))
632
681
  WHERE ($relationship_identifiers IS NULL OR rel.name in $relationship_identifiers)
633
682
  AND all(r IN relationships(paths_bidir) WHERE (%(filters)s))
683
+ AND n.uuid <> peer.uuid
634
684
  RETURN n, rel, peer, r1, r2, "bidirectional" as direction
635
685
  """ % {"filters": rels_filter}
636
686
 
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Generator, Optional, Union
8
8
  from infrahub_sdk.uuidt import UUIDT
9
9
 
10
10
  from infrahub.core.constants import RelationshipDirection, RelationshipStatus
11
+ from infrahub.core.constants.database import DatabaseEdgeType
11
12
  from infrahub.core.query import Query, QueryType
12
13
  from infrahub.core.query.subquery import build_subquery_filter, build_subquery_order
13
14
  from infrahub.core.timestamp import Timestamp
@@ -24,7 +25,7 @@ if TYPE_CHECKING:
24
25
  from infrahub.core.schema import RelationshipSchema
25
26
  from infrahub.database import InfrahubDatabase
26
27
 
27
- # pylint: disable=redefined-builtin
28
+ # pylint: disable=redefined-builtin,too-many-lines
28
29
 
29
30
 
30
31
  @dataclass
@@ -583,6 +584,7 @@ class RelationshipGetPeerQuery(Query):
583
584
  query = """
584
585
  MATCH (source_node:Node)%(arrow_left_start)s[:IS_RELATED]%(arrow_left_end)s(rl:Relationship { name: $rel_identifier })
585
586
  WHERE source_node.uuid IN $source_ids
587
+ WITH DISTINCT source_node, rl
586
588
  CALL {
587
589
  WITH rl, source_node
588
590
  MATCH path = (source_node)%(path)s(peer:Node)
@@ -659,10 +661,23 @@ class RelationshipGetPeerQuery(Query):
659
661
  # QUERY Properties
660
662
  # ----------------------------------------------------------------------------
661
663
  query = """
662
- MATCH (rl)-[rel_is_visible:IS_VISIBLE]-(is_visible)
663
- MATCH (rl)-[rel_is_protected:IS_PROTECTED]-(is_protected)
664
- WHERE all(r IN [ rel_is_visible, rel_is_protected] WHERE (%s))
665
- """ % (branch_filter,)
664
+ CALL {
665
+ WITH rl
666
+ MATCH (rl)-[r:IS_VISIBLE]-(is_visible)
667
+ WHERE %(branch_filter)s
668
+ RETURN r AS rel_is_visible, is_visible
669
+ ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
670
+ LIMIT 1
671
+ }
672
+ CALL {
673
+ WITH rl
674
+ MATCH (rl)-[r:IS_PROTECTED]-(is_protected)
675
+ WHERE %(branch_filter)s
676
+ RETURN r AS rel_is_protected, is_protected
677
+ ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
678
+ LIMIT 1
679
+ }
680
+ """ % {"branch_filter": branch_filter}
666
681
 
667
682
  self.add_to_query(query)
668
683
 
@@ -672,20 +687,24 @@ class RelationshipGetPeerQuery(Query):
672
687
  # We must query them one by one otherwise the second one won't return
673
688
  for node_prop in ["source", "owner"]:
674
689
  query = """
675
- WITH %s
676
- OPTIONAL MATCH (rl)-[rel_%s:HAS_%s]-(%s)
677
- WHERE all(r IN [ rel_%s ] WHERE (%s))
678
- """ % (
679
- ",".join(self.return_labels),
680
- node_prop,
681
- node_prop.upper(),
682
- node_prop,
683
- node_prop,
684
- branch_filter,
685
- )
690
+ CALL {
691
+ WITH rl
692
+ OPTIONAL MATCH (rl)-[r:HAS_%(node_prop_type)s]-(%(node_prop)s)
693
+ WHERE %(branch_filter)s
694
+ RETURN r AS rel_%(node_prop)s, %(node_prop)s
695
+ ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
696
+ LIMIT 1
697
+ }
698
+ """ % {
699
+ "node_prop": node_prop,
700
+ "node_prop_type": node_prop.upper(),
701
+ "branch_filter": branch_filter,
702
+ }
686
703
  self.add_to_query(query)
687
704
  self.update_return_labels([f"rel_{node_prop}", node_prop])
688
705
 
706
+ self.add_to_query("WITH " + ",".join(self.return_labels))
707
+
689
708
  # ----------------------------------------------------------------------------
690
709
  # ORDER Results
691
710
  # ----------------------------------------------------------------------------
@@ -932,3 +951,73 @@ class RelationshipCountPerNodeQuery(Query):
932
951
  data[node_id] = 0
933
952
 
934
953
  return data
954
+
955
+
956
+ class RelationshipDeleteAllQuery(Query):
957
+ """
958
+ Delete all relationships linked to a given node on a given branch at a given time. For every IS_RELATED edge:
959
+ - Set `to` time if an active edge exist on the same branch.
960
+ - Create `deleted` edge.
961
+ - Apply above to every edges linked to any connected Relationship node.
962
+ """
963
+
964
+ name = "node_delete_all_relationships"
965
+ type = QueryType.WRITE
966
+ insert_return = False
967
+
968
+ def __init__(self, node_id: str, **kwargs):
969
+ self.node_id = node_id
970
+ super().__init__(**kwargs)
971
+
972
+ async def query_init(self, db: InfrahubDatabase, **kwargs) -> None:
973
+ self.params["source_id"] = kwargs["node_id"]
974
+ self.params["branch"] = self.branch.name
975
+
976
+ self.params["rel_prop"] = {
977
+ "branch": self.branch.name,
978
+ "branch_level": self.branch.hierarchy_level,
979
+ "status": RelationshipStatus.DELETED.value,
980
+ "from": self.at.to_string(),
981
+ }
982
+
983
+ self.params["at"] = self.at.to_string()
984
+
985
+ active_rel_filter, rel_params = self.branch.get_query_filter_path(
986
+ at=self.at, variable_name="active_edge", branch_agnostic=self.branch_agnostic
987
+ )
988
+ self.params.update(rel_params)
989
+
990
+ query_match_relationships = """
991
+ MATCH (s:Node { uuid: $source_id })-[active_edge:IS_RELATED]-(rl:Relationship)
992
+ WHERE %(active_rel_filter)s AND active_edge.status = "active"
993
+ WITH DISTINCT rl
994
+ """ % {"active_rel_filter": active_rel_filter}
995
+
996
+ self.add_to_query(query_match_relationships)
997
+
998
+ for arrow_left, arrow_right in (("<-", "-"), ("-", "->")):
999
+ for edge_type in [
1000
+ DatabaseEdgeType.IS_RELATED.value,
1001
+ DatabaseEdgeType.IS_VISIBLE.value,
1002
+ DatabaseEdgeType.IS_PROTECTED.value,
1003
+ DatabaseEdgeType.HAS_OWNER.value,
1004
+ DatabaseEdgeType.HAS_SOURCE.value,
1005
+ ]:
1006
+ query = """
1007
+ CALL {
1008
+ WITH rl
1009
+ MATCH (rl)%(arrow_left)s[active_edge:%(edge_type)s]%(arrow_right)s(n)
1010
+ WHERE %(active_rel_filter)s AND active_edge.status ="active"
1011
+ CREATE (rl)%(arrow_left)s[deleted_edge:%(edge_type)s $rel_prop]%(arrow_right)s(n)
1012
+ SET deleted_edge.hierarchy = active_edge.hierarchy
1013
+ WITH active_edge
1014
+ WHERE active_edge.branch = $branch AND active_edge.to IS NULL
1015
+ SET active_edge.to = $at
1016
+ }
1017
+ """ % {
1018
+ "arrow_left": arrow_left,
1019
+ "arrow_right": arrow_right,
1020
+ "active_rel_filter": active_rel_filter,
1021
+ "edge_type": edge_type,
1022
+ }
1023
+ self.add_to_query(query)
@@ -276,6 +276,8 @@ class NumberPoolSetReserved(Query):
276
276
  query = """
277
277
  MATCH (pool:%(number_pool)s { uuid: $pool_id })
278
278
  MERGE (value:AttributeValue { value: $reserved, is_default: false })
279
+ WITH value, pool
280
+ LIMIT 1
279
281
  CREATE (pool)-[rel:IS_RESERVED $rel_prop]->(value)
280
282
  """ % {"number_pool": InfrahubKind.NUMBERPOOL}
281
283
 
@@ -884,6 +884,8 @@ class RelationshipManager:
884
884
  """If the attribute is branch aware, return the Branch object associated with this attribute
885
885
  If the attribute is branch agnostic return the Global Branch
886
886
 
887
+ Note that if this relationship is Aware and source node is Agnostic, it will return -global- branch.
888
+
887
889
  Returns:
888
890
  Branch:
889
891
  """
@@ -959,7 +961,7 @@ class RelationshipManager:
959
961
  self.has_fetched_relationships = True
960
962
 
961
963
  for peer_id in details.peer_ids_present_local_only:
962
- await self.remove(peer_id=peer_id, db=db)
964
+ await self.remove_locally(peer_id=peer_id, db=db)
963
965
 
964
966
  async def get(self, db: InfrahubDatabase) -> Relationship | list[Relationship] | None:
965
967
  rels = await self.get_relationships(db=db)
@@ -1077,22 +1079,17 @@ class RelationshipManager:
1077
1079
  for rel in self._relationships:
1078
1080
  await rel.resolve(db=db)
1079
1081
 
1080
- async def remove(
1082
+ async def remove_locally(
1081
1083
  self,
1082
1084
  peer_id: Union[str, UUID],
1083
1085
  db: InfrahubDatabase,
1084
- update_db: bool = False,
1085
1086
  ) -> bool:
1086
- """Remove a peer id from the local relationships list,
1087
- need to investigate if and when we should update the relationship in the database."""
1087
+ """Remove a peer id from the local relationships list"""
1088
1088
 
1089
1089
  for idx, rel in enumerate(await self.get_relationships(db=db)):
1090
1090
  if str(rel.peer_id) != str(peer_id):
1091
1091
  continue
1092
1092
 
1093
- if update_db:
1094
- await rel.delete(db=db)
1095
-
1096
1093
  self._relationships.pop(idx)
1097
1094
  return True
1098
1095
 
@@ -1109,14 +1106,13 @@ class RelationshipManager:
1109
1106
 
1110
1107
  # - Update the existing relationship if we are on the same branch
1111
1108
  rel_ids_per_branch = peer_data.rel_ids_per_branch()
1109
+
1110
+ # In which cases do we end up here and do not want to set `to` time?
1112
1111
  if branch.name in rel_ids_per_branch:
1113
1112
  await update_relationships_to([str(ri) for ri in rel_ids_per_branch[branch.name]], to=remove_at, db=db)
1114
1113
 
1115
1114
  # - Create a new rel of type DELETED if the existing relationship is on a different branch
1116
- rel_branches: set[str] = set()
1117
- if peer_data.rels:
1118
- rel_branches = {r.branch for r in peer_data.rels}
1119
- if rel_branches == {peer_data.branch}:
1115
+ if peer_data.rels and {r.branch for r in peer_data.rels} == {peer_data.branch}:
1120
1116
  return
1121
1117
 
1122
1118
  query = await RelationshipDataDeleteQuery.init(
@@ -223,6 +223,7 @@ core_models: dict[str, Any] = {
223
223
  "optional": True,
224
224
  "identifier": "group_member",
225
225
  "cardinality": "many",
226
+ "branch": BranchSupportType.AWARE,
226
227
  },
227
228
  {
228
229
  "name": "subscribers",
@@ -34,7 +34,7 @@ from infrahub.utils import InfrahubStringEnum
34
34
 
35
35
  from .constants import DatabaseType, Neo4jRuntime
36
36
  from .memgraph import DatabaseManagerMemgraph
37
- from .metrics import QUERY_EXECUTION_METRICS, TRANSACTION_RETRIES
37
+ from .metrics import CONNECTION_POOL_USAGE, QUERY_EXECUTION_METRICS, TRANSACTION_RETRIES
38
38
  from .neo4j import DatabaseManagerNeo4j
39
39
 
40
40
  if TYPE_CHECKING:
@@ -335,6 +335,14 @@ class InfrahubDatabase:
335
335
  context: dict[str, str] | None = None,
336
336
  type: QueryType | None = None, # pylint: disable=redefined-builtin
337
337
  ) -> tuple[list[Record], dict[str, Any]]:
338
+ connpool_usage = self._driver._pool.in_use_connection_count(self._driver._pool.address)
339
+ CONNECTION_POOL_USAGE.labels(self._driver._pool.address).set(float(connpool_usage))
340
+
341
+ if config.SETTINGS.database.max_concurrent_queries:
342
+ while connpool_usage > config.SETTINGS.database.max_concurrent_queries: # noqa: ASYNC110
343
+ await asyncio.sleep(config.SETTINGS.database.max_concurrent_queries_delay)
344
+ connpool_usage = self._driver._pool.in_use_connection_count(self._driver._pool.address)
345
+
338
346
  with trace.get_tracer(__name__).start_as_current_span("execute_db_query_with_metadata") as span:
339
347
  span.set_attribute("query", query)
340
348
  if name:
@@ -514,6 +522,7 @@ def retry_db_transaction(
514
522
  if exc.code != "Neo.ClientError.Statement.EntityNotFound":
515
523
  raise exc
516
524
  retry_time: float = random.randrange(100, 500) / 1000
525
+ log.exception("Retry handler caught database error")
517
526
  log.info(
518
527
  f"Retrying database transaction, attempt {attempt}/{config.SETTINGS.database.retry_limit}",
519
528
  retry_time=retry_time,
@@ -1,6 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- from prometheus_client import Counter, Histogram
3
+ from prometheus_client import Counter, Gauge, Histogram
4
4
 
5
5
  METRIC_PREFIX = "infrahub_db"
6
6
 
@@ -16,3 +16,9 @@ TRANSACTION_RETRIES = Counter(
16
16
  "Number of transaction that have been retried due to transcient error",
17
17
  labelnames=["name"],
18
18
  )
19
+
20
+ CONNECTION_POOL_USAGE = Gauge(
21
+ f"{METRIC_PREFIX}_last_connection_pool_usage",
22
+ "Number of last known active connections in the pool",
23
+ labelnames=["address"],
24
+ )
@@ -1,8 +1,10 @@
1
1
  from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer
2
2
  from infrahub.dependencies.interface import DependencyBuilder, DependencyBuilderContext
3
3
 
4
+ from .parent_node_adder import DiffParentNodeAdderDependency
5
+
4
6
 
5
7
  class DiffDeserializerDependency(DependencyBuilder[EnrichedDiffDeserializer]):
6
8
  @classmethod
7
9
  def build(cls, context: DependencyBuilderContext) -> EnrichedDiffDeserializer:
8
- return EnrichedDiffDeserializer()
10
+ return EnrichedDiffDeserializer(parent_adder=DiffParentNodeAdderDependency.build(context=context))
@@ -1,8 +1,10 @@
1
1
  from infrahub.core.diff.enricher.hierarchy import DiffHierarchyEnricher
2
2
  from infrahub.dependencies.interface import DependencyBuilder, DependencyBuilderContext
3
3
 
4
+ from ..parent_node_adder import DiffParentNodeAdderDependency
5
+
4
6
 
5
7
  class DiffHierarchyEnricherDependency(DependencyBuilder[DiffHierarchyEnricher]):
6
8
  @classmethod
7
9
  def build(cls, context: DependencyBuilderContext) -> DiffHierarchyEnricher:
8
- return DiffHierarchyEnricher(db=context.db)
10
+ return DiffHierarchyEnricher(db=context.db, parent_adder=DiffParentNodeAdderDependency.build(context=context))
@@ -0,0 +1,8 @@
1
+ from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder
2
+ from infrahub.dependencies.interface import DependencyBuilder, DependencyBuilderContext
3
+
4
+
5
+ class DiffParentNodeAdderDependency(DependencyBuilder[DiffParentNodeAdder]):
6
+ @classmethod
7
+ def build(cls, context: DependencyBuilderContext) -> DiffParentNodeAdder:
8
+ return DiffParentNodeAdder()
@@ -8,6 +8,7 @@ from starlette.background import BackgroundTasks
8
8
  from infrahub.core import registry
9
9
  from infrahub.core.timestamp import Timestamp
10
10
  from infrahub.exceptions import InitializationError
11
+ from infrahub.graphql.resolvers.many_relationship import ManyRelationshipResolver
11
12
  from infrahub.graphql.resolvers.single_relationship import SingleRelationshipResolver
12
13
  from infrahub.permissions import PermissionManager
13
14
 
@@ -35,6 +36,7 @@ class GraphqlContext:
35
36
  branch: Branch
36
37
  types: dict
37
38
  single_relationship_resolver: SingleRelationshipResolver
39
+ many_relationship_resolver: ManyRelationshipResolver
38
40
  at: Timestamp | None = None
39
41
  related_node_ids: set | None = None
40
42
  service: InfrahubServices | None = None
@@ -107,6 +109,7 @@ async def prepare_graphql_params(
107
109
  db=db,
108
110
  branch=branch,
109
111
  single_relationship_resolver=SingleRelationshipResolver(),
112
+ many_relationship_resolver=ManyRelationshipResolver(),
110
113
  at=Timestamp(at),
111
114
  types=gqlm.get_graphql_types(),
112
115
  related_node_ids=set(),
@@ -10,6 +10,8 @@ from infrahub.core.node import Node
10
10
  from infrahub.core.timestamp import Timestamp
11
11
  from infrahub.database import InfrahubDatabase
12
12
 
13
+ from .shared import to_frozen_set
14
+
13
15
 
14
16
  @dataclass
15
17
  class GetManyParams:
@@ -68,15 +70,3 @@ class NodeDataLoader(DataLoader[str, Node | None]):
68
70
  for node_id in keys:
69
71
  results.append(nodes_by_id.get(node_id, None))
70
72
  return results
71
-
72
-
73
- def to_frozen_set(to_freeze: dict[str, Any]) -> frozenset:
74
- freezing_dict = {}
75
- for k, v in to_freeze.items():
76
- if isinstance(v, dict):
77
- freezing_dict[k] = to_frozen_set(v)
78
- elif isinstance(v, (list, set)):
79
- freezing_dict[k] = frozenset(v)
80
- else:
81
- freezing_dict[k] = v
82
- return frozenset(freezing_dict)
@@ -0,0 +1,77 @@
1
+ from dataclasses import dataclass
2
+ from typing import Any
3
+
4
+ from aiodataloader import DataLoader
5
+
6
+ from infrahub.core.branch.models import Branch
7
+ from infrahub.core.manager import NodeManager
8
+ from infrahub.core.relationship.model import Relationship
9
+ from infrahub.core.schema.relationship_schema import RelationshipSchema
10
+ from infrahub.core.timestamp import Timestamp
11
+ from infrahub.database import InfrahubDatabase
12
+
13
+ from .shared import to_frozen_set
14
+
15
+
16
+ @dataclass
17
+ class QueryPeerParams:
18
+ branch: Branch | str
19
+ source_kind: str
20
+ schema: RelationshipSchema
21
+ filters: dict[str, Any]
22
+ fields: dict | None = None
23
+ at: Timestamp | str | None = None
24
+ branch_agnostic: bool = False
25
+
26
+ def __hash__(self) -> int:
27
+ frozen_fields: frozenset | None = None
28
+ if self.fields:
29
+ frozen_fields = to_frozen_set(self.fields)
30
+ frozen_filters = to_frozen_set(self.filters)
31
+ timestamp = Timestamp(self.at)
32
+ branch = self.branch.name if isinstance(self.branch, Branch) else self.branch
33
+ hash_str = "|".join(
34
+ [
35
+ str(hash(frozen_fields)),
36
+ str(hash(frozen_filters)),
37
+ timestamp.to_string(),
38
+ branch,
39
+ self.schema.name,
40
+ str(self.source_kind),
41
+ str(self.branch_agnostic),
42
+ ]
43
+ )
44
+ return hash(hash_str)
45
+
46
+
47
+ class PeerRelationshipsDataLoader(DataLoader[str, list[Relationship]]):
48
+ def __init__(self, db: InfrahubDatabase, query_params: QueryPeerParams, *args: Any, **kwargs: Any) -> None:
49
+ super().__init__(*args, **kwargs)
50
+ self.query_params = query_params
51
+ self.db = db
52
+
53
+ async def batch_load_fn(self, keys: list[Any]) -> list[list[Relationship]]: # pylint: disable=method-hidden
54
+ async with self.db.start_session() as db:
55
+ peer_rels = await NodeManager.query_peers(
56
+ db=db,
57
+ ids=keys,
58
+ source_kind=self.query_params.source_kind,
59
+ schema=self.query_params.schema,
60
+ filters=self.query_params.filters,
61
+ fields=self.query_params.fields,
62
+ at=self.query_params.at,
63
+ branch=self.query_params.branch,
64
+ branch_agnostic=self.query_params.branch_agnostic,
65
+ fetch_peers=True,
66
+ )
67
+ peer_rels_by_node_id: dict[str, list[Relationship]] = {}
68
+ for rel in peer_rels:
69
+ node_id = rel.node_id
70
+ if node_id not in peer_rels_by_node_id:
71
+ peer_rels_by_node_id[node_id] = []
72
+ peer_rels_by_node_id[node_id].append(rel)
73
+
74
+ results = []
75
+ for node_id in keys:
76
+ results.append(peer_rels_by_node_id.get(node_id, []))
77
+ return results
@@ -0,0 +1,13 @@
1
+ from typing import Any
2
+
3
+
4
+ def to_frozen_set(to_freeze: dict[str, Any]) -> frozenset:
5
+ freezing_dict = {}
6
+ for k, v in to_freeze.items():
7
+ if isinstance(v, dict):
8
+ freezing_dict[k] = to_frozen_set(v)
9
+ elif isinstance(v, (list, set)):
10
+ freezing_dict[k] = frozenset(v)
11
+ else:
12
+ freezing_dict[k] = v
13
+ return frozenset(freezing_dict)