infrahub-server 1.2.11__py3-none-any.whl → 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. infrahub/actions/constants.py +130 -0
  2. infrahub/actions/gather.py +114 -0
  3. infrahub/actions/models.py +243 -0
  4. infrahub/actions/parsers.py +104 -0
  5. infrahub/actions/schema.py +393 -0
  6. infrahub/actions/tasks.py +119 -0
  7. infrahub/actions/triggers.py +21 -0
  8. infrahub/branch/__init__.py +0 -0
  9. infrahub/branch/tasks.py +29 -0
  10. infrahub/branch/triggers.py +22 -0
  11. infrahub/cli/db.py +3 -4
  12. infrahub/computed_attribute/gather.py +3 -1
  13. infrahub/computed_attribute/tasks.py +23 -29
  14. infrahub/core/account.py +24 -47
  15. infrahub/core/attribute.py +13 -15
  16. infrahub/core/constants/__init__.py +10 -0
  17. infrahub/core/constants/database.py +1 -0
  18. infrahub/core/constants/infrahubkind.py +9 -0
  19. infrahub/core/constraint/node/runner.py +3 -1
  20. infrahub/core/convert_object_type/__init__.py +0 -0
  21. infrahub/core/convert_object_type/conversion.py +124 -0
  22. infrahub/core/convert_object_type/schema_mapping.py +56 -0
  23. infrahub/core/diff/coordinator.py +8 -1
  24. infrahub/core/diff/query/all_conflicts.py +1 -5
  25. infrahub/core/diff/query/artifact.py +10 -20
  26. infrahub/core/diff/query/delete_query.py +8 -4
  27. infrahub/core/diff/query/diff_get.py +3 -6
  28. infrahub/core/diff/query/field_specifiers.py +1 -1
  29. infrahub/core/diff/query/field_summary.py +2 -4
  30. infrahub/core/diff/query/merge.py +72 -125
  31. infrahub/core/diff/query/save.py +83 -68
  32. infrahub/core/diff/query/summary_counts_enricher.py +34 -54
  33. infrahub/core/diff/query/time_range_query.py +0 -1
  34. infrahub/core/diff/repository/repository.py +4 -0
  35. infrahub/core/graph/__init__.py +1 -1
  36. infrahub/core/manager.py +14 -11
  37. infrahub/core/migrations/graph/__init__.py +6 -0
  38. infrahub/core/migrations/graph/m003_relationship_parent_optional.py +1 -2
  39. infrahub/core/migrations/graph/m012_convert_account_generic.py +1 -1
  40. infrahub/core/migrations/graph/m013_convert_git_password_credential.py +2 -6
  41. infrahub/core/migrations/graph/m015_diff_format_update.py +1 -2
  42. infrahub/core/migrations/graph/m016_diff_delete_bug_fix.py +1 -2
  43. infrahub/core/migrations/graph/m019_restore_rels_to_time.py +11 -22
  44. infrahub/core/migrations/graph/m020_duplicate_edges.py +3 -6
  45. infrahub/core/migrations/graph/m021_missing_hierarchy_merge.py +1 -2
  46. infrahub/core/migrations/graph/m023_deduplicate_cardinality_one_relationships.py +2 -2
  47. infrahub/core/migrations/graph/m024_missing_hierarchy_backfill.py +1 -2
  48. infrahub/core/migrations/graph/m028_delete_diffs.py +1 -2
  49. infrahub/core/migrations/graph/m029_duplicates_cleanup.py +662 -0
  50. infrahub/core/migrations/graph/m030_illegal_edges.py +82 -0
  51. infrahub/core/migrations/query/attribute_add.py +14 -11
  52. infrahub/core/migrations/query/attribute_rename.py +6 -11
  53. infrahub/core/migrations/query/delete_element_in_schema.py +19 -17
  54. infrahub/core/migrations/query/node_duplicate.py +19 -21
  55. infrahub/core/migrations/query/relationship_duplicate.py +19 -18
  56. infrahub/core/migrations/schema/node_attribute_remove.py +4 -8
  57. infrahub/core/migrations/schema/node_remove.py +19 -20
  58. infrahub/core/models.py +29 -2
  59. infrahub/core/node/__init__.py +131 -28
  60. infrahub/core/node/base.py +1 -1
  61. infrahub/core/node/create.py +211 -0
  62. infrahub/core/node/resource_manager/number_pool.py +31 -5
  63. infrahub/core/node/standard.py +6 -1
  64. infrahub/core/path.py +15 -1
  65. infrahub/core/protocols.py +57 -0
  66. infrahub/core/protocols_base.py +3 -0
  67. infrahub/core/query/__init__.py +2 -2
  68. infrahub/core/query/delete.py +3 -3
  69. infrahub/core/query/diff.py +19 -32
  70. infrahub/core/query/ipam.py +10 -20
  71. infrahub/core/query/node.py +29 -47
  72. infrahub/core/query/relationship.py +55 -34
  73. infrahub/core/query/resource_manager.py +1 -2
  74. infrahub/core/query/standard_node.py +19 -5
  75. infrahub/core/query/subquery.py +2 -4
  76. infrahub/core/relationship/constraints/count.py +10 -9
  77. infrahub/core/relationship/constraints/interface.py +2 -1
  78. infrahub/core/relationship/constraints/peer_kind.py +2 -1
  79. infrahub/core/relationship/constraints/peer_parent.py +56 -0
  80. infrahub/core/relationship/constraints/peer_relatives.py +72 -0
  81. infrahub/core/relationship/constraints/profiles_kind.py +1 -1
  82. infrahub/core/relationship/model.py +4 -1
  83. infrahub/core/schema/__init__.py +2 -1
  84. infrahub/core/schema/attribute_parameters.py +160 -0
  85. infrahub/core/schema/attribute_schema.py +130 -7
  86. infrahub/core/schema/basenode_schema.py +27 -3
  87. infrahub/core/schema/definitions/core/__init__.py +29 -1
  88. infrahub/core/schema/definitions/core/group.py +45 -0
  89. infrahub/core/schema/definitions/core/resource_pool.py +9 -0
  90. infrahub/core/schema/definitions/internal.py +43 -5
  91. infrahub/core/schema/generated/attribute_schema.py +16 -3
  92. infrahub/core/schema/generated/relationship_schema.py +11 -1
  93. infrahub/core/schema/manager.py +7 -2
  94. infrahub/core/schema/schema_branch.py +109 -12
  95. infrahub/core/validators/__init__.py +15 -2
  96. infrahub/core/validators/attribute/choices.py +1 -3
  97. infrahub/core/validators/attribute/enum.py +1 -3
  98. infrahub/core/validators/attribute/kind.py +1 -3
  99. infrahub/core/validators/attribute/length.py +13 -7
  100. infrahub/core/validators/attribute/min_max.py +118 -0
  101. infrahub/core/validators/attribute/number_pool.py +106 -0
  102. infrahub/core/validators/attribute/optional.py +1 -4
  103. infrahub/core/validators/attribute/regex.py +5 -6
  104. infrahub/core/validators/attribute/unique.py +1 -3
  105. infrahub/core/validators/determiner.py +18 -2
  106. infrahub/core/validators/enum.py +12 -0
  107. infrahub/core/validators/node/hierarchy.py +3 -6
  108. infrahub/core/validators/query.py +1 -3
  109. infrahub/core/validators/relationship/count.py +6 -12
  110. infrahub/core/validators/relationship/optional.py +2 -4
  111. infrahub/core/validators/relationship/peer.py +177 -12
  112. infrahub/core/validators/tasks.py +1 -1
  113. infrahub/core/validators/uniqueness/query.py +5 -9
  114. infrahub/database/__init__.py +12 -4
  115. infrahub/database/validation.py +100 -0
  116. infrahub/dependencies/builder/constraint/grouped/node_runner.py +4 -0
  117. infrahub/dependencies/builder/constraint/relationship_manager/peer_parent.py +8 -0
  118. infrahub/dependencies/builder/constraint/relationship_manager/peer_relatives.py +8 -0
  119. infrahub/dependencies/builder/constraint/schema/aggregated.py +2 -0
  120. infrahub/dependencies/builder/constraint/schema/relationship_peer.py +8 -0
  121. infrahub/dependencies/builder/diff/deserializer.py +1 -1
  122. infrahub/dependencies/registry.py +4 -0
  123. infrahub/events/group_action.py +1 -0
  124. infrahub/events/models.py +1 -1
  125. infrahub/git/base.py +5 -3
  126. infrahub/git/integrator.py +96 -5
  127. infrahub/git/tasks.py +1 -0
  128. infrahub/graphql/analyzer.py +139 -18
  129. infrahub/graphql/manager.py +4 -0
  130. infrahub/graphql/mutations/action.py +164 -0
  131. infrahub/graphql/mutations/convert_object_type.py +71 -0
  132. infrahub/graphql/mutations/main.py +25 -176
  133. infrahub/graphql/mutations/proposed_change.py +20 -17
  134. infrahub/graphql/mutations/relationship.py +32 -0
  135. infrahub/graphql/mutations/resource_manager.py +63 -7
  136. infrahub/graphql/queries/convert_object_type_mapping.py +34 -0
  137. infrahub/graphql/queries/resource_manager.py +7 -1
  138. infrahub/graphql/resolvers/many_relationship.py +1 -1
  139. infrahub/graphql/resolvers/resolver.py +2 -2
  140. infrahub/graphql/resolvers/single_relationship.py +1 -1
  141. infrahub/graphql/schema.py +6 -0
  142. infrahub/menu/menu.py +34 -2
  143. infrahub/message_bus/messages/__init__.py +0 -10
  144. infrahub/message_bus/operations/__init__.py +0 -8
  145. infrahub/message_bus/operations/refresh/registry.py +4 -7
  146. infrahub/patch/queries/delete_duplicated_edges.py +45 -39
  147. infrahub/pools/models.py +14 -0
  148. infrahub/pools/number.py +5 -3
  149. infrahub/pools/registration.py +22 -0
  150. infrahub/pools/tasks.py +126 -0
  151. infrahub/prefect_server/models.py +1 -19
  152. infrahub/proposed_change/models.py +68 -3
  153. infrahub/proposed_change/tasks.py +911 -34
  154. infrahub/schema/__init__.py +0 -0
  155. infrahub/schema/tasks.py +27 -0
  156. infrahub/schema/triggers.py +23 -0
  157. infrahub/task_manager/models.py +10 -6
  158. infrahub/trigger/catalogue.py +6 -0
  159. infrahub/trigger/models.py +23 -6
  160. infrahub/trigger/setup.py +26 -2
  161. infrahub/trigger/tasks.py +4 -2
  162. infrahub/types.py +6 -0
  163. infrahub/webhook/tasks.py +6 -9
  164. infrahub/workflows/catalogue.py +103 -1
  165. infrahub_sdk/client.py +43 -10
  166. infrahub_sdk/ctl/generator.py +4 -4
  167. infrahub_sdk/ctl/repository.py +1 -1
  168. infrahub_sdk/node/__init__.py +39 -0
  169. infrahub_sdk/node/attribute.py +122 -0
  170. infrahub_sdk/node/constants.py +21 -0
  171. infrahub_sdk/{node.py → node/node.py} +158 -803
  172. infrahub_sdk/node/parsers.py +15 -0
  173. infrahub_sdk/node/property.py +24 -0
  174. infrahub_sdk/node/related_node.py +266 -0
  175. infrahub_sdk/node/relationship.py +302 -0
  176. infrahub_sdk/protocols.py +112 -0
  177. infrahub_sdk/protocols_base.py +34 -2
  178. infrahub_sdk/pytest_plugin/items/python_transform.py +2 -1
  179. infrahub_sdk/query_groups.py +17 -5
  180. infrahub_sdk/schema/main.py +1 -0
  181. infrahub_sdk/schema/repository.py +16 -0
  182. infrahub_sdk/spec/object.py +1 -1
  183. infrahub_sdk/store.py +1 -1
  184. infrahub_sdk/testing/schemas/car_person.py +1 -0
  185. infrahub_sdk/utils.py +7 -20
  186. infrahub_sdk/yaml.py +6 -5
  187. {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/METADATA +5 -5
  188. {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/RECORD +197 -168
  189. {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/WHEEL +1 -1
  190. infrahub_testcontainers/container.py +239 -65
  191. infrahub_testcontainers/docker-compose-cluster.test.yml +321 -0
  192. infrahub_testcontainers/docker-compose.test.yml +2 -1
  193. infrahub_testcontainers/helpers.py +23 -3
  194. infrahub_testcontainers/plugin.py +9 -0
  195. infrahub/message_bus/messages/check_generator_run.py +0 -26
  196. infrahub/message_bus/messages/finalize_validator_execution.py +0 -15
  197. infrahub/message_bus/messages/proposed_change/base_with_diff.py +0 -16
  198. infrahub/message_bus/messages/proposed_change/request_proposedchange_refreshartifacts.py +0 -11
  199. infrahub/message_bus/messages/request_generatordefinition_check.py +0 -20
  200. infrahub/message_bus/messages/request_proposedchange_pipeline.py +0 -23
  201. infrahub/message_bus/operations/check/__init__.py +0 -3
  202. infrahub/message_bus/operations/check/generator.py +0 -156
  203. infrahub/message_bus/operations/finalize/__init__.py +0 -3
  204. infrahub/message_bus/operations/finalize/validator.py +0 -133
  205. infrahub/message_bus/operations/requests/__init__.py +0 -9
  206. infrahub/message_bus/operations/requests/generator_definition.py +0 -140
  207. infrahub/message_bus/operations/requests/proposed_change.py +0 -629
  208. infrahub/patch/queries/consolidate_duplicated_nodes.py +0 -109
  209. /infrahub/{message_bus/messages/proposed_change → actions}/__init__.py +0 -0
  210. {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/LICENSE.txt +0 -0
  211. {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/entry_points.txt +0 -0
@@ -1,28 +1,18 @@
1
1
  from infrahub.message_bus import InfrahubMessage, InfrahubResponse
2
2
 
3
- from .check_generator_run import CheckGeneratorRun
4
- from .finalize_validator_execution import FinalizeValidatorExecution
5
3
  from .git_file_get import GitFileGet, GitFileGetResponse
6
4
  from .git_repository_connectivity import GitRepositoryConnectivity
7
- from .proposed_change.request_proposedchange_refreshartifacts import RequestProposedChangeRefreshArtifacts
8
5
  from .refresh_git_fetch import RefreshGitFetch
9
6
  from .refresh_registry_branches import RefreshRegistryBranches
10
7
  from .refresh_registry_rebasedbranch import RefreshRegistryRebasedBranch
11
- from .request_generatordefinition_check import RequestGeneratorDefinitionCheck
12
- from .request_proposedchange_pipeline import RequestProposedChangePipeline
13
8
  from .send_echo_request import SendEchoRequest, SendEchoRequestResponse
14
9
 
15
10
  MESSAGE_MAP: dict[str, type[InfrahubMessage]] = {
16
- "check.generator.run": CheckGeneratorRun,
17
- "finalize.validator.execution": FinalizeValidatorExecution,
18
11
  "git.file.get": GitFileGet,
19
12
  "git.repository.connectivity": GitRepositoryConnectivity,
20
13
  "refresh.git.fetch": RefreshGitFetch,
21
14
  "refresh.registry.branches": RefreshRegistryBranches,
22
15
  "refresh.registry.rebased_branch": RefreshRegistryRebasedBranch,
23
- "request.generator_definition.check": RequestGeneratorDefinitionCheck,
24
- "request.proposed_change.pipeline": RequestProposedChangePipeline,
25
- "request.proposed_change.refresh_artifacts": RequestProposedChangeRefreshArtifacts,
26
16
  "send.echo.request": SendEchoRequest,
27
17
  }
28
18
 
@@ -3,11 +3,8 @@ from prefect import Flow
3
3
 
4
4
  from infrahub.message_bus import RPCErrorResponse, messages
5
5
  from infrahub.message_bus.operations import (
6
- check,
7
- finalize,
8
6
  git,
9
7
  refresh,
10
- requests,
11
8
  send,
12
9
  )
13
10
  from infrahub.message_bus.types import MessageTTL
@@ -15,16 +12,11 @@ from infrahub.services import InfrahubServices
15
12
  from infrahub.tasks.check import set_check_status
16
13
 
17
14
  COMMAND_MAP = {
18
- "check.generator.run": check.generator.run,
19
- "finalize.validator.execution": finalize.validator.execution,
20
15
  "git.file.get": git.file.get,
21
16
  "git.repository.connectivity": git.repository.connectivity,
22
17
  "refresh.git.fetch": git.repository.fetch,
23
18
  "refresh.registry.branches": refresh.registry.branches,
24
19
  "refresh.registry.rebased_branch": refresh.registry.rebased_branch,
25
- "request.generator_definition.check": requests.generator_definition.check,
26
- "request.proposed_change.pipeline": requests.proposed_change.pipeline,
27
- "request.proposed_change.refresh_artifacts": requests.proposed_change.refresh_artifacts,
28
20
  "send.echo.request": send.echo.request,
29
21
  }
30
22
 
@@ -1,5 +1,3 @@
1
- from infrahub import lock
2
- from infrahub.core.registry import registry
3
1
  from infrahub.message_bus import messages
4
2
  from infrahub.services import InfrahubServices
5
3
  from infrahub.tasks.registry import refresh_branches
@@ -24,8 +22,7 @@ async def rebased_branch(message: messages.RefreshRegistryRebasedBranch, service
24
22
  )
25
23
  return
26
24
 
27
- async with lock.registry.local_schema_lock():
28
- service.log.info("Refreshing rebased branch")
29
- registry.branch[message.branch] = await registry.branch_object.get_by_name(
30
- name=message.branch, db=service.database
31
- )
25
+ async with service.database.start_session(read_only=True) as db:
26
+ await refresh_branches(db=db)
27
+
28
+ await service.component.refresh_schema_hash()
@@ -4,7 +4,7 @@ from .base import PatchQuery
4
4
 
5
5
  class DeleteDuplicatedEdgesPatchQuery(PatchQuery):
6
6
  """
7
- Find duplicated or overlapping edges of the same status, type, and branch to update and delete
7
+ For all Node vertices, find duplicated or overlapping edges of the same status, type, and branch to update and delete
8
8
  - one edge will be kept for each pair of nodes and a given status, type, and branch. it will be
9
9
  updated to have the earliest "from" and "to" times in this group
10
10
  - all the other duplicate/overlapping edges will be deleted
@@ -17,99 +17,105 @@ class DeleteDuplicatedEdgesPatchQuery(PatchQuery):
17
17
  async def plan(self) -> PatchPlan:
18
18
  query = """
19
19
  // ------------
20
- // Find node pairs that have duplicate edges
20
+ // Find vertex pairs that have duplicate edges
21
21
  // ------------
22
- MATCH (node_with_dup_edges:Node)-[edge]->(peer)
22
+ MATCH (node_with_dup_edges:Node)-[edge]-(peer)
23
23
  WITH node_with_dup_edges, type(edge) AS edge_type, edge.status AS edge_status, edge.branch AS edge_branch, peer, count(*) AS num_dup_edges
24
24
  WHERE num_dup_edges > 1
25
25
  WITH DISTINCT node_with_dup_edges, edge_type, edge_branch, peer
26
- CALL {
26
+ CALL (node_with_dup_edges, edge_type, edge_branch, peer) {
27
27
  // ------------
28
28
  // Get the earliest active and deleted edges for this branch
29
29
  // ------------
30
- WITH node_with_dup_edges, edge_type, edge_branch, peer
31
- MATCH (node_with_dup_edges)-[active_edge {branch: edge_branch, status: "active"}]->(peer)
30
+ OPTIONAL MATCH (node_with_dup_edges)-[active_edge {branch: edge_branch, status: "active"}]->(peer)
32
31
  WHERE type(active_edge) = edge_type
33
32
  WITH node_with_dup_edges, edge_type, edge_branch, peer, active_edge
34
33
  ORDER BY active_edge.from ASC
35
34
  WITH node_with_dup_edges, edge_type, edge_branch, peer, head(collect(active_edge.from)) AS active_from
36
- OPTIONAL MATCH (node_with_dup_edges)-[deleted_edge {branch: edge_branch, status: "deleted"}]->(peer)
35
+ OPTIONAL MATCH (node_with_dup_edges)-[deleted_edge {branch: edge_branch, status: "deleted"}]-(peer)
37
36
  WITH node_with_dup_edges, edge_type, edge_branch, peer, active_from, deleted_edge
38
37
  ORDER BY deleted_edge.from ASC
39
38
  WITH node_with_dup_edges, edge_type, edge_branch, peer, active_from, head(collect(deleted_edge.from)) AS deleted_from
40
39
  // ------------
41
40
  // Plan one active edge update with correct from and to times
42
41
  // ------------
43
- CALL {
44
- WITH node_with_dup_edges, edge_type, edge_branch, peer, active_from, deleted_from
45
- MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
42
+ CALL (node_with_dup_edges, edge_type, edge_branch, peer, active_from, deleted_from) {
43
+ OPTIONAL MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
46
44
  WHERE type(active_e) = edge_type
47
45
  WITH node_with_dup_edges, edge_type, edge_branch, peer, active_from, deleted_from, active_e
48
46
  ORDER BY %(id_func_name)s(active_e)
49
47
  LIMIT 1
50
48
  WITH active_e, properties(active_e) AS before_props, {from: active_from, to: deleted_from} AS prop_updates
51
- RETURN [
52
- {
53
- db_id: %(id_func_name)s(active_e), before_props: before_props, prop_updates: prop_updates
54
- }
55
- ] AS active_edges_to_update
49
+ RETURN CASE
50
+ WHEN active_e IS NOT NULL THEN [
51
+ {
52
+ db_id: %(id_func_name)s(active_e), before_props: before_props, prop_updates: prop_updates
53
+ }
54
+ ]
55
+ ELSE []
56
+ END AS active_edges_to_update
56
57
  }
57
58
  // ------------
58
59
  // Plan deletes for all the other active edges of this type on this branch
59
60
  // ------------
60
- CALL {
61
- WITH node_with_dup_edges, edge_type, edge_branch, peer
62
- MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
61
+ CALL (node_with_dup_edges, edge_type, edge_branch, peer) {
62
+ OPTIONAL MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
63
63
  WHERE type(active_e) = edge_type
64
64
  WITH node_with_dup_edges, peer, active_e
65
65
  ORDER BY %(id_func_name)s(active_e)
66
66
  SKIP 1
67
- RETURN collect(
68
- {
67
+ WITH CASE
68
+ WHEN active_e IS NOT NULL THEN {
69
69
  db_id: %(id_func_name)s(active_e),
70
- from_id: %(id_func_name)s(node_with_dup_edges),
71
- to_id: %(id_func_name)s(peer),
70
+ from_id: %(id_func_name)s(startNode(active_e)),
71
+ to_id: %(id_func_name)s(endNode(active_e)),
72
72
  edge_type: type(active_e),
73
73
  before_props: properties(active_e)
74
74
  }
75
- ) AS active_edges_to_delete
75
+ ELSE NULL
76
+ END AS serialized_edge
77
+ RETURN collect(serialized_edge) AS active_edges_to_delete
76
78
  }
77
79
  // ------------
78
80
  // Plan one deleted edge update with correct from time
79
81
  // ------------
80
- CALL {
81
- WITH node_with_dup_edges, edge_type, edge_branch, peer, deleted_from
82
- MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
82
+ CALL (node_with_dup_edges, edge_type, edge_branch, peer, deleted_from) {
83
+ OPTIONAL MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
83
84
  WHERE type(deleted_e) = edge_type
84
85
  WITH node_with_dup_edges, edge_type, edge_branch, peer, deleted_from, deleted_e
85
86
  ORDER BY %(id_func_name)s(deleted_e)
86
87
  LIMIT 1
87
88
  WITH deleted_e, properties(deleted_e) AS before_props, {from: deleted_from} AS prop_updates
88
- RETURN [
89
- {
90
- db_id: %(id_func_name)s(deleted_e), before_props: before_props, prop_updates: prop_updates
91
- }
92
- ] AS deleted_edges_to_update
89
+ RETURN CASE
90
+ WHEN deleted_e IS NOT NULL THEN [
91
+ {
92
+ db_id: %(id_func_name)s(deleted_e), before_props: before_props, prop_updates: prop_updates
93
+ }
94
+ ]
95
+ ELSE []
96
+ END AS deleted_edges_to_update
93
97
  }
94
98
  // ------------
95
99
  // Plan deletes for all the other deleted edges of this type on this branch
96
100
  // ------------
97
- CALL {
98
- WITH node_with_dup_edges, edge_type, edge_branch, peer
99
- MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
101
+ CALL (node_with_dup_edges, edge_type, edge_branch, peer) {
102
+ OPTIONAL MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
100
103
  WHERE type(deleted_e) = edge_type
101
104
  WITH node_with_dup_edges, peer, deleted_e
102
105
  ORDER BY %(id_func_name)s(deleted_e)
103
106
  SKIP 1
104
- RETURN collect(
105
- {
107
+ WITH CASE
108
+ WHEN deleted_e IS NOT NULL THEN {
106
109
  db_id: %(id_func_name)s(deleted_e),
107
- from_id: %(id_func_name)s(node_with_dup_edges),
108
- to_id: %(id_func_name)s(peer),
110
+ from_id: %(id_func_name)s(startNode(deleted_e)),
111
+ to_id: %(id_func_name)s(endNode(deleted_e)),
109
112
  edge_type: type(deleted_e),
110
113
  before_props: properties(deleted_e)
111
114
  }
112
- ) AS deleted_edges_to_delete
115
+ ELSE NULL
116
+ END AS serialized_edge
117
+
118
+ RETURN collect(serialized_edge) AS deleted_edges_to_delete
113
119
  }
114
120
  RETURN
115
121
  active_edges_to_update + deleted_edges_to_update AS edges_to_update,
@@ -0,0 +1,14 @@
1
+ from dataclasses import dataclass
2
+
3
+
4
+ @dataclass
5
+ class NumberPoolLockDefinition:
6
+ pool_id: str
7
+
8
+ @property
9
+ def lock_name(self) -> str:
10
+ return f"number-pool-creation-{self.pool_id}"
11
+
12
+ @property
13
+ def namespace_name(self) -> str:
14
+ return "number-pool"
infrahub/pools/number.py CHANGED
@@ -8,7 +8,7 @@ from infrahub.core.registry import registry
8
8
 
9
9
  if TYPE_CHECKING:
10
10
  from infrahub.core.branch import Branch
11
- from infrahub.core.protocols import CoreNode
11
+ from infrahub.core.node.resource_manager.number_pool import CoreNumberPool
12
12
  from infrahub.core.timestamp import Timestamp
13
13
  from infrahub.database import InfrahubDatabase
14
14
 
@@ -20,7 +20,9 @@ class UsedNumber:
20
20
 
21
21
 
22
22
  class NumberUtilizationGetter:
23
- def __init__(self, db: InfrahubDatabase, pool: CoreNode, branch: Branch, at: Timestamp | str | None = None) -> None:
23
+ def __init__(
24
+ self, db: InfrahubDatabase, pool: CoreNumberPool, branch: Branch, at: Timestamp | str | None = None
25
+ ) -> None:
24
26
  self.db = db
25
27
  self.at = at
26
28
  self.pool = pool
@@ -62,4 +64,4 @@ class NumberUtilizationGetter:
62
64
 
63
65
  @property
64
66
  def total_pool_size(self) -> int:
65
- return self.end_range - self.start_range + 1
67
+ return self.end_range - self.start_range + 1 - self.pool.get_attribute_nb_excluded_values()
@@ -0,0 +1,22 @@
1
+ from infrahub.core.registry import registry
2
+ from infrahub.exceptions import SchemaNotFoundError
3
+
4
+
5
+ def get_branches_with_schema_number_pool(kind: str, attribute_name: str) -> list[str]:
6
+ """Return branches where schema defined NumberPool exists"""
7
+
8
+ registered_branches = []
9
+ active_branches = registry.schema.get_branches()
10
+
11
+ for active_branch in active_branches:
12
+ try:
13
+ schema = registry.schema.get(name=kind, branch=active_branch)
14
+ except SchemaNotFoundError:
15
+ continue
16
+
17
+ if attribute_name in schema.attribute_names:
18
+ attribute = schema.get_attribute(name=attribute_name)
19
+ if attribute.kind == "NumberPool":
20
+ registered_branches.append(active_branch)
21
+
22
+ return registered_branches
@@ -0,0 +1,126 @@
1
+ from __future__ import annotations
2
+
3
+ from prefect import flow
4
+ from prefect.logging import get_run_logger
5
+
6
+ from infrahub import lock
7
+ from infrahub.context import InfrahubContext # noqa: TC001 needed for prefect flow
8
+ from infrahub.core.constants import InfrahubKind, NumberPoolType
9
+ from infrahub.core.manager import NodeManager
10
+ from infrahub.core.node import Node
11
+ from infrahub.core.protocols import CoreNumberPool
12
+ from infrahub.core.registry import registry
13
+ from infrahub.core.schema.attribute_parameters import NumberPoolParameters
14
+ from infrahub.exceptions import NodeNotFoundError
15
+ from infrahub.pools.models import NumberPoolLockDefinition
16
+ from infrahub.pools.registration import get_branches_with_schema_number_pool
17
+ from infrahub.services import InfrahubServices # noqa: TC001 needed for prefect flow
18
+
19
+
20
+ @flow(
21
+ name="validate-schema-number-pools",
22
+ flow_run_name="Validate schema number pools on {branch_name}",
23
+ )
24
+ async def validate_schema_number_pools(
25
+ branch_name: str, # noqa: ARG001
26
+ context: InfrahubContext, # noqa: ARG001
27
+ service: InfrahubServices,
28
+ ) -> None:
29
+ log = get_run_logger()
30
+
31
+ async with service.database.start_session() as dbs:
32
+ schema_number_pools = await NodeManager.query(
33
+ db=dbs, schema=CoreNumberPool, filters={"pool_type__value": NumberPoolType.SCHEMA.value}
34
+ )
35
+
36
+ for schema_number_pool in list(schema_number_pools):
37
+ defined_on_branches = get_branches_with_schema_number_pool(
38
+ kind=schema_number_pool.node.value, attribute_name=schema_number_pool.node_attribute.value
39
+ )
40
+ if registry.default_branch in defined_on_branches:
41
+ schema = registry.schema.get(name=schema_number_pool.node.value, branch=registry.default_branch)
42
+ attribute = schema.get_attribute(name=schema_number_pool.node_attribute.value)
43
+ number_pool_updated = False
44
+ if isinstance(attribute.parameters, NumberPoolParameters):
45
+ if schema_number_pool.start_range.value != attribute.parameters.start_range:
46
+ schema_number_pool.start_range.value = attribute.parameters.start_range
47
+ number_pool_updated = True
48
+ if schema_number_pool.end_range.value != attribute.parameters.end_range:
49
+ schema_number_pool.end_range.value = attribute.parameters.end_range
50
+ number_pool_updated = True
51
+
52
+ if number_pool_updated:
53
+ log.info(
54
+ f"Updating NumberPool={schema_number_pool.id} based on changes in the schema on {registry.default_branch}"
55
+ )
56
+ await schema_number_pool.save(db=service.database)
57
+
58
+ elif not defined_on_branches:
59
+ log.info(f"Deleting number pool (id={schema_number_pool.id}) as it is no longer defined in the schema")
60
+ await schema_number_pool.delete(db=service.database)
61
+
62
+ existing_pool_ids = [pool.id for pool in schema_number_pools]
63
+ for registry_branch in registry.schema.get_branches():
64
+ schema_branch = service.database.schema.get_schema_branch(name=registry_branch)
65
+
66
+ for generic_name in schema_branch.generic_names:
67
+ generic_node = schema_branch.get_generic(name=generic_name, duplicate=False)
68
+ for attribute_name in generic_node.attribute_names:
69
+ attribute = generic_node.get_attribute(name=attribute_name)
70
+ if isinstance(attribute.parameters, NumberPoolParameters) and attribute.parameters.number_pool_id:
71
+ if attribute.parameters.number_pool_id not in existing_pool_ids:
72
+ await _create_number_pool(
73
+ service=service,
74
+ number_pool_id=attribute.parameters.number_pool_id,
75
+ pool_node=generic_node.kind,
76
+ pool_attribute=attribute_name,
77
+ start_range=attribute.parameters.start_range,
78
+ end_range=attribute.parameters.end_range,
79
+ )
80
+ existing_pool_ids.append(attribute.parameters.number_pool_id)
81
+
82
+ for node_name in schema_branch.node_names:
83
+ node = schema_branch.get_node(name=node_name, duplicate=False)
84
+ for attribute_name in node.attribute_names:
85
+ attribute = node.get_attribute(name=attribute_name)
86
+ if isinstance(attribute.parameters, NumberPoolParameters) and attribute.parameters.number_pool_id:
87
+ if attribute.parameters.number_pool_id not in existing_pool_ids:
88
+ await _create_number_pool(
89
+ service=service,
90
+ number_pool_id=attribute.parameters.number_pool_id,
91
+ pool_node=node.kind,
92
+ pool_attribute=attribute_name,
93
+ start_range=attribute.parameters.start_range,
94
+ end_range=attribute.parameters.end_range,
95
+ )
96
+ existing_pool_ids.append(attribute.parameters.number_pool_id)
97
+
98
+
99
+ async def _create_number_pool(
100
+ service: InfrahubServices,
101
+ number_pool_id: str,
102
+ pool_node: str,
103
+ pool_attribute: str,
104
+ start_range: int,
105
+ end_range: int,
106
+ ) -> None:
107
+ lock_definition = NumberPoolLockDefinition(pool_id=number_pool_id)
108
+ async with lock.registry.get(name=lock_definition.lock_name, namespace=lock_definition.namespace_name, local=False):
109
+ async with service.database.start_session() as dbs:
110
+ try:
111
+ await registry.manager.get_one_by_id_or_default_filter(
112
+ db=dbs, id=str(number_pool_id), kind=CoreNumberPool
113
+ )
114
+ except NodeNotFoundError:
115
+ number_pool = await Node.init(db=dbs, schema=InfrahubKind.NUMBERPOOL, branch=registry.default_branch)
116
+ await number_pool.new(
117
+ db=dbs,
118
+ id=number_pool_id,
119
+ name=f"{pool_node}.{pool_attribute} [{number_pool_id}]",
120
+ node=pool_node,
121
+ node_attribute=pool_attribute,
122
+ start_range=start_range,
123
+ end_range=end_range,
124
+ pool_type=NumberPoolType.SCHEMA.value,
125
+ )
126
+ await number_pool.save(db=dbs)
@@ -1,18 +1,10 @@
1
- from typing import TYPE_CHECKING, Sequence, cast
2
-
3
- from prefect.server.database import PrefectDBInterface, db_injector
4
- from prefect.server.events.filters import EventFilter, EventNameFilter, EventOrder, EventRelatedFilter
1
+ from prefect.server.events.filters import EventFilter, EventNameFilter, EventOrder
5
2
  from prefect.server.events.schemas.events import ReceivedEvent
6
3
  from prefect.server.utilities.schemas import PrefectBaseModel
7
4
  from pydantic import BaseModel, Field
8
5
 
9
- if TYPE_CHECKING:
10
- from sqlalchemy.sql.expression import ColumnExpressionArgument
11
-
12
6
 
13
7
  class InfrahubEventFilter(EventFilter):
14
- matching_related: list[EventRelatedFilter] = Field(default_factory=list)
15
-
16
8
  def set_prefix(self) -> None:
17
9
  if self.event:
18
10
  if self.event.prefix is not None and "infrahub." not in self.event.prefix:
@@ -20,16 +12,6 @@ class InfrahubEventFilter(EventFilter):
20
12
  else:
21
13
  self.event = EventNameFilter(prefix=["infrahub."], name=[], exclude_prefix=None, exclude_name=None)
22
14
 
23
- @db_injector
24
- def build_where_clauses(self, db: PrefectDBInterface) -> Sequence["ColumnExpressionArgument[bool]"]:
25
- result = cast(list["ColumnExpressionArgument[bool]"], super().build_where_clauses())
26
- top_level_filter = self._scoped_event_resources(db)
27
- for matching_related in self.matching_related:
28
- matching_related._top_level_filter = top_level_filter
29
- result.extend(matching_related.build_where_clauses())
30
-
31
- return result
32
-
33
15
  @classmethod
34
16
  def default(cls) -> "InfrahubEventFilter":
35
17
  return cls(event=None, any_resource=None, resource=None, related=None, order=EventOrder.DESC)
@@ -1,10 +1,25 @@
1
+ import uuid
2
+
1
3
  from pydantic import BaseModel, ConfigDict, Field
2
4
 
3
- from infrahub.context import InfrahubContext
4
- from infrahub.message_bus.messages.proposed_change.base_with_diff import BaseProposedChangeWithDiffMessage
5
+ from infrahub.core.constants import CheckType
6
+ from infrahub.generators.models import ProposedChangeGeneratorDefinition
7
+ from infrahub.message_bus import InfrahubMessage
5
8
  from infrahub.message_bus.types import ProposedChangeArtifactDefinition, ProposedChangeBranchDiff
6
9
 
7
10
 
11
+ class BaseProposedChangeWithDiffMessage(InfrahubMessage):
12
+ """Sent trigger the refresh of artifacts that are impacted by the proposed change."""
13
+
14
+ model_config = ConfigDict(arbitrary_types_allowed=True)
15
+
16
+ proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
17
+ source_branch: str = Field(..., description="The source branch of the proposed change")
18
+ source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
19
+ destination_branch: str = Field(..., description="The destination branch of the proposed change")
20
+ branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")
21
+
22
+
8
23
  class RequestProposedChangeDataIntegrity(BaseProposedChangeWithDiffMessage):
9
24
  """Sent trigger data integrity checks for a proposed change"""
10
25
 
@@ -42,4 +57,54 @@ class RequestArtifactDefinitionCheck(BaseModel):
42
57
  source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
43
58
  destination_branch: str = Field(..., description="The target branch")
44
59
 
45
- context: InfrahubContext = Field(..., description="The context of the task")
60
+
61
+ class RunGeneratorAsCheckModel(BaseModel):
62
+ """A check that runs a generator."""
63
+
64
+ generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator definition")
65
+ generator_instance: str | None = Field(
66
+ default=None, description="The id of the generator instance if it previously existed"
67
+ )
68
+ commit: str = Field(..., description="The commit to target")
69
+ repository_id: str = Field(..., description="The unique ID of the Repository")
70
+ repository_name: str = Field(..., description="The name of the Repository")
71
+ repository_kind: str = Field(..., description="The kind of the Repository")
72
+ branch_name: str = Field(..., description="The branch where the check is run")
73
+ target_id: str = Field(..., description="The ID of the target object for this generator")
74
+ target_name: str = Field(..., description="Name of the generator target")
75
+ query: str = Field(..., description="The name of the query to use when collecting data")
76
+ variables: dict = Field(..., description="Input variables when running the generator")
77
+ validator_id: str = Field(..., description="The ID of the validator")
78
+ proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
79
+
80
+
81
+ class RequestGeneratorDefinitionCheck(BaseModel):
82
+ """Sent to trigger Generators to run for a proposed change."""
83
+
84
+ model_config = ConfigDict(arbitrary_types_allowed=True)
85
+
86
+ generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator Definition")
87
+ branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")
88
+ proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
89
+ source_branch: str = Field(..., description="The source branch")
90
+ source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
91
+ destination_branch: str = Field(..., description="The target branch")
92
+
93
+
94
+ class RequestProposedChangePipeline(BaseModel):
95
+ """Sent request the start of a pipeline connected to a proposed change."""
96
+
97
+ proposed_change: str = Field(..., description="The unique ID of the proposed change")
98
+ source_branch: str = Field(..., description="The source branch of the proposed change")
99
+ source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
100
+ destination_branch: str = Field(..., description="The destination branch of the proposed change")
101
+ check_type: CheckType = Field(
102
+ default=CheckType.ALL, description="Can be used to restrict the pipeline to a specific type of job"
103
+ )
104
+ pipeline_id: uuid.UUID = Field(
105
+ default_factory=uuid.uuid4, description="The unique ID of the execution of this pipeline"
106
+ )
107
+
108
+
109
+ class RequestProposedChangeRefreshArtifacts(BaseProposedChangeWithDiffMessage):
110
+ """Sent trigger the refresh of artifacts that are impacted by the proposed change."""