infrahub-server 1.3.6__py3-none-any.whl → 1.4.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. infrahub/api/internal.py +5 -0
  2. infrahub/artifacts/tasks.py +17 -22
  3. infrahub/branch/merge_mutation_checker.py +38 -0
  4. infrahub/cli/__init__.py +2 -2
  5. infrahub/cli/context.py +7 -3
  6. infrahub/cli/db.py +5 -41
  7. infrahub/cli/upgrade.py +7 -29
  8. infrahub/computed_attribute/tasks.py +36 -46
  9. infrahub/config.py +53 -2
  10. infrahub/constants/environment.py +1 -0
  11. infrahub/core/attribute.py +9 -7
  12. infrahub/core/branch/tasks.py +43 -41
  13. infrahub/core/constants/__init__.py +20 -6
  14. infrahub/core/constants/infrahubkind.py +2 -0
  15. infrahub/core/diff/coordinator.py +3 -1
  16. infrahub/core/diff/repository/repository.py +0 -8
  17. infrahub/core/diff/tasks.py +11 -8
  18. infrahub/core/graph/__init__.py +1 -1
  19. infrahub/core/graph/index.py +1 -2
  20. infrahub/core/graph/schema.py +50 -29
  21. infrahub/core/initialization.py +62 -33
  22. infrahub/core/ipam/tasks.py +4 -3
  23. infrahub/core/manager.py +2 -2
  24. infrahub/core/merge.py +8 -10
  25. infrahub/core/migrations/graph/__init__.py +4 -0
  26. infrahub/core/migrations/graph/m035_drop_attr_value_index.py +45 -0
  27. infrahub/core/migrations/graph/m036_index_attr_vals.py +577 -0
  28. infrahub/core/migrations/query/attribute_add.py +27 -2
  29. infrahub/core/migrations/query/node_duplicate.py +3 -26
  30. infrahub/core/migrations/schema/tasks.py +6 -5
  31. infrahub/core/node/proposed_change.py +43 -0
  32. infrahub/core/protocols.py +12 -0
  33. infrahub/core/query/attribute.py +32 -14
  34. infrahub/core/query/diff.py +11 -0
  35. infrahub/core/query/ipam.py +13 -7
  36. infrahub/core/query/node.py +51 -23
  37. infrahub/core/query/resource_manager.py +3 -3
  38. infrahub/core/relationship/model.py +13 -13
  39. infrahub/core/schema/basenode_schema.py +8 -0
  40. infrahub/core/schema/definitions/core/__init__.py +10 -1
  41. infrahub/core/schema/definitions/core/ipam.py +28 -2
  42. infrahub/core/schema/definitions/core/propose_change.py +15 -0
  43. infrahub/core/schema/definitions/core/webhook.py +3 -0
  44. infrahub/core/schema/generic_schema.py +10 -0
  45. infrahub/core/schema/manager.py +10 -1
  46. infrahub/core/schema/node_schema.py +22 -17
  47. infrahub/core/schema/profile_schema.py +8 -0
  48. infrahub/core/schema/schema_branch.py +9 -5
  49. infrahub/core/schema/template_schema.py +8 -0
  50. infrahub/core/validators/checks_runner.py +5 -5
  51. infrahub/core/validators/tasks.py +6 -7
  52. infrahub/core/validators/uniqueness/checker.py +4 -2
  53. infrahub/core/validators/uniqueness/model.py +1 -0
  54. infrahub/core/validators/uniqueness/query.py +57 -7
  55. infrahub/database/__init__.py +2 -1
  56. infrahub/events/__init__.py +18 -0
  57. infrahub/events/constants.py +7 -0
  58. infrahub/events/generator.py +29 -2
  59. infrahub/events/proposed_change_action.py +181 -0
  60. infrahub/generators/tasks.py +24 -20
  61. infrahub/git/base.py +4 -7
  62. infrahub/git/integrator.py +21 -12
  63. infrahub/git/repository.py +15 -30
  64. infrahub/git/tasks.py +121 -106
  65. infrahub/graphql/field_extractor.py +69 -0
  66. infrahub/graphql/manager.py +15 -11
  67. infrahub/graphql/mutations/account.py +2 -2
  68. infrahub/graphql/mutations/action.py +8 -2
  69. infrahub/graphql/mutations/artifact_definition.py +4 -1
  70. infrahub/graphql/mutations/branch.py +10 -5
  71. infrahub/graphql/mutations/graphql_query.py +2 -1
  72. infrahub/graphql/mutations/main.py +14 -8
  73. infrahub/graphql/mutations/menu.py +2 -1
  74. infrahub/graphql/mutations/proposed_change.py +225 -8
  75. infrahub/graphql/mutations/relationship.py +6 -1
  76. infrahub/graphql/mutations/repository.py +2 -1
  77. infrahub/graphql/mutations/tasks.py +7 -9
  78. infrahub/graphql/mutations/webhook.py +4 -1
  79. infrahub/graphql/parser.py +15 -6
  80. infrahub/graphql/queries/__init__.py +10 -1
  81. infrahub/graphql/queries/account.py +3 -3
  82. infrahub/graphql/queries/branch.py +2 -2
  83. infrahub/graphql/queries/diff/tree.py +3 -3
  84. infrahub/graphql/queries/event.py +13 -3
  85. infrahub/graphql/queries/ipam.py +23 -1
  86. infrahub/graphql/queries/proposed_change.py +84 -0
  87. infrahub/graphql/queries/relationship.py +2 -2
  88. infrahub/graphql/queries/resource_manager.py +3 -3
  89. infrahub/graphql/queries/search.py +3 -2
  90. infrahub/graphql/queries/status.py +3 -2
  91. infrahub/graphql/queries/task.py +2 -2
  92. infrahub/graphql/resolvers/ipam.py +440 -0
  93. infrahub/graphql/resolvers/many_relationship.py +4 -3
  94. infrahub/graphql/resolvers/resolver.py +5 -5
  95. infrahub/graphql/resolvers/single_relationship.py +3 -2
  96. infrahub/graphql/schema.py +25 -5
  97. infrahub/graphql/types/__init__.py +2 -2
  98. infrahub/graphql/types/attribute.py +3 -3
  99. infrahub/graphql/types/event.py +60 -0
  100. infrahub/groups/tasks.py +6 -6
  101. infrahub/lock.py +3 -2
  102. infrahub/menu/generator.py +8 -0
  103. infrahub/message_bus/operations/__init__.py +9 -12
  104. infrahub/message_bus/operations/git/file.py +6 -5
  105. infrahub/message_bus/operations/git/repository.py +12 -20
  106. infrahub/message_bus/operations/refresh/registry.py +15 -9
  107. infrahub/message_bus/operations/send/echo.py +7 -4
  108. infrahub/message_bus/types.py +1 -0
  109. infrahub/permissions/globals.py +1 -4
  110. infrahub/permissions/manager.py +8 -5
  111. infrahub/pools/prefix.py +7 -5
  112. infrahub/prefect_server/app.py +31 -0
  113. infrahub/prefect_server/bootstrap.py +18 -0
  114. infrahub/proposed_change/action_checker.py +206 -0
  115. infrahub/proposed_change/approval_revoker.py +40 -0
  116. infrahub/proposed_change/branch_diff.py +3 -1
  117. infrahub/proposed_change/checker.py +45 -0
  118. infrahub/proposed_change/constants.py +32 -2
  119. infrahub/proposed_change/tasks.py +182 -150
  120. infrahub/server.py +29 -17
  121. infrahub/services/__init__.py +13 -28
  122. infrahub/services/adapters/cache/__init__.py +4 -0
  123. infrahub/services/adapters/cache/nats.py +2 -0
  124. infrahub/services/adapters/cache/redis.py +3 -0
  125. infrahub/services/adapters/message_bus/__init__.py +0 -2
  126. infrahub/services/adapters/message_bus/local.py +1 -2
  127. infrahub/services/adapters/message_bus/nats.py +6 -8
  128. infrahub/services/adapters/message_bus/rabbitmq.py +7 -9
  129. infrahub/services/adapters/workflow/__init__.py +1 -0
  130. infrahub/services/adapters/workflow/local.py +1 -8
  131. infrahub/services/component.py +2 -1
  132. infrahub/task_manager/event.py +52 -0
  133. infrahub/task_manager/models.py +9 -0
  134. infrahub/tasks/artifact.py +6 -7
  135. infrahub/tasks/check.py +4 -7
  136. infrahub/telemetry/tasks.py +15 -18
  137. infrahub/transformations/tasks.py +10 -6
  138. infrahub/trigger/tasks.py +4 -3
  139. infrahub/types.py +4 -0
  140. infrahub/validators/events.py +7 -7
  141. infrahub/validators/tasks.py +6 -7
  142. infrahub/webhook/models.py +45 -45
  143. infrahub/webhook/tasks.py +25 -24
  144. infrahub/workers/dependencies.py +143 -0
  145. infrahub/workers/infrahub_async.py +19 -43
  146. infrahub/workflows/catalogue.py +16 -2
  147. infrahub/workflows/initialization.py +5 -4
  148. infrahub/workflows/models.py +2 -0
  149. infrahub_sdk/client.py +6 -6
  150. infrahub_sdk/ctl/repository.py +51 -0
  151. infrahub_sdk/ctl/schema.py +9 -9
  152. infrahub_sdk/protocols.py +40 -6
  153. {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/METADATA +6 -4
  154. {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/RECORD +162 -149
  155. infrahub_testcontainers/container.py +17 -0
  156. infrahub_testcontainers/docker-compose-cluster.test.yml +56 -1
  157. infrahub_testcontainers/docker-compose.test.yml +56 -1
  158. infrahub_testcontainers/helpers.py +4 -1
  159. infrahub/cli/db_commands/check_inheritance.py +0 -284
  160. /infrahub/{cli/db_commands/__init__.py → py.typed} +0 -0
  161. {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/LICENSE.txt +0 -0
  162. {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/WHEEL +0 -0
  163. {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/entry_points.txt +0 -0
@@ -59,6 +59,8 @@ PROJECT_ENV_VARIABLES: dict[str, str] = {
59
59
  "INFRAHUB_TESTING_PREFECT_UI_ENABLED": "true",
60
60
  "INFRAHUB_TESTING_DOCKER_PULL": "true",
61
61
  "INFRAHUB_TESTING_SCHEMA_STRICT_MODE": "true",
62
+ "INFRAHUB_TESTING_TASKMGR_API_WORKERS": "1",
63
+ "INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "0",
62
64
  }
63
65
 
64
66
 
@@ -144,6 +146,21 @@ class InfrahubDockerCompose(DockerCompose):
144
146
  "NEO4J_DOCKER_IMAGE": "neo4j:2025.03.0-enterprise",
145
147
  }
146
148
  )
149
+ if os.environ.get("INFRAHUB_TESTING_TASKMGR_SCALEOUT"):
150
+ PROJECT_ENV_VARIABLES.update(
151
+ {
152
+ "INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "1",
153
+ "PREFECT_MESSAGING_BROKER": "prefect_redis.messaging",
154
+ "PREFECT_MESSAGING_CACHE": "prefect_redis.messaging",
155
+ "PREFECT__SERVER_WEBSERVER_ONLY": "true",
156
+ "PREFECT_API_DATABASE_MIGRATE_ON_START": "false",
157
+ "PREFECT_API_BLOCKS_REGISTER_ON_START": "false",
158
+ "PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED": "false",
159
+ "PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED": "false",
160
+ "PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED": "false",
161
+ "PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED": "false",
162
+ }
163
+ )
147
164
 
148
165
  with env_file.open(mode="w", encoding="utf-8") as file:
149
166
  for key, value in PROJECT_ENV_VARIABLES.items():
@@ -166,13 +166,33 @@ services:
166
166
 
167
167
  task-manager:
168
168
  image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
169
- command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect
169
+ command: "${PREFECT_SERVER_COMMAND:-gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4200 'infrahub.prefect_server.app:create_infrahub_prefect()'}"
170
170
  depends_on:
171
171
  task-manager-db:
172
172
  condition: service_healthy
173
+ cache:
174
+ condition: service_healthy
173
175
  environment:
174
176
  PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
175
177
  PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
178
+
179
+ INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
180
+ PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
181
+ PREFECT_REDIS_MESSAGING_DB: "1"
182
+ PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
183
+ PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
184
+
185
+ PREFECT_MESSAGING_BROKER:
186
+ PREFECT_MESSAGING_CACHE:
187
+ PREFECT__SERVER_WEBSERVER_ONLY:
188
+ PREFECT_API_DATABASE_MIGRATE_ON_START:
189
+ PREFECT_API_BLOCKS_REGISTER_ON_START:
190
+ PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED:
191
+ PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED:
192
+ PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED:
193
+ PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED:
194
+
195
+ WEB_CONCURRENCY: ${INFRAHUB_TESTING_TASKMGR_API_WORKERS:-1}
176
196
  healthcheck:
177
197
  test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
178
198
  interval: 5s
@@ -182,8 +202,42 @@ services:
182
202
  ports:
183
203
  - ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
184
204
 
205
+ task-manager-background-svc:
206
+ deploy:
207
+ mode: replicated
208
+ replicas: ${INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS:-0}
209
+ image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
210
+ command: prefect server services start
211
+ restart: unless-stopped
212
+ depends_on:
213
+ task-manager:
214
+ condition: service_healthy
215
+ task-manager-db:
216
+ condition: service_healthy
217
+ cache:
218
+ condition: service_healthy
219
+ environment:
220
+ PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # This might be required because triggers actions service depends on an in-memory API server
221
+ PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
222
+
223
+ PREFECT_API_DATABASE_MIGRATE_ON_START: "false"
224
+ PREFECT_API_BLOCKS_REGISTER_ON_START: "false"
225
+ INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
226
+ PREFECT_MESSAGING_BROKER: prefect_redis.messaging
227
+ PREFECT_MESSAGING_CACHE: prefect_redis.messaging
228
+ PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
229
+ PREFECT_REDIS_MESSAGING_DB: "1"
230
+ PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
231
+ PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
232
+
233
+ PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED: "true"
234
+ PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED: "true"
235
+ PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED: "true"
236
+ PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED: "true"
237
+
185
238
  task-manager-db:
186
239
  image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
240
+ command: postgres -c 'max_connections=${INFRAHUB_TESTING_TASK_MANAGER_DB_MAX_CONNECTIONS:-100}'
187
241
  environment:
188
242
  - POSTGRES_USER=postgres
189
243
  - POSTGRES_PASSWORD=postgres
@@ -265,6 +319,7 @@ services:
265
319
  INFRAHUB_DB_PROTOCOL: ${INFRAHUB_TESTING_DB_PROTOCOL:-neo4j}
266
320
  INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
267
321
  INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
322
+ INFRAHUB_PAGINATION_SIZE: ${INFRAHUB_TESTING_PAGINATION_SIZE:-50}
268
323
  PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
269
324
  # Tracing
270
325
  INFRAHUB_TRACE_ENABLE: ${INFRAHUB_TRACE_ENABLE:-false}
@@ -77,13 +77,33 @@ services:
77
77
 
78
78
  task-manager:
79
79
  image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
80
- command: uvicorn --host 0.0.0.0 --port 4200 --factory infrahub.prefect_server.app:create_infrahub_prefect
80
+ command: "${PREFECT_SERVER_COMMAND:-gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4200 'infrahub.prefect_server.app:create_infrahub_prefect()'}"
81
81
  depends_on:
82
82
  task-manager-db:
83
83
  condition: service_healthy
84
+ cache:
85
+ condition: service_healthy
84
86
  environment:
85
87
  PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
86
88
  PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
89
+
90
+ INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
91
+ PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
92
+ PREFECT_REDIS_MESSAGING_DB: "1"
93
+ PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
94
+ PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
95
+
96
+ PREFECT_MESSAGING_BROKER:
97
+ PREFECT_MESSAGING_CACHE:
98
+ PREFECT__SERVER_WEBSERVER_ONLY:
99
+ PREFECT_API_DATABASE_MIGRATE_ON_START:
100
+ PREFECT_API_BLOCKS_REGISTER_ON_START:
101
+ PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED:
102
+ PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED:
103
+ PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED:
104
+ PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED:
105
+
106
+ WEB_CONCURRENCY: ${INFRAHUB_TESTING_TASKMGR_API_WORKERS:-1}
87
107
  healthcheck:
88
108
  test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
89
109
  interval: 5s
@@ -93,8 +113,42 @@ services:
93
113
  ports:
94
114
  - ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
95
115
 
116
+ task-manager-background-svc:
117
+ deploy:
118
+ mode: replicated
119
+ replicas: ${INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS:-0}
120
+ image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
121
+ command: prefect server services start
122
+ restart: unless-stopped
123
+ depends_on:
124
+ task-manager:
125
+ condition: service_healthy
126
+ task-manager-db:
127
+ condition: service_healthy
128
+ cache:
129
+ condition: service_healthy
130
+ environment:
131
+ PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # This might be required because triggers actions service depends on an in-memory API server
132
+ PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
133
+
134
+ PREFECT_API_DATABASE_MIGRATE_ON_START: "false"
135
+ PREFECT_API_BLOCKS_REGISTER_ON_START: "false"
136
+ INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
137
+ PREFECT_MESSAGING_BROKER: prefect_redis.messaging
138
+ PREFECT_MESSAGING_CACHE: prefect_redis.messaging
139
+ PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
140
+ PREFECT_REDIS_MESSAGING_DB: "1"
141
+ PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
142
+ PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
143
+
144
+ PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED: "true"
145
+ PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED: "true"
146
+ PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED: "true"
147
+ PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED: "true"
148
+
96
149
  task-manager-db:
97
150
  image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
151
+ command: postgres -c 'max_connections=${INFRAHUB_TESTING_TASK_MANAGER_DB_MAX_CONNECTIONS:-100}'
98
152
  environment:
99
153
  - POSTGRES_USER=postgres
100
154
  - POSTGRES_PASSWORD=postgres
@@ -172,6 +226,7 @@ services:
172
226
  INFRAHUB_DB_ADDRESS: ${INFRAHUB_TESTING_DB_ADDRESS:-database}
173
227
  INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
174
228
  INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
229
+ INFRAHUB_PAGINATION_SIZE: ${INFRAHUB_TESTING_PAGINATION_SIZE:-50}
175
230
  PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
176
231
  INFRAHUB_EXPERIMENTAL_VALUE_DB_INDEX:
177
232
  # Tracing
@@ -29,11 +29,14 @@ class TestInfrahubDocker:
29
29
  return result.stdout
30
30
 
31
31
  @staticmethod
32
- def execute_command(command: str, address: str, concurrent_execution: int = 10) -> subprocess.CompletedProcess[str]:
32
+ def execute_command(
33
+ command: str, address: str, concurrent_execution: int = 10, pagination_size: int = 50
34
+ ) -> subprocess.CompletedProcess[str]:
33
35
  env = os.environ.copy()
34
36
  env["INFRAHUB_ADDRESS"] = address
35
37
  env["INFRAHUB_API_TOKEN"] = PROJECT_ENV_VARIABLES["INFRAHUB_TESTING_INITIAL_ADMIN_TOKEN"]
36
38
  env["INFRAHUB_MAX_CONCURRENT_EXECUTION"] = f"{concurrent_execution}"
39
+ env["INFRAHUB_PAGINATION_SIZE"] = f"{pagination_size}"
37
40
  result = subprocess.run( # noqa: S602
38
41
  command, shell=True, capture_output=True, text=True, env=env, check=False
39
42
  )
@@ -1,284 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from collections import defaultdict
4
- from dataclasses import dataclass
5
- from typing import TYPE_CHECKING, Any
6
-
7
- from rich import print as rprint
8
- from rich.console import Console
9
- from rich.table import Table
10
-
11
- from infrahub.core import registry
12
- from infrahub.core.branch.models import Branch
13
- from infrahub.core.constants import InfrahubKind
14
- from infrahub.core.migrations.query.node_duplicate import NodeDuplicateQuery, SchemaNodeInfo
15
- from infrahub.core.query import Query, QueryType
16
- from infrahub.core.schema import SchemaRoot, internal_schema
17
- from infrahub.core.schema.manager import SchemaManager
18
- from infrahub.log import get_logger
19
-
20
- from ..constants import FAILED_BADGE, SUCCESS_BADGE
21
-
22
- if TYPE_CHECKING:
23
- from infrahub.core.schema.node_schema import NodeSchema
24
- from infrahub.database import InfrahubDatabase
25
-
26
- log = get_logger()
27
-
28
-
29
- class GetSchemaWithUpdatedInheritance(Query):
30
- """
31
- Get the name, namespace, and branch of any SchemaNodes with _updated_ inheritance
32
- This query will only return schemas that have had `inherit_from` updated after they were created
33
- """
34
-
35
- name = "get_schema_with_updated_inheritance"
36
- type = QueryType.READ
37
- insert_return = False
38
-
39
- async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
40
- query = """
41
- // find inherit_from attributes that have been updated
42
- MATCH p = (schema_node:SchemaNode)-[has_attr_e:HAS_ATTRIBUTE {status: "active"}]->(a:Attribute {name: "inherit_from"})
43
- WHERE has_attr_e.to IS NULL
44
- CALL (a) {
45
- // only get branches on which the value was updated, we can ignore the initial create
46
- MATCH (a)-[e:HAS_VALUE]->(:AttributeValue)
47
- ORDER BY e.from ASC
48
- // tail leaves out the earliest one, which is the initial create
49
- RETURN tail(collect(e.branch)) AS branches
50
- }
51
- WITH schema_node, a, branches
52
- WHERE size(branches) > 0
53
- UNWIND branches AS branch
54
- WITH DISTINCT schema_node, a, branch
55
-
56
- //get branch details
57
- CALL (branch) {
58
- MATCH (b:Branch {name: branch})
59
- RETURN b.branched_from AS branched_from, b.hierarchy_level AS branch_level
60
- }
61
-
62
- // get the namespace for the schema
63
- CALL (schema_node, a, branch, branched_from, branch_level) {
64
- MATCH (schema_node)-[e1:HAS_ATTRIBUTE]-(:Attribute {name: "namespace"})-[e2:HAS_VALUE]->(av)
65
- WHERE (
66
- e1.branch = branch OR
67
- (e1.branch_level < branch_level AND e1.from <= branched_from)
68
- ) AND e1.to IS NULL
69
- AND e1.status = "active"
70
- AND (
71
- e2.branch = branch OR
72
- (e2.branch_level < branch_level AND e2.from <= branched_from)
73
- ) AND e2.to IS NULL
74
- AND e2.status = "active"
75
- ORDER BY e2.branch_level DESC, e1.branch_level DESC, e2.from DESC, e1.from DESC
76
- RETURN av.value AS namespace
77
- LIMIT 1
78
- }
79
-
80
- // get the name for the schema
81
- CALL (schema_node, a, branch, branched_from, branch_level) {
82
- MATCH (schema_node)-[e1:HAS_ATTRIBUTE]-(:Attribute {name: "name"})-[e2:HAS_VALUE]->(av)
83
- WHERE (
84
- e1.branch = branch OR
85
- (e1.branch_level < branch_level AND e1.from <= branched_from)
86
- ) AND e1.to IS NULL
87
- AND e1.status = "active"
88
- AND (
89
- e2.branch = branch OR
90
- (e2.branch_level < branch_level AND e2.from <= branched_from)
91
- ) AND e2.to IS NULL
92
- AND e2.status = "active"
93
- ORDER BY e2.branch_level DESC, e1.branch_level DESC, e2.from DESC, e1.from DESC
94
- RETURN av.value AS name
95
- LIMIT 1
96
- }
97
- RETURN name, namespace, branch
98
- """
99
- self.return_labels = ["name", "namespace", "branch"]
100
- self.add_to_query(query)
101
-
102
- def get_updated_inheritance_kinds_by_branch(self) -> dict[str, list[str]]:
103
- kinds_by_branch: dict[str, list[str]] = defaultdict(list)
104
- for result in self.results:
105
- name = result.get_as_type(label="name", return_type=str)
106
- namespace = result.get_as_type(label="namespace", return_type=str)
107
- branch = result.get_as_type(label="branch", return_type=str)
108
- kinds_by_branch[branch].append(f"{namespace}{name}")
109
- return kinds_by_branch
110
-
111
-
112
- @dataclass
113
- class KindLabelCount:
114
- kind: str
115
- labels: frozenset[str]
116
- num_nodes: int
117
-
118
-
119
- @dataclass
120
- class KindLabelCountCorrected(KindLabelCount):
121
- node_schema: NodeSchema
122
-
123
-
124
- class GetAllKindsAndLabels(Query):
125
- """
126
- Get the kind, labels, and number of nodes for the given kinds and branch
127
- """
128
-
129
- name = "get_all_kinds_and_labels"
130
- type = QueryType.READ
131
- insert_return = False
132
-
133
- def __init__(self, kinds: list[str] | None = None, **kwargs: Any) -> None:
134
- super().__init__(**kwargs)
135
- self.kinds = kinds
136
-
137
- async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
138
- self.params["branch_name"] = self.branch.name
139
- self.params["branched_from"] = self.branch.get_branched_from()
140
- self.params["branch_level"] = self.branch.hierarchy_level
141
- kinds_str = "Node"
142
- if self.kinds:
143
- kinds_str = "|".join(self.kinds)
144
- query = """
145
- MATCH (n:%(kinds_str)s)-[r:IS_PART_OF]->(:Root)
146
- WHERE (
147
- r.branch = $branch_name OR
148
- (r.branch_level < $branch_level AND r.from <= $branched_from)
149
- )
150
- AND r.to IS NULL
151
- AND r.status = "active"
152
- RETURN DISTINCT n.kind AS kind, labels(n) AS labels, count(*) AS num_nodes
153
- ORDER BY kind ASC
154
- """ % {"kinds_str": kinds_str}
155
- self.return_labels = ["kind", "labels", "num_nodes"]
156
- self.add_to_query(query)
157
-
158
- def get_kind_label_counts(self) -> list[KindLabelCount]:
159
- kind_label_counts: list[KindLabelCount] = []
160
- for result in self.results:
161
- kind = result.get_as_type(label="kind", return_type=str)
162
- num_nodes = result.get_as_type(label="num_nodes", return_type=int)
163
- labels: list[str] = result.get_as_type(label="labels", return_type=list)
164
- # we can ignore the Node label and the label that matches the kind
165
- cleaned_labels = frozenset(str(lbl) for lbl in labels if lbl not in ["Node", "CoreNode", kind])
166
- kind_label_counts.append(KindLabelCount(kind=kind, labels=cleaned_labels, num_nodes=num_nodes))
167
- return kind_label_counts
168
-
169
-
170
- def display_kind_label_counts(kind_label_counts_by_branch: dict[str, list[KindLabelCountCorrected]]) -> None:
171
- console = Console()
172
-
173
- table = Table(title="Incorrect Inheritance Nodes")
174
-
175
- table.add_column("Branch")
176
- table.add_column("Kind")
177
- table.add_column("Incorrect Labels")
178
- table.add_column("Num Nodes")
179
-
180
- for branch_name, kind_label_counts in kind_label_counts_by_branch.items():
181
- for kind_label_count in kind_label_counts:
182
- table.add_row(
183
- branch_name, kind_label_count.kind, str(list(kind_label_count.labels)), str(kind_label_count.num_nodes)
184
- )
185
-
186
- console.print(table)
187
-
188
-
189
- async def check_inheritance(db: InfrahubDatabase, fix: bool = False) -> bool:
190
- """
191
- Run migrations to update the inheritance of any nodes with incorrect inheritance from a failed migration
192
- 1. Identifies node schemas that have had their inheritance updated after they were created
193
- a. includes the kind and branch of the inheritance update
194
- 2. Checks nodes of the given kinds on the given branch to verify their inheritance is correct
195
- 3. Displays counts of any kinds with incorrect inheritance on the given branch
196
- 4. If fix is True, runs migrations to update the inheritance of any nodes with incorrect inheritance
197
- on the correct branch
198
- """
199
-
200
- updated_inheritance_query = await GetSchemaWithUpdatedInheritance.init(db=db)
201
- await updated_inheritance_query.execute(db=db)
202
- updated_inheritance_kinds_by_branch = updated_inheritance_query.get_updated_inheritance_kinds_by_branch()
203
-
204
- if not updated_inheritance_kinds_by_branch:
205
- rprint(f"{SUCCESS_BADGE} No schemas have had their inheritance updated")
206
- return True
207
-
208
- schema_manager = SchemaManager()
209
- registry.schema = schema_manager
210
- schema = SchemaRoot(**internal_schema)
211
- schema_manager.register_schema(schema=schema)
212
- branches_by_name = {b.name: b for b in await Branch.get_list(db=db)}
213
-
214
- kind_label_counts_by_branch: dict[str, list[KindLabelCountCorrected]] = defaultdict(list)
215
- for branch_name, kinds in updated_inheritance_kinds_by_branch.items():
216
- rprint(f"Checking branch: {branch_name}", end="...")
217
- branch = branches_by_name[branch_name]
218
- schema_branch = await schema_manager.load_schema_from_db(db=db, branch=branch)
219
- kind_label_query = await GetAllKindsAndLabels.init(db=db, branch=branch, kinds=kinds)
220
- await kind_label_query.execute(db=db)
221
- kind_label_counts = kind_label_query.get_kind_label_counts()
222
-
223
- for kind_label_count in kind_label_counts:
224
- node_schema = schema_branch.get_node(name=kind_label_count.kind, duplicate=False)
225
- correct_labels = frozenset(node_schema.inherit_from)
226
- if kind_label_count.labels == correct_labels:
227
- continue
228
-
229
- kind_label_counts_by_branch[branch_name].append(
230
- KindLabelCountCorrected(
231
- kind=kind_label_count.kind,
232
- labels=kind_label_count.labels,
233
- num_nodes=kind_label_count.num_nodes,
234
- node_schema=node_schema,
235
- )
236
- )
237
- rprint("done")
238
-
239
- if not kind_label_counts_by_branch:
240
- rprint(f"{SUCCESS_BADGE} All nodes have the correct inheritance")
241
- return True
242
-
243
- display_kind_label_counts(kind_label_counts_by_branch)
244
-
245
- if not fix:
246
- rprint(f"{FAILED_BADGE} Use the --fix flag to fix the inheritance of any invalid nodes")
247
- return False
248
-
249
- for branch_name, kind_label_counts_corrected in kind_label_counts_by_branch.items():
250
- for kind_label_count in kind_label_counts_corrected:
251
- rprint(f"Fixing kind {kind_label_count.kind} on branch {branch_name}", end="...")
252
- node_schema = kind_label_count.node_schema
253
- migration_query = await NodeDuplicateQuery.init(
254
- db=db,
255
- branch=branches_by_name[branch_name],
256
- previous_node=SchemaNodeInfo(
257
- name=node_schema.name,
258
- namespace=node_schema.namespace,
259
- branch_support=node_schema.branch.value,
260
- labels=list(kind_label_count.labels) + [kind_label_count.kind, InfrahubKind.NODE],
261
- kind=kind_label_count.kind,
262
- ),
263
- new_node=SchemaNodeInfo(
264
- name=node_schema.name,
265
- namespace=node_schema.namespace,
266
- branch_support=node_schema.branch.value,
267
- labels=list(node_schema.inherit_from) + [kind_label_count.kind, InfrahubKind.NODE],
268
- kind=kind_label_count.kind,
269
- ),
270
- )
271
- await migration_query.execute(db=db)
272
- rprint("done")
273
-
274
- rprint(f"{SUCCESS_BADGE} All nodes have the correct inheritance")
275
-
276
- if registry.default_branch in kind_label_counts_by_branch:
277
- kinds = [kind_label_count.kind for kind_label_count in kind_label_counts_by_branch[registry.default_branch]]
278
- rprint(
279
- "[bold cyan]Note that migrations were run on the default branch for the following schema kinds: "
280
- f"{', '.join(kinds)}. You should rebase any branches that include/will include changes using "
281
- "the migrated schemas[/bold cyan]"
282
- )
283
-
284
- return True
File without changes