infrahub-server 1.3.6__py3-none-any.whl → 1.4.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/api/internal.py +5 -0
- infrahub/artifacts/tasks.py +17 -22
- infrahub/branch/merge_mutation_checker.py +38 -0
- infrahub/cli/__init__.py +2 -2
- infrahub/cli/context.py +7 -3
- infrahub/cli/db.py +5 -41
- infrahub/cli/upgrade.py +7 -29
- infrahub/computed_attribute/tasks.py +36 -46
- infrahub/config.py +53 -2
- infrahub/constants/environment.py +1 -0
- infrahub/core/attribute.py +9 -7
- infrahub/core/branch/tasks.py +43 -41
- infrahub/core/constants/__init__.py +20 -6
- infrahub/core/constants/infrahubkind.py +2 -0
- infrahub/core/diff/coordinator.py +3 -1
- infrahub/core/diff/repository/repository.py +0 -8
- infrahub/core/diff/tasks.py +11 -8
- infrahub/core/graph/__init__.py +1 -1
- infrahub/core/graph/index.py +1 -2
- infrahub/core/graph/schema.py +50 -29
- infrahub/core/initialization.py +62 -33
- infrahub/core/ipam/tasks.py +4 -3
- infrahub/core/manager.py +2 -2
- infrahub/core/merge.py +8 -10
- infrahub/core/migrations/graph/__init__.py +4 -0
- infrahub/core/migrations/graph/m035_drop_attr_value_index.py +45 -0
- infrahub/core/migrations/graph/m036_index_attr_vals.py +577 -0
- infrahub/core/migrations/query/attribute_add.py +27 -2
- infrahub/core/migrations/query/node_duplicate.py +3 -26
- infrahub/core/migrations/schema/tasks.py +6 -5
- infrahub/core/node/proposed_change.py +43 -0
- infrahub/core/protocols.py +12 -0
- infrahub/core/query/attribute.py +32 -14
- infrahub/core/query/diff.py +11 -0
- infrahub/core/query/ipam.py +13 -7
- infrahub/core/query/node.py +51 -23
- infrahub/core/query/resource_manager.py +3 -3
- infrahub/core/relationship/model.py +13 -13
- infrahub/core/schema/basenode_schema.py +8 -0
- infrahub/core/schema/definitions/core/__init__.py +10 -1
- infrahub/core/schema/definitions/core/ipam.py +28 -2
- infrahub/core/schema/definitions/core/propose_change.py +15 -0
- infrahub/core/schema/definitions/core/webhook.py +3 -0
- infrahub/core/schema/generic_schema.py +10 -0
- infrahub/core/schema/manager.py +10 -1
- infrahub/core/schema/node_schema.py +22 -17
- infrahub/core/schema/profile_schema.py +8 -0
- infrahub/core/schema/schema_branch.py +9 -5
- infrahub/core/schema/template_schema.py +8 -0
- infrahub/core/validators/checks_runner.py +5 -5
- infrahub/core/validators/tasks.py +6 -7
- infrahub/core/validators/uniqueness/checker.py +4 -2
- infrahub/core/validators/uniqueness/model.py +1 -0
- infrahub/core/validators/uniqueness/query.py +57 -7
- infrahub/database/__init__.py +2 -1
- infrahub/events/__init__.py +18 -0
- infrahub/events/constants.py +7 -0
- infrahub/events/generator.py +29 -2
- infrahub/events/proposed_change_action.py +181 -0
- infrahub/generators/tasks.py +24 -20
- infrahub/git/base.py +4 -7
- infrahub/git/integrator.py +21 -12
- infrahub/git/repository.py +15 -30
- infrahub/git/tasks.py +121 -106
- infrahub/graphql/field_extractor.py +69 -0
- infrahub/graphql/manager.py +15 -11
- infrahub/graphql/mutations/account.py +2 -2
- infrahub/graphql/mutations/action.py +8 -2
- infrahub/graphql/mutations/artifact_definition.py +4 -1
- infrahub/graphql/mutations/branch.py +10 -5
- infrahub/graphql/mutations/graphql_query.py +2 -1
- infrahub/graphql/mutations/main.py +14 -8
- infrahub/graphql/mutations/menu.py +2 -1
- infrahub/graphql/mutations/proposed_change.py +225 -8
- infrahub/graphql/mutations/relationship.py +6 -1
- infrahub/graphql/mutations/repository.py +2 -1
- infrahub/graphql/mutations/tasks.py +7 -9
- infrahub/graphql/mutations/webhook.py +4 -1
- infrahub/graphql/parser.py +15 -6
- infrahub/graphql/queries/__init__.py +10 -1
- infrahub/graphql/queries/account.py +3 -3
- infrahub/graphql/queries/branch.py +2 -2
- infrahub/graphql/queries/diff/tree.py +3 -3
- infrahub/graphql/queries/event.py +13 -3
- infrahub/graphql/queries/ipam.py +23 -1
- infrahub/graphql/queries/proposed_change.py +84 -0
- infrahub/graphql/queries/relationship.py +2 -2
- infrahub/graphql/queries/resource_manager.py +3 -3
- infrahub/graphql/queries/search.py +3 -2
- infrahub/graphql/queries/status.py +3 -2
- infrahub/graphql/queries/task.py +2 -2
- infrahub/graphql/resolvers/ipam.py +440 -0
- infrahub/graphql/resolvers/many_relationship.py +4 -3
- infrahub/graphql/resolvers/resolver.py +5 -5
- infrahub/graphql/resolvers/single_relationship.py +3 -2
- infrahub/graphql/schema.py +25 -5
- infrahub/graphql/types/__init__.py +2 -2
- infrahub/graphql/types/attribute.py +3 -3
- infrahub/graphql/types/event.py +60 -0
- infrahub/groups/tasks.py +6 -6
- infrahub/lock.py +3 -2
- infrahub/menu/generator.py +8 -0
- infrahub/message_bus/operations/__init__.py +9 -12
- infrahub/message_bus/operations/git/file.py +6 -5
- infrahub/message_bus/operations/git/repository.py +12 -20
- infrahub/message_bus/operations/refresh/registry.py +15 -9
- infrahub/message_bus/operations/send/echo.py +7 -4
- infrahub/message_bus/types.py +1 -0
- infrahub/permissions/globals.py +1 -4
- infrahub/permissions/manager.py +8 -5
- infrahub/pools/prefix.py +7 -5
- infrahub/prefect_server/app.py +31 -0
- infrahub/prefect_server/bootstrap.py +18 -0
- infrahub/proposed_change/action_checker.py +206 -0
- infrahub/proposed_change/approval_revoker.py +40 -0
- infrahub/proposed_change/branch_diff.py +3 -1
- infrahub/proposed_change/checker.py +45 -0
- infrahub/proposed_change/constants.py +32 -2
- infrahub/proposed_change/tasks.py +182 -150
- infrahub/server.py +29 -17
- infrahub/services/__init__.py +13 -28
- infrahub/services/adapters/cache/__init__.py +4 -0
- infrahub/services/adapters/cache/nats.py +2 -0
- infrahub/services/adapters/cache/redis.py +3 -0
- infrahub/services/adapters/message_bus/__init__.py +0 -2
- infrahub/services/adapters/message_bus/local.py +1 -2
- infrahub/services/adapters/message_bus/nats.py +6 -8
- infrahub/services/adapters/message_bus/rabbitmq.py +7 -9
- infrahub/services/adapters/workflow/__init__.py +1 -0
- infrahub/services/adapters/workflow/local.py +1 -8
- infrahub/services/component.py +2 -1
- infrahub/task_manager/event.py +52 -0
- infrahub/task_manager/models.py +9 -0
- infrahub/tasks/artifact.py +6 -7
- infrahub/tasks/check.py +4 -7
- infrahub/telemetry/tasks.py +15 -18
- infrahub/transformations/tasks.py +10 -6
- infrahub/trigger/tasks.py +4 -3
- infrahub/types.py +4 -0
- infrahub/validators/events.py +7 -7
- infrahub/validators/tasks.py +6 -7
- infrahub/webhook/models.py +45 -45
- infrahub/webhook/tasks.py +25 -24
- infrahub/workers/dependencies.py +143 -0
- infrahub/workers/infrahub_async.py +19 -43
- infrahub/workflows/catalogue.py +16 -2
- infrahub/workflows/initialization.py +5 -4
- infrahub/workflows/models.py +2 -0
- infrahub_sdk/client.py +6 -6
- infrahub_sdk/ctl/repository.py +51 -0
- infrahub_sdk/ctl/schema.py +9 -9
- infrahub_sdk/protocols.py +40 -6
- {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/METADATA +6 -4
- {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/RECORD +162 -149
- infrahub_testcontainers/container.py +17 -0
- infrahub_testcontainers/docker-compose-cluster.test.yml +56 -1
- infrahub_testcontainers/docker-compose.test.yml +56 -1
- infrahub_testcontainers/helpers.py +4 -1
- infrahub/cli/db_commands/check_inheritance.py +0 -284
- /infrahub/{cli/db_commands/__init__.py → py.typed} +0 -0
- {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/LICENSE.txt +0 -0
- {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/WHEEL +0 -0
- {infrahub_server-1.3.6.dist-info → infrahub_server-1.4.0b1.dist-info}/entry_points.txt +0 -0
|
@@ -59,6 +59,8 @@ PROJECT_ENV_VARIABLES: dict[str, str] = {
|
|
|
59
59
|
"INFRAHUB_TESTING_PREFECT_UI_ENABLED": "true",
|
|
60
60
|
"INFRAHUB_TESTING_DOCKER_PULL": "true",
|
|
61
61
|
"INFRAHUB_TESTING_SCHEMA_STRICT_MODE": "true",
|
|
62
|
+
"INFRAHUB_TESTING_TASKMGR_API_WORKERS": "1",
|
|
63
|
+
"INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "0",
|
|
62
64
|
}
|
|
63
65
|
|
|
64
66
|
|
|
@@ -144,6 +146,21 @@ class InfrahubDockerCompose(DockerCompose):
|
|
|
144
146
|
"NEO4J_DOCKER_IMAGE": "neo4j:2025.03.0-enterprise",
|
|
145
147
|
}
|
|
146
148
|
)
|
|
149
|
+
if os.environ.get("INFRAHUB_TESTING_TASKMGR_SCALEOUT"):
|
|
150
|
+
PROJECT_ENV_VARIABLES.update(
|
|
151
|
+
{
|
|
152
|
+
"INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS": "1",
|
|
153
|
+
"PREFECT_MESSAGING_BROKER": "prefect_redis.messaging",
|
|
154
|
+
"PREFECT_MESSAGING_CACHE": "prefect_redis.messaging",
|
|
155
|
+
"PREFECT__SERVER_WEBSERVER_ONLY": "true",
|
|
156
|
+
"PREFECT_API_DATABASE_MIGRATE_ON_START": "false",
|
|
157
|
+
"PREFECT_API_BLOCKS_REGISTER_ON_START": "false",
|
|
158
|
+
"PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED": "false",
|
|
159
|
+
"PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED": "false",
|
|
160
|
+
"PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED": "false",
|
|
161
|
+
"PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED": "false",
|
|
162
|
+
}
|
|
163
|
+
)
|
|
147
164
|
|
|
148
165
|
with env_file.open(mode="w", encoding="utf-8") as file:
|
|
149
166
|
for key, value in PROJECT_ENV_VARIABLES.items():
|
|
@@ -166,13 +166,33 @@ services:
|
|
|
166
166
|
|
|
167
167
|
task-manager:
|
|
168
168
|
image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
|
|
169
|
-
command: uvicorn
|
|
169
|
+
command: "${PREFECT_SERVER_COMMAND:-gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4200 'infrahub.prefect_server.app:create_infrahub_prefect()'}"
|
|
170
170
|
depends_on:
|
|
171
171
|
task-manager-db:
|
|
172
172
|
condition: service_healthy
|
|
173
|
+
cache:
|
|
174
|
+
condition: service_healthy
|
|
173
175
|
environment:
|
|
174
176
|
PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
|
|
175
177
|
PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
|
|
178
|
+
|
|
179
|
+
INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
|
|
180
|
+
PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
|
|
181
|
+
PREFECT_REDIS_MESSAGING_DB: "1"
|
|
182
|
+
PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
|
|
183
|
+
PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
|
|
184
|
+
|
|
185
|
+
PREFECT_MESSAGING_BROKER:
|
|
186
|
+
PREFECT_MESSAGING_CACHE:
|
|
187
|
+
PREFECT__SERVER_WEBSERVER_ONLY:
|
|
188
|
+
PREFECT_API_DATABASE_MIGRATE_ON_START:
|
|
189
|
+
PREFECT_API_BLOCKS_REGISTER_ON_START:
|
|
190
|
+
PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED:
|
|
191
|
+
PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED:
|
|
192
|
+
PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED:
|
|
193
|
+
PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED:
|
|
194
|
+
|
|
195
|
+
WEB_CONCURRENCY: ${INFRAHUB_TESTING_TASKMGR_API_WORKERS:-1}
|
|
176
196
|
healthcheck:
|
|
177
197
|
test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
|
|
178
198
|
interval: 5s
|
|
@@ -182,8 +202,42 @@ services:
|
|
|
182
202
|
ports:
|
|
183
203
|
- ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
|
|
184
204
|
|
|
205
|
+
task-manager-background-svc:
|
|
206
|
+
deploy:
|
|
207
|
+
mode: replicated
|
|
208
|
+
replicas: ${INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS:-0}
|
|
209
|
+
image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
|
|
210
|
+
command: prefect server services start
|
|
211
|
+
restart: unless-stopped
|
|
212
|
+
depends_on:
|
|
213
|
+
task-manager:
|
|
214
|
+
condition: service_healthy
|
|
215
|
+
task-manager-db:
|
|
216
|
+
condition: service_healthy
|
|
217
|
+
cache:
|
|
218
|
+
condition: service_healthy
|
|
219
|
+
environment:
|
|
220
|
+
PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # This might be required because triggers actions service depends on an in-memory API server
|
|
221
|
+
PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
|
|
222
|
+
|
|
223
|
+
PREFECT_API_DATABASE_MIGRATE_ON_START: "false"
|
|
224
|
+
PREFECT_API_BLOCKS_REGISTER_ON_START: "false"
|
|
225
|
+
INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
|
|
226
|
+
PREFECT_MESSAGING_BROKER: prefect_redis.messaging
|
|
227
|
+
PREFECT_MESSAGING_CACHE: prefect_redis.messaging
|
|
228
|
+
PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
|
|
229
|
+
PREFECT_REDIS_MESSAGING_DB: "1"
|
|
230
|
+
PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
|
|
231
|
+
PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
|
|
232
|
+
|
|
233
|
+
PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED: "true"
|
|
234
|
+
PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED: "true"
|
|
235
|
+
PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED: "true"
|
|
236
|
+
PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED: "true"
|
|
237
|
+
|
|
185
238
|
task-manager-db:
|
|
186
239
|
image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
|
|
240
|
+
command: postgres -c 'max_connections=${INFRAHUB_TESTING_TASK_MANAGER_DB_MAX_CONNECTIONS:-100}'
|
|
187
241
|
environment:
|
|
188
242
|
- POSTGRES_USER=postgres
|
|
189
243
|
- POSTGRES_PASSWORD=postgres
|
|
@@ -265,6 +319,7 @@ services:
|
|
|
265
319
|
INFRAHUB_DB_PROTOCOL: ${INFRAHUB_TESTING_DB_PROTOCOL:-neo4j}
|
|
266
320
|
INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
|
|
267
321
|
INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
|
|
322
|
+
INFRAHUB_PAGINATION_SIZE: ${INFRAHUB_TESTING_PAGINATION_SIZE:-50}
|
|
268
323
|
PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
|
|
269
324
|
# Tracing
|
|
270
325
|
INFRAHUB_TRACE_ENABLE: ${INFRAHUB_TRACE_ENABLE:-false}
|
|
@@ -77,13 +77,33 @@ services:
|
|
|
77
77
|
|
|
78
78
|
task-manager:
|
|
79
79
|
image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
|
|
80
|
-
command: uvicorn
|
|
80
|
+
command: "${PREFECT_SERVER_COMMAND:-gunicorn -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4200 'infrahub.prefect_server.app:create_infrahub_prefect()'}"
|
|
81
81
|
depends_on:
|
|
82
82
|
task-manager-db:
|
|
83
83
|
condition: service_healthy
|
|
84
|
+
cache:
|
|
85
|
+
condition: service_healthy
|
|
84
86
|
environment:
|
|
85
87
|
PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # enabling UI requires permissions, run container as root to enable UI
|
|
86
88
|
PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
|
|
89
|
+
|
|
90
|
+
INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
|
|
91
|
+
PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
|
|
92
|
+
PREFECT_REDIS_MESSAGING_DB: "1"
|
|
93
|
+
PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
|
|
94
|
+
PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
|
|
95
|
+
|
|
96
|
+
PREFECT_MESSAGING_BROKER:
|
|
97
|
+
PREFECT_MESSAGING_CACHE:
|
|
98
|
+
PREFECT__SERVER_WEBSERVER_ONLY:
|
|
99
|
+
PREFECT_API_DATABASE_MIGRATE_ON_START:
|
|
100
|
+
PREFECT_API_BLOCKS_REGISTER_ON_START:
|
|
101
|
+
PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED:
|
|
102
|
+
PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED:
|
|
103
|
+
PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED:
|
|
104
|
+
PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED:
|
|
105
|
+
|
|
106
|
+
WEB_CONCURRENCY: ${INFRAHUB_TESTING_TASKMGR_API_WORKERS:-1}
|
|
87
107
|
healthcheck:
|
|
88
108
|
test: /usr/local/bin/httpx http://localhost:4200/api/health || exit 1
|
|
89
109
|
interval: 5s
|
|
@@ -93,8 +113,42 @@ services:
|
|
|
93
113
|
ports:
|
|
94
114
|
- ${INFRAHUB_TESTING_TASK_MANAGER_PORT:-0}:4200
|
|
95
115
|
|
|
116
|
+
task-manager-background-svc:
|
|
117
|
+
deploy:
|
|
118
|
+
mode: replicated
|
|
119
|
+
replicas: ${INFRAHUB_TESTING_TASKMGR_BACKGROUND_SVC_REPLICAS:-0}
|
|
120
|
+
image: "${INFRAHUB_TESTING_DOCKER_IMAGE}:${INFRAHUB_TESTING_IMAGE_VERSION}"
|
|
121
|
+
command: prefect server services start
|
|
122
|
+
restart: unless-stopped
|
|
123
|
+
depends_on:
|
|
124
|
+
task-manager:
|
|
125
|
+
condition: service_healthy
|
|
126
|
+
task-manager-db:
|
|
127
|
+
condition: service_healthy
|
|
128
|
+
cache:
|
|
129
|
+
condition: service_healthy
|
|
130
|
+
environment:
|
|
131
|
+
PREFECT_UI_ENABLED: "${INFRAHUB_TESTING_PREFECT_UI_ENABLED}" # This might be required because triggers actions service depends on an in-memory API server
|
|
132
|
+
PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@task-manager-db:5432/prefect
|
|
133
|
+
|
|
134
|
+
PREFECT_API_DATABASE_MIGRATE_ON_START: "false"
|
|
135
|
+
PREFECT_API_BLOCKS_REGISTER_ON_START: "false"
|
|
136
|
+
INFRAHUB_CACHE_ADDRESS: ${INFRAHUB_TESTING_CACHE_ADDRESS}
|
|
137
|
+
PREFECT_MESSAGING_BROKER: prefect_redis.messaging
|
|
138
|
+
PREFECT_MESSAGING_CACHE: prefect_redis.messaging
|
|
139
|
+
PREFECT_REDIS_MESSAGING_HOST: "${INFRAHUB_TESTING_CACHE_ADDRESS:-cache}"
|
|
140
|
+
PREFECT_REDIS_MESSAGING_DB: "1"
|
|
141
|
+
PREFECT_REDIS_MESSAGING_CONSUMER_MIN_IDLE_TIME: "30"
|
|
142
|
+
PREFECT_REDIS_MESSAGING_PUBLISHER_BATCH_SIZE: "1"
|
|
143
|
+
|
|
144
|
+
PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED: "true"
|
|
145
|
+
PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED: "true"
|
|
146
|
+
PREFECT_SERVER_SERVICES_TRIGGERS_ENABLED: "true"
|
|
147
|
+
PREFECT_SERVER_SERVICES_TASK_RUN_RECORDER_ENABLED: "true"
|
|
148
|
+
|
|
96
149
|
task-manager-db:
|
|
97
150
|
image: "${POSTGRES_DOCKER_IMAGE:-postgres:16-alpine}"
|
|
151
|
+
command: postgres -c 'max_connections=${INFRAHUB_TESTING_TASK_MANAGER_DB_MAX_CONNECTIONS:-100}'
|
|
98
152
|
environment:
|
|
99
153
|
- POSTGRES_USER=postgres
|
|
100
154
|
- POSTGRES_PASSWORD=postgres
|
|
@@ -172,6 +226,7 @@ services:
|
|
|
172
226
|
INFRAHUB_DB_ADDRESS: ${INFRAHUB_TESTING_DB_ADDRESS:-database}
|
|
173
227
|
INFRAHUB_WORKFLOW_ADDRESS: ${INFRAHUB_TESTING_WORKFLOW_ADDRESS}
|
|
174
228
|
INFRAHUB_TIMEOUT: ${INFRAHUB_TESTING_TIMEOUT}
|
|
229
|
+
INFRAHUB_PAGINATION_SIZE: ${INFRAHUB_TESTING_PAGINATION_SIZE:-50}
|
|
175
230
|
PREFECT_API_URL: ${INFRAHUB_TESTING_PREFECT_API}
|
|
176
231
|
INFRAHUB_EXPERIMENTAL_VALUE_DB_INDEX:
|
|
177
232
|
# Tracing
|
|
@@ -29,11 +29,14 @@ class TestInfrahubDocker:
|
|
|
29
29
|
return result.stdout
|
|
30
30
|
|
|
31
31
|
@staticmethod
|
|
32
|
-
def execute_command(
|
|
32
|
+
def execute_command(
|
|
33
|
+
command: str, address: str, concurrent_execution: int = 10, pagination_size: int = 50
|
|
34
|
+
) -> subprocess.CompletedProcess[str]:
|
|
33
35
|
env = os.environ.copy()
|
|
34
36
|
env["INFRAHUB_ADDRESS"] = address
|
|
35
37
|
env["INFRAHUB_API_TOKEN"] = PROJECT_ENV_VARIABLES["INFRAHUB_TESTING_INITIAL_ADMIN_TOKEN"]
|
|
36
38
|
env["INFRAHUB_MAX_CONCURRENT_EXECUTION"] = f"{concurrent_execution}"
|
|
39
|
+
env["INFRAHUB_PAGINATION_SIZE"] = f"{pagination_size}"
|
|
37
40
|
result = subprocess.run( # noqa: S602
|
|
38
41
|
command, shell=True, capture_output=True, text=True, env=env, check=False
|
|
39
42
|
)
|
|
@@ -1,284 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from collections import defaultdict
|
|
4
|
-
from dataclasses import dataclass
|
|
5
|
-
from typing import TYPE_CHECKING, Any
|
|
6
|
-
|
|
7
|
-
from rich import print as rprint
|
|
8
|
-
from rich.console import Console
|
|
9
|
-
from rich.table import Table
|
|
10
|
-
|
|
11
|
-
from infrahub.core import registry
|
|
12
|
-
from infrahub.core.branch.models import Branch
|
|
13
|
-
from infrahub.core.constants import InfrahubKind
|
|
14
|
-
from infrahub.core.migrations.query.node_duplicate import NodeDuplicateQuery, SchemaNodeInfo
|
|
15
|
-
from infrahub.core.query import Query, QueryType
|
|
16
|
-
from infrahub.core.schema import SchemaRoot, internal_schema
|
|
17
|
-
from infrahub.core.schema.manager import SchemaManager
|
|
18
|
-
from infrahub.log import get_logger
|
|
19
|
-
|
|
20
|
-
from ..constants import FAILED_BADGE, SUCCESS_BADGE
|
|
21
|
-
|
|
22
|
-
if TYPE_CHECKING:
|
|
23
|
-
from infrahub.core.schema.node_schema import NodeSchema
|
|
24
|
-
from infrahub.database import InfrahubDatabase
|
|
25
|
-
|
|
26
|
-
log = get_logger()
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class GetSchemaWithUpdatedInheritance(Query):
|
|
30
|
-
"""
|
|
31
|
-
Get the name, namespace, and branch of any SchemaNodes with _updated_ inheritance
|
|
32
|
-
This query will only return schemas that have had `inherit_from` updated after they were created
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
name = "get_schema_with_updated_inheritance"
|
|
36
|
-
type = QueryType.READ
|
|
37
|
-
insert_return = False
|
|
38
|
-
|
|
39
|
-
async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
|
|
40
|
-
query = """
|
|
41
|
-
// find inherit_from attributes that have been updated
|
|
42
|
-
MATCH p = (schema_node:SchemaNode)-[has_attr_e:HAS_ATTRIBUTE {status: "active"}]->(a:Attribute {name: "inherit_from"})
|
|
43
|
-
WHERE has_attr_e.to IS NULL
|
|
44
|
-
CALL (a) {
|
|
45
|
-
// only get branches on which the value was updated, we can ignore the initial create
|
|
46
|
-
MATCH (a)-[e:HAS_VALUE]->(:AttributeValue)
|
|
47
|
-
ORDER BY e.from ASC
|
|
48
|
-
// tail leaves out the earliest one, which is the initial create
|
|
49
|
-
RETURN tail(collect(e.branch)) AS branches
|
|
50
|
-
}
|
|
51
|
-
WITH schema_node, a, branches
|
|
52
|
-
WHERE size(branches) > 0
|
|
53
|
-
UNWIND branches AS branch
|
|
54
|
-
WITH DISTINCT schema_node, a, branch
|
|
55
|
-
|
|
56
|
-
//get branch details
|
|
57
|
-
CALL (branch) {
|
|
58
|
-
MATCH (b:Branch {name: branch})
|
|
59
|
-
RETURN b.branched_from AS branched_from, b.hierarchy_level AS branch_level
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// get the namespace for the schema
|
|
63
|
-
CALL (schema_node, a, branch, branched_from, branch_level) {
|
|
64
|
-
MATCH (schema_node)-[e1:HAS_ATTRIBUTE]-(:Attribute {name: "namespace"})-[e2:HAS_VALUE]->(av)
|
|
65
|
-
WHERE (
|
|
66
|
-
e1.branch = branch OR
|
|
67
|
-
(e1.branch_level < branch_level AND e1.from <= branched_from)
|
|
68
|
-
) AND e1.to IS NULL
|
|
69
|
-
AND e1.status = "active"
|
|
70
|
-
AND (
|
|
71
|
-
e2.branch = branch OR
|
|
72
|
-
(e2.branch_level < branch_level AND e2.from <= branched_from)
|
|
73
|
-
) AND e2.to IS NULL
|
|
74
|
-
AND e2.status = "active"
|
|
75
|
-
ORDER BY e2.branch_level DESC, e1.branch_level DESC, e2.from DESC, e1.from DESC
|
|
76
|
-
RETURN av.value AS namespace
|
|
77
|
-
LIMIT 1
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
// get the name for the schema
|
|
81
|
-
CALL (schema_node, a, branch, branched_from, branch_level) {
|
|
82
|
-
MATCH (schema_node)-[e1:HAS_ATTRIBUTE]-(:Attribute {name: "name"})-[e2:HAS_VALUE]->(av)
|
|
83
|
-
WHERE (
|
|
84
|
-
e1.branch = branch OR
|
|
85
|
-
(e1.branch_level < branch_level AND e1.from <= branched_from)
|
|
86
|
-
) AND e1.to IS NULL
|
|
87
|
-
AND e1.status = "active"
|
|
88
|
-
AND (
|
|
89
|
-
e2.branch = branch OR
|
|
90
|
-
(e2.branch_level < branch_level AND e2.from <= branched_from)
|
|
91
|
-
) AND e2.to IS NULL
|
|
92
|
-
AND e2.status = "active"
|
|
93
|
-
ORDER BY e2.branch_level DESC, e1.branch_level DESC, e2.from DESC, e1.from DESC
|
|
94
|
-
RETURN av.value AS name
|
|
95
|
-
LIMIT 1
|
|
96
|
-
}
|
|
97
|
-
RETURN name, namespace, branch
|
|
98
|
-
"""
|
|
99
|
-
self.return_labels = ["name", "namespace", "branch"]
|
|
100
|
-
self.add_to_query(query)
|
|
101
|
-
|
|
102
|
-
def get_updated_inheritance_kinds_by_branch(self) -> dict[str, list[str]]:
|
|
103
|
-
kinds_by_branch: dict[str, list[str]] = defaultdict(list)
|
|
104
|
-
for result in self.results:
|
|
105
|
-
name = result.get_as_type(label="name", return_type=str)
|
|
106
|
-
namespace = result.get_as_type(label="namespace", return_type=str)
|
|
107
|
-
branch = result.get_as_type(label="branch", return_type=str)
|
|
108
|
-
kinds_by_branch[branch].append(f"{namespace}{name}")
|
|
109
|
-
return kinds_by_branch
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
@dataclass
|
|
113
|
-
class KindLabelCount:
|
|
114
|
-
kind: str
|
|
115
|
-
labels: frozenset[str]
|
|
116
|
-
num_nodes: int
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
@dataclass
|
|
120
|
-
class KindLabelCountCorrected(KindLabelCount):
|
|
121
|
-
node_schema: NodeSchema
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
class GetAllKindsAndLabels(Query):
|
|
125
|
-
"""
|
|
126
|
-
Get the kind, labels, and number of nodes for the given kinds and branch
|
|
127
|
-
"""
|
|
128
|
-
|
|
129
|
-
name = "get_all_kinds_and_labels"
|
|
130
|
-
type = QueryType.READ
|
|
131
|
-
insert_return = False
|
|
132
|
-
|
|
133
|
-
def __init__(self, kinds: list[str] | None = None, **kwargs: Any) -> None:
|
|
134
|
-
super().__init__(**kwargs)
|
|
135
|
-
self.kinds = kinds
|
|
136
|
-
|
|
137
|
-
async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
|
|
138
|
-
self.params["branch_name"] = self.branch.name
|
|
139
|
-
self.params["branched_from"] = self.branch.get_branched_from()
|
|
140
|
-
self.params["branch_level"] = self.branch.hierarchy_level
|
|
141
|
-
kinds_str = "Node"
|
|
142
|
-
if self.kinds:
|
|
143
|
-
kinds_str = "|".join(self.kinds)
|
|
144
|
-
query = """
|
|
145
|
-
MATCH (n:%(kinds_str)s)-[r:IS_PART_OF]->(:Root)
|
|
146
|
-
WHERE (
|
|
147
|
-
r.branch = $branch_name OR
|
|
148
|
-
(r.branch_level < $branch_level AND r.from <= $branched_from)
|
|
149
|
-
)
|
|
150
|
-
AND r.to IS NULL
|
|
151
|
-
AND r.status = "active"
|
|
152
|
-
RETURN DISTINCT n.kind AS kind, labels(n) AS labels, count(*) AS num_nodes
|
|
153
|
-
ORDER BY kind ASC
|
|
154
|
-
""" % {"kinds_str": kinds_str}
|
|
155
|
-
self.return_labels = ["kind", "labels", "num_nodes"]
|
|
156
|
-
self.add_to_query(query)
|
|
157
|
-
|
|
158
|
-
def get_kind_label_counts(self) -> list[KindLabelCount]:
|
|
159
|
-
kind_label_counts: list[KindLabelCount] = []
|
|
160
|
-
for result in self.results:
|
|
161
|
-
kind = result.get_as_type(label="kind", return_type=str)
|
|
162
|
-
num_nodes = result.get_as_type(label="num_nodes", return_type=int)
|
|
163
|
-
labels: list[str] = result.get_as_type(label="labels", return_type=list)
|
|
164
|
-
# we can ignore the Node label and the label that matches the kind
|
|
165
|
-
cleaned_labels = frozenset(str(lbl) for lbl in labels if lbl not in ["Node", "CoreNode", kind])
|
|
166
|
-
kind_label_counts.append(KindLabelCount(kind=kind, labels=cleaned_labels, num_nodes=num_nodes))
|
|
167
|
-
return kind_label_counts
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
def display_kind_label_counts(kind_label_counts_by_branch: dict[str, list[KindLabelCountCorrected]]) -> None:
|
|
171
|
-
console = Console()
|
|
172
|
-
|
|
173
|
-
table = Table(title="Incorrect Inheritance Nodes")
|
|
174
|
-
|
|
175
|
-
table.add_column("Branch")
|
|
176
|
-
table.add_column("Kind")
|
|
177
|
-
table.add_column("Incorrect Labels")
|
|
178
|
-
table.add_column("Num Nodes")
|
|
179
|
-
|
|
180
|
-
for branch_name, kind_label_counts in kind_label_counts_by_branch.items():
|
|
181
|
-
for kind_label_count in kind_label_counts:
|
|
182
|
-
table.add_row(
|
|
183
|
-
branch_name, kind_label_count.kind, str(list(kind_label_count.labels)), str(kind_label_count.num_nodes)
|
|
184
|
-
)
|
|
185
|
-
|
|
186
|
-
console.print(table)
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
async def check_inheritance(db: InfrahubDatabase, fix: bool = False) -> bool:
|
|
190
|
-
"""
|
|
191
|
-
Run migrations to update the inheritance of any nodes with incorrect inheritance from a failed migration
|
|
192
|
-
1. Identifies node schemas that have had their inheritance updated after they were created
|
|
193
|
-
a. includes the kind and branch of the inheritance update
|
|
194
|
-
2. Checks nodes of the given kinds on the given branch to verify their inheritance is correct
|
|
195
|
-
3. Displays counts of any kinds with incorrect inheritance on the given branch
|
|
196
|
-
4. If fix is True, runs migrations to update the inheritance of any nodes with incorrect inheritance
|
|
197
|
-
on the correct branch
|
|
198
|
-
"""
|
|
199
|
-
|
|
200
|
-
updated_inheritance_query = await GetSchemaWithUpdatedInheritance.init(db=db)
|
|
201
|
-
await updated_inheritance_query.execute(db=db)
|
|
202
|
-
updated_inheritance_kinds_by_branch = updated_inheritance_query.get_updated_inheritance_kinds_by_branch()
|
|
203
|
-
|
|
204
|
-
if not updated_inheritance_kinds_by_branch:
|
|
205
|
-
rprint(f"{SUCCESS_BADGE} No schemas have had their inheritance updated")
|
|
206
|
-
return True
|
|
207
|
-
|
|
208
|
-
schema_manager = SchemaManager()
|
|
209
|
-
registry.schema = schema_manager
|
|
210
|
-
schema = SchemaRoot(**internal_schema)
|
|
211
|
-
schema_manager.register_schema(schema=schema)
|
|
212
|
-
branches_by_name = {b.name: b for b in await Branch.get_list(db=db)}
|
|
213
|
-
|
|
214
|
-
kind_label_counts_by_branch: dict[str, list[KindLabelCountCorrected]] = defaultdict(list)
|
|
215
|
-
for branch_name, kinds in updated_inheritance_kinds_by_branch.items():
|
|
216
|
-
rprint(f"Checking branch: {branch_name}", end="...")
|
|
217
|
-
branch = branches_by_name[branch_name]
|
|
218
|
-
schema_branch = await schema_manager.load_schema_from_db(db=db, branch=branch)
|
|
219
|
-
kind_label_query = await GetAllKindsAndLabels.init(db=db, branch=branch, kinds=kinds)
|
|
220
|
-
await kind_label_query.execute(db=db)
|
|
221
|
-
kind_label_counts = kind_label_query.get_kind_label_counts()
|
|
222
|
-
|
|
223
|
-
for kind_label_count in kind_label_counts:
|
|
224
|
-
node_schema = schema_branch.get_node(name=kind_label_count.kind, duplicate=False)
|
|
225
|
-
correct_labels = frozenset(node_schema.inherit_from)
|
|
226
|
-
if kind_label_count.labels == correct_labels:
|
|
227
|
-
continue
|
|
228
|
-
|
|
229
|
-
kind_label_counts_by_branch[branch_name].append(
|
|
230
|
-
KindLabelCountCorrected(
|
|
231
|
-
kind=kind_label_count.kind,
|
|
232
|
-
labels=kind_label_count.labels,
|
|
233
|
-
num_nodes=kind_label_count.num_nodes,
|
|
234
|
-
node_schema=node_schema,
|
|
235
|
-
)
|
|
236
|
-
)
|
|
237
|
-
rprint("done")
|
|
238
|
-
|
|
239
|
-
if not kind_label_counts_by_branch:
|
|
240
|
-
rprint(f"{SUCCESS_BADGE} All nodes have the correct inheritance")
|
|
241
|
-
return True
|
|
242
|
-
|
|
243
|
-
display_kind_label_counts(kind_label_counts_by_branch)
|
|
244
|
-
|
|
245
|
-
if not fix:
|
|
246
|
-
rprint(f"{FAILED_BADGE} Use the --fix flag to fix the inheritance of any invalid nodes")
|
|
247
|
-
return False
|
|
248
|
-
|
|
249
|
-
for branch_name, kind_label_counts_corrected in kind_label_counts_by_branch.items():
|
|
250
|
-
for kind_label_count in kind_label_counts_corrected:
|
|
251
|
-
rprint(f"Fixing kind {kind_label_count.kind} on branch {branch_name}", end="...")
|
|
252
|
-
node_schema = kind_label_count.node_schema
|
|
253
|
-
migration_query = await NodeDuplicateQuery.init(
|
|
254
|
-
db=db,
|
|
255
|
-
branch=branches_by_name[branch_name],
|
|
256
|
-
previous_node=SchemaNodeInfo(
|
|
257
|
-
name=node_schema.name,
|
|
258
|
-
namespace=node_schema.namespace,
|
|
259
|
-
branch_support=node_schema.branch.value,
|
|
260
|
-
labels=list(kind_label_count.labels) + [kind_label_count.kind, InfrahubKind.NODE],
|
|
261
|
-
kind=kind_label_count.kind,
|
|
262
|
-
),
|
|
263
|
-
new_node=SchemaNodeInfo(
|
|
264
|
-
name=node_schema.name,
|
|
265
|
-
namespace=node_schema.namespace,
|
|
266
|
-
branch_support=node_schema.branch.value,
|
|
267
|
-
labels=list(node_schema.inherit_from) + [kind_label_count.kind, InfrahubKind.NODE],
|
|
268
|
-
kind=kind_label_count.kind,
|
|
269
|
-
),
|
|
270
|
-
)
|
|
271
|
-
await migration_query.execute(db=db)
|
|
272
|
-
rprint("done")
|
|
273
|
-
|
|
274
|
-
rprint(f"{SUCCESS_BADGE} All nodes have the correct inheritance")
|
|
275
|
-
|
|
276
|
-
if registry.default_branch in kind_label_counts_by_branch:
|
|
277
|
-
kinds = [kind_label_count.kind for kind_label_count in kind_label_counts_by_branch[registry.default_branch]]
|
|
278
|
-
rprint(
|
|
279
|
-
"[bold cyan]Note that migrations were run on the default branch for the following schema kinds: "
|
|
280
|
-
f"{', '.join(kinds)}. You should rebase any branches that include/will include changes using "
|
|
281
|
-
"the migrated schemas[/bold cyan]"
|
|
282
|
-
)
|
|
283
|
-
|
|
284
|
-
return True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|