infrahub-server 1.2.11__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/actions/constants.py +130 -0
- infrahub/actions/gather.py +114 -0
- infrahub/actions/models.py +243 -0
- infrahub/actions/parsers.py +104 -0
- infrahub/actions/schema.py +393 -0
- infrahub/actions/tasks.py +119 -0
- infrahub/actions/triggers.py +21 -0
- infrahub/branch/__init__.py +0 -0
- infrahub/branch/tasks.py +29 -0
- infrahub/branch/triggers.py +22 -0
- infrahub/cli/db.py +3 -4
- infrahub/computed_attribute/gather.py +3 -1
- infrahub/computed_attribute/tasks.py +23 -29
- infrahub/core/account.py +24 -47
- infrahub/core/attribute.py +13 -15
- infrahub/core/constants/__init__.py +10 -0
- infrahub/core/constants/database.py +1 -0
- infrahub/core/constants/infrahubkind.py +9 -0
- infrahub/core/constraint/node/runner.py +3 -1
- infrahub/core/convert_object_type/__init__.py +0 -0
- infrahub/core/convert_object_type/conversion.py +124 -0
- infrahub/core/convert_object_type/schema_mapping.py +56 -0
- infrahub/core/diff/coordinator.py +8 -1
- infrahub/core/diff/query/all_conflicts.py +1 -5
- infrahub/core/diff/query/artifact.py +10 -20
- infrahub/core/diff/query/delete_query.py +8 -4
- infrahub/core/diff/query/diff_get.py +3 -6
- infrahub/core/diff/query/field_specifiers.py +1 -1
- infrahub/core/diff/query/field_summary.py +2 -4
- infrahub/core/diff/query/merge.py +72 -125
- infrahub/core/diff/query/save.py +83 -68
- infrahub/core/diff/query/summary_counts_enricher.py +34 -54
- infrahub/core/diff/query/time_range_query.py +0 -1
- infrahub/core/diff/repository/repository.py +4 -0
- infrahub/core/graph/__init__.py +1 -1
- infrahub/core/manager.py +14 -11
- infrahub/core/migrations/graph/__init__.py +6 -0
- infrahub/core/migrations/graph/m003_relationship_parent_optional.py +1 -2
- infrahub/core/migrations/graph/m012_convert_account_generic.py +1 -1
- infrahub/core/migrations/graph/m013_convert_git_password_credential.py +2 -6
- infrahub/core/migrations/graph/m015_diff_format_update.py +1 -2
- infrahub/core/migrations/graph/m016_diff_delete_bug_fix.py +1 -2
- infrahub/core/migrations/graph/m019_restore_rels_to_time.py +11 -22
- infrahub/core/migrations/graph/m020_duplicate_edges.py +3 -6
- infrahub/core/migrations/graph/m021_missing_hierarchy_merge.py +1 -2
- infrahub/core/migrations/graph/m023_deduplicate_cardinality_one_relationships.py +2 -2
- infrahub/core/migrations/graph/m024_missing_hierarchy_backfill.py +1 -2
- infrahub/core/migrations/graph/m028_delete_diffs.py +1 -2
- infrahub/core/migrations/graph/m029_duplicates_cleanup.py +662 -0
- infrahub/core/migrations/graph/m030_illegal_edges.py +82 -0
- infrahub/core/migrations/query/attribute_add.py +14 -11
- infrahub/core/migrations/query/attribute_rename.py +6 -11
- infrahub/core/migrations/query/delete_element_in_schema.py +19 -17
- infrahub/core/migrations/query/node_duplicate.py +19 -21
- infrahub/core/migrations/query/relationship_duplicate.py +19 -18
- infrahub/core/migrations/schema/node_attribute_remove.py +4 -8
- infrahub/core/migrations/schema/node_remove.py +19 -20
- infrahub/core/models.py +29 -2
- infrahub/core/node/__init__.py +131 -28
- infrahub/core/node/base.py +1 -1
- infrahub/core/node/create.py +211 -0
- infrahub/core/node/resource_manager/number_pool.py +31 -5
- infrahub/core/node/standard.py +6 -1
- infrahub/core/path.py +15 -1
- infrahub/core/protocols.py +57 -0
- infrahub/core/protocols_base.py +3 -0
- infrahub/core/query/__init__.py +2 -2
- infrahub/core/query/delete.py +3 -3
- infrahub/core/query/diff.py +19 -32
- infrahub/core/query/ipam.py +10 -20
- infrahub/core/query/node.py +29 -47
- infrahub/core/query/relationship.py +55 -34
- infrahub/core/query/resource_manager.py +1 -2
- infrahub/core/query/standard_node.py +19 -5
- infrahub/core/query/subquery.py +2 -4
- infrahub/core/relationship/constraints/count.py +10 -9
- infrahub/core/relationship/constraints/interface.py +2 -1
- infrahub/core/relationship/constraints/peer_kind.py +2 -1
- infrahub/core/relationship/constraints/peer_parent.py +56 -0
- infrahub/core/relationship/constraints/peer_relatives.py +72 -0
- infrahub/core/relationship/constraints/profiles_kind.py +1 -1
- infrahub/core/relationship/model.py +4 -1
- infrahub/core/schema/__init__.py +2 -1
- infrahub/core/schema/attribute_parameters.py +160 -0
- infrahub/core/schema/attribute_schema.py +130 -7
- infrahub/core/schema/basenode_schema.py +27 -3
- infrahub/core/schema/definitions/core/__init__.py +29 -1
- infrahub/core/schema/definitions/core/group.py +45 -0
- infrahub/core/schema/definitions/core/resource_pool.py +9 -0
- infrahub/core/schema/definitions/internal.py +43 -5
- infrahub/core/schema/generated/attribute_schema.py +16 -3
- infrahub/core/schema/generated/relationship_schema.py +11 -1
- infrahub/core/schema/manager.py +7 -2
- infrahub/core/schema/schema_branch.py +109 -12
- infrahub/core/validators/__init__.py +15 -2
- infrahub/core/validators/attribute/choices.py +1 -3
- infrahub/core/validators/attribute/enum.py +1 -3
- infrahub/core/validators/attribute/kind.py +1 -3
- infrahub/core/validators/attribute/length.py +13 -7
- infrahub/core/validators/attribute/min_max.py +118 -0
- infrahub/core/validators/attribute/number_pool.py +106 -0
- infrahub/core/validators/attribute/optional.py +1 -4
- infrahub/core/validators/attribute/regex.py +5 -6
- infrahub/core/validators/attribute/unique.py +1 -3
- infrahub/core/validators/determiner.py +18 -2
- infrahub/core/validators/enum.py +12 -0
- infrahub/core/validators/node/hierarchy.py +3 -6
- infrahub/core/validators/query.py +1 -3
- infrahub/core/validators/relationship/count.py +6 -12
- infrahub/core/validators/relationship/optional.py +2 -4
- infrahub/core/validators/relationship/peer.py +177 -12
- infrahub/core/validators/tasks.py +1 -1
- infrahub/core/validators/uniqueness/query.py +5 -9
- infrahub/database/__init__.py +12 -4
- infrahub/database/validation.py +100 -0
- infrahub/dependencies/builder/constraint/grouped/node_runner.py +4 -0
- infrahub/dependencies/builder/constraint/relationship_manager/peer_parent.py +8 -0
- infrahub/dependencies/builder/constraint/relationship_manager/peer_relatives.py +8 -0
- infrahub/dependencies/builder/constraint/schema/aggregated.py +2 -0
- infrahub/dependencies/builder/constraint/schema/relationship_peer.py +8 -0
- infrahub/dependencies/builder/diff/deserializer.py +1 -1
- infrahub/dependencies/registry.py +4 -0
- infrahub/events/group_action.py +1 -0
- infrahub/events/models.py +1 -1
- infrahub/git/base.py +5 -3
- infrahub/git/integrator.py +96 -5
- infrahub/git/tasks.py +1 -0
- infrahub/graphql/analyzer.py +139 -18
- infrahub/graphql/manager.py +4 -0
- infrahub/graphql/mutations/action.py +164 -0
- infrahub/graphql/mutations/convert_object_type.py +71 -0
- infrahub/graphql/mutations/main.py +25 -176
- infrahub/graphql/mutations/proposed_change.py +20 -17
- infrahub/graphql/mutations/relationship.py +32 -0
- infrahub/graphql/mutations/resource_manager.py +63 -7
- infrahub/graphql/queries/convert_object_type_mapping.py +34 -0
- infrahub/graphql/queries/resource_manager.py +7 -1
- infrahub/graphql/resolvers/many_relationship.py +1 -1
- infrahub/graphql/resolvers/resolver.py +2 -2
- infrahub/graphql/resolvers/single_relationship.py +1 -1
- infrahub/graphql/schema.py +6 -0
- infrahub/menu/menu.py +34 -2
- infrahub/message_bus/messages/__init__.py +0 -10
- infrahub/message_bus/operations/__init__.py +0 -8
- infrahub/message_bus/operations/refresh/registry.py +4 -7
- infrahub/patch/queries/delete_duplicated_edges.py +45 -39
- infrahub/pools/models.py +14 -0
- infrahub/pools/number.py +5 -3
- infrahub/pools/registration.py +22 -0
- infrahub/pools/tasks.py +126 -0
- infrahub/prefect_server/models.py +1 -19
- infrahub/proposed_change/models.py +68 -3
- infrahub/proposed_change/tasks.py +911 -34
- infrahub/schema/__init__.py +0 -0
- infrahub/schema/tasks.py +27 -0
- infrahub/schema/triggers.py +23 -0
- infrahub/task_manager/models.py +10 -6
- infrahub/trigger/catalogue.py +6 -0
- infrahub/trigger/models.py +23 -6
- infrahub/trigger/setup.py +26 -2
- infrahub/trigger/tasks.py +4 -2
- infrahub/types.py +6 -0
- infrahub/webhook/tasks.py +6 -9
- infrahub/workflows/catalogue.py +103 -1
- infrahub_sdk/client.py +43 -10
- infrahub_sdk/ctl/generator.py +4 -4
- infrahub_sdk/ctl/repository.py +1 -1
- infrahub_sdk/node/__init__.py +39 -0
- infrahub_sdk/node/attribute.py +122 -0
- infrahub_sdk/node/constants.py +21 -0
- infrahub_sdk/{node.py → node/node.py} +158 -803
- infrahub_sdk/node/parsers.py +15 -0
- infrahub_sdk/node/property.py +24 -0
- infrahub_sdk/node/related_node.py +266 -0
- infrahub_sdk/node/relationship.py +302 -0
- infrahub_sdk/protocols.py +112 -0
- infrahub_sdk/protocols_base.py +34 -2
- infrahub_sdk/pytest_plugin/items/python_transform.py +2 -1
- infrahub_sdk/query_groups.py +17 -5
- infrahub_sdk/schema/main.py +1 -0
- infrahub_sdk/schema/repository.py +16 -0
- infrahub_sdk/spec/object.py +1 -1
- infrahub_sdk/store.py +1 -1
- infrahub_sdk/testing/schemas/car_person.py +1 -0
- infrahub_sdk/utils.py +7 -20
- infrahub_sdk/yaml.py +6 -5
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/METADATA +5 -5
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/RECORD +197 -168
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/WHEEL +1 -1
- infrahub_testcontainers/container.py +239 -65
- infrahub_testcontainers/docker-compose-cluster.test.yml +321 -0
- infrahub_testcontainers/docker-compose.test.yml +2 -1
- infrahub_testcontainers/helpers.py +23 -3
- infrahub_testcontainers/plugin.py +9 -0
- infrahub/message_bus/messages/check_generator_run.py +0 -26
- infrahub/message_bus/messages/finalize_validator_execution.py +0 -15
- infrahub/message_bus/messages/proposed_change/base_with_diff.py +0 -16
- infrahub/message_bus/messages/proposed_change/request_proposedchange_refreshartifacts.py +0 -11
- infrahub/message_bus/messages/request_generatordefinition_check.py +0 -20
- infrahub/message_bus/messages/request_proposedchange_pipeline.py +0 -23
- infrahub/message_bus/operations/check/__init__.py +0 -3
- infrahub/message_bus/operations/check/generator.py +0 -156
- infrahub/message_bus/operations/finalize/__init__.py +0 -3
- infrahub/message_bus/operations/finalize/validator.py +0 -133
- infrahub/message_bus/operations/requests/__init__.py +0 -9
- infrahub/message_bus/operations/requests/generator_definition.py +0 -140
- infrahub/message_bus/operations/requests/proposed_change.py +0 -629
- infrahub/patch/queries/consolidate_duplicated_nodes.py +0 -109
- /infrahub/{message_bus/messages/proposed_change → actions}/__init__.py +0 -0
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/LICENSE.txt +0 -0
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,28 +1,18 @@
|
|
|
1
1
|
from infrahub.message_bus import InfrahubMessage, InfrahubResponse
|
|
2
2
|
|
|
3
|
-
from .check_generator_run import CheckGeneratorRun
|
|
4
|
-
from .finalize_validator_execution import FinalizeValidatorExecution
|
|
5
3
|
from .git_file_get import GitFileGet, GitFileGetResponse
|
|
6
4
|
from .git_repository_connectivity import GitRepositoryConnectivity
|
|
7
|
-
from .proposed_change.request_proposedchange_refreshartifacts import RequestProposedChangeRefreshArtifacts
|
|
8
5
|
from .refresh_git_fetch import RefreshGitFetch
|
|
9
6
|
from .refresh_registry_branches import RefreshRegistryBranches
|
|
10
7
|
from .refresh_registry_rebasedbranch import RefreshRegistryRebasedBranch
|
|
11
|
-
from .request_generatordefinition_check import RequestGeneratorDefinitionCheck
|
|
12
|
-
from .request_proposedchange_pipeline import RequestProposedChangePipeline
|
|
13
8
|
from .send_echo_request import SendEchoRequest, SendEchoRequestResponse
|
|
14
9
|
|
|
15
10
|
MESSAGE_MAP: dict[str, type[InfrahubMessage]] = {
|
|
16
|
-
"check.generator.run": CheckGeneratorRun,
|
|
17
|
-
"finalize.validator.execution": FinalizeValidatorExecution,
|
|
18
11
|
"git.file.get": GitFileGet,
|
|
19
12
|
"git.repository.connectivity": GitRepositoryConnectivity,
|
|
20
13
|
"refresh.git.fetch": RefreshGitFetch,
|
|
21
14
|
"refresh.registry.branches": RefreshRegistryBranches,
|
|
22
15
|
"refresh.registry.rebased_branch": RefreshRegistryRebasedBranch,
|
|
23
|
-
"request.generator_definition.check": RequestGeneratorDefinitionCheck,
|
|
24
|
-
"request.proposed_change.pipeline": RequestProposedChangePipeline,
|
|
25
|
-
"request.proposed_change.refresh_artifacts": RequestProposedChangeRefreshArtifacts,
|
|
26
16
|
"send.echo.request": SendEchoRequest,
|
|
27
17
|
}
|
|
28
18
|
|
|
@@ -3,11 +3,8 @@ from prefect import Flow
|
|
|
3
3
|
|
|
4
4
|
from infrahub.message_bus import RPCErrorResponse, messages
|
|
5
5
|
from infrahub.message_bus.operations import (
|
|
6
|
-
check,
|
|
7
|
-
finalize,
|
|
8
6
|
git,
|
|
9
7
|
refresh,
|
|
10
|
-
requests,
|
|
11
8
|
send,
|
|
12
9
|
)
|
|
13
10
|
from infrahub.message_bus.types import MessageTTL
|
|
@@ -15,16 +12,11 @@ from infrahub.services import InfrahubServices
|
|
|
15
12
|
from infrahub.tasks.check import set_check_status
|
|
16
13
|
|
|
17
14
|
COMMAND_MAP = {
|
|
18
|
-
"check.generator.run": check.generator.run,
|
|
19
|
-
"finalize.validator.execution": finalize.validator.execution,
|
|
20
15
|
"git.file.get": git.file.get,
|
|
21
16
|
"git.repository.connectivity": git.repository.connectivity,
|
|
22
17
|
"refresh.git.fetch": git.repository.fetch,
|
|
23
18
|
"refresh.registry.branches": refresh.registry.branches,
|
|
24
19
|
"refresh.registry.rebased_branch": refresh.registry.rebased_branch,
|
|
25
|
-
"request.generator_definition.check": requests.generator_definition.check,
|
|
26
|
-
"request.proposed_change.pipeline": requests.proposed_change.pipeline,
|
|
27
|
-
"request.proposed_change.refresh_artifacts": requests.proposed_change.refresh_artifacts,
|
|
28
20
|
"send.echo.request": send.echo.request,
|
|
29
21
|
}
|
|
30
22
|
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
from infrahub import lock
|
|
2
|
-
from infrahub.core.registry import registry
|
|
3
1
|
from infrahub.message_bus import messages
|
|
4
2
|
from infrahub.services import InfrahubServices
|
|
5
3
|
from infrahub.tasks.registry import refresh_branches
|
|
@@ -24,8 +22,7 @@ async def rebased_branch(message: messages.RefreshRegistryRebasedBranch, service
|
|
|
24
22
|
)
|
|
25
23
|
return
|
|
26
24
|
|
|
27
|
-
async with
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
)
|
|
25
|
+
async with service.database.start_session(read_only=True) as db:
|
|
26
|
+
await refresh_branches(db=db)
|
|
27
|
+
|
|
28
|
+
await service.component.refresh_schema_hash()
|
|
@@ -4,7 +4,7 @@ from .base import PatchQuery
|
|
|
4
4
|
|
|
5
5
|
class DeleteDuplicatedEdgesPatchQuery(PatchQuery):
|
|
6
6
|
"""
|
|
7
|
-
|
|
7
|
+
For all Node vertices, find duplicated or overlapping edges of the same status, type, and branch to update and delete
|
|
8
8
|
- one edge will be kept for each pair of nodes and a given status, type, and branch. it will be
|
|
9
9
|
updated to have the earliest "from" and "to" times in this group
|
|
10
10
|
- all the other duplicate/overlapping edges will be deleted
|
|
@@ -17,99 +17,105 @@ class DeleteDuplicatedEdgesPatchQuery(PatchQuery):
|
|
|
17
17
|
async def plan(self) -> PatchPlan:
|
|
18
18
|
query = """
|
|
19
19
|
// ------------
|
|
20
|
-
// Find
|
|
20
|
+
// Find vertex pairs that have duplicate edges
|
|
21
21
|
// ------------
|
|
22
|
-
MATCH (node_with_dup_edges:Node)-[edge]
|
|
22
|
+
MATCH (node_with_dup_edges:Node)-[edge]-(peer)
|
|
23
23
|
WITH node_with_dup_edges, type(edge) AS edge_type, edge.status AS edge_status, edge.branch AS edge_branch, peer, count(*) AS num_dup_edges
|
|
24
24
|
WHERE num_dup_edges > 1
|
|
25
25
|
WITH DISTINCT node_with_dup_edges, edge_type, edge_branch, peer
|
|
26
|
-
CALL {
|
|
26
|
+
CALL (node_with_dup_edges, edge_type, edge_branch, peer) {
|
|
27
27
|
// ------------
|
|
28
28
|
// Get the earliest active and deleted edges for this branch
|
|
29
29
|
// ------------
|
|
30
|
-
|
|
31
|
-
MATCH (node_with_dup_edges)-[active_edge {branch: edge_branch, status: "active"}]->(peer)
|
|
30
|
+
OPTIONAL MATCH (node_with_dup_edges)-[active_edge {branch: edge_branch, status: "active"}]->(peer)
|
|
32
31
|
WHERE type(active_edge) = edge_type
|
|
33
32
|
WITH node_with_dup_edges, edge_type, edge_branch, peer, active_edge
|
|
34
33
|
ORDER BY active_edge.from ASC
|
|
35
34
|
WITH node_with_dup_edges, edge_type, edge_branch, peer, head(collect(active_edge.from)) AS active_from
|
|
36
|
-
OPTIONAL MATCH (node_with_dup_edges)-[deleted_edge {branch: edge_branch, status: "deleted"}]
|
|
35
|
+
OPTIONAL MATCH (node_with_dup_edges)-[deleted_edge {branch: edge_branch, status: "deleted"}]-(peer)
|
|
37
36
|
WITH node_with_dup_edges, edge_type, edge_branch, peer, active_from, deleted_edge
|
|
38
37
|
ORDER BY deleted_edge.from ASC
|
|
39
38
|
WITH node_with_dup_edges, edge_type, edge_branch, peer, active_from, head(collect(deleted_edge.from)) AS deleted_from
|
|
40
39
|
// ------------
|
|
41
40
|
// Plan one active edge update with correct from and to times
|
|
42
41
|
// ------------
|
|
43
|
-
CALL {
|
|
44
|
-
|
|
45
|
-
MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
|
|
42
|
+
CALL (node_with_dup_edges, edge_type, edge_branch, peer, active_from, deleted_from) {
|
|
43
|
+
OPTIONAL MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
|
|
46
44
|
WHERE type(active_e) = edge_type
|
|
47
45
|
WITH node_with_dup_edges, edge_type, edge_branch, peer, active_from, deleted_from, active_e
|
|
48
46
|
ORDER BY %(id_func_name)s(active_e)
|
|
49
47
|
LIMIT 1
|
|
50
48
|
WITH active_e, properties(active_e) AS before_props, {from: active_from, to: deleted_from} AS prop_updates
|
|
51
|
-
RETURN
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
49
|
+
RETURN CASE
|
|
50
|
+
WHEN active_e IS NOT NULL THEN [
|
|
51
|
+
{
|
|
52
|
+
db_id: %(id_func_name)s(active_e), before_props: before_props, prop_updates: prop_updates
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
ELSE []
|
|
56
|
+
END AS active_edges_to_update
|
|
56
57
|
}
|
|
57
58
|
// ------------
|
|
58
59
|
// Plan deletes for all the other active edges of this type on this branch
|
|
59
60
|
// ------------
|
|
60
|
-
CALL {
|
|
61
|
-
|
|
62
|
-
MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
|
|
61
|
+
CALL (node_with_dup_edges, edge_type, edge_branch, peer) {
|
|
62
|
+
OPTIONAL MATCH (node_with_dup_edges)-[active_e {branch: edge_branch, status: "active"}]->(peer)
|
|
63
63
|
WHERE type(active_e) = edge_type
|
|
64
64
|
WITH node_with_dup_edges, peer, active_e
|
|
65
65
|
ORDER BY %(id_func_name)s(active_e)
|
|
66
66
|
SKIP 1
|
|
67
|
-
|
|
68
|
-
{
|
|
67
|
+
WITH CASE
|
|
68
|
+
WHEN active_e IS NOT NULL THEN {
|
|
69
69
|
db_id: %(id_func_name)s(active_e),
|
|
70
|
-
from_id: %(id_func_name)s(
|
|
71
|
-
to_id: %(id_func_name)s(
|
|
70
|
+
from_id: %(id_func_name)s(startNode(active_e)),
|
|
71
|
+
to_id: %(id_func_name)s(endNode(active_e)),
|
|
72
72
|
edge_type: type(active_e),
|
|
73
73
|
before_props: properties(active_e)
|
|
74
74
|
}
|
|
75
|
-
|
|
75
|
+
ELSE NULL
|
|
76
|
+
END AS serialized_edge
|
|
77
|
+
RETURN collect(serialized_edge) AS active_edges_to_delete
|
|
76
78
|
}
|
|
77
79
|
// ------------
|
|
78
80
|
// Plan one deleted edge update with correct from time
|
|
79
81
|
// ------------
|
|
80
|
-
CALL {
|
|
81
|
-
|
|
82
|
-
MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
|
|
82
|
+
CALL (node_with_dup_edges, edge_type, edge_branch, peer, deleted_from) {
|
|
83
|
+
OPTIONAL MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
|
|
83
84
|
WHERE type(deleted_e) = edge_type
|
|
84
85
|
WITH node_with_dup_edges, edge_type, edge_branch, peer, deleted_from, deleted_e
|
|
85
86
|
ORDER BY %(id_func_name)s(deleted_e)
|
|
86
87
|
LIMIT 1
|
|
87
88
|
WITH deleted_e, properties(deleted_e) AS before_props, {from: deleted_from} AS prop_updates
|
|
88
|
-
RETURN
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
89
|
+
RETURN CASE
|
|
90
|
+
WHEN deleted_e IS NOT NULL THEN [
|
|
91
|
+
{
|
|
92
|
+
db_id: %(id_func_name)s(deleted_e), before_props: before_props, prop_updates: prop_updates
|
|
93
|
+
}
|
|
94
|
+
]
|
|
95
|
+
ELSE []
|
|
96
|
+
END AS deleted_edges_to_update
|
|
93
97
|
}
|
|
94
98
|
// ------------
|
|
95
99
|
// Plan deletes for all the other deleted edges of this type on this branch
|
|
96
100
|
// ------------
|
|
97
|
-
CALL {
|
|
98
|
-
|
|
99
|
-
MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
|
|
101
|
+
CALL (node_with_dup_edges, edge_type, edge_branch, peer) {
|
|
102
|
+
OPTIONAL MATCH (node_with_dup_edges)-[deleted_e {branch: edge_branch, status: "deleted"}]->(peer)
|
|
100
103
|
WHERE type(deleted_e) = edge_type
|
|
101
104
|
WITH node_with_dup_edges, peer, deleted_e
|
|
102
105
|
ORDER BY %(id_func_name)s(deleted_e)
|
|
103
106
|
SKIP 1
|
|
104
|
-
|
|
105
|
-
{
|
|
107
|
+
WITH CASE
|
|
108
|
+
WHEN deleted_e IS NOT NULL THEN {
|
|
106
109
|
db_id: %(id_func_name)s(deleted_e),
|
|
107
|
-
from_id: %(id_func_name)s(
|
|
108
|
-
to_id: %(id_func_name)s(
|
|
110
|
+
from_id: %(id_func_name)s(startNode(deleted_e)),
|
|
111
|
+
to_id: %(id_func_name)s(endNode(deleted_e)),
|
|
109
112
|
edge_type: type(deleted_e),
|
|
110
113
|
before_props: properties(deleted_e)
|
|
111
114
|
}
|
|
112
|
-
|
|
115
|
+
ELSE NULL
|
|
116
|
+
END AS serialized_edge
|
|
117
|
+
|
|
118
|
+
RETURN collect(serialized_edge) AS deleted_edges_to_delete
|
|
113
119
|
}
|
|
114
120
|
RETURN
|
|
115
121
|
active_edges_to_update + deleted_edges_to_update AS edges_to_update,
|
infrahub/pools/models.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class NumberPoolLockDefinition:
|
|
6
|
+
pool_id: str
|
|
7
|
+
|
|
8
|
+
@property
|
|
9
|
+
def lock_name(self) -> str:
|
|
10
|
+
return f"number-pool-creation-{self.pool_id}"
|
|
11
|
+
|
|
12
|
+
@property
|
|
13
|
+
def namespace_name(self) -> str:
|
|
14
|
+
return "number-pool"
|
infrahub/pools/number.py
CHANGED
|
@@ -8,7 +8,7 @@ from infrahub.core.registry import registry
|
|
|
8
8
|
|
|
9
9
|
if TYPE_CHECKING:
|
|
10
10
|
from infrahub.core.branch import Branch
|
|
11
|
-
from infrahub.core.
|
|
11
|
+
from infrahub.core.node.resource_manager.number_pool import CoreNumberPool
|
|
12
12
|
from infrahub.core.timestamp import Timestamp
|
|
13
13
|
from infrahub.database import InfrahubDatabase
|
|
14
14
|
|
|
@@ -20,7 +20,9 @@ class UsedNumber:
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class NumberUtilizationGetter:
|
|
23
|
-
def __init__(
|
|
23
|
+
def __init__(
|
|
24
|
+
self, db: InfrahubDatabase, pool: CoreNumberPool, branch: Branch, at: Timestamp | str | None = None
|
|
25
|
+
) -> None:
|
|
24
26
|
self.db = db
|
|
25
27
|
self.at = at
|
|
26
28
|
self.pool = pool
|
|
@@ -62,4 +64,4 @@ class NumberUtilizationGetter:
|
|
|
62
64
|
|
|
63
65
|
@property
|
|
64
66
|
def total_pool_size(self) -> int:
|
|
65
|
-
return self.end_range - self.start_range + 1
|
|
67
|
+
return self.end_range - self.start_range + 1 - self.pool.get_attribute_nb_excluded_values()
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from infrahub.core.registry import registry
|
|
2
|
+
from infrahub.exceptions import SchemaNotFoundError
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def get_branches_with_schema_number_pool(kind: str, attribute_name: str) -> list[str]:
|
|
6
|
+
"""Return branches where schema defined NumberPool exists"""
|
|
7
|
+
|
|
8
|
+
registered_branches = []
|
|
9
|
+
active_branches = registry.schema.get_branches()
|
|
10
|
+
|
|
11
|
+
for active_branch in active_branches:
|
|
12
|
+
try:
|
|
13
|
+
schema = registry.schema.get(name=kind, branch=active_branch)
|
|
14
|
+
except SchemaNotFoundError:
|
|
15
|
+
continue
|
|
16
|
+
|
|
17
|
+
if attribute_name in schema.attribute_names:
|
|
18
|
+
attribute = schema.get_attribute(name=attribute_name)
|
|
19
|
+
if attribute.kind == "NumberPool":
|
|
20
|
+
registered_branches.append(active_branch)
|
|
21
|
+
|
|
22
|
+
return registered_branches
|
infrahub/pools/tasks.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from prefect import flow
|
|
4
|
+
from prefect.logging import get_run_logger
|
|
5
|
+
|
|
6
|
+
from infrahub import lock
|
|
7
|
+
from infrahub.context import InfrahubContext # noqa: TC001 needed for prefect flow
|
|
8
|
+
from infrahub.core.constants import InfrahubKind, NumberPoolType
|
|
9
|
+
from infrahub.core.manager import NodeManager
|
|
10
|
+
from infrahub.core.node import Node
|
|
11
|
+
from infrahub.core.protocols import CoreNumberPool
|
|
12
|
+
from infrahub.core.registry import registry
|
|
13
|
+
from infrahub.core.schema.attribute_parameters import NumberPoolParameters
|
|
14
|
+
from infrahub.exceptions import NodeNotFoundError
|
|
15
|
+
from infrahub.pools.models import NumberPoolLockDefinition
|
|
16
|
+
from infrahub.pools.registration import get_branches_with_schema_number_pool
|
|
17
|
+
from infrahub.services import InfrahubServices # noqa: TC001 needed for prefect flow
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@flow(
|
|
21
|
+
name="validate-schema-number-pools",
|
|
22
|
+
flow_run_name="Validate schema number pools on {branch_name}",
|
|
23
|
+
)
|
|
24
|
+
async def validate_schema_number_pools(
|
|
25
|
+
branch_name: str, # noqa: ARG001
|
|
26
|
+
context: InfrahubContext, # noqa: ARG001
|
|
27
|
+
service: InfrahubServices,
|
|
28
|
+
) -> None:
|
|
29
|
+
log = get_run_logger()
|
|
30
|
+
|
|
31
|
+
async with service.database.start_session() as dbs:
|
|
32
|
+
schema_number_pools = await NodeManager.query(
|
|
33
|
+
db=dbs, schema=CoreNumberPool, filters={"pool_type__value": NumberPoolType.SCHEMA.value}
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
for schema_number_pool in list(schema_number_pools):
|
|
37
|
+
defined_on_branches = get_branches_with_schema_number_pool(
|
|
38
|
+
kind=schema_number_pool.node.value, attribute_name=schema_number_pool.node_attribute.value
|
|
39
|
+
)
|
|
40
|
+
if registry.default_branch in defined_on_branches:
|
|
41
|
+
schema = registry.schema.get(name=schema_number_pool.node.value, branch=registry.default_branch)
|
|
42
|
+
attribute = schema.get_attribute(name=schema_number_pool.node_attribute.value)
|
|
43
|
+
number_pool_updated = False
|
|
44
|
+
if isinstance(attribute.parameters, NumberPoolParameters):
|
|
45
|
+
if schema_number_pool.start_range.value != attribute.parameters.start_range:
|
|
46
|
+
schema_number_pool.start_range.value = attribute.parameters.start_range
|
|
47
|
+
number_pool_updated = True
|
|
48
|
+
if schema_number_pool.end_range.value != attribute.parameters.end_range:
|
|
49
|
+
schema_number_pool.end_range.value = attribute.parameters.end_range
|
|
50
|
+
number_pool_updated = True
|
|
51
|
+
|
|
52
|
+
if number_pool_updated:
|
|
53
|
+
log.info(
|
|
54
|
+
f"Updating NumberPool={schema_number_pool.id} based on changes in the schema on {registry.default_branch}"
|
|
55
|
+
)
|
|
56
|
+
await schema_number_pool.save(db=service.database)
|
|
57
|
+
|
|
58
|
+
elif not defined_on_branches:
|
|
59
|
+
log.info(f"Deleting number pool (id={schema_number_pool.id}) as it is no longer defined in the schema")
|
|
60
|
+
await schema_number_pool.delete(db=service.database)
|
|
61
|
+
|
|
62
|
+
existing_pool_ids = [pool.id for pool in schema_number_pools]
|
|
63
|
+
for registry_branch in registry.schema.get_branches():
|
|
64
|
+
schema_branch = service.database.schema.get_schema_branch(name=registry_branch)
|
|
65
|
+
|
|
66
|
+
for generic_name in schema_branch.generic_names:
|
|
67
|
+
generic_node = schema_branch.get_generic(name=generic_name, duplicate=False)
|
|
68
|
+
for attribute_name in generic_node.attribute_names:
|
|
69
|
+
attribute = generic_node.get_attribute(name=attribute_name)
|
|
70
|
+
if isinstance(attribute.parameters, NumberPoolParameters) and attribute.parameters.number_pool_id:
|
|
71
|
+
if attribute.parameters.number_pool_id not in existing_pool_ids:
|
|
72
|
+
await _create_number_pool(
|
|
73
|
+
service=service,
|
|
74
|
+
number_pool_id=attribute.parameters.number_pool_id,
|
|
75
|
+
pool_node=generic_node.kind,
|
|
76
|
+
pool_attribute=attribute_name,
|
|
77
|
+
start_range=attribute.parameters.start_range,
|
|
78
|
+
end_range=attribute.parameters.end_range,
|
|
79
|
+
)
|
|
80
|
+
existing_pool_ids.append(attribute.parameters.number_pool_id)
|
|
81
|
+
|
|
82
|
+
for node_name in schema_branch.node_names:
|
|
83
|
+
node = schema_branch.get_node(name=node_name, duplicate=False)
|
|
84
|
+
for attribute_name in node.attribute_names:
|
|
85
|
+
attribute = node.get_attribute(name=attribute_name)
|
|
86
|
+
if isinstance(attribute.parameters, NumberPoolParameters) and attribute.parameters.number_pool_id:
|
|
87
|
+
if attribute.parameters.number_pool_id not in existing_pool_ids:
|
|
88
|
+
await _create_number_pool(
|
|
89
|
+
service=service,
|
|
90
|
+
number_pool_id=attribute.parameters.number_pool_id,
|
|
91
|
+
pool_node=node.kind,
|
|
92
|
+
pool_attribute=attribute_name,
|
|
93
|
+
start_range=attribute.parameters.start_range,
|
|
94
|
+
end_range=attribute.parameters.end_range,
|
|
95
|
+
)
|
|
96
|
+
existing_pool_ids.append(attribute.parameters.number_pool_id)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
async def _create_number_pool(
|
|
100
|
+
service: InfrahubServices,
|
|
101
|
+
number_pool_id: str,
|
|
102
|
+
pool_node: str,
|
|
103
|
+
pool_attribute: str,
|
|
104
|
+
start_range: int,
|
|
105
|
+
end_range: int,
|
|
106
|
+
) -> None:
|
|
107
|
+
lock_definition = NumberPoolLockDefinition(pool_id=number_pool_id)
|
|
108
|
+
async with lock.registry.get(name=lock_definition.lock_name, namespace=lock_definition.namespace_name, local=False):
|
|
109
|
+
async with service.database.start_session() as dbs:
|
|
110
|
+
try:
|
|
111
|
+
await registry.manager.get_one_by_id_or_default_filter(
|
|
112
|
+
db=dbs, id=str(number_pool_id), kind=CoreNumberPool
|
|
113
|
+
)
|
|
114
|
+
except NodeNotFoundError:
|
|
115
|
+
number_pool = await Node.init(db=dbs, schema=InfrahubKind.NUMBERPOOL, branch=registry.default_branch)
|
|
116
|
+
await number_pool.new(
|
|
117
|
+
db=dbs,
|
|
118
|
+
id=number_pool_id,
|
|
119
|
+
name=f"{pool_node}.{pool_attribute} [{number_pool_id}]",
|
|
120
|
+
node=pool_node,
|
|
121
|
+
node_attribute=pool_attribute,
|
|
122
|
+
start_range=start_range,
|
|
123
|
+
end_range=end_range,
|
|
124
|
+
pool_type=NumberPoolType.SCHEMA.value,
|
|
125
|
+
)
|
|
126
|
+
await number_pool.save(db=dbs)
|
|
@@ -1,18 +1,10 @@
|
|
|
1
|
-
from
|
|
2
|
-
|
|
3
|
-
from prefect.server.database import PrefectDBInterface, db_injector
|
|
4
|
-
from prefect.server.events.filters import EventFilter, EventNameFilter, EventOrder, EventRelatedFilter
|
|
1
|
+
from prefect.server.events.filters import EventFilter, EventNameFilter, EventOrder
|
|
5
2
|
from prefect.server.events.schemas.events import ReceivedEvent
|
|
6
3
|
from prefect.server.utilities.schemas import PrefectBaseModel
|
|
7
4
|
from pydantic import BaseModel, Field
|
|
8
5
|
|
|
9
|
-
if TYPE_CHECKING:
|
|
10
|
-
from sqlalchemy.sql.expression import ColumnExpressionArgument
|
|
11
|
-
|
|
12
6
|
|
|
13
7
|
class InfrahubEventFilter(EventFilter):
|
|
14
|
-
matching_related: list[EventRelatedFilter] = Field(default_factory=list)
|
|
15
|
-
|
|
16
8
|
def set_prefix(self) -> None:
|
|
17
9
|
if self.event:
|
|
18
10
|
if self.event.prefix is not None and "infrahub." not in self.event.prefix:
|
|
@@ -20,16 +12,6 @@ class InfrahubEventFilter(EventFilter):
|
|
|
20
12
|
else:
|
|
21
13
|
self.event = EventNameFilter(prefix=["infrahub."], name=[], exclude_prefix=None, exclude_name=None)
|
|
22
14
|
|
|
23
|
-
@db_injector
|
|
24
|
-
def build_where_clauses(self, db: PrefectDBInterface) -> Sequence["ColumnExpressionArgument[bool]"]:
|
|
25
|
-
result = cast(list["ColumnExpressionArgument[bool]"], super().build_where_clauses())
|
|
26
|
-
top_level_filter = self._scoped_event_resources(db)
|
|
27
|
-
for matching_related in self.matching_related:
|
|
28
|
-
matching_related._top_level_filter = top_level_filter
|
|
29
|
-
result.extend(matching_related.build_where_clauses())
|
|
30
|
-
|
|
31
|
-
return result
|
|
32
|
-
|
|
33
15
|
@classmethod
|
|
34
16
|
def default(cls) -> "InfrahubEventFilter":
|
|
35
17
|
return cls(event=None, any_resource=None, resource=None, related=None, order=EventOrder.DESC)
|
|
@@ -1,10 +1,25 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
|
|
1
3
|
from pydantic import BaseModel, ConfigDict, Field
|
|
2
4
|
|
|
3
|
-
from infrahub.
|
|
4
|
-
from infrahub.
|
|
5
|
+
from infrahub.core.constants import CheckType
|
|
6
|
+
from infrahub.generators.models import ProposedChangeGeneratorDefinition
|
|
7
|
+
from infrahub.message_bus import InfrahubMessage
|
|
5
8
|
from infrahub.message_bus.types import ProposedChangeArtifactDefinition, ProposedChangeBranchDiff
|
|
6
9
|
|
|
7
10
|
|
|
11
|
+
class BaseProposedChangeWithDiffMessage(InfrahubMessage):
|
|
12
|
+
"""Sent trigger the refresh of artifacts that are impacted by the proposed change."""
|
|
13
|
+
|
|
14
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
15
|
+
|
|
16
|
+
proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
|
|
17
|
+
source_branch: str = Field(..., description="The source branch of the proposed change")
|
|
18
|
+
source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
|
|
19
|
+
destination_branch: str = Field(..., description="The destination branch of the proposed change")
|
|
20
|
+
branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")
|
|
21
|
+
|
|
22
|
+
|
|
8
23
|
class RequestProposedChangeDataIntegrity(BaseProposedChangeWithDiffMessage):
|
|
9
24
|
"""Sent trigger data integrity checks for a proposed change"""
|
|
10
25
|
|
|
@@ -42,4 +57,54 @@ class RequestArtifactDefinitionCheck(BaseModel):
|
|
|
42
57
|
source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
|
|
43
58
|
destination_branch: str = Field(..., description="The target branch")
|
|
44
59
|
|
|
45
|
-
|
|
60
|
+
|
|
61
|
+
class RunGeneratorAsCheckModel(BaseModel):
|
|
62
|
+
"""A check that runs a generator."""
|
|
63
|
+
|
|
64
|
+
generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator definition")
|
|
65
|
+
generator_instance: str | None = Field(
|
|
66
|
+
default=None, description="The id of the generator instance if it previously existed"
|
|
67
|
+
)
|
|
68
|
+
commit: str = Field(..., description="The commit to target")
|
|
69
|
+
repository_id: str = Field(..., description="The unique ID of the Repository")
|
|
70
|
+
repository_name: str = Field(..., description="The name of the Repository")
|
|
71
|
+
repository_kind: str = Field(..., description="The kind of the Repository")
|
|
72
|
+
branch_name: str = Field(..., description="The branch where the check is run")
|
|
73
|
+
target_id: str = Field(..., description="The ID of the target object for this generator")
|
|
74
|
+
target_name: str = Field(..., description="Name of the generator target")
|
|
75
|
+
query: str = Field(..., description="The name of the query to use when collecting data")
|
|
76
|
+
variables: dict = Field(..., description="Input variables when running the generator")
|
|
77
|
+
validator_id: str = Field(..., description="The ID of the validator")
|
|
78
|
+
proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class RequestGeneratorDefinitionCheck(BaseModel):
|
|
82
|
+
"""Sent to trigger Generators to run for a proposed change."""
|
|
83
|
+
|
|
84
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
85
|
+
|
|
86
|
+
generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator Definition")
|
|
87
|
+
branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")
|
|
88
|
+
proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
|
|
89
|
+
source_branch: str = Field(..., description="The source branch")
|
|
90
|
+
source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
|
|
91
|
+
destination_branch: str = Field(..., description="The target branch")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class RequestProposedChangePipeline(BaseModel):
|
|
95
|
+
"""Sent request the start of a pipeline connected to a proposed change."""
|
|
96
|
+
|
|
97
|
+
proposed_change: str = Field(..., description="The unique ID of the proposed change")
|
|
98
|
+
source_branch: str = Field(..., description="The source branch of the proposed change")
|
|
99
|
+
source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
|
|
100
|
+
destination_branch: str = Field(..., description="The destination branch of the proposed change")
|
|
101
|
+
check_type: CheckType = Field(
|
|
102
|
+
default=CheckType.ALL, description="Can be used to restrict the pipeline to a specific type of job"
|
|
103
|
+
)
|
|
104
|
+
pipeline_id: uuid.UUID = Field(
|
|
105
|
+
default_factory=uuid.uuid4, description="The unique ID of the execution of this pipeline"
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class RequestProposedChangeRefreshArtifacts(BaseProposedChangeWithDiffMessage):
|
|
110
|
+
"""Sent trigger the refresh of artifacts that are impacted by the proposed change."""
|