infrahub-server 1.7.0rc0__py3-none-any.whl → 1.7.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/actions/gather.py +2 -2
- infrahub/api/query.py +3 -2
- infrahub/api/schema.py +5 -0
- infrahub/api/transformation.py +3 -3
- infrahub/cli/db.py +6 -2
- infrahub/computed_attribute/gather.py +2 -0
- infrahub/config.py +2 -2
- infrahub/core/attribute.py +21 -2
- infrahub/core/branch/models.py +11 -117
- infrahub/core/branch/tasks.py +7 -3
- infrahub/core/diff/merger/merger.py +5 -1
- infrahub/core/diff/model/path.py +43 -0
- infrahub/core/graph/__init__.py +1 -1
- infrahub/core/graph/index.py +2 -0
- infrahub/core/initialization.py +2 -1
- infrahub/core/ipam/resource_allocator.py +229 -0
- infrahub/core/migrations/graph/__init__.py +10 -0
- infrahub/core/migrations/graph/m014_remove_index_attr_value.py +3 -2
- infrahub/core/migrations/graph/m015_diff_format_update.py +3 -2
- infrahub/core/migrations/graph/m016_diff_delete_bug_fix.py +3 -2
- infrahub/core/migrations/graph/m017_add_core_profile.py +6 -4
- infrahub/core/migrations/graph/m018_uniqueness_nulls.py +3 -4
- infrahub/core/migrations/graph/m020_duplicate_edges.py +3 -3
- infrahub/core/migrations/graph/m025_uniqueness_nulls.py +3 -4
- infrahub/core/migrations/graph/m026_0000_prefix_fix.py +4 -5
- infrahub/core/migrations/graph/m028_delete_diffs.py +3 -2
- infrahub/core/migrations/graph/m029_duplicates_cleanup.py +3 -2
- infrahub/core/migrations/graph/m031_check_number_attributes.py +4 -3
- infrahub/core/migrations/graph/m032_cleanup_orphaned_branch_relationships.py +3 -2
- infrahub/core/migrations/graph/m034_find_orphaned_schema_fields.py +3 -2
- infrahub/core/migrations/graph/m035_orphan_relationships.py +3 -3
- infrahub/core/migrations/graph/m036_drop_attr_value_index.py +3 -2
- infrahub/core/migrations/graph/m037_index_attr_vals.py +3 -2
- infrahub/core/migrations/graph/m038_redo_0000_prefix_fix.py +4 -5
- infrahub/core/migrations/graph/m039_ipam_reconcile.py +3 -2
- infrahub/core/migrations/graph/m041_deleted_dup_edges.py +3 -2
- infrahub/core/migrations/graph/m042_profile_attrs_in_db.py +5 -4
- infrahub/core/migrations/graph/m043_create_hfid_display_label_in_db.py +12 -5
- infrahub/core/migrations/graph/m044_backfill_hfid_display_label_in_db.py +15 -4
- infrahub/core/migrations/graph/m045_backfill_hfid_display_label_in_db_profile_template.py +10 -4
- infrahub/core/migrations/graph/m046_fill_agnostic_hfid_display_labels.py +6 -5
- infrahub/core/migrations/graph/m047_backfill_or_null_display_label.py +19 -5
- infrahub/core/migrations/graph/m048_undelete_rel_props.py +6 -4
- infrahub/core/migrations/graph/m049_remove_is_visible_relationship.py +3 -3
- infrahub/core/migrations/graph/m050_backfill_vertex_metadata.py +3 -3
- infrahub/core/migrations/graph/m051_subtract_branched_from_microsecond.py +39 -0
- infrahub/core/migrations/graph/m052_fix_global_branch_level.py +51 -0
- infrahub/core/migrations/graph/m053_fix_branch_level_zero.py +61 -0
- infrahub/core/migrations/graph/m054_cleanup_orphaned_nodes.py +87 -0
- infrahub/core/migrations/graph/m055_remove_webhook_validate_certificates_default.py +86 -0
- infrahub/core/migrations/runner.py +6 -3
- infrahub/core/migrations/schema/attribute_kind_update.py +8 -11
- infrahub/core/migrations/schema/attribute_supports_profile.py +3 -8
- infrahub/core/migrations/schema/models.py +8 -0
- infrahub/core/migrations/schema/node_attribute_add.py +24 -29
- infrahub/core/migrations/schema/tasks.py +7 -1
- infrahub/core/migrations/shared.py +37 -30
- infrahub/core/node/__init__.py +2 -1
- infrahub/core/node/lock_utils.py +23 -2
- infrahub/core/node/resource_manager/ip_address_pool.py +5 -11
- infrahub/core/node/resource_manager/ip_prefix_pool.py +5 -21
- infrahub/core/node/resource_manager/number_pool.py +109 -39
- infrahub/core/query/__init__.py +7 -1
- infrahub/core/query/branch.py +18 -2
- infrahub/core/query/ipam.py +629 -40
- infrahub/core/query/node.py +128 -0
- infrahub/core/query/resource_manager.py +114 -1
- infrahub/core/relationship/model.py +9 -3
- infrahub/core/schema/attribute_parameters.py +28 -1
- infrahub/core/schema/attribute_schema.py +9 -2
- infrahub/core/schema/definitions/core/webhook.py +0 -1
- infrahub/core/schema/definitions/internal.py +7 -4
- infrahub/core/schema/manager.py +50 -38
- infrahub/core/validators/attribute/kind.py +5 -2
- infrahub/core/validators/determiner.py +4 -0
- infrahub/graphql/analyzer.py +3 -1
- infrahub/graphql/app.py +7 -10
- infrahub/graphql/execution.py +95 -0
- infrahub/graphql/manager.py +8 -2
- infrahub/graphql/mutations/proposed_change.py +15 -0
- infrahub/graphql/parser.py +10 -7
- infrahub/graphql/queries/ipam.py +20 -25
- infrahub/graphql/queries/search.py +29 -9
- infrahub/lock.py +7 -0
- infrahub/proposed_change/tasks.py +2 -0
- infrahub/services/adapters/cache/redis.py +7 -0
- infrahub/services/adapters/http/httpx.py +27 -0
- infrahub/trigger/catalogue.py +2 -0
- infrahub/trigger/models.py +73 -4
- infrahub/trigger/setup.py +1 -1
- infrahub/trigger/system.py +36 -0
- infrahub/webhook/models.py +4 -2
- infrahub/webhook/tasks.py +2 -2
- infrahub/workflows/initialization.py +2 -2
- infrahub_sdk/analyzer.py +2 -2
- infrahub_sdk/branch.py +12 -39
- infrahub_sdk/checks.py +4 -4
- infrahub_sdk/client.py +36 -0
- infrahub_sdk/ctl/cli_commands.py +2 -1
- infrahub_sdk/ctl/graphql.py +15 -4
- infrahub_sdk/ctl/utils.py +2 -2
- infrahub_sdk/enums.py +6 -0
- infrahub_sdk/graphql/renderers.py +21 -0
- infrahub_sdk/graphql/utils.py +85 -0
- infrahub_sdk/node/attribute.py +12 -2
- infrahub_sdk/node/constants.py +11 -0
- infrahub_sdk/node/metadata.py +69 -0
- infrahub_sdk/node/node.py +65 -14
- infrahub_sdk/node/property.py +3 -0
- infrahub_sdk/node/related_node.py +24 -1
- infrahub_sdk/node/relationship.py +10 -1
- infrahub_sdk/operation.py +2 -2
- infrahub_sdk/schema/repository.py +1 -2
- infrahub_sdk/transforms.py +2 -2
- infrahub_sdk/types.py +18 -2
- {infrahub_server-1.7.0rc0.dist-info → infrahub_server-1.7.2.dist-info}/METADATA +8 -8
- {infrahub_server-1.7.0rc0.dist-info → infrahub_server-1.7.2.dist-info}/RECORD +123 -114
- {infrahub_server-1.7.0rc0.dist-info → infrahub_server-1.7.2.dist-info}/entry_points.txt +0 -1
- infrahub_testcontainers/docker-compose-cluster.test.yml +16 -10
- infrahub_testcontainers/docker-compose.test.yml +11 -10
- infrahub_testcontainers/performance_test.py +1 -1
- infrahub/pools/address.py +0 -16
- {infrahub_server-1.7.0rc0.dist-info → infrahub_server-1.7.2.dist-info}/WHEEL +0 -0
- {infrahub_server-1.7.0rc0.dist-info → infrahub_server-1.7.2.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Sequence
|
|
4
|
+
|
|
5
|
+
from infrahub.core.migrations.shared import GraphMigration, MigrationInput, MigrationResult
|
|
6
|
+
|
|
7
|
+
from ...query import Query, QueryType
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from infrahub.database import InfrahubDatabase
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CleanupOrphanedNodesQuery(Query):
|
|
14
|
+
"""
|
|
15
|
+
Clean up orphaned Node vertices (no IS_PART_OF edge to Root) and their linked
|
|
16
|
+
Attributes and Relationships.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
name = "cleanup_orphaned_nodes"
|
|
20
|
+
type = QueryType.WRITE
|
|
21
|
+
insert_return = False
|
|
22
|
+
raise_error_if_empty = False
|
|
23
|
+
|
|
24
|
+
async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
|
|
25
|
+
query = """
|
|
26
|
+
// Delete attributes of orphaned nodes
|
|
27
|
+
MATCH (n:Node)
|
|
28
|
+
WHERE NOT exists((n)-[:IS_PART_OF]->(:Root))
|
|
29
|
+
OPTIONAL MATCH (n)-[:HAS_ATTRIBUTE]->(attr:Attribute)
|
|
30
|
+
WITH DISTINCT attr
|
|
31
|
+
CALL (attr) {
|
|
32
|
+
DETACH DELETE attr
|
|
33
|
+
} IN TRANSACTIONS
|
|
34
|
+
|
|
35
|
+
// reduce the results to a single row
|
|
36
|
+
WITH 1 AS one
|
|
37
|
+
LIMIT 1
|
|
38
|
+
|
|
39
|
+
// Delete relationships that will have < 2 Node peers after orphaned node removal
|
|
40
|
+
OPTIONAL MATCH (orphan:Node)-[:IS_RELATED]-(rel:Relationship)
|
|
41
|
+
WHERE NOT exists((orphan)-[:IS_PART_OF]->(:Root))
|
|
42
|
+
WITH DISTINCT rel
|
|
43
|
+
CALL (rel) {
|
|
44
|
+
OPTIONAL MATCH (rel)-[:IS_RELATED]-(peer:Node)
|
|
45
|
+
WHERE exists((peer)-[:IS_PART_OF]->(:Root))
|
|
46
|
+
WITH rel, count(peer) AS remaining_peers
|
|
47
|
+
WHERE remaining_peers < 2
|
|
48
|
+
DETACH DELETE rel
|
|
49
|
+
} IN TRANSACTIONS
|
|
50
|
+
|
|
51
|
+
// reduce the results to a single row
|
|
52
|
+
WITH 1 AS one
|
|
53
|
+
LIMIT 1
|
|
54
|
+
|
|
55
|
+
// Delete the orphaned nodes
|
|
56
|
+
MATCH (n:Node)
|
|
57
|
+
WHERE NOT exists((n)-[:IS_PART_OF]->(:Root))
|
|
58
|
+
CALL (n) {
|
|
59
|
+
DETACH DELETE n
|
|
60
|
+
} IN TRANSACTIONS
|
|
61
|
+
"""
|
|
62
|
+
self.add_to_query(query)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Migration054(GraphMigration):
|
|
66
|
+
"""
|
|
67
|
+
Clean up orphaned Node vertices that have no IS_PART_OF edge to Root.
|
|
68
|
+
|
|
69
|
+
This can happen when a branch-aware node is deleted during branch deletion,
|
|
70
|
+
but its branch-agnostic attributes or relationships are not properly cleaned up.
|
|
71
|
+
|
|
72
|
+
The migration:
|
|
73
|
+
1. DETACH DELETEs Attributes linked to orphaned nodes
|
|
74
|
+
2. DETACH DELETEs Relationships that would have < 2 Node peers after orphaned node removal
|
|
75
|
+
3. DETACH DELETEs the orphaned nodes themselves
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
name: str = "054_cleanup_orphaned_nodes"
|
|
79
|
+
minimum_version: int = 53
|
|
80
|
+
queries: Sequence[type[Query]] = [CleanupOrphanedNodesQuery]
|
|
81
|
+
|
|
82
|
+
async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult: # noqa: ARG002
|
|
83
|
+
return MigrationResult()
|
|
84
|
+
|
|
85
|
+
async def execute(self, migration_input: MigrationInput) -> MigrationResult:
|
|
86
|
+
# Override parent class to skip transaction in case there are many nodes to delete
|
|
87
|
+
return await self.do_execute(migration_input=migration_input)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Sequence
|
|
4
|
+
|
|
5
|
+
from infrahub.core.constants import NULL_VALUE
|
|
6
|
+
from infrahub.core.migrations.shared import MigrationResult
|
|
7
|
+
from infrahub.core.query import Query, QueryType
|
|
8
|
+
|
|
9
|
+
from ..shared import GraphMigration
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from infrahub.database import InfrahubDatabase
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Migration055Query01(Query):
|
|
16
|
+
"""Remove the default_value from CoreWebhook's validate_certificates attribute.
|
|
17
|
+
|
|
18
|
+
This migration finds the CoreWebhook SchemaGeneric, locates its validate_certificates
|
|
19
|
+
SchemaAttribute, and sets the default_value to NULL. This removes the previous default
|
|
20
|
+
of True.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
name = "migration_055_01"
|
|
24
|
+
type: QueryType = QueryType.WRITE
|
|
25
|
+
|
|
26
|
+
async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
|
|
27
|
+
self.params["at"] = self.at.to_string()
|
|
28
|
+
self.params["null_value"] = NULL_VALUE
|
|
29
|
+
|
|
30
|
+
query = """
|
|
31
|
+
// get the generic schema
|
|
32
|
+
MATCH p1 = (sg:SchemaGeneric)-[:HAS_ATTRIBUTE]->(:Attribute {name: "name"})-[:HAS_VALUE]->(:AttributeValueIndexed {value: "Webhook"})
|
|
33
|
+
WHERE all(r IN relationships(p1) WHERE r.status = "active" AND r.to IS NULL)
|
|
34
|
+
|
|
35
|
+
// for safety, also check that the sg is in the namespace "Core"
|
|
36
|
+
MATCH p2 = (sg)-[:HAS_ATTRIBUTE]->(:Attribute {name: "namespace"})-[:HAS_VALUE]->(:AttributeValueIndexed {value: "Core"})
|
|
37
|
+
WHERE all(r IN relationships(p2) WHERE r.status = "active" AND r.to IS NULL)
|
|
38
|
+
|
|
39
|
+
// there should only be 1 CoreWebhook schema generic
|
|
40
|
+
WITH sg
|
|
41
|
+
LIMIT 1
|
|
42
|
+
|
|
43
|
+
// find the validate_certificates attribute
|
|
44
|
+
MATCH p3 = (sg)-[:IS_RELATED]-(:Relationship {name: "schema__node__attributes"})
|
|
45
|
+
-[:IS_RELATED]-(sa:SchemaAttribute)
|
|
46
|
+
-[:HAS_ATTRIBUTE]->(:Attribute {name: "name"})
|
|
47
|
+
-[:HAS_VALUE]->(:AttributeValueIndexed {value: "validate_certificates"})
|
|
48
|
+
WHERE all(r IN relationships(p3) WHERE r.status = "active" AND r.to IS NULL)
|
|
49
|
+
// there should only be 1 validate_certificates attribute
|
|
50
|
+
WITH sa
|
|
51
|
+
LIMIT 1
|
|
52
|
+
|
|
53
|
+
// get the default_value Attribute
|
|
54
|
+
MATCH (sa)-[ha:HAS_ATTRIBUTE]->(default_value_attr:Attribute {name: "default_value"})-[hv:HAS_VALUE]->(default_value)
|
|
55
|
+
WHERE all(r IN [ha, hv] WHERE r.status = "active" AND r.to IS NULL)
|
|
56
|
+
LIMIT 1
|
|
57
|
+
|
|
58
|
+
// skip if it is already NULL
|
|
59
|
+
WITH sa, default_value_attr, hv, default_value
|
|
60
|
+
WHERE default_value.value <> $null_value
|
|
61
|
+
|
|
62
|
+
// close the HAS_VALUE edge for the current default value
|
|
63
|
+
SET hv.to = $at
|
|
64
|
+
|
|
65
|
+
// get the new value
|
|
66
|
+
MERGE (new_value:AttributeValue:AttributeValueIndexed {value: $null_value, is_default: true})
|
|
67
|
+
LIMIT 1
|
|
68
|
+
|
|
69
|
+
// link the new value
|
|
70
|
+
CREATE (default_value_attr)-[new_hv:HAS_VALUE]->(new_value)
|
|
71
|
+
SET new_hv = properties(hv)
|
|
72
|
+
SET new_hv.from = $at, new_hv.to = NULL
|
|
73
|
+
"""
|
|
74
|
+
self.add_to_query(query)
|
|
75
|
+
self.return_labels = ["new_value"]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Migration055(GraphMigration):
|
|
79
|
+
name: str = "055_remove_webhook_validate_certificates_default"
|
|
80
|
+
queries: Sequence[type[Query]] = [Migration055Query01]
|
|
81
|
+
minimum_version: int = 54
|
|
82
|
+
|
|
83
|
+
async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult: # noqa: ARG002
|
|
84
|
+
result = MigrationResult()
|
|
85
|
+
|
|
86
|
+
return result
|
|
@@ -7,10 +7,11 @@ from infrahub.core.constants import GLOBAL_BRANCH_NAME
|
|
|
7
7
|
from infrahub.core.migrations.graph import MIGRATIONS
|
|
8
8
|
|
|
9
9
|
from .exceptions import MigrationFailureError
|
|
10
|
-
from .shared import MigrationRequiringRebase
|
|
10
|
+
from .shared import MigrationInput, MigrationRequiringRebase
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
13
|
from infrahub.core.branch import Branch
|
|
14
|
+
from infrahub.core.timestamp import Timestamp
|
|
14
15
|
from infrahub.database import InfrahubDatabase
|
|
15
16
|
|
|
16
17
|
|
|
@@ -35,12 +36,14 @@ class MigrationRunner:
|
|
|
35
36
|
def has_migrations(self) -> bool:
|
|
36
37
|
return bool(self.applicable_migrations)
|
|
37
38
|
|
|
38
|
-
async def run(self, db: InfrahubDatabase) -> None:
|
|
39
|
+
async def run(self, db: InfrahubDatabase, at: Timestamp) -> None:
|
|
39
40
|
if not self.has_migrations():
|
|
40
41
|
return
|
|
41
42
|
|
|
42
43
|
for migration in self.applicable_migrations:
|
|
43
|
-
execution_result = await migration.execute_against_branch(
|
|
44
|
+
execution_result = await migration.execute_against_branch(
|
|
45
|
+
migration_input=MigrationInput(db=db, at=at), branch=self.branch
|
|
46
|
+
)
|
|
44
47
|
validation_result = None
|
|
45
48
|
|
|
46
49
|
if execution_result.success:
|
|
@@ -2,15 +2,13 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from typing import TYPE_CHECKING, Any, Sequence
|
|
4
4
|
|
|
5
|
-
from infrahub.core.constants import SYSTEM_USER_ID
|
|
6
5
|
from infrahub.types import is_large_attribute_type
|
|
7
6
|
|
|
8
7
|
from ..query import AttributeMigrationQuery, MigrationBaseQuery
|
|
9
|
-
from ..shared import AttributeSchemaMigration, MigrationResult
|
|
8
|
+
from ..shared import AttributeSchemaMigration, MigrationInput, MigrationResult
|
|
10
9
|
|
|
11
10
|
if TYPE_CHECKING:
|
|
12
11
|
from infrahub.core.branch.models import Branch
|
|
13
|
-
from infrahub.core.timestamp import Timestamp
|
|
14
12
|
from infrahub.database import InfrahubDatabase
|
|
15
13
|
|
|
16
14
|
|
|
@@ -40,7 +38,7 @@ class AttributeKindUpdateMigrationQuery(AttributeMigrationQuery):
|
|
|
40
38
|
// ------------
|
|
41
39
|
// start with all the Attribute vertices we might care about
|
|
42
40
|
// ------------
|
|
43
|
-
MATCH (n:%(
|
|
41
|
+
MATCH (n:%(schema_kinds)s)-[:HAS_ATTRIBUTE]->(attr:Attribute)
|
|
44
42
|
WHERE attr.name = $attr_name
|
|
45
43
|
WITH DISTINCT n, attr
|
|
46
44
|
|
|
@@ -76,7 +74,7 @@ CALL (av_is_default, av_value) {
|
|
|
76
74
|
// ------------
|
|
77
75
|
WITH 1 AS one
|
|
78
76
|
LIMIT 1
|
|
79
|
-
MATCH (n:%(
|
|
77
|
+
MATCH (n:%(schema_kinds)s)-[:HAS_ATTRIBUTE]->(attr:Attribute)
|
|
80
78
|
WHERE attr.name = $attr_name
|
|
81
79
|
WITH DISTINCT n, attr
|
|
82
80
|
|
|
@@ -94,7 +92,6 @@ CALL (n, attr) {
|
|
|
94
92
|
RETURN has_value_e, av
|
|
95
93
|
}
|
|
96
94
|
|
|
97
|
-
|
|
98
95
|
// ------------
|
|
99
96
|
// create and update the HAS_VALUE edges
|
|
100
97
|
// ------------
|
|
@@ -154,7 +151,9 @@ CALL (attr, n) {
|
|
|
154
151
|
SET n.updated_at = $at, n.updated_by = $user_id
|
|
155
152
|
}
|
|
156
153
|
""" % {
|
|
157
|
-
"
|
|
154
|
+
"schema_kinds": (
|
|
155
|
+
f"{self.migration.previous_schema.kind}|Profile{self.migration.previous_schema.kind}|Template{self.migration.previous_schema.kind}"
|
|
156
|
+
),
|
|
158
157
|
"branch_filter": branch_filter,
|
|
159
158
|
"new_attr_value_labels": new_attr_value_labels,
|
|
160
159
|
}
|
|
@@ -167,15 +166,13 @@ class AttributeKindUpdateMigration(AttributeSchemaMigration):
|
|
|
167
166
|
|
|
168
167
|
async def execute(
|
|
169
168
|
self,
|
|
170
|
-
|
|
169
|
+
migration_input: MigrationInput,
|
|
171
170
|
branch: Branch,
|
|
172
|
-
at: Timestamp | str | None = None,
|
|
173
171
|
queries: Sequence[type[MigrationBaseQuery]] | None = None,
|
|
174
|
-
user_id: str = SYSTEM_USER_ID,
|
|
175
172
|
) -> MigrationResult:
|
|
176
173
|
is_indexed_previous = is_large_attribute_type(self.previous_attribute_schema.kind)
|
|
177
174
|
is_indexed_new = is_large_attribute_type(self.new_attribute_schema.kind)
|
|
178
175
|
if is_indexed_previous is is_indexed_new:
|
|
179
176
|
return MigrationResult()
|
|
180
177
|
|
|
181
|
-
return await super().execute(
|
|
178
|
+
return await super().execute(migration_input=migration_input, branch=branch, queries=queries)
|
|
@@ -2,20 +2,17 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from typing import TYPE_CHECKING, Any, Sequence
|
|
4
4
|
|
|
5
|
-
from infrahub.core.constants import SYSTEM_USER_ID
|
|
6
5
|
from infrahub.core.migrations.query.attribute_remove import AttributeRemoveQuery
|
|
7
6
|
from infrahub.core.schema.generic_schema import GenericSchema
|
|
8
7
|
from infrahub.core.schema.node_schema import NodeSchema
|
|
9
8
|
|
|
10
9
|
from ..query import AttributeMigrationQuery, MigrationBaseQuery
|
|
11
10
|
from ..query.attribute_add import AttributeAddQuery
|
|
12
|
-
from ..shared import AttributeSchemaMigration, MigrationResult
|
|
11
|
+
from ..shared import AttributeSchemaMigration, MigrationInput, MigrationResult
|
|
13
12
|
|
|
14
13
|
if TYPE_CHECKING:
|
|
15
14
|
from infrahub.core.branch.models import Branch
|
|
16
15
|
from infrahub.core.schema import MainSchemaTypes
|
|
17
|
-
from infrahub.core.timestamp import Timestamp
|
|
18
|
-
from infrahub.database import InfrahubDatabase
|
|
19
16
|
|
|
20
17
|
|
|
21
18
|
def _get_node_kinds(schema: MainSchemaTypes) -> list[str]:
|
|
@@ -70,11 +67,9 @@ class AttributeSupportsProfileUpdateMigration(AttributeSchemaMigration):
|
|
|
70
67
|
|
|
71
68
|
async def execute(
|
|
72
69
|
self,
|
|
73
|
-
|
|
70
|
+
migration_input: MigrationInput,
|
|
74
71
|
branch: Branch,
|
|
75
|
-
at: Timestamp | str | None = None,
|
|
76
72
|
queries: Sequence[type[MigrationBaseQuery]] | None = None, # noqa: ARG002
|
|
77
|
-
user_id: str = SYSTEM_USER_ID,
|
|
78
73
|
) -> MigrationResult:
|
|
79
74
|
if (
|
|
80
75
|
# no change in whether the attribute should be used on profiles
|
|
@@ -89,4 +84,4 @@ class AttributeSupportsProfileUpdateMigration(AttributeSchemaMigration):
|
|
|
89
84
|
if not self.new_attribute_schema.support_profiles:
|
|
90
85
|
profiles_queries.append(ProfilesAttributeRemoveMigrationQuery)
|
|
91
86
|
|
|
92
|
-
return await super().execute(
|
|
87
|
+
return await super().execute(migration_input=migration_input, branch=branch, queries=profiles_queries)
|
|
@@ -7,6 +7,7 @@ from infrahub.core.constants import SYSTEM_USER_ID
|
|
|
7
7
|
from infrahub.core.models import SchemaUpdateMigrationInfo
|
|
8
8
|
from infrahub.core.path import SchemaPath
|
|
9
9
|
from infrahub.core.schema.schema_branch import SchemaBranch
|
|
10
|
+
from infrahub.core.timestamp import Timestamp
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
class SchemaApplyMigrationData(BaseModel):
|
|
@@ -16,6 +17,7 @@ class SchemaApplyMigrationData(BaseModel):
|
|
|
16
17
|
new_schema: SchemaBranch
|
|
17
18
|
previous_schema: SchemaBranch
|
|
18
19
|
migrations: list[SchemaUpdateMigrationInfo]
|
|
20
|
+
at: Timestamp
|
|
19
21
|
user_id: str = SYSTEM_USER_ID
|
|
20
22
|
|
|
21
23
|
@model_serializer()
|
|
@@ -26,6 +28,7 @@ class SchemaApplyMigrationData(BaseModel):
|
|
|
26
28
|
"new_schema": self.new_schema.to_dict_schema_object(),
|
|
27
29
|
"migrations": [migration.model_dump() for migration in self.migrations],
|
|
28
30
|
"user_id": self.user_id,
|
|
31
|
+
"at": self.at.to_string(),
|
|
29
32
|
}
|
|
30
33
|
|
|
31
34
|
@field_validator("new_schema", "previous_schema", mode="before")
|
|
@@ -33,6 +36,11 @@ class SchemaApplyMigrationData(BaseModel):
|
|
|
33
36
|
def validate_schema_branch(cls, value: Any) -> SchemaBranch:
|
|
34
37
|
return SchemaBranch.validate(data=value)
|
|
35
38
|
|
|
39
|
+
@field_validator("at", mode="before")
|
|
40
|
+
@classmethod
|
|
41
|
+
def validate_at(cls, value: Any) -> Timestamp:
|
|
42
|
+
return Timestamp(value)
|
|
43
|
+
|
|
36
44
|
|
|
37
45
|
class SchemaMigrationPathResponseData(BaseModel):
|
|
38
46
|
errors: list[str] = Field(default_factory=list)
|
|
@@ -3,16 +3,14 @@ from __future__ import annotations
|
|
|
3
3
|
from typing import TYPE_CHECKING, Any, Sequence
|
|
4
4
|
|
|
5
5
|
from infrahub.core import registry
|
|
6
|
-
from infrahub.core.constants import SYSTEM_USER_ID
|
|
7
6
|
from infrahub.core.node import Node
|
|
8
7
|
from infrahub.core.schema.generic_schema import GenericSchema
|
|
9
8
|
from infrahub.core.schema.node_schema import NodeSchema
|
|
10
|
-
from infrahub.exceptions import PoolExhaustedError
|
|
11
9
|
from infrahub.tasks.registry import update_branch_registry
|
|
12
10
|
|
|
13
11
|
from ..query import AttributeMigrationQuery, MigrationBaseQuery
|
|
14
12
|
from ..query.attribute_add import AttributeAddQuery
|
|
15
|
-
from ..shared import AttributeSchemaMigration, MigrationResult
|
|
13
|
+
from ..shared import AttributeSchemaMigration, MigrationInput, MigrationResult
|
|
16
14
|
|
|
17
15
|
if TYPE_CHECKING:
|
|
18
16
|
from infrahub.core.node.resource_manager.number_pool import CoreNumberPool
|
|
@@ -21,7 +19,6 @@ if TYPE_CHECKING:
|
|
|
21
19
|
from infrahub.database import InfrahubDatabase
|
|
22
20
|
|
|
23
21
|
from ...branch import Branch
|
|
24
|
-
from ...timestamp import Timestamp
|
|
25
22
|
|
|
26
23
|
|
|
27
24
|
class NodeAttributeAddMigrationQuery01(AttributeMigrationQuery, AttributeAddQuery):
|
|
@@ -62,32 +59,32 @@ class NodeAttributeAddMigration(AttributeSchemaMigration):
|
|
|
62
59
|
|
|
63
60
|
async def execute(
|
|
64
61
|
self,
|
|
65
|
-
|
|
62
|
+
migration_input: MigrationInput,
|
|
66
63
|
branch: Branch,
|
|
67
|
-
at: Timestamp | str | None = None,
|
|
68
64
|
queries: Sequence[type[MigrationBaseQuery]] | None = None,
|
|
69
|
-
user_id: str = SYSTEM_USER_ID,
|
|
70
65
|
) -> MigrationResult:
|
|
71
66
|
if self.new_attribute_schema.inherited is True:
|
|
72
67
|
return MigrationResult()
|
|
73
|
-
return await super().execute(
|
|
68
|
+
return await super().execute(migration_input=migration_input, branch=branch, queries=queries)
|
|
74
69
|
|
|
75
70
|
async def execute_post_queries(
|
|
76
71
|
self,
|
|
77
|
-
|
|
72
|
+
migration_input: MigrationInput,
|
|
78
73
|
result: MigrationResult,
|
|
79
74
|
branch: Branch,
|
|
80
|
-
at: Timestamp, # noqa: ARG002
|
|
81
|
-
user_id: str, # noqa: ARG002
|
|
82
75
|
) -> MigrationResult:
|
|
83
76
|
if self.new_attribute_schema.kind != "NumberPool":
|
|
84
77
|
return result
|
|
85
78
|
|
|
79
|
+
db = migration_input.db
|
|
80
|
+
at = migration_input.at
|
|
81
|
+
|
|
86
82
|
number_pool: CoreNumberPool = await Node.fetch_or_create_number_pool(
|
|
87
83
|
db=db,
|
|
88
84
|
branch=branch,
|
|
89
85
|
schema_node=self.new_schema, # type: ignore
|
|
90
86
|
schema_attribute=self.new_attribute_schema,
|
|
87
|
+
at=at,
|
|
91
88
|
)
|
|
92
89
|
|
|
93
90
|
await update_branch_registry(db=db, branch=branch)
|
|
@@ -96,23 +93,21 @@ class NodeAttributeAddMigration(AttributeSchemaMigration):
|
|
|
96
93
|
db=db, branch=branch, schema=self.new_schema, fields={"id": True, self.new_attribute_schema.name: True}
|
|
97
94
|
)
|
|
98
95
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
await
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
await node.save(db=db, fields=[self.new_attribute_schema.name])
|
|
96
|
+
async def allocate_numbers(db: InfrahubDatabase) -> None:
|
|
97
|
+
for node in nodes:
|
|
98
|
+
number = await number_pool.get_resource(
|
|
99
|
+
db=db, branch=branch, node=node, attribute=self.new_attribute_schema, at=at
|
|
100
|
+
)
|
|
101
|
+
attr = node.get_attribute(name=self.new_attribute_schema.name)
|
|
102
|
+
attr.value = number
|
|
103
|
+
attr.set_source(number_pool)
|
|
104
|
+
|
|
105
|
+
await node.save(db=db, fields=[self.new_attribute_schema.name], at=at)
|
|
106
|
+
|
|
107
|
+
if db.is_transaction:
|
|
108
|
+
await allocate_numbers(db=db)
|
|
109
|
+
else:
|
|
110
|
+
async with db.start_transaction() as dbt:
|
|
111
|
+
await allocate_numbers(db=dbt)
|
|
117
112
|
|
|
118
113
|
return result
|
|
@@ -10,6 +10,7 @@ from prefect.logging import get_run_logger
|
|
|
10
10
|
from infrahub.core.branch import Branch # noqa: TC001
|
|
11
11
|
from infrahub.core.constants import SYSTEM_USER_ID
|
|
12
12
|
from infrahub.core.migrations import MIGRATION_MAP
|
|
13
|
+
from infrahub.core.migrations.shared import MigrationInput
|
|
13
14
|
from infrahub.core.path import SchemaPath # noqa: TC001
|
|
14
15
|
from infrahub.workers.dependencies import get_database
|
|
15
16
|
from infrahub.workflows.utils import add_branch_tag
|
|
@@ -18,6 +19,7 @@ from .models import SchemaApplyMigrationData, SchemaMigrationPathResponseData
|
|
|
18
19
|
|
|
19
20
|
if TYPE_CHECKING:
|
|
20
21
|
from infrahub.core.schema import MainSchemaTypes
|
|
22
|
+
from infrahub.core.timestamp import Timestamp
|
|
21
23
|
from infrahub.database import InfrahubDatabase
|
|
22
24
|
|
|
23
25
|
|
|
@@ -59,6 +61,7 @@ async def schema_apply_migrations(message: SchemaApplyMigrationData) -> list[str
|
|
|
59
61
|
schema_path=migration.path,
|
|
60
62
|
database=await get_database(),
|
|
61
63
|
user_id=message.user_id,
|
|
64
|
+
at=message.at,
|
|
62
65
|
)
|
|
63
66
|
|
|
64
67
|
async for _, result in batch.execute():
|
|
@@ -79,6 +82,7 @@ async def schema_path_migrate(
|
|
|
79
82
|
migration_name: str,
|
|
80
83
|
schema_path: SchemaPath,
|
|
81
84
|
database: InfrahubDatabase,
|
|
85
|
+
at: Timestamp,
|
|
82
86
|
new_node_schema: MainSchemaTypes | None = None,
|
|
83
87
|
previous_node_schema: MainSchemaTypes | None = None,
|
|
84
88
|
user_id: str = SYSTEM_USER_ID,
|
|
@@ -104,7 +108,9 @@ async def schema_path_migrate(
|
|
|
104
108
|
previous_node_schema=previous_node_schema, # type: ignore[arg-type]
|
|
105
109
|
schema_path=schema_path,
|
|
106
110
|
)
|
|
107
|
-
execution_result = await migration.execute(
|
|
111
|
+
execution_result = await migration.execute(
|
|
112
|
+
migration_input=MigrationInput(db=db, at=at, user_id=user_id), branch=branch
|
|
113
|
+
)
|
|
108
114
|
|
|
109
115
|
log.info(f"Migration completed for {migration_name}")
|
|
110
116
|
log.debug(f"execution_result {execution_result}")
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from dataclasses import dataclass, field
|
|
3
4
|
from typing import TYPE_CHECKING, Any, Sequence
|
|
4
5
|
|
|
5
6
|
from pydantic import BaseModel, ConfigDict, Field
|
|
@@ -46,6 +47,13 @@ class MigrationResult(BaseModel):
|
|
|
46
47
|
return False
|
|
47
48
|
|
|
48
49
|
|
|
50
|
+
@dataclass
|
|
51
|
+
class MigrationInput:
|
|
52
|
+
db: InfrahubDatabase
|
|
53
|
+
at: Timestamp = field(default_factory=Timestamp)
|
|
54
|
+
user_id: str = SYSTEM_USER_ID
|
|
55
|
+
|
|
56
|
+
|
|
49
57
|
class SchemaMigration(BaseModel):
|
|
50
58
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
51
59
|
name: str = Field(..., description="Name of the migration")
|
|
@@ -59,37 +67,37 @@ class SchemaMigration(BaseModel):
|
|
|
59
67
|
|
|
60
68
|
async def execute_pre_queries(
|
|
61
69
|
self,
|
|
62
|
-
|
|
70
|
+
migration_input: MigrationInput, # noqa: ARG002
|
|
63
71
|
result: MigrationResult,
|
|
64
72
|
branch: Branch, # noqa: ARG002
|
|
65
|
-
at: Timestamp, # noqa: ARG002
|
|
66
|
-
user_id: str, # noqa: ARG002
|
|
67
73
|
) -> MigrationResult:
|
|
68
74
|
return result
|
|
69
75
|
|
|
70
76
|
async def execute_post_queries(
|
|
71
77
|
self,
|
|
72
|
-
|
|
78
|
+
migration_input: MigrationInput, # noqa: ARG002
|
|
73
79
|
result: MigrationResult,
|
|
74
80
|
branch: Branch, # noqa: ARG002
|
|
75
|
-
at: Timestamp, # noqa: ARG002
|
|
76
|
-
user_id: str, # noqa: ARG002
|
|
77
81
|
) -> MigrationResult:
|
|
78
82
|
return result
|
|
79
83
|
|
|
80
84
|
async def execute_queries(
|
|
81
85
|
self,
|
|
82
|
-
|
|
86
|
+
migration_input: MigrationInput,
|
|
83
87
|
result: MigrationResult,
|
|
84
88
|
branch: Branch,
|
|
85
|
-
at: Timestamp,
|
|
86
89
|
queries: Sequence[type[MigrationBaseQuery]],
|
|
87
|
-
user_id: str,
|
|
88
90
|
) -> MigrationResult:
|
|
89
91
|
for migration_query in queries:
|
|
90
92
|
try:
|
|
91
|
-
query = await migration_query.init(
|
|
92
|
-
|
|
93
|
+
query = await migration_query.init(
|
|
94
|
+
db=migration_input.db,
|
|
95
|
+
branch=branch,
|
|
96
|
+
at=migration_input.at,
|
|
97
|
+
migration=self,
|
|
98
|
+
user_id=migration_input.user_id,
|
|
99
|
+
)
|
|
100
|
+
await query.execute(db=migration_input.db)
|
|
93
101
|
result.nbr_migrations_executed += query.get_nbr_migrations_executed()
|
|
94
102
|
except Exception as exc:
|
|
95
103
|
result.errors.append(str(exc))
|
|
@@ -99,22 +107,20 @@ class SchemaMigration(BaseModel):
|
|
|
99
107
|
|
|
100
108
|
async def execute(
|
|
101
109
|
self,
|
|
102
|
-
|
|
110
|
+
migration_input: MigrationInput,
|
|
103
111
|
branch: Branch,
|
|
104
|
-
at: Timestamp | str | None = None,
|
|
105
112
|
queries: Sequence[type[MigrationBaseQuery]] | None = None,
|
|
106
|
-
user_id: str = SYSTEM_USER_ID,
|
|
107
113
|
) -> MigrationResult:
|
|
108
|
-
async with db.start_transaction() as ts:
|
|
114
|
+
async with migration_input.db.start_transaction() as ts:
|
|
109
115
|
result = MigrationResult()
|
|
110
|
-
|
|
116
|
+
txn_migration_input = MigrationInput(db=ts, at=migration_input.at, user_id=migration_input.user_id)
|
|
111
117
|
|
|
112
|
-
await self.execute_pre_queries(
|
|
118
|
+
await self.execute_pre_queries(migration_input=txn_migration_input, result=result, branch=branch)
|
|
113
119
|
queries_to_execute = queries or self.queries
|
|
114
120
|
await self.execute_queries(
|
|
115
|
-
|
|
121
|
+
migration_input=txn_migration_input, result=result, branch=branch, queries=queries_to_execute
|
|
116
122
|
)
|
|
117
|
-
await self.execute_post_queries(
|
|
123
|
+
await self.execute_post_queries(migration_input=txn_migration_input, result=result, branch=branch)
|
|
118
124
|
|
|
119
125
|
return result
|
|
120
126
|
|
|
@@ -174,16 +180,17 @@ class GraphMigration(BaseModel):
|
|
|
174
180
|
async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:
|
|
175
181
|
raise NotImplementedError
|
|
176
182
|
|
|
177
|
-
async def execute(self,
|
|
178
|
-
async with db.start_transaction() as ts:
|
|
179
|
-
|
|
183
|
+
async def execute(self, migration_input: MigrationInput) -> MigrationResult:
|
|
184
|
+
async with migration_input.db.start_transaction() as ts:
|
|
185
|
+
txn_migration_input = MigrationInput(db=ts, at=migration_input.at)
|
|
186
|
+
return await self.do_execute(migration_input=txn_migration_input)
|
|
180
187
|
|
|
181
|
-
async def do_execute(self,
|
|
188
|
+
async def do_execute(self, migration_input: MigrationInput) -> MigrationResult:
|
|
182
189
|
result = MigrationResult()
|
|
183
190
|
for migration_query in self.queries:
|
|
184
191
|
try:
|
|
185
|
-
query = await migration_query.init(db=db)
|
|
186
|
-
await query.execute(db=db)
|
|
192
|
+
query = await migration_query.init(db=migration_input.db, at=migration_input.at)
|
|
193
|
+
await query.execute(db=migration_input.db)
|
|
187
194
|
except Exception as exc:
|
|
188
195
|
result.errors.append(str(exc))
|
|
189
196
|
return result
|
|
@@ -216,14 +223,14 @@ class InternalSchemaMigration(BaseModel):
|
|
|
216
223
|
async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:
|
|
217
224
|
raise NotImplementedError
|
|
218
225
|
|
|
219
|
-
async def execute(self,
|
|
226
|
+
async def execute(self, migration_input: MigrationInput) -> MigrationResult:
|
|
220
227
|
result = MigrationResult()
|
|
221
228
|
|
|
222
229
|
default_branch = registry.get_branch_from_registry()
|
|
223
230
|
|
|
224
231
|
for migration in self.migrations:
|
|
225
232
|
try:
|
|
226
|
-
execution_result = await migration.execute(
|
|
233
|
+
execution_result = await migration.execute(migration_input=migration_input, branch=default_branch)
|
|
227
234
|
result.errors.extend(execution_result.errors)
|
|
228
235
|
except Exception as exc:
|
|
229
236
|
result.errors.append(str(exc))
|
|
@@ -243,7 +250,7 @@ class ArbitraryMigration(BaseModel):
|
|
|
243
250
|
async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:
|
|
244
251
|
raise NotImplementedError()
|
|
245
252
|
|
|
246
|
-
async def execute(self,
|
|
253
|
+
async def execute(self, migration_input: MigrationInput) -> MigrationResult:
|
|
247
254
|
raise NotImplementedError()
|
|
248
255
|
|
|
249
256
|
|
|
@@ -259,11 +266,11 @@ class MigrationRequiringRebase(BaseModel):
|
|
|
259
266
|
async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:
|
|
260
267
|
raise NotImplementedError()
|
|
261
268
|
|
|
262
|
-
async def execute_against_branch(self,
|
|
269
|
+
async def execute_against_branch(self, migration_input: MigrationInput, branch: Branch) -> MigrationResult:
|
|
263
270
|
"""Method that will be run against non-default branches, it assumes that the branches have been rebased."""
|
|
264
271
|
raise NotImplementedError()
|
|
265
272
|
|
|
266
|
-
async def execute(self,
|
|
273
|
+
async def execute(self, migration_input: MigrationInput) -> MigrationResult:
|
|
267
274
|
"""Method that will be run against the default branch."""
|
|
268
275
|
raise NotImplementedError()
|
|
269
276
|
|
infrahub/core/node/__init__.py
CHANGED
|
@@ -408,6 +408,7 @@ class Node(BaseNode, MetadataInterface, metaclass=BaseNodeMeta):
|
|
|
408
408
|
schema_node: NodeSchema | GenericSchema,
|
|
409
409
|
schema_attribute: AttributeSchema,
|
|
410
410
|
branch: Branch | None = None,
|
|
411
|
+
at: Timestamp | None = None,
|
|
411
412
|
) -> CoreNumberPool:
|
|
412
413
|
"""Fetch or create a number pool based on the schema attribute parameters.
|
|
413
414
|
|
|
@@ -456,7 +457,7 @@ class Node(BaseNode, MetadataInterface, metaclass=BaseNodeMeta):
|
|
|
456
457
|
end_range=number_pool_parameters.end_range,
|
|
457
458
|
pool_type=NumberPoolType.SCHEMA.value,
|
|
458
459
|
)
|
|
459
|
-
await number_pool.save(db=db)
|
|
460
|
+
await number_pool.save(db=db, at=at)
|
|
460
461
|
|
|
461
462
|
# Do a lookup of the number pool to get the correct mapped type from the registry
|
|
462
463
|
# without this we don't get access to the .get_resource() method.
|