infrahub-server 1.2.9rc0__py3-none-any.whl → 1.2.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/computed_attribute/models.py +13 -0
- infrahub/computed_attribute/tasks.py +48 -26
- infrahub/config.py +9 -0
- infrahub/core/attribute.py +43 -2
- infrahub/core/branch/models.py +8 -9
- infrahub/core/branch/tasks.py +0 -2
- infrahub/core/constants/infrahubkind.py +1 -0
- infrahub/core/constraint/node/runner.py +1 -1
- infrahub/core/diff/calculator.py +65 -11
- infrahub/core/diff/combiner.py +38 -31
- infrahub/core/diff/coordinator.py +44 -28
- infrahub/core/diff/data_check_synchronizer.py +3 -2
- infrahub/core/diff/enricher/hierarchy.py +36 -27
- infrahub/core/diff/ipam_diff_parser.py +5 -4
- infrahub/core/diff/merger/merger.py +46 -16
- infrahub/core/diff/merger/serializer.py +1 -0
- infrahub/core/diff/model/field_specifiers_map.py +64 -0
- infrahub/core/diff/model/path.py +58 -58
- infrahub/core/diff/parent_node_adder.py +14 -16
- infrahub/core/diff/query/drop_nodes.py +42 -0
- infrahub/core/diff/query/field_specifiers.py +8 -7
- infrahub/core/diff/query/filters.py +15 -1
- infrahub/core/diff/query/merge.py +264 -28
- infrahub/core/diff/query/save.py +6 -2
- infrahub/core/diff/query_parser.py +55 -65
- infrahub/core/diff/repository/deserializer.py +38 -24
- infrahub/core/diff/repository/repository.py +31 -12
- infrahub/core/diff/tasks.py +3 -3
- infrahub/core/graph/__init__.py +1 -1
- infrahub/core/migrations/graph/__init__.py +2 -0
- infrahub/core/migrations/graph/m027_delete_isolated_nodes.py +50 -0
- infrahub/core/migrations/graph/m028_delete_diffs.py +38 -0
- infrahub/core/node/resource_manager/ip_address_pool.py +6 -2
- infrahub/core/node/resource_manager/ip_prefix_pool.py +6 -2
- infrahub/core/protocols.py +4 -0
- infrahub/core/query/branch.py +27 -17
- infrahub/core/query/diff.py +169 -51
- infrahub/core/query/node.py +39 -5
- infrahub/core/query/relationship.py +105 -30
- infrahub/core/query/subquery.py +2 -2
- infrahub/core/relationship/model.py +1 -1
- infrahub/core/schema/definitions/core/__init__.py +8 -1
- infrahub/core/schema/definitions/core/resource_pool.py +20 -0
- infrahub/core/schema/schema_branch.py +3 -0
- infrahub/core/validators/tasks.py +1 -1
- infrahub/core/validators/uniqueness/query.py +7 -0
- infrahub/database/__init__.py +5 -4
- infrahub/graphql/app.py +1 -1
- infrahub/graphql/loaders/node.py +1 -1
- infrahub/graphql/loaders/peers.py +1 -1
- infrahub/graphql/mutations/proposed_change.py +1 -1
- infrahub/graphql/queries/diff/tree.py +2 -1
- infrahub/graphql/queries/relationship.py +1 -1
- infrahub/graphql/queries/task.py +10 -0
- infrahub/graphql/resolvers/many_relationship.py +4 -4
- infrahub/graphql/resolvers/resolver.py +4 -4
- infrahub/graphql/resolvers/single_relationship.py +2 -2
- infrahub/graphql/subscription/graphql_query.py +2 -2
- infrahub/graphql/types/branch.py +1 -1
- infrahub/graphql/types/task_log.py +3 -2
- infrahub/message_bus/operations/refresh/registry.py +1 -1
- infrahub/task_manager/task.py +44 -4
- infrahub/telemetry/database.py +1 -1
- infrahub/telemetry/tasks.py +1 -1
- infrahub/trigger/models.py +11 -1
- infrahub/trigger/setup.py +51 -15
- infrahub/trigger/tasks.py +1 -4
- infrahub/types.py +1 -1
- infrahub/webhook/models.py +2 -1
- infrahub/workflows/catalogue.py +9 -0
- infrahub/workflows/initialization.py +1 -3
- infrahub_sdk/timestamp.py +2 -2
- {infrahub_server-1.2.9rc0.dist-info → infrahub_server-1.2.11.dist-info}/METADATA +3 -3
- {infrahub_server-1.2.9rc0.dist-info → infrahub_server-1.2.11.dist-info}/RECORD +79 -75
- infrahub_testcontainers/docker-compose.test.yml +3 -3
- infrahub_testcontainers/performance_test.py +6 -3
- {infrahub_server-1.2.9rc0.dist-info → infrahub_server-1.2.11.dist-info}/LICENSE.txt +0 -0
- {infrahub_server-1.2.9rc0.dist-info → infrahub_server-1.2.11.dist-info}/WHEEL +0 -0
- {infrahub_server-1.2.9rc0.dist-info → infrahub_server-1.2.11.dist-info}/entry_points.txt +0 -0
|
@@ -118,6 +118,10 @@ class PythonTransformTarget:
|
|
|
118
118
|
class ComputedAttrJinja2TriggerDefinition(TriggerBranchDefinition):
|
|
119
119
|
type: TriggerType = TriggerType.COMPUTED_ATTR_JINJA2
|
|
120
120
|
computed_attribute: ComputedAttributeTarget
|
|
121
|
+
template_hash: str
|
|
122
|
+
|
|
123
|
+
def get_description(self) -> str:
|
|
124
|
+
return f"{super().get_description()} | hash:{self.template_hash}"
|
|
121
125
|
|
|
122
126
|
@classmethod
|
|
123
127
|
def from_computed_attribute(
|
|
@@ -139,6 +143,14 @@ class ComputedAttrJinja2TriggerDefinition(TriggerBranchDefinition):
|
|
|
139
143
|
# node creation events if the attribute is optional.
|
|
140
144
|
event_trigger.events.add(NodeCreatedEvent.event_name)
|
|
141
145
|
|
|
146
|
+
if (
|
|
147
|
+
computed_attribute.attribute.computed_attribute
|
|
148
|
+
and computed_attribute.attribute.computed_attribute.jinja2_template is None
|
|
149
|
+
) or not computed_attribute.attribute.computed_attribute:
|
|
150
|
+
raise ValueError("Jinja2 template is required for computed attribute")
|
|
151
|
+
|
|
152
|
+
template_hash = computed_attribute.attribute.computed_attribute.get_hash()
|
|
153
|
+
|
|
142
154
|
event_trigger.match = {"infrahub.node.kind": trigger_node.kind}
|
|
143
155
|
if branches_out_of_scope:
|
|
144
156
|
event_trigger.match["infrahub.branch.name"] = [f"!{branch}" for branch in branches_out_of_scope]
|
|
@@ -177,6 +189,7 @@ class ComputedAttrJinja2TriggerDefinition(TriggerBranchDefinition):
|
|
|
177
189
|
|
|
178
190
|
definition = cls(
|
|
179
191
|
name=f"{computed_attribute.key_name}{NAME_SEPARATOR}kind{NAME_SEPARATOR}{trigger_node.kind}",
|
|
192
|
+
template_hash=template_hash,
|
|
180
193
|
branch=branch,
|
|
181
194
|
computed_attribute=computed_attribute,
|
|
182
195
|
trigger=event_trigger,
|
|
@@ -14,7 +14,7 @@ from infrahub.core.registry import registry
|
|
|
14
14
|
from infrahub.events import BranchDeletedEvent
|
|
15
15
|
from infrahub.git.repository import get_initialized_repo
|
|
16
16
|
from infrahub.services import InfrahubServices # noqa: TC001 needed for prefect flow
|
|
17
|
-
from infrahub.trigger.models import TriggerType
|
|
17
|
+
from infrahub.trigger.models import TriggerSetupReport, TriggerType
|
|
18
18
|
from infrahub.trigger.setup import setup_triggers
|
|
19
19
|
from infrahub.workflows.catalogue import (
|
|
20
20
|
COMPUTED_ATTRIBUTE_PROCESS_JINJA2,
|
|
@@ -25,7 +25,11 @@ from infrahub.workflows.catalogue import (
|
|
|
25
25
|
from infrahub.workflows.utils import add_tags, wait_for_schema_to_converge
|
|
26
26
|
|
|
27
27
|
from .gather import gather_trigger_computed_attribute_jinja2, gather_trigger_computed_attribute_python
|
|
28
|
-
from .models import
|
|
28
|
+
from .models import (
|
|
29
|
+
ComputedAttrJinja2GraphQL,
|
|
30
|
+
ComputedAttrJinja2GraphQLResponse,
|
|
31
|
+
PythonTransformTarget,
|
|
32
|
+
)
|
|
29
33
|
|
|
30
34
|
if TYPE_CHECKING:
|
|
31
35
|
from infrahub.core.schema.computed_attribute import ComputedAttribute
|
|
@@ -159,10 +163,10 @@ async def trigger_update_python_computed_attributes(
|
|
|
159
163
|
|
|
160
164
|
|
|
161
165
|
@flow(
|
|
162
|
-
name="
|
|
163
|
-
flow_run_name="Update value for computed attribute {attribute_name}",
|
|
166
|
+
name="computed-attribute-jinja2-update-value",
|
|
167
|
+
flow_run_name="Update value for computed attribute {node_kind}:{attribute_name}",
|
|
164
168
|
)
|
|
165
|
-
async def
|
|
169
|
+
async def computed_attribute_jinja2_update_value(
|
|
166
170
|
branch_name: str,
|
|
167
171
|
obj: ComputedAttrJinja2GraphQLResponse,
|
|
168
172
|
node_kind: str,
|
|
@@ -246,7 +250,7 @@ async def process_jinja2(
|
|
|
246
250
|
batch = await service.client.create_batch()
|
|
247
251
|
for node in found:
|
|
248
252
|
batch.add(
|
|
249
|
-
task=
|
|
253
|
+
task=computed_attribute_jinja2_update_value,
|
|
250
254
|
branch_name=branch_name,
|
|
251
255
|
obj=node,
|
|
252
256
|
node_kind=node_schema.kind,
|
|
@@ -302,26 +306,33 @@ async def computed_attribute_setup_jinja2(
|
|
|
302
306
|
|
|
303
307
|
triggers = await gather_trigger_computed_attribute_jinja2()
|
|
304
308
|
|
|
305
|
-
for trigger in triggers:
|
|
306
|
-
if event_name != BranchDeletedEvent.event_name and trigger.branch == branch_name:
|
|
307
|
-
await service.workflow.submit_workflow(
|
|
308
|
-
workflow=TRIGGER_UPDATE_JINJA_COMPUTED_ATTRIBUTES,
|
|
309
|
-
context=context,
|
|
310
|
-
parameters={
|
|
311
|
-
"branch_name": trigger.branch,
|
|
312
|
-
"computed_attribute_name": trigger.computed_attribute.attribute.name,
|
|
313
|
-
"computed_attribute_kind": trigger.computed_attribute.kind,
|
|
314
|
-
},
|
|
315
|
-
)
|
|
316
|
-
|
|
317
309
|
# Configure all ComputedAttrJinja2Trigger in Prefect
|
|
318
310
|
async with get_client(sync_client=False) as prefect_client:
|
|
319
|
-
await setup_triggers(
|
|
311
|
+
report: TriggerSetupReport = await setup_triggers(
|
|
320
312
|
client=prefect_client,
|
|
321
313
|
triggers=triggers,
|
|
322
314
|
trigger_type=TriggerType.COMPUTED_ATTR_JINJA2,
|
|
315
|
+
force_update=False,
|
|
323
316
|
) # type: ignore[misc]
|
|
324
317
|
|
|
318
|
+
# Since we can have multiple trigger per NodeKind
|
|
319
|
+
# we need to extract the list of unique node that should be processed
|
|
320
|
+
unique_nodes: set[tuple[str, str, str]] = {
|
|
321
|
+
(trigger.branch, trigger.computed_attribute.kind, trigger.computed_attribute.attribute.name) # type: ignore[attr-defined]
|
|
322
|
+
for trigger in report.updated + report.created
|
|
323
|
+
}
|
|
324
|
+
for branch, kind, attribute_name in unique_nodes:
|
|
325
|
+
if event_name != BranchDeletedEvent.event_name and branch == branch_name:
|
|
326
|
+
await service.workflow.submit_workflow(
|
|
327
|
+
workflow=TRIGGER_UPDATE_JINJA_COMPUTED_ATTRIBUTES,
|
|
328
|
+
context=context,
|
|
329
|
+
parameters={
|
|
330
|
+
"branch_name": branch,
|
|
331
|
+
"computed_attribute_name": attribute_name,
|
|
332
|
+
"computed_attribute_kind": kind,
|
|
333
|
+
},
|
|
334
|
+
)
|
|
335
|
+
|
|
325
336
|
log.info(f"{len(triggers)} Computed Attribute for Jinja2 automation configuration completed")
|
|
326
337
|
|
|
327
338
|
|
|
@@ -347,18 +358,29 @@ async def computed_attribute_setup_python(
|
|
|
347
358
|
|
|
348
359
|
triggers_python, triggers_python_query = await gather_trigger_computed_attribute_python(db=db)
|
|
349
360
|
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
361
|
+
# Since we can have multiple trigger per NodeKind
|
|
362
|
+
# we need to extract the list of unique node that should be processed
|
|
363
|
+
# also
|
|
364
|
+
# Because the automation in Prefect doesn't capture all information about the computed attribute
|
|
365
|
+
# we can't tell right now if a given computed attribute has changed and need to be updated
|
|
366
|
+
unique_nodes: set[tuple[str, str, str]] = {
|
|
367
|
+
(
|
|
368
|
+
trigger.branch,
|
|
369
|
+
trigger.computed_attribute.computed_attribute.kind,
|
|
370
|
+
trigger.computed_attribute.computed_attribute.attribute.name,
|
|
371
|
+
)
|
|
372
|
+
for trigger in triggers_python
|
|
373
|
+
}
|
|
374
|
+
for branch, kind, attribute_name in unique_nodes:
|
|
375
|
+
if event_name != BranchDeletedEvent.event_name and branch == branch_name:
|
|
376
|
+
log.info(f"Triggering update for {kind}.{attribute_name} on {branch}")
|
|
355
377
|
await service.workflow.submit_workflow(
|
|
356
378
|
workflow=TRIGGER_UPDATE_PYTHON_COMPUTED_ATTRIBUTES,
|
|
357
379
|
context=context,
|
|
358
380
|
parameters={
|
|
359
381
|
"branch_name": branch_name,
|
|
360
|
-
"computed_attribute_name":
|
|
361
|
-
"computed_attribute_kind":
|
|
382
|
+
"computed_attribute_name": attribute_name,
|
|
383
|
+
"computed_attribute_kind": kind,
|
|
362
384
|
},
|
|
363
385
|
)
|
|
364
386
|
|
infrahub/config.py
CHANGED
|
@@ -269,6 +269,7 @@ class DatabaseSettings(BaseSettings):
|
|
|
269
269
|
address: str = "localhost"
|
|
270
270
|
port: int = 7687
|
|
271
271
|
database: str | None = Field(default=None, pattern=VALID_DATABASE_NAME_REGEX, description="Name of the database")
|
|
272
|
+
policy: str | None = Field(default=None, description="Routing policy for database connections")
|
|
272
273
|
tls_enabled: bool = Field(default=False, description="Indicates if TLS is enabled for the connection")
|
|
273
274
|
tls_insecure: bool = Field(default=False, description="Indicates if TLS certificates are verified")
|
|
274
275
|
tls_ca_file: str | None = Field(default=None, description="File path to CA cert or bundle in PEM format")
|
|
@@ -293,6 +294,14 @@ class DatabaseSettings(BaseSettings):
|
|
|
293
294
|
default=0.01, ge=0, description="Delay to add when max_concurrent_queries is reached."
|
|
294
295
|
)
|
|
295
296
|
|
|
297
|
+
@property
|
|
298
|
+
def database_uri(self) -> str:
|
|
299
|
+
"""Constructs the database URI based on the configuration settings."""
|
|
300
|
+
base_uri = f"{self.protocol}://{self.address}:{self.port}"
|
|
301
|
+
if self.policy is not None:
|
|
302
|
+
return f"{base_uri}?policy={self.policy}"
|
|
303
|
+
return base_uri
|
|
304
|
+
|
|
296
305
|
@property
|
|
297
306
|
def database_name(self) -> str:
|
|
298
307
|
return self.database or self.db_type.value
|
infrahub/core/attribute.py
CHANGED
|
@@ -782,6 +782,10 @@ class Dropdown(BaseAttribute):
|
|
|
782
782
|
|
|
783
783
|
return ""
|
|
784
784
|
|
|
785
|
+
@staticmethod
|
|
786
|
+
def get_allowed_property_in_path() -> list[str]:
|
|
787
|
+
return ["color", "description", "label", "value"]
|
|
788
|
+
|
|
785
789
|
@classmethod
|
|
786
790
|
def validate_content(cls, value: Any, name: str, schema: AttributeSchema) -> None:
|
|
787
791
|
"""Validate the content of the dropdown."""
|
|
@@ -817,7 +821,18 @@ class IPNetwork(BaseAttribute):
|
|
|
817
821
|
|
|
818
822
|
@staticmethod
|
|
819
823
|
def get_allowed_property_in_path() -> list[str]:
|
|
820
|
-
return [
|
|
824
|
+
return [
|
|
825
|
+
"binary_address",
|
|
826
|
+
"broadcast_address",
|
|
827
|
+
"hostmask",
|
|
828
|
+
"netmask",
|
|
829
|
+
"num_addresses",
|
|
830
|
+
"prefixlen",
|
|
831
|
+
"value",
|
|
832
|
+
"version",
|
|
833
|
+
"with_hostmask",
|
|
834
|
+
"with_netmask",
|
|
835
|
+
]
|
|
821
836
|
|
|
822
837
|
@property
|
|
823
838
|
def obj(self) -> ipaddress.IPv4Network | ipaddress.IPv6Network:
|
|
@@ -950,7 +965,17 @@ class IPHost(BaseAttribute):
|
|
|
950
965
|
|
|
951
966
|
@staticmethod
|
|
952
967
|
def get_allowed_property_in_path() -> list[str]:
|
|
953
|
-
return [
|
|
968
|
+
return [
|
|
969
|
+
"binary_address",
|
|
970
|
+
"hostmask",
|
|
971
|
+
"ip",
|
|
972
|
+
"netmask",
|
|
973
|
+
"prefixlen",
|
|
974
|
+
"value",
|
|
975
|
+
"version",
|
|
976
|
+
"with_hostmask",
|
|
977
|
+
"with_netmask",
|
|
978
|
+
]
|
|
954
979
|
|
|
955
980
|
@property
|
|
956
981
|
def obj(self) -> ipaddress.IPv4Interface | ipaddress.IPv6Interface:
|
|
@@ -1170,6 +1195,22 @@ class MacAddress(BaseAttribute):
|
|
|
1170
1195
|
"""Serialize the value as standard EUI-48 or EUI-64 before storing it in the database."""
|
|
1171
1196
|
return str(netaddr.EUI(addr=self.value))
|
|
1172
1197
|
|
|
1198
|
+
@staticmethod
|
|
1199
|
+
def get_allowed_property_in_path() -> list[str]:
|
|
1200
|
+
return [
|
|
1201
|
+
"bare",
|
|
1202
|
+
"binary",
|
|
1203
|
+
"dot_notation",
|
|
1204
|
+
"ei",
|
|
1205
|
+
"eui48",
|
|
1206
|
+
"eui64",
|
|
1207
|
+
"oui",
|
|
1208
|
+
"semicolon_notation",
|
|
1209
|
+
"split_notation",
|
|
1210
|
+
"value",
|
|
1211
|
+
"version",
|
|
1212
|
+
]
|
|
1213
|
+
|
|
1173
1214
|
|
|
1174
1215
|
class MacAddressOptional(MacAddress):
|
|
1175
1216
|
value: str | None
|
infrahub/core/branch/models.py
CHANGED
|
@@ -295,6 +295,7 @@ class Branch(StandardNode):
|
|
|
295
295
|
is_isolated: bool = True,
|
|
296
296
|
branch_agnostic: bool = False,
|
|
297
297
|
variable_name: str = "r",
|
|
298
|
+
params_prefix: str = "",
|
|
298
299
|
) -> tuple[str, dict]:
|
|
299
300
|
"""
|
|
300
301
|
Generate a CYPHER Query filter based on a path to query a part of the graph at a specific time and on a specific branch.
|
|
@@ -306,30 +307,28 @@ class Branch(StandardNode):
|
|
|
306
307
|
|
|
307
308
|
There is a currently an assumption that the relationship in the path will be named 'r'
|
|
308
309
|
"""
|
|
309
|
-
|
|
310
|
+
pp = params_prefix
|
|
310
311
|
params: dict[str, Any] = {}
|
|
311
312
|
at = Timestamp(at)
|
|
312
313
|
at_str = at.to_string()
|
|
313
314
|
if branch_agnostic:
|
|
314
|
-
filter_str = (
|
|
315
|
-
|
|
316
|
-
)
|
|
317
|
-
params["time1"] = at_str
|
|
315
|
+
filter_str = f"{variable_name}.from <= ${pp}time1 AND ({variable_name}.to IS NULL or {variable_name}.to >= ${pp}time1)"
|
|
316
|
+
params[f"{pp}time1"] = at_str
|
|
318
317
|
return filter_str, params
|
|
319
318
|
|
|
320
319
|
branches_times = self.get_branches_and_times_to_query_global(at=at_str, is_isolated=is_isolated)
|
|
321
320
|
|
|
322
321
|
for idx, (branch_name, time_to_query) in enumerate(branches_times.items()):
|
|
323
|
-
params[f"branch{idx}"] = list(branch_name)
|
|
324
|
-
params[f"time{idx}"] = time_to_query
|
|
322
|
+
params[f"{pp}branch{idx}"] = list(branch_name)
|
|
323
|
+
params[f"{pp}time{idx}"] = time_to_query
|
|
325
324
|
|
|
326
325
|
filters = []
|
|
327
326
|
for idx in range(len(branches_times)):
|
|
328
327
|
filters.append(
|
|
329
|
-
f"({variable_name}.branch IN $branch{idx} AND {variable_name}.from <= $time{idx} AND {variable_name}.to IS NULL)"
|
|
328
|
+
f"({variable_name}.branch IN ${pp}branch{idx} AND {variable_name}.from <= ${pp}time{idx} AND {variable_name}.to IS NULL)"
|
|
330
329
|
)
|
|
331
330
|
filters.append(
|
|
332
|
-
f"({variable_name}.branch IN $branch{idx} AND {variable_name}.from <= $time{idx} AND {variable_name}.to >= $time{idx})"
|
|
331
|
+
f"({variable_name}.branch IN ${pp}branch{idx} AND {variable_name}.from <= ${pp}time{idx} AND {variable_name}.to >= ${pp}time{idx})"
|
|
333
332
|
)
|
|
334
333
|
|
|
335
334
|
filter_str = "(" + "\n OR ".join(filters) + ")"
|
infrahub/core/branch/tasks.py
CHANGED
|
@@ -212,8 +212,6 @@ async def merge_branch(
|
|
|
212
212
|
|
|
213
213
|
merger: BranchMerger | None = None
|
|
214
214
|
async with lock.registry.global_graph_lock():
|
|
215
|
-
# await update_diff(model=RequestDiffUpdate(branch_name=obj.name))
|
|
216
|
-
|
|
217
215
|
diff_repository = await component_registry.get_component(DiffRepository, db=db, branch=obj)
|
|
218
216
|
diff_coordinator = await component_registry.get_component(DiffCoordinator, db=db, branch=obj)
|
|
219
217
|
diff_merger = await component_registry.get_component(DiffMerger, db=db, branch=obj)
|
|
@@ -26,7 +26,7 @@ class NodeConstraintRunner:
|
|
|
26
26
|
async def check(
|
|
27
27
|
self, node: Node, field_filters: list[str] | None = None, skip_uniqueness_check: bool = False
|
|
28
28
|
) -> None:
|
|
29
|
-
async with self.db.start_session() as db:
|
|
29
|
+
async with self.db.start_session(read_only=False) as db:
|
|
30
30
|
await node.resolve_relationships(db=db)
|
|
31
31
|
|
|
32
32
|
if not skip_uniqueness_check:
|
infrahub/core/diff/calculator.py
CHANGED
|
@@ -7,6 +7,7 @@ from infrahub.core.diff.query_parser import DiffQueryParser
|
|
|
7
7
|
from infrahub.core.query.diff import (
|
|
8
8
|
DiffCalculationQuery,
|
|
9
9
|
DiffFieldPathsQuery,
|
|
10
|
+
DiffMigratedKindNodesQuery,
|
|
10
11
|
DiffNodePathsQuery,
|
|
11
12
|
DiffPropertyPathsQuery,
|
|
12
13
|
)
|
|
@@ -14,7 +15,8 @@ from infrahub.core.timestamp import Timestamp
|
|
|
14
15
|
from infrahub.database import InfrahubDatabase
|
|
15
16
|
from infrahub.log import get_logger
|
|
16
17
|
|
|
17
|
-
from .model.
|
|
18
|
+
from .model.field_specifiers_map import NodeFieldSpecifierMap
|
|
19
|
+
from .model.path import CalculatedDiffs, DiffNode, DiffRoot, NodeIdentifier
|
|
18
20
|
|
|
19
21
|
log = get_logger()
|
|
20
22
|
|
|
@@ -26,8 +28,8 @@ class DiffCalculationRequest:
|
|
|
26
28
|
branch_from_time: Timestamp
|
|
27
29
|
from_time: Timestamp
|
|
28
30
|
to_time: Timestamp
|
|
29
|
-
current_node_field_specifiers:
|
|
30
|
-
new_node_field_specifiers:
|
|
31
|
+
current_node_field_specifiers: NodeFieldSpecifierMap | None = field(default=None)
|
|
32
|
+
new_node_field_specifiers: NodeFieldSpecifierMap | None = field(default=None)
|
|
31
33
|
|
|
32
34
|
|
|
33
35
|
class DiffCalculator:
|
|
@@ -58,7 +60,7 @@ class DiffCalculator:
|
|
|
58
60
|
)
|
|
59
61
|
log.info(f"Beginning one diff calculation query {limit=}, {offset=}")
|
|
60
62
|
await diff_query.execute(db=self.db)
|
|
61
|
-
log.info("Diff calculation query complete")
|
|
63
|
+
log.info(f"Diff calculation query complete {limit=}, {offset=}")
|
|
62
64
|
last_result = None
|
|
63
65
|
for query_result in diff_query.get_results():
|
|
64
66
|
diff_parser.read_result(query_result=query_result)
|
|
@@ -68,6 +70,56 @@ class DiffCalculator:
|
|
|
68
70
|
has_more_data = last_result.get_as_type("has_more_data", bool)
|
|
69
71
|
offset += limit
|
|
70
72
|
|
|
73
|
+
async def _apply_kind_migrated_nodes(
|
|
74
|
+
self, branch_diff: DiffRoot, calculation_request: DiffCalculationRequest
|
|
75
|
+
) -> None:
|
|
76
|
+
has_more_data = True
|
|
77
|
+
offset = 0
|
|
78
|
+
limit = config.SETTINGS.database.query_size_limit
|
|
79
|
+
diff_nodes_by_identifier = {n.identifier: n for n in branch_diff.nodes}
|
|
80
|
+
diff_nodes_to_add: list[DiffNode] = []
|
|
81
|
+
while has_more_data:
|
|
82
|
+
diff_query = await DiffMigratedKindNodesQuery.init(
|
|
83
|
+
db=self.db,
|
|
84
|
+
branch=calculation_request.diff_branch,
|
|
85
|
+
base_branch=calculation_request.base_branch,
|
|
86
|
+
diff_branch_from_time=calculation_request.branch_from_time,
|
|
87
|
+
diff_from=calculation_request.from_time,
|
|
88
|
+
diff_to=calculation_request.to_time,
|
|
89
|
+
limit=limit,
|
|
90
|
+
offset=offset,
|
|
91
|
+
)
|
|
92
|
+
log.info(f"Getting one batch of migrated kind nodes {limit=}, {offset=}")
|
|
93
|
+
await diff_query.execute(db=self.db)
|
|
94
|
+
log.info(f"Migrated kind nodes query complete {limit=}, {offset=}")
|
|
95
|
+
last_result = None
|
|
96
|
+
for migrated_kind_node in diff_query.get_migrated_kind_nodes():
|
|
97
|
+
migrated_kind_identifier = NodeIdentifier(
|
|
98
|
+
uuid=migrated_kind_node.uuid,
|
|
99
|
+
kind=migrated_kind_node.kind,
|
|
100
|
+
db_id=migrated_kind_node.db_id,
|
|
101
|
+
)
|
|
102
|
+
if migrated_kind_identifier in diff_nodes_by_identifier:
|
|
103
|
+
diff_node = diff_nodes_by_identifier[migrated_kind_identifier]
|
|
104
|
+
diff_node.is_node_kind_migration = True
|
|
105
|
+
continue
|
|
106
|
+
new_diff_node = DiffNode(
|
|
107
|
+
identifier=migrated_kind_identifier,
|
|
108
|
+
changed_at=migrated_kind_node.from_time,
|
|
109
|
+
action=migrated_kind_node.action,
|
|
110
|
+
is_node_kind_migration=True,
|
|
111
|
+
attributes=[],
|
|
112
|
+
relationships=[],
|
|
113
|
+
)
|
|
114
|
+
diff_nodes_by_identifier[migrated_kind_identifier] = new_diff_node
|
|
115
|
+
diff_nodes_to_add.append(new_diff_node)
|
|
116
|
+
last_result = migrated_kind_node
|
|
117
|
+
has_more_data = False
|
|
118
|
+
if last_result:
|
|
119
|
+
has_more_data = last_result.has_more_data
|
|
120
|
+
offset += limit
|
|
121
|
+
branch_diff.nodes.extend(diff_nodes_to_add)
|
|
122
|
+
|
|
71
123
|
async def calculate_diff(
|
|
72
124
|
self,
|
|
73
125
|
base_branch: Branch,
|
|
@@ -75,7 +127,7 @@ class DiffCalculator:
|
|
|
75
127
|
from_time: Timestamp,
|
|
76
128
|
to_time: Timestamp,
|
|
77
129
|
include_unchanged: bool = True,
|
|
78
|
-
previous_node_specifiers:
|
|
130
|
+
previous_node_specifiers: NodeFieldSpecifierMap | None = None,
|
|
79
131
|
) -> CalculatedDiffs:
|
|
80
132
|
if diff_branch.name == registry.default_branch:
|
|
81
133
|
diff_branch_from_time = from_time
|
|
@@ -91,7 +143,7 @@ class DiffCalculator:
|
|
|
91
143
|
)
|
|
92
144
|
node_limit = int(config.SETTINGS.database.query_size_limit / 10)
|
|
93
145
|
fields_limit = int(config.SETTINGS.database.query_size_limit / 3)
|
|
94
|
-
properties_limit =
|
|
146
|
+
properties_limit = config.SETTINGS.database.query_size_limit
|
|
95
147
|
|
|
96
148
|
calculation_request = DiffCalculationRequest(
|
|
97
149
|
base_branch=base_branch,
|
|
@@ -131,7 +183,7 @@ class DiffCalculator:
|
|
|
131
183
|
if base_branch.name != diff_branch.name:
|
|
132
184
|
current_node_field_specifiers = diff_parser.get_current_node_field_specifiers()
|
|
133
185
|
new_node_field_specifiers = diff_parser.get_new_node_field_specifiers()
|
|
134
|
-
|
|
186
|
+
base_calculation_request = DiffCalculationRequest(
|
|
135
187
|
base_branch=base_branch,
|
|
136
188
|
diff_branch=base_branch,
|
|
137
189
|
branch_from_time=diff_branch_from_time,
|
|
@@ -145,7 +197,7 @@ class DiffCalculator:
|
|
|
145
197
|
await self._run_diff_calculation_query(
|
|
146
198
|
diff_parser=diff_parser,
|
|
147
199
|
query_class=DiffNodePathsQuery,
|
|
148
|
-
calculation_request=
|
|
200
|
+
calculation_request=base_calculation_request,
|
|
149
201
|
limit=node_limit,
|
|
150
202
|
)
|
|
151
203
|
log.info("Diff node-level calculation queries for base complete")
|
|
@@ -154,7 +206,7 @@ class DiffCalculator:
|
|
|
154
206
|
await self._run_diff_calculation_query(
|
|
155
207
|
diff_parser=diff_parser,
|
|
156
208
|
query_class=DiffFieldPathsQuery,
|
|
157
|
-
calculation_request=
|
|
209
|
+
calculation_request=base_calculation_request,
|
|
158
210
|
limit=fields_limit,
|
|
159
211
|
)
|
|
160
212
|
log.info("Diff field-level calculation queries for base complete")
|
|
@@ -163,7 +215,7 @@ class DiffCalculator:
|
|
|
163
215
|
await self._run_diff_calculation_query(
|
|
164
216
|
diff_parser=diff_parser,
|
|
165
217
|
query_class=DiffPropertyPathsQuery,
|
|
166
|
-
calculation_request=
|
|
218
|
+
calculation_request=base_calculation_request,
|
|
167
219
|
limit=properties_limit,
|
|
168
220
|
)
|
|
169
221
|
log.info("Diff property-level calculation queries for base complete")
|
|
@@ -171,9 +223,11 @@ class DiffCalculator:
|
|
|
171
223
|
log.info("Parsing calculated diff")
|
|
172
224
|
diff_parser.parse(include_unchanged=include_unchanged)
|
|
173
225
|
log.info("Calculated diff parsed")
|
|
226
|
+
branch_diff = diff_parser.get_diff_root_for_branch(branch=diff_branch.name)
|
|
227
|
+
await self._apply_kind_migrated_nodes(branch_diff=branch_diff, calculation_request=calculation_request)
|
|
174
228
|
return CalculatedDiffs(
|
|
175
229
|
base_branch_name=base_branch.name,
|
|
176
230
|
diff_branch_name=diff_branch.name,
|
|
177
231
|
base_branch_diff=diff_parser.get_diff_root_for_branch(branch=base_branch.name),
|
|
178
|
-
diff_branch_diff=
|
|
232
|
+
diff_branch_diff=branch_diff,
|
|
179
233
|
)
|
infrahub/core/diff/combiner.py
CHANGED
|
@@ -14,6 +14,7 @@ from .model.path import (
|
|
|
14
14
|
EnrichedDiffRoot,
|
|
15
15
|
EnrichedDiffs,
|
|
16
16
|
EnrichedDiffSingleRelationship,
|
|
17
|
+
NodeIdentifier,
|
|
17
18
|
)
|
|
18
19
|
|
|
19
20
|
|
|
@@ -26,30 +27,35 @@ class NodePair:
|
|
|
26
27
|
class DiffCombiner:
|
|
27
28
|
def __init__(self) -> None:
|
|
28
29
|
# {child_uuid: (parent_uuid, parent_rel_name)}
|
|
29
|
-
self.
|
|
30
|
-
self.
|
|
31
|
-
self.
|
|
32
|
-
self.
|
|
33
|
-
self.
|
|
30
|
+
self._child_parent_identifier_map: dict[NodeIdentifier, tuple[NodeIdentifier, str]] = {}
|
|
31
|
+
self._parent_node_identifiers: set[NodeIdentifier] = set()
|
|
32
|
+
self._earlier_nodes_by_identifier: dict[NodeIdentifier, EnrichedDiffNode] = {}
|
|
33
|
+
self._later_nodes_by_identifier: dict[NodeIdentifier, EnrichedDiffNode] = {}
|
|
34
|
+
self._common_node_identifiers: set[NodeIdentifier] = set()
|
|
34
35
|
self._diff_branch_name: str | None = None
|
|
35
36
|
|
|
36
37
|
def _initialize(self, earlier_diff: EnrichedDiffRoot, later_diff: EnrichedDiffRoot) -> None:
|
|
37
38
|
self._diff_branch_name = earlier_diff.diff_branch_name
|
|
38
|
-
self.
|
|
39
|
-
self.
|
|
40
|
-
self.
|
|
41
|
-
self.
|
|
39
|
+
self._child_parent_identifier_map = {}
|
|
40
|
+
self._earlier_nodes_by_identifier = {}
|
|
41
|
+
self._later_nodes_by_identifier = {}
|
|
42
|
+
self._common_node_identifiers = set()
|
|
42
43
|
# map the parent of each node (if it exists), preference to the later diff
|
|
43
44
|
for diff_root in (earlier_diff, later_diff):
|
|
44
45
|
for child_node in diff_root.nodes:
|
|
45
46
|
for parent_rel in child_node.relationships:
|
|
46
47
|
for parent_node in parent_rel.nodes:
|
|
47
|
-
self.
|
|
48
|
+
self._child_parent_identifier_map[child_node.identifier] = (
|
|
49
|
+
parent_node.identifier,
|
|
50
|
+
parent_rel.name,
|
|
51
|
+
)
|
|
48
52
|
# UUIDs of all the parents, removing the stale parents from the earlier diff
|
|
49
|
-
self.
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
52
|
-
self.
|
|
53
|
+
self._parent_node_identifiers = {parent_tuple[0] for parent_tuple in self._child_parent_identifier_map.values()}
|
|
54
|
+
self._earlier_nodes_by_identifier = {n.identifier: n for n in earlier_diff.nodes}
|
|
55
|
+
self._later_nodes_by_identifier = {n.identifier: n for n in later_diff.nodes}
|
|
56
|
+
self._common_node_identifiers = set(self._earlier_nodes_by_identifier.keys()) & set(
|
|
57
|
+
self._later_nodes_by_identifier.keys()
|
|
58
|
+
)
|
|
53
59
|
|
|
54
60
|
@property
|
|
55
61
|
def diff_branch_name(self) -> str:
|
|
@@ -61,13 +67,13 @@ class DiffCombiner:
|
|
|
61
67
|
filtered_node_pairs: list[NodePair] = []
|
|
62
68
|
for earlier_node in earlier_diff.nodes:
|
|
63
69
|
later_node: EnrichedDiffNode | None = None
|
|
64
|
-
if earlier_node.
|
|
65
|
-
later_node = self.
|
|
70
|
+
if earlier_node.identifier in self._common_node_identifiers:
|
|
71
|
+
later_node = self._later_nodes_by_identifier[earlier_node.identifier]
|
|
66
72
|
# this is an out-of-date parent
|
|
67
73
|
if (
|
|
68
74
|
earlier_node.action is DiffAction.UNCHANGED
|
|
69
75
|
and (later_node is None or later_node.action is DiffAction.UNCHANGED)
|
|
70
|
-
and earlier_node.
|
|
76
|
+
and earlier_node.identifier not in self._parent_node_identifiers
|
|
71
77
|
):
|
|
72
78
|
continue
|
|
73
79
|
if later_node is None:
|
|
@@ -79,15 +85,15 @@ class DiffCombiner:
|
|
|
79
85
|
filtered_node_pairs.append(NodePair(earlier=earlier_node, later=later_node))
|
|
80
86
|
for later_node in later_diff.nodes:
|
|
81
87
|
# these have already been handled
|
|
82
|
-
if later_node.
|
|
88
|
+
if later_node.identifier in self._common_node_identifiers:
|
|
83
89
|
continue
|
|
84
90
|
filtered_node_pairs.append(NodePair(later=later_node))
|
|
85
91
|
return filtered_node_pairs
|
|
86
92
|
|
|
87
|
-
def _get_parent_relationship_name(self, node_id:
|
|
88
|
-
if node_id not in self.
|
|
93
|
+
def _get_parent_relationship_name(self, node_id: NodeIdentifier) -> str | None:
|
|
94
|
+
if node_id not in self._child_parent_identifier_map:
|
|
89
95
|
return None
|
|
90
|
-
return self.
|
|
96
|
+
return self._child_parent_identifier_map[node_id][1]
|
|
91
97
|
|
|
92
98
|
def _should_include(self, earlier: DiffAction, later: DiffAction) -> bool:
|
|
93
99
|
actions = {earlier, later}
|
|
@@ -284,7 +290,7 @@ class DiffCombiner:
|
|
|
284
290
|
self,
|
|
285
291
|
earlier_relationships: set[EnrichedDiffRelationship],
|
|
286
292
|
later_relationships: set[EnrichedDiffRelationship],
|
|
287
|
-
node_id:
|
|
293
|
+
node_id: NodeIdentifier,
|
|
288
294
|
) -> set[EnrichedDiffRelationship]:
|
|
289
295
|
earlier_rels_by_name = {rel.name: rel for rel in earlier_relationships}
|
|
290
296
|
later_rels_by_name = {rel.name: rel for rel in later_relationships}
|
|
@@ -365,7 +371,7 @@ class DiffCombiner:
|
|
|
365
371
|
combined_relationships = self._combine_relationships(
|
|
366
372
|
earlier_relationships=node_pair.earlier.relationships,
|
|
367
373
|
later_relationships=node_pair.later.relationships,
|
|
368
|
-
node_id=node_pair.later.
|
|
374
|
+
node_id=node_pair.later.identifier,
|
|
369
375
|
)
|
|
370
376
|
if all(ca.action is DiffAction.UNCHANGED for ca in combined_attributes) and all(
|
|
371
377
|
cr.action is DiffAction.UNCHANGED for cr in combined_relationships
|
|
@@ -380,15 +386,16 @@ class DiffCombiner:
|
|
|
380
386
|
combined_attributes
|
|
381
387
|
or combined_relationships
|
|
382
388
|
or combined_conflict
|
|
383
|
-
or node_pair.later.
|
|
389
|
+
or node_pair.later.identifier in self._parent_node_identifiers
|
|
384
390
|
):
|
|
385
391
|
combined_nodes.add(
|
|
386
392
|
EnrichedDiffNode(
|
|
387
|
-
|
|
388
|
-
kind=node_pair.later.kind,
|
|
393
|
+
identifier=node_pair.later.identifier,
|
|
389
394
|
label=node_pair.later.label,
|
|
390
395
|
changed_at=node_pair.later.changed_at or node_pair.earlier.changed_at,
|
|
391
396
|
action=combined_action,
|
|
397
|
+
is_node_kind_migration=node_pair.earlier.is_node_kind_migration
|
|
398
|
+
or node_pair.later.is_node_kind_migration,
|
|
392
399
|
path_identifier=node_pair.later.path_identifier,
|
|
393
400
|
attributes=combined_attributes,
|
|
394
401
|
relationships=combined_relationships,
|
|
@@ -398,12 +405,12 @@ class DiffCombiner:
|
|
|
398
405
|
return combined_nodes
|
|
399
406
|
|
|
400
407
|
def _link_child_nodes(self, nodes: Iterable[EnrichedDiffNode]) -> None:
|
|
401
|
-
|
|
402
|
-
for child_node in
|
|
403
|
-
if child_node.
|
|
408
|
+
nodes_by_identifier: dict[NodeIdentifier, EnrichedDiffNode] = {n.identifier: n for n in nodes}
|
|
409
|
+
for child_node in nodes_by_identifier.values():
|
|
410
|
+
if child_node.identifier not in self._child_parent_identifier_map:
|
|
404
411
|
continue
|
|
405
|
-
|
|
406
|
-
parent_node =
|
|
412
|
+
parent_identifier, parent_rel_name = self._child_parent_identifier_map[child_node.identifier]
|
|
413
|
+
parent_node = nodes_by_identifier[parent_identifier]
|
|
407
414
|
parent_rel = child_node.get_relationship(name=parent_rel_name)
|
|
408
415
|
parent_rel.nodes.add(parent_node)
|
|
409
416
|
|