infrahub-server 1.4.13__py3-none-any.whl → 1.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. infrahub/actions/tasks.py +208 -16
  2. infrahub/api/artifact.py +3 -0
  3. infrahub/api/diff/diff.py +1 -1
  4. infrahub/api/internal.py +2 -0
  5. infrahub/api/query.py +2 -0
  6. infrahub/api/schema.py +27 -3
  7. infrahub/auth.py +5 -5
  8. infrahub/cli/__init__.py +2 -0
  9. infrahub/cli/db.py +160 -157
  10. infrahub/cli/dev.py +118 -0
  11. infrahub/cli/upgrade.py +56 -9
  12. infrahub/computed_attribute/tasks.py +19 -7
  13. infrahub/config.py +7 -2
  14. infrahub/core/attribute.py +35 -24
  15. infrahub/core/branch/enums.py +1 -1
  16. infrahub/core/branch/models.py +9 -5
  17. infrahub/core/branch/needs_rebase_status.py +11 -0
  18. infrahub/core/branch/tasks.py +72 -10
  19. infrahub/core/changelog/models.py +2 -10
  20. infrahub/core/constants/__init__.py +4 -0
  21. infrahub/core/constants/infrahubkind.py +1 -0
  22. infrahub/core/convert_object_type/object_conversion.py +201 -0
  23. infrahub/core/convert_object_type/repository_conversion.py +89 -0
  24. infrahub/core/convert_object_type/schema_mapping.py +27 -3
  25. infrahub/core/diff/model/path.py +4 -0
  26. infrahub/core/diff/payload_builder.py +1 -1
  27. infrahub/core/diff/query/artifact.py +1 -0
  28. infrahub/core/diff/query/field_summary.py +1 -0
  29. infrahub/core/graph/__init__.py +1 -1
  30. infrahub/core/initialization.py +7 -4
  31. infrahub/core/manager.py +3 -81
  32. infrahub/core/migrations/__init__.py +3 -0
  33. infrahub/core/migrations/exceptions.py +4 -0
  34. infrahub/core/migrations/graph/__init__.py +11 -10
  35. infrahub/core/migrations/graph/load_schema_branch.py +21 -0
  36. infrahub/core/migrations/graph/m013_convert_git_password_credential.py +1 -1
  37. infrahub/core/migrations/graph/m037_index_attr_vals.py +11 -30
  38. infrahub/core/migrations/graph/m039_ipam_reconcile.py +9 -7
  39. infrahub/core/migrations/graph/m042_profile_attrs_in_db.py +147 -0
  40. infrahub/core/migrations/graph/m043_create_hfid_display_label_in_db.py +164 -0
  41. infrahub/core/migrations/graph/m044_backfill_hfid_display_label_in_db.py +864 -0
  42. infrahub/core/migrations/query/__init__.py +7 -8
  43. infrahub/core/migrations/query/attribute_add.py +8 -6
  44. infrahub/core/migrations/query/attribute_remove.py +134 -0
  45. infrahub/core/migrations/runner.py +54 -0
  46. infrahub/core/migrations/schema/attribute_kind_update.py +9 -3
  47. infrahub/core/migrations/schema/attribute_supports_profile.py +90 -0
  48. infrahub/core/migrations/schema/node_attribute_add.py +26 -5
  49. infrahub/core/migrations/schema/node_attribute_remove.py +13 -109
  50. infrahub/core/migrations/schema/node_kind_update.py +2 -1
  51. infrahub/core/migrations/schema/node_remove.py +2 -1
  52. infrahub/core/migrations/schema/placeholder_dummy.py +3 -2
  53. infrahub/core/migrations/shared.py +66 -19
  54. infrahub/core/models.py +2 -2
  55. infrahub/core/node/__init__.py +207 -54
  56. infrahub/core/node/create.py +53 -49
  57. infrahub/core/node/lock_utils.py +124 -0
  58. infrahub/core/node/node_property_attribute.py +230 -0
  59. infrahub/core/node/resource_manager/ip_address_pool.py +2 -1
  60. infrahub/core/node/resource_manager/ip_prefix_pool.py +2 -1
  61. infrahub/core/node/resource_manager/number_pool.py +2 -1
  62. infrahub/core/node/standard.py +1 -1
  63. infrahub/core/property.py +11 -0
  64. infrahub/core/protocols.py +8 -1
  65. infrahub/core/query/attribute.py +82 -15
  66. infrahub/core/query/ipam.py +16 -4
  67. infrahub/core/query/node.py +66 -188
  68. infrahub/core/query/relationship.py +44 -26
  69. infrahub/core/query/subquery.py +0 -8
  70. infrahub/core/relationship/model.py +69 -24
  71. infrahub/core/schema/__init__.py +56 -0
  72. infrahub/core/schema/attribute_schema.py +4 -2
  73. infrahub/core/schema/basenode_schema.py +42 -2
  74. infrahub/core/schema/definitions/core/__init__.py +2 -0
  75. infrahub/core/schema/definitions/core/check.py +1 -1
  76. infrahub/core/schema/definitions/core/generator.py +2 -0
  77. infrahub/core/schema/definitions/core/group.py +16 -2
  78. infrahub/core/schema/definitions/core/repository.py +7 -0
  79. infrahub/core/schema/definitions/core/transform.py +1 -1
  80. infrahub/core/schema/definitions/internal.py +12 -3
  81. infrahub/core/schema/generated/attribute_schema.py +2 -2
  82. infrahub/core/schema/generated/base_node_schema.py +6 -1
  83. infrahub/core/schema/manager.py +3 -0
  84. infrahub/core/schema/node_schema.py +1 -0
  85. infrahub/core/schema/relationship_schema.py +0 -1
  86. infrahub/core/schema/schema_branch.py +295 -10
  87. infrahub/core/schema/schema_branch_display.py +135 -0
  88. infrahub/core/schema/schema_branch_hfid.py +120 -0
  89. infrahub/core/validators/aggregated_checker.py +1 -1
  90. infrahub/database/graph.py +21 -0
  91. infrahub/display_labels/__init__.py +0 -0
  92. infrahub/display_labels/gather.py +48 -0
  93. infrahub/display_labels/models.py +240 -0
  94. infrahub/display_labels/tasks.py +192 -0
  95. infrahub/display_labels/triggers.py +22 -0
  96. infrahub/events/branch_action.py +27 -1
  97. infrahub/events/group_action.py +1 -1
  98. infrahub/events/node_action.py +1 -1
  99. infrahub/generators/constants.py +7 -0
  100. infrahub/generators/models.py +38 -12
  101. infrahub/generators/tasks.py +34 -16
  102. infrahub/git/base.py +38 -1
  103. infrahub/git/integrator.py +22 -14
  104. infrahub/graphql/api/dependencies.py +2 -4
  105. infrahub/graphql/api/endpoints.py +16 -6
  106. infrahub/graphql/app.py +2 -4
  107. infrahub/graphql/initialization.py +2 -3
  108. infrahub/graphql/manager.py +213 -137
  109. infrahub/graphql/middleware.py +12 -0
  110. infrahub/graphql/mutations/branch.py +16 -0
  111. infrahub/graphql/mutations/computed_attribute.py +110 -3
  112. infrahub/graphql/mutations/convert_object_type.py +44 -13
  113. infrahub/graphql/mutations/display_label.py +118 -0
  114. infrahub/graphql/mutations/generator.py +25 -7
  115. infrahub/graphql/mutations/hfid.py +125 -0
  116. infrahub/graphql/mutations/ipam.py +73 -41
  117. infrahub/graphql/mutations/main.py +61 -178
  118. infrahub/graphql/mutations/profile.py +195 -0
  119. infrahub/graphql/mutations/proposed_change.py +8 -1
  120. infrahub/graphql/mutations/relationship.py +2 -2
  121. infrahub/graphql/mutations/repository.py +22 -83
  122. infrahub/graphql/mutations/resource_manager.py +2 -2
  123. infrahub/graphql/mutations/webhook.py +1 -1
  124. infrahub/graphql/queries/resource_manager.py +1 -1
  125. infrahub/graphql/registry.py +173 -0
  126. infrahub/graphql/resolvers/resolver.py +2 -0
  127. infrahub/graphql/schema.py +8 -1
  128. infrahub/graphql/schema_sort.py +170 -0
  129. infrahub/graphql/types/branch.py +4 -1
  130. infrahub/graphql/types/enums.py +3 -0
  131. infrahub/groups/tasks.py +1 -1
  132. infrahub/hfid/__init__.py +0 -0
  133. infrahub/hfid/gather.py +48 -0
  134. infrahub/hfid/models.py +240 -0
  135. infrahub/hfid/tasks.py +191 -0
  136. infrahub/hfid/triggers.py +22 -0
  137. infrahub/lock.py +119 -42
  138. infrahub/locks/__init__.py +0 -0
  139. infrahub/locks/tasks.py +37 -0
  140. infrahub/patch/plan_writer.py +2 -2
  141. infrahub/permissions/constants.py +2 -0
  142. infrahub/profiles/__init__.py +0 -0
  143. infrahub/profiles/node_applier.py +101 -0
  144. infrahub/profiles/queries/__init__.py +0 -0
  145. infrahub/profiles/queries/get_profile_data.py +98 -0
  146. infrahub/profiles/tasks.py +63 -0
  147. infrahub/proposed_change/tasks.py +24 -5
  148. infrahub/repositories/__init__.py +0 -0
  149. infrahub/repositories/create_repository.py +113 -0
  150. infrahub/server.py +9 -1
  151. infrahub/services/__init__.py +8 -5
  152. infrahub/services/adapters/workflow/worker.py +5 -2
  153. infrahub/task_manager/event.py +5 -0
  154. infrahub/task_manager/models.py +7 -0
  155. infrahub/tasks/registry.py +6 -4
  156. infrahub/trigger/catalogue.py +4 -0
  157. infrahub/trigger/models.py +2 -0
  158. infrahub/trigger/setup.py +13 -4
  159. infrahub/trigger/tasks.py +6 -0
  160. infrahub/webhook/models.py +1 -1
  161. infrahub/workers/dependencies.py +3 -1
  162. infrahub/workers/infrahub_async.py +5 -1
  163. infrahub/workflows/catalogue.py +118 -3
  164. infrahub/workflows/initialization.py +21 -0
  165. infrahub/workflows/models.py +17 -2
  166. infrahub_sdk/branch.py +17 -8
  167. infrahub_sdk/checks.py +1 -1
  168. infrahub_sdk/client.py +376 -95
  169. infrahub_sdk/config.py +29 -2
  170. infrahub_sdk/convert_object_type.py +61 -0
  171. infrahub_sdk/ctl/branch.py +3 -0
  172. infrahub_sdk/ctl/check.py +2 -3
  173. infrahub_sdk/ctl/cli_commands.py +20 -12
  174. infrahub_sdk/ctl/config.py +8 -2
  175. infrahub_sdk/ctl/generator.py +6 -3
  176. infrahub_sdk/ctl/graphql.py +184 -0
  177. infrahub_sdk/ctl/repository.py +39 -1
  178. infrahub_sdk/ctl/schema.py +40 -10
  179. infrahub_sdk/ctl/task.py +110 -0
  180. infrahub_sdk/ctl/utils.py +4 -0
  181. infrahub_sdk/ctl/validate.py +5 -3
  182. infrahub_sdk/diff.py +4 -5
  183. infrahub_sdk/exceptions.py +2 -0
  184. infrahub_sdk/generator.py +7 -1
  185. infrahub_sdk/graphql/__init__.py +12 -0
  186. infrahub_sdk/graphql/constants.py +1 -0
  187. infrahub_sdk/graphql/plugin.py +85 -0
  188. infrahub_sdk/graphql/query.py +77 -0
  189. infrahub_sdk/{graphql.py → graphql/renderers.py} +88 -75
  190. infrahub_sdk/graphql/utils.py +40 -0
  191. infrahub_sdk/node/attribute.py +2 -0
  192. infrahub_sdk/node/node.py +28 -20
  193. infrahub_sdk/node/relationship.py +1 -3
  194. infrahub_sdk/playback.py +1 -2
  195. infrahub_sdk/protocols.py +54 -6
  196. infrahub_sdk/pytest_plugin/plugin.py +7 -4
  197. infrahub_sdk/pytest_plugin/utils.py +40 -0
  198. infrahub_sdk/repository.py +1 -2
  199. infrahub_sdk/schema/__init__.py +70 -4
  200. infrahub_sdk/schema/main.py +1 -0
  201. infrahub_sdk/schema/repository.py +8 -0
  202. infrahub_sdk/spec/models.py +7 -0
  203. infrahub_sdk/spec/object.py +54 -6
  204. infrahub_sdk/spec/processors/__init__.py +0 -0
  205. infrahub_sdk/spec/processors/data_processor.py +10 -0
  206. infrahub_sdk/spec/processors/factory.py +34 -0
  207. infrahub_sdk/spec/processors/range_expand_processor.py +56 -0
  208. infrahub_sdk/spec/range_expansion.py +118 -0
  209. infrahub_sdk/task/models.py +6 -4
  210. infrahub_sdk/timestamp.py +18 -6
  211. infrahub_sdk/transforms.py +1 -1
  212. {infrahub_server-1.4.13.dist-info → infrahub_server-1.5.0.dist-info}/METADATA +9 -10
  213. {infrahub_server-1.4.13.dist-info → infrahub_server-1.5.0.dist-info}/RECORD +221 -165
  214. infrahub_testcontainers/container.py +114 -2
  215. infrahub_testcontainers/docker-compose-cluster.test.yml +5 -0
  216. infrahub_testcontainers/docker-compose.test.yml +5 -0
  217. infrahub_testcontainers/models.py +2 -2
  218. infrahub_testcontainers/performance_test.py +4 -4
  219. infrahub/core/convert_object_type/conversion.py +0 -134
  220. {infrahub_server-1.4.13.dist-info → infrahub_server-1.5.0.dist-info}/LICENSE.txt +0 -0
  221. {infrahub_server-1.4.13.dist-info → infrahub_server-1.5.0.dist-info}/WHEEL +0 -0
  222. {infrahub_server-1.4.13.dist-info → infrahub_server-1.5.0.dist-info}/entry_points.txt +0 -0
infrahub/core/manager.py CHANGED
@@ -1,7 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  from copy import copy
4
- from functools import reduce
5
4
  from typing import TYPE_CHECKING, Any, Iterable, Literal, TypeVar, overload
6
5
 
7
6
  from infrahub_sdk.utils import deep_merge_dict, is_valid_uuid
@@ -11,9 +10,7 @@ from infrahub.core.node import Node
11
10
  from infrahub.core.node.delete_validator import NodeDeleteValidator
12
11
  from infrahub.core.query.node import (
13
12
  AttributeFromDB,
14
- AttributeNodePropertyFromDB,
15
13
  GroupedPeerNodes,
16
- NodeAttributesFromDB,
17
14
  NodeGetHierarchyQuery,
18
15
  NodeGetListQuery,
19
16
  NodeListGetAttributeQuery,
@@ -78,60 +75,6 @@ def get_schema(
78
75
  return node_schema
79
76
 
80
77
 
81
- class ProfileAttributeIndex:
82
- def __init__(
83
- self,
84
- profile_attributes_id_map: dict[str, NodeAttributesFromDB],
85
- profile_ids_by_node_id: dict[str, list[str]],
86
- ) -> None:
87
- self._profile_attributes_id_map = profile_attributes_id_map
88
- self._profile_ids_by_node_id = profile_ids_by_node_id
89
-
90
- def apply_profiles(self, node_data_dict: dict[str, Any]) -> dict[str, Any]:
91
- updated_data: dict[str, Any] = {**node_data_dict}
92
- node_id = node_data_dict.get("id")
93
- profile_ids = self._profile_ids_by_node_id.get(node_id, [])
94
- if not profile_ids:
95
- return updated_data
96
- profiles = [
97
- self._profile_attributes_id_map[p_id] for p_id in profile_ids if p_id in self._profile_attributes_id_map
98
- ]
99
-
100
- def get_profile_priority(nafd: NodeAttributesFromDB) -> tuple[int | float, str]:
101
- try:
102
- return (int(nafd.attrs.get("profile_priority").value), nafd.node.get("uuid"))
103
- except (TypeError, AttributeError):
104
- return (float("inf"), "")
105
-
106
- profiles.sort(key=get_profile_priority)
107
-
108
- for attr_name, attr_data in updated_data.items():
109
- if not isinstance(attr_data, AttributeFromDB):
110
- continue
111
- if not attr_data.is_default:
112
- continue
113
- profile_value, profile_uuid = None, None
114
- index = 0
115
-
116
- while profile_value is None and index <= (len(profiles) - 1):
117
- try:
118
- profile_value = profiles[index].attrs[attr_name].value
119
- if profile_value != "NULL":
120
- profile_uuid = profiles[index].node["uuid"]
121
- break
122
- profile_value = None
123
- except (IndexError, KeyError, AttributeError):
124
- ...
125
- index += 1
126
-
127
- if profile_value is not None:
128
- attr_data.value = profile_value
129
- attr_data.is_from_profile = True
130
- attr_data.is_default = False
131
- attr_data.node_properties["source"] = AttributeNodePropertyFromDB(uuid=profile_uuid, labels=[])
132
- return updated_data
133
-
134
-
135
78
  class NodeManager:
136
79
  @overload
137
80
  @classmethod
@@ -1132,21 +1075,11 @@ class NodeManager:
1132
1075
  )
1133
1076
  await query.execute(db=db)
1134
1077
  nodes_info_by_id: dict[str, NodeToProcess] = {node.node_uuid: node async for node in query.get_nodes(db=db)}
1135
- profile_ids_by_node_id = query.get_profile_ids_by_node_id()
1136
- all_profile_ids = reduce(
1137
- lambda all_ids, these_ids: all_ids | set(these_ids), profile_ids_by_node_id.values(), set()
1138
- )
1139
-
1140
- if fields and all_profile_ids:
1141
- if "profile_priority" not in fields:
1142
- fields["profile_priority"] = {}
1143
- if "value" not in fields["profile_priority"]:
1144
- fields["profile_priority"]["value"] = None
1145
1078
 
1146
1079
  # Query list of all Attributes
1147
1080
  query = await NodeListGetAttributeQuery.init(
1148
1081
  db=db,
1149
- ids=list(nodes_info_by_id.keys()) + list(all_profile_ids),
1082
+ ids=list(nodes_info_by_id.keys()),
1150
1083
  fields=fields,
1151
1084
  branch=branch,
1152
1085
  include_source=include_source,
@@ -1156,17 +1089,7 @@ class NodeManager:
1156
1089
  branch_agnostic=branch_agnostic,
1157
1090
  )
1158
1091
  await query.execute(db=db)
1159
- all_node_attributes = query.get_attributes_group_by_node()
1160
- profile_attributes: dict[str, dict[str, AttributeFromDB]] = {}
1161
- node_attributes: dict[str, dict[str, AttributeFromDB]] = {}
1162
- for node_id, attribute_dict in all_node_attributes.items():
1163
- if node_id in all_profile_ids:
1164
- profile_attributes[node_id] = attribute_dict
1165
- else:
1166
- node_attributes[node_id] = attribute_dict
1167
- profile_index = ProfileAttributeIndex(
1168
- profile_attributes_id_map=profile_attributes, profile_ids_by_node_id=profile_ids_by_node_id
1169
- )
1092
+ node_attributes = query.get_attributes_group_by_node()
1170
1093
 
1171
1094
  nodes: dict[str, Node] = {}
1172
1095
 
@@ -1195,11 +1118,10 @@ class NodeManager:
1195
1118
  for attr_name, attr in node_attributes[node_id].attrs.items():
1196
1119
  new_node_data[attr_name] = attr
1197
1120
 
1198
- new_node_data_with_profile_overrides = profile_index.apply_profiles(new_node_data)
1199
1121
  node_class = identify_node_class(node=node)
1200
1122
  node_branch = await registry.get_branch(db=db, branch=node.branch)
1201
1123
  item = await node_class.init(schema=node.schema, branch=node_branch, at=at, db=db)
1202
- await item.load(**new_node_data_with_profile_overrides, db=db)
1124
+ await item.load(**new_node_data, db=db)
1203
1125
 
1204
1126
  nodes[node_id] = item
1205
1127
 
@@ -1,5 +1,6 @@
1
1
  from .schema.attribute_kind_update import AttributeKindUpdateMigration
2
2
  from .schema.attribute_name_update import AttributeNameUpdateMigration
3
+ from .schema.attribute_supports_profile import AttributeSupportsProfileUpdateMigration
3
4
  from .schema.node_attribute_add import NodeAttributeAddMigration
4
5
  from .schema.node_attribute_remove import NodeAttributeRemoveMigration
5
6
  from .schema.node_kind_update import NodeKindUpdateMigration
@@ -19,6 +20,8 @@ MIGRATION_MAP: dict[str, type[SchemaMigration] | None] = {
19
20
  "attribute.name.update": AttributeNameUpdateMigration,
20
21
  "attribute.branch.update": None,
21
22
  "attribute.kind.update": AttributeKindUpdateMigration,
23
+ "attribute.optional.update": AttributeSupportsProfileUpdateMigration,
24
+ "attribute.read_only.update": AttributeSupportsProfileUpdateMigration,
22
25
  "relationship.branch.update": None,
23
26
  "relationship.direction.update": None,
24
27
  "relationship.identifier.update": PlaceholderDummyMigration,
@@ -0,0 +1,4 @@
1
+ class MigrationFailureError(Exception):
2
+ def __init__(self, errors: list[str]) -> None:
3
+ super().__init__()
4
+ self.errors = errors
@@ -43,13 +43,15 @@ from .m038_redo_0000_prefix_fix import Migration038
43
43
  from .m039_ipam_reconcile import Migration039
44
44
  from .m040_duplicated_attributes import Migration040
45
45
  from .m041_deleted_dup_edges import Migration041
46
+ from .m042_profile_attrs_in_db import Migration042
47
+ from .m043_create_hfid_display_label_in_db import Migration043
48
+ from .m044_backfill_hfid_display_label_in_db import Migration044
46
49
 
47
50
  if TYPE_CHECKING:
48
- from infrahub.core.root import Root
51
+ from ..shared import MigrationTypes
49
52
 
50
- from ..shared import ArbitraryMigration, GraphMigration, InternalSchemaMigration
51
53
 
52
- MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigration]] = [
54
+ MIGRATIONS: list[type[MigrationTypes]] = [
53
55
  Migration001,
54
56
  Migration002,
55
57
  Migration003,
@@ -91,25 +93,24 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
91
93
  Migration039,
92
94
  Migration040,
93
95
  Migration041,
96
+ Migration042,
97
+ Migration043,
98
+ Migration044,
94
99
  ]
95
100
 
96
101
 
97
- async def get_graph_migrations(
98
- root: Root,
99
- ) -> Sequence[GraphMigration | InternalSchemaMigration | ArbitraryMigration]:
102
+ async def get_graph_migrations(current_graph_version: int) -> Sequence[MigrationTypes]:
100
103
  applicable_migrations = []
101
104
  for migration_class in MIGRATIONS:
102
105
  migration = migration_class.init()
103
- if root.graph_version > migration.minimum_version:
106
+ if current_graph_version > migration.minimum_version:
104
107
  continue
105
108
  applicable_migrations.append(migration)
106
109
 
107
110
  return applicable_migrations
108
111
 
109
112
 
110
- def get_migration_by_number(
111
- migration_number: int | str,
112
- ) -> GraphMigration | InternalSchemaMigration | ArbitraryMigration:
113
+ def get_migration_by_number(migration_number: int | str) -> MigrationTypes:
113
114
  # Convert to string and pad with zeros if needed
114
115
  try:
115
116
  num = int(migration_number)
@@ -0,0 +1,21 @@
1
+ from infrahub.core import registry
2
+ from infrahub.core.branch import Branch
3
+ from infrahub.core.schema import SchemaRoot, internal_schema
4
+ from infrahub.core.schema.manager import SchemaManager
5
+ from infrahub.core.schema.schema_branch import SchemaBranch
6
+ from infrahub.database import InfrahubDatabase
7
+ from infrahub.exceptions import InitializationError
8
+
9
+
10
+ async def get_or_load_schema_branch(db: InfrahubDatabase, branch: Branch) -> SchemaBranch:
11
+ try:
12
+ if registry.schema.has_schema_branch(branch.name):
13
+ return registry.schema.get_schema_branch(branch.name)
14
+ except InitializationError:
15
+ schema_manager = SchemaManager()
16
+ registry.schema = schema_manager
17
+ internal_schema_root = SchemaRoot(**internal_schema)
18
+ registry.schema.register_schema(schema=internal_schema_root)
19
+ schema_branch = await registry.schema.load_schema_from_db(db=db, branch=branch)
20
+ registry.schema.set_schema_branch(name=branch.name, schema=schema_branch)
21
+ return schema_branch
@@ -286,7 +286,7 @@ class Migration013AddInternalStatusData(AttributeAddQuery):
286
286
  kwargs.pop("branch", None)
287
287
 
288
288
  super().__init__(
289
- node_kind="CoreGenericRepository",
289
+ node_kinds=["CoreGenericRepository"],
290
290
  attribute_name="internal_status",
291
291
  attribute_kind="Dropdown",
292
292
  branch_support=BranchSupportType.LOCAL.value,
@@ -3,13 +3,10 @@ from __future__ import annotations
3
3
  from dataclasses import dataclass
4
4
  from typing import TYPE_CHECKING, Any
5
5
 
6
- from rich.console import Console
7
-
8
6
  from infrahub.constants.database import IndexType
9
7
  from infrahub.core.attribute import MAX_STRING_LENGTH
10
- from infrahub.core.migrations.shared import MigrationResult
8
+ from infrahub.core.migrations.shared import MigrationResult, get_migration_console
11
9
  from infrahub.core.query import Query, QueryType
12
- from infrahub.core.timestamp import Timestamp
13
10
  from infrahub.database.index import IndexItem
14
11
  from infrahub.database.neo4j import IndexManagerNeo4j
15
12
  from infrahub.log import get_logger
@@ -467,13 +464,11 @@ class Migration037(ArbitraryMigration):
467
464
  return result
468
465
 
469
466
  async def execute(self, db: InfrahubDatabase) -> MigrationResult: # noqa: PLR0915
470
- console = Console()
467
+ console = get_migration_console()
471
468
  result = MigrationResult()
472
469
 
473
470
  # find the active schema attributes that have a LARGE_ATTRIBUTE_TYPE kind on all branches
474
- console.print(
475
- f"{Timestamp().to_string()} Determining schema attribute types and timestamps on all branches...", end=""
476
- )
471
+ console.print("Determining schema attribute types and timestamps on all branches...", end="")
477
472
  get_large_attribute_types_query = await GetLargeAttributeTypesQuery.init(db=db)
478
473
  await get_large_attribute_types_query.execute(db=db)
479
474
  schema_attribute_timeframes = get_large_attribute_types_query.get_large_attribute_type_timeframes()
@@ -481,10 +476,7 @@ class Migration037(ArbitraryMigration):
481
476
 
482
477
  # find which schema attributes are large_types in the default branch, but updated to non-large_type on other branches
483
478
  # {(kind, attr_name): SchemaAttributeTimeframe}
484
- console.print(
485
- f"{Timestamp().to_string()} Determining which schema attributes have been updated to non-large_type on non-default branches...",
486
- end="",
487
- )
479
+ console.print("Determining schema attribute updates on non-default branches...", end="")
488
480
  main_schema_attribute_timeframes_map: dict[tuple[str, str], SchemaAttributeTimeframe] = {}
489
481
  for schema_attr_time in schema_attribute_timeframes:
490
482
  if schema_attr_time.is_default_branch:
@@ -508,7 +500,7 @@ class Migration037(ArbitraryMigration):
508
500
  console.print("done")
509
501
 
510
502
  # drop the index on the AttributeValueNonIndexed vertex, there won't be any at this point anyway
511
- console.print(f"{Timestamp().to_string()} Dropping index on AttributeValueIndexed vertices...", end="")
503
+ console.print("Dropping index on AttributeValueIndexed vertices...", end="")
512
504
  index_manager = IndexManagerNeo4j(db=db)
513
505
  index_manager.init(nodes=[AV_INDEXED_INDEX], rels=[])
514
506
  await index_manager.drop()
@@ -516,7 +508,7 @@ class Migration037(ArbitraryMigration):
516
508
 
517
509
  # create the temporary non-indexed attribute value vertices for LARGE_ATTRIBUTE_TYPE attributes
518
510
  # start with default branch
519
- console.print(f"{Timestamp().to_string()} Update non-indexed attribute values with temporary label...", end="")
511
+ console.print("Creating temporary non-indexed attribute values for large attribute types...", end="")
520
512
  large_schema_attribute_timeframes = [
521
513
  schema_attr_time for schema_attr_time in schema_attribute_timeframes if schema_attr_time.is_large_type
522
514
  ]
@@ -528,10 +520,7 @@ class Migration037(ArbitraryMigration):
528
520
  console.print("done")
529
521
 
530
522
  # re-index attribute values on branches where the type was updated to non-large_type
531
- console.print(
532
- f"{Timestamp().to_string()} Indexing attribute values on branches where the attribute schema was updated to a non-large_type...",
533
- end="",
534
- )
523
+ console.print("Re-indexing attribute values on branches updated to non-large types...", end="")
535
524
  for schema_attr_time in large_type_reverts:
536
525
  revert_non_index_on_branch_query = await RevertNonIndexOnBranchQuery.init(
537
526
  db=db, schema_attribute_timeframe=schema_attr_time
@@ -540,27 +529,19 @@ class Migration037(ArbitraryMigration):
540
529
  console.print("done")
541
530
 
542
531
  # set the AttributeValue vertices to be AttributeValueIndexed
543
- console.print(
544
- f"{Timestamp().to_string()} Update all AttributeValue vertices to add the AttributeValueIndexed label...",
545
- end="",
546
- )
532
+ console.print("Adding AttributeValueIndexed label to AttributeValue vertices...", end="")
547
533
  set_attribute_value_indexed_query = await SetAttributeValueIndexedQuery.init(db=db)
548
534
  await set_attribute_value_indexed_query.execute(db=db)
549
535
  console.print("done")
550
536
 
551
537
  # set AttributeValueNonIndexed vertices to just AttributeValue
552
- console.print(
553
- f"{Timestamp().to_string()} Update all AttributeValueNonIndexed vertices to be AttributeValue (no index)...",
554
- end="",
555
- )
538
+ console.print("Restoring AttributeValue label on AttributeValueNonIndexed vertices...", end="")
556
539
  finalize_attribute_value_non_indexed_query = await FinalizeAttributeValueNonIndexedQuery.init(db=db)
557
540
  await finalize_attribute_value_non_indexed_query.execute(db=db)
558
541
  console.print("done")
559
542
 
560
543
  # de-index all attribute values too large to be indexed
561
- console.print(
562
- f"{Timestamp().to_string()} De-index any legacy attribute data that is too large to be indexed...", end=""
563
- )
544
+ console.print("De-indexing legacy attribute data exceeding index limits...", end="")
564
545
  de_index_large_attribute_values_query = await DeIndexLargeAttributeValuesQuery.init(
565
546
  db=db, max_value_size=MAX_STRING_LENGTH
566
547
  )
@@ -568,7 +549,7 @@ class Migration037(ArbitraryMigration):
568
549
  console.print("done")
569
550
 
570
551
  # add the index back to the AttributeValueNonIndexed vertex
571
- console.print(f"{Timestamp().to_string()} Add the index back to the AttributeValueIndexed label...", end="")
552
+ console.print("Adding index back to the AttributeValueIndexed label...", end="")
572
553
  index_manager = IndexManagerNeo4j(db=db)
573
554
  index_manager.init(nodes=[AV_INDEXED_INDEX], rels=[])
574
555
  await index_manager.add()
@@ -4,14 +4,13 @@ import ipaddress
4
4
  from dataclasses import dataclass
5
5
  from typing import TYPE_CHECKING, Any
6
6
 
7
- from rich.console import Console
8
7
  from rich.progress import Progress
9
8
 
10
9
  from infrahub.core.branch.models import Branch
11
10
  from infrahub.core.constants import InfrahubKind
12
11
  from infrahub.core.initialization import initialization
13
12
  from infrahub.core.ipam.reconciler import IpamReconciler
14
- from infrahub.core.migrations.shared import MigrationResult
13
+ from infrahub.core.migrations.shared import MigrationResult, get_migration_console
15
14
  from infrahub.core.query import Query, QueryType
16
15
  from infrahub.lock import initialize_lock
17
16
  from infrahub.log import get_logger
@@ -235,13 +234,13 @@ class Migration039(ArbitraryMigration):
235
234
  return MigrationResult()
236
235
 
237
236
  async def execute(self, db: InfrahubDatabase) -> MigrationResult:
238
- console = Console()
237
+ console = get_migration_console()
239
238
  result = MigrationResult()
240
239
  # load schemas from database into registry
241
240
  initialize_lock()
242
241
  await initialization(db=db)
243
242
 
244
- console.print("Identifying IP prefixes/addresses to reconcile...", end="")
243
+ console.print("Identifying IP prefixes and addresses to reconcile...", end="")
245
244
  find_nodes_query = await FindNodesToReconcileQuery.init(db=db)
246
245
  await find_nodes_query.execute(db=db)
247
246
  console.print("done")
@@ -250,16 +249,17 @@ class Migration039(ArbitraryMigration):
250
249
  # reconciler cannot correctly handle a prefix that is its own parent
251
250
  ip_node_details_list = find_nodes_query.get_nodes_to_reconcile()
252
251
  uuids_to_check = {ip_node_details.node_uuid for ip_node_details in ip_node_details_list}
253
- console.print(f"{len(ip_node_details_list)} IP prefixes/addresses will be reconciled")
252
+ console.log(f"{len(ip_node_details_list)} IP prefixes or addresses will be reconciled.")
254
253
 
255
- console.print("Deleting any self-parent relationships...", end="")
254
+ console.print("Deleting self-parent relationships prior to reconciliation...", end="")
256
255
  delete_self_parent_relationships_query = await DeleteSelfParentRelationshipsQuery.init(
257
256
  db=db, uuids_to_check=list(uuids_to_check)
258
257
  )
259
258
  await delete_self_parent_relationships_query.execute(db=db)
260
259
  console.print("done")
261
260
 
262
- with Progress() as progress:
261
+ console.log("Reconciling IP prefixes and addresses across branches...")
262
+ with Progress(console=console) as progress:
263
263
  reconcile_task = progress.add_task("Reconciling IP prefixes/addresses...", total=len(ip_node_details_list))
264
264
 
265
265
  for ip_node_details in ip_node_details_list:
@@ -271,4 +271,6 @@ class Migration039(ArbitraryMigration):
271
271
  )
272
272
  progress.update(reconcile_task, advance=1)
273
273
 
274
+ console.log("IP prefix and address reconciliation complete.")
275
+
274
276
  return result
@@ -0,0 +1,147 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any
4
+
5
+ from rich.progress import Progress
6
+
7
+ from infrahub.core.branch.models import Branch
8
+ from infrahub.core.initialization import get_root_node
9
+ from infrahub.core.manager import NodeManager
10
+ from infrahub.core.migrations.shared import MigrationResult, get_migration_console
11
+ from infrahub.core.query import Query, QueryType
12
+ from infrahub.core.timestamp import Timestamp
13
+ from infrahub.log import get_logger
14
+ from infrahub.profiles.node_applier import NodeProfilesApplier
15
+
16
+ from ..shared import MigrationRequiringRebase
17
+ from .load_schema_branch import get_or_load_schema_branch
18
+
19
+ if TYPE_CHECKING:
20
+ from infrahub.database import InfrahubDatabase
21
+
22
+ log = get_logger()
23
+
24
+
25
+ class GetUpdatedProfilesForBranchQuery(Query):
26
+ """
27
+ Get CoreProfile UUIDs with updated attributes on this branch
28
+ """
29
+
30
+ name = "get_profiles_by_branch"
31
+ type = QueryType.READ
32
+
33
+ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
34
+ self.params["branch"] = self.branch.name
35
+ query = """
36
+ MATCH (profile:CoreProfile)-[:HAS_ATTRIBUTE]->(attr:Attribute)-[e:HAS_VALUE]->(:AttributeValue)
37
+ WHERE e.branch = $branch
38
+ WITH DISTINCT profile.uuid AS profile_uuid
39
+ """
40
+ self.add_to_query(query)
41
+ self.return_labels = ["profile_uuid"]
42
+
43
+ def get_profile_ids(self) -> list[str]:
44
+ """Get list of updated profile UUIDs"""
45
+ return [result.get_as_type("profile_uuid", str) for result in self.get_results()]
46
+
47
+
48
+ class GetNodesWithProfileUpdatesForBranchQuery(Query):
49
+ """
50
+ Get Node UUIDs by which branches they have updated profiles on
51
+ """
52
+
53
+ name = "get_nodes_with_profile_updates_by_branch"
54
+ type = QueryType.READ
55
+
56
+ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None: # noqa: ARG002
57
+ self.params["branch"] = self.branch.name
58
+ query = """
59
+ MATCH (node:Node)-[e:IS_RELATED]->(:Relationship {name: "node__profile"})
60
+ WHERE NOT node:CoreProfile
61
+ AND e.branch = $branch
62
+ WITH DISTINCT node.uuid AS node_uuid
63
+ """
64
+ self.add_to_query(query)
65
+ self.return_labels = ["node_uuid"]
66
+
67
+ def get_node_ids(self) -> list[str]:
68
+ """Get list of updated node UUIDs"""
69
+ return [result.get_as_type("node_uuid", str) for result in self.get_results()]
70
+
71
+
72
+ class Migration042(MigrationRequiringRebase):
73
+ """
74
+ Save profile attribute values on each node using the profile in the database
75
+ For any profile that has updates on a given branch (including default branch)
76
+ - run NodeProfilesApplier.apply_profiles on each node related to the profile on that branch
77
+ For any node that has an updated relationship to a profile on a given branch
78
+ - run NodeProfilesApplier.apply_profiles on the node on that branch
79
+ """
80
+
81
+ name: str = "042_profile_attrs_in_db"
82
+ minimum_version: int = 41
83
+
84
+ def _get_profile_applier(self, db: InfrahubDatabase, branch: Branch) -> NodeProfilesApplier:
85
+ return NodeProfilesApplier(db=db, branch=branch)
86
+
87
+ async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult: # noqa: ARG002
88
+ return MigrationResult()
89
+
90
+ async def execute(self, db: InfrahubDatabase) -> MigrationResult:
91
+ root_node = await get_root_node(db=db, initialize=False)
92
+ default_branch_name = root_node.default_branch
93
+ default_branch = await Branch.get_by_name(db=db, name=default_branch_name)
94
+ return await self._do_execute_for_branch(db=db, branch=default_branch)
95
+
96
+ async def execute_against_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
97
+ return await self._do_execute_for_branch(db=db, branch=branch)
98
+
99
+ async def _do_execute_for_branch(self, db: InfrahubDatabase, branch: Branch) -> MigrationResult:
100
+ console = get_migration_console()
101
+ result = MigrationResult()
102
+ await get_or_load_schema_branch(db=db, branch=branch)
103
+
104
+ console.print(f"Gathering profiles for branch {branch.name}...", end="")
105
+ get_updated_profiles_for_branch_query = await GetUpdatedProfilesForBranchQuery.init(db=db, branch=branch)
106
+ await get_updated_profiles_for_branch_query.execute(db=db)
107
+ profile_ids = get_updated_profiles_for_branch_query.get_profile_ids()
108
+
109
+ profiles_map = await NodeManager.get_many(db=db, branch=branch, ids=list(profile_ids))
110
+ console.print("done")
111
+
112
+ node_ids_to_update: set[str] = set()
113
+ with Progress(console=console) as progress:
114
+ gather_nodes_task = progress.add_task(
115
+ f"Gathering affected objects for each profile on branch {branch.name}...", total=len(profiles_map)
116
+ )
117
+
118
+ for profile in profiles_map.values():
119
+ node_relationship_manager = profile.get_relationship("related_nodes")
120
+ node_peers = await node_relationship_manager.get_db_peers(db=db)
121
+ node_ids_to_update.update(str(peer.peer_id) for peer in node_peers)
122
+ progress.update(gather_nodes_task, advance=1)
123
+ console.log(f"Collected nodes impacted by profiles on branch {branch.name}.")
124
+
125
+ console.print("Identifying nodes with profile updates by branch...", end="")
126
+ get_nodes_with_profile_updates_by_branch_query = await GetNodesWithProfileUpdatesForBranchQuery.init(
127
+ db=db, branch=branch
128
+ )
129
+ await get_nodes_with_profile_updates_by_branch_query.execute(db=db)
130
+ node_ids_to_update.update(get_nodes_with_profile_updates_by_branch_query.get_node_ids())
131
+ console.print("done")
132
+
133
+ right_now = Timestamp()
134
+ console.log("Applying profiles to nodes...")
135
+ with Progress(console=console) as progress:
136
+ apply_task = progress.add_task("Applying profiles to nodes...", total=len(node_ids_to_update))
137
+ applier = self._get_profile_applier(db=db, branch=branch)
138
+ for node_id in node_ids_to_update:
139
+ node = await NodeManager.get_one(db=db, branch=branch, id=node_id, at=right_now)
140
+ if node:
141
+ updated_field_names = await applier.apply_profiles(node=node)
142
+ if updated_field_names:
143
+ await node.save(db=db, fields=updated_field_names, at=right_now)
144
+ progress.update(apply_task, advance=1)
145
+ console.log("Completed applying profiles to nodes.")
146
+
147
+ return result