infrahub-server 1.4.10__py3-none-any.whl → 1.5.0b0__py3-none-any.whl

infrahub/core/convert_object_type/object_conversion.py

@@ -0,0 +1,201 @@
+from typing import Any, assert_never
+
+from infrahub_sdk.convert_object_type import ConversionFieldInput, ConversionFieldValue
+
+from infrahub.core.attribute import BaseAttribute
+from infrahub.core.branch import Branch
+from infrahub.core.branch.enums import BranchStatus
+from infrahub.core.constants import BranchSupportType, RelationshipCardinality
+from infrahub.core.manager import NodeManager
+from infrahub.core.node import Node
+from infrahub.core.node.create import create_node
+from infrahub.core.query.relationship import GetAllPeersIds
+from infrahub.core.query.resource_manager import PoolChangeReserved
+from infrahub.core.relationship import RelationshipManager
+from infrahub.core.schema import NodeSchema
+from infrahub.core.timestamp import Timestamp
+from infrahub.database import InfrahubDatabase
+from infrahub.message_bus.messages import RefreshRegistryBranches
+from infrahub.tasks.registry import update_branch_registry
+from infrahub.workers.dependencies import get_message_bus
+
+
+def _get_conversion_field_raw_value(conv_field_value: ConversionFieldValue) -> Any:
+    if conv_field_value.attribute_value is not None:
+        return conv_field_value.attribute_value
+    if conv_field_value.peer_id is not None:
+        return conv_field_value.peer_id
+    if conv_field_value.peers_ids is not None:
+        return conv_field_value.peers_ids
+    raise ValueError("ConversionFieldValue has not been validated correctly.")
+
+
+async def get_out_rels_peers_ids(node: Node, db: InfrahubDatabase, at: Timestamp) -> list[str]:
+    all_peers_ids: list[str] = []
+    for name in node._relationships:
+        relm: RelationshipManager = getattr(node, name)
+        peers = await relm.get_db_peers(db=db, at=at)
+        all_peers_ids.extend([str(peer.peer_id) for peer in peers])
+    return all_peers_ids
+
+
+async def build_data_new_node(db: InfrahubDatabase, mapping: dict[str, ConversionFieldInput], node: Node) -> dict:
+    """Value of a given field on the target kind to convert is either an input source attribute/relationship of the source node,
+    or a raw value."""
+
+    data = {}
+    for dest_field_name, conv_field_input in mapping.items():
+        if conv_field_input.source_field is not None:
+            # Fetch the value of the corresponding field from the node being converted.
+            item = getattr(node, conv_field_input.source_field)
+            if isinstance(item, BaseAttribute):
+                data[dest_field_name] = item.value
+            elif isinstance(item, RelationshipManager):
+                if item.schema.cardinality == RelationshipCardinality.ONE:
+                    peer = await item.get_peer(db=db)
+                    if peer is not None:
+                        data[dest_field_name] = {"id": peer.id}
+                    # else, relationship is optional, and if the target relationship is mandatory an error will be raised during creation
+                elif item.schema.cardinality == RelationshipCardinality.MANY:
+                    data[dest_field_name] = [{"id": peer.id} for _, peer in (await item.get_peers(db=db)).items()]
+                else:
+                    assert_never(item.schema.cardinality)
+        elif conv_field_input.data is not None:
+            data[dest_field_name] = _get_conversion_field_raw_value(conv_field_input.data)
+        elif conv_field_input.use_default_value is True:
+            pass  # default value will be used automatically when creating the node
+        else:
+            raise ValueError("ConversionFieldInput has not been validated correctly.")
+    return data
+
+
+async def get_unidirectional_rels_peers_ids(
+    node: Node, branch: Branch, db: InfrahubDatabase, at: Timestamp
+) -> list[str]:
+    """
+    Returns peers ids of nodes connected to input `node` through an incoming unidirectional relationship.
+    """
+
+    out_rels_identifier = [rel.identifier for rel in node.get_schema().relationships]
+    branch_agnostic = node.get_schema().branch == BranchSupportType.AGNOSTIC
+    query = await GetAllPeersIds.init(
+        db=db,
+        node_id=node.id,
+        branch=branch,
+        exclude_identifiers=out_rels_identifier,
+        branch_agnostic=branch_agnostic,
+        at=at,
+    )
+    await query.execute(db=db)
+    return query.get_peers_uuids()
+
+
+async def _get_other_active_branches(db: InfrahubDatabase) -> list[Branch]:
+    branches = await Branch.get_list(db=db)
+    return [branch for branch in branches if not (branch.is_global or branch.is_default)]
+
+
+def _has_pass_thru_aware_attributes(node_schema: NodeSchema, mapping: dict[str, ConversionFieldInput]) -> bool:
+    aware_attributes = [attr for attr in node_schema.attributes if attr.branch != BranchSupportType.AGNOSTIC]
+    aware_attributes_pass_thru = [
+        attr.name for attr in aware_attributes if attr.name in mapping and mapping[attr.name].source_field is not None
+    ]
+    return len(aware_attributes_pass_thru) > 0
+
+
+async def validate_conversion(
+    deleted_node: Node, branch: Branch, db: InfrahubDatabase, timestamp_before_conversion: Timestamp
+) -> None:
+    deleted_node_out_rels_peer_ids = await get_out_rels_peers_ids(
+        node=deleted_node, db=db, at=timestamp_before_conversion
+    )
+    deleted_node_unidir_rels_peer_ids = await get_unidirectional_rels_peers_ids(
+        node=deleted_node, db=db, branch=branch, at=timestamp_before_conversion
+    )
+
+    # Make sure relationships with constraints are not broken by retrieving them
+    peers_ids = deleted_node_out_rels_peer_ids + deleted_node_unidir_rels_peer_ids
+    peers = await NodeManager.get_many(ids=peers_ids, db=db, prefetch_relationships=True, branch=branch)
+    for peer in peers.values():
+        peer.validate_relationships()
+
+
+async def convert_and_validate_object_type(
+    node: Node,
+    target_schema: NodeSchema,
+    mapping: dict[str, ConversionFieldInput],
+    branch: Branch,
+    db: InfrahubDatabase,
+) -> Node:
+    async with db.start_transaction() as dbt:
+        timestamp_before_conversion = Timestamp()
+        new_node = await convert_object_type(
+            node=node, target_schema=target_schema, mapping=mapping, branch=branch, db=dbt
+        )
+        await validate_conversion(
+            deleted_node=node, branch=branch, db=dbt, timestamp_before_conversion=timestamp_before_conversion
+        )
+
+    # Refresh outside the transaction otherwise other workers would pull outdated branch objects.
+    message_bus = await get_message_bus()
+    await message_bus.send(RefreshRegistryBranches())
+
+    return new_node
+
+
+async def convert_object_type(
+    node: Node,
+    target_schema: NodeSchema,
+    mapping: dict[str, ConversionFieldInput],
+    branch: Branch,
+    db: InfrahubDatabase,
+) -> Node:
+    """Delete the node and return the new created one. If creation fails, the node is not deleted, and raise an error.
+    An extra check is performed on input node peers relationships to make sure they are still valid."""
+
+    node_schema = node.get_schema()
+    if not isinstance(node_schema, NodeSchema):
+        raise ValueError(f"Only a node with a NodeSchema can be converted, got {type(node_schema)}")
+
+    # Delete the node, so we delete relationships with peers as well, which might temporarily break cardinality constraints
+    # but they should be restored when creating the new node.
+    deleted_nodes = await NodeManager.delete(db=db, branch=branch, nodes=[node], cascade_delete=False)
+    if len(deleted_nodes) != 1:
+        raise ValueError(f"Deleted {len(deleted_nodes)} nodes instead of 1")
+
+    data_new_node = await build_data_new_node(db, mapping, node)
+
+    if node_schema.branch == BranchSupportType.AGNOSTIC and _has_pass_thru_aware_attributes(
+        node_schema=node_schema, mapping=mapping
+    ):
+        if not branch.is_default:
+            raise ValueError(
+                f"Conversion of {node_schema.kind} is not allowed on branch {branch.name} because it is agnostic and has aware attributes"
+            )
+
+        # When converting an agnostic node with aware attributes, we need to put other branches in NEED_REBASE state
+        # as aware attributes do not exist in other branches after conversion
+        other_branches = await _get_other_active_branches(db=db)
+        for br in other_branches:
+            br.status = BranchStatus.NEED_REBASE
+            await br.save(db=db)
+            # Registry of other API workers are updated outside the transaction
+            await update_branch_registry(db=db, branch=br)
+
+    node_created = await create_node(
+        data=data_new_node,
+        db=db,
+        branch=branch,
+        schema=target_schema,
+    )
+
+    # If the node had some value reserved in any Pools / Resource Manager, we need to change the identifier of the reservation(s)
+    query = await PoolChangeReserved.init(
+        db=db,
+        existing_identifier=node.get_id(),
+        new_identifier=node_created.get_id(),
+        branch=branch,
+    )
+    await query.execute(db=db)
+
+    return node_created

infrahub/core/convert_object_type/repository_conversion.py

@@ -0,0 +1,89 @@
+from infrahub import lock
+from infrahub.core.branch import Branch
+from infrahub.core.constants.infrahubkind import REPOSITORYVALIDATOR, USERVALIDATOR
+from infrahub.core.convert_object_type.object_conversion import (
+    ConversionFieldInput,
+    convert_object_type,
+    validate_conversion,
+)
+from infrahub.core.manager import NodeManager
+from infrahub.core.node import Node
+from infrahub.core.protocols import CoreReadOnlyRepository, CoreRepository
+from infrahub.core.schema import NodeSchema
+from infrahub.core.timestamp import Timestamp
+from infrahub.database import InfrahubDatabase
+from infrahub.message_bus.messages import RefreshRegistryBranches
+from infrahub.repositories.create_repository import RepositoryFinalizer
+from infrahub.workers.dependencies import get_message_bus
+
+
+async def convert_repository_type(
+    repository: CoreRepository | CoreReadOnlyRepository,
+    target_schema: NodeSchema,
+    mapping: dict[str, ConversionFieldInput],
+    branch: Branch,
+    db: InfrahubDatabase,
+    repository_post_creator: RepositoryFinalizer,
+) -> Node:
+    """Delete the node and return the new created one. If creation fails, the node is not deleted, and raise an error.
+    An extra check is performed on input node peers relationships to make sure they are still valid."""
+
+    repo_name = repository.name.value
+    async with lock.registry.get(name=repo_name, namespace="repository"):
+        async with db.start_transaction() as dbt:
+            timestamp_before_conversion = Timestamp()
+
+            # Fetch validators before deleting the repository otherwise validator-repository would no longer exist
+            user_validators = await NodeManager.query(
+                db=dbt, schema=USERVALIDATOR, prefetch_relationships=True, filters={"repository__id": repository.id}
+            )
+            repository_validators = await NodeManager.query(
+                db=dbt,
+                schema=REPOSITORYVALIDATOR,
+                prefetch_relationships=True,
+                filters={"repository__id": repository.id},
+            )
+            new_repository = await convert_object_type(
+                node=repository,  # type: ignore[arg-type]
+                target_schema=target_schema,
+                mapping=mapping,
+                branch=branch,
+                db=dbt,
+            )
+
+            for user_validator in user_validators:
+                await user_validator.repository.update(db=dbt, data=new_repository)
+                await user_validator.repository.save(db=dbt)
+
+            for repository_validator in repository_validators:
+                await repository_validator.repository.update(db=dbt, data=new_repository)
+                await repository_validator.repository.save(db=dbt)
+
+            await validate_conversion(
+                deleted_node=repository,  # type: ignore[arg-type]
+                branch=branch,
+                db=dbt,
+                timestamp_before_conversion=timestamp_before_conversion,
+            )
+
+        # Refresh outside the transaction otherwise other workers would pull outdated branch objects.
+        message_bus = await get_message_bus()
+        await message_bus.send(RefreshRegistryBranches())
+
+        # Following call involve a potential update of `commit` value of the newly created repository
+        # that would be done from another database connection so it can't be performed within above transaction.
+        # Also note since the conversion can only be performed on main branch here, it is fine that we do it
+        # after having updating other branches status to NEEDS_REBASE.
+        await repository_post_creator.post_create(
+            branch=branch,
+            obj=new_repository,  # type: ignore
+            db=db,
+            delete_on_connectivity_failure=False,
+        )
+
+        # Delete the RepositoryGroup associated with the old repository, as a new one was created for the new repository.
+        repository_groups = (await repository.groups_objects.get_peers(db=db)).values()
+        for repository_group in repository_groups:
+            await NodeManager.delete(db=db, branch=branch, nodes=[repository_group], cascade_delete=False)
+
+    return new_repository

infrahub/core/convert_object_type/schema_mapping.py

@@ -1,6 +1,6 @@
 from pydantic import BaseModel
 
-from infrahub.core.constants import RelationshipCardinality
+from infrahub.core.constants import BranchSupportType, RelationshipCardinality
 from infrahub.core.schema import NodeSchema
 
 
@@ -13,6 +13,21 @@ class SchemaMappingValue(BaseModel):
 SchemaMapping = dict[str, SchemaMappingValue]
 
 
+def _are_branch_support_matching(
+    left_branch_support: BranchSupportType | None,
+    right_branch_support: BranchSupportType | None,
+) -> bool:
+    if left_branch_support == right_branch_support:
+        return True
+
+    local_aware = [BranchSupportType.AWARE, BranchSupportType.LOCAL]
+
+    if left_branch_support in local_aware and right_branch_support in local_aware:
+        return True
+
+    return False
+
+
 def get_schema_mapping(source_schema: NodeSchema, target_schema: NodeSchema) -> SchemaMapping:
     """
     Return fields mapping meant to be used for converting a node from `source_kind` to `target_kind`.
@@ -31,7 +46,11 @@ def get_schema_mapping(source_schema: NodeSchema, target_schema: NodeSchema) ->
     # Process attributes
     for target_attr in target_schema.attributes:
         source_attr = source_attrs.get(target_attr.name)
-        if source_attr and source_attr.kind == target_attr.kind:
+        if (
+            source_attr
+            and source_attr.kind == target_attr.kind
+            and _are_branch_support_matching(source_attr.branch, target_attr.branch)
+        ):
             target_field_to_source_field[target_attr.name] = SchemaMappingValue(
                 source_field_name=source_attr.name, is_mandatory=not target_attr.optional
             )
@@ -41,7 +60,12 @@ def get_schema_mapping(source_schema: NodeSchema, target_schema: NodeSchema) ->
     # Process relationships
     for target_rel in target_schema.relationships:
         source_rel = source_rels.get(target_rel.name)
-        if source_rel and source_rel.peer == target_rel.peer and source_rel.cardinality == target_rel.cardinality:
+        if (
+            source_rel
+            and source_rel.peer == target_rel.peer
+            and source_rel.cardinality == target_rel.cardinality
+            and _are_branch_support_matching(source_rel.branch, target_rel.branch)
+        ):
             target_field_to_source_field[target_rel.name] = SchemaMappingValue(
                 source_field_name=source_rel.name,
                 is_mandatory=not target_rel.optional,

infrahub/core/diff/query/artifact.py

@@ -148,7 +148,7 @@ CALL (target_node, definition_node){
         )
         RETURN
             target_artifact,
-            (trel1.status = "active" AND trel2.status = "active" AND drel1.status = "active" AND drel1.status = "active") AS artifact_is_active
+            (trel1.status = "active" AND trel2.status = "active" AND drel1.status = "active" AND drel2.status = "active") AS artifact_is_active
         ORDER BY trel1.from DESC, trel2.from DESC, drel1.from DESC, drel2.from DESC,
             trel1.status ASC, trel2.status ASC, drel1.status ASC, drel2.status ASC
         LIMIT 1

infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 39
+GRAPH_VERSION = 40

infrahub/core/initialization.py

@@ -34,7 +34,7 @@ from infrahub.database import InfrahubDatabase
 from infrahub.database.memgraph import IndexManagerMemgraph
 from infrahub.database.neo4j import IndexManagerNeo4j
 from infrahub.exceptions import DatabaseError
-from infrahub.graphql.manager import GraphQLSchemaManager
+from infrahub.graphql.manager import registry as graphql_registry
 from infrahub.log import get_logger
 from infrahub.menu.utils import create_default_menu
 from infrahub.permissions import PermissionBackend, get_or_create_global_permission
@@ -196,7 +196,7 @@ async def initialization(db: InfrahubDatabase, add_database_indexes: bool = Fals
 
     default_branch = registry.get_branch_from_registry(branch=registry.default_branch)
     schema_branch = registry.schema.get_schema_branch(name=default_branch.name)
-    gqlm = GraphQLSchemaManager.get_manager_for_branch(branch=default_branch, schema_branch=schema_branch)
+    gqlm = graphql_registry.get_manager_for_branch(branch=default_branch, schema_branch=schema_branch)
     gqlm.get_graphql_schema(
         include_query=True,
         include_mutation=True,

infrahub/core/manager.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from copy import copy
-from functools import reduce
 from typing import TYPE_CHECKING, Any, Iterable, Literal, TypeVar, overload
 
 from infrahub_sdk.utils import deep_merge_dict, is_valid_uuid
@@ -11,9 +10,7 @@ from infrahub.core.node import Node
 from infrahub.core.node.delete_validator import NodeDeleteValidator
 from infrahub.core.query.node import (
     AttributeFromDB,
-    AttributeNodePropertyFromDB,
     GroupedPeerNodes,
-    NodeAttributesFromDB,
     NodeGetHierarchyQuery,
     NodeGetListQuery,
     NodeListGetAttributeQuery,
@@ -75,60 +72,6 @@ def get_schema(
     return node_schema
 
 
-class ProfileAttributeIndex:
-    def __init__(
-        self,
-        profile_attributes_id_map: dict[str, NodeAttributesFromDB],
-        profile_ids_by_node_id: dict[str, list[str]],
-    ) -> None:
-        self._profile_attributes_id_map = profile_attributes_id_map
-        self._profile_ids_by_node_id = profile_ids_by_node_id
-
-    def apply_profiles(self, node_data_dict: dict[str, Any]) -> dict[str, Any]:
-        updated_data: dict[str, Any] = {**node_data_dict}
-        node_id = node_data_dict.get("id")
-        profile_ids = self._profile_ids_by_node_id.get(node_id, [])
-        if not profile_ids:
-            return updated_data
-        profiles = [
-            self._profile_attributes_id_map[p_id] for p_id in profile_ids if p_id in self._profile_attributes_id_map
-        ]
-
-        def get_profile_priority(nafd: NodeAttributesFromDB) -> tuple[int | float, str]:
-            try:
-                return (int(nafd.attrs.get("profile_priority").value), nafd.node.get("uuid"))
-            except (TypeError, AttributeError):
-                return (float("inf"), "")
-
-        profiles.sort(key=get_profile_priority)
-
-        for attr_name, attr_data in updated_data.items():
-            if not isinstance(attr_data, AttributeFromDB):
-                continue
-            if not attr_data.is_default:
-                continue
-            profile_value, profile_uuid = None, None
-            index = 0
-
-            while profile_value is None and index <= (len(profiles) - 1):
-                try:
-                    profile_value = profiles[index].attrs[attr_name].value
-                    if profile_value != "NULL":
-                        profile_uuid = profiles[index].node["uuid"]
-                        break
-                    profile_value = None
-                except (IndexError, KeyError, AttributeError):
-                    ...
-                index += 1
-
-            if profile_value is not None:
-                attr_data.value = profile_value
-                attr_data.is_from_profile = True
-                attr_data.is_default = False
-                attr_data.node_properties["source"] = AttributeNodePropertyFromDB(uuid=profile_uuid, labels=[])
-        return updated_data
-
-
 class NodeManager:
     @overload
     @classmethod
@@ -1129,21 +1072,11 @@ class NodeManager:
         )
         await query.execute(db=db)
         nodes_info_by_id: dict[str, NodeToProcess] = {node.node_uuid: node async for node in query.get_nodes(db=db)}
-        profile_ids_by_node_id = query.get_profile_ids_by_node_id()
-        all_profile_ids = reduce(
-            lambda all_ids, these_ids: all_ids | set(these_ids), profile_ids_by_node_id.values(), set()
-        )
-
-        if fields and all_profile_ids:
-            if "profile_priority" not in fields:
-                fields["profile_priority"] = {}
-            if "value" not in fields["profile_priority"]:
-                fields["profile_priority"]["value"] = None
 
         # Query list of all Attributes
         query = await NodeListGetAttributeQuery.init(
             db=db,
-            ids=list(nodes_info_by_id.keys()) + list(all_profile_ids),
+            ids=list(nodes_info_by_id.keys()),
             fields=fields,
             branch=branch,
             include_source=include_source,
@@ -1153,17 +1086,7 @@ class NodeManager:
             branch_agnostic=branch_agnostic,
         )
         await query.execute(db=db)
-        all_node_attributes = query.get_attributes_group_by_node()
-        profile_attributes: dict[str, dict[str, AttributeFromDB]] = {}
-        node_attributes: dict[str, dict[str, AttributeFromDB]] = {}
-        for node_id, attribute_dict in all_node_attributes.items():
-            if node_id in all_profile_ids:
-                profile_attributes[node_id] = attribute_dict
-            else:
-                node_attributes[node_id] = attribute_dict
-        profile_index = ProfileAttributeIndex(
-            profile_attributes_id_map=profile_attributes, profile_ids_by_node_id=profile_ids_by_node_id
-        )
+        node_attributes = query.get_attributes_group_by_node()
 
         nodes: dict[str, Node] = {}
 
@@ -1192,11 +1115,10 @@ class NodeManager:
                 for attr_name, attr in node_attributes[node_id].attrs.items():
                     new_node_data[attr_name] = attr
 
-            new_node_data_with_profile_overrides = profile_index.apply_profiles(new_node_data)
             node_class = identify_node_class(node=node)
             node_branch = await registry.get_branch(db=db, branch=node.branch)
             item = await node_class.init(schema=node.schema, branch=node_branch, at=at, db=db)
-            await item.load(**new_node_data_with_profile_overrides, db=db)
+            await item.load(**new_node_data, db=db)
 
             nodes[node_id] = item
 

infrahub/core/migrations/graph/__init__.py

@@ -41,6 +41,7 @@ from .m036_drop_attr_value_index import Migration036
 from .m037_index_attr_vals import Migration037
 from .m038_redo_0000_prefix_fix import Migration038
 from .m039_ipam_reconcile import Migration039
+from .m040_profile_attrs_in_db import Migration040
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -87,6 +88,7 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration037,
     Migration038,
     Migration039,
+    Migration040,
 ]