infrahub-server 1.3.5__tar.gz → 1.3.6__tar.gz

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: infrahub-server
-Version: 1.3.5
+Version: 1.3.6
 Summary: Infrahub is taking a new approach to Infrastructure Management by providing a new generation of datastore to organize and control all the data that defines how an infrastructure should run.
 License: Apache-2.0
 Author: OpsMill

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/backend/infrahub/cli/db.py

@@ -56,6 +56,7 @@ from infrahub.services.adapters.message_bus.local import BusSimulator
 from infrahub.services.adapters.workflow.local import WorkflowLocalExecution
 
 from .constants import ERROR_BADGE, FAILED_BADGE, SUCCESS_BADGE
+from .db_commands.check_inheritance import check_inheritance
 from .patch import patch_app
 
 
@@ -178,6 +179,30 @@ async def migrate_cmd(
     await dbdriver.close()
 
 
+@app.command(name="check-inheritance")
+async def check_inheritance_cmd(
+    ctx: typer.Context,
+    fix: bool = typer.Option(False, help="Fix the inheritance of any invalid nodes."),
+    config_file: str = typer.Argument("infrahub.toml", envvar="INFRAHUB_CONFIG"),
+) -> None:
+    """Check the database for any vertices with incorrect inheritance"""
+    logging.getLogger("infrahub").setLevel(logging.WARNING)
+    logging.getLogger("neo4j").setLevel(logging.ERROR)
+    logging.getLogger("prefect").setLevel(logging.ERROR)
+
+    config.load_and_exit(config_file_name=config_file)
+
+    context: CliContext = ctx.obj
+    dbdriver = await context.init_db(retry=1)
+    await initialize_registry(db=dbdriver)
+
+    success = await check_inheritance(db=dbdriver, fix=fix)
+    if not success:
+        raise typer.Exit(code=1)
+
+    await dbdriver.close()
+
+
 @app.command(name="update-core-schema")
 async def update_core_schema_cmd(
     ctx: typer.Context,

infrahub_server-1.3.6/backend/infrahub/cli/db_commands/check_inheritance.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from rich import print as rprint
+from rich.console import Console
+from rich.table import Table
+
+from infrahub.core import registry
+from infrahub.core.branch.models import Branch
+from infrahub.core.constants import InfrahubKind
+from infrahub.core.migrations.query.node_duplicate import NodeDuplicateQuery, SchemaNodeInfo
+from infrahub.core.query import Query, QueryType
+from infrahub.core.schema import SchemaRoot, internal_schema
+from infrahub.core.schema.manager import SchemaManager
+from infrahub.log import get_logger
+
+from ..constants import FAILED_BADGE, SUCCESS_BADGE
+
+if TYPE_CHECKING:
+    from infrahub.core.schema.node_schema import NodeSchema
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class GetSchemaWithUpdatedInheritance(Query):
+    """
+    Get the name, namespace, and branch of any SchemaNodes with _updated_ inheritance
+    This query will only return schemas that have had `inherit_from` updated after they were created
+    """
+
+    name = "get_schema_with_updated_inheritance"
+    type = QueryType.READ
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        query = """
+// find inherit_from attributes that have been updated
+MATCH p = (schema_node:SchemaNode)-[has_attr_e:HAS_ATTRIBUTE {status: "active"}]->(a:Attribute {name: "inherit_from"})
+WHERE has_attr_e.to IS NULL
+CALL (a) {
+  // only get branches on which the value was updated, we can ignore the initial create
+  MATCH (a)-[e:HAS_VALUE]->(:AttributeValue)
+  ORDER BY e.from ASC
+  // tail leaves out the earliest one, which is the initial create
+  RETURN tail(collect(e.branch)) AS branches
+}
+WITH schema_node, a, branches
+WHERE size(branches) > 0
+UNWIND branches AS branch
+WITH DISTINCT schema_node, a, branch
+
+//get branch details
+CALL (branch) {
+  MATCH (b:Branch {name: branch})
+  RETURN b.branched_from AS branched_from, b.hierarchy_level AS branch_level
+}
+
+// get the namespace for the schema
+CALL (schema_node, a, branch, branched_from, branch_level) {
+  MATCH (schema_node)-[e1:HAS_ATTRIBUTE]-(:Attribute {name: "namespace"})-[e2:HAS_VALUE]->(av)
+  WHERE (
+    e1.branch = branch OR
+    (e1.branch_level < branch_level AND e1.from <= branched_from)
+  ) AND e1.to IS NULL
+  AND e1.status = "active"
+  AND (
+    e2.branch = branch OR
+    (e2.branch_level < branch_level AND e2.from <= branched_from)
+  ) AND e2.to IS NULL
+  AND e2.status = "active"
+  ORDER BY e2.branch_level DESC, e1.branch_level DESC, e2.from DESC, e1.from DESC
+  RETURN av.value AS namespace
+  LIMIT 1
+}
+
+// get the name for the schema
+CALL (schema_node, a, branch, branched_from, branch_level) {
+  MATCH (schema_node)-[e1:HAS_ATTRIBUTE]-(:Attribute {name: "name"})-[e2:HAS_VALUE]->(av)
+  WHERE (
+    e1.branch = branch OR
+    (e1.branch_level < branch_level AND e1.from <= branched_from)
+  ) AND e1.to IS NULL
+  AND e1.status = "active"
+  AND (
+    e2.branch = branch OR
+    (e2.branch_level < branch_level AND e2.from <= branched_from)
+  ) AND e2.to IS NULL
+  AND e2.status = "active"
+  ORDER BY e2.branch_level DESC, e1.branch_level DESC, e2.from DESC, e1.from DESC
+  RETURN av.value AS name
+  LIMIT 1
+}
+RETURN name, namespace, branch
+"""
+        self.return_labels = ["name", "namespace", "branch"]
+        self.add_to_query(query)
+
+    def get_updated_inheritance_kinds_by_branch(self) -> dict[str, list[str]]:
+        kinds_by_branch: dict[str, list[str]] = defaultdict(list)
+        for result in self.results:
+            name = result.get_as_type(label="name", return_type=str)
+            namespace = result.get_as_type(label="namespace", return_type=str)
+            branch = result.get_as_type(label="branch", return_type=str)
+            kinds_by_branch[branch].append(f"{namespace}{name}")
+        return kinds_by_branch
+
+
+@dataclass
+class KindLabelCount:
+    kind: str
+    labels: frozenset[str]
+    num_nodes: int
+
+
+@dataclass
+class KindLabelCountCorrected(KindLabelCount):
+    node_schema: NodeSchema
+
+
+class GetAllKindsAndLabels(Query):
+    """
+    Get the kind, labels, and number of nodes for the given kinds and branch
+    """
+
+    name = "get_all_kinds_and_labels"
+    type = QueryType.READ
+    insert_return = False
+
+    def __init__(self, kinds: list[str] | None = None, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self.kinds = kinds
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        self.params["branch_name"] = self.branch.name
+        self.params["branched_from"] = self.branch.get_branched_from()
+        self.params["branch_level"] = self.branch.hierarchy_level
+        kinds_str = "Node"
+        if self.kinds:
+            kinds_str = "|".join(self.kinds)
+        query = """
+MATCH (n:%(kinds_str)s)-[r:IS_PART_OF]->(:Root)
+WHERE (
+    r.branch = $branch_name OR
+    (r.branch_level < $branch_level AND r.from <= $branched_from)
+)
+AND r.to IS NULL
+AND r.status = "active"
+RETURN DISTINCT n.kind AS kind, labels(n) AS labels, count(*) AS num_nodes
+ORDER BY kind ASC
+        """ % {"kinds_str": kinds_str}
+        self.return_labels = ["kind", "labels", "num_nodes"]
+        self.add_to_query(query)
+
+    def get_kind_label_counts(self) -> list[KindLabelCount]:
+        kind_label_counts: list[KindLabelCount] = []
+        for result in self.results:
+            kind = result.get_as_type(label="kind", return_type=str)
+            num_nodes = result.get_as_type(label="num_nodes", return_type=int)
+            labels: list[str] = result.get_as_type(label="labels", return_type=list)
+            # we can ignore the Node label and the label that matches the kind
+            cleaned_labels = frozenset(str(lbl) for lbl in labels if lbl not in ["Node", "CoreNode", kind])
+            kind_label_counts.append(KindLabelCount(kind=kind, labels=cleaned_labels, num_nodes=num_nodes))
+        return kind_label_counts
+
+
+def display_kind_label_counts(kind_label_counts_by_branch: dict[str, list[KindLabelCountCorrected]]) -> None:
+    console = Console()
+
+    table = Table(title="Incorrect Inheritance Nodes")
+
+    table.add_column("Branch")
+    table.add_column("Kind")
+    table.add_column("Incorrect Labels")
+    table.add_column("Num Nodes")
+
+    for branch_name, kind_label_counts in kind_label_counts_by_branch.items():
+        for kind_label_count in kind_label_counts:
+            table.add_row(
+                branch_name, kind_label_count.kind, str(list(kind_label_count.labels)), str(kind_label_count.num_nodes)
+            )
+
+    console.print(table)
+
+
+async def check_inheritance(db: InfrahubDatabase, fix: bool = False) -> bool:
+    """
+    Run migrations to update the inheritance of any nodes with incorrect inheritance from a failed migration
+    1. Identifies node schemas that have had their inheritance updated after they were created
+        a. includes the kind and branch of the inheritance update
+    2. Checks nodes of the given kinds on the given branch to verify their inheritance is correct
+    3. Displays counts of any kinds with incorrect inheritance on the given branch
+    4. If fix is True, runs migrations to update the inheritance of any nodes with incorrect inheritance
+        on the correct branch
+    """
+
+    updated_inheritance_query = await GetSchemaWithUpdatedInheritance.init(db=db)
+    await updated_inheritance_query.execute(db=db)
+    updated_inheritance_kinds_by_branch = updated_inheritance_query.get_updated_inheritance_kinds_by_branch()
+
+    if not updated_inheritance_kinds_by_branch:
+        rprint(f"{SUCCESS_BADGE} No schemas have had their inheritance updated")
+        return True
+
+    schema_manager = SchemaManager()
+    registry.schema = schema_manager
+    schema = SchemaRoot(**internal_schema)
+    schema_manager.register_schema(schema=schema)
+    branches_by_name = {b.name: b for b in await Branch.get_list(db=db)}
+
+    kind_label_counts_by_branch: dict[str, list[KindLabelCountCorrected]] = defaultdict(list)
+    for branch_name, kinds in updated_inheritance_kinds_by_branch.items():
+        rprint(f"Checking branch: {branch_name}", end="...")
+        branch = branches_by_name[branch_name]
+        schema_branch = await schema_manager.load_schema_from_db(db=db, branch=branch)
+        kind_label_query = await GetAllKindsAndLabels.init(db=db, branch=branch, kinds=kinds)
+        await kind_label_query.execute(db=db)
+        kind_label_counts = kind_label_query.get_kind_label_counts()
+
+        for kind_label_count in kind_label_counts:
+            node_schema = schema_branch.get_node(name=kind_label_count.kind, duplicate=False)
+            correct_labels = frozenset(node_schema.inherit_from)
+            if kind_label_count.labels == correct_labels:
+                continue
+
+            kind_label_counts_by_branch[branch_name].append(
+                KindLabelCountCorrected(
+                    kind=kind_label_count.kind,
+                    labels=kind_label_count.labels,
+                    num_nodes=kind_label_count.num_nodes,
+                    node_schema=node_schema,
+                )
+            )
+        rprint("done")
+
+    if not kind_label_counts_by_branch:
+        rprint(f"{SUCCESS_BADGE} All nodes have the correct inheritance")
+        return True
+
+    display_kind_label_counts(kind_label_counts_by_branch)
+
+    if not fix:
+        rprint(f"{FAILED_BADGE} Use the --fix flag to fix the inheritance of any invalid nodes")
+        return False
+
+    for branch_name, kind_label_counts_corrected in kind_label_counts_by_branch.items():
+        for kind_label_count in kind_label_counts_corrected:
+            rprint(f"Fixing kind {kind_label_count.kind} on branch {branch_name}", end="...")
+            node_schema = kind_label_count.node_schema
+            migration_query = await NodeDuplicateQuery.init(
+                db=db,
+                branch=branches_by_name[branch_name],
+                previous_node=SchemaNodeInfo(
+                    name=node_schema.name,
+                    namespace=node_schema.namespace,
+                    branch_support=node_schema.branch.value,
+                    labels=list(kind_label_count.labels) + [kind_label_count.kind, InfrahubKind.NODE],
+                    kind=kind_label_count.kind,
+                ),
+                new_node=SchemaNodeInfo(
+                    name=node_schema.name,
+                    namespace=node_schema.namespace,
+                    branch_support=node_schema.branch.value,
+                    labels=list(node_schema.inherit_from) + [kind_label_count.kind, InfrahubKind.NODE],
+                    kind=kind_label_count.kind,
+                ),
+            )
+            await migration_query.execute(db=db)
+            rprint("done")
+
+    rprint(f"{SUCCESS_BADGE} All nodes have the correct inheritance")
+
+    if registry.default_branch in kind_label_counts_by_branch:
+        kinds = [kind_label_count.kind for kind_label_count in kind_label_counts_by_branch[registry.default_branch]]
+        rprint(
+            "[bold cyan]Note that migrations were run on the default branch for the following schema kinds: "
+            f"{', '.join(kinds)}. You should rebase any branches that include/will include changes using "
+            "the migrated schemas[/bold cyan]"
+        )
+
+    return True

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/backend/infrahub/core/manager.py

@@ -400,7 +400,7 @@ class NodeManager:
 
         results = []
         for peer in peers_info:
-            result = await Relationship(schema=schema, branch=branch, at=at, node_id=peer.source_id).load(
+            result = Relationship(schema=schema, branch=branch, at=at, node_id=peer.source_id).load(
                 db=db,
                 id=peer.rel_node_id,
                 db_id=peer.rel_node_db_id,
@@ -408,7 +408,7 @@ class NodeManager:
                 data=peer,
             )
             if fetch_peers:
-                await result.set_peer(value=peer_nodes[peer.peer_id])
+                result.set_peer(value=peer_nodes[peer.peer_id])
             results.append(result)
 
         return results

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/backend/infrahub/core/migrations/query/node_duplicate.py

@@ -21,6 +21,13 @@ class SchemaNodeInfo(BaseModel):
 
 
 class NodeDuplicateQuery(Query):
+    """
+    Duplicates a Node to use a new kind or inheritance.
+    Creates a copy of each affected Node and sets the new kind/inheritance.
+    Adds duplicate edges to the new Node that match all the active edges on the old Node.
+    Sets all the edges on the old Node to deleted.
+    """
+
     name = "node_duplicate"
     type = QueryType.WRITE
     insert_return: bool = False
@@ -38,11 +45,26 @@ class NodeDuplicateQuery(Query):
 
     def render_match(self) -> str:
         labels_str = ":".join(self.previous_node.labels)
-        query = f"""
+        query = """
         // Find all the active nodes
-        MATCH (node:{labels_str})
+        MATCH (node:%(labels_str)s)
         WITH DISTINCT node
-        """
+        // ----------------
+        // Filter out nodes that have already been migrated
+        // ----------------
+        CALL (node) {
+            WITH labels(node) AS node_labels
+            UNWIND node_labels AS n_label
+            ORDER BY n_label ASC
+            WITH collect(n_label) AS sorted_labels
+
+            RETURN (
+                node.kind = $new_node.kind AND
+                sorted_labels = $new_sorted_labels
+            ) AS already_migrated
+        }
+        WITH node WHERE already_migrated = FALSE
+        """ % {"labels_str": labels_str}
 
         return query
 
@@ -111,6 +133,7 @@ class NodeDuplicateQuery(Query):
 
         self.params["new_node"] = self.new_node.model_dump()
         self.params["previous_node"] = self.previous_node.model_dump()
+        self.params["new_sorted_labels"] = sorted(self.new_node.labels + ["Node"])
 
         self.params["current_time"] = self.at.to_string()
         self.params["branch"] = self.branch.name

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/backend/infrahub/core/query/node.py

@@ -157,7 +157,20 @@ class NodeCreateAllQuery(NodeQuery):
         relationships: list[RelationshipCreateData] = []
         for rel_name in self.node._relationships:
             rel_manager: RelationshipManager = getattr(self.node, rel_name)
+            # Fetch all relationship peers through a single database call for performances.
+            peers = await rel_manager.get_peers(db=db, branch_agnostic=self.branch_agnostic)
+
             for rel in rel_manager._relationships:
+                try:
+                    rel.set_peer(value=peers[rel.get_peer_id()])
+                except KeyError:
+                    pass
+                except ValueError:
+                    # Relationship has not been initialized yet, it means the peer does not exist in db yet
+                    # typically because it will be allocated from a ressource pool. In that case, the peer
+                    # will be fetched using `rel.resolve` later.
+                    pass
+
                 rel_create_data = await rel.get_create_data(db=db, at=at)
                 if rel_create_data.peer_branch_level > deepest_branch_level or (
                     deepest_branch_name == GLOBAL_BRANCH_NAME and rel_create_data.peer_branch == registry.default_branch

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/backend/infrahub/core/relationship/model.py

@@ -166,11 +166,11 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
             return registry.get_global_branch()
         return self.branch
 
-    async def _process_data(self, data: dict | RelationshipPeerData | str) -> None:
+    def _process_data(self, data: dict | RelationshipPeerData | str) -> None:
         self.data = data
 
         if isinstance(data, RelationshipPeerData):
-            await self.set_peer(value=str(data.peer_id))
+            self.set_peer(value=str(data.peer_id))
 
             if not self.id and data.rel_node_id:
                 self.id = data.rel_node_id
@@ -187,7 +187,7 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
         elif isinstance(data, dict):
             for key, value in data.items():
                 if key in ["peer", "id"]:
-                    await self.set_peer(value=data.get(key, None))
+                    self.set_peer(value=data.get(key, None))
                 elif key == "hfid" and self.peer_id is None:
                     self.peer_hfid = value
                 elif key.startswith(PREFIX_PROPERTY) and key.replace(PREFIX_PROPERTY, "") in self._flag_properties:
@@ -198,7 +198,7 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
                     self.from_pool = value
 
         else:
-            await self.set_peer(value=data)
+            self.set_peer(value=data)
 
     async def new(
         self,
@@ -206,11 +206,11 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
         data: dict | RelationshipPeerData | Any = None,
         **kwargs: Any,  # noqa: ARG002
     ) -> Relationship:
-        await self._process_data(data=data)
+        self._process_data(data=data)
 
         return self
 
-    async def load(
+    def load(
         self,
         db: InfrahubDatabase,  # noqa: ARG002
         id: UUID | None = None,
@@ -223,7 +223,7 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
         self.id = id or self.id
         self.db_id = db_id or self.db_id
 
-        await self._process_data(data=data)
+        self._process_data(data=data)
 
         if updated_at and hash(self) != hash_before:
             self.updated_at = Timestamp(updated_at)
@@ -252,7 +252,7 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
         self._node_id = self._node.id
         return node
 
-    async def set_peer(self, value: str | Node) -> None:
+    def set_peer(self, value: str | Node) -> None:
         if isinstance(value, str):
             self.peer_id = value
         else:
@@ -433,7 +433,7 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
                 db=db, id=self.peer_id, branch=self.branch, kind=self.schema.peer, fields={"display_label": None}
             )
             if peer:
-                await self.set_peer(value=peer)
+                self.set_peer(value=peer)
 
         if not self.peer_id and self.peer_hfid:
             peer_schema = db.schema.get(name=self.schema.peer, branch=self.branch)
@@ -450,7 +450,7 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
                 fields={"display_label": None},
                 raise_on_error=True,
             )
-            await self.set_peer(value=peer)
+            self.set_peer(value=peer)
 
         if not self.peer_id and self.from_pool and "id" in self.from_pool:
             pool_id = str(self.from_pool.get("id"))
@@ -473,7 +473,7 @@ class Relationship(FlagPropertyMixin, NodePropertyMixin):
                     data_from_pool["identifier"] = f"hfid={hfid_str} rel={self.name}"
 
             assigned_peer: Node = await pool.get_resource(db=db, branch=self.branch, at=at, **data_from_pool)  # type: ignore[attr-defined]
-            await self.set_peer(value=assigned_peer)
+            self.set_peer(value=assigned_peer)
             self.set_source(value=pool.id)
 
     async def save(self, db: InfrahubDatabase, at: Timestamp | None = None) -> Self:
@@ -962,7 +962,7 @@ class RelationshipManager:
 
         for peer_id in details.peer_ids_present_database_only:
             self._relationships.append(
-                await Relationship(
+                Relationship(
                     schema=self.schema,
                     branch=self.branch,
                     at=at or self.at,
@@ -1050,7 +1050,7 @@ class RelationshipManager:
             if isinstance(item, dict) and item.get("id", None) in previous_relationships:
                 rel = previous_relationships[item["id"]]
                 hash_before = hash(rel)
-                await rel.load(data=item, db=db)
+                rel.load(data=item, db=db)
                 if hash(rel) != hash_before:
                     changed = True
                 self._relationships.append(rel)

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/backend/infrahub/graphql/mutations/relationship.py

@@ -233,7 +233,7 @@ class RelationshipRemove(Mutation):
                     # we should use RelationshipDataDeleteQuery to delete the relationship
                     # it would be more query efficient
                     rel = Relationship(schema=rel_schema, branch=graphql_context.branch, node=source)
-                    await rel.load(db=db, data=existing_peers[node_data.get("id")])
+                    rel.load(db=db, data=existing_peers[node_data.get("id")])
                     if group_event_type != GroupUpdateType.NONE:
                         peers.append(EventNode(id=rel.get_peer_id(), kind=nodes[rel.get_peer_id()].get_kind()))
                     node_changelog.delete_relationship(relationship=rel)

{infrahub_server-1.3.5 → infrahub_server-1.3.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "infrahub-server"
-version = "1.3.5"
+version = "1.3.6"
 description = "Infrahub is taking a new approach to Infrastructure Management by providing a new generation of datastore to organize and control all the data that defines how an infrastructure should run."
 authors = ["OpsMill <info@opsmill.com>"]
 readme = "README.md"