infrahub-server 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

infrahub/computed_attribute/tasks.py

@@ -6,6 +6,7 @@ from infrahub_sdk.protocols import (
     CoreNode,  # noqa: TC002
     CoreTransformPython,
 )
+from infrahub_sdk.template import Jinja2Template
 from prefect import flow
 from prefect.client.orchestration import get_client
 from prefect.logging import get_run_logger
@@ -16,7 +17,6 @@ from infrahub.core.registry import registry
 from infrahub.events import BranchDeletedEvent
 from infrahub.git.repository import get_initialized_repo
 from infrahub.services import InfrahubServices  # noqa: TC001  needed for prefect flow
-from infrahub.support.macro import MacroDefinition
 from infrahub.trigger.models import TriggerType
 from infrahub.trigger.setup import setup_triggers
 from infrahub.workflows.catalogue import (
@@ -173,15 +173,15 @@ async def update_computed_attribute_value_jinja2(
 
     await add_tags(branches=[branch_name], nodes=[obj.id], db_change=True)
 
-    macro_definition = MacroDefinition(macro=template_value)
-    my_filter = {}
-    for variable in macro_definition.variables:
+    jinja_template = Jinja2Template(template=template_value)
+    variables = {}
+    for variable in jinja_template.get_variables():
         components = variable.split("__")
         if len(components) == 2:
             property_name = components[0]
             property_value = components[1]
             attribute_property = getattr(obj, property_name)
-            my_filter[variable] = getattr(attribute_property, property_value)
+            variables[variable] = getattr(attribute_property, property_value)
         elif len(components) == 3:
             relationship_name = components[0]
             property_name = components[1]
@@ -189,11 +189,11 @@ async def update_computed_attribute_value_jinja2(
             relationship = getattr(obj, relationship_name)
             try:
                 attribute_property = getattr(relationship.peer, property_name)
-                my_filter[variable] = getattr(attribute_property, property_value)
+                variables[variable] = getattr(attribute_property, property_value)
             except ValueError:
-                my_filter[variable] = ""
+                variables[variable] = ""
 
-    value = macro_definition.render(variables=my_filter)
+    value = await jinja_template.render(variables=variables)
     existing_value = getattr(obj, attribute_name).value
     if value == existing_value:
         log.debug(f"Ignoring to update {obj} with existing value on {attribute_name}={value}")
@@ -311,35 +311,36 @@ async def trigger_update_jinja2_computed_attributes(
 async def computed_attribute_setup_jinja2(
     service: InfrahubServices, context: InfrahubContext, branch_name: str | None = None, event_name: str | None = None
 ) -> None:
-    log = get_run_logger()
+    async with service.database.start_session() as db:
+        log = get_run_logger()
 
-    if branch_name:
-        await add_tags(branches=[branch_name])
-        await wait_for_schema_to_converge(branch_name=branch_name, service=service, log=log)
-
-    triggers = await gather_trigger_computed_attribute_jinja2()
-
-    for trigger in triggers:
-        if event_name != BranchDeletedEvent.event_name and trigger.branch == branch_name:
-            await service.workflow.submit_workflow(
-                workflow=TRIGGER_UPDATE_JINJA_COMPUTED_ATTRIBUTES,
-                context=context,
-                parameters={
-                    "branch_name": trigger.branch,
-                    "computed_attribute_name": trigger.computed_attribute.attribute.name,
-                    "computed_attribute_kind": trigger.computed_attribute.kind,
-                },
-            )
+        if branch_name:
+            await add_tags(branches=[branch_name])
+            await wait_for_schema_to_converge(branch_name=branch_name, component=service.component, db=db, log=log)
 
-    # Configure all ComputedAttrJinja2Trigger in Prefect
-    async with get_client(sync_client=False) as prefect_client:
-        await setup_triggers(
-            client=prefect_client,
-            triggers=triggers,
-            trigger_type=TriggerType.COMPUTED_ATTR_JINJA2,
-        )  # type: ignore[misc]
+        triggers = await gather_trigger_computed_attribute_jinja2()
+
+        for trigger in triggers:
+            if event_name != BranchDeletedEvent.event_name and trigger.branch == branch_name:
+                await service.workflow.submit_workflow(
+                    workflow=TRIGGER_UPDATE_JINJA_COMPUTED_ATTRIBUTES,
+                    context=context,
+                    parameters={
+                        "branch_name": trigger.branch,
+                        "computed_attribute_name": trigger.computed_attribute.attribute.name,
+                        "computed_attribute_kind": trigger.computed_attribute.kind,
+                    },
+                )
 
-    log.info(f"{len(triggers)} Computed Attribute for Jinja2 automation configuration completed")
+        # Configure all ComputedAttrJinja2Trigger in Prefect
+        async with get_client(sync_client=False) as prefect_client:
+            await setup_triggers(
+                client=prefect_client,
+                triggers=triggers,
+                trigger_type=TriggerType.COMPUTED_ATTR_JINJA2,
+            )  # type: ignore[misc]
+
+        log.info(f"{len(triggers)} Computed Attribute for Jinja2 automation configuration completed")
 
 
 @flow(
@@ -353,46 +354,49 @@ async def computed_attribute_setup_python(
     event_name: str | None = None,
     commit: str | None = None,  # noqa: ARG001
 ) -> None:
-    log = get_run_logger()
+    async with service.database.start_session() as db:
+        log = get_run_logger()
+
+        branch_name = branch_name or registry.default_branch
 
-    branch_name = branch_name or registry.default_branch
+        if branch_name:
+            await add_tags(branches=[branch_name])
+            await wait_for_schema_to_converge(branch_name=branch_name, component=service.component, db=db, log=log)
 
-    if branch_name:
-        await add_tags(branches=[branch_name])
-        await wait_for_schema_to_converge(branch_name=branch_name, service=service, log=log)
+        triggers_python, triggers_python_query = await gather_trigger_computed_attribute_python(db=db)
 
-    triggers_python, triggers_python_query = await gather_trigger_computed_attribute_python(db=service.database)
+        for trigger in triggers_python:
+            if event_name != BranchDeletedEvent.event_name and trigger.branch == branch_name:
+                log.info(
+                    f"Triggering update for {trigger.computed_attribute.computed_attribute.attribute.name} on {branch_name}"
+                )
+                await service.workflow.submit_workflow(
+                    workflow=TRIGGER_UPDATE_PYTHON_COMPUTED_ATTRIBUTES,
+                    context=context,
+                    parameters={
+                        "branch_name": branch_name,
+                        "computed_attribute_name": trigger.computed_attribute.computed_attribute.attribute.name,
+                        "computed_attribute_kind": trigger.computed_attribute.computed_attribute.kind,
+                    },
+                )
 
-    for trigger in triggers_python:
-        if event_name != BranchDeletedEvent.event_name and trigger.branch == branch_name:
+        async with get_client(sync_client=False) as prefect_client:
+            await setup_triggers(
+                client=prefect_client,
+                triggers=triggers_python,
+                trigger_type=TriggerType.COMPUTED_ATTR_PYTHON,
+            )  # type: ignore[misc]
+            log.info(f"{len(triggers_python)} Computed Attribute for Python automation configuration completed")
+
+            await setup_triggers(
+                client=prefect_client,
+                triggers=triggers_python_query,
+                trigger_type=TriggerType.COMPUTED_ATTR_PYTHON_QUERY,
+            )  # type: ignore[misc]
             log.info(
-                f"Triggering update for {trigger.computed_attribute.computed_attribute.attribute.name} on {branch_name}"
-            )
-            await service.workflow.submit_workflow(
-                workflow=TRIGGER_UPDATE_PYTHON_COMPUTED_ATTRIBUTES,
-                context=context,
-                parameters={
-                    "branch_name": branch_name,
-                    "computed_attribute_name": trigger.computed_attribute.computed_attribute.attribute.name,
-                    "computed_attribute_kind": trigger.computed_attribute.computed_attribute.kind,
-                },
+                f"{len(triggers_python_query)} Computed Attribute for Python Query automation configuration completed"
             )
 
-    async with get_client(sync_client=False) as prefect_client:
-        await setup_triggers(
-            client=prefect_client,
-            triggers=triggers_python,
-            trigger_type=TriggerType.COMPUTED_ATTR_PYTHON,
-        )  # type: ignore[misc]
-        log.info(f"{len(triggers_python)} Computed Attribute for Python automation configuration completed")
-
-        await setup_triggers(
-            client=prefect_client,
-            triggers=triggers_python_query,
-            trigger_type=TriggerType.COMPUTED_ATTR_PYTHON_QUERY,
-        )  # type: ignore[misc]
-        log.info(f"{len(triggers_python_query)} Computed Attribute for Python Query automation configuration completed")
-
 
 @flow(
     name="query-computed-attribute-transform-targets",

infrahub/config.py

@@ -612,6 +612,9 @@ class SecuritySettings(BaseSettings):
     oauth2_provider_settings: SecurityOAuth2ProviderSettings = Field(default_factory=SecurityOAuth2ProviderSettings)
     oidc_providers: list[OIDCProvider] = Field(default_factory=list, description="The selected OIDC providers")
     oidc_provider_settings: SecurityOIDCProviderSettings = Field(default_factory=SecurityOIDCProviderSettings)
+    restrict_untrusted_jinja2_filters: bool = Field(
+        default=True, description="Indicates if untrusted Jinja2 filters should be disallowd for computed attributes"
+    )
     _oauth2_settings: dict[str, SecurityOAuth2Settings] = PrivateAttr(default_factory=dict)
     _oidc_settings: dict[str, SecurityOIDCSettings] = PrivateAttr(default_factory=dict)
     sso_user_default_group: str | None = Field(

infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 22
+GRAPH_VERSION = 24

infrahub/core/migrations/graph/__init__.py

@@ -24,6 +24,8 @@ from .m019_restore_rels_to_time import Migration019
 from .m020_duplicate_edges import Migration020
 from .m021_missing_hierarchy_merge import Migration021
 from .m022_add_generate_template_attr import Migration022
+from .m023_deduplicate_cardinality_one_relationships import Migration023
+from .m024_missing_hierarchy_backfill import Migration024
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -53,7 +55,8 @@ MIGRATIONS: list[type[GraphMigration | InternalSchemaMigration | ArbitraryMigrat
     Migration020,
     Migration021,
     Migration022,
-    # Migration023,  Enable this migration once it has been tested on bigger databases
+    Migration023,
+    Migration024,
 ]
 
 

infrahub/core/migrations/graph/m024_missing_hierarchy_backfill.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Sequence
+
+from infrahub.core import registry
+from infrahub.core.initialization import initialization
+from infrahub.core.migrations.shared import GraphMigration, MigrationResult
+from infrahub.lock import initialize_lock
+from infrahub.log import get_logger
+
+from ...query import Query, QueryType
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+log = get_logger()
+
+
+class BackfillMissingHierarchyQuery(Query):
+    name = "backfill_missing_hierarchy"
+    type = QueryType.WRITE
+    insert_return = False
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        # load schemas from database into registry
+        initialize_lock()
+        await initialization(db=db)
+        kind_hierarchy_map: dict[str, str] = {}
+        schema_branch = await registry.schema.load_schema_from_db(db=db)
+        for node_schema_kind in schema_branch.node_names:
+            node_schema = schema_branch.get_node(name=node_schema_kind, duplicate=False)
+            if node_schema.hierarchy:
+                kind_hierarchy_map[node_schema.kind] = node_schema.hierarchy
+
+        self.params = {"hierarchy_map": kind_hierarchy_map}
+        query = """
+        MATCH (r:Root)
+        WITH r.default_branch AS default_branch
+        MATCH (rel:Relationship {name: "parent__child"})-[e:IS_RELATED]-(n:Node)
+        WHERE e.hierarchy IS NULL
+        WITH DISTINCT rel, n, default_branch
+        CALL {
+            WITH rel, n, default_branch
+            MATCH (rel)-[e:IS_RELATED {branch: default_branch}]-(n)
+            RETURN e
+            ORDER BY e.from DESC
+            LIMIT 1
+        }
+        WITH rel, n, e
+        WHERE e.status = "active" AND e.hierarchy IS NULL
+        SET e.hierarchy = $hierarchy_map[n.kind]
+        """
+        self.add_to_query(query)
+
+
+class Migration024(GraphMigration):
+    """
+    A bug in diff merge logic caused the hierarchy information on IS_RELATED edges to be lost when merged into
+    main. This migration backfills the missing hierarchy data and accounts for the case when the branch that
+    created the data has been deleted.
+    """
+
+    name: str = "024_backfill_hierarchy"
+    minimum_version: int = 23
+    queries: Sequence[type[Query]] = [BackfillMissingHierarchyQuery]
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:  # noqa: ARG002
+        result = MigrationResult()
+        return result

infrahub/core/models.py

@@ -19,6 +19,8 @@ if TYPE_CHECKING:
     from infrahub.core.schema import MainSchemaTypes
     from infrahub.core.schema.schema_branch import SchemaBranch
 
+GENERIC_ATTRIBUTES_TO_IGNORE = ["namespace", "name", "branch"]
+
 
 class NodeKind(BaseModel):
     namespace: str
@@ -270,6 +272,10 @@ class SchemaUpdateValidationResult(BaseModel):
         field_info = schema.model_fields[node_field_name]
         field_update = str(field_info.json_schema_extra.get("update"))  # type: ignore[union-attr]
 
+        # No need to execute a migration for generic nodes attributes because they are not stored in the database
+        if schema.is_generic_schema and node_field_name in GENERIC_ATTRIBUTES_TO_IGNORE:
+            return
+
         schema_path = SchemaPath(  # type: ignore[call-arg]
             schema_kind=schema.kind,
             path_type=SchemaPathType.NODE,

infrahub/core/node/__init__.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Sequence, TypeVar, overload
 
+from infrahub_sdk.template import Jinja2Template
 from infrahub_sdk.utils import is_valid_uuid
 from infrahub_sdk.uuidt import UUIDT
 
@@ -24,7 +25,6 @@ from infrahub.core.query.node import NodeCheckIDQuery, NodeCreateAllQuery, NodeD
 from infrahub.core.schema import AttributeSchema, NodeSchema, ProfileSchema, RelationshipSchema, TemplateSchema
 from infrahub.core.timestamp import Timestamp
 from infrahub.exceptions import InitializationError, NodeNotFoundError, PoolExhaustedError, ValidationError
-from infrahub.support.macro import MacroDefinition
 from infrahub.types import ATTRIBUTE_TYPES
 
 from ...graphql.constants import KIND_GRAPHQL_FIELD_NAME
@@ -458,9 +458,9 @@ class Node(BaseNode, metaclass=BaseNodeMeta):
                     ValidationError({macro: f"{macro} is missing computational_logic for macro ({attr_schema.kind})"})
                 )
                 continue
-            macro_definition = MacroDefinition(macro=attr_schema.computed_attribute.jinja2_template)
 
-            for variable in macro_definition.variables:
+            jinja_template = Jinja2Template(template=attr_schema.computed_attribute.jinja2_template)
+            for variable in jinja_template.get_variables():
                 attribute_path = schema_branch.validate_schema_path(
                     node_schema=self._schema, path=variable, allowed_path_types=allowed_path_types
                 )
@@ -487,7 +487,7 @@ class Node(BaseNode, metaclass=BaseNodeMeta):
                     )
                     variables[variable] = attribute
 
-            content = macro_definition.render(variables=variables)
+            content = await jinja_template.render(variables=variables)
 
             generator_method_name = "_generate_attribute_default"
             if hasattr(self, f"generate_{attr_schema.name}"):

infrahub/core/node/constraints/grouped_uniqueness.py

@@ -225,16 +225,31 @@ class NodeGroupedUniquenessConstraint(NodeConstraintInterface):
             )
             violations.extend(schema_violations)
 
-        is_hfid_violated = any(violation.typ == UniquenessConstraintType.HFID for violation in violations)
+        hfid_violations = [violation for violation in violations if violation.typ == UniquenessConstraintType.HFID]
+        hfid_violation = hfid_violations[0] if len(hfid_violations) > 0 else None
 
-        for violation in violations:
-            if violation.typ == UniquenessConstraintType.STANDARD or (
-                violation.typ == UniquenessConstraintType.SUBSET_OF_HFID and not is_hfid_violated
-            ):
-                error_msg = f"Violates uniqueness constraint '{'-'.join(violation.fields)}'"
-                raise ValidationError(error_msg)
+        # If there are both a hfid violation and another one, in case of an upsert, we still want to update the node in case other violations are:
+        # - either on subset fields of hfid, which would be necessarily violated too
+        # - or on uniqueness constraints with a matching node id being the id of the hfid violation
 
         for violation in violations:
             if violation.typ == UniquenessConstraintType.HFID:
-                error_msg = f"Violates uniqueness constraint '{'-'.join(violation.fields)}'"
-                raise HFIDViolatedError(error_msg, matching_nodes_ids=violation.nodes_ids)
+                continue
+
+            if hfid_violation:
+                if violation.typ == UniquenessConstraintType.SUBSET_OF_HFID:
+                    continue
+
+                if (
+                    violation.typ == UniquenessConstraintType.STANDARD
+                    and len(violation.nodes_ids) == 1
+                    and next(iter(violation.nodes_ids)) == next(iter(hfid_violation.nodes_ids))
+                ):
+                    continue
+
+            error_msg = f"Violates uniqueness constraint '{'-'.join(violation.fields)}'"
+            raise ValidationError(error_msg)
+
+        if hfid_violation:
+            error_msg = f"Violates uniqueness constraint '{'-'.join(hfid_violation.fields)}'"
+            raise HFIDViolatedError(error_msg, matching_nodes_ids=hfid_violation.nodes_ids)

infrahub/core/query/ipam.py

@@ -362,7 +362,7 @@ class IPPrefixReconcileQuery(Query):
         # possible prefix: highest possible prefix length for a match
         possible_prefix_map: dict[str, int] = {}
         start_prefixlen = prefixlen if is_address else prefixlen - 1
-        for max_prefix_len in range(start_prefixlen, 0, -1):
+        for max_prefix_len in range(start_prefixlen, -1, -1):
             tmp_prefix = prefix_bin_host[:max_prefix_len]
             possible_prefix = tmp_prefix.ljust(self.ip_value.max_prefixlen, "0")
             if possible_prefix not in possible_prefix_map:

infrahub/core/query/node.py

@@ -8,6 +8,7 @@ from enum import Enum
 from typing import TYPE_CHECKING, Any, AsyncIterator, Generator
 
 from infrahub import config
+from infrahub.core import registry
 from infrahub.core.constants import (
     AttributeDBNodeType,
     RelationshipDirection,
@@ -713,14 +714,24 @@ class NodeGetKindQuery(Query):
         super().__init__(**kwargs)
 
     async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
+        self.params["ids"] = self.ids
         query = """
-        MATCH p = (root:Root)<-[r_root:IS_PART_OF]-(n:Node)
-        WHERE n.uuid IN $ids
+MATCH (n:Node)-[r:IS_PART_OF {status: "active"}]->(:Root)
+WHERE toString(n.uuid) IN $ids
+        """
+        # only add the branch filter logic if a branch is included in the query parameters
+        if branch := getattr(self, "branch", None):
+            branch = await registry.get_branch(db=db, branch=branch)
+            branch_filter, branch_params = branch.get_query_filter_path(at=self.at)
+            self.params.update(branch_params)
+            query += f"AND {branch_filter}"
+        query += """
+WITH n.uuid AS node_id, n.kind AS node_kind
+ORDER BY r.from DESC
+WITH node_id, head(collect(node_kind)) AS node_kind
         """
         self.add_to_query(query)
-        self.params["ids"] = self.ids
-
-        self.return_labels = ["n.uuid AS node_id", "n.kind AS node_kind"]
+        self.return_labels = ["node_id", "node_kind"]
 
     async def get_node_kind_map(self) -> dict[str, str]:
         node_kind_map: dict[str, str] = {}

infrahub/core/schema/schema_branch.py

@@ -6,6 +6,8 @@ from collections import defaultdict
 from itertools import chain, combinations
 from typing import Any
 
+from infrahub_sdk.template import Jinja2Template
+from infrahub_sdk.template.exceptions import JinjaTemplateError, JinjaTemplateOperationViolationError
 from infrahub_sdk.topological_sort import DependencyCycleExistsError, topological_sort
 from infrahub_sdk.utils import compare_lists, deep_merge_dict, duplicates, intersection
 from typing_extensions import Self
@@ -51,7 +53,6 @@ from infrahub.core.schema.definitions.core import core_profile_schema_definition
 from infrahub.core.validators import CONSTRAINT_VALIDATOR_MAP
 from infrahub.exceptions import SchemaNotFoundError, ValidationError
 from infrahub.log import get_logger
-from infrahub.support.macro import MacroDefinition
 from infrahub.types import ATTRIBUTE_TYPES
 from infrahub.utils import format_label
 from infrahub.visuals import select_color
@@ -1037,14 +1038,22 @@ class SchemaBranch:
                 | SchemaElementPathType.REL_ONE_MANDATORY_ATTR_WITH_PROP
                 | SchemaElementPathType.REL_ONE_ATTR_WITH_PROP
             )
+
+            jinja_template = Jinja2Template(template=attribute.computed_attribute.jinja2_template)
             try:
-                macro = MacroDefinition(macro=attribute.computed_attribute.jinja2_template)
-            except ValueError as exc:
+                variables = jinja_template.get_variables()
+                jinja_template.validate(restricted=config.SETTINGS.security.restrict_untrusted_jinja2_filters)
+            except JinjaTemplateOperationViolationError as exc:
+                raise ValueError(
+                    f"{node.kind}: Attribute {attribute.name!r} is assigned by a jinja2 template, but has an invalid template: {exc.message}"
+                ) from exc
+
+            except JinjaTemplateError as exc:
                 raise ValueError(
-                    f"{node.kind}: Attribute {attribute.name!r} is assigned by a jinja2 template, but has an invalid template"
+                    f"{node.kind}: Attribute {attribute.name!r} is assigned by a jinja2 template, but has an invalid template: : {exc.message}"
                 ) from exc
 
-            for variable in macro.variables:
+            for variable in variables:
                 try:
                     schema_path = self.validate_schema_path(
                         node_schema=node, path=variable, allowed_path_types=allowed_path_types

infrahub/exceptions.py

@@ -76,6 +76,32 @@ class RepositoryError(Error):
         super().__init__(self.message)
 
 
+class RepositoryConnectionError(RepositoryError):
+    def __init__(self, identifier: str, message: str | None = None) -> None:
+        super().__init__(
+            identifier=identifier,
+            message=message
+            or f"Unable to clone the repository {identifier}, please check the address and the credential",
+        )
+
+
+class RepositoryCredentialsError(RepositoryError):
+    def __init__(self, identifier: str, message: str | None = None) -> None:
+        super().__init__(
+            identifier=identifier,
+            message=message or f"Authentication failed for {identifier}, please validate the credentials.",
+        )
+
+
+class RepositoryInvalidBranchError(RepositoryError):
+    def __init__(self, identifier: str, branch_name: str, location: str, message: str | None = None) -> None:
+        super().__init__(
+            identifier=identifier,
+            message=message
+            or f"The branch {branch_name} isn't a valid branch for the repository {identifier} at {location}.",
+        )
+
+
 class RepositoryInvalidFileSystemError(RepositoryError):
     def __init__(
         self,
@@ -83,9 +109,11 @@ class RepositoryInvalidFileSystemError(RepositoryError):
         directory: Path,
         message: str | None = None,
     ) -> None:
-        super().__init__(identifier=identifier)
+        super().__init__(
+            identifier=identifier,
+            message=message or f"Invalid file system for {identifier}, Local directory {directory} missing.",
+        )
         self.directory = directory
-        self.message = message or f"Invalid file system for {identifier}, Local directory {directory} missing."
 
 
 class CommitNotFoundError(Error):