infrahub-server 1.4.9__py3-none-any.whl → 1.5.0b0__py3-none-any.whl

infrahub/lock.py

@@ -25,6 +25,7 @@ registry: InfrahubLockRegistry = None
 
 
 METRIC_PREFIX = "infrahub_lock"
+LOCK_PREFIX = "lock"
 
 LOCK_ACQUIRE_TIME_METRICS = Histogram(
     f"{METRIC_PREFIX}_acquire_seconds",
@@ -139,9 +140,9 @@ class InfrahubLock:
         if self.use_local:
             self.local = LocalLock()
         elif config.SETTINGS.cache.driver == config.CacheDriver.Redis:
-            self.remote = GlobalLock(redis=self.connection, name=f"lock.{self.name}")
+            self.remote = GlobalLock(redis=self.connection, name=f"{LOCK_PREFIX}.{self.name}")
         else:
-            self.remote = NATSLock(service=self.connection, name=f"lock.{self.name}")
+            self.remote = NATSLock(service=self.connection, name=f"{LOCK_PREFIX}.{self.name}")
 
     async def __aenter__(self):
         await self.acquire()
@@ -179,9 +180,54 @@ class InfrahubLock:
         return self.local.locked()
 
 
+class LockNameGenerator:
+    local = "local"
+    _global = "global"
+
+    def generate_name(self, name: str, namespace: str | None = None, local: bool | None = None) -> str:
+        if namespace is None and local is None:
+            return name
+
+        new_name = ""
+        if local is True:
+            new_name = f"{self.local}."
+        elif local is False:
+            new_name = f"{self._global}."
+
+        if namespace is not None:
+            new_name += f"{namespace}."
+        new_name += name
+
+        return new_name
+
+    def unpack_name(self, name: str) -> tuple[str, str | None, bool | None]:
+        local = None
+        namespace = None
+
+        parts = name.split(".")
+        if parts[0] == self.local:
+            local = True
+            parts = parts[1:]
+        elif parts[0] == self._global:
+            local = False
+            parts = parts[1:]
+
+        if len(parts) > 1:
+            namespace = parts[0]
+            original_name = ".".join(parts[1:])
+        else:
+            original_name = parts[0]
+
+        return original_name, namespace, local
+
+
 class InfrahubLockRegistry:
     def __init__(
-        self, token: str | None = None, local_only: bool = False, service: InfrahubServices | None = None
+        self,
+        token: str | None = None,
+        local_only: bool = False,
+        service: InfrahubServices | None = None,
+        name_generator: LockNameGenerator | None = None,
     ) -> None:
         if config.SETTINGS.cache.enable and not local_only:
             if config.SETTINGS.cache.driver == config.CacheDriver.Redis:
@@ -201,23 +247,7 @@ class InfrahubLockRegistry:
 
         self.token = token or str(uuid.uuid4())
         self.locks: dict[str, InfrahubLock] = {}
-
-    @classmethod
-    def _generate_name(cls, name: str, namespace: str | None = None, local: bool | None = None) -> str:
-        if namespace is None and local is None:
-            return name
-
-        new_name = ""
-        if local is True:
-            new_name = "local."
-        elif local is False:
-            new_name = "global."
-
-        if namespace is not None:
-            new_name += f"{namespace}."
-        new_name += name
-
-        return new_name
+        self.name_generator = name_generator or LockNameGenerator()
 
     def get_existing(
         self,
@@ -225,7 +255,7 @@ class InfrahubLockRegistry:
         namespace: str | None,
         local: bool | None = None,
     ) -> InfrahubLock | None:
-        lock_name = self._generate_name(name=name, namespace=namespace, local=local)
+        lock_name = self.name_generator.generate_name(name=name, namespace=namespace, local=local)
         if lock_name not in self.locks:
             return None
         return self.locks[lock_name]
@@ -233,7 +263,7 @@ class InfrahubLockRegistry:
     def get(
         self, name: str, namespace: str | None = None, local: bool | None = None, in_multi: bool = False
     ) -> InfrahubLock:
-        lock_name = self._generate_name(name=name, namespace=namespace, local=local)
+        lock_name = self.name_generator.generate_name(name=name, namespace=namespace, local=local)
         if lock_name not in self.locks:
             self.locks[lock_name] = InfrahubLock(name=lock_name, connection=self.connection, in_multi=in_multi)
         return self.locks[lock_name]
@@ -257,7 +287,3 @@ class InfrahubLockRegistry:
 def initialize_lock(local_only: bool = False, service: InfrahubServices | None = None) -> None:
     global registry
     registry = InfrahubLockRegistry(local_only=local_only, service=service)
-
-
-def build_object_lock_name(name: str) -> str:
-    return f"global.object.{name}"

infrahub/locks/tasks.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from prefect import flow
+from prefect.logging import get_run_logger
+
+from infrahub import config
+from infrahub.core.registry import registry
+from infrahub.core.timestamp import Timestamp
+from infrahub.lock import LOCK_PREFIX
+from infrahub.services import InfrahubServices  # noqa: TC001  needed for prefect flow
+
+
+@flow(
+    name="clean-up-deadlocks",
+    flow_run_name="Clean up deadlocks",
+)
+async def clean_up_deadlocks(service: InfrahubServices) -> None:
+    """Remove stale distributed locks left behind by inactive workers"""
+    keys = await service.cache.list_keys(filter_pattern=f"{LOCK_PREFIX}*")
+    if not keys:
+        return
+
+    log = get_run_logger()
+    values = await service.cache.get_values(keys=keys)
+    workers = await service.component.list_workers(branch=registry.default_branch, schema_hash=False)
+    workers_active = {worker.id for worker in workers if worker.active}
+
+    for key, value in zip(keys, values, strict=False):
+        if not key or not value:
+            continue
+
+        timestamp, worker_id = value.split("::", 1)
+        if worker_id not in workers_active and Timestamp() > Timestamp(timestamp).add(
+            minutes=config.SETTINGS.cache.clean_up_deadlocks_interval_mins
+        ):
+            await service.cache.delete(key)
+            log.info(f"Deleted deadlock key={key} worker={worker_id}")

infrahub/patch/plan_writer.py

@@ -1,6 +1,6 @@
 import json
 from dataclasses import asdict
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -10,7 +10,7 @@ from .models import EdgeToAdd, EdgeToDelete, EdgeToUpdate, PatchPlan, VertexToAd
 
 class PatchPlanWriter:
     def write(self, patches_directory: Path, patch_plan: PatchPlan) -> Path:
-        timestamp_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d-%H%M%S")
+        timestamp_str = datetime.now(tz=UTC).strftime("%Y%m%d-%H%M%S")
         patch_name = f"patch-{patch_plan.name}-{timestamp_str}"
         patch_plan_directory = patches_directory / Path(patch_name)
         if not patch_plan_directory.exists():

infrahub/profiles/node_applier.py

@@ -0,0 +1,101 @@
+from typing import Any
+
+from infrahub.core.attribute import BaseAttribute
+from infrahub.core.branch import Branch
+from infrahub.core.node import Node
+from infrahub.database import InfrahubDatabase
+
+from .queries.get_profile_data import GetProfileDataQuery, ProfileData
+
+
+class NodeProfilesApplier:
+    def __init__(self, db: InfrahubDatabase, branch: Branch):
+        self.db = db
+        self.branch = branch
+
+    async def _get_profile_ids(self, node: Node) -> list[str]:
+        try:
+            profiles_rel = node.get_relationship("profiles")
+        except ValueError:
+            return []
+        profile_rels = await profiles_rel.get_relationships(db=self.db)
+        return [pr.peer_id for pr in profile_rels if pr.peer_id]
+
+    async def _get_attr_names_for_profiles(self, node: Node) -> list[str]:
+        node_schema = node.get_schema()
+
+        # get the names of attributes that could be affected by profile changes
+        attr_names_for_profiles: list[str] = []
+        for attr_schema in node_schema.attributes:
+            attr_name = attr_schema.name
+            node_attr: BaseAttribute = getattr(node, attr_name)
+            if node_attr.is_from_profile or node_attr.is_default:
+                attr_names_for_profiles.append(attr_name)
+        return attr_names_for_profiles
+
+    async def _get_sorted_profile_data(
+        self, profile_ids: list[str], attr_names_for_profiles: list[str]
+    ) -> list[ProfileData]:
+        if not profile_ids:
+            return []
+        query = await GetProfileDataQuery.init(
+            db=self.db, branch=self.branch, profile_ids=profile_ids, attr_names=attr_names_for_profiles
+        )
+        await query.execute(db=self.db)
+        profile_data_list = query.get_profile_data()
+        return sorted(profile_data_list, key=lambda x: (x.priority, x.uuid))
+
+    def _apply_profile_to_attribute(self, node_attr: BaseAttribute, profile_value: Any, profile_id: str) -> bool:
+        is_changed = False
+        if node_attr.value != profile_value:
+            node_attr.value = profile_value
+            is_changed = True
+        if node_attr.is_default is not False:
+            node_attr.is_default = False
+            is_changed = True
+        if node_attr.is_from_profile is not True:
+            node_attr.is_from_profile = True
+            is_changed = True
+        if node_attr.source_id != profile_id:  # type: ignore[attr-defined]
+            node_attr.set_source(value=profile_id)
+            is_changed = True
+        return is_changed
+
+    def _remove_profile_from_attribute(self, node_attr: BaseAttribute) -> None:
+        node_attr.clear_source()
+        node_attr.value = node_attr.schema.default_value
+        node_attr.is_default = True
+        node_attr.is_from_profile = False
+
+    async def apply_profiles(self, node: Node) -> list[str]:
+        profile_ids = await self._get_profile_ids(node=node)
+        attr_names_for_profiles = await self._get_attr_names_for_profiles(node=node)
+
+        if not attr_names_for_profiles:
+            return []
+
+        # get profiles priorities and attribute values on branch
+        sorted_profile_data = await self._get_sorted_profile_data(
+            profile_ids=profile_ids, attr_names_for_profiles=attr_names_for_profiles
+        )
+
+        updated_field_names = []
+        # set attribute values/is_default/is_from_profile on nodes
+        for attr_name in attr_names_for_profiles:
+            has_profile_data = False
+            node_attr = node.get_attribute(attr_name)
+            for profile_data in sorted_profile_data:
+                profile_value = profile_data.attribute_values.get(attr_name)
+                if profile_value is not None:
+                    has_profile_data = True
+                    is_changed = False
+                    is_changed = self._apply_profile_to_attribute(
+                        node_attr=node_attr, profile_value=profile_value, profile_id=profile_data.uuid
+                    )
+                    if is_changed:
+                        updated_field_names.append(attr_name)
+                    break
+            if not has_profile_data and node_attr.is_from_profile:
+                self._remove_profile_from_attribute(node_attr=node_attr)
+                updated_field_names.append(attr_name)
+        return updated_field_names

infrahub/profiles/queries/get_profile_data.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+from typing import Any
+
+from infrahub.core.constants import NULL_VALUE
+from infrahub.core.query import Query, QueryType
+from infrahub.database import InfrahubDatabase
+
+
+@dataclass
+class ProfileData:
+    uuid: str
+    priority: float | int
+    attribute_values: dict[str, Any]
+
+
+class GetProfileDataQuery(Query):
+    type: QueryType = QueryType.READ
+    insert_return: bool = False
+
+    def __init__(self, *args: Any, profile_ids: list[str], attr_names: list[str], **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.profile_ids = profile_ids
+        self.attr_names = attr_names
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        branch_filter, branch_params = self.branch.get_query_filter_path(at=self.at.to_string())
+        self.params.update(branch_params)
+        self.params["profile_ids"] = self.profile_ids
+        self.params["attr_names"] = self.attr_names + ["profile_priority"]
+
+        query = """
+// --------------
+// get the Profile nodes
+// --------------
+MATCH (profile:Node)
+WHERE profile.uuid IN $profile_ids
+// --------------
+// make sure we only use the active ones
+// --------------
+CALL (profile) {
+    MATCH (profile)-[r:IS_PART_OF]->(:Root)
+    WHERE %(branch_filter)s
+    ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
+    RETURN r.status = "active" AS is_active
+}
+WITH profile
+WHERE is_active = TRUE
+// --------------
+// get the attributes that we care about
+// --------------
+MATCH (profile)-[:HAS_ATTRIBUTE]-(attr:Attribute)
+WHERE attr.name IN $attr_names
+WITH DISTINCT profile, attr
+CALL (profile, attr) {
+    MATCH (profile)-[r:HAS_ATTRIBUTE]->(attr)
+    WHERE %(branch_filter)s
+    ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
+    RETURN r.status = "active" AS is_active
+}
+WITH profile, attr, is_active
+WHERE is_active = TRUE
+// --------------
+// get the attribute values
+// --------------
+MATCH (attr)-[:HAS_VALUE]->(av:AttributeValue)
+WITH DISTINCT profile, attr, av
+CALL (attr, av) {
+    MATCH (attr)-[r:HAS_VALUE]->(av)
+    WHERE %(branch_filter)s
+    ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
+    RETURN r.status = "active" AS is_active
+}
+WITH profile, attr, av
+WHERE is_active = TRUE
+RETURN profile.uuid AS profile_uuid, attr.name AS attr_name, av.value AS attr_value
+        """ % {"branch_filter": branch_filter}
+        self.add_to_query(query)
+        self.return_labels = ["profile_uuid", "attr_name", "attr_value"]
+
+    def get_profile_data(self) -> list[ProfileData]:
+        profile_data_by_uuid: dict[str, ProfileData] = {}
+        for result in self.results:
+            profile_uuid = result.get_as_type(label="profile_uuid", return_type=str)
+            if profile_uuid not in profile_data_by_uuid:
+                profile_data_by_uuid[profile_uuid] = ProfileData(
+                    uuid=profile_uuid, priority=float("inf"), attribute_values={}
+                )
+            profile_data = profile_data_by_uuid[profile_uuid]
+            attr_name = result.get_as_type(label="attr_name", return_type=str)
+            attr_value: Any = result.get(label="attr_value")
+            if attr_value == NULL_VALUE:
+                attr_value = None
+            if attr_name == "profile_priority":
+                if attr_value is not None and not isinstance(attr_value, int):
+                    attr_value = int(attr_value)
+                profile_data.priority = attr_value
+            else:
+                profile_data.attribute_values[attr_name] = attr_value
+        return list(profile_data_by_uuid.values())

infrahub/profiles/tasks.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from prefect import flow
+from prefect.logging import get_run_logger
+
+from infrahub.workers.dependencies import get_client, get_workflow
+from infrahub.workflows.catalogue import PROFILE_REFRESH
+from infrahub.workflows.utils import add_tags
+
+REFRESH_PROFILES_MUTATION = """
+mutation RefreshProfiles(
+    $id: String!,
+  ) {
+  InfrahubProfilesRefresh(
+    data: {id: $id}
+  ) {
+    ok
+  }
+}
+"""
+
+
+@flow(
+    name="object-profiles-refresh",
+    flow_run_name="Refresh profiles for {node_id}",
+)
+async def object_profiles_refresh(
+    branch_name: str,
+    node_id: str,
+) -> None:
+    log = get_run_logger()
+    client = get_client()
+
+    await add_tags(branches=[branch_name], nodes=[node_id], db_change=True)
+    await client.execute_graphql(
+        query=REFRESH_PROFILES_MUTATION,
+        variables={"id": node_id},
+        branch_name=branch_name,
+    )
+    log.info(f"Profiles refreshed for {node_id}")
+
+
+@flow(
+    name="objects-profiles-refresh-multiple",
+    flow_run_name="Refresh profiles for multiple objects",
+)
+async def objects_profiles_refresh_multiple(
+    branch_name: str,
+    node_ids: list[str],
+) -> None:
+    log = get_run_logger()
+
+    await add_tags(branches=[branch_name])
+
+    for node_id in node_ids:
+        log.info(f"Requesting profile refresh for {node_id}")
+        await get_workflow().submit_workflow(
+            workflow=PROFILE_REFRESH,
+            parameters={
+                "branch_name": branch_name,
+                "node_id": node_id,
+            },
+        )

infrahub/repositories/create_repository.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, cast
+
+from infrahub.core.constants import RepositoryInternalStatus
+from infrahub.core.constants.infrahubkind import READONLYREPOSITORY, REPOSITORY
+from infrahub.core.protocols import CoreGenericRepository, CoreReadOnlyRepository, CoreRepository
+from infrahub.exceptions import ValidationError
+from infrahub.git.models import GitRepositoryAdd, GitRepositoryAddReadOnly
+from infrahub.log import get_logger
+from infrahub.message_bus import messages
+from infrahub.message_bus.messages.git_repository_connectivity import GitRepositoryConnectivityResponse
+from infrahub.workflows.catalogue import GIT_REPOSITORY_ADD, GIT_REPOSITORY_ADD_READ_ONLY
+
+if TYPE_CHECKING:
+    from infrahub.auth import AccountSession
+    from infrahub.context import InfrahubContext
+    from infrahub.core.branch import Branch
+    from infrahub.database import InfrahubDatabase
+    from infrahub.services import InfrahubServices
+
+log = get_logger()
+
+
+class RepositoryFinalizer:
+    def __init__(
+        self,
+        account_session: AccountSession,
+        services: InfrahubServices,
+        context: InfrahubContext,
+    ) -> None:
+        self.account_session = account_session
+        self.services = services
+        self.context = context
+
+    async def post_create(
+        self,
+        obj: CoreGenericRepository,
+        branch: Branch,
+        db: InfrahubDatabase,
+        delete_on_connectivity_failure: bool = True,
+    ) -> None:
+        """
+        Method meant to be called after a repository has been created in the database.
+        It mainly checks the connectivity to the remote repository and submit the workflow to create the repository in the local filesystem.
+        """
+
+        # If the connectivity is not good, we remove the repository to allow the user to add a new one
+        if delete_on_connectivity_failure:
+            message = messages.GitRepositoryConnectivity(
+                repository_name=obj.name.value,
+                repository_location=obj.location.value,
+            )
+            response = await self.services.message_bus.rpc(
+                message=message, response_class=GitRepositoryConnectivityResponse
+            )
+
+            if response.data.success is False:
+                await obj.delete(db=db)
+                raise ValidationError(response.data.message)
+
+        # If we are in the default branch, we set the sync status to Active
+        # If we are in another branch, we set the sync status to Staging
+        if branch.is_default:
+            obj.internal_status.value = RepositoryInternalStatus.ACTIVE.value
+        else:
+            obj.internal_status.value = RepositoryInternalStatus.STAGING.value
+        await obj.save(db=db)
+
+        # Create the new repository in the filesystem.
+        log.info("create_repository", name=obj.name.value)
+        authenticated_user = None
+        if self.account_session and self.account_session.authenticated:
+            authenticated_user = self.account_session.account_id
+
+        if obj.get_kind() == READONLYREPOSITORY:
+            obj = cast(CoreReadOnlyRepository, obj)
+            model = GitRepositoryAddReadOnly(
+                repository_id=obj.id,
+                repository_name=obj.name.value,
+                location=obj.location.value,
+                ref=obj.ref.value,
+                infrahub_branch_name=branch.name,
+                infrahub_branch_id=str(branch.get_uuid()),
+                internal_status=obj.internal_status.value,
+                created_by=authenticated_user,
+            )
+            await self.services.workflow.submit_workflow(
+                workflow=GIT_REPOSITORY_ADD_READ_ONLY,
+                context=self.context,
+                parameters={"model": model},
+            )
+
+        elif obj.get_kind() == REPOSITORY:
+            obj = cast(CoreRepository, obj)
+            git_repo_add_model = GitRepositoryAdd(
+                repository_id=obj.id,
+                repository_name=obj.name.value,
+                location=obj.location.value,
+                default_branch_name=obj.default_branch.value,
+                infrahub_branch_name=branch.name,
+                infrahub_branch_id=str(branch.get_uuid()),
+                internal_status=obj.internal_status.value,
+                created_by=authenticated_user,
+            )
+
+            await self.services.workflow.submit_workflow(
+                workflow=GIT_REPOSITORY_ADD,
+                context=self.context,
+                parameters={"model": git_repo_add_model},
+            )
+        else:
+            raise ValueError(f"Unknown repository kind: {obj.get_kind()}")

infrahub/tasks/registry.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
 from infrahub import lock
 from infrahub.core import registry
 from infrahub.core.constants import GLOBAL_BRANCH_NAME
+from infrahub.graphql.registry import registry as graphql_registry
 from infrahub.log import get_logger
 from infrahub.worker import WORKER_IDENTITY
 
@@ -20,9 +21,8 @@ def update_graphql_schema(branch: Branch, schema_branch: SchemaBranch) -> None:
     """
     Update the GraphQL schema for the given branch.
     """
-    from infrahub.graphql.manager import GraphQLSchemaManager
 
-    gqlm = GraphQLSchemaManager.get_manager_for_branch(branch=branch, schema_branch=schema_branch)
+    gqlm = graphql_registry.get_manager_for_branch(branch=branch, schema_branch=schema_branch)
     gqlm.get_graphql_schema(
         include_query=True,
         include_mutation=True,
@@ -67,6 +67,9 @@ async def update_branch_registry(db: InfrahubDatabase, branch: Branch) -> None:
                 worker=WORKER_IDENTITY,
             )
             registry.branch[branch.name] = branch
+        elif existing_branch.status != branch.status:
+            log.info(f"Updating registry branch cache for {branch.name=}")
+            registry.branch[branch.name] = branch
         return
 
     log.info(
@@ -89,7 +92,6 @@ async def refresh_branches(db: InfrahubDatabase) -> None:
     If a branch is already present with a different value for the hash
     We pull the new schema from the database and we update the registry.
     """
-    from infrahub.graphql.manager import GraphQLSchemaManager
 
     async with lock.registry.local_schema_lock():
         active_branches = await registry.branch_object.get_list(db=db)
@@ -106,7 +108,7 @@ async def refresh_branches(db: InfrahubDatabase) -> None:
 
         purged_branches = await registry.purge_inactive_branches(db=db, active_branches=active_branches)
         purged_branches.update(
-            GraphQLSchemaManager.purge_inactive(active_branches=[branch.name for branch in active_branches])
+            graphql_registry.purge_inactive(active_branches=[branch.name for branch in active_branches])
         )
         for branch_name in sorted(purged_branches):
             log.info(f"Removed branch {branch_name!r} from the registry", branch=branch_name, worker=WORKER_IDENTITY)

infrahub/webhook/models.py

@@ -231,7 +231,7 @@ class TransformWebhook(Webhook):
             commit=commit,
             location=f"{self.transform_file}::{self.transform_class}",
             convert_query_response=self.convert_query_response,
-            data={"data": data, **context.model_dump()},
+            data={"data": {"data": data, **context.model_dump()}},
             client=client,
         )  # type: ignore[misc]