infrahub-server 1.4.12__py3-none-any.whl → 1.5.0__py3-none-any.whl

infrahub/hfid/triggers.py

@@ -0,0 +1,22 @@
+from infrahub.events.branch_action import BranchDeletedEvent
+from infrahub.events.schema_action import SchemaUpdatedEvent
+from infrahub.trigger.models import BuiltinTriggerDefinition, EventTrigger, ExecuteWorkflow
+from infrahub.workflows.catalogue import HFID_SETUP
+
+TRIGGER_HFID_ALL_SCHEMA = BuiltinTriggerDefinition(
+    name="hfid-setup-all",
+    trigger=EventTrigger(events={SchemaUpdatedEvent.event_name, BranchDeletedEvent.event_name}),
+    actions=[
+        ExecuteWorkflow(
+            workflow=HFID_SETUP,
+            parameters={
+                "branch_name": "{{ event.resource['infrahub.branch.name'] }}",
+                "event_name": "{{ event.event }}",
+                "context": {
+                    "__prefect_kind": "json",
+                    "value": {"__prefect_kind": "jinja", "template": "{{ event.payload['context'] | tojson }}"},
+                },
+            },
+        ),
+    ],
+)

infrahub/lock.py

@@ -5,6 +5,7 @@ import time
 import uuid
 from asyncio import Lock as LocalLock
 from asyncio import sleep
+from contextvars import ContextVar
 from typing import TYPE_CHECKING
 
 import redis.asyncio as redis
@@ -25,6 +26,7 @@ registry: InfrahubLockRegistry = None
 
 
 METRIC_PREFIX = "infrahub_lock"
+LOCK_PREFIX = "lock"
 
 LOCK_ACQUIRE_TIME_METRICS = Histogram(
     f"{METRIC_PREFIX}_acquire_seconds",
@@ -49,9 +51,12 @@ GLOBAL_GRAPH_LOCK = "global.graph"
 class InfrahubMultiLock:
     """Context manager to allow multiple locks to be reserved together"""
 
-    def __init__(self, lock_registry: InfrahubLockRegistry, locks: list[str] | None = None) -> None:
+    def __init__(
+        self, lock_registry: InfrahubLockRegistry, locks: list[str] | None = None, metrics: bool = True
+    ) -> None:
         self.registry = lock_registry
         self.locks = locks or []
+        self.metrics = metrics
 
     async def __aenter__(self):
         await self.acquire()
@@ -66,11 +71,11 @@ class InfrahubMultiLock:
 
     async def acquire(self) -> None:
         for lock in self.locks:
-            await self.registry.get(name=lock).acquire()
+            await self.registry.get(name=lock, metrics=self.metrics).acquire()
 
     async def release(self) -> None:
         for lock in reversed(self.locks):
-            await self.registry.get(name=lock).release()
+            await self.registry.get(name=lock, metrics=self.metrics).release()
 
 
 class NATSLock:
@@ -97,10 +102,10 @@ class NATSLock:
         while True:
             if await self.do_acquire(token):
                 self.token = token
-                return True
+                return
             await sleep(0.1)  # default Redis GlobalLock value
 
-    async def do_acquire(self, token: str) -> bool:
+    async def do_acquire(self, token: str) -> bool | None:
         return await self.service.cache.set(key=self.name, value=token, not_exists=True)
 
     async def release(self) -> None:
@@ -122,16 +127,19 @@ class InfrahubLock:
         connection: redis.Redis | InfrahubServices | None = None,
         local: bool | None = None,
         in_multi: bool = False,
+        metrics: bool = True,
     ) -> None:
-        self.use_local: bool = local
+        self.use_local: bool | None = local
         self.local: LocalLock = None
         self.remote: GlobalLock = None
         self.name: str = name
         self.connection: redis.Redis | None = connection
         self.in_multi: bool = in_multi
         self.lock_type: str = "multi" if self.in_multi else "individual"
-        self.acquire_time: int | None = None
+        self._acquire_time: int | None = None
         self.event = asyncio.Event()
+        self._recursion_var: ContextVar[int | None] = ContextVar(f"infrahub_lock_recursion_{self.name}", default=None)
+        self.metrics = metrics
 
         if not self.connection or (self.use_local is None and name.startswith("local.")):
             self.use_local = True
@@ -139,9 +147,20 @@ class InfrahubLock:
         if self.use_local:
             self.local = LocalLock()
         elif config.SETTINGS.cache.driver == config.CacheDriver.Redis:
-            self.remote = GlobalLock(redis=self.connection, name=f"lock.{self.name}")
+            self.remote = GlobalLock(redis=self.connection, name=f"{LOCK_PREFIX}.{self.name}")
         else:
-            self.remote = NATSLock(service=self.connection, name=f"lock.{self.name}")
+            self.remote = NATSLock(service=self.connection, name=f"{LOCK_PREFIX}.{self.name}")
+
+    @property
+    def acquire_time(self) -> int:
+        if self._acquire_time is not None:
+            return self._acquire_time
+
+        raise ValueError("The lock has not been initialized")
+
+    @acquire_time.setter
+    def acquire_time(self, value: int) -> None:
+        self._acquire_time = value
 
     async def __aenter__(self):
         await self.acquire()
@@ -155,21 +174,47 @@ class InfrahubLock:
         await self.release()
 
     async def acquire(self) -> None:
-        with LOCK_ACQUIRE_TIME_METRICS.labels(self.name, self.lock_type).time():
-            if not self.use_local:
-                await self.remote.acquire(token=f"{current_timestamp()}::{WORKER_IDENTITY}")
-            else:
-                await self.local.acquire()
+        depth = self._recursion_var.get()
+        if depth is not None:
+            self._recursion_var.set(depth + 1)
+            return
+
+        if self.metrics:
+            with LOCK_ACQUIRE_TIME_METRICS.labels(self.name, self.lock_type).time():
+                if not self.use_local:
+                    await self.remote.acquire(token=f"{current_timestamp()}::{WORKER_IDENTITY}")
+                else:
+                    await self.local.acquire()
+        elif not self.use_local:
+            await self.remote.acquire(token=f"{current_timestamp()}::{WORKER_IDENTITY}")
+        else:
+            await self.local.acquire()
+
         self.acquire_time = time.time_ns()
         self.event.clear()
+        self._recursion_var.set(1)
 
     async def release(self) -> None:
-        duration_ns = time.time_ns() - self.acquire_time
-        LOCK_RESERVE_TIME_METRICS.labels(self.name, self.lock_type).observe(duration_ns / 1000000000)
+        depth = self._recursion_var.get()
+        if depth is None:
+            raise RuntimeError("Lock release attempted without ownership context.")
+
+        if depth > 1:
+            self._recursion_var.set(depth - 1)
+            return
+
+        if self.acquire_time is not None:
+            duration_ns = time.time_ns() - self.acquire_time
+            if self.metrics:
+                LOCK_RESERVE_TIME_METRICS.labels(self.name, self.lock_type).observe(duration_ns / 1000000000)
+            self.acquire_time = None
+
         if not self.use_local:
             await self.remote.release()
         else:
             self.local.release()
+
+        self._recursion_var.set(None)
         self.event.set()
 
     async def locked(self) -> bool:
@@ -179,9 +224,54 @@ class InfrahubLock:
         return self.local.locked()
 
 
+class LockNameGenerator:
+    local = "local"
+    _global = "global"
+
+    def generate_name(self, name: str, namespace: str | None = None, local: bool | None = None) -> str:
+        if namespace is None and local is None:
+            return name
+
+        new_name = ""
+        if local is True:
+            new_name = f"{self.local}."
+        elif local is False:
+            new_name = f"{self._global}."
+
+        if namespace is not None:
+            new_name += f"{namespace}."
+        new_name += name
+
+        return new_name
+
+    def unpack_name(self, name: str) -> tuple[str, str | None, bool | None]:
+        local = None
+        namespace = None
+
+        parts = name.split(".")
+        if parts[0] == self.local:
+            local = True
+            parts = parts[1:]
+        elif parts[0] == self._global:
+            local = False
+            parts = parts[1:]
+
+        if len(parts) > 1:
+            namespace = parts[0]
+            original_name = ".".join(parts[1:])
+        else:
+            original_name = parts[0]
+
+        return original_name, namespace, local
+
+
 class InfrahubLockRegistry:
     def __init__(
-        self, token: str | None = None, local_only: bool = False, service: InfrahubServices | None = None
+        self,
+        token: str | None = None,
+        local_only: bool = False,
+        service: InfrahubServices | None = None,
+        name_generator: LockNameGenerator | None = None,
     ) -> None:
         if config.SETTINGS.cache.enable and not local_only:
             if config.SETTINGS.cache.driver == config.CacheDriver.Redis:
@@ -201,23 +291,7 @@ class InfrahubLockRegistry:
 
         self.token = token or str(uuid.uuid4())
         self.locks: dict[str, InfrahubLock] = {}
-
-    @classmethod
-    def _generate_name(cls, name: str, namespace: str | None = None, local: bool | None = None) -> str:
-        if namespace is None and local is None:
-            return name
-
-        new_name = ""
-        if local is True:
-            new_name = "local."
-        elif local is False:
-            new_name = "global."
-
-        if namespace is not None:
-            new_name += f"{namespace}."
-        new_name += name
-
-        return new_name
+        self.name_generator = name_generator or LockNameGenerator()
 
     def get_existing(
         self,
@@ -225,17 +299,24 @@ class InfrahubLockRegistry:
         namespace: str | None,
         local: bool | None = None,
     ) -> InfrahubLock | None:
-        lock_name = self._generate_name(name=name, namespace=namespace, local=local)
+        lock_name = self.name_generator.generate_name(name=name, namespace=namespace, local=local)
         if lock_name not in self.locks:
             return None
         return self.locks[lock_name]
 
     def get(
-        self, name: str, namespace: str | None = None, local: bool | None = None, in_multi: bool = False
+        self,
+        name: str,
+        namespace: str | None = None,
+        local: bool | None = None,
+        in_multi: bool = False,
+        metrics: bool = True,
     ) -> InfrahubLock:
-        lock_name = self._generate_name(name=name, namespace=namespace, local=local)
+        lock_name = self.name_generator.generate_name(name=name, namespace=namespace, local=local)
         if lock_name not in self.locks:
-            self.locks[lock_name] = InfrahubLock(name=lock_name, connection=self.connection, in_multi=in_multi)
+            self.locks[lock_name] = InfrahubLock(
+                name=lock_name, connection=self.connection, in_multi=in_multi, metrics=metrics
+            )
         return self.locks[lock_name]
 
     def local_schema_lock(self) -> LocalLock:
@@ -257,7 +338,3 @@ class InfrahubLockRegistry:
 def initialize_lock(local_only: bool = False, service: InfrahubServices | None = None) -> None:
     global registry
     registry = InfrahubLockRegistry(local_only=local_only, service=service)
-
-
-def build_object_lock_name(name: str) -> str:
-    return f"global.object.{name}"

infrahub/locks/tasks.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from prefect import flow
+from prefect.logging import get_run_logger
+
+from infrahub import config
+from infrahub.core.registry import registry
+from infrahub.core.timestamp import Timestamp
+from infrahub.lock import LOCK_PREFIX
+from infrahub.services import InfrahubServices  # noqa: TC001  needed for prefect flow
+
+
+@flow(
+    name="clean-up-deadlocks",
+    flow_run_name="Clean up deadlocks",
+)
+async def clean_up_deadlocks(service: InfrahubServices) -> None:
+    """Remove stale distributed locks left behind by inactive workers"""
+    keys = await service.cache.list_keys(filter_pattern=f"{LOCK_PREFIX}*")
+    if not keys:
+        return
+
+    log = get_run_logger()
+    values = await service.cache.get_values(keys=keys)
+    workers = await service.component.list_workers(branch=registry.default_branch, schema_hash=False)
+    workers_active = {worker.id for worker in workers if worker.active}
+
+    for key, value in zip(keys, values, strict=False):
+        if not key or not value:
+            continue
+
+        timestamp, worker_id = value.split("::", 1)
+        if worker_id not in workers_active and Timestamp() > Timestamp(timestamp).add(
+            minutes=config.SETTINGS.cache.clean_up_deadlocks_interval_mins
+        ):
+            await service.cache.delete(key)
+            log.info(f"Deleted deadlock key={key} worker={worker_id}")

infrahub/message_bus/types.py

@@ -92,6 +92,7 @@ class ProposedChangeArtifactDefinition(BaseModel):
     query_name: str  # Deprecated
     query_id: str
     query_models: list[str]
+    query_payload: str = Field(..., description="GraphQL query")
     repository_id: str
     transform_kind: str
     template_path: str = Field(default="")

infrahub/patch/plan_writer.py

@@ -1,6 +1,6 @@
 import json
 from dataclasses import asdict
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -10,7 +10,7 @@ from .models import EdgeToAdd, EdgeToDelete, EdgeToUpdate, PatchPlan, VertexToAd
 
 class PatchPlanWriter:
     def write(self, patches_directory: Path, patch_plan: PatchPlan) -> Path:
-        timestamp_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d-%H%M%S")
+        timestamp_str = datetime.now(tz=UTC).strftime("%Y%m%d-%H%M%S")
         patch_name = f"patch-{patch_plan.name}-{timestamp_str}"
         patch_plan_directory = patches_directory / Path(patch_name)
         if not patch_plan_directory.exists():

infrahub/permissions/constants.py

@@ -30,6 +30,7 @@ GLOBAL_PERMISSION_DENIAL_MESSAGE = {
     GlobalPermissions.MANAGE_ACCOUNTS.value: "You are not allowed to manage user accounts, groups or roles",
     GlobalPermissions.MANAGE_PERMISSIONS.value: "You are not allowed to manage permissions",
     GlobalPermissions.MANAGE_REPOSITORIES.value: "You are not allowed to manage repositories",
+    GlobalPermissions.UPDATE_OBJECT_HFID_DISPLAY_LABEL.value: "You are not allowed to update human friendly IDs and display labels ad hoc",
 }
 
 GLOBAL_PERMISSION_DESCRIPTION = {
@@ -42,4 +43,5 @@ GLOBAL_PERMISSION_DESCRIPTION = {
     GlobalPermissions.MANAGE_PERMISSIONS: "Allow a user to manage permissions",
     GlobalPermissions.MANAGE_REPOSITORIES: "Allow a user to manage repositories",
     GlobalPermissions.SUPER_ADMIN: "Allow a user to do anything",
+    GlobalPermissions.UPDATE_OBJECT_HFID_DISPLAY_LABEL: "Allow a user to update objects' display labels and human friendly IDs ad hoc",
 }

infrahub/profiles/node_applier.py

@@ -0,0 +1,101 @@
+from typing import Any
+
+from infrahub.core.attribute import BaseAttribute
+from infrahub.core.branch import Branch
+from infrahub.core.node import Node
+from infrahub.database import InfrahubDatabase
+
+from .queries.get_profile_data import GetProfileDataQuery, ProfileData
+
+
+class NodeProfilesApplier:
+    def __init__(self, db: InfrahubDatabase, branch: Branch):
+        self.db = db
+        self.branch = branch
+
+    async def _get_profile_ids(self, node: Node) -> list[str]:
+        try:
+            profiles_rel = node.get_relationship("profiles")
+        except ValueError:
+            return []
+        profile_rels = await profiles_rel.get_relationships(db=self.db)
+        return [pr.peer_id for pr in profile_rels if pr.peer_id]
+
+    async def _get_attr_names_for_profiles(self, node: Node) -> list[str]:
+        node_schema = node.get_schema()
+
+        # get the names of attributes that could be affected by profile changes
+        attr_names_for_profiles: list[str] = []
+        for attr_schema in node_schema.attributes:
+            attr_name = attr_schema.name
+            node_attr: BaseAttribute = getattr(node, attr_name)
+            if node_attr.is_from_profile or node_attr.is_default:
+                attr_names_for_profiles.append(attr_name)
+        return attr_names_for_profiles
+
+    async def _get_sorted_profile_data(
+        self, profile_ids: list[str], attr_names_for_profiles: list[str]
+    ) -> list[ProfileData]:
+        if not profile_ids:
+            return []
+        query = await GetProfileDataQuery.init(
+            db=self.db, branch=self.branch, profile_ids=profile_ids, attr_names=attr_names_for_profiles
+        )
+        await query.execute(db=self.db)
+        profile_data_list = query.get_profile_data()
+        return sorted(profile_data_list, key=lambda x: (x.priority, x.uuid))
+
+    def _apply_profile_to_attribute(self, node_attr: BaseAttribute, profile_value: Any, profile_id: str) -> bool:
+        is_changed = False
+        if node_attr.value != profile_value:
+            node_attr.value = profile_value
+            is_changed = True
+        if node_attr.is_default is not False:
+            node_attr.is_default = False
+            is_changed = True
+        if node_attr.is_from_profile is not True:
+            node_attr.is_from_profile = True
+            is_changed = True
+        if node_attr.source_id != profile_id:  # type: ignore[attr-defined]
+            node_attr.set_source(value=profile_id)
+            is_changed = True
+        return is_changed
+
+    def _remove_profile_from_attribute(self, node_attr: BaseAttribute) -> None:
+        node_attr.clear_source()
+        node_attr.value = node_attr.schema.default_value
+        node_attr.is_default = True
+        node_attr.is_from_profile = False
+
+    async def apply_profiles(self, node: Node) -> list[str]:
+        profile_ids = await self._get_profile_ids(node=node)
+        attr_names_for_profiles = await self._get_attr_names_for_profiles(node=node)
+
+        if not attr_names_for_profiles:
+            return []
+
+        # get profiles priorities and attribute values on branch
+        sorted_profile_data = await self._get_sorted_profile_data(
+            profile_ids=profile_ids, attr_names_for_profiles=attr_names_for_profiles
+        )
+
+        updated_field_names = []
+        # set attribute values/is_default/is_from_profile on nodes
+        for attr_name in attr_names_for_profiles:
+            has_profile_data = False
+            node_attr = node.get_attribute(attr_name)
+            for profile_data in sorted_profile_data:
+                profile_value = profile_data.attribute_values.get(attr_name)
+                if profile_value is not None:
+                    has_profile_data = True
+                    is_changed = False
+                    is_changed = self._apply_profile_to_attribute(
+                        node_attr=node_attr, profile_value=profile_value, profile_id=profile_data.uuid
+                    )
+                    if is_changed:
+                        updated_field_names.append(attr_name)
+                    break
+            if not has_profile_data and node_attr.is_from_profile:
+                self._remove_profile_from_attribute(node_attr=node_attr)
+                updated_field_names.append(attr_name)
+        return updated_field_names

infrahub/profiles/queries/get_profile_data.py

@@ -0,0 +1,98 @@
+from dataclasses import dataclass
+from typing import Any
+
+from infrahub.core.constants import NULL_VALUE
+from infrahub.core.query import Query, QueryType
+from infrahub.database import InfrahubDatabase
+
+
+@dataclass
+class ProfileData:
+    uuid: str
+    priority: float | int
+    attribute_values: dict[str, Any]
+
+
+class GetProfileDataQuery(Query):
+    type: QueryType = QueryType.READ
+    insert_return: bool = False
+
+    def __init__(self, *args: Any, profile_ids: list[str], attr_names: list[str], **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.profile_ids = profile_ids
+        self.attr_names = attr_names
+
+    async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> None:  # noqa: ARG002
+        branch_filter, branch_params = self.branch.get_query_filter_path(at=self.at.to_string())
+        self.params.update(branch_params)
+        self.params["profile_ids"] = self.profile_ids
+        self.params["attr_names"] = self.attr_names + ["profile_priority"]
+
+        query = """
+// --------------
+// get the Profile nodes
+// --------------
+MATCH (profile:Node)
+WHERE profile.uuid IN $profile_ids
+// --------------
+// make sure we only use the active ones
+// --------------
+CALL (profile) {
+    MATCH (profile)-[r:IS_PART_OF]->(:Root)
+    WHERE %(branch_filter)s
+    ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
+    RETURN r.status = "active" AS is_active
+}
+WITH profile
+WHERE is_active = TRUE
+// --------------
+// get the attributes that we care about
+// --------------
+MATCH (profile)-[:HAS_ATTRIBUTE]-(attr:Attribute)
+WHERE attr.name IN $attr_names
+WITH DISTINCT profile, attr
+CALL (profile, attr) {
+    MATCH (profile)-[r:HAS_ATTRIBUTE]->(attr)
+    WHERE %(branch_filter)s
+    ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
+    RETURN r.status = "active" AS is_active
+}
+WITH profile, attr
+WHERE is_active = TRUE
+// --------------
+// get the attribute values
+// --------------
+CALL (attr) {
+    MATCH (attr)-[r:HAS_VALUE]->(av)
+    WHERE %(branch_filter)s
+    ORDER BY r.branch_level DESC, r.from DESC, r.status ASC
+    RETURN av, r.status = "active" AS is_active
+    LIMIT 1
+}
+WITH profile, attr, av
+WHERE is_active = TRUE
+RETURN profile.uuid AS profile_uuid, attr.name AS attr_name, av.value AS attr_value
+        """ % {"branch_filter": branch_filter}
+        self.add_to_query(query)
+        self.return_labels = ["profile_uuid", "attr_name", "attr_value"]
+
+    def get_profile_data(self) -> list[ProfileData]:
+        profile_data_by_uuid: dict[str, ProfileData] = {}
+        for result in self.results:
+            profile_uuid = result.get_as_type(label="profile_uuid", return_type=str)
+            if profile_uuid not in profile_data_by_uuid:
+                profile_data_by_uuid[profile_uuid] = ProfileData(
+                    uuid=profile_uuid, priority=float("inf"), attribute_values={}
+                )
+            profile_data = profile_data_by_uuid[profile_uuid]
+            attr_name = result.get_as_type(label="attr_name", return_type=str)
+            attr_value: Any = result.get(label="attr_value")
+            if attr_value == NULL_VALUE:
+                attr_value = None
+            if attr_name == "profile_priority":
+                if attr_value is not None and not isinstance(attr_value, int):
+                    attr_value = int(attr_value)
+                profile_data.priority = attr_value
+            else:
+                profile_data.attribute_values[attr_name] = attr_value
+        return list(profile_data_by_uuid.values())

infrahub/profiles/tasks.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from prefect import flow
+from prefect.logging import get_run_logger
+
+from infrahub.workers.dependencies import get_client, get_workflow
+from infrahub.workflows.catalogue import PROFILE_REFRESH
+from infrahub.workflows.utils import add_tags
+
+REFRESH_PROFILES_MUTATION = """
+mutation RefreshProfiles(
+    $id: String!,
+  ) {
+  InfrahubProfilesRefresh(
+    data: {id: $id}
+  ) {
+    ok
+  }
+}
+"""
+
+
+@flow(
+    name="object-profiles-refresh",
+    flow_run_name="Refresh profiles for {node_id}",
+)
+async def object_profiles_refresh(
+    branch_name: str,
+    node_id: str,
+) -> None:
+    log = get_run_logger()
+    client = get_client()
+
+    await add_tags(branches=[branch_name], nodes=[node_id], db_change=True)
+    await client.execute_graphql(
+        query=REFRESH_PROFILES_MUTATION,
+        variables={"id": node_id},
+        branch_name=branch_name,
+    )
+    log.info(f"Profiles refreshed for {node_id}")
+
+
+@flow(
+    name="objects-profiles-refresh-multiple",
+    flow_run_name="Refresh profiles for multiple objects",
+)
+async def objects_profiles_refresh_multiple(
+    branch_name: str,
+    node_ids: list[str],
+) -> None:
+    log = get_run_logger()
+
+    await add_tags(branches=[branch_name])
+
+    for node_id in node_ids:
+        log.info(f"Requesting profile refresh for {node_id}")
+        await get_workflow().submit_workflow(
+            workflow=PROFILE_REFRESH,
+            parameters={
+                "branch_name": branch_name,
+                "node_id": node_id,
+            },
+        )