infrahub-server 1.5.0b1__py3-none-any.whl → 1.5.0b2__py3-none-any.whl

infrahub/git/repository.py

@@ -2,6 +2,9 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+from cachetools import TTLCache
+from cachetools.keys import hashkey
+from cachetools_async import cached
 from git.exc import BadName, GitCommandError
 from infrahub_sdk.exceptions import GraphQLError
 from prefect import task
@@ -248,12 +251,13 @@ class InfrahubReadOnlyRepository(InfrahubRepositoryIntegrator):
         await self.update_commit_value(branch_name=self.infrahub_branch_name, commit=commit)
 
 
-@task(
-    name="Fetch repository commit",
-    description="Retrieve a git repository at a given commit, if it does not already exist locally",
-    cache_policy=NONE,
+@cached(
+    TTLCache(maxsize=100, ttl=30),
+    key=lambda *_, **kwargs: hashkey(
+        kwargs.get("repository_id"), kwargs.get("name"), kwargs.get("repository_kind"), kwargs.get("commit")
+    ),
 )
-async def get_initialized_repo(
+async def _get_initialized_repo(
     client: InfrahubClient, repository_id: str, name: str, repository_kind: str, commit: str | None = None
 ) -> InfrahubReadOnlyRepository | InfrahubRepository:
     if repository_kind == InfrahubKind.REPOSITORY:
@@ -263,3 +267,16 @@ async def get_initialized_repo(
         return await InfrahubReadOnlyRepository.init(id=repository_id, name=name, commit=commit, client=client)
 
     raise NotImplementedError(f"The repository kind {repository_kind} has not been implemented")
+
+
+@task(
+    name="Fetch repository commit",
+    description="Retrieve a git repository at a given commit, if it does not already exist locally",
+    cache_policy=NONE,
+)
+async def get_initialized_repo(
+    client: InfrahubClient, repository_id: str, name: str, repository_kind: str, commit: str | None = None
+) -> InfrahubReadOnlyRepository | InfrahubRepository:
+    return await _get_initialized_repo(
+        client=client, repository_id=repository_id, name=name, repository_kind=repository_kind, commit=commit
+    )

infrahub/git/tasks.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from infrahub_sdk import InfrahubClient
 from infrahub_sdk.protocols import (
     CoreArtifact,
@@ -14,7 +16,12 @@ from prefect.logging import get_run_logger
 
 from infrahub import lock
 from infrahub.context import InfrahubContext
-from infrahub.core.constants import InfrahubKind, RepositoryInternalStatus, ValidatorConclusion
+from infrahub.core.constants import (
+    InfrahubKind,
+    RepositoryInternalStatus,
+    RepositoryOperationalStatus,
+    ValidatorConclusion,
+)
 from infrahub.core.manager import NodeManager
 from infrahub.core.registry import registry
 from infrahub.exceptions import CheckError, RepositoryError
@@ -53,6 +60,7 @@ from .models import (
     UserCheckDefinitionData,
 )
 from .repository import InfrahubReadOnlyRepository, InfrahubRepository, get_initialized_repo
+from .utils import fetch_artifact_definition_targets, fetch_check_definition_targets
 
 
 @flow(
@@ -151,6 +159,39 @@ async def create_branch(branch: str, branch_id: str) -> None:
         pass
 
 
+@flow(name="sync-git-repo-with-origin", flow_run_name="Sync git repo with origin")
+async def sync_git_repo_with_origin_and_tag_on_failure(
+    client: InfrahubClient,
+    repository_id: str,
+    repository_name: str,
+    repository_location: str,
+    internal_status: str,
+    default_branch_name: str,
+    operational_status: str,
+    staging_branch: str | None = None,
+    infrahub_branch: str | None = None,
+) -> None:
+    repo = await InfrahubRepository.init(
+        id=repository_id,
+        name=repository_name,
+        location=repository_location,
+        client=client,
+        internal_status=internal_status,
+        default_branch_name=default_branch_name,
+    )
+
+    try:
+        await repo.sync(staging_branch=staging_branch)
+    except RepositoryError:
+        if operational_status == RepositoryOperationalStatus.ONLINE.value:
+            params: dict[str, Any] = {
+                "branches": [infrahub_branch] if infrahub_branch else [],
+                "nodes": [str(repository_id)],
+            }
+            await add_tags(**params)
+        raise
+
+
 @flow(name="git_repositories_sync", flow_run_name="Sync Git Repositories")
 async def sync_remote_repositories() -> None:
     log = get_run_logger()
@@ -203,7 +244,17 @@ async def sync_remote_repositories() -> None:
                     continue
 
             try:
-                await repo.sync(staging_branch=staging_branch)
+                await sync_git_repo_with_origin_and_tag_on_failure(
+                    client=client,
+                    repository_id=repository_data.repository.id,
+                    repository_name=repository_data.repository.name.value,
+                    repository_location=repository_data.repository.location.value,
+                    internal_status=active_internal_status,
+                    default_branch_name=repository_data.repository.default_branch.value,
+                    operational_status=repository_data.repository.operational_status.value,
+                    staging_branch=staging_branch,
+                    infrahub_branch=infrahub_branch,
+                )
                 # Tell workers to fetch to stay in sync
                 message = messages.RefreshGitFetch(
                     meta=Meta(initiator_id=WORKER_IDENTITY, request_id=get_log_data().get("request_id", "")),
@@ -323,9 +374,8 @@ async def generate_request_artifact_definition(
         kind=CoreArtifactDefinition, id=model.artifact_definition_id, branch=model.branch
     )
 
-    await artifact_definition.targets.fetch()
-    group = artifact_definition.targets.peer
-    await group.members.fetch()
+    group = await fetch_artifact_definition_targets(client=client, branch=model.branch, definition=artifact_definition)
+
     current_members = [member.id for member in group.members.peers]
 
     artifacts_by_member = {}
@@ -356,6 +406,7 @@ async def generate_request_artifact_definition(
         transform_location = f"{transform.file_path.value}::{transform.class_name.value}"
         convert_query_response = transform.convert_query_response.value
 
+    batch = await client.create_batch()
     for relationship in group.members.peers:
         member = relationship.peer
         artifact_id = artifacts_by_member.get(member.id)
@@ -376,6 +427,7 @@ async def generate_request_artifact_definition(
             repository_kind=repository.get_kind(),
             branch_name=model.branch,
             query=query.name.value,
+            query_id=query.id,
             variables=await member.extract(params=artifact_definition.parameters.value),
             target_id=member.id,
             target_name=member.display_label,
@@ -385,10 +437,16 @@ async def generate_request_artifact_definition(
             context=context,
         )
 
-        await get_workflow().submit_workflow(
-            workflow=REQUEST_ARTIFACT_GENERATE, context=context, parameters={"model": request_artifact_generate_model}
+        batch.add(
+            task=get_workflow().submit_workflow,
+            workflow=REQUEST_ARTIFACT_GENERATE,
+            context=context,
+            parameters={"model": request_artifact_generate_model},
         )
 
+    async for _, _ in batch.execute():
+        pass
+
 
 @flow(name="git-repository-pull-read-only", flow_run_name="Pull latest commit on {model.repository_name}")
 async def pull_read_only(model: GitRepositoryPullReadOnly) -> None:
@@ -569,9 +627,7 @@ async def trigger_repository_user_checks_definitions(model: UserCheckDefinitionD
 
     if definition.targets.id:
         # Check against a group of targets
-        await definition.targets.fetch()
-        group = definition.targets.peer
-        await group.members.fetch()
+        group = await fetch_check_definition_targets(client=client, branch=model.branch_name, definition=definition)
         check_models = []
         for relationship in group.members.peers:
             member = relationship.peer

infrahub/git/utils.py

@@ -1,9 +1,16 @@
-from typing import TYPE_CHECKING
+from collections import defaultdict
+from typing import TYPE_CHECKING, Any
+
+from infrahub_sdk import InfrahubClient
+from infrahub_sdk.node import RelationshipManager
+from infrahub_sdk.protocols import CoreArtifactDefinition, CoreCheckDefinition, CoreGroup
+from infrahub_sdk.types import Order
 
 from infrahub.core import registry
 from infrahub.core.constants import InfrahubKind
 from infrahub.core.manager import NodeManager
 from infrahub.database import InfrahubDatabase
+from infrahub.generators.models import ProposedChangeGeneratorDefinition
 
 from .models import RepositoryBranchInfo, RepositoryData
 
@@ -46,3 +53,118 @@ async def get_repositories_commit_per_branch(
             )
 
     return repositories
+
+
+def _collect_parameter_first_segments(params: Any) -> set[str]:
+    segments: set[str] = set()
+
+    def _walk(value: Any) -> None:
+        if isinstance(value, str):
+            segment = value.split("__", 1)[0]
+            if segment:
+                segments.add(segment)
+        elif isinstance(value, dict):
+            for nested in value.values():
+                _walk(nested)
+        elif isinstance(value, (list, tuple, set)):
+            for nested in value:
+                _walk(nested)
+
+    _walk(params)
+    return segments
+
+
+async def _prefetch_group_member_nodes(
+    client: InfrahubClient,
+    members: RelationshipManager,
+    branch: str,
+    required_fields: set[str],
+) -> None:
+    ids_per_kind: dict[str, set[str]] = defaultdict(set)
+    for peer in members.peers:
+        if peer.id and peer.typename:
+            ids_per_kind[peer.typename].add(peer.id)
+
+    if not ids_per_kind:
+        return
+
+    batch = await client.create_batch()
+
+    for kind, ids in ids_per_kind.items():
+        schema = await client.schema.get(kind=kind, branch=branch)
+
+        # FIXME: https://github.com/opsmill/infrahub-sdk-python/pull/205
+        valid_fields = set(schema.attribute_names) | set(schema.relationship_names)
+        keep_relationships = set(schema.relationship_names) & required_fields
+        cleaned_fields = valid_fields - required_fields
+
+        kwargs: dict[str, Any] = {
+            "kind": kind,
+            "ids": list(ids),
+            "branch": branch,
+            "exclude": list(cleaned_fields),
+            "populate_store": True,
+            "order": Order(disable=True),
+        }
+
+        if keep_relationships:
+            kwargs["include"] = list(keep_relationships)
+
+        batch.add(task=client.filters, **kwargs)
+
+    async for _ in batch.execute():
+        pass
+
+
+async def _fetch_definition_targets(
+    client: InfrahubClient,
+    branch: str,
+    group_id: str,
+    parameters: Any,
+) -> CoreGroup:
+    group = await client.get(
+        kind=CoreGroup,
+        id=group_id,
+        branch=branch,
+        include=["members"],
+    )
+
+    parameter_fields = _collect_parameter_first_segments(parameters)
+    await _prefetch_group_member_nodes(
+        client=client,
+        members=group.members,
+        branch=branch,
+        required_fields=parameter_fields,
+    )
+
+    return group
+
+
+async def fetch_artifact_definition_targets(
+    client: InfrahubClient,
+    branch: str,
+    definition: CoreArtifactDefinition,
+) -> CoreGroup:
+    return await _fetch_definition_targets(
+        client=client, branch=branch, group_id=definition.targets.id, parameters=definition.parameters.value
+    )
+
+
+async def fetch_check_definition_targets(
+    client: InfrahubClient,
+    branch: str,
+    definition: CoreCheckDefinition,
+) -> CoreGroup:
+    return await _fetch_definition_targets(
+        client=client, branch=branch, group_id=definition.targets.id, parameters=definition.parameters.value
+    )
+
+
+async def fetch_proposed_change_generator_definition_targets(
+    client: InfrahubClient,
+    branch: str,
+    definition: ProposedChangeGeneratorDefinition,
+) -> CoreGroup:
+    return await _fetch_definition_targets(
+        client=client, branch=branch, group_id=definition.group_id, parameters=definition.parameters
+    )

infrahub/graphql/api/endpoints.py

@@ -2,14 +2,15 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Query
 from fastapi.responses import PlainTextResponse
-from graphql import print_schema
+from graphql import parse, print_ast, print_schema
 from starlette.routing import Route, WebSocketRoute
 
 from infrahub.api.dependencies import get_branch_dep, get_current_user
 from infrahub.core import registry
 from infrahub.graphql.registry import registry as graphql_registry
+from infrahub.graphql.schema_sort import sort_schema_ast
 
 from .dependencies import build_graphql_app
 
@@ -27,11 +28,20 @@ router.routes.append(WebSocketRoute(path="/graphql", endpoint=graphql_app))
 router.routes.append(WebSocketRoute(path="/graphql/{branch_name:str}", endpoint=graphql_app))
 
 
-@router.get("/schema.graphql", include_in_schema=False)
+@router.get("/schema.graphql")
 async def get_graphql_schema(
-    branch: Branch = Depends(get_branch_dep), _: AccountSession = Depends(get_current_user)
+    branch: Branch = Depends(get_branch_dep),
+    _: AccountSession = Depends(get_current_user),
+    sort_schema: bool = Query(default=False, alias="sorted", description="Whether to sort the schema alphabetically."),
 ) -> PlainTextResponse:
     schema_branch = registry.schema.get_schema_branch(name=branch.name)
     gqlm = graphql_registry.get_manager_for_branch(branch=branch, schema_branch=schema_branch)
     graphql_schema = gqlm.get_graphql_schema()
+
+    if sort_schema:
+        schema_str = print_schema(graphql_schema)
+        schema_ast = parse(schema_str)
+        sorted_schema_ast = sort_schema_ast(schema_ast)
+        return PlainTextResponse(content=print_ast(sorted_schema_ast))
+
     return PlainTextResponse(content=print_schema(graphql_schema))

infrahub/graphql/manager.py

@@ -780,10 +780,7 @@ class GraphQLSchemaManager:
             attr_kind = get_attr_kind(schema, attr)
             attr_type = get_attribute_type(kind=attr_kind).get_graphql_update()
 
-            # A Field is not required if explicitly indicated or if a default value has been provided
-            required = not attr.optional if not attr.default_value else False
-
-            attrs[attr.name] = graphene.InputField(attr_type, required=required, description=attr.description)
+            attrs[attr.name] = graphene.InputField(attr_type, description=attr.description)
 
         for rel in schema.relationships:
             if rel.internal_peer or rel.read_only:
@@ -791,14 +788,12 @@ class GraphQLSchemaManager:
 
             input_type = self._get_related_input_type(relationship=rel)
 
-            required = not rel.optional
             if rel.cardinality == RelationshipCardinality.ONE:
-                attrs[rel.name] = graphene.InputField(input_type, required=required, description=rel.description)
+                attrs[rel.name] = graphene.InputField(input_type, description=rel.description)
 
             elif rel.cardinality == RelationshipCardinality.MANY:
-                attrs[rel.name] = graphene.InputField(
-                    graphene.List(input_type), required=required, description=rel.description
-                )
+                attrs[rel.name] = graphene.InputField(graphene.List(input_type), description=rel.description)
+
         input_name = f"{schema.kind}UpsertInput"
         md5hash = hashlib.md5(usedforsecurity=False)
         md5hash.update(f"{input_name}{schema.get_hash()}".encode())

infrahub/graphql/mutations/convert_object_type.py

@@ -10,9 +10,11 @@ from infrahub.core.convert_object_type.object_conversion import ConversionFieldI
 from infrahub.core.convert_object_type.repository_conversion import convert_repository_type
 from infrahub.core.convert_object_type.schema_mapping import get_schema_mapping
 from infrahub.core.manager import NodeManager
+from infrahub.exceptions import ValidationError
 from infrahub.repositories.create_repository import RepositoryFinalizer
 
 if TYPE_CHECKING:
+    from infrahub.core.attribute import BaseAttribute
     from infrahub.graphql.initialization import GraphqlContext
 
 
@@ -49,7 +51,9 @@ class ConvertObjectType(Mutation):
 
         fields_mapping: dict[str, ConversionFieldInput] = {}
         if not isinstance(data.fields_mapping, dict):
-            raise ValueError(f"Expected `fields_mapping` to be a `dict`, got {type(data.fields_mapping)}")
+            raise ValidationError(
+                input_value=f"Expected `fields_mapping` to be a `dict`, got {type(data.fields_mapping)}"
+            )
 
         for field_name, input_for_dest_field_str in data.fields_mapping.items():
             fields_mapping[field_name] = ConversionFieldInput(**input_for_dest_field_str)
@@ -57,6 +61,12 @@ class ConvertObjectType(Mutation):
         node_to_convert = await NodeManager.get_one(
             id=str(data.node_id), db=graphql_context.db, branch=graphql_context.branch
         )
+        for attribute_name in source_schema.attribute_names:
+            attribute: BaseAttribute = getattr(node_to_convert, attribute_name)
+            if attribute.is_from_profile:
+                raise ValidationError(
+                    input_value=f"The attribute '{attribute_name}' is from a profile, converting objects that use profiles is not yet supported."
+                )
 
         # Complete fields mapping with auto-mapping.
         mapping = get_schema_mapping(source_schema=source_schema, target_schema=target_schema)

infrahub/graphql/mutations/display_label.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any
 from graphene import Boolean, InputObjectType, Mutation, String
 
 from infrahub.core.account import ObjectPermission
-from infrahub.core.constants import PermissionAction, PermissionDecision
+from infrahub.core.constants import GlobalPermissions, PermissionAction, PermissionDecision
 from infrahub.core.manager import NodeManager
 from infrahub.core.registry import registry
 from infrahub.database import retry_db_transaction
@@ -15,6 +15,7 @@ from infrahub.exceptions import NodeNotFoundError, ValidationError
 from infrahub.graphql.context import apply_external_context
 from infrahub.graphql.types.context import ContextInput
 from infrahub.log import get_log_data
+from infrahub.permissions import define_global_permission_from_branch
 from infrahub.worker import WORKER_IDENTITY
 
 if TYPE_CHECKING:
@@ -52,15 +53,21 @@ class UpdateDisplayLabel(Mutation):
         if not node_schema.display_label:
             raise ValidationError(input_value=f"{node_schema.kind}.display_label has not been defined for this kind.")
 
-        graphql_context.active_permissions.raise_for_permission(
-            permission=ObjectPermission(
-                namespace=node_schema.namespace,
-                name=node_schema.name,
-                action=PermissionAction.UPDATE.value,
-                decision=PermissionDecision.ALLOW_DEFAULT.value
-                if graphql_context.branch.name == registry.default_branch
-                else PermissionDecision.ALLOW_OTHER.value,
-            )
+        graphql_context.active_permissions.raise_for_permissions(
+            permissions=[
+                define_global_permission_from_branch(
+                    permission=GlobalPermissions.UPDATE_OBJECT_HFID_DISPLAY_LABEL,
+                    branch_name=graphql_context.branch.name,
+                ),
+                ObjectPermission(
+                    namespace=node_schema.namespace,
+                    name=node_schema.name,
+                    action=PermissionAction.UPDATE.value,
+                    decision=PermissionDecision.ALLOW_DEFAULT.value
+                    if graphql_context.branch.name == registry.default_branch
+                    else PermissionDecision.ALLOW_OTHER.value,
+                ),
+            ]
         )
         await apply_external_context(graphql_context=graphql_context, context_input=context)
 

infrahub/graphql/mutations/hfid.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, cast
 from graphene import Boolean, InputObjectType, List, Mutation, NonNull, String
 
 from infrahub.core.account import ObjectPermission
-from infrahub.core.constants import PermissionAction, PermissionDecision
+from infrahub.core.constants import GlobalPermissions, PermissionAction, PermissionDecision
 from infrahub.core.manager import NodeManager
 from infrahub.core.registry import registry
 from infrahub.database import retry_db_transaction
@@ -15,6 +15,7 @@ from infrahub.exceptions import NodeNotFoundError, ValidationError
 from infrahub.graphql.context import apply_external_context
 from infrahub.graphql.types.context import ContextInput
 from infrahub.log import get_log_data
+from infrahub.permissions import define_global_permission_from_branch
 from infrahub.worker import WORKER_IDENTITY
 
 if TYPE_CHECKING:
@@ -61,15 +62,21 @@ class UpdateHFID(Mutation):
                 input_value=f"{node_schema.kind}.human_friendly_id requires {len(node_schema.human_friendly_id)} parts data has {len(updated_hfid)}"
             )
 
-        graphql_context.active_permissions.raise_for_permission(
-            permission=ObjectPermission(
-                namespace=node_schema.namespace,
-                name=node_schema.name,
-                action=PermissionAction.UPDATE.value,
-                decision=PermissionDecision.ALLOW_DEFAULT.value
-                if graphql_context.branch.name == registry.default_branch
-                else PermissionDecision.ALLOW_OTHER.value,
-            )
+        graphql_context.active_permissions.raise_for_permissions(
+            permissions=[
+                define_global_permission_from_branch(
+                    permission=GlobalPermissions.UPDATE_OBJECT_HFID_DISPLAY_LABEL,
+                    branch_name=graphql_context.branch.name,
+                ),
+                ObjectPermission(
+                    namespace=node_schema.namespace,
+                    name=node_schema.name,
+                    action=PermissionAction.UPDATE.value,
+                    decision=PermissionDecision.ALLOW_DEFAULT.value
+                    if graphql_context.branch.name == registry.default_branch
+                    else PermissionDecision.ALLOW_OTHER.value,
+                ),
+            ]
         )
         await apply_external_context(graphql_context=graphql_context, context_input=context)