infrahub-server 1.2.10__py3-none-any.whl → 1.3.0a0__py3-none-any.whl

infrahub_testcontainers/helpers.py

@@ -1,8 +1,10 @@
 import os
 import subprocess  # noqa: S404
+import uuid
 from pathlib import Path
 
 import pytest
+from prefect.client.orchestration import PrefectClient
 
 from infrahub_testcontainers import __version__ as infrahub_version
 
@@ -38,8 +40,8 @@ class TestInfrahubDocker:
 
     @pytest.fixture(scope="class")
     def tmp_directory(self, tmpdir_factory: pytest.TempdirFactory) -> Path:
-        directory = Path(str(tmpdir_factory.getbasetemp().strpath))
-        return directory
+        name = f"{self.__class__.__name__.lower()}_{uuid.uuid4().hex}"
+        return Path(str(tmpdir_factory.mktemp(name)))
 
     @pytest.fixture(scope="class")
     def remote_repos_dir(self, tmp_directory: Path) -> Path:
@@ -91,3 +93,7 @@ class TestInfrahubDocker:
     @pytest.fixture(scope="class")
     def task_manager_port(self, infrahub_app: dict[str, int]) -> int:
         return infrahub_app["task-manager"]
+
+    @pytest.fixture(scope="class")
+    def prefect_client(self, task_manager_port: int) -> PrefectClient:
+        return PrefectClient(api=f"http://localhost:{task_manager_port}/api/")

infrahub/message_bus/messages/check_generator_run.py

@@ -1,26 +0,0 @@
-from pydantic import Field
-
-from infrahub.context import InfrahubContext
-from infrahub.generators.models import ProposedChangeGeneratorDefinition
-from infrahub.message_bus import InfrahubMessage
-
-
-class CheckGeneratorRun(InfrahubMessage):
-    """A check that runs a generator."""
-
-    generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator definition")
-    generator_instance: str | None = Field(
-        default=None, description="The id of the generator instance if it previously existed"
-    )
-    commit: str = Field(..., description="The commit to target")
-    repository_id: str = Field(..., description="The unique ID of the Repository")
-    repository_name: str = Field(..., description="The name of the Repository")
-    repository_kind: str = Field(..., description="The kind of the Repository")
-    branch_name: str = Field(..., description="The branch where the check is run")
-    target_id: str = Field(..., description="The ID of the target object for this generator")
-    target_name: str = Field(..., description="Name of the generator target")
-    query: str = Field(..., description="The name of the query to use when collecting data")
-    variables: dict = Field(..., description="Input variables when running the generator")
-    validator_id: str = Field(..., description="The ID of the validator")
-    proposed_change: str | None = Field(None, description="The unique ID of the Proposed Change")
-    context: InfrahubContext = Field(..., description="The Infrahub context")

infrahub/message_bus/messages/finalize_validator_execution.py

@@ -1,15 +0,0 @@
-from pydantic import Field
-
-from infrahub.context import InfrahubContext
-from infrahub.message_bus import InfrahubMessage
-
-
-class FinalizeValidatorExecution(InfrahubMessage):
-    """Update the status of a validator after all checks have been completed."""
-
-    validator_id: str = Field(..., description="The id of the validator associated with this check")
-    validator_execution_id: str = Field(..., description="The id of current execution of the associated validator")
-    start_time: str = Field(..., description="Start time when the message was first created")
-    validator_type: str = Field(..., description="The type of validator to complete")
-    context: InfrahubContext = Field(..., description="The Infrahub context")
-    proposed_change: str = Field(..., description="The ID of the proposed change")

infrahub/message_bus/messages/proposed_change/base_with_diff.py

@@ -1,16 +0,0 @@
-from pydantic import ConfigDict, Field
-
-from infrahub.message_bus import InfrahubMessage
-from infrahub.message_bus.types import ProposedChangeBranchDiff
-
-
-class BaseProposedChangeWithDiffMessage(InfrahubMessage):
-    """Sent trigger the refresh of artifacts that are impacted by the proposed change."""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
-    source_branch: str = Field(..., description="The source branch of the proposed change")
-    source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
-    destination_branch: str = Field(..., description="The destination branch of the proposed change")
-    branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")

infrahub/message_bus/messages/proposed_change/request_proposedchange_refreshartifacts.py

@@ -1,11 +0,0 @@
-from pydantic import Field
-
-from infrahub.context import InfrahubContext
-
-from .base_with_diff import BaseProposedChangeWithDiffMessage
-
-
-class RequestProposedChangeRefreshArtifacts(BaseProposedChangeWithDiffMessage):
-    """Sent trigger the refresh of artifacts that are impacted by the proposed change."""
-
-    context: InfrahubContext = Field(..., description="The context of the task")

infrahub/message_bus/messages/request_generatordefinition_check.py

@@ -1,20 +0,0 @@
-from pydantic import ConfigDict, Field
-
-from infrahub.context import InfrahubContext
-from infrahub.generators.models import ProposedChangeGeneratorDefinition
-from infrahub.message_bus import InfrahubMessage
-from infrahub.message_bus.types import ProposedChangeBranchDiff
-
-
-class RequestGeneratorDefinitionCheck(InfrahubMessage):
-    """Sent to trigger Generators to run for a proposed change."""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator Definition")
-    branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")
-    proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
-    source_branch: str = Field(..., description="The source branch")
-    source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
-    destination_branch: str = Field(..., description="The target branch")
-    context: InfrahubContext = Field(..., description="The Infrahub context")

infrahub/message_bus/messages/request_proposedchange_pipeline.py

@@ -1,23 +0,0 @@
-import uuid
-
-from pydantic import Field
-
-from infrahub.context import InfrahubContext
-from infrahub.core.constants import CheckType
-from infrahub.message_bus import InfrahubMessage
-
-
-class RequestProposedChangePipeline(InfrahubMessage):
-    """Sent request the start of a pipeline connected to a proposed change."""
-
-    proposed_change: str = Field(..., description="The unique ID of the proposed change")
-    source_branch: str = Field(..., description="The source branch of the proposed change")
-    source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
-    destination_branch: str = Field(..., description="The destination branch of the proposed change")
-    check_type: CheckType = Field(
-        default=CheckType.ALL, description="Can be used to restrict the pipeline to a specific type of job"
-    )
-    context: InfrahubContext = Field(..., description="The context of the task")
-    pipeline_id: uuid.UUID = Field(
-        default_factory=uuid.uuid4, description="The unique ID of the execution of this pipeline"
-    )

infrahub/message_bus/operations/check/__init__.py

@@ -1,3 +0,0 @@
-from . import generator
-
-__all__ = ["generator"]

infrahub/message_bus/operations/check/generator.py

@@ -1,156 +0,0 @@
-from infrahub_sdk.exceptions import ModuleImportError
-from infrahub_sdk.node import InfrahubNode
-from infrahub_sdk.schema.repository import InfrahubGeneratorDefinitionConfig
-from prefect import flow
-from prefect.logging import get_run_logger
-
-from infrahub import lock
-from infrahub.core.constants import GeneratorInstanceStatus, InfrahubKind, ValidatorConclusion
-from infrahub.core.timestamp import Timestamp
-from infrahub.git.base import extract_repo_file_information
-from infrahub.git.repository import get_initialized_repo
-from infrahub.message_bus import messages
-from infrahub.services import InfrahubServices
-from infrahub.tasks.check import set_check_status
-from infrahub.workflows.utils import add_tags
-
-
-@flow(
-    name="git-repository-check-generator-run",
-    flow_run_name="Execute Generator {message.generator_definition.definition_name} for {message.target_name}",
-)
-async def run(message: messages.CheckGeneratorRun, service: InfrahubServices) -> None:
-    if message.proposed_change:
-        await add_tags(branches=[message.branch_name], nodes=[message.proposed_change], db_change=True)
-    else:
-        await add_tags(branches=[message.branch_name], nodes=[message.repository_id], db_change=True)
-
-    log = get_run_logger()
-
-    repository = await get_initialized_repo(
-        repository_id=message.repository_id,
-        name=message.repository_name,
-        service=service,
-        repository_kind=message.repository_kind,
-        commit=message.commit,
-    )
-
-    conclusion = ValidatorConclusion.SUCCESS
-
-    generator_definition = InfrahubGeneratorDefinitionConfig(
-        name=message.generator_definition.definition_name,
-        class_name=message.generator_definition.class_name,
-        file_path=message.generator_definition.file_path,
-        query=message.generator_definition.query_name,
-        targets=message.generator_definition.group_id,
-        convert_query_response=message.generator_definition.convert_query_response,
-    )
-
-    commit_worktree = repository.get_commit_worktree(commit=message.commit)
-
-    file_info = extract_repo_file_information(
-        full_filename=commit_worktree.directory / generator_definition.file_path,
-        repo_directory=repository.directory_root,
-        worktree_directory=commit_worktree.directory,
-    )
-    generator_instance = await _define_instance(message=message, service=service)
-
-    check_message = "Instance successfully generated"
-    try:
-        log.debug(f"repo information {file_info}")
-        log.debug(f"Root directory : {repository.directory_root}")
-        generator_class = generator_definition.load_class(
-            import_root=repository.directory_root, relative_path=file_info.relative_repo_path_dir
-        )
-
-        generator = generator_class(
-            query=generator_definition.query,
-            client=service.client,
-            branch=message.branch_name,
-            params=message.variables,
-            generator_instance=generator_instance.id,
-            convert_query_response=generator_definition.convert_query_response,
-            infrahub_node=InfrahubNode,
-        )
-        generator._init_client.request_context = message.context.to_request_context()
-        await generator.run(identifier=generator_definition.name)
-        generator_instance.status.value = GeneratorInstanceStatus.READY.value
-    except ModuleImportError as exc:
-        conclusion = ValidatorConclusion.FAILURE
-        generator_instance.status.value = GeneratorInstanceStatus.ERROR.value
-        check_message = f"Failed to import generator: {exc.message}"
-        log.exception(check_message, exc_info=exc)
-    except Exception as exc:
-        conclusion = ValidatorConclusion.FAILURE
-        generator_instance.status.value = GeneratorInstanceStatus.ERROR.value
-        check_message = f"Failed to execute generator: {str(exc)}"
-        log.exception(check_message, exc_info=exc)
-
-    log.info("Generator run completed, starting update")
-    await generator_instance.update(do_full_update=True)
-
-    check = None
-    existing_check = await service.client.filters(
-        kind=InfrahubKind.GENERATORCHECK, validator__ids=message.validator_id, instance__value=generator_instance.id
-    )
-    if existing_check:
-        check = existing_check[0]
-
-    if check:
-        check.created_at.value = Timestamp().to_string()
-        check.conclusion.value = conclusion.value
-        await check.save()
-    else:
-        check = await service.client.create(
-            kind=InfrahubKind.GENERATORCHECK,
-            data={
-                "name": message.target_name,
-                "origin": message.repository_id,
-                "kind": "GeneratorDefinition",
-                "validator": message.validator_id,
-                "created_at": Timestamp().to_string(),
-                "message": check_message,
-                "conclusion": conclusion.value,
-                "instance": generator_instance.id,
-            },
-        )
-        await check.save()
-
-    await set_check_status(message=message, conclusion=conclusion.value, service=service)
-
-
-async def _define_instance(message: messages.CheckGeneratorRun, service: InfrahubServices) -> InfrahubNode:
-    if message.generator_instance:
-        instance = await service.client.get(
-            kind=InfrahubKind.GENERATORINSTANCE, id=message.generator_instance, branch=message.branch_name
-        )
-        instance.status.value = GeneratorInstanceStatus.PENDING.value
-        await instance.update(do_full_update=True)
-
-    else:
-        async with lock.registry.get(
-            f"{message.target_id}-{message.generator_definition.definition_id}", namespace="generator"
-        ):
-            instances = await service.client.filters(
-                kind=InfrahubKind.GENERATORINSTANCE,
-                definition__ids=[message.generator_definition.definition_id],
-                object__ids=[message.target_id],
-                branch=message.branch_name,
-            )
-            if instances:
-                instance = instances[0]
-                instance.status.value = GeneratorInstanceStatus.PENDING.value
-                await instance.update(do_full_update=True)
-            else:
-                instance = await service.client.create(
-                    kind=InfrahubKind.GENERATORINSTANCE,
-                    branch=message.branch_name,
-                    data={
-                        "name": f"{message.generator_definition.definition_name}: {message.target_name}",
-                        "status": GeneratorInstanceStatus.PENDING.value,
-                        "object": message.target_id,
-                        "definition": message.generator_definition.definition_id,
-                    },
-                )
-                await instance.save()
-    return instance

infrahub/message_bus/operations/finalize/__init__.py

@@ -1,3 +0,0 @@
-from . import validator
-
-__all__ = ["validator"]

infrahub/message_bus/operations/finalize/validator.py

@@ -1,133 +0,0 @@
-from infrahub_sdk.protocols import (
-    CoreArtifactValidator,
-    CoreDataValidator,
-    CoreGeneratorValidator,
-    CoreRepositoryValidator,
-    CoreSchemaValidator,
-    CoreUserValidator,
-    CoreValidator,
-)
-from prefect import flow
-
-from infrahub import config
-from infrahub.core.constants import InfrahubKind, ValidatorConclusion
-from infrahub.core.timestamp import Timestamp
-from infrahub.log import get_logger
-from infrahub.message_bus import messages
-from infrahub.message_bus.types import KVTTL, MessageTTL
-from infrahub.services import InfrahubServices
-from infrahub.validators.events import send_failed_validator, send_passed_validator
-
-log = get_logger()
-
-
-@flow(name="validator-finalize-execution")
-async def execution(message: messages.FinalizeValidatorExecution, service: InfrahubServices) -> None:
-    """Monitors the status of checks associated with a validator and finalizes the conclusion of the validator
-
-    Based on the unique execution_id this function looks expects to see an entry in the cache for each check
-    associated with this validator. Upon seeing the result of a check the function will exclude it from further
-    checks and update the current conclusion of the validator if any of the checks failed.
-
-    The message will get rescheduled until the timeout has exceeded or until all checks are accounted for.
-    """
-    validator_type = get_validator_type(validator_type=message.validator_type)
-    validator = await service.client.get(kind=validator_type, id=message.validator_id)
-    checks_key = f"validator_execution_id:{message.validator_execution_id}:checks"
-    current_conclusion = validator.conclusion.value
-    if validator.state.value != "in_progress":
-        validator.state.value = "in_progress"
-        validator.started_at.value = Timestamp().to_string()
-        validator.completed_at.value = ""
-        await validator.save()
-
-    required_checks_data = await service.cache.get(key=checks_key) or ""
-    # Remove instances of empty checks
-    required_checks = [required_check for required_check in required_checks_data.split(",") if required_check]
-
-    completed_checks_data = await service.cache.list_keys(
-        filter_pattern=f"validator_execution_id:{message.validator_execution_id}:check_execution_id:*"
-    )
-    completed_checks = [check.split(":")[-1] for check in completed_checks_data]
-
-    missing_checks = [check for check in required_checks if check not in completed_checks]
-    checks_to_verify = [check for check in completed_checks if check in required_checks]
-    failed_check = False
-
-    for check in checks_to_verify:
-        conclusion = await service.cache.get(
-            f"validator_execution_id:{message.validator_execution_id}:check_execution_id:{check}"
-        )
-        if conclusion != "success":
-            failed_check = True
-
-    conclusion = "failure" if failed_check else "success"
-    if failed_check and current_conclusion != "failure":
-        validator.conclusion.value = "failure"
-        await validator.save()
-
-    if missing_checks:
-        remaining_checks = ",".join(missing_checks)
-        await service.cache.set(key=checks_key, value=remaining_checks, expires=KVTTL.TWO_HOURS)
-        current_time = Timestamp()
-        starting_time = Timestamp(message.start_time)
-        deadline = starting_time.add_delta(seconds=config.SETTINGS.miscellaneous.maximum_validator_execution_time)
-        if current_time < deadline:
-            log.debug(
-                "Still waiting for checks to complete",
-                missing_checks=missing_checks,
-                validator_id=message.validator_id,
-                validator_execution_id=message.validator_execution_id,
-            )
-            await service.message_bus.send(message=message, delay=MessageTTL.FIVE)
-            return
-
-        log.info(
-            "Timeout reached",
-            validator_id=message.validator_id,
-            validator_execution_id=message.validator_execution_id,
-        )
-        conclusion = "failure"
-
-    validator.state.value = "completed"
-    validator.completed_at.value = Timestamp().to_string()
-    validator.conclusion.value = conclusion
-    await validator.save()
-    if validator.conclusion.value == ValidatorConclusion.SUCCESS.value:
-        await send_passed_validator(
-            service=service, validator=validator, proposed_change_id=message.proposed_change, context=message.context
-        )
-    else:
-        await send_failed_validator(
-            service=service, validator=validator, proposed_change_id=message.proposed_change, context=message.context
-        )
-
-
-def get_validator_type(
-    validator_type: str,
-) -> (
-    type[CoreArtifactValidator]
-    | type[CoreDataValidator]
-    | type[CoreGeneratorValidator]
-    | type[CoreRepositoryValidator]
-    | type[CoreSchemaValidator]
-    | type[CoreUserValidator]
-    | type[CoreValidator]
-):
-    match validator_type:
-        case InfrahubKind.USERVALIDATOR:
-            validator_kind = CoreUserValidator
-        case InfrahubKind.SCHEMAVALIDATOR:
-            validator_kind = CoreSchemaValidator
-        case InfrahubKind.GENERATORVALIDATOR:
-            validator_kind = CoreGeneratorValidator
-        case InfrahubKind.REPOSITORYVALIDATOR:
-            validator_kind = CoreRepositoryValidator
-        case InfrahubKind.DATAVALIDATOR:
-            validator_kind = CoreDataValidator
-        case InfrahubKind.ARTIFACTVALIDATOR:
-            validator_kind = CoreArtifactValidator
-        case _:
-            validator_kind = CoreValidator
-
-    return validator_kind

infrahub/message_bus/operations/requests/__init__.py

@@ -1,9 +0,0 @@
-from . import (
-    generator_definition,
-    proposed_change,
-)
-
-__all__ = [
-    "generator_definition",
-    "proposed_change",
-]

infrahub/message_bus/operations/requests/generator_definition.py

@@ -1,140 +0,0 @@
-from infrahub_sdk.protocols import CoreGeneratorValidator
-from infrahub_sdk.uuidt import UUIDT
-from prefect import flow
-from prefect.logging import get_run_logger
-
-from infrahub.core.constants import InfrahubKind
-from infrahub.core.timestamp import Timestamp
-from infrahub.message_bus import InfrahubMessage, Meta, messages
-from infrahub.message_bus.types import KVTTL
-from infrahub.services import InfrahubServices
-from infrahub.validators.tasks import start_validator
-from infrahub.workflows.utils import add_tags
-
-
-@flow(
-    name="generator-definition-check",
-    flow_run_name="Validate Generator selection for {message.generator_definition.definition_name}",
-)
-async def check(message: messages.RequestGeneratorDefinitionCheck, service: InfrahubServices) -> None:
-    log = get_run_logger()
-    await add_tags(branches=[message.source_branch], nodes=[message.proposed_change])
-    events: list[InfrahubMessage] = []
-
-    proposed_change = await service.client.get(kind=InfrahubKind.PROPOSEDCHANGE, id=message.proposed_change)
-
-    validator_name = f"Generator Validator: {message.generator_definition.definition_name}"
-    validator_execution_id = str(UUIDT())
-    check_execution_ids: list[str] = []
-
-    await proposed_change.validations.fetch()
-
-    previous_validator: CoreGeneratorValidator | None = None
-    for relationship in proposed_change.validations.peers:
-        existing_validator = relationship.peer
-        if (
-            existing_validator.typename == InfrahubKind.GENERATORVALIDATOR
-            and existing_validator.definition.id == message.generator_definition.definition_id
-        ):
-            previous_validator = existing_validator
-
-    validator = await start_validator(
-        service=service,
-        validator=previous_validator,
-        validator_type=CoreGeneratorValidator,
-        proposed_change=message.proposed_change,
-        data={
-            "label": validator_name,
-            "definition": message.generator_definition.definition_id,
-        },
-        context=message.context,
-    )
-
-    group = await service.client.get(
-        kind=InfrahubKind.GENERICGROUP,
-        prefetch_relationships=True,
-        populate_store=True,
-        id=message.generator_definition.group_id,
-        branch=message.source_branch,
-    )
-    await group.members.fetch()
-
-    existing_instances = await service.client.filters(
-        kind=InfrahubKind.GENERATORINSTANCE,
-        definition__ids=[message.generator_definition.definition_id],
-        include=["object"],
-        branch=message.source_branch,
-    )
-    instance_by_member = {}
-    for instance in existing_instances:
-        instance_by_member[instance.object.peer.id] = instance.id
-
-    repository = message.branch_diff.get_repository(repository_id=message.generator_definition.repository_id)
-    requested_instances = 0
-    impacted_instances = message.branch_diff.get_subscribers_ids(kind=InfrahubKind.GENERATORINSTANCE)
-
-    for relationship in group.members.peers:
-        member = relationship.peer
-        generator_instance = instance_by_member.get(member.id)
-        if _run_generator(
-            instance_id=generator_instance,
-            managed_branch=message.source_branch_sync_with_git,
-            impacted_instances=impacted_instances,
-        ):
-            check_execution_id = str(UUIDT())
-            check_execution_ids.append(check_execution_id)
-            requested_instances += 1
-            log.info(f"Trigger execution of {message.generator_definition.definition_name} for {member.display_label}")
-            events.append(
-                messages.CheckGeneratorRun(
-                    context=message.context,
-                    generator_definition=message.generator_definition,
-                    generator_instance=generator_instance,
-                    commit=repository.source_commit,
-                    repository_id=repository.repository_id,
-                    repository_name=repository.repository_name,
-                    repository_kind=repository.kind,
-                    branch_name=message.source_branch,
-                    query=message.generator_definition.query_name,
-                    variables=member.extract(params=message.generator_definition.parameters),
-                    target_id=member.id,
-                    target_name=member.display_label,
-                    validator_id=validator.id,
-                    proposed_change=message.proposed_change,
-                    meta=Meta(validator_execution_id=validator_execution_id, check_execution_id=check_execution_id),
-                )
-            )
-
-    checks_in_execution = ",".join(check_execution_ids)
-    await service.cache.set(
-        key=f"validator_execution_id:{validator_execution_id}:checks",
-        value=checks_in_execution,
-        expires=KVTTL.TWO_HOURS,
-    )
-    events.append(
-        messages.FinalizeValidatorExecution(
-            start_time=Timestamp().to_string(),
-            validator_id=validator.id,
-            validator_execution_id=validator_execution_id,
-            validator_type=InfrahubKind.GENERATORVALIDATOR,
-            context=message.context,
-            proposed_change=message.proposed_change,
-        )
-    )
-    for event in events:
-        event.assign_meta(parent=message)
-        await service.message_bus.send(message=event)
-
-
-def _run_generator(instance_id: str | None, managed_branch: bool, impacted_instances: list[str]) -> bool:
-    """Returns a boolean to indicate if a generator instance needs to be executed
-    Will return true if:
-        * The instance_id wasn't set which could be that it's a new object that doesn't have a previous generator instance
-        * The source branch is set to sync with Git which would indicate that it could contain updates in git to the generator
-        * The instance_id exists in the impacted_instances list
-    Will return false if:
-        * The source branch is a not one that syncs with git and the instance_id exists and is not in the impacted list
-    """
-    if not instance_id or managed_branch:
-        return True
-    return instance_id in impacted_instances