infrahub-server 1.1.6__py3-none-any.whl → 1.2.0rc0__py3-none-any.whl

infrahub/proposed_change/tasks.py

@@ -17,16 +17,19 @@ from prefect.logging import get_run_logger
 from prefect.states import Completed, Failed
 
 from infrahub import config
+from infrahub.artifacts.models import CheckArtifactCreate
+from infrahub.context import InfrahubContext  # noqa: TC001  needed for prefect flow
 from infrahub.core import registry
 from infrahub.core.branch import Branch
 from infrahub.core.branch.tasks import merge_branch
-from infrahub.core.constants import InfrahubKind, RepositoryInternalStatus, ValidatorConclusion
+from infrahub.core.constants import InfrahubKind, RepositoryInternalStatus, ValidatorConclusion, ValidatorState
 from infrahub.core.diff.coordinator import DiffCoordinator
 from infrahub.core.diff.model.diff import DiffElementType, SchemaConflict
 from infrahub.core.diff.model.path import NodeDiffFieldSummary
 from infrahub.core.integrity.object_conflict.conflict_recorder import ObjectConflictValidatorRecorder
 from infrahub.core.protocols import CoreDataCheck, CoreValidator
 from infrahub.core.protocols import CoreProposedChange as InternalCoreProposedChange
+from infrahub.core.validators.checks_runner import run_checks_and_update_validator
 from infrahub.core.validators.determiner import ConstraintValidatorDeterminer
 from infrahub.core.validators.models.validate_migration import SchemaValidateMigrationData
 from infrahub.core.validators.tasks import schema_validate_migrations
@@ -39,6 +42,7 @@ from infrahub.message_bus import InfrahubMessage, messages
 from infrahub.message_bus.operations.requests.proposed_change import DefinitionSelect
 from infrahub.proposed_change.constants import ProposedChangeState
 from infrahub.proposed_change.models import (
+    RequestArtifactDefinitionCheck,
     RequestProposedChangeDataIntegrity,
     RequestProposedChangeRepositoryChecks,
     RequestProposedChangeRunGenerators,
@@ -46,8 +50,12 @@ from infrahub.proposed_change.models import (
     RequestProposedChangeUserTests,
 )
 from infrahub.pytest_plugin import InfrahubBackendPlugin
-from infrahub.services import services
-from infrahub.workflows.catalogue import COMPUTED_ATTRIBUTE_SETUP_PYTHON, REQUEST_PROPOSED_CHANGE_REPOSITORY_CHECKS
+from infrahub.services import InfrahubServices  # noqa: TC001  needed for prefect flow
+from infrahub.workflows.catalogue import (
+    COMPUTED_ATTRIBUTE_SETUP_PYTHON,
+    GIT_REPOSITORIES_CHECK_ARTIFACT_CREATE,
+    REQUEST_PROPOSED_CHANGE_REPOSITORY_CHECKS,
+)
 from infrahub.workflows.utils import add_tags
 
 if TYPE_CHECKING:
@@ -60,10 +68,10 @@ if TYPE_CHECKING:
 
 async def _proposed_change_transition_state(
     state: ProposedChangeState,
+    service: InfrahubServices,
     proposed_change: InternalCoreProposedChange | None = None,
     proposed_change_id: str | None = None,
 ) -> None:
-    service = services.service
     async with service.database.start_session() as db:
         if proposed_change is None and proposed_change_id:
             proposed_change = await registry.manager.get_one(
@@ -96,8 +104,12 @@ async def _proposed_change_transition_state(
     # on_crashed=[proposed_change_transition_open],  # type: ignore
     # on_cancellation=[proposed_change_transition_open],  # type: ignore
 )
-async def merge_proposed_change(proposed_change_id: str, proposed_change_name: str) -> State:  # pylint: disable=unused-argument
-    service = services.service
+async def merge_proposed_change(
+    proposed_change_id: str,
+    proposed_change_name: str,  # noqa: ARG001
+    context: InfrahubContext,
+    service: InfrahubServices,
+) -> State:
     log = get_run_logger()
 
     await add_tags(nodes=[proposed_change_id])
@@ -118,14 +130,16 @@ async def merge_proposed_change(proposed_change_id: str, proposed_change_name: s
                 and validation.conclusion.value.value != ValidatorConclusion.SUCCESS.value
             ):
                 # Ignoring Data integrity checks as they are handled again later
-                await _proposed_change_transition_state(proposed_change=proposed_change, state=ProposedChangeState.OPEN)
+                await _proposed_change_transition_state(
+                    proposed_change=proposed_change, state=ProposedChangeState.OPEN, service=service
+                )
                 return Failed(message="Unable to merge proposed change containing failing checks")
             if validator_kind == InfrahubKind.DATAVALIDATOR:
                 data_checks = await validation.checks.get_peers(db=db, peer_type=CoreDataCheck)
                 for check in data_checks.values():
                     if check.conflicts.value and not check.keep_branch.value:
                         await _proposed_change_transition_state(
-                            proposed_change=proposed_change, state=ProposedChangeState.OPEN
+                            proposed_change=proposed_change, state=ProposedChangeState.OPEN, service=service
                         )
                         return Failed(
                             message="Data conflicts found on branch and missing decisions about what branch to keep"
@@ -133,15 +147,19 @@ async def merge_proposed_change(proposed_change_id: str, proposed_change_name: s
 
         log.info("Proposed change is eligible to be merged")
         try:
-            await merge_branch(branch=source_branch.name)
+            await merge_branch(branch=source_branch.name, context=context, service=service)
         except MergeFailedError as exc:
-            await _proposed_change_transition_state(proposed_change=proposed_change, state=ProposedChangeState.OPEN)
+            await _proposed_change_transition_state(
+                proposed_change=proposed_change, state=ProposedChangeState.OPEN, service=service
+            )
             return Failed(message=f"Merge failure when trying to merge {exc.message}")
 
         log.info(f"Branch {source_branch.name} has been merged successfully")
 
-        await _proposed_change_transition_state(proposed_change=proposed_change, state=ProposedChangeState.MERGED)
-        await service.workflow.submit_workflow(workflow=COMPUTED_ATTRIBUTE_SETUP_PYTHON)
+        await _proposed_change_transition_state(
+            proposed_change=proposed_change, state=ProposedChangeState.MERGED, service=service
+        )
+        await service.workflow.submit_workflow(workflow=COMPUTED_ATTRIBUTE_SETUP_PYTHON, context=context)
         return Completed(message="proposed change merged successfully")
 
 
@@ -150,9 +168,7 @@ async def merge_proposed_change(proposed_change_id: str, proposed_change_name: s
     flow_run_name="Cancel all proposed change associated with branch {branch_name}",
     description="Cancel all Proposed change associated with a branch.",
 )
-async def cancel_proposed_changes_branch(branch_name: str) -> None:
-    service = services.service
-
+async def cancel_proposed_changes_branch(branch_name: str, service: InfrahubServices) -> None:
     await add_tags(branches=[branch_name])
 
     proposed_changed_opened = await service.client.filters(
@@ -169,13 +185,11 @@ async def cancel_proposed_changes_branch(branch_name: str) -> None:
     )
 
     for proposed_change in proposed_changed_opened + proposed_changed_closed:
-        await cancel_proposed_change(proposed_change=proposed_change)
-
+        await cancel_proposed_change(proposed_change=proposed_change, service=service)
 
-@task(name="Cancel a propose change", description="Cancel a propose change", cache_policy=NONE)
-async def cancel_proposed_change(proposed_change: CoreProposedChange) -> None:
-    service = services.service
 
+@task(name="Cancel a propose change", description="Cancel a propose change", cache_policy=NONE)  # type: ignore[arg-type]
+async def cancel_proposed_change(proposed_change: CoreProposedChange, service: InfrahubServices) -> None:
     await add_tags(nodes=[proposed_change.id])
     log = get_run_logger()
 
@@ -189,11 +203,12 @@ async def cancel_proposed_change(proposed_change: CoreProposedChange) -> None:
     name="proposed-changed-data-integrity",
     flow_run_name="Triggers data integrity check",
 )
-async def run_proposed_change_data_integrity_check(model: RequestProposedChangeDataIntegrity) -> None:
+async def run_proposed_change_data_integrity_check(
+    model: RequestProposedChangeDataIntegrity, service: InfrahubServices
+) -> None:
     """Triggers a data integrity validation check on the provided proposed change to start."""
     await add_tags(branches=[model.source_branch], nodes=[model.proposed_change])
 
-    service = services.service
     async with service.database.start_transaction() as dbt:
         destination_branch = await registry.get_branch(db=dbt, branch=model.destination_branch)
         source_branch = await registry.get_branch(db=dbt, branch=model.source_branch)
@@ -207,8 +222,9 @@ async def run_proposed_change_data_integrity_check(model: RequestProposedChangeD
     name="proposed-changed-run-generator",
     flow_run_name="Run generators",
 )
-async def run_generators(model: RequestProposedChangeRunGenerators) -> None:
-    service = services.service
+async def run_generators(
+    model: RequestProposedChangeRunGenerators, context: InfrahubContext, service: InfrahubServices
+) -> None:
     await add_tags(branches=[model.source_branch], nodes=[model.proposed_change], db_change=True)
 
     generators = await service.client.filters(
@@ -264,7 +280,7 @@ async def run_generators(model: RequestProposedChangeRunGenerators) -> None:
                 destination_branch=model.destination_branch,
             )
             msg.assign_meta(parent=model)
-            await service.send(message=msg)
+            await service.message_bus.send(message=msg)
 
     next_messages: list[InfrahubMessage] = []
     if model.refresh_artifacts:
@@ -287,12 +303,14 @@ async def run_generators(model: RequestProposedChangeRunGenerators) -> None:
             branch_diff=model.branch_diff,
         )
         await service.workflow.submit_workflow(
-            workflow=REQUEST_PROPOSED_CHANGE_REPOSITORY_CHECKS, parameters={"model": model_proposed_change_repo_checks}
+            workflow=REQUEST_PROPOSED_CHANGE_REPOSITORY_CHECKS,
+            context=context,
+            parameters={"model": model_proposed_change_repo_checks},
         )
 
     for next_msg in next_messages:
         next_msg.assign_meta(parent=model)
-        await service.send(message=next_msg)
+        await service.message_bus.send(message=next_msg)
 
 
 @flow(
@@ -300,11 +318,10 @@ async def run_generators(model: RequestProposedChangeRunGenerators) -> None:
     flow_run_name="Process schema integrity",
 )
 async def run_proposed_change_schema_integrity_check(
-    model: RequestProposedChangeSchemaIntegrity,
+    model: RequestProposedChangeSchemaIntegrity, service: InfrahubServices
 ) -> None:
     # For now, we retrieve the latest schema for each branch from the registry
     # In the future it would be good to generate the object SchemaUpdateValidationResult from message.branch_diff
-    service = services.service
     await add_tags(branches=[model.source_branch], nodes=[model.proposed_change])
 
     source_schema = registry.schema.get_schema_branch(name=model.source_branch).duplicate()
@@ -331,7 +348,8 @@ async def run_proposed_change_schema_integrity_check(
     responses = await schema_validate_migrations(
         message=SchemaValidateMigrationData(
             branch=source_branch, schema_branch=candidate_schema, constraints=list(constraints)
-        )
+        ),
+        service=service,
     )
 
     # TODO we need to report a failure if an error happened during the execution of a validator
@@ -393,8 +411,7 @@ async def _get_proposed_change_schema_integrity_constraints(
     name="proposed-changed-repository-checks",
     flow_run_name="Process user defined checks",
 )
-async def repository_checks(model: RequestProposedChangeRepositoryChecks) -> None:
-    service = services.service
+async def repository_checks(model: RequestProposedChangeRepositoryChecks, service: InfrahubServices) -> None:
     await add_tags(branches=[model.source_branch], nodes=[model.proposed_change])
 
     events: list[InfrahubMessage] = []
@@ -426,15 +443,14 @@ async def repository_checks(model: RequestProposedChangeRepositoryChecks) -> Non
         )
     for event in events:
         event.assign_meta(parent=model)
-        await service.send(message=event)
+        await service.message_bus.send(message=event)
 
 
 @flow(
     name="proposed-changed-user-tests",
     flow_run_name="Run unit tests in repositories",
 )
-async def run_proposed_change_user_tests(model: RequestProposedChangeUserTests) -> None:
-    service = services.service
+async def run_proposed_change_user_tests(model: RequestProposedChangeUserTests, service: InfrahubServices) -> None:
     log = get_run_logger()
     await add_tags(branches=[model.source_branch], nodes=[model.proposed_change])
     proposed_change = await service.client.get(kind=InfrahubKind.PROPOSEDCHANGE, id=model.proposed_change)
@@ -493,3 +509,125 @@ async def run_proposed_change_user_tests(model: RequestProposedChangeUserTests)
 
             return_code = await asyncio.to_thread(_execute, worktree_directory, repository, proposed_change)
             log.info(msg=f"repository_tests_completed return_code={return_code}")
+
+
+@flow(
+    name="artifacts-generation-validation",
+    flow_run_name="Validating generation of artifacts for {model.artifact_definition.definition_name}",
+)
+async def validate_artifacts_generation(model: RequestArtifactDefinitionCheck, service: InfrahubServices) -> None:
+    await add_tags(branches=[model.source_branch], nodes=[model.proposed_change], db_change=True)
+
+    log = get_run_logger()
+    artifact_definition = await service.client.get(
+        kind=InfrahubKind.ARTIFACTDEFINITION,
+        id=model.artifact_definition.definition_id,
+        branch=model.source_branch,
+    )
+    proposed_change = await service.client.get(kind=InfrahubKind.PROPOSEDCHANGE, id=model.proposed_change)
+
+    validator_name = f"Artifact Validator: {model.artifact_definition.definition_name}"
+
+    await proposed_change.validations.fetch()
+
+    validator = None
+    for relationship in proposed_change.validations.peers:
+        existing_validator = relationship.peer
+        if (
+            existing_validator.typename == InfrahubKind.ARTIFACTVALIDATOR
+            and existing_validator.definition.id == model.artifact_definition.definition_id
+        ):
+            validator = existing_validator
+
+    if validator:
+        validator.conclusion.value = ValidatorConclusion.UNKNOWN.value
+        validator.state.value = ValidatorState.QUEUED.value
+        validator.started_at.value = ""
+        validator.completed_at.value = ""
+        await validator.save()
+    else:
+        validator = await service.client.create(
+            kind=InfrahubKind.ARTIFACTVALIDATOR,
+            data={
+                "label": validator_name,
+                "proposed_change": model.proposed_change,
+                "definition": model.artifact_definition.definition_id,
+            },
+        )
+        await validator.save()
+
+    await artifact_definition.targets.fetch()
+    group = artifact_definition.targets.peer
+    await group.members.fetch()
+
+    existing_artifacts = await service.client.filters(
+        kind=InfrahubKind.ARTIFACT,
+        definition__ids=[model.artifact_definition.definition_id],
+        include=["object"],
+        branch=model.source_branch,
+    )
+    artifacts_by_member = {}
+    for artifact in existing_artifacts:
+        artifacts_by_member[artifact.object.peer.id] = artifact.id
+
+    repository = model.branch_diff.get_repository(repository_id=model.artifact_definition.repository_id)
+    impacted_artifacts = model.branch_diff.get_subscribers_ids(kind=InfrahubKind.ARTIFACT)
+
+    checks = []
+
+    for relationship in group.members.peers:
+        member = relationship.peer
+        artifact_id = artifacts_by_member.get(member.id)
+        if _should_render_artifact(
+            artifact_id=artifact_id,
+            managed_branch=model.source_branch_sync_with_git,
+            impacted_artifacts=impacted_artifacts,
+        ):
+            log.info(f"Trigger Artifact processing for {member.display_label}")
+
+            check_model = CheckArtifactCreate(
+                artifact_name=model.artifact_definition.artifact_name,
+                artifact_id=artifact_id,
+                artifact_definition=model.artifact_definition.definition_id,
+                commit=repository.source_commit,
+                content_type=model.artifact_definition.content_type,
+                transform_type=model.artifact_definition.transform_kind,
+                transform_location=model.artifact_definition.transform_location,
+                repository_id=repository.repository_id,
+                repository_name=repository.repository_name,
+                repository_kind=repository.kind,
+                branch_name=model.source_branch,
+                query=model.artifact_definition.query_name,
+                variables=member.extract(params=artifact_definition.parameters.value),
+                target_id=member.id,
+                target_name=member.display_label,
+                timeout=model.artifact_definition.timeout,
+                validator_id=validator.id,
+            )
+
+            checks.append(
+                service.workflow.execute_workflow(
+                    workflow=GIT_REPOSITORIES_CHECK_ARTIFACT_CREATE,
+                    parameters={"model": check_model},
+                    expected_return=ValidatorConclusion,
+                )
+            )
+
+    await run_checks_and_update_validator(checks, validator)
+
+
+def _should_render_artifact(artifact_id: str | None, managed_branch: bool, impacted_artifacts: list[str]) -> bool:  # noqa: ARG001
+    """Returns a boolean to indicate if an artifact should be generated or not.
+    Will return true if:
+        * The artifact_id wasn't set which could be that it's a new object that doesn't have a previous artifact
+        * The source brance is not data only which would indicate that it could contain updates in git to the transform
+        * The artifact_id exists in the impacted_artifacts list
+    Will return false if:
+        * The source branch is a data only branch and the artifact_id exists and is not in the impacted list
+    """
+
+    # if not artifact_id or managed_branch:
+    #    return True
+    # return artifact_id in impacted_artifacts
+    # Temporary workaround tracked in https://github.com/opsmill/infrahub/issues/4991
+    return True

infrahub/pytest_plugin.py

@@ -52,7 +52,7 @@ class InfrahubBackendPlugin:
 
         return validator, True
 
-    def pytest_collection_modifyitems(self, session: Session, config: Config, items: list[Item]) -> None:  # pylint: disable=unused-argument
+    def pytest_collection_modifyitems(self, session: Session, config: Config, items: list[Item]) -> None:  # noqa: ARG002
         """This function is called after item collection and gives the opportunity to work on the collection before sending the items for testing.
 
         All items without an "infrahub" marker will be discarded. Items will also be re-ordered to be run in a specific order:
@@ -82,7 +82,7 @@ class InfrahubBackendPlugin:
         filtered_items.sort(key=sort_key)
         items[:] = filtered_items
 
-    def pytest_collection_finish(self, session: Session) -> None:  # pylint: disable=unused-argument
+    def pytest_collection_finish(self, session: Session) -> None:  # noqa: ARG002
         """This function is called when tests have been collected and modified, meaning they are ready to be run."""
         self.proposed_change = self.client.get(kind=InfrahubKind.PROPOSEDCHANGE, id=self.proposed_change_id)
         self.proposed_change.validations.fetch()
@@ -95,7 +95,7 @@ class InfrahubBackendPlugin:
                 check = relationship.peer
                 self.checks[check.origin.value] = check
 
-    def pytest_runtestloop(self, session: Session) -> Optional[object]:  # pylint: disable=unused-argument,useless-return
+    def pytest_runtestloop(self, session: Session) -> Optional[object]:  # noqa: ARG002
         """This function is called when the test loop is being run."""
         self.validator.conclusion.value = "unknown"
         self.validator.state.value = "in_progress"
@@ -144,7 +144,7 @@ class InfrahubBackendPlugin:
         # Workaround for https://github.com/opsmill/infrahub/issues/2184
         check.update(do_full_update=True)
 
-    def pytest_sessionfinish(self, session: Session) -> None:  # pylint: disable=unused-argument
+    def pytest_sessionfinish(self, session: Session) -> None:  # noqa: ARG002
         """Set the final RepositoryValidator details after completing the test session."""
         conclusion = "success"
 

infrahub/server.py

@@ -33,7 +33,7 @@ from infrahub.graphql.api.endpoints import router as graphql_router
 from infrahub.lock import initialize_lock
 from infrahub.log import clear_log_context, get_logger, set_log_data
 from infrahub.middleware import InfrahubCORSMiddleware
-from infrahub.services import InfrahubServices, services
+from infrahub.services import InfrahubServices
 from infrahub.services.adapters.cache.nats import NATSCache
 from infrahub.services.adapters.cache.redis import RedisCache
 from infrahub.services.adapters.message_bus.nats import NATSMessageBus
@@ -70,26 +70,27 @@ async def app_initialization(application: FastAPI, enable_scheduler: bool = True
         if config.SETTINGS.workflow.driver == config.WorkflowDriver.WORKER
         else WorkflowLocalExecution()
     )
-
+    component_type = ComponentType.API_SERVER
     message_bus = config.OVERRIDE.message_bus or (
-        NATSMessageBus() if config.SETTINGS.broker.driver == config.BrokerDriver.NATS else RabbitMQMessageBus()
+        await NATSMessageBus.new(component_type=component_type)
+        if config.SETTINGS.broker.driver == config.BrokerDriver.NATS
+        else await RabbitMQMessageBus.new(component_type=component_type)
     )
     cache = config.OVERRIDE.cache or (
-        NATSCache() if config.SETTINGS.cache.driver == config.CacheDriver.NATS else RedisCache()
+        await NATSCache.new() if config.SETTINGS.cache.driver == config.CacheDriver.NATS else RedisCache()
     )
-    service = InfrahubServices(
+    service = await InfrahubServices.new(
         cache=cache,
         database=database,
         message_bus=message_bus,
         workflow=workflow,
-        component_type=ComponentType.API_SERVER,
+        component_type=component_type,
     )
-    await service.initialize()
     initialize_lock(service=service)
     # We must initialize DB after initialize lock and initialize lock depends on cache initialization
     async with application.state.db.start_session() as db:
         await initialization(db=db)
-    services.prepare(service=service)
+
     application.state.service = service
     application.state.response_delay = config.SETTINGS.miscellaneous.response_delay
     if enable_scheduler:
@@ -97,7 +98,7 @@ async def app_initialization(application: FastAPI, enable_scheduler: bool = True
 
 
 async def shutdown(application: FastAPI) -> None:
-    await services.service.shutdown()
+    await application.state.service.shutdown()
     await application.state.db.close()
 
 
@@ -118,7 +119,7 @@ app = FastAPI(
 )
 
 
-def server_request_hook(span: Span, scope: dict) -> None:  # pylint: disable=unused-argument
+def server_request_hook(span: Span, scope: dict) -> None:  # noqa: ARG001
     if span and span.is_recording():
         span.set_attribute("worker", WORKER_IDENTITY)
 
@@ -217,5 +218,5 @@ async def documentation() -> RedirectResponse:
 
 
 @app.get("/{rest_of_path:path}", include_in_schema=False)
-async def react_app(req: Request, rest_of_path: str) -> Response:  # pylint: disable=unused-argument
+async def react_app(req: Request, rest_of_path: str) -> Response:  # noqa: ARG001
     return templates.TemplateResponse("index.html", {"request": req})