infrahub-server 1.2.11__py3-none-any.whl → 1.3.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- infrahub/actions/constants.py +86 -0
- infrahub/actions/gather.py +114 -0
- infrahub/actions/models.py +241 -0
- infrahub/actions/parsers.py +104 -0
- infrahub/actions/schema.py +382 -0
- infrahub/actions/tasks.py +126 -0
- infrahub/actions/triggers.py +21 -0
- infrahub/cli/db.py +1 -2
- infrahub/core/account.py +24 -47
- infrahub/core/attribute.py +13 -15
- infrahub/core/constants/__init__.py +5 -0
- infrahub/core/constants/infrahubkind.py +9 -0
- infrahub/core/convert_object_type/__init__.py +0 -0
- infrahub/core/convert_object_type/conversion.py +122 -0
- infrahub/core/convert_object_type/schema_mapping.py +56 -0
- infrahub/core/diff/query/all_conflicts.py +1 -5
- infrahub/core/diff/query/artifact.py +10 -20
- infrahub/core/diff/query/diff_get.py +3 -6
- infrahub/core/diff/query/field_summary.py +2 -4
- infrahub/core/diff/query/merge.py +70 -123
- infrahub/core/diff/query/save.py +20 -32
- infrahub/core/diff/query/summary_counts_enricher.py +34 -54
- infrahub/core/manager.py +14 -11
- infrahub/core/migrations/graph/m003_relationship_parent_optional.py +1 -2
- infrahub/core/migrations/graph/m013_convert_git_password_credential.py +2 -4
- infrahub/core/migrations/graph/m019_restore_rels_to_time.py +11 -22
- infrahub/core/migrations/graph/m020_duplicate_edges.py +3 -6
- infrahub/core/migrations/graph/m021_missing_hierarchy_merge.py +1 -2
- infrahub/core/migrations/graph/m024_missing_hierarchy_backfill.py +1 -2
- infrahub/core/migrations/query/attribute_add.py +1 -2
- infrahub/core/migrations/query/attribute_rename.py +5 -10
- infrahub/core/migrations/query/delete_element_in_schema.py +19 -17
- infrahub/core/migrations/query/node_duplicate.py +19 -21
- infrahub/core/migrations/query/relationship_duplicate.py +19 -17
- infrahub/core/migrations/schema/node_attribute_remove.py +4 -8
- infrahub/core/migrations/schema/node_remove.py +19 -19
- infrahub/core/models.py +29 -2
- infrahub/core/node/__init__.py +90 -18
- infrahub/core/node/create.py +211 -0
- infrahub/core/node/resource_manager/number_pool.py +31 -5
- infrahub/core/node/standard.py +6 -1
- infrahub/core/protocols.py +56 -0
- infrahub/core/protocols_base.py +3 -0
- infrahub/core/query/__init__.py +2 -2
- infrahub/core/query/diff.py +19 -32
- infrahub/core/query/ipam.py +10 -20
- infrahub/core/query/node.py +28 -46
- infrahub/core/query/relationship.py +53 -32
- infrahub/core/query/resource_manager.py +1 -2
- infrahub/core/query/subquery.py +2 -4
- infrahub/core/relationship/model.py +3 -0
- infrahub/core/schema/__init__.py +2 -1
- infrahub/core/schema/attribute_parameters.py +160 -0
- infrahub/core/schema/attribute_schema.py +111 -8
- infrahub/core/schema/basenode_schema.py +25 -1
- infrahub/core/schema/definitions/core/__init__.py +29 -1
- infrahub/core/schema/definitions/core/group.py +45 -0
- infrahub/core/schema/definitions/internal.py +27 -4
- infrahub/core/schema/generated/attribute_schema.py +16 -3
- infrahub/core/schema/manager.py +3 -0
- infrahub/core/schema/schema_branch.py +67 -7
- infrahub/core/validators/__init__.py +13 -1
- infrahub/core/validators/attribute/choices.py +1 -3
- infrahub/core/validators/attribute/enum.py +1 -3
- infrahub/core/validators/attribute/kind.py +1 -3
- infrahub/core/validators/attribute/length.py +13 -7
- infrahub/core/validators/attribute/min_max.py +118 -0
- infrahub/core/validators/attribute/number_pool.py +106 -0
- infrahub/core/validators/attribute/optional.py +1 -4
- infrahub/core/validators/attribute/regex.py +5 -6
- infrahub/core/validators/attribute/unique.py +1 -3
- infrahub/core/validators/determiner.py +18 -2
- infrahub/core/validators/enum.py +12 -0
- infrahub/core/validators/node/hierarchy.py +3 -6
- infrahub/core/validators/query.py +1 -3
- infrahub/core/validators/relationship/count.py +6 -12
- infrahub/core/validators/relationship/optional.py +2 -4
- infrahub/core/validators/relationship/peer.py +3 -8
- infrahub/core/validators/uniqueness/query.py +5 -9
- infrahub/database/__init__.py +11 -2
- infrahub/events/group_action.py +1 -0
- infrahub/git/base.py +5 -3
- infrahub/git/integrator.py +102 -3
- infrahub/graphql/analyzer.py +139 -18
- infrahub/graphql/manager.py +4 -0
- infrahub/graphql/mutations/action.py +164 -0
- infrahub/graphql/mutations/convert_object_type.py +62 -0
- infrahub/graphql/mutations/main.py +24 -175
- infrahub/graphql/mutations/proposed_change.py +20 -17
- infrahub/graphql/mutations/resource_manager.py +62 -6
- infrahub/graphql/queries/convert_object_type_mapping.py +36 -0
- infrahub/graphql/queries/resource_manager.py +7 -1
- infrahub/graphql/schema.py +6 -0
- infrahub/menu/menu.py +31 -0
- infrahub/message_bus/messages/__init__.py +0 -10
- infrahub/message_bus/operations/__init__.py +0 -8
- infrahub/patch/queries/consolidate_duplicated_nodes.py +3 -6
- infrahub/patch/queries/delete_duplicated_edges.py +5 -10
- infrahub/pools/number.py +5 -3
- infrahub/prefect_server/models.py +1 -19
- infrahub/proposed_change/models.py +68 -3
- infrahub/proposed_change/tasks.py +907 -30
- infrahub/task_manager/models.py +10 -6
- infrahub/trigger/catalogue.py +2 -0
- infrahub/trigger/models.py +18 -2
- infrahub/trigger/tasks.py +3 -1
- infrahub/types.py +6 -0
- infrahub/workflows/catalogue.py +76 -0
- infrahub_sdk/client.py +43 -10
- infrahub_sdk/node/__init__.py +39 -0
- infrahub_sdk/node/attribute.py +122 -0
- infrahub_sdk/node/constants.py +21 -0
- infrahub_sdk/{node.py → node/node.py} +50 -749
- infrahub_sdk/node/parsers.py +15 -0
- infrahub_sdk/node/property.py +24 -0
- infrahub_sdk/node/related_node.py +266 -0
- infrahub_sdk/node/relationship.py +302 -0
- infrahub_sdk/protocols.py +112 -0
- infrahub_sdk/protocols_base.py +34 -2
- infrahub_sdk/query_groups.py +13 -2
- infrahub_sdk/schema/main.py +1 -0
- infrahub_sdk/schema/repository.py +16 -0
- infrahub_sdk/spec/object.py +1 -1
- infrahub_sdk/store.py +1 -1
- infrahub_sdk/testing/schemas/car_person.py +1 -0
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0b1.dist-info}/METADATA +4 -4
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0b1.dist-info}/RECORD +134 -122
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0b1.dist-info}/WHEEL +1 -1
- infrahub_testcontainers/container.py +0 -1
- infrahub_testcontainers/docker-compose.test.yml +1 -1
- infrahub_testcontainers/helpers.py +8 -2
- infrahub/message_bus/messages/check_generator_run.py +0 -26
- infrahub/message_bus/messages/finalize_validator_execution.py +0 -15
- infrahub/message_bus/messages/proposed_change/base_with_diff.py +0 -16
- infrahub/message_bus/messages/proposed_change/request_proposedchange_refreshartifacts.py +0 -11
- infrahub/message_bus/messages/request_generatordefinition_check.py +0 -20
- infrahub/message_bus/messages/request_proposedchange_pipeline.py +0 -23
- infrahub/message_bus/operations/check/__init__.py +0 -3
- infrahub/message_bus/operations/check/generator.py +0 -156
- infrahub/message_bus/operations/finalize/__init__.py +0 -3
- infrahub/message_bus/operations/finalize/validator.py +0 -133
- infrahub/message_bus/operations/requests/__init__.py +0 -9
- infrahub/message_bus/operations/requests/generator_definition.py +0 -140
- infrahub/message_bus/operations/requests/proposed_change.py +0 -629
- /infrahub/{message_bus/messages/proposed_change → actions}/__init__.py +0 -0
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0b1.dist-info}/LICENSE.txt +0 -0
- {infrahub_server-1.2.11.dist-info → infrahub_server-1.3.0b1.dist-info}/entry_points.txt +0 -0
|
@@ -28,7 +28,6 @@ INFRAHUB_SERVICES: dict[str, ContainerService] = {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
PROJECT_ENV_VARIABLES: dict[str, str] = {
|
|
31
|
-
"NEO4J_DOCKER_IMAGE": "neo4j:5.20.0-community",
|
|
32
31
|
"MESSAGE_QUEUE_DOCKER_IMAGE": "rabbitmq:3.13.7-management",
|
|
33
32
|
"CACHE_DOCKER_IMAGE": "redis:7.2.4",
|
|
34
33
|
"INFRAHUB_TESTING_DOCKER_IMAGE": "registry.opsmill.io/opsmill/infrahub",
|
|
@@ -50,7 +50,7 @@ services:
|
|
|
50
50
|
limits:
|
|
51
51
|
cpus: ${INFRAHUB_TESTING_DB_CPU_LIMIT:-0.0}
|
|
52
52
|
memory: ${INFRAHUB_TESTING_DB_MEMORY_LIMIT:-0}
|
|
53
|
-
image: ${NEO4J_DOCKER_IMAGE:-neo4j:
|
|
53
|
+
image: ${NEO4J_DOCKER_IMAGE:-neo4j:2025.03.0-community}
|
|
54
54
|
restart: unless-stopped
|
|
55
55
|
environment:
|
|
56
56
|
NEO4J_AUTH: neo4j/admin
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import subprocess # noqa: S404
|
|
3
|
+
import uuid
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import pytest
|
|
7
|
+
from prefect.client.orchestration import PrefectClient
|
|
6
8
|
|
|
7
9
|
from infrahub_testcontainers import __version__ as infrahub_version
|
|
8
10
|
|
|
@@ -38,8 +40,8 @@ class TestInfrahubDocker:
|
|
|
38
40
|
|
|
39
41
|
@pytest.fixture(scope="class")
|
|
40
42
|
def tmp_directory(self, tmpdir_factory: pytest.TempdirFactory) -> Path:
|
|
41
|
-
|
|
42
|
-
return
|
|
43
|
+
name = f"{self.__class__.__name__.lower()}_{uuid.uuid4().hex}"
|
|
44
|
+
return Path(str(tmpdir_factory.mktemp(name)))
|
|
43
45
|
|
|
44
46
|
@pytest.fixture(scope="class")
|
|
45
47
|
def remote_repos_dir(self, tmp_directory: Path) -> Path:
|
|
@@ -91,3 +93,7 @@ class TestInfrahubDocker:
|
|
|
91
93
|
@pytest.fixture(scope="class")
|
|
92
94
|
def task_manager_port(self, infrahub_app: dict[str, int]) -> int:
|
|
93
95
|
return infrahub_app["task-manager"]
|
|
96
|
+
|
|
97
|
+
@pytest.fixture(scope="class")
|
|
98
|
+
def prefect_client(self, task_manager_port: int) -> PrefectClient:
|
|
99
|
+
return PrefectClient(api=f"http://localhost:{task_manager_port}/api/")
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
from pydantic import Field
|
|
2
|
-
|
|
3
|
-
from infrahub.context import InfrahubContext
|
|
4
|
-
from infrahub.generators.models import ProposedChangeGeneratorDefinition
|
|
5
|
-
from infrahub.message_bus import InfrahubMessage
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class CheckGeneratorRun(InfrahubMessage):
|
|
9
|
-
"""A check that runs a generator."""
|
|
10
|
-
|
|
11
|
-
generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator definition")
|
|
12
|
-
generator_instance: str | None = Field(
|
|
13
|
-
default=None, description="The id of the generator instance if it previously existed"
|
|
14
|
-
)
|
|
15
|
-
commit: str = Field(..., description="The commit to target")
|
|
16
|
-
repository_id: str = Field(..., description="The unique ID of the Repository")
|
|
17
|
-
repository_name: str = Field(..., description="The name of the Repository")
|
|
18
|
-
repository_kind: str = Field(..., description="The kind of the Repository")
|
|
19
|
-
branch_name: str = Field(..., description="The branch where the check is run")
|
|
20
|
-
target_id: str = Field(..., description="The ID of the target object for this generator")
|
|
21
|
-
target_name: str = Field(..., description="Name of the generator target")
|
|
22
|
-
query: str = Field(..., description="The name of the query to use when collecting data")
|
|
23
|
-
variables: dict = Field(..., description="Input variables when running the generator")
|
|
24
|
-
validator_id: str = Field(..., description="The ID of the validator")
|
|
25
|
-
proposed_change: str | None = Field(None, description="The unique ID of the Proposed Change")
|
|
26
|
-
context: InfrahubContext = Field(..., description="The Infrahub context")
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
from pydantic import Field
|
|
2
|
-
|
|
3
|
-
from infrahub.context import InfrahubContext
|
|
4
|
-
from infrahub.message_bus import InfrahubMessage
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class FinalizeValidatorExecution(InfrahubMessage):
|
|
8
|
-
"""Update the status of a validator after all checks have been completed."""
|
|
9
|
-
|
|
10
|
-
validator_id: str = Field(..., description="The id of the validator associated with this check")
|
|
11
|
-
validator_execution_id: str = Field(..., description="The id of current execution of the associated validator")
|
|
12
|
-
start_time: str = Field(..., description="Start time when the message was first created")
|
|
13
|
-
validator_type: str = Field(..., description="The type of validator to complete")
|
|
14
|
-
context: InfrahubContext = Field(..., description="The Infrahub context")
|
|
15
|
-
proposed_change: str = Field(..., description="The ID of the proposed change")
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
from pydantic import ConfigDict, Field
|
|
2
|
-
|
|
3
|
-
from infrahub.message_bus import InfrahubMessage
|
|
4
|
-
from infrahub.message_bus.types import ProposedChangeBranchDiff
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class BaseProposedChangeWithDiffMessage(InfrahubMessage):
|
|
8
|
-
"""Sent trigger the refresh of artifacts that are impacted by the proposed change."""
|
|
9
|
-
|
|
10
|
-
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
11
|
-
|
|
12
|
-
proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
|
|
13
|
-
source_branch: str = Field(..., description="The source branch of the proposed change")
|
|
14
|
-
source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
|
|
15
|
-
destination_branch: str = Field(..., description="The destination branch of the proposed change")
|
|
16
|
-
branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
from pydantic import Field
|
|
2
|
-
|
|
3
|
-
from infrahub.context import InfrahubContext
|
|
4
|
-
|
|
5
|
-
from .base_with_diff import BaseProposedChangeWithDiffMessage
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class RequestProposedChangeRefreshArtifacts(BaseProposedChangeWithDiffMessage):
|
|
9
|
-
"""Sent trigger the refresh of artifacts that are impacted by the proposed change."""
|
|
10
|
-
|
|
11
|
-
context: InfrahubContext = Field(..., description="The context of the task")
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
from pydantic import ConfigDict, Field
|
|
2
|
-
|
|
3
|
-
from infrahub.context import InfrahubContext
|
|
4
|
-
from infrahub.generators.models import ProposedChangeGeneratorDefinition
|
|
5
|
-
from infrahub.message_bus import InfrahubMessage
|
|
6
|
-
from infrahub.message_bus.types import ProposedChangeBranchDiff
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class RequestGeneratorDefinitionCheck(InfrahubMessage):
|
|
10
|
-
"""Sent to trigger Generators to run for a proposed change."""
|
|
11
|
-
|
|
12
|
-
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
13
|
-
|
|
14
|
-
generator_definition: ProposedChangeGeneratorDefinition = Field(..., description="The Generator Definition")
|
|
15
|
-
branch_diff: ProposedChangeBranchDiff = Field(..., description="The calculated diff between the two branches")
|
|
16
|
-
proposed_change: str = Field(..., description="The unique ID of the Proposed Change")
|
|
17
|
-
source_branch: str = Field(..., description="The source branch")
|
|
18
|
-
source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
|
|
19
|
-
destination_branch: str = Field(..., description="The target branch")
|
|
20
|
-
context: InfrahubContext = Field(..., description="The Infrahub context")
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import uuid
|
|
2
|
-
|
|
3
|
-
from pydantic import Field
|
|
4
|
-
|
|
5
|
-
from infrahub.context import InfrahubContext
|
|
6
|
-
from infrahub.core.constants import CheckType
|
|
7
|
-
from infrahub.message_bus import InfrahubMessage
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class RequestProposedChangePipeline(InfrahubMessage):
|
|
11
|
-
"""Sent request the start of a pipeline connected to a proposed change."""
|
|
12
|
-
|
|
13
|
-
proposed_change: str = Field(..., description="The unique ID of the proposed change")
|
|
14
|
-
source_branch: str = Field(..., description="The source branch of the proposed change")
|
|
15
|
-
source_branch_sync_with_git: bool = Field(..., description="Indicates if the source branch should sync with git")
|
|
16
|
-
destination_branch: str = Field(..., description="The destination branch of the proposed change")
|
|
17
|
-
check_type: CheckType = Field(
|
|
18
|
-
default=CheckType.ALL, description="Can be used to restrict the pipeline to a specific type of job"
|
|
19
|
-
)
|
|
20
|
-
context: InfrahubContext = Field(..., description="The context of the task")
|
|
21
|
-
pipeline_id: uuid.UUID = Field(
|
|
22
|
-
default_factory=uuid.uuid4, description="The unique ID of the execution of this pipeline"
|
|
23
|
-
)
|
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
from infrahub_sdk.exceptions import ModuleImportError
|
|
2
|
-
from infrahub_sdk.node import InfrahubNode
|
|
3
|
-
from infrahub_sdk.schema.repository import InfrahubGeneratorDefinitionConfig
|
|
4
|
-
from prefect import flow
|
|
5
|
-
from prefect.logging import get_run_logger
|
|
6
|
-
|
|
7
|
-
from infrahub import lock
|
|
8
|
-
from infrahub.core.constants import GeneratorInstanceStatus, InfrahubKind, ValidatorConclusion
|
|
9
|
-
from infrahub.core.timestamp import Timestamp
|
|
10
|
-
from infrahub.git.base import extract_repo_file_information
|
|
11
|
-
from infrahub.git.repository import get_initialized_repo
|
|
12
|
-
from infrahub.message_bus import messages
|
|
13
|
-
from infrahub.services import InfrahubServices
|
|
14
|
-
from infrahub.tasks.check import set_check_status
|
|
15
|
-
from infrahub.workflows.utils import add_tags
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@flow(
|
|
19
|
-
name="git-repository-check-generator-run",
|
|
20
|
-
flow_run_name="Execute Generator {message.generator_definition.definition_name} for {message.target_name}",
|
|
21
|
-
)
|
|
22
|
-
async def run(message: messages.CheckGeneratorRun, service: InfrahubServices) -> None:
|
|
23
|
-
if message.proposed_change:
|
|
24
|
-
await add_tags(branches=[message.branch_name], nodes=[message.proposed_change], db_change=True)
|
|
25
|
-
else:
|
|
26
|
-
await add_tags(branches=[message.branch_name], nodes=[message.repository_id], db_change=True)
|
|
27
|
-
|
|
28
|
-
log = get_run_logger()
|
|
29
|
-
|
|
30
|
-
repository = await get_initialized_repo(
|
|
31
|
-
repository_id=message.repository_id,
|
|
32
|
-
name=message.repository_name,
|
|
33
|
-
service=service,
|
|
34
|
-
repository_kind=message.repository_kind,
|
|
35
|
-
commit=message.commit,
|
|
36
|
-
)
|
|
37
|
-
|
|
38
|
-
conclusion = ValidatorConclusion.SUCCESS
|
|
39
|
-
|
|
40
|
-
generator_definition = InfrahubGeneratorDefinitionConfig(
|
|
41
|
-
name=message.generator_definition.definition_name,
|
|
42
|
-
class_name=message.generator_definition.class_name,
|
|
43
|
-
file_path=message.generator_definition.file_path,
|
|
44
|
-
query=message.generator_definition.query_name,
|
|
45
|
-
targets=message.generator_definition.group_id,
|
|
46
|
-
convert_query_response=message.generator_definition.convert_query_response,
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
commit_worktree = repository.get_commit_worktree(commit=message.commit)
|
|
50
|
-
|
|
51
|
-
file_info = extract_repo_file_information(
|
|
52
|
-
full_filename=commit_worktree.directory / generator_definition.file_path,
|
|
53
|
-
repo_directory=repository.directory_root,
|
|
54
|
-
worktree_directory=commit_worktree.directory,
|
|
55
|
-
)
|
|
56
|
-
generator_instance = await _define_instance(message=message, service=service)
|
|
57
|
-
|
|
58
|
-
check_message = "Instance successfully generated"
|
|
59
|
-
try:
|
|
60
|
-
log.debug(f"repo information {file_info}")
|
|
61
|
-
log.debug(f"Root directory : {repository.directory_root}")
|
|
62
|
-
generator_class = generator_definition.load_class(
|
|
63
|
-
import_root=repository.directory_root, relative_path=file_info.relative_repo_path_dir
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
generator = generator_class(
|
|
67
|
-
query=generator_definition.query,
|
|
68
|
-
client=service.client,
|
|
69
|
-
branch=message.branch_name,
|
|
70
|
-
params=message.variables,
|
|
71
|
-
generator_instance=generator_instance.id,
|
|
72
|
-
convert_query_response=generator_definition.convert_query_response,
|
|
73
|
-
infrahub_node=InfrahubNode,
|
|
74
|
-
)
|
|
75
|
-
generator._init_client.request_context = message.context.to_request_context()
|
|
76
|
-
await generator.run(identifier=generator_definition.name)
|
|
77
|
-
generator_instance.status.value = GeneratorInstanceStatus.READY.value
|
|
78
|
-
except ModuleImportError as exc:
|
|
79
|
-
conclusion = ValidatorConclusion.FAILURE
|
|
80
|
-
generator_instance.status.value = GeneratorInstanceStatus.ERROR.value
|
|
81
|
-
check_message = f"Failed to import generator: {exc.message}"
|
|
82
|
-
log.exception(check_message, exc_info=exc)
|
|
83
|
-
except Exception as exc:
|
|
84
|
-
conclusion = ValidatorConclusion.FAILURE
|
|
85
|
-
generator_instance.status.value = GeneratorInstanceStatus.ERROR.value
|
|
86
|
-
check_message = f"Failed to execute generator: {str(exc)}"
|
|
87
|
-
log.exception(check_message, exc_info=exc)
|
|
88
|
-
|
|
89
|
-
log.info("Generator run completed, starting update")
|
|
90
|
-
await generator_instance.update(do_full_update=True)
|
|
91
|
-
|
|
92
|
-
check = None
|
|
93
|
-
existing_check = await service.client.filters(
|
|
94
|
-
kind=InfrahubKind.GENERATORCHECK, validator__ids=message.validator_id, instance__value=generator_instance.id
|
|
95
|
-
)
|
|
96
|
-
if existing_check:
|
|
97
|
-
check = existing_check[0]
|
|
98
|
-
|
|
99
|
-
if check:
|
|
100
|
-
check.created_at.value = Timestamp().to_string()
|
|
101
|
-
check.conclusion.value = conclusion.value
|
|
102
|
-
await check.save()
|
|
103
|
-
else:
|
|
104
|
-
check = await service.client.create(
|
|
105
|
-
kind=InfrahubKind.GENERATORCHECK,
|
|
106
|
-
data={
|
|
107
|
-
"name": message.target_name,
|
|
108
|
-
"origin": message.repository_id,
|
|
109
|
-
"kind": "GeneratorDefinition",
|
|
110
|
-
"validator": message.validator_id,
|
|
111
|
-
"created_at": Timestamp().to_string(),
|
|
112
|
-
"message": check_message,
|
|
113
|
-
"conclusion": conclusion.value,
|
|
114
|
-
"instance": generator_instance.id,
|
|
115
|
-
},
|
|
116
|
-
)
|
|
117
|
-
await check.save()
|
|
118
|
-
|
|
119
|
-
await set_check_status(message=message, conclusion=conclusion.value, service=service)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
async def _define_instance(message: messages.CheckGeneratorRun, service: InfrahubServices) -> InfrahubNode:
|
|
123
|
-
if message.generator_instance:
|
|
124
|
-
instance = await service.client.get(
|
|
125
|
-
kind=InfrahubKind.GENERATORINSTANCE, id=message.generator_instance, branch=message.branch_name
|
|
126
|
-
)
|
|
127
|
-
instance.status.value = GeneratorInstanceStatus.PENDING.value
|
|
128
|
-
await instance.update(do_full_update=True)
|
|
129
|
-
|
|
130
|
-
else:
|
|
131
|
-
async with lock.registry.get(
|
|
132
|
-
f"{message.target_id}-{message.generator_definition.definition_id}", namespace="generator"
|
|
133
|
-
):
|
|
134
|
-
instances = await service.client.filters(
|
|
135
|
-
kind=InfrahubKind.GENERATORINSTANCE,
|
|
136
|
-
definition__ids=[message.generator_definition.definition_id],
|
|
137
|
-
object__ids=[message.target_id],
|
|
138
|
-
branch=message.branch_name,
|
|
139
|
-
)
|
|
140
|
-
if instances:
|
|
141
|
-
instance = instances[0]
|
|
142
|
-
instance.status.value = GeneratorInstanceStatus.PENDING.value
|
|
143
|
-
await instance.update(do_full_update=True)
|
|
144
|
-
else:
|
|
145
|
-
instance = await service.client.create(
|
|
146
|
-
kind=InfrahubKind.GENERATORINSTANCE,
|
|
147
|
-
branch=message.branch_name,
|
|
148
|
-
data={
|
|
149
|
-
"name": f"{message.generator_definition.definition_name}: {message.target_name}",
|
|
150
|
-
"status": GeneratorInstanceStatus.PENDING.value,
|
|
151
|
-
"object": message.target_id,
|
|
152
|
-
"definition": message.generator_definition.definition_id,
|
|
153
|
-
},
|
|
154
|
-
)
|
|
155
|
-
await instance.save()
|
|
156
|
-
return instance
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
from infrahub_sdk.protocols import (
|
|
2
|
-
CoreArtifactValidator,
|
|
3
|
-
CoreDataValidator,
|
|
4
|
-
CoreGeneratorValidator,
|
|
5
|
-
CoreRepositoryValidator,
|
|
6
|
-
CoreSchemaValidator,
|
|
7
|
-
CoreUserValidator,
|
|
8
|
-
CoreValidator,
|
|
9
|
-
)
|
|
10
|
-
from prefect import flow
|
|
11
|
-
|
|
12
|
-
from infrahub import config
|
|
13
|
-
from infrahub.core.constants import InfrahubKind, ValidatorConclusion
|
|
14
|
-
from infrahub.core.timestamp import Timestamp
|
|
15
|
-
from infrahub.log import get_logger
|
|
16
|
-
from infrahub.message_bus import messages
|
|
17
|
-
from infrahub.message_bus.types import KVTTL, MessageTTL
|
|
18
|
-
from infrahub.services import InfrahubServices
|
|
19
|
-
from infrahub.validators.events import send_failed_validator, send_passed_validator
|
|
20
|
-
|
|
21
|
-
log = get_logger()
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
@flow(name="validator-finalize-execution")
|
|
25
|
-
async def execution(message: messages.FinalizeValidatorExecution, service: InfrahubServices) -> None:
|
|
26
|
-
"""Monitors the status of checks associated with a validator and finalizes the conclusion of the validator
|
|
27
|
-
|
|
28
|
-
Based on the unique execution_id this function looks expects to see an entry in the cache for each check
|
|
29
|
-
associated with this validator. Upon seeing the result of a check the function will exclude it from further
|
|
30
|
-
checks and update the current conclusion of the validator if any of the checks failed.
|
|
31
|
-
|
|
32
|
-
The message will get rescheduled until the timeout has exceeded or until all checks are accounted for.
|
|
33
|
-
"""
|
|
34
|
-
validator_type = get_validator_type(validator_type=message.validator_type)
|
|
35
|
-
validator = await service.client.get(kind=validator_type, id=message.validator_id)
|
|
36
|
-
checks_key = f"validator_execution_id:{message.validator_execution_id}:checks"
|
|
37
|
-
current_conclusion = validator.conclusion.value
|
|
38
|
-
if validator.state.value != "in_progress":
|
|
39
|
-
validator.state.value = "in_progress"
|
|
40
|
-
validator.started_at.value = Timestamp().to_string()
|
|
41
|
-
validator.completed_at.value = ""
|
|
42
|
-
await validator.save()
|
|
43
|
-
|
|
44
|
-
required_checks_data = await service.cache.get(key=checks_key) or ""
|
|
45
|
-
# Remove instances of empty checks
|
|
46
|
-
required_checks = [required_check for required_check in required_checks_data.split(",") if required_check]
|
|
47
|
-
|
|
48
|
-
completed_checks_data = await service.cache.list_keys(
|
|
49
|
-
filter_pattern=f"validator_execution_id:{message.validator_execution_id}:check_execution_id:*"
|
|
50
|
-
)
|
|
51
|
-
completed_checks = [check.split(":")[-1] for check in completed_checks_data]
|
|
52
|
-
|
|
53
|
-
missing_checks = [check for check in required_checks if check not in completed_checks]
|
|
54
|
-
checks_to_verify = [check for check in completed_checks if check in required_checks]
|
|
55
|
-
failed_check = False
|
|
56
|
-
|
|
57
|
-
for check in checks_to_verify:
|
|
58
|
-
conclusion = await service.cache.get(
|
|
59
|
-
f"validator_execution_id:{message.validator_execution_id}:check_execution_id:{check}"
|
|
60
|
-
)
|
|
61
|
-
if conclusion != "success":
|
|
62
|
-
failed_check = True
|
|
63
|
-
|
|
64
|
-
conclusion = "failure" if failed_check else "success"
|
|
65
|
-
if failed_check and current_conclusion != "failure":
|
|
66
|
-
validator.conclusion.value = "failure"
|
|
67
|
-
await validator.save()
|
|
68
|
-
|
|
69
|
-
if missing_checks:
|
|
70
|
-
remaining_checks = ",".join(missing_checks)
|
|
71
|
-
await service.cache.set(key=checks_key, value=remaining_checks, expires=KVTTL.TWO_HOURS)
|
|
72
|
-
current_time = Timestamp()
|
|
73
|
-
starting_time = Timestamp(message.start_time)
|
|
74
|
-
deadline = starting_time.add_delta(seconds=config.SETTINGS.miscellaneous.maximum_validator_execution_time)
|
|
75
|
-
if current_time < deadline:
|
|
76
|
-
log.debug(
|
|
77
|
-
"Still waiting for checks to complete",
|
|
78
|
-
missing_checks=missing_checks,
|
|
79
|
-
validator_id=message.validator_id,
|
|
80
|
-
validator_execution_id=message.validator_execution_id,
|
|
81
|
-
)
|
|
82
|
-
await service.message_bus.send(message=message, delay=MessageTTL.FIVE)
|
|
83
|
-
return
|
|
84
|
-
|
|
85
|
-
log.info(
|
|
86
|
-
"Timeout reached",
|
|
87
|
-
validator_id=message.validator_id,
|
|
88
|
-
validator_execution_id=message.validator_execution_id,
|
|
89
|
-
)
|
|
90
|
-
conclusion = "failure"
|
|
91
|
-
|
|
92
|
-
validator.state.value = "completed"
|
|
93
|
-
validator.completed_at.value = Timestamp().to_string()
|
|
94
|
-
validator.conclusion.value = conclusion
|
|
95
|
-
await validator.save()
|
|
96
|
-
if validator.conclusion.value == ValidatorConclusion.SUCCESS.value:
|
|
97
|
-
await send_passed_validator(
|
|
98
|
-
service=service, validator=validator, proposed_change_id=message.proposed_change, context=message.context
|
|
99
|
-
)
|
|
100
|
-
else:
|
|
101
|
-
await send_failed_validator(
|
|
102
|
-
service=service, validator=validator, proposed_change_id=message.proposed_change, context=message.context
|
|
103
|
-
)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
def get_validator_type(
|
|
107
|
-
validator_type: str,
|
|
108
|
-
) -> (
|
|
109
|
-
type[CoreArtifactValidator]
|
|
110
|
-
| type[CoreDataValidator]
|
|
111
|
-
| type[CoreGeneratorValidator]
|
|
112
|
-
| type[CoreRepositoryValidator]
|
|
113
|
-
| type[CoreSchemaValidator]
|
|
114
|
-
| type[CoreUserValidator]
|
|
115
|
-
| type[CoreValidator]
|
|
116
|
-
):
|
|
117
|
-
match validator_type:
|
|
118
|
-
case InfrahubKind.USERVALIDATOR:
|
|
119
|
-
validator_kind = CoreUserValidator
|
|
120
|
-
case InfrahubKind.SCHEMAVALIDATOR:
|
|
121
|
-
validator_kind = CoreSchemaValidator
|
|
122
|
-
case InfrahubKind.GENERATORVALIDATOR:
|
|
123
|
-
validator_kind = CoreGeneratorValidator
|
|
124
|
-
case InfrahubKind.REPOSITORYVALIDATOR:
|
|
125
|
-
validator_kind = CoreRepositoryValidator
|
|
126
|
-
case InfrahubKind.DATAVALIDATOR:
|
|
127
|
-
validator_kind = CoreDataValidator
|
|
128
|
-
case InfrahubKind.ARTIFACTVALIDATOR:
|
|
129
|
-
validator_kind = CoreArtifactValidator
|
|
130
|
-
case _:
|
|
131
|
-
validator_kind = CoreValidator
|
|
132
|
-
|
|
133
|
-
return validator_kind
|
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
from infrahub_sdk.protocols import CoreGeneratorValidator
|
|
2
|
-
from infrahub_sdk.uuidt import UUIDT
|
|
3
|
-
from prefect import flow
|
|
4
|
-
from prefect.logging import get_run_logger
|
|
5
|
-
|
|
6
|
-
from infrahub.core.constants import InfrahubKind
|
|
7
|
-
from infrahub.core.timestamp import Timestamp
|
|
8
|
-
from infrahub.message_bus import InfrahubMessage, Meta, messages
|
|
9
|
-
from infrahub.message_bus.types import KVTTL
|
|
10
|
-
from infrahub.services import InfrahubServices
|
|
11
|
-
from infrahub.validators.tasks import start_validator
|
|
12
|
-
from infrahub.workflows.utils import add_tags
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
@flow(
|
|
16
|
-
name="generator-definition-check",
|
|
17
|
-
flow_run_name="Validate Generator selection for {message.generator_definition.definition_name}",
|
|
18
|
-
)
|
|
19
|
-
async def check(message: messages.RequestGeneratorDefinitionCheck, service: InfrahubServices) -> None:
|
|
20
|
-
log = get_run_logger()
|
|
21
|
-
await add_tags(branches=[message.source_branch], nodes=[message.proposed_change])
|
|
22
|
-
events: list[InfrahubMessage] = []
|
|
23
|
-
|
|
24
|
-
proposed_change = await service.client.get(kind=InfrahubKind.PROPOSEDCHANGE, id=message.proposed_change)
|
|
25
|
-
|
|
26
|
-
validator_name = f"Generator Validator: {message.generator_definition.definition_name}"
|
|
27
|
-
validator_execution_id = str(UUIDT())
|
|
28
|
-
check_execution_ids: list[str] = []
|
|
29
|
-
|
|
30
|
-
await proposed_change.validations.fetch()
|
|
31
|
-
|
|
32
|
-
previous_validator: CoreGeneratorValidator | None = None
|
|
33
|
-
for relationship in proposed_change.validations.peers:
|
|
34
|
-
existing_validator = relationship.peer
|
|
35
|
-
if (
|
|
36
|
-
existing_validator.typename == InfrahubKind.GENERATORVALIDATOR
|
|
37
|
-
and existing_validator.definition.id == message.generator_definition.definition_id
|
|
38
|
-
):
|
|
39
|
-
previous_validator = existing_validator
|
|
40
|
-
|
|
41
|
-
validator = await start_validator(
|
|
42
|
-
service=service,
|
|
43
|
-
validator=previous_validator,
|
|
44
|
-
validator_type=CoreGeneratorValidator,
|
|
45
|
-
proposed_change=message.proposed_change,
|
|
46
|
-
data={
|
|
47
|
-
"label": validator_name,
|
|
48
|
-
"definition": message.generator_definition.definition_id,
|
|
49
|
-
},
|
|
50
|
-
context=message.context,
|
|
51
|
-
)
|
|
52
|
-
|
|
53
|
-
group = await service.client.get(
|
|
54
|
-
kind=InfrahubKind.GENERICGROUP,
|
|
55
|
-
prefetch_relationships=True,
|
|
56
|
-
populate_store=True,
|
|
57
|
-
id=message.generator_definition.group_id,
|
|
58
|
-
branch=message.source_branch,
|
|
59
|
-
)
|
|
60
|
-
await group.members.fetch()
|
|
61
|
-
|
|
62
|
-
existing_instances = await service.client.filters(
|
|
63
|
-
kind=InfrahubKind.GENERATORINSTANCE,
|
|
64
|
-
definition__ids=[message.generator_definition.definition_id],
|
|
65
|
-
include=["object"],
|
|
66
|
-
branch=message.source_branch,
|
|
67
|
-
)
|
|
68
|
-
instance_by_member = {}
|
|
69
|
-
for instance in existing_instances:
|
|
70
|
-
instance_by_member[instance.object.peer.id] = instance.id
|
|
71
|
-
|
|
72
|
-
repository = message.branch_diff.get_repository(repository_id=message.generator_definition.repository_id)
|
|
73
|
-
requested_instances = 0
|
|
74
|
-
impacted_instances = message.branch_diff.get_subscribers_ids(kind=InfrahubKind.GENERATORINSTANCE)
|
|
75
|
-
|
|
76
|
-
for relationship in group.members.peers:
|
|
77
|
-
member = relationship.peer
|
|
78
|
-
generator_instance = instance_by_member.get(member.id)
|
|
79
|
-
if _run_generator(
|
|
80
|
-
instance_id=generator_instance,
|
|
81
|
-
managed_branch=message.source_branch_sync_with_git,
|
|
82
|
-
impacted_instances=impacted_instances,
|
|
83
|
-
):
|
|
84
|
-
check_execution_id = str(UUIDT())
|
|
85
|
-
check_execution_ids.append(check_execution_id)
|
|
86
|
-
requested_instances += 1
|
|
87
|
-
log.info(f"Trigger execution of {message.generator_definition.definition_name} for {member.display_label}")
|
|
88
|
-
events.append(
|
|
89
|
-
messages.CheckGeneratorRun(
|
|
90
|
-
context=message.context,
|
|
91
|
-
generator_definition=message.generator_definition,
|
|
92
|
-
generator_instance=generator_instance,
|
|
93
|
-
commit=repository.source_commit,
|
|
94
|
-
repository_id=repository.repository_id,
|
|
95
|
-
repository_name=repository.repository_name,
|
|
96
|
-
repository_kind=repository.kind,
|
|
97
|
-
branch_name=message.source_branch,
|
|
98
|
-
query=message.generator_definition.query_name,
|
|
99
|
-
variables=member.extract(params=message.generator_definition.parameters),
|
|
100
|
-
target_id=member.id,
|
|
101
|
-
target_name=member.display_label,
|
|
102
|
-
validator_id=validator.id,
|
|
103
|
-
proposed_change=message.proposed_change,
|
|
104
|
-
meta=Meta(validator_execution_id=validator_execution_id, check_execution_id=check_execution_id),
|
|
105
|
-
)
|
|
106
|
-
)
|
|
107
|
-
|
|
108
|
-
checks_in_execution = ",".join(check_execution_ids)
|
|
109
|
-
await service.cache.set(
|
|
110
|
-
key=f"validator_execution_id:{validator_execution_id}:checks",
|
|
111
|
-
value=checks_in_execution,
|
|
112
|
-
expires=KVTTL.TWO_HOURS,
|
|
113
|
-
)
|
|
114
|
-
events.append(
|
|
115
|
-
messages.FinalizeValidatorExecution(
|
|
116
|
-
start_time=Timestamp().to_string(),
|
|
117
|
-
validator_id=validator.id,
|
|
118
|
-
validator_execution_id=validator_execution_id,
|
|
119
|
-
validator_type=InfrahubKind.GENERATORVALIDATOR,
|
|
120
|
-
context=message.context,
|
|
121
|
-
proposed_change=message.proposed_change,
|
|
122
|
-
)
|
|
123
|
-
)
|
|
124
|
-
for event in events:
|
|
125
|
-
event.assign_meta(parent=message)
|
|
126
|
-
await service.message_bus.send(message=event)
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
def _run_generator(instance_id: str | None, managed_branch: bool, impacted_instances: list[str]) -> bool:
|
|
130
|
-
"""Returns a boolean to indicate if a generator instance needs to be executed
|
|
131
|
-
Will return true if:
|
|
132
|
-
* The instance_id wasn't set which could be that it's a new object that doesn't have a previous generator instance
|
|
133
|
-
* The source branch is set to sync with Git which would indicate that it could contain updates in git to the generator
|
|
134
|
-
* The instance_id exists in the impacted_instances list
|
|
135
|
-
Will return false if:
|
|
136
|
-
* The source branch is a not one that syncs with git and the instance_id exists and is not in the impacted list
|
|
137
|
-
"""
|
|
138
|
-
if not instance_id or managed_branch:
|
|
139
|
-
return True
|
|
140
|
-
return instance_id in impacted_instances
|